@senzops/apm-node 1.2.7 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/CHANGELOG.md +9 -0
  2. package/README.md +479 -398
  3. package/dist/index.d.mts +5 -0
  4. package/dist/index.d.ts +5 -0
  5. package/dist/index.global.js +1 -1
  6. package/dist/index.global.js.map +1 -1
  7. package/dist/index.js +1 -1
  8. package/dist/index.js.map +1 -1
  9. package/dist/index.mjs +1 -1
  10. package/dist/index.mjs.map +1 -1
  11. package/dist/register.js +1 -1
  12. package/dist/register.js.map +1 -1
  13. package/dist/register.mjs +1 -1
  14. package/dist/register.mjs.map +1 -1
  15. package/package.json +1 -1
  16. package/src/core/client.ts +57 -0
  17. package/src/core/context.ts +71 -9
  18. package/src/core/transport.ts +20 -3
  19. package/src/core/types.ts +5 -1
  20. package/src/index.ts +4 -0
  21. package/src/instrumentation/amqplib.ts +371 -0
  22. package/src/instrumentation/anthropic.ts +245 -0
  23. package/src/instrumentation/aws-sdk.ts +403 -0
  24. package/src/instrumentation/azure-openai.ts +177 -0
  25. package/src/instrumentation/bunyan.ts +93 -0
  26. package/src/instrumentation/cassandra.ts +367 -0
  27. package/src/instrumentation/cohere.ts +227 -0
  28. package/src/instrumentation/connect.ts +200 -0
  29. package/src/instrumentation/dataloader.ts +291 -0
  30. package/src/instrumentation/dns.ts +220 -0
  31. package/src/instrumentation/firebase.ts +445 -0
  32. package/src/instrumentation/fs.ts +260 -0
  33. package/src/instrumentation/generic-pool.ts +317 -0
  34. package/src/instrumentation/google-genai.ts +426 -0
  35. package/src/instrumentation/graphql.ts +434 -0
  36. package/src/instrumentation/grpc.ts +666 -0
  37. package/src/instrumentation/hapi.ts +257 -0
  38. package/src/instrumentation/kafka.ts +360 -0
  39. package/src/instrumentation/knex.ts +249 -0
  40. package/src/instrumentation/lru-memoizer.ts +175 -0
  41. package/src/instrumentation/memcached.ts +190 -0
  42. package/src/instrumentation/mistral.ts +254 -0
  43. package/src/instrumentation/nestjs.ts +243 -0
  44. package/src/instrumentation/net.ts +171 -0
  45. package/src/instrumentation/openai.ts +281 -0
  46. package/src/instrumentation/pino.ts +170 -0
  47. package/src/instrumentation/restify.ts +213 -0
  48. package/src/instrumentation/runtime.ts +352 -0
  49. package/src/instrumentation/socketio.ts +272 -0
  50. package/src/instrumentation/tedious.ts +509 -0
  51. package/src/instrumentation/winston.ts +149 -0
  52. package/src/register.ts +22 -3
  53. package/src/wrappers/lambda.ts +417 -0
  54. package/tsup.config.ts +3 -3
  55. package/wiki.md +1547 -852
package/wiki.md CHANGED
@@ -1,852 +1,1547 @@
1
- # Senzor Node APM SDK Wiki
2
-
3
- This wiki contains detailed integration and operational guidance for `@senzops/apm-node`.
4
-
5
- The package is the native Senzor SDK for Node.js API services. It captures traces, spans, errors, logs, and task runs without requiring OpenTelemetry in the application.
6
-
7
- ## 1. Recommended Startup Patterns
8
-
9
- ### 1.1 Production Preload Mode
10
-
11
- Preload mode gives the SDK the best chance to capture all supported libraries because instrumentation is installed before the application imports framework, database, queue, and HTTP client modules.
12
-
13
- CommonJS:
14
-
15
- ```sh
16
- SENZOR_API_KEY=sz_apm_your_key_here node -r @senzops/apm-node/register server.js
17
- ```
18
-
19
- ESM:
20
-
21
- ```sh
22
- SENZOR_API_KEY=sz_apm_your_key_here node --import @senzops/apm-node/register server.mjs
23
- ```
24
-
25
- Docker:
26
-
27
- ```dockerfile
28
- ENV SENZOR_API_KEY=sz_apm_your_key_here
29
- CMD ["node", "-r", "@senzops/apm-node/register", "dist/server.js"]
30
- ```
31
-
32
- PM2:
33
-
34
- ```json
35
- {
36
- "apps": [
37
- {
38
- "name": "orders-api",
39
- "script": "dist/server.js",
40
- "node_args": "-r @senzops/apm-node/register",
41
- "env": {
42
- "SENZOR_API_KEY": "sz_apm_your_key_here",
43
- "NODE_ENV": "production"
44
- }
45
- }
46
- ]
47
- }
48
- ```
49
-
50
- ### 1.2 Programmatic Mode
51
-
52
- Programmatic mode is useful when your hosting platform does not support preload flags.
53
-
54
- Place `Senzor.init()` at the top of the entrypoint, before importing application modules when possible.
55
-
56
- ```ts
57
- import Senzor from '@senzops/apm-node';
58
-
59
- Senzor.init({
60
- apiKey: process.env.SENZOR_API_KEY!,
61
- endpoint: process.env.SENZOR_ENDPOINT,
62
- batchSize: 100,
63
- flushInterval: 10000,
64
- maxSpansPerTrace: 500
65
- });
66
-
67
- import './server';
68
- ```
69
-
70
- ## 2. Configuration Reference
71
-
72
- ```ts
73
- Senzor.init({
74
- apiKey: 'sz_apm_your_key_here',
75
- endpoint: 'https://api.senzor.dev',
76
- batchSize: 100,
77
- flushInterval: 10000,
78
- flushTimeoutMs: 5000,
79
- maxQueueSize: 10000,
80
- maxSpansPerTrace: 500,
81
- maxAttributeLength: 2048,
82
- maxAttributes: 64,
83
- captureHeaders: false,
84
- captureDbStatement: true,
85
- frameworkSpans: true,
86
- captureMiddlewareSpans: true,
87
- captureRouterSpans: true,
88
- captureLifecycleHookSpans: true,
89
- autoLogs: true,
90
- debug: false
91
- });
92
- ```
93
-
94
- | Option | Production recommendation |
95
- | --- | --- |
96
- | `apiKey` | Always pass through environment or secret manager. Do not hardcode. |
97
- | `endpoint` | Use `https://api.senzor.dev` unless your Senzor tenant provides a custom ingest URL. |
98
- | `batchSize` | Start with `100`. Increase for very high-throughput APIs if network overhead matters. |
99
- | `flushInterval` | Start with `10000` ms. Lower for short-lived processes. |
100
- | `flushTimeoutMs` | Keep between `3000` and `10000` ms for API services. |
101
- | `maxQueueSize` | Keep bounded. Increase only after checking memory pressure. |
102
- | `maxSpansPerTrace` | Keep `500` unless traces are expected to fan out heavily. |
103
- | `captureHeaders` | Keep `false` unless sanitized headers are required for debugging. |
104
- | `captureDbStatement` | Keep enabled only if SQL normalization is acceptable for your data policy. |
105
- | `autoLogs` | Keep enabled if correlated logs are wanted. Disable if another log pipeline already owns logs. |
106
- | `debug` | Use only during troubleshooting. |
107
-
108
- Enable only selected instrumentation:
109
-
110
- ```ts
111
- Senzor.init({
112
- apiKey: process.env.SENZOR_API_KEY!,
113
- instrumentations: ['http', 'fetch', 'pg', 'redis']
114
- });
115
- ```
116
-
117
- Disable auto-instrumentation while keeping manual APIs:
118
-
119
- ```ts
120
- Senzor.init({
121
- apiKey: process.env.SENZOR_API_KEY!,
122
- instrumentations: false
123
- });
124
- ```
125
-
126
- ## 3. Environment Variables
127
-
128
- The preload entrypoint supports these variables:
129
-
130
- ```sh
131
- SENZOR_API_KEY=sz_apm_your_key_here
132
- SENZOR_ENDPOINT=https://api.senzor.dev
133
- SENZOR_BATCH_SIZE=100
134
- SENZOR_FLUSH_INTERVAL=10000
135
- SENZOR_FLUSH_TIMEOUT_MS=5000
136
- SENZOR_MAX_QUEUE_SIZE=10000
137
- SENZOR_MAX_SPANS_PER_TRACE=500
138
- SENZOR_CAPTURE_HEADERS=false
139
- SENZOR_CAPTURE_DB_STATEMENT=true
140
- SENZOR_FRAMEWORK_SPANS=true
141
- SENZOR_CAPTURE_MIDDLEWARE_SPANS=true
142
- SENZOR_CAPTURE_ROUTER_SPANS=true
143
- SENZOR_CAPTURE_LIFECYCLE_HOOK_SPANS=true
144
- SENZOR_AUTO_LOGS=true
145
- SENZOR_DEBUG=false
146
- ```
147
-
148
- Accepted API key variables:
149
-
150
- - `SENZOR_API_KEY`
151
- - `SENZOR_APM_API_KEY`
152
- - `SENZOR_SERVICE_API_KEY`
153
-
154
- Accepted endpoint variables:
155
-
156
- - `SENZOR_ENDPOINT`
157
- - `SENZOR_APM_ENDPOINT`
158
-
159
- ## 4. Framework Integrations
160
-
161
- ### 4.1 Express and NestJS
162
-
163
- Preload mode automatically captures inbound HTTP requests. For better Express route names and error capture, add the Senzor middleware.
164
-
165
- ```ts
166
- import express from 'express';
167
- import Senzor from '@senzops/apm-node';
168
-
169
- Senzor.init({
170
- apiKey: process.env.SENZOR_API_KEY!
171
- });
172
-
173
- const app = express();
174
-
175
- app.use(Senzor.requestHandler());
176
-
177
- app.get('/orders/:orderId', async (req, res) => {
178
- res.json({ orderId: req.params.orderId });
179
- });
180
-
181
- app.use(Senzor.errorHandler());
182
-
183
- app.listen(3000);
184
- ```
185
-
186
- For NestJS on Express:
187
-
188
- ```ts
189
- import Senzor from '@senzops/apm-node';
190
- import { NestFactory } from '@nestjs/core';
191
- import { AppModule } from './app.module';
192
-
193
- Senzor.init({
194
- apiKey: process.env.SENZOR_API_KEY!
195
- });
196
-
197
- async function bootstrap() {
198
- const app = await NestFactory.create(AppModule);
199
- app.use(Senzor.requestHandler());
200
- await app.listen(3000);
201
- }
202
-
203
- bootstrap();
204
- ```
205
-
206
- ### 4.2 Fastify
207
-
208
- ```ts
209
- import Fastify from 'fastify';
210
- import Senzor from '@senzops/apm-node';
211
-
212
- const fastify = Fastify();
213
-
214
- fastify.register(Senzor.fastifyPlugin, {
215
- apiKey: process.env.SENZOR_API_KEY!
216
- });
217
-
218
- fastify.get('/orders/:orderId', async (request) => {
219
- return { orderId: request.params.orderId };
220
- });
221
-
222
- await fastify.listen({ port: 3000 });
223
- ```
224
-
225
- ### 4.3 Next.js App Router
226
-
227
- ```ts
228
- import Senzor from '@senzops/apm-node';
229
-
230
- Senzor.init({
231
- apiKey: process.env.SENZOR_API_KEY!
232
- });
233
-
234
- export const GET = Senzor.wrapNextRoute(async (request: Request) => {
235
- const url = new URL(request.url);
236
- return Response.json({ path: url.pathname });
237
- });
238
- ```
239
-
240
- ### 4.4 Next.js Pages Router
241
-
242
- ```ts
243
- import type { NextApiRequest, NextApiResponse } from 'next';
244
- import Senzor from '@senzops/apm-node';
245
-
246
- Senzor.init({
247
- apiKey: process.env.SENZOR_API_KEY!
248
- });
249
-
250
- export default Senzor.wrapNextPages(
251
- async function handler(req: NextApiRequest, res: NextApiResponse) {
252
- res.status(200).json({ ok: true });
253
- }
254
- );
255
- ```
256
-
257
- For serverless functions:
258
-
259
- ```ts
260
- await Senzor.flush();
261
- ```
262
-
263
- ### 4.5 H3, Nuxt, Nitro
264
-
265
- ```ts
266
- import Senzor from '@senzops/apm-node';
267
-
268
- Senzor.init({
269
- apiKey: process.env.SENZOR_API_KEY!
270
- });
271
-
272
- export default Senzor.wrapH3(
273
- defineEventHandler(async (event) => {
274
- return { ok: true };
275
- })
276
- );
277
- ```
278
-
279
- ### 4.6 Vanilla Node HTTP
280
-
281
- ```ts
282
- import http from 'http';
283
- import Senzor from '@senzops/apm-node';
284
-
285
- Senzor.init({
286
- apiKey: process.env.SENZOR_API_KEY!
287
- });
288
-
289
- const server = http.createServer(async (req, res) => {
290
- res.writeHead(200, { 'content-type': 'application/json' });
291
- res.end(JSON.stringify({ ok: true }));
292
- });
293
-
294
- server.listen(3000);
295
- ```
296
-
297
- With preload mode, no manual request wrapper is required.
298
-
299
- ## 5. Database and Cache Instrumentation
300
-
301
- ### 5.1 PostgreSQL
302
-
303
- Supported package: `pg`.
304
-
305
- ```ts
306
- import { Pool } from 'pg';
307
-
308
- const pool = new Pool({
309
- connectionString: process.env.DATABASE_URL
310
- });
311
-
312
- const result = await pool.query(
313
- 'SELECT id, email FROM users WHERE id = $1',
314
- [userId]
315
- );
316
- ```
317
-
318
- Captured span example:
319
-
320
- ```json
321
- {
322
- "name": "Postgres SELECT",
323
- "type": "db",
324
- "meta": {
325
- "operation": "SELECT",
326
- "db.system.name": "postgresql",
327
- "db.operation.name": "SELECT",
328
- "db.query.text": "SELECT id, email FROM users WHERE id = $?",
329
- "rowCount": 1,
330
- "library": "pg"
331
- }
332
- }
333
- ```
334
-
335
- ### 5.2 MongoDB
336
-
337
- Supported package: `mongodb`.
338
-
339
- ```ts
340
- const user = await db.collection('users').findOne({ _id: userId });
341
- const orders = await db.collection('orders').find({ userId }).toArray();
342
- ```
343
-
344
- Captured operations include common collection operations and cursor execution:
345
-
346
- - `findOne`
347
- - `find`
348
- - `aggregate`
349
- - `insertOne`
350
- - `insertMany`
351
- - `updateOne`
352
- - `updateMany`
353
- - `deleteOne`
354
- - `deleteMany`
355
- - `bulkWrite`
356
- - `countDocuments`
357
-
358
- ### 5.3 Mongoose
359
-
360
- Supported package: `mongoose`.
361
-
362
- ```ts
363
- const user = await User.findById(userId).exec();
364
- await user.save();
365
- ```
366
-
367
- Captured metadata includes model name, collection name, operation, and driver family.
368
-
369
- ### 5.4 MySQL and MySQL2
370
-
371
- Supported packages:
372
-
373
- - `mysql`
374
- - `mysql2`
375
-
376
- ```ts
377
- const [rows] = await connection.execute(
378
- 'SELECT id, email FROM users WHERE id = ?',
379
- [userId]
380
- );
381
- ```
382
-
383
- Captured metadata includes SQL operation, sanitized statement, row count when available, and package name.
384
-
385
- ### 5.5 Redis and ioredis
386
-
387
- Supported packages:
388
-
389
- - `redis`
390
- - `ioredis`
391
-
392
- ```ts
393
- await redis.get(`user:${userId}`);
394
- await redis.set(`user:${userId}`, JSON.stringify(user));
395
- ```
396
-
397
- Captured span names follow the command name:
398
-
399
- - `Redis GET`
400
- - `Redis SET`
401
- - `Redis HGETALL`
402
- - `Redis DEL`
403
-
404
- ## 6. HTTP Client Instrumentation
405
-
406
- The SDK propagates distributed tracing headers on outgoing calls:
407
-
408
- ```txt
409
- traceparent: 00-f3b2c2c9c70443f5a4b7f0ff6d5b9a17-9d8a4d5f17e24d2a-01
410
- x-senzor-trace-id: f3b2c2c9c70443f5a4b7f0ff6d5b9a17
411
- x-senzor-parent-span-id: 9d8a4d5f17e24d2a
412
- ```
413
-
414
- Supported clients:
415
-
416
- - Node `http.request`
417
- - Node `https.request`
418
- - Node `http.get`
419
- - Node `https.get`
420
- - Global `fetch`
421
- - `undici`
422
-
423
- Example:
424
-
425
- ```ts
426
- const response = await fetch('https://inventory.internal/items/sku_123');
427
- const data = await response.json();
428
- ```
429
-
430
- Captured span example:
431
-
432
- ```json
433
- {
434
- "name": "GET inventory.internal",
435
- "type": "http",
436
- "status": 200,
437
- "meta": {
438
- "method": "GET",
439
- "library": "fetch",
440
- "http.request.method": "GET",
441
- "url.path": "/items/sku_123",
442
- "server.address": "inventory.internal"
443
- }
444
- }
445
- ```
446
-
447
- ## 7. Logs and Errors
448
-
449
- ### 7.1 Correlated Console Logs
450
-
451
- ```ts
452
- console.info('order accepted', {
453
- orderId: 'ord_123',
454
- customerId: 'cus_456'
455
- });
456
- ```
457
-
458
- Captured log example:
459
-
460
- ```json
461
- {
462
- "message": "order accepted",
463
- "level": "info",
464
- "traceId": "f3b2c2c9c70443f5a4b7f0ff6d5b9a17",
465
- "attributes": {
466
- "orderId": "ord_123",
467
- "customerId": "cus_456"
468
- },
469
- "timestamp": "2026-05-16T15:30:00.000Z"
470
- }
471
- ```
472
-
473
- Disable console log capture:
474
-
475
- ```ts
476
- Senzor.init({
477
- apiKey: process.env.SENZOR_API_KEY!,
478
- autoLogs: false
479
- });
480
- ```
481
-
482
- ### 7.2 Manual Error Capture
483
-
484
- ```ts
485
- try {
486
- await chargeCustomer(customerId);
487
- } catch (error) {
488
- Senzor.captureException(error, {
489
- customerId,
490
- operation: 'charge_customer'
491
- });
492
- throw error;
493
- }
494
- ```
495
-
496
- Captured error example:
497
-
498
- ```json
499
- {
500
- "errorClass": "Error",
501
- "message": "Card declined",
502
- "traceId": "f3b2c2c9c70443f5a4b7f0ff6d5b9a17",
503
- "context": {
504
- "customerId": "cus_456",
505
- "operation": "charge_customer"
506
- },
507
- "timestamp": "2026-05-16T15:30:00.000Z"
508
- }
509
- ```
510
-
511
- Global handlers capture:
512
-
513
- - `uncaughtExceptionMonitor`
514
- - `uncaughtException`
515
- - `unhandledRejection`
516
- - `warning`
517
- - `multipleResolves`
518
- - `SIGTERM`
519
- - `SIGINT`
520
-
521
- ## 8. Background Tasks
522
-
523
- ### 8.1 Manual Task Wrapping
524
-
525
- ```ts
526
- const rebuildSearchIndex = Senzor.wrapTask(
527
- 'rebuild_search_index',
528
- 'custom',
529
- {
530
- metadata: {
531
- owner: 'search',
532
- schedule: 'manual'
533
- }
534
- },
535
- async () => {
536
- await rebuildIndex();
537
- }
538
- );
539
-
540
- await rebuildSearchIndex();
541
- ```
542
-
543
- ### 8.2 BullMQ
544
-
545
- When `bullmq` is installed and auto-instrumentation is enabled, worker jobs are captured as task runs.
546
-
547
- ```ts
548
- import { Worker } from 'bullmq';
549
-
550
- new Worker('billing', async (job) => {
551
- await sendInvoice(job.data.invoiceId);
552
- });
553
- ```
554
-
555
- Captured task fields include:
556
-
557
- - `taskName`
558
- - `taskType`
559
- - `queueDelay`
560
- - `attempts`
561
- - `isDeadLetter`
562
- - `metadata.jobId`
563
- - `metadata.queueName`
564
- - `resourceMetrics`
565
-
566
- ### 8.3 node-cron
567
-
568
- ```ts
569
- import cron from 'node-cron';
570
-
571
- cron.schedule(
572
- '*/5 * * * *',
573
- async () => {
574
- await syncReports();
575
- },
576
- {
577
- name: 'sync_reports'
578
- }
579
- );
580
- ```
581
-
582
- The SDK wraps scheduled handlers and sends each execution as a task run.
583
-
584
- ## 9. Manual Traces and Spans
585
-
586
- ### 9.1 Manual Trace
587
-
588
- Use `track()` when the runtime cannot use framework wrappers and you already have timing data.
589
-
590
- ```ts
591
- Senzor.track({
592
- method: 'POST',
593
- route: '/webhooks/payment',
594
- path: '/webhooks/payment',
595
- status: 202,
596
- duration: 18.7,
597
- spans: []
598
- });
599
- ```
600
-
601
- ### 9.2 Manual Span
602
-
603
- ```ts
604
- const span = Senzor.startSpan('reserve_inventory', 'function');
605
-
606
- try {
607
- await reserveInventory(orderId);
608
- span.end({ orderId }, 200);
609
- } catch (error) {
610
- span.end({ orderId, error: String(error) }, 500);
611
- throw error;
612
- }
613
- ```
614
-
615
- Span types:
616
-
617
- - `http`
618
- - `db`
619
- - `function`
620
- - `custom`
621
-
622
- ## 10. Payload Format
623
-
624
- ### 10.1 APM Ingest
625
-
626
- Endpoint:
627
-
628
- ```txt
629
- POST /api/ingest/apm
630
- ```
631
-
632
- Payload:
633
-
634
- ```json
635
- {
636
- "traces": [
637
- {
638
- "traceId": "f3b2c2c9c70443f5a4b7f0ff6d5b9a17",
639
- "parentTraceId": "8cb1f2e2f43e4b2b8b3411c7df81e7e0",
640
- "parentSpanId": "6c62c908a21d4f49",
641
- "method": "GET",
642
- "route": "/orders/:orderId",
643
- "path": "/orders/ord_123?include=items",
644
- "status": 200,
645
- "duration": 53.29,
646
- "ip": "203.0.113.10",
647
- "userAgent": "Mozilla/5.0",
648
- "timestamp": "2026-05-16T15:30:00.000Z",
649
- "spans": [
650
- {
651
- "spanId": "9d8a4d5f17e24d2a",
652
- "parentSpanId": "91f6c551d5a2403f",
653
- "name": "GET inventory.internal",
654
- "type": "http",
655
- "startTime": 5.41,
656
- "duration": 22.16,
657
- "status": 200,
658
- "meta": {
659
- "method": "GET",
660
- "library": "fetch",
661
- "http.request.method": "GET",
662
- "server.address": "inventory.internal"
663
- }
664
- }
665
- ]
666
- }
667
- ],
668
- "errors": [],
669
- "logs": []
670
- }
671
- ```
672
-
673
- ### 10.2 Task Ingest
674
-
675
- Endpoint:
676
-
677
- ```txt
678
- POST /api/ingest/task
679
- ```
680
-
681
- Payload:
682
-
683
- ```json
684
- {
685
- "runs": [
686
- {
687
- "runId": "3917bd35-b1d6-4e23-a1d2-d969e1a7d6a1",
688
- "taskName": "billing:send_invoice_email",
689
- "taskType": "queue",
690
- "status": "success",
691
- "duration": 188.3,
692
- "queueDelay": 92,
693
- "attempts": 1,
694
- "triggerTraceId": "f3b2c2c9c70443f5a4b7f0ff6d5b9a17",
695
- "metadata": {
696
- "jobId": "142",
697
- "queueName": "billing"
698
- },
699
- "resourceMetrics": {
700
- "memoryDeltaBytes": 1048576,
701
- "cpuUserUs": 12000,
702
- "cpuSystemUs": 3000
703
- },
704
- "isDeadLetter": false,
705
- "spans": [],
706
- "timestamp": "2026-05-16T15:30:00.000Z"
707
- }
708
- ],
709
- "errors": [],
710
- "logs": []
711
- }
712
- ```
713
-
714
- ## 11. Security, Privacy, and Cardinality
715
-
716
- The SDK is conservative by default.
717
-
718
- Sensitive keys are redacted from attributes, headers, logs, and error context:
719
-
720
- - `authorization`
721
- - `cookie`
722
- - `set-cookie`
723
- - `password`
724
- - `passwd`
725
- - `pwd`
726
- - `secret`
727
- - `token`
728
- - `api-key`
729
- - `x-api-key`
730
- - `access-token`
731
- - `refresh-token`
732
- - `client-secret`
733
- - `private-key`
734
-
735
- Cardinality controls:
736
-
737
- - URL routes are normalized, for example `/users/123` becomes `/users/:id`.
738
- - UUID-like IDs and Mongo ObjectIds are normalized in routes.
739
- - Attribute count and string length are bounded.
740
- - Span count per trace is bounded.
741
- - Queue size is bounded.
742
-
743
- Production recommendations:
744
-
745
- - Keep `captureHeaders` disabled unless required.
746
- - Never capture request bodies by default.
747
- - Avoid putting raw customer data into span names.
748
- - Use route templates and IDs in metadata, not in operation names.
749
-
750
- ## 12. Transport Behavior
751
-
752
- The SDK batches telemetry and sends it asynchronously.
753
-
754
- Behavior:
755
-
756
- - Queues are bounded with `maxQueueSize`.
757
- - Flush occurs on `batchSize` or `flushInterval`.
758
- - Ingest calls have a timeout with `flushTimeoutMs`.
759
- - Failed batches are requeued within queue limits.
760
- - SDK ingest requests are marked as internal so they are not traced recursively.
761
- - A best-effort flush runs during `beforeExit`.
762
-
763
- Serverless recommendation:
764
-
765
- ```ts
766
- await Senzor.flush();
767
- ```
768
-
769
- Call this after the response is prepared but before the function exits when deterministic delivery matters.
770
-
771
- ## 13. Troubleshooting
772
-
773
- ### Missing spans
774
-
775
- Check the startup order first. Preload mode should be used when possible:
776
-
777
- ```sh
778
- node -r @senzops/apm-node/register server.js
779
- ```
780
-
781
- If programmatic mode is used, initialize before importing `pg`, `mongodb`, `redis`, framework modules, or HTTP clients.
782
-
783
- ### Duplicate traces
784
-
785
- The SDK deduplicates active traces in common cases. If duplicates still appear, check whether the app is using both native preload mode and custom manual `track()` calls for the same request.
786
-
787
- ### No data in Senzor
788
-
789
- Verify:
790
-
791
- - `SENZOR_API_KEY` is present.
792
- - `endpoint` points to the expected Senzor environment.
793
- - The runtime can reach the ingest endpoint.
794
- - `Senzor.flush()` is called in short-lived runtimes.
795
- - `debug: true` shows flush attempts.
796
-
797
- ### High memory usage
798
-
799
- Tune:
800
-
801
- ```ts
802
- Senzor.init({
803
- apiKey: process.env.SENZOR_API_KEY!,
804
- maxQueueSize: 5000,
805
- maxSpansPerTrace: 250,
806
- maxAttributes: 32,
807
- maxAttributeLength: 1024
808
- });
809
- ```
810
-
811
- ### High-cardinality routes
812
-
813
- Use framework wrappers where possible so route templates are available:
814
-
815
- ```ts
816
- app.use(Senzor.requestHandler());
817
- ```
818
-
819
- For manual traces, pass normalized routes:
820
-
821
- ```ts
822
- Senzor.track({
823
- method: 'GET',
824
- route: '/users/:id',
825
- path: '/users/123',
826
- status: 200,
827
- duration: 12.4
828
- });
829
- ```
830
-
831
- ## 14. Build and Publish
832
-
833
- Build the package:
834
-
835
- ```sh
836
- npm run build
837
- ```
838
-
839
- The build produces:
840
-
841
- - CommonJS: `dist/index.js`
842
- - ESM: `dist/index.mjs`
843
- - Browser/global bundle: `dist/index.global.js`
844
- - Preload entrypoint: `dist/register.js` and `dist/register.mjs`
845
- - Type declarations: `dist/index.d.ts`, `dist/register.d.ts`
846
-
847
- Before publishing, verify:
848
-
849
- ```sh
850
- npx tsc --noEmit
851
- npm run build
852
- ```
1
+ # Senzor Node APM SDK — Complete Guide
2
+
3
+ Comprehensive integration, configuration, and operational reference for `@senzops/apm-node`.
4
+
5
+ The SDK captures distributed traces, child spans, errors, logs, background task runs, and runtime metrics from Node.js services and sends them to Senzor. It replaces OpenTelemetry auto-instrumentation with a zero-dependency, Senzor-native alternative.
6
+
7
+ **43 auto-instrumentations. 8 framework wrappers. Zero runtime dependencies.**
8
+
9
+ ---
10
+
11
+ ## Table of Contents
12
+
13
+ 1. [Getting Started](#1-getting-started)
14
+ 2. [Startup Modes](#2-startup-modes)
15
+ 3. [Framework Integration Guide](#3-framework-integration-guide)
16
+ 4. [AWS Lambda Integration](#4-aws-lambda-integration)
17
+ 5. [Cloudflare Workers Integration](#5-cloudflare-workers-integration)
18
+ 6. [Auto-Instrumentation Reference](#6-auto-instrumentation-reference)
19
+ 7. [AI / LLM SDK Instrumentation](#7-ai--llm-sdk-instrumentation)
20
+ 8. [Database & Cache Instrumentation](#8-database--cache-instrumentation)
21
+ 9. [Messaging & Queue Instrumentation](#9-messaging--queue-instrumentation)
22
+ 10. [Cloud Provider Instrumentation](#10-cloud-provider-instrumentation)
23
+ 11. [gRPC, GraphQL & Network](#11-grpc-graphql--network)
24
+ 12. [Log Correlation](#12-log-correlation)
25
+ 13. [Background Task Monitoring](#13-background-task-monitoring)
26
+ 14. [Manual Traces & Spans](#14-manual-traces--spans)
27
+ 15. [Error Tracking](#15-error-tracking)
28
+ 16. [Runtime Metrics](#16-runtime-metrics)
29
+ 17. [Distributed Tracing & Context Propagation](#17-distributed-tracing--context-propagation)
30
+ 18. [Configuration Reference](#18-configuration-reference)
31
+ 19. [Environment Variables](#19-environment-variables)
32
+ 20. [Security, Privacy & Cardinality](#20-security-privacy--cardinality)
33
+ 21. [Transport Behavior](#21-transport-behavior)
34
+ 22. [Ingestion Payload Format](#22-ingestion-payload-format)
35
+ 23. [Deployment Patterns](#23-deployment-patterns)
36
+ 24. [Troubleshooting](#24-troubleshooting)
37
+ 25. [Public API Reference](#25-public-api-reference)
38
+ 26. [Build & Publish](#26-build--publish)
39
+
40
+ ---
41
+
42
+ ## 1. Getting Started
43
+
44
+ ### Install
45
+
46
+ ```sh
47
+ npm install @senzops/apm-node
48
+ ```
49
+
50
+ ### Minimal Setup
51
+
52
+ ```ts
53
+ import Senzor from '@senzops/apm-node';
54
+
55
+ Senzor.init({ apiKey: process.env.SENZOR_API_KEY! });
56
+ ```
57
+
58
+ That's it. Once initialized, the SDK automatically instruments all supported libraries that your application imports. No additional configuration or per-library setup is required.
59
+
60
+ ---
61
+
62
+ ## 2. Startup Modes
63
+
64
+ ### 2.1 Preload Mode (Recommended for Production)
65
+
66
+ Preload mode gives the SDK the best coverage because it installs instrumentation hooks before the application imports any library.
67
+
68
+ **CommonJS:**
69
+
70
+ ```sh
71
+ SENZOR_API_KEY=sz_apm_xxx node -r @senzops/apm-node/register server.js
72
+ ```
73
+
74
+ **ESM:**
75
+
76
+ ```sh
77
+ SENZOR_API_KEY=sz_apm_xxx node --import @senzops/apm-node/register server.mjs
78
+ ```
79
+
80
+ **Docker:**
81
+
82
+ ```dockerfile
83
+ ENV SENZOR_API_KEY=sz_apm_xxx
84
+ CMD ["node", "-r", "@senzops/apm-node/register", "dist/server.js"]
85
+ ```
86
+
87
+ **PM2:**
88
+
89
+ ```json
90
+ {
91
+ "apps": [{
92
+ "name": "orders-api",
93
+ "script": "dist/server.js",
94
+ "node_args": "-r @senzops/apm-node/register",
95
+ "env": {
96
+ "SENZOR_API_KEY": "sz_apm_xxx",
97
+ "NODE_ENV": "production"
98
+ }
99
+ }]
100
+ }
101
+ ```
102
+
103
+ ### 2.2 Programmatic Mode
104
+
105
+ When preload flags are unavailable (some serverless platforms, custom runtimes), initialize at the very top of your entrypoint.
106
+
107
+ ```ts
108
+ // init.ts import this FIRST
109
+ import Senzor from '@senzops/apm-node';
110
+
111
+ Senzor.init({
112
+ apiKey: process.env.SENZOR_API_KEY!,
113
+ endpoint: process.env.SENZOR_ENDPOINT,
114
+ batchSize: 100,
115
+ flushInterval: 10000,
116
+ });
117
+
118
+ // Then import your app
119
+ import './server';
120
+ ```
121
+
122
+ ### 2.3 When to Use Which
123
+
124
+ | Scenario | Mode |
125
+ |----------|------|
126
+ | Standard Node.js server (Express, Fastify, Koa, etc.) | Preload |
127
+ | Docker, Kubernetes, PM2 | Preload |
128
+ | Next.js API routes | Programmatic + wrapper |
129
+ | Nuxt / Nitro / H3 | Programmatic + wrapper |
130
+ | Cloudflare Workers | Programmatic + wrapper |
131
+ | AWS Lambda | Programmatic + `wrapLambda`, or Lambda Layer with preload |
132
+ | Serverless without `NODE_OPTIONS` support | Programmatic |
133
+
134
+ ---
135
+
136
+ ## 3. Framework Integration Guide
137
+
138
+ ### 3.1 Express
139
+
140
+ Preload mode captures inbound HTTP automatically. Add middleware for route detection and error capture:
141
+
142
+ ```ts
143
+ import express from 'express';
144
+ import Senzor from '@senzops/apm-node';
145
+
146
+ Senzor.init({ apiKey: process.env.SENZOR_API_KEY! });
147
+
148
+ const app = express();
149
+
150
+ // MUST be the first middleware
151
+ app.use(Senzor.requestHandler());
152
+
153
+ app.get('/users/:id', async (req, res) => {
154
+ res.json({ id: req.params.id });
155
+ });
156
+
157
+ // MUST be the last middleware
158
+ app.use(Senzor.errorHandler());
159
+
160
+ app.listen(3000);
161
+ ```
162
+
163
+ ### 3.2 Fastify
164
+
165
+ ```ts
166
+ import Fastify from 'fastify';
167
+ import Senzor from '@senzops/apm-node';
168
+
169
+ const fastify = Fastify();
170
+
171
+ fastify.register(Senzor.fastifyPlugin, {
172
+ apiKey: process.env.SENZOR_API_KEY!,
173
+ });
174
+
175
+ fastify.get('/orders/:orderId', async (request) => {
176
+ return { orderId: request.params.orderId };
177
+ });
178
+
179
+ await fastify.listen({ port: 3000 });
180
+ ```
181
+
182
+ ### 3.3 NestJS
183
+
184
+ NestJS runs on Express (default) or Fastify. For Express:
185
+
186
+ ```ts
187
+ import Senzor from '@senzops/apm-node';
188
+ import { NestFactory } from '@nestjs/core';
189
+ import { AppModule } from './app.module';
190
+
191
+ // Initialize BEFORE creating the Nest app
192
+ Senzor.init({ apiKey: process.env.SENZOR_API_KEY! });
193
+
194
+ async function bootstrap() {
195
+ const app = await NestFactory.create(AppModule);
196
+ app.use(Senzor.requestHandler());
197
+ await app.listen(3000);
198
+ }
199
+
200
+ bootstrap();
201
+ ```
202
+
203
+ The NestJS instrumentation auto-captures:
204
+ - Controller and method resolution
205
+ - Guards, Interceptors, Pipes execution
206
+ - Exception filter processing
207
+
208
+ ### 3.4 Koa
209
+
210
+ ```ts
211
+ import Koa from 'koa';
212
+ import Senzor from '@senzops/apm-node';
213
+
214
+ Senzor.init({ apiKey: process.env.SENZOR_API_KEY! });
215
+
216
+ const app = new Koa();
217
+
218
+ app.use(async (ctx) => {
219
+ ctx.body = { ok: true };
220
+ });
221
+
222
+ app.listen(3000);
223
+ ```
224
+
225
+ With preload mode, Koa middleware stack is automatically instrumented.
226
+
227
+ ### 3.5 Hapi
228
+
229
+ ```ts
230
+ import Hapi from '@hapi/hapi';
231
+ import Senzor from '@senzops/apm-node';
232
+
233
+ Senzor.init({ apiKey: process.env.SENZOR_API_KEY! });
234
+
235
+ const server = Hapi.server({ port: 3000 });
236
+
237
+ server.route({
238
+ method: 'GET',
239
+ path: '/users/{id}',
240
+ handler: (request) => ({ id: request.params.id }),
241
+ });
242
+
243
+ await server.start();
244
+ ```
245
+
246
+ ### 3.6 Restify
247
+
248
+ ```ts
249
+ import restify from 'restify';
250
+ import Senzor from '@senzops/apm-node';
251
+
252
+ Senzor.init({ apiKey: process.env.SENZOR_API_KEY! });
253
+
254
+ const server = restify.createServer();
255
+
256
+ server.get('/users/:id', (req, res, next) => {
257
+ res.send({ id: req.params.id });
258
+ next();
259
+ });
260
+
261
+ server.listen(3000);
262
+ ```
263
+
264
+ ### 3.7 Next.js
265
+
266
+ **App Router:**
267
+
268
+ ```ts
269
+ import Senzor from '@senzops/apm-node';
270
+
271
+ Senzor.init({ apiKey: process.env.SENZOR_API_KEY! });
272
+
273
+ export const GET = Senzor.wrapNextRoute(async (request: Request) => {
274
+ return Response.json({ ok: true });
275
+ });
276
+ ```
277
+
278
+ **Pages Router:**
279
+
280
+ ```ts
281
+ import Senzor from '@senzops/apm-node';
282
+
283
+ Senzor.init({ apiKey: process.env.SENZOR_API_KEY! });
284
+
285
+ export default Senzor.wrapNextPages(async (req, res) => {
286
+ res.status(200).json({ ok: true });
287
+ });
288
+ ```
289
+
290
+ ### 3.8 H3 / Nuxt / Nitro
291
+
292
+ ```ts
293
+ import Senzor from '@senzops/apm-node';
294
+
295
+ Senzor.init({ apiKey: process.env.SENZOR_API_KEY! });
296
+
297
+ export default Senzor.wrapH3(defineEventHandler(async (event) => {
298
+ return { ok: true };
299
+ }));
300
+ ```
301
+
302
+ **Nitro Plugin (for Cloudflare Workers preset):**
303
+
304
+ ```ts
305
+ // server/plugins/senzor.ts
306
+ import { Senzor } from '@senzops/apm-node';
307
+
308
+ export default defineNitroPlugin((nitroApp) => {
309
+ Senzor.init({ apiKey: '<YOUR_APM_KEY>' });
310
+ Senzor.nitroPlugin(nitroApp);
311
+ });
312
+ ```
313
+
314
+ ### 3.9 Vanilla Node HTTP Server
315
+
316
+ ```ts
317
+ import http from 'http';
318
+ import Senzor from '@senzops/apm-node';
319
+
320
+ Senzor.init({ apiKey: process.env.SENZOR_API_KEY! });
321
+
322
+ const server = http.createServer((req, res) => {
323
+ res.writeHead(200, { 'content-type': 'application/json' });
324
+ res.end(JSON.stringify({ ok: true }));
325
+ });
326
+
327
+ server.listen(3000);
328
+ ```
329
+
330
+ With preload mode, no wrapper is needed — `http.createServer` is automatically instrumented.
331
+
332
+ ---
333
+
334
+ ## 4. AWS Lambda Integration
335
+
336
+ ### 4.1 Handler Wrapper (Recommended)
337
+
338
+ ```ts
339
+ import Senzor from '@senzops/apm-node';
340
+
341
+ Senzor.init({ apiKey: process.env.SENZOR_API_KEY! });
342
+
343
+ export const handler = Senzor.wrapLambda(async (event, context) => {
344
+ // Your Lambda logic
345
+ return { statusCode: 200, body: JSON.stringify({ ok: true }) };
346
+ });
347
+ ```
348
+
349
+ ### 4.2 What `wrapLambda` Does
350
+
351
+ **Cold Start Detection:**
352
+ The first invocation in each container is tagged with `faas.coldstart: true`. Subsequent warm invocations are tagged `false`.
353
+
354
+ **Trigger-Type Detection:**
355
+ The wrapper inspects the event shape and automatically detects:
356
+
357
+ | Trigger | Detection |
358
+ |---------|-----------|
359
+ | API Gateway v1 (REST API) | `event.httpMethod` or `event.requestContext.httpMethod` |
360
+ | API Gateway v2 (HTTP API) | `event.requestContext.http.method` |
361
+ | Application Load Balancer | `event.requestContext.elb` |
362
+ | SQS | `event.Records[0].eventSource === 'aws:sqs'` |
363
+ | SNS | `event.Records[0].EventSource === 'aws:sns'` |
364
+ | DynamoDB Streams | `event.Records[0].eventSource === 'aws:dynamodb'` |
365
+ | S3 | `event.Records[0].eventSource === 'aws:s3'` |
366
+ | EventBridge | `event.source` + `event.detail-type` + `event.detail` |
367
+ | Scheduled (CloudWatch Events) | `event.source === 'aws.events'` |
368
+
369
+ For HTTP triggers (API Gateway, ALB), the wrapper extracts method, path, headers, client IP, and status code. For messaging triggers, it extracts queue/topic names, batch sizes, and table names.
370
+
371
+ **Lambda Context Extraction:**
372
+ Every invocation captures these attributes:
373
+
374
+ | Attribute | Source |
375
+ |-----------|--------|
376
+ | `faas.name` | `context.functionName` |
377
+ | `faas.version` | `context.functionVersion` |
378
+ | `faas.execution` | `context.awsRequestId` |
379
+ | `faas.max_memory` | `context.memoryLimitInMB` |
380
+ | `faas.coldstart` | Module-level boolean flag |
381
+ | `faas.trigger` | `http`, `pubsub`, `datasource`, `timer`, `other` |
382
+ | `cloud.provider` | `aws` |
383
+ | `cloud.platform` | `aws_lambda` |
384
+ | `cloud.region` | `AWS_REGION` env var |
385
+ | `cloud.account.id` | Parsed from invoked ARN |
386
+ | `cloud.resource_id` | `context.invokedFunctionArn` |
387
+ | `aws.log.group.names` | `context.logGroupName` |
388
+
389
+ **Forced Flush:**
390
+ After every invocation, the wrapper calls `await Senzor.flush()` before returning to the Lambda runtime. This is critical because Lambda freezes the process immediately after the handler returns.
391
+
392
+ **Lambda Extensions API:**
393
+ The wrapper registers as an internal Lambda extension to receive `SHUTDOWN` lifecycle events. This provides a safety-net flush when the Lambda execution environment is being terminated.
394
+
395
+ ### 4.3 Lambda Layer Deployment (Zero Code Changes)
396
+
397
+ Package the SDK as a Lambda Layer and use `NODE_OPTIONS` for automatic initialization:
398
+
399
+ **Step 1: Create the Layer**
400
+
401
+ ```sh
402
+ mkdir -p layer/nodejs
403
+ cd layer/nodejs
404
+ npm init -y
405
+ npm install @senzops/apm-node
406
+ cd ..
407
+ zip -r senzor-apm-layer.zip nodejs/
408
+ ```
409
+
410
+ **Step 2: Publish the Layer**
411
+
412
+ ```sh
413
+ aws lambda publish-layer-version \
414
+ --layer-name senzor-apm-node \
415
+ --zip-file fileb://senzor-apm-layer.zip \
416
+ --compatible-runtimes nodejs18.x nodejs20.x nodejs22.x
417
+ ```
418
+
419
+ **Step 3: Attach to Your Function**
420
+
421
+ ```sh
422
+ aws lambda update-function-configuration \
423
+ --function-name my-function \
424
+ --layers arn:aws:lambda:us-east-1:123456789:layer:senzor-apm-node:1 \
425
+ --environment Variables="{SENZOR_API_KEY=sz_apm_xxx,NODE_OPTIONS=--require @senzops/apm-node/register}"
426
+ ```
427
+
428
+ With this approach, the SDK initializes via preload before your handler code runs. For HTTP triggers, you still need `Senzor.wrapLambda` in your handler for proper route detection and response status capture. For non-HTTP triggers, preload alone captures outgoing calls and database operations.
429
+
430
+ ### 4.4 Lambda Auto-Detection
431
+
432
+ When running inside Lambda (detected via `AWS_LAMBDA_FUNCTION_NAME` env var), the register entrypoint automatically applies these optimizations:
433
+
434
+ | Setting | Default | Lambda Override |
435
+ |---------|---------|-----------------|
436
+ | `runtimeMetrics` | `true` | `false` (meaningless per-invocation) |
437
+ | `batchSize` | `100` | `10` (short-lived invocations) |
438
+ | `flushInterval` | `10000` | `0` (flush on demand only) |
439
+
440
+ ### 4.5 Lambda with SAM / CDK / Serverless Framework
441
+
442
+ **AWS SAM (`template.yaml`):**
443
+
444
+ ```yaml
445
+ Globals:
446
+ Function:
447
+ Layers:
448
+ - !Ref SenzorLayer
449
+ Environment:
450
+ Variables:
451
+ SENZOR_API_KEY: !Ref SenzorApiKey
452
+ NODE_OPTIONS: "--require @senzops/apm-node/register"
453
+ ```
454
+
455
+ **Serverless Framework (`serverless.yml`):**
456
+
457
+ ```yaml
458
+ provider:
459
+ environment:
460
+ SENZOR_API_KEY: ${ssm:/senzor/api-key}
461
+ NODE_OPTIONS: "--require @senzops/apm-node/register"
462
+ layers:
463
+ - arn:aws:lambda:us-east-1:123456789:layer:senzor-apm-node:1
464
+ ```
465
+
466
+ **AWS CDK:**
467
+
468
+ ```ts
469
+ const fn = new lambda.Function(this, 'Handler', {
470
+ runtime: lambda.Runtime.NODEJS_20_X,
471
+ handler: 'index.handler',
472
+ code: lambda.Code.fromAsset('dist'),
473
+ layers: [senzorLayer],
474
+ environment: {
475
+ SENZOR_API_KEY: senzorApiKey.stringValue,
476
+ NODE_OPTIONS: '--require @senzops/apm-node/register',
477
+ },
478
+ });
479
+ ```
480
+
481
+ ---
482
+
483
+ ## 5. Cloudflare Workers Integration
484
+
485
+ ### 5.1 Direct Worker
486
+
487
+ ```ts
488
+ import Senzor from '@senzops/apm-node';
489
+
490
+ Senzor.init({ apiKey: '<YOUR_APM_KEY>' });
491
+
492
+ export default {
493
+ fetch: Senzor.worker(async (request, env, ctx) => {
494
+ const url = new URL(request.url);
495
+ return new Response(JSON.stringify({ path: url.pathname }), {
496
+ headers: { 'content-type': 'application/json' },
497
+ });
498
+ }),
499
+ };
500
+ ```
501
+
502
+ The worker wrapper automatically:
503
+ - Starts a trace with method, path, headers, and client IP
504
+ - Captures response status code
505
+ - Uses `ctx.waitUntil()` for non-blocking flush when available
506
+ - Falls back to `await flush()` otherwise
507
+
508
+ ### 5.2 Nitro + Cloudflare Workers
509
+
510
+ ```ts
511
+ // server/plugins/senzor.ts
512
+ import { Senzor } from '@senzops/apm-node';
513
+
514
+ export default defineNitroPlugin((nitroApp) => {
515
+ Senzor.init({ apiKey: '<YOUR_APM_KEY>' });
516
+ Senzor.nitroPlugin(nitroApp);
517
+ });
518
+ ```
519
+
520
+ ---
521
+
522
+ ## 6. Auto-Instrumentation Reference
523
+
524
+ Complete table of all 43+ auto-instrumentations. All activate automatically when the library is present in `node_modules` and imported by your application.
525
+
526
+ ### Web Frameworks & HTTP (10)
527
+
528
+ | Instrumentation Key | npm Package(s) | Captured Signals |
529
+ |---------------------|----------------|------------------|
530
+ | `http` | Node built-in `http`/`https` | Inbound requests, outbound calls, status codes, timing |
531
+ | `fetch` | Global `fetch` | Outbound HTTP, W3C Traceparent propagation |
532
+ | `undici` | `undici` | Outbound HTTP via Node's native client |
533
+ | `express` | `express` | Route matching, middleware chain, error capture |
534
+ | `fastify` | `fastify` | Route matching, hooks, lifecycle spans |
535
+ | `koa` | `koa` | Middleware stack, route detection |
536
+ | `nestjs` | `@nestjs/core` | Controllers, Guards, Interceptors, Pipes |
537
+ | `hapi` | `@hapi/hapi` | Route handling, request lifecycle |
538
+ | `restify` | `restify` | Route matching, handler chain |
539
+ | `connect` | `connect` | Middleware stack |
540
+
541
+ ### Databases & Cache (9)
542
+
543
+ | Instrumentation Key | npm Package(s) | Captured Signals |
544
+ |---------------------|----------------|------------------|
545
+ | `pg` | `pg` | Queries, prepared statements, row counts, sanitized SQL |
546
+ | `mongo` | `mongodb` | find, insert, update, delete, aggregate, bulkWrite, cursor |
547
+ | `mongoose` | `mongoose` | Model operations, model/collection names |
548
+ | `mysql` | `mysql`, `mysql2` | Queries, sanitized SQL, row counts |
549
+ | `redis` | `redis`, `ioredis` | All commands (GET, SET, HGETALL, DEL, etc.) |
550
+ | `knex` | `knex` | Query builder, raw queries, transactions, streaming |
551
+ | `tedious` | `tedious` | T-SQL queries, stored procedures, row counts, batch SQL |
552
+ | `cassandra` | `cassandra-driver` | CQL queries, batch operations, prepared statements, consistency |
553
+ | `memcached` | `memcached` | get, set, gets, cas, append, prepend, incr, decr, del, flush |
554
+
555
+ ### Messaging & Queues (5)
556
+
557
+ | Instrumentation Key | npm Package(s) | Captured Signals |
558
+ |---------------------|----------------|------------------|
559
+ | `kafka` | `kafkajs` | Producer send, consumer processing, topic, partition, offset |
560
+ | `amqplib` | `amqplib` | Publish, consume, ack/nack, queue, exchange, routing key |
561
+ | `socketio` | `socket.io` | Event emit/receive, namespace, room, acknowledgements |
562
+ | `bullmq` | `bullmq` | Worker jobs as task runs, queue delay, retries, dead-letter |
563
+ | `cron` | `node-cron` | Scheduled jobs as task runs, schedule expression |
564
+
565
+ ### AI / LLM SDKs (7)
566
+
567
+ | Instrumentation Key | npm Package(s) | Captured Signals |
568
+ |---------------------|----------------|------------------|
569
+ | `openai` | `openai` | Chat completions, embeddings, images, audio, assistants, token usage |
570
+ | `anthropic` | `@anthropic-ai/sdk` | Messages, completions, model, input/output tokens, stop reason |
571
+ | `google-genai` | `@google/generative-ai`, `@google-cloud/vertexai` | generateContent, chat, embeddings, countTokens, token usage |
572
+ | `azure-openai` | `@azure/openai` | Chat, completions, embeddings, images, audio (v1.x) |
573
+ | `cohere` | `cohere-ai` | Chat, generate, embed, rerank, classify, summarize, tokenize |
574
+ | `mistral` | `@mistralai/mistralai` | Chat, FIM, embeddings, model, token usage |
575
+ | `aws-sdk` (Bedrock) | `@aws-sdk/client-bedrock-runtime` | InvokeModel, Converse, token usage, model ID, finish reason |
576
+
577
+ ### Cloud & Infrastructure (3)
578
+
579
+ | Instrumentation Key | npm Package(s) | Captured Signals |
580
+ |---------------------|----------------|------------------|
581
+ | `aws-sdk` | `@aws-sdk/*`, `@smithy/smithy-client` | All AWS service calls: S3, DynamoDB, SQS, SNS, Lambda, SES, etc. |
582
+ | `firebase` | `firebase-admin`, `@google-cloud/firestore` | Firestore CRUD/queries, Auth (16 methods), FCM Messaging (9 methods) |
583
+ | `generic-pool` | `generic-pool` | Pool acquire/release, pool size, pending count, queue size |
584
+
585
+ ### Logging (3)
586
+
587
+ | Instrumentation Key | npm Package(s) | Captured Signals |
588
+ |---------------------|----------------|------------------|
589
+ | `pino` | `pino` | Trace/span ID injection into log records |
590
+ | `winston` | `winston` | Trace/span ID injection into transport output |
591
+ | `bunyan` | `bunyan` | Trace/span ID injection into serialized records |
592
+
593
+ ### RPC & Network (4)
594
+
595
+ | Instrumentation Key | npm Package(s) | Captured Signals |
596
+ |---------------------|----------------|------------------|
597
+ | `grpc` | `@grpc/grpc-js` | Unary/streaming calls, service/method, status codes, metadata |
598
+ | `graphql` | `graphql` | Resolver execution, operation name/type, field paths, errors |
599
+ | `dns` | Node built-in `dns` | Lookups, resolve, hostname, record types, timing |
600
+ | `net` | Node built-in `net` | TCP socket connect, data transfer, connection duration |
601
+
602
+ ### Utilities (3)
603
+
604
+ | Instrumentation Key | npm Package(s) | Captured Signals |
605
+ |---------------------|----------------|------------------|
606
+ | `dataloader` | `dataloader` | Batch load calls, batch size, individual key loads |
607
+ | `lru-memoizer` | `lru-memoizer` | Memoized function calls, cache hit/miss |
608
+ | `fs` | Node built-in `fs` | readFile, writeFile, stat, readdir, mkdir, unlink, rename, etc. |
609
+
610
+ ---
611
+
612
+ ## 7. AI / LLM SDK Instrumentation
613
+
614
+ All AI SDK instrumentations follow [OpenTelemetry GenAI semantic conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/).
615
+
616
+ ### Common Attributes Captured
617
+
618
+ | Attribute | Description |
619
+ |-----------|-------------|
620
+ | `gen_ai.system` | `openai`, `anthropic`, `google_ai`, `vertex_ai`, `azure_openai`, `cohere`, `mistral`, `aws_bedrock` |
621
+ | `gen_ai.request.model` | Model name from request (e.g., `gpt-4o`, `claude-sonnet-4-20250514`, `gemini-1.5-pro`) |
622
+ | `gen_ai.response.model` | Actual model from response (may differ for aliases) |
623
+ | `gen_ai.operation.name` | `chat`, `embeddings`, `images`, `audio`, `fim`, `rerank`, `classify` |
624
+ | `gen_ai.usage.input_tokens` | Prompt/input token count |
625
+ | `gen_ai.usage.output_tokens` | Completion/output token count |
626
+ | `gen_ai.usage.total_tokens` | Total tokens (when available) |
627
+ | `gen_ai.response.finish_reason` | `stop`, `length`, `tool_calls`, `end_turn`, etc. |
628
+
629
+ ### OpenAI
630
+
631
+ ```ts
632
+ import OpenAI from 'openai';
633
+
634
+ const openai = new OpenAI();
635
+ const response = await openai.chat.completions.create({
636
+ model: 'gpt-4o',
637
+ messages: [{ role: 'user', content: 'Hello' }],
638
+ });
639
+ // Span: "OpenAI chat gpt-4o" with token usage
640
+ ```
641
+
642
+ ### Anthropic
643
+
644
+ ```ts
645
+ import Anthropic from '@anthropic-ai/sdk';
646
+
647
+ const client = new Anthropic();
648
+ const message = await client.messages.create({
649
+ model: 'claude-sonnet-4-20250514',
650
+ max_tokens: 1024,
651
+ messages: [{ role: 'user', content: 'Hello' }],
652
+ });
653
+ // Span: "Anthropic messages claude-sonnet-4-20250514" with input/output tokens
654
+ ```
655
+
656
+ ### Google Gemini
657
+
658
+ ```ts
659
+ import { GoogleGenerativeAI } from '@google/generative-ai';
660
+
661
+ const genai = new GoogleGenerativeAI(process.env.GOOGLE_API_KEY!);
662
+ const model = genai.getGenerativeModel({ model: 'gemini-1.5-pro' });
663
+ const result = await model.generateContent('Hello');
664
+ // Span: "Gemini generateContent gemini-1.5-pro" with token usage
665
+ ```
666
+
667
+ ### Cohere
668
+
669
+ ```ts
670
+ import { CohereClient } from 'cohere-ai';
671
+
672
+ const cohere = new CohereClient({ token: process.env.COHERE_API_KEY! });
673
+ const response = await cohere.chat({ model: 'command-r-plus', message: 'Hello' });
674
+ // Span: "Cohere chat command-r-plus" with billed units
675
+ ```
676
+
677
+ ### Mistral
678
+
679
+ ```ts
680
+ import { Mistral } from '@mistralai/mistralai';
681
+
682
+ const mistral = new Mistral({ apiKey: process.env.MISTRAL_API_KEY! });
683
+ const response = await mistral.chat.complete({
684
+ model: 'mistral-large-latest',
685
+ messages: [{ role: 'user', content: 'Hello' }],
686
+ });
687
+ // Span: "Mistral chat mistral-large-latest" with token usage
688
+ ```
689
+
690
+ ### AWS Bedrock
691
+
692
+ AWS Bedrock calls made through `@aws-sdk/client-bedrock-runtime` are automatically captured with GenAI attributes via the AWS SDK instrumentation. Both InvokeModel and Converse APIs are supported.
693
+
694
+ ```ts
695
+ import { BedrockRuntimeClient, InvokeModelCommand } from '@aws-sdk/client-bedrock-runtime';
696
+
697
+ const client = new BedrockRuntimeClient({ region: 'us-east-1' });
698
+ const response = await client.send(new InvokeModelCommand({
699
+ modelId: 'anthropic.claude-3-sonnet-20240229-v1:0',
700
+ body: JSON.stringify({ messages: [{ role: 'user', content: 'Hello' }] }),
701
+ }));
702
+ // Span: "AWS BedrockRuntime InvokeModel" with gen_ai.* attributes and token usage
703
+ ```
704
+
705
+ ### Azure OpenAI (v1.x)
706
+
707
+ ```ts
708
+ import { OpenAIClient, AzureKeyCredential } from '@azure/openai';
709
+
710
+ const client = new OpenAIClient(endpoint, new AzureKeyCredential(key));
711
+ const response = await client.getChatCompletions('gpt-4-deployment', messages);
712
+ // Span: "Azure OpenAI chat gpt-4-deployment" with token usage
713
+ ```
714
+
715
+ Note: Azure OpenAI SDK v2+ wraps the standard `openai` npm package internally, which is already covered by the OpenAI instrumentation.
716
+
717
+ ---
718
+
719
+ ## 8. Database & Cache Instrumentation
720
+
721
+ ### PostgreSQL (`pg`)
722
+
723
+ ```ts
724
+ const result = await pool.query('SELECT * FROM users WHERE id = $1', [userId]);
725
+ ```
726
+
727
+ Span: `Postgres SELECT` with `db.system.name: 'postgresql'`, `db.operation.name: 'SELECT'`, sanitized SQL, row count.
728
+
729
+ ### MongoDB (`mongodb`)
730
+
731
+ ```ts
732
+ const user = await db.collection('users').findOne({ _id: userId });
733
+ ```
734
+
735
+ Span: `Mongo users findOne` with `db.system.name: 'mongodb'`, `db.collection.name: 'users'`, `db.operation.name: 'findOne'`.
736
+
737
+ ### Mongoose
738
+
739
+ ```ts
740
+ const user = await User.findById(userId).exec();
741
+ ```
742
+
743
+ Span includes model name, collection name, and operation.
744
+
745
+ ### MySQL / MySQL2
746
+
747
+ ```ts
748
+ const [rows] = await connection.execute('SELECT * FROM users WHERE id = ?', [userId]);
749
+ ```
750
+
751
+ Span: `MySQL SELECT` with sanitized SQL and row count.
752
+
753
+ ### Redis / ioredis
754
+
755
+ ```ts
756
+ await redis.get(`user:${userId}`);
757
+ ```
758
+
759
+ Span: `Redis GET` with command name and key.
760
+
761
+ ### Knex
762
+
763
+ ```ts
764
+ const users = await knex('users').where({ active: true }).select('*');
765
+ ```
766
+
767
+ Span: `Knex SELECT users` with query builder context, sanitized SQL, and binding count.
768
+
769
+ ### SQL Server (Tedious)
770
+
771
+ ```ts
772
+ const request = new Request('SELECT * FROM Users WHERE Id = @id', callback);
773
+ request.addParameter('id', TYPES.Int, userId);
774
+ connection.execSql(request);
775
+ ```
776
+
777
+ Span: `SQLServer SELECT` with sanitized T-SQL, row count, and stored procedure name.
778
+
779
+ ### Cassandra
780
+
781
+ ```ts
782
+ const result = await client.execute('SELECT * FROM users WHERE id = ?', [userId], { prepare: true });
783
+ ```
784
+
785
+ Span: `Cassandra SELECT` with `db.system.name: 'cassandra'`, keyspace, consistency level, sanitized CQL.
786
+
787
+ ### Memcached
788
+
789
+ ```ts
790
+ memcached.get('session:abc', (err, data) => { });
791
+ ```
792
+
793
+ Span: `Memcached GET` with command name and key.
794
+
795
+ ---
796
+
797
+ ## 9. Messaging & Queue Instrumentation
798
+
799
+ ### Kafka (kafkajs)
800
+
801
+ ```ts
802
+ // Producer
803
+ await producer.send({ topic: 'orders', messages: [{ value: JSON.stringify(order) }] });
804
+ // Span: "Kafka SEND orders" with topic, partition, message count
805
+
806
+ // Consumer
807
+ await consumer.run({
808
+ eachMessage: async ({ topic, message }) => { /* ... */ },
809
+ });
810
+ // Span: "Kafka RECEIVE orders" with topic, partition, offset
811
+ ```
812
+
813
+ ### RabbitMQ (amqplib)
814
+
815
+ ```ts
816
+ // Publish
817
+ channel.publish('exchange', 'routing.key', Buffer.from(JSON.stringify(data)));
818
+ // Span: "RabbitMQ PUBLISH exchange" with exchange, routing key
819
+
820
+ // Consume
821
+ channel.consume('queue-name', (msg) => { channel.ack(msg); });
822
+ // Span: "RabbitMQ RECEIVE queue-name" with queue name, delivery tag
823
+ ```
824
+
825
+ ### Socket.IO
826
+
827
+ ```ts
828
+ io.on('connection', (socket) => {
829
+ socket.on('message', (data) => { /* ... */ });
830
+ socket.emit('response', { ok: true });
831
+ });
832
+ // Spans for emit and receive with event names, namespace, room
833
+ ```
834
+
835
+ ---
836
+
837
+ ## 10. Cloud Provider Instrumentation
838
+
839
+ ### AWS SDK v3
840
+
841
+ All AWS services are instrumented through the single `Client.send()` dispatch point:
842
+
843
+ ```ts
844
+ import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
845
+
846
+ const s3 = new S3Client({ region: 'us-east-1' });
847
+ await s3.send(new PutObjectCommand({ Bucket: 'my-bucket', Key: 'file.txt', Body: data }));
848
+ // Span: "AWS S3 PutObject" with rpc.service, rpc.method, aws.request_id, aws.region
849
+ ```
850
+
851
+ Covers: S3, DynamoDB, SQS, SNS, Lambda, SES, CloudWatch, Kinesis, EventBridge, Secrets Manager, SSM, STS, IAM, EC2, ECS, EKS, RDS, ElastiCache, Redshift, Cognito, Route53, CloudFront, API Gateway, Step Functions, CodeBuild, KMS, Bedrock Runtime, and any other AWS service using SDK v3.
852
+
853
+ ### Firebase Admin
854
+
855
+ **Firestore:**
856
+
857
+ ```ts
858
+ const doc = await db.collection('users').doc('user123').get();
859
+ await db.collection('users').doc('user123').set({ name: 'Alice' });
860
+ const snapshot = await db.collection('users').where('active', '==', true).get();
861
+ // Spans: "Firestore GET users/user123", "Firestore SET users/user123", "Firestore QUERY users"
862
+ ```
863
+
864
+ Captured operations: GET, SET, ADD, UPDATE, DELETE, CREATE, LIST, QUERY, TX_GET, BATCH_COMMIT.
865
+
866
+ **Auth:**
867
+
868
+ ```ts
869
+ const user = await admin.auth().getUser(uid);
870
+ const token = await admin.auth().verifyIdToken(idToken);
871
+ // Spans: "Firebase Auth getUser", "Firebase Auth verifyIdToken"
872
+ ```
873
+
874
+ 16 methods captured: createUser, getUser, getUserByEmail, getUserByPhoneNumber, listUsers, deleteUser, deleteUsers, updateUser, verifyIdToken, verifySessionCookie, createSessionCookie, revokeRefreshTokens, setCustomUserClaims, generateEmailVerificationLink, generatePasswordResetLink, generateSignInWithEmailLink.
875
+
876
+ **FCM Messaging:**
877
+
878
+ ```ts
879
+ await admin.messaging().send({ notification: { title: 'Hi' }, token: deviceToken });
880
+ // Span: "Firebase FCM send" with success/failure counts
881
+ ```
882
+
883
+ 9 methods captured: send, sendEach, sendEachForMulticast, sendMulticast, sendToDevice, sendToTopic, sendToCondition, subscribeToTopic, unsubscribeFromTopic.
884
+
885
+ ---
886
+
887
+ ## 11. gRPC, GraphQL & Network
888
+
889
+ ### gRPC
890
+
891
+ ```ts
892
+ // Server and client calls automatically instrumented
893
+ // Span: "grpc /package.Service/Method" with rpc.system, rpc.service, rpc.method, status code
894
+ ```
895
+
896
+ Supports unary, client streaming, server streaming, and bidirectional streaming.
897
+
898
+ ### GraphQL
899
+
900
+ ```ts
901
+ // Resolver-level instrumentation
902
+ // Span: "GraphQL query GetUser" with operation name, operation type, field path
903
+ ```
904
+
905
+ Captures query/mutation/subscription operations and individual field resolver execution.
906
+
907
+ ### DNS
908
+
909
+ ```ts
910
+ import dns from 'dns';
911
+
912
+ dns.lookup('example.com', (err, address) => { /* ... */ });
913
+ // Span: "DNS lookup example.com" with hostname, record type
914
+ ```
915
+
916
+ ### Net (TCP Sockets)
917
+
918
+ ```ts
919
+ import net from 'net';
920
+
921
+ const socket = net.createConnection({ host: 'db.internal', port: 5432 });
922
+ // Span: "TCP CONNECT db.internal:5432" with host, port, connection duration
923
+ ```
924
+
925
+ ---
926
+
927
+ ## 12. Log Correlation
928
+
929
+ ### Console Logs
930
+
931
+ All `console.log`, `console.info`, `console.warn`, `console.error`, and `console.debug` calls are automatically captured and correlated with the active trace or task context.
932
+
933
+ ```ts
934
+ app.get('/users/:id', async (req, res) => {
935
+ console.info('fetching user', { userId: req.params.id });
936
+ // Log is captured with traceId, level "info", and attributes { userId: "123" }
937
+ const user = await getUser(req.params.id);
938
+ res.json(user);
939
+ });
940
+ ```
941
+
942
+ ### Structured Logging Libraries
943
+
944
+ Pino, Winston, and Bunyan get trace/span IDs automatically injected into log records:
945
+
946
+ ```ts
947
+ import pino from 'pino';
948
+ const logger = pino();
949
+
950
+ app.get('/users/:id', (req, res) => {
951
+ logger.info({ userId: req.params.id }, 'fetching user');
952
+ // Output includes: traceId, spanId alongside your log fields
953
+ });
954
+ ```
955
+
956
+ Disable console log capture:
957
+
958
+ ```ts
959
+ Senzor.init({ apiKey: '...', autoLogs: false });
960
+ ```
961
+
962
+ ---
963
+
964
+ ## 13. Background Task Monitoring
965
+
966
+ ### Auto-Instrumented
967
+
968
+ **BullMQ** workers are captured as task runs with queue delay, retry counts, dead-letter detection, and CPU/memory resource metrics.
969
+
970
+ **node-cron** scheduled jobs are captured as task runs with the cron schedule expression.
971
+
972
+ ### Manual Task Wrapping
973
+
974
+ ```ts
975
+ const processPayment = Senzor.wrapTask(
976
+ 'process_payment', // task name
977
+ 'custom', // type: 'cron' | 'queue' | 'pipeline' | 'custom'
978
+ { metadata: { owner: 'billing' } },
979
+ async (invoiceId: string) => {
980
+ await chargeCustomer(invoiceId);
981
+ }
982
+ );
983
+
984
+ await processPayment('inv_123');
985
+ ```
986
+
987
+ ### Low-Level Task API
988
+
989
+ ```ts
990
+ Senzor.startTask('rebuild_index', 'custom', { metadata: { trigger: 'manual' } }, async () => {
991
+ try {
992
+ await rebuildSearchIndex();
993
+ // endTask is called automatically on success
994
+ } catch (error) {
995
+ Senzor.captureException(error);
996
+ throw error; // endTask('failed') on throw
997
+ }
998
+ });
999
+ ```
1000
+
1001
+ Task payloads include:
1002
+ - `taskName`, `taskType`, `status`, `duration`
1003
+ - `queueDelay`, `attempts`, `isDeadLetter` (for BullMQ)
1004
+ - `resourceMetrics`: heap memory delta, CPU user/system time
1005
+ - `triggerTraceId`: links tasks triggered by HTTP requests
1006
+
1007
+ ---
1008
+
1009
+ ## 14. Manual Traces & Spans
1010
+
1011
+ ### Manual Span
1012
+
1013
+ ```ts
1014
+ const span = Senzor.startSpan('calculate_invoice', 'function');
1015
+ try {
1016
+ const total = await calculateInvoice(invoiceId);
1017
+ span.end({ invoiceId, total }, 200);
1018
+ } catch (error) {
1019
+ span.end({ invoiceId, error: String(error) }, 500);
1020
+ throw error;
1021
+ }
1022
+ ```
1023
+
1024
+ Span types: `http`, `db`, `function`, `custom`, `rpc`, `messaging`, `dns`, `net`.
1025
+
1026
+ ### Manual Trace
1027
+
1028
+ For environments where framework wrappers can't be used:
1029
+
1030
+ ```ts
1031
+ Senzor.track({
1032
+ method: 'POST',
1033
+ route: '/webhooks/payment',
1034
+ path: '/webhooks/payment',
1035
+ status: 202,
1036
+ duration: 18.7,
1037
+ spans: [],
1038
+ });
1039
+ ```
1040
+
1041
+ ---
1042
+
1043
+ ## 15. Error Tracking
1044
+
1045
+ ### Automatic
1046
+
1047
+ The SDK captures these global events:
1048
+
1049
+ | Event | Severity |
1050
+ |-------|----------|
1051
+ | `uncaughtExceptionMonitor` | fatal |
1052
+ | `uncaughtException` | fatal |
1053
+ | `unhandledRejection` | error |
1054
+ | `warning` | warning |
1055
+ | `multipleResolves` | warning |
1056
+ | `SIGTERM` | info |
1057
+ | `SIGINT` | info |
1058
+
1059
+ Each captured error includes: stack trace, process context (pid, platform, uptime, NODE_ENV), memory snapshot (RSS, heap, external), and SDK metadata.
1060
+
1061
+ ### Manual
1062
+
1063
+ ```ts
1064
+ try {
1065
+ await chargeCustomer(customerId);
1066
+ } catch (error) {
1067
+ Senzor.captureException(error, { customerId, operation: 'charge' });
1068
+ throw error;
1069
+ }
1070
+ ```
1071
+
1072
+ Errors are correlated with the active trace or task context.
1073
+
1074
+ ---
1075
+
1076
+ ## 16. Runtime Metrics
1077
+
1078
+ Collected every 15 seconds (configurable) and sent with trace batches.
1079
+
1080
+ | Metric | Description |
1081
+ |--------|-------------|
1082
+ | `eventLoop.lag.p50` | Event loop lag, 50th percentile (ms) |
1083
+ | `eventLoop.lag.p99` | Event loop lag, 99th percentile (ms) |
1084
+ | `eventLoop.lag.max` | Event loop lag, maximum observed (ms) |
1085
+ | `eventLoop.utilization` | Event loop utilization (0-1, from `perf_hooks` ELU) |
1086
+ | `gc.duration.minor` | GC pause time, minor collections (ms) |
1087
+ | `gc.duration.major` | GC pause time, major collections (ms) |
1088
+ | `gc.duration.incremental` | GC pause time, incremental marking (ms) |
1089
+ | `memory.heapUsed` | V8 heap used (bytes) |
1090
+ | `memory.heapTotal` | V8 heap total (bytes) |
1091
+ | `memory.rss` | Resident set size (bytes) |
1092
+ | `memory.external` | External memory (bytes) |
1093
+ | `memory.arrayBuffers` | ArrayBuffer memory (bytes) |
1094
+ | `activeHandles` | Open handles (file descriptors, sockets) |
1095
+ | `activeRequests` | Pending async requests |
1096
+
1097
+ Disable runtime metrics:
1098
+
1099
+ ```ts
1100
+ Senzor.init({ apiKey: '...', runtimeMetrics: false });
1101
+ ```
1102
+
1103
+ Adjust collection interval:
1104
+
1105
+ ```ts
1106
+ Senzor.init({ apiKey: '...', runtimeMetricsInterval: 30000 }); // 30 seconds
1107
+ ```
1108
+
1109
+ Runtime metrics are automatically disabled in AWS Lambda environments.
1110
+
1111
+ ---
1112
+
1113
+ ## 17. Distributed Tracing & Context Propagation
1114
+
1115
+ ### Outgoing Headers
1116
+
1117
+ The SDK injects these headers on all outgoing HTTP calls:
1118
+
1119
+ ```
1120
+ traceparent: 00-{traceId}-{spanId}-01
1121
+ x-senzor-trace-id: {traceId}
1122
+ x-senzor-parent-span-id: {spanId}
1123
+ ```
1124
+
1125
+ ### Incoming Headers
1126
+
1127
+ Incoming requests with `traceparent` or `x-senzor-trace-id` headers are linked as child traces, enabling cross-service distributed trace visualization.
1128
+
1129
+ ### Context Propagation
1130
+
1131
+ The SDK uses `AsyncLocalStorage` to maintain trace context across async boundaries. All child spans, logs, and errors within a request are automatically correlated to the correct trace without any manual threading.
1132
+
1133
+ ---
1134
+
1135
+ ## 18. Configuration Reference
1136
+
1137
+ ```ts
1138
+ Senzor.init({
1139
+ apiKey: 'sz_apm_xxx', // Required
1140
+ endpoint: 'https://api.senzor.dev', // Ingest endpoint
1141
+ batchSize: 100, // Flush threshold
1142
+ flushInterval: 10000, // Flush interval (ms)
1143
+ flushTimeoutMs: 5000, // Per-request timeout (ms)
1144
+ maxQueueSize: 10000, // Max queued items before drop
1145
+ maxSpansPerTrace: 500, // Max spans per trace
1146
+ maxAttributeLength: 2048, // Max string length
1147
+ maxAttributes: 64, // Max attributes per object
1148
+ captureHeaders: false, // Capture sanitized headers
1149
+ captureDbStatement: true, // Capture sanitized SQL
1150
+ instrumentations: true, // true | false | string[]
1151
+ frameworkSpans: true, // Framework middleware/handler spans
1152
+ captureMiddlewareSpans: true, // Middleware execution spans
1153
+ captureRouterSpans: true, // Router dispatch spans
1154
+ captureLifecycleHookSpans: true, // Lifecycle hook spans
1155
+ autoLogs: true, // Console log capture
1156
+ runtimeMetrics: true, // Runtime metrics collection
1157
+ runtimeMetricsInterval: 15000, // Collection interval (ms)
1158
+ debug: false, // SDK diagnostics
1159
+ });
1160
+ ```
1161
+
1162
+ | Option | Production Guidance |
1163
+ |--------|-------------------|
1164
+ | `apiKey` | Always use environment variables or secret manager. Never hardcode. |
1165
+ | `batchSize` | Start at `100`. Increase for high-throughput services. |
1166
+ | `flushInterval` | `10000` ms for servers. `0` for Lambda (flush on demand). |
1167
+ | `maxQueueSize` | Keep bounded. Increase only after checking memory pressure. |
1168
+ | `maxSpansPerTrace` | Keep `500` unless traces fan out heavily (e.g., batch operations). |
1169
+ | `captureHeaders` | Keep `false` unless needed for debugging. |
1170
+ | `debug` | Never enable in production unless actively troubleshooting. |
1171
+
1172
+ ### Selective Instrumentation
1173
+
1174
+ ```ts
1175
+ // Enable only specific instrumentations
1176
+ Senzor.init({
1177
+ apiKey: '...',
1178
+ instrumentations: ['http', 'fetch', 'pg', 'redis', 'openai'],
1179
+ });
1180
+
1181
+ // Disable all auto-instrumentation (manual APIs only)
1182
+ Senzor.init({
1183
+ apiKey: '...',
1184
+ instrumentations: false,
1185
+ });
1186
+ ```
1187
+
1188
+ ---
1189
+
1190
+ ## 19. Environment Variables
1191
+
1192
+ The preload entrypoint (`@senzops/apm-node/register`) reads:
1193
+
1194
+ | Variable | Maps To | Default |
1195
+ |----------|---------|---------|
1196
+ | `SENZOR_API_KEY` | `apiKey` | — |
1197
+ | `SENZOR_APM_API_KEY` | `apiKey` (alt) | — |
1198
+ | `SENZOR_SERVICE_API_KEY` | `apiKey` (alt) | — |
1199
+ | `SENZOR_ENDPOINT` | `endpoint` | `https://api.senzor.dev` |
1200
+ | `SENZOR_APM_ENDPOINT` | `endpoint` (alt) | — |
1201
+ | `SENZOR_DEBUG` | `debug` | `false` |
1202
+ | `SENZOR_AUTO_LOGS` | `autoLogs` | `true` |
1203
+ | `SENZOR_BATCH_SIZE` | `batchSize` | `100` (or `10` in Lambda) |
1204
+ | `SENZOR_FLUSH_INTERVAL` | `flushInterval` | `10000` (or `0` in Lambda) |
1205
+ | `SENZOR_FLUSH_TIMEOUT_MS` | `flushTimeoutMs` | `5000` |
1206
+ | `SENZOR_MAX_QUEUE_SIZE` | `maxQueueSize` | `10000` |
1207
+ | `SENZOR_MAX_SPANS_PER_TRACE` | `maxSpansPerTrace` | `500` |
1208
+ | `SENZOR_CAPTURE_HEADERS` | `captureHeaders` | `false` |
1209
+ | `SENZOR_CAPTURE_DB_STATEMENT` | `captureDbStatement` | `true` |
1210
+ | `SENZOR_FRAMEWORK_SPANS` | `frameworkSpans` | `true` |
1211
+ | `SENZOR_CAPTURE_MIDDLEWARE_SPANS` | `captureMiddlewareSpans` | `true` |
1212
+ | `SENZOR_CAPTURE_ROUTER_SPANS` | `captureRouterSpans` | `true` |
1213
+ | `SENZOR_CAPTURE_LIFECYCLE_HOOK_SPANS` | `captureLifecycleHookSpans` | `true` |
1214
+ | `SENZOR_RUNTIME_METRICS` | `runtimeMetrics` | `true` (auto `false` in Lambda) |
1215
+ | `SENZOR_RUNTIME_METRICS_INTERVAL` | `runtimeMetricsInterval` | `15000` |
1216
+
1217
+ Environment variable values always take precedence over programmatic configuration.
1218
+
1219
+ ---
1220
+
1221
+ ## 20. Security, Privacy & Cardinality
1222
+
1223
+ ### Automatic Redaction
1224
+
1225
+ These keys are automatically redacted from attributes, headers, logs, and error context:
1226
+
1227
+ `authorization`, `cookie`, `set-cookie`, `password`, `passwd`, `pwd`, `secret`, `token`, `api-key`, `x-api-key`, `access-token`, `refresh-token`, `client-secret`, `private-key`
1228
+
1229
+ ### Cardinality Controls
1230
+
1231
+ - URL routes are normalized: `/users/123` becomes `/users/:id`
1232
+ - UUIDs and MongoDB ObjectIDs are replaced with `:id` in route names
1233
+ - SQL statements are normalized to remove literal values
1234
+ - Attribute count and string length are bounded
1235
+ - Span count per trace is bounded
1236
+ - Queue size is bounded with oldest-first eviction
1237
+
1238
+ ### Recommendations
1239
+
1240
+ - Keep `captureHeaders` disabled unless required for debugging
1241
+ - Never capture request/response bodies
1242
+ - Use route templates (e.g., `/users/:id`) rather than actual paths in span names
1243
+ - Store IDs in metadata, not in operation names
1244
+ - Use `instrumentations: [...]` to disable instrumentations you don't need
1245
+
1246
+ ---
1247
+
1248
+ ## 21. Transport Behavior
1249
+
1250
+ The SDK batches telemetry and sends it asynchronously using two independent queues:
1251
+
1252
+ - **APM Queue**: traces, errors, logs, runtime metrics -> `/api/ingest/apm`
1253
+ - **Task Queue**: task runs, task errors, task logs -> `/api/ingest/task`
1254
+
1255
+ Behavior:
1256
+
1257
+ - Queues are bounded by `maxQueueSize` (oldest items evicted first)
1258
+ - Flush triggers on `batchSize` threshold or `flushInterval` timer
1259
+ - Each ingest request has a `flushTimeoutMs` timeout
1260
+ - Failed batches are requeued within queue limits (retry on next flush)
1261
+ - SDK ingest requests are marked as internal (not traced recursively)
1262
+ - A best-effort flush runs on `process.beforeExit`
1263
+ - In Lambda, flush is forced before handler return + Extensions API SHUTDOWN hook
1264
+
1265
+ ### Serverless Flush
1266
+
1267
+ Always call `await Senzor.flush()` before the function exits in serverless environments (unless using `wrapLambda` or `Senzor.worker()`, which handle this automatically).
1268
+
1269
+ ---
1270
+
1271
+ ## 22. Ingestion Payload Format
1272
+
1273
+ ### APM Ingest (`POST /api/ingest/apm`)
1274
+
1275
+ ```json
1276
+ {
1277
+ "traces": [{
1278
+ "traceId": "f3b2c2c9c70443f5a4b7f0ff6d5b9a17",
1279
+ "parentTraceId": "8cb1f2e2f43e4b2b8b3411c7df81e7e0",
1280
+ "parentSpanId": "6c62c908a21d4f49",
1281
+ "rootSpanId": "91f6c551d5a2403f",
1282
+ "method": "GET",
1283
+ "route": "/orders/:orderId",
1284
+ "path": "/orders/ord_123",
1285
+ "status": 200,
1286
+ "duration": 53.29,
1287
+ "ip": "203.0.113.10",
1288
+ "userAgent": "Mozilla/5.0",
1289
+ "timestamp": "2026-05-26T15:30:00.000Z",
1290
+ "spans": [{
1291
+ "spanId": "9d8a4d5f17e24d2a",
1292
+ "parentSpanId": "91f6c551d5a2403f",
1293
+ "name": "Postgres SELECT",
1294
+ "type": "db",
1295
+ "startTime": 5.41,
1296
+ "duration": 12.45,
1297
+ "status": 0,
1298
+ "meta": {
1299
+ "db.system.name": "postgresql",
1300
+ "db.operation.name": "SELECT",
1301
+ "db.query.text": "SELECT id FROM users WHERE id = $?"
1302
+ }
1303
+ }]
1304
+ }],
1305
+ "errors": [{
1306
+ "errorClass": "Error",
1307
+ "message": "Connection refused",
1308
+ "stackTrace": "Error: Connection refused\n at ...",
1309
+ "traceId": "f3b2c2c9c70443f5a4b7f0ff6d5b9a17",
1310
+ "context": { "operation": "db_connect" },
1311
+ "timestamp": "2026-05-26T15:30:00.000Z"
1312
+ }],
1313
+ "logs": [{
1314
+ "message": "order processed",
1315
+ "level": "info",
1316
+ "traceId": "f3b2c2c9c70443f5a4b7f0ff6d5b9a17",
1317
+ "attributes": { "orderId": "ord_123" },
1318
+ "timestamp": "2026-05-26T15:30:00.000Z"
1319
+ }],
1320
+ "runtimeMetrics": [{
1321
+ "timestamp": "2026-05-26T15:30:00.000Z",
1322
+ "eventLoop": { "lagP50": 1.2, "lagP99": 4.8, "lagMax": 12.1, "utilization": 0.34 },
1323
+ "gc": { "minor": 2.1, "major": 0, "incremental": 0.8 },
1324
+ "memory": { "heapUsed": 45000000, "heapTotal": 67000000, "rss": 89000000 }
1325
+ }]
1326
+ }
1327
+ ```
1328
+
1329
+ ### Task Ingest (`POST /api/ingest/task`)
1330
+
1331
+ ```json
1332
+ {
1333
+ "runs": [{
1334
+ "runId": "3917bd35-b1d6-4e23-a1d2-d969e1a7d6a1",
1335
+ "taskName": "billing:send_invoice_email",
1336
+ "taskType": "queue",
1337
+ "status": "success",
1338
+ "duration": 188.3,
1339
+ "queueDelay": 92,
1340
+ "attempts": 1,
1341
+ "triggerTraceId": "f3b2c2c9c70443f5a4b7f0ff6d5b9a17",
1342
+ "metadata": { "jobId": "142", "queueName": "billing" },
1343
+ "resourceMetrics": {
1344
+ "memoryDeltaBytes": 1048576,
1345
+ "cpuUserUs": 12000,
1346
+ "cpuSystemUs": 3000
1347
+ },
1348
+ "isDeadLetter": false,
1349
+ "spans": [],
1350
+ "timestamp": "2026-05-26T15:30:00.000Z"
1351
+ }],
1352
+ "errors": [],
1353
+ "logs": []
1354
+ }
1355
+ ```
1356
+
1357
+ ---
1358
+
1359
+ ## 23. Deployment Patterns
1360
+
1361
+ ### Docker
1362
+
1363
+ ```dockerfile
1364
+ FROM node:20-alpine
1365
+ WORKDIR /app
1366
+ COPY package*.json ./
1367
+ RUN npm ci --production
1368
+ COPY dist/ dist/
1369
+
1370
+ ENV SENZOR_API_KEY=sz_apm_xxx
1371
+ CMD ["node", "-r", "@senzops/apm-node/register", "dist/server.js"]
1372
+ ```
1373
+
1374
+ ### Kubernetes
1375
+
1376
+ ```yaml
1377
+ apiVersion: apps/v1
1378
+ kind: Deployment
1379
+ spec:
1380
+ template:
1381
+ spec:
1382
+ containers:
1383
+ - name: api
1384
+ image: my-api:latest
1385
+ env:
1386
+ - name: SENZOR_API_KEY
1387
+ valueFrom:
1388
+ secretKeyRef:
1389
+ name: senzor-secrets
1390
+ key: api-key
1391
+ - name: NODE_OPTIONS
1392
+ value: "-r @senzops/apm-node/register"
1393
+ ```
1394
+
1395
+ ### PM2
1396
+
1397
+ ```json
1398
+ {
1399
+ "apps": [{
1400
+ "name": "orders-api",
1401
+ "script": "dist/server.js",
1402
+ "node_args": "-r @senzops/apm-node/register",
1403
+ "env": {
1404
+ "SENZOR_API_KEY": "sz_apm_xxx",
1405
+ "NODE_ENV": "production"
1406
+ }
1407
+ }]
1408
+ }
1409
+ ```
1410
+
1411
+ ### AWS Lambda
1412
+
1413
+ See [Section 4: AWS Lambda Integration](#4-aws-lambda-integration).
1414
+
1415
+ ### Cloudflare Workers
1416
+
1417
+ See [Section 5: Cloudflare Workers Integration](#5-cloudflare-workers-integration).
1418
+
1419
+ ### Vercel (Serverless Functions)
1420
+
1421
+ ```ts
1422
+ // api/route.ts
1423
+ import Senzor from '@senzops/apm-node';
1424
+
1425
+ Senzor.init({ apiKey: process.env.SENZOR_API_KEY! });
1426
+
1427
+ export const GET = Senzor.wrapNextRoute(async (req) => {
1428
+ const result = await fetchData();
1429
+ return Response.json(result);
1430
+ });
1431
+ ```
1432
+
1433
+ ---
1434
+
1435
+ ## 24. Troubleshooting
1436
+
1437
+ ### Missing Spans
1438
+
1439
+ 1. **Check startup order.** Preload mode must load before application modules:
1440
+ ```sh
1441
+ node -r @senzops/apm-node/register server.js
1442
+ ```
1443
+ 2. **Programmatic mode:** Initialize `Senzor.init()` before importing `pg`, `mongodb`, `redis`, etc.
1444
+ 3. **Check if the library is supported** in the [instrumentation table](#6-auto-instrumentation-reference).
1445
+
1446
+ ### No Data in Senzor
1447
+
1448
+ 1. Verify `SENZOR_API_KEY` is set and valid.
1449
+ 2. Verify `endpoint` points to the correct Senzor environment.
1450
+ 3. Verify the runtime can reach the ingest endpoint (check firewall, VPC, DNS).
1451
+ 4. Enable `debug: true` to see flush attempts.
1452
+ 5. In serverless environments, ensure `Senzor.flush()` is called (or use `wrapLambda`/`worker()`).
1453
+
1454
+ ### Duplicate Traces
1455
+
1456
+ The SDK deduplicates active traces. If duplicates still appear:
1457
+ - Check whether both preload mode AND manual `track()` calls are active for the same request.
1458
+ - Ensure `Senzor.requestHandler()` is only attached once.
1459
+
1460
+ ### High Memory Usage
1461
+
1462
+ ```ts
1463
+ Senzor.init({
1464
+ apiKey: '...',
1465
+ maxQueueSize: 5000, // Reduce queue bounds
1466
+ maxSpansPerTrace: 250, // Reduce span retention
1467
+ maxAttributes: 32, // Reduce attribute count
1468
+ maxAttributeLength: 1024, // Reduce string lengths
1469
+ });
1470
+ ```
1471
+
1472
+ ### High-Cardinality Routes
1473
+
1474
+ Use framework wrappers that provide route templates:
1475
+
1476
+ ```ts
1477
+ app.use(Senzor.requestHandler()); // Captures /users/:id instead of /users/123
1478
+ ```
1479
+
1480
+ For manual traces, use normalized routes:
1481
+
1482
+ ```ts
1483
+ Senzor.track({ method: 'GET', route: '/users/:id', path: '/users/123', ... });
1484
+ ```
1485
+
1486
+ ### Lambda Cold Start Issues
1487
+
1488
+ - Ensure `Senzor.init()` or preload runs outside the handler (module scope).
1489
+ - Check that `NODE_OPTIONS` is set correctly for Lambda Layer deployments.
1490
+ - Use `Senzor.wrapLambda()` for proper cold start tagging.
1491
+
1492
+ ---
1493
+
1494
+ ## 25. Public API Reference
1495
+
1496
+ ```ts
1497
+ // Initialization
1498
+ Senzor.init(options: SenzorOptions) // Initialize SDK with API key
1499
+ Senzor.preload(options?: Partial<SenzorOptions>) // Pre-install hooks (used by register.ts)
1500
+
1501
+ // Telemetry
1502
+ Senzor.flush(): Promise<void> // Force flush all queued telemetry
1503
+ Senzor.track(data: object) // Send a manual trace
1504
+ Senzor.startSpan(name: string, type?: SpanType) // Start a manual child span
1505
+ Senzor.captureException(error: unknown, ctx?: object) // Capture an error
1506
+
1507
+ // Task Monitoring
1508
+ Senzor.wrapTask(name, type, options, fn) // Wrap function as monitored task
1509
+ Senzor.startTask(name, type, options, fn) // Start task monitoring context
1510
+
1511
+ // Framework Wrappers
1512
+ Senzor.requestHandler() // Express request middleware
1513
+ Senzor.errorHandler() // Express error middleware
1514
+ Senzor.fastifyPlugin // Fastify plugin
1515
+ Senzor.wrapNextRoute(handler) // Next.js App Router wrapper
1516
+ Senzor.wrapNextPages(handler) // Next.js Pages Router wrapper
1517
+ Senzor.wrapH3(handler) // H3/Nuxt/Nitro event handler wrapper
1518
+ Senzor.nitroPlugin // Nitro plugin for Cloudflare Workers
1519
+ Senzor.worker(handler) // Cloudflare Workers fetch handler wrapper
1520
+ Senzor.wrapLambda(handler) // AWS Lambda handler wrapper
1521
+ ```
1522
+
1523
+ ---
1524
+
1525
+ ## 26. Build & Publish
1526
+
1527
+ ```sh
1528
+ npm run build
1529
+ ```
1530
+
1531
+ Output:
1532
+
1533
+ | File | Format | Description |
1534
+ |------|--------|-------------|
1535
+ | `dist/index.js` | CommonJS | Main entry |
1536
+ | `dist/index.mjs` | ESM | Main entry |
1537
+ | `dist/index.global.js` | IIFE | Browser/CDN bundle |
1538
+ | `dist/register.js` | CommonJS | Preload entry |
1539
+ | `dist/register.mjs` | ESM | Preload entry |
1540
+ | `dist/index.d.ts` | TypeScript | Type declarations |
1541
+ | `dist/register.d.ts` | TypeScript | Type declarations |
1542
+
1543
+ Verify before publishing:
1544
+
1545
+ ```sh
1546
+ npx tsc --noEmit && npm run build
1547
+ ```