@data-fair/lib-node 2.17.1 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +77 -9
- package/events-queue.js +4 -4
- package/events-queue.js.map +1 -1
- package/http-agents.js +2 -1
- package/http-agents.js.map +1 -1
- package/locks.js +3 -3
- package/locks.js.map +1 -1
- package/log-err.d.ts +11 -0
- package/log-err.js +41 -0
- package/log-err.js.map +1 -0
- package/log-err.test.d.ts +1 -0
- package/log-err.test.js +64 -0
- package/log-err.test.js.map +1 -0
- package/log-worker.test.d.ts +1 -0
- package/log-worker.test.js +16 -0
- package/log-worker.test.js.map +1 -0
- package/log.d.ts +34 -0
- package/log.js +119 -0
- package/log.js.map +1 -0
- package/log.test.d.ts +1 -0
- package/log.test.js +158 -0
- package/log.test.js.map +1 -0
- package/mongo.js +5 -4
- package/mongo.js.map +1 -1
- package/observer.d.ts +0 -1
- package/observer.js +6 -25
- package/observer.js.map +1 -1
- package/package.json +9 -2
- package/session.js +2 -1
- package/session.js.map +1 -1
- package/ssrf.test.js +4 -2
- package/ssrf.test.js.map +1 -1
- package/test-fixtures/early-debug.d.ts +2 -0
- package/test-fixtures/early-debug.js +4 -0
- package/test-fixtures/early-debug.js.map +1 -0
- package/test-fixtures/log-worker.d.ts +1 -0
- package/test-fixtures/log-worker.js +6 -0
- package/test-fixtures/log-worker.js.map +1 -0
- package/test-spies.js +1 -1
- package/test-spies.js.map +1 -1
- package/upgrade-scripts.js +2 -1
- package/upgrade-scripts.js.map +1 -1
package/README.md
CHANGED
|
@@ -12,7 +12,8 @@ Peer dependencies, all declared optional: install the ones required by the modul
|
|
|
12
12
|
|
|
13
13
|
| Peer dependency | Needed by |
|
|
14
14
|
| --- | --- |
|
|
15
|
-
| `prom-client@15` | `observer.js`, `http-agents.js` and everything built on them (`axios.js`, `events-queue.js`…) |
|
|
15
|
+
| `prom-client@15` | `observer.js`, `http-agents.js` and everything built on them (`axios.js`, `events-queue.js`…); without it `log.js` does not count log lines |
|
|
16
|
+
| `pino-pretty@13` | pretty output of `log.js` in dev (falls back to JSON without it) |
|
|
16
17
|
| `mongodb@6` | `mongo.js`, `locks.js`, `upgrade-scripts.js` |
|
|
17
18
|
| `ws@8` | `ws-client.js` |
|
|
18
19
|
| `tough-cookie@5` | `axios-with-cookies.js`, `axios-auth.js` |
|
|
@@ -28,7 +29,8 @@ There is no root entry point: import each module by its path.
|
|
|
28
29
|
| `@data-fair/lib-node/ssrf.js` | SSRF checks for clients that don't use the agents above: `resolvePublicAddress()`, `publicLookup`, `SsrfError`. |
|
|
29
30
|
| `@data-fair/lib-node/axios-with-cookies.js` | Axios instance with a `tough-cookie` cookie jar. |
|
|
30
31
|
| `@data-fair/lib-node/axios-auth.js` | Axios instance logged into simple-directory with email/password. Integration tests only. |
|
|
31
|
-
| `@data-fair/lib-node/
|
|
32
|
+
| `@data-fair/lib-node/log.js` | The logger of the stack: JSON lines on stdout (pretty in dev), implicit context, metrics, `debug` output redirected. |
|
|
33
|
+
| `@data-fair/lib-node/observer.js` | Mini web server exposing Prometheus metrics and CPU/heap profiles. |
|
|
32
34
|
| `@data-fair/lib-node/mongo.js` | `Mongo` connection wrapper (default export: a singleton) with declarative named index management (`configure()`). |
|
|
33
35
|
| `@data-fair/lib-node/locks.js` | Distributed locks stored in a MongoDB `locks` collection with a TTL (default export: a singleton). |
|
|
34
36
|
| `@data-fair/lib-node/upgrade-scripts.js` | Runner for the database migration scripts of a service. |
|
|
@@ -64,6 +66,78 @@ For clients that don't use these agents (ftp, sftp, raw sockets, SDKs…), `@dat
|
|
|
64
66
|
|
|
65
67
|
See the [`data-fair-security` skill](https://github.com/data-fair/lib/tree/main/skills/data-fair-security) for the stack-wide rules.
|
|
66
68
|
|
|
69
|
+
## Logging
|
|
70
|
+
|
|
71
|
+
One logger for everything, imported where needed, never passed around:
|
|
72
|
+
|
|
73
|
+
```ts
|
|
74
|
+
import log, { withLogContext } from '@data-fair/lib-node/log.js'
|
|
75
|
+
|
|
76
|
+
log.info('dataset finalized', { datasetId })
|
|
77
|
+
log.info('memory deleted', { code: 'agents.memory.delete' })
|
|
78
|
+
log.warn('forbidden request', { code: 'forbidden-req', err })
|
|
79
|
+
log.alert('admin api key created', { code: 'df.apikeys.createadmin' })
|
|
80
|
+
log.error('failed to create mongo index', { code: 'mongo-index-fail', err, collection })
|
|
81
|
+
|
|
82
|
+
log.init({ service: 'data-fair' }) // optional, defaults to OTEL_SERVICE_NAME
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
Every method has the same signature `(msg, fields?)` and never throws. Do not use the keys written by pino (`level`, `time`, `msg`, `hostname`) as field names, they would be duplicated in the JSON line.
|
|
86
|
+
|
|
87
|
+
| Level | Use | Emitted | Counted |
|
|
88
|
+
| --- | --- | --- | --- |
|
|
89
|
+
| `debug` | through the `debug` module (see below) | only namespaces enabled by `DEBUG` | no |
|
|
90
|
+
| `info` | normal operations, audit trail | always | no |
|
|
91
|
+
| `warn` | client errors, suspicious but expected situations | always | yes |
|
|
92
|
+
| `alert` | events that are not failures but deserve the same scrutiny as errors (mostly security) | always | yes |
|
|
93
|
+
| `error` | internal failures only, never client errors | always | yes |
|
|
94
|
+
|
|
95
|
+
**Output**: one JSON object per line on stdout when `NODE_ENV=production`, pretty output otherwise (`LOG_FORMAT=json|pretty` overrides). Fields: `time`, `level` (label), `msg`, `hostname`, `service`, `code`, `err`, the context fields, the per-call fields, and `trace_id` / `span_id` / `trace_flags` when OpenTelemetry is active.
|
|
96
|
+
|
|
97
|
+
**Codes**: `code` is optional on every line. It must be a **static string** (`mongo-index-fail`, `df.apikeys.createadmin`): never put ids, hosts or other dynamic values in it, they go in other fields. It labels the counters.
|
|
98
|
+
|
|
99
|
+
**Errors**: pass them as the `err` field. They are serialized with a whitelist (`type`, `message`, `stack`, primitive `code`, `status`, `cause`, and `method` / `url` without query string for axios errors), so headers, request config and document content carried by errors never reach the logs. Messages are truncated to 1000 characters (the message of an `AxiosRequestError` starts with the response body).
|
|
100
|
+
|
|
101
|
+
**Metrics**: `warn`, `alert` and `error` lines increment `df_logs{level, code, host}`. Deprecated, kept during the transition: `df_internal_error{errorCode}` (error lines) and `df_events_log{level, code, host}` (coded warn and alert lines).
|
|
102
|
+
|
|
103
|
+
**Debug**: keep using the `debug` module (`import Debug from 'debug'`), `DEBUG=ns1,ns2*` works as before. When `log.js` is loaded, the output of the shared `debug` module is redirected into the logger as `debug` lines with a `ns` field. A third-party library bundling its own copy of `debug` is not redirected.
|
|
104
|
+
|
|
105
|
+
### Implicit context
|
|
106
|
+
|
|
107
|
+
The fields of the current context are added to every line. `session.middleware()` of `@data-fair/lib-express` opens a context per request with `ip`, `host`, `userId`, `userName`, `accountKey`, `accountDep`, `accountName`. Any code can add fields for a scope:
|
|
108
|
+
|
|
109
|
+
```ts
|
|
110
|
+
await withLogContext({ datasetId }, async () => {
|
|
111
|
+
// every line logged in here, even deep in the call stack, carries datasetId
|
|
112
|
+
})
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
Rules:
|
|
116
|
+
|
|
117
|
+
- only small immutable values (ids, keys, short names), never `req`, documents or buffers: they stay in memory as long as the scope is alive,
|
|
118
|
+
- create long-lived resources (pools, intervals, caches, queues) at startup, outside any scope, or they capture it and attribute their later lines to the wrong user,
|
|
119
|
+
- event listeners run in the context of the code that emits, not of the code that subscribed.
|
|
120
|
+
|
|
121
|
+
Worker threads have their own context: pass `getLogContext()` in the task payload and wrap the task with `withLogContext(payload.logContext, ...)` in the worker.
|
|
122
|
+
|
|
123
|
+
### OpenTelemetry
|
|
124
|
+
|
|
125
|
+
Nothing to do in the code. Activate the OpenTelemetry Operator auto-instrumentation on the Kubernetes side: its pino instrumentation adds `trace_id`, `span_id` and `trace_flags` to the lines, and the `traceparent` header is propagated on outgoing HTTP calls (third parties included). Set `OTEL_LOGS_EXPORTER=none` so that logs are not also shipped over OTLP. Without the operator (dev, tests, scripts) lines have no trace fields.
|
|
126
|
+
|
|
127
|
+
### Migration
|
|
128
|
+
|
|
129
|
+
| Before | After |
|
|
130
|
+
| --- | --- |
|
|
131
|
+
| `eventsLog.info(code, msg, ctx)` | `log.info(msg, { code })` |
|
|
132
|
+
| `eventsLog.warn(code, msg, ctx)` | `log.warn(msg, { code })` |
|
|
133
|
+
| `eventsLog.alert(code, msg, ctx)` | `log.alert(msg, { code })` |
|
|
134
|
+
| `internalError(code, err)` | `log.error(msg, { code, err })` |
|
|
135
|
+
| `console.log` / `console.warn` / `console.error` | `log.info` / `log.warn` / `log.error` with a code |
|
|
136
|
+
| `EVENTS_LOG_LEVEL` | removed, every level but debug is emitted |
|
|
137
|
+
| `import Debug from 'debug'` | unchanged |
|
|
138
|
+
|
|
139
|
+
User, account, ip and host of `eventsLog`'s context argument now come from the implicit context.
|
|
140
|
+
|
|
67
141
|
## Observer
|
|
68
142
|
|
|
69
143
|
A very light web server for observability, on port 9090 by default:
|
|
@@ -74,19 +148,13 @@ A very light web server for observability, on port 9090 by default:
|
|
|
74
148
|
- `GET /heap-snapshot`: full retained heap snapshot (blocks the event loop while V8 walks the heap).
|
|
75
149
|
|
|
76
150
|
```ts
|
|
77
|
-
import { startObserver, stopObserver
|
|
151
|
+
import { startObserver, stopObserver } from '@data-fair/lib-node/observer.js'
|
|
78
152
|
|
|
79
153
|
await startObserver() // or startObserver(port)
|
|
80
154
|
// ...
|
|
81
155
|
await stopObserver()
|
|
82
156
|
```
|
|
83
157
|
|
|
84
|
-
`internalError(errorCode, error, ...optionalParams)` increments the shared `df_internal_error` counter and logs the error. Use it for anomalies that should trigger alerts, not for client errors:
|
|
85
|
-
|
|
86
|
-
```ts
|
|
87
|
-
internalError('http', 'failure while serving http request', err)
|
|
88
|
-
```
|
|
89
|
-
|
|
90
158
|
Process metrics are plain `prom-client` metrics. A service-level metric (its value depends on a shared state, like the database, not on the activity of the current process) is registered in `servicePromRegistry`:
|
|
91
159
|
|
|
92
160
|
```ts
|
package/events-queue.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import log from './log.js';
|
|
2
2
|
// the events service is part of our own infrastructure (configuration, not user input)
|
|
3
3
|
import { privateAxiosInstance as axios } from './axios.js';
|
|
4
4
|
import Debug from 'debug';
|
|
@@ -20,7 +20,7 @@ export class EventsQueue {
|
|
|
20
20
|
// events is an optional service: without its url it is not deployed and events are dropped
|
|
21
21
|
let inactive = !!options.inactive || !options.eventsUrl;
|
|
22
22
|
if (!inactive && !options.eventsSecret) {
|
|
23
|
-
|
|
23
|
+
log.error('the events service url is set but its secret is missing, events are dropped', { code: 'events-queue' });
|
|
24
24
|
inactive = true;
|
|
25
25
|
}
|
|
26
26
|
this._options = { ...options, inactive };
|
|
@@ -50,7 +50,7 @@ export class EventsQueue {
|
|
|
50
50
|
await axios.post(this.options().eventsUrl + '/api/events', events, { headers: { 'x-secret-key': this.options().eventsSecret } });
|
|
51
51
|
}
|
|
52
52
|
catch (err) {
|
|
53
|
-
|
|
53
|
+
log.error('failure to push events', { code: 'events-queue-push-event', err });
|
|
54
54
|
// retry later
|
|
55
55
|
if (err.status >= 500 && events.length < 1000)
|
|
56
56
|
this.eventsQueue = events.concat(this.eventsQueue);
|
|
@@ -65,7 +65,7 @@ export class EventsQueue {
|
|
|
65
65
|
await axios.post(this.options().eventsUrl + '/api/notifications', notification, { headers: { 'x-secret-key': this.options().eventsSecret } });
|
|
66
66
|
}
|
|
67
67
|
catch (err) {
|
|
68
|
-
|
|
68
|
+
log.error('failure to push a notification', { code: 'events-queue-push-notif', err });
|
|
69
69
|
// retry later
|
|
70
70
|
if (err.status >= 500 && (notifications.length + this.notificationsQueue.length) < 1000) {
|
|
71
71
|
this.notificationsQueue.unshift(notification);
|
package/events-queue.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"events-queue.js","sourceRoot":"","sources":["events-queue.ts"],"names":[],"mappings":"AAEA,OAAO,
|
|
1
|
+
{"version":3,"file":"events-queue.js","sourceRoot":"","sources":["events-queue.ts"],"names":[],"mappings":"AAEA,OAAO,GAAG,MAAM,UAAU,CAAA;AAC1B,uFAAuF;AACvF,OAAO,EAAE,oBAAoB,IAAI,KAAK,EAAE,MAAM,YAAY,CAAA;AAC1D,OAAO,KAAK,MAAM,OAAO,CAAA;AAGzB,MAAM,KAAK,GAAG,KAAK,CAAC,cAAc,CAAC,CAAA;AAWnC,MAAM,OAAO,WAAW;IACtB,QAAQ,CAAgC;IACxC,WAAW,GAAY,EAAE,CAAA;IACzB,kBAAkB,GAAmB,EAAE,CAAA;IACvC,OAAO,GAAG,KAAK,CAAA;IACf,mBAAmB,CAA2B;IAE9C,OAAO;QACL,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAA;QACvE,OAAO,IAAI,CAAC,QAAQ,CAAA;IACtB,CAAC;IAED,KAAK,GAAG,KAAK,EAAE,OAA2B,EAAE,EAAE;QAC5C,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAM,CAAC,sBAAsB;QAChD,2FAA2F;QAC3F,IAAI,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,OAAO,CAAC,SAAS,CAAA;QACvD,IAAI,CAAC,QAAQ,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;YACvC,GAAG,CAAC,KAAK,CAAC,6EAA6E,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,CAAA;YAClH,QAAQ,GAAG,IAAI,CAAA;QACjB,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,EAAE,GAAG,OAAO,EAAE,QAAQ,EAAE,CAAA;QACxC,IAAI,CAAC,QAAQ;YAAE,IAAI,CAAC,IAAI,EAAE,CAAA;IAC5B,CAAC,CAAA;IAED,IAAI,GAAG,KAAK,IAAI,EAAE;QAChB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAA;QACnB,IAAI,IAAI,CAAC,mBAAmB;YAAE,MAAM,IAAI,CAAC,mBAAmB,CAAA;QAC5D,MAAM,IAAI,CAAC,KAAK,EAAE,CAAA;IACpB,CAAC,CAAA;IAED,KAAK,CAAC,IAAI;QACR,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACrB,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAA;YACvD,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,KAAK,EAAE,CAAA;YACvC,MAAM,IAAI,CAAC,mBAAmB,CAAA;YAC9B,IAAI,CAAC,mBAAmB,GAAG,SAAS,CAAA;QACtC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAA;YAC/B,IAAI,CAAC,WAAW,GAAG,EAAE,CAAA;YACrB,KAAK,CAAC,cAAc,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;YACpC,IAAI,CAAC;gBACH,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,SAAS,GAAG,aAAa,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,EAAE,cAAc,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,YAAY,EAAE,EAAE,CAAC,CAAA;YAClI,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAClB,GAAG,CAAC,KAAK,CAAC,wBAAwB,EAAE,EAAE,IAAI,EAAE,yBAAyB,EAAE,GAAG,EAAE,CAAC,CAAA;gBAC7E,cAAc;gBACd,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,IAAI,MAAM,CAAC,MAAM,GAAG,IAAI;oBAAE,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;YACnG,CAAC;QACH,CAAC;QACD,IAAI,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC;YACnC,MAAM,aAAa,GAAG,IAAI,CAAC,kBAAkB,CAAA;YAC7C,IAAI,CAAC,kBAAkB,GAAG,EAAE,CAAA;YAC5B,KAAK,CAAC,qBAAqB,EAAE,aAAa,CAAC,MAAM,CAAC,CAAA;YAClD,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;gBACzC,IAAI,CAAC;oBACH,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,SAAS,GAAG,oBAAoB,EAAE,YAAY,EAAE,EAAE,OAAO,EAAE,EAAE,cAAc,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,YAAY,EAAE,EAAE,CAAC,CAAA;gBAC/I,CAAC;gBAAC,OAAO,GAAQ,EAAE,CAAC;oBAClB,GAAG,CAAC,KAAK,CAAC,gCAAgC,EAAE,EAAE,IAAI,EAAE,yBAAyB,EAAE,GAAG,EAAE,CAAC,CAAA;oBACrF,cAAc;oBACd,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC;wBACxF,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,YAAY,CAAC,CAAA;oBAC/C,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,SAAS,CAAE,KAAgB,EAAE,YAA2B;QACtD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAA;QAC9B,IAAI,OAAO,CAAC,QAAQ;YAAE,OAAM;QAC5B,IAAI,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;QAClE,KAAe,CAAC,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;QAChD,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,CAAA;QACzB,IAAI,CAAC,KAAK,CAAC,UAAU,IAAI,YAAY,EAAE,IAAI,IAAI,YAAY,CAAC,OAAO,EAAE,CAAC;YACpE,IAAI,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBAC9B,KAAK,CAAC,UAAU,GAAG;oBACjB,IAAI,EAAE,EAAE,GAAG,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE;iBACpD,CAAA;YACH,CAAC;iBAAM,IAAI,YAAY,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,KAAK,KAAK,CAAC,MAAM,EAAE,IAAI,IAAI,YAAY,CAAC,OAAO,CAAC,EAAE,KAAK,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC;gBAC7I,KAAK,CAAC,UAAU,GAAG;oBACjB,IAAI,EAAE,EAAE,EAAE,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE;iBAC9E,CAAA;YACH,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,UAAU,GAAG;oBACjB,IAAI,EAAE,EAAE,EAAE,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE;iBACjE,CAAA;gBACD,IAAI,YAAY,CAAC,YAAY,EAAE,CAAC;oBAC9B,KAAK,CAAC,UAAU,CAAC,YAAY,GAAG;wBAC9B,EAAE,EAAE,YAAY,CAAC,YAAY,CAAC,EAAE;wBAChC,IAAI,EAAE,YAAY,CAAC,YAAY,CAAC,IAAI;wBACpC,UAAU,EAAE,YAAY,CAAC,YAAY,CAAC,UAAU;wBAChD,cAAc,EAAE,YAAY,CAAC,YAAY,CAAC,cAAc;qBACzD,CAAA;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QACD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,KAAc,CAAC,CAAA;IACvC,CAAC;IAED,gBAAgB,CAAE,YAA8B;QAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAA;QAC9B,IAAI,OAAO,CAAC,QAAQ;YAAE,OAAM;QAC5B,IAAI,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QACzE,YAA6B,CAAC,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;QAC9D,KAAK,CAAC,kBAAkB,EAAE,YAAY,CAAC,CAAA;QACvC,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,YAA4B,CAAC,CAAA;IAC5D,CAAC;CACF;AAED,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAA;AAErC,eAAe,WAAW,CAAA","sourcesContent":["import { type Event } from '@data-fair/lib-common-types/event/index.js'\nimport { type Notification } from '@data-fair/lib-common-types/notification/index.js'\nimport log from './log.js'\n// the events service is part of our own infrastructure (configuration, not user input)\nimport { privateAxiosInstance as axios } from './axios.js'\nimport Debug from 'debug'\nimport { SessionState } from '@data-fair/lib-common-types/session/index.js'\n\nconst debug = Debug('events-queue')\n\nexport type EventsQueueOptions = {\n eventsUrl?: string | null,\n eventsSecret?: string | null,\n inactive?: boolean\n}\n\nexport type PushEvent = Omit<Event, 'date'>\nexport type PushNotification = Omit<Notification, 'date'>\n\nexport class EventsQueue {\n _options: EventsQueueOptions | undefined\n eventsQueue: Event[] = []\n notificationsQueue: Notification[] = []\n stopped = false\n currentDrainPromise: Promise<void> | undefined\n\n options () {\n if (!this._options) throw new Error('events queue was not initialized')\n return this._options\n }\n\n start = async (options: EventsQueueOptions) => {\n if (this._options) return // already initialized\n // events is an optional service: without its url it is not deployed and events are dropped\n let inactive = !!options.inactive || !options.eventsUrl\n if (!inactive && !options.eventsSecret) {\n log.error('the events service url is set but its secret is missing, events are dropped', { code: 'events-queue' })\n inactive = true\n }\n this._options = { ...options, inactive }\n if (!inactive) this.loop()\n }\n\n stop = async () => {\n this.stopped = true\n if (this.currentDrainPromise) await this.currentDrainPromise\n await this.drain()\n }\n\n async loop () {\n while (!this.stopped) {\n await new Promise(resolve => setTimeout(resolve, 3000))\n this.currentDrainPromise = this.drain()\n await this.currentDrainPromise\n this.currentDrainPromise = undefined\n }\n }\n\n async drain () {\n if (this.eventsQueue.length) {\n const events = this.eventsQueue\n this.eventsQueue = []\n debug('drain events', events.length)\n try {\n await axios.post(this.options().eventsUrl + '/api/events', events, { headers: { 'x-secret-key': this.options().eventsSecret } })\n } catch (err: any) {\n log.error('failure to push events', { code: 'events-queue-push-event', err })\n // retry later\n if (err.status >= 500 && events.length < 1000) this.eventsQueue = events.concat(this.eventsQueue)\n }\n }\n if (this.notificationsQueue.length) {\n const notifications = this.notificationsQueue\n this.notificationsQueue = []\n debug('drain notifications', notifications.length)\n for (const notification of notifications) {\n try {\n await axios.post(this.options().eventsUrl + '/api/notifications', notification, { headers: { 'x-secret-key': this.options().eventsSecret } })\n } catch (err: any) {\n log.error('failure to push a notification', { code: 'events-queue-push-notif', err })\n // retry later\n if (err.status >= 500 && (notifications.length + this.notificationsQueue.length) < 1000) {\n this.notificationsQueue.unshift(notification)\n }\n }\n }\n }\n }\n\n pushEvent (event: PushEvent, sessionState?: SessionState) {\n const options = this.options()\n if (options.inactive) return\n if (this.stopped) throw new Error('events queue has been stopped');\n (event as Event).date = new Date().toISOString()\n debug('pushEvent', event)\n if (!event.originator && sessionState?.user && sessionState.account) {\n if (sessionState.user.asAdmin) {\n event.originator = {\n user: { ...sessionState.user.asAdmin, admin: true }\n }\n } else if (sessionState.user.adminMode && (sessionState.account.type !== event.sender?.type || sessionState.account.id !== event.sender?.id)) {\n event.originator = {\n user: { id: sessionState.user.id, name: sessionState.user.name, admin: true }\n }\n } else {\n event.originator = {\n user: { id: sessionState.user.id, name: sessionState.user.name }\n }\n if (sessionState.organization) {\n event.originator.organization = {\n id: sessionState.organization.id,\n name: sessionState.organization.name,\n department: sessionState.organization.department,\n departmentName: sessionState.organization.departmentName\n }\n }\n }\n }\n this.eventsQueue.push(event as Event)\n }\n\n pushNotification (notification: PushNotification) {\n const options = this.options()\n if (options.inactive) return\n if (this.stopped) throw new Error('notifications queue has been stopped');\n (notification as Notification).date = new Date().toISOString()\n debug('pushNotification', notification)\n this.notificationsQueue.push(notification as Notification)\n }\n}\n\nconst eventsQueue = new EventsQueue()\n\nexport default eventsQueue\n"]}
|
package/http-agents.js
CHANGED
|
@@ -12,6 +12,7 @@ import CacheableLookup from 'cacheable-lookup';
|
|
|
12
12
|
import HttpAgent, { HttpsAgent } from 'agentkeepalive';
|
|
13
13
|
import { Counter, Gauge } from 'prom-client';
|
|
14
14
|
import { SsrfHttpAgent, SsrfHttpsAgent, getSsrfRules, getProxyEnvName } from './ssrf.js';
|
|
15
|
+
import log from './log.js';
|
|
15
16
|
// not very high number of sockets but we don't want to saturate our reverse proxies
|
|
16
17
|
// if a higher number is needed the service should probably be scaled anyway
|
|
17
18
|
const keepaliveOpts = {
|
|
@@ -25,7 +26,7 @@ export const privateHttpsAgent = new HttpsAgent(keepaliveOpts);
|
|
|
25
26
|
const createPublicAgents = () => {
|
|
26
27
|
const proxyEnvName = getProxyEnvName();
|
|
27
28
|
if (proxyEnvName) {
|
|
28
|
-
|
|
29
|
+
log.warn(`${proxyEnvName} is defined, the protection against server side request forgery (SSRF) is disabled, outgoing requests are not checked against non public addresses`);
|
|
29
30
|
return { httpAgent: new HttpAgent(keepaliveOpts), httpsAgent: new HttpsAgent(keepaliveOpts) };
|
|
30
31
|
}
|
|
31
32
|
const rules = getSsrfRules();
|
package/http-agents.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http-agents.js","sourceRoot":"","sources":["http-agents.ts"],"names":[],"mappings":"AAAA,wFAAwF;AACxF,yDAAyD;AACzD,EAAE;AACF,iGAAiG;AACjG,4DAA4D;AAC5D,6FAA6F;AAC7F,oFAAoF;AACpF,EAAE;AACF,uHAAuH;AACvH,0GAA0G;AAE1G,OAAO,eAAe,MAAM,kBAAkB,CAAA;AAC9C,OAAO,SAAS,EAAE,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAA;AACtD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAA;AAC5C,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,WAAW,CAAA;
|
|
1
|
+
{"version":3,"file":"http-agents.js","sourceRoot":"","sources":["http-agents.ts"],"names":[],"mappings":"AAAA,wFAAwF;AACxF,yDAAyD;AACzD,EAAE;AACF,iGAAiG;AACjG,4DAA4D;AAC5D,6FAA6F;AAC7F,oFAAoF;AACpF,EAAE;AACF,uHAAuH;AACvH,0GAA0G;AAE1G,OAAO,eAAe,MAAM,kBAAkB,CAAA;AAC9C,OAAO,SAAS,EAAE,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAA;AACtD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAA;AAC5C,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,WAAW,CAAA;AACxF,OAAO,GAAG,MAAM,UAAU,CAAA;AAE1B,oFAAoF;AACpF,4EAA4E;AAC5E,MAAM,aAAa,GAAG;IACpB,UAAU,EAAE,CAAC;IACb,cAAc,EAAE,CAAC;IACjB,OAAO,EAAE,KAAK;IACd,iBAAiB,EAAE,KAAK;CACzB,CAAA;AAED,MAAM,CAAC,MAAM,gBAAgB,GAAG,IAAI,SAAS,CAAC,aAAa,CAAC,CAAA;AAC5D,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,UAAU,CAAC,aAAa,CAAC,CAAA;AAE9D,MAAM,kBAAkB,GAAG,GAAG,EAAE;IAC9B,MAAM,YAAY,GAAG,eAAe,EAAE,CAAA;IACtC,IAAI,YAAY,EAAE,CAAC;QACjB,GAAG,CAAC,IAAI,CAAC,GAAG,YAAY,oJAAoJ,CAAC,CAAA;QAC7K,OAAO,EAAE,SAAS,EAAE,IAAI,SAAS,CAAC,aAAa,CAAC,EAAE,UAAU,EAAE,IAAI,UAAU,CAAC,aAAa,CAAC,EAAE,CAAA;IAC/F,CAAC;IACD,MAAM,KAAK,GAAG,YAAY,EAAE,CAAA;IAC5B,OAAO,EAAE,SAAS,EAAE,IAAI,aAAa,CAAC,aAAa,EAAE,KAAK,CAAC,EAAE,UAAU,EAAE,IAAI,cAAc,CAAC,aAAa,EAAE,KAAK,CAAC,EAAE,CAAA;AACrH,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,kBAAkB,EAAE,CAAA;AAE7D,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAA;AAC7C,KAAK,MAAM,KAAK,IAAI,CAAC,SAAS,EAAE,UAAU,EAAE,gBAAgB,EAAE,iBAAiB,CAAC,EAAE,CAAC;IACjF,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;AAChC,CAAC;AAED,yBAAyB;AACzB,MAAM,MAAM,GAAG;IACb,MAAM,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE;IAC9C,OAAO,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,iBAAiB,EAAE;CAC9D,CAAA;AAED,MAAM,YAAY,GAAG,IAAI,KAAK,CAAC;IAC7B,IAAI,EAAE,uBAAuB;IAC7B,IAAI,EAAE,0CAA0C;IAChD,UAAU,EAAE,CAAC,SAAS,EAAE,UAAU,EAAE,MAAM,CAAC;CAC5C,CAAC,CAAA;AACF,MAAM,gBAAgB,GAAG,IAAI,KAAK,CAAC;IACjC,IAAI,EAAE,4BAA4B;IAClC,IAAI,EAAE,0CAA0C;IAChD,UAAU,EAAE,CAAC,SAAS,EAAE,UAAU,EAAE,MAAM,CAAC;CAC5C,CAAC,CAAA;AACF,MAAM,mBAAmB,GAAG,IAAI,OAAO,CAAC;IACtC,IAAI,EAAE,mCAAmC;IACzC,IAAI,EAAE,mDAAmD;IACzD,UAAU,EAAE,CAAC,SAAS,EAAE,UAAU,CAAC;CACpC,CAAC,CAAA;AACF,MAAM,cAAc,GAAG,IAAI,OAAO,CAAC;IACjC,IAAI,EAAE,6BAA6B;IACnC,IAAI,EAAE,oDAAoD;IAC1D,UAAU,EAAE,CAAC,SAAS,EAAE,UAAU,CAAC;CACpC,CAAC,CAAA;AAEF,gFAAgF;AAChF,gFAAgF;AAChF,gDAAgD;AAChD,WAAW,CAAC,GAAG,EAAE;IACf,KAAK,MAAM,OAAO,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAU,EAAE,CAAC;QACrD,KAAK,MAAM,QAAQ,IAAI,CAAC,MAAM,EAAE,OAAO,CAAU,EAAE,CAAC;YAClD,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,gBAAgB,EAAE,CAAA;YAChE,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;gBACvC,YAAY,CAAC,GAAG,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAA;YAC1E,CAAC;YACD,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,WAAW,EAAE,CAAC;gBAC3C,gBAAgB,CAAC,GAAG,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,WAAW,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAA;YAClF,CAAC;YACD,mBAAmB,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAA;YACjD,mBAAmB,CAAC,GAAG,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,WAAW,CAAC,iBAAiB,CAAC,CAAA;YAC7E,cAAc,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAA;YAC5C,cAAc,CAAC,GAAG,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,WAAW,CAAC,YAAY,CAAC,CAAA;QACrE,CAAC;IACH,CAAC;AACH,CAAC,EAAE,KAAK,CAAC,CAAC,KAAK,EAAE,CAAA","sourcesContent":["// use better DNS lookup than nodejs default and try to reduce number of socket openings\n// and protect against server side request forgery (SSRF)\n//\n// public agents (httpAgent / httpsAgent, the default) refuse to connect to non public addresses,\n// they must be used for every URL that a user can influence\n// private agents (privateHttpAgent / privateHttpsAgent) must only be used for URLs that come\n// from the configuration of the service (other services of the same infrastructure)\n//\n// SSRF_PUBLIC_IPS: comma separated IPs / CIDR ranges to consider public (an intranet source of a self-hosted instance)\n// SSRF_PRIVATE_IPS: comma separated IPs / CIDR ranges to consider private (public IPs of databases, etc.)\n\nimport CacheableLookup from 'cacheable-lookup'\nimport HttpAgent, { HttpsAgent } from 'agentkeepalive'\nimport { Counter, Gauge } from 'prom-client'\nimport { SsrfHttpAgent, SsrfHttpsAgent, getSsrfRules, getProxyEnvName } from './ssrf.js'\nimport log from './log.js'\n\n// not very high number of sockets but we don't want to saturate our reverse proxies\n// if a higher number is needed the service should probably be scaled anyway\nconst keepaliveOpts = {\n maxSockets: 8,\n maxFreeSockets: 8,\n timeout: 60000,\n freeSocketTimeout: 30000\n}\n\nexport const privateHttpAgent = new HttpAgent(keepaliveOpts)\nexport const privateHttpsAgent = new HttpsAgent(keepaliveOpts)\n\nconst createPublicAgents = () => {\n const proxyEnvName = getProxyEnvName()\n if (proxyEnvName) {\n log.warn(`${proxyEnvName} is defined, the protection against server side request forgery (SSRF) is disabled, outgoing requests are not checked against non public addresses`)\n return { httpAgent: new HttpAgent(keepaliveOpts), httpsAgent: new HttpsAgent(keepaliveOpts) }\n }\n const rules = getSsrfRules()\n return { httpAgent: new SsrfHttpAgent(keepaliveOpts, rules), httpsAgent: new SsrfHttpsAgent(keepaliveOpts, rules) }\n}\n\nexport const { httpAgent, httpsAgent } = createPublicAgents()\n\nconst cacheableLookup = new CacheableLookup()\nfor (const agent of [httpAgent, httpsAgent, privateHttpAgent, privateHttpsAgent]) {\n cacheableLookup.install(agent)\n}\n\n// monitor agent statuses\nconst agents = {\n public: { http: httpAgent, https: httpsAgent },\n private: { http: privateHttpAgent, https: privateHttpsAgent }\n}\n\nconst socketsGauge = new Gauge({\n name: 'df_http_agent_sockets',\n help: 'Number of open sockets in the http agent',\n labelNames: ['network', 'protocol', 'host']\n})\nconst freeSocketsGauge = new Gauge({\n name: 'df_http_agent_free_sockets',\n help: 'Number of free sockets in the http agent',\n labelNames: ['network', 'protocol', 'host']\n})\nconst createSocketCounter = new Counter({\n name: 'df_http_agent_create_socket_total',\n help: 'Total number of sockets created by the http agent',\n labelNames: ['network', 'protocol']\n})\nconst requestCounter = new Counter({\n name: 'df_http_agent_request_total',\n help: 'Total number of requests created by the http agent',\n labelNames: ['network', 'protocol']\n})\n\n// unref'd so that importing this module never keeps a process alive on its own:\n// a server is kept running by its own handles and still gets the metrics, while\n// CLIs, tests and build tools can exit normally\nsetInterval(() => {\n for (const network of ['public', 'private'] as const) {\n for (const protocol of ['http', 'https'] as const) {\n const agentStatus = agents[network][protocol].getCurrentStatus()\n for (const host in agentStatus.sockets) {\n socketsGauge.set({ network, protocol, host }, agentStatus.sockets[host])\n }\n for (const host in agentStatus.freeSockets) {\n freeSocketsGauge.set({ network, protocol, host }, agentStatus.freeSockets[host])\n }\n createSocketCounter.remove({ network, protocol })\n createSocketCounter.inc({ network, protocol }, agentStatus.createSocketCount)\n requestCounter.remove({ network, protocol })\n requestCounter.inc({ network, protocol }, agentStatus.requestCount)\n }\n }\n}, 60000).unref()\n"]}
|
package/locks.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { hostname } from 'node:os';
|
|
2
2
|
import { randomBytes } from 'node:crypto';
|
|
3
3
|
import Debug from 'debug';
|
|
4
|
-
import
|
|
4
|
+
import log from './log.js';
|
|
5
5
|
const debug = Debug('locks');
|
|
6
6
|
export class Locks {
|
|
7
7
|
pid;
|
|
@@ -28,14 +28,14 @@ export class Locks {
|
|
|
28
28
|
await this.collection.createIndex({ updatedAt: 1 }, { expireAfterSeconds: ttl });
|
|
29
29
|
}
|
|
30
30
|
catch (err) {
|
|
31
|
-
|
|
31
|
+
log.info('Failure to create TTL index. Probably because the value changed. Try to update it.');
|
|
32
32
|
db.command({ collMod: 'locks', index: { keyPattern: { updatedAt: 1 }, expireAfterSeconds: ttl } });
|
|
33
33
|
}
|
|
34
34
|
// prolongate lock acquired by this process while it is still active
|
|
35
35
|
// a mongo error here must not become an unhandled rejection (it would crash the process on any failover)
|
|
36
36
|
this.interval = setInterval(() => {
|
|
37
37
|
this.collection.updateMany({ pid: this.pid }, { $currentDate: { updatedAt: true } })
|
|
38
|
-
.catch(err =>
|
|
38
|
+
.catch(err => log.error('failure to prolongate locks', { code: 'locks-heartbeat', err }));
|
|
39
39
|
}, (ttl / 2) * 1000);
|
|
40
40
|
};
|
|
41
41
|
stop = async () => {
|
package/locks.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"locks.js","sourceRoot":"","sources":["locks.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AAClC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACzC,OAAO,KAAK,MAAM,OAAO,CAAA;AACzB,OAAO,
|
|
1
|
+
{"version":3,"file":"locks.js","sourceRoot":"","sources":["locks.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AAClC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACzC,OAAO,KAAK,MAAM,OAAO,CAAA;AACzB,OAAO,GAAG,MAAM,UAAU,CAAA;AAE1B,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,CAAA;AAE5B,MAAM,OAAO,KAAK;IAChB,GAAG,CAAQ;IACH,QAAQ,CAA4C;IACpD,GAAG,CAAgB;IAC3B,IAAI,EAAE;QACJ,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAA;QACjE,OAAO,IAAI,CAAC,GAAG,CAAA;IACjB,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAmF,OAAO,CAAC,CAAA;IACtH,CAAC;IAED;QACE,IAAI,CAAC,GAAG,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;QACzC,KAAK,CAAC,gBAAgB,EAAE,IAAI,CAAC,GAAG,CAAC,CAAA;IACnC,CAAC;IAED,KAAK,GAAG,KAAK,EAAE,EAAM,EAAE,GAAG,GAAG,EAAE,EAAE,EAAE;QACjC,IAAI,IAAI,CAAC,GAAG;YAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAA;QACpE,IAAI,CAAC,GAAG,GAAG,EAAE,CAAA;QACb,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAA;QAC7C,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE,EAAE,kBAAkB,EAAE,GAAG,EAAE,CAAC,CAAA;QAClF,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,IAAI,CAAC,oFAAoF,CAAC,CAAA;YAC9F,EAAE,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE,kBAAkB,EAAE,GAAG,EAAE,EAAE,CAAC,CAAA;QACpG,CAAC;QAED,oEAAoE;QACpE,yGAAyG;QACzG,IAAI,CAAC,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE;YAC/B,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,EAAE,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,CAAC;iBACjF,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,6BAA6B,EAAE,EAAE,IAAI,EAAE,iBAAiB,EAAE,GAAG,EAAE,CAAC,CAAC,CAAA;QAC7F,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAA;IACtB,CAAC,CAAA;IAED,IAAI,GAAG,KAAK,IAAI,EAAE;QAChB,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAC5B,IAAI,IAAI,CAAC,GAAG;YAAE,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;IACnE,CAAC,CAAA;IAED,OAAO,GAAG,KAAK,EAAE,GAAW,EAAE,MAAe,EAAoB,EAAE;QACjE,KAAK,CAAC,SAAS,EAAE,GAAG,EAAE,MAAM,CAAC,CAAA;QAC7B,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,EAAE,CAAC,CAAA;YAC5G,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,CAAC,CAAA;YACjF,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;gBACvD,MAAM,GAAG,CAAA;YACX,CAAC;YACD,KAAK,CAAC,YAAY,EAAE,GAAG,CAAC,CAAA;YACxB,OAAO,IAAI,CAAA;QACb,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,IAAI,GAAG,CAAC,IAAI,KAAK,KAAK;gBAAE,MAAM,GAAG,CAAA;YACjC,gDAAgD;YAChD,KAAK,CAAC,YAAY,EAAE,GAAG,CAAC,CAAA;YACxB,OAAO,KAAK,CAAA;QACd,CAAC;IACH,CAAC,CAAA;IAED,OAAO,GAAG,KAAK,EAAE,GAAW,EAAE,KAAK,GAAG,CAAC,EAAE,EAAE;QACzC,KAAK,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;QACrB,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,CAAA;YACrD,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,CAAC,CAAA;QAC3H,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;QACzD,CAAC;IACH,CAAC,CAAA;CACF;AAED,MAAM,KAAK,GAAG,IAAI,KAAK,EAAE,CAAA;AAEzB,eAAe,KAAK,CAAA","sourcesContent":["import type { Db } from 'mongodb'\nimport { hostname } from 'node:os'\nimport { randomBytes } from 'node:crypto'\nimport Debug from 'debug'\nimport log from './log.js'\n\nconst debug = Debug('locks')\n\nexport class Locks {\n pid: string\n private interval: ReturnType<typeof setInterval> | undefined\n private _db: Db | undefined\n get db () {\n if (!this._db) throw new Error('locks utils was not initialized')\n return this._db\n }\n\n get collection () {\n return this.db.collection<{ _id: string, pid: string, origin?: string, hostname: string, createdAt: Date }>('locks')\n }\n\n constructor () {\n this.pid = randomBytes(8).toString('hex')\n debug('locks with pid', this.pid)\n }\n\n start = async (db: Db, ttl = 60) => {\n if (this._db) throw new Error('locks utils was already initialized')\n this._db = db\n await this.collection.createIndex({ pid: 1 })\n try {\n await this.collection.createIndex({ updatedAt: 1 }, { expireAfterSeconds: ttl })\n } catch (err) {\n log.info('Failure to create TTL index. Probably because the value changed. Try to update it.')\n db.command({ collMod: 'locks', index: { keyPattern: { updatedAt: 1 }, expireAfterSeconds: ttl } })\n }\n\n // prolongate lock acquired by this process while it is still active\n // a mongo error here must not become an unhandled rejection (it would crash the process on any failover)\n this.interval = setInterval(() => {\n this.collection.updateMany({ pid: this.pid }, { $currentDate: { updatedAt: true } })\n .catch(err => log.error('failure to prolongate locks', { code: 'locks-heartbeat', err }))\n }, (ttl / 2) * 1000)\n }\n\n stop = async () => {\n clearInterval(this.interval)\n if (this._db) await this.collection.deleteMany({ pid: this.pid })\n }\n\n acquire = async (_id: string, origin?: string): Promise<boolean> => {\n debug('acquire', _id, origin)\n try {\n await this.collection.insertOne({ _id, pid: this.pid, origin, hostname: hostname(), createdAt: new Date() })\n try {\n await this.collection.updateOne({ _id }, { $currentDate: { updatedAt: true } })\n } catch (err) {\n await this.collection.deleteOne({ _id, pid: this.pid })\n throw err\n }\n debug('acquire ok', _id)\n return true\n } catch (err: any) {\n if (err.code !== 11000) throw err\n // duplicate means the lock was already acquired\n debug('acquire ko', _id)\n return false\n }\n }\n\n release = async (_id: string, delay = 0) => {\n debug('release', _id)\n if (delay) {\n const date = new Date((new Date()).getTime() + delay)\n await this.collection.updateOne({ _id, pid: this.pid }, { $unset: { pid: 1 }, $set: { delayed: true, updatedAt: date } })\n } else {\n await this.collection.deleteOne({ _id, pid: this.pid })\n }\n }\n}\n\nconst locks = new Locks()\n\nexport default locks\n"]}
|
package/log-err.d.ts
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export interface SerializedErr {
|
|
2
|
+
type?: string;
|
|
3
|
+
message: string;
|
|
4
|
+
stack?: string;
|
|
5
|
+
code?: string | number;
|
|
6
|
+
status?: number;
|
|
7
|
+
method?: string;
|
|
8
|
+
url?: string;
|
|
9
|
+
cause?: SerializedErr;
|
|
10
|
+
}
|
|
11
|
+
export declare function serializeErr(value: unknown, depth?: number): SerializedErr;
|
package/log-err.js
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
// whitelist serializer for errors in logs
|
|
2
|
+
// pino's standard serializer copies all enumerable properties, which leaks secrets for us
|
|
3
|
+
// (axios config headers, request and response bodies, mongo document content, etc.)
|
|
4
|
+
const maxDepth = 5;
|
|
5
|
+
// messages can embed response bodies (AxiosRequestError) and the stack repeats the message
|
|
6
|
+
const maxMessageLength = 1000;
|
|
7
|
+
const truncate = (text) => text.length > maxMessageLength ? text.slice(0, maxMessageLength) + '…' : text;
|
|
8
|
+
const truncateStackHeader = (stack) => {
|
|
9
|
+
const framesIndex = stack.indexOf('\n at ');
|
|
10
|
+
if (framesIndex === -1)
|
|
11
|
+
return truncate(stack);
|
|
12
|
+
return truncate(stack.slice(0, framesIndex)) + stack.slice(framesIndex);
|
|
13
|
+
};
|
|
14
|
+
const isPrimitiveCode = (value) => typeof value === 'string' || typeof value === 'number';
|
|
15
|
+
export function serializeErr(value, depth = 0) {
|
|
16
|
+
if (value === null || typeof value !== 'object')
|
|
17
|
+
return { message: truncate(String(value)) };
|
|
18
|
+
const err = value;
|
|
19
|
+
const s = { message: truncate(typeof err.message === 'string' ? err.message : String(value)) };
|
|
20
|
+
if (value instanceof Error)
|
|
21
|
+
s.type = err.name && err.name !== 'Error' ? err.name : err.constructor?.name ?? 'Error';
|
|
22
|
+
if (typeof err.stack === 'string')
|
|
23
|
+
s.stack = truncateStackHeader(err.stack);
|
|
24
|
+
if (isPrimitiveCode(err.code))
|
|
25
|
+
s.code = err.code;
|
|
26
|
+
const status = err.status ?? err.statusCode;
|
|
27
|
+
if (typeof status === 'number')
|
|
28
|
+
s.status = status;
|
|
29
|
+
if (err.isAxiosError === true || err.name === 'AxiosRequestError') {
|
|
30
|
+
if (typeof err.config?.method === 'string')
|
|
31
|
+
s.method = err.config.method;
|
|
32
|
+
if (typeof err.config?.url === 'string')
|
|
33
|
+
s.url = err.config.url.split('?')[0];
|
|
34
|
+
}
|
|
35
|
+
// the depth limit also stops circular causes
|
|
36
|
+
if (err.cause !== undefined && depth < maxDepth) {
|
|
37
|
+
s.cause = serializeErr(err.cause, depth + 1);
|
|
38
|
+
}
|
|
39
|
+
return s;
|
|
40
|
+
}
|
|
41
|
+
//# sourceMappingURL=log-err.js.map
|
package/log-err.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"log-err.js","sourceRoot":"","sources":["log-err.ts"],"names":[],"mappings":"AAAA,0CAA0C;AAC1C,0FAA0F;AAC1F,oFAAoF;AAapF,MAAM,QAAQ,GAAG,CAAC,CAAA;AAClB,2FAA2F;AAC3F,MAAM,gBAAgB,GAAG,IAAI,CAAA;AAE7B,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAA;AAEhH,MAAM,mBAAmB,GAAG,CAAC,KAAa,EAAE,EAAE;IAC5C,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAA;IAC9C,IAAI,WAAW,KAAK,CAAC,CAAC;QAAE,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAA;IAC9C,OAAO,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA;AACzE,CAAC,CAAA;AAED,MAAM,eAAe,GAAG,CAAC,KAAc,EAA4B,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,QAAQ,CAAA;AAE5H,MAAM,UAAU,YAAY,CAAE,KAAc,EAAE,KAAK,GAAG,CAAC;IACrD,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAA;IAC5F,MAAM,GAAG,GAAG,KAA4B,CAAA;IACxC,MAAM,CAAC,GAAkB,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAA;IAC7G,IAAI,KAAK,YAAY,KAAK;QAAE,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,IAAI,OAAO,CAAA;IACnH,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ;QAAE,CAAC,CAAC,KAAK,GAAG,mBAAmB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;IAC3E,IAAI,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC,IAAI,CAAA;IAChD,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,CAAA;IAC3C,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,CAAC,CAAC,MAAM,GAAG,MAAM,CAAA;IACjD,IAAI,GAAG,CAAC,YAAY,KAAK,IAAI,IAAI,GAAG,CAAC,IAAI,KAAK,mBAAmB,EAAE,CAAC;QAClE,IAAI,OAAO,GAAG,CAAC,MAAM,EAAE,MAAM,KAAK,QAAQ;YAAE,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAA;QACxE,IAAI,OAAO,GAAG,CAAC,MAAM,EAAE,GAAG,KAAK,QAAQ;YAAE,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;IAC/E,CAAC;IACD,6CAA6C;IAC7C,IAAI,GAAG,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,GAAG,QAAQ,EAAE,CAAC;QAChD,CAAC,CAAC,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAA;IAC9C,CAAC;IACD,OAAO,CAAC,CAAA;AACV,CAAC","sourcesContent":["// whitelist serializer for errors in logs\n// pino's standard serializer copies all enumerable properties, which leaks secrets for us\n// (axios config headers, request and response bodies, mongo document content, etc.)\n\nexport interface SerializedErr {\n type?: string\n message: string\n stack?: string\n code?: string | number\n status?: number\n method?: string\n url?: string\n cause?: SerializedErr\n}\n\nconst maxDepth = 5\n// messages can embed response bodies (AxiosRequestError) and the stack repeats the message\nconst maxMessageLength = 1000\n\nconst truncate = (text: string) => text.length > maxMessageLength ? text.slice(0, maxMessageLength) + '…' : text\n\nconst truncateStackHeader = (stack: string) => {\n const framesIndex = stack.indexOf('\\n at ')\n if (framesIndex === -1) return truncate(stack)\n return truncate(stack.slice(0, framesIndex)) + stack.slice(framesIndex)\n}\n\nconst isPrimitiveCode = (value: unknown): value is string | number => typeof value === 'string' || typeof value === 'number'\n\nexport function serializeErr (value: unknown, depth = 0): SerializedErr {\n if (value === null || typeof value !== 'object') return { message: truncate(String(value)) }\n const err = value as Record<string, any>\n const s: SerializedErr = { message: truncate(typeof err.message === 'string' ? err.message : String(value)) }\n if (value instanceof Error) s.type = err.name && err.name !== 'Error' ? err.name : err.constructor?.name ?? 'Error'\n if (typeof err.stack === 'string') s.stack = truncateStackHeader(err.stack)\n if (isPrimitiveCode(err.code)) s.code = err.code\n const status = err.status ?? err.statusCode\n if (typeof status === 'number') s.status = status\n if (err.isAxiosError === true || err.name === 'AxiosRequestError') {\n if (typeof err.config?.method === 'string') s.method = err.config.method\n if (typeof err.config?.url === 'string') s.url = err.config.url.split('?')[0]\n }\n // the depth limit also stops circular causes\n if (err.cause !== undefined && depth < maxDepth) {\n s.cause = serializeErr(err.cause, depth + 1)\n }\n return s\n}\n"]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/log-err.test.js
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { describe, it } from 'node:test';
|
|
2
|
+
import { strict as assert } from 'assert';
|
|
3
|
+
import { serializeErr } from './log-err.js';
|
|
4
|
+
describe('serializeErr', () => {
|
|
5
|
+
it('keeps only whitelisted fields of an Error', () => {
|
|
6
|
+
const err = Object.assign(new Error('boom'), { code: 'E1', status: 500, keyValue: { email: 'a@b.c' }, secret: 'x' });
|
|
7
|
+
const s = serializeErr(err);
|
|
8
|
+
assert.deepEqual(Object.keys(s).sort(), ['code', 'message', 'stack', 'status', 'type']);
|
|
9
|
+
assert.equal(s.type, 'Error');
|
|
10
|
+
assert.equal(s.message, 'boom');
|
|
11
|
+
});
|
|
12
|
+
it('uses the error name as type', () => {
|
|
13
|
+
class MongoServerError extends Error {
|
|
14
|
+
constructor(m) { super(m); this.name = 'MongoServerError'; }
|
|
15
|
+
}
|
|
16
|
+
assert.equal(serializeErr(new MongoServerError('dup')).type, 'MongoServerError');
|
|
17
|
+
});
|
|
18
|
+
it('maps statusCode to status', () => {
|
|
19
|
+
assert.equal(serializeErr(Object.assign(new Error('x'), { statusCode: 404 })).status, 404);
|
|
20
|
+
});
|
|
21
|
+
it('drops non primitive code', () => {
|
|
22
|
+
assert.equal(serializeErr(Object.assign(new Error('x'), { code: { a: 1 } })).code, undefined);
|
|
23
|
+
});
|
|
24
|
+
it('never leaks axios request headers and strips the query string', () => {
|
|
25
|
+
const err = Object.assign(new Error('timeout'), { isAxiosError: true, config: { method: 'get', url: 'https://x.org/p?apiKey=s3cret', headers: { Authorization: 'Bearer s3cret' } } });
|
|
26
|
+
const json = JSON.stringify(serializeErr(err));
|
|
27
|
+
assert.ok(!json.includes('s3cret'));
|
|
28
|
+
assert.ok(json.includes('"url":"https://x.org/p"'));
|
|
29
|
+
assert.ok(json.includes('"method":"get"'));
|
|
30
|
+
});
|
|
31
|
+
it('handles AxiosRequestError shape without bodies', () => {
|
|
32
|
+
const err = Object.assign(new Error('500 - fail'), { name: 'AxiosRequestError', status: 500, config: { method: 'post', url: 'http://a/b?k=v', data: 'body' }, data: { secret: 1 }, headers: { 'set-cookie': 'x' } });
|
|
33
|
+
const s = serializeErr(err);
|
|
34
|
+
assert.equal(s.url, 'http://a/b');
|
|
35
|
+
assert.equal(s.status, 500);
|
|
36
|
+
assert.ok(!JSON.stringify(s).includes('secret'));
|
|
37
|
+
assert.ok(!JSON.stringify(s).includes('body'));
|
|
38
|
+
});
|
|
39
|
+
it('truncates long messages and the message line of stacks', () => {
|
|
40
|
+
const body = JSON.stringify({ echoed: 'Bearer SECRET', doc: 'x'.repeat(5000) });
|
|
41
|
+
const err = Object.assign(new Error(`500 - ${body}`), { name: 'AxiosRequestError', status: 500 });
|
|
42
|
+
const s = serializeErr(err);
|
|
43
|
+
assert.ok(s.message.length <= 1001, `message length ${s.message.length}`);
|
|
44
|
+
assert.ok((s.stack ?? '').length < 2000, `stack length ${s.stack?.length}`);
|
|
45
|
+
assert.ok(s.stack?.includes('at '));
|
|
46
|
+
});
|
|
47
|
+
it('serializes cause recursively and stops on cycles', () => {
|
|
48
|
+
const a = new Error('a');
|
|
49
|
+
const b = new Error('b', { cause: a });
|
|
50
|
+
a.cause = b;
|
|
51
|
+
const s = serializeErr(b);
|
|
52
|
+
assert.equal(s.cause?.message, 'a');
|
|
53
|
+
assert.doesNotThrow(() => JSON.stringify(s));
|
|
54
|
+
});
|
|
55
|
+
it('accepts non Error values', () => {
|
|
56
|
+
assert.equal(serializeErr('plain').message, 'plain');
|
|
57
|
+
assert.equal(serializeErr(null).message, 'null');
|
|
58
|
+
assert.equal(serializeErr(undefined).message, 'undefined');
|
|
59
|
+
const s = serializeErr({ message: 'obj', token: 'tok' });
|
|
60
|
+
assert.equal(s.message, 'obj');
|
|
61
|
+
assert.ok(!JSON.stringify(s).includes('tok'));
|
|
62
|
+
});
|
|
63
|
+
});
|
|
64
|
+
//# sourceMappingURL=log-err.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"log-err.test.js","sourceRoot":"","sources":["log-err.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,WAAW,CAAA;AACxC,OAAO,EAAE,MAAM,IAAI,MAAM,EAAE,MAAM,QAAQ,CAAA;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAE3C,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC5B,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAA;QACpH,MAAM,CAAC,GAAG,YAAY,CAAC,GAAG,CAAC,CAAA;QAC3B,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAA;QACvF,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;QAC7B,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IACjC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;QACrC,MAAM,gBAAiB,SAAQ,KAAK;YAAG,YAAa,CAAS,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAA,CAAC,CAAC;SAAE;QAC7G,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAA;IAClF,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;QACnC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;IAC5F,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;QAClC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;IAC/F,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,+DAA+D,EAAE,GAAG,EAAE;QACvE,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,SAAS,CAAC,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,+BAA+B,EAAE,OAAO,EAAE,EAAE,aAAa,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC,CAAA;QACrL,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAA;QAC9C,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAA;QACnC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,yBAAyB,CAAC,CAAC,CAAA;QACnD,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,YAAY,CAAC,EAAE,EAAE,IAAI,EAAE,mBAAmB,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE,EAAE,CAAC,CAAA;QACpN,MAAM,CAAC,GAAG,YAAY,CAAC,GAAG,CAAC,CAAA;QAC3B,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,EAAE,YAAY,CAAC,CAAA;QACjC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;QAC3B,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAA;QAChD,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAA;IAChD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;QAChE,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,eAAe,EAAE,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QAC/E,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,SAAS,IAAI,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,mBAAmB,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAA;QACjG,MAAM,CAAC,GAAG,YAAY,CAAC,GAAG,CAAC,CAAA;QAC3B,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,EAAE,kBAAkB,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;QACzE,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,IAAI,EAAE,gBAAgB,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAA;QAC3E,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAA;IACrC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;QAC1D,MAAM,CAAC,GAAQ,IAAI,KAAK,CAAC,GAAG,CAAC,CAAA;QAC7B,MAAM,CAAC,GAAQ,IAAI,KAAK,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAA;QAC3C,CAAC,CAAC,KAAK,GAAG,CAAC,CAAA;QACX,MAAM,CAAC,GAAG,YAAY,CAAC,CAAC,CAAC,CAAA;QACzB,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,CAAC,CAAA;QACnC,MAAM,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAA;IAC9C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;QAClC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QACpD,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;QAChD,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,WAAW,CAAC,CAAA;QAC1D,MAAM,CAAC,GAAG,YAAY,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAA;QACxD,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;QAC9B,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAA;IAC/C,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA","sourcesContent":["import { describe, it } from 'node:test'\nimport { strict as assert } from 'assert'\nimport { serializeErr } from './log-err.js'\n\ndescribe('serializeErr', () => {\n it('keeps only whitelisted fields of an Error', () => {\n const err = Object.assign(new Error('boom'), { code: 'E1', status: 500, keyValue: { email: 'a@b.c' }, secret: 'x' })\n const s = serializeErr(err)\n assert.deepEqual(Object.keys(s).sort(), ['code', 'message', 'stack', 'status', 'type'])\n assert.equal(s.type, 'Error')\n assert.equal(s.message, 'boom')\n })\n\n it('uses the error name as type', () => {\n class MongoServerError extends Error { constructor (m: string) { super(m); this.name = 'MongoServerError' } }\n assert.equal(serializeErr(new MongoServerError('dup')).type, 'MongoServerError')\n })\n\n it('maps statusCode to status', () => {\n assert.equal(serializeErr(Object.assign(new Error('x'), { statusCode: 404 })).status, 404)\n })\n\n it('drops non primitive code', () => {\n assert.equal(serializeErr(Object.assign(new Error('x'), { code: { a: 1 } })).code, undefined)\n })\n\n it('never leaks axios request headers and strips the query string', () => {\n const err = Object.assign(new Error('timeout'), { isAxiosError: true, config: { method: 'get', url: 'https://x.org/p?apiKey=s3cret', headers: { Authorization: 'Bearer s3cret' } } })\n const json = JSON.stringify(serializeErr(err))\n assert.ok(!json.includes('s3cret'))\n assert.ok(json.includes('\"url\":\"https://x.org/p\"'))\n assert.ok(json.includes('\"method\":\"get\"'))\n })\n\n it('handles AxiosRequestError shape without bodies', () => {\n const err = Object.assign(new Error('500 - fail'), { name: 'AxiosRequestError', status: 500, config: { method: 'post', url: 'http://a/b?k=v', data: 'body' }, data: { secret: 1 }, headers: { 'set-cookie': 'x' } })\n const s = serializeErr(err)\n assert.equal(s.url, 'http://a/b')\n assert.equal(s.status, 500)\n assert.ok(!JSON.stringify(s).includes('secret'))\n assert.ok(!JSON.stringify(s).includes('body'))\n })\n\n it('truncates long messages and the message line of stacks', () => {\n const body = JSON.stringify({ echoed: 'Bearer SECRET', doc: 'x'.repeat(5000) })\n const err = Object.assign(new Error(`500 - ${body}`), { name: 'AxiosRequestError', status: 500 })\n const s = serializeErr(err)\n assert.ok(s.message.length <= 1001, `message length ${s.message.length}`)\n assert.ok((s.stack ?? '').length < 2000, `stack length ${s.stack?.length}`)\n assert.ok(s.stack?.includes('at '))\n })\n\n it('serializes cause recursively and stops on cycles', () => {\n const a: any = new Error('a')\n const b: any = new Error('b', { cause: a })\n a.cause = b\n const s = serializeErr(b)\n assert.equal(s.cause?.message, 'a')\n assert.doesNotThrow(() => JSON.stringify(s))\n })\n\n it('accepts non Error values', () => {\n assert.equal(serializeErr('plain').message, 'plain')\n assert.equal(serializeErr(null).message, 'null')\n assert.equal(serializeErr(undefined).message, 'undefined')\n const s = serializeErr({ message: 'obj', token: 'tok' })\n assert.equal(s.message, 'obj')\n assert.ok(!JSON.stringify(s).includes('tok'))\n })\n})\n"]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { describe, it } from 'node:test';
|
|
2
|
+
import { strict as assert } from 'assert';
|
|
3
|
+
import { Worker } from 'node:worker_threads';
|
|
4
|
+
import { getLogContext, withLogContext } from './log.js';
|
|
5
|
+
describe('log context in worker threads', () => {
|
|
6
|
+
it('is propagated through the task payload', async () => {
|
|
7
|
+
const line = await withLogContext({ datasetId: 'w1' }, () => new Promise((resolve, reject) => {
|
|
8
|
+
const worker = new Worker(new URL('./test-fixtures/log-worker.js', import.meta.url), { workerData: { logContext: getLogContext() } });
|
|
9
|
+
worker.once('message', resolve);
|
|
10
|
+
worker.once('error', reject);
|
|
11
|
+
}));
|
|
12
|
+
assert.equal(JSON.parse(line).datasetId, 'w1');
|
|
13
|
+
assert.equal(JSON.parse(line).msg, 'in worker');
|
|
14
|
+
});
|
|
15
|
+
});
|
|
16
|
+
//# sourceMappingURL=log-worker.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"log-worker.test.js","sourceRoot":"","sources":["log-worker.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,WAAW,CAAA;AACxC,OAAO,EAAE,MAAM,IAAI,MAAM,EAAE,MAAM,QAAQ,CAAA;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AAC5C,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,UAAU,CAAA;AAExD,QAAQ,CAAC,+BAA+B,EAAE,GAAG,EAAE;IAC7C,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;QACtD,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnG,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC,+BAA+B,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,UAAU,EAAE,EAAE,UAAU,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC,CAAA;YACrI,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;YAC/B,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;QAC9B,CAAC,CAAC,CAAC,CAAA;QACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;QAC9C,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,WAAW,CAAC,CAAA;IACjD,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA","sourcesContent":["import { describe, it } from 'node:test'\nimport { strict as assert } from 'assert'\nimport { Worker } from 'node:worker_threads'\nimport { getLogContext, withLogContext } from './log.js'\n\ndescribe('log context in worker threads', () => {\n it('is propagated through the task payload', async () => {\n const line = await withLogContext({ datasetId: 'w1' }, () => new Promise<string>((resolve, reject) => {\n const worker = new Worker(new URL('./test-fixtures/log-worker.js', import.meta.url), { workerData: { logContext: getLogContext() } })\n worker.once('message', resolve)\n worker.once('error', reject)\n }))\n assert.equal(JSON.parse(line).datasetId, 'w1')\n assert.equal(JSON.parse(line).msg, 'in worker')\n })\n})\n"]}
|
package/log.d.ts
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { Counter } from 'prom-client';
|
|
2
|
+
export type LogLevel = 'debug' | 'info' | 'warn' | 'alert' | 'error';
|
|
3
|
+
export type LogFields = {
|
|
4
|
+
code?: string;
|
|
5
|
+
err?: unknown;
|
|
6
|
+
[key: string]: unknown;
|
|
7
|
+
};
|
|
8
|
+
export interface Logger {
|
|
9
|
+
info(msg: string, fields?: LogFields): void;
|
|
10
|
+
warn(msg: string, fields?: LogFields): void;
|
|
11
|
+
alert(msg: string, fields?: LogFields): void;
|
|
12
|
+
error(msg: string, fields?: LogFields): void;
|
|
13
|
+
init(opts: {
|
|
14
|
+
service?: string;
|
|
15
|
+
}): void;
|
|
16
|
+
}
|
|
17
|
+
type LogFormat = 'json' | 'pretty';
|
|
18
|
+
export type LogContext = Record<string, string | number | boolean | undefined>;
|
|
19
|
+
/** run fn in a scope where every log line carries the fields of the parent scope plus these */
|
|
20
|
+
export declare function withLogContext<T>(fields: LogContext, fn: () => T): T;
|
|
21
|
+
/** a copy of the current context, for example to pass it to a worker thread in a task payload */
|
|
22
|
+
export declare function getLogContext(): LogContext;
|
|
23
|
+
/** @internal test seam, not API */
|
|
24
|
+
export declare function _resetLogger(opts?: {
|
|
25
|
+
destination?: {
|
|
26
|
+
write(s: string): unknown;
|
|
27
|
+
};
|
|
28
|
+
format?: LogFormat;
|
|
29
|
+
}): void;
|
|
30
|
+
export declare const logsCounter: Counter<'level' | 'code' | 'host'> | null;
|
|
31
|
+
export declare const internalErrorCounter: Counter<'errorCode'> | null;
|
|
32
|
+
export declare const eventsLogCounter: Counter<'level' | 'code' | 'host'> | null;
|
|
33
|
+
declare const log: Logger;
|
|
34
|
+
export default log;
|