@blamejs/core 0.7.18 → 0.7.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +427 -423
- package/README.md +150 -150
- package/bin/blamejs.js +0 -0
- package/index.js +310 -308
- package/lib/api-key.js +660 -660
- package/lib/api-snapshot.js +338 -338
- package/lib/app-shutdown.js +385 -385
- package/lib/app.js +365 -365
- package/lib/archive.js +250 -250
- package/lib/atomic-file.js +544 -544
- package/lib/audit-chain.js +177 -177
- package/lib/audit-sign.js +344 -344
- package/lib/audit-tools.js +677 -677
- package/lib/audit.js +766 -766
- package/lib/auth/jwt-external.js +365 -0
- package/lib/auth/jwt.js +337 -311
- package/lib/auth/lockout.js +436 -436
- package/lib/auth/oauth.js +721 -721
- package/lib/auth/passkey.js +181 -181
- package/lib/auth/password.js +628 -594
- package/lib/backup/bundle.js +217 -217
- package/lib/backup/crypto.js +176 -176
- package/lib/backup/index.js +515 -515
- package/lib/backup/manifest.js +282 -282
- package/lib/break-glass.js +1338 -1338
- package/lib/bundler.js +441 -441
- package/lib/cache-redis.js +256 -256
- package/lib/cache.js +1206 -1206
- package/lib/canonical-json.js +115 -115
- package/lib/chain-writer.js +234 -234
- package/lib/cli-helpers.js +206 -206
- package/lib/cli.js +2334 -2334
- package/lib/cluster-provider-db.js +317 -317
- package/lib/cluster-storage.js +226 -226
- package/lib/cluster.js +703 -703
- package/lib/config-drift.js +301 -301
- package/lib/consent.js +222 -222
- package/lib/constants.js +191 -191
- package/lib/cookies.js +350 -315
- package/lib/credential-hash.js +322 -322
- package/lib/crypto.js +266 -266
- package/lib/csv.js +275 -275
- package/lib/db-declare-row-policy.js +267 -267
- package/lib/db-declare-view.js +420 -420
- package/lib/db-query.js +406 -406
- package/lib/db-schema.js +319 -319
- package/lib/db.js +1288 -1288
- package/lib/deprecate.js +222 -222
- package/lib/dev.js +335 -335
- package/lib/dual-control.js +473 -473
- package/lib/error-page.js +420 -420
- package/lib/external-db-migrate.js +441 -441
- package/lib/external-db.js +1061 -1061
- package/lib/file-type.js +273 -273
- package/lib/forms.js +422 -422
- package/lib/framework-error.js +293 -293
- package/lib/framework-schema.js +717 -717
- package/lib/handlers.js +350 -350
- package/lib/http-client-cookie-jar.js +508 -508
- package/lib/http-client.js +1195 -1195
- package/lib/i18n.js +878 -878
- package/lib/jobs.js +185 -185
- package/lib/log-stream-cloudwatch.js +369 -369
- package/lib/log-stream-local.js +146 -146
- package/lib/log-stream-otlp-grpc.js +410 -410
- package/lib/log-stream-otlp.js +286 -286
- package/lib/log-stream-syslog.js +302 -302
- package/lib/log-stream-webhook.js +199 -199
- package/lib/log-stream.js +330 -330
- package/lib/log.js +500 -500
- package/lib/mail-bounce.js +528 -528
- package/lib/mail-dkim.js +369 -369
- package/lib/mail.js +981 -981
- package/lib/metrics.js +683 -683
- package/lib/middleware/api-encrypt.js +936 -936
- package/lib/middleware/attach-user.js +157 -157
- package/lib/middleware/bearer-auth.js +152 -0
- package/lib/middleware/body-parser.js +1170 -1170
- package/lib/middleware/bot-guard.js +178 -178
- package/lib/middleware/compression.js +452 -452
- package/lib/middleware/cors.js +314 -314
- package/lib/middleware/csp-nonce.js +348 -348
- package/lib/middleware/csrf-protect.js +399 -316
- package/lib/middleware/db-role-for.js +264 -264
- package/lib/middleware/fetch-metadata.js +129 -0
- package/lib/middleware/health.js +392 -392
- package/lib/middleware/index.js +85 -79
- package/lib/middleware/rate-limit.js +358 -358
- package/lib/middleware/request-id.js +61 -61
- package/lib/middleware/request-log.js +168 -168
- package/lib/middleware/require-auth.js +104 -104
- package/lib/middleware/security-headers.js +121 -116
- package/lib/middleware/sse.js +166 -166
- package/lib/migrations.js +383 -383
- package/lib/mtls-ca.js +518 -518
- package/lib/mtls-engine-default.js +481 -481
- package/lib/network-dns.js +632 -632
- package/lib/network-heartbeat.js +290 -290
- package/lib/network-nts.js +574 -574
- package/lib/network-proxy.js +265 -265
- package/lib/network-tls.js +328 -328
- package/lib/network.js +233 -233
- package/lib/notify.js +612 -612
- package/lib/ntp-check.js +229 -229
- package/lib/numeric-bounds.js +111 -111
- package/lib/object-store/azure-blob-bucket-ops.js +349 -349
- package/lib/object-store/azure-blob.js +488 -488
- package/lib/object-store/gcs-bucket-ops.js +351 -351
- package/lib/object-store/gcs.js +519 -519
- package/lib/object-store/http-put.js +153 -153
- package/lib/object-store/index.js +197 -197
- package/lib/object-store/sigv4-bucket-ops.js +1092 -1092
- package/lib/object-store/sigv4.js +903 -903
- package/lib/observability.js +151 -151
- package/lib/otel-export.js +269 -269
- package/lib/pagination.js +464 -464
- package/lib/parsers/index.js +80 -80
- package/lib/parsers/safe-env.js +642 -642
- package/lib/parsers/safe-ini.js +292 -292
- package/lib/parsers/safe-toml.js +784 -784
- package/lib/parsers/safe-xml.js +390 -390
- package/lib/parsers/safe-yaml.js +1015 -1015
- package/lib/permissions.js +708 -708
- package/lib/pqc-agent.js +87 -87
- package/lib/pqc-gate.js +279 -279
- package/lib/protobuf-encoder.js +190 -190
- package/lib/protocol-dispatcher.js +161 -161
- package/lib/pubsub-redis.js +167 -167
- package/lib/pubsub.js +429 -429
- package/lib/queue-local.js +476 -476
- package/lib/queue-redis.js +745 -745
- package/lib/queue-sqs.js +319 -319
- package/lib/queue.js +695 -695
- package/lib/redis-client.js +519 -519
- package/lib/request-helpers.js +340 -340
- package/lib/restore-bundle.js +237 -237
- package/lib/restore-rollback.js +259 -259
- package/lib/restore.js +409 -409
- package/lib/retry.js +376 -376
- package/lib/router.js +748 -748
- package/lib/safe-async.js +735 -735
- package/lib/safe-buffer.js +237 -237
- package/lib/safe-json.js +541 -541
- package/lib/safe-schema.js +1266 -1266
- package/lib/safe-url.js +159 -159
- package/lib/scheduler.js +706 -706
- package/lib/security-assert.js +373 -373
- package/lib/seeders.js +618 -618
- package/lib/session.js +535 -478
- package/lib/slug.js +269 -269
- package/lib/ssrf-guard.js +401 -401
- package/lib/static.js +7 -5
- package/lib/storage.js +471 -471
- package/lib/subject.js +281 -281
- package/lib/template.js +791 -791
- package/lib/testing.js +798 -798
- package/lib/time.js +310 -310
- package/lib/totp.js +302 -302
- package/lib/tracing.js +494 -494
- package/lib/uuid.js +132 -132
- package/lib/validate-opts.js +340 -340
- package/lib/vault/index.js +308 -308
- package/lib/vault/rotate.js +784 -784
- package/lib/vault/wrap.js +296 -296
- package/lib/vendor/noble-ciphers.cjs +9 -9
- package/lib/webhook.js +595 -595
- package/lib/websocket.js +1048 -1048
- package/package.json +77 -77
- package/sbom.cyclonedx.json +7 -7
package/lib/log-stream.js
CHANGED
|
@@ -1,330 +1,330 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
/**
|
|
3
|
-
* Log streaming dispatcher — operational logs to one or many sinks with
|
|
4
|
-
* redaction and bidirectional command channel.
|
|
5
|
-
*
|
|
6
|
-
* Same dispatcher pattern: backends configured per-name with a protocol
|
|
7
|
-
* + protocol-specific options. Built-in protocols:
|
|
8
|
-
*
|
|
9
|
-
* local — append-only file with rotation
|
|
10
|
-
* webhook — generic HTTP POST; covers Splunk HEC, Datadog, Sumo
|
|
11
|
-
* Logic, Loki, custom collectors that ingest JSON
|
|
12
|
-
* otlp — OpenTelemetry Protocol over HTTP/JSON; ResourceLogs
|
|
13
|
-
* envelope with severity mapping per OTel Logs Data
|
|
14
|
-
* Model. Operators with an OTel collector running
|
|
15
|
-
* (k8s, cloud) get standard log forwarding without a
|
|
16
|
-
* vendor-specific adapter.
|
|
17
|
-
* otlp-grpc — same OTel Logs Data Model but over gRPC (HTTP/2 +
|
|
18
|
-
* hand-encoded protobuf). Higher-throughput than the
|
|
19
|
-
* JSON variant; preferred for production deployments
|
|
20
|
-
* pushing >100K logs/s straight to a remote OTel
|
|
21
|
-
* collector. No protobuf parser ships in the bundle —
|
|
22
|
-
* the encoder is the framework's own (lib/protobuf-encoder.js).
|
|
23
|
-
* cloudwatch — AWS CloudWatch Logs (PutLogEvents) over HTTPS with
|
|
24
|
-
* SigV4. Pass { autoCreate: true } to have the framework
|
|
25
|
-
* issue CreateLogGroup + CreateLogStream on first emit
|
|
26
|
-
* (idempotent — ResourceAlreadyExistsException treated as
|
|
27
|
-
* success). Honors IAM role + STS session tokens. Respects
|
|
28
|
-
* the 10K-event / 1 MiB / 256 KiB-per-event AWS caps.
|
|
29
|
-
* syslog — RFC 5424 with octet-counting framing over UDP / TCP /
|
|
30
|
-
* TLS. UDP is best-effort; TCP/TLS buffer during socket
|
|
31
|
-
* reconnect and replay on connect. Default ports 514
|
|
32
|
-
* (UDP/TCP) and 6514 (TLS).
|
|
33
|
-
*
|
|
34
|
-
* Every emit goes through lib/redact.js BEFORE any sink sees it. PHI/PCI
|
|
35
|
-
* never reaches operational logs even on a misconfigured field name —
|
|
36
|
-
* pattern detectors catch credit-card-shaped values, JWTs, PEM blocks,
|
|
37
|
-
* AWS access keys, vault-sealed strings, SSN-shaped values, etc.
|
|
38
|
-
*
|
|
39
|
-
* Bidirectional command channel:
|
|
40
|
-
* logStream.onIncoming(handler) registers a handler for inbound events.
|
|
41
|
-
* Operators wire their HTTP route (or other transport — webhook receiver,
|
|
42
|
-
* SSE, message-queue subscriber) to call logStream.deliverIncoming(payload)
|
|
43
|
-
* which invokes registered handlers. The framework doesn't prescribe the
|
|
44
|
-
* transport — it provides the dispatch.
|
|
45
|
-
*
|
|
46
|
-
* Public API:
|
|
47
|
-
* logStream.init({ sinks: { name: { protocol, ... } }, classification? })
|
|
48
|
-
* logStream.emit(level, message, meta?) (sync — non-blocking)
|
|
49
|
-
* logStream.info(msg, meta?) / .warn / .error / .debug
|
|
50
|
-
* logStream.onIncoming(handler) (handler returns Promise)
|
|
51
|
-
* logStream.deliverIncoming(payload, opts?)
|
|
52
|
-
* logStream.shutdown()
|
|
53
|
-
* logStream.listSinks() → [{ name, protocol, stats }]
|
|
54
|
-
*/
|
|
55
|
-
var localProto = require("./log-stream-local");
|
|
56
|
-
var webhookProto = require("./log-stream-webhook");
|
|
57
|
-
var otlpProto = require("./log-stream-otlp");
|
|
58
|
-
var otlpGrpcProto = require("./log-stream-otlp-grpc");
|
|
59
|
-
var cloudwatchProto = require("./log-stream-cloudwatch");
|
|
60
|
-
var syslogProto = require("./log-stream-syslog");
|
|
61
|
-
var { boot } = require("./log");
|
|
62
|
-
var redactor = require("./redact");
|
|
63
|
-
var lazyRequire = require("./lazy-require");
|
|
64
|
-
var protocolDispatcher = require("./protocol-dispatcher");
|
|
65
|
-
var { LogStreamError } = require("./framework-error");
|
|
66
|
-
|
|
67
|
-
var _log = boot("log-stream");
|
|
68
|
-
|
|
69
|
-
var dispatcher = protocolDispatcher.create({
|
|
70
|
-
name: "log-stream",
|
|
71
|
-
errorClass: LogStreamError,
|
|
72
|
-
protocols: {
|
|
73
|
-
"local": localProto,
|
|
74
|
-
"webhook": webhookProto,
|
|
75
|
-
"otlp": otlpProto,
|
|
76
|
-
"otlp-grpc": otlpGrpcProto,
|
|
77
|
-
"cloudwatch": cloudwatchProto,
|
|
78
|
-
"syslog": syslogProto,
|
|
79
|
-
},
|
|
80
|
-
deferred: {},
|
|
81
|
-
fallbackProtocol: "local",
|
|
82
|
-
});
|
|
83
|
-
|
|
84
|
-
var LEVELS = ["debug", "info", "warn", "error"];
|
|
85
|
-
// Ordinal priority — higher = more severe. Values are spaced (1..4)
|
|
86
|
-
// only so future intermediate levels (e.g. 'notice' between info/warn)
|
|
87
|
-
// could slot in without renumbering the existing four.
|
|
88
|
-
var LEVEL_PRIORITY = { debug: 1, info: 2, warn: 3, error: 4 };
|
|
89
|
-
|
|
90
|
-
var _err = LogStreamError.factory;
|
|
91
|
-
|
|
92
|
-
var audit = lazyRequire(function () { return require("./audit"); });
|
|
93
|
-
|
|
94
|
-
var initialized = false;
|
|
95
|
-
var sinks = {};
|
|
96
|
-
// Pending emit promises, tracked so shutdown can drain them before
|
|
97
|
-
// closing sink fds. Without this, fire-and-forget emits queued just
|
|
98
|
-
// before shutdown raced with close() and silently dropped records.
|
|
99
|
-
var _inflight = new Set();
|
|
100
|
-
var minLevel = "info";
|
|
101
|
-
var incomingHandlers = [];
|
|
102
|
-
|
|
103
|
-
function init(opts) {
|
|
104
|
-
if (initialized) return;
|
|
105
|
-
if (!opts || !opts.sinks) throw new Error("logStream.init({ sinks }) is required");
|
|
106
|
-
|
|
107
|
-
sinks = {};
|
|
108
|
-
for (var name in opts.sinks) {
|
|
109
|
-
var cfg = opts.sinks[name];
|
|
110
|
-
var proto = dispatcher.resolve(cfg.protocol);
|
|
111
|
-
sinks[name] = {
|
|
112
|
-
name: name,
|
|
113
|
-
protocol: cfg.protocol,
|
|
114
|
-
raw: proto.create(cfg),
|
|
115
|
-
levelFilter: cfg.minLevel || null,
|
|
116
|
-
};
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
minLevel = (opts.minLevel || "info").toLowerCase();
|
|
120
|
-
initialized = true;
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
function _shouldEmit(level, sinkLevelFilter) {
|
|
124
|
-
var threshold = sinkLevelFilter ? LEVEL_PRIORITY[sinkLevelFilter] : LEVEL_PRIORITY[minLevel];
|
|
125
|
-
return LEVEL_PRIORITY[level] >= threshold;
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
function emit(level, message, meta) {
|
|
129
|
-
if (!initialized) return;
|
|
130
|
-
if (LEVELS.indexOf(level) === -1) {
|
|
131
|
-
throw _err("INVALID_LEVEL", "log level must be one of " + LEVELS.join(", "), true);
|
|
132
|
-
}
|
|
133
|
-
// Build the record. Redact metadata BEFORE distribution to any sink.
|
|
134
|
-
var record = {
|
|
135
|
-
ts: Date.now(),
|
|
136
|
-
level: level,
|
|
137
|
-
message: message == null ? null : String(message),
|
|
138
|
-
};
|
|
139
|
-
if (meta) {
|
|
140
|
-
record.meta = redactor.redact(meta);
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
// Fire-and-forget to all sinks. Sink errors don't bubble — they're
|
|
144
|
-
// captured by audit (system.log.sink-failure) so an external sink
|
|
145
|
-
// outage doesn't take down the app's request handlers. Pending
|
|
146
|
-
// emits are tracked in _inflight so shutdown() can drain them
|
|
147
|
-
// before closing fds (otherwise records queued just before
|
|
148
|
-
// shutdown would be lost when close() ran ahead of the microtask).
|
|
149
|
-
Object.keys(sinks).forEach(function (name) {
|
|
150
|
-
var sink = sinks[name];
|
|
151
|
-
if (!_shouldEmit(level, sink.levelFilter)) return;
|
|
152
|
-
var p = Promise.resolve()
|
|
153
|
-
.then(function () { return sink.raw.emit(record); })
|
|
154
|
-
.catch(function (e) {
|
|
155
|
-
audit().safeEmit({
|
|
156
|
-
action: "system.log.sink-failure",
|
|
157
|
-
outcome: "failure",
|
|
158
|
-
reason: (e && e.message) || String(e),
|
|
159
|
-
metadata: { sink: name, level: level },
|
|
160
|
-
});
|
|
161
|
-
});
|
|
162
|
-
_inflight.add(p);
|
|
163
|
-
p.then(function () { _inflight.delete(p); }, function () { _inflight.delete(p); });
|
|
164
|
-
});
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
function debug(message, meta) { emit("debug", message, meta); }
|
|
168
|
-
function info(message, meta) { emit("info", message, meta); }
|
|
169
|
-
function warn(message, meta) { emit("warn", message, meta); }
|
|
170
|
-
function error(message, meta) { emit("error", message, meta); }
|
|
171
|
-
|
|
172
|
-
// ---- Bidirectional incoming command channel ----
|
|
173
|
-
|
|
174
|
-
function onIncoming(handler) {
|
|
175
|
-
if (typeof handler !== "function") {
|
|
176
|
-
throw _err("INVALID_HANDLER", "onIncoming requires a function handler", true);
|
|
177
|
-
}
|
|
178
|
-
incomingHandlers.push(handler);
|
|
179
|
-
return function () {
|
|
180
|
-
var idx = incomingHandlers.indexOf(handler);
|
|
181
|
-
if (idx >= 0) incomingHandlers.splice(idx, 1);
|
|
182
|
-
};
|
|
183
|
-
}
|
|
184
|
-
|
|
185
|
-
async function deliverIncoming(payload, opts) {
|
|
186
|
-
opts = opts || {};
|
|
187
|
-
var redacted = redactor.redact(payload);
|
|
188
|
-
// Audit-log the inbound command BEFORE invoking handlers — even handler
|
|
189
|
-
// exceptions don't lose the receipt.
|
|
190
|
-
audit().safeEmit({
|
|
191
|
-
actor: opts.actor || {},
|
|
192
|
-
action: "system.log.incoming",
|
|
193
|
-
metadata: { payload: redacted, source: opts.source || null },
|
|
194
|
-
});
|
|
195
|
-
|
|
196
|
-
var results = [];
|
|
197
|
-
for (var i = 0; i < incomingHandlers.length; i++) {
|
|
198
|
-
try {
|
|
199
|
-
results.push({ ok: true, value: await incomingHandlers[i](payload, opts) });
|
|
200
|
-
} catch (e) {
|
|
201
|
-
results.push({ ok: false, error: (e && e.message) || String(e) });
|
|
202
|
-
}
|
|
203
|
-
}
|
|
204
|
-
return results;
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
async function shutdown() {
|
|
208
|
-
if (!initialized) return;
|
|
209
|
-
// Drain any in-flight emits before closing sink fds so records
|
|
210
|
-
// queued just before shutdown actually reach disk / the wire.
|
|
211
|
-
if (_inflight.size > 0) {
|
|
212
|
-
try { await Promise.all(Array.from(_inflight)); }
|
|
213
|
-
catch (_e) { /* sink errors already audited via the catch above */ }
|
|
214
|
-
}
|
|
215
|
-
for (var name in sinks) {
|
|
216
|
-
try {
|
|
217
|
-
if (typeof sinks[name].raw.close === "function") await sinks[name].raw.close();
|
|
218
|
-
} catch (_e) { /* best effort on shutdown */ }
|
|
219
|
-
}
|
|
220
|
-
sinks = {};
|
|
221
|
-
incomingHandlers = [];
|
|
222
|
-
initialized = false;
|
|
223
|
-
}
|
|
224
|
-
|
|
225
|
-
function listSinks() {
|
|
226
|
-
if (!initialized) return [];
|
|
227
|
-
return Object.keys(sinks).map(function (name) {
|
|
228
|
-
var s = sinks[name];
|
|
229
|
-
var stats = (typeof s.raw.stats === "function") ? s.raw.stats() : null;
|
|
230
|
-
return { name: name, protocol: s.protocol, stats: stats };
|
|
231
|
-
});
|
|
232
|
-
}
|
|
233
|
-
|
|
234
|
-
// ---- bootFromEnv ----
|
|
235
|
-
//
|
|
236
|
-
// Operator-friendly env-driven init that mirrors b.network.bootFromEnv.
|
|
237
|
-
// Reads BLAMEJS_LOG_STREAM_* env vars and constructs a single-sink
|
|
238
|
-
// configuration matching the operator's choice. Skipped silently when
|
|
239
|
-
// BLAMEJS_LOG_STREAM_PROTOCOL isn't set (operators using the in-code
|
|
240
|
-
// init() path keep their existing wiring).
|
|
241
|
-
//
|
|
242
|
-
// Recognised env vars:
|
|
243
|
-
// BLAMEJS_LOG_STREAM_PROTOCOL "local" | "webhook" | "otlp" | "cloudwatch"
|
|
244
|
-
// BLAMEJS_LOG_STREAM_MIN_LEVEL "debug" | "info" | "warn" | "error"
|
|
245
|
-
//
|
|
246
|
-
// webhook + otlp shared:
|
|
247
|
-
// BLAMEJS_LOG_STREAM_URL
|
|
248
|
-
// BLAMEJS_LOG_STREAM_TOKEN (auth: bearer)
|
|
249
|
-
// otlp-only:
|
|
250
|
-
// BLAMEJS_LOG_STREAM_SERVICE_NAME
|
|
251
|
-
// BLAMEJS_LOG_STREAM_SERVICE_VERSION
|
|
252
|
-
// cloudwatch-only (AWS_* are standard):
|
|
253
|
-
// AWS_REGION, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN
|
|
254
|
-
// BLAMEJS_LOG_STREAM_CLOUDWATCH_LOG_GROUP
|
|
255
|
-
// BLAMEJS_LOG_STREAM_CLOUDWATCH_LOG_STREAM
|
|
256
|
-
// local-only:
|
|
257
|
-
// BLAMEJS_LOG_STREAM_PATH
|
|
258
|
-
function bootFromEnv(opts) {
|
|
259
|
-
opts = opts || {};
|
|
260
|
-
var env = opts.env || process.env;
|
|
261
|
-
var proto = env.BLAMEJS_LOG_STREAM_PROTOCOL;
|
|
262
|
-
if (!proto) return false;
|
|
263
|
-
var sink = { protocol: proto };
|
|
264
|
-
if (proto === "webhook") {
|
|
265
|
-
sink.url = env.BLAMEJS_LOG_STREAM_URL;
|
|
266
|
-
if (env.BLAMEJS_LOG_STREAM_TOKEN) {
|
|
267
|
-
sink.auth = "bearer";
|
|
268
|
-
sink.token = env.BLAMEJS_LOG_STREAM_TOKEN;
|
|
269
|
-
}
|
|
270
|
-
} else if (proto === "otlp") {
|
|
271
|
-
sink.url = env.BLAMEJS_LOG_STREAM_URL;
|
|
272
|
-
sink.serviceName = env.BLAMEJS_LOG_STREAM_SERVICE_NAME || "blamejs";
|
|
273
|
-
sink.serviceVersion = env.BLAMEJS_LOG_STREAM_SERVICE_VERSION || null;
|
|
274
|
-
if (env.BLAMEJS_LOG_STREAM_TOKEN) {
|
|
275
|
-
sink.auth = "bearer";
|
|
276
|
-
sink.token = env.BLAMEJS_LOG_STREAM_TOKEN;
|
|
277
|
-
}
|
|
278
|
-
} else if (proto === "cloudwatch") {
|
|
279
|
-
sink.region = env.AWS_REGION;
|
|
280
|
-
sink.accessKeyId = env.AWS_ACCESS_KEY_ID;
|
|
281
|
-
sink.secretAccessKey = env.AWS_SECRET_ACCESS_KEY;
|
|
282
|
-
sink.sessionToken = env.AWS_SESSION_TOKEN || null;
|
|
283
|
-
sink.logGroupName = env.BLAMEJS_LOG_STREAM_CLOUDWATCH_LOG_GROUP;
|
|
284
|
-
sink.logStreamName = env.BLAMEJS_LOG_STREAM_CLOUDWATCH_LOG_STREAM;
|
|
285
|
-
} else if (proto === "local") {
|
|
286
|
-
sink.path = env.BLAMEJS_LOG_STREAM_PATH;
|
|
287
|
-
} else {
|
|
288
|
-
throw _err("BAD_OPT",
|
|
289
|
-
"BLAMEJS_LOG_STREAM_PROTOCOL='" + proto + "' is not one of " +
|
|
290
|
-
"local | webhook | otlp | cloudwatch (or a custom backend wired via init())");
|
|
291
|
-
}
|
|
292
|
-
init({
|
|
293
|
-
sinks: { primary: sink },
|
|
294
|
-
minLevel: env.BLAMEJS_LOG_STREAM_MIN_LEVEL || undefined,
|
|
295
|
-
});
|
|
296
|
-
return true;
|
|
297
|
-
}
|
|
298
|
-
|
|
299
|
-
function _resetForTest() {
|
|
300
|
-
Object.keys(sinks).forEach(function (n) {
|
|
301
|
-
try { if (sinks[n].raw.close) sinks[n].raw.close(); }
|
|
302
|
-
catch (e) {
|
|
303
|
-
// Test-reset path; best-effort — surface for diagnosability via
|
|
304
|
-
// the framework's boot logger so flaky teardown doesn't go silent.
|
|
305
|
-
_log.debug("reset-close-failed: " + (e && e.message || e));
|
|
306
|
-
}
|
|
307
|
-
});
|
|
308
|
-
sinks = {};
|
|
309
|
-
incomingHandlers = [];
|
|
310
|
-
initialized = false;
|
|
311
|
-
audit.reset();
|
|
312
|
-
}
|
|
313
|
-
|
|
314
|
-
module.exports = {
|
|
315
|
-
init: init,
|
|
316
|
-
bootFromEnv: bootFromEnv,
|
|
317
|
-
emit: emit,
|
|
318
|
-
debug: debug,
|
|
319
|
-
info: info,
|
|
320
|
-
warn: warn,
|
|
321
|
-
error: error,
|
|
322
|
-
onIncoming: onIncoming,
|
|
323
|
-
deliverIncoming: deliverIncoming,
|
|
324
|
-
shutdown: shutdown,
|
|
325
|
-
listSinks: listSinks,
|
|
326
|
-
LEVELS: LEVELS,
|
|
327
|
-
PROTOCOLS: dispatcher.protocols,
|
|
328
|
-
DEFERRED_PROTOCOLS: dispatcher.deferred,
|
|
329
|
-
_resetForTest: _resetForTest,
|
|
330
|
-
};
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Log streaming dispatcher — operational logs to one or many sinks with
|
|
4
|
+
* redaction and bidirectional command channel.
|
|
5
|
+
*
|
|
6
|
+
* Same dispatcher pattern: backends configured per-name with a protocol
|
|
7
|
+
* + protocol-specific options. Built-in protocols:
|
|
8
|
+
*
|
|
9
|
+
* local — append-only file with rotation
|
|
10
|
+
* webhook — generic HTTP POST; covers Splunk HEC, Datadog, Sumo
|
|
11
|
+
* Logic, Loki, custom collectors that ingest JSON
|
|
12
|
+
* otlp — OpenTelemetry Protocol over HTTP/JSON; ResourceLogs
|
|
13
|
+
* envelope with severity mapping per OTel Logs Data
|
|
14
|
+
* Model. Operators with an OTel collector running
|
|
15
|
+
* (k8s, cloud) get standard log forwarding without a
|
|
16
|
+
* vendor-specific adapter.
|
|
17
|
+
* otlp-grpc — same OTel Logs Data Model but over gRPC (HTTP/2 +
|
|
18
|
+
* hand-encoded protobuf). Higher-throughput than the
|
|
19
|
+
* JSON variant; preferred for production deployments
|
|
20
|
+
* pushing >100K logs/s straight to a remote OTel
|
|
21
|
+
* collector. No protobuf parser ships in the bundle —
|
|
22
|
+
* the encoder is the framework's own (lib/protobuf-encoder.js).
|
|
23
|
+
* cloudwatch — AWS CloudWatch Logs (PutLogEvents) over HTTPS with
|
|
24
|
+
* SigV4. Pass { autoCreate: true } to have the framework
|
|
25
|
+
* issue CreateLogGroup + CreateLogStream on first emit
|
|
26
|
+
* (idempotent — ResourceAlreadyExistsException treated as
|
|
27
|
+
* success). Honors IAM role + STS session tokens. Respects
|
|
28
|
+
* the 10K-event / 1 MiB / 256 KiB-per-event AWS caps.
|
|
29
|
+
* syslog — RFC 5424 with octet-counting framing over UDP / TCP /
|
|
30
|
+
* TLS. UDP is best-effort; TCP/TLS buffer during socket
|
|
31
|
+
* reconnect and replay on connect. Default ports 514
|
|
32
|
+
* (UDP/TCP) and 6514 (TLS).
|
|
33
|
+
*
|
|
34
|
+
* Every emit goes through lib/redact.js BEFORE any sink sees it. PHI/PCI
|
|
35
|
+
* never reaches operational logs even on a misconfigured field name —
|
|
36
|
+
* pattern detectors catch credit-card-shaped values, JWTs, PEM blocks,
|
|
37
|
+
* AWS access keys, vault-sealed strings, SSN-shaped values, etc.
|
|
38
|
+
*
|
|
39
|
+
* Bidirectional command channel:
|
|
40
|
+
* logStream.onIncoming(handler) registers a handler for inbound events.
|
|
41
|
+
* Operators wire their HTTP route (or other transport — webhook receiver,
|
|
42
|
+
* SSE, message-queue subscriber) to call logStream.deliverIncoming(payload)
|
|
43
|
+
* which invokes registered handlers. The framework doesn't prescribe the
|
|
44
|
+
* transport — it provides the dispatch.
|
|
45
|
+
*
|
|
46
|
+
* Public API:
|
|
47
|
+
* logStream.init({ sinks: { name: { protocol, ... } }, classification? })
|
|
48
|
+
* logStream.emit(level, message, meta?) (sync — non-blocking)
|
|
49
|
+
* logStream.info(msg, meta?) / .warn / .error / .debug
|
|
50
|
+
* logStream.onIncoming(handler) (handler returns Promise)
|
|
51
|
+
* logStream.deliverIncoming(payload, opts?)
|
|
52
|
+
* logStream.shutdown()
|
|
53
|
+
* logStream.listSinks() → [{ name, protocol, stats }]
|
|
54
|
+
*/
|
|
55
|
+
var localProto = require("./log-stream-local");
|
|
56
|
+
var webhookProto = require("./log-stream-webhook");
|
|
57
|
+
var otlpProto = require("./log-stream-otlp");
|
|
58
|
+
var otlpGrpcProto = require("./log-stream-otlp-grpc");
|
|
59
|
+
var cloudwatchProto = require("./log-stream-cloudwatch");
|
|
60
|
+
var syslogProto = require("./log-stream-syslog");
|
|
61
|
+
var { boot } = require("./log");
|
|
62
|
+
var redactor = require("./redact");
|
|
63
|
+
var lazyRequire = require("./lazy-require");
|
|
64
|
+
var protocolDispatcher = require("./protocol-dispatcher");
|
|
65
|
+
var { LogStreamError } = require("./framework-error");
|
|
66
|
+
|
|
67
|
+
var _log = boot("log-stream");
|
|
68
|
+
|
|
69
|
+
var dispatcher = protocolDispatcher.create({
|
|
70
|
+
name: "log-stream",
|
|
71
|
+
errorClass: LogStreamError,
|
|
72
|
+
protocols: {
|
|
73
|
+
"local": localProto,
|
|
74
|
+
"webhook": webhookProto,
|
|
75
|
+
"otlp": otlpProto,
|
|
76
|
+
"otlp-grpc": otlpGrpcProto,
|
|
77
|
+
"cloudwatch": cloudwatchProto,
|
|
78
|
+
"syslog": syslogProto,
|
|
79
|
+
},
|
|
80
|
+
deferred: {},
|
|
81
|
+
fallbackProtocol: "local",
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
var LEVELS = ["debug", "info", "warn", "error"];
|
|
85
|
+
// Ordinal priority — higher = more severe. Values are spaced (1..4)
|
|
86
|
+
// only so future intermediate levels (e.g. 'notice' between info/warn)
|
|
87
|
+
// could slot in without renumbering the existing four.
|
|
88
|
+
var LEVEL_PRIORITY = { debug: 1, info: 2, warn: 3, error: 4 };
|
|
89
|
+
|
|
90
|
+
var _err = LogStreamError.factory;
|
|
91
|
+
|
|
92
|
+
var audit = lazyRequire(function () { return require("./audit"); });
|
|
93
|
+
|
|
94
|
+
var initialized = false;
|
|
95
|
+
var sinks = {};
|
|
96
|
+
// Pending emit promises, tracked so shutdown can drain them before
|
|
97
|
+
// closing sink fds. Without this, fire-and-forget emits queued just
|
|
98
|
+
// before shutdown raced with close() and silently dropped records.
|
|
99
|
+
var _inflight = new Set();
|
|
100
|
+
var minLevel = "info";
|
|
101
|
+
var incomingHandlers = [];
|
|
102
|
+
|
|
103
|
+
function init(opts) {
|
|
104
|
+
if (initialized) return;
|
|
105
|
+
if (!opts || !opts.sinks) throw new Error("logStream.init({ sinks }) is required");
|
|
106
|
+
|
|
107
|
+
sinks = {};
|
|
108
|
+
for (var name in opts.sinks) {
|
|
109
|
+
var cfg = opts.sinks[name];
|
|
110
|
+
var proto = dispatcher.resolve(cfg.protocol);
|
|
111
|
+
sinks[name] = {
|
|
112
|
+
name: name,
|
|
113
|
+
protocol: cfg.protocol,
|
|
114
|
+
raw: proto.create(cfg),
|
|
115
|
+
levelFilter: cfg.minLevel || null,
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
minLevel = (opts.minLevel || "info").toLowerCase();
|
|
120
|
+
initialized = true;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function _shouldEmit(level, sinkLevelFilter) {
|
|
124
|
+
var threshold = sinkLevelFilter ? LEVEL_PRIORITY[sinkLevelFilter] : LEVEL_PRIORITY[minLevel];
|
|
125
|
+
return LEVEL_PRIORITY[level] >= threshold;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function emit(level, message, meta) {
|
|
129
|
+
if (!initialized) return;
|
|
130
|
+
if (LEVELS.indexOf(level) === -1) {
|
|
131
|
+
throw _err("INVALID_LEVEL", "log level must be one of " + LEVELS.join(", "), true);
|
|
132
|
+
}
|
|
133
|
+
// Build the record. Redact metadata BEFORE distribution to any sink.
|
|
134
|
+
var record = {
|
|
135
|
+
ts: Date.now(),
|
|
136
|
+
level: level,
|
|
137
|
+
message: message == null ? null : String(message),
|
|
138
|
+
};
|
|
139
|
+
if (meta) {
|
|
140
|
+
record.meta = redactor.redact(meta);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// Fire-and-forget to all sinks. Sink errors don't bubble — they're
|
|
144
|
+
// captured by audit (system.log.sink-failure) so an external sink
|
|
145
|
+
// outage doesn't take down the app's request handlers. Pending
|
|
146
|
+
// emits are tracked in _inflight so shutdown() can drain them
|
|
147
|
+
// before closing fds (otherwise records queued just before
|
|
148
|
+
// shutdown would be lost when close() ran ahead of the microtask).
|
|
149
|
+
Object.keys(sinks).forEach(function (name) {
|
|
150
|
+
var sink = sinks[name];
|
|
151
|
+
if (!_shouldEmit(level, sink.levelFilter)) return;
|
|
152
|
+
var p = Promise.resolve()
|
|
153
|
+
.then(function () { return sink.raw.emit(record); })
|
|
154
|
+
.catch(function (e) {
|
|
155
|
+
audit().safeEmit({
|
|
156
|
+
action: "system.log.sink-failure",
|
|
157
|
+
outcome: "failure",
|
|
158
|
+
reason: (e && e.message) || String(e),
|
|
159
|
+
metadata: { sink: name, level: level },
|
|
160
|
+
});
|
|
161
|
+
});
|
|
162
|
+
_inflight.add(p);
|
|
163
|
+
p.then(function () { _inflight.delete(p); }, function () { _inflight.delete(p); });
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function debug(message, meta) { emit("debug", message, meta); }
|
|
168
|
+
function info(message, meta) { emit("info", message, meta); }
|
|
169
|
+
function warn(message, meta) { emit("warn", message, meta); }
|
|
170
|
+
function error(message, meta) { emit("error", message, meta); }
|
|
171
|
+
|
|
172
|
+
// ---- Bidirectional incoming command channel ----
|
|
173
|
+
|
|
174
|
+
function onIncoming(handler) {
|
|
175
|
+
if (typeof handler !== "function") {
|
|
176
|
+
throw _err("INVALID_HANDLER", "onIncoming requires a function handler", true);
|
|
177
|
+
}
|
|
178
|
+
incomingHandlers.push(handler);
|
|
179
|
+
return function () {
|
|
180
|
+
var idx = incomingHandlers.indexOf(handler);
|
|
181
|
+
if (idx >= 0) incomingHandlers.splice(idx, 1);
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
async function deliverIncoming(payload, opts) {
|
|
186
|
+
opts = opts || {};
|
|
187
|
+
var redacted = redactor.redact(payload);
|
|
188
|
+
// Audit-log the inbound command BEFORE invoking handlers — even handler
|
|
189
|
+
// exceptions don't lose the receipt.
|
|
190
|
+
audit().safeEmit({
|
|
191
|
+
actor: opts.actor || {},
|
|
192
|
+
action: "system.log.incoming",
|
|
193
|
+
metadata: { payload: redacted, source: opts.source || null },
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
var results = [];
|
|
197
|
+
for (var i = 0; i < incomingHandlers.length; i++) {
|
|
198
|
+
try {
|
|
199
|
+
results.push({ ok: true, value: await incomingHandlers[i](payload, opts) });
|
|
200
|
+
} catch (e) {
|
|
201
|
+
results.push({ ok: false, error: (e && e.message) || String(e) });
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
return results;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
async function shutdown() {
|
|
208
|
+
if (!initialized) return;
|
|
209
|
+
// Drain any in-flight emits before closing sink fds so records
|
|
210
|
+
// queued just before shutdown actually reach disk / the wire.
|
|
211
|
+
if (_inflight.size > 0) {
|
|
212
|
+
try { await Promise.all(Array.from(_inflight)); }
|
|
213
|
+
catch (_e) { /* sink errors already audited via the catch above */ }
|
|
214
|
+
}
|
|
215
|
+
for (var name in sinks) {
|
|
216
|
+
try {
|
|
217
|
+
if (typeof sinks[name].raw.close === "function") await sinks[name].raw.close();
|
|
218
|
+
} catch (_e) { /* best effort on shutdown */ }
|
|
219
|
+
}
|
|
220
|
+
sinks = {};
|
|
221
|
+
incomingHandlers = [];
|
|
222
|
+
initialized = false;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function listSinks() {
|
|
226
|
+
if (!initialized) return [];
|
|
227
|
+
return Object.keys(sinks).map(function (name) {
|
|
228
|
+
var s = sinks[name];
|
|
229
|
+
var stats = (typeof s.raw.stats === "function") ? s.raw.stats() : null;
|
|
230
|
+
return { name: name, protocol: s.protocol, stats: stats };
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
// ---- bootFromEnv ----
|
|
235
|
+
//
|
|
236
|
+
// Operator-friendly env-driven init that mirrors b.network.bootFromEnv.
|
|
237
|
+
// Reads BLAMEJS_LOG_STREAM_* env vars and constructs a single-sink
|
|
238
|
+
// configuration matching the operator's choice. Skipped silently when
|
|
239
|
+
// BLAMEJS_LOG_STREAM_PROTOCOL isn't set (operators using the in-code
|
|
240
|
+
// init() path keep their existing wiring).
|
|
241
|
+
//
|
|
242
|
+
// Recognised env vars:
|
|
243
|
+
// BLAMEJS_LOG_STREAM_PROTOCOL "local" | "webhook" | "otlp" | "cloudwatch"
|
|
244
|
+
// BLAMEJS_LOG_STREAM_MIN_LEVEL "debug" | "info" | "warn" | "error"
|
|
245
|
+
//
|
|
246
|
+
// webhook + otlp shared:
|
|
247
|
+
// BLAMEJS_LOG_STREAM_URL
|
|
248
|
+
// BLAMEJS_LOG_STREAM_TOKEN (auth: bearer)
|
|
249
|
+
// otlp-only:
|
|
250
|
+
// BLAMEJS_LOG_STREAM_SERVICE_NAME
|
|
251
|
+
// BLAMEJS_LOG_STREAM_SERVICE_VERSION
|
|
252
|
+
// cloudwatch-only (AWS_* are standard):
|
|
253
|
+
// AWS_REGION, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN
|
|
254
|
+
// BLAMEJS_LOG_STREAM_CLOUDWATCH_LOG_GROUP
|
|
255
|
+
// BLAMEJS_LOG_STREAM_CLOUDWATCH_LOG_STREAM
|
|
256
|
+
// local-only:
|
|
257
|
+
// BLAMEJS_LOG_STREAM_PATH
|
|
258
|
+
function bootFromEnv(opts) {
|
|
259
|
+
opts = opts || {};
|
|
260
|
+
var env = opts.env || process.env;
|
|
261
|
+
var proto = env.BLAMEJS_LOG_STREAM_PROTOCOL;
|
|
262
|
+
if (!proto) return false;
|
|
263
|
+
var sink = { protocol: proto };
|
|
264
|
+
if (proto === "webhook") {
|
|
265
|
+
sink.url = env.BLAMEJS_LOG_STREAM_URL;
|
|
266
|
+
if (env.BLAMEJS_LOG_STREAM_TOKEN) {
|
|
267
|
+
sink.auth = "bearer";
|
|
268
|
+
sink.token = env.BLAMEJS_LOG_STREAM_TOKEN;
|
|
269
|
+
}
|
|
270
|
+
} else if (proto === "otlp") {
|
|
271
|
+
sink.url = env.BLAMEJS_LOG_STREAM_URL;
|
|
272
|
+
sink.serviceName = env.BLAMEJS_LOG_STREAM_SERVICE_NAME || "blamejs";
|
|
273
|
+
sink.serviceVersion = env.BLAMEJS_LOG_STREAM_SERVICE_VERSION || null;
|
|
274
|
+
if (env.BLAMEJS_LOG_STREAM_TOKEN) {
|
|
275
|
+
sink.auth = "bearer";
|
|
276
|
+
sink.token = env.BLAMEJS_LOG_STREAM_TOKEN;
|
|
277
|
+
}
|
|
278
|
+
} else if (proto === "cloudwatch") {
|
|
279
|
+
sink.region = env.AWS_REGION;
|
|
280
|
+
sink.accessKeyId = env.AWS_ACCESS_KEY_ID;
|
|
281
|
+
sink.secretAccessKey = env.AWS_SECRET_ACCESS_KEY;
|
|
282
|
+
sink.sessionToken = env.AWS_SESSION_TOKEN || null;
|
|
283
|
+
sink.logGroupName = env.BLAMEJS_LOG_STREAM_CLOUDWATCH_LOG_GROUP;
|
|
284
|
+
sink.logStreamName = env.BLAMEJS_LOG_STREAM_CLOUDWATCH_LOG_STREAM;
|
|
285
|
+
} else if (proto === "local") {
|
|
286
|
+
sink.path = env.BLAMEJS_LOG_STREAM_PATH;
|
|
287
|
+
} else {
|
|
288
|
+
throw _err("BAD_OPT",
|
|
289
|
+
"BLAMEJS_LOG_STREAM_PROTOCOL='" + proto + "' is not one of " +
|
|
290
|
+
"local | webhook | otlp | cloudwatch (or a custom backend wired via init())");
|
|
291
|
+
}
|
|
292
|
+
init({
|
|
293
|
+
sinks: { primary: sink },
|
|
294
|
+
minLevel: env.BLAMEJS_LOG_STREAM_MIN_LEVEL || undefined,
|
|
295
|
+
});
|
|
296
|
+
return true;
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
function _resetForTest() {
|
|
300
|
+
Object.keys(sinks).forEach(function (n) {
|
|
301
|
+
try { if (sinks[n].raw.close) sinks[n].raw.close(); }
|
|
302
|
+
catch (e) {
|
|
303
|
+
// Test-reset path; best-effort — surface for diagnosability via
|
|
304
|
+
// the framework's boot logger so flaky teardown doesn't go silent.
|
|
305
|
+
_log.debug("reset-close-failed: " + (e && e.message || e));
|
|
306
|
+
}
|
|
307
|
+
});
|
|
308
|
+
sinks = {};
|
|
309
|
+
incomingHandlers = [];
|
|
310
|
+
initialized = false;
|
|
311
|
+
audit.reset();
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
module.exports = {
|
|
315
|
+
init: init,
|
|
316
|
+
bootFromEnv: bootFromEnv,
|
|
317
|
+
emit: emit,
|
|
318
|
+
debug: debug,
|
|
319
|
+
info: info,
|
|
320
|
+
warn: warn,
|
|
321
|
+
error: error,
|
|
322
|
+
onIncoming: onIncoming,
|
|
323
|
+
deliverIncoming: deliverIncoming,
|
|
324
|
+
shutdown: shutdown,
|
|
325
|
+
listSinks: listSinks,
|
|
326
|
+
LEVELS: LEVELS,
|
|
327
|
+
PROTOCOLS: dispatcher.protocols,
|
|
328
|
+
DEFERRED_PROTOCOLS: dispatcher.deferred,
|
|
329
|
+
_resetForTest: _resetForTest,
|
|
330
|
+
};
|