@thotischner/observability-mcp 3.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/analysis/history.d.ts +36 -2
- package/dist/analysis/history.js +60 -2
- package/dist/analysis/history.test.js +46 -0
- package/dist/audit/sinks/s3.d.ts +61 -0
- package/dist/audit/sinks/s3.js +179 -0
- package/dist/audit/sinks/s3.test.d.ts +1 -0
- package/dist/audit/sinks/s3.test.js +175 -0
- package/dist/auth/csrf.d.ts +6 -0
- package/dist/auth/csrf.js +4 -0
- package/dist/auth/csrf.test.js +22 -0
- package/dist/auth/lockout.d.ts +72 -0
- package/dist/auth/lockout.js +134 -0
- package/dist/auth/lockout.test.d.ts +1 -0
- package/dist/auth/lockout.test.js +133 -0
- package/dist/auth/middleware.d.ts +5 -0
- package/dist/auth/middleware.js +6 -1
- package/dist/auth/middleware.test.js +31 -0
- package/dist/auth/password-policy.d.ts +52 -0
- package/dist/auth/password-policy.js +125 -0
- package/dist/auth/password-policy.test.d.ts +1 -0
- package/dist/auth/password-policy.test.js +111 -0
- package/dist/auth/policy/batch-dry-run.js +15 -0
- package/dist/auth/revocation.d.ts +93 -0
- package/dist/auth/revocation.js +193 -0
- package/dist/auth/revocation.test.d.ts +1 -0
- package/dist/auth/revocation.test.js +136 -0
- package/dist/auth/session.d.ts +7 -0
- package/dist/auth/session.js +6 -0
- package/dist/auth/session.test.js +21 -0
- package/dist/connectors/interface.d.ts +5 -1
- package/dist/connectors/loader.d.ts +8 -0
- package/dist/connectors/loader.js +49 -0
- package/dist/connectors/loki.d.ts +45 -1
- package/dist/connectors/loki.js +141 -8
- package/dist/connectors/loki.test.js +171 -1
- package/dist/connectors/manifest-hooks.test.d.ts +1 -0
- package/dist/connectors/manifest-hooks.test.js +206 -0
- package/dist/federation/registry.d.ts +27 -5
- package/dist/federation/registry.js +49 -4
- package/dist/federation/registry.test.js +79 -3
- package/dist/federation/upstream.d.ts +32 -6
- package/dist/federation/upstream.js +60 -12
- package/dist/federation/upstream.test.d.ts +1 -0
- package/dist/federation/upstream.test.js +118 -0
- package/dist/index.js +522 -67
- package/dist/metrics/self.d.ts +1 -0
- package/dist/metrics/self.js +8 -0
- package/dist/openapi.js +39 -0
- package/dist/openapi.test.js +1 -0
- package/dist/policy/redact.js +1 -1
- package/dist/postmortem/store.d.ts +34 -0
- package/dist/postmortem/store.js +113 -0
- package/dist/postmortem/store.test.d.ts +1 -0
- package/dist/postmortem/store.test.js +118 -0
- package/dist/scim/compliance.test.d.ts +1 -0
- package/dist/scim/compliance.test.js +169 -0
- package/dist/scim/factory.test.d.ts +1 -0
- package/dist/scim/factory.test.js +54 -0
- package/dist/scim/patch-ops.test.d.ts +1 -0
- package/dist/scim/patch-ops.test.js +100 -0
- package/dist/scim/redis-store.d.ts +38 -0
- package/dist/scim/redis-store.js +178 -0
- package/dist/scim/redis-store.test.d.ts +1 -0
- package/dist/scim/redis-store.test.js +138 -0
- package/dist/scim/routes.d.ts +27 -2
- package/dist/scim/routes.js +161 -15
- package/dist/scim/store.d.ts +40 -1
- package/dist/scim/store.js +23 -5
- package/dist/sdk/hook-wrappers.d.ts +39 -0
- package/dist/sdk/hook-wrappers.js +113 -0
- package/dist/sdk/hook-wrappers.test.d.ts +1 -0
- package/dist/sdk/hook-wrappers.test.js +204 -0
- package/dist/sdk/index.d.ts +13 -0
- package/dist/security/csp.d.ts +64 -0
- package/dist/security/csp.js +135 -0
- package/dist/security/csp.test.d.ts +1 -0
- package/dist/security/csp.test.js +97 -0
- package/dist/tools/detect-anomalies.d.ts +12 -1
- package/dist/tools/detect-anomalies.js +22 -2
- package/dist/tools/query-logs.d.ts +40 -0
- package/dist/tools/query-logs.js +69 -3
- package/dist/tools/topology.js +23 -5
- package/dist/tools/topology.test.js +45 -0
- package/dist/tools/validation.d.ts +13 -0
- package/dist/tools/validation.js +74 -0
- package/dist/tools/validation.test.js +54 -1
- package/dist/transport/transportSessionMap.d.ts +70 -0
- package/dist/transport/transportSessionMap.js +128 -0
- package/dist/transport/transportSessionMap.test.d.ts +1 -0
- package/dist/transport/transportSessionMap.test.js +111 -0
- package/dist/types.d.ts +48 -0
- package/dist/ui/index.html +898 -116
- package/package.json +1 -1
|
@@ -27,6 +27,14 @@ export interface AnomalyHistoryConfig {
|
|
|
27
27
|
requestTimeoutMs?: number;
|
|
28
28
|
/** Inject fetch for tests. */
|
|
29
29
|
fetchImpl?: typeof fetch;
|
|
30
|
+
/** In-process ring retention window (ms). Default 3 600 000 (1h).
|
|
31
|
+
* Powers the Health-tab sparkline; independent of remote-write. */
|
|
32
|
+
retentionMs?: number;
|
|
33
|
+
/** Hard cap on ring entries (defends memory under a score storm).
|
|
34
|
+
* Default 5 000. */
|
|
35
|
+
ringMax?: number;
|
|
36
|
+
/** Clock injection for tests. Returns epoch ms. */
|
|
37
|
+
now?: () => number;
|
|
30
38
|
}
|
|
31
39
|
export declare function fromEnv(env?: NodeJS.ProcessEnv): AnomalyHistoryConfig;
|
|
32
40
|
/**
|
|
@@ -42,7 +50,14 @@ export declare class AnomalyHistory {
|
|
|
42
50
|
private readonly maxBufferSize;
|
|
43
51
|
private readonly requestTimeoutMs;
|
|
44
52
|
private readonly fetchImpl;
|
|
53
|
+
private readonly retentionMs;
|
|
54
|
+
private readonly ringMax;
|
|
55
|
+
private readonly nowFn;
|
|
45
56
|
private buffer;
|
|
57
|
+
/** Bounded, time-windowed in-process tier of the sink — powers the
|
|
58
|
+
* Health-tab sparkline. Captured on every record() regardless of
|
|
59
|
+
* remote-write so the sparkline works out of the box. */
|
|
60
|
+
private ring;
|
|
46
61
|
private timer?;
|
|
47
62
|
private flushing;
|
|
48
63
|
constructor(cfg?: AnomalyHistoryConfig);
|
|
@@ -52,9 +67,28 @@ export declare class AnomalyHistory {
|
|
|
52
67
|
start(): void;
|
|
53
68
|
/** Stop the flush timer + flush one last time. */
|
|
54
69
|
stop(): Promise<void>;
|
|
55
|
-
/** Add one anomaly
|
|
56
|
-
*
|
|
70
|
+
/** Add one anomaly. Always captured into the in-process ring (powers
|
|
71
|
+
* the sparkline); additionally buffered for remote-write only when the
|
|
72
|
+
* sink is enabled. Triggers a synchronous flush past maxBufferSize. */
|
|
57
73
|
record(entry: AnomalyRecord): Promise<void>;
|
|
74
|
+
/** Append to the ring + evict anything outside the retention window or
|
|
75
|
+
* beyond the hard cap. O(n) prune is fine — anomaly records are
|
|
76
|
+
* infrequent and n is bounded by ringMax. */
|
|
77
|
+
private pushRing;
|
|
78
|
+
/**
|
|
79
|
+
* Recent records from the in-process ring, oldest-first. Filters by
|
|
80
|
+
* service and/or tenant when supplied and drops anything older than the
|
|
81
|
+
* retention window. Returns a fresh array (safe for the caller to map).
|
|
82
|
+
*/
|
|
83
|
+
recent(opts?: {
|
|
84
|
+
service?: string;
|
|
85
|
+
tenant?: string;
|
|
86
|
+
}): AnomalyRecord[];
|
|
87
|
+
/** Distinct services with at least one record in the retention window,
|
|
88
|
+
* optionally tenant-scoped. */
|
|
89
|
+
recentServices(tenant?: string): string[];
|
|
90
|
+
/** Retention window in ms — surfaced so the API/UI can label the range. */
|
|
91
|
+
get windowMs(): number;
|
|
58
92
|
/** Send the current buffer to the remote-write endpoint. Drops the
|
|
59
93
|
* buffer on success OR failure — history is best-effort. */
|
|
60
94
|
flush(): Promise<void>;
|
package/dist/analysis/history.js
CHANGED
|
@@ -45,7 +45,14 @@ export class AnomalyHistory {
|
|
|
45
45
|
maxBufferSize;
|
|
46
46
|
requestTimeoutMs;
|
|
47
47
|
fetchImpl;
|
|
48
|
+
retentionMs;
|
|
49
|
+
ringMax;
|
|
50
|
+
nowFn;
|
|
48
51
|
buffer = [];
|
|
52
|
+
/** Bounded, time-windowed in-process tier of the sink — powers the
|
|
53
|
+
* Health-tab sparkline. Captured on every record() regardless of
|
|
54
|
+
* remote-write so the sparkline works out of the box. */
|
|
55
|
+
ring = [];
|
|
49
56
|
timer;
|
|
50
57
|
flushing = false;
|
|
51
58
|
constructor(cfg = {}) {
|
|
@@ -56,6 +63,9 @@ export class AnomalyHistory {
|
|
|
56
63
|
this.maxBufferSize = cfg.maxBufferSize ?? 500;
|
|
57
64
|
this.requestTimeoutMs = cfg.requestTimeoutMs ?? 5_000;
|
|
58
65
|
this.fetchImpl = cfg.fetchImpl ?? fetch;
|
|
66
|
+
this.retentionMs = cfg.retentionMs ?? 60 * 60 * 1000;
|
|
67
|
+
this.ringMax = cfg.ringMax ?? 5_000;
|
|
68
|
+
this.nowFn = cfg.now ?? Date.now;
|
|
59
69
|
}
|
|
60
70
|
/** Whether the history sink is enabled (URL configured). */
|
|
61
71
|
isEnabled() {
|
|
@@ -84,9 +94,11 @@ export class AnomalyHistory {
|
|
|
84
94
|
if (this.isEnabled())
|
|
85
95
|
await this.flush().catch(() => undefined);
|
|
86
96
|
}
|
|
87
|
-
/** Add one anomaly
|
|
88
|
-
*
|
|
97
|
+
/** Add one anomaly. Always captured into the in-process ring (powers
|
|
98
|
+
* the sparkline); additionally buffered for remote-write only when the
|
|
99
|
+
* sink is enabled. Triggers a synchronous flush past maxBufferSize. */
|
|
89
100
|
async record(entry) {
|
|
101
|
+
this.pushRing(entry);
|
|
90
102
|
if (!this.isEnabled())
|
|
91
103
|
return;
|
|
92
104
|
this.buffer.push(entry);
|
|
@@ -94,6 +106,52 @@ export class AnomalyHistory {
|
|
|
94
106
|
await this.flush().catch(() => undefined);
|
|
95
107
|
}
|
|
96
108
|
}
|
|
109
|
+
/** Append to the ring + evict anything outside the retention window or
|
|
110
|
+
* beyond the hard cap. O(n) prune is fine — anomaly records are
|
|
111
|
+
* infrequent and n is bounded by ringMax. */
|
|
112
|
+
pushRing(entry) {
|
|
113
|
+
this.ring.push(entry);
|
|
114
|
+
const cutoff = this.nowFn() - this.retentionMs;
|
|
115
|
+
this.ring = this.ring.filter((r) => {
|
|
116
|
+
const t = Date.parse(r.ts);
|
|
117
|
+
return Number.isFinite(t) && t >= cutoff;
|
|
118
|
+
});
|
|
119
|
+
if (this.ring.length > this.ringMax) {
|
|
120
|
+
this.ring = this.ring.slice(this.ring.length - this.ringMax);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Recent records from the in-process ring, oldest-first. Filters by
|
|
125
|
+
* service and/or tenant when supplied and drops anything older than the
|
|
126
|
+
* retention window. Returns a fresh array (safe for the caller to map).
|
|
127
|
+
*/
|
|
128
|
+
recent(opts = {}) {
|
|
129
|
+
const cutoff = this.nowFn() - this.retentionMs;
|
|
130
|
+
return this.ring
|
|
131
|
+
.filter((r) => {
|
|
132
|
+
const t = Date.parse(r.ts);
|
|
133
|
+
if (!Number.isFinite(t) || t < cutoff)
|
|
134
|
+
return false;
|
|
135
|
+
if (opts.service && r.service !== opts.service)
|
|
136
|
+
return false;
|
|
137
|
+
if (opts.tenant && r.tenant !== opts.tenant)
|
|
138
|
+
return false;
|
|
139
|
+
return true;
|
|
140
|
+
})
|
|
141
|
+
.sort((a, b) => Date.parse(a.ts) - Date.parse(b.ts));
|
|
142
|
+
}
|
|
143
|
+
/** Distinct services with at least one record in the retention window,
|
|
144
|
+
* optionally tenant-scoped. */
|
|
145
|
+
recentServices(tenant) {
|
|
146
|
+
const seen = new Set();
|
|
147
|
+
for (const r of this.recent({ tenant }))
|
|
148
|
+
seen.add(r.service);
|
|
149
|
+
return [...seen];
|
|
150
|
+
}
|
|
151
|
+
/** Retention window in ms — surfaced so the API/UI can label the range. */
|
|
152
|
+
get windowMs() {
|
|
153
|
+
return this.retentionMs;
|
|
154
|
+
}
|
|
97
155
|
/** Send the current buffer to the remote-write endpoint. Drops the
|
|
98
156
|
* buffer on success OR failure — history is best-effort. */
|
|
99
157
|
async flush() {
|
|
@@ -71,6 +71,52 @@ test("flush: bearer token forwarded as Authorization header", async () => {
|
|
|
71
71
|
await h.flush();
|
|
72
72
|
assert.equal(captured.headers["authorization"], "Bearer tok-abc");
|
|
73
73
|
});
|
|
74
|
+
// --- Q21: in-process ring (Health-tab sparkline) ----------------------
|
|
75
|
+
const NOW = Date.parse("2026-06-06T12:00:00.000Z");
|
|
76
|
+
function at(msAgo) {
|
|
77
|
+
return new Date(NOW - msAgo).toISOString();
|
|
78
|
+
}
|
|
79
|
+
test("ring captures records even when remote-write is disabled", async () => {
|
|
80
|
+
const h = new AnomalyHistory({ now: () => NOW });
|
|
81
|
+
assert.equal(h.isEnabled(), false);
|
|
82
|
+
await h.record(entry({ ts: at(0), score: 0.5 }));
|
|
83
|
+
const recent = h.recent();
|
|
84
|
+
assert.equal(recent.length, 1);
|
|
85
|
+
assert.equal(recent[0].score, 0.5);
|
|
86
|
+
// ...but the remote-write buffer stays empty when disabled.
|
|
87
|
+
assert.equal(h.bufferSize(), 0);
|
|
88
|
+
});
|
|
89
|
+
test("recent: drops records outside the retention window", async () => {
|
|
90
|
+
const h = new AnomalyHistory({ now: () => NOW, retentionMs: 60 * 60 * 1000 });
|
|
91
|
+
await h.record(entry({ ts: at(30 * 60 * 1000) })); // 30m ago — kept
|
|
92
|
+
await h.record(entry({ ts: at(90 * 60 * 1000) })); // 90m ago — evicted on next push/prune
|
|
93
|
+
const recent = h.recent();
|
|
94
|
+
assert.equal(recent.length, 1);
|
|
95
|
+
assert.equal(recent[0].ts, at(30 * 60 * 1000));
|
|
96
|
+
});
|
|
97
|
+
test("recent: oldest-first and filterable by service + tenant", async () => {
|
|
98
|
+
const h = new AnomalyHistory({ now: () => NOW });
|
|
99
|
+
await h.record(entry({ ts: at(3000), service: "payment", tenant: "a", score: 0.1 }));
|
|
100
|
+
await h.record(entry({ ts: at(1000), service: "payment", tenant: "a", score: 0.3 }));
|
|
101
|
+
await h.record(entry({ ts: at(2000), service: "orders", tenant: "a", score: 0.2 }));
|
|
102
|
+
await h.record(entry({ ts: at(500), service: "payment", tenant: "b", score: 0.9 }));
|
|
103
|
+
const pay = h.recent({ service: "payment", tenant: "a" });
|
|
104
|
+
assert.deepEqual(pay.map((r) => r.score), [0.1, 0.3], "oldest-first, service+tenant filtered");
|
|
105
|
+
const tenantA = h.recentServices("a").sort();
|
|
106
|
+
assert.deepEqual(tenantA, ["orders", "payment"]);
|
|
107
|
+
assert.deepEqual(h.recentServices("b"), ["payment"]);
|
|
108
|
+
});
|
|
109
|
+
test("ring honours the hard cap (ringMax)", async () => {
|
|
110
|
+
const h = new AnomalyHistory({ now: () => NOW, ringMax: 3 });
|
|
111
|
+
for (let i = 0; i < 10; i++)
|
|
112
|
+
await h.record(entry({ ts: at(10_000 - i), score: i / 10 }));
|
|
113
|
+
const recent = h.recent();
|
|
114
|
+
assert.equal(recent.length, 3, "ring capped at ringMax");
|
|
115
|
+
});
|
|
116
|
+
test("windowMs surfaces the retention window", () => {
|
|
117
|
+
assert.equal(new AnomalyHistory({ retentionMs: 1234 }).windowMs, 1234);
|
|
118
|
+
assert.equal(new AnomalyHistory({}).windowMs, 60 * 60 * 1000);
|
|
119
|
+
});
|
|
74
120
|
test("flush: clears the buffer on success", async () => {
|
|
75
121
|
const h = new AnomalyHistory({
|
|
76
122
|
url: "https://tsdb/api/v1/write",
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import type { AuditEntry } from "../log.js";
|
|
2
|
+
import type { AuditSink } from "./types.js";
|
|
3
|
+
/** Minimal subset of the AWS SDK S3Client we depend on. */
|
|
4
|
+
export interface S3ClientLike {
|
|
5
|
+
send(command: unknown): Promise<unknown>;
|
|
6
|
+
}
|
|
7
|
+
export interface S3SinkOptions {
|
|
8
|
+
/** Target bucket. Required. */
|
|
9
|
+
bucket: string;
|
|
10
|
+
/** Region — required for AWS, ignored by MinIO. Default us-east-1. */
|
|
11
|
+
region?: string;
|
|
12
|
+
/**
|
|
13
|
+
* Object key prefix. Default empty. The final key is
|
|
14
|
+
* `<prefix>/YYYY/MM/DD/HH/<minute>-<seqStart>-<seqEnd>.jsonl`.
|
|
15
|
+
* Trailing slash optional.
|
|
16
|
+
*/
|
|
17
|
+
prefix?: string;
|
|
18
|
+
/**
|
|
19
|
+
* Override the AWS endpoint URL for S3-compatible backends
|
|
20
|
+
* (MinIO, R2, B2). Empty = use AWS regional endpoint.
|
|
21
|
+
*/
|
|
22
|
+
endpoint?: string;
|
|
23
|
+
/** Force path-style addressing — required for MinIO + B2. */
|
|
24
|
+
forcePathStyle?: boolean;
|
|
25
|
+
/** Flush every N milliseconds. Default 60000 (1 minute). */
|
|
26
|
+
flushIntervalMs?: number;
|
|
27
|
+
/** Flush when buffer holds N entries. Default 1000. */
|
|
28
|
+
maxBufferSize?: number;
|
|
29
|
+
/** Optional path for unrecoverable batches. */
|
|
30
|
+
deadLetterFile?: string;
|
|
31
|
+
/** Inject a client for unit tests. */
|
|
32
|
+
client?: S3ClientLike;
|
|
33
|
+
}
|
|
34
|
+
export declare class S3Sink implements AuditSink {
|
|
35
|
+
readonly name = "s3";
|
|
36
|
+
private readonly bucket;
|
|
37
|
+
private readonly region;
|
|
38
|
+
private readonly prefix;
|
|
39
|
+
private readonly endpoint?;
|
|
40
|
+
private readonly forcePathStyle;
|
|
41
|
+
private readonly flushIntervalMs;
|
|
42
|
+
private readonly maxBufferSize;
|
|
43
|
+
private readonly deadLetterFile?;
|
|
44
|
+
private client;
|
|
45
|
+
private putCommand;
|
|
46
|
+
private sdkLoadError;
|
|
47
|
+
private buffer;
|
|
48
|
+
private flushTimer;
|
|
49
|
+
private writeQueue;
|
|
50
|
+
constructor(opts: S3SinkOptions);
|
|
51
|
+
/** Resolve the SDK lazily so the gateway still boots if @aws-sdk/client-s3
|
|
52
|
+
* isn't installed. Called on the first flush attempt only. */
|
|
53
|
+
private ensureClient;
|
|
54
|
+
write(entry: AuditEntry): Promise<void>;
|
|
55
|
+
flush(): Promise<void>;
|
|
56
|
+
/** Stop the timer + flush remaining. Called on SIGTERM. */
|
|
57
|
+
shutdown(): Promise<void>;
|
|
58
|
+
private flushNow;
|
|
59
|
+
private buildKey;
|
|
60
|
+
private deadLetter;
|
|
61
|
+
}
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
// S3Sink: ship audit entries to an S3-compatible object store
|
|
2
|
+
// (AWS S3, MinIO, Cloudflare R2, Backblaze B2, Wasabi, …).
|
|
3
|
+
//
|
|
4
|
+
// Buffer audit entries in memory; every flush window (default 60 s)
|
|
5
|
+
// or whenever the buffer crosses the cap (default 1000 entries),
|
|
6
|
+
// concatenate as JSONL and PUT one object under
|
|
7
|
+
// `<prefix>/YYYY/MM/DD/HH/<minute>-<seqStart>-<seqEnd>.jsonl`.
|
|
8
|
+
//
|
|
9
|
+
// Why per-minute rollups, not per-entry? S3 charges per PUT (5x per
|
|
10
|
+
// LIST / 1x per GET). An audit-heavy gateway generates 100s of
|
|
11
|
+
// entries/minute — per-entry PUT is wasteful on cost AND on SDK
|
|
12
|
+
// concurrency. One PUT/min keeps the bill low and still lets the
|
|
13
|
+
// operator scrape a minute-grained timeline in the storage console.
|
|
14
|
+
//
|
|
15
|
+
// Failures dead-letter to a local JSONL so a recovered S3 backend
|
|
16
|
+
// can be replayed by an external tool. The on-disk audit chain stays
|
|
17
|
+
// the authoritative master.
|
|
18
|
+
import { appendFile } from "node:fs/promises";
|
|
19
|
+
let _sdk = null;
|
|
20
|
+
async function loadSdk() {
|
|
21
|
+
if (_sdk)
|
|
22
|
+
return _sdk;
|
|
23
|
+
// @aws-sdk/client-s3 is an optional runtime dependency — operators
|
|
24
|
+
// only pull it in when they enable this sink. The specifier is
|
|
25
|
+
// resolved at runtime so TypeScript doesn't require the types at
|
|
26
|
+
// build time (matches the AWS-connector lazy-load pattern from Q1).
|
|
27
|
+
const specifier = "@aws-sdk/client-s3";
|
|
28
|
+
const s3 = (await import(specifier));
|
|
29
|
+
_sdk = { S3Client: s3.S3Client, PutObjectCommand: s3.PutObjectCommand };
|
|
30
|
+
return _sdk;
|
|
31
|
+
}
|
|
32
|
+
export class S3Sink {
|
|
33
|
+
name = "s3";
|
|
34
|
+
bucket;
|
|
35
|
+
region;
|
|
36
|
+
prefix;
|
|
37
|
+
endpoint;
|
|
38
|
+
forcePathStyle;
|
|
39
|
+
flushIntervalMs;
|
|
40
|
+
maxBufferSize;
|
|
41
|
+
deadLetterFile;
|
|
42
|
+
client = null;
|
|
43
|
+
putCommand = null;
|
|
44
|
+
sdkLoadError = null;
|
|
45
|
+
buffer = [];
|
|
46
|
+
flushTimer = null;
|
|
47
|
+
writeQueue = Promise.resolve();
|
|
48
|
+
constructor(opts) {
|
|
49
|
+
if (!opts.bucket)
|
|
50
|
+
throw new Error("S3Sink: bucket is required");
|
|
51
|
+
this.bucket = opts.bucket;
|
|
52
|
+
this.region = opts.region ?? process.env.AWS_REGION ?? "us-east-1";
|
|
53
|
+
// Normalise to "prefix/" form (trailing slash) when non-empty so
|
|
54
|
+
// we don't ever build `<prefix>YYYY/...` with a missing slash.
|
|
55
|
+
const rawPrefix = (opts.prefix ?? "").replace(/^\/+|\/+$/g, "");
|
|
56
|
+
this.prefix = rawPrefix ? `${rawPrefix}/` : "";
|
|
57
|
+
this.endpoint = opts.endpoint || undefined;
|
|
58
|
+
this.forcePathStyle = opts.forcePathStyle ?? false;
|
|
59
|
+
this.flushIntervalMs = opts.flushIntervalMs ?? 60_000;
|
|
60
|
+
this.maxBufferSize = opts.maxBufferSize ?? 1_000;
|
|
61
|
+
this.deadLetterFile = opts.deadLetterFile;
|
|
62
|
+
if (opts.client) {
|
|
63
|
+
this.client = opts.client;
|
|
64
|
+
// Tests inject a fake client AND need PutObjectCommand to be
|
|
65
|
+
// construct-able as a plain class. We stub here so the
|
|
66
|
+
// command-pattern API surface still works.
|
|
67
|
+
const Stub = class {
|
|
68
|
+
input;
|
|
69
|
+
constructor(input) { this.input = input; }
|
|
70
|
+
};
|
|
71
|
+
Object.defineProperty(Stub, "name", { value: "PutObjectCommand" });
|
|
72
|
+
this.putCommand = Stub;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
/** Resolve the SDK lazily so the gateway still boots if @aws-sdk/client-s3
|
|
76
|
+
* isn't installed. Called on the first flush attempt only. */
|
|
77
|
+
async ensureClient() {
|
|
78
|
+
if (this.client && this.putCommand)
|
|
79
|
+
return true;
|
|
80
|
+
if (this.sdkLoadError)
|
|
81
|
+
return false;
|
|
82
|
+
try {
|
|
83
|
+
const sdk = await loadSdk();
|
|
84
|
+
const S3Client = sdk.S3Client;
|
|
85
|
+
this.client = new S3Client({
|
|
86
|
+
region: this.region,
|
|
87
|
+
endpoint: this.endpoint,
|
|
88
|
+
forcePathStyle: this.forcePathStyle,
|
|
89
|
+
});
|
|
90
|
+
this.putCommand = sdk.PutObjectCommand;
|
|
91
|
+
return true;
|
|
92
|
+
}
|
|
93
|
+
catch (err) {
|
|
94
|
+
this.sdkLoadError = err instanceof Error ? err.message : String(err);
|
|
95
|
+
console.warn("S3Sink: @aws-sdk/client-s3 not installed (%s) — entries will dead-letter", this.sdkLoadError);
|
|
96
|
+
return false;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
async write(entry) {
|
|
100
|
+
this.buffer.push(entry);
|
|
101
|
+
if (!this.flushTimer && this.flushIntervalMs > 0) {
|
|
102
|
+
this.flushTimer = setInterval(() => { void this.flush(); }, this.flushIntervalMs);
|
|
103
|
+
if (this.flushTimer && typeof this.flushTimer.unref === "function") {
|
|
104
|
+
this.flushTimer.unref();
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
if (this.buffer.length >= this.maxBufferSize) {
|
|
108
|
+
// Buffer cap reached — flush in background, never block record().
|
|
109
|
+
this.writeQueue = this.writeQueue.then(() => this.flushNow());
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
async flush() {
|
|
113
|
+
this.writeQueue = this.writeQueue.then(() => this.flushNow());
|
|
114
|
+
await this.writeQueue;
|
|
115
|
+
}
|
|
116
|
+
/** Stop the timer + flush remaining. Called on SIGTERM. */
|
|
117
|
+
async shutdown() {
|
|
118
|
+
if (this.flushTimer) {
|
|
119
|
+
clearInterval(this.flushTimer);
|
|
120
|
+
this.flushTimer = null;
|
|
121
|
+
}
|
|
122
|
+
await this.flush();
|
|
123
|
+
}
|
|
124
|
+
// --- internals ----------------------------------------------------
|
|
125
|
+
async flushNow() {
|
|
126
|
+
if (this.buffer.length === 0)
|
|
127
|
+
return;
|
|
128
|
+
const batch = this.buffer.splice(0, this.buffer.length);
|
|
129
|
+
const ok = await this.ensureClient();
|
|
130
|
+
if (!ok) {
|
|
131
|
+
await this.deadLetter(batch);
|
|
132
|
+
return;
|
|
133
|
+
}
|
|
134
|
+
const body = batch.map((e) => JSON.stringify(e)).join("\n") + "\n";
|
|
135
|
+
const key = this.buildKey(batch);
|
|
136
|
+
try {
|
|
137
|
+
const cmd = new this.putCommand({
|
|
138
|
+
Bucket: this.bucket,
|
|
139
|
+
Key: key,
|
|
140
|
+
Body: body,
|
|
141
|
+
ContentType: "application/x-ndjson",
|
|
142
|
+
// Server-side encryption when running on AWS. MinIO + R2 + B2
|
|
143
|
+
// ignore the header gracefully.
|
|
144
|
+
ServerSideEncryption: "AES256",
|
|
145
|
+
});
|
|
146
|
+
await this.client.send(cmd);
|
|
147
|
+
}
|
|
148
|
+
catch (err) {
|
|
149
|
+
console.warn("S3Sink: PUT s3://%s/%s failed: %s — batch dead-letters", this.bucket, key, err instanceof Error ? err.message : String(err));
|
|
150
|
+
await this.deadLetter(batch);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
buildKey(batch) {
|
|
154
|
+
// Derive the time bucket from the first entry's timestamp so a
|
|
155
|
+
// late-arriving entry stays grouped with its peers.
|
|
156
|
+
const t = new Date(batch[0]?.ts ?? new Date().toISOString());
|
|
157
|
+
const yyyy = String(t.getUTCFullYear());
|
|
158
|
+
const mm = String(t.getUTCMonth() + 1).padStart(2, "0");
|
|
159
|
+
const dd = String(t.getUTCDate()).padStart(2, "0");
|
|
160
|
+
const hh = String(t.getUTCHours()).padStart(2, "0");
|
|
161
|
+
const mi = String(t.getUTCMinutes()).padStart(2, "0");
|
|
162
|
+
const seqStart = batch[0]?.seq ?? 0;
|
|
163
|
+
const seqEnd = batch[batch.length - 1]?.seq ?? seqStart;
|
|
164
|
+
return `${this.prefix}${yyyy}/${mm}/${dd}/${hh}/${mi}-${seqStart}-${seqEnd}.jsonl`;
|
|
165
|
+
}
|
|
166
|
+
async deadLetter(batch) {
|
|
167
|
+
if (!this.deadLetterFile) {
|
|
168
|
+
console.warn("S3Sink: %d entries dropped (no deadLetterFile configured)", batch.length);
|
|
169
|
+
return;
|
|
170
|
+
}
|
|
171
|
+
try {
|
|
172
|
+
const body = batch.map((e) => JSON.stringify(e)).join("\n") + "\n";
|
|
173
|
+
await appendFile(this.deadLetterFile, body, "utf8");
|
|
174
|
+
}
|
|
175
|
+
catch (err) {
|
|
176
|
+
console.warn("S3Sink: DLQ write failed: %s", err instanceof Error ? err.message : String(err));
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import { test } from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { mkdtempSync, readFileSync } from "node:fs";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { S3Sink } from "./s3.js";
|
|
7
|
+
function entry(seq, ts, action = "write") {
|
|
8
|
+
return {
|
|
9
|
+
seq,
|
|
10
|
+
ts,
|
|
11
|
+
actor: { sub: "alice@example.com", name: "alice" },
|
|
12
|
+
tenant: "default",
|
|
13
|
+
resource: "sources",
|
|
14
|
+
action,
|
|
15
|
+
method: "POST",
|
|
16
|
+
path: `/api/sources/s${seq}`,
|
|
17
|
+
status: 201,
|
|
18
|
+
ip: "10.0.0.7",
|
|
19
|
+
prevHash: "0".repeat(64),
|
|
20
|
+
hash: String(seq).padStart(64, "0"),
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
function fakeClient() {
|
|
24
|
+
const sent = [];
|
|
25
|
+
let throws = false;
|
|
26
|
+
const client = {
|
|
27
|
+
sent,
|
|
28
|
+
setThrow(v) { throws = v; },
|
|
29
|
+
async send(command) {
|
|
30
|
+
if (throws)
|
|
31
|
+
throw new Error("S3 unavailable (test)");
|
|
32
|
+
const input = command.input;
|
|
33
|
+
sent.push({ Bucket: input.Bucket, Key: input.Key, Body: input.Body });
|
|
34
|
+
return { $metadata: { httpStatusCode: 200 } };
|
|
35
|
+
},
|
|
36
|
+
};
|
|
37
|
+
return client;
|
|
38
|
+
}
|
|
39
|
+
test("constructor: bucket is required", () => {
|
|
40
|
+
assert.throws(() => new S3Sink({ bucket: "" }), /bucket is required/);
|
|
41
|
+
});
|
|
42
|
+
test("write+flush: one entry → one PUT with correct key shape", async () => {
|
|
43
|
+
const client = fakeClient();
|
|
44
|
+
const sink = new S3Sink({
|
|
45
|
+
bucket: "audit-bucket",
|
|
46
|
+
prefix: "omcp",
|
|
47
|
+
flushIntervalMs: 0,
|
|
48
|
+
client,
|
|
49
|
+
});
|
|
50
|
+
await sink.write(entry(1, "2026-06-06T15:23:00Z"));
|
|
51
|
+
await sink.flush();
|
|
52
|
+
assert.equal(client.sent.length, 1);
|
|
53
|
+
const put = client.sent[0];
|
|
54
|
+
assert.equal(put.Bucket, "audit-bucket");
|
|
55
|
+
assert.equal(put.Key, "omcp/2026/06/06/15/23-1-1.jsonl");
|
|
56
|
+
assert.equal(put.Body.trim().split("\n").length, 1);
|
|
57
|
+
});
|
|
58
|
+
test("multiple entries are batched into one PUT on flush", async () => {
|
|
59
|
+
const client = fakeClient();
|
|
60
|
+
const sink = new S3Sink({ bucket: "b", flushIntervalMs: 0, client });
|
|
61
|
+
await sink.write(entry(1, "2026-06-06T15:23:00Z"));
|
|
62
|
+
await sink.write(entry(2, "2026-06-06T15:23:01Z"));
|
|
63
|
+
await sink.write(entry(3, "2026-06-06T15:23:02Z"));
|
|
64
|
+
await sink.flush();
|
|
65
|
+
assert.equal(client.sent.length, 1);
|
|
66
|
+
const lines = client.sent[0].Body.trim().split("\n");
|
|
67
|
+
assert.equal(lines.length, 3);
|
|
68
|
+
// Key range reflects the seq span
|
|
69
|
+
assert.match(client.sent[0].Key, /23-1-3\.jsonl$/);
|
|
70
|
+
});
|
|
71
|
+
test("empty buffer flush is a no-op", async () => {
|
|
72
|
+
const client = fakeClient();
|
|
73
|
+
const sink = new S3Sink({ bucket: "b", flushIntervalMs: 0, client });
|
|
74
|
+
await sink.flush();
|
|
75
|
+
assert.equal(client.sent.length, 0);
|
|
76
|
+
});
|
|
77
|
+
test("maxBufferSize triggers an out-of-band flush", async () => {
|
|
78
|
+
const client = fakeClient();
|
|
79
|
+
const sink = new S3Sink({ bucket: "b", flushIntervalMs: 0, maxBufferSize: 2, client });
|
|
80
|
+
await sink.write(entry(1, "2026-06-06T15:23:00Z"));
|
|
81
|
+
await sink.write(entry(2, "2026-06-06T15:23:00Z"));
|
|
82
|
+
// The second write crosses the cap → background flush
|
|
83
|
+
await sink.flush();
|
|
84
|
+
assert.equal(client.sent.length, 1);
|
|
85
|
+
});
|
|
86
|
+
test("PUT failure dead-letters the batch", async () => {
|
|
87
|
+
const client = fakeClient();
|
|
88
|
+
client.setThrow(true);
|
|
89
|
+
const dlq = join(mkdtempSync(join(tmpdir(), "omcp-s3-dlq-")), "dlq.jsonl");
|
|
90
|
+
const sink = new S3Sink({
|
|
91
|
+
bucket: "b",
|
|
92
|
+
flushIntervalMs: 0,
|
|
93
|
+
client,
|
|
94
|
+
deadLetterFile: dlq,
|
|
95
|
+
});
|
|
96
|
+
await sink.write(entry(1, "2026-06-06T15:23:00Z"));
|
|
97
|
+
await sink.write(entry(2, "2026-06-06T15:23:01Z"));
|
|
98
|
+
await sink.flush();
|
|
99
|
+
// No successful PUT
|
|
100
|
+
assert.equal(client.sent.length, 0);
|
|
101
|
+
// DLQ file has both entries
|
|
102
|
+
const lines = readFileSync(dlq, "utf8").trim().split("\n");
|
|
103
|
+
assert.equal(lines.length, 2);
|
|
104
|
+
assert.match(lines[0], /"seq":1/);
|
|
105
|
+
});
|
|
106
|
+
test("missing SDK + no client → dead-letter (no throw)", async () => {
|
|
107
|
+
// Sink with neither an injected client nor the SDK installed.
|
|
108
|
+
// We can't really uninstall the SDK in tests, but constructing the
|
|
109
|
+
// sink without `client` AND without ever populating sdkLoadError
|
|
110
|
+
// exercises the load path. Validate that flush handles the "no
|
|
111
|
+
// client" state by relying on DLQ.
|
|
112
|
+
const dlq = join(mkdtempSync(join(tmpdir(), "omcp-s3-dlq2-")), "dlq.jsonl");
|
|
113
|
+
// Forcibly break the SDK loader by trying an obviously absent bucket
|
|
114
|
+
// path with a fake client whose `send` throws "MissingCredentials" —
|
|
115
|
+
// the deadLetter path catches it.
|
|
116
|
+
const sink = new S3Sink({
|
|
117
|
+
bucket: "b",
|
|
118
|
+
flushIntervalMs: 0,
|
|
119
|
+
deadLetterFile: dlq,
|
|
120
|
+
client: {
|
|
121
|
+
async send() { throw new Error("MissingCredentials"); },
|
|
122
|
+
},
|
|
123
|
+
});
|
|
124
|
+
await sink.write(entry(1, "2026-06-06T15:23:00Z"));
|
|
125
|
+
await sink.flush();
|
|
126
|
+
const lines = readFileSync(dlq, "utf8").trim().split("\n");
|
|
127
|
+
assert.equal(lines.length, 1);
|
|
128
|
+
});
|
|
129
|
+
test("prefix handles both with-trailing and without-trailing slash", async () => {
|
|
130
|
+
const c1 = fakeClient();
|
|
131
|
+
const s1 = new S3Sink({ bucket: "b", prefix: "audits", flushIntervalMs: 0, client: c1 });
|
|
132
|
+
await s1.write(entry(1, "2026-06-06T15:00:00Z"));
|
|
133
|
+
await s1.flush();
|
|
134
|
+
assert.match(c1.sent[0].Key, /^audits\/2026\//);
|
|
135
|
+
const c2 = fakeClient();
|
|
136
|
+
const s2 = new S3Sink({ bucket: "b", prefix: "/audits/", flushIntervalMs: 0, client: c2 });
|
|
137
|
+
await s2.write(entry(1, "2026-06-06T15:00:00Z"));
|
|
138
|
+
await s2.flush();
|
|
139
|
+
assert.match(c2.sent[0].Key, /^audits\/2026\//);
|
|
140
|
+
assert.doesNotMatch(c2.sent[0].Key, /^\/audits/);
|
|
141
|
+
const c3 = fakeClient();
|
|
142
|
+
const s3 = new S3Sink({ bucket: "b", flushIntervalMs: 0, client: c3 });
|
|
143
|
+
await s3.write(entry(1, "2026-06-06T15:00:00Z"));
|
|
144
|
+
await s3.flush();
|
|
145
|
+
// No prefix → key starts directly with YYYY
|
|
146
|
+
assert.match(c3.sent[0].Key, /^2026\//);
|
|
147
|
+
});
|
|
148
|
+
test("flush serialises — concurrent calls don't lose entries", async () => {
|
|
149
|
+
const client = fakeClient();
|
|
150
|
+
const sink = new S3Sink({ bucket: "b", flushIntervalMs: 0, client });
|
|
151
|
+
for (let i = 1; i <= 50; i++)
|
|
152
|
+
await sink.write(entry(i, "2026-06-06T15:00:00Z"));
|
|
153
|
+
// Multiple parallel flushes
|
|
154
|
+
await Promise.all([sink.flush(), sink.flush(), sink.flush()]);
|
|
155
|
+
// All 50 entries land in one (or at most a few) PUTs; total lines = 50
|
|
156
|
+
const allLines = client.sent.flatMap((p) => p.Body.trim().split("\n"));
|
|
157
|
+
assert.equal(allLines.length, 50);
|
|
158
|
+
});
|
|
159
|
+
test("shutdown stops the timer + flushes the remainder", async () => {
|
|
160
|
+
const client = fakeClient();
|
|
161
|
+
const sink = new S3Sink({ bucket: "b", flushIntervalMs: 1_000_000, client });
|
|
162
|
+
await sink.write(entry(1, "2026-06-06T15:00:00Z"));
|
|
163
|
+
await sink.write(entry(2, "2026-06-06T15:00:01Z"));
|
|
164
|
+
await sink.shutdown();
|
|
165
|
+
assert.equal(client.sent.length, 1);
|
|
166
|
+
assert.equal(client.sent[0].Body.trim().split("\n").length, 2);
|
|
167
|
+
});
|
|
168
|
+
test("uses entry's own ts for the key bucket (not wall-clock)", async () => {
|
|
169
|
+
const client = fakeClient();
|
|
170
|
+
const sink = new S3Sink({ bucket: "b", flushIntervalMs: 0, client });
|
|
171
|
+
// Entry from 5 minutes earlier — key must reflect THAT minute.
|
|
172
|
+
await sink.write(entry(1, "2026-06-06T15:18:00Z"));
|
|
173
|
+
await sink.flush();
|
|
174
|
+
assert.match(client.sent[0].Key, /15\/18-1-1\.jsonl$/);
|
|
175
|
+
});
|
package/dist/auth/csrf.d.ts
CHANGED
|
@@ -13,6 +13,12 @@ export interface CsrfConfig {
|
|
|
13
13
|
/** Set cookies with `Secure` flag. Default mirrors the existing
|
|
14
14
|
* session-cookie behaviour: only when the request is on https. */
|
|
15
15
|
secureCookie: (req: Request) => boolean;
|
|
16
|
+
/** Optional predicate to exempt specific requests from CSRF entirely.
|
|
17
|
+
* Used for unauthenticated browser-initiated POSTs that can't carry a
|
|
18
|
+
* token by construction — e.g. CSP violation reports, which the browser
|
|
19
|
+
* sends with no credentials and no custom headers. Keep this list
|
|
20
|
+
* minimal: an exempt endpoint must be safe to accept cross-site. */
|
|
21
|
+
skip?: (req: Request) => boolean;
|
|
16
22
|
}
|
|
17
23
|
export declare function csrfBypassFromEnv(env?: NodeJS.ProcessEnv): boolean;
|
|
18
24
|
/** Issue a fresh token cookie if the request doesn't already carry a
|
package/dist/auth/csrf.js
CHANGED
|
@@ -69,6 +69,10 @@ export function buildCsrfEnforcer(cfg) {
|
|
|
69
69
|
next();
|
|
70
70
|
return;
|
|
71
71
|
}
|
|
72
|
+
if (cfg.skip?.(req)) {
|
|
73
|
+
next();
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
72
76
|
if (cfg.bypassBearer) {
|
|
73
77
|
const auth = req.headers["authorization"];
|
|
74
78
|
if (typeof auth === "string" && /^Bearer\s+/i.test(auth)) {
|