@sleep2agi/commhub-server 0.9.0-preview.0 → 0.9.0-preview.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -2
- package/src/ack-create-request.test.ts +331 -0
- package/src/api-nodes-shape.test.ts +172 -0
- package/src/auth-kdf-migration.test.ts +378 -0
- package/src/auth-tokens.test.ts +18 -9
- package/src/auth.ts +65 -5
- package/src/config-apply-sec1.test.ts +723 -0
- package/src/config-apply-validate.test.ts +260 -0
- package/src/config-apply-validate.ts +171 -0
- package/src/create-node-validate.test.ts +187 -0
- package/src/create-node-validate.ts +206 -0
- package/src/create-node.test.ts +354 -0
- package/src/create-node.ts +346 -0
- package/src/cross-tenant-injection.test.ts +110 -0
- package/src/db.ts +499 -8
- package/src/index.ts +247 -19
- package/src/lifecycle-guard.ts +50 -0
- package/src/list-host-supervisors.test.ts +314 -0
- package/src/probe-validate.test.ts +221 -0
- package/src/probe-validate.ts +104 -0
- package/src/probe.test.ts +80 -0
- package/src/probe.ts +200 -0
- package/src/retention.test.ts +450 -0
- package/src/retention.ts +269 -0
- package/src/shared/probe-host-allowlist-drift.test.ts +54 -0
- package/src/shared/probe-host-allowlist.ts +128 -0
- package/src/shared/reserved-env-drift.test.ts +43 -0
- package/src/shared/reserved-env.test.ts +35 -0
- package/src/shared/reserved-env.ts +31 -0
- package/src/stale-sweeper.test.ts +181 -0
- package/src/stale-sweeper.ts +114 -0
- package/src/stop-delete-node.test.ts +720 -0
- package/src/tools.ts +2285 -64
- package/src/update-provider.test.ts +234 -0
- package/src/vault.test.ts +200 -0
- package/src/vault.ts +156 -0
package/src/retention.ts
ADDED
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
// Round-2/4 review ② — retention sweep + incremental VACUUM.
|
|
2
|
+
//
|
|
3
|
+
// Background maintenance for the commhub database. Multi-tenant
|
|
4
|
+
// public-facing hubs (e.g. <hub-domain>) write to high-frequency tables
|
|
5
|
+
// on every heartbeat (agent_telemetry, task_events, completions) — left
|
|
6
|
+
// unswept these grow unbounded, slow down indexed scans, and bloat the
|
|
7
|
+
// on-disk footprint. SQLite never reclaims freed pages without VACUUM,
|
|
8
|
+
// so a row delete by itself doesn't shrink the file.
|
|
9
|
+
//
|
|
10
|
+
// This module exposes:
|
|
11
|
+
// - sweepRetention(cfg): one-shot sweep across high-growth tables
|
|
12
|
+
// - startRetentionSweeper(): wires a 1-hour setInterval, fires once
|
|
13
|
+
// at startup with an immediate (non-blocking) sweep
|
|
14
|
+
//
|
|
15
|
+
// Defaults are conservative — long-tail debugging traces survive a
|
|
16
|
+
// week. Operators can override per env (COMMHUB_RETENTION_*_DAYS) for
|
|
17
|
+
// hot hubs.
|
|
18
|
+
//
|
|
19
|
+
// Run isolated tests against retention.test.ts:
|
|
20
|
+
// COMMHUB_DB=/tmp/test-retention.db bun test src/retention.test.ts
|
|
21
|
+
|
|
22
|
+
import { db } from "./db.js";
|
|
23
|
+
|
|
24
|
+
export type RetentionConfig = {
|
|
25
|
+
// High-frequency telemetry; default = 7 days.
|
|
26
|
+
// Heartbeat is ~30s → 50 agents × 86400 s/day ÷ 30 s = 144000 rows/day.
|
|
27
|
+
// At 7d that's ~1M rows. Indexed scans stay fast; raw disk = a few MB.
|
|
28
|
+
telemetryDays: number;
|
|
29
|
+
// Task state-change audit; default = 30 days.
|
|
30
|
+
// One row per status transition. Useful for incident forensics.
|
|
31
|
+
taskEventsDays: number;
|
|
32
|
+
// Acked inbox; default = 7 days post-ack.
|
|
33
|
+
// Once acked the row is no longer hot — keep a week for debugging.
|
|
34
|
+
// Unacked rows are NEVER swept (could be in-flight delivery).
|
|
35
|
+
ackedInboxDays: number;
|
|
36
|
+
// Completed/cancelled/failed tasks; default = 30 days.
|
|
37
|
+
// Active tasks (created/delivered/acked/running/replied less than
|
|
38
|
+
// 30 days old) are NEVER swept.
|
|
39
|
+
terminalTasksDays: number;
|
|
40
|
+
// Audit log; default = 90 days.
|
|
41
|
+
// Longer retention because audit_log is a security artifact.
|
|
42
|
+
auditLogDays: number;
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
export type SweepResult = {
|
|
46
|
+
startedAt: string;
|
|
47
|
+
durationMs: number;
|
|
48
|
+
deletes: {
|
|
49
|
+
agent_telemetry: number;
|
|
50
|
+
task_events: number;
|
|
51
|
+
inbox: number;
|
|
52
|
+
tasks: number;
|
|
53
|
+
audit_log: number;
|
|
54
|
+
};
|
|
55
|
+
vacuum: {
|
|
56
|
+
walCheckpointPagesMoved: number | null;
|
|
57
|
+
incrementalFreedPages: number | null;
|
|
58
|
+
errored: boolean;
|
|
59
|
+
};
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
const DEFAULTS: RetentionConfig = {
|
|
63
|
+
telemetryDays: 7,
|
|
64
|
+
taskEventsDays: 30,
|
|
65
|
+
ackedInboxDays: 7,
|
|
66
|
+
terminalTasksDays: 30,
|
|
67
|
+
auditLogDays: 90,
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
function readEnvDays(name: string, fallback: number): number {
|
|
71
|
+
const raw = process.env[name];
|
|
72
|
+
if (raw === undefined || raw === "") return fallback;
|
|
73
|
+
const n = Number(raw);
|
|
74
|
+
// Negative = disable that table's sweep entirely (operator opt-out).
|
|
75
|
+
// NaN / non-finite falls back to default — guard against fat-fingered env.
|
|
76
|
+
if (!Number.isFinite(n)) return fallback;
|
|
77
|
+
return n;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export function readRetentionConfig(): RetentionConfig {
|
|
81
|
+
return {
|
|
82
|
+
telemetryDays: readEnvDays("COMMHUB_RETENTION_TELEMETRY_DAYS", DEFAULTS.telemetryDays),
|
|
83
|
+
taskEventsDays: readEnvDays("COMMHUB_RETENTION_TASK_EVENTS_DAYS", DEFAULTS.taskEventsDays),
|
|
84
|
+
ackedInboxDays: readEnvDays("COMMHUB_RETENTION_INBOX_ACKED_DAYS", DEFAULTS.ackedInboxDays),
|
|
85
|
+
terminalTasksDays: readEnvDays("COMMHUB_RETENTION_TASKS_TERMINAL_DAYS", DEFAULTS.terminalTasksDays),
|
|
86
|
+
auditLogDays: readEnvDays("COMMHUB_RETENTION_AUDIT_DAYS", DEFAULTS.auditLogDays),
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function sweepOne(
|
|
91
|
+
sql: string,
|
|
92
|
+
params: (string | number)[],
|
|
93
|
+
days: number,
|
|
94
|
+
): number {
|
|
95
|
+
// days < 0 → operator opt-out for this table; skip the DELETE.
|
|
96
|
+
// days === 0 is a legitimate "sweep everything older than now"
|
|
97
|
+
// (effectively a TRUNCATE); we honour it.
|
|
98
|
+
if (days < 0) return 0;
|
|
99
|
+
try {
|
|
100
|
+
const res = db.run(sql, params);
|
|
101
|
+
return res.changes ?? 0;
|
|
102
|
+
} catch (e: any) {
|
|
103
|
+
console.log(`[commhub retention] sweep failed: ${e?.message ?? e}`);
|
|
104
|
+
return 0;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
export function sweepRetention(cfg: RetentionConfig = readRetentionConfig()): SweepResult {
|
|
109
|
+
const startedAt = new Date().toISOString();
|
|
110
|
+
const t0 = Date.now();
|
|
111
|
+
|
|
112
|
+
const telemetry = sweepOne(
|
|
113
|
+
`DELETE FROM agent_telemetry WHERE created_at < datetime('now', ?1)`,
|
|
114
|
+
[`-${cfg.telemetryDays} days`],
|
|
115
|
+
cfg.telemetryDays,
|
|
116
|
+
);
|
|
117
|
+
const taskEvents = sweepOne(
|
|
118
|
+
`DELETE FROM task_events WHERE created_at < datetime('now', ?1)`,
|
|
119
|
+
[`-${cfg.taskEventsDays} days`],
|
|
120
|
+
cfg.taskEventsDays,
|
|
121
|
+
);
|
|
122
|
+
// Only sweep ACKED inbox rows — unacked may be in-flight delivery
|
|
123
|
+
// for an agent that's offline but coming back. Acked rows have
|
|
124
|
+
// served their purpose and exist only for debugging.
|
|
125
|
+
const inbox = sweepOne(
|
|
126
|
+
`DELETE FROM inbox WHERE acked = 1 AND created_at < datetime('now', ?1)`,
|
|
127
|
+
[`-${cfg.ackedInboxDays} days`],
|
|
128
|
+
cfg.ackedInboxDays,
|
|
129
|
+
);
|
|
130
|
+
// Terminal-task sweep — split into two DELETEs:
|
|
131
|
+
//
|
|
132
|
+
// (a) "safe terminals": cancelled / failed / expired / (theoretical)
|
|
133
|
+
// completed have no chain-ancestor risk and are safe to reap
|
|
134
|
+
// unconditionally past the horizon. NOTE: as of this writing
|
|
135
|
+
// 'completed' is dead code — no code path sets it (every reply
|
|
136
|
+
// uses 'replied'/'failed'/'cancelled' via send_reply or
|
|
137
|
+
// chainReplyToParent). Kept in the IN-list as a forward guard:
|
|
138
|
+
// if someone introduces a 'completed' write later, the sweep
|
|
139
|
+
// picks it up without code changes.
|
|
140
|
+
//
|
|
141
|
+
// (b) "replied" — was excluded entirely in the original PR, which
|
|
142
|
+
// missed the goal: every fulfilled task lives at 'replied'
|
|
143
|
+
// status, so the tasks table's MAIN GROWTH SOURCE would never
|
|
144
|
+
// be reclaimed. Reviewer caught this. Fix: child-ref-aware
|
|
145
|
+
// delete — reap an old replied row ONLY IF no other task
|
|
146
|
+
// references it as parent_task_id. A row with no children is
|
|
147
|
+
// not a chain ancestor and is safe to drop.
|
|
148
|
+
//
|
|
149
|
+
// **Time field** (both DELETEs): use COALESCE(completed_at, created_at).
|
|
150
|
+
// Terminal status means the server set `completed_at` at the
|
|
151
|
+
// reply/fail/cancel moment. Sweeping on `created_at` alone would
|
|
152
|
+
// reap a task created 60d ago but completed today the instant it
|
|
153
|
+
// entered terminal state — zero post-terminal retention. The
|
|
154
|
+
// COALESCE falls back to created_at for legacy rows that pre-date
|
|
155
|
+
// the completed_at column (back-compat).
|
|
156
|
+
const tasksSafeTerminal = sweepOne(
|
|
157
|
+
`DELETE FROM tasks
|
|
158
|
+
WHERE status IN ('completed', 'cancelled', 'failed', 'expired')
|
|
159
|
+
AND COALESCE(completed_at, created_at) < datetime('now', ?1)`,
|
|
160
|
+
[`-${cfg.terminalTasksDays} days`],
|
|
161
|
+
cfg.terminalTasksDays,
|
|
162
|
+
);
|
|
163
|
+
const tasksRepliedChildSafe = sweepOne(
|
|
164
|
+
`DELETE FROM tasks
|
|
165
|
+
WHERE status = 'replied'
|
|
166
|
+
AND COALESCE(completed_at, created_at) < datetime('now', ?1)
|
|
167
|
+
AND NOT EXISTS (
|
|
168
|
+
SELECT 1 FROM tasks AS child
|
|
169
|
+
WHERE child.parent_task_id = tasks.task_id
|
|
170
|
+
)`,
|
|
171
|
+
[`-${cfg.terminalTasksDays} days`],
|
|
172
|
+
cfg.terminalTasksDays,
|
|
173
|
+
);
|
|
174
|
+
const tasks = tasksSafeTerminal + tasksRepliedChildSafe;
|
|
175
|
+
const auditLog = sweepOne(
|
|
176
|
+
`DELETE FROM audit_log WHERE created_at < datetime('now', ?1)`,
|
|
177
|
+
[`-${cfg.auditLogDays} days`],
|
|
178
|
+
cfg.auditLogDays,
|
|
179
|
+
);
|
|
180
|
+
|
|
181
|
+
// VACUUM:
|
|
182
|
+
// - wal_checkpoint(TRUNCATE): trims the WAL file. Works on every
|
|
183
|
+
// SQLite DB regardless of auto_vacuum setting. Useful after
|
|
184
|
+
// big DELETEs so the WAL doesn't stay bloated until the next
|
|
185
|
+
// natural checkpoint.
|
|
186
|
+
// - incremental_vacuum: ONLY reclaims main-DB pages on databases
|
|
187
|
+
// created with `PRAGMA auto_vacuum = INCREMENTAL`. On legacy
|
|
188
|
+
// DBs (the default mode 0) this is a no-op and the main file
|
|
189
|
+
// does NOT shrink — freed pages stay in the freelist for
|
|
190
|
+
// future inserts. Operators who want disk reclamation on old
|
|
191
|
+
// DBs must run a one-time blocking `VACUUM` themselves (out of
|
|
192
|
+
// scope for the sweeper, which deliberately stays non-
|
|
193
|
+
// blocking).
|
|
194
|
+
let walCheckpointPagesMoved: number | null = null;
|
|
195
|
+
let incrementalFreedPages: number | null = null;
|
|
196
|
+
let errored = false;
|
|
197
|
+
try {
|
|
198
|
+
// wal_checkpoint(TRUNCATE) returns a row { busy, log, checkpointed }.
|
|
199
|
+
// We expose `log` (page count moved into the main DB) for ops visibility.
|
|
200
|
+
const checkpoint = db.get<{ busy: number; log: number; checkpointed: number }>(
|
|
201
|
+
"PRAGMA wal_checkpoint(TRUNCATE)",
|
|
202
|
+
);
|
|
203
|
+
walCheckpointPagesMoved = checkpoint?.log ?? null;
|
|
204
|
+
} catch (e: any) {
|
|
205
|
+
console.log(`[commhub retention] wal_checkpoint failed: ${e?.message ?? e}`);
|
|
206
|
+
errored = true;
|
|
207
|
+
}
|
|
208
|
+
try {
|
|
209
|
+
// incremental_vacuum is a no-op on databases not created with
|
|
210
|
+
// PRAGMA auto_vacuum = INCREMENTAL. For existing prod DBs that
|
|
211
|
+
// weren't, the operator would have to run a full one-time VACUUM
|
|
212
|
+
// to migrate — out of scope for the sweeper.
|
|
213
|
+
const freelistBefore = db.get<{ freelist_count: number }>("PRAGMA freelist_count");
|
|
214
|
+
db.exec("PRAGMA incremental_vacuum");
|
|
215
|
+
const freelistAfter = db.get<{ freelist_count: number }>("PRAGMA freelist_count");
|
|
216
|
+
incrementalFreedPages =
|
|
217
|
+
(freelistBefore?.freelist_count ?? 0) - (freelistAfter?.freelist_count ?? 0);
|
|
218
|
+
} catch (e: any) {
|
|
219
|
+
console.log(`[commhub retention] incremental_vacuum failed: ${e?.message ?? e}`);
|
|
220
|
+
errored = true;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
return {
|
|
224
|
+
startedAt,
|
|
225
|
+
durationMs: Date.now() - t0,
|
|
226
|
+
deletes: {
|
|
227
|
+
agent_telemetry: telemetry,
|
|
228
|
+
task_events: taskEvents,
|
|
229
|
+
inbox,
|
|
230
|
+
tasks,
|
|
231
|
+
audit_log: auditLog,
|
|
232
|
+
},
|
|
233
|
+
vacuum: { walCheckpointPagesMoved, incrementalFreedPages, errored },
|
|
234
|
+
};
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// Used by index.ts to wire the periodic sweeper. Returns the timer so
|
|
238
|
+
// callers can stop it in tests / shutdown.
|
|
239
|
+
export function startRetentionSweeper(
|
|
240
|
+
intervalMs = 60 * 60 * 1000, // 1 hour
|
|
241
|
+
cfg?: RetentionConfig,
|
|
242
|
+
): ReturnType<typeof setInterval> {
|
|
243
|
+
const sweep = () => {
|
|
244
|
+
try {
|
|
245
|
+
const r = sweepRetention(cfg);
|
|
246
|
+
const totalDeletes =
|
|
247
|
+
r.deletes.agent_telemetry +
|
|
248
|
+
r.deletes.task_events +
|
|
249
|
+
r.deletes.inbox +
|
|
250
|
+
r.deletes.tasks +
|
|
251
|
+
r.deletes.audit_log;
|
|
252
|
+
if (totalDeletes > 0 || r.vacuum.errored) {
|
|
253
|
+
console.log(
|
|
254
|
+
`[commhub retention] swept in ${r.durationMs}ms — ` +
|
|
255
|
+
`telemetry=${r.deletes.agent_telemetry} events=${r.deletes.task_events} ` +
|
|
256
|
+
`inbox=${r.deletes.inbox} tasks=${r.deletes.tasks} audit=${r.deletes.audit_log} ` +
|
|
257
|
+
`walPages=${r.vacuum.walCheckpointPagesMoved} freedPages=${r.vacuum.incrementalFreedPages}`,
|
|
258
|
+
);
|
|
259
|
+
}
|
|
260
|
+
} catch (e: any) {
|
|
261
|
+
console.log(`[commhub retention] sweep top-level failed: ${e?.message ?? e}`);
|
|
262
|
+
}
|
|
263
|
+
};
|
|
264
|
+
// Fire once on startup so a long-running hub eventually catches up
|
|
265
|
+
// even if it's restarted just after the previous sweep. setImmediate
|
|
266
|
+
// (vs synchronous) keeps server startup from blocking on the sweep.
|
|
267
|
+
setImmediate(sweep);
|
|
268
|
+
return setInterval(sweep, intervalMs);
|
|
269
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { readFileSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import {
|
|
5
|
+
VENDOR_HOST_ALLOWLIST as HUB_ALLOW,
|
|
6
|
+
FORBIDDEN_IPV4_RE as HUB_V4,
|
|
7
|
+
FORBIDDEN_IPV6_RE as HUB_V6,
|
|
8
|
+
} from "./probe-host-allowlist.js";
|
|
9
|
+
|
|
10
|
+
// RFC-028 P1 fold-in #1 — hub-and-daemon probe-host-allowlist drift
|
|
11
|
+
// guard. Mirrors the G9 reserved-env drift test (RFC-026 §4.4.7).
|
|
12
|
+
// Two enforcement modes:
|
|
13
|
+
// 1. byte-identical source files
|
|
14
|
+
// 2. runtime set-equality of the imported constants (vendor regex
|
|
15
|
+
// source strings + private-IP regex source strings)
|
|
16
|
+
// If a future contributor edits one side without the other, this test
|
|
17
|
+
// breaks the merge — the daemon-side compromised-hub defense depends
|
|
18
|
+
// on the two lists being identical.
|
|
19
|
+
|
|
20
|
+
const HUB_PATH = join(import.meta.dir, "probe-host-allowlist.ts");
|
|
21
|
+
const DAEMON_PATH = join(
|
|
22
|
+
import.meta.dir,
|
|
23
|
+
"..", "..", "..", "agent-node", "src", "shared", "probe-host-allowlist.ts",
|
|
24
|
+
);
|
|
25
|
+
|
|
26
|
+
describe("RFC-028 P1 — hub/daemon probe-host-allowlist drift guard", () => {
|
|
27
|
+
test("byte-identical source files", () => {
|
|
28
|
+
const hub = readFileSync(HUB_PATH, "utf-8");
|
|
29
|
+
const daemon = readFileSync(DAEMON_PATH, "utf-8");
|
|
30
|
+
expect(daemon).toBe(hub);
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
test("VENDOR_HOST_ALLOWLIST regex sources equal at runtime", async () => {
|
|
34
|
+
const daemonMod = await import(DAEMON_PATH);
|
|
35
|
+
const daemonAllow = daemonMod.VENDOR_HOST_ALLOWLIST as Record<string, ReadonlyArray<RegExp>>;
|
|
36
|
+
// vendor set equality
|
|
37
|
+
expect(new Set(Object.keys(daemonAllow))).toEqual(new Set(Object.keys(HUB_ALLOW)));
|
|
38
|
+
// per-vendor regex source array equality (order matters — both should
|
|
39
|
+
// be authored in identical order to keep diffs minimal)
|
|
40
|
+
for (const vendor of Object.keys(HUB_ALLOW)) {
|
|
41
|
+
const hubSources = HUB_ALLOW[vendor].map(r => r.source);
|
|
42
|
+
const daemonSources = daemonAllow[vendor].map(r => r.source);
|
|
43
|
+
expect(daemonSources).toEqual(hubSources);
|
|
44
|
+
}
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
test("FORBIDDEN_IPV4_RE + FORBIDDEN_IPV6_RE sources equal at runtime", async () => {
|
|
48
|
+
const daemonMod = await import(DAEMON_PATH);
|
|
49
|
+
const daemonV4 = (daemonMod.FORBIDDEN_IPV4_RE as ReadonlyArray<RegExp>).map(r => r.source);
|
|
50
|
+
const daemonV6 = (daemonMod.FORBIDDEN_IPV6_RE as ReadonlyArray<RegExp>).map(r => r.source);
|
|
51
|
+
expect(daemonV4).toEqual(HUB_V4.map(r => r.source));
|
|
52
|
+
expect(daemonV6).toEqual(HUB_V6.map(r => r.source));
|
|
53
|
+
});
|
|
54
|
+
});
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
// RFC-028 P1 §4.4 — shared probe SSRF guards.
|
|
2
|
+
// Host allowlist + private IP block + validateBaseUrl.
|
|
3
|
+
//
|
|
4
|
+
// MUST be byte-identical between hub (server/src/shared/) and daemon
|
|
5
|
+
// (agent-node/src/shared/). Enforced by probe-host-allowlist-drift.test.ts
|
|
6
|
+
// (same pattern as reserved-env.ts G9 drift guard, RFC-026 §4.4.7).
|
|
7
|
+
//
|
|
8
|
+
// Why mirror not symlink: hub and daemon are separate npm packages
|
|
9
|
+
// installed independently on different machines. A drift-checked source
|
|
10
|
+
// copy survives `npm pack` + global install; a symlink does not.
|
|
11
|
+
//
|
|
12
|
+
// Threat model the daemon re-check defends against:
|
|
13
|
+
// - Compromised hub records a malicious base_url in providers/spec
|
|
14
|
+
// - Hub bypassed validateBaseUrl (regression / different code path)
|
|
15
|
+
// - On-wire injection between hub→daemon SSE
|
|
16
|
+
// Daemon trusts process.execPath + this file's content, NOT what
|
|
17
|
+
// `get_probe_request` claims. If hub-side check is identical, daemon
|
|
18
|
+
// re-check is a no-op; if hub-side ever loosens, daemon stays strict.
|
|
19
|
+
|
|
20
|
+
export class ProbeValidationError extends Error {
|
|
21
|
+
constructor(public code: string, public detail?: Record<string, unknown>) {
|
|
22
|
+
super(`${code}${detail ? ` ${JSON.stringify(detail)}` : ""}`);
|
|
23
|
+
this.name = "ProbeValidationError";
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// ── §4.4.1 per-vendor host allowlist ──────────────────────────────
|
|
28
|
+
// `anthropic` vendor accepts Anthropic + curated 3rd-party Anthropic-
|
|
29
|
+
// compatible endpoints that Vincent / N站马 ship live providers
|
|
30
|
+
// against (DeepSeek + MiniMax both expose `/v1/messages` with the
|
|
31
|
+
// Anthropic protocol surface). Whitelist semantics preserved — NOT
|
|
32
|
+
// allow-any. New entries land here via PR with security review
|
|
33
|
+
// (network-admin-configurable allowlist is backlog: needs strong
|
|
34
|
+
// IP-range / shape guard before admins can edit).
|
|
35
|
+
//
|
|
36
|
+
// Per 通信龙 (RFC-028 P1.5+): the host whitelist is defense layer 1;
|
|
37
|
+
// layer 2 is the DNS-resolved IP block (FORBIDDEN_IPV{4,6}_RE below),
|
|
38
|
+
// applied AFTER allowlist passes. Any new host gets BOTH layers
|
|
39
|
+
// automatically — host in allowlist + DNS resolves to private/metadata
|
|
40
|
+
// IP still surfaces `probe_resolve_unsafe_ip`.
|
|
41
|
+
//
|
|
42
|
+
// P2 will add vendors (openai/zai/openrouter/qwen) as separate keys.
|
|
43
|
+
export const VENDOR_HOST_ALLOWLIST: Record<string, ReadonlyArray<RegExp>> = {
|
|
44
|
+
anthropic: [
|
|
45
|
+
/^api\.anthropic\.com$/,
|
|
46
|
+
/^api\.deepseek\.com$/, // Anthropic-compatible /v1/messages
|
|
47
|
+
/^api\.minimax\.chat$/, // MiniMax legacy domain
|
|
48
|
+
/^api\.minimax\.io$/, // MiniMax new domain (Vincent live use)
|
|
49
|
+
],
|
|
50
|
+
// P2:
|
|
51
|
+
// openai: [/^api\.openai\.com$/],
|
|
52
|
+
// zai: [/^api\.z\.ai$/, /^open\.bigmodel\.cn$/],
|
|
53
|
+
// openrouter: [/^openrouter\.ai$/],
|
|
54
|
+
// qwen: [/^dashscope(-intl)?\.aliyuncs\.com$/],
|
|
55
|
+
};
|
|
56
|
+
export const SUPPORTED_VENDORS = Object.keys(VENDOR_HOST_ALLOWLIST);
|
|
57
|
+
|
|
58
|
+
// ── §4.4.2 private/reserved IP block (anti SSRF) ──────────────────
|
|
59
|
+
export const FORBIDDEN_IPV4_RE: ReadonlyArray<RegExp> = [
|
|
60
|
+
/^10\./,
|
|
61
|
+
/^127\./,
|
|
62
|
+
/^172\.(1[6-9]|2[0-9]|3[0-1])\./,
|
|
63
|
+
/^192\.168\./,
|
|
64
|
+
/^169\.254\./, // link-local + 169.254.169.254 cloud metadata
|
|
65
|
+
/^100\.(6[4-9]|[7-9][0-9]|1[01][0-9]|12[0-7])\./, // CGNAT
|
|
66
|
+
/^0\./,
|
|
67
|
+
/^22[4-9]\./, /^23[0-9]\./, // multicast
|
|
68
|
+
/^24[0-9]\./, /^25[0-5]\./, // experimental
|
|
69
|
+
];
|
|
70
|
+
export const FORBIDDEN_IPV6_RE: ReadonlyArray<RegExp> = [
|
|
71
|
+
/^::1$/,
|
|
72
|
+
/^::$/,
|
|
73
|
+
/^fe80:/i, /^fc00:/i, /^fd00:/i,
|
|
74
|
+
];
|
|
75
|
+
|
|
76
|
+
export function isLoopbackHost(host: string): boolean {
|
|
77
|
+
return host === "localhost" || host === "127.0.0.1" || host === "::1";
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export function isForbiddenIp(ip: string): boolean {
|
|
81
|
+
// IPv4-mapped IPv6: ::ffff:10.0.0.1 — recurse on inner v4
|
|
82
|
+
if (ip.toLowerCase().startsWith("::ffff:")) {
|
|
83
|
+
return isForbiddenIp(ip.slice(7));
|
|
84
|
+
}
|
|
85
|
+
// Simple v4/v6 distinction by presence of '.' / ':'
|
|
86
|
+
if (ip.includes(":") && !ip.includes(".")) {
|
|
87
|
+
return FORBIDDEN_IPV6_RE.some(re => re.test(ip));
|
|
88
|
+
}
|
|
89
|
+
return FORBIDDEN_IPV4_RE.some(re => re.test(ip));
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// ── §4.4.3 validateBaseUrl — boot/pre-fetch URL+host+vendor check ─
|
|
93
|
+
/** Pure, no I/O. Called by hub (upsert_provider write path) AND
|
|
94
|
+
* daemon (handleProbeDoorbell re-check after get_probe_request).
|
|
95
|
+
* allowLoopback: dev-only opt-in honoring http://localhost / 127.0.0.1.
|
|
96
|
+
* Production daemon MUST NOT set allowLoopback=true. */
|
|
97
|
+
export function validateBaseUrl(
|
|
98
|
+
vendor: string,
|
|
99
|
+
baseUrl: string,
|
|
100
|
+
opts: { allowLoopback?: boolean } = {},
|
|
101
|
+
): void {
|
|
102
|
+
let u: URL;
|
|
103
|
+
try { u = new URL(baseUrl); }
|
|
104
|
+
catch {
|
|
105
|
+
throw new ProbeValidationError("probe_base_url_invalid", {
|
|
106
|
+
reason: "not a valid URL", baseUrl: baseUrl.slice(0, 100),
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
if (u.protocol !== "https:") {
|
|
111
|
+
if (!opts.allowLoopback || !isLoopbackHost(u.hostname) || u.protocol !== "http:") {
|
|
112
|
+
throw new ProbeValidationError("probe_base_url_invalid", {
|
|
113
|
+
reason: "must be https (or http+loopback with dev opt-in)",
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
const allowed = VENDOR_HOST_ALLOWLIST[vendor];
|
|
118
|
+
if (!allowed) {
|
|
119
|
+
throw new ProbeValidationError("vendor_not_supported", {
|
|
120
|
+
vendor, supported: SUPPORTED_VENDORS,
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
if (!allowed.some(re => re.test(u.hostname))) {
|
|
124
|
+
throw new ProbeValidationError("probe_target_forbidden", {
|
|
125
|
+
vendor, host: u.hostname, allowed: allowed.map(r => r.source),
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { readFileSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import {
|
|
5
|
+
RESERVED_ENV_KEYS_EXACT as HUB_EXACT,
|
|
6
|
+
RESERVED_ENV_PREFIXES as HUB_PREFIXES,
|
|
7
|
+
} from "./reserved-env.js";
|
|
8
|
+
|
|
9
|
+
// RFC-026 v4 §4.4.7 G9 drift guard — denylist MUST be hub-and-daemon
|
|
10
|
+
// identical, or attacker walks the looser layer. Two enforcement modes:
|
|
11
|
+
//
|
|
12
|
+
// 1. byte-identical source files (this test)
|
|
13
|
+
// 2. runtime set-equality of the imported constants
|
|
14
|
+
//
|
|
15
|
+
// We assert both. If a future contributor edits one side without the
|
|
16
|
+
// other, this test breaks the merge.
|
|
17
|
+
|
|
18
|
+
const HUB_PATH = join(import.meta.dir, "reserved-env.ts");
|
|
19
|
+
const DAEMON_PATH = join(import.meta.dir, "..", "..", "..", "agent-node", "src", "shared", "reserved-env.ts");
|
|
20
|
+
|
|
21
|
+
describe("RFC-026 v4 §4.4.7 G9 — hub/daemon reserved-env drift guard", () => {
|
|
22
|
+
test("byte-identical source files", () => {
|
|
23
|
+
const hub = readFileSync(HUB_PATH, "utf-8");
|
|
24
|
+
const daemon = readFileSync(DAEMON_PATH, "utf-8");
|
|
25
|
+
expect(daemon).toBe(hub);
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
test("imported constants are set-equal at runtime", async () => {
|
|
29
|
+
// Use a dynamic file import path so this test breaks if the file
|
|
30
|
+
// moves but still references the daemon copy.
|
|
31
|
+
const daemonMod = await import(DAEMON_PATH);
|
|
32
|
+
const daemonExact = daemonMod.RESERVED_ENV_KEYS_EXACT as Set<string>;
|
|
33
|
+
const daemonPrefixes = daemonMod.RESERVED_ENV_PREFIXES as ReadonlyArray<string>;
|
|
34
|
+
|
|
35
|
+
// EXACT set equality
|
|
36
|
+
expect(daemonExact.size).toBe(HUB_EXACT.size);
|
|
37
|
+
for (const k of HUB_EXACT) expect(daemonExact.has(k)).toBe(true);
|
|
38
|
+
for (const k of daemonExact) expect(HUB_EXACT.has(k)).toBe(true);
|
|
39
|
+
|
|
40
|
+
// PREFIX array order-independent equality
|
|
41
|
+
expect(new Set(daemonPrefixes)).toEqual(new Set(HUB_PREFIXES));
|
|
42
|
+
});
|
|
43
|
+
});
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { RESERVED_ENV_KEYS_EXACT, RESERVED_ENV_PREFIXES, isReservedEnvKey } from "./reserved-env.js";
|
|
3
|
+
|
|
4
|
+
describe("RFC-026 v4 §4.4.7 reserved env denylist (B1)", () => {
|
|
5
|
+
test("exact PATH/HOME/LANG/NODE_OPTIONS blocked", () => {
|
|
6
|
+
for (const k of ["PATH", "HOME", "LANG", "LC_ALL", "SHELL", "USER", "LOGNAME", "NODE_OPTIONS", "IFS", "TMPDIR"]) {
|
|
7
|
+
expect(isReservedEnvKey(k)).toBe(true);
|
|
8
|
+
}
|
|
9
|
+
});
|
|
10
|
+
test("LD_ prefix family blocked (Linux dynamic loader)", () => {
|
|
11
|
+
for (const k of ["LD_PRELOAD", "LD_LIBRARY_PATH", "LD_AUDIT", "LD_"]) {
|
|
12
|
+
expect(isReservedEnvKey(k)).toBe(true);
|
|
13
|
+
}
|
|
14
|
+
});
|
|
15
|
+
test("DYLD_ prefix blocked (macOS)", () => {
|
|
16
|
+
expect(isReservedEnvKey("DYLD_INSERT_LIBRARIES")).toBe(true);
|
|
17
|
+
expect(isReservedEnvKey("DYLD_LIBRARY_PATH")).toBe(true);
|
|
18
|
+
});
|
|
19
|
+
test("BUN_/NPM_/NPM_CONFIG_/NODE_ prefix blocked", () => {
|
|
20
|
+
expect(isReservedEnvKey("BUN_INSTALL")).toBe(true);
|
|
21
|
+
expect(isReservedEnvKey("NPM_TOKEN")).toBe(true);
|
|
22
|
+
expect(isReservedEnvKey("NPM_CONFIG_REGISTRY")).toBe(true);
|
|
23
|
+
expect(isReservedEnvKey("NODE_PATH")).toBe(true);
|
|
24
|
+
expect(isReservedEnvKey("NODE_TLS_REJECT_UNAUTHORIZED")).toBe(true);
|
|
25
|
+
});
|
|
26
|
+
test("legitimate API key style is NOT blocked", () => {
|
|
27
|
+
for (const k of ["ANTHROPIC_API_KEY", "OPENAI_API_KEY", "TELEGRAM_BOT_TOKEN", "DEEPSEEK_API_KEY", "MY_APP_CONFIG"]) {
|
|
28
|
+
expect(isReservedEnvKey(k)).toBe(false);
|
|
29
|
+
}
|
|
30
|
+
});
|
|
31
|
+
test("set shape sanity — both EXACT and PREFIX non-empty + frozen-ish", () => {
|
|
32
|
+
expect(RESERVED_ENV_KEYS_EXACT.size).toBeGreaterThan(10);
|
|
33
|
+
expect(RESERVED_ENV_PREFIXES.length).toBeGreaterThan(3);
|
|
34
|
+
});
|
|
35
|
+
});
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// RFC-026 v4 §4.4.7 B1 — reserved env-key denylist (single source of
|
|
2
|
+
// truth for hub + daemon).
|
|
3
|
+
//
|
|
4
|
+
// Scope: env_refs naming a system/process-model sensitive key (PATH /
|
|
5
|
+
// LD_PRELOAD / NODE_OPTIONS / BUN_* / NPM_* / ...) MUST be rejected
|
|
6
|
+
// before the secret value ever reaches fork(), because such a key in
|
|
7
|
+
// the child env is effectively arbitrary code execution.
|
|
8
|
+
//
|
|
9
|
+
// G9 (drift guard): this module is the ONE source. Daemon-side runtime
|
|
10
|
+
// imports the same constants. A divergence between hub and daemon would
|
|
11
|
+
// let attacker target the looser layer; CI test asserts set equality.
|
|
12
|
+
|
|
13
|
+
export const RESERVED_ENV_KEYS_EXACT = new Set<string>([
|
|
14
|
+
"PATH", "HOME", "LANG", "LC_ALL", "SHELL", "USER", "LOGNAME",
|
|
15
|
+
"NODE_OPTIONS", "IFS", "PS1", "PS4", "ENV", "BASH_ENV",
|
|
16
|
+
"CDPATH", "PROMPT_COMMAND", "TMPDIR",
|
|
17
|
+
]);
|
|
18
|
+
|
|
19
|
+
// Prefix-match denied. Order matters only for clarity (LD_ catches
|
|
20
|
+
// LD_PRELOAD / LD_LIBRARY_PATH / LD_AUDIT; DYLD_ catches DYLD_INSERT_
|
|
21
|
+
// LIBRARIES; BUN_ / NPM_ / NPM_CONFIG_ / NODE_ catch ecosystem-level
|
|
22
|
+
// runtime config).
|
|
23
|
+
export const RESERVED_ENV_PREFIXES: ReadonlyArray<string> = [
|
|
24
|
+
"LD_", "DYLD_", "BUN_", "NPM_", "NPM_CONFIG_", "NODE_",
|
|
25
|
+
];
|
|
26
|
+
|
|
27
|
+
export function isReservedEnvKey(k: string): boolean {
|
|
28
|
+
if (RESERVED_ENV_KEYS_EXACT.has(k)) return true;
|
|
29
|
+
for (const p of RESERVED_ENV_PREFIXES) if (k.startsWith(p)) return true;
|
|
30
|
+
return false;
|
|
31
|
+
}
|