@obsunified/collector 2.0.1 → 2.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/dist/auth.d.ts +1 -1
  2. package/dist/auth.js +1 -1
  3. package/dist/{chunk-UGWDOGGH.js → chunk-KMEZKGSO.js} +22 -10
  4. package/dist/chunk-KMEZKGSO.js.map +1 -0
  5. package/dist/index.d.ts +322 -26
  6. package/dist/index.js +1478 -415
  7. package/dist/index.js.map +1 -1
  8. package/dist/{ingest-auth-HFVO1zIp.d.ts → ingest-auth-C5-8Mly5.d.ts} +9 -0
  9. package/package.json +5 -5
  10. package/src/analyses/derive.ts +26 -25
  11. package/src/analyses/investigations.ts +8 -8
  12. package/src/analyses/tier0.ts +22 -22
  13. package/src/auth/dashboard-auth.test.ts +791 -0
  14. package/src/auth/dashboard-auth.ts +187 -38
  15. package/src/auth/login-rate-limit.test.ts +129 -0
  16. package/src/auth/login-rate-limit.ts +143 -0
  17. package/src/framework/body-limit.test.ts +165 -0
  18. package/src/framework/collector.ts +355 -115
  19. package/src/framework/env.ts +9 -0
  20. package/src/framework/limits.ts +31 -0
  21. package/src/framework/scheduled-handlers.test.ts +342 -0
  22. package/src/index.ts +16 -1
  23. package/src/lib/action-retention.test.ts +472 -0
  24. package/src/lib/action-retention.ts +420 -0
  25. package/src/lib/actions-store.ts +6 -12
  26. package/src/lib/ai-store/calls.ts +2 -2
  27. package/src/lib/ai-store/sessions.ts +3 -3
  28. package/src/lib/ai-store/spans.ts +1 -1
  29. package/src/lib/ai-store/types.ts +2 -1
  30. package/src/lib/alerts-store.test.ts +153 -0
  31. package/src/lib/alerts-store.ts +122 -21
  32. package/src/lib/analyses-runner.test.ts +75 -0
  33. package/src/lib/analyses-runner.ts +60 -3
  34. package/src/lib/analyses-store-latest.test.ts +61 -0
  35. package/src/lib/analyses-store.ts +17 -10
  36. package/src/lib/blob-store.ts +1 -1
  37. package/src/lib/housekeeping.test.ts +73 -0
  38. package/src/lib/housekeeping.ts +93 -0
  39. package/src/lib/identity-index/action-lookups.ts +6 -6
  40. package/src/lib/identity-index/key-lookups.ts +2 -2
  41. package/src/lib/identity-index/user-lookup.ts +8 -5
  42. package/src/lib/identity-index.test.ts +2 -3
  43. package/src/lib/metrics-store-series.test.ts +99 -0
  44. package/src/lib/metrics-store.test.ts +23 -0
  45. package/src/lib/metrics-store.ts +27 -3
  46. package/src/lib/projects-store.test.ts +25 -0
  47. package/src/lib/projects-store.ts +4 -3
  48. package/src/lib/sql-db-dialect.test.ts +12 -3
  49. package/src/lib/sql-db-postgres.integration.test.ts +1225 -0
  50. package/src/lib/sql-db-postgres.test.ts +314 -0
  51. package/src/lib/sql-db-postgres.ts +392 -54
  52. package/src/lib/sql-db-sqlite-time-windows.test.ts +194 -0
  53. package/src/lib/sql-db.ts +40 -6
  54. package/src/lib/test-utils/action-graph-fixtures.ts +307 -0
  55. package/src/lib/test-utils/node-sqlite-db.ts +188 -0
  56. package/src/lib/webhook-url.test.ts +122 -0
  57. package/src/lib/webhook-url.ts +155 -0
  58. package/src/migrations/041_ingest_key_hash_text.sql +5 -0
  59. package/src/migrations/042_session_replay_project_composite_pk.sql +52 -0
  60. package/src/migrations/043_metric_exemplars_text_timestamps.sql +5 -0
  61. package/src/migrations/044_actions_started_at_index.sql +10 -0
  62. package/src/migrations/045_user_profiles_project_composite_pk.sql +40 -0
  63. package/src/migrations/046_action_graph_project_composite_pk.sql +255 -0
  64. package/src/migrations/047_ai_span_payloads_project_composite_pk.sql +61 -0
  65. package/src/migrations/048_telemetry_spans_project_composite_pk.sql +92 -0
  66. package/src/migrations/049_alert_state_notified.sql +17 -0
  67. package/src/migrations/alert-state-notified.test.ts +111 -0
  68. package/src/migrations/parity.test.ts +64 -0
  69. package/src/migrations/project-scoped-keys.test.ts +432 -0
  70. package/src/migrations-postgres/009_projects_and_keys.sql +4 -4
  71. package/src/migrations-postgres/026_obs_dashboard_project_seed.sql +1 -1
  72. package/src/migrations-postgres/035_metric_exemplars.sql +5 -2
  73. package/src/migrations-postgres/041_ingest_key_hash_text.sql +32 -0
  74. package/src/migrations-postgres/042_session_replay_project_composite_pk.sql +16 -0
  75. package/src/migrations-postgres/043_metric_exemplars_text_timestamps.sql +47 -0
  76. package/src/migrations-postgres/044_actions_started_at_index.sql +8 -0
  77. package/src/migrations-postgres/045_user_profiles_project_composite_pk.sql +33 -0
  78. package/src/migrations-postgres/046_action_graph_project_composite_pk.sql +39 -0
  79. package/src/migrations-postgres/047_ai_span_payloads_project_composite_pk.sql +30 -0
  80. package/src/migrations-postgres/048_telemetry_spans_project_composite_pk.sql +37 -0
  81. package/src/migrations-postgres/049_alert_state_notified.sql +10 -0
  82. package/src/migrations-postgres/README.md +58 -5
  83. package/src/otlp/decode/body.test.ts +76 -0
  84. package/src/otlp/decode/body.ts +67 -6
  85. package/src/plugins/_json.ts +21 -0
  86. package/src/plugins/action-graph-processor.ts +6 -12
  87. package/src/plugins/ai-receiver.ts +5 -12
  88. package/src/plugins/ai-span-payloads-processor.ts +1 -2
  89. package/src/plugins/alerts-evaluator-retention.test.ts +170 -0
  90. package/src/plugins/alerts-evaluator.test.ts +464 -3
  91. package/src/plugins/alerts-evaluator.ts +165 -41
  92. package/src/plugins/alerts-routes.ts +10 -6
  93. package/src/plugins/ask-routes.ts +4 -5
  94. package/src/plugins/connected-routes.test.ts +106 -0
  95. package/src/plugins/connected-routes.ts +14 -5
  96. package/src/plugins/dashboard-routes.test.ts +120 -0
  97. package/src/plugins/dashboard-routes.ts +11 -2
  98. package/src/plugins/eval-cases-routes.ts +8 -17
  99. package/src/plugins/identity-receiver.ts +9 -11
  100. package/src/plugins/json-body.test.ts +91 -0
  101. package/src/plugins/onboarding-routes.test.ts +105 -0
  102. package/src/plugins/onboarding-routes.ts +26 -11
  103. package/src/plugins/projects-routes.ts +2 -1
  104. package/src/plugins/replay-isolation.test.ts +922 -0
  105. package/src/plugins/replay-receiver.ts +8 -8
  106. package/src/plugins/usage-receiver.ts +3 -6
  107. package/dist/chunk-UGWDOGGH.js.map +0 -1
@@ -21,6 +21,12 @@ import { EVIDENCE_REFERENCE_CONTRACT } from "@obsunified/types";
21
21
  import { alertEvidenceReferences } from "./evidence-references";
22
22
  import { randomHex } from "./hash";
23
23
  import type { SqlDb } from "./sql-db";
24
+ import { assertSafeWebhookUrl, WebhookUrlError } from "./webhook-url";
25
+
26
+ export interface AlertsStoreOptions {
27
+ /** Permit webhook targets on loopback / private / link-local hosts. */
28
+ allowPrivateWebhookTargets?: boolean;
29
+ }
24
30
 
25
31
  const parseJsonField = <T>(value: unknown): T => {
26
32
  if (typeof value === "string") return JSON.parse(value) as T;
@@ -79,7 +85,10 @@ export function compareValue(
79
85
  }
80
86
 
81
87
  export class AlertsStore {
82
- constructor(private readonly db: SqlDb) {}
88
+ constructor(
89
+ private readonly db: SqlDb,
90
+ private readonly options: AlertsStoreOptions = {},
91
+ ) {}
83
92
 
84
93
  validateInput(input: AlertRuleInput): void {
85
94
  if (!input.name?.trim()) throw new Error("name is required");
@@ -96,8 +105,16 @@ export class AlertsStore {
96
105
  for (const ch of input.channels) {
97
106
  if (ch.type !== "webhook")
98
107
  throw new Error(`unknown channel type: ${ch.type}`);
99
- if (!ch.url || !/^https?:\/\//.test(ch.url))
108
+ if (!ch.url || typeof ch.url !== "string")
100
109
  throw new Error("channel.url must be http(s)://");
110
+ try {
111
+ assertSafeWebhookUrl(ch.url, {
112
+ allowPrivateNetworks: this.options.allowPrivateWebhookTargets,
113
+ });
114
+ } catch (err) {
115
+ if (err instanceof WebhookUrlError) throw new Error(err.message);
116
+ throw err;
117
+ }
101
118
  }
102
119
  }
103
120
 
@@ -217,18 +234,30 @@ export class AlertsStore {
217
234
  }
218
235
 
219
236
  async deleteRule(id: string, projectId: string): Promise<boolean> {
220
- const result = await this.db
221
- .prepare(`DELETE FROM alert_rules WHERE id = ? AND project_id = ?`)
237
+ // Only a rule the caller's project owns takes its state and
238
+ // evaluations with it; a miss must not touch another project's rows.
239
+ // Children go first because alert_state / alert_evaluations reference
240
+ // alert_rules(id) and SQLite enforces that on D1 and node:sqlite.
241
+ const owned = await this.db
242
+ .prepare(
243
+ `SELECT 1 AS ok FROM alert_rules WHERE id = ? AND project_id = ?`,
244
+ )
222
245
  .bind(id, projectId)
223
- .run();
224
- // Also clean up state and evaluations for this rule.
246
+ .first<{ ok: number }>();
247
+ if (!owned) return false;
225
248
  await this.db
226
- .prepare(`DELETE FROM alert_state WHERE rule_id = ?`)
227
- .bind(id)
249
+ .prepare(`DELETE FROM alert_state WHERE rule_id = ? AND project_id = ?`)
250
+ .bind(id, projectId)
228
251
  .run();
229
252
  await this.db
230
- .prepare(`DELETE FROM alert_evaluations WHERE rule_id = ?`)
231
- .bind(id)
253
+ .prepare(
254
+ `DELETE FROM alert_evaluations WHERE rule_id = ? AND project_id = ?`,
255
+ )
256
+ .bind(id, projectId)
257
+ .run();
258
+ const result = await this.db
259
+ .prepare(`DELETE FROM alert_rules WHERE id = ? AND project_id = ?`)
260
+ .bind(id, projectId)
232
261
  .run();
233
262
  return (result.meta?.changes ?? 0) > 0;
234
263
  }
@@ -242,28 +271,85 @@ export class AlertsStore {
242
271
 
243
272
  // ── State ──
244
273
 
245
- async getState(ruleId: string): Promise<AlertStateRow | null> {
274
+ async getState(
275
+ ruleId: string,
276
+ projectId: string,
277
+ ): Promise<AlertStateRow | null> {
246
278
  const row = await this.db
247
- .prepare(`SELECT * FROM alert_state WHERE rule_id = ?`)
248
- .bind(ruleId)
279
+ .prepare(`SELECT * FROM alert_state WHERE rule_id = ? AND project_id = ?`)
280
+ .bind(ruleId, projectId)
249
281
  .first<AlertStateRow>();
250
282
  return row ?? null;
251
283
  }
252
284
 
285
+ /**
286
+ * Record a state transition with its notification still outstanding
287
+ * (`notified = 0`); `markNotified` flips it once a channel accepts the
288
+ * alert. On a database that has not applied migration 049 the column is
289
+ * missing: the transition is still written (legacy statement) so alerting
290
+ * keeps working through a rolling deploy, and the evaluator treats a row
291
+ * without the column as delivered.
292
+ */
253
293
  async transitionState(
254
294
  ruleId: string,
255
295
  projectId: string,
256
296
  newState: AlertState,
257
297
  now: string,
258
298
  ): Promise<void> {
259
- await this.db
260
- .prepare(
261
- `INSERT INTO alert_state (rule_id, project_id, current_state, last_state_change)
299
+ try {
300
+ await this.db
301
+ .prepare(
302
+ `INSERT INTO alert_state (rule_id, project_id, current_state, last_state_change, notified)
303
+ VALUES (?, ?, ?, ?, 0)
304
+ ON CONFLICT(rule_id) DO UPDATE SET
305
+ current_state = excluded.current_state,
306
+ last_state_change = excluded.last_state_change,
307
+ notified = 0`,
308
+ )
309
+ .bind(ruleId, projectId, newState, now)
310
+ .run();
311
+ } catch (err) {
312
+ if (!isMissingNotifiedColumn(err)) throw err;
313
+ await this.db
314
+ .prepare(
315
+ `INSERT INTO alert_state (rule_id, project_id, current_state, last_state_change)
262
316
  VALUES (?, ?, ?, ?)
263
- ON CONFLICT(rule_id) DO UPDATE SET current_state = excluded.current_state, last_state_change = excluded.last_state_change`,
264
- )
265
- .bind(ruleId, projectId, newState, now)
266
- .run();
317
+ ON CONFLICT(rule_id) DO UPDATE SET
318
+ current_state = excluded.current_state,
319
+ last_state_change = excluded.last_state_change`,
320
+ )
321
+ .bind(ruleId, projectId, newState, now)
322
+ .run();
323
+ }
324
+ }
325
+
326
+ /**
327
+ * Mark the transition `(state, lastStateChange)` as delivered. Scoped to
328
+ * the exact transition the caller notified for: if the rule changed state
329
+ * again while the webhook was in flight, the newer transition keeps its
330
+ * own outstanding flag. Returns false when nothing matched (stale or
331
+ * already marked). On a pre-049 schema there is nothing to mark.
332
+ */
333
+ async markNotified(
334
+ ruleId: string,
335
+ projectId: string,
336
+ state: AlertState,
337
+ lastStateChange: string,
338
+ ): Promise<boolean> {
339
+ try {
340
+ const result = await this.db
341
+ .prepare(
342
+ `UPDATE alert_state SET notified = 1
343
+ WHERE rule_id = ? AND project_id = ? AND current_state = ?
344
+ AND last_state_change = ? AND notified = 0`,
345
+ )
346
+ .bind(ruleId, projectId, state, lastStateChange)
347
+ .run();
348
+ return (result.meta?.changes ?? 0) > 0;
349
+ } catch (err) {
350
+ if (isMissingNotifiedColumn(err)) return false;
351
+ throw err;
352
+ }
267
353
  }
268
354
 
269
355
  async recordEvaluation(
@@ -448,7 +534,9 @@ export class AlertsStore {
448
534
  sql += ` AND model_name = ?`;
449
535
  binds.push(q.model);
450
536
  }
451
- if (q.isError === true) sql += ` AND is_error = 1`;
537
+ // Bare boolean test: INTEGER 0/1 on D1, BOOLEAN on Postgres (where
538
+ // `is_error = 1` is a type error).
539
+ if (q.isError === true) sql += ` AND is_error`;
452
540
  const row = await this.db
453
541
  .prepare(sql)
454
542
  .bind(...binds)
@@ -456,3 +544,16 @@ export class AlertsStore {
456
544
  return row?.c ?? 0;
457
545
  }
458
546
  }
547
+
548
+ /**
549
+ * SQLite: "table alert_state has no column named notified" / "no such column:
550
+ * notified"; Postgres: `column "notified" of relation "alert_state" does not
551
+ * exist` (42703). Anything else is a real failure.
552
+ */
553
+ const isMissingNotifiedColumn = (err: unknown): boolean => {
554
+ const message = err instanceof Error ? err.message : String(err);
555
+ return (
556
+ /notified/i.test(message) &&
557
+ /no such column|has no column|does not exist/i.test(message)
558
+ );
559
+ };
@@ -0,0 +1,75 @@
1
+ /**
2
+ * `runAllDueAnalyses` used to hard-code project `default`, so analyses for
3
+ * every other project never refreshed and alert rules bound to them always
4
+ * evaluated against nothing. These tests run the orchestrator against the
5
+ * real migrated schema and against a database whose projects table fails.
6
+ */
7
+
8
+ import { describe, expect, it } from "vitest";
9
+ import type { CollectorEnv } from "../framework/env";
10
+ import { runAllDueAnalyses } from "./analyses-runner";
11
+ import { ProjectsStore } from "./projects-store";
12
+ import { MemSqlDb } from "./test-utils/mem-sql-db";
13
+ import { createMigratedSqliteDb } from "./test-utils/node-sqlite-db";
14
+
15
+ const silent = { debug() {}, info() {}, warn() {}, error() {} };
16
+
17
+ describe("runAllDueAnalyses", () => {
18
+ it("refreshes and runs analyses for every project, not just default", async () => {
19
+ const db = createMigratedSqliteDb();
20
+ const beta = await new ProjectsStore(db).createProject({
21
+ name: "Beta",
22
+ slug: "beta",
23
+ });
24
+
25
+ const summary = await runAllDueAnalyses({
26
+ env: { DB: db } as unknown as CollectorEnv,
27
+ retentionHours: 24,
28
+ logger: silent,
29
+ });
30
+
31
+ expect(summary.failed).toBe(0);
32
+ expect(summary.ran).toBeGreaterThan(0);
33
+ const projects = await db
34
+ .prepare(
35
+ `SELECT DISTINCT project_id FROM analysis_results ORDER BY project_id`,
36
+ )
37
+ .all<{ project_id: string }>();
38
+ const ids = projects.results.map((r) => r.project_id);
39
+ // 011 seeds `default`, 026 seeds `obs-dashboard`.
40
+ expect(ids).toEqual(
41
+ expect.arrayContaining(["default", "obs-dashboard", beta.id]),
42
+ );
43
+
44
+ const perProject = await db
45
+ .prepare(
46
+ `SELECT project_id, COUNT(*) AS n FROM analysis_results GROUP BY project_id`,
47
+ )
48
+ .all<{ project_id: string; n: number }>();
49
+ const counts = new Map(perProject.results.map((r) => [r.project_id, r.n]));
50
+ expect(counts.get(beta.id)).toBe(counts.get("default"));
51
+ db.close();
52
+ });
53
+
54
+ it("falls back to the default project when projects cannot be listed", async () => {
55
+ const db = new MemSqlDb({
56
+ all: (sql) => {
57
+ if (sql.includes("FROM projects")) throw new Error("no projects table");
58
+ return [];
59
+ },
60
+ first: () => null,
61
+ });
62
+
63
+ const summary = await runAllDueAnalyses({
64
+ env: { DB: db } as unknown as CollectorEnv,
65
+ retentionHours: 24,
66
+ logger: silent,
67
+ });
68
+
69
+ expect(summary).toMatchObject({ ran: 0, failed: 0 });
70
+ expect(summary.refreshed).toBeGreaterThan(0);
71
+ const due = db.callsMatching("FROM analysis_definitions");
72
+ expect(due.length).toBeGreaterThan(0);
73
+ expect(due.every((c) => c.binds[0] === "default")).toBe(true);
74
+ });
75
+ });
@@ -42,6 +42,7 @@ import {
42
42
  evaluateGate,
43
43
  type NarrativeIntent,
44
44
  } from "./narrate-gate";
45
+ import { ProjectsStore } from "./projects-store";
45
46
  import { type SqlDb, sqlDbFor } from "./sql-db";
46
47
 
47
48
  export interface AnalysisRunContext {
@@ -253,8 +254,10 @@ async function narratePass(
253
254
  * in-process registry (so newly-derived Tier 1 analyses become visible
254
255
  * to the dashboard within one tick), then runs whatever's due.
255
256
  *
256
- * Stage 1 has a single project ("default"). When a projects table is
257
- * available we'll iterate it here.
257
+ * Runs once per project in the `projects` table (falling back to the
258
+ * `default` project when the table is empty or unreadable), so analyses and
259
+ * the alert rules bound to them refresh for every tenant, not just the
260
+ * first one.
258
261
  */
259
262
  export async function runAllDueAnalyses(
260
263
  ctx: {
@@ -274,7 +277,61 @@ export async function runAllDueAnalyses(
274
277
  const logger = ctx.logger ?? consoleLogger;
275
278
  const db = sqlDbFor(ctx.env);
276
279
  const store = new AnalysesStore(db);
277
- const projectId = "default";
280
+ const totals = { ran: 0, failed: 0, refreshed: 0, narrated: 0, skipped: 0 };
281
+ for (const projectId of await listAnalysisProjects(db, logger)) {
282
+ const summary = await runDueAnalysesForProject(projectId, {
283
+ ctx,
284
+ db,
285
+ store,
286
+ logger,
287
+ });
288
+ totals.ran += summary.ran;
289
+ totals.failed += summary.failed;
290
+ totals.refreshed += summary.refreshed;
291
+ totals.narrated += summary.narrated;
292
+ totals.skipped += summary.skipped ?? 0;
293
+ }
294
+ return totals;
295
+ }
296
+
297
+ /** Project ids to run analyses for; `default` when none can be listed. */
298
+ async function listAnalysisProjects(
299
+ db: SqlDb,
300
+ logger: Logger,
301
+ ): Promise<string[]> {
302
+ try {
303
+ const ids = (await new ProjectsStore(db).listProjects())
304
+ .map((p) => p.id)
305
+ .filter((id) => typeof id === "string" && id.length > 0);
306
+ if (ids.length > 0) return ids;
307
+ } catch (error) {
308
+ logger.error("[analyses] failed to list projects; running default only", {
309
+ error: error instanceof Error ? error.message : String(error),
310
+ });
311
+ }
312
+ return ["default"];
313
+ }
314
+
315
+ async function runDueAnalysesForProject(
316
+ projectId: string,
317
+ deps: {
318
+ ctx: {
319
+ env: CollectorEnv;
320
+ retentionHours: number;
321
+ tracer?: ChildSpanRunner;
322
+ };
323
+ db: SqlDb;
324
+ store: AnalysesStore;
325
+ logger: Logger;
326
+ },
327
+ ): Promise<{
328
+ ran: number;
329
+ failed: number;
330
+ refreshed: number;
331
+ narrated: number;
332
+ skipped?: number;
333
+ }> {
334
+ const { ctx, db, store, logger } = deps;
278
335
  const now = Date.now();
279
336
  const expiresAt = now + ctx.retentionHours * 3600 * 1000;
280
337
 
@@ -0,0 +1,61 @@
1
+ import type { AnalysisResult } from "@obsunified/types";
2
+ import { describe, expect, it } from "vitest";
3
+ import { AnalysesStore } from "./analyses-store";
4
+ import { createMigratedSqliteDb } from "./test-utils/node-sqlite-db";
5
+
6
+ const result = (
7
+ analysisId: string,
8
+ generatedAt: string,
9
+ status: AnalysisResult["status"],
10
+ ): AnalysisResult => ({
11
+ analysisId,
12
+ projectId: "alpha",
13
+ generatedAt,
14
+ paramsHash: null,
15
+ status,
16
+ primaryValue: 1,
17
+ baselineValue: 0,
18
+ deltaPct: null,
19
+ payload: {},
20
+ narrative: null,
21
+ narrativeSignature: null,
22
+ durationMs: 1,
23
+ });
24
+
25
+ describe("AnalysesStore.getLatestResultsBulk", () => {
26
+ it("returns the newest result per analysis from a single bounded query", async () => {
27
+ const db = createMigratedSqliteDb();
28
+ const store = new AnalysesStore(db);
29
+ const expires = Date.now() + 3_600_000;
30
+ for (const [id, at, status] of [
31
+ ["a", "2026-10-13T01:00:00.000Z", "ok"],
32
+ ["a", "2026-10-13T03:00:00.000Z", "critical"],
33
+ ["a", "2026-10-13T02:00:00.000Z", "warn"],
34
+ ["b", "2026-10-13T02:30:00.000Z", "warn"],
35
+ ["c", "2026-10-13T02:30:00.000Z", "ok"],
36
+ ] as const) {
37
+ await store.insertResult(result(id, at, status), expires);
38
+ }
39
+ // Another project with the same ids must not bleed through.
40
+ await store.insertResult(
41
+ { ...result("a", "2026-10-13T09:00:00.000Z", "ok"), projectId: "beta" },
42
+ expires,
43
+ );
44
+
45
+ const latest = await store.getLatestResultsBulk("alpha", ["a", "b", "zz"]);
46
+ expect([...latest.keys()].sort()).toEqual(["a", "b"]);
47
+ expect(latest.get("a")?.status).toBe("critical");
48
+ expect(latest.get("a")?.generatedAt).toBe("2026-10-13T03:00:00.000Z");
49
+ expect(latest.get("b")?.status).toBe("warn");
50
+
51
+ // The query itself must not return every retained row.
52
+ const rows = await db
53
+ .prepare(
54
+ `SELECT COUNT(*) AS n FROM analysis_results WHERE project_id = ? AND analysis_id = ?`,
55
+ )
56
+ .bind("alpha", "a")
57
+ .first<{ n: number }>();
58
+ expect(rows?.n).toBe(3);
59
+ db.close();
60
+ });
61
+ });
@@ -293,20 +293,27 @@ export class AnalysesStore {
293
293
  );
294
294
  if (analysisIds.length === 0) return new Map();
295
295
 
296
- // Pull a recent window of rows for the requested analyses then group by
297
- // analysis_id, taking the row with the largest generated_at per group.
298
- // One query, no per-analysis round-trips.
296
+ // One row per analysis: join against the per-analysis MAX(generated_at)
297
+ // so the database returns only the newest result for each id instead
298
+ // of every retained row (a 60 s refresh over a 72 h window is ~4k rows
299
+ // per analysis). Ties on generated_at are broken in JS below.
299
300
  const placeholders = analysisIds.map(() => "?").join(", ");
300
301
  const result = await this.db
301
302
  .prepare(
302
- `SELECT id, project_id, analysis_id, generated_at, params_hash,
303
- status, primary_value, baseline_value, delta_pct, payload_json,
304
- narrative, narrative_signature, duration_ms, expires_at
305
- FROM analysis_results
306
- WHERE project_id = ? AND analysis_id IN (${placeholders})
307
- ORDER BY generated_at DESC`,
303
+ `SELECT r.id, r.project_id, r.analysis_id, r.generated_at, r.params_hash,
304
+ r.status, r.primary_value, r.baseline_value, r.delta_pct, r.payload_json,
305
+ r.narrative, r.narrative_signature, r.duration_ms, r.expires_at
306
+ FROM analysis_results r
307
+ JOIN (
308
+ SELECT analysis_id, MAX(generated_at) AS latest_at
309
+ FROM analysis_results
310
+ WHERE project_id = ? AND analysis_id IN (${placeholders})
311
+ GROUP BY analysis_id
312
+ ) m ON m.analysis_id = r.analysis_id AND m.latest_at = r.generated_at
313
+ WHERE r.project_id = ?
314
+ ORDER BY r.generated_at DESC, r.id DESC`,
308
315
  )
309
- .bind(projectId, ...analysisIds)
316
+ .bind(projectId, ...analysisIds, projectId)
310
317
  .all<ResultRow>();
311
318
 
312
319
  const latest = new Map<string, AnalysisResult>();
@@ -6,7 +6,7 @@
6
6
  * - `R2BlobStore` — Cloudflare R2 binding (default on Workers).
7
7
  * - `S3BlobStore` (sibling file) — AWS S3 / MinIO / any S3-compatible.
8
8
  *
9
- * Replay chunks live under `replays/<session_id>/<chunk_id>`; pprof
9
+ * Replay chunks live under `replays/<project_id>/<session_id>/<chunk_id>`; pprof
10
10
  * profiles live under `profiles/<service>/<trace_id>/<span_id>`. The
11
11
  * adapter never sees these paths — they're constructed by the callers
12
12
  * (`replay-receiver`, `profile-routes`).
@@ -0,0 +1,73 @@
1
+ import { describe, expect, it } from "vitest";
2
+ import { purgeSecondaryRows } from "./housekeeping";
3
+ import { createMigratedSqliteDb } from "./test-utils/node-sqlite-db";
4
+
5
+ const HOUR = 3_600_000;
6
+ const NOW = Date.UTC(2026, 9, 13, 12, 0, 0);
7
+ const iso = (hoursAgo: number) => new Date(NOW - hoursAgo * HOUR).toISOString();
8
+
9
+ describe("purgeSecondaryRows", () => {
10
+ it("removes only rows no API window or ingest path can still reach", async () => {
11
+ const db = createMigratedSqliteDb();
12
+ const raw = db.raw;
13
+ raw.exec(`
14
+ INSERT INTO projects (id, name, slug, created_at) VALUES ('alpha', 'A', 'a', '${iso(1000)}');
15
+ INSERT INTO alert_rules (id, project_id, name, signal, query_json, threshold, window_mins, comparison, channels_json, enabled, created_at, updated_at)
16
+ VALUES ('r1', 'alpha', 'r', 'spans', '{}', 1, 5, '>', '[]', 1, '${iso(1000)}', '${iso(1000)}');
17
+ `);
18
+ const ev = raw.prepare(
19
+ `INSERT INTO alert_evaluations (id, rule_id, project_id, evaluated_at, value, state, notified) VALUES (?, 'r1', 'alpha', ?, 1, 'ok', 0)`,
20
+ );
21
+ ev.run("old", iso(721));
22
+ ev.run("edge", iso(719));
23
+ const ask = raw.prepare(
24
+ `INSERT INTO ask_evidence_events (project_id, analysis_id, asked_at) VALUES ('alpha', 'x', ?)`,
25
+ );
26
+ ask.run(NOW - 31 * 24 * HOUR);
27
+ ask.run(NOW - 29 * 24 * HOUR);
28
+ const series = raw.prepare(
29
+ `INSERT INTO metric_series (id, project_id, name, type, identity, created_at) VALUES (?, 'alpha', 'm', 'gauge', ?, ?)`,
30
+ );
31
+ series.run("orphan-old", "i1", iso(100)); // no points, older than window
32
+ series.run("orphan-new", "i2", iso(1)); // no points but just created
33
+ series.run("live", "i3", iso(100)); // old but still has a point
34
+ raw
35
+ .prepare(
36
+ `INSERT INTO metric_point (id, series_id, project_id, ts_ns, received_at, expires_at) VALUES ('p1', 'live', 'alpha', '1', ?, ?)`,
37
+ )
38
+ .run(iso(1), iso(-1));
39
+ const ref = raw.prepare(
40
+ `INSERT INTO evidence_retrieval_refs (ref_id, project_id, kind, anchor_kind, anchor_id, source, expires_at) VALUES (?, 'alpha', 'logs', 'trace', 't', 'test', ?)`,
41
+ );
42
+ ref.run("expired", iso(1));
43
+ ref.run("fresh", iso(-1));
44
+ ref.run("no-expiry", null);
45
+ const exp = raw.prepare(
46
+ `INSERT INTO evidence_ref_expansions (id, ref_id, project_id, kind, operation, result_status) VALUES (?, ?, 'alpha', 'logs', 'expand', 'ok')`,
47
+ );
48
+ exp.run("e1", "expired");
49
+ exp.run("e2", "fresh");
50
+
51
+ const summary = await purgeSecondaryRows(db, {
52
+ retentionHours: 72,
53
+ now: () => NOW,
54
+ });
55
+ expect(summary).toEqual({
56
+ alert_evaluations: 1,
57
+ ask_evidence_events: 1,
58
+ metric_series: 1,
59
+ evidence_refs: 1,
60
+ evidence_ref_expansions: 1,
61
+ });
62
+ const ids = (sql: string) =>
63
+ (raw.prepare(sql).all() as Array<{ id: string }>).map((r) => r.id).sort();
64
+ expect(ids("SELECT id FROM alert_evaluations")).toEqual(["edge"]);
65
+ expect(ids("SELECT id FROM metric_series")).toEqual(["live", "orphan-new"]);
66
+ expect(ids("SELECT ref_id AS id FROM evidence_retrieval_refs")).toEqual([
67
+ "fresh",
68
+ "no-expiry",
69
+ ]);
70
+ expect(ids("SELECT id FROM evidence_ref_expansions")).toEqual(["e2"]);
71
+ db.close();
72
+ });
73
+ });
@@ -0,0 +1,93 @@
1
+ /**
2
+ * Retention for the secondary tables the hourly sweep did not cover. Each
3
+ * rule is tied to an existing contract rather than a new policy:
4
+ *
5
+ * - `alert_evaluations`: `/internal/alerts/evaluations` accepts at most a
6
+ * 720-hour window, so rows older than that are unreachable through the API.
7
+ * - `ask_evidence_events`: feeds `getTopAskedAnalyses`, whose trailing window
8
+ * is 7 days; rows older than 30 days cannot influence it.
9
+ * - `metric_series`: identity rows whose points have all expired and that
10
+ * were created before the retention window. They hold no data; the next
11
+ * ingest of that identity recreates the row.
12
+ * - `evidence_retrieval_refs`: rows carry their own `expires_at` (copied from
13
+ * the signal they reference); expansions follow their ref.
14
+ */
15
+
16
+ import type { SqlDb } from "./sql-db";
17
+
18
+ export const ALERT_EVALUATIONS_MAX_AGE_HOURS = 720;
19
+ export const ASK_EVIDENCE_EVENTS_MAX_AGE_HOURS = 30 * 24;
20
+
21
+ export interface HousekeepingSummary {
22
+ alert_evaluations: number;
23
+ ask_evidence_events: number;
24
+ metric_series: number;
25
+ evidence_refs: number;
26
+ evidence_ref_expansions: number;
27
+ }
28
+
29
+ export interface HousekeepingOptions {
30
+ /** Window that bounds orphaned metric series; same value the main sweep uses. */
31
+ retentionHours: number;
32
+ /** Clock override for tests. */
33
+ now?: () => number;
34
+ }
35
+
36
+ const changes = (r: { meta?: { changes?: number } }) => r.meta?.changes ?? 0;
37
+
38
+ export async function purgeSecondaryRows(
39
+ db: SqlDb,
40
+ options: HousekeepingOptions,
41
+ ): Promise<HousekeepingSummary> {
42
+ const nowMs = options.now?.() ?? Date.now();
43
+ const isoBefore = (hours: number) =>
44
+ new Date(nowMs - hours * 3_600_000).toISOString();
45
+
46
+ const alertEvaluations = await db
47
+ .prepare(`DELETE FROM alert_evaluations WHERE evaluated_at < ?`)
48
+ .bind(isoBefore(ALERT_EVALUATIONS_MAX_AGE_HOURS))
49
+ .run();
50
+
51
+ const askEvents = await db
52
+ .prepare(`DELETE FROM ask_evidence_events WHERE asked_at < ?`)
53
+ .bind(nowMs - ASK_EVIDENCE_EVENTS_MAX_AGE_HOURS * 3_600_000)
54
+ .run();
55
+
56
+ // Points and exemplars for the window are purged by MetricsStore first;
57
+ // a series created before the window with no remaining points is inert.
58
+ const series = await db
59
+ .prepare(
60
+ `DELETE FROM metric_series
61
+ WHERE created_at < ?
62
+ AND NOT EXISTS (
63
+ SELECT 1 FROM metric_point p WHERE p.series_id = metric_series.id
64
+ )`,
65
+ )
66
+ .bind(isoBefore(options.retentionHours))
67
+ .run();
68
+
69
+ const refs = await db
70
+ .prepare(
71
+ `DELETE FROM evidence_retrieval_refs
72
+ WHERE expires_at IS NOT NULL AND expires_at < ?`,
73
+ )
74
+ .bind(isoBefore(0))
75
+ .run();
76
+ const expansions = await db
77
+ .prepare(
78
+ `DELETE FROM evidence_ref_expansions
79
+ WHERE NOT EXISTS (
80
+ SELECT 1 FROM evidence_retrieval_refs r
81
+ WHERE r.ref_id = evidence_ref_expansions.ref_id
82
+ )`,
83
+ )
84
+ .run();
85
+
86
+ return {
87
+ alert_evaluations: changes(alertEvaluations),
88
+ ask_evidence_events: changes(askEvents),
89
+ metric_series: changes(series),
90
+ evidence_refs: changes(refs),
91
+ evidence_ref_expansions: changes(expansions),
92
+ };
93
+ }
@@ -313,10 +313,10 @@ export async function manifestByAction(
313
313
  .prepare(
314
314
  `SELECT session_id, first_chunk_at, last_chunk_at, chunk_count, events_count
315
315
  FROM session_replay_metadata
316
- WHERE session_id IN (${sessionPlaceholders})
316
+ WHERE project_id = ? AND session_id IN (${sessionPlaceholders})
317
317
  ORDER BY last_chunk_at DESC LIMIT 1`,
318
318
  )
319
- .bind(...sessionIds)
319
+ .bind(projectId, ...sessionIds)
320
320
  .first<{
321
321
  session_id: string;
322
322
  first_chunk_at: string;
@@ -546,10 +546,10 @@ export async function manifestByAgentRun(
546
546
  .prepare(
547
547
  `SELECT session_id, first_chunk_at, last_chunk_at, chunk_count, events_count
548
548
  FROM session_replay_metadata
549
- WHERE session_id IN (${sessionPlaceholders})
549
+ WHERE project_id = ? AND session_id IN (${sessionPlaceholders})
550
550
  ORDER BY last_chunk_at DESC LIMIT 1`,
551
551
  )
552
- .bind(...sessionIds)
552
+ .bind(projectId, ...sessionIds)
553
553
  .first<{
554
554
  session_id: string;
555
555
  first_chunk_at: string;
@@ -767,10 +767,10 @@ export async function manifestByActor(
767
767
  .prepare(
768
768
  `SELECT session_id, first_chunk_at, last_chunk_at, chunk_count, events_count
769
769
  FROM session_replay_metadata
770
- WHERE session_id IN (${sessionPlaceholders})
770
+ WHERE project_id = ? AND session_id IN (${sessionPlaceholders})
771
771
  ORDER BY last_chunk_at DESC LIMIT 1`,
772
772
  )
773
- .bind(...sessionIds)
773
+ .bind(projectId, ...sessionIds)
774
774
  .first<{
775
775
  session_id: string;
776
776
  first_chunk_at: string;