@obsunified/collector 2.0.1 → 2.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth.d.ts +1 -1
- package/dist/auth.js +1 -1
- package/dist/{chunk-UGWDOGGH.js → chunk-KMEZKGSO.js} +22 -10
- package/dist/chunk-KMEZKGSO.js.map +1 -0
- package/dist/index.d.ts +322 -26
- package/dist/index.js +1478 -415
- package/dist/index.js.map +1 -1
- package/dist/{ingest-auth-HFVO1zIp.d.ts → ingest-auth-C5-8Mly5.d.ts} +9 -0
- package/package.json +5 -5
- package/src/analyses/derive.ts +26 -25
- package/src/analyses/investigations.ts +8 -8
- package/src/analyses/tier0.ts +22 -22
- package/src/auth/dashboard-auth.test.ts +791 -0
- package/src/auth/dashboard-auth.ts +187 -38
- package/src/auth/login-rate-limit.test.ts +129 -0
- package/src/auth/login-rate-limit.ts +143 -0
- package/src/framework/body-limit.test.ts +165 -0
- package/src/framework/collector.ts +355 -115
- package/src/framework/env.ts +9 -0
- package/src/framework/limits.ts +31 -0
- package/src/framework/scheduled-handlers.test.ts +342 -0
- package/src/index.ts +16 -1
- package/src/lib/action-retention.test.ts +472 -0
- package/src/lib/action-retention.ts +420 -0
- package/src/lib/actions-store.ts +6 -12
- package/src/lib/ai-store/calls.ts +2 -2
- package/src/lib/ai-store/sessions.ts +3 -3
- package/src/lib/ai-store/spans.ts +1 -1
- package/src/lib/ai-store/types.ts +2 -1
- package/src/lib/alerts-store.test.ts +153 -0
- package/src/lib/alerts-store.ts +122 -21
- package/src/lib/analyses-runner.test.ts +75 -0
- package/src/lib/analyses-runner.ts +60 -3
- package/src/lib/analyses-store-latest.test.ts +61 -0
- package/src/lib/analyses-store.ts +17 -10
- package/src/lib/blob-store.ts +1 -1
- package/src/lib/housekeeping.test.ts +73 -0
- package/src/lib/housekeeping.ts +93 -0
- package/src/lib/identity-index/action-lookups.ts +6 -6
- package/src/lib/identity-index/key-lookups.ts +2 -2
- package/src/lib/identity-index/user-lookup.ts +8 -5
- package/src/lib/identity-index.test.ts +2 -3
- package/src/lib/metrics-store-series.test.ts +99 -0
- package/src/lib/metrics-store.test.ts +23 -0
- package/src/lib/metrics-store.ts +27 -3
- package/src/lib/projects-store.test.ts +25 -0
- package/src/lib/projects-store.ts +4 -3
- package/src/lib/sql-db-dialect.test.ts +12 -3
- package/src/lib/sql-db-postgres.integration.test.ts +1225 -0
- package/src/lib/sql-db-postgres.test.ts +314 -0
- package/src/lib/sql-db-postgres.ts +392 -54
- package/src/lib/sql-db-sqlite-time-windows.test.ts +194 -0
- package/src/lib/sql-db.ts +40 -6
- package/src/lib/test-utils/action-graph-fixtures.ts +307 -0
- package/src/lib/test-utils/node-sqlite-db.ts +188 -0
- package/src/lib/webhook-url.test.ts +122 -0
- package/src/lib/webhook-url.ts +155 -0
- package/src/migrations/041_ingest_key_hash_text.sql +5 -0
- package/src/migrations/042_session_replay_project_composite_pk.sql +52 -0
- package/src/migrations/043_metric_exemplars_text_timestamps.sql +5 -0
- package/src/migrations/044_actions_started_at_index.sql +10 -0
- package/src/migrations/045_user_profiles_project_composite_pk.sql +40 -0
- package/src/migrations/046_action_graph_project_composite_pk.sql +255 -0
- package/src/migrations/047_ai_span_payloads_project_composite_pk.sql +61 -0
- package/src/migrations/048_telemetry_spans_project_composite_pk.sql +92 -0
- package/src/migrations/049_alert_state_notified.sql +17 -0
- package/src/migrations/alert-state-notified.test.ts +111 -0
- package/src/migrations/parity.test.ts +64 -0
- package/src/migrations/project-scoped-keys.test.ts +432 -0
- package/src/migrations-postgres/009_projects_and_keys.sql +4 -4
- package/src/migrations-postgres/026_obs_dashboard_project_seed.sql +1 -1
- package/src/migrations-postgres/035_metric_exemplars.sql +5 -2
- package/src/migrations-postgres/041_ingest_key_hash_text.sql +32 -0
- package/src/migrations-postgres/042_session_replay_project_composite_pk.sql +16 -0
- package/src/migrations-postgres/043_metric_exemplars_text_timestamps.sql +47 -0
- package/src/migrations-postgres/044_actions_started_at_index.sql +8 -0
- package/src/migrations-postgres/045_user_profiles_project_composite_pk.sql +33 -0
- package/src/migrations-postgres/046_action_graph_project_composite_pk.sql +39 -0
- package/src/migrations-postgres/047_ai_span_payloads_project_composite_pk.sql +30 -0
- package/src/migrations-postgres/048_telemetry_spans_project_composite_pk.sql +37 -0
- package/src/migrations-postgres/049_alert_state_notified.sql +10 -0
- package/src/migrations-postgres/README.md +58 -5
- package/src/otlp/decode/body.test.ts +76 -0
- package/src/otlp/decode/body.ts +67 -6
- package/src/plugins/_json.ts +21 -0
- package/src/plugins/action-graph-processor.ts +6 -12
- package/src/plugins/ai-receiver.ts +5 -12
- package/src/plugins/ai-span-payloads-processor.ts +1 -2
- package/src/plugins/alerts-evaluator-retention.test.ts +170 -0
- package/src/plugins/alerts-evaluator.test.ts +464 -3
- package/src/plugins/alerts-evaluator.ts +165 -41
- package/src/plugins/alerts-routes.ts +10 -6
- package/src/plugins/ask-routes.ts +4 -5
- package/src/plugins/connected-routes.test.ts +106 -0
- package/src/plugins/connected-routes.ts +14 -5
- package/src/plugins/dashboard-routes.test.ts +120 -0
- package/src/plugins/dashboard-routes.ts +11 -2
- package/src/plugins/eval-cases-routes.ts +8 -17
- package/src/plugins/identity-receiver.ts +9 -11
- package/src/plugins/json-body.test.ts +91 -0
- package/src/plugins/onboarding-routes.test.ts +105 -0
- package/src/plugins/onboarding-routes.ts +26 -11
- package/src/plugins/projects-routes.ts +2 -1
- package/src/plugins/replay-isolation.test.ts +922 -0
- package/src/plugins/replay-receiver.ts +8 -8
- package/src/plugins/usage-receiver.ts +3 -6
- package/dist/chunk-UGWDOGGH.js.map +0 -1
package/src/lib/alerts-store.ts
CHANGED
|
@@ -21,6 +21,12 @@ import { EVIDENCE_REFERENCE_CONTRACT } from "@obsunified/types";
|
|
|
21
21
|
import { alertEvidenceReferences } from "./evidence-references";
|
|
22
22
|
import { randomHex } from "./hash";
|
|
23
23
|
import type { SqlDb } from "./sql-db";
|
|
24
|
+
import { assertSafeWebhookUrl, WebhookUrlError } from "./webhook-url";
|
|
25
|
+
|
|
26
|
+
export interface AlertsStoreOptions {
|
|
27
|
+
/** Permit webhook targets on loopback / private / link-local hosts. */
|
|
28
|
+
allowPrivateWebhookTargets?: boolean;
|
|
29
|
+
}
|
|
24
30
|
|
|
25
31
|
const parseJsonField = <T>(value: unknown): T => {
|
|
26
32
|
if (typeof value === "string") return JSON.parse(value) as T;
|
|
@@ -79,7 +85,10 @@ export function compareValue(
|
|
|
79
85
|
}
|
|
80
86
|
|
|
81
87
|
export class AlertsStore {
|
|
82
|
-
constructor(
|
|
88
|
+
constructor(
|
|
89
|
+
private readonly db: SqlDb,
|
|
90
|
+
private readonly options: AlertsStoreOptions = {},
|
|
91
|
+
) {}
|
|
83
92
|
|
|
84
93
|
validateInput(input: AlertRuleInput): void {
|
|
85
94
|
if (!input.name?.trim()) throw new Error("name is required");
|
|
@@ -96,8 +105,16 @@ export class AlertsStore {
|
|
|
96
105
|
for (const ch of input.channels) {
|
|
97
106
|
if (ch.type !== "webhook")
|
|
98
107
|
throw new Error(`unknown channel type: ${ch.type}`);
|
|
99
|
-
if (!ch.url ||
|
|
108
|
+
if (!ch.url || typeof ch.url !== "string")
|
|
100
109
|
throw new Error("channel.url must be http(s)://");
|
|
110
|
+
try {
|
|
111
|
+
assertSafeWebhookUrl(ch.url, {
|
|
112
|
+
allowPrivateNetworks: this.options.allowPrivateWebhookTargets,
|
|
113
|
+
});
|
|
114
|
+
} catch (err) {
|
|
115
|
+
if (err instanceof WebhookUrlError) throw new Error(err.message);
|
|
116
|
+
throw err;
|
|
117
|
+
}
|
|
101
118
|
}
|
|
102
119
|
}
|
|
103
120
|
|
|
@@ -217,18 +234,30 @@ export class AlertsStore {
|
|
|
217
234
|
}
|
|
218
235
|
|
|
219
236
|
async deleteRule(id: string, projectId: string): Promise<boolean> {
|
|
220
|
-
|
|
221
|
-
|
|
237
|
+
// Only a rule the caller's project owns takes its state and
|
|
238
|
+
// evaluations with it; a miss must not touch another project's rows.
|
|
239
|
+
// Children go first because alert_state / alert_evaluations reference
|
|
240
|
+
// alert_rules(id) and SQLite enforces that on D1 and node:sqlite.
|
|
241
|
+
const owned = await this.db
|
|
242
|
+
.prepare(
|
|
243
|
+
`SELECT 1 AS ok FROM alert_rules WHERE id = ? AND project_id = ?`,
|
|
244
|
+
)
|
|
222
245
|
.bind(id, projectId)
|
|
223
|
-
.
|
|
224
|
-
|
|
246
|
+
.first<{ ok: number }>();
|
|
247
|
+
if (!owned) return false;
|
|
225
248
|
await this.db
|
|
226
|
-
.prepare(`DELETE FROM alert_state WHERE rule_id = ?`)
|
|
227
|
-
.bind(id)
|
|
249
|
+
.prepare(`DELETE FROM alert_state WHERE rule_id = ? AND project_id = ?`)
|
|
250
|
+
.bind(id, projectId)
|
|
228
251
|
.run();
|
|
229
252
|
await this.db
|
|
230
|
-
.prepare(
|
|
231
|
-
|
|
253
|
+
.prepare(
|
|
254
|
+
`DELETE FROM alert_evaluations WHERE rule_id = ? AND project_id = ?`,
|
|
255
|
+
)
|
|
256
|
+
.bind(id, projectId)
|
|
257
|
+
.run();
|
|
258
|
+
const result = await this.db
|
|
259
|
+
.prepare(`DELETE FROM alert_rules WHERE id = ? AND project_id = ?`)
|
|
260
|
+
.bind(id, projectId)
|
|
232
261
|
.run();
|
|
233
262
|
return (result.meta?.changes ?? 0) > 0;
|
|
234
263
|
}
|
|
@@ -242,28 +271,85 @@ export class AlertsStore {
|
|
|
242
271
|
|
|
243
272
|
// ── State ──
|
|
244
273
|
|
|
245
|
-
async getState(
|
|
274
|
+
async getState(
|
|
275
|
+
ruleId: string,
|
|
276
|
+
projectId: string,
|
|
277
|
+
): Promise<AlertStateRow | null> {
|
|
246
278
|
const row = await this.db
|
|
247
|
-
.prepare(`SELECT * FROM alert_state WHERE rule_id = ?`)
|
|
248
|
-
.bind(ruleId)
|
|
279
|
+
.prepare(`SELECT * FROM alert_state WHERE rule_id = ? AND project_id = ?`)
|
|
280
|
+
.bind(ruleId, projectId)
|
|
249
281
|
.first<AlertStateRow>();
|
|
250
282
|
return row ?? null;
|
|
251
283
|
}
|
|
252
284
|
|
|
285
|
+
/**
|
|
286
|
+
* Record a state transition with its notification still outstanding
|
|
287
|
+
* (`notified = 0`); `markNotified` flips it once a channel accepts the
|
|
288
|
+
* alert. On a database that has not applied migration 049 the column is
|
|
289
|
+
* missing: the transition is still written (legacy statement) so alerting
|
|
290
|
+
* keeps working through a rolling deploy, and the evaluator treats a row
|
|
291
|
+
* without the column as delivered.
|
|
292
|
+
*/
|
|
253
293
|
async transitionState(
|
|
254
294
|
ruleId: string,
|
|
255
295
|
projectId: string,
|
|
256
296
|
newState: AlertState,
|
|
257
297
|
now: string,
|
|
258
298
|
): Promise<void> {
|
|
259
|
-
|
|
260
|
-
.
|
|
261
|
-
|
|
299
|
+
try {
|
|
300
|
+
await this.db
|
|
301
|
+
.prepare(
|
|
302
|
+
`INSERT INTO alert_state (rule_id, project_id, current_state, last_state_change, notified)
|
|
303
|
+
VALUES (?, ?, ?, ?, 0)
|
|
304
|
+
ON CONFLICT(rule_id) DO UPDATE SET
|
|
305
|
+
current_state = excluded.current_state,
|
|
306
|
+
last_state_change = excluded.last_state_change,
|
|
307
|
+
notified = 0`,
|
|
308
|
+
)
|
|
309
|
+
.bind(ruleId, projectId, newState, now)
|
|
310
|
+
.run();
|
|
311
|
+
} catch (err) {
|
|
312
|
+
if (!isMissingNotifiedColumn(err)) throw err;
|
|
313
|
+
await this.db
|
|
314
|
+
.prepare(
|
|
315
|
+
`INSERT INTO alert_state (rule_id, project_id, current_state, last_state_change)
|
|
262
316
|
VALUES (?, ?, ?, ?)
|
|
263
|
-
ON CONFLICT(rule_id) DO UPDATE SET
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
317
|
+
ON CONFLICT(rule_id) DO UPDATE SET
|
|
318
|
+
current_state = excluded.current_state,
|
|
319
|
+
last_state_change = excluded.last_state_change`,
|
|
320
|
+
)
|
|
321
|
+
.bind(ruleId, projectId, newState, now)
|
|
322
|
+
.run();
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
/**
|
|
327
|
+
* Mark the transition `(state, lastStateChange)` as delivered. Scoped to
|
|
328
|
+
* the exact transition the caller notified for: if the rule changed state
|
|
329
|
+
* again while the webhook was in flight, the newer transition keeps its
|
|
330
|
+
* own outstanding flag. Returns false when nothing matched (stale or
|
|
331
|
+
* already marked). On a pre-049 schema there is nothing to mark.
|
|
332
|
+
*/
|
|
333
|
+
async markNotified(
|
|
334
|
+
ruleId: string,
|
|
335
|
+
projectId: string,
|
|
336
|
+
state: AlertState,
|
|
337
|
+
lastStateChange: string,
|
|
338
|
+
): Promise<boolean> {
|
|
339
|
+
try {
|
|
340
|
+
const result = await this.db
|
|
341
|
+
.prepare(
|
|
342
|
+
`UPDATE alert_state SET notified = 1
|
|
343
|
+
WHERE rule_id = ? AND project_id = ? AND current_state = ?
|
|
344
|
+
AND last_state_change = ? AND notified = 0`,
|
|
345
|
+
)
|
|
346
|
+
.bind(ruleId, projectId, state, lastStateChange)
|
|
347
|
+
.run();
|
|
348
|
+
return (result.meta?.changes ?? 0) > 0;
|
|
349
|
+
} catch (err) {
|
|
350
|
+
if (isMissingNotifiedColumn(err)) return false;
|
|
351
|
+
throw err;
|
|
352
|
+
}
|
|
267
353
|
}
|
|
268
354
|
|
|
269
355
|
async recordEvaluation(
|
|
@@ -448,7 +534,9 @@ export class AlertsStore {
|
|
|
448
534
|
sql += ` AND model_name = ?`;
|
|
449
535
|
binds.push(q.model);
|
|
450
536
|
}
|
|
451
|
-
|
|
537
|
+
// Bare boolean test: INTEGER 0/1 on D1, BOOLEAN on Postgres (where
|
|
538
|
+
// `is_error = 1` is a type error).
|
|
539
|
+
if (q.isError === true) sql += ` AND is_error`;
|
|
452
540
|
const row = await this.db
|
|
453
541
|
.prepare(sql)
|
|
454
542
|
.bind(...binds)
|
|
@@ -456,3 +544,16 @@ export class AlertsStore {
|
|
|
456
544
|
return row?.c ?? 0;
|
|
457
545
|
}
|
|
458
546
|
}
|
|
547
|
+
|
|
548
|
+
/**
|
|
549
|
+
* SQLite: "table alert_state has no column named notified" / "no such column:
|
|
550
|
+
* notified"; Postgres: `column "notified" of relation "alert_state" does not
|
|
551
|
+
* exist` (42703). Anything else is a real failure.
|
|
552
|
+
*/
|
|
553
|
+
const isMissingNotifiedColumn = (err: unknown): boolean => {
|
|
554
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
555
|
+
return (
|
|
556
|
+
/notified/i.test(message) &&
|
|
557
|
+
/no such column|has no column|does not exist/i.test(message)
|
|
558
|
+
);
|
|
559
|
+
};
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `runAllDueAnalyses` used to hard-code project `default`, so analyses for
|
|
3
|
+
* every other project never refreshed and alert rules bound to them always
|
|
4
|
+
* evaluated against nothing. These tests run the orchestrator against the
|
|
5
|
+
* real migrated schema and against a database whose projects table fails.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { describe, expect, it } from "vitest";
|
|
9
|
+
import type { CollectorEnv } from "../framework/env";
|
|
10
|
+
import { runAllDueAnalyses } from "./analyses-runner";
|
|
11
|
+
import { ProjectsStore } from "./projects-store";
|
|
12
|
+
import { MemSqlDb } from "./test-utils/mem-sql-db";
|
|
13
|
+
import { createMigratedSqliteDb } from "./test-utils/node-sqlite-db";
|
|
14
|
+
|
|
15
|
+
const silent = { debug() {}, info() {}, warn() {}, error() {} };
|
|
16
|
+
|
|
17
|
+
describe("runAllDueAnalyses", () => {
|
|
18
|
+
it("refreshes and runs analyses for every project, not just default", async () => {
|
|
19
|
+
const db = createMigratedSqliteDb();
|
|
20
|
+
const beta = await new ProjectsStore(db).createProject({
|
|
21
|
+
name: "Beta",
|
|
22
|
+
slug: "beta",
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
const summary = await runAllDueAnalyses({
|
|
26
|
+
env: { DB: db } as unknown as CollectorEnv,
|
|
27
|
+
retentionHours: 24,
|
|
28
|
+
logger: silent,
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
expect(summary.failed).toBe(0);
|
|
32
|
+
expect(summary.ran).toBeGreaterThan(0);
|
|
33
|
+
const projects = await db
|
|
34
|
+
.prepare(
|
|
35
|
+
`SELECT DISTINCT project_id FROM analysis_results ORDER BY project_id`,
|
|
36
|
+
)
|
|
37
|
+
.all<{ project_id: string }>();
|
|
38
|
+
const ids = projects.results.map((r) => r.project_id);
|
|
39
|
+
// 011 seeds `default`, 026 seeds `obs-dashboard`.
|
|
40
|
+
expect(ids).toEqual(
|
|
41
|
+
expect.arrayContaining(["default", "obs-dashboard", beta.id]),
|
|
42
|
+
);
|
|
43
|
+
|
|
44
|
+
const perProject = await db
|
|
45
|
+
.prepare(
|
|
46
|
+
`SELECT project_id, COUNT(*) AS n FROM analysis_results GROUP BY project_id`,
|
|
47
|
+
)
|
|
48
|
+
.all<{ project_id: string; n: number }>();
|
|
49
|
+
const counts = new Map(perProject.results.map((r) => [r.project_id, r.n]));
|
|
50
|
+
expect(counts.get(beta.id)).toBe(counts.get("default"));
|
|
51
|
+
db.close();
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
it("falls back to the default project when projects cannot be listed", async () => {
|
|
55
|
+
const db = new MemSqlDb({
|
|
56
|
+
all: (sql) => {
|
|
57
|
+
if (sql.includes("FROM projects")) throw new Error("no projects table");
|
|
58
|
+
return [];
|
|
59
|
+
},
|
|
60
|
+
first: () => null,
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
const summary = await runAllDueAnalyses({
|
|
64
|
+
env: { DB: db } as unknown as CollectorEnv,
|
|
65
|
+
retentionHours: 24,
|
|
66
|
+
logger: silent,
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
expect(summary).toMatchObject({ ran: 0, failed: 0 });
|
|
70
|
+
expect(summary.refreshed).toBeGreaterThan(0);
|
|
71
|
+
const due = db.callsMatching("FROM analysis_definitions");
|
|
72
|
+
expect(due.length).toBeGreaterThan(0);
|
|
73
|
+
expect(due.every((c) => c.binds[0] === "default")).toBe(true);
|
|
74
|
+
});
|
|
75
|
+
});
|
|
@@ -42,6 +42,7 @@ import {
|
|
|
42
42
|
evaluateGate,
|
|
43
43
|
type NarrativeIntent,
|
|
44
44
|
} from "./narrate-gate";
|
|
45
|
+
import { ProjectsStore } from "./projects-store";
|
|
45
46
|
import { type SqlDb, sqlDbFor } from "./sql-db";
|
|
46
47
|
|
|
47
48
|
export interface AnalysisRunContext {
|
|
@@ -253,8 +254,10 @@ async function narratePass(
|
|
|
253
254
|
* in-process registry (so newly-derived Tier 1 analyses become visible
|
|
254
255
|
* to the dashboard within one tick), then runs whatever's due.
|
|
255
256
|
*
|
|
256
|
-
*
|
|
257
|
-
*
|
|
257
|
+
* Runs once per project in the `projects` table (falling back to the
|
|
258
|
+
* `default` project when the table is empty or unreadable), so analyses and
|
|
259
|
+
* the alert rules bound to them refresh for every tenant, not just the
|
|
260
|
+
* first one.
|
|
258
261
|
*/
|
|
259
262
|
export async function runAllDueAnalyses(
|
|
260
263
|
ctx: {
|
|
@@ -274,7 +277,61 @@ export async function runAllDueAnalyses(
|
|
|
274
277
|
const logger = ctx.logger ?? consoleLogger;
|
|
275
278
|
const db = sqlDbFor(ctx.env);
|
|
276
279
|
const store = new AnalysesStore(db);
|
|
277
|
-
const
|
|
280
|
+
const totals = { ran: 0, failed: 0, refreshed: 0, narrated: 0, skipped: 0 };
|
|
281
|
+
for (const projectId of await listAnalysisProjects(db, logger)) {
|
|
282
|
+
const summary = await runDueAnalysesForProject(projectId, {
|
|
283
|
+
ctx,
|
|
284
|
+
db,
|
|
285
|
+
store,
|
|
286
|
+
logger,
|
|
287
|
+
});
|
|
288
|
+
totals.ran += summary.ran;
|
|
289
|
+
totals.failed += summary.failed;
|
|
290
|
+
totals.refreshed += summary.refreshed;
|
|
291
|
+
totals.narrated += summary.narrated;
|
|
292
|
+
totals.skipped += summary.skipped ?? 0;
|
|
293
|
+
}
|
|
294
|
+
return totals;
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
/** Project ids to run analyses for; `default` when none can be listed. */
|
|
298
|
+
async function listAnalysisProjects(
|
|
299
|
+
db: SqlDb,
|
|
300
|
+
logger: Logger,
|
|
301
|
+
): Promise<string[]> {
|
|
302
|
+
try {
|
|
303
|
+
const ids = (await new ProjectsStore(db).listProjects())
|
|
304
|
+
.map((p) => p.id)
|
|
305
|
+
.filter((id) => typeof id === "string" && id.length > 0);
|
|
306
|
+
if (ids.length > 0) return ids;
|
|
307
|
+
} catch (error) {
|
|
308
|
+
logger.error("[analyses] failed to list projects; running default only", {
|
|
309
|
+
error: error instanceof Error ? error.message : String(error),
|
|
310
|
+
});
|
|
311
|
+
}
|
|
312
|
+
return ["default"];
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
async function runDueAnalysesForProject(
|
|
316
|
+
projectId: string,
|
|
317
|
+
deps: {
|
|
318
|
+
ctx: {
|
|
319
|
+
env: CollectorEnv;
|
|
320
|
+
retentionHours: number;
|
|
321
|
+
tracer?: ChildSpanRunner;
|
|
322
|
+
};
|
|
323
|
+
db: SqlDb;
|
|
324
|
+
store: AnalysesStore;
|
|
325
|
+
logger: Logger;
|
|
326
|
+
},
|
|
327
|
+
): Promise<{
|
|
328
|
+
ran: number;
|
|
329
|
+
failed: number;
|
|
330
|
+
refreshed: number;
|
|
331
|
+
narrated: number;
|
|
332
|
+
skipped?: number;
|
|
333
|
+
}> {
|
|
334
|
+
const { ctx, db, store, logger } = deps;
|
|
278
335
|
const now = Date.now();
|
|
279
336
|
const expiresAt = now + ctx.retentionHours * 3600 * 1000;
|
|
280
337
|
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import type { AnalysisResult } from "@obsunified/types";
|
|
2
|
+
import { describe, expect, it } from "vitest";
|
|
3
|
+
import { AnalysesStore } from "./analyses-store";
|
|
4
|
+
import { createMigratedSqliteDb } from "./test-utils/node-sqlite-db";
|
|
5
|
+
|
|
6
|
+
const result = (
|
|
7
|
+
analysisId: string,
|
|
8
|
+
generatedAt: string,
|
|
9
|
+
status: AnalysisResult["status"],
|
|
10
|
+
): AnalysisResult => ({
|
|
11
|
+
analysisId,
|
|
12
|
+
projectId: "alpha",
|
|
13
|
+
generatedAt,
|
|
14
|
+
paramsHash: null,
|
|
15
|
+
status,
|
|
16
|
+
primaryValue: 1,
|
|
17
|
+
baselineValue: 0,
|
|
18
|
+
deltaPct: null,
|
|
19
|
+
payload: {},
|
|
20
|
+
narrative: null,
|
|
21
|
+
narrativeSignature: null,
|
|
22
|
+
durationMs: 1,
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
describe("AnalysesStore.getLatestResultsBulk", () => {
|
|
26
|
+
it("returns the newest result per analysis from a single bounded query", async () => {
|
|
27
|
+
const db = createMigratedSqliteDb();
|
|
28
|
+
const store = new AnalysesStore(db);
|
|
29
|
+
const expires = Date.now() + 3_600_000;
|
|
30
|
+
for (const [id, at, status] of [
|
|
31
|
+
["a", "2026-10-13T01:00:00.000Z", "ok"],
|
|
32
|
+
["a", "2026-10-13T03:00:00.000Z", "critical"],
|
|
33
|
+
["a", "2026-10-13T02:00:00.000Z", "warn"],
|
|
34
|
+
["b", "2026-10-13T02:30:00.000Z", "warn"],
|
|
35
|
+
["c", "2026-10-13T02:30:00.000Z", "ok"],
|
|
36
|
+
] as const) {
|
|
37
|
+
await store.insertResult(result(id, at, status), expires);
|
|
38
|
+
}
|
|
39
|
+
// Another project with the same ids must not bleed through.
|
|
40
|
+
await store.insertResult(
|
|
41
|
+
{ ...result("a", "2026-10-13T09:00:00.000Z", "ok"), projectId: "beta" },
|
|
42
|
+
expires,
|
|
43
|
+
);
|
|
44
|
+
|
|
45
|
+
const latest = await store.getLatestResultsBulk("alpha", ["a", "b", "zz"]);
|
|
46
|
+
expect([...latest.keys()].sort()).toEqual(["a", "b"]);
|
|
47
|
+
expect(latest.get("a")?.status).toBe("critical");
|
|
48
|
+
expect(latest.get("a")?.generatedAt).toBe("2026-10-13T03:00:00.000Z");
|
|
49
|
+
expect(latest.get("b")?.status).toBe("warn");
|
|
50
|
+
|
|
51
|
+
// The query itself must not return every retained row.
|
|
52
|
+
const rows = await db
|
|
53
|
+
.prepare(
|
|
54
|
+
`SELECT COUNT(*) AS n FROM analysis_results WHERE project_id = ? AND analysis_id = ?`,
|
|
55
|
+
)
|
|
56
|
+
.bind("alpha", "a")
|
|
57
|
+
.first<{ n: number }>();
|
|
58
|
+
expect(rows?.n).toBe(3);
|
|
59
|
+
db.close();
|
|
60
|
+
});
|
|
61
|
+
});
|
|
@@ -293,20 +293,27 @@ export class AnalysesStore {
|
|
|
293
293
|
);
|
|
294
294
|
if (analysisIds.length === 0) return new Map();
|
|
295
295
|
|
|
296
|
-
//
|
|
297
|
-
//
|
|
298
|
-
//
|
|
296
|
+
// One row per analysis: join against the per-analysis MAX(generated_at)
|
|
297
|
+
// so the database returns only the newest result for each id instead
|
|
298
|
+
// of every retained row (a 60 s refresh over a 72 h window is ~4k rows
|
|
299
|
+
// per analysis). Ties on generated_at are broken in JS below.
|
|
299
300
|
const placeholders = analysisIds.map(() => "?").join(", ");
|
|
300
301
|
const result = await this.db
|
|
301
302
|
.prepare(
|
|
302
|
-
`SELECT id, project_id, analysis_id, generated_at, params_hash,
|
|
303
|
-
status, primary_value, baseline_value, delta_pct, payload_json,
|
|
304
|
-
narrative, narrative_signature, duration_ms, expires_at
|
|
305
|
-
FROM analysis_results
|
|
306
|
-
|
|
307
|
-
|
|
303
|
+
`SELECT r.id, r.project_id, r.analysis_id, r.generated_at, r.params_hash,
|
|
304
|
+
r.status, r.primary_value, r.baseline_value, r.delta_pct, r.payload_json,
|
|
305
|
+
r.narrative, r.narrative_signature, r.duration_ms, r.expires_at
|
|
306
|
+
FROM analysis_results r
|
|
307
|
+
JOIN (
|
|
308
|
+
SELECT analysis_id, MAX(generated_at) AS latest_at
|
|
309
|
+
FROM analysis_results
|
|
310
|
+
WHERE project_id = ? AND analysis_id IN (${placeholders})
|
|
311
|
+
GROUP BY analysis_id
|
|
312
|
+
) m ON m.analysis_id = r.analysis_id AND m.latest_at = r.generated_at
|
|
313
|
+
WHERE r.project_id = ?
|
|
314
|
+
ORDER BY r.generated_at DESC, r.id DESC`,
|
|
308
315
|
)
|
|
309
|
-
.bind(projectId, ...analysisIds)
|
|
316
|
+
.bind(projectId, ...analysisIds, projectId)
|
|
310
317
|
.all<ResultRow>();
|
|
311
318
|
|
|
312
319
|
const latest = new Map<string, AnalysisResult>();
|
package/src/lib/blob-store.ts
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* - `R2BlobStore` — Cloudflare R2 binding (default on Workers).
|
|
7
7
|
* - `S3BlobStore` (sibling file) — AWS S3 / MinIO / any S3-compatible.
|
|
8
8
|
*
|
|
9
|
-
* Replay chunks live under `replays/<session_id>/<chunk_id>`; pprof
|
|
9
|
+
* Replay chunks live under `replays/<project_id>/<session_id>/<chunk_id>`; pprof
|
|
10
10
|
* profiles live under `profiles/<service>/<trace_id>/<span_id>`. The
|
|
11
11
|
* adapter never sees these paths — they're constructed by the callers
|
|
12
12
|
* (`replay-receiver`, `profile-routes`).
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
import { purgeSecondaryRows } from "./housekeeping";
|
|
3
|
+
import { createMigratedSqliteDb } from "./test-utils/node-sqlite-db";
|
|
4
|
+
|
|
5
|
+
const HOUR = 3_600_000;
|
|
6
|
+
const NOW = Date.UTC(2026, 9, 13, 12, 0, 0);
|
|
7
|
+
const iso = (hoursAgo: number) => new Date(NOW - hoursAgo * HOUR).toISOString();
|
|
8
|
+
|
|
9
|
+
describe("purgeSecondaryRows", () => {
|
|
10
|
+
it("removes only rows no API window or ingest path can still reach", async () => {
|
|
11
|
+
const db = createMigratedSqliteDb();
|
|
12
|
+
const raw = db.raw;
|
|
13
|
+
raw.exec(`
|
|
14
|
+
INSERT INTO projects (id, name, slug, created_at) VALUES ('alpha', 'A', 'a', '${iso(1000)}');
|
|
15
|
+
INSERT INTO alert_rules (id, project_id, name, signal, query_json, threshold, window_mins, comparison, channels_json, enabled, created_at, updated_at)
|
|
16
|
+
VALUES ('r1', 'alpha', 'r', 'spans', '{}', 1, 5, '>', '[]', 1, '${iso(1000)}', '${iso(1000)}');
|
|
17
|
+
`);
|
|
18
|
+
const ev = raw.prepare(
|
|
19
|
+
`INSERT INTO alert_evaluations (id, rule_id, project_id, evaluated_at, value, state, notified) VALUES (?, 'r1', 'alpha', ?, 1, 'ok', 0)`,
|
|
20
|
+
);
|
|
21
|
+
ev.run("old", iso(721));
|
|
22
|
+
ev.run("edge", iso(719));
|
|
23
|
+
const ask = raw.prepare(
|
|
24
|
+
`INSERT INTO ask_evidence_events (project_id, analysis_id, asked_at) VALUES ('alpha', 'x', ?)`,
|
|
25
|
+
);
|
|
26
|
+
ask.run(NOW - 31 * 24 * HOUR);
|
|
27
|
+
ask.run(NOW - 29 * 24 * HOUR);
|
|
28
|
+
const series = raw.prepare(
|
|
29
|
+
`INSERT INTO metric_series (id, project_id, name, type, identity, created_at) VALUES (?, 'alpha', 'm', 'gauge', ?, ?)`,
|
|
30
|
+
);
|
|
31
|
+
series.run("orphan-old", "i1", iso(100)); // no points, older than window
|
|
32
|
+
series.run("orphan-new", "i2", iso(1)); // no points but just created
|
|
33
|
+
series.run("live", "i3", iso(100)); // old but still has a point
|
|
34
|
+
raw
|
|
35
|
+
.prepare(
|
|
36
|
+
`INSERT INTO metric_point (id, series_id, project_id, ts_ns, received_at, expires_at) VALUES ('p1', 'live', 'alpha', '1', ?, ?)`,
|
|
37
|
+
)
|
|
38
|
+
.run(iso(1), iso(-1));
|
|
39
|
+
const ref = raw.prepare(
|
|
40
|
+
`INSERT INTO evidence_retrieval_refs (ref_id, project_id, kind, anchor_kind, anchor_id, source, expires_at) VALUES (?, 'alpha', 'logs', 'trace', 't', 'test', ?)`,
|
|
41
|
+
);
|
|
42
|
+
ref.run("expired", iso(1));
|
|
43
|
+
ref.run("fresh", iso(-1));
|
|
44
|
+
ref.run("no-expiry", null);
|
|
45
|
+
const exp = raw.prepare(
|
|
46
|
+
`INSERT INTO evidence_ref_expansions (id, ref_id, project_id, kind, operation, result_status) VALUES (?, ?, 'alpha', 'logs', 'expand', 'ok')`,
|
|
47
|
+
);
|
|
48
|
+
exp.run("e1", "expired");
|
|
49
|
+
exp.run("e2", "fresh");
|
|
50
|
+
|
|
51
|
+
const summary = await purgeSecondaryRows(db, {
|
|
52
|
+
retentionHours: 72,
|
|
53
|
+
now: () => NOW,
|
|
54
|
+
});
|
|
55
|
+
expect(summary).toEqual({
|
|
56
|
+
alert_evaluations: 1,
|
|
57
|
+
ask_evidence_events: 1,
|
|
58
|
+
metric_series: 1,
|
|
59
|
+
evidence_refs: 1,
|
|
60
|
+
evidence_ref_expansions: 1,
|
|
61
|
+
});
|
|
62
|
+
const ids = (sql: string) =>
|
|
63
|
+
(raw.prepare(sql).all() as Array<{ id: string }>).map((r) => r.id).sort();
|
|
64
|
+
expect(ids("SELECT id FROM alert_evaluations")).toEqual(["edge"]);
|
|
65
|
+
expect(ids("SELECT id FROM metric_series")).toEqual(["live", "orphan-new"]);
|
|
66
|
+
expect(ids("SELECT ref_id AS id FROM evidence_retrieval_refs")).toEqual([
|
|
67
|
+
"fresh",
|
|
68
|
+
"no-expiry",
|
|
69
|
+
]);
|
|
70
|
+
expect(ids("SELECT id FROM evidence_ref_expansions")).toEqual(["e2"]);
|
|
71
|
+
db.close();
|
|
72
|
+
});
|
|
73
|
+
});
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Retention for the secondary tables the hourly sweep did not cover. Each
|
|
3
|
+
* rule is tied to an existing contract rather than a new policy:
|
|
4
|
+
*
|
|
5
|
+
* - `alert_evaluations`: `/internal/alerts/evaluations` accepts at most a
|
|
6
|
+
* 720-hour window, so rows older than that are unreachable through the API.
|
|
7
|
+
* - `ask_evidence_events`: feeds `getTopAskedAnalyses`, whose trailing window
|
|
8
|
+
* is 7 days; rows older than 30 days cannot influence it.
|
|
9
|
+
* - `metric_series`: identity rows whose points have all expired and that
|
|
10
|
+
* were created before the retention window. They hold no data; the next
|
|
11
|
+
* ingest of that identity recreates the row.
|
|
12
|
+
* - `evidence_retrieval_refs`: rows carry their own `expires_at` (copied from
|
|
13
|
+
* the signal they reference); expansions follow their ref.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import type { SqlDb } from "./sql-db";
|
|
17
|
+
|
|
18
|
+
export const ALERT_EVALUATIONS_MAX_AGE_HOURS = 720;
|
|
19
|
+
export const ASK_EVIDENCE_EVENTS_MAX_AGE_HOURS = 30 * 24;
|
|
20
|
+
|
|
21
|
+
export interface HousekeepingSummary {
|
|
22
|
+
alert_evaluations: number;
|
|
23
|
+
ask_evidence_events: number;
|
|
24
|
+
metric_series: number;
|
|
25
|
+
evidence_refs: number;
|
|
26
|
+
evidence_ref_expansions: number;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export interface HousekeepingOptions {
|
|
30
|
+
/** Window that bounds orphaned metric series; same value the main sweep uses. */
|
|
31
|
+
retentionHours: number;
|
|
32
|
+
/** Clock override for tests. */
|
|
33
|
+
now?: () => number;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const changes = (r: { meta?: { changes?: number } }) => r.meta?.changes ?? 0;
|
|
37
|
+
|
|
38
|
+
export async function purgeSecondaryRows(
|
|
39
|
+
db: SqlDb,
|
|
40
|
+
options: HousekeepingOptions,
|
|
41
|
+
): Promise<HousekeepingSummary> {
|
|
42
|
+
const nowMs = options.now?.() ?? Date.now();
|
|
43
|
+
const isoBefore = (hours: number) =>
|
|
44
|
+
new Date(nowMs - hours * 3_600_000).toISOString();
|
|
45
|
+
|
|
46
|
+
const alertEvaluations = await db
|
|
47
|
+
.prepare(`DELETE FROM alert_evaluations WHERE evaluated_at < ?`)
|
|
48
|
+
.bind(isoBefore(ALERT_EVALUATIONS_MAX_AGE_HOURS))
|
|
49
|
+
.run();
|
|
50
|
+
|
|
51
|
+
const askEvents = await db
|
|
52
|
+
.prepare(`DELETE FROM ask_evidence_events WHERE asked_at < ?`)
|
|
53
|
+
.bind(nowMs - ASK_EVIDENCE_EVENTS_MAX_AGE_HOURS * 3_600_000)
|
|
54
|
+
.run();
|
|
55
|
+
|
|
56
|
+
// Points and exemplars for the window are purged by MetricsStore first;
|
|
57
|
+
// a series created before the window with no remaining points is inert.
|
|
58
|
+
const series = await db
|
|
59
|
+
.prepare(
|
|
60
|
+
`DELETE FROM metric_series
|
|
61
|
+
WHERE created_at < ?
|
|
62
|
+
AND NOT EXISTS (
|
|
63
|
+
SELECT 1 FROM metric_point p WHERE p.series_id = metric_series.id
|
|
64
|
+
)`,
|
|
65
|
+
)
|
|
66
|
+
.bind(isoBefore(options.retentionHours))
|
|
67
|
+
.run();
|
|
68
|
+
|
|
69
|
+
const refs = await db
|
|
70
|
+
.prepare(
|
|
71
|
+
`DELETE FROM evidence_retrieval_refs
|
|
72
|
+
WHERE expires_at IS NOT NULL AND expires_at < ?`,
|
|
73
|
+
)
|
|
74
|
+
.bind(isoBefore(0))
|
|
75
|
+
.run();
|
|
76
|
+
const expansions = await db
|
|
77
|
+
.prepare(
|
|
78
|
+
`DELETE FROM evidence_ref_expansions
|
|
79
|
+
WHERE NOT EXISTS (
|
|
80
|
+
SELECT 1 FROM evidence_retrieval_refs r
|
|
81
|
+
WHERE r.ref_id = evidence_ref_expansions.ref_id
|
|
82
|
+
)`,
|
|
83
|
+
)
|
|
84
|
+
.run();
|
|
85
|
+
|
|
86
|
+
return {
|
|
87
|
+
alert_evaluations: changes(alertEvaluations),
|
|
88
|
+
ask_evidence_events: changes(askEvents),
|
|
89
|
+
metric_series: changes(series),
|
|
90
|
+
evidence_refs: changes(refs),
|
|
91
|
+
evidence_ref_expansions: changes(expansions),
|
|
92
|
+
};
|
|
93
|
+
}
|
|
@@ -313,10 +313,10 @@ export async function manifestByAction(
|
|
|
313
313
|
.prepare(
|
|
314
314
|
`SELECT session_id, first_chunk_at, last_chunk_at, chunk_count, events_count
|
|
315
315
|
FROM session_replay_metadata
|
|
316
|
-
WHERE session_id IN (${sessionPlaceholders})
|
|
316
|
+
WHERE project_id = ? AND session_id IN (${sessionPlaceholders})
|
|
317
317
|
ORDER BY last_chunk_at DESC LIMIT 1`,
|
|
318
318
|
)
|
|
319
|
-
.bind(...sessionIds)
|
|
319
|
+
.bind(projectId, ...sessionIds)
|
|
320
320
|
.first<{
|
|
321
321
|
session_id: string;
|
|
322
322
|
first_chunk_at: string;
|
|
@@ -546,10 +546,10 @@ export async function manifestByAgentRun(
|
|
|
546
546
|
.prepare(
|
|
547
547
|
`SELECT session_id, first_chunk_at, last_chunk_at, chunk_count, events_count
|
|
548
548
|
FROM session_replay_metadata
|
|
549
|
-
WHERE session_id IN (${sessionPlaceholders})
|
|
549
|
+
WHERE project_id = ? AND session_id IN (${sessionPlaceholders})
|
|
550
550
|
ORDER BY last_chunk_at DESC LIMIT 1`,
|
|
551
551
|
)
|
|
552
|
-
.bind(...sessionIds)
|
|
552
|
+
.bind(projectId, ...sessionIds)
|
|
553
553
|
.first<{
|
|
554
554
|
session_id: string;
|
|
555
555
|
first_chunk_at: string;
|
|
@@ -767,10 +767,10 @@ export async function manifestByActor(
|
|
|
767
767
|
.prepare(
|
|
768
768
|
`SELECT session_id, first_chunk_at, last_chunk_at, chunk_count, events_count
|
|
769
769
|
FROM session_replay_metadata
|
|
770
|
-
WHERE session_id IN (${sessionPlaceholders})
|
|
770
|
+
WHERE project_id = ? AND session_id IN (${sessionPlaceholders})
|
|
771
771
|
ORDER BY last_chunk_at DESC LIMIT 1`,
|
|
772
772
|
)
|
|
773
|
-
.bind(...sessionIds)
|
|
773
|
+
.bind(projectId, ...sessionIds)
|
|
774
774
|
.first<{
|
|
775
775
|
session_id: string;
|
|
776
776
|
first_chunk_at: string;
|