@obsunified/collector 2.0.1 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. package/dist/auth.d.ts +1 -1
  2. package/dist/auth.js +1 -1
  3. package/dist/{chunk-UGWDOGGH.js → chunk-KMEZKGSO.js} +22 -10
  4. package/dist/chunk-KMEZKGSO.js.map +1 -0
  5. package/dist/{chunk-XKQTVIRE.js → chunk-ZSF5XP3B.js} +8 -2
  6. package/dist/chunk-ZSF5XP3B.js.map +1 -0
  7. package/dist/index.d.ts +630 -27
  8. package/dist/index.js +5819 -853
  9. package/dist/index.js.map +1 -1
  10. package/dist/{ingest-auth-HFVO1zIp.d.ts → ingest-auth-C5-8Mly5.d.ts} +9 -0
  11. package/dist/{openai-FJYOQ7WO.js → openai-GSJZGFGL.js} +2 -2
  12. package/package.json +5 -5
  13. package/src/__tests__/autosre-closed-loop.test.ts +1275 -0
  14. package/src/analyses/derive.ts +26 -25
  15. package/src/analyses/investigations.ts +8 -8
  16. package/src/analyses/tier0.ts +22 -22
  17. package/src/auth/dashboard-auth.test.ts +791 -0
  18. package/src/auth/dashboard-auth.ts +187 -38
  19. package/src/auth/login-rate-limit.test.ts +129 -0
  20. package/src/auth/login-rate-limit.ts +143 -0
  21. package/src/framework/body-limit.test.ts +165 -0
  22. package/src/framework/collector.ts +355 -115
  23. package/src/framework/env.ts +9 -0
  24. package/src/framework/limits.ts +31 -0
  25. package/src/framework/scheduled-handlers.test.ts +342 -0
  26. package/src/index.ts +59 -1
  27. package/src/lib/action-retention.test.ts +472 -0
  28. package/src/lib/action-retention.ts +420 -0
  29. package/src/lib/actions-store.ts +6 -12
  30. package/src/lib/ai-store/calls.ts +2 -2
  31. package/src/lib/ai-store/sessions.ts +3 -3
  32. package/src/lib/ai-store/spans.ts +1 -1
  33. package/src/lib/ai-store/types.ts +2 -1
  34. package/src/lib/alerts-store.test.ts +153 -0
  35. package/src/lib/alerts-store.ts +122 -21
  36. package/src/lib/analyses-runner.test.ts +75 -0
  37. package/src/lib/analyses-runner.ts +69 -3
  38. package/src/lib/analyses-store-latest.test.ts +61 -0
  39. package/src/lib/analyses-store.ts +17 -10
  40. package/src/lib/autosre-change-correlation.ts +780 -0
  41. package/src/lib/autosre-findings.ts +725 -0
  42. package/src/lib/autosre-ontology.ts +914 -0
  43. package/src/lib/autosre-remediation-gate.ts +1663 -0
  44. package/src/lib/blob-store.ts +1 -1
  45. package/src/lib/eval-cases-store.ts +63 -1
  46. package/src/lib/evidence-references.ts +6 -0
  47. package/src/lib/housekeeping.test.ts +153 -0
  48. package/src/lib/housekeeping.ts +193 -0
  49. package/src/lib/identity-index/action-lookups.ts +6 -6
  50. package/src/lib/identity-index/key-lookups.ts +2 -2
  51. package/src/lib/identity-index/user-lookup.ts +8 -5
  52. package/src/lib/identity-index.test.ts +2 -3
  53. package/src/lib/metrics-store-series.test.ts +99 -0
  54. package/src/lib/metrics-store.test.ts +23 -0
  55. package/src/lib/metrics-store.ts +27 -3
  56. package/src/lib/projects-store.test.ts +25 -0
  57. package/src/lib/projects-store.ts +4 -3
  58. package/src/lib/sql-db-dialect.test.ts +12 -3
  59. package/src/lib/sql-db-postgres.integration.test.ts +1225 -0
  60. package/src/lib/sql-db-postgres.test.ts +314 -0
  61. package/src/lib/sql-db-postgres.ts +392 -54
  62. package/src/lib/sql-db-sqlite-time-windows.test.ts +194 -0
  63. package/src/lib/sql-db.ts +40 -6
  64. package/src/lib/store/ingest.ts +94 -6
  65. package/src/lib/test-utils/action-graph-fixtures.ts +307 -0
  66. package/src/lib/test-utils/node-sqlite-db.ts +188 -0
  67. package/src/lib/webhook-url.test.ts +122 -0
  68. package/src/lib/webhook-url.ts +155 -0
  69. package/src/migrations/041_ingest_key_hash_text.sql +5 -0
  70. package/src/migrations/042_session_replay_project_composite_pk.sql +52 -0
  71. package/src/migrations/043_metric_exemplars_text_timestamps.sql +5 -0
  72. package/src/migrations/044_actions_started_at_index.sql +10 -0
  73. package/src/migrations/045_user_profiles_project_composite_pk.sql +40 -0
  74. package/src/migrations/046_action_graph_project_composite_pk.sql +255 -0
  75. package/src/migrations/047_ai_span_payloads_project_composite_pk.sql +61 -0
  76. package/src/migrations/048_telemetry_spans_project_composite_pk.sql +92 -0
  77. package/src/migrations/049_alert_state_notified.sql +17 -0
  78. package/src/migrations/050_autosre_causal_ontology.sql +216 -0
  79. package/src/migrations/alert-state-notified.test.ts +111 -0
  80. package/src/migrations/parity.test.ts +68 -0
  81. package/src/migrations/project-scoped-keys.test.ts +432 -0
  82. package/src/migrations-postgres/009_projects_and_keys.sql +4 -4
  83. package/src/migrations-postgres/026_obs_dashboard_project_seed.sql +1 -1
  84. package/src/migrations-postgres/035_metric_exemplars.sql +5 -2
  85. package/src/migrations-postgres/041_ingest_key_hash_text.sql +32 -0
  86. package/src/migrations-postgres/042_session_replay_project_composite_pk.sql +16 -0
  87. package/src/migrations-postgres/043_metric_exemplars_text_timestamps.sql +47 -0
  88. package/src/migrations-postgres/044_actions_started_at_index.sql +8 -0
  89. package/src/migrations-postgres/045_user_profiles_project_composite_pk.sql +33 -0
  90. package/src/migrations-postgres/046_action_graph_project_composite_pk.sql +39 -0
  91. package/src/migrations-postgres/047_ai_span_payloads_project_composite_pk.sql +30 -0
  92. package/src/migrations-postgres/048_telemetry_spans_project_composite_pk.sql +37 -0
  93. package/src/migrations-postgres/049_alert_state_notified.sql +10 -0
  94. package/src/migrations-postgres/050_autosre_causal_ontology.sql +216 -0
  95. package/src/migrations-postgres/README.md +58 -5
  96. package/src/otlp/decode/body.test.ts +76 -0
  97. package/src/otlp/decode/body.ts +67 -6
  98. package/src/plugins/_json.ts +21 -0
  99. package/src/plugins/action-graph-processor.ts +6 -12
  100. package/src/plugins/ai-receiver.ts +5 -12
  101. package/src/plugins/ai-span-payloads-processor.ts +1 -2
  102. package/src/plugins/alerts-evaluator-retention.test.ts +170 -0
  103. package/src/plugins/alerts-evaluator.test.ts +464 -3
  104. package/src/plugins/alerts-evaluator.ts +181 -41
  105. package/src/plugins/alerts-routes.ts +10 -6
  106. package/src/plugins/ask-routes.ts +4 -5
  107. package/src/plugins/autosre-routes.ts +1019 -0
  108. package/src/plugins/connected-routes/manifest.ts +13 -1
  109. package/src/plugins/connected-routes.test.ts +106 -0
  110. package/src/plugins/connected-routes.ts +285 -5
  111. package/src/plugins/dashboard-routes.test.ts +120 -0
  112. package/src/plugins/dashboard-routes.ts +11 -2
  113. package/src/plugins/default-span-enrichment.ts +13 -1
  114. package/src/plugins/eval-cases-routes.ts +65 -19
  115. package/src/plugins/evidence-retrieval-routes.ts +300 -4
  116. package/src/plugins/identity-receiver.ts +9 -11
  117. package/src/plugins/json-body.test.ts +91 -0
  118. package/src/plugins/onboarding-routes.test.ts +105 -0
  119. package/src/plugins/onboarding-routes.ts +26 -11
  120. package/src/plugins/projects-routes.ts +2 -1
  121. package/src/plugins/replay-isolation.test.ts +922 -0
  122. package/src/plugins/replay-receiver.ts +8 -8
  123. package/src/plugins/usage-receiver.ts +3 -6
  124. package/dist/chunk-UGWDOGGH.js.map +0 -1
  125. package/dist/chunk-XKQTVIRE.js.map +0 -1
  126. /package/dist/{openai-FJYOQ7WO.js.map → openai-GSJZGFGL.js.map} +0 -0
@@ -0,0 +1,16 @@
1
+ -- Migration 042: session replay composite primary key (project_id, session_id)
2
+ --
3
+ -- Fixes cross-project session replay metadata collisions. Previously,
4
+ -- session_replay_metadata used session_id alone as primary key while
5
+ -- project_id was added in migration 010. Replay receiver upserts used
6
+ -- ON CONFLICT(session_id), causing identical session IDs from different
7
+ -- projects to overwrite or increment each other.
8
+ --
9
+ -- This forward migration drops the existing single-column primary key constraint
10
+ -- and establishes PRIMARY KEY (project_id, session_id) while preserving all
11
+ -- existing rows and indexes.
12
+
13
+ UPDATE session_replay_metadata SET project_id = 'default' WHERE project_id IS NULL;
14
+ ALTER TABLE session_replay_metadata ALTER COLUMN project_id SET NOT NULL;
15
+ ALTER TABLE session_replay_metadata DROP CONSTRAINT IF EXISTS session_replay_metadata_pkey;
16
+ ALTER TABLE session_replay_metadata ADD PRIMARY KEY (project_id, session_id);
@@ -0,0 +1,47 @@
1
+ -- Convert metric_exemplars.received_at / expires_at from TIMESTAMPTZ to TEXT
2
+ -- on databases migrated with the original Postgres 035.
3
+ --
4
+ -- Every other retention table stores timestamps as TEXT holding
5
+ -- Date#toISOString() output, and postgresDialect renders now() as ISO text
6
+ -- (to_char(..., 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')). Against TIMESTAMPTZ
7
+ -- columns MetricsStore.purgeExpired failed with
8
+ -- operator does not exist: timestamp with time zone < text
9
+ -- so metric exemplars were never purged on Postgres.
10
+ --
11
+ -- Existing rows are preserved: each value is rendered in UTC in the exact
12
+ -- shape the collector writes (millisecond precision, trailing "Z"), so the
13
+ -- converted text compares correctly against freshly ingested rows and the
14
+ -- dialect's now(). idx_metric_exemplars_expires is rebuilt by Postgres as
15
+ -- part of the type change.
16
+ --
17
+ -- Fresh installs already create both columns as TEXT in 035 and this is a
18
+ -- no-op. Re-running is also a no-op.
19
+ DO $$
20
+ BEGIN
21
+ IF EXISTS (
22
+ SELECT 1
23
+ FROM information_schema.columns
24
+ WHERE table_schema = current_schema()
25
+ AND table_name = 'metric_exemplars'
26
+ AND column_name = 'received_at'
27
+ AND data_type = 'timestamp with time zone'
28
+ ) THEN
29
+ ALTER TABLE metric_exemplars
30
+ ALTER COLUMN received_at TYPE TEXT
31
+ USING to_char(received_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"');
32
+ END IF;
33
+
34
+ IF EXISTS (
35
+ SELECT 1
36
+ FROM information_schema.columns
37
+ WHERE table_schema = current_schema()
38
+ AND table_name = 'metric_exemplars'
39
+ AND column_name = 'expires_at'
40
+ AND data_type = 'timestamp with time zone'
41
+ ) THEN
42
+ ALTER TABLE metric_exemplars
43
+ ALTER COLUMN expires_at TYPE TEXT
44
+ USING to_char(expires_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"');
45
+ END IF;
46
+ END
47
+ $$;
@@ -0,0 +1,8 @@
1
+ -- RFC 0010 — action graph retention.
2
+ --
3
+ -- Mirrors ../migrations/044_actions_started_at_index.sql. `started_at` is
4
+ -- ISO-8601 TEXT on both backends, so the retention sweep compares it against
5
+ -- a text cutoff rendered the same way `postgresDialect.sinceHours` does.
6
+
7
+ CREATE INDEX IF NOT EXISTS idx_actions_started_at
8
+ ON actions (started_at);
@@ -0,0 +1,33 @@
1
+ -- Migration 045: user_profiles composite primary key (project_id, user_id)
2
+ --
3
+ -- Mirrors ../migrations/045_user_profiles_project_composite_pk.sql. The 006
4
+ -- table used `user_id TEXT PRIMARY KEY`; 010 added project_id outside the
5
+ -- key, so /v1/identify's ON CONFLICT(user_id) upsert let one project rewrite
6
+ -- another project's profile. Swaps the key for (project_id, user_id) in
7
+ -- place, keeping every row and index. The old key was unique, so the
8
+ -- composite key cannot collide.
9
+ --
10
+ -- Idempotent: the block looks up the current primary key by definition and
11
+ -- does nothing when it is already (project_id, user_id).
12
+
13
+ UPDATE user_profiles SET project_id = 'default' WHERE project_id IS NULL;
14
+ ALTER TABLE user_profiles ALTER COLUMN project_id SET NOT NULL;
15
+
16
+ DO $$
17
+ DECLARE
18
+ pk_name TEXT;
19
+ pk_def TEXT;
20
+ BEGIN
21
+ SELECT c.conname, pg_get_constraintdef(c.oid)
22
+ INTO pk_name, pk_def
23
+ FROM pg_constraint c
24
+ WHERE c.conrelid = 'user_profiles'::regclass AND c.contype = 'p';
25
+
26
+ IF pk_def IS DISTINCT FROM 'PRIMARY KEY (project_id, user_id)' THEN
27
+ IF pk_name IS NOT NULL THEN
28
+ EXECUTE format('ALTER TABLE user_profiles DROP CONSTRAINT %I', pk_name);
29
+ END IF;
30
+ ALTER TABLE user_profiles ADD PRIMARY KEY (project_id, user_id);
31
+ END IF;
32
+ END
33
+ $$;
@@ -0,0 +1,39 @@
1
+ -- Migration 046: action graph tables keyed by (project_id, id)
2
+ --
3
+ -- Mirrors ../migrations/046_action_graph_project_composite_pk.sql. The 031
4
+ -- spine and leaf tables used `id TEXT PRIMARY KEY`; action ids are
5
+ -- client-supplied and leaf ids are raw OTel span ids, so two projects can
6
+ -- emit the same id and the ON CONFLICT(id) upserts let one tenant overwrite
7
+ -- another's row. Swaps each key for (project_id, id) in place, keeping every
8
+ -- row and index (031, 038, 044). The old keys were unique, so the composite
9
+ -- keys cannot collide.
10
+ --
11
+ -- Idempotent: each table is skipped when its primary key is already
12
+ -- (project_id, id).
13
+
14
+ DO $$
15
+ DECLARE
16
+ t TEXT;
17
+ pk_name TEXT;
18
+ pk_def TEXT;
19
+ BEGIN
20
+ FOREACH t IN ARRAY ARRAY[
21
+ 'actions', 'agent_runs', 'tool_calls', 'retrieval_events',
22
+ 'eval_results', 'artifacts'
23
+ ] LOOP
24
+ pk_name := NULL;
25
+ pk_def := NULL;
26
+ SELECT c.conname, pg_get_constraintdef(c.oid)
27
+ INTO pk_name, pk_def
28
+ FROM pg_constraint c
29
+ WHERE c.conrelid = t::regclass AND c.contype = 'p';
30
+
31
+ IF pk_def IS DISTINCT FROM 'PRIMARY KEY (project_id, id)' THEN
32
+ IF pk_name IS NOT NULL THEN
33
+ EXECUTE format('ALTER TABLE %I DROP CONSTRAINT %I', t, pk_name);
34
+ END IF;
35
+ EXECUTE format('ALTER TABLE %I ADD PRIMARY KEY (project_id, id)', t);
36
+ END IF;
37
+ END LOOP;
38
+ END
39
+ $$;
@@ -0,0 +1,30 @@
1
+ -- Migration 047: ai_span_payloads composite primary key
2
+ -- (project_id, trace_id, span_id)
3
+ --
4
+ -- Mirrors ../migrations/047_ai_span_payloads_project_composite_pk.sql. 019
5
+ -- keyed the payload side table by (trace_id, span_id) only; the processor's
6
+ -- ON CONFLICT(trace_id, span_id) upsert let a second project replace the
7
+ -- first project's prompt/completion payloads. Swaps the key for
8
+ -- (project_id, trace_id, span_id) in place, keeping every row and index.
9
+ -- The old key was unique, so the composite key cannot collide.
10
+ --
11
+ -- Idempotent: does nothing when the key is already composite.
12
+
13
+ DO $$
14
+ DECLARE
15
+ pk_name TEXT;
16
+ pk_def TEXT;
17
+ BEGIN
18
+ SELECT c.conname, pg_get_constraintdef(c.oid)
19
+ INTO pk_name, pk_def
20
+ FROM pg_constraint c
21
+ WHERE c.conrelid = 'ai_span_payloads'::regclass AND c.contype = 'p';
22
+
23
+ IF pk_def IS DISTINCT FROM 'PRIMARY KEY (project_id, trace_id, span_id)' THEN
24
+ IF pk_name IS NOT NULL THEN
25
+ EXECUTE format('ALTER TABLE ai_span_payloads DROP CONSTRAINT %I', pk_name);
26
+ END IF;
27
+ ALTER TABLE ai_span_payloads ADD PRIMARY KEY (project_id, trace_id, span_id);
28
+ END IF;
29
+ END
30
+ $$;
@@ -0,0 +1,37 @@
1
+ -- Migration 048: telemetry_spans composite primary key
2
+ -- (project_id, trace_id, span_id)
3
+ --
4
+ -- Mirrors ../migrations/048_telemetry_spans_project_composite_pk.sql. 001
5
+ -- keyed spans by (trace_id, span_id); 010 added project_id outside the key,
6
+ -- so ingest's INSERT OR IGNORE (ON CONFLICT DO NOTHING here) silently
7
+ -- dropped a span whose ids already existed under another project. Swaps the
8
+ -- key for (project_id, trace_id, span_id) in place, keeping every row and
9
+ -- index. The old key was unique, so the composite key cannot collide.
10
+ --
11
+ -- Idempotent: does nothing when the key is already composite.
12
+ --
13
+ -- Operational note: ADD PRIMARY KEY builds a new unique index under an
14
+ -- ACCESS EXCLUSIVE lock. Retention keeps the table bounded, but on a busy
15
+ -- database apply it in a quiet window.
16
+
17
+ UPDATE telemetry_spans SET project_id = 'default' WHERE project_id IS NULL;
18
+ ALTER TABLE telemetry_spans ALTER COLUMN project_id SET NOT NULL;
19
+
20
+ DO $$
21
+ DECLARE
22
+ pk_name TEXT;
23
+ pk_def TEXT;
24
+ BEGIN
25
+ SELECT c.conname, pg_get_constraintdef(c.oid)
26
+ INTO pk_name, pk_def
27
+ FROM pg_constraint c
28
+ WHERE c.conrelid = 'telemetry_spans'::regclass AND c.contype = 'p';
29
+
30
+ IF pk_def IS DISTINCT FROM 'PRIMARY KEY (project_id, trace_id, span_id)' THEN
31
+ IF pk_name IS NOT NULL THEN
32
+ EXECUTE format('ALTER TABLE telemetry_spans DROP CONSTRAINT %I', pk_name);
33
+ END IF;
34
+ ALTER TABLE telemetry_spans ADD PRIMARY KEY (project_id, trace_id, span_id);
35
+ END IF;
36
+ END
37
+ $$;
@@ -0,0 +1,10 @@
1
+ -- Migration 049: alert_state.notified
2
+ --
3
+ -- Mirrors ../migrations/049_alert_state_notified.sql. INTEGER 0/1 like the
4
+ -- sibling alert_evaluations.notified (not BOOLEAN), so the shared store SQL
5
+ -- can test and set it the same way on both backends.
6
+ --
7
+ -- Backfill: existing rows default to 1 (delivered); the pre-049 evaluator
8
+ -- only stored a transition after the channels accepted it. Idempotent.
9
+
10
+ ALTER TABLE alert_state ADD COLUMN IF NOT EXISTS notified INTEGER NOT NULL DEFAULT 1;
@@ -0,0 +1,216 @@
1
+ -- ============================================================================
2
+ -- Migration 050: AutoSRE Causal Ontology, Change Correlation & Incident Memory
3
+ -- (RFC 0013 — PostgreSQL mirror)
4
+ -- ============================================================================
5
+
6
+ -- 0. Denormalize environment, version, and target_node_id onto existing tables
7
+ -- for O(log N) cohort rate lift and cross-trace contention queries.
8
+ ALTER TABLE telemetry_spans ADD COLUMN IF NOT EXISTS environment TEXT NOT NULL DEFAULT 'default';
9
+ ALTER TABLE telemetry_spans ADD COLUMN IF NOT EXISTS service_version TEXT;
10
+ ALTER TABLE telemetry_spans ADD COLUMN IF NOT EXISTS commit_sha TEXT;
11
+ ALTER TABLE telemetry_spans ADD COLUMN IF NOT EXISTS target_node_id TEXT;
12
+ ALTER TABLE telemetry_spans ADD COLUMN IF NOT EXISTS error_signature TEXT;
13
+
14
+ CREATE INDEX IF NOT EXISTS idx_spans_cohort_attribution
15
+ ON telemetry_spans (project_id, environment, service_name, service_version, received_at DESC);
16
+
17
+ CREATE INDEX IF NOT EXISTS idx_spans_contention_lookup
18
+ ON telemetry_spans (project_id, environment, target_node_id, start_time, end_time)
19
+ WHERE target_node_id IS NOT NULL;
20
+
21
+ ALTER TABLE actions ADD COLUMN IF NOT EXISTS environment TEXT NOT NULL DEFAULT 'default';
22
+ ALTER TABLE actions ADD COLUMN IF NOT EXISTS service_version TEXT;
23
+ ALTER TABLE actions ADD COLUMN IF NOT EXISTS target_node_id TEXT;
24
+
25
+ -- Extend eval_runs and add eval_case_results for verifiable PairedCounterfactualProof
26
+ ALTER TABLE eval_runs ADD COLUMN IF NOT EXISTS environment TEXT NOT NULL DEFAULT 'default';
27
+ ALTER TABLE eval_runs ADD COLUMN IF NOT EXISTS service_version TEXT;
28
+ ALTER TABLE eval_runs ADD COLUMN IF NOT EXISTS commit_sha TEXT;
29
+ ALTER TABLE eval_runs ADD COLUMN IF NOT EXISTS config_digest TEXT;
30
+ ALTER TABLE eval_runs ADD COLUMN IF NOT EXISTS input_payload_hash TEXT;
31
+ ALTER TABLE eval_runs ADD COLUMN IF NOT EXISTS failure_fingerprint TEXT;
32
+
33
+ ALTER TABLE eval_case_results ADD COLUMN IF NOT EXISTS input_payload_hash TEXT;
34
+ ALTER TABLE eval_case_results ADD COLUMN IF NOT EXISTS failure_fingerprint TEXT;
35
+
36
+ CREATE INDEX IF NOT EXISTS idx_eval_case_results_counterfactual
37
+ ON eval_case_results (project_id, eval_case_id, input_payload_hash, passed);
38
+
39
+ -- 1. Environment-Scoped Persistent Operational Ontology Nodes
40
+ CREATE TABLE IF NOT EXISTS ontology_nodes (
41
+ project_id TEXT NOT NULL DEFAULT 'default',
42
+ environment TEXT NOT NULL DEFAULT 'default',
43
+ id TEXT NOT NULL,
44
+ kind TEXT NOT NULL,
45
+ name TEXT NOT NULL,
46
+ namespace_or_endpoint TEXT NOT NULL DEFAULT 'default',
47
+ is_scoped_identity INTEGER NOT NULL DEFAULT 1,
48
+ current_version TEXT,
49
+ current_commit_sha TEXT,
50
+ current_config_digest TEXT,
51
+ attributes_json TEXT,
52
+ first_seen_at TEXT NOT NULL,
53
+ last_seen_at TEXT NOT NULL,
54
+ PRIMARY KEY (project_id, environment, id)
55
+ );
56
+
57
+ -- 2. Directed Operational Ontology Edges (Cycles Permitted; Handled via Tarjan SCC)
58
+ CREATE TABLE IF NOT EXISTS ontology_edges (
59
+ project_id TEXT NOT NULL DEFAULT 'default',
60
+ environment TEXT NOT NULL DEFAULT 'default',
61
+ id TEXT NOT NULL,
62
+ source_node_id TEXT NOT NULL,
63
+ predicate TEXT NOT NULL,
64
+ target_node_id TEXT NOT NULL,
65
+ relationship_certainty TEXT NOT NULL DEFAULT 'exact',
66
+ call_count INTEGER NOT NULL DEFAULT 0,
67
+ error_count INTEGER NOT NULL DEFAULT 0,
68
+ last_duration_ms DOUBLE PRECISION,
69
+ first_seen_at TEXT NOT NULL,
70
+ last_seen_at TEXT NOT NULL,
71
+ PRIMARY KEY (project_id, environment, id)
72
+ );
73
+
74
+ CREATE INDEX IF NOT EXISTS idx_ontology_edges_source
75
+ ON ontology_edges (project_id, environment, source_node_id);
76
+
77
+ CREATE INDEX IF NOT EXISTS idx_ontology_edges_target
78
+ ON ontology_edges (project_id, environment, target_node_id);
79
+
80
+ -- 3. Explicit & Inferred Change Events
81
+ CREATE TABLE IF NOT EXISTS change_events (
82
+ project_id TEXT NOT NULL DEFAULT 'default',
83
+ environment TEXT NOT NULL DEFAULT 'default',
84
+ id TEXT NOT NULL,
85
+ target_node_id TEXT NOT NULL,
86
+ kind TEXT NOT NULL,
87
+ source TEXT NOT NULL,
88
+ relationship_certainty TEXT NOT NULL DEFAULT 'exact',
89
+ summary TEXT NOT NULL,
90
+ service_name TEXT,
91
+ version TEXT,
92
+ previous_version TEXT,
93
+ commit_sha TEXT,
94
+ previous_commit_sha TEXT,
95
+ config_digest TEXT,
96
+ repository_url TEXT,
97
+ pull_request_url TEXT,
98
+ actor_id TEXT,
99
+ caused_by_action_id TEXT,
100
+ agent_run_id TEXT,
101
+ touched_paths_json TEXT,
102
+ attributes_json TEXT,
103
+ occurred_at TEXT NOT NULL,
104
+ received_at TEXT NOT NULL,
105
+ expires_at TEXT,
106
+ PRIMARY KEY (project_id, environment, id)
107
+ );
108
+
109
+ CREATE INDEX IF NOT EXISTS idx_change_events_node_time
110
+ ON change_events (project_id, environment, target_node_id, occurred_at DESC);
111
+
112
+ -- 4. Durable Incident Investigations & Runner Lease Coordination
113
+ CREATE TABLE IF NOT EXISTS incident_investigations (
114
+ project_id TEXT NOT NULL DEFAULT 'default',
115
+ environment TEXT NOT NULL DEFAULT 'default',
116
+ id TEXT NOT NULL,
117
+ dedup_key TEXT NOT NULL,
118
+ status TEXT NOT NULL DEFAULT 'open',
119
+ target_node_id TEXT NOT NULL,
120
+ symptom_anchor_kind TEXT NOT NULL,
121
+ symptom_anchor_id TEXT NOT NULL,
122
+ symptom_fingerprint TEXT,
123
+ alert_count INTEGER NOT NULL DEFAULT 1,
124
+ latest_bundle_json TEXT,
125
+ lease_owner_id TEXT,
126
+ lease_expires_at TEXT,
127
+ dispatch_attempts INTEGER NOT NULL DEFAULT 0,
128
+ last_dispatched_at TEXT,
129
+ opened_at TEXT NOT NULL,
130
+ updated_at TEXT NOT NULL,
131
+ resolved_at TEXT,
132
+ expires_at TEXT,
133
+ PRIMARY KEY (project_id, environment, id)
134
+ );
135
+
136
+ CREATE INDEX IF NOT EXISTS idx_incidents_active_dedup
137
+ ON incident_investigations (project_id, environment, dedup_key, status);
138
+
139
+ -- 5. Closed-Loop Investigation Findings (Separating Recovery vs Causal Proof)
140
+ CREATE TABLE IF NOT EXISTS investigation_findings (
141
+ project_id TEXT NOT NULL DEFAULT 'default',
142
+ environment TEXT NOT NULL DEFAULT 'default',
143
+ id TEXT NOT NULL,
144
+ incident_id TEXT,
145
+ fingerprint TEXT NOT NULL,
146
+ fingerprint_input_json TEXT NOT NULL,
147
+ title TEXT NOT NULL,
148
+ summary TEXT NOT NULL,
149
+ causal_proof_tier TEXT NOT NULL DEFAULT 'unverified_locus',
150
+ confidence DOUBLE PRECISION NOT NULL,
151
+ confidence_breakdown_json TEXT NOT NULL,
152
+ symptom_entity_kind TEXT NOT NULL,
153
+ symptom_entity_id TEXT NOT NULL,
154
+ fault_locus_entity_kind TEXT NOT NULL,
155
+ fault_locus_entity_id TEXT NOT NULL,
156
+ root_cause_entity_kind TEXT,
157
+ root_cause_entity_id TEXT,
158
+ contributing_change_id TEXT,
159
+ cohort_verdict TEXT,
160
+ contending_trace_id TEXT,
161
+ causal_chain_json TEXT NOT NULL,
162
+ cited_evidence_ids_json TEXT NOT NULL,
163
+ code_references_json TEXT,
164
+ counterfactual_proof_json TEXT,
165
+ recovery_status TEXT NOT NULL DEFAULT 'not_applicable',
166
+ causal_verification_status TEXT NOT NULL DEFAULT 'unverified',
167
+ watchdog_step INTEGER NOT NULL DEFAULT 0,
168
+ baseline_request_rate_per_min DOUBLE PRECISION,
169
+ next_watchdog_check_at TEXT,
170
+ actor_type TEXT NOT NULL,
171
+ actor_id TEXT NOT NULL,
172
+ agent_run_id TEXT,
173
+ created_at TEXT NOT NULL,
174
+ updated_at TEXT NOT NULL,
175
+ expires_at TEXT,
176
+ PRIMARY KEY (project_id, environment, id)
177
+ );
178
+
179
+ CREATE INDEX IF NOT EXISTS idx_findings_fingerprint
180
+ ON investigation_findings (project_id, environment, fingerprint, created_at DESC);
181
+
182
+ CREATE INDEX IF NOT EXISTS idx_findings_watchdog
183
+ ON investigation_findings (project_id, recovery_status, next_watchdog_check_at);
184
+
185
+ -- 6. Pre-Execution Remediation Proposals & Single-Use Grants
186
+ CREATE TABLE IF NOT EXISTS remediation_proposals (
187
+ project_id TEXT NOT NULL DEFAULT 'default',
188
+ environment TEXT NOT NULL DEFAULT 'default',
189
+ id TEXT NOT NULL,
190
+ incident_id TEXT NOT NULL,
191
+ finding_id TEXT NOT NULL,
192
+ idempotency_key TEXT NOT NULL,
193
+ action_kind TEXT NOT NULL,
194
+ target_node_id TEXT NOT NULL,
195
+ expected_current_state_json TEXT NOT NULL,
196
+ desired_target_state_json TEXT NOT NULL,
197
+ requested_autonomy TEXT NOT NULL,
198
+ effective_autonomy TEXT NOT NULL,
199
+ policy_decision_reason TEXT NOT NULL,
200
+ status TEXT NOT NULL DEFAULT 'proposed',
201
+ grant_token_hash TEXT,
202
+ grant_expires_at TEXT,
203
+ approved_by_user_id TEXT,
204
+ approved_at TEXT,
205
+ claimed_by_runner_id TEXT,
206
+ claimed_at TEXT,
207
+ claim_expires_at TEXT,
208
+ remediation_agent_run_id TEXT,
209
+ resulting_change_id TEXT,
210
+ failure_reason TEXT,
211
+ created_at TEXT NOT NULL,
212
+ updated_at TEXT NOT NULL,
213
+ expires_at TEXT,
214
+ PRIMARY KEY (project_id, environment, id),
215
+ UNIQUE (project_id, environment, idempotency_key)
216
+ );
@@ -32,9 +32,62 @@ remaining migrations are tracked in
32
32
  [`docs/postgres-migration.md`](../../../../docs/postgres-migration.md); they'll
33
33
  land alongside the first hosted-Postgres deployment.
34
34
 
35
- ## What's NOT translated
35
+ ## Column types the stores depend on
36
36
 
37
- `009_projects_and_keys.sql` uses SQLite's `BLOB` type for the hashed API key.
38
- Postgres uses `BYTEA` and the column-name rewrite is in
39
- `009_projects_and_keys.sql` here (a non-mechanical translation; review carefully
40
- when porting forward).
37
+ - `ingest_keys.key_hash` is `TEXT` on both backends. `ProjectsStore` binds the
38
+ SHA-256 hex digest as a string; an earlier Postgres version of `009` declared
39
+ the column `BYTEA`, which silently stored the ASCII bytes of the hex text and
40
+ made the `026` seed key unreachable. `041_ingest_key_hash_text.sql` converts
41
+ databases migrated with that version back to `TEXT` (fresh installs get
42
+ `TEXT` from `009` directly and `041` is a no-op). The D1 `041` file is an
43
+ intentional no-op that only keeps the numbering in parity.
44
+ - Timestamp columns (`received_at`, `expires_at`, …) are `TEXT` holding
45
+ `Date#toISOString()` output. Postgres will not compare `text` to
46
+ `timestamptz`, so `postgresDialect` and `translateD1Sql` render every time
47
+ window as ISO text (`to_char(…, 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')`).
48
+ `metric_exemplars` originally declared `received_at`/`expires_at` as
49
+ `TIMESTAMPTZ`, which made `MetricsStore.purgeExpired` fail with
50
+ `operator does not exist: timestamp with time zone < text`;
51
+ `043_metric_exemplars_text_timestamps.sql` converts those columns to UTC
52
+ ISO `TEXT` in place (fresh installs get `TEXT` from `035` and `043` is a
53
+ no-op; the D1 `043` is a numbering no-op).
54
+ - `actions.started_at` is ISO `TEXT` and drives action-graph retention (the
55
+ spine has no `expires_at`). `044_actions_started_at_index.sql` adds
56
+ `idx_actions_started_at` on both backends so the hourly sweep reads expired
57
+ trees in index order. Deletes run per tree inside `PostgresAdapter.batch`
58
+ (one transaction) and re-check eligibility in every statement, but under
59
+ READ COMMITTED a concurrent insert into an already-expired tree can still
60
+ land between that tree's leaf deletes and its spine delete; see the header
61
+ of `src/lib/action-retention.ts`.
62
+ - Row keys are project-scoped. `045`–`048` swap the single-column primary
63
+ keys for composite ones in place (`user_profiles (project_id, user_id)`;
64
+ `actions`, `agent_runs`, `tool_calls`, `retrieval_events`, `eval_results`,
65
+ `artifacts (project_id, id)`; `ai_span_payloads` and
66
+ `telemetry_spans (project_id, trace_id, span_id)`). Each file looks up the
67
+ current primary key by definition and is a no-op once the key is
68
+ composite; the D1 siblings rebuild the tables through `*_new` copies. The
69
+ stores upsert with `ON CONFLICT (project_id, …)`, so both backends must
70
+ carry these migrations before deploying collector 2.1.
71
+ - `049` adds `alert_state.notified INTEGER` (0/1, like
72
+ `alert_evaluations.notified`): whether the current state's transition has
73
+ been delivered to the rule's channels. Existing rows backfill to 1. The
74
+ store falls back to the pre-049 statement when the column is missing, so
75
+ the collector keeps evaluating during a rolling deploy, but it cannot
76
+ retry undelivered notifications until 049 is applied.
77
+ - Flags (`is_error`, `is_bot`, `payload_capture_enabled`) are `BOOLEAN` here
78
+ and `INTEGER 0/1` on D1. Shared SQL must test them as bare booleans
79
+ (`CASE WHEN is_error THEN 1 ELSE 0 END`, never `is_error = 1` or
80
+ `SUM(is_error)`). Hosts call `configurePostgresTypeParsers(pg.types)` so
81
+ BOOLEAN reads back as `0/1`, BIGINT/NUMERIC as numbers and JSON/JSONB as
82
+ text, matching what D1 returns.
83
+
84
+ ## Verifying against a real database
85
+
86
+ `src/lib/sql-db-postgres.integration.test.ts` applies this directory into an
87
+ empty scratch database, then exercises ingest-key auth, AI/usage ingest, the
88
+ retention purge and every built-in analysis through `PostgresAdapter`:
89
+
90
+ ```bash
91
+ OBS_PG_TEST_URL=postgres://obs:obs@127.0.0.1:5432/obs_scratch \
92
+ pnpm --filter @obsunified/collector exec vitest run src/lib/sql-db-postgres.integration.test.ts
93
+ ```
@@ -0,0 +1,76 @@
1
+ import { gzipSync } from "node:zlib";
2
+ import { Hono } from "hono";
3
+ import { describe, expect, it } from "vitest";
4
+ import { OtlpDecodeError, type ReadBodyLimits, readOtlpBody } from "./body";
5
+
6
+ const MIB = 1024 * 1024;
7
+
8
+ const read = async (
9
+ body: BodyInit,
10
+ headers: Record<string, string>,
11
+ limits: ReadBodyLimits = {},
12
+ ) => {
13
+ const app = new Hono().post("/t", async (c) => {
14
+ try {
15
+ const result = await readOtlpBody(c, limits);
16
+ return c.json({
17
+ bytes: result.bytes.byteLength,
18
+ wire: result.wireFormat,
19
+ });
20
+ } catch (err) {
21
+ if (err instanceof OtlpDecodeError)
22
+ return c.json({ error: err.message }, err.status);
23
+ throw err;
24
+ }
25
+ });
26
+ const res = await app.request("/t", { method: "POST", body, headers });
27
+ return {
28
+ status: res.status,
29
+ body: (await res.json()) as Record<string, unknown>,
30
+ };
31
+ };
32
+
33
+ describe("readOtlpBody limits", () => {
34
+ it("decompresses a normal gzip payload", async () => {
35
+ const json = JSON.stringify({ resourceSpans: [] });
36
+ const res = await read(gzipSync(json), {
37
+ "content-type": "application/json",
38
+ "content-encoding": "gzip",
39
+ });
40
+ expect(res).toEqual({
41
+ status: 200,
42
+ body: { bytes: json.length, wire: "json" },
43
+ });
44
+ });
45
+
46
+ it("refuses a gzip bomb once the decompressed cap is crossed", async () => {
47
+ // 8 MiB of zeros gzips to a few KiB; the reader must stop at 1 MiB.
48
+ const bomb = gzipSync(Buffer.alloc(8 * MIB));
49
+ expect(bomb.byteLength).toBeLessThan(64 * 1024);
50
+ const res = await read(
51
+ bomb,
52
+ { "content-type": "application/x-protobuf", "content-encoding": "gzip" },
53
+ { maxDecompressedBytes: MIB },
54
+ );
55
+ expect(res.status).toBe(413);
56
+ expect(res.body.error).toMatch(/Decompressed payload too large/);
57
+ });
58
+
59
+ it("refuses a raw body over the configured cap", async () => {
60
+ const res = await read(
61
+ "x".repeat(2048),
62
+ { "content-type": "application/json" },
63
+ { maxBodyBytes: 1024 },
64
+ );
65
+ expect(res.status).toBe(413);
66
+ expect(res.body.error).toMatch(/Payload too large \(max 1024 bytes\)/);
67
+ });
68
+
69
+ it("reports malformed gzip as 400, not 500", async () => {
70
+ const res = await read("definitely not gzip", {
71
+ "content-type": "application/json",
72
+ "content-encoding": "gzip",
73
+ });
74
+ expect(res.status).toBe(400);
75
+ });
76
+ });