@obsunified/collector 2.0.1 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth.d.ts +1 -1
- package/dist/auth.js +1 -1
- package/dist/{chunk-UGWDOGGH.js → chunk-KMEZKGSO.js} +22 -10
- package/dist/chunk-KMEZKGSO.js.map +1 -0
- package/dist/{chunk-XKQTVIRE.js → chunk-ZSF5XP3B.js} +8 -2
- package/dist/chunk-ZSF5XP3B.js.map +1 -0
- package/dist/index.d.ts +630 -27
- package/dist/index.js +5819 -853
- package/dist/index.js.map +1 -1
- package/dist/{ingest-auth-HFVO1zIp.d.ts → ingest-auth-C5-8Mly5.d.ts} +9 -0
- package/dist/{openai-FJYOQ7WO.js → openai-GSJZGFGL.js} +2 -2
- package/package.json +5 -5
- package/src/__tests__/autosre-closed-loop.test.ts +1275 -0
- package/src/analyses/derive.ts +26 -25
- package/src/analyses/investigations.ts +8 -8
- package/src/analyses/tier0.ts +22 -22
- package/src/auth/dashboard-auth.test.ts +791 -0
- package/src/auth/dashboard-auth.ts +187 -38
- package/src/auth/login-rate-limit.test.ts +129 -0
- package/src/auth/login-rate-limit.ts +143 -0
- package/src/framework/body-limit.test.ts +165 -0
- package/src/framework/collector.ts +355 -115
- package/src/framework/env.ts +9 -0
- package/src/framework/limits.ts +31 -0
- package/src/framework/scheduled-handlers.test.ts +342 -0
- package/src/index.ts +59 -1
- package/src/lib/action-retention.test.ts +472 -0
- package/src/lib/action-retention.ts +420 -0
- package/src/lib/actions-store.ts +6 -12
- package/src/lib/ai-store/calls.ts +2 -2
- package/src/lib/ai-store/sessions.ts +3 -3
- package/src/lib/ai-store/spans.ts +1 -1
- package/src/lib/ai-store/types.ts +2 -1
- package/src/lib/alerts-store.test.ts +153 -0
- package/src/lib/alerts-store.ts +122 -21
- package/src/lib/analyses-runner.test.ts +75 -0
- package/src/lib/analyses-runner.ts +69 -3
- package/src/lib/analyses-store-latest.test.ts +61 -0
- package/src/lib/analyses-store.ts +17 -10
- package/src/lib/autosre-change-correlation.ts +780 -0
- package/src/lib/autosre-findings.ts +725 -0
- package/src/lib/autosre-ontology.ts +914 -0
- package/src/lib/autosre-remediation-gate.ts +1663 -0
- package/src/lib/blob-store.ts +1 -1
- package/src/lib/eval-cases-store.ts +63 -1
- package/src/lib/evidence-references.ts +6 -0
- package/src/lib/housekeeping.test.ts +153 -0
- package/src/lib/housekeeping.ts +193 -0
- package/src/lib/identity-index/action-lookups.ts +6 -6
- package/src/lib/identity-index/key-lookups.ts +2 -2
- package/src/lib/identity-index/user-lookup.ts +8 -5
- package/src/lib/identity-index.test.ts +2 -3
- package/src/lib/metrics-store-series.test.ts +99 -0
- package/src/lib/metrics-store.test.ts +23 -0
- package/src/lib/metrics-store.ts +27 -3
- package/src/lib/projects-store.test.ts +25 -0
- package/src/lib/projects-store.ts +4 -3
- package/src/lib/sql-db-dialect.test.ts +12 -3
- package/src/lib/sql-db-postgres.integration.test.ts +1225 -0
- package/src/lib/sql-db-postgres.test.ts +314 -0
- package/src/lib/sql-db-postgres.ts +392 -54
- package/src/lib/sql-db-sqlite-time-windows.test.ts +194 -0
- package/src/lib/sql-db.ts +40 -6
- package/src/lib/store/ingest.ts +94 -6
- package/src/lib/test-utils/action-graph-fixtures.ts +307 -0
- package/src/lib/test-utils/node-sqlite-db.ts +188 -0
- package/src/lib/webhook-url.test.ts +122 -0
- package/src/lib/webhook-url.ts +155 -0
- package/src/migrations/041_ingest_key_hash_text.sql +5 -0
- package/src/migrations/042_session_replay_project_composite_pk.sql +52 -0
- package/src/migrations/043_metric_exemplars_text_timestamps.sql +5 -0
- package/src/migrations/044_actions_started_at_index.sql +10 -0
- package/src/migrations/045_user_profiles_project_composite_pk.sql +40 -0
- package/src/migrations/046_action_graph_project_composite_pk.sql +255 -0
- package/src/migrations/047_ai_span_payloads_project_composite_pk.sql +61 -0
- package/src/migrations/048_telemetry_spans_project_composite_pk.sql +92 -0
- package/src/migrations/049_alert_state_notified.sql +17 -0
- package/src/migrations/050_autosre_causal_ontology.sql +216 -0
- package/src/migrations/alert-state-notified.test.ts +111 -0
- package/src/migrations/parity.test.ts +68 -0
- package/src/migrations/project-scoped-keys.test.ts +432 -0
- package/src/migrations-postgres/009_projects_and_keys.sql +4 -4
- package/src/migrations-postgres/026_obs_dashboard_project_seed.sql +1 -1
- package/src/migrations-postgres/035_metric_exemplars.sql +5 -2
- package/src/migrations-postgres/041_ingest_key_hash_text.sql +32 -0
- package/src/migrations-postgres/042_session_replay_project_composite_pk.sql +16 -0
- package/src/migrations-postgres/043_metric_exemplars_text_timestamps.sql +47 -0
- package/src/migrations-postgres/044_actions_started_at_index.sql +8 -0
- package/src/migrations-postgres/045_user_profiles_project_composite_pk.sql +33 -0
- package/src/migrations-postgres/046_action_graph_project_composite_pk.sql +39 -0
- package/src/migrations-postgres/047_ai_span_payloads_project_composite_pk.sql +30 -0
- package/src/migrations-postgres/048_telemetry_spans_project_composite_pk.sql +37 -0
- package/src/migrations-postgres/049_alert_state_notified.sql +10 -0
- package/src/migrations-postgres/050_autosre_causal_ontology.sql +216 -0
- package/src/migrations-postgres/README.md +58 -5
- package/src/otlp/decode/body.test.ts +76 -0
- package/src/otlp/decode/body.ts +67 -6
- package/src/plugins/_json.ts +21 -0
- package/src/plugins/action-graph-processor.ts +6 -12
- package/src/plugins/ai-receiver.ts +5 -12
- package/src/plugins/ai-span-payloads-processor.ts +1 -2
- package/src/plugins/alerts-evaluator-retention.test.ts +170 -0
- package/src/plugins/alerts-evaluator.test.ts +464 -3
- package/src/plugins/alerts-evaluator.ts +181 -41
- package/src/plugins/alerts-routes.ts +10 -6
- package/src/plugins/ask-routes.ts +4 -5
- package/src/plugins/autosre-routes.ts +1019 -0
- package/src/plugins/connected-routes/manifest.ts +13 -1
- package/src/plugins/connected-routes.test.ts +106 -0
- package/src/plugins/connected-routes.ts +285 -5
- package/src/plugins/dashboard-routes.test.ts +120 -0
- package/src/plugins/dashboard-routes.ts +11 -2
- package/src/plugins/default-span-enrichment.ts +13 -1
- package/src/plugins/eval-cases-routes.ts +65 -19
- package/src/plugins/evidence-retrieval-routes.ts +300 -4
- package/src/plugins/identity-receiver.ts +9 -11
- package/src/plugins/json-body.test.ts +91 -0
- package/src/plugins/onboarding-routes.test.ts +105 -0
- package/src/plugins/onboarding-routes.ts +26 -11
- package/src/plugins/projects-routes.ts +2 -1
- package/src/plugins/replay-isolation.test.ts +922 -0
- package/src/plugins/replay-receiver.ts +8 -8
- package/src/plugins/usage-receiver.ts +3 -6
- package/dist/chunk-UGWDOGGH.js.map +0 -1
- package/dist/chunk-XKQTVIRE.js.map +0 -1
- /package/dist/{openai-FJYOQ7WO.js.map → openai-GSJZGFGL.js.map} +0 -0
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
-- Migration 042: session replay composite primary key (project_id, session_id)
|
|
2
|
+
--
|
|
3
|
+
-- Fixes cross-project session replay metadata collisions. Previously,
|
|
4
|
+
-- session_replay_metadata used session_id alone as primary key while
|
|
5
|
+
-- project_id was added in migration 010. Replay receiver upserts used
|
|
6
|
+
-- ON CONFLICT(session_id), causing identical session IDs from different
|
|
7
|
+
-- projects to overwrite or increment each other.
|
|
8
|
+
--
|
|
9
|
+
-- This forward migration drops the existing single-column primary key constraint
|
|
10
|
+
-- and establishes PRIMARY KEY (project_id, session_id) while preserving all
|
|
11
|
+
-- existing rows and indexes.
|
|
12
|
+
|
|
13
|
+
UPDATE session_replay_metadata SET project_id = 'default' WHERE project_id IS NULL;
|
|
14
|
+
ALTER TABLE session_replay_metadata ALTER COLUMN project_id SET NOT NULL;
|
|
15
|
+
ALTER TABLE session_replay_metadata DROP CONSTRAINT IF EXISTS session_replay_metadata_pkey;
|
|
16
|
+
ALTER TABLE session_replay_metadata ADD PRIMARY KEY (project_id, session_id);
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
-- Convert metric_exemplars.received_at / expires_at from TIMESTAMPTZ to TEXT
|
|
2
|
+
-- on databases migrated with the original Postgres 035.
|
|
3
|
+
--
|
|
4
|
+
-- Every other retention table stores timestamps as TEXT holding
|
|
5
|
+
-- Date#toISOString() output, and postgresDialect renders now() as ISO text
|
|
6
|
+
-- (to_char(..., 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')). Against TIMESTAMPTZ
|
|
7
|
+
-- columns MetricsStore.purgeExpired failed with
|
|
8
|
+
-- operator does not exist: timestamp with time zone < text
|
|
9
|
+
-- so metric exemplars were never purged on Postgres.
|
|
10
|
+
--
|
|
11
|
+
-- Existing rows are preserved: each value is rendered in UTC in the exact
|
|
12
|
+
-- shape the collector writes (millisecond precision, trailing "Z"), so the
|
|
13
|
+
-- converted text compares correctly against freshly ingested rows and the
|
|
14
|
+
-- dialect's now(). idx_metric_exemplars_expires is rebuilt by Postgres as
|
|
15
|
+
-- part of the type change.
|
|
16
|
+
--
|
|
17
|
+
-- Fresh installs already create both columns as TEXT in 035 and this is a
|
|
18
|
+
-- no-op. Re-running is also a no-op.
|
|
19
|
+
DO $$
|
|
20
|
+
BEGIN
|
|
21
|
+
IF EXISTS (
|
|
22
|
+
SELECT 1
|
|
23
|
+
FROM information_schema.columns
|
|
24
|
+
WHERE table_schema = current_schema()
|
|
25
|
+
AND table_name = 'metric_exemplars'
|
|
26
|
+
AND column_name = 'received_at'
|
|
27
|
+
AND data_type = 'timestamp with time zone'
|
|
28
|
+
) THEN
|
|
29
|
+
ALTER TABLE metric_exemplars
|
|
30
|
+
ALTER COLUMN received_at TYPE TEXT
|
|
31
|
+
USING to_char(received_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"');
|
|
32
|
+
END IF;
|
|
33
|
+
|
|
34
|
+
IF EXISTS (
|
|
35
|
+
SELECT 1
|
|
36
|
+
FROM information_schema.columns
|
|
37
|
+
WHERE table_schema = current_schema()
|
|
38
|
+
AND table_name = 'metric_exemplars'
|
|
39
|
+
AND column_name = 'expires_at'
|
|
40
|
+
AND data_type = 'timestamp with time zone'
|
|
41
|
+
) THEN
|
|
42
|
+
ALTER TABLE metric_exemplars
|
|
43
|
+
ALTER COLUMN expires_at TYPE TEXT
|
|
44
|
+
USING to_char(expires_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"');
|
|
45
|
+
END IF;
|
|
46
|
+
END
|
|
47
|
+
$$;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
-- RFC 0010 — action graph retention.
|
|
2
|
+
--
|
|
3
|
+
-- Mirrors ../migrations/044_actions_started_at_index.sql. `started_at` is
|
|
4
|
+
-- ISO-8601 TEXT on both backends, so the retention sweep compares it against
|
|
5
|
+
-- a text cutoff rendered the same way `postgresDialect.sinceHours` does.
|
|
6
|
+
|
|
7
|
+
CREATE INDEX IF NOT EXISTS idx_actions_started_at
|
|
8
|
+
ON actions (started_at);
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
-- Migration 045: user_profiles composite primary key (project_id, user_id)
|
|
2
|
+
--
|
|
3
|
+
-- Mirrors ../migrations/045_user_profiles_project_composite_pk.sql. The 006
|
|
4
|
+
-- table used `user_id TEXT PRIMARY KEY`; 010 added project_id outside the
|
|
5
|
+
-- key, so /v1/identify's ON CONFLICT(user_id) upsert let one project rewrite
|
|
6
|
+
-- another project's profile. Swaps the key for (project_id, user_id) in
|
|
7
|
+
-- place, keeping every row and index. The old key was unique, so the
|
|
8
|
+
-- composite key cannot collide.
|
|
9
|
+
--
|
|
10
|
+
-- Idempotent: the block looks up the current primary key by definition and
|
|
11
|
+
-- does nothing when it is already (project_id, user_id).
|
|
12
|
+
|
|
13
|
+
UPDATE user_profiles SET project_id = 'default' WHERE project_id IS NULL;
|
|
14
|
+
ALTER TABLE user_profiles ALTER COLUMN project_id SET NOT NULL;
|
|
15
|
+
|
|
16
|
+
DO $$
|
|
17
|
+
DECLARE
|
|
18
|
+
pk_name TEXT;
|
|
19
|
+
pk_def TEXT;
|
|
20
|
+
BEGIN
|
|
21
|
+
SELECT c.conname, pg_get_constraintdef(c.oid)
|
|
22
|
+
INTO pk_name, pk_def
|
|
23
|
+
FROM pg_constraint c
|
|
24
|
+
WHERE c.conrelid = 'user_profiles'::regclass AND c.contype = 'p';
|
|
25
|
+
|
|
26
|
+
IF pk_def IS DISTINCT FROM 'PRIMARY KEY (project_id, user_id)' THEN
|
|
27
|
+
IF pk_name IS NOT NULL THEN
|
|
28
|
+
EXECUTE format('ALTER TABLE user_profiles DROP CONSTRAINT %I', pk_name);
|
|
29
|
+
END IF;
|
|
30
|
+
ALTER TABLE user_profiles ADD PRIMARY KEY (project_id, user_id);
|
|
31
|
+
END IF;
|
|
32
|
+
END
|
|
33
|
+
$$;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
-- Migration 046: action graph tables keyed by (project_id, id)
|
|
2
|
+
--
|
|
3
|
+
-- Mirrors ../migrations/046_action_graph_project_composite_pk.sql. The 031
|
|
4
|
+
-- spine and leaf tables used `id TEXT PRIMARY KEY`; action ids are
|
|
5
|
+
-- client-supplied and leaf ids are raw OTel span ids, so two projects can
|
|
6
|
+
-- emit the same id and the ON CONFLICT(id) upserts let one tenant overwrite
|
|
7
|
+
-- another's row. Swaps each key for (project_id, id) in place, keeping every
|
|
8
|
+
-- row and index (031, 038, 044). The old keys were unique, so the composite
|
|
9
|
+
-- keys cannot collide.
|
|
10
|
+
--
|
|
11
|
+
-- Idempotent: each table is skipped when its primary key is already
|
|
12
|
+
-- (project_id, id).
|
|
13
|
+
|
|
14
|
+
DO $$
|
|
15
|
+
DECLARE
|
|
16
|
+
t TEXT;
|
|
17
|
+
pk_name TEXT;
|
|
18
|
+
pk_def TEXT;
|
|
19
|
+
BEGIN
|
|
20
|
+
FOREACH t IN ARRAY ARRAY[
|
|
21
|
+
'actions', 'agent_runs', 'tool_calls', 'retrieval_events',
|
|
22
|
+
'eval_results', 'artifacts'
|
|
23
|
+
] LOOP
|
|
24
|
+
pk_name := NULL;
|
|
25
|
+
pk_def := NULL;
|
|
26
|
+
SELECT c.conname, pg_get_constraintdef(c.oid)
|
|
27
|
+
INTO pk_name, pk_def
|
|
28
|
+
FROM pg_constraint c
|
|
29
|
+
WHERE c.conrelid = t::regclass AND c.contype = 'p';
|
|
30
|
+
|
|
31
|
+
IF pk_def IS DISTINCT FROM 'PRIMARY KEY (project_id, id)' THEN
|
|
32
|
+
IF pk_name IS NOT NULL THEN
|
|
33
|
+
EXECUTE format('ALTER TABLE %I DROP CONSTRAINT %I', t, pk_name);
|
|
34
|
+
END IF;
|
|
35
|
+
EXECUTE format('ALTER TABLE %I ADD PRIMARY KEY (project_id, id)', t);
|
|
36
|
+
END IF;
|
|
37
|
+
END LOOP;
|
|
38
|
+
END
|
|
39
|
+
$$;
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
-- Migration 047: ai_span_payloads composite primary key
|
|
2
|
+
-- (project_id, trace_id, span_id)
|
|
3
|
+
--
|
|
4
|
+
-- Mirrors ../migrations/047_ai_span_payloads_project_composite_pk.sql. 019
|
|
5
|
+
-- keyed the payload side table by (trace_id, span_id) only; the processor's
|
|
6
|
+
-- ON CONFLICT(trace_id, span_id) upsert let a second project replace the
|
|
7
|
+
-- first project's prompt/completion payloads. Swaps the key for
|
|
8
|
+
-- (project_id, trace_id, span_id) in place, keeping every row and index.
|
|
9
|
+
-- The old key was unique, so the composite key cannot collide.
|
|
10
|
+
--
|
|
11
|
+
-- Idempotent: does nothing when the key is already composite.
|
|
12
|
+
|
|
13
|
+
DO $$
|
|
14
|
+
DECLARE
|
|
15
|
+
pk_name TEXT;
|
|
16
|
+
pk_def TEXT;
|
|
17
|
+
BEGIN
|
|
18
|
+
SELECT c.conname, pg_get_constraintdef(c.oid)
|
|
19
|
+
INTO pk_name, pk_def
|
|
20
|
+
FROM pg_constraint c
|
|
21
|
+
WHERE c.conrelid = 'ai_span_payloads'::regclass AND c.contype = 'p';
|
|
22
|
+
|
|
23
|
+
IF pk_def IS DISTINCT FROM 'PRIMARY KEY (project_id, trace_id, span_id)' THEN
|
|
24
|
+
IF pk_name IS NOT NULL THEN
|
|
25
|
+
EXECUTE format('ALTER TABLE ai_span_payloads DROP CONSTRAINT %I', pk_name);
|
|
26
|
+
END IF;
|
|
27
|
+
ALTER TABLE ai_span_payloads ADD PRIMARY KEY (project_id, trace_id, span_id);
|
|
28
|
+
END IF;
|
|
29
|
+
END
|
|
30
|
+
$$;
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
-- Migration 048: telemetry_spans composite primary key
|
|
2
|
+
-- (project_id, trace_id, span_id)
|
|
3
|
+
--
|
|
4
|
+
-- Mirrors ../migrations/048_telemetry_spans_project_composite_pk.sql. 001
|
|
5
|
+
-- keyed spans by (trace_id, span_id); 010 added project_id outside the key,
|
|
6
|
+
-- so ingest's INSERT OR IGNORE (ON CONFLICT DO NOTHING here) silently
|
|
7
|
+
-- dropped a span whose ids already existed under another project. Swaps the
|
|
8
|
+
-- key for (project_id, trace_id, span_id) in place, keeping every row and
|
|
9
|
+
-- index. The old key was unique, so the composite key cannot collide.
|
|
10
|
+
--
|
|
11
|
+
-- Idempotent: does nothing when the key is already composite.
|
|
12
|
+
--
|
|
13
|
+
-- Operational note: ADD PRIMARY KEY builds a new unique index under an
|
|
14
|
+
-- ACCESS EXCLUSIVE lock. Retention keeps the table bounded, but on a busy
|
|
15
|
+
-- database apply it in a quiet window.
|
|
16
|
+
|
|
17
|
+
UPDATE telemetry_spans SET project_id = 'default' WHERE project_id IS NULL;
|
|
18
|
+
ALTER TABLE telemetry_spans ALTER COLUMN project_id SET NOT NULL;
|
|
19
|
+
|
|
20
|
+
DO $$
|
|
21
|
+
DECLARE
|
|
22
|
+
pk_name TEXT;
|
|
23
|
+
pk_def TEXT;
|
|
24
|
+
BEGIN
|
|
25
|
+
SELECT c.conname, pg_get_constraintdef(c.oid)
|
|
26
|
+
INTO pk_name, pk_def
|
|
27
|
+
FROM pg_constraint c
|
|
28
|
+
WHERE c.conrelid = 'telemetry_spans'::regclass AND c.contype = 'p';
|
|
29
|
+
|
|
30
|
+
IF pk_def IS DISTINCT FROM 'PRIMARY KEY (project_id, trace_id, span_id)' THEN
|
|
31
|
+
IF pk_name IS NOT NULL THEN
|
|
32
|
+
EXECUTE format('ALTER TABLE telemetry_spans DROP CONSTRAINT %I', pk_name);
|
|
33
|
+
END IF;
|
|
34
|
+
ALTER TABLE telemetry_spans ADD PRIMARY KEY (project_id, trace_id, span_id);
|
|
35
|
+
END IF;
|
|
36
|
+
END
|
|
37
|
+
$$;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
-- Migration 049: alert_state.notified
|
|
2
|
+
--
|
|
3
|
+
-- Mirrors ../migrations/049_alert_state_notified.sql. INTEGER 0/1 like the
|
|
4
|
+
-- sibling alert_evaluations.notified (not BOOLEAN), so the shared store SQL
|
|
5
|
+
-- can test and set it the same way on both backends.
|
|
6
|
+
--
|
|
7
|
+
-- Backfill: existing rows default to 1 (delivered); the pre-049 evaluator
|
|
8
|
+
-- only stored a transition after the channels accepted it. Idempotent.
|
|
9
|
+
|
|
10
|
+
ALTER TABLE alert_state ADD COLUMN IF NOT EXISTS notified INTEGER NOT NULL DEFAULT 1;
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
-- ============================================================================
|
|
2
|
+
-- Migration 050: AutoSRE Causal Ontology, Change Correlation & Incident Memory
|
|
3
|
+
-- (RFC 0013 — PostgreSQL mirror)
|
|
4
|
+
-- ============================================================================
|
|
5
|
+
|
|
6
|
+
-- 0. Denormalize environment, version, and target_node_id onto existing tables
|
|
7
|
+
-- for O(log N) cohort rate lift and cross-trace contention queries.
|
|
8
|
+
ALTER TABLE telemetry_spans ADD COLUMN IF NOT EXISTS environment TEXT NOT NULL DEFAULT 'default';
|
|
9
|
+
ALTER TABLE telemetry_spans ADD COLUMN IF NOT EXISTS service_version TEXT;
|
|
10
|
+
ALTER TABLE telemetry_spans ADD COLUMN IF NOT EXISTS commit_sha TEXT;
|
|
11
|
+
ALTER TABLE telemetry_spans ADD COLUMN IF NOT EXISTS target_node_id TEXT;
|
|
12
|
+
ALTER TABLE telemetry_spans ADD COLUMN IF NOT EXISTS error_signature TEXT;
|
|
13
|
+
|
|
14
|
+
CREATE INDEX IF NOT EXISTS idx_spans_cohort_attribution
|
|
15
|
+
ON telemetry_spans (project_id, environment, service_name, service_version, received_at DESC);
|
|
16
|
+
|
|
17
|
+
CREATE INDEX IF NOT EXISTS idx_spans_contention_lookup
|
|
18
|
+
ON telemetry_spans (project_id, environment, target_node_id, start_time, end_time)
|
|
19
|
+
WHERE target_node_id IS NOT NULL;
|
|
20
|
+
|
|
21
|
+
ALTER TABLE actions ADD COLUMN IF NOT EXISTS environment TEXT NOT NULL DEFAULT 'default';
|
|
22
|
+
ALTER TABLE actions ADD COLUMN IF NOT EXISTS service_version TEXT;
|
|
23
|
+
ALTER TABLE actions ADD COLUMN IF NOT EXISTS target_node_id TEXT;
|
|
24
|
+
|
|
25
|
+
-- Extend eval_runs and add eval_case_results for verifiable PairedCounterfactualProof
|
|
26
|
+
ALTER TABLE eval_runs ADD COLUMN IF NOT EXISTS environment TEXT NOT NULL DEFAULT 'default';
|
|
27
|
+
ALTER TABLE eval_runs ADD COLUMN IF NOT EXISTS service_version TEXT;
|
|
28
|
+
ALTER TABLE eval_runs ADD COLUMN IF NOT EXISTS commit_sha TEXT;
|
|
29
|
+
ALTER TABLE eval_runs ADD COLUMN IF NOT EXISTS config_digest TEXT;
|
|
30
|
+
ALTER TABLE eval_runs ADD COLUMN IF NOT EXISTS input_payload_hash TEXT;
|
|
31
|
+
ALTER TABLE eval_runs ADD COLUMN IF NOT EXISTS failure_fingerprint TEXT;
|
|
32
|
+
|
|
33
|
+
ALTER TABLE eval_case_results ADD COLUMN IF NOT EXISTS input_payload_hash TEXT;
|
|
34
|
+
ALTER TABLE eval_case_results ADD COLUMN IF NOT EXISTS failure_fingerprint TEXT;
|
|
35
|
+
|
|
36
|
+
CREATE INDEX IF NOT EXISTS idx_eval_case_results_counterfactual
|
|
37
|
+
ON eval_case_results (project_id, eval_case_id, input_payload_hash, passed);
|
|
38
|
+
|
|
39
|
+
-- 1. Environment-Scoped Persistent Operational Ontology Nodes
|
|
40
|
+
CREATE TABLE IF NOT EXISTS ontology_nodes (
|
|
41
|
+
project_id TEXT NOT NULL DEFAULT 'default',
|
|
42
|
+
environment TEXT NOT NULL DEFAULT 'default',
|
|
43
|
+
id TEXT NOT NULL,
|
|
44
|
+
kind TEXT NOT NULL,
|
|
45
|
+
name TEXT NOT NULL,
|
|
46
|
+
namespace_or_endpoint TEXT NOT NULL DEFAULT 'default',
|
|
47
|
+
is_scoped_identity INTEGER NOT NULL DEFAULT 1,
|
|
48
|
+
current_version TEXT,
|
|
49
|
+
current_commit_sha TEXT,
|
|
50
|
+
current_config_digest TEXT,
|
|
51
|
+
attributes_json TEXT,
|
|
52
|
+
first_seen_at TEXT NOT NULL,
|
|
53
|
+
last_seen_at TEXT NOT NULL,
|
|
54
|
+
PRIMARY KEY (project_id, environment, id)
|
|
55
|
+
);
|
|
56
|
+
|
|
57
|
+
-- 2. Directed Operational Ontology Edges (Cycles Permitted; Handled via Tarjan SCC)
|
|
58
|
+
CREATE TABLE IF NOT EXISTS ontology_edges (
|
|
59
|
+
project_id TEXT NOT NULL DEFAULT 'default',
|
|
60
|
+
environment TEXT NOT NULL DEFAULT 'default',
|
|
61
|
+
id TEXT NOT NULL,
|
|
62
|
+
source_node_id TEXT NOT NULL,
|
|
63
|
+
predicate TEXT NOT NULL,
|
|
64
|
+
target_node_id TEXT NOT NULL,
|
|
65
|
+
relationship_certainty TEXT NOT NULL DEFAULT 'exact',
|
|
66
|
+
call_count INTEGER NOT NULL DEFAULT 0,
|
|
67
|
+
error_count INTEGER NOT NULL DEFAULT 0,
|
|
68
|
+
last_duration_ms DOUBLE PRECISION,
|
|
69
|
+
first_seen_at TEXT NOT NULL,
|
|
70
|
+
last_seen_at TEXT NOT NULL,
|
|
71
|
+
PRIMARY KEY (project_id, environment, id)
|
|
72
|
+
);
|
|
73
|
+
|
|
74
|
+
CREATE INDEX IF NOT EXISTS idx_ontology_edges_source
|
|
75
|
+
ON ontology_edges (project_id, environment, source_node_id);
|
|
76
|
+
|
|
77
|
+
CREATE INDEX IF NOT EXISTS idx_ontology_edges_target
|
|
78
|
+
ON ontology_edges (project_id, environment, target_node_id);
|
|
79
|
+
|
|
80
|
+
-- 3. Explicit & Inferred Change Events
|
|
81
|
+
CREATE TABLE IF NOT EXISTS change_events (
|
|
82
|
+
project_id TEXT NOT NULL DEFAULT 'default',
|
|
83
|
+
environment TEXT NOT NULL DEFAULT 'default',
|
|
84
|
+
id TEXT NOT NULL,
|
|
85
|
+
target_node_id TEXT NOT NULL,
|
|
86
|
+
kind TEXT NOT NULL,
|
|
87
|
+
source TEXT NOT NULL,
|
|
88
|
+
relationship_certainty TEXT NOT NULL DEFAULT 'exact',
|
|
89
|
+
summary TEXT NOT NULL,
|
|
90
|
+
service_name TEXT,
|
|
91
|
+
version TEXT,
|
|
92
|
+
previous_version TEXT,
|
|
93
|
+
commit_sha TEXT,
|
|
94
|
+
previous_commit_sha TEXT,
|
|
95
|
+
config_digest TEXT,
|
|
96
|
+
repository_url TEXT,
|
|
97
|
+
pull_request_url TEXT,
|
|
98
|
+
actor_id TEXT,
|
|
99
|
+
caused_by_action_id TEXT,
|
|
100
|
+
agent_run_id TEXT,
|
|
101
|
+
touched_paths_json TEXT,
|
|
102
|
+
attributes_json TEXT,
|
|
103
|
+
occurred_at TEXT NOT NULL,
|
|
104
|
+
received_at TEXT NOT NULL,
|
|
105
|
+
expires_at TEXT,
|
|
106
|
+
PRIMARY KEY (project_id, environment, id)
|
|
107
|
+
);
|
|
108
|
+
|
|
109
|
+
CREATE INDEX IF NOT EXISTS idx_change_events_node_time
|
|
110
|
+
ON change_events (project_id, environment, target_node_id, occurred_at DESC);
|
|
111
|
+
|
|
112
|
+
-- 4. Durable Incident Investigations & Runner Lease Coordination
|
|
113
|
+
CREATE TABLE IF NOT EXISTS incident_investigations (
|
|
114
|
+
project_id TEXT NOT NULL DEFAULT 'default',
|
|
115
|
+
environment TEXT NOT NULL DEFAULT 'default',
|
|
116
|
+
id TEXT NOT NULL,
|
|
117
|
+
dedup_key TEXT NOT NULL,
|
|
118
|
+
status TEXT NOT NULL DEFAULT 'open',
|
|
119
|
+
target_node_id TEXT NOT NULL,
|
|
120
|
+
symptom_anchor_kind TEXT NOT NULL,
|
|
121
|
+
symptom_anchor_id TEXT NOT NULL,
|
|
122
|
+
symptom_fingerprint TEXT,
|
|
123
|
+
alert_count INTEGER NOT NULL DEFAULT 1,
|
|
124
|
+
latest_bundle_json TEXT,
|
|
125
|
+
lease_owner_id TEXT,
|
|
126
|
+
lease_expires_at TEXT,
|
|
127
|
+
dispatch_attempts INTEGER NOT NULL DEFAULT 0,
|
|
128
|
+
last_dispatched_at TEXT,
|
|
129
|
+
opened_at TEXT NOT NULL,
|
|
130
|
+
updated_at TEXT NOT NULL,
|
|
131
|
+
resolved_at TEXT,
|
|
132
|
+
expires_at TEXT,
|
|
133
|
+
PRIMARY KEY (project_id, environment, id)
|
|
134
|
+
);
|
|
135
|
+
|
|
136
|
+
CREATE INDEX IF NOT EXISTS idx_incidents_active_dedup
|
|
137
|
+
ON incident_investigations (project_id, environment, dedup_key, status);
|
|
138
|
+
|
|
139
|
+
-- 5. Closed-Loop Investigation Findings (Separating Recovery vs Causal Proof)
|
|
140
|
+
CREATE TABLE IF NOT EXISTS investigation_findings (
|
|
141
|
+
project_id TEXT NOT NULL DEFAULT 'default',
|
|
142
|
+
environment TEXT NOT NULL DEFAULT 'default',
|
|
143
|
+
id TEXT NOT NULL,
|
|
144
|
+
incident_id TEXT,
|
|
145
|
+
fingerprint TEXT NOT NULL,
|
|
146
|
+
fingerprint_input_json TEXT NOT NULL,
|
|
147
|
+
title TEXT NOT NULL,
|
|
148
|
+
summary TEXT NOT NULL,
|
|
149
|
+
causal_proof_tier TEXT NOT NULL DEFAULT 'unverified_locus',
|
|
150
|
+
confidence DOUBLE PRECISION NOT NULL,
|
|
151
|
+
confidence_breakdown_json TEXT NOT NULL,
|
|
152
|
+
symptom_entity_kind TEXT NOT NULL,
|
|
153
|
+
symptom_entity_id TEXT NOT NULL,
|
|
154
|
+
fault_locus_entity_kind TEXT NOT NULL,
|
|
155
|
+
fault_locus_entity_id TEXT NOT NULL,
|
|
156
|
+
root_cause_entity_kind TEXT,
|
|
157
|
+
root_cause_entity_id TEXT,
|
|
158
|
+
contributing_change_id TEXT,
|
|
159
|
+
cohort_verdict TEXT,
|
|
160
|
+
contending_trace_id TEXT,
|
|
161
|
+
causal_chain_json TEXT NOT NULL,
|
|
162
|
+
cited_evidence_ids_json TEXT NOT NULL,
|
|
163
|
+
code_references_json TEXT,
|
|
164
|
+
counterfactual_proof_json TEXT,
|
|
165
|
+
recovery_status TEXT NOT NULL DEFAULT 'not_applicable',
|
|
166
|
+
causal_verification_status TEXT NOT NULL DEFAULT 'unverified',
|
|
167
|
+
watchdog_step INTEGER NOT NULL DEFAULT 0,
|
|
168
|
+
baseline_request_rate_per_min DOUBLE PRECISION,
|
|
169
|
+
next_watchdog_check_at TEXT,
|
|
170
|
+
actor_type TEXT NOT NULL,
|
|
171
|
+
actor_id TEXT NOT NULL,
|
|
172
|
+
agent_run_id TEXT,
|
|
173
|
+
created_at TEXT NOT NULL,
|
|
174
|
+
updated_at TEXT NOT NULL,
|
|
175
|
+
expires_at TEXT,
|
|
176
|
+
PRIMARY KEY (project_id, environment, id)
|
|
177
|
+
);
|
|
178
|
+
|
|
179
|
+
CREATE INDEX IF NOT EXISTS idx_findings_fingerprint
|
|
180
|
+
ON investigation_findings (project_id, environment, fingerprint, created_at DESC);
|
|
181
|
+
|
|
182
|
+
CREATE INDEX IF NOT EXISTS idx_findings_watchdog
|
|
183
|
+
ON investigation_findings (project_id, recovery_status, next_watchdog_check_at);
|
|
184
|
+
|
|
185
|
+
-- 6. Pre-Execution Remediation Proposals & Single-Use Grants
|
|
186
|
+
CREATE TABLE IF NOT EXISTS remediation_proposals (
|
|
187
|
+
project_id TEXT NOT NULL DEFAULT 'default',
|
|
188
|
+
environment TEXT NOT NULL DEFAULT 'default',
|
|
189
|
+
id TEXT NOT NULL,
|
|
190
|
+
incident_id TEXT NOT NULL,
|
|
191
|
+
finding_id TEXT NOT NULL,
|
|
192
|
+
idempotency_key TEXT NOT NULL,
|
|
193
|
+
action_kind TEXT NOT NULL,
|
|
194
|
+
target_node_id TEXT NOT NULL,
|
|
195
|
+
expected_current_state_json TEXT NOT NULL,
|
|
196
|
+
desired_target_state_json TEXT NOT NULL,
|
|
197
|
+
requested_autonomy TEXT NOT NULL,
|
|
198
|
+
effective_autonomy TEXT NOT NULL,
|
|
199
|
+
policy_decision_reason TEXT NOT NULL,
|
|
200
|
+
status TEXT NOT NULL DEFAULT 'proposed',
|
|
201
|
+
grant_token_hash TEXT,
|
|
202
|
+
grant_expires_at TEXT,
|
|
203
|
+
approved_by_user_id TEXT,
|
|
204
|
+
approved_at TEXT,
|
|
205
|
+
claimed_by_runner_id TEXT,
|
|
206
|
+
claimed_at TEXT,
|
|
207
|
+
claim_expires_at TEXT,
|
|
208
|
+
remediation_agent_run_id TEXT,
|
|
209
|
+
resulting_change_id TEXT,
|
|
210
|
+
failure_reason TEXT,
|
|
211
|
+
created_at TEXT NOT NULL,
|
|
212
|
+
updated_at TEXT NOT NULL,
|
|
213
|
+
expires_at TEXT,
|
|
214
|
+
PRIMARY KEY (project_id, environment, id),
|
|
215
|
+
UNIQUE (project_id, environment, idempotency_key)
|
|
216
|
+
);
|
|
@@ -32,9 +32,62 @@ remaining migrations are tracked in
|
|
|
32
32
|
[`docs/postgres-migration.md`](../../../../docs/postgres-migration.md); they'll
|
|
33
33
|
land alongside the first hosted-Postgres deployment.
|
|
34
34
|
|
|
35
|
-
##
|
|
35
|
+
## Column types the stores depend on
|
|
36
36
|
|
|
37
|
-
`
|
|
38
|
-
|
|
39
|
-
`
|
|
40
|
-
|
|
37
|
+
- `ingest_keys.key_hash` is `TEXT` on both backends. `ProjectsStore` binds the
|
|
38
|
+
SHA-256 hex digest as a string; an earlier Postgres version of `009` declared
|
|
39
|
+
the column `BYTEA`, which silently stored the ASCII bytes of the hex text and
|
|
40
|
+
made the `026` seed key unreachable. `041_ingest_key_hash_text.sql` converts
|
|
41
|
+
databases migrated with that version back to `TEXT` (fresh installs get
|
|
42
|
+
`TEXT` from `009` directly and `041` is a no-op). The D1 `041` file is an
|
|
43
|
+
intentional no-op that only keeps the numbering in parity.
|
|
44
|
+
- Timestamp columns (`received_at`, `expires_at`, …) are `TEXT` holding
|
|
45
|
+
`Date#toISOString()` output. Postgres will not compare `text` to
|
|
46
|
+
`timestamptz`, so `postgresDialect` and `translateD1Sql` render every time
|
|
47
|
+
window as ISO text (`to_char(…, 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')`).
|
|
48
|
+
`metric_exemplars` originally declared `received_at`/`expires_at` as
|
|
49
|
+
`TIMESTAMPTZ`, which made `MetricsStore.purgeExpired` fail with
|
|
50
|
+
`operator does not exist: timestamp with time zone < text`;
|
|
51
|
+
`043_metric_exemplars_text_timestamps.sql` converts those columns to UTC
|
|
52
|
+
ISO `TEXT` in place (fresh installs get `TEXT` from `035` and `043` is a
|
|
53
|
+
no-op; the D1 `043` is a numbering no-op).
|
|
54
|
+
- `actions.started_at` is ISO `TEXT` and drives action-graph retention (the
|
|
55
|
+
spine has no `expires_at`). `044_actions_started_at_index.sql` adds
|
|
56
|
+
`idx_actions_started_at` on both backends so the hourly sweep reads expired
|
|
57
|
+
trees in index order. Deletes run per tree inside `PostgresAdapter.batch`
|
|
58
|
+
(one transaction) and re-check eligibility in every statement, but under
|
|
59
|
+
READ COMMITTED a concurrent insert into an already-expired tree can still
|
|
60
|
+
land between that tree's leaf deletes and its spine delete; see the header
|
|
61
|
+
of `src/lib/action-retention.ts`.
|
|
62
|
+
- Row keys are project-scoped. `045`–`048` swap the single-column primary
|
|
63
|
+
keys for composite ones in place (`user_profiles (project_id, user_id)`;
|
|
64
|
+
`actions`, `agent_runs`, `tool_calls`, `retrieval_events`, `eval_results`,
|
|
65
|
+
`artifacts (project_id, id)`; `ai_span_payloads` and
|
|
66
|
+
`telemetry_spans (project_id, trace_id, span_id)`). Each file looks up the
|
|
67
|
+
current primary key by definition and is a no-op once the key is
|
|
68
|
+
composite; the D1 siblings rebuild the tables through `*_new` copies. The
|
|
69
|
+
stores upsert with `ON CONFLICT (project_id, …)`, so both backends must
|
|
70
|
+
carry these migrations before deploying collector 2.1.
|
|
71
|
+
- `049` adds `alert_state.notified INTEGER` (0/1, like
|
|
72
|
+
`alert_evaluations.notified`): whether the current state's transition has
|
|
73
|
+
been delivered to the rule's channels. Existing rows backfill to 1. The
|
|
74
|
+
store falls back to the pre-049 statement when the column is missing, so
|
|
75
|
+
the collector keeps evaluating during a rolling deploy, but it cannot
|
|
76
|
+
retry undelivered notifications until 049 is applied.
|
|
77
|
+
- Flags (`is_error`, `is_bot`, `payload_capture_enabled`) are `BOOLEAN` here
|
|
78
|
+
and `INTEGER 0/1` on D1. Shared SQL must test them as bare booleans
|
|
79
|
+
(`CASE WHEN is_error THEN 1 ELSE 0 END`, never `is_error = 1` or
|
|
80
|
+
`SUM(is_error)`). Hosts call `configurePostgresTypeParsers(pg.types)` so
|
|
81
|
+
BOOLEAN reads back as `0/1`, BIGINT/NUMERIC as numbers and JSON/JSONB as
|
|
82
|
+
text, matching what D1 returns.
|
|
83
|
+
|
|
84
|
+
## Verifying against a real database
|
|
85
|
+
|
|
86
|
+
`src/lib/sql-db-postgres.integration.test.ts` applies this directory into an
|
|
87
|
+
empty scratch database, then exercises ingest-key auth, AI/usage ingest, the
|
|
88
|
+
retention purge and every built-in analysis through `PostgresAdapter`:
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
OBS_PG_TEST_URL=postgres://obs:obs@127.0.0.1:5432/obs_scratch \
|
|
92
|
+
pnpm --filter @obsunified/collector exec vitest run src/lib/sql-db-postgres.integration.test.ts
|
|
93
|
+
```
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { gzipSync } from "node:zlib";
|
|
2
|
+
import { Hono } from "hono";
|
|
3
|
+
import { describe, expect, it } from "vitest";
|
|
4
|
+
import { OtlpDecodeError, type ReadBodyLimits, readOtlpBody } from "./body";
|
|
5
|
+
|
|
6
|
+
const MIB = 1024 * 1024;
|
|
7
|
+
|
|
8
|
+
const read = async (
|
|
9
|
+
body: BodyInit,
|
|
10
|
+
headers: Record<string, string>,
|
|
11
|
+
limits: ReadBodyLimits = {},
|
|
12
|
+
) => {
|
|
13
|
+
const app = new Hono().post("/t", async (c) => {
|
|
14
|
+
try {
|
|
15
|
+
const result = await readOtlpBody(c, limits);
|
|
16
|
+
return c.json({
|
|
17
|
+
bytes: result.bytes.byteLength,
|
|
18
|
+
wire: result.wireFormat,
|
|
19
|
+
});
|
|
20
|
+
} catch (err) {
|
|
21
|
+
if (err instanceof OtlpDecodeError)
|
|
22
|
+
return c.json({ error: err.message }, err.status);
|
|
23
|
+
throw err;
|
|
24
|
+
}
|
|
25
|
+
});
|
|
26
|
+
const res = await app.request("/t", { method: "POST", body, headers });
|
|
27
|
+
return {
|
|
28
|
+
status: res.status,
|
|
29
|
+
body: (await res.json()) as Record<string, unknown>,
|
|
30
|
+
};
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
describe("readOtlpBody limits", () => {
|
|
34
|
+
it("decompresses a normal gzip payload", async () => {
|
|
35
|
+
const json = JSON.stringify({ resourceSpans: [] });
|
|
36
|
+
const res = await read(gzipSync(json), {
|
|
37
|
+
"content-type": "application/json",
|
|
38
|
+
"content-encoding": "gzip",
|
|
39
|
+
});
|
|
40
|
+
expect(res).toEqual({
|
|
41
|
+
status: 200,
|
|
42
|
+
body: { bytes: json.length, wire: "json" },
|
|
43
|
+
});
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
it("refuses a gzip bomb once the decompressed cap is crossed", async () => {
|
|
47
|
+
// 8 MiB of zeros gzips to a few KiB; the reader must stop at 1 MiB.
|
|
48
|
+
const bomb = gzipSync(Buffer.alloc(8 * MIB));
|
|
49
|
+
expect(bomb.byteLength).toBeLessThan(64 * 1024);
|
|
50
|
+
const res = await read(
|
|
51
|
+
bomb,
|
|
52
|
+
{ "content-type": "application/x-protobuf", "content-encoding": "gzip" },
|
|
53
|
+
{ maxDecompressedBytes: MIB },
|
|
54
|
+
);
|
|
55
|
+
expect(res.status).toBe(413);
|
|
56
|
+
expect(res.body.error).toMatch(/Decompressed payload too large/);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
it("refuses a raw body over the configured cap", async () => {
|
|
60
|
+
const res = await read(
|
|
61
|
+
"x".repeat(2048),
|
|
62
|
+
{ "content-type": "application/json" },
|
|
63
|
+
{ maxBodyBytes: 1024 },
|
|
64
|
+
);
|
|
65
|
+
expect(res.status).toBe(413);
|
|
66
|
+
expect(res.body.error).toMatch(/Payload too large \(max 1024 bytes\)/);
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
it("reports malformed gzip as 400, not 500", async () => {
|
|
70
|
+
const res = await read("definitely not gzip", {
|
|
71
|
+
"content-type": "application/json",
|
|
72
|
+
"content-encoding": "gzip",
|
|
73
|
+
});
|
|
74
|
+
expect(res.status).toBe(400);
|
|
75
|
+
});
|
|
76
|
+
});
|