@obsunified/collector 2.0.1 → 2.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth.d.ts +1 -1
- package/dist/auth.js +1 -1
- package/dist/{chunk-UGWDOGGH.js → chunk-KMEZKGSO.js} +22 -10
- package/dist/chunk-KMEZKGSO.js.map +1 -0
- package/dist/index.d.ts +322 -26
- package/dist/index.js +1478 -415
- package/dist/index.js.map +1 -1
- package/dist/{ingest-auth-HFVO1zIp.d.ts → ingest-auth-C5-8Mly5.d.ts} +9 -0
- package/package.json +5 -5
- package/src/analyses/derive.ts +26 -25
- package/src/analyses/investigations.ts +8 -8
- package/src/analyses/tier0.ts +22 -22
- package/src/auth/dashboard-auth.test.ts +791 -0
- package/src/auth/dashboard-auth.ts +187 -38
- package/src/auth/login-rate-limit.test.ts +129 -0
- package/src/auth/login-rate-limit.ts +143 -0
- package/src/framework/body-limit.test.ts +165 -0
- package/src/framework/collector.ts +355 -115
- package/src/framework/env.ts +9 -0
- package/src/framework/limits.ts +31 -0
- package/src/framework/scheduled-handlers.test.ts +342 -0
- package/src/index.ts +16 -1
- package/src/lib/action-retention.test.ts +472 -0
- package/src/lib/action-retention.ts +420 -0
- package/src/lib/actions-store.ts +6 -12
- package/src/lib/ai-store/calls.ts +2 -2
- package/src/lib/ai-store/sessions.ts +3 -3
- package/src/lib/ai-store/spans.ts +1 -1
- package/src/lib/ai-store/types.ts +2 -1
- package/src/lib/alerts-store.test.ts +153 -0
- package/src/lib/alerts-store.ts +122 -21
- package/src/lib/analyses-runner.test.ts +75 -0
- package/src/lib/analyses-runner.ts +60 -3
- package/src/lib/analyses-store-latest.test.ts +61 -0
- package/src/lib/analyses-store.ts +17 -10
- package/src/lib/blob-store.ts +1 -1
- package/src/lib/housekeeping.test.ts +73 -0
- package/src/lib/housekeeping.ts +93 -0
- package/src/lib/identity-index/action-lookups.ts +6 -6
- package/src/lib/identity-index/key-lookups.ts +2 -2
- package/src/lib/identity-index/user-lookup.ts +8 -5
- package/src/lib/identity-index.test.ts +2 -3
- package/src/lib/metrics-store-series.test.ts +99 -0
- package/src/lib/metrics-store.test.ts +23 -0
- package/src/lib/metrics-store.ts +27 -3
- package/src/lib/projects-store.test.ts +25 -0
- package/src/lib/projects-store.ts +4 -3
- package/src/lib/sql-db-dialect.test.ts +12 -3
- package/src/lib/sql-db-postgres.integration.test.ts +1225 -0
- package/src/lib/sql-db-postgres.test.ts +314 -0
- package/src/lib/sql-db-postgres.ts +392 -54
- package/src/lib/sql-db-sqlite-time-windows.test.ts +194 -0
- package/src/lib/sql-db.ts +40 -6
- package/src/lib/test-utils/action-graph-fixtures.ts +307 -0
- package/src/lib/test-utils/node-sqlite-db.ts +188 -0
- package/src/lib/webhook-url.test.ts +122 -0
- package/src/lib/webhook-url.ts +155 -0
- package/src/migrations/041_ingest_key_hash_text.sql +5 -0
- package/src/migrations/042_session_replay_project_composite_pk.sql +52 -0
- package/src/migrations/043_metric_exemplars_text_timestamps.sql +5 -0
- package/src/migrations/044_actions_started_at_index.sql +10 -0
- package/src/migrations/045_user_profiles_project_composite_pk.sql +40 -0
- package/src/migrations/046_action_graph_project_composite_pk.sql +255 -0
- package/src/migrations/047_ai_span_payloads_project_composite_pk.sql +61 -0
- package/src/migrations/048_telemetry_spans_project_composite_pk.sql +92 -0
- package/src/migrations/049_alert_state_notified.sql +17 -0
- package/src/migrations/alert-state-notified.test.ts +111 -0
- package/src/migrations/parity.test.ts +64 -0
- package/src/migrations/project-scoped-keys.test.ts +432 -0
- package/src/migrations-postgres/009_projects_and_keys.sql +4 -4
- package/src/migrations-postgres/026_obs_dashboard_project_seed.sql +1 -1
- package/src/migrations-postgres/035_metric_exemplars.sql +5 -2
- package/src/migrations-postgres/041_ingest_key_hash_text.sql +32 -0
- package/src/migrations-postgres/042_session_replay_project_composite_pk.sql +16 -0
- package/src/migrations-postgres/043_metric_exemplars_text_timestamps.sql +47 -0
- package/src/migrations-postgres/044_actions_started_at_index.sql +8 -0
- package/src/migrations-postgres/045_user_profiles_project_composite_pk.sql +33 -0
- package/src/migrations-postgres/046_action_graph_project_composite_pk.sql +39 -0
- package/src/migrations-postgres/047_ai_span_payloads_project_composite_pk.sql +30 -0
- package/src/migrations-postgres/048_telemetry_spans_project_composite_pk.sql +37 -0
- package/src/migrations-postgres/049_alert_state_notified.sql +10 -0
- package/src/migrations-postgres/README.md +58 -5
- package/src/otlp/decode/body.test.ts +76 -0
- package/src/otlp/decode/body.ts +67 -6
- package/src/plugins/_json.ts +21 -0
- package/src/plugins/action-graph-processor.ts +6 -12
- package/src/plugins/ai-receiver.ts +5 -12
- package/src/plugins/ai-span-payloads-processor.ts +1 -2
- package/src/plugins/alerts-evaluator-retention.test.ts +170 -0
- package/src/plugins/alerts-evaluator.test.ts +464 -3
- package/src/plugins/alerts-evaluator.ts +165 -41
- package/src/plugins/alerts-routes.ts +10 -6
- package/src/plugins/ask-routes.ts +4 -5
- package/src/plugins/connected-routes.test.ts +106 -0
- package/src/plugins/connected-routes.ts +14 -5
- package/src/plugins/dashboard-routes.test.ts +120 -0
- package/src/plugins/dashboard-routes.ts +11 -2
- package/src/plugins/eval-cases-routes.ts +8 -17
- package/src/plugins/identity-receiver.ts +9 -11
- package/src/plugins/json-body.test.ts +91 -0
- package/src/plugins/onboarding-routes.test.ts +105 -0
- package/src/plugins/onboarding-routes.ts +26 -11
- package/src/plugins/projects-routes.ts +2 -1
- package/src/plugins/replay-isolation.test.ts +922 -0
- package/src/plugins/replay-receiver.ts +8 -8
- package/src/plugins/usage-receiver.ts +3 -6
- package/dist/chunk-UGWDOGGH.js.map +0 -1
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
-- Migration 046: action graph tables keyed by (project_id, id)
|
|
2
|
+
--
|
|
3
|
+
-- The RFC 0010 spine and leaf tables (031) used `id TEXT PRIMARY KEY`.
|
|
4
|
+
-- Action ids are client-supplied (`obs.action.id`) and leaf ids are raw
|
|
5
|
+
-- OTel span ids, so two projects can legitimately emit the same id. The
|
|
6
|
+
-- stores upserted with ON CONFLICT(id) DO UPDATE SET project_id = ..., which
|
|
7
|
+
-- let one tenant move or overwrite another tenant's row.
|
|
8
|
+
--
|
|
9
|
+
-- Rebuilds all six tables with PRIMARY KEY (project_id, id), preserving every
|
|
10
|
+
-- row, column default and index (031, 038, 044). The old key was unique, so
|
|
11
|
+
-- the composite key cannot collide. Re-running the file is a no-op in effect.
|
|
12
|
+
|
|
13
|
+
-- 1. actions
|
|
14
|
+
DROP TABLE IF EXISTS actions_new;
|
|
15
|
+
|
|
16
|
+
CREATE TABLE actions_new (
|
|
17
|
+
project_id TEXT NOT NULL,
|
|
18
|
+
id TEXT NOT NULL,
|
|
19
|
+
root_action_id TEXT NOT NULL,
|
|
20
|
+
caused_by_action_id TEXT,
|
|
21
|
+
actor_type TEXT NOT NULL,
|
|
22
|
+
actor_id TEXT,
|
|
23
|
+
action_kind TEXT NOT NULL,
|
|
24
|
+
name TEXT,
|
|
25
|
+
status TEXT NOT NULL DEFAULT 'ok',
|
|
26
|
+
started_at TEXT NOT NULL,
|
|
27
|
+
ended_at TEXT,
|
|
28
|
+
duration_ms INTEGER,
|
|
29
|
+
trace_id TEXT,
|
|
30
|
+
span_id TEXT,
|
|
31
|
+
session_id TEXT,
|
|
32
|
+
interaction_id TEXT,
|
|
33
|
+
user_id TEXT,
|
|
34
|
+
agent_run_id TEXT,
|
|
35
|
+
step_id TEXT,
|
|
36
|
+
tool_call_id TEXT,
|
|
37
|
+
prompt_version TEXT,
|
|
38
|
+
model_name TEXT,
|
|
39
|
+
provider TEXT,
|
|
40
|
+
total_cost_usd REAL,
|
|
41
|
+
attrs_json TEXT,
|
|
42
|
+
PRIMARY KEY (project_id, id)
|
|
43
|
+
);
|
|
44
|
+
|
|
45
|
+
INSERT INTO actions_new (
|
|
46
|
+
project_id, id, root_action_id, caused_by_action_id, actor_type, actor_id,
|
|
47
|
+
action_kind, name, status, started_at, ended_at, duration_ms, trace_id,
|
|
48
|
+
span_id, session_id, interaction_id, user_id, agent_run_id, step_id,
|
|
49
|
+
tool_call_id, prompt_version, model_name, provider, total_cost_usd, attrs_json
|
|
50
|
+
)
|
|
51
|
+
SELECT
|
|
52
|
+
project_id, id, root_action_id, caused_by_action_id, actor_type, actor_id,
|
|
53
|
+
action_kind, name, COALESCE(status, 'ok'), started_at, ended_at, duration_ms,
|
|
54
|
+
trace_id, span_id, session_id, interaction_id, user_id, agent_run_id, step_id,
|
|
55
|
+
tool_call_id, prompt_version, model_name, provider, total_cost_usd, attrs_json
|
|
56
|
+
FROM actions;
|
|
57
|
+
|
|
58
|
+
DROP TABLE actions;
|
|
59
|
+
|
|
60
|
+
ALTER TABLE actions_new RENAME TO actions;
|
|
61
|
+
|
|
62
|
+
CREATE INDEX IF NOT EXISTS idx_actions_project_root
|
|
63
|
+
ON actions (project_id, root_action_id, started_at);
|
|
64
|
+
CREATE INDEX IF NOT EXISTS idx_actions_project_actor
|
|
65
|
+
ON actions (project_id, actor_type, actor_id, started_at DESC);
|
|
66
|
+
CREATE INDEX IF NOT EXISTS idx_actions_project_trace
|
|
67
|
+
ON actions (project_id, trace_id)
|
|
68
|
+
WHERE trace_id IS NOT NULL;
|
|
69
|
+
CREATE INDEX IF NOT EXISTS idx_actions_project_interaction
|
|
70
|
+
ON actions (project_id, interaction_id)
|
|
71
|
+
WHERE interaction_id IS NOT NULL;
|
|
72
|
+
CREATE INDEX IF NOT EXISTS idx_actions_project_agent_run
|
|
73
|
+
ON actions (project_id, agent_run_id, started_at)
|
|
74
|
+
WHERE agent_run_id IS NOT NULL;
|
|
75
|
+
CREATE INDEX IF NOT EXISTS idx_actions_started_at
|
|
76
|
+
ON actions (started_at);
|
|
77
|
+
|
|
78
|
+
-- 2. agent_runs
|
|
79
|
+
DROP TABLE IF EXISTS agent_runs_new;
|
|
80
|
+
|
|
81
|
+
CREATE TABLE agent_runs_new (
|
|
82
|
+
project_id TEXT NOT NULL,
|
|
83
|
+
id TEXT NOT NULL,
|
|
84
|
+
agent_id TEXT NOT NULL,
|
|
85
|
+
agent_name TEXT NOT NULL,
|
|
86
|
+
agent_version TEXT NOT NULL,
|
|
87
|
+
goal TEXT,
|
|
88
|
+
outcome TEXT,
|
|
89
|
+
autonomy_level TEXT NOT NULL,
|
|
90
|
+
status TEXT NOT NULL DEFAULT 'running',
|
|
91
|
+
error_message TEXT,
|
|
92
|
+
total_cost_usd REAL DEFAULT 0.0,
|
|
93
|
+
total_duration_ms INTEGER,
|
|
94
|
+
metadata_json TEXT,
|
|
95
|
+
PRIMARY KEY (project_id, id)
|
|
96
|
+
);
|
|
97
|
+
|
|
98
|
+
INSERT INTO agent_runs_new (
|
|
99
|
+
project_id, id, agent_id, agent_name, agent_version, goal, outcome,
|
|
100
|
+
autonomy_level, status, error_message, total_cost_usd, total_duration_ms,
|
|
101
|
+
metadata_json
|
|
102
|
+
)
|
|
103
|
+
SELECT
|
|
104
|
+
project_id, id, agent_id, agent_name, agent_version, goal, outcome,
|
|
105
|
+
autonomy_level, COALESCE(status, 'running'), error_message, total_cost_usd,
|
|
106
|
+
total_duration_ms, metadata_json
|
|
107
|
+
FROM agent_runs;
|
|
108
|
+
|
|
109
|
+
DROP TABLE agent_runs;
|
|
110
|
+
|
|
111
|
+
ALTER TABLE agent_runs_new RENAME TO agent_runs;
|
|
112
|
+
|
|
113
|
+
CREATE INDEX IF NOT EXISTS idx_agent_runs_project_agent
|
|
114
|
+
ON agent_runs (project_id, agent_id, status);
|
|
115
|
+
|
|
116
|
+
-- 3. tool_calls (031 columns + 038 evidence columns)
|
|
117
|
+
DROP TABLE IF EXISTS tool_calls_new;
|
|
118
|
+
|
|
119
|
+
CREATE TABLE tool_calls_new (
|
|
120
|
+
project_id TEXT NOT NULL,
|
|
121
|
+
id TEXT NOT NULL,
|
|
122
|
+
action_id TEXT NOT NULL,
|
|
123
|
+
tool_name TEXT NOT NULL,
|
|
124
|
+
args_hash TEXT NOT NULL,
|
|
125
|
+
result_hash TEXT NOT NULL,
|
|
126
|
+
error_type TEXT,
|
|
127
|
+
side_effect INTEGER NOT NULL DEFAULT 0,
|
|
128
|
+
approval_state TEXT,
|
|
129
|
+
args_redacted TEXT,
|
|
130
|
+
result_redacted TEXT,
|
|
131
|
+
mcp_audit_json TEXT,
|
|
132
|
+
mutation_before_json TEXT,
|
|
133
|
+
mutation_after_json TEXT,
|
|
134
|
+
mutation_diff_json TEXT,
|
|
135
|
+
mutation_artifact_id TEXT,
|
|
136
|
+
PRIMARY KEY (project_id, id)
|
|
137
|
+
);
|
|
138
|
+
|
|
139
|
+
INSERT INTO tool_calls_new (
|
|
140
|
+
project_id, id, action_id, tool_name, args_hash, result_hash, error_type,
|
|
141
|
+
side_effect, approval_state, args_redacted, result_redacted, mcp_audit_json,
|
|
142
|
+
mutation_before_json, mutation_after_json, mutation_diff_json,
|
|
143
|
+
mutation_artifact_id
|
|
144
|
+
)
|
|
145
|
+
SELECT
|
|
146
|
+
project_id, id, action_id, tool_name, args_hash, result_hash, error_type,
|
|
147
|
+
COALESCE(side_effect, 0), approval_state, args_redacted, result_redacted,
|
|
148
|
+
mcp_audit_json, mutation_before_json, mutation_after_json,
|
|
149
|
+
mutation_diff_json, mutation_artifact_id
|
|
150
|
+
FROM tool_calls;
|
|
151
|
+
|
|
152
|
+
DROP TABLE tool_calls;
|
|
153
|
+
|
|
154
|
+
ALTER TABLE tool_calls_new RENAME TO tool_calls;
|
|
155
|
+
|
|
156
|
+
CREATE INDEX IF NOT EXISTS idx_tool_calls_action
|
|
157
|
+
ON tool_calls (action_id);
|
|
158
|
+
CREATE INDEX IF NOT EXISTS idx_tool_calls_name
|
|
159
|
+
ON tool_calls (project_id, tool_name);
|
|
160
|
+
|
|
161
|
+
-- 4. retrieval_events
|
|
162
|
+
DROP TABLE IF EXISTS retrieval_events_new;
|
|
163
|
+
|
|
164
|
+
CREATE TABLE retrieval_events_new (
|
|
165
|
+
project_id TEXT NOT NULL,
|
|
166
|
+
id TEXT NOT NULL,
|
|
167
|
+
action_id TEXT NOT NULL,
|
|
168
|
+
retriever_name TEXT NOT NULL,
|
|
169
|
+
query_hash TEXT NOT NULL,
|
|
170
|
+
documents_json TEXT,
|
|
171
|
+
total_results INTEGER NOT NULL DEFAULT 0,
|
|
172
|
+
max_relevance_score REAL,
|
|
173
|
+
duration_ms INTEGER,
|
|
174
|
+
PRIMARY KEY (project_id, id)
|
|
175
|
+
);
|
|
176
|
+
|
|
177
|
+
INSERT INTO retrieval_events_new (
|
|
178
|
+
project_id, id, action_id, retriever_name, query_hash, documents_json,
|
|
179
|
+
total_results, max_relevance_score, duration_ms
|
|
180
|
+
)
|
|
181
|
+
SELECT
|
|
182
|
+
project_id, id, action_id, retriever_name, query_hash, documents_json,
|
|
183
|
+
COALESCE(total_results, 0), max_relevance_score, duration_ms
|
|
184
|
+
FROM retrieval_events;
|
|
185
|
+
|
|
186
|
+
DROP TABLE retrieval_events;
|
|
187
|
+
|
|
188
|
+
ALTER TABLE retrieval_events_new RENAME TO retrieval_events;
|
|
189
|
+
|
|
190
|
+
CREATE INDEX IF NOT EXISTS idx_retrieval_events_action
|
|
191
|
+
ON retrieval_events (action_id);
|
|
192
|
+
|
|
193
|
+
-- 5. eval_results
|
|
194
|
+
DROP TABLE IF EXISTS eval_results_new;
|
|
195
|
+
|
|
196
|
+
CREATE TABLE eval_results_new (
|
|
197
|
+
project_id TEXT NOT NULL,
|
|
198
|
+
id TEXT NOT NULL,
|
|
199
|
+
action_id TEXT NOT NULL,
|
|
200
|
+
evaluator_name TEXT NOT NULL,
|
|
201
|
+
evaluator_version TEXT NOT NULL,
|
|
202
|
+
score REAL,
|
|
203
|
+
passed INTEGER NOT NULL DEFAULT 1,
|
|
204
|
+
reasoning TEXT,
|
|
205
|
+
rubric_json TEXT,
|
|
206
|
+
PRIMARY KEY (project_id, id)
|
|
207
|
+
);
|
|
208
|
+
|
|
209
|
+
INSERT INTO eval_results_new (
|
|
210
|
+
project_id, id, action_id, evaluator_name, evaluator_version, score, passed,
|
|
211
|
+
reasoning, rubric_json
|
|
212
|
+
)
|
|
213
|
+
SELECT
|
|
214
|
+
project_id, id, action_id, evaluator_name, evaluator_version, score,
|
|
215
|
+
COALESCE(passed, 1), reasoning, rubric_json
|
|
216
|
+
FROM eval_results;
|
|
217
|
+
|
|
218
|
+
DROP TABLE eval_results;
|
|
219
|
+
|
|
220
|
+
ALTER TABLE eval_results_new RENAME TO eval_results;
|
|
221
|
+
|
|
222
|
+
CREATE INDEX IF NOT EXISTS idx_eval_results_action
|
|
223
|
+
ON eval_results (action_id);
|
|
224
|
+
|
|
225
|
+
-- 6. artifacts
|
|
226
|
+
DROP TABLE IF EXISTS artifacts_new;
|
|
227
|
+
|
|
228
|
+
CREATE TABLE artifacts_new (
|
|
229
|
+
project_id TEXT NOT NULL,
|
|
230
|
+
id TEXT NOT NULL,
|
|
231
|
+
action_id TEXT NOT NULL,
|
|
232
|
+
artifact_name TEXT NOT NULL,
|
|
233
|
+
artifact_type TEXT NOT NULL,
|
|
234
|
+
storage_ref TEXT,
|
|
235
|
+
size_bytes INTEGER,
|
|
236
|
+
sha256_hash TEXT,
|
|
237
|
+
content_preview TEXT,
|
|
238
|
+
PRIMARY KEY (project_id, id)
|
|
239
|
+
);
|
|
240
|
+
|
|
241
|
+
INSERT INTO artifacts_new (
|
|
242
|
+
project_id, id, action_id, artifact_name, artifact_type, storage_ref,
|
|
243
|
+
size_bytes, sha256_hash, content_preview
|
|
244
|
+
)
|
|
245
|
+
SELECT
|
|
246
|
+
project_id, id, action_id, artifact_name, artifact_type, storage_ref,
|
|
247
|
+
size_bytes, sha256_hash, content_preview
|
|
248
|
+
FROM artifacts;
|
|
249
|
+
|
|
250
|
+
DROP TABLE artifacts;
|
|
251
|
+
|
|
252
|
+
ALTER TABLE artifacts_new RENAME TO artifacts;
|
|
253
|
+
|
|
254
|
+
CREATE INDEX IF NOT EXISTS idx_artifacts_action
|
|
255
|
+
ON artifacts (action_id);
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
-- Migration 047: ai_span_payloads composite primary key
|
|
2
|
+
-- (project_id, trace_id, span_id)
|
|
3
|
+
--
|
|
4
|
+
-- 019 keyed the payload side table by (trace_id, span_id) only, and the
|
|
5
|
+
-- processor upserted with ON CONFLICT(trace_id, span_id) DO UPDATE SET
|
|
6
|
+
-- project_id = excluded.project_id. Trace and span ids are produced by the
|
|
7
|
+
-- client SDK, so a second project emitting the same ids replaced the first
|
|
8
|
+
-- project's prompt/completion payloads and took ownership of the row.
|
|
9
|
+
--
|
|
10
|
+
-- Rebuilds the table with PRIMARY KEY (project_id, trace_id, span_id),
|
|
11
|
+
-- preserving every row and every index (019, 021, 031). Columns come from
|
|
12
|
+
-- 019 + 021 + 027 + 031 + 032. Re-running the file is a no-op in effect.
|
|
13
|
+
|
|
14
|
+
DROP TABLE IF EXISTS ai_span_payloads_new;
|
|
15
|
+
|
|
16
|
+
CREATE TABLE ai_span_payloads_new (
|
|
17
|
+
project_id TEXT NOT NULL,
|
|
18
|
+
trace_id TEXT NOT NULL,
|
|
19
|
+
span_id TEXT NOT NULL,
|
|
20
|
+
span_kind TEXT NOT NULL,
|
|
21
|
+
input_json TEXT,
|
|
22
|
+
output_json TEXT,
|
|
23
|
+
received_at TEXT NOT NULL,
|
|
24
|
+
expires_at TEXT NOT NULL,
|
|
25
|
+
session_id TEXT,
|
|
26
|
+
user_id TEXT,
|
|
27
|
+
interaction_id TEXT,
|
|
28
|
+
action_id TEXT,
|
|
29
|
+
input_hash TEXT,
|
|
30
|
+
output_hash TEXT,
|
|
31
|
+
PRIMARY KEY (project_id, trace_id, span_id)
|
|
32
|
+
);
|
|
33
|
+
|
|
34
|
+
INSERT INTO ai_span_payloads_new (
|
|
35
|
+
project_id, trace_id, span_id, span_kind, input_json, output_json,
|
|
36
|
+
received_at, expires_at, session_id, user_id, interaction_id, action_id,
|
|
37
|
+
input_hash, output_hash
|
|
38
|
+
)
|
|
39
|
+
SELECT
|
|
40
|
+
project_id, trace_id, span_id, span_kind, input_json, output_json,
|
|
41
|
+
received_at, expires_at, session_id, user_id, interaction_id, action_id,
|
|
42
|
+
input_hash, output_hash
|
|
43
|
+
FROM ai_span_payloads;
|
|
44
|
+
|
|
45
|
+
DROP TABLE ai_span_payloads;
|
|
46
|
+
|
|
47
|
+
ALTER TABLE ai_span_payloads_new RENAME TO ai_span_payloads;
|
|
48
|
+
|
|
49
|
+
CREATE INDEX IF NOT EXISTS idx_ai_payloads_received
|
|
50
|
+
ON ai_span_payloads (received_at DESC);
|
|
51
|
+
CREATE INDEX IF NOT EXISTS idx_ai_payloads_project
|
|
52
|
+
ON ai_span_payloads (project_id, received_at DESC);
|
|
53
|
+
CREATE INDEX IF NOT EXISTS idx_ai_payloads_kind
|
|
54
|
+
ON ai_span_payloads (project_id, span_kind, received_at DESC);
|
|
55
|
+
CREATE INDEX IF NOT EXISTS idx_ai_payloads_session
|
|
56
|
+
ON ai_span_payloads (project_id, session_id, received_at DESC);
|
|
57
|
+
CREATE INDEX IF NOT EXISTS idx_ai_payloads_user
|
|
58
|
+
ON ai_span_payloads (project_id, user_id, received_at DESC);
|
|
59
|
+
CREATE INDEX IF NOT EXISTS idx_ai_payloads_action
|
|
60
|
+
ON ai_span_payloads (project_id, action_id)
|
|
61
|
+
WHERE action_id IS NOT NULL;
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
-- Migration 048: telemetry_spans composite primary key
|
|
2
|
+
-- (project_id, trace_id, span_id)
|
|
3
|
+
--
|
|
4
|
+
-- 001 keyed spans by (trace_id, span_id); project_id arrived in 010 without
|
|
5
|
+
-- joining the key. Ingest uses INSERT OR IGNORE, so a span whose
|
|
6
|
+
-- (trace_id, span_id) already existed under another project was silently
|
|
7
|
+
-- dropped, and the first project to write a given id owned it for every
|
|
8
|
+
-- tenant.
|
|
9
|
+
--
|
|
10
|
+
-- Rebuilds the table with PRIMARY KEY (project_id, trace_id, span_id),
|
|
11
|
+
-- preserving every row, every column default (001, 010, 017, 022, 027, 029)
|
|
12
|
+
-- and every index (001, 010, 022, 023, 027, 029). The old key was unique,
|
|
13
|
+
-- so the composite key cannot collide. Re-running the file is a no-op in
|
|
14
|
+
-- effect.
|
|
15
|
+
--
|
|
16
|
+
-- Operational note: this copies the whole spans table once. Retention keeps
|
|
17
|
+
-- the table bounded, but on a busy D1 database apply it in a quiet window.
|
|
18
|
+
|
|
19
|
+
DROP TABLE IF EXISTS telemetry_spans_new;
|
|
20
|
+
|
|
21
|
+
CREATE TABLE telemetry_spans_new (
|
|
22
|
+
project_id TEXT NOT NULL DEFAULT 'default',
|
|
23
|
+
trace_id TEXT NOT NULL,
|
|
24
|
+
span_id TEXT NOT NULL,
|
|
25
|
+
parent_span_id TEXT,
|
|
26
|
+
trace_state TEXT,
|
|
27
|
+
service_name TEXT,
|
|
28
|
+
scope_name TEXT,
|
|
29
|
+
scope_version TEXT,
|
|
30
|
+
span_name TEXT NOT NULL,
|
|
31
|
+
span_kind INTEGER,
|
|
32
|
+
status_code INTEGER,
|
|
33
|
+
status_message TEXT,
|
|
34
|
+
start_time TEXT NOT NULL,
|
|
35
|
+
end_time TEXT NOT NULL,
|
|
36
|
+
duration_ms REAL NOT NULL,
|
|
37
|
+
attributes_json TEXT,
|
|
38
|
+
dropped_attributes_count INTEGER NOT NULL DEFAULT 0,
|
|
39
|
+
resource_attributes_json TEXT,
|
|
40
|
+
events_json TEXT,
|
|
41
|
+
dropped_events_count INTEGER NOT NULL DEFAULT 0,
|
|
42
|
+
links_json TEXT,
|
|
43
|
+
dropped_links_count INTEGER NOT NULL DEFAULT 0,
|
|
44
|
+
received_at TEXT NOT NULL,
|
|
45
|
+
expires_at TEXT NOT NULL,
|
|
46
|
+
session_id TEXT,
|
|
47
|
+
interaction_id TEXT,
|
|
48
|
+
telemetry_sdk_name TEXT,
|
|
49
|
+
PRIMARY KEY (project_id, trace_id, span_id)
|
|
50
|
+
);
|
|
51
|
+
|
|
52
|
+
INSERT INTO telemetry_spans_new (
|
|
53
|
+
project_id, trace_id, span_id, parent_span_id, trace_state, service_name,
|
|
54
|
+
scope_name, scope_version, span_name, span_kind, status_code, status_message,
|
|
55
|
+
start_time, end_time, duration_ms, attributes_json, dropped_attributes_count,
|
|
56
|
+
resource_attributes_json, events_json, dropped_events_count, links_json,
|
|
57
|
+
dropped_links_count, received_at, expires_at, session_id, interaction_id,
|
|
58
|
+
telemetry_sdk_name
|
|
59
|
+
)
|
|
60
|
+
SELECT
|
|
61
|
+
COALESCE(project_id, 'default'), trace_id, span_id, parent_span_id,
|
|
62
|
+
trace_state, service_name, scope_name, scope_version, span_name, span_kind,
|
|
63
|
+
status_code, status_message, start_time, end_time, duration_ms,
|
|
64
|
+
attributes_json, COALESCE(dropped_attributes_count, 0),
|
|
65
|
+
resource_attributes_json, events_json, COALESCE(dropped_events_count, 0),
|
|
66
|
+
links_json, COALESCE(dropped_links_count, 0), received_at, expires_at,
|
|
67
|
+
session_id, interaction_id, telemetry_sdk_name
|
|
68
|
+
FROM telemetry_spans;
|
|
69
|
+
|
|
70
|
+
DROP TABLE telemetry_spans;
|
|
71
|
+
|
|
72
|
+
ALTER TABLE telemetry_spans_new RENAME TO telemetry_spans;
|
|
73
|
+
|
|
74
|
+
CREATE INDEX IF NOT EXISTS idx_spans_received_at ON telemetry_spans (received_at DESC);
|
|
75
|
+
CREATE INDEX IF NOT EXISTS idx_spans_trace_id ON telemetry_spans (trace_id);
|
|
76
|
+
CREATE INDEX IF NOT EXISTS idx_spans_service ON telemetry_spans (service_name);
|
|
77
|
+
CREATE INDEX IF NOT EXISTS idx_spans_status ON telemetry_spans (status_code);
|
|
78
|
+
CREATE INDEX IF NOT EXISTS idx_spans_root ON telemetry_spans (parent_span_id, received_at DESC);
|
|
79
|
+
CREATE INDEX IF NOT EXISTS idx_spans_project_received ON telemetry_spans (project_id, received_at DESC);
|
|
80
|
+
CREATE INDEX IF NOT EXISTS idx_spans_with_links
|
|
81
|
+
ON telemetry_spans (project_id, received_at DESC)
|
|
82
|
+
WHERE links_json IS NOT NULL AND links_json != '[]';
|
|
83
|
+
CREATE INDEX IF NOT EXISTS idx_spans_project_service_received
|
|
84
|
+
ON telemetry_spans (project_id, service_name, received_at DESC);
|
|
85
|
+
CREATE INDEX IF NOT EXISTS idx_spans_session
|
|
86
|
+
ON telemetry_spans (project_id, session_id, received_at DESC);
|
|
87
|
+
CREATE INDEX IF NOT EXISTS idx_spans_interaction
|
|
88
|
+
ON telemetry_spans (project_id, interaction_id, received_at DESC)
|
|
89
|
+
WHERE interaction_id IS NOT NULL;
|
|
90
|
+
CREATE INDEX IF NOT EXISTS idx_spans_sdk_name
|
|
91
|
+
ON telemetry_spans (project_id, telemetry_sdk_name, received_at DESC)
|
|
92
|
+
WHERE telemetry_sdk_name IS NOT NULL;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
-- Migration 049: alert_state.notified
|
|
2
|
+
--
|
|
3
|
+
-- The evaluator records a state transition before it notifies the rule's
|
|
4
|
+
-- channels, and retries the notification on later ticks while it is still
|
|
5
|
+
-- outstanding. That "outstanding" bit used to be inferred from
|
|
6
|
+
-- alert_evaluations, which retention purges after 720 hours — so a state
|
|
7
|
+
-- older than the retention window looked undelivered again and was re-sent
|
|
8
|
+
-- every tick. The flag lives on the state row instead and is never purged.
|
|
9
|
+
--
|
|
10
|
+
-- Backfill: every existing row is marked delivered (DEFAULT 1). Before this
|
|
11
|
+
-- column existed the evaluator only wrote a transition after the channels
|
|
12
|
+
-- had accepted it, so no stored state can be awaiting a notification.
|
|
13
|
+
--
|
|
14
|
+
-- Re-running fails with "duplicate column name", which the D1 runner treats
|
|
15
|
+
-- as already applied.
|
|
16
|
+
|
|
17
|
+
ALTER TABLE alert_state ADD COLUMN notified INTEGER NOT NULL DEFAULT 1;
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Migration 049 — `alert_state.notified`, applied to the real SQLite/D1
|
|
3
|
+
* schema, plus the store's behaviour on a database that has not applied it.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { describe, expect, it } from "vitest";
|
|
7
|
+
import { AlertsStore } from "../lib/alerts-store";
|
|
8
|
+
import { ProjectsStore } from "../lib/projects-store";
|
|
9
|
+
import {
|
|
10
|
+
applySqliteMigrations,
|
|
11
|
+
createMigratedSqliteDb,
|
|
12
|
+
} from "../lib/test-utils/node-sqlite-db";
|
|
13
|
+
|
|
14
|
+
const ruleInput = {
|
|
15
|
+
name: "errors",
|
|
16
|
+
signal: "spans" as const,
|
|
17
|
+
query: {},
|
|
18
|
+
threshold: 1,
|
|
19
|
+
windowMins: 5,
|
|
20
|
+
comparison: ">" as const,
|
|
21
|
+
channels: [{ type: "webhook" as const, url: "https://hooks.example.com/a" }],
|
|
22
|
+
enabled: true,
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
describe("migration 049 (alert_state.notified)", () => {
|
|
26
|
+
it("adds the column and backfills existing state rows as delivered", async () => {
|
|
27
|
+
const db = createMigratedSqliteDb({ upTo: "048" });
|
|
28
|
+
const project = await new ProjectsStore(db).createProject({
|
|
29
|
+
name: "A",
|
|
30
|
+
slug: "a",
|
|
31
|
+
});
|
|
32
|
+
const store = new AlertsStore(db);
|
|
33
|
+
const rule = await store.createRule(project.id, ruleInput);
|
|
34
|
+
// The pre-049 evaluator only wrote a transition after delivery.
|
|
35
|
+
db.raw
|
|
36
|
+
.prepare(
|
|
37
|
+
`INSERT INTO alert_state (rule_id, project_id, current_state, last_state_change)
|
|
38
|
+
VALUES (?, ?, 'firing', '2026-05-31T00:00:00.000Z')`,
|
|
39
|
+
)
|
|
40
|
+
.run(rule.id, project.id);
|
|
41
|
+
const columnsBefore = db.raw
|
|
42
|
+
.prepare("PRAGMA table_info(alert_state)")
|
|
43
|
+
.all()
|
|
44
|
+
.map((c) => (c as { name: string }).name);
|
|
45
|
+
expect(columnsBefore).not.toContain("notified");
|
|
46
|
+
|
|
47
|
+
expect(applySqliteMigrations(db.raw, { from: "049" })).toEqual([
|
|
48
|
+
"049_alert_state_notified.sql",
|
|
49
|
+
]);
|
|
50
|
+
|
|
51
|
+
const notified = db.raw
|
|
52
|
+
.prepare("PRAGMA table_info(alert_state)")
|
|
53
|
+
.all()
|
|
54
|
+
.find((c) => (c as { name: string }).name === "notified") as {
|
|
55
|
+
type: string;
|
|
56
|
+
notnull: number;
|
|
57
|
+
dflt_value: string;
|
|
58
|
+
};
|
|
59
|
+
expect(notified).toMatchObject({
|
|
60
|
+
type: "INTEGER",
|
|
61
|
+
notnull: 1,
|
|
62
|
+
dflt_value: "1",
|
|
63
|
+
});
|
|
64
|
+
expect(await store.getState(rule.id, project.id)).toMatchObject({
|
|
65
|
+
current_state: "firing",
|
|
66
|
+
notified: 1,
|
|
67
|
+
});
|
|
68
|
+
// Other state is untouched and the new write path works afterwards.
|
|
69
|
+
await store.transitionState(
|
|
70
|
+
rule.id,
|
|
71
|
+
project.id,
|
|
72
|
+
"ok",
|
|
73
|
+
"2026-05-31T01:00:00.000Z",
|
|
74
|
+
);
|
|
75
|
+
expect((await store.getState(rule.id, project.id))?.notified).toBe(0);
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
it("keeps transitions working on a database that has not applied 049 yet", async () => {
|
|
79
|
+
const db = createMigratedSqliteDb({ upTo: "048" });
|
|
80
|
+
const project = await new ProjectsStore(db).createProject({
|
|
81
|
+
name: "A",
|
|
82
|
+
slug: "a",
|
|
83
|
+
});
|
|
84
|
+
const store = new AlertsStore(db);
|
|
85
|
+
const rule = await store.createRule(project.id, ruleInput);
|
|
86
|
+
const t1 = "2026-05-31T00:00:00.000Z";
|
|
87
|
+
|
|
88
|
+
// Falls back to the legacy statement instead of failing the tick.
|
|
89
|
+
await store.transitionState(rule.id, project.id, "firing", t1);
|
|
90
|
+
const row = await store.getState(rule.id, project.id);
|
|
91
|
+
expect(row).toMatchObject({
|
|
92
|
+
current_state: "firing",
|
|
93
|
+
last_state_change: t1,
|
|
94
|
+
});
|
|
95
|
+
expect(row?.notified).toBeUndefined();
|
|
96
|
+
// Nothing to acknowledge; the evaluator treats the row as delivered.
|
|
97
|
+
expect(await store.markNotified(rule.id, project.id, "firing", t1)).toBe(
|
|
98
|
+
false,
|
|
99
|
+
);
|
|
100
|
+
// Upserting over the existing row also works.
|
|
101
|
+
await store.transitionState(
|
|
102
|
+
rule.id,
|
|
103
|
+
project.id,
|
|
104
|
+
"ok",
|
|
105
|
+
"2026-05-31T01:00:00.000Z",
|
|
106
|
+
);
|
|
107
|
+
expect((await store.getState(rule.id, project.id))?.current_state).toBe(
|
|
108
|
+
"ok",
|
|
109
|
+
);
|
|
110
|
+
});
|
|
111
|
+
});
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Postgres migrations in `../migrations-postgres/` must mirror the
|
|
3
|
+
* SQLite/D1 files here one-to-one (see migrations-postgres/README.md
|
|
4
|
+
* "Numbering"). Both runners apply files by sorted name and record them in
|
|
5
|
+
* `schema_migrations`, so a number that exists on one side only would drift
|
|
6
|
+
* the two schemas apart silently.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { readdirSync, readFileSync } from "node:fs";
|
|
10
|
+
import path from "node:path";
|
|
11
|
+
import { fileURLToPath } from "node:url";
|
|
12
|
+
import { describe, expect, it } from "vitest";
|
|
13
|
+
|
|
14
|
+
const here = path.dirname(fileURLToPath(import.meta.url));
|
|
15
|
+
const sqliteDir = here;
|
|
16
|
+
const postgresDir = path.resolve(here, "../migrations-postgres");
|
|
17
|
+
|
|
18
|
+
const sqlFiles = (dir: string) =>
|
|
19
|
+
readdirSync(dir)
|
|
20
|
+
.filter((f) => f.endsWith(".sql"))
|
|
21
|
+
.sort();
|
|
22
|
+
|
|
23
|
+
/** Tables the retention purge deletes from by comparing `expires_at` to text. */
|
|
24
|
+
const RETENTION_TABLES = [
|
|
25
|
+
"telemetry_spans",
|
|
26
|
+
"logs",
|
|
27
|
+
"usage_events",
|
|
28
|
+
"ai_calls",
|
|
29
|
+
"metric_point",
|
|
30
|
+
"metric_exemplars",
|
|
31
|
+
];
|
|
32
|
+
|
|
33
|
+
describe("migration parity", () => {
|
|
34
|
+
it("has the same numbered files on SQLite/D1 and Postgres", () => {
|
|
35
|
+
const sqlite = sqlFiles(sqliteDir);
|
|
36
|
+
const postgres = sqlFiles(postgresDir);
|
|
37
|
+
expect(postgres).toEqual(sqlite);
|
|
38
|
+
const prefixes = sqlite.map((f) => f.slice(0, 3));
|
|
39
|
+
expect(new Set(prefixes).size).toBe(prefixes.length);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
it("declares retention timestamps as TEXT in every Postgres CREATE TABLE", () => {
|
|
43
|
+
// postgresDialect.now() renders ISO text; Postgres refuses
|
|
44
|
+
// `timestamptz < text`, so any retention table created with
|
|
45
|
+
// TIMESTAMPTZ breaks purgeExpired (the 035 metric_exemplars bug).
|
|
46
|
+
const offenders: string[] = [];
|
|
47
|
+
for (const file of sqlFiles(postgresDir)) {
|
|
48
|
+
const sql = readFileSync(path.join(postgresDir, file), "utf8");
|
|
49
|
+
const tables = sql.matchAll(
|
|
50
|
+
/CREATE TABLE(?: IF NOT EXISTS)?\s+(\w+)\s*\(([^;]*?)\);/gis,
|
|
51
|
+
);
|
|
52
|
+
for (const [, table, body] of tables) {
|
|
53
|
+
if (!RETENTION_TABLES.includes(table)) continue;
|
|
54
|
+
for (const column of ["received_at", "expires_at"]) {
|
|
55
|
+
const decl = new RegExp(`^\\s*${column}\\s+(\\w+)`, "im").exec(body);
|
|
56
|
+
if (decl && decl[1].toUpperCase() !== "TEXT") {
|
|
57
|
+
offenders.push(`${file}: ${table}.${column} ${decl[1]}`);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
expect(offenders).toEqual([]);
|
|
63
|
+
});
|
|
64
|
+
});
|