@catalyst-cloud/schema 0.1.39 → 0.1.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/events/current-state.ts +31 -0
- package/src/events/index.ts +4 -0
- package/src/events/registry.ts +14 -2
- package/src/index.ts +21 -0
- package/src/migrations.generated.ts +18 -0
- package/src/mirror.ts +114 -0
- package/src/telemetry/phase-lifecycle-point.ts +2 -29
- package/src/telemetry/point.ts +11 -134
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@catalyst-cloud/schema",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.41",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Typed Drizzle schema = single source of truth for the per-tenant Mirror DO SQLite store (CTC-13 / ADR-0002). Shared by the mirror Worker, the host-sync replica, and the browser OPFS replica.",
|
|
6
6
|
"license": "MIT",
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// current-state.ts — CTC-355. The THIRD admission gate, and the reason it did not exist.
|
|
2
|
+
//
|
|
3
|
+
// `classifyEventType` sorts every declared name into exactly one of three disjoint buckets, and until
|
|
4
|
+
// now only two of them had a door: `appendEventWithOutbox` (apps/mirror/src/do/event-backbone.ts)
|
|
5
|
+
// admits `durable`, `assertEmittable` (./telemetry/point.ts) admits `telemetry`, and `current-state`
|
|
6
|
+
// was admitted by NOTHING — declared, unproduced, unconsumed, the "declared but unfed lane" shape
|
|
7
|
+
// ADR-0029 names. This is that door: the fleet-liveness heartbeat ingest lane
|
|
8
|
+
// (apps/mirror/src/coordination/heartbeat-validate.ts) calls this to refuse anything that is not a
|
|
9
|
+
// heartbeat-shaped current-state name, symmetrically with how the durable backbone and the telemetry
|
|
10
|
+
// sink already refuse everything that is NOT theirs.
|
|
11
|
+
//
|
|
12
|
+
// Three disjoint buckets, three gates, one per bucket — the symmetry is the point. The other two gates
|
|
13
|
+
// stay exactly as they are: a heartbeat is still refused by the durable backbone and by the telemetry
|
|
14
|
+
// sink, which is correct — updating mutable NOW is this gate's job alone.
|
|
15
|
+
|
|
16
|
+
import { classifyEventType } from "./registry.js";
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Refuse to admit a name that is not `classifyEventType(name) === "current-state"`. Throws — never
|
|
20
|
+
* returns a falsy sentinel (AGENTS.md: a helper that degrades to "fine" hides the defect, and a caller
|
|
21
|
+
* cannot degrade a refusal into a silent skip when the refusal is an exception).
|
|
22
|
+
*/
|
|
23
|
+
export function assertCurrentState(name: string): void {
|
|
24
|
+
const klass = classifyEventType(name);
|
|
25
|
+
if (klass !== "current-state") {
|
|
26
|
+
throw new Error(
|
|
27
|
+
`event type ${JSON.stringify(name)} is classified "${klass}" — only current-state (heartbeat) ` +
|
|
28
|
+
`names may update mutable NOW. See packages/schema/src/events/registry.ts.`,
|
|
29
|
+
);
|
|
30
|
+
}
|
|
31
|
+
}
|
package/src/events/index.ts
CHANGED
|
@@ -6,3 +6,7 @@ export * from "./types.js";
|
|
|
6
6
|
export * from "./type-grammar.js";
|
|
7
7
|
export * from "./registry.js";
|
|
8
8
|
export * from "./retention.js";
|
|
9
|
+
// CTC-355 — the THIRD admission gate: durable/current-state/telemetry are the three disjoint buckets
|
|
10
|
+
// classifyEventType partitions the vocabulary into; the other two already had a door
|
|
11
|
+
// (appendEventWithOutbox, ../telemetry/point.ts's assertEmittable). This is current-state's.
|
|
12
|
+
export * from "./current-state.js";
|
package/src/events/registry.ts
CHANGED
|
@@ -185,12 +185,14 @@ export const durableEventTypes = {
|
|
|
185
185
|
wakes: { consumerClass: "dispatcher" },
|
|
186
186
|
},
|
|
187
187
|
"fleet.health.degraded": {
|
|
188
|
-
rationale:
|
|
188
|
+
rationale:
|
|
189
|
+
"A fleet-wide health boundary crossing — paired with fleet.health.recovered. ⭐ CTC-355 is its producer: the per-tenant absence-based liveness evaluator (apps/mirror/src/do/fleet-liveness-evaluate.ts's evaluateLiveness, invoked from MirrorDO.alarm()) derives it from monitor.heartbeat ABSENCE past a per-host grace window — exactly the derivation currentStateEventTypes' boundaryEvents list above was typed against. ⚠️ `ingestVerb` below is unchanged and now under-describes this name: the producer is an in-DO evaluator, not an ingest verb. The field is descriptive metadata with no runtime consumer (verified: nothing outside this registry file reads it) — widening its closed union for one unread field would touch ~90 spec literals.",
|
|
189
190
|
ingestVerb: "coordination.publish",
|
|
190
191
|
wakes: { consumerClass: "host-channel" },
|
|
191
192
|
},
|
|
192
193
|
"fleet.health.recovered": {
|
|
193
|
-
rationale:
|
|
194
|
+
rationale:
|
|
195
|
+
"The paired boundary-crossing fact to fleet.health.degraded. Same producer (CTC-355's liveness evaluator) — fired when a degraded host's heartbeat resumes inside a later evaluation pass.",
|
|
194
196
|
ingestVerb: "coordination.publish",
|
|
195
197
|
wakes: false,
|
|
196
198
|
},
|
|
@@ -1910,6 +1912,16 @@ export const telemetryEventTypes = {
|
|
|
1910
1912
|
"account.ratelimit.sampled": {
|
|
1911
1913
|
reason: "A periodic provider rate-limit budget reading — a gauge, not a state transition.",
|
|
1912
1914
|
},
|
|
1915
|
+
"catalyst.fleet.liveness.evaluated": {
|
|
1916
|
+
reason:
|
|
1917
|
+
"CTC-355 — one point per per-tenant liveness evaluation pass, carrying the live/degraded/unknown counts. Gauge-shaped: the rollup is the signal, no individual pass has identity. The BOUNDARY crossings are durable (fleet.health.degraded/recovered below); this is the pass itself.",
|
|
1918
|
+
sampling: { kind: "counted" },
|
|
1919
|
+
},
|
|
1920
|
+
"catalyst.fleet.anomaly.detected": {
|
|
1921
|
+
reason:
|
|
1922
|
+
"CTC-355 — one point per NEWLY-OPENED fleet anomaly (rate_spike/flatline), never per re-detection of an already-open one. The anomaly ROW (fleet_anomalies) is the durable record of what is wrong now; this is the detection event.",
|
|
1923
|
+
sampling: { kind: "counted" },
|
|
1924
|
+
},
|
|
1913
1925
|
"account.ratelimit.unsampled": {
|
|
1914
1926
|
reason:
|
|
1915
1927
|
"The raw (pre-sampling) rate-limit reading behind account.ratelimit.sampled — same gauge family.",
|
package/src/index.ts
CHANGED
|
@@ -50,6 +50,10 @@ import {
|
|
|
50
50
|
deployment_statuses,
|
|
51
51
|
pr_review_threads,
|
|
52
52
|
check_suites,
|
|
53
|
+
fleet_host_liveness,
|
|
54
|
+
fleet_anomalies,
|
|
55
|
+
fleet_activity_rate,
|
|
56
|
+
fleet_health_sends,
|
|
53
57
|
} from "./mirror.js";
|
|
54
58
|
|
|
55
59
|
export * from "./mirror.js";
|
|
@@ -152,6 +156,17 @@ export const mirrorSchema = {
|
|
|
152
156
|
// CTC-667 item 4: the check SUITE's own rollup — the highest-volume signal in CTL's census and the
|
|
153
157
|
// one every phase agent's CI wait blocks on.
|
|
154
158
|
check_suites,
|
|
159
|
+
// CTC-355: per-host fleet liveness (mutable NOW) + open anomaly signals. Real tables the DO creates
|
|
160
|
+
// (migrations must produce them) and change-feed entities (see feedEntityTables below) — the
|
|
161
|
+
// dashboard is a live consumer of both.
|
|
162
|
+
fleet_host_liveness,
|
|
163
|
+
fleet_anomalies,
|
|
164
|
+
// CTC-355: the rolling activity-rate bucket table (anomaly baseline source) and the fleet-health
|
|
165
|
+
// push re-fire gate. Same posture as fleet_activity_pruned / pull_sweeps — real tables the DO
|
|
166
|
+
// creates, deliberately absent from feedEntityTables/nonFeedEntityTables (pure bookkeeping no
|
|
167
|
+
// consumer should ever see over the feed).
|
|
168
|
+
fleet_activity_rate,
|
|
169
|
+
fleet_health_sends,
|
|
155
170
|
} as const;
|
|
156
171
|
|
|
157
172
|
/**
|
|
@@ -225,6 +240,12 @@ export const feedEntityTables = {
|
|
|
225
240
|
// CTC-667 item 4: GitHub's OWN check-suite rollup. ⛔ The row is contractual; a derivation from
|
|
226
241
|
// `check_runs` is NOT — see the table's own header for the two reasons it cannot agree with GitHub.
|
|
227
242
|
check_suites,
|
|
243
|
+
// CTC-355: per-host fleet liveness + open anomaly signals. On this plane deliberately — the
|
|
244
|
+
// dashboard's fleet-liveness panel is a legitimate account-wide reader, same posture as
|
|
245
|
+
// fleet_activity directly above (both are BROWSER_ONLY at the changefeed layer — see
|
|
246
|
+
// apps/mirror/src/do/changefeed.ts's BROWSER_ONLY_ENTITIES).
|
|
247
|
+
fleet_host_liveness,
|
|
248
|
+
fleet_anomalies,
|
|
228
249
|
} as const;
|
|
229
250
|
|
|
230
251
|
/**
|
|
@@ -256,6 +256,20 @@ export const MIRROR_MIGRATIONS = {
|
|
|
256
256
|
tag: "0034_special_onslaught",
|
|
257
257
|
breakpoints: true,
|
|
258
258
|
},
|
|
259
|
+
{
|
|
260
|
+
idx: 35,
|
|
261
|
+
version: "6",
|
|
262
|
+
when: 1788324536096,
|
|
263
|
+
tag: "0035_tricky_swarm",
|
|
264
|
+
breakpoints: true,
|
|
265
|
+
},
|
|
266
|
+
{
|
|
267
|
+
idx: 36,
|
|
268
|
+
version: "6",
|
|
269
|
+
when: 1788333712249,
|
|
270
|
+
tag: "0036_watery_hercules",
|
|
271
|
+
breakpoints: true,
|
|
272
|
+
},
|
|
259
273
|
],
|
|
260
274
|
},
|
|
261
275
|
migrations: {
|
|
@@ -326,5 +340,9 @@ export const MIRROR_MIGRATIONS = {
|
|
|
326
340
|
"ALTER TABLE `labels` ADD `team_id` text;--> statement-breakpoint\nCREATE INDEX `idx_labels_team` ON `labels` (`team_id`);",
|
|
327
341
|
"0034_special_onslaught":
|
|
328
342
|
"CREATE INDEX `idx_processed_events_source_received` ON `processed_events` (`source`,`received_at`);",
|
|
343
|
+
"0035_tricky_swarm":
|
|
344
|
+
"ALTER TABLE `agent_activities` ADD `action` text;--> statement-breakpoint\nALTER TABLE `agent_activities` ADD `parameter` text;--> statement-breakpoint\nALTER TABLE `agent_activities` ADD `result` text;--> statement-breakpoint\nUPDATE agent_activities\n SET action = json_extract(raw, '$.content.action'),\n parameter = json_extract(raw, '$.content.parameter'),\n result = json_extract(raw, '$.content.result')\n WHERE raw IS NOT NULL\n AND json_valid(raw)\n AND json_extract(raw, '$.content.type') = 'action';\n",
|
|
345
|
+
"0036_watery_hercules":
|
|
346
|
+
"CREATE TABLE `fleet_activity_rate` (\n\t`host_id` text NOT NULL,\n\t`ticket` text NOT NULL,\n\t`bucket_start_ms` integer NOT NULL,\n\t`events` integer NOT NULL,\n\tPRIMARY KEY(`host_id`, `ticket`, `bucket_start_ms`)\n);\n--> statement-breakpoint\nCREATE TABLE `fleet_anomalies` (\n\t`kind` text NOT NULL,\n\t`host_id` text NOT NULL,\n\t`ticket` text NOT NULL,\n\t`detected_at_ms` integer NOT NULL,\n\t`observed` integer NOT NULL,\n\t`baseline` integer NOT NULL,\n\t`detail` text,\n\t`updated_at` integer NOT NULL,\n\tPRIMARY KEY(`kind`, `host_id`, `ticket`)\n);\n--> statement-breakpoint\nCREATE TABLE `fleet_health_sends` (\n\t`host_id` text NOT NULL,\n\t`status` text NOT NULL,\n\t`status_changed_at_ms` integer NOT NULL,\n\t`sent_at_ms` integer NOT NULL,\n\t`delivered` integer NOT NULL,\n\t`failed` integer NOT NULL,\n\tPRIMARY KEY(`host_id`, `status`, `status_changed_at_ms`)\n);\n--> statement-breakpoint\nCREATE TABLE `fleet_host_liveness` (\n\t`host_id` text PRIMARY KEY NOT NULL,\n\t`status` text NOT NULL,\n\t`reason` text,\n\t`last_heartbeat_ms` integer,\n\t`last_activity_ms` integer,\n\t`last_seen_ms` integer,\n\t`period_ms` integer,\n\t`grace_ms` integer,\n\t`agent_version` text,\n\t`status_changed_at_ms` integer,\n\t`updated_at` integer NOT NULL\n);\n",
|
|
329
347
|
},
|
|
330
348
|
} as const;
|
package/src/mirror.ts
CHANGED
|
@@ -295,6 +295,14 @@ export const agent_activities = sqliteTable(
|
|
|
295
295
|
signal: text("signal"),
|
|
296
296
|
created_at: integer("created_at"),
|
|
297
297
|
updated_at: integer("updated_at"),
|
|
298
|
+
/** CTC-1089 — Linear's ActionContent fields, promoted out of `raw`. An action activity carries
|
|
299
|
+
* NO `body` (write-proxy/agent-activity.ts's AgentActivityContent), so before these columns the
|
|
300
|
+
* relay's phase-start / gate / artifact / pr-opened narration reached the client as the bare word
|
|
301
|
+
* "action" with nothing after it. Same call CTC-529 made for archived_at/dismissed_at: a render
|
|
302
|
+
* that branches on a field must not json_extract it out of an opaque blob. */
|
|
303
|
+
action: text("action"),
|
|
304
|
+
parameter: text("parameter"),
|
|
305
|
+
result: text("result"),
|
|
298
306
|
raw: text("raw"),
|
|
299
307
|
},
|
|
300
308
|
(t) => [index("idx_agent_activities_session_created").on(t.session_id, t.created_at)],
|
|
@@ -1358,3 +1366,109 @@ export const push_subscriptions = sqliteTable(
|
|
|
1358
1366
|
// PK), keyed off issues.assignee_id — the same shape idx_workflow_states_team indexes on team_id.
|
|
1359
1367
|
(t) => [index("idx_push_subscriptions_member").on(t.member_id)],
|
|
1360
1368
|
);
|
|
1369
|
+
|
|
1370
|
+
/**
|
|
1371
|
+
* CTC-355 — MUTABLE NOW for one fleet host's liveness. One row per host_id.
|
|
1372
|
+
*
|
|
1373
|
+
* ⛔ THIS IS NOT `fleet_activity`, AND THE DIFFERENCE IS THE WHOLE POINT. `fleet_activity` advances
|
|
1374
|
+
* only when a host does TICKET WORK, so an alive-but-idle host freezes its watermark — the exact trap
|
|
1375
|
+
* `apps/web/src/lib/sync-health.ts` documents for `last_ingest_ms` (CTC-144 added a heartbeat
|
|
1376
|
+
* precisely because of it). Liveness is asserted by a HEARTBEAT and by nothing else.
|
|
1377
|
+
*
|
|
1378
|
+
* ⚠️ `last_activity_ms` folds in ONE-WAY. Publishing a coordination event is positive proof of life,
|
|
1379
|
+
* so it advances `last_seen_ms`; its ABSENCE proves nothing and never drives a verdict.
|
|
1380
|
+
*
|
|
1381
|
+
* ⭐ `status` is TRI-STATE and `unknown` is load-bearing. A host that has never heartbeated is
|
|
1382
|
+
* `unknown` regardless of how much activity it has produced — because on the day this shipped NO host
|
|
1383
|
+
* emitted `monitor.heartbeat`, and a binary model would have marked the entire fleet degraded and
|
|
1384
|
+
* pushed a notification for every one of them.
|
|
1385
|
+
*/
|
|
1386
|
+
export const fleet_host_liveness = sqliteTable("fleet_host_liveness", {
|
|
1387
|
+
host_id: text("host_id").primaryKey(),
|
|
1388
|
+
/** 'live' | 'degraded' | 'unknown' — see FleetLivenessStatus (do/fleet-liveness-evaluate.ts). */
|
|
1389
|
+
status: text("status").notNull(),
|
|
1390
|
+
/** Why, when status is 'unknown': 'no_heartbeat_lane' | 'clock_skew'. Null otherwise. */
|
|
1391
|
+
reason: text("reason"),
|
|
1392
|
+
/** Last accepted heartbeat receipt (the DO's OWN clock). NULL ⇒ never heartbeated ⇒ unknown. */
|
|
1393
|
+
last_heartbeat_ms: integer("last_heartbeat_ms"),
|
|
1394
|
+
/** Last accepted coordination-event receipt for this host (DO clock). Positive proof only. */
|
|
1395
|
+
last_activity_ms: integer("last_activity_ms"),
|
|
1396
|
+
/** max(last_heartbeat_ms, last_activity_ms) — the value the grace window is measured against. */
|
|
1397
|
+
last_seen_ms: integer("last_seen_ms"),
|
|
1398
|
+
/** The cadence the host DECLARED, clamped on ingest. Drives the per-host grace window. */
|
|
1399
|
+
period_ms: integer("period_ms"),
|
|
1400
|
+
/** The window in force for this host, materialized so the UI reports the SAME number the evaluator
|
|
1401
|
+
* used rather than re-deriving one that could disagree. */
|
|
1402
|
+
grace_ms: integer("grace_ms"),
|
|
1403
|
+
/** The host's self-reported agent/SDK version, verbatim. Display only. */
|
|
1404
|
+
agent_version: text("agent_version"),
|
|
1405
|
+
/** When `status` last CHANGED — not when the row was last touched. The two differ on every
|
|
1406
|
+
* heartbeat of a healthy host, and the UI needs the former ("degraded for 20 minutes"). */
|
|
1407
|
+
status_changed_at_ms: integer("status_changed_at_ms"),
|
|
1408
|
+
updated_at: integer("updated_at").notNull(), // upsertRow's last-write-wins guard column
|
|
1409
|
+
});
|
|
1410
|
+
|
|
1411
|
+
/**
|
|
1412
|
+
* CTC-355 — an open anomaly signal. One row per (kind, host_id, ticket); DELETED when the condition
|
|
1413
|
+
* clears, so this table is "what is wrong now", never a history.
|
|
1414
|
+
*
|
|
1415
|
+
* ⚠️ TICKET IS `''` FOR TENANT-SCOPE KINDS (flatline), never NULL — it is a PK column, and SQLite
|
|
1416
|
+
* treats NULLs as distinct in a UNIQUE index, so a nullable PK member would let the same flatline
|
|
1417
|
+
* insert unboundedly many times.
|
|
1418
|
+
*/
|
|
1419
|
+
export const fleet_anomalies = sqliteTable(
|
|
1420
|
+
"fleet_anomalies",
|
|
1421
|
+
{
|
|
1422
|
+
kind: text("kind").notNull(), // 'rate_spike' | 'flatline'
|
|
1423
|
+
host_id: text("host_id").notNull(), // '' for tenant-scope kinds
|
|
1424
|
+
ticket: text("ticket").notNull(), // '' for tenant-scope kinds
|
|
1425
|
+
detected_at_ms: integer("detected_at_ms").notNull(),
|
|
1426
|
+
/** The measured value and the baseline it was judged against — so the UI can SHOW the arithmetic
|
|
1427
|
+
* instead of asserting a verdict the reader has to trust. */
|
|
1428
|
+
observed: integer("observed").notNull(),
|
|
1429
|
+
baseline: integer("baseline").notNull(),
|
|
1430
|
+
detail: text("detail"),
|
|
1431
|
+
updated_at: integer("updated_at").notNull(),
|
|
1432
|
+
},
|
|
1433
|
+
(t) => [primaryKey({ columns: [t.kind, t.host_id, t.ticket] })],
|
|
1434
|
+
);
|
|
1435
|
+
|
|
1436
|
+
/**
|
|
1437
|
+
* CTC-355 — bounded rolling activity counters, the ONLY baseline source for anomaly detection. 15-min
|
|
1438
|
+
* buckets, 24h retention: ≤96 rows per active (host, ticket), and a pair stops producing rows the
|
|
1439
|
+
* moment it stops publishing. Bookkeeping — on NEITHER entity list, like fleet_activity_pruned.
|
|
1440
|
+
*
|
|
1441
|
+
* ⛔ NOT Analytics Engine. AE applies query-time sampling, and a sampled-away point is
|
|
1442
|
+
* indistinguishable from an absent one — which for an absence-based detector is a FALSE ALARM, not a
|
|
1443
|
+
* rounding error. This table is how it is dodged rather than compensated for.
|
|
1444
|
+
*/
|
|
1445
|
+
export const fleet_activity_rate = sqliteTable(
|
|
1446
|
+
"fleet_activity_rate",
|
|
1447
|
+
{
|
|
1448
|
+
host_id: text("host_id").notNull(),
|
|
1449
|
+
ticket: text("ticket").notNull(),
|
|
1450
|
+
bucket_start_ms: integer("bucket_start_ms").notNull(),
|
|
1451
|
+
events: integer("events").notNull(),
|
|
1452
|
+
},
|
|
1453
|
+
(t) => [primaryKey({ columns: [t.host_id, t.ticket, t.bucket_start_ms] })],
|
|
1454
|
+
);
|
|
1455
|
+
|
|
1456
|
+
/**
|
|
1457
|
+
* CTC-355 — the durable "has anybody been told?" record for fleet-health notifications, modelled on
|
|
1458
|
+
* the CTC-764 ask-push re-fire gate. Keyed by the transition, so a re-evaluation that re-derives the
|
|
1459
|
+
* same verdict cannot re-notify, and a genuine flap (degraded → live → degraded) can.
|
|
1460
|
+
*/
|
|
1461
|
+
export const fleet_health_sends = sqliteTable(
|
|
1462
|
+
"fleet_health_sends",
|
|
1463
|
+
{
|
|
1464
|
+
host_id: text("host_id").notNull(),
|
|
1465
|
+
/** The status transitioned INTO. */
|
|
1466
|
+
status: text("status").notNull(),
|
|
1467
|
+
/** The status_changed_at_ms this send was for — what makes a flap re-notifiable. */
|
|
1468
|
+
status_changed_at_ms: integer("status_changed_at_ms").notNull(),
|
|
1469
|
+
sent_at_ms: integer("sent_at_ms").notNull(),
|
|
1470
|
+
delivered: integer("delivered").notNull(), // count of endpoints that accepted
|
|
1471
|
+
failed: integer("failed").notNull(),
|
|
1472
|
+
},
|
|
1473
|
+
(t) => [primaryKey({ columns: [t.host_id, t.status, t.status_changed_at_ms] })],
|
|
1474
|
+
);
|
|
@@ -37,12 +37,6 @@ export interface PhaseLifecycleFields {
|
|
|
37
37
|
phase: string;
|
|
38
38
|
nonce: number;
|
|
39
39
|
executor: string;
|
|
40
|
-
/** CTC-1417 — `owner/name`, when the caller's dispatch data names one. Absent, never guessed: a
|
|
41
|
-
* refusal that never resolved to a repo must not claim one. */
|
|
42
|
-
repo?: string | null;
|
|
43
|
-
/** CTC-1417 — the Linear team key the work unit belongs to, when the caller knows it. */
|
|
44
|
-
team?: string | null;
|
|
45
|
-
projectId?: string | null;
|
|
46
40
|
substrate?: string;
|
|
47
41
|
provider?: string;
|
|
48
42
|
branch?: string;
|
|
@@ -53,30 +47,16 @@ export interface PhaseLifecycleFields {
|
|
|
53
47
|
reason?: string;
|
|
54
48
|
}
|
|
55
49
|
|
|
56
|
-
/** A ticket identifier's team key IS its prefix — `CTC-1417` belongs to team `CTC`. That is not a
|
|
57
|
-
* guess: it is how Linear constructs the identifier and how this repo's own dispatch queue
|
|
58
|
-
* partitions (`listPublishedTeams`, `explainEligibleWork`). Restated here (rather than imported from
|
|
59
|
-
* ci-run-telemetry.ts's `TICKET_SHAPE`, which tests a whole different input) so the derivation is
|
|
60
|
-
* ANCHORED: a value that is not identifier-shaped yields null and the dimension stays absent, rather
|
|
61
|
-
* than a leading substring of something arbitrary being reported as a team. */
|
|
62
|
-
const TICKET_IDENTIFIER = /^([A-Z][A-Z0-9]{1,9})-\d+$/;
|
|
63
|
-
|
|
64
|
-
export function teamFromTicket(ticket: string): string | null {
|
|
65
|
-
return TICKET_IDENTIFIER.exec(ticket)?.[1] ?? null;
|
|
66
|
-
}
|
|
67
|
-
|
|
68
50
|
/**
|
|
69
51
|
* Build one `phase.lifecycle` TelemetryPoint. Throws (never a falsy sentinel) on an unrecognized
|
|
70
52
|
* `status` — the same posture as `assertEmittable` for the event name.
|
|
71
53
|
*/
|
|
72
54
|
export function buildPhaseLifecyclePoint(fields: PhaseLifecycleFields): TelemetryPoint {
|
|
73
55
|
assertKnownStatus(fields.status);
|
|
74
|
-
// CTC-1417 — `ticket` and `phase` are TYPED COLUMNS now (see the point's own dimensions), so they
|
|
75
|
-
// are no longer repeated here as free text. Nothing is lost: `traceId` still spells
|
|
76
|
-
// `ticket:phase:nonce`, and the columns are what an AE query can actually filter and group on.
|
|
77
|
-
// What stays in `detail` is exactly what has no column of its own.
|
|
78
56
|
const detailParts = [
|
|
79
57
|
`status=${fields.status}`,
|
|
58
|
+
`ticket=${fields.ticket}`,
|
|
59
|
+
`phase=${fields.phase}`,
|
|
80
60
|
`nonce=${fields.nonce}`,
|
|
81
61
|
`executor=${fields.executor}`,
|
|
82
62
|
...(fields.substrate !== undefined ? [`substrate=${fields.substrate}`] : []),
|
|
@@ -95,12 +75,5 @@ export function buildPhaseLifecyclePoint(fields: PhaseLifecycleFields): Telemetr
|
|
|
95
75
|
traceId: `${fields.ticket}:${fields.phase}:${fields.nonce}`,
|
|
96
76
|
durationMs: fields.durationMs ?? null,
|
|
97
77
|
count: 1,
|
|
98
|
-
ticket: fields.ticket,
|
|
99
|
-
phase: fields.phase,
|
|
100
|
-
repo: fields.repo ?? null,
|
|
101
|
-
// An explicit `team` from the caller wins; otherwise derive it from the ticket, which every one
|
|
102
|
-
// of these call sites already has. Null when the ticket is not identifier-shaped.
|
|
103
|
-
team: fields.team ?? teamFromTicket(fields.ticket),
|
|
104
|
-
projectId: fields.projectId ?? null,
|
|
105
78
|
};
|
|
106
79
|
}
|
package/src/telemetry/point.ts
CHANGED
|
@@ -18,10 +18,6 @@ export const MAX_POINTS_PER_BATCH = 25;
|
|
|
18
18
|
export const MAX_INDEX_BYTES = 96;
|
|
19
19
|
/** ClickHouse/AE per-blob byte cap. The bounded excerpt (`detail`) is budgeted against this. */
|
|
20
20
|
export const MAX_EXCERPT_BYTES = 1_024;
|
|
21
|
-
/** Per-DIMENSION blob cap (CTC-1417). A dimension is an identifier — an `owner/name`, a team key, a
|
|
22
|
-
* `CTC-NNN`, a phase name — never prose, so it is budgeted far below `MAX_EXCERPT_BYTES`. The cap is
|
|
23
|
-
* a guard against a malformed caller, not an expected truncation point. */
|
|
24
|
-
export const MAX_DIMENSION_BYTES = 256;
|
|
25
21
|
|
|
26
22
|
function byteLen(s: string): number {
|
|
27
23
|
return new TextEncoder().encode(s).length;
|
|
@@ -36,18 +32,12 @@ function byteLen(s: string): number {
|
|
|
36
32
|
* of carrying its own — one truncator, not two that could drift.
|
|
37
33
|
*/
|
|
38
34
|
export function boundExcerpt(raw: string): string {
|
|
39
|
-
return boundToBytes(raw, MAX_EXCERPT_BYTES);
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
/** The byte-budgeted, UTF-8-boundary-safe truncator {@link boundExcerpt} and the CTC-1417 dimension
|
|
43
|
-
* bounder both use — one implementation so the two budgets can never drift into two truncators. */
|
|
44
|
-
function boundToBytes(raw: string, cap: number): string {
|
|
45
35
|
const bytes = byteLen(raw);
|
|
46
|
-
if (bytes <=
|
|
36
|
+
if (bytes <= MAX_EXCERPT_BYTES) return raw;
|
|
47
37
|
// Reserve marker room using the WORST-CASE omitted count (the whole payload); the real marker is
|
|
48
|
-
// never longer, so the final excerpt is guaranteed ≤
|
|
38
|
+
// never longer, so the final excerpt is guaranteed ≤ MAX_EXCERPT_BYTES.
|
|
49
39
|
const markerReserve = byteLen(`…[+${bytes} bytes]`);
|
|
50
|
-
const keepBudget = Math.max(0,
|
|
40
|
+
const keepBudget = Math.max(0, MAX_EXCERPT_BYTES - markerReserve);
|
|
51
41
|
const kept = new TextEncoder().encode(raw).slice(0, keepBudget);
|
|
52
42
|
// Lenient decode → a split trailing multi-byte becomes U+FFFD; strip it so output is valid + smaller.
|
|
53
43
|
let decoded = new TextDecoder().decode(kept);
|
|
@@ -68,82 +58,11 @@ export function resolveSampling(name: string): SamplingClass {
|
|
|
68
58
|
return spec?.sampling ?? DEFAULT_SAMPLING;
|
|
69
59
|
}
|
|
70
60
|
|
|
71
|
-
/**
|
|
72
|
-
* The uniform tenant dimensions every telemetry point carries (CTC-1417), on top of the `account_id`
|
|
73
|
-
* that is already the AE index. These are the keys an operator segments by — "which tenant, which
|
|
74
|
-
* repo, which team, which unit of work" — so they are TYPED COLUMNS, not text packed into `detail`:
|
|
75
|
-
* Analytics Engine cannot filter or GROUP BY on a substring of a free-text blob, which is exactly why
|
|
76
|
-
* `phase.lifecycle`/`ci.run.completed`/`work.backlog.sampled` were unqueryable by ticket or team.
|
|
77
|
-
*
|
|
78
|
-
* Every field is optional and every ABSENT field means "not applicable to this event", never a
|
|
79
|
-
* default. A platform-level event (fleet-wide polling, a webhook that never resolved to a tenant work
|
|
80
|
-
* unit) leaves them unset rather than inventing a repo — a fabricated dimension is worse than a
|
|
81
|
-
* missing one, because it is indistinguishable from a real measurement downstream.
|
|
82
|
-
*
|
|
83
|
-
* ⚠️ `ticket` and `phase` are deliberately in this set even though they are HIGH-cardinality relative
|
|
84
|
-
* to `repo`/`team`. That is safe HERE and only here: AE columns and Iceberg columns are not label
|
|
85
|
-
* sets, so a per-ticket value costs a column value, not a new time series. The same two identifiers
|
|
86
|
-
* must stay BODY-level on the OTLP path (Loki promotes resource attributes to labels — see
|
|
87
|
-
* docs/observability.md), which is why THEY live on this sink envelope and not in
|
|
88
|
-
* `@catalyst-cloud/observability`'s `Dimensions` (ADR-0011), which is the label-safe vocabulary.
|
|
89
|
-
* `repo`, `team` and `project_id` belong to BOTH — they are low-cardinality, so they are canonical
|
|
90
|
-
* dimensions over there (`team` added by CTC-1417, once OTL-96 made the collector index it) and
|
|
91
|
-
* columns here. `ticket` and `phase` are the two that are columns here ONLY.
|
|
92
|
-
*/
|
|
93
|
-
export interface TelemetryDimensions {
|
|
94
|
-
/** `owner/name` — the same spelling `normalize/github.ts` derives from `repository.full_name`. */
|
|
95
|
-
readonly repo?: string | null;
|
|
96
|
-
/** Linear team key (e.g. `CTC`), the dispatch-queue partition. */
|
|
97
|
-
readonly team?: string | null;
|
|
98
|
-
readonly projectId?: string | null;
|
|
99
|
-
/** Ticket identifier (e.g. `CTC-1417`) — set only where a unit of work exists. */
|
|
100
|
-
readonly ticket?: string | null;
|
|
101
|
-
/** Relay phase (e.g. `implement`) — set only where a unit of work exists. */
|
|
102
|
-
readonly phase?: string | null;
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
/**
|
|
106
|
-
* The AE hot leg's positional blob layout, declared once (CTC-1417). A positional wire format has no
|
|
107
|
-
* names on it: inserting or reordering a column silently rewrites the meaning of every row already
|
|
108
|
-
* written, and nothing fails. So this array is APPEND-ONLY, and
|
|
109
|
-
* `packages/schema/test/telemetry/point-dimensions.test.ts` pins it as a drift oracle — a change has
|
|
110
|
-
* to be a deliberate edit to that expectation, never a side effect.
|
|
111
|
-
*
|
|
112
|
-
* Readers index by these positions: `apps/mirror/src/telemetry-sink/sql-proxy.ts` (`blob1`..`blob10`).
|
|
113
|
-
*/
|
|
114
|
-
export const TELEMETRY_BLOB_COLUMNS = [
|
|
115
|
-
"service",
|
|
116
|
-
"name",
|
|
117
|
-
"outcome",
|
|
118
|
-
"trace_id",
|
|
119
|
-
"detail",
|
|
120
|
-
"repo",
|
|
121
|
-
"team",
|
|
122
|
-
"project_id",
|
|
123
|
-
"ticket",
|
|
124
|
-
"phase",
|
|
125
|
-
] as const;
|
|
126
|
-
|
|
127
|
-
/** An absent dimension on the AE leg. AE blobs are positional, so a dimension cannot simply be
|
|
128
|
-
* omitted without shifting its neighbours; `""` is the encoding of "not applicable". None of the
|
|
129
|
-
* five dimensions is ever legitimately the empty string, so this is unambiguous. (The Iceberg leg is
|
|
130
|
-
* named rather than positional and uses a real `null` instead — see `toTelemetryStreamRecord`.) */
|
|
131
|
-
function dimensionBlob(value: string | null | undefined): string {
|
|
132
|
-
if (value == null || value === "") return "";
|
|
133
|
-
return boundToBytes(value, MAX_DIMENSION_BYTES);
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
/** An absent dimension on the named cold leg: a real SQL `null`, distinguishable from a value. */
|
|
137
|
-
function dimensionColumn(value: string | null | undefined): string | null {
|
|
138
|
-
if (value == null || value === "") return null;
|
|
139
|
-
return boundToBytes(value, MAX_DIMENSION_BYTES);
|
|
140
|
-
}
|
|
141
|
-
|
|
142
61
|
/**
|
|
143
62
|
* One telemetry point — the shared envelope both the AE hot leg and the Iceberg cold leg derive
|
|
144
63
|
* their layout from (D3: same vocabulary, different volume).
|
|
145
64
|
*/
|
|
146
|
-
export interface TelemetryPoint
|
|
65
|
+
export interface TelemetryPoint {
|
|
147
66
|
/** Closed-registry name, e.g. "mirror.webhook.received". Low cardinality by construction. */
|
|
148
67
|
readonly name: TelemetryName;
|
|
149
68
|
/** service.name — "catalyst-cloud.mirror" today. */
|
|
@@ -202,21 +121,7 @@ export function toTelemetryDataPoint(p: TelemetryPoint, accountId: string): Tele
|
|
|
202
121
|
}
|
|
203
122
|
return {
|
|
204
123
|
indexes: [accountId],
|
|
205
|
-
blobs: [
|
|
206
|
-
p.service,
|
|
207
|
-
p.name,
|
|
208
|
-
p.outcome,
|
|
209
|
-
p.traceId,
|
|
210
|
-
boundExcerpt(p.detail),
|
|
211
|
-
// CTC-1417 — appended at fixed positions 6-10 (TELEMETRY_BLOB_COLUMNS). Always emitted, even
|
|
212
|
-
// when empty, so the array length is constant and a future append can never land on a position
|
|
213
|
-
// an older reader already gives a different meaning to.
|
|
214
|
-
dimensionBlob(p.repo),
|
|
215
|
-
dimensionBlob(p.team),
|
|
216
|
-
dimensionBlob(p.projectId),
|
|
217
|
-
dimensionBlob(p.ticket),
|
|
218
|
-
dimensionBlob(p.phase),
|
|
219
|
-
],
|
|
124
|
+
blobs: [p.service, p.name, p.outcome, p.traceId, boundExcerpt(p.detail)],
|
|
220
125
|
doubles: p.durationMs == null ? [p.count] : [p.count, p.durationMs],
|
|
221
126
|
};
|
|
222
127
|
}
|
|
@@ -237,14 +142,6 @@ export interface TelemetryStreamRecord {
|
|
|
237
142
|
readonly detail: string | null;
|
|
238
143
|
readonly duration_ms: number | null;
|
|
239
144
|
readonly count: number;
|
|
240
|
-
// CTC-1417 — the uniform tenant dimensions, named (the cold leg is not positional). ADR-0048's
|
|
241
|
-
// binding condition 1 holds unchanged: these are IDENTIFIERS, never payload bodies, and the cold
|
|
242
|
-
// leg is permanent, so nothing that could carry tenant content may join them.
|
|
243
|
-
readonly repo: string | null;
|
|
244
|
-
readonly team: string | null;
|
|
245
|
-
readonly project_id: string | null;
|
|
246
|
-
readonly ticket: string | null;
|
|
247
|
-
readonly phase: string | null;
|
|
248
145
|
readonly [key: string]: unknown;
|
|
249
146
|
}
|
|
250
147
|
|
|
@@ -265,41 +162,21 @@ export function toTelemetryStreamRecord(
|
|
|
265
162
|
detail: p.detail ? boundExcerpt(p.detail) : null,
|
|
266
163
|
duration_ms: p.durationMs,
|
|
267
164
|
count: p.count,
|
|
268
|
-
repo: dimensionColumn(p.repo),
|
|
269
|
-
team: dimensionColumn(p.team),
|
|
270
|
-
project_id: dimensionColumn(p.projectId),
|
|
271
|
-
ticket: dimensionColumn(p.ticket),
|
|
272
|
-
phase: dimensionColumn(p.phase),
|
|
273
165
|
};
|
|
274
166
|
}
|
|
275
167
|
|
|
276
168
|
/**
|
|
277
|
-
* Fold points sharing an identical (name, outcome
|
|
278
|
-
*
|
|
279
|
-
*
|
|
280
|
-
*
|
|
281
|
-
*
|
|
282
|
-
*
|
|
283
|
-
* ⚠️ CTC-1417 widened the fold key from (name, outcome) to include the five tenant dimensions. It had
|
|
284
|
-
* to: with the dimensions as columns, folding two points that differ only by `team` would emit ONE
|
|
285
|
-
* row whose `team` column names whichever point arrived first and whose `count` is the sum of both —
|
|
286
|
-
* a wrong value, not a missing one. `work-backlog-telemetry.ts` documented this hazard as a
|
|
287
|
-
* caller-side rule ("one emitTelemetry call per point"); the key now enforces it here instead, so a
|
|
288
|
-
* caller that batches distinct dimensions gets correct rows rather than silently merged ones.
|
|
169
|
+
* Fold points sharing an identical (name, outcome) pair into a single point carrying the summed
|
|
170
|
+
* count and the FIRST traceId seen for that pair — the direct answer to ADR-0039's measurement (one
|
|
171
|
+
* fact logged 14,256 times). Order-preserving on first occurrence. Applies identically regardless of
|
|
172
|
+
* sampling class, to BOTH legs (D3/D4) — a `counted` name's "never stored individually" guarantee
|
|
173
|
+
* comes from this stage, not from `applySampling`.
|
|
289
174
|
*/
|
|
290
175
|
export function collapse(points: readonly TelemetryPoint[]): TelemetryPoint[] {
|
|
291
176
|
const order: string[] = [];
|
|
292
177
|
const byKey = new Map<string, TelemetryPoint>();
|
|
293
178
|
for (const p of points) {
|
|
294
|
-
const key =
|
|
295
|
-
p.name,
|
|
296
|
-
p.outcome,
|
|
297
|
-
p.repo ?? "",
|
|
298
|
-
p.team ?? "",
|
|
299
|
-
p.projectId ?? "",
|
|
300
|
-
p.ticket ?? "",
|
|
301
|
-
p.phase ?? "",
|
|
302
|
-
].join("\u0000");
|
|
179
|
+
const key = `${p.name}\u0000${p.outcome}`;
|
|
303
180
|
const existing = byKey.get(key);
|
|
304
181
|
if (existing) {
|
|
305
182
|
byKey.set(key, { ...existing, count: existing.count + p.count });
|