@catalyst-cloud/schema 0.1.38 → 0.1.39

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@catalyst-cloud/schema",
-  "version": "0.1.38",
+  "version": "0.1.39",
   "type": "module",
   "description": "Typed Drizzle schema = single source of truth for the per-tenant Mirror DO SQLite store (CTC-13 / ADR-0002). Shared by the mirror Worker, the host-sync replica, and the browser OPFS replica.",
   "license": "MIT",

package/src/telemetry/phase-lifecycle-point.ts

@@ -37,6 +37,12 @@ export interface PhaseLifecycleFields {
   phase: string;
   nonce: number;
   executor: string;
+  /** CTC-1417 — `owner/name`, when the caller's dispatch data names one. Absent, never guessed: a
+   *  refusal that never resolved to a repo must not claim one. */
+  repo?: string | null;
+  /** CTC-1417 — the Linear team key the work unit belongs to, when the caller knows it. */
+  team?: string | null;
+  projectId?: string | null;
   substrate?: string;
   provider?: string;
   branch?: string;
@@ -47,16 +53,30 @@ export interface PhaseLifecycleFields {
   reason?: string;
 }
 
+/** A ticket identifier's team key IS its prefix — `CTC-1417` belongs to team `CTC`. That is not a
+ *  guess: it is how Linear constructs the identifier and how this repo's own dispatch queue
+ *  partitions (`listPublishedTeams`, `explainEligibleWork`). Restated here (rather than imported from
+ *  ci-run-telemetry.ts's `TICKET_SHAPE`, which tests a whole different input) so the derivation is
+ *  ANCHORED: a value that is not identifier-shaped yields null and the dimension stays absent, rather
+ *  than a leading substring of something arbitrary being reported as a team. */
+const TICKET_IDENTIFIER = /^([A-Z][A-Z0-9]{1,9})-\d+$/;
+
+export function teamFromTicket(ticket: string): string | null {
+  return TICKET_IDENTIFIER.exec(ticket)?.[1] ?? null;
+}
+
 /**
  * Build one `phase.lifecycle` TelemetryPoint. Throws (never a falsy sentinel) on an unrecognized
  * `status` — the same posture as `assertEmittable` for the event name.
  */
 export function buildPhaseLifecyclePoint(fields: PhaseLifecycleFields): TelemetryPoint {
   assertKnownStatus(fields.status);
+  // CTC-1417 — `ticket` and `phase` are TYPED COLUMNS now (see the point's own dimensions), so they
+  // are no longer repeated here as free text. Nothing is lost: `traceId` still spells
+  // `ticket:phase:nonce`, and the columns are what an AE query can actually filter and group on.
+  // What stays in `detail` is exactly what has no column of its own.
   const detailParts = [
     `status=${fields.status}`,
-    `ticket=${fields.ticket}`,
-    `phase=${fields.phase}`,
     `nonce=${fields.nonce}`,
     `executor=${fields.executor}`,
     ...(fields.substrate !== undefined ? [`substrate=${fields.substrate}`] : []),
@@ -75,5 +95,12 @@ export function buildPhaseLifecyclePoint(fields: PhaseLifecycleFields): Telemetr
     traceId: `${fields.ticket}:${fields.phase}:${fields.nonce}`,
     durationMs: fields.durationMs ?? null,
     count: 1,
+    ticket: fields.ticket,
+    phase: fields.phase,
+    repo: fields.repo ?? null,
+    // An explicit `team` from the caller wins; otherwise derive it from the ticket, which every one
+    // of these call sites already has. Null when the ticket is not identifier-shaped.
+    team: fields.team ?? teamFromTicket(fields.ticket),
+    projectId: fields.projectId ?? null,
   };
 }

package/src/telemetry/point.ts

@@ -18,6 +18,10 @@ export const MAX_POINTS_PER_BATCH = 25;
 export const MAX_INDEX_BYTES = 96;
 /** ClickHouse/AE per-blob byte cap. The bounded excerpt (`detail`) is budgeted against this. */
 export const MAX_EXCERPT_BYTES = 1_024;
+/** Per-DIMENSION blob cap (CTC-1417). A dimension is an identifier — an `owner/name`, a team key, a
+ *  `CTC-NNN`, a phase name — never prose, so it is budgeted far below `MAX_EXCERPT_BYTES`. The cap is
+ *  a guard against a malformed caller, not an expected truncation point. */
+export const MAX_DIMENSION_BYTES = 256;
 
 function byteLen(s: string): number {
   return new TextEncoder().encode(s).length;
@@ -32,12 +36,18 @@ function byteLen(s: string): number {
  * of carrying its own — one truncator, not two that could drift.
  */
 export function boundExcerpt(raw: string): string {
+  return boundToBytes(raw, MAX_EXCERPT_BYTES);
+}
+
+/** The byte-budgeted, UTF-8-boundary-safe truncator {@link boundExcerpt} and the CTC-1417 dimension
+ *  bounder both use — one implementation so the two budgets can never drift into two truncators. */
+function boundToBytes(raw: string, cap: number): string {
   const bytes = byteLen(raw);
-  if (bytes <= MAX_EXCERPT_BYTES) return raw;
+  if (bytes <= cap) return raw;
   // Reserve marker room using the WORST-CASE omitted count (the whole payload); the real marker is
-  // never longer, so the final excerpt is guaranteed ≤ MAX_EXCERPT_BYTES.
+  // never longer, so the final excerpt is guaranteed ≤ cap.
   const markerReserve = byteLen(`…[+${bytes} bytes]`);
-  const keepBudget = Math.max(0, MAX_EXCERPT_BYTES - markerReserve);
+  const keepBudget = Math.max(0, cap - markerReserve);
   const kept = new TextEncoder().encode(raw).slice(0, keepBudget);
   // Lenient decode → a split trailing multi-byte becomes U+FFFD; strip it so output is valid + smaller.
   let decoded = new TextDecoder().decode(kept);
@@ -58,11 +68,82 @@ export function resolveSampling(name: string): SamplingClass {
   return spec?.sampling ?? DEFAULT_SAMPLING;
 }
 
+/**
+ * The uniform tenant dimensions every telemetry point carries (CTC-1417), on top of the `account_id`
+ * that is already the AE index. These are the keys an operator segments by — "which tenant, which
+ * repo, which team, which unit of work" — so they are TYPED COLUMNS, not text packed into `detail`:
+ * Analytics Engine cannot filter or GROUP BY on a substring of a free-text blob, which is exactly why
+ * `phase.lifecycle`/`ci.run.completed`/`work.backlog.sampled` were unqueryable by ticket or team.
+ *
+ * Every field is optional and every ABSENT field means "not applicable to this event", never a
+ * default. A platform-level event (fleet-wide polling, a webhook that never resolved to a tenant work
+ * unit) leaves them unset rather than inventing a repo — a fabricated dimension is worse than a
+ * missing one, because it is indistinguishable from a real measurement downstream.
+ *
+ * ⚠️ `ticket` and `phase` are deliberately in this set even though they are HIGH-cardinality relative
+ * to `repo`/`team`. That is safe HERE and only here: AE columns and Iceberg columns are not label
+ * sets, so a per-ticket value costs a column value, not a new time series. The same two identifiers
+ * must stay BODY-level on the OTLP path (Loki promotes resource attributes to labels — see
+ * docs/observability.md), which is why THEY live on this sink envelope and not in
+ * `@catalyst-cloud/observability`'s `Dimensions` (ADR-0011), which is the label-safe vocabulary.
+ * `repo`, `team` and `project_id` belong to BOTH — they are low-cardinality, so they are canonical
+ * dimensions over there (`team` added by CTC-1417, once OTL-96 made the collector index it) and
+ * columns here. `ticket` and `phase` are the two that are columns here ONLY.
+ */
+export interface TelemetryDimensions {
+  /** `owner/name` — the same spelling `normalize/github.ts` derives from `repository.full_name`. */
+  readonly repo?: string | null;
+  /** Linear team key (e.g. `CTC`), the dispatch-queue partition. */
+  readonly team?: string | null;
+  readonly projectId?: string | null;
+  /** Ticket identifier (e.g. `CTC-1417`) — set only where a unit of work exists. */
+  readonly ticket?: string | null;
+  /** Relay phase (e.g. `implement`) — set only where a unit of work exists. */
+  readonly phase?: string | null;
+}
+
+/**
+ * The AE hot leg's positional blob layout, declared once (CTC-1417). A positional wire format has no
+ * names on it: inserting or reordering a column silently rewrites the meaning of every row already
+ * written, and nothing fails. So this array is APPEND-ONLY, and
+ * `packages/schema/test/telemetry/point-dimensions.test.ts` pins it as a drift oracle — a change has
+ * to be a deliberate edit to that expectation, never a side effect.
+ *
+ * Readers index by these positions: `apps/mirror/src/telemetry-sink/sql-proxy.ts` (`blob1`..`blob10`).
+ */
+export const TELEMETRY_BLOB_COLUMNS = [
+  "service",
+  "name",
+  "outcome",
+  "trace_id",
+  "detail",
+  "repo",
+  "team",
+  "project_id",
+  "ticket",
+  "phase",
+] as const;
+
+/** An absent dimension on the AE leg. AE blobs are positional, so a dimension cannot simply be
+ *  omitted without shifting its neighbours; `""` is the encoding of "not applicable". None of the
+ *  five dimensions is ever legitimately the empty string, so this is unambiguous. (The Iceberg leg is
+ *  named rather than positional and uses a real `null` instead — see `toTelemetryStreamRecord`.) */
+function dimensionBlob(value: string | null | undefined): string {
+  if (value == null || value === "") return "";
+  return boundToBytes(value, MAX_DIMENSION_BYTES);
+}
+
+/** An absent dimension on the named cold leg: a real SQL `null`, distinguishable from a value. */
+function dimensionColumn(value: string | null | undefined): string | null {
+  if (value == null || value === "") return null;
+  return boundToBytes(value, MAX_DIMENSION_BYTES);
+}
+
 /**
  * One telemetry point — the shared envelope both the AE hot leg and the Iceberg cold leg derive
  * their layout from (D3: same vocabulary, different volume).
  */
-export interface TelemetryPoint {
+export interface TelemetryPoint extends TelemetryDimensions {
   /** Closed-registry name, e.g. "mirror.webhook.received". Low cardinality by construction. */
   readonly name: TelemetryName;
   /** service.name — "catalyst-cloud.mirror" today. */
@@ -121,7 +202,21 @@ export function toTelemetryDataPoint(p: TelemetryPoint, accountId: string): Tele
   }
   return {
     indexes: [accountId],
-    blobs: [p.service, p.name, p.outcome, p.traceId, boundExcerpt(p.detail)],
+    blobs: [
+      p.service,
+      p.name,
+      p.outcome,
+      p.traceId,
+      boundExcerpt(p.detail),
+      // CTC-1417 — appended at fixed positions 6-10 (TELEMETRY_BLOB_COLUMNS). Always emitted, even
+      // when empty, so the array length is constant and a future append can never land on a position
+      // an older reader already gives a different meaning to.
+      dimensionBlob(p.repo),
+      dimensionBlob(p.team),
+      dimensionBlob(p.projectId),
+      dimensionBlob(p.ticket),
+      dimensionBlob(p.phase),
+    ],
     doubles: p.durationMs == null ? [p.count] : [p.count, p.durationMs],
   };
 }
@@ -142,6 +237,14 @@ export interface TelemetryStreamRecord {
   readonly detail: string | null;
   readonly duration_ms: number | null;
   readonly count: number;
+  // CTC-1417 — the uniform tenant dimensions, named (the cold leg is not positional). ADR-0048's
+  // binding condition 1 holds unchanged: these are IDENTIFIERS, never payload bodies, and the cold
+  // leg is permanent, so nothing that could carry tenant content may join them.
+  readonly repo: string | null;
+  readonly team: string | null;
+  readonly project_id: string | null;
+  readonly ticket: string | null;
+  readonly phase: string | null;
   readonly [key: string]: unknown;
 }
 
@@ -162,21 +265,41 @@ export function toTelemetryStreamRecord(
     detail: p.detail ? boundExcerpt(p.detail) : null,
     duration_ms: p.durationMs,
     count: p.count,
+    repo: dimensionColumn(p.repo),
+    team: dimensionColumn(p.team),
+    project_id: dimensionColumn(p.projectId),
+    ticket: dimensionColumn(p.ticket),
+    phase: dimensionColumn(p.phase),
   };
 }
 
 /**
- * Fold points sharing an identical (name, outcome) pair into a single point carrying the summed
- * count and the FIRST traceId seen for that pair — the direct answer to ADR-0039's measurement (one
- * fact logged 14,256 times). Order-preserving on first occurrence. Applies identically regardless of
- * sampling class, to BOTH legs (D3/D4) — a `counted` name's "never stored individually" guarantee
- * comes from this stage, not from `applySampling`.
+ * Fold points sharing an identical (name, outcome, dimensions) tuple into a single point carrying
+ * the summed count and the FIRST traceId seen for that tuple — the direct answer to ADR-0039's
+ * measurement (one fact logged 14,256 times). Order-preserving on first occurrence. Applies
+ * identically regardless of sampling class, to BOTH legs (D3/D4) — a `counted` name's "never stored
+ * individually" guarantee comes from this stage, not from `applySampling`.
+ *
+ * ⚠️ CTC-1417 widened the fold key from (name, outcome) to include the five tenant dimensions. It had
+ * to: with the dimensions as columns, folding two points that differ only by `team` would emit ONE
+ * row whose `team` column names whichever point arrived first and whose `count` is the sum of both —
+ * a wrong value, not a missing one. `work-backlog-telemetry.ts` documented this hazard as a
+ * caller-side rule ("one emitTelemetry call per point"); the key now enforces it here instead, so a
+ * caller that batches distinct dimensions gets correct rows rather than silently merged ones.
  */
 export function collapse(points: readonly TelemetryPoint[]): TelemetryPoint[] {
   const order: string[] = [];
   const byKey = new Map<string, TelemetryPoint>();
   for (const p of points) {
-    const key = `${p.name}\u0000${p.outcome}`;
+    const key = [
+      p.name,
+      p.outcome,
+      p.repo ?? "",
+      p.team ?? "",
+      p.projectId ?? "",
+      p.ticket ?? "",
+      p.phase ?? "",
+    ].join("\u0000");
     const existing = byKey.get(key);
     if (existing) {
       byKey.set(key, { ...existing, count: existing.count + p.count });