@catalyst-cloud/schema 0.1.24 → 0.1.25

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@catalyst-cloud/schema",
-  "version": "0.1.24",
+  "version": "0.1.25",
   "type": "module",
   "description": "Typed Drizzle schema = single source of truth for the per-tenant Mirror DO SQLite store (CTC-13 / ADR-0002). Shared by the mirror Worker, the host-sync replica, and the browser OPFS replica.",
   "license": "MIT",

package/src/events/registry.ts

@@ -1767,19 +1767,35 @@ export const currentStateEventTypes = {
   },
 } as const satisfies Record<string, CurrentStateEventSpec>;
 
+/**
+ * CTC-962 (ADR-0048) — how much of a telemetry name is STORED by the cloud-owned sink. Absent from a
+ * spec ⇒ `DEFAULT_SAMPLING` (packages/schema/src/telemetry/point.ts) — never "unbounded". Collapse
+ * (repeat-folding, both legs) applies regardless of class; head sampling (`sampled`) applies to the
+ * Analytics Engine hot leg ONLY — the Pipelines→R2 Iceberg cold leg is unsampled by design (D3): a
+ * durable RECORD you sampled is not a record.
+ */
+export type SamplingClass =
+  | { kind: "always" } // rare, high-value: errors, boundary events — never dropped by sampling.
+  | { kind: "sampled"; oneIn: number } // head sampling for high-volume names (hot leg only).
+  | { kind: "counted" }; // no individual value; the collapse stage's rollup point IS the record.
+
 /** Why a type is excluded from the durable contract — the fact a consumer would lose if this reason
  *  is wrong, per the phase-3 producer-inventory bar ("names the failure someone would notice"). */
 export interface TelemetryEventSpec {
   reason: string;
+  /** CTC-962 (ADR-0048) — the sampling class the cloud-owned telemetry sink applies to this name. */
+  sampling?: SamplingClass;
 }
 
 /**
- * ⛔ EXILED from the durable contract (ADR-0039 binding condition 1). 73 types, the volume-dominant
- * side of Finding 3's split (92.3% of August's named lines were telemetry-shaped in the plan's
- * measurement) — ticks, gauges, process samples, read traces, forwarding lag, "would-*" dry-run
- * counterfactuals, and retry-storm noise that collapses to a single durable boundary fact elsewhere
- * in this registry (e.g. linear.write.proxy.failed's 140k+ lines/mo collapses to the ONE durable
- * linear.label.retry-exhausted terminal fact).
+ * ⛔ EXILED from the durable contract (ADR-0039 binding condition 1). 70 fleet/orchestrator-origin
+ * types (this doc comment previously said 73 — stale; measured, not re-derived, so it does not drift
+ * again), the volume-dominant side of Finding 3's split (92.3% of August's named lines were
+ * telemetry-shaped in the plan's measurement) — ticks, gauges, process samples, read traces,
+ * forwarding lag, "would-*" dry-run counterfactuals, and retry-storm noise that collapses to a single
+ * durable boundary fact elsewhere in this registry (e.g. linear.write.proxy.failed's 140k+ lines/mo
+ * collapses to the ONE durable linear.label.retry-exhausted terminal fact). CTC-962 (ADR-0048) added
+ * 5 mirror-ingress names on top of those 70 (measured total: 75) — see the `mirror.*` entries below.
  */
 export const telemetryEventTypes = {
   "account.ratelimit.sampled": {
@@ -1874,6 +1890,35 @@ export const telemetryEventTypes = {
     reason:
       "A producer bug: 14 lines shipped with an empty `event.name`. Not a real type — flagged in the phase-3 producer inventory as a defect for the sibling repo to fix (a JSONL writer must never emit an unnamed line), not something the durable registry can carry forward as a name.",
   },
+  // CTC-962 (ADR-0048) — the mirror-ingress signals the cloud-owned telemetry sink's first producer
+  // emits. `rejected` / `unroutable` / `dead-lettered` are boundary facts that could argue for
+  // DURABLE classification instead (see the plan's D4 flagged question) — classified telemetry here
+  // deliberately, with an ADR-0039 registry change filed as a follow-up rather than decided silently.
+  "mirror.ingest.committed": {
+    reason:
+      "A successfully-committed ingest envelope — high volume by construction (one per accepted webhook), and the durable fact it corresponds to (the event_log row itself) already exists; this is a volume signal for the sink, not a second copy of the durable record. Launched UNSAMPLED (Ryan, 2026-08-28): measured volume (~6.7K/day) sits well inside AE's included allowance; re-introduce sampled(N) here when volume warrants — the machinery stays.",
+    sampling: { kind: "always" },
+  },
+  "mirror.ingest.dead-lettered": {
+    reason:
+      "An accepted-then-lost ingest envelope (CTC-294's silent-hole alarm) — rare and high-value; every occurrence is a delivery that reached neither change_log nor event_log and must be individually visible, not sampled away.",
+    sampling: { kind: "always" },
+  },
+  "mirror.webhook.received": {
+    reason:
+      "Every accepted provider webhook delivery — the highest-volume mirror-ingress signal by construction, one per delivery from two providers across every connected tenant. Launched UNSAMPLED (Ryan, 2026-08-28): measured volume (~6.7K/day) sits well inside AE's included allowance; re-introduce sampled(N) here when volume warrants — the machinery stays.",
+    sampling: { kind: "always" },
+  },
+  "mirror.webhook.rejected": {
+    reason:
+      "A webhook that failed signature/timestamp/JSON verification — each occurrence is independently actionable (which gate, which tenant), so it is not sampled. ⚠️ NOT rare: measured at 11,027/24h on staging against 6,710 `received` over the same window (CTC-962 Phase 6 — docs/observability.md's noise-budget table). The `always` class here is a deliberately-paid cold-tier cost justified by per-occurrence diagnostic value, NOT an assumption that this name is low-volume.",
+    sampling: { kind: "always" },
+  },
+  "mirror.webhook.unroutable": {
+    reason:
+      "A verified webhook naming no connected account — carries the routing key that failed, the entire diagnostic value of the signal; sampling it away would hide which workspace is mis-bound.",
+    sampling: { kind: "always" },
+  },
   "orphans.reap-requested": {
     reason:
       "Phase-agent worker-directory garbage collection. phase.terminal.reap-requested alone is 10,015 lines/mo — a periodic sweep re-requesting cleanup of already-terminal workers, not a discrete coordination fact worth replay.",

package/src/index.ts

@@ -82,6 +82,11 @@ export {
 // packages/schema/src/events/index.ts's own header for what phase 2+ adds to this barrel.
 export * from "./events/index.js";
 
+// CTC-962 (ADR-0048) — the cloud-owned telemetry sink's shared envelope: TelemetryPoint, the
+// admission gate over the events registry above, and the pure AE/Iceberg record builders both the
+// mirror Worker and any future out-of-Worker emitter import.
+export * from "./telemetry/point.js";
+
 /** Every table in the Mirror DO store — pass as `drizzle(storage, { schema: mirrorSchema })`. */
 export const mirrorSchema = {
   issues,

package/src/telemetry/point.ts

@@ -0,0 +1,209 @@
+// point.ts — CTC-962 (ADR-0048). The telemetry envelope the cloud-owned sink's two-leg producer
+// emits, and the pure functions that turn a batch of them into an Analytics Engine data point (hot,
+// sampled) or a Pipelines/Iceberg stream record (cold, unsampled). Zero Workers-runtime dependency —
+// same discipline as ../events/type-grammar.ts — so the mirror Worker and any future out-of-Worker
+// emitter (runner, index-host) both import this from a plain Node/browser-safe module.
+
+import { telemetryEventTypes, classifyEventType, type SamplingClass } from "../events/registry.js";
+
+export type { SamplingClass } from "../events/registry.js";
+
+/** The wire `name` of a telemetry point — the keys of telemetryEventTypes, closed by construction. */
+export type TelemetryName = keyof typeof telemetryEventTypes;
+
+// AE platform caps — the SAME values as apps/mirror/src/event-stream/validate.ts, which re-exports
+// them (and `boundExcerpt` below) from this module rather than carrying a second definition, so the
+// event-stream and telemetry excerpt truncators can never drift apart (CTC-962 plan D-notes).
+export const MAX_POINTS_PER_BATCH = 25;
+export const MAX_INDEX_BYTES = 96;
+/** ClickHouse/AE per-blob byte cap. The bounded excerpt (`detail`) is budgeted against this. */
+export const MAX_EXCERPT_BYTES = 1_024;
+
+function byteLen(s: string): number {
+  return new TextEncoder().encode(s).length;
+}
+
+/**
+ * Bound a would-be-blob string to the AE per-blob cap, appending a stable elision marker that names
+ * how many bytes were dropped. Truncation is on a UTF-8 CHARACTER boundary: a naive byte slice can
+ * split a multi-byte sequence, so the kept bytes are decoded leniently and a trailing partial
+ * (U+FFFD) is dropped so the excerpt is always valid text AND stays under the byte budget. Lifted
+ * here (CTC-962) from apps/mirror/src/event-stream/validate.ts, which now imports this copy instead
+ * of carrying its own — one truncator, not two that could drift.
+ */
+export function boundExcerpt(raw: string): string {
+  const bytes = byteLen(raw);
+  if (bytes <= MAX_EXCERPT_BYTES) return raw;
+  // Reserve marker room using the WORST-CASE omitted count (the whole payload); the real marker is
+  // never longer, so the final excerpt is guaranteed ≤ MAX_EXCERPT_BYTES.
+  const markerReserve = byteLen(`…[+${bytes} bytes]`);
+  const keepBudget = Math.max(0, MAX_EXCERPT_BYTES - markerReserve);
+  const kept = new TextEncoder().encode(raw).slice(0, keepBudget);
+  // Lenient decode → a split trailing multi-byte becomes U+FFFD; strip it so output is valid + smaller.
+  let decoded = new TextDecoder().decode(kept);
+  if (decoded.endsWith("�")) decoded = decoded.slice(0, -1);
+  const omitted = bytes - byteLen(decoded);
+  return `${decoded}…[+${omitted} bytes]`;
+}
+
+/** Conservative default for a telemetry name with no explicit `sampling` spec — never "unbounded".
+ *  70 pre-existing fleet/orchestrator names carry no per-name budget yet (see registry.ts); defaulting
+ *  them to `always` would import their full volume into the hot tier the moment anything emits them. */
+export const DEFAULT_SAMPLING: SamplingClass = { kind: "sampled", oneIn: 100 };
+
+/** Resolve a telemetry name's sampling class — its own spec's, or DEFAULT_SAMPLING. Never undefined,
+ *  so a name added to the registry without a `sampling` field still resolves conservatively. */
+export function resolveSampling(name: string): SamplingClass {
+  const spec = (telemetryEventTypes as Record<string, { sampling?: SamplingClass }>)[name];
+  return spec?.sampling ?? DEFAULT_SAMPLING;
+}
+
+/**
+ * One telemetry point — the shared envelope both the AE hot leg and the Iceberg cold leg derive
+ * their layout from (D3: same vocabulary, different volume).
+ */
+export interface TelemetryPoint {
+  /** Closed-registry name, e.g. "mirror.webhook.received". Low cardinality by construction. */
+  readonly name: TelemetryName;
+  /** service.name — "catalyst-cloud.mirror" today. */
+  readonly service: string;
+  readonly outcome: "ok" | "error" | "skip";
+  /** BOUNDED ≤1024-byte excerpt. Dimensions and identifiers only — never tenant content (ADR-0048
+   *  binding condition 1). */
+  readonly detail: string;
+  /** ADR-0011 trace_id, so a cloud row joins the same correlation key as every other signal. */
+  readonly traceId: string;
+  readonly durationMs: number | null;
+  /** >1 when this point is a collapsed repeat (see `collapse`). */
+  readonly count: number;
+}
+
+/**
+ * Refuse to emit a name that is not `classifyEventType(name) === "telemetry"` — the D4 admission
+ * gate. A `durable` or `current-state` name must never be downgraded into this sampled, truncating,
+ * ~90-day-hot tier; an `unclassified` name has never been reviewed for either bucket. Throws rather
+ * than returning a falsy sentinel (AGENTS.md: a helper that degrades to "fine" hides the defect).
+ */
+export function assertEmittable(name: string): void {
+  const cls = classifyEventType(name);
+  if (cls !== "telemetry") {
+    throw new Error(
+      `telemetry sink refuses "${name}": classifyEventType() says "${cls}", not "telemetry" ` +
+        `(packages/schema/src/events/registry.ts is the one registry — see ADR-0048 binding condition 2)`,
+    );
+  }
+}
+
+/**
+ * Structurally matches Cloudflare's `AnalyticsEngineDataPoint` (`{indexes,blobs,doubles}` of
+ * `(ArrayBuffer|string|null)[]`/`number[]`) without importing `@cloudflare/workers-types` — this
+ * module stays Workers-free. `env.TELEMETRY.writeDataPoint()` accepts this value structurally.
+ */
+export interface TelemetryAeDataPoint {
+  readonly indexes: string[];
+  readonly blobs: string[];
+  readonly doubles: number[];
+}
+
+/**
+ * The ONE positional AE layout for the hot leg — mirrors event-stream/validate.ts's
+ * toEventStreamDataPoint exactly (positional blobs, the same index-byte guard). `doubles` is `[count,
+ * durationMs?]`: `count` is ALWAYS present so its position never shifts, and `durationMs` — genuinely
+ * unknown for most callers (e.g. `mirror.ingest.dead-lettered` never sets `telemetryDurationMs`) — is
+ * appended only when known, the same "drop the optional trailing double when null" convention
+ * event-stream/sql-proxy.ts's `doubles:[seq]` already uses. Encoding an unknown duration as `0`
+ * (the previous behavior) would have AE and the hot-leg feed report a false zero-latency instead of
+ * "no duration recorded", indistinguishable from a real fast operation.
+ */
+export function toTelemetryDataPoint(p: TelemetryPoint, accountId: string): TelemetryAeDataPoint {
+  if (byteLen(accountId) > MAX_INDEX_BYTES) {
+    throw new Error(`accountId exceeds ${MAX_INDEX_BYTES}-byte AE index cap`);
+  }
+  return {
+    indexes: [accountId],
+    blobs: [p.service, p.name, p.outcome, p.traceId, boundExcerpt(p.detail)],
+    doubles: p.durationMs == null ? [p.count] : [p.count, p.durationMs],
+  };
+}
+
+/** The named-column Iceberg stream record — the same fields as toTelemetryDataPoint, typed and with
+ *  `account_id` first (D6: the cold tier's tenant fence is this column, not an R2 key prefix). The
+ *  index signature is there ONLY so this satisfies `cloudflare:pipelines`' `PipelineRecord` generic
+ *  constraint (`Record<string, unknown>`) without importing `@cloudflare/workers-types` here — every
+ *  named field above is still a normal, closed, typed property; `toTelemetryStreamRecord` is the
+ *  single producer of this shape and never adds an extra key. */
+export interface TelemetryStreamRecord {
+  readonly account_id: string;
+  readonly ts: string;
+  readonly service: string;
+  readonly name: string;
+  readonly outcome: string;
+  readonly trace_id: string | null;
+  readonly detail: string | null;
+  readonly duration_ms: number | null;
+  readonly count: number;
+  readonly [key: string]: unknown;
+}
+
+/** Build the cold-leg (Iceberg) record for one point. Unsampled by construction — callers pass the
+ *  full retained batch here, never the AE-sampled subset (D3). */
+export function toTelemetryStreamRecord(
+  p: TelemetryPoint,
+  accountId: string,
+  ts: string,
+): TelemetryStreamRecord {
+  return {
+    account_id: accountId,
+    ts,
+    service: p.service,
+    name: p.name,
+    outcome: p.outcome,
+    trace_id: p.traceId || null,
+    detail: p.detail ? boundExcerpt(p.detail) : null,
+    duration_ms: p.durationMs,
+    count: p.count,
+  };
+}
+
+/**
+ * Fold points sharing an identical (name, outcome) pair into a single point carrying the summed
+ * count and the FIRST traceId seen for that pair — the direct answer to ADR-0039's measurement (one
+ * fact logged 14,256 times). Order-preserving on first occurrence. Applies identically regardless of
+ * sampling class, to BOTH legs (D3/D4) — a `counted` name's "never stored individually" guarantee
+ * comes from this stage, not from `applySampling`.
+ */
+export function collapse(points: readonly TelemetryPoint[]): TelemetryPoint[] {
+  const order: string[] = [];
+  const byKey = new Map<string, TelemetryPoint>();
+  for (const p of points) {
+    const key = `${p.name}\u0000${p.outcome}`;
+    const existing = byKey.get(key);
+    if (existing) {
+      byKey.set(key, { ...existing, count: existing.count + p.count });
+    } else {
+      order.push(key);
+      byKey.set(key, { ...p });
+    }
+  }
+  return order.map((k) => byKey.get(k)!);
+}
+
+/** Injectable RNG surface — a `() => number` in [0, 1), the same shape as `Math.random`. */
+export type Rng = () => number;
+
+/**
+ * Apply the AE-only head-sampling policy (D3: sampling applies to the hot leg only — the cold leg
+ * gets the full `collapse()`d batch, unsampled). `always` and `counted` names always survive here —
+ * `counted`'s "no individual point" guarantee is `collapse`'s job (see its doc), not this function's;
+ * only `sampled(N)` is probabilistic, surviving with probability 1/N against the injected `rng`.
+ */
+export function applySampling(
+  points: readonly TelemetryPoint[],
+  rng: Rng = Math.random,
+): TelemetryPoint[] {
+  return points.filter((p) => {
+    const cls = resolveSampling(p.name);
+    if (cls.kind === "sampled") return rng() < 1 / cls.oneIn;
+    return true;
+  });
+}