@noego/telemetry 0.0.3 → 0.1.0

package/dist/projector/index.d.ts

@@ -0,0 +1,246 @@
+import { Database } from 'better-sqlite3';
+import { TraceSource, TraceSourceDescription } from '@noego/trace/sqlite';
+import { M as MetricDefinition } from '../metric.types-D8aOAnUm.js';
+import '../json.types-BqS77fcN.js';
+
+/**
+ * Declarative trace projection definitions (spec 04 §6).
+ *
+ * A projection describes how raw trace events become historical time-series
+ * observations. Definitions are immutable by (name, version) once historical
+ * data exists: changing meaning requires a version bump.
+ */
+type TraceAggregate = 'count' | 'sum' | 'min' | 'max' | 'avg' | 'last';
+type TraceMeasurement = {
+    /** Numeric field read from the trace event's context. */
+    field: string;
+    aggregates: readonly TraceAggregate[];
+};
+type TraceCardinalityPolicy = {
+    /** Bound on distinct dimension-value combinations per projection. */
+    maxDimensionKeys?: number;
+    /** At the bound: 'reject' records a projection failure; 'other' folds into an __other__ bucket. */
+    overflow?: 'reject' | 'other';
+};
+type TraceProjection = {
+    name: string;
+    version: number;
+    /**
+     * Trace event name selector (TraceEvent.event). Exact match, or a glob:
+     * `*` matches one dot-separated segment, `**` matches any remainder
+     * (e.g. 'render-cycle.**' matches 'render-cycle.turns-changed').
+     */
+    event: string;
+    /** Optional source filter (TraceEvent.source). */
+    source?: string;
+    /** Context fields promoted to bounded historical dimensions. */
+    groupBy?: readonly string[];
+    measurements: readonly TraceMeasurement[];
+    /** Bucket resolutions in milliseconds (e.g. 60_000, 3_600_000). */
+    resolutions: readonly number[];
+    /**
+     * Behavior when a measurement field is absent/non-numeric on a matching
+     * event: 'record-failure' (default) records a bounded projection failure and
+     * skips the event; 'skip' omits the field but still counts the event.
+     */
+    missingField?: 'skip' | 'record-failure';
+    cardinality?: TraceCardinalityPolicy;
+};
+/** Validates a projection definition, throwing on structural errors. */
+declare function validateTraceProjection(projection: TraceProjection): void;
+
+type TraceProjectorOptions = {
+    /** Bounded reader over the raw trace store (owned by @noego/trace). */
+    source: TraceSource;
+    /** Historical SQL destination. Owned by telemetry; separate from raw traces. */
+    destination: Database;
+    projections: readonly TraceProjection[];
+    /** Raw events read per page; default 500. */
+    pageSize?: number;
+    now?: () => number;
+};
+/**
+ * Why a pass produced no bucket updates (spec 05 §5.4). A pass with buckets
+ * reports `buckets-updated`; every other value names the reason so "cursor
+ * advancing, zero buckets" can never be mistaken for health.
+ */
+type TraceProjectorProgress = 'buckets-updated' | 'caught-up-no-new-source' | 'new-source-no-selector-match' | 'matching-source-no-bucket';
+/** Why the durable cursor was discarded before this pass. */
+type TraceSourceReset = 'store-epoch-changed' | 'cursor-beyond-source';
+type TraceProjectorRunResult = {
+    pages: number;
+    eventsRead: number;
+    /** Events that matched at least one projection selector. */
+    selectorMatches: number;
+    bucketsUpdated: number;
+    failures: number;
+    progress: TraceProjectorProgress;
+    /** Cursor after the pass (rowid in the source store); null when nothing was ever projected. */
+    cursorRowId: number | null;
+    /** Newest source rowid at the start of the pass; null when the source cannot describe itself. */
+    sourceHighRowId: number | null;
+    sourceEpoch: string | null;
+    sourceReset: TraceSourceReset | null;
+};
+/** Backfill reprojects a range; it never touches the forward cursor, so it reports only volume. */
+type TraceBackfillResult = Pick<TraceProjectorRunResult, 'pages' | 'eventsRead' | 'bucketsUpdated' | 'failures'>;
+type TraceBucketQuery = {
+    projection: string;
+    version: number;
+    resolutionMs: number;
+    fromMs?: number;
+    toMs?: number;
+    /** Exact-match dimension filter. */
+    dimensions?: Record<string, string>;
+};
+type TraceBucketRow = {
+    projection: string;
+    version: number;
+    resolutionMs: number;
+    bucketStartMs: number;
+    dimensions: Record<string, string>;
+    count: number;
+    values: Record<string, MeasurementState>;
+    /** Computed views: avg per field where sum+count exist. */
+    averages: Record<string, number>;
+};
+type MeasurementState = {
+    count: number;
+    sum: number;
+    min: number;
+    max: number;
+    /** Last observed value in projector processing order. */
+    last: number;
+};
+/**
+ * Downstream trace projector (spec 04): reads retained raw traces through the
+ * TraceSource contract and materializes idempotent historical buckets in SQL.
+ *
+ * The projector is never part of trace ingestion. Buckets are keyed by
+ * (projection, version, resolution, bucket start, dimensions) and bucketed by
+ * the event's producer-assigned occurrence time — never by projector run time
+ * or insert time. Each page commits bucket updates, failures, and the scan
+ * cursor in one destination transaction, so a crash resumes exactly at the
+ * durable cursor with no double counting. The insertion-ordered source cursor
+ * means a late-delivered event is still observed and folded into its
+ * historical occurrence-time bucket.
+ */
+declare class TraceProjector {
+    private readonly source;
+    private readonly db;
+    private readonly projections;
+    private readonly pageSize;
+    private readonly now;
+    constructor(options: TraceProjectorOptions);
+    /**
+     * Drains all unprojected raw traces, page by page, committing progress per
+     * page.
+     *
+     * The durable cursor is a source rowid, so it is only meaningful against
+     * the store file it was taken from. When the source describes itself, a
+     * cursor whose epoch differs from the store's — or that lies beyond the
+     * store's newest row — belongs to a store that has since been recreated
+     * (boot reset, manual deletion) and is discarded, restarting the scan at
+     * row 0. 2026-09-03: the live projector sat at rowid 22.6M over a rebuilt
+     * store holding rows 1–440k, reported "caught up" every minute, and wrote
+     * no buckets for ten hours.
+     */
+    runOnce(): TraceProjectorRunResult;
+    /** Newest completed bucket start per projection at the given resolution — the freshness signal. */
+    newestBucketStartMs(resolutionMs: number): number | null;
+    /**
+     * Rebuilds one projection's derived buckets for an occurrence-time range
+     * from retained raw traces (spec 04 §14–15): clears coverage for the range,
+     * then reprojects every retained matching event. Uses the same extraction
+     * code as forward projection; does not touch the forward scan cursor.
+     */
+    backfill(options: {
+        projection: string;
+        version: number;
+        fromOccurredAtMs: number;
+        toOccurredAtMs: number;
+    }): TraceBackfillResult;
+    /** Historical bucket query with explicit projection/version/resolution. */
+    query(query: TraceBucketQuery): readonly TraceBucketRow[];
+    /** Bounded projection failures, newest first. */
+    failures(limit?: number): readonly {
+        projection: string;
+        version: number;
+        traceId: string;
+        reason: string;
+    }[];
+    private matches;
+    private applyEvent;
+    private admitDimensions;
+    private updateBucket;
+    private recordFailure;
+    private loadProgress;
+    private saveProgress;
+    private registerDefinitions;
+    private initializeSchema;
+}
+/**
+ * Matches a projection event selector against an event name. Exact match, or
+ * glob where `*` matches exactly one dot-separated segment and a trailing
+ * `**` matches any remainder (including empty).
+ */
+declare function eventSelectorMatches(selector: string, eventName: string): boolean;
+/** Durable scan progress: the source rowid plus the epoch of the store it was read from. */
+type ScanProgress = {
+    rowId: number;
+    /** Null for progress written before epochs existed; such a cursor is validated by extent only. */
+    storeEpoch: string | null;
+};
+/**
+ * Decide whether the persisted cursor still refers to the store in front of
+ * us. Epoch mismatch is definitive. Without an epoch (legacy progress, or a
+ * source that cannot describe itself) fall back to extent: a rowid beyond the
+ * newest row cannot have been read from this store.
+ */
+declare function detectSourceReset(progress: ScanProgress | null, description: TraceSourceDescription | null): TraceSourceReset | null;
+
+type TraceProjectorProcessOptions = {
+    /** Path to the raw trace SQLite written by @noego/trace (traces.db). */
+    tracesDbPath: string;
+    /** Path to the historical projection SQLite destination. */
+    destinationDbPath: string;
+    /** Projection definitions; or use configurationPath to migrate legacy metrics. */
+    projections?: readonly TraceProjection[];
+    /** telemetry.config.yml whose metric definitions become projections. */
+    configurationPath?: string;
+    /** Raw events per page; default 500. */
+    pageSize?: number;
+};
+type TraceProjectorProcess = {
+    runOnce(): TraceProjectorRunResult;
+    projector: TraceProjector;
+    close(): void;
+};
+/**
+ * Builds a runnable projector process over file-based stores (spec 04 §5).
+ *
+ * This is the separate-process execution mode: it opens the raw trace store
+ * read-side and its own historical destination, resumes from the durable
+ * cursor, and can be run continuously, on a schedule, or as an explicit
+ * maintenance command. Its lifecycle never gates raw trace collection.
+ */
+declare function createTraceProjectorProcess(options: TraceProjectorProcessOptions): TraceProjectorProcess;
+
+/**
+ * Migration of resolved telemetry.config.yml metric definitions into
+ * downstream trace projection definitions (spec 04 §18).
+ *
+ * The configuration parser has already split selectors into an event glob and
+ * groupBy dimensions and resolved per-metric bucket resolutions, so this is a
+ * shape mapping. Names and versions carry over so historical intent is
+ * preserved; the projection bucket store is new, so no numeric continuity
+ * with the legacy sample/rollup tables is implied.
+ */
+declare function metricDefinitionToTraceProjection(metric: MetricDefinition, options?: {
+    fallbackResolutions?: readonly number[];
+}): TraceProjection;
+declare function metricDefinitionsToTraceProjections(metrics: readonly MetricDefinition[], options?: {
+    fallbackResolutions?: readonly number[];
+}): readonly TraceProjection[];
+
+export { type TraceAggregate, type TraceBackfillResult, type TraceBucketQuery, type TraceBucketRow, type TraceCardinalityPolicy, type TraceMeasurement, type TraceProjection, TraceProjector, type TraceProjectorOptions, type TraceProjectorProcess, type TraceProjectorProcessOptions, type TraceProjectorProgress, type TraceProjectorRunResult, type TraceSourceReset, createTraceProjectorProcess, detectSourceReset, eventSelectorMatches, metricDefinitionToTraceProjection, metricDefinitionsToTraceProjections, validateTraceProjection };