@smart-data-engines/sde 0.1.0-dev.0

package/dist/telemetry.d.ts

@@ -0,0 +1,468 @@
+/**
+ * Measuring what the application actually does, without ever seeing what it does it to.
+ *
+ * This is the input a placement decision is made from, so its shape matters more than its
+ * precision. Three constraints shaped everything here.
+ *
+ * **It carries no values.** A record is keyed by an operation shape, which is assembled from the
+ * structure of a call and never sees its arguments. There is no code path by which a customer's row
+ * reaches a telemetry record, which is why this file can be read by a client and believed.
+ *
+ * **It cannot cost anything.** Routing already has a one percent budget for the whole library and
+ * recording happens on the same path. So: no string formatting per record, no stack walking, and a
+ * histogram rather than a list of samples. There is no lock, and that is not a shortcut - this
+ * runtime has one thread per record, so the trade the reference implementation makes (a lock taken
+ * only when a shape is first seen) has nothing to buy here.
+ *
+ * **It cannot fail the caller.** Every entry point goes through `guard`. A bug in an aggregation
+ * counter must not take down somebody's request - and the failure is counted, so it is not
+ * invisible either.
+ *
+ * The histogram deserves a word, because it is the one deliberate loss of precision. Latency lands
+ * in exponential buckets, so a percentile read out of it is approximate - within one bucket width,
+ * which is a factor of two at the extremes. That is ample for the decision it feeds: a planner
+ * cares whether a group's reads are microseconds or milliseconds, not whether p99 is 412 or 431
+ * microseconds.
+ */
+import type { Group } from './groups.js';
+import type { LogicalModel } from './model.js';
+import type { ShapeKind } from './shapes.js';
+/** 1 microsecond to about 17 s, doubling. Enough to tell a cache hit from a full scan. */
+export declare const BUCKET_COUNT = 25;
+export declare const BUCKET_BASE_NS = 1000;
+/** Exponential-bucket histogram. Fixed memory, O(1) record, approximate percentiles. */
+export declare class Histogram {
+    readonly buckets: number[];
+    count: number;
+    total: number;
+    /**
+     * Put one duration in its bucket. **Integer arithmetic only, and that is the point.**
+     *
+     * The obvious form is `Math.floor(Math.log2(ns / BUCKET_BASE_NS)) + 1`, which is the same
+     * function and the wrong way to compute it in a library that has to agree with another
+     * implementation. `log2` is not required by IEEE 754 to be correctly rounded, so two runtimes may
+     * differ in the last bit - and one bit at a power-of-two boundary is a different bucket, which is
+     * a different p99 for identical traffic, in a number a placement decision is made from.
+     *
+     * The bit length of the integer quotient is exact everywhere. `Math.clz32` is defined on the
+     * 32-bit value, and the quotient is below 2^24 in every case that is not clamped, so the clamp is
+     * checked first rather than relying on the coercion.
+     *
+     * **The vectors cannot see the difference, and that is worth saying rather than implying.**
+     * Measured: the logarithm form passes every vector in `telemetry/`, because glibc's `log2` and
+     * V8's are both exact at a power of two - the two runtimes we have agree, and the hazard is a
+     * *third* libm that does not. A property no output can distinguish on the machines available is
+     * not one a vector can hold, so it is held statically: `telemetry.test.ts` refuses a logarithm in
+     * this file.
+     */
+    record(nanoseconds: number): void;
+    /**
+     * Approximate percentile in milliseconds, or null if nothing was recorded.
+     *
+     * Returns the *upper* edge of the bucket the percentile falls in. Rounding up rather than
+     * interpolating is deliberate: a placement decision made on an optimistic latency figure is the
+     * wrong kind of wrong.
+     */
+    percentileMs(fraction: number): number | null;
+    merge(other: Histogram): void;
+}
+/** What a read filtered on: the fields compared by equality and the field a range bounded. */
+export interface ReadPredicates {
+    readonly equal: readonly string[];
+    readonly ranged: string;
+}
+/** What was observed for one operation shape. No values, by construction. */
+export declare class ShapeStats {
+    readonly shapeId: string;
+    readonly group: string;
+    readonly entity: string;
+    readonly kind: ShapeKind;
+    calls: number;
+    rows: number;
+    errors: number;
+    readonly latency: Histogram;
+    /**
+     * Calls by what they filtered on - equality fields and the bounded field (`''` for none) - names
+     * only, never values. Two reads of one shape can want different key orders; only an operation
+     * that takes a `where` reports this, so writes and point reads leave it empty.
+     */
+    readonly filtered: Map<string, {
+        predicates: ReadPredicates;
+        calls: number;
+    }>;
+    constructor(shapeId: string, group: string, entity: string, kind: ShapeKind);
+    record(nanoseconds: number, rows: number, failed: boolean, predicates?: ReadPredicates): void;
+}
+/**
+ * What was observed writing one row to one derived copy. No values, by construction.
+ *
+ * Deliberately **not** a `ShapeStats`, and that is the load-bearing decision. A fan-out is not an
+ * operation the application asked for - it is the library keeping a copy current - so recording it
+ * as a shape would add a write to the very counters a placement is scored on: `read_write_ratio`
+ * would move because a copy exists, and a group with one copy would look twice as write-heavy as
+ * the same group without one. It is also not part of `GroupFeatures`: a copy's freshness does not
+ * score a placement, it reports the health of one already made.
+ */
+export declare class FanOutStats {
+    readonly group: string;
+    readonly materialization: string;
+    writes: number;
+    /** Rows that did not reach the copy. Absence, not lateness - see `CopyFreshness`. */
+    failures: number;
+    readonly latency: Histogram;
+    constructor(group: string, materialization: string);
+    record(nanoseconds: number, failed: boolean): void;
+}
+/**
+ * How far behind one derived copy is, measured.
+ *
+ * A derived copy in this library is maintained by the fan-out in a write, in the client's own
+ * process, straight after the source. There is no asynchronous replication anywhere, so there is no
+ * queue to fall behind in. Two things can therefore be true of a copy, and only two: it is **late**
+ * by at most the duration of that one write, or the write **failed** and the row is absent rather
+ * than late. Both are reported, because a copy missing a thousand rows can have an excellent p99
+ * and a client told only the percentile reads "0.9 ms behind" off a copy that is missing yesterday.
+ */
+export interface CopyFreshness {
+    readonly group: string;
+    readonly materialization: string;
+    readonly writes: number;
+    readonly failures: number;
+    readonly lagP50Ms: number | null;
+    readonly lagP99Ms: number | null;
+    /** Whether every write reached the copy in this window. */
+    readonly complete: boolean;
+}
+export declare function copyFreshnessRecord(copy: CopyFreshness): Record<string, unknown>;
+/**
+ * The contract between telemetry and the planner.
+ *
+ * `null` means *unknown*, which is not zero and is treated differently by the planner. Anything
+ * unknown also appears in `missing`, so a reader never has to infer absence from a null.
+ */
+export interface GroupFeatures {
+    /**
+     * How many operations were observed. A count, never a value.
+     *
+     * Present because a planner comparing two groups has to know which one carries the traffic:
+     * without it, an idle group and the group serving every request are equally important. It is also
+     * evidence about the features themselves - a read/write ratio derived from twelve calls is
+     * arithmetic, not a measurement.
+     */
+    readonly calls: number;
+    readonly readWriteRatio: number | null;
+    readonly shapeMix: Readonly<Record<string, number>>;
+    readonly latencyP50Ms: number | null;
+    readonly latencyP99Ms: number | null;
+    readonly resultCardinalityP50: number | null;
+    readonly resultCardinalityP99: number | null;
+    readonly totalBytes: number | null;
+    readonly dailyGrowthBytes: number | null;
+    readonly indexToTableRatio: number | null;
+    readonly pkAccessShare: number | null;
+    readonly hasTimeDimension: boolean;
+    readonly timeFilteredShare: number | null;
+    readonly distinctShapes: number;
+    readonly writeBurstiness: number | null;
+    readonly errorShare: number | null;
+    readonly missing: readonly string[];
+    readonly complete: boolean;
+}
+/**
+ * The document key for every measured field, and the property that carries it.
+ *
+ * Two spellings because two things are being named: the **document** is the format, shared with
+ * every other implementation and therefore `snake_case`; the **property** is this language's, and a
+ * library that reads like transliterated Python is a worse library in TypeScript and no better an
+ * SDE one.
+ *
+ * The reference implementation derives its equivalent from the dataclass at runtime, which is not
+ * available here - types are erased before the code runs. So the list is the source and the type is
+ * checked against it, which is the same guarantee in the other direction and is enforced by the
+ * compiler rather than by a test: see `FIELD_LIST_IS_TOTAL`.
+ */
+export declare const MEASURED_FIELDS: readonly [readonly ["calls", "calls"], readonly ["read_write_ratio", "readWriteRatio"], readonly ["shape_mix", "shapeMix"], readonly ["latency_p50_ms", "latencyP50Ms"], readonly ["latency_p99_ms", "latencyP99Ms"], readonly ["result_cardinality_p50", "resultCardinalityP50"], readonly ["result_cardinality_p99", "resultCardinalityP99"], readonly ["total_bytes", "totalBytes"], readonly ["daily_growth_bytes", "dailyGrowthBytes"], readonly ["index_to_table_ratio", "indexToTableRatio"], readonly ["pk_access_share", "pkAccessShare"], readonly ["has_time_dimension", "hasTimeDimension"], readonly ["time_filtered_share", "timeFilteredShare"], readonly ["distinct_shapes", "distinctShapes"], readonly ["write_burstiness", "writeBurstiness"], readonly ["error_share", "errorShare"]];
+type MeasuredProperty = (typeof MEASURED_FIELDS)[number][1];
+type Bookkeeping = 'missing' | 'complete';
+/**
+ * A compile-time ratchet in both directions.
+ *
+ * Add a field to `GroupFeatures` and forget the list, and the second element stops being
+ * assignable; put a name in the list that is not a field, and the first does. `tsc` fails either
+ * way, which is where this belongs: the control plane's reader had a hand-written field list
+ * against a dataclass where every field has a default, so the next field added to the library would
+ * have been recorded as *measured* rather than missing. That defect had to be found by mutation.
+ * This one cannot exist.
+ */
+export declare const FIELD_LIST_IS_TOTAL: [
+    MeasuredProperty extends keyof GroupFeatures ? true : never,
+    Exclude<keyof GroupFeatures, Bookkeeping> extends MeasuredProperty ? true : never
+];
+/**
+ * The feature vector as the document that crosses the boundary to the control plane.
+ *
+ * **A field with no value is omitted, and `missing` is what says so.** Emitting a null would work
+ * too - the reader treats absent and null alike - but omitting is the honest spelling of "not
+ * measured", and it keeps this function from having an opinion about what a null means.
+ */
+export declare function featuresRecord(features: GroupFeatures): Record<string, unknown>;
+/** The bucket a write's rows are counted in for `write_burstiness`. */
+export declare const SECOND_NS = 1000000000;
+/** The kinds of read that take a `where` and so report what they filtered on (`filtered_on`). */
+export declare const FILTERED_KINDS: ReadonlySet<string>;
+/**
+ * The shortest span a daily growth is projected from.
+ *
+ * A day projected from a few seconds of writes is a number with no basis: a demonstration writing a
+ * hundred rows in two seconds would claim gigabytes a day. An hour is where the projection
+ * multiplies a measurement by 24 rather than by thousands.
+ */
+export declare const GROWTH_MIN_NS = 3600000000000;
+/** The unit growth is projected to, and how long the recorder keeps a group's storage samples. */
+export declare const DAY_NS = 86400000000000;
+/**
+ * One group's bytes on its source materialisation at one moment, from the engine's catalogue.
+ *
+ * Numbers only. `totalBytes` is everything the group's tables occupy, indexes included;
+ * `secondaryIndexBytes` is the part that is indexes other than the one enforcing the key - the part
+ * a physical design adds and can remove. `atNs` is the recorder's clock.
+ */
+export interface StorageSample {
+    readonly group: string;
+    readonly atNs: number;
+    readonly totalBytes: number;
+    readonly secondaryIndexBytes: number;
+}
+/** One group's size on its source materialisation, as the engine's catalogue gave it. */
+export interface StorageSize {
+    readonly group: string;
+    readonly materialization: string;
+    readonly engine: string;
+    readonly totalBytes: number;
+    readonly secondaryIndexBytes: number;
+}
+/**
+ * Why a group's size stayed unknown: its engine's adapter has no catalogue to read, a table the map
+ * names does not exist, the catalogue refused the login (ClickHouse without the `system.parts`
+ * grant) or the read failed otherwise.
+ */
+export declare const STORAGE_UNAVAILABLE: readonly ["failed", "missing_table", "refused", "unsupported"];
+export type StorageUnavailable = (typeof STORAGE_UNAVAILABLE)[number];
+/**
+ * What `Session.measureStorage` measured, and why the rest stayed unknown - a class of reason per
+ * group, never the engine's message.
+ */
+export interface StorageMeasurement {
+    readonly sizes: readonly StorageSize[];
+    readonly unavailable: Readonly<Record<string, StorageUnavailable>>;
+}
+/**
+ * A catalogue size as an exact safe integer, or a refusal. Drivers hand a 64-bit count over as text;
+ * a size past 2^53 bytes would round, and a rounded size is a wrong one.
+ */
+export declare function exactBytes(value: unknown): number;
+/**
+ * One aggregation period, ready to send.
+ *
+ * `complete` is false when the application could not collect part of the period, or when the buffer
+ * dropped windows. A window that is not complete is still sent - the planner needs to know traffic
+ * existed - but it may not be used to justify a migration.
+ */
+export interface Window {
+    readonly modelVersion: string;
+    readonly startedNs: number;
+    readonly endedNs: number;
+    readonly shapes: readonly ShapeStats[];
+    readonly complete: boolean;
+    readonly droppedWindows: number;
+    /**
+     * What the fan-out to each derived copy did. Empty when the group has no copy, which is the
+     * ordinary case - a copy exists during a migration and while a derived materialisation is in the
+     * map, not otherwise.
+     */
+    readonly fanned: readonly FanOutStats[];
+    /**
+     * The storage samples taken during this window - between the roll that opened it and the one
+     * that closed it, by the recorder's own state rather than by comparing clock readings, so a
+     * sample taken at the instant of a roll belongs to exactly one window.
+     */
+    readonly storage: readonly StorageSample[];
+    /**
+     * Every storage sample the recorder kept when this window closed: this window's and earlier ones
+     * up to a day old, which a daily growth is projected against.
+     */
+    readonly storageHistory: readonly StorageSample[];
+    /** Rows written by successful writes, per group, per whole second of the window from its start. */
+    readonly writeSeconds: ReadonlyMap<string, ReadonlyMap<number, number>>;
+}
+/**
+ * How far behind each of this group's derived copies ran, sorted by materialisation.
+ *
+ * Read out of the histogram rather than stored, like every other percentile here. A stored
+ * percentile is a second copy of a fact that changes when the samples do.
+ */
+export declare function windowCopies(window: Window, group: string): CopyFreshness[];
+/**
+ * What each operation shape of one group measured, in the model's enumeration order.
+ *
+ * The evidence a physical design can point at. A group's features say that 62% of its calls were
+ * range reads; only this says *which field* they ranged over, which is the fact a key order or a
+ * partition is chosen from. Still no values: a shape is the structure of a call, and the fields it
+ * names are the model's field names (digests, when the client hashes them).
+ *
+ * The descriptor - entity, kind, fields, target - comes from the model's own enumeration by
+ * identifier, never from the recorder, and `target` appears only on a relation walk. A record for an
+ * identifier the model does not enumerate is refused rather than described by guesswork. Every
+ * number is an integer count or a bucket edge divided by a million, like the rest of the document.
+ */
+export declare function windowShapes(window: Window, model: LogicalModel, group: string): Record<string, unknown>[];
+export interface FeatureOptions {
+    readonly hasTimeDimension?: boolean;
+    /**
+     * Per entity, the fields of a time type (`timeFields`), which `time_filtered_share` is counted
+     * against. Without it that share stays unknown: which fields are times is a fact about the model,
+     * and the window does not guess it.
+     */
+    readonly timeFields?: ReadonlyMap<string, ReadonlySet<string>>;
+    /**
+     * A range read's shape identifier and the field its shape ranges over, which settles a range read
+     * whose filters were not reported.
+     */
+    readonly rangeFields?: ReadonlyMap<string, string>;
+}
+/** Fold this window's records for one group into the planner's feature vector. */
+export declare function windowFeatures(window: Window, group: string, options?: FeatureOptions): GroupFeatures;
+/**
+ * This window as the document the control plane reads. Numbers, never rows.
+ *
+ * The model is required and it is checked. Only one fact is read from it - whether a group carries a
+ * time dimension, which is decided by declared *type* and never by a field's name - but a window
+ * serialised against the wrong model would attach that fact to the wrong groups and claim
+ * `has_time_dimension: false` for a group that has one. False is a claim; a measurement this
+ * library cannot make has to be absent.
+ *
+ * **This document is deliberately not canonical, and that needs saying because section 1 of the
+ * format contract rejects floating point outright.** Its reason is that a float's textual form
+ * differs between languages, and almost every number here is a float. This document is not signed,
+ * not hashed and never compared for equality, so the rule it breaks does not apply - but the thing
+ * that makes the `telemetry/` family checkable is narrower and worth stating: every number here is
+ * either **a ratio of two integers** or **a bucket edge divided by a million**, and IEEE 754
+ * requires division to be correctly rounded. So two languages compute the same double from the same
+ * traffic even where they would print it differently, which is why those vectors compare numbers
+ * rather than bytes.
+ */
+export declare function windowRecord(window: Window, model: LogicalModel): Record<string, unknown>;
+/**
+ * Does any entity in this group carry a time dimension?
+ *
+ * Decided by the declared type and never by the field's name. A name is not evidence: `created_at`
+ * typed as a string is a string, and treating it as a timestamp would have the planner recommend
+ * time partitioning on a column no engine can range-scan usefully. And a client may hash identifier
+ * names so that we never see them - a derivation that read names would silently produce different
+ * answers with hashing on and off, which is the property the hashed-model vectors forbid.
+ */
+export declare function hasTimeDimension(model: LogicalModel, group: Group): boolean;
+/**
+ * Each entity of this group and its fields of a time type - by type, never by name.
+ *
+ * What `time_filtered_share` counts against, for the reasons `hasTimeDimension` gives: a
+ * `created_at` typed as a string is not a time, and a hashed model must answer the same.
+ */
+export declare function timeFields(model: LogicalModel, group: Group): Map<string, Set<string>>;
+export interface StorageOptions {
+    readonly group: string;
+    readonly totalBytes: number;
+    readonly secondaryIndexBytes: number;
+}
+export interface RecordOptions {
+    readonly shapeId: string;
+    readonly group: string;
+    readonly entity: string;
+    readonly kind: ShapeKind;
+    readonly nanoseconds: number;
+    readonly rows?: number;
+    readonly failed?: boolean;
+    /**
+     * For an operation that takes a `where`: the fields it compared by equality. Absent for one that
+     * does not filter at all, and then nothing about filters is recorded.
+     */
+    readonly equal?: readonly string[];
+    /** The field a range bounded, for a filtered read. */
+    readonly ranged?: string | null;
+}
+export interface FanOutOptions {
+    readonly group: string;
+    readonly materialization: string;
+    readonly nanoseconds: number;
+    readonly failed?: boolean;
+}
+/**
+ * Accumulates records, rolls windows, and drops telemetry rather than anything else.
+ *
+ * No lock, because this runtime does not need one - and that is worth stating rather than leaving
+ * as an absence. The reference implementation takes one only when a shape is first seen or a window
+ * rolls, and accepts that two threads racing on the same shape can lose a call from a count. Here
+ * there is nothing to race: the cost this class is allowed is zero either way.
+ */
+export declare class Recorder {
+    private readonly modelVersion;
+    private readonly maxWindows;
+    private readonly clock;
+    private current;
+    private fanned;
+    private writes;
+    private storage;
+    private storageWindow;
+    private startedNs;
+    private windows;
+    private dropped;
+    private incomplete;
+    /**
+     * `clock` returns nanoseconds and defaults to a monotonic one; tests and the conformance vectors
+     * pass their own, because a write's second and a sample's age are read from it.
+     */
+    constructor(modelVersion: string, maxWindows?: number, clock?: () => number);
+    /** Record one operation. Never throws. */
+    record(options: RecordOptions): void;
+    /**
+     * Record one group's size, as the engine's catalogue reported it. Never throws.
+     *
+     * `Session.measureStorage` calls this for every group it could measure. A sample is kept for a
+     * day, across windows, because growth is a property of time rather than of one window; a sample
+     * that is not two non-negative safe integers with the index part inside the total is dropped,
+     * never guessed into shape.
+     */
+    recordStorage(options: StorageOptions): void;
+    /**
+     * Record one write to one derived copy. Never throws.
+     *
+     * A separate entry point from `record` rather than a shape kind, because a fan-out is not an
+     * operation the application asked for - see `FanOutStats`.
+     */
+    recordFanOut(options: FanOutOptions): void;
+    /**
+     * Close the current period and queue it.
+     *
+     * Returns the window, or undefined if nothing was recorded in it.
+     */
+    roll(): Window | undefined;
+    pending(): readonly Window[];
+    /**
+     * Called by the application when part of the current period was not recorded.
+     *
+     * The flag travels in `Window.complete` and the planner reads it, because a window missing a
+     * slice of the traffic must not be scored as though it were the whole period. Nothing here
+     * decides when that happened: the application collects these windows and hands them over, and
+     * only it knows whether a collection was skipped.
+     */
+    markIncomplete(): void;
+    /**
+     * Drop the oldest `count` windows once the application has taken them.
+     *
+     * The delivering party is the client's own process, and there is no other. This library never
+     * opens a connection to us - the buffer is read with `pending`, written wherever the application
+     * writes it, and that file is what we are handed.
+     */
+    acknowledge(count: number): void;
+}
+export {};