@harperfast/harper 5.2.13 → 5.3.0-beta.1

package/dist/resources/derivedIndexRegistry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"derivedIndexRegistry.js","sourceRoot":"","sources":["../../resources/derivedIndexRegistry.ts"],"names":[],"mappings":";;AAIA,gEAqCC;AAED,kEAEC;AAGD,gEAOC;AAvDD,MAAM,aAAa,GAAG,IAAI,OAAO,EAA+B,CAAC;AACjE,MAAM,UAAU,GAAG,IAAI,OAAO,EAAwD,CAAC;AAEvF,kFAAkF;AAClF,SAAgB,0BAA0B,CACzC,UAAkB,EAClB,QAA0B,EAC1B,SAAoC;IAEpC,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC7C,IAAI,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAC3C,IAAI,CAAC,MAAM;QAAE,aAAa,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,MAAM,GAAG,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC;IACjE,KAAK,MAAM,OAAO,IAAI,kBAAkB;QAAE,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9F,IAAI,MAAgE,CAAC;IACrE,IAAI,SAAS,EAAE,CAAC;QACf,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACpC,IAAI,CAAC,MAAM;YAAE,UAAU,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,MAAM,GAAG,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC;QAC9D,KAAK,MAAM,OAAO,IAAI,kBAAkB,EAAE,CAAC;YAC1C,IAAI,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAClC,IAAI,CAAC,OAAO;gBAAE,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC;YAClD,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACzB,CAAC;IACF,CAAC;IACD,IAAI,UAAU,GAAG,IAAI,CAAC;IACtB,OAAO,GAAG,EAAE;QACX,IAAI,CAAC,UAAU;YAAE,OAAO;QACxB,UAAU,GAAG,KAAK,CAAC;QACnB,KAAK,MAAM,OAAO,IAAI,kBAAkB,EAAE,CAAC;YAC1C,MAAM,KAAK,GAAG,MAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACnC,IAAI,KAAK,KAAK,CAAC;gBAAE,MAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;iBACpC,IAAI,KAAK;gBAAE,MAAO,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;YAChD,IAAI,SAAS,IAAI,MAAM,EAAE,CAAC;gBACzB,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBACpC,MAAM,KAAK,GAAG,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;gBAChD,IAAI,OAAO,IAAI,KAAK,IAAI,CAAC;oBAAE,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;gBACpD,IAAI,OAAO,EAAE,MAAM,KAAK,CAAC;oBAAE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACnD,CAAC;QACF,CAAC;QACD,IAAI,MAAO,CAAC,IAAI,KAAK,CAAC;YAAE,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACzD,IAAI,MAAM,EAAE,IAAI,KAAK,CAAC;YAAE,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACvD,CAAC,CAAC;AACH,CAAC;AAED,SAAgB,2BAA2B,CAAC,UAAkB,EAAE,OAAe;IAC9E,OAAO,aAAa,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC;AAC7D,CAAC;AAED,0GAA0G;AAC1G,SAAgB,0BAA0B,CAAC,UAAkB,EAAE,OAAe;IAC7E,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;IACzD,IAAI,CAAC,OAAO;QAAE,OAAO;IACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5B,IAAI,MAAM;YAAE,OAAO,MAAM,CAAC;IAC3B,CAAC;AACF,CAAC"}

package/dist/resources/derivedIndexRuntime.d.ts

@@ -0,0 +1,215 @@
+import type { RocksDatabase } from '@harperfast/rocksdb-js';
+import type { Id } from './ResourceInterface.ts';
+import type { RocksTransactionLogStore } from './RocksTransactionLogStore.ts';
+export declare const DERIVED_INDEX_ACCEPTED = 1;
+export declare const DERIVED_INDEX_DEFERRED = 0;
+export declare const DERIVED_INDEX_FAILED = -1;
+export type DerivedIndexDeliveryResult = typeof DERIVED_INDEX_ACCEPTED | typeof DERIVED_INDEX_DEFERRED | typeof DERIVED_INDEX_FAILED;
+export type DerivedIndexCursor = {
+    format: 1;
+    logs: Record<string, number>;
+    coverage?: DerivedIndexPositions;
+};
+export type DerivedIndexPositions = Record<string, {
+    sequence: number;
+    offset: number;
+} | null>;
+export type DerivedIndexCoverage = {
+    state: 'current' | 'bounded' | 'unknown';
+    maxLagMilliseconds: number;
+    lagUpperBoundMilliseconds?: number;
+};
+export declare function derivedIndexTime(store: RocksDatabase): bigint;
+export type DerivedIndexState = {
+    kind: 'record';
+    version: number;
+    projection: unknown;
+} | {
+    kind: 'absent';
+}
+/** The projection rejected the record (a 4xx-classified error); the backend removes any entry and counts it. */
+ | {
+    kind: 'unindexable';
+    version: number;
+    reason: string;
+};
+export type DerivedIndexMutation = {
+    tableId: number;
+    recordId: Id;
+    logVersion: number;
+    state: DerivedIndexState;
+};
+export type DerivedIndexTransaction = {
+    logName: string;
+    timestamp: number;
+    mutations: DerivedIndexMutation[];
+};
+export type DerivedIndexBatch = {
+    ownerEpoch: bigint;
+    transactions: DerivedIndexTransaction[];
+    /**
+     * Last-write-wins view over the distinct `(tableId, writeKeyId(recordId))` keys of the batch, in
+     * first-occurrence order, each carrying the last `logVersion` and the same resolved `state`
+     * object as its occurrences in `transactions`.
+     */
+    records: DerivedIndexMutation[];
+    /**
+     * Cursor vector this batch completes. Absent on a rebuild scan chunk: such a batch advances no
+     * cursor and the backend's durable cursor must stay `undefined` until a batch carrying `through`
+     * has been made durable.
+     */
+    through?: DerivedIndexCursor;
+    bytes: number;
+    rebuild?: true;
+};
+export type DerivedIndexFlushReason = 'age' | 'threshold' | 'shutdown';
+export type DerivedIndexReadinessState = 'unknown' | 'ready' | 'rebuilding' | 'needs-rebuild' | 'unavailable';
+/**
+ * Why an index is not `ready`, as a code every worker can read from shared memory. The owner's log
+ * line carries the full message; the code is what a peer or a successor can act on.
+ */
+export type DerivedIndexReadinessReason = 'none' | 'cursor-missing' | 'cursor-unoffered' | 'log-missing' | 'log-retention' | 'log-corrupt' | 'reload' | 'backend-failed' | 'runner-failed' | 'condemned' | 'shutdown-failed' | 'rebuild-failed' | 'rebuild-exhausted' | 'rebuild-requested';
+export type DerivedIndexReadiness = {
+    state: DerivedIndexReadinessState;
+    reason?: DerivedIndexReadinessReason;
+    /** The most recently minted owner epoch; a backend fences queued work against it through `isOwnerEpoch`. */
+    ownerEpoch: bigint;
+    rebuildAttempts: number;
+};
+export interface DerivedIndexBackendHost {
+    /** True while `epoch` is the most recently minted owner epoch for this backend. */
+    isOwnerEpoch(epoch: bigint): boolean;
+    getReadiness(): DerivedIndexReadiness;
+}
+/**
+ * A backend queues expensive work and publishes durability later: `deliver()` may only enqueue, a
+ * barrier completes asynchronously, and the durable cursor trails delivery. Work that survives a
+ * method return is the safety boundary, so every backend provides the epoch fence, the barrier
+ * request and the quiescence handshake the handoff protocol needs, even one that happens to complete
+ * everything before returning.
+ */
+export interface DerivedIndexBackend {
+    readonly id: string;
+    /** Receives the epoch fence and readiness reader before any delivery. */
+    attach(host: DerivedIndexBackendHost): void;
+    getDurableCursor(): DerivedIndexCursor | undefined;
+    /** Persist coverage only after its offered cursor is durable; reset must remove it with the cursor. */
+    publishCoverage?(positions: DerivedIndexPositions, ownerEpoch: bigint): void;
+    deliver(batch: DerivedIndexBatch): DerivedIndexDeliveryResult;
+    /** Request a durability barrier; the backend completes it and wakes through `onStateChange`. */
+    flush(reason: DerivedIndexFlushReason): void | Promise<void>;
+    /**
+     * Stop accepting work for `ownerEpoch`, settle or discard what is queued, and resolve once nothing
+     * further will be applied or published for it. A rejection keeps the runner lock held.
+     */
+    shutdown(ownerEpoch: bigint): void | Promise<void>;
+    onStateChange(wake: (change?: DerivedIndexBackendStateChange) => void): () => void;
+    /**
+     * Destroy index state and the durable cursor; `getDurableCursor()` must return `undefined`
+     * afterwards. Crash safety is the backend's: its first durable action must invalidate the cursor
+     * (or the generation the cursor belongs to) before anything destructive, so an interrupted reset
+     * reopens as cursorless rather than as a valid cursor over partially destroyed state. Shared
+     * readiness is process memory and is no evidence after a restart.
+     */
+    reset?(ownerEpoch: bigint): void | Promise<void>;
+}
+export type DerivedIndexBackendStateChange = 'changed' | 'accepted-work-lost' | 'failed';
+export type DerivedIndexRunnerOptions = {
+    maxTransactionsPerTurn?: number;
+    maxBytesPerTurn?: number;
+    maxMillisecondsPerTurn?: number;
+    /** Hard bound on distinct records resolved per chunk; an oversized transaction is cut here. */
+    maxChunkRecords?: number;
+    /** Estimated payload bytes after which a chunk stops adding complete transactions. */
+    maxChunkBytes?: number;
+    maxAcceptedBatchesAhead?: number;
+    maxFlushAgeMilliseconds?: number;
+    flushAfterMutations?: number;
+    flushAfterBytes?: number;
+    rebuildBackoffMilliseconds?: number;
+    maxRebuildBackoffMilliseconds?: number;
+    maxRebuildAttempts?: number;
+    /**
+     * Cadence on which a non-owner re-tries the runner lock when no wake reaches it: the fallback for
+     * an owner that died without releasing, whose lock is released natively with nothing to notify.
+     */
+    lockRetryMilliseconds?: number;
+    /**
+     * Opt-in writer backpressure: while the index is further behind than this, user writes to its
+     * tables fail with a retryable 503 on every worker. 0 (the default) means no policy.
+     */
+    maxLagMilliseconds?: number;
+};
+export type DerivedIndexRegistration = {
+    backend: DerivedIndexBackend;
+    projections: ReadonlyMap<number, (record: unknown) => unknown>;
+    options?: DerivedIndexRunnerOptions;
+};
+/**
+ * `size` is the stored byte size of the record when known; it bounds the projection's size without
+ * serializing it. Resolve a missing, deleted or evicted record as `undefined`; a present entry is
+ * projected as-is, so an undecodable body fails closed instead of silently leaving the index.
+ */
+export type DerivedIndexRecord = {
+    version: number;
+    value: unknown;
+    size?: number;
+} | undefined;
+/** A scan record whose `value` is null or undefined is a tombstone, and one whose `recordId` is a symbol is a Harper-internal store entry; neither is indexed. */
+export type DerivedIndexScanRecord = {
+    recordId: Id;
+    version: number;
+    value: unknown;
+    size?: number;
+};
+export type DerivedIndexRuntimeOptions = DerivedIndexRunnerOptions & {
+    idleGraceMilliseconds?: number;
+    now?: () => number;
+    /** Iterates every current record of a table for the rebuild scan; without it a rebuild cannot run. */
+    scanRecords?: (tableId: number) => Iterable<DerivedIndexScanRecord>;
+};
+export type DerivedIndexRunnerStatus = {
+    state: 'idle' | 'running' | 'deferred' | 'waiting-durable' | 'stopped' | 'rebuilding';
+    ownerEpoch?: bigint;
+} | {
+    state: 'needs-rebuild' | 'unavailable';
+    reason: string;
+    ownerEpoch?: bigint;
+};
+export type DerivedIndexRunnerMetrics = {
+    readiness: DerivedIndexReadiness;
+    acceptedBatches: number;
+    acceptedBytes: number;
+    acceptedMutations: number;
+    deferredBytes: number;
+    oldestAcceptedAgeMilliseconds: number;
+    /** Lag between the latest transaction this runner has read and the durable cursor; blind while parked. */
+    cursorLagMilliseconds: number;
+    /** How long the runner has been parked on backend backpressure or the durability ceiling. */
+    stalledMilliseconds: number;
+    unindexableRecords: number;
+    rebuildAttempts: number;
+    rebuiltRecords: number;
+    /** How long the current epoch's quiescence (backend shutdown) has been pending. */
+    quiescenceAgeMilliseconds: number;
+};
+export declare const READINESS_BYTES: number;
+export declare class DerivedIndexRuntime {
+    #private;
+    constructor(logStore: RocksTransactionLogStore, resolveRecord: (tableId: number, recordId: Id) => DerivedIndexRecord, options?: DerivedIndexRuntimeOptions);
+    register(registration: DerivedIndexRegistration): () => Promise<void>;
+    wake(): void;
+    getStatus(backendId: string): DerivedIndexRunnerStatus | undefined;
+    /** Shared readiness published by whichever worker owns the index; readable on every worker. */
+    getReadiness(backendId: string): DerivedIndexReadiness;
+    getMetrics(backendId: string): DerivedIndexRunnerMetrics | undefined;
+    waitForCoverage(backendId: string, since: bigint, timeout: number, signal?: AbortSignal): Promise<void>;
+    /** Force a rebuild (or retry one that became `unavailable`). Returns false when the backend cannot be rebuilt by the runtime. */
+    requestRebuild(backendId: string): boolean;
+    /** Resolves once every runner has released ownership and its backend shutdown has settled. */
+    stop(): Promise<void>;
+}
+/** Read an index's shared readiness on any worker, without a registered runtime. */
+export declare function readDerivedIndexReadiness(logStore: RocksTransactionLogStore, backendId: string): DerivedIndexReadiness;
+export declare function sameDerivedIndexPositions(left: DerivedIndexPositions, right: DerivedIndexPositions): boolean;
+export declare function readDerivedIndexCoverage(logStore: RocksTransactionLogStore, backendId: string, loadCursor: () => DerivedIndexCursor | undefined, maxLagMilliseconds: number): DerivedIndexCoverage;