@eltjs/elt 0.0.0-stage → 0.1.0

package/dist/core/stream.js

@@ -0,0 +1,86 @@
+import { Deduplication } from './deduplication.js';
+import { FileReference } from './file-read.js';
+// A source's immutable description of a resource, without extraction behavior.
+export class Stream {
+    name;
+    jsonSchema;
+    primaryKey;
+    supportedSyncModes;
+    supportsFileTransfer;
+    // Incremental progress is opaque source state, so copies select no cursorField.
+    sourceDefinedCursor;
+    // Incremental reads may emit DELETE messages keyed by primaryKey.
+    emitsDeletes;
+    // The source reads the stream once per value of these primaryKey fields and
+    // keeps each partition's state apart; every record carries its partition.
+    partitionKey;
+    // The upstream drops records once this timestamp field passes its retention,
+    // without a deletion: a record that vanished before the read's horizon
+    // expired, and its row stays loaded.
+    expiresBy;
+    constructor({ name, jsonSchema, primaryKey = [], supportedSyncModes, supportsFileTransfer, sourceDefinedCursor, emitsDeletes, partitionKey, expiresBy, }) {
+        if (!name || name.includes('\0'))
+            throw new TypeError('Invalid stream name');
+        if (!Array.isArray(supportedSyncModes) ||
+            supportedSyncModes.length === 0 ||
+            !supportedSyncModes.every((mode) => mode === 'full_refresh' || mode === 'incremental') ||
+            new Set(supportedSyncModes).size !== supportedSyncModes.length)
+            throw new TypeError('A stream requires distinct supported sync modes');
+        this.name = name;
+        if (supportsFileTransfer !== undefined && supportsFileTransfer !== true)
+            throw new TypeError('supportsFileTransfer must be true when declared');
+        this.supportsFileTransfer = supportsFileTransfer;
+        if (sourceDefinedCursor !== undefined && sourceDefinedCursor !== true)
+            throw new TypeError('sourceDefinedCursor must be true when declared');
+        if (emitsDeletes !== undefined && emitsDeletes !== true)
+            throw new TypeError('emitsDeletes must be true when declared');
+        if ((sourceDefinedCursor || emitsDeletes) &&
+            !supportedSyncModes.includes('incremental'))
+            throw new TypeError('A source-defined cursor or deletions require incremental support');
+        if (emitsDeletes && primaryKey.length === 0)
+            throw new TypeError('A stream that emits deletions requires a primaryKey');
+        this.sourceDefinedCursor = sourceDefinedCursor;
+        this.emitsDeletes = emitsDeletes;
+        this.jsonSchema = structuredClone(jsonSchema);
+        // Snapshot nested metadata too, so changes in a source cannot rewrite a pipeline.
+        const seen = new WeakSet();
+        const freeze = (value) => {
+            if (value === null || typeof value !== 'object' || seen.has(value))
+                return;
+            seen.add(value);
+            for (const child of Object.values(value))
+                freeze(child);
+            Object.freeze(value);
+        };
+        freeze(this.jsonSchema);
+        this.primaryKey = Object.freeze([...primaryKey]);
+        this.supportedSyncModes = Object.freeze([...supportedSyncModes]);
+        if (partitionKey !== undefined) {
+            if (!Array.isArray(partitionKey) ||
+                partitionKey.length === 0 ||
+                new Set(partitionKey).size !== partitionKey.length ||
+                !partitionKey.every((field) => primaryKey.includes(field)))
+                throw new TypeError('partitionKey fields must be distinct members of primaryKey');
+            this.partitionKey = Object.freeze([...partitionKey]);
+            new Deduplication(this, this.partitionKey);
+        }
+        if (expiresBy !== undefined) {
+            const field = jsonSchema.properties?.[expiresBy];
+            if (!emitsDeletes ||
+                field === undefined ||
+                field.type !== 'string' ||
+                field.format !== 'date-time')
+                throw new TypeError('expiresBy must name a non-null date-time property of a stream that emits deletions');
+            // Snapshots compare expiry with a millisecond horizon as text.
+            if (field.precision !== undefined && field.precision !== 3)
+                throw new TypeError(`expiresBy ${expiresBy} must keep milliseconds; it declares precision ${field.precision}`);
+            this.expiresBy = expiresBy;
+        }
+        Object.freeze(this);
+    }
+    get file() {
+        if (this.supportsFileTransfer !== true)
+            throw new TypeError(`Stream ${this.name} does not support file extraction`);
+        return new FileReference(this);
+    }
+}

package/dist/core/target-description.d.ts

@@ -0,0 +1,15 @@
+import type { CopyConfiguration } from './copy-configuration.ts';
+import type { FileRead } from './file-read.ts';
+export type DescribedColumn = {
+    readonly field: string;
+    readonly name: string;
+    readonly fileRead?: FileRead;
+    readonly storesFile: boolean;
+};
+export type TargetDescription = {
+    readonly meaning: string | null;
+    readonly table: string;
+    readonly columns: Readonly<Record<string, string | null>>;
+    readonly missing: readonly string[];
+};
+export declare function describeTarget<Column extends DescribedColumn>(configuration: CopyConfiguration, columns: readonly Column[], storedFile: (column: Column) => string): TargetDescription;

package/dist/core/target-description.js

@@ -0,0 +1,73 @@
+// JSON Schema annotations are optional; absent ones describe nothing.
+function annotation(value) {
+    if (value === undefined)
+        return null;
+    if (typeof value !== 'string' ||
+        value.includes('\0') ||
+        !value.isWellFormed())
+        throw new TypeError('JSON Schema description must be well-formed text without NUL');
+    return value;
+}
+const loading = {
+    append: 'Every accepted observation is appended; source keys may repeat.',
+    overwrite: 'Each full refresh replaces the table with its accepted records.',
+    append_dedup: 'Accepted observations reconcile rows by the copy key.',
+    overwrite_dedup: 'Each full refresh replaces the table with deduplicated records.',
+};
+// What every destination tells readers about a copy's target: the stream's
+// meaning, how it is loaded, and each column from the stream's schema. Only
+// where original file bytes are kept differs by destination.
+export function describeTarget(configuration, columns, storedFile) {
+    const { stream } = configuration;
+    const meaning = annotation(stream.jsonSchema.description);
+    const lines = [`Source stream: ${stream.name}.`];
+    if (meaning !== null)
+        lines.push(`Source record meaning: ${meaning}`);
+    lines.push(`Extraction: ${configuration.syncMode}. Loading: ${configuration.destinationSyncMode}. ${loading[configuration.destinationSyncMode]}`);
+    const named = (field) => columns.find((column) => column.field === field)?.name ?? field;
+    if (configuration.dedupPolicy !== undefined) {
+        const deduplication = configuration.deduplication();
+        const { primaryKey, cursorField } = deduplication;
+        lines.push(`Copy key: ${primaryKey.map(named).join(', ')}.`);
+        lines.push(configuration.dedupPolicy === 'replace' || cursorField === undefined
+            ? 'For a repeated key, the newest extracted record wins.'
+            : `For a repeated key, the greatest ${named(cursorField)} wins; equal cursors retain the first accepted record. Text cursors compare ${textOrdering(deduplication.format(cursorField))}.`);
+    }
+    const { properties } = stream.jsonSchema;
+    const missing = meaning === null ? ['the stream'] : [];
+    const describe = (column) => {
+        if (column.storesFile)
+            return storedFile(column);
+        if (column.fileRead?.parser !== undefined)
+            return `Text extracted from the source file by parser ${column.fileRead.parser.identity}. NULL when the source file is unavailable or the parser returns no text.`;
+        if (column.fileRead?.file.storage !== undefined)
+            return `${column.fileRead.file.storage.reference} NULL when the source file is unavailable.`;
+        const description = annotation(properties?.[column.field]?.description);
+        if (description === null)
+            missing.push(column.field);
+        return description;
+    };
+    const described = Object.fromEntries(columns.map((column) => {
+        const description = describe(column);
+        if (column.name === column.field)
+            return [column.name, description];
+        const source = `Source field: ${JSON.stringify(column.field)}.`;
+        return [
+            column.name,
+            description === null ? source : `${description} ${source}`,
+        ];
+    }));
+    described.loaded_at =
+        'Start time of the load that last wrote this row, not the source modification time or the most recent successful sync.';
+    return Object.freeze({
+        meaning,
+        table: lines.join('\n'),
+        columns: Object.freeze(described),
+        missing: Object.freeze(missing),
+    });
+}
+// How a text cursor compares: as its declared format orders it, or by byte
+// order, as a plain string compares, when it declares none.
+function textOrdering(format) {
+    return format === null ? 'by byte order' : format.ordering;
+}

package/dist/core/target.d.ts

@@ -0,0 +1,5 @@
+import { FileRead } from './file-read.ts';
+export declare abstract class Target {
+    readonly fileReads: readonly FileRead[];
+    constructor(fileReads?: readonly FileRead[]);
+}

package/dist/core/target.js

@@ -0,0 +1,14 @@
+import { FileRead } from './file-read.js';
+// A destination declaration supplies its source requirements without doing I/O.
+export class Target {
+    fileReads;
+    constructor(fileReads = []) {
+        if (!Array.isArray(fileReads) ||
+            !fileReads.every((read) => read instanceof FileRead))
+            throw new TypeError('Targets require FileRead declarations');
+        const names = fileReads.map((read) => read.name.toLowerCase());
+        if (new Set(names).size !== names.length)
+            throw new TypeError('Duplicate file field names');
+        this.fileReads = Object.freeze([...fileReads]);
+    }
+}

package/dist/core/upkeep.d.ts

@@ -0,0 +1,4 @@
+export type Upkeep = {
+    readonly interval: number;
+    readonly run: (signal: AbortSignal) => Promise<void>;
+};

package/dist/core/upkeep.js

@@ -0,0 +1 @@
+export {};

package/dist/core/writer.d.ts

@@ -0,0 +1,41 @@
+import type { Partition } from './partition.ts';
+import type { FailureType, KeyValue } from './source.ts';
+import type { Stream } from './stream.ts';
+export type WriteCount = {
+    readonly count: number;
+    readonly deleted: number;
+};
+export declare class TargetOwnedError extends TypeError {
+    name: string;
+    constructor(target: string, owner: string, writer: string);
+}
+export type LoadFailure = {
+    readonly partition: Partition | null;
+    readonly error: unknown;
+    readonly failureType: FailureType;
+};
+export type WriteOperation = {
+    readonly type: 'RECORD';
+    readonly data: unknown;
+} | {
+    readonly type: 'DELETE';
+    readonly key: Readonly<Record<string, KeyValue>>;
+} | {
+    readonly type: 'RESET';
+    readonly partition: Partition | null;
+};
+export type FieldValues = (field: string) => AsyncIterable<unknown>;
+export type Stage = AsyncDisposable & {
+    readonly fresh: boolean;
+    readonly reloading: boolean;
+    complete(): Promise<void>;
+    values: FieldValues;
+    apply(operation: WriteOperation): Promise<void>;
+    commit(): Promise<void>;
+    discard(): Promise<void>;
+};
+export declare abstract class Writer {
+    readonly stream: Stream;
+    constructor(stream: Stream);
+    abstract clear(writer: string, committed?: (values: FieldValues) => Promise<void>): Promise<void>;
+}

package/dist/core/writer.js

@@ -0,0 +1,17 @@
+// A target has one writer, as in Airbyte: an overwrite replaces the whole
+// target and a snapshot deletes keys it once saw, so a second writer's rows
+// would be lost. Clearing the target releases it.
+export class TargetOwnedError extends TypeError {
+    name = 'TargetOwnedError';
+    constructor(target, owner, writer) {
+        super(`Target ${target} is written by ${owner}; ${writer} cannot write it. A target has one writer; clear the owning copy, or drop the target, to reassign it.`);
+    }
+}
+// A destination's loading strategy for one stream; its Load prepares the
+// stream's Stage.
+export class Writer {
+    stream;
+    constructor(stream) {
+        this.stream = stream;
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,40 @@
+export { Catalog } from './core/catalog.ts';
+export type { PipelineClock } from './core/clock.ts';
+export { Connection } from './core/connection.ts';
+export { Copy, type CopyOutcome, type CopyResult } from './core/copy.ts';
+export { CopyConfiguration, type DedupPolicy, } from './core/copy-configuration.ts';
+export type { Deduplication } from './core/deduplication.ts';
+export { Destination, type DestinationSyncMode, type Load, } from './core/destination.ts';
+export { DerivedDataset, type RefreshTrigger } from './core/derived-dataset.ts';
+export { DocumentParser } from './core/document-parser.ts';
+export { DocumentText } from './core/document-text.ts';
+export { type LoadedRow, Rule, type RuleColumn, type RuleResult, } from './core/rule.ts';
+export { CopyProgress, PassFailed, PassFinished, PassStarted, passChannels, } from './core/pass-channels.ts';
+export { FileContent } from './core/file-content.ts';
+export { fileExtension } from './core/file-extension.ts';
+export { FileRead, FileReference } from './core/file-read.ts';
+export { FileStorage } from './core/file-storage.ts';
+export { isCalendarDate } from './core/formats/calendar-date-format.ts';
+export { type DeclaredFormat, declaredFormat, } from './core/formats/declared-format.ts';
+export { Int64Format } from './core/formats/int64-format.ts';
+export { LocalTimestampFormat } from './core/formats/local-timestamp-format.ts';
+export { isTimestamp } from './core/formats/timestamp-format.ts';
+export { Identifiers } from './core/identifiers.ts';
+export type { Partition, PartitionState } from './core/partition.ts';
+export { type Pass, Pipeline, PipelineError, WatchError, } from './core/pipeline.ts';
+export { type ReaderRelation, readerCatalog } from './core/reader-relation.ts';
+export { type ReloadMode, type StoredFit, reloadMode, } from './core/reload-mode.ts';
+export { StreamChangeError } from './core/stream-change.ts';
+export { type FieldSchema, type Properties, type RecordDraft, type SchemaRecord, type StreamSchema, validateRecords, } from './core/record-validation.ts';
+export { diffGroupedSnapshot, diffSnapshot, type KeptGroup, type SnapshotDiffOptions, } from './core/snapshot.ts';
+export { type DeleteMessage, type ExtractionCoverage, type FailureType, type JsonValue, type KeyValue, type ReadMessage, type RecordMessage, type ResetMessage, Source, type SourceMessage, type SourceReadOptions, type StateMessage, StreamStatus, } from './core/source.ts';
+export { type ChangeSignal, type Check, Gauge } from './core/change-signal.ts';
+export { type Upkeep } from './core/upkeep.ts';
+export { Stream, type SyncMode } from './core/stream.ts';
+export { Target } from './core/target.ts';
+export { type DescribedColumn, describeTarget, type TargetDescription, } from './core/target-description.ts';
+export { type FieldValues, type LoadFailure, type Stage, TargetOwnedError, type WriteCount, type WriteOperation, Writer, } from './core/writer.ts';
+export { type CheckpointBinding, type CheckpointRun, type CheckpointSession, CheckpointStore, type StoredCheckpoint, } from './state/checkpoint-store.ts';
+export { copyStatus, type DeclaredCopy, errorMessage, passError, passFailureType, passStatus, type RecordedPass, recordedFailure, type StreamCheck, SyncHistory, type SyncStatus, } from './state/sync-history.ts';
+export { syncHistoryRelations } from './state/sync-history-relations.ts';
+export { LocalFiles } from './storage/local-files.ts';

package/dist/index.js

@@ -0,0 +1,36 @@
+export { Catalog } from './core/catalog.js';
+export { Connection } from './core/connection.js';
+export { Copy } from './core/copy.js';
+export { CopyConfiguration, } from './core/copy-configuration.js';
+export { Destination, } from './core/destination.js';
+export { DerivedDataset } from './core/derived-dataset.js';
+export { DocumentParser } from './core/document-parser.js';
+export { DocumentText } from './core/document-text.js';
+export { Rule, } from './core/rule.js';
+export { CopyProgress, PassFailed, PassFinished, PassStarted, passChannels, } from './core/pass-channels.js';
+export { FileContent } from './core/file-content.js';
+export { fileExtension } from './core/file-extension.js';
+export { FileRead, FileReference } from './core/file-read.js';
+export { FileStorage } from './core/file-storage.js';
+export { isCalendarDate } from './core/formats/calendar-date-format.js';
+export { declaredFormat, } from './core/formats/declared-format.js';
+export { Int64Format } from './core/formats/int64-format.js';
+export { LocalTimestampFormat } from './core/formats/local-timestamp-format.js';
+export { isTimestamp } from './core/formats/timestamp-format.js';
+export { Identifiers } from './core/identifiers.js';
+export { Pipeline, PipelineError, WatchError, } from './core/pipeline.js';
+export { readerCatalog } from './core/reader-relation.js';
+export { reloadMode, } from './core/reload-mode.js';
+export { StreamChangeError } from './core/stream-change.js';
+export { validateRecords, } from './core/record-validation.js';
+export { diffGroupedSnapshot, diffSnapshot, } from './core/snapshot.js';
+export { Source, StreamStatus, } from './core/source.js';
+export { Gauge } from './core/change-signal.js';
+export { Stream } from './core/stream.js';
+export { Target } from './core/target.js';
+export { describeTarget, } from './core/target-description.js';
+export { TargetOwnedError, Writer, } from './core/writer.js';
+export { CheckpointStore, } from './state/checkpoint-store.js';
+export { copyStatus, errorMessage, passError, passFailureType, passStatus, recordedFailure, SyncHistory, } from './state/sync-history.js';
+export { syncHistoryRelations } from './state/sync-history-relations.js';
+export { LocalFiles } from './storage/local-files.js';

package/dist/state/checkpoint-store.d.ts

@@ -0,0 +1,29 @@
+export type StoredCheckpoint = {
+    readonly binding: string;
+    readonly reloading: boolean;
+};
+export type CheckpointSession = {
+    read(id: string): Promise<StoredCheckpoint | undefined>;
+    state(id: string): Promise<string | undefined>;
+    save(id: string, checkpoint: StoredCheckpoint & {
+        readonly state: string;
+    }): Promise<void>;
+    reloaded(id: string): Promise<void>;
+    remove(id: string): Promise<void>;
+};
+export type CheckpointBinding = {
+    readonly copy: object;
+    readonly shape: object;
+};
+export type CheckpointRun = {
+    state(id: string): Promise<unknown>;
+    reloading(id: string): boolean;
+    save(id: string, state: string, reloading: boolean): Promise<void>;
+    reloaded(id: string): Promise<void>;
+};
+export declare abstract class CheckpointStore {
+    run<T>(bindings: ReadonlyMap<string, CheckpointBinding>, work: (run: CheckpointRun) => Promise<T>): Promise<T>;
+    reset(id: string): Promise<void>;
+    clear(id: string, drop: () => Promise<void>): Promise<void>;
+    protected abstract session<T>(ids: readonly string[], work: (session: CheckpointSession) => Promise<T>): Promise<T>;
+}

package/dist/state/checkpoint-store.js

@@ -0,0 +1,81 @@
+import { isDeepStrictEqual } from 'node:util';
+import { StreamChangeError, changedDeclarations, } from '../core/stream-change.js';
+// Checkpoints belong to a replication ID, not to a shared Stream or warehouse
+// table. As Airbyte's platform keeps state apart from any destination, the
+// orchestrator keeps them here, and saves each one the destination
+// acknowledged, so a run that fails later resumes from its last commit.
+export class CheckpointStore {
+    // Holds every replication's lock for the whole run.
+    async run(bindings, work) {
+        const ids = [...bindings.keys()].sort();
+        return this.session(ids, async (session) => {
+            const checkpoints = new Map();
+            for (const [id, { copy, shape }] of bindings) {
+                const binding = JSON.stringify([copy, shape]);
+                const saved = await session.read(id);
+                const [savedCopy, savedShape] = saved === undefined ? [] : JSON.parse(saved.binding);
+                const [currentCopy, currentShape] = JSON.parse(binding);
+                let refused;
+                if (saved !== undefined) {
+                    const declarations = changedDeclarations(savedShape, currentShape);
+                    if (!isDeepStrictEqual(savedCopy, currentCopy))
+                        refused = new TypeError(`Checkpoint binding changed for ${id}; reset it or use a new copy ID`);
+                    else if (declarations.length > 0)
+                        refused = new StreamChangeError(id, declarations);
+                }
+                checkpoints.set(id, {
+                    binding,
+                    reloading: saved?.reloading ?? false,
+                    refused,
+                });
+            }
+            const checkpoint = (id) => {
+                const found = checkpoints.get(id);
+                if (found === undefined)
+                    throw new TypeError(`Checkpoint ${id} is not part of this run`);
+                if (found.refused !== undefined)
+                    throw found.refused;
+                return found;
+            };
+            const durably = async (id, save) => {
+                try {
+                    await save();
+                }
+                catch (cause) {
+                    throw new Error(`Checkpoint ${id} was not saved after the destination committed; the next run replays from the last saved checkpoint`, { cause });
+                }
+            };
+            return work({
+                state: async (id) => {
+                    checkpoint(id);
+                    const state = await session.state(id);
+                    return state === undefined ? null : JSON.parse(state);
+                },
+                reloading: (id) => checkpoint(id).reloading,
+                save: (id, state, reloading) => {
+                    const { binding } = checkpoint(id);
+                    return durably(id, () => session.save(id, { binding, reloading, state }));
+                },
+                reloaded: (id) => {
+                    checkpoint(id);
+                    return durably(id, () => session.reloaded(id));
+                },
+            });
+        });
+    }
+    // Forgets progress but keeps the loaded rows: the next run reloads
+    // everything, as Airbyte's refresh that keeps records. It also accepts a
+    // stream whose declarations changed, keeping its rows under them.
+    async reset(id) {
+        await this.session([id], (session) => session.remove(id));
+    }
+    // Airbyte's Clear: removes the data, then the checkpoint, under the
+    // replication's lock. A crash between the two leaves a checkpoint beside an
+    // emptied target, and clear can simply run again.
+    async clear(id, drop) {
+        await this.session([id], async (session) => {
+            await drop();
+            await session.remove(id);
+        });
+    }
+}

package/dist/state/sync-history-relations.d.ts

@@ -0,0 +1,72 @@
+export declare const syncHistoryRelations: {
+    readonly sync_attempts: {
+        readonly name: 'sync_attempts';
+        readonly description: 'One row per pass of a connection. A run passes every stream the connection selected; a watch pass reads only the streams whose change signals changed, so an attempt vouches only for its own extraction_coverage rows. History is retained, including failures and unfinished attempts. A successful pass may write zero rows. A connection that stopped, a failed change signal or a failed upkeep records a failed attempt without outcomes. Setup/authentication before the pipeline is constructed is not observed. This records extraction/load outcomes, not subsequent mart publication.';
+        readonly columns: {
+            attempt_id: string;
+            connector: string;
+            source: string;
+            started_at: string;
+            completed_at: string;
+            status: string;
+            error: string;
+            failure_type: string;
+        };
+    };
+    readonly extraction_coverage: {
+        readonly name: 'extraction_coverage';
+        readonly description: 'One row per selected stream per attempt, declared before extraction even when there are no records. Description and selection describe the requested export scope, not observed record dates or proof of upstream completeness. Only succeeded confirms that this copy completed that declared pass; partial/failed/running never establish complete coverage. Earlier successful declarations remain available. Raw target names identify storage, not reader grants; discover readable content in catalog.';
+        readonly columns: {
+            readonly attempt_id: string;
+            readonly connector: string;
+            readonly source: string;
+            readonly started_at: string;
+            readonly completed_at: 'Time the whole attempt was recorded as complete; NULL while unfinished. The copy may have finished earlier. Not record modification time or loaded_at.';
+            readonly stream: 'Source stream name, unique within one attempt.';
+            readonly target_schema: 'Schema of the raw destination table (main in SQLite); this metadata does not grant access to it.';
+            readonly target_table: 'Raw destination table name within target_schema.';
+            readonly target_exists: 'Whether the raw relation exists now, independent of this historical attempt. Existence does not prove successful extraction, current contents, or reader access.';
+            readonly sync_mode: 'Requested source mode: incremental or full_refresh. Incremental copies can resume saved state; the selection describes their configured scope, not every request made on this pass.';
+            readonly destination_sync_mode: 'Requested load mode, such as append_dedup. Partial passes can leave previously committed changes in the destination.';
+            readonly description: 'Connector-owned explanation of the scope, date boundaries, selection keys and source limitations for this stream.';
+            readonly selection: 'Structured configured selection, interpreted using description. Not computed from rows. Empty object means no additional configured selection, not unlimited upstream history.';
+            readonly status: 'running: no outcome recorded; succeeded: copy completed with no failures, even with zero changes; partial: failures after committed writes/deletes; failed: failures without committed row changes, or an error without outcomes; cancelled: the run was stopped on request before this copy ended, keeping what it committed. Consult failures for affected partitions.';
+            readonly written_count: 'Accepted record operations committed by this copy during this pass, including deduplication no-ops. Not changed-row or total-record counts. Zero is valid after success. NULL means no counts were reported.';
+            readonly deleted_count: 'Accepted deletion operations committed by this copy during this pass, including already-absent keys. Not a count of rows actually removed. NULL means no counts were reported.';
+            readonly failures: 'Array of {partition, error, failure_type, stacktrace} from pass outcomes. A null partition denotes a whole-stream or non-partition-specific failure; failure_type is config or system, as in sync_attempts. error is the message; stacktrace is the error with its stack and every cause it chains, such as the code of a refused connection behind "fetch failed". Empty array means none recorded; check status before assuming success.';
+        };
+    };
+    readonly sync_status: {
+        readonly name: 'sync_status';
+        readonly description: 'One row per connector with its latest started attempt, independently its most recently completed successful pass, and how current its data is. A watch pass covers only the streams that changed; see stream_status for each stream. A later failed, partial or unfinished pass does not erase prior success. Join extraction_coverage using the appropriate attempt ID: requested scope can change. NULL success means none recorded. Judge freshness by current_as_of: last_successful_sync_at measures sync completion, not service liveness or source completeness.';
+        readonly columns: {
+            readonly connector: string;
+            readonly latest_attempt_id: 'Most recently started attempt for this connector; join sync_attempts or extraction_coverage by attempt_id.';
+            readonly started_at: string;
+            readonly completed_at: string;
+            readonly status: string;
+            readonly error: string;
+            readonly failure_type: string;
+            readonly last_successful_attempt_id: 'Most recently completed all-copies-successful attempt; NULL if none. Its coverage may differ from the latest attempt.';
+            readonly last_successful_sync_at: 'Completion time of that successful pass, advanced even if no rows changed. NULL if no successful pass was recorded. Never inferred from loaded_at or record modification dates.';
+            readonly current_as_of: "The oldest current_as_of among the connector's streams in stream_status: everything it loaded matched its upstream at least as of this time. NULL until every stream has loaded once. An old value means its data is that old: no pass or watch has refreshed it since, a change failed to load, or the app that writes its upstream has not written it since, as WhatsApp while it is closed.";
+        };
+    };
+    readonly stream_status: {
+        readonly name: 'stream_status';
+        readonly description: 'One row per connector and stream: the latest attempt that declared the stream, independently the last attempt in which its copy succeeded, and how current the stream is. A watch pass reads only the streams its source reported changed, so a connector can succeed in sync_status while one of its streams last failed, and an unchanged stream keeps an old last_successful_sync_at while its watch keeps it current, up to when its source says its upstream was last current; judge a stream here, by current_as_of. NULL success means that stream never completed.';
+        readonly columns: {
+            readonly connector: string;
+            readonly stream: 'Source stream name.';
+            readonly target_schema: 'Schema of the raw destination table in the latest declaration (main in SQLite); this metadata does not grant access to it.';
+            readonly target_table: 'Raw destination table name within target_schema.';
+            readonly latest_attempt_id: 'Most recently started attempt that declared this stream; join extraction_coverage by attempt_id and stream.';
+            readonly started_at: string;
+            readonly completed_at: string;
+            readonly status: "This stream's copy status in the latest attempt: running, succeeded, partial, failed or cancelled. Other streams of the same attempt may differ.";
+            readonly last_successful_attempt_id: "Most recent completed attempt in which this stream's copy succeeded; NULL if none.";
+            readonly last_successful_sync_at: 'Completion time of that attempt, advanced even if no rows changed. NULL if this stream never succeeded. Never inferred from loaded_at or record modification dates.';
+            readonly current_as_of: 'When the stream last matched its upstream: when its last successful pass began reading, or a later check by a watch that found the stream unchanged since, whichever is later, and never later than when its source says the upstream itself was last current, as an app writes its store only while it runs. A watch records its checks about once a minute. NULL if the stream never succeeded. A stream whose latest change failed to load keeps the time before that change, so its age is how stale it is; compare it with the current time to judge freshness.';
+        };
+    };
+};

package/dist/state/sync-history-relations.js

@@ -0,0 +1,74 @@
+const attempt = {
+    attempt_id: 'Pass identifier. Join to extraction_coverage.attempt_id for the exact declarations and copy outcomes of this pass.',
+    connector: 'Connection name declared by the application. Each attempt is one pass: one read of the streams the connection selected.',
+    source: 'Source identity, not credentials or a record identifier.',
+    started_at: 'Database time when the attempt and its declared coverage were recorded, before the pass read anything. Not a source record modification or row load time.',
+    completed_at: 'Database time when the pass outcomes were recorded. NULL means no completion was recorded; this is not evidence that a process is alive.',
+    status: 'running: no completion recorded; succeeded: all selected copies completed, including empty or unchanged reads; partial: failures with some successful copies or committed writes/deletes; failed: failures without that progress; cancelled: the run was stopped on request, and error says why; what it committed before stays. No health of a watch is implied.',
+    error: 'Pipeline error message, or NULL after success or before completion. Per-copy failures and partitions are in extraction_coverage.',
+    failure_type: "Whose the failure is to fix, as the source classified it: config when at least one failure needs the user to change what they set up or granted, such as macOS access to the app's store; system for every other failure, which the user cannot fix by granting or configuring. NULL after success, before completion, or when cancelled.",
+};
+// How readers see the sync history: what each pass declared and loaded.
+export const syncHistoryRelations = {
+    sync_attempts: {
+        name: 'sync_attempts',
+        description: 'One row per pass of a connection. A run passes every stream the connection selected; a watch pass reads only the streams whose change signals changed, so an attempt vouches only for its own extraction_coverage rows. History is retained, including failures and unfinished attempts. A successful pass may write zero rows. A connection that stopped, a failed change signal or a failed upkeep records a failed attempt without outcomes. Setup/authentication before the pipeline is constructed is not observed. This records extraction/load outcomes, not subsequent mart publication.',
+        columns: attempt,
+    },
+    extraction_coverage: {
+        name: 'extraction_coverage',
+        description: 'One row per selected stream per attempt, declared before extraction even when there are no records. Description and selection describe the requested export scope, not observed record dates or proof of upstream completeness. Only succeeded confirms that this copy completed that declared pass; partial/failed/running never establish complete coverage. Earlier successful declarations remain available. Raw target names identify storage, not reader grants; discover readable content in catalog.',
+        columns: {
+            attempt_id: attempt.attempt_id,
+            connector: attempt.connector,
+            source: attempt.source,
+            started_at: attempt.started_at,
+            completed_at: 'Time the whole attempt was recorded as complete; NULL while unfinished. The copy may have finished earlier. Not record modification time or loaded_at.',
+            stream: 'Source stream name, unique within one attempt.',
+            target_schema: 'Schema of the raw destination table (main in SQLite); this metadata does not grant access to it.',
+            target_table: 'Raw destination table name within target_schema.',
+            target_exists: 'Whether the raw relation exists now, independent of this historical attempt. Existence does not prove successful extraction, current contents, or reader access.',
+            sync_mode: 'Requested source mode: incremental or full_refresh. Incremental copies can resume saved state; the selection describes their configured scope, not every request made on this pass.',
+            destination_sync_mode: 'Requested load mode, such as append_dedup. Partial passes can leave previously committed changes in the destination.',
+            description: 'Connector-owned explanation of the scope, date boundaries, selection keys and source limitations for this stream.',
+            selection: 'Structured configured selection, interpreted using description. Not computed from rows. Empty object means no additional configured selection, not unlimited upstream history.',
+            status: 'running: no outcome recorded; succeeded: copy completed with no failures, even with zero changes; partial: failures after committed writes/deletes; failed: failures without committed row changes, or an error without outcomes; cancelled: the run was stopped on request before this copy ended, keeping what it committed. Consult failures for affected partitions.',
+            written_count: 'Accepted record operations committed by this copy during this pass, including deduplication no-ops. Not changed-row or total-record counts. Zero is valid after success. NULL means no counts were reported.',
+            deleted_count: 'Accepted deletion operations committed by this copy during this pass, including already-absent keys. Not a count of rows actually removed. NULL means no counts were reported.',
+            failures: 'Array of {partition, error, failure_type, stacktrace} from pass outcomes. A null partition denotes a whole-stream or non-partition-specific failure; failure_type is config or system, as in sync_attempts. error is the message; stacktrace is the error with its stack and every cause it chains, such as the code of a refused connection behind "fetch failed". Empty array means none recorded; check status before assuming success.',
+        },
+    },
+    sync_status: {
+        name: 'sync_status',
+        description: 'One row per connector with its latest started attempt, independently its most recently completed successful pass, and how current its data is. A watch pass covers only the streams that changed; see stream_status for each stream. A later failed, partial or unfinished pass does not erase prior success. Join extraction_coverage using the appropriate attempt ID: requested scope can change. NULL success means none recorded. Judge freshness by current_as_of: last_successful_sync_at measures sync completion, not service liveness or source completeness.',
+        columns: {
+            connector: attempt.connector,
+            latest_attempt_id: 'Most recently started attempt for this connector; join sync_attempts or extraction_coverage by attempt_id.',
+            started_at: attempt.started_at,
+            completed_at: attempt.completed_at,
+            status: attempt.status,
+            error: attempt.error,
+            failure_type: attempt.failure_type,
+            last_successful_attempt_id: 'Most recently completed all-copies-successful attempt; NULL if none. Its coverage may differ from the latest attempt.',
+            last_successful_sync_at: 'Completion time of that successful pass, advanced even if no rows changed. NULL if no successful pass was recorded. Never inferred from loaded_at or record modification dates.',
+            current_as_of: "The oldest current_as_of among the connector's streams in stream_status: everything it loaded matched its upstream at least as of this time. NULL until every stream has loaded once. An old value means its data is that old: no pass or watch has refreshed it since, a change failed to load, or the app that writes its upstream has not written it since, as WhatsApp while it is closed.",
+        },
+    },
+    stream_status: {
+        name: 'stream_status',
+        description: 'One row per connector and stream: the latest attempt that declared the stream, independently the last attempt in which its copy succeeded, and how current the stream is. A watch pass reads only the streams its source reported changed, so a connector can succeed in sync_status while one of its streams last failed, and an unchanged stream keeps an old last_successful_sync_at while its watch keeps it current, up to when its source says its upstream was last current; judge a stream here, by current_as_of. NULL success means that stream never completed.',
+        columns: {
+            connector: attempt.connector,
+            stream: 'Source stream name.',
+            target_schema: 'Schema of the raw destination table in the latest declaration (main in SQLite); this metadata does not grant access to it.',
+            target_table: 'Raw destination table name within target_schema.',
+            latest_attempt_id: 'Most recently started attempt that declared this stream; join extraction_coverage by attempt_id and stream.',
+            started_at: attempt.started_at,
+            completed_at: attempt.completed_at,
+            status: "This stream's copy status in the latest attempt: running, succeeded, partial, failed or cancelled. Other streams of the same attempt may differ.",
+            last_successful_attempt_id: "Most recent completed attempt in which this stream's copy succeeded; NULL if none.",
+            last_successful_sync_at: 'Completion time of that attempt, advanced even if no rows changed. NULL if this stream never succeeded. Never inferred from loaded_at or record modification dates.',
+            current_as_of: 'When the stream last matched its upstream: when its last successful pass began reading, or a later check by a watch that found the stream unchanged since, whichever is later, and never later than when its source says the upstream itself was last current, as an app writes its store only while it runs. A watch records its checks about once a minute. NULL if the stream never succeeded. A stream whose latest change failed to load keeps the time before that change, so its age is how stale it is; compare it with the current time to judge freshness.',
+        },
+    },
+};