@eltjs/elt 0.0.0-stage → 0.1.0

package/dist/core/catalog.d.ts

@@ -0,0 +1,7 @@
+import { Stream } from './stream.ts';
+export declare class Catalog {
+    readonly streams: readonly Stream[];
+    constructor(streams: Iterable<Stream>);
+    get(name: string): Stream;
+    select(streams: Iterable<Stream>): Catalog;
+}

package/dist/core/catalog.js

@@ -0,0 +1,23 @@
+import { Stream } from './stream.js';
+// Available streams and their logical schemas. A selection is another catalog.
+export class Catalog {
+    streams;
+    constructor(streams) {
+        this.streams = Object.freeze(Array.from(streams));
+        if (!this.streams.every((stream) => stream instanceof Stream))
+            throw new TypeError('A catalog requires stream descriptions');
+        if (new Set(this.streams.map((stream) => stream.name)).size !==
+            this.streams.length)
+            throw new TypeError('A catalog cannot contain duplicate stream names');
+        Object.freeze(this);
+    }
+    get(name) {
+        const stream = this.streams.find((stream) => stream.name === name);
+        if (!stream)
+            throw new TypeError(`Unknown stream: ${name}`);
+        return stream;
+    }
+    select(streams) {
+        return new Catalog(Array.from(streams, (stream) => this.get(stream.name)));
+    }
+}

package/dist/core/change-signal.d.ts

@@ -0,0 +1,9 @@
+import type { Gauge } from '@zukhruf/async';
+import type { Stream } from './stream.ts';
+export { type Check, Gauge } from '@zukhruf/async';
+export type ChangeSignal = {
+    readonly gauge: Gauge;
+    readonly streams: readonly Stream[];
+    readonly interval: number;
+    readonly timeout: number;
+};

package/dist/core/change-signal.js

@@ -0,0 +1,4 @@
+// A change signal names a gauge, so elt exports the gauge and what its read
+// takes: a source that depends on elt alone builds its signals with the one
+// Gauge class that the pipeline reads.
+export { Gauge } from '@zukhruf/async';

package/dist/core/clock.d.ts

@@ -0,0 +1,6 @@
+import type { Clock } from '@zukhruf/async';
+export type PipelineClock = Clock & {
+    readonly Date: {
+        now(): number;
+    };
+};

package/dist/core/clock.js

@@ -0,0 +1 @@
+export {};

package/dist/core/connection.d.ts

@@ -0,0 +1,20 @@
+import type { CheckpointStore } from '../state/checkpoint-store.ts';
+import { Copy } from './copy.ts';
+import type { Destination } from './destination.ts';
+import type { Source } from './source.ts';
+import type { Target as DestinationTarget } from './target.ts';
+export declare class Connection<Target extends DestinationTarget> {
+    readonly name: string;
+    readonly source: Source;
+    readonly destination: Destination<Target>;
+    readonly checkpoints?: CheckpointStore;
+    readonly steps: readonly Copy<Target>[];
+    constructor({ name, source, destination, steps, checkpoints, }: {
+        name: string;
+        source: Source;
+        destination: Destination<Target>;
+        steps: readonly Copy<NoInfer<Target>>[];
+        checkpoints?: CheckpointStore;
+    });
+    validate(): void;
+}

package/dist/core/connection.js

@@ -0,0 +1,31 @@
+import { Copy } from './copy.js';
+// Airbyte's connection: one source's copies into one destination, with the
+// checkpoints that resume them. Its name is what readers of sync history see.
+export class Connection {
+    name;
+    source;
+    destination;
+    checkpoints;
+    steps;
+    constructor({ name, source, destination, steps, checkpoints, }) {
+        if (!name.trim() || name.includes('\0'))
+            throw new TypeError('A connection requires a name');
+        if (!steps.every((step) => step instanceof Copy))
+            throw new TypeError('Connection steps must be Copy declarations');
+        this.name = name;
+        this.source = source;
+        this.destination = destination;
+        this.checkpoints = checkpoints;
+        this.steps = Object.freeze([...steps]);
+        Object.freeze(this);
+    }
+    validate() {
+        // One read routes messages and states by stream, as Airbyte's configured
+        // catalog lists each stream once.
+        const streams = this.steps.map((copy) => copy.from.name);
+        if (new Set(streams).size !== streams.length)
+            throw new TypeError(`Connection ${this.name} copies each stream once`);
+        for (const copy of this.steps)
+            copy.validate(this.source, this.destination, this.checkpoints);
+    }
+}

package/dist/core/copy-configuration.d.ts

@@ -0,0 +1,24 @@
+import { Deduplication } from './deduplication.ts';
+import type { DestinationSyncMode } from './destination.ts';
+import { FileRead } from './file-read.ts';
+import { Stream, type SyncMode } from './stream.ts';
+export type DedupPolicy = 'cursor_newer' | 'replace';
+export declare class CopyConfiguration {
+    readonly stream: Stream;
+    readonly fileReads: readonly FileRead[];
+    readonly syncMode: SyncMode;
+    readonly destinationSyncMode: DestinationSyncMode;
+    readonly cursorField?: string;
+    readonly primaryKey?: readonly string[];
+    readonly dedupPolicy?: DedupPolicy;
+    constructor(from: Stream, { syncMode, destinationSyncMode, cursorField, primaryKey, dedupPolicy, }: {
+        syncMode: SyncMode;
+        destinationSyncMode: DestinationSyncMode;
+        cursorField?: string;
+        primaryKey?: readonly string[];
+        dedupPolicy?: DedupPolicy;
+    }, fileReads?: readonly FileRead[]);
+    validate(stream: Stream): void;
+    validateSelection(): void;
+    deduplication(): Deduplication;
+}

package/dist/core/copy-configuration.js

@@ -0,0 +1,120 @@
+import { Deduplication } from './deduplication.js';
+import { FileRead } from './file-read.js';
+import { Stream } from './stream.js';
+// A copy's immutable selection, separate from discovery metadata and runtime state.
+export class CopyConfiguration {
+    stream;
+    fileReads;
+    syncMode;
+    destinationSyncMode;
+    cursorField;
+    primaryKey;
+    // Set only for deduplicating loads; cursor_newer unless the copy selects replace.
+    dedupPolicy;
+    constructor(from, { syncMode, destinationSyncMode, cursorField, primaryKey, dedupPolicy, }, fileReads = []) {
+        if (!(from instanceof Stream))
+            throw new TypeError('Copy requires a stream description');
+        if (!Array.isArray(fileReads) ||
+            !fileReads.every((read) => read instanceof FileRead))
+            throw new TypeError('Copy requires FileRead declarations');
+        if (new Set(fileReads.map((read) => read.name.toLowerCase())).size !==
+            fileReads.length)
+            throw new TypeError('Duplicate file field names');
+        if (typeof syncMode !== 'string' || typeof destinationSyncMode !== 'string')
+            throw new TypeError('Both syncMode and destinationSyncMode are required');
+        if (primaryKey !== undefined && !Array.isArray(primaryKey))
+            throw new TypeError('primaryKey must be an array of field names');
+        // As Airbyte's source_defined_primary_key, a key the stream declares is
+        // the key; a copy selects one only for a stream without.
+        if (primaryKey !== undefined && from.primaryKey.length > 0)
+            throw new TypeError(`Stream ${from.name} defines its own primary key; omit primaryKey`);
+        const deduplicating = destinationSyncMode === 'append_dedup' ||
+            destinationSyncMode === 'overwrite_dedup';
+        this.stream = from;
+        this.fileReads = Object.freeze([...fileReads]);
+        this.syncMode = syncMode;
+        this.destinationSyncMode = destinationSyncMode;
+        this.cursorField = cursorField;
+        const key = deduplicating && from.primaryKey.length > 0
+            ? from.primaryKey
+            : primaryKey;
+        this.primaryKey = key === undefined ? undefined : Object.freeze([...key]);
+        // A source-defined cursor leaves no field to compare, so the newest extraction wins.
+        this.dedupPolicy = deduplicating
+            ? (dedupPolicy ?? (from.sourceDefinedCursor ? 'replace' : 'cursor_newer'))
+            : dedupPolicy;
+        Object.freeze(this);
+    }
+    validate(stream) {
+        if (!this.stream.supportedSyncModes.includes(this.syncMode) ||
+            !stream.supportedSyncModes.includes(this.syncMode))
+            throw new TypeError(`Stream ${stream.name} does not support ${this.syncMode}`);
+        this.validateSelection();
+        for (const read of this.fileReads)
+            read.validate(stream);
+    }
+    validateSelection() {
+        for (const read of this.fileReads)
+            read.validate(this.stream);
+        const { syncMode, destinationSyncMode, cursorField, primaryKey, dedupPolicy, } = this;
+        if (syncMode !== 'full_refresh' && syncMode !== 'incremental')
+            throw new TypeError(`Unsupported syncMode: ${syncMode}`);
+        if (syncMode === 'incremental' &&
+            (destinationSyncMode === 'overwrite' ||
+                destinationSyncMode === 'overwrite_dedup'))
+            throw new TypeError('Incremental extraction cannot use overwrite loading');
+        if (syncMode === 'full_refresh' && destinationSyncMode === 'append_dedup')
+            throw new TypeError('Full refresh deduplication requires overwrite_dedup');
+        const { sourceDefinedCursor, emitsDeletes } = this.stream;
+        if (sourceDefinedCursor && cursorField !== undefined)
+            throw new TypeError(`Stream ${this.stream.name} defines its own cursor; omit cursorField`);
+        // A cursor field the stream dropped is a breaking change, as Airbyte
+        // treats a removed cursor; the copy must choose another.
+        if (cursorField !== undefined &&
+            !Object.hasOwn(this.stream.jsonSchema.properties ?? {}, cursorField))
+            throw new TypeError(`Stream ${this.stream.name} does not describe cursor field ${cursorField}`);
+        if (syncMode === 'incremental' &&
+            !sourceDefinedCursor &&
+            (typeof cursorField !== 'string' || !cursorField))
+            throw new TypeError('Incremental extraction requires cursorField');
+        if (emitsDeletes &&
+            syncMode === 'incremental' &&
+            destinationSyncMode !== 'append_dedup')
+            throw new TypeError(`Stream ${this.stream.name} emits deletions; incremental copies require append_dedup`);
+        if (dedupPolicy !== undefined &&
+            dedupPolicy !== 'cursor_newer' &&
+            dedupPolicy !== 'replace')
+            throw new TypeError(`Unsupported dedupPolicy: ${dedupPolicy}`);
+        if (destinationSyncMode === 'append_dedup' ||
+            destinationSyncMode === 'overwrite_dedup') {
+            if (primaryKey === undefined)
+                throw new TypeError(`Stream ${this.stream.name} declares no primary key; select primaryKey`);
+            if (dedupPolicy === 'cursor_newer' && cursorField === undefined)
+                throw new TypeError(sourceDefinedCursor
+                    ? `Stream ${this.stream.name} has no cursor field to compare; deduplicate with dedupPolicy 'replace'`
+                    : 'cursor_newer deduplication requires cursorField');
+            new Deduplication(this.stream, primaryKey, cursorField);
+            // A cursor inside the primary key is equal on every conflict, so the
+            // cursor_newer guard can never fire and restated facts load as no-ops.
+            if (dedupPolicy === 'cursor_newer' &&
+                cursorField !== undefined &&
+                primaryKey.includes(cursorField))
+                throw new TypeError(`Cursor field ${cursorField} is part of the primary key, so cursor_newer can never update a conflicting row; select dedupPolicy 'replace' to let the newest extraction win`);
+        }
+        else {
+            if (primaryKey !== undefined)
+                throw new TypeError('Select primaryKey only for deduplication loading');
+            if (dedupPolicy !== undefined)
+                throw new TypeError('Select dedupPolicy only for deduplication loading');
+        }
+        if (syncMode === 'full_refresh' &&
+            destinationSyncMode !== 'overwrite_dedup' &&
+            cursorField !== undefined)
+            throw new TypeError('Full refresh append/overwrite does not use cursorField');
+    }
+    deduplication() {
+        if (this.primaryKey === undefined)
+            throw new TypeError('Deduplication requires a primaryKey');
+        return new Deduplication(this.stream, this.primaryKey, this.cursorField);
+    }
+}

package/dist/core/copy.d.ts

@@ -0,0 +1,27 @@
+import type { CheckpointStore } from '../state/checkpoint-store.ts';
+import { CopyConfiguration } from './copy-configuration.ts';
+import type { Destination } from './destination.ts';
+import type { Source } from './source.ts';
+import type { Stream } from './stream.ts';
+import { Target as DestinationTarget } from './target.ts';
+import type { LoadFailure, WriteCount } from './writer.ts';
+export type CopyResult<Target extends DestinationTarget> = WriteCount & {
+    readonly copy: Copy<Target>;
+};
+export type CopyOutcome<Target extends DestinationTarget> = CopyResult<Target> & {
+    readonly failures: readonly LoadFailure[];
+    readonly cancelled: boolean;
+};
+export declare function describeFailures<Target extends DestinationTarget>({ copy, failures, }: CopyOutcome<Target>): string;
+export declare class Copy<Target extends DestinationTarget> {
+    readonly configuration: CopyConfiguration;
+    readonly to: Target;
+    readonly id?: string;
+    constructor(from: Stream, to: Target, modes?: ConstructorParameters<typeof CopyConfiguration>[1] & {
+        id?: string;
+    });
+    get from(): Stream;
+    writer(source: Source): string;
+    validate(source: Source, destination: Destination<Target>, checkpoints?: CheckpointStore): void;
+    clear(source: Source, destination: Destination<Target>, checkpoints?: CheckpointStore): Promise<void>;
+}

package/dist/core/copy.js

@@ -0,0 +1,56 @@
+import { errorMessage } from '../state/sync-history.js';
+import { CopyConfiguration } from './copy-configuration.js';
+import { FileTransfer } from './file-transfer.js';
+import { Target as DestinationTarget } from './target.js';
+export function describeFailures({ copy, failures, }) {
+    return failures
+        .map(({ partition, error }) => `${copy.id ?? copy.from.name}${partition === null ? '' : ` ${JSON.stringify(partition)}`}: ${errorMessage(error)}`)
+        .join('; ');
+}
+// One configured transfer from a source stream to a destination declaration.
+export class Copy {
+    configuration;
+    to;
+    id;
+    constructor(from, to, modes = {
+        syncMode: 'full_refresh',
+        destinationSyncMode: 'overwrite',
+    }) {
+        if (!(to instanceof DestinationTarget))
+            throw new TypeError('Copy requires a destination target');
+        this.configuration = new CopyConfiguration(from, modes, to.fileReads);
+        this.to = to;
+        if (modes.id !== undefined && (!modes.id || modes.id.includes('\0')))
+            throw new TypeError('Copy id must be nonempty text');
+        this.id = modes.id;
+        Object.freeze(this);
+    }
+    get from() {
+        return this.configuration.stream;
+    }
+    // The target's writer: the declared id names the replication; without one,
+    // the source and stream do.
+    writer(source) {
+        return JSON.stringify(this.id !== undefined
+            ? { copy: this.id }
+            : { source: source.identity, stream: this.from.name });
+    }
+    validate(source, destination, checkpoints) {
+        source.validate(this.configuration);
+        destination.validate(this.configuration, this.to);
+        if (this.configuration.syncMode === 'incremental' &&
+            (this.id === undefined || checkpoints === undefined))
+            throw new TypeError('Incremental copies require a stable id and checkpoint store');
+    }
+    // Empties the target and removes this copy's checkpoint together, so the
+    // next run reloads from scratch.
+    async clear(source, destination, checkpoints) {
+        this.validate(source, destination, checkpoints);
+        const files = new FileTransfer(this.configuration.fileReads, destination.location(this.to), this.writer(source));
+        const drop = () => destination.clear(this.configuration, this.to, this.writer(source), (values) => files.reconcile(values));
+        if (this.id !== undefined && checkpoints !== undefined)
+            await checkpoints.clear(this.id, drop);
+        else
+            await drop();
+    }
+}

package/dist/core/deduplication.d.ts

@@ -0,0 +1,14 @@
+import { type DeclaredFormat } from './formats/declared-format.ts';
+import type { Stream } from './stream.ts';
+export declare class Deduplication {
+    #private;
+    readonly stream: Stream;
+    readonly primaryKey: readonly string[];
+    readonly cursorField?: string;
+    constructor(stream: Stream, primaryKey: readonly string[], cursorField?: string);
+    format(field: string): DeclaredFormat | null;
+    type(field: string): 'string' | 'number' | 'integer' | 'boolean';
+    value(record: unknown, field: string): string | number | boolean;
+    key(record: unknown): string;
+    cursor(record: unknown): string | number;
+}

package/dist/core/deduplication.js

@@ -0,0 +1,96 @@
+import { declaredFormat, } from './formats/declared-format.js';
+// The selected logical identity and ordering rule, independent of storage constraints.
+export class Deduplication {
+    stream;
+    primaryKey;
+    // Absent when the newest extraction always wins (dedupPolicy replace).
+    cursorField;
+    #formats = new Map();
+    constructor(stream, primaryKey, cursorField) {
+        this.stream = stream;
+        this.cursorField = cursorField;
+        if (!Array.isArray(primaryKey) ||
+            primaryKey.length === 0 ||
+            new Set(primaryKey).size !== primaryKey.length)
+            throw new TypeError('Deduplication requires distinct primaryKey fields');
+        this.primaryKey = Object.freeze([...primaryKey]);
+        for (const field of this.primaryKey)
+            this.#declare(field);
+        if (cursorField !== undefined) {
+            if (!['string', 'number', 'integer'].includes(this.type(cursorField)))
+                throw new TypeError('Deduplication cursor must be text or numeric');
+            const format = this.#declare(cursorField);
+            if (format?.orderable === false)
+                throw new TypeError(`Deduplication cursor ${cursorField} cannot order ${format.name} values`);
+        }
+        Object.freeze(this);
+    }
+    #declare(field) {
+        this.type(field);
+        const schema = this.stream.jsonSchema.properties?.[field];
+        const format = schema === undefined ? null : declaredFormat(schema);
+        this.#formats.set(field, format);
+        return format;
+    }
+    // The format of a key or cursor field, null when it declares none.
+    format(field) {
+        const format = this.#formats.get(field);
+        if (format === undefined)
+            throw new TypeError(`${field} is not a key or cursor field`);
+        return format;
+    }
+    type(field) {
+        const properties = this.stream.jsonSchema.properties;
+        if (typeof field !== 'string' ||
+            !field ||
+            properties === null ||
+            typeof properties !== 'object' ||
+            !Object.hasOwn(properties, field))
+            throw new TypeError(`Schema must describe key/cursor field ${field}`);
+        const schema = Reflect.get(properties, field);
+        const type = schema !== null && typeof schema === 'object'
+            ? Reflect.get(schema, 'type')
+            : undefined;
+        if (type !== 'string' &&
+            type !== 'number' &&
+            type !== 'integer' &&
+            type !== 'boolean')
+            throw new TypeError(`Key/cursor field ${field} requires one non-null scalar schema type`);
+        return type;
+    }
+    value(record, field) {
+        if (record === null ||
+            typeof record !== 'object' ||
+            !Object.hasOwn(record, field))
+            throw new TypeError(`Record is missing key/cursor field ${field}`);
+        const value = Reflect.get(record, field);
+        const type = this.type(field);
+        const format = this.format(field);
+        if ((type === 'string' &&
+            typeof value === 'string' &&
+            value.isWellFormed() &&
+            (format === null || format.accepts(value))) ||
+            (type === 'boolean' && typeof value === 'boolean') ||
+            (type === 'number' &&
+                typeof value === 'number' &&
+                Number.isFinite(value)) ||
+            (type === 'integer' &&
+                typeof value === 'number' &&
+                Number.isSafeInteger(value)))
+            return value;
+        throw new TypeError(format === null
+            ? `Key/cursor field ${field} requires non-null ${type}`
+            : `Key/cursor field ${field} requires a canonical ${format.name} value`);
+    }
+    key(record) {
+        return JSON.stringify(this.primaryKey.map((field) => this.value(record, field)));
+    }
+    cursor(record) {
+        if (this.cursorField === undefined)
+            throw new TypeError('Deduplication has no cursor field');
+        const value = this.value(record, this.cursorField);
+        if (typeof value === 'boolean')
+            throw new TypeError('Cursor cannot be boolean');
+        return value;
+    }
+}

package/dist/core/derived-dataset.d.ts

@@ -0,0 +1,19 @@
+import type { Connection } from './connection.ts';
+import type { Target as DestinationTarget } from './target.ts';
+export type RefreshTrigger = {
+    readonly kind: 'commit';
+    readonly tried: Set<string>;
+} | {
+    readonly kind: 'pass';
+    readonly tried: ReadonlySet<string>;
+};
+export declare abstract class DerivedDataset<Target extends DestinationTarget> {
+    abstract readonly name: string;
+    validate(_connections: readonly Connection<Target>[]): void;
+    abstract refresh(options: {
+        readonly connections: readonly Connection<Target>[];
+        readonly earlier: readonly DerivedDataset<Target>[];
+        readonly signal?: AbortSignal;
+        readonly trigger: RefreshTrigger;
+    }): Promise<void>;
+}

package/dist/core/derived-dataset.js

@@ -0,0 +1,10 @@
+// Data a pipeline keeps in step with what its copies load, and can always
+// rebuild from it (Kleppmann's derived data; Estuary's derivations), such as a
+// view over loaded tables or the text of the files a copy stored. Each kind
+// owns how it makes its data; the pipeline owns when: in the background after
+// each commit of a copy, after each pass of a watch, and after the passes of a
+// run. A refresh can run while a copy loads, so it reads only what committed.
+export class DerivedDataset {
+    // Refuses, without I/O, a declaration the pipeline's copies cannot feed.
+    validate(_connections) { }
+}

package/dist/core/derived-refreshes.d.ts

@@ -0,0 +1,15 @@
+import type { Connection } from './connection.ts';
+import type { DerivedDataset, RefreshTrigger } from './derived-dataset.ts';
+import type { Target as DestinationTarget } from './target.ts';
+type Kind = RefreshTrigger['kind'];
+export declare class DerivedRefreshes<Target extends DestinationTarget> {
+    #private;
+    constructor(datasets: readonly DerivedDataset<Target>[], connections: readonly Connection<Target>[], signal: AbortSignal | undefined, stop: (error: unknown) => void);
+    get failure(): {
+        readonly error: unknown;
+    } | undefined;
+    request(kind: Kind): void;
+    settled(): Promise<void>;
+    refresh(): Promise<void>;
+}
+export {};

package/dist/core/derived-refreshes.js

@@ -0,0 +1,120 @@
+import { Latch } from '@zukhruf/async';
+// No refresh runs.
+class Idle {
+}
+// A refresh runs; done opens once it ended.
+class Running {
+    done;
+    constructor(done) {
+        this.done = done;
+    }
+}
+// A refresh runs and a copy asked for one more while it ran: one follows, of
+// the strongest kind asked, since a pass refresh does what a commit refresh
+// does and more.
+class Queued {
+    done;
+    kind;
+    constructor(done, kind) {
+        this.done = done;
+        this.kind = kind;
+    }
+}
+// No refresh failed yet.
+class Healthy {
+}
+// The first refresh failure that the watch's own stop did not cause.
+class Failed {
+    error;
+    constructor(error) {
+        this.error = error;
+    }
+}
+// A pipeline's refreshes of its derived datasets. In the background, one
+// refresh runs at a time, and a copy that commits while one runs asks for one
+// more, so no pass waits on them, as a build watcher keeps one rebuild queued
+// and Spark's default trigger starts the next micro-batch once the last one
+// ended. The first background failure is told to stop and starts no more
+// background refreshes; a refresh that signal's abort ended is no failure.
+export class DerivedRefreshes {
+    #datasets;
+    #connections;
+    #signal;
+    #stop;
+    #state = new Idle();
+    // Each dataset's keys that commit refreshes tried since the last pass
+    // refresh; the memory of the passes that run, which ends with them.
+    #tried = new Map();
+    #health = new Healthy();
+    constructor(datasets, connections, signal, stop) {
+        this.#datasets = datasets;
+        this.#connections = connections;
+        this.#signal = signal;
+        this.#stop = stop;
+    }
+    get failure() {
+        return this.#health instanceof Failed ? this.#health : undefined;
+    }
+    request(kind) {
+        if (this.#datasets.length === 0 || this.#health instanceof Failed)
+            return;
+        if (this.#state instanceof Idle) {
+            const done = new Latch();
+            this.#state = new Running(done);
+            void this.#run(done, kind);
+            return;
+        }
+        this.#state = new Queued(this.#state.done, this.#state instanceof Queued && this.#state.kind === 'pass'
+            ? 'pass'
+            : kind);
+    }
+    async settled() {
+        if (this.#state instanceof Idle)
+            return;
+        await this.#state.done.wait();
+    }
+    // Once the background refresh ended, refreshes every dataset once more for
+    // the passes that ended, and rejects with its failure, as the end of a run.
+    async refresh() {
+        await this.settled();
+        await this.#refresh('pass');
+    }
+    // Every dataset in declared order, each after the ones it can read.
+    async #refresh(kind) {
+        for (const [index, dataset] of this.#datasets.entries()) {
+            const tried = this.#tried.get(dataset) ?? new Set();
+            this.#tried.set(dataset, tried);
+            await dataset.refresh({
+                connections: this.#connections,
+                earlier: this.#datasets.slice(0, index),
+                signal: this.#signal,
+                trigger: { kind, tried },
+            });
+        }
+        // The passes it follows ended, and their memory with them.
+        if (kind === 'pass')
+            this.#tried.clear();
+    }
+    // Refreshes once, and once more while a copy asked for it.
+    async #run(done, first) {
+        try {
+            for (let kind = first;;) {
+                await this.#refresh(kind);
+                if (!(this.#state instanceof Queued) || this.#signal?.aborted)
+                    break;
+                kind = this.#state.kind;
+                this.#state = new Running(done);
+            }
+        }
+        catch (error) {
+            if (this.#signal?.aborted !== true) {
+                this.#health = new Failed(error);
+                this.#stop(error);
+            }
+        }
+        finally {
+            this.#state = new Idle();
+            done.open();
+        }
+    }
+}

package/dist/core/destination.d.ts

@@ -0,0 +1,22 @@
+import type { PipelineClock } from './clock.ts';
+import type { CopyConfiguration } from './copy-configuration.ts';
+import type { Target as DestinationTarget } from './target.ts';
+import type { FieldValues, Stage, Writer } from './writer.ts';
+export type DestinationSyncMode = 'append' | 'overwrite' | 'append_dedup' | 'overwrite_dedup';
+export type Load<Target extends DestinationTarget> = AsyncDisposable & {
+    prepare(configuration: CopyConfiguration, target: Target, binding: {
+        readonly writer: string;
+        readonly reloading: boolean;
+    }): Promise<Stage>;
+};
+export declare abstract class Destination<Target extends DestinationTarget> {
+    abstract readonly supportedDestinationSyncModes: readonly DestinationSyncMode[];
+    abstract location(target: Target): string;
+    validate(configuration: CopyConfiguration, target: Target): void;
+    protected validateConfiguration(configuration: CopyConfiguration, target: Target): void;
+    abstract createWriter(configuration: CopyConfiguration, target: Target): Writer;
+    abstract load(options: {
+        clock: PipelineClock;
+    }): Promise<Load<Target>>;
+    clear(configuration: CopyConfiguration, target: Target, writer: string, committed?: (values: FieldValues) => Promise<void>): Promise<void>;
+}