@eltjs/elt 0.0.0-stage → 0.1.0

package/dist/core/record-validation.js

@@ -0,0 +1,128 @@
+import { declaredFormat, } from './formats/declared-format.js';
+const itemTypes = new Set(['string', 'integer', 'number', 'boolean']);
+const scalarTypes = new Set([...itemTypes, 'null']);
+const typesOf = (field) => typeof field.type === 'string' ? [field.type] : field.type;
+const isArrayField = (field) => typesOf(field).includes('array');
+// Whether a scalar declares only scalar types.
+function scalarSupported(field) {
+    const { type } = field;
+    const types = Array.isArray(type) ? type : [type];
+    return (field.items === undefined && types.every((each) => scalarTypes.has(each)));
+}
+// Whether an object item declares scalar fields and names each one required.
+function objectSupported(items) {
+    const { properties, required } = items;
+    if (properties === null || typeof properties !== 'object')
+        return false;
+    const names = Object.keys(properties);
+    return (names.length > 0 &&
+        Array.isArray(required) &&
+        required.length === names.length &&
+        names.every((name) => required.includes(name)) &&
+        Object.values(properties).every(scalarSupported));
+}
+// Whether a field declares a type validateRecords can check: scalar types, or
+// an array (alone or with null) of one non-null scalar type or of objects of
+// scalar fields.
+function supported(field) {
+    if (!isArrayField(field))
+        return scalarSupported(field);
+    const types = typesOf(field);
+    const { items } = field;
+    return (types.every((type) => type === 'array' || type === 'null') &&
+        // Constraints on an array field belong on its items.
+        field.enum === undefined &&
+        field.format === undefined &&
+        field.precision === undefined &&
+        field.scale === undefined &&
+        field.contentEncoding === undefined &&
+        field.minimum === undefined &&
+        field.maximum === undefined &&
+        field.minLength === undefined &&
+        items !== null &&
+        typeof items === 'object' &&
+        (items.type === 'object'
+            ? objectSupported(items)
+            : itemTypes.has(items.type)));
+}
+// Whether a non-null value satisfies a scalar schema's type and constraints.
+function scalarValid(field, types, format, value) {
+    const typed = types.some((type) => type === 'integer'
+        ? Number.isSafeInteger(value)
+        : type === 'number'
+            ? typeof value === 'number' && Number.isFinite(value)
+            : (type === 'string' || type === 'boolean') && typeof value === type);
+    return (typed &&
+        (field.enum === undefined ||
+            field.enum.some((member) => member === value)) &&
+        !(typeof value === 'number' &&
+            ((field.minimum !== undefined && value < field.minimum) ||
+                (field.maximum !== undefined && value > field.maximum))) &&
+        !(typeof value === 'string' &&
+            field.minLength !== undefined &&
+            value.length < field.minLength) &&
+        (format === null || (typeof value === 'string' && format.accepts(value))));
+}
+// A check of one scalar value against its schema, nulls included when the
+// schema allows them; the format is read once.
+function scalarCheck(field) {
+    const types = typesOf(field);
+    const format = declaredFormat(field);
+    return (value) => (value === null && types.includes('null')) ||
+        scalarValid(field, types, format, value);
+}
+// A check of one item of an array field.
+function itemCheck(items) {
+    if (items.type !== 'object')
+        return scalarCheck(items);
+    const fields = Object.entries(items.properties).map(([name, field]) => [name, scalarCheck(field)]);
+    return (item) => item !== null &&
+        typeof item === 'object' &&
+        !Array.isArray(item) &&
+        Object.keys(item).length === fields.length &&
+        fields.every(([name, valid]) => Object.hasOwn(item, name) && valid(Reflect.get(item, name)));
+}
+// A check of one field: a scalar, or an array whose every item is valid.
+function fieldCheck(field) {
+    if (!isArrayField(field))
+        return scalarCheck(field);
+    const item = itemCheck(field.items);
+    const nullable = typesOf(field).includes('null');
+    return (value) => (value === null && nullable) || (Array.isArray(value) && value.every(item));
+}
+// Every property is required and a record carries exactly those properties, so
+// a projection that drops or misspells a field fails here instead of loading.
+export function validateRecords(stream, records, source) {
+    if (!Array.isArray(records))
+        throw new TypeError(`${source} returned invalid ${stream.name} records`);
+    const { properties } = stream.jsonSchema;
+    if (properties === undefined)
+        throw new TypeError(`Stream ${stream.name} declares no properties to validate`);
+    const fields = Object.entries(properties).map(([name, schema]) => {
+        const unsupported = (cause) => new TypeError(`Stream ${stream.name}.${name} declares an unsupported type`, { cause });
+        if (!supported(schema))
+            throw unsupported();
+        try {
+            return { name, valid: fieldCheck(schema) };
+        }
+        catch (error) {
+            throw unsupported(error);
+        }
+    });
+    const valid = [];
+    for (const record of records) {
+        assertRecord(record, fields, stream.name, source);
+        valid.push(record);
+    }
+    return valid;
+}
+function assertRecord(record, fields, stream, source) {
+    if (record === null ||
+        typeof record !== 'object' ||
+        Array.isArray(record) ||
+        Object.keys(record).length !== fields.length)
+        throw new TypeError(`${source} returned an invalid ${stream} record`);
+    for (const { name, valid } of fields)
+        if (!valid(Reflect.get(record, name)))
+            throw new TypeError(`${source} returned invalid ${stream}.${name}`);
+}

package/dist/core/reload-mode.d.ts

@@ -0,0 +1,9 @@
+import type { DestinationSyncMode } from './destination.ts';
+export type StoredFit = 'missing' | 'stale' | 'fits';
+export type ReloadMode = 'load' | 'create' | 'evolve' | 'reload' | 'continue';
+export declare function reloadMode({ reloading, destinationSyncMode, target, hidden, }: {
+    readonly reloading: boolean;
+    readonly destinationSyncMode: DestinationSyncMode;
+    readonly target: StoredFit;
+    readonly hidden: StoredFit;
+}): ReloadMode;

package/dist/core/reload-mode.js

@@ -0,0 +1,17 @@
+// Every destination decides alike: an open reload continues into a hidden
+// target that still fits, or starts over in a new one, and a stored target
+// that no longer fits the stream evolves to it. An overwrite keeps none of
+// the stored rows, so it reloads that target instead, as Airbyte's truncate
+// syncs load a new table and swap it in.
+export function reloadMode({ reloading, destinationSyncMode, target, hidden, }) {
+    if (reloading)
+        return hidden === 'fits' ? 'continue' : 'reload';
+    if (target === 'missing')
+        return 'create';
+    if (target === 'stale') {
+        const overwrites = destinationSyncMode === 'overwrite' ||
+            destinationSyncMode === 'overwrite_dedup';
+        return overwrites ? 'reload' : 'evolve';
+    }
+    return 'load';
+}

package/dist/core/replication.d.ts

@@ -0,0 +1,8 @@
+import type { CheckpointStore } from '../state/checkpoint-store.ts';
+import type { PipelineClock } from './clock.ts';
+import type { Connection } from './connection.ts';
+import type { Copy, CopyOutcome } from './copy.ts';
+import type { Destination } from './destination.ts';
+import { type Source } from './source.ts';
+import type { Target as DestinationTarget } from './target.ts';
+export declare function replicate<Target extends DestinationTarget>(source: Source, destination: Destination<Target>, checkpoints: CheckpointStore | undefined, copies: readonly Copy<Target>[], connection: Connection<Target>, committed: () => void, signal: AbortSignal | undefined, clock: PipelineClock): Promise<CopyOutcome<Target>[]>;

package/dist/core/replication.js

@@ -0,0 +1,478 @@
+import { FileTransfer } from './file-transfer.js';
+import { CopyProgress, passChannels } from './pass-channels.js';
+import { StreamStatus, } from './source.js';
+// A copy that has not reported how it ended.
+class Reporting {
+}
+// A copy that reported how it ended, once.
+class Settled {
+}
+// One copy's side of a replication: its stage, what it committed, what is
+// pending since the last commit, and what did not load.
+class Replicated {
+    stage;
+    emitted = { count: 0, deleted: 0 };
+    committed = { count: 0, deleted: 0 };
+    pending = { count: 0, deleted: 0 };
+    failures = [];
+    // Nothing commits after a failed read until the next checkpoint: a full
+    // refresh has none, so any failure keeps the previous target.
+    failed = false;
+    // Nothing was applied since the last commit.
+    clean = false;
+    // The stage failed; the stream's remaining messages are ignored.
+    broken = false;
+    started = false;
+    ended = false;
+    // The copy's last run left a reload open.
+    reloading = false;
+    // The run's signal stopped the copy before it ended.
+    cancelled = false;
+    // Whether the saved checkpoint describes the copy's stage: not once the
+    // stage is a target this load creates or reloads, until the copy saves.
+    resumes = true;
+    #progress = new Reporting();
+    copy;
+    connection;
+    // Told each time the copy made rows visible in its target.
+    published;
+    files;
+    #source;
+    constructor(copy, connection, source, destination, published) {
+        this.copy = copy;
+        this.connection = connection;
+        this.published = published;
+        this.#source = source;
+        this.files = new FileTransfer(copy.configuration.fileReads, destination.location(copy.to), copy.writer(source));
+    }
+    get stream() {
+        return this.copy.from;
+    }
+    // What did not load, of the kind the source says the error is.
+    recordFailure(partition, error) {
+        this.failures.push({
+            partition,
+            error,
+            failureType: this.#source.failureType(error),
+        });
+    }
+    outcome() {
+        return {
+            copy: this.copy,
+            ...this.committed,
+            failures: this.failures,
+            cancelled: this.cancelled,
+        };
+    }
+    report() {
+        this.#notify('running');
+    }
+    settle() {
+        if (this.#progress instanceof Settled)
+            return;
+        this.#progress = new Settled();
+        this.#notify(this.failures.length === 0 ? 'complete' : 'incomplete');
+    }
+    #notify(status) {
+        passChannels.progress.publish(new CopyProgress(this.connection, this.copy, status, { ...this.emitted }, { ...this.committed }));
+    }
+}
+// Airbyte's replication worker: one read covers every copy's stream, each
+// stream's messages go to its own stage, and a checkpoint is saved only after
+// its stage committed. A copy that fails does not stop the others. Once signal
+// aborts, as Airbyte's worker stops a cancelled sync, no further message is
+// applied: each copy still open keeps what it committed and is cancelled.
+// committed is told each time a copy made rows visible in its target: at
+// each commit, and when a reload replaced the target.
+export async function replicate(source, destination, checkpoints, copies, connection, committed, signal, clock) {
+    const replications = copies.map((copy) => new Replicated(copy, connection, source, destination, committed));
+    const bindings = new Map();
+    for (const { copy } of replications)
+        if (copy.configuration.syncMode === 'incremental' && copy.id !== undefined)
+            bindings.set(copy.id, {
+                copy: {
+                    source: source.identity,
+                    target: destination.location(copy.to),
+                    selection: selection(copy.configuration),
+                },
+                shape: shape(copy.from),
+            });
+    try {
+        if (checkpoints === undefined || bindings.size === 0)
+            await transfer(source, destination, null, replications, signal, clock);
+        else
+            await checkpoints.run(bindings, (run) => transfer(source, destination, run, replications, signal, clock));
+    }
+    catch (error) {
+        for (const replication of replications)
+            if (!replication.ended) {
+                replication.cancelled = cancelledBy(signal, error);
+                fail(replication, error);
+            }
+    }
+    // A broken stream the source never ended settles here, after the read.
+    for (const replication of replications)
+        replication.settle();
+    return replications.map((replication) => replication.outcome());
+}
+async function transfer(source, destination, run, replications, signal, clock) {
+    const byStream = new Map(replications.map((replication) => [replication.stream.name, replication]));
+    const incremental = (replication) => replication.copy.configuration.syncMode === 'incremental';
+    const checkpoint = (replication) => {
+        if (run === null || replication.copy.id === undefined)
+            throw new TypeError('Incremental copies require a stable id and checkpoint store');
+        return { run, id: replication.copy.id };
+    };
+    // The source asks for a stream's state when it starts reading that stream.
+    const states = {
+        get: async (name) => {
+            const replication = byStream.get(name);
+            if (replication === undefined ||
+                !incremental(replication) ||
+                !replication.resumes)
+                return null;
+            const { run, id } = checkpoint(replication);
+            return run.state(id);
+        },
+    };
+    const reading = [];
+    for (const replication of replications) {
+        if (!incremental(replication)) {
+            reading.push(replication);
+            continue;
+        }
+        try {
+            const { run, id } = checkpoint(replication);
+            replication.reloading = run.reloading(id);
+            reading.push(replication);
+        }
+        catch (error) {
+            fail(replication, error);
+        }
+    }
+    if (reading.length === 0)
+        return;
+    signal?.throwIfAborted();
+    await using load = await destination.load({ clock });
+    // Every target is prepared before anything is read; a copy whose target is
+    // refused reads nothing.
+    const prepared = [];
+    for (const replication of reading)
+        try {
+            replication.stage = await load.prepare(replication.copy.configuration, replication.copy.to, {
+                writer: replication.copy.writer(source),
+                reloading: replication.reloading,
+            });
+            // A target this load creates or reloads holds nothing to resume from.
+            if (replication.stage.fresh)
+                replication.resumes = false;
+            await replication.files.reconcile(replication.stage.values);
+            prepared.push(replication);
+        }
+        catch (error) {
+            if (replication.stage !== undefined)
+                await breakStage(replication, error);
+            else
+                fail(replication, error);
+        }
+    if (prepared.length === 0)
+        return;
+    try {
+        for await (const message of source.read(prepared.map(({ copy }) => copy.configuration), states, { signal, clock })) {
+            signal?.throwIfAborted();
+            const replication = byStream.get(validStream(message));
+            if (replication === undefined || !prepared.includes(replication))
+                throw new TypeError(`Source emitted an unselected stream: ${message.stream}`);
+            if (replication.ended)
+                throw new TypeError(`Source emitted ${message.stream} after it ended`);
+            try {
+                if (message instanceof StreamStatus) {
+                    if (message.status === 'STARTED') {
+                        if (replication.started)
+                            throw new TypeError(`Source started ${message.stream} twice`);
+                        replication.started = true;
+                        replication.report();
+                    }
+                    else if (message.status === 'FAILED') {
+                        if (replication.broken)
+                            continue;
+                        await started(replication).discard();
+                        replication.pending.count = 0;
+                        replication.pending.deleted = 0;
+                        replication.failed = true;
+                        replication.recordFailure(message.partition, message.error);
+                        await replication.files.reconcile(started(replication).values);
+                        replication.report();
+                    }
+                    else {
+                        replication.ended = true;
+                        if (replication.broken) {
+                            replication.settle();
+                            continue;
+                        }
+                        const stage = started(replication);
+                        // The first commit also makes the prepared target itself durable,
+                        // so an empty overwrite still clears it.
+                        if (!replication.failed && !replication.clean)
+                            await commit(replication, signal);
+                        // A reload replaces the target only once every partition read.
+                        if (replication.failures.length === 0 && stage.reloading) {
+                            await stage.complete();
+                            replication.published();
+                            if (incremental(replication)) {
+                                const { run, id } = checkpoint(replication);
+                                if (replication.resumes)
+                                    await run.reloaded(id);
+                                else
+                                    await run.save(id, 'null', false);
+                            }
+                        }
+                        replication.stage = undefined;
+                        await stage[Symbol.asyncDispose]();
+                        replication.settle();
+                    }
+                    continue;
+                }
+                if (replication.broken)
+                    continue;
+                const operation = validOperation(replication.stream, message);
+                if (operation.type === 'STATE') {
+                    if (!incremental(replication))
+                        throw new TypeError(`Full refresh stream ${message.stream} emitted a checkpoint`);
+                    await commit(replication, signal);
+                    const { run, id } = checkpoint(replication);
+                    await run.save(id, operation.state, started(replication).reloading);
+                    replication.resumes = true;
+                }
+                else {
+                    await started(replication).apply(operation.type === 'RECORD'
+                        ? {
+                            ...operation,
+                            data: await replication.files.record(operation.data),
+                        }
+                        : operation);
+                    replication.clean = false;
+                    if (operation.type === 'RECORD') {
+                        replication.pending.count++;
+                        replication.emitted.count++;
+                    }
+                    else if (operation.type === 'DELETE') {
+                        replication.pending.deleted++;
+                        replication.emitted.deleted++;
+                    }
+                    replication.report();
+                }
+            }
+            catch (error) {
+                replication.cancelled = cancelledBy(signal, error);
+                await breakStage(replication, error);
+            }
+        }
+        for (const replication of prepared)
+            if (!replication.ended && !replication.broken)
+                await breakStage(replication, new TypeError(`Source did not end stream ${replication.stream.name}`));
+    }
+    catch (error) {
+        for (const replication of prepared)
+            if (!replication.ended && !replication.broken) {
+                replication.cancelled = cancelledBy(signal, error);
+                await breakStage(replication, error);
+            }
+    }
+}
+// Whether error is the abort of the run's signal, not a failure.
+function cancelledBy(signal, error) {
+    return signal?.aborted === true && error === signal.reason;
+}
+// What a copy selects of its stream, apart from the stream's shape: changing
+// it refuses the checkpoint. A key the stream declares is part of its shape.
+function selection(configuration) {
+    const { stream, fileReads, syncMode, destinationSyncMode, cursorField, primaryKey, dedupPolicy, } = configuration;
+    return {
+        fileReads,
+        syncMode,
+        destinationSyncMode,
+        cursorField,
+        dedupPolicy,
+        primaryKey: stream.primaryKey.length > 0 ? undefined : primaryKey,
+    };
+}
+// The stream as it shapes the target, without the descriptions that only
+// annotate it. A changed field evolves the target; a changed declaration
+// stops the copy (see stream-change.ts).
+function shape(stream) {
+    const { description: _, properties, ...schema } = stream.jsonSchema;
+    const undescribed = ({ description: __, ...rest }) => rest;
+    return {
+        ...stream,
+        jsonSchema: {
+            ...schema,
+            properties: properties &&
+                Object.fromEntries(Object.entries(properties).map(([name, { items, ...field }]) => [
+                    name,
+                    { ...undescribed(field), items: items && undescribed(items) },
+                ])),
+        },
+    };
+}
+function started(replication) {
+    if (!replication.started || replication.stage === undefined)
+        throw new TypeError(`Source sent ${replication.stream.name} a message before starting it`);
+    return replication.stage;
+}
+async function commit(replication, signal) {
+    await replication.files.flush();
+    signal?.throwIfAborted();
+    await started(replication).commit();
+    replication.clean = true;
+    replication.failed = false;
+    replication.committed.count += replication.pending.count;
+    replication.committed.deleted += replication.pending.deleted;
+    replication.pending.count = 0;
+    replication.pending.deleted = 0;
+    await replication.files.reconcile(started(replication).values);
+    // The commit ends once its stored files are reconciled: a refresh that
+    // starts earlier could read a file the reconcile still needs.
+    replication.published();
+    replication.report();
+}
+function fail(replication, error) {
+    replication.recordFailure(null, error);
+    replication.broken = true;
+    replication.ended = true;
+    replication.settle();
+}
+// Fails a stream whose stage or checkpoint failed: its uncommitted rows are
+// dropped with the stage, what it committed before stays, and its later
+// messages are ignored until the source ends it.
+async function breakStage(replication, error) {
+    replication.recordFailure(null, error);
+    replication.broken = true;
+    const { stage } = replication;
+    replication.stage = undefined;
+    try {
+        await stage?.[Symbol.asyncDispose]();
+    }
+    catch (cause) {
+        replication.recordFailure(null, cause);
+    }
+}
+function validStream(message) {
+    if (message === null ||
+        typeof message !== 'object' ||
+        typeof message.stream !== 'string')
+        throw new TypeError('Source must emit records with stream and data, DELETE, RESET or STATE messages');
+    return message.stream;
+}
+// A STATE's state is checked and kept as the JSON text it is saved as.
+function validOperation(stream, message) {
+    if ('type' in message &&
+        message.type === 'STATE' &&
+        Object.hasOwn(message, 'state') &&
+        !Object.hasOwn(message, 'data')) {
+        if (!lossless(message.state))
+            throw new TypeError('Checkpoint state must be losslessly JSON serializable');
+        wellFormed(message.state, 'Checkpoint state');
+        return { type: 'STATE', state: JSON.stringify(message.state) };
+    }
+    if ('type' in message &&
+        message.type === 'DELETE' &&
+        Object.hasOwn(message, 'key') &&
+        !Object.hasOwn(message, 'data'))
+        return { type: 'DELETE', key: deletionKey(stream, message.key) };
+    if ('type' in message &&
+        message.type === 'RESET' &&
+        Object.hasOwn(message, 'partition') &&
+        !Object.hasOwn(message, 'data')) {
+        if (!stream.emitsDeletes)
+            throw new TypeError(`Stream ${stream.name} does not emit deletions, so it cannot reset`);
+        if ((message.partition === null) !== (stream.partitionKey === undefined))
+            throw new TypeError(`A reset of ${stream.name} is scoped to a partition exactly when the stream is partitioned`);
+        return { type: 'RESET', partition: message.partition };
+    }
+    if (!('type' in message) &&
+        'data' in message &&
+        Object.hasOwn(message, 'data')) {
+        if (Object.hasOwn(message, 'file'))
+            throw new TypeError('Destinations cannot receive source file paths');
+        wellFormed(message.data, 'Record');
+        return { type: 'RECORD', data: message.data };
+    }
+    throw new TypeError('Source must emit records with stream and data, DELETE, RESET or STATE messages');
+}
+// Whether JSON carries value exactly, as JSON.parse(JSON.stringify(value))
+// strictly deep-equals it: null, a boolean, a string, a finite number other
+// than -0, an array with every index and no other field, or an object of
+// Object.prototype, none with an enumerable symbol key. A walk, not a round
+// trip, so checking a state as large as a snapshot copies none of it.
+function lossless(value) {
+    switch (typeof value) {
+        case 'boolean':
+        case 'string':
+            return true;
+        case 'number':
+            return Number.isFinite(value) && !Object.is(value, -0);
+        case 'object': {
+            if (value === null)
+                return true;
+            if (Object.getOwnPropertySymbols(value).some((key) => Object.prototype.propertyIsEnumerable.call(value, key)))
+                return false;
+            const fields = Object.keys(value);
+            if (Array.isArray(value)) {
+                if (Object.getPrototypeOf(value) !== Array.prototype ||
+                    fields.length !== value.length)
+                    return false;
+                for (let index = 0; index < value.length; index++)
+                    if (!Object.hasOwn(value, index) || !lossless(value[index]))
+                        return false;
+                return true;
+            }
+            return (Object.getPrototypeOf(value) === Object.prototype &&
+                fields.every((field) => lossless(Reflect.get(value, field))));
+        }
+        default:
+            return false;
+    }
+}
+// Every store keeps text as Unicode: a lone surrogate would load as U+FFFD
+// (SQLite) or be refused mid-load (Postgres), so no text reaches one.
+function wellFormed(value, path) {
+    if (typeof value === 'string') {
+        if (!value.isWellFormed())
+            throw new TypeError(`${path} has a lone surrogate`);
+    }
+    else if (Array.isArray(value))
+        for (const [index, item] of value.entries())
+            wellFormed(item, `${path}[${index}]`);
+    else if (value !== null &&
+        typeof value === 'object' &&
+        (Object.getPrototypeOf(value) === Object.prototype ||
+            Object.getPrototypeOf(value) === null))
+        for (const [name, item] of Object.entries(value)) {
+            if (!name.isWellFormed())
+                throw new TypeError(`${path} has a field name with a lone surrogate`);
+            wellFormed(item, `${path} field ${name}`);
+        }
+}
+function deletionKey(stream, key) {
+    const { name, primaryKey, emitsDeletes } = stream;
+    if (!emitsDeletes)
+        throw new TypeError(`Stream ${name} does not emit deletions`);
+    if (key === null ||
+        typeof key !== 'object' ||
+        Array.isArray(key) ||
+        Object.getPrototypeOf(key) !== Object.prototype ||
+        Object.keys(key).length !== primaryKey.length ||
+        !primaryKey.every((field) => Object.hasOwn(key, field)))
+        throw new TypeError(`DELETE for ${name} must carry exactly its primary key ${JSON.stringify(primaryKey)}`);
+    const snapshot = {};
+    for (const field of primaryKey) {
+        const value = Reflect.get(key, field);
+        if (!(typeof value === 'string' && value.isWellFormed()) &&
+            !(typeof value === 'number' && Number.isFinite(value)) &&
+            typeof value !== 'boolean')
+            throw new TypeError(`DELETE for ${name} has an invalid ${field}`);
+        snapshot[field] = value;
+    }
+    return Object.freeze(snapshot);
+}

package/dist/core/rule.d.ts

@@ -0,0 +1,29 @@
+export type LoadedRow = Readonly<Record<string, unknown>>;
+export type RuleResult<Output> = {
+    readonly value: Output;
+} | {
+    readonly empty: true;
+} | {
+    readonly error: string;
+} | {
+    readonly gone: true;
+};
+export type RuleColumn = {
+    readonly description: string;
+    readonly type: 'text' | 'blob';
+};
+export declare abstract class Rule<Input, Output> {
+    abstract readonly key: {
+        readonly name: string;
+        readonly description: string;
+    };
+    abstract readonly output: Readonly<Record<string, RuleColumn>>;
+    abstract readonly batch: number;
+    abstract readonly concurrency: number;
+    abstract read(row: LoadedRow): {
+        readonly key: string;
+        readonly input: Input;
+    } | null;
+    abstract identity(input: Input): Promise<string | null>;
+    abstract compute(inputs: readonly Input[], signal?: AbortSignal): Promise<readonly RuleResult<Readonly<Record<string, Output>>>[]>;
+}

package/dist/core/rule.js

@@ -0,0 +1,6 @@
+// How a computed dataset turns each input into its stored output, such as a
+// file's text or a listing's vector (a Strategy). Its identity names what made
+// an output, the parser's format or the model, so an output made otherwise
+// is made again, as pgai's vectorizer re-embeds rows after a model change.
+export class Rule {
+}