@eltjs/elt 0.0.0-stage → 0.1.1

package/dist/core/formats/temporal-format.js

@@ -0,0 +1,23 @@
+import { FieldFormat } from './field-format.js';
+// A time spelling whose fraction of a second has exactly `precision` digits,
+// none at precision 0, so the values of one field share a width and sort by
+// bytes as they do in time. Precision defaults to milliseconds.
+export class TemporalFormat extends FieldFormat {
+    precision;
+    constructor(precision = 3) {
+        super();
+        if (!Number.isInteger(precision) || precision < 0 || precision > 9)
+            throw new TypeError('Temporal precision must be an integer from 0 to 9 fraction digits');
+        this.precision = precision;
+    }
+    // The pattern of the seconds' fraction this precision requires.
+    get fraction() {
+        return this.precision === 0 ? '' : `\\.\\d{${this.precision}}`;
+    }
+    // Whether a `YYYY-MM-DDTHH:MM:SS` names a real second of the calendar.
+    real(second) {
+        const instant = `${second}.000Z`;
+        const time = Date.parse(instant);
+        return Number.isFinite(time) && new Date(time).toISOString() === instant;
+    }
+}

package/dist/core/formats/timestamp-format.d.ts

@@ -0,0 +1,7 @@
+import { TemporalFormat } from './temporal-format.ts';
+export declare class TimestampFormat extends TemporalFormat {
+    #private;
+    readonly name = "date-time";
+    accepts(value: string): boolean;
+}
+export declare function isTimestamp(value: unknown): value is string;

package/dist/core/formats/timestamp-format.js

@@ -0,0 +1,14 @@
+import { TemporalFormat } from './temporal-format.js';
+// format: 'date-time': a UTC instant ending in Z.
+export class TimestampFormat extends TemporalFormat {
+    name = 'date-time';
+    #pattern = new RegExp(`^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}${this.fraction}Z$`);
+    accepts(value) {
+        return this.#pattern.test(value) && this.real(value.slice(0, 19));
+    }
+}
+const milliseconds = new TimestampFormat();
+// A UTC timestamp exactly as Date#toISOString writes it.
+export function isTimestamp(value) {
+    return typeof value === 'string' && milliseconds.accepts(value);
+}

package/dist/core/identifiers.d.ts

@@ -0,0 +1,10 @@
+export declare abstract class Identifiers {
+    #private;
+    protected abstract readonly maxBytes: number;
+    protected abstract readonly ownColumns: readonly string[];
+    abstract key(name: string): string;
+    columns<Column extends {
+        readonly field: string;
+    }>(columns: readonly Column[]): readonly (readonly [Column, string])[];
+    relation(name: string): string;
+}

package/dist/core/identifiers.js

@@ -0,0 +1,51 @@
+import { createHash } from 'node:crypto';
+function hash(name) {
+    return createHash('sha256').update(name).digest('hex').slice(0, 8);
+}
+// Airbyte's rule: decomposed marks dropped, every run of whitespace and every
+// other character outside [A-Za-z0-9_] replaced by `_`, case kept, and a
+// leading digit prefixed with `_`. A name starting with `_elt_` drops its
+// first `_`, since elt names what it creates for itself that way.
+function sanitize(name) {
+    const clean = name
+        .normalize('NFKD')
+        .replace(/\p{M}/gu, '')
+        .replace(/\s+/gu, '_')
+        .replace(/[^A-Za-z0-9_]/gu, '_')
+        .replace(/^_(?=elt_)/i, '');
+    if (clean === '')
+        return '_';
+    return /^[0-9]/.test(clean) ? `_${clean}` : clean;
+}
+// How a destination names the columns, tables and views it creates from a
+// stream, so that no name it cannot hold, or holds for itself, fails a load.
+export class Identifiers {
+    // Each column with its name, in field order: a sanitized name another
+    // field already took is tried again with `_1`, `_2` and so on after the
+    // field.
+    columns(columns) {
+        const taken = new Set(this.ownColumns.map((name) => this.key(name)));
+        return columns.map((column) => {
+            const { field } = column;
+            let name = this.#fit(sanitize(field), field);
+            for (let suffix = 1; taken.has(this.key(name)); suffix += 1)
+                name = this.#fit(sanitize(`${field}_${suffix}`), `${field}_${suffix}`);
+            taken.add(this.key(name));
+            return [column, name];
+        });
+    }
+    // A table or view sees no other name to collide with, so one that
+    // sanitizing changed keeps a hash of what it was asked for.
+    relation(name) {
+        const clean = sanitize(name);
+        return clean === name
+            ? this.#fit(name, name)
+            : this.#fit(`${clean}_${hash(name)}`, name);
+    }
+    // A sanitized name is ASCII, so cutting it by bytes keeps whole characters.
+    #fit(name, original) {
+        if (Buffer.byteLength(name) <= this.maxBytes)
+            return name;
+        return `${name.slice(0, this.maxBytes - 9)}_${hash(original)}`;
+    }
+}

package/dist/core/interleave.d.ts

@@ -0,0 +1 @@
+export declare function interleave<T>(generators: readonly AsyncGenerator<T>[], concurrency: number): AsyncGenerator<T>;

package/dist/core/interleave.js

@@ -0,0 +1,38 @@
+// Up to concurrency generators run together, each value yielded as it
+// arrives. A generator is asked for its next value only after the consumer
+// took its last one, so whatever that value refers to stays valid until the
+// consumer advances, whichever generator it came from.
+export async function* interleave(generators, concurrency) {
+    const waiting = [...generators];
+    const reading = new Map();
+    const next = (generator) => reading.set(generator, generator.next().then((result) => ({ generator, result })));
+    const start = () => {
+        const generator = waiting.shift();
+        if (generator !== undefined)
+            next(generator);
+    };
+    try {
+        while (reading.size < concurrency && waiting.length > 0)
+            start();
+        while (reading.size > 0) {
+            const { generator, result } = await Promise.race(reading.values());
+            if (result.done) {
+                reading.delete(generator);
+                start();
+                continue;
+            }
+            // Keep the yielding generator tracked so an early return closes it too.
+            yield result.value;
+            reading.delete(generator);
+            next(generator);
+        }
+    }
+    finally {
+        for (const [generator, pending] of reading) {
+            await pending.catch(() => undefined);
+            await generator.return(undefined);
+        }
+        for (const generator of waiting)
+            await generator.return(undefined);
+    }
+}

package/dist/core/partition.d.ts

@@ -0,0 +1,16 @@
+import { Deduplication } from './deduplication.ts';
+import type { KeyValue } from './source.ts';
+import type { Stream } from './stream.ts';
+export type Partition = Readonly<Record<string, KeyValue>>;
+export type PartitionState = {
+    readonly partition: Partition;
+    readonly state: unknown;
+};
+export declare function partitionIdentity(stream: Stream): Deduplication;
+export declare function assertPartitions(stream: Stream, partitions: readonly Partition[]): void;
+export declare function readPartition(identity: Deduplication, value: unknown): {
+    readonly key: string;
+    readonly partition: Partition;
+};
+export declare function assertInPartition(stream: Stream, partition: Partition, value: unknown): void;
+export declare function readPartitionStates(stream: Stream, state: unknown): Map<string, PartitionState>;

package/dist/core/partition.js

@@ -0,0 +1,60 @@
+import { Deduplication } from './deduplication.js';
+// The partitionKey fields reuse deduplication's scalar key rules and encoding.
+export function partitionIdentity(stream) {
+    if (stream.partitionKey === undefined)
+        throw new TypeError(`Stream ${stream.name} is not partitioned`);
+    return new Deduplication(stream, stream.partitionKey);
+}
+export function assertPartitions(stream, partitions) {
+    const identity = partitionIdentity(stream);
+    // A stream with no partitions reads nothing, as an Airbyte partition
+    // router with no values gives no slices.
+    if (!Array.isArray(partitions))
+        throw new TypeError(`Stream ${stream.name} lists no partitions`);
+    const keys = new Set();
+    for (const partition of partitions) {
+        const { key } = readPartition(identity, partition);
+        if (keys.has(key))
+            throw new TypeError(`Stream ${stream.name} lists partition ${key} twice`);
+        keys.add(key);
+    }
+}
+// Exactly the partitionKey fields, each a valid non-null scalar.
+export function readPartition(identity, value) {
+    if (value === null ||
+        typeof value !== 'object' ||
+        Object.keys(value).length !== identity.primaryKey.length)
+        throw new TypeError(`Stream ${identity.stream.name} partitions carry exactly ${JSON.stringify(identity.primaryKey)}`);
+    return {
+        key: identity.key(value),
+        partition: Object.freeze(Object.fromEntries(identity.primaryKey.map((field) => [
+            field,
+            identity.value(value, field),
+        ]))),
+    };
+}
+// Every record and deletion names the partition it was read for, so a
+// connector cannot load rows without, or under the wrong, partition value.
+export function assertInPartition(stream, partition, value) {
+    for (const [field, expected] of Object.entries(partition)) {
+        const actual = value !== null && typeof value === 'object'
+            ? Reflect.get(value, field)
+            : undefined;
+        if (actual !== expected)
+            throw new TypeError(`Stream ${stream.name} record for partition ${JSON.stringify(partition)} carries ${field} ${JSON.stringify(actual)}`);
+    }
+}
+// The checkpoint of a partitioned stream, as Source.read wrote it: each
+// partition's own source state.
+export function readPartitionStates(stream, state) {
+    const identity = partitionIdentity(stream);
+    const partitions = Reflect.get(Object(state), 'partitions');
+    if (partitions === undefined)
+        return new Map();
+    if (!Array.isArray(partitions))
+        throw new TypeError(`Stream ${stream.name} checkpoint holds partitions that are not a list`);
+    return new Map(partitions.map((entry) => {
+        const { key, partition } = readPartition(identity, Reflect.get(Object(entry), 'partition'));
+        return [key, { partition, state: Reflect.get(Object(entry), 'state') }];
+    }));
+}

package/dist/core/pass-channels.d.ts

@@ -0,0 +1,46 @@
+import type { Connection } from './connection.ts';
+import type { Copy, CopyOutcome } from './copy.ts';
+import type { FailureType } from './source.ts';
+import type { Target as DestinationTarget } from './target.ts';
+import type { WriteCount } from './writer.ts';
+export declare const passChannels: {
+    start: import("node:diagnostics_channel").Channel<any, any>;
+    progress: import("node:diagnostics_channel").Channel<any, any>;
+    finish: import("node:diagnostics_channel").Channel<any, any>;
+    failure: import("node:diagnostics_channel").Channel<any, any>;
+};
+/** A pass began: its sync history recorded the attempt. */
+export declare class PassStarted {
+    readonly connection: Connection<DestinationTarget>;
+    constructor(connection: Connection<DestinationTarget>);
+}
+/**
+ * One copy of a running pass so far, counted as Airbyte's sync stats tracker
+ * counts a stream: what the source emitted and what the destination
+ * committed. It ends once, complete or incomplete, as Airbyte's stream status
+ * does.
+ */
+export declare class CopyProgress {
+    readonly connection: Connection<DestinationTarget>;
+    readonly copy: Copy<DestinationTarget>;
+    readonly status: 'running' | 'complete' | 'incomplete';
+    readonly emitted: WriteCount;
+    readonly committed: WriteCount;
+    constructor(connection: Connection<DestinationTarget>, copy: Copy<DestinationTarget>, status: 'running' | 'complete' | 'incomplete', emitted: WriteCount, committed: WriteCount);
+}
+/** A pass ended, once its sync history recorded each copy's outcome. */
+export declare class PassFinished {
+    readonly connection: Connection<DestinationTarget>;
+    readonly outcomes: readonly CopyOutcome<DestinationTarget>[];
+    constructor(connection: Connection<DestinationTarget>, outcomes: readonly CopyOutcome<DestinationTarget>[]);
+}
+/**
+ * A connection produced no pass, such as an invalid connection or a failed
+ * change signal, once its sync history recorded why.
+ */
+export declare class PassFailed {
+    readonly connection: Connection<DestinationTarget>;
+    readonly error: unknown;
+    readonly failureType: FailureType;
+    constructor(connection: Connection<DestinationTarget>, error: unknown, failureType: FailureType);
+}

package/dist/core/pass-channels.js

@@ -0,0 +1,65 @@
+import { channel } from 'node:diagnostics_channel';
+// What each pass publishes while it runs, on node:diagnostics_channel
+// channels, as Node asks a module to publish what it does
+// (doc/api/diagnostics_channel.md) and as Got and undici publish their
+// requests. A subscriber gets each message as unknown and tells it by its
+// class. A subscriber that throws never stops the pass: Node reports the error
+// as an uncaught exception. Messages carry no scope: a host that runs several
+// passes tells its own by running them in an AsyncLocalStorage, as Node calls
+// a subscriber in the async context of the publisher.
+export const passChannels = {
+    start: channel('elt:pass.start'),
+    progress: channel('elt:copy.progress'),
+    finish: channel('elt:pass.finish'),
+    failure: channel('elt:pass.failure'),
+};
+/** A pass began: its sync history recorded the attempt. */
+export class PassStarted {
+    connection;
+    constructor(connection) {
+        this.connection = connection;
+    }
+}
+/**
+ * One copy of a running pass so far, counted as Airbyte's sync stats tracker
+ * counts a stream: what the source emitted and what the destination
+ * committed. It ends once, complete or incomplete, as Airbyte's stream status
+ * does.
+ */
+export class CopyProgress {
+    connection;
+    copy;
+    status;
+    emitted;
+    committed;
+    constructor(connection, copy, status, emitted, committed) {
+        this.connection = connection;
+        this.copy = copy;
+        this.status = status;
+        this.emitted = emitted;
+        this.committed = committed;
+    }
+}
+/** A pass ended, once its sync history recorded each copy's outcome. */
+export class PassFinished {
+    connection;
+    outcomes;
+    constructor(connection, outcomes) {
+        this.connection = connection;
+        this.outcomes = outcomes;
+    }
+}
+/**
+ * A connection produced no pass, such as an invalid connection or a failed
+ * change signal, once its sync history recorded why.
+ */
+export class PassFailed {
+    connection;
+    error;
+    failureType;
+    constructor(connection, error, failureType) {
+        this.connection = connection;
+        this.error = error;
+        this.failureType = failureType;
+    }
+}

package/dist/core/pipeline.d.ts

@@ -0,0 +1,37 @@
+import { type SyncHistory } from '../state/sync-history.ts';
+import type { PipelineClock } from './clock.ts';
+import { Connection } from './connection.ts';
+import { type Copy, type CopyOutcome, type CopyResult } from './copy.ts';
+import type { DerivedDataset } from './derived-dataset.ts';
+import type { Target as DestinationTarget } from './target.ts';
+export type Pass<Target extends DestinationTarget> = {
+    readonly connection: Connection<Target>;
+    readonly outcomes: readonly CopyOutcome<Target>[];
+};
+export declare class PipelineError<Target extends DestinationTarget> extends AggregateError {
+    name: string;
+    readonly results: readonly CopyOutcome<Target>[];
+    constructor(results: readonly CopyOutcome<Target>[]);
+}
+export declare class WatchError extends AggregateError {
+    name: string;
+}
+export declare class Pipeline<Target extends DestinationTarget> {
+    #private;
+    readonly connections: readonly Connection<Target>[];
+    readonly history: SyncHistory<Target>;
+    readonly derived: readonly DerivedDataset<Target>[];
+    constructor({ connections, history, derived, clock, }: {
+        connections: readonly Connection<Target>[];
+        history: SyncHistory<NoInfer<Target>>;
+        derived?: readonly DerivedDataset<NoInfer<Target>>[];
+        clock: PipelineClock;
+    });
+    clear(steps?: readonly Copy<Target>[]): Promise<void>;
+    run(options?: {
+        readonly signal?: AbortSignal;
+    }): Promise<CopyResult<Target>[]>;
+    watch({ signal, }: {
+        signal: AbortSignal;
+    }): AsyncGenerator<Pass<Target>>;
+}