@eltjs/elt 0.0.0-stage → 0.1.1

package/dist/state/sync-history.d.ts

@@ -0,0 +1,37 @@
+import type { Connection } from '../core/connection.ts';
+import { type Copy, type CopyOutcome } from '../core/copy.ts';
+import type { ExtractionCoverage, FailureType } from '../core/source.ts';
+import type { Target as DestinationTarget } from '../core/target.ts';
+import type { LoadFailure } from '../core/writer.ts';
+export type SyncStatus = 'succeeded' | 'partial' | 'failed' | 'cancelled';
+export type DeclaredCopy<Target extends DestinationTarget> = {
+    readonly copy: Copy<Target>;
+    readonly coverage: ExtractionCoverage;
+};
+export type StreamCheck = {
+    readonly stream: string;
+    readonly at: Date;
+};
+export type RecordedPass<Target extends DestinationTarget> = {
+    finish(outcomes: readonly CopyOutcome<Target>[], upstream: ReadonlyMap<string, Date>, at: Date): Promise<void>;
+    fail(error: unknown, failureType: FailureType, at: Date): Promise<void>;
+};
+export declare abstract class SyncHistory<Target extends DestinationTarget> {
+    #private;
+    validate(_connection: Connection<Target>): void;
+    abstract install(connections: readonly Connection<Target>[], at: Date): Promise<void>;
+    abstract begin(connection: Connection<Target>, copies: readonly DeclaredCopy<Target>[], at: Date): Promise<RecordedPass<Target>>;
+    checked(connection: Connection<Target>, at: Date, current: (at: Date) => Promise<readonly StreamCheck[]>): Promise<void>;
+    protected abstract recordChecks(connection: Connection<Target>, checks: readonly StreamCheck[]): Promise<void>;
+}
+export declare function recordedFailure({ partition, error, failureType, }: LoadFailure): {
+    partition: Readonly<Record<string, import("../core/source.ts").KeyValue>> | null;
+    error: string;
+    failure_type: FailureType;
+    stacktrace: string;
+};
+export declare function errorMessage(error: unknown): string;
+export declare function copyStatus({ count, deleted, failures, cancelled, }: CopyOutcome<DestinationTarget>): SyncStatus;
+export declare function passStatus(outcomes: readonly CopyOutcome<DestinationTarget>[]): SyncStatus;
+export declare function passFailureType(outcomes: readonly CopyOutcome<DestinationTarget>[]): FailureType | null;
+export declare function passError(outcomes: readonly CopyOutcome<DestinationTarget>[]): string | null;

package/dist/state/sync-history.js

@@ -0,0 +1,80 @@
+import { inspect } from 'node:util';
+import { describeFailures } from '../core/copy.js';
+// How often a history records the checks that found a connection's streams
+// unchanged. A source checks about every second; a reader needs
+// current_as_of only to about a minute, and every record is a write.
+const checkRecordMs = 60_000;
+// As Airbyte's platform keeps each connection's jobs and attempts apart from
+// any connector, the orchestrator records every pass here. A pass is one
+// connection's read over the copies it selected. A history reads no clock:
+// the pipeline gives the time of each event it records, from its own clock.
+export class SyncHistory {
+    // When each connection's checks were last recorded.
+    #checkedAt = new Map();
+    // Refuses, without I/O, a connection this history cannot record.
+    validate(_connection) { }
+    // At that moment a watch's source checked its streams and found them
+    // unchanged since their last copy loaded, so readers see them current as of
+    // then though no pass read them, as a Debezium heartbeat moves a connector's
+    // position with no change to carry. A connection's first check is recorded
+    // at once, and the next at most once a minute; current gives the streams
+    // to record only then, since asking a source can cost a process call.
+    async checked(connection, at, current) {
+        const last = this.#checkedAt.get(connection.name);
+        if (last !== undefined && at.getTime() - last < checkRecordMs)
+            return;
+        this.#checkedAt.set(connection.name, at.getTime());
+        await this.recordChecks(connection, await current(at));
+    }
+}
+// A failure as a history records it in a copy's failures. stacktrace is the
+// error as Node renders it, with its stack and every cause it chains, such as
+// the code of a refused connection behind "fetch failed", as Airbyte keeps a
+// FailureReason's stacktrace beside its message.
+export function recordedFailure({ partition, error, failureType, }) {
+    return {
+        partition,
+        error: errorMessage(error),
+        failure_type: failureType,
+        stacktrace: inspect(error),
+    };
+}
+// The message of a thrown value: an Error's message, or the value as a string,
+// since JavaScript can throw a value that is not an Error. Error.isError also
+// knows an Error from another realm, which instanceof Error does not.
+export function errorMessage(error) {
+    return Error.isError(error) ? error.message : String(error);
+}
+export function copyStatus({ count, deleted, failures, cancelled, }) {
+    if (cancelled)
+        return 'cancelled';
+    if (failures.length === 0)
+        return 'succeeded';
+    return count + deleted > 0 ? 'partial' : 'failed';
+}
+export function passStatus(outcomes) {
+    if (outcomes.some(({ cancelled }) => cancelled))
+        return 'cancelled';
+    if (outcomes.every(({ failures }) => failures.length === 0))
+        return 'succeeded';
+    return outcomes.some((outcome) => copyStatus(outcome) !== 'failed')
+        ? 'partial'
+        : 'failed';
+}
+// Whose a pass's failures are to fix: config when any one is the user's, null
+// when nothing failed or the run was stopped on request.
+export function passFailureType(outcomes) {
+    const status = passStatus(outcomes);
+    if (status === 'succeeded' || status === 'cancelled')
+        return null;
+    return outcomes.some(({ failures }) => failures.some(({ failureType }) => failureType === 'config'))
+        ? 'config'
+        : 'system';
+}
+// What did not load in a pass, or null when every copy loaded completely.
+export function passError(outcomes) {
+    const incomplete = outcomes.filter(({ failures }) => failures.length > 0);
+    if (incomplete.length === 0)
+        return null;
+    return `The following copies did not load completely: ${incomplete.map(describeFailures).join('; ')}`;
+}

package/dist/storage/local-files.d.ts

@@ -0,0 +1,15 @@
+import type { FileContent } from '../core/file-content.ts';
+import { FileStorage } from '../core/file-storage.ts';
+export declare class LocalFiles extends FileStorage {
+    readonly directory: string;
+    readonly identity: string;
+    readonly reference = "Path of a copy of the source file bytes, relative to the folder the load stores files in. Named by content hash, keeping a short source extension, and kept only while a row references it. Not the source file name, a URL or extracted text.";
+    constructor({ directory }: {
+        directory: string;
+    });
+    private scopeFolder;
+    private inspect;
+    save(scope: string, content: FileContent): Promise<string>;
+    flush(scope: string): Promise<void>;
+    retain(scope: string, references: ReadonlySet<string>): Promise<void>;
+}

package/dist/storage/local-files.js

@@ -0,0 +1,121 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { link, lstat, mkdir, open, readdir, rm } from 'node:fs/promises';
+import { dirname, join, resolve } from 'node:path';
+import { fileExtension } from '../core/file-extension.js';
+import { FileStorage } from '../core/file-storage.js';
+async function syncDirectory(path) {
+    await using directory = await open(path, 'r');
+    await directory.sync();
+}
+export class LocalFiles extends FileStorage {
+    directory;
+    identity;
+    reference = 'Path of a copy of the source file bytes, relative to the folder the load stores files in. Named by content hash, keeping a short source extension, and kept only while a row references it. Not the source file name, a URL or extracted text.';
+    constructor({ directory }) {
+        super();
+        if (typeof directory !== 'string' ||
+            directory.length === 0 ||
+            directory.includes('\0') ||
+            !directory.isWellFormed())
+            throw new TypeError('LocalFiles requires a directory');
+        this.directory = resolve(directory);
+        // References are relative to the folder, so the folder is no part of the
+        // identity: a store moved with its files keeps every reference.
+        this.identity = JSON.stringify({ type: 'local-files' });
+        Object.freeze(this);
+    }
+    // The scope's folder, relative to the storage folder.
+    scopeFolder(scope) {
+        return join('.elt-files', createHash('sha256').update(scope).digest('hex'));
+    }
+    async inspect(path) {
+        const info = await lstat(path).catch((error) => {
+            if (error.code !== 'ENOENT')
+                throw error;
+            return undefined;
+        });
+        if (info === undefined)
+            return false;
+        if (!info.isDirectory())
+            throw new TypeError(`Managed attachment directories must be real directories: ${path}`);
+        return true;
+    }
+    async save(scope, content) {
+        const folder = this.scopeFolder(scope);
+        const directory = join(this.directory, folder);
+        // Sync newly created directory entries as well as the file publication.
+        const created = await mkdir(directory, { recursive: true });
+        for (const path of [this.directory, dirname(directory), directory])
+            await this.inspect(path);
+        if (created !== undefined) {
+            let path = directory;
+            while (path !== dirname(created)) {
+                await syncDirectory(path);
+                path = dirname(path);
+            }
+            await syncDirectory(path);
+        }
+        const temporary = join(directory, `.pending-${randomUUID()}`);
+        try {
+            const hash = createHash('sha256');
+            {
+                await using file = await open(temporary, 'wx', 0o600);
+                for await (const chunk of content.chunks(4 * 1024 * 1024)) {
+                    hash.update(chunk);
+                    await file.writeFile(chunk);
+                }
+                await file.sync();
+            }
+            const name = hash.digest('hex') + fileExtension(content.path);
+            const path = join(directory, name);
+            await link(temporary, path).catch(async (error) => {
+                if (error.code !== 'EEXIST')
+                    throw error;
+                if (!(await lstat(path)).isFile())
+                    throw new TypeError(`Attachment reference is not a regular file: ${path}`);
+            });
+            return join(folder, name);
+        }
+        finally {
+            await rm(temporary, { force: true });
+        }
+    }
+    async flush(scope) {
+        await syncDirectory(join(this.directory, this.scopeFolder(scope))).catch((error) => {
+            if (error.code !== 'ENOENT')
+                throw error;
+        });
+    }
+    async retain(scope, references) {
+        const folder = this.scopeFolder(scope);
+        const directory = join(this.directory, folder);
+        for (const path of [this.directory, dirname(directory), directory]) {
+            if (await this.inspect(path))
+                continue;
+            if (references.size > 0)
+                throw new Error(`Stored attachment directory is missing: ${directory}`);
+            return;
+        }
+        // ponytail: scan this field's managed directory at commit; use an object
+        // index if attachment counts make reconciliation expensive.
+        const missing = new Set(references);
+        const obsolete = [];
+        // Only this scope's saves write here, so every file no row references is
+        // obsolete, a crashed save's .pending- file included.
+        for (const entry of await readdir(directory, { withFileTypes: true })) {
+            const path = join(directory, entry.name);
+            if (!entry.isFile())
+                throw new TypeError(`Managed attachment is not a regular file: ${path}`);
+            const reference = join(folder, entry.name);
+            if (references.has(reference))
+                missing.delete(reference);
+            else
+                obsolete.push(path);
+        }
+        if (missing.size > 0)
+            throw new Error(`Stored attachment is missing or outside its scope: ${[...missing][0]}`);
+        for (const path of obsolete)
+            await rm(path);
+        await syncDirectory(directory);
+    }
+}

package/package.json

@@ -1,6 +1,50 @@
 {
   "name": "@eltjs/elt",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.1",
+  "description": "A pipeline that copies the data of apps and services into stores that SQL can read: sources, streams, checkpoints, watches and derived datasets.",
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/ezzabuzaid/elt",
+    "directory": "packages/elt"
+  },
+  "bugs": {
+    "url": "https://github.com/ezzabuzaid/elt/issues"
+  },
+  "homepage": "https://github.com/ezzabuzaid/elt/tree/main/packages/elt#readme",
+  "keywords": [
+    "elt",
+    "etl",
+    "pipeline",
+    "data-pipeline",
+    "change-data-capture",
+    "sync"
+  ],
+  "type": "module",
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "!dist/spec/**",
+    "!**/*.tsbuildinfo",
+    "!**/*.test.*"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "engines": {
+    "node": ">=24"
+  },
+  "dependencies": {
+    "@zukhruf/async": "^0.8.10",
+    "@zukhruf/snapshot-differential": "^0.8.10"
+  },
+  "devDependencies": {
+    "@zukhruf/testing": "^0.8.10"
+  }
+}