@eltjs/elt 0.0.0-stage → 0.1.1

package/dist/core/destination.js

@@ -0,0 +1,17 @@
+import { isDeepStrictEqual } from 'node:util';
+export class Destination {
+    // Validate declarations without storage I/O; destinations check their own targets.
+    validate(configuration, target) {
+        this.createWriter(configuration, target);
+    }
+    validateConfiguration(configuration, target) {
+        configuration.validateSelection();
+        if (!isDeepStrictEqual(configuration.fileReads, target.fileReads))
+            throw new TypeError('Copy file reads must match the destination target');
+        if (!this.supportedDestinationSyncModes.includes(configuration.destinationSyncMode))
+            throw new TypeError(`Destination does not support ${configuration.destinationSyncMode}`);
+    }
+    async clear(configuration, target, writer, committed) {
+        await this.createWriter(configuration, target).clear(writer, committed);
+    }
+}

package/dist/core/document-parser.d.ts

@@ -0,0 +1,6 @@
+export declare abstract class DocumentParser {
+    readonly identity: string;
+    constructor(identity: string);
+    abstract parse(path: string, signal?: AbortSignal): Promise<string | null>;
+    identityFor(_path: string): Promise<string | null>;
+}

package/dist/core/document-parser.js

@@ -0,0 +1,15 @@
+// A source-side Strategy: one staged document becomes one content value.
+export class DocumentParser {
+    identity;
+    constructor(identity) {
+        this.identity = identity;
+        if (!identity || identity.includes('\0'))
+            throw new TypeError('A parser requires a stable identity');
+    }
+    // A composite parser can identify the selected format independently, so
+    // changing an audio model does not invalidate unrelated document text.
+    // null means this parser does not handle the file; existing text can stay.
+    identityFor(_path) {
+        return Promise.resolve(this.identity);
+    }
+}

package/dist/core/document-text.d.ts

@@ -0,0 +1,28 @@
+import type { DocumentParser } from './document-parser.ts';
+import { type LoadedRow, Rule, type RuleResult } from './rule.ts';
+export declare class DocumentText extends Rule<string, string> {
+    #private;
+    readonly key: {
+        name: string;
+        description: string;
+    };
+    readonly output: {
+        readonly content: {
+            readonly type: 'text';
+            readonly description: 'The text the parser extracted. NULL unless status is computed.';
+        };
+    };
+    readonly batch = 1;
+    readonly concurrency: number;
+    constructor({ parser, files, column, }: {
+        parser: () => DocumentParser;
+        files: string;
+        column: string;
+    });
+    read(row: LoadedRow): {
+        readonly key: string;
+        readonly input: string;
+    } | null;
+    identity(ref: string): Promise<string | null>;
+    compute(refs: readonly string[], signal?: AbortSignal): Promise<readonly RuleResult<Readonly<Record<string, string>>>[]>;
+}

package/dist/core/document-text.js

@@ -0,0 +1,72 @@
+import { lstat } from 'node:fs/promises';
+import { availableParallelism } from 'node:os';
+import { join } from 'node:path';
+import { errorMessage } from '../state/sync-history.js';
+import { Rule } from './rule.js';
+// The text of each file a copy stored, read by a DocumentParser, so readers
+// can search attachments with SQL. A stored file's ref names its copy by the
+// SHA-256 of its bytes, so the ref is both the key and the input, and rows of
+// different tables that hold one file share its text.
+export class DocumentText extends Rule {
+    key = {
+        name: 'ref',
+        description: "Path of the stored copy, relative to the import's files folder: the value of the file column of every row whose file it is. Join that column to it.",
+    };
+    output = {
+        content: {
+            type: 'text',
+            description: 'The text the parser extracted. NULL unless status is computed.',
+        },
+    };
+    // One file per call, as many at once as the Mac has cores; an image read
+    // beyond the OCR workers waits in the host's pool.
+    batch = 1;
+    concurrency = availableParallelism();
+    #parser;
+    #files;
+    #column;
+    // parser: asked once per refresh, so a host's changed choice, such as a
+    // speech model, applies from the next refresh. files: the folder the
+    // column's refs are relative to.
+    constructor({ parser, files, column, }) {
+        super();
+        this.#parser = parser;
+        this.#files = files;
+        this.#column = column;
+        Object.freeze(this);
+    }
+    read(row) {
+        const ref = row[this.#column];
+        return typeof ref === 'string' ? { key: ref, input: ref } : null;
+    }
+    identity(ref) {
+        return this.#parser().identityFor(join(this.#files, ref));
+    }
+    async compute(refs, signal) {
+        const parser = this.#parser();
+        return Promise.all(refs.map(async (ref) => {
+            const file = join(this.#files, ref);
+            try {
+                const content = await parser.parse(file, signal);
+                return content === null ? { empty: true } : { value: { content } };
+            }
+            catch (failure) {
+                if (signal?.aborted ||
+                    (failure instanceof Error && failure.name === 'AbortError'))
+                    return { gone: true };
+                // A commit removed the copy after the refresh listed it; its rows
+                // are gone too, so the file is not a failure.
+                if (await removed(file))
+                    return { gone: true };
+                return { error: errorMessage(failure) };
+            }
+        }));
+    }
+}
+async function removed(path) {
+    return lstat(path).then(() => false, (error) => {
+        if (error.code === 'ENOENT')
+            return true;
+        throw error;
+    });
+}

package/dist/core/file-content.d.ts

@@ -0,0 +1,5 @@
+export declare class FileContent {
+    readonly path: string;
+    constructor(path: string);
+    chunks(size: number): AsyncGenerator<Uint8Array>;
+}

package/dist/core/file-content.js

@@ -0,0 +1,24 @@
+import { open } from 'node:fs/promises';
+// A source file's bytes, read in bounded chunks so no file is ever held in
+// memory whole. Valid until the consumer advances past the record carrying it.
+export class FileContent {
+    path;
+    constructor(path) {
+        this.path = path;
+        Object.freeze(this);
+    }
+    async *chunks(size) {
+        if (!Number.isSafeInteger(size) || size <= 0)
+            throw new TypeError('Chunk size must be a positive integer');
+        await using file = await open(this.path);
+        let remaining = (await file.stat()).size;
+        while (remaining > 0) {
+            const chunk = new Uint8Array(Math.min(size, remaining));
+            const { bytesRead } = await file.read(chunk, 0, chunk.length, null);
+            if (bytesRead === 0)
+                throw new Error(`File ${this.path} shrank while it was read`);
+            remaining -= bytesRead;
+            yield chunk.subarray(0, bytesRead);
+        }
+    }
+}

package/dist/core/file-extension.d.ts

@@ -0,0 +1 @@
+export declare function fileExtension(name: string): string;

package/dist/core/file-extension.js

@@ -0,0 +1,9 @@
+import { extname } from 'node:path';
+// The extension a file named name keeps when a source stages it or a file
+// storage stores it: its own when it is 1 to 16 letters or digits, else none.
+// A name from an upstream is untrusted, and one byte past 255 in a file name
+// fails the write; a parser still picks a file's format by a short extension.
+export function fileExtension(name) {
+    const extension = extname(name);
+    return /^\.[a-zA-Z0-9]{1,16}$/.test(extension) ? extension : '';
+}

package/dist/core/file-read.d.ts

@@ -0,0 +1,20 @@
+import { DocumentParser } from './document-parser.ts';
+import { FileStorage } from './file-storage.ts';
+import type { Stream } from './stream.ts';
+export declare class FileReference {
+    readonly stream: Stream;
+    readonly storage?: FileStorage;
+    constructor(stream: Stream, storage?: FileStorage);
+    store(storage: FileStorage): FileReference;
+}
+export declare class FileRead {
+    readonly name: string;
+    readonly file: FileReference;
+    readonly parser?: DocumentParser;
+    readonly parserIdentity?: string;
+    readonly storageIdentity?: string;
+    constructor(name: string, file: FileReference, parser?: DocumentParser);
+    validate(stream: Stream): void;
+    toJSON(): object;
+    get outputType(): 'text' | 'bytes';
+}

package/dist/core/file-read.js

@@ -0,0 +1,79 @@
+import { DocumentParser } from './document-parser.js';
+import { FileStorage } from './file-storage.js';
+// A source resource, not a local path. Only the source can export its contents.
+export class FileReference {
+    stream;
+    storage;
+    constructor(stream, storage) {
+        this.stream = stream;
+        this.storage = storage;
+        if (storage !== undefined &&
+            (!(storage instanceof FileStorage) ||
+                typeof storage.identity !== 'string' ||
+                storage.identity.length === 0 ||
+                !storage.identity.isWellFormed() ||
+                storage.identity.includes('\0')))
+            throw new TypeError('File storage requires a stable text identity');
+        Object.freeze(this);
+    }
+    store(storage) {
+        return new FileReference(this.stream, storage);
+    }
+}
+// A named extraction request. A stored file becomes a scalar reference before
+// its destination receives it; parsing and byte reads remain separate choices.
+export class FileRead {
+    name;
+    file;
+    parser;
+    parserIdentity;
+    storageIdentity;
+    constructor(name, file, parser) {
+        this.name = name;
+        this.file = file;
+        this.parser = parser;
+        if (!name || name.includes('\0'))
+            throw new TypeError('Invalid file field name');
+        if (!(file instanceof FileReference))
+            throw new TypeError('File extraction requires a source file reference');
+        if (parser !== undefined && !(parser instanceof DocumentParser))
+            throw new TypeError('parser must be a DocumentParser');
+        if (parser !== undefined && file.storage !== undefined)
+            throw new TypeError('Select parsing and storage as separate file fields');
+        this.parserIdentity = parser?.identity;
+        this.storageIdentity = file.storage?.identity;
+        Object.freeze(this);
+    }
+    validate(stream) {
+        if (this.file.stream.name !== stream.name)
+            throw new TypeError('File reference belongs to another stream');
+        if (this.file.stream.supportsFileTransfer !== true ||
+            stream.supportsFileTransfer !== true)
+            throw new TypeError(`Stream ${stream.name} does not support file extraction`);
+        if (this.parser?.identity !== this.parserIdentity)
+            throw new TypeError('Parser identity changed after configuration');
+        if (this.file.storage?.identity !== this.storageIdentity)
+            throw new TypeError('File storage identity changed after configuration');
+        const { properties } = stream.jsonSchema;
+        if (stream.jsonSchema.type !== 'object' ||
+            properties === null ||
+            typeof properties !== 'object' ||
+            Array.isArray(properties))
+            throw new TypeError('File extraction requires an object metadata schema');
+        if (Object.keys(properties).some((name) => name.toLowerCase() === this.name.toLowerCase()))
+            throw new TypeError('File field collides with source metadata');
+    }
+    toJSON() {
+        return {
+            name: this.name,
+            stream: this.file.stream.name,
+            parser: this.parserIdentity,
+            storage: this.storageIdentity,
+        };
+    }
+    get outputType() {
+        return this.parser !== undefined || this.file.storage !== undefined
+            ? 'text'
+            : 'bytes';
+    }
+}

package/dist/core/file-storage.d.ts

@@ -0,0 +1,8 @@
+import type { FileContent } from './file-content.ts';
+export declare abstract class FileStorage {
+    abstract readonly identity: string;
+    abstract readonly reference: string;
+    abstract save(scope: string, content: FileContent): Promise<string>;
+    abstract flush(scope: string): Promise<void>;
+    abstract retain(scope: string, references: ReadonlySet<string>): Promise<void>;
+}

package/dist/core/file-storage.js

@@ -0,0 +1,4 @@
+// Owns attachment bytes, independently of the destination that holds references.
+// Each scope belongs to one target field. Never remove another scope's objects.
+export class FileStorage {
+}

package/dist/core/file-transfer.d.ts

@@ -0,0 +1,12 @@
+import type { FileRead } from './file-read.ts';
+import type { FieldValues } from './writer.ts';
+export declare class FileTransfer {
+    readonly reads: readonly FileRead[];
+    readonly location: string;
+    readonly writer: string;
+    constructor(reads: readonly FileRead[], location: string, writer: string);
+    private scope;
+    record(record: unknown): Promise<unknown>;
+    flush(): Promise<void>;
+    reconcile(values: FieldValues): Promise<void>;
+}

package/dist/core/file-transfer.js

@@ -0,0 +1,65 @@
+import { FileContent } from './file-content.js';
+// The shared transfer owns coordination. Stores own bytes; destinations only
+// expose the scalar values their rows actually retain after deduplication.
+export class FileTransfer {
+    reads;
+    location;
+    writer;
+    constructor(reads, location, writer) {
+        this.reads = reads;
+        this.location = location;
+        this.writer = writer;
+    }
+    scope(read) {
+        return JSON.stringify({
+            location: this.location,
+            writer: this.writer,
+            field: read.name,
+        });
+    }
+    async record(record) {
+        let data = record;
+        for (const read of this.reads) {
+            const storage = read.file.storage;
+            if (storage === undefined)
+                continue;
+            const content = Reflect.get(Object(data), read.name);
+            if (content === null)
+                continue;
+            if (!(content instanceof FileContent))
+                throw new TypeError('Stored files require source file content');
+            const reference = await storage.save(this.scope(read), content);
+            if (typeof reference !== 'string' ||
+                !reference ||
+                reference.includes('\0') ||
+                !reference.isWellFormed())
+                throw new TypeError('File storage must return a nonempty text reference');
+            data = { ...Object(data), [read.name]: reference };
+        }
+        return data;
+    }
+    async flush() {
+        for (const read of this.reads) {
+            const storage = read.file.storage;
+            if (storage === undefined)
+                continue;
+            await storage.flush(this.scope(read));
+        }
+    }
+    async reconcile(values) {
+        for (const read of this.reads) {
+            const storage = read.file.storage;
+            if (storage === undefined)
+                continue;
+            const references = new Set();
+            for await (const value of values(read.name)) {
+                if (value === null)
+                    continue;
+                if (typeof value !== 'string')
+                    throw new TypeError('Stored file references must be text or null');
+                references.add(value);
+            }
+            await storage.retain(this.scope(read), references);
+        }
+    }
+}

package/dist/core/formats/base64-format.d.ts

@@ -0,0 +1,6 @@
+import { FieldFormat } from './field-format.ts';
+export declare class Base64Format extends FieldFormat {
+    readonly name = "base64";
+    readonly orderable = false;
+    accepts(value: string): boolean;
+}

package/dist/core/formats/base64-format.js

@@ -0,0 +1,13 @@
+import { FieldFormat } from './field-format.js';
+// contentEncoding: 'base64': bytes as padded RFC 4648 base64, the one
+// spelling its bytes have. The alphabet does not sort as bytes do, so it
+// cannot be a cursor.
+export class Base64Format extends FieldFormat {
+    name = 'base64';
+    orderable = false;
+    accepts(value) {
+        return (/^[A-Za-z0-9+/]*={0,2}$/.test(value) &&
+            value.length % 4 === 0 &&
+            Buffer.from(value, 'base64').toString('base64') === value);
+    }
+}

package/dist/core/formats/calendar-date-format.d.ts

@@ -0,0 +1,6 @@
+import { FieldFormat } from './field-format.ts';
+export declare class CalendarDateFormat extends FieldFormat {
+    readonly name = "date";
+    accepts(value: string): boolean;
+}
+export declare function isCalendarDate(value: unknown): value is string;

package/dist/core/formats/calendar-date-format.js

@@ -0,0 +1,13 @@
+import { FieldFormat } from './field-format.js';
+import { isTimestamp } from './timestamp-format.js';
+// format: 'date': a day of the Gregorian calendar.
+export class CalendarDateFormat extends FieldFormat {
+    name = 'date';
+    accepts(value) {
+        return (/^\d{4}-\d{2}-\d{2}$/.test(value) && isTimestamp(`${value}T00:00:00.000Z`));
+    }
+}
+const day = new CalendarDateFormat();
+export function isCalendarDate(value) {
+    return typeof value === 'string' && day.accepts(value);
+}

package/dist/core/formats/decimal-format.d.ts

@@ -0,0 +1,9 @@
+import { FieldFormat } from './field-format.ts';
+export declare class DecimalFormat extends FieldFormat {
+    readonly name = "decimal";
+    readonly orderable = false;
+    readonly precision: number | undefined;
+    readonly scale: number | undefined;
+    constructor(precision?: number, scale?: number);
+    accepts(value: string): boolean;
+}

package/dist/core/formats/decimal-format.js

@@ -0,0 +1,41 @@
+import { FieldFormat } from './field-format.js';
+// format: 'decimal': an exact decimal number as text, never through a binary
+// float. With a scale, a value has exactly that many fraction digits, as SQL
+// writes NUMERIC(p,s) as text; without one, its fraction has no trailing
+// zeros. Either way each value has one spelling, so equal keys are equal
+// text. Text does not sort as the numbers do, so it cannot be a cursor.
+export class DecimalFormat extends FieldFormat {
+    name = 'decimal';
+    orderable = false;
+    // Most significant digits a value has, before and after the point.
+    precision;
+    scale;
+    constructor(precision, scale) {
+        super();
+        if (precision !== undefined &&
+            (!Number.isSafeInteger(precision) || precision < 1))
+            throw new TypeError('Decimal precision must be a positive integer');
+        if (scale !== undefined &&
+            (precision === undefined ||
+                !Number.isSafeInteger(scale) ||
+                scale < 0 ||
+                scale > precision))
+            throw new TypeError('Decimal scale must be an integer from 0 to its precision');
+        this.precision = precision;
+        this.scale = scale;
+    }
+    accepts(value) {
+        const match = /^-?(0|[1-9]\d*)(?:\.(\d+))?$/.exec(value);
+        if (match === null)
+            return false;
+        const [, integer = '', fraction = ''] = match;
+        if (value.startsWith('-') && /^0+$/.test(integer + fraction))
+            return false;
+        if (this.scale === undefined
+            ? fraction.endsWith('0')
+            : fraction.length !== this.scale)
+            return false;
+        return (this.precision === undefined ||
+            (integer === '0' ? 0 : integer.length) + fraction.length <= this.precision);
+    }
+}

package/dist/core/formats/declared-format.d.ts

@@ -0,0 +1,12 @@
+import type { ScalarSchema } from '../record-validation.ts';
+import { Base64Format } from './base64-format.ts';
+import { CalendarDateFormat } from './calendar-date-format.ts';
+import { DecimalFormat } from './decimal-format.ts';
+import { Int64Format } from './int64-format.ts';
+import { LocalTimeFormat } from './local-time-format.ts';
+import { LocalTimestampFormat } from './local-timestamp-format.ts';
+import { TimestampFormat } from './timestamp-format.ts';
+export type DeclaredFormat = TimestampFormat | LocalTimestampFormat | LocalTimeFormat | CalendarDateFormat | Int64Format | DecimalFormat | Base64Format;
+export declare function declaredFormat(schema: ScalarSchema & {
+    readonly type: string | readonly string[];
+}): DeclaredFormat | null;

package/dist/core/formats/declared-format.js

@@ -0,0 +1,48 @@
+import { Base64Format } from './base64-format.js';
+import { CalendarDateFormat } from './calendar-date-format.js';
+import { DecimalFormat } from './decimal-format.js';
+import { Int64Format } from './int64-format.js';
+import { LocalTimeFormat } from './local-time-format.js';
+import { LocalTimestampFormat } from './local-timestamp-format.js';
+import { TimestampFormat } from './timestamp-format.js';
+// The format a field or array item declares, with the keywords that refine
+// it; null for one that declares none. Every format spells a string.
+export function declaredFormat(schema) {
+    const { format, precision, scale, contentEncoding } = schema;
+    if (format === undefined && contentEncoding === undefined) {
+        if (precision !== undefined || scale !== undefined)
+            throw new TypeError('precision and scale refine a format');
+        return null;
+    }
+    const types = typeof schema.type === 'string' ? [schema.type] : schema.type;
+    if (types.filter((type) => type !== 'null').join() !== 'string')
+        throw new TypeError(`${format === undefined ? `contentEncoding ${contentEncoding}` : format} spells a string`);
+    if (contentEncoding !== undefined) {
+        if (contentEncoding !== 'base64' ||
+            format !== undefined ||
+            precision !== undefined ||
+            scale !== undefined)
+            throw new TypeError('contentEncoding is base64 alone');
+        return new Base64Format();
+    }
+    if (scale !== undefined && format !== 'decimal')
+        throw new TypeError('scale refines a decimal');
+    if (precision !== undefined && (format === 'date' || format === 'int64'))
+        throw new TypeError(`format ${format} has no precision`);
+    switch (format) {
+        case 'date-time':
+            return new TimestampFormat(precision);
+        case 'date-time-local':
+            return new LocalTimestampFormat(precision);
+        case 'time-local':
+            return new LocalTimeFormat(precision);
+        case 'date':
+            return new CalendarDateFormat();
+        case 'int64':
+            return new Int64Format();
+        case 'decimal':
+            return new DecimalFormat(precision, scale);
+        default:
+            throw new TypeError(`Unsupported format ${String(format)}`);
+    }
+}

package/dist/core/formats/field-format.d.ts

@@ -0,0 +1,6 @@
+export declare abstract class FieldFormat {
+    abstract readonly name: string;
+    readonly orderable: boolean;
+    readonly ordering: string;
+    abstract accepts(value: string): boolean;
+}

package/dist/core/formats/field-format.js

@@ -0,0 +1,10 @@
+// The canonical spelling a string field declares through `format` or
+// `contentEncoding`, and how two canonical values order when a copy compares
+// them as cursors.
+export class FieldFormat {
+    // False when byte order is not the values' order, so the field cannot be a
+    // cursor.
+    orderable = true;
+    // How cursors of this format compare, as target descriptions state it.
+    ordering = 'by byte order';
+}

package/dist/core/formats/int64-format.d.ts

@@ -0,0 +1,6 @@
+import { FieldFormat } from './field-format.ts';
+export declare class Int64Format extends FieldFormat {
+    readonly name = "int64";
+    readonly ordering = "as 64-bit integers";
+    accepts(value: string): boolean;
+}

package/dist/core/formats/int64-format.js

@@ -0,0 +1,16 @@
+import { FieldFormat } from './field-format.js';
+const min = -(2n ** 63n);
+const max = 2n ** 63n - 1n;
+// format: 'int64': a signed 64-bit integer as decimal text without leading
+// zeros, so a value past Number.MAX_SAFE_INTEGER keeps every digit through
+// JSON. Its text does not sort as its value, so cursors compare the integers.
+export class Int64Format extends FieldFormat {
+    name = 'int64';
+    ordering = 'as 64-bit integers';
+    accepts(value) {
+        if (!/^-?(0|[1-9]\d*)$/.test(value) || value === '-0')
+            return false;
+        const integer = BigInt(value);
+        return integer >= min && integer <= max;
+    }
+}

package/dist/core/formats/local-time-format.d.ts

@@ -0,0 +1,6 @@
+import { TemporalFormat } from './temporal-format.ts';
+export declare class LocalTimeFormat extends TemporalFormat {
+    #private;
+    readonly name = "time-local";
+    accepts(value: string): boolean;
+}

package/dist/core/formats/local-time-format.js

@@ -0,0 +1,10 @@
+import { TemporalFormat } from './temporal-format.js';
+// format: 'time-local': a time of day with no zone, as SQL's TIME stores it.
+// JSON Schema's own `time` requires an offset, so it cannot carry one.
+export class LocalTimeFormat extends TemporalFormat {
+    name = 'time-local';
+    #pattern = new RegExp(`^\\d{2}:\\d{2}:\\d{2}${this.fraction}$`);
+    accepts(value) {
+        return (this.#pattern.test(value) && this.real(`1970-01-01T${value.slice(0, 8)}`));
+    }
+}

package/dist/core/formats/local-timestamp-format.d.ts

@@ -0,0 +1,6 @@
+import { TemporalFormat } from './temporal-format.ts';
+export declare class LocalTimestampFormat extends TemporalFormat {
+    #private;
+    readonly name = "date-time-local";
+    accepts(value: string): boolean;
+}

package/dist/core/formats/local-timestamp-format.js

@@ -0,0 +1,10 @@
+import { TemporalFormat } from './temporal-format.js';
+// format: 'date-time-local': a wall-clock date and time with no zone, as
+// SQL's TIMESTAMP WITHOUT TIME ZONE stores it.
+export class LocalTimestampFormat extends TemporalFormat {
+    name = 'date-time-local';
+    #pattern = new RegExp(`^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}${this.fraction}$`);
+    accepts(value) {
+        return this.#pattern.test(value) && this.real(value.slice(0, 19));
+    }
+}

package/dist/core/formats/temporal-format.d.ts

@@ -0,0 +1,7 @@
+import { FieldFormat } from './field-format.ts';
+export declare abstract class TemporalFormat extends FieldFormat {
+    readonly precision: number;
+    constructor(precision?: number);
+    protected get fraction(): string;
+    protected real(second: string): boolean;
+}