@eltjs/elt 0.1.2 → 0.2.0

package/README.md

@@ -1058,7 +1058,7 @@ Two rules exist:
 ## When a derived dataset fails
 
 - **A rule failure** is the output of its input: the status `failed` and the reason. The load and the run continue, and the next refresh tries the input again. See [ADR 0004](./docs/adr/0004-a-rule-failure-stays-on-its-output.md).
-- **A dataset failure**, for example SQL that SQLite refuses, rejects the run as itself, or it stops the watch.
+- **A dataset failure**, for example SQL that SQLite refuses, rejects the run, or it stops the watch, with a `DerivedDatasetError`. Its `dataset` names the dataset, and its `cause` is the error of the store. The passes that a stopped watch ended are `cancelled` in the sync history, and their records do not name the failure.
 - **A copy that failed** keeps its last committed rows. The derived data of those rows stays as it was.
 
 ## The same concepts in other systems

package/dist/core/derived-dataset.d.ts

@@ -7,6 +7,11 @@ export type RefreshTrigger = {
     readonly kind: 'pass';
     readonly tried: ReadonlySet<string>;
 };
+export declare class DerivedDatasetError extends Error {
+    name: string;
+    readonly dataset: string;
+    constructor(dataset: string, cause: unknown);
+}
 export declare abstract class DerivedDataset<Target extends DestinationTarget> {
     abstract readonly name: string;
     validate(_connections: readonly Connection<Target>[]): void;

package/dist/core/derived-dataset.js

@@ -1,3 +1,17 @@
+import { errorMessage } from '../state/sync-history.js';
+// A dataset's own failure, such as SQL its store refuses, named for whoever
+// reads it in a log: the cause alone, such as "no such column: x", does not say
+// which dataset of the pipeline failed.
+export class DerivedDatasetError extends Error {
+    name = 'DerivedDatasetError';
+    dataset;
+    constructor(dataset, cause) {
+        super(`Derived dataset ${dataset} failed: ${errorMessage(cause)}`, {
+            cause,
+        });
+        this.dataset = dataset;
+    }
+}
 // Data a pipeline keeps in step with what its copies load, and can always
 // rebuild from it (Kleppmann's derived data; Estuary's derivations), such as a
 // view over loaded tables or the text of the files a copy stored. Each kind

package/dist/core/derived-refreshes.d.ts

@@ -1,10 +1,10 @@
 import type { Connection } from './connection.ts';
-import type { DerivedDataset, RefreshTrigger } from './derived-dataset.ts';
+import { type DerivedDataset, type RefreshTrigger } from './derived-dataset.ts';
 import type { Target as DestinationTarget } from './target.ts';
 type Kind = RefreshTrigger['kind'];
 export declare class DerivedRefreshes<Target extends DestinationTarget> {
     #private;
-    constructor(datasets: readonly DerivedDataset<Target>[], connections: readonly Connection<Target>[], signal: AbortSignal | undefined, stop: (error: unknown) => void);
+    constructor(datasets: readonly DerivedDataset<Target>[], connections: readonly Connection<Target>[], signal: AbortSignal | undefined, stop: () => void);
     get failure(): {
         readonly error: unknown;
     } | undefined;

package/dist/core/derived-refreshes.js

@@ -1,4 +1,5 @@
 import { Latch } from '@zukhruf/async';
+import { DerivedDatasetError, } from './derived-dataset.js';
 // No refresh runs.
 class Idle {
 }
@@ -84,12 +85,20 @@ export class DerivedRefreshes {
         for (const [index, dataset] of this.#datasets.entries()) {
             const tried = this.#tried.get(dataset) ?? new Set();
             this.#tried.set(dataset, tried);
-            await dataset.refresh({
-                connections: this.#connections,
-                earlier: this.#datasets.slice(0, index),
-                signal: this.#signal,
-                trigger: { kind, tried },
-            });
+            try {
+                await dataset.refresh({
+                    connections: this.#connections,
+                    earlier: this.#datasets.slice(0, index),
+                    signal: this.#signal,
+                    trigger: { kind, tried },
+                });
+            }
+            catch (error) {
+                // A stop is not the dataset's failure.
+                if (this.#signal?.aborted === true)
+                    throw error;
+                throw new DerivedDatasetError(dataset.name, error);
+            }
         }
         // The passes it follows ended, and their memory with them.
         if (kind === 'pass')
@@ -109,7 +118,7 @@ export class DerivedRefreshes {
         catch (error) {
             if (this.#signal?.aborted !== true) {
                 this.#health = new Failed(error);
-                this.#stop(error);
+                this.#stop();
             }
         }
         finally {

package/dist/core/document-text.js

@@ -52,7 +52,7 @@ export class DocumentText extends Rule {
             }
             catch (failure) {
                 if (signal?.aborted ||
-                    (failure instanceof Error && failure.name === 'AbortError'))
+                    (Error.isError(failure) && failure.name === 'AbortError'))
                     return { gone: true };
                 // A commit removed the copy after the refresh listed it; its rows
                 // are gone too, so the file is not a failure.

package/dist/core/identifiers.js

@@ -1,6 +1,6 @@
-import { createHash } from 'node:crypto';
-function hash(name) {
-    return createHash('sha256').update(name).digest('hex').slice(0, 8);
+import { hash } from 'node:crypto';
+function shortHash(name) {
+    return hash('sha256', name).slice(0, 8);
 }
 // Airbyte's rule: decomposed marks dropped, every run of whitespace and every
 // other character outside [A-Za-z0-9_] replaced by `_`, case kept, and a
@@ -40,12 +40,12 @@ export class Identifiers {
         const clean = sanitize(name);
         return clean === name
             ? this.#fit(name, name)
-            : this.#fit(`${clean}_${hash(name)}`, name);
+            : this.#fit(`${clean}_${shortHash(name)}`, name);
     }
     // A sanitized name is ASCII, so cutting it by bytes keeps whole characters.
     #fit(name, original) {
         if (Buffer.byteLength(name) <= this.maxBytes)
             return name;
-        return `${name.slice(0, this.maxBytes - 9)}_${hash(original)}`;
+        return `${name.slice(0, this.maxBytes - 9)}_${shortHash(original)}`;
     }
 }

package/dist/core/pipeline.js

@@ -128,7 +128,10 @@ export class Pipeline {
         const controller = new AbortController();
         const watching = AbortSignal.any([signal, controller.signal]);
         const stopped = [];
-        const refreshes = new DerivedRefreshes(this.derived, this.connections, watching, (error) => controller.abort(error));
+        const refreshes = new DerivedRefreshes(this.derived, this.connections, watching, 
+        // With no reason: the passes it stops were not refused by this failure,
+        // and no sync history records a dataset's failure (ADR 0004).
+        () => controller.abort());
         const passes = interleave(this.connections
             .filter(({ steps }) => steps.length > 0)
             .map((connection) => this.#watch(connection, watching, controller, stopped, () => refreshes.request('commit'))), Number.POSITIVE_INFINITY);

package/dist/index.d.ts

@@ -5,7 +5,7 @@ export { Copy, type CopyOutcome, type CopyResult } from './core/copy.ts';
 export { CopyConfiguration, type DedupPolicy, } from './core/copy-configuration.ts';
 export type { Deduplication } from './core/deduplication.ts';
 export { Destination, type DestinationSyncMode, type Load, } from './core/destination.ts';
-export { DerivedDataset, type RefreshTrigger } from './core/derived-dataset.ts';
+export { DerivedDataset, DerivedDatasetError, type RefreshTrigger, } from './core/derived-dataset.ts';
 export { DocumentParser } from './core/document-parser.ts';
 export { DocumentText } from './core/document-text.ts';
 export { type LoadedRow, Rule, type RuleColumn, type RuleResult, } from './core/rule.ts';

package/dist/index.js

@@ -3,7 +3,7 @@ export { Connection } from './core/connection.js';
 export { Copy } from './core/copy.js';
 export { CopyConfiguration, } from './core/copy-configuration.js';
 export { Destination, } from './core/destination.js';
-export { DerivedDataset } from './core/derived-dataset.js';
+export { DerivedDataset, DerivedDatasetError, } from './core/derived-dataset.js';
 export { DocumentParser } from './core/document-parser.js';
 export { DocumentText } from './core/document-text.js';
 export { Rule, } from './core/rule.js';

package/dist/storage/local-files.js

@@ -1,4 +1,4 @@
-import { createHash, randomUUID } from 'node:crypto';
+import { createHash, hash, randomUUID } from 'node:crypto';
 import { link, lstat, mkdir, open, readdir, rm } from 'node:fs/promises';
 import { dirname, join, resolve } from 'node:path';
 import { fileExtension } from '../core/file-extension.js';
@@ -26,7 +26,7 @@ export class LocalFiles extends FileStorage {
     }
     // The scope's folder, relative to the storage folder.
     scopeFolder(scope) {
-        return join('.elt-files', createHash('sha256').update(scope).digest('hex'));
+        return join('.elt-files', hash('sha256', scope));
     }
     async inspect(path) {
         const info = await lstat(path).catch((error) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@eltjs/elt",
-  "version": "0.1.2",
+  "version": "0.2.0",
   "description": "A pipeline that copies the data of apps and services into stores that SQL can read: sources, streams, checkpoints, watches and derived datasets.",
   "license": "Apache-2.0",
   "repository": {