@eltjs/elt 0.1.3 → 0.3.0

package/README.md

@@ -718,7 +718,7 @@ When a change signal fails:
 
 - A failed change signal fails only its streams. The other streams of the connection continue to load. A failed stream is not current as of the check.
 - The sync history records the failure one time, when the change signal starts to fail, as a failed pass of its streams with the failure type of the error.
-- After a `system` failure, such as a connection that its upstream dropped, the next check reads the gauge again, as Prometheus scrapes a failed target again at its next interval. A check that reads a new value loads the stream again.
+- After a `system` failure, such as a connection that its upstream dropped, the next check reads the gauge again, as Prometheus scrapes a failed target again at its next interval. The first check that reads the gauge loads its streams again, also when the value did not change. Thus the sync history has a pass after the failed one, and a stream that failed for one check does not show as failed until its next change.
 - After a `config` failure, the watch reads that gauge no more, and its streams stay failed, because only the user can fix it. A new watch reads the gauge again.
 - An upkeep that fails is recorded in the same way, as a failed pass of the streams of the connection. After a `system` failure it runs again at its next interval. After a `config` failure it runs no more in this watch.
 - Change signals that break the contract stop their connection: a change signal names a stream that the connection did not select, a selected stream is in no change signal, the change signals of one stream have two intervals or two timeouts, or an interval or a timeout is not a positive number of milliseconds. A `changeSignals()` that throws also stops its connection. The sync history records it as a failed attempt. The other connections continue.
@@ -826,7 +826,7 @@ The destination contract:
 
 - Declarations are frozen, and you can use them again.
 - `Destination.createWriter(configuration, target)` selects a strategy of the storage with no I/O.
-- `Destination.load()` opens the one hold of the pass on the storage.
+- `Destination.load({ clock, connection })` opens the one hold of the pass on the storage. `connection` is the name of the connection whose pass opens it, so a destination that records its loads can name each one.
 - `load.prepare(configuration, target, { writer, reloading })` gives each stream its `Stage`. A `Stage` has `apply`, `commit` and `discard`. It also has `fresh`, which is true when the target holds none of the earlier rows of the copy, so the copy reads from no checkpoint; `reloading`, which is true while commits go to a hidden target; and `complete`, which puts that hidden target in the place of the target after the stream ends with no failure.
 - Each destination uses `reloadMode` to decide what to do with the target: load into it, make it, evolve a stored target that the stream does not fit now ([shape changes](#shape-changes)), open a reload for an overwrite whose stored target does not fit now, or continue the reload that the last run of the copy left open. A `RESET` of the full stream also opens a reload.
 - Destinations own connections, files and publication. `Copy` and `Pipeline` have no SQL or file system code of a load.
@@ -1058,7 +1058,7 @@ Two rules exist:
 ## When a derived dataset fails
 
 - **A rule failure** is the output of its input: the status `failed` and the reason. The load and the run continue, and the next refresh tries the input again. See [ADR 0004](./docs/adr/0004-a-rule-failure-stays-on-its-output.md).
-- **A dataset failure**, for example SQL that SQLite refuses, rejects the run as itself, or it stops the watch.
+- **A dataset failure**, for example SQL that SQLite refuses, rejects the run, or it stops the watch, with a `DerivedDatasetError`. Its `dataset` names the dataset, and its `cause` is the error of the store. The passes that a stopped watch ended are `cancelled` in the sync history, and their records do not name the failure.
 - **A copy that failed** keeps its last committed rows. The derived data of those rows stays as it was.
 
 ## The same concepts in other systems

package/dist/core/copy.d.ts

@@ -1,4 +1,6 @@
 import type { CheckpointStore } from '../state/checkpoint-store.ts';
+import type { PipelineClock } from './clock.ts';
+import type { Connection } from './connection.ts';
 import { CopyConfiguration } from './copy-configuration.ts';
 import type { Destination } from './destination.ts';
 import type { Source } from './source.ts';
@@ -23,5 +25,5 @@ export declare class Copy<Target extends DestinationTarget> {
     get from(): Stream;
     writer(source: Source): string;
     validate(source: Source, destination: Destination<Target>, checkpoints?: CheckpointStore): void;
-    clear(source: Source, destination: Destination<Target>, checkpoints?: CheckpointStore): Promise<void>;
+    clear({ name, source, destination, checkpoints }: Connection<Target>, clock: PipelineClock): Promise<void>;
 }

package/dist/core/copy.js

@@ -44,10 +44,11 @@ export class Copy {
     }
     // Empties the target and removes this copy's checkpoint together, so the
     // next run reloads from scratch.
-    async clear(source, destination, checkpoints) {
+    async clear({ name, source, destination, checkpoints }, clock) {
         this.validate(source, destination, checkpoints);
-        const files = new FileTransfer(this.configuration.fileReads, destination.location(this.to), this.writer(source));
-        const drop = () => destination.clear(this.configuration, this.to, this.writer(source), (values) => files.reconcile(values));
+        const writer = this.writer(source);
+        const files = new FileTransfer(this.configuration.fileReads, destination.location(this.to), writer);
+        const drop = () => destination.clear(this.configuration, this.to, { writer, clock, connection: name }, (values) => files.reconcile(values));
         if (this.id !== undefined && checkpoints !== undefined)
             await checkpoints.clear(this.id, drop);
         else

package/dist/core/derived-dataset.d.ts

@@ -1,3 +1,4 @@
+import type { PipelineClock } from './clock.ts';
 import type { Connection } from './connection.ts';
 import type { Target as DestinationTarget } from './target.ts';
 export type RefreshTrigger = {
@@ -7,11 +8,17 @@ export type RefreshTrigger = {
     readonly kind: 'pass';
     readonly tried: ReadonlySet<string>;
 };
+export declare class DerivedDatasetError extends Error {
+    name: string;
+    readonly dataset: string;
+    constructor(dataset: string, cause: unknown);
+}
 export declare abstract class DerivedDataset<Target extends DestinationTarget> {
     abstract readonly name: string;
     validate(_connections: readonly Connection<Target>[]): void;
     abstract refresh(options: {
         readonly connections: readonly Connection<Target>[];
+        readonly clock: PipelineClock;
         readonly earlier: readonly DerivedDataset<Target>[];
         readonly signal?: AbortSignal;
         readonly trigger: RefreshTrigger;

package/dist/core/derived-dataset.js

@@ -1,3 +1,17 @@
+import { errorMessage } from '../state/sync-history.js';
+// A dataset's own failure, such as SQL its store refuses, named for whoever
+// reads it in a log: the cause alone, such as "no such column: x", does not say
+// which dataset of the pipeline failed.
+export class DerivedDatasetError extends Error {
+    name = 'DerivedDatasetError';
+    dataset;
+    constructor(dataset, cause) {
+        super(`Derived dataset ${dataset} failed: ${errorMessage(cause)}`, {
+            cause,
+        });
+        this.dataset = dataset;
+    }
+}
 // Data a pipeline keeps in step with what its copies load, and can always
 // rebuild from it (Kleppmann's derived data; Estuary's derivations), such as a
 // view over loaded tables or the text of the files a copy stored. Each kind

package/dist/core/derived-refreshes.d.ts

@@ -1,10 +1,11 @@
+import type { PipelineClock } from './clock.ts';
 import type { Connection } from './connection.ts';
-import type { DerivedDataset, RefreshTrigger } from './derived-dataset.ts';
+import { type DerivedDataset, type RefreshTrigger } from './derived-dataset.ts';
 import type { Target as DestinationTarget } from './target.ts';
 type Kind = RefreshTrigger['kind'];
 export declare class DerivedRefreshes<Target extends DestinationTarget> {
     #private;
-    constructor(datasets: readonly DerivedDataset<Target>[], connections: readonly Connection<Target>[], signal: AbortSignal | undefined, stop: (error: unknown) => void);
+    constructor(datasets: readonly DerivedDataset<Target>[], connections: readonly Connection<Target>[], clock: PipelineClock, signal: AbortSignal | undefined, stop: () => void);
     get failure(): {
         readonly error: unknown;
     } | undefined;

package/dist/core/derived-refreshes.js

@@ -1,4 +1,5 @@
 import { Latch } from '@zukhruf/async';
+import { DerivedDatasetError, } from './derived-dataset.js';
 // No refresh runs.
 class Idle {
 }
@@ -39,6 +40,7 @@ class Failed {
 export class DerivedRefreshes {
     #datasets;
     #connections;
+    #clock;
     #signal;
     #stop;
     #state = new Idle();
@@ -46,9 +48,10 @@ export class DerivedRefreshes {
     // refresh; the memory of the passes that run, which ends with them.
     #tried = new Map();
     #health = new Healthy();
-    constructor(datasets, connections, signal, stop) {
+    constructor(datasets, connections, clock, signal, stop) {
         this.#datasets = datasets;
         this.#connections = connections;
+        this.#clock = clock;
         this.#signal = signal;
         this.#stop = stop;
     }
@@ -84,12 +87,21 @@ export class DerivedRefreshes {
         for (const [index, dataset] of this.#datasets.entries()) {
             const tried = this.#tried.get(dataset) ?? new Set();
             this.#tried.set(dataset, tried);
-            await dataset.refresh({
-                connections: this.#connections,
-                earlier: this.#datasets.slice(0, index),
-                signal: this.#signal,
-                trigger: { kind, tried },
-            });
+            try {
+                await dataset.refresh({
+                    connections: this.#connections,
+                    clock: this.#clock,
+                    earlier: this.#datasets.slice(0, index),
+                    signal: this.#signal,
+                    trigger: { kind, tried },
+                });
+            }
+            catch (error) {
+                // A stop is not the dataset's failure.
+                if (this.#signal?.aborted === true)
+                    throw error;
+                throw new DerivedDatasetError(dataset.name, error);
+            }
         }
         // The passes it follows ended, and their memory with them.
         if (kind === 'pass')
@@ -109,7 +121,7 @@ export class DerivedRefreshes {
         catch (error) {
             if (this.#signal?.aborted !== true) {
                 this.#health = new Failed(error);
-                this.#stop(error);
+                this.#stop();
             }
         }
         finally {

package/dist/core/destination.d.ts

@@ -1,7 +1,7 @@
 import type { PipelineClock } from './clock.ts';
 import type { CopyConfiguration } from './copy-configuration.ts';
 import type { Target as DestinationTarget } from './target.ts';
-import type { FieldValues, Stage, Writer } from './writer.ts';
+import type { Clearing, FieldValues, Stage, Writer } from './writer.ts';
 export type DestinationSyncMode = 'append' | 'overwrite' | 'append_dedup' | 'overwrite_dedup';
 export type Load<Target extends DestinationTarget> = AsyncDisposable & {
     prepare(configuration: CopyConfiguration, target: Target, binding: {
@@ -17,6 +17,7 @@ export declare abstract class Destination<Target extends DestinationTarget> {
     abstract createWriter(configuration: CopyConfiguration, target: Target): Writer;
     abstract load(options: {
         clock: PipelineClock;
+        connection: string;
     }): Promise<Load<Target>>;
-    clear(configuration: CopyConfiguration, target: Target, writer: string, committed?: (values: FieldValues) => Promise<void>): Promise<void>;
+    clear(configuration: CopyConfiguration, target: Target, clearing: Clearing, committed?: (values: FieldValues) => Promise<void>): Promise<void>;
 }

package/dist/core/destination.js

@@ -11,7 +11,7 @@ export class Destination {
         if (!this.supportedDestinationSyncModes.includes(configuration.destinationSyncMode))
             throw new TypeError(`Destination does not support ${configuration.destinationSyncMode}`);
     }
-    async clear(configuration, target, writer, committed) {
-        await this.createWriter(configuration, target).clear(writer, committed);
+    async clear(configuration, target, clearing, committed) {
+        await this.createWriter(configuration, target).clear(clearing, committed);
     }
 }

package/dist/core/pipeline.js

@@ -72,7 +72,7 @@ export class Pipeline {
             const connection = this.connections.find(({ steps }) => steps.includes(copy));
             if (connection === undefined)
                 throw new TypeError("Only this pipeline's copies can be cleared");
-            await copy.clear(connection.source, connection.destination, connection.checkpoints);
+            await copy.clear(connection, this.#clock);
         }
     }
     // Once signal aborts, each pass stops at its next message and is recorded as
@@ -87,7 +87,7 @@ export class Pipeline {
     async run(options) {
         const signal = options?.signal;
         await this.#declare();
-        const refreshes = new DerivedRefreshes(this.derived, this.connections, signal, () => { });
+        const refreshes = new DerivedRefreshes(this.derived, this.connections, this.#clock, signal, () => { });
         let passes;
         try {
             passes = await Promise.allSettled(this.connections.map((connection) => this.#pass(connection, connection.steps, signal, () => refreshes.request('commit'))));
@@ -128,7 +128,10 @@ export class Pipeline {
         const controller = new AbortController();
         const watching = AbortSignal.any([signal, controller.signal]);
         const stopped = [];
-        const refreshes = new DerivedRefreshes(this.derived, this.connections, watching, (error) => controller.abort(error));
+        const refreshes = new DerivedRefreshes(this.derived, this.connections, this.#clock, watching, 
+        // With no reason: the passes it stops were not refused by this failure,
+        // and no sync history records a dataset's failure (ADR 0004).
+        () => controller.abort());
         const passes = interleave(this.connections
             .filter(({ steps }) => steps.length > 0)
             .map((connection) => this.#watch(connection, watching, controller, stopped, () => refreshes.request('commit'))), Number.POSITIVE_INFINITY);
@@ -275,8 +278,12 @@ export class Pipeline {
                                 }
                                 continue;
                             }
-                            failing.delete(name);
-                            if (outcome.state === 'changed') {
+                            // A stream whose change signal failed loads again when a check
+                            // reads the signal again, also with no new value: the history
+                            // shows the latest attempt of a stream, and only a pass follows
+                            // the failed one (ADR 0020).
+                            const recovered = failing.delete(name);
+                            if (outcome.state === 'changed' || recovered) {
                                 unchangedAt.delete(name);
                                 pending.add(name);
                             }

package/dist/core/replication.js

@@ -99,9 +99,9 @@ export async function replicate(source, destination, checkpoints, copies, connec
             });
     try {
         if (checkpoints === undefined || bindings.size === 0)
-            await transfer(source, destination, null, replications, signal, clock);
+            await transfer(source, destination, connection, null, replications, signal, clock);
         else
-            await checkpoints.run(bindings, (run) => transfer(source, destination, run, replications, signal, clock));
+            await checkpoints.run(bindings, (run) => transfer(source, destination, connection, run, replications, signal, clock));
     }
     catch (error) {
         for (const replication of replications)
@@ -115,7 +115,7 @@ export async function replicate(source, destination, checkpoints, copies, connec
         replication.settle();
     return replications.map((replication) => replication.outcome());
 }
-async function transfer(source, destination, run, replications, signal, clock) {
+async function transfer(source, destination, connection, run, replications, signal, clock) {
     const byStream = new Map(replications.map((replication) => [replication.stream.name, replication]));
     const incremental = (replication) => replication.copy.configuration.syncMode === 'incremental';
     const checkpoint = (replication) => {
@@ -153,7 +153,10 @@ async function transfer(source, destination, run, replications, signal, clock) {
     if (reading.length === 0)
         return;
     signal?.throwIfAborted();
-    await using load = await destination.load({ clock });
+    await using load = await destination.load({
+        clock,
+        connection: connection.name,
+    });
     // Every target is prepared before anything is read; a copy whose target is
     // refused reads nothing.
     const prepared = [];

package/dist/core/writer.d.ts

@@ -1,3 +1,4 @@
+import type { PipelineClock } from './clock.ts';
 import type { Partition } from './partition.ts';
 import type { FailureType, KeyValue } from './source.ts';
 import type { Stream } from './stream.ts';
@@ -37,5 +38,10 @@ export type Stage = AsyncDisposable & {
 export declare abstract class Writer {
     readonly stream: Stream;
     constructor(stream: Stream);
-    abstract clear(writer: string, committed?: (values: FieldValues) => Promise<void>): Promise<void>;
+    abstract clear(clearing: Clearing, committed?: (values: FieldValues) => Promise<void>): Promise<void>;
 }
+export type Clearing = {
+    readonly writer: string;
+    readonly clock: PipelineClock;
+    readonly connection: string;
+};

package/dist/index.d.ts

@@ -5,7 +5,7 @@ export { Copy, type CopyOutcome, type CopyResult } from './core/copy.ts';
 export { CopyConfiguration, type DedupPolicy, } from './core/copy-configuration.ts';
 export type { Deduplication } from './core/deduplication.ts';
 export { Destination, type DestinationSyncMode, type Load, } from './core/destination.ts';
-export { DerivedDataset, type RefreshTrigger } from './core/derived-dataset.ts';
+export { DerivedDataset, DerivedDatasetError, type RefreshTrigger, } from './core/derived-dataset.ts';
 export { DocumentParser } from './core/document-parser.ts';
 export { DocumentText } from './core/document-text.ts';
 export { type LoadedRow, Rule, type RuleColumn, type RuleResult, } from './core/rule.ts';
@@ -33,7 +33,7 @@ export { type Upkeep } from './core/upkeep.ts';
 export { Stream, type SyncMode } from './core/stream.ts';
 export { Target } from './core/target.ts';
 export { type DescribedColumn, describeTarget, type TargetDescription, } from './core/target-description.ts';
-export { type FieldValues, type LoadFailure, type Stage, TargetOwnedError, type WriteCount, type WriteOperation, Writer, } from './core/writer.ts';
+export { type Clearing, type FieldValues, type LoadFailure, type Stage, TargetOwnedError, type WriteCount, type WriteOperation, Writer, } from './core/writer.ts';
 export { type CheckpointBinding, type CheckpointRun, type CheckpointSession, CheckpointStore, type StoredCheckpoint, } from './state/checkpoint-store.ts';
 export { copyStatus, type DeclaredCopy, errorMessage, passError, passFailureType, passStatus, type RecordedPass, recordedFailure, type StreamCheck, SyncHistory, type SyncStatus, } from './state/sync-history.ts';
 export { syncHistoryRelations } from './state/sync-history-relations.ts';

package/dist/index.js

@@ -3,7 +3,7 @@ export { Connection } from './core/connection.js';
 export { Copy } from './core/copy.js';
 export { CopyConfiguration, } from './core/copy-configuration.js';
 export { Destination, } from './core/destination.js';
-export { DerivedDataset } from './core/derived-dataset.js';
+export { DerivedDataset, DerivedDatasetError, } from './core/derived-dataset.js';
 export { DocumentParser } from './core/document-parser.js';
 export { DocumentText } from './core/document-text.js';
 export { Rule, } from './core/rule.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@eltjs/elt",
-  "version": "0.1.3",
+  "version": "0.3.0",
   "description": "A pipeline that copies the data of apps and services into stores that SQL can read: sources, streams, checkpoints, watches and derived datasets.",
   "license": "Apache-2.0",
   "repository": {