@chkit/plugin-ingest 0.2.0-beta.8

package/dist/incremental.js

@@ -0,0 +1,72 @@
+import { IngestConfigError } from './errors.js';
+/** No incremental selection: every execution reads the whole source. */
+export function fullSync() {
+    return {
+        id: 'chkit.full_sync',
+        version: 1,
+        parseState: () => undefined,
+        plan: () => undefined,
+    };
+}
+/**
+ * Bundled timestamp strategy. The upper bound is fixed to the execution cutoff
+ * and `range.to` becomes the committed watermark only after the whole window
+ * has sink evidence.
+ */
+export function timestampWindow(options) {
+    if (options.from === undefined) {
+        if (!(options.start instanceof Date) || !Number.isFinite(options.start.getTime())) {
+            throw new IngestConfigError('timestampWindow: start must be a valid Date.');
+        }
+        if (!Number.isFinite(options.overlapMs ?? 0) || (options.overlapMs ?? 0) < 0) {
+            throw new IngestConfigError('timestampWindow: overlapMs must be a finite non-negative number.');
+        }
+    }
+    else if (options.start !== undefined || options.overlapMs !== undefined) {
+        throw new IngestConfigError('timestampWindow: use either start/overlapMs or from, not both.');
+    }
+    return {
+        id: 'chkit.timestamp_window',
+        version: 1,
+        parseState(raw) {
+            if (typeof raw !== 'object' || raw === null || !('watermark' in raw) || typeof raw.watermark !== 'string') {
+                throw new IngestConfigError('timestampWindow: committed state has no string "watermark".');
+            }
+            if (Number.isNaN(Date.parse(raw.watermark))) {
+                throw new IngestConfigError(`timestampWindow: committed watermark "${raw.watermark}" is not a timestamp.`);
+            }
+            return { watermark: raw.watermark };
+        },
+        plan({ state, cutoff, range }) {
+            const to = range?.to ?? cutoff;
+            const watermark = state ? new Date(state.watermark) : undefined;
+            const from = range?.from ?? (options.from
+                ? options.from({ watermark, cutoff })
+                : new Date(watermark ? watermark.getTime() - (options.overlapMs ?? 0) : options.start.getTime()));
+            if (!Number.isFinite(from.getTime()) || !Number.isFinite(to.getTime())) {
+                throw new IngestConfigError('timestampWindow: planned bounds must be valid dates.');
+            }
+            if (from.getTime() > to.getTime()) {
+                throw new IngestConfigError(`timestampWindow: planned lower bound ${from.toISOString()} is after upper bound ${to.toISOString()}.`);
+            }
+            return { from, to };
+        },
+        complete({ selection }) {
+            return { watermark: selection.to.toISOString() };
+        },
+    };
+}
+/**
+ * Generic provider-owned state (compound cursor, change token, snapshot id…).
+ * The reader receives the committed state as its selection and advances it by
+ * yielding chunks that carry complete candidate state.
+ */
+export function cursorState(options) {
+    return {
+        id: options.id,
+        version: options.version,
+        parseState: options.parse,
+        plan: ({ state }) => state,
+    };
+}
+//# sourceMappingURL=incremental.js.map

package/dist/incremental.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"incremental.js","sourceRoot":"","sources":["../src/incremental.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAmC/C,wEAAwE;AACxE,MAAM,UAAU,QAAQ;IACtB,OAAO;QACL,EAAE,EAAE,iBAAiB;QACrB,OAAO,EAAE,CAAC;QACV,UAAU,EAAE,GAAG,EAAE,CAAC,SAAS;QAC3B,IAAI,EAAE,GAAG,EAAE,CAAC,SAAS;KACtB,CAAA;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAC7B,OAA+B;IAE/B,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC/B,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,YAAY,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;YAClF,MAAM,IAAI,iBAAiB,CAAC,8CAA8C,CAAC,CAAA;QAC7E,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7E,MAAM,IAAI,iBAAiB,CAAC,kEAAkE,CAAC,CAAA;QACjG,CAAC;IACH,CAAC;SAAM,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QAC1E,MAAM,IAAI,iBAAiB,CAAC,gEAAgE,CAAC,CAAA;IAC/F,CAAC;IACD,OAAO;QACL,EAAE,EAAE,wBAAwB;QAC5B,OAAO,EAAE,CAAC;QACV,UAAU,CAAC,GAAG;YACZ,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,IAAI,CAAC,CAAC,WAAW,IAAI,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;gBAC1G,MAAM,IAAI,iBAAiB,CAAC,6DAA6D,CAAC,CAAA;YAC5F,CAAC;YACD,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;gBAC5C,MAAM,IAAI,iBAAiB,CAAC,yCAAyC,GAAG,CAAC,SAAS,uBAAuB,CAAC,CAAA;YAC5G,CAAC;YACD,OAAO,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,CAAA;QACrC,CAAC;QACD,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE;YAC3B,MAAM,EAAE,GAAG,KAAK,EAAE,EAAE,IAAI,MAAM,CAAA;YAC9B,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;YAC/D,MAAM,IAAI,GAAG,KAAK,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI;gBACvC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;gBACrC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;YACnG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;gBACvE,MAAM,IAAI,iBAAiB,CAAC,sDAAsD,CAAC,CAAA;YACrF,CAAC;YACD,IAAI,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC;gBAClC,MAAM,IAAI,iBAAiB,CACzB,wCAAwC,IAAI,CAAC,WAAW,EAAE,yBAAyB,EAAE,CAAC,WAAW,EAAE,GAAG,CACvG,CAAA;YACH,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,CAAA;QACrB,CAAC;QACD,QAAQ,CAAC,EAAE,SAAS,EAAE;YACpB,OAAO,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC,WAAW,EAAE,EAAE,CAAA;QAClD,CAAC;KACF,CAAA;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAAS,OAAmC;IACrE,OAAO;QACL,EAAE,EAAE,OAAO,CAAC,EAAE;QACd,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,UAAU,EAAE,OAAO,CAAC,KAAK;QACzB,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK;KAC3B,CAAA;AACH,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,11 @@
+export { ingest, createIngestPlugin, checkGraph, type IngestPlugin, type IngestPluginOptions } from './plugin.js';
+export { defineStream, definePipeline, selectStreams, type SelectedStream } from './registry.js';
+export { fullSync, timestampWindow, cursorState, type TimestampRange, type TimestampWindowOptions, type TimestampWindowState } from './incremental.js';
+export { paginate, type Page } from './paginate.js';
+export { simpleLoader } from './loader.js';
+export { ingestionColumns, rawTable, rawRows, createClickHouseDestination, type RawRow } from './destination.js';
+export { createClickHouseJournal } from './journal.js';
+export { runIngestion, type BackfillRequest, type ExecutionEnv, type ExecutionRequest } from './executor.js';
+export { HttpError, FetchFailure, IngestConfigError } from './errors.js';
+export type { DestinationAdapter, ErrorClassifier, ExecutionResult, FailureClass, FetchContext, IncrementalStrategy, Journal, Loader, LoaderContext, LoaderFactory, PipelineDefinition, ReadContext, RetryOptions, Row, SinkReceipt, SourceChunk, StreamDefinition, StreamResult, } from './types.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,kBAAkB,EAAE,UAAU,EAAE,KAAK,YAAY,EAAE,KAAK,mBAAmB,EAAE,MAAM,aAAa,CAAA;AACjH,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,EAAE,KAAK,cAAc,EAAE,MAAM,eAAe,CAAA;AAChG,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,WAAW,EAAE,KAAK,cAAc,EAAE,KAAK,sBAAsB,EAAE,KAAK,oBAAoB,EAAE,MAAM,kBAAkB,CAAA;AACtJ,OAAO,EAAE,QAAQ,EAAE,KAAK,IAAI,EAAE,MAAM,eAAe,CAAA;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,OAAO,EAAE,2BAA2B,EAAE,KAAK,MAAM,EAAE,MAAM,kBAAkB,CAAA;AAChH,OAAO,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAA;AACtD,OAAO,EAAE,YAAY,EAAE,KAAK,eAAe,EAAE,KAAK,YAAY,EAAE,KAAK,gBAAgB,EAAE,MAAM,eAAe,CAAA;AAC5G,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AACxE,YAAY,EACV,kBAAkB,EAClB,eAAe,EACf,eAAe,EACf,YAAY,EACZ,YAAY,EACZ,mBAAmB,EACnB,OAAO,EACP,MAAM,EACN,aAAa,EACb,aAAa,EACb,kBAAkB,EAClB,WAAW,EACX,YAAY,EACZ,GAAG,EACH,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,YAAY,GACb,MAAM,YAAY,CAAA"}

package/dist/index.js

@@ -0,0 +1,10 @@
+export { ingest, createIngestPlugin, checkGraph } from './plugin.js';
+export { defineStream, definePipeline, selectStreams } from './registry.js';
+export { fullSync, timestampWindow, cursorState } from './incremental.js';
+export { paginate } from './paginate.js';
+export { simpleLoader } from './loader.js';
+export { ingestionColumns, rawTable, rawRows, createClickHouseDestination } from './destination.js';
+export { createClickHouseJournal } from './journal.js';
+export { runIngestion } from './executor.js';
+export { HttpError, FetchFailure, IngestConfigError } from './errors.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,kBAAkB,EAAE,UAAU,EAA+C,MAAM,aAAa,CAAA;AACjH,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,EAAuB,MAAM,eAAe,CAAA;AAChG,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,WAAW,EAA+E,MAAM,kBAAkB,CAAA;AACtJ,OAAO,EAAE,QAAQ,EAAa,MAAM,eAAe,CAAA;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,OAAO,EAAE,2BAA2B,EAAe,MAAM,kBAAkB,CAAA;AAChH,OAAO,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAA;AACtD,OAAO,EAAE,YAAY,EAAkE,MAAM,eAAe,CAAA;AAC5G,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA"}

package/dist/journal.d.ts

@@ -0,0 +1,44 @@
+import { type ClickHouseExecutor } from '@chkit/clickhouse';
+import type { CommittedCheckpoint, Journal, JournalEvent } from './types.js';
+export declare const DEFAULT_JOURNAL_TABLE = "_chkit_ingestion_journal";
+export type JournalRow = {
+    target_id: string;
+    namespace_id: string;
+    event_seq: string;
+    event_id: string;
+    payload_hash: string;
+    event_at: string;
+    event_kind: string;
+    run_id: string;
+    work_id: string;
+    attempt_no: number;
+    batch_id: string;
+    expected_checkpoint_version: string;
+    checkpoint_version: string;
+    checkpoint_json: string;
+    work_state: string;
+    sink_evidence: string;
+    retry_at: string | null;
+    error_class: string;
+    detail_json: string;
+};
+export interface ClickHouseJournalOptions {
+    /** Must allow concurrent queries: use a stateless executor, not a session-bound one. */
+    executor: ClickHouseExecutor;
+    database: string;
+    targetId: string;
+    table?: string;
+    now?: () => Date;
+}
+/**
+ * Append-only, target-linked journal. It is the sole authority for durable
+ * ingestion control state: checkpoints are read as a projection over
+ * `batch_committed` facts and never stored anywhere else.
+ */
+export declare function createClickHouseJournal(options: ClickHouseJournalOptions): Journal;
+export declare function toJournalRow(event: JournalEvent, targetId: string, at: Date): JournalRow;
+/** Stable key order so equal values always serialize identically. */
+export declare function canonicalJson(value: unknown): string;
+export declare function digest(parts: readonly string[]): string;
+export declare function emptyCheckpoint(): CommittedCheckpoint;
+//# sourceMappingURL=journal.d.ts.map

package/dist/journal.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"journal.d.ts","sourceRoot":"","sources":["../src/journal.ts"],"names":[],"mappings":"AAEA,OAAO,EAAgB,KAAK,kBAAkB,EAAE,MAAM,mBAAmB,CAAA;AAGzE,OAAO,KAAK,EAAsB,mBAAmB,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAEhG,eAAO,MAAM,qBAAqB,6BAA6B,CAAA;AAI/D,MAAM,MAAM,UAAU,GAAG;IACvB,SAAS,EAAE,MAAM,CAAA;IACjB,YAAY,EAAE,MAAM,CAAA;IACpB,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;IAChB,YAAY,EAAE,MAAM,CAAA;IACpB,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,MAAM,CAAA;IACf,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,MAAM,CAAA;IAChB,2BAA2B,EAAE,MAAM,CAAA;IACnC,kBAAkB,EAAE,MAAM,CAAA;IAC1B,eAAe,EAAE,MAAM,CAAA;IACvB,UAAU,EAAE,MAAM,CAAA;IAClB,aAAa,EAAE,MAAM,CAAA;IACrB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,MAAM,CAAA;CACpB,CAAA;AAED,MAAM,WAAW,wBAAwB;IACvC,wFAAwF;IACxF,QAAQ,EAAE,kBAAkB,CAAA;IAC5B,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,GAAG,CAAC,EAAE,MAAM,IAAI,CAAA;CACjB;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,wBAAwB,GAAG,OAAO,CA8GlF;AAED,wBAAgB,YAAY,CAAC,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,GAAG,UAAU,CAwCxF;AAkBD,qEAAqE;AACrE,wBAAgB,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAGpD;AAED,wBAAgB,MAAM,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAQvD;AAED,wBAAgB,eAAe,IAAI,mBAAmB,CAErD"}

package/dist/journal.js

@@ -0,0 +1,222 @@
+import { createHash } from 'node:crypto';
+import { waitForTable } from '@chkit/clickhouse';
+import { IngestConfigError } from './errors.js';
+export const DEFAULT_JOURNAL_TABLE = '_chkit_ingestion_journal';
+const TABLE_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/;
+/**
+ * Append-only, target-linked journal. It is the sole authority for durable
+ * ingestion control state: checkpoints are read as a projection over
+ * `batch_committed` facts and never stored anywhere else.
+ */
+export function createClickHouseJournal(options) {
+    const table = options.table ?? DEFAULT_JOURNAL_TABLE;
+    if (!TABLE_NAME_PATTERN.test(table))
+        throw new IngestConfigError(`Invalid journal table name "${table}".`);
+    if (!TABLE_NAME_PATTERN.test(options.database)) {
+        throw new IngestConfigError(`Invalid journal database name "${options.database}".`);
+    }
+    const qualified = `\`${options.database}\`.\`${table}\``;
+    const now = options.now ?? (() => new Date());
+    return {
+        async ensure() {
+            await options.executor.command(journalTableSql(qualified));
+            // Replicated/shared catalogs (ObsessionDB) can acknowledge the CREATE
+            // before the replica serving the next request knows the table.
+            await waitForTable(options.executor, options.database, table);
+        },
+        async append(event) {
+            const row = toJournalRow(event, options.targetId, now());
+            await options.executor.insert({
+                table: `${options.database}.${table}`,
+                values: [row],
+                // A retried append of the same deterministic fact is suppressed while
+                // the deduplication window lasts; readers canonicalize by event_id anyway.
+                settings: { insert_deduplication_token: row.event_id, async_insert: 0 },
+            });
+        },
+        async readCheckpoint(namespaceId) {
+            // Physical retry duplicates are allowed, so facts are canonicalized per
+            // sequence number first. The history is then validated before anything is
+            // projected from it: a checkpoint read from a damaged journal is worthless.
+            const facts = `SELECT
+    event_seq,
+    uniqExact(event_id) AS owners,
+    uniqExact(payload_hash) AS payloads,
+    any(event_kind) AS fact_kind,
+    any(work_state) AS fact_work_state,
+    any(expected_checkpoint_version) AS fact_expected,
+    any(checkpoint_version) AS fact_version,
+    any(checkpoint_json) AS fact_checkpoint
+  FROM ${qualified}
+  WHERE target_id = ${sqlString(options.targetId)} AND namespace_id = ${sqlString(namespaceId)}
+  GROUP BY event_seq`;
+            const settings = { select_sequential_consistency: '1' };
+            const [health, transitions] = await Promise.all([
+                options.executor.query(`SELECT
+  max(event_seq) AS head_seq,
+  maxIf(event_seq, fact_kind = 'work_finished' AND fact_work_state = 'succeeded') AS last_success_seq,
+  count() AS sequences,
+  countIf(owners > 1) AS conflicting_owners,
+  countIf(payloads > 1) AS drifted,
+  argMaxIf(fact_version, event_seq, fact_kind = 'batch_committed') AS checkpoint_version,
+  argMaxIf(fact_checkpoint, event_seq, fact_kind = 'batch_committed') AS checkpoint_json
+FROM (${facts})`, settings),
+                // Every commit must start from the version the previous commit produced,
+                // advance it by at most one, and only change the envelope when it advances.
+                // The two reads are not one snapshot; that is sound because V1 runs a
+                // single executor process and reads a namespace before appending to it.
+                options.executor.query(`SELECT countIf(
+  fact_expected != previous_version OR fact_version < fact_expected OR fact_version > fact_expected + 1
+  OR (fact_version = fact_expected AND fact_checkpoint != previous_checkpoint)
+) AS invalid
+FROM (
+  SELECT
+    fact_expected,
+    fact_version,
+    fact_checkpoint,
+    lagInFrame(fact_version, 1, toUInt64(0)) OVER (ORDER BY event_seq ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS previous_version,
+    lagInFrame(fact_checkpoint, 1, '') OVER (ORDER BY event_seq ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS previous_checkpoint
+  FROM (${facts})
+  WHERE fact_kind = 'batch_committed'
+)`, settings),
+            ]);
+            const row = health[0];
+            if (!row || Number(row.sequences) === 0)
+                return emptyCheckpoint();
+            const problems = [
+                Number(row.head_seq) !== Number(row.sequences) ? `sequence gap (head ${row.head_seq}, ${row.sequences} facts)` : '',
+                Number(row.conflicting_owners) > 0 ? `${row.conflicting_owners} sequence number(s) owned by conflicting facts` : '',
+                Number(row.drifted) > 0 ? `${row.drifted} fact(s) with drifting payloads` : '',
+                Number(transitions[0]?.invalid ?? 0) > 0 ? `${transitions[0]?.invalid} invalid checkpoint transition(s)` : '',
+            ].filter((problem) => problem !== '');
+            if (problems.length > 0) {
+                throw new Error(`Ingestion journal for "${namespaceId}" is not a valid history: ${problems.join('; ')}. Refusing to project a checkpoint from it; more than one executor process may have been active.`);
+            }
+            return {
+                version: Number(row.checkpoint_version),
+                envelope: parseEnvelope(row.checkpoint_json),
+                headSeq: Number(row.head_seq),
+                lastSuccessSeq: Number(row.last_success_seq),
+            };
+        },
+    };
+}
+export function toJournalRow(event, targetId, at) {
+    const checkpointJson = event.checkpoint ? canonicalJson(event.checkpoint) : '';
+    const detailJson = canonicalJson(event.detail);
+    // Identity covers what makes the fact unique; the payload hash covers every
+    // authoritative field a retry of that same fact must reproduce. Only the
+    // physical append time (event_at) is excluded.
+    const eventId = digest([targetId, event.namespaceId, String(event.eventSeq), event.eventKind, event.workId, event.batchId, String(event.attemptNo)]);
+    const payload = digest([
+        eventId,
+        String(event.expectedCheckpointVersion),
+        String(event.checkpointVersion),
+        checkpointJson,
+        event.workState,
+        event.sinkEvidence,
+        event.errorClass,
+        event.runId,
+        event.retryAt ? event.retryAt.toISOString() : '',
+        detailJson,
+    ]);
+    return {
+        target_id: targetId,
+        namespace_id: event.namespaceId,
+        event_seq: String(event.eventSeq),
+        event_id: eventId,
+        payload_hash: BigInt(`0x${payload.slice(0, 16)}`).toString(),
+        event_at: toClickHouseDateTime(at),
+        event_kind: event.eventKind,
+        run_id: event.runId,
+        work_id: event.workId,
+        attempt_no: event.attemptNo,
+        batch_id: event.batchId,
+        expected_checkpoint_version: String(event.expectedCheckpointVersion),
+        checkpoint_version: String(event.checkpointVersion),
+        checkpoint_json: checkpointJson,
+        work_state: event.workState,
+        sink_evidence: event.sinkEvidence,
+        retry_at: event.retryAt ? toClickHouseDateTime(event.retryAt) : null,
+        error_class: event.errorClass,
+        detail_json: detailJson,
+    };
+}
+function parseEnvelope(json) {
+    if (json === '')
+        return undefined;
+    const parsed = JSON.parse(json);
+    if (typeof parsed !== 'object' ||
+        parsed === null ||
+        !('strategy' in parsed) ||
+        typeof parsed.strategy !== 'string' ||
+        !('version' in parsed) ||
+        typeof parsed.version !== 'number') {
+        throw new Error('Committed checkpoint is not a valid ChKit checkpoint envelope.');
+    }
+    return { strategy: parsed.strategy, version: parsed.version, state: 'state' in parsed ? parsed.state : undefined };
+}
+/** Stable key order so equal values always serialize identically. */
+export function canonicalJson(value) {
+    // JSON.stringify(undefined) is undefined, not a string.
+    return JSON.stringify(sortKeys(value)) ?? 'null';
+}
+export function digest(parts) {
+    const hash = createHash('sha256');
+    for (const part of parts) {
+        hash.update(String(part.length));
+        hash.update(':');
+        hash.update(part);
+    }
+    return hash.digest('hex');
+}
+export function emptyCheckpoint() {
+    return { version: 0, envelope: undefined, headSeq: 0, lastSuccessSeq: 0 };
+}
+function journalTableSql(qualified) {
+    return `CREATE TABLE IF NOT EXISTS ${qualified}
+(
+    target_id LowCardinality(String),
+    namespace_id String,
+    event_seq UInt64,
+    event_id String,
+    payload_hash UInt64,
+    event_at DateTime64(6, 'UTC'),
+    event_kind LowCardinality(String),
+    run_id String,
+    work_id String,
+    attempt_no UInt32,
+    batch_id String,
+    expected_checkpoint_version UInt64,
+    checkpoint_version UInt64,
+    checkpoint_json String,
+    work_state LowCardinality(String),
+    sink_evidence LowCardinality(String),
+    retry_at Nullable(DateTime64(6, 'UTC')),
+    error_class LowCardinality(String),
+    detail_json String
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMM(event_at)
+ORDER BY (target_id, namespace_id, event_seq, event_id)`;
+}
+function sortKeys(value) {
+    if (Array.isArray(value))
+        return value.map(sortKeys);
+    if (value instanceof Date)
+        return value.toISOString();
+    if (typeof value === 'object' && value !== null) {
+        return Object.fromEntries(Object.entries(value)
+            .filter(([, entry]) => entry !== undefined)
+            .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
+            .map(([key, entry]) => [key, sortKeys(entry)]));
+    }
+    return value;
+}
+function toClickHouseDateTime(date) {
+    return date.toISOString().replace('T', ' ').replace('Z', '');
+}
+function sqlString(value) {
+    return `'${value.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`;
+}
+//# sourceMappingURL=journal.js.map

package/dist/journal.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"journal.js","sourceRoot":"","sources":["../src/journal.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AAExC,OAAO,EAAE,YAAY,EAA2B,MAAM,mBAAmB,CAAA;AAEzE,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAG/C,MAAM,CAAC,MAAM,qBAAqB,GAAG,0BAA0B,CAAA;AAC/D,MAAM,kBAAkB,GAAG,0BAA0B,CAAA;AAkCrD;;;;GAIG;AACH,MAAM,UAAU,uBAAuB,CAAC,OAAiC;IACvE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,qBAAqB,CAAA;IACpD,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,iBAAiB,CAAC,+BAA+B,KAAK,IAAI,CAAC,CAAA;IAC1G,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/C,MAAM,IAAI,iBAAiB,CAAC,kCAAkC,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAA;IACrF,CAAC;IACD,MAAM,SAAS,GAAG,KAAK,OAAO,CAAC,QAAQ,QAAQ,KAAK,IAAI,CAAA;IACxD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAA;IAE7C,OAAO;QACL,KAAK,CAAC,MAAM;YACV,MAAM,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC,CAAA;YAC1D,sEAAsE;YACtE,+DAA+D;YAC/D,MAAM,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;QAC/D,CAAC;QAED,KAAK,CAAC,MAAM,CAAC,KAAK;YAChB,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,EAAE,OAAO,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAA;YACxD,MAAM,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAC5B,KAAK,EAAE,GAAG,OAAO,CAAC,QAAQ,IAAI,KAAK,EAAE;gBACrC,MAAM,EAAE,CAAC,GAAG,CAAC;gBACb,sEAAsE;gBACtE,2EAA2E;gBAC3E,QAAQ,EAAE,EAAE,0BAA0B,EAAE,GAAG,CAAC,QAAQ,EAAE,YAAY,EAAE,CAAC,EAAE;aACxE,CAAC,CAAA;QACJ,CAAC;QAED,KAAK,CAAC,cAAc,CAAC,WAAW;YAC9B,wEAAwE;YACxE,0EAA0E;YAC1E,4EAA4E;YAC5E,MAAM,KAAK,GAAG;;;;;;;;;SASX,SAAS;sBACI,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,uBAAuB,SAAS,CAAC,WAAW,CAAC;qBACzE,CAAA;YACf,MAAM,QAAQ,GAAG,EAAE,6BAA6B,EAAE,GAAG,EAAE,CAAA;YACvD,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;gBAC9C,OAAO,CAAC,QAAQ,CAAC,KAAK,CASpB;;;;;;;;QAQF,KAAK,GAAG,EACN,QAAQ,CACT;gBACD,yEAAyE;gBACzE,4EAA4E;gBAC5E,sEAAsE;gBACtE,wEAAwE;gBACxE,OAAO,CAAC,QAAQ,CAAC,KAAK,CACpB;;;;;;;;;;;UAWA,KAAK;;EAEb,EACQ,QAAQ,CACT;aACF,CAAC,CAAA;YACF,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,CAAA;YACrB,IAAI,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC;gBAAE,OAAO,eAAe,EAAE,CAAA;YAEjE,MAAM,QAAQ,GAAG;gBACf,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,sBAAsB,GAAG,CAAC,QAAQ,KAAK,GAAG,CAAC,SAAS,SAAS,CAAC,CAAC,CAAC,EAAE;gBACnH,MAAM,CAAC,GAAG,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,kBAAkB,gDAAgD,CAAC,CAAC,CAAC,EAAE;gBACnH,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,OAAO,iCAAiC,CAAC,CAAC,CAAC,EAAE;gBAC9E,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,OAAO,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,OAAO,mCAAmC,CAAC,CAAC,CAAC,EAAE;aAC9G,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,KAAK,EAAE,CAAC,CAAA;YACrC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,MAAM,IAAI,KAAK,CACb,0BAA0B,WAAW,6BAA6B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,kGAAkG,CACxL,CAAA;YACH,CAAC;YACD,OAAO;gBACL,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,kBAAkB,CAAC;gBACvC,QAAQ,EAAE,aAAa,CAAC,GAAG,CAAC,eAAe,CAAC;gBAC5C,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAC7B,cAAc,EAAE,MAAM,CAAC,GAAG,CAAC,gBAAgB,CAAC;aAC7C,CAAA;QACH,CAAC;KACF,CAAA;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAmB,EAAE,QAAgB,EAAE,EAAQ;IAC1E,MAAM,cAAc,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IAC9E,MAAM,UAAU,GAAG,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;IAC9C,4EAA4E;IAC5E,yEAAyE;IACzE,+CAA+C;IAC/C,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,QAAQ,EAAE,KAAK,CAAC,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA;IACpJ,MAAM,OAAO,GAAG,MAAM,CAAC;QACrB,OAAO;QACP,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC;QACvC,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC;QAC/B,cAAc;QACd,KAAK,CAAC,SAAS;QACf,KAAK,CAAC,YAAY;QAClB,KAAK,CAAC,UAAU;QAChB,KAAK,CAAC,KAAK;QACX,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE;QAChD,UAAU;KACX,CAAC,CAAA;IACF,OAAO;QACL,SAAS,EAAE,QAAQ;QACnB,YAAY,EAAE,KAAK,CAAC,WAAW;QAC/B,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;QACjC,QAAQ,EAAE,OAAO;QACjB,YAAY,EAAE,MAAM,CAAC,KAAK,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE;QAC5D,QAAQ,EAAE,oBAAoB,CAAC,EAAE,CAAC;QAClC,UAAU,EAAE,KAAK,CAAC,SAAS;QAC3B,MAAM,EAAE,KAAK,CAAC,KAAK;QACnB,OAAO,EAAE,KAAK,CAAC,MAAM;QACrB,UAAU,EAAE,KAAK,CAAC,SAAS;QAC3B,QAAQ,EAAE,KAAK,CAAC,OAAO;QACvB,2BAA2B,EAAE,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC;QACpE,kBAAkB,EAAE,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC;QACnD,eAAe,EAAE,cAAc;QAC/B,UAAU,EAAE,KAAK,CAAC,SAAS;QAC3B,aAAa,EAAE,KAAK,CAAC,YAAY;QACjC,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,oBAAoB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI;QACpE,WAAW,EAAE,KAAK,CAAC,UAAU;QAC7B,WAAW,EAAE,UAAU;KACxB,CAAA;AACH,CAAC;AAED,SAAS,aAAa,CAAC,IAAY;IACjC,IAAI,IAAI,KAAK,EAAE;QAAE,OAAO,SAAS,CAAA;IACjC,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IACxC,IACE,OAAO,MAAM,KAAK,QAAQ;QAC1B,MAAM,KAAK,IAAI;QACf,CAAC,CAAC,UAAU,IAAI,MAAM,CAAC;QACvB,OAAO,MAAM,CAAC,QAAQ,KAAK,QAAQ;QACnC,CAAC,CAAC,SAAS,IAAI,MAAM,CAAC;QACtB,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,EAClC,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAA;IACnF,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,EAAE,CAAA;AACpH,CAAC;AAED,qEAAqE;AACrE,MAAM,UAAU,aAAa,CAAC,KAAc;IAC1C,wDAAwD;IACxD,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,MAAM,CAAA;AAClD,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,KAAwB;IAC7C,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAA;IACjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA;QAChC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QAChB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IACnB,CAAC;IACD,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAC3B,CAAC;AAED,MAAM,UAAU,eAAe;IAC7B,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,CAAA;AAC3E,CAAC;AAED,SAAS,eAAe,CAAC,SAAiB;IACxC,OAAO,8BAA8B,SAAS;;;;;;;;;;;;;;;;;;;;;;;;wDAwBQ,CAAA;AACxD,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;IACpD,IAAI,KAAK,YAAY,IAAI;QAAE,OAAO,KAAK,CAAC,WAAW,EAAE,CAAA;IACrD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QAChD,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC;aAClB,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC;aAC1C,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;aAChD,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CACjD,CAAA;IACH,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAU;IACtC,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAA;AAC9D,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,CAAA;AACjE,CAAC"}

package/dist/loader.d.ts

@@ -0,0 +1,12 @@
+import type { LoaderFactory } from './types.js';
+export interface SimpleLoaderOptions {
+    /** Rows per physical INSERT. A larger batch is split into deterministic chunks. */
+    maxRowsPerInsert?: number;
+}
+/**
+ * Direct at-least-once destination writes. Every write unit reuses one stable
+ * token across retries: a batch written by one INSERT has one token, a
+ * deterministically chunked batch has one distinct token per chunk.
+ */
+export declare function simpleLoader(options?: SimpleLoaderOptions): LoaderFactory;
+//# sourceMappingURL=loader.d.ts.map

package/dist/loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../src/loader.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAE/C,MAAM,WAAW,mBAAmB;IAClC,mFAAmF;IACnF,gBAAgB,CAAC,EAAE,MAAM,CAAA;CAC1B;AAID;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,OAAO,GAAE,mBAAwB,GAAG,aAAa,CA8B7E"}

package/dist/loader.js

@@ -0,0 +1,37 @@
+import { BATCH_ID_COLUMN, INGESTED_AT_COLUMN, RUN_ID_COLUMN } from './destination.js';
+const DEFAULT_MAX_ROWS_PER_INSERT = 50_000;
+/**
+ * Direct at-least-once destination writes. Every write unit reuses one stable
+ * token across retries: a batch written by one INSERT has one token, a
+ * deterministically chunked batch has one distinct token per chunk.
+ */
+export function simpleLoader(options = {}) {
+    const maxRows = options.maxRowsPerInsert ?? DEFAULT_MAX_ROWS_PER_INSERT;
+    return (ctx) => {
+        let rows = 0;
+        let writeUnits = 0;
+        return {
+            ctx,
+            async write(batch) {
+                for (let offset = 0, unit = 0; offset < batch.rows.length; offset += maxRows, unit += 1) {
+                    ctx.signal.throwIfAborted();
+                    const slice = batch.rows.slice(offset, offset + maxRows).map((row) => {
+                        // Publication time is destination-owned: never let a mapped row supply it.
+                        const { [INGESTED_AT_COLUMN]: _ignored, ...authored } = row;
+                        return { ...authored, [BATCH_ID_COLUMN]: batch.batchId, [RUN_ID_COLUMN]: ctx.runId };
+                    });
+                    await ctx.destination.insert({ table: ctx.table, rows: slice, token: `${batch.batchId}:${unit}` });
+                    rows += slice.length;
+                    writeUnits += 1;
+                }
+            },
+            async finalize() {
+                return { evidence: writeUnits === 0 ? 'none_required' : 'clickhouse_ack', rows, writeUnits };
+            },
+            async abort() {
+                // Direct writes cannot be withdrawn; replay reuses the same tokens.
+            },
+        };
+    };
+}
+//# sourceMappingURL=loader.js.map

package/dist/loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.js","sourceRoot":"","sources":["../src/loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAQrF,MAAM,2BAA2B,GAAG,MAAM,CAAA;AAE1C;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,UAA+B,EAAE;IAC5D,MAAM,OAAO,GAAG,OAAO,CAAC,gBAAgB,IAAI,2BAA2B,CAAA;IAEvE,OAAO,CAAC,GAAG,EAAE,EAAE;QACb,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,IAAI,UAAU,GAAG,CAAC,CAAA;QAElB,OAAO;YACL,GAAG;YACH,KAAK,CAAC,KAAK,CAAC,KAAK;gBACf,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,IAAI,OAAO,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC;oBACxF,GAAG,CAAC,MAAM,CAAC,cAAc,EAAE,CAAA;oBAC3B,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;wBACnE,2EAA2E;wBAC3E,MAAM,EAAE,CAAC,kBAAkB,CAAC,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE,GAAG,GAAG,CAAA;wBAC3D,OAAO,EAAE,GAAG,QAAQ,EAAE,CAAC,eAAe,CAAC,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,aAAa,CAAC,EAAE,GAAG,CAAC,KAAK,EAAE,CAAA;oBACtF,CAAC,CAAC,CAAA;oBACF,MAAM,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,KAAK,CAAC,OAAO,IAAI,IAAI,EAAE,EAAE,CAAC,CAAA;oBAClG,IAAI,IAAI,KAAK,CAAC,MAAM,CAAA;oBACpB,UAAU,IAAI,CAAC,CAAA;gBACjB,CAAC;YACH,CAAC;YACD,KAAK,CAAC,QAAQ;gBACZ,OAAO,EAAE,QAAQ,EAAE,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,gBAAgB,EAAE,IAAI,EAAE,UAAU,EAAE,CAAA;YAC9F,CAAC;YACD,KAAK,CAAC,KAAK;gBACT,oEAAoE;YACtE,CAAC;SACF,CAAA;IACH,CAAC,CAAA;AACH,CAAC"}

package/dist/paginate.d.ts

@@ -0,0 +1,20 @@
+import type { AttemptOptions, FetchContext } from './types.js';
+export interface Page<TItem, TCursor> {
+    items: readonly TItem[];
+    /** Continuation for the next request; `undefined` ends the sequence. */
+    next: TCursor | undefined;
+}
+export interface PaginateOptions<TItem, TCursor> {
+    context: FetchContext;
+    fetchPage(cursor: TCursor | undefined, signal: AbortSignal): Promise<Page<TItem, TCursor>>;
+    initial?: TCursor;
+    label?: AttemptOptions['label'];
+}
+/**
+ * Pull-based pagination over one retryable Promise step per page. Every request
+ * runs through the executor attempt capability. A continuation that was already
+ * seen (compared by canonical serialization) is rejected, so a cyclic provider
+ * cursor cannot loop forever; other non-progress is bounded by execution budgets.
+ */
+export declare function paginate<TItem, TCursor>(options: PaginateOptions<TItem, TCursor>): AsyncGenerator<readonly TItem[], void, void>;
+//# sourceMappingURL=paginate.d.ts.map

package/dist/paginate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"paginate.d.ts","sourceRoot":"","sources":["../src/paginate.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAE9D,MAAM,WAAW,IAAI,CAAC,KAAK,EAAE,OAAO;IAClC,KAAK,EAAE,SAAS,KAAK,EAAE,CAAA;IACvB,wEAAwE;IACxE,IAAI,EAAE,OAAO,GAAG,SAAS,CAAA;CAC1B;AAED,MAAM,WAAW,eAAe,CAAC,KAAK,EAAE,OAAO;IAC7C,OAAO,EAAE,YAAY,CAAA;IACrB,SAAS,CAAC,MAAM,EAAE,OAAO,GAAG,SAAS,EAAE,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAA;IAC1F,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,KAAK,CAAC,EAAE,cAAc,CAAC,OAAO,CAAC,CAAA;CAChC;AAED;;;;;GAKG;AACH,wBAAuB,QAAQ,CAAC,KAAK,EAAE,OAAO,EAC5C,OAAO,EAAE,eAAe,CAAC,KAAK,EAAE,OAAO,CAAC,GACvC,cAAc,CAAC,SAAS,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,CAoB9C"}

package/dist/paginate.js

@@ -0,0 +1,29 @@
+import { canonicalJson } from './journal.js';
+/**
+ * Pull-based pagination over one retryable Promise step per page. Every request
+ * runs through the executor attempt capability. A continuation that was already
+ * seen (compared by canonical serialization) is rejected, so a cyclic provider
+ * cursor cannot loop forever; other non-progress is bounded by execution budgets.
+ */
+export async function* paginate(options) {
+    const seen = new Set();
+    let cursor = options.initial;
+    while (true) {
+        options.context.signal.throwIfAborted();
+        const current = cursor;
+        const page = await options.context.attempt((signal) => options.fetchPage(current, signal), {
+            label: options.label,
+        });
+        if (page.items.length > 0)
+            yield page.items;
+        if (page.next === undefined || page.next === null)
+            return;
+        const key = canonicalJson(page.next);
+        if (seen.has(key)) {
+            throw new Error(`paginate: provider returned a repeated continuation ${key}; refusing to loop.`);
+        }
+        seen.add(key);
+        cursor = page.next;
+    }
+}
+//# sourceMappingURL=paginate.js.map

package/dist/paginate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"paginate.js","sourceRoot":"","sources":["../src/paginate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAA;AAgB5C;;;;;GAKG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,QAAQ,CAC7B,OAAwC;IAExC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,IAAI,MAAM,GAAG,OAAO,CAAC,OAAO,CAAA;IAE5B,OAAO,IAAI,EAAE,CAAC;QACZ,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,EAAE,CAAA;QACvC,MAAM,OAAO,GAAG,MAAM,CAAA;QACtB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE;YACzF,KAAK,EAAE,OAAO,CAAC,KAAK;SACrB,CAAC,CAAA;QACF,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,MAAM,IAAI,CAAC,KAAK,CAAA;QAC3C,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI;YAAE,OAAM;QAEzD,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,uDAAuD,GAAG,qBAAqB,CAAC,CAAA;QAClG,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACb,MAAM,GAAG,IAAI,CAAC,IAAI,CAAA;IACpB,CAAC;AACH,CAAC"}

package/dist/plugin.d.ts

@@ -0,0 +1,110 @@
+import { type ClickHouseExecutor } from '@chkit/clickhouse';
+import { type ChxInlinePluginRegistration, type ResolvedChxConfig } from '@chkit/core';
+import { z } from 'zod';
+import type { PipelineDefinition } from './types.js';
+declare const IngestOptionsSchema: z.ZodObject<{
+    journalTable: z.ZodDefault<z.ZodString>;
+    maxDurationSeconds: z.ZodDefault<z.ZodNumber>;
+    prefetchBatches: z.ZodDefault<z.ZodNumber>;
+}, z.core.$strip>;
+type IngestOptions = z.infer<typeof IngestOptionsSchema>;
+export type IngestPluginOptions = Partial<IngestOptions>;
+interface IngestPluginCommandContext {
+    args: string[];
+    flags: Record<string, string | string[] | boolean | undefined>;
+    jsonMode: boolean;
+    options: IngestOptions;
+    config: ResolvedChxConfig;
+    configPath: string;
+    print: (value: unknown) => void;
+    pluginContext?: {
+        executor: ClickHouseExecutor;
+        hasExecutor: boolean;
+    };
+}
+export declare function createIngestPlugin(options?: IngestPluginOptions): {
+    manifest: {
+        name: "ingest";
+        apiVersion: 1;
+    };
+    optionsSchema: import("@chkit/core").SafeParseable<{
+        journalTable: string;
+        maxDurationSeconds: number;
+        prefetchBatches: number;
+    }>;
+    commands: ({
+        name: string;
+        description: string;
+        flags: readonly [{
+            readonly name: "--tag";
+            readonly type: "string[]";
+            readonly description: "Exact tag every selected stream must carry (repeatable, AND)";
+            readonly placeholder: "<tag>";
+        }, {
+            readonly name: "--backfill";
+            readonly type: "string";
+            readonly description: "Stable backfill id; uses an isolated checkpoint namespace";
+            readonly placeholder: "<id>";
+        }, {
+            readonly name: "--from";
+            readonly type: "string";
+            readonly description: "Backfill range lower bound (ISO timestamp)";
+            readonly placeholder: "<timestamp>";
+        }, {
+            readonly name: "--to";
+            readonly type: "string";
+            readonly description: "Backfill range upper bound (ISO timestamp)";
+            readonly placeholder: "<timestamp>";
+        }, {
+            readonly name: "--max-duration";
+            readonly type: "string";
+            readonly description: "Execution budget in seconds";
+            readonly placeholder: "<seconds>";
+        }];
+        optionsSchema: import("@chkit/core").SafeParseable<{
+            journalTable: string;
+            maxDurationSeconds: number;
+            prefetchBatches: number;
+        }>;
+        run: (context: IngestPluginCommandContext) => Promise<number | undefined>;
+    } | {
+        name: string;
+        description: string;
+        flags: readonly [{
+            readonly name: "--tag";
+            readonly type: "string[]";
+            readonly description: "Exact tag every selected stream must carry (repeatable, AND)";
+            readonly placeholder: "<tag>";
+        }];
+        optionsSchema: import("@chkit/core").SafeParseable<{
+            journalTable: string;
+            maxDurationSeconds: number;
+            prefetchBatches: number;
+        }>;
+        run: (context: IngestPluginCommandContext) => Promise<number | undefined>;
+    })[];
+    hooks: {
+        onCheck(context: {
+            config: ResolvedChxConfig;
+        }): Promise<{
+            plugin: string;
+            evaluated: boolean;
+            ok: boolean;
+            findings: {
+                code: string;
+                message: string;
+                severity: "info" | "warn" | "error";
+            }[];
+        }>;
+    };
+};
+export type IngestPlugin = ReturnType<typeof createIngestPlugin>;
+export declare function ingest(options?: IngestPluginOptions): ChxInlinePluginRegistration<IngestPlugin, IngestPluginOptions>;
+/** Local, zero-network graph checks: they hold in `--offline` mode too. */
+export declare function checkGraph(pipelines: readonly PipelineDefinition[]): {
+    code: string;
+    message: string;
+    severity: "info" | "warn" | "error";
+}[];
+export {};
+//# sourceMappingURL=plugin.d.ts.map

package/dist/plugin.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"plugin.d.ts","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AAEA,OAAO,EAAqC,KAAK,kBAAkB,EAAE,MAAM,mBAAmB,CAAA;AAC9F,OAAO,EAIL,KAAK,2BAA2B,EAChC,KAAK,iBAAiB,EACvB,MAAM,aAAa,CAAA;AAEpB,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAOvB,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAA;AAIpD,QAAA,MAAM,mBAAmB;;;;iBAKvB,CAAA;AACF,KAAK,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAA;AAExD,MAAM,MAAM,mBAAmB,GAAG,OAAO,CAAC,aAAa,CAAC,CAAA;AAcxD,UAAU,0BAA0B;IAClC,IAAI,EAAE,MAAM,EAAE,CAAA;IACd,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,OAAO,GAAG,SAAS,CAAC,CAAA;IAC9D,QAAQ,EAAE,OAAO,CAAA;IACjB,OAAO,EAAE,aAAa,CAAA;IACtB,MAAM,EAAE,iBAAiB,CAAA;IACzB,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,CAAA;IAC/B,aAAa,CAAC,EAAE;QAAE,QAAQ,EAAE,kBAAkB,CAAC;QAAC,WAAW,EAAE,OAAO,CAAA;KAAE,CAAA;CACvE;AAID,wBAAgB,kBAAkB,CAAC,OAAO,GAAE,mBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;yBAwHvC;YAAE,MAAM,EAAE,iBAAiB,CAAA;SAAE;;;;;sBAqB1B,MAAM;yBAAW,MAAM;0BAAY,MAAM,GAAG,MAAM,GAAG,OAAO;;;;EAV3F;AAED,MAAM,MAAM,YAAY,GAAG,UAAU,CAAC,OAAO,kBAAkB,CAAC,CAAA;AAEhE,wBAAgB,MAAM,CAAC,OAAO,GAAE,mBAAwB,GAAG,2BAA2B,CAAC,YAAY,EAAE,mBAAmB,CAAC,CAExH;AAED,2EAA2E;AAC3E,wBAAgB,UAAU,CAAC,SAAS,EAAE,SAAS,kBAAkB,EAAE;UACnC,MAAM;aAAW,MAAM;cAAY,MAAM,GAAG,MAAM,GAAG,OAAO;IAkB3F"}