@chkit/plugin-ingest 0.2.0-beta.8

package/dist/plugin.js

@@ -0,0 +1,255 @@
+import process from 'node:process';
+import { createStatelessClickHouseExecutor } from '@chkit/clickhouse';
+import { createPluginRunner, defineFlags, withFactoryDefaults, } from '@chkit/core';
+import { loadDefinitionModules } from '@chkit/core/schema-loader';
+import { z } from 'zod';
+import { BATCH_ID_COLUMN, createClickHouseDestination, INGESTED_AT_COLUMN, RUN_ID_COLUMN } from './destination.js';
+import { IngestConfigError } from './errors.js';
+import { runIngestion } from './executor.js';
+import { createClickHouseJournal, DEFAULT_JOURNAL_TABLE } from './journal.js';
+import { collectPipelines, selectStreams } from './registry.js';
+const REQUIRED_COLUMNS = [BATCH_ID_COLUMN, RUN_ID_COLUMN, INGESTED_AT_COLUMN];
+const IngestOptionsSchema = z.object({
+    journalTable: z.string().min(1).default(DEFAULT_JOURNAL_TABLE),
+    /** Host-default execution budget in seconds. */
+    maxDurationSeconds: z.number().positive().default(3600),
+    prefetchBatches: z.number().int().positive().default(1),
+});
+const SELECTION_FLAGS = defineFlags([
+    { name: '--tag', type: 'string[]', description: 'Exact tag every selected stream must carry (repeatable, AND)', placeholder: '<tag>' },
+]);
+const RUN_FLAGS = defineFlags([
+    ...SELECTION_FLAGS,
+    { name: '--backfill', type: 'string', description: 'Stable backfill id; uses an isolated checkpoint namespace', placeholder: '<id>' },
+    { name: '--from', type: 'string', description: 'Backfill range lower bound (ISO timestamp)', placeholder: '<timestamp>' },
+    { name: '--to', type: 'string', description: 'Backfill range upper bound (ISO timestamp)', placeholder: '<timestamp>' },
+    { name: '--max-duration', type: 'string', description: 'Execution budget in seconds', placeholder: '<seconds>' },
+]);
+const runCommand = createPluginRunner({ configErrorClass: IngestConfigError });
+export function createIngestPlugin(options = {}) {
+    const optionsSchema = withFactoryDefaults(IngestOptionsSchema, options);
+    return {
+        manifest: { name: 'ingest', apiVersion: 1 },
+        optionsSchema,
+        commands: [
+            {
+                name: 'run',
+                description: 'Execute the selected ingestion streams from their journaled checkpoints',
+                flags: RUN_FLAGS,
+                optionsSchema,
+                run: runCommand({
+                    command: 'run',
+                    label: 'Ingest run',
+                    fn: async (context) => {
+                        const selected = await loadSelection(context);
+                        const backfill = parseBackfill(context.flags);
+                        const maxDurationSeconds = parseMaxDuration(context.flags['--max-duration']) ?? context.options.maxDurationSeconds;
+                        const target = openTarget(context);
+                        const abort = new AbortController();
+                        const onSignal = () => abort.abort(new DOMException('Interrupted', 'AbortError'));
+                        process.once('SIGINT', onSignal);
+                        process.once('SIGTERM', onSignal);
+                        try {
+                            const result = await runIngestion({ selected, backfill }, {
+                                journal: createClickHouseJournal({
+                                    executor: target.executor,
+                                    database: target.database,
+                                    targetId: target.targetId,
+                                    table: context.options.journalTable,
+                                }),
+                                destination: createClickHouseDestination(target.executor),
+                                signal: abort.signal,
+                                maxDurationMs: maxDurationSeconds * 1000,
+                                prefetchBatches: context.options.prefetchBatches,
+                                log: context.jsonMode ? undefined : (message) => context.print(message),
+                            });
+                            if (context.jsonMode) {
+                                context.print({ command: 'run', ...result });
+                            }
+                            else {
+                                const rows = result.streams.reduce((sum, stream) => sum + stream.rows, 0);
+                                context.print(`Ingest run ${result.runId}: ${result.ok ? 'ok' : 'incomplete'} (${result.streams.length} streams, ${rows} rows)`);
+                            }
+                            return result.ok ? 0 : 1;
+                        }
+                        finally {
+                            process.off('SIGINT', onSignal);
+                            process.off('SIGTERM', onSignal);
+                            await target.close();
+                        }
+                    },
+                }),
+            },
+            {
+                name: 'list',
+                description: 'List the ingestion streams in the loaded definition graph',
+                flags: SELECTION_FLAGS,
+                optionsSchema,
+                run: runCommand({
+                    command: 'list',
+                    label: 'Ingest list',
+                    fn: async (context) => {
+                        const selected = await loadSelection(context);
+                        const streams = selected.map(describeStream);
+                        if (context.jsonMode) {
+                            context.print({ ok: true, command: 'list', streams });
+                        }
+                        else {
+                            for (const stream of streams) {
+                                context.print(`${stream.streamId} -> ${stream.destination} [${stream.tags.join(', ')}]`);
+                            }
+                        }
+                        return 0;
+                    },
+                }),
+            },
+            {
+                name: 'status',
+                description: 'Show the committed checkpoint of each selected stream',
+                flags: SELECTION_FLAGS,
+                optionsSchema,
+                run: runCommand({
+                    command: 'status',
+                    label: 'Ingest status',
+                    fn: async (context) => {
+                        const selected = await loadSelection(context);
+                        const target = openTarget(context);
+                        try {
+                            const journal = createClickHouseJournal({
+                                executor: target.executor,
+                                database: target.database,
+                                targetId: target.targetId,
+                                table: context.options.journalTable,
+                            });
+                            await journal.ensure();
+                            const streams = await Promise.all(selected.map(async (entry) => {
+                                const checkpoint = await journal.readCheckpoint(entry.stream.id);
+                                return { ...describeStream(entry), checkpointVersion: checkpoint.version, checkpoint: checkpoint.envelope ?? null };
+                            }));
+                            if (context.jsonMode) {
+                                context.print({ ok: true, command: 'status', targetId: target.targetId, streams });
+                            }
+                            else {
+                                for (const stream of streams) {
+                                    context.print(`${stream.streamId}: v${stream.checkpointVersion} ${stream.checkpoint ? JSON.stringify(stream.checkpoint.state) : '(no checkpoint)'}`);
+                                }
+                            }
+                            return 0;
+                        }
+                        finally {
+                            await target.close();
+                        }
+                    },
+                }),
+            },
+        ],
+        hooks: {
+            async onCheck(context) {
+                const findings = checkGraph(await loadGraph(context.config));
+                return {
+                    plugin: 'ingest',
+                    evaluated: true,
+                    ok: findings.every((finding) => finding.severity !== 'error'),
+                    findings,
+                };
+            },
+        },
+    };
+}
+export function ingest(options = {}) {
+    return { plugin: createIngestPlugin(options), name: 'ingest', enabled: true, options };
+}
+/** Local, zero-network graph checks: they hold in `--offline` mode too. */
+export function checkGraph(pipelines) {
+    const findings = [];
+    if (pipelines.length === 0) {
+        findings.push({ code: 'ingest_no_pipelines', message: 'No ingestion pipeline is exported by the project entry.', severity: 'warn' });
+    }
+    for (const pipeline of pipelines) {
+        for (const stream of pipeline.streams) {
+            const columns = new Set(stream.destination.columns.map((column) => column.name));
+            const missing = REQUIRED_COLUMNS.filter((name) => !columns.has(name));
+            if (missing.length > 0) {
+                findings.push({
+                    code: 'ingest_missing_metadata_columns',
+                    message: `Stream "${stream.id}" destination ${stream.destination.database}.${stream.destination.name} is missing ${missing.join(', ')}. Spread ingestionColumns into its columns.`,
+                    severity: 'error',
+                });
+            }
+        }
+    }
+    return findings;
+}
+// Discover values exported from the configured entry (or legacy schema files).
+async function loadGraph(config) {
+    if (config.schema.length === 0)
+        return [];
+    return collectPipelines(await loadDefinitionModules(config.schema, { cwd: process.cwd() }));
+}
+async function loadSelection(context) {
+    const pipelines = await loadGraph(context.config);
+    if (pipelines.length === 0) {
+        throw new IngestConfigError('No ingestion pipeline is exported. Export a definePipeline(...) value from the module configured as "entry".');
+    }
+    const tags = context.flags['--tag'];
+    return selectStreams(pipelines, Array.isArray(tags) ? tags : typeof tags === 'string' ? [tags] : []);
+}
+function openTarget(context) {
+    const clickhouse = context.config.clickhouse;
+    // Ingestion requires JSONEachRow and per-insert deduplication settings.
+    // The host executor contract does not guarantee either capability.
+    // Streams fetch, load and journal concurrently, so the executor must not be
+    // bound to one ClickHouse HTTP session (a session allows one in-flight query).
+    if (clickhouse) {
+        const executor = createStatelessClickHouseExecutor(clickhouse);
+        return { executor, database: clickhouse.database, targetId: targetIdOf(clickhouse.url, clickhouse.database), close: () => executor.close() };
+    }
+    throw new IngestConfigError('Ingestion requires a direct ClickHouse connection for JSON rows and deduplication settings. Configure clickhouse in your clickhouse.config.ts; a host-provided executor is not supported.');
+}
+function targetIdOf(url, database) {
+    try {
+        return `${new URL(url).host}/${database}`;
+    }
+    catch {
+        return `${url}/${database}`;
+    }
+}
+function parseBackfill(flags) {
+    const id = flags['--backfill'];
+    const from = parseTimestamp(flags['--from'], '--from');
+    const to = parseTimestamp(flags['--to'], '--to');
+    if (typeof id !== 'string') {
+        if (from || to)
+            throw new IngestConfigError('--from and --to require --backfill <id>: explicit ranges never touch the scheduled checkpoint.');
+        return undefined;
+    }
+    if (!/^[A-Za-z0-9][A-Za-z0-9_.-]*$/.test(id))
+        throw new IngestConfigError(`Invalid --backfill id "${id}".`);
+    return { id, from, to };
+}
+function parseTimestamp(raw, flag) {
+    if (typeof raw !== 'string')
+        return undefined;
+    const date = new Date(raw);
+    if (Number.isNaN(date.getTime()))
+        throw new IngestConfigError(`Invalid timestamp for ${flag}: ${raw}`);
+    return date;
+}
+function parseMaxDuration(raw) {
+    if (typeof raw !== 'string')
+        return undefined;
+    const seconds = Number(raw);
+    if (!Number.isFinite(seconds) || seconds <= 0)
+        throw new IngestConfigError(`Invalid value for --max-duration: ${raw}`);
+    return seconds;
+}
+function describeStream(entry) {
+    return {
+        streamId: entry.stream.id,
+        pipelineId: entry.pipeline.id,
+        destination: `${entry.stream.destination.database}.${entry.stream.destination.name}`,
+        strategy: `${entry.stream.incremental.id}@${entry.stream.incremental.version}`,
+        tags: [...entry.effectiveTags],
+    };
+}
+//# sourceMappingURL=plugin.js.map

package/dist/plugin.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"plugin.js","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,cAAc,CAAA;AAElC,OAAO,EAAE,iCAAiC,EAA2B,MAAM,mBAAmB,CAAA;AAC9F,OAAO,EACL,kBAAkB,EAClB,WAAW,EACX,mBAAmB,GAGpB,MAAM,aAAa,CAAA;AACpB,OAAO,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAA;AACjE,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,OAAO,EAAE,eAAe,EAAE,2BAA2B,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAClH,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAC/C,OAAO,EAAE,YAAY,EAAwB,MAAM,eAAe,CAAA;AAClE,OAAO,EAAE,uBAAuB,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAA;AAC7E,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAuB,MAAM,eAAe,CAAA;AAGpF,MAAM,gBAAgB,GAAG,CAAC,eAAe,EAAE,aAAa,EAAE,kBAAkB,CAAC,CAAA;AAE7E,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IACnC,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,qBAAqB,CAAC;IAC9D,gDAAgD;IAChD,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IACvD,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;CACxD,CAAC,CAAA;AAKF,MAAM,eAAe,GAAG,WAAW,CAAC;IAClC,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,8DAA8D,EAAE,WAAW,EAAE,OAAO,EAAE;CAC9H,CAAC,CAAA;AAEX,MAAM,SAAS,GAAG,WAAW,CAAC;IAC5B,GAAG,eAAe;IAClB,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,2DAA2D,EAAE,WAAW,EAAE,MAAM,EAAE;IACrI,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,4CAA4C,EAAE,WAAW,EAAE,aAAa,EAAE;IACzH,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,4CAA4C,EAAE,WAAW,EAAE,aAAa,EAAE;IACvH,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,6BAA6B,EAAE,WAAW,EAAE,WAAW,EAAE;CACxG,CAAC,CAAA;AAaX,MAAM,UAAU,GAAG,kBAAkB,CAA6B,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,CAAC,CAAA;AAE1G,MAAM,UAAU,kBAAkB,CAAC,UAA+B,EAAE;IAClE,MAAM,aAAa,GAAG,mBAAmB,CAAC,mBAAmB,EAAE,OAAO,CAAC,CAAA;IAEvE,OAAO;QACL,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAiB,EAAE,UAAU,EAAE,CAAU,EAAE;QAC7D,aAAa;QACb,QAAQ,EAAE;YACR;gBACE,IAAI,EAAE,KAAK;gBACX,WAAW,EAAE,yEAAyE;gBACtF,KAAK,EAAE,SAAS;gBAChB,aAAa;gBACb,GAAG,EAAE,UAAU,CAAC;oBACd,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,YAAY;oBACnB,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;wBACpB,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,CAAA;wBAC7C,MAAM,QAAQ,GAAG,aAAa,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;wBAC7C,MAAM,kBAAkB,GAAG,gBAAgB,CAAC,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAA;wBAClH,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,CAAA;wBAClC,MAAM,KAAK,GAAG,IAAI,eAAe,EAAE,CAAA;wBACnC,MAAM,QAAQ,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,YAAY,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC,CAAA;wBACjF,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAA;wBAChC,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAA;wBAEjC,IAAI,CAAC;4BACH,MAAM,MAAM,GAAG,MAAM,YAAY,CAC/B,EAAE,QAAQ,EAAE,QAAQ,EAAE,EACtB;gCACE,OAAO,EAAE,uBAAuB,CAAC;oCAC/B,QAAQ,EAAE,MAAM,CAAC,QAAQ;oCACzB,QAAQ,EAAE,MAAM,CAAC,QAAQ;oCACzB,QAAQ,EAAE,MAAM,CAAC,QAAQ;oCACzB,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,YAAY;iCACpC,CAAC;gCACF,WAAW,EAAE,2BAA2B,CAAC,MAAM,CAAC,QAAQ,CAAC;gCACzD,MAAM,EAAE,KAAK,CAAC,MAAM;gCACpB,aAAa,EAAE,kBAAkB,GAAG,IAAI;gCACxC,eAAe,EAAE,OAAO,CAAC,OAAO,CAAC,eAAe;gCAChD,GAAG,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC;6BACxE,CACF,CAAA;4BACD,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gCACrB,OAAO,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,MAAM,EAAE,CAAC,CAAA;4BAC9C,CAAC;iCAAM,CAAC;gCACN,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA;gCACzE,OAAO,CAAC,KAAK,CAAC,cAAc,MAAM,CAAC,KAAK,KAAK,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,KAAK,MAAM,CAAC,OAAO,CAAC,MAAM,aAAa,IAAI,QAAQ,CAAC,CAAA;4BAClI,CAAC;4BACD,OAAO,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;wBAC1B,CAAC;gCAAS,CAAC;4BACT,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAA;4BAC/B,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAA;4BAChC,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;wBACtB,CAAC;oBACH,CAAC;iBACF,CAAC;aACH;YACD;gBACE,IAAI,EAAE,MAAM;gBACZ,WAAW,EAAE,2DAA2D;gBACxE,KAAK,EAAE,eAAe;gBACtB,aAAa;gBACb,GAAG,EAAE,UAAU,CAAC;oBACd,OAAO,EAAE,MAAM;oBACf,KAAK,EAAE,aAAa;oBACpB,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;wBACpB,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,CAAA;wBAC7C,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAC,CAAA;wBAC5C,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;4BACrB,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAA;wBACvD,CAAC;6BAAM,CAAC;4BACN,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gCAC7B,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,QAAQ,OAAO,MAAM,CAAC,WAAW,KAAK,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;4BAC1F,CAAC;wBACH,CAAC;wBACD,OAAO,CAAC,CAAA;oBACV,CAAC;iBACF,CAAC;aACH;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,uDAAuD;gBACpE,KAAK,EAAE,eAAe;gBACtB,aAAa;gBACb,GAAG,EAAE,UAAU,CAAC;oBACd,OAAO,EAAE,QAAQ;oBACjB,KAAK,EAAE,eAAe;oBACtB,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;wBACpB,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,CAAA;wBAC7C,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,CAAA;wBAClC,IAAI,CAAC;4BACH,MAAM,OAAO,GAAG,uBAAuB,CAAC;gCACtC,QAAQ,EAAE,MAAM,CAAC,QAAQ;gCACzB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gCACzB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gCACzB,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,YAAY;6BACpC,CAAC,CAAA;4BACF,MAAM,OAAO,CAAC,MAAM,EAAE,CAAA;4BACtB,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;gCAC3B,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;gCAChE,OAAO,EAAE,GAAG,cAAc,CAAC,KAAK,CAAC,EAAE,iBAAiB,EAAE,UAAU,CAAC,OAAO,EAAE,UAAU,EAAE,UAAU,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAA;4BACrH,CAAC,CAAC,CACH,CAAA;4BACD,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gCACrB,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAA;4BACpF,CAAC;iCAAM,CAAC;gCACN,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;oCAC7B,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,QAAQ,MAAM,MAAM,CAAC,iBAAiB,IAAI,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,iBAAiB,EAAE,CAAC,CAAA;gCACtJ,CAAC;4BACH,CAAC;4BACD,OAAO,CAAC,CAAA;wBACV,CAAC;gCAAS,CAAC;4BACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;wBACtB,CAAC;oBACH,CAAC;iBACF,CAAC;aACH;SACF;QACD,KAAK,EAAE;YACL,KAAK,CAAC,OAAO,CAAC,OAAsC;gBAClD,MAAM,QAAQ,GAAG,UAAU,CAAC,MAAM,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAA;gBAC5D,OAAO;oBACL,MAAM,EAAE,QAAQ;oBAChB,SAAS,EAAE,IAAI;oBACf,EAAE,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC;oBAC7D,QAAQ;iBACT,CAAA;YACH,CAAC;SACF;KACF,CAAA;AACH,CAAC;AAID,MAAM,UAAU,MAAM,CAAC,UAA+B,EAAE;IACtD,OAAO,EAAE,MAAM,EAAE,kBAAkB,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAA;AACxF,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,UAAU,CAAC,SAAwC;IACjE,MAAM,QAAQ,GAAkF,EAAE,CAAA;IAClG,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,OAAO,EAAE,yDAAyD,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IACtI,CAAC;IACD,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;YACtC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAA;YAChF,MAAM,OAAO,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAA;YACrE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,QAAQ,CAAC,IAAI,CAAC;oBACZ,IAAI,EAAE,iCAAiC;oBACvC,OAAO,EAAE,WAAW,MAAM,CAAC,EAAE,iBAAiB,MAAM,CAAC,WAAW,CAAC,QAAQ,IAAI,MAAM,CAAC,WAAW,CAAC,IAAI,eAAe,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,6CAA6C;oBAClL,QAAQ,EAAE,OAAO;iBAClB,CAAC,CAAA;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,+EAA+E;AAC/E,KAAK,UAAU,SAAS,CAAC,MAAyB;IAChD,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAA;IACzC,OAAO,gBAAgB,CAAC,MAAM,qBAAqB,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAA;AAC7F,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,OAAmC;IAC9D,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;IACjD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,iBAAiB,CAAC,8GAA8G,CAAC,CAAA;IAC7I,CAAC;IACD,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;IACnC,OAAO,aAAa,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;AACtG,CAAC;AAED,SAAS,UAAU,CAAC,OAAmC;IACrD,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAA;IAC5C,wEAAwE;IACxE,mEAAmE;IACnE,4EAA4E;IAC5E,+EAA+E;IAC/E,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,QAAQ,GAAG,iCAAiC,CAAC,UAAU,CAAC,CAAA;QAC9D,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,EAAE,QAAQ,EAAE,UAAU,CAAC,UAAU,CAAC,GAAG,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAA;IAC9I,CAAC;IACD,MAAM,IAAI,iBAAiB,CAAC,2LAA2L,CAAC,CAAA;AAC1N,CAAC;AAED,SAAS,UAAU,CAAC,GAAW,EAAE,QAAgB;IAC/C,IAAI,CAAC;QACH,OAAO,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAA;IAC3C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,GAAG,IAAI,QAAQ,EAAE,CAAA;IAC7B,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,KAA0C;IAC/D,MAAM,EAAE,GAAG,KAAK,CAAC,YAAY,CAAC,CAAA;IAC9B,MAAM,IAAI,GAAG,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,CAAA;IACtD,MAAM,EAAE,GAAG,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAA;IAChD,IAAI,OAAO,EAAE,KAAK,QAAQ,EAAE,CAAC;QAC3B,IAAI,IAAI,IAAI,EAAE;YAAE,MAAM,IAAI,iBAAiB,CAAC,gGAAgG,CAAC,CAAA;QAC7I,OAAO,SAAS,CAAA;IAClB,CAAC;IACD,IAAI,CAAC,8BAA8B,CAAC,IAAI,CAAC,EAAE,CAAC;QAAE,MAAM,IAAI,iBAAiB,CAAC,0BAA0B,EAAE,IAAI,CAAC,CAAA;IAC3G,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAA;AACzB,CAAC;AAED,SAAS,cAAc,CAAC,GAA4C,EAAE,IAAY;IAChF,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAA;IAC7C,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,CAAA;IAC1B,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QAAE,MAAM,IAAI,iBAAiB,CAAC,yBAAyB,IAAI,KAAK,GAAG,EAAE,CAAC,CAAA;IACtG,OAAO,IAAI,CAAA;AACb,CAAC;AAED,SAAS,gBAAgB,CAAC,GAA4C;IACpE,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAA;IAC7C,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,CAAA;IAC3B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,IAAI,CAAC;QAAE,MAAM,IAAI,iBAAiB,CAAC,qCAAqC,GAAG,EAAE,CAAC,CAAA;IACtH,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,SAAS,cAAc,CAAC,KAAqB;IAC3C,OAAO;QACL,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE;QACzB,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,EAAE;QAC7B,WAAW,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE;QACpF,QAAQ,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,EAAE;QAC9E,IAAI,EAAE,CAAC,GAAG,KAAK,CAAC,aAAa,CAAC;KAC/B,CAAA;AACH,CAAC"}

package/dist/queue.d.ts

@@ -0,0 +1,9 @@
+export interface BoundedQueue<T> {
+    /** Waits while the queue is full, which is what applies backpressure to the reader. */
+    push(item: T, signal: AbortSignal): Promise<void>;
+    /** Resolves `undefined` once the queue is closed and drained. */
+    pop(signal: AbortSignal): Promise<T | undefined>;
+    close(): void;
+}
+export declare function createBoundedQueue<T>(capacity: number): BoundedQueue<T>;
+//# sourceMappingURL=queue.d.ts.map

package/dist/queue.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["../src/queue.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,YAAY,CAAC,CAAC;IAC7B,uFAAuF;IACvF,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACjD,iEAAiE;IACjE,GAAG,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,CAAA;IAChD,KAAK,IAAI,IAAI,CAAA;CACd;AAED,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,CA+CvE"}

package/dist/queue.js

@@ -0,0 +1,47 @@
+export function createBoundedQueue(capacity) {
+    const limit = Math.max(1, capacity);
+    const items = [];
+    let closed = false;
+    let wake = [];
+    const notify = () => {
+        const waiting = wake;
+        wake = [];
+        for (const resolve of waiting)
+            resolve();
+    };
+    const wait = (signal) => new Promise((resolve, reject) => {
+        const onAbort = () => reject(signal.reason);
+        signal.addEventListener('abort', onAbort, { once: true });
+        wake.push(() => {
+            signal.removeEventListener('abort', onAbort);
+            resolve();
+        });
+    });
+    return {
+        async push(item, signal) {
+            while (items.length >= limit) {
+                signal.throwIfAborted();
+                await wait(signal);
+            }
+            signal.throwIfAborted();
+            items.push(item);
+            notify();
+        },
+        async pop(signal) {
+            while (items.length === 0) {
+                if (closed)
+                    return undefined;
+                signal.throwIfAborted();
+                await wait(signal);
+            }
+            const item = items.shift();
+            notify();
+            return item;
+        },
+        close() {
+            closed = true;
+            notify();
+        },
+    };
+}
+//# sourceMappingURL=queue.js.map

package/dist/queue.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"queue.js","sourceRoot":"","sources":["../src/queue.ts"],"names":[],"mappings":"AAQA,MAAM,UAAU,kBAAkB,CAAI,QAAgB;IACpD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAA;IACnC,MAAM,KAAK,GAAQ,EAAE,CAAA;IACrB,IAAI,MAAM,GAAG,KAAK,CAAA;IAClB,IAAI,IAAI,GAAsB,EAAE,CAAA;IAEhC,MAAM,MAAM,GAAG,GAAG,EAAE;QAClB,MAAM,OAAO,GAAG,IAAI,CAAA;QACpB,IAAI,GAAG,EAAE,CAAA;QACT,KAAK,MAAM,OAAO,IAAI,OAAO;YAAE,OAAO,EAAE,CAAA;IAC1C,CAAC,CAAA;IAED,MAAM,IAAI,GAAG,CAAC,MAAmB,EAAE,EAAE,CACnC,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACpC,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;QAC3C,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;QACzD,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;YACb,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;YAC5C,OAAO,EAAE,CAAA;QACX,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEJ,OAAO;QACL,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM;YACrB,OAAO,KAAK,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC;gBAC7B,MAAM,CAAC,cAAc,EAAE,CAAA;gBACvB,MAAM,IAAI,CAAC,MAAM,CAAC,CAAA;YACpB,CAAC;YACD,MAAM,CAAC,cAAc,EAAE,CAAA;YACvB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAChB,MAAM,EAAE,CAAA;QACV,CAAC;QACD,KAAK,CAAC,GAAG,CAAC,MAAM;YACd,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC1B,IAAI,MAAM;oBAAE,OAAO,SAAS,CAAA;gBAC5B,MAAM,CAAC,cAAc,EAAE,CAAA;gBACvB,MAAM,IAAI,CAAC,MAAM,CAAC,CAAA;YACpB,CAAC;YACD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAE,CAAA;YAC1B,MAAM,EAAE,CAAA;YACR,OAAO,IAAI,CAAA;QACb,CAAC;QACD,KAAK;YACH,MAAM,GAAG,IAAI,CAAA;YACb,MAAM,EAAE,CAAA;QACV,CAAC;KACF,CAAA;AACH,CAAC"}

package/dist/registry.d.ts

@@ -0,0 +1,48 @@
+import type { TableDefinition } from '@chkit/core';
+import type { AnyStreamDefinition, ErrorClassifier, IncrementalStrategy, LoaderFactory, PipelineDefinition, ReadContext, RetryOptions, Row, SourceChunk, StreamBudget, StreamDefinition } from './types.js';
+export interface StreamInput<TRow extends Row, TState, TSelection> {
+    id: string;
+    destination: TableDefinition;
+    tags?: readonly string[];
+    incremental: IncrementalStrategy<TState, TSelection>;
+    read: (context: ReadContext<TSelection, TState>) => AsyncIterable<SourceChunk<TRow, TState>>;
+    loader?: LoaderFactory;
+    retry?: RetryOptions;
+    batchSize?: number;
+    budget?: StreamBudget;
+    classifyError?: ErrorClassifier;
+}
+export interface FullSyncStreamInput<TRow extends Row> extends Omit<StreamInput<TRow, undefined, undefined>, 'incremental'> {
+    incremental?: undefined;
+}
+export interface PipelineInput {
+    id: string;
+    tags?: readonly string[];
+    streams: readonly AnyStreamDefinition[];
+    maxStreams?: number;
+    maxFetches?: number;
+    maxLoads?: number;
+    retry?: RetryOptions;
+}
+export interface SelectedStream {
+    stream: AnyStreamDefinition;
+    pipeline: PipelineDefinition;
+    effectiveTags: readonly string[];
+}
+export declare function defineStream<TRow extends Row>(input: FullSyncStreamInput<TRow>): StreamDefinition<TRow, undefined, undefined>;
+export declare function defineStream<TRow extends Row, TState, TSelection>(input: StreamInput<TRow, TState, TSelection>): StreamDefinition<TRow, TState, TSelection>;
+/**
+ * Define a non-durable named group of streams. Pipeline identity never
+ * participates in checkpoint or batch identity, so moving a stream between
+ * pipelines does not reset its state.
+ */
+export declare function definePipeline(input: PipelineInput): PipelineDefinition;
+/** Only exported pipeline values participate; aliases of the same value count once. */
+export declare function collectPipelines(modules: readonly Record<string, unknown>[]): PipelineDefinition[];
+/**
+ * Repeated exact case-sensitive tags with AND semantics over the union of
+ * pipeline and stream tags plus derived `pipeline:<id>` and `stream:<id>`.
+ * No filter selects the complete graph; an explicit empty selection throws.
+ */
+export declare function selectStreams(pipelines: readonly PipelineDefinition[], tags: readonly string[]): SelectedStream[];
+//# sourceMappingURL=registry.d.ts.map

package/dist/registry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../src/registry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAIlD,OAAO,KAAK,EACV,mBAAmB,EACnB,eAAe,EACf,mBAAmB,EACnB,aAAa,EACb,kBAAkB,EAClB,WAAW,EACX,YAAY,EACZ,GAAG,EACH,WAAW,EACX,YAAY,EACZ,gBAAgB,EACjB,MAAM,YAAY,CAAA;AAInB,MAAM,WAAW,WAAW,CAAC,IAAI,SAAS,GAAG,EAAE,MAAM,EAAE,UAAU;IAC/D,EAAE,EAAE,MAAM,CAAA;IACV,WAAW,EAAE,eAAe,CAAA;IAC5B,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IACxB,WAAW,EAAE,mBAAmB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;IACpD,IAAI,EAAE,CAAC,OAAO,EAAE,WAAW,CAAC,UAAU,EAAE,MAAM,CAAC,KAAK,aAAa,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAA;IAC5F,MAAM,CAAC,EAAE,aAAa,CAAA;IACtB,KAAK,CAAC,EAAE,YAAY,CAAA;IACpB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,CAAC,EAAE,YAAY,CAAA;IACrB,aAAa,CAAC,EAAE,eAAe,CAAA;CAChC;AAED,MAAM,WAAW,mBAAmB,CAAC,IAAI,SAAS,GAAG,CACnD,SAAQ,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE,aAAa,CAAC;IACpE,WAAW,CAAC,EAAE,SAAS,CAAA;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IACxB,OAAO,EAAE,SAAS,mBAAmB,EAAE,CAAA;IACvC,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,KAAK,CAAC,EAAE,YAAY,CAAA;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,mBAAmB,CAAA;IAC3B,QAAQ,EAAE,kBAAkB,CAAA;IAC5B,aAAa,EAAE,SAAS,MAAM,EAAE,CAAA;CACjC;AAED,wBAAgB,YAAY,CAAC,IAAI,SAAS,GAAG,EAAE,KAAK,EAAE,mBAAmB,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,CAAC,CAAA;AAC9H,wBAAgB,YAAY,CAAC,IAAI,SAAS,GAAG,EAAE,MAAM,EAAE,UAAU,EAC/D,KAAK,EAAE,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,GAC3C,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,CAAA;AAuB7C;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,aAAa,GAAG,kBAAkB,CAsBvE;AAqBD,uFAAuF;AACvF,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,kBAAkB,EAAE,CAMlG;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,SAAS,EAAE,SAAS,kBAAkB,EAAE,EAAE,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,cAAc,EAAE,CAmBjH"}

package/dist/registry.js

@@ -0,0 +1,109 @@
+import { IngestConfigError } from './errors.js';
+import { fullSync } from './incremental.js';
+const STREAM_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.:-]*$/;
+export function defineStream(input) {
+    assertValidId('stream', input.id);
+    if (input.batchSize !== undefined && (!Number.isInteger(input.batchSize) || input.batchSize <= 0)) {
+        throw new IngestConfigError(`Stream "${input.id}": batchSize must be a positive integer.`);
+    }
+    return {
+        kind: 'ingest_stream',
+        id: input.id,
+        destination: input.destination,
+        tags: dedupe(input.tags ?? []),
+        incremental: input.incremental ?? fullSync(),
+        read: input.read,
+        loader: input.loader,
+        retry: input.retry,
+        batchSize: input.batchSize,
+        budget: input.budget,
+        classifyError: input.classifyError,
+    };
+}
+/**
+ * Define a non-durable named group of streams. Pipeline identity never
+ * participates in checkpoint or batch identity, so moving a stream between
+ * pipelines does not reset its state.
+ */
+export function definePipeline(input) {
+    assertValidId('pipeline', input.id);
+    const pipeline = {
+        kind: 'ingest_pipeline',
+        id: input.id,
+        tags: dedupe(input.tags ?? []),
+        streams: [...input.streams],
+        maxStreams: positiveCeiling(input.id, 'maxStreams', input.maxStreams, 4),
+        maxFetches: positiveCeiling(input.id, 'maxFetches', input.maxFetches, 4),
+        maxLoads: positiveCeiling(input.id, 'maxLoads', input.maxLoads, 2),
+        retry: input.retry,
+    };
+    const seen = new Set();
+    for (const stream of pipeline.streams) {
+        if (seen.has(stream.id)) {
+            throw new IngestConfigError(`Pipeline "${pipeline.id}" lists stream "${stream.id}" more than once.`);
+        }
+        seen.add(stream.id);
+    }
+    return pipeline;
+}
+/** Validate identities within this graph, without process-wide registration. */
+function validatePipelines(pipelines) {
+    const ids = new Set();
+    const owners = new Map();
+    for (const pipeline of pipelines) {
+        if (ids.has(pipeline.id))
+            throw new IngestConfigError(`Duplicate pipeline id "${pipeline.id}".`);
+        ids.add(pipeline.id);
+        for (const stream of pipeline.streams) {
+            const owner = owners.get(stream.id);
+            if (owner !== undefined) {
+                throw new IngestConfigError(`Stream id "${stream.id}" occurs in pipelines "${owner}" and "${pipeline.id}". Stream ids must be globally unique.`);
+            }
+            owners.set(stream.id, pipeline.id);
+        }
+    }
+}
+/** Only exported pipeline values participate; aliases of the same value count once. */
+export function collectPipelines(modules) {
+    const pipelines = [...new Set(modules.flatMap((mod) => Object.values(mod).filter((value) => typeof value === 'object' && value !== null && 'kind' in value && value.kind === 'ingest_pipeline')))];
+    validatePipelines(pipelines);
+    return pipelines;
+}
+/**
+ * Repeated exact case-sensitive tags with AND semantics over the union of
+ * pipeline and stream tags plus derived `pipeline:<id>` and `stream:<id>`.
+ * No filter selects the complete graph; an explicit empty selection throws.
+ */
+export function selectStreams(pipelines, tags) {
+    validatePipelines(pipelines);
+    const wanted = dedupe(tags);
+    const all = pipelines.flatMap((pipeline) => pipeline.streams.map((stream) => ({
+        stream,
+        pipeline,
+        effectiveTags: dedupe([...pipeline.tags, ...stream.tags, `pipeline:${pipeline.id}`, `stream:${stream.id}`]),
+    })));
+    if (wanted.length === 0)
+        return all;
+    const selected = all.filter((entry) => wanted.every((tag) => entry.effectiveTags.includes(tag)));
+    if (selected.length === 0) {
+        throw new IngestConfigError(`No stream matches every requested tag: ${wanted.map((tag) => `--tag ${tag}`).join(' ')}. Nothing was executed.`);
+    }
+    return selected;
+}
+function assertValidId(kind, id) {
+    if (!STREAM_ID_PATTERN.test(id)) {
+        throw new IngestConfigError(`Invalid ${kind} id "${id}". Use letters, digits, "_", ".", ":" or "-", starting with a letter or digit.`);
+    }
+}
+function positiveCeiling(pipelineId, name, value, fallback) {
+    if (value === undefined)
+        return fallback;
+    if (!Number.isInteger(value) || value <= 0) {
+        throw new IngestConfigError(`Pipeline "${pipelineId}": ${name} must be a positive integer.`);
+    }
+    return value;
+}
+function dedupe(values) {
+    return [...new Set(values)];
+}
+//# sourceMappingURL=registry.js.map

package/dist/registry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"registry.js","sourceRoot":"","sources":["../src/registry.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAC/C,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAe3C,MAAM,iBAAiB,GAAG,+BAA+B,CAAA;AAwCzD,MAAM,UAAU,YAAY,CAC1B,KAAoE;IAEpE,aAAa,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,CAAC,CAAA;IACjC,IAAI,KAAK,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,SAAS,IAAI,CAAC,CAAC,EAAE,CAAC;QAClG,MAAM,IAAI,iBAAiB,CAAC,WAAW,KAAK,CAAC,EAAE,0CAA0C,CAAC,CAAA;IAC5F,CAAC;IACD,OAAO;QACL,IAAI,EAAE,eAAe;QACrB,EAAE,EAAE,KAAK,CAAC,EAAE;QACZ,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC;QAC9B,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,QAAQ,EAAE;QAC5C,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,aAAa,EAAE,KAAK,CAAC,aAAa;KACnC,CAAA;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,KAAoB;IACjD,aAAa,CAAC,UAAU,EAAE,KAAK,CAAC,EAAE,CAAC,CAAA;IACnC,MAAM,QAAQ,GAAuB;QACnC,IAAI,EAAE,iBAAiB;QACvB,EAAE,EAAE,KAAK,CAAC,EAAE;QACZ,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC;QAC9B,OAAO,EAAE,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC;QAC3B,UAAU,EAAE,eAAe,CAAC,KAAK,CAAC,EAAE,EAAE,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC;QACxE,UAAU,EAAE,eAAe,CAAC,KAAK,CAAC,EAAE,EAAE,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC;QACxE,QAAQ,EAAE,eAAe,CAAC,KAAK,CAAC,EAAE,EAAE,UAAU,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;QAClE,KAAK,EAAE,KAAK,CAAC,KAAK;KACnB,CAAA;IAED,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QACtC,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,iBAAiB,CAAC,aAAa,QAAQ,CAAC,EAAE,mBAAmB,MAAM,CAAC,EAAE,mBAAmB,CAAC,CAAA;QACtG,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;IACrB,CAAC;IAED,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,gFAAgF;AAChF,SAAS,iBAAiB,CAAC,SAAwC;IACjE,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAA;IAC7B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAA;IACxC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,IAAI,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAAE,MAAM,IAAI,iBAAiB,CAAC,0BAA0B,QAAQ,CAAC,EAAE,IAAI,CAAC,CAAA;QAChG,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAA;QACpB,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;YACtC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;YACnC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACxB,MAAM,IAAI,iBAAiB,CACzB,cAAc,MAAM,CAAC,EAAE,0BAA0B,KAAK,UAAU,QAAQ,CAAC,EAAE,wCAAwC,CACpH,CAAA;YACH,CAAC;YACD,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAA;QACpC,CAAC;IACH,CAAC;AACH,CAAC;AAED,uFAAuF;AACvF,MAAM,UAAU,gBAAgB,CAAC,OAA2C;IAC1E,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAC9E,CAAC,KAAK,EAA+B,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,iBAAiB,CAC3I,CAAC,CAAC,CAAC,CAAA;IACJ,iBAAiB,CAAC,SAAS,CAAC,CAAA;IAC5B,OAAO,SAAS,CAAA;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,SAAwC,EAAE,IAAuB;IAC7F,iBAAiB,CAAC,SAAS,CAAC,CAAA;IAC5B,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAA;IAC3B,MAAM,GAAG,GAAqB,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE,CAC3D,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAChC,MAAM;QACN,QAAQ;QACR,aAAa,EAAE,MAAM,CAAC,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,GAAG,MAAM,CAAC,IAAI,EAAE,YAAY,QAAQ,CAAC,EAAE,EAAE,EAAE,UAAU,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;KAC5G,CAAC,CAAC,CACJ,CAAA;IACD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,GAAG,CAAA;IAEnC,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;IAChG,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,iBAAiB,CACzB,0CAA0C,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,SAAS,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,yBAAyB,CACjH,CAAA;IACH,CAAC;IACD,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,SAAS,aAAa,CAAC,IAA2B,EAAE,EAAU;IAC5D,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,iBAAiB,CACzB,WAAW,IAAI,QAAQ,EAAE,gFAAgF,CAC1G,CAAA;IACH,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,UAAkB,EAAE,IAAY,EAAE,KAAyB,EAAE,QAAgB;IACpG,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAA;IACxC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QAC3C,MAAM,IAAI,iBAAiB,CAAC,aAAa,UAAU,MAAM,IAAI,8BAA8B,CAAC,CAAA;IAC9F,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,MAAM,CAAC,MAAyB;IACvC,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAA;AAC7B,CAAC"}

package/dist/retry.d.ts

@@ -0,0 +1,12 @@
+import type { ErrorClassifier, RetryContext, RetryOptions } from './types.js';
+interface AttemptEnv {
+    signal: AbortSignal;
+    classifier: ErrorClassifier | undefined;
+    onRetry: (context: RetryContext, retryAfterMs: number | undefined) => Promise<void> | void;
+}
+/** Pipeline default, partially overridden by the stream. */
+export declare function mergeRetry(...layers: Array<RetryOptions | undefined>): RetryOptions;
+/** p-retry owns retry budgets and backoff; this adapter adds provider classification and Retry-After. */
+export declare function runAttempt<T>(operation: (attemptNumber: number) => Promise<T>, policy: RetryOptions, env: AttemptEnv): Promise<T>;
+export {};
+//# sourceMappingURL=retry.d.ts.map

package/dist/retry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"retry.d.ts","sourceRoot":"","sources":["../src/retry.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,eAAe,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAW7E,UAAU,UAAU;IAClB,MAAM,EAAE,WAAW,CAAA;IACnB,UAAU,EAAE,eAAe,GAAG,SAAS,CAAA;IACvC,OAAO,EAAE,CAAC,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,GAAG,SAAS,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;CAC3F;AAED,4DAA4D;AAC5D,wBAAgB,UAAU,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,YAAY,GAAG,SAAS,CAAC,GAAG,YAAY,CASnF;AAED,yGAAyG;AACzG,wBAAgB,UAAU,CAAC,CAAC,EAC1B,SAAS,EAAE,CAAC,aAAa,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,EAChD,MAAM,EAAE,YAAY,EACpB,GAAG,EAAE,UAAU,GACd,OAAO,CAAC,CAAC,CAAC,CAqCZ"}

package/dist/retry.js

@@ -0,0 +1,69 @@
+import { setTimeout as sleep } from 'node:timers/promises';
+import pRetry, { AbortError } from 'p-retry';
+import { classifyFailure, FetchFailure, IngestConfigError } from './errors.js';
+const DEFAULT_RETRY = {
+    retries: 5,
+    factor: 2,
+    minTimeout: 1000,
+    maxTimeout: 60_000,
+    randomize: true,
+    maxRetryTime: 10 * 60_000,
+};
+/** Pipeline default, partially overridden by the stream. */
+export function mergeRetry(...layers) {
+    const merged = {};
+    for (const layer of layers) {
+        if (!layer)
+            continue;
+        for (const [key, value] of Object.entries(layer)) {
+            if (value !== undefined)
+                Object.assign(merged, { [key]: value });
+        }
+    }
+    return merged;
+}
+/** p-retry owns retry budgets and backoff; this adapter adds provider classification and Retry-After. */
+export function runAttempt(operation, policy, env) {
+    const options = { ...DEFAULT_RETRY, ...mergeRetry(policy) };
+    const deadline = performance.now() + options.maxRetryTime;
+    let notBefore = 0;
+    return pRetry(async (attemptNumber) => {
+        // Backoff and Retry-After overlap: only wait out the remaining provider embargo.
+        const remaining = notBefore - performance.now();
+        if (remaining > 0)
+            await sleep(remaining, undefined, { signal: env.signal });
+        env.signal.throwIfAborted();
+        try {
+            return await operation(attemptNumber);
+        }
+        catch (cause) {
+            // A nested source attempt already exhausted its retries. Never replay it
+            // through the reader's coarser retry boundary.
+            if (cause instanceof FetchFailure || cause instanceof IngestConfigError)
+                throw new AbortError(cause);
+            const failure = new FetchFailure(cause, classifyFailure(cause, env.signal, env.classifier));
+            if (failure.classification.kind === 'cancelled' || failure.classification.kind === 'permanent')
+                throw new AbortError(failure);
+            throw failure;
+        }
+    }, {
+        ...options,
+        signal: env.signal,
+        shouldConsumeRetry: ({ error, ...context }) => error instanceof FetchFailure ? (policy.shouldConsumeRetry?.({ ...context, error }) ?? true) : true,
+        shouldRetry: async ({ error, ...context }) => {
+            env.signal.throwIfAborted();
+            if (!(error instanceof FetchFailure))
+                return false;
+            const retryContext = { ...context, error };
+            if (policy.shouldRetry && !(await policy.shouldRetry(retryContext)))
+                return false;
+            const retryAfterMs = error.classification.kind === 'rate_limited' ? error.classification.retryAfterMs : undefined;
+            notBefore = performance.now() + (retryAfterMs ?? 0);
+            if (notBefore > deadline)
+                return false;
+            await env.onRetry(retryContext, retryAfterMs);
+            return true;
+        },
+    });
+}
+//# sourceMappingURL=retry.js.map

package/dist/retry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"retry.js","sourceRoot":"","sources":["../src/retry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAE1D,OAAO,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AAE5C,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAG9E,MAAM,aAAa,GAAG;IACpB,OAAO,EAAE,CAAC;IACV,MAAM,EAAE,CAAC;IACT,UAAU,EAAE,IAAI;IAChB,UAAU,EAAE,MAAM;IAClB,SAAS,EAAE,IAAI;IACf,YAAY,EAAE,EAAE,GAAG,MAAM;CAC1B,CAAA;AAQD,4DAA4D;AAC5D,MAAM,UAAU,UAAU,CAAC,GAAG,MAAuC;IACnE,MAAM,MAAM,GAAiB,EAAE,CAAA;IAC/B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,CAAC,KAAK;YAAE,SAAQ;QACpB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACjD,IAAI,KAAK,KAAK,SAAS;gBAAE,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,CAAA;QAClE,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC;AAED,yGAAyG;AACzG,MAAM,UAAU,UAAU,CACxB,SAAgD,EAChD,MAAoB,EACpB,GAAe;IAEf,MAAM,OAAO,GAAG,EAAE,GAAG,aAAa,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,EAAE,CAAA;IAC3D,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,YAAY,CAAA;IACzD,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,OAAO,MAAM,CAAC,KAAK,EAAE,aAAa,EAAE,EAAE;QACpC,iFAAiF;QACjF,MAAM,SAAS,GAAG,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;QAC/C,IAAI,SAAS,GAAG,CAAC;YAAE,MAAM,KAAK,CAAC,SAAS,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAA;QAC5E,GAAG,CAAC,MAAM,CAAC,cAAc,EAAE,CAAA;QAC3B,IAAI,CAAC;YACH,OAAO,MAAM,SAAS,CAAC,aAAa,CAAC,CAAA;QACvC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,yEAAyE;YACzE,+CAA+C;YAC/C,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,iBAAiB;gBAAE,MAAM,IAAI,UAAU,CAAC,KAAK,CAAC,CAAA;YACpG,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC,CAAA;YAC3F,IAAI,OAAO,CAAC,cAAc,CAAC,IAAI,KAAK,WAAW,IAAI,OAAO,CAAC,cAAc,CAAC,IAAI,KAAK,WAAW;gBAAE,MAAM,IAAI,UAAU,CAAC,OAAO,CAAC,CAAA;YAC7H,MAAM,OAAO,CAAA;QACf,CAAC;IACH,CAAC,EAAE;QACD,GAAG,OAAO;QACV,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,kBAAkB,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,EAAE,EAAE,CAC5C,KAAK,YAAY,YAAY,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;QACrG,WAAW,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,EAAE,EAAE;YAC3C,GAAG,CAAC,MAAM,CAAC,cAAc,EAAE,CAAA;YAC3B,IAAI,CAAC,CAAC,KAAK,YAAY,YAAY,CAAC;gBAAE,OAAO,KAAK,CAAA;YAClD,MAAM,YAAY,GAAG,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,CAAA;YAC1C,IAAI,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,MAAM,MAAM,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;gBAAE,OAAO,KAAK,CAAA;YACjF,MAAM,YAAY,GAAG,KAAK,CAAC,cAAc,CAAC,IAAI,KAAK,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,CAAA;YACjH,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,CAAC,YAAY,IAAI,CAAC,CAAC,CAAA;YACnD,IAAI,SAAS,GAAG,QAAQ;gBAAE,OAAO,KAAK,CAAA;YACtC,MAAM,GAAG,CAAC,OAAO,CAAC,YAAY,EAAE,YAAY,CAAC,CAAA;YAC7C,OAAO,IAAI,CAAA;QACb,CAAC;KACF,CAAC,CAAA;AACJ,CAAC"}

package/dist/testing.d.ts

@@ -0,0 +1,16 @@
+import { type JournalRow } from './journal.js';
+import type { DestinationAdapter, Journal, JournalEvent, Row } from './types.js';
+export interface MemoryJournal extends Journal {
+    readonly events: JournalEvent[];
+    readonly rows: JournalRow[];
+}
+export interface MemoryDestination extends DestinationAdapter {
+    /** Physical rows per `database.table`, after token deduplication. */
+    readonly tables: Map<string, Row[]>;
+    readonly tokens: string[];
+}
+/** In-memory journal with the same projection semantics as the ClickHouse one. */
+export declare function createMemoryJournal(): MemoryJournal;
+/** In-memory destination that honours `insert_deduplication_token` like ClickHouse. */
+export declare function createMemoryDestination(): MemoryDestination;
+//# sourceMappingURL=testing.d.ts.map

package/dist/testing.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"testing.d.ts","sourceRoot":"","sources":["../src/testing.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiC,KAAK,UAAU,EAAE,MAAM,cAAc,CAAA;AAC7E,OAAO,KAAK,EAAuB,kBAAkB,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,EAAE,MAAM,YAAY,CAAA;AAErG,MAAM,WAAW,aAAc,SAAQ,OAAO;IAC5C,QAAQ,CAAC,MAAM,EAAE,YAAY,EAAE,CAAA;IAC/B,QAAQ,CAAC,IAAI,EAAE,UAAU,EAAE,CAAA;CAC5B;AAED,MAAM,WAAW,iBAAkB,SAAQ,kBAAkB;IAC3D,qEAAqE;IACrE,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAA;IACnC,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAC1B;AAED,kFAAkF;AAClF,wBAAgB,mBAAmB,IAAI,aAAa,CAyBnD;AAED,uFAAuF;AACvF,wBAAgB,uBAAuB,IAAI,iBAAiB,CAa3D"}