@chkit/plugin-ingest 0.2.0-beta.8

package/dist/testing.js

@@ -0,0 +1,46 @@
+import { emptyCheckpoint, toJournalRow } from './journal.js';
+/** In-memory journal with the same projection semantics as the ClickHouse one. */
+export function createMemoryJournal() {
+    const events = [];
+    const rows = [];
+    return {
+        events,
+        rows,
+        async ensure() { },
+        async append(event) {
+            events.push(event);
+            rows.push(toJournalRow(event, 'memory', new Date(0)));
+        },
+        async readCheckpoint(namespaceId) {
+            const scoped = events.filter((event) => event.namespaceId === namespaceId);
+            if (scoped.length === 0)
+                return emptyCheckpoint();
+            const headSeq = Math.max(...scoped.map((event) => event.eventSeq));
+            const lastSuccessSeq = Math.max(0, ...scoped
+                .filter((event) => event.eventKind === 'work_finished' && event.workState === 'succeeded')
+                .map((event) => event.eventSeq));
+            const committed = scoped
+                .filter((event) => event.eventKind === 'batch_committed')
+                .sort((a, b) => a.checkpointVersion - b.checkpointVersion || a.eventSeq - b.eventSeq)
+                .at(-1);
+            return { version: committed?.checkpointVersion ?? 0, envelope: committed?.checkpoint, headSeq, lastSuccessSeq };
+        },
+    };
+}
+/** In-memory destination that honours `insert_deduplication_token` like ClickHouse. */
+export function createMemoryDestination() {
+    const tables = new Map();
+    const tokens = [];
+    return {
+        tables,
+        tokens,
+        async insert({ table, rows, token }) {
+            const key = `${table.database}.${table.name}`;
+            if (tokens.includes(`${key}:${token}`))
+                return;
+            tokens.push(`${key}:${token}`);
+            tables.set(key, [...(tables.get(key) ?? []), ...rows]);
+        },
+    };
+}
+//# sourceMappingURL=testing.js.map

package/dist/testing.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"testing.js","sourceRoot":"","sources":["../src/testing.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,YAAY,EAAmB,MAAM,cAAc,CAAA;AAc7E,kFAAkF;AAClF,MAAM,UAAU,mBAAmB;IACjC,MAAM,MAAM,GAAmB,EAAE,CAAA;IACjC,MAAM,IAAI,GAAiB,EAAE,CAAA;IAC7B,OAAO;QACL,MAAM;QACN,IAAI;QACJ,KAAK,CAAC,MAAM,KAAI,CAAC;QACjB,KAAK,CAAC,MAAM,CAAC,KAAK;YAChB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAClB,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,QAAQ,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QACvD,CAAC;QACD,KAAK,CAAC,cAAc,CAAC,WAAW;YAC9B,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,KAAK,WAAW,CAAC,CAAA;YAC1E,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,eAAe,EAAE,CAAA;YACjD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAA;YAClE,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,MAAM;iBACzC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,KAAK,eAAe,IAAI,KAAK,CAAC,SAAS,KAAK,WAAW,CAAC;iBACzF,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAA;YAClC,MAAM,SAAS,GAAG,MAAM;iBACrB,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,KAAK,iBAAiB,CAAC;iBACxD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,GAAG,CAAC,CAAC,iBAAiB,IAAI,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC;iBACpF,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;YACT,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,iBAAiB,IAAI,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,cAAc,EAAE,CAAA;QACjH,CAAC;KACF,CAAA;AACH,CAAC;AAED,uFAAuF;AACvF,MAAM,UAAU,uBAAuB;IACrC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAiB,CAAA;IACvC,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,OAAO;QACL,MAAM;QACN,MAAM;QACN,KAAK,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE;YACjC,MAAM,GAAG,GAAG,GAAG,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAA;YAC7C,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC;gBAAE,OAAM;YAC9C,MAAM,CAAC,IAAI,CAAC,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC,CAAA;YAC9B,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,CAAA;QACxD,CAAC;KACF,CAAA;AACH,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,221 @@
+import type { Options as PRetryOptions, RetryContext as PRetryContext } from 'p-retry';
+import type { TableDefinition } from '@chkit/core';
+import type { FetchFailure } from './errors.js';
+/** JSON-compatible column values; the ClickHouse destination encodes bigint as decimal strings. */
+export type Row = Record<string, unknown>;
+/**
+ * One bounded unit produced by a stream reader. Rows are already shaped for the
+ * destination table: mapping is ordinary code inside the reader.
+ *
+ * `state` is optional and, when present, must be the COMPLETE candidate
+ * provider state that is safe to commit once every row up to and including
+ * this chunk has sink evidence. The executor never infers ordering: a reader
+ * that cannot claim a safe frontier simply omits `state`.
+ */
+export interface SourceChunk<TRow extends Row = Row, TState = unknown> {
+    rows: readonly TRow[];
+    state?: TState;
+    /**
+     * Stable, non-secret identity of the logical source interval this chunk
+     * covers (a page cursor, an id range, a day…). Declare it only when a replay
+     * of the same interval is the same logical write: batch identity then ignores
+     * row content, so mutable provider fields cannot defeat retry deduplication.
+     * Without it, identity falls back to a content hash, which prefers a possible
+     * duplicate over suppressing rows that changed between attempts.
+     */
+    id?: string;
+}
+export interface AttemptOptions {
+    /** Short non-secret label used in journal and telemetry, e.g. `GET /v2/objects`. */
+    label?: string;
+}
+/** Source request capabilities, independent of selection and checkpoint types. */
+export interface FetchContext {
+    signal: AbortSignal;
+    /**
+     * Run one source operation under executor authority: fetch permit, retry
+     * policy, failure classification and cancellation. Operations that bypass it
+     * only get coarse recovery (reader recreation from the last checkpoint).
+     */
+    attempt<T>(operation: (signal: AbortSignal) => Promise<T>, options?: AttemptOptions): Promise<T>;
+}
+export interface ReadContext<TSelection, TState> extends FetchContext {
+    streamId: string;
+    /** What to read this execution, planned by the incremental strategy. */
+    selection: TSelection;
+    /** Last committed provider state, already validated by the strategy. */
+    state: TState | undefined;
+    /** Immutable cutoff shared by every stream selected in this execution. */
+    cutoff: Date;
+}
+/**
+ * Provider-owned incremental strategy inside the ChKit-owned checkpoint
+ * envelope `{ strategy, version, state }`. Core treats `state` as opaque.
+ */
+export interface IncrementalStrategy<TState, TSelection> {
+    readonly id: string;
+    readonly version: number;
+    /** Validate previously committed state. Throw when it cannot be trusted. */
+    parseState(raw: unknown): TState;
+    plan(input: PlanInput<TState>): TSelection;
+    /**
+     * Candidate state once the WHOLE selection has sink evidence. Return
+     * `undefined` to leave the checkpoint untouched.
+     */
+    complete?(input: {
+        state: TState | undefined;
+        selection: TSelection;
+    }): TState | undefined;
+}
+export interface PlanInput<TState> {
+    state: TState | undefined;
+    cutoff: Date;
+    /** Explicit historical range from a backfill invocation. */
+    range: {
+        from: Date | undefined;
+        to: Date | undefined;
+    } | undefined;
+}
+/** p-retry context with the original provider failure and its classification. */
+export type RetryContext = Omit<PRetryContext, 'error'> & {
+    error: FetchFailure;
+};
+export type RetryOptions = Pick<PRetryOptions, 'retries' | 'factor' | 'minTimeout' | 'maxTimeout' | 'randomize' | 'maxRetryTime'> & {
+    shouldRetry?: (context: RetryContext) => boolean | Promise<boolean>;
+    shouldConsumeRetry?: (context: RetryContext) => boolean | Promise<boolean>;
+};
+export interface StreamBudget {
+    /** Maximum source chunks pulled in one execution of this stream. */
+    maxChunks?: number;
+    /** Largest chunk a reader may yield. A bigger chunk fails the stream instead of being buffered. */
+    maxChunkRows?: number;
+}
+export interface StreamDefinition<TRow extends Row = Row, TState = unknown, TSelection = unknown> {
+    readonly kind: 'ingest_stream';
+    /** Globally stable identity. Owns the checkpoint; never derived from the pipeline. */
+    readonly id: string;
+    readonly destination: TableDefinition;
+    readonly tags: readonly string[];
+    readonly incremental: IncrementalStrategy<TState, TSelection>;
+    readonly read: (context: ReadContext<TSelection, TState>) => AsyncIterable<SourceChunk<TRow, TState>>;
+    readonly loader: LoaderFactory | undefined;
+    readonly retry: RetryOptions | undefined;
+    readonly batchSize: number | undefined;
+    readonly budget: StreamBudget | undefined;
+    readonly classifyError: ErrorClassifier | undefined;
+}
+export type AnyStreamDefinition = StreamDefinition<any, any, any>;
+export interface PipelineDefinition {
+    readonly kind: 'ingest_pipeline';
+    readonly id: string;
+    readonly tags: readonly string[];
+    readonly streams: readonly AnyStreamDefinition[];
+    readonly maxStreams: number;
+    readonly maxFetches: number;
+    readonly maxLoads: number;
+    readonly retry: RetryOptions | undefined;
+}
+export type FailureClass = {
+    kind: 'cancelled';
+} | {
+    kind: 'rate_limited';
+    retryAfterMs: number | undefined;
+} | {
+    kind: 'transient';
+} | {
+    kind: 'permanent';
+} | {
+    kind: 'unknown';
+};
+export type ErrorClassifier = (cause: unknown, fallback: FailureClass) => FailureClass | undefined;
+export type SinkEvidenceKind = 'clickhouse_ack' | 'none_required';
+export interface SinkReceipt {
+    evidence: SinkEvidenceKind;
+    rows: number;
+    writeUnits: number;
+}
+export interface LoadBatch {
+    /** Stable across every retry and crash replay of the same logical batch. */
+    batchId: string;
+    rows: readonly Row[];
+}
+export interface LoaderContext {
+    streamId: string;
+    runId: string;
+    table: TableDefinition;
+    destination: DestinationAdapter;
+    signal: AbortSignal;
+}
+export interface Loader {
+    readonly ctx: LoaderContext;
+    write(batch: LoadBatch): Promise<void>;
+    finalize(): Promise<SinkReceipt>;
+    abort(reason: unknown): Promise<void>;
+}
+export type LoaderFactory = (ctx: LoaderContext) => Loader;
+/** Low-level destination writes performed on behalf of a portable Loader. */
+export interface DestinationAdapter {
+    insert(input: {
+        table: TableDefinition;
+        rows: readonly Row[];
+        /** Deterministic `insert_deduplication_token` for this write unit. */
+        token: string;
+    }): Promise<void>;
+}
+export type JournalEventKind = 'run_started' | 'work_planned' | 'attempt_started' | 'retry_scheduled' | 'batch_committed' | 'work_finished' | 'run_finished';
+export type WorkState = '' | 'planned' | 'running' | 'succeeded' | 'failed' | 'budget_exhausted' | 'cancelled';
+export interface CheckpointEnvelope {
+    strategy: string;
+    version: number;
+    state: unknown;
+}
+export interface JournalEvent {
+    namespaceId: string;
+    eventSeq: number;
+    eventKind: JournalEventKind;
+    runId: string;
+    workId: string;
+    attemptNo: number;
+    batchId: string;
+    expectedCheckpointVersion: number;
+    checkpointVersion: number;
+    checkpoint: CheckpointEnvelope | undefined;
+    workState: WorkState;
+    sinkEvidence: SinkEvidenceKind | '';
+    retryAt: Date | undefined;
+    errorClass: string;
+    detail: Record<string, unknown>;
+}
+export interface CommittedCheckpoint {
+    version: number;
+    envelope: CheckpointEnvelope | undefined;
+    /** Highest journal sequence observed for the namespace (any event kind). */
+    headSeq: number;
+    /** Last successful work_finished sequence, or zero. Separates new syncs from replays. */
+    lastSuccessSeq: number;
+}
+/** Authoritative append-only control state. Checkpoints are projections of it. */
+export interface Journal {
+    ensure(): Promise<void>;
+    append(event: JournalEvent): Promise<void>;
+    readCheckpoint(namespaceId: string): Promise<CommittedCheckpoint>;
+}
+export type StreamOutcome = 'succeeded' | 'failed' | 'budget_exhausted' | 'cancelled';
+export interface StreamResult {
+    streamId: string;
+    pipelineId: string;
+    namespaceId: string;
+    outcome: StreamOutcome;
+    rows: number;
+    batches: number;
+    chunks: number;
+    checkpointVersion: number;
+    error: string | undefined;
+}
+export interface ExecutionResult {
+    runId: string;
+    cutoff: string;
+    streams: StreamResult[];
+    ok: boolean;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,IAAI,aAAa,EAAE,YAAY,IAAI,aAAa,EAAE,MAAM,SAAS,CAAA;AAEtF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAElD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAI/C,mGAAmG;AACnG,MAAM,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;AAEzC;;;;;;;;GAQG;AACH,MAAM,WAAW,WAAW,CAAC,IAAI,SAAS,GAAG,GAAG,GAAG,EAAE,MAAM,GAAG,OAAO;IACnE,IAAI,EAAE,SAAS,IAAI,EAAE,CAAA;IACrB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;;;;;;;OAOG;IACH,EAAE,CAAC,EAAE,MAAM,CAAA;CACZ;AAED,MAAM,WAAW,cAAc;IAC7B,oFAAoF;IACpF,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,kFAAkF;AAClF,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,WAAW,CAAA;IACnB;;;;OAIG;IACH,OAAO,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;CACjG;AAED,MAAM,WAAW,WAAW,CAAC,UAAU,EAAE,MAAM,CAAE,SAAQ,YAAY;IACnE,QAAQ,EAAE,MAAM,CAAA;IAChB,wEAAwE;IACxE,SAAS,EAAE,UAAU,CAAA;IACrB,wEAAwE;IACxE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAA;IACzB,0EAA0E;IAC1E,MAAM,EAAE,IAAI,CAAA;CACb;AAED;;;GAGG;AACH,MAAM,WAAW,mBAAmB,CAAC,MAAM,EAAE,UAAU;IACrD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,4EAA4E;IAC5E,UAAU,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,CAAA;IAChC,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,GAAG,UAAU,CAAA;IAC1C;;;OAGG;IACH,QAAQ,CAAC,CAAC,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;QAAC,SAAS,EAAE,UAAU,CAAA;KAAE,GAAG,MAAM,GAAG,SAAS,CAAA;CAC3F;AAED,MAAM,WAAW,SAAS,CAAC,MAAM;IAC/B,KAAK,EAAE,MAAM,GAAG,SAAS,CAAA;IACzB,MAAM,EAAE,IAAI,CAAA;IACZ,4DAA4D;IAC5D,KAAK,EAAE;QAAE,IAAI,EAAE,IAAI,GAAG,SAAS,CAAC;QAAC,EAAE,EAAE,IAAI,GAAG,SAAS,CAAA;KAAE,GAAG,SAAS,CAAA;CACpE;AAED,iFAAiF;AACjF,MAAM,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,EAAE,OAAO,CAAC,GAAG;IAAE,KAAK,EAAE,YAAY,CAAA;CAAE,CAAA;AAEjF,MAAM,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,EAC3C,SAAS,GAAG,QAAQ,GAAG,YAAY,GAAG,YAAY,GAAG,WAAW,GAAG,cAAc,CAClF,GAAG;IACF,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;IACnE,kBAAkB,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;CAC3E,CAAA;AAED,MAAM,WAAW,YAAY;IAC3B,oEAAoE;IACpE,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,mGAAmG;IACnG,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB;AAED,MAAM,WAAW,gBAAgB,CAAC,IAAI,SAAS,GAAG,GAAG,GAAG,EAAE,MAAM,GAAG,OAAO,EAAE,UAAU,GAAG,OAAO;IAC9F,QAAQ,CAAC,IAAI,EAAE,eAAe,CAAA;IAC9B,sFAAsF;IACtF,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,WAAW,EAAE,eAAe,CAAA;IACrC,QAAQ,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,CAAA;IAChC,QAAQ,CAAC,WAAW,EAAE,mBAAmB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;IAC7D,QAAQ,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,WAAW,CAAC,UAAU,EAAE,MAAM,CAAC,KAAK,aAAa,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAA;IACrG,QAAQ,CAAC,MAAM,EAAE,aAAa,GAAG,SAAS,CAAA;IAC1C,QAAQ,CAAC,KAAK,EAAE,YAAY,GAAG,SAAS,CAAA;IACxC,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,CAAA;IACtC,QAAQ,CAAC,MAAM,EAAE,YAAY,GAAG,SAAS,CAAA;IACzC,QAAQ,CAAC,aAAa,EAAE,eAAe,GAAG,SAAS,CAAA;CACpD;AAGD,MAAM,MAAM,mBAAmB,GAAG,gBAAgB,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;AAEjE,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,IAAI,EAAE,iBAAiB,CAAA;IAChC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,CAAA;IAChC,QAAQ,CAAC,OAAO,EAAE,SAAS,mBAAmB,EAAE,CAAA;IAChD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,KAAK,EAAE,YAAY,GAAG,SAAS,CAAA;CACzC;AAID,MAAM,MAAM,YAAY,GACpB;IAAE,IAAI,EAAE,WAAW,CAAA;CAAE,GACrB;IAAE,IAAI,EAAE,cAAc,CAAC;IAAC,YAAY,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE,GAC1D;IAAE,IAAI,EAAE,WAAW,CAAA;CAAE,GACrB;IAAE,IAAI,EAAE,WAAW,CAAA;CAAE,GACrB;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,CAAA;AAEvB,MAAM,MAAM,eAAe,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,KAAK,YAAY,GAAG,SAAS,CAAA;AAIlG,MAAM,MAAM,gBAAgB,GAAG,gBAAgB,GAAG,eAAe,CAAA;AAEjE,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,gBAAgB,CAAA;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,SAAS;IACxB,4EAA4E;IAC5E,OAAO,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,SAAS,GAAG,EAAE,CAAA;CACrB;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,eAAe,CAAA;IACtB,WAAW,EAAE,kBAAkB,CAAA;IAC/B,MAAM,EAAE,WAAW,CAAA;CACpB;AAED,MAAM,WAAW,MAAM;IACrB,QAAQ,CAAC,GAAG,EAAE,aAAa,CAAA;IAC3B,KAAK,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACtC,QAAQ,IAAI,OAAO,CAAC,WAAW,CAAC,CAAA;IAChC,KAAK,CAAC,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CACtC;AAED,MAAM,MAAM,aAAa,GAAG,CAAC,GAAG,EAAE,aAAa,KAAK,MAAM,CAAA;AAE1D,6EAA6E;AAC7E,MAAM,WAAW,kBAAkB;IACjC,MAAM,CAAC,KAAK,EAAE;QACZ,KAAK,EAAE,eAAe,CAAA;QACtB,IAAI,EAAE,SAAS,GAAG,EAAE,CAAA;QACpB,sEAAsE;QACtE,KAAK,EAAE,MAAM,CAAA;KACd,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAClB;AAID,MAAM,MAAM,gBAAgB,GACxB,aAAa,GACb,cAAc,GACd,iBAAiB,GACjB,iBAAiB,GACjB,iBAAiB,GACjB,eAAe,GACf,cAAc,CAAA;AAElB,MAAM,MAAM,SAAS,GAAG,EAAE,GAAG,SAAS,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,GAAG,kBAAkB,GAAG,WAAW,CAAA;AAE9G,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,MAAM,CAAA;IAChB,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,OAAO,CAAA;CACf;AAED,MAAM,WAAW,YAAY;IAC3B,WAAW,EAAE,MAAM,CAAA;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,gBAAgB,CAAA;IAC3B,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;IACf,yBAAyB,EAAE,MAAM,CAAA;IACjC,iBAAiB,EAAE,MAAM,CAAA;IACzB,UAAU,EAAE,kBAAkB,GAAG,SAAS,CAAA;IAC1C,SAAS,EAAE,SAAS,CAAA;IACpB,YAAY,EAAE,gBAAgB,GAAG,EAAE,CAAA;IACnC,OAAO,EAAE,IAAI,GAAG,SAAS,CAAA;IACzB,UAAU,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAChC;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,kBAAkB,GAAG,SAAS,CAAA;IACxC,4EAA4E;IAC5E,OAAO,EAAE,MAAM,CAAA;IACf,yFAAyF;IACzF,cAAc,EAAE,MAAM,CAAA;CACvB;AAED,kFAAkF;AAClF,MAAM,WAAW,OAAO;IACtB,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;IACvB,MAAM,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAC1C,cAAc,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAA;CAClE;AAID,MAAM,MAAM,aAAa,GAAG,WAAW,GAAG,QAAQ,GAAG,kBAAkB,GAAG,WAAW,CAAA;AAErF,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;IACnB,OAAO,EAAE,aAAa,CAAA;IACtB,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,iBAAiB,EAAE,MAAM,CAAA;IACzB,KAAK,EAAE,MAAM,GAAG,SAAS,CAAA;CAC1B;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,YAAY,EAAE,CAAA;IACvB,EAAE,EAAE,OAAO,CAAA;CACZ"}

package/dist/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}

package/package.json

@@ -0,0 +1,57 @@
+{
+  "name": "@chkit/plugin-ingest",
+  "version": "0.2.0-beta.8",
+  "description": "Scheduled pull ingestion into ClickHouse with journaled checkpoints for chkit",
+  "license": "MIT",
+  "author": "ObsessionDB",
+  "keywords": [
+    "clickhouse",
+    "ingestion",
+    "etl",
+    "chkit-plugin"
+  ],
+  "homepage": "https://chkit.obsessiondb.com",
+  "bugs": {
+    "url": "https://github.com/obsessiondb/chkit/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/obsessiondb/chkit.git",
+    "directory": "packages/plugin-ingest"
+  },
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "source": "./src/index.ts",
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./testing": {
+      "source": "./src/testing.ts",
+      "types": "./dist/testing.d.ts",
+      "default": "./dist/testing.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "dev": "tsc -p tsconfig.json --watch",
+    "typecheck": "tsc -p tsconfig.json --noEmit",
+    "lint": "biome lint src",
+    "test": "bun test src",
+    "test:env": "doppler run --project chkit --config ci -- bun test src",
+    "clean": "rm -rf dist"
+  },
+  "dependencies": {
+    "@chkit/clickhouse": "0.2.0-beta.8",
+    "@chkit/core": "0.2.0-beta.8",
+    "@opentelemetry/api": "^1.9.1",
+    "p-limit": "^7.2.0",
+    "p-retry": "^7.1.1",
+    "zod": "^4.3.6"
+  }
+}