khotan-data 0.0.1

package/dist/drizzle.d.ts

@@ -0,0 +1,120 @@
+import { D as DataRecord, E as Extractor, b as Loader } from './types-EpLTQcN2.js';
+
+/**
+ * Create an extractor from any Drizzle select query.
+ *
+ * Pass a function that returns the query — this makes the extractor
+ * re-runnable and avoids consuming a one-shot promise.
+ *
+ * @example
+ * ```ts
+ * import { fromQuery } from "khotan-data/drizzle";
+ * import { db } from "@/db";
+ * import { users } from "@/db/schema";
+ * import { eq } from "drizzle-orm";
+ *
+ * const extractor = fromQuery("active-users", () =>
+ *   db.select().from(users).where(eq(users.active, true))
+ * );
+ * ```
+ */
+declare function fromQuery<T extends DataRecord>(name: string, queryFn: () => PromiseLike<T[]>): Extractor<T>;
+/**
+ * Create an extractor from a Drizzle query that streams results in
+ * chunks. Use this for large tables where materializing all rows
+ * at once is too expensive.
+ *
+ * @example
+ * ```ts
+ * const extractor = fromQueryCursor("all-events", async function* () {
+ *   let offset = 0;
+ *   const limit = 5000;
+ *   while (true) {
+ *     const batch = await db.select().from(events).limit(limit).offset(offset);
+ *     if (batch.length === 0) break;
+ *     yield* batch;
+ *     offset += limit;
+ *   }
+ * });
+ * ```
+ */
+declare function fromQueryCursor<T extends DataRecord>(name: string, generatorFn: () => AsyncIterable<T>): Extractor<T>;
+/**
+ * Create an extractor from a paginated Drizzle query. Automatically
+ * handles offset-based pagination so you don't have to write the loop.
+ *
+ * @example
+ * ```ts
+ * const extractor = fromQueryPaginated("all-users", {
+ *   pageSize: 2000,
+ *   query: (limit, offset) => db.select().from(users).limit(limit).offset(offset),
+ * });
+ * ```
+ */
+declare function fromQueryPaginated<T extends DataRecord>(name: string, opts: {
+    query: (limit: number, offset: number) => PromiseLike<T[]>;
+    pageSize?: number;
+}): Extractor<T>;
+
+interface ToDrizzleOptions {
+    /**
+     * Max records per INSERT statement. When a batch exceeds this,
+     * it's automatically split into sub-batches to stay within Postgres
+     * parameter limits. Defaults to auto-calculated from columnsPerRow.
+     */
+    maxRowsPerStatement?: number;
+    /**
+     * Number of columns per row. Used to auto-calculate maxRowsPerStatement
+     * to stay under Postgres' 65535 parameter limit.
+     * If not provided, falls back to maxRowsPerStatement or 1000.
+     */
+    columnsPerRow?: number;
+}
+/**
+ * Create a loader that writes records using a Drizzle insert/upsert.
+ *
+ * You provide the write function — this keeps the loader decoupled from
+ * specific Drizzle driver types while giving you full control over
+ * insert/upsert/conflict behavior.
+ *
+ * Automatically sub-batches to stay within Postgres' 65535 parameter limit.
+ *
+ * @example
+ * ```ts
+ * import { toDrizzle } from "khotan-data/drizzle";
+ * import { db } from "@/db";
+ * import { processedUsers } from "@/db/schema";
+ *
+ * // Simple insert
+ * const loader = toDrizzle("insert-users", (rows) =>
+ *   db.insert(processedUsers).values(rows)
+ * );
+ *
+ * // Upsert
+ * const loader = toDrizzle("upsert-users", (rows) =>
+ *   db.insert(processedUsers).values(rows).onConflictDoUpdate({
+ *     target: processedUsers.id,
+ *     set: { name: sql`excluded.name`, updatedAt: new Date() },
+ *   })
+ * );
+ * ```
+ */
+declare function toDrizzle<T extends DataRecord>(name: string, writeFn: (records: T[]) => PromiseLike<unknown>, options?: ToDrizzleOptions): Loader<T>;
+/**
+ * Create a loader that writes records inside a Drizzle transaction.
+ *
+ * All sub-batches for a single load call are wrapped in a single
+ * transaction — if any batch fails, the entire load is rolled back.
+ *
+ * @example
+ * ```ts
+ * const loader = toDrizzleTx("tx-insert", db, (tx, rows) =>
+ *   tx.insert(processedUsers).values(rows)
+ * );
+ * ```
+ */
+declare function toDrizzleTx<T extends DataRecord>(name: string, db: {
+    transaction: <R>(fn: (tx: never) => Promise<R>) => Promise<R>;
+}, writeFn: (tx: never, records: T[]) => PromiseLike<unknown>, options?: ToDrizzleOptions): Loader<T>;
+
+export { fromQuery, fromQueryCursor, fromQueryPaginated, toDrizzle, toDrizzleTx };

package/dist/drizzle.js

@@ -0,0 +1,3 @@
+export { fromQuery, fromQueryCursor, fromQueryPaginated, toDrizzle, toDrizzleTx } from './chunk-42CNRMAQ.js';
+//# sourceMappingURL=drizzle.js.map
+//# sourceMappingURL=drizzle.js.map

package/dist/drizzle.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"names":[],"mappings":"","file":"drizzle.js"}

package/dist/index.cjs

@@ -0,0 +1,107 @@
+'use strict';
+
+var chunk6PDC7DFX_cjs = require('./chunk-6PDC7DFX.cjs');
+var chunk6R4QVX2Q_cjs = require('./chunk-6R4QVX2Q.cjs');
+var chunkFRRSW3TN_cjs = require('./chunk-FRRSW3TN.cjs');
+
+// src/extractors.ts
+function createExtractor(name, fn) {
+  return {
+    name,
+    extract: fn
+  };
+}
+function fromArray(name, data) {
+  return createExtractor(name, async function* () {
+    for (const record of data) {
+      yield record;
+    }
+  });
+}
+function fromIterable(name, iterable) {
+  return createExtractor(name, () => iterable);
+}
+
+// src/loaders.ts
+function createLoader(name, fn) {
+  return { name, load: fn };
+}
+function toArray(name, target) {
+  return createLoader(name, async (records) => {
+    target.push(...records);
+    return { recordsLoaded: records.length, errors: [] };
+  });
+}
+function toConsole(name) {
+  return createLoader(name, async (records) => {
+    for (const record of records) {
+      console.log(JSON.stringify(record));
+    }
+    return { recordsLoaded: records.length, errors: [] };
+  });
+}
+
+Object.defineProperty(exports, "Pipeline", {
+  enumerable: true,
+  get: function () { return chunk6PDC7DFX_cjs.Pipeline; }
+});
+Object.defineProperty(exports, "compose", {
+  enumerable: true,
+  get: function () { return chunk6R4QVX2Q_cjs.compose; }
+});
+Object.defineProperty(exports, "createTransformer", {
+  enumerable: true,
+  get: function () { return chunk6R4QVX2Q_cjs.createTransformer; }
+});
+Object.defineProperty(exports, "filter", {
+  enumerable: true,
+  get: function () { return chunk6R4QVX2Q_cjs.filter; }
+});
+Object.defineProperty(exports, "flatMap", {
+  enumerable: true,
+  get: function () { return chunk6R4QVX2Q_cjs.flatMap; }
+});
+Object.defineProperty(exports, "map", {
+  enumerable: true,
+  get: function () { return chunk6R4QVX2Q_cjs.map; }
+});
+Object.defineProperty(exports, "omit", {
+  enumerable: true,
+  get: function () { return chunk6R4QVX2Q_cjs.omit; }
+});
+Object.defineProperty(exports, "pick", {
+  enumerable: true,
+  get: function () { return chunk6R4QVX2Q_cjs.pick; }
+});
+Object.defineProperty(exports, "rename", {
+  enumerable: true,
+  get: function () { return chunk6R4QVX2Q_cjs.rename; }
+});
+Object.defineProperty(exports, "fromQuery", {
+  enumerable: true,
+  get: function () { return chunkFRRSW3TN_cjs.fromQuery; }
+});
+Object.defineProperty(exports, "fromQueryCursor", {
+  enumerable: true,
+  get: function () { return chunkFRRSW3TN_cjs.fromQueryCursor; }
+});
+Object.defineProperty(exports, "fromQueryPaginated", {
+  enumerable: true,
+  get: function () { return chunkFRRSW3TN_cjs.fromQueryPaginated; }
+});
+Object.defineProperty(exports, "toDrizzle", {
+  enumerable: true,
+  get: function () { return chunkFRRSW3TN_cjs.toDrizzle; }
+});
+Object.defineProperty(exports, "toDrizzleTx", {
+  enumerable: true,
+  get: function () { return chunkFRRSW3TN_cjs.toDrizzleTx; }
+});
+exports.createExtractor = createExtractor;
+exports.createLoader = createLoader;
+exports.fromArray = fromArray;
+exports.fromIterable = fromIterable;
+exports.toArray = toArray;
+exports.toConsole = toConsole;
+//# sourceMappingURL=index.cjs.map
+//# sourceMappingURL=index.cjs.map

package/dist/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/extractors.ts","../src/loaders.ts"],"names":[],"mappings":";;;;;;;AAKO,SAAS,eAAA,CACd,MACA,EAAA,EACc;AACd,EAAA,OAAO;AAAA,IACL,IAAA;AAAA,IACA,OAAA,EAAS;AAAA,GACX;AACF;AAMO,SAAS,SAAA,CACd,MACA,IAAA,EACc;AACd,EAAA,OAAO,eAAA,CAAgB,MAAM,mBAAmB;AAC9C,IAAA,KAAA,MAAW,UAAU,IAAA,EAAM;AACzB,MAAA,MAAM,MAAA;AAAA,IACR;AAAA,EACF,CAAC,CAAA;AACH;AAKO,SAAS,YAAA,CACd,MACA,QAAA,EACc;AACd,EAAA,OAAO,eAAA,CAAgB,IAAA,EAAM,MAAM,QAAQ,CAAA;AAC7C;;;ACjCO,SAAS,YAAA,CACd,MACA,EAAA,EACW;AACX,EAAA,OAAO,EAAE,IAAA,EAAM,IAAA,EAAM,EAAA,EAAG;AAC1B;AAMO,SAAS,OAAA,CACd,MACA,MAAA,EACW;AACX,EAAA,OAAO,YAAA,CAAa,IAAA,EAAM,OAAO,OAAA,KAAY;AAC3C,IAAA,MAAA,CAAO,IAAA,CAAK,GAAG,OAAO,CAAA;AACtB,IAAA,OAAO,EAAE,aAAA,EAAe,OAAA,CAAQ,MAAA,EAAQ,MAAA,EAAQ,EAAC,EAAE;AAAA,EACrD,CAAC,CAAA;AACH;AAKO,SAAS,UAAgC,IAAA,EAAyB;AACvE,EAAA,OAAO,YAAA,CAAa,IAAA,EAAM,OAAO,OAAA,KAAY;AAC3C,IAAA,KAAA,MAAW,UAAU,OAAA,EAAS;AAC5B,MAAA,OAAA,CAAQ,GAAA,CAAI,IAAA,CAAK,SAAA,CAAU,MAAM,CAAC,CAAA;AAAA,IACpC;AACA,IAAA,OAAO,EAAE,aAAA,EAAe,OAAA,CAAQ,MAAA,EAAQ,MAAA,EAAQ,EAAC,EAAE;AAAA,EACrD,CAAC,CAAA;AACH","file":"index.cjs","sourcesContent":["import type { DataRecord, Extractor } from \"./types.js\";\n\n/**\n * Create a custom extractor from an async generator function.\n */\nexport function createExtractor<T extends DataRecord>(\n  name: string,\n  fn: () => AsyncIterable<T>,\n): Extractor<T> {\n  return {\n    name,\n    extract: fn,\n  };\n}\n\n/**\n * Create an extractor from an in-memory array. Useful for testing\n * and for small datasets that are already loaded.\n */\nexport function fromArray<T extends DataRecord>(\n  name: string,\n  data: T[],\n): Extractor<T> {\n  return createExtractor(name, async function* () {\n    for (const record of data) {\n      yield record;\n    }\n  });\n}\n\n/**\n * Create an extractor from any async iterable source.\n */\nexport function fromIterable<T extends DataRecord>(\n  name: string,\n  iterable: AsyncIterable<T>,\n): Extractor<T> {\n  return createExtractor(name, () => iterable);\n}\n","import type { DataRecord, Loader, LoadResult } from \"./types.js\";\n\n/**\n * Create a custom loader from a function.\n */\nexport function createLoader<T extends DataRecord>(\n  name: string,\n  fn: (records: T[]) => Promise<LoadResult>,\n): Loader<T> {\n  return { name, load: fn };\n}\n\n/**\n * Load records into an in-memory array. Useful for testing\n * and collecting pipeline output.\n */\nexport function toArray<T extends DataRecord>(\n  name: string,\n  target: T[],\n): Loader<T> {\n  return createLoader(name, async (records) => {\n    target.push(...records);\n    return { recordsLoaded: records.length, errors: [] };\n  });\n}\n\n/**\n * Load records by logging them to the console. Useful for debugging.\n */\nexport function toConsole<T extends DataRecord>(name: string): Loader<T> {\n  return createLoader(name, async (records) => {\n    for (const record of records) {\n      console.log(JSON.stringify(record));\n    }\n    return { recordsLoaded: records.length, errors: [] };\n  });\n}\n"]}

package/dist/index.d.cts

@@ -0,0 +1,35 @@
+import { D as DataRecord, E as Extractor, a as LoadResult, b as Loader } from './types-EpLTQcN2.cjs';
+export { L as LoadError, P as PipelineEvent, c as PipelineEventListener, d as PipelineOptions, e as PipelineResult, f as PipelineStepError, T as Transformer } from './types-EpLTQcN2.cjs';
+export { Pipeline } from './pipeline.cjs';
+export { compose, createTransformer, filter, flatMap, map, omit, pick, rename } from './transform.cjs';
+export { fromQuery, fromQueryCursor, fromQueryPaginated, toDrizzle, toDrizzleTx } from './drizzle.cjs';
+
+/**
+ * Create a custom extractor from an async generator function.
+ */
+declare function createExtractor<T extends DataRecord>(name: string, fn: () => AsyncIterable<T>): Extractor<T>;
+/**
+ * Create an extractor from an in-memory array. Useful for testing
+ * and for small datasets that are already loaded.
+ */
+declare function fromArray<T extends DataRecord>(name: string, data: T[]): Extractor<T>;
+/**
+ * Create an extractor from any async iterable source.
+ */
+declare function fromIterable<T extends DataRecord>(name: string, iterable: AsyncIterable<T>): Extractor<T>;
+
+/**
+ * Create a custom loader from a function.
+ */
+declare function createLoader<T extends DataRecord>(name: string, fn: (records: T[]) => Promise<LoadResult>): Loader<T>;
+/**
+ * Load records into an in-memory array. Useful for testing
+ * and collecting pipeline output.
+ */
+declare function toArray<T extends DataRecord>(name: string, target: T[]): Loader<T>;
+/**
+ * Load records by logging them to the console. Useful for debugging.
+ */
+declare function toConsole<T extends DataRecord>(name: string): Loader<T>;
+
+export { DataRecord, Extractor, LoadResult, Loader, createExtractor, createLoader, fromArray, fromIterable, toArray, toConsole };

package/dist/index.d.ts

@@ -0,0 +1,35 @@
+import { D as DataRecord, E as Extractor, a as LoadResult, b as Loader } from './types-EpLTQcN2.js';
+export { L as LoadError, P as PipelineEvent, c as PipelineEventListener, d as PipelineOptions, e as PipelineResult, f as PipelineStepError, T as Transformer } from './types-EpLTQcN2.js';
+export { Pipeline } from './pipeline.js';
+export { compose, createTransformer, filter, flatMap, map, omit, pick, rename } from './transform.js';
+export { fromQuery, fromQueryCursor, fromQueryPaginated, toDrizzle, toDrizzleTx } from './drizzle.js';
+
+/**
+ * Create a custom extractor from an async generator function.
+ */
+declare function createExtractor<T extends DataRecord>(name: string, fn: () => AsyncIterable<T>): Extractor<T>;
+/**
+ * Create an extractor from an in-memory array. Useful for testing
+ * and for small datasets that are already loaded.
+ */
+declare function fromArray<T extends DataRecord>(name: string, data: T[]): Extractor<T>;
+/**
+ * Create an extractor from any async iterable source.
+ */
+declare function fromIterable<T extends DataRecord>(name: string, iterable: AsyncIterable<T>): Extractor<T>;
+
+/**
+ * Create a custom loader from a function.
+ */
+declare function createLoader<T extends DataRecord>(name: string, fn: (records: T[]) => Promise<LoadResult>): Loader<T>;
+/**
+ * Load records into an in-memory array. Useful for testing
+ * and collecting pipeline output.
+ */
+declare function toArray<T extends DataRecord>(name: string, target: T[]): Loader<T>;
+/**
+ * Load records by logging them to the console. Useful for debugging.
+ */
+declare function toConsole<T extends DataRecord>(name: string): Loader<T>;
+
+export { DataRecord, Extractor, LoadResult, Loader, createExtractor, createLoader, fromArray, fromIterable, toArray, toConsole };

package/dist/index.js

@@ -0,0 +1,44 @@
+export { Pipeline } from './chunk-TK4HD4XA.js';
+export { compose, createTransformer, filter, flatMap, map, omit, pick, rename } from './chunk-NVPI7OV3.js';
+export { fromQuery, fromQueryCursor, fromQueryPaginated, toDrizzle, toDrizzleTx } from './chunk-42CNRMAQ.js';
+
+// src/extractors.ts
+function createExtractor(name, fn) {
+  return {
+    name,
+    extract: fn
+  };
+}
+function fromArray(name, data) {
+  return createExtractor(name, async function* () {
+    for (const record of data) {
+      yield record;
+    }
+  });
+}
+function fromIterable(name, iterable) {
+  return createExtractor(name, () => iterable);
+}
+
+// src/loaders.ts
+function createLoader(name, fn) {
+  return { name, load: fn };
+}
+function toArray(name, target) {
+  return createLoader(name, async (records) => {
+    target.push(...records);
+    return { recordsLoaded: records.length, errors: [] };
+  });
+}
+function toConsole(name) {
+  return createLoader(name, async (records) => {
+    for (const record of records) {
+      console.log(JSON.stringify(record));
+    }
+    return { recordsLoaded: records.length, errors: [] };
+  });
+}
+
+export { createExtractor, createLoader, fromArray, fromIterable, toArray, toConsole };
+//# sourceMappingURL=index.js.map
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/extractors.ts","../src/loaders.ts"],"names":[],"mappings":";;;;;AAKO,SAAS,eAAA,CACd,MACA,EAAA,EACc;AACd,EAAA,OAAO;AAAA,IACL,IAAA;AAAA,IACA,OAAA,EAAS;AAAA,GACX;AACF;AAMO,SAAS,SAAA,CACd,MACA,IAAA,EACc;AACd,EAAA,OAAO,eAAA,CAAgB,MAAM,mBAAmB;AAC9C,IAAA,KAAA,MAAW,UAAU,IAAA,EAAM;AACzB,MAAA,MAAM,MAAA;AAAA,IACR;AAAA,EACF,CAAC,CAAA;AACH;AAKO,SAAS,YAAA,CACd,MACA,QAAA,EACc;AACd,EAAA,OAAO,eAAA,CAAgB,IAAA,EAAM,MAAM,QAAQ,CAAA;AAC7C;;;ACjCO,SAAS,YAAA,CACd,MACA,EAAA,EACW;AACX,EAAA,OAAO,EAAE,IAAA,EAAM,IAAA,EAAM,EAAA,EAAG;AAC1B;AAMO,SAAS,OAAA,CACd,MACA,MAAA,EACW;AACX,EAAA,OAAO,YAAA,CAAa,IAAA,EAAM,OAAO,OAAA,KAAY;AAC3C,IAAA,MAAA,CAAO,IAAA,CAAK,GAAG,OAAO,CAAA;AACtB,IAAA,OAAO,EAAE,aAAA,EAAe,OAAA,CAAQ,MAAA,EAAQ,MAAA,EAAQ,EAAC,EAAE;AAAA,EACrD,CAAC,CAAA;AACH;AAKO,SAAS,UAAgC,IAAA,EAAyB;AACvE,EAAA,OAAO,YAAA,CAAa,IAAA,EAAM,OAAO,OAAA,KAAY;AAC3C,IAAA,KAAA,MAAW,UAAU,OAAA,EAAS;AAC5B,MAAA,OAAA,CAAQ,GAAA,CAAI,IAAA,CAAK,SAAA,CAAU,MAAM,CAAC,CAAA;AAAA,IACpC;AACA,IAAA,OAAO,EAAE,aAAA,EAAe,OAAA,CAAQ,MAAA,EAAQ,MAAA,EAAQ,EAAC,EAAE;AAAA,EACrD,CAAC,CAAA;AACH","file":"index.js","sourcesContent":["import type { DataRecord, Extractor } from \"./types.js\";\n\n/**\n * Create a custom extractor from an async generator function.\n */\nexport function createExtractor<T extends DataRecord>(\n  name: string,\n  fn: () => AsyncIterable<T>,\n): Extractor<T> {\n  return {\n    name,\n    extract: fn,\n  };\n}\n\n/**\n * Create an extractor from an in-memory array. Useful for testing\n * and for small datasets that are already loaded.\n */\nexport function fromArray<T extends DataRecord>(\n  name: string,\n  data: T[],\n): Extractor<T> {\n  return createExtractor(name, async function* () {\n    for (const record of data) {\n      yield record;\n    }\n  });\n}\n\n/**\n * Create an extractor from any async iterable source.\n */\nexport function fromIterable<T extends DataRecord>(\n  name: string,\n  iterable: AsyncIterable<T>,\n): Extractor<T> {\n  return createExtractor(name, () => iterable);\n}\n","import type { DataRecord, Loader, LoadResult } from \"./types.js\";\n\n/**\n * Create a custom loader from a function.\n */\nexport function createLoader<T extends DataRecord>(\n  name: string,\n  fn: (records: T[]) => Promise<LoadResult>,\n): Loader<T> {\n  return { name, load: fn };\n}\n\n/**\n * Load records into an in-memory array. Useful for testing\n * and collecting pipeline output.\n */\nexport function toArray<T extends DataRecord>(\n  name: string,\n  target: T[],\n): Loader<T> {\n  return createLoader(name, async (records) => {\n    target.push(...records);\n    return { recordsLoaded: records.length, errors: [] };\n  });\n}\n\n/**\n * Load records by logging them to the console. Useful for debugging.\n */\nexport function toConsole<T extends DataRecord>(name: string): Loader<T> {\n  return createLoader(name, async (records) => {\n    for (const record of records) {\n      console.log(JSON.stringify(record));\n    }\n    return { recordsLoaded: records.length, errors: [] };\n  });\n}\n"]}

package/dist/pipeline.cjs

@@ -0,0 +1,12 @@
+'use strict';
+
+var chunk6PDC7DFX_cjs = require('./chunk-6PDC7DFX.cjs');
+
+
+
+Object.defineProperty(exports, "Pipeline", {
+  enumerable: true,
+  get: function () { return chunk6PDC7DFX_cjs.Pipeline; }
+});
+//# sourceMappingURL=pipeline.cjs.map
+//# sourceMappingURL=pipeline.cjs.map

package/dist/pipeline.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"names":[],"mappings":"","file":"pipeline.cjs"}

package/dist/pipeline.d.cts

@@ -0,0 +1,43 @@
+import { D as DataRecord, E as Extractor, T as Transformer, b as Loader, c as PipelineEventListener, d as PipelineOptions, e as PipelineResult } from './types-EpLTQcN2.cjs';
+export { P as PipelineEvent, f as PipelineStepError } from './types-EpLTQcN2.cjs';
+
+/**
+ * A composable, type-safe ETL pipeline builder.
+ *
+ * @example
+ * ```ts
+ * const result = await Pipeline.create("my-pipeline")
+ *   .extract(myExtractor)
+ *   .transform(myTransformer)
+ *   .load(myLoader)
+ *   .run();
+ * ```
+ */
+declare class Pipeline<TCurrent extends DataRecord = DataRecord> {
+    #private;
+    private constructor();
+    static create(name: string): Pipeline;
+    get name(): string;
+    /**
+     * Set the data source for this pipeline.
+     */
+    extract<T extends DataRecord>(extractor: Extractor<T>): Pipeline<T>;
+    /**
+     * Add a transformation step.
+     */
+    transform<TOutput extends DataRecord>(transformer: Transformer<TCurrent, TOutput>): Pipeline<TOutput>;
+    /**
+     * Add a load destination.
+     */
+    load(loader: Loader<TCurrent>): Pipeline<TCurrent>;
+    /**
+     * Subscribe to pipeline events.
+     */
+    on(listener: PipelineEventListener): Pipeline<TCurrent>;
+    /**
+     * Execute the pipeline.
+     */
+    run(options?: PipelineOptions): Promise<PipelineResult>;
+}
+
+export { Pipeline, PipelineEventListener, PipelineOptions, PipelineResult };

package/dist/pipeline.d.ts

@@ -0,0 +1,43 @@
+import { D as DataRecord, E as Extractor, T as Transformer, b as Loader, c as PipelineEventListener, d as PipelineOptions, e as PipelineResult } from './types-EpLTQcN2.js';
+export { P as PipelineEvent, f as PipelineStepError } from './types-EpLTQcN2.js';
+
+/**
+ * A composable, type-safe ETL pipeline builder.
+ *
+ * @example
+ * ```ts
+ * const result = await Pipeline.create("my-pipeline")
+ *   .extract(myExtractor)
+ *   .transform(myTransformer)
+ *   .load(myLoader)
+ *   .run();
+ * ```
+ */
+declare class Pipeline<TCurrent extends DataRecord = DataRecord> {
+    #private;
+    private constructor();
+    static create(name: string): Pipeline;
+    get name(): string;
+    /**
+     * Set the data source for this pipeline.
+     */
+    extract<T extends DataRecord>(extractor: Extractor<T>): Pipeline<T>;
+    /**
+     * Add a transformation step.
+     */
+    transform<TOutput extends DataRecord>(transformer: Transformer<TCurrent, TOutput>): Pipeline<TOutput>;
+    /**
+     * Add a load destination.
+     */
+    load(loader: Loader<TCurrent>): Pipeline<TCurrent>;
+    /**
+     * Subscribe to pipeline events.
+     */
+    on(listener: PipelineEventListener): Pipeline<TCurrent>;
+    /**
+     * Execute the pipeline.
+     */
+    run(options?: PipelineOptions): Promise<PipelineResult>;
+}
+
+export { Pipeline, PipelineEventListener, PipelineOptions, PipelineResult };

package/dist/pipeline.js

@@ -0,0 +1,3 @@
+export { Pipeline } from './chunk-TK4HD4XA.js';
+//# sourceMappingURL=pipeline.js.map
+//# sourceMappingURL=pipeline.js.map

package/dist/pipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"names":[],"mappings":"","file":"pipeline.js"}

package/dist/transform.cjs

@@ -0,0 +1,40 @@
+'use strict';
+
+var chunk6R4QVX2Q_cjs = require('./chunk-6R4QVX2Q.cjs');
+
+
+
+Object.defineProperty(exports, "compose", {
+  enumerable: true,
+  get: function () { return chunk6R4QVX2Q_cjs.compose; }
+});
+Object.defineProperty(exports, "createTransformer", {
+  enumerable: true,
+  get: function () { return chunk6R4QVX2Q_cjs.createTransformer; }
+});
+Object.defineProperty(exports, "filter", {
+  enumerable: true,
+  get: function () { return chunk6R4QVX2Q_cjs.filter; }
+});
+Object.defineProperty(exports, "flatMap", {
+  enumerable: true,
+  get: function () { return chunk6R4QVX2Q_cjs.flatMap; }
+});
+Object.defineProperty(exports, "map", {
+  enumerable: true,
+  get: function () { return chunk6R4QVX2Q_cjs.map; }
+});
+Object.defineProperty(exports, "omit", {
+  enumerable: true,
+  get: function () { return chunk6R4QVX2Q_cjs.omit; }
+});
+Object.defineProperty(exports, "pick", {
+  enumerable: true,
+  get: function () { return chunk6R4QVX2Q_cjs.pick; }
+});
+Object.defineProperty(exports, "rename", {
+  enumerable: true,
+  get: function () { return chunk6R4QVX2Q_cjs.rename; }
+});
+//# sourceMappingURL=transform.cjs.map
+//# sourceMappingURL=transform.cjs.map

package/dist/transform.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"names":[],"mappings":"","file":"transform.cjs"}

package/dist/transform.d.cts

@@ -0,0 +1,38 @@
+import { D as DataRecord, T as Transformer } from './types-EpLTQcN2.cjs';
+
+/**
+ * Create a custom transformer from a function.
+ */
+declare function createTransformer<TInput extends DataRecord = DataRecord, TOutput extends DataRecord = DataRecord>(name: string, fn: (record: TInput) => TOutput | TOutput[] | Promise<TOutput | TOutput[]>): Transformer<TInput, TOutput>;
+/**
+ * Transform each record using a mapping function.
+ */
+declare function map<TInput extends DataRecord = DataRecord, TOutput extends DataRecord = DataRecord>(name: string, fn: (record: TInput) => TOutput | Promise<TOutput>): Transformer<TInput, TOutput>;
+/**
+ * Filter records based on a predicate. Records that don't match
+ * are dropped (returned as empty array).
+ */
+declare function filter<T extends DataRecord = DataRecord>(name: string, predicate: (record: T) => boolean | Promise<boolean>): Transformer<T, T>;
+/**
+ * Transform each record into zero or more records.
+ */
+declare function flatMap<TInput extends DataRecord = DataRecord, TOutput extends DataRecord = DataRecord>(name: string, fn: (record: TInput) => TOutput[] | Promise<TOutput[]>): Transformer<TInput, TOutput>;
+/**
+ * Pick specific keys from each record.
+ */
+declare function pick<T extends DataRecord, K extends keyof T & string>(name: string, keys: K[]): Transformer<T, Pick<T, K> & DataRecord>;
+/**
+ * Omit specific keys from each record.
+ */
+declare function omit<T extends DataRecord, K extends keyof T & string>(name: string, keys: K[]): Transformer<T, Omit<T, K> & DataRecord>;
+/**
+ * Rename keys in each record.
+ */
+declare function rename<T extends DataRecord>(name: string, mapping: Record<string, string>): Transformer<T>;
+/**
+ * Compose multiple transformers into a single transformer that
+ * applies them in sequence.
+ */
+declare function compose<T extends DataRecord>(name: string, transformers: Transformer[]): Transformer<T>;
+
+export { Transformer, compose, createTransformer, filter, flatMap, map, omit, pick, rename };

package/dist/transform.d.ts

@@ -0,0 +1,38 @@
+import { D as DataRecord, T as Transformer } from './types-EpLTQcN2.js';
+
+/**
+ * Create a custom transformer from a function.
+ */
+declare function createTransformer<TInput extends DataRecord = DataRecord, TOutput extends DataRecord = DataRecord>(name: string, fn: (record: TInput) => TOutput | TOutput[] | Promise<TOutput | TOutput[]>): Transformer<TInput, TOutput>;
+/**
+ * Transform each record using a mapping function.
+ */
+declare function map<TInput extends DataRecord = DataRecord, TOutput extends DataRecord = DataRecord>(name: string, fn: (record: TInput) => TOutput | Promise<TOutput>): Transformer<TInput, TOutput>;
+/**
+ * Filter records based on a predicate. Records that don't match
+ * are dropped (returned as empty array).
+ */
+declare function filter<T extends DataRecord = DataRecord>(name: string, predicate: (record: T) => boolean | Promise<boolean>): Transformer<T, T>;
+/**
+ * Transform each record into zero or more records.
+ */
+declare function flatMap<TInput extends DataRecord = DataRecord, TOutput extends DataRecord = DataRecord>(name: string, fn: (record: TInput) => TOutput[] | Promise<TOutput[]>): Transformer<TInput, TOutput>;
+/**
+ * Pick specific keys from each record.
+ */
+declare function pick<T extends DataRecord, K extends keyof T & string>(name: string, keys: K[]): Transformer<T, Pick<T, K> & DataRecord>;
+/**
+ * Omit specific keys from each record.
+ */
+declare function omit<T extends DataRecord, K extends keyof T & string>(name: string, keys: K[]): Transformer<T, Omit<T, K> & DataRecord>;
+/**
+ * Rename keys in each record.
+ */
+declare function rename<T extends DataRecord>(name: string, mapping: Record<string, string>): Transformer<T>;
+/**
+ * Compose multiple transformers into a single transformer that
+ * applies them in sequence.
+ */
+declare function compose<T extends DataRecord>(name: string, transformers: Transformer[]): Transformer<T>;
+
+export { Transformer, compose, createTransformer, filter, flatMap, map, omit, pick, rename };

package/dist/transform.js

@@ -0,0 +1,3 @@
+export { compose, createTransformer, filter, flatMap, map, omit, pick, rename } from './chunk-NVPI7OV3.js';
+//# sourceMappingURL=transform.js.map
+//# sourceMappingURL=transform.js.map

package/dist/transform.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"names":[],"mappings":"","file":"transform.js"}

package/dist/types-EpLTQcN2.d.cts

@@ -0,0 +1,54 @@
+/** A single record flowing through a pipeline. */
+type DataRecord = Record<string, unknown>;
+/** Extracts data from a source, yielding records lazily. */
+interface Extractor<T extends DataRecord = DataRecord> {
+    readonly name: string;
+    extract(): AsyncIterable<T>;
+}
+/** Transforms a record into zero or more output records. */
+interface Transformer<TInput extends DataRecord = DataRecord, TOutput extends DataRecord = DataRecord> {
+    readonly name: string;
+    transform(record: TInput): TOutput | TOutput[] | Promise<TOutput | TOutput[]>;
+}
+/** Loads a batch of records into a destination. */
+interface Loader<T extends DataRecord = DataRecord> {
+    readonly name: string;
+    load(records: T[]): Promise<LoadResult>;
+}
+interface LoadResult {
+    readonly recordsLoaded: number;
+    readonly errors: LoadError[];
+}
+interface LoadError {
+    readonly record: DataRecord;
+    readonly error: Error;
+}
+/** Events emitted during pipeline execution. */
+interface PipelineEvent {
+    readonly type: "pipeline:start" | "pipeline:end" | "step:start" | "step:end" | "record:extracted" | "record:transformed" | "record:loaded" | "error";
+    readonly timestamp: Date;
+    readonly stepName?: string;
+    readonly data?: unknown;
+}
+type PipelineEventListener = (event: PipelineEvent) => void;
+interface PipelineOptions {
+    /** Records per load batch. Defaults to 1000. */
+    readonly batchSize?: number;
+    /** Keep processing after errors instead of throwing. */
+    readonly continueOnError?: boolean;
+    /** Cancel the pipeline. */
+    readonly signal?: AbortSignal;
+}
+interface PipelineResult {
+    readonly recordsProcessed: number;
+    readonly recordsLoaded: number;
+    readonly errors: PipelineStepError[];
+    readonly duration: number;
+}
+interface PipelineStepError {
+    readonly stepName: string;
+    readonly error: Error;
+    readonly record?: DataRecord;
+}
+
+export type { DataRecord as D, Extractor as E, LoadError as L, PipelineEvent as P, Transformer as T, LoadResult as a, Loader as b, PipelineEventListener as c, PipelineOptions as d, PipelineResult as e, PipelineStepError as f };