khotan-data 0.0.1

package/dist/chunk-FRRSW3TN.cjs

@@ -0,0 +1,105 @@
+'use strict';
+
+// src/drizzle-extract.ts
+function fromQuery(name, queryFn) {
+  return {
+    name,
+    async *extract() {
+      const rows = await queryFn();
+      for (const row of rows) {
+        yield row;
+      }
+    }
+  };
+}
+function fromQueryCursor(name, generatorFn) {
+  return {
+    name,
+    extract: generatorFn
+  };
+}
+function fromQueryPaginated(name, opts) {
+  const pageSize = opts.pageSize ?? 1e3;
+  return {
+    name,
+    async *extract() {
+      let offset = 0;
+      for (; ; ) {
+        const rows = await opts.query(pageSize, offset);
+        if (rows.length === 0) break;
+        for (const row of rows) {
+          yield row;
+        }
+        if (rows.length < pageSize) break;
+        offset += pageSize;
+      }
+    }
+  };
+}
+
+// src/drizzle-load.ts
+var PG_MAX_PARAMETERS = 65535;
+function resolveMaxRows(options) {
+  if (options?.maxRowsPerStatement) return options.maxRowsPerStatement;
+  if (options?.columnsPerRow) {
+    return Math.floor(PG_MAX_PARAMETERS / options.columnsPerRow);
+  }
+  return 1e3;
+}
+function toDrizzle(name, writeFn, options) {
+  const maxRows = resolveMaxRows(options);
+  return {
+    name,
+    async load(records) {
+      const errors = [];
+      let loaded = 0;
+      for (let i = 0; i < records.length; i += maxRows) {
+        const chunk = records.slice(i, i + maxRows);
+        try {
+          await writeFn(chunk);
+          loaded += chunk.length;
+        } catch (err) {
+          const error = err instanceof Error ? err : new Error(String(err));
+          for (const record of chunk) {
+            errors.push({ record, error });
+          }
+        }
+      }
+      return { recordsLoaded: loaded, errors };
+    }
+  };
+}
+function toDrizzleTx(name, db, writeFn, options) {
+  const maxRows = resolveMaxRows(options);
+  return {
+    name,
+    async load(records) {
+      const errors = [];
+      let loaded = 0;
+      try {
+        await db.transaction(async (tx) => {
+          for (let i = 0; i < records.length; i += maxRows) {
+            const chunk = records.slice(i, i + maxRows);
+            await writeFn(tx, chunk);
+            loaded += chunk.length;
+          }
+        });
+      } catch (err) {
+        const error = err instanceof Error ? err : new Error(String(err));
+        for (const record of records) {
+          errors.push({ record, error });
+        }
+        loaded = 0;
+      }
+      return { recordsLoaded: loaded, errors };
+    }
+  };
+}
+
+exports.fromQuery = fromQuery;
+exports.fromQueryCursor = fromQueryCursor;
+exports.fromQueryPaginated = fromQueryPaginated;
+exports.toDrizzle = toDrizzle;
+exports.toDrizzleTx = toDrizzleTx;
+//# sourceMappingURL=chunk-FRRSW3TN.cjs.map
+//# sourceMappingURL=chunk-FRRSW3TN.cjs.map

package/dist/chunk-FRRSW3TN.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/drizzle-extract.ts","../src/drizzle-load.ts"],"names":[],"mappings":";;;AAoBO,SAAS,SAAA,CACd,MACA,OAAA,EACc;AACd,EAAA,OAAO;AAAA,IACL,IAAA;AAAA,IACA,OAAO,OAAA,GAAU;AACf,MAAA,MAAM,IAAA,GAAO,MAAM,OAAA,EAAQ;AAC3B,MAAA,KAAA,MAAW,OAAO,IAAA,EAAM;AACtB,QAAA,MAAM,GAAA;AAAA,MACR;AAAA,IACF;AAAA,GACF;AACF;AAqBO,SAAS,eAAA,CACd,MACA,WAAA,EACc;AACd,EAAA,OAAO;AAAA,IACL,IAAA;AAAA,IACA,OAAA,EAAS;AAAA,GACX;AACF;AAcO,SAAS,kBAAA,CACd,MACA,IAAA,EAIc;AACd,EAAA,MAAM,QAAA,GAAW,KAAK,QAAA,IAAY,GAAA;AAClC,EAAA,OAAO;AAAA,IACL,IAAA;AAAA,IACA,OAAO,OAAA,GAAU;AACf,MAAA,IAAI,MAAA,GAAS,CAAA;AACb,MAAA,WAAS;AACP,QAAA,MAAM,IAAA,GAAO,MAAM,IAAA,CAAK,KAAA,CAAM,UAAU,MAAM,CAAA;AAC9C,QAAA,IAAI,IAAA,CAAK,WAAW,CAAA,EAAG;AACvB,QAAA,KAAA,MAAW,OAAO,IAAA,EAAM;AACtB,UAAA,MAAM,GAAA;AAAA,QACR;AACA,QAAA,IAAI,IAAA,CAAK,SAAS,QAAA,EAAU;AAC5B,QAAA,MAAA,IAAU,QAAA;AAAA,MACZ;AAAA,IACF;AAAA,GACF;AACF;;;ACjGA,IAAM,iBAAA,GAAoB,KAAA;AAiB1B,SAAS,eAAe,OAAA,EAAoC;AAC1D,EAAA,IAAI,OAAA,EAAS,mBAAA,EAAqB,OAAO,OAAA,CAAQ,mBAAA;AACjD,EAAA,IAAI,SAAS,aAAA,EAAe;AAC1B,IAAA,OAAO,IAAA,CAAK,KAAA,CAAM,iBAAA,GAAoB,OAAA,CAAQ,aAAa,CAAA;AAAA,EAC7D;AACA,EAAA,OAAO,GAAA;AACT;AA+BO,SAAS,SAAA,CACd,IAAA,EACA,OAAA,EACA,OAAA,EACW;AACX,EAAA,MAAM,OAAA,GAAU,eAAe,OAAO,CAAA;AAEtC,EAAA,OAAO;AAAA,IACL,IAAA;AAAA,IACA,MAAM,KAAK,OAAA,EAAmC;AAC5C,MAAA,MAAM,SAA+B,EAAC;AACtC,MAAA,IAAI,MAAA,GAAS,CAAA;AAEb,MAAA,KAAA,IAAS,IAAI,CAAA,EAAG,CAAA,GAAI,OAAA,CAAQ,MAAA,EAAQ,KAAK,OAAA,EAAS;AAChD,QAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,KAAA,CAAM,CAAA,EAAG,IAAI,OAAO,CAAA;AAC1C,QAAA,IAAI;AACF,UAAA,MAAM,QAAQ,KAAK,CAAA;AACnB,UAAA,MAAA,IAAU,KAAA,CAAM,MAAA;AAAA,QAClB,SAAS,GAAA,EAAK;AACZ,UAAA,MAAM,KAAA,GAAQ,eAAe,KAAA,GAAQ,GAAA,GAAM,IAAI,KAAA,CAAM,MAAA,CAAO,GAAG,CAAC,CAAA;AAChE,UAAA,KAAA,MAAW,UAAU,KAAA,EAAO;AAC1B,YAAA,MAAA,CAAO,IAAA,CAAK,EAAE,MAAA,EAAQ,KAAA,EAAO,CAAA;AAAA,UAC/B;AAAA,QACF;AAAA,MACF;AAEA,MAAA,OAAO,EAAE,aAAA,EAAe,MAAA,EAAQ,MAAA,EAAO;AAAA,IACzC;AAAA,GACF;AACF;AAeO,SAAS,WAAA,CACd,IAAA,EACA,EAAA,EACA,OAAA,EACA,OAAA,EACW;AACX,EAAA,MAAM,OAAA,GAAU,eAAe,OAAO,CAAA;AAEtC,EAAA,OAAO;AAAA,IACL,IAAA;AAAA,IACA,MAAM,KAAK,OAAA,EAAmC;AAC5C,MAAA,MAAM,SAA+B,EAAC;AACtC,MAAA,IAAI,MAAA,GAAS,CAAA;AAEb,MAAA,IAAI;AACF,QAAA,MAAM,EAAA,CAAG,WAAA,CAAY,OAAO,EAAA,KAAO;AACjC,UAAA,KAAA,IAAS,IAAI,CAAA,EAAG,CAAA,GAAI,OAAA,CAAQ,MAAA,EAAQ,KAAK,OAAA,EAAS;AAChD,YAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,KAAA,CAAM,CAAA,EAAG,IAAI,OAAO,CAAA;AAC1C,YAAA,MAAM,OAAA,CAAQ,IAAI,KAAK,CAAA;AACvB,YAAA,MAAA,IAAU,KAAA,CAAM,MAAA;AAAA,UAClB;AAAA,QACF,CAAC,CAAA;AAAA,MACH,SAAS,GAAA,EAAK;AACZ,QAAA,MAAM,KAAA,GAAQ,eAAe,KAAA,GAAQ,GAAA,GAAM,IAAI,KAAA,CAAM,MAAA,CAAO,GAAG,CAAC,CAAA;AAChE,QAAA,KAAA,MAAW,UAAU,OAAA,EAAS;AAC5B,UAAA,MAAA,CAAO,IAAA,CAAK,EAAE,MAAA,EAAQ,KAAA,EAAO,CAAA;AAAA,QAC/B;AACA,QAAA,MAAA,GAAS,CAAA;AAAA,MACX;AAEA,MAAA,OAAO,EAAE,aAAA,EAAe,MAAA,EAAQ,MAAA,EAAO;AAAA,IACzC;AAAA,GACF;AACF","file":"chunk-FRRSW3TN.cjs","sourcesContent":["import type { DataRecord, Extractor } from \"./types.js\";\n\n/**\n * Create an extractor from any Drizzle select query.\n *\n * Pass a function that returns the query — this makes the extractor\n * re-runnable and avoids consuming a one-shot promise.\n *\n * @example\n * ```ts\n * import { fromQuery } from \"khotan-data/drizzle\";\n * import { db } from \"@/db\";\n * import { users } from \"@/db/schema\";\n * import { eq } from \"drizzle-orm\";\n *\n * const extractor = fromQuery(\"active-users\", () =>\n *   db.select().from(users).where(eq(users.active, true))\n * );\n * ```\n */\nexport function fromQuery<T extends DataRecord>(\n  name: string,\n  queryFn: () => PromiseLike<T[]>,\n): Extractor<T> {\n  return {\n    name,\n    async *extract() {\n      const rows = await queryFn();\n      for (const row of rows) {\n        yield row;\n      }\n    },\n  };\n}\n\n/**\n * Create an extractor from a Drizzle query that streams results in\n * chunks. Use this for large tables where materializing all rows\n * at once is too expensive.\n *\n * @example\n * ```ts\n * const extractor = fromQueryCursor(\"all-events\", async function* () {\n *   let offset = 0;\n *   const limit = 5000;\n *   while (true) {\n *     const batch = await db.select().from(events).limit(limit).offset(offset);\n *     if (batch.length === 0) break;\n *     yield* batch;\n *     offset += limit;\n *   }\n * });\n * ```\n */\nexport function fromQueryCursor<T extends DataRecord>(\n  name: string,\n  generatorFn: () => AsyncIterable<T>,\n): Extractor<T> {\n  return {\n    name,\n    extract: generatorFn,\n  };\n}\n\n/**\n * Create an extractor from a paginated Drizzle query. Automatically\n * handles offset-based pagination so you don't have to write the loop.\n *\n * @example\n * ```ts\n * const extractor = fromQueryPaginated(\"all-users\", {\n *   pageSize: 2000,\n *   query: (limit, offset) => db.select().from(users).limit(limit).offset(offset),\n * });\n * ```\n */\nexport function fromQueryPaginated<T extends DataRecord>(\n  name: string,\n  opts: {\n    query: (limit: number, offset: number) => PromiseLike<T[]>;\n    pageSize?: number;\n  },\n): Extractor<T> {\n  const pageSize = opts.pageSize ?? 1000;\n  return {\n    name,\n    async *extract() {\n      let offset = 0;\n      for (;;) {\n        const rows = await opts.query(pageSize, offset);\n        if (rows.length === 0) break;\n        for (const row of rows) {\n          yield row;\n        }\n        if (rows.length < pageSize) break;\n        offset += pageSize;\n      }\n    },\n  };\n}\n","import type { DataRecord, Loader, LoadResult } from \"./types.js\";\n\nconst PG_MAX_PARAMETERS = 65535;\n\ninterface ToDrizzleOptions {\n  /**\n   * Max records per INSERT statement. When a batch exceeds this,\n   * it's automatically split into sub-batches to stay within Postgres\n   * parameter limits. Defaults to auto-calculated from columnsPerRow.\n   */\n  maxRowsPerStatement?: number;\n  /**\n   * Number of columns per row. Used to auto-calculate maxRowsPerStatement\n   * to stay under Postgres' 65535 parameter limit.\n   * If not provided, falls back to maxRowsPerStatement or 1000.\n   */\n  columnsPerRow?: number;\n}\n\nfunction resolveMaxRows(options?: ToDrizzleOptions): number {\n  if (options?.maxRowsPerStatement) return options.maxRowsPerStatement;\n  if (options?.columnsPerRow) {\n    return Math.floor(PG_MAX_PARAMETERS / options.columnsPerRow);\n  }\n  return 1000;\n}\n\n/**\n * Create a loader that writes records using a Drizzle insert/upsert.\n *\n * You provide the write function — this keeps the loader decoupled from\n * specific Drizzle driver types while giving you full control over\n * insert/upsert/conflict behavior.\n *\n * Automatically sub-batches to stay within Postgres' 65535 parameter limit.\n *\n * @example\n * ```ts\n * import { toDrizzle } from \"khotan-data/drizzle\";\n * import { db } from \"@/db\";\n * import { processedUsers } from \"@/db/schema\";\n *\n * // Simple insert\n * const loader = toDrizzle(\"insert-users\", (rows) =>\n *   db.insert(processedUsers).values(rows)\n * );\n *\n * // Upsert\n * const loader = toDrizzle(\"upsert-users\", (rows) =>\n *   db.insert(processedUsers).values(rows).onConflictDoUpdate({\n *     target: processedUsers.id,\n *     set: { name: sql`excluded.name`, updatedAt: new Date() },\n *   })\n * );\n * ```\n */\nexport function toDrizzle<T extends DataRecord>(\n  name: string,\n  writeFn: (records: T[]) => PromiseLike<unknown>,\n  options?: ToDrizzleOptions,\n): Loader<T> {\n  const maxRows = resolveMaxRows(options);\n\n  return {\n    name,\n    async load(records: T[]): Promise<LoadResult> {\n      const errors: LoadResult[\"errors\"] = [];\n      let loaded = 0;\n\n      for (let i = 0; i < records.length; i += maxRows) {\n        const chunk = records.slice(i, i + maxRows);\n        try {\n          await writeFn(chunk);\n          loaded += chunk.length;\n        } catch (err) {\n          const error = err instanceof Error ? err : new Error(String(err));\n          for (const record of chunk) {\n            errors.push({ record, error });\n          }\n        }\n      }\n\n      return { recordsLoaded: loaded, errors };\n    },\n  };\n}\n\n/**\n * Create a loader that writes records inside a Drizzle transaction.\n *\n * All sub-batches for a single load call are wrapped in a single\n * transaction — if any batch fails, the entire load is rolled back.\n *\n * @example\n * ```ts\n * const loader = toDrizzleTx(\"tx-insert\", db, (tx, rows) =>\n *   tx.insert(processedUsers).values(rows)\n * );\n * ```\n */\nexport function toDrizzleTx<T extends DataRecord>(\n  name: string,\n  db: { transaction: <R>(fn: (tx: never) => Promise<R>) => Promise<R> },\n  writeFn: (tx: never, records: T[]) => PromiseLike<unknown>,\n  options?: ToDrizzleOptions,\n): Loader<T> {\n  const maxRows = resolveMaxRows(options);\n\n  return {\n    name,\n    async load(records: T[]): Promise<LoadResult> {\n      const errors: LoadResult[\"errors\"] = [];\n      let loaded = 0;\n\n      try {\n        await db.transaction(async (tx) => {\n          for (let i = 0; i < records.length; i += maxRows) {\n            const chunk = records.slice(i, i + maxRows);\n            await writeFn(tx, chunk);\n            loaded += chunk.length;\n          }\n        });\n      } catch (err) {\n        const error = err instanceof Error ? err : new Error(String(err));\n        for (const record of records) {\n          errors.push({ record, error });\n        }\n        loaded = 0;\n      }\n\n      return { recordsLoaded: loaded, errors };\n    },\n  };\n}\n"]}

package/dist/chunk-NVPI7OV3.js

@@ -0,0 +1,71 @@
+// src/transformers.ts
+function createTransformer(name, fn) {
+  return { name, transform: fn };
+}
+function map(name, fn) {
+  return createTransformer(name, fn);
+}
+function filter(name, predicate) {
+  return createTransformer(name, async (record) => {
+    const keep = await predicate(record);
+    return keep ? record : [];
+  });
+}
+function flatMap(name, fn) {
+  return createTransformer(name, fn);
+}
+function pick(name, keys) {
+  return createTransformer(name, (record) => {
+    const result = {};
+    for (const key of keys) {
+      if (key in record) {
+        result[key] = record[key];
+      }
+    }
+    return result;
+  });
+}
+function omit(name, keys) {
+  const keySet = new Set(keys);
+  return createTransformer(name, (record) => {
+    const result = {};
+    for (const [key, value] of Object.entries(record)) {
+      if (!keySet.has(key)) {
+        result[key] = value;
+      }
+    }
+    return result;
+  });
+}
+function rename(name, mapping) {
+  return createTransformer(name, (record) => {
+    const result = {};
+    for (const [key, value] of Object.entries(record)) {
+      const newKey = mapping[key] ?? key;
+      result[newKey] = value;
+    }
+    return result;
+  });
+}
+function compose(name, transformers) {
+  return createTransformer(name, async (record) => {
+    let records = [record];
+    for (const transformer of transformers) {
+      const nextRecords = [];
+      for (const r of records) {
+        const result = await transformer.transform(r);
+        if (Array.isArray(result)) {
+          nextRecords.push(...result);
+        } else {
+          nextRecords.push(result);
+        }
+      }
+      records = nextRecords;
+    }
+    return records;
+  });
+}
+
+export { compose, createTransformer, filter, flatMap, map, omit, pick, rename };
+//# sourceMappingURL=chunk-NVPI7OV3.js.map
+//# sourceMappingURL=chunk-NVPI7OV3.js.map

package/dist/chunk-NVPI7OV3.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/transformers.ts"],"names":[],"mappings":";AAKO,SAAS,iBAAA,CAId,MACA,EAAA,EAC8B;AAC9B,EAAA,OAAO,EAAE,IAAA,EAAM,SAAA,EAAW,EAAA,EAAG;AAC/B;AAKO,SAAS,GAAA,CAId,MACA,EAAA,EAC8B;AAC9B,EAAA,OAAO,iBAAA,CAAmC,MAAM,EAAE,CAAA;AACpD;AAMO,SAAS,MAAA,CACd,MACA,SAAA,EACmB;AACnB,EAAA,OAAO,iBAAA,CAAkB,IAAA,EAAM,OAAO,MAAA,KAAc;AAClD,IAAA,MAAM,IAAA,GAAO,MAAM,SAAA,CAAU,MAAM,CAAA;AACnC,IAAA,OAAO,IAAA,GAAO,SAAS,EAAC;AAAA,EAC1B,CAAC,CAAA;AACH;AAKO,SAAS,OAAA,CAId,MACA,EAAA,EAC8B;AAC9B,EAAA,OAAO,iBAAA,CAAmC,MAAM,EAAE,CAAA;AACpD;AAKO,SAAS,IAAA,CACd,MACA,IAAA,EACyC;AACzC,EAAA,OAAO,iBAAA,CAA8C,IAAA,EAAM,CAAC,MAAA,KAAc;AACxE,IAAA,MAAM,SAAS,EAAC;AAChB,IAAA,KAAA,MAAW,OAAO,IAAA,EAAM;AACtB,MAAA,IAAI,OAAO,MAAA,EAAQ;AACjB,QAAC,MAAA,CAAmC,GAAG,CAAA,GAAI,MAAA,CAAO,GAAG,CAAA;AAAA,MACvD;AAAA,IACF;AACA,IAAA,OAAO,MAAA;AAAA,EACT,CAAC,CAAA;AACH;AAKO,SAAS,IAAA,CACd,MACA,IAAA,EACyC;AACzC,EAAA,MAAM,MAAA,GAAS,IAAI,GAAA,CAAY,IAAI,CAAA;AACnC,EAAA,OAAO,iBAAA,CAA8C,IAAA,EAAM,CAAC,MAAA,KAAc;AACxE,IAAA,MAAM,SAAS,EAAC;AAChB,IAAA,KAAA,MAAW,CAAC,GAAA,EAAK,KAAK,KAAK,MAAA,CAAO,OAAA,CAAQ,MAAM,CAAA,EAAG;AACjD,MAAA,IAAI,CAAC,MAAA,CAAO,GAAA,CAAI,GAAG,CAAA,EAAG;AACpB,QAAC,MAAA,CAAmC,GAAG,CAAA,GAAI,KAAA;AAAA,MAC7C;AAAA,IACF;AACA,IAAA,OAAO,MAAA;AAAA,EACT,CAAC,CAAA;AACH;AAKO,SAAS,MAAA,CACd,MACA,OAAA,EACgB;AAChB,EAAA,OAAO,iBAAA,CAAqB,IAAA,EAAM,CAAC,MAAA,KAAc;AAC/C,IAAA,MAAM,SAAqB,EAAC;AAC5B,IAAA,KAAA,MAAW,CAAC,GAAA,EAAK,KAAK,KAAK,MAAA,CAAO,OAAA,CAAQ,MAAM,CAAA,EAAG;AACjD,MAAA,MAAM,MAAA,GAAS,OAAA,CAAQ,GAAG,CAAA,IAAK,GAAA;AAC/B,MAAA,MAAA,CAAO,MAAM,CAAA,GAAI,KAAA;AAAA,IACnB;AACA,IAAA,OAAO,MAAA;AAAA,EACT,CAAC,CAAA;AACH;AAMO,SAAS,OAAA,CACd,MACA,YAAA,EACgB;AAChB,EAAA,OAAO,iBAAA,CAAqB,IAAA,EAAM,OAAO,MAAA,KAAc;AACrD,IAAA,IAAI,OAAA,GAAwB,CAAC,MAAM,CAAA;AAEnC,IAAA,KAAA,MAAW,eAAe,YAAA,EAAc;AACtC,MAAA,MAAM,cAA4B,EAAC;AACnC,MAAA,KAAA,MAAW,KAAK,OAAA,EAAS;AACvB,QAAA,MAAM,MAAA,GAAS,MAAM,WAAA,CAAY,SAAA,CAAU,CAAC,CAAA;AAC5C,QAAA,IAAI,KAAA,CAAM,OAAA,CAAQ,MAAM,CAAA,EAAG;AACzB,UAAA,WAAA,CAAY,IAAA,CAAK,GAAG,MAAM,CAAA;AAAA,QAC5B,CAAA,MAAO;AACL,UAAA,WAAA,CAAY,KAAK,MAAM,CAAA;AAAA,QACzB;AAAA,MACF;AACA,MAAA,OAAA,GAAU,WAAA;AAAA,IACZ;AAEA,IAAA,OAAO,OAAA;AAAA,EACT,CAAC,CAAA;AACH","file":"chunk-NVPI7OV3.js","sourcesContent":["import type { DataRecord, Transformer } from \"./types.js\";\n\n/**\n * Create a custom transformer from a function.\n */\nexport function createTransformer<\n  TInput extends DataRecord = DataRecord,\n  TOutput extends DataRecord = DataRecord,\n>(\n  name: string,\n  fn: (record: TInput) => TOutput | TOutput[] | Promise<TOutput | TOutput[]>,\n): Transformer<TInput, TOutput> {\n  return { name, transform: fn };\n}\n\n/**\n * Transform each record using a mapping function.\n */\nexport function map<\n  TInput extends DataRecord = DataRecord,\n  TOutput extends DataRecord = DataRecord,\n>(\n  name: string,\n  fn: (record: TInput) => TOutput | Promise<TOutput>,\n): Transformer<TInput, TOutput> {\n  return createTransformer<TInput, TOutput>(name, fn);\n}\n\n/**\n * Filter records based on a predicate. Records that don't match\n * are dropped (returned as empty array).\n */\nexport function filter<T extends DataRecord = DataRecord>(\n  name: string,\n  predicate: (record: T) => boolean | Promise<boolean>,\n): Transformer<T, T> {\n  return createTransformer(name, async (record: T) => {\n    const keep = await predicate(record);\n    return keep ? record : [];\n  });\n}\n\n/**\n * Transform each record into zero or more records.\n */\nexport function flatMap<\n  TInput extends DataRecord = DataRecord,\n  TOutput extends DataRecord = DataRecord,\n>(\n  name: string,\n  fn: (record: TInput) => TOutput[] | Promise<TOutput[]>,\n): Transformer<TInput, TOutput> {\n  return createTransformer<TInput, TOutput>(name, fn);\n}\n\n/**\n * Pick specific keys from each record.\n */\nexport function pick<T extends DataRecord, K extends keyof T & string>(\n  name: string,\n  keys: K[],\n): Transformer<T, Pick<T, K> & DataRecord> {\n  return createTransformer<T, Pick<T, K> & DataRecord>(name, (record: T) => {\n    const result = {} as Pick<T, K> & DataRecord;\n    for (const key of keys) {\n      if (key in record) {\n        (result as Record<string, unknown>)[key] = record[key];\n      }\n    }\n    return result;\n  });\n}\n\n/**\n * Omit specific keys from each record.\n */\nexport function omit<T extends DataRecord, K extends keyof T & string>(\n  name: string,\n  keys: K[],\n): Transformer<T, Omit<T, K> & DataRecord> {\n  const keySet = new Set<string>(keys);\n  return createTransformer<T, Omit<T, K> & DataRecord>(name, (record: T) => {\n    const result = {} as Omit<T, K> & DataRecord;\n    for (const [key, value] of Object.entries(record)) {\n      if (!keySet.has(key)) {\n        (result as Record<string, unknown>)[key] = value;\n      }\n    }\n    return result;\n  });\n}\n\n/**\n * Rename keys in each record.\n */\nexport function rename<T extends DataRecord>(\n  name: string,\n  mapping: Record<string, string>,\n): Transformer<T> {\n  return createTransformer<T>(name, (record: T) => {\n    const result: DataRecord = {};\n    for (const [key, value] of Object.entries(record)) {\n      const newKey = mapping[key] ?? key;\n      result[newKey] = value;\n    }\n    return result;\n  });\n}\n\n/**\n * Compose multiple transformers into a single transformer that\n * applies them in sequence.\n */\nexport function compose<T extends DataRecord>(\n  name: string,\n  transformers: Transformer[],\n): Transformer<T> {\n  return createTransformer<T>(name, async (record: T) => {\n    let records: DataRecord[] = [record];\n\n    for (const transformer of transformers) {\n      const nextRecords: DataRecord[] = [];\n      for (const r of records) {\n        const result = await transformer.transform(r);\n        if (Array.isArray(result)) {\n          nextRecords.push(...result);\n        } else {\n          nextRecords.push(result);\n        }\n      }\n      records = nextRecords;\n    }\n\n    return records;\n  });\n}\n"]}

package/dist/chunk-TK4HD4XA.js

@@ -0,0 +1,213 @@
+// src/pipeline-builder.ts
+var Pipeline = class _Pipeline {
+  #name;
+  #extractor;
+  #transformers;
+  #loaders;
+  #listeners;
+  constructor(name, extractor, transformers, loaders, listeners) {
+    this.#name = name;
+    this.#extractor = extractor;
+    this.#transformers = transformers;
+    this.#loaders = loaders;
+    this.#listeners = listeners;
+  }
+  static create(name) {
+    return new _Pipeline(name, null, [], [], []);
+  }
+  get name() {
+    return this.#name;
+  }
+  /**
+   * Set the data source for this pipeline.
+   */
+  extract(extractor) {
+    return new _Pipeline(
+      this.#name,
+      extractor,
+      this.#transformers,
+      this.#loaders,
+      this.#listeners
+    );
+  }
+  /**
+   * Add a transformation step.
+   */
+  transform(transformer) {
+    return new _Pipeline(
+      this.#name,
+      this.#extractor,
+      [...this.#transformers, transformer],
+      this.#loaders,
+      this.#listeners
+    );
+  }
+  /**
+   * Add a load destination.
+   */
+  load(loader) {
+    return new _Pipeline(
+      this.#name,
+      this.#extractor,
+      this.#transformers,
+      [...this.#loaders, loader],
+      this.#listeners
+    );
+  }
+  /**
+   * Subscribe to pipeline events.
+   */
+  on(listener) {
+    return new _Pipeline(
+      this.#name,
+      this.#extractor,
+      this.#transformers,
+      this.#loaders,
+      [...this.#listeners, listener]
+    );
+  }
+  /**
+   * Execute the pipeline.
+   */
+  async run(options = {}) {
+    if (!this.#extractor) {
+      throw new Error(
+        `Pipeline "${this.#name}" has no extractor. Call .extract() before .run().`
+      );
+    }
+    if (this.#loaders.length === 0) {
+      throw new Error(
+        `Pipeline "${this.#name}" has no loaders. Call .load() before .run().`
+      );
+    }
+    const { batchSize = 1e3, continueOnError = false, signal } = options;
+    const startTime = performance.now();
+    const errors = [];
+    let recordsProcessed = 0;
+    let recordsLoaded = 0;
+    this.#emit({
+      type: "pipeline:start",
+      timestamp: /* @__PURE__ */ new Date(),
+      data: { name: this.#name }
+    });
+    let batch = [];
+    const flushBatch = async () => {
+      if (batch.length === 0) return;
+      for (const loader of this.#loaders) {
+        try {
+          this.#emit({
+            type: "step:start",
+            timestamp: /* @__PURE__ */ new Date(),
+            stepName: loader.name
+          });
+          const result = await loader.load(batch);
+          recordsLoaded += result.recordsLoaded;
+          for (const err of result.errors) {
+            errors.push({
+              stepName: loader.name,
+              error: err.error,
+              record: err.record
+            });
+          }
+          this.#emit({
+            type: "step:end",
+            timestamp: /* @__PURE__ */ new Date(),
+            stepName: loader.name,
+            data: result
+          });
+        } catch (err) {
+          const error = err instanceof Error ? err : new Error(String(err));
+          errors.push({ stepName: loader.name, error });
+          if (!continueOnError) {
+            throw error;
+          }
+        }
+      }
+      batch = [];
+    };
+    try {
+      for await (const raw of this.#extractor.extract()) {
+        if (signal?.aborted) {
+          break;
+        }
+        this.#emit({
+          type: "record:extracted",
+          timestamp: /* @__PURE__ */ new Date(),
+          stepName: this.#extractor.name,
+          data: raw
+        });
+        let records = [raw];
+        for (const transformer of this.#transformers) {
+          const nextRecords = [];
+          for (const record of records) {
+            try {
+              this.#emit({
+                type: "step:start",
+                timestamp: /* @__PURE__ */ new Date(),
+                stepName: transformer.name
+              });
+              const result = await transformer.transform(record);
+              const transformed = Array.isArray(result) ? result : [result];
+              nextRecords.push(...transformed);
+              this.#emit({
+                type: "record:transformed",
+                timestamp: /* @__PURE__ */ new Date(),
+                stepName: transformer.name,
+                data: transformed
+              });
+            } catch (err) {
+              const error = err instanceof Error ? err : new Error(String(err));
+              errors.push({
+                stepName: transformer.name,
+                error,
+                record
+              });
+              this.#emit({
+                type: "error",
+                timestamp: /* @__PURE__ */ new Date(),
+                stepName: transformer.name,
+                data: error
+              });
+              if (!continueOnError) {
+                throw error;
+              }
+            }
+          }
+          records = nextRecords;
+        }
+        batch.push(...records);
+        recordsProcessed += records.length;
+        if (batch.length >= batchSize) {
+          await flushBatch();
+        }
+      }
+      await flushBatch();
+    } catch {
+      if (!continueOnError) {
+        const duration2 = performance.now() - startTime;
+        this.#emit({
+          type: "pipeline:end",
+          timestamp: /* @__PURE__ */ new Date(),
+          data: { recordsProcessed, recordsLoaded, errors, duration: duration2 }
+        });
+        return { recordsProcessed, recordsLoaded, errors, duration: duration2 };
+      }
+    }
+    const duration = performance.now() - startTime;
+    this.#emit({
+      type: "pipeline:end",
+      timestamp: /* @__PURE__ */ new Date(),
+      data: { recordsProcessed, recordsLoaded, errors, duration }
+    });
+    return { recordsProcessed, recordsLoaded, errors, duration };
+  }
+  #emit(event) {
+    for (const listener of this.#listeners) {
+      listener(event);
+    }
+  }
+};
+
+export { Pipeline };
+//# sourceMappingURL=chunk-TK4HD4XA.js.map
+//# sourceMappingURL=chunk-TK4HD4XA.js.map

package/dist/chunk-TK4HD4XA.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/pipeline-builder.ts"],"names":["duration"],"mappings":";AAuBO,IAAM,QAAA,GAAN,MAAM,SAAA,CAAmD;AAAA,EACrD,KAAA;AAAA,EACA,UAAA;AAAA,EACA,aAAA;AAAA,EACA,QAAA;AAAA,EACA,UAAA;AAAA,EAED,WAAA,CACN,IAAA,EACA,SAAA,EACA,YAAA,EACA,SACA,SAAA,EACA;AACA,IAAA,IAAA,CAAK,KAAA,GAAQ,IAAA;AACb,IAAA,IAAA,CAAK,UAAA,GAAa,SAAA;AAClB,IAAA,IAAA,CAAK,aAAA,GAAgB,YAAA;AACrB,IAAA,IAAA,CAAK,QAAA,GAAW,OAAA;AAChB,IAAA,IAAA,CAAK,UAAA,GAAa,SAAA;AAAA,EACpB;AAAA,EAEA,OAAO,OAAO,IAAA,EAAwB;AACpC,IAAA,OAAO,IAAI,UAAS,IAAA,EAAM,IAAA,EAAM,EAAC,EAAG,EAAC,EAAG,EAAE,CAAA;AAAA,EAC5C;AAAA,EAEA,IAAI,IAAA,GAAe;AACjB,IAAA,OAAO,IAAA,CAAK,KAAA;AAAA,EACd;AAAA;AAAA;AAAA;AAAA,EAKA,QAA8B,SAAA,EAAsC;AAClE,IAAA,OAAO,IAAI,SAAA;AAAA,MACT,IAAA,CAAK,KAAA;AAAA,MACL,SAAA;AAAA,MACA,IAAA,CAAK,aAAA;AAAA,MACL,IAAA,CAAK,QAAA;AAAA,MACL,IAAA,CAAK;AAAA,KACP;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,UACE,WAAA,EACmB;AACnB,IAAA,OAAO,IAAI,SAAA;AAAA,MACT,IAAA,CAAK,KAAA;AAAA,MACL,IAAA,CAAK,UAAA;AAAA,MACL,CAAC,GAAG,IAAA,CAAK,aAAA,EAAe,WAAW,CAAA;AAAA,MACnC,IAAA,CAAK,QAAA;AAAA,MACL,IAAA,CAAK;AAAA,KACP;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,KAAK,MAAA,EAA8C;AACjD,IAAA,OAAO,IAAI,SAAA;AAAA,MACT,IAAA,CAAK,KAAA;AAAA,MACL,IAAA,CAAK,UAAA;AAAA,MACL,IAAA,CAAK,aAAA;AAAA,MACL,CAAC,GAAG,IAAA,CAAK,QAAA,EAAU,MAAM,CAAA;AAAA,MACzB,IAAA,CAAK;AAAA,KACP;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,GAAG,QAAA,EAAqD;AACtD,IAAA,OAAO,IAAI,SAAA;AAAA,MACT,IAAA,CAAK,KAAA;AAAA,MACL,IAAA,CAAK,UAAA;AAAA,MACL,IAAA,CAAK,aAAA;AAAA,MACL,IAAA,CAAK,QAAA;AAAA,MACL,CAAC,GAAG,IAAA,CAAK,UAAA,EAAY,QAAQ;AAAA,KAC/B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,GAAA,CAAI,OAAA,GAA2B,EAAC,EAA4B;AAChE,IAAA,IAAI,CAAC,KAAK,UAAA,EAAY;AACpB,MAAA,MAAM,IAAI,KAAA;AAAA,QACR,CAAA,UAAA,EAAa,KAAK,KAAK,CAAA,kDAAA;AAAA,OACzB;AAAA,IACF;AAEA,IAAA,IAAI,IAAA,CAAK,QAAA,CAAS,MAAA,KAAW,CAAA,EAAG;AAC9B,MAAA,MAAM,IAAI,KAAA;AAAA,QACR,CAAA,UAAA,EAAa,KAAK,KAAK,CAAA,6CAAA;AAAA,OACzB;AAAA,IACF;AAEA,IAAA,MAAM,EAAE,SAAA,GAAY,GAAA,EAAM,eAAA,GAAkB,KAAA,EAAO,QAAO,GAAI,OAAA;AAE9D,IAAA,MAAM,SAAA,GAAY,YAAY,GAAA,EAAI;AAClC,IAAA,MAAM,SAA8B,EAAC;AACrC,IAAA,IAAI,gBAAA,GAAmB,CAAA;AACvB,IAAA,IAAI,aAAA,GAAgB,CAAA;AAEpB,IAAA,IAAA,CAAK,KAAA,CAAM;AAAA,MACT,IAAA,EAAM,gBAAA;AAAA,MACN,SAAA,sBAAe,IAAA,EAAK;AAAA,MACpB,IAAA,EAAM,EAAE,IAAA,EAAM,IAAA,CAAK,KAAA;AAAM,KAC1B,CAAA;AAED,IAAA,IAAI,QAAsB,EAAC;AAE3B,IAAA,MAAM,aAAa,YAA2B;AAC5C,MAAA,IAAI,KAAA,CAAM,WAAW,CAAA,EAAG;AAExB,MAAA,KAAA,MAAW,MAAA,IAAU,KAAK,QAAA,EAAU;AAClC,QAAA,IAAI;AACF,UAAA,IAAA,CAAK,KAAA,CAAM;AAAA,YACT,IAAA,EAAM,YAAA;AAAA,YACN,SAAA,sBAAe,IAAA,EAAK;AAAA,YACpB,UAAU,MAAA,CAAO;AAAA,WAClB,CAAA;AAED,UAAA,MAAM,MAAA,GAAS,MAAM,MAAA,CAAO,IAAA,CAAK,KAAK,CAAA;AACtC,UAAA,aAAA,IAAiB,MAAA,CAAO,aAAA;AAExB,UAAA,KAAA,MAAW,GAAA,IAAO,OAAO,MAAA,EAAQ;AAC/B,YAAA,MAAA,CAAO,IAAA,CAAK;AAAA,cACV,UAAU,MAAA,CAAO,IAAA;AAAA,cACjB,OAAO,GAAA,CAAI,KAAA;AAAA,cACX,QAAQ,GAAA,CAAI;AAAA,aACb,CAAA;AAAA,UACH;AAEA,UAAA,IAAA,CAAK,KAAA,CAAM;AAAA,YACT,IAAA,EAAM,UAAA;AAAA,YACN,SAAA,sBAAe,IAAA,EAAK;AAAA,YACpB,UAAU,MAAA,CAAO,IAAA;AAAA,YACjB,IAAA,EAAM;AAAA,WACP,CAAA;AAAA,QACH,SAAS,GAAA,EAAK;AACZ,UAAA,MAAM,KAAA,GAAQ,eAAe,KAAA,GAAQ,GAAA,GAAM,IAAI,KAAA,CAAM,MAAA,CAAO,GAAG,CAAC,CAAA;AAChE,UAAA,MAAA,CAAO,KAAK,EAAE,QAAA,EAAU,MAAA,CAAO,IAAA,EAAM,OAAO,CAAA;AAE5C,UAAA,IAAI,CAAC,eAAA,EAAiB;AACpB,YAAA,MAAM,KAAA;AAAA,UACR;AAAA,QACF;AAAA,MACF;AAEA,MAAA,KAAA,GAAQ,EAAC;AAAA,IACX,CAAA;AAEA,IAAA,IAAI;AACF,MAAA,WAAA,MAAiB,GAAA,IAAO,IAAA,CAAK,UAAA,CAAW,OAAA,EAAQ,EAAG;AACjD,QAAA,IAAI,QAAQ,OAAA,EAAS;AACnB,UAAA;AAAA,QACF;AAEA,QAAA,IAAA,CAAK,KAAA,CAAM;AAAA,UACT,IAAA,EAAM,kBAAA;AAAA,UACN,SAAA,sBAAe,IAAA,EAAK;AAAA,UACpB,QAAA,EAAU,KAAK,UAAA,CAAW,IAAA;AAAA,UAC1B,IAAA,EAAM;AAAA,SACP,CAAA;AAED,QAAA,IAAI,OAAA,GAAwB,CAAC,GAAG,CAAA;AAEhC,QAAA,KAAA,MAAW,WAAA,IAAe,KAAK,aAAA,EAAe;AAC5C,UAAA,MAAM,cAA4B,EAAC;AAEnC,UAAA,KAAA,MAAW,UAAU,OAAA,EAAS;AAC5B,YAAA,IAAI;AACF,cAAA,IAAA,CAAK,KAAA,CAAM;AAAA,gBACT,IAAA,EAAM,YAAA;AAAA,gBACN,SAAA,sBAAe,IAAA,EAAK;AAAA,gBACpB,UAAU,WAAA,CAAY;AAAA,eACvB,CAAA;AAED,cAAA,MAAM,MAAA,GAAS,MAAM,WAAA,CAAY,SAAA,CAAU,MAAM,CAAA;AACjD,cAAA,MAAM,cAAc,KAAA,CAAM,OAAA,CAAQ,MAAM,CAAA,GAAI,MAAA,GAAS,CAAC,MAAM,CAAA;AAC5D,cAAA,WAAA,CAAY,IAAA,CAAK,GAAG,WAAW,CAAA;AAE/B,cAAA,IAAA,CAAK,KAAA,CAAM;AAAA,gBACT,IAAA,EAAM,oBAAA;AAAA,gBACN,SAAA,sBAAe,IAAA,EAAK;AAAA,gBACpB,UAAU,WAAA,CAAY,IAAA;AAAA,gBACtB,IAAA,EAAM;AAAA,eACP,CAAA;AAAA,YACH,SAAS,GAAA,EAAK;AACZ,cAAA,MAAM,KAAA,GAAQ,eAAe,KAAA,GAAQ,GAAA,GAAM,IAAI,KAAA,CAAM,MAAA,CAAO,GAAG,CAAC,CAAA;AAChE,cAAA,MAAA,CAAO,IAAA,CAAK;AAAA,gBACV,UAAU,WAAA,CAAY,IAAA;AAAA,gBACtB,KAAA;AAAA,gBACA;AAAA,eACD,CAAA;AAED,cAAA,IAAA,CAAK,KAAA,CAAM;AAAA,gBACT,IAAA,EAAM,OAAA;AAAA,gBACN,SAAA,sBAAe,IAAA,EAAK;AAAA,gBACpB,UAAU,WAAA,CAAY,IAAA;AAAA,gBACtB,IAAA,EAAM;AAAA,eACP,CAAA;AAED,cAAA,IAAI,CAAC,eAAA,EAAiB;AACpB,gBAAA,MAAM,KAAA;AAAA,cACR;AAAA,YACF;AAAA,UACF;AAEA,UAAA,OAAA,GAAU,WAAA;AAAA,QACZ;AAEA,QAAA,KAAA,CAAM,IAAA,CAAK,GAAG,OAAO,CAAA;AACrB,QAAA,gBAAA,IAAoB,OAAA,CAAQ,MAAA;AAE5B,QAAA,IAAI,KAAA,CAAM,UAAU,SAAA,EAAW;AAC7B,UAAA,MAAM,UAAA,EAAW;AAAA,QACnB;AAAA,MACF;AAEA,MAAA,MAAM,UAAA,EAAW;AAAA,IACnB,CAAA,CAAA,MAAQ;AACN,MAAA,IAAI,CAAC,eAAA,EAAiB;AACpB,QAAA,MAAMA,SAAAA,GAAW,WAAA,CAAY,GAAA,EAAI,GAAI,SAAA;AACrC,QAAA,IAAA,CAAK,KAAA,CAAM;AAAA,UACT,IAAA,EAAM,cAAA;AAAA,UACN,SAAA,sBAAe,IAAA,EAAK;AAAA,UACpB,MAAM,EAAE,gBAAA,EAAkB,aAAA,EAAe,MAAA,EAAQ,UAAAA,SAAAA;AAAS,SAC3D,CAAA;AACD,QAAA,OAAO,EAAE,gBAAA,EAAkB,aAAA,EAAe,MAAA,EAAQ,UAAAA,SAAAA,EAAS;AAAA,MAC7D;AAAA,IACF;AAEA,IAAA,MAAM,QAAA,GAAW,WAAA,CAAY,GAAA,EAAI,GAAI,SAAA;AAErC,IAAA,IAAA,CAAK,KAAA,CAAM;AAAA,MACT,IAAA,EAAM,cAAA;AAAA,MACN,SAAA,sBAAe,IAAA,EAAK;AAAA,MACpB,IAAA,EAAM,EAAE,gBAAA,EAAkB,aAAA,EAAe,QAAQ,QAAA;AAAS,KAC3D,CAAA;AAED,IAAA,OAAO,EAAE,gBAAA,EAAkB,aAAA,EAAe,MAAA,EAAQ,QAAA,EAAS;AAAA,EAC7D;AAAA,EAEA,MAAM,KAAA,EAKG;AACP,IAAA,KAAA,MAAW,QAAA,IAAY,KAAK,UAAA,EAAY;AACtC,MAAA,QAAA,CAAS,KAA6C,CAAA;AAAA,IACxD;AAAA,EACF;AACF","file":"chunk-TK4HD4XA.js","sourcesContent":["import type {\n  DataRecord,\n  Extractor,\n  Loader,\n  PipelineEventListener,\n  PipelineOptions,\n  PipelineResult,\n  PipelineStepError,\n  Transformer,\n} from \"./types.js\";\n\n/**\n * A composable, type-safe ETL pipeline builder.\n *\n * @example\n * ```ts\n * const result = await Pipeline.create(\"my-pipeline\")\n *   .extract(myExtractor)\n *   .transform(myTransformer)\n *   .load(myLoader)\n *   .run();\n * ```\n */\nexport class Pipeline<TCurrent extends DataRecord = DataRecord> {\n  readonly #name: string;\n  readonly #extractor: Extractor | null;\n  readonly #transformers: Transformer[];\n  readonly #loaders: Loader[];\n  readonly #listeners: PipelineEventListener[];\n\n  private constructor(\n    name: string,\n    extractor: Extractor | null,\n    transformers: Transformer[],\n    loaders: Loader[],\n    listeners: PipelineEventListener[],\n  ) {\n    this.#name = name;\n    this.#extractor = extractor;\n    this.#transformers = transformers;\n    this.#loaders = loaders;\n    this.#listeners = listeners;\n  }\n\n  static create(name: string): Pipeline {\n    return new Pipeline(name, null, [], [], []);\n  }\n\n  get name(): string {\n    return this.#name;\n  }\n\n  /**\n   * Set the data source for this pipeline.\n   */\n  extract<T extends DataRecord>(extractor: Extractor<T>): Pipeline<T> {\n    return new Pipeline(\n      this.#name,\n      extractor,\n      this.#transformers,\n      this.#loaders,\n      this.#listeners,\n    );\n  }\n\n  /**\n   * Add a transformation step.\n   */\n  transform<TOutput extends DataRecord>(\n    transformer: Transformer<TCurrent, TOutput>,\n  ): Pipeline<TOutput> {\n    return new Pipeline(\n      this.#name,\n      this.#extractor,\n      [...this.#transformers, transformer],\n      this.#loaders,\n      this.#listeners,\n    );\n  }\n\n  /**\n   * Add a load destination.\n   */\n  load(loader: Loader<TCurrent>): Pipeline<TCurrent> {\n    return new Pipeline(\n      this.#name,\n      this.#extractor,\n      this.#transformers,\n      [...this.#loaders, loader],\n      this.#listeners,\n    );\n  }\n\n  /**\n   * Subscribe to pipeline events.\n   */\n  on(listener: PipelineEventListener): Pipeline<TCurrent> {\n    return new Pipeline(\n      this.#name,\n      this.#extractor,\n      this.#transformers,\n      this.#loaders,\n      [...this.#listeners, listener],\n    );\n  }\n\n  /**\n   * Execute the pipeline.\n   */\n  async run(options: PipelineOptions = {}): Promise<PipelineResult> {\n    if (!this.#extractor) {\n      throw new Error(\n        `Pipeline \"${this.#name}\" has no extractor. Call .extract() before .run().`,\n      );\n    }\n\n    if (this.#loaders.length === 0) {\n      throw new Error(\n        `Pipeline \"${this.#name}\" has no loaders. Call .load() before .run().`,\n      );\n    }\n\n    const { batchSize = 1000, continueOnError = false, signal } = options;\n\n    const startTime = performance.now();\n    const errors: PipelineStepError[] = [];\n    let recordsProcessed = 0;\n    let recordsLoaded = 0;\n\n    this.#emit({\n      type: \"pipeline:start\",\n      timestamp: new Date(),\n      data: { name: this.#name },\n    });\n\n    let batch: DataRecord[] = [];\n\n    const flushBatch = async (): Promise<void> => {\n      if (batch.length === 0) return;\n\n      for (const loader of this.#loaders) {\n        try {\n          this.#emit({\n            type: \"step:start\",\n            timestamp: new Date(),\n            stepName: loader.name,\n          });\n\n          const result = await loader.load(batch);\n          recordsLoaded += result.recordsLoaded;\n\n          for (const err of result.errors) {\n            errors.push({\n              stepName: loader.name,\n              error: err.error,\n              record: err.record,\n            });\n          }\n\n          this.#emit({\n            type: \"step:end\",\n            timestamp: new Date(),\n            stepName: loader.name,\n            data: result,\n          });\n        } catch (err) {\n          const error = err instanceof Error ? err : new Error(String(err));\n          errors.push({ stepName: loader.name, error });\n\n          if (!continueOnError) {\n            throw error;\n          }\n        }\n      }\n\n      batch = [];\n    };\n\n    try {\n      for await (const raw of this.#extractor.extract()) {\n        if (signal?.aborted) {\n          break;\n        }\n\n        this.#emit({\n          type: \"record:extracted\",\n          timestamp: new Date(),\n          stepName: this.#extractor.name,\n          data: raw,\n        });\n\n        let records: DataRecord[] = [raw];\n\n        for (const transformer of this.#transformers) {\n          const nextRecords: DataRecord[] = [];\n\n          for (const record of records) {\n            try {\n              this.#emit({\n                type: \"step:start\",\n                timestamp: new Date(),\n                stepName: transformer.name,\n              });\n\n              const result = await transformer.transform(record);\n              const transformed = Array.isArray(result) ? result : [result];\n              nextRecords.push(...transformed);\n\n              this.#emit({\n                type: \"record:transformed\",\n                timestamp: new Date(),\n                stepName: transformer.name,\n                data: transformed,\n              });\n            } catch (err) {\n              const error = err instanceof Error ? err : new Error(String(err));\n              errors.push({\n                stepName: transformer.name,\n                error,\n                record,\n              });\n\n              this.#emit({\n                type: \"error\",\n                timestamp: new Date(),\n                stepName: transformer.name,\n                data: error,\n              });\n\n              if (!continueOnError) {\n                throw error;\n              }\n            }\n          }\n\n          records = nextRecords;\n        }\n\n        batch.push(...records);\n        recordsProcessed += records.length;\n\n        if (batch.length >= batchSize) {\n          await flushBatch();\n        }\n      }\n\n      await flushBatch();\n    } catch {\n      if (!continueOnError) {\n        const duration = performance.now() - startTime;\n        this.#emit({\n          type: \"pipeline:end\",\n          timestamp: new Date(),\n          data: { recordsProcessed, recordsLoaded, errors, duration },\n        });\n        return { recordsProcessed, recordsLoaded, errors, duration };\n      }\n    }\n\n    const duration = performance.now() - startTime;\n\n    this.#emit({\n      type: \"pipeline:end\",\n      timestamp: new Date(),\n      data: { recordsProcessed, recordsLoaded, errors, duration },\n    });\n\n    return { recordsProcessed, recordsLoaded, errors, duration };\n  }\n\n  #emit(event: {\n    type: string;\n    timestamp: Date;\n    stepName?: string;\n    data?: unknown;\n  }): void {\n    for (const listener of this.#listeners) {\n      listener(event as Parameters<PipelineEventListener>[0]);\n    }\n  }\n}\n"]}

package/dist/drizzle.cjs

@@ -0,0 +1,28 @@
+'use strict';
+
+var chunkFRRSW3TN_cjs = require('./chunk-FRRSW3TN.cjs');
+
+
+
+Object.defineProperty(exports, "fromQuery", {
+  enumerable: true,
+  get: function () { return chunkFRRSW3TN_cjs.fromQuery; }
+});
+Object.defineProperty(exports, "fromQueryCursor", {
+  enumerable: true,
+  get: function () { return chunkFRRSW3TN_cjs.fromQueryCursor; }
+});
+Object.defineProperty(exports, "fromQueryPaginated", {
+  enumerable: true,
+  get: function () { return chunkFRRSW3TN_cjs.fromQueryPaginated; }
+});
+Object.defineProperty(exports, "toDrizzle", {
+  enumerable: true,
+  get: function () { return chunkFRRSW3TN_cjs.toDrizzle; }
+});
+Object.defineProperty(exports, "toDrizzleTx", {
+  enumerable: true,
+  get: function () { return chunkFRRSW3TN_cjs.toDrizzleTx; }
+});
+//# sourceMappingURL=drizzle.cjs.map
+//# sourceMappingURL=drizzle.cjs.map

package/dist/drizzle.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"names":[],"mappings":"","file":"drizzle.cjs"}

package/dist/drizzle.d.cts

@@ -0,0 +1,120 @@
+import { D as DataRecord, E as Extractor, b as Loader } from './types-EpLTQcN2.cjs';
+
+/**
+ * Create an extractor from any Drizzle select query.
+ *
+ * Pass a function that returns the query — this makes the extractor
+ * re-runnable and avoids consuming a one-shot promise.
+ *
+ * @example
+ * ```ts
+ * import { fromQuery } from "khotan-data/drizzle";
+ * import { db } from "@/db";
+ * import { users } from "@/db/schema";
+ * import { eq } from "drizzle-orm";
+ *
+ * const extractor = fromQuery("active-users", () =>
+ *   db.select().from(users).where(eq(users.active, true))
+ * );
+ * ```
+ */
+declare function fromQuery<T extends DataRecord>(name: string, queryFn: () => PromiseLike<T[]>): Extractor<T>;
+/**
+ * Create an extractor from a Drizzle query that streams results in
+ * chunks. Use this for large tables where materializing all rows
+ * at once is too expensive.
+ *
+ * @example
+ * ```ts
+ * const extractor = fromQueryCursor("all-events", async function* () {
+ *   let offset = 0;
+ *   const limit = 5000;
+ *   while (true) {
+ *     const batch = await db.select().from(events).limit(limit).offset(offset);
+ *     if (batch.length === 0) break;
+ *     yield* batch;
+ *     offset += limit;
+ *   }
+ * });
+ * ```
+ */
+declare function fromQueryCursor<T extends DataRecord>(name: string, generatorFn: () => AsyncIterable<T>): Extractor<T>;
+/**
+ * Create an extractor from a paginated Drizzle query. Automatically
+ * handles offset-based pagination so you don't have to write the loop.
+ *
+ * @example
+ * ```ts
+ * const extractor = fromQueryPaginated("all-users", {
+ *   pageSize: 2000,
+ *   query: (limit, offset) => db.select().from(users).limit(limit).offset(offset),
+ * });
+ * ```
+ */
+declare function fromQueryPaginated<T extends DataRecord>(name: string, opts: {
+    query: (limit: number, offset: number) => PromiseLike<T[]>;
+    pageSize?: number;
+}): Extractor<T>;
+
+interface ToDrizzleOptions {
+    /**
+     * Max records per INSERT statement. When a batch exceeds this,
+     * it's automatically split into sub-batches to stay within Postgres
+     * parameter limits. Defaults to auto-calculated from columnsPerRow.
+     */
+    maxRowsPerStatement?: number;
+    /**
+     * Number of columns per row. Used to auto-calculate maxRowsPerStatement
+     * to stay under Postgres' 65535 parameter limit.
+     * If not provided, falls back to maxRowsPerStatement or 1000.
+     */
+    columnsPerRow?: number;
+}
+/**
+ * Create a loader that writes records using a Drizzle insert/upsert.
+ *
+ * You provide the write function — this keeps the loader decoupled from
+ * specific Drizzle driver types while giving you full control over
+ * insert/upsert/conflict behavior.
+ *
+ * Automatically sub-batches to stay within Postgres' 65535 parameter limit.
+ *
+ * @example
+ * ```ts
+ * import { toDrizzle } from "khotan-data/drizzle";
+ * import { db } from "@/db";
+ * import { processedUsers } from "@/db/schema";
+ *
+ * // Simple insert
+ * const loader = toDrizzle("insert-users", (rows) =>
+ *   db.insert(processedUsers).values(rows)
+ * );
+ *
+ * // Upsert
+ * const loader = toDrizzle("upsert-users", (rows) =>
+ *   db.insert(processedUsers).values(rows).onConflictDoUpdate({
+ *     target: processedUsers.id,
+ *     set: { name: sql`excluded.name`, updatedAt: new Date() },
+ *   })
+ * );
+ * ```
+ */
+declare function toDrizzle<T extends DataRecord>(name: string, writeFn: (records: T[]) => PromiseLike<unknown>, options?: ToDrizzleOptions): Loader<T>;
+/**
+ * Create a loader that writes records inside a Drizzle transaction.
+ *
+ * All sub-batches for a single load call are wrapped in a single
+ * transaction — if any batch fails, the entire load is rolled back.
+ *
+ * @example
+ * ```ts
+ * const loader = toDrizzleTx("tx-insert", db, (tx, rows) =>
+ *   tx.insert(processedUsers).values(rows)
+ * );
+ * ```
+ */
+declare function toDrizzleTx<T extends DataRecord>(name: string, db: {
+    transaction: <R>(fn: (tx: never) => Promise<R>) => Promise<R>;
+}, writeFn: (tx: never, records: T[]) => PromiseLike<unknown>, options?: ToDrizzleOptions): Loader<T>;
+
+export { fromQuery, fromQueryCursor, fromQueryPaginated, toDrizzle, toDrizzleTx };