@nest-batch/core 0.2.0

package/dist/src/io/file-readers.js

@@ -0,0 +1,167 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: Object.getOwnPropertyDescriptor(all, name).get
+    });
+}
+_export(exports, {
+    get CsvFileItemReader () {
+        return CsvFileItemReader;
+    },
+    get JsonlFileItemReader () {
+        return JsonlFileItemReader;
+    },
+    get RestartableFileLineReader () {
+        return RestartableFileLineReader;
+    },
+    get parseDelimitedLine () {
+        return parseDelimitedLine;
+    },
+    get splitLines () {
+        return splitLines;
+    }
+});
+const _promises = require("node:fs/promises");
+const _checkpoint = require("./checkpoint");
+let RestartableFileLineReader = class RestartableFileLineReader {
+    options;
+    lines = [];
+    index = 0;
+    constructor(options){
+        this.options = options;
+    }
+    async open(context) {
+        const raw = await (0, _promises.readFile)(this.options.path, this.options.encoding ?? 'utf8');
+        this.lines = splitLines(raw);
+        this.index = (0, _checkpoint.readCheckpointNumber)(context, this.checkpointKey, 'index', this.options.skipLines ?? 0);
+    }
+    async read() {
+        while(this.index < this.lines.length){
+            const lineIndex = this.index;
+            const line = this.lines[this.index];
+            this.index += 1;
+            if (this.options.skipBlankLines === true && line.trim().length === 0) continue;
+            if (this.options.mapLine !== undefined) {
+                const mapped = await this.options.mapLine(line, lineIndex);
+                if (mapped === null) continue;
+                return mapped;
+            }
+            return line;
+        }
+        return null;
+    }
+    update(context) {
+        return (0, _checkpoint.writeCheckpoint)(context, this.checkpointKey, {
+            index: this.index
+        });
+    }
+    close() {
+        this.lines = [];
+    }
+    get checkpointKey() {
+        return this.options.checkpointKey ?? `file-line:${this.options.path}`;
+    }
+};
+let CsvFileItemReader = class CsvFileItemReader {
+    options;
+    lines = [];
+    headers = [];
+    index = 0;
+    constructor(options){
+        this.options = options;
+    }
+    async open(context) {
+        const raw = await (0, _promises.readFile)(this.options.path, this.options.encoding ?? 'utf8');
+        this.lines = splitLines(raw);
+        const hasHeader = this.options.hasHeader ?? this.options.headers === undefined;
+        this.headers = this.options.headers ?? (this.lines.length > 0 ? parseDelimitedLine(this.lines[0], this.delimiter) : []);
+        const firstDataLine = hasHeader ? 1 : 0;
+        this.index = (0, _checkpoint.readCheckpointNumber)(context, this.checkpointKey, 'index', firstDataLine);
+    }
+    async read() {
+        while(this.index < this.lines.length){
+            const lineIndex = this.index;
+            const line = this.lines[this.index];
+            this.index += 1;
+            if (this.options.skipBlankLines !== false && line.trim().length === 0) continue;
+            const fields = parseDelimitedLine(line, this.delimiter);
+            const row = {};
+            for(let i = 0; i < this.headers.length; i += 1){
+                row[this.headers[i]] = fields[i] ?? '';
+            }
+            if (this.options.mapRow !== undefined) {
+                const mapped = await this.options.mapRow(row, lineIndex);
+                if (mapped === null) continue;
+                return mapped;
+            }
+            return row;
+        }
+        return null;
+    }
+    update(context) {
+        return (0, _checkpoint.writeCheckpoint)(context, this.checkpointKey, {
+            index: this.index
+        });
+    }
+    close() {
+        this.lines = [];
+        this.headers = [];
+    }
+    get checkpointKey() {
+        return this.options.checkpointKey ?? `csv:${this.options.path}`;
+    }
+    get delimiter() {
+        return this.options.delimiter ?? ',';
+    }
+};
+let JsonlFileItemReader = class JsonlFileItemReader extends RestartableFileLineReader {
+    constructor(options){
+        super({
+            ...options,
+            skipBlankLines: true,
+            mapLine: (line)=>JSON.parse(line)
+        });
+    }
+};
+function splitLines(raw) {
+    const text = typeof raw === 'string' ? raw : raw.toString('utf8');
+    const withoutBom = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
+    return withoutBom.replace(/\r\n/g, '\n').replace(/\r/g, '\n').split('\n');
+}
+function parseDelimitedLine(line, delimiter = ',') {
+    const out = [];
+    let field = '';
+    let quoted = false;
+    for(let i = 0; i < line.length; i += 1){
+        const ch = line[i];
+        if (quoted) {
+            if (ch === '"' && line[i + 1] === '"') {
+                field += '"';
+                i += 1;
+            } else if (ch === '"') {
+                quoted = false;
+            } else {
+                field += ch;
+            }
+            continue;
+        }
+        if (ch === '"') {
+            quoted = true;
+            continue;
+        }
+        if (ch === delimiter) {
+            out.push(field);
+            field = '';
+            continue;
+        }
+        field += ch;
+    }
+    out.push(field);
+    return out;
+}
+
+//# sourceMappingURL=file-readers.js.map

package/dist/src/io/file-readers.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/io/file-readers.ts"],"sourcesContent":["import { readFile } from 'node:fs/promises';\n\nimport type { ExecutionContext, ItemReader, ItemStream } from '../core';\n\nimport { readCheckpointNumber, writeCheckpoint } from './checkpoint';\n\nexport interface RestartableFileLineReaderOptions<T = string> {\n  readonly path: string;\n  readonly encoding?: BufferEncoding;\n  readonly checkpointKey?: string;\n  readonly skipLines?: number;\n  readonly skipBlankLines?: boolean;\n  readonly mapLine?: (line: string, lineIndex: number) => T | null | Promise<T | null>;\n}\n\nexport class RestartableFileLineReader<T = string>\n  implements ItemReader<T>, ItemStream\n{\n  private lines: string[] = [];\n  private index = 0;\n\n  constructor(private readonly options: RestartableFileLineReaderOptions<T>) {}\n\n  async open(context: ExecutionContext): Promise<void> {\n    const raw = await readFile(this.options.path, this.options.encoding ?? 'utf8');\n    this.lines = splitLines(raw);\n    this.index = readCheckpointNumber(\n      context,\n      this.checkpointKey,\n      'index',\n      this.options.skipLines ?? 0,\n    );\n  }\n\n  async read(): Promise<T | null> {\n    while (this.index < this.lines.length) {\n      const lineIndex = this.index;\n      const line = this.lines[this.index]!;\n      this.index += 1;\n      if (this.options.skipBlankLines === true && line.trim().length === 0) continue;\n      if (this.options.mapLine !== undefined) {\n        const mapped = await this.options.mapLine(line, lineIndex);\n        if (mapped === null) continue;\n        return mapped;\n      }\n      return line as T;\n    }\n    return null;\n  }\n\n  update(context: ExecutionContext): ExecutionContext {\n    return writeCheckpoint(context, this.checkpointKey, { index: this.index });\n  }\n\n  close(): void {\n    this.lines = [];\n  }\n\n  private get checkpointKey(): string {\n    return this.options.checkpointKey ?? `file-line:${this.options.path}`;\n  }\n}\n\nexport interface CsvFileItemReaderOptions<T extends Record<string, unknown>> {\n  readonly path: string;\n  readonly encoding?: BufferEncoding;\n  readonly checkpointKey?: string;\n  readonly delimiter?: string;\n  readonly headers?: readonly string[];\n  readonly hasHeader?: boolean;\n  readonly skipBlankLines?: boolean;\n  readonly mapRow?: (\n    row: Record<string, string>,\n    lineIndex: number,\n  ) => T | null | Promise<T | null>;\n}\n\nexport class CsvFileItemReader<T extends Record<string, unknown> = Record<string, string>>\n  implements ItemReader<T>, ItemStream\n{\n  private lines: string[] = [];\n  private headers: readonly string[] = [];\n  private index = 0;\n\n  constructor(private readonly options: CsvFileItemReaderOptions<T>) {}\n\n  async open(context: ExecutionContext): Promise<void> {\n    const raw = await readFile(this.options.path, this.options.encoding ?? 'utf8');\n    this.lines = splitLines(raw);\n    const hasHeader = this.options.hasHeader ?? this.options.headers === undefined;\n    this.headers =\n      this.options.headers ??\n      (this.lines.length > 0\n        ? parseDelimitedLine(this.lines[0]!, this.delimiter)\n        : []);\n    const firstDataLine = hasHeader ? 1 : 0;\n    this.index = readCheckpointNumber(context, this.checkpointKey, 'index', firstDataLine);\n  }\n\n  async read(): Promise<T | null> {\n    while (this.index < this.lines.length) {\n      const lineIndex = this.index;\n      const line = this.lines[this.index]!;\n      this.index += 1;\n      if (this.options.skipBlankLines !== false && line.trim().length === 0) continue;\n      const fields = parseDelimitedLine(line, this.delimiter);\n      const row: Record<string, string> = {};\n      for (let i = 0; i < this.headers.length; i += 1) {\n        row[this.headers[i]!] = fields[i] ?? '';\n      }\n      if (this.options.mapRow !== undefined) {\n        const mapped = await this.options.mapRow(row, lineIndex);\n        if (mapped === null) continue;\n        return mapped;\n      }\n      return row as T;\n    }\n    return null;\n  }\n\n  update(context: ExecutionContext): ExecutionContext {\n    return writeCheckpoint(context, this.checkpointKey, { index: this.index });\n  }\n\n  close(): void {\n    this.lines = [];\n    this.headers = [];\n  }\n\n  private get checkpointKey(): string {\n    return this.options.checkpointKey ?? `csv:${this.options.path}`;\n  }\n\n  private get delimiter(): string {\n    return this.options.delimiter ?? ',';\n  }\n}\n\nexport interface JsonlFileItemReaderOptions<T> {\n  readonly path: string;\n  readonly encoding?: BufferEncoding;\n  readonly checkpointKey?: string;\n}\n\nexport class JsonlFileItemReader<T = unknown> extends RestartableFileLineReader<T> {\n  constructor(options: JsonlFileItemReaderOptions<T>) {\n    super({\n      ...options,\n      skipBlankLines: true,\n      mapLine: (line) => JSON.parse(line) as T,\n    });\n  }\n}\n\nexport function splitLines(raw: string | Buffer): string[] {\n  const text = typeof raw === 'string' ? raw : raw.toString('utf8');\n  const withoutBom = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;\n  return withoutBom.replace(/\\r\\n/g, '\\n').replace(/\\r/g, '\\n').split('\\n');\n}\n\nexport function parseDelimitedLine(line: string, delimiter = ','): string[] {\n  const out: string[] = [];\n  let field = '';\n  let quoted = false;\n  for (let i = 0; i < line.length; i += 1) {\n    const ch = line[i]!;\n    if (quoted) {\n      if (ch === '\"' && line[i + 1] === '\"') {\n        field += '\"';\n        i += 1;\n      } else if (ch === '\"') {\n        quoted = false;\n      } else {\n        field += ch;\n      }\n      continue;\n    }\n    if (ch === '\"') {\n      quoted = true;\n      continue;\n    }\n    if (ch === delimiter) {\n      out.push(field);\n      field = '';\n      continue;\n    }\n    field += ch;\n  }\n  out.push(field);\n  return out;\n}\n"],"names":["CsvFileItemReader","JsonlFileItemReader","RestartableFileLineReader","parseDelimitedLine","splitLines","lines","index","options","open","context","raw","readFile","path","encoding","readCheckpointNumber","checkpointKey","skipLines","read","length","lineIndex","line","skipBlankLines","trim","mapLine","undefined","mapped","update","writeCheckpoint","close","headers","hasHeader","delimiter","firstDataLine","fields","row","i","mapRow","JSON","parse","text","toString","withoutBom","charCodeAt","slice","replace","split","out","field","quoted","ch","push"],"mappings":";;;;;;;;;;;QA6EaA;eAAAA;;QAmEAC;eAAAA;;QAjIAC;eAAAA;;QAiJGC;eAAAA;;QANAC;eAAAA;;;0BA1JS;4BAI6B;AAW/C,IAAA,AAAMF,4BAAN,MAAMA;;IAGHG,QAAkB,EAAE,CAAC;IACrBC,QAAQ,EAAE;IAElB,YAAY,AAAiBC,OAA4C,CAAE;aAA9CA,UAAAA;IAA+C;IAE5E,MAAMC,KAAKC,OAAyB,EAAiB;QACnD,MAAMC,MAAM,MAAMC,IAAAA,kBAAQ,EAAC,IAAI,CAACJ,OAAO,CAACK,IAAI,EAAE,IAAI,CAACL,OAAO,CAACM,QAAQ,IAAI;QACvE,IAAI,CAACR,KAAK,GAAGD,WAAWM;QACxB,IAAI,CAACJ,KAAK,GAAGQ,IAAAA,gCAAoB,EAC/BL,SACA,IAAI,CAACM,aAAa,EAClB,SACA,IAAI,CAACR,OAAO,CAACS,SAAS,IAAI;IAE9B;IAEA,MAAMC,OAA0B;QAC9B,MAAO,IAAI,CAACX,KAAK,GAAG,IAAI,CAACD,KAAK,CAACa,MAAM,CAAE;YACrC,MAAMC,YAAY,IAAI,CAACb,KAAK;YAC5B,MAAMc,OAAO,IAAI,CAACf,KAAK,CAAC,IAAI,CAACC,KAAK,CAAC;YACnC,IAAI,CAACA,KAAK,IAAI;YACd,IAAI,IAAI,CAACC,OAAO,CAACc,cAAc,KAAK,QAAQD,KAAKE,IAAI,GAAGJ,MAAM,KAAK,GAAG;YACtE,IAAI,IAAI,CAACX,OAAO,CAACgB,OAAO,KAAKC,WAAW;gBACtC,MAAMC,SAAS,MAAM,IAAI,CAAClB,OAAO,CAACgB,OAAO,CAACH,MAAMD;gBAChD,IAAIM,WAAW,MAAM;gBACrB,OAAOA;YACT;YACA,OAAOL;QACT;QACA,OAAO;IACT;IAEAM,OAAOjB,OAAyB,EAAoB;QAClD,OAAOkB,IAAAA,2BAAe,EAAClB,SAAS,IAAI,CAACM,aAAa,EAAE;YAAET,OAAO,IAAI,CAACA,KAAK;QAAC;IAC1E;IAEAsB,QAAc;QACZ,IAAI,CAACvB,KAAK,GAAG,EAAE;IACjB;IAEA,IAAYU,gBAAwB;QAClC,OAAO,IAAI,CAACR,OAAO,CAACQ,aAAa,IAAI,CAAC,UAAU,EAAE,IAAI,CAACR,OAAO,CAACK,IAAI,EAAE;IACvE;AACF;AAgBO,IAAA,AAAMZ,oBAAN,MAAMA;;IAGHK,QAAkB,EAAE,CAAC;IACrBwB,UAA6B,EAAE,CAAC;IAChCvB,QAAQ,EAAE;IAElB,YAAY,AAAiBC,OAAoC,CAAE;aAAtCA,UAAAA;IAAuC;IAEpE,MAAMC,KAAKC,OAAyB,EAAiB;QACnD,MAAMC,MAAM,MAAMC,IAAAA,kBAAQ,EAAC,IAAI,CAACJ,OAAO,CAACK,IAAI,EAAE,IAAI,CAACL,OAAO,CAACM,QAAQ,IAAI;QACvE,IAAI,CAACR,KAAK,GAAGD,WAAWM;QACxB,MAAMoB,YAAY,IAAI,CAACvB,OAAO,CAACuB,SAAS,IAAI,IAAI,CAACvB,OAAO,CAACsB,OAAO,KAAKL;QACrE,IAAI,CAACK,OAAO,GACV,IAAI,CAACtB,OAAO,CAACsB,OAAO,IACnB,CAAA,IAAI,CAACxB,KAAK,CAACa,MAAM,GAAG,IACjBf,mBAAmB,IAAI,CAACE,KAAK,CAAC,EAAE,EAAG,IAAI,CAAC0B,SAAS,IACjD,EAAE,AAAD;QACP,MAAMC,gBAAgBF,YAAY,IAAI;QACtC,IAAI,CAACxB,KAAK,GAAGQ,IAAAA,gCAAoB,EAACL,SAAS,IAAI,CAACM,aAAa,EAAE,SAASiB;IAC1E;IAEA,MAAMf,OAA0B;QAC9B,MAAO,IAAI,CAACX,KAAK,GAAG,IAAI,CAACD,KAAK,CAACa,MAAM,CAAE;YACrC,MAAMC,YAAY,IAAI,CAACb,KAAK;YAC5B,MAAMc,OAAO,IAAI,CAACf,KAAK,CAAC,IAAI,CAACC,KAAK,CAAC;YACnC,IAAI,CAACA,KAAK,IAAI;YACd,IAAI,IAAI,CAACC,OAAO,CAACc,cAAc,KAAK,SAASD,KAAKE,IAAI,GAAGJ,MAAM,KAAK,GAAG;YACvE,MAAMe,SAAS9B,mBAAmBiB,MAAM,IAAI,CAACW,SAAS;YACtD,MAAMG,MAA8B,CAAC;YACrC,IAAK,IAAIC,IAAI,GAAGA,IAAI,IAAI,CAACN,OAAO,CAACX,MAAM,EAAEiB,KAAK,EAAG;gBAC/CD,GAAG,CAAC,IAAI,CAACL,OAAO,CAACM,EAAE,CAAE,GAAGF,MAAM,CAACE,EAAE,IAAI;YACvC;YACA,IAAI,IAAI,CAAC5B,OAAO,CAAC6B,MAAM,KAAKZ,WAAW;gBACrC,MAAMC,SAAS,MAAM,IAAI,CAAClB,OAAO,CAAC6B,MAAM,CAACF,KAAKf;gBAC9C,IAAIM,WAAW,MAAM;gBACrB,OAAOA;YACT;YACA,OAAOS;QACT;QACA,OAAO;IACT;IAEAR,OAAOjB,OAAyB,EAAoB;QAClD,OAAOkB,IAAAA,2BAAe,EAAClB,SAAS,IAAI,CAACM,aAAa,EAAE;YAAET,OAAO,IAAI,CAACA,KAAK;QAAC;IAC1E;IAEAsB,QAAc;QACZ,IAAI,CAACvB,KAAK,GAAG,EAAE;QACf,IAAI,CAACwB,OAAO,GAAG,EAAE;IACnB;IAEA,IAAYd,gBAAwB;QAClC,OAAO,IAAI,CAACR,OAAO,CAACQ,aAAa,IAAI,CAAC,IAAI,EAAE,IAAI,CAACR,OAAO,CAACK,IAAI,EAAE;IACjE;IAEA,IAAYmB,YAAoB;QAC9B,OAAO,IAAI,CAACxB,OAAO,CAACwB,SAAS,IAAI;IACnC;AACF;AAQO,IAAA,AAAM9B,sBAAN,MAAMA,4BAAyCC;IACpD,YAAYK,OAAsC,CAAE;QAClD,KAAK,CAAC;YACJ,GAAGA,OAAO;YACVc,gBAAgB;YAChBE,SAAS,CAACH,OAASiB,KAAKC,KAAK,CAAClB;QAChC;IACF;AACF;AAEO,SAAShB,WAAWM,GAAoB;IAC7C,MAAM6B,OAAO,OAAO7B,QAAQ,WAAWA,MAAMA,IAAI8B,QAAQ,CAAC;IAC1D,MAAMC,aAAaF,KAAKG,UAAU,CAAC,OAAO,SAASH,KAAKI,KAAK,CAAC,KAAKJ;IACnE,OAAOE,WAAWG,OAAO,CAAC,SAAS,MAAMA,OAAO,CAAC,OAAO,MAAMC,KAAK,CAAC;AACtE;AAEO,SAAS1C,mBAAmBiB,IAAY,EAAEW,YAAY,GAAG;IAC9D,MAAMe,MAAgB,EAAE;IACxB,IAAIC,QAAQ;IACZ,IAAIC,SAAS;IACb,IAAK,IAAIb,IAAI,GAAGA,IAAIf,KAAKF,MAAM,EAAEiB,KAAK,EAAG;QACvC,MAAMc,KAAK7B,IAAI,CAACe,EAAE;QAClB,IAAIa,QAAQ;YACV,IAAIC,OAAO,OAAO7B,IAAI,CAACe,IAAI,EAAE,KAAK,KAAK;gBACrCY,SAAS;gBACTZ,KAAK;YACP,OAAO,IAAIc,OAAO,KAAK;gBACrBD,SAAS;YACX,OAAO;gBACLD,SAASE;YACX;YACA;QACF;QACA,IAAIA,OAAO,KAAK;YACdD,SAAS;YACT;QACF;QACA,IAAIC,OAAOlB,WAAW;YACpBe,IAAII,IAAI,CAACH;YACTA,QAAQ;YACR;QACF;QACAA,SAASE;IACX;IACAH,IAAII,IAAI,CAACH;IACT,OAAOD;AACT"}

package/dist/src/io/file-writers.d.ts

@@ -0,0 +1,31 @@
+import type { ExecutionContext, ItemStream, ItemWriter, WriterResult } from '../core';
+export interface JsonlFileItemWriterOptions {
+    readonly path: string;
+    readonly encoding?: BufferEncoding;
+}
+export declare class JsonlFileItemWriter<T = unknown> implements ItemWriter<T> {
+    private readonly options;
+    constructor(options: JsonlFileItemWriterOptions);
+    write(items: T[]): Promise<void>;
+}
+export interface CsvFileItemWriterOptions<T extends Record<string, unknown>> {
+    readonly path: string;
+    readonly headers: readonly (keyof T & string)[];
+    readonly encoding?: BufferEncoding;
+    readonly delimiter?: string;
+    readonly checkpointKey?: string;
+    readonly writeHeader?: boolean;
+}
+export declare class CsvFileItemWriter<T extends Record<string, unknown>> implements ItemWriter<T>, ItemStream {
+    private readonly options;
+    private headerWritten;
+    constructor(options: CsvFileItemWriterOptions<T>);
+    open(context: ExecutionContext): void;
+    write(items: T[]): Promise<WriterResult | void>;
+    update(context: ExecutionContext): ExecutionContext;
+    close(): void;
+    private get checkpointKey();
+    private get delimiter();
+}
+export declare function escapeCsvValue(value: unknown, delimiter?: string): string;
+//# sourceMappingURL=file-writers.d.ts.map

package/dist/src/io/file-writers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-writers.d.ts","sourceRoot":"","sources":["../../../src/io/file-writers.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,gBAAgB,EAAE,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAKtF,MAAM,WAAW,0BAA0B;IACzC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,CAAC,EAAE,cAAc,CAAC;CACpC;AAED,qBAAa,mBAAmB,CAAC,CAAC,GAAG,OAAO,CAAE,YAAW,UAAU,CAAC,CAAC,CAAC;IACxD,OAAO,CAAC,QAAQ,CAAC,OAAO;gBAAP,OAAO,EAAE,0BAA0B;IAE1D,KAAK,CAAC,KAAK,EAAE,CAAC,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CAKvC;AAED,MAAM,WAAW,wBAAwB,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IACzE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,EAAE,CAAC;IAChD,QAAQ,CAAC,QAAQ,CAAC,EAAE,cAAc,CAAC;IACnC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,qBAAa,iBAAiB,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAC9D,YAAW,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU;IAIxB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAFpC,OAAO,CAAC,aAAa,CAAS;gBAED,OAAO,EAAE,wBAAwB,CAAC,CAAC,CAAC;IAEjE,IAAI,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI;IAS/B,KAAK,CAAC,KAAK,EAAE,CAAC,EAAE,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;IAqBrD,MAAM,CAAC,OAAO,EAAE,gBAAgB,GAAG,gBAAgB;IAMnD,KAAK,IAAI,IAAI;IAIb,OAAO,KAAK,aAAa,GAExB;IAED,OAAO,KAAK,SAAS,GAEpB;CACF;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,SAAM,GAAG,MAAM,CAWtE"}

package/dist/src/io/file-writers.js

@@ -0,0 +1,80 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: Object.getOwnPropertyDescriptor(all, name).get
+    });
+}
+_export(exports, {
+    get CsvFileItemWriter () {
+        return CsvFileItemWriter;
+    },
+    get JsonlFileItemWriter () {
+        return JsonlFileItemWriter;
+    },
+    get escapeCsvValue () {
+        return escapeCsvValue;
+    }
+});
+const _promises = require("node:fs/promises");
+const _checkpoint = require("./checkpoint");
+const _filereaders = require("./file-readers");
+let JsonlFileItemWriter = class JsonlFileItemWriter {
+    options;
+    constructor(options){
+        this.options = options;
+    }
+    async write(items) {
+        if (items.length === 0) return;
+        const body = `${items.map((item)=>JSON.stringify(item)).join('\n')}\n`;
+        await (0, _promises.appendFile)(this.options.path, body, this.options.encoding ?? 'utf8');
+    }
+};
+let CsvFileItemWriter = class CsvFileItemWriter {
+    options;
+    headerWritten = false;
+    constructor(options){
+        this.options = options;
+    }
+    open(context) {
+        this.headerWritten = (0, _checkpoint.readCheckpointValue)(context, this.checkpointKey, 'headerWritten', false);
+    }
+    async write(items) {
+        if (items.length === 0) return;
+        const lines = [];
+        if ((this.options.writeHeader ?? true) && !this.headerWritten) {
+            lines.push(this.options.headers.map((h)=>escapeCsvValue(h, this.delimiter)).join(this.delimiter));
+            this.headerWritten = true;
+        }
+        for (const item of items){
+            lines.push(this.options.headers.map((h)=>escapeCsvValue(item[h], this.delimiter)).join(this.delimiter));
+        }
+        await (0, _promises.appendFile)(this.options.path, `${lines.join('\n')}\n`, this.options.encoding ?? 'utf8');
+    }
+    update(context) {
+        return (0, _checkpoint.writeCheckpoint)(context, this.checkpointKey, {
+            headerWritten: this.headerWritten
+        });
+    }
+    close() {
+        return undefined;
+    }
+    get checkpointKey() {
+        return this.options.checkpointKey ?? `csv-writer:${this.options.path}`;
+    }
+    get delimiter() {
+        return this.options.delimiter ?? ',';
+    }
+};
+function escapeCsvValue(value, delimiter = ',') {
+    const text = value === null || value === undefined ? '' : String(value);
+    if (text.includes('"') || text.includes('\n') || text.includes('\r') || (0, _filereaders.parseDelimitedLine)(text, delimiter).length > 1) {
+        return `"${text.replace(/"/g, '""')}"`;
+    }
+    return text;
+}
+
+//# sourceMappingURL=file-writers.js.map

package/dist/src/io/file-writers.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/io/file-writers.ts"],"sourcesContent":["import { appendFile } from 'node:fs/promises';\n\nimport type { ExecutionContext, ItemStream, ItemWriter, WriterResult } from '../core';\n\nimport { readCheckpointValue, writeCheckpoint } from './checkpoint';\nimport { parseDelimitedLine } from './file-readers';\n\nexport interface JsonlFileItemWriterOptions {\n  readonly path: string;\n  readonly encoding?: BufferEncoding;\n}\n\nexport class JsonlFileItemWriter<T = unknown> implements ItemWriter<T> {\n  constructor(private readonly options: JsonlFileItemWriterOptions) {}\n\n  async write(items: T[]): Promise<void> {\n    if (items.length === 0) return;\n    const body = `${items.map((item) => JSON.stringify(item)).join('\\n')}\\n`;\n    await appendFile(this.options.path, body, this.options.encoding ?? 'utf8');\n  }\n}\n\nexport interface CsvFileItemWriterOptions<T extends Record<string, unknown>> {\n  readonly path: string;\n  readonly headers: readonly (keyof T & string)[];\n  readonly encoding?: BufferEncoding;\n  readonly delimiter?: string;\n  readonly checkpointKey?: string;\n  readonly writeHeader?: boolean;\n}\n\nexport class CsvFileItemWriter<T extends Record<string, unknown>>\n  implements ItemWriter<T>, ItemStream\n{\n  private headerWritten = false;\n\n  constructor(private readonly options: CsvFileItemWriterOptions<T>) {}\n\n  open(context: ExecutionContext): void {\n    this.headerWritten = readCheckpointValue(\n      context,\n      this.checkpointKey,\n      'headerWritten',\n      false,\n    );\n  }\n\n  async write(items: T[]): Promise<WriterResult | void> {\n    if (items.length === 0) return;\n    const lines: string[] = [];\n    if ((this.options.writeHeader ?? true) && !this.headerWritten) {\n      lines.push(this.options.headers.map((h) => escapeCsvValue(h, this.delimiter)).join(this.delimiter));\n      this.headerWritten = true;\n    }\n    for (const item of items) {\n      lines.push(\n        this.options.headers\n          .map((h) => escapeCsvValue(item[h], this.delimiter))\n          .join(this.delimiter),\n      );\n    }\n    await appendFile(\n      this.options.path,\n      `${lines.join('\\n')}\\n`,\n      this.options.encoding ?? 'utf8',\n    );\n  }\n\n  update(context: ExecutionContext): ExecutionContext {\n    return writeCheckpoint(context, this.checkpointKey, {\n      headerWritten: this.headerWritten,\n    });\n  }\n\n  close(): void {\n    return undefined;\n  }\n\n  private get checkpointKey(): string {\n    return this.options.checkpointKey ?? `csv-writer:${this.options.path}`;\n  }\n\n  private get delimiter(): string {\n    return this.options.delimiter ?? ',';\n  }\n}\n\nexport function escapeCsvValue(value: unknown, delimiter = ','): string {\n  const text = value === null || value === undefined ? '' : String(value);\n  if (\n    text.includes('\"') ||\n    text.includes('\\n') ||\n    text.includes('\\r') ||\n    parseDelimitedLine(text, delimiter).length > 1\n  ) {\n    return `\"${text.replace(/\"/g, '\"\"')}\"`;\n  }\n  return text;\n}\n"],"names":["CsvFileItemWriter","JsonlFileItemWriter","escapeCsvValue","options","write","items","length","body","map","item","JSON","stringify","join","appendFile","path","encoding","headerWritten","open","context","readCheckpointValue","checkpointKey","lines","writeHeader","push","headers","h","delimiter","update","writeCheckpoint","close","undefined","value","text","String","includes","parseDelimitedLine","replace"],"mappings":";;;;;;;;;;;QA+BaA;eAAAA;;QAnBAC;eAAAA;;QA2EGC;eAAAA;;;0BAvFW;4BAI0B;6BAClB;AAO5B,IAAA,AAAMD,sBAAN,MAAMA;;IACX,YAAY,AAAiBE,OAAmC,CAAE;aAArCA,UAAAA;IAAsC;IAEnE,MAAMC,MAAMC,KAAU,EAAiB;QACrC,IAAIA,MAAMC,MAAM,KAAK,GAAG;QACxB,MAAMC,OAAO,GAAGF,MAAMG,GAAG,CAAC,CAACC,OAASC,KAAKC,SAAS,CAACF,OAAOG,IAAI,CAAC,MAAM,EAAE,CAAC;QACxE,MAAMC,IAAAA,oBAAU,EAAC,IAAI,CAACV,OAAO,CAACW,IAAI,EAAEP,MAAM,IAAI,CAACJ,OAAO,CAACY,QAAQ,IAAI;IACrE;AACF;AAWO,IAAA,AAAMf,oBAAN,MAAMA;;IAGHgB,gBAAgB,MAAM;IAE9B,YAAY,AAAiBb,OAAoC,CAAE;aAAtCA,UAAAA;IAAuC;IAEpEc,KAAKC,OAAyB,EAAQ;QACpC,IAAI,CAACF,aAAa,GAAGG,IAAAA,+BAAmB,EACtCD,SACA,IAAI,CAACE,aAAa,EAClB,iBACA;IAEJ;IAEA,MAAMhB,MAAMC,KAAU,EAAgC;QACpD,IAAIA,MAAMC,MAAM,KAAK,GAAG;QACxB,MAAMe,QAAkB,EAAE;QAC1B,IAAI,AAAC,CAAA,IAAI,CAAClB,OAAO,CAACmB,WAAW,IAAI,IAAG,KAAM,CAAC,IAAI,CAACN,aAAa,EAAE;YAC7DK,MAAME,IAAI,CAAC,IAAI,CAACpB,OAAO,CAACqB,OAAO,CAAChB,GAAG,CAAC,CAACiB,IAAMvB,eAAeuB,GAAG,IAAI,CAACC,SAAS,GAAGd,IAAI,CAAC,IAAI,CAACc,SAAS;YACjG,IAAI,CAACV,aAAa,GAAG;QACvB;QACA,KAAK,MAAMP,QAAQJ,MAAO;YACxBgB,MAAME,IAAI,CACR,IAAI,CAACpB,OAAO,CAACqB,OAAO,CACjBhB,GAAG,CAAC,CAACiB,IAAMvB,eAAeO,IAAI,CAACgB,EAAE,EAAE,IAAI,CAACC,SAAS,GACjDd,IAAI,CAAC,IAAI,CAACc,SAAS;QAE1B;QACA,MAAMb,IAAAA,oBAAU,EACd,IAAI,CAACV,OAAO,CAACW,IAAI,EACjB,GAAGO,MAAMT,IAAI,CAAC,MAAM,EAAE,CAAC,EACvB,IAAI,CAACT,OAAO,CAACY,QAAQ,IAAI;IAE7B;IAEAY,OAAOT,OAAyB,EAAoB;QAClD,OAAOU,IAAAA,2BAAe,EAACV,SAAS,IAAI,CAACE,aAAa,EAAE;YAClDJ,eAAe,IAAI,CAACA,aAAa;QACnC;IACF;IAEAa,QAAc;QACZ,OAAOC;IACT;IAEA,IAAYV,gBAAwB;QAClC,OAAO,IAAI,CAACjB,OAAO,CAACiB,aAAa,IAAI,CAAC,WAAW,EAAE,IAAI,CAACjB,OAAO,CAACW,IAAI,EAAE;IACxE;IAEA,IAAYY,YAAoB;QAC9B,OAAO,IAAI,CAACvB,OAAO,CAACuB,SAAS,IAAI;IACnC;AACF;AAEO,SAASxB,eAAe6B,KAAc,EAAEL,YAAY,GAAG;IAC5D,MAAMM,OAAOD,UAAU,QAAQA,UAAUD,YAAY,KAAKG,OAAOF;IACjE,IACEC,KAAKE,QAAQ,CAAC,QACdF,KAAKE,QAAQ,CAAC,SACdF,KAAKE,QAAQ,CAAC,SACdC,IAAAA,+BAAkB,EAACH,MAAMN,WAAWpB,MAAM,GAAG,GAC7C;QACA,OAAO,CAAC,CAAC,EAAE0B,KAAKI,OAAO,CAAC,MAAM,MAAM,CAAC,CAAC;IACxC;IACA,OAAOJ;AACT"}

package/dist/src/io/index.d.ts

@@ -0,0 +1,6 @@
+export * from './checkpoint';
+export * from './file-readers';
+export * from './file-writers';
+export * from './s3';
+export * from './database';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/io/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/io/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,MAAM,CAAC;AACrB,cAAc,YAAY,CAAC"}

package/dist/src/io/index.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+_export_star(require("./checkpoint"), exports);
+_export_star(require("./file-readers"), exports);
+_export_star(require("./file-writers"), exports);
+_export_star(require("./s3"), exports);
+_export_star(require("./database"), exports);
+function _export_star(from, to) {
+    Object.keys(from).forEach(function(k) {
+        if (k !== "default" && !Object.prototype.hasOwnProperty.call(to, k)) {
+            Object.defineProperty(to, k, {
+                enumerable: true,
+                get: function() {
+                    return from[k];
+                }
+            });
+        }
+    });
+    return from;
+}
+
+//# sourceMappingURL=index.js.map

package/dist/src/io/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/io/index.ts"],"sourcesContent":["export * from './checkpoint';\nexport * from './file-readers';\nexport * from './file-writers';\nexport * from './s3';\nexport * from './database';\n"],"names":[],"mappings":";;;;qBAAc;qBACA;qBACA;qBACA;qBACA"}

package/dist/src/io/s3.d.ts

@@ -0,0 +1,50 @@
+import type { ExecutionContext, ItemReader, ItemStream, ItemWriter } from '../core';
+export interface S3ObjectLocation {
+    readonly bucket: string;
+    readonly key: string;
+}
+export interface S3GetObjectInput extends S3ObjectLocation {
+}
+export interface S3PutObjectInput extends S3ObjectLocation {
+    readonly body: string | Uint8Array;
+    readonly contentType?: string;
+}
+export interface S3ObjectClient {
+    getObject(input: S3GetObjectInput): Promise<{
+        readonly body: string | Uint8Array;
+    }>;
+    putObject(input: S3PutObjectInput): Promise<void>;
+}
+export interface S3JsonlItemReaderOptions<T> extends S3ObjectLocation {
+    readonly client: S3ObjectClient;
+    readonly checkpointKey?: string;
+    readonly mapItem?: (item: unknown, lineIndex: number) => T | Promise<T>;
+}
+export declare class S3JsonlItemReader<T = unknown> implements ItemReader<T>, ItemStream {
+    private readonly options;
+    private lines;
+    private index;
+    constructor(options: S3JsonlItemReaderOptions<T>);
+    open(context: ExecutionContext): Promise<void>;
+    read(): Promise<T | null>;
+    update(context: ExecutionContext): ExecutionContext;
+    close(): void;
+    private get checkpointKey();
+}
+export interface S3ChunkJsonlItemWriterOptions {
+    readonly client: S3ObjectClient;
+    readonly bucket: string;
+    readonly keyPrefix: string;
+    readonly checkpointKey?: string;
+}
+export declare class S3ChunkJsonlItemWriter<T = unknown> implements ItemWriter<T>, ItemStream {
+    private readonly options;
+    private chunkIndex;
+    constructor(options: S3ChunkJsonlItemWriterOptions);
+    open(context: ExecutionContext): void;
+    write(items: T[]): Promise<void>;
+    update(context: ExecutionContext): ExecutionContext;
+    close(): void;
+    private get checkpointKey();
+}
+//# sourceMappingURL=s3.d.ts.map

package/dist/src/io/s3.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"s3.d.ts","sourceRoot":"","sources":["../../../src/io/s3.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAKpF,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,gBAAiB,SAAQ,gBAAgB;CAAG;AAE7D,MAAM,WAAW,gBAAiB,SAAQ,gBAAgB;IACxD,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,UAAU,CAAC;IACnC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,CAAC,KAAK,EAAE,gBAAgB,GAAG,OAAO,CAAC;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,UAAU,CAAA;KAAE,CAAC,CAAC;IACpF,SAAS,CAAC,KAAK,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACnD;AAED,MAAM,WAAW,wBAAwB,CAAC,CAAC,CAAE,SAAQ,gBAAgB;IACnE,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,KAAK,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;CACzE;AAED,qBAAa,iBAAiB,CAAC,CAAC,GAAG,OAAO,CAAE,YAAW,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU;IAIlE,OAAO,CAAC,QAAQ,CAAC,OAAO;IAHpC,OAAO,CAAC,KAAK,CAAgB;IAC7B,OAAO,CAAC,KAAK,CAAK;gBAEW,OAAO,EAAE,wBAAwB,CAAC,CAAC,CAAC;IAE3D,IAAI,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IAS9C,IAAI,IAAI,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAc/B,MAAM,CAAC,OAAO,EAAE,gBAAgB,GAAG,gBAAgB;IAInD,KAAK,IAAI,IAAI;IAIb,OAAO,KAAK,aAAa,GAExB;CACF;AAED,MAAM,WAAW,6BAA6B;IAC5C,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;CACjC;AAED,qBAAa,sBAAsB,CAAC,CAAC,GAAG,OAAO,CAAE,YAAW,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU;IAGvE,OAAO,CAAC,QAAQ,CAAC,OAAO;IAFpC,OAAO,CAAC,UAAU,CAAK;gBAEM,OAAO,EAAE,6BAA6B;IAEnE,IAAI,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI;IAI/B,KAAK,CAAC,KAAK,EAAE,CAAC,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAatC,MAAM,CAAC,OAAO,EAAE,gBAAgB,GAAG,gBAAgB;IAMnD,KAAK,IAAI,IAAI;IAIb,OAAO,KAAK,aAAa,GAExB;CACF"}

package/dist/src/io/s3.js

@@ -0,0 +1,96 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: Object.getOwnPropertyDescriptor(all, name).get
+    });
+}
+_export(exports, {
+    get S3ChunkJsonlItemWriter () {
+        return S3ChunkJsonlItemWriter;
+    },
+    get S3JsonlItemReader () {
+        return S3JsonlItemReader;
+    }
+});
+const _checkpoint = require("./checkpoint");
+const _filereaders = require("./file-readers");
+let S3JsonlItemReader = class S3JsonlItemReader {
+    options;
+    lines = [];
+    index = 0;
+    constructor(options){
+        this.options = options;
+    }
+    async open(context) {
+        const object = await this.options.client.getObject({
+            bucket: this.options.bucket,
+            key: this.options.key
+        });
+        this.lines = (0, _filereaders.splitLines)(bodyToString(object.body));
+        this.index = (0, _checkpoint.readCheckpointNumber)(context, this.checkpointKey, 'index', 0);
+    }
+    async read() {
+        while(this.index < this.lines.length){
+            const lineIndex = this.index;
+            const line = this.lines[this.index];
+            this.index += 1;
+            if (line.trim().length === 0) continue;
+            const parsed = JSON.parse(line);
+            return this.options.mapItem !== undefined ? this.options.mapItem(parsed, lineIndex) : parsed;
+        }
+        return null;
+    }
+    update(context) {
+        return (0, _checkpoint.writeCheckpoint)(context, this.checkpointKey, {
+            index: this.index
+        });
+    }
+    close() {
+        this.lines = [];
+    }
+    get checkpointKey() {
+        return this.options.checkpointKey ?? `s3-jsonl:${this.options.bucket}/${this.options.key}`;
+    }
+};
+let S3ChunkJsonlItemWriter = class S3ChunkJsonlItemWriter {
+    options;
+    chunkIndex = 0;
+    constructor(options){
+        this.options = options;
+    }
+    open(context) {
+        this.chunkIndex = (0, _checkpoint.readCheckpointNumber)(context, this.checkpointKey, 'chunkIndex', 0);
+    }
+    async write(items) {
+        if (items.length === 0) return;
+        const key = `${this.options.keyPrefix.replace(/\/+$/g, '')}/chunk-${String(this.chunkIndex).padStart(6, '0')}.jsonl`;
+        const body = `${items.map((item)=>JSON.stringify(item)).join('\n')}\n`;
+        await this.options.client.putObject({
+            bucket: this.options.bucket,
+            key,
+            body,
+            contentType: 'application/x-ndjson'
+        });
+        this.chunkIndex += 1;
+    }
+    update(context) {
+        return (0, _checkpoint.writeCheckpoint)(context, this.checkpointKey, {
+            chunkIndex: this.chunkIndex
+        });
+    }
+    close() {
+        return undefined;
+    }
+    get checkpointKey() {
+        return this.options.checkpointKey ?? `s3-jsonl-writer:${this.options.bucket}/${this.options.keyPrefix}`;
+    }
+};
+function bodyToString(body) {
+    return typeof body === 'string' ? body : Buffer.from(body).toString('utf8');
+}
+
+//# sourceMappingURL=s3.js.map

package/dist/src/io/s3.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/io/s3.ts"],"sourcesContent":["import type { ExecutionContext, ItemReader, ItemStream, ItemWriter } from '../core';\n\nimport { readCheckpointNumber, writeCheckpoint } from './checkpoint';\nimport { splitLines } from './file-readers';\n\nexport interface S3ObjectLocation {\n  readonly bucket: string;\n  readonly key: string;\n}\n\nexport interface S3GetObjectInput extends S3ObjectLocation {}\n\nexport interface S3PutObjectInput extends S3ObjectLocation {\n  readonly body: string | Uint8Array;\n  readonly contentType?: string;\n}\n\nexport interface S3ObjectClient {\n  getObject(input: S3GetObjectInput): Promise<{ readonly body: string | Uint8Array }>;\n  putObject(input: S3PutObjectInput): Promise<void>;\n}\n\nexport interface S3JsonlItemReaderOptions<T> extends S3ObjectLocation {\n  readonly client: S3ObjectClient;\n  readonly checkpointKey?: string;\n  readonly mapItem?: (item: unknown, lineIndex: number) => T | Promise<T>;\n}\n\nexport class S3JsonlItemReader<T = unknown> implements ItemReader<T>, ItemStream {\n  private lines: string[] = [];\n  private index = 0;\n\n  constructor(private readonly options: S3JsonlItemReaderOptions<T>) {}\n\n  async open(context: ExecutionContext): Promise<void> {\n    const object = await this.options.client.getObject({\n      bucket: this.options.bucket,\n      key: this.options.key,\n    });\n    this.lines = splitLines(bodyToString(object.body));\n    this.index = readCheckpointNumber(context, this.checkpointKey, 'index', 0);\n  }\n\n  async read(): Promise<T | null> {\n    while (this.index < this.lines.length) {\n      const lineIndex = this.index;\n      const line = this.lines[this.index]!;\n      this.index += 1;\n      if (line.trim().length === 0) continue;\n      const parsed = JSON.parse(line) as unknown;\n      return this.options.mapItem !== undefined\n        ? this.options.mapItem(parsed, lineIndex)\n        : (parsed as T);\n    }\n    return null;\n  }\n\n  update(context: ExecutionContext): ExecutionContext {\n    return writeCheckpoint(context, this.checkpointKey, { index: this.index });\n  }\n\n  close(): void {\n    this.lines = [];\n  }\n\n  private get checkpointKey(): string {\n    return this.options.checkpointKey ?? `s3-jsonl:${this.options.bucket}/${this.options.key}`;\n  }\n}\n\nexport interface S3ChunkJsonlItemWriterOptions {\n  readonly client: S3ObjectClient;\n  readonly bucket: string;\n  readonly keyPrefix: string;\n  readonly checkpointKey?: string;\n}\n\nexport class S3ChunkJsonlItemWriter<T = unknown> implements ItemWriter<T>, ItemStream {\n  private chunkIndex = 0;\n\n  constructor(private readonly options: S3ChunkJsonlItemWriterOptions) {}\n\n  open(context: ExecutionContext): void {\n    this.chunkIndex = readCheckpointNumber(context, this.checkpointKey, 'chunkIndex', 0);\n  }\n\n  async write(items: T[]): Promise<void> {\n    if (items.length === 0) return;\n    const key = `${this.options.keyPrefix.replace(/\\/+$/g, '')}/chunk-${String(this.chunkIndex).padStart(6, '0')}.jsonl`;\n    const body = `${items.map((item) => JSON.stringify(item)).join('\\n')}\\n`;\n    await this.options.client.putObject({\n      bucket: this.options.bucket,\n      key,\n      body,\n      contentType: 'application/x-ndjson',\n    });\n    this.chunkIndex += 1;\n  }\n\n  update(context: ExecutionContext): ExecutionContext {\n    return writeCheckpoint(context, this.checkpointKey, {\n      chunkIndex: this.chunkIndex,\n    });\n  }\n\n  close(): void {\n    return undefined;\n  }\n\n  private get checkpointKey(): string {\n    return this.options.checkpointKey ?? `s3-jsonl-writer:${this.options.bucket}/${this.options.keyPrefix}`;\n  }\n}\n\nfunction bodyToString(body: string | Uint8Array): string {\n  return typeof body === 'string' ? body : Buffer.from(body).toString('utf8');\n}\n"],"names":["S3ChunkJsonlItemWriter","S3JsonlItemReader","lines","index","options","open","context","object","client","getObject","bucket","key","splitLines","bodyToString","body","readCheckpointNumber","checkpointKey","read","length","lineIndex","line","trim","parsed","JSON","parse","mapItem","undefined","update","writeCheckpoint","close","chunkIndex","write","items","keyPrefix","replace","String","padStart","map","item","stringify","join","putObject","contentType","Buffer","from","toString"],"mappings":";;;;;;;;;;;QA6EaA;eAAAA;;QAjDAC;eAAAA;;;4BA1ByC;6BAC3B;AAyBpB,IAAA,AAAMA,oBAAN,MAAMA;;IACHC,QAAkB,EAAE,CAAC;IACrBC,QAAQ,EAAE;IAElB,YAAY,AAAiBC,OAAoC,CAAE;aAAtCA,UAAAA;IAAuC;IAEpE,MAAMC,KAAKC,OAAyB,EAAiB;QACnD,MAAMC,SAAS,MAAM,IAAI,CAACH,OAAO,CAACI,MAAM,CAACC,SAAS,CAAC;YACjDC,QAAQ,IAAI,CAACN,OAAO,CAACM,MAAM;YAC3BC,KAAK,IAAI,CAACP,OAAO,CAACO,GAAG;QACvB;QACA,IAAI,CAACT,KAAK,GAAGU,IAAAA,uBAAU,EAACC,aAAaN,OAAOO,IAAI;QAChD,IAAI,CAACX,KAAK,GAAGY,IAAAA,gCAAoB,EAACT,SAAS,IAAI,CAACU,aAAa,EAAE,SAAS;IAC1E;IAEA,MAAMC,OAA0B;QAC9B,MAAO,IAAI,CAACd,KAAK,GAAG,IAAI,CAACD,KAAK,CAACgB,MAAM,CAAE;YACrC,MAAMC,YAAY,IAAI,CAAChB,KAAK;YAC5B,MAAMiB,OAAO,IAAI,CAAClB,KAAK,CAAC,IAAI,CAACC,KAAK,CAAC;YACnC,IAAI,CAACA,KAAK,IAAI;YACd,IAAIiB,KAAKC,IAAI,GAAGH,MAAM,KAAK,GAAG;YAC9B,MAAMI,SAASC,KAAKC,KAAK,CAACJ;YAC1B,OAAO,IAAI,CAAChB,OAAO,CAACqB,OAAO,KAAKC,YAC5B,IAAI,CAACtB,OAAO,CAACqB,OAAO,CAACH,QAAQH,aAC5BG;QACP;QACA,OAAO;IACT;IAEAK,OAAOrB,OAAyB,EAAoB;QAClD,OAAOsB,IAAAA,2BAAe,EAACtB,SAAS,IAAI,CAACU,aAAa,EAAE;YAAEb,OAAO,IAAI,CAACA,KAAK;QAAC;IAC1E;IAEA0B,QAAc;QACZ,IAAI,CAAC3B,KAAK,GAAG,EAAE;IACjB;IAEA,IAAYc,gBAAwB;QAClC,OAAO,IAAI,CAACZ,OAAO,CAACY,aAAa,IAAI,CAAC,SAAS,EAAE,IAAI,CAACZ,OAAO,CAACM,MAAM,CAAC,CAAC,EAAE,IAAI,CAACN,OAAO,CAACO,GAAG,EAAE;IAC5F;AACF;AASO,IAAA,AAAMX,yBAAN,MAAMA;;IACH8B,aAAa,EAAE;IAEvB,YAAY,AAAiB1B,OAAsC,CAAE;aAAxCA,UAAAA;IAAyC;IAEtEC,KAAKC,OAAyB,EAAQ;QACpC,IAAI,CAACwB,UAAU,GAAGf,IAAAA,gCAAoB,EAACT,SAAS,IAAI,CAACU,aAAa,EAAE,cAAc;IACpF;IAEA,MAAMe,MAAMC,KAAU,EAAiB;QACrC,IAAIA,MAAMd,MAAM,KAAK,GAAG;QACxB,MAAMP,MAAM,GAAG,IAAI,CAACP,OAAO,CAAC6B,SAAS,CAACC,OAAO,CAAC,SAAS,IAAI,OAAO,EAAEC,OAAO,IAAI,CAACL,UAAU,EAAEM,QAAQ,CAAC,GAAG,KAAK,MAAM,CAAC;QACpH,MAAMtB,OAAO,GAAGkB,MAAMK,GAAG,CAAC,CAACC,OAASf,KAAKgB,SAAS,CAACD,OAAOE,IAAI,CAAC,MAAM,EAAE,CAAC;QACxE,MAAM,IAAI,CAACpC,OAAO,CAACI,MAAM,CAACiC,SAAS,CAAC;YAClC/B,QAAQ,IAAI,CAACN,OAAO,CAACM,MAAM;YAC3BC;YACAG;YACA4B,aAAa;QACf;QACA,IAAI,CAACZ,UAAU,IAAI;IACrB;IAEAH,OAAOrB,OAAyB,EAAoB;QAClD,OAAOsB,IAAAA,2BAAe,EAACtB,SAAS,IAAI,CAACU,aAAa,EAAE;YAClDc,YAAY,IAAI,CAACA,UAAU;QAC7B;IACF;IAEAD,QAAc;QACZ,OAAOH;IACT;IAEA,IAAYV,gBAAwB;QAClC,OAAO,IAAI,CAACZ,OAAO,CAACY,aAAa,IAAI,CAAC,gBAAgB,EAAE,IAAI,CAACZ,OAAO,CAACM,MAAM,CAAC,CAAC,EAAE,IAAI,CAACN,OAAO,CAAC6B,SAAS,EAAE;IACzG;AACF;AAEA,SAASpB,aAAaC,IAAyB;IAC7C,OAAO,OAAOA,SAAS,WAAWA,OAAO6B,OAAOC,IAAI,CAAC9B,MAAM+B,QAAQ,CAAC;AACtE"}

package/dist/src/listeners/builtin-listeners.d.ts

@@ -0,0 +1,77 @@
+/**
+ * Emits a one-line `log` / `warn` entry for every lifecycle event. The method
+ * names match the 7 `LifecyclePhaseKind` values plus the 3 `SkipSubKind`
+ * variants, so callers can register any of them under a `ResolverMap` key
+ * like `before:step:LoggingListener` and the invoker will dispatch correctly.
+ */
+export declare class LoggingListener {
+    private readonly logger;
+    beforeJob(ctx: {
+        jobExecutionId: string;
+    }): Promise<void>;
+    afterJob(ctx: {
+        jobExecutionId: string;
+    }, result: {
+        status: string;
+    }): Promise<void>;
+    beforeStep(ctx: {
+        jobExecutionId: string;
+        stepExecutionId: string;
+    }): Promise<void>;
+    afterStep(ctx: {
+        jobExecutionId: string;
+        stepExecutionId: string;
+    }, result: {
+        status: string;
+        exitCode: string;
+    }): Promise<void>;
+    onSkipInRead(err: unknown, item: unknown): Promise<void>;
+    onSkipInProcess(item: unknown, err: unknown): Promise<void>;
+    onSkipInWrite(items: unknown[], err: unknown): Promise<void>;
+}
+/**
+ * Per-step read / write / skip counters. The counts are keyed by
+ * `stepExecutionId`, so multiple step executions within the same job are
+ * tracked independently. Callers can read the latest counts via
+ * `getCounts(stepExecutionId)`.
+ */
+export declare class MetricsListener {
+    private readonly stepCounts;
+    /**
+     * Store the counts reported by the step result. Missing fields default to 0
+     * so a partial result (e.g. a tasklet step that has no read/write/skip
+     * counts) does not pollute the metric with `NaN`/`undefined`.
+     */
+    afterStep(ctx: {
+        stepExecutionId: string;
+    }, result: {
+        readCount?: number;
+        writeCount?: number;
+        skipCount?: number;
+        status: string;
+    }): Promise<void>;
+    /** Returns the latest recorded counts for the given step, or `undefined` if
+     *  the step has not been observed yet. */
+    getCounts(stepExecutionId: string): {
+        read: number;
+        write: number;
+        skip: number;
+    } | undefined;
+}
+/**
+ * Records the wall-clock duration of each step. `beforeStep` captures
+ * `Date.now()`; `afterStep` returns the elapsed milliseconds (or 0 if no
+ * matching `beforeStep` was observed — this keeps the listener idempotent
+ * even when invoked out of order, e.g. after a process restart that replayed
+ * a partial log).
+ */
+export declare class TimingListener {
+    private readonly startTimes;
+    beforeStep(ctx: {
+        stepExecutionId: string;
+    }): Promise<void>;
+    afterStep(ctx: {
+        stepExecutionId: string;
+    }): Promise<number>;
+}
+//# sourceMappingURL=builtin-listeners.d.ts.map

package/dist/src/listeners/builtin-listeners.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"builtin-listeners.d.ts","sourceRoot":"","sources":["../../../src/listeners/builtin-listeners.ts"],"names":[],"mappings":"AAkBA;;;;;GAKG;AACH,qBACa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoC;IAGrD,SAAS,CAAC,GAAG,EAAE;QAAE,cAAc,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAIzD,QAAQ,CACZ,GAAG,EAAE;QAAE,cAAc,EAAE,MAAM,CAAA;KAAE,EAC/B,MAAM,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,GACzB,OAAO,CAAC,IAAI,CAAC;IAKV,UAAU,CAAC,GAAG,EAAE;QACpB,cAAc,EAAE,MAAM,CAAC;QACvB,eAAe,EAAE,MAAM,CAAC;KACzB,GAAG,OAAO,CAAC,IAAI,CAAC;IAIX,SAAS,CACb,GAAG,EAAE;QAAE,cAAc,EAAE,MAAM,CAAC;QAAC,eAAe,EAAE,MAAM,CAAA;KAAE,EACxD,MAAM,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,GAC3C,OAAO,CAAC,IAAI,CAAC;IAOV,YAAY,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAMxD,eAAe,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAM3D,aAAa,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,GAAG,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;CAKnE;AAMD;;;;;GAKG;AACH,qBACa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,UAAU,CAGvB;IAEJ;;;;OAIG;IACG,SAAS,CACb,GAAG,EAAE;QAAE,eAAe,EAAE,MAAM,CAAA;KAAE,EAChC,MAAM,EAAE;QACN,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,MAAM,EAAE,MAAM,CAAC;KAChB,GACA,OAAO,CAAC,IAAI,CAAC;IAQhB;8CAC0C;IAC1C,SAAS,CACP,eAAe,EAAE,MAAM,GACtB;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS;CAG7D;AAMD;;;;;;GAMG;AACH,qBACa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA6B;IAElD,UAAU,CAAC,GAAG,EAAE;QAAE,eAAe,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAI3D,SAAS,CAAC,GAAG,EAAE;QAAE,eAAe,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,CAAC;CAOnE"}