@nest-batch/core 0.2.0

package/src/io/database.ts

@@ -0,0 +1,114 @@
+import type { ExecutionContext, ItemReader, ItemStream, ItemWriter, WriterResult } from '../core';
+import type { JsonValue } from '../core/execution-context';
+
+import { readCheckpointNumber, readCheckpointValue, writeCheckpoint } from './checkpoint';
+
+export interface DatabasePagingItemReaderOptions<T> {
+  readonly pageSize: number;
+  readonly checkpointKey?: string;
+  fetchPage(args: { offset: number; limit: number }): Promise<readonly T[]>;
+}
+
+export class DatabasePagingItemReader<T> implements ItemReader<T>, ItemStream {
+  private offset = 0;
+  private buffer: readonly T[] = [];
+  private bufferIndex = 0;
+
+  constructor(private readonly options: DatabasePagingItemReaderOptions<T>) {}
+
+  open(context: ExecutionContext): void {
+    this.offset = readCheckpointNumber(context, this.checkpointKey, 'offset', 0);
+    this.buffer = [];
+    this.bufferIndex = 0;
+  }
+
+  async read(): Promise<T | null> {
+    if (this.bufferIndex >= this.buffer.length) {
+      this.buffer = await this.options.fetchPage({
+        offset: this.offset,
+        limit: this.options.pageSize,
+      });
+      this.bufferIndex = 0;
+      if (this.buffer.length === 0) return null;
+    }
+    const item = this.buffer[this.bufferIndex]!;
+    this.bufferIndex += 1;
+    this.offset += 1;
+    return item;
+  }
+
+  update(context: ExecutionContext): ExecutionContext {
+    return writeCheckpoint(context, this.checkpointKey, { offset: this.offset });
+  }
+
+  close(): void {
+    this.buffer = [];
+  }
+
+  private get checkpointKey(): string {
+    return this.options.checkpointKey ?? 'database-paging-reader';
+  }
+}
+
+export interface DatabaseCursorItemReaderOptions<T> {
+  readonly pageSize: number;
+  readonly checkpointKey?: string;
+  readonly initialCursor?: JsonValue;
+  fetchAfter(cursor: JsonValue, limit: number): Promise<readonly T[]>;
+  getCursor(item: T): JsonValue;
+}
+
+export class DatabaseCursorItemReader<T> implements ItemReader<T>, ItemStream {
+  private cursor: JsonValue = null;
+  private buffer: readonly T[] = [];
+  private bufferIndex = 0;
+
+  constructor(private readonly options: DatabaseCursorItemReaderOptions<T>) {}
+
+  open(context: ExecutionContext): void {
+    this.cursor = readCheckpointValue(
+      context,
+      this.checkpointKey,
+      'cursor',
+      this.options.initialCursor ?? null,
+    );
+    this.buffer = [];
+    this.bufferIndex = 0;
+  }
+
+  async read(): Promise<T | null> {
+    if (this.bufferIndex >= this.buffer.length) {
+      this.buffer = await this.options.fetchAfter(this.cursor, this.options.pageSize);
+      this.bufferIndex = 0;
+      if (this.buffer.length === 0) return null;
+    }
+    const item = this.buffer[this.bufferIndex]!;
+    this.bufferIndex += 1;
+    this.cursor = this.options.getCursor(item);
+    return item;
+  }
+
+  update(context: ExecutionContext): ExecutionContext {
+    return writeCheckpoint(context, this.checkpointKey, { cursor: this.cursor });
+  }
+
+  close(): void {
+    this.buffer = [];
+  }
+
+  private get checkpointKey(): string {
+    return this.options.checkpointKey ?? 'database-cursor-reader';
+  }
+}
+
+export interface DatabaseBatchItemWriterOptions<T> {
+  writeBatch(items: readonly T[]): Promise<WriterResult | void>;
+}
+
+export class DatabaseBatchItemWriter<T> implements ItemWriter<T> {
+  constructor(private readonly options: DatabaseBatchItemWriterOptions<T>) {}
+
+  write(items: T[]): Promise<WriterResult | void> {
+    return this.options.writeBatch(items);
+  }
+}

package/src/io/file-readers.ts

@@ -0,0 +1,191 @@
+import { readFile } from 'node:fs/promises';
+
+import type { ExecutionContext, ItemReader, ItemStream } from '../core';
+
+import { readCheckpointNumber, writeCheckpoint } from './checkpoint';
+
+export interface RestartableFileLineReaderOptions<T = string> {
+  readonly path: string;
+  readonly encoding?: BufferEncoding;
+  readonly checkpointKey?: string;
+  readonly skipLines?: number;
+  readonly skipBlankLines?: boolean;
+  readonly mapLine?: (line: string, lineIndex: number) => T | null | Promise<T | null>;
+}
+
+export class RestartableFileLineReader<T = string>
+  implements ItemReader<T>, ItemStream
+{
+  private lines: string[] = [];
+  private index = 0;
+
+  constructor(private readonly options: RestartableFileLineReaderOptions<T>) {}
+
+  async open(context: ExecutionContext): Promise<void> {
+    const raw = await readFile(this.options.path, this.options.encoding ?? 'utf8');
+    this.lines = splitLines(raw);
+    this.index = readCheckpointNumber(
+      context,
+      this.checkpointKey,
+      'index',
+      this.options.skipLines ?? 0,
+    );
+  }
+
+  async read(): Promise<T | null> {
+    while (this.index < this.lines.length) {
+      const lineIndex = this.index;
+      const line = this.lines[this.index]!;
+      this.index += 1;
+      if (this.options.skipBlankLines === true && line.trim().length === 0) continue;
+      if (this.options.mapLine !== undefined) {
+        const mapped = await this.options.mapLine(line, lineIndex);
+        if (mapped === null) continue;
+        return mapped;
+      }
+      return line as T;
+    }
+    return null;
+  }
+
+  update(context: ExecutionContext): ExecutionContext {
+    return writeCheckpoint(context, this.checkpointKey, { index: this.index });
+  }
+
+  close(): void {
+    this.lines = [];
+  }
+
+  private get checkpointKey(): string {
+    return this.options.checkpointKey ?? `file-line:${this.options.path}`;
+  }
+}
+
+export interface CsvFileItemReaderOptions<T extends Record<string, unknown>> {
+  readonly path: string;
+  readonly encoding?: BufferEncoding;
+  readonly checkpointKey?: string;
+  readonly delimiter?: string;
+  readonly headers?: readonly string[];
+  readonly hasHeader?: boolean;
+  readonly skipBlankLines?: boolean;
+  readonly mapRow?: (
+    row: Record<string, string>,
+    lineIndex: number,
+  ) => T | null | Promise<T | null>;
+}
+
+export class CsvFileItemReader<T extends Record<string, unknown> = Record<string, string>>
+  implements ItemReader<T>, ItemStream
+{
+  private lines: string[] = [];
+  private headers: readonly string[] = [];
+  private index = 0;
+
+  constructor(private readonly options: CsvFileItemReaderOptions<T>) {}
+
+  async open(context: ExecutionContext): Promise<void> {
+    const raw = await readFile(this.options.path, this.options.encoding ?? 'utf8');
+    this.lines = splitLines(raw);
+    const hasHeader = this.options.hasHeader ?? this.options.headers === undefined;
+    this.headers =
+      this.options.headers ??
+      (this.lines.length > 0
+        ? parseDelimitedLine(this.lines[0]!, this.delimiter)
+        : []);
+    const firstDataLine = hasHeader ? 1 : 0;
+    this.index = readCheckpointNumber(context, this.checkpointKey, 'index', firstDataLine);
+  }
+
+  async read(): Promise<T | null> {
+    while (this.index < this.lines.length) {
+      const lineIndex = this.index;
+      const line = this.lines[this.index]!;
+      this.index += 1;
+      if (this.options.skipBlankLines !== false && line.trim().length === 0) continue;
+      const fields = parseDelimitedLine(line, this.delimiter);
+      const row: Record<string, string> = {};
+      for (let i = 0; i < this.headers.length; i += 1) {
+        row[this.headers[i]!] = fields[i] ?? '';
+      }
+      if (this.options.mapRow !== undefined) {
+        const mapped = await this.options.mapRow(row, lineIndex);
+        if (mapped === null) continue;
+        return mapped;
+      }
+      return row as T;
+    }
+    return null;
+  }
+
+  update(context: ExecutionContext): ExecutionContext {
+    return writeCheckpoint(context, this.checkpointKey, { index: this.index });
+  }
+
+  close(): void {
+    this.lines = [];
+    this.headers = [];
+  }
+
+  private get checkpointKey(): string {
+    return this.options.checkpointKey ?? `csv:${this.options.path}`;
+  }
+
+  private get delimiter(): string {
+    return this.options.delimiter ?? ',';
+  }
+}
+
+export interface JsonlFileItemReaderOptions<T> {
+  readonly path: string;
+  readonly encoding?: BufferEncoding;
+  readonly checkpointKey?: string;
+}
+
+export class JsonlFileItemReader<T = unknown> extends RestartableFileLineReader<T> {
+  constructor(options: JsonlFileItemReaderOptions<T>) {
+    super({
+      ...options,
+      skipBlankLines: true,
+      mapLine: (line) => JSON.parse(line) as T,
+    });
+  }
+}
+
+export function splitLines(raw: string | Buffer): string[] {
+  const text = typeof raw === 'string' ? raw : raw.toString('utf8');
+  const withoutBom = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
+  return withoutBom.replace(/\r\n/g, '\n').replace(/\r/g, '\n').split('\n');
+}
+
+export function parseDelimitedLine(line: string, delimiter = ','): string[] {
+  const out: string[] = [];
+  let field = '';
+  let quoted = false;
+  for (let i = 0; i < line.length; i += 1) {
+    const ch = line[i]!;
+    if (quoted) {
+      if (ch === '"' && line[i + 1] === '"') {
+        field += '"';
+        i += 1;
+      } else if (ch === '"') {
+        quoted = false;
+      } else {
+        field += ch;
+      }
+      continue;
+    }
+    if (ch === '"') {
+      quoted = true;
+      continue;
+    }
+    if (ch === delimiter) {
+      out.push(field);
+      field = '';
+      continue;
+    }
+    field += ch;
+  }
+  out.push(field);
+  return out;
+}

package/src/io/file-writers.ts

@@ -0,0 +1,99 @@
+import { appendFile } from 'node:fs/promises';
+
+import type { ExecutionContext, ItemStream, ItemWriter, WriterResult } from '../core';
+
+import { readCheckpointValue, writeCheckpoint } from './checkpoint';
+import { parseDelimitedLine } from './file-readers';
+
+export interface JsonlFileItemWriterOptions {
+  readonly path: string;
+  readonly encoding?: BufferEncoding;
+}
+
+export class JsonlFileItemWriter<T = unknown> implements ItemWriter<T> {
+  constructor(private readonly options: JsonlFileItemWriterOptions) {}
+
+  async write(items: T[]): Promise<void> {
+    if (items.length === 0) return;
+    const body = `${items.map((item) => JSON.stringify(item)).join('\n')}\n`;
+    await appendFile(this.options.path, body, this.options.encoding ?? 'utf8');
+  }
+}
+
+export interface CsvFileItemWriterOptions<T extends Record<string, unknown>> {
+  readonly path: string;
+  readonly headers: readonly (keyof T & string)[];
+  readonly encoding?: BufferEncoding;
+  readonly delimiter?: string;
+  readonly checkpointKey?: string;
+  readonly writeHeader?: boolean;
+}
+
+export class CsvFileItemWriter<T extends Record<string, unknown>>
+  implements ItemWriter<T>, ItemStream
+{
+  private headerWritten = false;
+
+  constructor(private readonly options: CsvFileItemWriterOptions<T>) {}
+
+  open(context: ExecutionContext): void {
+    this.headerWritten = readCheckpointValue(
+      context,
+      this.checkpointKey,
+      'headerWritten',
+      false,
+    );
+  }
+
+  async write(items: T[]): Promise<WriterResult | void> {
+    if (items.length === 0) return;
+    const lines: string[] = [];
+    if ((this.options.writeHeader ?? true) && !this.headerWritten) {
+      lines.push(this.options.headers.map((h) => escapeCsvValue(h, this.delimiter)).join(this.delimiter));
+      this.headerWritten = true;
+    }
+    for (const item of items) {
+      lines.push(
+        this.options.headers
+          .map((h) => escapeCsvValue(item[h], this.delimiter))
+          .join(this.delimiter),
+      );
+    }
+    await appendFile(
+      this.options.path,
+      `${lines.join('\n')}\n`,
+      this.options.encoding ?? 'utf8',
+    );
+  }
+
+  update(context: ExecutionContext): ExecutionContext {
+    return writeCheckpoint(context, this.checkpointKey, {
+      headerWritten: this.headerWritten,
+    });
+  }
+
+  close(): void {
+    return undefined;
+  }
+
+  private get checkpointKey(): string {
+    return this.options.checkpointKey ?? `csv-writer:${this.options.path}`;
+  }
+
+  private get delimiter(): string {
+    return this.options.delimiter ?? ',';
+  }
+}
+
+export function escapeCsvValue(value: unknown, delimiter = ','): string {
+  const text = value === null || value === undefined ? '' : String(value);
+  if (
+    text.includes('"') ||
+    text.includes('\n') ||
+    text.includes('\r') ||
+    parseDelimitedLine(text, delimiter).length > 1
+  ) {
+    return `"${text.replace(/"/g, '""')}"`;
+  }
+  return text;
+}

package/src/io/index.ts

@@ -0,0 +1,5 @@
+export * from './checkpoint';
+export * from './file-readers';
+export * from './file-writers';
+export * from './s3';
+export * from './database';

package/src/io/s3.ts

@@ -0,0 +1,117 @@
+import type { ExecutionContext, ItemReader, ItemStream, ItemWriter } from '../core';
+
+import { readCheckpointNumber, writeCheckpoint } from './checkpoint';
+import { splitLines } from './file-readers';
+
+export interface S3ObjectLocation {
+  readonly bucket: string;
+  readonly key: string;
+}
+
+export interface S3GetObjectInput extends S3ObjectLocation {}
+
+export interface S3PutObjectInput extends S3ObjectLocation {
+  readonly body: string | Uint8Array;
+  readonly contentType?: string;
+}
+
+export interface S3ObjectClient {
+  getObject(input: S3GetObjectInput): Promise<{ readonly body: string | Uint8Array }>;
+  putObject(input: S3PutObjectInput): Promise<void>;
+}
+
+export interface S3JsonlItemReaderOptions<T> extends S3ObjectLocation {
+  readonly client: S3ObjectClient;
+  readonly checkpointKey?: string;
+  readonly mapItem?: (item: unknown, lineIndex: number) => T | Promise<T>;
+}
+
+export class S3JsonlItemReader<T = unknown> implements ItemReader<T>, ItemStream {
+  private lines: string[] = [];
+  private index = 0;
+
+  constructor(private readonly options: S3JsonlItemReaderOptions<T>) {}
+
+  async open(context: ExecutionContext): Promise<void> {
+    const object = await this.options.client.getObject({
+      bucket: this.options.bucket,
+      key: this.options.key,
+    });
+    this.lines = splitLines(bodyToString(object.body));
+    this.index = readCheckpointNumber(context, this.checkpointKey, 'index', 0);
+  }
+
+  async read(): Promise<T | null> {
+    while (this.index < this.lines.length) {
+      const lineIndex = this.index;
+      const line = this.lines[this.index]!;
+      this.index += 1;
+      if (line.trim().length === 0) continue;
+      const parsed = JSON.parse(line) as unknown;
+      return this.options.mapItem !== undefined
+        ? this.options.mapItem(parsed, lineIndex)
+        : (parsed as T);
+    }
+    return null;
+  }
+
+  update(context: ExecutionContext): ExecutionContext {
+    return writeCheckpoint(context, this.checkpointKey, { index: this.index });
+  }
+
+  close(): void {
+    this.lines = [];
+  }
+
+  private get checkpointKey(): string {
+    return this.options.checkpointKey ?? `s3-jsonl:${this.options.bucket}/${this.options.key}`;
+  }
+}
+
+export interface S3ChunkJsonlItemWriterOptions {
+  readonly client: S3ObjectClient;
+  readonly bucket: string;
+  readonly keyPrefix: string;
+  readonly checkpointKey?: string;
+}
+
+export class S3ChunkJsonlItemWriter<T = unknown> implements ItemWriter<T>, ItemStream {
+  private chunkIndex = 0;
+
+  constructor(private readonly options: S3ChunkJsonlItemWriterOptions) {}
+
+  open(context: ExecutionContext): void {
+    this.chunkIndex = readCheckpointNumber(context, this.checkpointKey, 'chunkIndex', 0);
+  }
+
+  async write(items: T[]): Promise<void> {
+    if (items.length === 0) return;
+    const key = `${this.options.keyPrefix.replace(/\/+$/g, '')}/chunk-${String(this.chunkIndex).padStart(6, '0')}.jsonl`;
+    const body = `${items.map((item) => JSON.stringify(item)).join('\n')}\n`;
+    await this.options.client.putObject({
+      bucket: this.options.bucket,
+      key,
+      body,
+      contentType: 'application/x-ndjson',
+    });
+    this.chunkIndex += 1;
+  }
+
+  update(context: ExecutionContext): ExecutionContext {
+    return writeCheckpoint(context, this.checkpointKey, {
+      chunkIndex: this.chunkIndex,
+    });
+  }
+
+  close(): void {
+    return undefined;
+  }
+
+  private get checkpointKey(): string {
+    return this.options.checkpointKey ?? `s3-jsonl-writer:${this.options.bucket}/${this.options.keyPrefix}`;
+  }
+}
+
+function bodyToString(body: string | Uint8Array): string {
+  return typeof body === 'string' ? body : Buffer.from(body).toString('utf8');
+}

package/src/listeners/builtin-listeners.ts

@@ -0,0 +1,151 @@
+/**
+ * Reference built-in listeners — drop-in `Injectable` classes that mirror common
+ * batch lifecycle conventions (logging, metrics, timing). Each listener exposes
+ * the lifecycle methods consumed by `ListenerInvoker` and uses the canonical
+ * `ResolverMap` key format (`${phase}:${kind}:${name}`), so a user can simply
+ * instantiate one of these classes and register the bound methods under the
+ * desired phase/kind pairs.
+ *
+ * The classes are intentionally framework-agnostic — they do not depend on
+ * the execution pipeline, the `ResolverMap` shape, or any module wiring. They
+ * only depend on `@nestjs/common` for `Logger` / `Injectable`.
+ */
+import { Injectable, Logger } from '@nestjs/common';
+
+// ---------------------------------------------------------------------------
+// LoggingListener
+// ---------------------------------------------------------------------------
+
+/**
+ * Emits a one-line `log` / `warn` entry for every lifecycle event. The method
+ * names match the 7 `LifecyclePhaseKind` values plus the 3 `SkipSubKind`
+ * variants, so callers can register any of them under a `ResolverMap` key
+ * like `before:step:LoggingListener` and the invoker will dispatch correctly.
+ */
+@Injectable()
+export class LoggingListener {
+  private readonly logger = new Logger(LoggingListener.name);
+
+  // -- Job -----------------------------------------------------------------
+  async beforeJob(ctx: { jobExecutionId: string }): Promise<void> {
+    this.logger.log(`Job ${ctx.jobExecutionId} starting`);
+  }
+
+  async afterJob(
+    ctx: { jobExecutionId: string },
+    result: { status: string },
+  ): Promise<void> {
+    this.logger.log(`Job ${ctx.jobExecutionId} ${result.status}`);
+  }
+
+  // -- Step ----------------------------------------------------------------
+  async beforeStep(ctx: {
+    jobExecutionId: string;
+    stepExecutionId: string;
+  }): Promise<void> {
+    this.logger.log(`Step ${ctx.stepExecutionId} starting`);
+  }
+
+  async afterStep(
+    ctx: { jobExecutionId: string; stepExecutionId: string },
+    result: { status: string; exitCode: string },
+  ): Promise<void> {
+    this.logger.log(
+      `Step ${ctx.stepExecutionId} ${result.status} (${result.exitCode})`,
+    );
+  }
+
+  // -- Skip ----------------------------------------------------------------
+  async onSkipInRead(err: unknown, item: unknown): Promise<void> {
+    this.logger.warn(
+      `Skipped read: ${(err as Error).message} (item=${String(item)})`,
+    );
+  }
+
+  async onSkipInProcess(item: unknown, err: unknown): Promise<void> {
+    this.logger.warn(
+      `Skipped process: ${(err as Error).message} (item=${String(item)})`,
+    );
+  }
+
+  async onSkipInWrite(items: unknown[], err: unknown): Promise<void> {
+    this.logger.warn(
+      `Skipped write of ${items.length} items: ${(err as Error).message}`,
+    );
+  }
+}
+
+// ---------------------------------------------------------------------------
+// MetricsListener
+// ---------------------------------------------------------------------------
+
+/**
+ * Per-step read / write / skip counters. The counts are keyed by
+ * `stepExecutionId`, so multiple step executions within the same job are
+ * tracked independently. Callers can read the latest counts via
+ * `getCounts(stepExecutionId)`.
+ */
+@Injectable()
+export class MetricsListener {
+  private readonly stepCounts = new Map<
+    string,
+    { read: number; write: number; skip: number }
+  >();
+
+  /**
+   * Store the counts reported by the step result. Missing fields default to 0
+   * so a partial result (e.g. a tasklet step that has no read/write/skip
+   * counts) does not pollute the metric with `NaN`/`undefined`.
+   */
+  async afterStep(
+    ctx: { stepExecutionId: string },
+    result: {
+      readCount?: number;
+      writeCount?: number;
+      skipCount?: number;
+      status: string;
+    },
+  ): Promise<void> {
+    this.stepCounts.set(ctx.stepExecutionId, {
+      read: result.readCount ?? 0,
+      write: result.writeCount ?? 0,
+      skip: result.skipCount ?? 0,
+    });
+  }
+
+  /** Returns the latest recorded counts for the given step, or `undefined` if
+   *  the step has not been observed yet. */
+  getCounts(
+    stepExecutionId: string,
+  ): { read: number; write: number; skip: number } | undefined {
+    return this.stepCounts.get(stepExecutionId);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// TimingListener
+// ---------------------------------------------------------------------------
+
+/**
+ * Records the wall-clock duration of each step. `beforeStep` captures
+ * `Date.now()`; `afterStep` returns the elapsed milliseconds (or 0 if no
+ * matching `beforeStep` was observed — this keeps the listener idempotent
+ * even when invoked out of order, e.g. after a process restart that replayed
+ * a partial log).
+ */
+@Injectable()
+export class TimingListener {
+  private readonly startTimes = new Map<string, number>();
+
+  async beforeStep(ctx: { stepExecutionId: string }): Promise<void> {
+    this.startTimes.set(ctx.stepExecutionId, Date.now());
+  }
+
+  async afterStep(ctx: { stepExecutionId: string }): Promise<number> {
+    const start = this.startTimes.get(ctx.stepExecutionId);
+    if (start !== undefined) {
+      return Date.now() - start;
+    }
+    return 0;
+  }
+}

package/src/listeners/index.ts

@@ -0,0 +1,7 @@
+/**
+ * Public surface for the reference built-in listeners.
+ *
+ * Re-exports `LoggingListener`, `MetricsListener`, and `TimingListener` so
+ * consumers can do `import { LoggingListener } from '@nest-batch/core'`.
+ */
+export * from './builtin-listeners';