@bulkimport/core 0.1.0

package/dist/index.js

@@ -0,0 +1,777 @@
+// src/domain/model/ImportStatus.ts
+var ImportStatus = {
+  CREATED: "CREATED",
+  PREVIEWING: "PREVIEWING",
+  PREVIEWED: "PREVIEWED",
+  PROCESSING: "PROCESSING",
+  PAUSED: "PAUSED",
+  COMPLETED: "COMPLETED",
+  ABORTED: "ABORTED",
+  FAILED: "FAILED"
+};
+var VALID_TRANSITIONS = {
+  [ImportStatus.CREATED]: [ImportStatus.PREVIEWING, ImportStatus.PROCESSING],
+  [ImportStatus.PREVIEWING]: [ImportStatus.PREVIEWED, ImportStatus.FAILED],
+  [ImportStatus.PREVIEWED]: [ImportStatus.PROCESSING],
+  [ImportStatus.PROCESSING]: [ImportStatus.PAUSED, ImportStatus.COMPLETED, ImportStatus.ABORTED, ImportStatus.FAILED],
+  [ImportStatus.PAUSED]: [ImportStatus.PROCESSING, ImportStatus.ABORTED],
+  [ImportStatus.COMPLETED]: [],
+  [ImportStatus.ABORTED]: [],
+  [ImportStatus.FAILED]: []
+};
+function canTransition(from, to) {
+  return VALID_TRANSITIONS[from].includes(to);
+}
+
+// src/domain/model/Batch.ts
+function createBatch(id, index, records) {
+  return {
+    id,
+    index,
+    status: "PENDING",
+    records,
+    processedCount: 0,
+    failedCount: 0
+  };
+}
+
+// src/domain/model/Record.ts
+function createPendingRecord(index, raw) {
+  return {
+    index,
+    raw,
+    parsed: raw,
+    status: "pending",
+    errors: []
+  };
+}
+function markRecordValid(record, parsed) {
+  return { ...record, parsed, status: "valid", errors: [] };
+}
+function markRecordInvalid(record, errors) {
+  return { ...record, status: "invalid", errors };
+}
+function markRecordProcessed(record) {
+  return { ...record, status: "processed" };
+}
+function markRecordFailed(record, error) {
+  return { ...record, status: "failed", processingError: error };
+}
+
+// src/domain/model/ValidationResult.ts
+function validResult() {
+  return { isValid: true, errors: [] };
+}
+function invalidResult(errors) {
+  return { isValid: false, errors };
+}
+
+// src/domain/services/SchemaValidator.ts
+var EMAIL_PATTERN = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
+var SchemaValidator = class {
+  constructor(schema) {
+    this.schema = schema;
+  }
+  validate(record) {
+    const errors = [];
+    for (const field of this.schema.fields) {
+      const value = record[field.name];
+      const fieldErrors = this.validateField(field, value);
+      errors.push(...fieldErrors);
+    }
+    if (this.schema.strict) {
+      const definedFields = new Set(this.schema.fields.map((f) => f.name));
+      for (const key of Object.keys(record)) {
+        if (!definedFields.has(key)) {
+          errors.push({
+            field: key,
+            message: `Unknown field '${key}' is not allowed in strict mode`,
+            code: "UNKNOWN_FIELD",
+            value: record[key]
+          });
+        }
+      }
+    }
+    return errors.length === 0 ? validResult() : invalidResult(errors);
+  }
+  applyTransforms(record) {
+    const transformed = { ...record };
+    for (const field of this.schema.fields) {
+      if (field.transform && transformed[field.name] !== void 0) {
+        transformed[field.name] = field.transform(transformed[field.name]);
+      }
+      if (transformed[field.name] === void 0 && field.defaultValue !== void 0) {
+        transformed[field.name] = field.defaultValue;
+      }
+    }
+    return transformed;
+  }
+  validateField(field, value) {
+    const errors = [];
+    if (this.isEmpty(value)) {
+      if (field.required) {
+        errors.push({
+          field: field.name,
+          message: `Field '${field.name}' is required`,
+          code: "REQUIRED",
+          value
+        });
+      }
+      return errors;
+    }
+    if (field.type !== "custom") {
+      const typeError = this.validateType(field, value);
+      if (typeError) {
+        errors.push(typeError);
+        return errors;
+      }
+    }
+    if (field.pattern) {
+      const stringValue = String(value);
+      if (!field.pattern.test(stringValue)) {
+        errors.push({
+          field: field.name,
+          message: `Field '${field.name}' does not match pattern ${String(field.pattern)}`,
+          code: "PATTERN_MISMATCH",
+          value
+        });
+      }
+    }
+    if (field.customValidator) {
+      const result = field.customValidator(value);
+      if (!result.valid) {
+        errors.push({
+          field: field.name,
+          message: result.message ?? `Custom validation failed for field '${field.name}'`,
+          code: "CUSTOM_VALIDATION",
+          value
+        });
+      }
+    }
+    return errors;
+  }
+  validateType(field, value) {
+    const stringValue = String(value);
+    switch (field.type) {
+      case "number": {
+        const num = Number(stringValue);
+        if (isNaN(num)) {
+          return {
+            field: field.name,
+            message: `Field '${field.name}' must be a number`,
+            code: "TYPE_MISMATCH",
+            value
+          };
+        }
+        return null;
+      }
+      case "boolean": {
+        const lower = stringValue.toLowerCase();
+        if (!["true", "false", "1", "0", "yes", "no"].includes(lower)) {
+          return {
+            field: field.name,
+            message: `Field '${field.name}' must be a boolean`,
+            code: "TYPE_MISMATCH",
+            value
+          };
+        }
+        return null;
+      }
+      case "date": {
+        const date = new Date(stringValue);
+        if (isNaN(date.getTime())) {
+          return {
+            field: field.name,
+            message: `Field '${field.name}' must be a valid date`,
+            code: "TYPE_MISMATCH",
+            value
+          };
+        }
+        return null;
+      }
+      case "email": {
+        if (!EMAIL_PATTERN.test(stringValue)) {
+          return {
+            field: field.name,
+            message: `Field '${field.name}' must be a valid email`,
+            code: "TYPE_MISMATCH",
+            value
+          };
+        }
+        return null;
+      }
+      case "string":
+        return null;
+      default:
+        return null;
+    }
+  }
+  isEmpty(value) {
+    return value === void 0 || value === null || value === "";
+  }
+};
+
+// src/application/EventBus.ts
+var EventBus = class {
+  constructor() {
+    this.handlers = /* @__PURE__ */ new Map();
+  }
+  on(type, handler) {
+    const existing = this.handlers.get(type) ?? /* @__PURE__ */ new Set();
+    existing.add(handler);
+    this.handlers.set(type, existing);
+  }
+  off(type, handler) {
+    const existing = this.handlers.get(type);
+    if (existing) {
+      existing.delete(handler);
+    }
+  }
+  emit(event) {
+    const handlers = this.handlers.get(event.type);
+    if (handlers) {
+      for (const handler of handlers) {
+        handler(event);
+      }
+    }
+  }
+};
+
+// src/infrastructure/state/InMemoryStateStore.ts
+var InMemoryStateStore = class {
+  constructor() {
+    this.jobs = /* @__PURE__ */ new Map();
+    this.records = /* @__PURE__ */ new Map();
+  }
+  saveJobState(job) {
+    this.jobs.set(job.id, job);
+    return Promise.resolve();
+  }
+  getJobState(jobId) {
+    return Promise.resolve(this.jobs.get(jobId) ?? null);
+  }
+  updateBatchState(jobId, batchId, state) {
+    const job = this.jobs.get(jobId);
+    if (!job) return Promise.resolve();
+    const batches = job.batches.map(
+      (b) => b.id === batchId ? { ...b, status: state.status, processedCount: state.processedCount, failedCount: state.failedCount } : b
+    );
+    this.jobs.set(jobId, { ...job, batches });
+    return Promise.resolve();
+  }
+  saveProcessedRecord(jobId, _batchId, record) {
+    const key = jobId;
+    const existing = this.records.get(key) ?? [];
+    const index = existing.findIndex((r) => r.index === record.index);
+    if (index >= 0) {
+      existing[index] = record;
+    } else {
+      existing.push(record);
+    }
+    this.records.set(key, existing);
+    return Promise.resolve();
+  }
+  getFailedRecords(jobId) {
+    const all = this.records.get(jobId) ?? [];
+    return Promise.resolve(all.filter((r) => r.status === "failed" || r.status === "invalid"));
+  }
+  getPendingRecords(jobId) {
+    const all = this.records.get(jobId) ?? [];
+    return Promise.resolve(all.filter((r) => r.status === "pending" || r.status === "valid"));
+  }
+  getProcessedRecords(jobId) {
+    const all = this.records.get(jobId) ?? [];
+    return Promise.resolve(all.filter((r) => r.status === "processed"));
+  }
+  getProgress(jobId) {
+    const job = this.jobs.get(jobId);
+    const all = this.records.get(jobId) ?? [];
+    const processed = all.filter((r) => r.status === "processed").length;
+    const failed = all.filter((r) => r.status === "failed" || r.status === "invalid").length;
+    const total = job?.totalRecords ?? all.length;
+    const pending = total - processed - failed;
+    const elapsed = job?.startedAt ? Date.now() - job.startedAt : 0;
+    const currentBatch = job?.batches.filter((b) => b.status === "COMPLETED").length ?? 0;
+    const totalBatches = job?.batches.length ?? 0;
+    return Promise.resolve({
+      totalRecords: total,
+      processedRecords: processed,
+      failedRecords: failed,
+      pendingRecords: pending,
+      percentage: total > 0 ? Math.round(processed / total * 100) : 0,
+      currentBatch,
+      totalBatches,
+      elapsedMs: elapsed
+    });
+  }
+};
+
+// src/BulkImport.ts
+var BulkImport = class {
+  constructor(config) {
+    this.source = null;
+    this.parser = null;
+    this.status = "CREATED";
+    this.batches = [];
+    this.allRecords = [];
+    this.totalRecords = 0;
+    this.abortController = null;
+    this.pausePromise = null;
+    this.config = {
+      ...config,
+      batchSize: config.batchSize ?? 100,
+      continueOnError: config.continueOnError ?? false
+    };
+    this.validator = new SchemaValidator(config.schema);
+    this.eventBus = new EventBus();
+    this.stateStore = config.stateStore ?? new InMemoryStateStore();
+    this.jobId = crypto.randomUUID();
+  }
+  from(source, parser) {
+    this.source = source;
+    this.parser = parser;
+    return this;
+  }
+  on(type, handler) {
+    this.eventBus.on(type, handler);
+    return this;
+  }
+  async preview(maxRecords = 10) {
+    this.assertSourceConfigured();
+    this.transitionTo("PREVIEWING");
+    const records = await this.parseRecords(maxRecords);
+    const validRecords = [];
+    const invalidRecords = [];
+    const columns = /* @__PURE__ */ new Set();
+    for (const record of records) {
+      for (const key of Object.keys(record.raw)) {
+        columns.add(key);
+      }
+      const transformed = this.validator.applyTransforms(record.raw);
+      const result = this.validator.validate(transformed);
+      if (result.isValid) {
+        validRecords.push(markRecordValid(record, transformed));
+      } else {
+        invalidRecords.push(markRecordInvalid(record, result.errors));
+      }
+    }
+    this.transitionTo("PREVIEWED");
+    return {
+      validRecords,
+      invalidRecords,
+      totalSampled: records.length,
+      columns: [...columns]
+    };
+  }
+  async start(processor) {
+    this.assertSourceConfigured();
+    this.assertCanStart();
+    this.transitionTo("PROCESSING");
+    this.abortController = new AbortController();
+    this.startedAt = Date.now();
+    const allRawRecords = await this.parseRecords();
+    this.totalRecords = allRawRecords.length;
+    this.batches = this.splitIntoBatches(allRawRecords);
+    await this.saveState();
+    this.eventBus.emit({
+      type: "import:started",
+      jobId: this.jobId,
+      totalRecords: this.totalRecords,
+      totalBatches: this.batches.length,
+      timestamp: Date.now()
+    });
+    try {
+      for (let i = 0; i < this.batches.length; i++) {
+        if (this.abortController.signal.aborted) break;
+        await this.checkPause();
+        const batch = this.batches[i];
+        if (!batch) break;
+        await this.processBatch(batch, processor);
+      }
+      if (!this.abortController.signal.aborted && this.status !== "ABORTED") {
+        this.transitionTo("COMPLETED");
+        const summary = this.buildSummary();
+        this.eventBus.emit({
+          type: "import:completed",
+          jobId: this.jobId,
+          summary,
+          timestamp: Date.now()
+        });
+      }
+    } catch (error) {
+      if (this.status !== "ABORTED") {
+        this.transitionTo("FAILED");
+        this.eventBus.emit({
+          type: "import:failed",
+          jobId: this.jobId,
+          error: error instanceof Error ? error.message : String(error),
+          timestamp: Date.now()
+        });
+      }
+    }
+    await this.saveState();
+  }
+  async pause() {
+    if (this.status !== "PROCESSING") {
+      throw new Error(`Cannot pause import from status '${this.status}'`);
+    }
+    this.transitionTo("PAUSED");
+    this.pausePromise = this.createPausePromise();
+    const progress = this.buildProgress();
+    this.eventBus.emit({
+      type: "import:paused",
+      jobId: this.jobId,
+      progress,
+      timestamp: Date.now()
+    });
+    await this.saveState();
+  }
+  resume() {
+    if (this.status === "ABORTED") {
+      throw new Error("Cannot resume an aborted import");
+    }
+    if (this.status !== "PAUSED") {
+      throw new Error(`Cannot resume import from status '${this.status}'`);
+    }
+    this.transitionTo("PROCESSING");
+    if (this.pausePromise) {
+      this.pausePromise.resolve();
+      this.pausePromise = null;
+    }
+  }
+  async abort() {
+    if (this.status !== "PROCESSING" && this.status !== "PAUSED") {
+      throw new Error(`Cannot abort import from status '${this.status}'`);
+    }
+    this.transitionTo("ABORTED");
+    this.abortController?.abort();
+    if (this.pausePromise) {
+      this.pausePromise.resolve();
+      this.pausePromise = null;
+    }
+    const progress = this.buildProgress();
+    this.eventBus.emit({
+      type: "import:aborted",
+      jobId: this.jobId,
+      progress,
+      timestamp: Date.now()
+    });
+    await this.saveState();
+  }
+  getStatus() {
+    return {
+      state: this.status,
+      progress: this.buildProgress(),
+      batches: this.batches
+    };
+  }
+  getFailedRecords() {
+    return this.allRecords.filter((r) => r.status === "failed" || r.status === "invalid");
+  }
+  getPendingRecords() {
+    return this.allRecords.filter((r) => r.status === "pending" || r.status === "valid");
+  }
+  getJobId() {
+    return this.jobId;
+  }
+  // --- Private methods ---
+  async processBatch(batch, processor) {
+    const batchIndex = this.batches.indexOf(batch);
+    this.updateBatchStatus(batch.id, "PROCESSING");
+    this.eventBus.emit({
+      type: "batch:started",
+      jobId: this.jobId,
+      batchId: batch.id,
+      batchIndex,
+      recordCount: batch.records.length,
+      timestamp: Date.now()
+    });
+    let processedCount = 0;
+    let failedCount = 0;
+    for (const record of batch.records) {
+      if (this.abortController?.signal.aborted) break;
+      await this.checkPause();
+      const transformed = this.validator.applyTransforms(record.raw);
+      const validation = this.validator.validate(transformed);
+      if (!validation.isValid) {
+        const invalidRecord = markRecordInvalid(record, validation.errors);
+        this.updateRecord(record.index, invalidRecord);
+        failedCount++;
+        this.eventBus.emit({
+          type: "record:failed",
+          jobId: this.jobId,
+          batchId: batch.id,
+          recordIndex: record.index,
+          error: validation.errors.map((e) => e.message).join("; "),
+          record: invalidRecord,
+          timestamp: Date.now()
+        });
+        if (!this.config.continueOnError) {
+          throw new Error(`Validation failed for record ${String(record.index)}`);
+        }
+        continue;
+      }
+      const validRecord = markRecordValid(record, transformed);
+      try {
+        const context = {
+          jobId: this.jobId,
+          batchId: batch.id,
+          batchIndex,
+          recordIndex: record.index,
+          totalRecords: this.totalRecords,
+          signal: this.abortController?.signal ?? new AbortController().signal
+        };
+        await processor(validRecord.parsed, context);
+        const processed = markRecordProcessed(validRecord);
+        this.updateRecord(record.index, processed);
+        processedCount++;
+        this.eventBus.emit({
+          type: "record:processed",
+          jobId: this.jobId,
+          batchId: batch.id,
+          recordIndex: record.index,
+          timestamp: Date.now()
+        });
+      } catch (error) {
+        const failedRecord = markRecordFailed(validRecord, error instanceof Error ? error.message : String(error));
+        this.updateRecord(record.index, failedRecord);
+        failedCount++;
+        this.eventBus.emit({
+          type: "record:failed",
+          jobId: this.jobId,
+          batchId: batch.id,
+          recordIndex: record.index,
+          error: error instanceof Error ? error.message : String(error),
+          record: failedRecord,
+          timestamp: Date.now()
+        });
+        if (!this.config.continueOnError) {
+          throw error;
+        }
+      }
+    }
+    this.updateBatchStatus(batch.id, "COMPLETED", processedCount, failedCount);
+    this.eventBus.emit({
+      type: "batch:completed",
+      jobId: this.jobId,
+      batchId: batch.id,
+      batchIndex,
+      processedCount,
+      failedCount,
+      totalCount: batch.records.length,
+      timestamp: Date.now()
+    });
+    this.emitProgress();
+  }
+  async parseRecords(maxRecords) {
+    const source = this.source;
+    const parser = this.parser;
+    if (!source || !parser) {
+      throw new Error("Source and parser must be configured. Call .from(source, parser) first.");
+    }
+    const records = [];
+    let index = 0;
+    for await (const chunk of source.read()) {
+      for await (const raw of parser.parse(chunk)) {
+        if (maxRecords !== void 0 && records.length >= maxRecords) {
+          return records;
+        }
+        records.push(createPendingRecord(index, raw));
+        index++;
+      }
+    }
+    return records;
+  }
+  splitIntoBatches(records) {
+    const batches = [];
+    const { batchSize } = this.config;
+    for (let i = 0; i < records.length; i += batchSize) {
+      const batchRecords = records.slice(i, i + batchSize);
+      const batchId = crypto.randomUUID();
+      batches.push(createBatch(batchId, batches.length, batchRecords));
+    }
+    this.allRecords = [...records];
+    return batches;
+  }
+  updateRecord(index, record) {
+    const pos = this.allRecords.findIndex((r) => r.index === index);
+    if (pos >= 0) {
+      this.allRecords[pos] = record;
+    }
+  }
+  updateBatchStatus(batchId, status, processedCount, failedCount) {
+    this.batches = this.batches.map(
+      (b) => b.id === batchId ? {
+        ...b,
+        status,
+        processedCount: processedCount ?? b.processedCount,
+        failedCount: failedCount ?? b.failedCount
+      } : b
+    );
+  }
+  transitionTo(newStatus) {
+    if (!canTransition(this.status, newStatus)) {
+      throw new Error(`Invalid state transition: ${this.status} \u2192 ${newStatus}`);
+    }
+    this.status = newStatus;
+  }
+  buildProgress() {
+    const processed = this.allRecords.filter((r) => r.status === "processed").length;
+    const failed = this.allRecords.filter((r) => r.status === "failed" || r.status === "invalid").length;
+    const pending = this.totalRecords - processed - failed;
+    const completedBatches = this.batches.filter((b) => b.status === "COMPLETED").length;
+    const elapsed = this.startedAt ? Date.now() - this.startedAt : 0;
+    return {
+      totalRecords: this.totalRecords,
+      processedRecords: processed,
+      failedRecords: failed,
+      pendingRecords: pending,
+      percentage: this.totalRecords > 0 ? Math.round(processed / this.totalRecords * 100) : 0,
+      currentBatch: completedBatches,
+      totalBatches: this.batches.length,
+      elapsedMs: elapsed
+    };
+  }
+  buildSummary() {
+    const processed = this.allRecords.filter((r) => r.status === "processed").length;
+    const failed = this.allRecords.filter((r) => r.status === "failed" || r.status === "invalid").length;
+    const skipped = this.totalRecords - processed - failed;
+    const elapsed = this.startedAt ? Date.now() - this.startedAt : 0;
+    return { total: this.totalRecords, processed, failed, skipped, elapsedMs: elapsed };
+  }
+  emitProgress() {
+    this.eventBus.emit({
+      type: "import:progress",
+      jobId: this.jobId,
+      progress: this.buildProgress(),
+      timestamp: Date.now()
+    });
+  }
+  async saveState() {
+    const state = {
+      id: this.jobId,
+      config: {
+        schema: this.config.schema,
+        batchSize: this.config.batchSize,
+        continueOnError: this.config.continueOnError
+      },
+      status: this.status,
+      batches: this.batches,
+      totalRecords: this.totalRecords,
+      startedAt: this.startedAt
+    };
+    await this.stateStore.saveJobState(state);
+  }
+  async checkPause() {
+    if (this.pausePromise) {
+      await this.pausePromise.promise;
+    }
+  }
+  createPausePromise() {
+    let resolveRef;
+    const promise = new Promise((resolve) => {
+      resolveRef = resolve;
+    });
+    return { resolve: resolveRef, promise };
+  }
+  assertSourceConfigured() {
+    if (!this.source || !this.parser) {
+      throw new Error("Source and parser must be configured. Call .from(source, parser) first.");
+    }
+  }
+  assertCanStart() {
+    if (this.status !== "PREVIEWED" && this.status !== "CREATED") {
+      throw new Error(`Cannot start import from status '${this.status}'`);
+    }
+  }
+};
+
+// src/domain/model/BatchStatus.ts
+var BatchStatus = {
+  PENDING: "PENDING",
+  PROCESSING: "PROCESSING",
+  PAUSED: "PAUSED",
+  COMPLETED: "COMPLETED",
+  FAILED: "FAILED"
+};
+
+// src/infrastructure/parsers/CsvParser.ts
+import Papa from "papaparse";
+var CsvParser = class {
+  constructor(options) {
+    this.options = {
+      delimiter: options?.delimiter,
+      encoding: options?.encoding ?? "utf-8",
+      hasHeader: options?.hasHeader ?? true
+    };
+  }
+  *parse(data) {
+    const content = typeof data === "string" ? data : data.toString("utf-8");
+    const result = Papa.parse(content, {
+      header: this.options.hasHeader,
+      delimiter: this.options.delimiter || void 0,
+      skipEmptyLines: true,
+      dynamicTyping: false
+    });
+    for (const row of result.data) {
+      if (this.isEmptyRow(row)) continue;
+      yield row;
+    }
+  }
+  detect(sample) {
+    const content = typeof sample === "string" ? sample : sample.toString("utf-8");
+    const firstLines = content.split("\n").slice(0, 5).join("\n");
+    const delimiters = [",", ";", "	", "|"];
+    let bestDelimiter = ",";
+    let maxColumns = 0;
+    for (const delimiter of delimiters) {
+      const result = Papa.parse(firstLines, { delimiter, header: false });
+      const firstRow = result.data[0];
+      if (firstRow && firstRow.length > maxColumns) {
+        maxColumns = firstRow.length;
+        bestDelimiter = delimiter;
+      }
+    }
+    return {
+      delimiter: bestDelimiter,
+      encoding: "utf-8",
+      hasHeader: true
+    };
+  }
+  isEmptyRow(row) {
+    return Object.values(row).every((v) => v === null || v === void 0 || v === "");
+  }
+};
+
+// src/infrastructure/sources/BufferSource.ts
+var BufferSource = class {
+  constructor(data, metadata) {
+    this.content = typeof data === "string" ? data : data.toString("utf-8");
+    this.meta = {
+      fileName: metadata?.fileName ?? "buffer-input",
+      fileSize: this.content.length,
+      mimeType: metadata?.mimeType ?? "text/plain"
+    };
+  }
+  async *read() {
+    yield await Promise.resolve(this.content);
+  }
+  sample(maxBytes) {
+    if (maxBytes && maxBytes < this.content.length) {
+      return Promise.resolve(this.content.slice(0, maxBytes));
+    }
+    return Promise.resolve(this.content);
+  }
+  metadata() {
+    return this.meta;
+  }
+};
+export {
+  BatchStatus,
+  BufferSource,
+  BulkImport,
+  CsvParser,
+  ImportStatus,
+  InMemoryStateStore
+};
+//# sourceMappingURL=index.js.map