@bulkimport/core 0.1.0

package/dist/index.d.ts

@@ -0,0 +1,335 @@
+type FieldType = 'string' | 'number' | 'boolean' | 'date' | 'email' | 'custom';
+interface ValidationFieldResult {
+    valid: boolean;
+    message?: string;
+}
+interface FieldDefinition {
+    readonly name: string;
+    readonly type: FieldType;
+    readonly required: boolean;
+    readonly pattern?: RegExp;
+    readonly customValidator?: (value: unknown) => ValidationFieldResult;
+    readonly transform?: (value: unknown) => unknown;
+    readonly defaultValue?: unknown;
+}
+
+interface SchemaDefinition {
+    readonly fields: readonly FieldDefinition[];
+    readonly strict?: boolean;
+    readonly skipEmptyRows?: boolean;
+}
+
+declare const ImportStatus: {
+    readonly CREATED: "CREATED";
+    readonly PREVIEWING: "PREVIEWING";
+    readonly PREVIEWED: "PREVIEWED";
+    readonly PROCESSING: "PROCESSING";
+    readonly PAUSED: "PAUSED";
+    readonly COMPLETED: "COMPLETED";
+    readonly ABORTED: "ABORTED";
+    readonly FAILED: "FAILED";
+};
+type ImportStatus = (typeof ImportStatus)[keyof typeof ImportStatus];
+
+declare const BatchStatus: {
+    readonly PENDING: "PENDING";
+    readonly PROCESSING: "PROCESSING";
+    readonly PAUSED: "PAUSED";
+    readonly COMPLETED: "COMPLETED";
+    readonly FAILED: "FAILED";
+};
+type BatchStatus = (typeof BatchStatus)[keyof typeof BatchStatus];
+
+type ValidationErrorCode = 'REQUIRED' | 'TYPE_MISMATCH' | 'PATTERN_MISMATCH' | 'CUSTOM_VALIDATION' | 'UNKNOWN_FIELD';
+interface ValidationError {
+    readonly field: string;
+    readonly message: string;
+    readonly code: ValidationErrorCode;
+    readonly value?: unknown;
+}
+interface ValidationResult {
+    readonly isValid: boolean;
+    readonly errors: readonly ValidationError[];
+}
+
+type RecordStatus = 'valid' | 'invalid' | 'processed' | 'failed' | 'pending';
+interface RawRecord {
+    readonly [key: string]: unknown;
+}
+interface ProcessedRecord {
+    readonly index: number;
+    readonly raw: RawRecord;
+    readonly parsed: RawRecord;
+    readonly status: RecordStatus;
+    readonly errors: readonly ValidationError[];
+    readonly processingError?: string;
+}
+
+interface Batch {
+    readonly id: string;
+    readonly index: number;
+    readonly status: BatchStatus;
+    readonly records: readonly ProcessedRecord[];
+    readonly processedCount: number;
+    readonly failedCount: number;
+}
+
+interface ImportJobConfig {
+    readonly id?: string;
+    readonly schema: SchemaDefinition;
+    readonly batchSize: number;
+    readonly maxConcurrentBatches?: number;
+    readonly continueOnError?: boolean;
+}
+interface ImportJobState {
+    readonly id: string;
+    readonly config: ImportJobConfig;
+    readonly status: ImportStatus;
+    readonly batches: readonly Batch[];
+    readonly totalRecords: number;
+    readonly startedAt?: number;
+    readonly completedAt?: number;
+}
+interface ImportProgress {
+    readonly totalRecords: number;
+    readonly processedRecords: number;
+    readonly failedRecords: number;
+    readonly pendingRecords: number;
+    readonly percentage: number;
+    readonly currentBatch: number;
+    readonly totalBatches: number;
+    readonly elapsedMs: number;
+    readonly estimatedRemainingMs?: number;
+}
+interface ImportSummary {
+    readonly total: number;
+    readonly processed: number;
+    readonly failed: number;
+    readonly skipped: number;
+    readonly elapsedMs: number;
+}
+interface PreviewResult {
+    readonly validRecords: readonly ProcessedRecord[];
+    readonly invalidRecords: readonly ProcessedRecord[];
+    readonly totalSampled: number;
+    readonly columns: readonly string[];
+}
+
+interface ParserOptions {
+    readonly delimiter?: string;
+    readonly encoding?: string;
+    readonly hasHeader?: boolean;
+}
+interface SourceParser {
+    parse(data: string | Buffer): AsyncIterable<RawRecord> | Iterable<RawRecord>;
+    detect?(sample: string | Buffer): ParserOptions;
+}
+
+interface SourceMetadata {
+    readonly fileName?: string;
+    readonly fileSize?: number;
+    readonly mimeType?: string;
+}
+interface DataSource {
+    read(): AsyncIterable<string | Buffer>;
+    sample(maxBytes?: number): Promise<string | Buffer>;
+    metadata(): SourceMetadata;
+}
+
+interface BatchState {
+    readonly batchId: string;
+    readonly status: BatchStatus;
+    readonly processedCount: number;
+    readonly failedCount: number;
+}
+interface StateStore {
+    saveJobState(job: ImportJobState): Promise<void>;
+    getJobState(jobId: string): Promise<ImportJobState | null>;
+    updateBatchState(jobId: string, batchId: string, state: BatchState): Promise<void>;
+    saveProcessedRecord(jobId: string, batchId: string, record: ProcessedRecord): Promise<void>;
+    getFailedRecords(jobId: string): Promise<readonly ProcessedRecord[]>;
+    getPendingRecords(jobId: string): Promise<readonly ProcessedRecord[]>;
+    getProcessedRecords(jobId: string): Promise<readonly ProcessedRecord[]>;
+    getProgress(jobId: string): Promise<ImportProgress>;
+}
+
+interface ProcessingContext {
+    readonly jobId: string;
+    readonly batchId: string;
+    readonly batchIndex: number;
+    readonly recordIndex: number;
+    readonly totalRecords: number;
+    readonly signal: AbortSignal;
+}
+type RecordProcessorFn = (record: RawRecord, context: ProcessingContext) => Promise<void>;
+
+interface ImportStartedEvent {
+    readonly type: 'import:started';
+    readonly jobId: string;
+    readonly totalRecords: number;
+    readonly totalBatches: number;
+    readonly timestamp: number;
+}
+interface ImportCompletedEvent {
+    readonly type: 'import:completed';
+    readonly jobId: string;
+    readonly summary: ImportSummary;
+    readonly timestamp: number;
+}
+interface ImportPausedEvent {
+    readonly type: 'import:paused';
+    readonly jobId: string;
+    readonly progress: ImportProgress;
+    readonly timestamp: number;
+}
+interface ImportAbortedEvent {
+    readonly type: 'import:aborted';
+    readonly jobId: string;
+    readonly progress: ImportProgress;
+    readonly timestamp: number;
+}
+interface ImportFailedEvent {
+    readonly type: 'import:failed';
+    readonly jobId: string;
+    readonly error: string;
+    readonly timestamp: number;
+}
+interface ImportProgressEvent {
+    readonly type: 'import:progress';
+    readonly jobId: string;
+    readonly progress: ImportProgress;
+    readonly timestamp: number;
+}
+interface BatchStartedEvent {
+    readonly type: 'batch:started';
+    readonly jobId: string;
+    readonly batchId: string;
+    readonly batchIndex: number;
+    readonly recordCount: number;
+    readonly timestamp: number;
+}
+interface BatchCompletedEvent {
+    readonly type: 'batch:completed';
+    readonly jobId: string;
+    readonly batchId: string;
+    readonly batchIndex: number;
+    readonly processedCount: number;
+    readonly failedCount: number;
+    readonly totalCount: number;
+    readonly timestamp: number;
+}
+interface BatchFailedEvent {
+    readonly type: 'batch:failed';
+    readonly jobId: string;
+    readonly batchId: string;
+    readonly batchIndex: number;
+    readonly error: string;
+    readonly timestamp: number;
+}
+interface RecordProcessedEvent {
+    readonly type: 'record:processed';
+    readonly jobId: string;
+    readonly batchId: string;
+    readonly recordIndex: number;
+    readonly timestamp: number;
+}
+interface RecordFailedEvent {
+    readonly type: 'record:failed';
+    readonly jobId: string;
+    readonly batchId: string;
+    readonly recordIndex: number;
+    readonly error: string;
+    readonly record: ProcessedRecord;
+    readonly timestamp: number;
+}
+type DomainEvent = ImportStartedEvent | ImportCompletedEvent | ImportPausedEvent | ImportAbortedEvent | ImportFailedEvent | ImportProgressEvent | BatchStartedEvent | BatchCompletedEvent | BatchFailedEvent | RecordProcessedEvent | RecordFailedEvent;
+type EventType = DomainEvent['type'];
+type EventPayload<T extends EventType> = Extract<DomainEvent, {
+    type: T;
+}>;
+
+interface BulkImportConfig {
+    readonly schema: SchemaDefinition;
+    readonly batchSize?: number;
+    readonly maxConcurrentBatches?: number;
+    readonly continueOnError?: boolean;
+    readonly stateStore?: StateStore;
+}
+declare class BulkImport {
+    private readonly config;
+    private readonly validator;
+    private readonly eventBus;
+    private readonly stateStore;
+    private source;
+    private parser;
+    private jobId;
+    private status;
+    private batches;
+    private allRecords;
+    private totalRecords;
+    private startedAt?;
+    private abortController;
+    private pausePromise;
+    constructor(config: BulkImportConfig);
+    from(source: DataSource, parser: SourceParser): this;
+    on<T extends EventType>(type: T, handler: (event: EventPayload<T>) => void): this;
+    preview(maxRecords?: number): Promise<PreviewResult>;
+    start(processor: RecordProcessorFn): Promise<void>;
+    pause(): Promise<void>;
+    resume(): void;
+    abort(): Promise<void>;
+    getStatus(): {
+        state: ImportStatus;
+        progress: ImportProgress;
+        batches: readonly Batch[];
+    };
+    getFailedRecords(): readonly ProcessedRecord[];
+    getPendingRecords(): readonly ProcessedRecord[];
+    getJobId(): string;
+    private processBatch;
+    private parseRecords;
+    private splitIntoBatches;
+    private updateRecord;
+    private updateBatchStatus;
+    private transitionTo;
+    private buildProgress;
+    private buildSummary;
+    private emitProgress;
+    private saveState;
+    private checkPause;
+    private createPausePromise;
+    private assertSourceConfigured;
+    private assertCanStart;
+}
+
+declare class CsvParser implements SourceParser {
+    private readonly options;
+    constructor(options?: Partial<ParserOptions>);
+    parse(data: string | Buffer): Iterable<RawRecord>;
+    detect(sample: string | Buffer): ParserOptions;
+    private isEmptyRow;
+}
+
+declare class BufferSource implements DataSource {
+    private readonly content;
+    private readonly meta;
+    constructor(data: string | Buffer, metadata?: Partial<SourceMetadata>);
+    read(): AsyncIterable<string>;
+    sample(maxBytes?: number): Promise<string>;
+    metadata(): SourceMetadata;
+}
+
+declare class InMemoryStateStore implements StateStore {
+    private jobs;
+    private records;
+    saveJobState(job: ImportJobState): Promise<void>;
+    getJobState(jobId: string): Promise<ImportJobState | null>;
+    updateBatchState(jobId: string, batchId: string, state: BatchState): Promise<void>;
+    saveProcessedRecord(jobId: string, _batchId: string, record: ProcessedRecord): Promise<void>;
+    getFailedRecords(jobId: string): Promise<readonly ProcessedRecord[]>;
+    getPendingRecords(jobId: string): Promise<readonly ProcessedRecord[]>;
+    getProcessedRecords(jobId: string): Promise<readonly ProcessedRecord[]>;
+    getProgress(jobId: string): Promise<ImportProgress>;
+}
+
+export { type Batch, type BatchCompletedEvent, type BatchFailedEvent, type BatchStartedEvent, BatchStatus, BufferSource, BulkImport, type BulkImportConfig, CsvParser, type DataSource, type DomainEvent, type EventPayload, type EventType, type FieldDefinition, type FieldType, type ImportAbortedEvent, type ImportCompletedEvent, type ImportFailedEvent, type ImportJobConfig, type ImportJobState, type ImportPausedEvent, type ImportProgress, type ImportProgressEvent, type ImportStartedEvent, ImportStatus, type ImportSummary, InMemoryStateStore, type ParserOptions, type PreviewResult, type ProcessedRecord, type ProcessingContext, type RawRecord, type RecordFailedEvent, type RecordProcessedEvent, type RecordProcessorFn, type RecordStatus, type SchemaDefinition, type SourceMetadata, type SourceParser, type StateStore, type ValidationError, type ValidationErrorCode, type ValidationFieldResult, type ValidationResult };