convert-buddy-js 0.12.0 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/unified-stream.js
CHANGED
|
@@ -8,6 +8,7 @@ class StreamControllerImpl {
|
|
|
8
8
|
pauseResolver = null;
|
|
9
9
|
recordBuffer = [];
|
|
10
10
|
input;
|
|
11
|
+
finalStats = null;
|
|
11
12
|
constructor(input, options) {
|
|
12
13
|
this.input = input;
|
|
13
14
|
this.options = {
|
|
@@ -40,7 +41,7 @@ class StreamControllerImpl {
|
|
|
40
41
|
this.resume();
|
|
41
42
|
}
|
|
42
43
|
stats() {
|
|
43
|
-
return this.buddy?.stats() ?? null;
|
|
44
|
+
return this.finalStats ?? this.buddy?.stats() ?? null;
|
|
44
45
|
}
|
|
45
46
|
get recordCount() {
|
|
46
47
|
return this._recordCount;
|
|
@@ -291,6 +292,9 @@ class StreamControllerImpl {
|
|
|
291
292
|
});
|
|
292
293
|
}
|
|
293
294
|
cleanup() {
|
|
295
|
+
if (this.buddy && !this.finalStats) {
|
|
296
|
+
this.finalStats = this.buddy.stats();
|
|
297
|
+
}
|
|
294
298
|
this.buddy = null;
|
|
295
299
|
this.pauseResolver = null;
|
|
296
300
|
this.recordBuffer = [];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/unified-stream.ts"],"sourcesContent":["/**\r\n * Unified Streaming API for ConvertBuddy\r\n * \r\n * This is the PRODUCTION-READY isomorphic streaming interface.\r\n * Works identically in browser and Node.js environments.\r\n * \r\n * Design Principles:\r\n * 1. Batch-based processing for high performance\r\n * 2. Full backpressure control (pause/resume/abort)\r\n * 3. Direct stats access through controller\r\n * 4. Zero-copy when possible (WASM-parsed records preferred)\r\n */\r\n\r\nimport { ConvertBuddy } from \"./index.js\";\r\nimport type { Stats, Format, ConvertBuddyOptions } from \"./index.js\";\r\n\r\n// ============================================================================\r\n// Types\r\n// ============================================================================\r\n\r\n/**\r\n * Unified input type - works in both Node.js and Browser\r\n */\r\nexport type StreamInput = \r\n | string // URL or raw data\r\n | Uint8Array // Binary data\r\n | ArrayBuffer // Binary data\r\n | ReadableStream<Uint8Array> // Web Streams API (both environments)\r\n | AsyncIterable<Uint8Array> // Async iterator\r\n | Blob // Browser File/Blob\r\n | NodeReadableCompat; // Node.js Readable (if available)\r\n\r\n/**\r\n * Minimal interface for Node.js Readable stream compatibility\r\n * We use duck-typing to avoid importing node:stream in browser builds\r\n */\r\ninterface NodeReadableCompat {\r\n read?: () => unknown;\r\n on?: (event: string, listener: (...args: unknown[]) => void) => unknown;\r\n [Symbol.asyncIterator]?: () => AsyncIterator<unknown>;\r\n}\r\n\r\n/**\r\n * Options for stream processing\r\n */\r\nexport interface StreamOptions extends Omit<ConvertBuddyOptions, 'onProgress'> {\r\n /**\r\n * Called with batches of parsed records. If async, backpressure is automatically applied.\r\n * Return `false` to pause processing (call controller.resume() to continue).\r\n * \r\n * @param records - Array of records (batch size controlled by recordBatchSize)\r\n * @param stats - Current conversion statistics\r\n */\r\n onRecords?: (records: unknown[], stats: Stats) => void | boolean | Promise<void | boolean>;\r\n\r\n /**\r\n * Called with raw output bytes before parsing into records.\r\n * Useful for writing directly to a file/stream while also processing records.\r\n */\r\n onData?: (bytes: Uint8Array) => void | Promise<void>;\r\n\r\n /**\r\n * Called periodically with progress stats.\r\n * Frequency controlled by `progressIntervalRecords`.\r\n */\r\n onProgress?: (stats: Stats, recordCount: number) => void;\r\n\r\n /**\r\n * Trigger onProgress callback every N records. Default: 1000\r\n */\r\n progressIntervalRecords?: number;\r\n\r\n /**\r\n * Number of records per batch when using onRecords callback or iterator.\r\n * Lower = more responsive, Higher = better throughput\r\n * Default: 100\r\n */\r\n recordBatchSize?: number;\r\n}\r\n\r\n/**\r\n * Stream controller returned by ConvertBuddy.stream()\r\n * Provides pause/resume/abort controls and stats access\r\n */\r\nexport interface StreamController {\r\n /** Pause the stream processing */\r\n pause(): void;\r\n \r\n /** Resume the stream processing */\r\n resume(): void;\r\n \r\n /** Abort the stream processing */\r\n abort(): void;\r\n \r\n /** Get current statistics */\r\n stats(): Stats | null;\r\n \r\n /** Get current record count */\r\n recordCount: number;\r\n \r\n /** Check if currently paused */\r\n isPaused: boolean;\r\n \r\n /** Check if aborted */\r\n isAborted: boolean;\r\n \r\n /** Async iterator over record batches */\r\n [Symbol.asyncIterator](): AsyncIterator<unknown[]>;\r\n}\r\n\r\n/**\r\n * Result from processing a stream\r\n */\r\nexport interface StreamResult {\r\n stats: Stats;\r\n recordCount: number;\r\n aborted: boolean;\r\n}\r\n\r\n// ============================================================================\r\n// StreamControllerImpl Class\r\n// ============================================================================\r\n\r\nclass StreamControllerImpl implements StreamController {\r\n private options: StreamOptions;\r\n private buddy: ConvertBuddy | null = null;\r\n private _aborted = false;\r\n private _paused = false;\r\n private _recordCount = 0;\r\n private pauseResolver: (() => void) | null = null;\r\n private recordBuffer: unknown[] = [];\r\n private input: StreamInput;\r\n\r\n constructor(input: StreamInput, options: StreamOptions) {\r\n this.input = input;\r\n this.options = {\r\n outputFormat: 'ndjson',\r\n progressIntervalRecords: 1000,\r\n recordBatchSize: 100,\r\n profile: true, // Always track stats\r\n ...options,\r\n };\r\n }\r\n\r\n // ==========================================================================\r\n // Public API: Control Methods\r\n // ==========================================================================\r\n\r\n pause(): void {\r\n this._paused = true;\r\n this.buddy?.pause();\r\n }\r\n\r\n resume(): void {\r\n this._paused = false;\r\n this.buddy?.resume();\r\n if (this.pauseResolver) {\r\n this.pauseResolver();\r\n this.pauseResolver = null;\r\n }\r\n }\r\n\r\n abort(): void {\r\n this._aborted = true;\r\n this.buddy?.abort();\r\n this.resume(); // Unblock any paused iteration\r\n }\r\n\r\n stats(): Stats | null {\r\n return this.buddy?.stats() ?? null;\r\n }\r\n\r\n get recordCount(): number {\r\n return this._recordCount;\r\n }\r\n\r\n get isPaused(): boolean {\r\n return this._paused;\r\n }\r\n\r\n get isAborted(): boolean {\r\n return this._aborted;\r\n }\r\n\r\n // ==========================================================================\r\n // Public API: Async Iterator\r\n // ==========================================================================\r\n\r\n async *[Symbol.asyncIterator](): AsyncIterator<unknown[]> {\r\n try {\r\n this.buddy = await ConvertBuddy.create(this.options as ConvertBuddyOptions);\r\n const iterator = this.toAsyncIterator(this.input);\r\n const batchSize = this.options.recordBatchSize || 100;\r\n\r\n for await (const chunk of iterator) {\r\n if (this._aborted) return;\r\n await this.waitIfPaused();\r\n\r\n const records = this.processChunkForRecords(chunk);\r\n this.recordBuffer.push(...records);\r\n\r\n // Yield batches\r\n while (this.recordBuffer.length >= batchSize) {\r\n const batch = this.recordBuffer.splice(0, batchSize);\r\n this._recordCount += batch.length;\r\n this.maybeEmitProgress();\r\n yield batch;\r\n }\r\n }\r\n\r\n // Finalize - process any remaining records\r\n if (!this._aborted && this.buddy) {\r\n const finalOutput = this.buddy.finish();\r\n const finalRecords = this.parseRecordsFromOutput(finalOutput);\r\n this.recordBuffer.push(...finalRecords);\r\n\r\n // Yield remaining batches\r\n while (this.recordBuffer.length > 0) {\r\n const batch = this.recordBuffer.splice(0, batchSize);\r\n this._recordCount += batch.length;\r\n this.maybeEmitProgress();\r\n yield batch;\r\n }\r\n }\r\n } finally {\r\n this.cleanup();\r\n }\r\n }\r\n\r\n // ==========================================================================\r\n // Internal: Process with callbacks\r\n // ==========================================================================\r\n\r\n async processInternal(): Promise<StreamResult> {\r\n try {\r\n this.buddy = await ConvertBuddy.create(this.options as ConvertBuddyOptions);\r\n const iterator = this.toAsyncIterator(this.input);\r\n const batchSize = this.options.recordBatchSize || 100;\r\n\r\n for await (const chunk of iterator) {\r\n if (this._aborted) break;\r\n await this.waitIfPaused();\r\n\r\n await this.processChunk(chunk);\r\n\r\n // Process buffered records in batches\r\n while (this.recordBuffer.length >= batchSize) {\r\n const batch = this.recordBuffer.splice(0, batchSize);\r\n await this.emitRecordBatch(batch);\r\n }\r\n }\r\n\r\n // Finalize\r\n if (!this._aborted && this.buddy) {\r\n const finalOutput = this.buddy.finish();\r\n if (finalOutput.length > 0) {\r\n await this.emitOutput(finalOutput);\r\n }\r\n\r\n // Emit remaining records\r\n if (this.recordBuffer.length > 0) {\r\n await this.emitRecordBatch(this.recordBuffer.splice(0));\r\n }\r\n }\r\n\r\n return {\r\n stats: this.buddy?.stats() ?? this.emptyStats(),\r\n recordCount: this._recordCount,\r\n aborted: this._aborted,\r\n };\r\n } finally {\r\n this.cleanup();\r\n }\r\n }\r\n\r\n // ==========================================================================\r\n // Private: Input Normalization\r\n // ==========================================================================\r\n\r\n private async *toAsyncIterator(input: StreamInput): AsyncGenerator<Uint8Array> {\r\n // String: URL or raw data\r\n if (typeof input === 'string') {\r\n if (input.startsWith('http://') || input.startsWith('https://')) {\r\n const response = await fetch(input);\r\n if (!response.ok) throw new Error(`Fetch failed: ${response.status} ${response.statusText}`);\r\n if (!response.body) throw new Error('Response body is null');\r\n yield* this.fromReadableStream(response.body);\r\n return;\r\n }\r\n yield new TextEncoder().encode(input);\r\n return;\r\n }\r\n\r\n if (input instanceof Uint8Array) {\r\n yield input;\r\n return;\r\n }\r\n\r\n if (input instanceof ArrayBuffer) {\r\n yield new Uint8Array(input);\r\n return;\r\n }\r\n\r\n if (this.isReadableStream(input)) {\r\n yield* this.fromReadableStream(input);\r\n return;\r\n }\r\n\r\n if (typeof Blob !== 'undefined' && input instanceof Blob) {\r\n const stream = input.stream() as ReadableStream<Uint8Array>;\r\n yield* this.fromReadableStream(stream);\r\n return;\r\n }\r\n\r\n if (this.isAsyncIterable(input)) {\r\n for await (const chunk of input) {\r\n yield this.toUint8Array(chunk);\r\n }\r\n return;\r\n }\r\n\r\n if (this.isNodeReadable(input)) {\r\n yield* this.fromNodeReadable(input);\r\n return;\r\n }\r\n\r\n throw new Error('Unsupported input type');\r\n }\r\n\r\n private isReadableStream(value: unknown): value is ReadableStream<Uint8Array> {\r\n return typeof ReadableStream !== 'undefined' && value instanceof ReadableStream;\r\n }\r\n\r\n private isAsyncIterable(value: unknown): value is AsyncIterable<unknown> {\r\n return value != null && typeof (value as AsyncIterable<unknown>)[Symbol.asyncIterator] === 'function';\r\n }\r\n\r\n private isNodeReadable(value: unknown): value is NodeReadableCompat {\r\n const v = value as NodeReadableCompat;\r\n return v != null && typeof v.on === 'function' && typeof v.read === 'function';\r\n }\r\n\r\n private async *fromReadableStream(stream: ReadableStream<Uint8Array>): AsyncGenerator<Uint8Array> {\r\n const reader = stream.getReader();\r\n try {\r\n while (true) {\r\n const { done, value } = await reader.read();\r\n if (done) break;\r\n yield value;\r\n }\r\n } finally {\r\n reader.releaseLock();\r\n }\r\n }\r\n\r\n private async *fromNodeReadable(stream: NodeReadableCompat): AsyncGenerator<Uint8Array> {\r\n if (stream[Symbol.asyncIterator]) {\r\n const iter = stream[Symbol.asyncIterator]!();\r\n for await (const chunk of { [Symbol.asyncIterator]: () => iter }) {\r\n yield this.toUint8Array(chunk);\r\n }\r\n } else {\r\n throw new Error('Node.js Readable stream must be async iterable');\r\n }\r\n }\r\n\r\n private toUint8Array(chunk: unknown): Uint8Array {\r\n if (chunk instanceof Uint8Array) return chunk;\r\n if (typeof Buffer !== 'undefined' && Buffer.isBuffer(chunk)) return new Uint8Array(chunk);\r\n if (typeof chunk === 'string') return new TextEncoder().encode(chunk);\r\n if (chunk instanceof ArrayBuffer) return new Uint8Array(chunk);\r\n throw new Error('Unexpected chunk type');\r\n }\r\n\r\n // ==========================================================================\r\n // Private: Processing\r\n // ==========================================================================\r\n\r\n private async processChunk(chunk: Uint8Array): Promise<void> {\r\n if (!this.buddy) return;\r\n\r\n const needsRecords = !!this.options.onRecords;\r\n const result = this.buddy.pushWithRecords(chunk, needsRecords);\r\n\r\n if (typeof result === 'object' && 'output' in result) {\r\n await this.emitOutput(result.output, result.records);\r\n } else if (result.length > 0) {\r\n await this.emitOutput(result);\r\n }\r\n }\r\n\r\n private processChunkForRecords(chunk: Uint8Array): unknown[] {\r\n if (!this.buddy) return [];\r\n\r\n const result = this.buddy.pushWithRecords(chunk, true);\r\n \r\n if (typeof result === 'object' && 'output' in result) {\r\n if (result.records && result.records.length > 0) {\r\n return result.records;\r\n }\r\n return this.parseRecordsFromOutput(result.output);\r\n }\r\n \r\n return this.parseRecordsFromOutput(result);\r\n }\r\n\r\n private async emitOutput(output: Uint8Array, wasmRecords?: unknown[]): Promise<void> {\r\n // Emit raw data\r\n if (this.options.onData) {\r\n const maybePromise = this.options.onData(output);\r\n if (maybePromise instanceof Promise) await maybePromise;\r\n }\r\n\r\n // Buffer records for batching\r\n if (this.options.onRecords) {\r\n const records = wasmRecords && wasmRecords.length > 0 \r\n ? wasmRecords \r\n : this.parseRecordsFromOutput(output);\r\n \r\n this.recordBuffer.push(...records);\r\n }\r\n }\r\n\r\n private async emitRecordBatch(batch: unknown[]): Promise<void> {\r\n if (!this.options.onRecords || batch.length === 0) return;\r\n\r\n const stats = this.buddy?.stats() ?? this.emptyStats();\r\n const result = this.options.onRecords(batch, stats);\r\n\r\n this._recordCount += batch.length;\r\n this.maybeEmitProgress();\r\n\r\n if (result instanceof Promise) {\r\n const shouldContinue = await result;\r\n if (shouldContinue === false) this.pause();\r\n } else if (result === false) {\r\n this.pause();\r\n }\r\n }\r\n\r\n /**\r\n * Parse records from output bytes - FALLBACK ONLY\r\n * Now WASM returns pre-parsed records for all formats!\r\n */\r\n private parseRecordsFromOutput(output: Uint8Array): unknown[] {\r\n if (output.length === 0) return [];\r\n\r\n const format = this.options.outputFormat || 'ndjson';\r\n const text = new TextDecoder().decode(output);\r\n\r\n try {\r\n switch (format) {\r\n case 'ndjson':\r\n return text\r\n .split('\\n')\r\n .filter(line => line.trim().length > 0)\r\n .map(line => JSON.parse(line));\r\n \r\n case 'json':\r\n const parsed = JSON.parse(text);\r\n return Array.isArray(parsed) ? parsed : [parsed];\r\n \r\n case 'csv':\r\n case 'xml':\r\n // These should now be parsed by WASM!\r\n // If we reach here, WASM didn't return records\r\n console.warn('[StreamController] WASM did not return parsed records for', format);\r\n return [];\r\n \r\n default:\r\n return [];\r\n }\r\n } catch {\r\n return []; // Partial data - normal during streaming\r\n }\r\n }\r\n\r\n private maybeEmitProgress(): void {\r\n const interval = this.options.progressIntervalRecords || 1000;\r\n if (this.options.onProgress && this._recordCount % interval === 0) {\r\n const stats = this.buddy?.stats() ?? this.emptyStats();\r\n this.options.onProgress(stats, this._recordCount);\r\n }\r\n }\r\n\r\n private async waitIfPaused(): Promise<void> {\r\n if (!this._paused) return;\r\n await new Promise<void>(resolve => {\r\n this.pauseResolver = resolve;\r\n });\r\n }\r\n\r\n private cleanup(): void {\r\n this.buddy = null;\r\n this.pauseResolver = null;\r\n this.recordBuffer = [];\r\n }\r\n\r\n private emptyStats(): Stats {\r\n return {\r\n bytesIn: 0,\r\n bytesOut: 0,\r\n chunksIn: 0,\r\n recordsProcessed: 0,\r\n parseTimeMs: 0,\r\n transformTimeMs: 0,\r\n writeTimeMs: 0,\r\n maxBufferSize: 0,\r\n currentPartialSize: 0,\r\n throughputMbPerSec: 0,\r\n };\r\n }\r\n}\r\n\r\n// ============================================================================\r\n// Convenience Functions\r\n// ============================================================================\r\n\r\n/**\r\n * Process a stream with callbacks\r\n * Returns controller for pause/resume/stats access\r\n * \r\n * @example\r\n * ```ts\r\n * const result = await processStream(file, {\r\n * inputFormat: 'csv',\r\n * outputFormat: 'json',\r\n * onRecords: (records, stats) => {\r\n * console.log(`Batch of ${records.length} records`);\r\n * console.log(`Throughput: ${stats.throughputMbPerSec} MB/s`);\r\n * },\r\n * });\r\n * console.log('Total records:', result.recordCount);\r\n * ```\r\n */\r\nexport async function processStream(\r\n input: StreamInput,\r\n options: StreamOptions\r\n): Promise<StreamResult> {\r\n const controller = new StreamControllerImpl(input, options);\r\n return controller.processInternal();\r\n}\r\n\r\n/**\r\n * Create a stream controller for manual control\r\n * \r\n * @example\r\n * ```ts\r\n * const controller = createStreamController(file, {\r\n * inputFormat: 'csv',\r\n * outputFormat: 'json',\r\n * });\r\n * \r\n * // Process with full control\r\n * for await (const batch of controller) {\r\n * console.log(`Processing ${batch.length} records`);\r\n * console.log('Stats:', controller.stats());\r\n * \r\n * if (needsSlowdown) {\r\n * controller.pause();\r\n * await sleep(1000);\r\n * controller.resume();\r\n * }\r\n * }\r\n * ```\r\n */\r\nexport function createStreamController(\r\n input: StreamInput,\r\n options: StreamOptions\r\n): StreamController {\r\n return new StreamControllerImpl(input, options);\r\n}\r\n\r\n/**\r\n * Helper: Create an async iterator over record batches\r\n * \r\n * @example\r\n * ```ts\r\n * for await (const batch of streamRecords(file, {\r\n * inputFormat: 'csv',\r\n * outputFormat: 'json',\r\n * recordBatchSize: 50, // 50 records per batch\r\n * })) {\r\n * await Promise.all(batch.map(r => saveRecord(r)));\r\n * }\r\n * ```\r\n */\r\nexport function streamRecords(\r\n input: StreamInput,\r\n options: StreamOptions\r\n): AsyncIterableIterator<unknown[]> {\r\n const controller = new StreamControllerImpl(input, options);\r\n return controller[Symbol.asyncIterator]() as AsyncIterableIterator<unknown[]>;\r\n}\r\n"],"mappings":"AAaA,SAAS,oBAAoB;AA8G7B,MAAM,qBAAiD;AAAA,EAC7C;AAAA,EACA,QAA6B;AAAA,EAC7B,WAAW;AAAA,EACX,UAAU;AAAA,EACV,eAAe;AAAA,EACf,gBAAqC;AAAA,EACrC,eAA0B,CAAC;AAAA,EAC3B;AAAA,EAER,YAAY,OAAoB,SAAwB;AACtD,SAAK,QAAQ;AACb,SAAK,UAAU;AAAA,MACb,cAAc;AAAA,MACd,yBAAyB;AAAA,MACzB,iBAAiB;AAAA,MACjB,SAAS;AAAA;AAAA,MACT,GAAG;AAAA,IACL;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAMA,QAAc;AACZ,SAAK,UAAU;AACf,SAAK,OAAO,MAAM;AAAA,EACpB;AAAA,EAEA,SAAe;AACb,SAAK,UAAU;AACf,SAAK,OAAO,OAAO;AACnB,QAAI,KAAK,eAAe;AACtB,WAAK,cAAc;AACnB,WAAK,gBAAgB;AAAA,IACvB;AAAA,EACF;AAAA,EAEA,QAAc;AACZ,SAAK,WAAW;AAChB,SAAK,OAAO,MAAM;AAClB,SAAK,OAAO;AAAA,EACd;AAAA,EAEA,QAAsB;AACpB,WAAO,KAAK,OAAO,MAAM,KAAK;AAAA,EAChC;AAAA,EAEA,IAAI,cAAsB;AACxB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,WAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,YAAqB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA,EAMA,QAAQ,OAAO,aAAa,IAA8B;AACxD,QAAI;AACF,WAAK,QAAQ,MAAM,aAAa,OAAO,KAAK,OAA8B;AAC1E,YAAM,WAAW,KAAK,gBAAgB,KAAK,KAAK;AAChD,YAAM,YAAY,KAAK,QAAQ,mBAAmB;AAElD,uBAAiB,SAAS,UAAU;AAClC,YAAI,KAAK,SAAU;AACnB,cAAM,KAAK,aAAa;AAExB,cAAM,UAAU,KAAK,uBAAuB,KAAK;AACjD,aAAK,aAAa,KAAK,GAAG,OAAO;AAGjC,eAAO,KAAK,aAAa,UAAU,WAAW;AAC5C,gBAAM,QAAQ,KAAK,aAAa,OAAO,GAAG,SAAS;AACnD,eAAK,gBAAgB,MAAM;AAC3B,eAAK,kBAAkB;AACvB,gBAAM;AAAA,QACR;AAAA,MACF;AAGA,UAAI,CAAC,KAAK,YAAY,KAAK,OAAO;AAChC,cAAM,cAAc,KAAK,MAAM,OAAO;AACtC,cAAM,eAAe,KAAK,uBAAuB,WAAW;AAC5D,aAAK,aAAa,KAAK,GAAG,YAAY;AAGtC,eAAO,KAAK,aAAa,SAAS,GAAG;AACnC,gBAAM,QAAQ,KAAK,aAAa,OAAO,GAAG,SAAS;AACnD,eAAK,gBAAgB,MAAM;AAC3B,eAAK,kBAAkB;AACvB,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF,UAAE;AACA,WAAK,QAAQ;AAAA,IACf;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,kBAAyC;AAC7C,QAAI;AACF,WAAK,QAAQ,MAAM,aAAa,OAAO,KAAK,OAA8B;AAC1E,YAAM,WAAW,KAAK,gBAAgB,KAAK,KAAK;AAChD,YAAM,YAAY,KAAK,QAAQ,mBAAmB;AAElD,uBAAiB,SAAS,UAAU;AAClC,YAAI,KAAK,SAAU;AACnB,cAAM,KAAK,aAAa;AAExB,cAAM,KAAK,aAAa,KAAK;AAG7B,eAAO,KAAK,aAAa,UAAU,WAAW;AAC5C,gBAAM,QAAQ,KAAK,aAAa,OAAO,GAAG,SAAS;AACnD,gBAAM,KAAK,gBAAgB,KAAK;AAAA,QAClC;AAAA,MACF;AAGA,UAAI,CAAC,KAAK,YAAY,KAAK,OAAO;AAChC,cAAM,cAAc,KAAK,MAAM,OAAO;AACtC,YAAI,YAAY,SAAS,GAAG;AAC1B,gBAAM,KAAK,WAAW,WAAW;AAAA,QACnC;AAGA,YAAI,KAAK,aAAa,SAAS,GAAG;AAChC,gBAAM,KAAK,gBAAgB,KAAK,aAAa,OAAO,CAAC,CAAC;AAAA,QACxD;AAAA,MACF;AAEA,aAAO;AAAA,QACL,OAAO,KAAK,OAAO,MAAM,KAAK,KAAK,WAAW;AAAA,QAC9C,aAAa,KAAK;AAAA,QAClB,SAAS,KAAK;AAAA,MAChB;AAAA,IACF,UAAE;AACA,WAAK,QAAQ;AAAA,IACf;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAMA,OAAe,gBAAgB,OAAgD;AAE7E,QAAI,OAAO,UAAU,UAAU;AAC7B,UAAI,MAAM,WAAW,SAAS,KAAK,MAAM,WAAW,UAAU,GAAG;AAC/D,cAAM,WAAW,MAAM,MAAM,KAAK;AAClC,YAAI,CAAC,SAAS,GAAI,OAAM,IAAI,MAAM,iBAAiB,SAAS,MAAM,IAAI,SAAS,UAAU,EAAE;AAC3F,YAAI,CAAC,SAAS,KAAM,OAAM,IAAI,MAAM,uBAAuB;AAC3D,eAAO,KAAK,mBAAmB,SAAS,IAAI;AAC5C;AAAA,MACF;AACA,YAAM,IAAI,YAAY,EAAE,OAAO,KAAK;AACpC;AAAA,IACF;AAEA,QAAI,iBAAiB,YAAY;AAC/B,YAAM;AACN;AAAA,IACF;AAEA,QAAI,iBAAiB,aAAa;AAChC,YAAM,IAAI,WAAW,KAAK;AAC1B;AAAA,IACF;AAEA,QAAI,KAAK,iBAAiB,KAAK,GAAG;AAChC,aAAO,KAAK,mBAAmB,KAAK;AACpC;AAAA,IACF;AAEA,QAAI,OAAO,SAAS,eAAe,iBAAiB,MAAM;AACxD,YAAM,SAAS,MAAM,OAAO;AAC5B,aAAO,KAAK,mBAAmB,MAAM;AACrC;AAAA,IACF;AAEA,QAAI,KAAK,gBAAgB,KAAK,GAAG;AAC/B,uBAAiB,SAAS,OAAO;AAC/B,cAAM,KAAK,aAAa,KAAK;AAAA,MAC/B;AACA;AAAA,IACF;AAEA,QAAI,KAAK,eAAe,KAAK,GAAG;AAC9B,aAAO,KAAK,iBAAiB,KAAK;AAClC;AAAA,IACF;AAEA,UAAM,IAAI,MAAM,wBAAwB;AAAA,EAC1C;AAAA,EAEQ,iBAAiB,OAAqD;AAC5E,WAAO,OAAO,mBAAmB,eAAe,iBAAiB;AAAA,EACnE;AAAA,EAEQ,gBAAgB,OAAiD;AACvE,WAAO,SAAS,QAAQ,OAAQ,MAAiC,OAAO,aAAa,MAAM;AAAA,EAC7F;AAAA,EAEQ,eAAe,OAA6C;AAClE,UAAM,IAAI;AACV,WAAO,KAAK,QAAQ,OAAO,EAAE,OAAO,cAAc,OAAO,EAAE,SAAS;AAAA,EACtE;AAAA,EAEA,OAAe,mBAAmB,QAAgE;AAChG,UAAM,SAAS,OAAO,UAAU;AAChC,QAAI;AACF,aAAO,MAAM;AACX,cAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,YAAI,KAAM;AACV,cAAM;AAAA,MACR;AAAA,IACF,UAAE;AACA,aAAO,YAAY;AAAA,IACrB;AAAA,EACF;AAAA,EAEA,OAAe,iBAAiB,QAAwD;AACtF,QAAI,OAAO,OAAO,aAAa,GAAG;AAChC,YAAM,OAAO,OAAO,OAAO,aAAa,EAAG;AAC3C,uBAAiB,SAAS,EAAE,CAAC,OAAO,aAAa,GAAG,MAAM,KAAK,GAAG;AAChE,cAAM,KAAK,aAAa,KAAK;AAAA,MAC/B;AAAA,IACF,OAAO;AACL,YAAM,IAAI,MAAM,gDAAgD;AAAA,IAClE;AAAA,EACF;AAAA,EAEQ,aAAa,OAA4B;AAC/C,QAAI,iBAAiB,WAAY,QAAO;AACxC,QAAI,OAAO,WAAW,eAAe,OAAO,SAAS,KAAK,EAAG,QAAO,IAAI,WAAW,KAAK;AACxF,QAAI,OAAO,UAAU,SAAU,QAAO,IAAI,YAAY,EAAE,OAAO,KAAK;AACpE,QAAI,iBAAiB,YAAa,QAAO,IAAI,WAAW,KAAK;AAC7D,UAAM,IAAI,MAAM,uBAAuB;AAAA,EACzC;AAAA;AAAA;AAAA;AAAA,EAMA,MAAc,aAAa,OAAkC;AAC3D,QAAI,CAAC,KAAK,MAAO;AAEjB,UAAM,eAAe,CAAC,CAAC,KAAK,QAAQ;AACpC,UAAM,SAAS,KAAK,MAAM,gBAAgB,OAAO,YAAY;AAE7D,QAAI,OAAO,WAAW,YAAY,YAAY,QAAQ;AACpD,YAAM,KAAK,WAAW,OAAO,QAAQ,OAAO,OAAO;AAAA,IACrD,WAAW,OAAO,SAAS,GAAG;AAC5B,YAAM,KAAK,WAAW,MAAM;AAAA,IAC9B;AAAA,EACF;AAAA,EAEQ,uBAAuB,OAA8B;AAC3D,QAAI,CAAC,KAAK,MAAO,QAAO,CAAC;AAEzB,UAAM,SAAS,KAAK,MAAM,gBAAgB,OAAO,IAAI;AAErD,QAAI,OAAO,WAAW,YAAY,YAAY,QAAQ;AACpD,UAAI,OAAO,WAAW,OAAO,QAAQ,SAAS,GAAG;AAC/C,eAAO,OAAO;AAAA,MAChB;AACA,aAAO,KAAK,uBAAuB,OAAO,MAAM;AAAA,IAClD;AAEA,WAAO,KAAK,uBAAuB,MAAM;AAAA,EAC3C;AAAA,EAEA,MAAc,WAAW,QAAoB,aAAwC;AAEnF,QAAI,KAAK,QAAQ,QAAQ;AACvB,YAAM,eAAe,KAAK,QAAQ,OAAO,MAAM;AAC/C,UAAI,wBAAwB,QAAS,OAAM;AAAA,IAC7C;AAGA,QAAI,KAAK,QAAQ,WAAW;AAC1B,YAAM,UAAU,eAAe,YAAY,SAAS,IAChD,cACA,KAAK,uBAAuB,MAAM;AAEtC,WAAK,aAAa,KAAK,GAAG,OAAO;AAAA,IACnC;AAAA,EACF;AAAA,EAEA,MAAc,gBAAgB,OAAiC;AAC7D,QAAI,CAAC,KAAK,QAAQ,aAAa,MAAM,WAAW,EAAG;AAEnD,UAAM,QAAQ,KAAK,OAAO,MAAM,KAAK,KAAK,WAAW;AACrD,UAAM,SAAS,KAAK,QAAQ,UAAU,OAAO,KAAK;AAElD,SAAK,gBAAgB,MAAM;AAC3B,SAAK,kBAAkB;AAEvB,QAAI,kBAAkB,SAAS;AAC7B,YAAM,iBAAiB,MAAM;AAC7B,UAAI,mBAAmB,MAAO,MAAK,MAAM;AAAA,IAC3C,WAAW,WAAW,OAAO;AAC3B,WAAK,MAAM;AAAA,IACb;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,uBAAuB,QAA+B;AAC5D,QAAI,OAAO,WAAW,EAAG,QAAO,CAAC;AAEjC,UAAM,SAAS,KAAK,QAAQ,gBAAgB;AAC5C,UAAM,OAAO,IAAI,YAAY,EAAE,OAAO,MAAM;AAE5C,QAAI;AACF,cAAQ,QAAQ;AAAA,QACd,KAAK;AACH,iBAAO,KACJ,MAAM,IAAI,EACV,OAAO,UAAQ,KAAK,KAAK,EAAE,SAAS,CAAC,EACrC,IAAI,UAAQ,KAAK,MAAM,IAAI,CAAC;AAAA,QAEjC,KAAK;AACH,gBAAM,SAAS,KAAK,MAAM,IAAI;AAC9B,iBAAO,MAAM,QAAQ,MAAM,IAAI,SAAS,CAAC,MAAM;AAAA,QAEjD,KAAK;AAAA,QACL,KAAK;AAGH,kBAAQ,KAAK,6DAA6D,MAAM;AAChF,iBAAO,CAAC;AAAA,QAEV;AACE,iBAAO,CAAC;AAAA,MACZ;AAAA,IACF,QAAQ;AACN,aAAO,CAAC;AAAA,IACV;AAAA,EACF;AAAA,EAEQ,oBAA0B;AAChC,UAAM,WAAW,KAAK,QAAQ,2BAA2B;AACzD,QAAI,KAAK,QAAQ,cAAc,KAAK,eAAe,aAAa,GAAG;AACjE,YAAM,QAAQ,KAAK,OAAO,MAAM,KAAK,KAAK,WAAW;AACrD,WAAK,QAAQ,WAAW,OAAO,KAAK,YAAY;AAAA,IAClD;AAAA,EACF;AAAA,EAEA,MAAc,eAA8B;AAC1C,QAAI,CAAC,KAAK,QAAS;AACnB,UAAM,IAAI,QAAc,aAAW;AACjC,WAAK,gBAAgB;AAAA,IACvB,CAAC;AAAA,EACH;AAAA,EAEQ,UAAgB;AACtB,SAAK,QAAQ;AACb,SAAK,gBAAgB;AACrB,SAAK,eAAe,CAAC;AAAA,EACvB;AAAA,EAEQ,aAAoB;AAC1B,WAAO;AAAA,MACL,SAAS;AAAA,MACT,UAAU;AAAA,MACV,UAAU;AAAA,MACV,kBAAkB;AAAA,MAClB,aAAa;AAAA,MACb,iBAAiB;AAAA,MACjB,aAAa;AAAA,MACb,eAAe;AAAA,MACf,oBAAoB;AAAA,MACpB,oBAAoB;AAAA,IACtB;AAAA,EACF;AACF;AAuBA,eAAsB,cACpB,OACA,SACuB;AACvB,QAAM,aAAa,IAAI,qBAAqB,OAAO,OAAO;AAC1D,SAAO,WAAW,gBAAgB;AACpC;AAyBO,SAAS,uBACd,OACA,SACkB;AAClB,SAAO,IAAI,qBAAqB,OAAO,OAAO;AAChD;AAgBO,SAAS,cACd,OACA,SACkC;AAClC,QAAM,aAAa,IAAI,qBAAqB,OAAO,OAAO;AAC1D,SAAO,WAAW,OAAO,aAAa,EAAE;AAC1C;","names":[]}
|
|
1
|
+
{"version":3,"sources":["../src/unified-stream.ts"],"sourcesContent":["/**\r\n * Unified Streaming API for ConvertBuddy\r\n * \r\n * This is the PRODUCTION-READY isomorphic streaming interface.\r\n * Works identically in browser and Node.js environments.\r\n * \r\n * Design Principles:\r\n * 1. Batch-based processing for high performance\r\n * 2. Full backpressure control (pause/resume/abort)\r\n * 3. Direct stats access through controller\r\n * 4. Zero-copy when possible (WASM-parsed records preferred)\r\n */\r\n\r\nimport { ConvertBuddy } from \"./index.js\";\r\nimport type { Stats, Format, ConvertBuddyOptions } from \"./index.js\";\r\n\r\n// ============================================================================\r\n// Types\r\n// ============================================================================\r\n\r\n/**\r\n * Unified input type - works in both Node.js and Browser\r\n */\r\nexport type StreamInput = \r\n | string // URL or raw data\r\n | Uint8Array // Binary data\r\n | ArrayBuffer // Binary data\r\n | ReadableStream<Uint8Array> // Web Streams API (both environments)\r\n | AsyncIterable<Uint8Array> // Async iterator\r\n | Blob // Browser File/Blob\r\n | NodeReadableCompat; // Node.js Readable (if available)\r\n\r\n/**\r\n * Minimal interface for Node.js Readable stream compatibility\r\n * We use duck-typing to avoid importing node:stream in browser builds\r\n */\r\ninterface NodeReadableCompat {\r\n read?: () => unknown;\r\n on?: (event: string, listener: (...args: unknown[]) => void) => unknown;\r\n [Symbol.asyncIterator]?: () => AsyncIterator<unknown>;\r\n}\r\n\r\n/**\r\n * Options for stream processing\r\n */\r\nexport interface StreamOptions extends Omit<ConvertBuddyOptions, 'onProgress'> {\r\n /**\r\n * Called with batches of parsed records. If async, backpressure is automatically applied.\r\n * Return `false` to pause processing (call controller.resume() to continue).\r\n * \r\n * @param records - Array of records (batch size controlled by recordBatchSize)\r\n * @param stats - Current conversion statistics\r\n */\r\n onRecords?: (records: unknown[], stats: Stats) => void | boolean | Promise<void | boolean>;\r\n\r\n /**\r\n * Called with raw output bytes before parsing into records.\r\n * Useful for writing directly to a file/stream while also processing records.\r\n */\r\n onData?: (bytes: Uint8Array) => void | Promise<void>;\r\n\r\n /**\r\n * Called periodically with progress stats.\r\n * Frequency controlled by `progressIntervalRecords`.\r\n */\r\n onProgress?: (stats: Stats, recordCount: number) => void;\r\n\r\n /**\r\n * Trigger onProgress callback every N records. Default: 1000\r\n */\r\n progressIntervalRecords?: number;\r\n\r\n /**\r\n * Number of records per batch when using onRecords callback or iterator.\r\n * Lower = more responsive, Higher = better throughput\r\n * Default: 100\r\n */\r\n recordBatchSize?: number;\r\n}\r\n\r\n/**\r\n * Stream controller returned by ConvertBuddy.stream()\r\n * Provides pause/resume/abort controls and stats access\r\n */\r\nexport interface StreamController {\r\n /** Pause the stream processing */\r\n pause(): void;\r\n \r\n /** Resume the stream processing */\r\n resume(): void;\r\n \r\n /** Abort the stream processing */\r\n abort(): void;\r\n \r\n /** Get current statistics */\r\n stats(): Stats | null;\r\n \r\n /** Get current record count */\r\n recordCount: number;\r\n \r\n /** Check if currently paused */\r\n isPaused: boolean;\r\n \r\n /** Check if aborted */\r\n isAborted: boolean;\r\n \r\n /** Async iterator over record batches */\r\n [Symbol.asyncIterator](): AsyncIterator<unknown[]>;\r\n}\r\n\r\n/**\r\n * Result from processing a stream\r\n */\r\nexport interface StreamResult {\r\n stats: Stats;\r\n recordCount: number;\r\n aborted: boolean;\r\n}\r\n\r\n// ============================================================================\r\n// StreamControllerImpl Class\r\n// ============================================================================\r\n\r\nclass StreamControllerImpl implements StreamController {\r\n private options: StreamOptions;\r\n private buddy: ConvertBuddy | null = null;\r\n private _aborted = false;\r\n private _paused = false;\r\n private _recordCount = 0;\r\n private pauseResolver: (() => void) | null = null;\r\n private recordBuffer: unknown[] = [];\r\n private input: StreamInput;\r\n private finalStats: Stats | null = null;\r\n\r\n constructor(input: StreamInput, options: StreamOptions) {\r\n this.input = input;\r\n this.options = {\r\n outputFormat: 'ndjson',\r\n progressIntervalRecords: 1000,\r\n recordBatchSize: 100,\r\n profile: true, // Always track stats\r\n ...options,\r\n };\r\n }\r\n\r\n // ==========================================================================\r\n // Public API: Control Methods\r\n // ==========================================================================\r\n\r\n pause(): void {\r\n this._paused = true;\r\n this.buddy?.pause();\r\n }\r\n\r\n resume(): void {\r\n this._paused = false;\r\n this.buddy?.resume();\r\n if (this.pauseResolver) {\r\n this.pauseResolver();\r\n this.pauseResolver = null;\r\n }\r\n }\r\n\r\n abort(): void {\r\n this._aborted = true;\r\n this.buddy?.abort();\r\n this.resume(); // Unblock any paused iteration\r\n }\r\n\r\n stats(): Stats | null {\r\n return this.finalStats ?? this.buddy?.stats() ?? null;\r\n }\r\n\r\n get recordCount(): number {\r\n return this._recordCount;\r\n }\r\n\r\n get isPaused(): boolean {\r\n return this._paused;\r\n }\r\n\r\n get isAborted(): boolean {\r\n return this._aborted;\r\n }\r\n\r\n // ==========================================================================\r\n // Public API: Async Iterator\r\n // ==========================================================================\r\n\r\n async *[Symbol.asyncIterator](): AsyncIterator<unknown[]> {\r\n try {\r\n this.buddy = await ConvertBuddy.create(this.options as ConvertBuddyOptions);\r\n const iterator = this.toAsyncIterator(this.input);\r\n const batchSize = this.options.recordBatchSize || 100;\r\n\r\n for await (const chunk of iterator) {\r\n if (this._aborted) return;\r\n await this.waitIfPaused();\r\n\r\n const records = this.processChunkForRecords(chunk);\r\n this.recordBuffer.push(...records);\r\n\r\n // Yield batches\r\n while (this.recordBuffer.length >= batchSize) {\r\n const batch = this.recordBuffer.splice(0, batchSize);\r\n this._recordCount += batch.length;\r\n this.maybeEmitProgress();\r\n yield batch;\r\n }\r\n }\r\n\r\n // Finalize - process any remaining records\r\n if (!this._aborted && this.buddy) {\r\n const finalOutput = this.buddy.finish();\r\n const finalRecords = this.parseRecordsFromOutput(finalOutput);\r\n this.recordBuffer.push(...finalRecords);\r\n\r\n // Yield remaining batches\r\n while (this.recordBuffer.length > 0) {\r\n const batch = this.recordBuffer.splice(0, batchSize);\r\n this._recordCount += batch.length;\r\n this.maybeEmitProgress();\r\n yield batch;\r\n }\r\n }\r\n } finally {\r\n this.cleanup();\r\n }\r\n }\r\n\r\n // ==========================================================================\r\n // Internal: Process with callbacks\r\n // ==========================================================================\r\n\r\n async processInternal(): Promise<StreamResult> {\r\n try {\r\n this.buddy = await ConvertBuddy.create(this.options as ConvertBuddyOptions);\r\n const iterator = this.toAsyncIterator(this.input);\r\n const batchSize = this.options.recordBatchSize || 100;\r\n\r\n for await (const chunk of iterator) {\r\n if (this._aborted) break;\r\n await this.waitIfPaused();\r\n\r\n await this.processChunk(chunk);\r\n\r\n // Process buffered records in batches\r\n while (this.recordBuffer.length >= batchSize) {\r\n const batch = this.recordBuffer.splice(0, batchSize);\r\n await this.emitRecordBatch(batch);\r\n }\r\n }\r\n\r\n // Finalize\r\n if (!this._aborted && this.buddy) {\r\n const finalOutput = this.buddy.finish();\r\n if (finalOutput.length > 0) {\r\n await this.emitOutput(finalOutput);\r\n }\r\n\r\n // Emit remaining records\r\n if (this.recordBuffer.length > 0) {\r\n await this.emitRecordBatch(this.recordBuffer.splice(0));\r\n }\r\n }\r\n\r\n return {\r\n stats: this.buddy?.stats() ?? this.emptyStats(),\r\n recordCount: this._recordCount,\r\n aborted: this._aborted,\r\n };\r\n } finally {\r\n this.cleanup();\r\n }\r\n }\r\n\r\n // ==========================================================================\r\n // Private: Input Normalization\r\n // ==========================================================================\r\n\r\n private async *toAsyncIterator(input: StreamInput): AsyncGenerator<Uint8Array> {\r\n // String: URL or raw data\r\n if (typeof input === 'string') {\r\n if (input.startsWith('http://') || input.startsWith('https://')) {\r\n const response = await fetch(input);\r\n if (!response.ok) throw new Error(`Fetch failed: ${response.status} ${response.statusText}`);\r\n if (!response.body) throw new Error('Response body is null');\r\n yield* this.fromReadableStream(response.body);\r\n return;\r\n }\r\n yield new TextEncoder().encode(input);\r\n return;\r\n }\r\n\r\n if (input instanceof Uint8Array) {\r\n yield input;\r\n return;\r\n }\r\n\r\n if (input instanceof ArrayBuffer) {\r\n yield new Uint8Array(input);\r\n return;\r\n }\r\n\r\n if (this.isReadableStream(input)) {\r\n yield* this.fromReadableStream(input);\r\n return;\r\n }\r\n\r\n if (typeof Blob !== 'undefined' && input instanceof Blob) {\r\n const stream = input.stream() as ReadableStream<Uint8Array>;\r\n yield* this.fromReadableStream(stream);\r\n return;\r\n }\r\n\r\n if (this.isAsyncIterable(input)) {\r\n for await (const chunk of input) {\r\n yield this.toUint8Array(chunk);\r\n }\r\n return;\r\n }\r\n\r\n if (this.isNodeReadable(input)) {\r\n yield* this.fromNodeReadable(input);\r\n return;\r\n }\r\n\r\n throw new Error('Unsupported input type');\r\n }\r\n\r\n private isReadableStream(value: unknown): value is ReadableStream<Uint8Array> {\r\n return typeof ReadableStream !== 'undefined' && value instanceof ReadableStream;\r\n }\r\n\r\n private isAsyncIterable(value: unknown): value is AsyncIterable<unknown> {\r\n return value != null && typeof (value as AsyncIterable<unknown>)[Symbol.asyncIterator] === 'function';\r\n }\r\n\r\n private isNodeReadable(value: unknown): value is NodeReadableCompat {\r\n const v = value as NodeReadableCompat;\r\n return v != null && typeof v.on === 'function' && typeof v.read === 'function';\r\n }\r\n\r\n private async *fromReadableStream(stream: ReadableStream<Uint8Array>): AsyncGenerator<Uint8Array> {\r\n const reader = stream.getReader();\r\n try {\r\n while (true) {\r\n const { done, value } = await reader.read();\r\n if (done) break;\r\n yield value;\r\n }\r\n } finally {\r\n reader.releaseLock();\r\n }\r\n }\r\n\r\n private async *fromNodeReadable(stream: NodeReadableCompat): AsyncGenerator<Uint8Array> {\r\n if (stream[Symbol.asyncIterator]) {\r\n const iter = stream[Symbol.asyncIterator]!();\r\n for await (const chunk of { [Symbol.asyncIterator]: () => iter }) {\r\n yield this.toUint8Array(chunk);\r\n }\r\n } else {\r\n throw new Error('Node.js Readable stream must be async iterable');\r\n }\r\n }\r\n\r\n private toUint8Array(chunk: unknown): Uint8Array {\r\n if (chunk instanceof Uint8Array) return chunk;\r\n if (typeof Buffer !== 'undefined' && Buffer.isBuffer(chunk)) return new Uint8Array(chunk);\r\n if (typeof chunk === 'string') return new TextEncoder().encode(chunk);\r\n if (chunk instanceof ArrayBuffer) return new Uint8Array(chunk);\r\n throw new Error('Unexpected chunk type');\r\n }\r\n\r\n // ==========================================================================\r\n // Private: Processing\r\n // ==========================================================================\r\n\r\n private async processChunk(chunk: Uint8Array): Promise<void> {\r\n if (!this.buddy) return;\r\n\r\n const needsRecords = !!this.options.onRecords;\r\n const result = this.buddy.pushWithRecords(chunk, needsRecords);\r\n\r\n if (typeof result === 'object' && 'output' in result) {\r\n await this.emitOutput(result.output, result.records);\r\n } else if (result.length > 0) {\r\n await this.emitOutput(result);\r\n }\r\n }\r\n\r\n private processChunkForRecords(chunk: Uint8Array): unknown[] {\r\n if (!this.buddy) return [];\r\n\r\n const result = this.buddy.pushWithRecords(chunk, true);\r\n \r\n if (typeof result === 'object' && 'output' in result) {\r\n if (result.records && result.records.length > 0) {\r\n return result.records;\r\n }\r\n return this.parseRecordsFromOutput(result.output);\r\n }\r\n \r\n return this.parseRecordsFromOutput(result);\r\n }\r\n\r\n private async emitOutput(output: Uint8Array, wasmRecords?: unknown[]): Promise<void> {\r\n // Emit raw data\r\n if (this.options.onData) {\r\n const maybePromise = this.options.onData(output);\r\n if (maybePromise instanceof Promise) await maybePromise;\r\n }\r\n\r\n // Buffer records for batching\r\n if (this.options.onRecords) {\r\n const records = wasmRecords && wasmRecords.length > 0 \r\n ? wasmRecords \r\n : this.parseRecordsFromOutput(output);\r\n \r\n this.recordBuffer.push(...records);\r\n }\r\n }\r\n\r\n private async emitRecordBatch(batch: unknown[]): Promise<void> {\r\n if (!this.options.onRecords || batch.length === 0) return;\r\n\r\n const stats = this.buddy?.stats() ?? this.emptyStats();\r\n const result = this.options.onRecords(batch, stats);\r\n\r\n this._recordCount += batch.length;\r\n this.maybeEmitProgress();\r\n\r\n if (result instanceof Promise) {\r\n const shouldContinue = await result;\r\n if (shouldContinue === false) this.pause();\r\n } else if (result === false) {\r\n this.pause();\r\n }\r\n }\r\n\r\n /**\r\n * Parse records from output bytes - FALLBACK ONLY\r\n * Now WASM returns pre-parsed records for all formats!\r\n */\r\n private parseRecordsFromOutput(output: Uint8Array): unknown[] {\r\n if (output.length === 0) return [];\r\n\r\n const format = this.options.outputFormat || 'ndjson';\r\n const text = new TextDecoder().decode(output);\r\n\r\n try {\r\n switch (format) {\r\n case 'ndjson':\r\n return text\r\n .split('\\n')\r\n .filter(line => line.trim().length > 0)\r\n .map(line => JSON.parse(line));\r\n \r\n case 'json':\r\n const parsed = JSON.parse(text);\r\n return Array.isArray(parsed) ? parsed : [parsed];\r\n \r\n case 'csv':\r\n case 'xml':\r\n // These should now be parsed by WASM!\r\n // If we reach here, WASM didn't return records\r\n console.warn('[StreamController] WASM did not return parsed records for', format);\r\n return [];\r\n \r\n default:\r\n return [];\r\n }\r\n } catch {\r\n return []; // Partial data - normal during streaming\r\n }\r\n }\r\n\r\n private maybeEmitProgress(): void {\r\n const interval = this.options.progressIntervalRecords || 1000;\r\n if (this.options.onProgress && this._recordCount % interval === 0) {\r\n const stats = this.buddy?.stats() ?? this.emptyStats();\r\n this.options.onProgress(stats, this._recordCount);\r\n }\r\n }\r\n\r\n private async waitIfPaused(): Promise<void> {\r\n if (!this._paused) return;\r\n await new Promise<void>(resolve => {\r\n this.pauseResolver = resolve;\r\n });\r\n }\r\n\r\n private cleanup(): void {\r\n // Store final stats before clearing buddy\r\n if (this.buddy && !this.finalStats) {\r\n this.finalStats = this.buddy.stats();\r\n }\r\n this.buddy = null;\r\n this.pauseResolver = null;\r\n this.recordBuffer = [];\r\n }\r\n\r\n private emptyStats(): Stats {\r\n return {\r\n bytesIn: 0,\r\n bytesOut: 0,\r\n chunksIn: 0,\r\n recordsProcessed: 0,\r\n parseTimeMs: 0,\r\n transformTimeMs: 0,\r\n writeTimeMs: 0,\r\n maxBufferSize: 0,\r\n currentPartialSize: 0,\r\n throughputMbPerSec: 0,\r\n };\r\n }\r\n}\r\n\r\n// ============================================================================\r\n// Convenience Functions\r\n// ============================================================================\r\n\r\n/**\r\n * Process a stream with callbacks\r\n * Returns controller for pause/resume/stats access\r\n * \r\n * @example\r\n * ```ts\r\n * const result = await processStream(file, {\r\n * inputFormat: 'csv',\r\n * outputFormat: 'json',\r\n * onRecords: (records, stats) => {\r\n * console.log(`Batch of ${records.length} records`);\r\n * console.log(`Throughput: ${stats.throughputMbPerSec} MB/s`);\r\n * },\r\n * });\r\n * console.log('Total records:', result.recordCount);\r\n * ```\r\n */\r\nexport async function processStream(\r\n input: StreamInput,\r\n options: StreamOptions\r\n): Promise<StreamResult> {\r\n const controller = new StreamControllerImpl(input, options);\r\n return controller.processInternal();\r\n}\r\n\r\n/**\r\n * Create a stream controller for manual control\r\n * \r\n * @example\r\n * ```ts\r\n * const controller = createStreamController(file, {\r\n * inputFormat: 'csv',\r\n * outputFormat: 'json',\r\n * });\r\n * \r\n * // Process with full control\r\n * for await (const batch of controller) {\r\n * console.log(`Processing ${batch.length} records`);\r\n * console.log('Stats:', controller.stats());\r\n * \r\n * if (needsSlowdown) {\r\n * controller.pause();\r\n * await sleep(1000);\r\n * controller.resume();\r\n * }\r\n * }\r\n * ```\r\n */\r\nexport function createStreamController(\r\n input: StreamInput,\r\n options: StreamOptions\r\n): StreamController {\r\n return new StreamControllerImpl(input, options);\r\n}\r\n\r\n/**\r\n * Helper: Create an async iterator over record batches\r\n * \r\n * @example\r\n * ```ts\r\n * for await (const batch of streamRecords(file, {\r\n * inputFormat: 'csv',\r\n * outputFormat: 'json',\r\n * recordBatchSize: 50, // 50 records per batch\r\n * })) {\r\n * await Promise.all(batch.map(r => saveRecord(r)));\r\n * }\r\n * ```\r\n */\r\nexport function streamRecords(\r\n input: StreamInput,\r\n options: StreamOptions\r\n): AsyncIterableIterator<unknown[]> {\r\n const controller = new StreamControllerImpl(input, options);\r\n return controller[Symbol.asyncIterator]() as AsyncIterableIterator<unknown[]>;\r\n}\r\n"],"mappings":"AAaA,SAAS,oBAAoB;AA8G7B,MAAM,qBAAiD;AAAA,EAC7C;AAAA,EACA,QAA6B;AAAA,EAC7B,WAAW;AAAA,EACX,UAAU;AAAA,EACV,eAAe;AAAA,EACf,gBAAqC;AAAA,EACrC,eAA0B,CAAC;AAAA,EAC3B;AAAA,EACA,aAA2B;AAAA,EAEnC,YAAY,OAAoB,SAAwB;AACtD,SAAK,QAAQ;AACb,SAAK,UAAU;AAAA,MACb,cAAc;AAAA,MACd,yBAAyB;AAAA,MACzB,iBAAiB;AAAA,MACjB,SAAS;AAAA;AAAA,MACT,GAAG;AAAA,IACL;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAMA,QAAc;AACZ,SAAK,UAAU;AACf,SAAK,OAAO,MAAM;AAAA,EACpB;AAAA,EAEA,SAAe;AACb,SAAK,UAAU;AACf,SAAK,OAAO,OAAO;AACnB,QAAI,KAAK,eAAe;AACtB,WAAK,cAAc;AACnB,WAAK,gBAAgB;AAAA,IACvB;AAAA,EACF;AAAA,EAEA,QAAc;AACZ,SAAK,WAAW;AAChB,SAAK,OAAO,MAAM;AAClB,SAAK,OAAO;AAAA,EACd;AAAA,EAEA,QAAsB;AACpB,WAAO,KAAK,cAAc,KAAK,OAAO,MAAM,KAAK;AAAA,EACnD;AAAA,EAEA,IAAI,cAAsB;AACxB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,WAAoB;AACtB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,YAAqB;AACvB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA,EAMA,QAAQ,OAAO,aAAa,IAA8B;AACxD,QAAI;AACF,WAAK,QAAQ,MAAM,aAAa,OAAO,KAAK,OAA8B;AAC1E,YAAM,WAAW,KAAK,gBAAgB,KAAK,KAAK;AAChD,YAAM,YAAY,KAAK,QAAQ,mBAAmB;AAElD,uBAAiB,SAAS,UAAU;AAClC,YAAI,KAAK,SAAU;AACnB,cAAM,KAAK,aAAa;AAExB,cAAM,UAAU,KAAK,uBAAuB,KAAK;AACjD,aAAK,aAAa,KAAK,GAAG,OAAO;AAGjC,eAAO,KAAK,aAAa,UAAU,WAAW;AAC5C,gBAAM,QAAQ,KAAK,aAAa,OAAO,GAAG,SAAS;AACnD,eAAK,gBAAgB,MAAM;AAC3B,eAAK,kBAAkB;AACvB,gBAAM;AAAA,QACR;AAAA,MACF;AAGA,UAAI,CAAC,KAAK,YAAY,KAAK,OAAO;AAChC,cAAM,cAAc,KAAK,MAAM,OAAO;AACtC,cAAM,eAAe,KAAK,uBAAuB,WAAW;AAC5D,aAAK,aAAa,KAAK,GAAG,YAAY;AAGtC,eAAO,KAAK,aAAa,SAAS,GAAG;AACnC,gBAAM,QAAQ,KAAK,aAAa,OAAO,GAAG,SAAS;AACnD,eAAK,gBAAgB,MAAM;AAC3B,eAAK,kBAAkB;AACvB,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF,UAAE;AACA,WAAK,QAAQ;AAAA,IACf;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,kBAAyC;AAC7C,QAAI;AACF,WAAK,QAAQ,MAAM,aAAa,OAAO,KAAK,OAA8B;AAC1E,YAAM,WAAW,KAAK,gBAAgB,KAAK,KAAK;AAChD,YAAM,YAAY,KAAK,QAAQ,mBAAmB;AAElD,uBAAiB,SAAS,UAAU;AAClC,YAAI,KAAK,SAAU;AACnB,cAAM,KAAK,aAAa;AAExB,cAAM,KAAK,aAAa,KAAK;AAG7B,eAAO,KAAK,aAAa,UAAU,WAAW;AAC5C,gBAAM,QAAQ,KAAK,aAAa,OAAO,GAAG,SAAS;AACnD,gBAAM,KAAK,gBAAgB,KAAK;AAAA,QAClC;AAAA,MACF;AAGA,UAAI,CAAC,KAAK,YAAY,KAAK,OAAO;AAChC,cAAM,cAAc,KAAK,MAAM,OAAO;AACtC,YAAI,YAAY,SAAS,GAAG;AAC1B,gBAAM,KAAK,WAAW,WAAW;AAAA,QACnC;AAGA,YAAI,KAAK,aAAa,SAAS,GAAG;AAChC,gBAAM,KAAK,gBAAgB,KAAK,aAAa,OAAO,CAAC,CAAC;AAAA,QACxD;AAAA,MACF;AAEA,aAAO;AAAA,QACL,OAAO,KAAK,OAAO,MAAM,KAAK,KAAK,WAAW;AAAA,QAC9C,aAAa,KAAK;AAAA,QAClB,SAAS,KAAK;AAAA,MAChB;AAAA,IACF,UAAE;AACA,WAAK,QAAQ;AAAA,IACf;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAMA,OAAe,gBAAgB,OAAgD;AAE7E,QAAI,OAAO,UAAU,UAAU;AAC7B,UAAI,MAAM,WAAW,SAAS,KAAK,MAAM,WAAW,UAAU,GAAG;AAC/D,cAAM,WAAW,MAAM,MAAM,KAAK;AAClC,YAAI,CAAC,SAAS,GAAI,OAAM,IAAI,MAAM,iBAAiB,SAAS,MAAM,IAAI,SAAS,UAAU,EAAE;AAC3F,YAAI,CAAC,SAAS,KAAM,OAAM,IAAI,MAAM,uBAAuB;AAC3D,eAAO,KAAK,mBAAmB,SAAS,IAAI;AAC5C;AAAA,MACF;AACA,YAAM,IAAI,YAAY,EAAE,OAAO,KAAK;AACpC;AAAA,IACF;AAEA,QAAI,iBAAiB,YAAY;AAC/B,YAAM;AACN;AAAA,IACF;AAEA,QAAI,iBAAiB,aAAa;AAChC,YAAM,IAAI,WAAW,KAAK;AAC1B;AAAA,IACF;AAEA,QAAI,KAAK,iBAAiB,KAAK,GAAG;AAChC,aAAO,KAAK,mBAAmB,KAAK;AACpC;AAAA,IACF;AAEA,QAAI,OAAO,SAAS,eAAe,iBAAiB,MAAM;AACxD,YAAM,SAAS,MAAM,OAAO;AAC5B,aAAO,KAAK,mBAAmB,MAAM;AACrC;AAAA,IACF;AAEA,QAAI,KAAK,gBAAgB,KAAK,GAAG;AAC/B,uBAAiB,SAAS,OAAO;AAC/B,cAAM,KAAK,aAAa,KAAK;AAAA,MAC/B;AACA;AAAA,IACF;AAEA,QAAI,KAAK,eAAe,KAAK,GAAG;AAC9B,aAAO,KAAK,iBAAiB,KAAK;AAClC;AAAA,IACF;AAEA,UAAM,IAAI,MAAM,wBAAwB;AAAA,EAC1C;AAAA,EAEQ,iBAAiB,OAAqD;AAC5E,WAAO,OAAO,mBAAmB,eAAe,iBAAiB;AAAA,EACnE;AAAA,EAEQ,gBAAgB,OAAiD;AACvE,WAAO,SAAS,QAAQ,OAAQ,MAAiC,OAAO,aAAa,MAAM;AAAA,EAC7F;AAAA,EAEQ,eAAe,OAA6C;AAClE,UAAM,IAAI;AACV,WAAO,KAAK,QAAQ,OAAO,EAAE,OAAO,cAAc,OAAO,EAAE,SAAS;AAAA,EACtE;AAAA,EAEA,OAAe,mBAAmB,QAAgE;AAChG,UAAM,SAAS,OAAO,UAAU;AAChC,QAAI;AACF,aAAO,MAAM;AACX,cAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,YAAI,KAAM;AACV,cAAM;AAAA,MACR;AAAA,IACF,UAAE;AACA,aAAO,YAAY;AAAA,IACrB;AAAA,EACF;AAAA,EAEA,OAAe,iBAAiB,QAAwD;AACtF,QAAI,OAAO,OAAO,aAAa,GAAG;AAChC,YAAM,OAAO,OAAO,OAAO,aAAa,EAAG;AAC3C,uBAAiB,SAAS,EAAE,CAAC,OAAO,aAAa,GAAG,MAAM,KAAK,GAAG;AAChE,cAAM,KAAK,aAAa,KAAK;AAAA,MAC/B;AAAA,IACF,OAAO;AACL,YAAM,IAAI,MAAM,gDAAgD;AAAA,IAClE;AAAA,EACF;AAAA,EAEQ,aAAa,OAA4B;AAC/C,QAAI,iBAAiB,WAAY,QAAO;AACxC,QAAI,OAAO,WAAW,eAAe,OAAO,SAAS,KAAK,EAAG,QAAO,IAAI,WAAW,KAAK;AACxF,QAAI,OAAO,UAAU,SAAU,QAAO,IAAI,YAAY,EAAE,OAAO,KAAK;AACpE,QAAI,iBAAiB,YAAa,QAAO,IAAI,WAAW,KAAK;AAC7D,UAAM,IAAI,MAAM,uBAAuB;AAAA,EACzC;AAAA;AAAA;AAAA;AAAA,EAMA,MAAc,aAAa,OAAkC;AAC3D,QAAI,CAAC,KAAK,MAAO;AAEjB,UAAM,eAAe,CAAC,CAAC,KAAK,QAAQ;AACpC,UAAM,SAAS,KAAK,MAAM,gBAAgB,OAAO,YAAY;AAE7D,QAAI,OAAO,WAAW,YAAY,YAAY,QAAQ;AACpD,YAAM,KAAK,WAAW,OAAO,QAAQ,OAAO,OAAO;AAAA,IACrD,WAAW,OAAO,SAAS,GAAG;AAC5B,YAAM,KAAK,WAAW,MAAM;AAAA,IAC9B;AAAA,EACF;AAAA,EAEQ,uBAAuB,OAA8B;AAC3D,QAAI,CAAC,KAAK,MAAO,QAAO,CAAC;AAEzB,UAAM,SAAS,KAAK,MAAM,gBAAgB,OAAO,IAAI;AAErD,QAAI,OAAO,WAAW,YAAY,YAAY,QAAQ;AACpD,UAAI,OAAO,WAAW,OAAO,QAAQ,SAAS,GAAG;AAC/C,eAAO,OAAO;AAAA,MAChB;AACA,aAAO,KAAK,uBAAuB,OAAO,MAAM;AAAA,IAClD;AAEA,WAAO,KAAK,uBAAuB,MAAM;AAAA,EAC3C;AAAA,EAEA,MAAc,WAAW,QAAoB,aAAwC;AAEnF,QAAI,KAAK,QAAQ,QAAQ;AACvB,YAAM,eAAe,KAAK,QAAQ,OAAO,MAAM;AAC/C,UAAI,wBAAwB,QAAS,OAAM;AAAA,IAC7C;AAGA,QAAI,KAAK,QAAQ,WAAW;AAC1B,YAAM,UAAU,eAAe,YAAY,SAAS,IAChD,cACA,KAAK,uBAAuB,MAAM;AAEtC,WAAK,aAAa,KAAK,GAAG,OAAO;AAAA,IACnC;AAAA,EACF;AAAA,EAEA,MAAc,gBAAgB,OAAiC;AAC7D,QAAI,CAAC,KAAK,QAAQ,aAAa,MAAM,WAAW,EAAG;AAEnD,UAAM,QAAQ,KAAK,OAAO,MAAM,KAAK,KAAK,WAAW;AACrD,UAAM,SAAS,KAAK,QAAQ,UAAU,OAAO,KAAK;AAElD,SAAK,gBAAgB,MAAM;AAC3B,SAAK,kBAAkB;AAEvB,QAAI,kBAAkB,SAAS;AAC7B,YAAM,iBAAiB,MAAM;AAC7B,UAAI,mBAAmB,MAAO,MAAK,MAAM;AAAA,IAC3C,WAAW,WAAW,OAAO;AAC3B,WAAK,MAAM;AAAA,IACb;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,uBAAuB,QAA+B;AAC5D,QAAI,OAAO,WAAW,EAAG,QAAO,CAAC;AAEjC,UAAM,SAAS,KAAK,QAAQ,gBAAgB;AAC5C,UAAM,OAAO,IAAI,YAAY,EAAE,OAAO,MAAM;AAE5C,QAAI;AACF,cAAQ,QAAQ;AAAA,QACd,KAAK;AACH,iBAAO,KACJ,MAAM,IAAI,EACV,OAAO,UAAQ,KAAK,KAAK,EAAE,SAAS,CAAC,EACrC,IAAI,UAAQ,KAAK,MAAM,IAAI,CAAC;AAAA,QAEjC,KAAK;AACH,gBAAM,SAAS,KAAK,MAAM,IAAI;AAC9B,iBAAO,MAAM,QAAQ,MAAM,IAAI,SAAS,CAAC,MAAM;AAAA,QAEjD,KAAK;AAAA,QACL,KAAK;AAGH,kBAAQ,KAAK,6DAA6D,MAAM;AAChF,iBAAO,CAAC;AAAA,QAEV;AACE,iBAAO,CAAC;AAAA,MACZ;AAAA,IACF,QAAQ;AACN,aAAO,CAAC;AAAA,IACV;AAAA,EACF;AAAA,EAEQ,oBAA0B;AAChC,UAAM,WAAW,KAAK,QAAQ,2BAA2B;AACzD,QAAI,KAAK,QAAQ,cAAc,KAAK,eAAe,aAAa,GAAG;AACjE,YAAM,QAAQ,KAAK,OAAO,MAAM,KAAK,KAAK,WAAW;AACrD,WAAK,QAAQ,WAAW,OAAO,KAAK,YAAY;AAAA,IAClD;AAAA,EACF;AAAA,EAEA,MAAc,eAA8B;AAC1C,QAAI,CAAC,KAAK,QAAS;AACnB,UAAM,IAAI,QAAc,aAAW;AACjC,WAAK,gBAAgB;AAAA,IACvB,CAAC;AAAA,EACH;AAAA,EAEQ,UAAgB;AAEtB,QAAI,KAAK,SAAS,CAAC,KAAK,YAAY;AAClC,WAAK,aAAa,KAAK,MAAM,MAAM;AAAA,IACrC;AACA,SAAK,QAAQ;AACb,SAAK,gBAAgB;AACrB,SAAK,eAAe,CAAC;AAAA,EACvB;AAAA,EAEQ,aAAoB;AAC1B,WAAO;AAAA,MACL,SAAS;AAAA,MACT,UAAU;AAAA,MACV,UAAU;AAAA,MACV,kBAAkB;AAAA,MAClB,aAAa;AAAA,MACb,iBAAiB;AAAA,MACjB,aAAa;AAAA,MACb,eAAe;AAAA,MACf,oBAAoB;AAAA,MACpB,oBAAoB;AAAA,IACtB;AAAA,EACF;AACF;AAuBA,eAAsB,cACpB,OACA,SACuB;AACvB,QAAM,aAAa,IAAI,qBAAqB,OAAO,OAAO;AAC1D,SAAO,WAAW,gBAAgB;AACpC;AAyBO,SAAS,uBACd,OACA,SACkB;AAClB,SAAO,IAAI,qBAAqB,OAAO,OAAO;AAChD;AAgBO,SAAS,cACd,OACA,SACkC;AAClC,QAAM,aAAa,IAAI,qBAAqB,OAAO,OAAO;AAC1D,SAAO,WAAW,OAAO,aAAa,EAAE;AAC1C;","names":[]}
|