@open-mercato/core 0.6.6 → 0.6.7-develop.6553.1.2ec7246e04

package/src/modules/sync_excel/lib/adapters/customers.ts

@@ -17,8 +17,8 @@ import { CustomerAddress, CustomerEntity } from '../../../customers/data/entitie
 import { Attachment } from '../../../attachments/data/entities'
 import { CustomFieldDef } from '../../../entities/data/entities'
 import { SyncExcelUpload } from '../../data/entities'
-import { parseCsvDocument, type CsvPreviewRow } from '../parser'
-import { readSyncExcelUploadBuffer } from '../upload-storage'
+import { parseCsvDocumentBatches, parseCsvStreamMetadata, type CsvPreviewRow } from '../parser'
+import { createSyncExcelUploadReadStream } from '../upload-storage'
 import { E } from '#generated/entities.ids.generated'
 
 type SyncExcelCursor = {
@@ -785,20 +785,16 @@ async function resolveExistingPersonId(params: {
   return params.emailDedupeIndex.get(params.email) ?? null
 }
 
+function mappingHasEmailField(mapping: DataMapping): boolean {
+  return mapping.fields.some((field) => field.localField === 'person.primaryEmail')
+}
+
 async function buildEmailDedupeIndex(params: {
   em: EntityManager
-  rows: CsvPreviewRow[]
   mapping: DataMapping
   scope: TenantScope
 }): Promise<EmailDedupeIndex> {
-  const emails = new Set<string>()
-  for (const row of params.rows) {
-    const rowValues = mapRowValues(row, params.mapping.fields)
-    const email = rowValues.values.primaryEmail
-    if (email) emails.add(email)
-  }
-
-  if (emails.size === 0) return new Map()
+  if (!mappingHasEmailField(params.mapping)) return new Map()
 
   const candidates = await findWithDecryption(
     params.em,
@@ -821,7 +817,7 @@ async function buildEmailDedupeIndex(params: {
   const index: EmailDedupeIndex = new Map()
   for (const candidate of candidates) {
     const email = normalizeEmail(candidate.primaryEmail)
-    if (!email || !emails.has(email) || index.has(email)) continue
+    if (!email || index.has(email)) continue
     index.set(email, candidate.id)
   }
   return index
@@ -1043,26 +1039,28 @@ export const syncExcelCustomersAdapter: DataSyncAdapter = {
         throw new Error('CSV upload attachment could not be found.')
       }
 
-      const fileBuffer = await readSyncExcelUploadBuffer(attachment)
-      const document = parseCsvDocument(fileBuffer)
+      const document = await parseCsvStreamMetadata(await createSyncExcelUploadReadStream(attachment))
       const startOffset = cursor?.uploadId === upload.id ? cursor.offset : 0
       const commandContext = buildCommandContext(container, input.scope)
       const customFieldDefinitions = await loadImportCustomFieldDefinitions(em, input.scope)
       const emailDedupeIndex = await buildEmailDedupeIndex({
         em,
-        rows: document.rows.slice(startOffset),
         mapping: input.mapping,
         scope: input.scope,
       })
+      let batchIndex = 0
 
-      for (let offset = startOffset, batchIndex = 0; offset < document.rows.length; offset += input.batchSize, batchIndex += 1) {
-        const batchRows = document.rows.slice(offset, offset + input.batchSize)
+      for await (const batch of parseCsvDocumentBatches(await createSyncExcelUploadReadStream(attachment), {
+        batchSize: input.batchSize,
+        startOffset,
+      })) {
+        const batchRows = batch.rows
         const items: ImportItem[] = []
 
         for (let index = 0; index < batchRows.length; index += 1) {
           items.push(await processRow({
             row: batchRows[index],
-            rowNumber: offset + index + 1,
+            rowNumber: batch.rowStart + index + 1,
             mapping: input.mapping,
             scope: input.scope,
             commandBus,
@@ -1074,17 +1072,17 @@ export const syncExcelCustomersAdapter: DataSyncAdapter = {
           }))
         }
 
-        const nextOffset = offset + batchRows.length
         yield {
           items,
-          cursor: createCursor(upload.id, nextOffset),
-          hasMore: nextOffset < document.rows.length,
+          cursor: createCursor(upload.id, batch.nextOffset),
+          hasMore: batch.nextOffset < document.totalRows,
           totalEstimate: document.totalRows,
           processedCount: batchRows.length,
           refreshCoverageEntityTypes: CUSTOMER_IMPORT_COVERAGE_ENTITY_TYPES,
           batchIndex,
-          message: `Processed ${nextOffset} of ${document.totalRows} CSV rows`,
+          message: `Processed ${batch.nextOffset} of ${document.totalRows} CSV rows`,
         }
+        batchIndex += 1
       }
 
       upload.status = 'completed'

package/src/modules/sync_excel/lib/parser.ts

@@ -1,3 +1,5 @@
+import { TextDecoder } from 'util'
+
 export type CsvPreviewRow = Record<string, string | null>
 
 export type CsvPreview = {
@@ -16,10 +18,36 @@ export type CsvDocument = {
   encoding: 'utf-8'
 }
 
+export type CsvStreamMetadata = {
+  headers: string[]
+  totalRows: number
+  delimiter: ',' | ';'
+  encoding: 'utf-8'
+}
+
+export type CsvDocumentBatch = {
+  headers: string[]
+  rows: CsvPreviewRow[]
+  rowStart: number
+  nextOffset: number
+  delimiter: ',' | ';'
+  encoding: 'utf-8'
+}
+
 type ParseCsvPreviewOptions = {
   maxRows?: number
 }
 
+type ParseCsvBatchOptions = {
+  batchSize: number
+  startOffset?: number
+}
+
+type CsvInputChunk = Buffer | Uint8Array | string
+
+const CSV_DELIMITER_PROBE_ROWS = 5
+const CSV_DELIMITER_PROBE_MAX_CHARS = 64 * 1024
+
 function stripUtf8Bom(value: string): string {
   return value.charCodeAt(0) === 0xfeff ? value.slice(1) : value
 }
@@ -131,6 +159,223 @@ function rowsToObjects(headers: string[], rows: string[][], maxRows: number): Cs
   })
 }
 
+async function createCsvRowStream(input: AsyncIterable<CsvInputChunk>): Promise<{
+  delimiter: ',' | ';'
+  rows: AsyncIterable<string[]>
+}> {
+  const iterator = input[Symbol.asyncIterator]()
+  const decoder = new TextDecoder('utf-8')
+  const prefixTexts: string[] = []
+  let probeText = ''
+
+  while (probeText.length < CSV_DELIMITER_PROBE_MAX_CHARS) {
+    const next = await iterator.next()
+    if (next.done) break
+
+    const text = typeof next.value === 'string' ? next.value : decoder.decode(next.value, { stream: true })
+    prefixTexts.push(text)
+    probeText += text
+
+    if (parseCsvText(probeText, ',').length >= CSV_DELIMITER_PROBE_ROWS) {
+      break
+    }
+  }
+
+  const delimiter = detectCsvDelimiter(probeText)
+
+  async function* decodedChunks(): AsyncIterable<string> {
+    for (const text of prefixTexts) {
+      yield text
+    }
+    while (true) {
+      const next = await iterator.next()
+      if (next.done) break
+      yield typeof next.value === 'string' ? next.value : decoder.decode(next.value, { stream: true })
+    }
+    const finalChunk = decoder.decode()
+    if (finalChunk.length > 0) {
+      yield finalChunk
+    }
+  }
+
+  return {
+    delimiter,
+    rows: parseCsvRowsFromTextChunks(decodedChunks(), delimiter),
+  }
+}
+
+async function* parseCsvRowsFromTextChunks(
+  chunks: AsyncIterable<string>,
+  delimiter: ',' | ';',
+): AsyncIterable<string[]> {
+  let currentRow: string[] = []
+  let currentValue = ''
+  let inQuotes = false
+  let isFirstChunk = true
+  let previousWasCarriageReturn = false
+  let pendingQuote = false
+  let pendingQuoteWasInQuotes = false
+
+  const completeRow = (): string[] | null => {
+    currentRow.push(currentValue)
+    const completedRow = currentRow
+    currentRow = []
+    currentValue = ''
+    return completedRow.some((value) => value.trim().length > 0) ? completedRow : null
+  }
+
+  for await (const chunk of chunks) {
+    const text = isFirstChunk ? stripUtf8Bom(chunk) : chunk
+    isFirstChunk = false
+
+    for (let index = 0; index < text.length; index += 1) {
+      const character = text[index]
+
+      if (pendingQuote) {
+        pendingQuote = false
+        if (pendingQuoteWasInQuotes && character === '"') {
+          currentValue += '"'
+          continue
+        }
+        inQuotes = !pendingQuoteWasInQuotes
+      }
+
+      if (previousWasCarriageReturn) {
+        previousWasCarriageReturn = false
+        if (character === '\n') {
+          continue
+        }
+      }
+
+      if (character === '"') {
+        if (index + 1 >= text.length) {
+          pendingQuote = true
+          pendingQuoteWasInQuotes = inQuotes
+          continue
+        }
+        if (inQuotes && text[index + 1] === '"') {
+          currentValue += '"'
+          index += 1
+          continue
+        }
+        inQuotes = !inQuotes
+        continue
+      }
+
+      if (!inQuotes && character === delimiter) {
+        currentRow.push(currentValue)
+        currentValue = ''
+        continue
+      }
+
+      if (character === '\r') {
+        previousWasCarriageReturn = true
+        if (inQuotes) {
+          currentValue += '\n'
+        } else {
+          const row = completeRow()
+          if (row) yield row
+        }
+        continue
+      }
+
+      if (character === '\n') {
+        if (inQuotes) {
+          currentValue += '\n'
+        } else {
+          const row = completeRow()
+          if (row) yield row
+        }
+        continue
+      }
+
+      currentValue += character
+    }
+  }
+
+  if (currentValue.length > 0 || currentRow.length > 0) {
+    const row = completeRow()
+    if (row) yield row
+  }
+}
+
+export async function parseCsvStreamMetadata(input: AsyncIterable<CsvInputChunk>): Promise<CsvStreamMetadata> {
+  const stream = await createCsvRowStream(input)
+  let headers: string[] = []
+  let hasHeaders = false
+  let totalRows = 0
+
+  for await (const row of stream.rows) {
+    if (!hasHeaders) {
+      headers = normalizeHeaders(row)
+      hasHeaders = true
+      continue
+    }
+    totalRows += 1
+  }
+
+  return {
+    headers,
+    totalRows,
+    delimiter: stream.delimiter,
+    encoding: 'utf-8',
+  }
+}
+
+export async function* parseCsvDocumentBatches(
+  input: AsyncIterable<CsvInputChunk>,
+  options: ParseCsvBatchOptions,
+): AsyncIterable<CsvDocumentBatch> {
+  const stream = await createCsvRowStream(input)
+  const batchSize = Math.max(1, Math.floor(options.batchSize))
+  const startOffset = Math.max(0, Math.floor(options.startOffset ?? 0))
+  let headers: string[] | null = null
+  let rowOffset = 0
+  let batchRows: string[][] = []
+  let batchStart = startOffset
+
+  for await (const row of stream.rows) {
+    if (!headers) {
+      headers = normalizeHeaders(row)
+      continue
+    }
+
+    if (rowOffset < startOffset) {
+      rowOffset += 1
+      continue
+    }
+
+    if (batchRows.length === 0) {
+      batchStart = rowOffset
+    }
+    batchRows.push(row)
+    rowOffset += 1
+
+    if (batchRows.length >= batchSize) {
+      yield {
+        headers,
+        rows: rowsToObjects(headers, batchRows, batchRows.length),
+        rowStart: batchStart,
+        nextOffset: rowOffset,
+        delimiter: stream.delimiter,
+        encoding: 'utf-8',
+      }
+      batchRows = []
+    }
+  }
+
+  if (headers && batchRows.length > 0) {
+    yield {
+      headers,
+      rows: rowsToObjects(headers, batchRows, batchRows.length),
+      rowStart: batchStart,
+      nextOffset: rowOffset,
+      delimiter: stream.delimiter,
+      encoding: 'utf-8',
+    }
+  }
+}
+
 export function parseCsvPreview(buffer: Buffer, options: ParseCsvPreviewOptions = {}): CsvPreview {
   const text = buffer.toString('utf-8')
   const delimiter = detectCsvDelimiter(text)

package/src/modules/sync_excel/lib/upload-storage.ts

@@ -1,4 +1,5 @@
-import { promises as fs } from 'fs'
+import { createReadStream, promises as fs } from 'fs'
+import { Readable } from 'stream'
 import { randomUUID } from 'crypto'
 import type { EntityManager } from '@mikro-orm/postgresql'
 import { Attachment, AttachmentPartition } from '../../attachments/data/entities'
@@ -79,3 +80,24 @@ export async function readSyncExcelUploadBuffer(
     throw error
   }
 }
+
+export async function createSyncExcelUploadReadStream(
+  attachment: Pick<Attachment, 'partitionCode' | 'storagePath' | 'storageDriver' | 'storageMetadata'>,
+): Promise<Readable> {
+  const inlineCsvBase64 = attachment.storageMetadata?.inlineCsvBase64
+
+  try {
+    const absolutePath = resolveAttachmentAbsolutePath(
+      attachment.partitionCode,
+      attachment.storagePath,
+      attachment.storageDriver,
+    )
+    await fs.access(absolutePath)
+    return createReadStream(absolutePath)
+  } catch (error) {
+    if (typeof inlineCsvBase64 === 'string' && inlineCsvBase64.length > 0) {
+      return Readable.from([Buffer.from(inlineCsvBase64, 'base64')])
+    }
+    throw error
+  }
+}