@open-mercato/core 0.6.2-develop.3330.1.8ad2b3b84c → 0.6.2-develop.3366.1.4692954e44

package/src/modules/sync_excel/lib/column-detector.ts

@@ -0,0 +1,91 @@
+import type { DataMapping, FieldMapping } from '../../data_sync/lib/adapter'
+
+const PERSON_FIELD_ALIASES: Record<string, string[]> = {
+  'person.externalId': ['record id', 'external id', 'lead id'],
+  'person.firstName': ['first name', 'firstname', 'given name'],
+  'person.lastName': ['last name', 'lastname', 'surname', 'family name'],
+  'person.displayName': ['lead name', 'display name', 'full name', 'name'],
+  'person.primaryEmail': ['email', 'primary email', 'email address'],
+  'person.primaryPhone': ['mobile', 'phone', 'mobile phone', 'telephone'],
+  'person.jobTitle': ['title', 'job title', 'position'],
+  'person.status': ['lead status', 'status'],
+  'person.source': ['lead source', 'source'],
+  'person.description': ['description', 'notes', 'comment'],
+  'address.name': ['address label', 'address name', 'label'],
+  'address.purpose': ['address purpose', 'address type'],
+  'address.companyName': ['address company', 'address company name'],
+  'address.addressLine1': ['address line 1', 'street address', 'street', 'street 1'],
+  'address.addressLine2': ['address line 2', 'street 2'],
+  'address.buildingNumber': ['building number', 'building no', 'house number'],
+  'address.flatNumber': ['flat number', 'apartment number', 'unit number'],
+  'address.city': ['city', 'town'],
+  'address.region': ['region', 'state', 'province'],
+  'address.postalCode': ['postal code', 'zip code', 'postcode'],
+  'address.country': ['country'],
+  'address.latitude': ['latitude', 'lat'],
+  'address.longitude': ['longitude', 'lng', 'lon'],
+}
+
+function normalizeLabel(value: string): string {
+  return value
+    .trim()
+    .replace(/([a-z0-9])([A-Z])/g, '$1 $2')
+    .toLowerCase()
+    .replace(/[_-]+/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim()
+}
+
+function buildFieldMapping(externalField: string, localField: string): FieldMapping {
+  if (localField === 'person.externalId') {
+    return {
+      externalField,
+      localField,
+      mappingKind: 'external_id',
+      dedupeRole: 'primary',
+    }
+  }
+
+  if (localField === 'person.primaryEmail') {
+    return {
+      externalField,
+      localField,
+      mappingKind: 'core',
+      dedupeRole: 'secondary',
+    }
+  }
+
+  return {
+    externalField,
+    localField,
+    mappingKind: 'core',
+  }
+}
+
+export function detectCustomersPersonMapping(columns: string[]): DataMapping & { unmappedColumns: string[] } {
+  const fields: FieldMapping[] = []
+  const unmappedColumns: string[] = []
+
+  for (const column of columns) {
+    const normalizedColumn = normalizeLabel(column)
+    const matchingField = Object.entries(PERSON_FIELD_ALIASES).find(([, aliases]) => aliases.includes(normalizedColumn))
+
+    if (!matchingField) {
+      unmappedColumns.push(column)
+      continue
+    }
+
+    fields.push(buildFieldMapping(column, matchingField[0]))
+  }
+
+  const hasExternalId = fields.some((field) => field.localField === 'person.externalId')
+  const hasEmail = fields.some((field) => field.localField === 'person.primaryEmail')
+
+  return {
+    entityType: 'customers.person',
+    matchStrategy: hasExternalId ? 'externalId' : hasEmail ? 'email' : 'custom',
+    matchField: hasExternalId ? 'person.externalId' : hasEmail ? 'person.primaryEmail' : undefined,
+    fields,
+    unmappedColumns,
+  }
+}

package/src/modules/sync_excel/lib/mapping.ts

@@ -0,0 +1,19 @@
+import type { DataMapping, FieldMapping } from '../../data_sync/lib/adapter'
+import { detectCustomersPersonMapping } from './column-detector'
+
+export type SuggestedMapping = DataMapping & {
+  unmappedColumns: string[]
+}
+
+export function buildSuggestedMapping(entityType: string, headers: string[]): SuggestedMapping {
+  if (entityType === 'customers.person') {
+    return detectCustomersPersonMapping(headers)
+  }
+
+  return {
+    entityType,
+    matchStrategy: 'custom',
+    fields: [] as FieldMapping[],
+    unmappedColumns: headers.slice(),
+  }
+}

package/src/modules/sync_excel/lib/parser.ts

@@ -0,0 +1,165 @@
+export type CsvPreviewRow = Record<string, string | null>
+
+export type CsvPreview = {
+  headers: string[]
+  sampleRows: CsvPreviewRow[]
+  totalRows: number
+  delimiter: ',' | ';'
+  encoding: 'utf-8'
+}
+
+export type CsvDocument = {
+  headers: string[]
+  rows: CsvPreviewRow[]
+  totalRows: number
+  delimiter: ',' | ';'
+  encoding: 'utf-8'
+}
+
+type ParseCsvPreviewOptions = {
+  maxRows?: number
+}
+
+function stripUtf8Bom(value: string): string {
+  return value.charCodeAt(0) === 0xfeff ? value.slice(1) : value
+}
+
+function countFields(line: string, delimiter: ',' | ';'): number {
+  let count = 1
+  let inQuotes = false
+
+  for (let index = 0; index < line.length; index += 1) {
+    const character = line[index]
+    if (character === '"') {
+      if (inQuotes && line[index + 1] === '"') {
+        index += 1
+        continue
+      }
+      inQuotes = !inQuotes
+      continue
+    }
+    if (!inQuotes && character === delimiter) {
+      count += 1
+    }
+  }
+
+  return count
+}
+
+export function detectCsvDelimiter(text: string): ',' | ';' {
+  const normalized = stripUtf8Bom(text).replace(/\r\n/g, '\n').replace(/\r/g, '\n')
+  const lines = normalized
+    .split('\n')
+    .map((line) => line.trim())
+    .filter((line) => line.length > 0)
+    .slice(0, 5)
+
+  if (lines.length === 0) return ','
+
+  const commaScore = lines.reduce((accumulator, line) => accumulator + countFields(line, ','), 0)
+  const semicolonScore = lines.reduce((accumulator, line) => accumulator + countFields(line, ';'), 0)
+
+  return semicolonScore > commaScore ? ';' : ','
+}
+
+export function parseCsvText(text: string, delimiter: ',' | ';'): string[][] {
+  const normalized = stripUtf8Bom(text).replace(/\r\n/g, '\n').replace(/\r/g, '\n')
+  const rows: string[][] = []
+  let currentRow: string[] = []
+  let currentValue = ''
+  let inQuotes = false
+
+  for (let index = 0; index < normalized.length; index += 1) {
+    const character = normalized[index]
+
+    if (character === '"') {
+      if (inQuotes && normalized[index + 1] === '"') {
+        currentValue += '"'
+        index += 1
+        continue
+      }
+      inQuotes = !inQuotes
+      continue
+    }
+
+    if (!inQuotes && character === delimiter) {
+      currentRow.push(currentValue)
+      currentValue = ''
+      continue
+    }
+
+    if (!inQuotes && character === '\n') {
+      currentRow.push(currentValue)
+      const isMeaningfulRow = currentRow.some((value) => value.trim().length > 0)
+      if (isMeaningfulRow) {
+        rows.push(currentRow)
+      }
+      currentRow = []
+      currentValue = ''
+      continue
+    }
+
+    currentValue += character
+  }
+
+  if (currentValue.length > 0 || currentRow.length > 0) {
+    currentRow.push(currentValue)
+    const isMeaningfulRow = currentRow.some((value) => value.trim().length > 0)
+    if (isMeaningfulRow) {
+      rows.push(currentRow)
+    }
+  }
+
+  return rows
+}
+
+function normalizeHeaders(headers: string[]): string[] {
+  return headers.map((header, index) => {
+    const trimmed = header.trim()
+    return trimmed.length > 0 ? trimmed : `Column ${index + 1}`
+  })
+}
+
+function rowsToObjects(headers: string[], rows: string[][], maxRows: number): CsvPreviewRow[] {
+  return rows.slice(0, maxRows).map((row) => {
+    const record: CsvPreviewRow = {}
+    headers.forEach((header, index) => {
+      const value = row[index]
+      record[header] = typeof value === 'string' && value.length > 0 ? value : null
+    })
+    return record
+  })
+}
+
+export function parseCsvPreview(buffer: Buffer, options: ParseCsvPreviewOptions = {}): CsvPreview {
+  const text = buffer.toString('utf-8')
+  const delimiter = detectCsvDelimiter(text)
+  const rows = parseCsvText(text, delimiter)
+  const [rawHeaders = [], ...dataRows] = rows
+  const headers = normalizeHeaders(rawHeaders)
+  const maxRows = options.maxRows ?? 5
+
+  return {
+    headers,
+    sampleRows: rowsToObjects(headers, dataRows, maxRows),
+    totalRows: dataRows.length,
+    delimiter,
+    encoding: 'utf-8',
+  }
+}
+
+export function parseCsvDocument(buffer: Buffer): CsvDocument {
+  const text = buffer.toString('utf-8')
+  const delimiter = detectCsvDelimiter(text)
+  const rows = parseCsvText(text, delimiter)
+  const [rawHeaders = [], ...dataRows] = rows
+  const headers = normalizeHeaders(rawHeaders)
+
+  return {
+    headers,
+    rows: rowsToObjects(headers, dataRows, dataRows.length),
+    totalRows: dataRows.length,
+    delimiter,
+    encoding: 'utf-8',
+  }
+}

package/src/modules/sync_excel/lib/scope.ts

@@ -0,0 +1,63 @@
+import type { AwilixContainer } from 'awilix'
+import type { AuthContext } from '@open-mercato/shared/lib/auth/server'
+import { isAllOrganizationsSelection } from '@open-mercato/core/modules/directory/constants'
+import { getSelectedOrganizationFromRequest, resolveOrganizationScopeForRequest } from '@open-mercato/core/modules/directory/utils/organizationScope'
+import type { TenantScope } from '../../data_sync/lib/adapter'
+
+export type SyncExcelConcreteScopeResult =
+  | { ok: true; scope: TenantScope }
+  | { ok: false; status: 401 | 422; error: string }
+
+const CONCRETE_ORGANIZATION_REQUIRED_ERROR = 'Select a concrete organization before importing CSV.'
+
+function normalizeSelectedOrganizationId(value: string | null | undefined): string | null {
+  if (typeof value !== 'string') return null
+  const trimmed = value.trim()
+  return trimmed.length > 0 ? trimmed : null
+}
+
+export async function resolveSyncExcelConcreteScope(params: {
+  auth: AuthContext
+  container: AwilixContainer
+  request: Request
+}): Promise<SyncExcelConcreteScopeResult> {
+  const { auth, container, request } = params
+  if (!auth?.tenantId) {
+    return { ok: false, status: 401, error: 'Unauthorized' }
+  }
+
+  const requestedSelectedId = normalizeSelectedOrganizationId(getSelectedOrganizationFromRequest(request))
+  if (!requestedSelectedId || isAllOrganizationsSelection(requestedSelectedId)) {
+    return { ok: false, status: 422, error: CONCRETE_ORGANIZATION_REQUIRED_ERROR }
+  }
+
+  const organizationScope = await resolveOrganizationScopeForRequest({
+    container,
+    auth,
+    request,
+    selectedId: requestedSelectedId,
+  })
+
+  const selectedId = organizationScope.selectedId
+  const tenantId = organizationScope.tenantId ?? auth.tenantId
+  const filterIds = organizationScope.filterIds
+  const selectedIsResolvable = Boolean(
+    selectedId
+    && tenantId
+    && Array.isArray(filterIds)
+    && selectedId === requestedSelectedId
+    && filterIds.includes(selectedId),
+  )
+
+  if (!selectedIsResolvable || !selectedId || !tenantId) {
+    return { ok: false, status: 422, error: CONCRETE_ORGANIZATION_REQUIRED_ERROR }
+  }
+
+  return {
+    ok: true,
+    scope: {
+      organizationId: selectedId,
+      tenantId,
+    },
+  }
+}

package/src/modules/sync_excel/lib/upload-storage.ts

@@ -0,0 +1,79 @@
+import { promises as fs } from 'fs'
+import { randomUUID } from 'crypto'
+import type { EntityManager } from '@mikro-orm/postgresql'
+import { Attachment, AttachmentPartition } from '../../attachments/data/entities'
+import { buildAttachmentFileUrl } from '../../attachments/lib/imageUrls'
+import { ensureDefaultPartitions } from '../../attachments/lib/partitions'
+import { resolveAttachmentAbsolutePath, storePartitionFile } from '../../attachments/lib/storage'
+
+const SYNC_EXCEL_ATTACHMENT_ENTITY_ID = 'sync_excel:upload'
+const SYNC_EXCEL_PARTITION_CODE = 'privateAttachments'
+
+export async function createSyncExcelUploadAttachment(input: {
+  em: EntityManager
+  uploadId: string
+  organizationId: string
+  tenantId: string
+  fileName: string
+  mimeType: string
+  buffer: Buffer
+}): Promise<Attachment> {
+  await ensureDefaultPartitions(input.em)
+
+  // Attachment partitions are global storage configuration, not tenant-scoped payload data.
+  const partition = await input.em.findOne(AttachmentPartition, { code: SYNC_EXCEL_PARTITION_CODE })
+  if (!partition) {
+    throw new Error('Storage partition is not configured.')
+  }
+
+  const stored = await storePartitionFile({
+    partitionCode: partition.code,
+    orgId: input.organizationId,
+    tenantId: input.tenantId,
+    fileName: input.fileName,
+    buffer: input.buffer,
+  })
+
+  const attachmentId = randomUUID()
+  const attachment = input.em.create(Attachment, {
+    id: attachmentId,
+    entityId: SYNC_EXCEL_ATTACHMENT_ENTITY_ID,
+    recordId: input.uploadId,
+    organizationId: input.organizationId,
+    tenantId: input.tenantId,
+    partitionCode: partition.code,
+    fileName: input.fileName,
+    mimeType: input.mimeType,
+    fileSize: input.buffer.length,
+    storageDriver: partition.storageDriver || 'local',
+    storagePath: stored.storagePath,
+    url: buildAttachmentFileUrl(attachmentId),
+    storageMetadata: {
+      module: 'sync_excel',
+      temporary: true,
+      uploadId: input.uploadId,
+      inlineCsvBase64: input.buffer.toString('base64'),
+    },
+  })
+
+  input.em.persist(attachment)
+  await input.em.flush()
+
+  return attachment
+}
+
+export async function readSyncExcelUploadBuffer(
+  attachment: Pick<Attachment, 'partitionCode' | 'storagePath' | 'storageDriver' | 'storageMetadata'>,
+): Promise<Buffer> {
+  const inlineCsvBase64 = attachment.storageMetadata?.inlineCsvBase64
+  if (typeof inlineCsvBase64 === 'string' && inlineCsvBase64.length > 0) {
+    return Buffer.from(inlineCsvBase64, 'base64')
+  }
+
+  const absolutePath = resolveAttachmentAbsolutePath(
+    attachment.partitionCode,
+    attachment.storagePath,
+    attachment.storageDriver,
+  )
+  return fs.readFile(absolutePath)
+}

package/src/modules/sync_excel/migrations/.snapshot-open-mercato.json

@@ -0,0 +1,199 @@
+{
+  "namespaces": [
+    "public"
+  ],
+  "name": "public",
+  "tables": [
+    {
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "uuid",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": false,
+          "default": "gen_random_uuid()",
+          "mappedType": "uuid"
+        },
+        "attachment_id": {
+          "name": "attachment_id",
+          "type": "uuid",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": false,
+          "mappedType": "uuid"
+        },
+        "filename": {
+          "name": "filename",
+          "type": "text",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": false,
+          "mappedType": "text"
+        },
+        "mime_type": {
+          "name": "mime_type",
+          "type": "text",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": false,
+          "mappedType": "text"
+        },
+        "file_size": {
+          "name": "file_size",
+          "type": "int",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": false,
+          "mappedType": "integer"
+        },
+        "entity_type": {
+          "name": "entity_type",
+          "type": "text",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": false,
+          "mappedType": "text"
+        },
+        "delimiter": {
+          "name": "delimiter",
+          "type": "text",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": true,
+          "mappedType": "text"
+        },
+        "encoding": {
+          "name": "encoding",
+          "type": "text",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": true,
+          "mappedType": "text"
+        },
+        "headers": {
+          "name": "headers",
+          "type": "jsonb",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": false,
+          "mappedType": "json"
+        },
+        "sample_rows": {
+          "name": "sample_rows",
+          "type": "jsonb",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": false,
+          "mappedType": "json"
+        },
+        "total_rows": {
+          "name": "total_rows",
+          "type": "int",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": false,
+          "mappedType": "integer"
+        },
+        "status": {
+          "name": "status",
+          "type": "text",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": false,
+          "default": "'uploaded'",
+          "mappedType": "text"
+        },
+        "sync_run_id": {
+          "name": "sync_run_id",
+          "type": "uuid",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": true,
+          "mappedType": "uuid"
+        },
+        "organization_id": {
+          "name": "organization_id",
+          "type": "uuid",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": false,
+          "mappedType": "uuid"
+        },
+        "tenant_id": {
+          "name": "tenant_id",
+          "type": "uuid",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": false,
+          "mappedType": "uuid"
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "timestamptz",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": false,
+          "length": 6,
+          "mappedType": "datetime"
+        },
+        "updated_at": {
+          "name": "updated_at",
+          "type": "timestamptz",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": false,
+          "length": 6,
+          "mappedType": "datetime"
+        }
+      },
+      "name": "sync_excel_uploads",
+      "schema": "public",
+      "indexes": [
+        {
+          "keyName": "sync_excel_uploads_organization_id_tenant_id_status_index",
+          "columnNames": [
+            "organization_id",
+            "tenant_id",
+            "status"
+          ],
+          "composite": true,
+          "constraint": false,
+          "primary": false,
+          "unique": false
+        },
+        {
+          "keyName": "sync_excel_uploads_pkey",
+          "columnNames": [
+            "id"
+          ],
+          "composite": false,
+          "constraint": true,
+          "primary": true,
+          "unique": true
+        }
+      ],
+      "checks": [],
+      "foreignKeys": {},
+      "nativeEnums": {}
+    }
+  ],
+  "nativeEnums": {}
+}

package/src/modules/sync_excel/migrations/Migration20260329181120_sync_excel.ts

@@ -0,0 +1,10 @@
+import { Migration } from '@mikro-orm/migrations';
+
+export class Migration20260329181120_sync_excel extends Migration {
+
+  override async up(): Promise<void> {
+    this.addSql(`create table "sync_excel_uploads" ("id" uuid not null default gen_random_uuid(), "attachment_id" uuid not null, "filename" text not null, "mime_type" text not null, "file_size" int not null, "entity_type" text not null, "delimiter" text null, "encoding" text null, "headers" jsonb not null, "sample_rows" jsonb not null, "total_rows" int not null, "status" text not null default 'uploaded', "sync_run_id" uuid null, "organization_id" uuid not null, "tenant_id" uuid not null, "created_at" timestamptz not null, "updated_at" timestamptz not null, constraint "sync_excel_uploads_pkey" primary key ("id"));`);
+    this.addSql(`create index "sync_excel_uploads_organization_id_tenant_id_status_index" on "sync_excel_uploads" ("organization_id", "tenant_id", "status");`);
+  }
+
+}

package/src/modules/sync_excel/setup.ts

@@ -0,0 +1,20 @@
+import type { ModuleSetupConfig } from '@open-mercato/shared/modules/setup'
+import type { CredentialsService } from '../integrations/lib/credentials-service'
+import type { IntegrationStateService } from '../integrations/lib/state-service'
+
+export const setup: ModuleSetupConfig = {
+  defaultRoleFeatures: {
+    superadmin: ['sync_excel.view', 'sync_excel.run'],
+    admin: ['sync_excel.view', 'sync_excel.run'],
+  },
+  async seedDefaults({ tenantId, organizationId, container }) {
+    const credentialsService = container.resolve('integrationCredentialsService') as CredentialsService
+    const integrationStateService = container.resolve('integrationStateService') as IntegrationStateService
+    const scope = { tenantId, organizationId }
+
+    await credentialsService.save('sync_excel', {}, scope)
+    await integrationStateService.upsert('sync_excel', { isEnabled: true }, scope)
+  },
+}
+
+export default setup