@open-mercato/core 0.6.8-develop.6993.1.3fc50f3409 → 0.6.8-develop.7000.1.0ae682bad1

package/src/modules/data_sync/lib/sync-engine.ts

@@ -1,5 +1,4 @@
 import type { EntityManager } from '@mikro-orm/postgresql'
-import { getIntegration } from '@open-mercato/shared/modules/integrations/types'
 import type { CredentialsService } from '../../integrations/lib/credentials-service'
 import type { IntegrationLogService } from '../../integrations/lib/log-service'
 import type { IntegrationStateService } from '../../integrations/lib/state-service'
@@ -7,14 +6,22 @@ import type { ProgressService } from '../../progress/lib/progressService'
 import { STALE_JOB_TIMEOUT_SECONDS } from '../../progress/lib/progressService'
 import { refreshCoverageSnapshot } from '../../query_index/lib/coverage'
 import { emitDataSyncEvent } from '../events'
-import type { DataSyncAdapter, DataMapping, ExportBatch, ImportBatch } from './adapter'
+import type { DataSyncAdapter, DataMapping, ExportBatch, ImportBatch, RunParameterValue } from './adapter'
 import { getDataSyncAdapter, resolveProviderKey } from './adapter-registry'
 import type { SyncRunService } from './sync-run-service'
 import { SyncRunOwnershipConflictError } from './sync-run-service'
+import { forEachBatch } from './batch-stream'
 import { createLogger } from '@open-mercato/shared/lib/logger'
+import {
+  captureTelemetryTrace,
+  type TelemetrySpanAttributes,
+} from '@open-mercato/shared/lib/telemetry/runtime'
+import type { SyncRun } from '../data/entities'
 
 const logger = createLogger('data_sync').child({ component: 'sync-engine' })
 
+type RunParameters = Record<string, RunParameterValue>
+
 type SyncScope = {
   organizationId: string
   tenantId: string
@@ -30,6 +37,19 @@ type EngineDeps = {
   progressService: ProgressService
 }
 
+/** Repeated on every batch span so a rooted batch trace identifies its run on its own. */
+function runSpanAttributes(run: SyncRun, providerKey: string, scope: SyncScope): TelemetrySpanAttributes {
+  return {
+    'data_sync.run_id': run.id,
+    'data_sync.integration_id': run.integrationId,
+    'data_sync.provider_key': providerKey,
+    'data_sync.entity_type': run.entityType,
+    'data_sync.direction': run.direction,
+    'om.tenant_id': scope.tenantId,
+    'om.organization_id': scope.organizationId,
+  }
+}
+
 function applyImportCounters(batch: ImportBatch): Pick<Required<SyncCounterDelta>, 'createdCount' | 'updatedCount' | 'skippedCount' | 'failedCount'> {
   let createdCount = 0
   let updatedCount = 0
@@ -539,66 +559,96 @@ export function createSyncEngine(deps: EngineDeps) {
       let processedCount = await seedProcessedCount(run.progressJobId, scope)
       let totalCount: number | null = null
       let committedBatches = activeRun.batchesCompleted ?? 0
+      // Captured while the triggering job's span is still the active one, so
+      // every rooted batch trace can link back to it.
+      const runTrace = captureTelemetryTrace()
+      const spanAttributes = runSpanAttributes(run, providerKey, scope)
 
       try {
-        for await (const batch of withHeartbeat(
-          adapter.streamImport({
-            entityType: run.entityType,
-            cursor: run.cursor ?? undefined,
-            batchSize,
-            credentials,
-            mapping,
-            scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },
-            runId: run.id,
-          }),
-          makeHeartbeatTick(run.progressJobId, scope),
-          HEARTBEAT_TICK_MS,
-        )) {
-          if (run.progressJobId && await progressService.isCancellationRequested(run.progressJobId, scope.tenantId, scope.organizationId)) {
-            await finalizeRun(run.id, 'cancelled', scope, undefined, operationalTelemetry)
-            return
-          }
-
-          const delta = applyImportCounters(batch)
-          const processedBatchCount = batch.processedCount ?? batch.items.length
-          processedCount += processedBatchCount
-          totalCount = batch.totalEstimate ?? totalCount
-
-          await syncRunService.commitBatchProgress(
-            run.id,
-            {
-              ...delta,
-              batchesCompleted: 1,
-            },
-            batch.cursor,
-            scope,
-            { expectedBatchesCompleted: committedBatches, persistSharedCursor },
-          )
-          committedBatches += 1
-
-          await updateProgress(run.progressJobId, processedCount, totalCount, scope)
-          await refreshCoverageSnapshots(batch.refreshCoverageEntityTypes, scope)
-          await logImportItemFailures(run.id, run.integrationId, batch.items, scope)
-
-          await writeOperationalLog({
-            integrationId: run.integrationId,
-            runId: run.id,
-            level: 'info',
-            message: batch.message?.trim().length
-              ? batch.message.trim()
-              : `Processed import batch ${batch.batchIndex}`,
-            scope,
-            enabled: operationalTelemetry,
-            payload: {
-              operationalStatus: 'running',
-              summary: `Processed ${processedCount}${totalCount ? ` of ${totalCount}` : ''} rows so far.`,
-              processedCount,
-              batchSize: batch.items.length,
-              processedBatchCount,
-              cursor: batch.cursor,
-            },
-          })
-        }
+        const streamResult = await forEachBatch(
+          withHeartbeat(
+            adapter.streamImport({
+              entityType: run.entityType,
+              cursor: run.cursor ?? undefined,
+              batchSize,
+              credentials,
+              mapping,
+              scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },
+              runId: run.id,
+              parameters: (run.parameters ?? {}) as RunParameters,
+            }),
+            makeHeartbeatTick(run.progressJobId, scope),
+            HEARTBEAT_TICK_MS,
+          ),
+          {
+            spanName: 'data_sync.import.batch',
+            drainSpanName: 'data_sync.import.drain',
+            attributes: spanAttributes,
+            linkTo: runTrace,
+          },
+          async (batch, span) => {
+            span.setAttributes({
+              'data_sync.batch_index': batch.batchIndex,
+              'data_sync.batch_size': batch.items.length,
+            })
+
+            if (run.progressJobId && await progressService.isCancellationRequested(run.progressJobId, scope.tenantId, scope.organizationId)) {
+              await finalizeRun(run.id, 'cancelled', scope, undefined, operationalTelemetry)
+              return 'stop'
+            }
+
+            const delta = applyImportCounters(batch)
+            const processedBatchCount = batch.processedCount ?? batch.items.length
+            processedCount += processedBatchCount
+            totalCount = batch.totalEstimate ?? totalCount
+
+            span.setAttributes({
+              'data_sync.processed_count': processedBatchCount,
+              'data_sync.created_count': delta.createdCount,
+              'data_sync.updated_count': delta.updatedCount,
+              'data_sync.skipped_count': delta.skippedCount,
+              'data_sync.failed_count': delta.failedCount,
+            })
+
+            await syncRunService.commitBatchProgress(
+              run.id,
+              {
+                ...delta,
+                batchesCompleted: 1,
+              },
+              batch.cursor,
+              scope,
+              { expectedBatchesCompleted: committedBatches, persistSharedCursor },
+            )
+            committedBatches += 1
+
+            await updateProgress(run.progressJobId, processedCount, totalCount, scope)
+            await refreshCoverageSnapshots(batch.refreshCoverageEntityTypes, scope)
+            await logImportItemFailures(run.id, run.integrationId, batch.items, scope)
+
+            await writeOperationalLog({
+              integrationId: run.integrationId,
+              runId: run.id,
+              level: 'info',
+              message: batch.message?.trim().length
+                ? batch.message.trim()
+                : `Processed import batch ${batch.batchIndex}`,
+              scope,
+              enabled: operationalTelemetry,
+              payload: {
+                operationalStatus: 'running',
+                summary: `Processed ${processedCount}${totalCount ? ` of ${totalCount}` : ''} rows so far.`,
+                processedCount,
+                batchSize: batch.items.length,
+                processedBatchCount,
+                cursor: batch.cursor,
+              },
+            })
+
+            return 'continue'
+          },
+        )
+        if (streamResult === 'stopped') return
       } catch (error) {
         if (error instanceof SyncRunOwnershipConflictError) {
           logger.warn('Yielding import run to a concurrent worker that already advanced it', {
@@ -703,62 +753,91 @@ export function createSyncEngine(deps: EngineDeps) {
       const mapping = await resolveMapping(adapter, run.entityType, scope)
       let processedCount = await seedProcessedCount(run.progressJobId, scope)
       let committedBatches = activeRun.batchesCompleted ?? 0
+      // Captured while the triggering job's span is still the active one, so
+      // every rooted batch trace can link back to it.
+      const runTrace = captureTelemetryTrace()
+      const spanAttributes = runSpanAttributes(run, providerKey, scope)
 
       try {
-        for await (const batch of withHeartbeat(
-          adapter.streamExport({
-            entityType: run.entityType,
-            cursor: run.cursor ?? undefined,
-            batchSize,
-            credentials,
-            mapping,
-            scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },
-            runId: run.id,
-          }),
-          makeHeartbeatTick(run.progressJobId, scope),
-          HEARTBEAT_TICK_MS,
-        )) {
-          if (run.progressJobId && await progressService.isCancellationRequested(run.progressJobId, scope.tenantId, scope.organizationId)) {
-            await finalizeRun(run.id, 'cancelled', scope, undefined, operationalTelemetry)
-            return
-          }
-
-          const delta = applyExportCounters(batch)
-          processedCount += delta.processedCount
-
-          await syncRunService.commitBatchProgress(
-            run.id,
-            {
-              createdCount: 0,
-              updatedCount: delta.updatedCount,
-              skippedCount: delta.skippedCount,
-              failedCount: delta.failedCount,
-              batchesCompleted: 1,
-            },
-            batch.cursor,
-            scope,
-            { expectedBatchesCompleted: committedBatches, persistSharedCursor },
-          )
-          committedBatches += 1
-          await updateProgress(run.progressJobId, processedCount, null, scope)
-          await logExportItemFailures(run.id, run.integrationId, batch.results, scope)
-
-          await writeOperationalLog({
-            integrationId: run.integrationId,
-            runId: run.id,
-            level: 'info',
-            message: `Processed export batch ${batch.batchIndex}`,
-            scope,
-            enabled: operationalTelemetry,
-            payload: {
-              operationalStatus: 'running',
-              summary: `Processed ${processedCount} export items so far.`,
-              processedCount,
-              batchSize: batch.results.length,
-              cursor: batch.cursor,
-            },
-          })
-        }
+        const streamResult = await forEachBatch(
+          withHeartbeat(
+            adapter.streamExport({
+              entityType: run.entityType,
+              cursor: run.cursor ?? undefined,
+              batchSize,
+              credentials,
+              mapping,
+              scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },
+              runId: run.id,
+              parameters: (run.parameters ?? {}) as RunParameters,
+            }),
+            makeHeartbeatTick(run.progressJobId, scope),
+            HEARTBEAT_TICK_MS,
+          ),
+          {
+            spanName: 'data_sync.export.batch',
+            drainSpanName: 'data_sync.export.drain',
+            attributes: spanAttributes,
+            linkTo: runTrace,
+          },
+          async (batch, span) => {
+            span.setAttributes({
+              'data_sync.batch_index': batch.batchIndex,
+              'data_sync.batch_size': batch.results.length,
+            })
+
+            if (run.progressJobId && await progressService.isCancellationRequested(run.progressJobId, scope.tenantId, scope.organizationId)) {
+              await finalizeRun(run.id, 'cancelled', scope, undefined, operationalTelemetry)
+              return 'stop'
+            }
+
+            const delta = applyExportCounters(batch)
+            processedCount += delta.processedCount
+
+            span.setAttributes({
+              'data_sync.processed_count': delta.processedCount,
+              'data_sync.updated_count': delta.updatedCount,
+              'data_sync.skipped_count': delta.skippedCount,
+              'data_sync.failed_count': delta.failedCount,
+            })
+
+            await syncRunService.commitBatchProgress(
+              run.id,
+              {
+                createdCount: 0,
+                updatedCount: delta.updatedCount,
+                skippedCount: delta.skippedCount,
+                failedCount: delta.failedCount,
+                batchesCompleted: 1,
+              },
+              batch.cursor,
+              scope,
+              { expectedBatchesCompleted: committedBatches, persistSharedCursor },
+            )
+            committedBatches += 1
+            await updateProgress(run.progressJobId, processedCount, null, scope)
+            await logExportItemFailures(run.id, run.integrationId, batch.results, scope)
+
+            await writeOperationalLog({
+              integrationId: run.integrationId,
+              runId: run.id,
+              level: 'info',
+              message: `Processed export batch ${batch.batchIndex}`,
+              scope,
+              enabled: operationalTelemetry,
+              payload: {
+                operationalStatus: 'running',
+                summary: `Processed ${processedCount} export items so far.`,
+                processedCount,
+                batchSize: batch.results.length,
+                cursor: batch.cursor,
+              },
+            })
+
+            return 'continue'
+          },
+        )
+        if (streamResult === 'stopped') return
       } catch (error) {
         if (error instanceof SyncRunOwnershipConflictError) {
           logger.warn('Yielding export run to a concurrent worker that already advanced it', {

package/src/modules/data_sync/lib/sync-run-service.ts

@@ -115,6 +115,7 @@ export function createSyncRunService(em: EntityManager) {
       triggeredBy?: string | null
       progressJobId?: string | null
       jobId?: string | null
+      parameters?: Record<string, unknown> | null
     }, scope: SyncScope): Promise<SyncRun> {
       const row = em.create(SyncRun, {
         integrationId: input.integrationId,
@@ -126,6 +127,7 @@ export function createSyncRunService(em: EntityManager) {
         triggeredBy: input.triggeredBy,
         progressJobId: input.progressJobId,
         jobId: input.jobId,
+        parameters: input.parameters ?? null,
         organizationId: scope.organizationId,
         tenantId: scope.tenantId,
       })

package/src/modules/data_sync/migrations/.snapshot-open-mercato.json

@@ -668,6 +668,22 @@
           "comment": null,
           "enumItems": [],
           "mappedType": "integer"
+        },
+        "parameters": {
+          "name": "parameters",
+          "type": "jsonb",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": true,
+          "unique": false,
+          "length": null,
+          "precision": null,
+          "scale": null,
+          "default": null,
+          "comment": null,
+          "enumItems": [],
+          "mappedType": "json"
         }
       },
       "indexes": [
@@ -994,4 +1010,4 @@
   ],
   "views": [],
   "nativeEnums": {}
-}
+}

package/src/modules/data_sync/migrations/Migration20260810120000.ts

@@ -0,0 +1,13 @@
+import { Migration } from '@mikro-orm/migrations';
+
+export class Migration20260810120000 extends Migration {
+
+  override async up(): Promise<void> {
+    this.addSql(`alter table "sync_runs" add column "parameters" jsonb null;`);
+  }
+
+  override async down(): Promise<void> {
+    this.addSql(`alter table "sync_runs" drop column "parameters";`);
+  }
+
+}

package/src/modules/data_sync/workers/sync-scheduled.ts

@@ -7,6 +7,10 @@ import type { SyncRunService } from '../lib/sync-run-service'
 import { SyncSchedule } from '../data/entities'
 import { startDataSyncRun } from '../lib/start-run'
 import { resolveAdapterForIntegration, resolveStartCursor } from '../lib/start-cursor'
+import { normalizeRunParameters } from '../lib/run-parameters'
+import { createLogger } from '@open-mercato/shared/lib/logger'
+
+const logger = createLogger('data_sync').child({ component: 'sync-scheduled' })
 
 type ScheduledSyncPayload = {
   scheduleId: string
@@ -64,17 +68,43 @@ export default async function handle(job: QueuedJob<ScheduledSyncPayload>, ctx:
     return
   }
 
+  const adapter = resolveAdapterForIntegration(schedule.integrationId)
+
   const cursor = schedule.fullSync
     ? null
     : await resolveStartCursor({
         syncRunService,
-        adapter: resolveAdapterForIntegration(schedule.integrationId),
+        adapter,
         integrationId: schedule.integrationId,
         entityType: schedule.entityType,
         direction: schedule.direction,
         scope: job.payload.scope,
       })
 
+  // A schedule carries no parameter form, but the adapter's declared defaults
+  // still apply: normalizing an empty input materializes exactly those, so a
+  // scheduled run reaches the adapter with the same set a manual run would
+  // rather than an empty object. A declaration whose own default is invalid
+  // stops the run instead of handing over a half-applied set.
+  const normalizedParameters = normalizeRunParameters(
+    adapter?.runParameters,
+    schedule.direction,
+    null,
+    schedule.entityType,
+  )
+  if (!normalizedParameters.ok) {
+    logger.error('Scheduled run skipped: adapter declares invalid run parameter defaults', {
+      scheduleId: schedule.id,
+      integrationId: schedule.integrationId,
+      entityType: schedule.entityType,
+      keys: normalizedParameters.errors.map((error) => error.key),
+    })
+    return
+  }
+  const parameters = Object.keys(normalizedParameters.values).length > 0
+    ? normalizedParameters.values
+    : null
+
   schedule.lastRunAt = new Date()
   await em.flush()
 
@@ -88,6 +118,7 @@ export default async function handle(job: QueuedJob<ScheduledSyncPayload>, ctx:
       direction: schedule.direction,
       cursor,
       triggeredBy: 'scheduler',
+      parameters,
     },
   })
 }

package/src/modules/inbox_ops/api/emails/[id]/reprocess/route.ts

@@ -75,6 +75,10 @@ export async function POST(req: Request) {
         organizationId: email.organizationId,
         forwardedByAddress: email.forwardedByAddress,
         subject: email.subject,
+      }, {
+        persistent: true,
+        tenantId: email.tenantId,
+        organizationId: email.organizationId,
       })
     } catch (eventError) {
       logger.error('Failed to emit events', { err: eventError })

package/src/modules/inbox_ops/api/extract/route.ts

@@ -70,6 +70,10 @@ export async function POST(req: Request) {
         organizationId: ctx.organizationId,
         forwardedByAddress: submitterEmail,
         subject: title || 'Text extraction',
+      }, {
+        persistent: true,
+        tenantId: ctx.tenantId,
+        organizationId: ctx.organizationId,
       })
     } catch (eventError) {
       logger.error('Failed to emit email.received event', { err: eventError })

package/src/modules/inbox_ops/api/webhook/inbound.ts

@@ -396,6 +396,10 @@ export async function POST(req: Request) {
       organizationId: settings.organizationId,
       forwardedByAddress: parsed.from.email,
       subject: parsed.subject,
+    }, {
+      persistent: true,
+      tenantId: settings.tenantId,
+      organizationId: settings.organizationId,
     })
   } catch (eventError) {
     logger.error('Failed to emit email.received event', { err: eventError })