@open-mercato/core 0.6.8-develop.6985.1.fb93574faa → 0.6.8-develop.6987.1.02f619470c

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,5 @@
1
1
  import { getIntegration } from "@open-mercato/shared/modules/integrations/types";
2
+ import { STALE_JOB_TIMEOUT_SECONDS } from "../../progress/lib/progressService.js";
2
3
  import { refreshCoverageSnapshot } from "../../query_index/lib/coverage.js";
3
4
  import { emitDataSyncEvent } from "../events.js";
4
5
  import { getDataSyncAdapter } from "./adapter-registry.js";
@@ -37,6 +38,25 @@ function applyExportCounters(batch) {
37
38
  processedCount: batch.results.length
38
39
  };
39
40
  }
41
+ const HEARTBEAT_TICK_MS = STALE_JOB_TIMEOUT_SECONDS * 1e3 / 4;
42
+ async function* withHeartbeat(source, tick, intervalMs) {
43
+ const iterator = source[Symbol.asyncIterator]();
44
+ try {
45
+ while (true) {
46
+ const timer = setInterval(tick, intervalMs);
47
+ let result;
48
+ try {
49
+ result = await iterator.next();
50
+ } finally {
51
+ clearInterval(timer);
52
+ }
53
+ if (result.done) return;
54
+ yield result.value;
55
+ }
56
+ } finally {
57
+ await iterator.return?.();
58
+ }
59
+ }
40
60
  function createSyncEngine(deps) {
41
61
  const { syncRunService, integrationCredentialsService, integrationLogService, integrationStateService, progressService } = deps;
42
62
  async function resolveMapping(adapter, entityType, scope) {
@@ -60,6 +80,37 @@ function createSyncEngine(deps) {
60
80
  }
61
81
  );
62
82
  }
83
+ async function seedProcessedCount(progressJobId, scope) {
84
+ if (!progressJobId) return 0;
85
+ const job = await progressService.getJob(progressJobId, {
86
+ tenantId: scope.tenantId,
87
+ organizationId: scope.organizationId,
88
+ userId: scope.userId
89
+ });
90
+ return job?.processedCount ?? 0;
91
+ }
92
+ function makeHeartbeatTick(progressJobId, scope) {
93
+ const touchJobHeartbeat = progressService.touchJobHeartbeat?.bind(progressService);
94
+ if (!progressJobId || !touchJobHeartbeat) return () => {
95
+ };
96
+ let inFlight = false;
97
+ return () => {
98
+ if (inFlight) return;
99
+ inFlight = true;
100
+ touchJobHeartbeat(progressJobId, {
101
+ tenantId: scope.tenantId,
102
+ organizationId: scope.organizationId,
103
+ userId: scope.userId
104
+ }).catch((error) => {
105
+ logger.warn("Progress heartbeat failed", {
106
+ progressJobId,
107
+ error: error instanceof Error ? error.message : String(error)
108
+ });
109
+ }).finally(() => {
110
+ inFlight = false;
111
+ });
112
+ };
113
+ }
63
114
  async function refreshCoverageSnapshots(entityTypes, scope) {
64
115
  if (!entityTypes || entityTypes.length === 0) return;
65
116
  await Promise.allSettled(
@@ -358,19 +409,23 @@ function createSyncEngine(deps) {
358
409
  });
359
410
  }
360
411
  const mapping = await resolveMapping(adapter, run.entityType, scope);
361
- let processedCount = 0;
412
+ let processedCount = await seedProcessedCount(run.progressJobId, scope);
362
413
  let totalCount = null;
363
414
  let committedBatches = activeRun.batchesCompleted ?? 0;
364
415
  try {
365
- for await (const batch of adapter.streamImport({
366
- entityType: run.entityType,
367
- cursor: run.cursor ?? void 0,
368
- batchSize,
369
- credentials,
370
- mapping,
371
- scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },
372
- runId: run.id
373
- })) {
416
+ for await (const batch of withHeartbeat(
417
+ adapter.streamImport({
418
+ entityType: run.entityType,
419
+ cursor: run.cursor ?? void 0,
420
+ batchSize,
421
+ credentials,
422
+ mapping,
423
+ scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },
424
+ runId: run.id
425
+ }),
426
+ makeHeartbeatTick(run.progressJobId, scope),
427
+ HEARTBEAT_TICK_MS
428
+ )) {
374
429
  if (run.progressJobId && await progressService.isCancellationRequested(run.progressJobId, scope.tenantId, scope.organizationId)) {
375
430
  await finalizeRun(run.id, "cancelled", scope, void 0, operationalTelemetry);
376
431
  return;
@@ -501,18 +556,22 @@ function createSyncEngine(deps) {
501
556
  });
502
557
  }
503
558
  const mapping = await resolveMapping(adapter, run.entityType, scope);
504
- let processedCount = 0;
559
+ let processedCount = await seedProcessedCount(run.progressJobId, scope);
505
560
  let committedBatches = activeRun.batchesCompleted ?? 0;
506
561
  try {
507
- for await (const batch of adapter.streamExport({
508
- entityType: run.entityType,
509
- cursor: run.cursor ?? void 0,
510
- batchSize,
511
- credentials,
512
- mapping,
513
- scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },
514
- runId: run.id
515
- })) {
562
+ for await (const batch of withHeartbeat(
563
+ adapter.streamExport({
564
+ entityType: run.entityType,
565
+ cursor: run.cursor ?? void 0,
566
+ batchSize,
567
+ credentials,
568
+ mapping,
569
+ scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },
570
+ runId: run.id
571
+ }),
572
+ makeHeartbeatTick(run.progressJobId, scope),
573
+ HEARTBEAT_TICK_MS
574
+ )) {
516
575
  if (run.progressJobId && await progressService.isCancellationRequested(run.progressJobId, scope.tenantId, scope.organizationId)) {
517
576
  await finalizeRun(run.id, "cancelled", scope, void 0, operationalTelemetry);
518
577
  return;
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "version": 3,
3
3
  "sources": ["../../../../src/modules/data_sync/lib/sync-engine.ts"],
4
- "sourcesContent": ["import type { EntityManager } from '@mikro-orm/postgresql'\nimport { getIntegration } from '@open-mercato/shared/modules/integrations/types'\nimport type { CredentialsService } from '../../integrations/lib/credentials-service'\nimport type { IntegrationLogService } from '../../integrations/lib/log-service'\nimport type { IntegrationStateService } from '../../integrations/lib/state-service'\nimport type { ProgressService } from '../../progress/lib/progressService'\nimport { refreshCoverageSnapshot } from '../../query_index/lib/coverage'\nimport { emitDataSyncEvent } from '../events'\nimport type { DataSyncAdapter, DataMapping, ExportBatch, ImportBatch } from './adapter'\nimport { getDataSyncAdapter } from './adapter-registry'\nimport type { SyncRunService } from './sync-run-service'\nimport { SyncRunOwnershipConflictError } from './sync-run-service'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\n\nconst logger = createLogger('data_sync').child({ component: 'sync-engine' })\n\ntype SyncScope = {\n organizationId: string\n tenantId: string\n userId?: string | null\n}\n\ntype EngineDeps = {\n em: EntityManager\n syncRunService: SyncRunService\n integrationCredentialsService: CredentialsService\n integrationLogService: IntegrationLogService\n integrationStateService?: IntegrationStateService\n progressService: ProgressService\n}\n\nfunction resolveProviderKey(integrationId: string): string {\n return getIntegration(integrationId)?.providerKey ?? integrationId\n}\n\nfunction applyImportCounters(batch: ImportBatch): Pick<Required<SyncCounterDelta>, 'createdCount' | 'updatedCount' | 'skippedCount' | 'failedCount'> {\n let createdCount = 0\n let updatedCount = 0\n let skippedCount = 0\n let failedCount = 0\n\n for (const item of batch.items) {\n if (item.action === 'create') createdCount += 1\n else if (item.action === 'update') updatedCount += 1\n else if (item.action === 'failed') failedCount += 1\n else skippedCount += 1\n }\n\n return { createdCount, updatedCount, skippedCount, failedCount }\n}\n\ntype SyncCounterDelta = {\n createdCount?: number\n updatedCount?: number\n skippedCount?: number\n failedCount?: number\n processedCount: number\n}\n\nfunction applyExportCounters(batch: ExportBatch): SyncCounterDelta {\n let failedCount = 0\n let skippedCount = 0\n let updatedCount = 0\n\n for (const result of batch.results) {\n if (result.status === 'error') failedCount += 1\n else if (result.status === 'skipped') skippedCount += 1\n else updatedCount += 1\n }\n\n return {\n failedCount,\n skippedCount,\n updatedCount,\n processedCount: batch.results.length,\n }\n}\n\nexport function createSyncEngine(deps: EngineDeps) {\n const { syncRunService, integrationCredentialsService, integrationLogService, integrationStateService, progressService } = deps\n\n async function resolveMapping(adapter: DataSyncAdapter, entityType: string, scope: SyncScope): Promise<DataMapping> {\n return adapter.getMapping({\n entityType,\n scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },\n })\n }\n\n async function updateProgress(progressJobId: string | null | undefined, processedCount: number, totalCount: number | null, scope: SyncScope): Promise<void> {\n if (!progressJobId) return\n\n await progressService.updateProgress(\n progressJobId,\n {\n processedCount,\n totalCount: totalCount ?? undefined,\n },\n {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n },\n )\n }\n\n async function refreshCoverageSnapshots(entityTypes: string[] | undefined, scope: SyncScope): Promise<void> {\n if (!entityTypes || entityTypes.length === 0) return\n\n await Promise.allSettled(\n Array.from(new Set(entityTypes.filter((value) => typeof value === 'string' && value.trim().length > 0)))\n .map((entityType) => refreshCoverageSnapshot(deps.em, {\n entityType,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n })),\n )\n }\n\n async function logImportItemFailures(\n runId: string,\n integrationId: string,\n items: ImportBatch['items'],\n scope: SyncScope,\n ): Promise<void> {\n const failedItems = items.filter((item) => item.action === 'failed')\n for (const item of failedItems) {\n const errorMessage = typeof item.data.errorMessage === 'string' && item.data.errorMessage.trim().length > 0\n ? item.data.errorMessage.trim()\n : 'Import item failed'\n const sourceProductUuid = typeof item.data.sourceProductUuid === 'string' && item.data.sourceProductUuid.trim().length > 0\n ? item.data.sourceProductUuid.trim()\n : null\n const sourceIdentifier = typeof item.data.sourceIdentifier === 'string' && item.data.sourceIdentifier.trim().length > 0\n ? item.data.sourceIdentifier.trim()\n : null\n const message = [\n `Failed to import item ${item.externalId}`,\n sourceProductUuid ? `(uuid: ${sourceProductUuid})` : null,\n sourceIdentifier ? `(identifier: ${sourceIdentifier})` : null,\n `: ${errorMessage}`,\n ].filter((part) => part !== null).join(' ')\n\n await integrationLogService.write(\n {\n integrationId,\n runId,\n level: 'error',\n message,\n payload: item.data,\n },\n scope,\n )\n }\n }\n\n async function logExportItemFailures(\n runId: string,\n integrationId: string,\n results: ExportBatch['results'],\n scope: SyncScope,\n ): Promise<void> {\n const failedResults = results.filter((result) => result.status === 'error' && result.error)\n for (const result of failedResults) {\n const label = result.externalId ? `${result.externalId} (id: ${result.localId})` : result.localId\n const errorMessage = result.error!.split('\\n')[0]\n const message = `Failed to export item ${label}: ${errorMessage}`\n\n await integrationLogService.write(\n {\n integrationId,\n runId,\n level: 'error',\n message,\n payload: { kind: 'export-item-failure', summary: result.error },\n },\n scope,\n )\n }\n }\n\n async function writeOperationalLog(params: {\n integrationId: string\n runId: string\n level: 'info' | 'warn' | 'error'\n message: string\n scope: SyncScope\n enabled: boolean\n payload?: Record<string, unknown>\n }): Promise<void> {\n if (!params.enabled) return\n\n await integrationLogService.write(\n {\n integrationId: params.integrationId,\n runId: params.runId,\n level: params.level,\n message: params.message,\n payload: params.payload,\n },\n params.scope,\n )\n }\n\n async function updateOperationalState(params: {\n integrationId: string\n status: 'healthy' | 'degraded' | 'unhealthy'\n scope: SyncScope\n enabled: boolean\n }): Promise<void> {\n if (!params.enabled || !integrationStateService) return\n\n await integrationStateService.upsert(\n params.integrationId,\n {\n lastHealthStatus: params.status,\n lastHealthCheckedAt: new Date(),\n },\n params.scope,\n )\n }\n\n async function finalizeRun(\n runId: string,\n status: 'completed' | 'failed' | 'cancelled',\n scope: SyncScope,\n error?: string,\n operationalTelemetry = false,\n ): Promise<void> {\n const existingRun = await syncRunService.getRun(runId, scope)\n const alreadyFinalizedWithSameStatus = existingRun?.status === status\n && (status === 'completed' || status === 'failed' || status === 'cancelled')\n\n const run = await syncRunService.markStatus(runId, status, scope, error)\n if (!run) return\n\n if (alreadyFinalizedWithSameStatus) {\n return\n }\n\n if (run.status !== status) {\n // `markStatus` refuses a terminal -> different-terminal transition and\n // returns the row unchanged, so the run is already finished under another\n // delivery of this job. Everything below \u2014 the progress job, the\n // operational log and the lifecycle event \u2014 would describe the wrong\n // outcome, and `data_sync.run.failed` is dispatched to tenant webhooks.\n // A displaced worker stays silent instead.\n logger.warn('Skipping finalization of a sync run another worker already finalized', {\n runId,\n requestedStatus: status,\n actualStatus: run.status,\n })\n return\n }\n\n if (run.progressJobId) {\n if (status === 'completed') {\n await progressService.completeJob(\n run.progressJobId,\n {\n resultSummary: {\n createdCount: run.createdCount,\n updatedCount: run.updatedCount,\n skippedCount: run.skippedCount,\n failedCount: run.failedCount,\n batchesCompleted: run.batchesCompleted,\n },\n },\n {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n },\n )\n } else if (status === 'failed') {\n await progressService.failJob(\n run.progressJobId,\n {\n errorMessage: error ?? 'Sync run failed',\n },\n {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n },\n )\n } else if (status === 'cancelled') {\n await progressService.markCancelled(\n run.progressJobId,\n {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n },\n )\n }\n }\n\n if (status === 'completed') {\n await updateOperationalState({\n integrationId: run.integrationId,\n status: 'healthy',\n scope,\n enabled: operationalTelemetry,\n })\n await writeOperationalLog({\n integrationId: run.integrationId,\n runId: run.id,\n level: 'info',\n message: 'Sync run completed',\n scope,\n enabled: operationalTelemetry,\n payload: {\n operationalStatus: 'completed',\n summary: `Sync completed with ${run.createdCount} created, ${run.updatedCount} updated, ${run.failedCount} failed.`,\n createdCount: run.createdCount,\n updatedCount: run.updatedCount,\n skippedCount: run.skippedCount,\n failedCount: run.failedCount,\n batchesCompleted: run.batchesCompleted,\n },\n })\n } else if (status === 'cancelled') {\n await updateOperationalState({\n integrationId: run.integrationId,\n status: 'degraded',\n scope,\n enabled: operationalTelemetry,\n })\n await writeOperationalLog({\n integrationId: run.integrationId,\n runId: run.id,\n level: 'warn',\n message: 'Sync run cancelled',\n scope,\n enabled: operationalTelemetry,\n payload: {\n operationalStatus: 'cancelled',\n summary: 'The sync run was cancelled before completion.',\n },\n })\n } else {\n await updateOperationalState({\n integrationId: run.integrationId,\n status: 'unhealthy',\n scope,\n enabled: operationalTelemetry,\n })\n await writeOperationalLog({\n integrationId: run.integrationId,\n runId: run.id,\n level: 'error',\n message: error ?? 'Sync run failed',\n scope,\n enabled: operationalTelemetry,\n payload: {\n operationalStatus: 'failed',\n summary: error ?? 'The sync run failed.',\n },\n })\n }\n\n if (status === 'completed') {\n await emitDataSyncEvent('data_sync.run.completed', {\n runId,\n integrationId: run.integrationId,\n entityType: run.entityType,\n direction: run.direction,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n })\n return\n }\n\n if (status === 'cancelled') {\n await emitDataSyncEvent('data_sync.run.cancelled', {\n runId,\n integrationId: run.integrationId,\n entityType: run.entityType,\n direction: run.direction,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n })\n return\n }\n\n await emitDataSyncEvent('data_sync.run.failed', {\n runId,\n integrationId: run.integrationId,\n entityType: run.entityType,\n direction: run.direction,\n error: error ?? null,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n })\n }\n\n return {\n async runImport(runId: string, batchSize: number, scope: SyncScope): Promise<void> {\n const run = await syncRunService.getRun(runId, scope)\n if (!run) {\n logger.warn('Skipping stale import job for missing run', { runId })\n return\n }\n if (run.status === 'cancelled') {\n if (run.progressJobId) {\n await progressService.markCancelled(run.progressJobId, {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n })\n }\n return\n }\n\n const providerKey = resolveProviderKey(run.integrationId)\n const adapter = getDataSyncAdapter(providerKey)\n if (!adapter?.streamImport) {\n throw new Error(`No import adapter registered for provider ${providerKey}`)\n }\n const operationalTelemetry = adapter.operationalTelemetry === true\n\n const credentials = await integrationCredentialsService.resolve(run.integrationId, scope)\n if (!credentials) {\n throw new Error(`Integration ${run.integrationId} is missing credentials`)\n }\n\n // A run already `running` means a stalled job was redelivered, so this is\n // a resume rather than a first start. Consumers see one `started` event per\n // delivery either way; the flag is what lets them tell the two apart.\n const resumed = run.status === 'running'\n const activeRun = await syncRunService.markStatus(run.id, 'running', scope)\n if (!activeRun || activeRun.status !== 'running') {\n return\n }\n await emitDataSyncEvent('data_sync.run.started', {\n runId: run.id,\n integrationId: run.integrationId,\n entityType: run.entityType,\n direction: run.direction,\n resumed,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n })\n await updateOperationalState({\n integrationId: run.integrationId,\n status: 'degraded',\n scope,\n enabled: operationalTelemetry,\n })\n await writeOperationalLog({\n integrationId: run.integrationId,\n runId: run.id,\n level: 'info',\n message: 'Sync run started',\n scope,\n enabled: operationalTelemetry,\n payload: {\n operationalStatus: 'running',\n summary: `Import run started for ${run.entityType}.`,\n entityType: run.entityType,\n direction: run.direction,\n },\n })\n\n if (run.progressJobId) {\n await progressService.startJob(run.progressJobId, {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n })\n }\n\n const mapping = await resolveMapping(adapter, run.entityType, scope)\n let processedCount = 0\n let totalCount: number | null = null\n let committedBatches = activeRun.batchesCompleted ?? 0\n\n try {\n for await (const batch of adapter.streamImport({\n entityType: run.entityType,\n cursor: run.cursor ?? undefined,\n batchSize,\n credentials,\n mapping,\n scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },\n runId: run.id,\n })) {\n if (run.progressJobId && await progressService.isCancellationRequested(run.progressJobId, scope.tenantId, scope.organizationId)) {\n await finalizeRun(run.id, 'cancelled', scope, undefined, operationalTelemetry)\n return\n }\n\n const delta = applyImportCounters(batch)\n const processedBatchCount = batch.processedCount ?? batch.items.length\n processedCount += processedBatchCount\n totalCount = batch.totalEstimate ?? totalCount\n\n await syncRunService.commitBatchProgress(\n run.id,\n {\n ...delta,\n batchesCompleted: 1,\n },\n batch.cursor,\n scope,\n committedBatches,\n )\n committedBatches += 1\n\n await updateProgress(run.progressJobId, processedCount, totalCount, scope)\n await refreshCoverageSnapshots(batch.refreshCoverageEntityTypes, scope)\n await logImportItemFailures(run.id, run.integrationId, batch.items, scope)\n\n await writeOperationalLog({\n integrationId: run.integrationId,\n runId: run.id,\n level: 'info',\n message: batch.message?.trim().length\n ? batch.message.trim()\n : `Processed import batch ${batch.batchIndex}`,\n scope,\n enabled: operationalTelemetry,\n payload: {\n operationalStatus: 'running',\n summary: `Processed ${processedCount}${totalCount ? ` of ${totalCount}` : ''} rows so far.`,\n processedCount,\n batchSize: batch.items.length,\n processedBatchCount,\n cursor: batch.cursor,\n },\n })\n }\n } catch (error) {\n if (error instanceof SyncRunOwnershipConflictError) {\n logger.warn('Yielding import run to a concurrent worker that already advanced it', {\n runId: run.id,\n expectedBatchesCompleted: error.expectedBatchesCompleted,\n })\n return\n }\n const message = error instanceof Error ? error.message : 'Sync import failed'\n await integrationLogService.write(\n {\n integrationId: run.integrationId,\n runId: run.id,\n level: 'error',\n message,\n },\n scope,\n )\n await finalizeRun(run.id, 'failed', scope, message, operationalTelemetry)\n return\n }\n\n await finalizeRun(run.id, 'completed', scope, undefined, operationalTelemetry)\n },\n\n async runExport(runId: string, batchSize: number, scope: SyncScope): Promise<void> {\n const run = await syncRunService.getRun(runId, scope)\n if (!run) {\n logger.warn('Skipping stale export job for missing run', { runId })\n return\n }\n if (run.status === 'cancelled') {\n if (run.progressJobId) {\n await progressService.markCancelled(run.progressJobId, {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n })\n }\n return\n }\n\n const providerKey = resolveProviderKey(run.integrationId)\n const adapter = getDataSyncAdapter(providerKey)\n if (!adapter?.streamExport) {\n throw new Error(`No export adapter registered for provider ${providerKey}`)\n }\n const operationalTelemetry = adapter.operationalTelemetry === true\n\n const credentials = await integrationCredentialsService.resolve(run.integrationId, scope)\n if (!credentials) {\n throw new Error(`Integration ${run.integrationId} is missing credentials`)\n }\n\n // A run already `running` means a stalled job was redelivered, so this is\n // a resume rather than a first start. Consumers see one `started` event per\n // delivery either way; the flag is what lets them tell the two apart.\n const resumed = run.status === 'running'\n const activeRun = await syncRunService.markStatus(run.id, 'running', scope)\n if (!activeRun || activeRun.status !== 'running') {\n return\n }\n await emitDataSyncEvent('data_sync.run.started', {\n runId: run.id,\n integrationId: run.integrationId,\n entityType: run.entityType,\n direction: run.direction,\n resumed,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n })\n await updateOperationalState({\n integrationId: run.integrationId,\n status: 'degraded',\n scope,\n enabled: operationalTelemetry,\n })\n await writeOperationalLog({\n integrationId: run.integrationId,\n runId: run.id,\n level: 'info',\n message: 'Sync run started',\n scope,\n enabled: operationalTelemetry,\n payload: {\n operationalStatus: 'running',\n summary: `Export run started for ${run.entityType}.`,\n entityType: run.entityType,\n direction: run.direction,\n },\n })\n\n if (run.progressJobId) {\n await progressService.startJob(run.progressJobId, {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n })\n }\n\n const mapping = await resolveMapping(adapter, run.entityType, scope)\n let processedCount = 0\n let committedBatches = activeRun.batchesCompleted ?? 0\n\n try {\n for await (const batch of adapter.streamExport({\n entityType: run.entityType,\n cursor: run.cursor ?? undefined,\n batchSize,\n credentials,\n mapping,\n scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },\n runId: run.id,\n })) {\n if (run.progressJobId && await progressService.isCancellationRequested(run.progressJobId, scope.tenantId, scope.organizationId)) {\n await finalizeRun(run.id, 'cancelled', scope, undefined, operationalTelemetry)\n return\n }\n\n const delta = applyExportCounters(batch)\n processedCount += delta.processedCount\n\n await syncRunService.commitBatchProgress(\n run.id,\n {\n createdCount: 0,\n updatedCount: delta.updatedCount,\n skippedCount: delta.skippedCount,\n failedCount: delta.failedCount,\n batchesCompleted: 1,\n },\n batch.cursor,\n scope,\n committedBatches,\n )\n committedBatches += 1\n await updateProgress(run.progressJobId, processedCount, null, scope)\n await logExportItemFailures(run.id, run.integrationId, batch.results, scope)\n\n await writeOperationalLog({\n integrationId: run.integrationId,\n runId: run.id,\n level: 'info',\n message: `Processed export batch ${batch.batchIndex}`,\n scope,\n enabled: operationalTelemetry,\n payload: {\n operationalStatus: 'running',\n summary: `Processed ${processedCount} export items so far.`,\n processedCount,\n batchSize: batch.results.length,\n cursor: batch.cursor,\n },\n })\n }\n } catch (error) {\n if (error instanceof SyncRunOwnershipConflictError) {\n logger.warn('Yielding export run to a concurrent worker that already advanced it', {\n runId: run.id,\n expectedBatchesCompleted: error.expectedBatchesCompleted,\n })\n return\n }\n const message = error instanceof Error ? error.message : 'Sync export failed'\n await integrationLogService.write(\n {\n integrationId: run.integrationId,\n runId: run.id,\n level: 'error',\n message,\n },\n scope,\n )\n await finalizeRun(run.id, 'failed', scope, message, operationalTelemetry)\n return\n }\n\n await finalizeRun(run.id, 'completed', scope, undefined, operationalTelemetry)\n },\n }\n}\n\nexport type SyncEngine = ReturnType<typeof createSyncEngine>\n"],
5
- "mappings": "AACA,SAAS,sBAAsB;AAK/B,SAAS,+BAA+B;AACxC,SAAS,yBAAyB;AAElC,SAAS,0BAA0B;AAEnC,SAAS,qCAAqC;AAC9C,SAAS,oBAAoB;AAE7B,MAAM,SAAS,aAAa,WAAW,EAAE,MAAM,EAAE,WAAW,cAAc,CAAC;AAiB3E,SAAS,mBAAmB,eAA+B;AACzD,SAAO,eAAe,aAAa,GAAG,eAAe;AACvD;AAEA,SAAS,oBAAoB,OAAwH;AACnJ,MAAI,eAAe;AACnB,MAAI,eAAe;AACnB,MAAI,eAAe;AACnB,MAAI,cAAc;AAElB,aAAW,QAAQ,MAAM,OAAO;AAC9B,QAAI,KAAK,WAAW,SAAU,iBAAgB;AAAA,aACrC,KAAK,WAAW,SAAU,iBAAgB;AAAA,aAC1C,KAAK,WAAW,SAAU,gBAAe;AAAA,QAC7C,iBAAgB;AAAA,EACvB;AAEA,SAAO,EAAE,cAAc,cAAc,cAAc,YAAY;AACjE;AAUA,SAAS,oBAAoB,OAAsC;AACjE,MAAI,cAAc;AAClB,MAAI,eAAe;AACnB,MAAI,eAAe;AAEnB,aAAW,UAAU,MAAM,SAAS;AAClC,QAAI,OAAO,WAAW,QAAS,gBAAe;AAAA,aACrC,OAAO,WAAW,UAAW,iBAAgB;AAAA,QACjD,iBAAgB;AAAA,EACvB;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA,gBAAgB,MAAM,QAAQ;AAAA,EAChC;AACF;AAEO,SAAS,iBAAiB,MAAkB;AACjD,QAAM,EAAE,gBAAgB,+BAA+B,uBAAuB,yBAAyB,gBAAgB,IAAI;AAE3H,iBAAe,eAAe,SAA0B,YAAoB,OAAwC;AAClH,WAAO,QAAQ,WAAW;AAAA,MACxB;AAAA,MACA,OAAO,EAAE,gBAAgB,MAAM,gBAAgB,UAAU,MAAM,SAAS;AAAA,IAC1E,CAAC;AAAA,EACH;AAEA,iBAAe,eAAe,eAA0C,gBAAwB,YAA2B,OAAiC;AAC1J,QAAI,CAAC,cAAe;AAEpB,UAAM,gBAAgB;AAAA,MACpB;AAAA,MACA;AAAA,QACE;AAAA,QACA,YAAY,cAAc;AAAA,MAC5B;AAAA,MACA;AAAA,QACE,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,QACtB,QAAQ,MAAM;AAAA,MAChB;AAAA,IACF;AAAA,EACF;AAEA,iBAAe,yBAAyB,aAAmC,OAAiC;AAC1G,QAAI,CAAC,eAAe,YAAY,WAAW,EAAG;AAE9C,UAAM,QAAQ;AAAA,MACZ,MAAM,KAAK,IAAI,IAAI,YAAY,OAAO,CAAC,UAAU,OAAO,UAAU,YAAY,MAAM,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,EACpG,IAAI,CAAC,eAAe,wBAAwB,KAAK,IAAI;AAAA,QACpD;AAAA,QACA,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,MACxB,CAAC,CAAC;AAAA,IACN;AAAA,EACF;AAEA,iBAAe,sBACb,OACA,eACA,OACA,OACe;AACf,UAAM,cAAc,MAAM,OAAO,CAAC,SAAS,KAAK,WAAW,QAAQ;AACnE,eAAW,QAAQ,aAAa;AAC9B,YAAM,eAAe,OAAO,KAAK,KAAK,iBAAiB,YAAY,KAAK,KAAK,aAAa,KAAK,EAAE,SAAS,IACtG,KAAK,KAAK,aAAa,KAAK,IAC5B;AACJ,YAAM,oBAAoB,OAAO,KAAK,KAAK,sBAAsB,YAAY,KAAK,KAAK,kBAAkB,KAAK,EAAE,SAAS,IACrH,KAAK,KAAK,kBAAkB,KAAK,IACjC;AACJ,YAAM,mBAAmB,OAAO,KAAK,KAAK,qBAAqB,YAAY,KAAK,KAAK,iBAAiB,KAAK,EAAE,SAAS,IAClH,KAAK,KAAK,iBAAiB,KAAK,IAChC;AACJ,YAAM,UAAU;AAAA,QACd,yBAAyB,KAAK,UAAU;AAAA,QACxC,oBAAoB,UAAU,iBAAiB,MAAM;AAAA,QACrD,mBAAmB,gBAAgB,gBAAgB,MAAM;AAAA,QACzD,KAAK,YAAY;AAAA,MACnB,EAAE,OAAO,CAAC,SAAS,SAAS,IAAI,EAAE,KAAK,GAAG;AAE1C,YAAM,sBAAsB;AAAA,QAC1B;AAAA,UACE;AAAA,UACA;AAAA,UACA,OAAO;AAAA,UACP;AAAA,UACA,SAAS,KAAK;AAAA,QAChB;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,iBAAe,sBACb,OACA,eACA,SACA,OACe;AACf,UAAM,gBAAgB,QAAQ,OAAO,CAAC,WAAW,OAAO,WAAW,WAAW,OAAO,KAAK;AAC1F,eAAW,UAAU,eAAe;AAClC,YAAM,QAAQ,OAAO,aAAa,GAAG,OAAO,UAAU,SAAS,OAAO,OAAO,MAAM,OAAO;AAC1F,YAAM,eAAe,OAAO,MAAO,MAAM,IAAI,EAAE,CAAC;AAChD,YAAM,UAAU,yBAAyB,KAAK,KAAK,YAAY;AAE/D,YAAM,sBAAsB;AAAA,QAC1B;AAAA,UACE;AAAA,UACA;AAAA,UACA,OAAO;AAAA,UACP;AAAA,UACA,SAAS,EAAE,MAAM,uBAAuB,SAAS,OAAO,MAAM;AAAA,QAChE;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,iBAAe,oBAAoB,QAQjB;AAChB,QAAI,CAAC,OAAO,QAAS;AAErB,UAAM,sBAAsB;AAAA,MAC1B;AAAA,QACE,eAAe,OAAO;AAAA,QACtB,OAAO,OAAO;AAAA,QACd,OAAO,OAAO;AAAA,QACd,SAAS,OAAO;AAAA,QAChB,SAAS,OAAO;AAAA,MAClB;AAAA,MACA,OAAO;AAAA,IACT;AAAA,EACF;AAEA,iBAAe,uBAAuB,QAKpB;AAChB,QAAI,CAAC,OAAO,WAAW,CAAC,wBAAyB;AAEjD,UAAM,wBAAwB;AAAA,MAC5B,OAAO;AAAA,MACP;AAAA,QACE,kBAAkB,OAAO;AAAA,QACzB,qBAAqB,oBAAI,KAAK;AAAA,MAChC;AAAA,MACA,OAAO;AAAA,IACT;AAAA,EACF;AAEA,iBAAe,YACb,OACA,QACA,OACA,OACA,uBAAuB,OACR;AACf,UAAM,cAAc,MAAM,eAAe,OAAO,OAAO,KAAK;AAC5D,UAAM,iCAAiC,aAAa,WAAW,WACzD,WAAW,eAAe,WAAW,YAAY,WAAW;AAElE,UAAM,MAAM,MAAM,eAAe,WAAW,OAAO,QAAQ,OAAO,KAAK;AACvE,QAAI,CAAC,IAAK;AAEV,QAAI,gCAAgC;AAClC;AAAA,IACF;AAEA,QAAI,IAAI,WAAW,QAAQ;AAOzB,aAAO,KAAK,wEAAwE;AAAA,QAClF;AAAA,QACA,iBAAiB;AAAA,QACjB,cAAc,IAAI;AAAA,MACpB,CAAC;AACD;AAAA,IACF;AAEA,QAAI,IAAI,eAAe;AACrB,UAAI,WAAW,aAAa;AAC1B,cAAM,gBAAgB;AAAA,UACpB,IAAI;AAAA,UACJ;AAAA,YACE,eAAe;AAAA,cACb,cAAc,IAAI;AAAA,cAClB,cAAc,IAAI;AAAA,cAClB,cAAc,IAAI;AAAA,cAClB,aAAa,IAAI;AAAA,cACjB,kBAAkB,IAAI;AAAA,YACxB;AAAA,UACF;AAAA,UACA;AAAA,YACE,UAAU,MAAM;AAAA,YAChB,gBAAgB,MAAM;AAAA,YACtB,QAAQ,MAAM;AAAA,UAChB;AAAA,QACF;AAAA,MACF,WAAW,WAAW,UAAU;AAC9B,cAAM,gBAAgB;AAAA,UACpB,IAAI;AAAA,UACJ;AAAA,YACE,cAAc,SAAS;AAAA,UACzB;AAAA,UACA;AAAA,YACE,UAAU,MAAM;AAAA,YAChB,gBAAgB,MAAM;AAAA,YACtB,QAAQ,MAAM;AAAA,UAChB;AAAA,QACF;AAAA,MACF,WAAW,WAAW,aAAa;AACjC,cAAM,gBAAgB;AAAA,UACpB,IAAI;AAAA,UACJ;AAAA,YACE,UAAU,MAAM;AAAA,YAChB,gBAAgB,MAAM;AAAA,YACtB,QAAQ,MAAM;AAAA,UAChB;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,WAAW,aAAa;AAC1B,YAAM,uBAAuB;AAAA,QAC3B,eAAe,IAAI;AAAA,QACnB,QAAQ;AAAA,QACR;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AACD,YAAM,oBAAoB;AAAA,QACxB,eAAe,IAAI;AAAA,QACnB,OAAO,IAAI;AAAA,QACX,OAAO;AAAA,QACP,SAAS;AAAA,QACT;AAAA,QACA,SAAS;AAAA,QACT,SAAS;AAAA,UACP,mBAAmB;AAAA,UACnB,SAAS,uBAAuB,IAAI,YAAY,aAAa,IAAI,YAAY,aAAa,IAAI,WAAW;AAAA,UACzG,cAAc,IAAI;AAAA,UAClB,cAAc,IAAI;AAAA,UAClB,cAAc,IAAI;AAAA,UAClB,aAAa,IAAI;AAAA,UACjB,kBAAkB,IAAI;AAAA,QACxB;AAAA,MACF,CAAC;AAAA,IACH,WAAW,WAAW,aAAa;AACjC,YAAM,uBAAuB;AAAA,QAC3B,eAAe,IAAI;AAAA,QACnB,QAAQ;AAAA,QACR;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AACD,YAAM,oBAAoB;AAAA,QACxB,eAAe,IAAI;AAAA,QACnB,OAAO,IAAI;AAAA,QACX,OAAO;AAAA,QACP,SAAS;AAAA,QACT;AAAA,QACA,SAAS;AAAA,QACT,SAAS;AAAA,UACP,mBAAmB;AAAA,UACnB,SAAS;AAAA,QACX;AAAA,MACF,CAAC;AAAA,IACH,OAAO;AACL,YAAM,uBAAuB;AAAA,QAC3B,eAAe,IAAI;AAAA,QACnB,QAAQ;AAAA,QACR;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AACD,YAAM,oBAAoB;AAAA,QACxB,eAAe,IAAI;AAAA,QACnB,OAAO,IAAI;AAAA,QACX,OAAO;AAAA,QACP,SAAS,SAAS;AAAA,QAClB;AAAA,QACA,SAAS;AAAA,QACT,SAAS;AAAA,UACP,mBAAmB;AAAA,UACnB,SAAS,SAAS;AAAA,QACpB;AAAA,MACF,CAAC;AAAA,IACH;AAEA,QAAI,WAAW,aAAa;AAC1B,YAAM,kBAAkB,2BAA2B;AAAA,QACjD;AAAA,QACA,eAAe,IAAI;AAAA,QACnB,YAAY,IAAI;AAAA,QAChB,WAAW,IAAI;AAAA,QACf,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,MACxB,CAAC;AACD;AAAA,IACF;AAEA,QAAI,WAAW,aAAa;AAC1B,YAAM,kBAAkB,2BAA2B;AAAA,QACjD;AAAA,QACA,eAAe,IAAI;AAAA,QACnB,YAAY,IAAI;AAAA,QAChB,WAAW,IAAI;AAAA,QACf,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,MACxB,CAAC;AACD;AAAA,IACF;AAEA,UAAM,kBAAkB,wBAAwB;AAAA,MAC9C;AAAA,MACA,eAAe,IAAI;AAAA,MACnB,YAAY,IAAI;AAAA,MAChB,WAAW,IAAI;AAAA,MACf,OAAO,SAAS;AAAA,MAChB,UAAU,MAAM;AAAA,MAChB,gBAAgB,MAAM;AAAA,IACxB,CAAC;AAAA,EACH;AAEA,SAAO;AAAA,IACL,MAAM,UAAU,OAAe,WAAmB,OAAiC;AACjF,YAAM,MAAM,MAAM,eAAe,OAAO,OAAO,KAAK;AACpD,UAAI,CAAC,KAAK;AACR,eAAO,KAAK,6CAA6C,EAAE,MAAM,CAAC;AAClE;AAAA,MACF;AACA,UAAI,IAAI,WAAW,aAAa;AAC9B,YAAI,IAAI,eAAe;AACrB,gBAAM,gBAAgB,cAAc,IAAI,eAAe;AAAA,YACrD,UAAU,MAAM;AAAA,YAChB,gBAAgB,MAAM;AAAA,YACtB,QAAQ,MAAM;AAAA,UAChB,CAAC;AAAA,QACH;AACA;AAAA,MACF;AAEA,YAAM,cAAc,mBAAmB,IAAI,aAAa;AACxD,YAAM,UAAU,mBAAmB,WAAW;AAC9C,UAAI,CAAC,SAAS,cAAc;AAC1B,cAAM,IAAI,MAAM,6CAA6C,WAAW,EAAE;AAAA,MAC5E;AACA,YAAM,uBAAuB,QAAQ,yBAAyB;AAE9D,YAAM,cAAc,MAAM,8BAA8B,QAAQ,IAAI,eAAe,KAAK;AACxF,UAAI,CAAC,aAAa;AAChB,cAAM,IAAI,MAAM,eAAe,IAAI,aAAa,yBAAyB;AAAA,MAC3E;AAKA,YAAM,UAAU,IAAI,WAAW;AAC/B,YAAM,YAAY,MAAM,eAAe,WAAW,IAAI,IAAI,WAAW,KAAK;AAC1E,UAAI,CAAC,aAAa,UAAU,WAAW,WAAW;AAChD;AAAA,MACF;AACA,YAAM,kBAAkB,yBAAyB;AAAA,QAC/C,OAAO,IAAI;AAAA,QACX,eAAe,IAAI;AAAA,QACnB,YAAY,IAAI;AAAA,QAChB,WAAW,IAAI;AAAA,QACf;AAAA,QACA,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,MACxB,CAAC;AACD,YAAM,uBAAuB;AAAA,QAC3B,eAAe,IAAI;AAAA,QACnB,QAAQ;AAAA,QACR;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AACD,YAAM,oBAAoB;AAAA,QACxB,eAAe,IAAI;AAAA,QACnB,OAAO,IAAI;AAAA,QACX,OAAO;AAAA,QACP,SAAS;AAAA,QACT;AAAA,QACA,SAAS;AAAA,QACT,SAAS;AAAA,UACP,mBAAmB;AAAA,UACnB,SAAS,0BAA0B,IAAI,UAAU;AAAA,UACjD,YAAY,IAAI;AAAA,UAChB,WAAW,IAAI;AAAA,QACjB;AAAA,MACF,CAAC;AAED,UAAI,IAAI,eAAe;AACrB,cAAM,gBAAgB,SAAS,IAAI,eAAe;AAAA,UAChD,UAAU,MAAM;AAAA,UAChB,gBAAgB,MAAM;AAAA,UACtB,QAAQ,MAAM;AAAA,QAChB,CAAC;AAAA,MACH;AAEA,YAAM,UAAU,MAAM,eAAe,SAAS,IAAI,YAAY,KAAK;AACnE,UAAI,iBAAiB;AACrB,UAAI,aAA4B;AAChC,UAAI,mBAAmB,UAAU,oBAAoB;AAErD,UAAI;AACF,yBAAiB,SAAS,QAAQ,aAAa;AAAA,UAC7C,YAAY,IAAI;AAAA,UAChB,QAAQ,IAAI,UAAU;AAAA,UACtB;AAAA,UACA;AAAA,UACA;AAAA,UACA,OAAO,EAAE,gBAAgB,MAAM,gBAAgB,UAAU,MAAM,SAAS;AAAA,UACxE,OAAO,IAAI;AAAA,QACb,CAAC,GAAG;AACF,cAAI,IAAI,iBAAiB,MAAM,gBAAgB,wBAAwB,IAAI,eAAe,MAAM,UAAU,MAAM,cAAc,GAAG;AAC/H,kBAAM,YAAY,IAAI,IAAI,aAAa,OAAO,QAAW,oBAAoB;AAC7E;AAAA,UACF;AAEA,gBAAM,QAAQ,oBAAoB,KAAK;AACvC,gBAAM,sBAAsB,MAAM,kBAAkB,MAAM,MAAM;AAChE,4BAAkB;AAClB,uBAAa,MAAM,iBAAiB;AAEpC,gBAAM,eAAe;AAAA,YACnB,IAAI;AAAA,YACJ;AAAA,cACE,GAAG;AAAA,cACH,kBAAkB;AAAA,YACpB;AAAA,YACA,MAAM;AAAA,YACN;AAAA,YACA;AAAA,UACF;AACA,8BAAoB;AAEpB,gBAAM,eAAe,IAAI,eAAe,gBAAgB,YAAY,KAAK;AACzE,gBAAM,yBAAyB,MAAM,4BAA4B,KAAK;AACtE,gBAAM,sBAAsB,IAAI,IAAI,IAAI,eAAe,MAAM,OAAO,KAAK;AAEzE,gBAAM,oBAAoB;AAAA,YACxB,eAAe,IAAI;AAAA,YACnB,OAAO,IAAI;AAAA,YACX,OAAO;AAAA,YACP,SAAS,MAAM,SAAS,KAAK,EAAE,SAC3B,MAAM,QAAQ,KAAK,IACnB,0BAA0B,MAAM,UAAU;AAAA,YAC9C;AAAA,YACA,SAAS;AAAA,YACT,SAAS;AAAA,cACP,mBAAmB;AAAA,cACnB,SAAS,aAAa,cAAc,GAAG,aAAa,OAAO,UAAU,KAAK,EAAE;AAAA,cAC5E;AAAA,cACA,WAAW,MAAM,MAAM;AAAA,cACvB;AAAA,cACA,QAAQ,MAAM;AAAA,YAChB;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF,SAAS,OAAO;AACd,YAAI,iBAAiB,+BAA+B;AAClD,iBAAO,KAAK,uEAAuE;AAAA,YACjF,OAAO,IAAI;AAAA,YACX,0BAA0B,MAAM;AAAA,UAClC,CAAC;AACD;AAAA,QACF;AACA,cAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU;AACzD,cAAM,sBAAsB;AAAA,UAC1B;AAAA,YACE,eAAe,IAAI;AAAA,YACnB,OAAO,IAAI;AAAA,YACX,OAAO;AAAA,YACP;AAAA,UACF;AAAA,UACA;AAAA,QACF;AACA,cAAM,YAAY,IAAI,IAAI,UAAU,OAAO,SAAS,oBAAoB;AACxE;AAAA,MACF;AAEA,YAAM,YAAY,IAAI,IAAI,aAAa,OAAO,QAAW,oBAAoB;AAAA,IAC/E;AAAA,IAEA,MAAM,UAAU,OAAe,WAAmB,OAAiC;AACjF,YAAM,MAAM,MAAM,eAAe,OAAO,OAAO,KAAK;AACpD,UAAI,CAAC,KAAK;AACR,eAAO,KAAK,6CAA6C,EAAE,MAAM,CAAC;AAClE;AAAA,MACF;AACA,UAAI,IAAI,WAAW,aAAa;AAC9B,YAAI,IAAI,eAAe;AACrB,gBAAM,gBAAgB,cAAc,IAAI,eAAe;AAAA,YACrD,UAAU,MAAM;AAAA,YAChB,gBAAgB,MAAM;AAAA,YACtB,QAAQ,MAAM;AAAA,UAChB,CAAC;AAAA,QACH;AACA;AAAA,MACF;AAEA,YAAM,cAAc,mBAAmB,IAAI,aAAa;AACxD,YAAM,UAAU,mBAAmB,WAAW;AAC9C,UAAI,CAAC,SAAS,cAAc;AAC1B,cAAM,IAAI,MAAM,6CAA6C,WAAW,EAAE;AAAA,MAC5E;AACA,YAAM,uBAAuB,QAAQ,yBAAyB;AAE9D,YAAM,cAAc,MAAM,8BAA8B,QAAQ,IAAI,eAAe,KAAK;AACxF,UAAI,CAAC,aAAa;AAChB,cAAM,IAAI,MAAM,eAAe,IAAI,aAAa,yBAAyB;AAAA,MAC3E;AAKA,YAAM,UAAU,IAAI,WAAW;AAC/B,YAAM,YAAY,MAAM,eAAe,WAAW,IAAI,IAAI,WAAW,KAAK;AAC1E,UAAI,CAAC,aAAa,UAAU,WAAW,WAAW;AAChD;AAAA,MACF;AACA,YAAM,kBAAkB,yBAAyB;AAAA,QAC/C,OAAO,IAAI;AAAA,QACX,eAAe,IAAI;AAAA,QACnB,YAAY,IAAI;AAAA,QAChB,WAAW,IAAI;AAAA,QACf;AAAA,QACA,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,MACxB,CAAC;AACD,YAAM,uBAAuB;AAAA,QAC3B,eAAe,IAAI;AAAA,QACnB,QAAQ;AAAA,QACR;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AACD,YAAM,oBAAoB;AAAA,QACxB,eAAe,IAAI;AAAA,QACnB,OAAO,IAAI;AAAA,QACX,OAAO;AAAA,QACP,SAAS;AAAA,QACT;AAAA,QACA,SAAS;AAAA,QACT,SAAS;AAAA,UACP,mBAAmB;AAAA,UACnB,SAAS,0BAA0B,IAAI,UAAU;AAAA,UACjD,YAAY,IAAI;AAAA,UAChB,WAAW,IAAI;AAAA,QACjB;AAAA,MACF,CAAC;AAED,UAAI,IAAI,eAAe;AACrB,cAAM,gBAAgB,SAAS,IAAI,eAAe;AAAA,UAChD,UAAU,MAAM;AAAA,UAChB,gBAAgB,MAAM;AAAA,UACtB,QAAQ,MAAM;AAAA,QAChB,CAAC;AAAA,MACH;AAEA,YAAM,UAAU,MAAM,eAAe,SAAS,IAAI,YAAY,KAAK;AACnE,UAAI,iBAAiB;AACrB,UAAI,mBAAmB,UAAU,oBAAoB;AAErD,UAAI;AACF,yBAAiB,SAAS,QAAQ,aAAa;AAAA,UAC7C,YAAY,IAAI;AAAA,UAChB,QAAQ,IAAI,UAAU;AAAA,UACtB;AAAA,UACA;AAAA,UACA;AAAA,UACA,OAAO,EAAE,gBAAgB,MAAM,gBAAgB,UAAU,MAAM,SAAS;AAAA,UACxE,OAAO,IAAI;AAAA,QACb,CAAC,GAAG;AACF,cAAI,IAAI,iBAAiB,MAAM,gBAAgB,wBAAwB,IAAI,eAAe,MAAM,UAAU,MAAM,cAAc,GAAG;AAC/H,kBAAM,YAAY,IAAI,IAAI,aAAa,OAAO,QAAW,oBAAoB;AAC7E;AAAA,UACF;AAEA,gBAAM,QAAQ,oBAAoB,KAAK;AACvC,4BAAkB,MAAM;AAExB,gBAAM,eAAe;AAAA,YACnB,IAAI;AAAA,YACJ;AAAA,cACE,cAAc;AAAA,cACd,cAAc,MAAM;AAAA,cACpB,cAAc,MAAM;AAAA,cACpB,aAAa,MAAM;AAAA,cACnB,kBAAkB;AAAA,YACpB;AAAA,YACA,MAAM;AAAA,YACN;AAAA,YACA;AAAA,UACF;AACA,8BAAoB;AACpB,gBAAM,eAAe,IAAI,eAAe,gBAAgB,MAAM,KAAK;AACnE,gBAAM,sBAAsB,IAAI,IAAI,IAAI,eAAe,MAAM,SAAS,KAAK;AAE3E,gBAAM,oBAAoB;AAAA,YACxB,eAAe,IAAI;AAAA,YACnB,OAAO,IAAI;AAAA,YACX,OAAO;AAAA,YACP,SAAS,0BAA0B,MAAM,UAAU;AAAA,YACnD;AAAA,YACA,SAAS;AAAA,YACT,SAAS;AAAA,cACP,mBAAmB;AAAA,cACnB,SAAS,aAAa,cAAc;AAAA,cACpC;AAAA,cACA,WAAW,MAAM,QAAQ;AAAA,cACzB,QAAQ,MAAM;AAAA,YAChB;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF,SAAS,OAAO;AACd,YAAI,iBAAiB,+BAA+B;AAClD,iBAAO,KAAK,uEAAuE;AAAA,YACjF,OAAO,IAAI;AAAA,YACX,0BAA0B,MAAM;AAAA,UAClC,CAAC;AACD;AAAA,QACF;AACA,cAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU;AACzD,cAAM,sBAAsB;AAAA,UAC1B;AAAA,YACE,eAAe,IAAI;AAAA,YACnB,OAAO,IAAI;AAAA,YACX,OAAO;AAAA,YACP;AAAA,UACF;AAAA,UACA;AAAA,QACF;AACA,cAAM,YAAY,IAAI,IAAI,UAAU,OAAO,SAAS,oBAAoB;AACxE;AAAA,MACF;AAEA,YAAM,YAAY,IAAI,IAAI,aAAa,OAAO,QAAW,oBAAoB;AAAA,IAC/E;AAAA,EACF;AACF;",
4
+ "sourcesContent": ["import type { EntityManager } from '@mikro-orm/postgresql'\nimport { getIntegration } from '@open-mercato/shared/modules/integrations/types'\nimport type { CredentialsService } from '../../integrations/lib/credentials-service'\nimport type { IntegrationLogService } from '../../integrations/lib/log-service'\nimport type { IntegrationStateService } from '../../integrations/lib/state-service'\nimport type { ProgressService } from '../../progress/lib/progressService'\nimport { STALE_JOB_TIMEOUT_SECONDS } from '../../progress/lib/progressService'\nimport { refreshCoverageSnapshot } from '../../query_index/lib/coverage'\nimport { emitDataSyncEvent } from '../events'\nimport type { DataSyncAdapter, DataMapping, ExportBatch, ImportBatch } from './adapter'\nimport { getDataSyncAdapter } from './adapter-registry'\nimport type { SyncRunService } from './sync-run-service'\nimport { SyncRunOwnershipConflictError } from './sync-run-service'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\n\nconst logger = createLogger('data_sync').child({ component: 'sync-engine' })\n\ntype SyncScope = {\n organizationId: string\n tenantId: string\n userId?: string | null\n}\n\ntype EngineDeps = {\n em: EntityManager\n syncRunService: SyncRunService\n integrationCredentialsService: CredentialsService\n integrationLogService: IntegrationLogService\n integrationStateService?: IntegrationStateService\n progressService: ProgressService\n}\n\nfunction resolveProviderKey(integrationId: string): string {\n return getIntegration(integrationId)?.providerKey ?? integrationId\n}\n\nfunction applyImportCounters(batch: ImportBatch): Pick<Required<SyncCounterDelta>, 'createdCount' | 'updatedCount' | 'skippedCount' | 'failedCount'> {\n let createdCount = 0\n let updatedCount = 0\n let skippedCount = 0\n let failedCount = 0\n\n for (const item of batch.items) {\n if (item.action === 'create') createdCount += 1\n else if (item.action === 'update') updatedCount += 1\n else if (item.action === 'failed') failedCount += 1\n else skippedCount += 1\n }\n\n return { createdCount, updatedCount, skippedCount, failedCount }\n}\n\ntype SyncCounterDelta = {\n createdCount?: number\n updatedCount?: number\n skippedCount?: number\n failedCount?: number\n processedCount: number\n}\n\nfunction applyExportCounters(batch: ExportBatch): SyncCounterDelta {\n let failedCount = 0\n let skippedCount = 0\n let updatedCount = 0\n\n for (const result of batch.results) {\n if (result.status === 'error') failedCount += 1\n else if (result.status === 'skipped') skippedCount += 1\n else updatedCount += 1\n }\n\n return {\n failedCount,\n skippedCount,\n updatedCount,\n processedCount: batch.results.length,\n }\n}\n\n// Adapter batches can legitimately outlast the stale-job sweep window (slow upstream\n// APIs), so the engine must heartbeat while a batch is still being produced.\nconst HEARTBEAT_TICK_MS = (STALE_JOB_TIMEOUT_SECONDS * 1000) / 4\n\n// Runs `tick` on an interval only while the source iterator is pending, so heartbeats\n// stop the moment the producer dies and genuinely stale jobs still get swept. The outer\n// finally closes the adapter generator on early exits (cancellation, ownership conflict).\nasync function* withHeartbeat<T>(source: AsyncIterable<T>, tick: () => void, intervalMs: number): AsyncGenerator<T, void, undefined> {\n const iterator = source[Symbol.asyncIterator]()\n try {\n while (true) {\n const timer = setInterval(tick, intervalMs)\n let result: IteratorResult<T>\n try {\n result = await iterator.next()\n } finally {\n clearInterval(timer)\n }\n if (result.done) return\n yield result.value\n }\n } finally {\n await iterator.return?.()\n }\n}\n\nexport function createSyncEngine(deps: EngineDeps) {\n const { syncRunService, integrationCredentialsService, integrationLogService, integrationStateService, progressService } = deps\n\n async function resolveMapping(adapter: DataSyncAdapter, entityType: string, scope: SyncScope): Promise<DataMapping> {\n return adapter.getMapping({\n entityType,\n scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },\n })\n }\n\n async function updateProgress(progressJobId: string | null | undefined, processedCount: number, totalCount: number | null, scope: SyncScope): Promise<void> {\n if (!progressJobId) return\n\n await progressService.updateProgress(\n progressJobId,\n {\n processedCount,\n totalCount: totalCount ?? undefined,\n },\n {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n },\n )\n }\n\n // On redelivery the progress counter must resume where the last delivery left off the\n // same way committedBatches does \u2014 updateProgress writes absolute counts, so starting at\n // zero would regress the visible count. The progress job's own processedCount is the only\n // persisted value already in the engine's unit: `batch.processedCount ?? items.length`,\n // i.e. source records. The run's created/updated/skipped/failed counters count emitted\n // items, which adapters may explode several-per-source-record (Akeneo yields a product\n // plus its variants), so seeding from them would overshoot the total and pin the bar.\n async function seedProcessedCount(progressJobId: string | null | undefined, scope: SyncScope): Promise<number> {\n if (!progressJobId) return 0\n const job = await progressService.getJob(progressJobId, {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n })\n return job?.processedCount ?? 0\n }\n\n function makeHeartbeatTick(progressJobId: string | null | undefined, scope: SyncScope): () => void {\n const touchJobHeartbeat = progressService.touchJobHeartbeat?.bind(progressService)\n if (!progressJobId || !touchJobHeartbeat) return () => {}\n let inFlight = false\n return () => {\n if (inFlight) return\n inFlight = true\n touchJobHeartbeat(progressJobId, {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n })\n .catch((error) => {\n logger.warn('Progress heartbeat failed', {\n progressJobId,\n error: error instanceof Error ? error.message : String(error),\n })\n })\n .finally(() => {\n inFlight = false\n })\n }\n }\n\n async function refreshCoverageSnapshots(entityTypes: string[] | undefined, scope: SyncScope): Promise<void> {\n if (!entityTypes || entityTypes.length === 0) return\n\n await Promise.allSettled(\n Array.from(new Set(entityTypes.filter((value) => typeof value === 'string' && value.trim().length > 0)))\n .map((entityType) => refreshCoverageSnapshot(deps.em, {\n entityType,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n })),\n )\n }\n\n async function logImportItemFailures(\n runId: string,\n integrationId: string,\n items: ImportBatch['items'],\n scope: SyncScope,\n ): Promise<void> {\n const failedItems = items.filter((item) => item.action === 'failed')\n for (const item of failedItems) {\n const errorMessage = typeof item.data.errorMessage === 'string' && item.data.errorMessage.trim().length > 0\n ? item.data.errorMessage.trim()\n : 'Import item failed'\n const sourceProductUuid = typeof item.data.sourceProductUuid === 'string' && item.data.sourceProductUuid.trim().length > 0\n ? item.data.sourceProductUuid.trim()\n : null\n const sourceIdentifier = typeof item.data.sourceIdentifier === 'string' && item.data.sourceIdentifier.trim().length > 0\n ? item.data.sourceIdentifier.trim()\n : null\n const message = [\n `Failed to import item ${item.externalId}`,\n sourceProductUuid ? `(uuid: ${sourceProductUuid})` : null,\n sourceIdentifier ? `(identifier: ${sourceIdentifier})` : null,\n `: ${errorMessage}`,\n ].filter((part) => part !== null).join(' ')\n\n await integrationLogService.write(\n {\n integrationId,\n runId,\n level: 'error',\n message,\n payload: item.data,\n },\n scope,\n )\n }\n }\n\n async function logExportItemFailures(\n runId: string,\n integrationId: string,\n results: ExportBatch['results'],\n scope: SyncScope,\n ): Promise<void> {\n const failedResults = results.filter((result) => result.status === 'error' && result.error)\n for (const result of failedResults) {\n const label = result.externalId ? `${result.externalId} (id: ${result.localId})` : result.localId\n const errorMessage = result.error!.split('\\n')[0]\n const message = `Failed to export item ${label}: ${errorMessage}`\n\n await integrationLogService.write(\n {\n integrationId,\n runId,\n level: 'error',\n message,\n payload: { kind: 'export-item-failure', summary: result.error },\n },\n scope,\n )\n }\n }\n\n async function writeOperationalLog(params: {\n integrationId: string\n runId: string\n level: 'info' | 'warn' | 'error'\n message: string\n scope: SyncScope\n enabled: boolean\n payload?: Record<string, unknown>\n }): Promise<void> {\n if (!params.enabled) return\n\n await integrationLogService.write(\n {\n integrationId: params.integrationId,\n runId: params.runId,\n level: params.level,\n message: params.message,\n payload: params.payload,\n },\n params.scope,\n )\n }\n\n async function updateOperationalState(params: {\n integrationId: string\n status: 'healthy' | 'degraded' | 'unhealthy'\n scope: SyncScope\n enabled: boolean\n }): Promise<void> {\n if (!params.enabled || !integrationStateService) return\n\n await integrationStateService.upsert(\n params.integrationId,\n {\n lastHealthStatus: params.status,\n lastHealthCheckedAt: new Date(),\n },\n params.scope,\n )\n }\n\n async function finalizeRun(\n runId: string,\n status: 'completed' | 'failed' | 'cancelled',\n scope: SyncScope,\n error?: string,\n operationalTelemetry = false,\n ): Promise<void> {\n const existingRun = await syncRunService.getRun(runId, scope)\n const alreadyFinalizedWithSameStatus = existingRun?.status === status\n && (status === 'completed' || status === 'failed' || status === 'cancelled')\n\n const run = await syncRunService.markStatus(runId, status, scope, error)\n if (!run) return\n\n if (alreadyFinalizedWithSameStatus) {\n return\n }\n\n if (run.status !== status) {\n // `markStatus` refuses a terminal -> different-terminal transition and\n // returns the row unchanged, so the run is already finished under another\n // delivery of this job. Everything below \u2014 the progress job, the\n // operational log and the lifecycle event \u2014 would describe the wrong\n // outcome, and `data_sync.run.failed` is dispatched to tenant webhooks.\n // A displaced worker stays silent instead.\n logger.warn('Skipping finalization of a sync run another worker already finalized', {\n runId,\n requestedStatus: status,\n actualStatus: run.status,\n })\n return\n }\n\n if (run.progressJobId) {\n if (status === 'completed') {\n await progressService.completeJob(\n run.progressJobId,\n {\n resultSummary: {\n createdCount: run.createdCount,\n updatedCount: run.updatedCount,\n skippedCount: run.skippedCount,\n failedCount: run.failedCount,\n batchesCompleted: run.batchesCompleted,\n },\n },\n {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n },\n )\n } else if (status === 'failed') {\n await progressService.failJob(\n run.progressJobId,\n {\n errorMessage: error ?? 'Sync run failed',\n },\n {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n },\n )\n } else if (status === 'cancelled') {\n await progressService.markCancelled(\n run.progressJobId,\n {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n },\n )\n }\n }\n\n if (status === 'completed') {\n await updateOperationalState({\n integrationId: run.integrationId,\n status: 'healthy',\n scope,\n enabled: operationalTelemetry,\n })\n await writeOperationalLog({\n integrationId: run.integrationId,\n runId: run.id,\n level: 'info',\n message: 'Sync run completed',\n scope,\n enabled: operationalTelemetry,\n payload: {\n operationalStatus: 'completed',\n summary: `Sync completed with ${run.createdCount} created, ${run.updatedCount} updated, ${run.failedCount} failed.`,\n createdCount: run.createdCount,\n updatedCount: run.updatedCount,\n skippedCount: run.skippedCount,\n failedCount: run.failedCount,\n batchesCompleted: run.batchesCompleted,\n },\n })\n } else if (status === 'cancelled') {\n await updateOperationalState({\n integrationId: run.integrationId,\n status: 'degraded',\n scope,\n enabled: operationalTelemetry,\n })\n await writeOperationalLog({\n integrationId: run.integrationId,\n runId: run.id,\n level: 'warn',\n message: 'Sync run cancelled',\n scope,\n enabled: operationalTelemetry,\n payload: {\n operationalStatus: 'cancelled',\n summary: 'The sync run was cancelled before completion.',\n },\n })\n } else {\n await updateOperationalState({\n integrationId: run.integrationId,\n status: 'unhealthy',\n scope,\n enabled: operationalTelemetry,\n })\n await writeOperationalLog({\n integrationId: run.integrationId,\n runId: run.id,\n level: 'error',\n message: error ?? 'Sync run failed',\n scope,\n enabled: operationalTelemetry,\n payload: {\n operationalStatus: 'failed',\n summary: error ?? 'The sync run failed.',\n },\n })\n }\n\n if (status === 'completed') {\n await emitDataSyncEvent('data_sync.run.completed', {\n runId,\n integrationId: run.integrationId,\n entityType: run.entityType,\n direction: run.direction,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n })\n return\n }\n\n if (status === 'cancelled') {\n await emitDataSyncEvent('data_sync.run.cancelled', {\n runId,\n integrationId: run.integrationId,\n entityType: run.entityType,\n direction: run.direction,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n })\n return\n }\n\n await emitDataSyncEvent('data_sync.run.failed', {\n runId,\n integrationId: run.integrationId,\n entityType: run.entityType,\n direction: run.direction,\n error: error ?? null,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n })\n }\n\n return {\n async runImport(runId: string, batchSize: number, scope: SyncScope): Promise<void> {\n const run = await syncRunService.getRun(runId, scope)\n if (!run) {\n logger.warn('Skipping stale import job for missing run', { runId })\n return\n }\n if (run.status === 'cancelled') {\n if (run.progressJobId) {\n await progressService.markCancelled(run.progressJobId, {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n })\n }\n return\n }\n\n const providerKey = resolveProviderKey(run.integrationId)\n const adapter = getDataSyncAdapter(providerKey)\n if (!adapter?.streamImport) {\n throw new Error(`No import adapter registered for provider ${providerKey}`)\n }\n const operationalTelemetry = adapter.operationalTelemetry === true\n\n const credentials = await integrationCredentialsService.resolve(run.integrationId, scope)\n if (!credentials) {\n throw new Error(`Integration ${run.integrationId} is missing credentials`)\n }\n\n // A run already `running` means a stalled job was redelivered, so this is\n // a resume rather than a first start. Consumers see one `started` event per\n // delivery either way; the flag is what lets them tell the two apart.\n const resumed = run.status === 'running'\n const activeRun = await syncRunService.markStatus(run.id, 'running', scope)\n if (!activeRun || activeRun.status !== 'running') {\n return\n }\n await emitDataSyncEvent('data_sync.run.started', {\n runId: run.id,\n integrationId: run.integrationId,\n entityType: run.entityType,\n direction: run.direction,\n resumed,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n })\n await updateOperationalState({\n integrationId: run.integrationId,\n status: 'degraded',\n scope,\n enabled: operationalTelemetry,\n })\n await writeOperationalLog({\n integrationId: run.integrationId,\n runId: run.id,\n level: 'info',\n message: 'Sync run started',\n scope,\n enabled: operationalTelemetry,\n payload: {\n operationalStatus: 'running',\n summary: `Import run started for ${run.entityType}.`,\n entityType: run.entityType,\n direction: run.direction,\n },\n })\n\n if (run.progressJobId) {\n await progressService.startJob(run.progressJobId, {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n })\n }\n\n const mapping = await resolveMapping(adapter, run.entityType, scope)\n let processedCount = await seedProcessedCount(run.progressJobId, scope)\n let totalCount: number | null = null\n let committedBatches = activeRun.batchesCompleted ?? 0\n\n try {\n for await (const batch of withHeartbeat(\n adapter.streamImport({\n entityType: run.entityType,\n cursor: run.cursor ?? undefined,\n batchSize,\n credentials,\n mapping,\n scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },\n runId: run.id,\n }),\n makeHeartbeatTick(run.progressJobId, scope),\n HEARTBEAT_TICK_MS,\n )) {\n if (run.progressJobId && await progressService.isCancellationRequested(run.progressJobId, scope.tenantId, scope.organizationId)) {\n await finalizeRun(run.id, 'cancelled', scope, undefined, operationalTelemetry)\n return\n }\n\n const delta = applyImportCounters(batch)\n const processedBatchCount = batch.processedCount ?? batch.items.length\n processedCount += processedBatchCount\n totalCount = batch.totalEstimate ?? totalCount\n\n await syncRunService.commitBatchProgress(\n run.id,\n {\n ...delta,\n batchesCompleted: 1,\n },\n batch.cursor,\n scope,\n committedBatches,\n )\n committedBatches += 1\n\n await updateProgress(run.progressJobId, processedCount, totalCount, scope)\n await refreshCoverageSnapshots(batch.refreshCoverageEntityTypes, scope)\n await logImportItemFailures(run.id, run.integrationId, batch.items, scope)\n\n await writeOperationalLog({\n integrationId: run.integrationId,\n runId: run.id,\n level: 'info',\n message: batch.message?.trim().length\n ? batch.message.trim()\n : `Processed import batch ${batch.batchIndex}`,\n scope,\n enabled: operationalTelemetry,\n payload: {\n operationalStatus: 'running',\n summary: `Processed ${processedCount}${totalCount ? ` of ${totalCount}` : ''} rows so far.`,\n processedCount,\n batchSize: batch.items.length,\n processedBatchCount,\n cursor: batch.cursor,\n },\n })\n }\n } catch (error) {\n if (error instanceof SyncRunOwnershipConflictError) {\n logger.warn('Yielding import run to a concurrent worker that already advanced it', {\n runId: run.id,\n expectedBatchesCompleted: error.expectedBatchesCompleted,\n })\n return\n }\n const message = error instanceof Error ? error.message : 'Sync import failed'\n await integrationLogService.write(\n {\n integrationId: run.integrationId,\n runId: run.id,\n level: 'error',\n message,\n },\n scope,\n )\n await finalizeRun(run.id, 'failed', scope, message, operationalTelemetry)\n return\n }\n\n await finalizeRun(run.id, 'completed', scope, undefined, operationalTelemetry)\n },\n\n async runExport(runId: string, batchSize: number, scope: SyncScope): Promise<void> {\n const run = await syncRunService.getRun(runId, scope)\n if (!run) {\n logger.warn('Skipping stale export job for missing run', { runId })\n return\n }\n if (run.status === 'cancelled') {\n if (run.progressJobId) {\n await progressService.markCancelled(run.progressJobId, {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n })\n }\n return\n }\n\n const providerKey = resolveProviderKey(run.integrationId)\n const adapter = getDataSyncAdapter(providerKey)\n if (!adapter?.streamExport) {\n throw new Error(`No export adapter registered for provider ${providerKey}`)\n }\n const operationalTelemetry = adapter.operationalTelemetry === true\n\n const credentials = await integrationCredentialsService.resolve(run.integrationId, scope)\n if (!credentials) {\n throw new Error(`Integration ${run.integrationId} is missing credentials`)\n }\n\n // A run already `running` means a stalled job was redelivered, so this is\n // a resume rather than a first start. Consumers see one `started` event per\n // delivery either way; the flag is what lets them tell the two apart.\n const resumed = run.status === 'running'\n const activeRun = await syncRunService.markStatus(run.id, 'running', scope)\n if (!activeRun || activeRun.status !== 'running') {\n return\n }\n await emitDataSyncEvent('data_sync.run.started', {\n runId: run.id,\n integrationId: run.integrationId,\n entityType: run.entityType,\n direction: run.direction,\n resumed,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n })\n await updateOperationalState({\n integrationId: run.integrationId,\n status: 'degraded',\n scope,\n enabled: operationalTelemetry,\n })\n await writeOperationalLog({\n integrationId: run.integrationId,\n runId: run.id,\n level: 'info',\n message: 'Sync run started',\n scope,\n enabled: operationalTelemetry,\n payload: {\n operationalStatus: 'running',\n summary: `Export run started for ${run.entityType}.`,\n entityType: run.entityType,\n direction: run.direction,\n },\n })\n\n if (run.progressJobId) {\n await progressService.startJob(run.progressJobId, {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n })\n }\n\n const mapping = await resolveMapping(adapter, run.entityType, scope)\n let processedCount = await seedProcessedCount(run.progressJobId, scope)\n let committedBatches = activeRun.batchesCompleted ?? 0\n\n try {\n for await (const batch of withHeartbeat(\n adapter.streamExport({\n entityType: run.entityType,\n cursor: run.cursor ?? undefined,\n batchSize,\n credentials,\n mapping,\n scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },\n runId: run.id,\n }),\n makeHeartbeatTick(run.progressJobId, scope),\n HEARTBEAT_TICK_MS,\n )) {\n if (run.progressJobId && await progressService.isCancellationRequested(run.progressJobId, scope.tenantId, scope.organizationId)) {\n await finalizeRun(run.id, 'cancelled', scope, undefined, operationalTelemetry)\n return\n }\n\n const delta = applyExportCounters(batch)\n processedCount += delta.processedCount\n\n await syncRunService.commitBatchProgress(\n run.id,\n {\n createdCount: 0,\n updatedCount: delta.updatedCount,\n skippedCount: delta.skippedCount,\n failedCount: delta.failedCount,\n batchesCompleted: 1,\n },\n batch.cursor,\n scope,\n committedBatches,\n )\n committedBatches += 1\n await updateProgress(run.progressJobId, processedCount, null, scope)\n await logExportItemFailures(run.id, run.integrationId, batch.results, scope)\n\n await writeOperationalLog({\n integrationId: run.integrationId,\n runId: run.id,\n level: 'info',\n message: `Processed export batch ${batch.batchIndex}`,\n scope,\n enabled: operationalTelemetry,\n payload: {\n operationalStatus: 'running',\n summary: `Processed ${processedCount} export items so far.`,\n processedCount,\n batchSize: batch.results.length,\n cursor: batch.cursor,\n },\n })\n }\n } catch (error) {\n if (error instanceof SyncRunOwnershipConflictError) {\n logger.warn('Yielding export run to a concurrent worker that already advanced it', {\n runId: run.id,\n expectedBatchesCompleted: error.expectedBatchesCompleted,\n })\n return\n }\n const message = error instanceof Error ? error.message : 'Sync export failed'\n await integrationLogService.write(\n {\n integrationId: run.integrationId,\n runId: run.id,\n level: 'error',\n message,\n },\n scope,\n )\n await finalizeRun(run.id, 'failed', scope, message, operationalTelemetry)\n return\n }\n\n await finalizeRun(run.id, 'completed', scope, undefined, operationalTelemetry)\n },\n }\n}\n\nexport type SyncEngine = ReturnType<typeof createSyncEngine>\n"],
5
+ "mappings": "AACA,SAAS,sBAAsB;AAK/B,SAAS,iCAAiC;AAC1C,SAAS,+BAA+B;AACxC,SAAS,yBAAyB;AAElC,SAAS,0BAA0B;AAEnC,SAAS,qCAAqC;AAC9C,SAAS,oBAAoB;AAE7B,MAAM,SAAS,aAAa,WAAW,EAAE,MAAM,EAAE,WAAW,cAAc,CAAC;AAiB3E,SAAS,mBAAmB,eAA+B;AACzD,SAAO,eAAe,aAAa,GAAG,eAAe;AACvD;AAEA,SAAS,oBAAoB,OAAwH;AACnJ,MAAI,eAAe;AACnB,MAAI,eAAe;AACnB,MAAI,eAAe;AACnB,MAAI,cAAc;AAElB,aAAW,QAAQ,MAAM,OAAO;AAC9B,QAAI,KAAK,WAAW,SAAU,iBAAgB;AAAA,aACrC,KAAK,WAAW,SAAU,iBAAgB;AAAA,aAC1C,KAAK,WAAW,SAAU,gBAAe;AAAA,QAC7C,iBAAgB;AAAA,EACvB;AAEA,SAAO,EAAE,cAAc,cAAc,cAAc,YAAY;AACjE;AAUA,SAAS,oBAAoB,OAAsC;AACjE,MAAI,cAAc;AAClB,MAAI,eAAe;AACnB,MAAI,eAAe;AAEnB,aAAW,UAAU,MAAM,SAAS;AAClC,QAAI,OAAO,WAAW,QAAS,gBAAe;AAAA,aACrC,OAAO,WAAW,UAAW,iBAAgB;AAAA,QACjD,iBAAgB;AAAA,EACvB;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA,gBAAgB,MAAM,QAAQ;AAAA,EAChC;AACF;AAIA,MAAM,oBAAqB,4BAA4B,MAAQ;AAK/D,gBAAgB,cAAiB,QAA0B,MAAkB,YAAwD;AACnI,QAAM,WAAW,OAAO,OAAO,aAAa,EAAE;AAC9C,MAAI;AACF,WAAO,MAAM;AACX,YAAM,QAAQ,YAAY,MAAM,UAAU;AAC1C,UAAI;AACJ,UAAI;AACF,iBAAS,MAAM,SAAS,KAAK;AAAA,MAC/B,UAAE;AACA,sBAAc,KAAK;AAAA,MACrB;AACA,UAAI,OAAO,KAAM;AACjB,YAAM,OAAO;AAAA,IACf;AAAA,EACF,UAAE;AACA,UAAM,SAAS,SAAS;AAAA,EAC1B;AACF;AAEO,SAAS,iBAAiB,MAAkB;AACjD,QAAM,EAAE,gBAAgB,+BAA+B,uBAAuB,yBAAyB,gBAAgB,IAAI;AAE3H,iBAAe,eAAe,SAA0B,YAAoB,OAAwC;AAClH,WAAO,QAAQ,WAAW;AAAA,MACxB;AAAA,MACA,OAAO,EAAE,gBAAgB,MAAM,gBAAgB,UAAU,MAAM,SAAS;AAAA,IAC1E,CAAC;AAAA,EACH;AAEA,iBAAe,eAAe,eAA0C,gBAAwB,YAA2B,OAAiC;AAC1J,QAAI,CAAC,cAAe;AAEpB,UAAM,gBAAgB;AAAA,MACpB;AAAA,MACA;AAAA,QACE;AAAA,QACA,YAAY,cAAc;AAAA,MAC5B;AAAA,MACA;AAAA,QACE,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,QACtB,QAAQ,MAAM;AAAA,MAChB;AAAA,IACF;AAAA,EACF;AASA,iBAAe,mBAAmB,eAA0C,OAAmC;AAC7G,QAAI,CAAC,cAAe,QAAO;AAC3B,UAAM,MAAM,MAAM,gBAAgB,OAAO,eAAe;AAAA,MACtD,UAAU,MAAM;AAAA,MAChB,gBAAgB,MAAM;AAAA,MACtB,QAAQ,MAAM;AAAA,IAChB,CAAC;AACD,WAAO,KAAK,kBAAkB;AAAA,EAChC;AAEA,WAAS,kBAAkB,eAA0C,OAA8B;AACjG,UAAM,oBAAoB,gBAAgB,mBAAmB,KAAK,eAAe;AACjF,QAAI,CAAC,iBAAiB,CAAC,kBAAmB,QAAO,MAAM;AAAA,IAAC;AACxD,QAAI,WAAW;AACf,WAAO,MAAM;AACX,UAAI,SAAU;AACd,iBAAW;AACX,wBAAkB,eAAe;AAAA,QAC/B,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,QACtB,QAAQ,MAAM;AAAA,MAChB,CAAC,EACE,MAAM,CAAC,UAAU;AAChB,eAAO,KAAK,6BAA6B;AAAA,UACvC;AAAA,UACA,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AAAA,MACH,CAAC,EACA,QAAQ,MAAM;AACb,mBAAW;AAAA,MACb,CAAC;AAAA,IACL;AAAA,EACF;AAEA,iBAAe,yBAAyB,aAAmC,OAAiC;AAC1G,QAAI,CAAC,eAAe,YAAY,WAAW,EAAG;AAE9C,UAAM,QAAQ;AAAA,MACZ,MAAM,KAAK,IAAI,IAAI,YAAY,OAAO,CAAC,UAAU,OAAO,UAAU,YAAY,MAAM,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,EACpG,IAAI,CAAC,eAAe,wBAAwB,KAAK,IAAI;AAAA,QACpD;AAAA,QACA,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,MACxB,CAAC,CAAC;AAAA,IACN;AAAA,EACF;AAEA,iBAAe,sBACb,OACA,eACA,OACA,OACe;AACf,UAAM,cAAc,MAAM,OAAO,CAAC,SAAS,KAAK,WAAW,QAAQ;AACnE,eAAW,QAAQ,aAAa;AAC9B,YAAM,eAAe,OAAO,KAAK,KAAK,iBAAiB,YAAY,KAAK,KAAK,aAAa,KAAK,EAAE,SAAS,IACtG,KAAK,KAAK,aAAa,KAAK,IAC5B;AACJ,YAAM,oBAAoB,OAAO,KAAK,KAAK,sBAAsB,YAAY,KAAK,KAAK,kBAAkB,KAAK,EAAE,SAAS,IACrH,KAAK,KAAK,kBAAkB,KAAK,IACjC;AACJ,YAAM,mBAAmB,OAAO,KAAK,KAAK,qBAAqB,YAAY,KAAK,KAAK,iBAAiB,KAAK,EAAE,SAAS,IAClH,KAAK,KAAK,iBAAiB,KAAK,IAChC;AACJ,YAAM,UAAU;AAAA,QACd,yBAAyB,KAAK,UAAU;AAAA,QACxC,oBAAoB,UAAU,iBAAiB,MAAM;AAAA,QACrD,mBAAmB,gBAAgB,gBAAgB,MAAM;AAAA,QACzD,KAAK,YAAY;AAAA,MACnB,EAAE,OAAO,CAAC,SAAS,SAAS,IAAI,EAAE,KAAK,GAAG;AAE1C,YAAM,sBAAsB;AAAA,QAC1B;AAAA,UACE;AAAA,UACA;AAAA,UACA,OAAO;AAAA,UACP;AAAA,UACA,SAAS,KAAK;AAAA,QAChB;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,iBAAe,sBACb,OACA,eACA,SACA,OACe;AACf,UAAM,gBAAgB,QAAQ,OAAO,CAAC,WAAW,OAAO,WAAW,WAAW,OAAO,KAAK;AAC1F,eAAW,UAAU,eAAe;AAClC,YAAM,QAAQ,OAAO,aAAa,GAAG,OAAO,UAAU,SAAS,OAAO,OAAO,MAAM,OAAO;AAC1F,YAAM,eAAe,OAAO,MAAO,MAAM,IAAI,EAAE,CAAC;AAChD,YAAM,UAAU,yBAAyB,KAAK,KAAK,YAAY;AAE/D,YAAM,sBAAsB;AAAA,QAC1B;AAAA,UACE;AAAA,UACA;AAAA,UACA,OAAO;AAAA,UACP;AAAA,UACA,SAAS,EAAE,MAAM,uBAAuB,SAAS,OAAO,MAAM;AAAA,QAChE;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,iBAAe,oBAAoB,QAQjB;AAChB,QAAI,CAAC,OAAO,QAAS;AAErB,UAAM,sBAAsB;AAAA,MAC1B;AAAA,QACE,eAAe,OAAO;AAAA,QACtB,OAAO,OAAO;AAAA,QACd,OAAO,OAAO;AAAA,QACd,SAAS,OAAO;AAAA,QAChB,SAAS,OAAO;AAAA,MAClB;AAAA,MACA,OAAO;AAAA,IACT;AAAA,EACF;AAEA,iBAAe,uBAAuB,QAKpB;AAChB,QAAI,CAAC,OAAO,WAAW,CAAC,wBAAyB;AAEjD,UAAM,wBAAwB;AAAA,MAC5B,OAAO;AAAA,MACP;AAAA,QACE,kBAAkB,OAAO;AAAA,QACzB,qBAAqB,oBAAI,KAAK;AAAA,MAChC;AAAA,MACA,OAAO;AAAA,IACT;AAAA,EACF;AAEA,iBAAe,YACb,OACA,QACA,OACA,OACA,uBAAuB,OACR;AACf,UAAM,cAAc,MAAM,eAAe,OAAO,OAAO,KAAK;AAC5D,UAAM,iCAAiC,aAAa,WAAW,WACzD,WAAW,eAAe,WAAW,YAAY,WAAW;AAElE,UAAM,MAAM,MAAM,eAAe,WAAW,OAAO,QAAQ,OAAO,KAAK;AACvE,QAAI,CAAC,IAAK;AAEV,QAAI,gCAAgC;AAClC;AAAA,IACF;AAEA,QAAI,IAAI,WAAW,QAAQ;AAOzB,aAAO,KAAK,wEAAwE;AAAA,QAClF;AAAA,QACA,iBAAiB;AAAA,QACjB,cAAc,IAAI;AAAA,MACpB,CAAC;AACD;AAAA,IACF;AAEA,QAAI,IAAI,eAAe;AACrB,UAAI,WAAW,aAAa;AAC1B,cAAM,gBAAgB;AAAA,UACpB,IAAI;AAAA,UACJ;AAAA,YACE,eAAe;AAAA,cACb,cAAc,IAAI;AAAA,cAClB,cAAc,IAAI;AAAA,cAClB,cAAc,IAAI;AAAA,cAClB,aAAa,IAAI;AAAA,cACjB,kBAAkB,IAAI;AAAA,YACxB;AAAA,UACF;AAAA,UACA;AAAA,YACE,UAAU,MAAM;AAAA,YAChB,gBAAgB,MAAM;AAAA,YACtB,QAAQ,MAAM;AAAA,UAChB;AAAA,QACF;AAAA,MACF,WAAW,WAAW,UAAU;AAC9B,cAAM,gBAAgB;AAAA,UACpB,IAAI;AAAA,UACJ;AAAA,YACE,cAAc,SAAS;AAAA,UACzB;AAAA,UACA;AAAA,YACE,UAAU,MAAM;AAAA,YAChB,gBAAgB,MAAM;AAAA,YACtB,QAAQ,MAAM;AAAA,UAChB;AAAA,QACF;AAAA,MACF,WAAW,WAAW,aAAa;AACjC,cAAM,gBAAgB;AAAA,UACpB,IAAI;AAAA,UACJ;AAAA,YACE,UAAU,MAAM;AAAA,YAChB,gBAAgB,MAAM;AAAA,YACtB,QAAQ,MAAM;AAAA,UAChB;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,WAAW,aAAa;AAC1B,YAAM,uBAAuB;AAAA,QAC3B,eAAe,IAAI;AAAA,QACnB,QAAQ;AAAA,QACR;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AACD,YAAM,oBAAoB;AAAA,QACxB,eAAe,IAAI;AAAA,QACnB,OAAO,IAAI;AAAA,QACX,OAAO;AAAA,QACP,SAAS;AAAA,QACT;AAAA,QACA,SAAS;AAAA,QACT,SAAS;AAAA,UACP,mBAAmB;AAAA,UACnB,SAAS,uBAAuB,IAAI,YAAY,aAAa,IAAI,YAAY,aAAa,IAAI,WAAW;AAAA,UACzG,cAAc,IAAI;AAAA,UAClB,cAAc,IAAI;AAAA,UAClB,cAAc,IAAI;AAAA,UAClB,aAAa,IAAI;AAAA,UACjB,kBAAkB,IAAI;AAAA,QACxB;AAAA,MACF,CAAC;AAAA,IACH,WAAW,WAAW,aAAa;AACjC,YAAM,uBAAuB;AAAA,QAC3B,eAAe,IAAI;AAAA,QACnB,QAAQ;AAAA,QACR;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AACD,YAAM,oBAAoB;AAAA,QACxB,eAAe,IAAI;AAAA,QACnB,OAAO,IAAI;AAAA,QACX,OAAO;AAAA,QACP,SAAS;AAAA,QACT;AAAA,QACA,SAAS;AAAA,QACT,SAAS;AAAA,UACP,mBAAmB;AAAA,UACnB,SAAS;AAAA,QACX;AAAA,MACF,CAAC;AAAA,IACH,OAAO;AACL,YAAM,uBAAuB;AAAA,QAC3B,eAAe,IAAI;AAAA,QACnB,QAAQ;AAAA,QACR;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AACD,YAAM,oBAAoB;AAAA,QACxB,eAAe,IAAI;AAAA,QACnB,OAAO,IAAI;AAAA,QACX,OAAO;AAAA,QACP,SAAS,SAAS;AAAA,QAClB;AAAA,QACA,SAAS;AAAA,QACT,SAAS;AAAA,UACP,mBAAmB;AAAA,UACnB,SAAS,SAAS;AAAA,QACpB;AAAA,MACF,CAAC;AAAA,IACH;AAEA,QAAI,WAAW,aAAa;AAC1B,YAAM,kBAAkB,2BAA2B;AAAA,QACjD;AAAA,QACA,eAAe,IAAI;AAAA,QACnB,YAAY,IAAI;AAAA,QAChB,WAAW,IAAI;AAAA,QACf,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,MACxB,CAAC;AACD;AAAA,IACF;AAEA,QAAI,WAAW,aAAa;AAC1B,YAAM,kBAAkB,2BAA2B;AAAA,QACjD;AAAA,QACA,eAAe,IAAI;AAAA,QACnB,YAAY,IAAI;AAAA,QAChB,WAAW,IAAI;AAAA,QACf,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,MACxB,CAAC;AACD;AAAA,IACF;AAEA,UAAM,kBAAkB,wBAAwB;AAAA,MAC9C;AAAA,MACA,eAAe,IAAI;AAAA,MACnB,YAAY,IAAI;AAAA,MAChB,WAAW,IAAI;AAAA,MACf,OAAO,SAAS;AAAA,MAChB,UAAU,MAAM;AAAA,MAChB,gBAAgB,MAAM;AAAA,IACxB,CAAC;AAAA,EACH;AAEA,SAAO;AAAA,IACL,MAAM,UAAU,OAAe,WAAmB,OAAiC;AACjF,YAAM,MAAM,MAAM,eAAe,OAAO,OAAO,KAAK;AACpD,UAAI,CAAC,KAAK;AACR,eAAO,KAAK,6CAA6C,EAAE,MAAM,CAAC;AAClE;AAAA,MACF;AACA,UAAI,IAAI,WAAW,aAAa;AAC9B,YAAI,IAAI,eAAe;AACrB,gBAAM,gBAAgB,cAAc,IAAI,eAAe;AAAA,YACrD,UAAU,MAAM;AAAA,YAChB,gBAAgB,MAAM;AAAA,YACtB,QAAQ,MAAM;AAAA,UAChB,CAAC;AAAA,QACH;AACA;AAAA,MACF;AAEA,YAAM,cAAc,mBAAmB,IAAI,aAAa;AACxD,YAAM,UAAU,mBAAmB,WAAW;AAC9C,UAAI,CAAC,SAAS,cAAc;AAC1B,cAAM,IAAI,MAAM,6CAA6C,WAAW,EAAE;AAAA,MAC5E;AACA,YAAM,uBAAuB,QAAQ,yBAAyB;AAE9D,YAAM,cAAc,MAAM,8BAA8B,QAAQ,IAAI,eAAe,KAAK;AACxF,UAAI,CAAC,aAAa;AAChB,cAAM,IAAI,MAAM,eAAe,IAAI,aAAa,yBAAyB;AAAA,MAC3E;AAKA,YAAM,UAAU,IAAI,WAAW;AAC/B,YAAM,YAAY,MAAM,eAAe,WAAW,IAAI,IAAI,WAAW,KAAK;AAC1E,UAAI,CAAC,aAAa,UAAU,WAAW,WAAW;AAChD;AAAA,MACF;AACA,YAAM,kBAAkB,yBAAyB;AAAA,QAC/C,OAAO,IAAI;AAAA,QACX,eAAe,IAAI;AAAA,QACnB,YAAY,IAAI;AAAA,QAChB,WAAW,IAAI;AAAA,QACf;AAAA,QACA,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,MACxB,CAAC;AACD,YAAM,uBAAuB;AAAA,QAC3B,eAAe,IAAI;AAAA,QACnB,QAAQ;AAAA,QACR;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AACD,YAAM,oBAAoB;AAAA,QACxB,eAAe,IAAI;AAAA,QACnB,OAAO,IAAI;AAAA,QACX,OAAO;AAAA,QACP,SAAS;AAAA,QACT;AAAA,QACA,SAAS;AAAA,QACT,SAAS;AAAA,UACP,mBAAmB;AAAA,UACnB,SAAS,0BAA0B,IAAI,UAAU;AAAA,UACjD,YAAY,IAAI;AAAA,UAChB,WAAW,IAAI;AAAA,QACjB;AAAA,MACF,CAAC;AAED,UAAI,IAAI,eAAe;AACrB,cAAM,gBAAgB,SAAS,IAAI,eAAe;AAAA,UAChD,UAAU,MAAM;AAAA,UAChB,gBAAgB,MAAM;AAAA,UACtB,QAAQ,MAAM;AAAA,QAChB,CAAC;AAAA,MACH;AAEA,YAAM,UAAU,MAAM,eAAe,SAAS,IAAI,YAAY,KAAK;AACnE,UAAI,iBAAiB,MAAM,mBAAmB,IAAI,eAAe,KAAK;AACtE,UAAI,aAA4B;AAChC,UAAI,mBAAmB,UAAU,oBAAoB;AAErD,UAAI;AACF,yBAAiB,SAAS;AAAA,UACxB,QAAQ,aAAa;AAAA,YACnB,YAAY,IAAI;AAAA,YAChB,QAAQ,IAAI,UAAU;AAAA,YACtB;AAAA,YACA;AAAA,YACA;AAAA,YACA,OAAO,EAAE,gBAAgB,MAAM,gBAAgB,UAAU,MAAM,SAAS;AAAA,YACxE,OAAO,IAAI;AAAA,UACb,CAAC;AAAA,UACD,kBAAkB,IAAI,eAAe,KAAK;AAAA,UAC1C;AAAA,QACF,GAAG;AACD,cAAI,IAAI,iBAAiB,MAAM,gBAAgB,wBAAwB,IAAI,eAAe,MAAM,UAAU,MAAM,cAAc,GAAG;AAC/H,kBAAM,YAAY,IAAI,IAAI,aAAa,OAAO,QAAW,oBAAoB;AAC7E;AAAA,UACF;AAEA,gBAAM,QAAQ,oBAAoB,KAAK;AACvC,gBAAM,sBAAsB,MAAM,kBAAkB,MAAM,MAAM;AAChE,4BAAkB;AAClB,uBAAa,MAAM,iBAAiB;AAEpC,gBAAM,eAAe;AAAA,YACnB,IAAI;AAAA,YACJ;AAAA,cACE,GAAG;AAAA,cACH,kBAAkB;AAAA,YACpB;AAAA,YACA,MAAM;AAAA,YACN;AAAA,YACA;AAAA,UACF;AACA,8BAAoB;AAEpB,gBAAM,eAAe,IAAI,eAAe,gBAAgB,YAAY,KAAK;AACzE,gBAAM,yBAAyB,MAAM,4BAA4B,KAAK;AACtE,gBAAM,sBAAsB,IAAI,IAAI,IAAI,eAAe,MAAM,OAAO,KAAK;AAEzE,gBAAM,oBAAoB;AAAA,YACxB,eAAe,IAAI;AAAA,YACnB,OAAO,IAAI;AAAA,YACX,OAAO;AAAA,YACP,SAAS,MAAM,SAAS,KAAK,EAAE,SAC3B,MAAM,QAAQ,KAAK,IACnB,0BAA0B,MAAM,UAAU;AAAA,YAC9C;AAAA,YACA,SAAS;AAAA,YACT,SAAS;AAAA,cACP,mBAAmB;AAAA,cACnB,SAAS,aAAa,cAAc,GAAG,aAAa,OAAO,UAAU,KAAK,EAAE;AAAA,cAC5E;AAAA,cACA,WAAW,MAAM,MAAM;AAAA,cACvB;AAAA,cACA,QAAQ,MAAM;AAAA,YAChB;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF,SAAS,OAAO;AACd,YAAI,iBAAiB,+BAA+B;AAClD,iBAAO,KAAK,uEAAuE;AAAA,YACjF,OAAO,IAAI;AAAA,YACX,0BAA0B,MAAM;AAAA,UAClC,CAAC;AACD;AAAA,QACF;AACA,cAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU;AACzD,cAAM,sBAAsB;AAAA,UAC1B;AAAA,YACE,eAAe,IAAI;AAAA,YACnB,OAAO,IAAI;AAAA,YACX,OAAO;AAAA,YACP;AAAA,UACF;AAAA,UACA;AAAA,QACF;AACA,cAAM,YAAY,IAAI,IAAI,UAAU,OAAO,SAAS,oBAAoB;AACxE;AAAA,MACF;AAEA,YAAM,YAAY,IAAI,IAAI,aAAa,OAAO,QAAW,oBAAoB;AAAA,IAC/E;AAAA,IAEA,MAAM,UAAU,OAAe,WAAmB,OAAiC;AACjF,YAAM,MAAM,MAAM,eAAe,OAAO,OAAO,KAAK;AACpD,UAAI,CAAC,KAAK;AACR,eAAO,KAAK,6CAA6C,EAAE,MAAM,CAAC;AAClE;AAAA,MACF;AACA,UAAI,IAAI,WAAW,aAAa;AAC9B,YAAI,IAAI,eAAe;AACrB,gBAAM,gBAAgB,cAAc,IAAI,eAAe;AAAA,YACrD,UAAU,MAAM;AAAA,YAChB,gBAAgB,MAAM;AAAA,YACtB,QAAQ,MAAM;AAAA,UAChB,CAAC;AAAA,QACH;AACA;AAAA,MACF;AAEA,YAAM,cAAc,mBAAmB,IAAI,aAAa;AACxD,YAAM,UAAU,mBAAmB,WAAW;AAC9C,UAAI,CAAC,SAAS,cAAc;AAC1B,cAAM,IAAI,MAAM,6CAA6C,WAAW,EAAE;AAAA,MAC5E;AACA,YAAM,uBAAuB,QAAQ,yBAAyB;AAE9D,YAAM,cAAc,MAAM,8BAA8B,QAAQ,IAAI,eAAe,KAAK;AACxF,UAAI,CAAC,aAAa;AAChB,cAAM,IAAI,MAAM,eAAe,IAAI,aAAa,yBAAyB;AAAA,MAC3E;AAKA,YAAM,UAAU,IAAI,WAAW;AAC/B,YAAM,YAAY,MAAM,eAAe,WAAW,IAAI,IAAI,WAAW,KAAK;AAC1E,UAAI,CAAC,aAAa,UAAU,WAAW,WAAW;AAChD;AAAA,MACF;AACA,YAAM,kBAAkB,yBAAyB;AAAA,QAC/C,OAAO,IAAI;AAAA,QACX,eAAe,IAAI;AAAA,QACnB,YAAY,IAAI;AAAA,QAChB,WAAW,IAAI;AAAA,QACf;AAAA,QACA,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,MACxB,CAAC;AACD,YAAM,uBAAuB;AAAA,QAC3B,eAAe,IAAI;AAAA,QACnB,QAAQ;AAAA,QACR;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AACD,YAAM,oBAAoB;AAAA,QACxB,eAAe,IAAI;AAAA,QACnB,OAAO,IAAI;AAAA,QACX,OAAO;AAAA,QACP,SAAS;AAAA,QACT;AAAA,QACA,SAAS;AAAA,QACT,SAAS;AAAA,UACP,mBAAmB;AAAA,UACnB,SAAS,0BAA0B,IAAI,UAAU;AAAA,UACjD,YAAY,IAAI;AAAA,UAChB,WAAW,IAAI;AAAA,QACjB;AAAA,MACF,CAAC;AAED,UAAI,IAAI,eAAe;AACrB,cAAM,gBAAgB,SAAS,IAAI,eAAe;AAAA,UAChD,UAAU,MAAM;AAAA,UAChB,gBAAgB,MAAM;AAAA,UACtB,QAAQ,MAAM;AAAA,QAChB,CAAC;AAAA,MACH;AAEA,YAAM,UAAU,MAAM,eAAe,SAAS,IAAI,YAAY,KAAK;AACnE,UAAI,iBAAiB,MAAM,mBAAmB,IAAI,eAAe,KAAK;AACtE,UAAI,mBAAmB,UAAU,oBAAoB;AAErD,UAAI;AACF,yBAAiB,SAAS;AAAA,UACxB,QAAQ,aAAa;AAAA,YACnB,YAAY,IAAI;AAAA,YAChB,QAAQ,IAAI,UAAU;AAAA,YACtB;AAAA,YACA;AAAA,YACA;AAAA,YACA,OAAO,EAAE,gBAAgB,MAAM,gBAAgB,UAAU,MAAM,SAAS;AAAA,YACxE,OAAO,IAAI;AAAA,UACb,CAAC;AAAA,UACD,kBAAkB,IAAI,eAAe,KAAK;AAAA,UAC1C;AAAA,QACF,GAAG;AACD,cAAI,IAAI,iBAAiB,MAAM,gBAAgB,wBAAwB,IAAI,eAAe,MAAM,UAAU,MAAM,cAAc,GAAG;AAC/H,kBAAM,YAAY,IAAI,IAAI,aAAa,OAAO,QAAW,oBAAoB;AAC7E;AAAA,UACF;AAEA,gBAAM,QAAQ,oBAAoB,KAAK;AACvC,4BAAkB,MAAM;AAExB,gBAAM,eAAe;AAAA,YACnB,IAAI;AAAA,YACJ;AAAA,cACE,cAAc;AAAA,cACd,cAAc,MAAM;AAAA,cACpB,cAAc,MAAM;AAAA,cACpB,aAAa,MAAM;AAAA,cACnB,kBAAkB;AAAA,YACpB;AAAA,YACA,MAAM;AAAA,YACN;AAAA,YACA;AAAA,UACF;AACA,8BAAoB;AACpB,gBAAM,eAAe,IAAI,eAAe,gBAAgB,MAAM,KAAK;AACnE,gBAAM,sBAAsB,IAAI,IAAI,IAAI,eAAe,MAAM,SAAS,KAAK;AAE3E,gBAAM,oBAAoB;AAAA,YACxB,eAAe,IAAI;AAAA,YACnB,OAAO,IAAI;AAAA,YACX,OAAO;AAAA,YACP,SAAS,0BAA0B,MAAM,UAAU;AAAA,YACnD;AAAA,YACA,SAAS;AAAA,YACT,SAAS;AAAA,cACP,mBAAmB;AAAA,cACnB,SAAS,aAAa,cAAc;AAAA,cACpC;AAAA,cACA,WAAW,MAAM,QAAQ;AAAA,cACzB,QAAQ,MAAM;AAAA,YAChB;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF,SAAS,OAAO;AACd,YAAI,iBAAiB,+BAA+B;AAClD,iBAAO,KAAK,uEAAuE;AAAA,YACjF,OAAO,IAAI;AAAA,YACX,0BAA0B,MAAM;AAAA,UAClC,CAAC;AACD;AAAA,QACF;AACA,cAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU;AACzD,cAAM,sBAAsB;AAAA,UAC1B;AAAA,YACE,eAAe,IAAI;AAAA,YACnB,OAAO,IAAI;AAAA,YACX,OAAO;AAAA,YACP;AAAA,UACF;AAAA,UACA;AAAA,QACF;AACA,cAAM,YAAY,IAAI,IAAI,UAAU,OAAO,SAAS,oBAAoB;AACxE;AAAA,MACF;AAEA,YAAM,YAAY,IAAI,IAAI,aAAa,OAAO,QAAW,oBAAoB;AAAA,IAC/E;AAAA,EACF;AACF;",
6
6
  "names": []
7
7
  }
@@ -1,6 +1,7 @@
1
1
  const HEARTBEAT_INTERVAL_MS = 5e3;
2
2
  const STALE_JOB_TIMEOUT_SECONDS = 60;
3
3
  const STALE_PENDING_TIMEOUT_SECONDS = 900;
4
+ const STALE_SWEEP_ERROR_PREFIX = "Job stale:";
4
5
  function calculateEta(processedCount, totalCount, startedAt) {
5
6
  if (processedCount === 0 || totalCount === 0) return null;
6
7
  const elapsedMs = Date.now() - startedAt.getTime();
@@ -17,6 +18,7 @@ export {
17
18
  HEARTBEAT_INTERVAL_MS,
18
19
  STALE_JOB_TIMEOUT_SECONDS,
19
20
  STALE_PENDING_TIMEOUT_SECONDS,
21
+ STALE_SWEEP_ERROR_PREFIX,
20
22
  calculateEta,
21
23
  calculateProgressPercent
22
24
  };
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "version": 3,
3
3
  "sources": ["../../../../src/modules/progress/lib/progressService.ts"],
4
- "sourcesContent": ["import type { ProgressJob } from '../data/entities'\nimport type { CreateProgressJobInput, UpdateProgressInput, CompleteJobInput, FailJobInput } from '../data/validators'\n\nexport interface ProgressServiceContext {\n tenantId: string\n organizationId?: string | null\n userId?: string | null\n}\n\nexport interface ProgressService {\n createJob(input: CreateProgressJobInput, ctx: ProgressServiceContext): Promise<ProgressJob>\n startJob(jobId: string, ctx: ProgressServiceContext): Promise<ProgressJob>\n updateProgress(jobId: string, input: UpdateProgressInput, ctx: ProgressServiceContext): Promise<ProgressJob>\n incrementProgress(jobId: string, delta: number, ctx: ProgressServiceContext): Promise<ProgressJob>\n completeJob(jobId: string, input: CompleteJobInput | undefined, ctx: ProgressServiceContext): Promise<ProgressJob>\n failJob(jobId: string, input: FailJobInput, ctx: ProgressServiceContext): Promise<ProgressJob>\n cancelJob(jobId: string, ctx: ProgressServiceContext): Promise<ProgressJob>\n markCancelled(jobId: string, ctx: ProgressServiceContext): Promise<ProgressJob>\n isCancellationRequested(jobId: string, tenantId: string, organizationId?: string | null): Promise<boolean>\n getActiveJobs(ctx: ProgressServiceContext): Promise<ProgressJob[]>\n getRecentlyCompletedJobs(ctx: ProgressServiceContext, sinceSeconds?: number): Promise<ProgressJob[]>\n getJob(jobId: string, ctx: ProgressServiceContext): Promise<ProgressJob | null>\n markStaleJobsFailed(tenantId: string, timeoutSeconds?: number, organizationId?: string | null): Promise<number>\n}\n\nexport const HEARTBEAT_INTERVAL_MS = 5000\nexport const STALE_JOB_TIMEOUT_SECONDS = 60\nexport const STALE_PENDING_TIMEOUT_SECONDS = 900\n\nexport function calculateEta(\n processedCount: number,\n totalCount: number,\n startedAt: Date,\n): number | null {\n if (processedCount === 0 || totalCount === 0) return null\n\n const elapsedMs = Date.now() - startedAt.getTime()\n const rate = processedCount / elapsedMs\n const remaining = totalCount - processedCount\n\n if (rate <= 0) return null\n\n return Math.ceil(remaining / rate / 1000)\n}\n\nexport function calculateProgressPercent(processedCount: number, totalCount: number | null): number {\n if (!totalCount || totalCount <= 0) return 0\n return Math.min(100, Math.round((processedCount / totalCount) * 100))\n}\n"],
5
- "mappings": "AAyBO,MAAM,wBAAwB;AAC9B,MAAM,4BAA4B;AAClC,MAAM,gCAAgC;AAEtC,SAAS,aACd,gBACA,YACA,WACe;AACf,MAAI,mBAAmB,KAAK,eAAe,EAAG,QAAO;AAErD,QAAM,YAAY,KAAK,IAAI,IAAI,UAAU,QAAQ;AACjD,QAAM,OAAO,iBAAiB;AAC9B,QAAM,YAAY,aAAa;AAE/B,MAAI,QAAQ,EAAG,QAAO;AAEtB,SAAO,KAAK,KAAK,YAAY,OAAO,GAAI;AAC1C;AAEO,SAAS,yBAAyB,gBAAwB,YAAmC;AAClG,MAAI,CAAC,cAAc,cAAc,EAAG,QAAO;AAC3C,SAAO,KAAK,IAAI,KAAK,KAAK,MAAO,iBAAiB,aAAc,GAAG,CAAC;AACtE;",
4
+ "sourcesContent": ["import type { ProgressJob } from '../data/entities'\nimport type { CreateProgressJobInput, UpdateProgressInput, CompleteJobInput, FailJobInput } from '../data/validators'\n\nexport interface ProgressServiceContext {\n tenantId: string\n organizationId?: string | null\n userId?: string | null\n}\n\nexport interface ProgressService {\n createJob(input: CreateProgressJobInput, ctx: ProgressServiceContext): Promise<ProgressJob>\n startJob(jobId: string, ctx: ProgressServiceContext): Promise<ProgressJob>\n updateProgress(jobId: string, input: UpdateProgressInput, ctx: ProgressServiceContext): Promise<ProgressJob>\n incrementProgress(jobId: string, delta: number, ctx: ProgressServiceContext): Promise<ProgressJob>\n completeJob(jobId: string, input: CompleteJobInput | undefined, ctx: ProgressServiceContext): Promise<ProgressJob>\n failJob(jobId: string, input: FailJobInput, ctx: ProgressServiceContext): Promise<ProgressJob>\n cancelJob(jobId: string, ctx: ProgressServiceContext): Promise<ProgressJob>\n markCancelled(jobId: string, ctx: ProgressServiceContext): Promise<ProgressJob>\n isCancellationRequested(jobId: string, tenantId: string, organizationId?: string | null): Promise<boolean>\n getActiveJobs(ctx: ProgressServiceContext): Promise<ProgressJob[]>\n getRecentlyCompletedJobs(ctx: ProgressServiceContext, sinceSeconds?: number): Promise<ProgressJob[]>\n getJob(jobId: string, ctx: ProgressServiceContext): Promise<ProgressJob | null>\n markStaleJobsFailed(tenantId: string, timeoutSeconds?: number, organizationId?: string | null): Promise<number>\n // Optional so third-party ProgressService implementations keep compiling; callers must\n // optional-chain. Runs on a forked EntityManager, so it is safe to call while the shared\n // request/worker EM is mid-transaction (e.g. from a keepalive timer around adapter I/O).\n touchJobHeartbeat?(jobId: string, ctx: ProgressServiceContext): Promise<void>\n}\n\nexport const HEARTBEAT_INTERVAL_MS = 5000\nexport const STALE_JOB_TIMEOUT_SECONDS = 60\nexport const STALE_PENDING_TIMEOUT_SECONDS = 900\n\n// Every `errorMessage` the stale sweep writes starts with this, so recovery paths can tell\n// a job the sweep gave up on from one that failed for a real reason and must keep its\n// diagnostics. Declared here so the sweep and the revive filter cannot drift apart.\nexport const STALE_SWEEP_ERROR_PREFIX = 'Job stale:'\n\nexport function calculateEta(\n processedCount: number,\n totalCount: number,\n startedAt: Date,\n): number | null {\n if (processedCount === 0 || totalCount === 0) return null\n\n const elapsedMs = Date.now() - startedAt.getTime()\n const rate = processedCount / elapsedMs\n const remaining = totalCount - processedCount\n\n if (rate <= 0) return null\n\n return Math.ceil(remaining / rate / 1000)\n}\n\nexport function calculateProgressPercent(processedCount: number, totalCount: number | null): number {\n if (!totalCount || totalCount <= 0) return 0\n return Math.min(100, Math.round((processedCount / totalCount) * 100))\n}\n"],
5
+ "mappings": "AA6BO,MAAM,wBAAwB;AAC9B,MAAM,4BAA4B;AAClC,MAAM,gCAAgC;AAKtC,MAAM,2BAA2B;AAEjC,SAAS,aACd,gBACA,YACA,WACe;AACf,MAAI,mBAAmB,KAAK,eAAe,EAAG,QAAO;AAErD,QAAM,YAAY,KAAK,IAAI,IAAI,UAAU,QAAQ;AACjD,QAAM,OAAO,iBAAiB;AAC9B,QAAM,YAAY,aAAa;AAE/B,MAAI,QAAQ,EAAG,QAAO;AAEtB,SAAO,KAAK,KAAK,YAAY,OAAO,GAAI;AAC1C;AAEO,SAAS,yBAAyB,gBAAwB,YAAmC;AAClG,MAAI,CAAC,cAAc,cAAc,EAAG,QAAO;AAC3C,SAAO,KAAK,IAAI,KAAK,KAAK,MAAO,iBAAiB,aAAc,GAAG,CAAC;AACtE;",
6
6
  "names": []
7
7
  }
@@ -5,7 +5,8 @@ import {
5
5
  calculateProgressPercent,
6
6
  HEARTBEAT_INTERVAL_MS,
7
7
  STALE_JOB_TIMEOUT_SECONDS,
8
- STALE_PENDING_TIMEOUT_SECONDS
8
+ STALE_PENDING_TIMEOUT_SECONDS,
9
+ STALE_SWEEP_ERROR_PREFIX
9
10
  } from "./progressService.js";
10
11
  import { PROGRESS_EVENTS } from "./events.js";
11
12
  import { findOneWithDecryption } from "@open-mercato/shared/lib/encryption/find";
@@ -90,6 +91,37 @@ function createProgressService(em, eventBus) {
90
91
  }
91
92
  return {};
92
93
  }
94
+ async function startJobViaCas(targetEm, job, ctx, options = {}) {
95
+ const now = /* @__PURE__ */ new Date();
96
+ const startedAt = job.startedAt ?? now;
97
+ const filter = {
98
+ ...jobScopeFilter(job.id, ctx),
99
+ status: { $in: START_FROM_STATUSES }
100
+ };
101
+ if (options.staleSweptOnly) filter.errorMessage = { $like: `${STALE_SWEEP_ERROR_PREFIX}%` };
102
+ const affected = await targetEm.nativeUpdate(ProgressJob, filter, {
103
+ status: "running",
104
+ startedAt,
105
+ heartbeatAt: now,
106
+ finishedAt: null,
107
+ errorMessage: null,
108
+ errorStack: null,
109
+ updatedAt: now
110
+ });
111
+ if (affected === 0) return null;
112
+ job.status = "running";
113
+ job.startedAt = startedAt;
114
+ job.heartbeatAt = now;
115
+ job.finishedAt = null;
116
+ job.errorMessage = null;
117
+ job.errorStack = null;
118
+ await eventBus.emit(PROGRESS_EVENTS.JOB_STARTED, {
119
+ ...buildJobPayload(job),
120
+ tenantId: ctx.tenantId,
121
+ organizationId: job.organizationId ?? null
122
+ });
123
+ return job;
124
+ }
93
125
  async function persistAndMaybeBroadcast(entry, ctx) {
94
126
  const job = entry.job;
95
127
  const now = Date.now();
@@ -106,14 +138,28 @@ function createProgressService(em, eventBus) {
106
138
  updatedAt: /* @__PURE__ */ new Date(),
107
139
  ...buildBufferedCountData(entry)
108
140
  };
109
- const affected = await em.nativeUpdate(ProgressJob, {
141
+ const writableFilter = {
110
142
  ...jobScopeFilter(job.id, ctx),
111
143
  status: { $in: PROGRESS_WRITABLE_STATUSES }
112
- }, data);
144
+ };
145
+ let affected = await em.nativeUpdate(ProgressJob, writableFilter, data);
113
146
  if (affected === 0) {
114
- forgetJobThrottle(job.id);
115
147
  const fresh = await loadFreshJob(job.id, ctx);
116
- return fresh ?? job;
148
+ let revived = false;
149
+ if (fresh && fresh.status === "failed" && await startJobViaCas(em, fresh, ctx, { staleSweptOnly: true })) {
150
+ revived = true;
151
+ job.status = "running";
152
+ job.startedAt = fresh.startedAt;
153
+ job.finishedAt = null;
154
+ job.errorMessage = null;
155
+ job.errorStack = null;
156
+ affected = await em.nativeUpdate(ProgressJob, writableFilter, data);
157
+ }
158
+ if (affected === 0) {
159
+ forgetJobThrottle(job.id);
160
+ const latest = revived ? await loadFreshJob(job.id, ctx) : fresh;
161
+ return latest ?? job;
162
+ }
117
163
  }
118
164
  const persistedJob = usesAtomicIncrement ? await loadFreshJob(job.id, ctx) ?? job : job;
119
165
  entry.job = persistedJob;
@@ -161,42 +207,36 @@ function createProgressService(em, eventBus) {
161
207
  if (job.status === "running" || job.status === "cancelled" || job.status === "completed") {
162
208
  return job;
163
209
  }
210
+ const started = await startJobViaCas(em, job, ctx);
211
+ if (started) return started;
212
+ const fresh = await loadFreshJob(jobId, ctx);
213
+ return fresh ?? job;
214
+ },
215
+ async touchJobHeartbeat(jobId, ctx) {
216
+ const fork = em.fork();
164
217
  const now = /* @__PURE__ */ new Date();
165
- const affected = await em.nativeUpdate(ProgressJob, {
218
+ const affected = await fork.nativeUpdate(ProgressJob, {
166
219
  ...jobScopeFilter(jobId, ctx),
167
- status: { $in: START_FROM_STATUSES }
220
+ status: { $in: PROGRESS_WRITABLE_STATUSES }
168
221
  }, {
169
- status: "running",
170
- startedAt: now,
171
222
  heartbeatAt: now,
172
- finishedAt: null,
173
- errorMessage: null,
174
- errorStack: null,
175
223
  updatedAt: now
176
224
  });
177
- if (affected === 0) {
178
- const fresh = await loadFreshJob(jobId, ctx);
179
- return fresh ?? job;
180
- }
181
- job.status = "running";
182
- job.startedAt = now;
183
- job.heartbeatAt = now;
184
- job.finishedAt = null;
185
- job.errorMessage = null;
186
- job.errorStack = null;
187
- await eventBus.emit(PROGRESS_EVENTS.JOB_STARTED, {
188
- ...buildJobPayload(job),
189
- tenantId: ctx.tenantId,
190
- organizationId: job.organizationId ?? null
191
- });
192
- return job;
225
+ if (affected > 0) return;
226
+ const job = await fork.findOne(ProgressJob, jobScopeFilter(jobId, ctx), { disableIdentityMap: true });
227
+ if (!job || job.status !== "failed") return;
228
+ await startJobViaCas(fork, job, ctx, { staleSweptOnly: true });
193
229
  },
194
230
  async updateProgress(jobId, input, ctx) {
195
231
  const entry = await ensureThrottleEntry(jobId, ctx);
196
232
  const job = entry.job;
197
233
  if (TERMINAL_STATUSES.includes(job.status)) {
198
- forgetJobThrottle(jobId);
199
- return job;
234
+ const revived = job.status === "failed" && await startJobViaCas(em, job, ctx, { staleSweptOnly: true });
235
+ if (!revived) {
236
+ forgetJobThrottle(jobId);
237
+ return job;
238
+ }
239
+ entry.lastPersistedAt = 0;
200
240
  }
201
241
  if (input.processedCount !== void 0) {
202
242
  job.processedCount = input.processedCount;
@@ -226,8 +266,12 @@ function createProgressService(em, eventBus) {
226
266
  const entry = await ensureThrottleEntry(jobId, ctx);
227
267
  const job = entry.job;
228
268
  if (TERMINAL_STATUSES.includes(job.status)) {
229
- forgetJobThrottle(jobId);
230
- return job;
269
+ const revived = job.status === "failed" && await startJobViaCas(em, job, ctx, { staleSweptOnly: true });
270
+ if (!revived) {
271
+ forgetJobThrottle(jobId);
272
+ return job;
273
+ }
274
+ entry.lastPersistedAt = 0;
231
275
  }
232
276
  job.processedCount += delta;
233
277
  entry.pendingDelta += delta;
@@ -486,7 +530,7 @@ function createProgressService(em, eventBus) {
486
530
  };
487
531
  const staleRunning = await em.find(ProgressJob, { ...scope, ...staleFilter }, { disableIdentityMap: true });
488
532
  for (const job of staleRunning) {
489
- const errorMessage = `Job stale: no heartbeat for ${timeoutSeconds} seconds`;
533
+ const errorMessage = `${STALE_SWEEP_ERROR_PREFIX} no heartbeat for ${timeoutSeconds} seconds`;
490
534
  const affected = await em.nativeUpdate(ProgressJob, {
491
535
  id: job.id,
492
536
  tenantId: job.tenantId,
@@ -517,7 +561,7 @@ function createProgressService(em, eventBus) {
517
561
  };
518
562
  const stalePending = await em.find(ProgressJob, { ...scope, ...stalePendingFilter }, { disableIdentityMap: true });
519
563
  for (const job of stalePending) {
520
- const errorMessage = `Job stale: never started within ${STALE_PENDING_TIMEOUT_SECONDS} seconds`;
564
+ const errorMessage = `${STALE_SWEEP_ERROR_PREFIX} never started within ${STALE_PENDING_TIMEOUT_SECONDS} seconds`;
521
565
  const affected = await em.nativeUpdate(ProgressJob, {
522
566
  id: job.id,
523
567
  tenantId: job.tenantId,
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "version": 3,
3
3
  "sources": ["../../../../src/modules/progress/lib/progressServiceImpl.ts"],
4
- "sourcesContent": ["import type { EntityManager } from '@mikro-orm/postgresql'\nimport { raw } from '@mikro-orm/core'\nimport type { EntityData, FilterQuery } from '@mikro-orm/core'\nimport { ProgressJob, type ProgressJobStatus } from '../data/entities'\nimport type { ProgressService, ProgressServiceContext } from './progressService'\nimport {\n calculateEta,\n calculateProgressPercent,\n HEARTBEAT_INTERVAL_MS,\n STALE_JOB_TIMEOUT_SECONDS,\n STALE_PENDING_TIMEOUT_SECONDS,\n} from './progressService'\nimport { PROGRESS_EVENTS } from './events'\nimport { findOneWithDecryption } from '@open-mercato/shared/lib/encryption/find'\n\nconst DEFAULT_BROADCAST_MIN_INTERVAL_MS = 250\n\n// Minimum elapsed time between coalesced `progress.job.updated` broadcasts for a\n// single job. Bulk workers call updateProgress/incrementProgress once per record, and\n// every emit of the `clientBroadcast: true` event pays a serialized pg_notify roundtrip\n// plus a tenant-wide SSE fan-out. Setting the knob to 0 restores per-record emission.\n// Persistence is throttled separately: heartbeats hit the database at least every\n// HEARTBEAT_INTERVAL_MS regardless of this knob, so a high broadcast interval can\n// never starve the stale-job sweep of heartbeats.\nfunction resolveBroadcastMinIntervalMs(): number {\n const rawValue = process.env.OM_PROGRESS_BROADCAST_MIN_INTERVAL_MS\n if (rawValue == null || rawValue.trim() === '') return DEFAULT_BROADCAST_MIN_INTERVAL_MS\n const parsed = Number.parseInt(rawValue, 10)\n if (!Number.isFinite(parsed) || parsed < 0) return DEFAULT_BROADCAST_MIN_INTERVAL_MS\n return parsed\n}\n\n// Statuses a given transition is allowed to move FROM. Every write below is a\n// compare-and-swap (`nativeUpdate` guarded on status), so concurrent writers in other\n// processes can never resurrect a terminal job or double-apply a transition \u2014 the\n// UPDATE that matches zero rows simply lost the race and must not emit events.\nconst TERMINAL_STATUSES: readonly ProgressJobStatus[] = ['completed', 'failed', 'cancelled']\nconst PROGRESS_WRITABLE_STATUSES: readonly ProgressJobStatus[] = ['pending', 'running']\n// `failed` is restartable/completable so a job wrongly swept as stale (worker alive but\n// slow) or retried by an at-least-once queue converges to its true outcome.\nconst START_FROM_STATUSES: readonly ProgressJobStatus[] = ['pending', 'failed']\nconst COMPLETE_FROM_STATUSES: readonly ProgressJobStatus[] = ['pending', 'running', 'failed']\nconst FAIL_FROM_STATUSES: readonly ProgressJobStatus[] = ['pending', 'running']\nconst CANCEL_FROM_STATUSES: readonly ProgressJobStatus[] = ['pending', 'running', 'failed']\n\ntype JobUpdateThrottleEntry = {\n job: ProgressJob\n lastBroadcastAt: number\n lastPersistedAt: number\n lastBroadcastPercent: number\n pendingDelta: number\n absoluteCountsPending: boolean\n}\n\nfunction buildJobPayload(job: ProgressJob): Record<string, unknown> {\n return {\n jobId: job.id,\n jobType: job.jobType,\n name: job.name,\n description: job.description ?? null,\n status: job.status,\n progressPercent: job.progressPercent,\n processedCount: job.processedCount,\n totalCount: job.totalCount ?? null,\n etaSeconds: job.etaSeconds ?? null,\n cancellable: job.cancellable,\n meta: job.meta ?? null,\n startedAt: job.startedAt?.toISOString() ?? null,\n finishedAt: job.finishedAt?.toISOString() ?? null,\n }\n}\n\nexport function createProgressService(em: EntityManager, eventBus: { emit: (event: string, payload: Record<string, unknown>) => Promise<void> }): ProgressService {\n const broadcastMinIntervalMs = resolveBroadcastMinIntervalMs()\n // Per-job coalescing state, scoped to this service instance (request/worker scope).\n // The cached entity is a DETACHED snapshot (loaded with disableIdentityMap) that doubles\n // as the in-memory buffer: intermediate updates mutate it without touching the shared\n // identity map or UnitOfWork, so an unrelated em.flush() elsewhere can never write the\n // buffered values. Persistence happens exclusively through the CAS nativeUpdate below.\n const jobUpdateThrottle = new Map<string, JobUpdateThrottleEntry>()\n\n function jobScopeFilter(jobId: string, ctx: ProgressServiceContext) {\n return {\n id: jobId,\n tenantId: ctx.tenantId,\n ...(ctx.organizationId ? { organizationId: ctx.organizationId } : {}),\n }\n }\n\n async function loadFreshJob(jobId: string, ctx: ProgressServiceContext): Promise<ProgressJob | null> {\n return em.findOne(ProgressJob, jobScopeFilter(jobId, ctx), { disableIdentityMap: true })\n }\n\n async function ensureThrottleEntry(jobId: string, ctx: ProgressServiceContext): Promise<JobUpdateThrottleEntry> {\n const cached = jobUpdateThrottle.get(jobId)\n if (cached) return cached\n const job = await em.findOneOrFail(ProgressJob, jobScopeFilter(jobId, ctx), { disableIdentityMap: true })\n const entry: JobUpdateThrottleEntry = {\n job,\n lastBroadcastAt: 0,\n lastPersistedAt: 0,\n lastBroadcastPercent: job.progressPercent,\n pendingDelta: 0,\n absoluteCountsPending: false,\n }\n jobUpdateThrottle.set(jobId, entry)\n return entry\n }\n\n function forgetJobThrottle(jobId: string) {\n jobUpdateThrottle.delete(jobId)\n }\n\n function buildBufferedCountData(entry: JobUpdateThrottleEntry): EntityData<ProgressJob> {\n if (entry.absoluteCountsPending || !Number.isSafeInteger(entry.pendingDelta)) {\n return { processedCount: entry.job.processedCount }\n }\n if (entry.pendingDelta !== 0) {\n const processedCount = raw(`processed_count + ${entry.pendingDelta}`)\n const totalCount = entry.job.totalCount\n if (Number.isSafeInteger(totalCount) && totalCount != null && totalCount > 0) {\n return {\n processedCount,\n progressPercent: raw(\n `least(100, round(((processed_count + ${entry.pendingDelta})::numeric / ${totalCount}) * 100))`,\n ),\n }\n }\n return { processedCount }\n }\n return {}\n }\n\n async function persistAndMaybeBroadcast(entry: JobUpdateThrottleEntry, ctx: ProgressServiceContext): Promise<ProgressJob> {\n const job = entry.job\n const now = Date.now()\n const shouldBroadcast =\n broadcastMinIntervalMs <= 0 ||\n now - entry.lastBroadcastAt >= broadcastMinIntervalMs ||\n Math.abs(job.progressPercent - entry.lastBroadcastPercent) >= 1\n const shouldPersist = shouldBroadcast || now - entry.lastPersistedAt >= HEARTBEAT_INTERVAL_MS\n\n if (!shouldPersist) return job\n\n const usesAtomicIncrement =\n !entry.absoluteCountsPending && Number.isSafeInteger(entry.pendingDelta) && entry.pendingDelta !== 0\n const data: EntityData<ProgressJob> = {\n progressPercent: job.progressPercent,\n totalCount: job.totalCount ?? null,\n etaSeconds: job.etaSeconds ?? null,\n meta: job.meta ?? null,\n heartbeatAt: job.heartbeatAt ?? new Date(),\n updatedAt: new Date(),\n ...buildBufferedCountData(entry),\n }\n const affected = await em.nativeUpdate(ProgressJob, {\n ...jobScopeFilter(job.id, ctx),\n status: { $in: PROGRESS_WRITABLE_STATUSES as ProgressJobStatus[] },\n } as FilterQuery<ProgressJob>, data)\n\n if (affected === 0) {\n // The job reached a terminal state in another process; stop writing to it.\n forgetJobThrottle(job.id)\n const fresh = await loadFreshJob(job.id, ctx)\n return fresh ?? job\n }\n\n const persistedJob = usesAtomicIncrement ? (await loadFreshJob(job.id, ctx)) ?? job : job\n entry.job = persistedJob\n entry.pendingDelta = 0\n entry.absoluteCountsPending = false\n entry.lastPersistedAt = now\n\n if (shouldBroadcast) {\n await eventBus.emit(PROGRESS_EVENTS.JOB_UPDATED, {\n ...buildJobPayload(persistedJob),\n tenantId: ctx.tenantId,\n organizationId: persistedJob.organizationId ?? null,\n })\n entry.lastBroadcastAt = now\n entry.lastBroadcastPercent = persistedJob.progressPercent\n }\n\n return persistedJob\n }\n\n return {\n async createJob(input, ctx) {\n const job = em.create(ProgressJob, {\n jobType: input.jobType,\n name: input.name,\n description: input.description,\n totalCount: input.totalCount,\n cancellable: input.cancellable ?? false,\n meta: input.meta,\n parentJobId: input.parentJobId,\n partitionIndex: input.partitionIndex,\n partitionCount: input.partitionCount,\n startedByUserId: ctx.userId,\n tenantId: ctx.tenantId,\n organizationId: ctx.organizationId,\n status: 'pending',\n })\n\n await em.persist(job).flush()\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_CREATED, {\n ...buildJobPayload(job),\n tenantId: ctx.tenantId,\n organizationId: ctx.organizationId,\n })\n\n return job\n },\n\n async startJob(jobId, ctx) {\n const job = await em.findOneOrFail(ProgressJob, jobScopeFilter(jobId, ctx), { disableIdentityMap: true })\n if (job.status === 'running' || job.status === 'cancelled' || job.status === 'completed') {\n return job\n }\n\n const now = new Date()\n const affected = await em.nativeUpdate(ProgressJob, {\n ...jobScopeFilter(jobId, ctx),\n status: { $in: START_FROM_STATUSES as ProgressJobStatus[] },\n } as FilterQuery<ProgressJob>, {\n status: 'running',\n startedAt: now,\n heartbeatAt: now,\n finishedAt: null,\n errorMessage: null,\n errorStack: null,\n updatedAt: now,\n })\n\n if (affected === 0) {\n const fresh = await loadFreshJob(jobId, ctx)\n return fresh ?? job\n }\n\n job.status = 'running'\n job.startedAt = now\n job.heartbeatAt = now\n job.finishedAt = null\n job.errorMessage = null\n job.errorStack = null\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_STARTED, {\n ...buildJobPayload(job),\n tenantId: ctx.tenantId,\n organizationId: job.organizationId ?? null,\n })\n\n return job\n },\n\n async updateProgress(jobId, input, ctx) {\n const entry = await ensureThrottleEntry(jobId, ctx)\n const job = entry.job\n if (TERMINAL_STATUSES.includes(job.status)) {\n forgetJobThrottle(jobId)\n return job\n }\n\n if (input.processedCount !== undefined) {\n job.processedCount = input.processedCount\n entry.absoluteCountsPending = true\n entry.pendingDelta = 0\n }\n if (input.totalCount !== undefined) {\n job.totalCount = input.totalCount\n }\n if (input.meta !== undefined) {\n job.meta = { ...job.meta, ...input.meta }\n }\n\n if (input.progressPercent !== undefined) {\n job.progressPercent = input.progressPercent\n } else if (job.totalCount) {\n job.progressPercent = calculateProgressPercent(job.processedCount, job.totalCount)\n }\n\n if (input.etaSeconds !== undefined) {\n job.etaSeconds = input.etaSeconds\n } else if (job.startedAt && job.totalCount) {\n job.etaSeconds = calculateEta(job.processedCount, job.totalCount, job.startedAt)\n }\n\n job.heartbeatAt = new Date()\n\n return persistAndMaybeBroadcast(entry, ctx)\n },\n\n async incrementProgress(jobId, delta, ctx) {\n const entry = await ensureThrottleEntry(jobId, ctx)\n const job = entry.job\n if (TERMINAL_STATUSES.includes(job.status)) {\n forgetJobThrottle(jobId)\n return job\n }\n\n job.processedCount += delta\n entry.pendingDelta += delta\n job.heartbeatAt = new Date()\n\n if (job.totalCount) {\n job.progressPercent = calculateProgressPercent(job.processedCount, job.totalCount)\n if (job.startedAt) {\n job.etaSeconds = calculateEta(job.processedCount, job.totalCount, job.startedAt)\n }\n }\n\n return persistAndMaybeBroadcast(entry, ctx)\n },\n\n async completeJob(jobId, input, ctx) {\n const job = await loadFreshJob(jobId, ctx)\n if (!job) throw new Error(`Job ${jobId} not found`)\n if (job.status === 'cancelled' || job.status === 'completed') {\n forgetJobThrottle(jobId)\n return job\n }\n\n const entry = jobUpdateThrottle.get(jobId)\n const snapshot = entry?.job ?? job\n const now = new Date()\n const data: EntityData<ProgressJob> = {\n status: 'completed',\n finishedAt: now,\n etaSeconds: 0,\n updatedAt: now,\n ...(input?.resultSummary ? { resultSummary: input.resultSummary } : {}),\n ...(entry\n ? {\n totalCount: snapshot.totalCount ?? null,\n meta: snapshot.meta ?? null,\n ...buildBufferedCountData(entry),\n }\n : {}),\n progressPercent: 100,\n }\n\n const affected = await em.nativeUpdate(ProgressJob, {\n ...jobScopeFilter(jobId, ctx),\n status: { $in: COMPLETE_FROM_STATUSES as ProgressJobStatus[] },\n } as FilterQuery<ProgressJob>, data)\n forgetJobThrottle(jobId)\n\n if (affected === 0) {\n const fresh = await loadFreshJob(jobId, ctx)\n return fresh ?? job\n }\n\n snapshot.status = 'completed'\n snapshot.finishedAt = now\n snapshot.progressPercent = 100\n snapshot.etaSeconds = 0\n if (input?.resultSummary) {\n snapshot.resultSummary = input.resultSummary\n }\n\n const persistedJob = (await loadFreshJob(jobId, ctx)) ?? snapshot\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_COMPLETED, {\n ...buildJobPayload(persistedJob),\n resultSummary: persistedJob.resultSummary,\n tenantId: ctx.tenantId,\n organizationId: persistedJob.organizationId ?? null,\n })\n\n return persistedJob\n },\n\n async failJob(jobId, input, ctx) {\n const job = await loadFreshJob(jobId, ctx)\n if (!job) throw new Error(`Job ${jobId} not found`)\n if (TERMINAL_STATUSES.includes(job.status)) {\n forgetJobThrottle(jobId)\n return job\n }\n\n const entry = jobUpdateThrottle.get(jobId)\n const snapshot = entry?.job ?? job\n const now = new Date()\n const data: EntityData<ProgressJob> = {\n status: 'failed',\n finishedAt: now,\n errorMessage: input.errorMessage,\n errorStack: input.errorStack,\n ...(input.resultSummary ? { resultSummary: input.resultSummary } : {}),\n updatedAt: now,\n ...(entry\n ? {\n totalCount: snapshot.totalCount ?? null,\n meta: snapshot.meta ?? null,\n ...buildBufferedCountData(entry),\n }\n : {}),\n }\n\n const affected = await em.nativeUpdate(ProgressJob, {\n ...jobScopeFilter(jobId, ctx),\n status: { $in: FAIL_FROM_STATUSES as ProgressJobStatus[] },\n } as FilterQuery<ProgressJob>, data)\n forgetJobThrottle(jobId)\n\n if (affected === 0) {\n const fresh = await loadFreshJob(jobId, ctx)\n return fresh ?? job\n }\n\n snapshot.status = 'failed'\n snapshot.finishedAt = now\n snapshot.errorMessage = input.errorMessage\n snapshot.errorStack = input.errorStack\n if (input.resultSummary) {\n snapshot.resultSummary = input.resultSummary\n }\n\n const persistedJob = (await loadFreshJob(jobId, ctx)) ?? snapshot\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_FAILED, {\n ...buildJobPayload(persistedJob),\n errorMessage: persistedJob.errorMessage,\n tenantId: ctx.tenantId,\n organizationId: persistedJob.organizationId ?? null,\n })\n\n return persistedJob\n },\n\n async cancelJob(jobId, ctx) {\n const job = await em.findOneOrFail(ProgressJob, jobScopeFilter(jobId, ctx), { disableIdentityMap: true })\n if (TERMINAL_STATUSES.includes(job.status)) {\n // The job finished while the user clicked cancel \u2014 a benign race, not an error.\n return job\n }\n if (!job.cancellable) {\n throw new Error(`Job ${jobId} is not cancellable`)\n }\n\n const now = new Date()\n const cancelledNow = await em.nativeUpdate(ProgressJob, {\n ...jobScopeFilter(jobId, ctx),\n cancellable: true,\n status: 'pending',\n } as FilterQuery<ProgressJob>, {\n status: 'cancelled',\n cancelRequestedAt: now,\n cancelledByUserId: ctx.userId ?? null,\n finishedAt: now,\n etaSeconds: 0,\n updatedAt: now,\n })\n\n let requested = 0\n if (cancelledNow === 0) {\n requested = await em.nativeUpdate(ProgressJob, {\n ...jobScopeFilter(jobId, ctx),\n cancellable: true,\n status: 'running',\n } as FilterQuery<ProgressJob>, {\n cancelRequestedAt: now,\n cancelledByUserId: ctx.userId ?? null,\n updatedAt: now,\n })\n }\n\n if (cancelledNow === 0 && requested === 0) {\n // Raced into a terminal state between the read and the CAS.\n const fresh = await loadFreshJob(jobId, ctx)\n return fresh ?? job\n }\n\n forgetJobThrottle(jobId)\n if (cancelledNow > 0) {\n job.status = 'cancelled'\n job.finishedAt = now\n job.etaSeconds = 0\n }\n job.cancelRequestedAt = now\n job.cancelledByUserId = ctx.userId ?? null\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_CANCELLED, {\n ...buildJobPayload(job),\n tenantId: ctx.tenantId,\n organizationId: job.organizationId ?? null,\n })\n\n return job\n },\n\n async markCancelled(jobId, ctx) {\n const job = await loadFreshJob(jobId, ctx)\n if (!job) throw new Error(`Job ${jobId} not found`)\n if (job.status === 'cancelled' || job.status === 'completed') {\n forgetJobThrottle(jobId)\n return job\n }\n\n const now = new Date()\n const cancelRequestedAt = job.cancelRequestedAt ?? now\n const cancelledByUserId = job.cancelledByUserId ?? ctx.userId ?? null\n const finishedAt = job.finishedAt ?? now\n\n const affected = await em.nativeUpdate(ProgressJob, {\n ...jobScopeFilter(jobId, ctx),\n status: { $in: CANCEL_FROM_STATUSES as ProgressJobStatus[] },\n } as FilterQuery<ProgressJob>, {\n status: 'cancelled',\n cancelRequestedAt,\n cancelledByUserId,\n finishedAt,\n etaSeconds: 0,\n updatedAt: now,\n })\n forgetJobThrottle(jobId)\n\n if (affected === 0) {\n const fresh = await loadFreshJob(jobId, ctx)\n return fresh ?? job\n }\n\n job.status = 'cancelled'\n job.cancelRequestedAt = cancelRequestedAt\n job.cancelledByUserId = cancelledByUserId\n job.finishedAt = finishedAt\n job.etaSeconds = 0\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_CANCELLED, {\n ...buildJobPayload(job),\n tenantId: ctx.tenantId,\n organizationId: job.organizationId ?? null,\n })\n\n return job\n },\n\n async isCancellationRequested(jobId, tenantId, organizationId) {\n // disableIdentityMap forces a fresh read: a managed copy of the job in this\n // EntityManager (loaded by updateProgress) must never mask a cancellation\n // requested from another process.\n const job = await findOneWithDecryption(em, ProgressJob, {\n id: jobId,\n tenantId,\n ...(organizationId ? { organizationId } : {}),\n }, { disableIdentityMap: true })\n return job?.cancelRequestedAt != null\n },\n\n async getActiveJobs(ctx) {\n return em.find(ProgressJob, {\n tenantId: ctx.tenantId,\n ...(ctx.organizationId ? { organizationId: ctx.organizationId } : {}),\n status: { $in: ['pending', 'running'] },\n parentJobId: null,\n }, {\n orderBy: { createdAt: 'DESC' },\n limit: 50,\n })\n },\n\n async getRecentlyCompletedJobs(ctx, sinceSeconds = 30) {\n const cutoff = new Date(Date.now() - sinceSeconds * 1000)\n return em.find(ProgressJob, {\n tenantId: ctx.tenantId,\n ...(ctx.organizationId ? { organizationId: ctx.organizationId } : {}),\n status: { $in: ['completed', 'failed'] },\n finishedAt: { $gte: cutoff },\n parentJobId: null,\n }, {\n orderBy: { finishedAt: 'DESC' },\n limit: 10,\n })\n },\n\n async getJob(jobId, ctx) {\n return em.findOne(ProgressJob, {\n id: jobId,\n tenantId: ctx.tenantId,\n ...(ctx.organizationId ? { organizationId: ctx.organizationId } : {}),\n })\n },\n\n async markStaleJobsFailed(tenantId: string, timeoutSeconds = STALE_JOB_TIMEOUT_SECONDS, organizationId?: string | null) {\n const now = new Date()\n const cutoff = new Date(now.getTime() - timeoutSeconds * 1000)\n const scope = {\n tenantId,\n ...(organizationId ? { organizationId } : {}),\n }\n let failedCount = 0\n\n const staleFilter = {\n status: 'running' as ProgressJobStatus,\n $or: [\n { heartbeatAt: { $lt: cutoff } },\n {\n heartbeatAt: null,\n startedAt: { $lt: cutoff },\n },\n ],\n }\n const staleRunning = await em.find(ProgressJob, { ...scope, ...staleFilter }, { disableIdentityMap: true })\n\n for (const job of staleRunning) {\n const errorMessage = `Job stale: no heartbeat for ${timeoutSeconds} seconds`\n // Per-row CAS (re-checking the staleness condition): exactly one concurrent\n // sweeper wins, and a heartbeat that landed after the SELECT aborts the failure.\n const affected = await em.nativeUpdate(ProgressJob, {\n id: job.id,\n tenantId: job.tenantId,\n ...staleFilter,\n } as FilterQuery<ProgressJob>, {\n status: 'failed',\n finishedAt: now,\n errorMessage,\n updatedAt: now,\n })\n if (affected === 0) continue\n\n failedCount += 1\n job.status = 'failed'\n job.finishedAt = now\n job.errorMessage = errorMessage\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_FAILED, {\n ...buildJobPayload(job),\n errorMessage: job.errorMessage,\n tenantId: job.tenantId,\n stale: true,\n organizationId: job.organizationId ?? null,\n })\n }\n\n // Jobs stuck in `pending` (enqueue failed, worker died before startJob) have no\n // heartbeat, so they need their own sweep or they stay in the active list forever.\n // A late queue delivery still recovers such a job: startJob transitions failed \u2192 running.\n const pendingCutoff = new Date(now.getTime() - STALE_PENDING_TIMEOUT_SECONDS * 1000)\n const stalePendingFilter = {\n status: 'pending' as ProgressJobStatus,\n createdAt: { $lt: pendingCutoff },\n }\n const stalePending = await em.find(ProgressJob, { ...scope, ...stalePendingFilter }, { disableIdentityMap: true })\n\n for (const job of stalePending) {\n const errorMessage = `Job stale: never started within ${STALE_PENDING_TIMEOUT_SECONDS} seconds`\n const affected = await em.nativeUpdate(ProgressJob, {\n id: job.id,\n tenantId: job.tenantId,\n ...stalePendingFilter,\n } as FilterQuery<ProgressJob>, {\n status: 'failed',\n finishedAt: now,\n errorMessage,\n updatedAt: now,\n })\n if (affected === 0) continue\n\n failedCount += 1\n job.status = 'failed'\n job.finishedAt = now\n job.errorMessage = errorMessage\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_FAILED, {\n ...buildJobPayload(job),\n errorMessage: job.errorMessage,\n tenantId: job.tenantId,\n stale: true,\n organizationId: job.organizationId ?? null,\n })\n }\n\n return failedCount\n },\n }\n}\n"],
5
- "mappings": "AACA,SAAS,WAAW;AAEpB,SAAS,mBAA2C;AAEpD;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,uBAAuB;AAChC,SAAS,6BAA6B;AAEtC,MAAM,oCAAoC;AAS1C,SAAS,gCAAwC;AAC/C,QAAM,WAAW,QAAQ,IAAI;AAC7B,MAAI,YAAY,QAAQ,SAAS,KAAK,MAAM,GAAI,QAAO;AACvD,QAAM,SAAS,OAAO,SAAS,UAAU,EAAE;AAC3C,MAAI,CAAC,OAAO,SAAS,MAAM,KAAK,SAAS,EAAG,QAAO;AACnD,SAAO;AACT;AAMA,MAAM,oBAAkD,CAAC,aAAa,UAAU,WAAW;AAC3F,MAAM,6BAA2D,CAAC,WAAW,SAAS;AAGtF,MAAM,sBAAoD,CAAC,WAAW,QAAQ;AAC9E,MAAM,yBAAuD,CAAC,WAAW,WAAW,QAAQ;AAC5F,MAAM,qBAAmD,CAAC,WAAW,SAAS;AAC9E,MAAM,uBAAqD,CAAC,WAAW,WAAW,QAAQ;AAW1F,SAAS,gBAAgB,KAA2C;AAClE,SAAO;AAAA,IACL,OAAO,IAAI;AAAA,IACX,SAAS,IAAI;AAAA,IACb,MAAM,IAAI;AAAA,IACV,aAAa,IAAI,eAAe;AAAA,IAChC,QAAQ,IAAI;AAAA,IACZ,iBAAiB,IAAI;AAAA,IACrB,gBAAgB,IAAI;AAAA,IACpB,YAAY,IAAI,cAAc;AAAA,IAC9B,YAAY,IAAI,cAAc;AAAA,IAC9B,aAAa,IAAI;AAAA,IACjB,MAAM,IAAI,QAAQ;AAAA,IAClB,WAAW,IAAI,WAAW,YAAY,KAAK;AAAA,IAC3C,YAAY,IAAI,YAAY,YAAY,KAAK;AAAA,EAC/C;AACF;AAEO,SAAS,sBAAsB,IAAmB,UAAyG;AAChK,QAAM,yBAAyB,8BAA8B;AAM7D,QAAM,oBAAoB,oBAAI,IAAoC;AAElE,WAAS,eAAe,OAAe,KAA6B;AAClE,WAAO;AAAA,MACL,IAAI;AAAA,MACJ,UAAU,IAAI;AAAA,MACd,GAAI,IAAI,iBAAiB,EAAE,gBAAgB,IAAI,eAAe,IAAI,CAAC;AAAA,IACrE;AAAA,EACF;AAEA,iBAAe,aAAa,OAAe,KAA0D;AACnG,WAAO,GAAG,QAAQ,aAAa,eAAe,OAAO,GAAG,GAAG,EAAE,oBAAoB,KAAK,CAAC;AAAA,EACzF;AAEA,iBAAe,oBAAoB,OAAe,KAA8D;AAC9G,UAAM,SAAS,kBAAkB,IAAI,KAAK;AAC1C,QAAI,OAAQ,QAAO;AACnB,UAAM,MAAM,MAAM,GAAG,cAAc,aAAa,eAAe,OAAO,GAAG,GAAG,EAAE,oBAAoB,KAAK,CAAC;AACxG,UAAM,QAAgC;AAAA,MACpC;AAAA,MACA,iBAAiB;AAAA,MACjB,iBAAiB;AAAA,MACjB,sBAAsB,IAAI;AAAA,MAC1B,cAAc;AAAA,MACd,uBAAuB;AAAA,IACzB;AACA,sBAAkB,IAAI,OAAO,KAAK;AAClC,WAAO;AAAA,EACT;AAEA,WAAS,kBAAkB,OAAe;AACxC,sBAAkB,OAAO,KAAK;AAAA,EAChC;AAEA,WAAS,uBAAuB,OAAwD;AACtF,QAAI,MAAM,yBAAyB,CAAC,OAAO,cAAc,MAAM,YAAY,GAAG;AAC5E,aAAO,EAAE,gBAAgB,MAAM,IAAI,eAAe;AAAA,IACpD;AACA,QAAI,MAAM,iBAAiB,GAAG;AAC5B,YAAM,iBAAiB,IAAI,qBAAqB,MAAM,YAAY,EAAE;AACpE,YAAM,aAAa,MAAM,IAAI;AAC7B,UAAI,OAAO,cAAc,UAAU,KAAK,cAAc,QAAQ,aAAa,GAAG;AAC5E,eAAO;AAAA,UACL;AAAA,UACA,iBAAiB;AAAA,YACf,wCAAwC,MAAM,YAAY,gBAAgB,UAAU;AAAA,UACtF;AAAA,QACF;AAAA,MACF;AACA,aAAO,EAAE,eAAe;AAAA,IAC1B;AACA,WAAO,CAAC;AAAA,EACV;AAEA,iBAAe,yBAAyB,OAA+B,KAAmD;AACxH,UAAM,MAAM,MAAM;AAClB,UAAM,MAAM,KAAK,IAAI;AACrB,UAAM,kBACJ,0BAA0B,KAC1B,MAAM,MAAM,mBAAmB,0BAC/B,KAAK,IAAI,IAAI,kBAAkB,MAAM,oBAAoB,KAAK;AAChE,UAAM,gBAAgB,mBAAmB,MAAM,MAAM,mBAAmB;AAExE,QAAI,CAAC,cAAe,QAAO;AAE3B,UAAM,sBACJ,CAAC,MAAM,yBAAyB,OAAO,cAAc,MAAM,YAAY,KAAK,MAAM,iBAAiB;AACrG,UAAM,OAAgC;AAAA,MACpC,iBAAiB,IAAI;AAAA,MACrB,YAAY,IAAI,cAAc;AAAA,MAC9B,YAAY,IAAI,cAAc;AAAA,MAC9B,MAAM,IAAI,QAAQ;AAAA,MAClB,aAAa,IAAI,eAAe,oBAAI,KAAK;AAAA,MACzC,WAAW,oBAAI,KAAK;AAAA,MACpB,GAAG,uBAAuB,KAAK;AAAA,IACjC;AACA,UAAM,WAAW,MAAM,GAAG,aAAa,aAAa;AAAA,MAClD,GAAG,eAAe,IAAI,IAAI,GAAG;AAAA,MAC7B,QAAQ,EAAE,KAAK,2BAAkD;AAAA,IACnE,GAA+B,IAAI;AAEnC,QAAI,aAAa,GAAG;AAElB,wBAAkB,IAAI,EAAE;AACxB,YAAM,QAAQ,MAAM,aAAa,IAAI,IAAI,GAAG;AAC5C,aAAO,SAAS;AAAA,IAClB;AAEA,UAAM,eAAe,sBAAuB,MAAM,aAAa,IAAI,IAAI,GAAG,KAAM,MAAM;AACtF,UAAM,MAAM;AACZ,UAAM,eAAe;AACrB,UAAM,wBAAwB;AAC9B,UAAM,kBAAkB;AAExB,QAAI,iBAAiB;AACnB,YAAM,SAAS,KAAK,gBAAgB,aAAa;AAAA,QAC/C,GAAG,gBAAgB,YAAY;AAAA,QAC/B,UAAU,IAAI;AAAA,QACd,gBAAgB,aAAa,kBAAkB;AAAA,MACjD,CAAC;AACD,YAAM,kBAAkB;AACxB,YAAM,uBAAuB,aAAa;AAAA,IAC5C;AAEA,WAAO;AAAA,EACT;AAEA,SAAO;AAAA,IACL,MAAM,UAAU,OAAO,KAAK;AAC1B,YAAM,MAAM,GAAG,OAAO,aAAa;AAAA,QACjC,SAAS,MAAM;AAAA,QACf,MAAM,MAAM;AAAA,QACZ,aAAa,MAAM;AAAA,QACnB,YAAY,MAAM;AAAA,QAClB,aAAa,MAAM,eAAe;AAAA,QAClC,MAAM,MAAM;AAAA,QACZ,aAAa,MAAM;AAAA,QACnB,gBAAgB,MAAM;AAAA,QACtB,gBAAgB,MAAM;AAAA,QACtB,iBAAiB,IAAI;AAAA,QACrB,UAAU,IAAI;AAAA,QACd,gBAAgB,IAAI;AAAA,QACpB,QAAQ;AAAA,MACV,CAAC;AAED,YAAM,GAAG,QAAQ,GAAG,EAAE,MAAM;AAE5B,YAAM,SAAS,KAAK,gBAAgB,aAAa;AAAA,QAC/C,GAAG,gBAAgB,GAAG;AAAA,QACtB,UAAU,IAAI;AAAA,QACd,gBAAgB,IAAI;AAAA,MACtB,CAAC;AAED,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,SAAS,OAAO,KAAK;AACzB,YAAM,MAAM,MAAM,GAAG,cAAc,aAAa,eAAe,OAAO,GAAG,GAAG,EAAE,oBAAoB,KAAK,CAAC;AACxG,UAAI,IAAI,WAAW,aAAa,IAAI,WAAW,eAAe,IAAI,WAAW,aAAa;AACxF,eAAO;AAAA,MACT;AAEA,YAAM,MAAM,oBAAI,KAAK;AACrB,YAAM,WAAW,MAAM,GAAG,aAAa,aAAa;AAAA,QAClD,GAAG,eAAe,OAAO,GAAG;AAAA,QAC5B,QAAQ,EAAE,KAAK,oBAA2C;AAAA,MAC5D,GAA+B;AAAA,QAC7B,QAAQ;AAAA,QACR,WAAW;AAAA,QACX,aAAa;AAAA,QACb,YAAY;AAAA,QACZ,cAAc;AAAA,QACd,YAAY;AAAA,QACZ,WAAW;AAAA,MACb,CAAC;AAED,UAAI,aAAa,GAAG;AAClB,cAAM,QAAQ,MAAM,aAAa,OAAO,GAAG;AAC3C,eAAO,SAAS;AAAA,MAClB;AAEA,UAAI,SAAS;AACb,UAAI,YAAY;AAChB,UAAI,cAAc;AAClB,UAAI,aAAa;AACjB,UAAI,eAAe;AACnB,UAAI,aAAa;AAEjB,YAAM,SAAS,KAAK,gBAAgB,aAAa;AAAA,QAC/C,GAAG,gBAAgB,GAAG;AAAA,QACtB,UAAU,IAAI;AAAA,QACd,gBAAgB,IAAI,kBAAkB;AAAA,MACxC,CAAC;AAED,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,eAAe,OAAO,OAAO,KAAK;AACtC,YAAM,QAAQ,MAAM,oBAAoB,OAAO,GAAG;AAClD,YAAM,MAAM,MAAM;AAClB,UAAI,kBAAkB,SAAS,IAAI,MAAM,GAAG;AAC1C,0BAAkB,KAAK;AACvB,eAAO;AAAA,MACT;AAEA,UAAI,MAAM,mBAAmB,QAAW;AACtC,YAAI,iBAAiB,MAAM;AAC3B,cAAM,wBAAwB;AAC9B,cAAM,eAAe;AAAA,MACvB;AACA,UAAI,MAAM,eAAe,QAAW;AAClC,YAAI,aAAa,MAAM;AAAA,MACzB;AACA,UAAI,MAAM,SAAS,QAAW;AAC5B,YAAI,OAAO,EAAE,GAAG,IAAI,MAAM,GAAG,MAAM,KAAK;AAAA,MAC1C;AAEA,UAAI,MAAM,oBAAoB,QAAW;AACvC,YAAI,kBAAkB,MAAM;AAAA,MAC9B,WAAW,IAAI,YAAY;AACzB,YAAI,kBAAkB,yBAAyB,IAAI,gBAAgB,IAAI,UAAU;AAAA,MACnF;AAEA,UAAI,MAAM,eAAe,QAAW;AAClC,YAAI,aAAa,MAAM;AAAA,MACzB,WAAW,IAAI,aAAa,IAAI,YAAY;AAC1C,YAAI,aAAa,aAAa,IAAI,gBAAgB,IAAI,YAAY,IAAI,SAAS;AAAA,MACjF;AAEA,UAAI,cAAc,oBAAI,KAAK;AAE3B,aAAO,yBAAyB,OAAO,GAAG;AAAA,IAC5C;AAAA,IAEA,MAAM,kBAAkB,OAAO,OAAO,KAAK;AACzC,YAAM,QAAQ,MAAM,oBAAoB,OAAO,GAAG;AAClD,YAAM,MAAM,MAAM;AAClB,UAAI,kBAAkB,SAAS,IAAI,MAAM,GAAG;AAC1C,0BAAkB,KAAK;AACvB,eAAO;AAAA,MACT;AAEA,UAAI,kBAAkB;AACtB,YAAM,gBAAgB;AACtB,UAAI,cAAc,oBAAI,KAAK;AAE3B,UAAI,IAAI,YAAY;AAClB,YAAI,kBAAkB,yBAAyB,IAAI,gBAAgB,IAAI,UAAU;AACjF,YAAI,IAAI,WAAW;AACjB,cAAI,aAAa,aAAa,IAAI,gBAAgB,IAAI,YAAY,IAAI,SAAS;AAAA,QACjF;AAAA,MACF;AAEA,aAAO,yBAAyB,OAAO,GAAG;AAAA,IAC5C;AAAA,IAEA,MAAM,YAAY,OAAO,OAAO,KAAK;AACnC,YAAM,MAAM,MAAM,aAAa,OAAO,GAAG;AACzC,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,OAAO,KAAK,YAAY;AAClD,UAAI,IAAI,WAAW,eAAe,IAAI,WAAW,aAAa;AAC5D,0BAAkB,KAAK;AACvB,eAAO;AAAA,MACT;AAEA,YAAM,QAAQ,kBAAkB,IAAI,KAAK;AACzC,YAAM,WAAW,OAAO,OAAO;AAC/B,YAAM,MAAM,oBAAI,KAAK;AACrB,YAAM,OAAgC;AAAA,QACpC,QAAQ;AAAA,QACR,YAAY;AAAA,QACZ,YAAY;AAAA,QACZ,WAAW;AAAA,QACX,GAAI,OAAO,gBAAgB,EAAE,eAAe,MAAM,cAAc,IAAI,CAAC;AAAA,QACrE,GAAI,QACA;AAAA,UACE,YAAY,SAAS,cAAc;AAAA,UACnC,MAAM,SAAS,QAAQ;AAAA,UACvB,GAAG,uBAAuB,KAAK;AAAA,QACjC,IACA,CAAC;AAAA,QACL,iBAAiB;AAAA,MACnB;AAEA,YAAM,WAAW,MAAM,GAAG,aAAa,aAAa;AAAA,QAClD,GAAG,eAAe,OAAO,GAAG;AAAA,QAC5B,QAAQ,EAAE,KAAK,uBAA8C;AAAA,MAC/D,GAA+B,IAAI;AACnC,wBAAkB,KAAK;AAEvB,UAAI,aAAa,GAAG;AAClB,cAAM,QAAQ,MAAM,aAAa,OAAO,GAAG;AAC3C,eAAO,SAAS;AAAA,MAClB;AAEA,eAAS,SAAS;AAClB,eAAS,aAAa;AACtB,eAAS,kBAAkB;AAC3B,eAAS,aAAa;AACtB,UAAI,OAAO,eAAe;AACxB,iBAAS,gBAAgB,MAAM;AAAA,MACjC;AAEA,YAAM,eAAgB,MAAM,aAAa,OAAO,GAAG,KAAM;AAEzD,YAAM,SAAS,KAAK,gBAAgB,eAAe;AAAA,QACjD,GAAG,gBAAgB,YAAY;AAAA,QAC/B,eAAe,aAAa;AAAA,QAC5B,UAAU,IAAI;AAAA,QACd,gBAAgB,aAAa,kBAAkB;AAAA,MACjD,CAAC;AAED,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,QAAQ,OAAO,OAAO,KAAK;AAC/B,YAAM,MAAM,MAAM,aAAa,OAAO,GAAG;AACzC,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,OAAO,KAAK,YAAY;AAClD,UAAI,kBAAkB,SAAS,IAAI,MAAM,GAAG;AAC1C,0BAAkB,KAAK;AACvB,eAAO;AAAA,MACT;AAEA,YAAM,QAAQ,kBAAkB,IAAI,KAAK;AACzC,YAAM,WAAW,OAAO,OAAO;AAC/B,YAAM,MAAM,oBAAI,KAAK;AACrB,YAAM,OAAgC;AAAA,QACpC,QAAQ;AAAA,QACR,YAAY;AAAA,QACZ,cAAc,MAAM;AAAA,QACpB,YAAY,MAAM;AAAA,QAClB,GAAI,MAAM,gBAAgB,EAAE,eAAe,MAAM,cAAc,IAAI,CAAC;AAAA,QACpE,WAAW;AAAA,QACX,GAAI,QACA;AAAA,UACE,YAAY,SAAS,cAAc;AAAA,UACnC,MAAM,SAAS,QAAQ;AAAA,UACvB,GAAG,uBAAuB,KAAK;AAAA,QACjC,IACA,CAAC;AAAA,MACP;AAEA,YAAM,WAAW,MAAM,GAAG,aAAa,aAAa;AAAA,QAClD,GAAG,eAAe,OAAO,GAAG;AAAA,QAC5B,QAAQ,EAAE,KAAK,mBAA0C;AAAA,MAC3D,GAA+B,IAAI;AACnC,wBAAkB,KAAK;AAEvB,UAAI,aAAa,GAAG;AAClB,cAAM,QAAQ,MAAM,aAAa,OAAO,GAAG;AAC3C,eAAO,SAAS;AAAA,MAClB;AAEA,eAAS,SAAS;AAClB,eAAS,aAAa;AACtB,eAAS,eAAe,MAAM;AAC9B,eAAS,aAAa,MAAM;AAC5B,UAAI,MAAM,eAAe;AACvB,iBAAS,gBAAgB,MAAM;AAAA,MACjC;AAEA,YAAM,eAAgB,MAAM,aAAa,OAAO,GAAG,KAAM;AAEzD,YAAM,SAAS,KAAK,gBAAgB,YAAY;AAAA,QAC9C,GAAG,gBAAgB,YAAY;AAAA,QAC/B,cAAc,aAAa;AAAA,QAC3B,UAAU,IAAI;AAAA,QACd,gBAAgB,aAAa,kBAAkB;AAAA,MACjD,CAAC;AAED,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,UAAU,OAAO,KAAK;AAC1B,YAAM,MAAM,MAAM,GAAG,cAAc,aAAa,eAAe,OAAO,GAAG,GAAG,EAAE,oBAAoB,KAAK,CAAC;AACxG,UAAI,kBAAkB,SAAS,IAAI,MAAM,GAAG;AAE1C,eAAO;AAAA,MACT;AACA,UAAI,CAAC,IAAI,aAAa;AACpB,cAAM,IAAI,MAAM,OAAO,KAAK,qBAAqB;AAAA,MACnD;AAEA,YAAM,MAAM,oBAAI,KAAK;AACrB,YAAM,eAAe,MAAM,GAAG,aAAa,aAAa;AAAA,QACtD,GAAG,eAAe,OAAO,GAAG;AAAA,QAC5B,aAAa;AAAA,QACb,QAAQ;AAAA,MACV,GAA+B;AAAA,QAC7B,QAAQ;AAAA,QACR,mBAAmB;AAAA,QACnB,mBAAmB,IAAI,UAAU;AAAA,QACjC,YAAY;AAAA,QACZ,YAAY;AAAA,QACZ,WAAW;AAAA,MACb,CAAC;AAED,UAAI,YAAY;AAChB,UAAI,iBAAiB,GAAG;AACtB,oBAAY,MAAM,GAAG,aAAa,aAAa;AAAA,UAC7C,GAAG,eAAe,OAAO,GAAG;AAAA,UAC5B,aAAa;AAAA,UACb,QAAQ;AAAA,QACV,GAA+B;AAAA,UAC7B,mBAAmB;AAAA,UACnB,mBAAmB,IAAI,UAAU;AAAA,UACjC,WAAW;AAAA,QACb,CAAC;AAAA,MACH;AAEA,UAAI,iBAAiB,KAAK,cAAc,GAAG;AAEzC,cAAM,QAAQ,MAAM,aAAa,OAAO,GAAG;AAC3C,eAAO,SAAS;AAAA,MAClB;AAEA,wBAAkB,KAAK;AACvB,UAAI,eAAe,GAAG;AACpB,YAAI,SAAS;AACb,YAAI,aAAa;AACjB,YAAI,aAAa;AAAA,MACnB;AACA,UAAI,oBAAoB;AACxB,UAAI,oBAAoB,IAAI,UAAU;AAEtC,YAAM,SAAS,KAAK,gBAAgB,eAAe;AAAA,QACjD,GAAG,gBAAgB,GAAG;AAAA,QACtB,UAAU,IAAI;AAAA,QACd,gBAAgB,IAAI,kBAAkB;AAAA,MACxC,CAAC;AAED,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,cAAc,OAAO,KAAK;AAC9B,YAAM,MAAM,MAAM,aAAa,OAAO,GAAG;AACzC,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,OAAO,KAAK,YAAY;AAClD,UAAI,IAAI,WAAW,eAAe,IAAI,WAAW,aAAa;AAC5D,0BAAkB,KAAK;AACvB,eAAO;AAAA,MACT;AAEA,YAAM,MAAM,oBAAI,KAAK;AACrB,YAAM,oBAAoB,IAAI,qBAAqB;AACnD,YAAM,oBAAoB,IAAI,qBAAqB,IAAI,UAAU;AACjE,YAAM,aAAa,IAAI,cAAc;AAErC,YAAM,WAAW,MAAM,GAAG,aAAa,aAAa;AAAA,QAClD,GAAG,eAAe,OAAO,GAAG;AAAA,QAC5B,QAAQ,EAAE,KAAK,qBAA4C;AAAA,MAC7D,GAA+B;AAAA,QAC7B,QAAQ;AAAA,QACR;AAAA,QACA;AAAA,QACA;AAAA,QACA,YAAY;AAAA,QACZ,WAAW;AAAA,MACb,CAAC;AACD,wBAAkB,KAAK;AAEvB,UAAI,aAAa,GAAG;AAClB,cAAM,QAAQ,MAAM,aAAa,OAAO,GAAG;AAC3C,eAAO,SAAS;AAAA,MAClB;AAEA,UAAI,SAAS;AACb,UAAI,oBAAoB;AACxB,UAAI,oBAAoB;AACxB,UAAI,aAAa;AACjB,UAAI,aAAa;AAEjB,YAAM,SAAS,KAAK,gBAAgB,eAAe;AAAA,QACjD,GAAG,gBAAgB,GAAG;AAAA,QACtB,UAAU,IAAI;AAAA,QACd,gBAAgB,IAAI,kBAAkB;AAAA,MACxC,CAAC;AAED,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,wBAAwB,OAAO,UAAU,gBAAgB;AAI7D,YAAM,MAAM,MAAM,sBAAsB,IAAI,aAAa;AAAA,QACvD,IAAI;AAAA,QACJ;AAAA,QACA,GAAI,iBAAiB,EAAE,eAAe,IAAI,CAAC;AAAA,MAC7C,GAAG,EAAE,oBAAoB,KAAK,CAAC;AAC/B,aAAO,KAAK,qBAAqB;AAAA,IACnC;AAAA,IAEA,MAAM,cAAc,KAAK;AACvB,aAAO,GAAG,KAAK,aAAa;AAAA,QAC1B,UAAU,IAAI;AAAA,QACd,GAAI,IAAI,iBAAiB,EAAE,gBAAgB,IAAI,eAAe,IAAI,CAAC;AAAA,QACnE,QAAQ,EAAE,KAAK,CAAC,WAAW,SAAS,EAAE;AAAA,QACtC,aAAa;AAAA,MACf,GAAG;AAAA,QACD,SAAS,EAAE,WAAW,OAAO;AAAA,QAC7B,OAAO;AAAA,MACT,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,yBAAyB,KAAK,eAAe,IAAI;AACrD,YAAM,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI,eAAe,GAAI;AACxD,aAAO,GAAG,KAAK,aAAa;AAAA,QAC1B,UAAU,IAAI;AAAA,QACd,GAAI,IAAI,iBAAiB,EAAE,gBAAgB,IAAI,eAAe,IAAI,CAAC;AAAA,QACnE,QAAQ,EAAE,KAAK,CAAC,aAAa,QAAQ,EAAE;AAAA,QACvC,YAAY,EAAE,MAAM,OAAO;AAAA,QAC3B,aAAa;AAAA,MACf,GAAG;AAAA,QACD,SAAS,EAAE,YAAY,OAAO;AAAA,QAC9B,OAAO;AAAA,MACT,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,OAAO,OAAO,KAAK;AACvB,aAAO,GAAG,QAAQ,aAAa;AAAA,QAC7B,IAAI;AAAA,QACJ,UAAU,IAAI;AAAA,QACd,GAAI,IAAI,iBAAiB,EAAE,gBAAgB,IAAI,eAAe,IAAI,CAAC;AAAA,MACrE,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,oBAAoB,UAAkB,iBAAiB,2BAA2B,gBAAgC;AACtH,YAAM,MAAM,oBAAI,KAAK;AACrB,YAAM,SAAS,IAAI,KAAK,IAAI,QAAQ,IAAI,iBAAiB,GAAI;AAC7D,YAAM,QAAQ;AAAA,QACZ;AAAA,QACA,GAAI,iBAAiB,EAAE,eAAe,IAAI,CAAC;AAAA,MAC7C;AACA,UAAI,cAAc;AAElB,YAAM,cAAc;AAAA,QAClB,QAAQ;AAAA,QACR,KAAK;AAAA,UACH,EAAE,aAAa,EAAE,KAAK,OAAO,EAAE;AAAA,UAC/B;AAAA,YACE,aAAa;AAAA,YACb,WAAW,EAAE,KAAK,OAAO;AAAA,UAC3B;AAAA,QACF;AAAA,MACF;AACA,YAAM,eAAe,MAAM,GAAG,KAAK,aAAa,EAAE,GAAG,OAAO,GAAG,YAAY,GAAG,EAAE,oBAAoB,KAAK,CAAC;AAE1G,iBAAW,OAAO,cAAc;AAC9B,cAAM,eAAe,+BAA+B,cAAc;AAGlE,cAAM,WAAW,MAAM,GAAG,aAAa,aAAa;AAAA,UAClD,IAAI,IAAI;AAAA,UACR,UAAU,IAAI;AAAA,UACd,GAAG;AAAA,QACL,GAA+B;AAAA,UAC7B,QAAQ;AAAA,UACR,YAAY;AAAA,UACZ;AAAA,UACA,WAAW;AAAA,QACb,CAAC;AACD,YAAI,aAAa,EAAG;AAEpB,uBAAe;AACf,YAAI,SAAS;AACb,YAAI,aAAa;AACjB,YAAI,eAAe;AAEnB,cAAM,SAAS,KAAK,gBAAgB,YAAY;AAAA,UAC9C,GAAG,gBAAgB,GAAG;AAAA,UACtB,cAAc,IAAI;AAAA,UAClB,UAAU,IAAI;AAAA,UACd,OAAO;AAAA,UACP,gBAAgB,IAAI,kBAAkB;AAAA,QACxC,CAAC;AAAA,MACH;AAKA,YAAM,gBAAgB,IAAI,KAAK,IAAI,QAAQ,IAAI,gCAAgC,GAAI;AACnF,YAAM,qBAAqB;AAAA,QACzB,QAAQ;AAAA,QACR,WAAW,EAAE,KAAK,cAAc;AAAA,MAClC;AACA,YAAM,eAAe,MAAM,GAAG,KAAK,aAAa,EAAE,GAAG,OAAO,GAAG,mBAAmB,GAAG,EAAE,oBAAoB,KAAK,CAAC;AAEjH,iBAAW,OAAO,cAAc;AAC9B,cAAM,eAAe,mCAAmC,6BAA6B;AACrF,cAAM,WAAW,MAAM,GAAG,aAAa,aAAa;AAAA,UAClD,IAAI,IAAI;AAAA,UACR,UAAU,IAAI;AAAA,UACd,GAAG;AAAA,QACL,GAA+B;AAAA,UAC7B,QAAQ;AAAA,UACR,YAAY;AAAA,UACZ;AAAA,UACA,WAAW;AAAA,QACb,CAAC;AACD,YAAI,aAAa,EAAG;AAEpB,uBAAe;AACf,YAAI,SAAS;AACb,YAAI,aAAa;AACjB,YAAI,eAAe;AAEnB,cAAM,SAAS,KAAK,gBAAgB,YAAY;AAAA,UAC9C,GAAG,gBAAgB,GAAG;AAAA,UACtB,cAAc,IAAI;AAAA,UAClB,UAAU,IAAI;AAAA,UACd,OAAO;AAAA,UACP,gBAAgB,IAAI,kBAAkB;AAAA,QACxC,CAAC;AAAA,MACH;AAEA,aAAO;AAAA,IACT;AAAA,EACF;AACF;",
4
+ "sourcesContent": ["import type { EntityManager } from '@mikro-orm/postgresql'\nimport { raw } from '@mikro-orm/core'\nimport type { EntityData, FilterQuery } from '@mikro-orm/core'\nimport { ProgressJob, type ProgressJobStatus } from '../data/entities'\nimport type { ProgressService, ProgressServiceContext } from './progressService'\nimport {\n calculateEta,\n calculateProgressPercent,\n HEARTBEAT_INTERVAL_MS,\n STALE_JOB_TIMEOUT_SECONDS,\n STALE_PENDING_TIMEOUT_SECONDS,\n STALE_SWEEP_ERROR_PREFIX,\n} from './progressService'\nimport { PROGRESS_EVENTS } from './events'\nimport { findOneWithDecryption } from '@open-mercato/shared/lib/encryption/find'\n\nconst DEFAULT_BROADCAST_MIN_INTERVAL_MS = 250\n\n// Minimum elapsed time between coalesced `progress.job.updated` broadcasts for a\n// single job. Bulk workers call updateProgress/incrementProgress once per record, and\n// every emit of the `clientBroadcast: true` event pays a serialized pg_notify roundtrip\n// plus a tenant-wide SSE fan-out. Setting the knob to 0 restores per-record emission.\n// Persistence is throttled separately: heartbeats hit the database at least every\n// HEARTBEAT_INTERVAL_MS regardless of this knob, so a high broadcast interval can\n// never starve the stale-job sweep of heartbeats.\nfunction resolveBroadcastMinIntervalMs(): number {\n const rawValue = process.env.OM_PROGRESS_BROADCAST_MIN_INTERVAL_MS\n if (rawValue == null || rawValue.trim() === '') return DEFAULT_BROADCAST_MIN_INTERVAL_MS\n const parsed = Number.parseInt(rawValue, 10)\n if (!Number.isFinite(parsed) || parsed < 0) return DEFAULT_BROADCAST_MIN_INTERVAL_MS\n return parsed\n}\n\n// Statuses a given transition is allowed to move FROM. Every write below is a\n// compare-and-swap (`nativeUpdate` guarded on status), so concurrent writers in other\n// processes can never resurrect a terminal job or double-apply a transition \u2014 the\n// UPDATE that matches zero rows simply lost the race and must not emit events.\nconst TERMINAL_STATUSES: readonly ProgressJobStatus[] = ['completed', 'failed', 'cancelled']\nconst PROGRESS_WRITABLE_STATUSES: readonly ProgressJobStatus[] = ['pending', 'running']\n// `failed` is restartable/completable so a job wrongly swept as stale (worker alive but\n// slow) or retried by an at-least-once queue converges to its true outcome.\nconst START_FROM_STATUSES: readonly ProgressJobStatus[] = ['pending', 'failed']\nconst COMPLETE_FROM_STATUSES: readonly ProgressJobStatus[] = ['pending', 'running', 'failed']\nconst FAIL_FROM_STATUSES: readonly ProgressJobStatus[] = ['pending', 'running']\nconst CANCEL_FROM_STATUSES: readonly ProgressJobStatus[] = ['pending', 'running', 'failed']\n\ntype JobUpdateThrottleEntry = {\n job: ProgressJob\n lastBroadcastAt: number\n lastPersistedAt: number\n lastBroadcastPercent: number\n pendingDelta: number\n absoluteCountsPending: boolean\n}\n\nfunction buildJobPayload(job: ProgressJob): Record<string, unknown> {\n return {\n jobId: job.id,\n jobType: job.jobType,\n name: job.name,\n description: job.description ?? null,\n status: job.status,\n progressPercent: job.progressPercent,\n processedCount: job.processedCount,\n totalCount: job.totalCount ?? null,\n etaSeconds: job.etaSeconds ?? null,\n cancellable: job.cancellable,\n meta: job.meta ?? null,\n startedAt: job.startedAt?.toISOString() ?? null,\n finishedAt: job.finishedAt?.toISOString() ?? null,\n }\n}\n\nexport function createProgressService(em: EntityManager, eventBus: { emit: (event: string, payload: Record<string, unknown>) => Promise<void> }): ProgressService {\n const broadcastMinIntervalMs = resolveBroadcastMinIntervalMs()\n // Per-job coalescing state, scoped to this service instance (request/worker scope).\n // The cached entity is a DETACHED snapshot (loaded with disableIdentityMap) that doubles\n // as the in-memory buffer: intermediate updates mutate it without touching the shared\n // identity map or UnitOfWork, so an unrelated em.flush() elsewhere can never write the\n // buffered values. Persistence happens exclusively through the CAS nativeUpdate below.\n const jobUpdateThrottle = new Map<string, JobUpdateThrottleEntry>()\n\n function jobScopeFilter(jobId: string, ctx: ProgressServiceContext) {\n return {\n id: jobId,\n tenantId: ctx.tenantId,\n ...(ctx.organizationId ? { organizationId: ctx.organizationId } : {}),\n }\n }\n\n async function loadFreshJob(jobId: string, ctx: ProgressServiceContext): Promise<ProgressJob | null> {\n return em.findOne(ProgressJob, jobScopeFilter(jobId, ctx), { disableIdentityMap: true })\n }\n\n async function ensureThrottleEntry(jobId: string, ctx: ProgressServiceContext): Promise<JobUpdateThrottleEntry> {\n const cached = jobUpdateThrottle.get(jobId)\n if (cached) return cached\n const job = await em.findOneOrFail(ProgressJob, jobScopeFilter(jobId, ctx), { disableIdentityMap: true })\n const entry: JobUpdateThrottleEntry = {\n job,\n lastBroadcastAt: 0,\n lastPersistedAt: 0,\n lastBroadcastPercent: job.progressPercent,\n pendingDelta: 0,\n absoluteCountsPending: false,\n }\n jobUpdateThrottle.set(jobId, entry)\n return entry\n }\n\n function forgetJobThrottle(jobId: string) {\n jobUpdateThrottle.delete(jobId)\n }\n\n function buildBufferedCountData(entry: JobUpdateThrottleEntry): EntityData<ProgressJob> {\n if (entry.absoluteCountsPending || !Number.isSafeInteger(entry.pendingDelta)) {\n return { processedCount: entry.job.processedCount }\n }\n if (entry.pendingDelta !== 0) {\n const processedCount = raw(`processed_count + ${entry.pendingDelta}`)\n const totalCount = entry.job.totalCount\n if (Number.isSafeInteger(totalCount) && totalCount != null && totalCount > 0) {\n return {\n processedCount,\n progressPercent: raw(\n `least(100, round(((processed_count + ${entry.pendingDelta})::numeric / ${totalCount}) * 100))`,\n ),\n }\n }\n return { processedCount }\n }\n return {}\n }\n\n // The start transition (START_FROM_STATUSES) doubles as the recovery path for jobs a\n // stale sweep flipped to `failed` while their producer was alive but slow. Callers pass\n // the EM to run against so the forked heartbeat path can reuse the same CAS.\n //\n // `staleSweptOnly` is for the revive paths: a live write proves the producer is alive,\n // but it does NOT prove the recorded failure was the sweep's. Narrowing the CAS to rows\n // the sweep tagged keeps a genuine failure's message and stack intact instead of\n // resurrecting the job and erasing why it died. `startJob` stays unrestricted so an\n // at-least-once queue can still retry a genuinely failed job from the top.\n async function startJobViaCas(\n targetEm: EntityManager,\n job: ProgressJob,\n ctx: ProgressServiceContext,\n options: { staleSweptOnly?: boolean } = {},\n ): Promise<ProgressJob | null> {\n const now = new Date()\n // Progress counts are absolute and survive across deliveries, so the elapsed window\n // calculateEta divides them by has to survive too. Resetting it on every revive makes\n // a job with hours of work left report seconds remaining.\n const startedAt = job.startedAt ?? now\n const filter: Record<string, unknown> = {\n ...jobScopeFilter(job.id, ctx),\n status: { $in: START_FROM_STATUSES as ProgressJobStatus[] },\n }\n if (options.staleSweptOnly) filter.errorMessage = { $like: `${STALE_SWEEP_ERROR_PREFIX}%` }\n const affected = await targetEm.nativeUpdate(ProgressJob, filter as FilterQuery<ProgressJob>, {\n status: 'running',\n startedAt,\n heartbeatAt: now,\n finishedAt: null,\n errorMessage: null,\n errorStack: null,\n updatedAt: now,\n })\n if (affected === 0) return null\n\n job.status = 'running'\n job.startedAt = startedAt\n job.heartbeatAt = now\n job.finishedAt = null\n job.errorMessage = null\n job.errorStack = null\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_STARTED, {\n ...buildJobPayload(job),\n tenantId: ctx.tenantId,\n organizationId: job.organizationId ?? null,\n })\n\n return job\n }\n\n async function persistAndMaybeBroadcast(entry: JobUpdateThrottleEntry, ctx: ProgressServiceContext): Promise<ProgressJob> {\n const job = entry.job\n const now = Date.now()\n const shouldBroadcast =\n broadcastMinIntervalMs <= 0 ||\n now - entry.lastBroadcastAt >= broadcastMinIntervalMs ||\n Math.abs(job.progressPercent - entry.lastBroadcastPercent) >= 1\n const shouldPersist = shouldBroadcast || now - entry.lastPersistedAt >= HEARTBEAT_INTERVAL_MS\n\n if (!shouldPersist) return job\n\n const usesAtomicIncrement =\n !entry.absoluteCountsPending && Number.isSafeInteger(entry.pendingDelta) && entry.pendingDelta !== 0\n const data: EntityData<ProgressJob> = {\n progressPercent: job.progressPercent,\n totalCount: job.totalCount ?? null,\n etaSeconds: job.etaSeconds ?? null,\n meta: job.meta ?? null,\n heartbeatAt: job.heartbeatAt ?? new Date(),\n updatedAt: new Date(),\n ...buildBufferedCountData(entry),\n }\n const writableFilter = {\n ...jobScopeFilter(job.id, ctx),\n status: { $in: PROGRESS_WRITABLE_STATUSES as ProgressJobStatus[] },\n } as FilterQuery<ProgressJob>\n let affected = await em.nativeUpdate(ProgressJob, writableFilter, data)\n\n if (affected === 0) {\n // A stale sweep may have flipped a live job to `failed` between writes. Revive it\n // through the start CAS and retry once so the buffered delta isn't silently dropped.\n const fresh = await loadFreshJob(job.id, ctx)\n let revived = false\n if (fresh && fresh.status === 'failed' && (await startJobViaCas(em, fresh, ctx, { staleSweptOnly: true }))) {\n revived = true\n job.status = 'running'\n job.startedAt = fresh.startedAt\n job.finishedAt = null\n job.errorMessage = null\n job.errorStack = null\n affected = await em.nativeUpdate(ProgressJob, writableFilter, data)\n }\n if (affected === 0) {\n // The job reached a genuinely terminal state in another process; stop writing to it.\n forgetJobThrottle(job.id)\n const latest = revived ? await loadFreshJob(job.id, ctx) : fresh\n return latest ?? job\n }\n }\n\n const persistedJob = usesAtomicIncrement ? (await loadFreshJob(job.id, ctx)) ?? job : job\n entry.job = persistedJob\n entry.pendingDelta = 0\n entry.absoluteCountsPending = false\n entry.lastPersistedAt = now\n\n if (shouldBroadcast) {\n await eventBus.emit(PROGRESS_EVENTS.JOB_UPDATED, {\n ...buildJobPayload(persistedJob),\n tenantId: ctx.tenantId,\n organizationId: persistedJob.organizationId ?? null,\n })\n entry.lastBroadcastAt = now\n entry.lastBroadcastPercent = persistedJob.progressPercent\n }\n\n return persistedJob\n }\n\n return {\n async createJob(input, ctx) {\n const job = em.create(ProgressJob, {\n jobType: input.jobType,\n name: input.name,\n description: input.description,\n totalCount: input.totalCount,\n cancellable: input.cancellable ?? false,\n meta: input.meta,\n parentJobId: input.parentJobId,\n partitionIndex: input.partitionIndex,\n partitionCount: input.partitionCount,\n startedByUserId: ctx.userId,\n tenantId: ctx.tenantId,\n organizationId: ctx.organizationId,\n status: 'pending',\n })\n\n await em.persist(job).flush()\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_CREATED, {\n ...buildJobPayload(job),\n tenantId: ctx.tenantId,\n organizationId: ctx.organizationId,\n })\n\n return job\n },\n\n async startJob(jobId, ctx) {\n const job = await em.findOneOrFail(ProgressJob, jobScopeFilter(jobId, ctx), { disableIdentityMap: true })\n if (job.status === 'running' || job.status === 'cancelled' || job.status === 'completed') {\n return job\n }\n\n const started = await startJobViaCas(em, job, ctx)\n if (started) return started\n\n const fresh = await loadFreshJob(jobId, ctx)\n return fresh ?? job\n },\n\n async touchJobHeartbeat(jobId, ctx) {\n // Forked EM: keepalive timers call this while the shared EM may be mid-transaction\n // inside a producer's own writes, so the heartbeat must not join that unit of work.\n const fork = em.fork()\n const now = new Date()\n const affected = await fork.nativeUpdate(ProgressJob, {\n ...jobScopeFilter(jobId, ctx),\n status: { $in: PROGRESS_WRITABLE_STATUSES as ProgressJobStatus[] },\n } as FilterQuery<ProgressJob>, {\n heartbeatAt: now,\n updatedAt: now,\n })\n if (affected > 0) return\n\n const job = await fork.findOne(ProgressJob, jobScopeFilter(jobId, ctx), { disableIdentityMap: true })\n if (!job || job.status !== 'failed') return\n // A live producer heartbeating a `failed` job means a stale sweep false-positived;\n // revive through the start CAS so the card recovers without waiting for a batch boundary.\n await startJobViaCas(fork, job, ctx, { staleSweptOnly: true })\n },\n\n async updateProgress(jobId, input, ctx) {\n const entry = await ensureThrottleEntry(jobId, ctx)\n const job = entry.job\n if (TERMINAL_STATUSES.includes(job.status)) {\n // `failed` may be a false-positive stale sweep on a slow-but-alive producer; the\n // fact that this write is happening proves the producer is alive, so revive and\n // apply the update. `completed`/`cancelled` stay terminal, and a failure the sweep\n // did not write keeps its diagnostics (see startJobViaCas).\n const revived = job.status === 'failed' && (await startJobViaCas(em, job, ctx, { staleSweptOnly: true }))\n if (!revived) {\n forgetJobThrottle(jobId)\n return job\n }\n entry.lastPersistedAt = 0\n }\n\n if (input.processedCount !== undefined) {\n job.processedCount = input.processedCount\n entry.absoluteCountsPending = true\n entry.pendingDelta = 0\n }\n if (input.totalCount !== undefined) {\n job.totalCount = input.totalCount\n }\n if (input.meta !== undefined) {\n job.meta = { ...job.meta, ...input.meta }\n }\n\n if (input.progressPercent !== undefined) {\n job.progressPercent = input.progressPercent\n } else if (job.totalCount) {\n job.progressPercent = calculateProgressPercent(job.processedCount, job.totalCount)\n }\n\n if (input.etaSeconds !== undefined) {\n job.etaSeconds = input.etaSeconds\n } else if (job.startedAt && job.totalCount) {\n job.etaSeconds = calculateEta(job.processedCount, job.totalCount, job.startedAt)\n }\n\n job.heartbeatAt = new Date()\n\n return persistAndMaybeBroadcast(entry, ctx)\n },\n\n async incrementProgress(jobId, delta, ctx) {\n const entry = await ensureThrottleEntry(jobId, ctx)\n const job = entry.job\n if (TERMINAL_STATUSES.includes(job.status)) {\n const revived = job.status === 'failed' && (await startJobViaCas(em, job, ctx, { staleSweptOnly: true }))\n if (!revived) {\n forgetJobThrottle(jobId)\n return job\n }\n entry.lastPersistedAt = 0\n }\n\n job.processedCount += delta\n entry.pendingDelta += delta\n job.heartbeatAt = new Date()\n\n if (job.totalCount) {\n job.progressPercent = calculateProgressPercent(job.processedCount, job.totalCount)\n if (job.startedAt) {\n job.etaSeconds = calculateEta(job.processedCount, job.totalCount, job.startedAt)\n }\n }\n\n return persistAndMaybeBroadcast(entry, ctx)\n },\n\n async completeJob(jobId, input, ctx) {\n const job = await loadFreshJob(jobId, ctx)\n if (!job) throw new Error(`Job ${jobId} not found`)\n if (job.status === 'cancelled' || job.status === 'completed') {\n forgetJobThrottle(jobId)\n return job\n }\n\n const entry = jobUpdateThrottle.get(jobId)\n const snapshot = entry?.job ?? job\n const now = new Date()\n const data: EntityData<ProgressJob> = {\n status: 'completed',\n finishedAt: now,\n etaSeconds: 0,\n updatedAt: now,\n ...(input?.resultSummary ? { resultSummary: input.resultSummary } : {}),\n ...(entry\n ? {\n totalCount: snapshot.totalCount ?? null,\n meta: snapshot.meta ?? null,\n ...buildBufferedCountData(entry),\n }\n : {}),\n progressPercent: 100,\n }\n\n const affected = await em.nativeUpdate(ProgressJob, {\n ...jobScopeFilter(jobId, ctx),\n status: { $in: COMPLETE_FROM_STATUSES as ProgressJobStatus[] },\n } as FilterQuery<ProgressJob>, data)\n forgetJobThrottle(jobId)\n\n if (affected === 0) {\n const fresh = await loadFreshJob(jobId, ctx)\n return fresh ?? job\n }\n\n snapshot.status = 'completed'\n snapshot.finishedAt = now\n snapshot.progressPercent = 100\n snapshot.etaSeconds = 0\n if (input?.resultSummary) {\n snapshot.resultSummary = input.resultSummary\n }\n\n const persistedJob = (await loadFreshJob(jobId, ctx)) ?? snapshot\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_COMPLETED, {\n ...buildJobPayload(persistedJob),\n resultSummary: persistedJob.resultSummary,\n tenantId: ctx.tenantId,\n organizationId: persistedJob.organizationId ?? null,\n })\n\n return persistedJob\n },\n\n async failJob(jobId, input, ctx) {\n const job = await loadFreshJob(jobId, ctx)\n if (!job) throw new Error(`Job ${jobId} not found`)\n if (TERMINAL_STATUSES.includes(job.status)) {\n forgetJobThrottle(jobId)\n return job\n }\n\n const entry = jobUpdateThrottle.get(jobId)\n const snapshot = entry?.job ?? job\n const now = new Date()\n const data: EntityData<ProgressJob> = {\n status: 'failed',\n finishedAt: now,\n errorMessage: input.errorMessage,\n errorStack: input.errorStack,\n ...(input.resultSummary ? { resultSummary: input.resultSummary } : {}),\n updatedAt: now,\n ...(entry\n ? {\n totalCount: snapshot.totalCount ?? null,\n meta: snapshot.meta ?? null,\n ...buildBufferedCountData(entry),\n }\n : {}),\n }\n\n const affected = await em.nativeUpdate(ProgressJob, {\n ...jobScopeFilter(jobId, ctx),\n status: { $in: FAIL_FROM_STATUSES as ProgressJobStatus[] },\n } as FilterQuery<ProgressJob>, data)\n forgetJobThrottle(jobId)\n\n if (affected === 0) {\n const fresh = await loadFreshJob(jobId, ctx)\n return fresh ?? job\n }\n\n snapshot.status = 'failed'\n snapshot.finishedAt = now\n snapshot.errorMessage = input.errorMessage\n snapshot.errorStack = input.errorStack\n if (input.resultSummary) {\n snapshot.resultSummary = input.resultSummary\n }\n\n const persistedJob = (await loadFreshJob(jobId, ctx)) ?? snapshot\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_FAILED, {\n ...buildJobPayload(persistedJob),\n errorMessage: persistedJob.errorMessage,\n tenantId: ctx.tenantId,\n organizationId: persistedJob.organizationId ?? null,\n })\n\n return persistedJob\n },\n\n async cancelJob(jobId, ctx) {\n const job = await em.findOneOrFail(ProgressJob, jobScopeFilter(jobId, ctx), { disableIdentityMap: true })\n if (TERMINAL_STATUSES.includes(job.status)) {\n // The job finished while the user clicked cancel \u2014 a benign race, not an error.\n return job\n }\n if (!job.cancellable) {\n throw new Error(`Job ${jobId} is not cancellable`)\n }\n\n const now = new Date()\n const cancelledNow = await em.nativeUpdate(ProgressJob, {\n ...jobScopeFilter(jobId, ctx),\n cancellable: true,\n status: 'pending',\n } as FilterQuery<ProgressJob>, {\n status: 'cancelled',\n cancelRequestedAt: now,\n cancelledByUserId: ctx.userId ?? null,\n finishedAt: now,\n etaSeconds: 0,\n updatedAt: now,\n })\n\n let requested = 0\n if (cancelledNow === 0) {\n requested = await em.nativeUpdate(ProgressJob, {\n ...jobScopeFilter(jobId, ctx),\n cancellable: true,\n status: 'running',\n } as FilterQuery<ProgressJob>, {\n cancelRequestedAt: now,\n cancelledByUserId: ctx.userId ?? null,\n updatedAt: now,\n })\n }\n\n if (cancelledNow === 0 && requested === 0) {\n // Raced into a terminal state between the read and the CAS.\n const fresh = await loadFreshJob(jobId, ctx)\n return fresh ?? job\n }\n\n forgetJobThrottle(jobId)\n if (cancelledNow > 0) {\n job.status = 'cancelled'\n job.finishedAt = now\n job.etaSeconds = 0\n }\n job.cancelRequestedAt = now\n job.cancelledByUserId = ctx.userId ?? null\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_CANCELLED, {\n ...buildJobPayload(job),\n tenantId: ctx.tenantId,\n organizationId: job.organizationId ?? null,\n })\n\n return job\n },\n\n async markCancelled(jobId, ctx) {\n const job = await loadFreshJob(jobId, ctx)\n if (!job) throw new Error(`Job ${jobId} not found`)\n if (job.status === 'cancelled' || job.status === 'completed') {\n forgetJobThrottle(jobId)\n return job\n }\n\n const now = new Date()\n const cancelRequestedAt = job.cancelRequestedAt ?? now\n const cancelledByUserId = job.cancelledByUserId ?? ctx.userId ?? null\n const finishedAt = job.finishedAt ?? now\n\n const affected = await em.nativeUpdate(ProgressJob, {\n ...jobScopeFilter(jobId, ctx),\n status: { $in: CANCEL_FROM_STATUSES as ProgressJobStatus[] },\n } as FilterQuery<ProgressJob>, {\n status: 'cancelled',\n cancelRequestedAt,\n cancelledByUserId,\n finishedAt,\n etaSeconds: 0,\n updatedAt: now,\n })\n forgetJobThrottle(jobId)\n\n if (affected === 0) {\n const fresh = await loadFreshJob(jobId, ctx)\n return fresh ?? job\n }\n\n job.status = 'cancelled'\n job.cancelRequestedAt = cancelRequestedAt\n job.cancelledByUserId = cancelledByUserId\n job.finishedAt = finishedAt\n job.etaSeconds = 0\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_CANCELLED, {\n ...buildJobPayload(job),\n tenantId: ctx.tenantId,\n organizationId: job.organizationId ?? null,\n })\n\n return job\n },\n\n async isCancellationRequested(jobId, tenantId, organizationId) {\n // disableIdentityMap forces a fresh read: a managed copy of the job in this\n // EntityManager (loaded by updateProgress) must never mask a cancellation\n // requested from another process.\n const job = await findOneWithDecryption(em, ProgressJob, {\n id: jobId,\n tenantId,\n ...(organizationId ? { organizationId } : {}),\n }, { disableIdentityMap: true })\n return job?.cancelRequestedAt != null\n },\n\n async getActiveJobs(ctx) {\n return em.find(ProgressJob, {\n tenantId: ctx.tenantId,\n ...(ctx.organizationId ? { organizationId: ctx.organizationId } : {}),\n status: { $in: ['pending', 'running'] },\n parentJobId: null,\n }, {\n orderBy: { createdAt: 'DESC' },\n limit: 50,\n })\n },\n\n async getRecentlyCompletedJobs(ctx, sinceSeconds = 30) {\n const cutoff = new Date(Date.now() - sinceSeconds * 1000)\n return em.find(ProgressJob, {\n tenantId: ctx.tenantId,\n ...(ctx.organizationId ? { organizationId: ctx.organizationId } : {}),\n status: { $in: ['completed', 'failed'] },\n finishedAt: { $gte: cutoff },\n parentJobId: null,\n }, {\n orderBy: { finishedAt: 'DESC' },\n limit: 10,\n })\n },\n\n async getJob(jobId, ctx) {\n return em.findOne(ProgressJob, {\n id: jobId,\n tenantId: ctx.tenantId,\n ...(ctx.organizationId ? { organizationId: ctx.organizationId } : {}),\n })\n },\n\n async markStaleJobsFailed(tenantId: string, timeoutSeconds = STALE_JOB_TIMEOUT_SECONDS, organizationId?: string | null) {\n const now = new Date()\n const cutoff = new Date(now.getTime() - timeoutSeconds * 1000)\n const scope = {\n tenantId,\n ...(organizationId ? { organizationId } : {}),\n }\n let failedCount = 0\n\n const staleFilter = {\n status: 'running' as ProgressJobStatus,\n $or: [\n { heartbeatAt: { $lt: cutoff } },\n {\n heartbeatAt: null,\n startedAt: { $lt: cutoff },\n },\n ],\n }\n const staleRunning = await em.find(ProgressJob, { ...scope, ...staleFilter }, { disableIdentityMap: true })\n\n for (const job of staleRunning) {\n const errorMessage = `${STALE_SWEEP_ERROR_PREFIX} no heartbeat for ${timeoutSeconds} seconds`\n // Per-row CAS (re-checking the staleness condition): exactly one concurrent\n // sweeper wins, and a heartbeat that landed after the SELECT aborts the failure.\n const affected = await em.nativeUpdate(ProgressJob, {\n id: job.id,\n tenantId: job.tenantId,\n ...staleFilter,\n } as FilterQuery<ProgressJob>, {\n status: 'failed',\n finishedAt: now,\n errorMessage,\n updatedAt: now,\n })\n if (affected === 0) continue\n\n failedCount += 1\n job.status = 'failed'\n job.finishedAt = now\n job.errorMessage = errorMessage\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_FAILED, {\n ...buildJobPayload(job),\n errorMessage: job.errorMessage,\n tenantId: job.tenantId,\n stale: true,\n organizationId: job.organizationId ?? null,\n })\n }\n\n // Jobs stuck in `pending` (enqueue failed, worker died before startJob) have no\n // heartbeat, so they need their own sweep or they stay in the active list forever.\n // A late queue delivery still recovers such a job: startJob transitions failed \u2192 running.\n const pendingCutoff = new Date(now.getTime() - STALE_PENDING_TIMEOUT_SECONDS * 1000)\n const stalePendingFilter = {\n status: 'pending' as ProgressJobStatus,\n createdAt: { $lt: pendingCutoff },\n }\n const stalePending = await em.find(ProgressJob, { ...scope, ...stalePendingFilter }, { disableIdentityMap: true })\n\n for (const job of stalePending) {\n const errorMessage = `${STALE_SWEEP_ERROR_PREFIX} never started within ${STALE_PENDING_TIMEOUT_SECONDS} seconds`\n const affected = await em.nativeUpdate(ProgressJob, {\n id: job.id,\n tenantId: job.tenantId,\n ...stalePendingFilter,\n } as FilterQuery<ProgressJob>, {\n status: 'failed',\n finishedAt: now,\n errorMessage,\n updatedAt: now,\n })\n if (affected === 0) continue\n\n failedCount += 1\n job.status = 'failed'\n job.finishedAt = now\n job.errorMessage = errorMessage\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_FAILED, {\n ...buildJobPayload(job),\n errorMessage: job.errorMessage,\n tenantId: job.tenantId,\n stale: true,\n organizationId: job.organizationId ?? null,\n })\n }\n\n return failedCount\n },\n }\n}\n"],
5
+ "mappings": "AACA,SAAS,WAAW;AAEpB,SAAS,mBAA2C;AAEpD;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,uBAAuB;AAChC,SAAS,6BAA6B;AAEtC,MAAM,oCAAoC;AAS1C,SAAS,gCAAwC;AAC/C,QAAM,WAAW,QAAQ,IAAI;AAC7B,MAAI,YAAY,QAAQ,SAAS,KAAK,MAAM,GAAI,QAAO;AACvD,QAAM,SAAS,OAAO,SAAS,UAAU,EAAE;AAC3C,MAAI,CAAC,OAAO,SAAS,MAAM,KAAK,SAAS,EAAG,QAAO;AACnD,SAAO;AACT;AAMA,MAAM,oBAAkD,CAAC,aAAa,UAAU,WAAW;AAC3F,MAAM,6BAA2D,CAAC,WAAW,SAAS;AAGtF,MAAM,sBAAoD,CAAC,WAAW,QAAQ;AAC9E,MAAM,yBAAuD,CAAC,WAAW,WAAW,QAAQ;AAC5F,MAAM,qBAAmD,CAAC,WAAW,SAAS;AAC9E,MAAM,uBAAqD,CAAC,WAAW,WAAW,QAAQ;AAW1F,SAAS,gBAAgB,KAA2C;AAClE,SAAO;AAAA,IACL,OAAO,IAAI;AAAA,IACX,SAAS,IAAI;AAAA,IACb,MAAM,IAAI;AAAA,IACV,aAAa,IAAI,eAAe;AAAA,IAChC,QAAQ,IAAI;AAAA,IACZ,iBAAiB,IAAI;AAAA,IACrB,gBAAgB,IAAI;AAAA,IACpB,YAAY,IAAI,cAAc;AAAA,IAC9B,YAAY,IAAI,cAAc;AAAA,IAC9B,aAAa,IAAI;AAAA,IACjB,MAAM,IAAI,QAAQ;AAAA,IAClB,WAAW,IAAI,WAAW,YAAY,KAAK;AAAA,IAC3C,YAAY,IAAI,YAAY,YAAY,KAAK;AAAA,EAC/C;AACF;AAEO,SAAS,sBAAsB,IAAmB,UAAyG;AAChK,QAAM,yBAAyB,8BAA8B;AAM7D,QAAM,oBAAoB,oBAAI,IAAoC;AAElE,WAAS,eAAe,OAAe,KAA6B;AAClE,WAAO;AAAA,MACL,IAAI;AAAA,MACJ,UAAU,IAAI;AAAA,MACd,GAAI,IAAI,iBAAiB,EAAE,gBAAgB,IAAI,eAAe,IAAI,CAAC;AAAA,IACrE;AAAA,EACF;AAEA,iBAAe,aAAa,OAAe,KAA0D;AACnG,WAAO,GAAG,QAAQ,aAAa,eAAe,OAAO,GAAG,GAAG,EAAE,oBAAoB,KAAK,CAAC;AAAA,EACzF;AAEA,iBAAe,oBAAoB,OAAe,KAA8D;AAC9G,UAAM,SAAS,kBAAkB,IAAI,KAAK;AAC1C,QAAI,OAAQ,QAAO;AACnB,UAAM,MAAM,MAAM,GAAG,cAAc,aAAa,eAAe,OAAO,GAAG,GAAG,EAAE,oBAAoB,KAAK,CAAC;AACxG,UAAM,QAAgC;AAAA,MACpC;AAAA,MACA,iBAAiB;AAAA,MACjB,iBAAiB;AAAA,MACjB,sBAAsB,IAAI;AAAA,MAC1B,cAAc;AAAA,MACd,uBAAuB;AAAA,IACzB;AACA,sBAAkB,IAAI,OAAO,KAAK;AAClC,WAAO;AAAA,EACT;AAEA,WAAS,kBAAkB,OAAe;AACxC,sBAAkB,OAAO,KAAK;AAAA,EAChC;AAEA,WAAS,uBAAuB,OAAwD;AACtF,QAAI,MAAM,yBAAyB,CAAC,OAAO,cAAc,MAAM,YAAY,GAAG;AAC5E,aAAO,EAAE,gBAAgB,MAAM,IAAI,eAAe;AAAA,IACpD;AACA,QAAI,MAAM,iBAAiB,GAAG;AAC5B,YAAM,iBAAiB,IAAI,qBAAqB,MAAM,YAAY,EAAE;AACpE,YAAM,aAAa,MAAM,IAAI;AAC7B,UAAI,OAAO,cAAc,UAAU,KAAK,cAAc,QAAQ,aAAa,GAAG;AAC5E,eAAO;AAAA,UACL;AAAA,UACA,iBAAiB;AAAA,YACf,wCAAwC,MAAM,YAAY,gBAAgB,UAAU;AAAA,UACtF;AAAA,QACF;AAAA,MACF;AACA,aAAO,EAAE,eAAe;AAAA,IAC1B;AACA,WAAO,CAAC;AAAA,EACV;AAWA,iBAAe,eACb,UACA,KACA,KACA,UAAwC,CAAC,GACZ;AAC7B,UAAM,MAAM,oBAAI,KAAK;AAIrB,UAAM,YAAY,IAAI,aAAa;AACnC,UAAM,SAAkC;AAAA,MACtC,GAAG,eAAe,IAAI,IAAI,GAAG;AAAA,MAC7B,QAAQ,EAAE,KAAK,oBAA2C;AAAA,IAC5D;AACA,QAAI,QAAQ,eAAgB,QAAO,eAAe,EAAE,OAAO,GAAG,wBAAwB,IAAI;AAC1F,UAAM,WAAW,MAAM,SAAS,aAAa,aAAa,QAAoC;AAAA,MAC5F,QAAQ;AAAA,MACR;AAAA,MACA,aAAa;AAAA,MACb,YAAY;AAAA,MACZ,cAAc;AAAA,MACd,YAAY;AAAA,MACZ,WAAW;AAAA,IACb,CAAC;AACD,QAAI,aAAa,EAAG,QAAO;AAE3B,QAAI,SAAS;AACb,QAAI,YAAY;AAChB,QAAI,cAAc;AAClB,QAAI,aAAa;AACjB,QAAI,eAAe;AACnB,QAAI,aAAa;AAEjB,UAAM,SAAS,KAAK,gBAAgB,aAAa;AAAA,MAC/C,GAAG,gBAAgB,GAAG;AAAA,MACtB,UAAU,IAAI;AAAA,MACd,gBAAgB,IAAI,kBAAkB;AAAA,IACxC,CAAC;AAED,WAAO;AAAA,EACT;AAEA,iBAAe,yBAAyB,OAA+B,KAAmD;AACxH,UAAM,MAAM,MAAM;AAClB,UAAM,MAAM,KAAK,IAAI;AACrB,UAAM,kBACJ,0BAA0B,KAC1B,MAAM,MAAM,mBAAmB,0BAC/B,KAAK,IAAI,IAAI,kBAAkB,MAAM,oBAAoB,KAAK;AAChE,UAAM,gBAAgB,mBAAmB,MAAM,MAAM,mBAAmB;AAExE,QAAI,CAAC,cAAe,QAAO;AAE3B,UAAM,sBACJ,CAAC,MAAM,yBAAyB,OAAO,cAAc,MAAM,YAAY,KAAK,MAAM,iBAAiB;AACrG,UAAM,OAAgC;AAAA,MACpC,iBAAiB,IAAI;AAAA,MACrB,YAAY,IAAI,cAAc;AAAA,MAC9B,YAAY,IAAI,cAAc;AAAA,MAC9B,MAAM,IAAI,QAAQ;AAAA,MAClB,aAAa,IAAI,eAAe,oBAAI,KAAK;AAAA,MACzC,WAAW,oBAAI,KAAK;AAAA,MACpB,GAAG,uBAAuB,KAAK;AAAA,IACjC;AACA,UAAM,iBAAiB;AAAA,MACrB,GAAG,eAAe,IAAI,IAAI,GAAG;AAAA,MAC7B,QAAQ,EAAE,KAAK,2BAAkD;AAAA,IACnE;AACA,QAAI,WAAW,MAAM,GAAG,aAAa,aAAa,gBAAgB,IAAI;AAEtE,QAAI,aAAa,GAAG;AAGlB,YAAM,QAAQ,MAAM,aAAa,IAAI,IAAI,GAAG;AAC5C,UAAI,UAAU;AACd,UAAI,SAAS,MAAM,WAAW,YAAa,MAAM,eAAe,IAAI,OAAO,KAAK,EAAE,gBAAgB,KAAK,CAAC,GAAI;AAC1G,kBAAU;AACV,YAAI,SAAS;AACb,YAAI,YAAY,MAAM;AACtB,YAAI,aAAa;AACjB,YAAI,eAAe;AACnB,YAAI,aAAa;AACjB,mBAAW,MAAM,GAAG,aAAa,aAAa,gBAAgB,IAAI;AAAA,MACpE;AACA,UAAI,aAAa,GAAG;AAElB,0BAAkB,IAAI,EAAE;AACxB,cAAM,SAAS,UAAU,MAAM,aAAa,IAAI,IAAI,GAAG,IAAI;AAC3D,eAAO,UAAU;AAAA,MACnB;AAAA,IACF;AAEA,UAAM,eAAe,sBAAuB,MAAM,aAAa,IAAI,IAAI,GAAG,KAAM,MAAM;AACtF,UAAM,MAAM;AACZ,UAAM,eAAe;AACrB,UAAM,wBAAwB;AAC9B,UAAM,kBAAkB;AAExB,QAAI,iBAAiB;AACnB,YAAM,SAAS,KAAK,gBAAgB,aAAa;AAAA,QAC/C,GAAG,gBAAgB,YAAY;AAAA,QAC/B,UAAU,IAAI;AAAA,QACd,gBAAgB,aAAa,kBAAkB;AAAA,MACjD,CAAC;AACD,YAAM,kBAAkB;AACxB,YAAM,uBAAuB,aAAa;AAAA,IAC5C;AAEA,WAAO;AAAA,EACT;AAEA,SAAO;AAAA,IACL,MAAM,UAAU,OAAO,KAAK;AAC1B,YAAM,MAAM,GAAG,OAAO,aAAa;AAAA,QACjC,SAAS,MAAM;AAAA,QACf,MAAM,MAAM;AAAA,QACZ,aAAa,MAAM;AAAA,QACnB,YAAY,MAAM;AAAA,QAClB,aAAa,MAAM,eAAe;AAAA,QAClC,MAAM,MAAM;AAAA,QACZ,aAAa,MAAM;AAAA,QACnB,gBAAgB,MAAM;AAAA,QACtB,gBAAgB,MAAM;AAAA,QACtB,iBAAiB,IAAI;AAAA,QACrB,UAAU,IAAI;AAAA,QACd,gBAAgB,IAAI;AAAA,QACpB,QAAQ;AAAA,MACV,CAAC;AAED,YAAM,GAAG,QAAQ,GAAG,EAAE,MAAM;AAE5B,YAAM,SAAS,KAAK,gBAAgB,aAAa;AAAA,QAC/C,GAAG,gBAAgB,GAAG;AAAA,QACtB,UAAU,IAAI;AAAA,QACd,gBAAgB,IAAI;AAAA,MACtB,CAAC;AAED,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,SAAS,OAAO,KAAK;AACzB,YAAM,MAAM,MAAM,GAAG,cAAc,aAAa,eAAe,OAAO,GAAG,GAAG,EAAE,oBAAoB,KAAK,CAAC;AACxG,UAAI,IAAI,WAAW,aAAa,IAAI,WAAW,eAAe,IAAI,WAAW,aAAa;AACxF,eAAO;AAAA,MACT;AAEA,YAAM,UAAU,MAAM,eAAe,IAAI,KAAK,GAAG;AACjD,UAAI,QAAS,QAAO;AAEpB,YAAM,QAAQ,MAAM,aAAa,OAAO,GAAG;AAC3C,aAAO,SAAS;AAAA,IAClB;AAAA,IAEA,MAAM,kBAAkB,OAAO,KAAK;AAGlC,YAAM,OAAO,GAAG,KAAK;AACrB,YAAM,MAAM,oBAAI,KAAK;AACrB,YAAM,WAAW,MAAM,KAAK,aAAa,aAAa;AAAA,QACpD,GAAG,eAAe,OAAO,GAAG;AAAA,QAC5B,QAAQ,EAAE,KAAK,2BAAkD;AAAA,MACnE,GAA+B;AAAA,QAC7B,aAAa;AAAA,QACb,WAAW;AAAA,MACb,CAAC;AACD,UAAI,WAAW,EAAG;AAElB,YAAM,MAAM,MAAM,KAAK,QAAQ,aAAa,eAAe,OAAO,GAAG,GAAG,EAAE,oBAAoB,KAAK,CAAC;AACpG,UAAI,CAAC,OAAO,IAAI,WAAW,SAAU;AAGrC,YAAM,eAAe,MAAM,KAAK,KAAK,EAAE,gBAAgB,KAAK,CAAC;AAAA,IAC/D;AAAA,IAEA,MAAM,eAAe,OAAO,OAAO,KAAK;AACtC,YAAM,QAAQ,MAAM,oBAAoB,OAAO,GAAG;AAClD,YAAM,MAAM,MAAM;AAClB,UAAI,kBAAkB,SAAS,IAAI,MAAM,GAAG;AAK1C,cAAM,UAAU,IAAI,WAAW,YAAa,MAAM,eAAe,IAAI,KAAK,KAAK,EAAE,gBAAgB,KAAK,CAAC;AACvG,YAAI,CAAC,SAAS;AACZ,4BAAkB,KAAK;AACvB,iBAAO;AAAA,QACT;AACA,cAAM,kBAAkB;AAAA,MAC1B;AAEA,UAAI,MAAM,mBAAmB,QAAW;AACtC,YAAI,iBAAiB,MAAM;AAC3B,cAAM,wBAAwB;AAC9B,cAAM,eAAe;AAAA,MACvB;AACA,UAAI,MAAM,eAAe,QAAW;AAClC,YAAI,aAAa,MAAM;AAAA,MACzB;AACA,UAAI,MAAM,SAAS,QAAW;AAC5B,YAAI,OAAO,EAAE,GAAG,IAAI,MAAM,GAAG,MAAM,KAAK;AAAA,MAC1C;AAEA,UAAI,MAAM,oBAAoB,QAAW;AACvC,YAAI,kBAAkB,MAAM;AAAA,MAC9B,WAAW,IAAI,YAAY;AACzB,YAAI,kBAAkB,yBAAyB,IAAI,gBAAgB,IAAI,UAAU;AAAA,MACnF;AAEA,UAAI,MAAM,eAAe,QAAW;AAClC,YAAI,aAAa,MAAM;AAAA,MACzB,WAAW,IAAI,aAAa,IAAI,YAAY;AAC1C,YAAI,aAAa,aAAa,IAAI,gBAAgB,IAAI,YAAY,IAAI,SAAS;AAAA,MACjF;AAEA,UAAI,cAAc,oBAAI,KAAK;AAE3B,aAAO,yBAAyB,OAAO,GAAG;AAAA,IAC5C;AAAA,IAEA,MAAM,kBAAkB,OAAO,OAAO,KAAK;AACzC,YAAM,QAAQ,MAAM,oBAAoB,OAAO,GAAG;AAClD,YAAM,MAAM,MAAM;AAClB,UAAI,kBAAkB,SAAS,IAAI,MAAM,GAAG;AAC1C,cAAM,UAAU,IAAI,WAAW,YAAa,MAAM,eAAe,IAAI,KAAK,KAAK,EAAE,gBAAgB,KAAK,CAAC;AACvG,YAAI,CAAC,SAAS;AACZ,4BAAkB,KAAK;AACvB,iBAAO;AAAA,QACT;AACA,cAAM,kBAAkB;AAAA,MAC1B;AAEA,UAAI,kBAAkB;AACtB,YAAM,gBAAgB;AACtB,UAAI,cAAc,oBAAI,KAAK;AAE3B,UAAI,IAAI,YAAY;AAClB,YAAI,kBAAkB,yBAAyB,IAAI,gBAAgB,IAAI,UAAU;AACjF,YAAI,IAAI,WAAW;AACjB,cAAI,aAAa,aAAa,IAAI,gBAAgB,IAAI,YAAY,IAAI,SAAS;AAAA,QACjF;AAAA,MACF;AAEA,aAAO,yBAAyB,OAAO,GAAG;AAAA,IAC5C;AAAA,IAEA,MAAM,YAAY,OAAO,OAAO,KAAK;AACnC,YAAM,MAAM,MAAM,aAAa,OAAO,GAAG;AACzC,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,OAAO,KAAK,YAAY;AAClD,UAAI,IAAI,WAAW,eAAe,IAAI,WAAW,aAAa;AAC5D,0BAAkB,KAAK;AACvB,eAAO;AAAA,MACT;AAEA,YAAM,QAAQ,kBAAkB,IAAI,KAAK;AACzC,YAAM,WAAW,OAAO,OAAO;AAC/B,YAAM,MAAM,oBAAI,KAAK;AACrB,YAAM,OAAgC;AAAA,QACpC,QAAQ;AAAA,QACR,YAAY;AAAA,QACZ,YAAY;AAAA,QACZ,WAAW;AAAA,QACX,GAAI,OAAO,gBAAgB,EAAE,eAAe,MAAM,cAAc,IAAI,CAAC;AAAA,QACrE,GAAI,QACA;AAAA,UACE,YAAY,SAAS,cAAc;AAAA,UACnC,MAAM,SAAS,QAAQ;AAAA,UACvB,GAAG,uBAAuB,KAAK;AAAA,QACjC,IACA,CAAC;AAAA,QACL,iBAAiB;AAAA,MACnB;AAEA,YAAM,WAAW,MAAM,GAAG,aAAa,aAAa;AAAA,QAClD,GAAG,eAAe,OAAO,GAAG;AAAA,QAC5B,QAAQ,EAAE,KAAK,uBAA8C;AAAA,MAC/D,GAA+B,IAAI;AACnC,wBAAkB,KAAK;AAEvB,UAAI,aAAa,GAAG;AAClB,cAAM,QAAQ,MAAM,aAAa,OAAO,GAAG;AAC3C,eAAO,SAAS;AAAA,MAClB;AAEA,eAAS,SAAS;AAClB,eAAS,aAAa;AACtB,eAAS,kBAAkB;AAC3B,eAAS,aAAa;AACtB,UAAI,OAAO,eAAe;AACxB,iBAAS,gBAAgB,MAAM;AAAA,MACjC;AAEA,YAAM,eAAgB,MAAM,aAAa,OAAO,GAAG,KAAM;AAEzD,YAAM,SAAS,KAAK,gBAAgB,eAAe;AAAA,QACjD,GAAG,gBAAgB,YAAY;AAAA,QAC/B,eAAe,aAAa;AAAA,QAC5B,UAAU,IAAI;AAAA,QACd,gBAAgB,aAAa,kBAAkB;AAAA,MACjD,CAAC;AAED,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,QAAQ,OAAO,OAAO,KAAK;AAC/B,YAAM,MAAM,MAAM,aAAa,OAAO,GAAG;AACzC,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,OAAO,KAAK,YAAY;AAClD,UAAI,kBAAkB,SAAS,IAAI,MAAM,GAAG;AAC1C,0BAAkB,KAAK;AACvB,eAAO;AAAA,MACT;AAEA,YAAM,QAAQ,kBAAkB,IAAI,KAAK;AACzC,YAAM,WAAW,OAAO,OAAO;AAC/B,YAAM,MAAM,oBAAI,KAAK;AACrB,YAAM,OAAgC;AAAA,QACpC,QAAQ;AAAA,QACR,YAAY;AAAA,QACZ,cAAc,MAAM;AAAA,QACpB,YAAY,MAAM;AAAA,QAClB,GAAI,MAAM,gBAAgB,EAAE,eAAe,MAAM,cAAc,IAAI,CAAC;AAAA,QACpE,WAAW;AAAA,QACX,GAAI,QACA;AAAA,UACE,YAAY,SAAS,cAAc;AAAA,UACnC,MAAM,SAAS,QAAQ;AAAA,UACvB,GAAG,uBAAuB,KAAK;AAAA,QACjC,IACA,CAAC;AAAA,MACP;AAEA,YAAM,WAAW,MAAM,GAAG,aAAa,aAAa;AAAA,QAClD,GAAG,eAAe,OAAO,GAAG;AAAA,QAC5B,QAAQ,EAAE,KAAK,mBAA0C;AAAA,MAC3D,GAA+B,IAAI;AACnC,wBAAkB,KAAK;AAEvB,UAAI,aAAa,GAAG;AAClB,cAAM,QAAQ,MAAM,aAAa,OAAO,GAAG;AAC3C,eAAO,SAAS;AAAA,MAClB;AAEA,eAAS,SAAS;AAClB,eAAS,aAAa;AACtB,eAAS,eAAe,MAAM;AAC9B,eAAS,aAAa,MAAM;AAC5B,UAAI,MAAM,eAAe;AACvB,iBAAS,gBAAgB,MAAM;AAAA,MACjC;AAEA,YAAM,eAAgB,MAAM,aAAa,OAAO,GAAG,KAAM;AAEzD,YAAM,SAAS,KAAK,gBAAgB,YAAY;AAAA,QAC9C,GAAG,gBAAgB,YAAY;AAAA,QAC/B,cAAc,aAAa;AAAA,QAC3B,UAAU,IAAI;AAAA,QACd,gBAAgB,aAAa,kBAAkB;AAAA,MACjD,CAAC;AAED,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,UAAU,OAAO,KAAK;AAC1B,YAAM,MAAM,MAAM,GAAG,cAAc,aAAa,eAAe,OAAO,GAAG,GAAG,EAAE,oBAAoB,KAAK,CAAC;AACxG,UAAI,kBAAkB,SAAS,IAAI,MAAM,GAAG;AAE1C,eAAO;AAAA,MACT;AACA,UAAI,CAAC,IAAI,aAAa;AACpB,cAAM,IAAI,MAAM,OAAO,KAAK,qBAAqB;AAAA,MACnD;AAEA,YAAM,MAAM,oBAAI,KAAK;AACrB,YAAM,eAAe,MAAM,GAAG,aAAa,aAAa;AAAA,QACtD,GAAG,eAAe,OAAO,GAAG;AAAA,QAC5B,aAAa;AAAA,QACb,QAAQ;AAAA,MACV,GAA+B;AAAA,QAC7B,QAAQ;AAAA,QACR,mBAAmB;AAAA,QACnB,mBAAmB,IAAI,UAAU;AAAA,QACjC,YAAY;AAAA,QACZ,YAAY;AAAA,QACZ,WAAW;AAAA,MACb,CAAC;AAED,UAAI,YAAY;AAChB,UAAI,iBAAiB,GAAG;AACtB,oBAAY,MAAM,GAAG,aAAa,aAAa;AAAA,UAC7C,GAAG,eAAe,OAAO,GAAG;AAAA,UAC5B,aAAa;AAAA,UACb,QAAQ;AAAA,QACV,GAA+B;AAAA,UAC7B,mBAAmB;AAAA,UACnB,mBAAmB,IAAI,UAAU;AAAA,UACjC,WAAW;AAAA,QACb,CAAC;AAAA,MACH;AAEA,UAAI,iBAAiB,KAAK,cAAc,GAAG;AAEzC,cAAM,QAAQ,MAAM,aAAa,OAAO,GAAG;AAC3C,eAAO,SAAS;AAAA,MAClB;AAEA,wBAAkB,KAAK;AACvB,UAAI,eAAe,GAAG;AACpB,YAAI,SAAS;AACb,YAAI,aAAa;AACjB,YAAI,aAAa;AAAA,MACnB;AACA,UAAI,oBAAoB;AACxB,UAAI,oBAAoB,IAAI,UAAU;AAEtC,YAAM,SAAS,KAAK,gBAAgB,eAAe;AAAA,QACjD,GAAG,gBAAgB,GAAG;AAAA,QACtB,UAAU,IAAI;AAAA,QACd,gBAAgB,IAAI,kBAAkB;AAAA,MACxC,CAAC;AAED,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,cAAc,OAAO,KAAK;AAC9B,YAAM,MAAM,MAAM,aAAa,OAAO,GAAG;AACzC,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,OAAO,KAAK,YAAY;AAClD,UAAI,IAAI,WAAW,eAAe,IAAI,WAAW,aAAa;AAC5D,0BAAkB,KAAK;AACvB,eAAO;AAAA,MACT;AAEA,YAAM,MAAM,oBAAI,KAAK;AACrB,YAAM,oBAAoB,IAAI,qBAAqB;AACnD,YAAM,oBAAoB,IAAI,qBAAqB,IAAI,UAAU;AACjE,YAAM,aAAa,IAAI,cAAc;AAErC,YAAM,WAAW,MAAM,GAAG,aAAa,aAAa;AAAA,QAClD,GAAG,eAAe,OAAO,GAAG;AAAA,QAC5B,QAAQ,EAAE,KAAK,qBAA4C;AAAA,MAC7D,GAA+B;AAAA,QAC7B,QAAQ;AAAA,QACR;AAAA,QACA;AAAA,QACA;AAAA,QACA,YAAY;AAAA,QACZ,WAAW;AAAA,MACb,CAAC;AACD,wBAAkB,KAAK;AAEvB,UAAI,aAAa,GAAG;AAClB,cAAM,QAAQ,MAAM,aAAa,OAAO,GAAG;AAC3C,eAAO,SAAS;AAAA,MAClB;AAEA,UAAI,SAAS;AACb,UAAI,oBAAoB;AACxB,UAAI,oBAAoB;AACxB,UAAI,aAAa;AACjB,UAAI,aAAa;AAEjB,YAAM,SAAS,KAAK,gBAAgB,eAAe;AAAA,QACjD,GAAG,gBAAgB,GAAG;AAAA,QACtB,UAAU,IAAI;AAAA,QACd,gBAAgB,IAAI,kBAAkB;AAAA,MACxC,CAAC;AAED,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,wBAAwB,OAAO,UAAU,gBAAgB;AAI7D,YAAM,MAAM,MAAM,sBAAsB,IAAI,aAAa;AAAA,QACvD,IAAI;AAAA,QACJ;AAAA,QACA,GAAI,iBAAiB,EAAE,eAAe,IAAI,CAAC;AAAA,MAC7C,GAAG,EAAE,oBAAoB,KAAK,CAAC;AAC/B,aAAO,KAAK,qBAAqB;AAAA,IACnC;AAAA,IAEA,MAAM,cAAc,KAAK;AACvB,aAAO,GAAG,KAAK,aAAa;AAAA,QAC1B,UAAU,IAAI;AAAA,QACd,GAAI,IAAI,iBAAiB,EAAE,gBAAgB,IAAI,eAAe,IAAI,CAAC;AAAA,QACnE,QAAQ,EAAE,KAAK,CAAC,WAAW,SAAS,EAAE;AAAA,QACtC,aAAa;AAAA,MACf,GAAG;AAAA,QACD,SAAS,EAAE,WAAW,OAAO;AAAA,QAC7B,OAAO;AAAA,MACT,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,yBAAyB,KAAK,eAAe,IAAI;AACrD,YAAM,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI,eAAe,GAAI;AACxD,aAAO,GAAG,KAAK,aAAa;AAAA,QAC1B,UAAU,IAAI;AAAA,QACd,GAAI,IAAI,iBAAiB,EAAE,gBAAgB,IAAI,eAAe,IAAI,CAAC;AAAA,QACnE,QAAQ,EAAE,KAAK,CAAC,aAAa,QAAQ,EAAE;AAAA,QACvC,YAAY,EAAE,MAAM,OAAO;AAAA,QAC3B,aAAa;AAAA,MACf,GAAG;AAAA,QACD,SAAS,EAAE,YAAY,OAAO;AAAA,QAC9B,OAAO;AAAA,MACT,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,OAAO,OAAO,KAAK;AACvB,aAAO,GAAG,QAAQ,aAAa;AAAA,QAC7B,IAAI;AAAA,QACJ,UAAU,IAAI;AAAA,QACd,GAAI,IAAI,iBAAiB,EAAE,gBAAgB,IAAI,eAAe,IAAI,CAAC;AAAA,MACrE,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,oBAAoB,UAAkB,iBAAiB,2BAA2B,gBAAgC;AACtH,YAAM,MAAM,oBAAI,KAAK;AACrB,YAAM,SAAS,IAAI,KAAK,IAAI,QAAQ,IAAI,iBAAiB,GAAI;AAC7D,YAAM,QAAQ;AAAA,QACZ;AAAA,QACA,GAAI,iBAAiB,EAAE,eAAe,IAAI,CAAC;AAAA,MAC7C;AACA,UAAI,cAAc;AAElB,YAAM,cAAc;AAAA,QAClB,QAAQ;AAAA,QACR,KAAK;AAAA,UACH,EAAE,aAAa,EAAE,KAAK,OAAO,EAAE;AAAA,UAC/B;AAAA,YACE,aAAa;AAAA,YACb,WAAW,EAAE,KAAK,OAAO;AAAA,UAC3B;AAAA,QACF;AAAA,MACF;AACA,YAAM,eAAe,MAAM,GAAG,KAAK,aAAa,EAAE,GAAG,OAAO,GAAG,YAAY,GAAG,EAAE,oBAAoB,KAAK,CAAC;AAE1G,iBAAW,OAAO,cAAc;AAC9B,cAAM,eAAe,GAAG,wBAAwB,qBAAqB,cAAc;AAGnF,cAAM,WAAW,MAAM,GAAG,aAAa,aAAa;AAAA,UAClD,IAAI,IAAI;AAAA,UACR,UAAU,IAAI;AAAA,UACd,GAAG;AAAA,QACL,GAA+B;AAAA,UAC7B,QAAQ;AAAA,UACR,YAAY;AAAA,UACZ;AAAA,UACA,WAAW;AAAA,QACb,CAAC;AACD,YAAI,aAAa,EAAG;AAEpB,uBAAe;AACf,YAAI,SAAS;AACb,YAAI,aAAa;AACjB,YAAI,eAAe;AAEnB,cAAM,SAAS,KAAK,gBAAgB,YAAY;AAAA,UAC9C,GAAG,gBAAgB,GAAG;AAAA,UACtB,cAAc,IAAI;AAAA,UAClB,UAAU,IAAI;AAAA,UACd,OAAO;AAAA,UACP,gBAAgB,IAAI,kBAAkB;AAAA,QACxC,CAAC;AAAA,MACH;AAKA,YAAM,gBAAgB,IAAI,KAAK,IAAI,QAAQ,IAAI,gCAAgC,GAAI;AACnF,YAAM,qBAAqB;AAAA,QACzB,QAAQ;AAAA,QACR,WAAW,EAAE,KAAK,cAAc;AAAA,MAClC;AACA,YAAM,eAAe,MAAM,GAAG,KAAK,aAAa,EAAE,GAAG,OAAO,GAAG,mBAAmB,GAAG,EAAE,oBAAoB,KAAK,CAAC;AAEjH,iBAAW,OAAO,cAAc;AAC9B,cAAM,eAAe,GAAG,wBAAwB,yBAAyB,6BAA6B;AACtG,cAAM,WAAW,MAAM,GAAG,aAAa,aAAa;AAAA,UAClD,IAAI,IAAI;AAAA,UACR,UAAU,IAAI;AAAA,UACd,GAAG;AAAA,QACL,GAA+B;AAAA,UAC7B,QAAQ;AAAA,UACR,YAAY;AAAA,UACZ;AAAA,UACA,WAAW;AAAA,QACb,CAAC;AACD,YAAI,aAAa,EAAG;AAEpB,uBAAe;AACf,YAAI,SAAS;AACb,YAAI,aAAa;AACjB,YAAI,eAAe;AAEnB,cAAM,SAAS,KAAK,gBAAgB,YAAY;AAAA,UAC9C,GAAG,gBAAgB,GAAG;AAAA,UACtB,cAAc,IAAI;AAAA,UAClB,UAAU,IAAI;AAAA,UACd,OAAO;AAAA,UACP,gBAAgB,IAAI,kBAAkB;AAAA,QACxC,CAAC;AAAA,MACH;AAEA,aAAO;AAAA,IACT;AAAA,EACF;AACF;",
6
6
  "names": []
7
7
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@open-mercato/core",
3
- "version": "0.6.8-develop.6985.1.fb93574faa",
3
+ "version": "0.6.8-develop.6987.1.02f619470c",
4
4
  "license": "MIT",
5
5
  "type": "module",
6
6
  "main": "./dist/index.js",
@@ -254,16 +254,16 @@
254
254
  "zod": "^4.4.3"
255
255
  },
256
256
  "peerDependencies": {
257
- "@open-mercato/ai-assistant": "0.6.8-develop.6985.1.fb93574faa",
258
- "@open-mercato/shared": "0.6.8-develop.6985.1.fb93574faa",
259
- "@open-mercato/ui": "0.6.8-develop.6985.1.fb93574faa",
257
+ "@open-mercato/ai-assistant": "0.6.8-develop.6987.1.02f619470c",
258
+ "@open-mercato/shared": "0.6.8-develop.6987.1.02f619470c",
259
+ "@open-mercato/ui": "0.6.8-develop.6987.1.02f619470c",
260
260
  "react": "^19.0.0",
261
261
  "react-dom": "^19.0.0"
262
262
  },
263
263
  "devDependencies": {
264
- "@open-mercato/ai-assistant": "0.6.8-develop.6985.1.fb93574faa",
265
- "@open-mercato/shared": "0.6.8-develop.6985.1.fb93574faa",
266
- "@open-mercato/ui": "0.6.8-develop.6985.1.fb93574faa",
264
+ "@open-mercato/ai-assistant": "0.6.8-develop.6987.1.02f619470c",
265
+ "@open-mercato/shared": "0.6.8-develop.6987.1.02f619470c",
266
+ "@open-mercato/ui": "0.6.8-develop.6987.1.02f619470c",
267
267
  "@testing-library/dom": "^10.4.1",
268
268
  "@testing-library/jest-dom": "^7.0.0",
269
269
  "@testing-library/react": "^16.3.1",
@@ -172,6 +172,8 @@ Data sync providers can leverage the **Unified Module Extension System (UMES)**
172
172
 
173
173
  - `ProgressTopBar` and sync-run detail pages use `progress.job.*` SSE updates for live progress.
174
174
  - Create `ProgressJob` in `run`/`retry` endpoints; start/update/complete/fail in `sync-engine`.
175
+ - The engine heartbeats (`touchJobHeartbeat`, forked-EM) on a timer while an adapter batch is being produced, because batches can outlast the 60s stale-job sweep — keep the `withHeartbeat` wrapper around `streamImport`/`streamExport` when touching the batch loops.
176
+ - On redelivery the progress counter is seeded from the progress job's own `processedCount` (`progressService.getJob`), mirroring how `committedBatches` resumes — never reset it to zero (`updateProgress` writes absolute counts) and never seed it from the run's `created/updated/skipped/failed` columns: those count emitted items, while progress counts source records (`batch.processedCount`), and adapters may emit several items per source record.
175
177
  - Include `progressJob` details in run detail response.
176
178
  - SSE DOM bridge forwards only events with `clientBroadcast: true`.
177
179
  - `progress.job.*` events are marked `clientBroadcast: true` and must reach the browser from both web and worker processes.
@@ -4,6 +4,7 @@ import type { CredentialsService } from '../../integrations/lib/credentials-serv
4
4
  import type { IntegrationLogService } from '../../integrations/lib/log-service'
5
5
  import type { IntegrationStateService } from '../../integrations/lib/state-service'
6
6
  import type { ProgressService } from '../../progress/lib/progressService'
7
+ import { STALE_JOB_TIMEOUT_SECONDS } from '../../progress/lib/progressService'
7
8
  import { refreshCoverageSnapshot } from '../../query_index/lib/coverage'
8
9
  import { emitDataSyncEvent } from '../events'
9
10
  import type { DataSyncAdapter, DataMapping, ExportBatch, ImportBatch } from './adapter'
@@ -76,6 +77,32 @@ function applyExportCounters(batch: ExportBatch): SyncCounterDelta {
76
77
  }
77
78
  }
78
79
 
80
+ // Adapter batches can legitimately outlast the stale-job sweep window (slow upstream
81
+ // APIs), so the engine must heartbeat while a batch is still being produced.
82
+ const HEARTBEAT_TICK_MS = (STALE_JOB_TIMEOUT_SECONDS * 1000) / 4
83
+
84
+ // Runs `tick` on an interval only while the source iterator is pending, so heartbeats
85
+ // stop the moment the producer dies and genuinely stale jobs still get swept. The outer
86
+ // finally closes the adapter generator on early exits (cancellation, ownership conflict).
87
+ async function* withHeartbeat<T>(source: AsyncIterable<T>, tick: () => void, intervalMs: number): AsyncGenerator<T, void, undefined> {
88
+ const iterator = source[Symbol.asyncIterator]()
89
+ try {
90
+ while (true) {
91
+ const timer = setInterval(tick, intervalMs)
92
+ let result: IteratorResult<T>
93
+ try {
94
+ result = await iterator.next()
95
+ } finally {
96
+ clearInterval(timer)
97
+ }
98
+ if (result.done) return
99
+ yield result.value
100
+ }
101
+ } finally {
102
+ await iterator.return?.()
103
+ }
104
+ }
105
+
79
106
  export function createSyncEngine(deps: EngineDeps) {
80
107
  const { syncRunService, integrationCredentialsService, integrationLogService, integrationStateService, progressService } = deps
81
108
 
@@ -103,6 +130,47 @@ export function createSyncEngine(deps: EngineDeps) {
103
130
  )
104
131
  }
105
132
 
133
+ // On redelivery the progress counter must resume where the last delivery left off the
134
+ // same way committedBatches does — updateProgress writes absolute counts, so starting at
135
+ // zero would regress the visible count. The progress job's own processedCount is the only
136
+ // persisted value already in the engine's unit: `batch.processedCount ?? items.length`,
137
+ // i.e. source records. The run's created/updated/skipped/failed counters count emitted
138
+ // items, which adapters may explode several-per-source-record (Akeneo yields a product
139
+ // plus its variants), so seeding from them would overshoot the total and pin the bar.
140
+ async function seedProcessedCount(progressJobId: string | null | undefined, scope: SyncScope): Promise<number> {
141
+ if (!progressJobId) return 0
142
+ const job = await progressService.getJob(progressJobId, {
143
+ tenantId: scope.tenantId,
144
+ organizationId: scope.organizationId,
145
+ userId: scope.userId,
146
+ })
147
+ return job?.processedCount ?? 0
148
+ }
149
+
150
+ function makeHeartbeatTick(progressJobId: string | null | undefined, scope: SyncScope): () => void {
151
+ const touchJobHeartbeat = progressService.touchJobHeartbeat?.bind(progressService)
152
+ if (!progressJobId || !touchJobHeartbeat) return () => {}
153
+ let inFlight = false
154
+ return () => {
155
+ if (inFlight) return
156
+ inFlight = true
157
+ touchJobHeartbeat(progressJobId, {
158
+ tenantId: scope.tenantId,
159
+ organizationId: scope.organizationId,
160
+ userId: scope.userId,
161
+ })
162
+ .catch((error) => {
163
+ logger.warn('Progress heartbeat failed', {
164
+ progressJobId,
165
+ error: error instanceof Error ? error.message : String(error),
166
+ })
167
+ })
168
+ .finally(() => {
169
+ inFlight = false
170
+ })
171
+ }
172
+ }
173
+
106
174
  async function refreshCoverageSnapshots(entityTypes: string[] | undefined, scope: SyncScope): Promise<void> {
107
175
  if (!entityTypes || entityTypes.length === 0) return
108
176
 
@@ -471,20 +539,24 @@ export function createSyncEngine(deps: EngineDeps) {
471
539
  }
472
540
 
473
541
  const mapping = await resolveMapping(adapter, run.entityType, scope)
474
- let processedCount = 0
542
+ let processedCount = await seedProcessedCount(run.progressJobId, scope)
475
543
  let totalCount: number | null = null
476
544
  let committedBatches = activeRun.batchesCompleted ?? 0
477
545
 
478
546
  try {
479
- for await (const batch of adapter.streamImport({
480
- entityType: run.entityType,
481
- cursor: run.cursor ?? undefined,
482
- batchSize,
483
- credentials,
484
- mapping,
485
- scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },
486
- runId: run.id,
487
- })) {
547
+ for await (const batch of withHeartbeat(
548
+ adapter.streamImport({
549
+ entityType: run.entityType,
550
+ cursor: run.cursor ?? undefined,
551
+ batchSize,
552
+ credentials,
553
+ mapping,
554
+ scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },
555
+ runId: run.id,
556
+ }),
557
+ makeHeartbeatTick(run.progressJobId, scope),
558
+ HEARTBEAT_TICK_MS,
559
+ )) {
488
560
  if (run.progressJobId && await progressService.isCancellationRequested(run.progressJobId, scope.tenantId, scope.organizationId)) {
489
561
  await finalizeRun(run.id, 'cancelled', scope, undefined, operationalTelemetry)
490
562
  return
@@ -631,19 +703,23 @@ export function createSyncEngine(deps: EngineDeps) {
631
703
  }
632
704
 
633
705
  const mapping = await resolveMapping(adapter, run.entityType, scope)
634
- let processedCount = 0
706
+ let processedCount = await seedProcessedCount(run.progressJobId, scope)
635
707
  let committedBatches = activeRun.batchesCompleted ?? 0
636
708
 
637
709
  try {
638
- for await (const batch of adapter.streamExport({
639
- entityType: run.entityType,
640
- cursor: run.cursor ?? undefined,
641
- batchSize,
642
- credentials,
643
- mapping,
644
- scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },
645
- runId: run.id,
646
- })) {
710
+ for await (const batch of withHeartbeat(
711
+ adapter.streamExport({
712
+ entityType: run.entityType,
713
+ cursor: run.cursor ?? undefined,
714
+ batchSize,
715
+ credentials,
716
+ mapping,
717
+ scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },
718
+ runId: run.id,
719
+ }),
720
+ makeHeartbeatTick(run.progressJobId, scope),
721
+ HEARTBEAT_TICK_MS,
722
+ )) {
647
723
  if (run.progressJobId && await progressService.isCancellationRequested(run.progressJobId, scope.tenantId, scope.organizationId)) {
648
724
  await finalizeRun(run.id, 'cancelled', scope, undefined, operationalTelemetry)
649
725
  return
@@ -93,7 +93,8 @@ Use stable, grep-friendly ids:
93
93
  `progressService` is safe to run across many app/worker instances. Do not reintroduce read-modify-write transitions:
94
94
 
95
95
  - **Every status transition is a status-guarded `nativeUpdate` (CAS).** An update that matches zero rows lost the race — it MUST NOT emit events or overwrite the row. Allowed transitions: start from `pending|failed` (an already-running start is an idempotent no-op; `failed` allows queue retries and recovery from a wrong stale sweep), complete from `pending|running|failed`, fail from `pending|running`, cancel from `pending|running|failed`.
96
- - **Progress writes are guarded on `status IN ('pending','running')`** — once another process finishes/cancels a job, buffered updaters stop writing to it.
96
+ - **Progress writes are guarded on `status IN ('pending','running')`** — once another process finishes/cancels a job, buffered updaters stop writing to it. Exception: when a write path finds the job `failed`, it attempts exactly one revival through the start CAS (a live producer writing progress proves a stale sweep false-positived) before giving up. That revive CAS is narrowed to rows whose `errorMessage` starts with `STALE_SWEEP_ERROR_PREFIX`, so a job that failed for a real reason keeps its message and stack instead of being resurrected by a racing buffered writer; `completed`/`cancelled` are never revived, and genuinely dead producers never write again, so real sweeps stick. Only `startJob` uses the unrestricted CAS, because a queue retry restarts the whole unit of work. A revive never resets `startedAt` — progress counts are absolute across deliveries, so the elapsed window `calculateEta` divides them by must be too.
97
+ - **Long single units of work MUST call `touchJobHeartbeat`** — when one batch/step can outlast `STALE_JOB_TIMEOUT_SECONDS` (60s), run a keepalive that calls the optional `touchJobHeartbeat(jobId, ctx)` while awaiting it (see `withHeartbeat` in `data_sync/lib/sync-engine.ts`). It writes only `heartbeat_at`/`updated_at` on a **forked** EntityManager, so it is safe while the producer's own EM is mid-transaction, and it self-heals a falsely-swept job via the start CAS. Always optional-chain it (`progressService.touchJobHeartbeat?.(...)`) — third-party implementations may not provide it. The "no timers for durable work" rule below bans replacing queue delivery with timers, not an in-process keepalive around a pending await.
97
98
  - **`incrementProgress` deltas persist as atomic SQL increments** (`processed_count + n`), and the service reloads the database winner before returning or emitting, so concurrent writers never lose or report stale counts.
98
99
  - **Update-path reads use `disableIdentityMap: true`** — `isCancellationRequested` and lifecycle reads must always see fresh cross-process state, never a stale managed entity.
99
100
  - **The stale sweep (`markStaleJobsFailed`) re-checks staleness per row inside the CAS**, so concurrent sweepers emit exactly one `JOB_FAILED` per job, and it also fails `pending` jobs that never started within `STALE_PENDING_TIMEOUT_SECONDS` (a late queue delivery recovers them via `startJob`'s `failed → running` transition).
@@ -21,12 +21,21 @@ export interface ProgressService {
21
21
  getRecentlyCompletedJobs(ctx: ProgressServiceContext, sinceSeconds?: number): Promise<ProgressJob[]>
22
22
  getJob(jobId: string, ctx: ProgressServiceContext): Promise<ProgressJob | null>
23
23
  markStaleJobsFailed(tenantId: string, timeoutSeconds?: number, organizationId?: string | null): Promise<number>
24
+ // Optional so third-party ProgressService implementations keep compiling; callers must
25
+ // optional-chain. Runs on a forked EntityManager, so it is safe to call while the shared
26
+ // request/worker EM is mid-transaction (e.g. from a keepalive timer around adapter I/O).
27
+ touchJobHeartbeat?(jobId: string, ctx: ProgressServiceContext): Promise<void>
24
28
  }
25
29
 
26
30
  export const HEARTBEAT_INTERVAL_MS = 5000
27
31
  export const STALE_JOB_TIMEOUT_SECONDS = 60
28
32
  export const STALE_PENDING_TIMEOUT_SECONDS = 900
29
33
 
34
+ // Every `errorMessage` the stale sweep writes starts with this, so recovery paths can tell
35
+ // a job the sweep gave up on from one that failed for a real reason and must keep its
36
+ // diagnostics. Declared here so the sweep and the revive filter cannot drift apart.
37
+ export const STALE_SWEEP_ERROR_PREFIX = 'Job stale:'
38
+
30
39
  export function calculateEta(
31
40
  processedCount: number,
32
41
  totalCount: number,
@@ -9,6 +9,7 @@ import {
9
9
  HEARTBEAT_INTERVAL_MS,
10
10
  STALE_JOB_TIMEOUT_SECONDS,
11
11
  STALE_PENDING_TIMEOUT_SECONDS,
12
+ STALE_SWEEP_ERROR_PREFIX,
12
13
  } from './progressService'
13
14
  import { PROGRESS_EVENTS } from './events'
14
15
  import { findOneWithDecryption } from '@open-mercato/shared/lib/encryption/find'
@@ -131,6 +132,58 @@ export function createProgressService(em: EntityManager, eventBus: { emit: (even
131
132
  return {}
132
133
  }
133
134
 
135
+ // The start transition (START_FROM_STATUSES) doubles as the recovery path for jobs a
136
+ // stale sweep flipped to `failed` while their producer was alive but slow. Callers pass
137
+ // the EM to run against so the forked heartbeat path can reuse the same CAS.
138
+ //
139
+ // `staleSweptOnly` is for the revive paths: a live write proves the producer is alive,
140
+ // but it does NOT prove the recorded failure was the sweep's. Narrowing the CAS to rows
141
+ // the sweep tagged keeps a genuine failure's message and stack intact instead of
142
+ // resurrecting the job and erasing why it died. `startJob` stays unrestricted so an
143
+ // at-least-once queue can still retry a genuinely failed job from the top.
144
+ async function startJobViaCas(
145
+ targetEm: EntityManager,
146
+ job: ProgressJob,
147
+ ctx: ProgressServiceContext,
148
+ options: { staleSweptOnly?: boolean } = {},
149
+ ): Promise<ProgressJob | null> {
150
+ const now = new Date()
151
+ // Progress counts are absolute and survive across deliveries, so the elapsed window
152
+ // calculateEta divides them by has to survive too. Resetting it on every revive makes
153
+ // a job with hours of work left report seconds remaining.
154
+ const startedAt = job.startedAt ?? now
155
+ const filter: Record<string, unknown> = {
156
+ ...jobScopeFilter(job.id, ctx),
157
+ status: { $in: START_FROM_STATUSES as ProgressJobStatus[] },
158
+ }
159
+ if (options.staleSweptOnly) filter.errorMessage = { $like: `${STALE_SWEEP_ERROR_PREFIX}%` }
160
+ const affected = await targetEm.nativeUpdate(ProgressJob, filter as FilterQuery<ProgressJob>, {
161
+ status: 'running',
162
+ startedAt,
163
+ heartbeatAt: now,
164
+ finishedAt: null,
165
+ errorMessage: null,
166
+ errorStack: null,
167
+ updatedAt: now,
168
+ })
169
+ if (affected === 0) return null
170
+
171
+ job.status = 'running'
172
+ job.startedAt = startedAt
173
+ job.heartbeatAt = now
174
+ job.finishedAt = null
175
+ job.errorMessage = null
176
+ job.errorStack = null
177
+
178
+ await eventBus.emit(PROGRESS_EVENTS.JOB_STARTED, {
179
+ ...buildJobPayload(job),
180
+ tenantId: ctx.tenantId,
181
+ organizationId: job.organizationId ?? null,
182
+ })
183
+
184
+ return job
185
+ }
186
+
134
187
  async function persistAndMaybeBroadcast(entry: JobUpdateThrottleEntry, ctx: ProgressServiceContext): Promise<ProgressJob> {
135
188
  const job = entry.job
136
189
  const now = Date.now()
@@ -153,16 +206,32 @@ export function createProgressService(em: EntityManager, eventBus: { emit: (even
153
206
  updatedAt: new Date(),
154
207
  ...buildBufferedCountData(entry),
155
208
  }
156
- const affected = await em.nativeUpdate(ProgressJob, {
209
+ const writableFilter = {
157
210
  ...jobScopeFilter(job.id, ctx),
158
211
  status: { $in: PROGRESS_WRITABLE_STATUSES as ProgressJobStatus[] },
159
- } as FilterQuery<ProgressJob>, data)
212
+ } as FilterQuery<ProgressJob>
213
+ let affected = await em.nativeUpdate(ProgressJob, writableFilter, data)
160
214
 
161
215
  if (affected === 0) {
162
- // The job reached a terminal state in another process; stop writing to it.
163
- forgetJobThrottle(job.id)
216
+ // A stale sweep may have flipped a live job to `failed` between writes. Revive it
217
+ // through the start CAS and retry once so the buffered delta isn't silently dropped.
164
218
  const fresh = await loadFreshJob(job.id, ctx)
165
- return fresh ?? job
219
+ let revived = false
220
+ if (fresh && fresh.status === 'failed' && (await startJobViaCas(em, fresh, ctx, { staleSweptOnly: true }))) {
221
+ revived = true
222
+ job.status = 'running'
223
+ job.startedAt = fresh.startedAt
224
+ job.finishedAt = null
225
+ job.errorMessage = null
226
+ job.errorStack = null
227
+ affected = await em.nativeUpdate(ProgressJob, writableFilter, data)
228
+ }
229
+ if (affected === 0) {
230
+ // The job reached a genuinely terminal state in another process; stop writing to it.
231
+ forgetJobThrottle(job.id)
232
+ const latest = revived ? await loadFreshJob(job.id, ctx) : fresh
233
+ return latest ?? job
234
+ }
166
235
  }
167
236
 
168
237
  const persistedJob = usesAtomicIncrement ? (await loadFreshJob(job.id, ctx)) ?? job : job
@@ -219,47 +288,48 @@ export function createProgressService(em: EntityManager, eventBus: { emit: (even
219
288
  return job
220
289
  }
221
290
 
291
+ const started = await startJobViaCas(em, job, ctx)
292
+ if (started) return started
293
+
294
+ const fresh = await loadFreshJob(jobId, ctx)
295
+ return fresh ?? job
296
+ },
297
+
298
+ async touchJobHeartbeat(jobId, ctx) {
299
+ // Forked EM: keepalive timers call this while the shared EM may be mid-transaction
300
+ // inside a producer's own writes, so the heartbeat must not join that unit of work.
301
+ const fork = em.fork()
222
302
  const now = new Date()
223
- const affected = await em.nativeUpdate(ProgressJob, {
303
+ const affected = await fork.nativeUpdate(ProgressJob, {
224
304
  ...jobScopeFilter(jobId, ctx),
225
- status: { $in: START_FROM_STATUSES as ProgressJobStatus[] },
305
+ status: { $in: PROGRESS_WRITABLE_STATUSES as ProgressJobStatus[] },
226
306
  } as FilterQuery<ProgressJob>, {
227
- status: 'running',
228
- startedAt: now,
229
307
  heartbeatAt: now,
230
- finishedAt: null,
231
- errorMessage: null,
232
- errorStack: null,
233
308
  updatedAt: now,
234
309
  })
310
+ if (affected > 0) return
235
311
 
236
- if (affected === 0) {
237
- const fresh = await loadFreshJob(jobId, ctx)
238
- return fresh ?? job
239
- }
240
-
241
- job.status = 'running'
242
- job.startedAt = now
243
- job.heartbeatAt = now
244
- job.finishedAt = null
245
- job.errorMessage = null
246
- job.errorStack = null
247
-
248
- await eventBus.emit(PROGRESS_EVENTS.JOB_STARTED, {
249
- ...buildJobPayload(job),
250
- tenantId: ctx.tenantId,
251
- organizationId: job.organizationId ?? null,
252
- })
253
-
254
- return job
312
+ const job = await fork.findOne(ProgressJob, jobScopeFilter(jobId, ctx), { disableIdentityMap: true })
313
+ if (!job || job.status !== 'failed') return
314
+ // A live producer heartbeating a `failed` job means a stale sweep false-positived;
315
+ // revive through the start CAS so the card recovers without waiting for a batch boundary.
316
+ await startJobViaCas(fork, job, ctx, { staleSweptOnly: true })
255
317
  },
256
318
 
257
319
  async updateProgress(jobId, input, ctx) {
258
320
  const entry = await ensureThrottleEntry(jobId, ctx)
259
321
  const job = entry.job
260
322
  if (TERMINAL_STATUSES.includes(job.status)) {
261
- forgetJobThrottle(jobId)
262
- return job
323
+ // `failed` may be a false-positive stale sweep on a slow-but-alive producer; the
324
+ // fact that this write is happening proves the producer is alive, so revive and
325
+ // apply the update. `completed`/`cancelled` stay terminal, and a failure the sweep
326
+ // did not write keeps its diagnostics (see startJobViaCas).
327
+ const revived = job.status === 'failed' && (await startJobViaCas(em, job, ctx, { staleSweptOnly: true }))
328
+ if (!revived) {
329
+ forgetJobThrottle(jobId)
330
+ return job
331
+ }
332
+ entry.lastPersistedAt = 0
263
333
  }
264
334
 
265
335
  if (input.processedCount !== undefined) {
@@ -295,8 +365,12 @@ export function createProgressService(em: EntityManager, eventBus: { emit: (even
295
365
  const entry = await ensureThrottleEntry(jobId, ctx)
296
366
  const job = entry.job
297
367
  if (TERMINAL_STATUSES.includes(job.status)) {
298
- forgetJobThrottle(jobId)
299
- return job
368
+ const revived = job.status === 'failed' && (await startJobViaCas(em, job, ctx, { staleSweptOnly: true }))
369
+ if (!revived) {
370
+ forgetJobThrottle(jobId)
371
+ return job
372
+ }
373
+ entry.lastPersistedAt = 0
300
374
  }
301
375
 
302
376
  job.processedCount += delta
@@ -604,7 +678,7 @@ export function createProgressService(em: EntityManager, eventBus: { emit: (even
604
678
  const staleRunning = await em.find(ProgressJob, { ...scope, ...staleFilter }, { disableIdentityMap: true })
605
679
 
606
680
  for (const job of staleRunning) {
607
- const errorMessage = `Job stale: no heartbeat for ${timeoutSeconds} seconds`
681
+ const errorMessage = `${STALE_SWEEP_ERROR_PREFIX} no heartbeat for ${timeoutSeconds} seconds`
608
682
  // Per-row CAS (re-checking the staleness condition): exactly one concurrent
609
683
  // sweeper wins, and a heartbeat that landed after the SELECT aborts the failure.
610
684
  const affected = await em.nativeUpdate(ProgressJob, {
@@ -644,7 +718,7 @@ export function createProgressService(em: EntityManager, eventBus: { emit: (even
644
718
  const stalePending = await em.find(ProgressJob, { ...scope, ...stalePendingFilter }, { disableIdentityMap: true })
645
719
 
646
720
  for (const job of stalePending) {
647
- const errorMessage = `Job stale: never started within ${STALE_PENDING_TIMEOUT_SECONDS} seconds`
721
+ const errorMessage = `${STALE_SWEEP_ERROR_PREFIX} never started within ${STALE_PENDING_TIMEOUT_SECONDS} seconds`
648
722
  const affected = await em.nativeUpdate(ProgressJob, {
649
723
  id: job.id,
650
724
  tenantId: job.tenantId,