@open-mercato/core 0.6.8-develop.6986.1.3adb0d0df6 → 0.6.8-develop.6987.1.02f619470c
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/modules/data_sync/lib/sync-engine.js +79 -20
- package/dist/modules/data_sync/lib/sync-engine.js.map +2 -2
- package/dist/modules/progress/lib/progressService.js +2 -0
- package/dist/modules/progress/lib/progressService.js.map +2 -2
- package/dist/modules/progress/lib/progressServiceImpl.js +78 -34
- package/dist/modules/progress/lib/progressServiceImpl.js.map +2 -2
- package/package.json +7 -7
- package/src/modules/data_sync/AGENTS.md +2 -0
- package/src/modules/data_sync/lib/sync-engine.ts +96 -20
- package/src/modules/progress/AGENTS.md +2 -1
- package/src/modules/progress/lib/progressService.ts +9 -0
- package/src/modules/progress/lib/progressServiceImpl.ts +111 -37
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { getIntegration } from "@open-mercato/shared/modules/integrations/types";
|
|
2
|
+
import { STALE_JOB_TIMEOUT_SECONDS } from "../../progress/lib/progressService.js";
|
|
2
3
|
import { refreshCoverageSnapshot } from "../../query_index/lib/coverage.js";
|
|
3
4
|
import { emitDataSyncEvent } from "../events.js";
|
|
4
5
|
import { getDataSyncAdapter } from "./adapter-registry.js";
|
|
@@ -37,6 +38,25 @@ function applyExportCounters(batch) {
|
|
|
37
38
|
processedCount: batch.results.length
|
|
38
39
|
};
|
|
39
40
|
}
|
|
41
|
+
const HEARTBEAT_TICK_MS = STALE_JOB_TIMEOUT_SECONDS * 1e3 / 4;
|
|
42
|
+
async function* withHeartbeat(source, tick, intervalMs) {
|
|
43
|
+
const iterator = source[Symbol.asyncIterator]();
|
|
44
|
+
try {
|
|
45
|
+
while (true) {
|
|
46
|
+
const timer = setInterval(tick, intervalMs);
|
|
47
|
+
let result;
|
|
48
|
+
try {
|
|
49
|
+
result = await iterator.next();
|
|
50
|
+
} finally {
|
|
51
|
+
clearInterval(timer);
|
|
52
|
+
}
|
|
53
|
+
if (result.done) return;
|
|
54
|
+
yield result.value;
|
|
55
|
+
}
|
|
56
|
+
} finally {
|
|
57
|
+
await iterator.return?.();
|
|
58
|
+
}
|
|
59
|
+
}
|
|
40
60
|
function createSyncEngine(deps) {
|
|
41
61
|
const { syncRunService, integrationCredentialsService, integrationLogService, integrationStateService, progressService } = deps;
|
|
42
62
|
async function resolveMapping(adapter, entityType, scope) {
|
|
@@ -60,6 +80,37 @@ function createSyncEngine(deps) {
|
|
|
60
80
|
}
|
|
61
81
|
);
|
|
62
82
|
}
|
|
83
|
+
async function seedProcessedCount(progressJobId, scope) {
|
|
84
|
+
if (!progressJobId) return 0;
|
|
85
|
+
const job = await progressService.getJob(progressJobId, {
|
|
86
|
+
tenantId: scope.tenantId,
|
|
87
|
+
organizationId: scope.organizationId,
|
|
88
|
+
userId: scope.userId
|
|
89
|
+
});
|
|
90
|
+
return job?.processedCount ?? 0;
|
|
91
|
+
}
|
|
92
|
+
function makeHeartbeatTick(progressJobId, scope) {
|
|
93
|
+
const touchJobHeartbeat = progressService.touchJobHeartbeat?.bind(progressService);
|
|
94
|
+
if (!progressJobId || !touchJobHeartbeat) return () => {
|
|
95
|
+
};
|
|
96
|
+
let inFlight = false;
|
|
97
|
+
return () => {
|
|
98
|
+
if (inFlight) return;
|
|
99
|
+
inFlight = true;
|
|
100
|
+
touchJobHeartbeat(progressJobId, {
|
|
101
|
+
tenantId: scope.tenantId,
|
|
102
|
+
organizationId: scope.organizationId,
|
|
103
|
+
userId: scope.userId
|
|
104
|
+
}).catch((error) => {
|
|
105
|
+
logger.warn("Progress heartbeat failed", {
|
|
106
|
+
progressJobId,
|
|
107
|
+
error: error instanceof Error ? error.message : String(error)
|
|
108
|
+
});
|
|
109
|
+
}).finally(() => {
|
|
110
|
+
inFlight = false;
|
|
111
|
+
});
|
|
112
|
+
};
|
|
113
|
+
}
|
|
63
114
|
async function refreshCoverageSnapshots(entityTypes, scope) {
|
|
64
115
|
if (!entityTypes || entityTypes.length === 0) return;
|
|
65
116
|
await Promise.allSettled(
|
|
@@ -358,19 +409,23 @@ function createSyncEngine(deps) {
|
|
|
358
409
|
});
|
|
359
410
|
}
|
|
360
411
|
const mapping = await resolveMapping(adapter, run.entityType, scope);
|
|
361
|
-
let processedCount =
|
|
412
|
+
let processedCount = await seedProcessedCount(run.progressJobId, scope);
|
|
362
413
|
let totalCount = null;
|
|
363
414
|
let committedBatches = activeRun.batchesCompleted ?? 0;
|
|
364
415
|
try {
|
|
365
|
-
for await (const batch of
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
416
|
+
for await (const batch of withHeartbeat(
|
|
417
|
+
adapter.streamImport({
|
|
418
|
+
entityType: run.entityType,
|
|
419
|
+
cursor: run.cursor ?? void 0,
|
|
420
|
+
batchSize,
|
|
421
|
+
credentials,
|
|
422
|
+
mapping,
|
|
423
|
+
scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },
|
|
424
|
+
runId: run.id
|
|
425
|
+
}),
|
|
426
|
+
makeHeartbeatTick(run.progressJobId, scope),
|
|
427
|
+
HEARTBEAT_TICK_MS
|
|
428
|
+
)) {
|
|
374
429
|
if (run.progressJobId && await progressService.isCancellationRequested(run.progressJobId, scope.tenantId, scope.organizationId)) {
|
|
375
430
|
await finalizeRun(run.id, "cancelled", scope, void 0, operationalTelemetry);
|
|
376
431
|
return;
|
|
@@ -501,18 +556,22 @@ function createSyncEngine(deps) {
|
|
|
501
556
|
});
|
|
502
557
|
}
|
|
503
558
|
const mapping = await resolveMapping(adapter, run.entityType, scope);
|
|
504
|
-
let processedCount =
|
|
559
|
+
let processedCount = await seedProcessedCount(run.progressJobId, scope);
|
|
505
560
|
let committedBatches = activeRun.batchesCompleted ?? 0;
|
|
506
561
|
try {
|
|
507
|
-
for await (const batch of
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
562
|
+
for await (const batch of withHeartbeat(
|
|
563
|
+
adapter.streamExport({
|
|
564
|
+
entityType: run.entityType,
|
|
565
|
+
cursor: run.cursor ?? void 0,
|
|
566
|
+
batchSize,
|
|
567
|
+
credentials,
|
|
568
|
+
mapping,
|
|
569
|
+
scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },
|
|
570
|
+
runId: run.id
|
|
571
|
+
}),
|
|
572
|
+
makeHeartbeatTick(run.progressJobId, scope),
|
|
573
|
+
HEARTBEAT_TICK_MS
|
|
574
|
+
)) {
|
|
516
575
|
if (run.progressJobId && await progressService.isCancellationRequested(run.progressJobId, scope.tenantId, scope.organizationId)) {
|
|
517
576
|
await finalizeRun(run.id, "cancelled", scope, void 0, operationalTelemetry);
|
|
518
577
|
return;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../../src/modules/data_sync/lib/sync-engine.ts"],
|
|
4
|
-
"sourcesContent": ["import type { EntityManager } from '@mikro-orm/postgresql'\nimport { getIntegration } from '@open-mercato/shared/modules/integrations/types'\nimport type { CredentialsService } from '../../integrations/lib/credentials-service'\nimport type { IntegrationLogService } from '../../integrations/lib/log-service'\nimport type { IntegrationStateService } from '../../integrations/lib/state-service'\nimport type { ProgressService } from '../../progress/lib/progressService'\nimport { refreshCoverageSnapshot } from '../../query_index/lib/coverage'\nimport { emitDataSyncEvent } from '../events'\nimport type { DataSyncAdapter, DataMapping, ExportBatch, ImportBatch } from './adapter'\nimport { getDataSyncAdapter } from './adapter-registry'\nimport type { SyncRunService } from './sync-run-service'\nimport { SyncRunOwnershipConflictError } from './sync-run-service'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\n\nconst logger = createLogger('data_sync').child({ component: 'sync-engine' })\n\ntype SyncScope = {\n organizationId: string\n tenantId: string\n userId?: string | null\n}\n\ntype EngineDeps = {\n em: EntityManager\n syncRunService: SyncRunService\n integrationCredentialsService: CredentialsService\n integrationLogService: IntegrationLogService\n integrationStateService?: IntegrationStateService\n progressService: ProgressService\n}\n\nfunction resolveProviderKey(integrationId: string): string {\n return getIntegration(integrationId)?.providerKey ?? integrationId\n}\n\nfunction applyImportCounters(batch: ImportBatch): Pick<Required<SyncCounterDelta>, 'createdCount' | 'updatedCount' | 'skippedCount' | 'failedCount'> {\n let createdCount = 0\n let updatedCount = 0\n let skippedCount = 0\n let failedCount = 0\n\n for (const item of batch.items) {\n if (item.action === 'create') createdCount += 1\n else if (item.action === 'update') updatedCount += 1\n else if (item.action === 'failed') failedCount += 1\n else skippedCount += 1\n }\n\n return { createdCount, updatedCount, skippedCount, failedCount }\n}\n\ntype SyncCounterDelta = {\n createdCount?: number\n updatedCount?: number\n skippedCount?: number\n failedCount?: number\n processedCount: number\n}\n\nfunction applyExportCounters(batch: ExportBatch): SyncCounterDelta {\n let failedCount = 0\n let skippedCount = 0\n let updatedCount = 0\n\n for (const result of batch.results) {\n if (result.status === 'error') failedCount += 1\n else if (result.status === 'skipped') skippedCount += 1\n else updatedCount += 1\n }\n\n return {\n failedCount,\n skippedCount,\n updatedCount,\n processedCount: batch.results.length,\n }\n}\n\nexport function createSyncEngine(deps: EngineDeps) {\n const { syncRunService, integrationCredentialsService, integrationLogService, integrationStateService, progressService } = deps\n\n async function resolveMapping(adapter: DataSyncAdapter, entityType: string, scope: SyncScope): Promise<DataMapping> {\n return adapter.getMapping({\n entityType,\n scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },\n })\n }\n\n async function updateProgress(progressJobId: string | null | undefined, processedCount: number, totalCount: number | null, scope: SyncScope): Promise<void> {\n if (!progressJobId) return\n\n await progressService.updateProgress(\n progressJobId,\n {\n processedCount,\n totalCount: totalCount ?? undefined,\n },\n {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n },\n )\n }\n\n async function refreshCoverageSnapshots(entityTypes: string[] | undefined, scope: SyncScope): Promise<void> {\n if (!entityTypes || entityTypes.length === 0) return\n\n await Promise.allSettled(\n Array.from(new Set(entityTypes.filter((value) => typeof value === 'string' && value.trim().length > 0)))\n .map((entityType) => refreshCoverageSnapshot(deps.em, {\n entityType,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n })),\n )\n }\n\n async function logImportItemFailures(\n runId: string,\n integrationId: string,\n items: ImportBatch['items'],\n scope: SyncScope,\n ): Promise<void> {\n const failedItems = items.filter((item) => item.action === 'failed')\n for (const item of failedItems) {\n const errorMessage = typeof item.data.errorMessage === 'string' && item.data.errorMessage.trim().length > 0\n ? item.data.errorMessage.trim()\n : 'Import item failed'\n const sourceProductUuid = typeof item.data.sourceProductUuid === 'string' && item.data.sourceProductUuid.trim().length > 0\n ? item.data.sourceProductUuid.trim()\n : null\n const sourceIdentifier = typeof item.data.sourceIdentifier === 'string' && item.data.sourceIdentifier.trim().length > 0\n ? item.data.sourceIdentifier.trim()\n : null\n const message = [\n `Failed to import item ${item.externalId}`,\n sourceProductUuid ? `(uuid: ${sourceProductUuid})` : null,\n sourceIdentifier ? `(identifier: ${sourceIdentifier})` : null,\n `: ${errorMessage}`,\n ].filter((part) => part !== null).join(' ')\n\n await integrationLogService.write(\n {\n integrationId,\n runId,\n level: 'error',\n message,\n payload: item.data,\n },\n scope,\n )\n }\n }\n\n async function logExportItemFailures(\n runId: string,\n integrationId: string,\n results: ExportBatch['results'],\n scope: SyncScope,\n ): Promise<void> {\n const failedResults = results.filter((result) => result.status === 'error' && result.error)\n for (const result of failedResults) {\n const label = result.externalId ? `${result.externalId} (id: ${result.localId})` : result.localId\n const errorMessage = result.error!.split('\\n')[0]\n const message = `Failed to export item ${label}: ${errorMessage}`\n\n await integrationLogService.write(\n {\n integrationId,\n runId,\n level: 'error',\n message,\n payload: { kind: 'export-item-failure', summary: result.error },\n },\n scope,\n )\n }\n }\n\n async function writeOperationalLog(params: {\n integrationId: string\n runId: string\n level: 'info' | 'warn' | 'error'\n message: string\n scope: SyncScope\n enabled: boolean\n payload?: Record<string, unknown>\n }): Promise<void> {\n if (!params.enabled) return\n\n await integrationLogService.write(\n {\n integrationId: params.integrationId,\n runId: params.runId,\n level: params.level,\n message: params.message,\n payload: params.payload,\n },\n params.scope,\n )\n }\n\n async function updateOperationalState(params: {\n integrationId: string\n status: 'healthy' | 'degraded' | 'unhealthy'\n scope: SyncScope\n enabled: boolean\n }): Promise<void> {\n if (!params.enabled || !integrationStateService) return\n\n await integrationStateService.upsert(\n params.integrationId,\n {\n lastHealthStatus: params.status,\n lastHealthCheckedAt: new Date(),\n },\n params.scope,\n )\n }\n\n async function finalizeRun(\n runId: string,\n status: 'completed' | 'failed' | 'cancelled',\n scope: SyncScope,\n error?: string,\n operationalTelemetry = false,\n ): Promise<void> {\n const existingRun = await syncRunService.getRun(runId, scope)\n const alreadyFinalizedWithSameStatus = existingRun?.status === status\n && (status === 'completed' || status === 'failed' || status === 'cancelled')\n\n const run = await syncRunService.markStatus(runId, status, scope, error)\n if (!run) return\n\n if (alreadyFinalizedWithSameStatus) {\n return\n }\n\n if (run.status !== status) {\n // `markStatus` refuses a terminal -> different-terminal transition and\n // returns the row unchanged, so the run is already finished under another\n // delivery of this job. Everything below \u2014 the progress job, the\n // operational log and the lifecycle event \u2014 would describe the wrong\n // outcome, and `data_sync.run.failed` is dispatched to tenant webhooks.\n // A displaced worker stays silent instead.\n logger.warn('Skipping finalization of a sync run another worker already finalized', {\n runId,\n requestedStatus: status,\n actualStatus: run.status,\n })\n return\n }\n\n if (run.progressJobId) {\n if (status === 'completed') {\n await progressService.completeJob(\n run.progressJobId,\n {\n resultSummary: {\n createdCount: run.createdCount,\n updatedCount: run.updatedCount,\n skippedCount: run.skippedCount,\n failedCount: run.failedCount,\n batchesCompleted: run.batchesCompleted,\n },\n },\n {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n },\n )\n } else if (status === 'failed') {\n await progressService.failJob(\n run.progressJobId,\n {\n errorMessage: error ?? 'Sync run failed',\n },\n {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n },\n )\n } else if (status === 'cancelled') {\n await progressService.markCancelled(\n run.progressJobId,\n {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n },\n )\n }\n }\n\n if (status === 'completed') {\n await updateOperationalState({\n integrationId: run.integrationId,\n status: 'healthy',\n scope,\n enabled: operationalTelemetry,\n })\n await writeOperationalLog({\n integrationId: run.integrationId,\n runId: run.id,\n level: 'info',\n message: 'Sync run completed',\n scope,\n enabled: operationalTelemetry,\n payload: {\n operationalStatus: 'completed',\n summary: `Sync completed with ${run.createdCount} created, ${run.updatedCount} updated, ${run.failedCount} failed.`,\n createdCount: run.createdCount,\n updatedCount: run.updatedCount,\n skippedCount: run.skippedCount,\n failedCount: run.failedCount,\n batchesCompleted: run.batchesCompleted,\n },\n })\n } else if (status === 'cancelled') {\n await updateOperationalState({\n integrationId: run.integrationId,\n status: 'degraded',\n scope,\n enabled: operationalTelemetry,\n })\n await writeOperationalLog({\n integrationId: run.integrationId,\n runId: run.id,\n level: 'warn',\n message: 'Sync run cancelled',\n scope,\n enabled: operationalTelemetry,\n payload: {\n operationalStatus: 'cancelled',\n summary: 'The sync run was cancelled before completion.',\n },\n })\n } else {\n await updateOperationalState({\n integrationId: run.integrationId,\n status: 'unhealthy',\n scope,\n enabled: operationalTelemetry,\n })\n await writeOperationalLog({\n integrationId: run.integrationId,\n runId: run.id,\n level: 'error',\n message: error ?? 'Sync run failed',\n scope,\n enabled: operationalTelemetry,\n payload: {\n operationalStatus: 'failed',\n summary: error ?? 'The sync run failed.',\n },\n })\n }\n\n if (status === 'completed') {\n await emitDataSyncEvent('data_sync.run.completed', {\n runId,\n integrationId: run.integrationId,\n entityType: run.entityType,\n direction: run.direction,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n })\n return\n }\n\n if (status === 'cancelled') {\n await emitDataSyncEvent('data_sync.run.cancelled', {\n runId,\n integrationId: run.integrationId,\n entityType: run.entityType,\n direction: run.direction,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n })\n return\n }\n\n await emitDataSyncEvent('data_sync.run.failed', {\n runId,\n integrationId: run.integrationId,\n entityType: run.entityType,\n direction: run.direction,\n error: error ?? null,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n })\n }\n\n return {\n async runImport(runId: string, batchSize: number, scope: SyncScope): Promise<void> {\n const run = await syncRunService.getRun(runId, scope)\n if (!run) {\n logger.warn('Skipping stale import job for missing run', { runId })\n return\n }\n if (run.status === 'cancelled') {\n if (run.progressJobId) {\n await progressService.markCancelled(run.progressJobId, {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n })\n }\n return\n }\n\n const providerKey = resolveProviderKey(run.integrationId)\n const adapter = getDataSyncAdapter(providerKey)\n if (!adapter?.streamImport) {\n throw new Error(`No import adapter registered for provider ${providerKey}`)\n }\n const operationalTelemetry = adapter.operationalTelemetry === true\n\n const credentials = await integrationCredentialsService.resolve(run.integrationId, scope)\n if (!credentials) {\n throw new Error(`Integration ${run.integrationId} is missing credentials`)\n }\n\n // A run already `running` means a stalled job was redelivered, so this is\n // a resume rather than a first start. Consumers see one `started` event per\n // delivery either way; the flag is what lets them tell the two apart.\n const resumed = run.status === 'running'\n const activeRun = await syncRunService.markStatus(run.id, 'running', scope)\n if (!activeRun || activeRun.status !== 'running') {\n return\n }\n await emitDataSyncEvent('data_sync.run.started', {\n runId: run.id,\n integrationId: run.integrationId,\n entityType: run.entityType,\n direction: run.direction,\n resumed,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n })\n await updateOperationalState({\n integrationId: run.integrationId,\n status: 'degraded',\n scope,\n enabled: operationalTelemetry,\n })\n await writeOperationalLog({\n integrationId: run.integrationId,\n runId: run.id,\n level: 'info',\n message: 'Sync run started',\n scope,\n enabled: operationalTelemetry,\n payload: {\n operationalStatus: 'running',\n summary: `Import run started for ${run.entityType}.`,\n entityType: run.entityType,\n direction: run.direction,\n },\n })\n\n if (run.progressJobId) {\n await progressService.startJob(run.progressJobId, {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n })\n }\n\n const mapping = await resolveMapping(adapter, run.entityType, scope)\n let processedCount = 0\n let totalCount: number | null = null\n let committedBatches = activeRun.batchesCompleted ?? 0\n\n try {\n for await (const batch of adapter.streamImport({\n entityType: run.entityType,\n cursor: run.cursor ?? undefined,\n batchSize,\n credentials,\n mapping,\n scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },\n runId: run.id,\n })) {\n if (run.progressJobId && await progressService.isCancellationRequested(run.progressJobId, scope.tenantId, scope.organizationId)) {\n await finalizeRun(run.id, 'cancelled', scope, undefined, operationalTelemetry)\n return\n }\n\n const delta = applyImportCounters(batch)\n const processedBatchCount = batch.processedCount ?? batch.items.length\n processedCount += processedBatchCount\n totalCount = batch.totalEstimate ?? totalCount\n\n await syncRunService.commitBatchProgress(\n run.id,\n {\n ...delta,\n batchesCompleted: 1,\n },\n batch.cursor,\n scope,\n committedBatches,\n )\n committedBatches += 1\n\n await updateProgress(run.progressJobId, processedCount, totalCount, scope)\n await refreshCoverageSnapshots(batch.refreshCoverageEntityTypes, scope)\n await logImportItemFailures(run.id, run.integrationId, batch.items, scope)\n\n await writeOperationalLog({\n integrationId: run.integrationId,\n runId: run.id,\n level: 'info',\n message: batch.message?.trim().length\n ? batch.message.trim()\n : `Processed import batch ${batch.batchIndex}`,\n scope,\n enabled: operationalTelemetry,\n payload: {\n operationalStatus: 'running',\n summary: `Processed ${processedCount}${totalCount ? ` of ${totalCount}` : ''} rows so far.`,\n processedCount,\n batchSize: batch.items.length,\n processedBatchCount,\n cursor: batch.cursor,\n },\n })\n }\n } catch (error) {\n if (error instanceof SyncRunOwnershipConflictError) {\n logger.warn('Yielding import run to a concurrent worker that already advanced it', {\n runId: run.id,\n expectedBatchesCompleted: error.expectedBatchesCompleted,\n })\n return\n }\n const message = error instanceof Error ? error.message : 'Sync import failed'\n await integrationLogService.write(\n {\n integrationId: run.integrationId,\n runId: run.id,\n level: 'error',\n message,\n },\n scope,\n )\n await finalizeRun(run.id, 'failed', scope, message, operationalTelemetry)\n return\n }\n\n await finalizeRun(run.id, 'completed', scope, undefined, operationalTelemetry)\n },\n\n async runExport(runId: string, batchSize: number, scope: SyncScope): Promise<void> {\n const run = await syncRunService.getRun(runId, scope)\n if (!run) {\n logger.warn('Skipping stale export job for missing run', { runId })\n return\n }\n if (run.status === 'cancelled') {\n if (run.progressJobId) {\n await progressService.markCancelled(run.progressJobId, {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n })\n }\n return\n }\n\n const providerKey = resolveProviderKey(run.integrationId)\n const adapter = getDataSyncAdapter(providerKey)\n if (!adapter?.streamExport) {\n throw new Error(`No export adapter registered for provider ${providerKey}`)\n }\n const operationalTelemetry = adapter.operationalTelemetry === true\n\n const credentials = await integrationCredentialsService.resolve(run.integrationId, scope)\n if (!credentials) {\n throw new Error(`Integration ${run.integrationId} is missing credentials`)\n }\n\n // A run already `running` means a stalled job was redelivered, so this is\n // a resume rather than a first start. Consumers see one `started` event per\n // delivery either way; the flag is what lets them tell the two apart.\n const resumed = run.status === 'running'\n const activeRun = await syncRunService.markStatus(run.id, 'running', scope)\n if (!activeRun || activeRun.status !== 'running') {\n return\n }\n await emitDataSyncEvent('data_sync.run.started', {\n runId: run.id,\n integrationId: run.integrationId,\n entityType: run.entityType,\n direction: run.direction,\n resumed,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n })\n await updateOperationalState({\n integrationId: run.integrationId,\n status: 'degraded',\n scope,\n enabled: operationalTelemetry,\n })\n await writeOperationalLog({\n integrationId: run.integrationId,\n runId: run.id,\n level: 'info',\n message: 'Sync run started',\n scope,\n enabled: operationalTelemetry,\n payload: {\n operationalStatus: 'running',\n summary: `Export run started for ${run.entityType}.`,\n entityType: run.entityType,\n direction: run.direction,\n },\n })\n\n if (run.progressJobId) {\n await progressService.startJob(run.progressJobId, {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n })\n }\n\n const mapping = await resolveMapping(adapter, run.entityType, scope)\n let processedCount = 0\n let committedBatches = activeRun.batchesCompleted ?? 0\n\n try {\n for await (const batch of adapter.streamExport({\n entityType: run.entityType,\n cursor: run.cursor ?? undefined,\n batchSize,\n credentials,\n mapping,\n scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },\n runId: run.id,\n })) {\n if (run.progressJobId && await progressService.isCancellationRequested(run.progressJobId, scope.tenantId, scope.organizationId)) {\n await finalizeRun(run.id, 'cancelled', scope, undefined, operationalTelemetry)\n return\n }\n\n const delta = applyExportCounters(batch)\n processedCount += delta.processedCount\n\n await syncRunService.commitBatchProgress(\n run.id,\n {\n createdCount: 0,\n updatedCount: delta.updatedCount,\n skippedCount: delta.skippedCount,\n failedCount: delta.failedCount,\n batchesCompleted: 1,\n },\n batch.cursor,\n scope,\n committedBatches,\n )\n committedBatches += 1\n await updateProgress(run.progressJobId, processedCount, null, scope)\n await logExportItemFailures(run.id, run.integrationId, batch.results, scope)\n\n await writeOperationalLog({\n integrationId: run.integrationId,\n runId: run.id,\n level: 'info',\n message: `Processed export batch ${batch.batchIndex}`,\n scope,\n enabled: operationalTelemetry,\n payload: {\n operationalStatus: 'running',\n summary: `Processed ${processedCount} export items so far.`,\n processedCount,\n batchSize: batch.results.length,\n cursor: batch.cursor,\n },\n })\n }\n } catch (error) {\n if (error instanceof SyncRunOwnershipConflictError) {\n logger.warn('Yielding export run to a concurrent worker that already advanced it', {\n runId: run.id,\n expectedBatchesCompleted: error.expectedBatchesCompleted,\n })\n return\n }\n const message = error instanceof Error ? error.message : 'Sync export failed'\n await integrationLogService.write(\n {\n integrationId: run.integrationId,\n runId: run.id,\n level: 'error',\n message,\n },\n scope,\n )\n await finalizeRun(run.id, 'failed', scope, message, operationalTelemetry)\n return\n }\n\n await finalizeRun(run.id, 'completed', scope, undefined, operationalTelemetry)\n },\n }\n}\n\nexport type SyncEngine = ReturnType<typeof createSyncEngine>\n"],
|
|
5
|
-
"mappings": "AACA,SAAS,sBAAsB;AAK/B,SAAS,+BAA+B;AACxC,SAAS,yBAAyB;AAElC,SAAS,0BAA0B;AAEnC,SAAS,qCAAqC;AAC9C,SAAS,oBAAoB;AAE7B,MAAM,SAAS,aAAa,WAAW,EAAE,MAAM,EAAE,WAAW,cAAc,CAAC;AAiB3E,SAAS,mBAAmB,eAA+B;AACzD,SAAO,eAAe,aAAa,GAAG,eAAe;AACvD;AAEA,SAAS,oBAAoB,OAAwH;AACnJ,MAAI,eAAe;AACnB,MAAI,eAAe;AACnB,MAAI,eAAe;AACnB,MAAI,cAAc;AAElB,aAAW,QAAQ,MAAM,OAAO;AAC9B,QAAI,KAAK,WAAW,SAAU,iBAAgB;AAAA,aACrC,KAAK,WAAW,SAAU,iBAAgB;AAAA,aAC1C,KAAK,WAAW,SAAU,gBAAe;AAAA,QAC7C,iBAAgB;AAAA,EACvB;AAEA,SAAO,EAAE,cAAc,cAAc,cAAc,YAAY;AACjE;AAUA,SAAS,oBAAoB,OAAsC;AACjE,MAAI,cAAc;AAClB,MAAI,eAAe;AACnB,MAAI,eAAe;AAEnB,aAAW,UAAU,MAAM,SAAS;AAClC,QAAI,OAAO,WAAW,QAAS,gBAAe;AAAA,aACrC,OAAO,WAAW,UAAW,iBAAgB;AAAA,QACjD,iBAAgB;AAAA,EACvB;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA,gBAAgB,MAAM,QAAQ;AAAA,EAChC;AACF;AAEO,SAAS,iBAAiB,MAAkB;AACjD,QAAM,EAAE,gBAAgB,+BAA+B,uBAAuB,yBAAyB,gBAAgB,IAAI;AAE3H,iBAAe,eAAe,SAA0B,YAAoB,OAAwC;AAClH,WAAO,QAAQ,WAAW;AAAA,MACxB;AAAA,MACA,OAAO,EAAE,gBAAgB,MAAM,gBAAgB,UAAU,MAAM,SAAS;AAAA,IAC1E,CAAC;AAAA,EACH;AAEA,iBAAe,eAAe,eAA0C,gBAAwB,YAA2B,OAAiC;AAC1J,QAAI,CAAC,cAAe;AAEpB,UAAM,gBAAgB;AAAA,MACpB;AAAA,MACA;AAAA,QACE;AAAA,QACA,YAAY,cAAc;AAAA,MAC5B;AAAA,MACA;AAAA,QACE,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,QACtB,QAAQ,MAAM;AAAA,MAChB;AAAA,IACF;AAAA,EACF;AAEA,iBAAe,yBAAyB,aAAmC,OAAiC;AAC1G,QAAI,CAAC,eAAe,YAAY,WAAW,EAAG;AAE9C,UAAM,QAAQ;AAAA,MACZ,MAAM,KAAK,IAAI,IAAI,YAAY,OAAO,CAAC,UAAU,OAAO,UAAU,YAAY,MAAM,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,EACpG,IAAI,CAAC,eAAe,wBAAwB,KAAK,IAAI;AAAA,QACpD;AAAA,QACA,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,MACxB,CAAC,CAAC;AAAA,IACN;AAAA,EACF;AAEA,iBAAe,sBACb,OACA,eACA,OACA,OACe;AACf,UAAM,cAAc,MAAM,OAAO,CAAC,SAAS,KAAK,WAAW,QAAQ;AACnE,eAAW,QAAQ,aAAa;AAC9B,YAAM,eAAe,OAAO,KAAK,KAAK,iBAAiB,YAAY,KAAK,KAAK,aAAa,KAAK,EAAE,SAAS,IACtG,KAAK,KAAK,aAAa,KAAK,IAC5B;AACJ,YAAM,oBAAoB,OAAO,KAAK,KAAK,sBAAsB,YAAY,KAAK,KAAK,kBAAkB,KAAK,EAAE,SAAS,IACrH,KAAK,KAAK,kBAAkB,KAAK,IACjC;AACJ,YAAM,mBAAmB,OAAO,KAAK,KAAK,qBAAqB,YAAY,KAAK,KAAK,iBAAiB,KAAK,EAAE,SAAS,IAClH,KAAK,KAAK,iBAAiB,KAAK,IAChC;AACJ,YAAM,UAAU;AAAA,QACd,yBAAyB,KAAK,UAAU;AAAA,QACxC,oBAAoB,UAAU,iBAAiB,MAAM;AAAA,QACrD,mBAAmB,gBAAgB,gBAAgB,MAAM;AAAA,QACzD,KAAK,YAAY;AAAA,MACnB,EAAE,OAAO,CAAC,SAAS,SAAS,IAAI,EAAE,KAAK,GAAG;AAE1C,YAAM,sBAAsB;AAAA,QAC1B;AAAA,UACE;AAAA,UACA;AAAA,UACA,OAAO;AAAA,UACP;AAAA,UACA,SAAS,KAAK;AAAA,QAChB;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,iBAAe,sBACb,OACA,eACA,SACA,OACe;AACf,UAAM,gBAAgB,QAAQ,OAAO,CAAC,WAAW,OAAO,WAAW,WAAW,OAAO,KAAK;AAC1F,eAAW,UAAU,eAAe;AAClC,YAAM,QAAQ,OAAO,aAAa,GAAG,OAAO,UAAU,SAAS,OAAO,OAAO,MAAM,OAAO;AAC1F,YAAM,eAAe,OAAO,MAAO,MAAM,IAAI,EAAE,CAAC;AAChD,YAAM,UAAU,yBAAyB,KAAK,KAAK,YAAY;AAE/D,YAAM,sBAAsB;AAAA,QAC1B;AAAA,UACE;AAAA,UACA;AAAA,UACA,OAAO;AAAA,UACP;AAAA,UACA,SAAS,EAAE,MAAM,uBAAuB,SAAS,OAAO,MAAM;AAAA,QAChE;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,iBAAe,oBAAoB,QAQjB;AAChB,QAAI,CAAC,OAAO,QAAS;AAErB,UAAM,sBAAsB;AAAA,MAC1B;AAAA,QACE,eAAe,OAAO;AAAA,QACtB,OAAO,OAAO;AAAA,QACd,OAAO,OAAO;AAAA,QACd,SAAS,OAAO;AAAA,QAChB,SAAS,OAAO;AAAA,MAClB;AAAA,MACA,OAAO;AAAA,IACT;AAAA,EACF;AAEA,iBAAe,uBAAuB,QAKpB;AAChB,QAAI,CAAC,OAAO,WAAW,CAAC,wBAAyB;AAEjD,UAAM,wBAAwB;AAAA,MAC5B,OAAO;AAAA,MACP;AAAA,QACE,kBAAkB,OAAO;AAAA,QACzB,qBAAqB,oBAAI,KAAK;AAAA,MAChC;AAAA,MACA,OAAO;AAAA,IACT;AAAA,EACF;AAEA,iBAAe,YACb,OACA,QACA,OACA,OACA,uBAAuB,OACR;AACf,UAAM,cAAc,MAAM,eAAe,OAAO,OAAO,KAAK;AAC5D,UAAM,iCAAiC,aAAa,WAAW,WACzD,WAAW,eAAe,WAAW,YAAY,WAAW;AAElE,UAAM,MAAM,MAAM,eAAe,WAAW,OAAO,QAAQ,OAAO,KAAK;AACvE,QAAI,CAAC,IAAK;AAEV,QAAI,gCAAgC;AAClC;AAAA,IACF;AAEA,QAAI,IAAI,WAAW,QAAQ;AAOzB,aAAO,KAAK,wEAAwE;AAAA,QAClF;AAAA,QACA,iBAAiB;AAAA,QACjB,cAAc,IAAI;AAAA,MACpB,CAAC;AACD;AAAA,IACF;AAEA,QAAI,IAAI,eAAe;AACrB,UAAI,WAAW,aAAa;AAC1B,cAAM,gBAAgB;AAAA,UACpB,IAAI;AAAA,UACJ;AAAA,YACE,eAAe;AAAA,cACb,cAAc,IAAI;AAAA,cAClB,cAAc,IAAI;AAAA,cAClB,cAAc,IAAI;AAAA,cAClB,aAAa,IAAI;AAAA,cACjB,kBAAkB,IAAI;AAAA,YACxB;AAAA,UACF;AAAA,UACA;AAAA,YACE,UAAU,MAAM;AAAA,YAChB,gBAAgB,MAAM;AAAA,YACtB,QAAQ,MAAM;AAAA,UAChB;AAAA,QACF;AAAA,MACF,WAAW,WAAW,UAAU;AAC9B,cAAM,gBAAgB;AAAA,UACpB,IAAI;AAAA,UACJ;AAAA,YACE,cAAc,SAAS;AAAA,UACzB;AAAA,UACA;AAAA,YACE,UAAU,MAAM;AAAA,YAChB,gBAAgB,MAAM;AAAA,YACtB,QAAQ,MAAM;AAAA,UAChB;AAAA,QACF;AAAA,MACF,WAAW,WAAW,aAAa;AACjC,cAAM,gBAAgB;AAAA,UACpB,IAAI;AAAA,UACJ;AAAA,YACE,UAAU,MAAM;AAAA,YAChB,gBAAgB,MAAM;AAAA,YACtB,QAAQ,MAAM;AAAA,UAChB;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,WAAW,aAAa;AAC1B,YAAM,uBAAuB;AAAA,QAC3B,eAAe,IAAI;AAAA,QACnB,QAAQ;AAAA,QACR;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AACD,YAAM,oBAAoB;AAAA,QACxB,eAAe,IAAI;AAAA,QACnB,OAAO,IAAI;AAAA,QACX,OAAO;AAAA,QACP,SAAS;AAAA,QACT;AAAA,QACA,SAAS;AAAA,QACT,SAAS;AAAA,UACP,mBAAmB;AAAA,UACnB,SAAS,uBAAuB,IAAI,YAAY,aAAa,IAAI,YAAY,aAAa,IAAI,WAAW;AAAA,UACzG,cAAc,IAAI;AAAA,UAClB,cAAc,IAAI;AAAA,UAClB,cAAc,IAAI;AAAA,UAClB,aAAa,IAAI;AAAA,UACjB,kBAAkB,IAAI;AAAA,QACxB;AAAA,MACF,CAAC;AAAA,IACH,WAAW,WAAW,aAAa;AACjC,YAAM,uBAAuB;AAAA,QAC3B,eAAe,IAAI;AAAA,QACnB,QAAQ;AAAA,QACR;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AACD,YAAM,oBAAoB;AAAA,QACxB,eAAe,IAAI;AAAA,QACnB,OAAO,IAAI;AAAA,QACX,OAAO;AAAA,QACP,SAAS;AAAA,QACT;AAAA,QACA,SAAS;AAAA,QACT,SAAS;AAAA,UACP,mBAAmB;AAAA,UACnB,SAAS;AAAA,QACX;AAAA,MACF,CAAC;AAAA,IACH,OAAO;AACL,YAAM,uBAAuB;AAAA,QAC3B,eAAe,IAAI;AAAA,QACnB,QAAQ;AAAA,QACR;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AACD,YAAM,oBAAoB;AAAA,QACxB,eAAe,IAAI;AAAA,QACnB,OAAO,IAAI;AAAA,QACX,OAAO;AAAA,QACP,SAAS,SAAS;AAAA,QAClB;AAAA,QACA,SAAS;AAAA,QACT,SAAS;AAAA,UACP,mBAAmB;AAAA,UACnB,SAAS,SAAS;AAAA,QACpB;AAAA,MACF,CAAC;AAAA,IACH;AAEA,QAAI,WAAW,aAAa;AAC1B,YAAM,kBAAkB,2BAA2B;AAAA,QACjD;AAAA,QACA,eAAe,IAAI;AAAA,QACnB,YAAY,IAAI;AAAA,QAChB,WAAW,IAAI;AAAA,QACf,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,MACxB,CAAC;AACD;AAAA,IACF;AAEA,QAAI,WAAW,aAAa;AAC1B,YAAM,kBAAkB,2BAA2B;AAAA,QACjD;AAAA,QACA,eAAe,IAAI;AAAA,QACnB,YAAY,IAAI;AAAA,QAChB,WAAW,IAAI;AAAA,QACf,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,MACxB,CAAC;AACD;AAAA,IACF;AAEA,UAAM,kBAAkB,wBAAwB;AAAA,MAC9C;AAAA,MACA,eAAe,IAAI;AAAA,MACnB,YAAY,IAAI;AAAA,MAChB,WAAW,IAAI;AAAA,MACf,OAAO,SAAS;AAAA,MAChB,UAAU,MAAM;AAAA,MAChB,gBAAgB,MAAM;AAAA,IACxB,CAAC;AAAA,EACH;AAEA,SAAO;AAAA,IACL,MAAM,UAAU,OAAe,WAAmB,OAAiC;AACjF,YAAM,MAAM,MAAM,eAAe,OAAO,OAAO,KAAK;AACpD,UAAI,CAAC,KAAK;AACR,eAAO,KAAK,6CAA6C,EAAE,MAAM,CAAC;AAClE;AAAA,MACF;AACA,UAAI,IAAI,WAAW,aAAa;AAC9B,YAAI,IAAI,eAAe;AACrB,gBAAM,gBAAgB,cAAc,IAAI,eAAe;AAAA,YACrD,UAAU,MAAM;AAAA,YAChB,gBAAgB,MAAM;AAAA,YACtB,QAAQ,MAAM;AAAA,UAChB,CAAC;AAAA,QACH;AACA;AAAA,MACF;AAEA,YAAM,cAAc,mBAAmB,IAAI,aAAa;AACxD,YAAM,UAAU,mBAAmB,WAAW;AAC9C,UAAI,CAAC,SAAS,cAAc;AAC1B,cAAM,IAAI,MAAM,6CAA6C,WAAW,EAAE;AAAA,MAC5E;AACA,YAAM,uBAAuB,QAAQ,yBAAyB;AAE9D,YAAM,cAAc,MAAM,8BAA8B,QAAQ,IAAI,eAAe,KAAK;AACxF,UAAI,CAAC,aAAa;AAChB,cAAM,IAAI,MAAM,eAAe,IAAI,aAAa,yBAAyB;AAAA,MAC3E;AAKA,YAAM,UAAU,IAAI,WAAW;AAC/B,YAAM,YAAY,MAAM,eAAe,WAAW,IAAI,IAAI,WAAW,KAAK;AAC1E,UAAI,CAAC,aAAa,UAAU,WAAW,WAAW;AAChD;AAAA,MACF;AACA,YAAM,kBAAkB,yBAAyB;AAAA,QAC/C,OAAO,IAAI;AAAA,QACX,eAAe,IAAI;AAAA,QACnB,YAAY,IAAI;AAAA,QAChB,WAAW,IAAI;AAAA,QACf;AAAA,QACA,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,MACxB,CAAC;AACD,YAAM,uBAAuB;AAAA,QAC3B,eAAe,IAAI;AAAA,QACnB,QAAQ;AAAA,QACR;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AACD,YAAM,oBAAoB;AAAA,QACxB,eAAe,IAAI;AAAA,QACnB,OAAO,IAAI;AAAA,QACX,OAAO;AAAA,QACP,SAAS;AAAA,QACT;AAAA,QACA,SAAS;AAAA,QACT,SAAS;AAAA,UACP,mBAAmB;AAAA,UACnB,SAAS,0BAA0B,IAAI,UAAU;AAAA,UACjD,YAAY,IAAI;AAAA,UAChB,WAAW,IAAI;AAAA,QACjB;AAAA,MACF,CAAC;AAED,UAAI,IAAI,eAAe;AACrB,cAAM,gBAAgB,SAAS,IAAI,eAAe;AAAA,UAChD,UAAU,MAAM;AAAA,UAChB,gBAAgB,MAAM;AAAA,UACtB,QAAQ,MAAM;AAAA,QAChB,CAAC;AAAA,MACH;AAEA,YAAM,UAAU,MAAM,eAAe,SAAS,IAAI,YAAY,KAAK;AACnE,UAAI,iBAAiB;AACrB,UAAI,aAA4B;AAChC,UAAI,mBAAmB,UAAU,oBAAoB;AAErD,UAAI;AACF,yBAAiB,SAAS,QAAQ,aAAa;AAAA,UAC7C,YAAY,IAAI;AAAA,UAChB,QAAQ,IAAI,UAAU;AAAA,UACtB;AAAA,UACA;AAAA,UACA;AAAA,UACA,OAAO,EAAE,gBAAgB,MAAM,gBAAgB,UAAU,MAAM,SAAS;AAAA,UACxE,OAAO,IAAI;AAAA,QACb,CAAC,GAAG;AACF,cAAI,IAAI,iBAAiB,MAAM,gBAAgB,wBAAwB,IAAI,eAAe,MAAM,UAAU,MAAM,cAAc,GAAG;AAC/H,kBAAM,YAAY,IAAI,IAAI,aAAa,OAAO,QAAW,oBAAoB;AAC7E;AAAA,UACF;AAEA,gBAAM,QAAQ,oBAAoB,KAAK;AACvC,gBAAM,sBAAsB,MAAM,kBAAkB,MAAM,MAAM;AAChE,4BAAkB;AAClB,uBAAa,MAAM,iBAAiB;AAEpC,gBAAM,eAAe;AAAA,YACnB,IAAI;AAAA,YACJ;AAAA,cACE,GAAG;AAAA,cACH,kBAAkB;AAAA,YACpB;AAAA,YACA,MAAM;AAAA,YACN;AAAA,YACA;AAAA,UACF;AACA,8BAAoB;AAEpB,gBAAM,eAAe,IAAI,eAAe,gBAAgB,YAAY,KAAK;AACzE,gBAAM,yBAAyB,MAAM,4BAA4B,KAAK;AACtE,gBAAM,sBAAsB,IAAI,IAAI,IAAI,eAAe,MAAM,OAAO,KAAK;AAEzE,gBAAM,oBAAoB;AAAA,YACxB,eAAe,IAAI;AAAA,YACnB,OAAO,IAAI;AAAA,YACX,OAAO;AAAA,YACP,SAAS,MAAM,SAAS,KAAK,EAAE,SAC3B,MAAM,QAAQ,KAAK,IACnB,0BAA0B,MAAM,UAAU;AAAA,YAC9C;AAAA,YACA,SAAS;AAAA,YACT,SAAS;AAAA,cACP,mBAAmB;AAAA,cACnB,SAAS,aAAa,cAAc,GAAG,aAAa,OAAO,UAAU,KAAK,EAAE;AAAA,cAC5E;AAAA,cACA,WAAW,MAAM,MAAM;AAAA,cACvB;AAAA,cACA,QAAQ,MAAM;AAAA,YAChB;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF,SAAS,OAAO;AACd,YAAI,iBAAiB,+BAA+B;AAClD,iBAAO,KAAK,uEAAuE;AAAA,YACjF,OAAO,IAAI;AAAA,YACX,0BAA0B,MAAM;AAAA,UAClC,CAAC;AACD;AAAA,QACF;AACA,cAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU;AACzD,cAAM,sBAAsB;AAAA,UAC1B;AAAA,YACE,eAAe,IAAI;AAAA,YACnB,OAAO,IAAI;AAAA,YACX,OAAO;AAAA,YACP;AAAA,UACF;AAAA,UACA;AAAA,QACF;AACA,cAAM,YAAY,IAAI,IAAI,UAAU,OAAO,SAAS,oBAAoB;AACxE;AAAA,MACF;AAEA,YAAM,YAAY,IAAI,IAAI,aAAa,OAAO,QAAW,oBAAoB;AAAA,IAC/E;AAAA,IAEA,MAAM,UAAU,OAAe,WAAmB,OAAiC;AACjF,YAAM,MAAM,MAAM,eAAe,OAAO,OAAO,KAAK;AACpD,UAAI,CAAC,KAAK;AACR,eAAO,KAAK,6CAA6C,EAAE,MAAM,CAAC;AAClE;AAAA,MACF;AACA,UAAI,IAAI,WAAW,aAAa;AAC9B,YAAI,IAAI,eAAe;AACrB,gBAAM,gBAAgB,cAAc,IAAI,eAAe;AAAA,YACrD,UAAU,MAAM;AAAA,YAChB,gBAAgB,MAAM;AAAA,YACtB,QAAQ,MAAM;AAAA,UAChB,CAAC;AAAA,QACH;AACA;AAAA,MACF;AAEA,YAAM,cAAc,mBAAmB,IAAI,aAAa;AACxD,YAAM,UAAU,mBAAmB,WAAW;AAC9C,UAAI,CAAC,SAAS,cAAc;AAC1B,cAAM,IAAI,MAAM,6CAA6C,WAAW,EAAE;AAAA,MAC5E;AACA,YAAM,uBAAuB,QAAQ,yBAAyB;AAE9D,YAAM,cAAc,MAAM,8BAA8B,QAAQ,IAAI,eAAe,KAAK;AACxF,UAAI,CAAC,aAAa;AAChB,cAAM,IAAI,MAAM,eAAe,IAAI,aAAa,yBAAyB;AAAA,MAC3E;AAKA,YAAM,UAAU,IAAI,WAAW;AAC/B,YAAM,YAAY,MAAM,eAAe,WAAW,IAAI,IAAI,WAAW,KAAK;AAC1E,UAAI,CAAC,aAAa,UAAU,WAAW,WAAW;AAChD;AAAA,MACF;AACA,YAAM,kBAAkB,yBAAyB;AAAA,QAC/C,OAAO,IAAI;AAAA,QACX,eAAe,IAAI;AAAA,QACnB,YAAY,IAAI;AAAA,QAChB,WAAW,IAAI;AAAA,QACf;AAAA,QACA,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,MACxB,CAAC;AACD,YAAM,uBAAuB;AAAA,QAC3B,eAAe,IAAI;AAAA,QACnB,QAAQ;AAAA,QACR;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AACD,YAAM,oBAAoB;AAAA,QACxB,eAAe,IAAI;AAAA,QACnB,OAAO,IAAI;AAAA,QACX,OAAO;AAAA,QACP,SAAS;AAAA,QACT;AAAA,QACA,SAAS;AAAA,QACT,SAAS;AAAA,UACP,mBAAmB;AAAA,UACnB,SAAS,0BAA0B,IAAI,UAAU;AAAA,UACjD,YAAY,IAAI;AAAA,UAChB,WAAW,IAAI;AAAA,QACjB;AAAA,MACF,CAAC;AAED,UAAI,IAAI,eAAe;AACrB,cAAM,gBAAgB,SAAS,IAAI,eAAe;AAAA,UAChD,UAAU,MAAM;AAAA,UAChB,gBAAgB,MAAM;AAAA,UACtB,QAAQ,MAAM;AAAA,QAChB,CAAC;AAAA,MACH;AAEA,YAAM,UAAU,MAAM,eAAe,SAAS,IAAI,YAAY,KAAK;AACnE,UAAI,iBAAiB;AACrB,UAAI,mBAAmB,UAAU,oBAAoB;AAErD,UAAI;AACF,yBAAiB,SAAS,QAAQ,aAAa;AAAA,UAC7C,YAAY,IAAI;AAAA,UAChB,QAAQ,IAAI,UAAU;AAAA,UACtB;AAAA,UACA;AAAA,UACA;AAAA,UACA,OAAO,EAAE,gBAAgB,MAAM,gBAAgB,UAAU,MAAM,SAAS;AAAA,UACxE,OAAO,IAAI;AAAA,QACb,CAAC,GAAG;AACF,cAAI,IAAI,iBAAiB,MAAM,gBAAgB,wBAAwB,IAAI,eAAe,MAAM,UAAU,MAAM,cAAc,GAAG;AAC/H,kBAAM,YAAY,IAAI,IAAI,aAAa,OAAO,QAAW,oBAAoB;AAC7E;AAAA,UACF;AAEA,gBAAM,QAAQ,oBAAoB,KAAK;AACvC,4BAAkB,MAAM;AAExB,gBAAM,eAAe;AAAA,YACnB,IAAI;AAAA,YACJ;AAAA,cACE,cAAc;AAAA,cACd,cAAc,MAAM;AAAA,cACpB,cAAc,MAAM;AAAA,cACpB,aAAa,MAAM;AAAA,cACnB,kBAAkB;AAAA,YACpB;AAAA,YACA,MAAM;AAAA,YACN;AAAA,YACA;AAAA,UACF;AACA,8BAAoB;AACpB,gBAAM,eAAe,IAAI,eAAe,gBAAgB,MAAM,KAAK;AACnE,gBAAM,sBAAsB,IAAI,IAAI,IAAI,eAAe,MAAM,SAAS,KAAK;AAE3E,gBAAM,oBAAoB;AAAA,YACxB,eAAe,IAAI;AAAA,YACnB,OAAO,IAAI;AAAA,YACX,OAAO;AAAA,YACP,SAAS,0BAA0B,MAAM,UAAU;AAAA,YACnD;AAAA,YACA,SAAS;AAAA,YACT,SAAS;AAAA,cACP,mBAAmB;AAAA,cACnB,SAAS,aAAa,cAAc;AAAA,cACpC;AAAA,cACA,WAAW,MAAM,QAAQ;AAAA,cACzB,QAAQ,MAAM;AAAA,YAChB;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF,SAAS,OAAO;AACd,YAAI,iBAAiB,+BAA+B;AAClD,iBAAO,KAAK,uEAAuE;AAAA,YACjF,OAAO,IAAI;AAAA,YACX,0BAA0B,MAAM;AAAA,UAClC,CAAC;AACD;AAAA,QACF;AACA,cAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU;AACzD,cAAM,sBAAsB;AAAA,UAC1B;AAAA,YACE,eAAe,IAAI;AAAA,YACnB,OAAO,IAAI;AAAA,YACX,OAAO;AAAA,YACP;AAAA,UACF;AAAA,UACA;AAAA,QACF;AACA,cAAM,YAAY,IAAI,IAAI,UAAU,OAAO,SAAS,oBAAoB;AACxE;AAAA,MACF;AAEA,YAAM,YAAY,IAAI,IAAI,aAAa,OAAO,QAAW,oBAAoB;AAAA,IAC/E;AAAA,EACF;AACF;",
|
|
4
|
+
"sourcesContent": ["import type { EntityManager } from '@mikro-orm/postgresql'\nimport { getIntegration } from '@open-mercato/shared/modules/integrations/types'\nimport type { CredentialsService } from '../../integrations/lib/credentials-service'\nimport type { IntegrationLogService } from '../../integrations/lib/log-service'\nimport type { IntegrationStateService } from '../../integrations/lib/state-service'\nimport type { ProgressService } from '../../progress/lib/progressService'\nimport { STALE_JOB_TIMEOUT_SECONDS } from '../../progress/lib/progressService'\nimport { refreshCoverageSnapshot } from '../../query_index/lib/coverage'\nimport { emitDataSyncEvent } from '../events'\nimport type { DataSyncAdapter, DataMapping, ExportBatch, ImportBatch } from './adapter'\nimport { getDataSyncAdapter } from './adapter-registry'\nimport type { SyncRunService } from './sync-run-service'\nimport { SyncRunOwnershipConflictError } from './sync-run-service'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\n\nconst logger = createLogger('data_sync').child({ component: 'sync-engine' })\n\ntype SyncScope = {\n organizationId: string\n tenantId: string\n userId?: string | null\n}\n\ntype EngineDeps = {\n em: EntityManager\n syncRunService: SyncRunService\n integrationCredentialsService: CredentialsService\n integrationLogService: IntegrationLogService\n integrationStateService?: IntegrationStateService\n progressService: ProgressService\n}\n\nfunction resolveProviderKey(integrationId: string): string {\n return getIntegration(integrationId)?.providerKey ?? integrationId\n}\n\nfunction applyImportCounters(batch: ImportBatch): Pick<Required<SyncCounterDelta>, 'createdCount' | 'updatedCount' | 'skippedCount' | 'failedCount'> {\n let createdCount = 0\n let updatedCount = 0\n let skippedCount = 0\n let failedCount = 0\n\n for (const item of batch.items) {\n if (item.action === 'create') createdCount += 1\n else if (item.action === 'update') updatedCount += 1\n else if (item.action === 'failed') failedCount += 1\n else skippedCount += 1\n }\n\n return { createdCount, updatedCount, skippedCount, failedCount }\n}\n\ntype SyncCounterDelta = {\n createdCount?: number\n updatedCount?: number\n skippedCount?: number\n failedCount?: number\n processedCount: number\n}\n\nfunction applyExportCounters(batch: ExportBatch): SyncCounterDelta {\n let failedCount = 0\n let skippedCount = 0\n let updatedCount = 0\n\n for (const result of batch.results) {\n if (result.status === 'error') failedCount += 1\n else if (result.status === 'skipped') skippedCount += 1\n else updatedCount += 1\n }\n\n return {\n failedCount,\n skippedCount,\n updatedCount,\n processedCount: batch.results.length,\n }\n}\n\n// Adapter batches can legitimately outlast the stale-job sweep window (slow upstream\n// APIs), so the engine must heartbeat while a batch is still being produced.\nconst HEARTBEAT_TICK_MS = (STALE_JOB_TIMEOUT_SECONDS * 1000) / 4\n\n// Runs `tick` on an interval only while the source iterator is pending, so heartbeats\n// stop the moment the producer dies and genuinely stale jobs still get swept. The outer\n// finally closes the adapter generator on early exits (cancellation, ownership conflict).\nasync function* withHeartbeat<T>(source: AsyncIterable<T>, tick: () => void, intervalMs: number): AsyncGenerator<T, void, undefined> {\n const iterator = source[Symbol.asyncIterator]()\n try {\n while (true) {\n const timer = setInterval(tick, intervalMs)\n let result: IteratorResult<T>\n try {\n result = await iterator.next()\n } finally {\n clearInterval(timer)\n }\n if (result.done) return\n yield result.value\n }\n } finally {\n await iterator.return?.()\n }\n}\n\nexport function createSyncEngine(deps: EngineDeps) {\n const { syncRunService, integrationCredentialsService, integrationLogService, integrationStateService, progressService } = deps\n\n async function resolveMapping(adapter: DataSyncAdapter, entityType: string, scope: SyncScope): Promise<DataMapping> {\n return adapter.getMapping({\n entityType,\n scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },\n })\n }\n\n async function updateProgress(progressJobId: string | null | undefined, processedCount: number, totalCount: number | null, scope: SyncScope): Promise<void> {\n if (!progressJobId) return\n\n await progressService.updateProgress(\n progressJobId,\n {\n processedCount,\n totalCount: totalCount ?? undefined,\n },\n {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n },\n )\n }\n\n // On redelivery the progress counter must resume where the last delivery left off the\n // same way committedBatches does \u2014 updateProgress writes absolute counts, so starting at\n // zero would regress the visible count. The progress job's own processedCount is the only\n // persisted value already in the engine's unit: `batch.processedCount ?? items.length`,\n // i.e. source records. The run's created/updated/skipped/failed counters count emitted\n // items, which adapters may explode several-per-source-record (Akeneo yields a product\n // plus its variants), so seeding from them would overshoot the total and pin the bar.\n async function seedProcessedCount(progressJobId: string | null | undefined, scope: SyncScope): Promise<number> {\n if (!progressJobId) return 0\n const job = await progressService.getJob(progressJobId, {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n })\n return job?.processedCount ?? 0\n }\n\n function makeHeartbeatTick(progressJobId: string | null | undefined, scope: SyncScope): () => void {\n const touchJobHeartbeat = progressService.touchJobHeartbeat?.bind(progressService)\n if (!progressJobId || !touchJobHeartbeat) return () => {}\n let inFlight = false\n return () => {\n if (inFlight) return\n inFlight = true\n touchJobHeartbeat(progressJobId, {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n })\n .catch((error) => {\n logger.warn('Progress heartbeat failed', {\n progressJobId,\n error: error instanceof Error ? error.message : String(error),\n })\n })\n .finally(() => {\n inFlight = false\n })\n }\n }\n\n async function refreshCoverageSnapshots(entityTypes: string[] | undefined, scope: SyncScope): Promise<void> {\n if (!entityTypes || entityTypes.length === 0) return\n\n await Promise.allSettled(\n Array.from(new Set(entityTypes.filter((value) => typeof value === 'string' && value.trim().length > 0)))\n .map((entityType) => refreshCoverageSnapshot(deps.em, {\n entityType,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n })),\n )\n }\n\n async function logImportItemFailures(\n runId: string,\n integrationId: string,\n items: ImportBatch['items'],\n scope: SyncScope,\n ): Promise<void> {\n const failedItems = items.filter((item) => item.action === 'failed')\n for (const item of failedItems) {\n const errorMessage = typeof item.data.errorMessage === 'string' && item.data.errorMessage.trim().length > 0\n ? item.data.errorMessage.trim()\n : 'Import item failed'\n const sourceProductUuid = typeof item.data.sourceProductUuid === 'string' && item.data.sourceProductUuid.trim().length > 0\n ? item.data.sourceProductUuid.trim()\n : null\n const sourceIdentifier = typeof item.data.sourceIdentifier === 'string' && item.data.sourceIdentifier.trim().length > 0\n ? item.data.sourceIdentifier.trim()\n : null\n const message = [\n `Failed to import item ${item.externalId}`,\n sourceProductUuid ? `(uuid: ${sourceProductUuid})` : null,\n sourceIdentifier ? `(identifier: ${sourceIdentifier})` : null,\n `: ${errorMessage}`,\n ].filter((part) => part !== null).join(' ')\n\n await integrationLogService.write(\n {\n integrationId,\n runId,\n level: 'error',\n message,\n payload: item.data,\n },\n scope,\n )\n }\n }\n\n async function logExportItemFailures(\n runId: string,\n integrationId: string,\n results: ExportBatch['results'],\n scope: SyncScope,\n ): Promise<void> {\n const failedResults = results.filter((result) => result.status === 'error' && result.error)\n for (const result of failedResults) {\n const label = result.externalId ? `${result.externalId} (id: ${result.localId})` : result.localId\n const errorMessage = result.error!.split('\\n')[0]\n const message = `Failed to export item ${label}: ${errorMessage}`\n\n await integrationLogService.write(\n {\n integrationId,\n runId,\n level: 'error',\n message,\n payload: { kind: 'export-item-failure', summary: result.error },\n },\n scope,\n )\n }\n }\n\n async function writeOperationalLog(params: {\n integrationId: string\n runId: string\n level: 'info' | 'warn' | 'error'\n message: string\n scope: SyncScope\n enabled: boolean\n payload?: Record<string, unknown>\n }): Promise<void> {\n if (!params.enabled) return\n\n await integrationLogService.write(\n {\n integrationId: params.integrationId,\n runId: params.runId,\n level: params.level,\n message: params.message,\n payload: params.payload,\n },\n params.scope,\n )\n }\n\n async function updateOperationalState(params: {\n integrationId: string\n status: 'healthy' | 'degraded' | 'unhealthy'\n scope: SyncScope\n enabled: boolean\n }): Promise<void> {\n if (!params.enabled || !integrationStateService) return\n\n await integrationStateService.upsert(\n params.integrationId,\n {\n lastHealthStatus: params.status,\n lastHealthCheckedAt: new Date(),\n },\n params.scope,\n )\n }\n\n async function finalizeRun(\n runId: string,\n status: 'completed' | 'failed' | 'cancelled',\n scope: SyncScope,\n error?: string,\n operationalTelemetry = false,\n ): Promise<void> {\n const existingRun = await syncRunService.getRun(runId, scope)\n const alreadyFinalizedWithSameStatus = existingRun?.status === status\n && (status === 'completed' || status === 'failed' || status === 'cancelled')\n\n const run = await syncRunService.markStatus(runId, status, scope, error)\n if (!run) return\n\n if (alreadyFinalizedWithSameStatus) {\n return\n }\n\n if (run.status !== status) {\n // `markStatus` refuses a terminal -> different-terminal transition and\n // returns the row unchanged, so the run is already finished under another\n // delivery of this job. Everything below \u2014 the progress job, the\n // operational log and the lifecycle event \u2014 would describe the wrong\n // outcome, and `data_sync.run.failed` is dispatched to tenant webhooks.\n // A displaced worker stays silent instead.\n logger.warn('Skipping finalization of a sync run another worker already finalized', {\n runId,\n requestedStatus: status,\n actualStatus: run.status,\n })\n return\n }\n\n if (run.progressJobId) {\n if (status === 'completed') {\n await progressService.completeJob(\n run.progressJobId,\n {\n resultSummary: {\n createdCount: run.createdCount,\n updatedCount: run.updatedCount,\n skippedCount: run.skippedCount,\n failedCount: run.failedCount,\n batchesCompleted: run.batchesCompleted,\n },\n },\n {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n },\n )\n } else if (status === 'failed') {\n await progressService.failJob(\n run.progressJobId,\n {\n errorMessage: error ?? 'Sync run failed',\n },\n {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n },\n )\n } else if (status === 'cancelled') {\n await progressService.markCancelled(\n run.progressJobId,\n {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n },\n )\n }\n }\n\n if (status === 'completed') {\n await updateOperationalState({\n integrationId: run.integrationId,\n status: 'healthy',\n scope,\n enabled: operationalTelemetry,\n })\n await writeOperationalLog({\n integrationId: run.integrationId,\n runId: run.id,\n level: 'info',\n message: 'Sync run completed',\n scope,\n enabled: operationalTelemetry,\n payload: {\n operationalStatus: 'completed',\n summary: `Sync completed with ${run.createdCount} created, ${run.updatedCount} updated, ${run.failedCount} failed.`,\n createdCount: run.createdCount,\n updatedCount: run.updatedCount,\n skippedCount: run.skippedCount,\n failedCount: run.failedCount,\n batchesCompleted: run.batchesCompleted,\n },\n })\n } else if (status === 'cancelled') {\n await updateOperationalState({\n integrationId: run.integrationId,\n status: 'degraded',\n scope,\n enabled: operationalTelemetry,\n })\n await writeOperationalLog({\n integrationId: run.integrationId,\n runId: run.id,\n level: 'warn',\n message: 'Sync run cancelled',\n scope,\n enabled: operationalTelemetry,\n payload: {\n operationalStatus: 'cancelled',\n summary: 'The sync run was cancelled before completion.',\n },\n })\n } else {\n await updateOperationalState({\n integrationId: run.integrationId,\n status: 'unhealthy',\n scope,\n enabled: operationalTelemetry,\n })\n await writeOperationalLog({\n integrationId: run.integrationId,\n runId: run.id,\n level: 'error',\n message: error ?? 'Sync run failed',\n scope,\n enabled: operationalTelemetry,\n payload: {\n operationalStatus: 'failed',\n summary: error ?? 'The sync run failed.',\n },\n })\n }\n\n if (status === 'completed') {\n await emitDataSyncEvent('data_sync.run.completed', {\n runId,\n integrationId: run.integrationId,\n entityType: run.entityType,\n direction: run.direction,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n })\n return\n }\n\n if (status === 'cancelled') {\n await emitDataSyncEvent('data_sync.run.cancelled', {\n runId,\n integrationId: run.integrationId,\n entityType: run.entityType,\n direction: run.direction,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n })\n return\n }\n\n await emitDataSyncEvent('data_sync.run.failed', {\n runId,\n integrationId: run.integrationId,\n entityType: run.entityType,\n direction: run.direction,\n error: error ?? null,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n })\n }\n\n return {\n async runImport(runId: string, batchSize: number, scope: SyncScope): Promise<void> {\n const run = await syncRunService.getRun(runId, scope)\n if (!run) {\n logger.warn('Skipping stale import job for missing run', { runId })\n return\n }\n if (run.status === 'cancelled') {\n if (run.progressJobId) {\n await progressService.markCancelled(run.progressJobId, {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n })\n }\n return\n }\n\n const providerKey = resolveProviderKey(run.integrationId)\n const adapter = getDataSyncAdapter(providerKey)\n if (!adapter?.streamImport) {\n throw new Error(`No import adapter registered for provider ${providerKey}`)\n }\n const operationalTelemetry = adapter.operationalTelemetry === true\n\n const credentials = await integrationCredentialsService.resolve(run.integrationId, scope)\n if (!credentials) {\n throw new Error(`Integration ${run.integrationId} is missing credentials`)\n }\n\n // A run already `running` means a stalled job was redelivered, so this is\n // a resume rather than a first start. Consumers see one `started` event per\n // delivery either way; the flag is what lets them tell the two apart.\n const resumed = run.status === 'running'\n const activeRun = await syncRunService.markStatus(run.id, 'running', scope)\n if (!activeRun || activeRun.status !== 'running') {\n return\n }\n await emitDataSyncEvent('data_sync.run.started', {\n runId: run.id,\n integrationId: run.integrationId,\n entityType: run.entityType,\n direction: run.direction,\n resumed,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n })\n await updateOperationalState({\n integrationId: run.integrationId,\n status: 'degraded',\n scope,\n enabled: operationalTelemetry,\n })\n await writeOperationalLog({\n integrationId: run.integrationId,\n runId: run.id,\n level: 'info',\n message: 'Sync run started',\n scope,\n enabled: operationalTelemetry,\n payload: {\n operationalStatus: 'running',\n summary: `Import run started for ${run.entityType}.`,\n entityType: run.entityType,\n direction: run.direction,\n },\n })\n\n if (run.progressJobId) {\n await progressService.startJob(run.progressJobId, {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n })\n }\n\n const mapping = await resolveMapping(adapter, run.entityType, scope)\n let processedCount = await seedProcessedCount(run.progressJobId, scope)\n let totalCount: number | null = null\n let committedBatches = activeRun.batchesCompleted ?? 0\n\n try {\n for await (const batch of withHeartbeat(\n adapter.streamImport({\n entityType: run.entityType,\n cursor: run.cursor ?? undefined,\n batchSize,\n credentials,\n mapping,\n scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },\n runId: run.id,\n }),\n makeHeartbeatTick(run.progressJobId, scope),\n HEARTBEAT_TICK_MS,\n )) {\n if (run.progressJobId && await progressService.isCancellationRequested(run.progressJobId, scope.tenantId, scope.organizationId)) {\n await finalizeRun(run.id, 'cancelled', scope, undefined, operationalTelemetry)\n return\n }\n\n const delta = applyImportCounters(batch)\n const processedBatchCount = batch.processedCount ?? batch.items.length\n processedCount += processedBatchCount\n totalCount = batch.totalEstimate ?? totalCount\n\n await syncRunService.commitBatchProgress(\n run.id,\n {\n ...delta,\n batchesCompleted: 1,\n },\n batch.cursor,\n scope,\n committedBatches,\n )\n committedBatches += 1\n\n await updateProgress(run.progressJobId, processedCount, totalCount, scope)\n await refreshCoverageSnapshots(batch.refreshCoverageEntityTypes, scope)\n await logImportItemFailures(run.id, run.integrationId, batch.items, scope)\n\n await writeOperationalLog({\n integrationId: run.integrationId,\n runId: run.id,\n level: 'info',\n message: batch.message?.trim().length\n ? batch.message.trim()\n : `Processed import batch ${batch.batchIndex}`,\n scope,\n enabled: operationalTelemetry,\n payload: {\n operationalStatus: 'running',\n summary: `Processed ${processedCount}${totalCount ? ` of ${totalCount}` : ''} rows so far.`,\n processedCount,\n batchSize: batch.items.length,\n processedBatchCount,\n cursor: batch.cursor,\n },\n })\n }\n } catch (error) {\n if (error instanceof SyncRunOwnershipConflictError) {\n logger.warn('Yielding import run to a concurrent worker that already advanced it', {\n runId: run.id,\n expectedBatchesCompleted: error.expectedBatchesCompleted,\n })\n return\n }\n const message = error instanceof Error ? error.message : 'Sync import failed'\n await integrationLogService.write(\n {\n integrationId: run.integrationId,\n runId: run.id,\n level: 'error',\n message,\n },\n scope,\n )\n await finalizeRun(run.id, 'failed', scope, message, operationalTelemetry)\n return\n }\n\n await finalizeRun(run.id, 'completed', scope, undefined, operationalTelemetry)\n },\n\n async runExport(runId: string, batchSize: number, scope: SyncScope): Promise<void> {\n const run = await syncRunService.getRun(runId, scope)\n if (!run) {\n logger.warn('Skipping stale export job for missing run', { runId })\n return\n }\n if (run.status === 'cancelled') {\n if (run.progressJobId) {\n await progressService.markCancelled(run.progressJobId, {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n })\n }\n return\n }\n\n const providerKey = resolveProviderKey(run.integrationId)\n const adapter = getDataSyncAdapter(providerKey)\n if (!adapter?.streamExport) {\n throw new Error(`No export adapter registered for provider ${providerKey}`)\n }\n const operationalTelemetry = adapter.operationalTelemetry === true\n\n const credentials = await integrationCredentialsService.resolve(run.integrationId, scope)\n if (!credentials) {\n throw new Error(`Integration ${run.integrationId} is missing credentials`)\n }\n\n // A run already `running` means a stalled job was redelivered, so this is\n // a resume rather than a first start. Consumers see one `started` event per\n // delivery either way; the flag is what lets them tell the two apart.\n const resumed = run.status === 'running'\n const activeRun = await syncRunService.markStatus(run.id, 'running', scope)\n if (!activeRun || activeRun.status !== 'running') {\n return\n }\n await emitDataSyncEvent('data_sync.run.started', {\n runId: run.id,\n integrationId: run.integrationId,\n entityType: run.entityType,\n direction: run.direction,\n resumed,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n })\n await updateOperationalState({\n integrationId: run.integrationId,\n status: 'degraded',\n scope,\n enabled: operationalTelemetry,\n })\n await writeOperationalLog({\n integrationId: run.integrationId,\n runId: run.id,\n level: 'info',\n message: 'Sync run started',\n scope,\n enabled: operationalTelemetry,\n payload: {\n operationalStatus: 'running',\n summary: `Export run started for ${run.entityType}.`,\n entityType: run.entityType,\n direction: run.direction,\n },\n })\n\n if (run.progressJobId) {\n await progressService.startJob(run.progressJobId, {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n })\n }\n\n const mapping = await resolveMapping(adapter, run.entityType, scope)\n let processedCount = await seedProcessedCount(run.progressJobId, scope)\n let committedBatches = activeRun.batchesCompleted ?? 0\n\n try {\n for await (const batch of withHeartbeat(\n adapter.streamExport({\n entityType: run.entityType,\n cursor: run.cursor ?? undefined,\n batchSize,\n credentials,\n mapping,\n scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },\n runId: run.id,\n }),\n makeHeartbeatTick(run.progressJobId, scope),\n HEARTBEAT_TICK_MS,\n )) {\n if (run.progressJobId && await progressService.isCancellationRequested(run.progressJobId, scope.tenantId, scope.organizationId)) {\n await finalizeRun(run.id, 'cancelled', scope, undefined, operationalTelemetry)\n return\n }\n\n const delta = applyExportCounters(batch)\n processedCount += delta.processedCount\n\n await syncRunService.commitBatchProgress(\n run.id,\n {\n createdCount: 0,\n updatedCount: delta.updatedCount,\n skippedCount: delta.skippedCount,\n failedCount: delta.failedCount,\n batchesCompleted: 1,\n },\n batch.cursor,\n scope,\n committedBatches,\n )\n committedBatches += 1\n await updateProgress(run.progressJobId, processedCount, null, scope)\n await logExportItemFailures(run.id, run.integrationId, batch.results, scope)\n\n await writeOperationalLog({\n integrationId: run.integrationId,\n runId: run.id,\n level: 'info',\n message: `Processed export batch ${batch.batchIndex}`,\n scope,\n enabled: operationalTelemetry,\n payload: {\n operationalStatus: 'running',\n summary: `Processed ${processedCount} export items so far.`,\n processedCount,\n batchSize: batch.results.length,\n cursor: batch.cursor,\n },\n })\n }\n } catch (error) {\n if (error instanceof SyncRunOwnershipConflictError) {\n logger.warn('Yielding export run to a concurrent worker that already advanced it', {\n runId: run.id,\n expectedBatchesCompleted: error.expectedBatchesCompleted,\n })\n return\n }\n const message = error instanceof Error ? error.message : 'Sync export failed'\n await integrationLogService.write(\n {\n integrationId: run.integrationId,\n runId: run.id,\n level: 'error',\n message,\n },\n scope,\n )\n await finalizeRun(run.id, 'failed', scope, message, operationalTelemetry)\n return\n }\n\n await finalizeRun(run.id, 'completed', scope, undefined, operationalTelemetry)\n },\n }\n}\n\nexport type SyncEngine = ReturnType<typeof createSyncEngine>\n"],
|
|
5
|
+
"mappings": "AACA,SAAS,sBAAsB;AAK/B,SAAS,iCAAiC;AAC1C,SAAS,+BAA+B;AACxC,SAAS,yBAAyB;AAElC,SAAS,0BAA0B;AAEnC,SAAS,qCAAqC;AAC9C,SAAS,oBAAoB;AAE7B,MAAM,SAAS,aAAa,WAAW,EAAE,MAAM,EAAE,WAAW,cAAc,CAAC;AAiB3E,SAAS,mBAAmB,eAA+B;AACzD,SAAO,eAAe,aAAa,GAAG,eAAe;AACvD;AAEA,SAAS,oBAAoB,OAAwH;AACnJ,MAAI,eAAe;AACnB,MAAI,eAAe;AACnB,MAAI,eAAe;AACnB,MAAI,cAAc;AAElB,aAAW,QAAQ,MAAM,OAAO;AAC9B,QAAI,KAAK,WAAW,SAAU,iBAAgB;AAAA,aACrC,KAAK,WAAW,SAAU,iBAAgB;AAAA,aAC1C,KAAK,WAAW,SAAU,gBAAe;AAAA,QAC7C,iBAAgB;AAAA,EACvB;AAEA,SAAO,EAAE,cAAc,cAAc,cAAc,YAAY;AACjE;AAUA,SAAS,oBAAoB,OAAsC;AACjE,MAAI,cAAc;AAClB,MAAI,eAAe;AACnB,MAAI,eAAe;AAEnB,aAAW,UAAU,MAAM,SAAS;AAClC,QAAI,OAAO,WAAW,QAAS,gBAAe;AAAA,aACrC,OAAO,WAAW,UAAW,iBAAgB;AAAA,QACjD,iBAAgB;AAAA,EACvB;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA,gBAAgB,MAAM,QAAQ;AAAA,EAChC;AACF;AAIA,MAAM,oBAAqB,4BAA4B,MAAQ;AAK/D,gBAAgB,cAAiB,QAA0B,MAAkB,YAAwD;AACnI,QAAM,WAAW,OAAO,OAAO,aAAa,EAAE;AAC9C,MAAI;AACF,WAAO,MAAM;AACX,YAAM,QAAQ,YAAY,MAAM,UAAU;AAC1C,UAAI;AACJ,UAAI;AACF,iBAAS,MAAM,SAAS,KAAK;AAAA,MAC/B,UAAE;AACA,sBAAc,KAAK;AAAA,MACrB;AACA,UAAI,OAAO,KAAM;AACjB,YAAM,OAAO;AAAA,IACf;AAAA,EACF,UAAE;AACA,UAAM,SAAS,SAAS;AAAA,EAC1B;AACF;AAEO,SAAS,iBAAiB,MAAkB;AACjD,QAAM,EAAE,gBAAgB,+BAA+B,uBAAuB,yBAAyB,gBAAgB,IAAI;AAE3H,iBAAe,eAAe,SAA0B,YAAoB,OAAwC;AAClH,WAAO,QAAQ,WAAW;AAAA,MACxB;AAAA,MACA,OAAO,EAAE,gBAAgB,MAAM,gBAAgB,UAAU,MAAM,SAAS;AAAA,IAC1E,CAAC;AAAA,EACH;AAEA,iBAAe,eAAe,eAA0C,gBAAwB,YAA2B,OAAiC;AAC1J,QAAI,CAAC,cAAe;AAEpB,UAAM,gBAAgB;AAAA,MACpB;AAAA,MACA;AAAA,QACE;AAAA,QACA,YAAY,cAAc;AAAA,MAC5B;AAAA,MACA;AAAA,QACE,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,QACtB,QAAQ,MAAM;AAAA,MAChB;AAAA,IACF;AAAA,EACF;AASA,iBAAe,mBAAmB,eAA0C,OAAmC;AAC7G,QAAI,CAAC,cAAe,QAAO;AAC3B,UAAM,MAAM,MAAM,gBAAgB,OAAO,eAAe;AAAA,MACtD,UAAU,MAAM;AAAA,MAChB,gBAAgB,MAAM;AAAA,MACtB,QAAQ,MAAM;AAAA,IAChB,CAAC;AACD,WAAO,KAAK,kBAAkB;AAAA,EAChC;AAEA,WAAS,kBAAkB,eAA0C,OAA8B;AACjG,UAAM,oBAAoB,gBAAgB,mBAAmB,KAAK,eAAe;AACjF,QAAI,CAAC,iBAAiB,CAAC,kBAAmB,QAAO,MAAM;AAAA,IAAC;AACxD,QAAI,WAAW;AACf,WAAO,MAAM;AACX,UAAI,SAAU;AACd,iBAAW;AACX,wBAAkB,eAAe;AAAA,QAC/B,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,QACtB,QAAQ,MAAM;AAAA,MAChB,CAAC,EACE,MAAM,CAAC,UAAU;AAChB,eAAO,KAAK,6BAA6B;AAAA,UACvC;AAAA,UACA,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AAAA,MACH,CAAC,EACA,QAAQ,MAAM;AACb,mBAAW;AAAA,MACb,CAAC;AAAA,IACL;AAAA,EACF;AAEA,iBAAe,yBAAyB,aAAmC,OAAiC;AAC1G,QAAI,CAAC,eAAe,YAAY,WAAW,EAAG;AAE9C,UAAM,QAAQ;AAAA,MACZ,MAAM,KAAK,IAAI,IAAI,YAAY,OAAO,CAAC,UAAU,OAAO,UAAU,YAAY,MAAM,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,EACpG,IAAI,CAAC,eAAe,wBAAwB,KAAK,IAAI;AAAA,QACpD;AAAA,QACA,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,MACxB,CAAC,CAAC;AAAA,IACN;AAAA,EACF;AAEA,iBAAe,sBACb,OACA,eACA,OACA,OACe;AACf,UAAM,cAAc,MAAM,OAAO,CAAC,SAAS,KAAK,WAAW,QAAQ;AACnE,eAAW,QAAQ,aAAa;AAC9B,YAAM,eAAe,OAAO,KAAK,KAAK,iBAAiB,YAAY,KAAK,KAAK,aAAa,KAAK,EAAE,SAAS,IACtG,KAAK,KAAK,aAAa,KAAK,IAC5B;AACJ,YAAM,oBAAoB,OAAO,KAAK,KAAK,sBAAsB,YAAY,KAAK,KAAK,kBAAkB,KAAK,EAAE,SAAS,IACrH,KAAK,KAAK,kBAAkB,KAAK,IACjC;AACJ,YAAM,mBAAmB,OAAO,KAAK,KAAK,qBAAqB,YAAY,KAAK,KAAK,iBAAiB,KAAK,EAAE,SAAS,IAClH,KAAK,KAAK,iBAAiB,KAAK,IAChC;AACJ,YAAM,UAAU;AAAA,QACd,yBAAyB,KAAK,UAAU;AAAA,QACxC,oBAAoB,UAAU,iBAAiB,MAAM;AAAA,QACrD,mBAAmB,gBAAgB,gBAAgB,MAAM;AAAA,QACzD,KAAK,YAAY;AAAA,MACnB,EAAE,OAAO,CAAC,SAAS,SAAS,IAAI,EAAE,KAAK,GAAG;AAE1C,YAAM,sBAAsB;AAAA,QAC1B;AAAA,UACE;AAAA,UACA;AAAA,UACA,OAAO;AAAA,UACP;AAAA,UACA,SAAS,KAAK;AAAA,QAChB;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,iBAAe,sBACb,OACA,eACA,SACA,OACe;AACf,UAAM,gBAAgB,QAAQ,OAAO,CAAC,WAAW,OAAO,WAAW,WAAW,OAAO,KAAK;AAC1F,eAAW,UAAU,eAAe;AAClC,YAAM,QAAQ,OAAO,aAAa,GAAG,OAAO,UAAU,SAAS,OAAO,OAAO,MAAM,OAAO;AAC1F,YAAM,eAAe,OAAO,MAAO,MAAM,IAAI,EAAE,CAAC;AAChD,YAAM,UAAU,yBAAyB,KAAK,KAAK,YAAY;AAE/D,YAAM,sBAAsB;AAAA,QAC1B;AAAA,UACE;AAAA,UACA;AAAA,UACA,OAAO;AAAA,UACP;AAAA,UACA,SAAS,EAAE,MAAM,uBAAuB,SAAS,OAAO,MAAM;AAAA,QAChE;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,iBAAe,oBAAoB,QAQjB;AAChB,QAAI,CAAC,OAAO,QAAS;AAErB,UAAM,sBAAsB;AAAA,MAC1B;AAAA,QACE,eAAe,OAAO;AAAA,QACtB,OAAO,OAAO;AAAA,QACd,OAAO,OAAO;AAAA,QACd,SAAS,OAAO;AAAA,QAChB,SAAS,OAAO;AAAA,MAClB;AAAA,MACA,OAAO;AAAA,IACT;AAAA,EACF;AAEA,iBAAe,uBAAuB,QAKpB;AAChB,QAAI,CAAC,OAAO,WAAW,CAAC,wBAAyB;AAEjD,UAAM,wBAAwB;AAAA,MAC5B,OAAO;AAAA,MACP;AAAA,QACE,kBAAkB,OAAO;AAAA,QACzB,qBAAqB,oBAAI,KAAK;AAAA,MAChC;AAAA,MACA,OAAO;AAAA,IACT;AAAA,EACF;AAEA,iBAAe,YACb,OACA,QACA,OACA,OACA,uBAAuB,OACR;AACf,UAAM,cAAc,MAAM,eAAe,OAAO,OAAO,KAAK;AAC5D,UAAM,iCAAiC,aAAa,WAAW,WACzD,WAAW,eAAe,WAAW,YAAY,WAAW;AAElE,UAAM,MAAM,MAAM,eAAe,WAAW,OAAO,QAAQ,OAAO,KAAK;AACvE,QAAI,CAAC,IAAK;AAEV,QAAI,gCAAgC;AAClC;AAAA,IACF;AAEA,QAAI,IAAI,WAAW,QAAQ;AAOzB,aAAO,KAAK,wEAAwE;AAAA,QAClF;AAAA,QACA,iBAAiB;AAAA,QACjB,cAAc,IAAI;AAAA,MACpB,CAAC;AACD;AAAA,IACF;AAEA,QAAI,IAAI,eAAe;AACrB,UAAI,WAAW,aAAa;AAC1B,cAAM,gBAAgB;AAAA,UACpB,IAAI;AAAA,UACJ;AAAA,YACE,eAAe;AAAA,cACb,cAAc,IAAI;AAAA,cAClB,cAAc,IAAI;AAAA,cAClB,cAAc,IAAI;AAAA,cAClB,aAAa,IAAI;AAAA,cACjB,kBAAkB,IAAI;AAAA,YACxB;AAAA,UACF;AAAA,UACA;AAAA,YACE,UAAU,MAAM;AAAA,YAChB,gBAAgB,MAAM;AAAA,YACtB,QAAQ,MAAM;AAAA,UAChB;AAAA,QACF;AAAA,MACF,WAAW,WAAW,UAAU;AAC9B,cAAM,gBAAgB;AAAA,UACpB,IAAI;AAAA,UACJ;AAAA,YACE,cAAc,SAAS;AAAA,UACzB;AAAA,UACA;AAAA,YACE,UAAU,MAAM;AAAA,YAChB,gBAAgB,MAAM;AAAA,YACtB,QAAQ,MAAM;AAAA,UAChB;AAAA,QACF;AAAA,MACF,WAAW,WAAW,aAAa;AACjC,cAAM,gBAAgB;AAAA,UACpB,IAAI;AAAA,UACJ;AAAA,YACE,UAAU,MAAM;AAAA,YAChB,gBAAgB,MAAM;AAAA,YACtB,QAAQ,MAAM;AAAA,UAChB;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,WAAW,aAAa;AAC1B,YAAM,uBAAuB;AAAA,QAC3B,eAAe,IAAI;AAAA,QACnB,QAAQ;AAAA,QACR;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AACD,YAAM,oBAAoB;AAAA,QACxB,eAAe,IAAI;AAAA,QACnB,OAAO,IAAI;AAAA,QACX,OAAO;AAAA,QACP,SAAS;AAAA,QACT;AAAA,QACA,SAAS;AAAA,QACT,SAAS;AAAA,UACP,mBAAmB;AAAA,UACnB,SAAS,uBAAuB,IAAI,YAAY,aAAa,IAAI,YAAY,aAAa,IAAI,WAAW;AAAA,UACzG,cAAc,IAAI;AAAA,UAClB,cAAc,IAAI;AAAA,UAClB,cAAc,IAAI;AAAA,UAClB,aAAa,IAAI;AAAA,UACjB,kBAAkB,IAAI;AAAA,QACxB;AAAA,MACF,CAAC;AAAA,IACH,WAAW,WAAW,aAAa;AACjC,YAAM,uBAAuB;AAAA,QAC3B,eAAe,IAAI;AAAA,QACnB,QAAQ;AAAA,QACR;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AACD,YAAM,oBAAoB;AAAA,QACxB,eAAe,IAAI;AAAA,QACnB,OAAO,IAAI;AAAA,QACX,OAAO;AAAA,QACP,SAAS;AAAA,QACT;AAAA,QACA,SAAS;AAAA,QACT,SAAS;AAAA,UACP,mBAAmB;AAAA,UACnB,SAAS;AAAA,QACX;AAAA,MACF,CAAC;AAAA,IACH,OAAO;AACL,YAAM,uBAAuB;AAAA,QAC3B,eAAe,IAAI;AAAA,QACnB,QAAQ;AAAA,QACR;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AACD,YAAM,oBAAoB;AAAA,QACxB,eAAe,IAAI;AAAA,QACnB,OAAO,IAAI;AAAA,QACX,OAAO;AAAA,QACP,SAAS,SAAS;AAAA,QAClB;AAAA,QACA,SAAS;AAAA,QACT,SAAS;AAAA,UACP,mBAAmB;AAAA,UACnB,SAAS,SAAS;AAAA,QACpB;AAAA,MACF,CAAC;AAAA,IACH;AAEA,QAAI,WAAW,aAAa;AAC1B,YAAM,kBAAkB,2BAA2B;AAAA,QACjD;AAAA,QACA,eAAe,IAAI;AAAA,QACnB,YAAY,IAAI;AAAA,QAChB,WAAW,IAAI;AAAA,QACf,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,MACxB,CAAC;AACD;AAAA,IACF;AAEA,QAAI,WAAW,aAAa;AAC1B,YAAM,kBAAkB,2BAA2B;AAAA,QACjD;AAAA,QACA,eAAe,IAAI;AAAA,QACnB,YAAY,IAAI;AAAA,QAChB,WAAW,IAAI;AAAA,QACf,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,MACxB,CAAC;AACD;AAAA,IACF;AAEA,UAAM,kBAAkB,wBAAwB;AAAA,MAC9C;AAAA,MACA,eAAe,IAAI;AAAA,MACnB,YAAY,IAAI;AAAA,MAChB,WAAW,IAAI;AAAA,MACf,OAAO,SAAS;AAAA,MAChB,UAAU,MAAM;AAAA,MAChB,gBAAgB,MAAM;AAAA,IACxB,CAAC;AAAA,EACH;AAEA,SAAO;AAAA,IACL,MAAM,UAAU,OAAe,WAAmB,OAAiC;AACjF,YAAM,MAAM,MAAM,eAAe,OAAO,OAAO,KAAK;AACpD,UAAI,CAAC,KAAK;AACR,eAAO,KAAK,6CAA6C,EAAE,MAAM,CAAC;AAClE;AAAA,MACF;AACA,UAAI,IAAI,WAAW,aAAa;AAC9B,YAAI,IAAI,eAAe;AACrB,gBAAM,gBAAgB,cAAc,IAAI,eAAe;AAAA,YACrD,UAAU,MAAM;AAAA,YAChB,gBAAgB,MAAM;AAAA,YACtB,QAAQ,MAAM;AAAA,UAChB,CAAC;AAAA,QACH;AACA;AAAA,MACF;AAEA,YAAM,cAAc,mBAAmB,IAAI,aAAa;AACxD,YAAM,UAAU,mBAAmB,WAAW;AAC9C,UAAI,CAAC,SAAS,cAAc;AAC1B,cAAM,IAAI,MAAM,6CAA6C,WAAW,EAAE;AAAA,MAC5E;AACA,YAAM,uBAAuB,QAAQ,yBAAyB;AAE9D,YAAM,cAAc,MAAM,8BAA8B,QAAQ,IAAI,eAAe,KAAK;AACxF,UAAI,CAAC,aAAa;AAChB,cAAM,IAAI,MAAM,eAAe,IAAI,aAAa,yBAAyB;AAAA,MAC3E;AAKA,YAAM,UAAU,IAAI,WAAW;AAC/B,YAAM,YAAY,MAAM,eAAe,WAAW,IAAI,IAAI,WAAW,KAAK;AAC1E,UAAI,CAAC,aAAa,UAAU,WAAW,WAAW;AAChD;AAAA,MACF;AACA,YAAM,kBAAkB,yBAAyB;AAAA,QAC/C,OAAO,IAAI;AAAA,QACX,eAAe,IAAI;AAAA,QACnB,YAAY,IAAI;AAAA,QAChB,WAAW,IAAI;AAAA,QACf;AAAA,QACA,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,MACxB,CAAC;AACD,YAAM,uBAAuB;AAAA,QAC3B,eAAe,IAAI;AAAA,QACnB,QAAQ;AAAA,QACR;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AACD,YAAM,oBAAoB;AAAA,QACxB,eAAe,IAAI;AAAA,QACnB,OAAO,IAAI;AAAA,QACX,OAAO;AAAA,QACP,SAAS;AAAA,QACT;AAAA,QACA,SAAS;AAAA,QACT,SAAS;AAAA,UACP,mBAAmB;AAAA,UACnB,SAAS,0BAA0B,IAAI,UAAU;AAAA,UACjD,YAAY,IAAI;AAAA,UAChB,WAAW,IAAI;AAAA,QACjB;AAAA,MACF,CAAC;AAED,UAAI,IAAI,eAAe;AACrB,cAAM,gBAAgB,SAAS,IAAI,eAAe;AAAA,UAChD,UAAU,MAAM;AAAA,UAChB,gBAAgB,MAAM;AAAA,UACtB,QAAQ,MAAM;AAAA,QAChB,CAAC;AAAA,MACH;AAEA,YAAM,UAAU,MAAM,eAAe,SAAS,IAAI,YAAY,KAAK;AACnE,UAAI,iBAAiB,MAAM,mBAAmB,IAAI,eAAe,KAAK;AACtE,UAAI,aAA4B;AAChC,UAAI,mBAAmB,UAAU,oBAAoB;AAErD,UAAI;AACF,yBAAiB,SAAS;AAAA,UACxB,QAAQ,aAAa;AAAA,YACnB,YAAY,IAAI;AAAA,YAChB,QAAQ,IAAI,UAAU;AAAA,YACtB;AAAA,YACA;AAAA,YACA;AAAA,YACA,OAAO,EAAE,gBAAgB,MAAM,gBAAgB,UAAU,MAAM,SAAS;AAAA,YACxE,OAAO,IAAI;AAAA,UACb,CAAC;AAAA,UACD,kBAAkB,IAAI,eAAe,KAAK;AAAA,UAC1C;AAAA,QACF,GAAG;AACD,cAAI,IAAI,iBAAiB,MAAM,gBAAgB,wBAAwB,IAAI,eAAe,MAAM,UAAU,MAAM,cAAc,GAAG;AAC/H,kBAAM,YAAY,IAAI,IAAI,aAAa,OAAO,QAAW,oBAAoB;AAC7E;AAAA,UACF;AAEA,gBAAM,QAAQ,oBAAoB,KAAK;AACvC,gBAAM,sBAAsB,MAAM,kBAAkB,MAAM,MAAM;AAChE,4BAAkB;AAClB,uBAAa,MAAM,iBAAiB;AAEpC,gBAAM,eAAe;AAAA,YACnB,IAAI;AAAA,YACJ;AAAA,cACE,GAAG;AAAA,cACH,kBAAkB;AAAA,YACpB;AAAA,YACA,MAAM;AAAA,YACN;AAAA,YACA;AAAA,UACF;AACA,8BAAoB;AAEpB,gBAAM,eAAe,IAAI,eAAe,gBAAgB,YAAY,KAAK;AACzE,gBAAM,yBAAyB,MAAM,4BAA4B,KAAK;AACtE,gBAAM,sBAAsB,IAAI,IAAI,IAAI,eAAe,MAAM,OAAO,KAAK;AAEzE,gBAAM,oBAAoB;AAAA,YACxB,eAAe,IAAI;AAAA,YACnB,OAAO,IAAI;AAAA,YACX,OAAO;AAAA,YACP,SAAS,MAAM,SAAS,KAAK,EAAE,SAC3B,MAAM,QAAQ,KAAK,IACnB,0BAA0B,MAAM,UAAU;AAAA,YAC9C;AAAA,YACA,SAAS;AAAA,YACT,SAAS;AAAA,cACP,mBAAmB;AAAA,cACnB,SAAS,aAAa,cAAc,GAAG,aAAa,OAAO,UAAU,KAAK,EAAE;AAAA,cAC5E;AAAA,cACA,WAAW,MAAM,MAAM;AAAA,cACvB;AAAA,cACA,QAAQ,MAAM;AAAA,YAChB;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF,SAAS,OAAO;AACd,YAAI,iBAAiB,+BAA+B;AAClD,iBAAO,KAAK,uEAAuE;AAAA,YACjF,OAAO,IAAI;AAAA,YACX,0BAA0B,MAAM;AAAA,UAClC,CAAC;AACD;AAAA,QACF;AACA,cAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU;AACzD,cAAM,sBAAsB;AAAA,UAC1B;AAAA,YACE,eAAe,IAAI;AAAA,YACnB,OAAO,IAAI;AAAA,YACX,OAAO;AAAA,YACP;AAAA,UACF;AAAA,UACA;AAAA,QACF;AACA,cAAM,YAAY,IAAI,IAAI,UAAU,OAAO,SAAS,oBAAoB;AACxE;AAAA,MACF;AAEA,YAAM,YAAY,IAAI,IAAI,aAAa,OAAO,QAAW,oBAAoB;AAAA,IAC/E;AAAA,IAEA,MAAM,UAAU,OAAe,WAAmB,OAAiC;AACjF,YAAM,MAAM,MAAM,eAAe,OAAO,OAAO,KAAK;AACpD,UAAI,CAAC,KAAK;AACR,eAAO,KAAK,6CAA6C,EAAE,MAAM,CAAC;AAClE;AAAA,MACF;AACA,UAAI,IAAI,WAAW,aAAa;AAC9B,YAAI,IAAI,eAAe;AACrB,gBAAM,gBAAgB,cAAc,IAAI,eAAe;AAAA,YACrD,UAAU,MAAM;AAAA,YAChB,gBAAgB,MAAM;AAAA,YACtB,QAAQ,MAAM;AAAA,UAChB,CAAC;AAAA,QACH;AACA;AAAA,MACF;AAEA,YAAM,cAAc,mBAAmB,IAAI,aAAa;AACxD,YAAM,UAAU,mBAAmB,WAAW;AAC9C,UAAI,CAAC,SAAS,cAAc;AAC1B,cAAM,IAAI,MAAM,6CAA6C,WAAW,EAAE;AAAA,MAC5E;AACA,YAAM,uBAAuB,QAAQ,yBAAyB;AAE9D,YAAM,cAAc,MAAM,8BAA8B,QAAQ,IAAI,eAAe,KAAK;AACxF,UAAI,CAAC,aAAa;AAChB,cAAM,IAAI,MAAM,eAAe,IAAI,aAAa,yBAAyB;AAAA,MAC3E;AAKA,YAAM,UAAU,IAAI,WAAW;AAC/B,YAAM,YAAY,MAAM,eAAe,WAAW,IAAI,IAAI,WAAW,KAAK;AAC1E,UAAI,CAAC,aAAa,UAAU,WAAW,WAAW;AAChD;AAAA,MACF;AACA,YAAM,kBAAkB,yBAAyB;AAAA,QAC/C,OAAO,IAAI;AAAA,QACX,eAAe,IAAI;AAAA,QACnB,YAAY,IAAI;AAAA,QAChB,WAAW,IAAI;AAAA,QACf;AAAA,QACA,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,MACxB,CAAC;AACD,YAAM,uBAAuB;AAAA,QAC3B,eAAe,IAAI;AAAA,QACnB,QAAQ;AAAA,QACR;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AACD,YAAM,oBAAoB;AAAA,QACxB,eAAe,IAAI;AAAA,QACnB,OAAO,IAAI;AAAA,QACX,OAAO;AAAA,QACP,SAAS;AAAA,QACT;AAAA,QACA,SAAS;AAAA,QACT,SAAS;AAAA,UACP,mBAAmB;AAAA,UACnB,SAAS,0BAA0B,IAAI,UAAU;AAAA,UACjD,YAAY,IAAI;AAAA,UAChB,WAAW,IAAI;AAAA,QACjB;AAAA,MACF,CAAC;AAED,UAAI,IAAI,eAAe;AACrB,cAAM,gBAAgB,SAAS,IAAI,eAAe;AAAA,UAChD,UAAU,MAAM;AAAA,UAChB,gBAAgB,MAAM;AAAA,UACtB,QAAQ,MAAM;AAAA,QAChB,CAAC;AAAA,MACH;AAEA,YAAM,UAAU,MAAM,eAAe,SAAS,IAAI,YAAY,KAAK;AACnE,UAAI,iBAAiB,MAAM,mBAAmB,IAAI,eAAe,KAAK;AACtE,UAAI,mBAAmB,UAAU,oBAAoB;AAErD,UAAI;AACF,yBAAiB,SAAS;AAAA,UACxB,QAAQ,aAAa;AAAA,YACnB,YAAY,IAAI;AAAA,YAChB,QAAQ,IAAI,UAAU;AAAA,YACtB;AAAA,YACA;AAAA,YACA;AAAA,YACA,OAAO,EAAE,gBAAgB,MAAM,gBAAgB,UAAU,MAAM,SAAS;AAAA,YACxE,OAAO,IAAI;AAAA,UACb,CAAC;AAAA,UACD,kBAAkB,IAAI,eAAe,KAAK;AAAA,UAC1C;AAAA,QACF,GAAG;AACD,cAAI,IAAI,iBAAiB,MAAM,gBAAgB,wBAAwB,IAAI,eAAe,MAAM,UAAU,MAAM,cAAc,GAAG;AAC/H,kBAAM,YAAY,IAAI,IAAI,aAAa,OAAO,QAAW,oBAAoB;AAC7E;AAAA,UACF;AAEA,gBAAM,QAAQ,oBAAoB,KAAK;AACvC,4BAAkB,MAAM;AAExB,gBAAM,eAAe;AAAA,YACnB,IAAI;AAAA,YACJ;AAAA,cACE,cAAc;AAAA,cACd,cAAc,MAAM;AAAA,cACpB,cAAc,MAAM;AAAA,cACpB,aAAa,MAAM;AAAA,cACnB,kBAAkB;AAAA,YACpB;AAAA,YACA,MAAM;AAAA,YACN;AAAA,YACA;AAAA,UACF;AACA,8BAAoB;AACpB,gBAAM,eAAe,IAAI,eAAe,gBAAgB,MAAM,KAAK;AACnE,gBAAM,sBAAsB,IAAI,IAAI,IAAI,eAAe,MAAM,SAAS,KAAK;AAE3E,gBAAM,oBAAoB;AAAA,YACxB,eAAe,IAAI;AAAA,YACnB,OAAO,IAAI;AAAA,YACX,OAAO;AAAA,YACP,SAAS,0BAA0B,MAAM,UAAU;AAAA,YACnD;AAAA,YACA,SAAS;AAAA,YACT,SAAS;AAAA,cACP,mBAAmB;AAAA,cACnB,SAAS,aAAa,cAAc;AAAA,cACpC;AAAA,cACA,WAAW,MAAM,QAAQ;AAAA,cACzB,QAAQ,MAAM;AAAA,YAChB;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF,SAAS,OAAO;AACd,YAAI,iBAAiB,+BAA+B;AAClD,iBAAO,KAAK,uEAAuE;AAAA,YACjF,OAAO,IAAI;AAAA,YACX,0BAA0B,MAAM;AAAA,UAClC,CAAC;AACD;AAAA,QACF;AACA,cAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU;AACzD,cAAM,sBAAsB;AAAA,UAC1B;AAAA,YACE,eAAe,IAAI;AAAA,YACnB,OAAO,IAAI;AAAA,YACX,OAAO;AAAA,YACP;AAAA,UACF;AAAA,UACA;AAAA,QACF;AACA,cAAM,YAAY,IAAI,IAAI,UAAU,OAAO,SAAS,oBAAoB;AACxE;AAAA,MACF;AAEA,YAAM,YAAY,IAAI,IAAI,aAAa,OAAO,QAAW,oBAAoB;AAAA,IAC/E;AAAA,EACF;AACF;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
const HEARTBEAT_INTERVAL_MS = 5e3;
|
|
2
2
|
const STALE_JOB_TIMEOUT_SECONDS = 60;
|
|
3
3
|
const STALE_PENDING_TIMEOUT_SECONDS = 900;
|
|
4
|
+
const STALE_SWEEP_ERROR_PREFIX = "Job stale:";
|
|
4
5
|
function calculateEta(processedCount, totalCount, startedAt) {
|
|
5
6
|
if (processedCount === 0 || totalCount === 0) return null;
|
|
6
7
|
const elapsedMs = Date.now() - startedAt.getTime();
|
|
@@ -17,6 +18,7 @@ export {
|
|
|
17
18
|
HEARTBEAT_INTERVAL_MS,
|
|
18
19
|
STALE_JOB_TIMEOUT_SECONDS,
|
|
19
20
|
STALE_PENDING_TIMEOUT_SECONDS,
|
|
21
|
+
STALE_SWEEP_ERROR_PREFIX,
|
|
20
22
|
calculateEta,
|
|
21
23
|
calculateProgressPercent
|
|
22
24
|
};
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../../src/modules/progress/lib/progressService.ts"],
|
|
4
|
-
"sourcesContent": ["import type { ProgressJob } from '../data/entities'\nimport type { CreateProgressJobInput, UpdateProgressInput, CompleteJobInput, FailJobInput } from '../data/validators'\n\nexport interface ProgressServiceContext {\n tenantId: string\n organizationId?: string | null\n userId?: string | null\n}\n\nexport interface ProgressService {\n createJob(input: CreateProgressJobInput, ctx: ProgressServiceContext): Promise<ProgressJob>\n startJob(jobId: string, ctx: ProgressServiceContext): Promise<ProgressJob>\n updateProgress(jobId: string, input: UpdateProgressInput, ctx: ProgressServiceContext): Promise<ProgressJob>\n incrementProgress(jobId: string, delta: number, ctx: ProgressServiceContext): Promise<ProgressJob>\n completeJob(jobId: string, input: CompleteJobInput | undefined, ctx: ProgressServiceContext): Promise<ProgressJob>\n failJob(jobId: string, input: FailJobInput, ctx: ProgressServiceContext): Promise<ProgressJob>\n cancelJob(jobId: string, ctx: ProgressServiceContext): Promise<ProgressJob>\n markCancelled(jobId: string, ctx: ProgressServiceContext): Promise<ProgressJob>\n isCancellationRequested(jobId: string, tenantId: string, organizationId?: string | null): Promise<boolean>\n getActiveJobs(ctx: ProgressServiceContext): Promise<ProgressJob[]>\n getRecentlyCompletedJobs(ctx: ProgressServiceContext, sinceSeconds?: number): Promise<ProgressJob[]>\n getJob(jobId: string, ctx: ProgressServiceContext): Promise<ProgressJob | null>\n markStaleJobsFailed(tenantId: string, timeoutSeconds?: number, organizationId?: string | null): Promise<number>\n}\n\nexport const HEARTBEAT_INTERVAL_MS = 5000\nexport const STALE_JOB_TIMEOUT_SECONDS = 60\nexport const STALE_PENDING_TIMEOUT_SECONDS = 900\n\nexport function calculateEta(\n processedCount: number,\n totalCount: number,\n startedAt: Date,\n): number | null {\n if (processedCount === 0 || totalCount === 0) return null\n\n const elapsedMs = Date.now() - startedAt.getTime()\n const rate = processedCount / elapsedMs\n const remaining = totalCount - processedCount\n\n if (rate <= 0) return null\n\n return Math.ceil(remaining / rate / 1000)\n}\n\nexport function calculateProgressPercent(processedCount: number, totalCount: number | null): number {\n if (!totalCount || totalCount <= 0) return 0\n return Math.min(100, Math.round((processedCount / totalCount) * 100))\n}\n"],
|
|
5
|
-
"mappings": "
|
|
4
|
+
"sourcesContent": ["import type { ProgressJob } from '../data/entities'\nimport type { CreateProgressJobInput, UpdateProgressInput, CompleteJobInput, FailJobInput } from '../data/validators'\n\nexport interface ProgressServiceContext {\n tenantId: string\n organizationId?: string | null\n userId?: string | null\n}\n\nexport interface ProgressService {\n createJob(input: CreateProgressJobInput, ctx: ProgressServiceContext): Promise<ProgressJob>\n startJob(jobId: string, ctx: ProgressServiceContext): Promise<ProgressJob>\n updateProgress(jobId: string, input: UpdateProgressInput, ctx: ProgressServiceContext): Promise<ProgressJob>\n incrementProgress(jobId: string, delta: number, ctx: ProgressServiceContext): Promise<ProgressJob>\n completeJob(jobId: string, input: CompleteJobInput | undefined, ctx: ProgressServiceContext): Promise<ProgressJob>\n failJob(jobId: string, input: FailJobInput, ctx: ProgressServiceContext): Promise<ProgressJob>\n cancelJob(jobId: string, ctx: ProgressServiceContext): Promise<ProgressJob>\n markCancelled(jobId: string, ctx: ProgressServiceContext): Promise<ProgressJob>\n isCancellationRequested(jobId: string, tenantId: string, organizationId?: string | null): Promise<boolean>\n getActiveJobs(ctx: ProgressServiceContext): Promise<ProgressJob[]>\n getRecentlyCompletedJobs(ctx: ProgressServiceContext, sinceSeconds?: number): Promise<ProgressJob[]>\n getJob(jobId: string, ctx: ProgressServiceContext): Promise<ProgressJob | null>\n markStaleJobsFailed(tenantId: string, timeoutSeconds?: number, organizationId?: string | null): Promise<number>\n // Optional so third-party ProgressService implementations keep compiling; callers must\n // optional-chain. Runs on a forked EntityManager, so it is safe to call while the shared\n // request/worker EM is mid-transaction (e.g. from a keepalive timer around adapter I/O).\n touchJobHeartbeat?(jobId: string, ctx: ProgressServiceContext): Promise<void>\n}\n\nexport const HEARTBEAT_INTERVAL_MS = 5000\nexport const STALE_JOB_TIMEOUT_SECONDS = 60\nexport const STALE_PENDING_TIMEOUT_SECONDS = 900\n\n// Every `errorMessage` the stale sweep writes starts with this, so recovery paths can tell\n// a job the sweep gave up on from one that failed for a real reason and must keep its\n// diagnostics. Declared here so the sweep and the revive filter cannot drift apart.\nexport const STALE_SWEEP_ERROR_PREFIX = 'Job stale:'\n\nexport function calculateEta(\n processedCount: number,\n totalCount: number,\n startedAt: Date,\n): number | null {\n if (processedCount === 0 || totalCount === 0) return null\n\n const elapsedMs = Date.now() - startedAt.getTime()\n const rate = processedCount / elapsedMs\n const remaining = totalCount - processedCount\n\n if (rate <= 0) return null\n\n return Math.ceil(remaining / rate / 1000)\n}\n\nexport function calculateProgressPercent(processedCount: number, totalCount: number | null): number {\n if (!totalCount || totalCount <= 0) return 0\n return Math.min(100, Math.round((processedCount / totalCount) * 100))\n}\n"],
|
|
5
|
+
"mappings": "AA6BO,MAAM,wBAAwB;AAC9B,MAAM,4BAA4B;AAClC,MAAM,gCAAgC;AAKtC,MAAM,2BAA2B;AAEjC,SAAS,aACd,gBACA,YACA,WACe;AACf,MAAI,mBAAmB,KAAK,eAAe,EAAG,QAAO;AAErD,QAAM,YAAY,KAAK,IAAI,IAAI,UAAU,QAAQ;AACjD,QAAM,OAAO,iBAAiB;AAC9B,QAAM,YAAY,aAAa;AAE/B,MAAI,QAAQ,EAAG,QAAO;AAEtB,SAAO,KAAK,KAAK,YAAY,OAAO,GAAI;AAC1C;AAEO,SAAS,yBAAyB,gBAAwB,YAAmC;AAClG,MAAI,CAAC,cAAc,cAAc,EAAG,QAAO;AAC3C,SAAO,KAAK,IAAI,KAAK,KAAK,MAAO,iBAAiB,aAAc,GAAG,CAAC;AACtE;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
|
@@ -5,7 +5,8 @@ import {
|
|
|
5
5
|
calculateProgressPercent,
|
|
6
6
|
HEARTBEAT_INTERVAL_MS,
|
|
7
7
|
STALE_JOB_TIMEOUT_SECONDS,
|
|
8
|
-
STALE_PENDING_TIMEOUT_SECONDS
|
|
8
|
+
STALE_PENDING_TIMEOUT_SECONDS,
|
|
9
|
+
STALE_SWEEP_ERROR_PREFIX
|
|
9
10
|
} from "./progressService.js";
|
|
10
11
|
import { PROGRESS_EVENTS } from "./events.js";
|
|
11
12
|
import { findOneWithDecryption } from "@open-mercato/shared/lib/encryption/find";
|
|
@@ -90,6 +91,37 @@ function createProgressService(em, eventBus) {
|
|
|
90
91
|
}
|
|
91
92
|
return {};
|
|
92
93
|
}
|
|
94
|
+
async function startJobViaCas(targetEm, job, ctx, options = {}) {
|
|
95
|
+
const now = /* @__PURE__ */ new Date();
|
|
96
|
+
const startedAt = job.startedAt ?? now;
|
|
97
|
+
const filter = {
|
|
98
|
+
...jobScopeFilter(job.id, ctx),
|
|
99
|
+
status: { $in: START_FROM_STATUSES }
|
|
100
|
+
};
|
|
101
|
+
if (options.staleSweptOnly) filter.errorMessage = { $like: `${STALE_SWEEP_ERROR_PREFIX}%` };
|
|
102
|
+
const affected = await targetEm.nativeUpdate(ProgressJob, filter, {
|
|
103
|
+
status: "running",
|
|
104
|
+
startedAt,
|
|
105
|
+
heartbeatAt: now,
|
|
106
|
+
finishedAt: null,
|
|
107
|
+
errorMessage: null,
|
|
108
|
+
errorStack: null,
|
|
109
|
+
updatedAt: now
|
|
110
|
+
});
|
|
111
|
+
if (affected === 0) return null;
|
|
112
|
+
job.status = "running";
|
|
113
|
+
job.startedAt = startedAt;
|
|
114
|
+
job.heartbeatAt = now;
|
|
115
|
+
job.finishedAt = null;
|
|
116
|
+
job.errorMessage = null;
|
|
117
|
+
job.errorStack = null;
|
|
118
|
+
await eventBus.emit(PROGRESS_EVENTS.JOB_STARTED, {
|
|
119
|
+
...buildJobPayload(job),
|
|
120
|
+
tenantId: ctx.tenantId,
|
|
121
|
+
organizationId: job.organizationId ?? null
|
|
122
|
+
});
|
|
123
|
+
return job;
|
|
124
|
+
}
|
|
93
125
|
async function persistAndMaybeBroadcast(entry, ctx) {
|
|
94
126
|
const job = entry.job;
|
|
95
127
|
const now = Date.now();
|
|
@@ -106,14 +138,28 @@ function createProgressService(em, eventBus) {
|
|
|
106
138
|
updatedAt: /* @__PURE__ */ new Date(),
|
|
107
139
|
...buildBufferedCountData(entry)
|
|
108
140
|
};
|
|
109
|
-
const
|
|
141
|
+
const writableFilter = {
|
|
110
142
|
...jobScopeFilter(job.id, ctx),
|
|
111
143
|
status: { $in: PROGRESS_WRITABLE_STATUSES }
|
|
112
|
-
}
|
|
144
|
+
};
|
|
145
|
+
let affected = await em.nativeUpdate(ProgressJob, writableFilter, data);
|
|
113
146
|
if (affected === 0) {
|
|
114
|
-
forgetJobThrottle(job.id);
|
|
115
147
|
const fresh = await loadFreshJob(job.id, ctx);
|
|
116
|
-
|
|
148
|
+
let revived = false;
|
|
149
|
+
if (fresh && fresh.status === "failed" && await startJobViaCas(em, fresh, ctx, { staleSweptOnly: true })) {
|
|
150
|
+
revived = true;
|
|
151
|
+
job.status = "running";
|
|
152
|
+
job.startedAt = fresh.startedAt;
|
|
153
|
+
job.finishedAt = null;
|
|
154
|
+
job.errorMessage = null;
|
|
155
|
+
job.errorStack = null;
|
|
156
|
+
affected = await em.nativeUpdate(ProgressJob, writableFilter, data);
|
|
157
|
+
}
|
|
158
|
+
if (affected === 0) {
|
|
159
|
+
forgetJobThrottle(job.id);
|
|
160
|
+
const latest = revived ? await loadFreshJob(job.id, ctx) : fresh;
|
|
161
|
+
return latest ?? job;
|
|
162
|
+
}
|
|
117
163
|
}
|
|
118
164
|
const persistedJob = usesAtomicIncrement ? await loadFreshJob(job.id, ctx) ?? job : job;
|
|
119
165
|
entry.job = persistedJob;
|
|
@@ -161,42 +207,36 @@ function createProgressService(em, eventBus) {
|
|
|
161
207
|
if (job.status === "running" || job.status === "cancelled" || job.status === "completed") {
|
|
162
208
|
return job;
|
|
163
209
|
}
|
|
210
|
+
const started = await startJobViaCas(em, job, ctx);
|
|
211
|
+
if (started) return started;
|
|
212
|
+
const fresh = await loadFreshJob(jobId, ctx);
|
|
213
|
+
return fresh ?? job;
|
|
214
|
+
},
|
|
215
|
+
async touchJobHeartbeat(jobId, ctx) {
|
|
216
|
+
const fork = em.fork();
|
|
164
217
|
const now = /* @__PURE__ */ new Date();
|
|
165
|
-
const affected = await
|
|
218
|
+
const affected = await fork.nativeUpdate(ProgressJob, {
|
|
166
219
|
...jobScopeFilter(jobId, ctx),
|
|
167
|
-
status: { $in:
|
|
220
|
+
status: { $in: PROGRESS_WRITABLE_STATUSES }
|
|
168
221
|
}, {
|
|
169
|
-
status: "running",
|
|
170
|
-
startedAt: now,
|
|
171
222
|
heartbeatAt: now,
|
|
172
|
-
finishedAt: null,
|
|
173
|
-
errorMessage: null,
|
|
174
|
-
errorStack: null,
|
|
175
223
|
updatedAt: now
|
|
176
224
|
});
|
|
177
|
-
if (affected
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
}
|
|
181
|
-
job.status = "running";
|
|
182
|
-
job.startedAt = now;
|
|
183
|
-
job.heartbeatAt = now;
|
|
184
|
-
job.finishedAt = null;
|
|
185
|
-
job.errorMessage = null;
|
|
186
|
-
job.errorStack = null;
|
|
187
|
-
await eventBus.emit(PROGRESS_EVENTS.JOB_STARTED, {
|
|
188
|
-
...buildJobPayload(job),
|
|
189
|
-
tenantId: ctx.tenantId,
|
|
190
|
-
organizationId: job.organizationId ?? null
|
|
191
|
-
});
|
|
192
|
-
return job;
|
|
225
|
+
if (affected > 0) return;
|
|
226
|
+
const job = await fork.findOne(ProgressJob, jobScopeFilter(jobId, ctx), { disableIdentityMap: true });
|
|
227
|
+
if (!job || job.status !== "failed") return;
|
|
228
|
+
await startJobViaCas(fork, job, ctx, { staleSweptOnly: true });
|
|
193
229
|
},
|
|
194
230
|
async updateProgress(jobId, input, ctx) {
|
|
195
231
|
const entry = await ensureThrottleEntry(jobId, ctx);
|
|
196
232
|
const job = entry.job;
|
|
197
233
|
if (TERMINAL_STATUSES.includes(job.status)) {
|
|
198
|
-
|
|
199
|
-
|
|
234
|
+
const revived = job.status === "failed" && await startJobViaCas(em, job, ctx, { staleSweptOnly: true });
|
|
235
|
+
if (!revived) {
|
|
236
|
+
forgetJobThrottle(jobId);
|
|
237
|
+
return job;
|
|
238
|
+
}
|
|
239
|
+
entry.lastPersistedAt = 0;
|
|
200
240
|
}
|
|
201
241
|
if (input.processedCount !== void 0) {
|
|
202
242
|
job.processedCount = input.processedCount;
|
|
@@ -226,8 +266,12 @@ function createProgressService(em, eventBus) {
|
|
|
226
266
|
const entry = await ensureThrottleEntry(jobId, ctx);
|
|
227
267
|
const job = entry.job;
|
|
228
268
|
if (TERMINAL_STATUSES.includes(job.status)) {
|
|
229
|
-
|
|
230
|
-
|
|
269
|
+
const revived = job.status === "failed" && await startJobViaCas(em, job, ctx, { staleSweptOnly: true });
|
|
270
|
+
if (!revived) {
|
|
271
|
+
forgetJobThrottle(jobId);
|
|
272
|
+
return job;
|
|
273
|
+
}
|
|
274
|
+
entry.lastPersistedAt = 0;
|
|
231
275
|
}
|
|
232
276
|
job.processedCount += delta;
|
|
233
277
|
entry.pendingDelta += delta;
|
|
@@ -486,7 +530,7 @@ function createProgressService(em, eventBus) {
|
|
|
486
530
|
};
|
|
487
531
|
const staleRunning = await em.find(ProgressJob, { ...scope, ...staleFilter }, { disableIdentityMap: true });
|
|
488
532
|
for (const job of staleRunning) {
|
|
489
|
-
const errorMessage =
|
|
533
|
+
const errorMessage = `${STALE_SWEEP_ERROR_PREFIX} no heartbeat for ${timeoutSeconds} seconds`;
|
|
490
534
|
const affected = await em.nativeUpdate(ProgressJob, {
|
|
491
535
|
id: job.id,
|
|
492
536
|
tenantId: job.tenantId,
|
|
@@ -517,7 +561,7 @@ function createProgressService(em, eventBus) {
|
|
|
517
561
|
};
|
|
518
562
|
const stalePending = await em.find(ProgressJob, { ...scope, ...stalePendingFilter }, { disableIdentityMap: true });
|
|
519
563
|
for (const job of stalePending) {
|
|
520
|
-
const errorMessage =
|
|
564
|
+
const errorMessage = `${STALE_SWEEP_ERROR_PREFIX} never started within ${STALE_PENDING_TIMEOUT_SECONDS} seconds`;
|
|
521
565
|
const affected = await em.nativeUpdate(ProgressJob, {
|
|
522
566
|
id: job.id,
|
|
523
567
|
tenantId: job.tenantId,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../../src/modules/progress/lib/progressServiceImpl.ts"],
|
|
4
|
-
"sourcesContent": ["import type { EntityManager } from '@mikro-orm/postgresql'\nimport { raw } from '@mikro-orm/core'\nimport type { EntityData, FilterQuery } from '@mikro-orm/core'\nimport { ProgressJob, type ProgressJobStatus } from '../data/entities'\nimport type { ProgressService, ProgressServiceContext } from './progressService'\nimport {\n calculateEta,\n calculateProgressPercent,\n HEARTBEAT_INTERVAL_MS,\n STALE_JOB_TIMEOUT_SECONDS,\n STALE_PENDING_TIMEOUT_SECONDS,\n} from './progressService'\nimport { PROGRESS_EVENTS } from './events'\nimport { findOneWithDecryption } from '@open-mercato/shared/lib/encryption/find'\n\nconst DEFAULT_BROADCAST_MIN_INTERVAL_MS = 250\n\n// Minimum elapsed time between coalesced `progress.job.updated` broadcasts for a\n// single job. Bulk workers call updateProgress/incrementProgress once per record, and\n// every emit of the `clientBroadcast: true` event pays a serialized pg_notify roundtrip\n// plus a tenant-wide SSE fan-out. Setting the knob to 0 restores per-record emission.\n// Persistence is throttled separately: heartbeats hit the database at least every\n// HEARTBEAT_INTERVAL_MS regardless of this knob, so a high broadcast interval can\n// never starve the stale-job sweep of heartbeats.\nfunction resolveBroadcastMinIntervalMs(): number {\n const rawValue = process.env.OM_PROGRESS_BROADCAST_MIN_INTERVAL_MS\n if (rawValue == null || rawValue.trim() === '') return DEFAULT_BROADCAST_MIN_INTERVAL_MS\n const parsed = Number.parseInt(rawValue, 10)\n if (!Number.isFinite(parsed) || parsed < 0) return DEFAULT_BROADCAST_MIN_INTERVAL_MS\n return parsed\n}\n\n// Statuses a given transition is allowed to move FROM. Every write below is a\n// compare-and-swap (`nativeUpdate` guarded on status), so concurrent writers in other\n// processes can never resurrect a terminal job or double-apply a transition \u2014 the\n// UPDATE that matches zero rows simply lost the race and must not emit events.\nconst TERMINAL_STATUSES: readonly ProgressJobStatus[] = ['completed', 'failed', 'cancelled']\nconst PROGRESS_WRITABLE_STATUSES: readonly ProgressJobStatus[] = ['pending', 'running']\n// `failed` is restartable/completable so a job wrongly swept as stale (worker alive but\n// slow) or retried by an at-least-once queue converges to its true outcome.\nconst START_FROM_STATUSES: readonly ProgressJobStatus[] = ['pending', 'failed']\nconst COMPLETE_FROM_STATUSES: readonly ProgressJobStatus[] = ['pending', 'running', 'failed']\nconst FAIL_FROM_STATUSES: readonly ProgressJobStatus[] = ['pending', 'running']\nconst CANCEL_FROM_STATUSES: readonly ProgressJobStatus[] = ['pending', 'running', 'failed']\n\ntype JobUpdateThrottleEntry = {\n job: ProgressJob\n lastBroadcastAt: number\n lastPersistedAt: number\n lastBroadcastPercent: number\n pendingDelta: number\n absoluteCountsPending: boolean\n}\n\nfunction buildJobPayload(job: ProgressJob): Record<string, unknown> {\n return {\n jobId: job.id,\n jobType: job.jobType,\n name: job.name,\n description: job.description ?? null,\n status: job.status,\n progressPercent: job.progressPercent,\n processedCount: job.processedCount,\n totalCount: job.totalCount ?? null,\n etaSeconds: job.etaSeconds ?? null,\n cancellable: job.cancellable,\n meta: job.meta ?? null,\n startedAt: job.startedAt?.toISOString() ?? null,\n finishedAt: job.finishedAt?.toISOString() ?? null,\n }\n}\n\nexport function createProgressService(em: EntityManager, eventBus: { emit: (event: string, payload: Record<string, unknown>) => Promise<void> }): ProgressService {\n const broadcastMinIntervalMs = resolveBroadcastMinIntervalMs()\n // Per-job coalescing state, scoped to this service instance (request/worker scope).\n // The cached entity is a DETACHED snapshot (loaded with disableIdentityMap) that doubles\n // as the in-memory buffer: intermediate updates mutate it without touching the shared\n // identity map or UnitOfWork, so an unrelated em.flush() elsewhere can never write the\n // buffered values. Persistence happens exclusively through the CAS nativeUpdate below.\n const jobUpdateThrottle = new Map<string, JobUpdateThrottleEntry>()\n\n function jobScopeFilter(jobId: string, ctx: ProgressServiceContext) {\n return {\n id: jobId,\n tenantId: ctx.tenantId,\n ...(ctx.organizationId ? { organizationId: ctx.organizationId } : {}),\n }\n }\n\n async function loadFreshJob(jobId: string, ctx: ProgressServiceContext): Promise<ProgressJob | null> {\n return em.findOne(ProgressJob, jobScopeFilter(jobId, ctx), { disableIdentityMap: true })\n }\n\n async function ensureThrottleEntry(jobId: string, ctx: ProgressServiceContext): Promise<JobUpdateThrottleEntry> {\n const cached = jobUpdateThrottle.get(jobId)\n if (cached) return cached\n const job = await em.findOneOrFail(ProgressJob, jobScopeFilter(jobId, ctx), { disableIdentityMap: true })\n const entry: JobUpdateThrottleEntry = {\n job,\n lastBroadcastAt: 0,\n lastPersistedAt: 0,\n lastBroadcastPercent: job.progressPercent,\n pendingDelta: 0,\n absoluteCountsPending: false,\n }\n jobUpdateThrottle.set(jobId, entry)\n return entry\n }\n\n function forgetJobThrottle(jobId: string) {\n jobUpdateThrottle.delete(jobId)\n }\n\n function buildBufferedCountData(entry: JobUpdateThrottleEntry): EntityData<ProgressJob> {\n if (entry.absoluteCountsPending || !Number.isSafeInteger(entry.pendingDelta)) {\n return { processedCount: entry.job.processedCount }\n }\n if (entry.pendingDelta !== 0) {\n const processedCount = raw(`processed_count + ${entry.pendingDelta}`)\n const totalCount = entry.job.totalCount\n if (Number.isSafeInteger(totalCount) && totalCount != null && totalCount > 0) {\n return {\n processedCount,\n progressPercent: raw(\n `least(100, round(((processed_count + ${entry.pendingDelta})::numeric / ${totalCount}) * 100))`,\n ),\n }\n }\n return { processedCount }\n }\n return {}\n }\n\n async function persistAndMaybeBroadcast(entry: JobUpdateThrottleEntry, ctx: ProgressServiceContext): Promise<ProgressJob> {\n const job = entry.job\n const now = Date.now()\n const shouldBroadcast =\n broadcastMinIntervalMs <= 0 ||\n now - entry.lastBroadcastAt >= broadcastMinIntervalMs ||\n Math.abs(job.progressPercent - entry.lastBroadcastPercent) >= 1\n const shouldPersist = shouldBroadcast || now - entry.lastPersistedAt >= HEARTBEAT_INTERVAL_MS\n\n if (!shouldPersist) return job\n\n const usesAtomicIncrement =\n !entry.absoluteCountsPending && Number.isSafeInteger(entry.pendingDelta) && entry.pendingDelta !== 0\n const data: EntityData<ProgressJob> = {\n progressPercent: job.progressPercent,\n totalCount: job.totalCount ?? null,\n etaSeconds: job.etaSeconds ?? null,\n meta: job.meta ?? null,\n heartbeatAt: job.heartbeatAt ?? new Date(),\n updatedAt: new Date(),\n ...buildBufferedCountData(entry),\n }\n const affected = await em.nativeUpdate(ProgressJob, {\n ...jobScopeFilter(job.id, ctx),\n status: { $in: PROGRESS_WRITABLE_STATUSES as ProgressJobStatus[] },\n } as FilterQuery<ProgressJob>, data)\n\n if (affected === 0) {\n // The job reached a terminal state in another process; stop writing to it.\n forgetJobThrottle(job.id)\n const fresh = await loadFreshJob(job.id, ctx)\n return fresh ?? job\n }\n\n const persistedJob = usesAtomicIncrement ? (await loadFreshJob(job.id, ctx)) ?? job : job\n entry.job = persistedJob\n entry.pendingDelta = 0\n entry.absoluteCountsPending = false\n entry.lastPersistedAt = now\n\n if (shouldBroadcast) {\n await eventBus.emit(PROGRESS_EVENTS.JOB_UPDATED, {\n ...buildJobPayload(persistedJob),\n tenantId: ctx.tenantId,\n organizationId: persistedJob.organizationId ?? null,\n })\n entry.lastBroadcastAt = now\n entry.lastBroadcastPercent = persistedJob.progressPercent\n }\n\n return persistedJob\n }\n\n return {\n async createJob(input, ctx) {\n const job = em.create(ProgressJob, {\n jobType: input.jobType,\n name: input.name,\n description: input.description,\n totalCount: input.totalCount,\n cancellable: input.cancellable ?? false,\n meta: input.meta,\n parentJobId: input.parentJobId,\n partitionIndex: input.partitionIndex,\n partitionCount: input.partitionCount,\n startedByUserId: ctx.userId,\n tenantId: ctx.tenantId,\n organizationId: ctx.organizationId,\n status: 'pending',\n })\n\n await em.persist(job).flush()\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_CREATED, {\n ...buildJobPayload(job),\n tenantId: ctx.tenantId,\n organizationId: ctx.organizationId,\n })\n\n return job\n },\n\n async startJob(jobId, ctx) {\n const job = await em.findOneOrFail(ProgressJob, jobScopeFilter(jobId, ctx), { disableIdentityMap: true })\n if (job.status === 'running' || job.status === 'cancelled' || job.status === 'completed') {\n return job\n }\n\n const now = new Date()\n const affected = await em.nativeUpdate(ProgressJob, {\n ...jobScopeFilter(jobId, ctx),\n status: { $in: START_FROM_STATUSES as ProgressJobStatus[] },\n } as FilterQuery<ProgressJob>, {\n status: 'running',\n startedAt: now,\n heartbeatAt: now,\n finishedAt: null,\n errorMessage: null,\n errorStack: null,\n updatedAt: now,\n })\n\n if (affected === 0) {\n const fresh = await loadFreshJob(jobId, ctx)\n return fresh ?? job\n }\n\n job.status = 'running'\n job.startedAt = now\n job.heartbeatAt = now\n job.finishedAt = null\n job.errorMessage = null\n job.errorStack = null\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_STARTED, {\n ...buildJobPayload(job),\n tenantId: ctx.tenantId,\n organizationId: job.organizationId ?? null,\n })\n\n return job\n },\n\n async updateProgress(jobId, input, ctx) {\n const entry = await ensureThrottleEntry(jobId, ctx)\n const job = entry.job\n if (TERMINAL_STATUSES.includes(job.status)) {\n forgetJobThrottle(jobId)\n return job\n }\n\n if (input.processedCount !== undefined) {\n job.processedCount = input.processedCount\n entry.absoluteCountsPending = true\n entry.pendingDelta = 0\n }\n if (input.totalCount !== undefined) {\n job.totalCount = input.totalCount\n }\n if (input.meta !== undefined) {\n job.meta = { ...job.meta, ...input.meta }\n }\n\n if (input.progressPercent !== undefined) {\n job.progressPercent = input.progressPercent\n } else if (job.totalCount) {\n job.progressPercent = calculateProgressPercent(job.processedCount, job.totalCount)\n }\n\n if (input.etaSeconds !== undefined) {\n job.etaSeconds = input.etaSeconds\n } else if (job.startedAt && job.totalCount) {\n job.etaSeconds = calculateEta(job.processedCount, job.totalCount, job.startedAt)\n }\n\n job.heartbeatAt = new Date()\n\n return persistAndMaybeBroadcast(entry, ctx)\n },\n\n async incrementProgress(jobId, delta, ctx) {\n const entry = await ensureThrottleEntry(jobId, ctx)\n const job = entry.job\n if (TERMINAL_STATUSES.includes(job.status)) {\n forgetJobThrottle(jobId)\n return job\n }\n\n job.processedCount += delta\n entry.pendingDelta += delta\n job.heartbeatAt = new Date()\n\n if (job.totalCount) {\n job.progressPercent = calculateProgressPercent(job.processedCount, job.totalCount)\n if (job.startedAt) {\n job.etaSeconds = calculateEta(job.processedCount, job.totalCount, job.startedAt)\n }\n }\n\n return persistAndMaybeBroadcast(entry, ctx)\n },\n\n async completeJob(jobId, input, ctx) {\n const job = await loadFreshJob(jobId, ctx)\n if (!job) throw new Error(`Job ${jobId} not found`)\n if (job.status === 'cancelled' || job.status === 'completed') {\n forgetJobThrottle(jobId)\n return job\n }\n\n const entry = jobUpdateThrottle.get(jobId)\n const snapshot = entry?.job ?? job\n const now = new Date()\n const data: EntityData<ProgressJob> = {\n status: 'completed',\n finishedAt: now,\n etaSeconds: 0,\n updatedAt: now,\n ...(input?.resultSummary ? { resultSummary: input.resultSummary } : {}),\n ...(entry\n ? {\n totalCount: snapshot.totalCount ?? null,\n meta: snapshot.meta ?? null,\n ...buildBufferedCountData(entry),\n }\n : {}),\n progressPercent: 100,\n }\n\n const affected = await em.nativeUpdate(ProgressJob, {\n ...jobScopeFilter(jobId, ctx),\n status: { $in: COMPLETE_FROM_STATUSES as ProgressJobStatus[] },\n } as FilterQuery<ProgressJob>, data)\n forgetJobThrottle(jobId)\n\n if (affected === 0) {\n const fresh = await loadFreshJob(jobId, ctx)\n return fresh ?? job\n }\n\n snapshot.status = 'completed'\n snapshot.finishedAt = now\n snapshot.progressPercent = 100\n snapshot.etaSeconds = 0\n if (input?.resultSummary) {\n snapshot.resultSummary = input.resultSummary\n }\n\n const persistedJob = (await loadFreshJob(jobId, ctx)) ?? snapshot\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_COMPLETED, {\n ...buildJobPayload(persistedJob),\n resultSummary: persistedJob.resultSummary,\n tenantId: ctx.tenantId,\n organizationId: persistedJob.organizationId ?? null,\n })\n\n return persistedJob\n },\n\n async failJob(jobId, input, ctx) {\n const job = await loadFreshJob(jobId, ctx)\n if (!job) throw new Error(`Job ${jobId} not found`)\n if (TERMINAL_STATUSES.includes(job.status)) {\n forgetJobThrottle(jobId)\n return job\n }\n\n const entry = jobUpdateThrottle.get(jobId)\n const snapshot = entry?.job ?? job\n const now = new Date()\n const data: EntityData<ProgressJob> = {\n status: 'failed',\n finishedAt: now,\n errorMessage: input.errorMessage,\n errorStack: input.errorStack,\n ...(input.resultSummary ? { resultSummary: input.resultSummary } : {}),\n updatedAt: now,\n ...(entry\n ? {\n totalCount: snapshot.totalCount ?? null,\n meta: snapshot.meta ?? null,\n ...buildBufferedCountData(entry),\n }\n : {}),\n }\n\n const affected = await em.nativeUpdate(ProgressJob, {\n ...jobScopeFilter(jobId, ctx),\n status: { $in: FAIL_FROM_STATUSES as ProgressJobStatus[] },\n } as FilterQuery<ProgressJob>, data)\n forgetJobThrottle(jobId)\n\n if (affected === 0) {\n const fresh = await loadFreshJob(jobId, ctx)\n return fresh ?? job\n }\n\n snapshot.status = 'failed'\n snapshot.finishedAt = now\n snapshot.errorMessage = input.errorMessage\n snapshot.errorStack = input.errorStack\n if (input.resultSummary) {\n snapshot.resultSummary = input.resultSummary\n }\n\n const persistedJob = (await loadFreshJob(jobId, ctx)) ?? snapshot\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_FAILED, {\n ...buildJobPayload(persistedJob),\n errorMessage: persistedJob.errorMessage,\n tenantId: ctx.tenantId,\n organizationId: persistedJob.organizationId ?? null,\n })\n\n return persistedJob\n },\n\n async cancelJob(jobId, ctx) {\n const job = await em.findOneOrFail(ProgressJob, jobScopeFilter(jobId, ctx), { disableIdentityMap: true })\n if (TERMINAL_STATUSES.includes(job.status)) {\n // The job finished while the user clicked cancel \u2014 a benign race, not an error.\n return job\n }\n if (!job.cancellable) {\n throw new Error(`Job ${jobId} is not cancellable`)\n }\n\n const now = new Date()\n const cancelledNow = await em.nativeUpdate(ProgressJob, {\n ...jobScopeFilter(jobId, ctx),\n cancellable: true,\n status: 'pending',\n } as FilterQuery<ProgressJob>, {\n status: 'cancelled',\n cancelRequestedAt: now,\n cancelledByUserId: ctx.userId ?? null,\n finishedAt: now,\n etaSeconds: 0,\n updatedAt: now,\n })\n\n let requested = 0\n if (cancelledNow === 0) {\n requested = await em.nativeUpdate(ProgressJob, {\n ...jobScopeFilter(jobId, ctx),\n cancellable: true,\n status: 'running',\n } as FilterQuery<ProgressJob>, {\n cancelRequestedAt: now,\n cancelledByUserId: ctx.userId ?? null,\n updatedAt: now,\n })\n }\n\n if (cancelledNow === 0 && requested === 0) {\n // Raced into a terminal state between the read and the CAS.\n const fresh = await loadFreshJob(jobId, ctx)\n return fresh ?? job\n }\n\n forgetJobThrottle(jobId)\n if (cancelledNow > 0) {\n job.status = 'cancelled'\n job.finishedAt = now\n job.etaSeconds = 0\n }\n job.cancelRequestedAt = now\n job.cancelledByUserId = ctx.userId ?? null\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_CANCELLED, {\n ...buildJobPayload(job),\n tenantId: ctx.tenantId,\n organizationId: job.organizationId ?? null,\n })\n\n return job\n },\n\n async markCancelled(jobId, ctx) {\n const job = await loadFreshJob(jobId, ctx)\n if (!job) throw new Error(`Job ${jobId} not found`)\n if (job.status === 'cancelled' || job.status === 'completed') {\n forgetJobThrottle(jobId)\n return job\n }\n\n const now = new Date()\n const cancelRequestedAt = job.cancelRequestedAt ?? now\n const cancelledByUserId = job.cancelledByUserId ?? ctx.userId ?? null\n const finishedAt = job.finishedAt ?? now\n\n const affected = await em.nativeUpdate(ProgressJob, {\n ...jobScopeFilter(jobId, ctx),\n status: { $in: CANCEL_FROM_STATUSES as ProgressJobStatus[] },\n } as FilterQuery<ProgressJob>, {\n status: 'cancelled',\n cancelRequestedAt,\n cancelledByUserId,\n finishedAt,\n etaSeconds: 0,\n updatedAt: now,\n })\n forgetJobThrottle(jobId)\n\n if (affected === 0) {\n const fresh = await loadFreshJob(jobId, ctx)\n return fresh ?? job\n }\n\n job.status = 'cancelled'\n job.cancelRequestedAt = cancelRequestedAt\n job.cancelledByUserId = cancelledByUserId\n job.finishedAt = finishedAt\n job.etaSeconds = 0\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_CANCELLED, {\n ...buildJobPayload(job),\n tenantId: ctx.tenantId,\n organizationId: job.organizationId ?? null,\n })\n\n return job\n },\n\n async isCancellationRequested(jobId, tenantId, organizationId) {\n // disableIdentityMap forces a fresh read: a managed copy of the job in this\n // EntityManager (loaded by updateProgress) must never mask a cancellation\n // requested from another process.\n const job = await findOneWithDecryption(em, ProgressJob, {\n id: jobId,\n tenantId,\n ...(organizationId ? { organizationId } : {}),\n }, { disableIdentityMap: true })\n return job?.cancelRequestedAt != null\n },\n\n async getActiveJobs(ctx) {\n return em.find(ProgressJob, {\n tenantId: ctx.tenantId,\n ...(ctx.organizationId ? { organizationId: ctx.organizationId } : {}),\n status: { $in: ['pending', 'running'] },\n parentJobId: null,\n }, {\n orderBy: { createdAt: 'DESC' },\n limit: 50,\n })\n },\n\n async getRecentlyCompletedJobs(ctx, sinceSeconds = 30) {\n const cutoff = new Date(Date.now() - sinceSeconds * 1000)\n return em.find(ProgressJob, {\n tenantId: ctx.tenantId,\n ...(ctx.organizationId ? { organizationId: ctx.organizationId } : {}),\n status: { $in: ['completed', 'failed'] },\n finishedAt: { $gte: cutoff },\n parentJobId: null,\n }, {\n orderBy: { finishedAt: 'DESC' },\n limit: 10,\n })\n },\n\n async getJob(jobId, ctx) {\n return em.findOne(ProgressJob, {\n id: jobId,\n tenantId: ctx.tenantId,\n ...(ctx.organizationId ? { organizationId: ctx.organizationId } : {}),\n })\n },\n\n async markStaleJobsFailed(tenantId: string, timeoutSeconds = STALE_JOB_TIMEOUT_SECONDS, organizationId?: string | null) {\n const now = new Date()\n const cutoff = new Date(now.getTime() - timeoutSeconds * 1000)\n const scope = {\n tenantId,\n ...(organizationId ? { organizationId } : {}),\n }\n let failedCount = 0\n\n const staleFilter = {\n status: 'running' as ProgressJobStatus,\n $or: [\n { heartbeatAt: { $lt: cutoff } },\n {\n heartbeatAt: null,\n startedAt: { $lt: cutoff },\n },\n ],\n }\n const staleRunning = await em.find(ProgressJob, { ...scope, ...staleFilter }, { disableIdentityMap: true })\n\n for (const job of staleRunning) {\n const errorMessage = `Job stale: no heartbeat for ${timeoutSeconds} seconds`\n // Per-row CAS (re-checking the staleness condition): exactly one concurrent\n // sweeper wins, and a heartbeat that landed after the SELECT aborts the failure.\n const affected = await em.nativeUpdate(ProgressJob, {\n id: job.id,\n tenantId: job.tenantId,\n ...staleFilter,\n } as FilterQuery<ProgressJob>, {\n status: 'failed',\n finishedAt: now,\n errorMessage,\n updatedAt: now,\n })\n if (affected === 0) continue\n\n failedCount += 1\n job.status = 'failed'\n job.finishedAt = now\n job.errorMessage = errorMessage\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_FAILED, {\n ...buildJobPayload(job),\n errorMessage: job.errorMessage,\n tenantId: job.tenantId,\n stale: true,\n organizationId: job.organizationId ?? null,\n })\n }\n\n // Jobs stuck in `pending` (enqueue failed, worker died before startJob) have no\n // heartbeat, so they need their own sweep or they stay in the active list forever.\n // A late queue delivery still recovers such a job: startJob transitions failed \u2192 running.\n const pendingCutoff = new Date(now.getTime() - STALE_PENDING_TIMEOUT_SECONDS * 1000)\n const stalePendingFilter = {\n status: 'pending' as ProgressJobStatus,\n createdAt: { $lt: pendingCutoff },\n }\n const stalePending = await em.find(ProgressJob, { ...scope, ...stalePendingFilter }, { disableIdentityMap: true })\n\n for (const job of stalePending) {\n const errorMessage = `Job stale: never started within ${STALE_PENDING_TIMEOUT_SECONDS} seconds`\n const affected = await em.nativeUpdate(ProgressJob, {\n id: job.id,\n tenantId: job.tenantId,\n ...stalePendingFilter,\n } as FilterQuery<ProgressJob>, {\n status: 'failed',\n finishedAt: now,\n errorMessage,\n updatedAt: now,\n })\n if (affected === 0) continue\n\n failedCount += 1\n job.status = 'failed'\n job.finishedAt = now\n job.errorMessage = errorMessage\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_FAILED, {\n ...buildJobPayload(job),\n errorMessage: job.errorMessage,\n tenantId: job.tenantId,\n stale: true,\n organizationId: job.organizationId ?? null,\n })\n }\n\n return failedCount\n },\n }\n}\n"],
|
|
5
|
-
"mappings": "AACA,SAAS,WAAW;AAEpB,SAAS,mBAA2C;AAEpD;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,uBAAuB;AAChC,SAAS,6BAA6B;AAEtC,MAAM,oCAAoC;AAS1C,SAAS,gCAAwC;AAC/C,QAAM,WAAW,QAAQ,IAAI;AAC7B,MAAI,YAAY,QAAQ,SAAS,KAAK,MAAM,GAAI,QAAO;AACvD,QAAM,SAAS,OAAO,SAAS,UAAU,EAAE;AAC3C,MAAI,CAAC,OAAO,SAAS,MAAM,KAAK,SAAS,EAAG,QAAO;AACnD,SAAO;AACT;AAMA,MAAM,oBAAkD,CAAC,aAAa,UAAU,WAAW;AAC3F,MAAM,6BAA2D,CAAC,WAAW,SAAS;AAGtF,MAAM,sBAAoD,CAAC,WAAW,QAAQ;AAC9E,MAAM,yBAAuD,CAAC,WAAW,WAAW,QAAQ;AAC5F,MAAM,qBAAmD,CAAC,WAAW,SAAS;AAC9E,MAAM,uBAAqD,CAAC,WAAW,WAAW,QAAQ;AAW1F,SAAS,gBAAgB,KAA2C;AAClE,SAAO;AAAA,IACL,OAAO,IAAI;AAAA,IACX,SAAS,IAAI;AAAA,IACb,MAAM,IAAI;AAAA,IACV,aAAa,IAAI,eAAe;AAAA,IAChC,QAAQ,IAAI;AAAA,IACZ,iBAAiB,IAAI;AAAA,IACrB,gBAAgB,IAAI;AAAA,IACpB,YAAY,IAAI,cAAc;AAAA,IAC9B,YAAY,IAAI,cAAc;AAAA,IAC9B,aAAa,IAAI;AAAA,IACjB,MAAM,IAAI,QAAQ;AAAA,IAClB,WAAW,IAAI,WAAW,YAAY,KAAK;AAAA,IAC3C,YAAY,IAAI,YAAY,YAAY,KAAK;AAAA,EAC/C;AACF;AAEO,SAAS,sBAAsB,IAAmB,UAAyG;AAChK,QAAM,yBAAyB,8BAA8B;AAM7D,QAAM,oBAAoB,oBAAI,IAAoC;AAElE,WAAS,eAAe,OAAe,KAA6B;AAClE,WAAO;AAAA,MACL,IAAI;AAAA,MACJ,UAAU,IAAI;AAAA,MACd,GAAI,IAAI,iBAAiB,EAAE,gBAAgB,IAAI,eAAe,IAAI,CAAC;AAAA,IACrE;AAAA,EACF;AAEA,iBAAe,aAAa,OAAe,KAA0D;AACnG,WAAO,GAAG,QAAQ,aAAa,eAAe,OAAO,GAAG,GAAG,EAAE,oBAAoB,KAAK,CAAC;AAAA,EACzF;AAEA,iBAAe,oBAAoB,OAAe,KAA8D;AAC9G,UAAM,SAAS,kBAAkB,IAAI,KAAK;AAC1C,QAAI,OAAQ,QAAO;AACnB,UAAM,MAAM,MAAM,GAAG,cAAc,aAAa,eAAe,OAAO,GAAG,GAAG,EAAE,oBAAoB,KAAK,CAAC;AACxG,UAAM,QAAgC;AAAA,MACpC;AAAA,MACA,iBAAiB;AAAA,MACjB,iBAAiB;AAAA,MACjB,sBAAsB,IAAI;AAAA,MAC1B,cAAc;AAAA,MACd,uBAAuB;AAAA,IACzB;AACA,sBAAkB,IAAI,OAAO,KAAK;AAClC,WAAO;AAAA,EACT;AAEA,WAAS,kBAAkB,OAAe;AACxC,sBAAkB,OAAO,KAAK;AAAA,EAChC;AAEA,WAAS,uBAAuB,OAAwD;AACtF,QAAI,MAAM,yBAAyB,CAAC,OAAO,cAAc,MAAM,YAAY,GAAG;AAC5E,aAAO,EAAE,gBAAgB,MAAM,IAAI,eAAe;AAAA,IACpD;AACA,QAAI,MAAM,iBAAiB,GAAG;AAC5B,YAAM,iBAAiB,IAAI,qBAAqB,MAAM,YAAY,EAAE;AACpE,YAAM,aAAa,MAAM,IAAI;AAC7B,UAAI,OAAO,cAAc,UAAU,KAAK,cAAc,QAAQ,aAAa,GAAG;AAC5E,eAAO;AAAA,UACL;AAAA,UACA,iBAAiB;AAAA,YACf,wCAAwC,MAAM,YAAY,gBAAgB,UAAU;AAAA,UACtF;AAAA,QACF;AAAA,MACF;AACA,aAAO,EAAE,eAAe;AAAA,IAC1B;AACA,WAAO,CAAC;AAAA,EACV;AAEA,iBAAe,yBAAyB,OAA+B,KAAmD;AACxH,UAAM,MAAM,MAAM;AAClB,UAAM,MAAM,KAAK,IAAI;AACrB,UAAM,kBACJ,0BAA0B,KAC1B,MAAM,MAAM,mBAAmB,0BAC/B,KAAK,IAAI,IAAI,kBAAkB,MAAM,oBAAoB,KAAK;AAChE,UAAM,gBAAgB,mBAAmB,MAAM,MAAM,mBAAmB;AAExE,QAAI,CAAC,cAAe,QAAO;AAE3B,UAAM,sBACJ,CAAC,MAAM,yBAAyB,OAAO,cAAc,MAAM,YAAY,KAAK,MAAM,iBAAiB;AACrG,UAAM,OAAgC;AAAA,MACpC,iBAAiB,IAAI;AAAA,MACrB,YAAY,IAAI,cAAc;AAAA,MAC9B,YAAY,IAAI,cAAc;AAAA,MAC9B,MAAM,IAAI,QAAQ;AAAA,MAClB,aAAa,IAAI,eAAe,oBAAI,KAAK;AAAA,MACzC,WAAW,oBAAI,KAAK;AAAA,MACpB,GAAG,uBAAuB,KAAK;AAAA,IACjC;AACA,UAAM,WAAW,MAAM,GAAG,aAAa,aAAa;AAAA,MAClD,GAAG,eAAe,IAAI,IAAI,GAAG;AAAA,MAC7B,QAAQ,EAAE,KAAK,2BAAkD;AAAA,IACnE,GAA+B,IAAI;AAEnC,QAAI,aAAa,GAAG;AAElB,wBAAkB,IAAI,EAAE;AACxB,YAAM,QAAQ,MAAM,aAAa,IAAI,IAAI,GAAG;AAC5C,aAAO,SAAS;AAAA,IAClB;AAEA,UAAM,eAAe,sBAAuB,MAAM,aAAa,IAAI,IAAI,GAAG,KAAM,MAAM;AACtF,UAAM,MAAM;AACZ,UAAM,eAAe;AACrB,UAAM,wBAAwB;AAC9B,UAAM,kBAAkB;AAExB,QAAI,iBAAiB;AACnB,YAAM,SAAS,KAAK,gBAAgB,aAAa;AAAA,QAC/C,GAAG,gBAAgB,YAAY;AAAA,QAC/B,UAAU,IAAI;AAAA,QACd,gBAAgB,aAAa,kBAAkB;AAAA,MACjD,CAAC;AACD,YAAM,kBAAkB;AACxB,YAAM,uBAAuB,aAAa;AAAA,IAC5C;AAEA,WAAO;AAAA,EACT;AAEA,SAAO;AAAA,IACL,MAAM,UAAU,OAAO,KAAK;AAC1B,YAAM,MAAM,GAAG,OAAO,aAAa;AAAA,QACjC,SAAS,MAAM;AAAA,QACf,MAAM,MAAM;AAAA,QACZ,aAAa,MAAM;AAAA,QACnB,YAAY,MAAM;AAAA,QAClB,aAAa,MAAM,eAAe;AAAA,QAClC,MAAM,MAAM;AAAA,QACZ,aAAa,MAAM;AAAA,QACnB,gBAAgB,MAAM;AAAA,QACtB,gBAAgB,MAAM;AAAA,QACtB,iBAAiB,IAAI;AAAA,QACrB,UAAU,IAAI;AAAA,QACd,gBAAgB,IAAI;AAAA,QACpB,QAAQ;AAAA,MACV,CAAC;AAED,YAAM,GAAG,QAAQ,GAAG,EAAE,MAAM;AAE5B,YAAM,SAAS,KAAK,gBAAgB,aAAa;AAAA,QAC/C,GAAG,gBAAgB,GAAG;AAAA,QACtB,UAAU,IAAI;AAAA,QACd,gBAAgB,IAAI;AAAA,MACtB,CAAC;AAED,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,SAAS,OAAO,KAAK;AACzB,YAAM,MAAM,MAAM,GAAG,cAAc,aAAa,eAAe,OAAO,GAAG,GAAG,EAAE,oBAAoB,KAAK,CAAC;AACxG,UAAI,IAAI,WAAW,aAAa,IAAI,WAAW,eAAe,IAAI,WAAW,aAAa;AACxF,eAAO;AAAA,MACT;AAEA,YAAM,MAAM,oBAAI,KAAK;AACrB,YAAM,WAAW,MAAM,GAAG,aAAa,aAAa;AAAA,QAClD,GAAG,eAAe,OAAO,GAAG;AAAA,QAC5B,QAAQ,EAAE,KAAK,oBAA2C;AAAA,MAC5D,GAA+B;AAAA,QAC7B,QAAQ;AAAA,QACR,WAAW;AAAA,QACX,aAAa;AAAA,QACb,YAAY;AAAA,QACZ,cAAc;AAAA,QACd,YAAY;AAAA,QACZ,WAAW;AAAA,MACb,CAAC;AAED,UAAI,aAAa,GAAG;AAClB,cAAM,QAAQ,MAAM,aAAa,OAAO,GAAG;AAC3C,eAAO,SAAS;AAAA,MAClB;AAEA,UAAI,SAAS;AACb,UAAI,YAAY;AAChB,UAAI,cAAc;AAClB,UAAI,aAAa;AACjB,UAAI,eAAe;AACnB,UAAI,aAAa;AAEjB,YAAM,SAAS,KAAK,gBAAgB,aAAa;AAAA,QAC/C,GAAG,gBAAgB,GAAG;AAAA,QACtB,UAAU,IAAI;AAAA,QACd,gBAAgB,IAAI,kBAAkB;AAAA,MACxC,CAAC;AAED,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,eAAe,OAAO,OAAO,KAAK;AACtC,YAAM,QAAQ,MAAM,oBAAoB,OAAO,GAAG;AAClD,YAAM,MAAM,MAAM;AAClB,UAAI,kBAAkB,SAAS,IAAI,MAAM,GAAG;AAC1C,0BAAkB,KAAK;AACvB,eAAO;AAAA,MACT;AAEA,UAAI,MAAM,mBAAmB,QAAW;AACtC,YAAI,iBAAiB,MAAM;AAC3B,cAAM,wBAAwB;AAC9B,cAAM,eAAe;AAAA,MACvB;AACA,UAAI,MAAM,eAAe,QAAW;AAClC,YAAI,aAAa,MAAM;AAAA,MACzB;AACA,UAAI,MAAM,SAAS,QAAW;AAC5B,YAAI,OAAO,EAAE,GAAG,IAAI,MAAM,GAAG,MAAM,KAAK;AAAA,MAC1C;AAEA,UAAI,MAAM,oBAAoB,QAAW;AACvC,YAAI,kBAAkB,MAAM;AAAA,MAC9B,WAAW,IAAI,YAAY;AACzB,YAAI,kBAAkB,yBAAyB,IAAI,gBAAgB,IAAI,UAAU;AAAA,MACnF;AAEA,UAAI,MAAM,eAAe,QAAW;AAClC,YAAI,aAAa,MAAM;AAAA,MACzB,WAAW,IAAI,aAAa,IAAI,YAAY;AAC1C,YAAI,aAAa,aAAa,IAAI,gBAAgB,IAAI,YAAY,IAAI,SAAS;AAAA,MACjF;AAEA,UAAI,cAAc,oBAAI,KAAK;AAE3B,aAAO,yBAAyB,OAAO,GAAG;AAAA,IAC5C;AAAA,IAEA,MAAM,kBAAkB,OAAO,OAAO,KAAK;AACzC,YAAM,QAAQ,MAAM,oBAAoB,OAAO,GAAG;AAClD,YAAM,MAAM,MAAM;AAClB,UAAI,kBAAkB,SAAS,IAAI,MAAM,GAAG;AAC1C,0BAAkB,KAAK;AACvB,eAAO;AAAA,MACT;AAEA,UAAI,kBAAkB;AACtB,YAAM,gBAAgB;AACtB,UAAI,cAAc,oBAAI,KAAK;AAE3B,UAAI,IAAI,YAAY;AAClB,YAAI,kBAAkB,yBAAyB,IAAI,gBAAgB,IAAI,UAAU;AACjF,YAAI,IAAI,WAAW;AACjB,cAAI,aAAa,aAAa,IAAI,gBAAgB,IAAI,YAAY,IAAI,SAAS;AAAA,QACjF;AAAA,MACF;AAEA,aAAO,yBAAyB,OAAO,GAAG;AAAA,IAC5C;AAAA,IAEA,MAAM,YAAY,OAAO,OAAO,KAAK;AACnC,YAAM,MAAM,MAAM,aAAa,OAAO,GAAG;AACzC,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,OAAO,KAAK,YAAY;AAClD,UAAI,IAAI,WAAW,eAAe,IAAI,WAAW,aAAa;AAC5D,0BAAkB,KAAK;AACvB,eAAO;AAAA,MACT;AAEA,YAAM,QAAQ,kBAAkB,IAAI,KAAK;AACzC,YAAM,WAAW,OAAO,OAAO;AAC/B,YAAM,MAAM,oBAAI,KAAK;AACrB,YAAM,OAAgC;AAAA,QACpC,QAAQ;AAAA,QACR,YAAY;AAAA,QACZ,YAAY;AAAA,QACZ,WAAW;AAAA,QACX,GAAI,OAAO,gBAAgB,EAAE,eAAe,MAAM,cAAc,IAAI,CAAC;AAAA,QACrE,GAAI,QACA;AAAA,UACE,YAAY,SAAS,cAAc;AAAA,UACnC,MAAM,SAAS,QAAQ;AAAA,UACvB,GAAG,uBAAuB,KAAK;AAAA,QACjC,IACA,CAAC;AAAA,QACL,iBAAiB;AAAA,MACnB;AAEA,YAAM,WAAW,MAAM,GAAG,aAAa,aAAa;AAAA,QAClD,GAAG,eAAe,OAAO,GAAG;AAAA,QAC5B,QAAQ,EAAE,KAAK,uBAA8C;AAAA,MAC/D,GAA+B,IAAI;AACnC,wBAAkB,KAAK;AAEvB,UAAI,aAAa,GAAG;AAClB,cAAM,QAAQ,MAAM,aAAa,OAAO,GAAG;AAC3C,eAAO,SAAS;AAAA,MAClB;AAEA,eAAS,SAAS;AAClB,eAAS,aAAa;AACtB,eAAS,kBAAkB;AAC3B,eAAS,aAAa;AACtB,UAAI,OAAO,eAAe;AACxB,iBAAS,gBAAgB,MAAM;AAAA,MACjC;AAEA,YAAM,eAAgB,MAAM,aAAa,OAAO,GAAG,KAAM;AAEzD,YAAM,SAAS,KAAK,gBAAgB,eAAe;AAAA,QACjD,GAAG,gBAAgB,YAAY;AAAA,QAC/B,eAAe,aAAa;AAAA,QAC5B,UAAU,IAAI;AAAA,QACd,gBAAgB,aAAa,kBAAkB;AAAA,MACjD,CAAC;AAED,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,QAAQ,OAAO,OAAO,KAAK;AAC/B,YAAM,MAAM,MAAM,aAAa,OAAO,GAAG;AACzC,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,OAAO,KAAK,YAAY;AAClD,UAAI,kBAAkB,SAAS,IAAI,MAAM,GAAG;AAC1C,0BAAkB,KAAK;AACvB,eAAO;AAAA,MACT;AAEA,YAAM,QAAQ,kBAAkB,IAAI,KAAK;AACzC,YAAM,WAAW,OAAO,OAAO;AAC/B,YAAM,MAAM,oBAAI,KAAK;AACrB,YAAM,OAAgC;AAAA,QACpC,QAAQ;AAAA,QACR,YAAY;AAAA,QACZ,cAAc,MAAM;AAAA,QACpB,YAAY,MAAM;AAAA,QAClB,GAAI,MAAM,gBAAgB,EAAE,eAAe,MAAM,cAAc,IAAI,CAAC;AAAA,QACpE,WAAW;AAAA,QACX,GAAI,QACA;AAAA,UACE,YAAY,SAAS,cAAc;AAAA,UACnC,MAAM,SAAS,QAAQ;AAAA,UACvB,GAAG,uBAAuB,KAAK;AAAA,QACjC,IACA,CAAC;AAAA,MACP;AAEA,YAAM,WAAW,MAAM,GAAG,aAAa,aAAa;AAAA,QAClD,GAAG,eAAe,OAAO,GAAG;AAAA,QAC5B,QAAQ,EAAE,KAAK,mBAA0C;AAAA,MAC3D,GAA+B,IAAI;AACnC,wBAAkB,KAAK;AAEvB,UAAI,aAAa,GAAG;AAClB,cAAM,QAAQ,MAAM,aAAa,OAAO,GAAG;AAC3C,eAAO,SAAS;AAAA,MAClB;AAEA,eAAS,SAAS;AAClB,eAAS,aAAa;AACtB,eAAS,eAAe,MAAM;AAC9B,eAAS,aAAa,MAAM;AAC5B,UAAI,MAAM,eAAe;AACvB,iBAAS,gBAAgB,MAAM;AAAA,MACjC;AAEA,YAAM,eAAgB,MAAM,aAAa,OAAO,GAAG,KAAM;AAEzD,YAAM,SAAS,KAAK,gBAAgB,YAAY;AAAA,QAC9C,GAAG,gBAAgB,YAAY;AAAA,QAC/B,cAAc,aAAa;AAAA,QAC3B,UAAU,IAAI;AAAA,QACd,gBAAgB,aAAa,kBAAkB;AAAA,MACjD,CAAC;AAED,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,UAAU,OAAO,KAAK;AAC1B,YAAM,MAAM,MAAM,GAAG,cAAc,aAAa,eAAe,OAAO,GAAG,GAAG,EAAE,oBAAoB,KAAK,CAAC;AACxG,UAAI,kBAAkB,SAAS,IAAI,MAAM,GAAG;AAE1C,eAAO;AAAA,MACT;AACA,UAAI,CAAC,IAAI,aAAa;AACpB,cAAM,IAAI,MAAM,OAAO,KAAK,qBAAqB;AAAA,MACnD;AAEA,YAAM,MAAM,oBAAI,KAAK;AACrB,YAAM,eAAe,MAAM,GAAG,aAAa,aAAa;AAAA,QACtD,GAAG,eAAe,OAAO,GAAG;AAAA,QAC5B,aAAa;AAAA,QACb,QAAQ;AAAA,MACV,GAA+B;AAAA,QAC7B,QAAQ;AAAA,QACR,mBAAmB;AAAA,QACnB,mBAAmB,IAAI,UAAU;AAAA,QACjC,YAAY;AAAA,QACZ,YAAY;AAAA,QACZ,WAAW;AAAA,MACb,CAAC;AAED,UAAI,YAAY;AAChB,UAAI,iBAAiB,GAAG;AACtB,oBAAY,MAAM,GAAG,aAAa,aAAa;AAAA,UAC7C,GAAG,eAAe,OAAO,GAAG;AAAA,UAC5B,aAAa;AAAA,UACb,QAAQ;AAAA,QACV,GAA+B;AAAA,UAC7B,mBAAmB;AAAA,UACnB,mBAAmB,IAAI,UAAU;AAAA,UACjC,WAAW;AAAA,QACb,CAAC;AAAA,MACH;AAEA,UAAI,iBAAiB,KAAK,cAAc,GAAG;AAEzC,cAAM,QAAQ,MAAM,aAAa,OAAO,GAAG;AAC3C,eAAO,SAAS;AAAA,MAClB;AAEA,wBAAkB,KAAK;AACvB,UAAI,eAAe,GAAG;AACpB,YAAI,SAAS;AACb,YAAI,aAAa;AACjB,YAAI,aAAa;AAAA,MACnB;AACA,UAAI,oBAAoB;AACxB,UAAI,oBAAoB,IAAI,UAAU;AAEtC,YAAM,SAAS,KAAK,gBAAgB,eAAe;AAAA,QACjD,GAAG,gBAAgB,GAAG;AAAA,QACtB,UAAU,IAAI;AAAA,QACd,gBAAgB,IAAI,kBAAkB;AAAA,MACxC,CAAC;AAED,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,cAAc,OAAO,KAAK;AAC9B,YAAM,MAAM,MAAM,aAAa,OAAO,GAAG;AACzC,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,OAAO,KAAK,YAAY;AAClD,UAAI,IAAI,WAAW,eAAe,IAAI,WAAW,aAAa;AAC5D,0BAAkB,KAAK;AACvB,eAAO;AAAA,MACT;AAEA,YAAM,MAAM,oBAAI,KAAK;AACrB,YAAM,oBAAoB,IAAI,qBAAqB;AACnD,YAAM,oBAAoB,IAAI,qBAAqB,IAAI,UAAU;AACjE,YAAM,aAAa,IAAI,cAAc;AAErC,YAAM,WAAW,MAAM,GAAG,aAAa,aAAa;AAAA,QAClD,GAAG,eAAe,OAAO,GAAG;AAAA,QAC5B,QAAQ,EAAE,KAAK,qBAA4C;AAAA,MAC7D,GAA+B;AAAA,QAC7B,QAAQ;AAAA,QACR;AAAA,QACA;AAAA,QACA;AAAA,QACA,YAAY;AAAA,QACZ,WAAW;AAAA,MACb,CAAC;AACD,wBAAkB,KAAK;AAEvB,UAAI,aAAa,GAAG;AAClB,cAAM,QAAQ,MAAM,aAAa,OAAO,GAAG;AAC3C,eAAO,SAAS;AAAA,MAClB;AAEA,UAAI,SAAS;AACb,UAAI,oBAAoB;AACxB,UAAI,oBAAoB;AACxB,UAAI,aAAa;AACjB,UAAI,aAAa;AAEjB,YAAM,SAAS,KAAK,gBAAgB,eAAe;AAAA,QACjD,GAAG,gBAAgB,GAAG;AAAA,QACtB,UAAU,IAAI;AAAA,QACd,gBAAgB,IAAI,kBAAkB;AAAA,MACxC,CAAC;AAED,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,wBAAwB,OAAO,UAAU,gBAAgB;AAI7D,YAAM,MAAM,MAAM,sBAAsB,IAAI,aAAa;AAAA,QACvD,IAAI;AAAA,QACJ;AAAA,QACA,GAAI,iBAAiB,EAAE,eAAe,IAAI,CAAC;AAAA,MAC7C,GAAG,EAAE,oBAAoB,KAAK,CAAC;AAC/B,aAAO,KAAK,qBAAqB;AAAA,IACnC;AAAA,IAEA,MAAM,cAAc,KAAK;AACvB,aAAO,GAAG,KAAK,aAAa;AAAA,QAC1B,UAAU,IAAI;AAAA,QACd,GAAI,IAAI,iBAAiB,EAAE,gBAAgB,IAAI,eAAe,IAAI,CAAC;AAAA,QACnE,QAAQ,EAAE,KAAK,CAAC,WAAW,SAAS,EAAE;AAAA,QACtC,aAAa;AAAA,MACf,GAAG;AAAA,QACD,SAAS,EAAE,WAAW,OAAO;AAAA,QAC7B,OAAO;AAAA,MACT,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,yBAAyB,KAAK,eAAe,IAAI;AACrD,YAAM,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI,eAAe,GAAI;AACxD,aAAO,GAAG,KAAK,aAAa;AAAA,QAC1B,UAAU,IAAI;AAAA,QACd,GAAI,IAAI,iBAAiB,EAAE,gBAAgB,IAAI,eAAe,IAAI,CAAC;AAAA,QACnE,QAAQ,EAAE,KAAK,CAAC,aAAa,QAAQ,EAAE;AAAA,QACvC,YAAY,EAAE,MAAM,OAAO;AAAA,QAC3B,aAAa;AAAA,MACf,GAAG;AAAA,QACD,SAAS,EAAE,YAAY,OAAO;AAAA,QAC9B,OAAO;AAAA,MACT,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,OAAO,OAAO,KAAK;AACvB,aAAO,GAAG,QAAQ,aAAa;AAAA,QAC7B,IAAI;AAAA,QACJ,UAAU,IAAI;AAAA,QACd,GAAI,IAAI,iBAAiB,EAAE,gBAAgB,IAAI,eAAe,IAAI,CAAC;AAAA,MACrE,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,oBAAoB,UAAkB,iBAAiB,2BAA2B,gBAAgC;AACtH,YAAM,MAAM,oBAAI,KAAK;AACrB,YAAM,SAAS,IAAI,KAAK,IAAI,QAAQ,IAAI,iBAAiB,GAAI;AAC7D,YAAM,QAAQ;AAAA,QACZ;AAAA,QACA,GAAI,iBAAiB,EAAE,eAAe,IAAI,CAAC;AAAA,MAC7C;AACA,UAAI,cAAc;AAElB,YAAM,cAAc;AAAA,QAClB,QAAQ;AAAA,QACR,KAAK;AAAA,UACH,EAAE,aAAa,EAAE,KAAK,OAAO,EAAE;AAAA,UAC/B;AAAA,YACE,aAAa;AAAA,YACb,WAAW,EAAE,KAAK,OAAO;AAAA,UAC3B;AAAA,QACF;AAAA,MACF;AACA,YAAM,eAAe,MAAM,GAAG,KAAK,aAAa,EAAE,GAAG,OAAO,GAAG,YAAY,GAAG,EAAE,oBAAoB,KAAK,CAAC;AAE1G,iBAAW,OAAO,cAAc;AAC9B,cAAM,eAAe,+BAA+B,cAAc;AAGlE,cAAM,WAAW,MAAM,GAAG,aAAa,aAAa;AAAA,UAClD,IAAI,IAAI;AAAA,UACR,UAAU,IAAI;AAAA,UACd,GAAG;AAAA,QACL,GAA+B;AAAA,UAC7B,QAAQ;AAAA,UACR,YAAY;AAAA,UACZ;AAAA,UACA,WAAW;AAAA,QACb,CAAC;AACD,YAAI,aAAa,EAAG;AAEpB,uBAAe;AACf,YAAI,SAAS;AACb,YAAI,aAAa;AACjB,YAAI,eAAe;AAEnB,cAAM,SAAS,KAAK,gBAAgB,YAAY;AAAA,UAC9C,GAAG,gBAAgB,GAAG;AAAA,UACtB,cAAc,IAAI;AAAA,UAClB,UAAU,IAAI;AAAA,UACd,OAAO;AAAA,UACP,gBAAgB,IAAI,kBAAkB;AAAA,QACxC,CAAC;AAAA,MACH;AAKA,YAAM,gBAAgB,IAAI,KAAK,IAAI,QAAQ,IAAI,gCAAgC,GAAI;AACnF,YAAM,qBAAqB;AAAA,QACzB,QAAQ;AAAA,QACR,WAAW,EAAE,KAAK,cAAc;AAAA,MAClC;AACA,YAAM,eAAe,MAAM,GAAG,KAAK,aAAa,EAAE,GAAG,OAAO,GAAG,mBAAmB,GAAG,EAAE,oBAAoB,KAAK,CAAC;AAEjH,iBAAW,OAAO,cAAc;AAC9B,cAAM,eAAe,mCAAmC,6BAA6B;AACrF,cAAM,WAAW,MAAM,GAAG,aAAa,aAAa;AAAA,UAClD,IAAI,IAAI;AAAA,UACR,UAAU,IAAI;AAAA,UACd,GAAG;AAAA,QACL,GAA+B;AAAA,UAC7B,QAAQ;AAAA,UACR,YAAY;AAAA,UACZ;AAAA,UACA,WAAW;AAAA,QACb,CAAC;AACD,YAAI,aAAa,EAAG;AAEpB,uBAAe;AACf,YAAI,SAAS;AACb,YAAI,aAAa;AACjB,YAAI,eAAe;AAEnB,cAAM,SAAS,KAAK,gBAAgB,YAAY;AAAA,UAC9C,GAAG,gBAAgB,GAAG;AAAA,UACtB,cAAc,IAAI;AAAA,UAClB,UAAU,IAAI;AAAA,UACd,OAAO;AAAA,UACP,gBAAgB,IAAI,kBAAkB;AAAA,QACxC,CAAC;AAAA,MACH;AAEA,aAAO;AAAA,IACT;AAAA,EACF;AACF;",
|
|
4
|
+
"sourcesContent": ["import type { EntityManager } from '@mikro-orm/postgresql'\nimport { raw } from '@mikro-orm/core'\nimport type { EntityData, FilterQuery } from '@mikro-orm/core'\nimport { ProgressJob, type ProgressJobStatus } from '../data/entities'\nimport type { ProgressService, ProgressServiceContext } from './progressService'\nimport {\n calculateEta,\n calculateProgressPercent,\n HEARTBEAT_INTERVAL_MS,\n STALE_JOB_TIMEOUT_SECONDS,\n STALE_PENDING_TIMEOUT_SECONDS,\n STALE_SWEEP_ERROR_PREFIX,\n} from './progressService'\nimport { PROGRESS_EVENTS } from './events'\nimport { findOneWithDecryption } from '@open-mercato/shared/lib/encryption/find'\n\nconst DEFAULT_BROADCAST_MIN_INTERVAL_MS = 250\n\n// Minimum elapsed time between coalesced `progress.job.updated` broadcasts for a\n// single job. Bulk workers call updateProgress/incrementProgress once per record, and\n// every emit of the `clientBroadcast: true` event pays a serialized pg_notify roundtrip\n// plus a tenant-wide SSE fan-out. Setting the knob to 0 restores per-record emission.\n// Persistence is throttled separately: heartbeats hit the database at least every\n// HEARTBEAT_INTERVAL_MS regardless of this knob, so a high broadcast interval can\n// never starve the stale-job sweep of heartbeats.\nfunction resolveBroadcastMinIntervalMs(): number {\n const rawValue = process.env.OM_PROGRESS_BROADCAST_MIN_INTERVAL_MS\n if (rawValue == null || rawValue.trim() === '') return DEFAULT_BROADCAST_MIN_INTERVAL_MS\n const parsed = Number.parseInt(rawValue, 10)\n if (!Number.isFinite(parsed) || parsed < 0) return DEFAULT_BROADCAST_MIN_INTERVAL_MS\n return parsed\n}\n\n// Statuses a given transition is allowed to move FROM. Every write below is a\n// compare-and-swap (`nativeUpdate` guarded on status), so concurrent writers in other\n// processes can never resurrect a terminal job or double-apply a transition \u2014 the\n// UPDATE that matches zero rows simply lost the race and must not emit events.\nconst TERMINAL_STATUSES: readonly ProgressJobStatus[] = ['completed', 'failed', 'cancelled']\nconst PROGRESS_WRITABLE_STATUSES: readonly ProgressJobStatus[] = ['pending', 'running']\n// `failed` is restartable/completable so a job wrongly swept as stale (worker alive but\n// slow) or retried by an at-least-once queue converges to its true outcome.\nconst START_FROM_STATUSES: readonly ProgressJobStatus[] = ['pending', 'failed']\nconst COMPLETE_FROM_STATUSES: readonly ProgressJobStatus[] = ['pending', 'running', 'failed']\nconst FAIL_FROM_STATUSES: readonly ProgressJobStatus[] = ['pending', 'running']\nconst CANCEL_FROM_STATUSES: readonly ProgressJobStatus[] = ['pending', 'running', 'failed']\n\ntype JobUpdateThrottleEntry = {\n job: ProgressJob\n lastBroadcastAt: number\n lastPersistedAt: number\n lastBroadcastPercent: number\n pendingDelta: number\n absoluteCountsPending: boolean\n}\n\nfunction buildJobPayload(job: ProgressJob): Record<string, unknown> {\n return {\n jobId: job.id,\n jobType: job.jobType,\n name: job.name,\n description: job.description ?? null,\n status: job.status,\n progressPercent: job.progressPercent,\n processedCount: job.processedCount,\n totalCount: job.totalCount ?? null,\n etaSeconds: job.etaSeconds ?? null,\n cancellable: job.cancellable,\n meta: job.meta ?? null,\n startedAt: job.startedAt?.toISOString() ?? null,\n finishedAt: job.finishedAt?.toISOString() ?? null,\n }\n}\n\nexport function createProgressService(em: EntityManager, eventBus: { emit: (event: string, payload: Record<string, unknown>) => Promise<void> }): ProgressService {\n const broadcastMinIntervalMs = resolveBroadcastMinIntervalMs()\n // Per-job coalescing state, scoped to this service instance (request/worker scope).\n // The cached entity is a DETACHED snapshot (loaded with disableIdentityMap) that doubles\n // as the in-memory buffer: intermediate updates mutate it without touching the shared\n // identity map or UnitOfWork, so an unrelated em.flush() elsewhere can never write the\n // buffered values. Persistence happens exclusively through the CAS nativeUpdate below.\n const jobUpdateThrottle = new Map<string, JobUpdateThrottleEntry>()\n\n function jobScopeFilter(jobId: string, ctx: ProgressServiceContext) {\n return {\n id: jobId,\n tenantId: ctx.tenantId,\n ...(ctx.organizationId ? { organizationId: ctx.organizationId } : {}),\n }\n }\n\n async function loadFreshJob(jobId: string, ctx: ProgressServiceContext): Promise<ProgressJob | null> {\n return em.findOne(ProgressJob, jobScopeFilter(jobId, ctx), { disableIdentityMap: true })\n }\n\n async function ensureThrottleEntry(jobId: string, ctx: ProgressServiceContext): Promise<JobUpdateThrottleEntry> {\n const cached = jobUpdateThrottle.get(jobId)\n if (cached) return cached\n const job = await em.findOneOrFail(ProgressJob, jobScopeFilter(jobId, ctx), { disableIdentityMap: true })\n const entry: JobUpdateThrottleEntry = {\n job,\n lastBroadcastAt: 0,\n lastPersistedAt: 0,\n lastBroadcastPercent: job.progressPercent,\n pendingDelta: 0,\n absoluteCountsPending: false,\n }\n jobUpdateThrottle.set(jobId, entry)\n return entry\n }\n\n function forgetJobThrottle(jobId: string) {\n jobUpdateThrottle.delete(jobId)\n }\n\n function buildBufferedCountData(entry: JobUpdateThrottleEntry): EntityData<ProgressJob> {\n if (entry.absoluteCountsPending || !Number.isSafeInteger(entry.pendingDelta)) {\n return { processedCount: entry.job.processedCount }\n }\n if (entry.pendingDelta !== 0) {\n const processedCount = raw(`processed_count + ${entry.pendingDelta}`)\n const totalCount = entry.job.totalCount\n if (Number.isSafeInteger(totalCount) && totalCount != null && totalCount > 0) {\n return {\n processedCount,\n progressPercent: raw(\n `least(100, round(((processed_count + ${entry.pendingDelta})::numeric / ${totalCount}) * 100))`,\n ),\n }\n }\n return { processedCount }\n }\n return {}\n }\n\n // The start transition (START_FROM_STATUSES) doubles as the recovery path for jobs a\n // stale sweep flipped to `failed` while their producer was alive but slow. Callers pass\n // the EM to run against so the forked heartbeat path can reuse the same CAS.\n //\n // `staleSweptOnly` is for the revive paths: a live write proves the producer is alive,\n // but it does NOT prove the recorded failure was the sweep's. Narrowing the CAS to rows\n // the sweep tagged keeps a genuine failure's message and stack intact instead of\n // resurrecting the job and erasing why it died. `startJob` stays unrestricted so an\n // at-least-once queue can still retry a genuinely failed job from the top.\n async function startJobViaCas(\n targetEm: EntityManager,\n job: ProgressJob,\n ctx: ProgressServiceContext,\n options: { staleSweptOnly?: boolean } = {},\n ): Promise<ProgressJob | null> {\n const now = new Date()\n // Progress counts are absolute and survive across deliveries, so the elapsed window\n // calculateEta divides them by has to survive too. Resetting it on every revive makes\n // a job with hours of work left report seconds remaining.\n const startedAt = job.startedAt ?? now\n const filter: Record<string, unknown> = {\n ...jobScopeFilter(job.id, ctx),\n status: { $in: START_FROM_STATUSES as ProgressJobStatus[] },\n }\n if (options.staleSweptOnly) filter.errorMessage = { $like: `${STALE_SWEEP_ERROR_PREFIX}%` }\n const affected = await targetEm.nativeUpdate(ProgressJob, filter as FilterQuery<ProgressJob>, {\n status: 'running',\n startedAt,\n heartbeatAt: now,\n finishedAt: null,\n errorMessage: null,\n errorStack: null,\n updatedAt: now,\n })\n if (affected === 0) return null\n\n job.status = 'running'\n job.startedAt = startedAt\n job.heartbeatAt = now\n job.finishedAt = null\n job.errorMessage = null\n job.errorStack = null\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_STARTED, {\n ...buildJobPayload(job),\n tenantId: ctx.tenantId,\n organizationId: job.organizationId ?? null,\n })\n\n return job\n }\n\n async function persistAndMaybeBroadcast(entry: JobUpdateThrottleEntry, ctx: ProgressServiceContext): Promise<ProgressJob> {\n const job = entry.job\n const now = Date.now()\n const shouldBroadcast =\n broadcastMinIntervalMs <= 0 ||\n now - entry.lastBroadcastAt >= broadcastMinIntervalMs ||\n Math.abs(job.progressPercent - entry.lastBroadcastPercent) >= 1\n const shouldPersist = shouldBroadcast || now - entry.lastPersistedAt >= HEARTBEAT_INTERVAL_MS\n\n if (!shouldPersist) return job\n\n const usesAtomicIncrement =\n !entry.absoluteCountsPending && Number.isSafeInteger(entry.pendingDelta) && entry.pendingDelta !== 0\n const data: EntityData<ProgressJob> = {\n progressPercent: job.progressPercent,\n totalCount: job.totalCount ?? null,\n etaSeconds: job.etaSeconds ?? null,\n meta: job.meta ?? null,\n heartbeatAt: job.heartbeatAt ?? new Date(),\n updatedAt: new Date(),\n ...buildBufferedCountData(entry),\n }\n const writableFilter = {\n ...jobScopeFilter(job.id, ctx),\n status: { $in: PROGRESS_WRITABLE_STATUSES as ProgressJobStatus[] },\n } as FilterQuery<ProgressJob>\n let affected = await em.nativeUpdate(ProgressJob, writableFilter, data)\n\n if (affected === 0) {\n // A stale sweep may have flipped a live job to `failed` between writes. Revive it\n // through the start CAS and retry once so the buffered delta isn't silently dropped.\n const fresh = await loadFreshJob(job.id, ctx)\n let revived = false\n if (fresh && fresh.status === 'failed' && (await startJobViaCas(em, fresh, ctx, { staleSweptOnly: true }))) {\n revived = true\n job.status = 'running'\n job.startedAt = fresh.startedAt\n job.finishedAt = null\n job.errorMessage = null\n job.errorStack = null\n affected = await em.nativeUpdate(ProgressJob, writableFilter, data)\n }\n if (affected === 0) {\n // The job reached a genuinely terminal state in another process; stop writing to it.\n forgetJobThrottle(job.id)\n const latest = revived ? await loadFreshJob(job.id, ctx) : fresh\n return latest ?? job\n }\n }\n\n const persistedJob = usesAtomicIncrement ? (await loadFreshJob(job.id, ctx)) ?? job : job\n entry.job = persistedJob\n entry.pendingDelta = 0\n entry.absoluteCountsPending = false\n entry.lastPersistedAt = now\n\n if (shouldBroadcast) {\n await eventBus.emit(PROGRESS_EVENTS.JOB_UPDATED, {\n ...buildJobPayload(persistedJob),\n tenantId: ctx.tenantId,\n organizationId: persistedJob.organizationId ?? null,\n })\n entry.lastBroadcastAt = now\n entry.lastBroadcastPercent = persistedJob.progressPercent\n }\n\n return persistedJob\n }\n\n return {\n async createJob(input, ctx) {\n const job = em.create(ProgressJob, {\n jobType: input.jobType,\n name: input.name,\n description: input.description,\n totalCount: input.totalCount,\n cancellable: input.cancellable ?? false,\n meta: input.meta,\n parentJobId: input.parentJobId,\n partitionIndex: input.partitionIndex,\n partitionCount: input.partitionCount,\n startedByUserId: ctx.userId,\n tenantId: ctx.tenantId,\n organizationId: ctx.organizationId,\n status: 'pending',\n })\n\n await em.persist(job).flush()\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_CREATED, {\n ...buildJobPayload(job),\n tenantId: ctx.tenantId,\n organizationId: ctx.organizationId,\n })\n\n return job\n },\n\n async startJob(jobId, ctx) {\n const job = await em.findOneOrFail(ProgressJob, jobScopeFilter(jobId, ctx), { disableIdentityMap: true })\n if (job.status === 'running' || job.status === 'cancelled' || job.status === 'completed') {\n return job\n }\n\n const started = await startJobViaCas(em, job, ctx)\n if (started) return started\n\n const fresh = await loadFreshJob(jobId, ctx)\n return fresh ?? job\n },\n\n async touchJobHeartbeat(jobId, ctx) {\n // Forked EM: keepalive timers call this while the shared EM may be mid-transaction\n // inside a producer's own writes, so the heartbeat must not join that unit of work.\n const fork = em.fork()\n const now = new Date()\n const affected = await fork.nativeUpdate(ProgressJob, {\n ...jobScopeFilter(jobId, ctx),\n status: { $in: PROGRESS_WRITABLE_STATUSES as ProgressJobStatus[] },\n } as FilterQuery<ProgressJob>, {\n heartbeatAt: now,\n updatedAt: now,\n })\n if (affected > 0) return\n\n const job = await fork.findOne(ProgressJob, jobScopeFilter(jobId, ctx), { disableIdentityMap: true })\n if (!job || job.status !== 'failed') return\n // A live producer heartbeating a `failed` job means a stale sweep false-positived;\n // revive through the start CAS so the card recovers without waiting for a batch boundary.\n await startJobViaCas(fork, job, ctx, { staleSweptOnly: true })\n },\n\n async updateProgress(jobId, input, ctx) {\n const entry = await ensureThrottleEntry(jobId, ctx)\n const job = entry.job\n if (TERMINAL_STATUSES.includes(job.status)) {\n // `failed` may be a false-positive stale sweep on a slow-but-alive producer; the\n // fact that this write is happening proves the producer is alive, so revive and\n // apply the update. `completed`/`cancelled` stay terminal, and a failure the sweep\n // did not write keeps its diagnostics (see startJobViaCas).\n const revived = job.status === 'failed' && (await startJobViaCas(em, job, ctx, { staleSweptOnly: true }))\n if (!revived) {\n forgetJobThrottle(jobId)\n return job\n }\n entry.lastPersistedAt = 0\n }\n\n if (input.processedCount !== undefined) {\n job.processedCount = input.processedCount\n entry.absoluteCountsPending = true\n entry.pendingDelta = 0\n }\n if (input.totalCount !== undefined) {\n job.totalCount = input.totalCount\n }\n if (input.meta !== undefined) {\n job.meta = { ...job.meta, ...input.meta }\n }\n\n if (input.progressPercent !== undefined) {\n job.progressPercent = input.progressPercent\n } else if (job.totalCount) {\n job.progressPercent = calculateProgressPercent(job.processedCount, job.totalCount)\n }\n\n if (input.etaSeconds !== undefined) {\n job.etaSeconds = input.etaSeconds\n } else if (job.startedAt && job.totalCount) {\n job.etaSeconds = calculateEta(job.processedCount, job.totalCount, job.startedAt)\n }\n\n job.heartbeatAt = new Date()\n\n return persistAndMaybeBroadcast(entry, ctx)\n },\n\n async incrementProgress(jobId, delta, ctx) {\n const entry = await ensureThrottleEntry(jobId, ctx)\n const job = entry.job\n if (TERMINAL_STATUSES.includes(job.status)) {\n const revived = job.status === 'failed' && (await startJobViaCas(em, job, ctx, { staleSweptOnly: true }))\n if (!revived) {\n forgetJobThrottle(jobId)\n return job\n }\n entry.lastPersistedAt = 0\n }\n\n job.processedCount += delta\n entry.pendingDelta += delta\n job.heartbeatAt = new Date()\n\n if (job.totalCount) {\n job.progressPercent = calculateProgressPercent(job.processedCount, job.totalCount)\n if (job.startedAt) {\n job.etaSeconds = calculateEta(job.processedCount, job.totalCount, job.startedAt)\n }\n }\n\n return persistAndMaybeBroadcast(entry, ctx)\n },\n\n async completeJob(jobId, input, ctx) {\n const job = await loadFreshJob(jobId, ctx)\n if (!job) throw new Error(`Job ${jobId} not found`)\n if (job.status === 'cancelled' || job.status === 'completed') {\n forgetJobThrottle(jobId)\n return job\n }\n\n const entry = jobUpdateThrottle.get(jobId)\n const snapshot = entry?.job ?? job\n const now = new Date()\n const data: EntityData<ProgressJob> = {\n status: 'completed',\n finishedAt: now,\n etaSeconds: 0,\n updatedAt: now,\n ...(input?.resultSummary ? { resultSummary: input.resultSummary } : {}),\n ...(entry\n ? {\n totalCount: snapshot.totalCount ?? null,\n meta: snapshot.meta ?? null,\n ...buildBufferedCountData(entry),\n }\n : {}),\n progressPercent: 100,\n }\n\n const affected = await em.nativeUpdate(ProgressJob, {\n ...jobScopeFilter(jobId, ctx),\n status: { $in: COMPLETE_FROM_STATUSES as ProgressJobStatus[] },\n } as FilterQuery<ProgressJob>, data)\n forgetJobThrottle(jobId)\n\n if (affected === 0) {\n const fresh = await loadFreshJob(jobId, ctx)\n return fresh ?? job\n }\n\n snapshot.status = 'completed'\n snapshot.finishedAt = now\n snapshot.progressPercent = 100\n snapshot.etaSeconds = 0\n if (input?.resultSummary) {\n snapshot.resultSummary = input.resultSummary\n }\n\n const persistedJob = (await loadFreshJob(jobId, ctx)) ?? snapshot\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_COMPLETED, {\n ...buildJobPayload(persistedJob),\n resultSummary: persistedJob.resultSummary,\n tenantId: ctx.tenantId,\n organizationId: persistedJob.organizationId ?? null,\n })\n\n return persistedJob\n },\n\n async failJob(jobId, input, ctx) {\n const job = await loadFreshJob(jobId, ctx)\n if (!job) throw new Error(`Job ${jobId} not found`)\n if (TERMINAL_STATUSES.includes(job.status)) {\n forgetJobThrottle(jobId)\n return job\n }\n\n const entry = jobUpdateThrottle.get(jobId)\n const snapshot = entry?.job ?? job\n const now = new Date()\n const data: EntityData<ProgressJob> = {\n status: 'failed',\n finishedAt: now,\n errorMessage: input.errorMessage,\n errorStack: input.errorStack,\n ...(input.resultSummary ? { resultSummary: input.resultSummary } : {}),\n updatedAt: now,\n ...(entry\n ? {\n totalCount: snapshot.totalCount ?? null,\n meta: snapshot.meta ?? null,\n ...buildBufferedCountData(entry),\n }\n : {}),\n }\n\n const affected = await em.nativeUpdate(ProgressJob, {\n ...jobScopeFilter(jobId, ctx),\n status: { $in: FAIL_FROM_STATUSES as ProgressJobStatus[] },\n } as FilterQuery<ProgressJob>, data)\n forgetJobThrottle(jobId)\n\n if (affected === 0) {\n const fresh = await loadFreshJob(jobId, ctx)\n return fresh ?? job\n }\n\n snapshot.status = 'failed'\n snapshot.finishedAt = now\n snapshot.errorMessage = input.errorMessage\n snapshot.errorStack = input.errorStack\n if (input.resultSummary) {\n snapshot.resultSummary = input.resultSummary\n }\n\n const persistedJob = (await loadFreshJob(jobId, ctx)) ?? snapshot\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_FAILED, {\n ...buildJobPayload(persistedJob),\n errorMessage: persistedJob.errorMessage,\n tenantId: ctx.tenantId,\n organizationId: persistedJob.organizationId ?? null,\n })\n\n return persistedJob\n },\n\n async cancelJob(jobId, ctx) {\n const job = await em.findOneOrFail(ProgressJob, jobScopeFilter(jobId, ctx), { disableIdentityMap: true })\n if (TERMINAL_STATUSES.includes(job.status)) {\n // The job finished while the user clicked cancel \u2014 a benign race, not an error.\n return job\n }\n if (!job.cancellable) {\n throw new Error(`Job ${jobId} is not cancellable`)\n }\n\n const now = new Date()\n const cancelledNow = await em.nativeUpdate(ProgressJob, {\n ...jobScopeFilter(jobId, ctx),\n cancellable: true,\n status: 'pending',\n } as FilterQuery<ProgressJob>, {\n status: 'cancelled',\n cancelRequestedAt: now,\n cancelledByUserId: ctx.userId ?? null,\n finishedAt: now,\n etaSeconds: 0,\n updatedAt: now,\n })\n\n let requested = 0\n if (cancelledNow === 0) {\n requested = await em.nativeUpdate(ProgressJob, {\n ...jobScopeFilter(jobId, ctx),\n cancellable: true,\n status: 'running',\n } as FilterQuery<ProgressJob>, {\n cancelRequestedAt: now,\n cancelledByUserId: ctx.userId ?? null,\n updatedAt: now,\n })\n }\n\n if (cancelledNow === 0 && requested === 0) {\n // Raced into a terminal state between the read and the CAS.\n const fresh = await loadFreshJob(jobId, ctx)\n return fresh ?? job\n }\n\n forgetJobThrottle(jobId)\n if (cancelledNow > 0) {\n job.status = 'cancelled'\n job.finishedAt = now\n job.etaSeconds = 0\n }\n job.cancelRequestedAt = now\n job.cancelledByUserId = ctx.userId ?? null\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_CANCELLED, {\n ...buildJobPayload(job),\n tenantId: ctx.tenantId,\n organizationId: job.organizationId ?? null,\n })\n\n return job\n },\n\n async markCancelled(jobId, ctx) {\n const job = await loadFreshJob(jobId, ctx)\n if (!job) throw new Error(`Job ${jobId} not found`)\n if (job.status === 'cancelled' || job.status === 'completed') {\n forgetJobThrottle(jobId)\n return job\n }\n\n const now = new Date()\n const cancelRequestedAt = job.cancelRequestedAt ?? now\n const cancelledByUserId = job.cancelledByUserId ?? ctx.userId ?? null\n const finishedAt = job.finishedAt ?? now\n\n const affected = await em.nativeUpdate(ProgressJob, {\n ...jobScopeFilter(jobId, ctx),\n status: { $in: CANCEL_FROM_STATUSES as ProgressJobStatus[] },\n } as FilterQuery<ProgressJob>, {\n status: 'cancelled',\n cancelRequestedAt,\n cancelledByUserId,\n finishedAt,\n etaSeconds: 0,\n updatedAt: now,\n })\n forgetJobThrottle(jobId)\n\n if (affected === 0) {\n const fresh = await loadFreshJob(jobId, ctx)\n return fresh ?? job\n }\n\n job.status = 'cancelled'\n job.cancelRequestedAt = cancelRequestedAt\n job.cancelledByUserId = cancelledByUserId\n job.finishedAt = finishedAt\n job.etaSeconds = 0\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_CANCELLED, {\n ...buildJobPayload(job),\n tenantId: ctx.tenantId,\n organizationId: job.organizationId ?? null,\n })\n\n return job\n },\n\n async isCancellationRequested(jobId, tenantId, organizationId) {\n // disableIdentityMap forces a fresh read: a managed copy of the job in this\n // EntityManager (loaded by updateProgress) must never mask a cancellation\n // requested from another process.\n const job = await findOneWithDecryption(em, ProgressJob, {\n id: jobId,\n tenantId,\n ...(organizationId ? { organizationId } : {}),\n }, { disableIdentityMap: true })\n return job?.cancelRequestedAt != null\n },\n\n async getActiveJobs(ctx) {\n return em.find(ProgressJob, {\n tenantId: ctx.tenantId,\n ...(ctx.organizationId ? { organizationId: ctx.organizationId } : {}),\n status: { $in: ['pending', 'running'] },\n parentJobId: null,\n }, {\n orderBy: { createdAt: 'DESC' },\n limit: 50,\n })\n },\n\n async getRecentlyCompletedJobs(ctx, sinceSeconds = 30) {\n const cutoff = new Date(Date.now() - sinceSeconds * 1000)\n return em.find(ProgressJob, {\n tenantId: ctx.tenantId,\n ...(ctx.organizationId ? { organizationId: ctx.organizationId } : {}),\n status: { $in: ['completed', 'failed'] },\n finishedAt: { $gte: cutoff },\n parentJobId: null,\n }, {\n orderBy: { finishedAt: 'DESC' },\n limit: 10,\n })\n },\n\n async getJob(jobId, ctx) {\n return em.findOne(ProgressJob, {\n id: jobId,\n tenantId: ctx.tenantId,\n ...(ctx.organizationId ? { organizationId: ctx.organizationId } : {}),\n })\n },\n\n async markStaleJobsFailed(tenantId: string, timeoutSeconds = STALE_JOB_TIMEOUT_SECONDS, organizationId?: string | null) {\n const now = new Date()\n const cutoff = new Date(now.getTime() - timeoutSeconds * 1000)\n const scope = {\n tenantId,\n ...(organizationId ? { organizationId } : {}),\n }\n let failedCount = 0\n\n const staleFilter = {\n status: 'running' as ProgressJobStatus,\n $or: [\n { heartbeatAt: { $lt: cutoff } },\n {\n heartbeatAt: null,\n startedAt: { $lt: cutoff },\n },\n ],\n }\n const staleRunning = await em.find(ProgressJob, { ...scope, ...staleFilter }, { disableIdentityMap: true })\n\n for (const job of staleRunning) {\n const errorMessage = `${STALE_SWEEP_ERROR_PREFIX} no heartbeat for ${timeoutSeconds} seconds`\n // Per-row CAS (re-checking the staleness condition): exactly one concurrent\n // sweeper wins, and a heartbeat that landed after the SELECT aborts the failure.\n const affected = await em.nativeUpdate(ProgressJob, {\n id: job.id,\n tenantId: job.tenantId,\n ...staleFilter,\n } as FilterQuery<ProgressJob>, {\n status: 'failed',\n finishedAt: now,\n errorMessage,\n updatedAt: now,\n })\n if (affected === 0) continue\n\n failedCount += 1\n job.status = 'failed'\n job.finishedAt = now\n job.errorMessage = errorMessage\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_FAILED, {\n ...buildJobPayload(job),\n errorMessage: job.errorMessage,\n tenantId: job.tenantId,\n stale: true,\n organizationId: job.organizationId ?? null,\n })\n }\n\n // Jobs stuck in `pending` (enqueue failed, worker died before startJob) have no\n // heartbeat, so they need their own sweep or they stay in the active list forever.\n // A late queue delivery still recovers such a job: startJob transitions failed \u2192 running.\n const pendingCutoff = new Date(now.getTime() - STALE_PENDING_TIMEOUT_SECONDS * 1000)\n const stalePendingFilter = {\n status: 'pending' as ProgressJobStatus,\n createdAt: { $lt: pendingCutoff },\n }\n const stalePending = await em.find(ProgressJob, { ...scope, ...stalePendingFilter }, { disableIdentityMap: true })\n\n for (const job of stalePending) {\n const errorMessage = `${STALE_SWEEP_ERROR_PREFIX} never started within ${STALE_PENDING_TIMEOUT_SECONDS} seconds`\n const affected = await em.nativeUpdate(ProgressJob, {\n id: job.id,\n tenantId: job.tenantId,\n ...stalePendingFilter,\n } as FilterQuery<ProgressJob>, {\n status: 'failed',\n finishedAt: now,\n errorMessage,\n updatedAt: now,\n })\n if (affected === 0) continue\n\n failedCount += 1\n job.status = 'failed'\n job.finishedAt = now\n job.errorMessage = errorMessage\n\n await eventBus.emit(PROGRESS_EVENTS.JOB_FAILED, {\n ...buildJobPayload(job),\n errorMessage: job.errorMessage,\n tenantId: job.tenantId,\n stale: true,\n organizationId: job.organizationId ?? null,\n })\n }\n\n return failedCount\n },\n }\n}\n"],
|
|
5
|
+
"mappings": "AACA,SAAS,WAAW;AAEpB,SAAS,mBAA2C;AAEpD;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,uBAAuB;AAChC,SAAS,6BAA6B;AAEtC,MAAM,oCAAoC;AAS1C,SAAS,gCAAwC;AAC/C,QAAM,WAAW,QAAQ,IAAI;AAC7B,MAAI,YAAY,QAAQ,SAAS,KAAK,MAAM,GAAI,QAAO;AACvD,QAAM,SAAS,OAAO,SAAS,UAAU,EAAE;AAC3C,MAAI,CAAC,OAAO,SAAS,MAAM,KAAK,SAAS,EAAG,QAAO;AACnD,SAAO;AACT;AAMA,MAAM,oBAAkD,CAAC,aAAa,UAAU,WAAW;AAC3F,MAAM,6BAA2D,CAAC,WAAW,SAAS;AAGtF,MAAM,sBAAoD,CAAC,WAAW,QAAQ;AAC9E,MAAM,yBAAuD,CAAC,WAAW,WAAW,QAAQ;AAC5F,MAAM,qBAAmD,CAAC,WAAW,SAAS;AAC9E,MAAM,uBAAqD,CAAC,WAAW,WAAW,QAAQ;AAW1F,SAAS,gBAAgB,KAA2C;AAClE,SAAO;AAAA,IACL,OAAO,IAAI;AAAA,IACX,SAAS,IAAI;AAAA,IACb,MAAM,IAAI;AAAA,IACV,aAAa,IAAI,eAAe;AAAA,IAChC,QAAQ,IAAI;AAAA,IACZ,iBAAiB,IAAI;AAAA,IACrB,gBAAgB,IAAI;AAAA,IACpB,YAAY,IAAI,cAAc;AAAA,IAC9B,YAAY,IAAI,cAAc;AAAA,IAC9B,aAAa,IAAI;AAAA,IACjB,MAAM,IAAI,QAAQ;AAAA,IAClB,WAAW,IAAI,WAAW,YAAY,KAAK;AAAA,IAC3C,YAAY,IAAI,YAAY,YAAY,KAAK;AAAA,EAC/C;AACF;AAEO,SAAS,sBAAsB,IAAmB,UAAyG;AAChK,QAAM,yBAAyB,8BAA8B;AAM7D,QAAM,oBAAoB,oBAAI,IAAoC;AAElE,WAAS,eAAe,OAAe,KAA6B;AAClE,WAAO;AAAA,MACL,IAAI;AAAA,MACJ,UAAU,IAAI;AAAA,MACd,GAAI,IAAI,iBAAiB,EAAE,gBAAgB,IAAI,eAAe,IAAI,CAAC;AAAA,IACrE;AAAA,EACF;AAEA,iBAAe,aAAa,OAAe,KAA0D;AACnG,WAAO,GAAG,QAAQ,aAAa,eAAe,OAAO,GAAG,GAAG,EAAE,oBAAoB,KAAK,CAAC;AAAA,EACzF;AAEA,iBAAe,oBAAoB,OAAe,KAA8D;AAC9G,UAAM,SAAS,kBAAkB,IAAI,KAAK;AAC1C,QAAI,OAAQ,QAAO;AACnB,UAAM,MAAM,MAAM,GAAG,cAAc,aAAa,eAAe,OAAO,GAAG,GAAG,EAAE,oBAAoB,KAAK,CAAC;AACxG,UAAM,QAAgC;AAAA,MACpC;AAAA,MACA,iBAAiB;AAAA,MACjB,iBAAiB;AAAA,MACjB,sBAAsB,IAAI;AAAA,MAC1B,cAAc;AAAA,MACd,uBAAuB;AAAA,IACzB;AACA,sBAAkB,IAAI,OAAO,KAAK;AAClC,WAAO;AAAA,EACT;AAEA,WAAS,kBAAkB,OAAe;AACxC,sBAAkB,OAAO,KAAK;AAAA,EAChC;AAEA,WAAS,uBAAuB,OAAwD;AACtF,QAAI,MAAM,yBAAyB,CAAC,OAAO,cAAc,MAAM,YAAY,GAAG;AAC5E,aAAO,EAAE,gBAAgB,MAAM,IAAI,eAAe;AAAA,IACpD;AACA,QAAI,MAAM,iBAAiB,GAAG;AAC5B,YAAM,iBAAiB,IAAI,qBAAqB,MAAM,YAAY,EAAE;AACpE,YAAM,aAAa,MAAM,IAAI;AAC7B,UAAI,OAAO,cAAc,UAAU,KAAK,cAAc,QAAQ,aAAa,GAAG;AAC5E,eAAO;AAAA,UACL;AAAA,UACA,iBAAiB;AAAA,YACf,wCAAwC,MAAM,YAAY,gBAAgB,UAAU;AAAA,UACtF;AAAA,QACF;AAAA,MACF;AACA,aAAO,EAAE,eAAe;AAAA,IAC1B;AACA,WAAO,CAAC;AAAA,EACV;AAWA,iBAAe,eACb,UACA,KACA,KACA,UAAwC,CAAC,GACZ;AAC7B,UAAM,MAAM,oBAAI,KAAK;AAIrB,UAAM,YAAY,IAAI,aAAa;AACnC,UAAM,SAAkC;AAAA,MACtC,GAAG,eAAe,IAAI,IAAI,GAAG;AAAA,MAC7B,QAAQ,EAAE,KAAK,oBAA2C;AAAA,IAC5D;AACA,QAAI,QAAQ,eAAgB,QAAO,eAAe,EAAE,OAAO,GAAG,wBAAwB,IAAI;AAC1F,UAAM,WAAW,MAAM,SAAS,aAAa,aAAa,QAAoC;AAAA,MAC5F,QAAQ;AAAA,MACR;AAAA,MACA,aAAa;AAAA,MACb,YAAY;AAAA,MACZ,cAAc;AAAA,MACd,YAAY;AAAA,MACZ,WAAW;AAAA,IACb,CAAC;AACD,QAAI,aAAa,EAAG,QAAO;AAE3B,QAAI,SAAS;AACb,QAAI,YAAY;AAChB,QAAI,cAAc;AAClB,QAAI,aAAa;AACjB,QAAI,eAAe;AACnB,QAAI,aAAa;AAEjB,UAAM,SAAS,KAAK,gBAAgB,aAAa;AAAA,MAC/C,GAAG,gBAAgB,GAAG;AAAA,MACtB,UAAU,IAAI;AAAA,MACd,gBAAgB,IAAI,kBAAkB;AAAA,IACxC,CAAC;AAED,WAAO;AAAA,EACT;AAEA,iBAAe,yBAAyB,OAA+B,KAAmD;AACxH,UAAM,MAAM,MAAM;AAClB,UAAM,MAAM,KAAK,IAAI;AACrB,UAAM,kBACJ,0BAA0B,KAC1B,MAAM,MAAM,mBAAmB,0BAC/B,KAAK,IAAI,IAAI,kBAAkB,MAAM,oBAAoB,KAAK;AAChE,UAAM,gBAAgB,mBAAmB,MAAM,MAAM,mBAAmB;AAExE,QAAI,CAAC,cAAe,QAAO;AAE3B,UAAM,sBACJ,CAAC,MAAM,yBAAyB,OAAO,cAAc,MAAM,YAAY,KAAK,MAAM,iBAAiB;AACrG,UAAM,OAAgC;AAAA,MACpC,iBAAiB,IAAI;AAAA,MACrB,YAAY,IAAI,cAAc;AAAA,MAC9B,YAAY,IAAI,cAAc;AAAA,MAC9B,MAAM,IAAI,QAAQ;AAAA,MAClB,aAAa,IAAI,eAAe,oBAAI,KAAK;AAAA,MACzC,WAAW,oBAAI,KAAK;AAAA,MACpB,GAAG,uBAAuB,KAAK;AAAA,IACjC;AACA,UAAM,iBAAiB;AAAA,MACrB,GAAG,eAAe,IAAI,IAAI,GAAG;AAAA,MAC7B,QAAQ,EAAE,KAAK,2BAAkD;AAAA,IACnE;AACA,QAAI,WAAW,MAAM,GAAG,aAAa,aAAa,gBAAgB,IAAI;AAEtE,QAAI,aAAa,GAAG;AAGlB,YAAM,QAAQ,MAAM,aAAa,IAAI,IAAI,GAAG;AAC5C,UAAI,UAAU;AACd,UAAI,SAAS,MAAM,WAAW,YAAa,MAAM,eAAe,IAAI,OAAO,KAAK,EAAE,gBAAgB,KAAK,CAAC,GAAI;AAC1G,kBAAU;AACV,YAAI,SAAS;AACb,YAAI,YAAY,MAAM;AACtB,YAAI,aAAa;AACjB,YAAI,eAAe;AACnB,YAAI,aAAa;AACjB,mBAAW,MAAM,GAAG,aAAa,aAAa,gBAAgB,IAAI;AAAA,MACpE;AACA,UAAI,aAAa,GAAG;AAElB,0BAAkB,IAAI,EAAE;AACxB,cAAM,SAAS,UAAU,MAAM,aAAa,IAAI,IAAI,GAAG,IAAI;AAC3D,eAAO,UAAU;AAAA,MACnB;AAAA,IACF;AAEA,UAAM,eAAe,sBAAuB,MAAM,aAAa,IAAI,IAAI,GAAG,KAAM,MAAM;AACtF,UAAM,MAAM;AACZ,UAAM,eAAe;AACrB,UAAM,wBAAwB;AAC9B,UAAM,kBAAkB;AAExB,QAAI,iBAAiB;AACnB,YAAM,SAAS,KAAK,gBAAgB,aAAa;AAAA,QAC/C,GAAG,gBAAgB,YAAY;AAAA,QAC/B,UAAU,IAAI;AAAA,QACd,gBAAgB,aAAa,kBAAkB;AAAA,MACjD,CAAC;AACD,YAAM,kBAAkB;AACxB,YAAM,uBAAuB,aAAa;AAAA,IAC5C;AAEA,WAAO;AAAA,EACT;AAEA,SAAO;AAAA,IACL,MAAM,UAAU,OAAO,KAAK;AAC1B,YAAM,MAAM,GAAG,OAAO,aAAa;AAAA,QACjC,SAAS,MAAM;AAAA,QACf,MAAM,MAAM;AAAA,QACZ,aAAa,MAAM;AAAA,QACnB,YAAY,MAAM;AAAA,QAClB,aAAa,MAAM,eAAe;AAAA,QAClC,MAAM,MAAM;AAAA,QACZ,aAAa,MAAM;AAAA,QACnB,gBAAgB,MAAM;AAAA,QACtB,gBAAgB,MAAM;AAAA,QACtB,iBAAiB,IAAI;AAAA,QACrB,UAAU,IAAI;AAAA,QACd,gBAAgB,IAAI;AAAA,QACpB,QAAQ;AAAA,MACV,CAAC;AAED,YAAM,GAAG,QAAQ,GAAG,EAAE,MAAM;AAE5B,YAAM,SAAS,KAAK,gBAAgB,aAAa;AAAA,QAC/C,GAAG,gBAAgB,GAAG;AAAA,QACtB,UAAU,IAAI;AAAA,QACd,gBAAgB,IAAI;AAAA,MACtB,CAAC;AAED,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,SAAS,OAAO,KAAK;AACzB,YAAM,MAAM,MAAM,GAAG,cAAc,aAAa,eAAe,OAAO,GAAG,GAAG,EAAE,oBAAoB,KAAK,CAAC;AACxG,UAAI,IAAI,WAAW,aAAa,IAAI,WAAW,eAAe,IAAI,WAAW,aAAa;AACxF,eAAO;AAAA,MACT;AAEA,YAAM,UAAU,MAAM,eAAe,IAAI,KAAK,GAAG;AACjD,UAAI,QAAS,QAAO;AAEpB,YAAM,QAAQ,MAAM,aAAa,OAAO,GAAG;AAC3C,aAAO,SAAS;AAAA,IAClB;AAAA,IAEA,MAAM,kBAAkB,OAAO,KAAK;AAGlC,YAAM,OAAO,GAAG,KAAK;AACrB,YAAM,MAAM,oBAAI,KAAK;AACrB,YAAM,WAAW,MAAM,KAAK,aAAa,aAAa;AAAA,QACpD,GAAG,eAAe,OAAO,GAAG;AAAA,QAC5B,QAAQ,EAAE,KAAK,2BAAkD;AAAA,MACnE,GAA+B;AAAA,QAC7B,aAAa;AAAA,QACb,WAAW;AAAA,MACb,CAAC;AACD,UAAI,WAAW,EAAG;AAElB,YAAM,MAAM,MAAM,KAAK,QAAQ,aAAa,eAAe,OAAO,GAAG,GAAG,EAAE,oBAAoB,KAAK,CAAC;AACpG,UAAI,CAAC,OAAO,IAAI,WAAW,SAAU;AAGrC,YAAM,eAAe,MAAM,KAAK,KAAK,EAAE,gBAAgB,KAAK,CAAC;AAAA,IAC/D;AAAA,IAEA,MAAM,eAAe,OAAO,OAAO,KAAK;AACtC,YAAM,QAAQ,MAAM,oBAAoB,OAAO,GAAG;AAClD,YAAM,MAAM,MAAM;AAClB,UAAI,kBAAkB,SAAS,IAAI,MAAM,GAAG;AAK1C,cAAM,UAAU,IAAI,WAAW,YAAa,MAAM,eAAe,IAAI,KAAK,KAAK,EAAE,gBAAgB,KAAK,CAAC;AACvG,YAAI,CAAC,SAAS;AACZ,4BAAkB,KAAK;AACvB,iBAAO;AAAA,QACT;AACA,cAAM,kBAAkB;AAAA,MAC1B;AAEA,UAAI,MAAM,mBAAmB,QAAW;AACtC,YAAI,iBAAiB,MAAM;AAC3B,cAAM,wBAAwB;AAC9B,cAAM,eAAe;AAAA,MACvB;AACA,UAAI,MAAM,eAAe,QAAW;AAClC,YAAI,aAAa,MAAM;AAAA,MACzB;AACA,UAAI,MAAM,SAAS,QAAW;AAC5B,YAAI,OAAO,EAAE,GAAG,IAAI,MAAM,GAAG,MAAM,KAAK;AAAA,MAC1C;AAEA,UAAI,MAAM,oBAAoB,QAAW;AACvC,YAAI,kBAAkB,MAAM;AAAA,MAC9B,WAAW,IAAI,YAAY;AACzB,YAAI,kBAAkB,yBAAyB,IAAI,gBAAgB,IAAI,UAAU;AAAA,MACnF;AAEA,UAAI,MAAM,eAAe,QAAW;AAClC,YAAI,aAAa,MAAM;AAAA,MACzB,WAAW,IAAI,aAAa,IAAI,YAAY;AAC1C,YAAI,aAAa,aAAa,IAAI,gBAAgB,IAAI,YAAY,IAAI,SAAS;AAAA,MACjF;AAEA,UAAI,cAAc,oBAAI,KAAK;AAE3B,aAAO,yBAAyB,OAAO,GAAG;AAAA,IAC5C;AAAA,IAEA,MAAM,kBAAkB,OAAO,OAAO,KAAK;AACzC,YAAM,QAAQ,MAAM,oBAAoB,OAAO,GAAG;AAClD,YAAM,MAAM,MAAM;AAClB,UAAI,kBAAkB,SAAS,IAAI,MAAM,GAAG;AAC1C,cAAM,UAAU,IAAI,WAAW,YAAa,MAAM,eAAe,IAAI,KAAK,KAAK,EAAE,gBAAgB,KAAK,CAAC;AACvG,YAAI,CAAC,SAAS;AACZ,4BAAkB,KAAK;AACvB,iBAAO;AAAA,QACT;AACA,cAAM,kBAAkB;AAAA,MAC1B;AAEA,UAAI,kBAAkB;AACtB,YAAM,gBAAgB;AACtB,UAAI,cAAc,oBAAI,KAAK;AAE3B,UAAI,IAAI,YAAY;AAClB,YAAI,kBAAkB,yBAAyB,IAAI,gBAAgB,IAAI,UAAU;AACjF,YAAI,IAAI,WAAW;AACjB,cAAI,aAAa,aAAa,IAAI,gBAAgB,IAAI,YAAY,IAAI,SAAS;AAAA,QACjF;AAAA,MACF;AAEA,aAAO,yBAAyB,OAAO,GAAG;AAAA,IAC5C;AAAA,IAEA,MAAM,YAAY,OAAO,OAAO,KAAK;AACnC,YAAM,MAAM,MAAM,aAAa,OAAO,GAAG;AACzC,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,OAAO,KAAK,YAAY;AAClD,UAAI,IAAI,WAAW,eAAe,IAAI,WAAW,aAAa;AAC5D,0BAAkB,KAAK;AACvB,eAAO;AAAA,MACT;AAEA,YAAM,QAAQ,kBAAkB,IAAI,KAAK;AACzC,YAAM,WAAW,OAAO,OAAO;AAC/B,YAAM,MAAM,oBAAI,KAAK;AACrB,YAAM,OAAgC;AAAA,QACpC,QAAQ;AAAA,QACR,YAAY;AAAA,QACZ,YAAY;AAAA,QACZ,WAAW;AAAA,QACX,GAAI,OAAO,gBAAgB,EAAE,eAAe,MAAM,cAAc,IAAI,CAAC;AAAA,QACrE,GAAI,QACA;AAAA,UACE,YAAY,SAAS,cAAc;AAAA,UACnC,MAAM,SAAS,QAAQ;AAAA,UACvB,GAAG,uBAAuB,KAAK;AAAA,QACjC,IACA,CAAC;AAAA,QACL,iBAAiB;AAAA,MACnB;AAEA,YAAM,WAAW,MAAM,GAAG,aAAa,aAAa;AAAA,QAClD,GAAG,eAAe,OAAO,GAAG;AAAA,QAC5B,QAAQ,EAAE,KAAK,uBAA8C;AAAA,MAC/D,GAA+B,IAAI;AACnC,wBAAkB,KAAK;AAEvB,UAAI,aAAa,GAAG;AAClB,cAAM,QAAQ,MAAM,aAAa,OAAO,GAAG;AAC3C,eAAO,SAAS;AAAA,MAClB;AAEA,eAAS,SAAS;AAClB,eAAS,aAAa;AACtB,eAAS,kBAAkB;AAC3B,eAAS,aAAa;AACtB,UAAI,OAAO,eAAe;AACxB,iBAAS,gBAAgB,MAAM;AAAA,MACjC;AAEA,YAAM,eAAgB,MAAM,aAAa,OAAO,GAAG,KAAM;AAEzD,YAAM,SAAS,KAAK,gBAAgB,eAAe;AAAA,QACjD,GAAG,gBAAgB,YAAY;AAAA,QAC/B,eAAe,aAAa;AAAA,QAC5B,UAAU,IAAI;AAAA,QACd,gBAAgB,aAAa,kBAAkB;AAAA,MACjD,CAAC;AAED,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,QAAQ,OAAO,OAAO,KAAK;AAC/B,YAAM,MAAM,MAAM,aAAa,OAAO,GAAG;AACzC,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,OAAO,KAAK,YAAY;AAClD,UAAI,kBAAkB,SAAS,IAAI,MAAM,GAAG;AAC1C,0BAAkB,KAAK;AACvB,eAAO;AAAA,MACT;AAEA,YAAM,QAAQ,kBAAkB,IAAI,KAAK;AACzC,YAAM,WAAW,OAAO,OAAO;AAC/B,YAAM,MAAM,oBAAI,KAAK;AACrB,YAAM,OAAgC;AAAA,QACpC,QAAQ;AAAA,QACR,YAAY;AAAA,QACZ,cAAc,MAAM;AAAA,QACpB,YAAY,MAAM;AAAA,QAClB,GAAI,MAAM,gBAAgB,EAAE,eAAe,MAAM,cAAc,IAAI,CAAC;AAAA,QACpE,WAAW;AAAA,QACX,GAAI,QACA;AAAA,UACE,YAAY,SAAS,cAAc;AAAA,UACnC,MAAM,SAAS,QAAQ;AAAA,UACvB,GAAG,uBAAuB,KAAK;AAAA,QACjC,IACA,CAAC;AAAA,MACP;AAEA,YAAM,WAAW,MAAM,GAAG,aAAa,aAAa;AAAA,QAClD,GAAG,eAAe,OAAO,GAAG;AAAA,QAC5B,QAAQ,EAAE,KAAK,mBAA0C;AAAA,MAC3D,GAA+B,IAAI;AACnC,wBAAkB,KAAK;AAEvB,UAAI,aAAa,GAAG;AAClB,cAAM,QAAQ,MAAM,aAAa,OAAO,GAAG;AAC3C,eAAO,SAAS;AAAA,MAClB;AAEA,eAAS,SAAS;AAClB,eAAS,aAAa;AACtB,eAAS,eAAe,MAAM;AAC9B,eAAS,aAAa,MAAM;AAC5B,UAAI,MAAM,eAAe;AACvB,iBAAS,gBAAgB,MAAM;AAAA,MACjC;AAEA,YAAM,eAAgB,MAAM,aAAa,OAAO,GAAG,KAAM;AAEzD,YAAM,SAAS,KAAK,gBAAgB,YAAY;AAAA,QAC9C,GAAG,gBAAgB,YAAY;AAAA,QAC/B,cAAc,aAAa;AAAA,QAC3B,UAAU,IAAI;AAAA,QACd,gBAAgB,aAAa,kBAAkB;AAAA,MACjD,CAAC;AAED,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,UAAU,OAAO,KAAK;AAC1B,YAAM,MAAM,MAAM,GAAG,cAAc,aAAa,eAAe,OAAO,GAAG,GAAG,EAAE,oBAAoB,KAAK,CAAC;AACxG,UAAI,kBAAkB,SAAS,IAAI,MAAM,GAAG;AAE1C,eAAO;AAAA,MACT;AACA,UAAI,CAAC,IAAI,aAAa;AACpB,cAAM,IAAI,MAAM,OAAO,KAAK,qBAAqB;AAAA,MACnD;AAEA,YAAM,MAAM,oBAAI,KAAK;AACrB,YAAM,eAAe,MAAM,GAAG,aAAa,aAAa;AAAA,QACtD,GAAG,eAAe,OAAO,GAAG;AAAA,QAC5B,aAAa;AAAA,QACb,QAAQ;AAAA,MACV,GAA+B;AAAA,QAC7B,QAAQ;AAAA,QACR,mBAAmB;AAAA,QACnB,mBAAmB,IAAI,UAAU;AAAA,QACjC,YAAY;AAAA,QACZ,YAAY;AAAA,QACZ,WAAW;AAAA,MACb,CAAC;AAED,UAAI,YAAY;AAChB,UAAI,iBAAiB,GAAG;AACtB,oBAAY,MAAM,GAAG,aAAa,aAAa;AAAA,UAC7C,GAAG,eAAe,OAAO,GAAG;AAAA,UAC5B,aAAa;AAAA,UACb,QAAQ;AAAA,QACV,GAA+B;AAAA,UAC7B,mBAAmB;AAAA,UACnB,mBAAmB,IAAI,UAAU;AAAA,UACjC,WAAW;AAAA,QACb,CAAC;AAAA,MACH;AAEA,UAAI,iBAAiB,KAAK,cAAc,GAAG;AAEzC,cAAM,QAAQ,MAAM,aAAa,OAAO,GAAG;AAC3C,eAAO,SAAS;AAAA,MAClB;AAEA,wBAAkB,KAAK;AACvB,UAAI,eAAe,GAAG;AACpB,YAAI,SAAS;AACb,YAAI,aAAa;AACjB,YAAI,aAAa;AAAA,MACnB;AACA,UAAI,oBAAoB;AACxB,UAAI,oBAAoB,IAAI,UAAU;AAEtC,YAAM,SAAS,KAAK,gBAAgB,eAAe;AAAA,QACjD,GAAG,gBAAgB,GAAG;AAAA,QACtB,UAAU,IAAI;AAAA,QACd,gBAAgB,IAAI,kBAAkB;AAAA,MACxC,CAAC;AAED,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,cAAc,OAAO,KAAK;AAC9B,YAAM,MAAM,MAAM,aAAa,OAAO,GAAG;AACzC,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,OAAO,KAAK,YAAY;AAClD,UAAI,IAAI,WAAW,eAAe,IAAI,WAAW,aAAa;AAC5D,0BAAkB,KAAK;AACvB,eAAO;AAAA,MACT;AAEA,YAAM,MAAM,oBAAI,KAAK;AACrB,YAAM,oBAAoB,IAAI,qBAAqB;AACnD,YAAM,oBAAoB,IAAI,qBAAqB,IAAI,UAAU;AACjE,YAAM,aAAa,IAAI,cAAc;AAErC,YAAM,WAAW,MAAM,GAAG,aAAa,aAAa;AAAA,QAClD,GAAG,eAAe,OAAO,GAAG;AAAA,QAC5B,QAAQ,EAAE,KAAK,qBAA4C;AAAA,MAC7D,GAA+B;AAAA,QAC7B,QAAQ;AAAA,QACR;AAAA,QACA;AAAA,QACA;AAAA,QACA,YAAY;AAAA,QACZ,WAAW;AAAA,MACb,CAAC;AACD,wBAAkB,KAAK;AAEvB,UAAI,aAAa,GAAG;AAClB,cAAM,QAAQ,MAAM,aAAa,OAAO,GAAG;AAC3C,eAAO,SAAS;AAAA,MAClB;AAEA,UAAI,SAAS;AACb,UAAI,oBAAoB;AACxB,UAAI,oBAAoB;AACxB,UAAI,aAAa;AACjB,UAAI,aAAa;AAEjB,YAAM,SAAS,KAAK,gBAAgB,eAAe;AAAA,QACjD,GAAG,gBAAgB,GAAG;AAAA,QACtB,UAAU,IAAI;AAAA,QACd,gBAAgB,IAAI,kBAAkB;AAAA,MACxC,CAAC;AAED,aAAO;AAAA,IACT;AAAA,IAEA,MAAM,wBAAwB,OAAO,UAAU,gBAAgB;AAI7D,YAAM,MAAM,MAAM,sBAAsB,IAAI,aAAa;AAAA,QACvD,IAAI;AAAA,QACJ;AAAA,QACA,GAAI,iBAAiB,EAAE,eAAe,IAAI,CAAC;AAAA,MAC7C,GAAG,EAAE,oBAAoB,KAAK,CAAC;AAC/B,aAAO,KAAK,qBAAqB;AAAA,IACnC;AAAA,IAEA,MAAM,cAAc,KAAK;AACvB,aAAO,GAAG,KAAK,aAAa;AAAA,QAC1B,UAAU,IAAI;AAAA,QACd,GAAI,IAAI,iBAAiB,EAAE,gBAAgB,IAAI,eAAe,IAAI,CAAC;AAAA,QACnE,QAAQ,EAAE,KAAK,CAAC,WAAW,SAAS,EAAE;AAAA,QACtC,aAAa;AAAA,MACf,GAAG;AAAA,QACD,SAAS,EAAE,WAAW,OAAO;AAAA,QAC7B,OAAO;AAAA,MACT,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,yBAAyB,KAAK,eAAe,IAAI;AACrD,YAAM,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI,eAAe,GAAI;AACxD,aAAO,GAAG,KAAK,aAAa;AAAA,QAC1B,UAAU,IAAI;AAAA,QACd,GAAI,IAAI,iBAAiB,EAAE,gBAAgB,IAAI,eAAe,IAAI,CAAC;AAAA,QACnE,QAAQ,EAAE,KAAK,CAAC,aAAa,QAAQ,EAAE;AAAA,QACvC,YAAY,EAAE,MAAM,OAAO;AAAA,QAC3B,aAAa;AAAA,MACf,GAAG;AAAA,QACD,SAAS,EAAE,YAAY,OAAO;AAAA,QAC9B,OAAO;AAAA,MACT,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,OAAO,OAAO,KAAK;AACvB,aAAO,GAAG,QAAQ,aAAa;AAAA,QAC7B,IAAI;AAAA,QACJ,UAAU,IAAI;AAAA,QACd,GAAI,IAAI,iBAAiB,EAAE,gBAAgB,IAAI,eAAe,IAAI,CAAC;AAAA,MACrE,CAAC;AAAA,IACH;AAAA,IAEA,MAAM,oBAAoB,UAAkB,iBAAiB,2BAA2B,gBAAgC;AACtH,YAAM,MAAM,oBAAI,KAAK;AACrB,YAAM,SAAS,IAAI,KAAK,IAAI,QAAQ,IAAI,iBAAiB,GAAI;AAC7D,YAAM,QAAQ;AAAA,QACZ;AAAA,QACA,GAAI,iBAAiB,EAAE,eAAe,IAAI,CAAC;AAAA,MAC7C;AACA,UAAI,cAAc;AAElB,YAAM,cAAc;AAAA,QAClB,QAAQ;AAAA,QACR,KAAK;AAAA,UACH,EAAE,aAAa,EAAE,KAAK,OAAO,EAAE;AAAA,UAC/B;AAAA,YACE,aAAa;AAAA,YACb,WAAW,EAAE,KAAK,OAAO;AAAA,UAC3B;AAAA,QACF;AAAA,MACF;AACA,YAAM,eAAe,MAAM,GAAG,KAAK,aAAa,EAAE,GAAG,OAAO,GAAG,YAAY,GAAG,EAAE,oBAAoB,KAAK,CAAC;AAE1G,iBAAW,OAAO,cAAc;AAC9B,cAAM,eAAe,GAAG,wBAAwB,qBAAqB,cAAc;AAGnF,cAAM,WAAW,MAAM,GAAG,aAAa,aAAa;AAAA,UAClD,IAAI,IAAI;AAAA,UACR,UAAU,IAAI;AAAA,UACd,GAAG;AAAA,QACL,GAA+B;AAAA,UAC7B,QAAQ;AAAA,UACR,YAAY;AAAA,UACZ;AAAA,UACA,WAAW;AAAA,QACb,CAAC;AACD,YAAI,aAAa,EAAG;AAEpB,uBAAe;AACf,YAAI,SAAS;AACb,YAAI,aAAa;AACjB,YAAI,eAAe;AAEnB,cAAM,SAAS,KAAK,gBAAgB,YAAY;AAAA,UAC9C,GAAG,gBAAgB,GAAG;AAAA,UACtB,cAAc,IAAI;AAAA,UAClB,UAAU,IAAI;AAAA,UACd,OAAO;AAAA,UACP,gBAAgB,IAAI,kBAAkB;AAAA,QACxC,CAAC;AAAA,MACH;AAKA,YAAM,gBAAgB,IAAI,KAAK,IAAI,QAAQ,IAAI,gCAAgC,GAAI;AACnF,YAAM,qBAAqB;AAAA,QACzB,QAAQ;AAAA,QACR,WAAW,EAAE,KAAK,cAAc;AAAA,MAClC;AACA,YAAM,eAAe,MAAM,GAAG,KAAK,aAAa,EAAE,GAAG,OAAO,GAAG,mBAAmB,GAAG,EAAE,oBAAoB,KAAK,CAAC;AAEjH,iBAAW,OAAO,cAAc;AAC9B,cAAM,eAAe,GAAG,wBAAwB,yBAAyB,6BAA6B;AACtG,cAAM,WAAW,MAAM,GAAG,aAAa,aAAa;AAAA,UAClD,IAAI,IAAI;AAAA,UACR,UAAU,IAAI;AAAA,UACd,GAAG;AAAA,QACL,GAA+B;AAAA,UAC7B,QAAQ;AAAA,UACR,YAAY;AAAA,UACZ;AAAA,UACA,WAAW;AAAA,QACb,CAAC;AACD,YAAI,aAAa,EAAG;AAEpB,uBAAe;AACf,YAAI,SAAS;AACb,YAAI,aAAa;AACjB,YAAI,eAAe;AAEnB,cAAM,SAAS,KAAK,gBAAgB,YAAY;AAAA,UAC9C,GAAG,gBAAgB,GAAG;AAAA,UACtB,cAAc,IAAI;AAAA,UAClB,UAAU,IAAI;AAAA,UACd,OAAO;AAAA,UACP,gBAAgB,IAAI,kBAAkB;AAAA,QACxC,CAAC;AAAA,MACH;AAEA,aAAO;AAAA,IACT;AAAA,EACF;AACF;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@open-mercato/core",
|
|
3
|
-
"version": "0.6.8-develop.
|
|
3
|
+
"version": "0.6.8-develop.6987.1.02f619470c",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -254,16 +254,16 @@
|
|
|
254
254
|
"zod": "^4.4.3"
|
|
255
255
|
},
|
|
256
256
|
"peerDependencies": {
|
|
257
|
-
"@open-mercato/ai-assistant": "0.6.8-develop.
|
|
258
|
-
"@open-mercato/shared": "0.6.8-develop.
|
|
259
|
-
"@open-mercato/ui": "0.6.8-develop.
|
|
257
|
+
"@open-mercato/ai-assistant": "0.6.8-develop.6987.1.02f619470c",
|
|
258
|
+
"@open-mercato/shared": "0.6.8-develop.6987.1.02f619470c",
|
|
259
|
+
"@open-mercato/ui": "0.6.8-develop.6987.1.02f619470c",
|
|
260
260
|
"react": "^19.0.0",
|
|
261
261
|
"react-dom": "^19.0.0"
|
|
262
262
|
},
|
|
263
263
|
"devDependencies": {
|
|
264
|
-
"@open-mercato/ai-assistant": "0.6.8-develop.
|
|
265
|
-
"@open-mercato/shared": "0.6.8-develop.
|
|
266
|
-
"@open-mercato/ui": "0.6.8-develop.
|
|
264
|
+
"@open-mercato/ai-assistant": "0.6.8-develop.6987.1.02f619470c",
|
|
265
|
+
"@open-mercato/shared": "0.6.8-develop.6987.1.02f619470c",
|
|
266
|
+
"@open-mercato/ui": "0.6.8-develop.6987.1.02f619470c",
|
|
267
267
|
"@testing-library/dom": "^10.4.1",
|
|
268
268
|
"@testing-library/jest-dom": "^7.0.0",
|
|
269
269
|
"@testing-library/react": "^16.3.1",
|
|
@@ -172,6 +172,8 @@ Data sync providers can leverage the **Unified Module Extension System (UMES)**
|
|
|
172
172
|
|
|
173
173
|
- `ProgressTopBar` and sync-run detail pages use `progress.job.*` SSE updates for live progress.
|
|
174
174
|
- Create `ProgressJob` in `run`/`retry` endpoints; start/update/complete/fail in `sync-engine`.
|
|
175
|
+
- The engine heartbeats (`touchJobHeartbeat`, forked-EM) on a timer while an adapter batch is being produced, because batches can outlast the 60s stale-job sweep — keep the `withHeartbeat` wrapper around `streamImport`/`streamExport` when touching the batch loops.
|
|
176
|
+
- On redelivery the progress counter is seeded from the progress job's own `processedCount` (`progressService.getJob`), mirroring how `committedBatches` resumes — never reset it to zero (`updateProgress` writes absolute counts) and never seed it from the run's `created/updated/skipped/failed` columns: those count emitted items, while progress counts source records (`batch.processedCount`), and adapters may emit several items per source record.
|
|
175
177
|
- Include `progressJob` details in run detail response.
|
|
176
178
|
- SSE DOM bridge forwards only events with `clientBroadcast: true`.
|
|
177
179
|
- `progress.job.*` events are marked `clientBroadcast: true` and must reach the browser from both web and worker processes.
|
|
@@ -4,6 +4,7 @@ import type { CredentialsService } from '../../integrations/lib/credentials-serv
|
|
|
4
4
|
import type { IntegrationLogService } from '../../integrations/lib/log-service'
|
|
5
5
|
import type { IntegrationStateService } from '../../integrations/lib/state-service'
|
|
6
6
|
import type { ProgressService } from '../../progress/lib/progressService'
|
|
7
|
+
import { STALE_JOB_TIMEOUT_SECONDS } from '../../progress/lib/progressService'
|
|
7
8
|
import { refreshCoverageSnapshot } from '../../query_index/lib/coverage'
|
|
8
9
|
import { emitDataSyncEvent } from '../events'
|
|
9
10
|
import type { DataSyncAdapter, DataMapping, ExportBatch, ImportBatch } from './adapter'
|
|
@@ -76,6 +77,32 @@ function applyExportCounters(batch: ExportBatch): SyncCounterDelta {
|
|
|
76
77
|
}
|
|
77
78
|
}
|
|
78
79
|
|
|
80
|
+
// Adapter batches can legitimately outlast the stale-job sweep window (slow upstream
|
|
81
|
+
// APIs), so the engine must heartbeat while a batch is still being produced.
|
|
82
|
+
const HEARTBEAT_TICK_MS = (STALE_JOB_TIMEOUT_SECONDS * 1000) / 4
|
|
83
|
+
|
|
84
|
+
// Runs `tick` on an interval only while the source iterator is pending, so heartbeats
|
|
85
|
+
// stop the moment the producer dies and genuinely stale jobs still get swept. The outer
|
|
86
|
+
// finally closes the adapter generator on early exits (cancellation, ownership conflict).
|
|
87
|
+
async function* withHeartbeat<T>(source: AsyncIterable<T>, tick: () => void, intervalMs: number): AsyncGenerator<T, void, undefined> {
|
|
88
|
+
const iterator = source[Symbol.asyncIterator]()
|
|
89
|
+
try {
|
|
90
|
+
while (true) {
|
|
91
|
+
const timer = setInterval(tick, intervalMs)
|
|
92
|
+
let result: IteratorResult<T>
|
|
93
|
+
try {
|
|
94
|
+
result = await iterator.next()
|
|
95
|
+
} finally {
|
|
96
|
+
clearInterval(timer)
|
|
97
|
+
}
|
|
98
|
+
if (result.done) return
|
|
99
|
+
yield result.value
|
|
100
|
+
}
|
|
101
|
+
} finally {
|
|
102
|
+
await iterator.return?.()
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
79
106
|
export function createSyncEngine(deps: EngineDeps) {
|
|
80
107
|
const { syncRunService, integrationCredentialsService, integrationLogService, integrationStateService, progressService } = deps
|
|
81
108
|
|
|
@@ -103,6 +130,47 @@ export function createSyncEngine(deps: EngineDeps) {
|
|
|
103
130
|
)
|
|
104
131
|
}
|
|
105
132
|
|
|
133
|
+
// On redelivery the progress counter must resume where the last delivery left off the
|
|
134
|
+
// same way committedBatches does — updateProgress writes absolute counts, so starting at
|
|
135
|
+
// zero would regress the visible count. The progress job's own processedCount is the only
|
|
136
|
+
// persisted value already in the engine's unit: `batch.processedCount ?? items.length`,
|
|
137
|
+
// i.e. source records. The run's created/updated/skipped/failed counters count emitted
|
|
138
|
+
// items, which adapters may explode several-per-source-record (Akeneo yields a product
|
|
139
|
+
// plus its variants), so seeding from them would overshoot the total and pin the bar.
|
|
140
|
+
async function seedProcessedCount(progressJobId: string | null | undefined, scope: SyncScope): Promise<number> {
|
|
141
|
+
if (!progressJobId) return 0
|
|
142
|
+
const job = await progressService.getJob(progressJobId, {
|
|
143
|
+
tenantId: scope.tenantId,
|
|
144
|
+
organizationId: scope.organizationId,
|
|
145
|
+
userId: scope.userId,
|
|
146
|
+
})
|
|
147
|
+
return job?.processedCount ?? 0
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function makeHeartbeatTick(progressJobId: string | null | undefined, scope: SyncScope): () => void {
|
|
151
|
+
const touchJobHeartbeat = progressService.touchJobHeartbeat?.bind(progressService)
|
|
152
|
+
if (!progressJobId || !touchJobHeartbeat) return () => {}
|
|
153
|
+
let inFlight = false
|
|
154
|
+
return () => {
|
|
155
|
+
if (inFlight) return
|
|
156
|
+
inFlight = true
|
|
157
|
+
touchJobHeartbeat(progressJobId, {
|
|
158
|
+
tenantId: scope.tenantId,
|
|
159
|
+
organizationId: scope.organizationId,
|
|
160
|
+
userId: scope.userId,
|
|
161
|
+
})
|
|
162
|
+
.catch((error) => {
|
|
163
|
+
logger.warn('Progress heartbeat failed', {
|
|
164
|
+
progressJobId,
|
|
165
|
+
error: error instanceof Error ? error.message : String(error),
|
|
166
|
+
})
|
|
167
|
+
})
|
|
168
|
+
.finally(() => {
|
|
169
|
+
inFlight = false
|
|
170
|
+
})
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
106
174
|
async function refreshCoverageSnapshots(entityTypes: string[] | undefined, scope: SyncScope): Promise<void> {
|
|
107
175
|
if (!entityTypes || entityTypes.length === 0) return
|
|
108
176
|
|
|
@@ -471,20 +539,24 @@ export function createSyncEngine(deps: EngineDeps) {
|
|
|
471
539
|
}
|
|
472
540
|
|
|
473
541
|
const mapping = await resolveMapping(adapter, run.entityType, scope)
|
|
474
|
-
let processedCount =
|
|
542
|
+
let processedCount = await seedProcessedCount(run.progressJobId, scope)
|
|
475
543
|
let totalCount: number | null = null
|
|
476
544
|
let committedBatches = activeRun.batchesCompleted ?? 0
|
|
477
545
|
|
|
478
546
|
try {
|
|
479
|
-
for await (const batch of
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
547
|
+
for await (const batch of withHeartbeat(
|
|
548
|
+
adapter.streamImport({
|
|
549
|
+
entityType: run.entityType,
|
|
550
|
+
cursor: run.cursor ?? undefined,
|
|
551
|
+
batchSize,
|
|
552
|
+
credentials,
|
|
553
|
+
mapping,
|
|
554
|
+
scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },
|
|
555
|
+
runId: run.id,
|
|
556
|
+
}),
|
|
557
|
+
makeHeartbeatTick(run.progressJobId, scope),
|
|
558
|
+
HEARTBEAT_TICK_MS,
|
|
559
|
+
)) {
|
|
488
560
|
if (run.progressJobId && await progressService.isCancellationRequested(run.progressJobId, scope.tenantId, scope.organizationId)) {
|
|
489
561
|
await finalizeRun(run.id, 'cancelled', scope, undefined, operationalTelemetry)
|
|
490
562
|
return
|
|
@@ -631,19 +703,23 @@ export function createSyncEngine(deps: EngineDeps) {
|
|
|
631
703
|
}
|
|
632
704
|
|
|
633
705
|
const mapping = await resolveMapping(adapter, run.entityType, scope)
|
|
634
|
-
let processedCount =
|
|
706
|
+
let processedCount = await seedProcessedCount(run.progressJobId, scope)
|
|
635
707
|
let committedBatches = activeRun.batchesCompleted ?? 0
|
|
636
708
|
|
|
637
709
|
try {
|
|
638
|
-
for await (const batch of
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
710
|
+
for await (const batch of withHeartbeat(
|
|
711
|
+
adapter.streamExport({
|
|
712
|
+
entityType: run.entityType,
|
|
713
|
+
cursor: run.cursor ?? undefined,
|
|
714
|
+
batchSize,
|
|
715
|
+
credentials,
|
|
716
|
+
mapping,
|
|
717
|
+
scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },
|
|
718
|
+
runId: run.id,
|
|
719
|
+
}),
|
|
720
|
+
makeHeartbeatTick(run.progressJobId, scope),
|
|
721
|
+
HEARTBEAT_TICK_MS,
|
|
722
|
+
)) {
|
|
647
723
|
if (run.progressJobId && await progressService.isCancellationRequested(run.progressJobId, scope.tenantId, scope.organizationId)) {
|
|
648
724
|
await finalizeRun(run.id, 'cancelled', scope, undefined, operationalTelemetry)
|
|
649
725
|
return
|
|
@@ -93,7 +93,8 @@ Use stable, grep-friendly ids:
|
|
|
93
93
|
`progressService` is safe to run across many app/worker instances. Do not reintroduce read-modify-write transitions:
|
|
94
94
|
|
|
95
95
|
- **Every status transition is a status-guarded `nativeUpdate` (CAS).** An update that matches zero rows lost the race — it MUST NOT emit events or overwrite the row. Allowed transitions: start from `pending|failed` (an already-running start is an idempotent no-op; `failed` allows queue retries and recovery from a wrong stale sweep), complete from `pending|running|failed`, fail from `pending|running`, cancel from `pending|running|failed`.
|
|
96
|
-
- **Progress writes are guarded on `status IN ('pending','running')`** — once another process finishes/cancels a job, buffered updaters stop writing to it.
|
|
96
|
+
- **Progress writes are guarded on `status IN ('pending','running')`** — once another process finishes/cancels a job, buffered updaters stop writing to it. Exception: when a write path finds the job `failed`, it attempts exactly one revival through the start CAS (a live producer writing progress proves a stale sweep false-positived) before giving up. That revive CAS is narrowed to rows whose `errorMessage` starts with `STALE_SWEEP_ERROR_PREFIX`, so a job that failed for a real reason keeps its message and stack instead of being resurrected by a racing buffered writer; `completed`/`cancelled` are never revived, and genuinely dead producers never write again, so real sweeps stick. Only `startJob` uses the unrestricted CAS, because a queue retry restarts the whole unit of work. A revive never resets `startedAt` — progress counts are absolute across deliveries, so the elapsed window `calculateEta` divides them by must be too.
|
|
97
|
+
- **Long single units of work MUST call `touchJobHeartbeat`** — when one batch/step can outlast `STALE_JOB_TIMEOUT_SECONDS` (60s), run a keepalive that calls the optional `touchJobHeartbeat(jobId, ctx)` while awaiting it (see `withHeartbeat` in `data_sync/lib/sync-engine.ts`). It writes only `heartbeat_at`/`updated_at` on a **forked** EntityManager, so it is safe while the producer's own EM is mid-transaction, and it self-heals a falsely-swept job via the start CAS. Always optional-chain it (`progressService.touchJobHeartbeat?.(...)`) — third-party implementations may not provide it. The "no timers for durable work" rule below bans replacing queue delivery with timers, not an in-process keepalive around a pending await.
|
|
97
98
|
- **`incrementProgress` deltas persist as atomic SQL increments** (`processed_count + n`), and the service reloads the database winner before returning or emitting, so concurrent writers never lose or report stale counts.
|
|
98
99
|
- **Update-path reads use `disableIdentityMap: true`** — `isCancellationRequested` and lifecycle reads must always see fresh cross-process state, never a stale managed entity.
|
|
99
100
|
- **The stale sweep (`markStaleJobsFailed`) re-checks staleness per row inside the CAS**, so concurrent sweepers emit exactly one `JOB_FAILED` per job, and it also fails `pending` jobs that never started within `STALE_PENDING_TIMEOUT_SECONDS` (a late queue delivery recovers them via `startJob`'s `failed → running` transition).
|
|
@@ -21,12 +21,21 @@ export interface ProgressService {
|
|
|
21
21
|
getRecentlyCompletedJobs(ctx: ProgressServiceContext, sinceSeconds?: number): Promise<ProgressJob[]>
|
|
22
22
|
getJob(jobId: string, ctx: ProgressServiceContext): Promise<ProgressJob | null>
|
|
23
23
|
markStaleJobsFailed(tenantId: string, timeoutSeconds?: number, organizationId?: string | null): Promise<number>
|
|
24
|
+
// Optional so third-party ProgressService implementations keep compiling; callers must
|
|
25
|
+
// optional-chain. Runs on a forked EntityManager, so it is safe to call while the shared
|
|
26
|
+
// request/worker EM is mid-transaction (e.g. from a keepalive timer around adapter I/O).
|
|
27
|
+
touchJobHeartbeat?(jobId: string, ctx: ProgressServiceContext): Promise<void>
|
|
24
28
|
}
|
|
25
29
|
|
|
26
30
|
export const HEARTBEAT_INTERVAL_MS = 5000
|
|
27
31
|
export const STALE_JOB_TIMEOUT_SECONDS = 60
|
|
28
32
|
export const STALE_PENDING_TIMEOUT_SECONDS = 900
|
|
29
33
|
|
|
34
|
+
// Every `errorMessage` the stale sweep writes starts with this, so recovery paths can tell
|
|
35
|
+
// a job the sweep gave up on from one that failed for a real reason and must keep its
|
|
36
|
+
// diagnostics. Declared here so the sweep and the revive filter cannot drift apart.
|
|
37
|
+
export const STALE_SWEEP_ERROR_PREFIX = 'Job stale:'
|
|
38
|
+
|
|
30
39
|
export function calculateEta(
|
|
31
40
|
processedCount: number,
|
|
32
41
|
totalCount: number,
|
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
HEARTBEAT_INTERVAL_MS,
|
|
10
10
|
STALE_JOB_TIMEOUT_SECONDS,
|
|
11
11
|
STALE_PENDING_TIMEOUT_SECONDS,
|
|
12
|
+
STALE_SWEEP_ERROR_PREFIX,
|
|
12
13
|
} from './progressService'
|
|
13
14
|
import { PROGRESS_EVENTS } from './events'
|
|
14
15
|
import { findOneWithDecryption } from '@open-mercato/shared/lib/encryption/find'
|
|
@@ -131,6 +132,58 @@ export function createProgressService(em: EntityManager, eventBus: { emit: (even
|
|
|
131
132
|
return {}
|
|
132
133
|
}
|
|
133
134
|
|
|
135
|
+
// The start transition (START_FROM_STATUSES) doubles as the recovery path for jobs a
|
|
136
|
+
// stale sweep flipped to `failed` while their producer was alive but slow. Callers pass
|
|
137
|
+
// the EM to run against so the forked heartbeat path can reuse the same CAS.
|
|
138
|
+
//
|
|
139
|
+
// `staleSweptOnly` is for the revive paths: a live write proves the producer is alive,
|
|
140
|
+
// but it does NOT prove the recorded failure was the sweep's. Narrowing the CAS to rows
|
|
141
|
+
// the sweep tagged keeps a genuine failure's message and stack intact instead of
|
|
142
|
+
// resurrecting the job and erasing why it died. `startJob` stays unrestricted so an
|
|
143
|
+
// at-least-once queue can still retry a genuinely failed job from the top.
|
|
144
|
+
async function startJobViaCas(
|
|
145
|
+
targetEm: EntityManager,
|
|
146
|
+
job: ProgressJob,
|
|
147
|
+
ctx: ProgressServiceContext,
|
|
148
|
+
options: { staleSweptOnly?: boolean } = {},
|
|
149
|
+
): Promise<ProgressJob | null> {
|
|
150
|
+
const now = new Date()
|
|
151
|
+
// Progress counts are absolute and survive across deliveries, so the elapsed window
|
|
152
|
+
// calculateEta divides them by has to survive too. Resetting it on every revive makes
|
|
153
|
+
// a job with hours of work left report seconds remaining.
|
|
154
|
+
const startedAt = job.startedAt ?? now
|
|
155
|
+
const filter: Record<string, unknown> = {
|
|
156
|
+
...jobScopeFilter(job.id, ctx),
|
|
157
|
+
status: { $in: START_FROM_STATUSES as ProgressJobStatus[] },
|
|
158
|
+
}
|
|
159
|
+
if (options.staleSweptOnly) filter.errorMessage = { $like: `${STALE_SWEEP_ERROR_PREFIX}%` }
|
|
160
|
+
const affected = await targetEm.nativeUpdate(ProgressJob, filter as FilterQuery<ProgressJob>, {
|
|
161
|
+
status: 'running',
|
|
162
|
+
startedAt,
|
|
163
|
+
heartbeatAt: now,
|
|
164
|
+
finishedAt: null,
|
|
165
|
+
errorMessage: null,
|
|
166
|
+
errorStack: null,
|
|
167
|
+
updatedAt: now,
|
|
168
|
+
})
|
|
169
|
+
if (affected === 0) return null
|
|
170
|
+
|
|
171
|
+
job.status = 'running'
|
|
172
|
+
job.startedAt = startedAt
|
|
173
|
+
job.heartbeatAt = now
|
|
174
|
+
job.finishedAt = null
|
|
175
|
+
job.errorMessage = null
|
|
176
|
+
job.errorStack = null
|
|
177
|
+
|
|
178
|
+
await eventBus.emit(PROGRESS_EVENTS.JOB_STARTED, {
|
|
179
|
+
...buildJobPayload(job),
|
|
180
|
+
tenantId: ctx.tenantId,
|
|
181
|
+
organizationId: job.organizationId ?? null,
|
|
182
|
+
})
|
|
183
|
+
|
|
184
|
+
return job
|
|
185
|
+
}
|
|
186
|
+
|
|
134
187
|
async function persistAndMaybeBroadcast(entry: JobUpdateThrottleEntry, ctx: ProgressServiceContext): Promise<ProgressJob> {
|
|
135
188
|
const job = entry.job
|
|
136
189
|
const now = Date.now()
|
|
@@ -153,16 +206,32 @@ export function createProgressService(em: EntityManager, eventBus: { emit: (even
|
|
|
153
206
|
updatedAt: new Date(),
|
|
154
207
|
...buildBufferedCountData(entry),
|
|
155
208
|
}
|
|
156
|
-
const
|
|
209
|
+
const writableFilter = {
|
|
157
210
|
...jobScopeFilter(job.id, ctx),
|
|
158
211
|
status: { $in: PROGRESS_WRITABLE_STATUSES as ProgressJobStatus[] },
|
|
159
|
-
} as FilterQuery<ProgressJob
|
|
212
|
+
} as FilterQuery<ProgressJob>
|
|
213
|
+
let affected = await em.nativeUpdate(ProgressJob, writableFilter, data)
|
|
160
214
|
|
|
161
215
|
if (affected === 0) {
|
|
162
|
-
//
|
|
163
|
-
|
|
216
|
+
// A stale sweep may have flipped a live job to `failed` between writes. Revive it
|
|
217
|
+
// through the start CAS and retry once so the buffered delta isn't silently dropped.
|
|
164
218
|
const fresh = await loadFreshJob(job.id, ctx)
|
|
165
|
-
|
|
219
|
+
let revived = false
|
|
220
|
+
if (fresh && fresh.status === 'failed' && (await startJobViaCas(em, fresh, ctx, { staleSweptOnly: true }))) {
|
|
221
|
+
revived = true
|
|
222
|
+
job.status = 'running'
|
|
223
|
+
job.startedAt = fresh.startedAt
|
|
224
|
+
job.finishedAt = null
|
|
225
|
+
job.errorMessage = null
|
|
226
|
+
job.errorStack = null
|
|
227
|
+
affected = await em.nativeUpdate(ProgressJob, writableFilter, data)
|
|
228
|
+
}
|
|
229
|
+
if (affected === 0) {
|
|
230
|
+
// The job reached a genuinely terminal state in another process; stop writing to it.
|
|
231
|
+
forgetJobThrottle(job.id)
|
|
232
|
+
const latest = revived ? await loadFreshJob(job.id, ctx) : fresh
|
|
233
|
+
return latest ?? job
|
|
234
|
+
}
|
|
166
235
|
}
|
|
167
236
|
|
|
168
237
|
const persistedJob = usesAtomicIncrement ? (await loadFreshJob(job.id, ctx)) ?? job : job
|
|
@@ -219,47 +288,48 @@ export function createProgressService(em: EntityManager, eventBus: { emit: (even
|
|
|
219
288
|
return job
|
|
220
289
|
}
|
|
221
290
|
|
|
291
|
+
const started = await startJobViaCas(em, job, ctx)
|
|
292
|
+
if (started) return started
|
|
293
|
+
|
|
294
|
+
const fresh = await loadFreshJob(jobId, ctx)
|
|
295
|
+
return fresh ?? job
|
|
296
|
+
},
|
|
297
|
+
|
|
298
|
+
async touchJobHeartbeat(jobId, ctx) {
|
|
299
|
+
// Forked EM: keepalive timers call this while the shared EM may be mid-transaction
|
|
300
|
+
// inside a producer's own writes, so the heartbeat must not join that unit of work.
|
|
301
|
+
const fork = em.fork()
|
|
222
302
|
const now = new Date()
|
|
223
|
-
const affected = await
|
|
303
|
+
const affected = await fork.nativeUpdate(ProgressJob, {
|
|
224
304
|
...jobScopeFilter(jobId, ctx),
|
|
225
|
-
status: { $in:
|
|
305
|
+
status: { $in: PROGRESS_WRITABLE_STATUSES as ProgressJobStatus[] },
|
|
226
306
|
} as FilterQuery<ProgressJob>, {
|
|
227
|
-
status: 'running',
|
|
228
|
-
startedAt: now,
|
|
229
307
|
heartbeatAt: now,
|
|
230
|
-
finishedAt: null,
|
|
231
|
-
errorMessage: null,
|
|
232
|
-
errorStack: null,
|
|
233
308
|
updatedAt: now,
|
|
234
309
|
})
|
|
310
|
+
if (affected > 0) return
|
|
235
311
|
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
job.status = 'running'
|
|
242
|
-
job.startedAt = now
|
|
243
|
-
job.heartbeatAt = now
|
|
244
|
-
job.finishedAt = null
|
|
245
|
-
job.errorMessage = null
|
|
246
|
-
job.errorStack = null
|
|
247
|
-
|
|
248
|
-
await eventBus.emit(PROGRESS_EVENTS.JOB_STARTED, {
|
|
249
|
-
...buildJobPayload(job),
|
|
250
|
-
tenantId: ctx.tenantId,
|
|
251
|
-
organizationId: job.organizationId ?? null,
|
|
252
|
-
})
|
|
253
|
-
|
|
254
|
-
return job
|
|
312
|
+
const job = await fork.findOne(ProgressJob, jobScopeFilter(jobId, ctx), { disableIdentityMap: true })
|
|
313
|
+
if (!job || job.status !== 'failed') return
|
|
314
|
+
// A live producer heartbeating a `failed` job means a stale sweep false-positived;
|
|
315
|
+
// revive through the start CAS so the card recovers without waiting for a batch boundary.
|
|
316
|
+
await startJobViaCas(fork, job, ctx, { staleSweptOnly: true })
|
|
255
317
|
},
|
|
256
318
|
|
|
257
319
|
async updateProgress(jobId, input, ctx) {
|
|
258
320
|
const entry = await ensureThrottleEntry(jobId, ctx)
|
|
259
321
|
const job = entry.job
|
|
260
322
|
if (TERMINAL_STATUSES.includes(job.status)) {
|
|
261
|
-
|
|
262
|
-
|
|
323
|
+
// `failed` may be a false-positive stale sweep on a slow-but-alive producer; the
|
|
324
|
+
// fact that this write is happening proves the producer is alive, so revive and
|
|
325
|
+
// apply the update. `completed`/`cancelled` stay terminal, and a failure the sweep
|
|
326
|
+
// did not write keeps its diagnostics (see startJobViaCas).
|
|
327
|
+
const revived = job.status === 'failed' && (await startJobViaCas(em, job, ctx, { staleSweptOnly: true }))
|
|
328
|
+
if (!revived) {
|
|
329
|
+
forgetJobThrottle(jobId)
|
|
330
|
+
return job
|
|
331
|
+
}
|
|
332
|
+
entry.lastPersistedAt = 0
|
|
263
333
|
}
|
|
264
334
|
|
|
265
335
|
if (input.processedCount !== undefined) {
|
|
@@ -295,8 +365,12 @@ export function createProgressService(em: EntityManager, eventBus: { emit: (even
|
|
|
295
365
|
const entry = await ensureThrottleEntry(jobId, ctx)
|
|
296
366
|
const job = entry.job
|
|
297
367
|
if (TERMINAL_STATUSES.includes(job.status)) {
|
|
298
|
-
|
|
299
|
-
|
|
368
|
+
const revived = job.status === 'failed' && (await startJobViaCas(em, job, ctx, { staleSweptOnly: true }))
|
|
369
|
+
if (!revived) {
|
|
370
|
+
forgetJobThrottle(jobId)
|
|
371
|
+
return job
|
|
372
|
+
}
|
|
373
|
+
entry.lastPersistedAt = 0
|
|
300
374
|
}
|
|
301
375
|
|
|
302
376
|
job.processedCount += delta
|
|
@@ -604,7 +678,7 @@ export function createProgressService(em: EntityManager, eventBus: { emit: (even
|
|
|
604
678
|
const staleRunning = await em.find(ProgressJob, { ...scope, ...staleFilter }, { disableIdentityMap: true })
|
|
605
679
|
|
|
606
680
|
for (const job of staleRunning) {
|
|
607
|
-
const errorMessage =
|
|
681
|
+
const errorMessage = `${STALE_SWEEP_ERROR_PREFIX} no heartbeat for ${timeoutSeconds} seconds`
|
|
608
682
|
// Per-row CAS (re-checking the staleness condition): exactly one concurrent
|
|
609
683
|
// sweeper wins, and a heartbeat that landed after the SELECT aborts the failure.
|
|
610
684
|
const affected = await em.nativeUpdate(ProgressJob, {
|
|
@@ -644,7 +718,7 @@ export function createProgressService(em: EntityManager, eventBus: { emit: (even
|
|
|
644
718
|
const stalePending = await em.find(ProgressJob, { ...scope, ...stalePendingFilter }, { disableIdentityMap: true })
|
|
645
719
|
|
|
646
720
|
for (const job of stalePending) {
|
|
647
|
-
const errorMessage =
|
|
721
|
+
const errorMessage = `${STALE_SWEEP_ERROR_PREFIX} never started within ${STALE_PENDING_TIMEOUT_SECONDS} seconds`
|
|
648
722
|
const affected = await em.nativeUpdate(ProgressJob, {
|
|
649
723
|
id: job.id,
|
|
650
724
|
tenantId: job.tenantId,
|