@open-mercato/core 0.4.6-develop-a88276bc52 → 0.4.6-develop-806a2ed6b9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +10 -0
- package/dist/generated/entities/integration_credentials/index.js +19 -0
- package/dist/generated/entities/integration_credentials/index.js.map +7 -0
- package/dist/generated/entities/integration_log/index.js +27 -0
- package/dist/generated/entities/integration_log/index.js.map +7 -0
- package/dist/generated/entities/integration_state/index.js +27 -0
- package/dist/generated/entities/integration_state/index.js.map +7 -0
- package/dist/generated/entities/sync_cursor/index.js +19 -0
- package/dist/generated/entities/sync_cursor/index.js.map +7 -0
- package/dist/generated/entities/sync_external_id_mapping/index.js +27 -0
- package/dist/generated/entities/sync_external_id_mapping/index.js.map +7 -0
- package/dist/generated/entities/sync_mapping/index.js +19 -0
- package/dist/generated/entities/sync_mapping/index.js.map +7 -0
- package/dist/generated/entities/sync_run/index.js +45 -0
- package/dist/generated/entities/sync_run/index.js.map +7 -0
- package/dist/generated/entities/sync_schedule/index.js +35 -0
- package/dist/generated/entities/sync_schedule/index.js.map +7 -0
- package/dist/generated/entities.ids.generated.js +14 -0
- package/dist/generated/entities.ids.generated.js.map +2 -2
- package/dist/generated/entity-fields-registry.js +16 -0
- package/dist/generated/entity-fields-registry.js.map +2 -2
- package/dist/modules/data_sync/acl.js +11 -0
- package/dist/modules/data_sync/acl.js.map +7 -0
- package/dist/modules/data_sync/api/mappings/[id]/route.js +137 -0
- package/dist/modules/data_sync/api/mappings/[id]/route.js.map +7 -0
- package/dist/modules/data_sync/api/mappings/route.js +132 -0
- package/dist/modules/data_sync/api/mappings/route.js.map +7 -0
- package/dist/modules/data_sync/api/run.js +87 -0
- package/dist/modules/data_sync/api/run.js.map +7 -0
- package/dist/modules/data_sync/api/runs/[id]/cancel.js +49 -0
- package/dist/modules/data_sync/api/runs/[id]/cancel.js.map +7 -0
- package/dist/modules/data_sync/api/runs/[id]/retry.js +93 -0
- package/dist/modules/data_sync/api/runs/[id]/retry.js.map +7 -0
- package/dist/modules/data_sync/api/runs/[id]/route.js +69 -0
- package/dist/modules/data_sync/api/runs/[id]/route.js.map +7 -0
- package/dist/modules/data_sync/api/runs.js +66 -0
- package/dist/modules/data_sync/api/runs.js.map +7 -0
- package/dist/modules/data_sync/api/validate.js +66 -0
- package/dist/modules/data_sync/api/validate.js.map +7 -0
- package/dist/modules/data_sync/backend/data-sync/page.js +216 -0
- package/dist/modules/data_sync/backend/data-sync/page.js.map +7 -0
- package/dist/modules/data_sync/backend/data-sync/page.meta.js +25 -0
- package/dist/modules/data_sync/backend/data-sync/page.meta.js.map +7 -0
- package/dist/modules/data_sync/backend/data-sync/runs/[id]/page.js +178 -0
- package/dist/modules/data_sync/backend/data-sync/runs/[id]/page.js.map +7 -0
- package/dist/modules/data_sync/backend/data-sync/runs/[id]/page.meta.js +14 -0
- package/dist/modules/data_sync/backend/data-sync/runs/[id]/page.meta.js.map +7 -0
- package/dist/modules/data_sync/data/entities.js +228 -0
- package/dist/modules/data_sync/data/entities.js.map +7 -0
- package/dist/modules/data_sync/data/validators.js +32 -0
- package/dist/modules/data_sync/data/validators.js.map +7 -0
- package/dist/modules/data_sync/di.js +26 -0
- package/dist/modules/data_sync/di.js.map +7 -0
- package/dist/modules/data_sync/events.js +16 -0
- package/dist/modules/data_sync/events.js.map +7 -0
- package/dist/modules/data_sync/index.js +9 -0
- package/dist/modules/data_sync/index.js.map +7 -0
- package/dist/modules/data_sync/lib/adapter-registry.js +16 -0
- package/dist/modules/data_sync/lib/adapter-registry.js.map +7 -0
- package/dist/modules/data_sync/lib/adapter.js +1 -0
- package/dist/modules/data_sync/lib/adapter.js.map +7 -0
- package/dist/modules/data_sync/lib/id-mapping.js +79 -0
- package/dist/modules/data_sync/lib/id-mapping.js.map +7 -0
- package/dist/modules/data_sync/lib/queue.js +17 -0
- package/dist/modules/data_sync/lib/queue.js.map +7 -0
- package/dist/modules/data_sync/lib/sync-engine.js +309 -0
- package/dist/modules/data_sync/lib/sync-engine.js.map +7 -0
- package/dist/modules/data_sync/lib/sync-run-service.js +148 -0
- package/dist/modules/data_sync/lib/sync-run-service.js.map +7 -0
- package/dist/modules/data_sync/migrations/Migration20260304113737.js +17 -0
- package/dist/modules/data_sync/migrations/Migration20260304113737.js.map +7 -0
- package/dist/modules/data_sync/setup.js +13 -0
- package/dist/modules/data_sync/setup.js.map +7 -0
- package/dist/modules/data_sync/workers/sync-export.js +14 -0
- package/dist/modules/data_sync/workers/sync-export.js.map +7 -0
- package/dist/modules/data_sync/workers/sync-import.js +14 -0
- package/dist/modules/data_sync/workers/sync-import.js.map +7 -0
- package/dist/modules/data_sync/workers/sync-scheduled.js +63 -0
- package/dist/modules/data_sync/workers/sync-scheduled.js.map +7 -0
- package/dist/modules/entities/lib/encryptionDefaults.js +4 -0
- package/dist/modules/entities/lib/encryptionDefaults.js.map +2 -2
- package/dist/modules/integrations/acl.js +4 -1
- package/dist/modules/integrations/acl.js.map +2 -2
- package/dist/modules/integrations/api/[id]/credentials/route.js +127 -0
- package/dist/modules/integrations/api/[id]/credentials/route.js.map +7 -0
- package/dist/modules/integrations/api/[id]/health/route.js +46 -0
- package/dist/modules/integrations/api/[id]/health/route.js.map +7 -0
- package/dist/modules/integrations/api/[id]/route.js +65 -0
- package/dist/modules/integrations/api/[id]/route.js.map +7 -0
- package/dist/modules/integrations/api/[id]/state/route.js +109 -0
- package/dist/modules/integrations/api/[id]/state/route.js.map +7 -0
- package/dist/modules/integrations/api/[id]/version/route.js +117 -0
- package/dist/modules/integrations/api/[id]/version/route.js.map +7 -0
- package/dist/modules/integrations/api/guards.js +31 -0
- package/dist/modules/integrations/api/guards.js.map +7 -0
- package/dist/modules/integrations/api/logs/route.js +60 -0
- package/dist/modules/integrations/api/logs/route.js.map +7 -0
- package/dist/modules/integrations/api/openapi.js +25 -0
- package/dist/modules/integrations/api/openapi.js.map +7 -0
- package/dist/modules/integrations/api/route.js +68 -0
- package/dist/modules/integrations/api/route.js.map +7 -0
- package/dist/modules/integrations/backend/integrations/[id]/page.js +313 -0
- package/dist/modules/integrations/backend/integrations/[id]/page.js.map +7 -0
- package/dist/modules/integrations/backend/integrations/[id]/page.meta.js +15 -0
- package/dist/modules/integrations/backend/integrations/[id]/page.meta.js.map +7 -0
- package/dist/modules/integrations/backend/integrations/bundle/[id]/page.js +189 -0
- package/dist/modules/integrations/backend/integrations/bundle/[id]/page.js.map +7 -0
- package/dist/modules/integrations/backend/integrations/bundle/[id]/page.meta.js +15 -0
- package/dist/modules/integrations/backend/integrations/bundle/[id]/page.meta.js.map +7 -0
- package/dist/modules/integrations/backend/integrations/page.js +212 -0
- package/dist/modules/integrations/backend/integrations/page.js.map +7 -0
- package/dist/modules/integrations/backend/integrations/page.meta.js +22 -0
- package/dist/modules/integrations/backend/integrations/page.meta.js.map +7 -0
- package/dist/modules/integrations/data/enrichers.js +27 -12
- package/dist/modules/integrations/data/enrichers.js.map +2 -2
- package/dist/modules/integrations/data/entities.js +136 -1
- package/dist/modules/integrations/data/entities.js.map +2 -2
- package/dist/modules/integrations/data/validators.js +36 -0
- package/dist/modules/integrations/data/validators.js.map +7 -0
- package/dist/modules/integrations/di.js +24 -0
- package/dist/modules/integrations/di.js.map +7 -0
- package/dist/modules/integrations/events.js +19 -0
- package/dist/modules/integrations/events.js.map +7 -0
- package/dist/modules/integrations/lib/credentials-service.js +159 -0
- package/dist/modules/integrations/lib/credentials-service.js.map +7 -0
- package/dist/modules/integrations/lib/health-service.js +37 -0
- package/dist/modules/integrations/lib/health-service.js.map +7 -0
- package/dist/modules/integrations/lib/log-service.js +66 -0
- package/dist/modules/integrations/lib/log-service.js.map +7 -0
- package/dist/modules/integrations/lib/registry-service.js +33 -0
- package/dist/modules/integrations/lib/registry-service.js.map +7 -0
- package/dist/modules/integrations/lib/state-service.js +55 -0
- package/dist/modules/integrations/lib/state-service.js.map +7 -0
- package/dist/modules/integrations/lib/types.js +1 -0
- package/dist/modules/integrations/lib/types.js.map +7 -0
- package/dist/modules/integrations/migrations/Migration20260304113737.js +19 -0
- package/dist/modules/integrations/migrations/Migration20260304113737.js.map +7 -0
- package/dist/modules/integrations/setup.js +2 -2
- package/dist/modules/integrations/setup.js.map +2 -2
- package/dist/modules/integrations/widgets/injection-table.js.map +1 -1
- package/dist/modules/integrations/workers/log-pruner.js +18 -0
- package/dist/modules/integrations/workers/log-pruner.js.map +7 -0
- package/generated/entities/integration_credentials/index.ts +8 -0
- package/generated/entities/integration_log/index.ts +12 -0
- package/generated/entities/integration_state/index.ts +12 -0
- package/generated/entities/sync_cursor/index.ts +8 -0
- package/generated/entities/sync_external_id_mapping/index.ts +12 -0
- package/generated/entities/sync_mapping/index.ts +8 -0
- package/generated/entities/sync_run/index.ts +21 -0
- package/generated/entities/sync_schedule/index.ts +16 -0
- package/generated/entities.ids.generated.ts +14 -0
- package/generated/entity-fields-registry.ts +16 -0
- package/package.json +2 -2
- package/src/modules/data_sync/AGENTS.md +157 -0
- package/src/modules/data_sync/acl.ts +7 -0
- package/src/modules/data_sync/api/mappings/[id]/route.ts +158 -0
- package/src/modules/data_sync/api/mappings/route.ts +144 -0
- package/src/modules/data_sync/api/run.ts +97 -0
- package/src/modules/data_sync/api/runs/[id]/cancel.ts +57 -0
- package/src/modules/data_sync/api/runs/[id]/retry.ts +108 -0
- package/src/modules/data_sync/api/runs/[id]/route.ts +81 -0
- package/src/modules/data_sync/api/runs.ts +69 -0
- package/src/modules/data_sync/api/validate.ts +73 -0
- package/src/modules/data_sync/backend/data-sync/page.meta.ts +21 -0
- package/src/modules/data_sync/backend/data-sync/page.tsx +244 -0
- package/src/modules/data_sync/backend/data-sync/runs/[id]/page.meta.ts +10 -0
- package/src/modules/data_sync/backend/data-sync/runs/[id]/page.tsx +278 -0
- package/src/modules/data_sync/data/entities.ts +180 -0
- package/src/modules/data_sync/data/validators.ts +35 -0
- package/src/modules/data_sync/di.ts +38 -0
- package/src/modules/data_sync/events.ts +12 -0
- package/src/modules/data_sync/i18n/de.json +48 -0
- package/src/modules/data_sync/i18n/en.json +48 -0
- package/src/modules/data_sync/i18n/es.json +48 -0
- package/src/modules/data_sync/i18n/pl.json +48 -0
- package/src/modules/data_sync/index.ts +5 -0
- package/src/modules/data_sync/lib/adapter-registry.ts +15 -0
- package/src/modules/data_sync/lib/adapter.ts +90 -0
- package/src/modules/data_sync/lib/id-mapping.ts +95 -0
- package/src/modules/data_sync/lib/queue.ts +19 -0
- package/src/modules/data_sync/lib/sync-engine.ts +375 -0
- package/src/modules/data_sync/lib/sync-run-service.ts +187 -0
- package/src/modules/data_sync/migrations/.snapshot-open-mercato.json +653 -0
- package/src/modules/data_sync/migrations/Migration20260304113737.ts +19 -0
- package/src/modules/data_sync/setup.ts +11 -0
- package/src/modules/data_sync/workers/sync-export.ts +27 -0
- package/src/modules/data_sync/workers/sync-import.ts +27 -0
- package/src/modules/data_sync/workers/sync-scheduled.ts +84 -0
- package/src/modules/entities/lib/encryptionDefaults.ts +4 -0
- package/src/modules/integrations/AGENTS.md +160 -0
- package/src/modules/integrations/acl.ts +3 -0
- package/src/modules/integrations/api/[id]/credentials/route.ts +142 -0
- package/src/modules/integrations/api/[id]/health/route.ts +53 -0
- package/src/modules/integrations/api/[id]/route.ts +76 -0
- package/src/modules/integrations/api/[id]/state/route.ts +121 -0
- package/src/modules/integrations/api/[id]/version/route.ts +132 -0
- package/src/modules/integrations/api/guards.ts +59 -0
- package/src/modules/integrations/api/logs/route.ts +63 -0
- package/src/modules/integrations/api/openapi.ts +22 -0
- package/src/modules/integrations/api/route.ts +73 -0
- package/src/modules/integrations/backend/integrations/[id]/page.meta.ts +11 -0
- package/src/modules/integrations/backend/integrations/[id]/page.tsx +424 -0
- package/src/modules/integrations/backend/integrations/bundle/[id]/page.meta.ts +11 -0
- package/src/modules/integrations/backend/integrations/bundle/[id]/page.tsx +249 -0
- package/src/modules/integrations/backend/integrations/page.meta.ts +18 -0
- package/src/modules/integrations/backend/integrations/page.tsx +296 -0
- package/src/modules/integrations/data/enrichers.ts +35 -18
- package/src/modules/integrations/data/entities.ts +114 -5
- package/src/modules/integrations/data/validators.ts +41 -0
- package/src/modules/integrations/di.ts +31 -0
- package/src/modules/integrations/events.ts +17 -0
- package/src/modules/integrations/i18n/de.json +70 -0
- package/src/modules/integrations/i18n/en.json +70 -0
- package/src/modules/integrations/i18n/es.json +70 -0
- package/src/modules/integrations/i18n/pl.json +70 -0
- package/src/modules/integrations/lib/credentials-service.ts +204 -0
- package/src/modules/integrations/lib/health-service.ts +59 -0
- package/src/modules/integrations/lib/log-service.ts +84 -0
- package/src/modules/integrations/lib/registry-service.ts +42 -0
- package/src/modules/integrations/lib/state-service.ts +64 -0
- package/src/modules/integrations/lib/types.ts +4 -0
- package/src/modules/integrations/migrations/.snapshot-open-mercato.json +582 -0
- package/src/modules/integrations/migrations/Migration20260304113737.ts +21 -0
- package/src/modules/integrations/setup.ts +2 -2
- package/src/modules/integrations/widgets/injection-table.ts +1 -1
- package/src/modules/integrations/workers/log-pruner.ts +30 -0
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../../src/modules/data_sync/index.ts"],
|
|
4
|
+
"sourcesContent": ["export const metadata = {\n id: 'data_sync',\n title: 'Data Sync',\n description: 'Streaming data sync hub for import/export integrations.',\n}\n"],
|
|
5
|
+
"mappings": "AAAO,MAAM,WAAW;AAAA,EACtB,IAAI;AAAA,EACJ,OAAO;AAAA,EACP,aAAa;AACf;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
const adapters = /* @__PURE__ */ new Map();
|
|
2
|
+
function registerDataSyncAdapter(adapter) {
|
|
3
|
+
adapters.set(adapter.providerKey, adapter);
|
|
4
|
+
}
|
|
5
|
+
function getDataSyncAdapter(providerKey) {
|
|
6
|
+
return adapters.get(providerKey);
|
|
7
|
+
}
|
|
8
|
+
function getAllDataSyncAdapters() {
|
|
9
|
+
return Array.from(adapters.values());
|
|
10
|
+
}
|
|
11
|
+
export {
|
|
12
|
+
getAllDataSyncAdapters,
|
|
13
|
+
getDataSyncAdapter,
|
|
14
|
+
registerDataSyncAdapter
|
|
15
|
+
};
|
|
16
|
+
//# sourceMappingURL=adapter-registry.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../../../src/modules/data_sync/lib/adapter-registry.ts"],
|
|
4
|
+
"sourcesContent": ["import type { DataSyncAdapter } from './adapter'\n\nconst adapters = new Map<string, DataSyncAdapter>()\n\nexport function registerDataSyncAdapter(adapter: DataSyncAdapter): void {\n adapters.set(adapter.providerKey, adapter)\n}\n\nexport function getDataSyncAdapter(providerKey: string): DataSyncAdapter | undefined {\n return adapters.get(providerKey)\n}\n\nexport function getAllDataSyncAdapters(): DataSyncAdapter[] {\n return Array.from(adapters.values())\n}\n"],
|
|
5
|
+
"mappings": "AAEA,MAAM,WAAW,oBAAI,IAA6B;AAE3C,SAAS,wBAAwB,SAAgC;AACtE,WAAS,IAAI,QAAQ,aAAa,OAAO;AAC3C;AAEO,SAAS,mBAAmB,aAAkD;AACnF,SAAO,SAAS,IAAI,WAAW;AACjC;AAEO,SAAS,yBAA4C;AAC1D,SAAO,MAAM,KAAK,SAAS,OAAO,CAAC;AACrC;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
//# sourceMappingURL=adapter.js.map
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { findOneWithDecryption } from "@open-mercato/shared/lib/encryption/find";
|
|
2
|
+
import { SyncExternalIdMapping } from "../../integrations/data/entities.js";
|
|
3
|
+
function createExternalIdMappingService(em) {
|
|
4
|
+
return {
|
|
5
|
+
async lookupLocalId(integrationId, entityType, externalId, scope) {
|
|
6
|
+
const row = await findOneWithDecryption(
|
|
7
|
+
em,
|
|
8
|
+
SyncExternalIdMapping,
|
|
9
|
+
{
|
|
10
|
+
integrationId,
|
|
11
|
+
internalEntityType: entityType,
|
|
12
|
+
externalId,
|
|
13
|
+
organizationId: scope.organizationId,
|
|
14
|
+
tenantId: scope.tenantId,
|
|
15
|
+
deletedAt: null
|
|
16
|
+
},
|
|
17
|
+
void 0,
|
|
18
|
+
scope
|
|
19
|
+
);
|
|
20
|
+
return row?.internalEntityId ?? null;
|
|
21
|
+
},
|
|
22
|
+
async lookupExternalId(integrationId, entityType, localId, scope) {
|
|
23
|
+
const row = await findOneWithDecryption(
|
|
24
|
+
em,
|
|
25
|
+
SyncExternalIdMapping,
|
|
26
|
+
{
|
|
27
|
+
integrationId,
|
|
28
|
+
internalEntityType: entityType,
|
|
29
|
+
internalEntityId: localId,
|
|
30
|
+
organizationId: scope.organizationId,
|
|
31
|
+
tenantId: scope.tenantId,
|
|
32
|
+
deletedAt: null
|
|
33
|
+
},
|
|
34
|
+
void 0,
|
|
35
|
+
scope
|
|
36
|
+
);
|
|
37
|
+
return row?.externalId ?? null;
|
|
38
|
+
},
|
|
39
|
+
async storeExternalIdMapping(integrationId, entityType, localId, externalId, scope) {
|
|
40
|
+
const existing = await findOneWithDecryption(
|
|
41
|
+
em,
|
|
42
|
+
SyncExternalIdMapping,
|
|
43
|
+
{
|
|
44
|
+
integrationId,
|
|
45
|
+
internalEntityType: entityType,
|
|
46
|
+
internalEntityId: localId,
|
|
47
|
+
organizationId: scope.organizationId,
|
|
48
|
+
tenantId: scope.tenantId,
|
|
49
|
+
deletedAt: null
|
|
50
|
+
},
|
|
51
|
+
void 0,
|
|
52
|
+
scope
|
|
53
|
+
);
|
|
54
|
+
if (existing) {
|
|
55
|
+
existing.externalId = externalId;
|
|
56
|
+
existing.syncStatus = "synced";
|
|
57
|
+
existing.lastSyncedAt = /* @__PURE__ */ new Date();
|
|
58
|
+
await em.flush();
|
|
59
|
+
return existing;
|
|
60
|
+
}
|
|
61
|
+
const created = em.create(SyncExternalIdMapping, {
|
|
62
|
+
integrationId,
|
|
63
|
+
internalEntityType: entityType,
|
|
64
|
+
internalEntityId: localId,
|
|
65
|
+
externalId,
|
|
66
|
+
syncStatus: "synced",
|
|
67
|
+
lastSyncedAt: /* @__PURE__ */ new Date(),
|
|
68
|
+
organizationId: scope.organizationId,
|
|
69
|
+
tenantId: scope.tenantId
|
|
70
|
+
});
|
|
71
|
+
await em.persistAndFlush(created);
|
|
72
|
+
return created;
|
|
73
|
+
}
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
export {
|
|
77
|
+
createExternalIdMappingService
|
|
78
|
+
};
|
|
79
|
+
//# sourceMappingURL=id-mapping.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../../../src/modules/data_sync/lib/id-mapping.ts"],
|
|
4
|
+
"sourcesContent": ["import type { EntityManager } from '@mikro-orm/postgresql'\nimport { findOneWithDecryption } from '@open-mercato/shared/lib/encryption/find'\nimport { SyncExternalIdMapping } from '../../integrations/data/entities'\n\ntype MappingScope = {\n organizationId: string\n tenantId: string\n}\n\nexport function createExternalIdMappingService(em: EntityManager) {\n return {\n async lookupLocalId(integrationId: string, entityType: string, externalId: string, scope: MappingScope): Promise<string | null> {\n const row = await findOneWithDecryption(\n em,\n SyncExternalIdMapping,\n {\n integrationId,\n internalEntityType: entityType,\n externalId,\n organizationId: scope.organizationId,\n tenantId: scope.tenantId,\n deletedAt: null,\n },\n undefined,\n scope,\n )\n return row?.internalEntityId ?? null\n },\n\n async lookupExternalId(integrationId: string, entityType: string, localId: string, scope: MappingScope): Promise<string | null> {\n const row = await findOneWithDecryption(\n em,\n SyncExternalIdMapping,\n {\n integrationId,\n internalEntityType: entityType,\n internalEntityId: localId,\n organizationId: scope.organizationId,\n tenantId: scope.tenantId,\n deletedAt: null,\n },\n undefined,\n scope,\n )\n return row?.externalId ?? null\n },\n\n async storeExternalIdMapping(\n integrationId: string,\n entityType: string,\n localId: string,\n externalId: string,\n scope: MappingScope,\n ): Promise<SyncExternalIdMapping> {\n const existing = await findOneWithDecryption(\n em,\n SyncExternalIdMapping,\n {\n integrationId,\n internalEntityType: entityType,\n internalEntityId: localId,\n organizationId: scope.organizationId,\n tenantId: scope.tenantId,\n deletedAt: null,\n },\n undefined,\n scope,\n )\n\n if (existing) {\n existing.externalId = externalId\n existing.syncStatus = 'synced'\n existing.lastSyncedAt = new Date()\n await em.flush()\n return existing\n }\n\n const created = em.create(SyncExternalIdMapping, {\n integrationId,\n internalEntityType: entityType,\n internalEntityId: localId,\n externalId,\n syncStatus: 'synced',\n lastSyncedAt: new Date(),\n organizationId: scope.organizationId,\n tenantId: scope.tenantId,\n })\n\n await em.persistAndFlush(created)\n return created\n },\n }\n}\n\nexport type ExternalIdMappingService = ReturnType<typeof createExternalIdMappingService>\n"],
|
|
5
|
+
"mappings": "AACA,SAAS,6BAA6B;AACtC,SAAS,6BAA6B;AAO/B,SAAS,+BAA+B,IAAmB;AAChE,SAAO;AAAA,IACL,MAAM,cAAc,eAAuB,YAAoB,YAAoB,OAA6C;AAC9H,YAAM,MAAM,MAAM;AAAA,QAChB;AAAA,QACA;AAAA,QACA;AAAA,UACA;AAAA,UACA,oBAAoB;AAAA,UACpB;AAAA,UACA,gBAAgB,MAAM;AAAA,UACtB,UAAU,MAAM;AAAA,UAChB,WAAW;AAAA,QACX;AAAA,QACA;AAAA,QACA;AAAA,MACF;AACA,aAAO,KAAK,oBAAoB;AAAA,IAClC;AAAA,IAEA,MAAM,iBAAiB,eAAuB,YAAoB,SAAiB,OAA6C;AAC9H,YAAM,MAAM,MAAM;AAAA,QAChB;AAAA,QACA;AAAA,QACA;AAAA,UACA;AAAA,UACA,oBAAoB;AAAA,UACpB,kBAAkB;AAAA,UAClB,gBAAgB,MAAM;AAAA,UACtB,UAAU,MAAM;AAAA,UAChB,WAAW;AAAA,QACX;AAAA,QACA;AAAA,QACA;AAAA,MACF;AACA,aAAO,KAAK,cAAc;AAAA,IAC5B;AAAA,IAEA,MAAM,uBACJ,eACA,YACA,SACA,YACA,OACgC;AAChC,YAAM,WAAW,MAAM;AAAA,QACrB;AAAA,QACA;AAAA,QACA;AAAA,UACA;AAAA,UACA,oBAAoB;AAAA,UACpB,kBAAkB;AAAA,UAClB,gBAAgB,MAAM;AAAA,UACtB,UAAU,MAAM;AAAA,UAChB,WAAW;AAAA,QACX;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAEA,UAAI,UAAU;AACZ,iBAAS,aAAa;AACtB,iBAAS,aAAa;AACtB,iBAAS,eAAe,oBAAI,KAAK;AACjC,cAAM,GAAG,MAAM;AACf,eAAO;AAAA,MACT;AAEA,YAAM,UAAU,GAAG,OAAO,uBAAuB;AAAA,QAC/C;AAAA,QACA,oBAAoB;AAAA,QACpB,kBAAkB;AAAA,QAClB;AAAA,QACA,YAAY;AAAA,QACZ,cAAc,oBAAI,KAAK;AAAA,QACvB,gBAAgB,MAAM;AAAA,QACtB,UAAU,MAAM;AAAA,MAClB,CAAC;AAED,YAAM,GAAG,gBAAgB,OAAO;AAChC,aAAO;AAAA,IACT;AAAA,EACF;AACF;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { createQueue } from "@open-mercato/queue";
|
|
2
|
+
import { getRedisUrl } from "@open-mercato/shared/lib/redis/connection";
|
|
3
|
+
const queues = /* @__PURE__ */ new Map();
|
|
4
|
+
function getSyncQueue(queueName) {
|
|
5
|
+
const existing = queues.get(queueName);
|
|
6
|
+
if (existing) return existing;
|
|
7
|
+
const created = process.env.QUEUE_STRATEGY === "async" ? createQueue(queueName, "async", {
|
|
8
|
+
connection: { url: getRedisUrl("QUEUE") },
|
|
9
|
+
concurrency: Math.max(1, Number.parseInt(process.env.DATA_SYNC_QUEUE_CONCURRENCY ?? "5", 10) || 5)
|
|
10
|
+
}) : createQueue(queueName, "local");
|
|
11
|
+
queues.set(queueName, created);
|
|
12
|
+
return created;
|
|
13
|
+
}
|
|
14
|
+
export {
|
|
15
|
+
getSyncQueue
|
|
16
|
+
};
|
|
17
|
+
//# sourceMappingURL=queue.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../../../src/modules/data_sync/lib/queue.ts"],
|
|
4
|
+
"sourcesContent": ["import { createQueue, type Queue } from '@open-mercato/queue'\nimport { getRedisUrl } from '@open-mercato/shared/lib/redis/connection'\n\nconst queues = new Map<string, Queue<Record<string, unknown>>>()\n\nexport function getSyncQueue(queueName: string): Queue<Record<string, unknown>> {\n const existing = queues.get(queueName)\n if (existing) return existing\n\n const created = process.env.QUEUE_STRATEGY === 'async'\n ? createQueue<Record<string, unknown>>(queueName, 'async', {\n connection: { url: getRedisUrl('QUEUE') },\n concurrency: Math.max(1, Number.parseInt(process.env.DATA_SYNC_QUEUE_CONCURRENCY ?? '5', 10) || 5),\n })\n : createQueue<Record<string, unknown>>(queueName, 'local')\n\n queues.set(queueName, created)\n return created\n}\n"],
|
|
5
|
+
"mappings": "AAAA,SAAS,mBAA+B;AACxC,SAAS,mBAAmB;AAE5B,MAAM,SAAS,oBAAI,IAA4C;AAExD,SAAS,aAAa,WAAmD;AAC9E,QAAM,WAAW,OAAO,IAAI,SAAS;AACrC,MAAI,SAAU,QAAO;AAErB,QAAM,UAAU,QAAQ,IAAI,mBAAmB,UAC3C,YAAqC,WAAW,SAAS;AAAA,IACzD,YAAY,EAAE,KAAK,YAAY,OAAO,EAAE;AAAA,IACxC,aAAa,KAAK,IAAI,GAAG,OAAO,SAAS,QAAQ,IAAI,+BAA+B,KAAK,EAAE,KAAK,CAAC;AAAA,EACnG,CAAC,IACC,YAAqC,WAAW,OAAO;AAE3D,SAAO,IAAI,WAAW,OAAO;AAC7B,SAAO;AACT;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
import { getIntegration } from "@open-mercato/shared/modules/integrations/types";
|
|
2
|
+
import { emitDataSyncEvent } from "../events.js";
|
|
3
|
+
import { getDataSyncAdapter } from "./adapter-registry.js";
|
|
4
|
+
function resolveProviderKey(integrationId) {
|
|
5
|
+
return getIntegration(integrationId)?.providerKey ?? integrationId;
|
|
6
|
+
}
|
|
7
|
+
function applyImportCounters(batch) {
|
|
8
|
+
let createdCount = 0;
|
|
9
|
+
let updatedCount = 0;
|
|
10
|
+
let skippedCount = 0;
|
|
11
|
+
let failedCount = 0;
|
|
12
|
+
for (const item of batch.items) {
|
|
13
|
+
if (item.action === "create") createdCount += 1;
|
|
14
|
+
else if (item.action === "update") updatedCount += 1;
|
|
15
|
+
else skippedCount += 1;
|
|
16
|
+
}
|
|
17
|
+
return { createdCount, updatedCount, skippedCount, failedCount };
|
|
18
|
+
}
|
|
19
|
+
function applyExportCounters(batch) {
|
|
20
|
+
let failedCount = 0;
|
|
21
|
+
let skippedCount = 0;
|
|
22
|
+
let updatedCount = 0;
|
|
23
|
+
for (const result of batch.results) {
|
|
24
|
+
if (result.status === "error") failedCount += 1;
|
|
25
|
+
else if (result.status === "skipped") skippedCount += 1;
|
|
26
|
+
else updatedCount += 1;
|
|
27
|
+
}
|
|
28
|
+
return {
|
|
29
|
+
failedCount,
|
|
30
|
+
skippedCount,
|
|
31
|
+
updatedCount,
|
|
32
|
+
processedCount: batch.results.length
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
function createSyncEngine(deps) {
|
|
36
|
+
const { syncRunService, integrationCredentialsService, integrationLogService, progressService } = deps;
|
|
37
|
+
async function resolveMapping(adapter, entityType, scope) {
|
|
38
|
+
return adapter.getMapping({
|
|
39
|
+
entityType,
|
|
40
|
+
scope: { organizationId: scope.organizationId, tenantId: scope.tenantId }
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
async function updateProgress(progressJobId, processedCount, totalCount, scope) {
|
|
44
|
+
if (!progressJobId) return;
|
|
45
|
+
await progressService.updateProgress(
|
|
46
|
+
progressJobId,
|
|
47
|
+
{
|
|
48
|
+
processedCount,
|
|
49
|
+
totalCount: totalCount ?? void 0
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
tenantId: scope.tenantId,
|
|
53
|
+
organizationId: scope.organizationId,
|
|
54
|
+
userId: scope.userId
|
|
55
|
+
}
|
|
56
|
+
);
|
|
57
|
+
}
|
|
58
|
+
async function finalizeRun(runId, status, scope, error) {
|
|
59
|
+
const run = await syncRunService.markStatus(runId, status, scope, error);
|
|
60
|
+
if (!run) return;
|
|
61
|
+
if (run.progressJobId) {
|
|
62
|
+
if (status === "completed") {
|
|
63
|
+
await progressService.completeJob(
|
|
64
|
+
run.progressJobId,
|
|
65
|
+
{
|
|
66
|
+
resultSummary: {
|
|
67
|
+
createdCount: run.createdCount,
|
|
68
|
+
updatedCount: run.updatedCount,
|
|
69
|
+
skippedCount: run.skippedCount,
|
|
70
|
+
failedCount: run.failedCount,
|
|
71
|
+
batchesCompleted: run.batchesCompleted
|
|
72
|
+
}
|
|
73
|
+
},
|
|
74
|
+
{
|
|
75
|
+
tenantId: scope.tenantId,
|
|
76
|
+
organizationId: scope.organizationId,
|
|
77
|
+
userId: scope.userId
|
|
78
|
+
}
|
|
79
|
+
);
|
|
80
|
+
} else if (status === "failed") {
|
|
81
|
+
await progressService.failJob(
|
|
82
|
+
run.progressJobId,
|
|
83
|
+
{
|
|
84
|
+
errorMessage: error ?? "Sync run failed"
|
|
85
|
+
},
|
|
86
|
+
{
|
|
87
|
+
tenantId: scope.tenantId,
|
|
88
|
+
organizationId: scope.organizationId,
|
|
89
|
+
userId: scope.userId
|
|
90
|
+
}
|
|
91
|
+
);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
if (status === "completed") {
|
|
95
|
+
await emitDataSyncEvent("data_sync.run.completed", {
|
|
96
|
+
runId,
|
|
97
|
+
integrationId: run.integrationId,
|
|
98
|
+
entityType: run.entityType,
|
|
99
|
+
direction: run.direction,
|
|
100
|
+
tenantId: scope.tenantId,
|
|
101
|
+
organizationId: scope.organizationId
|
|
102
|
+
});
|
|
103
|
+
return;
|
|
104
|
+
}
|
|
105
|
+
if (status === "cancelled") {
|
|
106
|
+
await emitDataSyncEvent("data_sync.run.cancelled", {
|
|
107
|
+
runId,
|
|
108
|
+
integrationId: run.integrationId,
|
|
109
|
+
entityType: run.entityType,
|
|
110
|
+
direction: run.direction,
|
|
111
|
+
tenantId: scope.tenantId,
|
|
112
|
+
organizationId: scope.organizationId
|
|
113
|
+
});
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
await emitDataSyncEvent("data_sync.run.failed", {
|
|
117
|
+
runId,
|
|
118
|
+
integrationId: run.integrationId,
|
|
119
|
+
entityType: run.entityType,
|
|
120
|
+
direction: run.direction,
|
|
121
|
+
error: error ?? null,
|
|
122
|
+
tenantId: scope.tenantId,
|
|
123
|
+
organizationId: scope.organizationId
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
return {
|
|
127
|
+
async runImport(runId, batchSize, scope) {
|
|
128
|
+
const run = await syncRunService.getRun(runId, scope);
|
|
129
|
+
if (!run) throw new Error(`Sync run ${runId} not found`);
|
|
130
|
+
const providerKey = resolveProviderKey(run.integrationId);
|
|
131
|
+
const adapter = getDataSyncAdapter(providerKey);
|
|
132
|
+
if (!adapter?.streamImport) {
|
|
133
|
+
throw new Error(`No import adapter registered for provider ${providerKey}`);
|
|
134
|
+
}
|
|
135
|
+
const credentials = await integrationCredentialsService.resolve(run.integrationId, scope);
|
|
136
|
+
if (!credentials) {
|
|
137
|
+
throw new Error(`Integration ${run.integrationId} is missing credentials`);
|
|
138
|
+
}
|
|
139
|
+
await syncRunService.markStatus(run.id, "running", scope);
|
|
140
|
+
await emitDataSyncEvent("data_sync.run.started", {
|
|
141
|
+
runId: run.id,
|
|
142
|
+
integrationId: run.integrationId,
|
|
143
|
+
entityType: run.entityType,
|
|
144
|
+
direction: run.direction,
|
|
145
|
+
tenantId: scope.tenantId,
|
|
146
|
+
organizationId: scope.organizationId
|
|
147
|
+
});
|
|
148
|
+
if (run.progressJobId) {
|
|
149
|
+
await progressService.startJob(run.progressJobId, {
|
|
150
|
+
tenantId: scope.tenantId,
|
|
151
|
+
organizationId: scope.organizationId,
|
|
152
|
+
userId: scope.userId
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
const mapping = await resolveMapping(adapter, run.entityType, scope);
|
|
156
|
+
let processedCount = 0;
|
|
157
|
+
let totalCount = null;
|
|
158
|
+
try {
|
|
159
|
+
for await (const batch of adapter.streamImport({
|
|
160
|
+
entityType: run.entityType,
|
|
161
|
+
cursor: run.cursor ?? void 0,
|
|
162
|
+
batchSize,
|
|
163
|
+
credentials,
|
|
164
|
+
mapping,
|
|
165
|
+
scope: { organizationId: scope.organizationId, tenantId: scope.tenantId }
|
|
166
|
+
})) {
|
|
167
|
+
if (run.progressJobId && await progressService.isCancellationRequested(run.progressJobId)) {
|
|
168
|
+
await finalizeRun(run.id, "cancelled", scope);
|
|
169
|
+
return;
|
|
170
|
+
}
|
|
171
|
+
const delta = applyImportCounters(batch);
|
|
172
|
+
processedCount += batch.items.length;
|
|
173
|
+
totalCount = batch.totalEstimate ?? totalCount;
|
|
174
|
+
await syncRunService.updateCounts(
|
|
175
|
+
run.id,
|
|
176
|
+
{
|
|
177
|
+
...delta,
|
|
178
|
+
batchesCompleted: 1
|
|
179
|
+
},
|
|
180
|
+
scope
|
|
181
|
+
);
|
|
182
|
+
await syncRunService.updateCursor(run.id, batch.cursor, scope);
|
|
183
|
+
await updateProgress(run.progressJobId, processedCount, totalCount, scope);
|
|
184
|
+
await integrationLogService.write(
|
|
185
|
+
{
|
|
186
|
+
integrationId: run.integrationId,
|
|
187
|
+
runId: run.id,
|
|
188
|
+
level: "info",
|
|
189
|
+
message: `Processed import batch ${batch.batchIndex}`,
|
|
190
|
+
payload: {
|
|
191
|
+
processedCount,
|
|
192
|
+
batchSize: batch.items.length,
|
|
193
|
+
cursor: batch.cursor
|
|
194
|
+
}
|
|
195
|
+
},
|
|
196
|
+
scope
|
|
197
|
+
);
|
|
198
|
+
}
|
|
199
|
+
} catch (error) {
|
|
200
|
+
const message = error instanceof Error ? error.message : "Sync import failed";
|
|
201
|
+
await integrationLogService.write(
|
|
202
|
+
{
|
|
203
|
+
integrationId: run.integrationId,
|
|
204
|
+
runId: run.id,
|
|
205
|
+
level: "error",
|
|
206
|
+
message
|
|
207
|
+
},
|
|
208
|
+
scope
|
|
209
|
+
);
|
|
210
|
+
await finalizeRun(run.id, "failed", scope, message);
|
|
211
|
+
return;
|
|
212
|
+
}
|
|
213
|
+
await finalizeRun(run.id, "completed", scope);
|
|
214
|
+
},
|
|
215
|
+
async runExport(runId, batchSize, scope) {
|
|
216
|
+
const run = await syncRunService.getRun(runId, scope);
|
|
217
|
+
if (!run) throw new Error(`Sync run ${runId} not found`);
|
|
218
|
+
const providerKey = resolveProviderKey(run.integrationId);
|
|
219
|
+
const adapter = getDataSyncAdapter(providerKey);
|
|
220
|
+
if (!adapter?.streamExport) {
|
|
221
|
+
throw new Error(`No export adapter registered for provider ${providerKey}`);
|
|
222
|
+
}
|
|
223
|
+
const credentials = await integrationCredentialsService.resolve(run.integrationId, scope);
|
|
224
|
+
if (!credentials) {
|
|
225
|
+
throw new Error(`Integration ${run.integrationId} is missing credentials`);
|
|
226
|
+
}
|
|
227
|
+
await syncRunService.markStatus(run.id, "running", scope);
|
|
228
|
+
await emitDataSyncEvent("data_sync.run.started", {
|
|
229
|
+
runId: run.id,
|
|
230
|
+
integrationId: run.integrationId,
|
|
231
|
+
entityType: run.entityType,
|
|
232
|
+
direction: run.direction,
|
|
233
|
+
tenantId: scope.tenantId,
|
|
234
|
+
organizationId: scope.organizationId
|
|
235
|
+
});
|
|
236
|
+
if (run.progressJobId) {
|
|
237
|
+
await progressService.startJob(run.progressJobId, {
|
|
238
|
+
tenantId: scope.tenantId,
|
|
239
|
+
organizationId: scope.organizationId,
|
|
240
|
+
userId: scope.userId
|
|
241
|
+
});
|
|
242
|
+
}
|
|
243
|
+
const mapping = await resolveMapping(adapter, run.entityType, scope);
|
|
244
|
+
let processedCount = 0;
|
|
245
|
+
try {
|
|
246
|
+
for await (const batch of adapter.streamExport({
|
|
247
|
+
entityType: run.entityType,
|
|
248
|
+
cursor: run.cursor ?? void 0,
|
|
249
|
+
batchSize,
|
|
250
|
+
credentials,
|
|
251
|
+
mapping,
|
|
252
|
+
scope: { organizationId: scope.organizationId, tenantId: scope.tenantId }
|
|
253
|
+
})) {
|
|
254
|
+
if (run.progressJobId && await progressService.isCancellationRequested(run.progressJobId)) {
|
|
255
|
+
await finalizeRun(run.id, "cancelled", scope);
|
|
256
|
+
return;
|
|
257
|
+
}
|
|
258
|
+
const delta = applyExportCounters(batch);
|
|
259
|
+
processedCount += delta.processedCount;
|
|
260
|
+
await syncRunService.updateCounts(
|
|
261
|
+
run.id,
|
|
262
|
+
{
|
|
263
|
+
createdCount: 0,
|
|
264
|
+
updatedCount: delta.updatedCount,
|
|
265
|
+
skippedCount: delta.skippedCount,
|
|
266
|
+
failedCount: delta.failedCount,
|
|
267
|
+
batchesCompleted: 1
|
|
268
|
+
},
|
|
269
|
+
scope
|
|
270
|
+
);
|
|
271
|
+
await syncRunService.updateCursor(run.id, batch.cursor, scope);
|
|
272
|
+
await updateProgress(run.progressJobId, processedCount, null, scope);
|
|
273
|
+
await integrationLogService.write(
|
|
274
|
+
{
|
|
275
|
+
integrationId: run.integrationId,
|
|
276
|
+
runId: run.id,
|
|
277
|
+
level: "info",
|
|
278
|
+
message: `Processed export batch ${batch.batchIndex}`,
|
|
279
|
+
payload: {
|
|
280
|
+
processedCount,
|
|
281
|
+
batchSize: batch.results.length,
|
|
282
|
+
cursor: batch.cursor
|
|
283
|
+
}
|
|
284
|
+
},
|
|
285
|
+
scope
|
|
286
|
+
);
|
|
287
|
+
}
|
|
288
|
+
} catch (error) {
|
|
289
|
+
const message = error instanceof Error ? error.message : "Sync export failed";
|
|
290
|
+
await integrationLogService.write(
|
|
291
|
+
{
|
|
292
|
+
integrationId: run.integrationId,
|
|
293
|
+
runId: run.id,
|
|
294
|
+
level: "error",
|
|
295
|
+
message
|
|
296
|
+
},
|
|
297
|
+
scope
|
|
298
|
+
);
|
|
299
|
+
await finalizeRun(run.id, "failed", scope, message);
|
|
300
|
+
return;
|
|
301
|
+
}
|
|
302
|
+
await finalizeRun(run.id, "completed", scope);
|
|
303
|
+
}
|
|
304
|
+
};
|
|
305
|
+
}
|
|
306
|
+
export {
|
|
307
|
+
createSyncEngine
|
|
308
|
+
};
|
|
309
|
+
//# sourceMappingURL=sync-engine.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../../../src/modules/data_sync/lib/sync-engine.ts"],
|
|
4
|
+
"sourcesContent": ["import type { EntityManager } from '@mikro-orm/postgresql'\nimport { getIntegration } from '@open-mercato/shared/modules/integrations/types'\nimport type { CredentialsService } from '../../integrations/lib/credentials-service'\nimport type { IntegrationLogService } from '../../integrations/lib/log-service'\nimport type { ProgressService } from '../../progress/lib/progressService'\nimport { emitDataSyncEvent } from '../events'\nimport type { DataSyncAdapter, DataMapping, ExportBatch, ImportBatch } from './adapter'\nimport { getDataSyncAdapter } from './adapter-registry'\nimport type { SyncRunService } from './sync-run-service'\n\ntype SyncScope = {\n organizationId: string\n tenantId: string\n userId?: string | null\n}\n\ntype EngineDeps = {\n em: EntityManager\n syncRunService: SyncRunService\n integrationCredentialsService: CredentialsService\n integrationLogService: IntegrationLogService\n progressService: ProgressService\n}\n\nfunction resolveProviderKey(integrationId: string): string {\n return getIntegration(integrationId)?.providerKey ?? integrationId\n}\n\nfunction applyImportCounters(batch: ImportBatch): Pick<Required<SyncCounterDelta>, 'createdCount' | 'updatedCount' | 'skippedCount' | 'failedCount'> {\n let createdCount = 0\n let updatedCount = 0\n let skippedCount = 0\n let failedCount = 0\n\n for (const item of batch.items) {\n if (item.action === 'create') createdCount += 1\n else if (item.action === 'update') updatedCount += 1\n else skippedCount += 1\n }\n\n return { createdCount, updatedCount, skippedCount, failedCount }\n}\n\ntype SyncCounterDelta = {\n createdCount?: number\n updatedCount?: number\n skippedCount?: number\n failedCount?: number\n processedCount: number\n}\n\nfunction applyExportCounters(batch: ExportBatch): SyncCounterDelta {\n let failedCount = 0\n let skippedCount = 0\n let updatedCount = 0\n\n for (const result of batch.results) {\n if (result.status === 'error') failedCount += 1\n else if (result.status === 'skipped') skippedCount += 1\n else updatedCount += 1\n }\n\n return {\n failedCount,\n skippedCount,\n updatedCount,\n processedCount: batch.results.length,\n }\n}\n\nexport function createSyncEngine(deps: EngineDeps) {\n const { syncRunService, integrationCredentialsService, integrationLogService, progressService } = deps\n\n async function resolveMapping(adapter: DataSyncAdapter, entityType: string, scope: SyncScope): Promise<DataMapping> {\n return adapter.getMapping({\n entityType,\n scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },\n })\n }\n\n async function updateProgress(progressJobId: string | null | undefined, processedCount: number, totalCount: number | null, scope: SyncScope): Promise<void> {\n if (!progressJobId) return\n\n await progressService.updateProgress(\n progressJobId,\n {\n processedCount,\n totalCount: totalCount ?? undefined,\n },\n {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n },\n )\n }\n\n async function finalizeRun(runId: string, status: 'completed' | 'failed' | 'cancelled', scope: SyncScope, error?: string): Promise<void> {\n const run = await syncRunService.markStatus(runId, status, scope, error)\n if (!run) return\n\n if (run.progressJobId) {\n if (status === 'completed') {\n await progressService.completeJob(\n run.progressJobId,\n {\n resultSummary: {\n createdCount: run.createdCount,\n updatedCount: run.updatedCount,\n skippedCount: run.skippedCount,\n failedCount: run.failedCount,\n batchesCompleted: run.batchesCompleted,\n },\n },\n {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n },\n )\n } else if (status === 'failed') {\n await progressService.failJob(\n run.progressJobId,\n {\n errorMessage: error ?? 'Sync run failed',\n },\n {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n },\n )\n }\n }\n\n if (status === 'completed') {\n await emitDataSyncEvent('data_sync.run.completed', {\n runId,\n integrationId: run.integrationId,\n entityType: run.entityType,\n direction: run.direction,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n })\n return\n }\n\n if (status === 'cancelled') {\n await emitDataSyncEvent('data_sync.run.cancelled', {\n runId,\n integrationId: run.integrationId,\n entityType: run.entityType,\n direction: run.direction,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n })\n return\n }\n\n await emitDataSyncEvent('data_sync.run.failed', {\n runId,\n integrationId: run.integrationId,\n entityType: run.entityType,\n direction: run.direction,\n error: error ?? null,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n })\n }\n\n return {\n async runImport(runId: string, batchSize: number, scope: SyncScope): Promise<void> {\n const run = await syncRunService.getRun(runId, scope)\n if (!run) throw new Error(`Sync run ${runId} not found`)\n\n const providerKey = resolveProviderKey(run.integrationId)\n const adapter = getDataSyncAdapter(providerKey)\n if (!adapter?.streamImport) {\n throw new Error(`No import adapter registered for provider ${providerKey}`)\n }\n\n const credentials = await integrationCredentialsService.resolve(run.integrationId, scope)\n if (!credentials) {\n throw new Error(`Integration ${run.integrationId} is missing credentials`)\n }\n\n await syncRunService.markStatus(run.id, 'running', scope)\n await emitDataSyncEvent('data_sync.run.started', {\n runId: run.id,\n integrationId: run.integrationId,\n entityType: run.entityType,\n direction: run.direction,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n })\n\n if (run.progressJobId) {\n await progressService.startJob(run.progressJobId, {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n })\n }\n\n const mapping = await resolveMapping(adapter, run.entityType, scope)\n let processedCount = 0\n let totalCount: number | null = null\n\n try {\n for await (const batch of adapter.streamImport({\n entityType: run.entityType,\n cursor: run.cursor ?? undefined,\n batchSize,\n credentials,\n mapping,\n scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },\n })) {\n if (run.progressJobId && await progressService.isCancellationRequested(run.progressJobId)) {\n await finalizeRun(run.id, 'cancelled', scope)\n return\n }\n\n const delta = applyImportCounters(batch)\n processedCount += batch.items.length\n totalCount = batch.totalEstimate ?? totalCount\n\n await syncRunService.updateCounts(\n run.id,\n {\n ...delta,\n batchesCompleted: 1,\n },\n scope,\n )\n await syncRunService.updateCursor(run.id, batch.cursor, scope)\n\n await updateProgress(run.progressJobId, processedCount, totalCount, scope)\n\n await integrationLogService.write(\n {\n integrationId: run.integrationId,\n runId: run.id,\n level: 'info',\n message: `Processed import batch ${batch.batchIndex}`,\n payload: {\n processedCount,\n batchSize: batch.items.length,\n cursor: batch.cursor,\n },\n },\n scope,\n )\n }\n } catch (error) {\n const message = error instanceof Error ? error.message : 'Sync import failed'\n await integrationLogService.write(\n {\n integrationId: run.integrationId,\n runId: run.id,\n level: 'error',\n message,\n },\n scope,\n )\n await finalizeRun(run.id, 'failed', scope, message)\n return\n }\n\n await finalizeRun(run.id, 'completed', scope)\n },\n\n async runExport(runId: string, batchSize: number, scope: SyncScope): Promise<void> {\n const run = await syncRunService.getRun(runId, scope)\n if (!run) throw new Error(`Sync run ${runId} not found`)\n\n const providerKey = resolveProviderKey(run.integrationId)\n const adapter = getDataSyncAdapter(providerKey)\n if (!adapter?.streamExport) {\n throw new Error(`No export adapter registered for provider ${providerKey}`)\n }\n\n const credentials = await integrationCredentialsService.resolve(run.integrationId, scope)\n if (!credentials) {\n throw new Error(`Integration ${run.integrationId} is missing credentials`)\n }\n\n await syncRunService.markStatus(run.id, 'running', scope)\n await emitDataSyncEvent('data_sync.run.started', {\n runId: run.id,\n integrationId: run.integrationId,\n entityType: run.entityType,\n direction: run.direction,\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n })\n\n if (run.progressJobId) {\n await progressService.startJob(run.progressJobId, {\n tenantId: scope.tenantId,\n organizationId: scope.organizationId,\n userId: scope.userId,\n })\n }\n\n const mapping = await resolveMapping(adapter, run.entityType, scope)\n let processedCount = 0\n\n try {\n for await (const batch of adapter.streamExport({\n entityType: run.entityType,\n cursor: run.cursor ?? undefined,\n batchSize,\n credentials,\n mapping,\n scope: { organizationId: scope.organizationId, tenantId: scope.tenantId },\n })) {\n if (run.progressJobId && await progressService.isCancellationRequested(run.progressJobId)) {\n await finalizeRun(run.id, 'cancelled', scope)\n return\n }\n\n const delta = applyExportCounters(batch)\n processedCount += delta.processedCount\n\n await syncRunService.updateCounts(\n run.id,\n {\n createdCount: 0,\n updatedCount: delta.updatedCount,\n skippedCount: delta.skippedCount,\n failedCount: delta.failedCount,\n batchesCompleted: 1,\n },\n scope,\n )\n\n await syncRunService.updateCursor(run.id, batch.cursor, scope)\n await updateProgress(run.progressJobId, processedCount, null, scope)\n\n await integrationLogService.write(\n {\n integrationId: run.integrationId,\n runId: run.id,\n level: 'info',\n message: `Processed export batch ${batch.batchIndex}`,\n payload: {\n processedCount,\n batchSize: batch.results.length,\n cursor: batch.cursor,\n },\n },\n scope,\n )\n }\n } catch (error) {\n const message = error instanceof Error ? error.message : 'Sync export failed'\n await integrationLogService.write(\n {\n integrationId: run.integrationId,\n runId: run.id,\n level: 'error',\n message,\n },\n scope,\n )\n await finalizeRun(run.id, 'failed', scope, message)\n return\n }\n\n await finalizeRun(run.id, 'completed', scope)\n },\n }\n}\n\nexport type SyncEngine = ReturnType<typeof createSyncEngine>\n"],
|
|
5
|
+
"mappings": "AACA,SAAS,sBAAsB;AAI/B,SAAS,yBAAyB;AAElC,SAAS,0BAA0B;AAiBnC,SAAS,mBAAmB,eAA+B;AACzD,SAAO,eAAe,aAAa,GAAG,eAAe;AACvD;AAEA,SAAS,oBAAoB,OAAwH;AACnJ,MAAI,eAAe;AACnB,MAAI,eAAe;AACnB,MAAI,eAAe;AACnB,MAAI,cAAc;AAElB,aAAW,QAAQ,MAAM,OAAO;AAC9B,QAAI,KAAK,WAAW,SAAU,iBAAgB;AAAA,aACrC,KAAK,WAAW,SAAU,iBAAgB;AAAA,QAC9C,iBAAgB;AAAA,EACvB;AAEA,SAAO,EAAE,cAAc,cAAc,cAAc,YAAY;AACjE;AAUA,SAAS,oBAAoB,OAAsC;AACjE,MAAI,cAAc;AAClB,MAAI,eAAe;AACnB,MAAI,eAAe;AAEnB,aAAW,UAAU,MAAM,SAAS;AAClC,QAAI,OAAO,WAAW,QAAS,gBAAe;AAAA,aACrC,OAAO,WAAW,UAAW,iBAAgB;AAAA,QACjD,iBAAgB;AAAA,EACvB;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA,gBAAgB,MAAM,QAAQ;AAAA,EAChC;AACF;AAEO,SAAS,iBAAiB,MAAkB;AACjD,QAAM,EAAE,gBAAgB,+BAA+B,uBAAuB,gBAAgB,IAAI;AAElG,iBAAe,eAAe,SAA0B,YAAoB,OAAwC;AAClH,WAAO,QAAQ,WAAW;AAAA,MACxB;AAAA,MACA,OAAO,EAAE,gBAAgB,MAAM,gBAAgB,UAAU,MAAM,SAAS;AAAA,IAC1E,CAAC;AAAA,EACH;AAEA,iBAAe,eAAe,eAA0C,gBAAwB,YAA2B,OAAiC;AAC1J,QAAI,CAAC,cAAe;AAEpB,UAAM,gBAAgB;AAAA,MACpB;AAAA,MACA;AAAA,QACE;AAAA,QACA,YAAY,cAAc;AAAA,MAC5B;AAAA,MACA;AAAA,QACE,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,QACtB,QAAQ,MAAM;AAAA,MAChB;AAAA,IACF;AAAA,EACF;AAEA,iBAAe,YAAY,OAAe,QAA8C,OAAkB,OAA+B;AACvI,UAAM,MAAM,MAAM,eAAe,WAAW,OAAO,QAAQ,OAAO,KAAK;AACvE,QAAI,CAAC,IAAK;AAEV,QAAI,IAAI,eAAe;AACrB,UAAI,WAAW,aAAa;AAC1B,cAAM,gBAAgB;AAAA,UACpB,IAAI;AAAA,UACJ;AAAA,YACE,eAAe;AAAA,cACb,cAAc,IAAI;AAAA,cAClB,cAAc,IAAI;AAAA,cAClB,cAAc,IAAI;AAAA,cAClB,aAAa,IAAI;AAAA,cACjB,kBAAkB,IAAI;AAAA,YACxB;AAAA,UACF;AAAA,UACA;AAAA,YACE,UAAU,MAAM;AAAA,YAChB,gBAAgB,MAAM;AAAA,YACtB,QAAQ,MAAM;AAAA,UAChB;AAAA,QACF;AAAA,MACF,WAAW,WAAW,UAAU;AAC9B,cAAM,gBAAgB;AAAA,UACpB,IAAI;AAAA,UACJ;AAAA,YACE,cAAc,SAAS;AAAA,UACzB;AAAA,UACA;AAAA,YACE,UAAU,MAAM;AAAA,YAChB,gBAAgB,MAAM;AAAA,YACtB,QAAQ,MAAM;AAAA,UAChB;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,WAAW,aAAa;AAC1B,YAAM,kBAAkB,2BAA2B;AAAA,QACjD;AAAA,QACA,eAAe,IAAI;AAAA,QACnB,YAAY,IAAI;AAAA,QAChB,WAAW,IAAI;AAAA,QACf,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,MACxB,CAAC;AACD;AAAA,IACF;AAEA,QAAI,WAAW,aAAa;AAC1B,YAAM,kBAAkB,2BAA2B;AAAA,QACjD;AAAA,QACA,eAAe,IAAI;AAAA,QACnB,YAAY,IAAI;AAAA,QAChB,WAAW,IAAI;AAAA,QACf,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,MACxB,CAAC;AACD;AAAA,IACF;AAEA,UAAM,kBAAkB,wBAAwB;AAAA,MAC9C;AAAA,MACA,eAAe,IAAI;AAAA,MACnB,YAAY,IAAI;AAAA,MAChB,WAAW,IAAI;AAAA,MACf,OAAO,SAAS;AAAA,MAChB,UAAU,MAAM;AAAA,MAChB,gBAAgB,MAAM;AAAA,IACxB,CAAC;AAAA,EACH;AAEA,SAAO;AAAA,IACL,MAAM,UAAU,OAAe,WAAmB,OAAiC;AACjF,YAAM,MAAM,MAAM,eAAe,OAAO,OAAO,KAAK;AACpD,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,YAAY,KAAK,YAAY;AAEvD,YAAM,cAAc,mBAAmB,IAAI,aAAa;AACxD,YAAM,UAAU,mBAAmB,WAAW;AAC9C,UAAI,CAAC,SAAS,cAAc;AAC1B,cAAM,IAAI,MAAM,6CAA6C,WAAW,EAAE;AAAA,MAC5E;AAEA,YAAM,cAAc,MAAM,8BAA8B,QAAQ,IAAI,eAAe,KAAK;AACxF,UAAI,CAAC,aAAa;AAChB,cAAM,IAAI,MAAM,eAAe,IAAI,aAAa,yBAAyB;AAAA,MAC3E;AAEA,YAAM,eAAe,WAAW,IAAI,IAAI,WAAW,KAAK;AACxD,YAAM,kBAAkB,yBAAyB;AAAA,QAC/C,OAAO,IAAI;AAAA,QACX,eAAe,IAAI;AAAA,QACnB,YAAY,IAAI;AAAA,QAChB,WAAW,IAAI;AAAA,QACf,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,MACxB,CAAC;AAED,UAAI,IAAI,eAAe;AACrB,cAAM,gBAAgB,SAAS,IAAI,eAAe;AAAA,UAChD,UAAU,MAAM;AAAA,UAChB,gBAAgB,MAAM;AAAA,UACtB,QAAQ,MAAM;AAAA,QAChB,CAAC;AAAA,MACH;AAEA,YAAM,UAAU,MAAM,eAAe,SAAS,IAAI,YAAY,KAAK;AACnE,UAAI,iBAAiB;AACrB,UAAI,aAA4B;AAEhC,UAAI;AACF,yBAAiB,SAAS,QAAQ,aAAa;AAAA,UAC7C,YAAY,IAAI;AAAA,UAChB,QAAQ,IAAI,UAAU;AAAA,UACtB;AAAA,UACA;AAAA,UACA;AAAA,UACA,OAAO,EAAE,gBAAgB,MAAM,gBAAgB,UAAU,MAAM,SAAS;AAAA,QAC1E,CAAC,GAAG;AACF,cAAI,IAAI,iBAAiB,MAAM,gBAAgB,wBAAwB,IAAI,aAAa,GAAG;AACzF,kBAAM,YAAY,IAAI,IAAI,aAAa,KAAK;AAC5C;AAAA,UACF;AAEA,gBAAM,QAAQ,oBAAoB,KAAK;AACvC,4BAAkB,MAAM,MAAM;AAC9B,uBAAa,MAAM,iBAAiB;AAEpC,gBAAM,eAAe;AAAA,YACnB,IAAI;AAAA,YACJ;AAAA,cACE,GAAG;AAAA,cACH,kBAAkB;AAAA,YACpB;AAAA,YACA;AAAA,UACF;AACA,gBAAM,eAAe,aAAa,IAAI,IAAI,MAAM,QAAQ,KAAK;AAE7D,gBAAM,eAAe,IAAI,eAAe,gBAAgB,YAAY,KAAK;AAEzE,gBAAM,sBAAsB;AAAA,YAC1B;AAAA,cACE,eAAe,IAAI;AAAA,cACnB,OAAO,IAAI;AAAA,cACX,OAAO;AAAA,cACP,SAAS,0BAA0B,MAAM,UAAU;AAAA,cACnD,SAAS;AAAA,gBACP;AAAA,gBACA,WAAW,MAAM,MAAM;AAAA,gBACvB,QAAQ,MAAM;AAAA,cAChB;AAAA,YACF;AAAA,YACA;AAAA,UACF;AAAA,QACF;AAAA,MACF,SAAS,OAAO;AACd,cAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU;AACzD,cAAM,sBAAsB;AAAA,UAC1B;AAAA,YACE,eAAe,IAAI;AAAA,YACnB,OAAO,IAAI;AAAA,YACX,OAAO;AAAA,YACP;AAAA,UACF;AAAA,UACA;AAAA,QACF;AACA,cAAM,YAAY,IAAI,IAAI,UAAU,OAAO,OAAO;AAClD;AAAA,MACF;AAEA,YAAM,YAAY,IAAI,IAAI,aAAa,KAAK;AAAA,IAC9C;AAAA,IAEA,MAAM,UAAU,OAAe,WAAmB,OAAiC;AACjF,YAAM,MAAM,MAAM,eAAe,OAAO,OAAO,KAAK;AACpD,UAAI,CAAC,IAAK,OAAM,IAAI,MAAM,YAAY,KAAK,YAAY;AAEvD,YAAM,cAAc,mBAAmB,IAAI,aAAa;AACxD,YAAM,UAAU,mBAAmB,WAAW;AAC9C,UAAI,CAAC,SAAS,cAAc;AAC1B,cAAM,IAAI,MAAM,6CAA6C,WAAW,EAAE;AAAA,MAC5E;AAEA,YAAM,cAAc,MAAM,8BAA8B,QAAQ,IAAI,eAAe,KAAK;AACxF,UAAI,CAAC,aAAa;AAChB,cAAM,IAAI,MAAM,eAAe,IAAI,aAAa,yBAAyB;AAAA,MAC3E;AAEA,YAAM,eAAe,WAAW,IAAI,IAAI,WAAW,KAAK;AACxD,YAAM,kBAAkB,yBAAyB;AAAA,QAC/C,OAAO,IAAI;AAAA,QACX,eAAe,IAAI;AAAA,QACnB,YAAY,IAAI;AAAA,QAChB,WAAW,IAAI;AAAA,QACf,UAAU,MAAM;AAAA,QAChB,gBAAgB,MAAM;AAAA,MACxB,CAAC;AAED,UAAI,IAAI,eAAe;AACrB,cAAM,gBAAgB,SAAS,IAAI,eAAe;AAAA,UAChD,UAAU,MAAM;AAAA,UAChB,gBAAgB,MAAM;AAAA,UACtB,QAAQ,MAAM;AAAA,QAChB,CAAC;AAAA,MACH;AAEA,YAAM,UAAU,MAAM,eAAe,SAAS,IAAI,YAAY,KAAK;AACnE,UAAI,iBAAiB;AAErB,UAAI;AACF,yBAAiB,SAAS,QAAQ,aAAa;AAAA,UAC7C,YAAY,IAAI;AAAA,UAChB,QAAQ,IAAI,UAAU;AAAA,UACtB;AAAA,UACA;AAAA,UACA;AAAA,UACA,OAAO,EAAE,gBAAgB,MAAM,gBAAgB,UAAU,MAAM,SAAS;AAAA,QAC1E,CAAC,GAAG;AACF,cAAI,IAAI,iBAAiB,MAAM,gBAAgB,wBAAwB,IAAI,aAAa,GAAG;AACzF,kBAAM,YAAY,IAAI,IAAI,aAAa,KAAK;AAC5C;AAAA,UACF;AAEA,gBAAM,QAAQ,oBAAoB,KAAK;AACvC,4BAAkB,MAAM;AAExB,gBAAM,eAAe;AAAA,YACnB,IAAI;AAAA,YACJ;AAAA,cACE,cAAc;AAAA,cACd,cAAc,MAAM;AAAA,cACpB,cAAc,MAAM;AAAA,cACpB,aAAa,MAAM;AAAA,cACnB,kBAAkB;AAAA,YACpB;AAAA,YACA;AAAA,UACF;AAEA,gBAAM,eAAe,aAAa,IAAI,IAAI,MAAM,QAAQ,KAAK;AAC7D,gBAAM,eAAe,IAAI,eAAe,gBAAgB,MAAM,KAAK;AAEnE,gBAAM,sBAAsB;AAAA,YAC1B;AAAA,cACE,eAAe,IAAI;AAAA,cACnB,OAAO,IAAI;AAAA,cACX,OAAO;AAAA,cACP,SAAS,0BAA0B,MAAM,UAAU;AAAA,cACnD,SAAS;AAAA,gBACP;AAAA,gBACA,WAAW,MAAM,QAAQ;AAAA,gBACzB,QAAQ,MAAM;AAAA,cAChB;AAAA,YACF;AAAA,YACA;AAAA,UACF;AAAA,QACF;AAAA,MACF,SAAS,OAAO;AACd,cAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU;AACzD,cAAM,sBAAsB;AAAA,UAC1B;AAAA,YACE,eAAe,IAAI;AAAA,YACnB,OAAO,IAAI;AAAA,YACX,OAAO;AAAA,YACP;AAAA,UACF;AAAA,UACA;AAAA,QACF;AACA,cAAM,YAAY,IAAI,IAAI,UAAU,OAAO,OAAO;AAClD;AAAA,MACF;AAEA,YAAM,YAAY,IAAI,IAAI,aAAa,KAAK;AAAA,IAC9C;AAAA,EACF;AACF;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|