@fullstackhouse/open-mercato-data-sync-durable 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. package/README.md +86 -0
  2. package/dist/engine/durable-run.js +130 -0
  3. package/dist/engine/durable-run.js.map +7 -0
  4. package/dist/index.js +5 -0
  5. package/dist/index.js.map +7 -0
  6. package/dist/kinds/data-sync-run.js +57 -0
  7. package/dist/kinds/data-sync-run.js.map +7 -0
  8. package/dist/mirror-version.js +5 -0
  9. package/dist/mirror-version.js.map +7 -0
  10. package/dist/mirror.manifest.json +65 -0
  11. package/dist/modules/data_sync/acl.js +8 -0
  12. package/dist/modules/data_sync/acl.js.map +7 -0
  13. package/dist/modules/data_sync/api/mappings/[id]/route.js +17 -0
  14. package/dist/modules/data_sync/api/mappings/[id]/route.js.map +7 -0
  15. package/dist/modules/data_sync/api/mappings/route.js +15 -0
  16. package/dist/modules/data_sync/api/mappings/route.js.map +7 -0
  17. package/dist/modules/data_sync/api/options.js +13 -0
  18. package/dist/modules/data_sync/api/options.js.map +7 -0
  19. package/dist/modules/data_sync/api/run.js +59 -0
  20. package/dist/modules/data_sync/api/run.js.map +7 -0
  21. package/dist/modules/data_sync/api/runs/[id]/cancel.js +13 -0
  22. package/dist/modules/data_sync/api/runs/[id]/cancel.js.map +7 -0
  23. package/dist/modules/data_sync/api/runs/[id]/retry.js +13 -0
  24. package/dist/modules/data_sync/api/runs/[id]/retry.js.map +7 -0
  25. package/dist/modules/data_sync/api/runs/[id]/route.js +13 -0
  26. package/dist/modules/data_sync/api/runs/[id]/route.js.map +7 -0
  27. package/dist/modules/data_sync/api/runs.js +13 -0
  28. package/dist/modules/data_sync/api/runs.js.map +7 -0
  29. package/dist/modules/data_sync/api/schedules/[id]/route.js +17 -0
  30. package/dist/modules/data_sync/api/schedules/[id]/route.js.map +7 -0
  31. package/dist/modules/data_sync/api/schedules/route.js +15 -0
  32. package/dist/modules/data_sync/api/schedules/route.js.map +7 -0
  33. package/dist/modules/data_sync/api/schedules/serialize.js +6 -0
  34. package/dist/modules/data_sync/api/schedules/serialize.js.map +7 -0
  35. package/dist/modules/data_sync/api/validate.js +13 -0
  36. package/dist/modules/data_sync/api/validate.js.map +7 -0
  37. package/dist/modules/data_sync/backend/data-sync/page.js +7 -0
  38. package/dist/modules/data_sync/backend/data-sync/page.js.map +7 -0
  39. package/dist/modules/data_sync/backend/data-sync/page.meta.js +6 -0
  40. package/dist/modules/data_sync/backend/data-sync/page.meta.js.map +7 -0
  41. package/dist/modules/data_sync/backend/data-sync/runs/[id]/page.js +7 -0
  42. package/dist/modules/data_sync/backend/data-sync/runs/[id]/page.js.map +7 -0
  43. package/dist/modules/data_sync/backend/data-sync/runs/[id]/page.meta.js +6 -0
  44. package/dist/modules/data_sync/backend/data-sync/runs/[id]/page.meta.js.map +7 -0
  45. package/dist/modules/data_sync/components/IntegrationScheduleTab.js +9 -0
  46. package/dist/modules/data_sync/components/IntegrationScheduleTab.js.map +7 -0
  47. package/dist/modules/data_sync/components/RunParameterFields.js +13 -0
  48. package/dist/modules/data_sync/components/RunParameterFields.js.map +7 -0
  49. package/dist/modules/data_sync/data/entities.js +9 -0
  50. package/dist/modules/data_sync/data/entities.js.map +7 -0
  51. package/dist/modules/data_sync/data/validators.js +12 -0
  52. package/dist/modules/data_sync/data/validators.js.map +7 -0
  53. package/dist/modules/data_sync/di.js +33 -0
  54. package/dist/modules/data_sync/di.js.map +7 -0
  55. package/dist/modules/data_sync/events.js +9 -0
  56. package/dist/modules/data_sync/events.js.map +7 -0
  57. package/dist/modules/data_sync/extension-points.js +8 -0
  58. package/dist/modules/data_sync/extension-points.js.map +7 -0
  59. package/dist/modules/data_sync/i18n/de.json +135 -0
  60. package/dist/modules/data_sync/i18n/en.json +135 -0
  61. package/dist/modules/data_sync/i18n/es.json +135 -0
  62. package/dist/modules/data_sync/i18n/ko.json +135 -0
  63. package/dist/modules/data_sync/i18n/pl.json +135 -0
  64. package/dist/modules/data_sync/index.js +6 -0
  65. package/dist/modules/data_sync/index.js.map +7 -0
  66. package/dist/modules/data_sync/lib/abandoned-run.js +6 -0
  67. package/dist/modules/data_sync/lib/abandoned-run.js.map +7 -0
  68. package/dist/modules/data_sync/lib/adapter-registry.js +10 -0
  69. package/dist/modules/data_sync/lib/adapter-registry.js.map +7 -0
  70. package/dist/modules/data_sync/lib/adapter.js +2 -0
  71. package/dist/modules/data_sync/lib/adapter.js.map +7 -0
  72. package/dist/modules/data_sync/lib/adopt-on-delivery.js +35 -0
  73. package/dist/modules/data_sync/lib/adopt-on-delivery.js.map +7 -0
  74. package/dist/modules/data_sync/lib/batch-stream.js +6 -0
  75. package/dist/modules/data_sync/lib/batch-stream.js.map +7 -0
  76. package/dist/modules/data_sync/lib/id-mapping.js +6 -0
  77. package/dist/modules/data_sync/lib/id-mapping.js.map +7 -0
  78. package/dist/modules/data_sync/lib/queue-policy.js +11 -0
  79. package/dist/modules/data_sync/lib/queue-policy.js.map +7 -0
  80. package/dist/modules/data_sync/lib/queue.js +6 -0
  81. package/dist/modules/data_sync/lib/queue.js.map +7 -0
  82. package/dist/modules/data_sync/lib/run-parameters.js +8 -0
  83. package/dist/modules/data_sync/lib/run-parameters.js.map +7 -0
  84. package/dist/modules/data_sync/lib/start-cursor.js +8 -0
  85. package/dist/modules/data_sync/lib/start-cursor.js.map +7 -0
  86. package/dist/modules/data_sync/lib/start-run.js +51 -0
  87. package/dist/modules/data_sync/lib/start-run.js.map +7 -0
  88. package/dist/modules/data_sync/lib/sync-engine.js +6 -0
  89. package/dist/modules/data_sync/lib/sync-engine.js.map +7 -0
  90. package/dist/modules/data_sync/lib/sync-run-service.js +7 -0
  91. package/dist/modules/data_sync/lib/sync-run-service.js.map +7 -0
  92. package/dist/modules/data_sync/lib/sync-schedule-service.js +6 -0
  93. package/dist/modules/data_sync/lib/sync-schedule-service.js.map +7 -0
  94. package/dist/modules/data_sync/lib/syncRunStatus.js +9 -0
  95. package/dist/modules/data_sync/lib/syncRunStatus.js.map +7 -0
  96. package/dist/modules/data_sync/lib/version-guard.js +14 -0
  97. package/dist/modules/data_sync/lib/version-guard.js.map +7 -0
  98. package/dist/modules/data_sync/migrations/Migration20260304113737.js +6 -0
  99. package/dist/modules/data_sync/migrations/Migration20260304113737.js.map +7 -0
  100. package/dist/modules/data_sync/migrations/Migration20260810120000.js +6 -0
  101. package/dist/modules/data_sync/migrations/Migration20260810120000.js.map +7 -0
  102. package/dist/modules/data_sync/setup.js +8 -0
  103. package/dist/modules/data_sync/setup.js.map +7 -0
  104. package/dist/modules/data_sync/workers/sync-export.js +19 -0
  105. package/dist/modules/data_sync/workers/sync-export.js.map +7 -0
  106. package/dist/modules/data_sync/workers/sync-import.js +19 -0
  107. package/dist/modules/data_sync/workers/sync-import.js.map +7 -0
  108. package/dist/modules/data_sync/workers/sync-scheduled.js +8 -0
  109. package/dist/modules/data_sync/workers/sync-scheduled.js.map +7 -0
  110. package/generated/entities/sync_cursor/index.ts +8 -0
  111. package/generated/entities/sync_mapping/index.ts +8 -0
  112. package/generated/entities/sync_run/index.ts +22 -0
  113. package/generated/entities/sync_schedule/index.ts +16 -0
  114. package/generated/entities.ids.generated.ts +13 -0
  115. package/package.json +135 -0
  116. package/src/compat/mirror-shape.test.ts +106 -0
  117. package/src/compat/seams.test.ts +97 -0
  118. package/src/engine/durable-run.ts +231 -0
  119. package/src/index.ts +4 -0
  120. package/src/kinds/data-sync-run.ts +104 -0
  121. package/src/mirror-version.ts +3 -0
  122. package/src/mirror.manifest.json +65 -0
  123. package/src/modules/data_sync/__integration__/TC-DSD-001.spec.ts +80 -0
  124. package/src/modules/data_sync/acl.ts +5 -0
  125. package/src/modules/data_sync/api/mappings/[id]/route.ts +12 -0
  126. package/src/modules/data_sync/api/mappings/route.ts +11 -0
  127. package/src/modules/data_sync/api/options.ts +10 -0
  128. package/src/modules/data_sync/api/run.ts +103 -0
  129. package/src/modules/data_sync/api/runs/[id]/cancel.ts +10 -0
  130. package/src/modules/data_sync/api/runs/[id]/retry.ts +10 -0
  131. package/src/modules/data_sync/api/runs/[id]/route.ts +10 -0
  132. package/src/modules/data_sync/api/runs.ts +10 -0
  133. package/src/modules/data_sync/api/schedules/[id]/route.ts +12 -0
  134. package/src/modules/data_sync/api/schedules/route.ts +11 -0
  135. package/src/modules/data_sync/api/schedules/serialize.ts +4 -0
  136. package/src/modules/data_sync/api/validate.ts +10 -0
  137. package/src/modules/data_sync/backend/data-sync/page.meta.ts +4 -0
  138. package/src/modules/data_sync/backend/data-sync/page.tsx +5 -0
  139. package/src/modules/data_sync/backend/data-sync/runs/[id]/page.meta.ts +4 -0
  140. package/src/modules/data_sync/backend/data-sync/runs/[id]/page.tsx +5 -0
  141. package/src/modules/data_sync/components/IntegrationScheduleTab.tsx +6 -0
  142. package/src/modules/data_sync/components/RunParameterFields.tsx +5 -0
  143. package/src/modules/data_sync/data/entities.ts +4 -0
  144. package/src/modules/data_sync/data/validators.ts +4 -0
  145. package/src/modules/data_sync/di.ts +57 -0
  146. package/src/modules/data_sync/events.ts +5 -0
  147. package/src/modules/data_sync/extension-points.ts +5 -0
  148. package/src/modules/data_sync/i18n/de.json +135 -0
  149. package/src/modules/data_sync/i18n/en.json +135 -0
  150. package/src/modules/data_sync/i18n/es.json +135 -0
  151. package/src/modules/data_sync/i18n/ko.json +135 -0
  152. package/src/modules/data_sync/i18n/pl.json +135 -0
  153. package/src/modules/data_sync/index.ts +4 -0
  154. package/src/modules/data_sync/lib/adapter-registry.ts +4 -0
  155. package/src/modules/data_sync/lib/adapter.ts +3 -0
  156. package/src/modules/data_sync/lib/adopt-on-delivery.ts +69 -0
  157. package/src/modules/data_sync/lib/batch-stream.ts +4 -0
  158. package/src/modules/data_sync/lib/id-mapping.ts +4 -0
  159. package/src/modules/data_sync/lib/queue-policy.ts +4 -0
  160. package/src/modules/data_sync/lib/queue.ts +4 -0
  161. package/src/modules/data_sync/lib/run-parameters.ts +4 -0
  162. package/src/modules/data_sync/lib/start-cursor.ts +4 -0
  163. package/src/modules/data_sync/lib/start-run.ts +106 -0
  164. package/src/modules/data_sync/lib/sync-engine.ts +4 -0
  165. package/src/modules/data_sync/lib/sync-run-service.ts +4 -0
  166. package/src/modules/data_sync/lib/sync-schedule-service.ts +4 -0
  167. package/src/modules/data_sync/lib/syncRunStatus.ts +4 -0
  168. package/src/modules/data_sync/lib/version-guard.ts +27 -0
  169. package/src/modules/data_sync/migrations/.snapshot-open-mercato.json +1013 -0
  170. package/src/modules/data_sync/migrations/Migration20260304113737.ts +4 -0
  171. package/src/modules/data_sync/migrations/Migration20260810120000.ts +4 -0
  172. package/src/modules/data_sync/setup.ts +5 -0
  173. package/src/modules/data_sync/workers/sync-export.ts +20 -0
  174. package/src/modules/data_sync/workers/sync-import.ts +20 -0
  175. package/src/modules/data_sync/workers/sync-scheduled.ts +5 -0
package/package.json ADDED
@@ -0,0 +1,135 @@
1
+ {
2
+ "name": "@fullstackhouse/open-mercato-data-sync-durable",
3
+ "version": "0.1.0",
4
+ "description": "Drop-in replacement for Open Mercato's core data_sync module that runs sync runs as durable at-least-once work (via @fullstackhouse/open-mercato-durable-work): survives deploys and worker kills, resumes from the committed cursor, retries transient errors at the batch, never leaves a run 'running' forever. Same tables, API, adapter contract, events and UI as core; swap one line in modules.ts.",
5
+ "type": "module",
6
+ "main": "./dist/index.js",
7
+ "license": "MIT",
8
+ "author": "Full Stack House",
9
+ "repository": {
10
+ "type": "git",
11
+ "url": "https://github.com/fullstackhouse/open-mercato-durable.git",
12
+ "directory": "packages/data-sync-durable"
13
+ },
14
+ "keywords": [
15
+ "open-mercato",
16
+ "data-sync",
17
+ "durable",
18
+ "at-least-once",
19
+ "background-jobs"
20
+ ],
21
+ "publishConfig": {
22
+ "access": "public",
23
+ "provenance": true
24
+ },
25
+ "files": [
26
+ "dist",
27
+ "src",
28
+ "generated",
29
+ "README.md"
30
+ ],
31
+ "scripts": {
32
+ "gen:mirror": "node scripts/gen-mirror.mjs",
33
+ "prebuild": "node scripts/gen-mirror.mjs --check",
34
+ "build": "node build.mjs",
35
+ "prepack": "yarn build",
36
+ "watch": "node watch.mjs",
37
+ "test": "vitest run",
38
+ "test:watch": "vitest",
39
+ "typecheck": "tsc --noEmit",
40
+ "lint": "eslint src scripts"
41
+ },
42
+ "exports": {
43
+ ".": {
44
+ "types": "./src/index.ts",
45
+ "default": "./dist/index.js"
46
+ },
47
+ "./*.ts": {
48
+ "types": "./src/*.ts",
49
+ "default": "./dist/*.js"
50
+ },
51
+ "./*.tsx": {
52
+ "types": "./src/*.tsx",
53
+ "default": "./dist/*.js"
54
+ },
55
+ "./*.json": "./src/*.json",
56
+ "./*": {
57
+ "types": [
58
+ "./src/*.ts",
59
+ "./src/*.tsx"
60
+ ],
61
+ "default": "./dist/*.js"
62
+ },
63
+ "./*/*.json": "./src/*/*.json",
64
+ "./*/*": {
65
+ "types": [
66
+ "./src/*/*.ts",
67
+ "./src/*/*.tsx"
68
+ ],
69
+ "default": "./dist/*/*.js"
70
+ },
71
+ "./*/*/*.json": "./src/*/*/*.json",
72
+ "./*/*/*": {
73
+ "types": [
74
+ "./src/*/*/*.ts",
75
+ "./src/*/*/*.tsx"
76
+ ],
77
+ "default": "./dist/*/*/*.js"
78
+ },
79
+ "./*/*/*/*.json": "./src/*/*/*/*.json",
80
+ "./*/*/*/*": {
81
+ "types": [
82
+ "./src/*/*/*/*.ts",
83
+ "./src/*/*/*/*.tsx"
84
+ ],
85
+ "default": "./dist/*/*/*/*.js"
86
+ },
87
+ "./*/*/*/*/*.json": "./src/*/*/*/*/*.json",
88
+ "./*/*/*/*/*": {
89
+ "types": [
90
+ "./src/*/*/*/*/*.ts",
91
+ "./src/*/*/*/*/*.tsx"
92
+ ],
93
+ "default": "./dist/*/*/*/*/*.js"
94
+ }
95
+ },
96
+ "peerDependencies": {
97
+ "@fullstackhouse/open-mercato-durable-work": "^0.1.0",
98
+ "@mikro-orm/core": ">=7.0.0",
99
+ "@mikro-orm/migrations": ">=7.0.0",
100
+ "@mikro-orm/postgresql": ">=7.0.0",
101
+ "@open-mercato/core": ">=0.7.0 <0.8.0",
102
+ "@open-mercato/queue": ">=0.7.0 <0.8.0",
103
+ "@open-mercato/shared": ">=0.7.0 <0.8.0",
104
+ "@open-mercato/ui": ">=0.7.0 <0.8.0",
105
+ "next": ">=15.0.0",
106
+ "react": ">=19.0.0",
107
+ "react-dom": ">=19.0.0",
108
+ "zod": ">=3.25.0"
109
+ },
110
+ "devDependencies": {
111
+ "@eslint/js": "^9.0.0",
112
+ "@fullstackhouse/open-mercato-durable-work": "^0.1.0",
113
+ "@mikro-orm/core": "^7.1.8",
114
+ "@mikro-orm/migrations": "^7.1.8",
115
+ "@mikro-orm/postgresql": "^7.1.8",
116
+ "@open-mercato/core": "0.7.0",
117
+ "@open-mercato/queue": "0.7.0",
118
+ "@open-mercato/shared": "0.7.0",
119
+ "@open-mercato/ui": "0.7.0",
120
+ "@types/node": "^24.10.1",
121
+ "@types/react": "^19.0.0",
122
+ "awilix": "^13.0.5",
123
+ "esbuild": "^0.25.0",
124
+ "eslint": "^9.0.0",
125
+ "glob": "^13.0.6",
126
+ "globals": "^16.0.0",
127
+ "next": "16.3.0",
128
+ "react": "19.2.8",
129
+ "react-dom": "19.2.8",
130
+ "typescript": "^5.9.3",
131
+ "typescript-eslint": "^8.0.0",
132
+ "vitest": "^3.0.0",
133
+ "zod": "^4.4.3"
134
+ }
135
+ }
@@ -0,0 +1,106 @@
1
+ import { createRequire } from 'node:module'
2
+ import fs from 'node:fs'
3
+ import path from 'node:path'
4
+
5
+ import manifest from '../mirror.manifest.json'
6
+
7
+ // Guards the three findings of the phase-0 mirror spike (docs/adr/0006). Each is a case
8
+ // where the Open Mercato generator reads a module's own file statically and therefore
9
+ // cannot see through `export * from '@open-mercato/core/...'`. Losing any of them is
10
+ // silent — the app boots, the module is registered, and only the affected contract is
11
+ // missing — so they are asserted rather than trusted.
12
+ const pkgDir = path.resolve(__dirname, '..', '..')
13
+ const moduleDir = path.join(pkgDir, 'src', 'modules', 'data_sync')
14
+ const require = createRequire(path.join(pkgDir, 'package.json'))
15
+
16
+ function corePackageRoot(): string {
17
+ // The `exports` map rewrites "./package.json" to "./src/package.json"; walk up instead.
18
+ let dir = path.dirname(require.resolve('@open-mercato/core'))
19
+ for (;;) {
20
+ const file = path.join(dir, 'package.json')
21
+ if (fs.existsSync(file) && JSON.parse(fs.readFileSync(file, 'utf8')).name === '@open-mercato/core') return dir
22
+ const parent = path.dirname(dir)
23
+ if (parent === dir) throw new Error('could not locate @open-mercato/core')
24
+ dir = parent
25
+ }
26
+ }
27
+
28
+ const coreModuleDir = path.join(corePackageRoot(), 'src', 'modules', 'data_sync')
29
+ const HTTP_METHODS = ['GET', 'POST', 'PUT', 'PATCH', 'DELETE']
30
+
31
+ function walk(dir: string, rel = ''): string[] {
32
+ return fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
33
+ const next = rel ? `${rel}/${entry.name}` : entry.name
34
+ if (entry.isDirectory()) return entry.name.startsWith('__') ? [] : walk(path.join(dir, entry.name), next)
35
+ return /\.tsx?$/.test(entry.name) ? [next] : []
36
+ })
37
+ }
38
+
39
+ const coreApiFiles = walk(path.join(coreModuleDir, 'api')).map((rel) => `api/${rel}`)
40
+ // Mirrors `detectExportedHttpMethods` in @open-mercato/cli: declarations and named
41
+ // re-export blocks, both by regex over the file's own text.
42
+ function methodsOf(source: string): string[] {
43
+ const found = new Set<string>()
44
+ for (const m of HTTP_METHODS) {
45
+ if (new RegExp(`export\\s+(?:async\\s+)?function\\s+${m}\\b|export\\s+(?:const|let|var)\\s+${m}\\b`).test(source)) found.add(m)
46
+ }
47
+ for (const match of source.matchAll(/export\s*\{([^}]+)\}/g)) {
48
+ for (const part of (match[1] ?? '').split(',')) {
49
+ const name = part.trim().split(/\s+as\s+/i).pop()?.trim()
50
+ if (name && HTTP_METHODS.includes(name)) found.add(name)
51
+ }
52
+ }
53
+ return HTTP_METHODS.filter((m) => found.has(m))
54
+ }
55
+
56
+ describe('data_sync mirror shape', () => {
57
+ it('mirrors the core version the manifest was generated from', () => {
58
+ expect(manifest.coreVersion).toBe(JSON.parse(fs.readFileSync(path.join(corePackageRoot(), 'package.json'), 'utf8')).version)
59
+ })
60
+
61
+ it('covers every core api file', () => {
62
+ expect(coreApiFiles.length).toBeGreaterThan(0)
63
+ for (const rel of coreApiFiles) expect(fs.existsSync(path.join(moduleDir, rel))).toBe(true)
64
+ })
65
+
66
+ // Finding 1: `detectExportedHttpMethods` is a regex over the file's own text.
67
+ it('names every HTTP method core exports, so the route registers with the same methods', () => {
68
+ for (const rel of coreApiFiles) {
69
+ const expected = methodsOf(fs.readFileSync(path.join(coreModuleDir, rel), 'utf8'))
70
+ if (!expected.length) continue
71
+ expect({ rel, methods: methodsOf(fs.readFileSync(path.join(moduleDir, rel), 'utf8')) }).toEqual({ rel, methods: expected })
72
+ }
73
+ })
74
+
75
+ // Finding 2: route `metadata` is read by AST, and carries requireAuth/requireFeatures.
76
+ it('copies the route metadata literal rather than re-exporting it', () => {
77
+ for (const rel of coreApiFiles) {
78
+ const core = fs.readFileSync(path.join(coreModuleDir, rel), 'utf8')
79
+ if (!methodsOf(core).length || !/export\s+const\s+metadata\s*=\s*\{/.test(core)) continue
80
+ expect({ rel, copied: /export const metadata = \{/.test(fs.readFileSync(path.join(moduleDir, rel), 'utf8')) })
81
+ .toEqual({ rel, copied: true })
82
+ }
83
+ })
84
+
85
+ // Finding 3: entity ids and fields come from @Entity() classes a stub does not have,
86
+ // so they ship as the package-level `generated/` descriptor the CLI looks for.
87
+ it('ships an entity descriptor listing every entity core declares', () => {
88
+ const coreEntities = [...fs.readFileSync(path.join(coreModuleDir, 'data', 'entities.ts'), 'utf8')
89
+ .matchAll(/export\s+class\s+(\w+)/g)].map((m) => m[1].replace(/([a-z0-9])([A-Z])/g, '$1_$2').toLowerCase())
90
+ expect(coreEntities.length).toBeGreaterThan(0)
91
+
92
+ const ids = fs.readFileSync(path.join(pkgDir, 'generated', 'entities.ids.generated.ts'), 'utf8')
93
+ for (const entity of coreEntities) {
94
+ expect({ entity, declared: ids.includes(`"${entity}": "data_sync:${entity}"`) }).toEqual({ entity, declared: true })
95
+ expect(fs.existsSync(path.join(pkgDir, 'generated', 'entities', entity, 'index.ts'))).toBe(true)
96
+ }
97
+ })
98
+
99
+ // `data/entities.ts` must stay a re-export: the app spreads our namespace into MikroORM's
100
+ // entity list, so copying the classes would register duplicates instead of core's.
101
+ it('keeps the entities module a re-export so core keeps class identity', () => {
102
+ const ours = fs.readFileSync(path.join(moduleDir, 'data', 'entities.ts'), 'utf8')
103
+ expect(ours).toContain("export * from '@open-mercato/core/modules/data_sync/data/entities'")
104
+ expect(ours).not.toMatch(/export\s+class\s/)
105
+ })
106
+ })
@@ -0,0 +1,97 @@
1
+ // The seams this package depends on, asserted against the installed `@open-mercato/core`.
2
+ //
3
+ // The adopter does not fork core's engine; it decorates three methods and relies on one
4
+ // documented branch of `finalizeRun` (ADR 0004). That is a far smaller coupling than a fork —
5
+ // but only if it is *checked*, because every one of these could be changed upstream in a way
6
+ // that compiles fine and silently stops the durable behaviour:
7
+ //
8
+ // - if `markStatus` stopped returning the row unchanged on a refused terminal transition,
9
+ // core would write the terminal state itself and the durable one would arrive second
10
+ // - if the batch loop stopped consulting `isCancellationRequested`, a slice would have no
11
+ // hand-back point and could only be interrupted between whole runs
12
+ // - if `commitBatchProgress` changed shape, cursor commits would stop being fenced
13
+ //
14
+ // These run in the `compat` CI lane against both the released and the development channel, so
15
+ // a core release that moves any of them fails here rather than in a host.
16
+
17
+ import { createRequire } from 'node:module'
18
+ import fs from 'node:fs'
19
+ import path from 'node:path'
20
+
21
+ const require = createRequire(path.join(__dirname, '..', '..', 'package.json'))
22
+
23
+ function coreRoot(): string {
24
+ let dir = path.dirname(require.resolve('@open-mercato/core'))
25
+ for (;;) {
26
+ const manifest = path.join(dir, 'package.json')
27
+ if (fs.existsSync(manifest) && JSON.parse(fs.readFileSync(manifest, 'utf8')).name === '@open-mercato/core') return dir
28
+ const parent = path.dirname(dir)
29
+ if (parent === dir) throw new Error('could not locate @open-mercato/core')
30
+ dir = parent
31
+ }
32
+ }
33
+
34
+ const read = (relative: string): string => fs.readFileSync(path.join(coreRoot(), 'src', 'modules', 'data_sync', relative), 'utf8')
35
+
36
+ describe('the seams the durable adopter decorates', () => {
37
+ const runService = read('lib/sync-run-service.ts')
38
+ const engine = read('lib/sync-engine.ts')
39
+
40
+ it('still exposes the three run-service methods the adopter wraps', () => {
41
+ for (const method of ['getRun', 'markStatus', 'commitBatchProgress']) {
42
+ expect({ method, present: new RegExp(`\\b${method}\\s*\\(`).test(runService) }).toEqual({ method, present: true })
43
+ }
44
+ })
45
+
46
+ it('still returns the row unchanged when a terminal transition is refused', () => {
47
+ // This is what lets the adopter record a terminal outcome instead of performing it. Without
48
+ // it, core writes the run's terminal state itself and the durable transition arrives second
49
+ // — two writers, and the domain row no longer moves with the job row.
50
+ expect(runService).toMatch(/isTerminal\s*&&\s*row\.status\s*!==\s*status/)
51
+ expect(runService).toMatch(/if \(isTerminal && row\.status !== status\) \{\s*\n\s*return row/)
52
+ })
53
+
54
+ it('still stays silent when the status it wrote is not the status it asked for', () => {
55
+ // `finalizeRun`'s "another worker already finalized this" branch. It is what suppresses the
56
+ // progress write, the operational log and the lifecycle event, so the durable terminal
57
+ // transition can emit them exactly once instead.
58
+ expect(engine).toMatch(/const run = await syncRunService\.markStatus\(runId, status, scope, error\)/)
59
+ expect(engine).toMatch(/if \(run\.status !== status\) \{/)
60
+ })
61
+
62
+ it('still asks whether to cancel at every batch boundary', () => {
63
+ // The hand-back point. Without it a slice has nowhere to stop, and the mechanism's
64
+ // headline property — a deploy does not kill a multi-day run — is gone.
65
+ //
66
+ // Asserted as a property rather than a phrasing, because the phrasing has already moved:
67
+ // `develop` puts a background poller in front of the check
68
+ // (`cancellation.signal.aborted || (run.progressJobId && await isCancellationRequested(…))`).
69
+ // The seam is unaffected — the question is still asked per batch and a true answer still
70
+ // stops the stream — and a probe that failed on the wording would cry wolf on every
71
+ // upstream refactor while missing the change that actually matters.
72
+ const checks = engine.match(/progressService\.isCancellationRequested\(/g) ?? []
73
+ expect(checks.length).toBeGreaterThanOrEqual(2) // import and export
74
+
75
+ for (const branch of engine.split('isCancellationRequested(').slice(1)) {
76
+ // Every use either leads to a stop, or feeds the poller that aborts to the same end.
77
+ const leadsToStop = /return 'stop'/.test(branch.slice(0, 400))
78
+ const feedsThePoller = /controller\.abort\(\)/.test(branch.slice(0, 400))
79
+ expect(leadsToStop || feedsThePoller).toBe(true)
80
+ }
81
+ })
82
+
83
+ it('still ends the stream cleanly when a batch says stop', () => {
84
+ expect(engine).toMatch(/streamResult === 'stopped'/)
85
+ })
86
+
87
+ it('still commits the cursor through the run service, batch by batch', () => {
88
+ expect(engine).toMatch(/syncRunService\.commitBatchProgress\(/)
89
+ })
90
+
91
+ it('still consults the cancellation check only for runs that have a progress job', () => {
92
+ // Recorded because it is a real constraint on the adopter rather than an incidental
93
+ // detail: a run created without a progress job has no batch boundary to stop at, so the
94
+ // start path must always create one.
95
+ expect(engine).toMatch(/run\.progressJobId && await progressService\.isCancellationRequested/)
96
+ })
97
+ })
@@ -0,0 +1,231 @@
1
+ // How a `data_sync` run becomes durable work without core's engine being changed or copied.
2
+ //
3
+ // Core's engine already survives being displaced by another worker: it asks a cancellation
4
+ // question at every batch boundary, and it stays silent when a terminal write is refused. A
5
+ // durable adopter is exactly a worker in that position, so all this does is answer those two
6
+ // questions differently — and route the cursor commit through the lease fence.
7
+ //
8
+ // See docs/adr/0004: the original plan was to fork the batch loop. Reading the engine showed
9
+ // the seams were already there.
10
+
11
+ import type { SliceContext, SliceOutcome, SqlExecutor } from '@fullstackhouse/open-mercato-durable-work'
12
+
13
+ import { SeamBrokenError } from '../modules/data_sync/lib/version-guard'
14
+
15
+ export type SyncScope = { tenantId: string; organizationId: string | null; userId?: string | null }
16
+ export type SyncTerminalStatus = 'completed' | 'failed' | 'cancelled'
17
+
18
+ /** The methods this decorator *wraps*, not the whole service: core's engine calls plenty more,
19
+ * and they are passed through untouched. Anything not listed here is delegated. */
20
+ export type SyncRunServiceLike = {
21
+ [method: string]: unknown
22
+ getRun(runId: string, scope: SyncScope): Promise<{ status: string; progressJobId?: string | null } | null>
23
+ markStatus(runId: string, status: string, scope: SyncScope, error?: string): Promise<unknown>
24
+ commitBatchProgress(
25
+ runId: string,
26
+ delta: Record<string, unknown>,
27
+ cursor: unknown,
28
+ scope: SyncScope,
29
+ options?: Record<string, unknown>,
30
+ ): Promise<unknown>
31
+ }
32
+
33
+ export type ProgressServiceLike = {
34
+ [method: string]: unknown
35
+ isCancellationRequested(progressJobId: string, tenantId: string, organizationId: string | null): Promise<boolean>
36
+ }
37
+
38
+ /** What core's engine tried to do, captured rather than applied. */
39
+ export type CapturedOutcome = { status: SyncTerminalStatus; error?: string } | null
40
+
41
+ export type SliceRecorder = {
42
+ runService: SyncRunServiceLike
43
+ progressService: ProgressServiceLike
44
+ /** What the engine tried to finalize the run as, if anything. */
45
+ captured(): CapturedOutcome
46
+ /** Why the slice stopped early, when it did. */
47
+ stopReason(): 'budget' | 'cancelled' | null
48
+ /** Batches whose cursor this slice committed. Zero means the slice made no progress. */
49
+ committedBatches(): number
50
+ }
51
+
52
+ /**
53
+ * Wraps core's two services for the duration of one slice.
54
+ *
55
+ * @param ctx the slice this recorder belongs to; its lease is what fences the cursor commits
56
+ */
57
+ export function recordSlice(
58
+ ctx: SliceContext,
59
+ runService: SyncRunServiceLike,
60
+ progressService: ProgressServiceLike,
61
+ ): SliceRecorder {
62
+ let captured: CapturedOutcome = null
63
+ let stopReason: 'budget' | 'cancelled' | null = null
64
+ let committed = 0
65
+
66
+ return {
67
+ captured: () => captured,
68
+ stopReason: () => stopReason,
69
+ committedBatches: () => committed,
70
+
71
+ // Spread, not rebuilt.
72
+ //
73
+ // Core's engine calls far more than the three methods decorated here — `startJob`,
74
+ // `updateProgress`, `touchJobHeartbeat`, `markCancelled` and others on the progress
75
+ // service, and more of the run service besides. An object carrying only the overrides
76
+ // looks fine to TypeScript through a structural type and then fails at the first
77
+ // undecorated call, mid-run.
78
+ runService: {
79
+ ...runService,
80
+
81
+ /**
82
+ * Records a terminal transition instead of performing it, and answers with the run
83
+ * unchanged.
84
+ *
85
+ * Core's `finalizeRun` compares what comes back to what it asked for, and stays silent
86
+ * when they differ — the branch it has for "another worker already finalized this". So
87
+ * this both captures the outcome and suppresses the progress write, the operational log
88
+ * and the lifecycle event, leaving all three to the durable terminal transition, which
89
+ * writes them in the same transaction as the job's own terminal state.
90
+ */
91
+ async markStatus(runId, status, scope, error) {
92
+ if (status === 'completed' || status === 'failed' || status === 'cancelled') {
93
+ captured = { status, error }
94
+ return runService.getRun(runId, scope)
95
+ }
96
+ return runService.markStatus(runId, status, scope, error)
97
+ },
98
+
99
+ /**
100
+ * Commits the batch cursor under the lease.
101
+ *
102
+ * This is the write that must not outlive the right to make it: a worker whose lease
103
+ * expired mid-batch would otherwise advance the cursor of a run another worker is now
104
+ * driving, and the two would interleave over one stream.
105
+ */
106
+ async commitBatchProgress(runId, delta, cursor, scope, options) {
107
+ const result = await ctx.fencedWrite(async () => runService.commitBatchProgress(runId, delta, cursor, scope, options))
108
+ committed += 1
109
+ // A committed unit of work resets the failure and orphan budgets: a run that is making
110
+ // progress has not earned the suspicion those counters represent, however many times
111
+ // it was interrupted getting there.
112
+ await ctx.heartbeat({ committed: true })
113
+ return result
114
+ },
115
+ },
116
+
117
+ progressService: {
118
+ ...progressService,
119
+
120
+ /**
121
+ * Mirrors the run's counters onto the durable job as they move.
122
+ *
123
+ * Core reports progress here after every committed batch. Without this the operator API
124
+ * shows a job that is plainly running with `0 of null` processed, and the one place
125
+ * somebody looks to see whether a multi-day backfill is advancing tells them nothing.
126
+ */
127
+ async updateProgress(progressJobId: string, patch: { processedCount?: number; totalCount?: number | null }, scope: unknown) {
128
+ await ctx
129
+ .heartbeat({ processedCount: patch?.processedCount, totalCount: patch?.totalCount ?? null })
130
+ .catch(() => undefined)
131
+ const inner = progressService.updateProgress as
132
+ | ((id: string, patch: unknown, scope: unknown) => Promise<unknown>)
133
+ | undefined
134
+ return inner?.call(progressService, progressJobId, patch, scope)
135
+ },
136
+
137
+ /**
138
+ * Core asks this once per batch and stops the stream cleanly when it is true. That makes
139
+ * it the slice's hand-back point as well as its cancellation point — the two need the
140
+ * same clean stop, and differ only in what happens afterwards.
141
+ */
142
+ async isCancellationRequested(progressJobId, tenantId, organizationId) {
143
+ if (await progressService.isCancellationRequested(progressJobId, tenantId, organizationId)) {
144
+ stopReason = 'cancelled'
145
+ return true
146
+ }
147
+ if (ctx.signal.aborted || ctx.shouldYield()) {
148
+ stopReason = 'budget'
149
+ return true
150
+ }
151
+ return false
152
+ },
153
+ },
154
+ }
155
+ }
156
+
157
+ /** A run that ended `failed` inside core's engine. Thrown so the durable error taxonomy and the
158
+ * retry budget apply to it, instead of the run being thrown away at the first blip. */
159
+ export class SyncRunFailedError extends Error {
160
+ readonly durableErrorClass = 'transient' as const
161
+ constructor(
162
+ readonly runId: string,
163
+ message: string,
164
+ ) {
165
+ super(message)
166
+ this.name = 'SyncRunFailedError'
167
+ }
168
+ }
169
+
170
+ /**
171
+ * Turns what the recorder saw into the outcome the mechanism understands.
172
+ *
173
+ * @param runStatus the run's status after the slice, used only to tell "already finished" from
174
+ * "core finalized this itself" — see the seam check below
175
+ */
176
+ export function outcomeOf(recorder: SliceRecorder, runId: string, runStatus?: string): SliceOutcome {
177
+ const captured = recorder.captured()
178
+
179
+ if (recorder.stopReason() === 'budget') return 'budget'
180
+ if (recorder.stopReason() === 'cancelled') return 'cancelled'
181
+
182
+ if (!captured) {
183
+ // Nothing was recorded. Two very different situations look the same from here, and telling
184
+ // them apart is the whole point:
185
+ //
186
+ // - the run was already terminal, or gone, before this slice started — nothing to do
187
+ // - core finalized the run itself, without going through the decorated `markStatus`
188
+ //
189
+ // The second means the seam this package rests on has moved (ADR 0004), and reporting it
190
+ // as success would leave a job that says `completed` beside a run that says `failed`.
191
+ // Committed batches are what distinguishes them: a slice that did real work and then found
192
+ // the run terminal without recording anything did not simply arrive late.
193
+ if (recorder.committedBatches() > 0 && runStatus && runStatus !== 'running' && runStatus !== 'pending') {
194
+ throw new SeamBrokenError(runId, `the run reached "${runStatus}" without the adopter recording it`)
195
+ }
196
+ return 'drained'
197
+ }
198
+ if (captured.status === 'failed') throw new SyncRunFailedError(runId, captured.error ?? 'Sync run failed')
199
+ if (captured.status === 'cancelled') return 'cancelled'
200
+ return 'drained'
201
+ }
202
+
203
+ /** Maps a durable terminal state onto the run's own, inside the terminal transaction. */
204
+ export async function mirrorRunStatus(
205
+ tx: SqlExecutor,
206
+ runId: string,
207
+ status: SyncTerminalStatus,
208
+ errorMessage: string | null,
209
+ ): Promise<{ matched: number }> {
210
+ // Fenced on the run still being open, so a run finished by another path is not overwritten —
211
+ // "mirrored" means the domain row agrees, and a row that already disagrees for a good reason
212
+ // must not be forced.
213
+ const result = await tx.query(
214
+ `update sync_runs
215
+ set status = $2, last_error = $3, updated_at = now()
216
+ where id = $1 and deleted_at is null and status in ('pending','running')`,
217
+ [runId, status, errorMessage],
218
+ )
219
+ return { matched: result.rowCount }
220
+ }
221
+
222
+ /** Re-opens a run when an operator re-drives its job. The mirror image of the above. */
223
+ export async function reopenRun(tx: SqlExecutor, runId: string): Promise<{ matched: number }> {
224
+ const result = await tx.query(
225
+ `update sync_runs
226
+ set status = 'running', last_error = null, updated_at = now()
227
+ where id = $1 and deleted_at is null and status in ('failed','pending','running')`,
228
+ [runId],
229
+ )
230
+ return { matched: result.rowCount }
231
+ }
package/src/index.ts ADDED
@@ -0,0 +1,4 @@
1
+ // Public API of @fullstackhouse/open-mercato-data-sync-durable.
2
+ // The host loads ./modules/data_sync as module id `data_sync` in place of core's:
3
+ // { id: 'data_sync', from: '@fullstackhouse/open-mercato-data-sync-durable' }
4
+ export { metadata } from './modules/data_sync/index'