@elaraai/e3-core 0.0.2-beta.5 → 0.0.2-beta.50

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. package/README.md +25 -22
  2. package/dist/src/dataflow/api-compat.d.ts +90 -0
  3. package/dist/src/dataflow/api-compat.d.ts.map +1 -0
  4. package/dist/src/dataflow/api-compat.js +139 -0
  5. package/dist/src/dataflow/api-compat.js.map +1 -0
  6. package/dist/src/dataflow/index.d.ts +18 -0
  7. package/dist/src/dataflow/index.d.ts.map +1 -0
  8. package/dist/src/dataflow/index.js +23 -0
  9. package/dist/src/dataflow/index.js.map +1 -0
  10. package/dist/src/dataflow/orchestrator/LocalOrchestrator.d.ts +76 -0
  11. package/dist/src/dataflow/orchestrator/LocalOrchestrator.d.ts.map +1 -0
  12. package/dist/src/dataflow/orchestrator/LocalOrchestrator.js +729 -0
  13. package/dist/src/dataflow/orchestrator/LocalOrchestrator.js.map +1 -0
  14. package/dist/src/dataflow/orchestrator/index.d.ts +12 -0
  15. package/dist/src/dataflow/orchestrator/index.d.ts.map +1 -0
  16. package/dist/src/dataflow/orchestrator/index.js +12 -0
  17. package/dist/src/dataflow/orchestrator/index.js.map +1 -0
  18. package/dist/src/dataflow/orchestrator/interfaces.d.ts +163 -0
  19. package/dist/src/dataflow/orchestrator/interfaces.d.ts.map +1 -0
  20. package/dist/src/dataflow/orchestrator/interfaces.js +52 -0
  21. package/dist/src/dataflow/orchestrator/interfaces.js.map +1 -0
  22. package/dist/src/dataflow/state-store/FileStateStore.d.ts +67 -0
  23. package/dist/src/dataflow/state-store/FileStateStore.d.ts.map +1 -0
  24. package/dist/src/dataflow/state-store/FileStateStore.js +300 -0
  25. package/dist/src/dataflow/state-store/FileStateStore.js.map +1 -0
  26. package/dist/src/dataflow/state-store/InMemoryStateStore.d.ts +42 -0
  27. package/dist/src/dataflow/state-store/InMemoryStateStore.d.ts.map +1 -0
  28. package/dist/src/dataflow/state-store/InMemoryStateStore.js +229 -0
  29. package/dist/src/dataflow/state-store/InMemoryStateStore.js.map +1 -0
  30. package/dist/src/dataflow/state-store/index.d.ts +13 -0
  31. package/dist/src/dataflow/state-store/index.d.ts.map +1 -0
  32. package/dist/src/dataflow/state-store/index.js +13 -0
  33. package/dist/src/dataflow/state-store/index.js.map +1 -0
  34. package/dist/src/dataflow/state-store/interfaces.d.ts +159 -0
  35. package/dist/src/dataflow/state-store/interfaces.d.ts.map +1 -0
  36. package/dist/src/dataflow/state-store/interfaces.js +6 -0
  37. package/dist/src/dataflow/state-store/interfaces.js.map +1 -0
  38. package/dist/src/dataflow/steps.d.ts +222 -0
  39. package/dist/src/dataflow/steps.d.ts.map +1 -0
  40. package/dist/src/dataflow/steps.js +707 -0
  41. package/dist/src/dataflow/steps.js.map +1 -0
  42. package/dist/src/dataflow/types.d.ts +127 -0
  43. package/dist/src/dataflow/types.d.ts.map +1 -0
  44. package/dist/src/dataflow/types.js +7 -0
  45. package/dist/src/dataflow/types.js.map +1 -0
  46. package/dist/src/dataflow.d.ts +113 -38
  47. package/dist/src/dataflow.d.ts.map +1 -1
  48. package/dist/src/dataflow.js +269 -416
  49. package/dist/src/dataflow.js.map +1 -1
  50. package/dist/src/dataset-refs.d.ts +124 -0
  51. package/dist/src/dataset-refs.d.ts.map +1 -0
  52. package/dist/src/dataset-refs.js +319 -0
  53. package/dist/src/dataset-refs.js.map +1 -0
  54. package/dist/src/errors.d.ts +39 -9
  55. package/dist/src/errors.d.ts.map +1 -1
  56. package/dist/src/errors.js +51 -8
  57. package/dist/src/errors.js.map +1 -1
  58. package/dist/src/execution/LocalTaskRunner.d.ts +73 -0
  59. package/dist/src/execution/LocalTaskRunner.d.ts.map +1 -0
  60. package/dist/src/execution/LocalTaskRunner.js +399 -0
  61. package/dist/src/execution/LocalTaskRunner.js.map +1 -0
  62. package/dist/src/execution/MockTaskRunner.d.ts +49 -0
  63. package/dist/src/execution/MockTaskRunner.d.ts.map +1 -0
  64. package/dist/src/execution/MockTaskRunner.js +54 -0
  65. package/dist/src/execution/MockTaskRunner.js.map +1 -0
  66. package/dist/src/execution/index.d.ts +16 -0
  67. package/dist/src/execution/index.d.ts.map +1 -0
  68. package/dist/src/execution/index.js +8 -0
  69. package/dist/src/execution/index.js.map +1 -0
  70. package/dist/src/execution/interfaces.d.ts +246 -0
  71. package/dist/src/execution/interfaces.d.ts.map +1 -0
  72. package/dist/src/execution/interfaces.js +6 -0
  73. package/dist/src/execution/interfaces.js.map +1 -0
  74. package/dist/src/execution/processHelpers.d.ts +20 -0
  75. package/dist/src/execution/processHelpers.d.ts.map +1 -0
  76. package/dist/src/execution/processHelpers.js +62 -0
  77. package/dist/src/execution/processHelpers.js.map +1 -0
  78. package/dist/src/executions.d.ts +71 -104
  79. package/dist/src/executions.d.ts.map +1 -1
  80. package/dist/src/executions.js +110 -476
  81. package/dist/src/executions.js.map +1 -1
  82. package/dist/src/index.d.ts +20 -10
  83. package/dist/src/index.d.ts.map +1 -1
  84. package/dist/src/index.js +48 -18
  85. package/dist/src/index.js.map +1 -1
  86. package/dist/src/objects.d.ts +7 -53
  87. package/dist/src/objects.d.ts.map +1 -1
  88. package/dist/src/objects.js +13 -232
  89. package/dist/src/objects.js.map +1 -1
  90. package/dist/src/packages.d.ts +41 -14
  91. package/dist/src/packages.d.ts.map +1 -1
  92. package/dist/src/packages.js +145 -88
  93. package/dist/src/packages.js.map +1 -1
  94. package/dist/src/storage/in-memory/InMemoryRepoStore.d.ts +35 -0
  95. package/dist/src/storage/in-memory/InMemoryRepoStore.d.ts.map +1 -0
  96. package/dist/src/storage/in-memory/InMemoryRepoStore.js +107 -0
  97. package/dist/src/storage/in-memory/InMemoryRepoStore.js.map +1 -0
  98. package/dist/src/storage/in-memory/InMemoryStorage.d.ts +139 -0
  99. package/dist/src/storage/in-memory/InMemoryStorage.d.ts.map +1 -0
  100. package/dist/src/storage/in-memory/InMemoryStorage.js +439 -0
  101. package/dist/src/storage/in-memory/InMemoryStorage.js.map +1 -0
  102. package/dist/src/storage/in-memory/index.d.ts +12 -0
  103. package/dist/src/storage/in-memory/index.d.ts.map +1 -0
  104. package/dist/src/storage/in-memory/index.js +12 -0
  105. package/dist/src/storage/in-memory/index.js.map +1 -0
  106. package/dist/src/storage/index.d.ts +18 -0
  107. package/dist/src/storage/index.d.ts.map +1 -0
  108. package/dist/src/storage/index.js +10 -0
  109. package/dist/src/storage/index.js.map +1 -0
  110. package/dist/src/storage/interfaces.d.ts +581 -0
  111. package/dist/src/storage/interfaces.d.ts.map +1 -0
  112. package/dist/src/storage/interfaces.js +6 -0
  113. package/dist/src/storage/interfaces.js.map +1 -0
  114. package/dist/src/storage/local/LocalBackend.d.ts +56 -0
  115. package/dist/src/storage/local/LocalBackend.d.ts.map +1 -0
  116. package/dist/src/storage/local/LocalBackend.js +145 -0
  117. package/dist/src/storage/local/LocalBackend.js.map +1 -0
  118. package/dist/src/storage/local/LocalDatasetRefStore.d.ts +22 -0
  119. package/dist/src/storage/local/LocalDatasetRefStore.d.ts.map +1 -0
  120. package/dist/src/storage/local/LocalDatasetRefStore.js +118 -0
  121. package/dist/src/storage/local/LocalDatasetRefStore.js.map +1 -0
  122. package/dist/src/storage/local/LocalLockService.d.ts +111 -0
  123. package/dist/src/storage/local/LocalLockService.d.ts.map +1 -0
  124. package/dist/src/storage/local/LocalLockService.js +364 -0
  125. package/dist/src/storage/local/LocalLockService.js.map +1 -0
  126. package/dist/src/storage/local/LocalLogStore.d.ts +23 -0
  127. package/dist/src/storage/local/LocalLogStore.d.ts.map +1 -0
  128. package/dist/src/storage/local/LocalLogStore.js +66 -0
  129. package/dist/src/storage/local/LocalLogStore.js.map +1 -0
  130. package/dist/src/storage/local/LocalObjectStore.d.ts +55 -0
  131. package/dist/src/storage/local/LocalObjectStore.d.ts.map +1 -0
  132. package/dist/src/storage/local/LocalObjectStore.js +300 -0
  133. package/dist/src/storage/local/LocalObjectStore.js.map +1 -0
  134. package/dist/src/storage/local/LocalRefStore.d.ts +50 -0
  135. package/dist/src/storage/local/LocalRefStore.d.ts.map +1 -0
  136. package/dist/src/storage/local/LocalRefStore.js +337 -0
  137. package/dist/src/storage/local/LocalRefStore.js.map +1 -0
  138. package/dist/src/storage/local/LocalRepoStore.d.ts +55 -0
  139. package/dist/src/storage/local/LocalRepoStore.d.ts.map +1 -0
  140. package/dist/src/storage/local/LocalRepoStore.js +365 -0
  141. package/dist/src/storage/local/LocalRepoStore.js.map +1 -0
  142. package/dist/src/storage/local/gc.d.ts +92 -0
  143. package/dist/src/storage/local/gc.d.ts.map +1 -0
  144. package/dist/src/storage/local/gc.js +377 -0
  145. package/dist/src/storage/local/gc.js.map +1 -0
  146. package/dist/src/storage/local/index.d.ts +18 -0
  147. package/dist/src/storage/local/index.d.ts.map +1 -0
  148. package/dist/src/storage/local/index.js +18 -0
  149. package/dist/src/storage/local/index.js.map +1 -0
  150. package/dist/src/storage/local/localHelpers.d.ts +25 -0
  151. package/dist/src/storage/local/localHelpers.d.ts.map +1 -0
  152. package/dist/src/storage/local/localHelpers.js +69 -0
  153. package/dist/src/storage/local/localHelpers.js.map +1 -0
  154. package/dist/src/{repository.d.ts → storage/local/repository.d.ts} +8 -4
  155. package/dist/src/storage/local/repository.d.ts.map +1 -0
  156. package/dist/src/{repository.js → storage/local/repository.js} +31 -29
  157. package/dist/src/storage/local/repository.js.map +1 -0
  158. package/dist/src/tasks.d.ts +16 -10
  159. package/dist/src/tasks.d.ts.map +1 -1
  160. package/dist/src/tasks.js +35 -41
  161. package/dist/src/tasks.js.map +1 -1
  162. package/dist/src/test-helpers.d.ts +5 -4
  163. package/dist/src/test-helpers.d.ts.map +1 -1
  164. package/dist/src/test-helpers.js +9 -21
  165. package/dist/src/test-helpers.js.map +1 -1
  166. package/dist/src/transfer/InMemoryTransferBackend.d.ts +75 -0
  167. package/dist/src/transfer/InMemoryTransferBackend.d.ts.map +1 -0
  168. package/dist/src/transfer/InMemoryTransferBackend.js +211 -0
  169. package/dist/src/transfer/InMemoryTransferBackend.js.map +1 -0
  170. package/dist/src/transfer/index.d.ts +9 -0
  171. package/dist/src/transfer/index.d.ts.map +1 -0
  172. package/dist/src/transfer/index.js +11 -0
  173. package/dist/src/transfer/index.js.map +1 -0
  174. package/dist/src/transfer/interfaces.d.ts +103 -0
  175. package/dist/src/transfer/interfaces.d.ts.map +1 -0
  176. package/dist/src/transfer/interfaces.js +6 -0
  177. package/dist/src/transfer/interfaces.js.map +1 -0
  178. package/dist/src/transfer/process.d.ts +55 -0
  179. package/dist/src/transfer/process.d.ts.map +1 -0
  180. package/dist/src/transfer/process.js +144 -0
  181. package/dist/src/transfer/process.js.map +1 -0
  182. package/dist/src/transfer/types.d.ts +106 -0
  183. package/dist/src/transfer/types.d.ts.map +1 -0
  184. package/dist/src/transfer/types.js +61 -0
  185. package/dist/src/transfer/types.js.map +1 -0
  186. package/dist/src/trees.d.ts +147 -59
  187. package/dist/src/trees.d.ts.map +1 -1
  188. package/dist/src/trees.js +372 -419
  189. package/dist/src/trees.js.map +1 -1
  190. package/dist/src/uuid.d.ts +26 -0
  191. package/dist/src/uuid.d.ts.map +1 -0
  192. package/dist/src/uuid.js +80 -0
  193. package/dist/src/uuid.js.map +1 -0
  194. package/dist/src/workspaceStatus.d.ts +6 -4
  195. package/dist/src/workspaceStatus.d.ts.map +1 -1
  196. package/dist/src/workspaceStatus.js +46 -60
  197. package/dist/src/workspaceStatus.js.map +1 -1
  198. package/dist/src/workspaces.d.ts +46 -47
  199. package/dist/src/workspaces.d.ts.map +1 -1
  200. package/dist/src/workspaces.js +281 -221
  201. package/dist/src/workspaces.js.map +1 -1
  202. package/package.json +4 -4
  203. package/dist/src/gc.d.ts +0 -54
  204. package/dist/src/gc.d.ts.map +0 -1
  205. package/dist/src/gc.js +0 -233
  206. package/dist/src/gc.js.map +0 -1
  207. package/dist/src/repository.d.ts.map +0 -1
  208. package/dist/src/repository.js.map +0 -1
  209. package/dist/src/workspaceLock.d.ts +0 -67
  210. package/dist/src/workspaceLock.d.ts.map +0 -1
  211. package/dist/src/workspaceLock.js +0 -217
  212. package/dist/src/workspaceLock.js.map +0 -1
@@ -0,0 +1,211 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Licensed under BSL 1.1. See LICENSE for details.
4
+ */
5
+ /**
6
+ * In-memory TransferBackend implementation.
7
+ *
8
+ * Stores transfer records in memory Maps. When `storage` and `getRepoPath`
9
+ * are provided, `execute()` performs real background processing via the
10
+ * shared handlers. Without them, falls back to mock behavior for tests.
11
+ */
12
+ /* eslint-disable @typescript-eslint/require-await */
13
+ import { randomUUID } from 'node:crypto';
14
+ import { mkdir } from 'node:fs/promises';
15
+ import { join } from 'node:path';
16
+ import { tmpdir } from 'node:os';
17
+ import { variant } from '@elaraai/east';
18
+ import { handleProcessExport, handleProcessImport } from './process.js';
19
+ const STAGING_DIR = join(tmpdir(), 'e3-transfers');
20
+ // =============================================================================
21
+ // Dataset Upload
22
+ // =============================================================================
23
+ class InMemoryDatasetUploadStore {
24
+ baseUrl;
25
+ records = new Map();
26
+ constructor(baseUrl) {
27
+ this.baseUrl = baseUrl;
28
+ }
29
+ async create(id, record) {
30
+ this.records.set(id, record);
31
+ }
32
+ async get(id) {
33
+ return this.records.get(id) ?? null;
34
+ }
35
+ async delete(id) {
36
+ this.records.delete(id);
37
+ }
38
+ async getUploadUrl(id, _repo, _hash) {
39
+ return `${this.baseUrl}/api/uploads/${id}`;
40
+ }
41
+ async commitObject(_repo, _hash, uploadId) {
42
+ // Mock — just remove the record. Real verification happens in integration tests.
43
+ this.records.delete(uploadId);
44
+ }
45
+ clear() {
46
+ this.records.clear();
47
+ }
48
+ }
49
+ // =============================================================================
50
+ // Dataset Download
51
+ // =============================================================================
52
+ class InMemoryDatasetDownloadStore {
53
+ baseUrl;
54
+ records = new Map();
55
+ constructor(baseUrl) {
56
+ this.baseUrl = baseUrl;
57
+ }
58
+ async getDownloadUrl(repo, hash) {
59
+ const id = randomUUID();
60
+ this.records.set(id, { repo, hash });
61
+ return `${this.baseUrl}/api/downloads/${id}`;
62
+ }
63
+ async get(id) {
64
+ return this.records.get(id) ?? null;
65
+ }
66
+ async delete(id) {
67
+ this.records.delete(id);
68
+ }
69
+ clear() {
70
+ this.records.clear();
71
+ }
72
+ }
73
+ // =============================================================================
74
+ // Package Import
75
+ // =============================================================================
76
+ class InMemoryPackageImportStore {
77
+ baseUrl;
78
+ storage;
79
+ getRepoPath;
80
+ records = new Map();
81
+ executing = new Set();
82
+ constructor(baseUrl, storage, getRepoPath) {
83
+ this.baseUrl = baseUrl;
84
+ this.storage = storage;
85
+ this.getRepoPath = getRepoPath;
86
+ }
87
+ async create(id, record) {
88
+ this.records.set(id, record);
89
+ }
90
+ async get(id) {
91
+ return this.records.get(id) ?? null;
92
+ }
93
+ async updateStatus(id, status) {
94
+ const record = this.records.get(id);
95
+ if (!record)
96
+ throw new Error(`Package import ${id} not found`);
97
+ this.records.set(id, { ...record, status });
98
+ }
99
+ async delete(id) {
100
+ this.records.delete(id);
101
+ }
102
+ async getUploadUrl(id, _repo) {
103
+ return `${this.baseUrl}/api/uploads/${id}`;
104
+ }
105
+ async execute(id, repo) {
106
+ const record = this.records.get(id);
107
+ if (!record)
108
+ throw new Error(`Package import ${id} not found`);
109
+ if (this.executing.has(id))
110
+ return;
111
+ this.executing.add(id);
112
+ if (!this.storage || !this.getRepoPath) {
113
+ // Mock fallback for tests that don't provide storage
114
+ await this.updateStatus(id, variant('completed', {
115
+ name: 'mock',
116
+ version: '0.0.0',
117
+ packageHash: 'mock',
118
+ objectCount: 0n,
119
+ }));
120
+ this.executing.delete(id);
121
+ return;
122
+ }
123
+ const zipPath = join(STAGING_DIR, `${id}.zip.partial`);
124
+ await mkdir(STAGING_DIR, { recursive: true });
125
+ void handleProcessImport({ storage: this.storage, importStore: this }, { id, repo: this.getRepoPath(repo), zipPath }).catch(() => {
126
+ // Error already recorded in job status by handleProcessImport
127
+ }).finally(() => {
128
+ this.executing.delete(id);
129
+ });
130
+ }
131
+ clear() {
132
+ this.records.clear();
133
+ }
134
+ }
135
+ // =============================================================================
136
+ // Package Export
137
+ // =============================================================================
138
+ class InMemoryPackageExportStore {
139
+ baseUrl;
140
+ storage;
141
+ getRepoPath;
142
+ records = new Map();
143
+ executing = new Set();
144
+ constructor(baseUrl, storage, getRepoPath) {
145
+ this.baseUrl = baseUrl;
146
+ this.storage = storage;
147
+ this.getRepoPath = getRepoPath;
148
+ }
149
+ async create(id, record) {
150
+ this.records.set(id, record);
151
+ }
152
+ async get(id) {
153
+ return this.records.get(id) ?? null;
154
+ }
155
+ async updateStatus(id, status) {
156
+ const record = this.records.get(id);
157
+ if (!record)
158
+ throw new Error(`Package export ${id} not found`);
159
+ this.records.set(id, { ...record, status });
160
+ }
161
+ async delete(id) {
162
+ this.records.delete(id);
163
+ }
164
+ async getDownloadUrl(id, _repo) {
165
+ return `${this.baseUrl}/api/downloads/${id}`;
166
+ }
167
+ async execute(id, repo) {
168
+ const record = this.records.get(id);
169
+ if (!record)
170
+ throw new Error(`Package export ${id} not found`);
171
+ if (this.executing.has(id))
172
+ return;
173
+ this.executing.add(id);
174
+ if (!this.storage || !this.getRepoPath) {
175
+ // Mock fallback for tests that don't provide storage
176
+ await this.updateStatus(id, variant('completed', { size: 0n }));
177
+ this.executing.delete(id);
178
+ return;
179
+ }
180
+ const zipPath = join(STAGING_DIR, `${id}.zip`);
181
+ await mkdir(STAGING_DIR, { recursive: true });
182
+ void handleProcessExport({ storage: this.storage, exportStore: this }, { id, repo: this.getRepoPath(repo), zipPath }).catch(() => {
183
+ // Error already recorded in job status by handleProcessExport
184
+ }).finally(() => {
185
+ this.executing.delete(id);
186
+ });
187
+ }
188
+ clear() {
189
+ this.records.clear();
190
+ }
191
+ }
192
+ export class InMemoryTransferBackend {
193
+ datasetUpload;
194
+ datasetDownload;
195
+ packageImport;
196
+ packageExport;
197
+ constructor(options) {
198
+ const baseUrl = options.baseUrl ?? '';
199
+ this.datasetUpload = new InMemoryDatasetUploadStore(baseUrl);
200
+ this.datasetDownload = new InMemoryDatasetDownloadStore(baseUrl);
201
+ this.packageImport = new InMemoryPackageImportStore(baseUrl, options.storage, options.getRepoPath);
202
+ this.packageExport = new InMemoryPackageExportStore(baseUrl, options.storage, options.getRepoPath);
203
+ }
204
+ clear() {
205
+ this.datasetUpload.clear();
206
+ this.datasetDownload.clear();
207
+ this.packageImport.clear();
208
+ this.packageExport.clear();
209
+ }
210
+ }
211
+ //# sourceMappingURL=InMemoryTransferBackend.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"InMemoryTransferBackend.js","sourceRoot":"","sources":["../../../src/transfer/InMemoryTransferBackend.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;GAMG;AAEH,qDAAqD;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAWxC,OAAO,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAExE,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,cAAc,CAAC,CAAC;AAEnD,gFAAgF;AAChF,iBAAiB;AACjB,gFAAgF;AAEhF,MAAM,0BAA0B;IAGD;IAFZ,OAAO,GAAG,IAAI,GAAG,EAAyB,CAAC;IAE5D,YAA6B,OAAe;QAAf,YAAO,GAAP,OAAO,CAAQ;IAAG,CAAC;IAEhD,KAAK,CAAC,MAAM,CAAC,EAAU,EAAE,MAAqB;QAC5C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EAAU;QAClB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAU;QACrB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,EAAU,EAAE,KAAa,EAAE,KAAa;QACzD,OAAO,GAAG,IAAI,CAAC,OAAO,gBAAgB,EAAE,EAAE,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,KAAa,EAAE,QAAgB;QAC/D,iFAAiF;QACjF,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IAED,KAAK;QACH,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;CACF;AAED,gFAAgF;AAChF,mBAAmB;AACnB,gFAAgF;AAEhF,MAAM,4BAA4B;IAGH;IAFZ,OAAO,GAAG,IAAI,GAAG,EAA0C,CAAC;IAE7E,YAA6B,OAAe;QAAf,YAAO,GAAP,OAAO,CAAQ;IAAG,CAAC;IAEhD,KAAK,CAAC,cAAc,CAAC,IAAY,EAAE,IAAY;QAC7C,MAAM,EAAE,GAAG,UAAU,EAAE,CAAC;QACxB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QACrC,OAAO,GAAG,IAAI,CAAC,OAAO,kBAAkB,EAAE,EAAE,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EAAU;QAClB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAU;QACrB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC1B,CAAC;IAED,KAAK;QACH,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;CACF;AAED,gFAAgF;AAChF,iBAAiB;AACjB,gFAAgF;AAEhF,MAAM,0BAA0B;IAKX;IACA;IACA;IANF,OAAO,GAAG,IAAI,GAAG,EAAyB,CAAC;IAC3C,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/C,YACmB,OAAe,EACf,OAAwB,EACxB,WAAsC;QAFtC,YAAO,GAAP,OAAO,CAAQ;QACf,YAAO,GAAP,OAAO,CAAiB;QACxB,gBAAW,GAAX,WAAW,CAA2B;IACtD,CAAC;IAEJ,KAAK,CAAC,MAAM,CAAC,EAAU,EAAE,MAAqB;QAC5C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EAAU;QAClB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,EAAU,EAAE,MAA+B;QAC5D,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACpC,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,EAAE,YAAY,CAAC,CAAC;QAC/D,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAU;QACrB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,EAAU,EAAE,KAAa;QAC1C,OAAO,GAAG,IAAI,CAAC,OAAO,gBAAgB,EAAE,EAAE,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,EAAU,EAAE,IAAY;QACpC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACpC,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,EAAE,YAAY,CAAC,CAAC;QAE/D,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;YAAE,OAAO;QACnC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAEvB,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACvC,qDAAqD;YACrD,MAAM,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,OAAO,CAAC,WAAW,EAAE;gBAC/C,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,OAAO;gBAChB,WAAW,EAAE,MAAM;gBACnB,WAAW,EAAE,EAAE;aAChB,CAAC,CAAC,CAAC;YACJ,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAC1B,OAAO;QACT,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,EAAE,GAAG,EAAE,cAAc,CAAC,CAAC;QACvD,MAAM,KAAK,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,KAAK,mBAAmB,CACtB,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,EAC5C,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,CAC9C,CAAC,KAAK,CAAC,GAAG,EAAE;YACX,8DAA8D;QAChE,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE;YACd,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC5B,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK;QACH,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;CACF;AAED,gFAAgF;AAChF,iBAAiB;AACjB,gFAAgF;AAEhF,MAAM,0BAA0B;IAKX;IACA;IACA;IANF,OAAO,GAAG,IAAI,GAAG,EAAyB,CAAC;IAC3C,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/C,YACmB,OAAe,EACf,OAAwB,EACxB,WAAsC;QAFtC,YAAO,GAAP,OAAO,CAAQ;QACf,YAAO,GAAP,OAAO,CAAiB;QACxB,gBAAW,GAAX,WAAW,CAA2B;IACtD,CAAC;IAEJ,KAAK,CAAC,MAAM,CAAC,EAAU,EAAE,MAAqB;QAC5C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EAAU;QAClB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,EAAU,EAAE,MAA+B;QAC5D,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACpC,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,EAAE,YAAY,CAAC,CAAC;QAC/D,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAU;QACrB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,EAAU,EAAE,KAAa;QAC5C,OAAO,GAAG,IAAI,CAAC,OAAO,kBAAkB,EAAE,EAAE,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,EAAU,EAAE,IAAY;QACpC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACpC,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,EAAE,YAAY,CAAC,CAAC;QAE/D,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;YAAE,OAAO;QACnC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAEvB,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACvC,qDAAqD;YACrD,MAAM,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,OAAO,CAAC,WAAW,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;YAChE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAC1B,OAAO;QACT,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;QAC/C,MAAM,KAAK,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,KAAK,mBAAmB,CACtB,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,EAC5C,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,CAC9C,CAAC,KAAK,CAAC,GAAG,EAAE;YACX,8DAA8D;QAChE,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE;YACd,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC5B,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK;QACH,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;CACF;AAYD,MAAM,OAAO,uBAAuB;IACzB,aAAa,CAA6B;IAC1C,eAAe,CAA+B;IAC9C,aAAa,CAA6B;IAC1C,aAAa,CAA6B;IAEnD,YAAY,OAAuC;QACjD,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,aAAa,GAAG,IAAI,0BAA0B,CAAC,OAAO,CAAC,CAAC;QAC7D,IAAI,CAAC,eAAe,GAAG,IAAI,4BAA4B,CAAC,OAAO,CAAC,CAAC;QACjE,IAAI,CAAC,aAAa,GAAG,IAAI,0BAA0B,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;QACnG,IAAI,CAAC,aAAa,GAAG,IAAI,0BAA0B,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;IACrG,CAAC;IAED,KAAK;QACH,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC7B,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;IAC7B,CAAC;CACF"}
@@ -0,0 +1,9 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Licensed under BSL 1.1. See LICENSE for details.
4
+ */
5
+ export { DatasetUploadType, type DatasetUpload, PackageImportType, PackageImportProgressType, PackageImportStatusType, type PackageImport, PackageExportType, PackageExportProgressType, PackageExportStatusType, type PackageExport, } from './types.js';
6
+ export { type DatasetUploadStore, type DatasetDownloadStore, type PackageImportStore, type PackageExportStore, type TransferBackend, } from './interfaces.js';
7
+ export { InMemoryTransferBackend, type InMemoryTransferBackendOptions, } from './InMemoryTransferBackend.js';
8
+ export { handleProcessExport, handleProcessImport, type ProcessExportDeps, type ProcessExportInput, type ProcessImportDeps, type ProcessImportInput, } from './process.js';
9
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/transfer/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EACL,iBAAiB,EACjB,KAAK,aAAa,EAClB,iBAAiB,EACjB,yBAAyB,EACzB,uBAAuB,EACvB,KAAK,aAAa,EAClB,iBAAiB,EACjB,yBAAyB,EACzB,uBAAuB,EACvB,KAAK,aAAa,GACnB,MAAM,YAAY,CAAC;AAGpB,OAAO,EACL,KAAK,kBAAkB,EACvB,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EACvB,KAAK,eAAe,GACrB,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EACL,uBAAuB,EACvB,KAAK,8BAA8B,GACpC,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EACL,mBAAmB,EACnB,mBAAmB,EACnB,KAAK,iBAAiB,EACtB,KAAK,kBAAkB,EACvB,KAAK,iBAAiB,EACtB,KAAK,kBAAkB,GACxB,MAAM,cAAc,CAAC"}
@@ -0,0 +1,11 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Licensed under BSL 1.1. See LICENSE for details.
4
+ */
5
+ // Types
6
+ export { DatasetUploadType, PackageImportType, PackageImportProgressType, PackageImportStatusType, PackageExportType, PackageExportProgressType, PackageExportStatusType, } from './types.js';
7
+ // InMemory implementation
8
+ export { InMemoryTransferBackend, } from './InMemoryTransferBackend.js';
9
+ // Shared processing handlers
10
+ export { handleProcessExport, handleProcessImport, } from './process.js';
11
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/transfer/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,QAAQ;AACR,OAAO,EACL,iBAAiB,EAEjB,iBAAiB,EACjB,yBAAyB,EACzB,uBAAuB,EAEvB,iBAAiB,EACjB,yBAAyB,EACzB,uBAAuB,GAExB,MAAM,YAAY,CAAC;AAWpB,0BAA0B;AAC1B,OAAO,EACL,uBAAuB,GAExB,MAAM,8BAA8B,CAAC;AAEtC,6BAA6B;AAC7B,OAAO,EACL,mBAAmB,EACnB,mBAAmB,GAKpB,MAAM,cAAc,CAAC"}
@@ -0,0 +1,103 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Licensed under BSL 1.1. See LICENSE for details.
4
+ */
5
+ /**
6
+ * Transfer backend interfaces for presigned URL object transfer.
7
+ *
8
+ * TransferBackend is a separate top-level interface (not part of StorageBackend).
9
+ * It provides cloud-agnostic abstractions for uploading/downloading datasets and
10
+ * packages via presigned URLs. Implementations:
11
+ * - InMemoryTransferBackend (local server, tests)
12
+ * - S3DynamoTransferBackend (AWS cloud, future)
13
+ */
14
+ import type { DatasetUpload, PackageImport, PackageExport } from './types.js';
15
+ /**
16
+ * Manages staged dataset uploads.
17
+ *
18
+ * Flow: create → getUploadUrl → (client uploads) → commitObject → delete
19
+ */
20
+ export interface DatasetUploadStore {
21
+ create(id: string, record: DatasetUpload): Promise<void>;
22
+ get(id: string): Promise<DatasetUpload | null>;
23
+ delete(id: string): Promise<void>;
24
+ /**
25
+ * URL the client PUTs bytes to. The upload ID is embedded in the URL
26
+ * so concurrent uploads to the same hash are unambiguous.
27
+ */
28
+ getUploadUrl(id: string, repo: string, hash: string): Promise<string>;
29
+ /**
30
+ * Verify the upload and make the object visible in the catalogue.
31
+ * On success, the object is queryable via storage.objects.read(repo, hash).
32
+ * On failure, throws — caller should clean up the transfer record.
33
+ */
34
+ commitObject(repo: string, hash: string, uploadId: string): Promise<void>;
35
+ }
36
+ /**
37
+ * Generates download URLs for dataset objects.
38
+ *
39
+ * Local: creates temporary records so `/api/downloads/:id` can look up the object.
40
+ * Cloud: returns presigned S3 URLs directly; `get`/`delete` are never called.
41
+ */
42
+ export interface DatasetDownloadStore {
43
+ /** Create a temporary download record and return the URL. */
44
+ getDownloadUrl(repo: string, hash: string): Promise<string>;
45
+ /** Look up a download record by ID (for local server data handlers). */
46
+ get(id: string): Promise<{
47
+ repo: string;
48
+ hash: string;
49
+ } | null>;
50
+ /** Clean up after serving. */
51
+ delete(id: string): Promise<void>;
52
+ }
53
+ /**
54
+ * Manages package import lifecycle: upload zip → process → poll completion.
55
+ *
56
+ * Flow: create → getUploadUrl → (client uploads) → execute → poll get → delete
57
+ */
58
+ export interface PackageImportStore {
59
+ create(id: string, record: PackageImport): Promise<void>;
60
+ get(id: string): Promise<PackageImport | null>;
61
+ updateStatus(id: string, status: PackageImport['status']): Promise<void>;
62
+ delete(id: string): Promise<void>;
63
+ /** URL the client PUTs zip bytes to. */
64
+ getUploadUrl(id: string, repo: string): Promise<string>;
65
+ /**
66
+ * Dispatch processing.
67
+ * Local: calls packageImport() inline, updates status to completed/failed.
68
+ * Cloud: invokes background processor asynchronously.
69
+ */
70
+ execute(id: string, repo: string): Promise<void>;
71
+ }
72
+ /**
73
+ * Manages package export lifecycle: trigger → process → poll → download.
74
+ *
75
+ * Flow: create → execute → poll get → getDownloadUrl → delete
76
+ */
77
+ export interface PackageExportStore {
78
+ create(id: string, record: PackageExport): Promise<void>;
79
+ get(id: string): Promise<PackageExport | null>;
80
+ updateStatus(id: string, status: PackageExport['status']): Promise<void>;
81
+ delete(id: string): Promise<void>;
82
+ /** URL the client GETs zip bytes from. */
83
+ getDownloadUrl(id: string, repo: string): Promise<string>;
84
+ /**
85
+ * Dispatch processing.
86
+ * Local: calls packageExport() inline, updates status to completed/failed.
87
+ * Cloud: invokes background processor asynchronously.
88
+ */
89
+ execute(id: string, repo: string): Promise<void>;
90
+ }
91
+ /**
92
+ * Cloud-agnostic transfer backend for presigned URL object transfer.
93
+ *
94
+ * Separate from StorageBackend — depends on it for actual object/ref operations
95
+ * but has its own lifecycle (staging, jobs, URLs).
96
+ */
97
+ export interface TransferBackend {
98
+ readonly datasetUpload: DatasetUploadStore;
99
+ readonly datasetDownload: DatasetDownloadStore;
100
+ readonly packageImport: PackageImportStore;
101
+ readonly packageExport: PackageExportStore;
102
+ }
103
+ //# sourceMappingURL=interfaces.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"interfaces.d.ts","sourceRoot":"","sources":["../../../src/transfer/interfaces.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAM9E;;;;GAIG;AACH,MAAM,WAAW,kBAAkB;IACjC,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzD,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC;IAC/C,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAElC;;;OAGG;IACH,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAEtE;;;;OAIG;IACH,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3E;AAMD;;;;;GAKG;AACH,MAAM,WAAW,oBAAoB;IACnC,6DAA6D;IAC7D,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC5D,wEAAwE;IACxE,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IAChE,8BAA8B;IAC9B,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACnC;AAMD;;;;GAIG;AACH,MAAM,WAAW,kBAAkB;IACjC,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzD,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC;IAC/C,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzE,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAElC,wCAAwC;IACxC,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAExD;;;;OAIG;IACH,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAClD;AAMD;;;;GAIG;AACH,MAAM,WAAW,kBAAkB;IACjC,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzD,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC;IAC/C,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzE,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAElC,0CAA0C;IAC1C,cAAc,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAE1D;;;;OAIG;IACH,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAClD;AAMD;;;;;GAKG;AACH,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,aAAa,EAAE,kBAAkB,CAAC;IAC3C,QAAQ,CAAC,eAAe,EAAE,oBAAoB,CAAC;IAC/C,QAAQ,CAAC,aAAa,EAAE,kBAAkB,CAAC;IAC3C,QAAQ,CAAC,aAAa,EAAE,kBAAkB,CAAC;CAC5C"}
@@ -0,0 +1,6 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Licensed under BSL 1.1. See LICENSE for details.
4
+ */
5
+ export {};
6
+ //# sourceMappingURL=interfaces.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"interfaces.js","sourceRoot":"","sources":["../../../src/transfer/interfaces.ts"],"names":[],"mappings":"AAAA;;;GAGG"}
@@ -0,0 +1,55 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Licensed under BSL 1.1. See LICENSE for details.
4
+ */
5
+ import type { StorageBackend } from '../storage/index.js';
6
+ import type { PackageExportStore, PackageImportStore } from './interfaces.js';
7
+ /** Dependencies for handleProcessExport. */
8
+ export interface ProcessExportDeps {
9
+ storage: StorageBackend;
10
+ exportStore: PackageExportStore;
11
+ }
12
+ /** Input for handleProcessExport. */
13
+ export interface ProcessExportInput {
14
+ id: string;
15
+ repo: string;
16
+ zipPath: string;
17
+ }
18
+ /**
19
+ * Processes a package or workspace export job.
20
+ *
21
+ * Gets the export record, determines whether this is a package or workspace
22
+ * export (based on the `workspace` field), runs the appropriate export
23
+ * function, and updates the status to completed or failed.
24
+ *
25
+ * @param deps - Storage backend and export store
26
+ * @param input - Job ID, repository path, and output zip path
27
+ *
28
+ * @throws Re-throws errors after updating status to failed and cleaning up
29
+ */
30
+ export declare function handleProcessExport(deps: ProcessExportDeps, input: ProcessExportInput): Promise<void>;
31
+ /** Dependencies for handleProcessImport. */
32
+ export interface ProcessImportDeps {
33
+ storage: StorageBackend;
34
+ importStore: PackageImportStore;
35
+ }
36
+ /** Input for handleProcessImport. */
37
+ export interface ProcessImportInput {
38
+ id: string;
39
+ repo: string;
40
+ zipPath: string;
41
+ }
42
+ /**
43
+ * Processes a package import job.
44
+ *
45
+ * Gets the import record, verifies the file size matches, runs packageImport,
46
+ * and updates the status to completed or failed. Cleans up the staging zip
47
+ * file in all cases.
48
+ *
49
+ * @param deps - Storage backend and import store
50
+ * @param input - Job ID, repository path, and staging zip path
51
+ *
52
+ * @throws Re-throws errors after updating status to failed
53
+ */
54
+ export declare function handleProcessImport(deps: ProcessImportDeps, input: ProcessImportInput): Promise<void>;
55
+ //# sourceMappingURL=process.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"process.d.ts","sourceRoot":"","sources":["../../../src/transfer/process.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAgBH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,KAAK,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AA6C9E,4CAA4C;AAC5C,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,cAAc,CAAC;IACxB,WAAW,EAAE,kBAAkB,CAAC;CACjC;AAED,qCAAqC;AACrC,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,mBAAmB,CACvC,IAAI,EAAE,iBAAiB,EACvB,KAAK,EAAE,kBAAkB,GACxB,OAAO,CAAC,IAAI,CAAC,CAiCf;AAMD,4CAA4C;AAC5C,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,cAAc,CAAC;IACxB,WAAW,EAAE,kBAAkB,CAAC;CACjC;AAED,qCAAqC;AACrC,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,mBAAmB,CACvC,IAAI,EAAE,iBAAiB,EACvB,KAAK,EAAE,kBAAkB,GACxB,OAAO,CAAC,IAAI,CAAC,CAwCf"}
@@ -0,0 +1,144 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Licensed under BSL 1.1. See LICENSE for details.
4
+ */
5
+ /**
6
+ * Shared processing handlers for package import/export.
7
+ *
8
+ * These are cloud-agnostic handlers that perform the actual import/export work.
9
+ * Used by both the local InMemoryTransferBackend and cloud backends
10
+ * (e.g. AWS Lambda/Step Functions).
11
+ */
12
+ import { stat, unlink } from 'node:fs/promises';
13
+ import { variant } from '@elaraai/east';
14
+ import { packageExport } from '../packages.js';
15
+ import { workspaceExport } from '../workspaces.js';
16
+ import { packageImport } from '../packages.js';
17
+ // =============================================================================
18
+ // Throttled progress callback
19
+ // =============================================================================
20
+ /**
21
+ * Creates a progress callback that throttles updates to at most once per interval.
22
+ *
23
+ * @param fn - The function to call with throttled updates
24
+ * @param intervalMs - Minimum interval between calls in milliseconds
25
+ * @returns A throttled version of the progress callback
26
+ */
27
+ function throttledProgress(fn, intervalMs = 1000) {
28
+ let lastCall = 0;
29
+ let pending = null;
30
+ const throttled = async (progress) => {
31
+ const now = Date.now();
32
+ if (now - lastCall >= intervalMs) {
33
+ lastCall = now;
34
+ pending = null;
35
+ await fn(progress);
36
+ }
37
+ else {
38
+ pending = progress;
39
+ }
40
+ };
41
+ throttled.flush = async () => {
42
+ if (pending) {
43
+ await fn(pending);
44
+ pending = null;
45
+ }
46
+ };
47
+ return throttled;
48
+ }
49
+ /**
50
+ * Processes a package or workspace export job.
51
+ *
52
+ * Gets the export record, determines whether this is a package or workspace
53
+ * export (based on the `workspace` field), runs the appropriate export
54
+ * function, and updates the status to completed or failed.
55
+ *
56
+ * @param deps - Storage backend and export store
57
+ * @param input - Job ID, repository path, and output zip path
58
+ *
59
+ * @throws Re-throws errors after updating status to failed and cleaning up
60
+ */
61
+ export async function handleProcessExport(deps, input) {
62
+ const { storage, exportStore } = deps;
63
+ const { id, repo, zipPath } = input;
64
+ const record = await exportStore.get(id);
65
+ if (!record)
66
+ throw new Error(`Export record ${id} not found`);
67
+ const onProgress = throttledProgress(async ({ objectsProcessed }) => {
68
+ await exportStore.updateStatus(id, variant('processing', variant('exporting', { objectsProcessed: BigInt(objectsProcessed) })));
69
+ });
70
+ try {
71
+ if (record.workspace.type === 'some') {
72
+ await workspaceExport(storage, repo, record.workspace.value, zipPath, record.name, record.version, {
73
+ onProgress,
74
+ });
75
+ }
76
+ else {
77
+ await packageExport(storage, repo, record.name, record.version, zipPath, {
78
+ onProgress,
79
+ });
80
+ }
81
+ await onProgress.flush();
82
+ const fileStat = await stat(zipPath);
83
+ await exportStore.updateStatus(id, variant('completed', {
84
+ size: BigInt(fileStat.size),
85
+ }));
86
+ }
87
+ catch (err) {
88
+ await unlink(zipPath).catch(() => { });
89
+ const message = err instanceof Error ? err.message : String(err);
90
+ await exportStore.updateStatus(id, variant('failed', { message }));
91
+ throw err;
92
+ }
93
+ }
94
+ /**
95
+ * Processes a package import job.
96
+ *
97
+ * Gets the import record, verifies the file size matches, runs packageImport,
98
+ * and updates the status to completed or failed. Cleans up the staging zip
99
+ * file in all cases.
100
+ *
101
+ * @param deps - Storage backend and import store
102
+ * @param input - Job ID, repository path, and staging zip path
103
+ *
104
+ * @throws Re-throws errors after updating status to failed
105
+ */
106
+ export async function handleProcessImport(deps, input) {
107
+ const { storage, importStore } = deps;
108
+ const { id, repo, zipPath } = input;
109
+ const record = await importStore.get(id);
110
+ if (!record)
111
+ throw new Error(`Import record ${id} not found`);
112
+ // Verify file size matches
113
+ const fileStat = await stat(zipPath);
114
+ if (BigInt(fileStat.size) !== record.size) {
115
+ const message = `size mismatch: expected ${record.size}, got ${fileStat.size}`;
116
+ await importStore.updateStatus(id, variant('failed', { message }));
117
+ await unlink(zipPath).catch(() => { });
118
+ throw new Error(message);
119
+ }
120
+ try {
121
+ const onProgress = throttledProgress(async ({ objectsProcessed }) => {
122
+ await importStore.updateStatus(id, variant('processing', variant('importing', { objectsProcessed: BigInt(objectsProcessed) })));
123
+ });
124
+ const result = await packageImport(storage, repo, zipPath, {
125
+ onProgress,
126
+ });
127
+ await onProgress.flush();
128
+ await importStore.updateStatus(id, variant('completed', {
129
+ name: result.name,
130
+ version: result.version,
131
+ packageHash: result.packageHash,
132
+ objectCount: BigInt(result.objectCount),
133
+ }));
134
+ }
135
+ catch (err) {
136
+ const message = err instanceof Error ? err.message : String(err);
137
+ await importStore.updateStatus(id, variant('failed', { message }));
138
+ throw err;
139
+ }
140
+ finally {
141
+ await unlink(zipPath).catch(() => { });
142
+ }
143
+ }
144
+ //# sourceMappingURL=process.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"process.js","sourceRoot":"","sources":["../../../src/transfer/process.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;GAMG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAI/C,gFAAgF;AAChF,8BAA8B;AAC9B,gFAAgF;AAEhF;;;;;;GAMG;AACH,SAAS,iBAAiB,CACxB,EAA6D,EAC7D,UAAU,GAAG,IAAI;IAEjB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,OAAO,GAAwC,IAAI,CAAC;IAExD,MAAM,SAAS,GAAG,KAAK,EAAE,QAAsC,EAAE,EAAE;QACjE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,GAAG,GAAG,QAAQ,IAAI,UAAU,EAAE,CAAC;YACjC,QAAQ,GAAG,GAAG,CAAC;YACf,OAAO,GAAG,IAAI,CAAC;YACf,MAAM,EAAE,CAAC,QAAQ,CAAC,CAAC;QACrB,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,QAAQ,CAAC;QACrB,CAAC;IACH,CAAC,CAAC;IAEF,SAAS,CAAC,KAAK,GAAG,KAAK,IAAI,EAAE;QAC3B,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC;YAClB,OAAO,GAAG,IAAI,CAAC;QACjB,CAAC;IACH,CAAC,CAAC;IAEF,OAAO,SAAS,CAAC;AACnB,CAAC;AAmBD;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,IAAuB,EACvB,KAAyB;IAEzB,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,IAAI,CAAC;IACtC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC;IAEpC,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,iBAAiB,EAAE,YAAY,CAAC,CAAC;IAE9D,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,EAAE,EAAE,gBAAgB,EAAE,EAAE,EAAE;QAClE,MAAM,WAAW,CAAC,YAAY,CAAC,EAAE,EAC/B,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC,WAAW,EAAE,EAAE,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACjG,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,IAAI,MAAM,CAAC,SAAS,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACrC,MAAM,eAAe,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,OAAO,EAAE;gBACjG,UAAU;aACX,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,MAAM,aAAa,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE;gBACvE,UAAU;aACX,CAAC,CAAC;QACL,CAAC;QACD,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,CAAC;QACrC,MAAM,WAAW,CAAC,YAAY,CAAC,EAAE,EAAE,OAAO,CAAC,WAAW,EAAE;YACtD,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;SAC5B,CAAC,CAAC,CAAC;IACN,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,MAAM,WAAW,CAAC,YAAY,CAAC,EAAE,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QACnE,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAmBD;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,IAAuB,EACvB,KAAyB;IAEzB,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,IAAI,CAAC;IACtC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC;IAEpC,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,iBAAiB,EAAE,YAAY,CAAC,CAAC;IAE9D,2BAA2B;IAC3B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC;QAC1C,MAAM,OAAO,GAAG,2BAA2B,MAAM,CAAC,IAAI,SAAS,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/E,MAAM,WAAW,CAAC,YAAY,CAAC,EAAE,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QACnE,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,EAAE,EAAE,gBAAgB,EAAE,EAAE,EAAE;YAClE,MAAM,WAAW,CAAC,YAAY,CAAC,EAAE,EAC/B,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC,WAAW,EAAE,EAAE,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACjG,CAAC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE;YACzD,UAAU;SACX,CAAC,CAAC;QAEH,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC;QACzB,MAAM,WAAW,CAAC,YAAY,CAAC,EAAE,EAAE,OAAO,CAAC,WAAW,EAAE;YACtD,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC;SACxC,CAAC,CAAC,CAAC;IACN,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,MAAM,WAAW,CAAC,YAAY,CAAC,EAAE,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QACnE,MAAM,GAAG,CAAC;IACZ,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACxC,CAAC;AACH,CAAC"}