@elaraai/e3-core 0.0.2-beta.4 → 0.0.2-beta.41

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (208) hide show
  1. package/README.md +25 -22
  2. package/dist/src/dataflow/api-compat.d.ts +90 -0
  3. package/dist/src/dataflow/api-compat.d.ts.map +1 -0
  4. package/dist/src/dataflow/api-compat.js +139 -0
  5. package/dist/src/dataflow/api-compat.js.map +1 -0
  6. package/dist/src/dataflow/index.d.ts +18 -0
  7. package/dist/src/dataflow/index.d.ts.map +1 -0
  8. package/dist/src/dataflow/index.js +23 -0
  9. package/dist/src/dataflow/index.js.map +1 -0
  10. package/dist/src/dataflow/orchestrator/LocalOrchestrator.d.ts +76 -0
  11. package/dist/src/dataflow/orchestrator/LocalOrchestrator.d.ts.map +1 -0
  12. package/dist/src/dataflow/orchestrator/LocalOrchestrator.js +695 -0
  13. package/dist/src/dataflow/orchestrator/LocalOrchestrator.js.map +1 -0
  14. package/dist/src/dataflow/orchestrator/index.d.ts +12 -0
  15. package/dist/src/dataflow/orchestrator/index.d.ts.map +1 -0
  16. package/dist/src/dataflow/orchestrator/index.js +12 -0
  17. package/dist/src/dataflow/orchestrator/index.js.map +1 -0
  18. package/dist/src/dataflow/orchestrator/interfaces.d.ts +163 -0
  19. package/dist/src/dataflow/orchestrator/interfaces.d.ts.map +1 -0
  20. package/dist/src/dataflow/orchestrator/interfaces.js +52 -0
  21. package/dist/src/dataflow/orchestrator/interfaces.js.map +1 -0
  22. package/dist/src/dataflow/state-store/FileStateStore.d.ts +67 -0
  23. package/dist/src/dataflow/state-store/FileStateStore.d.ts.map +1 -0
  24. package/dist/src/dataflow/state-store/FileStateStore.js +300 -0
  25. package/dist/src/dataflow/state-store/FileStateStore.js.map +1 -0
  26. package/dist/src/dataflow/state-store/InMemoryStateStore.d.ts +42 -0
  27. package/dist/src/dataflow/state-store/InMemoryStateStore.d.ts.map +1 -0
  28. package/dist/src/dataflow/state-store/InMemoryStateStore.js +229 -0
  29. package/dist/src/dataflow/state-store/InMemoryStateStore.js.map +1 -0
  30. package/dist/src/dataflow/state-store/index.d.ts +13 -0
  31. package/dist/src/dataflow/state-store/index.d.ts.map +1 -0
  32. package/dist/src/dataflow/state-store/index.js +13 -0
  33. package/dist/src/dataflow/state-store/index.js.map +1 -0
  34. package/dist/src/dataflow/state-store/interfaces.d.ts +159 -0
  35. package/dist/src/dataflow/state-store/interfaces.d.ts.map +1 -0
  36. package/dist/src/dataflow/state-store/interfaces.js +6 -0
  37. package/dist/src/dataflow/state-store/interfaces.js.map +1 -0
  38. package/dist/src/dataflow/steps.d.ts +222 -0
  39. package/dist/src/dataflow/steps.d.ts.map +1 -0
  40. package/dist/src/dataflow/steps.js +707 -0
  41. package/dist/src/dataflow/steps.js.map +1 -0
  42. package/dist/src/dataflow/types.d.ts +127 -0
  43. package/dist/src/dataflow/types.d.ts.map +1 -0
  44. package/dist/src/dataflow/types.js +7 -0
  45. package/dist/src/dataflow/types.js.map +1 -0
  46. package/dist/src/dataflow.d.ts +113 -38
  47. package/dist/src/dataflow.d.ts.map +1 -1
  48. package/dist/src/dataflow.js +269 -416
  49. package/dist/src/dataflow.js.map +1 -1
  50. package/dist/src/dataset-refs.d.ts +124 -0
  51. package/dist/src/dataset-refs.d.ts.map +1 -0
  52. package/dist/src/dataset-refs.js +319 -0
  53. package/dist/src/dataset-refs.js.map +1 -0
  54. package/dist/src/errors.d.ts +39 -9
  55. package/dist/src/errors.d.ts.map +1 -1
  56. package/dist/src/errors.js +51 -8
  57. package/dist/src/errors.js.map +1 -1
  58. package/dist/src/execution/LocalTaskRunner.d.ts +73 -0
  59. package/dist/src/execution/LocalTaskRunner.d.ts.map +1 -0
  60. package/dist/src/execution/LocalTaskRunner.js +399 -0
  61. package/dist/src/execution/LocalTaskRunner.js.map +1 -0
  62. package/dist/src/execution/MockTaskRunner.d.ts +49 -0
  63. package/dist/src/execution/MockTaskRunner.d.ts.map +1 -0
  64. package/dist/src/execution/MockTaskRunner.js +54 -0
  65. package/dist/src/execution/MockTaskRunner.js.map +1 -0
  66. package/dist/src/execution/index.d.ts +16 -0
  67. package/dist/src/execution/index.d.ts.map +1 -0
  68. package/dist/src/execution/index.js +8 -0
  69. package/dist/src/execution/index.js.map +1 -0
  70. package/dist/src/execution/interfaces.d.ts +246 -0
  71. package/dist/src/execution/interfaces.d.ts.map +1 -0
  72. package/dist/src/execution/interfaces.js +6 -0
  73. package/dist/src/execution/interfaces.js.map +1 -0
  74. package/dist/src/execution/processHelpers.d.ts +20 -0
  75. package/dist/src/execution/processHelpers.d.ts.map +1 -0
  76. package/dist/src/execution/processHelpers.js +62 -0
  77. package/dist/src/execution/processHelpers.js.map +1 -0
  78. package/dist/src/executions.d.ts +71 -104
  79. package/dist/src/executions.d.ts.map +1 -1
  80. package/dist/src/executions.js +110 -476
  81. package/dist/src/executions.js.map +1 -1
  82. package/dist/src/index.d.ts +19 -9
  83. package/dist/src/index.d.ts.map +1 -1
  84. package/dist/src/index.js +48 -18
  85. package/dist/src/index.js.map +1 -1
  86. package/dist/src/objects.d.ts +8 -51
  87. package/dist/src/objects.d.ts.map +1 -1
  88. package/dist/src/objects.js +13 -230
  89. package/dist/src/objects.js.map +1 -1
  90. package/dist/src/packages.d.ts +22 -14
  91. package/dist/src/packages.d.ts.map +1 -1
  92. package/dist/src/packages.js +134 -88
  93. package/dist/src/packages.js.map +1 -1
  94. package/dist/src/storage/in-memory/InMemoryRepoStore.d.ts +35 -0
  95. package/dist/src/storage/in-memory/InMemoryRepoStore.d.ts.map +1 -0
  96. package/dist/src/storage/in-memory/InMemoryRepoStore.js +107 -0
  97. package/dist/src/storage/in-memory/InMemoryRepoStore.js.map +1 -0
  98. package/dist/src/storage/in-memory/InMemoryStorage.d.ts +139 -0
  99. package/dist/src/storage/in-memory/InMemoryStorage.d.ts.map +1 -0
  100. package/dist/src/storage/in-memory/InMemoryStorage.js +439 -0
  101. package/dist/src/storage/in-memory/InMemoryStorage.js.map +1 -0
  102. package/dist/src/storage/in-memory/index.d.ts +12 -0
  103. package/dist/src/storage/in-memory/index.d.ts.map +1 -0
  104. package/dist/src/storage/in-memory/index.js +12 -0
  105. package/dist/src/storage/in-memory/index.js.map +1 -0
  106. package/dist/src/storage/index.d.ts +18 -0
  107. package/dist/src/storage/index.d.ts.map +1 -0
  108. package/dist/src/storage/index.js +10 -0
  109. package/dist/src/storage/index.js.map +1 -0
  110. package/dist/src/storage/interfaces.d.ts +581 -0
  111. package/dist/src/storage/interfaces.d.ts.map +1 -0
  112. package/dist/src/storage/interfaces.js +6 -0
  113. package/dist/src/storage/interfaces.js.map +1 -0
  114. package/dist/src/storage/local/LocalBackend.d.ts +56 -0
  115. package/dist/src/storage/local/LocalBackend.d.ts.map +1 -0
  116. package/dist/src/storage/local/LocalBackend.js +145 -0
  117. package/dist/src/storage/local/LocalBackend.js.map +1 -0
  118. package/dist/src/storage/local/LocalDatasetRefStore.d.ts +22 -0
  119. package/dist/src/storage/local/LocalDatasetRefStore.d.ts.map +1 -0
  120. package/dist/src/storage/local/LocalDatasetRefStore.js +118 -0
  121. package/dist/src/storage/local/LocalDatasetRefStore.js.map +1 -0
  122. package/dist/src/storage/local/LocalLockService.d.ts +111 -0
  123. package/dist/src/storage/local/LocalLockService.d.ts.map +1 -0
  124. package/dist/src/storage/local/LocalLockService.js +355 -0
  125. package/dist/src/storage/local/LocalLockService.js.map +1 -0
  126. package/dist/src/storage/local/LocalLogStore.d.ts +23 -0
  127. package/dist/src/storage/local/LocalLogStore.d.ts.map +1 -0
  128. package/dist/src/storage/local/LocalLogStore.js +66 -0
  129. package/dist/src/storage/local/LocalLogStore.js.map +1 -0
  130. package/dist/src/storage/local/LocalObjectStore.d.ts +55 -0
  131. package/dist/src/storage/local/LocalObjectStore.d.ts.map +1 -0
  132. package/dist/src/storage/local/LocalObjectStore.js +300 -0
  133. package/dist/src/storage/local/LocalObjectStore.js.map +1 -0
  134. package/dist/src/storage/local/LocalRefStore.d.ts +50 -0
  135. package/dist/src/storage/local/LocalRefStore.d.ts.map +1 -0
  136. package/dist/src/storage/local/LocalRefStore.js +337 -0
  137. package/dist/src/storage/local/LocalRefStore.js.map +1 -0
  138. package/dist/src/storage/local/LocalRepoStore.d.ts +55 -0
  139. package/dist/src/storage/local/LocalRepoStore.d.ts.map +1 -0
  140. package/dist/src/storage/local/LocalRepoStore.js +365 -0
  141. package/dist/src/storage/local/LocalRepoStore.js.map +1 -0
  142. package/dist/src/storage/local/gc.d.ts +92 -0
  143. package/dist/src/storage/local/gc.d.ts.map +1 -0
  144. package/dist/src/storage/local/gc.js +377 -0
  145. package/dist/src/storage/local/gc.js.map +1 -0
  146. package/dist/src/storage/local/index.d.ts +18 -0
  147. package/dist/src/storage/local/index.d.ts.map +1 -0
  148. package/dist/src/storage/local/index.js +18 -0
  149. package/dist/src/storage/local/index.js.map +1 -0
  150. package/dist/src/storage/local/localHelpers.d.ts +25 -0
  151. package/dist/src/storage/local/localHelpers.d.ts.map +1 -0
  152. package/dist/src/storage/local/localHelpers.js +69 -0
  153. package/dist/src/storage/local/localHelpers.js.map +1 -0
  154. package/dist/src/{repository.d.ts → storage/local/repository.d.ts} +8 -4
  155. package/dist/src/storage/local/repository.d.ts.map +1 -0
  156. package/dist/src/{repository.js → storage/local/repository.js} +31 -29
  157. package/dist/src/storage/local/repository.js.map +1 -0
  158. package/dist/src/tasks.d.ts +16 -10
  159. package/dist/src/tasks.d.ts.map +1 -1
  160. package/dist/src/tasks.js +35 -41
  161. package/dist/src/tasks.js.map +1 -1
  162. package/dist/src/test-helpers.d.ts +5 -4
  163. package/dist/src/test-helpers.d.ts.map +1 -1
  164. package/dist/src/test-helpers.js +9 -21
  165. package/dist/src/test-helpers.js.map +1 -1
  166. package/dist/src/transfer/InMemoryTransferBackend.d.ts +66 -0
  167. package/dist/src/transfer/InMemoryTransferBackend.d.ts.map +1 -0
  168. package/dist/src/transfer/InMemoryTransferBackend.js +166 -0
  169. package/dist/src/transfer/InMemoryTransferBackend.js.map +1 -0
  170. package/dist/src/transfer/index.d.ts +8 -0
  171. package/dist/src/transfer/index.d.ts.map +1 -0
  172. package/dist/src/transfer/index.js +9 -0
  173. package/dist/src/transfer/index.js.map +1 -0
  174. package/dist/src/transfer/interfaces.d.ts +103 -0
  175. package/dist/src/transfer/interfaces.d.ts.map +1 -0
  176. package/dist/src/transfer/interfaces.js +6 -0
  177. package/dist/src/transfer/interfaces.js.map +1 -0
  178. package/dist/src/transfer/types.d.ts +79 -0
  179. package/dist/src/transfer/types.d.ts.map +1 -0
  180. package/dist/src/transfer/types.js +58 -0
  181. package/dist/src/transfer/types.js.map +1 -0
  182. package/dist/src/trees.d.ts +147 -59
  183. package/dist/src/trees.d.ts.map +1 -1
  184. package/dist/src/trees.js +372 -419
  185. package/dist/src/trees.js.map +1 -1
  186. package/dist/src/uuid.d.ts +26 -0
  187. package/dist/src/uuid.d.ts.map +1 -0
  188. package/dist/src/uuid.js +80 -0
  189. package/dist/src/uuid.js.map +1 -0
  190. package/dist/src/workspaceStatus.d.ts +6 -4
  191. package/dist/src/workspaceStatus.d.ts.map +1 -1
  192. package/dist/src/workspaceStatus.js +43 -49
  193. package/dist/src/workspaceStatus.js.map +1 -1
  194. package/dist/src/workspaces.d.ts +35 -47
  195. package/dist/src/workspaces.d.ts.map +1 -1
  196. package/dist/src/workspaces.js +194 -156
  197. package/dist/src/workspaces.js.map +1 -1
  198. package/package.json +4 -4
  199. package/dist/src/gc.d.ts +0 -54
  200. package/dist/src/gc.d.ts.map +0 -1
  201. package/dist/src/gc.js +0 -233
  202. package/dist/src/gc.js.map +0 -1
  203. package/dist/src/repository.d.ts.map +0 -1
  204. package/dist/src/repository.js.map +0 -1
  205. package/dist/src/workspaceLock.d.ts +0 -67
  206. package/dist/src/workspaceLock.d.ts.map +0 -1
  207. package/dist/src/workspaceLock.js +0 -217
  208. package/dist/src/workspaceLock.js.map +0 -1
@@ -0,0 +1,377 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Licensed under BSL 1.1. See LICENSE for details.
4
+ */
5
+ /**
6
+ * Shared garbage collection algorithm for e3 repositories.
7
+ *
8
+ * Uses mark-and-sweep:
9
+ * 1. collectAllRoots: Collect root hashes from all root scan methods
10
+ * 2. markReachable: DFS through object graph via BEAST2 schema-aware traversal
11
+ * 3. sweepBatch: Pure decision function — identify unreachable objects to delete
12
+ * 4. repoGc: Driver that calls all phases in sequence
13
+ *
14
+ * These functions work with any StorageBackend — no instanceof checks.
15
+ * Cloud-specific concerns (S3 reachable set persistence, orphaned version cleanup)
16
+ * are handled in the cloud Lambda handlers.
17
+ */
18
+ import * as fs from 'fs/promises';
19
+ import * as path from 'path';
20
+ import { tmpdir } from 'os';
21
+ import { decodeBeast2 } from '@elaraai/east';
22
+ // =============================================================================
23
+ // Shared Algorithm Functions
24
+ // =============================================================================
25
+ /**
26
+ * Collect all root hashes from packages, workspaces, and executions.
27
+ *
28
+ * Calls each gcScan*Roots method with pagination support.
29
+ * Adding a new root scan method to RepoStore requires updating this function.
30
+ */
31
+ export async function collectAllRoots(store, repo) {
32
+ const roots = new Set();
33
+ const scanAll = async (scan) => {
34
+ let cursor;
35
+ do {
36
+ const result = await scan(repo, cursor);
37
+ for (const hash of result.roots) {
38
+ roots.add(hash);
39
+ }
40
+ cursor = result.cursor;
41
+ } while (cursor !== undefined);
42
+ };
43
+ await scanAll(store.gcScanPackageRoots.bind(store));
44
+ await scanAll(store.gcScanWorkspaceRoots.bind(store));
45
+ await scanAll(store.gcScanExecutionRoots.bind(store));
46
+ return roots;
47
+ }
48
+ /**
49
+ * Trace the object graph from roots using iterative DFS with schema-aware traversal.
50
+ *
51
+ * Decodes each object using BEAST2 self-describing format and extracts child
52
+ * hashes based on the detected object type (Package, Task, or Tree). Objects
53
+ * known to be leaves (values, IR blobs) are marked reachable without reading.
54
+ *
55
+ * @param readObject - Function to read an object by hash (returns null if missing)
56
+ * @param roots - Set of root hashes to start from
57
+ * @returns Set of all reachable hashes
58
+ */
59
+ export async function markReachable(readObject, roots) {
60
+ const reachable = new Set();
61
+ const stack = [...roots];
62
+ while (stack.length > 0) {
63
+ const hash = stack.pop();
64
+ if (reachable.has(hash))
65
+ continue;
66
+ const data = await readObject(hash);
67
+ if (!data)
68
+ continue;
69
+ reachable.add(hash);
70
+ // Schema-aware child extraction
71
+ let children;
72
+ try {
73
+ const decoded = decodeBeast2(Buffer.from(data));
74
+ children = extractChildren(decoded.type, decoded.value);
75
+ }
76
+ catch {
77
+ continue; // Not valid BEAST2 or unknown format — treat as leaf
78
+ }
79
+ for (const child of children) {
80
+ if (reachable.has(child.hash))
81
+ continue;
82
+ if (child.isLeaf) {
83
+ reachable.add(child.hash); // Mark without reading
84
+ }
85
+ else {
86
+ stack.push(child.hash);
87
+ }
88
+ }
89
+ }
90
+ return reachable;
91
+ }
92
+ // =============================================================================
93
+ // Type Detection Helpers
94
+ // =============================================================================
95
+ // EastTypeValue is a variant object: { type: string, value: any }
96
+ // For Struct: type.type === "Struct", type.value is Array<{ name: string, type: EastTypeValue }>
97
+ // For Variant: type.type === "Variant", type.value is Array<{ name: string, type: EastTypeValue }>
98
+ /**
99
+ * Check if a decoded EastTypeValue represents a PackageObject.
100
+ * PackageObject is a Struct with fields: tasks (Dict<String,String>), data (Struct)
101
+ */
102
+ function isPackageObjectShape(type) {
103
+ if (type.type !== 'Struct')
104
+ return false;
105
+ const fields = type.value;
106
+ const names = new Set(fields.map(f => f.name));
107
+ return names.has('tasks') && names.has('data');
108
+ }
109
+ /**
110
+ * Check if a decoded EastTypeValue represents a TaskObject.
111
+ * TaskObject is a Struct with fields: commandIr, inputs, output
112
+ */
113
+ function isTaskObjectShape(type) {
114
+ if (type.type !== 'Struct')
115
+ return false;
116
+ const fields = type.value;
117
+ const names = new Set(fields.map(f => f.name));
118
+ return names.has('commandIr') && names.has('inputs') && names.has('output');
119
+ }
120
+ /**
121
+ * Check if a field type is a DataRef (Variant with cases: unassigned, null, value, tree).
122
+ */
123
+ function isDataRefFieldType(fieldType) {
124
+ if (fieldType.type !== 'Variant')
125
+ return false;
126
+ const cases = fieldType.value;
127
+ const names = new Set(cases.map(c => c.name));
128
+ return names.has('tree') && names.has('value') && names.has('unassigned') && names.has('null');
129
+ }
130
+ /**
131
+ * Check if a decoded EastTypeValue represents a TreeObject.
132
+ * A tree is a Struct where every field is a DataRef variant.
133
+ */
134
+ function isTreeObjectShape(type) {
135
+ if (type.type !== 'Struct')
136
+ return false;
137
+ const fields = type.value;
138
+ return fields.length > 0 && fields.every(f => isDataRefFieldType(f.type));
139
+ }
140
+ /**
141
+ * Extract child hashes from a decoded BEAST2 object based on its type.
142
+ * Returns children with isLeaf flag to avoid reading leaf objects.
143
+ */
144
+ function extractChildren(type, value) {
145
+ const t = type;
146
+ const children = [];
147
+ if (isPackageObjectShape(t)) {
148
+ const pkg = value;
149
+ for (const taskHash of pkg.tasks.values()) {
150
+ children.push({ hash: taskHash, isLeaf: false });
151
+ }
152
+ // Extract value hashes from inline per-dataset refs
153
+ if (pkg.data.refs instanceof Map) {
154
+ for (const ref of pkg.data.refs.values()) {
155
+ if (ref.type === 'value' && typeof ref.value?.hash === 'string') {
156
+ children.push({ hash: ref.value.hash, isLeaf: true });
157
+ }
158
+ }
159
+ }
160
+ return children;
161
+ }
162
+ if (isTaskObjectShape(t)) {
163
+ const task = value;
164
+ children.push({ hash: task.commandIr, isLeaf: true }); // IR is a leaf
165
+ return children;
166
+ }
167
+ if (isTreeObjectShape(t)) {
168
+ const tree = value;
169
+ for (const ref of Object.values(tree)) {
170
+ if (ref.type === 'tree') {
171
+ children.push({ hash: ref.value, isLeaf: false }); // subtree needs traversal
172
+ }
173
+ else if (ref.type === 'value') {
174
+ children.push({ hash: ref.value, isLeaf: true }); // value is a leaf
175
+ }
176
+ // 'unassigned' and 'null': no hash to follow
177
+ }
178
+ return children;
179
+ }
180
+ return []; // Unknown type: leaf, no children
181
+ }
182
+ /**
183
+ * Pure decision function: determine which objects to delete.
184
+ *
185
+ * No side effects — trivially testable. Caller decides whether to
186
+ * actually delete (supports dry-run by skipping gcDeleteObjects).
187
+ *
188
+ * @param objects - Object entries from gcScanObjects
189
+ * @param reachable - Set of reachable hashes from markReachable
190
+ * @param minAge - Minimum age in ms; objects younger than this are skipped
191
+ * @returns Decision result with toDelete list and stats
192
+ */
193
+ export function sweepBatch(objects, reachable, minAge) {
194
+ const now = Date.now();
195
+ const toDelete = [];
196
+ let retained = 0;
197
+ let skippedYoung = 0;
198
+ let bytesFreed = 0;
199
+ for (const obj of objects) {
200
+ if (reachable.has(obj.hash)) {
201
+ retained++;
202
+ continue;
203
+ }
204
+ const age = now - obj.lastModified;
205
+ if (minAge > 0 && age < minAge) {
206
+ skippedYoung++;
207
+ continue;
208
+ }
209
+ toDelete.push(obj.hash);
210
+ bytesFreed += obj.size;
211
+ }
212
+ return { toDelete, retained, skippedYoung, bytesFreed };
213
+ }
214
+ // =============================================================================
215
+ // Local Driver
216
+ // =============================================================================
217
+ /**
218
+ * Run garbage collection on an e3 repository.
219
+ *
220
+ * Works with any StorageBackend — no instanceof checks.
221
+ *
222
+ * @param storage - Storage backend
223
+ * @param repo - Repository identifier
224
+ * @param options - GC options
225
+ * @returns GC result with statistics
226
+ */
227
+ export async function repoGc(storage, repo, options = {}) {
228
+ const minAge = options.minAge ?? 60000;
229
+ const dryRun = options.dryRun ?? false;
230
+ // Step 1: Collect all root hashes
231
+ const roots = await collectAllRoots(storage.repos, repo);
232
+ // Step 2: Mark all reachable objects
233
+ const readObject = async (hash) => {
234
+ try {
235
+ return await storage.objects.read(repo, hash);
236
+ }
237
+ catch {
238
+ return null;
239
+ }
240
+ };
241
+ const reachable = await markReachable(readObject, roots);
242
+ // Step 3: Scan and sweep objects
243
+ let totalDeleted = 0;
244
+ let totalRetained = 0;
245
+ let totalSkippedYoung = 0;
246
+ let totalBytesFreed = 0;
247
+ let cursor;
248
+ do {
249
+ const scan = await storage.repos.gcScanObjects(repo, cursor);
250
+ const result = sweepBatch(scan.objects, reachable, minAge);
251
+ totalRetained += result.retained;
252
+ totalSkippedYoung += result.skippedYoung;
253
+ totalBytesFreed += result.bytesFreed;
254
+ if (!dryRun && result.toDelete.length > 0) {
255
+ await storage.repos.gcDeleteObjects(repo, result.toDelete);
256
+ }
257
+ totalDeleted += result.toDelete.length;
258
+ cursor = scan.cursor;
259
+ } while (cursor !== undefined);
260
+ // Step 4: Clean up orphaned .partial files (local-only concern)
261
+ let deletedPartials = 0;
262
+ let partialSkippedYoung = 0;
263
+ try {
264
+ const partialResult = await cleanupPartials(repo, minAge, dryRun);
265
+ deletedPartials = partialResult.deleted;
266
+ partialSkippedYoung = partialResult.skippedYoung;
267
+ }
268
+ catch {
269
+ // Not a fatal error
270
+ }
271
+ // Step 5: Clean up orphaned transfer staging files
272
+ try {
273
+ const transferResult = await cleanupTransferStaging(minAge, dryRun);
274
+ deletedPartials += transferResult.deleted;
275
+ partialSkippedYoung += transferResult.skippedYoung;
276
+ }
277
+ catch {
278
+ // Not a fatal error
279
+ }
280
+ return {
281
+ deletedObjects: totalDeleted,
282
+ deletedPartials,
283
+ retainedObjects: totalRetained,
284
+ skippedYoung: totalSkippedYoung + partialSkippedYoung,
285
+ bytesFreed: totalBytesFreed,
286
+ };
287
+ }
288
+ /**
289
+ * Clean up orphaned .partial staging files in the objects directory.
290
+ * This is a local-only concern — cloud storage doesn't use .partial files.
291
+ */
292
+ async function cleanupPartials(repoPath, minAge, dryRun) {
293
+ const objectsDir = path.join(repoPath, 'objects');
294
+ const now = Date.now();
295
+ let deleted = 0;
296
+ let skippedYoung = 0;
297
+ try {
298
+ const subdirs = await fs.readdir(objectsDir);
299
+ for (const subdir of subdirs) {
300
+ if (!/^[a-f0-9]{2}$/.test(subdir))
301
+ continue;
302
+ const subdirPath = path.join(objectsDir, subdir);
303
+ try {
304
+ const stat = await fs.stat(subdirPath);
305
+ if (!stat.isDirectory())
306
+ continue;
307
+ }
308
+ catch {
309
+ continue;
310
+ }
311
+ const files = await fs.readdir(subdirPath);
312
+ for (const file of files) {
313
+ if (!file.endsWith('.partial'))
314
+ continue;
315
+ const filePath = path.join(subdirPath, file);
316
+ try {
317
+ const fileStat = await fs.stat(filePath);
318
+ const age = now - fileStat.mtimeMs;
319
+ if (minAge > 0 && age < minAge) {
320
+ skippedYoung++;
321
+ continue;
322
+ }
323
+ if (!dryRun) {
324
+ await fs.unlink(filePath);
325
+ }
326
+ deleted++;
327
+ }
328
+ catch {
329
+ // Skip files we can't stat or delete
330
+ }
331
+ }
332
+ }
333
+ }
334
+ catch {
335
+ // Objects directory doesn't exist
336
+ }
337
+ return { deleted, skippedYoung };
338
+ }
339
+ /**
340
+ * Clean up orphaned transfer staging files from the OS temp directory.
341
+ * These are created by the transfer upload flow and should be cleaned up
342
+ * after the transfer completes, but may be left behind on crashes.
343
+ */
344
+ async function cleanupTransferStaging(minAge, dryRun) {
345
+ const stagingDir = path.join(tmpdir(), 'e3-transfers');
346
+ const now = Date.now();
347
+ let deleted = 0;
348
+ let skippedYoung = 0;
349
+ try {
350
+ const files = await fs.readdir(stagingDir);
351
+ for (const file of files) {
352
+ if (!file.endsWith('.partial'))
353
+ continue;
354
+ const filePath = path.join(stagingDir, file);
355
+ try {
356
+ const fileStat = await fs.stat(filePath);
357
+ const age = now - fileStat.mtimeMs;
358
+ if (minAge > 0 && age < minAge) {
359
+ skippedYoung++;
360
+ continue;
361
+ }
362
+ if (!dryRun) {
363
+ await fs.unlink(filePath);
364
+ }
365
+ deleted++;
366
+ }
367
+ catch {
368
+ // Skip files we can't stat or delete
369
+ }
370
+ }
371
+ }
372
+ catch {
373
+ // Staging directory doesn't exist — nothing to clean
374
+ }
375
+ return { deleted, skippedYoung };
376
+ }
377
+ //# sourceMappingURL=gc.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"gc.js","sourceRoot":"","sources":["../../../../src/storage/local/gc.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAmD7C,gFAAgF;AAChF,6BAA6B;AAC7B,gFAAgF;AAEhF;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,KAAgB,EAAE,IAAY;IAClE,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAEhC,MAAM,OAAO,GAAG,KAAK,EAAE,IAAmE,EAAE,EAAE;QAC5F,IAAI,MAAe,CAAC;QACpB,GAAG,CAAC;YACF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YACxC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBAChC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAClB,CAAC;YACD,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QACzB,CAAC,QAAQ,MAAM,KAAK,SAAS,EAAE;IACjC,CAAC,CAAC;IAEF,MAAM,OAAO,CAAC,KAAK,CAAC,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IACpD,MAAM,OAAO,CAAC,KAAK,CAAC,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IACtD,MAAM,OAAO,CAAC,KAAK,CAAC,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAEtD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,UAAwD,EACxD,KAAkB;IAElB,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IACpC,MAAM,KAAK,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;IAEzB,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,EAAG,CAAC;QAC1B,IAAI,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,SAAS;QAElC,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAEpB,gCAAgC;QAChC,IAAI,QAA6C,CAAC;QAClD,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YAChD,QAAQ,GAAG,eAAe,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QAC1D,CAAC;QAAC,MAAM,CAAC;YACP,SAAS,CAAC,qDAAqD;QACjE,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;YAC7B,IAAI,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;gBAAE,SAAS;YACxC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;gBACjB,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,uBAAuB;YACpD,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,gFAAgF;AAChF,yBAAyB;AACzB,gFAAgF;AAEhF,kEAAkE;AAClE,iGAAiG;AACjG,mGAAmG;AAEnG;;;GAGG;AACH,SAAS,oBAAoB,CAAC,IAAS;IACrC,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAsC,CAAC;IAC3D,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC/C,OAAO,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;AACjD,CAAC;AAED;;;GAGG;AACH,SAAS,iBAAiB,CAAC,IAAS;IAClC,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAsC,CAAC;IAC3D,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC/C,OAAO,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;AAC9E,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CAAC,SAAc;IACxC,IAAI,SAAS,CAAC,IAAI,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC;IAC/C,MAAM,KAAK,GAAG,SAAS,CAAC,KAAsC,CAAC;IAC/D,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC9C,OAAO,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;AACjG,CAAC;AAED;;;GAGG;AACH,SAAS,iBAAiB,CAAC,IAAS;IAClC,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAsC,CAAC;IAC3D,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;AAC5E,CAAC;AAED;;;GAGG;AACH,SAAS,eAAe,CACtB,IAAa,EACb,KAAc;IAEd,MAAM,CAAC,GAAG,IAAW,CAAC;IACtB,MAAM,QAAQ,GAAwC,EAAE,CAAC;IAEzD,IAAI,oBAAoB,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,KAAuH,CAAC;QACpI,KAAK,MAAM,QAAQ,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YAC1C,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QACnD,CAAC;QACD,oDAAoD;QACpD,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,YAAY,GAAG,EAAE,CAAC;YACjC,KAAK,MAAM,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;gBACzC,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO,IAAI,OAAO,GAAG,CAAC,KAAK,EAAE,IAAI,KAAK,QAAQ,EAAE,CAAC;oBAChE,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;gBACxD,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,IAAI,iBAAiB,CAAC,CAAC,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,KAA8B,CAAC;QAC5C,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,eAAe;QACtE,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,IAAI,iBAAiB,CAAC,CAAC,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,KAAqD,CAAC;QACnE,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YACtC,IAAI,GAAG,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACxB,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,KAAe,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,0BAA0B;YACzF,CAAC;iBAAM,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;gBAChC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,KAAe,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,kBAAkB;YAChF,CAAC;YACD,6CAA6C;QAC/C,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,OAAO,EAAE,CAAC,CAAC,kCAAkC;AAC/C,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,UAAU,CACxB,OAAwB,EACxB,SAAsB,EACtB,MAAc;IAEd,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,QAAQ,EAAE,CAAC;YACX,SAAS;QACX,CAAC;QACD,MAAM,GAAG,GAAG,GAAG,GAAG,GAAG,CAAC,YAAY,CAAC;QACnC,IAAI,MAAM,GAAG,CAAC,IAAI,GAAG,GAAG,MAAM,EAAE,CAAC;YAC/B,YAAY,EAAE,CAAC;YACf,SAAS;QACX,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACxB,UAAU,IAAI,GAAG,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,YAAY,EAAE,UAAU,EAAE,CAAC;AAC1D,CAAC;AAED,gFAAgF;AAChF,eAAe;AACf,gFAAgF;AAEhF;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,OAAuB,EACvB,IAAY,EACZ,UAAqB,EAAE;IAEvB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC;IACvC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC;IAEvC,kCAAkC;IAClC,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAEzD,qCAAqC;IACrC,MAAM,UAAU,GAAG,KAAK,EAAE,IAAY,EAA8B,EAAE;QACpE,IAAI,CAAC;YACH,OAAO,MAAM,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAChD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,CAAC;IACF,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IAEzD,iCAAiC;IACjC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAC1B,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,MAAe,CAAC;IAEpB,GAAG,CAAC;QACF,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC7D,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAE3D,aAAa,IAAI,MAAM,CAAC,QAAQ,CAAC;QACjC,iBAAiB,IAAI,MAAM,CAAC,YAAY,CAAC;QACzC,eAAe,IAAI,MAAM,CAAC,UAAU,CAAC;QAErC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1C,MAAM,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC7D,CAAC;QACD,YAAY,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;QAEvC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC,QAAQ,MAAM,KAAK,SAAS,EAAE;IAE/B,gEAAgE;IAChE,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,mBAAmB,GAAG,CAAC,CAAC;IAC5B,IAAI,CAAC;QACH,MAAM,aAAa,GAAG,MAAM,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QAClE,eAAe,GAAG,aAAa,CAAC,OAAO,CAAC;QACxC,mBAAmB,GAAG,aAAa,CAAC,YAAY,CAAC;IACnD,CAAC;IAAC,MAAM,CAAC;QACP,oBAAoB;IACtB,CAAC;IAED,mDAAmD;IACnD,IAAI,CAAC;QACH,MAAM,cAAc,GAAG,MAAM,sBAAsB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACpE,eAAe,IAAI,cAAc,CAAC,OAAO,CAAC;QAC1C,mBAAmB,IAAI,cAAc,CAAC,YAAY,CAAC;IACrD,CAAC;IAAC,MAAM,CAAC;QACP,oBAAoB;IACtB,CAAC;IAED,OAAO;QACL,cAAc,EAAE,YAAY;QAC5B,eAAe;QACf,eAAe,EAAE,aAAa;QAC9B,YAAY,EAAE,iBAAiB,GAAG,mBAAmB;QACrD,UAAU,EAAE,eAAe;KAC5B,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,eAAe,CAC5B,QAAgB,EAChB,MAAc,EACd,MAAe;IAEf,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAClD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QAC7C,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC;gBAAE,SAAS;YAC5C,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YACjD,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBACvC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;oBAAE,SAAS;YACpC,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;YAED,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YAC3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;oBAAE,SAAS;gBACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;gBAC7C,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;oBACzC,MAAM,GAAG,GAAG,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC;oBACnC,IAAI,MAAM,GAAG,CAAC,IAAI,GAAG,GAAG,MAAM,EAAE,CAAC;wBAC/B,YAAY,EAAE,CAAC;wBACf,SAAS;oBACX,CAAC;oBACD,IAAI,CAAC,MAAM,EAAE,CAAC;wBACZ,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;oBAC5B,CAAC;oBACD,OAAO,EAAE,CAAC;gBACZ,CAAC;gBAAC,MAAM,CAAC;oBACP,qCAAqC;gBACvC,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,kCAAkC;IACpC,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC;AACnC,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,sBAAsB,CACnC,MAAc,EACd,MAAe;IAEf,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,cAAc,CAAC,CAAC;IACvD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QAC3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;gBAAE,SAAS;YACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;YAC7C,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACzC,MAAM,GAAG,GAAG,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC;gBACnC,IAAI,MAAM,GAAG,CAAC,IAAI,GAAG,GAAG,MAAM,EAAE,CAAC;oBAC/B,YAAY,EAAE,CAAC;oBACf,SAAS;gBACX,CAAC;gBACD,IAAI,CAAC,MAAM,EAAE,CAAC;oBACZ,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBAC5B,CAAC;gBACD,OAAO,EAAE,CAAC;YACZ,CAAC;YAAC,MAAM,CAAC;gBACP,qCAAqC;YACvC,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,qDAAqD;IACvD,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC;AACnC,CAAC"}
@@ -0,0 +1,18 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Licensed under BSL 1.1. See LICENSE for details.
4
+ */
5
+ /**
6
+ * Local filesystem implementation of StorageBackend.
7
+ *
8
+ * This wraps the existing e3-core filesystem functions to provide a
9
+ * StorageBackend that works with local repositories.
10
+ */
11
+ export { LocalStorage, LocalBackend } from './LocalBackend.js';
12
+ export { LocalObjectStore } from './LocalObjectStore.js';
13
+ export { LocalRefStore } from './LocalRefStore.js';
14
+ export { LocalLockService } from './LocalLockService.js';
15
+ export { LocalLogStore } from './LocalLogStore.js';
16
+ export { LocalRepoStore } from './LocalRepoStore.js';
17
+ export { LocalDatasetRefStore } from './LocalDatasetRefStore.js';
18
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/storage/local/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;GAKG;AAEH,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC"}
@@ -0,0 +1,18 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Licensed under BSL 1.1. See LICENSE for details.
4
+ */
5
+ /**
6
+ * Local filesystem implementation of StorageBackend.
7
+ *
8
+ * This wraps the existing e3-core filesystem functions to provide a
9
+ * StorageBackend that works with local repositories.
10
+ */
11
+ export { LocalStorage, LocalBackend } from './LocalBackend.js';
12
+ export { LocalObjectStore } from './LocalObjectStore.js';
13
+ export { LocalRefStore } from './LocalRefStore.js';
14
+ export { LocalLockService } from './LocalLockService.js';
15
+ export { LocalLogStore } from './LocalLogStore.js';
16
+ export { LocalRepoStore } from './LocalRepoStore.js';
17
+ export { LocalDatasetRefStore } from './LocalDatasetRefStore.js';
18
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/storage/local/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;GAKG;AAEH,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAC/D,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC"}
@@ -0,0 +1,25 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Licensed under BSL 1.1. See LICENSE for details.
4
+ */
5
+ /**
6
+ * Get the filesystem path for an object.
7
+ *
8
+ * @param repoPath - Path to e3 repository
9
+ * @param hash - SHA256 hash of the object
10
+ * @returns Filesystem path: objects/<hash[0..2]>/<hash[2..]>.beast2
11
+ */
12
+ export declare function objectPath(repoPath: string, hash: string): string;
13
+ /**
14
+ * Get the minimum unambiguous prefix length for an object hash.
15
+ *
16
+ * Scans the object store to find the shortest prefix of the given hash
17
+ * that uniquely identifies it among all stored objects.
18
+ *
19
+ * @param repoPath - Path to e3 repository
20
+ * @param hash - Full SHA256 hash of the object
21
+ * @param minLength - Minimum prefix length to return (default: 4)
22
+ * @returns Minimum unambiguous prefix length
23
+ */
24
+ export declare function objectAbbrev(repoPath: string, hash: string, minLength?: number): Promise<number>;
25
+ //# sourceMappingURL=localHelpers.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"localHelpers.d.ts","sourceRoot":"","sources":["../../../../src/storage/local/localHelpers.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAaH;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAIjE;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,YAAY,CAChC,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,EACZ,SAAS,GAAE,MAAU,GACpB,OAAO,CAAC,MAAM,CAAC,CAwCjB"}
@@ -0,0 +1,69 @@
1
+ /**
2
+ * Copyright (c) 2025 Elara AI Pty Ltd
3
+ * Licensed under BSL 1.1. See LICENSE for details.
4
+ */
5
+ /**
6
+ * Local filesystem helpers for e3 object storage.
7
+ *
8
+ * These functions are local-specific utilities for working with
9
+ * the filesystem-based object store. They are used by LocalObjectStore
10
+ * and other local storage components.
11
+ */
12
+ import * as fs from 'fs/promises';
13
+ import * as path from 'path';
14
+ /**
15
+ * Get the filesystem path for an object.
16
+ *
17
+ * @param repoPath - Path to e3 repository
18
+ * @param hash - SHA256 hash of the object
19
+ * @returns Filesystem path: objects/<hash[0..2]>/<hash[2..]>.beast2
20
+ */
21
+ export function objectPath(repoPath, hash) {
22
+ const dirName = hash.slice(0, 2);
23
+ const fileName = hash.slice(2) + '.beast2';
24
+ return path.join(repoPath, 'objects', dirName, fileName);
25
+ }
26
+ /**
27
+ * Get the minimum unambiguous prefix length for an object hash.
28
+ *
29
+ * Scans the object store to find the shortest prefix of the given hash
30
+ * that uniquely identifies it among all stored objects.
31
+ *
32
+ * @param repoPath - Path to e3 repository
33
+ * @param hash - Full SHA256 hash of the object
34
+ * @param minLength - Minimum prefix length to return (default: 4)
35
+ * @returns Minimum unambiguous prefix length
36
+ */
37
+ export async function objectAbbrev(repoPath, hash, minLength = 4) {
38
+ const objectsDir = path.join(repoPath, 'objects');
39
+ const targetPrefix = hash.slice(0, 2);
40
+ // Collect all hashes that share the same 2-char prefix directory
41
+ const hashes = [];
42
+ try {
43
+ const dirPath = path.join(objectsDir, targetPrefix);
44
+ const entries = await fs.readdir(dirPath);
45
+ for (const entry of entries) {
46
+ if (entry.endsWith('.beast2') && !entry.includes('.partial')) {
47
+ // Reconstruct full hash: dir prefix + filename without extension
48
+ const fullHash = targetPrefix + entry.slice(0, -7); // remove '.beast2'
49
+ hashes.push(fullHash);
50
+ }
51
+ }
52
+ }
53
+ catch {
54
+ // Directory doesn't exist - hash is unique at minimum length
55
+ return minLength;
56
+ }
57
+ // Find minimum length that disambiguates from all other hashes
58
+ let length = minLength;
59
+ while (length < hash.length) {
60
+ const prefix = hash.slice(0, length);
61
+ const conflicts = hashes.filter((h) => h !== hash && h.startsWith(prefix));
62
+ if (conflicts.length === 0) {
63
+ return length;
64
+ }
65
+ length++;
66
+ }
67
+ return hash.length;
68
+ }
69
+ //# sourceMappingURL=localHelpers.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"localHelpers.js","sourceRoot":"","sources":["../../../../src/storage/local/localHelpers.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B;;;;;;GAMG;AACH,MAAM,UAAU,UAAU,CAAC,QAAgB,EAAE,IAAY;IACvD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC;IAC3C,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;AAC3D,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,QAAgB,EAChB,IAAY,EACZ,YAAoB,CAAC;IAErB,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAClD,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAEtC,iEAAiE;IACjE,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;QACpD,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAE1C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC7D,iEAAiE;gBACjE,MAAM,QAAQ,GAAG,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,mBAAmB;gBACvE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,6DAA6D;QAC7D,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,+DAA+D;IAC/D,IAAI,MAAM,GAAG,SAAS,CAAC;IAEvB,OAAO,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QACrC,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAC7B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAC1C,CAAC;QAEF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,MAAM,EAAE,CAAC;IACX,CAAC;IAED,OAAO,IAAI,CAAC,MAAM,CAAC;AACrB,CAAC"}
@@ -7,28 +7,32 @@
7
7
  */
8
8
  export interface InitRepositoryResult {
9
9
  success: boolean;
10
- e3Dir: string;
10
+ repoPath: string;
11
11
  error?: Error;
12
12
  alreadyExists?: boolean;
13
13
  }
14
14
  /**
15
15
  * Initialize a new e3 repository
16
16
  *
17
- * Creates:
17
+ * Creates the repository directory structure:
18
18
  * - objects/
19
19
  * - packages/
20
20
  * - executions/
21
21
  * - workspaces/
22
22
  *
23
+ * The repository IS the specified directory - subdirectories are created directly within it.
24
+ *
23
25
  * Pure business logic - no UI dependencies
24
26
  */
25
27
  export declare function repoInit(repoPath: string): InitRepositoryResult;
26
28
  /**
27
29
  * Find the e3 repository directory
28
30
  *
29
- * Searches:
31
+ * Checks:
30
32
  * 1. E3_REPO environment variable
31
- * 2. Current directory and parents (like git)
33
+ * 2. The provided startPath (if given)
34
+ *
35
+ * Returns null if no valid repository is found.
32
36
  */
33
37
  export declare function repoFind(startPath?: string): string | null;
34
38
  /**
@@ -0,0 +1 @@
1
+ {"version":3,"file":"repository.d.ts","sourceRoot":"","sources":["../../../../src/storage/local/repository.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAYH;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,oBAAoB,CAwC/D;AAYD;;;;;;;;GAQG;AACH,wBAAgB,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAkB1D;AAED;;GAEG;AACH,wBAAgB,OAAO,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,CAQjD"}