@elie-laloum/outpost 4.0.0 → 4.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (240) hide show
  1. package/CHANGELOG.md +25 -0
  2. package/README.md +10 -6
  3. package/SECURITY.md +2 -0
  4. package/dist/adapters/agents/codex-adapter.js +2 -0
  5. package/dist/adapters/agents/codex-adapter.js.map +1 -1
  6. package/dist/adapters/agents/codex-provider.d.ts +2 -0
  7. package/dist/adapters/agents/codex-provider.js +32 -0
  8. package/dist/adapters/agents/codex-provider.js.map +1 -0
  9. package/dist/adapters/agents/codex-request.js +2 -1
  10. package/dist/adapters/agents/codex-request.js.map +1 -1
  11. package/dist/adapters/agents/gemini-request.js +2 -0
  12. package/dist/adapters/agents/gemini-request.js.map +1 -1
  13. package/dist/adapters/agents/settings.types.d.ts +5 -0
  14. package/dist/adapters/agents/settings.types.js.map +1 -1
  15. package/dist/adapters/models/openai-compatible.constants.d.ts +4 -0
  16. package/dist/adapters/models/openai-compatible.constants.js +10 -0
  17. package/dist/adapters/models/openai-compatible.constants.js.map +1 -0
  18. package/dist/adapters/models/openai-compatible.d.ts +3 -0
  19. package/dist/adapters/models/openai-compatible.js +78 -0
  20. package/dist/adapters/models/openai-compatible.js.map +1 -0
  21. package/dist/adapters/models/openai-compatible.types.d.ts +14 -0
  22. package/dist/adapters/models/openai-compatible.types.js +2 -0
  23. package/dist/adapters/models/openai-compatible.types.js.map +1 -0
  24. package/dist/adapters/models/openai-http.d.ts +1 -0
  25. package/dist/adapters/models/openai-http.js +30 -0
  26. package/dist/adapters/models/openai-http.js.map +1 -0
  27. package/dist/adapters/models/openai-request.d.ts +4 -0
  28. package/dist/adapters/models/openai-request.js +46 -0
  29. package/dist/adapters/models/openai-request.js.map +1 -0
  30. package/dist/adapters/models/openai-response.d.ts +3 -0
  31. package/dist/adapters/models/openai-response.js +65 -0
  32. package/dist/adapters/models/openai-response.js.map +1 -0
  33. package/dist/application/execution.types.d.ts +2 -0
  34. package/dist/application/execution.types.js.map +1 -1
  35. package/dist/application/outpost.types.d.ts +5 -0
  36. package/dist/application/outpost.types.js.map +1 -1
  37. package/dist/application/recovery-archive.d.ts +4 -0
  38. package/dist/application/recovery-archive.js +29 -0
  39. package/dist/application/recovery-archive.js.map +1 -0
  40. package/dist/application/recovery-archive.types.d.ts +10 -0
  41. package/dist/application/recovery-archive.types.js +2 -0
  42. package/dist/application/recovery-archive.types.js.map +1 -0
  43. package/dist/application/recovery-inspection.js +3 -0
  44. package/dist/application/recovery-inspection.js.map +1 -1
  45. package/dist/application/recovery-inspection.types.d.ts +2 -0
  46. package/dist/application/recovery-inspection.types.js.map +1 -1
  47. package/dist/application/recovery-retention.d.ts +2 -1
  48. package/dist/application/recovery-retention.js +9 -1
  49. package/dist/application/recovery-retention.js.map +1 -1
  50. package/dist/application/recovery-retention.types.d.ts +6 -0
  51. package/dist/application/recovery-retention.types.js.map +1 -1
  52. package/dist/application/remote-backup.js +8 -0
  53. package/dist/application/remote-backup.js.map +1 -1
  54. package/dist/application/remote-workspace.types.d.ts +2 -0
  55. package/dist/application/remote-workspace.types.js.map +1 -1
  56. package/dist/application/sandbox-agents.js +2 -1
  57. package/dist/application/sandbox-agents.js.map +1 -1
  58. package/dist/application/sandbox-dispatch.js +9 -0
  59. package/dist/application/sandbox-dispatch.js.map +1 -1
  60. package/dist/application/sandbox-provision.js +6 -0
  61. package/dist/application/sandbox-provision.js.map +1 -1
  62. package/dist/application/storage-reservation.js +3 -0
  63. package/dist/application/storage-reservation.js.map +1 -1
  64. package/dist/application/transport-inspection.constants.d.ts +2 -0
  65. package/dist/application/transport-inspection.constants.js +10 -0
  66. package/dist/application/transport-inspection.constants.js.map +1 -0
  67. package/dist/application/transport-inspection.d.ts +2 -0
  68. package/dist/application/transport-inspection.js +103 -0
  69. package/dist/application/transport-inspection.js.map +1 -0
  70. package/dist/application/transport-retention.d.ts +4 -0
  71. package/dist/application/transport-retention.js +153 -0
  72. package/dist/application/transport-retention.js.map +1 -0
  73. package/dist/cli/commands.js +55 -18
  74. package/dist/cli/commands.js.map +1 -1
  75. package/dist/cli/init-authentication.constants.d.ts +13 -0
  76. package/dist/cli/init-authentication.constants.js +10 -0
  77. package/dist/cli/init-authentication.constants.js.map +1 -0
  78. package/dist/cli/init-authentication.d.ts +4 -0
  79. package/dist/cli/init-authentication.js +43 -0
  80. package/dist/cli/init-authentication.js.map +1 -0
  81. package/dist/cli/init-command.js +74 -15
  82. package/dist/cli/init-command.js.map +1 -1
  83. package/dist/cli/main.constants.d.ts +102 -2
  84. package/dist/cli/main.constants.js +148 -46
  85. package/dist/cli/main.constants.js.map +1 -1
  86. package/dist/cli/scaffold-files.js +4 -3
  87. package/dist/cli/scaffold-files.js.map +1 -1
  88. package/dist/cli/scaffold-validation.js +19 -0
  89. package/dist/cli/scaffold-validation.js.map +1 -1
  90. package/dist/cli/scaffold.constants.d.ts +0 -5
  91. package/dist/cli/scaffold.constants.js +0 -5
  92. package/dist/cli/scaffold.constants.js.map +1 -1
  93. package/dist/cli/scaffold.js +13 -0
  94. package/dist/cli/scaffold.js.map +1 -1
  95. package/dist/cli/scaffold.types.d.ts +3 -0
  96. package/dist/cli/scaffold.types.js.map +1 -1
  97. package/dist/cli/starters.js +9 -2
  98. package/dist/cli/starters.js.map +1 -1
  99. package/dist/domain/conversation.types.d.ts +2 -0
  100. package/dist/domain/conversation.types.js.map +1 -1
  101. package/dist/domain/model.types.d.ts +15 -0
  102. package/dist/domain/model.types.js +2 -0
  103. package/dist/domain/model.types.js.map +1 -0
  104. package/dist/domain/transport.constants.d.ts +2 -0
  105. package/dist/domain/transport.constants.js +3 -0
  106. package/dist/domain/transport.constants.js.map +1 -0
  107. package/dist/domain/transport.d.ts +6 -0
  108. package/dist/domain/transport.js +21 -0
  109. package/dist/domain/transport.js.map +1 -0
  110. package/dist/domain/transport.types.d.ts +31 -0
  111. package/dist/domain/transport.types.js +2 -0
  112. package/dist/domain/transport.types.js.map +1 -0
  113. package/dist/index.d.ts +18 -1
  114. package/dist/index.js +8 -0
  115. package/dist/index.js.map +1 -1
  116. package/dist/infrastructure/artifact-store.js +10 -0
  117. package/dist/infrastructure/artifact-store.js.map +1 -1
  118. package/dist/infrastructure/artifact-store.types.d.ts +3 -1
  119. package/dist/infrastructure/artifact-store.types.js.map +1 -1
  120. package/dist/infrastructure/conversations/capture.js +1 -1
  121. package/dist/infrastructure/conversations/capture.js.map +1 -1
  122. package/dist/infrastructure/conversations.d.ts +2 -0
  123. package/dist/infrastructure/conversations.js +2 -0
  124. package/dist/infrastructure/conversations.js.map +1 -1
  125. package/dist/infrastructure/inspection-file.js +3 -3
  126. package/dist/infrastructure/inspection-file.js.map +1 -1
  127. package/dist/infrastructure/inspection-hash.js +4 -3
  128. package/dist/infrastructure/inspection-hash.js.map +1 -1
  129. package/dist/infrastructure/inspection-path.d.ts +1 -0
  130. package/dist/infrastructure/inspection-path.js +18 -0
  131. package/dist/infrastructure/inspection-path.js.map +1 -0
  132. package/dist/infrastructure/journal.js +6 -0
  133. package/dist/infrastructure/journal.js.map +1 -1
  134. package/dist/infrastructure/journal.types.d.ts +3 -0
  135. package/dist/infrastructure/journal.types.js.map +1 -1
  136. package/dist/infrastructure/local-transport.d.ts +3 -0
  137. package/dist/infrastructure/local-transport.js +146 -0
  138. package/dist/infrastructure/local-transport.js.map +1 -0
  139. package/dist/infrastructure/local-transport.types.d.ts +3 -0
  140. package/dist/infrastructure/local-transport.types.js +2 -0
  141. package/dist/infrastructure/local-transport.types.js.map +1 -0
  142. package/dist/infrastructure/resource-activity-inspection.d.ts +2 -1
  143. package/dist/infrastructure/resource-activity-inspection.js +2 -2
  144. package/dist/infrastructure/resource-activity-inspection.js.map +1 -1
  145. package/dist/infrastructure/resource-activity.js +42 -14
  146. package/dist/infrastructure/resource-activity.js.map +1 -1
  147. package/dist/infrastructure/resource-activity.types.d.ts +2 -0
  148. package/dist/infrastructure/resource-activity.types.js.map +1 -1
  149. package/dist/infrastructure/s3-transport.d.ts +4 -0
  150. package/dist/infrastructure/s3-transport.js +144 -0
  151. package/dist/infrastructure/s3-transport.js.map +1 -0
  152. package/dist/infrastructure/s3-transport.types.d.ts +6 -0
  153. package/dist/infrastructure/s3-transport.types.js +2 -0
  154. package/dist/infrastructure/s3-transport.types.js.map +1 -0
  155. package/dist/infrastructure/storage-inventory.types.d.ts +2 -1
  156. package/dist/infrastructure/storage-inventory.types.js.map +1 -1
  157. package/dist/infrastructure/storage-reservations.types.d.ts +2 -0
  158. package/dist/infrastructure/storage-reservations.types.js.map +1 -1
  159. package/dist/infrastructure/task-queue-bullmq-resources.d.ts +13 -0
  160. package/dist/infrastructure/task-queue-bullmq-resources.js +118 -0
  161. package/dist/infrastructure/task-queue-bullmq-resources.js.map +1 -0
  162. package/dist/infrastructure/task-queue-bullmq-state.d.ts +5 -0
  163. package/dist/infrastructure/task-queue-bullmq-state.js +26 -0
  164. package/dist/infrastructure/task-queue-bullmq-state.js.map +1 -0
  165. package/dist/infrastructure/task-queue-bullmq.constants.d.ts +7 -0
  166. package/dist/infrastructure/task-queue-bullmq.constants.js +79 -0
  167. package/dist/infrastructure/task-queue-bullmq.constants.js.map +1 -0
  168. package/dist/infrastructure/task-queue-bullmq.d.ts +3 -0
  169. package/dist/infrastructure/task-queue-bullmq.js +124 -0
  170. package/dist/infrastructure/task-queue-bullmq.js.map +1 -0
  171. package/dist/infrastructure/task-queue-bullmq.types.d.ts +17 -0
  172. package/dist/infrastructure/task-queue-bullmq.types.js +2 -0
  173. package/dist/infrastructure/task-queue-bullmq.types.js.map +1 -0
  174. package/dist/infrastructure/transport-archive.constants.d.ts +6 -0
  175. package/dist/infrastructure/transport-archive.constants.js +7 -0
  176. package/dist/infrastructure/transport-archive.constants.js.map +1 -0
  177. package/dist/infrastructure/transport-archive.d.ts +5 -0
  178. package/dist/infrastructure/transport-archive.js +157 -0
  179. package/dist/infrastructure/transport-archive.js.map +1 -0
  180. package/dist/infrastructure/transport-archive.types.d.ts +17 -0
  181. package/dist/infrastructure/transport-archive.types.js +2 -0
  182. package/dist/infrastructure/transport-archive.types.js.map +1 -0
  183. package/dist/infrastructure/transport-artifact-store.d.ts +3 -0
  184. package/dist/infrastructure/transport-artifact-store.js +31 -0
  185. package/dist/infrastructure/transport-artifact-store.js.map +1 -0
  186. package/dist/infrastructure/transport-artifact-store.types.d.ts +4 -0
  187. package/dist/infrastructure/transport-artifact-store.types.js +2 -0
  188. package/dist/infrastructure/transport-artifact-store.types.js.map +1 -0
  189. package/dist/infrastructure/transport-checkpoint.d.ts +5 -0
  190. package/dist/infrastructure/transport-checkpoint.js +105 -0
  191. package/dist/infrastructure/transport-checkpoint.js.map +1 -0
  192. package/dist/infrastructure/transport-checkpoint.types.d.ts +9 -0
  193. package/dist/infrastructure/transport-checkpoint.types.js +2 -0
  194. package/dist/infrastructure/transport-checkpoint.types.js.map +1 -0
  195. package/dist/infrastructure/transport-conversations.d.ts +4 -0
  196. package/dist/infrastructure/transport-conversations.js +69 -0
  197. package/dist/infrastructure/transport-conversations.js.map +1 -0
  198. package/dist/infrastructure/transport-conversations.types.d.ts +4 -0
  199. package/dist/infrastructure/transport-conversations.types.js +2 -0
  200. package/dist/infrastructure/transport-conversations.types.js.map +1 -0
  201. package/dist/infrastructure/transport-envelope.d.ts +7 -0
  202. package/dist/infrastructure/transport-envelope.js +47 -0
  203. package/dist/infrastructure/transport-envelope.js.map +1 -0
  204. package/dist/infrastructure/transport-journal.d.ts +6 -0
  205. package/dist/infrastructure/transport-journal.js +93 -0
  206. package/dist/infrastructure/transport-journal.js.map +1 -0
  207. package/dist/infrastructure/transport-journal.types.d.ts +10 -0
  208. package/dist/infrastructure/transport-journal.types.js +2 -0
  209. package/dist/infrastructure/transport-journal.types.js.map +1 -0
  210. package/dist/infrastructure/transport-json.d.ts +6 -0
  211. package/dist/infrastructure/transport-json.js +46 -0
  212. package/dist/infrastructure/transport-json.js.map +1 -0
  213. package/dist/infrastructure/transport-reservations.d.ts +5 -0
  214. package/dist/infrastructure/transport-reservations.js +89 -0
  215. package/dist/infrastructure/transport-reservations.js.map +1 -0
  216. package/dist/infrastructure/transport-reservations.types.d.ts +4 -0
  217. package/dist/infrastructure/transport-reservations.types.js +2 -0
  218. package/dist/infrastructure/transport-reservations.types.js.map +1 -0
  219. package/dist/infrastructure/transport.constants.d.ts +8 -0
  220. package/dist/infrastructure/transport.constants.js +9 -0
  221. package/dist/infrastructure/transport.constants.js.map +1 -0
  222. package/dist/infrastructure/workflow-checkpoint.js +7 -0
  223. package/dist/infrastructure/workflow-checkpoint.js.map +1 -1
  224. package/dist/infrastructure/workflow-checkpoint.types.d.ts +3 -1
  225. package/dist/infrastructure/workflow-checkpoint.types.js.map +1 -1
  226. package/dist/providers/agents.d.ts +1 -1
  227. package/dist/providers/agents.js.map +1 -1
  228. package/dist/providers/daytona-command.constants.d.ts +2 -0
  229. package/dist/providers/daytona-command.constants.js +13 -0
  230. package/dist/providers/daytona-command.constants.js.map +1 -0
  231. package/dist/providers/daytona-command.js +9 -3
  232. package/dist/providers/daytona-command.js.map +1 -1
  233. package/dist/providers/daytona-output.d.ts +4 -0
  234. package/dist/providers/daytona-output.js +37 -0
  235. package/dist/providers/daytona-output.js.map +1 -0
  236. package/dist/providers/daytona-terminal.js +3 -1
  237. package/dist/providers/daytona-terminal.js.map +1 -1
  238. package/dist/providers/vercel-command.js +2 -2
  239. package/dist/providers/vercel-command.js.map +1 -1
  240. package/package.json +31 -6
@@ -0,0 +1 @@
1
+ {"version":3,"file":"task-queue-bullmq.constants.js","sourceRoot":"","sources":["../../src/infrastructure/task-queue-bullmq.constants.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,mBAAmB,GAAG,SAAS,CAAC;AAC7C,MAAM,CAAC,MAAM,8BAA8B,GAAG,IAAI,CAAC;AACnD,MAAM,CAAC,MAAM,oBAAoB,GAAG,GAAG,CAAC;AACxC,MAAM,CAAC,MAAM,yBAAyB,GAAG,MAAM,CAAC;AAChD,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC;AACzC,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAEtC,MAAM,CAAC,MAAM,iBAAiB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAuEhC,CAAC","sourcesContent":["export const bullMQDefaultPrefix = \"outpost\";\nexport const bullMQDefaultStalledIntervalMs = 1000;\nexport const bullMQClaimBatchSize = 100;\nexport const bullMQConnectionTimeoutMs = 10_000;\nexport const bullMQConnectionRetries = 3;\nexport const bullMQRetryDelayMs = 100;\n\nexport const bullMQStateScript = `\nlocal operation = ARGV[1]\nlocal request = redis.call('HGET', KEYS[1], 'request')\nif operation == 'enqueue' then\n if request and request ~= ARGV[2] then\n return redis.error_reply('Queue identity already has a different request')\n end\n if not request then\n request = ARGV[2]\n redis.call('HSET', KEYS[1], 'request', request, 'deadline', ARGV[3], 'meta', '{\"status\":\"pending\",\"fence\":0}')\n end\nend\nif not request then\n if operation == 'get' then return nil end\n return redis.error_reply('Queue job does not exist')\nend\nlocal meta = cjson.decode(redis.call('HGET', KEYS[1], 'meta'))\nlocal clock = redis.call('TIME')\nlocal now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000)\nlocal deadline = tonumber(redis.call('HGET', KEYS[1], 'deadline'))\nlocal function live()\n return meta.status == 'pending' or meta.status == 'active'\nend\nlocal function save()\n redis.call('HSET', KEYS[1], 'meta', cjson.encode(meta))\nend\nif live() and deadline and deadline <= now then\n meta.status = 'cancelled'\n meta.fence = meta.fence + 1\n save()\nend\nlocal function owned()\n return meta.status == 'active' and meta.worker == ARGV[2]\n and meta.fence == tonumber(ARGV[3]) and meta.expires > now\n and redis.call('GET', KEYS[2]) == meta.token\nend\nlocal function extend(duration)\n meta.expires = now + duration\n if deadline then meta.expires = math.min(meta.expires, deadline) end\n redis.call('PEXPIRE', KEYS[2], meta.expires - now)\n redis.call('SREM', KEYS[3], ARGV[5])\nend\nif operation == 'claim' and live() then\n if redis.call('GET', KEYS[2]) ~= ARGV[3] then\n return redis.error_reply('Stale queue lease')\n end\n meta.status = 'active'\n meta.worker = ARGV[2]\n meta.token = ARGV[3]\n meta.fence = meta.fence + 1\n extend(tonumber(ARGV[4]))\n save()\nend\nif operation == 'renew' or operation == 'complete' then\n if not owned() then return redis.error_reply('Stale queue lease') end\n if operation == 'renew' then extend(tonumber(ARGV[4])) end\n if operation == 'complete' then\n meta.status = ARGV[5]\n redis.call('HSET', KEYS[1], 'result', ARGV[4])\n end\n save()\nend\nif operation == 'cancel' then\n if meta.fence ~= tonumber(ARGV[2]) then return redis.error_reply('Stale queue fence') end\n if live() then\n meta.status = 'cancelled'\n meta.fence = meta.fence + 1\n save()\n end\nend\nreturn {request, cjson.encode(meta), redis.call('HGET', KEYS[1], 'result') or ''}\n`;\n"]}
@@ -0,0 +1,3 @@
1
+ import type { BullMQTaskQueueOptions, BullMQTaskQueue } from "./task-queue-bullmq.types.ts";
2
+ export type { BullMQTaskQueueOptions, BullMQTaskQueue, } from "./task-queue-bullmq.types.ts";
3
+ export declare function bullmqTaskQueue(options: BullMQTaskQueueOptions): Promise<BullMQTaskQueue>;
@@ -0,0 +1,124 @@
1
+ import { randomUUID } from "node:crypto";
2
+ import { openBullMQResources } from "./task-queue-bullmq-resources.js";
3
+ import { queueClaim, queueLease, queueLeaseMs, queueNumber, queueRequest, queueResult, queueString, } from "../domain/task-queue.js";
4
+ import { bullMQKey, bullMQState } from "./task-queue-bullmq-state.js";
5
+ import { bullMQClaimBatchSize, bullMQStateScript, } from "./task-queue-bullmq.constants.js";
6
+ export async function bullmqTaskQueue(options) {
7
+ const resources = await openBullMQResources(options);
8
+ const { root, prefix, nativeName, nativeKey, handlerQueue, perform, report } = resources;
9
+ let cursor = 0;
10
+ const client = await root.client;
11
+ client.defineCommand("outpostQueueStateV1", {
12
+ numberOfKeys: 3,
13
+ lua: bullMQStateScript,
14
+ });
15
+ async function state(id, operation, args = [], handler = "") {
16
+ return bullMQState(await client.runCommand("outpostQueueStateV1", [
17
+ `${root.qualifiedName}:outpost:${bullMQKey(id)}`,
18
+ `${nativeKey(handler, id)}:lock`,
19
+ `${prefix}:${nativeName(handler)}:stalled`,
20
+ operation,
21
+ ...args,
22
+ ]));
23
+ }
24
+ async function required(id) {
25
+ const current = await state(id, "get");
26
+ if (!current)
27
+ throw new Error("Queue job does not exist");
28
+ return current;
29
+ }
30
+ async function settle(native, job, token) {
31
+ try {
32
+ if (job.status === "done") {
33
+ await native.moveToCompleted(job.result, token, false);
34
+ return;
35
+ }
36
+ await native.moveToFailed(new Error(job.result?.error ?? "Queue job cancelled"), token, false);
37
+ }
38
+ catch (error) {
39
+ report(error);
40
+ }
41
+ }
42
+ async function ownedOperation(input, operation, value, extra) {
43
+ const lease = queueLease(input);
44
+ const previous = await required(lease.id);
45
+ const current = await state(lease.id, operation, [lease.worker, lease.fence, value, extra], previous.job.handler);
46
+ if (!current)
47
+ throw new Error("Queue job does not exist");
48
+ return current;
49
+ }
50
+ return {
51
+ enqueue(input) {
52
+ return perform(async () => {
53
+ const request = queueRequest(input);
54
+ await state(request.id, "enqueue", [
55
+ JSON.stringify(request),
56
+ request.deadline ?? "",
57
+ ]);
58
+ const { queue } = await handlerQueue(request.handler);
59
+ await queue.add(request.handler, request, {
60
+ jobId: bullMQKey(request.id),
61
+ removeOnComplete: false,
62
+ removeOnFail: false,
63
+ });
64
+ return (await required(request.id)).job;
65
+ });
66
+ },
67
+ get(id) {
68
+ return perform(async () => (await state(queueString(id), "get"))?.job);
69
+ },
70
+ claim(input) {
71
+ return perform(async () => {
72
+ const claim = queueClaim(input);
73
+ const eligible = [...new Set(claim.handlers)];
74
+ const start = cursor++ % eligible.length;
75
+ for (let index = 0; index < eligible.length; index++) {
76
+ const handler = eligible[(start + index) % eligible.length];
77
+ const { worker } = await handlerQueue(handler);
78
+ for (let attempt = 0; attempt < bullMQClaimBatchSize; attempt++) {
79
+ const token = randomUUID();
80
+ const native = await worker.getNextJob(token, { block: false });
81
+ if (!native)
82
+ break;
83
+ const request = queueRequest(native.data);
84
+ const current = await state(request.id, "claim", [claim.worker, token, claim.leaseMs, native.id], handler);
85
+ if (!current)
86
+ throw new Error("Queue job does not exist");
87
+ if (current.job.status === "active")
88
+ return current.job;
89
+ await settle(native, current.job, token);
90
+ }
91
+ }
92
+ return undefined;
93
+ });
94
+ },
95
+ renew(lease, leaseMs) {
96
+ return perform(async () => {
97
+ return (await ownedOperation(lease, "renew", queueLeaseMs(leaseMs), bullMQKey(lease.id))).job;
98
+ });
99
+ },
100
+ complete(lease, input) {
101
+ return perform(async () => {
102
+ const result = queueResult(input);
103
+ const current = await ownedOperation(lease, "complete", JSON.stringify(result), result.error === undefined ? "done" : "failed");
104
+ const { queue } = await handlerQueue(current.job.handler);
105
+ const native = await queue.getJob(bullMQKey(lease.id));
106
+ if (native && current.token)
107
+ await settle(native, current.job, current.token);
108
+ return current.job;
109
+ });
110
+ },
111
+ cancel(id, fence) {
112
+ return perform(async () => {
113
+ const current = await state(queueString(id), "cancel", [
114
+ queueNumber(fence),
115
+ ]);
116
+ if (!current)
117
+ throw new Error("Queue job does not exist");
118
+ return current.job;
119
+ });
120
+ },
121
+ close: resources.close,
122
+ };
123
+ }
124
+ //# sourceMappingURL=task-queue-bullmq.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"task-queue-bullmq.js","sourceRoot":"","sources":["../../src/infrastructure/task-queue-bullmq.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AACvE,OAAO,EACL,UAAU,EACV,UAAU,EACV,YAAY,EACZ,WAAW,EACX,YAAY,EACZ,WAAW,EACX,WAAW,GACZ,MAAM,yBAAyB,CAAC;AAOjC,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AACtE,OAAO,EACL,oBAAoB,EACpB,iBAAiB,GAClB,MAAM,kCAAkC,CAAC;AAO1C,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,OAA+B;IAE/B,MAAM,SAAS,GAAG,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAC;IACrD,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,EAAE,GAC1E,SAAS,CAAC;IACZ,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC;IACjC,MAAM,CAAC,aAAa,CAAC,qBAAqB,EAAE;QAC1C,YAAY,EAAE,CAAC;QACf,GAAG,EAAE,iBAAiB;KACvB,CAAC,CAAC;IACH,KAAK,UAAU,KAAK,CAClB,EAAU,EACV,SAAiB,EACjB,OAAqC,EAAE,EACvC,OAAO,GAAG,EAAE;QAEZ,OAAO,WAAW,CAChB,MAAM,MAAM,CAAC,UAAU,CAAC,qBAAqB,EAAE;YAC7C,GAAG,IAAI,CAAC,aAAa,YAAY,SAAS,CAAC,EAAE,CAAC,EAAE;YAChD,GAAG,SAAS,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO;YAChC,GAAG,MAAM,IAAI,UAAU,CAAC,OAAO,CAAC,UAAU;YAC1C,SAAS;YACT,GAAG,IAAI;SACR,CAAC,CACH,CAAC;IACJ,CAAC;IACD,KAAK,UAAU,QAAQ,CAAC,EAAU;QAChC,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QACvC,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAC1D,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,KAAK,UAAU,MAAM,CAAC,MAAiB,EAAE,GAAa,EAAE,KAAa;QACnE,IAAI,CAAC;YACH,IAAI,GAAG,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;gBAC1B,MAAM,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,MAAO,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;gBACxD,OAAO;YACT,CAAC;YACD,MAAM,MAAM,CAAC,YAAY,CACvB,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,IAAI,qBAAqB,CAAC,EACrD,KAAK,EACL,KAAK,CACN,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,CAAC;QAChB,CAAC;IACH,CAAC;IACD,KAAK,UAAU,cAAc,CAC3B,KAAiB,EACjB,SAA+B,EAC/B,KAAsB,EACtB,KAAa;QAEb,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAChC,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,MAAM,KAAK,CACzB,KAAK,CAAC,EAAE,EACR,SAAS,EACT,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,EACzC,QAAQ,CAAC,GAAG,CAAC,OAAO,CACrB,CAAC;QACF,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAC1D,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,OAAO;QACL,OAAO,CAAC,KAAK;YACX,OAAO,OAAO,CAAC,KAAK,IAAI,EAAE;gBACxB,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpC,MAAM,KAAK,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE;oBACjC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;oBACvB,OAAO,CAAC,QAAQ,IAAI,EAAE;iBACvB,CAAC,CAAC;gBACH,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;gBACtD,MAAM,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE;oBACxC,KAAK,EAAE,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC5B,gBAAgB,EAAE,KAAK;oBACvB,YAAY,EAAE,KAAK;iBACpB,CAAC,CAAC;gBACH,OAAO,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;YAC1C,CAAC,CAAC,CAAC;QACL,CAAC;QACD,GAAG,CAAC,EAAE;YACJ,OAAO,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACzE,CAAC;QACD,KAAK,CAAC,KAAK;YACT,OAAO,OAAO,CAAC,KAAK,IAAI,EAAE;gBACxB,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;gBAChC,MAAM,QAAQ,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAC9C,MAAM,KAAK,GAAG,MAAM,EAAE,GAAG,QAAQ,CAAC,MAAM,CAAC;gBACzC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;oBACrD,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAE,CAAC;oBAC7D,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,CAAC;oBAC/C,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,oBAAoB,EAAE,OAAO,EAAE,EAAE,CAAC;wBAChE,MAAM,KAAK,GAAG,UAAU,EAAE,CAAC;wBAC3B,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;wBAChE,IAAI,CAAC,MAAM;4BAAE,MAAM;wBACnB,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;wBAC1C,MAAM,OAAO,GAAG,MAAM,KAAK,CACzB,OAAO,CAAC,EAAE,EACV,OAAO,EACP,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,EAAG,CAAC,EAChD,OAAO,CACR,CAAC;wBACF,IAAI,CAAC,OAAO;4BAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;wBAC1D,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,KAAK,QAAQ;4BAAE,OAAO,OAAO,CAAC,GAAG,CAAC;wBACxD,MAAM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;oBAC3C,CAAC;gBACH,CAAC;gBACD,OAAO,SAAS,CAAC;YACnB,CAAC,CAAC,CAAC;QACL,CAAC;QACD,KAAK,CAAC,KAAK,EAAE,OAAO;YAClB,OAAO,OAAO,CAAC,KAAK,IAAI,EAAE;gBACxB,OAAO,CACL,MAAM,cAAc,CAClB,KAAK,EACL,OAAO,EACP,YAAY,CAAC,OAAO,CAAC,EACrB,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CACpB,CACF,CAAC,GAAG,CAAC;YACR,CAAC,CAAC,CAAC;QACL,CAAC;QACD,QAAQ,CAAC,KAAK,EAAE,KAAK;YACnB,OAAO,OAAO,CAAC,KAAK,IAAI,EAAE;gBACxB,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;gBAClC,MAAM,OAAO,GAAG,MAAM,cAAc,CAClC,KAAK,EACL,UAAU,EACV,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EACtB,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAC/C,CAAC;gBACF,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBAC1D,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;gBACvD,IAAI,MAAM,IAAI,OAAO,CAAC,KAAK;oBACzB,MAAM,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;gBACnD,OAAO,OAAO,CAAC,GAAG,CAAC;YACrB,CAAC,CAAC,CAAC;QACL,CAAC;QACD,MAAM,CAAC,EAAE,EAAE,KAAK;YACd,OAAO,OAAO,CAAC,KAAK,IAAI,EAAE;gBACxB,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,QAAQ,EAAE;oBACrD,WAAW,CAAC,KAAK,CAAC;iBACnB,CAAC,CAAC;gBACH,IAAI,CAAC,OAAO;oBAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;gBAC1D,OAAO,OAAO,CAAC,GAAG,CAAC;YACrB,CAAC,CAAC,CAAC;QACL,CAAC;QACD,KAAK,EAAE,SAAS,CAAC,KAAK;KACvB,CAAC;AACJ,CAAC","sourcesContent":["import { randomUUID } from \"node:crypto\";\nimport { openBullMQResources } from \"./task-queue-bullmq-resources.ts\";\nimport {\n queueClaim,\n queueLease,\n queueLeaseMs,\n queueNumber,\n queueRequest,\n queueResult,\n queueString,\n} from \"../domain/task-queue.ts\";\nimport type { QueueJob, QueueLease } from \"../domain/task-queue.types.ts\";\nimport type {\n BullMQTaskQueueOptions,\n BullMQTaskQueue,\n BullMQJob,\n} from \"./task-queue-bullmq.types.ts\";\nimport { bullMQKey, bullMQState } from \"./task-queue-bullmq-state.ts\";\nimport {\n bullMQClaimBatchSize,\n bullMQStateScript,\n} from \"./task-queue-bullmq.constants.ts\";\n\nexport type {\n BullMQTaskQueueOptions,\n BullMQTaskQueue,\n} from \"./task-queue-bullmq.types.ts\";\n\nexport async function bullmqTaskQueue(\n options: BullMQTaskQueueOptions,\n): Promise<BullMQTaskQueue> {\n const resources = await openBullMQResources(options);\n const { root, prefix, nativeName, nativeKey, handlerQueue, perform, report } =\n resources;\n let cursor = 0;\n const client = await root.client;\n client.defineCommand(\"outpostQueueStateV1\", {\n numberOfKeys: 3,\n lua: bullMQStateScript,\n });\n async function state(\n id: string,\n operation: string,\n args: readonly (string | number)[] = [],\n handler = \"\",\n ) {\n return bullMQState(\n await client.runCommand(\"outpostQueueStateV1\", [\n `${root.qualifiedName}:outpost:${bullMQKey(id)}`,\n `${nativeKey(handler, id)}:lock`,\n `${prefix}:${nativeName(handler)}:stalled`,\n operation,\n ...args,\n ]),\n );\n }\n async function required(id: string) {\n const current = await state(id, \"get\");\n if (!current) throw new Error(\"Queue job does not exist\");\n return current;\n }\n async function settle(native: BullMQJob, job: QueueJob, token: string) {\n try {\n if (job.status === \"done\") {\n await native.moveToCompleted(job.result!, token, false);\n return;\n }\n await native.moveToFailed(\n new Error(job.result?.error ?? \"Queue job cancelled\"),\n token,\n false,\n );\n } catch (error) {\n report(error);\n }\n }\n async function ownedOperation(\n input: QueueLease,\n operation: \"renew\" | \"complete\",\n value: string | number,\n extra: string,\n ) {\n const lease = queueLease(input);\n const previous = await required(lease.id);\n const current = await state(\n lease.id,\n operation,\n [lease.worker, lease.fence, value, extra],\n previous.job.handler,\n );\n if (!current) throw new Error(\"Queue job does not exist\");\n return current;\n }\n return {\n enqueue(input) {\n return perform(async () => {\n const request = queueRequest(input);\n await state(request.id, \"enqueue\", [\n JSON.stringify(request),\n request.deadline ?? \"\",\n ]);\n const { queue } = await handlerQueue(request.handler);\n await queue.add(request.handler, request, {\n jobId: bullMQKey(request.id),\n removeOnComplete: false,\n removeOnFail: false,\n });\n return (await required(request.id)).job;\n });\n },\n get(id) {\n return perform(async () => (await state(queueString(id), \"get\"))?.job);\n },\n claim(input) {\n return perform(async () => {\n const claim = queueClaim(input);\n const eligible = [...new Set(claim.handlers)];\n const start = cursor++ % eligible.length;\n for (let index = 0; index < eligible.length; index++) {\n const handler = eligible[(start + index) % eligible.length]!;\n const { worker } = await handlerQueue(handler);\n for (let attempt = 0; attempt < bullMQClaimBatchSize; attempt++) {\n const token = randomUUID();\n const native = await worker.getNextJob(token, { block: false });\n if (!native) break;\n const request = queueRequest(native.data);\n const current = await state(\n request.id,\n \"claim\",\n [claim.worker, token, claim.leaseMs, native.id!],\n handler,\n );\n if (!current) throw new Error(\"Queue job does not exist\");\n if (current.job.status === \"active\") return current.job;\n await settle(native, current.job, token);\n }\n }\n return undefined;\n });\n },\n renew(lease, leaseMs) {\n return perform(async () => {\n return (\n await ownedOperation(\n lease,\n \"renew\",\n queueLeaseMs(leaseMs),\n bullMQKey(lease.id),\n )\n ).job;\n });\n },\n complete(lease, input) {\n return perform(async () => {\n const result = queueResult(input);\n const current = await ownedOperation(\n lease,\n \"complete\",\n JSON.stringify(result),\n result.error === undefined ? \"done\" : \"failed\",\n );\n const { queue } = await handlerQueue(current.job.handler);\n const native = await queue.getJob(bullMQKey(lease.id));\n if (native && current.token)\n await settle(native, current.job, current.token);\n return current.job;\n });\n },\n cancel(id, fence) {\n return perform(async () => {\n const current = await state(queueString(id), \"cancel\", [\n queueNumber(fence),\n ]);\n if (!current) throw new Error(\"Queue job does not exist\");\n return current.job;\n });\n },\n close: resources.close,\n };\n}\n"]}
@@ -0,0 +1,17 @@
1
+ import type { RedisOptions, Job, Queue, Worker } from "bullmq";
2
+ import type { TaskQueue, QueueRequest, QueueResult } from "../domain/task-queue.types.ts";
3
+ export interface BullMQTaskQueueOptions {
4
+ readonly name: string;
5
+ readonly connection: RedisOptions;
6
+ readonly prefix?: string;
7
+ readonly stalledIntervalMs?: number;
8
+ readonly onError?: (error: Error) => void;
9
+ }
10
+ export interface BullMQTaskQueue extends TaskQueue {
11
+ close(): Promise<void>;
12
+ }
13
+ export interface BullMQHandlerQueue {
14
+ readonly queue: Queue<QueueRequest, QueueResult>;
15
+ readonly worker: Worker<QueueRequest, QueueResult>;
16
+ }
17
+ export type BullMQJob = Job<QueueRequest, QueueResult>;
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=task-queue-bullmq.types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"task-queue-bullmq.types.js","sourceRoot":"","sources":["../../src/infrastructure/task-queue-bullmq.types.ts"],"names":[],"mappings":"","sourcesContent":["import type { RedisOptions, Job, Queue, Worker } from \"bullmq\";\nimport type {\n TaskQueue,\n QueueRequest,\n QueueResult,\n} from \"../domain/task-queue.types.ts\";\n\nexport interface BullMQTaskQueueOptions {\n readonly name: string;\n readonly connection: RedisOptions;\n readonly prefix?: string;\n readonly stalledIntervalMs?: number;\n readonly onError?: (error: Error) => void;\n}\nexport interface BullMQTaskQueue extends TaskQueue {\n close(): Promise<void>;\n}\nexport interface BullMQHandlerQueue {\n readonly queue: Queue<QueueRequest, QueueResult>;\n readonly worker: Worker<QueueRequest, QueueResult>;\n}\nexport type BullMQJob = Job<QueueRequest, QueueResult>;\n"]}
@@ -0,0 +1,6 @@
1
+ export declare const archiveDefaults: Readonly<{
2
+ chunkBytes: number;
3
+ maxBytes: number;
4
+ maxFiles: 100000;
5
+ manifestBytes: number;
6
+ }>;
@@ -0,0 +1,7 @@
1
+ export const archiveDefaults = Object.freeze({
2
+ chunkBytes: 4 * 1024 * 1024,
3
+ maxBytes: 1024 * 1024 * 1024,
4
+ maxFiles: 100_000,
5
+ manifestBytes: 16 * 1024 * 1024,
6
+ });
7
+ //# sourceMappingURL=transport-archive.constants.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"transport-archive.constants.js","sourceRoot":"","sources":["../../src/infrastructure/transport-archive.constants.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC;IAC3C,UAAU,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI;IAC3B,QAAQ,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI;IAC5B,QAAQ,EAAE,OAAO;IACjB,aAAa,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;CAChC,CAAC,CAAC","sourcesContent":["export const archiveDefaults = Object.freeze({\n chunkBytes: 4 * 1024 * 1024,\n maxBytes: 1024 * 1024 * 1024,\n maxFiles: 100_000,\n manifestBytes: 16 * 1024 * 1024,\n});\n"]}
@@ -0,0 +1,5 @@
1
+ import type { Transport, TransportReference } from "../domain/transport.types.ts";
2
+ import type { ArchiveManifest } from "./transport-archive.types.ts";
3
+ export declare function archiveFiles(transporter: Transport, root: string, paths: readonly string[], prefix: string, maxBytes?: number): Promise<TransportReference>;
4
+ export declare function archiveManifest(value: unknown): ArchiveManifest;
5
+ export declare function restoreArchiveFiles(transporter: Transport, reference: TransportReference, destination: string, maxBytes?: number): Promise<void>;
@@ -0,0 +1,157 @@
1
+ import { createHash } from "node:crypto";
2
+ import { lstat, open, readlink, mkdir, symlink, chmod } from "node:fs/promises";
3
+ import { dirname } from "node:path";
4
+ import { inspectionFileFlags } from "./inspection-file.constants.js";
5
+ import { invariant, positive } from "../domain/errors.js";
6
+ import { safeDestination } from "./files.js";
7
+ import { jsonBytes, jsonObject, readReference, transportReference, } from "./transport-json.js";
8
+ import { archiveDefaults as defaults } from "./transport-archive.constants.js";
9
+ const digest = (bytes) => createHash("sha256").update(bytes).digest("hex");
10
+ export async function archiveFiles(transporter, root, paths, prefix, maxBytes = defaults.maxBytes) {
11
+ positive(maxBytes, "maxBytes");
12
+ invariant(paths.length <= defaults.maxFiles && new Set(paths).size === paths.length, "Invalid archive path count");
13
+ const files = [];
14
+ let total = 0;
15
+ for (const path of paths) {
16
+ const source = await safeDestination(root, path);
17
+ const info = await lstat(source);
18
+ invariant(info.isFile() || info.isSymbolicLink(), "Unsupported archive entry");
19
+ const chunks = [];
20
+ let size = 0;
21
+ const publish = async (bytes) => {
22
+ total += bytes.length;
23
+ size += bytes.length;
24
+ invariant(total <= maxBytes, "Archive exceeds byte limit");
25
+ const reference = await transporter.write(`${prefix}/files/${files.length}/${chunks.length}`, bytes, { ifRevision: null });
26
+ chunks.push({ reference, sha256: digest(bytes), size: bytes.length });
27
+ };
28
+ if (info.isSymbolicLink())
29
+ await publish(Buffer.from(await readlink(source)));
30
+ if (info.isFile()) {
31
+ const file = await open(source, inspectionFileFlags);
32
+ try {
33
+ const buffer = Buffer.alloc(defaults.chunkBytes);
34
+ for (;;) {
35
+ const { bytesRead } = await file.read(buffer, 0, buffer.length, null);
36
+ if (!bytesRead)
37
+ break;
38
+ await publish(Buffer.from(buffer.subarray(0, bytesRead)));
39
+ }
40
+ const after = await file.stat();
41
+ invariant(info.ino === after.ino &&
42
+ info.dev === after.dev &&
43
+ info.size === size &&
44
+ info.mtimeMs === after.mtimeMs &&
45
+ info.ctimeMs === after.ctimeMs, "Archive source changed");
46
+ }
47
+ finally {
48
+ await file.close();
49
+ }
50
+ }
51
+ files.push({
52
+ path,
53
+ kind: info.isSymbolicLink() ? "symlink" : "file",
54
+ mode: info.mode & 0o777,
55
+ size,
56
+ chunks,
57
+ });
58
+ }
59
+ const manifest = { format: 1, files };
60
+ const bytes = jsonBytes(manifest);
61
+ invariant(bytes.length <= defaults.manifestBytes, "Archive manifest exceeds byte limit");
62
+ return transporter.write(`${prefix}/manifest`, bytes, { ifRevision: null });
63
+ }
64
+ export function archiveManifest(value) {
65
+ invariant(value &&
66
+ typeof value === "object" &&
67
+ "format" in value &&
68
+ value.format === 1 &&
69
+ "files" in value &&
70
+ Array.isArray(value.files) &&
71
+ value.files.length <= defaults.maxFiles, "Invalid archive manifest");
72
+ const files = value.files.map((file) => {
73
+ invariant(file &&
74
+ typeof file === "object" &&
75
+ "path" in file &&
76
+ typeof file.path === "string" &&
77
+ "kind" in file &&
78
+ (file.kind === "file" || file.kind === "symlink") &&
79
+ "mode" in file &&
80
+ typeof file.mode === "number" &&
81
+ Number.isInteger(file.mode) &&
82
+ file.mode >= 0 &&
83
+ file.mode <= 0o777 &&
84
+ "size" in file &&
85
+ typeof file.size === "number" &&
86
+ Number.isSafeInteger(file.size) &&
87
+ file.size >= 0 &&
88
+ "chunks" in file &&
89
+ Array.isArray(file.chunks), "Invalid archive file");
90
+ const chunks = file.chunks.map((chunk) => {
91
+ invariant(chunk &&
92
+ typeof chunk === "object" &&
93
+ "reference" in chunk &&
94
+ "sha256" in chunk &&
95
+ typeof chunk.sha256 === "string" &&
96
+ /^[a-f0-9]{64}$/.test(chunk.sha256) &&
97
+ "size" in chunk &&
98
+ typeof chunk.size === "number" &&
99
+ Number.isSafeInteger(chunk.size) &&
100
+ chunk.size >= 0 &&
101
+ chunk.size <= defaults.chunkBytes, "Invalid archive chunk");
102
+ return {
103
+ reference: transportReference(chunk.reference),
104
+ sha256: chunk.sha256,
105
+ size: chunk.size,
106
+ };
107
+ });
108
+ invariant(chunks.reduce((sum, chunk) => sum + chunk.size, 0) === file.size, "Archive file size mismatch");
109
+ return {
110
+ path: file.path,
111
+ kind: file.kind,
112
+ mode: file.mode,
113
+ size: file.size,
114
+ chunks,
115
+ };
116
+ });
117
+ invariant(new Set(files.map((file) => file.path)).size === files.length, "Duplicate archive paths");
118
+ return { format: 1, files };
119
+ }
120
+ export async function restoreArchiveFiles(transporter, reference, destination, maxBytes = defaults.maxBytes) {
121
+ positive(maxBytes, "maxBytes");
122
+ const manifest = archiveManifest(jsonObject(await readReference(transporter, reference, defaults.manifestBytes)));
123
+ const total = manifest.files.reduce((sum, file) => sum + file.size, 0);
124
+ invariant(Number.isSafeInteger(total) && total <= maxBytes, "Archive exceeds byte limit");
125
+ for (const file of manifest.files)
126
+ await safeDestination(destination, file.path);
127
+ await mkdir(destination, { mode: 0o700 });
128
+ for (const entry of manifest.files) {
129
+ const target = await safeDestination(destination, entry.path);
130
+ await mkdir(dirname(target), { recursive: true, mode: 0o700 });
131
+ const data = async (chunk) => {
132
+ const value = await readReference(transporter, chunk.reference, chunk.size);
133
+ invariant(value.bytes.length === chunk.size &&
134
+ digest(value.bytes) === chunk.sha256, "Archive chunk integrity mismatch");
135
+ return value.bytes;
136
+ };
137
+ if (entry.kind === "symlink") {
138
+ invariant(entry.size <= defaults.chunkBytes, "Archive symlink exceeds limit");
139
+ const chunks = [];
140
+ for (const chunk of entry.chunks)
141
+ chunks.push(await data(chunk));
142
+ await symlink(Buffer.concat(chunks).toString(), target);
143
+ continue;
144
+ }
145
+ const file = await open(target, "wx", 0o600);
146
+ try {
147
+ for (const chunk of entry.chunks)
148
+ await file.writeFile(await data(chunk));
149
+ await file.sync();
150
+ }
151
+ finally {
152
+ await file.close();
153
+ }
154
+ await chmod(target, entry.mode);
155
+ }
156
+ }
157
+ //# sourceMappingURL=transport-archive.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"transport-archive.js","sourceRoot":"","sources":["../../src/infrastructure/transport-archive.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAChF,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,mBAAmB,EAAE,MAAM,gCAAgC,CAAC;AAKrE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EACL,SAAS,EACT,UAAU,EACV,aAAa,EACb,kBAAkB,GACnB,MAAM,qBAAqB,CAAC;AAM7B,OAAO,EAAE,eAAe,IAAI,QAAQ,EAAE,MAAM,kCAAkC,CAAC;AAE/E,MAAM,MAAM,GAAG,CAAC,KAAiB,EAAE,EAAE,CACnC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAEnD,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,WAAsB,EACtB,IAAY,EACZ,KAAwB,EACxB,MAAc,EACd,QAAQ,GAAG,QAAQ,CAAC,QAAQ;IAE5B,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC/B,SAAS,CACP,KAAK,CAAC,MAAM,IAAI,QAAQ,CAAC,QAAQ,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,MAAM,EACzE,4BAA4B,CAC7B,CAAC;IACF,MAAM,KAAK,GAAkB,EAAE,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACjD,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC;QACjC,SAAS,CACP,IAAI,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,cAAc,EAAE,EACtC,2BAA2B,CAC5B,CAAC;QACF,MAAM,MAAM,GAAmB,EAAE,CAAC;QAClC,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,MAAM,OAAO,GAAG,KAAK,EAAE,KAAiB,EAAE,EAAE;YAC1C,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;YACtB,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;YACrB,SAAS,CAAC,KAAK,IAAI,QAAQ,EAAE,4BAA4B,CAAC,CAAC;YAC3D,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,KAAK,CACvC,GAAG,MAAM,UAAU,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,EAClD,KAAK,EACL,EAAE,UAAU,EAAE,IAAI,EAAE,CACrB,CAAC;YACF,MAAM,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QACxE,CAAC,CAAC;QACF,IAAI,IAAI,CAAC,cAAc,EAAE;YACvB,MAAM,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACrD,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;YAClB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC;YACrD,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;gBACjD,SAAS,CAAC;oBACR,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;oBACtE,IAAI,CAAC,SAAS;wBAAE,MAAM;oBACtB,MAAM,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC;gBAC5D,CAAC;gBACD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;gBAChC,SAAS,CACP,IAAI,CAAC,GAAG,KAAK,KAAK,CAAC,GAAG;oBACpB,IAAI,CAAC,GAAG,KAAK,KAAK,CAAC,GAAG;oBACtB,IAAI,CAAC,IAAI,KAAK,IAAI;oBAClB,IAAI,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO;oBAC9B,IAAI,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO,EAChC,wBAAwB,CACzB,CAAC;YACJ,CAAC;oBAAS,CAAC;gBACT,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACrB,CAAC;QACH,CAAC;QACD,KAAK,CAAC,IAAI,CAAC;YACT,IAAI;YACJ,IAAI,EAAE,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM;YAChD,IAAI,EAAE,IAAI,CAAC,IAAI,GAAG,KAAK;YACvB,IAAI;YACJ,MAAM;SACP,CAAC,CAAC;IACL,CAAC;IACD,MAAM,QAAQ,GAAoB,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC;IACvD,MAAM,KAAK,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;IAClC,SAAS,CACP,KAAK,CAAC,MAAM,IAAI,QAAQ,CAAC,aAAa,EACtC,qCAAqC,CACtC,CAAC;IACF,OAAO,WAAW,CAAC,KAAK,CAAC,GAAG,MAAM,WAAW,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;AAC9E,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,KAAc;IAC5C,SAAS,CACP,KAAK;QACH,OAAO,KAAK,KAAK,QAAQ;QACzB,QAAQ,IAAI,KAAK;QACjB,KAAK,CAAC,MAAM,KAAK,CAAC;QAClB,OAAO,IAAI,KAAK;QAChB,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC;QAC1B,KAAK,CAAC,KAAK,CAAC,MAAM,IAAI,QAAQ,CAAC,QAAQ,EACzC,0BAA0B,CAC3B,CAAC;IACF,MAAM,KAAK,GAAkB,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAa,EAAE,EAAE;QAC7D,SAAS,CACP,IAAI;YACF,OAAO,IAAI,KAAK,QAAQ;YACxB,MAAM,IAAI,IAAI;YACd,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ;YAC7B,MAAM,IAAI,IAAI;YACd,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC;YACjD,MAAM,IAAI,IAAI;YACd,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ;YAC7B,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;YAC3B,IAAI,CAAC,IAAI,IAAI,CAAC;YACd,IAAI,CAAC,IAAI,IAAI,KAAK;YAClB,MAAM,IAAI,IAAI;YACd,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ;YAC7B,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC;YAC/B,IAAI,CAAC,IAAI,IAAI,CAAC;YACd,QAAQ,IAAI,IAAI;YAChB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAC5B,sBAAsB,CACvB,CAAC;QACF,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAc,EAAgB,EAAE;YAC9D,SAAS,CACP,KAAK;gBACH,OAAO,KAAK,KAAK,QAAQ;gBACzB,WAAW,IAAI,KAAK;gBACpB,QAAQ,IAAI,KAAK;gBACjB,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ;gBAChC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;gBACnC,MAAM,IAAI,KAAK;gBACf,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ;gBAC9B,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC;gBAChC,KAAK,CAAC,IAAI,IAAI,CAAC;gBACf,KAAK,CAAC,IAAI,IAAI,QAAQ,CAAC,UAAU,EACnC,uBAAuB,CACxB,CAAC;YACF,OAAO;gBACL,SAAS,EAAE,kBAAkB,CAAC,KAAK,CAAC,SAAS,CAAC;gBAC9C,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,IAAI,EAAE,KAAK,CAAC,IAAI;aACjB,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,SAAS,CACP,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,EAChE,4BAA4B,CAC7B,CAAC;QACF,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM;SACP,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,SAAS,CACP,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,MAAM,EAC7D,yBAAyB,CAC1B,CAAC;IACF,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC;AAC9B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,WAAsB,EACtB,SAA6B,EAC7B,WAAmB,EACnB,QAAQ,GAAG,QAAQ,CAAC,QAAQ;IAE5B,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC/B,MAAM,QAAQ,GAAG,eAAe,CAC9B,UAAU,CACR,MAAM,aAAa,CAAC,WAAW,EAAE,SAAS,EAAE,QAAQ,CAAC,aAAa,CAAC,CACpE,CACF,CAAC;IACF,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACvE,SAAS,CACP,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,QAAQ,EAChD,4BAA4B,CAC7B,CAAC;IACF,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK;QAC/B,MAAM,eAAe,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAChD,MAAM,KAAK,CAAC,WAAW,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC1C,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,WAAW,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAC9D,MAAM,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/D,MAAM,IAAI,GAAG,KAAK,EAAE,KAAmB,EAAE,EAAE;YACzC,MAAM,KAAK,GAAG,MAAM,aAAa,CAC/B,WAAW,EACX,KAAK,CAAC,SAAS,EACf,KAAK,CAAC,IAAI,CACX,CAAC;YACF,SAAS,CACP,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,IAAI;gBAC/B,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,MAAM,EACtC,kCAAkC,CACnC,CAAC;YACF,OAAO,KAAK,CAAC,KAAK,CAAC;QACrB,CAAC,CAAC;QACF,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC7B,SAAS,CACP,KAAK,CAAC,IAAI,IAAI,QAAQ,CAAC,UAAU,EACjC,+BAA+B,CAChC,CAAC;YACF,MAAM,MAAM,GAAG,EAAE,CAAC;YAClB,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM;gBAAE,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;YACjE,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC,CAAC;YACxD,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QAC7C,IAAI,CAAC;YACH,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM;gBAAE,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;YAC1E,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QACpB,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;QACD,MAAM,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;AACH,CAAC","sourcesContent":["import { createHash } from \"node:crypto\";\nimport { lstat, open, readlink, mkdir, symlink, chmod } from \"node:fs/promises\";\nimport { dirname } from \"node:path\";\nimport { inspectionFileFlags } from \"./inspection-file.constants.ts\";\nimport type {\n Transport,\n TransportReference,\n} from \"../domain/transport.types.ts\";\nimport { invariant, positive } from \"../domain/errors.ts\";\nimport { safeDestination } from \"./files.ts\";\nimport {\n jsonBytes,\n jsonObject,\n readReference,\n transportReference,\n} from \"./transport-json.ts\";\nimport type {\n ArchiveManifest,\n ArchiveFile,\n ArchiveChunk,\n} from \"./transport-archive.types.ts\";\nimport { archiveDefaults as defaults } from \"./transport-archive.constants.ts\";\n\nconst digest = (bytes: Uint8Array) =>\n createHash(\"sha256\").update(bytes).digest(\"hex\");\n\nexport async function archiveFiles(\n transporter: Transport,\n root: string,\n paths: readonly string[],\n prefix: string,\n maxBytes = defaults.maxBytes,\n): Promise<TransportReference> {\n positive(maxBytes, \"maxBytes\");\n invariant(\n paths.length <= defaults.maxFiles && new Set(paths).size === paths.length,\n \"Invalid archive path count\",\n );\n const files: ArchiveFile[] = [];\n let total = 0;\n for (const path of paths) {\n const source = await safeDestination(root, path);\n const info = await lstat(source);\n invariant(\n info.isFile() || info.isSymbolicLink(),\n \"Unsupported archive entry\",\n );\n const chunks: ArchiveChunk[] = [];\n let size = 0;\n const publish = async (bytes: Uint8Array) => {\n total += bytes.length;\n size += bytes.length;\n invariant(total <= maxBytes, \"Archive exceeds byte limit\");\n const reference = await transporter.write(\n `${prefix}/files/${files.length}/${chunks.length}`,\n bytes,\n { ifRevision: null },\n );\n chunks.push({ reference, sha256: digest(bytes), size: bytes.length });\n };\n if (info.isSymbolicLink())\n await publish(Buffer.from(await readlink(source)));\n if (info.isFile()) {\n const file = await open(source, inspectionFileFlags);\n try {\n const buffer = Buffer.alloc(defaults.chunkBytes);\n for (;;) {\n const { bytesRead } = await file.read(buffer, 0, buffer.length, null);\n if (!bytesRead) break;\n await publish(Buffer.from(buffer.subarray(0, bytesRead)));\n }\n const after = await file.stat();\n invariant(\n info.ino === after.ino &&\n info.dev === after.dev &&\n info.size === size &&\n info.mtimeMs === after.mtimeMs &&\n info.ctimeMs === after.ctimeMs,\n \"Archive source changed\",\n );\n } finally {\n await file.close();\n }\n }\n files.push({\n path,\n kind: info.isSymbolicLink() ? \"symlink\" : \"file\",\n mode: info.mode & 0o777,\n size,\n chunks,\n });\n }\n const manifest: ArchiveManifest = { format: 1, files };\n const bytes = jsonBytes(manifest);\n invariant(\n bytes.length <= defaults.manifestBytes,\n \"Archive manifest exceeds byte limit\",\n );\n return transporter.write(`${prefix}/manifest`, bytes, { ifRevision: null });\n}\n\nexport function archiveManifest(value: unknown): ArchiveManifest {\n invariant(\n value &&\n typeof value === \"object\" &&\n \"format\" in value &&\n value.format === 1 &&\n \"files\" in value &&\n Array.isArray(value.files) &&\n value.files.length <= defaults.maxFiles,\n \"Invalid archive manifest\",\n );\n const files: ArchiveFile[] = value.files.map((file: unknown) => {\n invariant(\n file &&\n typeof file === \"object\" &&\n \"path\" in file &&\n typeof file.path === \"string\" &&\n \"kind\" in file &&\n (file.kind === \"file\" || file.kind === \"symlink\") &&\n \"mode\" in file &&\n typeof file.mode === \"number\" &&\n Number.isInteger(file.mode) &&\n file.mode >= 0 &&\n file.mode <= 0o777 &&\n \"size\" in file &&\n typeof file.size === \"number\" &&\n Number.isSafeInteger(file.size) &&\n file.size >= 0 &&\n \"chunks\" in file &&\n Array.isArray(file.chunks),\n \"Invalid archive file\",\n );\n const chunks = file.chunks.map((chunk: unknown): ArchiveChunk => {\n invariant(\n chunk &&\n typeof chunk === \"object\" &&\n \"reference\" in chunk &&\n \"sha256\" in chunk &&\n typeof chunk.sha256 === \"string\" &&\n /^[a-f0-9]{64}$/.test(chunk.sha256) &&\n \"size\" in chunk &&\n typeof chunk.size === \"number\" &&\n Number.isSafeInteger(chunk.size) &&\n chunk.size >= 0 &&\n chunk.size <= defaults.chunkBytes,\n \"Invalid archive chunk\",\n );\n return {\n reference: transportReference(chunk.reference),\n sha256: chunk.sha256,\n size: chunk.size,\n };\n });\n invariant(\n chunks.reduce((sum, chunk) => sum + chunk.size, 0) === file.size,\n \"Archive file size mismatch\",\n );\n return {\n path: file.path,\n kind: file.kind,\n mode: file.mode,\n size: file.size,\n chunks,\n };\n });\n invariant(\n new Set(files.map((file) => file.path)).size === files.length,\n \"Duplicate archive paths\",\n );\n return { format: 1, files };\n}\n\nexport async function restoreArchiveFiles(\n transporter: Transport,\n reference: TransportReference,\n destination: string,\n maxBytes = defaults.maxBytes,\n): Promise<void> {\n positive(maxBytes, \"maxBytes\");\n const manifest = archiveManifest(\n jsonObject(\n await readReference(transporter, reference, defaults.manifestBytes),\n ),\n );\n const total = manifest.files.reduce((sum, file) => sum + file.size, 0);\n invariant(\n Number.isSafeInteger(total) && total <= maxBytes,\n \"Archive exceeds byte limit\",\n );\n for (const file of manifest.files)\n await safeDestination(destination, file.path);\n await mkdir(destination, { mode: 0o700 });\n for (const entry of manifest.files) {\n const target = await safeDestination(destination, entry.path);\n await mkdir(dirname(target), { recursive: true, mode: 0o700 });\n const data = async (chunk: ArchiveChunk) => {\n const value = await readReference(\n transporter,\n chunk.reference,\n chunk.size,\n );\n invariant(\n value.bytes.length === chunk.size &&\n digest(value.bytes) === chunk.sha256,\n \"Archive chunk integrity mismatch\",\n );\n return value.bytes;\n };\n if (entry.kind === \"symlink\") {\n invariant(\n entry.size <= defaults.chunkBytes,\n \"Archive symlink exceeds limit\",\n );\n const chunks = [];\n for (const chunk of entry.chunks) chunks.push(await data(chunk));\n await symlink(Buffer.concat(chunks).toString(), target);\n continue;\n }\n const file = await open(target, \"wx\", 0o600);\n try {\n for (const chunk of entry.chunks) await file.writeFile(await data(chunk));\n await file.sync();\n } finally {\n await file.close();\n }\n await chmod(target, entry.mode);\n }\n}\n"]}
@@ -0,0 +1,17 @@
1
+ import type { TransportReference } from "../domain/transport.types.ts";
2
+ export interface ArchiveChunk {
3
+ readonly reference: TransportReference;
4
+ readonly sha256: string;
5
+ readonly size: number;
6
+ }
7
+ export interface ArchiveFile {
8
+ readonly path: string;
9
+ readonly kind: "file" | "symlink";
10
+ readonly mode: number;
11
+ readonly size: number;
12
+ readonly chunks: readonly ArchiveChunk[];
13
+ }
14
+ export interface ArchiveManifest {
15
+ readonly format: 1;
16
+ readonly files: readonly ArchiveFile[];
17
+ }
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=transport-archive.types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"transport-archive.types.js","sourceRoot":"","sources":["../../src/infrastructure/transport-archive.types.ts"],"names":[],"mappings":"","sourcesContent":["import type { TransportReference } from \"../domain/transport.types.ts\";\nexport interface ArchiveChunk {\n readonly reference: TransportReference;\n readonly sha256: string;\n readonly size: number;\n}\nexport interface ArchiveFile {\n readonly path: string;\n readonly kind: \"file\" | \"symlink\";\n readonly mode: number;\n readonly size: number;\n readonly chunks: readonly ArchiveChunk[];\n}\nexport interface ArchiveManifest {\n readonly format: 1;\n readonly files: readonly ArchiveFile[];\n}\n"]}
@@ -0,0 +1,3 @@
1
+ import type { ArtifactStore } from "../domain/artifact.types.ts";
2
+ import type { ArtifactStoreOptions } from "./transport-artifact-store.types.ts";
3
+ export declare function artifactStore(options: ArtifactStoreOptions): ArtifactStore;
@@ -0,0 +1,31 @@
1
+ import { artifactDigestPattern } from "../domain/artifact.constants.js";
2
+ import { immutableObject } from "./transport-json.js";
3
+ import { artifactMaxBytes } from "./artifact-store.constants.js";
4
+ import { readLimit } from "./transport-envelope.js";
5
+ export function artifactStore(options) {
6
+ const maxBytes = readLimit(options.maxBytes ?? artifactMaxBytes);
7
+ if (!maxBytes)
8
+ throw new Error("Artifact maxBytes must be positive");
9
+ const key = (id) => {
10
+ if (!artifactDigestPattern.test(id))
11
+ throw new Error("Invalid artifact storage ID");
12
+ return `artifacts/${id}.blob`;
13
+ };
14
+ return {
15
+ async put(id, input, signal) {
16
+ if (input.byteLength > maxBytes)
17
+ throw new Error("Artifact exceeds maxBytes");
18
+ await immutableObject(options.transporter, key(id), Buffer.from(input), signal);
19
+ },
20
+ async get(id, signal) {
21
+ const value = await options.transporter.read(key(id), {
22
+ maxBytes,
23
+ ...(signal ? { signal } : {}),
24
+ });
25
+ if (!value)
26
+ throw new Error("Artifact does not exist");
27
+ return value.bytes;
28
+ },
29
+ };
30
+ }
31
+ //# sourceMappingURL=transport-artifact-store.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"transport-artifact-store.js","sourceRoot":"","sources":["../../src/infrastructure/transport-artifact-store.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,iCAAiC,CAAC;AAExE,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AACjE,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAEpD,MAAM,UAAU,aAAa,CAAC,OAA6B;IACzD,MAAM,QAAQ,GAAG,SAAS,CAAC,OAAO,CAAC,QAAQ,IAAI,gBAAgB,CAAC,CAAC;IACjE,IAAI,CAAC,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;IACrE,MAAM,GAAG,GAAG,CAAC,EAAU,EAAE,EAAE;QACzB,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACjD,OAAO,aAAa,EAAE,OAAO,CAAC;IAChC,CAAC,CAAC;IACF,OAAO;QACL,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM;YACzB,IAAI,KAAK,CAAC,UAAU,GAAG,QAAQ;gBAC7B,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;YAC/C,MAAM,eAAe,CACnB,OAAO,CAAC,WAAW,EACnB,GAAG,CAAC,EAAE,CAAC,EACP,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAClB,MAAM,CACP,CAAC;QACJ,CAAC;QACD,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM;YAClB,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;gBACpD,QAAQ;gBACR,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC9B,CAAC,CAAC;YACH,IAAI,CAAC,KAAK;gBAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACvD,OAAO,KAAK,CAAC,KAAK,CAAC;QACrB,CAAC;KACF,CAAC;AACJ,CAAC","sourcesContent":["import type { ArtifactStore } from \"../domain/artifact.types.ts\";\nimport { artifactDigestPattern } from \"../domain/artifact.constants.ts\";\nimport type { ArtifactStoreOptions } from \"./transport-artifact-store.types.ts\";\nimport { immutableObject } from \"./transport-json.ts\";\nimport { artifactMaxBytes } from \"./artifact-store.constants.ts\";\nimport { readLimit } from \"./transport-envelope.ts\";\n\nexport function artifactStore(options: ArtifactStoreOptions): ArtifactStore {\n const maxBytes = readLimit(options.maxBytes ?? artifactMaxBytes);\n if (!maxBytes) throw new Error(\"Artifact maxBytes must be positive\");\n const key = (id: string) => {\n if (!artifactDigestPattern.test(id))\n throw new Error(\"Invalid artifact storage ID\");\n return `artifacts/${id}.blob`;\n };\n return {\n async put(id, input, signal) {\n if (input.byteLength > maxBytes)\n throw new Error(\"Artifact exceeds maxBytes\");\n await immutableObject(\n options.transporter,\n key(id),\n Buffer.from(input),\n signal,\n );\n },\n async get(id, signal) {\n const value = await options.transporter.read(key(id), {\n maxBytes,\n ...(signal ? { signal } : {}),\n });\n if (!value) throw new Error(\"Artifact does not exist\");\n return value.bytes;\n },\n };\n}\n"]}
@@ -0,0 +1,4 @@
1
+ import type { TransportStoreOptions } from "../domain/transport.types.ts";
2
+ export interface ArtifactStoreOptions extends TransportStoreOptions {
3
+ readonly maxBytes?: number;
4
+ }
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=transport-artifact-store.types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"transport-artifact-store.types.js","sourceRoot":"","sources":["../../src/infrastructure/transport-artifact-store.types.ts"],"names":[],"mappings":"","sourcesContent":["import type { TransportStoreOptions } from \"../domain/transport.types.ts\";\n\nexport interface ArtifactStoreOptions extends TransportStoreOptions {\n readonly maxBytes?: number;\n}\n"]}
@@ -0,0 +1,5 @@
1
+ import type { WorkflowCheckpointStore } from "../domain/workflow/checkpoint.types.ts";
2
+ import type { TransportStoreOptions } from "../domain/transport.types.ts";
3
+ import type { CheckpointRecoveryOptions } from "./transport-checkpoint.types.ts";
4
+ export declare function workflowCheckpointStore(options: TransportStoreOptions): WorkflowCheckpointStore;
5
+ export declare function recoverWorkflowCheckpoint(options: CheckpointRecoveryOptions): Promise<void>;
@@ -0,0 +1,105 @@
1
+ import { createHash, randomUUID } from "node:crypto";
2
+ import { TransportConflict } from "../domain/transport.js";
3
+ import { jsonBytes, jsonObject } from "./transport-json.js";
4
+ import { workflowCheckpointMaxBytes } from "./workflow-checkpoint.constants.js";
5
+ function key(runId) {
6
+ if (!runId.trim())
7
+ throw new Error("Checkpoint runId must not be empty");
8
+ return `checkpoints/${createHash("sha256").update(runId).digest("hex")}.json`;
9
+ }
10
+ function envelope(value) {
11
+ if (value === undefined)
12
+ return { owner: null };
13
+ if (!value ||
14
+ typeof value !== "object" ||
15
+ !("owner" in value) ||
16
+ (value.owner !== null && typeof value.owner !== "string"))
17
+ throw new Error("Invalid checkpoint ownership envelope");
18
+ return {
19
+ owner: value.owner,
20
+ ...("checkpoint" in value ? { checkpoint: value.checkpoint } : {}),
21
+ };
22
+ }
23
+ export function workflowCheckpointStore(options) {
24
+ const { transporter } = options;
25
+ return {
26
+ async acquire(runId) {
27
+ const target = key(runId);
28
+ const previous = await transporter.read(target, {
29
+ maxBytes: workflowCheckpointMaxBytes + 1024,
30
+ });
31
+ let data = envelope(jsonObject(previous));
32
+ if (data.owner !== null)
33
+ throw new Error("Workflow is already in use or ownership is unknown");
34
+ const owner = randomUUID();
35
+ data = { ...data, owner };
36
+ let revision = (await transporter.write(target, jsonBytes(data), {
37
+ ifRevision: previous?.revision ?? null,
38
+ })).revision;
39
+ let released = false;
40
+ let pending = Promise.resolve();
41
+ let closing;
42
+ async function owned() {
43
+ if (released)
44
+ throw new Error("Checkpoint lease has been released");
45
+ const current = await transporter.read(target, {
46
+ maxBytes: workflowCheckpointMaxBytes + 1024,
47
+ });
48
+ if (current?.revision !== revision ||
49
+ envelope(jsonObject(current)).owner !== owner)
50
+ throw new TransportConflict(target);
51
+ }
52
+ return {
53
+ async read() {
54
+ await pending;
55
+ await owned();
56
+ return structuredClone(data.checkpoint);
57
+ },
58
+ async write(checkpoint) {
59
+ if (closing)
60
+ return Promise.reject(new Error("Checkpoint lease has been released"));
61
+ const bytes = jsonBytes(checkpoint);
62
+ if (bytes.byteLength > workflowCheckpointMaxBytes)
63
+ return Promise.reject(new Error("Workflow checkpoint exceeds 16 MiB; store large artifacts separately"));
64
+ const next = {
65
+ owner,
66
+ checkpoint: JSON.parse(Buffer.from(bytes).toString()),
67
+ };
68
+ const operation = pending.then(async () => {
69
+ await owned();
70
+ revision = (await transporter.write(target, jsonBytes(next), {
71
+ ifRevision: revision,
72
+ })).revision;
73
+ data = next;
74
+ });
75
+ pending = operation.catch(() => { });
76
+ return operation;
77
+ },
78
+ release() {
79
+ closing ??= (async () => {
80
+ await pending;
81
+ if (released)
82
+ return;
83
+ await owned();
84
+ await transporter.write(target, jsonBytes({ ...data, owner: null }), { ifRevision: revision });
85
+ released = true;
86
+ })();
87
+ return closing;
88
+ },
89
+ };
90
+ },
91
+ };
92
+ }
93
+ export async function recoverWorkflowCheckpoint(options) {
94
+ const target = key(options.runId);
95
+ const current = await options.transporter.read(target, {
96
+ maxBytes: workflowCheckpointMaxBytes + 1024,
97
+ });
98
+ if (!current || current.revision !== options.revision)
99
+ throw new TransportConflict(target);
100
+ const data = envelope(jsonObject(current));
101
+ await options.transporter.write(target, jsonBytes({ ...data, owner: null }), {
102
+ ifRevision: current.revision,
103
+ });
104
+ }
105
+ //# sourceMappingURL=transport-checkpoint.js.map