@elie-laloum/outpost 4.1.0 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/README.md +9 -5
- package/SECURITY.md +2 -0
- package/dist/adapters/models/openai-compatible.constants.d.ts +4 -0
- package/dist/adapters/models/openai-compatible.constants.js +10 -0
- package/dist/adapters/models/openai-compatible.constants.js.map +1 -0
- package/dist/adapters/models/openai-compatible.d.ts +3 -0
- package/dist/adapters/models/openai-compatible.js +78 -0
- package/dist/adapters/models/openai-compatible.js.map +1 -0
- package/dist/adapters/models/openai-compatible.types.d.ts +14 -0
- package/dist/adapters/models/openai-compatible.types.js +2 -0
- package/dist/adapters/models/openai-compatible.types.js.map +1 -0
- package/dist/adapters/models/openai-http.d.ts +1 -0
- package/dist/adapters/models/openai-http.js +30 -0
- package/dist/adapters/models/openai-http.js.map +1 -0
- package/dist/adapters/models/openai-request.d.ts +4 -0
- package/dist/adapters/models/openai-request.js +46 -0
- package/dist/adapters/models/openai-request.js.map +1 -0
- package/dist/adapters/models/openai-response.d.ts +3 -0
- package/dist/adapters/models/openai-response.js +65 -0
- package/dist/adapters/models/openai-response.js.map +1 -0
- package/dist/application/execution.types.d.ts +2 -0
- package/dist/application/execution.types.js.map +1 -1
- package/dist/application/outpost.types.d.ts +5 -0
- package/dist/application/outpost.types.js.map +1 -1
- package/dist/application/recovery-archive.d.ts +4 -0
- package/dist/application/recovery-archive.js +29 -0
- package/dist/application/recovery-archive.js.map +1 -0
- package/dist/application/recovery-archive.types.d.ts +10 -0
- package/dist/application/recovery-archive.types.js +2 -0
- package/dist/application/recovery-archive.types.js.map +1 -0
- package/dist/application/recovery-inspection.js +3 -0
- package/dist/application/recovery-inspection.js.map +1 -1
- package/dist/application/recovery-inspection.types.d.ts +2 -0
- package/dist/application/recovery-inspection.types.js.map +1 -1
- package/dist/application/recovery-retention.d.ts +2 -1
- package/dist/application/recovery-retention.js +9 -1
- package/dist/application/recovery-retention.js.map +1 -1
- package/dist/application/recovery-retention.types.d.ts +6 -0
- package/dist/application/recovery-retention.types.js.map +1 -1
- package/dist/application/remote-backup.js +8 -0
- package/dist/application/remote-backup.js.map +1 -1
- package/dist/application/remote-workspace.types.d.ts +2 -0
- package/dist/application/remote-workspace.types.js.map +1 -1
- package/dist/application/sandbox-agents.js +2 -1
- package/dist/application/sandbox-agents.js.map +1 -1
- package/dist/application/sandbox-dispatch.js +9 -0
- package/dist/application/sandbox-dispatch.js.map +1 -1
- package/dist/application/sandbox-provision.js +6 -0
- package/dist/application/sandbox-provision.js.map +1 -1
- package/dist/application/storage-reservation.js +3 -0
- package/dist/application/storage-reservation.js.map +1 -1
- package/dist/application/transport-inspection.constants.d.ts +2 -0
- package/dist/application/transport-inspection.constants.js +10 -0
- package/dist/application/transport-inspection.constants.js.map +1 -0
- package/dist/application/transport-inspection.d.ts +2 -0
- package/dist/application/transport-inspection.js +103 -0
- package/dist/application/transport-inspection.js.map +1 -0
- package/dist/application/transport-retention.d.ts +4 -0
- package/dist/application/transport-retention.js +153 -0
- package/dist/application/transport-retention.js.map +1 -0
- package/dist/domain/conversation.types.d.ts +2 -0
- package/dist/domain/conversation.types.js.map +1 -1
- package/dist/domain/model.types.d.ts +15 -0
- package/dist/domain/model.types.js +2 -0
- package/dist/domain/model.types.js.map +1 -0
- package/dist/domain/transport.constants.d.ts +2 -0
- package/dist/domain/transport.constants.js +3 -0
- package/dist/domain/transport.constants.js.map +1 -0
- package/dist/domain/transport.d.ts +6 -0
- package/dist/domain/transport.js +21 -0
- package/dist/domain/transport.js.map +1 -0
- package/dist/domain/transport.types.d.ts +31 -0
- package/dist/domain/transport.types.js +2 -0
- package/dist/domain/transport.types.js.map +1 -0
- package/dist/index.d.ts +17 -0
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -1
- package/dist/infrastructure/artifact-store.js +10 -0
- package/dist/infrastructure/artifact-store.js.map +1 -1
- package/dist/infrastructure/artifact-store.types.d.ts +3 -1
- package/dist/infrastructure/artifact-store.types.js.map +1 -1
- package/dist/infrastructure/conversations/capture.js +1 -1
- package/dist/infrastructure/conversations/capture.js.map +1 -1
- package/dist/infrastructure/conversations.d.ts +2 -0
- package/dist/infrastructure/conversations.js +2 -0
- package/dist/infrastructure/conversations.js.map +1 -1
- package/dist/infrastructure/inspection-file.js +3 -3
- package/dist/infrastructure/inspection-file.js.map +1 -1
- package/dist/infrastructure/inspection-hash.js +4 -3
- package/dist/infrastructure/inspection-hash.js.map +1 -1
- package/dist/infrastructure/inspection-path.d.ts +1 -0
- package/dist/infrastructure/inspection-path.js +18 -0
- package/dist/infrastructure/inspection-path.js.map +1 -0
- package/dist/infrastructure/journal.js +6 -0
- package/dist/infrastructure/journal.js.map +1 -1
- package/dist/infrastructure/journal.types.d.ts +3 -0
- package/dist/infrastructure/journal.types.js.map +1 -1
- package/dist/infrastructure/local-transport.d.ts +3 -0
- package/dist/infrastructure/local-transport.js +146 -0
- package/dist/infrastructure/local-transport.js.map +1 -0
- package/dist/infrastructure/local-transport.types.d.ts +3 -0
- package/dist/infrastructure/local-transport.types.js +2 -0
- package/dist/infrastructure/local-transport.types.js.map +1 -0
- package/dist/infrastructure/resource-activity-inspection.d.ts +2 -1
- package/dist/infrastructure/resource-activity-inspection.js +2 -2
- package/dist/infrastructure/resource-activity-inspection.js.map +1 -1
- package/dist/infrastructure/resource-activity.js +42 -14
- package/dist/infrastructure/resource-activity.js.map +1 -1
- package/dist/infrastructure/resource-activity.types.d.ts +2 -0
- package/dist/infrastructure/resource-activity.types.js.map +1 -1
- package/dist/infrastructure/s3-transport.d.ts +4 -0
- package/dist/infrastructure/s3-transport.js +144 -0
- package/dist/infrastructure/s3-transport.js.map +1 -0
- package/dist/infrastructure/s3-transport.types.d.ts +6 -0
- package/dist/infrastructure/s3-transport.types.js +2 -0
- package/dist/infrastructure/s3-transport.types.js.map +1 -0
- package/dist/infrastructure/storage-inventory.types.d.ts +2 -1
- package/dist/infrastructure/storage-inventory.types.js.map +1 -1
- package/dist/infrastructure/storage-reservations.types.d.ts +2 -0
- package/dist/infrastructure/storage-reservations.types.js.map +1 -1
- package/dist/infrastructure/task-queue-bullmq-resources.d.ts +13 -0
- package/dist/infrastructure/task-queue-bullmq-resources.js +118 -0
- package/dist/infrastructure/task-queue-bullmq-resources.js.map +1 -0
- package/dist/infrastructure/task-queue-bullmq-state.d.ts +5 -0
- package/dist/infrastructure/task-queue-bullmq-state.js +26 -0
- package/dist/infrastructure/task-queue-bullmq-state.js.map +1 -0
- package/dist/infrastructure/task-queue-bullmq.constants.d.ts +7 -0
- package/dist/infrastructure/task-queue-bullmq.constants.js +79 -0
- package/dist/infrastructure/task-queue-bullmq.constants.js.map +1 -0
- package/dist/infrastructure/task-queue-bullmq.d.ts +3 -0
- package/dist/infrastructure/task-queue-bullmq.js +124 -0
- package/dist/infrastructure/task-queue-bullmq.js.map +1 -0
- package/dist/infrastructure/task-queue-bullmq.types.d.ts +17 -0
- package/dist/infrastructure/task-queue-bullmq.types.js +2 -0
- package/dist/infrastructure/task-queue-bullmq.types.js.map +1 -0
- package/dist/infrastructure/transport-archive.constants.d.ts +6 -0
- package/dist/infrastructure/transport-archive.constants.js +7 -0
- package/dist/infrastructure/transport-archive.constants.js.map +1 -0
- package/dist/infrastructure/transport-archive.d.ts +5 -0
- package/dist/infrastructure/transport-archive.js +157 -0
- package/dist/infrastructure/transport-archive.js.map +1 -0
- package/dist/infrastructure/transport-archive.types.d.ts +17 -0
- package/dist/infrastructure/transport-archive.types.js +2 -0
- package/dist/infrastructure/transport-archive.types.js.map +1 -0
- package/dist/infrastructure/transport-artifact-store.d.ts +3 -0
- package/dist/infrastructure/transport-artifact-store.js +31 -0
- package/dist/infrastructure/transport-artifact-store.js.map +1 -0
- package/dist/infrastructure/transport-artifact-store.types.d.ts +4 -0
- package/dist/infrastructure/transport-artifact-store.types.js +2 -0
- package/dist/infrastructure/transport-artifact-store.types.js.map +1 -0
- package/dist/infrastructure/transport-checkpoint.d.ts +5 -0
- package/dist/infrastructure/transport-checkpoint.js +105 -0
- package/dist/infrastructure/transport-checkpoint.js.map +1 -0
- package/dist/infrastructure/transport-checkpoint.types.d.ts +9 -0
- package/dist/infrastructure/transport-checkpoint.types.js +2 -0
- package/dist/infrastructure/transport-checkpoint.types.js.map +1 -0
- package/dist/infrastructure/transport-conversations.d.ts +4 -0
- package/dist/infrastructure/transport-conversations.js +69 -0
- package/dist/infrastructure/transport-conversations.js.map +1 -0
- package/dist/infrastructure/transport-conversations.types.d.ts +4 -0
- package/dist/infrastructure/transport-conversations.types.js +2 -0
- package/dist/infrastructure/transport-conversations.types.js.map +1 -0
- package/dist/infrastructure/transport-envelope.d.ts +7 -0
- package/dist/infrastructure/transport-envelope.js +47 -0
- package/dist/infrastructure/transport-envelope.js.map +1 -0
- package/dist/infrastructure/transport-journal.d.ts +6 -0
- package/dist/infrastructure/transport-journal.js +93 -0
- package/dist/infrastructure/transport-journal.js.map +1 -0
- package/dist/infrastructure/transport-journal.types.d.ts +10 -0
- package/dist/infrastructure/transport-journal.types.js +2 -0
- package/dist/infrastructure/transport-journal.types.js.map +1 -0
- package/dist/infrastructure/transport-json.d.ts +6 -0
- package/dist/infrastructure/transport-json.js +46 -0
- package/dist/infrastructure/transport-json.js.map +1 -0
- package/dist/infrastructure/transport-reservations.d.ts +5 -0
- package/dist/infrastructure/transport-reservations.js +89 -0
- package/dist/infrastructure/transport-reservations.js.map +1 -0
- package/dist/infrastructure/transport-reservations.types.d.ts +4 -0
- package/dist/infrastructure/transport-reservations.types.js +2 -0
- package/dist/infrastructure/transport-reservations.types.js.map +1 -0
- package/dist/infrastructure/transport.constants.d.ts +8 -0
- package/dist/infrastructure/transport.constants.js +9 -0
- package/dist/infrastructure/transport.constants.js.map +1 -0
- package/dist/infrastructure/workflow-checkpoint.js +7 -0
- package/dist/infrastructure/workflow-checkpoint.js.map +1 -1
- package/dist/infrastructure/workflow-checkpoint.types.d.ts +3 -1
- package/dist/infrastructure/workflow-checkpoint.types.js.map +1 -1
- package/package.json +26 -5
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { TransportConflict } from "../domain/transport.js";
|
|
3
|
+
import { jsonBytes, jsonObject } from "./transport-json.js";
|
|
4
|
+
import { workflowCheckpointMaxBytes } from "./workflow-checkpoint.constants.js";
|
|
5
|
+
function key(runId) {
|
|
6
|
+
if (!runId.trim())
|
|
7
|
+
throw new Error("Checkpoint runId must not be empty");
|
|
8
|
+
return `checkpoints/${createHash("sha256").update(runId).digest("hex")}.json`;
|
|
9
|
+
}
|
|
10
|
+
function envelope(value) {
|
|
11
|
+
if (value === undefined)
|
|
12
|
+
return { owner: null };
|
|
13
|
+
if (!value ||
|
|
14
|
+
typeof value !== "object" ||
|
|
15
|
+
!("owner" in value) ||
|
|
16
|
+
(value.owner !== null && typeof value.owner !== "string"))
|
|
17
|
+
throw new Error("Invalid checkpoint ownership envelope");
|
|
18
|
+
return {
|
|
19
|
+
owner: value.owner,
|
|
20
|
+
...("checkpoint" in value ? { checkpoint: value.checkpoint } : {}),
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
export function workflowCheckpointStore(options) {
|
|
24
|
+
const { transporter } = options;
|
|
25
|
+
return {
|
|
26
|
+
async acquire(runId) {
|
|
27
|
+
const target = key(runId);
|
|
28
|
+
const previous = await transporter.read(target, {
|
|
29
|
+
maxBytes: workflowCheckpointMaxBytes + 1024,
|
|
30
|
+
});
|
|
31
|
+
let data = envelope(jsonObject(previous));
|
|
32
|
+
if (data.owner !== null)
|
|
33
|
+
throw new Error("Workflow is already in use or ownership is unknown");
|
|
34
|
+
const owner = randomUUID();
|
|
35
|
+
data = { ...data, owner };
|
|
36
|
+
let revision = (await transporter.write(target, jsonBytes(data), {
|
|
37
|
+
ifRevision: previous?.revision ?? null,
|
|
38
|
+
})).revision;
|
|
39
|
+
let released = false;
|
|
40
|
+
let pending = Promise.resolve();
|
|
41
|
+
let closing;
|
|
42
|
+
async function owned() {
|
|
43
|
+
if (released)
|
|
44
|
+
throw new Error("Checkpoint lease has been released");
|
|
45
|
+
const current = await transporter.read(target, {
|
|
46
|
+
maxBytes: workflowCheckpointMaxBytes + 1024,
|
|
47
|
+
});
|
|
48
|
+
if (current?.revision !== revision ||
|
|
49
|
+
envelope(jsonObject(current)).owner !== owner)
|
|
50
|
+
throw new TransportConflict(target);
|
|
51
|
+
}
|
|
52
|
+
return {
|
|
53
|
+
async read() {
|
|
54
|
+
await pending;
|
|
55
|
+
await owned();
|
|
56
|
+
return structuredClone(data.checkpoint);
|
|
57
|
+
},
|
|
58
|
+
async write(checkpoint) {
|
|
59
|
+
if (closing)
|
|
60
|
+
return Promise.reject(new Error("Checkpoint lease has been released"));
|
|
61
|
+
const bytes = jsonBytes(checkpoint);
|
|
62
|
+
if (bytes.byteLength > workflowCheckpointMaxBytes)
|
|
63
|
+
return Promise.reject(new Error("Workflow checkpoint exceeds 16 MiB; store large artifacts separately"));
|
|
64
|
+
const next = {
|
|
65
|
+
owner,
|
|
66
|
+
checkpoint: JSON.parse(Buffer.from(bytes).toString()),
|
|
67
|
+
};
|
|
68
|
+
const operation = pending.then(async () => {
|
|
69
|
+
await owned();
|
|
70
|
+
revision = (await transporter.write(target, jsonBytes(next), {
|
|
71
|
+
ifRevision: revision,
|
|
72
|
+
})).revision;
|
|
73
|
+
data = next;
|
|
74
|
+
});
|
|
75
|
+
pending = operation.catch(() => { });
|
|
76
|
+
return operation;
|
|
77
|
+
},
|
|
78
|
+
release() {
|
|
79
|
+
closing ??= (async () => {
|
|
80
|
+
await pending;
|
|
81
|
+
if (released)
|
|
82
|
+
return;
|
|
83
|
+
await owned();
|
|
84
|
+
await transporter.write(target, jsonBytes({ ...data, owner: null }), { ifRevision: revision });
|
|
85
|
+
released = true;
|
|
86
|
+
})();
|
|
87
|
+
return closing;
|
|
88
|
+
},
|
|
89
|
+
};
|
|
90
|
+
},
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
export async function recoverWorkflowCheckpoint(options) {
|
|
94
|
+
const target = key(options.runId);
|
|
95
|
+
const current = await options.transporter.read(target, {
|
|
96
|
+
maxBytes: workflowCheckpointMaxBytes + 1024,
|
|
97
|
+
});
|
|
98
|
+
if (!current || current.revision !== options.revision)
|
|
99
|
+
throw new TransportConflict(target);
|
|
100
|
+
const data = envelope(jsonObject(current));
|
|
101
|
+
await options.transporter.write(target, jsonBytes({ ...data, owner: null }), {
|
|
102
|
+
ifRevision: current.revision,
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
//# sourceMappingURL=transport-checkpoint.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"transport-checkpoint.js","sourceRoot":"","sources":["../../src/infrastructure/transport-checkpoint.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,EAAE,0BAA0B,EAAE,MAAM,oCAAoC,CAAC;AAMhF,SAAS,GAAG,CAAC,KAAa;IACxB,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;IACzE,OAAO,eAAe,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC;AAChF,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IAChD,IACE,CAAC,KAAK;QACN,OAAO,KAAK,KAAK,QAAQ;QACzB,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC;QACnB,CAAC,KAAK,CAAC,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,CAAC;QAEzD,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,OAAO;QACL,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,GAAG,CAAC,YAAY,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACnE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,OAA8B;IAE9B,MAAM,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;IAChC,OAAO;QACL,KAAK,CAAC,OAAO,CAAC,KAAK;YACjB,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC;YAC1B,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE;gBAC9C,QAAQ,EAAE,0BAA0B,GAAG,IAAI;aAC5C,CAAC,CAAC;YACH,IAAI,IAAI,GAAG,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC1C,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI;gBACrB,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;YACxE,MAAM,KAAK,GAAG,UAAU,EAAE,CAAC;YAC3B,IAAI,GAAG,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,CAAC;YAC1B,IAAI,QAAQ,GAAG,CACb,MAAM,WAAW,CAAC,KAAK,CAAC,MAAM,EAAE,SAAS,CAAC,IAAI,CAAC,EAAE;gBAC/C,UAAU,EAAE,QAAQ,EAAE,QAAQ,IAAI,IAAI;aACvC,CAAC,CACH,CAAC,QAAQ,CAAC;YACX,IAAI,QAAQ,GAAG,KAAK,CAAC;YACrB,IAAI,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;YAChC,IAAI,OAAkC,CAAC;YACvC,KAAK,UAAU,KAAK;gBAClB,IAAI,QAAQ;oBAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;gBACpE,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE;oBAC7C,QAAQ,EAAE,0BAA0B,GAAG,IAAI;iBAC5C,CAAC,CAAC;gBACH,IACE,OAAO,EAAE,QAAQ,KAAK,QAAQ;oBAC9B,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,KAAK,KAAK;oBAE7C,MAAM,IAAI,iBAAiB,CAAC,MAAM,CAAC,CAAC;YACxC,CAAC;YACD,OAAO;gBACL,KAAK,CAAC,IAAI;oBACR,MAAM,OAAO,CAAC;oBACd,MAAM,KAAK,EAAE,CAAC;oBACd,OAAO,eAAe,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAC1C,CAAC;gBACD,KAAK,CAAC,KAAK,CAAC,UAAU;oBACpB,IAAI,OAAO;wBACT,OAAO,OAAO,CAAC,MAAM,CACnB,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAChD,CAAC;oBACJ,MAAM,KAAK,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC;oBACpC,IAAI,KAAK,CAAC,UAAU,GAAG,0BAA0B;wBAC/C,OAAO,OAAO,CAAC,MAAM,CACnB,IAAI,KAAK,CACP,sEAAsE,CACvE,CACF,CAAC;oBACJ,MAAM,IAAI,GAAG;wBACX,KAAK;wBACL,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE,CAAY;qBACjE,CAAC;oBACF,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;wBACxC,MAAM,KAAK,EAAE,CAAC;wBACd,QAAQ,GAAG,CACT,MAAM,WAAW,CAAC,KAAK,CAAC,MAAM,EAAE,SAAS,CAAC,IAAI,CAAC,EAAE;4BAC/C,UAAU,EAAE,QAAQ;yBACrB,CAAC,CACH,CAAC,QAAQ,CAAC;wBACX,IAAI,GAAG,IAAI,CAAC;oBACd,CAAC,CAAC,CAAC;oBACH,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;oBACpC,OAAO,SAAS,CAAC;gBACnB,CAAC;gBACD,OAAO;oBACL,OAAO,KAAK,CAAC,KAAK,IAAI,EAAE;wBACtB,MAAM,OAAO,CAAC;wBACd,IAAI,QAAQ;4BAAE,OAAO;wBACrB,MAAM,KAAK,EAAE,CAAC;wBACd,MAAM,WAAW,CAAC,KAAK,CACrB,MAAM,EACN,SAAS,CAAC,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EACnC,EAAE,UAAU,EAAE,QAAQ,EAAE,CACzB,CAAC;wBACF,QAAQ,GAAG,IAAI,CAAC;oBAClB,CAAC,CAAC,EAAE,CAAC;oBACL,OAAO,OAAO,CAAC;gBACjB,CAAC;aACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,OAAkC;IAElC,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAClC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE;QACrD,QAAQ,EAAE,0BAA0B,GAAG,IAAI;KAC5C,CAAC,CAAC;IACH,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,QAAQ;QACnD,MAAM,IAAI,iBAAiB,CAAC,MAAM,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3C,MAAM,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE;QAC3E,UAAU,EAAE,OAAO,CAAC,QAAQ;KAC7B,CAAC,CAAC;AACL,CAAC","sourcesContent":["import { createHash, randomUUID } from \"node:crypto\";\nimport type { WorkflowCheckpointStore } from \"../domain/workflow/checkpoint.types.ts\";\nimport type { TransportStoreOptions } from \"../domain/transport.types.ts\";\nimport { TransportConflict } from \"../domain/transport.ts\";\nimport { jsonBytes, jsonObject } from \"./transport-json.ts\";\nimport { workflowCheckpointMaxBytes } from \"./workflow-checkpoint.constants.ts\";\nimport type {\n CheckpointEnvelope,\n CheckpointRecoveryOptions,\n} from \"./transport-checkpoint.types.ts\";\n\nfunction key(runId: string): string {\n if (!runId.trim()) throw new Error(\"Checkpoint runId must not be empty\");\n return `checkpoints/${createHash(\"sha256\").update(runId).digest(\"hex\")}.json`;\n}\n\nfunction envelope(value: unknown): CheckpointEnvelope {\n if (value === undefined) return { owner: null };\n if (\n !value ||\n typeof value !== \"object\" ||\n !(\"owner\" in value) ||\n (value.owner !== null && typeof value.owner !== \"string\")\n )\n throw new Error(\"Invalid checkpoint ownership envelope\");\n return {\n owner: value.owner,\n ...(\"checkpoint\" in value ? { checkpoint: value.checkpoint } : {}),\n };\n}\n\nexport function workflowCheckpointStore(\n options: TransportStoreOptions,\n): WorkflowCheckpointStore {\n const { transporter } = options;\n return {\n async acquire(runId) {\n const target = key(runId);\n const previous = await transporter.read(target, {\n maxBytes: workflowCheckpointMaxBytes + 1024,\n });\n let data = envelope(jsonObject(previous));\n if (data.owner !== null)\n throw new Error(\"Workflow is already in use or ownership is unknown\");\n const owner = randomUUID();\n data = { ...data, owner };\n let revision = (\n await transporter.write(target, jsonBytes(data), {\n ifRevision: previous?.revision ?? null,\n })\n ).revision;\n let released = false;\n let pending = Promise.resolve();\n let closing: Promise<void> | undefined;\n async function owned() {\n if (released) throw new Error(\"Checkpoint lease has been released\");\n const current = await transporter.read(target, {\n maxBytes: workflowCheckpointMaxBytes + 1024,\n });\n if (\n current?.revision !== revision ||\n envelope(jsonObject(current)).owner !== owner\n )\n throw new TransportConflict(target);\n }\n return {\n async read() {\n await pending;\n await owned();\n return structuredClone(data.checkpoint);\n },\n async write(checkpoint) {\n if (closing)\n return Promise.reject(\n new Error(\"Checkpoint lease has been released\"),\n );\n const bytes = jsonBytes(checkpoint);\n if (bytes.byteLength > workflowCheckpointMaxBytes)\n return Promise.reject(\n new Error(\n \"Workflow checkpoint exceeds 16 MiB; store large artifacts separately\",\n ),\n );\n const next = {\n owner,\n checkpoint: JSON.parse(Buffer.from(bytes).toString()) as unknown,\n };\n const operation = pending.then(async () => {\n await owned();\n revision = (\n await transporter.write(target, jsonBytes(next), {\n ifRevision: revision,\n })\n ).revision;\n data = next;\n });\n pending = operation.catch(() => {});\n return operation;\n },\n release() {\n closing ??= (async () => {\n await pending;\n if (released) return;\n await owned();\n await transporter.write(\n target,\n jsonBytes({ ...data, owner: null }),\n { ifRevision: revision },\n );\n released = true;\n })();\n return closing;\n },\n };\n },\n };\n}\n\nexport async function recoverWorkflowCheckpoint(\n options: CheckpointRecoveryOptions,\n): Promise<void> {\n const target = key(options.runId);\n const current = await options.transporter.read(target, {\n maxBytes: workflowCheckpointMaxBytes + 1024,\n });\n if (!current || current.revision !== options.revision)\n throw new TransportConflict(target);\n const data = envelope(jsonObject(current));\n await options.transporter.write(target, jsonBytes({ ...data, owner: null }), {\n ifRevision: current.revision,\n });\n}\n"]}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { TransportStoreOptions } from "../domain/transport.types.ts";
|
|
2
|
+
export interface CheckpointRecoveryOptions extends TransportStoreOptions {
|
|
3
|
+
readonly runId: string;
|
|
4
|
+
readonly revision: string;
|
|
5
|
+
}
|
|
6
|
+
export interface CheckpointEnvelope {
|
|
7
|
+
readonly owner: string | null;
|
|
8
|
+
readonly checkpoint?: unknown;
|
|
9
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"transport-checkpoint.types.js","sourceRoot":"","sources":["../../src/infrastructure/transport-checkpoint.types.ts"],"names":[],"mappings":"","sourcesContent":["import type { TransportStoreOptions } from \"../domain/transport.types.ts\";\n\nexport interface CheckpointRecoveryOptions extends TransportStoreOptions {\n readonly runId: string;\n readonly revision: string;\n}\n\nexport interface CheckpointEnvelope {\n readonly owner: string | null;\n readonly checkpoint?: unknown;\n}\n"]}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { ConversationStore } from "../domain/conversation.types.ts";
|
|
2
|
+
import type { ConversationFormat } from "./conversations.types.ts";
|
|
3
|
+
import type { TransportConversationOptions } from "./transport-conversations.types.ts";
|
|
4
|
+
export declare function transportConversations(format: ConversationFormat, options: TransportConversationOptions): ConversationStore;
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { mkdir, mkdtemp } from "node:fs/promises";
|
|
3
|
+
import { dirname, join, relative } from "node:path";
|
|
4
|
+
import { invariant } from "../domain/errors.js";
|
|
5
|
+
import { transportKey } from "../domain/transport.js";
|
|
6
|
+
import { nativeConversations } from "./conversations/native-store.js";
|
|
7
|
+
import { files } from "./conversations/files.js";
|
|
8
|
+
import { validId } from "./conversations/paths.js";
|
|
9
|
+
import { jsonBytes, jsonObject, transportReference } from "./transport-json.js";
|
|
10
|
+
import { archiveFiles, restoreArchiveFiles } from "./transport-archive.js";
|
|
11
|
+
import { safeDestination } from "./files.js";
|
|
12
|
+
export function transportConversations(format, options) {
|
|
13
|
+
const native = nativeConversations(format);
|
|
14
|
+
const prefix = `conversations/${transportKey(options.namespace)}/${format}`;
|
|
15
|
+
const key = (id) => {
|
|
16
|
+
validId(id);
|
|
17
|
+
return `${prefix}/${id}/index`;
|
|
18
|
+
};
|
|
19
|
+
return {
|
|
20
|
+
name: `transport:${options.namespace}:${format}`,
|
|
21
|
+
async locate(id, repository) {
|
|
22
|
+
const current = await options.transporter.read(key(id));
|
|
23
|
+
invariant(current, "Conversation does not exist");
|
|
24
|
+
const value = jsonObject(current);
|
|
25
|
+
invariant(value &&
|
|
26
|
+
typeof value === "object" &&
|
|
27
|
+
"archive" in value &&
|
|
28
|
+
"file" in value &&
|
|
29
|
+
typeof value.file === "string", "Invalid conversation index");
|
|
30
|
+
const staging = join(repository, ".outpost", "recovery", "conversations");
|
|
31
|
+
await mkdir(staging, { recursive: true, mode: 0o700 });
|
|
32
|
+
const parent = await mkdtemp(join(staging, "materialized-"));
|
|
33
|
+
const destination = join(parent, "transcript");
|
|
34
|
+
await restoreArchiveFiles(options.transporter, transportReference(value.archive), destination);
|
|
35
|
+
const file = await safeDestination(destination, value.file);
|
|
36
|
+
return {
|
|
37
|
+
id,
|
|
38
|
+
file,
|
|
39
|
+
format,
|
|
40
|
+
reference: { key: current.key, revision: current.revision },
|
|
41
|
+
};
|
|
42
|
+
},
|
|
43
|
+
async capture(id, context) {
|
|
44
|
+
const target = key(id);
|
|
45
|
+
const previous = await options.transporter.read(target);
|
|
46
|
+
await mkdir(context.staging, { recursive: true, mode: 0o700 });
|
|
47
|
+
const home = await mkdtemp(join(context.staging, "captured-"));
|
|
48
|
+
const captured = await native.capture(id, { ...context, home });
|
|
49
|
+
const root = dirname(captured.file);
|
|
50
|
+
const selected = [
|
|
51
|
+
captured.file,
|
|
52
|
+
...(await files(join(root, id, "subagents"))),
|
|
53
|
+
];
|
|
54
|
+
const archive = await archiveFiles(options.transporter, root, selected.map((file) => relative(root, file).replaceAll("\\", "/")), `${prefix}/${id}/snapshots/${randomUUID()}`);
|
|
55
|
+
const reference = await options.transporter.write(target, jsonBytes({
|
|
56
|
+
archive,
|
|
57
|
+
file: relative(root, captured.file).replaceAll("\\", "/"),
|
|
58
|
+
}), { ifRevision: previous?.revision ?? null });
|
|
59
|
+
return {
|
|
60
|
+
...captured,
|
|
61
|
+
reference: { key: reference.key, revision: reference.revision },
|
|
62
|
+
};
|
|
63
|
+
},
|
|
64
|
+
restore(record, context) {
|
|
65
|
+
return native.restore(record, context);
|
|
66
|
+
},
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
//# sourceMappingURL=transport-conversations.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"transport-conversations.js","sourceRoot":"","sources":["../../src/infrastructure/transport-conversations.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAEpD,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAGtD,OAAO,EAAE,mBAAmB,EAAE,MAAM,iCAAiC,CAAC;AACtE,OAAO,EAAE,KAAK,EAAE,MAAM,0BAA0B,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAChF,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAE7C,MAAM,UAAU,sBAAsB,CACpC,MAA0B,EAC1B,OAAqC;IAErC,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,iBAAiB,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,MAAM,EAAE,CAAC;IAC5E,MAAM,GAAG,GAAG,CAAC,EAAU,EAAE,EAAE;QACzB,OAAO,CAAC,EAAE,CAAC,CAAC;QACZ,OAAO,GAAG,MAAM,IAAI,EAAE,QAAQ,CAAC;IACjC,CAAC,CAAC;IACF,OAAO;QACL,IAAI,EAAE,aAAa,OAAO,CAAC,SAAS,IAAI,MAAM,EAAE;QAChD,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,UAAU;YACzB,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;YACxD,SAAS,CAAC,OAAO,EAAE,6BAA6B,CAAC,CAAC;YAClD,MAAM,KAAK,GAAY,UAAU,CAAC,OAAO,CAAC,CAAC;YAC3C,SAAS,CACP,KAAK;gBACH,OAAO,KAAK,KAAK,QAAQ;gBACzB,SAAS,IAAI,KAAK;gBAClB,MAAM,IAAI,KAAK;gBACf,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,EAChC,4BAA4B,CAC7B,CAAC;YACF,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,eAAe,CAAC,CAAC;YAC1E,MAAM,KAAK,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YACvD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC;YAC7D,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;YAC/C,MAAM,mBAAmB,CACvB,OAAO,CAAC,WAAW,EACnB,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC,EACjC,WAAW,CACZ,CAAC;YACF,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC,WAAW,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAC5D,OAAO;gBACL,EAAE;gBACF,IAAI;gBACJ,MAAM;gBACN,SAAS,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE;aAC5D,CAAC;QACJ,CAAC;QACD,KAAK,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO;YACvB,MAAM,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC;YACvB,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACxD,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YAC/D,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC;YAC/D,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YAChE,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACpC,MAAM,QAAQ,GAAG;gBACf,QAAQ,CAAC,IAAI;gBACb,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,EAAE,WAAW,CAAC,CAAC,CAAC;aAC9C,CAAC;YACF,MAAM,OAAO,GAAG,MAAM,YAAY,CAChC,OAAO,CAAC,WAAW,EACnB,IAAI,EACJ,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,EAClE,GAAG,MAAM,IAAI,EAAE,cAAc,UAAU,EAAE,EAAE,CAC5C,CAAC;YACF,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,KAAK,CAC/C,MAAM,EACN,SAAS,CAAC;gBACR,OAAO;gBACP,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC;aAC1D,CAAC,EACF,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,IAAI,IAAI,EAAE,CAC3C,CAAC;YACF,OAAO;gBACL,GAAG,QAAQ;gBACX,SAAS,EAAE,EAAE,GAAG,EAAE,SAAS,CAAC,GAAG,EAAE,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE;aAChE,CAAC;QACJ,CAAC;QACD,OAAO,CAAC,MAAM,EAAE,OAAO;YACrB,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACzC,CAAC;KACF,CAAC;AACJ,CAAC","sourcesContent":["import { randomUUID } from \"node:crypto\";\nimport { mkdir, mkdtemp } from \"node:fs/promises\";\nimport { dirname, join, relative } from \"node:path\";\nimport type { ConversationStore } from \"../domain/conversation.types.ts\";\nimport { invariant } from \"../domain/errors.ts\";\nimport { transportKey } from \"../domain/transport.ts\";\nimport type { ConversationFormat } from \"./conversations.types.ts\";\nimport type { TransportConversationOptions } from \"./transport-conversations.types.ts\";\nimport { nativeConversations } from \"./conversations/native-store.ts\";\nimport { files } from \"./conversations/files.ts\";\nimport { validId } from \"./conversations/paths.ts\";\nimport { jsonBytes, jsonObject, transportReference } from \"./transport-json.ts\";\nimport { archiveFiles, restoreArchiveFiles } from \"./transport-archive.ts\";\nimport { safeDestination } from \"./files.ts\";\n\nexport function transportConversations(\n format: ConversationFormat,\n options: TransportConversationOptions,\n): ConversationStore {\n const native = nativeConversations(format);\n const prefix = `conversations/${transportKey(options.namespace)}/${format}`;\n const key = (id: string) => {\n validId(id);\n return `${prefix}/${id}/index`;\n };\n return {\n name: `transport:${options.namespace}:${format}`,\n async locate(id, repository) {\n const current = await options.transporter.read(key(id));\n invariant(current, \"Conversation does not exist\");\n const value: unknown = jsonObject(current);\n invariant(\n value &&\n typeof value === \"object\" &&\n \"archive\" in value &&\n \"file\" in value &&\n typeof value.file === \"string\",\n \"Invalid conversation index\",\n );\n const staging = join(repository, \".outpost\", \"recovery\", \"conversations\");\n await mkdir(staging, { recursive: true, mode: 0o700 });\n const parent = await mkdtemp(join(staging, \"materialized-\"));\n const destination = join(parent, \"transcript\");\n await restoreArchiveFiles(\n options.transporter,\n transportReference(value.archive),\n destination,\n );\n const file = await safeDestination(destination, value.file);\n return {\n id,\n file,\n format,\n reference: { key: current.key, revision: current.revision },\n };\n },\n async capture(id, context) {\n const target = key(id);\n const previous = await options.transporter.read(target);\n await mkdir(context.staging, { recursive: true, mode: 0o700 });\n const home = await mkdtemp(join(context.staging, \"captured-\"));\n const captured = await native.capture(id, { ...context, home });\n const root = dirname(captured.file);\n const selected = [\n captured.file,\n ...(await files(join(root, id, \"subagents\"))),\n ];\n const archive = await archiveFiles(\n options.transporter,\n root,\n selected.map((file) => relative(root, file).replaceAll(\"\\\\\", \"/\")),\n `${prefix}/${id}/snapshots/${randomUUID()}`,\n );\n const reference = await options.transporter.write(\n target,\n jsonBytes({\n archive,\n file: relative(root, captured.file).replaceAll(\"\\\\\", \"/\"),\n }),\n { ifRevision: previous?.revision ?? null },\n );\n return {\n ...captured,\n reference: { key: reference.key, revision: reference.revision },\n };\n },\n restore(record, context) {\n return native.restore(record, context);\n },\n };\n}\n"]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"transport-conversations.types.js","sourceRoot":"","sources":["../../src/infrastructure/transport-conversations.types.ts"],"names":[],"mappings":"","sourcesContent":["import type { TransportStoreOptions } from \"../domain/transport.types.ts\";\nexport interface TransportConversationOptions extends TransportStoreOptions {\n readonly namespace: string;\n}\n"]}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { TransportEntry, TransportObject } from "../domain/transport.types.ts";
|
|
2
|
+
export declare function encodeObject(key: string, bytes: Uint8Array): {
|
|
3
|
+
entry: TransportEntry;
|
|
4
|
+
data: Buffer<ArrayBuffer>;
|
|
5
|
+
};
|
|
6
|
+
export declare function decodeObject(data: Uint8Array, key: string, maxBytes: number): TransportObject;
|
|
7
|
+
export declare function readLimit(maxBytes?: number): number;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { transportDefaults } from "./transport.constants.js";
|
|
3
|
+
export function encodeObject(key, bytes) {
|
|
4
|
+
if (bytes.byteLength > transportDefaults.maxBytes)
|
|
5
|
+
throw new Error("Transport object exceeds 64 MiB; split large payloads into chunks");
|
|
6
|
+
const entry = {
|
|
7
|
+
key,
|
|
8
|
+
revision: randomUUID(),
|
|
9
|
+
size: bytes.byteLength,
|
|
10
|
+
modifiedAt: new Date().toISOString(),
|
|
11
|
+
};
|
|
12
|
+
const header = Buffer.alloc(transportDefaults.headerBytes, " ");
|
|
13
|
+
header.write(JSON.stringify(entry));
|
|
14
|
+
return { entry, data: Buffer.concat([header, bytes]) };
|
|
15
|
+
}
|
|
16
|
+
export function decodeObject(data, key, maxBytes) {
|
|
17
|
+
const buffer = Buffer.from(data);
|
|
18
|
+
const value = JSON.parse(buffer.subarray(0, transportDefaults.headerBytes).toString());
|
|
19
|
+
const bytes = buffer.subarray(transportDefaults.headerBytes);
|
|
20
|
+
if (!value ||
|
|
21
|
+
typeof value !== "object" ||
|
|
22
|
+
!("key" in value) ||
|
|
23
|
+
value.key !== key ||
|
|
24
|
+
!("revision" in value) ||
|
|
25
|
+
typeof value.revision !== "string" ||
|
|
26
|
+
!value.revision ||
|
|
27
|
+
!("size" in value) ||
|
|
28
|
+
value.size !== bytes.length ||
|
|
29
|
+
bytes.length > maxBytes ||
|
|
30
|
+
!("modifiedAt" in value) ||
|
|
31
|
+
typeof value.modifiedAt !== "string" ||
|
|
32
|
+
!Number.isFinite(Date.parse(value.modifiedAt)))
|
|
33
|
+
throw new Error("Invalid or oversized transport object");
|
|
34
|
+
return {
|
|
35
|
+
key,
|
|
36
|
+
revision: value.revision,
|
|
37
|
+
size: bytes.length,
|
|
38
|
+
modifiedAt: value.modifiedAt,
|
|
39
|
+
bytes,
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
export function readLimit(maxBytes = transportDefaults.maxBytes) {
|
|
43
|
+
if (!Number.isSafeInteger(maxBytes) || maxBytes < 0)
|
|
44
|
+
throw new Error("Invalid transport byte limit");
|
|
45
|
+
return maxBytes;
|
|
46
|
+
}
|
|
47
|
+
//# sourceMappingURL=transport-envelope.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"transport-envelope.js","sourceRoot":"","sources":["../../src/infrastructure/transport-envelope.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAKzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAE7D,MAAM,UAAU,YAAY,CAAC,GAAW,EAAE,KAAiB;IACzD,IAAI,KAAK,CAAC,UAAU,GAAG,iBAAiB,CAAC,QAAQ;QAC/C,MAAM,IAAI,KAAK,CACb,mEAAmE,CACpE,CAAC;IACJ,MAAM,KAAK,GAAmB;QAC5B,GAAG;QACH,QAAQ,EAAE,UAAU,EAAE;QACtB,IAAI,EAAE,KAAK,CAAC,UAAU;QACtB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACrC,CAAC;IACF,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;IAChE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IACpC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,IAAgB,EAChB,GAAW,EACX,QAAgB;IAEhB,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjC,MAAM,KAAK,GAAY,IAAI,CAAC,KAAK,CAC/B,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,iBAAiB,CAAC,WAAW,CAAC,CAAC,QAAQ,EAAE,CAC7D,CAAC;IACF,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;IAC7D,IACE,CAAC,KAAK;QACN,OAAO,KAAK,KAAK,QAAQ;QACzB,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC;QACjB,KAAK,CAAC,GAAG,KAAK,GAAG;QACjB,CAAC,CAAC,UAAU,IAAI,KAAK,CAAC;QACtB,OAAO,KAAK,CAAC,QAAQ,KAAK,QAAQ;QAClC,CAAC,KAAK,CAAC,QAAQ;QACf,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC;QAClB,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,MAAM;QAC3B,KAAK,CAAC,MAAM,GAAG,QAAQ;QACvB,CAAC,CAAC,YAAY,IAAI,KAAK,CAAC;QACxB,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ;QACpC,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAE9C,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,OAAO;QACL,GAAG;QACH,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,IAAI,EAAE,KAAK,CAAC,MAAM;QAClB,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,KAAK;KACN,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,QAAQ,GAAG,iBAAiB,CAAC,QAAQ;IAC7D,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC;QACjD,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAClD,OAAO,QAAQ,CAAC;AAClB,CAAC","sourcesContent":["import { randomUUID } from \"node:crypto\";\nimport type {\n TransportEntry,\n TransportObject,\n} from \"../domain/transport.types.ts\";\nimport { transportDefaults } from \"./transport.constants.ts\";\n\nexport function encodeObject(key: string, bytes: Uint8Array) {\n if (bytes.byteLength > transportDefaults.maxBytes)\n throw new Error(\n \"Transport object exceeds 64 MiB; split large payloads into chunks\",\n );\n const entry: TransportEntry = {\n key,\n revision: randomUUID(),\n size: bytes.byteLength,\n modifiedAt: new Date().toISOString(),\n };\n const header = Buffer.alloc(transportDefaults.headerBytes, \" \");\n header.write(JSON.stringify(entry));\n return { entry, data: Buffer.concat([header, bytes]) };\n}\n\nexport function decodeObject(\n data: Uint8Array,\n key: string,\n maxBytes: number,\n): TransportObject {\n const buffer = Buffer.from(data);\n const value: unknown = JSON.parse(\n buffer.subarray(0, transportDefaults.headerBytes).toString(),\n );\n const bytes = buffer.subarray(transportDefaults.headerBytes);\n if (\n !value ||\n typeof value !== \"object\" ||\n !(\"key\" in value) ||\n value.key !== key ||\n !(\"revision\" in value) ||\n typeof value.revision !== \"string\" ||\n !value.revision ||\n !(\"size\" in value) ||\n value.size !== bytes.length ||\n bytes.length > maxBytes ||\n !(\"modifiedAt\" in value) ||\n typeof value.modifiedAt !== \"string\" ||\n !Number.isFinite(Date.parse(value.modifiedAt))\n )\n throw new Error(\"Invalid or oversized transport object\");\n return {\n key,\n revision: value.revision,\n size: bytes.length,\n modifiedAt: value.modifiedAt,\n bytes,\n };\n}\n\nexport function readLimit(maxBytes = transportDefaults.maxBytes): number {\n if (!Number.isSafeInteger(maxBytes) || maxBytes < 0)\n throw new Error(\"Invalid transport byte limit\");\n return maxBytes;\n}\n"]}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { Transport } from "../domain/transport.types.ts";
|
|
2
|
+
import type { Journal } from "./journal.types.ts";
|
|
3
|
+
import type { JournalSnapshot, ReadJournalOptions } from "./transport-journal.types.ts";
|
|
4
|
+
export declare function journalSnapshot(value: unknown): JournalSnapshot;
|
|
5
|
+
export declare function transportJournal(transporter: Transport, verbose: boolean, label?: string): Promise<Journal>;
|
|
6
|
+
export declare function readJournal(options: ReadJournalOptions): Promise<readonly unknown[]>;
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { transportDefaults } from "./transport.constants.js";
|
|
2
|
+
import { readLimit } from "./transport-envelope.js";
|
|
3
|
+
import { randomUUID } from "node:crypto";
|
|
4
|
+
import { jsonBytes, jsonObject, readReference, transportReference, } from "./transport-json.js";
|
|
5
|
+
import { positive } from "../domain/errors.js";
|
|
6
|
+
export function journalSnapshot(value) {
|
|
7
|
+
if (!value ||
|
|
8
|
+
typeof value !== "object" ||
|
|
9
|
+
!("closed" in value) ||
|
|
10
|
+
typeof value.closed !== "boolean" ||
|
|
11
|
+
!("head" in value))
|
|
12
|
+
throw new Error("Invalid journal snapshot");
|
|
13
|
+
return {
|
|
14
|
+
closed: value.closed,
|
|
15
|
+
head: value.head === null ? null : transportReference(value.head),
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
export async function transportJournal(transporter, verbose, label) {
|
|
19
|
+
const prefix = `logs/${randomUUID()}`;
|
|
20
|
+
let head = null;
|
|
21
|
+
let current = await transporter.write(`${prefix}/index`, jsonBytes({ closed: false, head }), { ifRevision: null });
|
|
22
|
+
let closing;
|
|
23
|
+
let pending = Promise.resolve();
|
|
24
|
+
let failure;
|
|
25
|
+
const append = async (event) => {
|
|
26
|
+
const segment = await transporter.write(`${prefix}/${randomUUID()}`, jsonBytes({ previous: head, event }), { ifRevision: null });
|
|
27
|
+
current = await transporter.write(current.key, jsonBytes({ closed: false, head: segment }), { ifRevision: current.revision });
|
|
28
|
+
head = segment;
|
|
29
|
+
};
|
|
30
|
+
await append({ kind: "dispatch-start", at: new Date().toISOString(), label });
|
|
31
|
+
return {
|
|
32
|
+
get reference() {
|
|
33
|
+
return { key: current.key, revision: current.revision };
|
|
34
|
+
},
|
|
35
|
+
record(event) {
|
|
36
|
+
if (closing || (event.kind === "raw" && !verbose))
|
|
37
|
+
return;
|
|
38
|
+
try {
|
|
39
|
+
const value = structuredClone({
|
|
40
|
+
at: new Date().toISOString(),
|
|
41
|
+
...(label ? { label } : {}),
|
|
42
|
+
...event,
|
|
43
|
+
});
|
|
44
|
+
pending = pending
|
|
45
|
+
.then(async () => {
|
|
46
|
+
if (!failure)
|
|
47
|
+
await append(value);
|
|
48
|
+
})
|
|
49
|
+
.catch((error) => {
|
|
50
|
+
failure = error;
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
catch (error) {
|
|
54
|
+
failure = error;
|
|
55
|
+
}
|
|
56
|
+
},
|
|
57
|
+
close() {
|
|
58
|
+
closing ??= (async () => {
|
|
59
|
+
await pending;
|
|
60
|
+
if (failure)
|
|
61
|
+
throw failure;
|
|
62
|
+
current = await transporter.write(current.key, jsonBytes({ closed: true, head }), { ifRevision: current.revision });
|
|
63
|
+
})();
|
|
64
|
+
return closing;
|
|
65
|
+
},
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
export async function readJournal(options) {
|
|
69
|
+
const maxEntries = positive(options.maxEntries ?? transportDefaults.maxEntries, "maxEntries");
|
|
70
|
+
const snapshot = journalSnapshot(jsonObject(await readReference(options.transporter, options.reference)));
|
|
71
|
+
let next = snapshot.head;
|
|
72
|
+
let remaining = readLimit(options.maxBytes);
|
|
73
|
+
const events = [];
|
|
74
|
+
const seen = new Set();
|
|
75
|
+
while (next) {
|
|
76
|
+
if (events.length >= maxEntries || seen.has(next.key))
|
|
77
|
+
throw new Error("Journal entry limit or cycle");
|
|
78
|
+
seen.add(next.key);
|
|
79
|
+
const object = await readReference(options.transporter, next, remaining);
|
|
80
|
+
remaining -= object.bytes.length;
|
|
81
|
+
const segment = jsonObject(object);
|
|
82
|
+
if (!segment ||
|
|
83
|
+
typeof segment !== "object" ||
|
|
84
|
+
!("event" in segment) ||
|
|
85
|
+
!("previous" in segment))
|
|
86
|
+
throw new Error("Invalid journal segment");
|
|
87
|
+
events.push(segment.event);
|
|
88
|
+
next =
|
|
89
|
+
segment.previous === null ? null : transportReference(segment.previous);
|
|
90
|
+
}
|
|
91
|
+
return events.reverse();
|
|
92
|
+
}
|
|
93
|
+
//# sourceMappingURL=transport-journal.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"transport-journal.js","sourceRoot":"","sources":["../../src/infrastructure/transport-journal.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAUzC,OAAO,EACL,SAAS,EACT,UAAU,EACV,aAAa,EACb,kBAAkB,GACnB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAE/C,MAAM,UAAU,eAAe,CAAC,KAAc;IAC5C,IACE,CAAC,KAAK;QACN,OAAO,KAAK,KAAK,QAAQ;QACzB,CAAC,CAAC,QAAQ,IAAI,KAAK,CAAC;QACpB,OAAO,KAAK,CAAC,MAAM,KAAK,SAAS;QACjC,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC;QAElB,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;IAC9C,OAAO;QACL,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,IAAI,EAAE,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC;KAClE,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,WAAsB,EACtB,OAAgB,EAChB,KAAc;IAEd,MAAM,MAAM,GAAG,QAAQ,UAAU,EAAE,EAAE,CAAC;IACtC,IAAI,IAAI,GAA8B,IAAI,CAAC;IAC3C,IAAI,OAAO,GAAG,MAAM,WAAW,CAAC,KAAK,CACnC,GAAG,MAAM,QAAQ,EACjB,SAAS,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAClC,EAAE,UAAU,EAAE,IAAI,EAAE,CACrB,CAAC;IACF,IAAI,OAAkC,CAAC;IACvC,IAAI,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAChC,IAAI,OAAgB,CAAC;IACrB,MAAM,MAAM,GAAG,KAAK,EAAE,KAAc,EAAE,EAAE;QACtC,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,KAAK,CACrC,GAAG,MAAM,IAAI,UAAU,EAAE,EAAE,EAC3B,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EACpC,EAAE,UAAU,EAAE,IAAI,EAAE,CACrB,CAAC;QACF,OAAO,GAAG,MAAM,WAAW,CAAC,KAAK,CAC/B,OAAO,CAAC,GAAG,EACX,SAAS,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,EAC3C,EAAE,UAAU,EAAE,OAAO,CAAC,QAAQ,EAAE,CACjC,CAAC;QACF,IAAI,GAAG,OAAO,CAAC;IACjB,CAAC,CAAC;IACF,MAAM,MAAM,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;IAC9E,OAAO;QACL,IAAI,SAAS;YACX,OAAO,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;QAC1D,CAAC;QACD,MAAM,CAAC,KAAK;YACV,IAAI,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,KAAK,IAAI,CAAC,OAAO,CAAC;gBAAE,OAAO;YAC1D,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,eAAe,CAAC;oBAC5B,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBAC5B,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC3B,GAAG,KAAK;iBACT,CAAC,CAAC;gBACH,OAAO,GAAG,OAAO;qBACd,IAAI,CAAC,KAAK,IAAI,EAAE;oBACf,IAAI,CAAC,OAAO;wBAAE,MAAM,MAAM,CAAC,KAAK,CAAC,CAAC;gBACpC,CAAC,CAAC;qBACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;oBACf,OAAO,GAAG,KAAK,CAAC;gBAClB,CAAC,CAAC,CAAC;YACP,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,GAAG,KAAK,CAAC;YAClB,CAAC;QACH,CAAC;QACD,KAAK;YACH,OAAO,KAAK,CAAC,KAAK,IAAI,EAAE;gBACtB,MAAM,OAAO,CAAC;gBACd,IAAI,OAAO;oBAAE,MAAM,OAAO,CAAC;gBAC3B,OAAO,GAAG,MAAM,WAAW,CAAC,KAAK,CAC/B,OAAO,CAAC,GAAG,EACX,SAAS,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,EACjC,EAAE,UAAU,EAAE,OAAO,CAAC,QAAQ,EAAE,CACjC,CAAC;YACJ,CAAC,CAAC,EAAE,CAAC;YACL,OAAO,OAAO,CAAC;QACjB,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,OAA2B;IAE3B,MAAM,UAAU,GAAG,QAAQ,CACzB,OAAO,CAAC,UAAU,IAAI,iBAAiB,CAAC,UAAU,EAClD,YAAY,CACb,CAAC;IACF,MAAM,QAAQ,GAAG,eAAe,CAC9B,UAAU,CAAC,MAAM,aAAa,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CACxE,CAAC;IACF,IAAI,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;IACzB,IAAI,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAc,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,MAAM,CAAC,MAAM,IAAI,UAAU,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAClD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnB,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;QACzE,SAAS,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;QACjC,MAAM,OAAO,GAAY,UAAU,CAAC,MAAM,CAAC,CAAC;QAC5C,IACE,CAAC,OAAO;YACR,OAAO,OAAO,KAAK,QAAQ;YAC3B,CAAC,CAAC,OAAO,IAAI,OAAO,CAAC;YACrB,CAAC,CAAC,UAAU,IAAI,OAAO,CAAC;YAExB,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAC7C,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC3B,IAAI;YACF,OAAO,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,kBAAkB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC5E,CAAC;IACD,OAAO,MAAM,CAAC,OAAO,EAAE,CAAC;AAC1B,CAAC","sourcesContent":["import { transportDefaults } from \"./transport.constants.ts\";\nimport { readLimit } from \"./transport-envelope.ts\";\nimport { randomUUID } from \"node:crypto\";\nimport type {\n Transport,\n TransportReference,\n} from \"../domain/transport.types.ts\";\nimport type { Journal } from \"./journal.types.ts\";\nimport type {\n JournalSnapshot,\n ReadJournalOptions,\n} from \"./transport-journal.types.ts\";\nimport {\n jsonBytes,\n jsonObject,\n readReference,\n transportReference,\n} from \"./transport-json.ts\";\nimport { positive } from \"../domain/errors.ts\";\n\nexport function journalSnapshot(value: unknown): JournalSnapshot {\n if (\n !value ||\n typeof value !== \"object\" ||\n !(\"closed\" in value) ||\n typeof value.closed !== \"boolean\" ||\n !(\"head\" in value)\n )\n throw new Error(\"Invalid journal snapshot\");\n return {\n closed: value.closed,\n head: value.head === null ? null : transportReference(value.head),\n };\n}\n\nexport async function transportJournal(\n transporter: Transport,\n verbose: boolean,\n label?: string,\n): Promise<Journal> {\n const prefix = `logs/${randomUUID()}`;\n let head: TransportReference | null = null;\n let current = await transporter.write(\n `${prefix}/index`,\n jsonBytes({ closed: false, head }),\n { ifRevision: null },\n );\n let closing: Promise<void> | undefined;\n let pending = Promise.resolve();\n let failure: unknown;\n const append = async (event: unknown) => {\n const segment = await transporter.write(\n `${prefix}/${randomUUID()}`,\n jsonBytes({ previous: head, event }),\n { ifRevision: null },\n );\n current = await transporter.write(\n current.key,\n jsonBytes({ closed: false, head: segment }),\n { ifRevision: current.revision },\n );\n head = segment;\n };\n await append({ kind: \"dispatch-start\", at: new Date().toISOString(), label });\n return {\n get reference() {\n return { key: current.key, revision: current.revision };\n },\n record(event) {\n if (closing || (event.kind === \"raw\" && !verbose)) return;\n try {\n const value = structuredClone({\n at: new Date().toISOString(),\n ...(label ? { label } : {}),\n ...event,\n });\n pending = pending\n .then(async () => {\n if (!failure) await append(value);\n })\n .catch((error) => {\n failure = error;\n });\n } catch (error) {\n failure = error;\n }\n },\n close() {\n closing ??= (async () => {\n await pending;\n if (failure) throw failure;\n current = await transporter.write(\n current.key,\n jsonBytes({ closed: true, head }),\n { ifRevision: current.revision },\n );\n })();\n return closing;\n },\n };\n}\n\nexport async function readJournal(\n options: ReadJournalOptions,\n): Promise<readonly unknown[]> {\n const maxEntries = positive(\n options.maxEntries ?? transportDefaults.maxEntries,\n \"maxEntries\",\n );\n const snapshot = journalSnapshot(\n jsonObject(await readReference(options.transporter, options.reference)),\n );\n let next = snapshot.head;\n let remaining = readLimit(options.maxBytes);\n const events: unknown[] = [];\n const seen = new Set<string>();\n while (next) {\n if (events.length >= maxEntries || seen.has(next.key))\n throw new Error(\"Journal entry limit or cycle\");\n seen.add(next.key);\n const object = await readReference(options.transporter, next, remaining);\n remaining -= object.bytes.length;\n const segment: unknown = jsonObject(object);\n if (\n !segment ||\n typeof segment !== \"object\" ||\n !(\"event\" in segment) ||\n !(\"previous\" in segment)\n )\n throw new Error(\"Invalid journal segment\");\n events.push(segment.event);\n next =\n segment.previous === null ? null : transportReference(segment.previous);\n }\n return events.reverse();\n}\n"]}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { TransportReference, TransportStoreOptions } from "../domain/transport.types.ts";
|
|
2
|
+
export interface ReadJournalOptions extends TransportStoreOptions {
|
|
3
|
+
readonly reference: TransportReference;
|
|
4
|
+
readonly maxEntries?: number;
|
|
5
|
+
readonly maxBytes?: number;
|
|
6
|
+
}
|
|
7
|
+
export interface JournalSnapshot {
|
|
8
|
+
readonly closed: boolean;
|
|
9
|
+
readonly head: TransportReference | null;
|
|
10
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"transport-journal.types.js","sourceRoot":"","sources":["../../src/infrastructure/transport-journal.types.ts"],"names":[],"mappings":"","sourcesContent":["import type {\n TransportReference,\n TransportStoreOptions,\n} from \"../domain/transport.types.ts\";\nexport interface ReadJournalOptions extends TransportStoreOptions {\n readonly reference: TransportReference;\n readonly maxEntries?: number;\n readonly maxBytes?: number;\n}\nexport interface JournalSnapshot {\n readonly closed: boolean;\n readonly head: TransportReference | null;\n}\n"]}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { Transport, TransportObject, TransportReference } from "../domain/transport.types.ts";
|
|
2
|
+
export declare function jsonBytes(value: unknown): Uint8Array;
|
|
3
|
+
export declare function jsonObject(value: TransportObject | undefined): unknown;
|
|
4
|
+
export declare function immutableObject(transporter: Transport, key: string, bytes: Uint8Array, signal?: AbortSignal): Promise<import("../domain/transport.types.ts").TransportEntry>;
|
|
5
|
+
export declare function readReference(transporter: Transport, reference: TransportReference, maxBytes?: number): Promise<TransportObject>;
|
|
6
|
+
export declare function transportReference(value: unknown): TransportReference;
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { TransportConflict } from "../domain/transport.js";
|
|
2
|
+
export function jsonBytes(value) {
|
|
3
|
+
return Buffer.from(JSON.stringify(value));
|
|
4
|
+
}
|
|
5
|
+
export function jsonObject(value) {
|
|
6
|
+
return value
|
|
7
|
+
? JSON.parse(Buffer.from(value.bytes).toString("utf8"))
|
|
8
|
+
: undefined;
|
|
9
|
+
}
|
|
10
|
+
export async function immutableObject(transporter, key, bytes, signal) {
|
|
11
|
+
try {
|
|
12
|
+
return await transporter.write(key, bytes, {
|
|
13
|
+
ifRevision: null,
|
|
14
|
+
...(signal ? { signal } : {}),
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
catch (error) {
|
|
18
|
+
if (!(error instanceof TransportConflict))
|
|
19
|
+
throw error;
|
|
20
|
+
const existing = await transporter.read(key, {
|
|
21
|
+
maxBytes: bytes.byteLength,
|
|
22
|
+
...(signal ? { signal } : {}),
|
|
23
|
+
});
|
|
24
|
+
if (!existing || !Buffer.from(bytes).equals(existing.bytes))
|
|
25
|
+
throw new Error("Existing object content integrity mismatch");
|
|
26
|
+
return existing;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
export async function readReference(transporter, reference, maxBytes) {
|
|
30
|
+
const value = await transporter.read(reference.key, maxBytes === undefined ? {} : { maxBytes });
|
|
31
|
+
if (!value || value.revision !== reference.revision)
|
|
32
|
+
throw new TransportConflict(reference.key);
|
|
33
|
+
return value;
|
|
34
|
+
}
|
|
35
|
+
export function transportReference(value) {
|
|
36
|
+
if (!value ||
|
|
37
|
+
typeof value !== "object" ||
|
|
38
|
+
!("key" in value) ||
|
|
39
|
+
typeof value.key !== "string" ||
|
|
40
|
+
!("revision" in value) ||
|
|
41
|
+
typeof value.revision !== "string" ||
|
|
42
|
+
!value.revision)
|
|
43
|
+
throw new Error("Invalid transport reference");
|
|
44
|
+
return { key: value.key, revision: value.revision };
|
|
45
|
+
}
|
|
46
|
+
//# sourceMappingURL=transport-json.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"transport-json.js","sourceRoot":"","sources":["../../src/infrastructure/transport-json.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAE3D,MAAM,UAAU,SAAS,CAAC,KAAc;IACtC,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;AAC5C,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,KAAkC;IAC3D,OAAO,KAAK;QACV,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACvD,CAAC,CAAC,SAAS,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,WAAsB,EACtB,GAAW,EACX,KAAiB,EACjB,MAAoB;IAEpB,IAAI,CAAC;QACH,OAAO,MAAM,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE;YACzC,UAAU,EAAE,IAAI;YAChB,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC9B,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC,CAAC,KAAK,YAAY,iBAAiB,CAAC;YAAE,MAAM,KAAK,CAAC;QACvD,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE;YAC3C,QAAQ,EAAE,KAAK,CAAC,UAAU;YAC1B,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC9B,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;YACzD,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAChE,OAAO,QAAQ,CAAC;IAClB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,WAAsB,EACtB,SAA6B,EAC7B,QAAiB;IAEjB,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,IAAI,CAClC,SAAS,CAAC,GAAG,EACb,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAC3C,CAAC;IACF,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,CAAC,QAAQ;QACjD,MAAM,IAAI,iBAAiB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC7C,OAAO,KAAK,CAAC;AACf,CAAC;AACD,MAAM,UAAU,kBAAkB,CAAC,KAAc;IAC/C,IACE,CAAC,KAAK;QACN,OAAO,KAAK,KAAK,QAAQ;QACzB,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC;QACjB,OAAO,KAAK,CAAC,GAAG,KAAK,QAAQ;QAC7B,CAAC,CAAC,UAAU,IAAI,KAAK,CAAC;QACtB,OAAO,KAAK,CAAC,QAAQ,KAAK,QAAQ;QAClC,CAAC,KAAK,CAAC,QAAQ;QAEf,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;IACjD,OAAO,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;AACtD,CAAC","sourcesContent":["import type {\n Transport,\n TransportObject,\n TransportReference,\n} from \"../domain/transport.types.ts\";\nimport { TransportConflict } from \"../domain/transport.ts\";\n\nexport function jsonBytes(value: unknown): Uint8Array {\n return Buffer.from(JSON.stringify(value));\n}\n\nexport function jsonObject(value: TransportObject | undefined): unknown {\n return value\n ? JSON.parse(Buffer.from(value.bytes).toString(\"utf8\"))\n : undefined;\n}\n\nexport async function immutableObject(\n transporter: Transport,\n key: string,\n bytes: Uint8Array,\n signal?: AbortSignal,\n) {\n try {\n return await transporter.write(key, bytes, {\n ifRevision: null,\n ...(signal ? { signal } : {}),\n });\n } catch (error) {\n if (!(error instanceof TransportConflict)) throw error;\n const existing = await transporter.read(key, {\n maxBytes: bytes.byteLength,\n ...(signal ? { signal } : {}),\n });\n if (!existing || !Buffer.from(bytes).equals(existing.bytes))\n throw new Error(\"Existing object content integrity mismatch\");\n return existing;\n }\n}\n\nexport async function readReference(\n transporter: Transport,\n reference: TransportReference,\n maxBytes?: number,\n): Promise<TransportObject> {\n const value = await transporter.read(\n reference.key,\n maxBytes === undefined ? {} : { maxBytes },\n );\n if (!value || value.revision !== reference.revision)\n throw new TransportConflict(reference.key);\n return value;\n}\nexport function transportReference(value: unknown): TransportReference {\n if (\n !value ||\n typeof value !== \"object\" ||\n !(\"key\" in value) ||\n typeof value.key !== \"string\" ||\n !(\"revision\" in value) ||\n typeof value.revision !== \"string\" ||\n !value.revision\n )\n throw new Error(\"Invalid transport reference\");\n return { key: value.key, revision: value.revision };\n}\n"]}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { Transport } from "../domain/transport.types.ts";
|
|
2
|
+
import type { StorageReservation, StorageReservationOptions } from "./storage-reservations.types.ts";
|
|
3
|
+
import type { ReservationLedger } from "./transport-reservations.types.ts";
|
|
4
|
+
export declare function reservationLedger(value: unknown): ReservationLedger;
|
|
5
|
+
export declare function reserveTransportStorage(transporter: Transport, repository: string, options: StorageReservationOptions): Promise<StorageReservation>;
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { transportDefaults } from "./transport.constants.js";
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
3
|
+
import { TransportConflict } from "../domain/transport.js";
|
|
4
|
+
import { invariant, recordRecovery } from "../domain/errors.js";
|
|
5
|
+
import { jsonBytes, jsonObject } from "./transport-json.js";
|
|
6
|
+
const key = "reservations/ledger";
|
|
7
|
+
export function reservationLedger(value) {
|
|
8
|
+
if (value === undefined)
|
|
9
|
+
return { format: 1, reservations: {} };
|
|
10
|
+
invariant(value &&
|
|
11
|
+
typeof value === "object" &&
|
|
12
|
+
"format" in value &&
|
|
13
|
+
value.format === 1 &&
|
|
14
|
+
"reservations" in value &&
|
|
15
|
+
value.reservations &&
|
|
16
|
+
typeof value.reservations === "object" &&
|
|
17
|
+
!Array.isArray(value.reservations), "Invalid reservation ledger");
|
|
18
|
+
const entries = Object.entries(value.reservations);
|
|
19
|
+
invariant(entries.every(([id, bytes]) => /^[a-f0-9-]{36}$/.test(id) &&
|
|
20
|
+
typeof bytes === "number" &&
|
|
21
|
+
Number.isSafeInteger(bytes) &&
|
|
22
|
+
bytes >= 0), "Invalid reservation entry");
|
|
23
|
+
return { format: 1, reservations: Object.fromEntries(entries) };
|
|
24
|
+
}
|
|
25
|
+
export async function reserveTransportStorage(transporter, repository, options) {
|
|
26
|
+
invariant(Number.isSafeInteger(options.maxBytes) &&
|
|
27
|
+
options.maxBytes >= 0 &&
|
|
28
|
+
Number.isSafeInteger(options.reserveBytes) &&
|
|
29
|
+
options.reserveBytes >= 0, "Invalid reservation byte limit");
|
|
30
|
+
const id = randomUUID();
|
|
31
|
+
const maxEntries = options.maxEntries ?? transportDefaults.maxEntries;
|
|
32
|
+
invariant(Number.isSafeInteger(maxEntries) && maxEntries > 0, "Invalid reservation entry limit");
|
|
33
|
+
async function update(release) {
|
|
34
|
+
for (let attempt = 0; attempt < transportDefaults.mutationAttempts; attempt++) {
|
|
35
|
+
if (!release)
|
|
36
|
+
options.signal?.throwIfAborted();
|
|
37
|
+
const current = await transporter.read(key);
|
|
38
|
+
const ledger = reservationLedger(jsonObject(current));
|
|
39
|
+
const reservations = { ...ledger.reservations };
|
|
40
|
+
if (release)
|
|
41
|
+
delete reservations[id];
|
|
42
|
+
if (!release) {
|
|
43
|
+
let usage = 0, count = 0;
|
|
44
|
+
for await (const entry of transporter.list("", options.signal ? { signal: options.signal } : {})) {
|
|
45
|
+
invariant(++count <= maxEntries, "Reservation inventory is incomplete");
|
|
46
|
+
if (entry.key !== key)
|
|
47
|
+
usage += entry.size;
|
|
48
|
+
}
|
|
49
|
+
const reserved = Object.values(reservations).reduce((sum, value) => sum + value, 0);
|
|
50
|
+
invariant(usage + reserved + options.reserveBytes <= options.maxBytes, "Outpost storage quota admission refused");
|
|
51
|
+
reservations[id] = options.reserveBytes;
|
|
52
|
+
}
|
|
53
|
+
try {
|
|
54
|
+
await transporter.write(key, jsonBytes({ format: 1, reservations }), {
|
|
55
|
+
ifRevision: current?.revision ?? null,
|
|
56
|
+
...(!release && options.signal ? { signal: options.signal } : {}),
|
|
57
|
+
});
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
catch (error) {
|
|
61
|
+
if (!(error instanceof TransportConflict))
|
|
62
|
+
throw error;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
throw new Error("Storage reservation contention limit exceeded");
|
|
66
|
+
}
|
|
67
|
+
try {
|
|
68
|
+
await update(false);
|
|
69
|
+
}
|
|
70
|
+
catch (error) {
|
|
71
|
+
recordRecovery(error, { reservation: id });
|
|
72
|
+
throw error;
|
|
73
|
+
}
|
|
74
|
+
let released = false;
|
|
75
|
+
const release = async () => {
|
|
76
|
+
if (released)
|
|
77
|
+
return;
|
|
78
|
+
await update(true);
|
|
79
|
+
released = true;
|
|
80
|
+
};
|
|
81
|
+
return {
|
|
82
|
+
id,
|
|
83
|
+
repository,
|
|
84
|
+
reserveBytes: options.reserveBytes,
|
|
85
|
+
release,
|
|
86
|
+
[Symbol.asyncDispose]: release,
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
//# sourceMappingURL=transport-reservations.js.map
|