@yielded/agent-storage-memory 0.1.0-beta.167 → 0.1.0-beta.169
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/MemoryMessageDeliveryStore.d.mts +7 -3
- package/dist/MemoryMessageDeliveryStore.mjs +427 -64
- package/dist/MemoryMessageDeliveryStore.mjs.map +1 -1
- package/dist/MemoryScheduleStore.mjs +8 -8
- package/dist/MemoryScheduleStore.mjs.map +1 -1
- package/dist/MemorySemanticIndex.mjs +7 -9
- package/dist/MemorySemanticIndex.mjs.map +1 -1
- package/dist/MemorySubmissionLedger.d.mts +7 -4
- package/dist/MemorySubmissionLedger.mjs +761 -377
- package/dist/MemorySubmissionLedger.mjs.map +1 -1
- package/dist/MemorySubscriptionStore.mjs +23 -23
- package/dist/MemorySubscriptionStore.mjs.map +1 -1
- package/dist/MemoryThreadStore.d.mts +8 -4
- package/dist/MemoryThreadStore.mjs +1325 -277
- package/dist/MemoryThreadStore.mjs.map +1 -1
- package/dist/MemoryThreadStoreKernel-DYNDjlhO.d.mts +85 -0
- package/dist/WorkSelection-CmqTuws3.mjs +31 -0
- package/dist/WorkSelection-CmqTuws3.mjs.map +1 -0
- package/package.json +1 -1
- package/src/MemoryMessageDeliveryStore.ts +766 -230
- package/src/MemoryScheduleStore.ts +190 -200
- package/src/MemorySemanticIndex.ts +94 -95
- package/src/MemorySubmissionLedger.ts +1955 -1245
- package/src/MemorySubscriptionStore.ts +1122 -1117
- package/src/MemoryThreadStore.ts +2609 -628
- package/src/internal/MemoryThreadStoreKernel.ts +156 -0
- package/src/internal/WorkSelection.ts +29 -0
|
@@ -1,20 +1,325 @@
|
|
|
1
1
|
import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { n as MemoryThreadStoreKernel, t as boundedWorkSelection } from "./WorkSelection-CmqTuws3.mjs";
|
|
3
|
+
import { Context, Crypto, Effect, Layer, MutableRef, Option, PubSub, Ref, Schema, Semaphore, Stream } from "effect";
|
|
4
|
+
import { RunId, SubmissionId, ThreadId } from "@yielded/agent/identifiers";
|
|
5
|
+
import { submissionAbortRecordId, submissionInputRecordId, submissionSettlementRecordId } from "@yielded/agent/submission-ledger";
|
|
6
|
+
import { ThreadDeliveryImportReader, ThreadImport, ThreadImportReader, ThreadImportRejected, captureImportSource, finishThreadImport, invalidThreadArchive, makeThreadImportProgress, prepareImportPage, rebuildClosedSubmission, validateClosedRun, verifyImportedReferences } from "@yielded/agent/thread-import";
|
|
7
|
+
import { AppendConflict, AppendResult, CheckpointRejected, LoadCheckpointRequest, PROMPT_EVIDENCE_TAGS, PreparedAppend, SaveCheckpointRequest, ThreadExportRecord, ThreadExportRejected, ThreadExportSource, ThreadIdentity, ThreadIdentityRequest, ThreadMaterialization, ThreadNotMaterialized, ThreadObservation, ThreadPeerCountRequest, ThreadPromptRead, ThreadReadRequest, ThreadReader, ThreadStore, ThreadStoreError, ThreadTail, ThreadTailRequest, ThreadVerificationRequest, ThreadWorkerCapacity, ThreadWorkerCapacityRequest, canonicalBatchFitsTransfer, streamExport, transferPageFits } from "@yielded/agent/thread-store";
|
|
8
|
+
import { MAX_WORK_REBUILD_BYTES, MAX_WORK_REBUILD_RECORDS, ThreadWorkRequest, WORK_INDEX_VERSION, WorkDiscoveryUnavailable, WorkIndexRebuildRequest, WorkThreadsRequest, advanceWorkEntry, decodeWorkCursor, encodeWorkCursor, validateWorkPage, workIndexChanges } from "@yielded/agent/thread-work";
|
|
4
9
|
import { Base64 } from "effect/encoding";
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import { runIdForSubmission, subagentLineageRecordId, workerOriginRecordId } from "@yielded/agent/run-journal";
|
|
8
|
-
import {
|
|
10
|
+
import { canonicalJson, utf8ByteLength } from "@yielded/agent/digest";
|
|
11
|
+
import { CURRENT_RECORD_FORMAT, CanonicalRecordEnvelope, CanonicalSequence, MAX_RUN_EVIDENCE_BYTES, MAX_RUN_EVIDENCE_RECORDS, MAX_RUN_RECOVERY_SUFFIX_BYTES, MAX_RUN_RECOVERY_SUFFIX_RECORDS, MAX_RUN_TERMINAL_BYTES, ObservationOffset, PromptRecord, RUN_TERMINAL_RESERVE_RECORDS, RecordEnvelope } from "@yielded/agent/records";
|
|
12
|
+
import { runIdForSubmission, subagentLineageRecordId, toolCallSettledRecordId, workerInputRecordId, workerOriginRecordId } from "@yielded/agent/run-journal";
|
|
13
|
+
import { ExportBatch, ExportRecord, ExportedRecord, ReadRecord } from "@yielded/agent/record-format";
|
|
14
|
+
import { ProgressAppendReader, canonicalRecordBytes, canonicalRunIds, isWorkHandoff, prepareProgressAppend, validateProgressAppend, verifyRunContinuations } from "@yielded/agent/run-continuation";
|
|
15
|
+
import { StorageOwner, makeStorageOwner } from "@yielded/agent/run-storage";
|
|
16
|
+
import { MAX_ARCHIVE_RANGE_BYTES, MAX_ARCHIVE_RANGE_RECORDS, ThreadArchiveRange, ThreadArchiveRangePublish, ThreadArchiveRangeRead, ThreadArchiveRangeRequest, ThreadArchiveRangeSeal } from "@yielded/agent/thread-archive-range";
|
|
17
|
+
import { verifyThreadInvariants } from "@yielded/agent/thread-invariants";
|
|
18
|
+
import { ThreadExporterReader, checkExportSource, exportThreadPage } from "@yielded/agent/thread-transfer";
|
|
9
19
|
//#region src/MemoryThreadStore.ts
|
|
10
20
|
var MemoryThreadStore_exports = /* @__PURE__ */ __exportAll({
|
|
11
21
|
MemoryThreadStoreLive: () => MemoryThreadStoreLive,
|
|
12
22
|
memoryThreadStoreLayer: () => memoryThreadStoreLayer
|
|
13
23
|
});
|
|
14
24
|
const MAX_THREADS = 256;
|
|
15
|
-
const
|
|
25
|
+
const RECORD_CHUNK_SIZE = 1024;
|
|
26
|
+
const READ_PAGE_SIZE = 8;
|
|
16
27
|
const MAX_CHECKPOINTS_PER_THREAD = 1024;
|
|
17
28
|
const ThreadCapacity = Context.Reference("@effect-agent/storage-memory/MemoryThreadStore/ThreadCapacity", { defaultValue: () => MAX_THREADS });
|
|
29
|
+
const storedRecord = (thread, sequence) => thread.chunks.get(Math.floor((sequence - 1) / RECORD_CHUNK_SIZE))?.records[(sequence - 1) % RECORD_CHUNK_SIZE];
|
|
30
|
+
const isBatchBoundary = (thread, sequence) => {
|
|
31
|
+
if (sequence === 0) return true;
|
|
32
|
+
const record = storedRecord(thread, sequence);
|
|
33
|
+
return record !== void 0 && thread.batches.get(record.batchId)?.result.lastSequence === sequence;
|
|
34
|
+
};
|
|
35
|
+
const storedBytes = (thread, sequence) => thread.chunks.get(Math.floor((sequence - 1) / RECORD_CHUNK_SIZE))?.bytes[(sequence - 1) % RECORD_CHUNK_SIZE];
|
|
36
|
+
const storedPage = (thread, after, limit) => {
|
|
37
|
+
const page = [];
|
|
38
|
+
let bytes = 0;
|
|
39
|
+
for (let sequence = after + 1; sequence <= Math.min(thread.tailSequence, after + limit); sequence++) {
|
|
40
|
+
const record = storedRecord(thread, sequence);
|
|
41
|
+
if (record === void 0) throw new Error("Incomplete canonical chunk table");
|
|
42
|
+
const size = storedBytes(thread, sequence) ?? 0;
|
|
43
|
+
if (bytes + size > 33554432) break;
|
|
44
|
+
bytes += size;
|
|
45
|
+
page.push(record);
|
|
46
|
+
}
|
|
47
|
+
return page;
|
|
48
|
+
};
|
|
49
|
+
const retainRecords = (thread, records) => {
|
|
50
|
+
for (const entry of records) {
|
|
51
|
+
const chunkId = Math.floor((entry.sequence - 1) / RECORD_CHUNK_SIZE);
|
|
52
|
+
let chunk = thread.chunks.get(chunkId);
|
|
53
|
+
if (chunk === void 0) {
|
|
54
|
+
chunk = {
|
|
55
|
+
records: [],
|
|
56
|
+
bytes: []
|
|
57
|
+
};
|
|
58
|
+
thread.chunks.set(chunkId, chunk);
|
|
59
|
+
}
|
|
60
|
+
chunk.records.push(entry);
|
|
61
|
+
chunk.bytes.push(canonicalRecordBytes(entry.record));
|
|
62
|
+
}
|
|
63
|
+
};
|
|
64
|
+
const emptyIndexes = () => ({
|
|
65
|
+
toolDeclarations: /* @__PURE__ */ new Map(),
|
|
66
|
+
promptEvidence: [],
|
|
67
|
+
modelCompletions: [],
|
|
68
|
+
applicationInputs: [],
|
|
69
|
+
durableOwners: [],
|
|
70
|
+
windowBoundaries: /* @__PURE__ */ new Map(),
|
|
71
|
+
peerCount: 0,
|
|
72
|
+
workerRecords: /* @__PURE__ */ new Map(),
|
|
73
|
+
byId: /* @__PURE__ */ new Map(),
|
|
74
|
+
runInputs: /* @__PURE__ */ new Map(),
|
|
75
|
+
runRecords: /* @__PURE__ */ new Map(),
|
|
76
|
+
handoffs: [],
|
|
77
|
+
agentUpdates: [],
|
|
78
|
+
deliveryPredecessors: [],
|
|
79
|
+
operationRecords: /* @__PURE__ */ new Map()
|
|
80
|
+
});
|
|
81
|
+
const liveInputPosition = (entries, sequence) => {
|
|
82
|
+
let low = 0;
|
|
83
|
+
let high = entries.length;
|
|
84
|
+
while (low < high) {
|
|
85
|
+
const middle = Math.floor((low + high) / 2);
|
|
86
|
+
if (entries[middle].sequence < sequence) low = middle + 1;
|
|
87
|
+
else high = middle;
|
|
88
|
+
}
|
|
89
|
+
return low;
|
|
90
|
+
};
|
|
91
|
+
const changeLiveInput = (entries, entry, change) => {
|
|
92
|
+
const at = liveInputPosition(entries, entry.createdSequence);
|
|
93
|
+
if (change === 1) entries.splice(at, 0, {
|
|
94
|
+
sequence: entry.createdSequence,
|
|
95
|
+
id: entry.id
|
|
96
|
+
});
|
|
97
|
+
else if (entries[at]?.id === entry.id) entries.splice(at, 1);
|
|
98
|
+
};
|
|
99
|
+
const emptyWorkIndex = () => ({
|
|
100
|
+
version: WORK_INDEX_VERSION,
|
|
101
|
+
state: "ready",
|
|
102
|
+
through: ZERO_CANONICAL_SEQUENCE,
|
|
103
|
+
reporting: "none",
|
|
104
|
+
entryCount: 0,
|
|
105
|
+
entries: /* @__PURE__ */ new Map(),
|
|
106
|
+
messages: /* @__PURE__ */ new Map(),
|
|
107
|
+
foreign: {
|
|
108
|
+
child: 0,
|
|
109
|
+
worker: 0,
|
|
110
|
+
delivery: 0,
|
|
111
|
+
peer: 0
|
|
112
|
+
},
|
|
113
|
+
workers: /* @__PURE__ */ new Map(),
|
|
114
|
+
liveInputs: [],
|
|
115
|
+
inputsByWorker: /* @__PURE__ */ new Map()
|
|
116
|
+
});
|
|
117
|
+
const changeWorkCounts = (index, entry, change) => {
|
|
118
|
+
const owner = entry.owner;
|
|
119
|
+
if (owner._tag === "Child" || owner._tag === "Handoff" && (owner.kind === "reservation" || owner.kind === "child-accounting")) index.foreign.child += change;
|
|
120
|
+
if ([
|
|
121
|
+
"WorkerInput",
|
|
122
|
+
"WorkerEffects",
|
|
123
|
+
"Report"
|
|
124
|
+
].includes(owner._tag) || owner._tag === "Handoff" && owner.kind === "worker-stop") index.foreign.worker += change;
|
|
125
|
+
if (owner._tag === "Handoff" && [
|
|
126
|
+
"peer",
|
|
127
|
+
"update",
|
|
128
|
+
"report"
|
|
129
|
+
].includes(owner.kind)) index.foreign.delivery += change;
|
|
130
|
+
if (owner._tag === "Handoff" && owner.kind === "peer") index.foreign.peer += change;
|
|
131
|
+
if (owner._tag === "WorkerInput") {
|
|
132
|
+
changeLiveInput(index.liveInputs, entry, change);
|
|
133
|
+
let inputs = index.inputsByWorker.get(owner.workerThreadId);
|
|
134
|
+
if (inputs === void 0) index.inputsByWorker.set(owner.workerThreadId, inputs = []);
|
|
135
|
+
changeLiveInput(inputs, entry, change);
|
|
136
|
+
if (inputs.length === 0) index.inputsByWorker.delete(owner.workerThreadId);
|
|
137
|
+
let worker = index.workers.get(owner.workerThreadId);
|
|
138
|
+
if (worker === void 0) index.workers.set(owner.workerThreadId, worker = {
|
|
139
|
+
ordinary: 0,
|
|
140
|
+
update: 0,
|
|
141
|
+
effects: 0
|
|
142
|
+
});
|
|
143
|
+
if (owner.update) worker.update += change;
|
|
144
|
+
else worker.ordinary += change;
|
|
145
|
+
if (worker.ordinary + worker.update + worker.effects === 0) index.workers.delete(owner.workerThreadId);
|
|
146
|
+
}
|
|
147
|
+
};
|
|
148
|
+
/** Prepare only this batch's changes; commit under the writer gate with canonical publication. */
|
|
149
|
+
const prepareWork = Effect.fnUntraced(function* (previous, records, retained) {
|
|
150
|
+
const changes = /* @__PURE__ */ new Map();
|
|
151
|
+
let reporting = previous.reporting;
|
|
152
|
+
let through = previous.through;
|
|
153
|
+
for (const record of records) {
|
|
154
|
+
for (const change of yield* workIndexChanges(record, reporting)) if (change._tag === "WorkerMode") reporting = change.mode;
|
|
155
|
+
else if (change._tag === "Remove") {
|
|
156
|
+
const entry = changes.has(change.id) ? changes.get(change.id) : previous.entries.get(change.id);
|
|
157
|
+
if (change.stateRecordId === void 0 || entry?.stateReference._tag === "Canonical" && entry.stateReference.recordId === change.stateRecordId) changes.set(change.id, void 0);
|
|
158
|
+
} else {
|
|
159
|
+
const entry = advanceWorkEntry(changes.has(change.entry.id) ? changes.get(change.entry.id) : previous.entries.get(change.entry.id), change.entry);
|
|
160
|
+
changes.set(entry.id, entry.owner._tag === "Handoff" && entry.owner.messageId !== void 0 && retained(record.threadId, entry.owner.messageId) ? void 0 : entry);
|
|
161
|
+
}
|
|
162
|
+
through = record.sequence;
|
|
163
|
+
}
|
|
164
|
+
return () => {
|
|
165
|
+
const { entries, messages } = previous;
|
|
166
|
+
for (const [id, entry] of changes) {
|
|
167
|
+
const old = entries.get(id);
|
|
168
|
+
if (old !== void 0) changeWorkCounts(previous, old, -1);
|
|
169
|
+
if (old?.owner._tag === "Handoff" && old.owner.messageId !== void 0) {
|
|
170
|
+
const ids = messages.get(old.owner.messageId);
|
|
171
|
+
ids?.delete(id);
|
|
172
|
+
if (ids?.size === 0) messages.delete(old.owner.messageId);
|
|
173
|
+
}
|
|
174
|
+
if (entry === void 0) entries.delete(id);
|
|
175
|
+
else {
|
|
176
|
+
entries.set(id, entry);
|
|
177
|
+
changeWorkCounts(previous, entry, 1);
|
|
178
|
+
if (entry.owner._tag === "Handoff" && entry.owner.messageId !== void 0) {
|
|
179
|
+
const ids = messages.get(entry.owner.messageId);
|
|
180
|
+
if (ids === void 0) messages.set(entry.owner.messageId, /* @__PURE__ */ new Set([id]));
|
|
181
|
+
else ids.add(id);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
return {
|
|
186
|
+
...previous,
|
|
187
|
+
through,
|
|
188
|
+
reporting,
|
|
189
|
+
entryCount: entries.size
|
|
190
|
+
};
|
|
191
|
+
};
|
|
192
|
+
});
|
|
193
|
+
/** First sequence strictly after the cursor, without traversing a Run's earlier facts. */
|
|
194
|
+
const upperSequence = (records, sequence) => {
|
|
195
|
+
let low = 0;
|
|
196
|
+
let high = records.length;
|
|
197
|
+
while (low < high) {
|
|
198
|
+
const mid = Math.floor((low + high) / 2);
|
|
199
|
+
if (records[mid].sequence <= sequence) low = mid + 1;
|
|
200
|
+
else high = mid;
|
|
201
|
+
}
|
|
202
|
+
return low;
|
|
203
|
+
};
|
|
204
|
+
const appendToIndex = (index, key, entry) => {
|
|
205
|
+
const records = index.get(key);
|
|
206
|
+
if (records === void 0) index.set(key, [entry]);
|
|
207
|
+
else records.push(entry);
|
|
208
|
+
};
|
|
209
|
+
/** Only call at successful synchronous publication, or on indexes not yet reachable by readers. */
|
|
210
|
+
const commitIndex = (index, additions) => {
|
|
211
|
+
for (const [key, entries] of additions) {
|
|
212
|
+
const records = index.get(key);
|
|
213
|
+
if (records === void 0) index.set(key, entries);
|
|
214
|
+
else for (const entry of entries) records.push(entry);
|
|
215
|
+
}
|
|
216
|
+
};
|
|
217
|
+
const prepareIndexes = (previous, records) => {
|
|
218
|
+
let peerCount = previous.peerCount;
|
|
219
|
+
const workerRecords = /* @__PURE__ */ new Map();
|
|
220
|
+
const wasWorker = (previous.workerRecords.get("worker") ?? []).some(({ record }) => record.payload._tag === "WorkerOriginRecorded");
|
|
221
|
+
const isWorker = wasWorker || records.some(({ record }) => record.payload._tag === "WorkerOriginRecorded");
|
|
222
|
+
if (isWorker && !wasWorker) for (const entry of previous.byId.values()) {
|
|
223
|
+
const payload = entry.record.payload;
|
|
224
|
+
if ((payload._tag === "UserInputRecorded" || payload._tag === "SubmissionSettled") && payload.runId !== void 0) {
|
|
225
|
+
const key = `execution:${payload._tag}`;
|
|
226
|
+
if ((workerRecords.get(key)?.[0]?.sequence ?? 0) < entry.sequence) workerRecords.set(key, [entry]);
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
const workerAppends = /* @__PURE__ */ new Map();
|
|
230
|
+
const byId = /* @__PURE__ */ new Map();
|
|
231
|
+
const runInputs = /* @__PURE__ */ new Map();
|
|
232
|
+
const runRecords = /* @__PURE__ */ new Map();
|
|
233
|
+
const handoffs = [];
|
|
234
|
+
const agentUpdates = [];
|
|
235
|
+
const deliveryPredecessors = [];
|
|
236
|
+
const nativeAdds = emptyIndexes();
|
|
237
|
+
const operationAppends = /* @__PURE__ */ new Map();
|
|
238
|
+
for (const entry of records) {
|
|
239
|
+
byId.set(entry.record.recordId, entry);
|
|
240
|
+
const payload = entry.record.payload;
|
|
241
|
+
if (payload._tag === "ModelResponseRecorded") for (const operation of payload.toolOperations) {
|
|
242
|
+
const id = toolCallSettledRecordId(payload.runId, payload.turn, operation.toolCallId);
|
|
243
|
+
nativeAdds.toolDeclarations.set(id, previous.toolDeclarations.has(id) || nativeAdds.toolDeclarations.has(id) ? null : entry);
|
|
244
|
+
}
|
|
245
|
+
if (PROMPT_EVIDENCE_TAGS.some((tag) => tag === payload._tag)) nativeAdds.promptEvidence.push(entry);
|
|
246
|
+
if (payload._tag === "ModelCompleted") nativeAdds.modelCompletions.push(entry);
|
|
247
|
+
if (payload._tag === "UserInputRecorded") {
|
|
248
|
+
const message = payload.messageAdmission;
|
|
249
|
+
if (message === void 0 || !("_tag" in message) || message._tag !== "WorkerCompletion" && message._tag !== "WorkerUpdate") nativeAdds.applicationInputs.push(entry);
|
|
250
|
+
}
|
|
251
|
+
if (payload._tag === "UserInputRecorded" && payload.submissionId !== void 0 || [
|
|
252
|
+
"RunStarted",
|
|
253
|
+
"SubmissionSettled",
|
|
254
|
+
"AbortRequested",
|
|
255
|
+
"WorkerOriginRecorded",
|
|
256
|
+
"SubagentLineageRecorded"
|
|
257
|
+
].includes(payload._tag)) nativeAdds.durableOwners.push(entry);
|
|
258
|
+
if (payload._tag === "CompactionCreated" && payload.kind === "rollover") for (let height = 0; height < 53; height++) appendToIndex(nativeAdds.windowBoundaries, `${height}:${Math.floor(payload.coversThrough / 2 ** height)}`, entry);
|
|
259
|
+
for (const runId of canonicalRunIds(entry.record)) appendToIndex(runRecords, runId, entry);
|
|
260
|
+
if (isWorkHandoff(entry.record)) handoffs.push(entry);
|
|
261
|
+
if (payload._tag === "AgentUpdateEmitted") agentUpdates.push(entry);
|
|
262
|
+
if (payload._tag === "WorkerReportPrepared" || payload._tag === "AgentUpdateEmitted" && payload.delivery !== void 0) deliveryPredecessors.push(entry);
|
|
263
|
+
if ("runId" in payload && payload.runId !== void 0 && "toolCallId" in payload) {
|
|
264
|
+
appendToIndex(operationAppends, JSON.stringify([payload.runId, payload.toolCallId]), entry);
|
|
265
|
+
if (payload._tag === "ToolApprovalRequested" || payload._tag === "ToolApprovalDecided") appendToIndex(operationAppends, JSON.stringify([
|
|
266
|
+
payload.runId,
|
|
267
|
+
payload.toolCallId,
|
|
268
|
+
"approval"
|
|
269
|
+
]), entry);
|
|
270
|
+
if (payload._tag === "ToolCallSettled") appendToIndex(operationAppends, JSON.stringify([
|
|
271
|
+
payload.runId,
|
|
272
|
+
payload.toolCallId,
|
|
273
|
+
"settled"
|
|
274
|
+
]), entry);
|
|
275
|
+
}
|
|
276
|
+
if (payload._tag === "PeerMessagePrepared") peerCount++;
|
|
277
|
+
if (isWorker && (payload._tag === "UserInputRecorded" || payload._tag === "SubmissionSettled") && payload.runId !== void 0) workerRecords.set(`execution:${payload._tag}`, [entry]);
|
|
278
|
+
const workerKey = payload._tag === "SubtreeBudgetReserved" ? `subtree:${payload.sourceSubmissionId ?? ""}` : payload._tag === "SubagentJoined" ? `joined:${payload.runId}` : [
|
|
279
|
+
"ThreadCreated",
|
|
280
|
+
"WorkerOriginRecorded",
|
|
281
|
+
"SubagentLineageRecorded"
|
|
282
|
+
].includes(payload._tag) ? "worker" : void 0;
|
|
283
|
+
if (workerKey !== void 0) appendToIndex(workerAppends, workerKey, entry);
|
|
284
|
+
if (payload._tag === "WorkerStopRequested") appendToIndex(workerAppends, `stop:${payload.command.worker.threadId}`, entry);
|
|
285
|
+
if (payload._tag === "WorkerInputRequested" && payload.admission.sourceSubmissionId !== void 0) appendToIndex(workerAppends, `funding:${payload.admission.sourceSubmissionId}`, entry);
|
|
286
|
+
if (payload._tag === "WorkerInputCompleted") {
|
|
287
|
+
const admission = (previous.byId.get(workerInputRecordId(payload.messageId)) ?? byId.get(workerInputRecordId(payload.messageId)))?.record.payload;
|
|
288
|
+
if (admission?._tag === "WorkerInputRequested" && admission.admission.sourceSubmissionId !== void 0) appendToIndex(workerAppends, `funding:${admission.admission.sourceSubmissionId}`, entry);
|
|
289
|
+
}
|
|
290
|
+
if (payload._tag === "WorkerInputRefused") {
|
|
291
|
+
const reserved = (previous.byId.get(payload.reservation.recordId) ?? byId.get(payload.reservation.recordId))?.record.payload;
|
|
292
|
+
const sourceSubmissionId = reserved?._tag === "WorkerInputRequested" ? reserved.admission.sourceSubmissionId : reserved?._tag === "SubtreeBudgetReserved" ? reserved.sourceSubmissionId : void 0;
|
|
293
|
+
if (sourceSubmissionId !== void 0) appendToIndex(workerAppends, `funding:${sourceSubmissionId}`, entry);
|
|
294
|
+
}
|
|
295
|
+
if (payload._tag === "UserInputRecorded" && payload.kind === "user" && payload.runId !== void 0) runInputs.set(payload.runId, runInputs.has(payload.runId) || previous.runInputs.has(payload.runId) ? null : entry);
|
|
296
|
+
}
|
|
297
|
+
return {
|
|
298
|
+
indexes: {
|
|
299
|
+
...previous,
|
|
300
|
+
peerCount
|
|
301
|
+
},
|
|
302
|
+
commit: () => {
|
|
303
|
+
commitIndex(previous.runRecords, runRecords);
|
|
304
|
+
commitIndex(previous.operationRecords, operationAppends);
|
|
305
|
+
for (const [id, entry] of nativeAdds.toolDeclarations) previous.toolDeclarations.set(id, entry);
|
|
306
|
+
for (const key of [
|
|
307
|
+
"promptEvidence",
|
|
308
|
+
"modelCompletions",
|
|
309
|
+
"applicationInputs"
|
|
310
|
+
]) for (const entry of nativeAdds[key]) previous[key].push(entry);
|
|
311
|
+
commitIndex(previous.windowBoundaries, nativeAdds.windowBoundaries);
|
|
312
|
+
if (previous.durableOwners.length === 0 && nativeAdds.durableOwners[0] !== void 0) previous.durableOwners.push(nativeAdds.durableOwners[0]);
|
|
313
|
+
for (const [id, entry] of byId) previous.byId.set(id, entry);
|
|
314
|
+
for (const [id, entry] of runInputs) previous.runInputs.set(id, entry);
|
|
315
|
+
for (const [key, entries] of workerRecords) previous.workerRecords.set(key, entries);
|
|
316
|
+
commitIndex(previous.workerRecords, workerAppends);
|
|
317
|
+
for (const entry of handoffs) previous.handoffs.push(entry);
|
|
318
|
+
for (const entry of agentUpdates) previous.agentUpdates.push(entry);
|
|
319
|
+
for (const entry of deliveryPredecessors) previous.deliveryPredecessors.push(entry);
|
|
320
|
+
}
|
|
321
|
+
};
|
|
322
|
+
};
|
|
18
323
|
const storeError = (operation, message, cause) => cause === void 0 ? ThreadStoreError.make({
|
|
19
324
|
operation,
|
|
20
325
|
message
|
|
@@ -23,27 +328,38 @@ const storeError = (operation, message, cause) => cause === void 0 ? ThreadStore
|
|
|
23
328
|
message,
|
|
24
329
|
cause
|
|
25
330
|
});
|
|
26
|
-
const validate =
|
|
331
|
+
const validate = (schema, operation, value) => Schema.encodeUnknownEffect(schema)(value).pipe(Effect.flatMap(Schema.decodeUnknownEffect(schema)), Effect.mapError((error) => storeError(operation, `Invalid ${operation} request`, error)));
|
|
27
332
|
const decodeCanonicalSequence = Schema.decodeSync(CanonicalSequence);
|
|
28
333
|
const ZERO_CANONICAL_SEQUENCE = decodeCanonicalSequence(0);
|
|
29
|
-
const offsetSequence = Effect.
|
|
30
|
-
if (offset === void 0) return Effect.succeed(ZERO_CANONICAL_SEQUENCE);
|
|
334
|
+
const offsetSequence = Effect.fnUntraced(function* (threadId, offset) {
|
|
335
|
+
if (offset === void 0) return yield* Effect.succeed(ZERO_CANONICAL_SEQUENCE);
|
|
31
336
|
const prefix = `memory:v1:${Base64.encode(threadId)}:`;
|
|
32
337
|
const encodedSequence = offset.startsWith(prefix) ? offset.slice(prefix.length) : "";
|
|
33
|
-
if (!/^\d+$/.test(encodedSequence)) return Effect.fail(storeError("observe", "Malformed observation offset"));
|
|
338
|
+
if (!/^\d+$/.test(encodedSequence)) return yield* Effect.fail(storeError("observe", "Malformed observation offset"));
|
|
34
339
|
const sequence = Number(encodedSequence);
|
|
35
|
-
return Number.isSafeInteger(sequence) ? Schema.decodeEffect(CanonicalSequence)(sequence).pipe(Effect.mapError(() => storeError("observe", "Malformed observation offset"))) : Effect.fail(storeError("observe", "Malformed observation offset"));
|
|
340
|
+
return yield* Number.isSafeInteger(sequence) ? Schema.decodeEffect(CanonicalSequence)(sequence).pipe(Effect.mapError(() => storeError("observe", "Malformed observation offset"))) : Effect.fail(storeError("observe", "Malformed observation offset"));
|
|
36
341
|
});
|
|
37
342
|
const observationOffset = (threadId, sequence) => Schema.decodeSync(ObservationOffset)(`memory:v1:${Base64.encode(threadId)}:${sequence}`);
|
|
38
|
-
const
|
|
343
|
+
const batchEnvelopes = (threadId, batch, firstSequence) => batch.records.map((record, index) => {
|
|
344
|
+
const sequence = decodeCanonicalSequence(firstSequence + index);
|
|
345
|
+
return CanonicalRecordEnvelope.make({
|
|
346
|
+
threadId,
|
|
347
|
+
batchId: batch.batchId,
|
|
348
|
+
sequence,
|
|
349
|
+
offset: observationOffset(threadId, sequence),
|
|
350
|
+
record
|
|
351
|
+
});
|
|
352
|
+
});
|
|
353
|
+
const decodeStoredBatch = (batchJson, operation) => Schema.decodeEffect(Schema.fromJsonString(ExportBatch))(batchJson).pipe(Effect.mapError((cause) => storeError(operation, "Invalid canonical batch JSON", cause)));
|
|
354
|
+
const findThread = Effect.fnUntraced(function* (state, threadId) {
|
|
39
355
|
const thread = state.threads.get(threadId);
|
|
40
|
-
return thread === void 0 ? Effect.fail(ThreadNotMaterialized.make({ threadId })) : Effect.succeed(thread);
|
|
356
|
+
return yield* thread === void 0 ? Effect.fail(ThreadNotMaterialized.make({ threadId })) : Effect.succeed(thread);
|
|
41
357
|
});
|
|
42
358
|
const CheckpointVersionEnvelope = Schema.Struct({ checkpoint: Schema.Struct({
|
|
43
359
|
threadId: ThreadId,
|
|
44
360
|
schemaVersion: Schema.Natural
|
|
45
361
|
}) });
|
|
46
|
-
const validateCheckpointVersion = Effect.
|
|
362
|
+
const validateCheckpointVersion = Effect.fnUntraced(function* (value) {
|
|
47
363
|
const envelope = yield* Schema.decodeUnknownEffect(CheckpointVersionEnvelope)(value).pipe(Effect.mapError(() => storeError("saveCheckpoint", "Invalid saveCheckpoint request")));
|
|
48
364
|
if (envelope.checkpoint.schemaVersion !== 1) return yield* CheckpointRejected.make({
|
|
49
365
|
threadId: envelope.checkpoint.threadId,
|
|
@@ -54,79 +370,303 @@ const makeThreadStore = Effect.gen(function* () {
|
|
|
54
370
|
const maxThreads = yield* ThreadCapacity;
|
|
55
371
|
const crypto = yield* Crypto.Crypto;
|
|
56
372
|
const state = yield* Ref.make({ threads: /* @__PURE__ */ new Map() });
|
|
373
|
+
const gate = yield* Semaphore.make(1);
|
|
374
|
+
const owner = yield* makeStorageOwner(gate.withPermits(1));
|
|
375
|
+
const withMutation = owner.withMutation;
|
|
57
376
|
const updates = yield* PubSub.sliding(1);
|
|
377
|
+
let ledgerTransfer;
|
|
378
|
+
let deliveryTransfer;
|
|
379
|
+
const workState = yield* Ref.make(/* @__PURE__ */ new Map());
|
|
380
|
+
const workOwners = /* @__PURE__ */ new Map();
|
|
381
|
+
let deliveryLookup;
|
|
382
|
+
const retainedDelivery = (threadId, messageId) => deliveryLookup?.(threadId, messageId) ?? false;
|
|
383
|
+
const publishWork = (threadId, index) => MutableRef.get(workState.ref).set(threadId, index);
|
|
384
|
+
const retainDelivery = (threadId, messageId) => {
|
|
385
|
+
const index = MutableRef.get(workState.ref).get(threadId);
|
|
386
|
+
const ids = index?.messages.get(messageId);
|
|
387
|
+
if (index === void 0 || index.version !== WORK_INDEX_VERSION || index.entryCount !== index.entries.size || ids === void 0) return;
|
|
388
|
+
const { entries, messages } = index;
|
|
389
|
+
for (const id of ids) {
|
|
390
|
+
const entry = entries.get(id);
|
|
391
|
+
if (entry !== void 0) changeWorkCounts(index, entry, -1);
|
|
392
|
+
entries.delete(id);
|
|
393
|
+
}
|
|
394
|
+
messages.delete(messageId);
|
|
395
|
+
publishWork(threadId, {
|
|
396
|
+
...index,
|
|
397
|
+
entryCount: entries.size
|
|
398
|
+
});
|
|
399
|
+
};
|
|
400
|
+
const readyIndex = (threadId, tail) => {
|
|
401
|
+
const index = MutableRef.get(workState.ref).get(threadId);
|
|
402
|
+
if (index === void 0 || index.version !== WORK_INDEX_VERSION || index.entries.size !== index.entryCount) return Effect.fail(WorkDiscoveryUnavailable.make({
|
|
403
|
+
threadId,
|
|
404
|
+
reason: "missing-index"
|
|
405
|
+
}));
|
|
406
|
+
if (index.state !== "ready" || index.through !== tail) return Effect.fail(WorkDiscoveryUnavailable.make({
|
|
407
|
+
threadId,
|
|
408
|
+
reason: "incomplete-rebuild"
|
|
409
|
+
}));
|
|
410
|
+
return Effect.succeed(index);
|
|
411
|
+
};
|
|
412
|
+
const work = {
|
|
413
|
+
threads: (unvalidated) => withMutation(Effect.gen(function* () {
|
|
414
|
+
const request = yield* validate(WorkThreadsRequest, "work threads", unvalidated);
|
|
415
|
+
const current = yield* Ref.get(state);
|
|
416
|
+
const catalogue = yield* Ref.get(workState);
|
|
417
|
+
const candidates = boundedWorkSelection(request.limit, (id) => id, request.afterThreadId);
|
|
418
|
+
for (const [threadId, thread] of current.threads) {
|
|
419
|
+
const index = catalogue.get(threadId);
|
|
420
|
+
if (index === void 0 || index.version !== WORK_INDEX_VERSION || index.state !== "ready" || index.through !== thread.tailSequence || index.entryCount !== index.entries.size || index.entryCount > 0) candidates.add(threadId);
|
|
421
|
+
}
|
|
422
|
+
for (const owner of workOwners.values()) {
|
|
423
|
+
const page = yield* owner.threads(request);
|
|
424
|
+
for (const threadId of page.threadIds) candidates.add(threadId);
|
|
425
|
+
}
|
|
426
|
+
const threadIds = candidates.values;
|
|
427
|
+
const afterThreadId = threadIds.length === request.limit ? threadIds.at(-1) : void 0;
|
|
428
|
+
return {
|
|
429
|
+
threadIds,
|
|
430
|
+
...afterThreadId === void 0 ? {} : { afterThreadId }
|
|
431
|
+
};
|
|
432
|
+
})),
|
|
433
|
+
page: (unvalidated) => withMutation(Effect.gen(function* () {
|
|
434
|
+
const request = yield* validate(ThreadWorkRequest, "work page", unvalidated);
|
|
435
|
+
const cursor = yield* decodeWorkCursor(request).pipe(Effect.provideService(Crypto.Crypto, crypto));
|
|
436
|
+
const thread = (yield* Ref.get(state)).threads.get(request.threadId);
|
|
437
|
+
const index = yield* readyIndex(request.threadId, thread?.tailSequence ?? 0);
|
|
438
|
+
if (cursor.source === "canonical") {
|
|
439
|
+
const candidates = boundedWorkSelection(request.limit + 1, (entry) => entry.id, cursor.after);
|
|
440
|
+
for (const entry of index.entries.values()) candidates.add(entry);
|
|
441
|
+
const selected = candidates.values;
|
|
442
|
+
const entries = selected.slice(0, request.limit);
|
|
443
|
+
const after = selected.length > request.limit ? entries.at(-1)?.id : void 0;
|
|
444
|
+
return yield* validateWorkPage({
|
|
445
|
+
entries,
|
|
446
|
+
cursor: encodeWorkCursor({
|
|
447
|
+
version: WORK_INDEX_VERSION,
|
|
448
|
+
threadDigest: cursor.threadDigest,
|
|
449
|
+
source: after === void 0 ? "deliveries" : "canonical",
|
|
450
|
+
...after === void 0 ? {} : { after }
|
|
451
|
+
})
|
|
452
|
+
}, request.limit);
|
|
453
|
+
}
|
|
454
|
+
const page = yield* workOwners.get(cursor.source)?.page(request.threadId, cursor.after, request.limit) ?? Effect.succeed({ entries: [] });
|
|
455
|
+
const next = page.after === void 0 ? cursor.source === "admissions" ? "canonical" : void 0 : cursor.source;
|
|
456
|
+
return yield* validateWorkPage({
|
|
457
|
+
entries: [...page.entries],
|
|
458
|
+
...next === void 0 ? {} : { cursor: encodeWorkCursor({
|
|
459
|
+
version: WORK_INDEX_VERSION,
|
|
460
|
+
threadDigest: cursor.threadDigest,
|
|
461
|
+
source: next,
|
|
462
|
+
...page.after === void 0 ? {} : { after: page.after }
|
|
463
|
+
}) }
|
|
464
|
+
}, request.limit);
|
|
465
|
+
})),
|
|
466
|
+
rebuild: (unvalidated) => withMutation(Effect.gen(function* () {
|
|
467
|
+
const request = yield* validate(WorkIndexRebuildRequest, "rebuild work", unvalidated);
|
|
468
|
+
const current = yield* Ref.get(state);
|
|
469
|
+
const thread = current.threads.get(request.threadId);
|
|
470
|
+
const tail = thread?.tailSequence ?? ZERO_CANONICAL_SEQUENCE;
|
|
471
|
+
const prior = (yield* Ref.get(workState)).get(request.threadId);
|
|
472
|
+
const index = request.restart || prior === void 0 || prior.version !== WORK_INDEX_VERSION || prior.entries.size !== prior.entryCount || prior.state === "ready" && prior.through !== tail ? {
|
|
473
|
+
...emptyWorkIndex(),
|
|
474
|
+
state: "rebuilding",
|
|
475
|
+
rebuildingIndexes: emptyIndexes()
|
|
476
|
+
} : prior;
|
|
477
|
+
if (index.state === "ready") return {
|
|
478
|
+
version: WORK_INDEX_VERSION,
|
|
479
|
+
threadId: request.threadId,
|
|
480
|
+
state: "ready",
|
|
481
|
+
throughSequence: index.through,
|
|
482
|
+
tailSequence: tail,
|
|
483
|
+
processedRecords: 0,
|
|
484
|
+
processedBytes: 0
|
|
485
|
+
};
|
|
486
|
+
if (index.rebuildingIndexes === void 0) return yield* WorkDiscoveryUnavailable.make({
|
|
487
|
+
threadId: request.threadId,
|
|
488
|
+
reason: "incomplete-rebuild"
|
|
489
|
+
});
|
|
490
|
+
const rebuildingIndexes = index.rebuildingIndexes;
|
|
491
|
+
const records = [];
|
|
492
|
+
let bytes = 0;
|
|
493
|
+
const end = Math.min(tail, index.through + (request.limit ?? MAX_WORK_REBUILD_RECORDS));
|
|
494
|
+
for (let position = index.through; position < end; position++) {
|
|
495
|
+
const size = thread === void 0 ? void 0 : storedBytes(thread, position + 1);
|
|
496
|
+
if (size === void 0) return yield* storeError("rebuild work", "Canonical record byte length is missing");
|
|
497
|
+
if (bytes + size > MAX_WORK_REBUILD_BYTES) break;
|
|
498
|
+
const record = thread === void 0 ? void 0 : storedRecord(thread, position + 1);
|
|
499
|
+
if (record === void 0 || record.sequence !== position + 1) return yield* storeError("rebuild work", "Canonical rebuild sequence is incomplete");
|
|
500
|
+
bytes += size;
|
|
501
|
+
records.push(record);
|
|
502
|
+
}
|
|
503
|
+
if (index.through < tail && records.length === 0) return yield* storeError("rebuild work", "Canonical rebuild cannot advance within its byte bound");
|
|
504
|
+
const commitWork = yield* prepareWork(index, records, retainedDelivery).pipe(Effect.provideService(Crypto.Crypto, crypto));
|
|
505
|
+
const next = yield* Effect.sync(() => {
|
|
506
|
+
const native = prepareIndexes(rebuildingIndexes, records);
|
|
507
|
+
const rebuilt = commitWork();
|
|
508
|
+
const complete = rebuilt.through === tail;
|
|
509
|
+
const next = {
|
|
510
|
+
version: rebuilt.version,
|
|
511
|
+
through: rebuilt.through,
|
|
512
|
+
reporting: rebuilt.reporting,
|
|
513
|
+
entryCount: rebuilt.entryCount,
|
|
514
|
+
entries: rebuilt.entries,
|
|
515
|
+
messages: rebuilt.messages,
|
|
516
|
+
foreign: rebuilt.foreign,
|
|
517
|
+
workers: rebuilt.workers,
|
|
518
|
+
liveInputs: rebuilt.liveInputs,
|
|
519
|
+
inputsByWorker: rebuilt.inputsByWorker,
|
|
520
|
+
state: complete ? "ready" : "rebuilding",
|
|
521
|
+
...complete ? {} : { rebuildingIndexes: native.indexes }
|
|
522
|
+
};
|
|
523
|
+
native.commit();
|
|
524
|
+
if (thread !== void 0 && complete) MutableRef.set(state.ref, { threads: new Map(current.threads).set(request.threadId, {
|
|
525
|
+
...thread,
|
|
526
|
+
...native.indexes
|
|
527
|
+
}) });
|
|
528
|
+
publishWork(request.threadId, next);
|
|
529
|
+
return next;
|
|
530
|
+
});
|
|
531
|
+
return {
|
|
532
|
+
version: WORK_INDEX_VERSION,
|
|
533
|
+
threadId: request.threadId,
|
|
534
|
+
state: next.state,
|
|
535
|
+
throughSequence: next.through,
|
|
536
|
+
tailSequence: tail,
|
|
537
|
+
processedRecords: records.length,
|
|
538
|
+
processedBytes: bytes
|
|
539
|
+
};
|
|
540
|
+
}))
|
|
541
|
+
};
|
|
58
542
|
yield* Effect.addFinalizer(() => PubSub.shutdown(updates));
|
|
59
|
-
const materialize = Effect.
|
|
543
|
+
const materialize = Effect.fnUntraced(function* (unvalidated) {
|
|
60
544
|
const request = yield* validate(ThreadMaterialization, "materialize", unvalidated);
|
|
61
545
|
const decision = yield* Ref.modify(state, (current) => {
|
|
62
|
-
|
|
63
|
-
if (
|
|
64
|
-
if (request.producerEpoch < existing.producerEpoch) return [{
|
|
65
|
-
_tag: "failure",
|
|
66
|
-
error: FenceRejected.make({
|
|
67
|
-
threadId: request.threadId,
|
|
68
|
-
actualEpoch: existing.producerEpoch,
|
|
69
|
-
attemptedEpoch: request.producerEpoch
|
|
70
|
-
})
|
|
71
|
-
}, current];
|
|
72
|
-
if (request.producerEpoch === existing.producerEpoch) return [{ _tag: "success" }, current];
|
|
73
|
-
const threads = new Map(current.threads);
|
|
74
|
-
threads.set(request.threadId, {
|
|
75
|
-
...existing,
|
|
76
|
-
producerEpoch: request.producerEpoch
|
|
77
|
-
});
|
|
78
|
-
return [{ _tag: "success" }, { threads }];
|
|
79
|
-
}
|
|
80
|
-
if (current.threads.size >= maxThreads) return [{
|
|
546
|
+
if (current.threads.get(request.threadId) !== void 0) return [{ _tag: "success" }, current];
|
|
547
|
+
if (!MutableRef.get(workState.ref).has(request.threadId) && MutableRef.get(workState.ref).size >= maxThreads) return [{
|
|
81
548
|
_tag: "failure",
|
|
82
549
|
error: storeError("materialize", `In-memory thread limit ${maxThreads} exceeded`)
|
|
83
550
|
}, current];
|
|
551
|
+
if (!transferPageFits(request.threadId)) return [{
|
|
552
|
+
_tag: "failure",
|
|
553
|
+
error: storeError("materialize", "Thread identity exceeds its bounded transfer representation")
|
|
554
|
+
}, current];
|
|
84
555
|
const threads = new Map(current.threads);
|
|
85
556
|
threads.set(request.threadId, {
|
|
86
|
-
producerEpoch: request.producerEpoch,
|
|
87
557
|
tailSequence: ZERO_CANONICAL_SEQUENCE,
|
|
88
|
-
tailDigest: EMPTY_TAIL_DIGEST,
|
|
89
|
-
unverifiedWorkerInputs: /* @__PURE__ */ new Set(),
|
|
90
558
|
byId: /* @__PURE__ */ new Map(),
|
|
91
559
|
workerRecords: /* @__PURE__ */ new Map(),
|
|
92
560
|
peerCount: 0,
|
|
93
561
|
runInputs: /* @__PURE__ */ new Map(),
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
562
|
+
runRecords: /* @__PURE__ */ new Map(),
|
|
563
|
+
operationRecords: /* @__PURE__ */ new Map(),
|
|
564
|
+
handoffs: [],
|
|
565
|
+
agentUpdates: [],
|
|
566
|
+
deliveryPredecessors: [],
|
|
567
|
+
toolDeclarations: /* @__PURE__ */ new Map(),
|
|
568
|
+
promptEvidence: [],
|
|
569
|
+
modelCompletions: [],
|
|
570
|
+
applicationInputs: [],
|
|
571
|
+
durableOwners: [],
|
|
572
|
+
windowBoundaries: /* @__PURE__ */ new Map(),
|
|
573
|
+
ranges: /* @__PURE__ */ new Map(),
|
|
574
|
+
rangeStarts: [],
|
|
575
|
+
archiveContents: /* @__PURE__ */ new Map(),
|
|
576
|
+
chunks: /* @__PURE__ */ new Map(),
|
|
577
|
+
batchesByFirst: /* @__PURE__ */ new Map(),
|
|
97
578
|
batches: /* @__PURE__ */ new Map(),
|
|
98
|
-
|
|
579
|
+
executions: /* @__PURE__ */ new Map(),
|
|
580
|
+
executionCommits: /* @__PURE__ */ new Map(),
|
|
99
581
|
checkpoints: /* @__PURE__ */ new Map()
|
|
100
582
|
});
|
|
583
|
+
publishWork(request.threadId, emptyWorkIndex());
|
|
101
584
|
return [{ _tag: "success" }, { threads }];
|
|
102
585
|
});
|
|
103
586
|
if (decision._tag === "failure") return yield* decision.error;
|
|
104
|
-
})
|
|
105
|
-
const
|
|
106
|
-
const
|
|
107
|
-
const
|
|
587
|
+
});
|
|
588
|
+
const prepareAppend = Effect.fnUntraced(function* (unvalidated) {
|
|
589
|
+
const captured = yield* PreparedAppend.capture(unvalidated);
|
|
590
|
+
const request = captured;
|
|
591
|
+
const batchJson = captured.batchJson;
|
|
592
|
+
const batchBytes = new TextEncoder().encode(batchJson).byteLength;
|
|
593
|
+
if (batchBytes > 16777216) return yield* storeError("append", "Canonical batch exceeds 16 MiB");
|
|
594
|
+
const offsetBytes = 4 * Math.ceil(utf8ByteLength(request.threadId) / 3) + 36;
|
|
595
|
+
if (!canonicalBatchFitsTransfer(request.threadId, request.batch, batchBytes, offsetBytes)) return yield* storeError("append", "Canonical batch exceeds its bounded transfer representation");
|
|
596
|
+
return {
|
|
597
|
+
request,
|
|
598
|
+
batchJson
|
|
599
|
+
};
|
|
600
|
+
});
|
|
601
|
+
const appendPrepared = Effect.fnUntraced(function* ({ request, batchJson }) {
|
|
602
|
+
const thread = (yield* Ref.get(state)).threads.get(request.threadId);
|
|
603
|
+
const executionIntent = JSON.stringify({
|
|
604
|
+
batch: batchJson,
|
|
605
|
+
operational: (request.executions ?? []).flatMap((update) => update.facts.map((fact) => Schema.encodeSync(RecordEnvelope)(fact)))
|
|
606
|
+
});
|
|
607
|
+
const latestIntent = thread === void 0 ? void 0 : [...thread.executionCommits.values()].find((entry) => entry.batchId === request.batch.batchId);
|
|
608
|
+
if (thread !== void 0 && latestIntent === void 0 && request.batch.records.length === 0) for (const update of request.executions ?? []) {
|
|
609
|
+
const prior = thread.executions.get(update.checkpoint.progress.runId);
|
|
610
|
+
if (prior !== void 0 && update.previousRevision < prior.progress.revision && update.facts.every((fact) => prior.operational.some((known) => known.recordId === fact.recordId))) return yield* AppendConflict.make({
|
|
611
|
+
threadId: request.threadId,
|
|
612
|
+
batchId: request.batch.batchId,
|
|
613
|
+
reason: "stale-command"
|
|
614
|
+
});
|
|
615
|
+
}
|
|
616
|
+
if (thread !== void 0 && thread.tailSequence === request.expectedTailSequence && !thread.batches.has(request.batch.batchId) && latestIntent === void 0) {
|
|
617
|
+
const executions = request.executions ?? [];
|
|
618
|
+
if (new Set(executions.map((update) => update.checkpoint.progress.runId)).size !== executions.length) return yield* storeError("append", "Duplicate active execution identity");
|
|
619
|
+
for (const update of request.executions ?? []) {
|
|
620
|
+
const prior = thread.executions.get(update.checkpoint.progress.runId);
|
|
621
|
+
if ((prior?.progress.revision ?? 0) !== update.previousRevision) return yield* AppendConflict.make({
|
|
622
|
+
threadId: request.threadId,
|
|
623
|
+
batchId: request.batch.batchId,
|
|
624
|
+
reason: "tail",
|
|
625
|
+
actualTailSequence: thread.tailSequence
|
|
626
|
+
});
|
|
627
|
+
if (update.checkpoint.throughSequence !== thread.tailSequence + request.batch.records.length || update.checkpoint.batchId !== request.batch.batchId || prior !== void 0 && update.checkpoint.originalSequence !== prior.originalSequence) return yield* storeError("append", "Active checkpoint differs from its committed history boundary");
|
|
628
|
+
if (update.retire && ![...request.batch.records, ...(thread.runRecords.get(update.checkpoint.progress.runId) ?? []).map((entry) => entry.record)].some(({ payload }) => payload._tag === "SubmissionSettled" && payload.runId === update.checkpoint.progress.runId && payload.submissionId === update.checkpoint.progress.submissionId)) return yield* storeError("append", "Active state cannot retire without a canonical terminal outcome");
|
|
629
|
+
}
|
|
630
|
+
yield* validateProgressAppend(prepareProgressAppend(request.batch.records), request.executions).pipe(Effect.provideService(ProgressAppendReader, {
|
|
631
|
+
previous: (runId) => Effect.succeed(thread.executions.get(runId)?.progress),
|
|
632
|
+
initial: (next) => Effect.succeed((thread.runRecords.get(next.runId) ?? []).map((entry) => entry.record))
|
|
633
|
+
}));
|
|
634
|
+
}
|
|
635
|
+
const currentWork = (yield* Ref.get(workState)).get(request.threadId);
|
|
636
|
+
const preparedWork = thread !== void 0 && thread.tailSequence === request.expectedTailSequence && !thread.batches.has(request.batch.batchId) && latestIntent === void 0 && currentWork?.version === WORK_INDEX_VERSION && currentWork.state === "ready" && currentWork.through === thread.tailSequence && currentWork.entryCount === currentWork.entries.size ? yield* prepareWork(currentWork, batchEnvelopes(request.threadId, request.batch, decodeCanonicalSequence(thread.tailSequence + 1)), retainedDelivery).pipe(Effect.provideService(Crypto.Crypto, crypto)) : void 0;
|
|
108
637
|
const decision = yield* Effect.uninterruptible(Ref.modify(state, (current) => {
|
|
109
638
|
const thread = current.threads.get(request.threadId);
|
|
110
639
|
if (thread === void 0) return [{
|
|
111
640
|
_tag: "failure",
|
|
112
641
|
error: ThreadNotMaterialized.make({ threadId: request.threadId })
|
|
113
642
|
}, current];
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
643
|
+
const priorExecution = latestIntent;
|
|
644
|
+
if (priorExecution !== void 0) {
|
|
645
|
+
if (priorExecution.intent !== executionIntent) return [{
|
|
646
|
+
_tag: "failure",
|
|
647
|
+
error: AppendConflict.make({
|
|
648
|
+
threadId: request.threadId,
|
|
649
|
+
batchId: request.batch.batchId,
|
|
650
|
+
reason: "batch-content"
|
|
651
|
+
})
|
|
652
|
+
}, current];
|
|
653
|
+
return [{
|
|
654
|
+
_tag: "success",
|
|
655
|
+
result: AppendResult.make({
|
|
656
|
+
...priorExecution.result,
|
|
657
|
+
replayed: true
|
|
658
|
+
}),
|
|
659
|
+
records: []
|
|
660
|
+
}, current];
|
|
661
|
+
}
|
|
122
662
|
const previous = thread.batches.get(request.batch.batchId);
|
|
123
663
|
if (previous !== void 0) {
|
|
124
|
-
if (previous.
|
|
664
|
+
if (previous.batchJson !== batchJson) return [{
|
|
125
665
|
_tag: "failure",
|
|
126
666
|
error: AppendConflict.make({
|
|
127
667
|
threadId: request.threadId,
|
|
128
668
|
batchId: request.batch.batchId,
|
|
129
|
-
reason: "batch-
|
|
669
|
+
reason: "batch-content"
|
|
130
670
|
})
|
|
131
671
|
}, current];
|
|
132
672
|
return [{
|
|
@@ -134,29 +674,23 @@ const makeThreadStore = Effect.gen(function* () {
|
|
|
134
674
|
result: AppendResult.make({
|
|
135
675
|
firstSequence: previous.result.firstSequence,
|
|
136
676
|
lastSequence: previous.result.lastSequence,
|
|
137
|
-
tailDigest: previous.result.tailDigest,
|
|
138
677
|
replayed: true
|
|
139
678
|
}),
|
|
140
679
|
records: []
|
|
141
680
|
}, current];
|
|
142
681
|
}
|
|
143
|
-
if (request.expectedTailSequence !== thread.tailSequence
|
|
682
|
+
if (request.expectedTailSequence !== thread.tailSequence) return [{
|
|
144
683
|
_tag: "failure",
|
|
145
684
|
error: AppendConflict.make({
|
|
146
685
|
threadId: request.threadId,
|
|
147
686
|
batchId: request.batch.batchId,
|
|
148
687
|
reason: "tail",
|
|
149
|
-
actualTailSequence: thread.tailSequence
|
|
150
|
-
actualTailDigest: thread.tailDigest
|
|
688
|
+
actualTailSequence: thread.tailSequence
|
|
151
689
|
})
|
|
152
690
|
}, current];
|
|
153
|
-
if (thread.records.length + request.batch.records.length > MAX_RECORDS_PER_THREAD) return [{
|
|
154
|
-
_tag: "failure",
|
|
155
|
-
error: storeError("append", `In-memory record limit ${MAX_RECORDS_PER_THREAD} exceeded`)
|
|
156
|
-
}, current];
|
|
157
691
|
const batchRecordIds = /* @__PURE__ */ new Set();
|
|
158
692
|
for (const record of request.batch.records) {
|
|
159
|
-
if (thread.
|
|
693
|
+
if (thread.byId.has(record.recordId) || batchRecordIds.has(record.recordId)) return [{
|
|
160
694
|
_tag: "failure",
|
|
161
695
|
error: AppendConflict.make({
|
|
162
696
|
threadId: request.threadId,
|
|
@@ -166,186 +700,558 @@ const makeThreadStore = Effect.gen(function* () {
|
|
|
166
700
|
}, current];
|
|
167
701
|
batchRecordIds.add(record.recordId);
|
|
168
702
|
}
|
|
169
|
-
const records = request.batch
|
|
170
|
-
const sequence = decodeCanonicalSequence(thread.tailSequence + index + 1);
|
|
171
|
-
return CanonicalRecordEnvelope.make({
|
|
172
|
-
threadId: request.threadId,
|
|
173
|
-
batchId: request.batch.batchId,
|
|
174
|
-
sequence,
|
|
175
|
-
offset: observationOffset(request.threadId, sequence),
|
|
176
|
-
record
|
|
177
|
-
});
|
|
178
|
-
});
|
|
703
|
+
const records = batchEnvelopes(request.threadId, request.batch, decodeCanonicalSequence(thread.tailSequence + 1));
|
|
179
704
|
const lastSequence = decodeCanonicalSequence(thread.tailSequence + records.length);
|
|
180
705
|
const result = AppendResult.make({
|
|
181
|
-
firstSequence: decodeCanonicalSequence(thread.tailSequence + 1),
|
|
706
|
+
firstSequence: decodeCanonicalSequence(thread.tailSequence + (records.length > 0 ? 1 : 0)),
|
|
182
707
|
lastSequence,
|
|
183
|
-
tailDigest: digest,
|
|
184
708
|
replayed: false
|
|
185
709
|
});
|
|
186
|
-
const
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
const
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
"WorkerInputCompleted",
|
|
213
|
-
"WorkerStopRequested"
|
|
214
|
-
].includes(payload._tag) ? "worker" : void 0;
|
|
215
|
-
if (workerKey !== void 0) workerRecords.set(workerKey, [...workerRecords.get(workerKey) ?? [], entry]);
|
|
216
|
-
if (payload._tag === "UserInputRecorded" && payload.kind === "user" && payload.runId !== void 0) runInputs.set(payload.runId, runInputs.has(payload.runId) ? null : entry);
|
|
217
|
-
if (payload._tag === "ToolCallPrepared" || payload._tag === "ToolCallUnknown" || payload._tag === "ToolCallSettled") {
|
|
218
|
-
const key = JSON.stringify([payload.runId, payload.toolCallId]);
|
|
219
|
-
if (payload._tag === "ToolCallSettled") outstanding.delete(key);
|
|
220
|
-
else outstanding.set(key, [...(outstanding.get(key) ?? []).filter((prior) => payload._tag !== "ToolCallUnknown" || prior.record.payload._tag !== "ToolCallPrepared"), entry]);
|
|
221
|
-
}
|
|
222
|
-
if (payload._tag === "WorkerInputRequested") outstanding.set(`worker:${payload.admission.messageId}`, [...outstanding.get(`worker:${payload.admission.messageId}`) ?? [], entry]);
|
|
223
|
-
if (payload._tag === "WorkerInputCompleted") {
|
|
224
|
-
if (payload.effectsResolved) {
|
|
225
|
-
outstanding.delete(`worker:${payload.messageId}`);
|
|
226
|
-
unverifiedWorkerInputs.delete(payload.messageId);
|
|
227
|
-
} else unverifiedWorkerInputs.add(payload.messageId);
|
|
710
|
+
const indexes = prepareIndexes(thread, records);
|
|
711
|
+
const next = {
|
|
712
|
+
...thread,
|
|
713
|
+
...indexes.indexes,
|
|
714
|
+
tailSequence: lastSequence
|
|
715
|
+
};
|
|
716
|
+
if (records.length > 0) {
|
|
717
|
+
thread.batches.set(request.batch.batchId, {
|
|
718
|
+
result,
|
|
719
|
+
batchJson
|
|
720
|
+
});
|
|
721
|
+
thread.batchesByFirst.set(result.firstSequence, request.batch.batchId);
|
|
722
|
+
retainRecords(thread, records);
|
|
723
|
+
}
|
|
724
|
+
for (const update of request.executions ?? []) {
|
|
725
|
+
const runId = update.checkpoint.progress.runId;
|
|
726
|
+
if (update.retire) {
|
|
727
|
+
thread.executions.delete(runId);
|
|
728
|
+
thread.executionCommits.delete(runId);
|
|
729
|
+
} else {
|
|
730
|
+
thread.executions.set(runId, update.checkpoint);
|
|
731
|
+
thread.executionCommits.set(runId, {
|
|
732
|
+
batchId: request.batch.batchId,
|
|
733
|
+
intent: executionIntent,
|
|
734
|
+
result
|
|
735
|
+
});
|
|
228
736
|
}
|
|
229
737
|
}
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
workerRecords,
|
|
234
|
-
peerCount,
|
|
235
|
-
runInputs,
|
|
236
|
-
outstanding,
|
|
237
|
-
unverifiedWorkerInputs,
|
|
238
|
-
tailSequence: lastSequence,
|
|
239
|
-
tailDigest: digest,
|
|
240
|
-
records: [...thread.records, ...records],
|
|
241
|
-
recordIds,
|
|
242
|
-
batches,
|
|
243
|
-
tailDigests
|
|
244
|
-
});
|
|
738
|
+
indexes.commit();
|
|
739
|
+
if (preparedWork !== void 0) publishWork(request.threadId, preparedWork());
|
|
740
|
+
current.threads.set(request.threadId, next);
|
|
245
741
|
return [{
|
|
246
742
|
_tag: "success",
|
|
247
743
|
result,
|
|
248
744
|
records
|
|
249
|
-
},
|
|
745
|
+
}, current];
|
|
250
746
|
}).pipe(Effect.tap((decision) => decision._tag === "success" && decision.records.length > 0 ? PubSub.publish(updates, void 0) : Effect.void)));
|
|
251
747
|
if (decision._tag === "failure") return yield* decision.error;
|
|
252
748
|
return decision.result;
|
|
253
|
-
})
|
|
749
|
+
});
|
|
750
|
+
const append = Effect.fn("MemoryThreadStore.append")((request) => prepareAppend(request).pipe(Effect.flatMap((prepared) => owner.withProgress(prepared.request.threadId, withMutation(appendPrepared(prepared))))));
|
|
254
751
|
const countPeerMessages = Effect.fnUntraced(function* (request) {
|
|
255
752
|
yield* validate(ThreadPeerCountRequest, "countPeerMessages", request);
|
|
256
753
|
const thread = yield* findThread(yield* Ref.get(state), request.threadId);
|
|
257
|
-
|
|
754
|
+
const index = yield* readyIndex(request.threadId, thread.tailSequence).pipe(Effect.mapError((cause) => storeError("countPeerMessages", cause.message)));
|
|
755
|
+
return Math.min(index.foreign.peer + (deliveryTransfer?.pendingPeerCount(request.threadId, request.limit) ?? 0), request.limit);
|
|
258
756
|
});
|
|
259
|
-
const readSnapshot =
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
757
|
+
const readSnapshot = (threadId, afterSequence, limit) => Ref.get(state).pipe(Effect.flatMap((current) => findThread(current, threadId)), Effect.map((thread) => {
|
|
758
|
+
return storedPage(thread, afterSequence ?? ZERO_CANONICAL_SEQUENCE, limit);
|
|
759
|
+
}));
|
|
760
|
+
const selectedStream = (sources, after, through, maximum) => Stream.paginate({
|
|
761
|
+
after,
|
|
762
|
+
count: 0
|
|
763
|
+
}, (cursor) => Effect.gen(function* () {
|
|
764
|
+
const page = [];
|
|
765
|
+
let bytes = 0;
|
|
766
|
+
let last = cursor.after;
|
|
767
|
+
while (page.length < Math.min(READ_PAGE_SIZE, maximum - cursor.count)) {
|
|
768
|
+
let selected;
|
|
769
|
+
for (const source of sources) {
|
|
770
|
+
const next = source[upperSequence(source, last)];
|
|
771
|
+
if (next !== void 0 && next.sequence <= through && (selected === void 0 || next.sequence < selected.sequence)) selected = next;
|
|
772
|
+
}
|
|
773
|
+
if (selected === void 0) break;
|
|
774
|
+
const size = canonicalRecordBytes(selected.record);
|
|
775
|
+
if (size > 33554432) return yield* storeError("read", "Canonical value exceeds the page byte bound");
|
|
776
|
+
if (bytes + size > 33554432) break;
|
|
777
|
+
bytes += size;
|
|
778
|
+
page.push(selected);
|
|
779
|
+
last = selected.sequence;
|
|
780
|
+
}
|
|
781
|
+
const next = {
|
|
782
|
+
after: last,
|
|
783
|
+
count: cursor.count + page.length
|
|
784
|
+
};
|
|
785
|
+
return [page, page.length === 0 || next.count >= maximum ? Option.none() : Option.some(next)];
|
|
786
|
+
}));
|
|
787
|
+
const readFromThread = (threadId, thread, request) => Stream.unwrap(Effect.gen(function* () {
|
|
788
|
+
if (request.threadId !== threadId) return yield* storeError("read snapshot", "Thread differs from the captured snapshot");
|
|
789
|
+
if (!("selection" in request)) return Stream.paginate({
|
|
790
|
+
after: request.afterSequence ?? ZERO_CANONICAL_SEQUENCE,
|
|
791
|
+
count: 0
|
|
792
|
+
}, (cursor) => Effect.gen(function* () {
|
|
793
|
+
const page = storedPage(thread, cursor.after, Math.min(READ_PAGE_SIZE, request.limit - cursor.count));
|
|
794
|
+
const next = {
|
|
795
|
+
after: page.at(-1)?.sequence ?? cursor.after,
|
|
796
|
+
count: cursor.count + page.length
|
|
797
|
+
};
|
|
798
|
+
return [page, page.length === 0 || next.count >= request.limit ? Option.none() : Option.some(next)];
|
|
799
|
+
}));
|
|
800
|
+
const selection = request.selection;
|
|
801
|
+
if ("expectedTailSequence" in selection && selection.expectedTailSequence !== thread.tailSequence) return yield* storeError("selected read", "Canonical tail changed");
|
|
802
|
+
if ("throughSequence" in selection && selection.throughSequence > thread.tailSequence) return yield* storeError("selected read", "Captured sequence is ahead of canonical tail");
|
|
803
|
+
const through = "throughSequence" in selection ? selection.throughSequence : thread.tailSequence;
|
|
804
|
+
const after = request.page.afterSequence ?? 0;
|
|
805
|
+
let sources = [];
|
|
806
|
+
const one = (entry) => entry === void 0 || entry === null ? [] : [entry];
|
|
807
|
+
const latest = (entries) => one(entries[upperSequence(entries, through) - 1]);
|
|
808
|
+
switch (selection._tag) {
|
|
809
|
+
case "RecordId":
|
|
810
|
+
sources = [one(thread.byId.get(selection.recordId))];
|
|
811
|
+
break;
|
|
812
|
+
case "RunInput": {
|
|
813
|
+
const entry = thread.runInputs.get(selection.runId);
|
|
814
|
+
if (entry === null) return yield* storeError("selected read", "Ambiguous original Run input");
|
|
815
|
+
sources = [one(entry)];
|
|
816
|
+
break;
|
|
817
|
+
}
|
|
818
|
+
case "ToolDeclaration": {
|
|
819
|
+
const entry = thread.toolDeclarations.get(selection.settlementRecordId);
|
|
820
|
+
if (entry === null) return yield* storeError("selected read", "Ambiguous original tool declaration");
|
|
821
|
+
if (entry !== void 0) {
|
|
822
|
+
const payload = entry.record.payload;
|
|
823
|
+
if (payload._tag !== "ModelResponseRecorded" || !payload.toolOperations.some((operation) => toolCallSettledRecordId(payload.runId, payload.turn, operation.toolCallId) === selection.settlementRecordId)) return yield* storeError("selected read", "Invalid exact tool declaration pointer");
|
|
281
824
|
}
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
825
|
+
sources = [one(entry)];
|
|
826
|
+
break;
|
|
827
|
+
}
|
|
828
|
+
case "RunEvidence":
|
|
829
|
+
sources = [thread.runRecords.get(selection.runId) ?? [], [
|
|
830
|
+
submissionInputRecordId(selection.submissionId),
|
|
831
|
+
submissionAbortRecordId(selection.submissionId),
|
|
832
|
+
submissionSettlementRecordId(selection.submissionId)
|
|
833
|
+
].flatMap((id) => one(thread.byId.get(id))).sort((a, b) => a.sequence - b.sequence)];
|
|
834
|
+
break;
|
|
835
|
+
case "OperationEvidence":
|
|
836
|
+
sources = [
|
|
837
|
+
thread.operationRecords.get(JSON.stringify([selection.runId, selection.toolCallId])) ?? [],
|
|
838
|
+
...selection.approvalToolCallIds.flatMap((toolCallId) => toolCallId === selection.toolCallId ? [] : [thread.operationRecords.get(JSON.stringify([
|
|
839
|
+
selection.runId,
|
|
840
|
+
toolCallId,
|
|
841
|
+
"approval"
|
|
842
|
+
])) ?? []]),
|
|
843
|
+
one(thread.byId.get(selection.originRecordId))
|
|
293
844
|
];
|
|
845
|
+
break;
|
|
846
|
+
case "LastAgentUpdate":
|
|
847
|
+
sources = [latest(thread.agentUpdates)];
|
|
848
|
+
break;
|
|
849
|
+
case "DeliveryPredecessor":
|
|
850
|
+
sources = [latest(thread.deliveryPredecessors)];
|
|
851
|
+
break;
|
|
852
|
+
case "LatestModelCompleted":
|
|
853
|
+
sources = [latest(thread.modelCompletions)];
|
|
854
|
+
break;
|
|
855
|
+
case "LatestApplicationInput":
|
|
856
|
+
sources = [latest(thread.applicationInputs)];
|
|
857
|
+
break;
|
|
858
|
+
case "DurableHistoryOwner":
|
|
859
|
+
sources = [one(thread.durableOwners[0])];
|
|
860
|
+
break;
|
|
861
|
+
case "PromptEvidence":
|
|
862
|
+
sources = [thread.promptEvidence];
|
|
863
|
+
break;
|
|
864
|
+
case "WorkHandoffs":
|
|
865
|
+
sources = [thread.handoffs];
|
|
866
|
+
break;
|
|
867
|
+
case "ContextWindowBoundary": {
|
|
868
|
+
let start = 0;
|
|
869
|
+
let selected;
|
|
870
|
+
while (start < selection.atSequence) {
|
|
871
|
+
const remaining = selection.atSequence - start;
|
|
872
|
+
let height = Math.floor(Math.log2(remaining));
|
|
873
|
+
while (2 ** height > remaining) height--;
|
|
874
|
+
const width = 2 ** height;
|
|
875
|
+
const entries = thread.windowBoundaries.get(`${height}:${start / width}`) ?? [];
|
|
876
|
+
const candidate = entries[upperSequence(entries, through) - 1];
|
|
877
|
+
if (candidate !== void 0 && (selected === void 0 || candidate.sequence > selected.sequence)) selected = candidate;
|
|
878
|
+
start += width;
|
|
879
|
+
}
|
|
880
|
+
sources = [one(selected)];
|
|
881
|
+
break;
|
|
882
|
+
}
|
|
883
|
+
case "WorkerStop":
|
|
884
|
+
sources = [latest(thread.workerRecords.get(`stop:${selection.workerThreadId}`) ?? [])];
|
|
885
|
+
break;
|
|
886
|
+
case "WorkerExecution":
|
|
887
|
+
sources = [thread.workerRecords.get("execution:UserInputRecorded") ?? [], thread.workerRecords.get("execution:SubmissionSettled") ?? []];
|
|
888
|
+
break;
|
|
889
|
+
case "WorkerState":
|
|
890
|
+
sources = [thread.workerRecords.get("worker") ?? [], ...selection.sourceSubmissionId === void 0 ? [] : [
|
|
891
|
+
thread.workerRecords.get(`subtree:${selection.sourceSubmissionId}`) ?? [],
|
|
892
|
+
thread.workerRecords.get(`joined:${runIdForSubmission(selection.sourceSubmissionId)}`) ?? [],
|
|
893
|
+
thread.workerRecords.get(`funding:${selection.sourceSubmissionId}`) ?? []
|
|
894
|
+
]];
|
|
895
|
+
break;
|
|
896
|
+
case "LiveWorkerInputs": {
|
|
897
|
+
const index = yield* readyIndex(request.threadId, thread.tailSequence).pipe(Effect.mapError((e) => storeError("selected read", e.message)));
|
|
898
|
+
if (selection.throughSequence !== thread.tailSequence) return yield* storeError("selected read", "Live work requires the current exact tail");
|
|
899
|
+
const locators = selection.workerThreadId === void 0 ? index.liveInputs : index.inputsByWorker.get(selection.workerThreadId) ?? [];
|
|
900
|
+
const start = liveInputPosition(locators, after + 1);
|
|
901
|
+
const selected = [];
|
|
902
|
+
for (let at = start; at < Math.min(locators.length, start + request.page.limit); at++) {
|
|
903
|
+
const entry = index.entries.get(locators[at].id);
|
|
904
|
+
const origin = entry === void 0 ? void 0 : thread.byId.get(entry.originRecordId);
|
|
905
|
+
if (entry?.owner._tag !== "WorkerInput" || origin?.record.payload._tag !== "WorkerInputRequested" || origin.sequence !== locators[at].sequence || origin.record.payload.admission.origin.worker.threadId !== entry.owner.workerThreadId || origin.record.payload.admission.reportKind === "update" !== entry.owner.update) return yield* storeError("selected read", "Invalid live worker input locator or origin");
|
|
906
|
+
selected.push(origin);
|
|
907
|
+
}
|
|
908
|
+
sources = [selected];
|
|
909
|
+
break;
|
|
294
910
|
}
|
|
295
|
-
return Stream.fromIterable(records.filter((entry) => entry.sequence > (request.page.afterSequence ?? 0)).sort((a, b) => a.sequence - b.sequence).slice(0, request.page.limit));
|
|
296
911
|
}
|
|
297
|
-
|
|
912
|
+
return selectedStream(sources, after, through, request.page.limit);
|
|
913
|
+
}));
|
|
914
|
+
const read = (input) => Stream.unwrap(Effect.gen(function* () {
|
|
915
|
+
const request = yield* validate(ThreadReadRequest, "read", input);
|
|
916
|
+
const thread = yield* findThread(MutableRef.get(state.ref), request.threadId);
|
|
917
|
+
return readFromThread(request.threadId, thread, request);
|
|
918
|
+
}));
|
|
919
|
+
const decodePromptRecord = Schema.decodeUnknownEffect(PromptRecord);
|
|
920
|
+
const readPrompt = (input) => Stream.unwrap(Effect.gen(function* () {
|
|
921
|
+
const request = yield* validate(ThreadPromptRead, "readPrompt", input);
|
|
922
|
+
const thread = yield* findThread(MutableRef.get(state.ref), request.threadId);
|
|
923
|
+
if (request.throughSequence > thread.tailSequence) return yield* storeError("readPrompt", "Captured sequence is ahead of canonical tail");
|
|
924
|
+
const start = upperSequence(thread.promptEvidence, request.afterSequence ?? 0);
|
|
925
|
+
const records = [];
|
|
926
|
+
for (let at = start; at < Math.min(thread.promptEvidence.length, start + request.limit); at++) {
|
|
927
|
+
const entry = thread.promptEvidence[at];
|
|
928
|
+
if (entry.sequence > request.throughSequence) break;
|
|
929
|
+
const record = yield* decodePromptRecord(entry.record instanceof ExportedRecord ? entry.record.wire : entry.record).pipe(Effect.mapError((cause) => storeError("readPrompt", "Invalid prompt record", cause)));
|
|
930
|
+
records.push({
|
|
931
|
+
threadId: request.threadId,
|
|
932
|
+
sequence: entry.sequence,
|
|
933
|
+
record
|
|
934
|
+
});
|
|
935
|
+
}
|
|
298
936
|
return Stream.fromIterable(records);
|
|
299
937
|
}));
|
|
300
938
|
const observe = (unvalidated) => Stream.unwrap(Effect.gen(function* () {
|
|
301
939
|
const request = yield* validate(ThreadObservation, "observe", unvalidated);
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
}));
|
|
310
|
-
}));
|
|
311
|
-
const exportThread = Effect.fn("MemoryThreadStore.export")((unvalidated) => Effect.gen(function* () {
|
|
312
|
-
const request = yield* validate(ThreadExportRequest, "export", unvalidated);
|
|
313
|
-
const thread = yield* Ref.get(state).pipe(Effect.flatMap((current) => findThread(current, request.threadId)));
|
|
314
|
-
return ThreadExport.make({
|
|
315
|
-
format: "effect-agent/thread@1",
|
|
316
|
-
threadId: request.threadId,
|
|
317
|
-
tailSequence: thread.tailSequence,
|
|
318
|
-
tailDigest: thread.tailDigest,
|
|
319
|
-
records: thread.records
|
|
320
|
-
});
|
|
940
|
+
let after = yield* offsetSequence(request.threadId, request.afterOffset);
|
|
941
|
+
const subscription = yield* PubSub.subscribe(updates);
|
|
942
|
+
const drain = () => Stream.paginate(after, (cursor) => readSnapshot(request.threadId, cursor, READ_PAGE_SIZE).pipe(Effect.map((records) => {
|
|
943
|
+
after = records.at(-1)?.sequence ?? cursor;
|
|
944
|
+
return [records, records.length > 0 ? Option.some(after) : Option.none()];
|
|
945
|
+
})));
|
|
946
|
+
return drain().pipe(Stream.concat(Stream.fromEffectRepeat(PubSub.take(subscription)).pipe(Stream.flatMap(drain))));
|
|
321
947
|
}));
|
|
322
|
-
const
|
|
948
|
+
const emptyThread = () => ({
|
|
949
|
+
...emptyIndexes(),
|
|
950
|
+
tailSequence: ZERO_CANONICAL_SEQUENCE,
|
|
951
|
+
ranges: /* @__PURE__ */ new Map(),
|
|
952
|
+
rangeStarts: [],
|
|
953
|
+
archiveContents: /* @__PURE__ */ new Map(),
|
|
954
|
+
chunks: /* @__PURE__ */ new Map(),
|
|
955
|
+
batches: /* @__PURE__ */ new Map(),
|
|
956
|
+
executions: /* @__PURE__ */ new Map(),
|
|
957
|
+
executionCommits: /* @__PURE__ */ new Map(),
|
|
958
|
+
batchesByFirst: /* @__PURE__ */ new Map(),
|
|
959
|
+
checkpoints: /* @__PURE__ */ new Map()
|
|
960
|
+
});
|
|
961
|
+
const makeExportReader = (idleOnly) => ThreadExporterReader.of({
|
|
962
|
+
snapshot: Effect.fnUntraced(function* (threadId, captured) {
|
|
963
|
+
const thread = MutableRef.get(state.ref).threads.get(threadId);
|
|
964
|
+
const facts = ledgerTransfer === void 0 ? {
|
|
965
|
+
idle: true,
|
|
966
|
+
admissions: 0,
|
|
967
|
+
aborts: 0,
|
|
968
|
+
approvals: 0,
|
|
969
|
+
resolutions: 0
|
|
970
|
+
} : yield* ledgerTransfer.snapshot(threadId);
|
|
971
|
+
const delivered = deliveryTransfer?.snapshot(threadId) ?? {
|
|
972
|
+
idle: true,
|
|
973
|
+
deliveries: 0
|
|
974
|
+
};
|
|
975
|
+
if (thread === void 0 && facts.admissions === 0 && delivered.deliveries === 0) return yield* ThreadNotMaterialized.make({ threadId });
|
|
976
|
+
if (idleOnly) {
|
|
977
|
+
const index = yield* readyIndex(threadId, thread?.tailSequence ?? 0).pipe(Effect.mapError((e) => storeError("export", e.message)));
|
|
978
|
+
if (owner.hasOwner(threadId) || (thread?.executions.size ?? 0) > 0 || index.entries.size > 0 || !facts.idle || !delivered.idle) return yield* ThreadExportRejected.make({
|
|
979
|
+
threadId,
|
|
980
|
+
reason: "non-idle"
|
|
981
|
+
});
|
|
982
|
+
if (captured === void 0 && thread !== void 0) yield* Effect.gen(function* () {
|
|
983
|
+
for (const runId of thread.runRecords.keys()) yield* validateClosedRun(threadId, Schema.decodeSync(RunId)(runId));
|
|
984
|
+
if (ledgerTransfer !== void 0) {
|
|
985
|
+
const ledger = ledgerTransfer;
|
|
986
|
+
yield* Stream.runForEach(ledger.submissions(threadId), (submission) => Effect.gen(function* () {
|
|
987
|
+
const admission = yield* ledger.admission(threadId, submission.submissionId);
|
|
988
|
+
if (admission === void 0) return yield* storeError("export", "Missing immutable admission");
|
|
989
|
+
yield* rebuildClosedSubmission(admission);
|
|
990
|
+
}));
|
|
991
|
+
}
|
|
992
|
+
}).pipe(Effect.provide(ThreadImportReader.layer(importReader(threadId, thread))), Effect.mapError((cause) => Schema.is(ThreadImportRejected)(cause) ? ThreadExportRejected.make({
|
|
993
|
+
threadId,
|
|
994
|
+
reason: "non-idle"
|
|
995
|
+
}) : cause));
|
|
996
|
+
}
|
|
997
|
+
return {
|
|
998
|
+
format: CURRENT_RECORD_FORMAT,
|
|
999
|
+
threadId,
|
|
1000
|
+
tailSequence: thread?.tailSequence ?? ZERO_CANONICAL_SEQUENCE,
|
|
1001
|
+
snapshot: {
|
|
1002
|
+
admissions: facts.admissions,
|
|
1003
|
+
aborts: facts.aborts,
|
|
1004
|
+
approvals: facts.approvals,
|
|
1005
|
+
resolutions: facts.resolutions,
|
|
1006
|
+
deliveries: delivered.deliveries
|
|
1007
|
+
},
|
|
1008
|
+
...facts.workerSeal === void 0 ? {} : { workerSeal: facts.workerSeal }
|
|
1009
|
+
};
|
|
1010
|
+
}),
|
|
1011
|
+
batch: Effect.fnUntraced(function* (threadId, fromSequence) {
|
|
1012
|
+
const thread = yield* findThread(MutableRef.get(state.ref), threadId);
|
|
1013
|
+
const batchId = thread.batchesByFirst.get(fromSequence);
|
|
1014
|
+
const stored = batchId === void 0 ? void 0 : thread.batches.get(batchId);
|
|
1015
|
+
if (stored === void 0 || new TextEncoder().encode(stored.batchJson).byteLength > 16777216) return yield* storeError("export", "Missing or oversized canonical batch wire");
|
|
1016
|
+
const batch = yield* decodeStoredBatch(stored.batchJson, "export");
|
|
1017
|
+
const records = [];
|
|
1018
|
+
for (const [index, record] of batch.records.entries()) {
|
|
1019
|
+
const sequence = decodeCanonicalSequence(fromSequence + index);
|
|
1020
|
+
const exact = storedRecord(thread, sequence);
|
|
1021
|
+
if (exact === void 0 || exact.batchId !== batch.batchId || exact.record.recordId !== record.recordId) return yield* storeError("export", "Canonical batch index differs from its wire");
|
|
1022
|
+
records.push(ThreadExportRecord.make({
|
|
1023
|
+
threadId,
|
|
1024
|
+
batchId: batch.batchId,
|
|
1025
|
+
sequence,
|
|
1026
|
+
offset: observationOffset(threadId, sequence),
|
|
1027
|
+
record
|
|
1028
|
+
}));
|
|
1029
|
+
}
|
|
1030
|
+
return {
|
|
1031
|
+
batches: [{
|
|
1032
|
+
batchId: batch.batchId,
|
|
1033
|
+
producerId: batch.producerId
|
|
1034
|
+
}],
|
|
1035
|
+
records
|
|
1036
|
+
};
|
|
1037
|
+
}),
|
|
1038
|
+
facts: (threadId, section, after) => {
|
|
1039
|
+
const empty = Effect.succeed({
|
|
1040
|
+
facts: {
|
|
1041
|
+
admissions: [],
|
|
1042
|
+
commands: {
|
|
1043
|
+
aborts: [],
|
|
1044
|
+
approvals: [],
|
|
1045
|
+
resolutions: []
|
|
1046
|
+
},
|
|
1047
|
+
deliveries: []
|
|
1048
|
+
},
|
|
1049
|
+
after: after ?? ""
|
|
1050
|
+
});
|
|
1051
|
+
return section === "deliveries" ? deliveryTransfer?.facts(threadId, after) ?? empty : ledgerTransfer?.facts(threadId, section, after) ?? empty;
|
|
1052
|
+
}
|
|
1053
|
+
});
|
|
1054
|
+
const exportReader = makeExportReader(true);
|
|
1055
|
+
const verificationReader = makeExportReader(false);
|
|
1056
|
+
const exportThread = (request) => exportThreadPage(request).pipe(Effect.provideService(ThreadExporterReader, exportReader));
|
|
1057
|
+
const verificationPage = (request) => exportThreadPage(request).pipe(Effect.provideService(ThreadExporterReader, verificationReader));
|
|
1058
|
+
const exactRunRecords = (thread, runId) => Effect.gen(function* () {
|
|
1059
|
+
const records = [];
|
|
1060
|
+
let bytes = 0;
|
|
1061
|
+
const facts = thread.runRecords.get(runId) ?? [];
|
|
1062
|
+
if (facts.length > MAX_RUN_EVIDENCE_RECORDS + RUN_TERMINAL_RESERVE_RECORDS + MAX_RUN_RECOVERY_SUFFIX_RECORDS) return yield* storeError("Run evidence", "Run record bounds exceeded");
|
|
1063
|
+
for (const entry of facts) {
|
|
1064
|
+
bytes += canonicalRecordBytes(entry.record);
|
|
1065
|
+
if (bytes > MAX_RUN_EVIDENCE_BYTES + MAX_RUN_TERMINAL_BYTES + MAX_RUN_RECOVERY_SUFFIX_BYTES) return yield* storeError("Run evidence", "Run byte bounds exceeded");
|
|
1066
|
+
records.push(entry);
|
|
1067
|
+
}
|
|
1068
|
+
return records.sort((a, b) => a.sequence - b.sequence);
|
|
1069
|
+
});
|
|
1070
|
+
const exactRecordReader = (threadId, thread) => ({
|
|
1071
|
+
read: (input) => Stream.unwrap(validate(ThreadReadRequest, "read snapshot", input).pipe(Effect.map((request) => readFromThread(threadId, thread, request)))),
|
|
1072
|
+
record: (id) => Effect.succeed(thread.byId.get(id)),
|
|
1073
|
+
runRecords: (id) => exactRunRecords(thread, id),
|
|
1074
|
+
execution: (id) => Effect.succeed(thread.executions.get(id)),
|
|
1075
|
+
hasAgent: (id) => Effect.succeed((thread.workerRecords.get("worker") ?? []).some(({ record }) => record.payload._tag === "ThreadCreated" && record.payload.agentId === id))
|
|
1076
|
+
});
|
|
1077
|
+
const importReader = (id, thread) => ({
|
|
1078
|
+
...exactRecordReader(id, thread),
|
|
1079
|
+
admission: (sid) => ledgerTransfer?.admission(id, sid) ?? Effect.succeed(void 0),
|
|
1080
|
+
runOwner: (runId) => runId.startsWith("run:") ? ledgerTransfer?.admission(id, Schema.decodeSync(SubmissionId)(runId.slice(4))) ?? Effect.succeed(void 0) : Effect.succeed(void 0),
|
|
1081
|
+
commands: (sid) => ledgerTransfer?.commands(sid) ?? Effect.succeed({
|
|
1082
|
+
aborts: [],
|
|
1083
|
+
approvals: [],
|
|
1084
|
+
resolutions: []
|
|
1085
|
+
}),
|
|
1086
|
+
delivery: (messageId) => deliveryTransfer?.record(id, messageId) ?? Effect.succeed(void 0)
|
|
1087
|
+
});
|
|
1088
|
+
const importThread = (pages) => captureImportSource(pages, (captured) => withMutation(Effect.gen(function* () {
|
|
1089
|
+
const progress = makeThreadImportProgress();
|
|
1090
|
+
let staged;
|
|
1091
|
+
let stagedWork = emptyWorkIndex();
|
|
1092
|
+
let stagedLedger;
|
|
1093
|
+
let stagedDelivery;
|
|
1094
|
+
let existing;
|
|
1095
|
+
yield* Stream.runForEach(captured, (input) => Effect.gen(function* () {
|
|
1096
|
+
const page = yield* prepareImportPage(progress, input).pipe(Effect.provideService(Crypto.Crypto, crypto));
|
|
1097
|
+
const id = page.archive.threadId;
|
|
1098
|
+
if (staged === void 0) {
|
|
1099
|
+
existing = MutableRef.get(state.ref).threads.get(id);
|
|
1100
|
+
const existingWork = MutableRef.get(workState.ref).get(id);
|
|
1101
|
+
if (existing !== void 0 && (existing.tailSequence > 0 || existing.batches.size > 0 || existing.executions.size > 0 || existing.executionCommits.size > 0 || existing.checkpoints.size > 0 || existing.ranges.size > 0 || existing.archiveContents.size > 0) || existingWork !== void 0 && (existingWork.entries.size > 0 || existingWork.state !== "ready" || existingWork.through !== 0)) return yield* ThreadImportRejected.make({
|
|
1102
|
+
threadId: id,
|
|
1103
|
+
reason: "target-not-empty",
|
|
1104
|
+
message: "Destination canonical Thread is not empty"
|
|
1105
|
+
});
|
|
1106
|
+
if (!MutableRef.get(workState.ref).has(id) && MutableRef.get(workState.ref).size >= maxThreads) return yield* storeError("import", "Destination Thread capacity exceeded");
|
|
1107
|
+
staged = emptyThread();
|
|
1108
|
+
stagedLedger = ledgerTransfer === void 0 ? void 0 : yield* ledgerTransfer.startImport(id);
|
|
1109
|
+
stagedDelivery = deliveryTransfer === void 0 ? void 0 : yield* deliveryTransfer.startImport(id);
|
|
1110
|
+
if (page.archive.workerSeal !== void 0 && stagedLedger === void 0) return yield* ThreadImportRejected.make({
|
|
1111
|
+
threadId: id,
|
|
1112
|
+
reason: "unsupported-obligations",
|
|
1113
|
+
message: "A paired ledger is required to preserve the worker seal"
|
|
1114
|
+
});
|
|
1115
|
+
}
|
|
1116
|
+
if (page.archive.admissions.length > 0 && stagedLedger === void 0) return yield* ThreadImportRejected.make({
|
|
1117
|
+
threadId: id,
|
|
1118
|
+
reason: "unsupported-obligations",
|
|
1119
|
+
message: "A paired ledger is required for admission restore"
|
|
1120
|
+
});
|
|
1121
|
+
if (page.archive.deliveries.length > 0 && stagedDelivery === void 0) return yield* ThreadImportRejected.make({
|
|
1122
|
+
threadId: id,
|
|
1123
|
+
reason: "unsupported-obligations",
|
|
1124
|
+
message: "A paired delivery store is required for delivery restore"
|
|
1125
|
+
});
|
|
1126
|
+
for (const batch of page.batches) {
|
|
1127
|
+
if (!canonicalBatchFitsTransfer(id, batch.batch, utf8ByteLength(batch.batchJson), 4 * Math.ceil(utf8ByteLength(id) / 3) + 36)) return yield* ThreadImportRejected.make({
|
|
1128
|
+
threadId: id,
|
|
1129
|
+
reason: "unsupported-capacity",
|
|
1130
|
+
message: "Canonical batch exceeds its bounded transfer representation"
|
|
1131
|
+
});
|
|
1132
|
+
if (staged.batches.has(batch.batch.batchId)) return yield* invalidThreadArchive("Duplicate canonical batch identity", id);
|
|
1133
|
+
const records = page.records.filter((entry) => entry.batchId === batch.batch.batchId).map((entry) => CanonicalRecordEnvelope.make({
|
|
1134
|
+
...entry,
|
|
1135
|
+
offset: observationOffset(id, entry.sequence)
|
|
1136
|
+
}));
|
|
1137
|
+
for (const entry of records) if (staged.byId.has(entry.record.recordId)) return yield* invalidThreadArchive("Duplicate canonical record identity", id);
|
|
1138
|
+
const native = prepareIndexes(staged, records);
|
|
1139
|
+
const commitWork = yield* prepareWork(stagedWork, records, (_, messageId) => stagedDelivery?.has(messageId) ?? false).pipe(Effect.provideService(Crypto.Crypto, crypto));
|
|
1140
|
+
retainRecords(staged, records);
|
|
1141
|
+
native.commit();
|
|
1142
|
+
staged = {
|
|
1143
|
+
...staged,
|
|
1144
|
+
...native.indexes,
|
|
1145
|
+
tailSequence: batch.lastSequence
|
|
1146
|
+
};
|
|
1147
|
+
staged.batches.set(batch.batch.batchId, {
|
|
1148
|
+
batchJson: batch.batchJson,
|
|
1149
|
+
result: AppendResult.make({
|
|
1150
|
+
firstSequence: batch.firstSequence,
|
|
1151
|
+
lastSequence: batch.lastSequence,
|
|
1152
|
+
replayed: false
|
|
1153
|
+
})
|
|
1154
|
+
});
|
|
1155
|
+
staged.batchesByFirst.set(batch.firstSequence, batch.batch.batchId);
|
|
1156
|
+
stagedWork = commitWork();
|
|
1157
|
+
}
|
|
1158
|
+
if (stagedLedger !== void 0) yield* stagedLedger.stage(page);
|
|
1159
|
+
if (stagedDelivery !== void 0) yield* stagedDelivery.stage(page);
|
|
1160
|
+
for (const delivery of page.archive.deliveries) {
|
|
1161
|
+
for (const workId of stagedWork.messages.get(delivery.key.messageId) ?? []) {
|
|
1162
|
+
const entry = stagedWork.entries.get(workId);
|
|
1163
|
+
if (entry !== void 0) changeWorkCounts(stagedWork, entry, -1);
|
|
1164
|
+
stagedWork.entries.delete(workId);
|
|
1165
|
+
}
|
|
1166
|
+
stagedWork.messages.delete(delivery.key.messageId);
|
|
1167
|
+
stagedWork = {
|
|
1168
|
+
...stagedWork,
|
|
1169
|
+
entryCount: stagedWork.entries.size
|
|
1170
|
+
};
|
|
1171
|
+
}
|
|
1172
|
+
}));
|
|
1173
|
+
const result = yield* finishThreadImport(progress);
|
|
1174
|
+
if (staged === void 0) return yield* invalidThreadArchive("Missing transfer pages");
|
|
1175
|
+
const restoredThread = staged;
|
|
1176
|
+
const id = result.threadId;
|
|
1177
|
+
const fallback = {
|
|
1178
|
+
admission: () => Effect.succeed(void 0),
|
|
1179
|
+
runOwner: () => Effect.succeed(void 0),
|
|
1180
|
+
commands: () => Effect.succeed({
|
|
1181
|
+
aborts: [],
|
|
1182
|
+
approvals: [],
|
|
1183
|
+
resolutions: []
|
|
1184
|
+
})
|
|
1185
|
+
};
|
|
1186
|
+
const reader = {
|
|
1187
|
+
...exactRecordReader(id, restoredThread),
|
|
1188
|
+
...stagedLedger?.reader ?? fallback,
|
|
1189
|
+
delivery: stagedDelivery?.record ?? (() => Effect.succeed(void 0))
|
|
1190
|
+
};
|
|
1191
|
+
return yield* Effect.gen(function* () {
|
|
1192
|
+
for (const runId of restoredThread.runRecords.keys()) {
|
|
1193
|
+
yield* verifyRunContinuations(yield* reader.runRecords(Schema.decodeSync(RunId)(runId)), { verifyContext: false }).pipe(Effect.mapError((e) => invalidThreadArchive(e.message, id)));
|
|
1194
|
+
yield* validateClosedRun(id, Schema.decodeSync(RunId)(runId));
|
|
1195
|
+
}
|
|
1196
|
+
let lastInputQueue = -1;
|
|
1197
|
+
for (let sequence = 1; sequence <= restoredThread.tailSequence; sequence++) {
|
|
1198
|
+
const entry = storedRecord(restoredThread, sequence);
|
|
1199
|
+
if (entry === void 0) return yield* invalidThreadArchive("Incomplete canonical staging", id);
|
|
1200
|
+
yield* verifyImportedReferences(entry);
|
|
1201
|
+
const payload = entry.record.payload;
|
|
1202
|
+
if (payload._tag === "UserInputRecorded" && payload.submissionId !== void 0) {
|
|
1203
|
+
const admission = yield* reader.admission(payload.submissionId);
|
|
1204
|
+
if (admission === void 0 || admission.queueSequence <= lastInputQueue) return yield* invalidThreadArchive("Canonical inputs violate immutable admission order", id);
|
|
1205
|
+
lastInputQueue = admission.queueSequence;
|
|
1206
|
+
}
|
|
1207
|
+
if (payload._tag === "UserInputRecorded" && payload.submissionId !== void 0 && entry.record.recordId !== submissionInputRecordId(payload.submissionId)) return yield* invalidThreadArchive("Canonical input identity is ambiguous", id);
|
|
1208
|
+
if (payload._tag === "SubmissionSettled" && entry.record.recordId !== submissionSettlementRecordId(payload.submissionId)) return yield* invalidThreadArchive("Canonical settlement identity is ambiguous", id);
|
|
1209
|
+
if (payload._tag !== "WorkerInputCompleted" && "submissionId" in payload && payload.submissionId !== void 0 && (yield* reader.admission(payload.submissionId)) === void 0) return yield* invalidThreadArchive("Canonical submission has no admission", id);
|
|
1210
|
+
}
|
|
1211
|
+
if (stagedWork.entries.size > 0) return yield* ThreadImportRejected.make({
|
|
1212
|
+
threadId: id,
|
|
1213
|
+
reason: "unsupported-obligations",
|
|
1214
|
+
message: "Transfer contains unresolved work"
|
|
1215
|
+
});
|
|
1216
|
+
const commitLedger = stagedLedger === void 0 ? void 0 : yield* stagedLedger.prepareCommit(progress.manifest?.workerSeal);
|
|
1217
|
+
const commitDelivery = stagedDelivery === void 0 ? void 0 : yield* stagedDelivery.prepareCommit();
|
|
1218
|
+
const installed = restoredThread;
|
|
1219
|
+
yield* Effect.uninterruptible(Effect.sync(() => {
|
|
1220
|
+
commitLedger?.();
|
|
1221
|
+
commitDelivery?.();
|
|
1222
|
+
publishWork(id, stagedWork);
|
|
1223
|
+
MutableRef.get(state.ref).threads.set(id, installed);
|
|
1224
|
+
}).pipe(Effect.andThen(PubSub.publish(updates, void 0))));
|
|
1225
|
+
return result;
|
|
1226
|
+
}).pipe(Effect.provide(ThreadImportReader.layer(reader)), Effect.provideService(ThreadDeliveryImportReader, {
|
|
1227
|
+
admission: (threadId, sid) => threadId === id ? reader.admission(sid) : ledgerTransfer?.admission(threadId, sid) ?? Effect.succeed(void 0),
|
|
1228
|
+
record: (threadId, recordId) => Effect.succeed(threadId === id ? restoredThread.byId.get(recordId)?.record : MutableRef.get(state.ref).threads.get(threadId)?.byId.get(recordId)?.record)
|
|
1229
|
+
}));
|
|
1230
|
+
}).pipe(Effect.provideService(Crypto.Crypto, crypto))));
|
|
1231
|
+
const inspectTail = Effect.fnUntraced(function* (unvalidated) {
|
|
323
1232
|
const request = yield* validate(ThreadTailRequest, "inspectTail", unvalidated);
|
|
324
1233
|
const thread = yield* Ref.get(state).pipe(Effect.flatMap((current) => findThread(current, request.threadId)));
|
|
325
1234
|
return ThreadTail.make({
|
|
326
1235
|
threadId: request.threadId,
|
|
327
|
-
tailSequence: thread.tailSequence
|
|
328
|
-
tailDigest: thread.tailDigest,
|
|
329
|
-
producerEpoch: thread.producerEpoch
|
|
1236
|
+
tailSequence: thread.tailSequence
|
|
330
1237
|
});
|
|
331
|
-
})
|
|
332
|
-
const readIdentity = Effect.
|
|
1238
|
+
});
|
|
1239
|
+
const readIdentity = Effect.fnUntraced(function* (unvalidated) {
|
|
333
1240
|
const request = yield* validate(ThreadIdentityRequest, "readIdentity", unvalidated);
|
|
334
1241
|
const thread = yield* Ref.get(state).pipe(Effect.flatMap((current) => findThread(current, request.threadId)));
|
|
335
1242
|
const selected = [
|
|
336
|
-
thread
|
|
1243
|
+
storedRecord(thread, 1),
|
|
337
1244
|
thread.byId.get(workerOriginRecordId(request.threadId)),
|
|
338
1245
|
thread.byId.get(subagentLineageRecordId(request.threadId))
|
|
339
1246
|
].filter((entry) => entry !== void 0);
|
|
340
1247
|
return yield* ThreadIdentity.makeEffect({
|
|
341
1248
|
threadId: request.threadId,
|
|
342
1249
|
tailSequence: thread.tailSequence,
|
|
343
|
-
|
|
344
|
-
producerEpoch: thread.producerEpoch,
|
|
1250
|
+
hasAdmissions: ledgerTransfer !== void 0 && (yield* ledgerTransfer.snapshot(request.threadId)).admissions > 0,
|
|
345
1251
|
records: selected.filter((entry, index) => selected.findIndex((other) => other.record.recordId === entry.record.recordId) === index)
|
|
346
1252
|
}).pipe(Effect.mapError((cause) => storeError("readIdentity", "Invalid canonical identity", cause)));
|
|
347
1253
|
});
|
|
348
|
-
const saveCheckpoint = Effect.
|
|
1254
|
+
const saveCheckpoint = Effect.fnUntraced(function* (unvalidated) {
|
|
349
1255
|
yield* validateCheckpointVersion(unvalidated);
|
|
350
1256
|
const request = yield* validate(SaveCheckpointRequest, "saveCheckpoint", unvalidated);
|
|
351
1257
|
const decision = yield* Ref.modify(state, (current) => {
|
|
@@ -362,116 +1268,258 @@ const makeThreadStore = Effect.gen(function* () {
|
|
|
362
1268
|
reason: "ahead-of-tail"
|
|
363
1269
|
})
|
|
364
1270
|
}, current];
|
|
365
|
-
if (thread
|
|
1271
|
+
if (!isBatchBoundary(thread, checkpoint.throughSequence)) return [{
|
|
366
1272
|
_tag: "failure",
|
|
367
1273
|
error: CheckpointRejected.make({
|
|
368
1274
|
threadId: checkpoint.threadId,
|
|
369
|
-
reason: "
|
|
1275
|
+
reason: "not-batch-boundary"
|
|
370
1276
|
})
|
|
371
1277
|
}, current];
|
|
372
|
-
|
|
373
|
-
_tag: "failure",
|
|
374
|
-
error: storeError("saveCheckpoint", `In-memory checkpoint limit ${MAX_CHECKPOINTS_PER_THREAD} exceeded`)
|
|
375
|
-
}, current];
|
|
376
|
-
const checkpoints = new Map(thread.checkpoints);
|
|
1278
|
+
const checkpoints = thread.checkpoints;
|
|
377
1279
|
checkpoints.set(checkpoint.throughSequence, checkpoint);
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
checkpoints
|
|
382
|
-
}
|
|
383
|
-
return [{ _tag: "success" },
|
|
1280
|
+
if (checkpoints.size > MAX_CHECKPOINTS_PER_THREAD) {
|
|
1281
|
+
let oldest = checkpoint.throughSequence;
|
|
1282
|
+
for (const sequence of checkpoints.keys()) if (sequence < oldest) oldest = sequence;
|
|
1283
|
+
checkpoints.delete(oldest);
|
|
1284
|
+
}
|
|
1285
|
+
return [{ _tag: "success" }, current];
|
|
384
1286
|
});
|
|
385
1287
|
if (decision._tag === "failure") return yield* decision.error;
|
|
386
|
-
})
|
|
387
|
-
const loadCheckpoint = Effect.
|
|
1288
|
+
});
|
|
1289
|
+
const loadCheckpoint = Effect.fnUntraced(function* (unvalidated) {
|
|
388
1290
|
const request = yield* validate(LoadCheckpointRequest, "loadCheckpoint", unvalidated);
|
|
389
1291
|
const thread = yield* Ref.get(state).pipe(Effect.flatMap((current) => findThread(current, request.threadId)));
|
|
390
1292
|
const maximum = request.atOrBeforeSequence ?? thread.tailSequence;
|
|
391
1293
|
let selected;
|
|
392
1294
|
for (const [sequence, checkpoint] of thread.checkpoints) if (sequence <= maximum && (selected === void 0 || sequence > selected.throughSequence)) selected = checkpoint;
|
|
393
|
-
if (selected !== void 0 && thread
|
|
1295
|
+
if (selected !== void 0 && !isBatchBoundary(thread, selected.throughSequence)) return yield* CheckpointRejected.make({
|
|
394
1296
|
threadId: request.threadId,
|
|
395
|
-
reason: "
|
|
1297
|
+
reason: "not-batch-boundary"
|
|
396
1298
|
});
|
|
397
1299
|
return Option.fromNullishOr(selected);
|
|
398
|
-
}));
|
|
399
|
-
const saveRecoveryCheckpoint = Effect.fn("MemoryThreadStore.saveRecoveryCheckpoint")(function* (unvalidated) {
|
|
400
|
-
const request = yield* validate(SaveRecoveryCheckpointRequest, "saveRecoveryCheckpoint", unvalidated);
|
|
401
|
-
const decision = yield* Ref.modify(state, (current) => {
|
|
402
|
-
const checkpoint = request.checkpoint;
|
|
403
|
-
const thread = current.threads.get(checkpoint.threadId);
|
|
404
|
-
if (thread === void 0) return [{
|
|
405
|
-
_tag: "failure",
|
|
406
|
-
error: ThreadNotMaterialized.make({ threadId: checkpoint.threadId })
|
|
407
|
-
}, current];
|
|
408
|
-
if (thread.producerEpoch !== request.producerEpoch) return [{
|
|
409
|
-
_tag: "failure",
|
|
410
|
-
error: FenceRejected.make({
|
|
411
|
-
threadId: checkpoint.threadId,
|
|
412
|
-
actualEpoch: thread.producerEpoch,
|
|
413
|
-
attemptedEpoch: request.producerEpoch
|
|
414
|
-
})
|
|
415
|
-
}, current];
|
|
416
|
-
if (checkpoint.throughSequence > thread.tailSequence) return [{
|
|
417
|
-
_tag: "failure",
|
|
418
|
-
error: CheckpointRejected.make({
|
|
419
|
-
threadId: checkpoint.threadId,
|
|
420
|
-
reason: "ahead-of-tail"
|
|
421
|
-
})
|
|
422
|
-
}, current];
|
|
423
|
-
if (thread.tailDigests.get(checkpoint.throughSequence) !== checkpoint.tailDigest) return [{
|
|
424
|
-
_tag: "failure",
|
|
425
|
-
error: CheckpointRejected.make({
|
|
426
|
-
threadId: checkpoint.threadId,
|
|
427
|
-
reason: "digest-mismatch"
|
|
428
|
-
})
|
|
429
|
-
}, current];
|
|
430
|
-
if ((thread.recoveryCheckpoint?.throughSequence ?? -1) > checkpoint.throughSequence) return [{ _tag: "success" }, current];
|
|
431
|
-
const threads = new Map(current.threads);
|
|
432
|
-
threads.set(checkpoint.threadId, {
|
|
433
|
-
...thread,
|
|
434
|
-
recoveryCheckpoint: checkpoint
|
|
435
|
-
});
|
|
436
|
-
return [{ _tag: "success" }, { threads }];
|
|
437
|
-
});
|
|
438
|
-
if (decision._tag === "failure") return yield* decision.error;
|
|
439
1300
|
});
|
|
440
|
-
const
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
1301
|
+
const verifyRange = Effect.fnUntraced(function* (thread, range, source) {
|
|
1302
|
+
let sequence = range.firstSequence;
|
|
1303
|
+
let bytes = 0;
|
|
1304
|
+
let batches = 0;
|
|
1305
|
+
const captured = /* @__PURE__ */ new Map();
|
|
1306
|
+
while (sequence <= range.lastSequence) {
|
|
1307
|
+
const id = thread.batchesByFirst.get(sequence);
|
|
1308
|
+
const batch = id === void 0 ? void 0 : thread.batches.get(id);
|
|
1309
|
+
const text = id === void 0 ? void 0 : source === void 0 ? batch?.batchJson : source.get(id);
|
|
1310
|
+
if (id === void 0 || batch === void 0 || text === void 0 || batch.result.lastSequence > range.lastSequence) return yield* storeError("verify range", "Incomplete canonical batch range");
|
|
1311
|
+
bytes += new TextEncoder().encode(text).byteLength;
|
|
1312
|
+
batches++;
|
|
1313
|
+
if (bytes > MAX_ARCHIVE_RANGE_BYTES || batches > MAX_ARCHIVE_RANGE_RECORDS || text !== batch.batchJson) return yield* storeError("verify range", "Range wire or bounds differ from canonical storage");
|
|
1314
|
+
const wire = yield* Schema.decodeEffect(Schema.fromJsonString(ExportBatch))(text).pipe(Effect.mapError((cause) => storeError("verify range", "Invalid range wire", cause)));
|
|
1315
|
+
if (wire.batchId !== id || batch.result.firstSequence !== sequence || batch.result.lastSequence !== sequence + wire.records.length - 1) return yield* storeError("verify range", "Canonical batch span or identity differs");
|
|
1316
|
+
for (const [index, record] of wire.records.entries()) {
|
|
1317
|
+
const expected = storedRecord(thread, sequence + index);
|
|
1318
|
+
if (expected === void 0 || expected.threadId !== range.threadId || expected.sequence !== sequence + index || expected.batchId !== id || expected.record.recordId !== record.recordId) return yield* storeError("verify range", "Canonical record identity or order differs");
|
|
1319
|
+
const canonical = yield* Schema.encodeEffect(ReadRecord)(expected.record).pipe(Effect.mapError((cause) => storeError("verify range", "Invalid canonical record", cause)));
|
|
1320
|
+
const archived = yield* Schema.encodeEffect(ExportRecord)(record).pipe(Effect.mapError((cause) => storeError("verify range", "Invalid archive record", cause)));
|
|
1321
|
+
if (canonicalJson(canonical) !== canonicalJson(archived)) return yield* storeError("verify range", "Archive and canonical record bytes differ");
|
|
1322
|
+
}
|
|
1323
|
+
captured.set(id, text);
|
|
1324
|
+
sequence = decodeCanonicalSequence(batch.result.lastSequence + 1);
|
|
1325
|
+
}
|
|
1326
|
+
if (bytes !== range.byteCount || batches !== range.batchCount || range.recordCount !== range.lastSequence - range.firstSequence + 1) return yield* storeError("verify range", "Range spans or counts differ");
|
|
1327
|
+
return captured;
|
|
1328
|
+
});
|
|
1329
|
+
const archiveTarget = Effect.fnUntraced(function* (request) {
|
|
1330
|
+
return yield* findThread(MutableRef.get(state.ref), request.threadId);
|
|
1331
|
+
});
|
|
1332
|
+
const archives = {
|
|
1333
|
+
page: Effect.fnUntraced(function* (unvalidated) {
|
|
1334
|
+
const request = yield* validate(ThreadArchiveRangeRead, "range page", unvalidated);
|
|
1335
|
+
const thread = yield* findThread(MutableRef.get(state.ref), request.threadId);
|
|
1336
|
+
let low = 0;
|
|
1337
|
+
let high = thread.rangeStarts.length;
|
|
1338
|
+
while (low < high) {
|
|
1339
|
+
const middle = Math.floor((low + high) / 2);
|
|
1340
|
+
const range = thread.ranges.get(thread.rangeStarts[middle]);
|
|
1341
|
+
if (range === void 0) return yield* storeError("page ranges", "Range locator has no exact range");
|
|
1342
|
+
if (range.lastSequence <= (request.afterSequence ?? 0)) low = middle + 1;
|
|
1343
|
+
else high = middle;
|
|
1344
|
+
}
|
|
1345
|
+
const ranges = [];
|
|
1346
|
+
const limit = request.limit ?? 32;
|
|
1347
|
+
for (let at = low; at < Math.min(thread.rangeStarts.length, low + limit); at++) {
|
|
1348
|
+
const range = thread.ranges.get(thread.rangeStarts[at]);
|
|
1349
|
+
if (range === void 0) return yield* storeError("page ranges", "Range locator has no exact range");
|
|
1350
|
+
ranges.push(range);
|
|
1351
|
+
}
|
|
1352
|
+
const afterSequence = low + ranges.length < thread.rangeStarts.length ? ranges.at(-1)?.lastSequence : void 0;
|
|
1353
|
+
return {
|
|
1354
|
+
ranges,
|
|
1355
|
+
...afterSequence === void 0 ? {} : { afterSequence }
|
|
1356
|
+
};
|
|
1357
|
+
}),
|
|
1358
|
+
seal: (unvalidated) => withMutation(Effect.gen(function* () {
|
|
1359
|
+
const request = yield* validate(ThreadArchiveRangeSeal, "seal range", unvalidated);
|
|
1360
|
+
const thread = yield* archiveTarget(request);
|
|
1361
|
+
const previousStart = thread.rangeStarts.at(-1);
|
|
1362
|
+
const previous = previousStart === void 0 ? void 0 : thread.ranges.get(previousStart);
|
|
1363
|
+
if (previousStart !== void 0 && previous === void 0) return yield* storeError("seal range", "Range locator has no exact range");
|
|
1364
|
+
const firstSequence = (previous?.lastSequence ?? 0) + 1;
|
|
1365
|
+
if (firstSequence > thread.tailSequence) return Option.none();
|
|
1366
|
+
let nextSequence = firstSequence;
|
|
1367
|
+
let byteCount = 0;
|
|
1368
|
+
let batchCount = 0;
|
|
1369
|
+
while (nextSequence <= thread.tailSequence) {
|
|
1370
|
+
const batchId = thread.batchesByFirst.get(nextSequence);
|
|
1371
|
+
const batch = batchId === void 0 ? void 0 : thread.batches.get(batchId);
|
|
1372
|
+
if (batch === void 0) return yield* storeError("seal range", "Missing exact canonical batch");
|
|
1373
|
+
const bytes = utf8ByteLength(batch.batchJson);
|
|
1374
|
+
if (batchCount > 0 && (batch.result.lastSequence - firstSequence + 1 > MAX_ARCHIVE_RANGE_RECORDS || byteCount + bytes > MAX_ARCHIVE_RANGE_BYTES)) break;
|
|
1375
|
+
byteCount += bytes;
|
|
1376
|
+
batchCount++;
|
|
1377
|
+
nextSequence = batch.result.lastSequence + 1;
|
|
1378
|
+
}
|
|
1379
|
+
const range = ThreadArchiveRange.make({
|
|
1380
|
+
format: "effect-agent/thread-range@2",
|
|
1381
|
+
threadId: request.threadId,
|
|
1382
|
+
firstSequence: decodeCanonicalSequence(firstSequence),
|
|
1383
|
+
lastSequence: decodeCanonicalSequence(nextSequence - 1),
|
|
1384
|
+
recordCount: nextSequence - firstSequence,
|
|
1385
|
+
batchCount,
|
|
1386
|
+
byteCount,
|
|
1387
|
+
state: "sealed"
|
|
1388
|
+
});
|
|
1389
|
+
yield* verifyRange(thread, range);
|
|
1390
|
+
thread.ranges.set(firstSequence, range);
|
|
1391
|
+
thread.rangeStarts.push(firstSequence);
|
|
1392
|
+
return Option.some(range);
|
|
1393
|
+
})),
|
|
1394
|
+
verify: Effect.fnUntraced(function* (unvalidated) {
|
|
1395
|
+
const request = yield* validate(ThreadArchiveRangeRequest, "verify range", unvalidated);
|
|
1396
|
+
const thread = yield* findThread(MutableRef.get(state.ref), request.threadId);
|
|
1397
|
+
const range = thread.ranges.get(request.firstSequence);
|
|
1398
|
+
if (range === void 0) return yield* storeError("verify range", "Unknown sealed range");
|
|
1399
|
+
yield* verifyRange(thread, range, range.state === "archived" ? thread.archiveContents.get(range.firstSequence) : void 0);
|
|
1400
|
+
if (range.state === "archived" && !thread.archiveContents.has(range.firstSequence)) return yield* storeError("verify range", "Published archive contents are missing");
|
|
1401
|
+
return range;
|
|
1402
|
+
}),
|
|
1403
|
+
archive: (unvalidated) => withMutation(Effect.gen(function* () {
|
|
1404
|
+
const request = yield* validate(ThreadArchiveRangePublish, "archive range", unvalidated);
|
|
1405
|
+
const thread = yield* archiveTarget(request);
|
|
1406
|
+
const range = thread.ranges.get(request.firstSequence);
|
|
1407
|
+
if (range === void 0) return yield* storeError("archive range", "Unknown sealed range");
|
|
1408
|
+
if (range.state === "archived") {
|
|
1409
|
+
const contents = thread.archiveContents.get(range.firstSequence);
|
|
1410
|
+
if (contents === void 0) return yield* storeError("archive range", "Published archive contents are missing");
|
|
1411
|
+
yield* verifyRange(thread, range, contents);
|
|
1412
|
+
return range;
|
|
1413
|
+
}
|
|
1414
|
+
const contents = yield* verifyRange(thread, range);
|
|
1415
|
+
yield* verifyRange(thread, range, contents);
|
|
1416
|
+
const published = ThreadArchiveRange.make({
|
|
1417
|
+
...range,
|
|
1418
|
+
state: "archived",
|
|
1419
|
+
locator: `memory:${encodeURIComponent(request.threadId)}:${range.firstSequence}`
|
|
1420
|
+
});
|
|
1421
|
+
thread.archiveContents.set(range.firstSequence, contents);
|
|
1422
|
+
thread.ranges.set(range.firstSequence, published);
|
|
1423
|
+
return published;
|
|
1424
|
+
}))
|
|
1425
|
+
};
|
|
1426
|
+
const readWorkerCapacity = Effect.fnUntraced(function* (unvalidated) {
|
|
1427
|
+
const request = yield* validate(ThreadWorkerCapacityRequest, "worker capacity", unvalidated);
|
|
1428
|
+
const thread = yield* findThread(MutableRef.get(state.ref), request.threadId);
|
|
1429
|
+
if (request.expectedTailSequence !== thread.tailSequence) return yield* storeError("worker capacity", "Canonical tail changed");
|
|
1430
|
+
const index = yield* readyIndex(request.threadId, thread.tailSequence).pipe(Effect.mapError((e) => storeError("worker capacity", e.message)));
|
|
1431
|
+
const worker = index.workers.get(request.workerThreadId);
|
|
1432
|
+
return ThreadWorkerCapacity.make({
|
|
446
1433
|
threadId: request.threadId,
|
|
447
|
-
|
|
1434
|
+
tailSequence: thread.tailSequence,
|
|
1435
|
+
activeWorkers: Math.min(index.workers.size, request.activeLimit + 1),
|
|
1436
|
+
workerActive: worker !== void 0,
|
|
1437
|
+
pendingInputs: Math.min(request.update ? worker?.update ?? 0 : worker?.ordinary ?? 0, request.pendingLimit + 1)
|
|
448
1438
|
});
|
|
449
|
-
return Option.some(checkpoint);
|
|
450
1439
|
});
|
|
451
|
-
|
|
1440
|
+
const threadStore = ThreadStore.of({
|
|
1441
|
+
work,
|
|
1442
|
+
archives,
|
|
1443
|
+
verification: { verify: (unvalidated) => withMutation(Effect.gen(function* () {
|
|
1444
|
+
const request = yield* validate(ThreadVerificationRequest, "verify Thread", unvalidated);
|
|
1445
|
+
const thread = yield* findThread(MutableRef.get(state.ref), request.threadId);
|
|
1446
|
+
const id = request.threadId;
|
|
1447
|
+
const reader = importReader(id, thread);
|
|
1448
|
+
const submissions = () => ledgerTransfer?.submissions(id) ?? Stream.empty;
|
|
1449
|
+
const runs = Stream.fromIterable(thread.runRecords.keys()).pipe(Stream.map((runId) => Schema.decodeSync(RunId)(runId)));
|
|
1450
|
+
const checkpoint = yield* loadCheckpoint(LoadCheckpointRequest.make({ threadId: id })).pipe(Effect.mapError((cause) => Schema.is(CheckpointRejected)(cause) ? storeError("verify checkpoint", "Invalid checkpoint anchor", cause) : cause));
|
|
1451
|
+
return yield* verifyThreadInvariants({
|
|
1452
|
+
threadId: id,
|
|
1453
|
+
runs,
|
|
1454
|
+
executions: Stream.fromIterable(thread.executions.values()),
|
|
1455
|
+
submissions: submissions(),
|
|
1456
|
+
pages: streamExport({ threadId: id }).pipe(Stream.provideService(ThreadExportSource, { export: verificationPage }), Stream.mapError((cause) => Schema.is(ThreadExportRejected)(cause) ? storeError("verify Thread", "Internal verification snapshot changed") : cause)),
|
|
1457
|
+
...Option.isNone(checkpoint) ? {} : { checkpoint: checkpoint.value },
|
|
1458
|
+
checkpointsSupported: true,
|
|
1459
|
+
...request.requireAllSettled === void 0 ? {} : { requireAllSettled: request.requireAllSettled }
|
|
1460
|
+
}).pipe(Effect.provide(ThreadImportReader.layer(reader)), Effect.provideService(Crypto.Crypto, crypto));
|
|
1461
|
+
})) },
|
|
1462
|
+
readWorkerCapacity,
|
|
452
1463
|
readIdentity,
|
|
453
1464
|
countPeerMessages,
|
|
454
|
-
materialize,
|
|
1465
|
+
materialize: (request) => withMutation(materialize(request)),
|
|
455
1466
|
append,
|
|
1467
|
+
readExecution: (request) => withMutation(Effect.gen(function* () {
|
|
1468
|
+
const thread = (yield* Ref.get(state)).threads.get(request.threadId);
|
|
1469
|
+
if (thread === void 0) return yield* ThreadNotMaterialized.make({ threadId: request.threadId });
|
|
1470
|
+
return Option.fromUndefinedOr(thread.executions.get(request.runId));
|
|
1471
|
+
})),
|
|
456
1472
|
read,
|
|
1473
|
+
readPrompt,
|
|
457
1474
|
observe,
|
|
458
|
-
export: exportThread,
|
|
1475
|
+
export: (request) => checkExportSource(request.threadId).pipe(Effect.andThen(withMutation(exportThread(request)))),
|
|
459
1476
|
inspectTail,
|
|
460
1477
|
checkpoints: {
|
|
461
|
-
save: saveCheckpoint,
|
|
1478
|
+
save: (request) => withMutation(saveCheckpoint(request)),
|
|
462
1479
|
load: loadCheckpoint
|
|
463
|
-
},
|
|
464
|
-
recoveryCheckpoints: {
|
|
465
|
-
save: saveRecoveryCheckpoint,
|
|
466
|
-
load: loadRecoveryCheckpoint
|
|
467
1480
|
}
|
|
468
1481
|
});
|
|
1482
|
+
return Context.make(ThreadStore, threadStore).pipe(Context.add(StorageOwner, owner), Context.add(ThreadImport, { import: importThread }), Context.add(MemoryThreadStoreKernel, {
|
|
1483
|
+
owner,
|
|
1484
|
+
withMutation,
|
|
1485
|
+
retainDelivery,
|
|
1486
|
+
checkThreadCapacity: (threadId) => Effect.suspend(() => !MutableRef.get(workState.ref).has(threadId) && MutableRef.get(workState.ref).size >= maxThreads ? Effect.fail(storeError("Thread capacity", "Retained Thread quota exceeded")) : Effect.void),
|
|
1487
|
+
initializeWork: (threadId) => {
|
|
1488
|
+
if (!MutableRef.get(state.ref).threads.has(threadId) && !MutableRef.get(workState.ref).has(threadId)) publishWork(threadId, emptyWorkIndex());
|
|
1489
|
+
},
|
|
1490
|
+
registerWorkOwner: (kind, owner) => withMutation(Effect.sync(() => {
|
|
1491
|
+
if (workOwners.has(kind)) throw new Error(`MemoryThreadStore already has a ${kind} work owner`);
|
|
1492
|
+
workOwners.set(kind, owner);
|
|
1493
|
+
})),
|
|
1494
|
+
registerDeliveryLookup: (lookup) => withMutation(Effect.sync(() => {
|
|
1495
|
+
if (deliveryLookup !== void 0) throw new Error("MemoryThreadStore already has a delivery lookup");
|
|
1496
|
+
deliveryLookup = lookup;
|
|
1497
|
+
})),
|
|
1498
|
+
registerLedgerTransfer: (transfer) => withMutation(Effect.sync(() => {
|
|
1499
|
+
if (ledgerTransfer !== void 0) throw new Error("MemoryThreadStore already has a paired SubmissionLedger");
|
|
1500
|
+
ledgerTransfer = transfer;
|
|
1501
|
+
})),
|
|
1502
|
+
registerDeliveryTransfer: (transfer) => withMutation(Effect.sync(() => {
|
|
1503
|
+
if (deliveryTransfer !== void 0) throw new Error("MemoryThreadStore already has a paired delivery store");
|
|
1504
|
+
deliveryTransfer = transfer;
|
|
1505
|
+
})),
|
|
1506
|
+
prepareAppend,
|
|
1507
|
+
appendPrepared,
|
|
1508
|
+
record: (threadId, recordId) => Ref.get(state).pipe(Effect.map((current) => current.threads.get(threadId)?.byId.get(recordId)?.record)),
|
|
1509
|
+
toolCallResults: (threadId, runId, toolCallId) => Ref.get(state).pipe(Effect.map((current) => (current.threads.get(threadId)?.operationRecords.get(JSON.stringify([
|
|
1510
|
+
runId,
|
|
1511
|
+
toolCallId,
|
|
1512
|
+
"settled"
|
|
1513
|
+
])) ?? []).slice(0, 2).map((entry) => entry.record))),
|
|
1514
|
+
tail: (threadId) => inspectTail(ThreadTailRequest.make({ threadId }))
|
|
1515
|
+
}));
|
|
469
1516
|
});
|
|
470
1517
|
/**
|
|
471
1518
|
* In-memory canonical Thread persistence. Durable accepted work is served by the separate
|
|
472
|
-
* SubmissionLedger port; this Layer
|
|
1519
|
+
* SubmissionLedger port; this Layer provides ThreadStore, ThreadReader, and ThreadImport.
|
|
1520
|
+
* Importing admissions requires a MemorySubmissionLedger paired with this store.
|
|
473
1521
|
*/
|
|
474
|
-
const MemoryThreadStoreLive = Layer.
|
|
1522
|
+
const MemoryThreadStoreLive = ThreadReader.layer().pipe(Layer.provideMerge(Layer.effectContext(makeThreadStore)));
|
|
475
1523
|
/** Configure a finite retained Thread capacity. Invalid construction options throw immediately. */
|
|
476
1524
|
const memoryThreadStoreLayer = (options = {}) => MemoryThreadStoreLive.pipe(Layer.provide(Layer.succeed(ThreadCapacity)(Schema.decodeSync(Schema.Int.check(Schema.isGreaterThan(0), Schema.isLessThanOrEqualTo(65536)))(options.maxThreads ?? MAX_THREADS))));
|
|
477
1525
|
//#endregion
|