effect-agent 0.1.0-beta.155 → 0.1.0-beta.157
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{Admin-BjmupLN3.d.mts → Admin-BbTZHSLu.d.mts} +23 -11
- package/dist/{Agent-1DMSO_Fl.mjs → Agent-BMATXYsI.mjs} +9 -2
- package/dist/Agent-BMATXYsI.mjs.map +1 -0
- package/dist/{AgentRuntime-BAjT5gXl.d.mts → AgentRuntime-BPlAn5nA.d.mts} +6 -6
- package/dist/{AgentRuntime-IF_1GoUf.mjs → AgentRuntime-GQACxY_m.mjs} +21 -5
- package/dist/AgentRuntime-GQACxY_m.mjs.map +1 -0
- package/dist/{Digest-CaJ2RIsF.d.mts → Digest-CNHImGy0.d.mts} +2 -2
- package/dist/{DurableAgentRuntime-BQzYkFVy.mjs → DurableAgentRuntime-BAQvZK1e.mjs} +30 -18
- package/dist/DurableAgentRuntime-BAQvZK1e.mjs.map +1 -0
- package/dist/{EventSource-C8CKwQy5.d.mts → EventSource-CETtQTiH.d.mts} +3 -3
- package/dist/InMemory.d.mts +1 -1
- package/dist/{PreparedInputAdmission-CU1PzUVD.d.mts → PreparedInputAdmission-Swp9t800.d.mts} +4 -4
- package/dist/{Records-BrEpibum.d.mts → Records-BxGWRs-t.d.mts} +2 -2
- package/dist/{RunEvent-DV_llBcq.d.mts → RunEvent-D0aepYwN.d.mts} +2 -2
- package/dist/{RunEventSink-D5aVhISG.d.mts → RunEventSink-DsbcGXog.d.mts} +2 -2
- package/dist/{RunOptions-CkRYnlJa.d.mts → RunOptions-GjDlAuKh.d.mts} +5 -2
- package/dist/{Schedule-B_z7PMg-.d.mts → Schedule-B94iS2X3.d.mts} +2 -2
- package/dist/{Subagent-Dld4W1BY.d.mts → Subagent-CLaMeZKu.d.mts} +6 -6
- package/dist/{Subagent-CmFtZaeI.mjs → Subagent-JjcidJ7q.mjs} +2 -2
- package/dist/{Subagent-CmFtZaeI.mjs.map → Subagent-JjcidJ7q.mjs.map} +1 -1
- package/dist/{Subscription-BomcQ9_W.d.mts → Subscription-DaMCppa_.d.mts} +4 -4
- package/dist/{SubscriptionInput-gpZSR9vB.d.mts → SubscriptionInput-B0ZtrQoY.d.mts} +3 -3
- package/dist/{ThreadHistory-ZEu_ja6L.d.mts → ThreadHistory-FD-o4TsR.d.mts} +2 -2
- package/dist/{Usage-Dx0aknfk.d.mts → Usage-BNjPR7Zr.d.mts} +9 -1
- package/dist/{WorkerAdmission-CNbm4o_x.mjs → WorkerAdmission-N9hZV85Q.mjs} +2 -2
- package/dist/{WorkerAdmission-CNbm4o_x.mjs.map → WorkerAdmission-N9hZV85Q.mjs.map} +1 -1
- package/dist/{agent-registration-U3kmV7nW.mjs → agent-registration-BIOzAej4.mjs} +10 -1
- package/dist/agent-registration-BIOzAej4.mjs.map +1 -0
- package/dist/capabilities/Commands.d.mts +1 -1
- package/dist/capabilities/Redaction.d.mts +1 -1
- package/dist/capabilities/RunHooks.d.mts +1 -1
- package/dist/capabilities/Subagent.d.mts +1 -1
- package/dist/capabilities/Subagent.mjs +1 -1
- package/dist/capabilities/ToolDiscovery.d.mts +5 -0
- package/dist/capabilities/ToolDiscovery.mjs +18 -9
- package/dist/capabilities/ToolDiscovery.mjs.map +1 -1
- package/dist/capabilities/WebSearch.d.mts +5 -1
- package/dist/capabilities/WebSearch.mjs +9 -1
- package/dist/capabilities/WebSearch.mjs.map +1 -1
- package/dist/core/Agent.mjs +1 -1
- package/dist/core/RunEvent.d.mts +1 -1
- package/dist/core/Usage.d.mts +1 -1
- package/dist/core/Usage.mjs +22 -1
- package/dist/core/Usage.mjs.map +1 -1
- package/dist/durable/ActivityStore.d.mts +1 -1
- package/dist/durable/Admin.d.mts +1 -1
- package/dist/durable/AgentRegistration.d.mts +1 -1
- package/dist/durable/AgentRegistration.mjs +1 -1
- package/dist/durable/Certification.d.mts +1 -1
- package/dist/durable/CommittedActivity.d.mts +3 -3
- package/dist/durable/Digest.d.mts +1 -1
- package/dist/durable/DurableAgentRuntime.d.mts +1 -1
- package/dist/durable/DurableAgentRuntime.mjs +2 -2
- package/dist/durable/EventSource.d.mts +1 -1
- package/dist/durable/GitHubWorkflowSource.d.mts +3 -3
- package/dist/durable/LifecyclePublication.d.mts +1 -1
- package/dist/durable/LifecyclePublication.mjs +2 -2
- package/dist/durable/LifecyclePublication.mjs.map +1 -1
- package/dist/durable/MessageDelivery.d.mts +4 -4
- package/dist/durable/MessageDeliveryStoreConformance.d.mts +3 -3
- package/dist/durable/PersistentHistory.d.mts +2 -2
- package/dist/durable/PreparedInputAdmission.d.mts +1 -1
- package/dist/durable/Records.d.mts +1 -1
- package/dist/durable/Recovery.d.mts +1 -1
- package/dist/durable/RunJournal.d.mts +1 -1
- package/dist/durable/Schedule.d.mts +1 -1
- package/dist/durable/ScheduleStoreConformance.d.mts +1 -1
- package/dist/durable/ScheduleTransition.d.mts +1 -1
- package/dist/durable/Scheduling.d.mts +3 -3
- package/dist/durable/SqlMemoryStore.d.mts +715 -5
- package/dist/durable/SqlMemoryStore.mjs +313 -123
- package/dist/durable/SqlMemoryStore.mjs.map +1 -1
- package/dist/durable/SubmissionLedger.d.mts +1 -1
- package/dist/durable/SubmissionLedgerConformance.d.mts +2 -2
- package/dist/durable/SubmissionStatus.d.mts +1 -1
- package/dist/durable/Subscription.d.mts +1 -1
- package/dist/durable/SubscriptionInput.d.mts +1 -1
- package/dist/durable/SubscriptionInput.mjs +1 -1
- package/dist/durable/SubscriptionStoreConformance.d.mts +1 -1
- package/dist/durable/SubscriptionTools.d.mts +4 -4
- package/dist/durable/SubscriptionTransition.d.mts +1 -1
- package/dist/durable/Subscriptions.d.mts +6 -6
- package/dist/durable/ThreadContextHistory.d.mts +1 -1
- package/dist/durable/ThreadContextHistoryProjection.d.mts +1 -1
- package/dist/durable/ThreadInvariants.d.mts +2 -2
- package/dist/durable/ThreadProjection.d.mts +1 -1
- package/dist/durable/ThreadProjectionMaintenance.d.mts +1 -1
- package/dist/durable/ThreadStore.d.mts +1 -1
- package/dist/durable/ThreadStoreConformance.d.mts +1 -1
- package/dist/durable/WorkerAdmission.d.mts +2 -2
- package/dist/durable/WorkerAdmission.mjs +1 -1
- package/dist/durable/WorkerHost.d.mts +2 -2
- package/dist/engine/AgentRuntime.d.mts +1 -1
- package/dist/engine/AgentRuntime.mjs +1 -1
- package/dist/engine/RunEventSink.d.mts +1 -1
- package/dist/engine/RunOptions.d.mts +1 -1
- package/dist/engine/RunOptions.mjs +1 -0
- package/dist/engine/RunOptions.mjs.map +1 -1
- package/dist/engine/ThreadHistory.d.mts +1 -1
- package/dist/index.d.mts +15 -15
- package/dist/index.mjs +5 -5
- package/package.json +1 -1
- package/src/capabilities/ToolDiscovery.ts +24 -10
- package/src/capabilities/WebSearch.ts +15 -0
- package/src/core/Agent.ts +16 -2
- package/src/core/Usage.ts +41 -0
- package/src/durable/DurableAgentRuntime.ts +40 -12
- package/src/durable/LifecyclePublication.ts +15 -2
- package/src/durable/SqlMemoryStore.ts +462 -188
- package/src/durable/internal/agent-registration.ts +11 -0
- package/src/engine/RunOptions.ts +3 -0
- package/src/engine/internal/agent-runtime.ts +68 -5
- package/dist/Agent-1DMSO_Fl.mjs.map +0 -1
- package/dist/AgentRuntime-IF_1GoUf.mjs.map +0 -1
- package/dist/DurableAgentRuntime-BQzYkFVy.mjs.map +0 -1
- package/dist/agent-registration-U3kmV7nW.mjs.map +0 -1
|
@@ -18,6 +18,16 @@ const EncodedMemoryChange = Schema.Struct({
|
|
|
18
18
|
});
|
|
19
19
|
const equivalentContent = Schema.toEquivalence(Schema.toEncoded(MemoryWrite.Wire.members[0].fields.content));
|
|
20
20
|
const equivalentScopes = Schema.toEquivalence(MemoryWrite.Wire.members[0].fields.scopes);
|
|
21
|
+
/** Ordered commands committed atomically by the SQL memory adapter. */
|
|
22
|
+
const SqlMemoryWriteBatch = Schema.Array(MemoryWrite.Wire).check(Schema.isMaxLength(128));
|
|
23
|
+
/**
|
|
24
|
+
* SQL-only bulk writing. Results preserve input order, including exact operation replays.
|
|
25
|
+
* Commands see earlier staged revisions; any refusal rolls back the entire batch. Storage
|
|
26
|
+
* limits apply to every intermediate transition. Empty batches succeed without mutation.
|
|
27
|
+
* Supplied by the memory store Layers alongside MemoryWriter's single-command API.
|
|
28
|
+
*/
|
|
29
|
+
var SqlMemoryBatchWriter = class extends Context.Service()("@effect-agent/thread/SqlMemoryBatchWriter") {};
|
|
30
|
+
const SqlStorageOwner = Context.Reference("@effect-agent/thread/SqlStorageOwner", { defaultValue: () => void 0 });
|
|
21
31
|
/**
|
|
22
32
|
* Independent adapter limits. Counts and encoded bytes include durable operation receipts.
|
|
23
33
|
* Bytes count UTF-8 row payloads plus 128 bytes per row, excluding SQLite pages/indexes.
|
|
@@ -45,6 +55,61 @@ const UsageRow = Schema.Struct({
|
|
|
45
55
|
receipts: Schema.Natural,
|
|
46
56
|
bytes: Schema.Natural
|
|
47
57
|
});
|
|
58
|
+
var MemoryViews = class {
|
|
59
|
+
entries = /* @__PURE__ */ new Map();
|
|
60
|
+
bytes = 0;
|
|
61
|
+
clear = () => {
|
|
62
|
+
this.entries.clear();
|
|
63
|
+
this.bytes = 0;
|
|
64
|
+
};
|
|
65
|
+
get(key) {
|
|
66
|
+
const entry = this.entries.get(key);
|
|
67
|
+
if (entry === void 0) return void 0;
|
|
68
|
+
this.entries.delete(key);
|
|
69
|
+
this.entries.set(key, entry);
|
|
70
|
+
return entry.view;
|
|
71
|
+
}
|
|
72
|
+
set(key, view) {
|
|
73
|
+
const previous = this.entries.get(key);
|
|
74
|
+
if (previous !== void 0) this.bytes -= previous.bytes;
|
|
75
|
+
this.entries.delete(key);
|
|
76
|
+
const bytes = utf8ByteLength(key) + utf8ByteLength(JSON.stringify(view)) + 128;
|
|
77
|
+
if (bytes > 4194304) return;
|
|
78
|
+
this.entries.set(key, {
|
|
79
|
+
view,
|
|
80
|
+
bytes
|
|
81
|
+
});
|
|
82
|
+
this.bytes += bytes;
|
|
83
|
+
while (this.entries.size > 128 || this.bytes > 4194304) {
|
|
84
|
+
const oldest = this.entries.entries().next().value;
|
|
85
|
+
if (oldest === void 0) break;
|
|
86
|
+
this.bytes -= oldest[1].bytes;
|
|
87
|
+
this.entries.delete(oldest[0]);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
};
|
|
91
|
+
const ownedViews = /* @__PURE__ */ new WeakMap();
|
|
92
|
+
const memoryViews = Effect.fn("SqliteMemoryStore.memoryViews")(function* () {
|
|
93
|
+
const owner = yield* SqlStorageOwner;
|
|
94
|
+
if (owner === void 0) return void 0;
|
|
95
|
+
let views = ownedViews.get(owner);
|
|
96
|
+
if (views === void 0) {
|
|
97
|
+
views = new MemoryViews();
|
|
98
|
+
ownedViews.set(owner, views);
|
|
99
|
+
owner.invalidators.add(views.clear);
|
|
100
|
+
}
|
|
101
|
+
return views;
|
|
102
|
+
});
|
|
103
|
+
const documentViewKey = (key) => JSON.stringify([
|
|
104
|
+
"document",
|
|
105
|
+
key.namespace.address,
|
|
106
|
+
key.id
|
|
107
|
+
]);
|
|
108
|
+
const receiptViewKey = (write) => JSON.stringify([
|
|
109
|
+
"receipt",
|
|
110
|
+
write.key.namespace.address,
|
|
111
|
+
write.operationId
|
|
112
|
+
]);
|
|
48
113
|
const documentBytesSql = (row) => `length(CAST(${row}.namespace AS BLOB)) + length(CAST(${row}.source_id AS BLOB)) + length(CAST(${row}.document_json AS BLOB)) + 128`;
|
|
49
114
|
const receiptBytesSql = (row) => `length(CAST(${row}.namespace AS BLOB)) + length(CAST(${row}.source_id AS BLOB)) + length(CAST(${row}.operation_id AS BLOB)) + length(CAST(${row}.command_json AS BLOB)) + length(CAST(${row}.result_json AS BLOB)) + 128`;
|
|
50
115
|
const usageTriggers = [{
|
|
@@ -137,11 +202,16 @@ const validateReceiptResult = Effect.fn("SqliteMemoryStore.validateReceiptResult
|
|
|
137
202
|
});
|
|
138
203
|
const readUsage = Effect.fn("SqliteMemoryStore.readUsage")(function* () {
|
|
139
204
|
const sql = yield* SqlClientService.SqlClient;
|
|
205
|
+
const views = yield* memoryViews();
|
|
140
206
|
const operation = "read memory usage";
|
|
141
207
|
const rows = yield* query(sql`
|
|
142
208
|
SELECT documents, receipts, bytes FROM effect_agent_memory_usage_v1 WHERE singleton = 1
|
|
143
209
|
`, operation).pipe(Effect.flatMap((rows) => decodeRows(UsageRow, rows, operation)));
|
|
144
210
|
if (rows.length !== 1) return yield* storageError(operation, "corrupt");
|
|
211
|
+
views?.set("usage", {
|
|
212
|
+
_tag: "Usage",
|
|
213
|
+
value: rows[0]
|
|
214
|
+
});
|
|
145
215
|
return rows[0];
|
|
146
216
|
});
|
|
147
217
|
const initializeMemoryUsage = Effect.fn("SqliteMemoryStore.initializeUsage")(function* () {
|
|
@@ -190,9 +260,11 @@ const initializeMemoryUsage = Effect.fn("SqliteMemoryStore.initializeUsage")(fun
|
|
|
190
260
|
});
|
|
191
261
|
const initializeMemorySchema = Effect.fn("SqliteMemoryStore.initialize")(function* () {
|
|
192
262
|
const sql = yield* SqlClientService.SqlClient;
|
|
263
|
+
const owner = yield* SqlStorageOwner;
|
|
264
|
+
const transaction = owner === void 0 ? sql.withTransaction : owner.transaction;
|
|
193
265
|
const failpoint = yield* MemoryMutationFailpoint;
|
|
194
266
|
yield* failpoint.hit("memory:initialize:before");
|
|
195
|
-
yield*
|
|
267
|
+
yield* transaction(Effect.gen(function* () {
|
|
196
268
|
yield* sql`
|
|
197
269
|
CREATE TABLE IF NOT EXISTS effect_agent_memory_metadata (
|
|
198
270
|
component TEXT PRIMARY KEY NOT NULL,
|
|
@@ -258,155 +330,273 @@ const initializeMemorySchema = Effect.fn("SqliteMemoryStore.initialize")(functio
|
|
|
258
330
|
});
|
|
259
331
|
const makeMemoryReader = Effect.fn("SqliteMemoryStore.makeReader")(function* () {
|
|
260
332
|
const sql = yield* SqlClientService.SqlClient;
|
|
261
|
-
const
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
const
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
333
|
+
const owner = yield* SqlStorageOwner;
|
|
334
|
+
const views = yield* memoryViews();
|
|
335
|
+
const readDocuments = Effect.fn("SqliteMemoryStore.readDocuments")(function* (keys, operation) {
|
|
336
|
+
const found = /* @__PURE__ */ new Map();
|
|
337
|
+
const missing = /* @__PURE__ */ new Map();
|
|
338
|
+
for (const key of keys) {
|
|
339
|
+
const viewKey = documentViewKey(key);
|
|
340
|
+
const cached = views?.get(viewKey);
|
|
341
|
+
if (cached?._tag === "Document") found.set(viewKey, cached.value);
|
|
342
|
+
else missing.set(viewKey, key);
|
|
343
|
+
}
|
|
344
|
+
const pending = Array.from(missing);
|
|
345
|
+
for (let index = 0; index < pending.length; index += 50) {
|
|
346
|
+
const requested = new Map(pending.slice(index, index + 50));
|
|
347
|
+
const rawRows = yield* query(sql`
|
|
348
|
+
SELECT namespace, source_id, format_version, generation, revision, document_json,
|
|
349
|
+
length(CAST(namespace AS BLOB)) + length(CAST(source_id AS BLOB)) +
|
|
350
|
+
length(CAST(document_json AS BLOB)) + 128 AS stored_bytes
|
|
351
|
+
FROM effect_agent_memory_documents_v1
|
|
352
|
+
WHERE ${sql.or(Array.from(requested.values(), (key) => sql`(namespace = ${key.namespace.address} AND source_id = ${key.id})`))}
|
|
353
|
+
`, operation);
|
|
354
|
+
const rows = yield* decodeRows(MemoryDocumentRow, rawRows, operation);
|
|
355
|
+
for (const row of rows) {
|
|
356
|
+
const viewKey = JSON.stringify([
|
|
357
|
+
"document",
|
|
358
|
+
row.namespace,
|
|
359
|
+
row.source_id
|
|
360
|
+
]);
|
|
361
|
+
const key = requested.get(viewKey);
|
|
362
|
+
if (key === void 0 || found.has(viewKey)) return yield* storageError(operation, "corrupt");
|
|
363
|
+
if (row.format_version !== STORAGE_VERSION) return yield* storageError(operation, "incompatible");
|
|
364
|
+
const document = (yield* decodeVersionedJson(StoredMemoryResult, row.document_json, operation)).value;
|
|
365
|
+
yield* validateDocument(document, key, operation);
|
|
366
|
+
if (row.generation !== document.generation || row.revision !== document.source.revision) return yield* storageError(operation, "corrupt");
|
|
367
|
+
found.set(viewKey, {
|
|
368
|
+
document,
|
|
369
|
+
bytes: row.stored_bytes
|
|
370
|
+
});
|
|
371
|
+
}
|
|
372
|
+
for (const viewKey of requested.keys()) {
|
|
373
|
+
const value = found.get(viewKey) ?? null;
|
|
374
|
+
found.set(viewKey, value);
|
|
375
|
+
views?.set(viewKey, {
|
|
376
|
+
_tag: "Document",
|
|
377
|
+
value
|
|
378
|
+
});
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
return found;
|
|
382
|
+
}, (effect) => owner === void 0 ? effect : owner.read(effect));
|
|
282
383
|
return {
|
|
283
384
|
get: Effect.fn("SqliteMemoryStore.get")(function* (key) {
|
|
284
385
|
const decodedKey = yield* decodeInput(MemoryKey.Wire, key, "get memory document");
|
|
285
|
-
return (yield*
|
|
386
|
+
return (yield* readDocuments([decodedKey], "get memory document")).get(documentViewKey(decodedKey))?.document ?? null;
|
|
286
387
|
}),
|
|
287
|
-
|
|
388
|
+
readDocuments
|
|
288
389
|
};
|
|
289
390
|
});
|
|
290
391
|
const makeMemoryServices = Effect.fn("SqliteMemoryStore.make")(function* () {
|
|
291
392
|
const sql = yield* SqlClientService.SqlClient;
|
|
292
|
-
const
|
|
393
|
+
const owner = yield* SqlStorageOwner;
|
|
394
|
+
const views = yield* memoryViews();
|
|
395
|
+
const usage = readUsage().pipe(Effect.provideService(SqlClientService.SqlClient, sql), Effect.provideService(SqlStorageOwner, owner));
|
|
396
|
+
const transaction = (effect) => owner === void 0 ? sql.withTransaction(effect) : owner.transaction(effect).pipe(Effect.provideService(SqlClientService.SqlClient, sql));
|
|
293
397
|
const failpoint = yield* MemoryMutationFailpoint;
|
|
294
398
|
const limits = yield* decodeInput(Limits, yield* SqlMemoryLimits, "memory storage limits");
|
|
295
399
|
yield* initializeMemorySchema();
|
|
296
|
-
const { get,
|
|
297
|
-
const
|
|
298
|
-
const
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
const
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
400
|
+
const { get, readDocuments } = yield* makeMemoryReader();
|
|
401
|
+
const readReceipts = Effect.fn("SqliteMemoryStore.readReceipts")(function* (writes, operation) {
|
|
402
|
+
const found = /* @__PURE__ */ new Map();
|
|
403
|
+
const missing = /* @__PURE__ */ new Map();
|
|
404
|
+
for (const write of writes) {
|
|
405
|
+
const viewKey = receiptViewKey(write);
|
|
406
|
+
const cached = views?.get(viewKey);
|
|
407
|
+
if (cached?._tag === "Receipt") found.set(viewKey, cached.value);
|
|
408
|
+
else missing.set(viewKey, write);
|
|
409
|
+
}
|
|
410
|
+
const pending = Array.from(missing);
|
|
411
|
+
for (let index = 0; index < pending.length; index += 50) {
|
|
412
|
+
const requested = new Map(pending.slice(index, index + 50));
|
|
413
|
+
const rawRows = yield* query(sql`
|
|
414
|
+
SELECT namespace, operation_id, source_id, format_version, command_json, result_json
|
|
415
|
+
FROM effect_agent_memory_receipts_v1
|
|
416
|
+
WHERE ${sql.or(Array.from(requested.values(), (write) => sql`(namespace = ${write.key.namespace.address} AND operation_id = ${write.operationId})`))}
|
|
417
|
+
`, operation);
|
|
418
|
+
const rows = yield* decodeRows(MemoryReceiptRow, rawRows, operation);
|
|
419
|
+
for (const row of rows) {
|
|
420
|
+
const viewKey = JSON.stringify([
|
|
421
|
+
"receipt",
|
|
422
|
+
row.namespace,
|
|
423
|
+
row.operation_id
|
|
424
|
+
]);
|
|
425
|
+
if (!requested.has(viewKey) || found.has(viewKey)) return yield* storageError(operation, "corrupt");
|
|
426
|
+
if (row.format_version !== STORAGE_VERSION) return yield* storageError(operation, "incompatible");
|
|
427
|
+
const command = yield* decodeVersionedJson(StoredMemoryCommand, row.command_json, `${operation} command`);
|
|
428
|
+
const result = yield* decodeVersionedJson(StoredMemoryResult, row.result_json, `${operation} result`);
|
|
429
|
+
if (row.namespace !== command.value.key.namespace.address || row.operation_id !== command.value.operationId || row.source_id !== command.value.key.id || result.value.key.namespace.address !== command.value.key.namespace.address || result.value.key.id !== command.value.key.id) return yield* storageError(operation, "corrupt");
|
|
430
|
+
yield* validateDocument(result.value, command.value.key, operation);
|
|
431
|
+
yield* validateReceiptResult(command.value, result.value, operation);
|
|
432
|
+
found.set(viewKey, {
|
|
433
|
+
commandJson: row.command_json,
|
|
434
|
+
result: result.value
|
|
435
|
+
});
|
|
436
|
+
}
|
|
437
|
+
for (const viewKey of requested.keys()) {
|
|
438
|
+
const value = found.get(viewKey) ?? null;
|
|
439
|
+
found.set(viewKey, value);
|
|
440
|
+
views?.set(viewKey, {
|
|
441
|
+
_tag: "Receipt",
|
|
442
|
+
value
|
|
443
|
+
});
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
return found;
|
|
317
447
|
});
|
|
318
|
-
const
|
|
448
|
+
const changeMany = Effect.fn("SqliteMemoryStore.changeMany")(function* (writes) {
|
|
319
449
|
const operation = "change memory document";
|
|
320
|
-
const
|
|
321
|
-
|
|
450
|
+
const decodedWrites = yield* decodeInput(SqlMemoryWriteBatch, writes, operation);
|
|
451
|
+
if (decodedWrites.length === 0) return [];
|
|
452
|
+
const commands = yield* Effect.forEach(decodedWrites, (write) => encodeJson(StoredMemoryCommand, StoredMemoryCommand.make({
|
|
322
453
|
version: STORAGE_VERSION,
|
|
323
|
-
value:
|
|
324
|
-
}), "encode memory command")
|
|
454
|
+
value: write
|
|
455
|
+
}), "encode memory command").pipe(Effect.map((commandJson) => ({
|
|
456
|
+
write,
|
|
457
|
+
commandJson
|
|
458
|
+
}))));
|
|
325
459
|
yield* failpoint.hit("memory:change:before");
|
|
326
|
-
const transactionResult = yield*
|
|
327
|
-
const
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
460
|
+
const transactionResult = yield* transaction(Effect.gen(function* () {
|
|
461
|
+
const retainedReceipts = yield* readReceipts(decodedWrites, operation);
|
|
462
|
+
const retainedDocuments = yield* readDocuments(decodedWrites.filter((write) => retainedReceipts.get(receiptViewKey(write)) === null).map((write) => write.key), operation);
|
|
463
|
+
const documents = /* @__PURE__ */ new Map();
|
|
464
|
+
const receipts = /* @__PURE__ */ new Map();
|
|
465
|
+
const results = [];
|
|
466
|
+
let projectedUsage;
|
|
467
|
+
for (const { write, commandJson } of commands) {
|
|
468
|
+
const receiptKey = receiptViewKey(write);
|
|
469
|
+
const receipt = receipts.get(receiptKey) ?? retainedReceipts.get(receiptKey) ?? null;
|
|
470
|
+
if (receipt !== null) {
|
|
471
|
+
if (receipt.commandJson !== commandJson) return yield* MemoryOperationConflict.make({
|
|
472
|
+
key: write.key,
|
|
473
|
+
operationId: write.operationId
|
|
474
|
+
});
|
|
475
|
+
results.push(receipt.result);
|
|
476
|
+
continue;
|
|
477
|
+
}
|
|
478
|
+
const documentKey = documentViewKey(write.key);
|
|
479
|
+
const currentRow = documents.get(documentKey) ?? retainedDocuments.get(documentKey) ?? null;
|
|
480
|
+
const current = currentRow?.document ?? null;
|
|
481
|
+
const modifiedAt = yield* Clock.currentTimeMillis;
|
|
482
|
+
const next = yield* applyMemoryWrite(current, write, modifiedAt);
|
|
483
|
+
const resultJson = yield* encodeJson(StoredMemoryResult, StoredMemoryResult.make({
|
|
484
|
+
version: STORAGE_VERSION,
|
|
485
|
+
value: next
|
|
486
|
+
}), "encode memory result");
|
|
487
|
+
const documentJson = resultJson;
|
|
488
|
+
yield* validateEncodedChange({
|
|
489
|
+
commandJson,
|
|
490
|
+
documentJson,
|
|
491
|
+
resultJson
|
|
492
|
+
}, operation);
|
|
493
|
+
const bytes = utf8ByteLength;
|
|
494
|
+
const identityBytes = bytes(next.key.namespace.address) + bytes(next.key.id);
|
|
495
|
+
const resultBytes = bytes(resultJson);
|
|
496
|
+
const documentBytes = identityBytes + resultBytes + 128;
|
|
497
|
+
const receiptBytes = identityBytes + bytes(write.operationId) + bytes(commandJson) + resultBytes + 128;
|
|
498
|
+
if (Math.max(documentBytes, receiptBytes) > limits.maxRowBytes) return yield* storageError("memory row byte limit", "invalid-input");
|
|
499
|
+
const cachedUsage = views?.get("usage");
|
|
500
|
+
const used = projectedUsage ?? (cachedUsage?._tag === "Usage" ? cachedUsage.value : yield* usage);
|
|
501
|
+
if (used.bytes < (currentRow?.bytes ?? 0) || current !== null && used.documents === 0) return yield* storageError("read memory usage", "corrupt");
|
|
502
|
+
const maxReceipts = limits.maxReceipts - (write._tag === "Put" ? limits.reservedWithdrawalReceipts ?? 0 : 0);
|
|
503
|
+
const maxStorageBytes = limits.maxStorageBytes - (write._tag === "Put" ? limits.reservedWithdrawalBytes ?? 0 : 0);
|
|
504
|
+
if (used.documents > limits.maxDocuments - (current === null ? 1 : 0) || used.receipts >= maxReceipts || used.bytes - (currentRow?.bytes ?? 0) > maxStorageBytes - documentBytes - receiptBytes) return yield* storageError("memory storage limit", "invalid-input");
|
|
505
|
+
projectedUsage = {
|
|
506
|
+
documents: used.documents + (current === null ? 1 : 0),
|
|
507
|
+
receipts: used.receipts + 1,
|
|
508
|
+
bytes: used.bytes - (currentRow?.bytes ?? 0) + documentBytes + receiptBytes
|
|
336
509
|
};
|
|
510
|
+
documents.set(documentKey, {
|
|
511
|
+
document: next,
|
|
512
|
+
bytes: documentBytes,
|
|
513
|
+
documentJson
|
|
514
|
+
});
|
|
515
|
+
receipts.set(receiptKey, {
|
|
516
|
+
write,
|
|
517
|
+
commandJson,
|
|
518
|
+
resultJson,
|
|
519
|
+
result: next
|
|
520
|
+
});
|
|
521
|
+
results.push(next);
|
|
522
|
+
}
|
|
523
|
+
if (projectedUsage === void 0) return {
|
|
524
|
+
documents: results,
|
|
525
|
+
changed: false
|
|
526
|
+
};
|
|
527
|
+
const documentRows = Array.from(documents.values(), ({ document, documentJson }) => ({
|
|
528
|
+
namespace: document.key.namespace.address,
|
|
529
|
+
source_id: document.key.id,
|
|
530
|
+
format_version: STORAGE_VERSION,
|
|
531
|
+
generation: document.generation,
|
|
532
|
+
revision: document.source.revision,
|
|
533
|
+
document_json: documentJson
|
|
534
|
+
}));
|
|
535
|
+
for (let index = 0; index < documentRows.length; index += 16) {
|
|
536
|
+
const chunk = documentRows.slice(index, index + 16);
|
|
537
|
+
const changedRows = yield* sql`
|
|
538
|
+
INSERT INTO effect_agent_memory_documents_v1 ${sql.insert(chunk)}
|
|
539
|
+
ON CONFLICT (namespace, source_id) DO UPDATE SET
|
|
540
|
+
format_version = excluded.format_version,
|
|
541
|
+
generation = excluded.generation,
|
|
542
|
+
revision = excluded.revision,
|
|
543
|
+
document_json = excluded.document_json
|
|
544
|
+
RETURNING 1 AS changed
|
|
545
|
+
`;
|
|
546
|
+
const changed = yield* decodeRows(MemoryChangeCountRow, changedRows, operation);
|
|
547
|
+
if (changed.length !== chunk.length || changed.some((row) => row.changed !== 1)) return yield* storageError(operation, "corrupt");
|
|
337
548
|
}
|
|
338
|
-
const currentRow = yield* readDocument(decodedWrite.key, operation);
|
|
339
|
-
const current = currentRow?.document ?? null;
|
|
340
|
-
const modifiedAt = yield* Clock.currentTimeMillis;
|
|
341
|
-
const next = yield* applyMemoryWrite(current, decodedWrite, modifiedAt);
|
|
342
|
-
const resultJson = yield* encodeJson(StoredMemoryResult, StoredMemoryResult.make({
|
|
343
|
-
version: STORAGE_VERSION,
|
|
344
|
-
value: next
|
|
345
|
-
}), "encode memory result");
|
|
346
|
-
const documentJson = resultJson;
|
|
347
|
-
yield* validateEncodedChange({
|
|
348
|
-
commandJson,
|
|
349
|
-
documentJson,
|
|
350
|
-
resultJson
|
|
351
|
-
}, operation);
|
|
352
|
-
const bytes = utf8ByteLength;
|
|
353
|
-
const identityBytes = bytes(next.key.namespace.address) + bytes(next.key.id);
|
|
354
|
-
const resultBytes = bytes(resultJson);
|
|
355
|
-
const documentBytes = identityBytes + resultBytes + 128;
|
|
356
|
-
const receiptBytes = identityBytes + bytes(decodedWrite.operationId) + bytes(commandJson) + resultBytes + 128;
|
|
357
|
-
if (Math.max(documentBytes, receiptBytes) > limits.maxRowBytes) return yield* storageError("memory row byte limit", "invalid-input");
|
|
358
|
-
const used = yield* usage;
|
|
359
|
-
if (used.bytes < (currentRow?.bytes ?? 0) || current !== null && used.documents === 0) return yield* storageError("read memory usage", "corrupt");
|
|
360
|
-
const maxReceipts = limits.maxReceipts - (decodedWrite._tag === "Put" ? limits.reservedWithdrawalReceipts ?? 0 : 0);
|
|
361
|
-
const maxStorageBytes = limits.maxStorageBytes - (decodedWrite._tag === "Put" ? limits.reservedWithdrawalBytes ?? 0 : 0);
|
|
362
|
-
if (used.documents > limits.maxDocuments - (current === null ? 1 : 0) || used.receipts >= maxReceipts || used.bytes - (currentRow?.bytes ?? 0) > maxStorageBytes - documentBytes - receiptBytes) return yield* storageError("memory storage limit", "invalid-input");
|
|
363
|
-
if (current === null) yield* sql`
|
|
364
|
-
INSERT INTO effect_agent_memory_documents_v1 (
|
|
365
|
-
namespace, source_id, format_version, generation, revision, document_json
|
|
366
|
-
) VALUES (
|
|
367
|
-
${next.key.namespace.address}, ${next.key.id}, ${STORAGE_VERSION},
|
|
368
|
-
${next.generation}, ${next.source.revision}, ${documentJson}
|
|
369
|
-
)
|
|
370
|
-
`;
|
|
371
|
-
else yield* sql`
|
|
372
|
-
UPDATE effect_agent_memory_documents_v1
|
|
373
|
-
SET format_version = ${STORAGE_VERSION},
|
|
374
|
-
generation = ${next.generation},
|
|
375
|
-
revision = ${next.source.revision},
|
|
376
|
-
document_json = ${documentJson}
|
|
377
|
-
WHERE namespace = ${next.key.namespace.address}
|
|
378
|
-
AND source_id = ${next.key.id}
|
|
379
|
-
AND generation = ${current.generation}
|
|
380
|
-
AND revision = ${current.source.revision}
|
|
381
|
-
`;
|
|
382
|
-
const changedRows = yield* sql`
|
|
383
|
-
SELECT changes() AS changed
|
|
384
|
-
`;
|
|
385
|
-
const changed = yield* decodeRows(MemoryChangeCountRow, changedRows, operation);
|
|
386
|
-
if (changed.length !== 1 || changed[0].changed !== 1) return yield* storageError(operation, "corrupt");
|
|
387
549
|
yield* failpoint.hit("memory:change:after-state");
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
550
|
+
const receiptRows = Array.from(receipts.values(), ({ write, commandJson, resultJson }) => ({
|
|
551
|
+
namespace: write.key.namespace.address,
|
|
552
|
+
operation_id: write.operationId,
|
|
553
|
+
source_id: write.key.id,
|
|
554
|
+
format_version: STORAGE_VERSION,
|
|
555
|
+
command_json: commandJson,
|
|
556
|
+
result_json: resultJson
|
|
557
|
+
}));
|
|
558
|
+
for (let index = 0; index < receiptRows.length; index += 16) yield* sql`
|
|
559
|
+
INSERT INTO effect_agent_memory_receipts_v1 ${sql.insert(receiptRows.slice(index, index + 16))}
|
|
560
|
+
`;
|
|
396
561
|
yield* failpoint.hit("memory:change:after-receipt");
|
|
562
|
+
for (const [key, { document, bytes }] of documents) views?.set(key, {
|
|
563
|
+
_tag: "Document",
|
|
564
|
+
value: {
|
|
565
|
+
document,
|
|
566
|
+
bytes
|
|
567
|
+
}
|
|
568
|
+
});
|
|
569
|
+
for (const [key, { commandJson, result }] of receipts) views?.set(key, {
|
|
570
|
+
_tag: "Receipt",
|
|
571
|
+
value: {
|
|
572
|
+
commandJson,
|
|
573
|
+
result
|
|
574
|
+
}
|
|
575
|
+
});
|
|
576
|
+
views?.set("usage", {
|
|
577
|
+
_tag: "Usage",
|
|
578
|
+
value: projectedUsage
|
|
579
|
+
});
|
|
397
580
|
return {
|
|
398
|
-
|
|
581
|
+
documents: results,
|
|
399
582
|
changed: true
|
|
400
583
|
};
|
|
401
584
|
})).pipe(Effect.catchTag("SqlError", () => Effect.fail(storageError(operation))));
|
|
402
585
|
if (transactionResult.changed) yield* failpoint.hit("memory:change:after");
|
|
403
|
-
return transactionResult.
|
|
586
|
+
return transactionResult.documents;
|
|
587
|
+
});
|
|
588
|
+
const change = Effect.fn("SqliteMemoryStore.change")(function* (write) {
|
|
589
|
+
return (yield* changeMany([write]))[0];
|
|
404
590
|
});
|
|
405
|
-
|
|
591
|
+
const batchWriter = SqlMemoryBatchWriter.of({ changeMany: Effect.fn("SqlMemoryBatchWriter.changeMany")(function* (writes) {
|
|
592
|
+
const documents = yield* changeMany(writes);
|
|
593
|
+
return yield* Effect.forEach(documents, (document, index) => MemoryDocument.restore(writes[index].key.namespace, document));
|
|
594
|
+
}) });
|
|
595
|
+
return Context.make(MemoryReader, MemoryReader.fromAdapter({ get })).pipe(Context.add(MemoryWriter, MemoryWriter.fromAdapter({ change })), Context.add(SqlMemoryBatchWriter, batchWriter));
|
|
406
596
|
});
|
|
407
597
|
/** SQLite memory ports with the mutation failpoint kept injectable for recovery tests. */
|
|
408
598
|
const memoryStoreLayerWithFailpoints = Layer.effectContext(makeMemoryServices());
|
|
409
|
-
/** SQLite memory reader and writer with the production no-op
|
|
599
|
+
/** SQLite memory reader, single writer and batch writer with the production no-op failpoint. */
|
|
410
600
|
const memoryStoreLayer = memoryStoreLayerWithFailpoints.pipe(Layer.provide(MemoryMutationFailpoint.layer));
|
|
411
601
|
/** Reads an existing memory schema without writes, transactions, or mutation failpoints. */
|
|
412
602
|
const memoryReaderLayer = Layer.effect(MemoryReader, Effect.gen(function* () {
|
|
@@ -431,6 +621,6 @@ const memoryReaderLayer = Layer.effect(MemoryReader, Effect.gen(function* () {
|
|
|
431
621
|
return MemoryReader.fromAdapter({ get });
|
|
432
622
|
}));
|
|
433
623
|
//#endregion
|
|
434
|
-
export { SqlMemoryLimits, memoryReaderLayer, memoryStoreLayer, memoryStoreLayerWithFailpoints };
|
|
624
|
+
export { SqlMemoryBatchWriter, SqlMemoryLimits, SqlMemoryWriteBatch, SqlStorageOwner, memoryReaderLayer, memoryStoreLayer, memoryStoreLayerWithFailpoints };
|
|
435
625
|
|
|
436
626
|
//# sourceMappingURL=SqlMemoryStore.mjs.map
|