@tangleai/context 0.21.1 → 0.24.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/ledger.js CHANGED
@@ -1,4 +1,3 @@
1
- //@ts-check
2
1
  /**
3
2
  * The ledger: schema-validated, durable, addressable state that outlives
4
3
  * a context window.
@@ -23,7 +22,7 @@
23
22
  * - **Nothing enters without passing its schema**, compiled by
24
23
  * `JarenValidator` here at construction. A rejected write answers the
25
24
  * same `{ error, errors, inputSchema }` the toolbox answers with (one
26
- * implementation, in `check.js`) and never throws: a model that wrote
25
+ * implementation, in `check.ts`) and never throws: a model that wrote
27
26
  * a bad memory can read why and fix it.
28
27
  * - **Every mutation is reversible.** `snapshot()` before, `rollback()`
29
28
  * after — which is what will make model-proposed refinements safe to
@@ -55,98 +54,47 @@
55
54
  * slots, refinement's patchable state, and the website assistant's
56
55
  * durable memory — all on this one implementation.
57
56
  */
58
-
59
57
  import { JarenValidator } from '@jarenjs/validate';
60
58
  import { excerpt } from '@jarenjs/core/chunk';
61
59
  import { isVector, cosineSimilarity } from '@jarenjs/core/vector';
62
-
63
60
  import { checkOutcome } from '@jarenjs/core/check';
64
61
  import { invalidInput } from '@tangleai/models/check';
65
- import { validateClaimEvidence } from './evidence.js';
62
+ import { validateClaimEvidence } from "./evidence.js";
66
63
  import { AiError } from '@tangleai/models/errors';
67
- import { retainArchives } from './archive.js';
68
- import { jsonBytes, checkpointProgress, validateCheckpoint, goalPrompt } from './retention.js';
69
- import { atomicTask, isAtomicView } from './storage/transaction.js';
70
- import { createMemoryStorage } from './storage/memory.js';
71
- import { LEDGER_SCHEMAS } from './schemas/ledger.js';
72
-
73
- /** @typedef {import('./schemas/ledger.js').LedgerGoal} LedgerGoal */
74
- /** @typedef {import('./schemas/ledger.js').LedgerMemory} LedgerMemory */
75
- /** @typedef {import('./schemas/ledger.js').LedgerSkill} LedgerSkill */
76
- /** @typedef {import('./schemas/ledger.js').LedgerSlot} LedgerSlot */
77
- /** @typedef {import('./schemas/ledger.js').LedgerRejection} LedgerRejection */
78
- /** @typedef {import('./schemas/ledger.js').LedgerEmbeddingPair} LedgerEmbeddingPair */
79
- /** @typedef {import('./schemas/ledger.js').LedgerEmbeddedBy} LedgerEmbeddedBy */
80
- /** @typedef {import('@tangleai/models/embed').Embedder} Embedder */
81
-
82
- /**
83
- * What `recall({ near })` answers: the memories that carry a comparable
64
+ import { retainArchives } from "./archive.js";
65
+ import { jsonBytes, checkpointProgress, validateCheckpoint, goalPrompt } from "./retention.js";
66
+ import { atomicTask, isAtomicView } from "./storage/transaction.js";
67
+ import { createMemoryStorage } from "./storage/memory.js";
68
+ import { LEDGER_SCHEMAS } from "./schemas/ledger.js";
69
+ /** What `recall({ near })` answers: the memories that carry a comparable
84
70
  * vector, ranked by cosine similarity (descending; ties by recency, then
85
71
  * id), one score per memory in the same order, and the count of records
86
72
  * that passed the filter but carry no vector and were therefore skipped
87
- * — reported, never scored.
88
- * @typedef {object} LedgerRankedMemories
89
- * @property {LedgerMemory[]} memories
90
- * @property {number[]} scores
91
- * @property {number} skipped
92
- * @property {'sweep' | 'adapter'} via - which path answered: the ledger's
93
- * own read-and-rank, or the adapter's `rank` capability
94
- * @property {LedgerRanking} ranking - candidate selection provenance, not a quality guarantee
95
- */
96
-
97
- /**
98
- * What `recallSkills({ near })` answers — see {@link LedgerRankedMemories}.
99
- * @typedef {object} LedgerRankedSkills
100
- * @property {LedgerSkill[]} skills
101
- * @property {number[]} scores
102
- * @property {number} skipped
103
- * @property {'sweep' | 'adapter'} via - see {@link LedgerRankedMemories}
104
- * @property {LedgerRanking} ranking - see {@link LedgerRankedMemories}
105
- */
106
-
107
- /**
108
- * Optional storage rank metadata. Legacy adapters normalize to exhaustive.
73
+ * — reported, never scored. */
74
+ /** Optional storage rank metadata. Legacy adapters normalize to exhaustive.
109
75
  * Exhaustive means exact candidate selection, not that every record is returned.
110
- * Candidate count is the number returned before ledger filtering and capping.
111
- * @typedef {object} LedgerRanking
112
- * @property {string} algorithm
113
- * @property {boolean} exhaustive
114
- * @property {number} candidateCount
115
- */
116
-
117
- /**
118
- * The query both recalls take. `near` is the string to rank by meaning
76
+ * Candidate count is the number returned before ledger filtering and capping. */
77
+ /** The query both recalls take. `near` is the string to rank by meaning
119
78
  * against, and needs the embedder seam; `minScore` filters the ranked
120
79
  * result (cosine, in [-1, 1]); `tags` and `where` narrow the candidates
121
- * first, exactly as they do without `near`.
122
- * @typedef {object} LedgerQuery
123
- * @property {string[]} [tags]
124
- * @property {any} [where]
125
- * @property {number} [limit]
126
- * @property {string} [near]
127
- * @property {number} [minScore]
128
- */
129
-
80
+ * first, exactly as they do without `near`. */
130
81
  /** The key space. State and snapshots are separate prefixes on purpose:
131
82
  * a rollback wipes state and must not take the other snapshots with it. */
132
83
  const STATE = 'ai/state/';
133
84
  const SNAP = 'ai/snap/';
134
85
  const KEYS = {
135
- goal: `${STATE}goal/active`,
136
- goalArchive: `${STATE}goal/archive/`,
137
- memory: `${STATE}memory/`,
138
- skill: `${STATE}skill/`,
139
- slot: `${STATE}slot/`,
140
- slotContent: `${STATE}slot-content/`,
86
+ goal: `${STATE}goal/active`,
87
+ goalArchive: `${STATE}goal/archive/`,
88
+ memory: `${STATE}memory/`,
89
+ skill: `${STATE}skill/`,
90
+ slot: `${STATE}slot/`,
91
+ slotContent: `${STATE}slot-content/`,
141
92
  };
142
-
143
93
  /** Slot metadata carries a short excerpt so a root request can list a
144
94
  * hundred slots without carrying one slot's content. */
145
95
  const SLOT_EXCERPT_CHARS = 120;
146
-
147
96
  /** A zero-padded sequence, so `keys()` sorts lexicographically into order. */
148
97
  const seq = (n) => String(n).padStart(6, '0');
149
-
150
98
  /**
151
99
  * The next sequence number under a prefix: one past the highest that
152
100
  * still exists, read from the trailing digits of every key. Never the
@@ -155,21 +103,20 @@ const seq = (n) => String(n).padStart(6, '0');
155
103
  * key that happens to end in digits merely advances the sequence, which
156
104
  * costs nothing: uniqueness needs only that no existing key ends in the
157
105
  * number minted.
158
- * @param {string[]} keys - every key under the prefix
159
- * @returns {number}
106
+ * @param keys - every key under the prefix
160
107
  */
161
108
  function nextSequence(keys) {
162
- let highest = -1;
163
- for (const key of keys) {
164
- const match = /(\d+)$/.exec(key);
165
- if (match !== null) {
166
- const n = Number(match[1]);
167
- if (n > highest) highest = n;
109
+ let highest = -1;
110
+ for (const key of keys) {
111
+ const match = /(\d+)$/.exec(key);
112
+ if (match !== null) {
113
+ const n = Number(match[1]);
114
+ if (n > highest)
115
+ highest = n;
116
+ }
168
117
  }
169
- }
170
- return highest + 1;
118
+ return highest + 1;
171
119
  }
172
-
173
120
  /**
174
121
  * A record with its `undefined` members removed.
175
122
  *
@@ -179,50 +126,42 @@ function nextSequence(keys) {
179
126
  * present, and the write validates — then `JSON.stringify` drops it on
180
127
  * the way to storage and an unevidenced memory is durable. Stripping
181
128
  * first makes what is validated exactly what would be stored.
182
- * @param {Record<string, any>} record
183
- * @returns {any} the stripped record — `any` so each write site's declared
129
+ * @returns the stripped record — `any` so each write site's declared
184
130
  * return type (the record typedef) is the statement that binds, not an
185
131
  * inference from this generic helper
186
132
  */
187
133
  function defined(record) {
188
- const out = {};
189
- for (const [key, value] of Object.entries(record)) {
190
- if (value !== undefined) out[key] = value;
191
- }
192
- return out;
134
+ const out = {};
135
+ for (const [key, value] of Object.entries(record)) {
136
+ if (value !== undefined)
137
+ out[key] = value;
138
+ }
139
+ return out;
193
140
  }
194
-
195
141
  /**
196
142
  * Newest first, ties broken by id so two records written in the same
197
143
  * millisecond still come back in a stable order. A retrieval whose order
198
144
  * wobbled between calls would make everything downstream of it
199
145
  * unreproducible.
200
- * @param {any[]} records
201
146
  */
202
147
  function byRecency(records) {
203
- return [...records].sort(recencyOrder);
148
+ return [...records].sort(recencyOrder);
204
149
  }
205
-
206
150
  /**
207
151
  * The comparator behind {@link byRecency}: newer first, then id — the
208
152
  * same rule ranked recall falls back to between two equal scores.
209
- * @param {any} a
210
- * @param {any} b
211
153
  */
212
154
  function recencyOrder(a, b) {
213
- return a.at === b.at
214
- ? String(a.id ?? a.name).localeCompare(String(b.id ?? b.name))
215
- : String(b.at).localeCompare(String(a.at));
155
+ return a.at === b.at
156
+ ? String(a.id ?? a.name).localeCompare(String(b.id ?? b.name))
157
+ : String(b.at).localeCompare(String(a.at));
216
158
  }
217
-
218
159
  /**
219
160
  * The text a skill is embedded from: its name, when it applies and what
220
161
  * to do, one per line — what "near" means for a skill. A memory is
221
162
  * embedded from its `text` alone.
222
- * @param {{ name: string, when: string, instructions: string }} skill
223
163
  */
224
164
  const skillText = (skill) => `${skill.name}\n${skill.when}\n${skill.instructions}`;
225
-
226
165
  /**
227
166
  * Whether two vector identities name the same space: the same model at
228
167
  * the same width. The rule the ledger applies before any arithmetic
@@ -243,19 +182,16 @@ const skillText = (skill) => `${skill.name}\n${skill.when}\n${skill.instructions
243
182
  * produce vectors whose cosine is arithmetic without meaning, which is
244
183
  * why `dims` matching is necessary and never sufficient.
245
184
  *
246
- * @param {LedgerEmbeddedBy | undefined | null} a
247
- * @param {LedgerEmbeddedBy | undefined | null} b
248
- * @returns {boolean} whether both are identities naming one space
185
+ * @returns whether both are identities naming one space
249
186
  * @example
250
187
  * sameIdentity({ model: 'm', dims: 4 }, { model: 'm', dims: 4 }); // true
251
188
  * sameIdentity({ model: 'm', dims: 4 }, { model: 'm', dims: 8 }); // false — a re-embed, not a match
252
189
  * sameIdentity(undefined, undefined); // false — no space is not a shared space
253
190
  */
254
191
  export function sameIdentity(a, b) {
255
- return a !== undefined && a !== null && b !== undefined && b !== null
256
- && a.model === b.model && a.dims === b.dims;
192
+ return a !== undefined && a !== null && b !== undefined && b !== null
193
+ && a.model === b.model && a.dims === b.dims;
257
194
  }
258
-
259
195
  /**
260
196
  * An identity as a refusal names it — `"text-embedding-3-small (1536
261
197
  * dims)"`. Exported for the same reason as {@link sameIdentity}: it is
@@ -264,40 +200,18 @@ export function sameIdentity(a, b) {
264
200
  * key for collecting the DISTINCT identities a `rank` adapter owes its
265
201
  * caller.
266
202
  *
267
- * @param {LedgerEmbeddedBy} identity
268
- * @returns {string}
269
203
  */
270
204
  export function describeIdentity(identity) {
271
- return `${identity.model} (${identity.dims} dims)`;
205
+ return `${identity.model} (${identity.dims} dims)`;
272
206
  }
273
-
274
207
  /** The seam refusal, shared by every entry point that needs the embedder. */
275
208
  const SEAM_REFUSAL = 'needs the embedder seam — inject createEmbeddingClient(...) or any { embed, model, dims }';
276
-
277
209
  /** How many texts one `embedMissing` batch hands the seam. */
278
210
  const EMBED_BATCH = 64;
279
-
280
211
  /**
281
212
  * Create a ledger.
282
213
  *
283
- * @param {{ storage?: { get: (key: string) => Promise<any>,
284
- * set: (key: string, value: any) => Promise<void>,
285
- * delete: (key: string) => Promise<void>,
286
- * keys: (prefix?: string) => Promise<string[]>,
287
- * status?: () => any,
288
- * mutate?: import('./storage/transaction.js').StorageMutation,
289
- * rank?: (request: { prefix: string, vector: number[], model: string, dims: number,
290
- * limit?: number, minScore?: number }) => Promise<{ hits: { key: string, score: number }[],
291
- * skipped: number, identities: LedgerEmbeddedBy[], ranking?: LedgerRanking }> },
292
- * compileQuery?: (document: any) => (data: any) => any,
293
- * embedder?: Embedder,
294
- * embedOnWrite?: boolean,
295
- * validator?: any,
296
- * now?: () => string,
297
- * archiveLimits?: { maxItems?: number, maxBytes?: number },
298
- * goalLimits?: { maxEntries?: number, maxBytes?: number, maxChars?: number },
299
- * checkpointReducer?: (goal: any) => any,
300
- * artifacts?: import('./schemas/evidence.js').ArtifactRecord[] }} [options]
214
+ * @param [options]
301
215
  * - `storage` defaults to an in-memory adapter, so a ledger works with
302
216
  * nothing wired. Anything durable is the host's to inject. Its four
303
217
  * methods are the contract; an adapter that can rank vectors itself
@@ -323,1075 +237,1015 @@ const EMBED_BATCH = 64;
323
237
  * tests, exactly as the rest of the suite injects its environment).
324
238
  */
325
239
  export function createLedger(options = {}) {
326
- const storage = options.storage ?? createMemoryStorage();
327
- for (const [limits, names] of [[options.archiveLimits, ['maxItems', 'maxBytes']],
328
- [options.goalLimits, ['maxEntries', 'maxBytes', 'maxChars']]]) {
329
- for (const [name, value] of Object.entries(limits ?? {})) {
330
- if (!names.includes(name)) throw new TypeError(`unknown ledger budget '${name}'`);
331
- if (!Number.isSafeInteger(value) || value < 0) throw new TypeError(`${name} must be a non-negative safe integer`);
240
+ const storage = options.storage ?? createMemoryStorage();
241
+ for (const [limits, names] of [[options.archiveLimits, ['maxItems', 'maxBytes']],
242
+ [options.goalLimits, ['maxEntries', 'maxBytes', 'maxChars']]]) {
243
+ for (const [name, value] of Object.entries(limits ?? {})) {
244
+ if (!names.includes(name))
245
+ throw new TypeError(`unknown ledger budget '${name}'`);
246
+ if (!Number.isSafeInteger(value) || value < 0)
247
+ throw new TypeError(`${name} must be a non-negative safe integer`);
248
+ }
332
249
  }
333
- }
334
- const admittedArtifacts = options.artifacts === undefined ? undefined : JSON.parse(JSON.stringify(options.artifacts));
335
- if (admittedArtifacts !== undefined && !validateClaimEvidence({ version: 1, artifacts: admittedArtifacts,
336
- evidence: [], claims: [], visibleEvidence: [] }).valid) throw new TypeError('artifacts must be unique, valid admitted records');
337
- const compileQuery = typeof options.compileQuery === 'function' ? options.compileQuery : null;
338
- /** @type {Embedder | null} */
339
- const embedder = options.embedder ?? null;
340
- if (embedder !== null) {
341
- if (typeof embedder !== 'object' || typeof embedder.embed !== 'function')
342
- throw new AiError('AI0001', 'embedder: expected the seam { embed(texts) Promise<Float32Array[]>, model, dims }');
343
- if (typeof embedder.model !== 'string' || embedder.model === '')
344
- throw new AiError('AI0001', 'embedder: needs a model name it is half of every vector\'s identity');
345
- }
346
- const embedOnWrite = options.embedOnWrite === true;
347
- if (embedOnWrite && embedder === null)
348
- throw new AiError('AI0001', `embedOnWrite ${SEAM_REFUSAL}`);
349
- const now = options.now ?? (() => new Date().toISOString());
350
- // `unknownFormats: 'ignore'` is the library default; it is passed
351
- // EXPLICITLY because this project's convention for a schema it ships is
352
- // the opposite one (`'error'`, so a format nobody registered fails the
353
- // build instead of checking nothing), and this is the deliberate
354
- // exception. It has to be: the schemas declare `format: 'date-time'`
355
- // because that is what the values ARE, but the package may not depend
356
- // on `@jarenjs/formats` (D3), so no compiler for it can ever be
357
- // registered here. The enforcement is the `pattern` beside the format,
358
- // which needs nothing injected; a host passing its own
359
- // formats-registered `validator` gets the stricter check on top.
360
- const jaren = options.validator
361
- ?? new JarenValidator({
362
- skipErrors: false, collectErrors: true, unknownFormats: 'ignore',
363
- });
364
-
365
- /** Every kind's compiled check, built once. */
366
- const checks = Object.fromEntries(Object.entries(LEDGER_SCHEMAS)
367
- .map(([kind, schema]) => [kind, jaren.compile(schema)]));
368
-
369
- /** Compiled retrieval queries, keyed by their document. Recall runs on
370
- * every turn of a long conversation; compiling the same predicate each
371
- * time would be the kind of waste that only shows up under load. */
372
- const queryCache = new Map();
373
-
374
- /** The tail of the write queue: the promise every mutation waits on. */
375
- let queue = Promise.resolve();
376
-
377
- /**
378
- * Run one mutating operation after every one queued before it has
379
- * settled. Every write is a read-modify-write — mint an id from what
380
- * exists, then store; read the goal, then replace it — and two of them
381
- * interleaved read the same "what exists" and collide. One queue per
382
- * ledger makes a `Promise.all` of writes behave as the sequence it
383
- * reads as. A rejected operation does not stall the queue: the next
384
- * one runs regardless, and only its own caller sees the rejection.
385
- * @template T
386
- * @param {(view: any) => Promise<T>} task
387
- * @param {import('./storage/transaction.js').StorageScope} [scope]
388
- * @returns {Promise<T>}
389
- */
390
- function enqueue(task, scope = 'ai/') {
391
- const result = queue.then(() => atomicTask(storage, scope, task));
392
- queue = result.then(() => undefined, () => undefined);
393
- return result;
394
- }
395
-
396
- /**
397
- * Validate a complete record against its kind, or produce the standard
398
- * rejection. The record is built by the caller (defaults already
399
- * filled) so that what is validated is exactly what would be stored.
400
- *
401
- * Exported as well as used internally, because a caller that wants to
402
- * know whether a record WOULD be storable before storing it — a
403
- * model-proposed refinement, which has to reject an unevidenced memory
404
- * without writing one — must ask the same question the write asks,
405
- * against the same compiled schema. A second copy of these checks is
406
- * how "the ledger rejects it" and "the gate rejects it" would come to
407
- * mean different things.
408
- * @param {'goal'|'memory'|'skill'|'slot'} kind
409
- * @param {any} record
410
- * @returns {null | { error: string, errors: any[], inputSchema: any }}
411
- * null when the record is storable.
412
- */
413
- function validate(kind, record) {
414
- const outcome = checkOutcome(checks[kind](record));
415
- if (!outcome.valid) return invalidInput(kind, outcome, LEDGER_SCHEMAS[kind]);
416
- if (kind === 'goal') {
417
- const ids = new Set();
418
- const errors = [];
419
- const entries = [...(record.checkpoint?.sources ?? []), ...record.progress];
420
- for (const entry of entries) {
421
- if (entry.id !== undefined && ids.has(entry.id)) errors.push({ keyword: 'uniqueId', instancePath: '/progress', message: 'progress source ids must be unique' });
422
- if (entry.id !== undefined) ids.add(entry.id);
423
- }
424
- for (const source of record.checkpoint?.sources ?? []) {
425
- if (source.record >= record.checkpoint.records.length)
426
- errors.push({ keyword: 'reference', instancePath: '/checkpoint/sources', message: 'checkpoint source refers to a missing record' });
427
- }
428
- if (errors.length) return invalidInput(kind, { errors }, LEDGER_SCHEMAS[kind]);
250
+ const admittedArtifacts = options.artifacts === undefined ? undefined : JSON.parse(JSON.stringify(options.artifacts));
251
+ if (admittedArtifacts !== undefined && !validateClaimEvidence({
252
+ version: 1, artifacts: admittedArtifacts,
253
+ evidence: [], claims: [], visibleEvidence: []
254
+ }).valid)
255
+ throw new TypeError('artifacts must be unique, valid admitted records');
256
+ const compileQuery = typeof options.compileQuery === 'function' ? options.compileQuery : null;
257
+ const embedder = options.embedder ?? null;
258
+ if (embedder !== null) {
259
+ if (typeof embedder !== 'object' || typeof embedder.embed !== 'function')
260
+ throw new AiError('AI0001', 'embedder: expected the seam — { embed(texts) → Promise<Float32Array[]>, model, dims }');
261
+ if (typeof embedder.model !== 'string' || embedder.model === '')
262
+ throw new AiError('AI0001', 'embedder: needs a model name — it is half of every vector\'s identity');
429
263
  }
430
- if (kind === 'memory' && typeof record.evidence !== 'string') {
431
- const references = validateClaimEvidence(record.evidence, { artifacts: admittedArtifacts });
432
- if (!references.valid) return invalidInput(kind, { errors: references.errors.map((error) => ({
433
- ...error, keyword: error.code ?? error.keyword, instancePath: `/evidence${error.instancePath ?? error.docPath ?? ''}`,
434
- })) }, LEDGER_SCHEMAS[kind]);
264
+ const embedOnWrite = options.embedOnWrite === true;
265
+ if (embedOnWrite && embedder === null)
266
+ throw new AiError('AI0001', `embedOnWrite ${SEAM_REFUSAL}`);
267
+ const now = options.now ?? (() => new Date().toISOString());
268
+ // `unknownFormats: 'ignore'` is the library default; it is passed
269
+ // EXPLICITLY because this project's convention for a schema it ships is
270
+ // the opposite one (`'error'`, so a format nobody registered fails the
271
+ // build instead of checking nothing), and this is the deliberate
272
+ // exception. It has to be: the schemas declare `format: 'date-time'`
273
+ // because that is what the values ARE, but the package may not depend
274
+ // on `@jarenjs/formats` (D3), so no compiler for it can ever be
275
+ // registered here. The enforcement is the `pattern` beside the format,
276
+ // which needs nothing injected; a host passing its own
277
+ // formats-registered `validator` gets the stricter check on top.
278
+ const jaren = options.validator
279
+ ?? new JarenValidator({
280
+ skipErrors: false, collectErrors: true, unknownFormats: 'ignore',
281
+ });
282
+ /** Every kind's compiled check, built once. */
283
+ const checks = Object.fromEntries(Object.entries(LEDGER_SCHEMAS)
284
+ .map(([kind, schema]) => [kind, jaren.compile(schema)]));
285
+ /** Compiled retrieval queries, keyed by their document. Recall runs on
286
+ * every turn of a long conversation; compiling the same predicate each
287
+ * time would be the kind of waste that only shows up under load. */
288
+ const queryCache = new Map();
289
+ /** The tail of the write queue: the promise every mutation waits on. */
290
+ let queue = Promise.resolve();
291
+ /**
292
+ * Run one mutating operation after every one queued before it has
293
+ * settled. Every write is a read-modify-write — mint an id from what
294
+ * exists, then store; read the goal, then replace it — and two of them
295
+ * interleaved read the same "what exists" and collide. One queue per
296
+ * ledger makes a `Promise.all` of writes behave as the sequence it
297
+ * reads as. A rejected operation does not stall the queue: the next
298
+ * one runs regardless, and only its own caller sees the rejection.
299
+ * @template T
300
+ * @param [scope]
301
+ */
302
+ function enqueue(task, scope = 'ai/') {
303
+ const result = queue.then(() => atomicTask(storage, scope, task));
304
+ queue = result.then(() => undefined, () => undefined);
305
+ return result;
435
306
  }
436
- // the schema has said the pair is present together and shaped; what
437
- // it cannot say is that the vector IS what its identity declares —
438
- // `isVector` is the suite's one definition of that
439
- if (record.embedding !== undefined && !isVector(record.embedding, record.embeddedBy.dims)) {
440
- return invalidInput(kind, { errors: [{
441
- instancePath: '/embedding',
442
- keyword: 'embeddedBy',
443
- message: `must be exactly ${record.embeddedBy.dims} finite numbers, as embeddedBy.dims declares`,
444
- }] }, LEDGER_SCHEMAS[kind]);
307
+ /**
308
+ * Validate a complete record against its kind, or produce the standard
309
+ * rejection. The record is built by the caller (defaults already
310
+ * filled) so that what is validated is exactly what would be stored.
311
+ *
312
+ * Exported as well as used internally, because a caller that wants to
313
+ * know whether a record WOULD be storable before storing it — a
314
+ * model-proposed refinement, which has to reject an unevidenced memory
315
+ * without writing one — must ask the same question the write asks,
316
+ * against the same compiled schema. A second copy of these checks is
317
+ * how "the ledger rejects it" and "the gate rejects it" would come to
318
+ * mean different things.
319
+ * @returns
320
+ * null when the record is storable.
321
+ */
322
+ function validate(kind, record) {
323
+ const outcome = checkOutcome(checks[kind](record));
324
+ if (!outcome.valid)
325
+ return invalidInput(kind, outcome, LEDGER_SCHEMAS[kind]);
326
+ if (kind === 'goal') {
327
+ const ids = new Set();
328
+ const errors = [];
329
+ const entries = [...(record.checkpoint?.sources ?? []), ...record.progress];
330
+ for (const entry of entries) {
331
+ if (entry.id !== undefined && ids.has(entry.id))
332
+ errors.push({ keyword: 'uniqueId', instancePath: '/progress', message: 'progress source ids must be unique' });
333
+ if (entry.id !== undefined)
334
+ ids.add(entry.id);
335
+ }
336
+ for (const source of record.checkpoint?.sources ?? []) {
337
+ if (source.record >= record.checkpoint.records.length)
338
+ errors.push({ keyword: 'reference', instancePath: '/checkpoint/sources', message: 'checkpoint source refers to a missing record' });
339
+ }
340
+ if (errors.length)
341
+ return invalidInput(kind, { errors }, LEDGER_SCHEMAS[kind]);
342
+ }
343
+ if (kind === 'memory' && typeof record.evidence !== 'string') {
344
+ const references = validateClaimEvidence(record.evidence, { artifacts: admittedArtifacts });
345
+ if (!references.valid)
346
+ return invalidInput(kind, {
347
+ errors: references.errors.map((error) => ({
348
+ ...error, keyword: error.code ?? error.keyword, instancePath: `/evidence${error.instancePath ?? error.docPath ?? ''}`,
349
+ }))
350
+ }, LEDGER_SCHEMAS[kind]);
351
+ }
352
+ // the schema has said the pair is present together and shaped; what
353
+ // it cannot say is that the vector IS what its identity declares —
354
+ // `isVector` is the suite's one definition of that
355
+ if (record.embedding !== undefined && !isVector(record.embedding, record.embeddedBy.dims)) {
356
+ return invalidInput(kind, {
357
+ errors: [{
358
+ instancePath: '/embedding',
359
+ keyword: 'embeddedBy',
360
+ message: `must be exactly ${record.embeddedBy.dims} finite numbers, as embeddedBy.dims declares`,
361
+ }]
362
+ }, LEDGER_SCHEMAS[kind]);
363
+ }
364
+ return null;
445
365
  }
446
- return null;
447
- }
448
-
449
- /**
450
- * Embed texts through the seam, holding a host embedder to the
451
- * contract the shipped ones keep one finite vector per input, all of
452
- * one width and answering `{ vectors, identity }` or `{ error }`,
453
- * never a throw. The call sits inside the `try` so that a host `embed`
454
- * that throws synchronously lands in the same catch as one that
455
- * rejects. `identity.dims` is the embedder's settled width, or the
456
- * width this reply settled it at.
457
- * @param {string[]} texts
458
- * @returns {Promise<{ vectors: Float32Array[], identity: LedgerEmbeddedBy, error?: undefined }
459
- * | { error: string, vectors?: undefined, identity?: undefined }>}
460
- */
461
- async function embedThrough(texts) {
462
- if (embedder === null) return { error: SEAM_REFUSAL };
463
- try {
464
- const vectors = await embedder.embed(texts);
465
- if (!Array.isArray(vectors) || vectors.length !== texts.length) {
466
- return { error: `the embedder answered ${Array.isArray(vectors) ? vectors.length : 'no'}`
467
- + ` vectors for ${texts.length} inputs` };
468
- }
469
- const dims = embedder.dims ?? vectors[0].length;
470
- for (let i = 0; i < vectors.length; i++) {
471
- if (!isVector(vectors[i], dims))
472
- return { error: `the embedder answered something other than ${dims} finite numbers for input ${i}` };
473
- }
474
- return { vectors, identity: { model: embedder.model, dims } };
366
+ /**
367
+ * Embed texts through the seam, holding a host embedder to the
368
+ * contract the shipped ones keep — one finite vector per input, all of
369
+ * one width — and answering `{ vectors, identity }` or `{ error }`,
370
+ * never a throw. The call sits inside the `try` so that a host `embed`
371
+ * that throws synchronously lands in the same catch as one that
372
+ * rejects. `identity.dims` is the embedder's settled width, or the
373
+ * width this reply settled it at.
374
+ */
375
+ async function embedThrough(texts) {
376
+ if (embedder === null)
377
+ return { error: SEAM_REFUSAL };
378
+ try {
379
+ const vectors = await embedder.embed(texts);
380
+ if (!Array.isArray(vectors) || vectors.length !== texts.length) {
381
+ return {
382
+ error: `the embedder answered ${Array.isArray(vectors) ? vectors.length : 'no'}`
383
+ + ` vectors for ${texts.length} inputs`
384
+ };
385
+ }
386
+ const dims = embedder.dims ?? vectors[0].length;
387
+ for (let i = 0; i < vectors.length; i++) {
388
+ if (!isVector(vectors[i], dims))
389
+ return { error: `the embedder answered something other than ${dims} finite numbers for input ${i}` };
390
+ }
391
+ return { vectors, identity: { model: embedder.model, dims } };
392
+ }
393
+ catch (err) {
394
+ return { error: err?.message ?? String(err) };
395
+ }
475
396
  }
476
- catch (err) {
477
- return { error: /** @type {any} */ (err)?.message ?? String(err) };
397
+ /**
398
+ * The stored form of one seam vector on a record: plain numbers, with
399
+ * the identity beside them.
400
+ * @template {object} T
401
+ */
402
+ const withEmbedding = (record, vector, identity) => ({ ...record, embedding: Array.from(vector), embeddedBy: { ...identity } });
403
+ /**
404
+ * The `embedOnWrite` step: a record arriving without a vector is
405
+ * embedded inside its own write. A seam failure is reported on the
406
+ * RESULT, not the record — the record is stored un-embedded and the
407
+ * write succeeds, because one bad network call must not lose a memory;
408
+ * `embedMissing()` closes the gap later.
409
+ * @template {{ embedding?: number[] }} T
410
+ */
411
+ async function embedOnWriteStep(record, text) {
412
+ if (!embedOnWrite || record.embedding !== undefined)
413
+ return { record };
414
+ const answer = await embedThrough([text]);
415
+ if (answer.error !== undefined)
416
+ return { record, embedError: `the embedder seam failed — ${answer.error}; stored un-embedded` };
417
+ return { record: withEmbedding(record, answer.vectors[0], answer.identity) };
478
418
  }
479
- }
480
-
481
- /**
482
- * The stored form of one seam vector on a record: plain numbers, with
483
- * the identity beside them.
484
- * @template {object} T
485
- * @param {T} record
486
- * @param {Float32Array} vector
487
- * @param {LedgerEmbeddedBy} identity
488
- * @returns {T & { embedding: number[], embeddedBy: LedgerEmbeddedBy }}
489
- */
490
- const withEmbedding = (record, vector, identity) =>
491
- ({ ...record, embedding: Array.from(vector), embeddedBy: { ...identity } });
492
-
493
- /**
494
- * The `embedOnWrite` step: a record arriving without a vector is
495
- * embedded inside its own write. A seam failure is reported on the
496
- * RESULT, not the record — the record is stored un-embedded and the
497
- * write succeeds, because one bad network call must not lose a memory;
498
- * `embedMissing()` closes the gap later.
499
- * @template {{ embedding?: number[] }} T
500
- * @param {T} record
501
- * @param {string} text
502
- * @returns {Promise<{ record: T, embedError?: string }>}
503
- */
504
- async function embedOnWriteStep(record, text) {
505
- if (!embedOnWrite || record.embedding !== undefined) return { record };
506
- const answer = await embedThrough([text]);
507
- if (answer.error !== undefined)
508
- return { record, embedError: `the embedder seam failed — ${answer.error}; stored un-embedded` };
509
- return { record: withEmbedding(record, answer.vectors[0], answer.identity) };
510
- }
511
-
512
- /** Read every value under a prefix, in key order. */
513
- async function readAll(prefix, view = storage) {
514
- const keys = await view.keys(prefix);
515
- return Promise.all(keys.map((key) => view.get(key)));
516
- }
517
-
518
- /**
519
- * An id for a record the caller did not name. Derived from the
520
- * timestamp and one past the highest sequence of that kind still
521
- * stored, so it is unique within a ledger and reproducible under an
522
- * injected clock. Called only from inside the write queue: the mint
523
- * and the store that follows it are one step, so no second writer can
524
- * read the same highest in between.
525
- */
526
- async function mintId(kind, prefix, view) {
527
- const counter = `ai/counters/${kind}`;
528
- const n = Math.max(await view.get(counter) ?? 0, nextSequence(await view.keys(prefix)));
529
- if (!Number.isSafeInteger(n) || n >= Number.MAX_SAFE_INTEGER) throw new RangeError('ledger id sequence exhausted');
530
- await view.set(counter, n + 1);
531
- return `${kind}-${now()}-${seq(n)}`;
532
- }
533
-
534
- //#region goal
535
-
536
- /**
537
- * Set the active objective. Singular by construction: an existing goal
538
- * is marked `superseded` and moved to the archive rather than
539
- * overwritten, so the history of what this agent was asked to do
540
- * survives the change.
541
- * @param {{ objective: string, createdAt?: string, status?: string,
542
- * progress?: any[] }} input
543
- * @returns {Promise<LedgerGoal | LedgerRejection>} the stored goal, or a rejection
544
- */
545
- function setGoal(input) {
546
- return enqueue(async (storage) => {
547
- let goal = defined({
548
- objective: input?.objective,
549
- createdAt: input?.createdAt ?? now(),
550
- status: input?.status ?? 'active',
551
- progress: input?.progress ?? [],
552
- });
553
- const rejected = validate('goal', goal);
554
- if (rejected !== null) return rejected;
555
-
556
- const bounded = await boundGoal(goal, storage);
557
- if (bounded.error) return bounded;
558
- goal = bounded;
559
- const previous = await storage.get(KEYS.goal);
560
- if (previous !== undefined) {
561
- const archived = await storage.keys(KEYS.goalArchive);
562
- await storage.set(`${KEYS.goalArchive}${seq(nextSequence(archived))}`,
563
- { ...previous, status: 'superseded' });
564
- }
565
- await storage.set(KEYS.goal, goal);
566
- return goal;
567
- }, { prefixes: [KEYS.goalArchive], keys: [KEYS.goal, 'ai/counters/progress'] });
568
- }
569
-
570
- /**
571
- * The active objective, or null.
572
- * @returns {Promise<LedgerGoal | null>}
573
- */
574
- async function getGoal() {
575
- return (await storage.get(KEYS.goal)) ?? null;
576
- }
577
-
578
- /**
579
- * Every superseded goal, oldest first.
580
- * @returns {Promise<LedgerGoal[]>}
581
- */
582
- async function listArchivedGoals() {
583
- return readAll(KEYS.goalArchive);
584
- }
585
-
586
- /**
587
- * Append one evidenced progress entry to the active goal. Storage
588
- * only: nothing here reads a note, scores it, or decides that a goal
589
- * is finished — that judgement belongs to whatever drives the agent.
590
- * @param {{ note: string, evidence: string, at?: string }} entry
591
- * @returns {Promise<LedgerGoal | LedgerRejection | { error: string }>}
592
- * the goal with the entry appended; the bare `{ error }` is "no
593
- * active goal", which is a state problem, not a validation one
594
- */
595
- function recordProgress(entry) {
596
- return enqueue(async (storage) => {
597
- const goal = await storage.get(KEYS.goal);
598
- if (goal === undefined) return { error: 'no active goal — call setGoal first' };
599
- let next = {
600
- ...goal,
601
- progress: [...goal.progress, defined({
602
- at: entry?.at ?? now(),
603
- note: entry?.note,
604
- evidence: entry?.evidence,
605
- })],
606
- };
607
- const rejected = validate('goal', next);
608
- if (rejected !== null) return rejected;
609
- next = await boundGoal(next, storage);
610
- if (next.error) return next;
611
- await storage.set(KEYS.goal, next);
612
- return next;
613
- }, { keys: [KEYS.goal, 'ai/counters/progress'] });
614
- }
615
-
616
- /**
617
- * Change the active goal's status without superseding it — how a run
618
- * records that its objective finished or was abandoned.
619
- * @param {'active'|'done'|'abandoned'|'superseded'} status
620
- * @returns {Promise<LedgerGoal | LedgerRejection | { error: string }>}
621
- */
622
- function setGoalStatus(status) {
623
- return enqueue(async (storage) => {
624
- const goal = await storage.get(KEYS.goal);
625
- if (goal === undefined) return { error: 'no active goal — call setGoal first' };
626
- const next = { ...goal, status };
627
- const rejected = validate('goal', next);
628
- if (rejected !== null) return rejected;
629
- await storage.set(KEYS.goal, next);
630
- return next;
631
- }, { keys: [KEYS.goal] });
632
- }
633
-
634
- /** Bound a goal only through validated, lossless coverage, or visibly refuse. */
635
- async function boundGoal(goal, view) {
636
- const limits = options.goalLimits;
637
- if (!limits) return goal;
638
- if (typeof storage.mutate !== 'function' && !isAtomicView(storage))
639
- return { error: 'goal budgets require atomic storage', code: 'ATOMIC_REQUIRED' };
640
- const next = JSON.parse(JSON.stringify(goal));
641
- let counter = await view.get('ai/counters/progress') ?? 0;
642
- const ids = new Set([...(next.checkpoint?.sources ?? []), ...next.progress].map((entry) => entry.id));
643
- for (const entry of next.progress) {
644
- if (entry.id !== undefined) continue;
645
- while (ids.has(`progress-${seq(counter)}`)) counter++;
646
- if (!Number.isSafeInteger(counter) || counter >= Number.MAX_SAFE_INTEGER) throw new RangeError('progress sequence exhausted');
647
- entry.id = `progress-${seq(counter++)}`;
648
- ids.add(entry.id);
419
+ /** Read every value under a prefix, in key order. */
420
+ async function readAll(prefix, view = storage) {
421
+ const keys = await view.keys(prefix);
422
+ return Promise.all(keys.map((key) => view.get(key)));
649
423
  }
650
- const fits = (candidate) => candidate.progress.length <= (limits.maxEntries ?? Infinity)
651
- && jsonBytes(candidate) <= (limits.maxBytes ?? Infinity)
652
- && goalPrompt(candidate).length <= (limits.maxChars ?? Infinity);
653
- if (!fits(next)) {
654
- let checkpoint;
655
- try { checkpoint = options.checkpointReducer ? options.checkpointReducer(JSON.parse(JSON.stringify(next))) : checkpointProgress(next); }
656
- catch (error) { return { error: `checkpoint reducer failed: ${error instanceof Error ? error.message : String(error)}`, code: 'GOAL_CHECKPOINT' }; }
657
- const checked = validateCheckpoint(checkpoint, next);
658
- if (!checked.valid) return { error: 'invalid goal checkpoint', errors: checked.errors, code: 'GOAL_CHECKPOINT' };
659
- const compacted = { ...next, progress: [], checkpoint,
660
- retention: { version: 1, reason: 'goal-budget', retired: next.progress.map((entry) => entry.id) } };
661
- const rejected = validate('goal', compacted);
662
- if (rejected) return rejected;
663
- if (!fits(compacted)) return { error: 'goal budget cannot preserve the objective and evidence; host action required',
664
- code: 'GOAL_BUDGET', retention: { refused: true, bytes: jsonBytes(compacted), chars: goalPrompt(compacted).length, limits } };
665
- await view.set('ai/counters/progress', counter);
666
- return compacted;
424
+ /**
425
+ * An id for a record the caller did not name. Derived from the
426
+ * timestamp and one past the highest sequence of that kind still
427
+ * stored, so it is unique within a ledger and reproducible under an
428
+ * injected clock. Called only from inside the write queue: the mint
429
+ * and the store that follows it are one step, so no second writer can
430
+ * read the same highest in between.
431
+ */
432
+ async function mintId(kind, prefix, view) {
433
+ const counter = `ai/counters/${kind}`;
434
+ const n = Math.max(await view.get(counter) ?? 0, nextSequence(await view.keys(prefix)));
435
+ if (!Number.isSafeInteger(n) || n >= Number.MAX_SAFE_INTEGER)
436
+ throw new RangeError('ledger id sequence exhausted');
437
+ await view.set(counter, n + 1);
438
+ return `${kind}-${now()}-${seq(n)}`;
667
439
  }
668
- await view.set('ai/counters/progress', counter);
669
- return next;
670
- }
671
-
672
- /** Render the complete goal under the host's hard character budget. */
673
- async function composeGoal() {
674
- const goal = await getGoal();
675
- if (!goal || goal.status !== 'active') return { text: '' };
676
- const rejected = validate('goal', goal);
677
- if (rejected) return rejected;
678
- const text = goalPrompt(goal);
679
- if (text.length > (options.goalLimits?.maxChars ?? Infinity))
680
- return { error: 'goal prompt exceeds its character budget; host action required', code: 'GOAL_BUDGET' };
681
- return { text };
682
- }
683
-
684
- //#endregion
685
-
686
- //#region memories and skills
687
-
688
- /**
689
- * Store a fact worth carrying. `evidence` is required by the schema,
690
- * and the rejection says so: a memory without it is a guess, and a
691
- * ledger of guesses is worse than an empty one.
692
- *
693
- * The record is EVERY member of the input plus the generated defaults,
694
- * validated whole so a member the schema does not know is refused
695
- * with the same `additionalProperties` answer `validate()` gives,
696
- * never silently dropped. A write that quietly stored less than it
697
- * was handed would let "it was accepted" and "it is there" diverge.
698
- * @param {{ id?: string, text: string, evidence: LedgerMemory['evidence'],
699
- * tags?: string[], at?: string } & LedgerEmbeddingPair} input
700
- * `embedding` + `embeddedBy` travel together (plain numbers, never a
701
- * typed array — the pair type says so, and an orphan does not
702
- * compile); the vector must be exactly `embeddedBy.dims` finite
703
- * numbers or the write is rejected
704
- * @returns {Promise<(LedgerMemory & { embedError?: string }) | LedgerRejection>}
705
- * the record as stored (defaults filled, undefined members
706
- * stripped), or the rejection saying why nothing was. `embedError`
707
- * appears only under `embedOnWrite` when the seam failed: the record
708
- * is stored WITHOUT a vector and the member is not part of it
709
- */
710
- function addMemory(input) {
711
- let embedded;
712
- return enqueue(async (storage) => {
713
- const memory = defined({
714
- ...input,
715
- id: input?.id ?? '(pending)',
716
- tags: input?.tags ?? [],
717
- at: input?.at ?? now(),
718
- });
719
- const rejected = validate('memory', memory);
720
- if (rejected !== null) return rejected;
721
- if (input?.id === undefined || input?.id === null) memory.id = await mintId('memory', KEYS.memory, storage);
722
- embedded ??= await embedOnWriteStep(memory, memory.text);
723
- const record = { ...embedded.record, id: memory.id, at: memory.at };
724
- const { embedError } = embedded;
725
- await storage.set(`${KEYS.memory}${record.id}`, record);
726
- return embedError === undefined ? record : { ...record, embedError };
727
- }, { prefixes: [KEYS.memory], keys: ['ai/counters/memory'] });
728
- }
729
-
730
- /**
731
- * One memory by id, or null.
732
- * @param {string} id
733
- * @returns {Promise<LedgerMemory | null>}
734
- */
735
- async function getMemory(id) {
736
- return (await storage.get(`${KEYS.memory}${id}`)) ?? null;
737
- }
738
-
739
- /**
740
- * Every memory, newest first.
741
- * @returns {Promise<LedgerMemory[]>}
742
- */
743
- async function listMemories() {
744
- return byRecency(await readAll(KEYS.memory));
745
- }
746
-
747
- /**
748
- * Remove a memory. Answers whether one was there.
749
- * @param {string} id
750
- */
751
- function deleteMemory(id) {
752
- return enqueue(async (storage) => {
753
- const key = `${KEYS.memory}${id}`;
754
- const existed = (await storage.get(key)) !== undefined;
755
- await storage.delete(key);
756
- return existed;
757
- }, { keys: [`${KEYS.memory}${id}`] });
758
- }
759
-
760
- /**
761
- * Store a reusable recipe. Built and validated exactly as a memory is:
762
- * every input member, defaults filled, the whole record checked.
763
- * @param {{ id?: string, name: string, when: string,
764
- * instructions: string, tools?: string[], at?: string, program?: LedgerSkill['program'] }
765
- * & LedgerEmbeddingPair} input
766
- * @returns {Promise<(LedgerSkill & { embedError?: string }) | LedgerRejection>}
767
- * as `addMemory`; under `embedOnWrite` the skill is embedded from its
768
- * name, when and instructions
769
- */
770
- function addSkill(input) {
771
- let embedded;
772
- return enqueue(async (storage) => {
773
- const skill = defined({
774
- ...input,
775
- id: input?.id ?? '(pending)',
776
- tools: input?.tools ?? [],
777
- at: input?.at ?? now(),
778
- });
779
- const rejected = validate('skill', skill);
780
- if (rejected !== null) return rejected;
781
- if (input?.id === undefined || input?.id === null) skill.id = await mintId('skill', KEYS.skill, storage);
782
- embedded ??= await embedOnWriteStep(skill, skillText(skill));
783
- const record = { ...embedded.record, id: skill.id, at: skill.at };
784
- const { embedError } = embedded;
785
- await storage.set(`${KEYS.skill}${record.id}`, record);
786
- return embedError === undefined ? record : { ...record, embedError };
787
- }, { prefixes: [KEYS.skill], keys: ['ai/counters/skill'] });
788
- }
789
-
790
- /**
791
- * One skill by id, or null.
792
- * @param {string} id
793
- * @returns {Promise<LedgerSkill | null>}
794
- */
795
- async function getSkill(id) {
796
- return (await storage.get(`${KEYS.skill}${id}`)) ?? null;
797
- }
798
-
799
- /**
800
- * Every skill, newest first.
801
- * @returns {Promise<LedgerSkill[]>}
802
- */
803
- async function listSkills() {
804
- return byRecency(await readAll(KEYS.skill));
805
- }
806
-
807
- /**
808
- * Remove a skill. Answers whether one was there.
809
- * @param {string} id
810
- */
811
- function deleteSkill(id) {
812
- return enqueue(async (storage) => {
813
- const key = `${KEYS.skill}${id}`;
814
- const existed = (await storage.get(key)) !== undefined;
815
- await storage.delete(key);
816
- return existed;
817
- }, { keys: [`${KEYS.skill}${id}`] });
818
- }
819
-
820
- //#endregion
821
-
822
- //#region retrieval
823
-
824
- /**
825
- * The tag predicate as a query document. Jaren's comparison operators
826
- * are existentially lifted over sequences, so `$eq` between the record's
827
- * tag sequence and the requested one is exactly "the two overlap" —
828
- * one operator, no loop, and the same meaning as the fallback's
829
- * `some`/`includes`.
830
- * @param {string[]} tags
831
- */
832
- const tagPredicate = (tags) => ({ $eq: ['$it.tags[*]', { $seq: tags }] });
833
-
834
- /** Compile once per distinct query document. */
835
- function compiled(document) {
836
- const key = JSON.stringify(document);
837
- let run = queryCache.get(key);
838
- if (run === undefined) {
839
- run = compileQuery(document);
840
- queryCache.set(key, run);
440
+ //#region goal
441
+ /**
442
+ * Set the active objective. Singular by construction: an existing goal
443
+ * is marked `superseded` and moved to the archive rather than
444
+ * overwritten, so the history of what this agent was asked to do
445
+ * survives the change.
446
+ * @returns the stored goal, or a rejection
447
+ */
448
+ function setGoal(input) {
449
+ return enqueue(async (storage) => {
450
+ let goal = defined({
451
+ objective: input?.objective,
452
+ createdAt: input?.createdAt ?? now(),
453
+ status: input?.status ?? 'active',
454
+ progress: input?.progress ?? [],
455
+ });
456
+ const rejected = validate('goal', goal);
457
+ if (rejected !== null)
458
+ return rejected;
459
+ const bounded = await boundGoal(goal, storage);
460
+ if (bounded.error)
461
+ return bounded;
462
+ goal = bounded;
463
+ const previous = await storage.get(KEYS.goal);
464
+ if (previous !== undefined) {
465
+ const archived = await storage.keys(KEYS.goalArchive);
466
+ await storage.set(`${KEYS.goalArchive}${seq(nextSequence(archived))}`, { ...previous, status: 'superseded' });
467
+ }
468
+ await storage.set(KEYS.goal, goal);
469
+ return goal;
470
+ }, { prefixes: [KEYS.goalArchive], keys: [KEYS.goal, 'ai/counters/progress'] });
841
471
  }
842
- return run;
843
- }
844
-
845
- /**
846
- * Filter a record set. With the query seam wired this is a real jaren
847
- * query — one document, which a host that also wired `@jarenjs/db`
848
- * could push down to SQL unchanged. With the seam empty it is tag
849
- * match, and a caller predicate is REFUSED rather than ignored:
850
- * silently dropping a filter would answer the wrong question with a
851
- * straight face.
852
- */
853
- function filter(records, tags, where) {
854
- const parts = [];
855
- if (tags !== undefined && tags.length > 0) parts.push(tagPredicate(tags));
856
- if (where !== undefined && where !== null) parts.push(where);
857
-
858
- if (compileQuery === null) {
859
- if (where !== undefined && where !== null) {
860
- return { error: 'recall: a `where` predicate needs the compileQuery seam — '
861
- + 'inject compileJsonQuery from @jarenjs/json/query, or filter by tags only' };
862
- }
863
- if (parts.length === 0) return { value: records };
864
- return { value: records.filter((record) =>
865
- (record.tags ?? []).some((tag) => tags.includes(tag))) };
472
+ /**
473
+ * The active objective, or null.
474
+ */
475
+ async function getGoal() {
476
+ return (await storage.get(KEYS.goal)) ?? null;
866
477
  }
867
-
868
- if (parts.length === 0) return { value: records };
869
- const predicate = parts.length === 1 ? parts[0] : { $and: parts };
870
- try {
871
- const run = compiled([{ $for: { it: '$[*]' }, $where: predicate, $return: '$it' }]);
872
- return { value: run(records) };
478
+ /**
479
+ * Every superseded goal, oldest first.
480
+ */
481
+ async function listArchivedGoals() {
482
+ return readAll(KEYS.goalArchive);
873
483
  }
874
- catch (err) {
875
- // a caller's predicate that does not compile is content, not a
876
- // crash same posture as a tool call the model got wrong
877
- return { error: `recall: ${/** @type {any} */ (err)?.reason ?? /** @type {any} */ (err)?.message ?? err}` };
484
+ /**
485
+ * Append one evidenced progress entry to the active goal. Storage
486
+ * only: nothing here reads a note, scores it, or decides that a goal
487
+ * is finished that judgement belongs to whatever drives the agent.
488
+ * @returns
489
+ * the goal with the entry appended; the bare `{ error }` is "no
490
+ * active goal", which is a state problem, not a validation one
491
+ */
492
+ function recordProgress(entry) {
493
+ return enqueue(async (storage) => {
494
+ const goal = await storage.get(KEYS.goal);
495
+ if (goal === undefined)
496
+ return { error: 'no active goal — call setGoal first' };
497
+ let next = {
498
+ ...goal,
499
+ progress: [...goal.progress, defined({
500
+ at: entry?.at ?? now(),
501
+ note: entry?.note,
502
+ evidence: entry?.evidence,
503
+ })],
504
+ };
505
+ const rejected = validate('goal', next);
506
+ if (rejected !== null)
507
+ return rejected;
508
+ next = await boundGoal(next, storage);
509
+ if (next.error)
510
+ return next;
511
+ await storage.set(KEYS.goal, next);
512
+ return next;
513
+ }, { keys: [KEYS.goal, 'ai/counters/progress'] });
878
514
  }
879
- }
880
-
881
- /**
882
- * Retrieve by relevance: tag match, then recency, plus any predicate
883
- * the seam can evaluate.
884
- * @param {any[]} records
885
- * @param {LedgerQuery} query
886
- */
887
- function retrieve(records, query = {}) {
888
- const filtered = filter(records, query.tags, query.where);
889
- if (filtered.error !== undefined) return filtered;
890
- const ordered = byRecency(filtered.value);
891
- return typeof query.limit === 'number' ? ordered.slice(0, query.limit) : ordered;
892
- }
893
-
894
- /**
895
- * The mixture refusal, in one place, so the sweep and an adapter that
896
- * ranks for us word it identically and in the same order.
897
- * @param {string} name
898
- * @param {Map<string, LedgerEmbeddedBy>} found - every identity seen, in the order seen
899
- * @param {LedgerEmbeddedBy} identity - the query embedder's
900
- * @returns {{ error: string } | null}
901
- */
902
- function refuseMixture(name, found, identity) {
903
- if ([...found.values()].every((stored) => sameIdentity(stored, identity))) return null;
904
- return { error: `${name}: near cannot rank across embedders — the ledger holds vectors from`
905
- + ` ${[...found.keys()].join(', ')} and the query embedder is ${describeIdentity(identity)};`
906
- + ' rank through the embedder that wrote them, or re-embed every record with one embedder' };
907
- }
908
-
909
- /**
910
- * The last two steps both ranked paths share: the kernels score every
911
- * record the path selected, `minScore` filters and `limit` caps.
912
- *
913
- * The score is always `cosineSimilarity` over the record as stored —
914
- * never a number an adapter handed back — so the two paths cannot
915
- * report different similarities for the same record. Equal scores
916
- * fall back to recency, then id.
917
- * @param {any[]} records
918
- * @param {any} probe
919
- * @param {LedgerQuery} query
920
- * @param {'memories' | 'skills'} member
921
- * @param {number} skipped
922
- * @param {'sweep' | 'adapter'} via
923
- * @param {LedgerRanking} [ranking]
924
- */
925
- function rankedResult(records, probe, query, member, skipped, via,
926
- ranking = { algorithm: 'exact-cosine', exhaustive: true, candidateCount: records.length }) {
927
- const ranked = records
928
- .map((record) => ({ record, score: cosineSimilarity(probe, record.embedding) }))
929
- .sort((a, b) => (b.score - a.score) || recencyOrder(a.record, b.record));
930
- const kept = typeof query.minScore === 'number'
931
- ? ranked.filter((entry) => entry.score >= /** @type {number} */ (query.minScore))
932
- : ranked;
933
- const capped = typeof query.limit === 'number' ? kept.slice(0, query.limit) : kept;
934
- return {
935
- [member]: capped.map((entry) => entry.record),
936
- scores: capped.map((entry) => entry.score),
937
- skipped,
938
- via,
939
- ranking,
940
- };
941
- }
942
-
943
- /**
944
- * Retrieve by meaning: the records that pass the tag/where filter AND
945
- * carry a vector, ranked by cosine similarity to the embedded query.
946
- *
947
- * The refusals come before the arithmetic, in this order: `near` must
948
- * be a non-empty string; the seam must be wired (the `compileQuery`
949
- * precedent — an absent capability refuses, it never degrades); the
950
- * filter must compile; the seam must answer; and every candidate's
951
- * identity must equal the query embedder's. A mixture — two models in
952
- * the ledger, or a ledger embedded by one model and queried through
953
- * another is refused naming every identity found, because ranking
954
- * the matching subset would be a silent wrong answer. Records without
955
- * a vector are not scored (a fabricated score poisons a ranking) and
956
- * not hidden (a silent drop poisons trust): they are counted in
957
- * `skipped`.
958
- *
959
- * TWO paths reach that answer and both report which one ran. By
960
- * default the ledger reads every record under the prefix and ranks
961
- * them here. An adapter that declares `rank` is asked instead — it
962
- * knows how to narrow to the k best without handing over the whole
963
- * ledger — and is held to exactly the same contract: it reports the
964
- * distinct identities it holds so the mixture refusal is the ledger's,
965
- * it reports what it skipped, and the kernels re-score what it
966
- * returns. An adapter whose own ranking is approximate makes recall
967
- * approximate: the ledger can re-score what it is handed, never
968
- * recover a record the adapter did not return.
969
- *
970
- * `tags` and `where` narrow candidates the adapter knows nothing
971
- * about, so a query carrying either takes the sweep.
972
- *
973
- * Ranking is `cosineSimilarity` from the kernels, descending; equal
974
- * scores fall back to recency, then id, so the order is deterministic.
975
- * `minScore` filters the ranked list; `limit` caps what survives.
976
- * @param {LedgerQuery} query
977
- * @param {'recall' | 'recallSkills'} name - the entry point, for the refusal text
978
- * @param {'memories' | 'skills'} member - the result member the ranked records go under
979
- * @param {string} prefix - the key space the records live under
980
- * @returns {Promise<{ error: string }
981
- * | { [member: string]: any, scores: number[], skipped: number, via: 'sweep' | 'adapter' }>}
982
- */
983
- async function rankNear(query, name, member, prefix) {
984
- const { near } = query;
985
- if (typeof near !== 'string' || near === '')
986
- return { error: `${name}: near must be a non-empty string — the text to rank by meaning against` };
987
- if (embedder === null) return { error: `${name}: near ${SEAM_REFUSAL}` };
988
-
989
- const narrowed = query.tags !== undefined || query.where !== undefined;
990
- const delegated = typeof storage.rank === 'function' && !narrowed;
991
- // the sweep reads first so that a `where` without the compileQuery
992
- // seam still refuses before the embedder is called; the adapter path
993
- // has no filter to compile
994
- const filtered = delegated ? null : filter(await readAll(prefix), query.tags, query.where);
995
- if (filtered !== null && filtered.error !== undefined) return filtered;
996
-
997
- const answer = await embedThrough([near]);
998
- if (answer.error !== undefined) return { error: `${name}: the embedder seam failed — ${answer.error}` };
999
- const probe = answer.vectors[0];
1000
- const identity = answer.identity;
1001
-
1002
- if (delegated) {
1003
- const hits = await /** @type {any} */ (storage).rank({
1004
- prefix, vector: Array.from(probe), model: identity.model, dims: identity.dims,
1005
- limit: query.limit, minScore: query.minScore,
1006
- });
1007
- if (hits === null || typeof hits !== 'object' || !Array.isArray(hits.hits)
1008
- || !Array.isArray(hits.identities) || !Number.isSafeInteger(hits.skipped) || hits.skipped < 0) {
1009
- return { error: `${name}: the storage adapter's rank answered something other than`
1010
- + ' { hits, skipped, identities } — a capability that cannot be trusted to report what it'
1011
- + ' skipped is worse than one that is absent' };
1012
- }
1013
- const ranking = hits.ranking === undefined ? { algorithm: 'legacy-exact', exhaustive: true, candidateCount: hits.hits.length } : hits.ranking;
1014
- if (ranking === null || typeof ranking !== 'object' || typeof ranking.algorithm !== 'string' || ranking.algorithm.trim() === ''
1015
- || typeof ranking.exhaustive !== 'boolean' || ranking.candidateCount !== hits.hits.length)
1016
- return { error: `${name}: invalid storage rank metadata` };
1017
- const keys = new Set();
1018
- for (const hit of hits.hits) {
1019
- if (typeof hit?.key !== 'string' || !hit.key.startsWith(prefix) || hit.key.length === prefix.length
1020
- || keys.has(hit.key) || typeof hit.score !== 'number' || !Number.isFinite(hit.score))
1021
- return { error: `${name}: invalid or duplicate storage rank candidate` };
1022
- keys.add(hit.key);
1023
- }
1024
- /** @type {Map<string, LedgerEmbeddedBy>} */
1025
- const found = new Map();
1026
- for (const stored of hits.identities) {
1027
- if (typeof stored?.model !== 'string' || stored.model === '' || !Number.isSafeInteger(stored.dims) || stored.dims < 1)
1028
- return { error: `${name}: invalid storage rank identity` };
1029
- found.set(describeIdentity(stored), stored);
1030
- }
1031
- const records = await Promise.all(hits.hits.map((/** @type {any} */ hit) => storage.get(hit.key)));
1032
- for (let i = 0; i < records.length; i++) {
1033
- const record = records[i];
1034
- if (record == null) continue; // Deleted between selection and read.
1035
- if (`${prefix}${record.id}` !== hits.hits[i].key || !sameIdentity(record.embeddedBy, identity))
1036
- return { error: `${name}: storage rank candidate identity mismatch` };
1037
- if (!isVector(record.embedding, identity.dims))
1038
- return { error: `${name}: invalid storage rank candidate vector` };
1039
- found.set(describeIdentity(record.embeddedBy), record.embeddedBy);
1040
- }
1041
- const refused = refuseMixture(name, found, identity);
1042
- if (refused !== null) return refused;
1043
- // a key the adapter ranked and a read that no longer finds it is a
1044
- // record deleted in between, not a record without a vector
1045
- return rankedResult(records.filter((record) => record?.embedding !== undefined),
1046
- probe, query, member, hits.skipped, 'adapter',
1047
- { algorithm: ranking.algorithm, exhaustive: ranking.exhaustive, candidateCount: ranking.candidateCount });
515
+ /**
516
+ * Change the active goal's status without superseding it — how a run
517
+ * records that its objective finished or was abandoned.
518
+ */
519
+ function setGoalStatus(status) {
520
+ return enqueue(async (storage) => {
521
+ const goal = await storage.get(KEYS.goal);
522
+ if (goal === undefined)
523
+ return { error: 'no active goal — call setGoal first' };
524
+ const next = { ...goal, status };
525
+ const rejected = validate('goal', next);
526
+ if (rejected !== null)
527
+ return rejected;
528
+ await storage.set(KEYS.goal, next);
529
+ return next;
530
+ }, { keys: [KEYS.goal] });
531
+ }
532
+ /** Bound a goal only through validated, lossless coverage, or visibly refuse. */
533
+ async function boundGoal(goal, view) {
534
+ const limits = options.goalLimits;
535
+ if (!limits)
536
+ return goal;
537
+ if (typeof storage.mutate !== 'function' && !isAtomicView(storage))
538
+ return { error: 'goal budgets require atomic storage', code: 'ATOMIC_REQUIRED' };
539
+ const next = JSON.parse(JSON.stringify(goal));
540
+ let counter = await view.get('ai/counters/progress') ?? 0;
541
+ const ids = new Set([...(next.checkpoint?.sources ?? []), ...next.progress].map((entry) => entry.id));
542
+ for (const entry of next.progress) {
543
+ if (entry.id !== undefined)
544
+ continue;
545
+ while (ids.has(`progress-${seq(counter)}`))
546
+ counter++;
547
+ if (!Number.isSafeInteger(counter) || counter >= Number.MAX_SAFE_INTEGER)
548
+ throw new RangeError('progress sequence exhausted');
549
+ entry.id = `progress-${seq(counter++)}`;
550
+ ids.add(entry.id);
551
+ }
552
+ const fits = (candidate) => candidate.progress.length <= (limits.maxEntries ?? Infinity)
553
+ && jsonBytes(candidate) <= (limits.maxBytes ?? Infinity)
554
+ && goalPrompt(candidate).length <= (limits.maxChars ?? Infinity);
555
+ if (!fits(next)) {
556
+ let checkpoint;
557
+ try {
558
+ checkpoint = options.checkpointReducer ? options.checkpointReducer(JSON.parse(JSON.stringify(next))) : checkpointProgress(next);
559
+ }
560
+ catch (error) {
561
+ return { error: `checkpoint reducer failed: ${error instanceof Error ? error.message : String(error)}`, code: 'GOAL_CHECKPOINT' };
562
+ }
563
+ const checked = validateCheckpoint(checkpoint, next);
564
+ if (!checked.valid)
565
+ return { error: 'invalid goal checkpoint', errors: checked.errors, code: 'GOAL_CHECKPOINT' };
566
+ const compacted = {
567
+ ...next, progress: [], checkpoint,
568
+ retention: { version: 1, reason: 'goal-budget', retired: next.progress.map((entry) => entry.id) }
569
+ };
570
+ const rejected = validate('goal', compacted);
571
+ if (rejected)
572
+ return rejected;
573
+ if (!fits(compacted))
574
+ return {
575
+ error: 'goal budget cannot preserve the objective and evidence; host action required',
576
+ code: 'GOAL_BUDGET', retention: { refused: true, bytes: jsonBytes(compacted), chars: goalPrompt(compacted).length, limits }
577
+ };
578
+ await view.set('ai/counters/progress', counter);
579
+ return compacted;
580
+ }
581
+ await view.set('ai/counters/progress', counter);
582
+ return next;
583
+ }
584
+ /** Render the complete goal under the host's hard character budget. */
585
+ async function composeGoal() {
586
+ const goal = await getGoal();
587
+ if (!goal || goal.status !== 'active')
588
+ return { text: '' };
589
+ const rejected = validate('goal', goal);
590
+ if (rejected)
591
+ return rejected;
592
+ const text = goalPrompt(goal);
593
+ if (text.length > (options.goalLimits?.maxChars ?? Infinity))
594
+ return { error: 'goal prompt exceeds its character budget; host action required', code: 'GOAL_BUDGET' };
595
+ return { text };
596
+ }
597
+ //#endregion
598
+ //#region memories and skills
599
+ /**
600
+ * Store a fact worth carrying. `evidence` is required by the schema,
601
+ * and the rejection says so: a memory without it is a guess, and a
602
+ * ledger of guesses is worse than an empty one.
603
+ *
604
+ * The record is EVERY member of the input plus the generated defaults,
605
+ * validated whole — so a member the schema does not know is refused
606
+ * with the same `additionalProperties` answer `validate()` gives,
607
+ * never silently dropped. A write that quietly stored less than it
608
+ * was handed would let "it was accepted" and "it is there" diverge.
609
+ * `embedding` + `embeddedBy` travel together (plain numbers, never a
610
+ * typed array the pair type says so, and an orphan does not
611
+ * compile); the vector must be exactly `embeddedBy.dims` finite
612
+ * numbers or the write is rejected
613
+ * @returns
614
+ * the record as stored (defaults filled, undefined members
615
+ * stripped), or the rejection saying why nothing was. `embedError`
616
+ * appears only under `embedOnWrite` when the seam failed: the record
617
+ * is stored WITHOUT a vector and the member is not part of it
618
+ */
619
+ function addMemory(input) {
620
+ let embedded;
621
+ return enqueue(async (storage) => {
622
+ const memory = defined({
623
+ ...input,
624
+ id: input?.id ?? '(pending)',
625
+ tags: input?.tags ?? [],
626
+ at: input?.at ?? now(),
627
+ });
628
+ const rejected = validate('memory', memory);
629
+ if (rejected !== null)
630
+ return rejected;
631
+ if (input?.id === undefined || input?.id === null)
632
+ memory.id = await mintId('memory', KEYS.memory, storage);
633
+ embedded ??= await embedOnWriteStep(memory, memory.text);
634
+ const record = { ...embedded.record, id: memory.id, at: memory.at };
635
+ const { embedError } = embedded;
636
+ await storage.set(`${KEYS.memory}${record.id}`, record);
637
+ return embedError === undefined ? record : { ...record, embedError };
638
+ }, { prefixes: [KEYS.memory], keys: ['ai/counters/memory'] });
639
+ }
640
+ /**
641
+ * One memory by id, or null.
642
+ */
643
+ async function getMemory(id) {
644
+ return (await storage.get(`${KEYS.memory}${id}`)) ?? null;
645
+ }
646
+ /**
647
+ * Every memory, newest first.
648
+ */
649
+ async function listMemories() {
650
+ return byRecency(await readAll(KEYS.memory));
651
+ }
652
+ /**
653
+ * Remove a memory. Answers whether one was there.
654
+ */
655
+ function deleteMemory(id) {
656
+ return enqueue(async (storage) => {
657
+ const key = `${KEYS.memory}${id}`;
658
+ const existed = (await storage.get(key)) !== undefined;
659
+ await storage.delete(key);
660
+ return existed;
661
+ }, { keys: [`${KEYS.memory}${id}`] });
662
+ }
663
+ /**
664
+ * Store a reusable recipe. Built and validated exactly as a memory is:
665
+ * every input member, defaults filled, the whole record checked.
666
+ * @returns
667
+ * as `addMemory`; under `embedOnWrite` the skill is embedded from its
668
+ * name, when and instructions
669
+ */
670
+ function addSkill(input) {
671
+ let embedded;
672
+ return enqueue(async (storage) => {
673
+ const skill = defined({
674
+ ...input,
675
+ id: input?.id ?? '(pending)',
676
+ tools: input?.tools ?? [],
677
+ at: input?.at ?? now(),
678
+ });
679
+ const rejected = validate('skill', skill);
680
+ if (rejected !== null)
681
+ return rejected;
682
+ if (input?.id === undefined || input?.id === null)
683
+ skill.id = await mintId('skill', KEYS.skill, storage);
684
+ embedded ??= await embedOnWriteStep(skill, skillText(skill));
685
+ const record = { ...embedded.record, id: skill.id, at: skill.at };
686
+ const { embedError } = embedded;
687
+ await storage.set(`${KEYS.skill}${record.id}`, record);
688
+ return embedError === undefined ? record : { ...record, embedError };
689
+ }, { prefixes: [KEYS.skill], keys: ['ai/counters/skill'] });
690
+ }
691
+ /**
692
+ * One skill by id, or null.
693
+ */
694
+ async function getSkill(id) {
695
+ return (await storage.get(`${KEYS.skill}${id}`)) ?? null;
1048
696
  }
1049
-
1050
- // the identity check, before any math: every candidate's identity
1051
- // must be the query's, or nothing is ranked
1052
- /** @type {any[]} */
1053
- const candidates = [];
1054
- /** @type {Map<string, LedgerEmbeddedBy>} */
1055
- const found = new Map();
1056
- let skipped = 0;
1057
- for (const record of /** @type {any} */ (filtered).value) {
1058
- if (record.embedding === undefined) {
1059
- skipped += 1;
1060
- continue;
1061
- }
1062
- found.set(describeIdentity(record.embeddedBy), record.embeddedBy);
1063
- candidates.push(record);
697
+ /**
698
+ * Every skill, newest first.
699
+ */
700
+ async function listSkills() {
701
+ return byRecency(await readAll(KEYS.skill));
1064
702
  }
1065
- const refused = refuseMixture(name, found, identity);
1066
- if (refused !== null) return refused;
1067
- return rankedResult(candidates, probe, query, member, skipped, 'sweep');
1068
- }
1069
-
1070
- /**
1071
- * Memories relevant to a query. Without `near`: tag match, then
1072
- * recency — an array, newest first. With `near`: ranked by meaning
1073
- * through the embedder seam — `{ memories, scores, skipped, via }` — or
1074
- * the refusal that says why not.
1075
- * @param {LedgerQuery} [query]
1076
- * @returns {Promise<LedgerMemory[] | LedgerRankedMemories | { error: string }>}
1077
- * the `{ error }` is a query problem — no seam for a `where` or a
1078
- * `near`, a predicate that does not compile, a seam that failed, or
1079
- * a mixture of vector identities content, not a crash
1080
- */
1081
- async function recall(query = {}) {
1082
- return query.near === undefined
1083
- ? retrieve(await readAll(KEYS.memory), query)
1084
- : /** @type {any} */ (rankNear(query, 'recall', 'memories', KEYS.memory));
1085
- }
1086
-
1087
- /**
1088
- * Skills relevant to a query — what a host composes into a system
1089
- * prompt. The same two paths as `recall`; a skill's meaning is its
1090
- * name, when and instructions together.
1091
- * @param {LedgerQuery} [query]
1092
- * @returns {Promise<LedgerSkill[] | LedgerRankedSkills | { error: string }>}
1093
- */
1094
- async function recallSkills(query = {}) {
1095
- return query.near === undefined
1096
- ? retrieve(await readAll(KEYS.skill), query)
1097
- : /** @type {any} */ (rankNear(query, 'recallSkills', 'skills', KEYS.skill));
1098
- }
1099
-
1100
- /**
1101
- * The explicit sweep: embed every memory and skill that carries no
1102
- * vector, through the seam, in batches, inside the write chain — so no
1103
- * other write moves a record under it. Memories first, then skills,
1104
- * each in key order; `limit` caps how many records this run embeds,
1105
- * `batch` how many texts one seam call carries. Positive fractional batch sizes
1106
- * are floored to an integer of at least one.
1107
- *
1108
- * Honest about what it did not do. A second run over a swept ledger
1109
- * embeds zero and makes no seam call (the two-run check). A failing
1110
- * batch ends the run: the records embedded before it are written, the
1111
- * failed batch and everything after it stay un-embedded and are
1112
- * counted in `remaining`, and the error is surfaced once — never a
1113
- * throw that loses the batch, and never a per-record retry against a
1114
- * provider that just refused (the seam's own retry policy has already
1115
- * run). A ledger that already holds vectors under another identity is
1116
- * refused up front rather than turned into a mixture `recall({ near })`
1117
- * would then refuse.
1118
- * @param {{ limit?: number, batch?: number }} [options]
1119
- * @returns {Promise<{ embedded: number, remaining: number, error?: string }>}
1120
- */
1121
- function embedMissing(options = {}) {
1122
- const batches = new Map();
1123
- return enqueue(async (storage) => {
1124
- const batch = typeof options.batch === 'number' && options.batch > 0 ? Math.max(1, Math.floor(options.batch)) : EMBED_BATCH;
1125
- /** @type {Array<{ key: string, record: any, text: string }>} */
1126
- const pending = [];
1127
- /** @type {Map<string, LedgerEmbeddedBy>} */
1128
- const held = new Map();
1129
- for (const memory of await readAll(KEYS.memory, storage)) {
1130
- if (memory.embedding === undefined) pending.push({ key: `${KEYS.memory}${memory.id}`, record: memory, text: memory.text });
1131
- else held.set(describeIdentity(memory.embeddedBy), memory.embeddedBy);
1132
- }
1133
- for (const skill of await readAll(KEYS.skill, storage)) {
1134
- if (skill.embedding === undefined) pending.push({ key: `${KEYS.skill}${skill.id}`, record: skill, text: skillText(skill) });
1135
- else held.set(describeIdentity(skill.embeddedBy), skill.embeddedBy);
1136
- }
1137
- const total = pending.length;
1138
- if (embedder === null) return { embedded: 0, remaining: total, error: `embedMissing: ${SEAM_REFUSAL}` };
1139
-
1140
- /** The mixture refusal, once the embedder's width is known. */
1141
- const mixture = (/** @type {LedgerEmbeddedBy} */ identity) => {
1142
- const foreign = [...held.keys()].filter((key) => !sameIdentity(held.get(key), identity));
1143
- return foreign.length === 0 ? null
1144
- : { embedded: 0, remaining: total, error: 'embedMissing: would mix vector identities — the ledger'
1145
- + ` already holds vectors from ${foreign.join(', ')} and the embedder is ${describeIdentity(identity)};`
1146
- + ' sweep with the embedder that wrote them, or re-add those records without a vector first' };
1147
- };
1148
- if (embedder.dims !== undefined) {
1149
- const refused = mixture({ model: embedder.model, dims: embedder.dims });
1150
- if (refused !== null) return refused;
1151
- }
1152
-
1153
- const todo = typeof options.limit === 'number' ? pending.slice(0, Math.max(0, Math.floor(options.limit))) : pending;
1154
- let embedded = 0;
1155
- for (let i = 0; i < todo.length; i += batch) {
1156
- const slice = todo.slice(i, i + batch);
1157
- const texts = slice.map((entry) => entry.text), key = JSON.stringify(texts);
1158
- if (!batches.has(key)) batches.set(key, await embedThrough(texts));
1159
- const answer = batches.get(key);
703
+ /**
704
+ * Remove a skill. Answers whether one was there.
705
+ */
706
+ function deleteSkill(id) {
707
+ return enqueue(async (storage) => {
708
+ const key = `${KEYS.skill}${id}`;
709
+ const existed = (await storage.get(key)) !== undefined;
710
+ await storage.delete(key);
711
+ return existed;
712
+ }, { keys: [`${KEYS.skill}${id}`] });
713
+ }
714
+ //#endregion
715
+ //#region retrieval
716
+ /**
717
+ * The tag predicate as a query document. Jaren's comparison operators
718
+ * are existentially lifted over sequences, so `$eq` between the record's
719
+ * tag sequence and the requested one is exactly "the two overlap" —
720
+ * one operator, no loop, and the same meaning as the fallback's
721
+ * `some`/`includes`.
722
+ */
723
+ const tagPredicate = (tags) => ({ $eq: ['$it.tags[*]', { $seq: tags }] });
724
+ /** Compile once per distinct query document. */
725
+ function compiled(document) {
726
+ const key = JSON.stringify(document);
727
+ let run = queryCache.get(key);
728
+ if (run === undefined) {
729
+ run = compileQuery(document);
730
+ queryCache.set(key, run);
731
+ }
732
+ return run;
733
+ }
734
+ /**
735
+ * Filter a record set. With the query seam wired this is a real jaren
736
+ * query — one document, which a host that also wired `@jarenjs/db`
737
+ * could push down to SQL unchanged. With the seam empty it is tag
738
+ * match, and a caller predicate is REFUSED rather than ignored:
739
+ * silently dropping a filter would answer the wrong question with a
740
+ * straight face.
741
+ */
742
+ function filter(records, tags, where) {
743
+ const parts = [];
744
+ if (tags !== undefined && tags.length > 0)
745
+ parts.push(tagPredicate(tags));
746
+ if (where !== undefined && where !== null)
747
+ parts.push(where);
748
+ if (compileQuery === null) {
749
+ if (where !== undefined && where !== null) {
750
+ return {
751
+ error: 'recall: a `where` predicate needs the compileQuery seam '
752
+ + 'inject compileJsonQuery from @jarenjs/json/query, or filter by tags only'
753
+ };
754
+ }
755
+ if (parts.length === 0)
756
+ return { value: records };
757
+ return {
758
+ value: records.filter((record) => (record.tags ?? []).some((tag) => tags.includes(tag)))
759
+ };
760
+ }
761
+ if (parts.length === 0)
762
+ return { value: records };
763
+ const predicate = parts.length === 1 ? parts[0] : { $and: parts };
764
+ try {
765
+ const run = compiled([{ $for: { it: '$[*]' }, $where: predicate, $return: '$it' }]);
766
+ return { value: run(records) };
767
+ }
768
+ catch (err) {
769
+ // a caller's predicate that does not compile is content, not a
770
+ // crash — same posture as a tool call the model got wrong
771
+ return { error: `recall: ${err?.reason ?? err?.message ?? err}` };
772
+ }
773
+ }
774
+ /**
775
+ * Retrieve by relevance: tag match, then recency, plus any predicate
776
+ * the seam can evaluate.
777
+ */
778
+ function retrieve(records, query = {}) {
779
+ const filtered = filter(records, query.tags, query.where);
780
+ if (filtered.error !== undefined)
781
+ return filtered;
782
+ const ordered = byRecency(filtered.value);
783
+ return typeof query.limit === 'number' ? ordered.slice(0, query.limit) : ordered;
784
+ }
785
+ /**
786
+ * The mixture refusal, in one place, so the sweep and an adapter that
787
+ * ranks for us word it identically and in the same order.
788
+ * @param found - every identity seen, in the order seen
789
+ * @param identity - the query embedder's
790
+ */
791
+ function refuseMixture(name, found, identity) {
792
+ if ([...found.values()].every((stored) => sameIdentity(stored, identity)))
793
+ return null;
794
+ return {
795
+ error: `${name}: near cannot rank across embedders the ledger holds vectors from`
796
+ + ` ${[...found.keys()].join(', ')} and the query embedder is ${describeIdentity(identity)};`
797
+ + ' rank through the embedder that wrote them, or re-embed every record with one embedder'
798
+ };
799
+ }
800
+ /**
801
+ * The last two steps both ranked paths share: the kernels score every
802
+ * record the path selected, `minScore` filters and `limit` caps.
803
+ *
804
+ * The score is always `cosineSimilarity` over the record as stored —
805
+ * never a number an adapter handed back — so the two paths cannot
806
+ * report different similarities for the same record. Equal scores
807
+ * fall back to recency, then id.
808
+ * @param [ranking]
809
+ */
810
+ function rankedResult(records, probe, query, member, skipped, via, ranking = { algorithm: 'exact-cosine', exhaustive: true, candidateCount: records.length }) {
811
+ const ranked = records
812
+ .map((record) => ({ record, score: cosineSimilarity(probe, record.embedding) }))
813
+ .sort((a, b) => (b.score - a.score) || recencyOrder(a.record, b.record));
814
+ const kept = typeof query.minScore === 'number'
815
+ ? ranked.filter((entry) => entry.score >= query.minScore)
816
+ : ranked;
817
+ const capped = typeof query.limit === 'number' ? kept.slice(0, query.limit) : kept;
818
+ return {
819
+ [member]: capped.map((entry) => entry.record),
820
+ scores: capped.map((entry) => entry.score),
821
+ skipped,
822
+ via,
823
+ ranking,
824
+ };
825
+ }
826
+ /**
827
+ * Retrieve by meaning: the records that pass the tag/where filter AND
828
+ * carry a vector, ranked by cosine similarity to the embedded query.
829
+ *
830
+ * The refusals come before the arithmetic, in this order: `near` must
831
+ * be a non-empty string; the seam must be wired (the `compileQuery`
832
+ * precedent — an absent capability refuses, it never degrades); the
833
+ * filter must compile; the seam must answer; and every candidate's
834
+ * identity must equal the query embedder's. A mixture — two models in
835
+ * the ledger, or a ledger embedded by one model and queried through
836
+ * another — is refused naming every identity found, because ranking
837
+ * the matching subset would be a silent wrong answer. Records without
838
+ * a vector are not scored (a fabricated score poisons a ranking) and
839
+ * not hidden (a silent drop poisons trust): they are counted in
840
+ * `skipped`.
841
+ *
842
+ * TWO paths reach that answer and both report which one ran. By
843
+ * default the ledger reads every record under the prefix and ranks
844
+ * them here. An adapter that declares `rank` is asked instead — it
845
+ * knows how to narrow to the k best without handing over the whole
846
+ * ledger — and is held to exactly the same contract: it reports the
847
+ * distinct identities it holds so the mixture refusal is the ledger's,
848
+ * it reports what it skipped, and the kernels re-score what it
849
+ * returns. An adapter whose own ranking is approximate makes recall
850
+ * approximate: the ledger can re-score what it is handed, never
851
+ * recover a record the adapter did not return.
852
+ *
853
+ * `tags` and `where` narrow candidates the adapter knows nothing
854
+ * about, so a query carrying either takes the sweep.
855
+ *
856
+ * Ranking is `cosineSimilarity` from the kernels, descending; equal
857
+ * scores fall back to recency, then id, so the order is deterministic.
858
+ * `minScore` filters the ranked list; `limit` caps what survives.
859
+ * @param name - the entry point, for the refusal text
860
+ * @param member - the result member the ranked records go under
861
+ * @param prefix - the key space the records live under
862
+ */
863
+ async function rankNear(query, name, member, prefix) {
864
+ const { near } = query;
865
+ if (typeof near !== 'string' || near === '')
866
+ return { error: `${name}: near must be a non-empty string — the text to rank by meaning against` };
867
+ if (embedder === null)
868
+ return { error: `${name}: near ${SEAM_REFUSAL}` };
869
+ const narrowed = query.tags !== undefined || query.where !== undefined;
870
+ const delegated = typeof storage.rank === 'function' && !narrowed;
871
+ // the sweep reads first so that a `where` without the compileQuery
872
+ // seam still refuses before the embedder is called; the adapter path
873
+ // has no filter to compile
874
+ const filtered = delegated ? null : filter(await readAll(prefix), query.tags, query.where);
875
+ if (filtered !== null && filtered.error !== undefined)
876
+ return filtered;
877
+ const answer = await embedThrough([near]);
1160
878
  if (answer.error !== undefined)
1161
- return { embedded, remaining: total - embedded, error: `embedMissing: the embedder seam failed — ${answer.error}` };
1162
- if (i === 0) {
1163
- // a wire client settles its width on its first reply; the
1164
- // check that could not run up front runs here, before a write
1165
- const refused = mixture(answer.identity);
1166
- if (refused !== null) return refused;
879
+ return { error: `${name}: the embedder seam failed — ${answer.error}` };
880
+ const probe = answer.vectors[0];
881
+ const identity = answer.identity;
882
+ if (delegated) {
883
+ const hits = await storage.rank({
884
+ prefix, vector: Array.from(probe), model: identity.model, dims: identity.dims,
885
+ limit: query.limit, minScore: query.minScore,
886
+ });
887
+ if (hits === null || typeof hits !== 'object' || !Array.isArray(hits.hits)
888
+ || !Array.isArray(hits.identities) || !Number.isSafeInteger(hits.skipped) || hits.skipped < 0) {
889
+ return {
890
+ error: `${name}: the storage adapter's rank answered something other than`
891
+ + ' { hits, skipped, identities } — a capability that cannot be trusted to report what it'
892
+ + ' skipped is worse than one that is absent'
893
+ };
894
+ }
895
+ const ranking = hits.ranking === undefined ? { algorithm: 'legacy-exact', exhaustive: true, candidateCount: hits.hits.length } : hits.ranking;
896
+ if (ranking === null || typeof ranking !== 'object' || typeof ranking.algorithm !== 'string' || ranking.algorithm.trim() === ''
897
+ || typeof ranking.exhaustive !== 'boolean' || ranking.candidateCount !== hits.hits.length)
898
+ return { error: `${name}: invalid storage rank metadata` };
899
+ const keys = new Set();
900
+ for (const hit of hits.hits) {
901
+ if (typeof hit?.key !== 'string' || !hit.key.startsWith(prefix) || hit.key.length === prefix.length
902
+ || keys.has(hit.key) || typeof hit.score !== 'number' || !Number.isFinite(hit.score))
903
+ return { error: `${name}: invalid or duplicate storage rank candidate` };
904
+ keys.add(hit.key);
905
+ }
906
+ const found = new Map();
907
+ for (const stored of hits.identities) {
908
+ if (typeof stored?.model !== 'string' || stored.model === '' || !Number.isSafeInteger(stored.dims) || stored.dims < 1)
909
+ return { error: `${name}: invalid storage rank identity` };
910
+ found.set(describeIdentity(stored), stored);
911
+ }
912
+ const records = await Promise.all(hits.hits.map((hit) => storage.get(hit.key)));
913
+ for (let i = 0; i < records.length; i++) {
914
+ const record = records[i];
915
+ if (record == null)
916
+ continue; // Deleted between selection and read.
917
+ if (`${prefix}${record.id}` !== hits.hits[i].key || !sameIdentity(record.embeddedBy, identity))
918
+ return { error: `${name}: storage rank candidate identity mismatch` };
919
+ if (!isVector(record.embedding, identity.dims))
920
+ return { error: `${name}: invalid storage rank candidate vector` };
921
+ found.set(describeIdentity(record.embeddedBy), record.embeddedBy);
922
+ }
923
+ const refused = refuseMixture(name, found, identity);
924
+ if (refused !== null)
925
+ return refused;
926
+ // a key the adapter ranked and a read that no longer finds it is a
927
+ // record deleted in between, not a record without a vector
928
+ return rankedResult(records.filter((record) => record?.embedding !== undefined), probe, query, member, hits.skipped, 'adapter', { algorithm: ranking.algorithm, exhaustive: ranking.exhaustive, candidateCount: ranking.candidateCount });
1167
929
  }
1168
- for (let j = 0; j < slice.length; j++) {
1169
- await storage.set(slice[j].key, withEmbedding(slice[j].record, answer.vectors[j], answer.identity));
1170
- embedded += 1;
930
+ // the identity check, before any math: every candidate's identity
931
+ // must be the query's, or nothing is ranked
932
+ const candidates = [];
933
+ const found = new Map();
934
+ let skipped = 0;
935
+ for (const record of filtered.value) {
936
+ if (record.embedding === undefined) {
937
+ skipped += 1;
938
+ continue;
939
+ }
940
+ found.set(describeIdentity(record.embeddedBy), record.embeddedBy);
941
+ candidates.push(record);
1171
942
  }
1172
- }
1173
- return { embedded, remaining: total - embedded };
1174
- }, { prefixes: [KEYS.memory, KEYS.skill] });
1175
- }
1176
-
1177
- //#endregion
1178
-
1179
- //#region slots
1180
-
1181
- /**
1182
- * Write an addressable blob. The metadata and the content go to
1183
- * separate keys, and only the metadata is ever handed around: that
1184
- * separation is the point of a slot, and it is what will let a request
1185
- * name a hundred of them without carrying any of their content.
1186
- * @param {string} name
1187
- * @param {string} content
1188
- * @param {{ kind?: string, at?: string, count?: number, pinned?: boolean }} [meta]
1189
- * `count` is what the content HOLDS (lines, records, pieces) where
1190
- * the writer knows it; absent where it does not, rather than guessed.
1191
- * @returns {Promise<LedgerSlot | LedgerRejection>}
1192
- */
1193
- function putSlot(name, content, meta = {}) {
1194
- return enqueue(async (storage) => {
1195
- const text = typeof content === 'string' ? content : JSON.stringify(content ?? null);
1196
- const slot = defined({
1197
- name,
1198
- kind: meta.kind ?? 'text',
1199
- size: text.length,
1200
- excerpt: excerpt(text, SLOT_EXCERPT_CHARS),
1201
- at: meta.at ?? now(),
1202
- count: meta.count,
1203
- pinned: meta.pinned,
1204
- });
1205
- const rejected = validate('slot', slot);
1206
- if (rejected !== null) return rejected;
1207
- if (options.archiveLimits && (slot.kind === 'agent-round' || slot.kind === 'agent-round-index')) {
1208
- const result = await storeArchive(storage, [{ slot, text }]);
1209
- return result.error ? result : slot;
1210
- }
1211
- await storage.set(`${KEYS.slotContent}${name}`, text);
1212
- await storage.set(`${KEYS.slot}${name}`, slot);
1213
- return slot;
1214
- }, options.archiveLimits && (meta.kind === 'agent-round' || meta.kind === 'agent-round-index') ? 'ai/state/' : { keys: [`${KEYS.slot}${name}`, `${KEYS.slotContent}${name}`, `ai/state/evicted/${name}`] });
1215
- }
1216
-
1217
- /**
1218
- * A slot's metadata — never its content.
1219
- * @param {string} name
1220
- * @returns {Promise<LedgerSlot | null>}
1221
- */
1222
- async function getSlot(name) {
1223
- return (await storage.get(`${KEYS.slot}${name}`)) ?? null;
1224
- }
1225
-
1226
- /**
1227
- * A slot's content, or undefined. The one call that returns the bytes.
1228
- * @param {string} name
1229
- * @returns {Promise<string | undefined | { status: 'evicted', name: string, bytes: number, reason: string }>}
1230
- */
1231
- async function readSlot(name) {
1232
- return (await storage.get(`${KEYS.slotContent}${name}`))
1233
- ?? (await storage.get(`ai/state/evicted/${name}`));
1234
- }
1235
-
1236
- /**
1237
- * Every slot's metadata, newest first.
1238
- * @returns {Promise<LedgerSlot[]>}
1239
- */
1240
- async function listSlots() {
1241
- return byRecency(await readAll(KEYS.slot));
1242
- }
1243
-
1244
- /**
1245
- * Remove a slot and its content. Answers whether one was there.
1246
- * @param {string} name
1247
- */
1248
- function deleteSlot(name) {
1249
- return enqueue(async (storage) => {
1250
- const key = `${KEYS.slot}${name}`;
1251
- const existed = (await storage.get(key)) !== undefined;
1252
- await storage.delete(key);
1253
- await storage.delete(`${KEYS.slotContent}${name}`);
1254
- await storage.delete(`ai/state/evicted/${name}`);
1255
- return existed;
1256
- }, { keys: [`${KEYS.slot}${name}`, `${KEYS.slotContent}${name}`, `ai/state/evicted/${name}`] });
1257
- }
1258
-
1259
- /** Apply a complete archive plan only after retention accepts its exact bytes. */
1260
- async function storeArchive(view, entries, protection = {}) {
1261
- if (options.archiveLimits && typeof storage.mutate !== 'function' && !isAtomicView(storage))
1262
- return { error: 'archive budgets require atomic storage', code: 'ATOMIC_REQUIRED' };
1263
- const keys = await view.keys('ai/state/');
1264
- const current = Object.fromEntries(await Promise.all(keys.map(async (key) => [key, await view.get(key)])));
1265
- const planned = retainArchives(current, entries, options.archiveLimits ?? {}, protection);
1266
- if (planned.error) return planned;
1267
- for (const key of Object.keys(current)) if (!Object.hasOwn(planned.next, key)) await view.delete(key);
1268
- for (const [key, value] of Object.entries(planned.next))
1269
- if (JSON.stringify(value) !== JSON.stringify(current[key])) await view.set(key, value);
1270
- return { ok: true, retention: planned.report, footprint: planned.footprint };
1271
- }
1272
-
1273
- /** Atomically store rounds and their index before removing any transcript.
1274
- * `protection.immutable` refuses address collisions, including within one batch;
1275
- * ordinary host-named archives retain their replacement behavior. */
1276
- function putArchive(entries, protection = {}) {
1277
- return enqueue(async (view) => {
1278
- const records = [];
1279
- for (const entry of entries) {
1280
- if ((entry.kind !== 'agent-round' && entry.kind !== 'agent-round-index') || typeof entry.text !== 'string')
1281
- return { error: 'archive entries require round/index kind and string content', code: 'ARCHIVE_INPUT' };
1282
- const held = await view.get(`${KEYS.slot}${entry.name}`);
1283
- const unchanged = held?.kind === entry.kind && await view.get(`${KEYS.slotContent}${entry.name}`) === entry.text;
1284
- records.push({ text: entry.text, slot: unchanged ? held : {
1285
- name: entry.name, kind: entry.kind, size: entry.text.length,
1286
- excerpt: excerpt(entry.text, SLOT_EXCERPT_CHARS), at: now(),
1287
- ...(held?.pinned === undefined ? {} : { pinned: held.pinned }),
1288
- } });
1289
- }
1290
- for (const entry of records) {
1291
- const rejected = validate('slot', entry.slot);
1292
- if (rejected) return rejected;
1293
- }
1294
- return storeArchive(view, records, protection);
1295
- });
1296
- }
1297
-
1298
- /** Read the last durable archive decision and current host capability. */
1299
- async function retentionReport() {
1300
- return (await storage.get('ai/state/retention/archive')) ?? null;
1301
- }
1302
-
1303
- /** Intentionally remove all conversation archives, tombstones and their report. */
1304
- function clearArchives(prefix = '') {
1305
- return enqueue(async (view) => {
1306
- for (const slot of await readAll(KEYS.slot, view)) {
1307
- if (!slot.name.startsWith(prefix) || (slot.kind !== 'agent-round' && slot.kind !== 'agent-round-index')) continue;
1308
- await view.delete(`${KEYS.slot}${slot.name}`);
1309
- await view.delete(`${KEYS.slotContent}${slot.name}`);
1310
- }
1311
- for (const key of await view.keys(`ai/state/evicted/${prefix}`)) await view.delete(key);
1312
- const report = await view.get('ai/state/retention/archive');
1313
- if (prefix === '') await view.delete('ai/state/retention/archive');
1314
- else if (report) await view.set('ai/state/retention/archive', { ...report,
1315
- evicted: report.evicted.filter((entry) => !entry.name.startsWith(prefix)),
1316
- written: report.written.filter((name) => !name.startsWith(prefix)) });
1317
- return true;
1318
- });
1319
- }
1320
-
1321
- //#endregion
1322
-
1323
- //#region snapshots
1324
-
1325
- /**
1326
- * Record the whole state under a token. Cheap — a ledger is small JSON
1327
- * and it is what makes an automatic write safe to accept: a bad one
1328
- * is undone by its token, with no human reading a diff. The token
1329
- * continues the sequence already in storage, exactly as an id does: a
1330
- * second ledger over the same adapter — the next process over a
1331
- * durable one — mints the next token, never the first one's again.
1332
- * @returns {Promise<string>} an opaque token for `rollback`
1333
- */
1334
- function snapshot() {
1335
- return enqueue(async (storage) => {
1336
- const keys = await storage.keys(STATE);
1337
- const entries = await Promise.all(keys.map(async (key) => [key, await storage.get(key)]));
1338
- const token = `snap-${seq(nextSequence(await storage.keys(SNAP)))}`;
1339
- await storage.set(`${SNAP}${token}`, entries);
1340
- return token;
1341
- });
1342
- }
1343
-
1344
- /**
1345
- * Restore the state a token recorded. Every current state key is
1346
- * removed first, so a rollback is a restore and not a merge — a record
1347
- * created after the snapshot is gone afterwards, which is the only
1348
- * reading of "rollback" that can be relied on.
1349
- * @param {string} token
1350
- * @returns {Promise<true | { error: string }>}
1351
- */
1352
- function rollback(token) {
1353
- return enqueue(async (storage) => {
1354
- const entries = await storage.get(`${SNAP}${token}`);
1355
- if (entries === undefined) return { error: `unknown snapshot '${token}'` };
1356
- for (const key of await storage.keys(STATE)) await storage.delete(key);
1357
- for (const [key, value] of entries) await storage.set(key, value);
1358
- return true;
1359
- });
1360
- }
1361
-
1362
- //#endregion
1363
-
1364
- /**
1365
- * Commit guarded supplemental work against the exact document it read.
1366
- * Atomic adapters publish the snapshot and all writes together. A stale
1367
- * proposal is refused, never rebased onto different record indices.
1368
- * @param {any} expected
1369
- * @param {(ledger: any) => Promise<any>} work
1370
- */
1371
- function transaction(expected, work) {
1372
- return enqueue(async (view) => {
1373
- const scoped = createLedger({ ...options, artifacts: admittedArtifacts, storage: view });
1374
- const current = { goal: await scoped.getGoal(), memories: await scoped.listMemories(),
1375
- skills: await scoped.listSkills() };
1376
- if (JSON.stringify(current) !== JSON.stringify(expected))
1377
- return { error: 'the refinement was rejected: supplemental state changed; regenerate the proposal',
1378
- errors: [{ code: 'LEDGER_CONFLICT', docPath: '', message: 'supplemental state changed' }] };
1379
- const outcome = await work(scoped);
1380
- if (outcome?.error) throw Object.assign(new Error(outcome.error), { outcome });
1381
- return outcome;
1382
- });
1383
- }
1384
-
1385
- return {
1386
- storageStatus: () => typeof storage.status === 'function' ? storage.status()
1387
- : { concurrency: typeof storage.mutate === 'function' ? 'atomic' : 'single-writer', durability: 'host-defined', error: null },
1388
- concurrency: typeof storage.mutate === 'function' ? 'atomic' : 'single-writer',
1389
- validate,
1390
- setGoal, getGoal, listArchivedGoals, recordProgress, setGoalStatus, composeGoal,
1391
- addMemory, getMemory, listMemories, deleteMemory,
1392
- addSkill, getSkill, listSkills, deleteSkill,
1393
- recall, recallSkills, embedMissing,
1394
- putSlot, getSlot, readSlot, listSlots, deleteSlot, putArchive, clearArchives, retentionReport,
1395
- snapshot, rollback, transaction,
1396
- };
943
+ const refused = refuseMixture(name, found, identity);
944
+ if (refused !== null)
945
+ return refused;
946
+ return rankedResult(candidates, probe, query, member, skipped, 'sweep');
947
+ }
948
+ async function recall(query = {}) {
949
+ return query.near === undefined
950
+ ? retrieve(await readAll(KEYS.memory), query)
951
+ : rankNear(query, 'recall', 'memories', KEYS.memory);
952
+ }
953
+ async function recallSkills(query = {}) {
954
+ return query.near === undefined
955
+ ? retrieve(await readAll(KEYS.skill), query)
956
+ : rankNear(query, 'recallSkills', 'skills', KEYS.skill);
957
+ }
958
+ /**
959
+ * The explicit sweep: embed every memory and skill that carries no
960
+ * vector, through the seam, in batches, inside the write chain — so no
961
+ * other write moves a record under it. Memories first, then skills,
962
+ * each in key order; `limit` caps how many records this run embeds,
963
+ * `batch` how many texts one seam call carries. Positive fractional batch sizes
964
+ * are floored to an integer of at least one.
965
+ *
966
+ * Honest about what it did not do. A second run over a swept ledger
967
+ * embeds zero and makes no seam call (the two-run check). A failing
968
+ * batch ends the run: the records embedded before it are written, the
969
+ * failed batch and everything after it stay un-embedded and are
970
+ * counted in `remaining`, and the error is surfaced once — never a
971
+ * throw that loses the batch, and never a per-record retry against a
972
+ * provider that just refused (the seam's own retry policy has already
973
+ * run). A ledger that already holds vectors under another identity is
974
+ * refused up front rather than turned into a mixture `recall({ near })`
975
+ * would then refuse.
976
+ * @param [options]
977
+ */
978
+ function embedMissing(options = {}) {
979
+ const batches = new Map();
980
+ return enqueue(async (storage) => {
981
+ const batch = typeof options.batch === 'number' && options.batch > 0 ? Math.max(1, Math.floor(options.batch)) : EMBED_BATCH;
982
+ const pending = [];
983
+ const held = new Map();
984
+ for (const memory of await readAll(KEYS.memory, storage)) {
985
+ if (memory.embedding === undefined)
986
+ pending.push({ key: `${KEYS.memory}${memory.id}`, record: memory, text: memory.text });
987
+ else
988
+ held.set(describeIdentity(memory.embeddedBy), memory.embeddedBy);
989
+ }
990
+ for (const skill of await readAll(KEYS.skill, storage)) {
991
+ if (skill.embedding === undefined)
992
+ pending.push({ key: `${KEYS.skill}${skill.id}`, record: skill, text: skillText(skill) });
993
+ else
994
+ held.set(describeIdentity(skill.embeddedBy), skill.embeddedBy);
995
+ }
996
+ const total = pending.length;
997
+ if (embedder === null)
998
+ return { embedded: 0, remaining: total, error: `embedMissing: ${SEAM_REFUSAL}` };
999
+ /** The mixture refusal, once the embedder's width is known. */
1000
+ const mixture = (identity) => {
1001
+ const foreign = [...held.keys()].filter((key) => !sameIdentity(held.get(key), identity));
1002
+ return foreign.length === 0 ? null
1003
+ : {
1004
+ embedded: 0, remaining: total, error: 'embedMissing: would mix vector identities — the ledger'
1005
+ + ` already holds vectors from ${foreign.join(', ')} and the embedder is ${describeIdentity(identity)};`
1006
+ + ' sweep with the embedder that wrote them, or re-add those records without a vector first'
1007
+ };
1008
+ };
1009
+ if (embedder.dims !== undefined) {
1010
+ const refused = mixture({ model: embedder.model, dims: embedder.dims });
1011
+ if (refused !== null)
1012
+ return refused;
1013
+ }
1014
+ const todo = typeof options.limit === 'number' ? pending.slice(0, Math.max(0, Math.floor(options.limit))) : pending;
1015
+ let embedded = 0;
1016
+ for (let i = 0; i < todo.length; i += batch) {
1017
+ const slice = todo.slice(i, i + batch);
1018
+ const texts = slice.map((entry) => entry.text), key = JSON.stringify(texts);
1019
+ if (!batches.has(key))
1020
+ batches.set(key, await embedThrough(texts));
1021
+ const answer = batches.get(key);
1022
+ if (answer.error !== undefined)
1023
+ return { embedded, remaining: total - embedded, error: `embedMissing: the embedder seam failed — ${answer.error}` };
1024
+ if (i === 0) {
1025
+ // a wire client settles its width on its first reply; the
1026
+ // check that could not run up front runs here, before a write
1027
+ const refused = mixture(answer.identity);
1028
+ if (refused !== null)
1029
+ return refused;
1030
+ }
1031
+ for (let j = 0; j < slice.length; j++) {
1032
+ await storage.set(slice[j].key, withEmbedding(slice[j].record, answer.vectors[j], answer.identity));
1033
+ embedded += 1;
1034
+ }
1035
+ }
1036
+ return { embedded, remaining: total - embedded };
1037
+ }, { prefixes: [KEYS.memory, KEYS.skill] });
1038
+ }
1039
+ //#endregion
1040
+ //#region slots
1041
+ /**
1042
+ * Write an addressable blob. The metadata and the content go to
1043
+ * separate keys, and only the metadata is ever handed around: that
1044
+ * separation is the point of a slot, and it is what will let a request
1045
+ * name a hundred of them without carrying any of their content.
1046
+ * @param [meta]
1047
+ * `count` is what the content HOLDS (lines, records, pieces) where
1048
+ * the writer knows it; absent where it does not, rather than guessed.
1049
+ */
1050
+ function putSlot(name, content, meta = {}) {
1051
+ return enqueue(async (storage) => {
1052
+ const text = typeof content === 'string' ? content : JSON.stringify(content ?? null);
1053
+ const slot = defined({
1054
+ name,
1055
+ kind: meta.kind ?? 'text',
1056
+ size: text.length,
1057
+ excerpt: excerpt(text, SLOT_EXCERPT_CHARS),
1058
+ at: meta.at ?? now(),
1059
+ count: meta.count,
1060
+ pinned: meta.pinned,
1061
+ });
1062
+ const rejected = validate('slot', slot);
1063
+ if (rejected !== null)
1064
+ return rejected;
1065
+ if (options.archiveLimits && (slot.kind === 'agent-round' || slot.kind === 'agent-round-index')) {
1066
+ const result = await storeArchive(storage, [{ slot, text }]);
1067
+ return 'error' in result ? result : slot;
1068
+ }
1069
+ await storage.set(`${KEYS.slotContent}${name}`, text);
1070
+ await storage.set(`${KEYS.slot}${name}`, slot);
1071
+ return slot;
1072
+ }, options.archiveLimits && (meta.kind === 'agent-round' || meta.kind === 'agent-round-index') ? 'ai/state/' : { keys: [`${KEYS.slot}${name}`, `${KEYS.slotContent}${name}`, `ai/state/evicted/${name}`] });
1073
+ }
1074
+ /**
1075
+ * A slot's metadata — never its content.
1076
+ */
1077
+ async function getSlot(name) {
1078
+ return (await storage.get(`${KEYS.slot}${name}`)) ?? null;
1079
+ }
1080
+ /**
1081
+ * A slot's content, or undefined. The one call that returns the bytes.
1082
+ */
1083
+ async function readSlot(name) {
1084
+ return (await storage.get(`${KEYS.slotContent}${name}`))
1085
+ ?? (await storage.get(`ai/state/evicted/${name}`));
1086
+ }
1087
+ /**
1088
+ * Every slot's metadata, newest first.
1089
+ */
1090
+ async function listSlots() {
1091
+ return byRecency(await readAll(KEYS.slot));
1092
+ }
1093
+ /**
1094
+ * Remove a slot and its content. Answers whether one was there.
1095
+ */
1096
+ function deleteSlot(name) {
1097
+ return enqueue(async (storage) => {
1098
+ const key = `${KEYS.slot}${name}`;
1099
+ const existed = (await storage.get(key)) !== undefined;
1100
+ await storage.delete(key);
1101
+ await storage.delete(`${KEYS.slotContent}${name}`);
1102
+ await storage.delete(`ai/state/evicted/${name}`);
1103
+ return existed;
1104
+ }, { keys: [`${KEYS.slot}${name}`, `${KEYS.slotContent}${name}`, `ai/state/evicted/${name}`] });
1105
+ }
1106
+ /** Apply a complete archive plan only after retention accepts its exact bytes. */
1107
+ async function storeArchive(view, entries, protection = {}) {
1108
+ if (options.archiveLimits && typeof storage.mutate !== 'function' && !isAtomicView(storage))
1109
+ return { error: 'archive budgets require atomic storage', code: 'ATOMIC_REQUIRED' };
1110
+ const keys = await view.keys('ai/state/');
1111
+ const current = Object.fromEntries(await Promise.all(keys.map(async (key) => [key, await view.get(key)])));
1112
+ const planned = retainArchives(current, entries, options.archiveLimits ?? {}, protection);
1113
+ if ('error' in planned)
1114
+ return planned;
1115
+ for (const key of Object.keys(current))
1116
+ if (!Object.hasOwn(planned.next, key))
1117
+ await view.delete(key);
1118
+ for (const [key, value] of Object.entries(planned.next))
1119
+ if (JSON.stringify(value) !== JSON.stringify(current[key]))
1120
+ await view.set(key, value);
1121
+ return { ok: true, retention: planned.report, footprint: planned.footprint };
1122
+ }
1123
+ /** Atomically store rounds and their index before removing any transcript.
1124
+ * `protection.immutable` refuses address collisions, including within one batch;
1125
+ * ordinary host-named archives retain their replacement behavior. */
1126
+ function putArchive(entries, protection = {}) {
1127
+ return enqueue(async (view) => {
1128
+ const records = [];
1129
+ for (const entry of entries) {
1130
+ if ((entry.kind !== 'agent-round' && entry.kind !== 'agent-round-index') || typeof entry.text !== 'string')
1131
+ return { error: 'archive entries require round/index kind and string content', code: 'ARCHIVE_INPUT' };
1132
+ const held = await view.get(`${KEYS.slot}${entry.name}`);
1133
+ const unchanged = held?.kind === entry.kind && await view.get(`${KEYS.slotContent}${entry.name}`) === entry.text;
1134
+ records.push({
1135
+ text: entry.text, slot: unchanged ? held : {
1136
+ name: entry.name, kind: entry.kind, size: entry.text.length,
1137
+ excerpt: excerpt(entry.text, SLOT_EXCERPT_CHARS), at: now(),
1138
+ ...(held?.pinned === undefined ? {} : { pinned: held.pinned }),
1139
+ }
1140
+ });
1141
+ }
1142
+ for (const entry of records) {
1143
+ const rejected = validate('slot', entry.slot);
1144
+ if (rejected)
1145
+ return rejected;
1146
+ }
1147
+ return storeArchive(view, records, protection);
1148
+ });
1149
+ }
1150
+ /** Read the last durable archive decision and current host capability. */
1151
+ async function retentionReport() {
1152
+ return (await storage.get('ai/state/retention/archive')) ?? null;
1153
+ }
1154
+ /** Intentionally remove all conversation archives, tombstones and their report. */
1155
+ function clearArchives(prefix = '') {
1156
+ return enqueue(async (view) => {
1157
+ for (const slot of await readAll(KEYS.slot, view)) {
1158
+ if (!slot.name.startsWith(prefix) || (slot.kind !== 'agent-round' && slot.kind !== 'agent-round-index'))
1159
+ continue;
1160
+ await view.delete(`${KEYS.slot}${slot.name}`);
1161
+ await view.delete(`${KEYS.slotContent}${slot.name}`);
1162
+ }
1163
+ for (const key of await view.keys(`ai/state/evicted/${prefix}`))
1164
+ await view.delete(key);
1165
+ const report = await view.get('ai/state/retention/archive');
1166
+ if (prefix === '')
1167
+ await view.delete('ai/state/retention/archive');
1168
+ else if (report)
1169
+ await view.set('ai/state/retention/archive', {
1170
+ ...report,
1171
+ evicted: report.evicted.filter((entry) => !entry.name.startsWith(prefix)),
1172
+ written: report.written.filter((name) => !name.startsWith(prefix))
1173
+ });
1174
+ return true;
1175
+ });
1176
+ }
1177
+ //#endregion
1178
+ //#region snapshots
1179
+ /**
1180
+ * Record the whole state under a token. Cheap — a ledger is small JSON
1181
+ * — and it is what makes an automatic write safe to accept: a bad one
1182
+ * is undone by its token, with no human reading a diff. The token
1183
+ * continues the sequence already in storage, exactly as an id does: a
1184
+ * second ledger over the same adapter — the next process over a
1185
+ * durable one — mints the next token, never the first one's again.
1186
+ * @returns an opaque token for `rollback`
1187
+ */
1188
+ function snapshot() {
1189
+ return enqueue(async (storage) => {
1190
+ const keys = await storage.keys(STATE);
1191
+ const entries = await Promise.all(keys.map(async (key) => [key, await storage.get(key)]));
1192
+ const token = `snap-${seq(nextSequence(await storage.keys(SNAP)))}`;
1193
+ await storage.set(`${SNAP}${token}`, entries);
1194
+ return token;
1195
+ });
1196
+ }
1197
+ /**
1198
+ * Restore the state a token recorded. Every current state key is
1199
+ * removed first, so a rollback is a restore and not a merge — a record
1200
+ * created after the snapshot is gone afterwards, which is the only
1201
+ * reading of "rollback" that can be relied on.
1202
+ */
1203
+ function rollback(token) {
1204
+ return enqueue(async (storage) => {
1205
+ const entries = await storage.get(`${SNAP}${token}`);
1206
+ if (entries === undefined)
1207
+ return { error: `unknown snapshot '${token}'` };
1208
+ for (const key of await storage.keys(STATE))
1209
+ await storage.delete(key);
1210
+ for (const [key, value] of entries)
1211
+ await storage.set(key, value);
1212
+ return true;
1213
+ });
1214
+ }
1215
+ //#endregion
1216
+ /**
1217
+ * Commit guarded supplemental work against the exact document it read.
1218
+ * Atomic adapters publish the snapshot and all writes together. A stale
1219
+ * proposal is refused, never rebased onto different record indices.
1220
+ */
1221
+ function transaction(expected, work) {
1222
+ return enqueue(async (view) => {
1223
+ const scoped = createLedger({ ...options, artifacts: admittedArtifacts, storage: view });
1224
+ const current = {
1225
+ goal: await scoped.getGoal(), memories: await scoped.listMemories(),
1226
+ skills: await scoped.listSkills()
1227
+ };
1228
+ if (JSON.stringify(current) !== JSON.stringify(expected))
1229
+ return {
1230
+ error: 'the refinement was rejected: supplemental state changed; regenerate the proposal',
1231
+ errors: [{ code: 'LEDGER_CONFLICT', docPath: '', message: 'supplemental state changed' }]
1232
+ };
1233
+ const outcome = await work(scoped);
1234
+ if (outcome?.error)
1235
+ throw Object.assign(new Error(outcome.error), { outcome });
1236
+ return outcome;
1237
+ });
1238
+ }
1239
+ return {
1240
+ storageStatus: () => typeof storage.status === 'function' ? storage.status()
1241
+ : { concurrency: typeof storage.mutate === 'function' ? 'atomic' : 'single-writer', durability: 'host-defined', error: null },
1242
+ concurrency: typeof storage.mutate === 'function' ? 'atomic' : 'single-writer',
1243
+ validate,
1244
+ setGoal, getGoal, listArchivedGoals, recordProgress, setGoalStatus, composeGoal,
1245
+ addMemory, getMemory, listMemories, deleteMemory,
1246
+ addSkill, getSkill, listSkills, deleteSkill,
1247
+ recall, recallSkills, embedMissing,
1248
+ putSlot, getSlot, readSlot, listSlots, deleteSlot, putArchive, clearArchives, retentionReport,
1249
+ snapshot, rollback, transaction,
1250
+ };
1397
1251
  }