bare-agent 0.19.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -1
- package/bareagent.context.md +70 -1
- package/index.d.ts +6 -1
- package/index.js +7 -0
- package/package.json +4 -4
- package/src/planner.d.ts +21 -1
- package/src/planner.js +24 -4
- package/src/recurse-prompts.d.ts +22 -0
- package/src/recurse-prompts.js +71 -0
- package/src/recurse-retrieval.d.ts +174 -0
- package/src/recurse-retrieval.js +372 -0
- package/src/recurse-synthesize.d.ts +53 -0
- package/src/recurse-synthesize.js +97 -0
- package/src/recurse.d.ts +376 -0
- package/src/recurse.js +911 -0
|
@@ -0,0 +1,372 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// RLM_PRD §10 step 7 — retrieval wiring, to the §9.2.1 MEASURED task-shape model. Context is a HANDLE, never
|
|
4
|
+
// the whole corpus (RC-5), and the handle is chosen by the QUESTION'S SHAPE — there is no single retrieval
|
|
5
|
+
// winner:
|
|
6
|
+
//
|
|
7
|
+
// scan (DEFAULT) — process EVERY slice + LLM-judge per window + CODE-count. The only COMPLETE path: retrieval
|
|
8
|
+
// recall is structurally capped (BM25 at lexical hits, embeddings at the KNN cap), so for a
|
|
9
|
+
// "how many / all of them" ask a scan is the sole honest answer (§9.2.1 CORRECTION 2 — naive
|
|
10
|
+
// "search→count" silently undercounts 75–95%). Deterministic ORCHESTRATION, not a model call:
|
|
11
|
+
// the aggregation is CODE (never a model Finish/count — §9.1 flaw #2). Pure bareagent.
|
|
12
|
+
// search (needle) — litectx `recall` handle (embeddings ON, `fact`/`episode` — the KNN-nominate kinds; capped
|
|
13
|
+
// at KNN_K). For FINDING the few; CANNOT count. A Family-A worker calls it as a tool.
|
|
14
|
+
// exact (rule) — a deterministic code-side AND-term predicate filter over the slice-source (embeddings-free
|
|
15
|
+
// by construction — the "code-side predicate filter" half of §9.2.1). A worker tool.
|
|
16
|
+
//
|
|
17
|
+
// THE CORPUS FOR SCAN IS A GENERIC ARRAY SLICE-SOURCE (`opts.corpus`), NOT litectx: litectx has no exhaustive,
|
|
18
|
+
// rank-free enumerate verb today (every read is FTS-gated). The "corpus that already LIVES in litectx" case
|
|
19
|
+
// waits on the litectx `enumerate` verb (docs/01-product/litectx-enumerate-spec.md) and drops in behind this
|
|
20
|
+
// same slice-source socket with ZERO recurse changes — the same backend-agnostic stance as `remember`'s Store
|
|
21
|
+
// socket. Composes AROUND a Loop; NEVER imported by loop.js.
|
|
22
|
+
|
|
23
|
+
/** @typedef {import('../types').Provider} Provider */
|
|
24
|
+
/** @typedef {import('../types').ToolDef} ToolDef */
|
|
25
|
+
|
|
26
|
+
const { Loop } = require('./loop');
|
|
27
|
+
const { HaltError } = require('./errors');
|
|
28
|
+
|
|
29
|
+
// §9.2.1 LOCKED defaults. WINDOW is the one calibrated number (RECALL knee ≈ 8, per-model — calibrate via the
|
|
30
|
+
// active half-window probe, NOT context size); PASSES=2 (shuffled-boundary union → recall ~0.91, precision
|
|
31
|
+
// ~0.98). KNN_K is litectx's embeddings-nominate cap for the search handle.
|
|
32
|
+
const SCAN_WINDOW = 8;
|
|
33
|
+
const SCAN_PASSES = 2;
|
|
34
|
+
const KNN_K = 8;
|
|
35
|
+
|
|
36
|
+
// Completeness-contract guard (RC-9 applied to retrieval): a goal/contract implying COMPLETENESS must never be
|
|
37
|
+
// answered by a capped `search` (which cannot count). Auto-detection may only UPGRADE to scan (the safe
|
|
38
|
+
// direction), NEVER silently downgrade. Deliberately broad on the "how many / all" family the PRD names.
|
|
39
|
+
const COMPLETENESS_RE =
|
|
40
|
+
/\b(all|every|each|how many|count|counts|counting|total|number of|list all|exhaustive|complete(?:ly)?)\b/i;
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Does this text imply a completeness ("all / every / count / how many") ask? Used to upgrade a capped search
|
|
44
|
+
* to a scan (the only complete path). Conservative by design — a false positive only costs a thorough scan.
|
|
45
|
+
* @param {unknown} text
|
|
46
|
+
* @returns {boolean}
|
|
47
|
+
*/
|
|
48
|
+
function impliesCompleteness(text) {
|
|
49
|
+
return typeof text === 'string' && COMPLETENESS_RE.test(text);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* The §9.2-validated classify system prompt, GENERALIZED from the POC's hardcoded "SPORTS news" predicate to
|
|
54
|
+
* an arbitrary one. The load-bearing wording is verbatim ("Examine EACH item individually", "Output ONLY the
|
|
55
|
+
* IDs", "Comma-separated. If none, output 'none'. No count, no prose.") — the predicate is the only variable.
|
|
56
|
+
* It MUST return ids, not a count: counting is CODE's job (RC-5 / §9.1), so the judge never does arithmetic.
|
|
57
|
+
* @param {string} predicate - The task/goal the items are judged against.
|
|
58
|
+
* @returns {string}
|
|
59
|
+
*/
|
|
60
|
+
function classifySystem(predicate) {
|
|
61
|
+
return (
|
|
62
|
+
'You are a precise classifier. Examine EACH item individually. Each item is shown on its own line as ' +
|
|
63
|
+
'`<id> => <text>`. Output ONLY the IDs of the items that match the predicate below — copy each matching ' +
|
|
64
|
+
'item\'s <id> VERBATIM (the exact characters before " => "; an id may itself contain ":" — keep all of ' +
|
|
65
|
+
'it). Comma-separated. If none, output "none". No count, no prose.\n\nPREDICATE: ' +
|
|
66
|
+
predicate
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* @typedef {object} Slice
|
|
72
|
+
* @property {string} id - Stable, word-like id (no whitespace/commas — it is echoed back by the judge).
|
|
73
|
+
* @property {string} text - The item content shown to the judge.
|
|
74
|
+
*/
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Judge ONE window: run an isolated Loop with the classify prompt over the window's items, and intersect the
|
|
78
|
+
* returned ids with the ids actually SHOWN this window (RC-2 — a window's judge can only "match" what it was
|
|
79
|
+
* given; a hallucinated id from another window is dropped). A governance HaltError propagates (the caller turns
|
|
80
|
+
* it into a clean incomplete); any other Loop fault marks the window DEAD (→ RC-9 missingSlices), never a
|
|
81
|
+
* silent zero that would undercount.
|
|
82
|
+
* @param {string} predicate
|
|
83
|
+
* @param {Slice[]} window
|
|
84
|
+
* @param {{provider: Provider, ctx?: object, onLlmResult?: Function, policy?: Function, nonce: number}} opts
|
|
85
|
+
* @returns {Promise<{ids: string[]|null, dead: boolean}>}
|
|
86
|
+
* @throws {HaltError}
|
|
87
|
+
*/
|
|
88
|
+
async function judgeWindow(predicate, window, opts) {
|
|
89
|
+
const shown = new Set(window.map((r) => r.id));
|
|
90
|
+
const loop = new Loop({
|
|
91
|
+
provider: opts.provider,
|
|
92
|
+
system: classifySystem(predicate),
|
|
93
|
+
policy: opts.policy || undefined,
|
|
94
|
+
onLlmResult: opts.onLlmResult || undefined,
|
|
95
|
+
throwOnError: false,
|
|
96
|
+
});
|
|
97
|
+
const body = window.map((r) => `${r.id} => ${r.text}`).join('\n');
|
|
98
|
+
// A per-window nonce busts provider prompt-caching on live runs (identical windows across passes would
|
|
99
|
+
// otherwise be served from cache); harmless offline.
|
|
100
|
+
const user = `[run ${opts.nonce}] Find the items matching the predicate.\n\nITEMS:\n${body}\n\nReply with ONLY the comma-separated matching ids (or "none").`;
|
|
101
|
+
const out = await loop.run([{ role: 'user', content: user }], [], { ctx: opts.ctx });
|
|
102
|
+
if (typeof out.error === 'string' && out.error.startsWith('halt:')) {
|
|
103
|
+
throw new HaltError('[scan] judge halted by governance', { rule: out.error.slice('halt:'.length) });
|
|
104
|
+
}
|
|
105
|
+
if (out.error) return { ids: null, dead: true };
|
|
106
|
+
const tokens = String(out.text || '')
|
|
107
|
+
.split(/[\s,]+/)
|
|
108
|
+
.map((t) => t.trim())
|
|
109
|
+
.filter(Boolean);
|
|
110
|
+
// RC-2 intersect: keep only ids that were actually shown in THIS window.
|
|
111
|
+
return { ids: tokens.filter((t) => shown.has(t)), dead: false };
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Deterministic rotation by `k` — the shuffled-boundary mechanism for multi-pass union WITHOUT an RNG (keeps
|
|
116
|
+
* RC-3 determinism: same corpus + same passes ⇒ identical scan). Rotating the array changes which items share a
|
|
117
|
+
* window (and each item's within-window position), so an item under-recalled at one window's tail in pass 0
|
|
118
|
+
* lands mid-window in pass 1 — the §9.2.1 mechanism that lifts recall ~0.85 → ~0.93.
|
|
119
|
+
* @template T @param {T[]} arr @param {number} k @returns {T[]}
|
|
120
|
+
*/
|
|
121
|
+
function rotate(arr, k) {
|
|
122
|
+
const n = arr.length;
|
|
123
|
+
if (n === 0) return arr.slice();
|
|
124
|
+
const s = ((k % n) + n) % n;
|
|
125
|
+
return arr.slice(s).concat(arr.slice(0, s));
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* SCAN — process every slice, LLM-judge each window, union matching ids across windows AND passes, CODE-count
|
|
130
|
+
* the union. The default reliability mechanism (§9.2.1): the only path that does not silently undercount.
|
|
131
|
+
* RC-9: a dead window is recorded in `missingSlices`, never folded into the count as a zero.
|
|
132
|
+
* @param {string} predicate - The task the slices are judged against.
|
|
133
|
+
* @param {Slice[]} corpus - The array slice-source (already validated/normalized by the caller).
|
|
134
|
+
* @param {object} opts
|
|
135
|
+
* @param {Provider} opts.provider
|
|
136
|
+
* @param {number} [opts.window]
|
|
137
|
+
* @param {number} [opts.passes]
|
|
138
|
+
* @param {object} [opts.ctx]
|
|
139
|
+
* @param {Function} [opts.onLlmResult]
|
|
140
|
+
* @param {Function} [opts.policy]
|
|
141
|
+
* @returns {Promise<{matchedIds: string[], count: number, missingSlices: string[], window: number, passes: number, scanned: number}>}
|
|
142
|
+
* @throws {HaltError} a governance cap halted a window judge.
|
|
143
|
+
*/
|
|
144
|
+
async function scanCount(predicate, corpus, opts) {
|
|
145
|
+
const window = Number.isInteger(opts.window) && /** @type {number} */ (opts.window) > 0 ? /** @type {number} */ (opts.window) : SCAN_WINDOW;
|
|
146
|
+
const passes = Number.isInteger(opts.passes) && /** @type {number} */ (opts.passes) > 0 ? /** @type {number} */ (opts.passes) : SCAN_PASSES;
|
|
147
|
+
/** @type {Set<string>} */
|
|
148
|
+
const matched = new Set();
|
|
149
|
+
/** @type {string[]} */
|
|
150
|
+
const missingSlices = [];
|
|
151
|
+
let nonce = 0;
|
|
152
|
+
|
|
153
|
+
for (let p = 0; p < passes; p++) {
|
|
154
|
+
// Pass 0 = natural order; later passes shift the boundaries by a deterministic part-window offset so the
|
|
155
|
+
// windows group different items together (the union breaks the single-pass recall ceiling).
|
|
156
|
+
const order = p === 0 ? corpus : rotate(corpus, p * Math.max(1, Math.floor(window / passes)) + p);
|
|
157
|
+
for (let i = 0; i < order.length; i += window) {
|
|
158
|
+
const w = order.slice(i, i + window);
|
|
159
|
+
const { ids, dead } = await judgeWindow(predicate, w, { ...opts, nonce: nonce++ });
|
|
160
|
+
if (dead) {
|
|
161
|
+
missingSlices.push(`pass ${p} window@${i} (${w.length} items)`);
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
for (const id of /** @type {string[]} */ (ids)) matched.add(id);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
return { matchedIds: [...matched], count: matched.size, missingSlices, window, passes, scanned: corpus.length };
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Normalize a raw slice-source into validated `{id, text}` slices (drops malformed entries — a slice with no
|
|
172
|
+
* string id/text cannot be judged or counted, so it is excluded rather than silently miscounted).
|
|
173
|
+
* @param {unknown} corpus
|
|
174
|
+
* @returns {Slice[]}
|
|
175
|
+
*/
|
|
176
|
+
function normalizeCorpus(corpus) {
|
|
177
|
+
if (!Array.isArray(corpus)) return [];
|
|
178
|
+
return /** @type {Slice[]} */ (
|
|
179
|
+
corpus.filter((r) => r && typeof (/** @type {any} */ (r).id) === 'string' && typeof (/** @type {any} */ (r).text) === 'string')
|
|
180
|
+
);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/**
|
|
184
|
+
* The `search` handle tool (RC-5 needle path): litectx `recall` — embeddings ON, `fact`/`episode` (the
|
|
185
|
+
* KNN-nominate kinds), capped at `KNN_K`. For FINDING the relevant few; the description tells the worker it
|
|
186
|
+
* CANNOT count (the completeness guard catches a "how many" ask before this tool is ever offered). Returns the
|
|
187
|
+
* matched items' bodies as text the worker reads; never the whole corpus.
|
|
188
|
+
* @param {{recall: Function}} litectx
|
|
189
|
+
* @param {{kinds?: string[], n?: number}} [opts]
|
|
190
|
+
* @returns {ToolDef}
|
|
191
|
+
*/
|
|
192
|
+
function buildSearchTool(litectx, opts = {}) {
|
|
193
|
+
const kinds = Array.isArray(opts.kinds) && opts.kinds.length ? opts.kinds : ['fact', 'episode'];
|
|
194
|
+
const n = Number.isInteger(opts.n) && /** @type {number} */ (opts.n) > 0 ? /** @type {number} */ (opts.n) : KNN_K;
|
|
195
|
+
return {
|
|
196
|
+
name: 'search_memory',
|
|
197
|
+
description:
|
|
198
|
+
'Find the most relevant FEW stored items for a query (semantic recall, capped). Use to FIND specific ' +
|
|
199
|
+
'facts or episodes by meaning. It returns only the top matches — NOT all of them — so never use it to ' +
|
|
200
|
+
'count or enumerate "how many / all"; for that, the records are scanned in full instead.',
|
|
201
|
+
parameters: {
|
|
202
|
+
type: 'object',
|
|
203
|
+
properties: { query: { type: 'string', description: 'what to find (natural language)' } },
|
|
204
|
+
required: ['query'],
|
|
205
|
+
},
|
|
206
|
+
/** @param {{query?: string}} args */
|
|
207
|
+
execute: async (args) => {
|
|
208
|
+
const query = typeof args?.query === 'string' ? args.query.trim() : '';
|
|
209
|
+
if (!query) return '[error] search_memory requires a non-empty query';
|
|
210
|
+
const grouped = await litectx.recall(query, { kind: kinds, n, body: true });
|
|
211
|
+
const hits = kinds.flatMap((k) =>
|
|
212
|
+
(grouped && grouped[k] ? grouped[k] : []).map((h) => `[${k}] ${h.path}: ${h.body || ''}`),
|
|
213
|
+
);
|
|
214
|
+
return hits.length ? hits.join('\n') : 'no matches';
|
|
215
|
+
},
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* The `exact` handle tool (RC-5 rule path): a deterministic, embeddings-free code-side predicate filter over
|
|
221
|
+
* the slice-source — returns every record whose text contains ALL given terms (case-insensitive AND). This is
|
|
222
|
+
* the "code-side predicate filter" half of §9.2.1; it is complete over the slices it is given (no recall cap)
|
|
223
|
+
* but only as good as a lexical rule. (FTS-AND over a litectx instance is the alternative, but needs embeddings
|
|
224
|
+
* OFF to stay exact — deferred; the code-side filter is the embeddings-free path shipped now.)
|
|
225
|
+
* @param {Slice[]} corpus - The validated slice-source.
|
|
226
|
+
* @returns {ToolDef}
|
|
227
|
+
*/
|
|
228
|
+
function buildExactTool(corpus) {
|
|
229
|
+
const slices = normalizeCorpus(corpus);
|
|
230
|
+
return {
|
|
231
|
+
name: 'exact_match',
|
|
232
|
+
description:
|
|
233
|
+
'Filter the records by an EXACT lexical rule: returns every record whose text contains ALL of the given ' +
|
|
234
|
+
'terms (case-insensitive). Deterministic and complete over the records — use for precise rule-based ' +
|
|
235
|
+
'selection (e.g. records mentioning a specific identifier).',
|
|
236
|
+
parameters: {
|
|
237
|
+
type: 'object',
|
|
238
|
+
properties: { terms: { type: 'string', description: 'space-separated terms; ALL must be present' } },
|
|
239
|
+
required: ['terms'],
|
|
240
|
+
},
|
|
241
|
+
/** @param {{terms?: string}} args */
|
|
242
|
+
execute: async (args) => {
|
|
243
|
+
const terms = (typeof args?.terms === 'string' ? args.terms : '')
|
|
244
|
+
.toLowerCase()
|
|
245
|
+
.split(/\s+/)
|
|
246
|
+
.filter(Boolean);
|
|
247
|
+
if (!terms.length) return '[error] exact_match requires a non-empty terms string';
|
|
248
|
+
const hits = slices.filter((r) => {
|
|
249
|
+
const t = String(r.text).toLowerCase();
|
|
250
|
+
return terms.every((term) => t.includes(term));
|
|
251
|
+
});
|
|
252
|
+
return hits.length ? hits.map((r) => `${r.id}: ${r.text}`).join('\n') : 'no matches';
|
|
253
|
+
},
|
|
254
|
+
};
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/**
|
|
258
|
+
* The `scan` handle tool (RC-5 COMPLETE path, the per-query Family-A face of §10 step 7) — the deterministic
|
|
259
|
+
* counterpart to `search`/`exact` as a TOOL a worker may call per sub-query. Where `search_memory` returns the
|
|
260
|
+
* top FEW (capped, cannot count) and `exact_match` is a lexical rule, `scan_count` runs the full §9.2.1 scan
|
|
261
|
+
* (`scanCount`) over EVERY record and returns an exact, CODE-counted total — the only tool that does not silently
|
|
262
|
+
* undercount. The completeness routing lives in the DESCRIPTIONS, not a code-guard: this tool says "use for how
|
|
263
|
+
* many / all / count"; `search_memory` says "never use to count" — so a worker picks the complete path per
|
|
264
|
+
* sub-query (the shape can differ per sub-query with no adopter declaration). RC-9 honesty is preserved at the
|
|
265
|
+
* tool boundary: a dead window surfaces as an explicit `INCOMPLETE — the count is a floor`, never a clean number
|
|
266
|
+
* over a hole. A governance `HaltError` from the inner scan PROPAGATES (the Loop turns it into a clean halt —
|
|
267
|
+
* never wrapped to a `ToolError`).
|
|
268
|
+
* @param {Slice[] | (() => Promise<Slice[]>)} corpus - The slice-source (array or async, like `litectxCorpus`);
|
|
269
|
+
* materialized lazily on first call and cached for the tool's lifetime.
|
|
270
|
+
* @param {{provider: Provider, window?: number, passes?: number, ctx?: object, onLlmResult?: Function, policy?: Function}} opts
|
|
271
|
+
* @returns {ToolDef}
|
|
272
|
+
*/
|
|
273
|
+
function buildScanTool(corpus, opts) {
|
|
274
|
+
/** @type {Slice[]|null} */
|
|
275
|
+
let resolved = null; // materialize the source once (a re-scan re-reads the same set; RC-3 determinism)
|
|
276
|
+
return {
|
|
277
|
+
name: 'scan_count',
|
|
278
|
+
description:
|
|
279
|
+
'COUNT or list ALL records matching a predicate, completely. Use this whenever the question is "how many", ' +
|
|
280
|
+
'"all", "every", "count", or "total" over the records — it examines EVERY record (not just the top matches ' +
|
|
281
|
+
'like search) and returns an exact, code-counted total plus the matching ids. It is the only complete, ' +
|
|
282
|
+
'no-undercount path. COST: each call runs an LLM judge over EVERY record (a full-corpus pass) — it is the ' +
|
|
283
|
+
'most expensive handle, so call it once per population you need counted, not repeatedly or for needle ' +
|
|
284
|
+
'lookups (use search_memory for those). The right tool only when completeness matters.',
|
|
285
|
+
parameters: {
|
|
286
|
+
type: 'object',
|
|
287
|
+
properties: {
|
|
288
|
+
predicate: { type: 'string', description: 'natural-language description of which records match (what to count)' },
|
|
289
|
+
},
|
|
290
|
+
required: ['predicate'],
|
|
291
|
+
},
|
|
292
|
+
/** @param {{predicate?: string}} args */
|
|
293
|
+
execute: async (args) => {
|
|
294
|
+
const predicate = typeof args?.predicate === 'string' ? args.predicate.trim() : '';
|
|
295
|
+
if (!predicate) return '[error] scan_count requires a non-empty predicate';
|
|
296
|
+
if (resolved == null) {
|
|
297
|
+
const raw = typeof corpus === 'function' ? await corpus() : corpus;
|
|
298
|
+
resolved = normalizeCorpus(raw);
|
|
299
|
+
}
|
|
300
|
+
if (resolved.length === 0) return '[error] scan_count has no records to scan';
|
|
301
|
+
// HaltError from scanCount is INTENTIONALLY not caught — it must propagate so the Loop exits cleanly
|
|
302
|
+
// (governance), never be swallowed into a ToolError (the 0.18.0 invariant).
|
|
303
|
+
const scan = await scanCount(predicate, resolved, {
|
|
304
|
+
provider: opts.provider,
|
|
305
|
+
window: opts.window,
|
|
306
|
+
passes: opts.passes,
|
|
307
|
+
ctx: opts.ctx,
|
|
308
|
+
onLlmResult: opts.onLlmResult,
|
|
309
|
+
policy: opts.policy,
|
|
310
|
+
});
|
|
311
|
+
const head = `count=${scan.count} (scanned ${scan.scanned} records, window=${scan.window}, passes=${scan.passes})`;
|
|
312
|
+
// RC-9 at the tool boundary: a dead window means the count is a FLOOR — say so, never present a hole as exact.
|
|
313
|
+
const miss = scan.missingSlices.length
|
|
314
|
+
? `\nINCOMPLETE — ${scan.missingSlices.length} window(s) failed; the count is a floor, not exact (${scan.missingSlices.join('; ')})`
|
|
315
|
+
: '';
|
|
316
|
+
const ids = scan.matchedIds.length ? `\nmatching ids: ${scan.matchedIds.join(', ')}` : '';
|
|
317
|
+
return head + miss + ids;
|
|
318
|
+
},
|
|
319
|
+
};
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
// Default page size when materializing a litectx-resident corpus. Pages are an internal detail (the union/
|
|
323
|
+
// rotation scan needs the whole array in hand); a larger page = fewer round-trips, the caller's memory budget.
|
|
324
|
+
const ENUM_PAGE = 200;
|
|
325
|
+
|
|
326
|
+
/**
|
|
327
|
+
* Build a SCAN slice-source backed by a litectx-RESIDENT corpus (facts/episodes the agent already accrued) —
|
|
328
|
+
* the §10-step-7 deferral un-blocked by litectx 0.26's `enumerate` verb (spec:
|
|
329
|
+
* docs/01-product/litectx-enumerate-spec.md). Returns the generic async slice-source recurse's scan reads: a
|
|
330
|
+
* `() => Promise<Slice[]>` that pages through EVERY row of `kind` via `enumerate` (exhaustive — the rank-free
|
|
331
|
+
* read `recall` structurally cannot do) and maps each to `{id: item.path, text: item.body}`. recurse stays
|
|
332
|
+
* litectx-agnostic: it depends on this source SHAPE, never on litectx (same stance as `remember`'s Store
|
|
333
|
+
* socket) — an adopter can hand any `() => Promise<Slice[]>` (a DB, a file, an API) instead.
|
|
334
|
+
*
|
|
335
|
+
* Only for a corpus ALREADY in litectx for its own reasons — never ingest a fresh corpus just to enumerate it
|
|
336
|
+
* back (strictly worse than scanning the in-hand array; spec §1.1).
|
|
337
|
+
* @param {{enumerate: Function}} litectx
|
|
338
|
+
* @param {{kind?: 'fact'|'episode', pageSize?: number}} [opts]
|
|
339
|
+
* @returns {() => Promise<Slice[]>}
|
|
340
|
+
*/
|
|
341
|
+
function litectxCorpus(litectx, opts = {}) {
|
|
342
|
+
const kind = opts.kind === 'episode' ? 'episode' : 'fact'; // enumerate v1 is the memory axis (fact/episode)
|
|
343
|
+
const pageSize = Number.isInteger(opts.pageSize) && /** @type {number} */ (opts.pageSize) > 0 ? /** @type {number} */ (opts.pageSize) : ENUM_PAGE;
|
|
344
|
+
return async () => {
|
|
345
|
+
/** @type {Slice[]} */
|
|
346
|
+
const out = [];
|
|
347
|
+
let offset = 0;
|
|
348
|
+
for (;;) {
|
|
349
|
+
const page = await litectx.enumerate({ kind, offset, limit: pageSize, body: true });
|
|
350
|
+
for (const it of page.items) out.push({ id: String(it.path), text: it.body == null ? '' : String(it.body) });
|
|
351
|
+
if (page.nextOffset == null) break;
|
|
352
|
+
offset = page.nextOffset;
|
|
353
|
+
}
|
|
354
|
+
return out;
|
|
355
|
+
};
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
module.exports = {
|
|
359
|
+
scanCount,
|
|
360
|
+
judgeWindow,
|
|
361
|
+
classifySystem,
|
|
362
|
+
impliesCompleteness,
|
|
363
|
+
normalizeCorpus,
|
|
364
|
+
buildSearchTool,
|
|
365
|
+
buildExactTool,
|
|
366
|
+
buildScanTool,
|
|
367
|
+
litectxCorpus,
|
|
368
|
+
rotate,
|
|
369
|
+
SCAN_WINDOW,
|
|
370
|
+
SCAN_PASSES,
|
|
371
|
+
KNN_K,
|
|
372
|
+
};
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
export type Provider = import("../types").Provider;
|
|
2
|
+
/**
|
|
3
|
+
* The reducer dispatcher used by `recurse` (and, at build step 5, Family B over `runPlan` results[]).
|
|
4
|
+
* @param {string} task
|
|
5
|
+
* @param {any[]} results - The worker/child result values (NOT transcripts — copy-on-return holds upstream).
|
|
6
|
+
* @param {object} [opts]
|
|
7
|
+
* @param {Function} [opts.reduce] - A deterministic code-reduce: `({ task, results }) => any`. The §9.1
|
|
8
|
+
* aggregation path; takes precedence over `strategy`.
|
|
9
|
+
* @param {'concat'|'merge'} [opts.strategy='concat'] - Built-in strategy when no `reduce` fn is given.
|
|
10
|
+
* @param {Provider} [opts.provider] - Required for `merge`.
|
|
11
|
+
* @param {string|null} [opts.contract]
|
|
12
|
+
* @param {Function} [opts.onLlmResult]
|
|
13
|
+
* @param {Function} [opts.policy]
|
|
14
|
+
* @param {any} [opts.text] - The worker's own synthesized text (Family A), available to a `reduce` fn.
|
|
15
|
+
* @param {any[]} [opts.children] - The receipts nodes, available to a `reduce` fn.
|
|
16
|
+
* @param {any} [opts.ctx]
|
|
17
|
+
* @returns {Promise<any>}
|
|
18
|
+
*/
|
|
19
|
+
export function synthesize(task: string, results: any[], opts?: {
|
|
20
|
+
reduce?: Function | undefined;
|
|
21
|
+
strategy?: "concat" | "merge" | undefined;
|
|
22
|
+
provider?: import("../types").Provider | undefined;
|
|
23
|
+
contract?: string | null | undefined;
|
|
24
|
+
onLlmResult?: Function | undefined;
|
|
25
|
+
policy?: Function | undefined;
|
|
26
|
+
text?: any;
|
|
27
|
+
children?: any[] | undefined;
|
|
28
|
+
ctx?: any;
|
|
29
|
+
}): Promise<any>;
|
|
30
|
+
/**
|
|
31
|
+
* Deterministic lossless join — no LLM. The safe default when partials are independent and order-able.
|
|
32
|
+
* @param {any[]} results
|
|
33
|
+
* @returns {string}
|
|
34
|
+
*/
|
|
35
|
+
export function concatReduce(results: any[]): string;
|
|
36
|
+
/**
|
|
37
|
+
* Subjective merge — an ISOLATED Loop (fresh context window, a harsh synthesis persona) combines the partials
|
|
38
|
+
* into one answer. Use for prose / overlapping-findings synthesis; NEVER for arithmetic (§9.1 → use a
|
|
39
|
+
* `reduce` fn). Budget visibility: forwards usage to `onLlmResult`. A governance HaltError propagates clean
|
|
40
|
+
* (the caller decides), mirroring the Evaluator's agentic path.
|
|
41
|
+
* @param {string} task
|
|
42
|
+
* @param {any[]} results
|
|
43
|
+
* @param {{provider: Provider, contract?: string|null, onLlmResult?: Function, policy?: Function}} opts
|
|
44
|
+
* @returns {Promise<string>}
|
|
45
|
+
* @throws {HaltError} a governance cap halted the synthesis Loop.
|
|
46
|
+
*/
|
|
47
|
+
export function mergeReduce(task: string, results: any[], opts: {
|
|
48
|
+
provider: Provider;
|
|
49
|
+
contract?: string | null;
|
|
50
|
+
onLlmResult?: Function;
|
|
51
|
+
policy?: Function;
|
|
52
|
+
}): Promise<string>;
|
|
53
|
+
export const MERGE_PROMPT: string;
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// NB-3 — the synthesis/reduce step (RLM_PRD §4.3 / §10 step 4). A Family-A worker synthesizes in its own
|
|
4
|
+
// closing turn; Family B's `runPlan` returns results[] with nothing to combine them — this is that reducer,
|
|
5
|
+
// AND the seam a Family-A caller uses to OVERRIDE the model's synthesis when it shouldn't be trusted.
|
|
6
|
+
//
|
|
7
|
+
// §9.1 RESOLVED: numeric/AGGREGATION reduces must be DETERMINISTIC CODE — LLM arithmetic over found partials
|
|
8
|
+
// carried ~10–15% error even at FULL retrieval (spikes 1 & 2). So the `reduce` FUNCTION form is the
|
|
9
|
+
// aggregation path (the caller sums / maxes / dedups in JS); the Loop-driven `merge` strategy is reserved for
|
|
10
|
+
// genuinely SUBJECTIVE synthesis (combine prose, reconcile overlapping findings) where there is no exact
|
|
11
|
+
// answer to compute. `concat` is the lossless no-LLM default. The merge path is low-risk Loop reuse (the Loop
|
|
12
|
+
// is already live-proven) — the riskiest NB-3 assumption was "trust-code-over-LLM-arithmetic," already
|
|
13
|
+
// validated; this file does not re-POC it.
|
|
14
|
+
|
|
15
|
+
/** @typedef {import('../types').Provider} Provider */
|
|
16
|
+
|
|
17
|
+
const { Loop } = require('./loop');
|
|
18
|
+
const { HaltError } = require('./errors');
|
|
19
|
+
|
|
20
|
+
const MERGE_PROMPT =
|
|
21
|
+
'You are a synthesis engine. You are given a TASK and several PARTIAL RESULTS, each produced by a separate ' +
|
|
22
|
+
'worker over its own slice of the problem. Combine them into ONE coherent, de-duplicated answer to the ' +
|
|
23
|
+
'task. Preserve every distinct fact; drop exact duplicates; reconcile overlaps; do NOT invent anything not ' +
|
|
24
|
+
'present in the partials. Output only the combined answer.';
|
|
25
|
+
|
|
26
|
+
const asText = (/** @type {any} */ r) => (typeof r === 'string' ? r : JSON.stringify(r));
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Deterministic lossless join — no LLM. The safe default when partials are independent and order-able.
|
|
30
|
+
* @param {any[]} results
|
|
31
|
+
* @returns {string}
|
|
32
|
+
*/
|
|
33
|
+
function concatReduce(results) {
|
|
34
|
+
return results.map((r, i) => `### Part ${i + 1}\n${asText(r)}`).join('\n\n');
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Subjective merge — an ISOLATED Loop (fresh context window, a harsh synthesis persona) combines the partials
|
|
39
|
+
* into one answer. Use for prose / overlapping-findings synthesis; NEVER for arithmetic (§9.1 → use a
|
|
40
|
+
* `reduce` fn). Budget visibility: forwards usage to `onLlmResult`. A governance HaltError propagates clean
|
|
41
|
+
* (the caller decides), mirroring the Evaluator's agentic path.
|
|
42
|
+
* @param {string} task
|
|
43
|
+
* @param {any[]} results
|
|
44
|
+
* @param {{provider: Provider, contract?: string|null, onLlmResult?: Function, policy?: Function}} opts
|
|
45
|
+
* @returns {Promise<string>}
|
|
46
|
+
* @throws {HaltError} a governance cap halted the synthesis Loop.
|
|
47
|
+
*/
|
|
48
|
+
async function mergeReduce(task, results, opts) {
|
|
49
|
+
if (!opts || !opts.provider) throw new Error('[synthesize] strategy "merge" requires a provider');
|
|
50
|
+
const loop = new Loop({
|
|
51
|
+
provider: opts.provider,
|
|
52
|
+
system: MERGE_PROMPT,
|
|
53
|
+
policy: opts.policy || undefined,
|
|
54
|
+
onLlmResult: opts.onLlmResult || undefined,
|
|
55
|
+
throwOnError: false, // a synthesis fault surfaces below; a halt is re-thrown as a clean HaltError
|
|
56
|
+
});
|
|
57
|
+
const dod = opts.contract ? `\n\nDEFINITION OF DONE:\n${opts.contract}` : '';
|
|
58
|
+
const body =
|
|
59
|
+
`TASK:\n${task}${dod}\n\nPARTIAL RESULTS:\n` +
|
|
60
|
+
results.map((r, i) => `[${i + 1}] ${asText(r)}`).join('\n\n');
|
|
61
|
+
const out = await loop.run([{ role: 'user', content: body }]);
|
|
62
|
+
if (typeof out.error === 'string' && out.error.startsWith('halt:')) {
|
|
63
|
+
throw new HaltError('[synthesize] merge halted by governance', { rule: out.error.slice('halt:'.length) });
|
|
64
|
+
}
|
|
65
|
+
// A non-halt fault (e.g. provider error) is non-fatal here — fall back to the lossless concat rather than
|
|
66
|
+
// losing the partials entirely. recurse still reports honest completeness via its own paths.
|
|
67
|
+
return out.text || concatReduce(results);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* The reducer dispatcher used by `recurse` (and, at build step 5, Family B over `runPlan` results[]).
|
|
72
|
+
* @param {string} task
|
|
73
|
+
* @param {any[]} results - The worker/child result values (NOT transcripts — copy-on-return holds upstream).
|
|
74
|
+
* @param {object} [opts]
|
|
75
|
+
* @param {Function} [opts.reduce] - A deterministic code-reduce: `({ task, results }) => any`. The §9.1
|
|
76
|
+
* aggregation path; takes precedence over `strategy`.
|
|
77
|
+
* @param {'concat'|'merge'} [opts.strategy='concat'] - Built-in strategy when no `reduce` fn is given.
|
|
78
|
+
* @param {Provider} [opts.provider] - Required for `merge`.
|
|
79
|
+
* @param {string|null} [opts.contract]
|
|
80
|
+
* @param {Function} [opts.onLlmResult]
|
|
81
|
+
* @param {Function} [opts.policy]
|
|
82
|
+
* @param {any} [opts.text] - The worker's own synthesized text (Family A), available to a `reduce` fn.
|
|
83
|
+
* @param {any[]} [opts.children] - The receipts nodes, available to a `reduce` fn.
|
|
84
|
+
* @param {any} [opts.ctx]
|
|
85
|
+
* @returns {Promise<any>}
|
|
86
|
+
*/
|
|
87
|
+
async function synthesize(task, results, opts = {}) {
|
|
88
|
+
if (typeof opts.reduce === 'function') {
|
|
89
|
+
return opts.reduce({ task, results, text: opts.text, children: opts.children, ctx: opts.ctx });
|
|
90
|
+
}
|
|
91
|
+
const strategy = opts.strategy || 'concat';
|
|
92
|
+
if (strategy === 'merge') return mergeReduce(task, results, /** @type {any} */ (opts));
|
|
93
|
+
if (strategy === 'concat') return concatReduce(results);
|
|
94
|
+
throw new Error(`[synthesize] unknown strategy: ${strategy} (expected 'concat' | 'merge', or a reduce function)`);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
module.exports = { synthesize, concatReduce, mergeReduce, MERGE_PROMPT };
|