@agentpro/scroll 0.0.0-stage → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +55 -0
- package/LICENSE +21 -0
- package/README.md +293 -2
- package/SPEC.md +1079 -0
- package/bin/scroll.js +8 -0
- package/docs/prompt-caching.md +128 -0
- package/index.js +33 -0
- package/lib/audit.js +358 -0
- package/lib/auth.js +849 -0
- package/lib/chat-sdk/admin-controller.js +289 -0
- package/lib/chat-sdk/debouncer.js +94 -0
- package/lib/chat-sdk/group-grants.js +150 -0
- package/lib/chat-sdk/index.js +222 -0
- package/lib/chat-sdk/long-task-runner.js +413 -0
- package/lib/chat-session.js +1788 -0
- package/lib/chat-telemetry.js +557 -0
- package/lib/codex-app-server.js +851 -0
- package/lib/digest.js +112 -0
- package/lib/effects.js +157 -0
- package/lib/eval.js +408 -0
- package/lib/grounding.js +356 -0
- package/lib/installers.js +176 -0
- package/lib/knowledge.js +335 -0
- package/lib/ledger.js +38 -0
- package/lib/loop.js +197 -0
- package/lib/mcp-client.js +926 -0
- package/lib/memory-cli.js +66 -0
- package/lib/memory-tracker.js +1472 -0
- package/lib/memory.js +1279 -0
- package/lib/people-cli.js +81 -0
- package/lib/people.js +534 -0
- package/lib/permissions.js +272 -0
- package/lib/providers.js +1572 -0
- package/lib/runtime.js +928 -0
- package/lib/sandbox.js +744 -0
- package/lib/scheduler.js +402 -0
- package/lib/scroll.js +1225 -0
- package/lib/skill-cli.js +120 -0
- package/lib/skill-eval.js +156 -0
- package/lib/skills.js +860 -0
- package/lib/style.js +126 -0
- package/lib/tool-retriever.js +489 -0
- package/lib/tools.js +1833 -0
- package/lib/user.js +711 -0
- package/lib/vault.js +151 -0
- package/package.json +88 -4
- package/templates/agent/IDENTITY.md +56 -0
- package/templates/agent/SOUL.md +23 -0
- package/templates/agent/TOOLS.md +25 -0
- package/templates/agent/evals/case-02-citation-discipline.md +29 -0
- package/templates/agent/evals/case-03-boundary.md +27 -0
- package/templates/agent/evals/case-example.md +43 -0
- package/templates/agent/hard-rules.md +11 -0
- package/templates/agent/memory/lessons.md +5 -0
- package/templates/agent/memory/log/.gitkeep +0 -0
- package/templates/external-safety-contract.md +5 -0
- package/templates/long-task-planner.md +17 -0
- package/templates/work/LOOP.md +37 -0
- package/templates/work/WORK.md +50 -0
- package/types/chat.d.ts +61 -0
- package/types/index.d.ts +112 -0
package/lib/knowledge.js
ADDED
|
@@ -0,0 +1,335 @@
|
|
|
1
|
+
// SCROLL — deployment knowledge indexer & retriever (v1.10 §44).
|
|
2
|
+
//
|
|
3
|
+
// Why: large deployment knowledge bases (manuals, SOPs, tariffs, FAQs) cannot be dumped raw into
|
|
4
|
+
// every prompt — full-dump multiplies token cost and weakens reply grounding through cross-document
|
|
5
|
+
// number collisions. This module provides deterministic, zero-dependency section chunking and
|
|
6
|
+
// hybrid lexical retrieval (Unicode-folded BM25 + heading boost + bigram boost + exact code boost)
|
|
7
|
+
// over the deployment-root `knowledge/` folder (outside `agents/`, per §41).
|
|
8
|
+
//
|
|
9
|
+
// Pure ASCII source, zero dependencies, Node >= 20.
|
|
10
|
+
import crypto from 'node:crypto';
|
|
11
|
+
import fs from 'node:fs';
|
|
12
|
+
import path from 'node:path';
|
|
13
|
+
|
|
14
|
+
export const KNOWLEDGE_DEFAULTS = {
|
|
15
|
+
dir: 'knowledge',
|
|
16
|
+
maxChunkChars: 1600,
|
|
17
|
+
topK: 4,
|
|
18
|
+
maxChars: 6000,
|
|
19
|
+
extensions: ['.md', '.txt'],
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
// Fold diacritics to ASCII so accented languages match cleanly without non-ASCII literals in code
|
|
23
|
+
export function foldAscii(text) {
|
|
24
|
+
return String(text ?? '')
|
|
25
|
+
.normalize('NFD')
|
|
26
|
+
.replace(/[\u0300-\u036f]/g, '')
|
|
27
|
+
.replace(/\u0111/g, 'd')
|
|
28
|
+
.replace(/\u0110/g, 'D');
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// Common English + unambiguous ASCII-folded Vietnamese function words (pure ASCII).
|
|
32
|
+
// Avoid words whose ASCII-folded form collides with domain nouns (e.g. hoi=hoi nghi,
|
|
33
|
+
// bao=bao gia, thu=thu cung/phu thu, nha=nha hang, sao=5 sao, da=da).
|
|
34
|
+
const STOPWORDS = new Set([
|
|
35
|
+
'and', 'for', 'with', 'from', 'that', 'this', 'are', 'was', 'were', 'not', 'can', 'you',
|
|
36
|
+
'your', 'have', 'has', 'had', 'will', 'would', 'should', 'could', 'about', 'into', 'when', 'what',
|
|
37
|
+
'where', 'which', 'how', 'who', 'why', 'please', 'thanks', 'thank',
|
|
38
|
+
'va', 'cua', 'cho', 'voi', 'trong', 'nhung', 'cac', 'mot', 'nay', 'khi', 'thi', 'la', 'co',
|
|
39
|
+
'khong', 'duoc', 'de', 'tu', 'den', 'theo', 'hoac', 'neu', 'nhu', 've', 'tai', 'tren', 'duoi',
|
|
40
|
+
'sau', 'truoc', 'anh', 'chi', 'em', 'oi', 'nhe', 'vang', 'toi', 'minh', 'ben',
|
|
41
|
+
'nao', 'nhieu', 'cung', 'thoi',
|
|
42
|
+
]);
|
|
43
|
+
|
|
44
|
+
// Matches alphanumeric codes (SKU-A100, MICE-H2) AND uppercase hyphenated alpha codes (AUR-DLX, FDP-STD)
|
|
45
|
+
const CODE_TOKEN_RE = /(?<![A-Za-z0-9_-])(?:(?=[A-Z0-9_-]*\d)(?=[A-Z0-9_-]*[A-Z])[A-Z0-9]{2,}(?:[-_][A-Z0-9]+)*|[A-Z]{2,5}(?:-[A-Z0-9]{2,6})+)(?![A-Za-z0-9_-])/g;
|
|
46
|
+
|
|
47
|
+
function extractNormalizedAmounts(folded) {
|
|
48
|
+
const out = [];
|
|
49
|
+
// Grouped thousands: 2,250,000 or 2.250.000 -> 2250000
|
|
50
|
+
for (const m of folded.matchAll(/\b(\d{1,3}(?:[.,]\d{3})+)\b/g)) {
|
|
51
|
+
out.push(m[1].replace(/[.,]/g, ''));
|
|
52
|
+
}
|
|
53
|
+
// Million shorthand: 2,25 trieu / 2.25tr / 3,45tr -> 2250000 / 3450000
|
|
54
|
+
for (const m of folded.matchAll(/\b(\d{1,3})(?:[.,](\d{1,2}))?\s*(?:trieu|tr)\b/g)) {
|
|
55
|
+
const val = Math.round(Number(`${m[1]}.${m[2] || '0'}`) * 1_000_000);
|
|
56
|
+
if (Number.isFinite(val) && val >= 100_000) out.push(String(val));
|
|
57
|
+
}
|
|
58
|
+
// Thousand shorthand: 150k / 500k -> 150000 / 500000
|
|
59
|
+
for (const m of folded.matchAll(/\b(\d{1,4})k\b/g)) {
|
|
60
|
+
const val = Number(m[1]) * 1_000;
|
|
61
|
+
if (Number.isFinite(val) && val >= 10_000) out.push(String(val));
|
|
62
|
+
}
|
|
63
|
+
return out;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function tokenize(text) {
|
|
67
|
+
const folded = foldAscii(text).toLowerCase();
|
|
68
|
+
const raw = folded.match(/[a-z0-9]+/g) || [];
|
|
69
|
+
const base = raw.filter((t) => t.length >= 2 && !STOPWORDS.has(t));
|
|
70
|
+
const amounts = extractNormalizedAmounts(folded);
|
|
71
|
+
return amounts.length ? [...base, ...amounts] : base;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function extractCodes(text) {
|
|
75
|
+
const m = String(text ?? '').match(CODE_TOKEN_RE) || [];
|
|
76
|
+
return [...new Set(m.map((c) => c.toUpperCase()))];
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export function knowledgeConfig({ cwd = process.cwd(), dir = null, config = null } = {}) {
|
|
80
|
+
const k = (config && typeof config.knowledge === 'object' && config.knowledge) ? config.knowledge : {};
|
|
81
|
+
const rawDir = dir || k.dir || KNOWLEDGE_DEFAULTS.dir;
|
|
82
|
+
return {
|
|
83
|
+
cwd,
|
|
84
|
+
dir: path.isAbsolute(rawDir) ? rawDir : path.resolve(cwd, rawDir),
|
|
85
|
+
maxChunkChars: Number(k.max_chunk_chars || KNOWLEDGE_DEFAULTS.maxChunkChars),
|
|
86
|
+
topK: Number(k.top_k || KNOWLEDGE_DEFAULTS.topK),
|
|
87
|
+
maxChars: Number(k.max_chars || KNOWLEDGE_DEFAULTS.maxChars),
|
|
88
|
+
extensions: Array.isArray(k.extensions) && k.extensions.length ? k.extensions : KNOWLEDGE_DEFAULTS.extensions,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function walkKnowledgeFiles(rootDir, extensions) {
|
|
93
|
+
const out = [];
|
|
94
|
+
if (!fs.existsSync(rootDir)) return out;
|
|
95
|
+
const exts = new Set(extensions.map((e) => String(e).toLowerCase()));
|
|
96
|
+
(function walk(dir, relBase) {
|
|
97
|
+
let ents;
|
|
98
|
+
try { ents = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }
|
|
99
|
+
ents.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
|
|
100
|
+
for (const e of ents) {
|
|
101
|
+
if (e.name.startsWith('.') || e.isSymbolicLink()) continue;
|
|
102
|
+
const full = path.join(dir, e.name);
|
|
103
|
+
const rel = relBase ? `${relBase}/${e.name}` : e.name;
|
|
104
|
+
if (e.isDirectory()) walk(full, rel);
|
|
105
|
+
else if (e.isFile() && exts.has(path.extname(e.name).toLowerCase())) out.push({ full, rel });
|
|
106
|
+
}
|
|
107
|
+
})(rootDir, '');
|
|
108
|
+
return out;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export function indexKnowledge({ cwd = process.cwd(), dir = null, config = null, maxChunkChars = null } = {}) {
|
|
112
|
+
const cfg = knowledgeConfig({ cwd, dir, config });
|
|
113
|
+
const chunkCap = Math.max(400, Number(maxChunkChars || cfg.maxChunkChars));
|
|
114
|
+
const files = walkKnowledgeFiles(cfg.dir, cfg.extensions);
|
|
115
|
+
const chunks = [];
|
|
116
|
+
const h = crypto.createHash('sha256');
|
|
117
|
+
let totalBytes = 0;
|
|
118
|
+
let totalWords = 0;
|
|
119
|
+
let totalCorpusChars = 0;
|
|
120
|
+
|
|
121
|
+
for (const { full, rel } of files) {
|
|
122
|
+
const raw = fs.readFileSync(full, 'utf8').replace(/\r\n?/g, '\n');
|
|
123
|
+
h.update(`${rel}\0`).update(raw);
|
|
124
|
+
totalBytes += Buffer.byteLength(raw, 'utf8');
|
|
125
|
+
totalCorpusChars += raw.length;
|
|
126
|
+
totalWords += raw.trim() ? raw.trim().split(/\s+/).length : 0;
|
|
127
|
+
|
|
128
|
+
const lines = raw.split('\n');
|
|
129
|
+
const docTitle = (lines.find((l) => /^#\s+/.test(l)) || rel).replace(/^#\s+/, '').trim();
|
|
130
|
+
|
|
131
|
+
let curH2 = '';
|
|
132
|
+
let curH3 = '';
|
|
133
|
+
let buf = [];
|
|
134
|
+
|
|
135
|
+
const pushChunk = (heading, bodyText) => {
|
|
136
|
+
const clean = bodyText.trim();
|
|
137
|
+
if (!clean || clean.length < 30) return;
|
|
138
|
+
const text = `[Source: ${rel} | ${heading}]\n${clean}`;
|
|
139
|
+
const tokens = tokenize(`${docTitle} ${heading} ${clean}`);
|
|
140
|
+
const headingTokens = new Set(tokenize(`${docTitle} ${heading}`));
|
|
141
|
+
const codes = new Set(extractCodes(text));
|
|
142
|
+
const tf = new Map();
|
|
143
|
+
for (const tok of tokens) tf.set(tok, (tf.get(tok) || 0) + 1);
|
|
144
|
+
chunks.push({
|
|
145
|
+
id: `${rel}#${chunks.length + 1}`,
|
|
146
|
+
file: rel,
|
|
147
|
+
docTitle,
|
|
148
|
+
heading,
|
|
149
|
+
text,
|
|
150
|
+
folded: foldAscii(text).toLowerCase(),
|
|
151
|
+
headingFolded: foldAscii(`${docTitle} ${heading}`).toLowerCase(),
|
|
152
|
+
chars: text.length,
|
|
153
|
+
tokensCount: tokens.length || 1,
|
|
154
|
+
tf,
|
|
155
|
+
headingTokens,
|
|
156
|
+
codes,
|
|
157
|
+
});
|
|
158
|
+
};
|
|
159
|
+
|
|
160
|
+
const flush = () => {
|
|
161
|
+
const body = buf.join('\n').trim();
|
|
162
|
+
if (!body || body.length < 30) return;
|
|
163
|
+
const heading = [curH2, curH3].filter(Boolean).join(' > ') || docTitle;
|
|
164
|
+
if (body.length <= chunkCap) {
|
|
165
|
+
pushChunk(heading, body);
|
|
166
|
+
return;
|
|
167
|
+
}
|
|
168
|
+
// Sub-split oversized sections/tables/FAQ lists while repeating active Markdown table header
|
|
169
|
+
const secLines = body.split('\n');
|
|
170
|
+
let tableHeader = null;
|
|
171
|
+
let part = [];
|
|
172
|
+
let partLen = 0;
|
|
173
|
+
let partIdx = 1;
|
|
174
|
+
for (let i = 0; i < secLines.length; i++) {
|
|
175
|
+
const line = secLines[i];
|
|
176
|
+
if (i + 1 < secLines.length && /^\|.*\|$/.test(line.trim()) && /^\|[\s:|-]+\|$/.test(secLines[i + 1].trim())) {
|
|
177
|
+
tableHeader = [line, secLines[i + 1]];
|
|
178
|
+
} else if (!/^\|/.test(line.trim())) {
|
|
179
|
+
tableHeader = null;
|
|
180
|
+
}
|
|
181
|
+
if (partLen + line.length + 1 > chunkCap && part.length >= 2) {
|
|
182
|
+
pushChunk(`${heading} (part ${partIdx})`, part.join('\n'));
|
|
183
|
+
partIdx++;
|
|
184
|
+
part = (tableHeader && /^\|/.test(line.trim())) ? [...tableHeader, line] : [line];
|
|
185
|
+
partLen = part.reduce((s, l) => s + l.length + 1, 0);
|
|
186
|
+
} else {
|
|
187
|
+
part.push(line);
|
|
188
|
+
partLen += line.length + 1;
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
if (part.length) pushChunk(`${heading} (part ${partIdx})`, part.join('\n'));
|
|
192
|
+
};
|
|
193
|
+
|
|
194
|
+
for (const line of lines) {
|
|
195
|
+
if (/^##\s+/.test(line)) { flush(); curH2 = line.replace(/^##\s+/, '').trim(); curH3 = ''; buf = [line]; }
|
|
196
|
+
else if (/^###\s+/.test(line)) { flush(); curH3 = line.replace(/^###\s+/, '').trim(); buf = [line]; }
|
|
197
|
+
else buf.push(line);
|
|
198
|
+
}
|
|
199
|
+
flush();
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
const N = chunks.length || 1;
|
|
203
|
+
const df = new Map();
|
|
204
|
+
let sumLen = 0;
|
|
205
|
+
for (const ch of chunks) {
|
|
206
|
+
sumLen += ch.tokensCount;
|
|
207
|
+
for (const tok of ch.tf.keys()) df.set(tok, (df.get(tok) || 0) + 1);
|
|
208
|
+
}
|
|
209
|
+
const avgLen = sumLen / N;
|
|
210
|
+
const idf = new Map();
|
|
211
|
+
for (const [tok, count] of df.entries()) {
|
|
212
|
+
idf.set(tok, Math.log(1 + (N - count + 0.5) / (count + 0.5)));
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
return {
|
|
216
|
+
active: chunks.length > 0,
|
|
217
|
+
dir: cfg.dir,
|
|
218
|
+
hash: files.length ? h.digest('hex').slice(0, 16) : null,
|
|
219
|
+
filesCount: files.length,
|
|
220
|
+
totalBytes,
|
|
221
|
+
totalWords,
|
|
222
|
+
totalCorpusChars,
|
|
223
|
+
chunksCount: chunks.length,
|
|
224
|
+
chunks,
|
|
225
|
+
avgLen,
|
|
226
|
+
idf,
|
|
227
|
+
cfg,
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
export function retrieveKnowledge(index, query, {
|
|
232
|
+
topK = null,
|
|
233
|
+
maxChars = null,
|
|
234
|
+
prevQuery = '',
|
|
235
|
+
synonyms = null,
|
|
236
|
+
} = {}) {
|
|
237
|
+
const t0 = performance.now();
|
|
238
|
+
if (!index || !index.active) {
|
|
239
|
+
return { query: String(query ?? ''), hits: [], chunks: [], selectedChars: 0, totalCorpusChars: 0, durationMs: 0 };
|
|
240
|
+
}
|
|
241
|
+
const limitK = Math.max(1, Number(topK || index.cfg?.topK || KNOWLEDGE_DEFAULTS.topK));
|
|
242
|
+
const charCap = Math.max(600, Number(maxChars || index.cfg?.maxChars || KNOWLEDGE_DEFAULTS.maxChars));
|
|
243
|
+
|
|
244
|
+
let expanded = String(query ?? '');
|
|
245
|
+
if (Array.isArray(synonyms) && synonyms.length) {
|
|
246
|
+
const foldedQ = foldAscii(expanded).toLowerCase();
|
|
247
|
+
for (const rule of synonyms) {
|
|
248
|
+
if (rule && rule.pattern instanceof RegExp && rule.pattern.test(foldedQ) && rule.terms) {
|
|
249
|
+
expanded += ' ' + String(rule.terms);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
const qTokens = tokenize(expanded);
|
|
255
|
+
const isShortFollowUp = qTokens.length <= 6;
|
|
256
|
+
const prevTokens = (prevQuery && isShortFollowUp) ? tokenize(prevQuery).slice(0, 12) : [];
|
|
257
|
+
const qBigrams = [];
|
|
258
|
+
for (let i = 0; i < qTokens.length - 1; i++) qBigrams.push(`${qTokens[i]} ${qTokens[i + 1]}`);
|
|
259
|
+
const curCodes = extractCodes(expanded);
|
|
260
|
+
const prevCodes = (prevQuery && curCodes.length === 0 && isShortFollowUp) ? extractCodes(prevQuery) : [];
|
|
261
|
+
const k1 = 1.5;
|
|
262
|
+
const b = 0.75;
|
|
263
|
+
|
|
264
|
+
const scored = [];
|
|
265
|
+
for (const ch of index.chunks) {
|
|
266
|
+
let score = 0;
|
|
267
|
+
for (const tok of qTokens) {
|
|
268
|
+
const f = ch.tf.get(tok) || 0;
|
|
269
|
+
if (f > 0) {
|
|
270
|
+
const w = index.idf.get(tok) || 0.5;
|
|
271
|
+
score += w * ((f * (k1 + 1)) / (f + k1 * (1 - b + b * (ch.tokensCount / index.avgLen))));
|
|
272
|
+
if (ch.headingTokens.has(tok)) score += w * 1.1;
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
for (const tok of prevTokens) {
|
|
276
|
+
const f = ch.tf.get(tok) || 0;
|
|
277
|
+
if (f > 0) score += (index.idf.get(tok) || 0.5) * 0.2;
|
|
278
|
+
}
|
|
279
|
+
for (const code of curCodes) {
|
|
280
|
+
if (ch.codes.has(code)) score += 12.0;
|
|
281
|
+
}
|
|
282
|
+
for (const code of prevCodes) {
|
|
283
|
+
if (ch.codes.has(code)) score += 3.0;
|
|
284
|
+
}
|
|
285
|
+
for (const bg of qBigrams) {
|
|
286
|
+
if (ch.folded.includes(bg)) score += 1.6;
|
|
287
|
+
if (ch.headingFolded && ch.headingFolded.includes(bg)) score += 2.8;
|
|
288
|
+
}
|
|
289
|
+
if (score > 0) scored.push({ chunk: ch, score });
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
scored.sort((a, b) => b.score - a.score);
|
|
293
|
+
const selected = [];
|
|
294
|
+
let usedChars = 0;
|
|
295
|
+
for (const item of scored) {
|
|
296
|
+
if (selected.length >= limitK) break;
|
|
297
|
+
if (selected.length > 0 && usedChars + item.chunk.chars > charCap) continue;
|
|
298
|
+
selected.push(item);
|
|
299
|
+
usedChars += item.chunk.chars;
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
const durationMs = +(performance.now() - t0).toFixed(2);
|
|
303
|
+
const hits = selected.map((s) => ({
|
|
304
|
+
id: s.chunk.id,
|
|
305
|
+
file: s.chunk.file,
|
|
306
|
+
heading: s.chunk.heading,
|
|
307
|
+
score: +s.score.toFixed(2),
|
|
308
|
+
chars: s.chunk.chars,
|
|
309
|
+
text: s.chunk.text,
|
|
310
|
+
}));
|
|
311
|
+
return {
|
|
312
|
+
query: String(query ?? ''),
|
|
313
|
+
hits,
|
|
314
|
+
chunks: hits.map((h) => h.text),
|
|
315
|
+
selectedChars: usedChars,
|
|
316
|
+
totalCorpusChars: index.totalCorpusChars,
|
|
317
|
+
durationMs,
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
export function formatKnowledgeBlock(chunks, { hash = null } = {}) {
|
|
322
|
+
const list = Array.isArray(chunks) ? chunks.map(String).filter(Boolean) : [];
|
|
323
|
+
if (!list.length) return '';
|
|
324
|
+
const attr = hash ? ` snapshot="${hash}"` : '';
|
|
325
|
+
return `<retrieved-knowledge${attr}>\n${list.join('\n\n')}\n</retrieved-knowledge>`;
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
export function openKnowledge(opts = {}) {
|
|
329
|
+
const index = indexKnowledge(opts);
|
|
330
|
+
return {
|
|
331
|
+
...index,
|
|
332
|
+
retrieve: (query, rOpts = {}) => retrieveKnowledge(index, query, rOpts),
|
|
333
|
+
format: (chunks) => formatKnowledgeBlock(chunks, { hash: index.hash }),
|
|
334
|
+
};
|
|
335
|
+
}
|
package/lib/ledger.js
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
// SCROLL v1.5 — LEDGER: the human-readable tracking table that spans runs.
|
|
2
|
+
// `local://` = CSV/MD (core, zero dependency). External schemes write ledger.jsonl
|
|
3
|
+
// into loopDir for an app-layer ADAPTER to push (the core embeds no third-party SDK).
|
|
4
|
+
import fs from 'node:fs';
|
|
5
|
+
import path from 'node:path';
|
|
6
|
+
|
|
7
|
+
export const LEDGER_HEADER = ['ts', 'loop', 'iteration', 'task', 'status', 'tries', 'proof', 'digest'];
|
|
8
|
+
|
|
9
|
+
function csvCell(v) { const s = String(v == null ? '' : v); return /[",\n]/.test(s) ? '"' + s.replace(/"/g, '""') + '"' : s; }
|
|
10
|
+
|
|
11
|
+
// "local://runs/ledger.csv" → {scheme:'local', target:'runs/ledger.csv'}
|
|
12
|
+
export function parseLedger(spec) {
|
|
13
|
+
if (!spec) return null;
|
|
14
|
+
const m = String(spec).match(/^(\w+):\/\/(.*)$/);
|
|
15
|
+
return m ? { scheme: m[1], target: m[2] } : { scheme: 'local', target: String(spec) };
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
// Append one ledger row. Always writes ledger.jsonl (audit + adapter). `local` also writes CSV/MD directly.
|
|
19
|
+
export function appendLedgerRow(ledgerSpec, cwd, loopDir, row) {
|
|
20
|
+
const jsonlPath = path.join(loopDir, 'ledger.jsonl');
|
|
21
|
+
fs.appendFileSync(jsonlPath, JSON.stringify(row) + '\n');
|
|
22
|
+
const L = parseLedger(ledgerSpec);
|
|
23
|
+
if (!L) return { jsonl: jsonlPath, external: false };
|
|
24
|
+
if (L.scheme === 'local') {
|
|
25
|
+
const p = path.resolve(cwd, L.target);
|
|
26
|
+
fs.mkdirSync(path.dirname(p), { recursive: true });
|
|
27
|
+
if (/\.md$/.test(p)) {
|
|
28
|
+
if (!fs.existsSync(p)) fs.writeFileSync(p, '| ' + LEDGER_HEADER.join(' | ') + ' |\n| ' + LEDGER_HEADER.map(() => '---').join(' | ') + ' |\n');
|
|
29
|
+
fs.appendFileSync(p, '| ' + LEDGER_HEADER.map((h) => csvCell(row[h]).replace(/\|/g, '\\|')).join(' | ') + ' |\n');
|
|
30
|
+
} else {
|
|
31
|
+
if (!fs.existsSync(p)) fs.writeFileSync(p, LEDGER_HEADER.join(',') + '\n');
|
|
32
|
+
fs.appendFileSync(p, LEDGER_HEADER.map((h) => csvCell(row[h])).join(',') + '\n');
|
|
33
|
+
}
|
|
34
|
+
return { local: p, external: false };
|
|
35
|
+
}
|
|
36
|
+
// External schemes: an app-layer adapter reads ledger.jsonl and pushes it (proof: upload → URL → file field).
|
|
37
|
+
return { jsonl: jsonlPath, external: true, scheme: L.scheme, target: L.target };
|
|
38
|
+
}
|
package/lib/loop.js
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
// SCROLL — the OUTER loop (v1.4). This is the "loop engineering" layer.
|
|
2
|
+
//
|
|
3
|
+
// `scroll run` is the INNER loop: one harnessed execution of a WORK.md.
|
|
4
|
+
// `LOOP.md` + `scroll loop` is the OUTER loop: the system that decides WHEN to start an inner
|
|
5
|
+
// loop, WHERE it finds work, and WHEN to stop — so an agent finds its own work, does it, verifies
|
|
6
|
+
// it, and remembers it, without a human prompting each turn. No engine: it rides `scroll run`,
|
|
7
|
+
// the filesystem, and the host scheduler (cron / scheduled-tasks / HEARTBEAT).
|
|
8
|
+
//
|
|
9
|
+
// Safety is non-negotiable: a LOOP.md MUST declare at least one stop condition, the outer loop
|
|
10
|
+
// NEVER escalates privilege (every action inside still goes through the permission matrix), and a
|
|
11
|
+
// digest is written after every inner run.
|
|
12
|
+
import fs from 'node:fs';
|
|
13
|
+
import path from 'node:path';
|
|
14
|
+
import { execSync } from 'node:child_process';
|
|
15
|
+
import { parseFrontmatter } from './scroll.js';
|
|
16
|
+
import { writeDigest } from './digest.js';
|
|
17
|
+
import { appendLedgerRow } from './ledger.js';
|
|
18
|
+
|
|
19
|
+
export function parseLoop(raw) {
|
|
20
|
+
const { frontmatter, body } = parseFrontmatter(raw);
|
|
21
|
+
return { frontmatter: frontmatter || {}, goal: body || '' };
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
// Validate a LOOP.md. A loop with no stop condition is refused — that is the whole safety point.
|
|
25
|
+
export function validateLoop(fm) {
|
|
26
|
+
const errors = [];
|
|
27
|
+
if (!fm || typeof fm !== 'object') { errors.push('LOOP.md has no frontmatter'); return errors; }
|
|
28
|
+
if (!fm.id) errors.push('LOOP: missing id');
|
|
29
|
+
if (!fm.controller) errors.push('LOOP: missing controller (one owner)');
|
|
30
|
+
const sc = fm.stop_conditions || {};
|
|
31
|
+
const hasStop = ['max_runs_per_day', 'budget_usd_per_day', 'until', 'max_runs'].some((k) => sc[k] != null)
|
|
32
|
+
|| (Array.isArray(sc.halt_on) && sc.halt_on.length > 0);
|
|
33
|
+
if (!hasStop) errors.push('LOOP: stop_conditions must declare at least one limit (a loop with no stop condition is refused)');
|
|
34
|
+
if (!fm.work_source) errors.push('LOOP: missing work_source');
|
|
35
|
+
// v1.5 — new fields (all OPTIONAL; type-checked only when present → backward compatible).
|
|
36
|
+
if (fm.max_tries_per_task != null && !(Number(fm.max_tries_per_task) >= 1)) errors.push('LOOP: max_tries_per_task must be a number >= 1');
|
|
37
|
+
if (fm.ledger != null && typeof fm.ledger !== 'string') errors.push('LOOP: ledger must be a string (scheme://target)');
|
|
38
|
+
if (fm.planner != null && typeof fm.planner !== 'string') errors.push('LOOP: planner must be a string (command)');
|
|
39
|
+
return errors;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
// Resolve the WORK for one iteration from the loop's work_source.
|
|
43
|
+
// type: work_file → reads an existing WORK.md (returns { workFile })
|
|
44
|
+
// type: inline → uses fm.work (a WORK.md string) written to a temp file
|
|
45
|
+
// type: glob → turns matched files into one task each (returns { workFile } generated)
|
|
46
|
+
function buildIterationWork(fm, cwd, iterDir) {
|
|
47
|
+
const ws = fm.work_source || {};
|
|
48
|
+
if (ws.type === 'work_file' && ws.query) {
|
|
49
|
+
return { workFile: path.resolve(cwd, ws.query) };
|
|
50
|
+
}
|
|
51
|
+
if (ws.type === 'inline' && fm.work) {
|
|
52
|
+
const p = path.join(iterDir, 'WORK.md');
|
|
53
|
+
fs.writeFileSync(p, fm.work);
|
|
54
|
+
return { workFile: p };
|
|
55
|
+
}
|
|
56
|
+
if (ws.type === 'glob' && ws.query) {
|
|
57
|
+
const dir = path.resolve(cwd, path.dirname(ws.query));
|
|
58
|
+
const re = new RegExp('^' + path.basename(ws.query).replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*') + '$');
|
|
59
|
+
const matched = fs.existsSync(dir) ? fs.readdirSync(dir).filter((f) => re.test(f)) : [];
|
|
60
|
+
const tasks = matched.map((f, i) => [
|
|
61
|
+
'```task',
|
|
62
|
+
`id: item-${i + 1}`,
|
|
63
|
+
`title: process ${f}`,
|
|
64
|
+
'status: todo',
|
|
65
|
+
`owner: ${fm.controller}`,
|
|
66
|
+
`objective: Process the work item in ${f}.`,
|
|
67
|
+
'output_format: decision-ready markdown',
|
|
68
|
+
'boundaries: [stay within the item scope]',
|
|
69
|
+
'blockedBy: []',
|
|
70
|
+
i === matched.length - 1 ? 'final: true' : '',
|
|
71
|
+
'```',
|
|
72
|
+
].filter(Boolean).join('\n'));
|
|
73
|
+
const md = `---\ncontroller: ${fm.controller}\n---\n# generated by scroll loop\n\n${tasks.join('\n\n')}\n`;
|
|
74
|
+
const p = path.join(iterDir, 'WORK.md');
|
|
75
|
+
fs.writeFileSync(p, md);
|
|
76
|
+
return { workFile: p, items: matched.length };
|
|
77
|
+
}
|
|
78
|
+
return { workFile: null };
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function todayKey() { return new Date().toISOString().slice(0, 10); }
|
|
82
|
+
|
|
83
|
+
// Run the outer loop. `runWork` is injected so this stays testable without a provider.
|
|
84
|
+
// opts: { cwd, once, maxRuns, dryRun, providerName, model, runWork, sleepMs }
|
|
85
|
+
export async function runLoop(loopFile, opts = {}) {
|
|
86
|
+
const cwd = opts.cwd || process.cwd();
|
|
87
|
+
const raw = fs.readFileSync(path.resolve(cwd, loopFile), 'utf8');
|
|
88
|
+
const { frontmatter: fm, goal } = parseLoop(raw);
|
|
89
|
+
const errors = validateLoop(fm);
|
|
90
|
+
if (errors.length) { const e = new Error('invalid LOOP.md:\n ' + errors.join('\n ')); e.loopErrors = errors; throw e; }
|
|
91
|
+
|
|
92
|
+
const loopDir = path.join(cwd, 'runs', `loop-${fm.id}`);
|
|
93
|
+
fs.mkdirSync(loopDir, { recursive: true });
|
|
94
|
+
const statePath = path.join(loopDir, 'state.json');
|
|
95
|
+
const eventsPath = path.join(loopDir, 'loop-events.jsonl');
|
|
96
|
+
let state = { day: todayKey(), runsToday: 0, spentToday: 0, totalRuns: 0 };
|
|
97
|
+
if (fs.existsSync(statePath)) { try { state = JSON.parse(fs.readFileSync(statePath, 'utf8')); } catch { /* fresh */ } }
|
|
98
|
+
if (state.day !== todayKey()) state = { day: todayKey(), runsToday: 0, spentToday: 0, totalRuns: state.totalRuns || 0 };
|
|
99
|
+
|
|
100
|
+
const lemit = (type, data = {}) => {
|
|
101
|
+
fs.appendFileSync(eventsPath, JSON.stringify({ ts: new Date().toISOString(), loop: fm.id, type, data }) + '\n');
|
|
102
|
+
if (opts.verbose) console.log(` loop:${type}`, data.reason || data.runId || '');
|
|
103
|
+
};
|
|
104
|
+
lemit('loop_started', { id: fm.id, controller: fm.controller, goal: goal.slice(0, 200) });
|
|
105
|
+
|
|
106
|
+
// v1.5 — PLANNER (self-planning): run a command that generates the backlog BEFORE looping.
|
|
107
|
+
if (fm.planner && !opts.dryRun) {
|
|
108
|
+
try { execSync(String(fm.planner), { cwd, stdio: 'pipe' }); lemit('planner_ran', { planner: fm.planner }); }
|
|
109
|
+
catch (e) { lemit('planner_failed', { planner: fm.planner, error: String(e.message || e).slice(0, 200) }); }
|
|
110
|
+
}
|
|
111
|
+
const MAXT = Math.max(1, Number(fm.max_tries_per_task ?? 3)); // v1.5 — try-budget PER-TASK (default 3)
|
|
112
|
+
|
|
113
|
+
const sc = fm.stop_conditions || {};
|
|
114
|
+
const runWork = opts.runWork; // injected (lib/runtime.js runWork) — keeps loop provider-free for tests
|
|
115
|
+
if (!runWork) throw new Error('runLoop requires opts.runWork');
|
|
116
|
+
|
|
117
|
+
const stopCheck = () => {
|
|
118
|
+
if (sc.until && todayKey() > String(sc.until)) return `until ${sc.until} passed`;
|
|
119
|
+
if (sc.max_runs != null && state.totalRuns >= Number(sc.max_runs)) return `max_runs ${sc.max_runs} reached`;
|
|
120
|
+
if (sc.max_runs_per_day != null && state.runsToday >= Number(sc.max_runs_per_day)) return `max_runs_per_day ${sc.max_runs_per_day} reached`;
|
|
121
|
+
if (sc.budget_usd_per_day != null && state.spentToday >= Number(sc.budget_usd_per_day)) return `budget_usd_per_day ${sc.budget_usd_per_day} reached`;
|
|
122
|
+
return null;
|
|
123
|
+
};
|
|
124
|
+
|
|
125
|
+
const summary = { id: fm.id, iterations: 0, runs: [], stopped: null };
|
|
126
|
+
const haltOn = new Set(Array.isArray(sc.halt_on) ? sc.halt_on : []);
|
|
127
|
+
const maxIters = opts.once ? 1 : (opts.maxRuns != null ? Number(opts.maxRuns) : Number(sc.max_runs ?? sc.max_runs_per_day ?? 1));
|
|
128
|
+
|
|
129
|
+
for (let i = 0; i < maxIters; i++) {
|
|
130
|
+
const stop = stopCheck();
|
|
131
|
+
if (stop) { summary.stopped = stop; lemit('loop_stopped', { reason: stop }); break; }
|
|
132
|
+
|
|
133
|
+
const iterDir = path.join(loopDir, `iter-${state.totalRuns + 1}`);
|
|
134
|
+
fs.mkdirSync(iterDir, { recursive: true });
|
|
135
|
+
const { workFile, items } = buildIterationWork(fm, cwd, iterDir);
|
|
136
|
+
if (!workFile || !fs.existsSync(workFile)) { summary.stopped = 'no work available'; lemit('loop_stopped', { reason: 'no work' }); break; }
|
|
137
|
+
if (items === 0) { summary.stopped = 'work_source empty'; lemit('loop_idle', { reason: 'no items' }); break; }
|
|
138
|
+
|
|
139
|
+
lemit('iteration_started', { n: state.totalRuns + 1, workFile: path.relative(cwd, workFile) });
|
|
140
|
+
if (opts.dryRun) {
|
|
141
|
+
summary.runs.push({ iteration: state.totalRuns + 1, dryRun: true, workFile: path.relative(cwd, workFile) });
|
|
142
|
+
summary.iterations++; state.totalRuns++; state.runsToday++;
|
|
143
|
+
try { appendLedgerRow(fm.ledger, cwd, loopDir, { ts: new Date().toISOString(), loop: fm.id, iteration: state.totalRuns, task: path.basename(workFile), status: 'dry-run', tries: 0, proof: '', digest: '' }); } catch { /* ignore */ }
|
|
144
|
+
lemit('iteration_dry_run', { n: state.totalRuns });
|
|
145
|
+
continue;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
const runDir = path.join(iterDir, 'run');
|
|
149
|
+
const budget = fm.budget || {};
|
|
150
|
+
const caps = {};
|
|
151
|
+
if (budget.per_run && budget.per_run.max_usd) caps.maxUsd = Number(budget.per_run.max_usd);
|
|
152
|
+
|
|
153
|
+
// v1.5 — PER-TASK try-budget: up to MAXT attempts; stop early on pass (or gate_denied).
|
|
154
|
+
let res, tries = 0, passed = false, gateDenied = false;
|
|
155
|
+
for (tries = 1; tries <= MAXT; tries++) {
|
|
156
|
+
res = await runWork({
|
|
157
|
+
cwd, workFile, runDir,
|
|
158
|
+
providerName: opts.providerName || 'auto', model: opts.model,
|
|
159
|
+
language: fm.language, caps,
|
|
160
|
+
});
|
|
161
|
+
state.spentToday += (res.metrics && res.metrics.costUsd) || 0;
|
|
162
|
+
gateDenied = res.status === 'paused' && haltOn.has('gate_denied');
|
|
163
|
+
passed = !(res.verification && res.verification.passed === false);
|
|
164
|
+
if (passed || gateDenied) break;
|
|
165
|
+
if (tries < MAXT) lemit('task_retry', { n: state.totalRuns + 1, try: tries, of: MAXT });
|
|
166
|
+
}
|
|
167
|
+
state.runsToday++; state.totalRuns++; summary.iterations++;
|
|
168
|
+
const triesDone = (passed || gateDenied) ? tries : MAXT; // the for-loop overshoots tries by 1 when exhausted → clamp back to MAXT
|
|
169
|
+
|
|
170
|
+
const digestPath = path.join(runDir, 'digest.md');
|
|
171
|
+
const taskStatus = gateDenied ? 'gate_denied' : (passed ? 'done' : 'failed');
|
|
172
|
+
// v1.5 — LEDGER: one row per output (pass OR fail). On fail, proof is STILL recorded (evidence of the attempt).
|
|
173
|
+
const ledgerRow = {
|
|
174
|
+
ts: new Date().toISOString(), loop: fm.id, iteration: state.totalRuns,
|
|
175
|
+
task: path.basename(workFile), status: taskStatus, tries: triesDone,
|
|
176
|
+
proof: res.proof ? path.relative(cwd, path.resolve(cwd, String(res.proof))) : '',
|
|
177
|
+
digest: fs.existsSync(digestPath) ? path.relative(cwd, digestPath) : '',
|
|
178
|
+
};
|
|
179
|
+
try { appendLedgerRow(fm.ledger, cwd, loopDir, ledgerRow); lemit('ledger_row', ledgerRow); }
|
|
180
|
+
catch (e) { lemit('ledger_failed', { error: String(e.message || e).slice(0, 160) }); }
|
|
181
|
+
|
|
182
|
+
summary.runs.push({ iteration: state.totalRuns, status: res.status, taskStatus, verify: passed, tries: triesDone, proof: ledgerRow.proof || null, digest: ledgerRow.digest || null });
|
|
183
|
+
lemit('iteration_completed', { n: state.totalRuns, status: taskStatus, verify: passed, tries: triesDone, costUsd: res.metrics?.costUsd });
|
|
184
|
+
fs.writeFileSync(statePath, JSON.stringify(state, null, 2));
|
|
185
|
+
|
|
186
|
+
// v1.5 — FAIL-CONTINUE: gate_denied → halt immediately. verify_fail (after exhausting MAXT) → halt ONLY if 'verify_fail' ∈ halt_on; otherwise CONTINUE to the next task.
|
|
187
|
+
if (gateDenied) { summary.stopped = 'gate_denied'; lemit('loop_halted', { reason: summary.stopped }); break; }
|
|
188
|
+
if (!passed && haltOn.has('verify_fail')) { summary.stopped = `verify_fail after ${triesDone} tries`; lemit('loop_halted', { reason: summary.stopped }); break; }
|
|
189
|
+
if (!opts.once && i < maxIters - 1 && opts.sleepMs) await new Promise((r) => setTimeout(r, opts.sleepMs));
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
fs.writeFileSync(statePath, JSON.stringify(state, null, 2));
|
|
193
|
+
lemit('loop_finished', { iterations: summary.iterations, stopped: summary.stopped });
|
|
194
|
+
summary.eventsPath = eventsPath;
|
|
195
|
+
summary.loopDir = loopDir;
|
|
196
|
+
return summary;
|
|
197
|
+
}
|