@hexdecimal16/nexo 0.4.8 → 0.4.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/src/areas.js +3 -3
- package/cli/src/home-migrate.js +168 -0
- package/cli/src/main.js +3 -0
- package/cli/src/ui.js +1 -1
- package/core/src/engine/prompts.js +1 -1
- package/core/src/engine/run.js +2 -2
- package/core/src/index.js +1 -0
- package/core/src/learn/areas.js +22 -9
- package/core/src/learn/job.js +5 -4
- package/core/src/learn/wiki.js +12 -9
- package/core/src/optimize/optimize.js +1 -1
- package/core/src/optimize/scratchpad-wiki.js +7 -3
- package/core/src/providers/codex-p.js +378 -0
- package/core/src/providers/models.js +3 -1
- package/core/src/settings/settings.js +3 -2
- package/package.json +1 -1
- package/skills/bundled/wiki/SKILL.md +3 -3
- package/store/src/paths.js +2 -2
- package/tools/src/home-folders.js +36 -0
- package/tools/src/index.js +1 -0
- package/web/public/area.html +1 -1
- package/web/public/cost.html +1 -1
- package/web/public/home.html +1 -1
- package/web/public/index.html +1 -1
- package/web/public/judge.html +1 -1
- package/web/public/learning.html +1 -1
- package/web/public/optim.html +1 -1
- package/web/public/page.html +24 -0
- package/web/public/pages/area.ts +23 -13
- package/web/public/pages/home.ts +9 -3
- package/web/public/pages/page.ts +62 -0
- package/web/public/run.html +1 -1
- package/web/public/sessions.html +1 -1
- package/web/public/style.css +27 -0
- package/web/public/wikis.html +1 -1
- package/web/src/api.js +3 -3
- package/web/src/home-api.js +64 -7
- package/web/src/server.js +11 -1
package/cli/src/areas.js
CHANGED
|
@@ -10,7 +10,7 @@ export function areasCommand(args, io, options) {
|
|
|
10
10
|
throw usageError(AREAS_USAGE);
|
|
11
11
|
const paths = resolveNexoPaths(options.env ?? process.env);
|
|
12
12
|
ensureNexoDirs(paths);
|
|
13
|
-
const registry = readRegistry(paths.
|
|
13
|
+
const registry = readRegistry(paths.homeRoot);
|
|
14
14
|
if (sub === "list") {
|
|
15
15
|
if (registry.areas.length === 0) {
|
|
16
16
|
io.stdout.write("No areas yet.\n");
|
|
@@ -34,7 +34,7 @@ export function areasCommand(args, io, options) {
|
|
|
34
34
|
if ((sub !== "approve" && sub !== "reject") || !id)
|
|
35
35
|
throw usageError(AREAS_USAGE);
|
|
36
36
|
if (sub === "approve") {
|
|
37
|
-
writeRegistry(paths.
|
|
37
|
+
writeRegistry(paths.homeRoot, approveArea(registry, id));
|
|
38
38
|
io.stdout.write(`Approved area ${id}\n`);
|
|
39
39
|
return 0;
|
|
40
40
|
}
|
|
@@ -43,7 +43,7 @@ export function areasCommand(args, io, options) {
|
|
|
43
43
|
throw new NexoError({ code: "NOT_FOUND", message: `No area "${id}"`, context: { id } });
|
|
44
44
|
if (area.status !== "proposed")
|
|
45
45
|
throw new NexoError({ code: "CONFLICT", message: `Area "${id}" is active; only proposed areas can be rejected`, context: { id } });
|
|
46
|
-
writeRegistry(paths.
|
|
46
|
+
writeRegistry(paths.homeRoot, { ...registry, areas: registry.areas.filter((a) => a.id !== id) });
|
|
47
47
|
io.stdout.write(`Rejected area ${id}\n`);
|
|
48
48
|
return 0;
|
|
49
49
|
}
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { dirname, join, relative, resolve, sep } from "node:path";
|
|
3
|
+
import { parseArgs } from "node:util";
|
|
4
|
+
import { resolveNexoPaths } from "../../store/src/index.js";
|
|
5
|
+
import { NexoError } from "../../telemetry/src/index.js";
|
|
6
|
+
import { UNSORTED_AREA } from "../../tools/src/index.js";
|
|
7
|
+
import { usageError } from "./common.js";
|
|
8
|
+
const META_FILES = ["areas.json", "feed.jsonl", "decisions.jsonl"];
|
|
9
|
+
const MANIFEST = ".migrated-from-wikis.json";
|
|
10
|
+
const HOME_USAGE = "Usage: nexo home migrate [--apply | --undo]";
|
|
11
|
+
function isDir(p) {
|
|
12
|
+
try {
|
|
13
|
+
return statSync(p).isDirectory();
|
|
14
|
+
}
|
|
15
|
+
catch {
|
|
16
|
+
return false;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
/** Where each folder of the old wikis root goes: <home>/<area>/<name>, or <home>/unsorted/<name> when no active area lists it. */
|
|
20
|
+
export function planHomeMigration(oldRoot, newRoot) {
|
|
21
|
+
const plan = { folders: [], files: [], conflicts: [] };
|
|
22
|
+
if (!isDir(oldRoot))
|
|
23
|
+
return plan;
|
|
24
|
+
let areas = [];
|
|
25
|
+
try {
|
|
26
|
+
const parsed = JSON.parse(readFileSync(join(oldRoot, "areas.json"), "utf8"));
|
|
27
|
+
if (Array.isArray(parsed.areas))
|
|
28
|
+
areas = parsed.areas;
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
areas = [];
|
|
32
|
+
}
|
|
33
|
+
const owner = new Map();
|
|
34
|
+
for (const a of areas) {
|
|
35
|
+
if (a.status !== "active" || typeof a.id !== "string" || !/^[a-z0-9][a-z0-9-]*$/.test(a.id) || !Array.isArray(a.wikis))
|
|
36
|
+
continue;
|
|
37
|
+
for (const w of a.wikis)
|
|
38
|
+
if (typeof w === "string" && !owner.has(w))
|
|
39
|
+
owner.set(w, a.id);
|
|
40
|
+
}
|
|
41
|
+
for (const name of readdirSync(oldRoot).sort()) {
|
|
42
|
+
if (name.startsWith("."))
|
|
43
|
+
continue;
|
|
44
|
+
const from = join(oldRoot, name);
|
|
45
|
+
if (!isDir(from))
|
|
46
|
+
continue;
|
|
47
|
+
const to = join(newRoot, owner.get(name) ?? UNSORTED_AREA, name);
|
|
48
|
+
if (existsSync(to))
|
|
49
|
+
plan.conflicts.push(to);
|
|
50
|
+
else
|
|
51
|
+
plan.folders.push({ from, to });
|
|
52
|
+
}
|
|
53
|
+
for (const f of META_FILES) {
|
|
54
|
+
const from = join(oldRoot, f);
|
|
55
|
+
if (!existsSync(from))
|
|
56
|
+
continue;
|
|
57
|
+
const to = join(newRoot, f);
|
|
58
|
+
if (existsSync(to))
|
|
59
|
+
plan.conflicts.push(to);
|
|
60
|
+
else
|
|
61
|
+
plan.files.push({ from, to });
|
|
62
|
+
}
|
|
63
|
+
return plan;
|
|
64
|
+
}
|
|
65
|
+
function htmlFiles(dir, out = []) {
|
|
66
|
+
for (const f of readdirSync(dir)) {
|
|
67
|
+
if (f === ".git")
|
|
68
|
+
continue;
|
|
69
|
+
const p = join(dir, f);
|
|
70
|
+
if (isDir(p))
|
|
71
|
+
htmlFiles(p, out);
|
|
72
|
+
else if (f.endsWith(".html"))
|
|
73
|
+
out.push(p);
|
|
74
|
+
}
|
|
75
|
+
return out;
|
|
76
|
+
}
|
|
77
|
+
const within = (p, root) => p === root || p.startsWith(root + sep);
|
|
78
|
+
/** After folders moved from `from` to `to`, rewrites relative links into another moved folder so they still resolve. Returns links changed. */
|
|
79
|
+
export function rewriteMovedLinks(moves) {
|
|
80
|
+
let changed = 0;
|
|
81
|
+
for (const m of moves) {
|
|
82
|
+
if (!isDir(m.to))
|
|
83
|
+
continue;
|
|
84
|
+
for (const file of htmlFiles(m.to)) {
|
|
85
|
+
const oldFile = join(m.from, relative(m.to, file));
|
|
86
|
+
const text = readFileSync(file, "utf8");
|
|
87
|
+
let n = 0;
|
|
88
|
+
const next = text.replace(/href="([^"#?:]+\.html)([#?][^"]*)?"/g, (whole, href, tail) => {
|
|
89
|
+
if (href.startsWith("/"))
|
|
90
|
+
return whole;
|
|
91
|
+
const target = resolve(dirname(oldFile), href);
|
|
92
|
+
const owner = moves.find((x) => within(target, x.from));
|
|
93
|
+
if (!owner)
|
|
94
|
+
return whole;
|
|
95
|
+
const rel = relative(dirname(file), join(owner.to, relative(owner.from, target))).split(sep).join("/");
|
|
96
|
+
if (rel === href)
|
|
97
|
+
return whole;
|
|
98
|
+
n++;
|
|
99
|
+
return `href="${rel}${tail ?? ""}"`;
|
|
100
|
+
});
|
|
101
|
+
if (n > 0) {
|
|
102
|
+
writeFileSync(file, next);
|
|
103
|
+
changed += n;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
return changed;
|
|
108
|
+
}
|
|
109
|
+
function moveAll(moves, done) {
|
|
110
|
+
for (const m of moves) {
|
|
111
|
+
mkdirSync(dirname(m.to), { recursive: true });
|
|
112
|
+
renameSync(m.from, m.to);
|
|
113
|
+
done.push(m);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
/** `nexo home migrate`: moves ~/.nexo/wikis/<name> to ~/.nexo/home/<area>/<name>. Dry run by default; --apply moves, --undo reverses. */
|
|
117
|
+
export function homeCommand(args, io, options) {
|
|
118
|
+
const [sub, ...rest] = args;
|
|
119
|
+
if (sub !== "migrate")
|
|
120
|
+
throw usageError(HOME_USAGE);
|
|
121
|
+
const { values } = parseArgs({ args: rest, options: { apply: { type: "boolean" }, undo: { type: "boolean" } } });
|
|
122
|
+
if (values.apply && values.undo)
|
|
123
|
+
throw usageError(HOME_USAGE);
|
|
124
|
+
const paths = resolveNexoPaths(options.env ?? process.env);
|
|
125
|
+
const oldRoot = join(paths.home, "wikis");
|
|
126
|
+
const newRoot = paths.homeRoot;
|
|
127
|
+
const manifestFile = join(newRoot, MANIFEST);
|
|
128
|
+
if (values.undo) {
|
|
129
|
+
if (!existsSync(manifestFile))
|
|
130
|
+
throw new NexoError({ code: "NOT_FOUND", message: `No migration manifest at ${manifestFile}`, context: { manifest: manifestFile } });
|
|
131
|
+
const manifest = JSON.parse(readFileSync(manifestFile, "utf8"));
|
|
132
|
+
const back = [...manifest.moves].reverse().map((m) => ({ from: m.to, to: m.from }));
|
|
133
|
+
for (const m of back)
|
|
134
|
+
if (existsSync(m.to))
|
|
135
|
+
throw new NexoError({ code: "CONFLICT", message: `${m.to} already exists; undo would overwrite it`, context: { path: m.to } });
|
|
136
|
+
moveAll(back, []);
|
|
137
|
+
const links = rewriteMovedLinks(manifest.folders.map((m) => ({ from: m.to, to: m.from })));
|
|
138
|
+
rmSync(manifestFile);
|
|
139
|
+
io.stdout.write(`Moved ${back.length} entries back to ${oldRoot} and restored ${links} links.\n`);
|
|
140
|
+
return 0;
|
|
141
|
+
}
|
|
142
|
+
const plan = planHomeMigration(oldRoot, newRoot);
|
|
143
|
+
if (plan.folders.length + plan.files.length + plan.conflicts.length === 0) {
|
|
144
|
+
io.stdout.write(`Nothing to migrate in ${oldRoot}.\n`);
|
|
145
|
+
return 0;
|
|
146
|
+
}
|
|
147
|
+
for (const m of [...plan.folders, ...plan.files])
|
|
148
|
+
io.stdout.write(` ${relative(paths.home, m.from)} -> ${relative(paths.home, m.to)}\n`);
|
|
149
|
+
if (plan.conflicts.length > 0) {
|
|
150
|
+
throw new NexoError({ code: "CONFLICT", message: `Already exists: ${plan.conflicts.join(", ")}`, hint: "Move or remove those paths, then run again", context: { conflicts: plan.conflicts } });
|
|
151
|
+
}
|
|
152
|
+
if (!values.apply) {
|
|
153
|
+
io.stdout.write("Dry run: nothing moved. Run `nexo home migrate --apply` to move, and `--undo` to reverse it.\n");
|
|
154
|
+
return 0;
|
|
155
|
+
}
|
|
156
|
+
mkdirSync(newRoot, { recursive: true });
|
|
157
|
+
const done = [];
|
|
158
|
+
try {
|
|
159
|
+
moveAll([...plan.folders, ...plan.files], done);
|
|
160
|
+
}
|
|
161
|
+
finally {
|
|
162
|
+
const moved = new Set(done);
|
|
163
|
+
writeFileSync(manifestFile, `${JSON.stringify({ at: new Date().toISOString(), moves: done, folders: plan.folders.filter((m) => moved.has(m)) }, null, 2)}\n`);
|
|
164
|
+
}
|
|
165
|
+
const links = rewriteMovedLinks(plan.folders);
|
|
166
|
+
io.stdout.write(`Moved ${done.length} entries into ${newRoot} and rewrote ${links} links. Undo with: nexo home migrate --undo\n`);
|
|
167
|
+
return 0;
|
|
168
|
+
}
|
package/cli/src/main.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { homeCommand } from "./home-migrate.js";
|
|
1
2
|
import { areasCommand } from "./areas.js";
|
|
2
3
|
import { parseArgs } from "node:util";
|
|
3
4
|
import { judgeFamilyWarnings } from "../../core/src/index.js";
|
|
@@ -258,6 +259,8 @@ export async function runCli(argv, io, options = {}) {
|
|
|
258
259
|
return await initCommand(rest, io, options, stdio);
|
|
259
260
|
case "update":
|
|
260
261
|
return await updateCommand(io, options);
|
|
262
|
+
case "home":
|
|
263
|
+
return homeCommand(rest, io, options);
|
|
261
264
|
case "areas":
|
|
262
265
|
return areasCommand(rest, io, options);
|
|
263
266
|
case "sessions":
|
package/cli/src/ui.js
CHANGED
|
@@ -85,7 +85,7 @@ export async function runUi(options) {
|
|
|
85
85
|
judge,
|
|
86
86
|
learning: { enabled: config?.learning.enabled ?? false },
|
|
87
87
|
optimize: { minJudgeKappa: config?.optimize.minJudgeKappa ?? 0.6 },
|
|
88
|
-
wikisDir: paths.
|
|
88
|
+
wikisDir: paths.homeRoot,
|
|
89
89
|
companion: config ? companionModel(options, paths) : null,
|
|
90
90
|
});
|
|
91
91
|
}
|
|
@@ -17,7 +17,7 @@ export const DEFAULT_PLANNER_PROMPT = `Parallel work:
|
|
|
17
17
|
export const WEB_SEARCH_PROMPT = `Web research: your training data has a cutoff, so use web_search for anything that may have changed or that you are not sure of: current versions, APIs and docs, error messages, prices, news and other recent facts. Search with specific terms, read the best one to three hits with web_fetch, and cite the URLs you relied on in your answer. Do not search for things the workspace or a skill already answers.`;
|
|
18
18
|
/** The line that points the agent at the wikis directory; the wiki skill holds the rules for using it. */
|
|
19
19
|
export function wikiPrompt(wikisDir) {
|
|
20
|
-
return `Wikis: the user keeps persistent knowledge bases as folders of HTML pages under ${wikisDir}. When a task produces durable subject knowledge worth keeping (research findings, a source you read, a topic the user keeps returning to), load the wiki skill with load_skill and file it in the wiki that fits, creating a new wiki when none fits. Do not create or edit wikis for one-off questions, small talk or routine code changes. Load the wiki skill first whenever the task is about a wiki. The user's approved life areas are listed in ${wikisDir}/areas.json; file work in a wiki that belongs to the matching area, and never add or rename an area yourself, because new areas need the user's approval.`;
|
|
20
|
+
return `Wikis: the user keeps persistent knowledge bases as folders of HTML pages under ${wikisDir}, one level under their area (${wikisDir}/<area>/<folder>/). When a task produces durable subject knowledge worth keeping (research findings, a source you read, a topic the user keeps returning to), load the wiki skill with load_skill and file it in the wiki that fits, creating a new wiki when none fits. Do not create or edit wikis for one-off questions, small talk or routine code changes. Load the wiki skill first whenever the task is about a wiki. The user's approved life areas are listed in ${wikisDir}/areas.json; file work in a wiki that belongs to the matching area, and never add or rename an area yourself, because new areas need the user's approval.`;
|
|
21
21
|
}
|
|
22
22
|
export function renderSystemPrompt(parts) {
|
|
23
23
|
const sections = [parts.systemPrompt.replaceAll("{{workspace}}", parts.workspace)];
|
package/core/src/engine/run.js
CHANGED
|
@@ -538,7 +538,7 @@ export async function executeRun(env, input) {
|
|
|
538
538
|
bashTimeoutMs: config.tools.bashTimeoutMs,
|
|
539
539
|
nexoHome: deps.paths.home,
|
|
540
540
|
...(sharedWorkspace ? {} : { runWorkspace: runDir.workspaceDir }),
|
|
541
|
-
...(rollout ? {} : { wikiRoots: [deps.paths.
|
|
541
|
+
...(rollout ? {} : { wikiRoots: [deps.paths.homeRoot] }),
|
|
542
542
|
toolOutputsDir: runDir.toolOutputsDir,
|
|
543
543
|
// Large-output pointers in a session's history name files saved by earlier turns' runs.
|
|
544
544
|
earlierToolOutputsDirs: sessionId
|
|
@@ -640,7 +640,7 @@ export async function executeRun(env, input) {
|
|
|
640
640
|
plannerPrompt: tools.some((t) => t.name === "spawn") ? plannerPromptText : null,
|
|
641
641
|
webSearch: tools.some((t) => t.name === "web_search"),
|
|
642
642
|
workspace,
|
|
643
|
-
...(rollout ? {} : { wikisDir: deps.paths.
|
|
643
|
+
...(rollout ? {} : { wikisDir: deps.paths.homeRoot }),
|
|
644
644
|
skillIndex: formatSkillIndex(deps.skills),
|
|
645
645
|
...(rollout ? {} : { environment: environmentBlock({ ...(request.projectDir ? { projectDir: request.projectDir } : {}), now: new Date(), workspaceShared: sharedWorkspace !== null, ...(process.env.SHELL ? { shell: process.env.SHELL } : {}) }) }),
|
|
646
646
|
...(request.projectDir && !rollout ? { projectInstructions: loadProjectInstructions(request.projectDir, config.instructions).block } : {}),
|
package/core/src/index.js
CHANGED
|
@@ -19,6 +19,7 @@ export * from "./optimize/optimize.js";
|
|
|
19
19
|
export * from "./optimize/optimizer-run.js";
|
|
20
20
|
export * from "./optimize/reflect.js";
|
|
21
21
|
export * from "./providers/claude-p.js";
|
|
22
|
+
export * from "./providers/codex-p.js";
|
|
22
23
|
export * from "./providers/kiro-p.js";
|
|
23
24
|
export * from "./providers/models.js";
|
|
24
25
|
export * from "./scoring/calibration.js";
|
package/core/src/learn/areas.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { appendFileSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, statSync, writeFileSync } from "node:fs";
|
|
2
|
-
import { join } from "node:path";
|
|
2
|
+
import { isAbsolute, join, relative, sep } from "node:path";
|
|
3
3
|
import { dataBlock } from "../../../scorers/src/index.js";
|
|
4
4
|
import { NexoError } from "../../../telemetry/src/index.js";
|
|
5
|
+
import { listHomeFolders } from "../../../tools/src/index.js";
|
|
5
6
|
import { Type } from "typebox";
|
|
6
7
|
import { structuredLearnCall } from "./caller.js";
|
|
7
8
|
/** Life areas live in the wikis directory as data. Code never names an area; the user approves every new one. */
|
|
@@ -94,13 +95,24 @@ export function askUserAnswers(spans) {
|
|
|
94
95
|
})
|
|
95
96
|
.slice(-5);
|
|
96
97
|
}
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
98
|
+
/** HTML wiki pages (relative to the wikis dir) that the run's write and edit tool calls changed. */
|
|
99
|
+
export function wikiPagesTouched(spans, wikisDir) {
|
|
100
|
+
const out = [];
|
|
101
|
+
for (const s of spans) {
|
|
102
|
+
if (s.kind !== "tool_call" || s.status !== "ok" || (s.name !== "write" && s.name !== "edit"))
|
|
103
|
+
continue;
|
|
104
|
+
const path = (s.attrs ?? {}).args?.path;
|
|
105
|
+
if (typeof path !== "string" || !isAbsolute(path))
|
|
106
|
+
continue;
|
|
107
|
+
const rel = relative(wikisDir, path);
|
|
108
|
+
if (!rel || rel.startsWith("..") || isAbsolute(rel) || !rel.endsWith(".html"))
|
|
109
|
+
continue;
|
|
110
|
+
out.push(rel.split(sep).join("/"));
|
|
103
111
|
}
|
|
112
|
+
return out;
|
|
113
|
+
}
|
|
114
|
+
function listWikis(wikisDir) {
|
|
115
|
+
return listHomeFolders(wikisDir).map((f) => f.name);
|
|
104
116
|
}
|
|
105
117
|
export const AssignmentSchema = Type.Object({
|
|
106
118
|
areaId: Type.String({ minLength: 1, maxLength: 64 }),
|
|
@@ -131,7 +143,7 @@ export function assignPrompt(input) {
|
|
|
131
143
|
/** After a scored run: files it into an approved area (or the unsorted queue), appends a feed line and, for verified runs, the user's decisions. Does nothing until an area is active. */
|
|
132
144
|
export async function assignRun(deps, input) {
|
|
133
145
|
const none = { areaId: null, feed: false, decisions: 0, proposed: null, verified: false };
|
|
134
|
-
const wikisDir = deps.env.deps.paths?.
|
|
146
|
+
const wikisDir = deps.env.deps.paths?.homeRoot;
|
|
135
147
|
if (!wikisDir || !existsSync(wikisDir))
|
|
136
148
|
return none;
|
|
137
149
|
if (!input.output || input.output.trim().length < 50)
|
|
@@ -162,7 +174,8 @@ export async function assignRun(deps, input) {
|
|
|
162
174
|
}
|
|
163
175
|
}
|
|
164
176
|
const areaId = known?.id ?? null;
|
|
165
|
-
const
|
|
177
|
+
const pages = [...new Set([...wikiPagesTouched(input.spans, wikisDir), ...(input.extraPages ?? [])])].filter((x) => x.endsWith(".html")).slice(0, 5);
|
|
178
|
+
const base = { at: now, runId: input.run.id, sessionId: input.run.sessionId ?? null, areaId, wiki, pages };
|
|
166
179
|
let fed = false;
|
|
167
180
|
if (reply.feed.trim()) {
|
|
168
181
|
appendFileSync(join(wikisDir, FEED_FILE), `${JSON.stringify({ ...base, text: redact(reply.feed.trim()), verified })}\n`);
|
package/core/src/learn/job.js
CHANGED
|
@@ -90,9 +90,9 @@ export async function runLearnJob(env, job, options) {
|
|
|
90
90
|
output = "";
|
|
91
91
|
}
|
|
92
92
|
const feedback = latestFeedback(db, run.id);
|
|
93
|
-
const assignAreas = async () => {
|
|
93
|
+
const assignAreas = async (extraPages = []) => {
|
|
94
94
|
try {
|
|
95
|
-
await assignRun(deps, { run, output, spans, scores, cwd: run.sessionId ? getSessionRow(db, run.sessionId)?.cwd : undefined });
|
|
95
|
+
await assignRun(deps, { run, output, spans, scores, extraPages, cwd: run.sessionId ? getSessionRow(db, run.sessionId)?.cwd : undefined });
|
|
96
96
|
}
|
|
97
97
|
catch (err) {
|
|
98
98
|
if (options.signal.aborted)
|
|
@@ -194,13 +194,14 @@ export async function runLearnJob(env, job, options) {
|
|
|
194
194
|
throw error;
|
|
195
195
|
}
|
|
196
196
|
recorder.finish(writeSpan, { status: "ok", attrs: { result: curation } });
|
|
197
|
+
let curated = [];
|
|
197
198
|
try {
|
|
198
|
-
await curateWikiKnowledge(deps, run, output, spans);
|
|
199
|
+
curated = (await curateWikiKnowledge(deps, run, output, spans)).pages;
|
|
199
200
|
}
|
|
200
201
|
catch (err) {
|
|
201
202
|
logger.warn("wiki curation pass failed; continuing without it", { run_id: run.id, error: err });
|
|
202
203
|
}
|
|
203
|
-
await assignAreas();
|
|
204
|
+
await assignAreas(curated);
|
|
204
205
|
return { lessons: reflection.lessons.length, curation, costUsd, costUnknown };
|
|
205
206
|
}
|
|
206
207
|
catch (error) {
|
package/core/src/learn/wiki.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
|
|
2
|
-
import { dirname, join } from "node:path";
|
|
2
|
+
import { dirname, join, relative, sep } from "node:path";
|
|
3
3
|
import { dataBlock } from "../../../scorers/src/index.js";
|
|
4
|
-
import { resolveConfined, wrapWikiHtml } from "../../../tools/src/index.js";
|
|
4
|
+
import { findHomeFolder, listHomeFolders, resolveConfined, UNSORTED_AREA, wrapWikiHtml } from "../../../tools/src/index.js";
|
|
5
5
|
import { Type } from "typebox";
|
|
6
6
|
import { structuredLearnCall } from "./caller.js";
|
|
7
7
|
function titleFromPath(path) {
|
|
@@ -42,18 +42,18 @@ export function wikiCuratorPrompt(input) {
|
|
|
42
42
|
* Curates durable subject knowledge into persistent markdown wikis after a run.
|
|
43
43
|
*/
|
|
44
44
|
export async function curateWikiKnowledge(deps, run, output, spans) {
|
|
45
|
-
const wikisDir = deps.env.deps.paths?.
|
|
45
|
+
const wikisDir = deps.env.deps.paths?.homeRoot;
|
|
46
46
|
if (!wikisDir)
|
|
47
|
-
return { pagesCreated: 0, wikisTouched: [] };
|
|
47
|
+
return { pagesCreated: 0, wikisTouched: [], pages: [] };
|
|
48
48
|
if (!existsSync(wikisDir))
|
|
49
49
|
mkdirSync(wikisDir, { recursive: true });
|
|
50
50
|
// Skip if the run itself explicitly loaded the wiki skill (it managed its own wiki edits)
|
|
51
51
|
const usedWikiSkill = spans.some((s) => s.kind === "skill_load" && s.status === "ok" && s.attrs.args?.name === "wiki");
|
|
52
52
|
if (usedWikiSkill)
|
|
53
|
-
return { pagesCreated: 0, wikisTouched: [] };
|
|
53
|
+
return { pagesCreated: 0, wikisTouched: [], pages: [] };
|
|
54
54
|
// Only evaluate runs that have non-trivial instruction/output
|
|
55
55
|
if (!output || output.trim().length < 50)
|
|
56
|
-
return { pagesCreated: 0, wikisTouched: [] };
|
|
56
|
+
return { pagesCreated: 0, wikisTouched: [], pages: [] };
|
|
57
57
|
let existingWikis = [];
|
|
58
58
|
try {
|
|
59
59
|
existingWikis = readdirSync(wikisDir).filter((f) => {
|
|
@@ -70,6 +70,7 @@ export async function curateWikiKnowledge(deps, run, output, spans) {
|
|
|
70
70
|
catch {
|
|
71
71
|
existingWikis = [];
|
|
72
72
|
}
|
|
73
|
+
existingWikis = listHomeFolders(wikisDir).map((f) => f.name);
|
|
73
74
|
const prompt = wikiCuratorPrompt({
|
|
74
75
|
instruction: run.instruction,
|
|
75
76
|
output,
|
|
@@ -81,13 +82,14 @@ export async function curateWikiKnowledge(deps, run, output, spans) {
|
|
|
81
82
|
tier: "curator",
|
|
82
83
|
});
|
|
83
84
|
if (curation.pages.length === 0)
|
|
84
|
-
return { pagesCreated: 0, wikisTouched: [] };
|
|
85
|
+
return { pagesCreated: 0, wikisTouched: [], pages: [] };
|
|
85
86
|
let pagesCreated = 0;
|
|
87
|
+
const createdPages = [];
|
|
86
88
|
const wikisTouched = new Set();
|
|
87
89
|
const today = deps.env.now().toISOString().slice(0, 10);
|
|
88
90
|
for (const p of curation.pages) {
|
|
89
91
|
try {
|
|
90
|
-
const wikiDir = resolveConfined(wikisDir, p.wikiName);
|
|
92
|
+
const wikiDir = findHomeFolder(wikisDir, p.wikiName)?.dir ?? resolveConfined(wikisDir, `${UNSORTED_AREA}/${p.wikiName}`);
|
|
91
93
|
if (!existsSync(wikiDir)) {
|
|
92
94
|
mkdirSync(wikiDir, { recursive: true });
|
|
93
95
|
mkdirSync(join(wikiDir, "concepts"), { recursive: true });
|
|
@@ -106,6 +108,7 @@ export async function curateWikiKnowledge(deps, run, output, spans) {
|
|
|
106
108
|
const pageHtml = wrapWikiHtml({ title: titleFromPath(pagePath), summary: p.summary, updated: today, sources: [`run:${run.id}`], body: p.content });
|
|
107
109
|
writeFileSync(targetFile, pageHtml);
|
|
108
110
|
pagesCreated++;
|
|
111
|
+
createdPages.push(relative(wikisDir, targetFile).split(sep).join("/"));
|
|
109
112
|
wikisTouched.add(p.wikiName);
|
|
110
113
|
const logFile = join(wikiDir, "log.md");
|
|
111
114
|
if (existsSync(logFile)) {
|
|
@@ -117,5 +120,5 @@ export async function curateWikiKnowledge(deps, run, output, spans) {
|
|
|
117
120
|
deps.env.deps.logger.warn("failed to write curated wiki page", { wiki: p.wikiName, path: p.path, error: err });
|
|
118
121
|
}
|
|
119
122
|
}
|
|
120
|
-
return { pagesCreated, wikisTouched: [...wikisTouched] };
|
|
123
|
+
return { pagesCreated, wikisTouched: [...wikisTouched], pages: createdPages };
|
|
121
124
|
}
|
|
@@ -244,7 +244,7 @@ export async function startOptimization(baseEnv, options) {
|
|
|
244
244
|
ctx.reserveUsd = cases.holdout.length * Math.min(config.budgets.perRunUsd, 2 * dearest);
|
|
245
245
|
ctx.phase = "loop";
|
|
246
246
|
options.onEvent?.({ type: "phase", phase: "loop" });
|
|
247
|
-
const wikisDir = env.deps.paths?.
|
|
247
|
+
const wikisDir = env.deps.paths?.homeRoot;
|
|
248
248
|
const seededPad = wikisDir ? readScratchpad(wikisDir, target.key) : undefined;
|
|
249
249
|
const result = await runGepa({
|
|
250
250
|
adapter,
|
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
2
|
import { dirname, join } from "node:path";
|
|
3
3
|
import { emptyScratchpad, SCRATCHPAD_ENTRIES, scratchpadEntry } from "../../../optimize/src/index.js";
|
|
4
|
-
import { resolveConfined } from "../../../tools/src/index.js";
|
|
4
|
+
import { findHomeFolder, resolveConfined, UNSORTED_AREA } from "../../../tools/src/index.js";
|
|
5
|
+
/** The scratchpad wiki folder: wherever it already lives under an area, else <root>/unsorted/optimization. */
|
|
6
|
+
function scratchpadDir(root) {
|
|
7
|
+
return findHomeFolder(root, SCRATCHPAD_WIKI)?.dir ?? resolveConfined(root, `${UNSORTED_AREA}/${SCRATCHPAD_WIKI}`);
|
|
8
|
+
}
|
|
5
9
|
/** The wiki that holds what optimizations learned: one page per target, readable and editable in the Wikis UI. */
|
|
6
10
|
export const SCRATCHPAD_WIKI = "optimization";
|
|
7
11
|
/** The page of `targetKey` inside the wiki, e.g. `analyses/optim-skill-web-research.md`. */
|
|
@@ -28,7 +32,7 @@ export function parseScratchpad(markdown) {
|
|
|
28
32
|
/** The saved scratchpad of a target, or undefined when there is no page, it has no entries or it cannot be read. */
|
|
29
33
|
export function readScratchpad(wikisDir, targetKey) {
|
|
30
34
|
try {
|
|
31
|
-
const file = resolveConfined(
|
|
35
|
+
const file = resolveConfined(scratchpadDir(wikisDir), scratchpadPagePath(targetKey));
|
|
32
36
|
if (!existsSync(file))
|
|
33
37
|
return undefined;
|
|
34
38
|
const pad = parseScratchpad(readFileSync(file, "utf8"));
|
|
@@ -40,7 +44,7 @@ export function readScratchpad(wikisDir, targetKey) {
|
|
|
40
44
|
}
|
|
41
45
|
/** Writes the target's scratchpad page (creating the wiki on first use) and logs it; returns the page file. */
|
|
42
46
|
export function writeScratchpad(wikisDir, targetKey, pad, info) {
|
|
43
|
-
const wikiDir =
|
|
47
|
+
const wikiDir = scratchpadDir(wikisDir);
|
|
44
48
|
const { today } = info;
|
|
45
49
|
if (!existsSync(wikiDir)) {
|
|
46
50
|
for (const d of ["concepts", "entities", "sources", "analyses", "raw"])
|