@zenera/cli 1.1.23 → 1.1.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -2
- package/dist/ask.d.ts +51 -0
- package/dist/ask.js +143 -0
- package/dist/batch.d.ts +77 -0
- package/dist/batch.js +338 -0
- package/dist/command.d.ts +6 -0
- package/dist/commands/inspect.d.ts +11 -0
- package/dist/commands/inspect.js +388 -89
- package/dist/commands/list.js +128 -19
- package/dist/commands/memory.js +342 -12
- package/dist/commands/models.js +14 -2
- package/dist/commands/run.js +139 -34
- package/dist/commands/sandbox.js +72 -1
- package/dist/engine.d.ts +19 -0
- package/dist/engine.js +60 -8
- package/dist/liveness.js +6 -3
- package/dist/main.js +5 -2
- package/dist/podman.d.ts +16 -0
- package/dist/podman.js +115 -2
- package/dist/progress.d.ts +33 -0
- package/dist/progress.js +402 -0
- package/dist/request.d.ts +45 -0
- package/dist/request.js +300 -0
- package/dist/sandbox.js +10 -0
- package/dist/session.d.ts +26 -1
- package/dist/session.js +42 -3
- package/dist/term.d.ts +3 -0
- package/dist/term.js +17 -1
- package/dist/trace.d.ts +84 -0
- package/dist/trace.js +686 -0
- package/dist/validate.js +25 -3
- package/package.json +2 -2
- package/templates/editor/.github/copilot-instructions.md +11 -6
- package/templates/editor/.github/prompts/finetune.prompt.md +251 -0
- package/templates/editor/.github/skills/zen-cli/SKILL.md +9 -3
- package/templates/editor/.github/skills/zen-cli/references/check.md +7 -5
- package/templates/editor/.github/skills/zen-cli/references/inspect.md +262 -21
- package/templates/editor/.github/skills/zen-cli/references/memory.md +89 -19
- package/templates/editor/.github/skills/zen-cli/references/models.md +1 -1
- package/templates/editor/.github/skills/zen-cli/references/projects.md +12 -5
- package/templates/editor/.github/skills/zen-cli/references/run.md +201 -4
- package/templates/editor/.github/skills/zen-finetune/SKILL.md +1000 -0
- package/templates/editor/.github/skills/zen-finetune/scripts/collect.sh +432 -0
- package/templates/editor/.github/skills/zen-finetune/scripts/sample.sh +199 -0
- package/templates/editor/.github/skills/zen-inspect/SKILL.md +396 -0
- package/templates/editor/.github/skills/zen-memory/SKILL.md +74 -29
- package/templates/editor/.github/skills/zen-memory-warmup/SKILL.md +363 -531
- package/templates/editor/.github/skills/zen-sandbox/SKILL.md +19 -0
- package/templates/editor/.github/skills/zen-sandbox-capacity/SKILL.md +334 -0
- package/templates/editor/.github/skills/zen-sandbox-capacity/references/loadtest.md +181 -0
- package/templates/editor/.github/skills/zen-sandbox-capacity/scripts/preflight_sandbox.sh +457 -0
- package/templates/editor/.vscode/agents.schema.json +1 -1
- package/templates/editor/.vscode/settings.json +1 -0
- package/templates/project/agents/files-instructions.md +13 -0
- package/templates/project/agents/fork-instructions.md +27 -0
- package/templates/project/agents/memory-instructions.md +80 -7
- package/templates/project/gitignore +3 -0
package/dist/commands/list.js
CHANGED
|
@@ -1,21 +1,38 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
1
2
|
import { parse } from "../args.js";
|
|
2
3
|
// Run ids are timestamps by construction, so the newest id *is* the last-run
|
|
3
4
|
// time — no file has to be opened to learn it.
|
|
4
5
|
import { stampInstant } from "../ids.js";
|
|
5
|
-
import { openDir, Registry, summarize } from "../projects.js";
|
|
6
|
-
import { listSessions } from "../session.js";
|
|
7
|
-
import { ago, bold, count, dim, json, note, table, write, writeAll, yellow } from "../term.js";
|
|
8
|
-
const USAGE = 'zen list [--sessions] [--prune]';
|
|
6
|
+
import { openDir, Registry, runIds, sessionIds, sessionsDir, summarize, } from "../projects.js";
|
|
7
|
+
import { listSessions, readRunMeta, runPaths, sessionPaths } from "../session.js";
|
|
8
|
+
import { ago, bold, count, cyan, dim, json, note, red, table, usageError, write, writeAll, yellow, } from "../term.js";
|
|
9
|
+
const USAGE = 'zen list [--sessions] [--runs] [--limit <n>] [--prune]';
|
|
10
|
+
/** Recent runs shown before `--limit` is asked for. */
|
|
11
|
+
const RECENT = 20;
|
|
9
12
|
export const list = {
|
|
10
13
|
summary: 'Every known project: sessions, last run, whether one is live.',
|
|
11
14
|
usage: USAGE,
|
|
12
15
|
banner: { head: 'Zenera', accent: 'Projects', subtitle: 'Project Registry', hue: 'sky' },
|
|
13
16
|
details: [
|
|
17
|
+
" --sessions Also list each project's sessions.",
|
|
18
|
+
' --runs The most recent runs, newest first, across every project.',
|
|
19
|
+
' --limit <n> Runs to list. Default 20.',
|
|
20
|
+
' --prune Forget entries whose directory has gone away.',
|
|
21
|
+
'',
|
|
22
|
+
'--runs answers "what ran last", which is not a question about one',
|
|
23
|
+
'project, so on its own it replaces the table rather than following it.',
|
|
24
|
+
'',
|
|
14
25
|
'The registry is an index, not the truth. An entry whose directory has',
|
|
15
26
|
'gone away is shown dimmed rather than hidden; --prune forgets them.',
|
|
16
27
|
],
|
|
17
28
|
run: async (ctx) => {
|
|
18
|
-
const { values } = parse(ctx.args, {
|
|
29
|
+
const { values } = parse(ctx.args, {
|
|
30
|
+
sessions: { type: 'boolean' },
|
|
31
|
+
runs: { type: 'boolean' },
|
|
32
|
+
limit: { type: 'string' },
|
|
33
|
+
prune: { type: 'boolean' },
|
|
34
|
+
}, USAGE);
|
|
35
|
+
const limit = number(values.limit, '--limit') ?? RECENT;
|
|
19
36
|
const registry = await Registry.open();
|
|
20
37
|
if (values.prune) {
|
|
21
38
|
const gone = registry.prune();
|
|
@@ -29,8 +46,18 @@ export const list = {
|
|
|
29
46
|
summaries.push(await summarize(entry));
|
|
30
47
|
}
|
|
31
48
|
summaries.sort((a, b) => (b.lastRunAt ?? '').localeCompare(a.lastRunAt ?? ''));
|
|
49
|
+
// `--runs` asks a question about runs, so on its own it *replaces* the
|
|
50
|
+
// project table rather than following thirty rows of it. Asked together
|
|
51
|
+
// with `--sessions`, both are wanted and both are printed.
|
|
52
|
+
const projectTable = !values.runs || values.sessions;
|
|
32
53
|
if (ctx.json) {
|
|
33
|
-
|
|
54
|
+
const runs = values.runs ? await recent(summaries, limit).runs : undefined;
|
|
55
|
+
if (!projectTable) {
|
|
56
|
+
json(runs);
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
const projects = values.sessions ? await withSessions(summaries) : summaries;
|
|
60
|
+
json(runs ? { projects, runs } : projects);
|
|
34
61
|
return;
|
|
35
62
|
}
|
|
36
63
|
if (summaries.length === 0) {
|
|
@@ -38,20 +65,22 @@ export const list = {
|
|
|
38
65
|
note(dim('create one: zen init'));
|
|
39
66
|
return;
|
|
40
67
|
}
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
const
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
68
|
+
if (projectTable) {
|
|
69
|
+
const rows = [
|
|
70
|
+
[bold('NAME'), bold('SESSIONS'), bold('RUNS'), bold('LAST'), bold('PATH')],
|
|
71
|
+
];
|
|
72
|
+
for (const s of summaries) {
|
|
73
|
+
const style = s.present ? (x) => x : dim;
|
|
74
|
+
rows.push([
|
|
75
|
+
style(s.name) + (s.busy ? yellow(' •') : ''),
|
|
76
|
+
style(String(s.sessions)),
|
|
77
|
+
style(String(s.runs)),
|
|
78
|
+
style(ago(s.lastRunAt ? stampInstant(s.lastRunAt) : undefined)),
|
|
79
|
+
dim(s.present ? s.path : `${s.path} (missing)`),
|
|
80
|
+
]);
|
|
81
|
+
}
|
|
82
|
+
writeAll(table(rows));
|
|
53
83
|
}
|
|
54
|
-
writeAll(table(rows));
|
|
55
84
|
if (values.sessions) {
|
|
56
85
|
for (const s of summaries.filter((p) => p.present)) {
|
|
57
86
|
write('');
|
|
@@ -59,8 +88,88 @@ export const list = {
|
|
|
59
88
|
writeAll(await sessionRows(s));
|
|
60
89
|
}
|
|
61
90
|
}
|
|
91
|
+
if (values.runs) {
|
|
92
|
+
if (projectTable) {
|
|
93
|
+
write('');
|
|
94
|
+
}
|
|
95
|
+
await writeRuns(summaries, limit);
|
|
96
|
+
}
|
|
62
97
|
},
|
|
63
98
|
};
|
|
99
|
+
function keys(summaries) {
|
|
100
|
+
const out = [];
|
|
101
|
+
for (const project of summaries) {
|
|
102
|
+
if (!project.present) {
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
105
|
+
for (const session of sessionIds(project.path)) {
|
|
106
|
+
for (const id of runIds(join(sessionsDir(project.path), session))) {
|
|
107
|
+
out.push({ project, session, id });
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return out.sort((a, b) => b.id.localeCompare(a.id));
|
|
112
|
+
}
|
|
113
|
+
function recent(summaries, limit) {
|
|
114
|
+
const all = keys(summaries);
|
|
115
|
+
const runs = Promise.all(all.slice(0, limit).map(async (k) => {
|
|
116
|
+
const run = runPaths(sessionPaths(k.project.path, k.session), k.id);
|
|
117
|
+
const meta = await readRunMeta(run);
|
|
118
|
+
return {
|
|
119
|
+
id: k.id,
|
|
120
|
+
project: k.project.name,
|
|
121
|
+
session: k.session,
|
|
122
|
+
dir: run.dir,
|
|
123
|
+
startedAt: meta.startedAt ?? stampInstant(k.id),
|
|
124
|
+
finishedAt: meta.finishedAt,
|
|
125
|
+
agent: meta.agent,
|
|
126
|
+
stopReason: meta.stopReason,
|
|
127
|
+
error: meta.error,
|
|
128
|
+
};
|
|
129
|
+
}));
|
|
130
|
+
return { runs, found: all.length };
|
|
131
|
+
}
|
|
132
|
+
async function writeRuns(summaries, limit) {
|
|
133
|
+
const { runs, found } = recent(summaries, limit);
|
|
134
|
+
const rows = await runs;
|
|
135
|
+
if (rows.length === 0) {
|
|
136
|
+
note('no runs yet');
|
|
137
|
+
note(dim('start one: zen run'));
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
writeAll(table([
|
|
141
|
+
[
|
|
142
|
+
bold('RUN'),
|
|
143
|
+
bold('PROJECT'),
|
|
144
|
+
bold('SESSION'),
|
|
145
|
+
bold('AGENT'),
|
|
146
|
+
bold('OUTCOME'),
|
|
147
|
+
bold('WHEN'),
|
|
148
|
+
],
|
|
149
|
+
...rows.map((r) => [
|
|
150
|
+
r.id,
|
|
151
|
+
cyan(r.project),
|
|
152
|
+
dim(r.session),
|
|
153
|
+
r.agent ?? dim('—'),
|
|
154
|
+
r.error ? red('failed') : (r.stopReason ?? dim('—')),
|
|
155
|
+
ago(r.finishedAt ?? r.startedAt),
|
|
156
|
+
]),
|
|
157
|
+
]));
|
|
158
|
+
if (found > rows.length) {
|
|
159
|
+
note('');
|
|
160
|
+
note(dim(`${found - rows.length} more — raise --limit to see them`));
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
function number(text, flag) {
|
|
164
|
+
if (text === undefined) {
|
|
165
|
+
return undefined;
|
|
166
|
+
}
|
|
167
|
+
const value = Number(text);
|
|
168
|
+
if (!Number.isInteger(value) || value < 1) {
|
|
169
|
+
throw usageError(`${flag} takes a whole number of at least 1`, `got "${text}"`);
|
|
170
|
+
}
|
|
171
|
+
return value;
|
|
172
|
+
}
|
|
64
173
|
/**
|
|
65
174
|
* Run ids are timestamps by construction, so the newest id *is* the last-run
|
|
66
175
|
* time — no file has to be opened to learn it.
|
package/dist/commands/memory.js
CHANGED
|
@@ -1,13 +1,15 @@
|
|
|
1
|
-
import { ALL_AGENTS, buildMemoryReport, hostPath, memoryDir, MemoryIndex, MemoryStore, MergeConflicts, mergeMemories, readProjectConfig, renderMemoryHtml, } from '@zenera/neo';
|
|
1
|
+
import { ALL_AGENTS, buildMemoryReport, createEmbedder, GREP_FIELDS, grepMemory, hostPath, memoryDir, MemoryError, MemoryIndex, MemoryStore, MergeConflicts, mergeMemories, readProjectConfig, renderMemoryHtml, renderRecollection, } from '@zenera/neo';
|
|
2
2
|
import { spawn } from 'node:child_process';
|
|
3
3
|
import { existsSync } from 'node:fs';
|
|
4
4
|
import { writeFile } from 'node:fs/promises';
|
|
5
5
|
import { basename, isAbsolute, join, relative, resolve } from 'node:path';
|
|
6
6
|
import { parse } from "../args.js";
|
|
7
|
+
import { loadProjectEnv } from "../env.js";
|
|
8
|
+
import { KeyStore } from "../keys.js";
|
|
7
9
|
import { project as resolveProject } from "../resolve.js";
|
|
8
10
|
import { ago, bold, bytes, CliError, confirm, count, cyan, dim, EXIT, green, isInteractive, json, note, table, usageError, write, writeAll, yellow, } from "../term.js";
|
|
9
|
-
const USAGE = 'zen memory [stats|ls|show|export|merge|forget] [args] [options]';
|
|
10
|
-
const SUBCOMMANDS = ['stats', 'ls', 'show', 'export', 'merge', 'forget'];
|
|
11
|
+
const USAGE = 'zen memory [stats|ls|search|grep|show|export|merge|forget] [args] [options]';
|
|
12
|
+
const SUBCOMMANDS = ['stats', 'ls', 'search', 'grep', 'show', 'export', 'merge', 'forget'];
|
|
11
13
|
/** Enough to see the shape of it; the rest is a number. */
|
|
12
14
|
const LISTED = 30;
|
|
13
15
|
// ---------------------------------------------------------------------------
|
|
@@ -24,9 +26,11 @@ const LISTED = 30;
|
|
|
24
26
|
// an agent inside a run, and the mask exists to keep agents apart rather than
|
|
25
27
|
// to keep secrets from the person who owns the files.
|
|
26
28
|
//
|
|
27
|
-
//
|
|
28
|
-
//
|
|
29
|
-
//
|
|
29
|
+
// `search` is the one subcommand that contacts a model, and it has to: the
|
|
30
|
+
// question it answers is what an agent's recall returns, and a recall ranked
|
|
31
|
+
// any other way is a different answer wearing the same shape. Everything else
|
|
32
|
+
// is offline, so inspecting a graph whose credentials have gone missing —
|
|
33
|
+
// which is exactly when someone is debugging — still works.
|
|
30
34
|
// ---------------------------------------------------------------------------
|
|
31
35
|
export const memory = {
|
|
32
36
|
summary: 'What the agents remember, and getting rid of it.',
|
|
@@ -35,6 +39,8 @@ export const memory = {
|
|
|
35
39
|
details: [
|
|
36
40
|
' stats Size, vocabulary, and whether it is embedded.',
|
|
37
41
|
' ls Nodes, newest first. Changes nothing.',
|
|
42
|
+
' search <text> Recall it, the way an agent does. Ranked.',
|
|
43
|
+
' grep <pattern> Every node containing it, with the matching lines.',
|
|
38
44
|
' show <id> One node in full, with what it links to.',
|
|
39
45
|
' export [file] The whole graph as one HTML page.',
|
|
40
46
|
' merge <dir...> Fold other memories into this one.',
|
|
@@ -46,7 +52,16 @@ export const memory = {
|
|
|
46
52
|
' --audience <name> Only nodes committed under this label.',
|
|
47
53
|
' --files Only nodes that remember a file.',
|
|
48
54
|
' --stale Only nodes something has superseded.',
|
|
49
|
-
' --
|
|
55
|
+
' --all For `search` and `grep`: superseded nodes too, marked.',
|
|
56
|
+
' --regex Read the pattern as a regular expression, per line.',
|
|
57
|
+
' --case-sensitive Match case exactly. Off by default.',
|
|
58
|
+
' --in <text|metadata|file> Where to look. Repeatable. All three by default.',
|
|
59
|
+
' --ids-only Print bare ids, for piping into `zen memory show`.',
|
|
60
|
+
' --limit <n> Rows to list, or seeds to rank. Default 30, search 10.',
|
|
61
|
+
' --hops <n> How far `search` follows links from a seed. Default 2.',
|
|
62
|
+
' --nodes <n> Cap on the subgraph `search` returns. Default 25.',
|
|
63
|
+
' --min-score <n> Drop seeds scoring below this. Default 0.15.',
|
|
64
|
+
' --embedding <ref> Rank `search` with this model, not the project’s.',
|
|
50
65
|
' --out <file> Where `export` writes. Default memory.html.',
|
|
51
66
|
' --open Open the exported page.',
|
|
52
67
|
' --dry-run Say what `merge` would do, and stop.',
|
|
@@ -62,10 +77,33 @@ export const memory = {
|
|
|
62
77
|
'page: the node list on the left, the graph in the middle, and whatever',
|
|
63
78
|
'you click on the right, file contents and all.',
|
|
64
79
|
'',
|
|
80
|
+
'`search` is recall itself, run from here. It embeds the query with the',
|
|
81
|
+
'project’s own model and prints the block a model would have been given,',
|
|
82
|
+
'scores and all — so *why did it recall that?* has an answer, and so does',
|
|
83
|
+
'*why did it not?*:',
|
|
84
|
+
'',
|
|
85
|
+
' zen memory search ‘how do we deploy’ --audience reviewer',
|
|
86
|
+
'',
|
|
87
|
+
'Passing `--audience` recalls as an agent that sees only that label,',
|
|
88
|
+
'which is how a memory that is present but invisible gives itself away.',
|
|
89
|
+
'',
|
|
90
|
+
'`grep` is the counterpart. Recall ranks, so it answers “what is closest”',
|
|
91
|
+
'and can never answer “is this in here at all”; `grep` reads every node',
|
|
92
|
+
'exactly — text, metadata and the bytes of remembered files — and the',
|
|
93
|
+
'count it reports is the true one even when the list was cut:',
|
|
94
|
+
'',
|
|
95
|
+
' zen memory grep ‘staging.example.com’ --in metadata --in file',
|
|
96
|
+
'',
|
|
97
|
+
'Nothing that only reads takes the directory lock, so every subcommand',
|
|
98
|
+
'here but `merge` and `forget` works on a memory a run is writing, and on',
|
|
99
|
+
'the read-only /memory mount inside a sandbox.',
|
|
100
|
+
'',
|
|
65
101
|
'`merge` is for warming a memory in parallel. The lock is per directory,',
|
|
66
102
|
'so N runs write N memories; this folds them back into one:',
|
|
67
103
|
'',
|
|
68
|
-
' zen memory merge
|
|
104
|
+
' zen memory merge <batch-dir>/*/memory',
|
|
105
|
+
'',
|
|
106
|
+
'which is where `zen run batch` leaves them.',
|
|
69
107
|
],
|
|
70
108
|
run: async (ctx) => {
|
|
71
109
|
const { values, positionals } = parse(ctx.args, {
|
|
@@ -75,7 +113,16 @@ export const memory = {
|
|
|
75
113
|
audience: { type: 'string' },
|
|
76
114
|
files: { type: 'boolean' },
|
|
77
115
|
stale: { type: 'boolean' },
|
|
116
|
+
all: { type: 'boolean' },
|
|
117
|
+
regex: { type: 'boolean' },
|
|
118
|
+
'case-sensitive': { type: 'boolean' },
|
|
119
|
+
in: { type: 'string', multiple: true },
|
|
120
|
+
'ids-only': { type: 'boolean' },
|
|
78
121
|
limit: { type: 'string' },
|
|
122
|
+
hops: { type: 'string' },
|
|
123
|
+
nodes: { type: 'string' },
|
|
124
|
+
'min-score': { type: 'string' },
|
|
125
|
+
embedding: { type: 'string' },
|
|
79
126
|
out: { type: 'string' },
|
|
80
127
|
open: { type: 'boolean' },
|
|
81
128
|
yes: { type: 'boolean' },
|
|
@@ -90,13 +137,23 @@ export const memory = {
|
|
|
90
137
|
const rest = positionals.slice(1);
|
|
91
138
|
// `merge` is the one subcommand that may write a memory into existence;
|
|
92
139
|
// every other one is an inspector and a missing manifest is a mistake.
|
|
93
|
-
|
|
140
|
+
// And only the two that write take the lock: the moments a reader is
|
|
141
|
+
// most wanted are while a run is writing and against the read-only
|
|
142
|
+
// /memory mount, in both of which claiming it fails.
|
|
143
|
+
const opened = await open(ctx.cwd, values.project, values.dir, {
|
|
144
|
+
create: what === 'merge',
|
|
145
|
+
lock: what === 'merge' || what === 'forget',
|
|
146
|
+
});
|
|
94
147
|
try {
|
|
95
148
|
switch (what) {
|
|
96
149
|
case 'stats':
|
|
97
150
|
return stats(opened, ctx.json);
|
|
98
151
|
case 'ls':
|
|
99
152
|
return list(opened, values, ctx.json);
|
|
153
|
+
case 'search':
|
|
154
|
+
return await search(opened, rest, values, ctx.json);
|
|
155
|
+
case 'grep':
|
|
156
|
+
return await grep(opened, rest, values, ctx.json);
|
|
100
157
|
case 'show':
|
|
101
158
|
return show(opened, rest, ctx.json);
|
|
102
159
|
case 'export':
|
|
@@ -125,14 +182,19 @@ export const memory = {
|
|
|
125
182
|
* `create` drops the requirement that anything be there yet, for `merge`,
|
|
126
183
|
* which has somewhere to put what it reads even when the destination is a name
|
|
127
184
|
* nothing has written to.
|
|
185
|
+
*
|
|
186
|
+
* `lock` is off for a read that must not fail on a busy or read-only memory.
|
|
187
|
+
* The lock exists to stop two writers losing each other's edges; a reader that
|
|
188
|
+
* took it would only be refusing itself.
|
|
128
189
|
*/
|
|
129
|
-
async function open(cwd, want, at,
|
|
190
|
+
async function open(cwd, want, at, opts = {}) {
|
|
191
|
+
const { create = false, lock = true } = opts;
|
|
130
192
|
if (at) {
|
|
131
193
|
const dir = resolve(cwd, at);
|
|
132
194
|
if (!create && !existsSync(join(dir, 'manifest.json'))) {
|
|
133
195
|
throw usageError(`${dir} is not a memory`, 'no manifest.json in it');
|
|
134
196
|
}
|
|
135
|
-
return { store: await MemoryStore.open(dir), dir, project: basename(dir) };
|
|
197
|
+
return { store: await MemoryStore.open(dir, { lock }), dir, project: basename(dir) };
|
|
136
198
|
}
|
|
137
199
|
const project = await resolveProject({ cwd, project: want });
|
|
138
200
|
const { config } = readProjectConfig(project.dir);
|
|
@@ -143,7 +205,12 @@ async function open(cwd, want, at, create = false) {
|
|
|
143
205
|
if (!create && !existsSync(join(dir, 'manifest.json'))) {
|
|
144
206
|
throw usageError(`${project.name} has a memory configured but nothing in it yet`, 'it is written the first time an agent commits something');
|
|
145
207
|
}
|
|
146
|
-
return {
|
|
208
|
+
return {
|
|
209
|
+
store: await MemoryStore.open(dir, { lock }),
|
|
210
|
+
dir,
|
|
211
|
+
project: project.name,
|
|
212
|
+
at: { root: project.dir, config },
|
|
213
|
+
};
|
|
147
214
|
}
|
|
148
215
|
// ---------------------------------------------------------------------------
|
|
149
216
|
// stats
|
|
@@ -265,6 +332,269 @@ function list(o, flags, asJson) {
|
|
|
265
332
|
}
|
|
266
333
|
}
|
|
267
334
|
// ---------------------------------------------------------------------------
|
|
335
|
+
// search
|
|
336
|
+
// ---------------------------------------------------------------------------
|
|
337
|
+
/**
|
|
338
|
+
* Recall, run from outside a run. `ls` and `grep` read the graph; this asks it
|
|
339
|
+
* the question an agent asks, through the same ranker, the same traversal and
|
|
340
|
+
* the same renderer — so what it prints is the block a model would have been
|
|
341
|
+
* handed, scores and all.
|
|
342
|
+
*
|
|
343
|
+
* That is the only way to answer *why did it recall that*, and the harder *why
|
|
344
|
+
* did it not*: a memory that is present but ranked sixth looks exactly like a
|
|
345
|
+
* memory that was never written, until the score is on the screen.
|
|
346
|
+
*/
|
|
347
|
+
async function search(o, rest, flags, asJson) {
|
|
348
|
+
// Joined rather than taken singly. A query is prose, and this is the one
|
|
349
|
+
// subcommand whose whole argument is prose, so quoting it is the step
|
|
350
|
+
// everybody forgets.
|
|
351
|
+
const text = rest.join(' ').trim();
|
|
352
|
+
if (!text) {
|
|
353
|
+
throw usageError('search takes something to recall', 'zen memory search <text>');
|
|
354
|
+
}
|
|
355
|
+
// Recall either includes what has been superseded or does not; there is no
|
|
356
|
+
// "only", because a correction alone is not the subgraph anyone wanted.
|
|
357
|
+
if (flags.stale) {
|
|
358
|
+
throw usageError('search has no --stale', 'use --all to recall superseded nodes too');
|
|
359
|
+
}
|
|
360
|
+
const query = {
|
|
361
|
+
text,
|
|
362
|
+
kinds: flags.kind ? [flags.kind] : undefined,
|
|
363
|
+
limit: whole(flags.limit, '--limit'),
|
|
364
|
+
// Zero hops is a real request — the seeds and nothing they dragged in.
|
|
365
|
+
maxHops: whole(flags.hops, '--hops', 0),
|
|
366
|
+
maxNodes: whole(flags.nodes, '--nodes'),
|
|
367
|
+
minScore: threshold(flags['min-score']),
|
|
368
|
+
};
|
|
369
|
+
const ranker = await vectoriser(o, flags.embedding);
|
|
370
|
+
const sees = flags.audience ? [flags.audience] : audiences(o);
|
|
371
|
+
let rec;
|
|
372
|
+
try {
|
|
373
|
+
const index = new MemoryIndex({ store: o.store, embedder: ranker.embedder });
|
|
374
|
+
rec = await index.search(query, sees, { stale: flags.all });
|
|
375
|
+
}
|
|
376
|
+
catch (err) {
|
|
377
|
+
if (err instanceof MemoryError) {
|
|
378
|
+
throw new CliError(err.message, EXIT.usage, err.hint);
|
|
379
|
+
}
|
|
380
|
+
throw err;
|
|
381
|
+
}
|
|
382
|
+
if (asJson) {
|
|
383
|
+
json({
|
|
384
|
+
query,
|
|
385
|
+
sees,
|
|
386
|
+
// Named field by field: the ranker carries the embedder itself,
|
|
387
|
+
// which is a client object and no part of an answer.
|
|
388
|
+
ranking: { by: ranker.by, model: ranker.model, reason: ranker.reason },
|
|
389
|
+
seeds: rec.seeds,
|
|
390
|
+
truncated: rec.truncated,
|
|
391
|
+
nodes: rec.nodes.map((n) => ({
|
|
392
|
+
id: n.node.id,
|
|
393
|
+
kind: n.node.kind,
|
|
394
|
+
score: n.score,
|
|
395
|
+
seed: n.seed,
|
|
396
|
+
via: n.via,
|
|
397
|
+
text: n.node.text,
|
|
398
|
+
file: n.node.file ? hostPath(o.dir, n.node.file) : undefined,
|
|
399
|
+
})),
|
|
400
|
+
edges: rec.edges,
|
|
401
|
+
// Verbatim, because it is the thing that went into a prompt.
|
|
402
|
+
block: renderRecollection(rec, { tagged: false }),
|
|
403
|
+
});
|
|
404
|
+
return;
|
|
405
|
+
}
|
|
406
|
+
if (flags['ids-only']) {
|
|
407
|
+
writeAll(rec.nodes.map((n) => n.node.id));
|
|
408
|
+
return;
|
|
409
|
+
}
|
|
410
|
+
if (!rec.nodes.length) {
|
|
411
|
+
note(`nothing recalled for ${text}`);
|
|
412
|
+
ranking(ranker);
|
|
413
|
+
return;
|
|
414
|
+
}
|
|
415
|
+
write(renderRecollection(rec, { tagged: false }));
|
|
416
|
+
write();
|
|
417
|
+
ranking(ranker);
|
|
418
|
+
}
|
|
419
|
+
/**
|
|
420
|
+
* The embedder the project ranks with. Reproducing a recall means reproducing
|
|
421
|
+
* its vectors, so the ref is resolved through the same alias table a run uses
|
|
422
|
+
* and the keyring is materialised the same way.
|
|
423
|
+
*
|
|
424
|
+
* What cannot be built falls back to term overlap rather than failing, because
|
|
425
|
+
* a graph whose credentials have gone missing is one worth looking at. The
|
|
426
|
+
* fallback is a different ranking, though, so it is reported rather than passed
|
|
427
|
+
* off as recall.
|
|
428
|
+
*/
|
|
429
|
+
async function vectoriser(o, override) {
|
|
430
|
+
// A memory with no vectors cannot be ranked by meaning whatever we build:
|
|
431
|
+
// embedding a query against an empty block finds nothing, which would read
|
|
432
|
+
// as "nothing is remembered" rather than "there is nothing to rank with".
|
|
433
|
+
if (!o.store.embedding) {
|
|
434
|
+
return { by: 'term overlap', reason: 'this memory holds no vectors' };
|
|
435
|
+
}
|
|
436
|
+
const config = o.at?.config;
|
|
437
|
+
const declared = override ?? config?.memory?.embedding ?? config?.embedding ?? o.store.embedding.model;
|
|
438
|
+
// The project's own file first, then the keyring — the order `zen run`
|
|
439
|
+
// uses, so a query embeds here exactly when it would embed there.
|
|
440
|
+
if (o.at) {
|
|
441
|
+
loadProjectEnv(o.at.root);
|
|
442
|
+
}
|
|
443
|
+
(await KeyStore.open()).materialize();
|
|
444
|
+
try {
|
|
445
|
+
const ref = config?.embeddings?.[declared] ?? declared;
|
|
446
|
+
const embedder = createEmbedder(ref);
|
|
447
|
+
return { embedder, by: 'meaning', model: embedder.id };
|
|
448
|
+
}
|
|
449
|
+
catch (err) {
|
|
450
|
+
return { by: 'term overlap', reason: err instanceof Error ? err.message : String(err) };
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
/**
|
|
454
|
+
* Which ranking that was. A lexical fallback is not what an agent sees, and a
|
|
455
|
+
* block that looked like recall without being it would send someone chasing a
|
|
456
|
+
* difference that exists only in this command.
|
|
457
|
+
*/
|
|
458
|
+
function ranking(r) {
|
|
459
|
+
if (r.by === 'meaning') {
|
|
460
|
+
note(dim(`ranked by meaning · ${r.model}`));
|
|
461
|
+
return;
|
|
462
|
+
}
|
|
463
|
+
note(yellow('ranked by term overlap — this is not the order an agent sees'));
|
|
464
|
+
if (r.reason) {
|
|
465
|
+
note(dim(` ${r.reason}`));
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
/**
|
|
469
|
+
* Every label the graph uses. `recall` masks against a list rather than taking
|
|
470
|
+
* `undefined` for "no mask", so reading unmasked has to be spelled out — and
|
|
471
|
+
* spelling it out from the graph is exact, since a label nothing carries can
|
|
472
|
+
* hide nothing.
|
|
473
|
+
*/
|
|
474
|
+
function audiences(o) {
|
|
475
|
+
return [...new Set(o.store.graph.nodes().flatMap((n) => n.audience))];
|
|
476
|
+
}
|
|
477
|
+
/** Absent stays absent, so the engine's own default is the one on display. */
|
|
478
|
+
function whole(value, flag, min = 1) {
|
|
479
|
+
if (value === undefined) {
|
|
480
|
+
return undefined;
|
|
481
|
+
}
|
|
482
|
+
const n = Number(value);
|
|
483
|
+
if (!Number.isInteger(n) || n < min) {
|
|
484
|
+
throw usageError(`${flag} wants a whole number ${min} or more, got ${value}`, USAGE);
|
|
485
|
+
}
|
|
486
|
+
return n;
|
|
487
|
+
}
|
|
488
|
+
function threshold(value) {
|
|
489
|
+
if (value === undefined) {
|
|
490
|
+
return undefined;
|
|
491
|
+
}
|
|
492
|
+
const n = Number(value);
|
|
493
|
+
if (!Number.isFinite(n) || n < 0 || n > 1) {
|
|
494
|
+
throw usageError(`--min-score wants a number from 0 to 1, got ${value}`, USAGE);
|
|
495
|
+
}
|
|
496
|
+
return n;
|
|
497
|
+
}
|
|
498
|
+
// ---------------------------------------------------------------------------
|
|
499
|
+
// grep
|
|
500
|
+
// ---------------------------------------------------------------------------
|
|
501
|
+
/**
|
|
502
|
+
* The exhaustive read, and the counterpart to recall rather than a variant of
|
|
503
|
+
* it. Recall ranks, and a ranking returns the top of a list, so it cannot tell
|
|
504
|
+
* "there is nothing" from "nothing was close enough" — which is the question
|
|
505
|
+
* anyone actually has when they come to a memory looking for a name, a path or
|
|
506
|
+
* a command they think an agent wrote down.
|
|
507
|
+
*/
|
|
508
|
+
async function grep(o, rest, flags, asJson) {
|
|
509
|
+
const pattern = rest[0];
|
|
510
|
+
if (!pattern || rest.length > 1) {
|
|
511
|
+
throw usageError('grep takes exactly one pattern', 'zen memory grep <pattern>');
|
|
512
|
+
}
|
|
513
|
+
const limit = Number(flags.limit ?? LISTED);
|
|
514
|
+
if (!Number.isInteger(limit) || limit < 1) {
|
|
515
|
+
throw usageError(`--limit wants a positive whole number, got ${flags.limit}`, USAGE);
|
|
516
|
+
}
|
|
517
|
+
for (const field of flags.in ?? []) {
|
|
518
|
+
if (!GREP_FIELDS.includes(field)) {
|
|
519
|
+
throw usageError(`--in wants one of ${GREP_FIELDS.join(', ')}, got ${field}`, USAGE);
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
let res;
|
|
523
|
+
try {
|
|
524
|
+
res = await grepMemory(o.store, pattern, {
|
|
525
|
+
// Unmasked, like everything else here: this is the person who owns
|
|
526
|
+
// the files, not an agent inside a run.
|
|
527
|
+
kinds: flags.kind ? [flags.kind] : undefined,
|
|
528
|
+
audience: flags.audience,
|
|
529
|
+
files: flags.files,
|
|
530
|
+
in: flags.in,
|
|
531
|
+
stale: flags.stale ? 'only' : flags.all ? 'include' : 'exclude',
|
|
532
|
+
regex: flags.regex,
|
|
533
|
+
caseSensitive: flags['case-sensitive'],
|
|
534
|
+
limit,
|
|
535
|
+
});
|
|
536
|
+
}
|
|
537
|
+
catch (err) {
|
|
538
|
+
if (err instanceof MemoryError) {
|
|
539
|
+
throw new CliError(err.message, EXIT.usage, err.hint);
|
|
540
|
+
}
|
|
541
|
+
throw err;
|
|
542
|
+
}
|
|
543
|
+
if (asJson) {
|
|
544
|
+
json({
|
|
545
|
+
found: res.found,
|
|
546
|
+
truncated: res.truncated,
|
|
547
|
+
matches: res.matches.map((m) => ({
|
|
548
|
+
id: m.node.id,
|
|
549
|
+
kind: m.node.kind,
|
|
550
|
+
stale: m.stale,
|
|
551
|
+
file: m.node.file ? hostPath(o.dir, m.node.file) : undefined,
|
|
552
|
+
hits: m.hits,
|
|
553
|
+
more: m.more,
|
|
554
|
+
})),
|
|
555
|
+
unsearched: res.skipped,
|
|
556
|
+
});
|
|
557
|
+
return;
|
|
558
|
+
}
|
|
559
|
+
if (flags['ids-only']) {
|
|
560
|
+
writeAll(res.matches.map((m) => m.node.id));
|
|
561
|
+
return;
|
|
562
|
+
}
|
|
563
|
+
if (!res.found) {
|
|
564
|
+
note(`nothing matches ${pattern}`);
|
|
565
|
+
unsearched(res.skipped);
|
|
566
|
+
return;
|
|
567
|
+
}
|
|
568
|
+
for (const m of res.matches) {
|
|
569
|
+
write(cyan(m.node.id) +
|
|
570
|
+
' ' +
|
|
571
|
+
m.node.kind +
|
|
572
|
+
(m.stale ? ' ' + yellow('superseded') : '') +
|
|
573
|
+
(m.node.file ? ' ' + dim(m.node.file.path) : ''));
|
|
574
|
+
writeAll(table(m.hits.map((h) => [` ${dim(h.where + ':' + h.line)}`, clip(h.text, 120)])));
|
|
575
|
+
if (m.more) {
|
|
576
|
+
write(dim(` … ${count(m.more, 'more line')} in this node`));
|
|
577
|
+
}
|
|
578
|
+
write();
|
|
579
|
+
}
|
|
580
|
+
if (res.truncated) {
|
|
581
|
+
note(dim(`${res.found - res.matches.length} more nodes — raise --limit`));
|
|
582
|
+
}
|
|
583
|
+
unsearched(res.skipped);
|
|
584
|
+
}
|
|
585
|
+
/**
|
|
586
|
+
* Named rather than counted. A file that was not read is a hole in an answer
|
|
587
|
+
* whose whole value is that it has none, so the ids have to be printable.
|
|
588
|
+
*/
|
|
589
|
+
function unsearched(skipped) {
|
|
590
|
+
if (!skipped.length) {
|
|
591
|
+
return;
|
|
592
|
+
}
|
|
593
|
+
write();
|
|
594
|
+
note(yellow(`${count(skipped.length, 'file')} not searched`));
|
|
595
|
+
writeAll(table(skipped.map((s) => [` ${cyan(s.id)}`, dim(s.reason), s.path])));
|
|
596
|
+
}
|
|
597
|
+
// ---------------------------------------------------------------------------
|
|
268
598
|
// show
|
|
269
599
|
// ---------------------------------------------------------------------------
|
|
270
600
|
function show(o, ids, asJson) {
|
package/dist/commands/models.js
CHANGED
|
@@ -393,6 +393,18 @@ function verdict(probe) {
|
|
|
393
393
|
return dim('no answer');
|
|
394
394
|
}
|
|
395
395
|
}
|
|
396
|
+
/**
|
|
397
|
+
* What the provider said, in the verdict's own colour. Grey is for the half of
|
|
398
|
+
* the output nobody needs to read, and the one line explaining a refusal is not
|
|
399
|
+
* in that half.
|
|
400
|
+
*/
|
|
401
|
+
function reason(probe) {
|
|
402
|
+
const detail = probe.check.detail ?? '';
|
|
403
|
+
if (detail === '' || probe.check.state === 'live') {
|
|
404
|
+
return dim(detail);
|
|
405
|
+
}
|
|
406
|
+
return probe.check.state === 'dead' ? red(detail) : yellow(detail);
|
|
407
|
+
}
|
|
396
408
|
/**
|
|
397
409
|
* Which project and region a Vertex call was actually made in. A publisher
|
|
398
410
|
* model is only missing *somewhere*, and the somewhere is the part nobody
|
|
@@ -429,11 +441,11 @@ const test = async (ctx, args) => {
|
|
|
429
441
|
cyan(p.ref),
|
|
430
442
|
verdict(p),
|
|
431
443
|
dim(p.dimensions ? `${p.dimensions} dims` : ''),
|
|
432
|
-
|
|
444
|
+
reason(p),
|
|
433
445
|
])));
|
|
434
446
|
for (const p of probes) {
|
|
435
447
|
if (p.check.fix) {
|
|
436
|
-
note(
|
|
448
|
+
note(`${yellow('fix')} ${bold(p.ref)} ${p.check.fix}`);
|
|
437
449
|
}
|
|
438
450
|
}
|
|
439
451
|
// A Vertex refusal is about a model *in a project*, and the project is
|