botholomew 0.16.3 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -41
- package/package.json +3 -8
- package/src/chat/agent.ts +37 -40
- package/src/chat/session.ts +8 -8
- package/src/cli.ts +0 -2
- package/src/commands/capabilities.ts +32 -32
- package/src/commands/context.ts +124 -223
- package/src/commands/mcpx.ts +1 -1
- package/src/commands/nuke.ts +44 -15
- package/src/commands/prepare.ts +17 -13
- package/src/config/loader.ts +1 -8
- package/src/constants.ts +16 -32
- package/src/init/index.ts +11 -14
- package/src/mem/client.ts +17 -0
- package/src/{context → prompts}/capabilities.ts +11 -7
- package/src/schedules/store.ts +1 -1
- package/src/tasks/store.ts +1 -1
- package/src/threads/store.ts +1 -1
- package/src/tools/capabilities/refresh.ts +1 -1
- package/src/tools/membot/adapter.ts +111 -0
- package/src/tools/membot/copy.ts +59 -0
- package/src/tools/membot/count_lines.ts +53 -0
- package/src/tools/membot/edit.ts +72 -0
- package/src/tools/membot/exists.ts +54 -0
- package/src/tools/membot/index.ts +26 -0
- package/src/tools/{context → membot}/pipe.ts +34 -32
- package/src/tools/registry.ts +6 -37
- package/src/tools/tool.ts +6 -8
- package/src/tui/App.tsx +3 -4
- package/src/tui/components/ContextPanel.tsx +109 -226
- package/src/tui/components/HelpPanel.tsx +2 -2
- package/src/tui/components/MessageList.tsx +0 -1
- package/src/tui/components/StatusBar.tsx +0 -6
- package/src/tui/components/ThreadPanel.tsx +8 -7
- package/src/tui/wrapDetail.ts +11 -0
- package/src/worker/heartbeat.ts +0 -20
- package/src/worker/index.ts +11 -11
- package/src/worker/llm.ts +7 -9
- package/src/worker/prompt.ts +25 -13
- package/src/worker/spawn.ts +1 -1
- package/src/worker/tick.ts +10 -9
- package/src/commands/db.ts +0 -119
- package/src/commands/with-db.ts +0 -22
- package/src/context/chunker.ts +0 -275
- package/src/context/embedder-impl.ts +0 -100
- package/src/context/embedder.ts +0 -9
- package/src/context/fetcher-errors.ts +0 -8
- package/src/context/fetcher.ts +0 -515
- package/src/context/locks.ts +0 -146
- package/src/context/markdown-converter.ts +0 -186
- package/src/context/reindex.ts +0 -198
- package/src/context/store.ts +0 -841
- package/src/context/url-utils.ts +0 -25
- package/src/db/connection.ts +0 -255
- package/src/db/doctor.ts +0 -235
- package/src/db/embeddings.ts +0 -317
- package/src/db/query.ts +0 -56
- package/src/db/schema.ts +0 -93
- package/src/db/sql/1-core_tables.sql +0 -53
- package/src/db/sql/10-dedupe_context_items.sql +0 -26
- package/src/db/sql/11-rebuild_hnsw.sql +0 -8
- package/src/db/sql/12-workers.sql +0 -66
- package/src/db/sql/13-drive-paths.sql +0 -47
- package/src/db/sql/14-drop_hnsw_index.sql +0 -8
- package/src/db/sql/15-fts_index.sql +0 -8
- package/src/db/sql/16-source_url.sql +0 -7
- package/src/db/sql/17-worker_log_path.sql +0 -3
- package/src/db/sql/18-reset_embeddings_for_local.sql +0 -39
- package/src/db/sql/19-disk_backed_index.sql +0 -36
- package/src/db/sql/2-logging_tables.sql +0 -24
- package/src/db/sql/20-drop_db_tables_for_files.sql +0 -19
- package/src/db/sql/3-daemon_state.sql +0 -5
- package/src/db/sql/4-unique_context_path.sql +0 -1
- package/src/db/sql/5-reset_embeddings_for_openai.sql +0 -1
- package/src/db/sql/6-vss_index.sql +0 -7
- package/src/db/sql/7-drop_embeddings_fk.sql +0 -23
- package/src/db/sql/8-task_output.sql +0 -1
- package/src/db/sql/9-source-type.sql +0 -1
- package/src/tools/context/read-large-result.ts +0 -33
- package/src/tools/dir/create.ts +0 -47
- package/src/tools/dir/size.ts +0 -77
- package/src/tools/dir/tree.ts +0 -124
- package/src/tools/file/copy.ts +0 -73
- package/src/tools/file/count-lines.ts +0 -54
- package/src/tools/file/delete.ts +0 -83
- package/src/tools/file/edit.ts +0 -76
- package/src/tools/file/exists.ts +0 -33
- package/src/tools/file/info.ts +0 -66
- package/src/tools/file/move.ts +0 -66
- package/src/tools/file/read.ts +0 -67
- package/src/tools/file/write.ts +0 -58
- package/src/tools/search/fuse.ts +0 -96
- package/src/tools/search/index.ts +0 -127
- package/src/tools/search/regexp.ts +0 -82
- package/src/tools/search/semantic.ts +0 -167
- /package/src/{db → utils}/uuid.ts +0 -0
package/src/commands/context.ts
CHANGED
|
@@ -1,245 +1,146 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { copyFileSync, existsSync, mkdirSync, statSync } from "node:fs";
|
|
2
|
+
import { createRequire } from "node:module";
|
|
3
|
+
import { homedir } from "node:os";
|
|
2
4
|
import { join } from "node:path";
|
|
3
|
-
import
|
|
5
|
+
import { fileURLToPath } from "node:url";
|
|
4
6
|
import type { Command } from "commander";
|
|
5
|
-
import {
|
|
6
|
-
import { loadConfig } from "../config/loader.ts";
|
|
7
|
-
import { CONTEXT_DIR, getDbPath } from "../constants.ts";
|
|
8
|
-
import { fetchUrl } from "../context/fetcher.ts";
|
|
9
|
-
import { reindexContext } from "../context/reindex.ts";
|
|
10
|
-
import {
|
|
11
|
-
buildTree,
|
|
12
|
-
fileExists,
|
|
13
|
-
listContextDir,
|
|
14
|
-
type TreeNode,
|
|
15
|
-
writeContextFile,
|
|
16
|
-
} from "../context/store.ts";
|
|
17
|
-
import { withDb } from "../db/connection.ts";
|
|
18
|
-
import { indexStats } from "../db/embeddings.ts";
|
|
19
|
-
import { migrate } from "../db/schema.ts";
|
|
20
|
-
import { createMcpxClient } from "../mcpx/client.ts";
|
|
21
|
-
import {
|
|
22
|
-
type ContextFileMeta,
|
|
23
|
-
serializeContextFile,
|
|
24
|
-
} from "../utils/frontmatter.ts";
|
|
7
|
+
import { defaultCliName, OPERATIONS } from "membot";
|
|
25
8
|
import { logger } from "../utils/logger.ts";
|
|
26
9
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
.description(
|
|
31
|
-
"Inspect and manage the on-disk context/ tree (the agent's knowledge store)",
|
|
32
|
-
);
|
|
10
|
+
const require = createRequire(import.meta.url);
|
|
11
|
+
const ourPkg = require("../../package.json");
|
|
12
|
+
const membotPkg = require("membot/package.json");
|
|
33
13
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
14
|
+
// Soft warning rather than a hard error — membot's SDK API is stable within a
|
|
15
|
+
// minor version, and dev workspaces sometimes pin a newer copy.
|
|
16
|
+
const requested = (ourPkg.dependencies.membot as string).replace(/^[\^~]/, "");
|
|
17
|
+
if (!membotPkg.version.startsWith(requested.split(".")[0])) {
|
|
18
|
+
logger.warn(
|
|
19
|
+
`membot version drift: installed ${membotPkg.version}, expected ${ourPkg.dependencies.membot}`,
|
|
20
|
+
);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const MEMBOT_CLI = fileURLToPath(import.meta.resolve("membot/cli"));
|
|
24
|
+
|
|
25
|
+
function getDir(program: Command): string {
|
|
26
|
+
return program.opts().dir;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Slice process.argv from the token after "context" so flags (including
|
|
31
|
+
* --help) and positional args flow through to upstream membot verbatim.
|
|
32
|
+
*/
|
|
33
|
+
function getRawContextArgs(): string[] {
|
|
34
|
+
const idx = process.argv.indexOf("context");
|
|
35
|
+
return idx === -1 ? [] : process.argv.slice(idx + 1);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
async function runMembot(projectDir: string, args: string[]): Promise<number> {
|
|
39
|
+
// Point membot at <projectDir> as its data dir via the `--config` flag —
|
|
40
|
+
// each Botholomew project gets its own membot store at
|
|
41
|
+
// `<projectDir>/index.duckdb`. Forward stdio so the user sees the same
|
|
42
|
+
// output they would running `membot` directly.
|
|
43
|
+
const proc = Bun.spawn(["bun", MEMBOT_CLI, "--config", projectDir, ...args], {
|
|
44
|
+
stdout: "inherit",
|
|
45
|
+
stderr: "inherit",
|
|
46
|
+
stdin: "inherit",
|
|
47
|
+
});
|
|
48
|
+
return await proc.exited;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Copy the system-wide `~/.membot` data dir into the project, mirroring
|
|
53
|
+
* `botholomew mcpx import-global`. Useful when the user has built up a
|
|
54
|
+
* personal knowledge base globally and wants to seed a new project with it.
|
|
55
|
+
*
|
|
56
|
+
* Refuses to overwrite a non-empty project membot store unless `--force` is
|
|
57
|
+
* passed — accidentally clobbering an active project's index is much worse
|
|
58
|
+
* than re-running the import.
|
|
59
|
+
*/
|
|
60
|
+
function registerImportGlobal(parent: Command, program: Command): void {
|
|
61
|
+
parent
|
|
62
|
+
.command("import-global")
|
|
63
|
+
.description("Copy system-wide membot data (~/.membot) into this project")
|
|
44
64
|
.option(
|
|
45
|
-
"--
|
|
46
|
-
"
|
|
65
|
+
"-f, --force",
|
|
66
|
+
"Overwrite an existing index.duckdb in the project",
|
|
67
|
+
false,
|
|
47
68
|
)
|
|
48
|
-
.
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
const mcpxClient = await createMcpxClient(dir);
|
|
53
|
-
logger.info(`importing ${url}`);
|
|
54
|
-
try {
|
|
55
|
-
const fetched = await fetchUrl(url, config, mcpxClient, opts.prompt);
|
|
56
|
-
const dest = opts.path ?? deriveContextPath(url, fetched.source);
|
|
57
|
-
const meta: ContextFileMeta = {
|
|
58
|
-
source_url: url,
|
|
59
|
-
imported_at: new Date().toISOString(),
|
|
60
|
-
};
|
|
61
|
-
// Title falls back to the URL when fetcher couldn't extract one —
|
|
62
|
-
// skip it in that case to avoid duplicating source_url.
|
|
63
|
-
if (fetched.title && fetched.title !== url) {
|
|
64
|
-
meta.title = fetched.title;
|
|
65
|
-
}
|
|
66
|
-
const body = serializeContextFile(meta, fetched.content);
|
|
67
|
-
await writeContextFile(dir, dest, body, {
|
|
68
|
-
onConflict: opts.overwrite ? "overwrite" : "error",
|
|
69
|
-
});
|
|
70
|
-
logger.success(
|
|
71
|
-
`imported ${body.length} bytes → ${ansis.bold(`context/${dest}`)} (source: ${fetched.source ?? "http"})`,
|
|
72
|
-
);
|
|
73
|
-
|
|
74
|
-
// Reindex so the new file is searchable. reindexContext is
|
|
75
|
-
// incremental — files whose content_hash matches the index are
|
|
76
|
-
// skipped, so this only embeds the file we just wrote.
|
|
77
|
-
const dbPath = getDbPath(dir);
|
|
78
|
-
await withDb(dbPath, migrate);
|
|
79
|
-
const summary = await reindexContext(dir, config, dbPath, {
|
|
80
|
-
onProgress: (msg) => logger.dim(` ${msg}`),
|
|
81
|
-
});
|
|
82
|
-
logger.success(
|
|
83
|
-
`indexed: ${summary.added} added, ${summary.updated} updated, ${summary.unchanged} unchanged, ${summary.chunksWritten} chunks written`,
|
|
84
|
-
);
|
|
85
|
-
} catch (err) {
|
|
86
|
-
logger.error(
|
|
87
|
-
`import failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
88
|
-
);
|
|
69
|
+
.action((opts: { force: boolean }) => {
|
|
70
|
+
const globalDir = join(homedir(), ".membot");
|
|
71
|
+
if (!existsSync(globalDir)) {
|
|
72
|
+
logger.error("No global membot data found at ~/.membot");
|
|
89
73
|
process.exit(1);
|
|
90
|
-
} finally {
|
|
91
|
-
await mcpxClient?.close();
|
|
92
74
|
}
|
|
93
|
-
});
|
|
94
|
-
|
|
95
|
-
// ---- reindex -------------------------------------------------------------
|
|
96
|
-
context
|
|
97
|
-
.command("reindex")
|
|
98
|
-
.description(
|
|
99
|
-
"Walk context/ and reconcile the search index: embed new files, re-embed changed ones, drop rows for removed ones.",
|
|
100
|
-
)
|
|
101
|
-
.action(async () => {
|
|
102
|
-
const dir = program.opts().dir;
|
|
103
|
-
const config = await loadConfig(dir);
|
|
104
|
-
const dbPath = getDbPath(dir);
|
|
105
|
-
// The migrate() call ensures the index DB is initialized, including
|
|
106
|
-
// the context_index table from migration 19, before we try to write.
|
|
107
|
-
await withDb(dbPath, migrate);
|
|
108
|
-
const spinner = createSpinner("reindexing").start();
|
|
109
|
-
const summary = await reindexContext(dir, config, dbPath, {
|
|
110
|
-
onProgress: (msg) => spinner.update({ text: msg }),
|
|
111
|
-
});
|
|
112
|
-
const parts = [
|
|
113
|
-
`${summary.added} added`,
|
|
114
|
-
`${summary.updated} updated`,
|
|
115
|
-
`${summary.unchanged} unchanged`,
|
|
116
|
-
`${summary.removed} removed`,
|
|
117
|
-
`${summary.chunksWritten} chunks written`,
|
|
118
|
-
];
|
|
119
|
-
spinner.success({ text: parts.join(", ") });
|
|
120
|
-
});
|
|
121
75
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
.description("Render the context/ tree (or a subdirectory).")
|
|
126
|
-
.option(
|
|
127
|
-
"-d, --max-depth <n>",
|
|
128
|
-
"max directory depth to render",
|
|
129
|
-
Number.parseInt,
|
|
130
|
-
10,
|
|
131
|
-
)
|
|
132
|
-
.action(async (path: string | undefined, opts) => {
|
|
133
|
-
const dir = program.opts().dir;
|
|
134
|
-
const node = await buildTree(dir, path ?? "", opts.maxDepth);
|
|
135
|
-
console.log(renderTreeAnsi(node));
|
|
136
|
-
});
|
|
76
|
+
const projectDir = getDir(program);
|
|
77
|
+
const dest = (name: string) => join(projectDir, name);
|
|
78
|
+
const destDb = dest("index.duckdb");
|
|
137
79
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
const dir = program.opts().dir;
|
|
146
|
-
const dbPath = getDbPath(dir);
|
|
147
|
-
const exists = await fileExists(dir, "");
|
|
148
|
-
if (!exists) {
|
|
149
|
-
logger.dim(`context/ does not exist under ${dir}`);
|
|
150
|
-
return;
|
|
151
|
-
}
|
|
152
|
-
const entries = await listContextDir(dir, "", { recursive: true });
|
|
153
|
-
let files = 0;
|
|
154
|
-
let textual = 0;
|
|
155
|
-
let bytes = 0;
|
|
156
|
-
for (const e of entries) {
|
|
157
|
-
if (e.is_directory) continue;
|
|
158
|
-
files++;
|
|
159
|
-
if (e.is_textual) textual++;
|
|
160
|
-
try {
|
|
161
|
-
const st = await stat(join(dir, CONTEXT_DIR, e.path));
|
|
162
|
-
bytes += st.size;
|
|
163
|
-
} catch {
|
|
164
|
-
// file vanished mid-walk — skip
|
|
80
|
+
if (existsSync(destDb) && !opts.force) {
|
|
81
|
+
const stat = statSync(destDb);
|
|
82
|
+
if (stat.size > 0) {
|
|
83
|
+
logger.error(
|
|
84
|
+
`Refusing to overwrite ${destDb} (${stat.size} bytes). Pass --force to replace it.`,
|
|
85
|
+
);
|
|
86
|
+
process.exit(1);
|
|
165
87
|
}
|
|
166
88
|
}
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
` ${ansis.dim((label ?? "").padEnd(labelWidth))} ${value}`,
|
|
184
|
-
);
|
|
89
|
+
|
|
90
|
+
mkdirSync(projectDir, { recursive: true });
|
|
91
|
+
|
|
92
|
+
const filesToCopy = ["index.duckdb", "config.json"];
|
|
93
|
+
let copied = 0;
|
|
94
|
+
for (const file of filesToCopy) {
|
|
95
|
+
const src = join(globalDir, file);
|
|
96
|
+
if (!existsSync(src)) continue;
|
|
97
|
+
copyFileSync(src, dest(file));
|
|
98
|
+
logger.success(`Copied ${file}`);
|
|
99
|
+
copied++;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
if (copied === 0) {
|
|
103
|
+
logger.warn("No files found in ~/.membot to copy.");
|
|
104
|
+
return;
|
|
185
105
|
}
|
|
106
|
+
|
|
107
|
+
logger.success(
|
|
108
|
+
`Imported ${copied} file(s) from ~/.membot into ${projectDir}`,
|
|
109
|
+
);
|
|
186
110
|
});
|
|
187
111
|
}
|
|
188
112
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
function deriveContextPath(url: string, source: string | null): string {
|
|
196
|
-
const slug = slugifyUrl(url);
|
|
197
|
-
const root = source ?? "url";
|
|
198
|
-
return `${root}/${slug}.md`;
|
|
199
|
-
}
|
|
113
|
+
export function registerContextCommand(program: Command) {
|
|
114
|
+
const context = program
|
|
115
|
+
.command("context")
|
|
116
|
+
.description(
|
|
117
|
+
"Manage the project's knowledge store via membot (add, search, ls, read, …)",
|
|
118
|
+
);
|
|
200
119
|
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
parsed = new URL(url);
|
|
205
|
-
} catch {
|
|
206
|
-
return url.replace(/[^a-z0-9]+/gi, "-").slice(0, 80);
|
|
207
|
-
}
|
|
208
|
-
const path = parsed.pathname.replace(/^\/+|\/+$/g, "").replace(/\//g, "_");
|
|
209
|
-
const base = path || parsed.hostname;
|
|
210
|
-
return `${parsed.hostname}_${base}`
|
|
211
|
-
.replace(/[^a-z0-9._-]+/gi, "-")
|
|
212
|
-
.replace(/-+/g, "-")
|
|
213
|
-
.slice(0, 80);
|
|
214
|
-
}
|
|
120
|
+
// Botholomew-specific helpers first so they show up before the membot
|
|
121
|
+
// passthrough subcommands in --help.
|
|
122
|
+
registerImportGlobal(context, program);
|
|
215
123
|
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
const children = node.children;
|
|
232
|
-
children.forEach((c, i) => {
|
|
233
|
-
const last = i === children.length - 1;
|
|
234
|
-
lines.push(renderTreeAnsi(c, childPrefix, last, false));
|
|
235
|
-
});
|
|
124
|
+
// One Commander subcommand per membot Operation. We don't redeclare any
|
|
125
|
+
// flags — Commander hands the raw argv slice to membot, which owns the
|
|
126
|
+
// canonical schema.
|
|
127
|
+
for (const op of OPERATIONS) {
|
|
128
|
+
const name = defaultCliName(op);
|
|
129
|
+
context
|
|
130
|
+
.command(name)
|
|
131
|
+
.description(op.description.split("\n")[0] ?? op.description)
|
|
132
|
+
.allowUnknownOption(true)
|
|
133
|
+
.helpOption(false)
|
|
134
|
+
.argument("[args...]", "arguments forwarded to membot")
|
|
135
|
+
.action(async () => {
|
|
136
|
+
const exitCode = await runMembot(getDir(program), getRawContextArgs());
|
|
137
|
+
if (exitCode !== 0) process.exit(exitCode);
|
|
138
|
+
});
|
|
236
139
|
}
|
|
237
|
-
return lines.join("\n");
|
|
238
|
-
}
|
|
239
140
|
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
141
|
+
// `botholomew context` (no subcommand) → membot's default action.
|
|
142
|
+
context.action(async () => {
|
|
143
|
+
const exitCode = await runMembot(getDir(program), getRawContextArgs());
|
|
144
|
+
if (exitCode !== 0) process.exit(exitCode);
|
|
145
|
+
});
|
|
245
146
|
}
|
package/src/commands/mcpx.ts
CHANGED
|
@@ -7,8 +7,8 @@ import type { Command } from "commander";
|
|
|
7
7
|
import { createSpinner } from "nanospinner";
|
|
8
8
|
import { loadConfig } from "../config/loader.ts";
|
|
9
9
|
import { getMcpxDir } from "../constants.ts";
|
|
10
|
-
import { writeCapabilitiesFile } from "../context/capabilities.ts";
|
|
11
10
|
import { createMcpxClient } from "../mcpx/client.ts";
|
|
11
|
+
import { writeCapabilitiesFile } from "../prompts/capabilities.ts";
|
|
12
12
|
import { registerAllTools } from "../tools/registry.ts";
|
|
13
13
|
import { logger } from "../utils/logger.ts";
|
|
14
14
|
|
package/src/commands/nuke.ts
CHANGED
|
@@ -1,20 +1,16 @@
|
|
|
1
1
|
import { rm } from "node:fs/promises";
|
|
2
|
+
import { join } from "node:path";
|
|
2
3
|
import ansis from "ansis";
|
|
3
4
|
import type { Command } from "commander";
|
|
4
|
-
import {
|
|
5
|
-
|
|
6
|
-
getContextDir,
|
|
7
|
-
SCHEDULES_DIR,
|
|
8
|
-
TASKS_DIR,
|
|
9
|
-
THREADS_DIR,
|
|
10
|
-
} from "../constants.ts";
|
|
5
|
+
import { SCHEDULES_DIR, TASKS_DIR, THREADS_DIR } from "../constants.ts";
|
|
6
|
+
import { openMembot } from "../mem/client.ts";
|
|
11
7
|
import { deleteAllSchedules } from "../schedules/store.ts";
|
|
12
8
|
import { deleteAllTasks } from "../tasks/store.ts";
|
|
13
9
|
import { deleteAllThreads } from "../threads/store.ts";
|
|
14
10
|
import { logger } from "../utils/logger.ts";
|
|
15
11
|
import { listWorkers } from "../workers/store.ts";
|
|
16
12
|
|
|
17
|
-
type NukeScope = "
|
|
13
|
+
type NukeScope = "knowledge" | "tasks" | "schedules" | "threads" | "all";
|
|
18
14
|
|
|
19
15
|
async function ensureNoRunningWorkers(projectDir: string): Promise<boolean> {
|
|
20
16
|
const running = await listWorkers(projectDir, { status: "running" });
|
|
@@ -30,10 +26,43 @@ async function ensureNoRunningWorkers(projectDir: string): Promise<boolean> {
|
|
|
30
26
|
return true;
|
|
31
27
|
}
|
|
32
28
|
|
|
29
|
+
/**
|
|
30
|
+
* Erase the membot knowledge store: tombstone every current file and prune
|
|
31
|
+
* non-current versions. We do this through the SDK rather than `rm`ing
|
|
32
|
+
* `index.duckdb` directly so a concurrent open client (e.g. a tool call in
|
|
33
|
+
* mid-flight) doesn't lose data unpredictably; if the file is empty / missing
|
|
34
|
+
* we fall back to deleting it outright.
|
|
35
|
+
*/
|
|
36
|
+
async function nukeKnowledge(projectDir: string): Promise<void> {
|
|
37
|
+
const indexPath = join(projectDir, "index.duckdb");
|
|
38
|
+
try {
|
|
39
|
+
const mem = openMembot(projectDir);
|
|
40
|
+
try {
|
|
41
|
+
const list = await mem.list({ limit: 100_000 });
|
|
42
|
+
const paths = list.entries.map((e) => e.logical_path);
|
|
43
|
+
if (paths.length > 0) {
|
|
44
|
+
await mem.remove({ paths });
|
|
45
|
+
}
|
|
46
|
+
// Drop all versions older than now (i.e. everything we just tombstoned).
|
|
47
|
+
await mem.prune({ before: new Date().toISOString() });
|
|
48
|
+
logger.success(
|
|
49
|
+
`Cleared the membot knowledge store (${paths.length} entries removed, history pruned)`,
|
|
50
|
+
);
|
|
51
|
+
} finally {
|
|
52
|
+
await mem.close();
|
|
53
|
+
}
|
|
54
|
+
} catch (err) {
|
|
55
|
+
logger.warn(
|
|
56
|
+
`membot prune failed (${(err as Error).message}); removing ${indexPath} instead`,
|
|
57
|
+
);
|
|
58
|
+
await rm(indexPath, { force: true });
|
|
59
|
+
logger.success(`Removed ${indexPath}`);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
33
63
|
async function runNuke(projectDir: string, scope: NukeScope): Promise<void> {
|
|
34
|
-
if (scope === "
|
|
35
|
-
await
|
|
36
|
-
logger.success(`Removed ${CONTEXT_DIR}/ directory`);
|
|
64
|
+
if (scope === "knowledge" || scope === "all") {
|
|
65
|
+
await nukeKnowledge(projectDir);
|
|
37
66
|
}
|
|
38
67
|
if (scope === "tasks" || scope === "all") {
|
|
39
68
|
const n = await deleteAllTasks(projectDir);
|
|
@@ -71,7 +100,7 @@ function registerScope(
|
|
|
71
100
|
console.log(ansis.red.bold(`Nuke scope: ${scope}`));
|
|
72
101
|
console.log(
|
|
73
102
|
ansis.yellow(
|
|
74
|
-
`Re-run with --yes to confirm. This will delete
|
|
103
|
+
`Re-run with --yes to confirm. This will delete data; cannot be undone.`,
|
|
75
104
|
),
|
|
76
105
|
);
|
|
77
106
|
process.exit(1);
|
|
@@ -89,8 +118,8 @@ export function registerNukeCommand(program: Command) {
|
|
|
89
118
|
registerScope(
|
|
90
119
|
program,
|
|
91
120
|
nuke,
|
|
92
|
-
"
|
|
93
|
-
|
|
121
|
+
"knowledge",
|
|
122
|
+
"Erase the entire membot knowledge store (every current entry tombstoned and pruned)",
|
|
94
123
|
);
|
|
95
124
|
registerScope(
|
|
96
125
|
program,
|
|
@@ -114,6 +143,6 @@ export function registerNukeCommand(program: Command) {
|
|
|
114
143
|
program,
|
|
115
144
|
nuke,
|
|
116
145
|
"all",
|
|
117
|
-
"Erase all agent-writable data:
|
|
146
|
+
"Erase all agent-writable data: membot store, tasks/, schedules/, threads/",
|
|
118
147
|
);
|
|
119
148
|
}
|
package/src/commands/prepare.ts
CHANGED
|
@@ -1,21 +1,25 @@
|
|
|
1
1
|
import type { Command } from "commander";
|
|
2
2
|
import { loadConfig } from "../config/loader.ts";
|
|
3
|
-
import {
|
|
3
|
+
import { openMembot } from "../mem/client.ts";
|
|
4
4
|
import { logger } from "../utils/logger.ts";
|
|
5
|
-
import { withDb } from "./with-db.ts";
|
|
6
5
|
|
|
7
6
|
export function registerPrepareCommand(program: Command) {
|
|
8
7
|
program
|
|
9
8
|
.command("prepare")
|
|
10
|
-
.description(
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
9
|
+
.description(
|
|
10
|
+
"Verify the project is healthy: load config and open the membot knowledge store (triggers any first-run migration / model download).",
|
|
11
|
+
)
|
|
12
|
+
.action(async () => {
|
|
13
|
+
const projectDir = program.opts().dir as string;
|
|
14
|
+
logger.info("Preparing Botholomew...");
|
|
15
|
+
const config = await loadConfig(projectDir);
|
|
16
|
+
void config;
|
|
17
|
+
const mem = openMembot(projectDir);
|
|
18
|
+
try {
|
|
19
|
+
await mem.connect();
|
|
20
|
+
logger.success("membot knowledge store opened successfully");
|
|
21
|
+
} finally {
|
|
22
|
+
await mem.close();
|
|
23
|
+
}
|
|
24
|
+
});
|
|
21
25
|
}
|
package/src/config/loader.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { getConfigPath, getModelsDir } from "../constants.ts";
|
|
1
|
+
import { getConfigPath } from "../constants.ts";
|
|
3
2
|
import { setLogLevel } from "../utils/logger.ts";
|
|
4
3
|
import { type BotholomewConfig, DEFAULT_CONFIG } from "./schemas.ts";
|
|
5
4
|
|
|
@@ -23,12 +22,6 @@ export async function loadConfig(
|
|
|
23
22
|
|
|
24
23
|
setLogLevel(config.log_level);
|
|
25
24
|
|
|
26
|
-
const modelsDir = getModelsDir(projectDir);
|
|
27
|
-
mkdirSync(modelsDir, { recursive: true });
|
|
28
|
-
// Dynamic import keeps @huggingface/transformers (heavy, pulls ONNX runtime) out of commands that never embed.
|
|
29
|
-
const { setEmbeddingCacheDir } = await import("../context/embedder-impl.ts");
|
|
30
|
-
setEmbeddingCacheDir(modelsDir);
|
|
31
|
-
|
|
32
25
|
return config;
|
|
33
26
|
}
|
|
34
27
|
|
package/src/constants.ts
CHANGED
|
@@ -6,19 +6,22 @@ import { join } from "node:path";
|
|
|
6
6
|
*
|
|
7
7
|
* <projectDir>/
|
|
8
8
|
* config/config.json
|
|
9
|
-
*
|
|
9
|
+
* config.json membot config (separate from ours)
|
|
10
|
+
* index.duckdb membot-owned knowledge store
|
|
11
|
+
* prompts/*.md init seeds goals/beliefs/capabilities
|
|
10
12
|
* skills/*.md
|
|
11
13
|
* mcpx/servers.json
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
* tasks/.locks/<id>.lock O_EXCL claim files
|
|
16
|
-
* schedules/<id>.md schedules
|
|
14
|
+
* tasks/<id>.md tasks (status in frontmatter)
|
|
15
|
+
* tasks/.locks/<id>.lock O_EXCL claim files
|
|
16
|
+
* schedules/<id>.md
|
|
17
17
|
* schedules/.locks/<id>.lock
|
|
18
|
-
* threads/<YYYY-MM-DD>/<id>.csv
|
|
19
|
-
* workers/<id>.json
|
|
20
|
-
* logs/
|
|
21
|
-
*
|
|
18
|
+
* threads/<YYYY-MM-DD>/<id>.csv conversation history
|
|
19
|
+
* workers/<id>.json pidfile + heartbeat
|
|
20
|
+
* logs/ worker logs
|
|
21
|
+
*
|
|
22
|
+
* The agent's knowledge ("what used to be `context/`") now lives in
|
|
23
|
+
* `index.duckdb`, managed by the `membot` library. Tasks, schedules, threads,
|
|
24
|
+
* workers, prompts, and skills remain real files on disk.
|
|
22
25
|
*/
|
|
23
26
|
|
|
24
27
|
export const HOME_CONFIG_DIR = join(homedir(), ".botholomew");
|
|
@@ -32,14 +35,11 @@ export const DEFAULTS = {
|
|
|
32
35
|
UPDATE_CHECK_TIMEOUT_MS: 5_000,
|
|
33
36
|
} as const;
|
|
34
37
|
|
|
35
|
-
export const INDEX_DB_FILENAME = "index.duckdb";
|
|
36
38
|
export const CONFIG_DIR = "config";
|
|
37
39
|
export const CONFIG_FILENAME = "config.json";
|
|
38
40
|
export const PROMPTS_DIR = "prompts";
|
|
39
41
|
export const SKILLS_DIR = "skills";
|
|
40
42
|
export const MCPX_DIR = "mcpx";
|
|
41
|
-
export const MODELS_DIR = "models";
|
|
42
|
-
export const CONTEXT_DIR = "context";
|
|
43
43
|
export const TASKS_DIR = "tasks";
|
|
44
44
|
export const SCHEDULES_DIR = "schedules";
|
|
45
45
|
export const LOCKS_SUBDIR = ".locks";
|
|
@@ -47,24 +47,18 @@ export const LOGS_DIR = "logs";
|
|
|
47
47
|
export const WORKERS_DIR = "workers";
|
|
48
48
|
export const THREADS_DIR = "threads";
|
|
49
49
|
export const MCPX_SERVERS_FILENAME = "servers.json";
|
|
50
|
-
export const EMBEDDING_DIMENSION = 384;
|
|
51
|
-
export const EMBEDDING_MODEL = "Xenova/bge-small-en-v1.5";
|
|
52
50
|
|
|
53
51
|
/**
|
|
54
|
-
* Top-level areas tools must never touch directly.
|
|
55
|
-
*
|
|
52
|
+
* Top-level areas tools must never touch directly. Tasks/schedule lockfile
|
|
53
|
+
* dirs are kept off-limits because their `O_EXCL` claim semantics break if
|
|
54
|
+
* something else writes into them.
|
|
56
55
|
*/
|
|
57
56
|
export const PROTECTED_AREAS: ReadonlySet<string> = new Set([
|
|
58
|
-
MODELS_DIR,
|
|
59
57
|
LOGS_DIR,
|
|
60
58
|
`${TASKS_DIR}/${LOCKS_SUBDIR}`,
|
|
61
59
|
`${SCHEDULES_DIR}/${LOCKS_SUBDIR}`,
|
|
62
60
|
]);
|
|
63
61
|
|
|
64
|
-
export function getDbPath(projectDir: string): string {
|
|
65
|
-
return join(projectDir, INDEX_DB_FILENAME);
|
|
66
|
-
}
|
|
67
|
-
|
|
68
62
|
export function getWorkerLogsDir(projectDir: string): string {
|
|
69
63
|
return join(projectDir, LOGS_DIR);
|
|
70
64
|
}
|
|
@@ -92,12 +86,6 @@ export function getMcpxDir(projectDir: string): string {
|
|
|
92
86
|
return join(projectDir, MCPX_DIR);
|
|
93
87
|
}
|
|
94
88
|
|
|
95
|
-
export function getModelsDir(projectDir: string): string {
|
|
96
|
-
return (
|
|
97
|
-
process.env.BOTHOLOMEW_MODELS_DIR_OVERRIDE ?? join(projectDir, MODELS_DIR)
|
|
98
|
-
);
|
|
99
|
-
}
|
|
100
|
-
|
|
101
89
|
export function getSkillsDir(projectDir: string): string {
|
|
102
90
|
return join(projectDir, SKILLS_DIR);
|
|
103
91
|
}
|
|
@@ -106,10 +94,6 @@ export function getPromptsDir(projectDir: string): string {
|
|
|
106
94
|
return join(projectDir, PROMPTS_DIR);
|
|
107
95
|
}
|
|
108
96
|
|
|
109
|
-
export function getContextDir(projectDir: string): string {
|
|
110
|
-
return join(projectDir, CONTEXT_DIR);
|
|
111
|
-
}
|
|
112
|
-
|
|
113
97
|
export function getTasksDir(projectDir: string): string {
|
|
114
98
|
return join(projectDir, TASKS_DIR);
|
|
115
99
|
}
|