botholomew 0.12.5 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/README.md +91 -68
  2. package/package.json +2 -2
  3. package/src/chat/agent.ts +59 -86
  4. package/src/chat/session.ts +29 -25
  5. package/src/commands/capabilities.ts +1 -1
  6. package/src/commands/context.ts +178 -926
  7. package/src/commands/db.ts +9 -13
  8. package/src/commands/init.ts +4 -1
  9. package/src/commands/nuke.ts +57 -90
  10. package/src/commands/schedule.ts +103 -124
  11. package/src/commands/skill.ts +2 -2
  12. package/src/commands/task.ts +86 -95
  13. package/src/commands/thread.ts +107 -112
  14. package/src/commands/worker.ts +88 -88
  15. package/src/constants.ts +93 -16
  16. package/src/context/capabilities.ts +10 -10
  17. package/src/context/fetcher.ts +9 -10
  18. package/src/context/reindex.ts +189 -0
  19. package/src/context/store.ts +803 -0
  20. package/src/db/doctor.ts +1 -8
  21. package/src/db/embeddings.ts +227 -175
  22. package/src/db/sql/19-disk_backed_index.sql +36 -0
  23. package/src/db/sql/20-drop_db_tables_for_files.sql +19 -0
  24. package/src/fs/atomic.ts +217 -0
  25. package/src/fs/compat.ts +86 -0
  26. package/src/fs/sandbox.ts +293 -0
  27. package/src/init/index.ts +69 -52
  28. package/src/init/templates.ts +1 -1
  29. package/src/mcpx/client.ts +1 -1
  30. package/src/schedules/schema.ts +19 -0
  31. package/src/schedules/store.ts +296 -0
  32. package/src/skills/commands.ts +1 -3
  33. package/src/tasks/schema.ts +47 -0
  34. package/src/tasks/store.ts +486 -0
  35. package/src/threads/store.ts +559 -0
  36. package/src/tools/capabilities/refresh.ts +42 -21
  37. package/src/tools/context/pipe.ts +15 -71
  38. package/src/tools/context/update-beliefs.ts +3 -3
  39. package/src/tools/context/update-goals.ts +3 -3
  40. package/src/tools/dir/create.ts +26 -23
  41. package/src/tools/dir/size.ts +46 -17
  42. package/src/tools/dir/tree.ts +74 -279
  43. package/src/tools/file/copy.ts +50 -24
  44. package/src/tools/file/count-lines.ts +34 -10
  45. package/src/tools/file/delete.ts +53 -23
  46. package/src/tools/file/edit.ts +39 -14
  47. package/src/tools/file/exists.ts +12 -26
  48. package/src/tools/file/info.ts +27 -85
  49. package/src/tools/file/move.ts +39 -24
  50. package/src/tools/file/read.ts +32 -80
  51. package/src/tools/file/write.ts +14 -91
  52. package/src/tools/registry.ts +8 -7
  53. package/src/tools/schedule/create.ts +2 -2
  54. package/src/tools/schedule/list.ts +7 -3
  55. package/src/tools/search/fuse.ts +12 -33
  56. package/src/tools/search/index.ts +36 -43
  57. package/src/tools/search/regexp.ts +29 -17
  58. package/src/tools/search/semantic.ts +137 -51
  59. package/src/tools/skill/delete.ts +1 -1
  60. package/src/tools/skill/list.ts +1 -1
  61. package/src/tools/skill/write.ts +1 -1
  62. package/src/tools/task/create.ts +41 -16
  63. package/src/tools/task/delete.ts +3 -3
  64. package/src/tools/task/list.ts +6 -3
  65. package/src/tools/task/update.ts +31 -9
  66. package/src/tools/task/view.ts +6 -6
  67. package/src/tools/thread/list.ts +2 -2
  68. package/src/tools/thread/search.ts +208 -0
  69. package/src/tools/thread/view.ts +50 -5
  70. package/src/tools/tool.ts +5 -0
  71. package/src/tools/util/sleep.ts +77 -0
  72. package/src/tools/worker/spawn.ts +28 -14
  73. package/src/tui/App.tsx +12 -19
  74. package/src/tui/components/ContextPanel.tsx +83 -316
  75. package/src/tui/components/SchedulePanel.tsx +34 -48
  76. package/src/tui/components/SleepProgress.tsx +70 -0
  77. package/src/tui/components/StatusBar.tsx +15 -15
  78. package/src/tui/components/TaskPanel.tsx +34 -38
  79. package/src/tui/components/ThreadPanel.tsx +29 -38
  80. package/src/tui/components/ToolCall.tsx +10 -0
  81. package/src/tui/components/WorkerPanel.tsx +21 -19
  82. package/src/tui/markdown.ts +2 -8
  83. package/src/utils/title.ts +5 -7
  84. package/src/utils/v7-date.ts +47 -0
  85. package/src/worker/heartbeat.ts +46 -24
  86. package/src/worker/index.ts +13 -15
  87. package/src/worker/llm.ts +30 -37
  88. package/src/worker/prompt.ts +19 -41
  89. package/src/worker/schedules.ts +48 -69
  90. package/src/worker/spawn.ts +11 -11
  91. package/src/worker/tick.ts +39 -43
  92. package/src/workers/store.ts +247 -0
  93. package/src/commands/tools.ts +0 -367
  94. package/src/context/describer.ts +0 -140
  95. package/src/context/drives.ts +0 -110
  96. package/src/context/ingest.ts +0 -162
  97. package/src/context/refresh.ts +0 -183
  98. package/src/db/context.ts +0 -637
  99. package/src/db/daemon-state.ts +0 -6
  100. package/src/db/reembed.ts +0 -113
  101. package/src/db/schedules.ts +0 -213
  102. package/src/db/tasks.ts +0 -347
  103. package/src/db/threads.ts +0 -276
  104. package/src/db/workers.ts +0 -212
  105. package/src/tools/context/list-drives.ts +0 -36
  106. package/src/tools/context/refresh.ts +0 -165
  107. package/src/tools/context/search.ts +0 -54
package/README.md CHANGED
@@ -13,12 +13,14 @@ that works its way through a task queue — reading email, summarizing
13
13
  documents, researching topics, organizing notes, and maintaining context
14
14
  over time — while you sleep, work, or chat with it.
15
15
 
16
- Unlike coding agents, Botholomew has **no shell and no direct access to
17
- your filesystem**. It can't edit files on disk instead, it ingests local
18
- files, folders, and URLs into a DuckDB-backed context store that it can
19
- read, search, and summarize. External capabilities (email, Slack, the web,
20
- and hundreds of other services) are granted deliberately, per project,
21
- through MCP servers wired up via [MCPX](https://github.com/evantahler/mcpx).
16
+ Botholomew has **no shell and no access to your real filesystem**. The
17
+ agent's world is a sandboxed `context/` tree inside the project: it can
18
+ read, write, edit, and grep files there, but cannot escape upward,
19
+ follow symlinks, or touch anything outside. Local files and URLs are
20
+ brought in through `botholomew context add`. External capabilities
21
+ (email, Slack, the web, and hundreds of other services) are granted
22
+ deliberately, per project, through MCP servers wired up via
23
+ [MCPX](https://github.com/evantahler/mcpx).
22
24
 
23
25
  ---
24
26
 
@@ -27,13 +29,15 @@ through MCP servers wired up via [MCPX](https://github.com/evantahler/mcpx).
27
29
  - **Autonomous.** Background **workers** claim tasks, work them with Claude,
28
30
  and log every interaction. You can spawn one-shot workers on demand, a
29
31
  long-running `--persist` worker, or point cron at `botholomew worker run`.
30
- - **Portable.** Each project is a `.botholomew/` directory — markdown +
31
- DuckDB. Copy it, share it, check it in (or `.gitignore` it).
32
- - **Your data, your disk.** Project state tasks, threads, ingested
33
- context, embeddings lives in `.botholomew/`, indexed in DuckDB with
34
- BM25 keyword search and `array_cosine_distance` vector search. Model
35
- calls go direct to Anthropic and OpenAI; any further reach is scoped to
36
- the MCP servers you add.
32
+ - **Portable.** A project is just a directory of files — markdown for
33
+ prompts, tasks, schedules, and context; CSVs for conversation history.
34
+ Copy it, share it, `git diff` it, check it in (or `.gitignore` it).
35
+ - **Your data, your disk.** Tasks, schedules, threads, and the agent's
36
+ context tree are all real files you can `vim`, `grep`, and `git`.
37
+ DuckDB is demoted to a single search-index sidecar (`index.duckdb`)
38
+ that's fully derivable from disk and safe to delete. Model calls go
39
+ direct to Anthropic; any further reach is scoped to the MCP servers
40
+ you add.
37
41
  - **Extensible.** External tools come from MCP servers via
38
42
  [MCPX](https://github.com/evantahler/mcpx) — run them locally (Gmail,
39
43
  Slack, GitHub) or connect through an MCP gateway like
@@ -42,11 +46,12 @@ through MCP servers wired up via [MCPX](https://github.com/evantahler/mcpx).
42
46
  Reusable workflows are defined as markdown "skills" (slash commands)
43
47
  that the chat agent can also create, edit, and search at runtime.
44
48
  - **Safe by default.** The agent has no shell and no direct filesystem
45
- access. Out of the box, everything it can touch lives in `.botholomew/`;
46
- every external capability is a MCP server you explicitly add.
47
- - **Concurrent.** Many workers can run at once. Each registers itself in
48
- the DB and heartbeats; crashed workers get reaped and their tasks go
49
- back into the queue automatically.
49
+ access. Every path-taking tool is sandboxed to the project's `context/`
50
+ tree (NFC normalization + lstat-walk to reject symlinks at any level);
51
+ every external capability is an MCP server you explicitly add.
52
+ - **Concurrent.** Many workers can run at once. Each writes a pidfile
53
+ and heartbeats; tasks and schedules are claimed via `O_EXCL` lockfiles
54
+ and crashed workers get reaped automatically.
50
55
  - **Self-modifying.** The agent maintains its own `beliefs.md` and
51
56
  `goals.md` — it learns, updates its priors, and revises its goals as it
52
57
  works. It can also author its own slash-command skills mid-conversation,
@@ -88,7 +93,7 @@ bun run dev -- --help
88
93
  # 1. Initialize a project in the current directory
89
94
  botholomew init
90
95
 
91
- # 2. Add your Anthropic key to .botholomew/config.json, or export it
96
+ # 2. Add your Anthropic key to config/config.json, or export it
92
97
  export ANTHROPIC_API_KEY=sk-ant-...
93
98
  # Embeddings run locally — no API key required.
94
99
 
@@ -110,25 +115,40 @@ want Botholomew to advance on its own.
110
115
 
111
116
  ## What a project looks like
112
117
 
118
+ A project is the directory you ran `botholomew init` in. Every entity
119
+ the agent or worker touches is a real file you can `vim`, `grep`, and
120
+ `git diff`:
121
+
113
122
  ```
114
123
  my-project/
115
- .botholomew/
116
- soul.md # always-loaded identity (not agent-editable)
117
- beliefs.md # always-loaded, agent-editable priors
118
- goals.md # always-loaded, agent-editable goals
119
- capabilities.md # always-loaded, agent-editable tool inventory
120
- config.json # models, tick interval, API keys
121
- data.duckdb # tasks, schedules, context, embeddings, logs
122
- mcpx/servers.json # external MCP servers (Gmail, Slack, …)
123
- skills/ # slash commands (built-ins + user-defined)
124
- summarize.md
125
- standup.md
126
- capabilities.md
127
- logs/ # per-worker log files (one file per spawned worker)
128
- <worker-id>.log
124
+ config/config.json # models, tick interval, API keys
125
+ prompts/ # always-loaded markdown
126
+ soul.md # identity (not agent-editable)
127
+ beliefs.md # agent-editable priors
128
+ goals.md # agent-editable goals
129
+ capabilities.md # agent-editable tool inventory
130
+ skills/ # slash commands (built-ins + user-defined)
131
+ summarize.md
132
+ standup.md
133
+ capabilities.md
134
+ mcpx/servers.json # external MCP servers (Gmail, Slack, …)
135
+ models/ # local embedding model cache
136
+ context/ # agent-writable knowledge tree
137
+ tasks/ # one markdown file per task
138
+ <id>.md # status & metadata in frontmatter
139
+ .locks/<id>.lock # O_EXCL claim file (held by a worker)
140
+ schedules/ # one markdown file per schedule
141
+ <id>.md
142
+ .locks/<id>.lock
143
+ threads/<YYYY-MM-DD>/<id>.csv # full conversation history
144
+ workers/<id>.json # worker pidfile + heartbeat
145
+ logs/<YYYY-MM-DD>/<id>.log # per-worker logs
146
+ index.duckdb # search index sidecar (rebuildable; safe to delete)
129
147
  ```
130
148
 
131
- Everything the agent can touch is here. No surprises.
149
+ `index.duckdb` is the only opaque file; everything else is plain text.
150
+ Delete the index any time and `botholomew context reindex` rebuilds it
151
+ from `context/`.
132
152
 
133
153
  ---
134
154
 
@@ -138,19 +158,19 @@ Everything the agent can touch is here. No surprises.
138
158
 
139
159
  | Command | Purpose |
140
160
  |---|---|
141
- | `botholomew init` | Create `.botholomew/` with templates and a fresh database |
161
+ | `botholomew init` | Initialize the current directory as a project (refuses on iCloud/Dropbox/NFS without `--force`) |
142
162
  | `botholomew worker run\|start` | Run a worker (foreground or background); `--persist` for long-running, `--task-id <id>` to target one task |
143
163
  | `botholomew worker list\|status\|stop\|kill\|reap` | Inspect and manage running workers |
144
164
  | `botholomew chat` | Interactive Ink/React TUI |
145
- | `botholomew task list\|add\|view\|update\|reset\|delete` | Manage the task queue |
146
- | `botholomew schedule list\|add\|view\|enable\|disable\|trigger\|delete` | Recurring work |
147
- | `botholomew context add\|list\|search\|chunks\|refresh\|reembed\|delete` | Ingest & browse knowledge (files, folders, URLs); `reembed` rebuilds every vector after upgrading the embedding model; also exposes the agent's `read`/`write`/`tree`/`edit`/… tools as subcommands |
148
- | `botholomew capabilities` | Rescan built-in + MCPX tools and rewrite `.botholomew/capabilities.md` |
165
+ | `botholomew task list\|add\|view\|update\|reset\|delete` | Manage the task queue (markdown files in `tasks/`) |
166
+ | `botholomew schedule list\|add\|view\|enable\|disable\|trigger\|delete` | Recurring work (markdown files in `schedules/`) |
167
+ | `botholomew context add\|import\|tree\|stats\|reindex\|search\|read\|write\|edit\|move\|delete\|…` | Bring files/URLs into `context/`; rebuild the search index; expose the agent's file/dir tools as CLI subcommands |
168
+ | `botholomew capabilities` | Rescan built-in + MCPX tools and rewrite `prompts/capabilities.md` |
149
169
  | `botholomew mcpx servers\|list\|add\|remove\|info\|search\|exec\|ping\|auth\|deauth\|import-global\|…` | Configure external MCP servers (passthrough to `mcpx`) |
150
170
  | `botholomew skill list\|show\|create\|validate` | Manage slash-command skills |
151
- | `botholomew thread list\|view` | Browse the agent's interaction history |
152
- | `botholomew nuke context\|tasks\|schedules\|threads\|all` | Bulk-erase sections of the database |
153
- | `botholomew db doctor [--repair]` | Probe each table for primary-key index corruption; rebuild via EXPORT/IMPORT |
171
+ | `botholomew thread list\|view` | Browse the agent's conversation history (CSVs in `threads/`) |
172
+ | `botholomew nuke context\|tasks\|schedules\|threads\|all` | Bulk-erase project state |
173
+ | `botholomew db doctor [--repair]` | Probe the search-index DB; rebuild via EXPORT/IMPORT |
154
174
  | `botholomew upgrade` | Self-update |
155
175
 
156
176
  All `list` subcommands support `-l, --limit <n>` and `-o, --offset <n>` for pagination.
@@ -166,25 +186,25 @@ All `list` subcommands support `-l, --limit <n>` and `-o, --offset <n>` for pagi
166
186
  │ │ │ │ │ (optional)│
167
187
  └──────┬───────┘ └──────┬───────┘ └──────┬───────┘
168
188
  │ │ │
169
- │ enqueue tasks │ register + heartbeat │ fire
170
- │ browse history │ claim tasks │ `worker run`
189
+ │ enqueue tasks │ pidfile + heartbeat │ fire
190
+ │ browse history │ claim via O_EXCL lock │ `worker run`
171
191
  │ spawn_worker tool │ run LLM tool loops │ on a
172
- │ invoke skills │ reap dead peers │ schedule
173
- │ │ log to threads
192
+ │ invoke skills │ reap orphan locks │ schedule
193
+ │ │ log threads → CSV
174
194
  └────────────┬───────────┴────────────┬───────────┘
175
195
  │ │
176
- ┌─────▼────────────────────────▼─────┐
177
- DuckDB
178
- ┌───────────┐ ┌──────────────┐
179
- │ tasks │ │ context_items│ │
180
- schedules │ │ embeddings │ │
181
- workers │ (FTS+vector)│ │
182
- threads │ │ │
183
- └───────────┘ └──────────────┘
184
- └─────┬───────────────────────────────┘
185
-
186
-
187
- MCPX ─► Gmail, Slack, GitHub, Firecrawl, …
196
+ ┌──────▼────────────────────────▼──────┐
197
+ <project-root>/
198
+ tasks/<id>.md
199
+ │ schedules/<id>.md
200
+ threads/<date>/<id>.csv
201
+ workers/<id>.json
202
+ context/ ─► index.duckdb
203
+ (search sidecar)
204
+ └──────────────────┬────────────────────┘
205
+
206
+
207
+ MCPX ─► Gmail, Slack, GitHub, Firecrawl, …
188
208
  ```
189
209
 
190
210
  See [docs/architecture.md](docs/architecture.md) for a deeper tour.
@@ -205,17 +225,19 @@ Topics worth understanding in detail:
205
225
  - **[The TUI](docs/tui.md)** — the `botholomew chat` Ink/React terminal UI:
206
226
  eight tabs, slash-command autocomplete, message queue, tool-call
207
227
  visualization, and a live workers panel.
208
- - **[The virtual filesystem](docs/virtual-filesystem.md)** — why the agent's
209
- "files" are actually DuckDB rows, and how `context_read`/`context_write` work.
228
+ - **[Files & the sandbox](docs/files.md)** — the agent's `context/`
229
+ tree, the path sandbox (NFC + lstat-walk), and how
230
+ `context_read`/`context_write`/`context_edit` work.
210
231
  - **[Context & hybrid search](docs/context-and-search.md)** — LLM-driven
211
- chunking, OpenAI embeddings, and DuckDB BM25 + linear-scan vector
232
+ chunking, local embeddings, and DuckDB BM25 + linear-scan vector
212
233
  search merged with reciprocal rank fusion.
213
- - **[Tasks & schedules](docs/tasks-and-schedules.md)** — the claim loop, DAG
214
- validation, stale-task recovery, and natural-language recurring schedules.
234
+ - **[Tasks & schedules](docs/tasks-and-schedules.md)** — markdown
235
+ frontmatter as the source of truth, lockfile-based claim, DAG
236
+ validation, and natural-language recurring schedules.
215
237
  - **[The Tool class](docs/tools.md)** — one Zod definition, three consumers
216
238
  (Anthropic tool-use, Commander CLI, tests).
217
- - **[Persistent context](docs/persistent-context.md)** — `soul.md`,
218
- `beliefs.md`, `goals.md`, frontmatter flags, and agent self-modification.
239
+ - **[Prompts](docs/prompts.md)** — `soul.md`, `beliefs.md`, `goals.md`,
240
+ frontmatter flags, and agent self-modification.
219
241
  - **[Skills (slash commands)](docs/skills.md)** — reusable prompt templates
220
242
  with positional arguments and tab completion; the chat agent can also
221
243
  create, edit, and search them at runtime.
@@ -231,9 +253,10 @@ Topics worth understanding in detail:
231
253
  ## Tech stack
232
254
 
233
255
  - **[Bun](https://bun.sh)** + TypeScript
234
- - **[DuckDB](https://duckdb.org)** via `@duckdb/node-api` —
235
- `array_cosine_distance()` (core DuckDB) for vector search, plus the
236
- built-in FTS extension for BM25 keyword search
256
+ - **[DuckDB](https://duckdb.org)** via `@duckdb/node-api` — drives the
257
+ search-index sidecar only. `array_cosine_distance()` (core DuckDB) for
258
+ vector search, plus the built-in FTS extension for BM25 keyword
259
+ search; the index is rebuildable from `context/` at any time
237
260
  - **[Anthropic SDK](https://docs.anthropic.com/en/api/client-sdks)** for
238
261
  Claude — the reasoning model
239
262
  - **[`@huggingface/transformers`](https://huggingface.co/docs/transformers.js)**
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "botholomew",
3
- "version": "0.12.5",
3
+ "version": "0.14.0",
4
4
  "description": "An autonomous AI agent for knowledge work — works your task queue while you sleep.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -16,7 +16,7 @@
16
16
  "url": "https://github.com/evantahler/botholomew.git"
17
17
  },
18
18
  "scripts": {
19
- "dev": "bun run src/cli.ts",
19
+ "dev": "bun run src/cli.ts -d .botholomew",
20
20
  "dev:demo": "bun run src/cli.ts chat -p 'learn everything you can about me from the connected MCP services and then save what you'\\''ve learned about me to context'",
21
21
  "test": "bun test",
22
22
  "lint": "tsc --noEmit && biome check .",
package/src/chat/agent.ts CHANGED
@@ -8,10 +8,8 @@ import type {
8
8
  } from "@anthropic-ai/sdk/resources/messages";
9
9
  import type { McpxClient } from "@evantahler/mcpx";
10
10
  import type { BotholomewConfig } from "../config/schemas.ts";
11
- import { embedSingle } from "../context/embedder.ts";
12
11
  import { withDb } from "../db/connection.ts";
13
- import { hybridSearch } from "../db/embeddings.ts";
14
- import { logInteraction } from "../db/threads.ts";
12
+ import { logInteraction } from "../threads/store.ts";
15
13
  import { registerAllTools } from "../tools/registry.ts";
16
14
  import {
17
15
  getAllTools,
@@ -19,7 +17,6 @@ import {
19
17
  type ToolContext,
20
18
  toAnthropicTool,
21
19
  } from "../tools/tool.ts";
22
- import { logger } from "../utils/logger.ts";
23
20
  import { fitToContextWindow, getMaxInputTokens } from "../worker/context.ts";
24
21
  import { maybeStoreResult } from "../worker/large-results.ts";
25
22
  import { createLlmClient } from "../worker/llm-client.ts";
@@ -38,17 +35,15 @@ const CHAT_TOOL_NAMES = new Set([
38
35
  "create_task",
39
36
  "list_tasks",
40
37
  "view_task",
41
- "context_search",
42
38
  "context_info",
43
- "context_refresh",
44
39
  "context_tree",
45
- "context_list_drives",
46
40
  "context_read",
47
41
  "context_write",
48
42
  "context_edit",
49
43
  "search",
50
44
  "list_threads",
51
45
  "view_thread",
46
+ "search_threads",
52
47
  "create_schedule",
53
48
  "list_schedules",
54
49
  "update_beliefs",
@@ -67,6 +62,7 @@ const CHAT_TOOL_NAMES = new Set([
67
62
  "skill_edit",
68
63
  "skill_search",
69
64
  "skill_delete",
65
+ "sleep",
70
66
  ]);
71
67
 
72
68
  export function getChatTools() {
@@ -91,39 +87,14 @@ export async function buildChatSystemPrompt(
91
87
 
92
88
  prompt += await loadPersistentContext(projectDir, taskKeywords);
93
89
 
94
- const dbPath = options?.dbPath;
95
- const config = options?.config;
96
- if (dbPath && config && keywordSource) {
97
- try {
98
- const queryVec = await embedSingle(keywordSource, config);
99
- const results = await withDb(dbPath, (conn) =>
100
- hybridSearch(conn, keywordSource, queryVec, 5),
101
- );
102
-
103
- if (results.length > 0) {
104
- prompt += "## Relevant Context\n";
105
- for (const r of results) {
106
- const ref =
107
- r.drive && r.path ? `${r.drive}:${r.path}` : r.context_item_id;
108
- prompt += `### ${r.title} (${ref})\n`;
109
- if (r.chunk_content) {
110
- prompt += `${r.chunk_content.slice(0, 1000)}\n`;
111
- }
112
- prompt += "\n";
113
- }
114
- }
115
- } catch (err) {
116
- logger.debug(`Failed to load contextual embeddings: ${err}`);
117
- }
118
- }
119
-
120
90
  prompt += `## Instructions
121
91
  You are Botholomew, an AI agent personified by a wise owl. This is your interactive chat interface. Help the user manage tasks, review results from background worker activity, search context, and answer questions.
122
92
  You do NOT execute long-running work directly — enqueue tasks for a background worker instead using create_task, and spawn a worker via spawn_worker when the user wants the task run now.
123
- Use the available tools to look up tasks, threads, schedules, and context when the user asks about them. Context items live under a drive (disk / url / agent / google-docs / github /); use \`context_list_drives\` to discover which drives have content, then \`context_tree\`, \`context_info\`, \`context_search\`, or \`context_refresh\` as needed.
93
+ Use the available tools to look up tasks, threads, schedules, and context when the user asks about them. Files the agent can read and write live under \`context/\` as project-relative paths (e.g. \`notes/foo.md\`). Use \`context_tree\` to see what's there, \`search\` (hybrid regexp + semantic) to find content, then \`context_read\` / \`context_info\` to drill in.
94
+ Past conversations live in CSV files under \`threads/\`; use \`list_threads\`, \`search_threads\`, and \`view_thread\` to find and page through them.
124
95
  When multiple tool calls are independent of each other (i.e., one does not depend on the result of another), call them all in a single response. They will be executed in parallel, which is faster than calling them one at a time.
125
96
  You can update the agent's beliefs and goals files when the user asks you to.
126
- You can author and refine slash-command skills (reusable prompt templates stored in \`.botholomew/skills/\`) via \`skill_list\`, \`skill_search\`, \`skill_read\`, \`skill_write\`, \`skill_edit\`, and \`skill_delete\`. New or edited skills are usable as \`/<name>\` on the user's next message.
97
+ You can author and refine slash-command skills (reusable prompt templates stored in \`skills/\`) via \`skill_list\`, \`skill_search\`, \`skill_read\`, \`skill_write\`, \`skill_edit\`, and \`skill_delete\`. New or edited skills are usable as \`/<name>\` on the user's next message.
127
98
  Format your responses using Markdown. Use headings, bold, italic, lists, and code blocks to make your responses clear and well-structured.
128
99
  `;
129
100
 
@@ -133,19 +104,19 @@ Format your responses using Markdown. Use headings, bold, italic, lists, and cod
133
104
 
134
105
  ### Local context first
135
106
 
136
- **Before any MCP read, search local context.** Drive, Gmail, GitHub, URLs, and prior agent runs are usually already ingested — refetching is slower, costs tokens, and risks rate limits.
107
+ **Before any MCP read, search local context.** Files in \`context/\` (Gmail dumps, GitHub fetches, URL ingests, prior agent outputs) are usually already there — refetching is slower, costs tokens, and risks rate limits.
137
108
 
138
109
  Workflow for any "look up / find / read" intent:
139
110
 
140
- 1. \`search\` (hybrid regexp + semantic) or \`context_search\` (keyword), then \`context_read\` / \`context_tree\` to drill in.
141
- 2. If freshness matters, call \`context_info\` and check \`indexed_at\`. To re-pull a single stale item, use \`context_refresh\` rather than going to MCP for the whole document.
111
+ 1. \`search\` (hybrid regexp + semantic) over \`context/\`, then \`context_read\` / \`context_tree\` to drill in.
112
+ 2. If freshness matters, call \`context_info\` and check the file's mtime. To re-pull stale content, write fresh into \`context/\` (\`pipe_to_context\` from an \`mcp_exec\` call is the typical path) rather than going to MCP for the whole document on every question.
142
113
  3. Only call \`mcp_exec\` for reads when the data is genuinely missing locally **or** must be real-time (e.g., "what's on my calendar right now").
143
114
 
144
115
  Writes always go through MCP — sending an email, creating an issue, posting to Slack. Don't search context first for those.
145
116
 
146
117
  Examples:
147
118
  - "What does doc X say?" → \`search\` first.
148
- - "Any new emails from Y?" → check the \`gmail\` drive first; only hit Gmail MCP if the freshest indexed item is too old for the question.
119
+ - "Any new emails from Y?" → \`search\` for the sender under \`context/gmail/\` (or wherever you've been ingesting mail) before hitting Gmail MCP.
149
120
  - "Send an email to Y" → MCP write directly; no context lookup.
150
121
 
151
122
  ### Calling MCP tools
@@ -250,13 +221,11 @@ export async function runChatTurn(input: {
250
221
  // the whole tool loop to finish.
251
222
  const injections = callbacks.takeInjections?.() ?? [];
252
223
  for (const text of injections) {
253
- await withDb(dbPath, (conn) =>
254
- logInteraction(conn, threadId, {
255
- role: "user",
256
- kind: "message",
257
- content: text,
258
- }),
259
- );
224
+ await logInteraction(projectDir, threadId, {
225
+ role: "user",
226
+ kind: "message",
227
+ content: text,
228
+ });
260
229
  messages.push({ role: "user", content: text });
261
230
  }
262
231
 
@@ -327,15 +296,13 @@ export async function runChatTurn(input: {
327
296
  // `assistantText` is the right partial value). Deliberately drop any
328
297
  // partial tool_use blocks — they would be unmatched on the next turn.
329
298
  if (assistantText) {
330
- await withDb(dbPath, (conn) =>
331
- logInteraction(conn, threadId, {
332
- role: "assistant",
333
- kind: "message",
334
- content: assistantText,
335
- durationMs: Date.now() - startTime,
336
- tokenCount: 0,
337
- }),
338
- );
299
+ await logInteraction(projectDir, threadId, {
300
+ role: "assistant",
301
+ kind: "message",
302
+ content: assistantText,
303
+ durationMs: Date.now() - startTime,
304
+ tokenCount: 0,
305
+ });
339
306
  messages.push({ role: "assistant", content: assistantText });
340
307
  }
341
308
  return;
@@ -348,15 +315,13 @@ export async function runChatTurn(input: {
348
315
 
349
316
  // Log assistant text
350
317
  if (assistantText) {
351
- await withDb(dbPath, (conn) =>
352
- logInteraction(conn, threadId, {
353
- role: "assistant",
354
- kind: "message",
355
- content: assistantText,
356
- durationMs,
357
- tokenCount,
358
- }),
359
- );
318
+ await logInteraction(projectDir, threadId, {
319
+ role: "assistant",
320
+ kind: "message",
321
+ content: assistantText,
322
+ durationMs,
323
+ tokenCount,
324
+ });
360
325
  }
361
326
 
362
327
  // Check for tool calls
@@ -380,15 +345,13 @@ export async function runChatTurn(input: {
380
345
  callbacks.onToolStart(toolUse.id, toolUse.name, toolInput);
381
346
  }
382
347
 
383
- await withDb(dbPath, (conn) =>
384
- logInteraction(conn, threadId, {
385
- role: "assistant",
386
- kind: "tool_use",
387
- content: `Calling ${toolUse.name}`,
388
- toolName: toolUse.name,
389
- toolInput,
390
- }),
391
- );
348
+ await logInteraction(projectDir, threadId, {
349
+ role: "assistant",
350
+ kind: "tool_use",
351
+ content: `Calling ${toolUse.name}`,
352
+ toolName: toolUse.name,
353
+ toolInput,
354
+ });
392
355
  }
393
356
 
394
357
  // Execute all tools in parallel. Each tool call opens its own short-lived
@@ -402,6 +365,7 @@ export async function runChatTurn(input: {
402
365
  projectDir,
403
366
  config,
404
367
  mcpxClient,
368
+ shouldAbort: session ? () => session.aborted : undefined,
405
369
  });
406
370
  const durationMs = Date.now() - start;
407
371
  const stored = maybeStoreResult(toolUse.name, result.output);
@@ -422,15 +386,13 @@ export async function runChatTurn(input: {
422
386
  // Log results and collect tool_result messages
423
387
  const toolResults: ToolResultBlockParam[] = [];
424
388
  for (const { toolUse, result, durationMs, stored } of execResults) {
425
- await withDb(dbPath, (conn) =>
426
- logInteraction(conn, threadId, {
427
- role: "tool",
428
- kind: "tool_result",
429
- content: result.output,
430
- toolName: toolUse.name,
431
- durationMs,
432
- }),
433
- );
389
+ await logInteraction(projectDir, threadId, {
390
+ role: "tool",
391
+ kind: "tool_result",
392
+ content: result.output,
393
+ toolName: toolUse.name,
394
+ durationMs,
395
+ });
434
396
 
435
397
  toolResults.push({
436
398
  type: "tool_result",
@@ -451,6 +413,7 @@ interface ChatToolCallCtx {
451
413
  projectDir: string;
452
414
  config: Required<BotholomewConfig>;
453
415
  mcpxClient: McpxClient | null;
416
+ shouldAbort?: () => boolean;
454
417
  }
455
418
 
456
419
  async function executeChatToolCall(
@@ -474,10 +437,20 @@ async function executeChatToolCall(
474
437
  }
475
438
 
476
439
  try {
477
- const result = await withDb(baseCtx.dbPath, (conn) => {
478
- const ctx: ToolContext = { ...baseCtx, conn };
479
- return tool.execute(parsed.data, ctx);
480
- });
440
+ // `sleep` deliberately yields for up to an hour; opening a DuckDB
441
+ // connection for that whole window would hold the instance-level file
442
+ // lock and block any worker that also wants the DB. Run it without a
443
+ // connection — the tool doesn't touch the DB.
444
+ const runWithoutDb = tool.name === "sleep";
445
+ const result = runWithoutDb
446
+ ? await tool.execute(parsed.data, {
447
+ ...baseCtx,
448
+ conn: undefined as unknown as ToolContext["conn"],
449
+ })
450
+ : await withDb(baseCtx.dbPath, (conn) => {
451
+ const ctx: ToolContext = { ...baseCtx, conn };
452
+ return tool.execute(parsed.data, ctx);
453
+ });
481
454
  const isError =
482
455
  typeof result === "object" && result !== null && "is_error" in result
483
456
  ? (result as { is_error: boolean }).is_error