@orquentra/codebuddy 0.0.0-stage → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +269 -0
- package/CLAUDE_CODEX.md +433 -0
- package/LICENSE +21 -0
- package/README.md +1118 -2
- package/WORKFLOW.md +152 -0
- package/dist/chunk-3ZELDCR3.js +204 -0
- package/dist/chunk-3ZELDCR3.js.map +1 -0
- package/dist/chunk-5OWZVZSO.js +476 -0
- package/dist/chunk-5OWZVZSO.js.map +1 -0
- package/dist/chunk-DSN5CEHO.js +175 -0
- package/dist/chunk-DSN5CEHO.js.map +1 -0
- package/dist/chunk-GDJH7XJC.js +119 -0
- package/dist/chunk-GDJH7XJC.js.map +1 -0
- package/dist/chunk-IBMJYIEP.js +1578 -0
- package/dist/chunk-IBMJYIEP.js.map +1 -0
- package/dist/chunk-IE2OUO54.js +199 -0
- package/dist/chunk-IE2OUO54.js.map +1 -0
- package/dist/chunk-IHQM2ZDV.js +19 -0
- package/dist/chunk-IHQM2ZDV.js.map +1 -0
- package/dist/chunk-ITGRANYW.js +63 -0
- package/dist/chunk-ITGRANYW.js.map +1 -0
- package/dist/chunk-K63EOA4B.js +416 -0
- package/dist/chunk-K63EOA4B.js.map +1 -0
- package/dist/chunk-PQK3GKZH.js +98 -0
- package/dist/chunk-PQK3GKZH.js.map +1 -0
- package/dist/chunk-PYLH4MH5.js +34 -0
- package/dist/chunk-PYLH4MH5.js.map +1 -0
- package/dist/chunk-PZ5AY32C.js +10 -0
- package/dist/chunk-PZ5AY32C.js.map +1 -0
- package/dist/chunk-SYHB3AMM.js +8580 -0
- package/dist/chunk-SYHB3AMM.js.map +1 -0
- package/dist/chunk-TI5TQD6D.js +324 -0
- package/dist/chunk-TI5TQD6D.js.map +1 -0
- package/dist/chunk-WHZSVY2L.js +144 -0
- package/dist/chunk-WHZSVY2L.js.map +1 -0
- package/dist/cli/index.d.ts +1 -0
- package/dist/cli/index.js +15 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/codebuddy-D2xJ6hFk.d.ts +841 -0
- package/dist/config-S2TULOMK.js +11 -0
- package/dist/config-S2TULOMK.js.map +1 -0
- package/dist/index.d.ts +2527 -0
- package/dist/index.js +116 -0
- package/dist/index.js.map +1 -0
- package/dist/langgraph/index.d.ts +53 -0
- package/dist/langgraph/index.js +143 -0
- package/dist/langgraph/index.js.map +1 -0
- package/dist/lean-command-D6TZJEFX.js +13 -0
- package/dist/lean-command-D6TZJEFX.js.map +1 -0
- package/dist/main-ZZ3EQLYA.js +4575 -0
- package/dist/main-ZZ3EQLYA.js.map +1 -0
- package/dist/memory-NIBFAMY4.js +57 -0
- package/dist/memory-NIBFAMY4.js.map +1 -0
- package/dist/pretool-command-MQQ2UNQM.js +10 -0
- package/dist/pretool-command-MQQ2UNQM.js.map +1 -0
- package/dist/rerank-Q7MIFNKA.js +60 -0
- package/dist/rerank-Q7MIFNKA.js.map +1 -0
- package/dist/summaries-TSE44KUQ.js +18 -0
- package/dist/summaries-TSE44KUQ.js.map +1 -0
- package/dist/zvec-F27AR357.js +23 -0
- package/dist/zvec-F27AR357.js.map +1 -0
- package/docker-compose.yml +20 -0
- package/examples/claude-code/README.md +67 -0
- package/examples/claude-desktop/README.md +54 -0
- package/examples/claude-desktop/claude_desktop_config.json +13 -0
- package/examples/langgraph-agent/README.md +32 -0
- package/examples/langgraph-agent/index.ts +71 -0
- package/examples/multi-agent-handoff/README.md +35 -0
- package/examples/multi-agent-handoff/index.ts +68 -0
- package/package.json +108 -4
- package/src/db/migrations/0000_cultured_satana.sql +144 -0
- package/src/db/migrations/0001_embedding_lease.sql +3 -0
- package/src/db/migrations/0002_embedding_lease_token.sql +1 -0
- package/src/db/migrations/0003_embedding_space.sql +9 -0
- package/src/db/migrations/0004_markdown_reconcile.sql +5 -0
- package/src/db/migrations/0005_fact_propositions.sql +1 -0
- package/src/db/migrations/meta/0000_snapshot.json +1169 -0
- package/src/db/migrations/meta/0002_snapshot.json +1181 -0
- package/src/db/migrations/meta/0003_snapshot.json +1214 -0
- package/src/db/migrations/meta/0004_snapshot.json +1225 -0
- package/src/db/migrations/meta/0005_snapshot.json +1231 -0
- package/src/db/migrations/meta/_journal.json +48 -0
package/WORKFLOW.md
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
# CodeBuddy Workflow
|
|
2
|
+
|
|
3
|
+
This is the simple mental model for how CodeBuddy works.
|
|
4
|
+
|
|
5
|
+
## Layman View
|
|
6
|
+
|
|
7
|
+
CodeBuddy is a shared notebook for AI agents.
|
|
8
|
+
|
|
9
|
+
- Agents write memories into the notebook.
|
|
10
|
+
- CodeBuddy writes curated facts and summaries to Markdown and Postgres.
|
|
11
|
+
- Raw interactions and operational data remain in Postgres.
|
|
12
|
+
- Embeddings are prepared in the background so related memories can be found later.
|
|
13
|
+
- When an agent asks a question, CodeBuddy picks the most useful memories that fit inside the token budget.
|
|
14
|
+
- Agents can share facts with other agents through namespaces.
|
|
15
|
+
- The CLI helps you see whether the database, Hugging Face models, and memory index are healthy.
|
|
16
|
+
|
|
17
|
+
## Graphical Flow
|
|
18
|
+
|
|
19
|
+
```text
|
|
20
|
+
User or Agent
|
|
21
|
+
|
|
|
22
|
+
v
|
|
23
|
+
SDK / MCP Tool / CLI
|
|
24
|
+
|
|
|
25
|
+
v
|
|
26
|
+
CodeBuddy Core
|
|
27
|
+
|
|
|
28
|
+
+--> Markdown file store (facts + summaries)
|
|
29
|
+
| |
|
|
30
|
+
| +--> versioned YAML front matter
|
|
31
|
+
| +--> atomic write + fsync + rename
|
|
32
|
+
|
|
|
33
|
+
+--> Postgres transaction/index
|
|
34
|
+
| |
|
|
35
|
+
| +--> write interaction/fact/summary
|
|
36
|
+
| +--> create embedding row as pending
|
|
37
|
+
| +--> write audit record when needed
|
|
38
|
+
|
|
|
39
|
+
+--> In-process embedding worker
|
|
40
|
+
| |
|
|
41
|
+
| +--> Hugging Face resilience layer
|
|
42
|
+
| | |
|
|
43
|
+
| | +--> retry/backoff
|
|
44
|
+
| | +--> cold-start wait
|
|
45
|
+
| | +--> rate-limit queue
|
|
46
|
+
| | +--> fallback model
|
|
47
|
+
| |
|
|
48
|
+
| +--> update embedding status to ready or failed
|
|
49
|
+
|
|
|
50
|
+
v
|
|
51
|
+
Recall request
|
|
52
|
+
|
|
|
53
|
+
+--> recent session memory
|
|
54
|
+
+--> ready vector matches
|
|
55
|
+
+--> available summaries
|
|
56
|
+
+--> shared facts
|
|
57
|
+
|
|
|
58
|
+
v
|
|
59
|
+
Planner packs best items under token budget
|
|
60
|
+
|
|
|
61
|
+
v
|
|
62
|
+
{ system, messages, stats }
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
## Remember Flow In Detail
|
|
66
|
+
|
|
67
|
+
1. A caller sends `remember({ sessionId?, content, type?, idempotencyKey?, agentId? })`.
|
|
68
|
+
2. If `sessionId` is missing, CodeBuddy creates `sess_<ulid>`.
|
|
69
|
+
3. CodeBuddy computes a content hash if no idempotency key is provided.
|
|
70
|
+
4. It takes a Postgres advisory lock for `(namespace, sessionId)`.
|
|
71
|
+
5. In one transaction, it writes the memory row and creates an embedding row with `status=pending`.
|
|
72
|
+
6. For facts and summaries, it also writes a Markdown file.
|
|
73
|
+
7. If the same content or idempotency key already exists, it returns the original ID with `deduplicated: true`.
|
|
74
|
+
8. After the transaction, the in-process worker asks Hugging Face for embeddings.
|
|
75
|
+
9. The embedding row becomes `ready` or `failed`.
|
|
76
|
+
|
|
77
|
+
## Migration And Recovery Flow
|
|
78
|
+
|
|
79
|
+
```text
|
|
80
|
+
existing PostgreSQL memories
|
|
81
|
+
|
|
|
82
|
+
v
|
|
83
|
+
codebuddy migrate to-files (preview only)
|
|
84
|
+
|
|
|
85
|
+
v
|
|
86
|
+
codebuddy migrate to-files --write
|
|
87
|
+
|
|
|
88
|
+
v
|
|
89
|
+
.codebuddy/memory/**/*.md
|
|
90
|
+
|
|
|
91
|
+
v
|
|
92
|
+
codebuddy reindex --full (after DB recreation)
|
|
93
|
+
|
|
|
94
|
+
v
|
|
95
|
+
restored fact + summary rows and pending embeddings
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Migration never overwrites same-ID files. Content differences are reported as
|
|
99
|
+
checksum conflicts for manual review.
|
|
100
|
+
|
|
101
|
+
## Recall Flow In Detail
|
|
102
|
+
|
|
103
|
+
1. A caller sends `recall({ sessionId, query, budget?, callerModel?, conflictMode? })`.
|
|
104
|
+
2. If `callerModel` is present, CodeBuddy checks the budget against known model context windows.
|
|
105
|
+
3. The planner always includes a recency floor from the current session.
|
|
106
|
+
4. It searches ready embeddings for related facts and interactions.
|
|
107
|
+
5. If embeddings are still pending, it falls back to recency and available summaries.
|
|
108
|
+
6. It applies conflict behavior (`all`, `latest`, or `highest_confidence`) to facts sharing an explicit `propositionKey`, flagging unresolved ties in fact provenance.
|
|
109
|
+
7. It greedily packs the best items until the token budget is reached.
|
|
110
|
+
8. It returns prompt-ready context plus stats explaining what was included and skipped.
|
|
111
|
+
|
|
112
|
+
## Sharing Flow In Detail
|
|
113
|
+
|
|
114
|
+
```text
|
|
115
|
+
reference mode:
|
|
116
|
+
research-agent fact A ---> ops-agent reads fact A
|
|
117
|
+
source update ---> target sees update
|
|
118
|
+
source delete ---> target reference is tombstoned
|
|
119
|
+
|
|
120
|
+
snapshot mode:
|
|
121
|
+
research-agent fact A ---> copied as ops-agent fact B
|
|
122
|
+
source update ---> B does not change
|
|
123
|
+
source delete ---> B remains
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
## Doctor Flow
|
|
127
|
+
|
|
128
|
+
`codebuddy doctor` checks the things most likely to make the system feel broken:
|
|
129
|
+
|
|
130
|
+
- database connection
|
|
131
|
+
- `pgvector` extension
|
|
132
|
+
- vector index health
|
|
133
|
+
- Hugging Face model reachability
|
|
134
|
+
- gated model errors
|
|
135
|
+
- cold vs warm model state
|
|
136
|
+
- rate-limit pressure
|
|
137
|
+
- calls in the last 60 seconds, hour, and 24 hours
|
|
138
|
+
- daily budget estimate
|
|
139
|
+
- config file permissions
|
|
140
|
+
|
|
141
|
+
## Why Async Embeddings Matter
|
|
142
|
+
|
|
143
|
+
Hugging Face free-tier models can be cold or rate-limited. CodeBuddy should not block a memory write just because a model needs time to wake up. It stores the memory first, marks the embedding as pending, and lets recall use recency until vectors are ready.
|
|
144
|
+
|
|
145
|
+
## What The User Sees
|
|
146
|
+
|
|
147
|
+
- Writes return quickly and safely.
|
|
148
|
+
- Repeated writes do not duplicate memory.
|
|
149
|
+
- Recall works even if embeddings are still warming up.
|
|
150
|
+
- `stats` explains memory selection.
|
|
151
|
+
- `doctor` explains operational problems.
|
|
152
|
+
- Agent handoffs are explicit through namespaces and shares.
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
import {
|
|
2
|
+
NON_SOURCE_PATH,
|
|
3
|
+
TEST_PATH,
|
|
4
|
+
loadFreshIndex,
|
|
5
|
+
loadProjectArchitectureMap,
|
|
6
|
+
readBoundedSource
|
|
7
|
+
} from "./chunk-IBMJYIEP.js";
|
|
8
|
+
import {
|
|
9
|
+
chatJson
|
|
10
|
+
} from "./chunk-IE2OUO54.js";
|
|
11
|
+
import {
|
|
12
|
+
redactSensitive,
|
|
13
|
+
scanSensitive
|
|
14
|
+
} from "./chunk-GDJH7XJC.js";
|
|
15
|
+
import {
|
|
16
|
+
resolveLlmConfig
|
|
17
|
+
} from "./chunk-ITGRANYW.js";
|
|
18
|
+
import {
|
|
19
|
+
writeAtomic
|
|
20
|
+
} from "./chunk-K63EOA4B.js";
|
|
21
|
+
|
|
22
|
+
// src/llm/summaries.ts
|
|
23
|
+
import { mkdir, readFile } from "fs/promises";
|
|
24
|
+
import { join } from "path";
|
|
25
|
+
import { z } from "zod";
|
|
26
|
+
var SUMMARY_PROMPT_VERSION = 1;
|
|
27
|
+
var LIMITS = {
|
|
28
|
+
fileChars: 160,
|
|
29
|
+
symbolChars: 100,
|
|
30
|
+
signatures: 12,
|
|
31
|
+
headLines: 20,
|
|
32
|
+
/** ~600 tokens per file: long-line (minified) files are cut, never sent whole. */
|
|
33
|
+
headChars: 1600,
|
|
34
|
+
signatureChars: 160,
|
|
35
|
+
batchTokens: 5e3,
|
|
36
|
+
batchFiles: 8,
|
|
37
|
+
outputTokensPerFile: 120,
|
|
38
|
+
tokensPerMinute: 6e3,
|
|
39
|
+
timeoutMs: 2e4
|
|
40
|
+
};
|
|
41
|
+
var storePath = (root) => join(root, ".codebuddy", "cache", "llm-summaries.json");
|
|
42
|
+
async function readStore(root) {
|
|
43
|
+
try {
|
|
44
|
+
return JSON.parse(await readFile(storePath(root), "utf8"));
|
|
45
|
+
} catch {
|
|
46
|
+
return {};
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
async function writeStore(root, store) {
|
|
50
|
+
await mkdir(join(root, ".codebuddy", "cache"), { recursive: true });
|
|
51
|
+
await writeAtomic(storePath(root), `${JSON.stringify(store)}
|
|
52
|
+
`);
|
|
53
|
+
}
|
|
54
|
+
async function loadLlmSummaries(root, manifest) {
|
|
55
|
+
const store = await readStore(root);
|
|
56
|
+
return new Map(
|
|
57
|
+
Object.entries(store).filter(([path, entry]) => manifest.entries[path]?.hash === entry.hash)
|
|
58
|
+
);
|
|
59
|
+
}
|
|
60
|
+
var outputSchema = z.object({
|
|
61
|
+
files: z.array(
|
|
62
|
+
z.object({
|
|
63
|
+
path: z.string(),
|
|
64
|
+
summary: z.string(),
|
|
65
|
+
symbols: z.array(z.object({ name: z.string(), summary: z.string() })).default([])
|
|
66
|
+
})
|
|
67
|
+
)
|
|
68
|
+
});
|
|
69
|
+
var SYSTEM = [
|
|
70
|
+
"You describe source files for a code search index.",
|
|
71
|
+
'Answer with JSON only: {"files":[{"path":"...","summary":"...","symbols":[{"name":"...","summary":"..."}]}]}.',
|
|
72
|
+
`For each input file: one sentence (max ${LIMITS.fileChars} chars) saying what it does, using the words a developer would search for;`,
|
|
73
|
+
`for its main symbols (only names listed in "signatures"), one short phrase each (max ${LIMITS.symbolChars} chars).`,
|
|
74
|
+
"Never invent paths or symbol names."
|
|
75
|
+
].join(" ");
|
|
76
|
+
async function summarizeRepository(root, options = {}) {
|
|
77
|
+
const resolved = options.config ?? await resolveLlmConfig(root);
|
|
78
|
+
if (!resolved.enabled || !resolved.features.summaries) {
|
|
79
|
+
return { summarized: 0, skipped: 0, remaining: 0, stoppedBy: "disabled" };
|
|
80
|
+
}
|
|
81
|
+
const config = resolved;
|
|
82
|
+
const { manifest } = await loadFreshIndex(root);
|
|
83
|
+
const store = await readStore(root);
|
|
84
|
+
for (const path of Object.keys(store)) if (!manifest.entries[path]) delete store[path];
|
|
85
|
+
const pending = await orderByImportance(
|
|
86
|
+
root,
|
|
87
|
+
manifest,
|
|
88
|
+
Object.values(manifest.entries).filter((entry) => {
|
|
89
|
+
const done = store[entry.path];
|
|
90
|
+
return !(done?.hash === entry.hash && done.model === config.model && done.version === SUMMARY_PROMPT_VERSION);
|
|
91
|
+
})
|
|
92
|
+
);
|
|
93
|
+
const limit = options.limit ?? Number.POSITIVE_INFINITY;
|
|
94
|
+
let summarized = 0;
|
|
95
|
+
let skipped = 0;
|
|
96
|
+
let index = 0;
|
|
97
|
+
let stoppedBy = "done";
|
|
98
|
+
while (index < pending.length) {
|
|
99
|
+
if (options.signal?.aborted) {
|
|
100
|
+
stoppedBy = "aborted";
|
|
101
|
+
break;
|
|
102
|
+
}
|
|
103
|
+
if (summarized >= limit) {
|
|
104
|
+
stoppedBy = "limit";
|
|
105
|
+
break;
|
|
106
|
+
}
|
|
107
|
+
const batch = [];
|
|
108
|
+
let tokens = 0;
|
|
109
|
+
while (index < pending.length && batch.length < (options.batchFiles ?? LIMITS.batchFiles)) {
|
|
110
|
+
if (summarized + batch.length >= limit) break;
|
|
111
|
+
const entry = pending[index];
|
|
112
|
+
const input = await fileInput(root, entry);
|
|
113
|
+
if (input === null) {
|
|
114
|
+
skipped++;
|
|
115
|
+
index++;
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
const cost = Math.ceil(JSON.stringify(input).length / 4);
|
|
119
|
+
if (batch.length > 0 && tokens + cost > LIMITS.batchTokens - 600) break;
|
|
120
|
+
batch.push({ entry, input });
|
|
121
|
+
tokens += cost;
|
|
122
|
+
index++;
|
|
123
|
+
}
|
|
124
|
+
if (batch.length === 0) continue;
|
|
125
|
+
const started = Date.now();
|
|
126
|
+
const result = await chatJson(config, {
|
|
127
|
+
system: SYSTEM,
|
|
128
|
+
user: JSON.stringify({ files: batch.map(({ input }) => input) }),
|
|
129
|
+
schema: outputSchema,
|
|
130
|
+
maxTokens: LIMITS.outputTokensPerFile * batch.length,
|
|
131
|
+
deadlineMs: LIMITS.timeoutMs,
|
|
132
|
+
purpose: "background",
|
|
133
|
+
repositoryRoot: root
|
|
134
|
+
});
|
|
135
|
+
if (!result.ok) {
|
|
136
|
+
if (result.reason === "invalid" || result.reason === "http" && batch.length === 1) {
|
|
137
|
+
skipped += batch.length;
|
|
138
|
+
continue;
|
|
139
|
+
}
|
|
140
|
+
index -= batch.length;
|
|
141
|
+
stoppedBy = result.reason === "budget" ? "budget" : result.reason === "cooldown" || result.reason === "rate-limited" ? "cooldown" : "error";
|
|
142
|
+
break;
|
|
143
|
+
}
|
|
144
|
+
for (const { entry } of batch) {
|
|
145
|
+
const answer = result.value.files.find((file) => file.path === entry.path);
|
|
146
|
+
if (!answer) {
|
|
147
|
+
skipped++;
|
|
148
|
+
continue;
|
|
149
|
+
}
|
|
150
|
+
const known = new Set((entry.symbolTable ?? []).map((symbol) => symbol.name));
|
|
151
|
+
store[entry.path] = {
|
|
152
|
+
hash: entry.hash,
|
|
153
|
+
model: config.model,
|
|
154
|
+
version: SUMMARY_PROMPT_VERSION,
|
|
155
|
+
summary: answer.summary.trim().slice(0, LIMITS.fileChars),
|
|
156
|
+
symbols: Object.fromEntries(
|
|
157
|
+
answer.symbols.filter((symbol) => known.has(symbol.name)).map((symbol) => [symbol.name, symbol.summary.trim().slice(0, LIMITS.symbolChars)])
|
|
158
|
+
)
|
|
159
|
+
};
|
|
160
|
+
summarized++;
|
|
161
|
+
}
|
|
162
|
+
await writeStore(root, store);
|
|
163
|
+
options.onProgress?.(index, pending.length);
|
|
164
|
+
if (options.pace !== false) {
|
|
165
|
+
const wait = result.tokens / LIMITS.tokensPerMinute * 6e4 - (Date.now() - started);
|
|
166
|
+
if (wait > 0) await new Promise((resolve) => setTimeout(resolve, wait));
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
await writeStore(root, store);
|
|
170
|
+
return { summarized, skipped, remaining: pending.length - index, stoppedBy };
|
|
171
|
+
}
|
|
172
|
+
async function fileInput(root, entry) {
|
|
173
|
+
const source = await readBoundedSource(root, entry.path);
|
|
174
|
+
if (source === null || scanSensitive(source).sensitive) return null;
|
|
175
|
+
return {
|
|
176
|
+
path: entry.path,
|
|
177
|
+
language: entry.language,
|
|
178
|
+
summary: redactSensitive(entry.summary),
|
|
179
|
+
signatures: (entry.symbolTable ?? []).slice(0, LIMITS.signatures).map((symbol) => redactSensitive(symbol.signature.slice(0, LIMITS.signatureChars))),
|
|
180
|
+
head: redactSensitive(
|
|
181
|
+
source.split(/\r?\n/).slice(0, LIMITS.headLines).join("\n").slice(0, LIMITS.headChars)
|
|
182
|
+
)
|
|
183
|
+
};
|
|
184
|
+
}
|
|
185
|
+
async function orderByImportance(root, manifest, entries) {
|
|
186
|
+
const fanIn = /* @__PURE__ */ new Map();
|
|
187
|
+
try {
|
|
188
|
+
for (const edge of (await loadProjectArchitectureMap(root, manifest)).edges) {
|
|
189
|
+
fanIn.set(edge.to, (fanIn.get(edge.to) ?? 0) + 1);
|
|
190
|
+
}
|
|
191
|
+
} catch {
|
|
192
|
+
}
|
|
193
|
+
const tier = (path) => TEST_PATH.test(path) || NON_SOURCE_PATH.test(path) ? 1 : 0;
|
|
194
|
+
return [...entries].sort(
|
|
195
|
+
(left, right) => tier(left.path) - tier(right.path) || (fanIn.get(right.path) ?? 0) - (fanIn.get(left.path) ?? 0) || left.path.localeCompare(right.path)
|
|
196
|
+
);
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
export {
|
|
200
|
+
SUMMARY_PROMPT_VERSION,
|
|
201
|
+
loadLlmSummaries,
|
|
202
|
+
summarizeRepository
|
|
203
|
+
};
|
|
204
|
+
//# sourceMappingURL=chunk-3ZELDCR3.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/llm/summaries.ts"],"sourcesContent":["import { mkdir, readFile } from \"node:fs/promises\";\nimport { join } from \"node:path\";\nimport { z } from \"zod\";\nimport {\n loadProjectArchitectureMap,\n NON_SOURCE_PATH,\n readBoundedSource,\n TEST_PATH,\n} from \"../context/discovery.js\";\nimport { writeAtomic } from \"../core/markdown-store-fs.js\";\nimport { loadFreshIndex } from \"../indexer/engine.js\";\nimport type { IndexEntry, IndexManifest } from \"../indexer/types.js\";\nimport { redactSensitive, scanSensitive } from \"../memory-extract/sensitive.js\";\nimport { chatJson } from \"./client.js\";\nimport { type EnabledLlmConfig, resolveLlmConfig } from \"./config.js\";\n\n/**\n * One-line descriptions of files and their main symbols, written by the BYOK\n * model off the prompt path (`codebuddy llm summarize`, `watch`) and cached by\n * content hash. Retrieval reads them so a paraphrased task can match a file\n * whose own text never uses the task's words. Generated text is a hint for\n * ordering only; it never satisfies the evidence floor on its own.\n */\nexport const SUMMARY_PROMPT_VERSION = 1;\nconst LIMITS = {\n fileChars: 160,\n symbolChars: 100,\n signatures: 12,\n headLines: 20,\n /** ~600 tokens per file: long-line (minified) files are cut, never sent whole. */\n headChars: 1_600,\n signatureChars: 160,\n batchTokens: 5_000,\n batchFiles: 8,\n outputTokensPerFile: 120,\n tokensPerMinute: 6_000,\n timeoutMs: 20_000,\n} as const;\n\nexport type LlmSummary = {\n hash: string;\n model: string;\n version: number;\n summary: string;\n symbols: Record<string, string>;\n};\n\ntype Store = Record<string, LlmSummary>;\nconst storePath = (root: string) => join(root, \".codebuddy\", \"cache\", \"llm-summaries.json\");\n\nasync function readStore(root: string): Promise<Store> {\n try {\n return JSON.parse(await readFile(storePath(root), \"utf8\")) as Store;\n } catch {\n return {};\n }\n}\n\nasync function writeStore(root: string, store: Store): Promise<void> {\n await mkdir(join(root, \".codebuddy\", \"cache\"), { recursive: true });\n await writeAtomic(storePath(root), `${JSON.stringify(store)}\\n`);\n}\n\n/** Summaries whose content hash still matches the current index. */\nexport async function loadLlmSummaries(\n root: string,\n manifest: IndexManifest,\n): Promise<Map<string, LlmSummary>> {\n const store = await readStore(root);\n return new Map(\n Object.entries(store).filter(([path, entry]) => manifest.entries[path]?.hash === entry.hash),\n );\n}\n\nconst outputSchema = z.object({\n files: z.array(\n z.object({\n path: z.string(),\n summary: z.string(),\n symbols: z.array(z.object({ name: z.string(), summary: z.string() })).default([]),\n }),\n ),\n});\n\nconst SYSTEM = [\n \"You describe source files for a code search index.\",\n 'Answer with JSON only: {\"files\":[{\"path\":\"...\",\"summary\":\"...\",\"symbols\":[{\"name\":\"...\",\"summary\":\"...\"}]}]}.',\n `For each input file: one sentence (max ${LIMITS.fileChars} chars) saying what it does, using the words a developer would search for;`,\n `for its main symbols (only names listed in \"signatures\"), one short phrase each (max ${LIMITS.symbolChars} chars).`,\n \"Never invent paths or symbol names.\",\n].join(\" \");\n\nexport type SummarizeResult = {\n summarized: number;\n skipped: number;\n remaining: number;\n stoppedBy: \"done\" | \"limit\" | \"budget\" | \"cooldown\" | \"disabled\" | \"aborted\" | \"error\";\n};\n\nexport async function summarizeRepository(\n root: string,\n options: {\n limit?: number;\n onProgress?: (done: number, total: number) => void;\n signal?: AbortSignal;\n /** Injected config (tests, CLI); defaults to `resolveLlmConfig`. */\n config?: EnabledLlmConfig;\n /** Wait between batches to stay under the free tier's tokens/minute. */\n pace?: boolean;\n batchFiles?: number;\n } = {},\n): Promise<SummarizeResult> {\n const resolved = options.config ?? (await resolveLlmConfig(root));\n if (!resolved.enabled || !resolved.features.summaries) {\n return { summarized: 0, skipped: 0, remaining: 0, stoppedBy: \"disabled\" };\n }\n const config = resolved;\n const { manifest } = await loadFreshIndex(root);\n const store = await readStore(root);\n for (const path of Object.keys(store)) if (!manifest.entries[path]) delete store[path];\n\n const pending = await orderByImportance(\n root,\n manifest,\n Object.values(manifest.entries).filter((entry) => {\n const done = store[entry.path];\n return !(\n done?.hash === entry.hash &&\n done.model === config.model &&\n done.version === SUMMARY_PROMPT_VERSION\n );\n }),\n );\n const limit = options.limit ?? Number.POSITIVE_INFINITY;\n let summarized = 0;\n let skipped = 0;\n let index = 0;\n let stoppedBy: SummarizeResult[\"stoppedBy\"] = \"done\";\n\n while (index < pending.length) {\n if (options.signal?.aborted) {\n stoppedBy = \"aborted\";\n break;\n }\n if (summarized >= limit) {\n stoppedBy = \"limit\";\n break;\n }\n const batch: Array<{ entry: IndexEntry; input: FileInput }> = [];\n let tokens = 0;\n while (index < pending.length && batch.length < (options.batchFiles ?? LIMITS.batchFiles)) {\n if (summarized + batch.length >= limit) break;\n const entry = pending[index] as IndexEntry;\n const input = await fileInput(root, entry);\n if (input === null) {\n skipped++;\n index++;\n continue;\n }\n const cost = Math.ceil(JSON.stringify(input).length / 4);\n if (batch.length > 0 && tokens + cost > LIMITS.batchTokens - 600) break;\n batch.push({ entry, input });\n tokens += cost;\n index++;\n }\n if (batch.length === 0) continue;\n\n const started = Date.now();\n const result = await chatJson(config, {\n system: SYSTEM,\n user: JSON.stringify({ files: batch.map(({ input }) => input) }),\n schema: outputSchema,\n maxTokens: LIMITS.outputTokensPerFile * batch.length,\n deadlineMs: LIMITS.timeoutMs,\n purpose: \"background\",\n repositoryRoot: root,\n });\n if (!result.ok) {\n // A batch the provider rejects outright (bad output, or e.g. 413 for one\n // file) is skipped, not retried forever; limits and outages stop the run.\n if (result.reason === \"invalid\" || (result.reason === \"http\" && batch.length === 1)) {\n skipped += batch.length;\n continue;\n }\n index -= batch.length;\n stoppedBy =\n result.reason === \"budget\"\n ? \"budget\"\n : result.reason === \"cooldown\" || result.reason === \"rate-limited\"\n ? \"cooldown\"\n : \"error\";\n break;\n }\n for (const { entry } of batch) {\n const answer = result.value.files.find((file) => file.path === entry.path);\n if (!answer) {\n skipped++;\n continue;\n }\n const known = new Set((entry.symbolTable ?? []).map((symbol) => symbol.name));\n store[entry.path] = {\n hash: entry.hash,\n model: config.model,\n version: SUMMARY_PROMPT_VERSION,\n summary: answer.summary.trim().slice(0, LIMITS.fileChars),\n symbols: Object.fromEntries(\n answer.symbols\n .filter((symbol) => known.has(symbol.name))\n .map((symbol) => [symbol.name, symbol.summary.trim().slice(0, LIMITS.symbolChars)]),\n ),\n };\n summarized++;\n }\n await writeStore(root, store);\n options.onProgress?.(index, pending.length);\n if (options.pace !== false) {\n const wait = (result.tokens / LIMITS.tokensPerMinute) * 60_000 - (Date.now() - started);\n if (wait > 0) await new Promise((resolve) => setTimeout(resolve, wait));\n }\n }\n await writeStore(root, store);\n return { summarized, skipped, remaining: pending.length - index, stoppedBy };\n}\n\ntype FileInput = {\n path: string;\n language: string;\n summary: string;\n signatures: string[];\n head: string;\n};\n\n/** What is sent for one file; null for unreadable or secret-bearing files. */\nasync function fileInput(root: string, entry: IndexEntry): Promise<FileInput | null> {\n const source = await readBoundedSource(root, entry.path);\n if (source === null || scanSensitive(source).sensitive) return null;\n return {\n path: entry.path,\n language: entry.language,\n summary: redactSensitive(entry.summary),\n signatures: (entry.symbolTable ?? [])\n .slice(0, LIMITS.signatures)\n .map((symbol) => redactSensitive(symbol.signature.slice(0, LIMITS.signatureChars))),\n head: redactSensitive(\n source.split(/\\r?\\n/).slice(0, LIMITS.headLines).join(\"\\n\").slice(0, LIMITS.headChars),\n ),\n };\n}\n\n/** Most-imported source first; tests and non-source files last. */\nasync function orderByImportance(\n root: string,\n manifest: IndexManifest,\n entries: IndexEntry[],\n): Promise<IndexEntry[]> {\n const fanIn = new Map<string, number>();\n try {\n for (const edge of (await loadProjectArchitectureMap(root, manifest)).edges) {\n fanIn.set(edge.to, (fanIn.get(edge.to) ?? 0) + 1);\n }\n } catch {\n // No map: path order is still a usable order.\n }\n const tier = (path: string) => (TEST_PATH.test(path) || NON_SOURCE_PATH.test(path) ? 1 : 0);\n return [...entries].sort(\n (left, right) =>\n tier(left.path) - tier(right.path) ||\n (fanIn.get(right.path) ?? 0) - (fanIn.get(left.path) ?? 0) ||\n left.path.localeCompare(right.path),\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAAA,SAAS,OAAO,gBAAgB;AAChC,SAAS,YAAY;AACrB,SAAS,SAAS;AAqBX,IAAM,yBAAyB;AACtC,IAAM,SAAS;AAAA,EACb,WAAW;AAAA,EACX,aAAa;AAAA,EACb,YAAY;AAAA,EACZ,WAAW;AAAA;AAAA,EAEX,WAAW;AAAA,EACX,gBAAgB;AAAA,EAChB,aAAa;AAAA,EACb,YAAY;AAAA,EACZ,qBAAqB;AAAA,EACrB,iBAAiB;AAAA,EACjB,WAAW;AACb;AAWA,IAAM,YAAY,CAAC,SAAiB,KAAK,MAAM,cAAc,SAAS,oBAAoB;AAE1F,eAAe,UAAU,MAA8B;AACrD,MAAI;AACF,WAAO,KAAK,MAAM,MAAM,SAAS,UAAU,IAAI,GAAG,MAAM,CAAC;AAAA,EAC3D,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACF;AAEA,eAAe,WAAW,MAAc,OAA6B;AACnE,QAAM,MAAM,KAAK,MAAM,cAAc,OAAO,GAAG,EAAE,WAAW,KAAK,CAAC;AAClE,QAAM,YAAY,UAAU,IAAI,GAAG,GAAG,KAAK,UAAU,KAAK,CAAC;AAAA,CAAI;AACjE;AAGA,eAAsB,iBACpB,MACA,UACkC;AAClC,QAAM,QAAQ,MAAM,UAAU,IAAI;AAClC,SAAO,IAAI;AAAA,IACT,OAAO,QAAQ,KAAK,EAAE,OAAO,CAAC,CAAC,MAAM,KAAK,MAAM,SAAS,QAAQ,IAAI,GAAG,SAAS,MAAM,IAAI;AAAA,EAC7F;AACF;AAEA,IAAM,eAAe,EAAE,OAAO;AAAA,EAC5B,OAAO,EAAE;AAAA,IACP,EAAE,OAAO;AAAA,MACP,MAAM,EAAE,OAAO;AAAA,MACf,SAAS,EAAE,OAAO;AAAA,MAClB,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,GAAG,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;AAAA,IAClF,CAAC;AAAA,EACH;AACF,CAAC;AAED,IAAM,SAAS;AAAA,EACb;AAAA,EACA;AAAA,EACA,0CAA0C,OAAO,SAAS;AAAA,EAC1D,wFAAwF,OAAO,WAAW;AAAA,EAC1G;AACF,EAAE,KAAK,GAAG;AASV,eAAsB,oBACpB,MACA,UASI,CAAC,GACqB;AAC1B,QAAM,WAAW,QAAQ,UAAW,MAAM,iBAAiB,IAAI;AAC/D,MAAI,CAAC,SAAS,WAAW,CAAC,SAAS,SAAS,WAAW;AACrD,WAAO,EAAE,YAAY,GAAG,SAAS,GAAG,WAAW,GAAG,WAAW,WAAW;AAAA,EAC1E;AACA,QAAM,SAAS;AACf,QAAM,EAAE,SAAS,IAAI,MAAM,eAAe,IAAI;AAC9C,QAAM,QAAQ,MAAM,UAAU,IAAI;AAClC,aAAW,QAAQ,OAAO,KAAK,KAAK,EAAG,KAAI,CAAC,SAAS,QAAQ,IAAI,EAAG,QAAO,MAAM,IAAI;AAErF,QAAM,UAAU,MAAM;AAAA,IACpB;AAAA,IACA;AAAA,IACA,OAAO,OAAO,SAAS,OAAO,EAAE,OAAO,CAAC,UAAU;AAChD,YAAM,OAAO,MAAM,MAAM,IAAI;AAC7B,aAAO,EACL,MAAM,SAAS,MAAM,QACrB,KAAK,UAAU,OAAO,SACtB,KAAK,YAAY;AAAA,IAErB,CAAC;AAAA,EACH;AACA,QAAM,QAAQ,QAAQ,SAAS,OAAO;AACtC,MAAI,aAAa;AACjB,MAAI,UAAU;AACd,MAAI,QAAQ;AACZ,MAAI,YAA0C;AAE9C,SAAO,QAAQ,QAAQ,QAAQ;AAC7B,QAAI,QAAQ,QAAQ,SAAS;AAC3B,kBAAY;AACZ;AAAA,IACF;AACA,QAAI,cAAc,OAAO;AACvB,kBAAY;AACZ;AAAA,IACF;AACA,UAAM,QAAwD,CAAC;AAC/D,QAAI,SAAS;AACb,WAAO,QAAQ,QAAQ,UAAU,MAAM,UAAU,QAAQ,cAAc,OAAO,aAAa;AACzF,UAAI,aAAa,MAAM,UAAU,MAAO;AACxC,YAAM,QAAQ,QAAQ,KAAK;AAC3B,YAAM,QAAQ,MAAM,UAAU,MAAM,KAAK;AACzC,UAAI,UAAU,MAAM;AAClB;AACA;AACA;AAAA,MACF;AACA,YAAM,OAAO,KAAK,KAAK,KAAK,UAAU,KAAK,EAAE,SAAS,CAAC;AACvD,UAAI,MAAM,SAAS,KAAK,SAAS,OAAO,OAAO,cAAc,IAAK;AAClE,YAAM,KAAK,EAAE,OAAO,MAAM,CAAC;AAC3B,gBAAU;AACV;AAAA,IACF;AACA,QAAI,MAAM,WAAW,EAAG;AAExB,UAAM,UAAU,KAAK,IAAI;AACzB,UAAM,SAAS,MAAM,SAAS,QAAQ;AAAA,MACpC,QAAQ;AAAA,MACR,MAAM,KAAK,UAAU,EAAE,OAAO,MAAM,IAAI,CAAC,EAAE,MAAM,MAAM,KAAK,EAAE,CAAC;AAAA,MAC/D,QAAQ;AAAA,MACR,WAAW,OAAO,sBAAsB,MAAM;AAAA,MAC9C,YAAY,OAAO;AAAA,MACnB,SAAS;AAAA,MACT,gBAAgB;AAAA,IAClB,CAAC;AACD,QAAI,CAAC,OAAO,IAAI;AAGd,UAAI,OAAO,WAAW,aAAc,OAAO,WAAW,UAAU,MAAM,WAAW,GAAI;AACnF,mBAAW,MAAM;AACjB;AAAA,MACF;AACA,eAAS,MAAM;AACf,kBACE,OAAO,WAAW,WACd,WACA,OAAO,WAAW,cAAc,OAAO,WAAW,iBAChD,aACA;AACR;AAAA,IACF;AACA,eAAW,EAAE,MAAM,KAAK,OAAO;AAC7B,YAAM,SAAS,OAAO,MAAM,MAAM,KAAK,CAAC,SAAS,KAAK,SAAS,MAAM,IAAI;AACzE,UAAI,CAAC,QAAQ;AACX;AACA;AAAA,MACF;AACA,YAAM,QAAQ,IAAI,KAAK,MAAM,eAAe,CAAC,GAAG,IAAI,CAAC,WAAW,OAAO,IAAI,CAAC;AAC5E,YAAM,MAAM,IAAI,IAAI;AAAA,QAClB,MAAM,MAAM;AAAA,QACZ,OAAO,OAAO;AAAA,QACd,SAAS;AAAA,QACT,SAAS,OAAO,QAAQ,KAAK,EAAE,MAAM,GAAG,OAAO,SAAS;AAAA,QACxD,SAAS,OAAO;AAAA,UACd,OAAO,QACJ,OAAO,CAAC,WAAW,MAAM,IAAI,OAAO,IAAI,CAAC,EACzC,IAAI,CAAC,WAAW,CAAC,OAAO,MAAM,OAAO,QAAQ,KAAK,EAAE,MAAM,GAAG,OAAO,WAAW,CAAC,CAAC;AAAA,QACtF;AAAA,MACF;AACA;AAAA,IACF;AACA,UAAM,WAAW,MAAM,KAAK;AAC5B,YAAQ,aAAa,OAAO,QAAQ,MAAM;AAC1C,QAAI,QAAQ,SAAS,OAAO;AAC1B,YAAM,OAAQ,OAAO,SAAS,OAAO,kBAAmB,OAAU,KAAK,IAAI,IAAI;AAC/E,UAAI,OAAO,EAAG,OAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,IAAI,CAAC;AAAA,IACxE;AAAA,EACF;AACA,QAAM,WAAW,MAAM,KAAK;AAC5B,SAAO,EAAE,YAAY,SAAS,WAAW,QAAQ,SAAS,OAAO,UAAU;AAC7E;AAWA,eAAe,UAAU,MAAc,OAA8C;AACnF,QAAM,SAAS,MAAM,kBAAkB,MAAM,MAAM,IAAI;AACvD,MAAI,WAAW,QAAQ,cAAc,MAAM,EAAE,UAAW,QAAO;AAC/D,SAAO;AAAA,IACL,MAAM,MAAM;AAAA,IACZ,UAAU,MAAM;AAAA,IAChB,SAAS,gBAAgB,MAAM,OAAO;AAAA,IACtC,aAAa,MAAM,eAAe,CAAC,GAChC,MAAM,GAAG,OAAO,UAAU,EAC1B,IAAI,CAAC,WAAW,gBAAgB,OAAO,UAAU,MAAM,GAAG,OAAO,cAAc,CAAC,CAAC;AAAA,IACpF,MAAM;AAAA,MACJ,OAAO,MAAM,OAAO,EAAE,MAAM,GAAG,OAAO,SAAS,EAAE,KAAK,IAAI,EAAE,MAAM,GAAG,OAAO,SAAS;AAAA,IACvF;AAAA,EACF;AACF;AAGA,eAAe,kBACb,MACA,UACA,SACuB;AACvB,QAAM,QAAQ,oBAAI,IAAoB;AACtC,MAAI;AACF,eAAW,SAAS,MAAM,2BAA2B,MAAM,QAAQ,GAAG,OAAO;AAC3E,YAAM,IAAI,KAAK,KAAK,MAAM,IAAI,KAAK,EAAE,KAAK,KAAK,CAAC;AAAA,IAClD;AAAA,EACF,QAAQ;AAAA,EAER;AACA,QAAM,OAAO,CAAC,SAAkB,UAAU,KAAK,IAAI,KAAK,gBAAgB,KAAK,IAAI,IAAI,IAAI;AACzF,SAAO,CAAC,GAAG,OAAO,EAAE;AAAA,IAClB,CAAC,MAAM,UACL,KAAK,KAAK,IAAI,IAAI,KAAK,MAAM,IAAI,MAChC,MAAM,IAAI,MAAM,IAAI,KAAK,MAAM,MAAM,IAAI,KAAK,IAAI,KAAK,MACxD,KAAK,KAAK,cAAc,MAAM,IAAI;AAAA,EACtC;AACF;","names":[]}
|