@thincoder/core 0.9.5 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -0
- package/README.md +1 -1
- package/abort-provenance.mjs +10 -10
- package/advisor/citations.mjs +5 -5
- package/advisor/compaction.mjs +16 -15
- package/advisor/loop.mjs +12 -12
- package/advisor/messages.mjs +6 -6
- package/advisor/project-context.mjs +5 -5
- package/advisor/repos.mjs +3 -3
- package/advisor/run.mjs +22 -12
- package/advisor.mjs +4 -4
- package/agent/assemble.mjs +111 -0
- package/agent/chat-call.mjs +68 -0
- package/agent/child-marks.mjs +2 -2
- package/agent/completion.mjs +9 -2
- package/agent/continue-decision.mjs +25 -0
- package/agent/dispatch-gates.mjs +140 -0
- package/agent/dispatch-run.mjs +167 -0
- package/agent/dispatch.mjs +54 -296
- package/agent/family-tools.mjs +5 -5
- package/agent/helpers.mjs +53 -3
- package/agent/live-beat.mjs +49 -0
- package/agent/post-turn.mjs +4 -8
- package/agent/record-results.mjs +23 -20
- package/agent/relay-prefix.mjs +2 -2
- package/agent/run-stages.mjs +15 -11
- package/agent/run-start.mjs +111 -0
- package/agent/setup-reminders.mjs +9 -9
- package/agent/setup.mjs +32 -4
- package/agent/spawn-child.mjs +9 -9
- package/agent/suspension.mjs +89 -29
- package/agent/timers.mjs +113 -0
- package/agent/turn-loop.mjs +244 -0
- package/agent/write-gate.mjs +103 -6
- package/agent-tools/advisor-async.mjs +9 -9
- package/agent-tools/advisor-settle.mjs +12 -12
- package/agent-tools/advisor.mjs +36 -41
- package/agent-tools/async-discard.mjs +44 -1
- package/agent-tools/async-settle.mjs +21 -18
- package/agent-tools/audit-block.mjs +7 -7
- package/agent-tools/bash-async.mjs +281 -0
- package/agent-tools/batch-lifecycle.mjs +57 -18
- package/agent-tools/batch-paths.mjs +52 -8
- package/agent-tools/batch-skeleton.mjs +19 -1
- package/agent-tools/batch.mjs +12 -65
- package/agent-tools/checkpoint.mjs +121 -0
- package/agent-tools/child-permission.mjs +5 -3
- package/agent-tools/consult.mjs +56 -56
- package/agent-tools/context.mjs +4 -20
- package/agent-tools/design-token.mjs +3 -3
- package/agent-tools/digest-budget.mjs +1 -1
- package/agent-tools/eng.mjs +5 -5
- package/agent-tools/escalate-async.mjs +16 -11
- package/agent-tools/goal.mjs +3 -10
- package/agent-tools/panel-blocks.mjs +2 -2
- package/agent-tools/parent-channel.mjs +3 -13
- package/agent-tools/plan.mjs +3 -4
- package/agent-tools/read-history.mjs +12 -28
- package/agent-tools/recent-changes.mjs +4 -5
- package/agent-tools/review-facts.mjs +1 -1
- package/agent-tools/settings.mjs +47 -20
- package/agent-tools/skill.mjs +3 -5
- package/agent-tools/spawn-gates.mjs +3 -3
- package/agent-tools/subagent-actions-query.mjs +270 -0
- package/agent-tools/subagent-actions.mjs +21 -265
- package/agent-tools/subagent-async.mjs +25 -20
- package/agent-tools/subagent-panel.mjs +33 -4
- package/agent-tools/subagent-run.mjs +36 -21
- package/agent-tools/subagent-scheduler.mjs +28 -27
- package/agent-tools/subagent-spawn.mjs +45 -14
- package/agent-tools/subagent.mjs +33 -53
- package/agent-tools/task.mjs +3 -9
- package/agent-tools/timer.mjs +12 -12
- package/agent-tools/verify.mjs +14 -13
- package/agent-tools.mjs +0 -1
- package/agent.mjs +18 -355
- package/attachments.mjs +125 -0
- package/auto-think.mjs +1 -1
- package/config-presets.mjs +11 -6
- package/config-watch.mjs +83 -0
- package/config.mjs +17 -4
- package/context-degrade.mjs +81 -0
- package/context-echo.mjs +88 -0
- package/context-push.mjs +51 -0
- package/context.mjs +8 -187
- package/conventions.mjs +110 -161
- package/declaration.mjs +194 -0
- package/embedding.mjs +3 -1
- package/env-ref.mjs +81 -0
- package/explore-distill.mjs +2 -3
- package/file-links.mjs +53 -0
- package/file-refs.mjs +103 -0
- package/generate-title.mjs +28 -4
- package/history-window.mjs +17 -2
- package/i18n.mjs +2 -0
- package/index-discover.mjs +7 -7
- package/ledger-cmd.mjs +34 -126
- package/ledger-db.mjs +10 -2
- package/ledger-tools.mjs +188 -0
- package/ledger.mjs +9 -4
- package/log.mjs +12 -7
- package/manifest-discovery.mjs +131 -0
- package/manifest-schema.mjs +166 -0
- package/manifest.mjs +18 -217
- package/mcp.mjs +4 -1
- package/memory/code-search.mjs +149 -0
- package/memory/code-sync.mjs +89 -230
- package/memory/core.mjs +47 -32
- package/memory/docs.mjs +63 -262
- package/memory/file-list.mjs +89 -0
- package/memory/file-walk.mjs +7 -4
- package/memory/memory-tool.mjs +229 -0
- package/memory/schema.mjs +18 -3
- package/memory/sweep.mjs +289 -0
- package/memory/sync-tail.mjs +81 -0
- package/memory-status.mjs +69 -0
- package/memory.mjs +3 -0
- package/model-specs.mjs +16 -7
- package/notify-policy.mjs +51 -0
- package/package.json +1 -1
- package/peer-claims.mjs +2 -2
- package/peer-domains.mjs +7 -11
- package/peer-instances.mjs +9 -10
- package/process-probe-exec.mjs +187 -0
- package/process-probe.mjs +27 -175
- package/prompt-overlays.mjs +11 -10
- package/prompts/advisor-design.md +2 -2
- package/prompts/advisor-round2.md +1 -1
- package/prompts/advisor-round3.md +1 -1
- package/prompts/common.md +16 -8
- package/prompts/discipline-engineering.md +57 -16
- package/prompts/discipline-normal.md +6 -3
- package/prompts/persona-coder.md +0 -1
- package/prompts/persona-eng-coder.md +1 -5
- package/prompts/persona-eng-designer.md +2 -3
- package/prompts/persona-engineering.md +32 -11
- package/prompts/persona-explore.md +0 -1
- package/prompts/persona-normal.md +1 -1
- package/prompts/persona-plan.md +3 -4
- package/prompts/subagent-base.md +14 -0
- package/provider/core.mjs +11 -48
- package/provider/list-models.mjs +3 -0
- package/provider/responses-request.mjs +237 -0
- package/provider/responses.mjs +6 -228
- package/provider-flows.mjs +248 -0
- package/queued.mjs +90 -0
- package/rules.mjs +209 -2
- package/session-guard.mjs +27 -7
- package/session-lifecycle.mjs +90 -18
- package/session-rename.mjs +5 -1
- package/session-slot-claims.mjs +230 -0
- package/session-slot-scan.mjs +299 -0
- package/session-slot-verify.mjs +304 -0
- package/session-slot-write.mjs +74 -8
- package/session-slots-manifest.mjs +163 -241
- package/session-slots.mjs +99 -71
- package/session.mjs +30 -10
- package/shell-candidates.mjs +78 -0
- package/text-budget.mjs +2 -2
- package/think-off.mjs +49 -0
- package/token-ttl.mjs +15 -4
- package/tool-args.mjs +73 -0
- package/tool-docs/advisor.md +9 -0
- package/tool-docs/batch.md +1 -0
- package/tool-docs/code_search.md +1 -0
- package/tool-docs/consult_start.md +5 -0
- package/tool-docs/consult_stop.md +3 -0
- package/tool-docs/context.md +7 -0
- package/tool-docs/doc_search.md +1 -0
- package/tool-docs/edit.md +2 -13
- package/tool-docs/eng.md +1 -0
- package/tool-docs/execute.md +1 -9
- package/tool-docs/git.md +25 -51
- package/tool-docs/goal.md +6 -0
- package/tool-docs/ledger_add.md +1 -0
- package/tool-docs/ledger_close.md +1 -0
- package/tool-docs/ledger_count.md +1 -0
- package/tool-docs/ledger_query.md +1 -0
- package/tool-docs/ledger_update.md +1 -0
- package/tool-docs/main_history.md +1 -0
- package/tool-docs/memory.md +10 -0
- package/tool-docs/notify_parent.md +9 -0
- package/tool-docs/peer_instances.md +1 -0
- package/tool-docs/plan.md +5 -0
- package/tool-docs/process.md +7 -3
- package/tool-docs/read_history.md +7 -0
- package/tool-docs/recent_changes.md +1 -0
- package/tool-docs/repo_outline.md +1 -0
- package/tool-docs/settings.md +6 -0
- package/tool-docs/skill.md +5 -0
- package/tool-docs/subagent.md +31 -0
- package/tool-docs/task.md +7 -0
- package/tool-docs/timer.md +6 -0
- package/tool-docs/verify.md +1 -0
- package/tool-docs/wait_for.md +2 -1
- package/tools/bash.mjs +28 -4
- package/tools/edit-batch.mjs +4 -4
- package/tools/edit-diff.mjs +4 -23
- package/tools/exec-run.mjs +102 -4
- package/tools/execute.mjs +5 -2
- package/tools/file.mjs +4 -9
- package/tools/git-run.mjs +1 -1
- package/tools/git.mjs +0 -24
- package/tools/index.mjs +2 -1
- package/tools/linter.mjs +2 -1
- package/tools/node-child.mjs +21 -0
- package/tools/ops.mjs +50 -13
- package/tools/patch.mjs +4 -2
- package/tools/repomap.mjs +20 -9
- package/tools/shared.mjs +8 -6
- package/tools/web.mjs +5 -2
- package/tools/write-path.mjs +1 -1
- package/traces/trace-store.mjs +17 -14
- package/vision-reader.mjs +76 -0
- package/agent-tools/batch-segment.mjs +0 -18
package/memory/code-sync.mjs
CHANGED
|
@@ -1,44 +1,58 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* memory/code-sync.mjs — code index sync, retrieval
|
|
2
|
+
* memory/code-sync.mjs — code index sync, incremental update (retrieval / tool generation → code-search.mjs; 2026-10-01 split; §6.15 声明根集展开 2026-10-03)
|
|
3
3
|
*/
|
|
4
|
+
import { existsSync } from "node:fs"
|
|
4
5
|
import { readFile, stat } from "node:fs/promises"
|
|
5
6
|
import { join, relative } from "node:path"
|
|
6
|
-
import {
|
|
7
|
-
import { scanVectors, createTopK } from "./scan.mjs"
|
|
7
|
+
import { SCAN_YIELD_MS } from "./scan.mjs"
|
|
8
8
|
import { normalizeOrigin } from "./origin.mjs"
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
9
|
+
import { MAX_CODE_FILE_BYTES, MAX_DOC_FILE_BYTES } from "./schema.mjs"
|
|
10
|
+
import { ensureEmbeddings, invalidateStaleEmbeddings } from "./core.mjs"
|
|
11
11
|
import { detectLanguage, _upsertCodeFile, _upsertDocFile, yieldTick } from "./code-index.mjs"
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
12
|
+
import { isSkippedRelPath, extensionOf } from "./file-walk.mjs"
|
|
13
|
+
import { createRowBudget, rowCountOfPath, sweepStaleRows } from "./sync-tail.mjs"
|
|
14
|
+
import { loadProjectDeclaration, isExcludedRelPath, declaredPublicRoots } from "../conventions.mjs"
|
|
14
15
|
import { logEvent } from "../log.mjs"
|
|
15
|
-
import {
|
|
16
|
+
import { indexExtensions, listProjectFiles } from "./file-list.mjs"
|
|
16
17
|
|
|
17
18
|
const DIFF_FULL_SYNC_THRESHOLD = 200
|
|
18
|
-
const CODE_EMBED_BATCH = 64
|
|
19
19
|
|
|
20
|
-
/**
|
|
21
|
-
*
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
const
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
20
|
+
/** per-origin 锚键(§6.14 M1):`last_indexed_commit:<归一键>`——多 origin 互不覆盖;
|
|
21
|
+
* 旧单键在而本键缺 ⇒ 视为无锚(该 origin 全扫一次);旧键清退 = 全部 origin 落锚后(ops)。 */
|
|
22
|
+
const anchorKey = (origin) => `last_indexed_commit:${origin}`
|
|
23
|
+
|
|
24
|
+
/** 声明公共仓根集(§6.15——入口 dir 之外;缺位整根跳过 + 一行 logEvent;归一去重保序)。 */
|
|
25
|
+
export function declaredRootEntries(dir) {
|
|
26
|
+
const seen = new Set([normalizeOrigin(dir)])
|
|
27
|
+
const out = []
|
|
28
|
+
for (const abs of declaredPublicRoots(dir)) {
|
|
29
|
+
const o = normalizeOrigin(abs)
|
|
30
|
+
if (seen.has(o)) continue
|
|
31
|
+
seen.add(o)
|
|
32
|
+
if (!existsSync(abs)) { logEvent("index:root-missing", { dir, origin: o }); continue }
|
|
33
|
+
out.push(abs)
|
|
31
34
|
}
|
|
35
|
+
return out
|
|
32
36
|
}
|
|
33
37
|
|
|
34
|
-
/**
|
|
35
|
-
* git
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
38
|
+
/** §6.15 同步展开(git 入口):入口 dir + 声明公共仓逐根(一层不递归);回执 = 入口根(契约零改)。
|
|
39
|
+
* 单根内胆 = `gitSyncDir`(git diff 增量——原 `gitSync` 体,零语义改动);null = git 不可得。 */
|
|
40
|
+
export async function gitSync(memory, dir, opts = {}) {
|
|
41
|
+
const res = await gitSyncDir(memory, dir, opts)
|
|
42
|
+
if (res === null) return null // 入口 dir null 契约保持——公共仓随调用方回退(codeSync ∥ docSync)展开覆盖
|
|
43
|
+
for (const root of declaredRootEntries(dir)) {
|
|
44
|
+
if ((await gitSyncDir(memory, root, opts)) !== null) continue // 增量可行 ⇒ 增量(per-origin 锚既有)
|
|
45
|
+
await codeSyncDir(memory, root, opts) // 增量不可行 ⇒ 全扫(同入口款)
|
|
46
|
+
const { syncDocRoot } = await import("./docs.mjs")
|
|
47
|
+
await syncDocRoot(memory, root, opts)
|
|
48
|
+
}
|
|
49
|
+
return res
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** 单根内胆(git 增量径):原 `gitSync` 体——零语义改动。 */
|
|
53
|
+
async function gitSyncDir(memory, dir, { onProgress } = {}) {
|
|
41
54
|
const origin = normalizeOrigin(dir) // §6.11 写缝归一(git / 文件 I/O 用原样 dir;库面 origin 一律归一值)
|
|
55
|
+
const decl = loadProjectDeclaration(dir)
|
|
42
56
|
const { execFile: _execFile } = await import("node:child_process")
|
|
43
57
|
const { code: codeExts, doc: docExts } = indexExtensions(dir)
|
|
44
58
|
const gitRun = (args) => new Promise((resolve, reject) => {
|
|
@@ -51,13 +65,15 @@ export async function gitSync(memory, dir, { onProgress } = {}) {
|
|
|
51
65
|
let head
|
|
52
66
|
try { head = (await gitRun(["rev-parse", "HEAD"])).trim() } catch { return null }
|
|
53
67
|
|
|
54
|
-
const stored = memory.db.prepare(`SELECT value FROM meta WHERE key =
|
|
68
|
+
const stored = memory.db.prepare(`SELECT value FROM meta WHERE key = ?`).get(anchorKey(origin))?.value
|
|
55
69
|
if (!stored) return null
|
|
56
70
|
|
|
57
71
|
let diffOut
|
|
58
72
|
try {
|
|
59
|
-
|
|
60
|
-
|
|
73
|
+
// §6.14 M2:`--relative`——diff 表收在 cwd 子树内且路径为 cwd 相对(与索引 path ∕ `codeSync` 同形;
|
|
74
|
+
// 无它则子目录 origin 下 `join(dir, rel)` 错位:真改动入删除支 ∕ 错内容入库 = 静默陈旧)。
|
|
75
|
+
const committed = mergeDiff(await gitRun(["diff", "--name-only", "--relative", "--diff-filter=ACMRTD", stored, "HEAD"]))
|
|
76
|
+
const dirty = mergeDiff(await gitRun(["diff", "--name-only", "--relative", "--diff-filter=ACMRTD"]))
|
|
61
77
|
const lines = [...new Set([...committed, ...dirty])]
|
|
62
78
|
diffOut = lines
|
|
63
79
|
} catch {
|
|
@@ -66,9 +82,9 @@ export async function gitSync(memory, dir, { onProgress } = {}) {
|
|
|
66
82
|
|
|
67
83
|
if (diffOut.length > DIFF_FULL_SYNC_THRESHOLD) {
|
|
68
84
|
// diff too large, incremental is useless — fall back to full sync and update anchor
|
|
69
|
-
await
|
|
70
|
-
const {
|
|
71
|
-
await
|
|
85
|
+
await codeSyncDir(memory, dir, { onProgress })
|
|
86
|
+
const { syncDocRoot } = await import("./docs.mjs")
|
|
87
|
+
await syncDocRoot(memory, dir, { onProgress })
|
|
72
88
|
return { updated: -1, removed: 0, skipped: 0, failed: 0, errors: [], fallback: true }
|
|
73
89
|
}
|
|
74
90
|
|
|
@@ -79,7 +95,7 @@ export async function gitSync(memory, dir, { onProgress } = {}) {
|
|
|
79
95
|
const abs = join(dir, rel)
|
|
80
96
|
const ext = extensionOf(rel)
|
|
81
97
|
|
|
82
|
-
if (isSkippedRelPath(rel)) continue
|
|
98
|
+
if (isSkippedRelPath(rel) || isExcludedRelPath(rel, decl, dir)) continue
|
|
83
99
|
|
|
84
100
|
if (!codeExts.has(ext) && !docExts.has(ext)) { skipped++; continue }
|
|
85
101
|
|
|
@@ -115,8 +131,8 @@ export async function gitSync(memory, dir, { onProgress } = {}) {
|
|
|
115
131
|
}
|
|
116
132
|
|
|
117
133
|
if (failed === 0) {
|
|
118
|
-
memory.db.prepare(`INSERT INTO meta (key, value) VALUES (
|
|
119
|
-
ON CONFLICT (key) DO UPDATE SET value = excluded.value`).run(head)
|
|
134
|
+
memory.db.prepare(`INSERT INTO meta (key, value) VALUES (?, ?)
|
|
135
|
+
ON CONFLICT (key) DO UPDATE SET value = excluded.value`).run(anchorKey(origin), head)
|
|
120
136
|
}
|
|
121
137
|
|
|
122
138
|
onProgress?.({ phase: "done", total: diffOut.length, updated, removed, skipped, failed })
|
|
@@ -124,73 +140,19 @@ export async function gitSync(memory, dir, { onProgress } = {}) {
|
|
|
124
140
|
}
|
|
125
141
|
|
|
126
142
|
/**
|
|
127
|
-
*
|
|
128
|
-
*
|
|
129
|
-
* tracked + untracked-not-ignored, .gitignore respected). Projects WITHOUT git
|
|
130
|
-
* fall back to a filesystem walk (PORTABILITY FR15/P8) — before, the index was
|
|
131
|
-
* silently empty there. The walk cannot honour .gitignore (no git → no such
|
|
132
|
-
* concept) and skips the shared SKIP_DIRS/dot-directory set instead.
|
|
133
|
-
* Returns { entries, unlisted }: `entries` = { abs, rel } pairs (rel relative to
|
|
134
|
-
* dir); `unlisted` = files whose extension is in NO index list (count + sample),
|
|
135
|
-
* the visible signal behind "my .xyz files are not searchable" (PORTABILITY PO-9).
|
|
136
|
-
* `truncated` = the non-git walk hit its file cap (capped trees are logged, never
|
|
137
|
-
* silently indexed-partial). opts.maxFiles = the walk cap (test seam).
|
|
143
|
+
* §6.15 同步展开(写侧入口):入口 dir + 声明公共仓逐根(一层不递归);回执 = 入口根(契约零改)。
|
|
144
|
+
* 单根内胆 = `codeSyncDir`(原 `codeSync` 体——mtime 增量 + 行预算 + stale 收尾;opts seam 同 scan.mjs)。
|
|
138
145
|
*/
|
|
139
|
-
export async function
|
|
140
|
-
const
|
|
141
|
-
const
|
|
142
|
-
|
|
143
|
-
const tally = createUnlistedTally(new Set([...code, ...doc]))
|
|
144
|
-
|
|
145
|
-
const files = []
|
|
146
|
-
const finish = (truncated = false) => ({ entries: files, unlisted: tally.result(), truncated })
|
|
147
|
-
|
|
148
|
-
// Git listing when available; a non-git project falls through to the walk.
|
|
149
|
-
let gitTop
|
|
150
|
-
try {
|
|
151
|
-
gitTop = (await new Promise((resolve, reject) => {
|
|
152
|
-
_execFile("git", ["rev-parse", "--show-toplevel"], { cwd: dir, encoding: "utf8", timeout: 5000, windowsHide: true },
|
|
153
|
-
(err, stdout) => { if (err) reject(err); else resolve(stdout.trim()) })
|
|
154
|
-
})).replace(/\\/g, "/")
|
|
155
|
-
} catch {
|
|
156
|
-
// Not a git repo → filesystem walk (FR15: the index must still be usable).
|
|
157
|
-
const walked = await walkProjectFiles(dir, exts, { knownExts: new Set([...code, ...doc]), ...(maxFiles !== undefined ? { maxFiles } : {}) })
|
|
158
|
-
files.push(...walked.files)
|
|
159
|
-
// The walk's cap must stay visible end-to-end (file-walk.mjs: "never silently
|
|
160
|
-
// dropped") — a capped listing says so in the log, not only in a discarded flag.
|
|
161
|
-
if (walked.truncated) {
|
|
162
|
-
logEvent("index:truncated", { dir, files: walked.files.length, maxFiles: maxFiles ?? MAX_WALK_FILES })
|
|
163
|
-
}
|
|
164
|
-
return { entries: files, unlisted: walked.unlisted, truncated: walked.truncated }
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
try {
|
|
168
|
-
const raw = await new Promise((resolve, reject) => {
|
|
169
|
-
_execFile("git", ["ls-files", "--cached", "--others", "--exclude-standard"],
|
|
170
|
-
{ cwd: dir, encoding: "utf8", timeout: 15000, windowsHide: true, maxBuffer: 10 * 1024 * 1024 },
|
|
171
|
-
(err, stdout) => { if (err) reject(err); else resolve(stdout) })
|
|
172
|
-
})
|
|
173
|
-
for (const line of raw.trim().split("\n")) {
|
|
174
|
-
const p = line.trim()
|
|
175
|
-
if (!p) continue
|
|
176
|
-
const rel = p.replace(/\\/g, "/")
|
|
177
|
-
if (isSkippedRelPath(rel)) continue
|
|
178
|
-
const ext = extensionOf(rel)
|
|
179
|
-
if (!ext || !exts.has(ext)) { tally.note(rel); continue }
|
|
180
|
-
files.push({ abs: joinPath(dir, rel), rel })
|
|
181
|
-
}
|
|
182
|
-
} catch { /* ls-files failed */ }
|
|
183
|
-
|
|
184
|
-
return finish()
|
|
146
|
+
export async function codeSync(memory, dir, opts = {}) {
|
|
147
|
+
const res = await codeSyncDir(memory, dir, opts)
|
|
148
|
+
for (const root of declaredRootEntries(dir)) await codeSyncDir(memory, root, opts)
|
|
149
|
+
return res
|
|
185
150
|
}
|
|
186
151
|
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
* Sync code index: scan all source files under dir → chunk → upsert into code_chunks.
|
|
190
|
-
* Incremental by mtime — only rebuilds chunks for files that have changed.
|
|
191
|
-
*/
|
|
192
|
-
export async function codeSync(memory, dir, { onProgress } = {}) {
|
|
152
|
+
/** 单根内胆(code 全扫径):原 `codeSync` 体——零语义改动。 */
|
|
153
|
+
async function codeSyncDir(memory, dir, { onProgress, yieldFn = yieldTick, nowFn = Date.now, yieldMs = SCAN_YIELD_MS } = {}) {
|
|
193
154
|
const origin = normalizeOrigin(dir) // §6.11 写缝归一(遍历/git I/O 用原样 dir;库面 origin 一律归一值)
|
|
155
|
+
const decl = loadProjectDeclaration(dir)
|
|
194
156
|
const { code: exts } = indexExtensions(dir)
|
|
195
157
|
const { entries, unlisted } = await listProjectFiles(dir, exts)
|
|
196
158
|
const files = [] // { abs, rel, mtimeMs }
|
|
@@ -201,6 +163,8 @@ export async function codeSync(memory, dir, { onProgress } = {}) {
|
|
|
201
163
|
if (st.size > MAX_CODE_FILE_BYTES) { overSizeSkipped++; continue }
|
|
202
164
|
files.push({ abs, rel, mtimeMs: Math.floor(st.mtimeMs) })
|
|
203
165
|
}
|
|
166
|
+
// §6.14 P2:处理序钉死 = `rel` 字典序(现 walk ∕ git 列序不确定;预算跳过 = 列序后缀)
|
|
167
|
+
files.sort((a, b) => (a.rel < b.rel ? -1 : a.rel > b.rel ? 1 : 0))
|
|
204
168
|
|
|
205
169
|
const indexed = new Map(
|
|
206
170
|
memory.db.prepare(`SELECT path, mtime_ms FROM code_chunks WHERE origin = ?`).all(origin).map((r) => [r.path, r.mtime_ms])
|
|
@@ -209,6 +173,11 @@ export async function codeSync(memory, dir, { onProgress } = {}) {
|
|
|
209
173
|
|
|
210
174
|
onProgress?.({ phase: "scan", total: files.length, overSizeSkipped })
|
|
211
175
|
|
|
176
|
+
// §6.14 P2 行预算(基准 = 本 origin 库内行数 code+doc,开趟读一次;只停新增——存量行不失效;
|
|
177
|
+
// 跳过 = 不落行 ∕ 不记 mtime ⇒ 下趟重试)。WARN 一行可见(每趟至多一行)+ CAP 起跳过后列文件。
|
|
178
|
+
const budget = createRowBudget(memory, origin, "code", dir)
|
|
179
|
+
const budgetSkipped = []
|
|
180
|
+
|
|
212
181
|
let updated = 0, removed = 0, skipped = 0, failed = 0
|
|
213
182
|
const errors = []
|
|
214
183
|
for (let i = 0; i < files.length; i++) {
|
|
@@ -219,12 +188,16 @@ export async function codeSync(memory, dir, { onProgress } = {}) {
|
|
|
219
188
|
skipped++
|
|
220
189
|
continue
|
|
221
190
|
}
|
|
191
|
+
budget.note()
|
|
192
|
+
if (budget.over()) { budgetSkipped.push(rel); continue }
|
|
222
193
|
|
|
223
194
|
try {
|
|
195
|
+
const before = rowCountOfPath(memory, "code_chunks", origin, rel)
|
|
224
196
|
const text = await readFile(abs, "utf8")
|
|
225
197
|
const lines = text.split("\n")
|
|
226
198
|
const lang = detectLanguage(abs)
|
|
227
199
|
_upsertCodeFile(memory, origin, rel, lines, lang, mtimeMs)
|
|
200
|
+
budget.add(rowCountOfPath(memory, "code_chunks", origin, rel) - before) // 净增行数——预算与「库内行数」同刻度
|
|
228
201
|
updated++
|
|
229
202
|
} catch (e) {
|
|
230
203
|
failed++
|
|
@@ -237,154 +210,34 @@ export async function codeSync(memory, dir, { onProgress } = {}) {
|
|
|
237
210
|
}
|
|
238
211
|
}
|
|
239
212
|
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
memory.db.prepare(`DELETE FROM code_chunks WHERE origin = ? AND path = ?`).run(origin, stale)
|
|
243
|
-
removed++
|
|
244
|
-
}
|
|
245
|
-
}
|
|
213
|
+
// §6.14 B6 ∕ M3:收尾 stale 循环(同款让出 + 面① 收尾保护位)——机制单源 = `sync-tail.mjs`
|
|
214
|
+
removed = await sweepStaleRows(memory, { table: "code_chunks", origin, indexed, seen, decl, base: dir, yieldFn, nowFn, yieldMs })
|
|
246
215
|
|
|
247
|
-
onProgress?.({ phase: "done", total: files.length, updated, removed, skipped, failed, overSizeSkipped })
|
|
248
|
-
|
|
216
|
+
onProgress?.({ phase: "done", total: files.length, updated, removed, skipped, failed, overSizeSkipped, budgetSkipped: budgetSkipped.length })
|
|
217
|
+
// §6.14 B7 维护口:模型键失配的失效**执行**(读面只判定降级——执行住同步尾;覆盖 /reindex ∕ 桌面 ∕ VSC 构建)
|
|
218
|
+
if (memory.embedder) { try { invalidateStaleEmbeddings(memory) } catch { /* 非阻塞 */ } }
|
|
219
|
+
await markIndexedCommit(memory, dir) // 锚推进 = 同步完成条件(§6.14 M1——原 fire-and-forget 竞态,T12 ∕ T14 判别面需确定性)
|
|
249
220
|
if (unlisted.count > 0) {
|
|
250
221
|
logEvent("index:unlisted", { dir, kind: "code", count: unlisted.count, exts: unlisted.exts.map((e) => e.ext) })
|
|
251
222
|
}
|
|
252
|
-
return { updated, removed, skipped, failed, errors, total: files.length, overSizeSkipped, unlistedExts: unlisted }
|
|
223
|
+
return { updated, removed, skipped, failed, errors, total: files.length, overSizeSkipped, budgetSkipped, unlistedExts: unlisted }
|
|
253
224
|
}
|
|
254
225
|
|
|
255
|
-
/** Record current HEAD as the index anchor (
|
|
226
|
+
/** Record current HEAD as the index anchor (§6.14 M1: per-origin key `last_indexed_commit:<归一键>`); silently skip non-git repos */
|
|
256
227
|
export async function markIndexedCommit(memory, dir) {
|
|
257
228
|
try {
|
|
258
229
|
const { execFile } = await import("node:child_process")
|
|
230
|
+
const origin = normalizeOrigin(dir)
|
|
259
231
|
const head = await new Promise((resolve, reject) => {
|
|
260
232
|
execFile("git", ["rev-parse", "HEAD"], { cwd: dir, encoding: "utf8", timeout: 5000, windowsHide: true }, (err, stdout) => {
|
|
261
233
|
if (err) reject(err); else resolve(stdout)
|
|
262
234
|
})
|
|
263
235
|
})
|
|
264
|
-
memory.db.prepare(`INSERT INTO meta (key, value) VALUES (
|
|
265
|
-
ON CONFLICT (key) DO UPDATE SET value = excluded.value`).run(head.trim())
|
|
236
|
+
memory.db.prepare(`INSERT INTO meta (key, value) VALUES (?, ?)
|
|
237
|
+
ON CONFLICT (key) DO UPDATE SET value = excluded.value`).run(anchorKey(origin), head.trim())
|
|
266
238
|
} catch { /* not a git repo or git unavailable, skip */ }
|
|
267
239
|
}
|
|
268
240
|
|
|
269
|
-
/**
|
|
270
|
-
* Code search: FTS5(BM25) + optional vector cosine, RRF merged.
|
|
271
|
-
* Falls back to pure FTS when no embedder; falls back to pure vector when ftsQuery is empty and embedder is present.
|
|
272
|
-
*/
|
|
273
|
-
export async function codeSearch(memory, query, { limit = 5 } = {}) {
|
|
274
|
-
const ftsQuery = buildFtsQuery(query)
|
|
275
|
-
if (!ftsQuery && !memory.embedder) return []
|
|
276
|
-
|
|
277
|
-
// §6.11 读缝归一(单点取名——函数体内一律用归一值)
|
|
278
|
-
const codeOrigin = normalizeOrigin(memory.codeOrigin)
|
|
279
|
-
const ftsOriginFilter = codeOrigin ? `AND c.origin = ?` : ""
|
|
280
|
-
const vecOriginFilter = codeOrigin ? `AND origin = ?` : ""
|
|
281
|
-
const originParams = codeOrigin ? [codeOrigin] : []
|
|
282
|
-
|
|
283
|
-
const ftsList = ftsQuery ? memory.db.prepare(`
|
|
284
|
-
SELECT c.rowid, c.path, c.language, c.symbol_name, c.content, c.line_start, c.line_end, bm25(code_chunks_fts) AS rank
|
|
285
|
-
FROM code_chunks_fts JOIN code_chunks c ON c.rowid = code_chunks_fts.rowid
|
|
286
|
-
WHERE code_chunks_fts MATCH ? ${ftsOriginFilter}
|
|
287
|
-
ORDER BY rank LIMIT ?
|
|
288
|
-
`).all(ftsQuery, ...originParams, Math.max(limit * 4, 20)) : []
|
|
289
|
-
|
|
290
|
-
if (!memory.embedder) return ftsList.slice(0, limit)
|
|
291
|
-
|
|
292
|
-
try { await ensureEmbeddings(memory) } catch (e) {
|
|
293
|
-
console.error(`[code] embedding ensure failed, falling back to FTS-only: ${e.message}`)
|
|
294
|
-
return ftsList.slice(0, limit)
|
|
295
|
-
}
|
|
296
|
-
let qvec
|
|
297
|
-
try { [qvec] = await embed(memory.embedder, [query]) } catch (e) {
|
|
298
|
-
console.error(`[code] query embedding failed, falling back to FTS-only: ${e.message}`)
|
|
299
|
-
return ftsList.slice(0, limit)
|
|
300
|
-
}
|
|
301
|
-
// TUI-OOM-ROOTCAUSE(MEMORY.md §10.3):分块扫描 + 有界 top-K(原全表 .all()——峰值 = 块 + K)
|
|
302
|
-
// TUI 假死批(§6.10 修法 A1/A2):游标 = PK 去等值过滤前缀列(有 origin 过滤 ⇒ 2 元组;
|
|
303
|
-
// 无过滤 ⇒ 全 PK);scanVectors = async(让出)。SELECT 须携键列(游标值源)。
|
|
304
|
-
const cursorKey = codeOrigin ? ["path", "line_start"] : ["origin", "path", "line_start"]
|
|
305
|
-
const keyCols = cursorKey.join(", ")
|
|
306
|
-
const top = createTopK(Math.max(limit * 4, 20))
|
|
307
|
-
await scanVectors(memory.db, `SELECT rowid, ${keyCols}, embedding FROM code_chunks WHERE embedding IS NOT NULL ${vecOriginFilter}`, originParams, {
|
|
308
|
-
cursorKey,
|
|
309
|
-
onRow: (r) => top.push({ id: r.rowid, rowid: r.rowid, score: cosine(qvec, fromBlob(r.embedding)) }),
|
|
310
|
-
})
|
|
311
|
-
const vecList = top.list().map((c) => ({ rowid: c.id, score: c.score }))
|
|
312
|
-
|
|
313
|
-
const K = 60
|
|
314
|
-
const scores = new Map()
|
|
315
|
-
ftsList.forEach((r, i) => scores.set(r.rowid, (scores.get(r.rowid) ?? 0) + 1 / (K + i + 1)))
|
|
316
|
-
vecList.forEach((r, i) => scores.set(r.rowid, (scores.get(r.rowid) ?? 0) + 1 / (K + i + 1)))
|
|
317
|
-
|
|
318
|
-
const fetchChunk = memory.db.prepare(`
|
|
319
|
-
SELECT path, language, symbol_name, content, line_start, line_end FROM code_chunks WHERE rowid = ?
|
|
320
|
-
`)
|
|
321
|
-
const sorted = [...scores.entries()]
|
|
322
|
-
.sort((a, b) => b[1] - a[1])
|
|
323
|
-
.slice(0, limit)
|
|
324
|
-
return sorted
|
|
325
|
-
.map(([rowid, score]) => {
|
|
326
|
-
const chunk = fetchChunk.get(rowid)
|
|
327
|
-
if (!chunk) return null
|
|
328
|
-
chunk._score = Math.round(score * 100) / 100
|
|
329
|
-
return chunk
|
|
330
|
-
})
|
|
331
|
-
.filter(Boolean)
|
|
332
|
-
}
|
|
333
|
-
|
|
334
|
-
/** Lazily backfill missing vectors for code_chunks. Guarded against concurrent calls. */
|
|
335
|
-
let _codeEmbedLock = null
|
|
336
|
-
export function ensureCodeEmbeddings(memory) {
|
|
337
|
-
if (_codeEmbedLock) return _codeEmbedLock
|
|
338
|
-
_codeEmbedLock = _runEnsureCodeEmbeddings(memory).finally(() => { _codeEmbedLock = null })
|
|
339
|
-
return _codeEmbedLock
|
|
340
|
-
}
|
|
341
|
-
|
|
342
|
-
async function _runEnsureCodeEmbeddings(memory) {
|
|
343
|
-
if (!memory.embedder) return
|
|
344
|
-
const modelKey = memory.embedder.model
|
|
345
|
-
const stored = memory.db.prepare(`SELECT value FROM meta WHERE key = 'code_embedding_model'`).get()?.value
|
|
346
|
-
if (stored !== modelKey) {
|
|
347
|
-
memory.db.prepare(`UPDATE code_chunks SET embedding = NULL`).run()
|
|
348
|
-
memory.db.prepare(`INSERT INTO meta (key, value) VALUES ('code_embedding_model', ?)
|
|
349
|
-
ON CONFLICT (key) DO UPDATE SET value = excluded.value`).run(modelKey)
|
|
350
|
-
}
|
|
351
|
-
|
|
352
|
-
const pending = memory.db.prepare(`SELECT rowid, path, symbol_name, content FROM code_chunks WHERE embedding IS NULL LIMIT ${CODE_EMBED_BATCH}`).all()
|
|
353
|
-
if (pending.length === 0) return
|
|
354
|
-
|
|
355
|
-
const texts = pending.map((r) => `${r.path}${r.symbol_name ? " :: " + r.symbol_name : ""}\n${safeSliceUTF16(r.content, EMBED_TEXT_MAX_LEN)}`)
|
|
356
|
-
const vecs = await embed(memory.embedder, texts)
|
|
357
|
-
|
|
358
|
-
const update = memory.db.prepare(`UPDATE code_chunks SET embedding = ? WHERE rowid = ?`)
|
|
359
|
-
pending.forEach((r, i) => update.run(toBlob(vecs[i]), r.rowid))
|
|
360
|
-
}
|
|
361
|
-
|
|
362
|
-
/** Generate the code_search tool (read-only). */
|
|
363
|
-
export function codeSearchTool(memory) {
|
|
364
|
-
return {
|
|
365
|
-
name: "code_search",
|
|
366
|
-
description:
|
|
367
|
-
"Search the project's source code for relevant code. Use this to find functions, classes, or code patterns across the codebase. Supports natural language queries and code snippets. Returns matching code chunks with file paths and line numbers. Prefer doc_search for the intended design (design docs, conventions); code_search for the implementation as written. " +
|
|
368
|
-
"For what was said in a session (conversation/chat history), use read_history.",
|
|
369
|
-
parameters: {
|
|
370
|
-
type: "object",
|
|
371
|
-
properties: {
|
|
372
|
-
query: { type: "string", description: "Natural language or code snippet to search for" },
|
|
373
|
-
limit: { type: "number", description: "Max results (default 5)" },
|
|
374
|
-
},
|
|
375
|
-
required: ["query"],
|
|
376
|
-
},
|
|
377
|
-
readonly: true,
|
|
378
|
-
async execute(args) {
|
|
379
|
-
const results = await codeSearch(memory, args.query, { limit: args.limit ?? 5 })
|
|
380
|
-
if (results.length === 0) return "(no matching code)"
|
|
381
|
-
return results.map((r) =>
|
|
382
|
-
`${r.path}${r.symbol_name ? ` :: ${r.symbol_name}` : ""} (L${r.line_start}-L${r.line_end}, relevance ${r._score?.toFixed(2) ?? "?"}):\n${r.content.slice(0, 2000)}`
|
|
383
|
-
).join("\n\n---\n\n")
|
|
384
|
-
},
|
|
385
|
-
}
|
|
386
|
-
}
|
|
387
|
-
|
|
388
241
|
/**
|
|
389
242
|
* Single-file incremental reindex: called after write/edit/delete, only rebuilds this one path.
|
|
390
243
|
*/
|
|
@@ -394,6 +247,7 @@ export async function reindexFile(memory, cwd, absPath) {
|
|
|
394
247
|
const rel = relative(cwd, absPath).replaceAll("\\", "/")
|
|
395
248
|
if (rel === ".." || rel.startsWith("../")) return
|
|
396
249
|
if (isSkippedRelPath(rel)) return
|
|
250
|
+
if (isExcludedRelPath(rel, loadProjectDeclaration(cwd), cwd)) return // §6.14 面① 第三起效点(单文件缝零写)
|
|
397
251
|
// Declared extensions count for the single-file path too (union with built-ins).
|
|
398
252
|
const { code: codeExts, doc: docExts } = indexExtensions(cwd)
|
|
399
253
|
|
|
@@ -425,3 +279,8 @@ export async function reindexFile(memory, cwd, absPath) {
|
|
|
425
279
|
try { await ensureEmbeddings(memory) } catch { /* embedding failure is non-blocking */ }
|
|
426
280
|
}
|
|
427
281
|
}
|
|
282
|
+
|
|
283
|
+
// ─── 迁出面(2026-10-01 拆分批 · #755 ∥ #786):检索面 → `memory/code-search.mjs`;清单面 →
|
|
284
|
+
// `memory/file-list.mjs`——经本档转口保名(消费面 / 批内件 import 面零改),四起效点调用形零改。──────
|
|
285
|
+
export { codeSearch, ensureCodeEmbeddings, codeSearchTool } from "./code-search.mjs"
|
|
286
|
+
export { listProjectFiles, indexExtensions } from "./file-list.mjs"
|
package/memory/core.mjs
CHANGED
|
@@ -52,10 +52,15 @@ export async function search(memory, query, { limit = 5 } = {}) {
|
|
|
52
52
|
if (!memory.embedder) return ftsList.slice(0, limit)
|
|
53
53
|
|
|
54
54
|
// ---- vector channel ----
|
|
55
|
-
|
|
55
|
+
let ensured
|
|
56
|
+
try { ensured = await ensureEmbeddings(memory) } catch (e) {
|
|
56
57
|
console.error(`[memory] embedding ensure failed, falling back to FTS-only: ${e.message}`)
|
|
57
58
|
return ftsList.slice(0, limit)
|
|
58
59
|
}
|
|
60
|
+
if (ensured?.mismatch) { // §6.14 B7 读面零写:模型键失配 ⇒ 本面向量通道降级 FTS-only + 一行可见(失效执行归维护口)
|
|
61
|
+
console.warn("[memory] embedding model mismatch — vector channel degraded to FTS-only this run (rebuild runs at maintenance: sync tail / reindex)")
|
|
62
|
+
return ftsList.slice(0, limit)
|
|
63
|
+
}
|
|
59
64
|
let qvec
|
|
60
65
|
try { [qvec] = await embed(memory.embedder, [query]) } catch (e) {
|
|
61
66
|
console.error(`[memory] query embedding failed, falling back to FTS-only: ${e.message}`)
|
|
@@ -143,7 +148,9 @@ export function fetchEntry(memory, uid) {
|
|
|
143
148
|
|
|
144
149
|
/**
|
|
145
150
|
* Lazy embedding: batch-compute vectors for entries that don't have them yet (slow first time, zero cost thereafter).
|
|
146
|
-
* When the embedding model changes
|
|
151
|
+
* When the embedding model changes the READ face performs ZERO writes (§6.14 B7): it only reports the
|
|
152
|
+
* mismatch back (`{ mismatch }`) so callers degrade to FTS-only; invalidation (clear + re-key) runs at
|
|
153
|
+
* the maintenance entry `invalidateStaleEmbeddings` (sync tail ∕ /reindex ∕ desktop ∕ VSC build).
|
|
147
154
|
* Guarded by a module-level lock — concurrent fire-and-forget callers share the same promise,
|
|
148
155
|
* so embedding API calls are never duplicated.
|
|
149
156
|
*/
|
|
@@ -157,39 +164,47 @@ export function ensureEmbeddings(memory) {
|
|
|
157
164
|
async function _runEnsureEmbeddings(memory) {
|
|
158
165
|
const modelKey = memory.embedder.model
|
|
159
166
|
const stored = memory.db.prepare(`SELECT value FROM meta WHERE key = 'embedding_model'`).get()?.value
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
memory.db.prepare(`
|
|
164
|
-
memory.db.prepare(`
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
167
|
+
// §6.14 B7 读面零写:失配 ⇒ **不执行**失效(不清向量 ∕ 不落键——执行归维护口);本面回降级信号。
|
|
168
|
+
const mismatch = stored !== modelKey
|
|
169
|
+
if (!mismatch) {
|
|
170
|
+
const pendingEntries = memory.db.prepare(`SELECT id, title, content FROM entries WHERE embedding IS NULL LIMIT ${EMBED_BATCH_SIZE}`).all()
|
|
171
|
+
const pendingFiles = memory.db.prepare(`SELECT rowid, title, content FROM files WHERE embedding IS NULL LIMIT ${EMBED_BATCH_SIZE}`).all()
|
|
172
|
+
if (pendingEntries.length + pendingFiles.length > 0) {
|
|
173
|
+
const items = [...pendingEntries, ...pendingFiles]
|
|
174
|
+
const texts = items.map((r) => `${r.title}\n${safeSliceUTF16(r.content, EMBED_TEXT_MAX_LEN)}`)
|
|
175
|
+
const vecs = await embed(memory.embedder, texts)
|
|
176
|
+
const updateEntry = memory.db.prepare(`UPDATE entries SET embedding = ? WHERE id = ?`)
|
|
177
|
+
pendingEntries.forEach((r, i) => updateEntry.run(toBlob(vecs[i]), r.id))
|
|
178
|
+
const updateFile = memory.db.prepare(`UPDATE files SET embedding = ? WHERE rowid = ?`)
|
|
179
|
+
pendingFiles.forEach((r, i) => updateFile.run(toBlob(vecs[pendingEntries.length + i]), r.rowid))
|
|
180
|
+
}
|
|
170
181
|
}
|
|
182
|
+
// 逐面判定 + 回填(code ∕ doc 面自持模型键——同判据;失配面同样零写回降级信号)
|
|
183
|
+
const { ensureCodeEmbeddings } = await import("./code-sync.mjs")
|
|
184
|
+
const { ensureDocEmbeddings } = await import("./docs.mjs")
|
|
185
|
+
const code = await ensureCodeEmbeddings(memory)
|
|
186
|
+
const doc = await ensureDocEmbeddings(memory)
|
|
187
|
+
return { mismatch, code: Boolean(code?.mismatch), doc: Boolean(doc?.mismatch) }
|
|
188
|
+
}
|
|
171
189
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
190
|
+
/**
|
|
191
|
+
* 维护口失效**执行**(§6.14 B7 ∕ D-MEM25):模型键失配的面 ⇒ 清该面向量 + 落键(一次性重写)。
|
|
192
|
+
* **读面禁调**(search ∕ codeSearch ∕ docSearch 只判定降级);调用点 = 同步尾(codeSync ∕ docSync——
|
|
193
|
+
* 覆盖 /reindex ∕ 桌面 ∕ VSC 构建三条维护路径)。返回被清的面键(空数组 = 无失配,零写)。
|
|
194
|
+
*/
|
|
195
|
+
export function invalidateStaleEmbeddings(memory) {
|
|
196
|
+
if (!memory.embedder) return []
|
|
197
|
+
const modelKey = memory.embedder.model
|
|
198
|
+
const upsert = memory.db.prepare(`INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value = excluded.value`)
|
|
199
|
+
const faces = [["embedding_model", ["entries", "files"]], ["code_embedding_model", ["code_chunks"]], ["doc_embedding_model", ["doc_chunks"]]]
|
|
200
|
+
const invalidated = []
|
|
201
|
+
for (const [key, tables] of faces) {
|
|
202
|
+
if (memory.db.prepare(`SELECT value FROM meta WHERE key = ?`).get(key)?.value === modelKey) continue
|
|
203
|
+
for (const t of tables) memory.db.prepare(`UPDATE ${t} SET embedding = NULL`).run()
|
|
204
|
+
upsert.run(key, modelKey)
|
|
205
|
+
invalidated.push(key)
|
|
179
206
|
}
|
|
180
|
-
|
|
181
|
-
const items = [...pendingEntries, ...pendingFiles]
|
|
182
|
-
const texts = items.map((r) => `${r.title}\n${safeSliceUTF16(r.content, EMBED_TEXT_MAX_LEN)}`)
|
|
183
|
-
const vecs = await embed(memory.embedder, texts)
|
|
184
|
-
|
|
185
|
-
const updateEntry = memory.db.prepare(`UPDATE entries SET embedding = ? WHERE id = ?`)
|
|
186
|
-
pendingEntries.forEach((r, i) => updateEntry.run(toBlob(vecs[i]), r.id))
|
|
187
|
-
const updateFile = memory.db.prepare(`UPDATE files SET embedding = ? WHERE rowid = ?`)
|
|
188
|
-
pendingFiles.forEach((r, i) => updateFile.run(toBlob(vecs[pendingEntries.length + i]), r.rowid))
|
|
189
|
-
|
|
190
|
-
// After each batch of embeddings, also backfill code and doc chunks
|
|
191
|
-
await (await import("./code-sync.mjs")).ensureCodeEmbeddings(memory)
|
|
192
|
-
await (await import("./docs.mjs")).ensureDocEmbeddings(memory)
|
|
207
|
+
return invalidated
|
|
193
208
|
}
|
|
194
209
|
|
|
195
210
|
/**
|