minicode-ai 0.9.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +935 -0
- package/LICENSE +21 -0
- package/README.md +200 -0
- package/cli/args.ts +195 -0
- package/cli/auto-update.ts +108 -0
- package/cli/commands/auth.ts +166 -0
- package/cli/commands/config.ts +382 -0
- package/cli/commands/doctor.ts +128 -0
- package/cli/commands/exec.ts +179 -0
- package/cli/commands/mcp.ts +35 -0
- package/cli/commands/memory.ts +107 -0
- package/cli/commands/pricing.ts +100 -0
- package/cli/commands/providers.ts +162 -0
- package/cli/commands/sessions.ts +128 -0
- package/cli/commands/skills.ts +61 -0
- package/cli/commands/stats.ts +82 -0
- package/cli/commands.ts +317 -0
- package/cli/index.ts +539 -0
- package/cli/model-manager.ts +112 -0
- package/cli/provider-manager.ts +111 -0
- package/cli/repl.ts +734 -0
- package/cli/router.ts +124 -0
- package/cli/setup.ts +780 -0
- package/cli/wizard.ts +21 -0
- package/docs/ARCHITECTURE.html +1222 -0
- package/docs/README.md +109 -0
- package/docs/architecture.md +44 -0
- package/docs/changelog.md +3 -0
- package/package.json +91 -0
- package/src/agents/pool.ts +43 -0
- package/src/app/mentions.ts +50 -0
- package/src/app/provider-layer.ts +118 -0
- package/src/app/rag-layer.ts +105 -0
- package/src/app/session.ts +154 -0
- package/src/app/tool-layer.ts +56 -0
- package/src/config.ts +372 -0
- package/src/constants.ts +119 -0
- package/src/hooks/run.ts +89 -0
- package/src/lib/atomic-write.ts +88 -0
- package/src/lib/db-path.ts +36 -0
- package/src/lib/git-hardening.ts +91 -0
- package/src/lib/ignore.ts +58 -0
- package/src/lib/keystore.ts +202 -0
- package/src/lib/net.ts +70 -0
- package/src/lib/safe-open.ts +138 -0
- package/src/lib/session-id.ts +23 -0
- package/src/lib/trash.ts +38 -0
- package/src/lib/trusted-exec.ts +81 -0
- package/src/lsp/client.ts +395 -0
- package/src/mcp/client.ts +400 -0
- package/src/mcp/http-transport.ts +344 -0
- package/src/mcp/server.ts +538 -0
- package/src/mcp/transport.ts +150 -0
- package/src/memory/files.ts +159 -0
- package/src/memory/vector.ts +775 -0
- package/src/policy/allowlist.ts +67 -0
- package/src/policy/bash-guard.ts +465 -0
- package/src/policy/compaction.ts +271 -0
- package/src/policy/context.ts +161 -0
- package/src/policy/executor.ts +199 -0
- package/src/policy/jail.ts +127 -0
- package/src/policy/permission.ts +517 -0
- package/src/policy/pricing.ts +336 -0
- package/src/policy/ratelimit.ts +54 -0
- package/src/policy/sandbox-policy.ts +157 -0
- package/src/policy/scrub.ts +138 -0
- package/src/policy/update-check.ts +200 -0
- package/src/policy/usage.ts +292 -0
- package/src/policy/verifier.ts +181 -0
- package/src/providers/anthropic.ts +401 -0
- package/src/providers/auth-store.ts +139 -0
- package/src/providers/build.ts +136 -0
- package/src/providers/detect.ts +126 -0
- package/src/providers/effort.ts +165 -0
- package/src/providers/guards.ts +107 -0
- package/src/providers/oauth.ts +326 -0
- package/src/providers/presets.ts +122 -0
- package/src/providers/provision.ts +298 -0
- package/src/providers/responses.ts +371 -0
- package/src/providers/router.ts +247 -0
- package/src/repo/repomap.ts +470 -0
- package/src/sandbox/docker.ts +129 -0
- package/src/sandbox/os.ts +193 -0
- package/src/session/checkpoint.ts +543 -0
- package/src/session/journal.ts +1347 -0
- package/src/session/persistence.ts +446 -0
- package/src/session/shadow-git.ts +351 -0
- package/src/session/turn-marker.ts +118 -0
- package/src/skills/loader.ts +117 -0
- package/src/telemetry/trace.ts +283 -0
- package/src/tools/ask_user.ts +54 -0
- package/src/tools/bash.ts +400 -0
- package/src/tools/code_run.ts +96 -0
- package/src/tools/delete_file.ts +31 -0
- package/src/tools/edit.ts +204 -0
- package/src/tools/git.ts +301 -0
- package/src/tools/glob.ts +90 -0
- package/src/tools/grep.ts +281 -0
- package/src/tools/hashline.ts +66 -0
- package/src/tools/index.ts +103 -0
- package/src/tools/lsp.ts +346 -0
- package/src/tools/mcp_call.ts +163 -0
- package/src/tools/memory.ts +144 -0
- package/src/tools/move_file.ts +47 -0
- package/src/tools/patch.ts +86 -0
- package/src/tools/read_file.ts +130 -0
- package/src/tools/read_image.ts +80 -0
- package/src/tools/submit_result.ts +49 -0
- package/src/tools/task.ts +346 -0
- package/src/tools/todo.ts +201 -0
- package/src/tools/web_fetch.ts +161 -0
- package/src/tools/web_search.ts +156 -0
- package/src/tools/write_file.ts +42 -0
- package/src/ui/approval/prompt.ts +76 -0
- package/src/ui/assistant/simple.ts +568 -0
- package/src/ui/assistant/turn-status.ts +303 -0
- package/src/ui/contract.ts +55 -0
- package/src/ui/footer.ts +105 -0
- package/src/ui/input/input.ts +805 -0
- package/src/ui/input/prompt-engine.ts +606 -0
- package/src/ui/render/collapse.ts +84 -0
- package/src/ui/render/detail.ts +24 -0
- package/src/ui/render/diff.ts +181 -0
- package/src/ui/render/errors.ts +239 -0
- package/src/ui/render/format.ts +54 -0
- package/src/ui/render/highlight.ts +294 -0
- package/src/ui/render/markdown.ts +120 -0
- package/src/ui/render/money.ts +12 -0
- package/src/ui/render/reasoning.ts +20 -0
- package/src/ui/render/sanitize.ts +80 -0
- package/src/ui/render/table.ts +88 -0
- package/src/ui/render/theme.ts +339 -0
- package/src/ui/render/width.ts +256 -0
- package/src/ui/render/wrap.ts +120 -0
- package/src/ui/runtime/bus-debug.ts +83 -0
- package/src/ui/runtime/chrome.ts +301 -0
- package/src/ui/runtime/spinner.ts +77 -0
- package/src/ui/runtime/statusline.ts +227 -0
- package/src/ui/screens/model-manager.ts +434 -0
- package/src/ui/screens/overlay.ts +61 -0
- package/src/ui/screens/picker.ts +307 -0
- package/src/ui/screens/provider-manager.ts +450 -0
- package/src/ui/screens/wizard.ts +132 -0
- package/vendor/minicore/VENDOR.md +29 -0
- package/vendor/minicore/package.json +21 -0
- package/vendor/minicore/src/core/budget.ts +26 -0
- package/vendor/minicore/src/core/compact.ts +76 -0
- package/vendor/minicore/src/core/errors.ts +60 -0
- package/vendor/minicore/src/core/events.ts +74 -0
- package/vendor/minicore/src/core/executor.ts +139 -0
- package/vendor/minicore/src/core/history.ts +32 -0
- package/vendor/minicore/src/core/index.ts +42 -0
- package/vendor/minicore/src/core/loop.ts +344 -0
- package/vendor/minicore/src/core/permission.ts +24 -0
- package/vendor/minicore/src/core/provider.ts +39 -0
- package/vendor/minicore/src/core/recovery.ts +34 -0
- package/vendor/minicore/src/core/session.ts +327 -0
- package/vendor/minicore/src/core/snapshot.ts +142 -0
- package/vendor/minicore/src/core/tokens.ts +111 -0
- package/vendor/minicore/src/core/tool.ts +118 -0
- package/vendor/minicore/src/core/types.ts +50 -0
- package/vendor/minicore/src/providers/openai-compat.ts +366 -0
|
@@ -0,0 +1,775 @@
|
|
|
1
|
+
import { Database } from "bun:sqlite"
|
|
2
|
+
import { Buffer } from "node:buffer"
|
|
3
|
+
import { randomUUID } from "node:crypto"
|
|
4
|
+
import { LIMITS } from "../constants.ts"
|
|
5
|
+
import { resolveDbPath } from "../lib/db-path.ts"
|
|
6
|
+
import { isPrivateHostWithDns } from "../lib/net.ts"
|
|
7
|
+
import { scrubSecrets } from "../policy/scrub.ts"
|
|
8
|
+
|
|
9
|
+
const dbPath = (cwd?: string) => resolveDbPath("vector.db", cwd)
|
|
10
|
+
|
|
11
|
+
const initializedPaths = new Set<string>()
|
|
12
|
+
|
|
13
|
+
function open(cwd?: string): Database {
|
|
14
|
+
return openAt(dbPath(cwd))
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* DB global eksplisit untuk scope="global" (audit 2026-09-16 B9).
|
|
19
|
+
* Sebelumnya scope global jatuh ke open(cwd) = resolveDbPath yang memilih
|
|
20
|
+
* LOKAL bila .minicode ada — env MINICODE_MEMORY_SCOPE=global yang
|
|
21
|
+
* didokumentasikan tak pernah dihormati. Global yang belum pernah ada
|
|
22
|
+
* dilewati (fallback resolusi normal) agar baca tak membuat DB kosong.
|
|
23
|
+
*/
|
|
24
|
+
function openGlobalDb(fallbackCwd?: string): Database {
|
|
25
|
+
try {
|
|
26
|
+
const { join } = require("node:path") as typeof import("node:path")
|
|
27
|
+
const { existsSync } = require("node:fs") as typeof import("node:fs")
|
|
28
|
+
const { homeDir } = require("../lib/db-path.ts") as typeof import("../lib/db-path.ts")
|
|
29
|
+
const g = join(homeDir(), ".minicode", "vector.db")
|
|
30
|
+
if (existsSync(g)) return openAt(g)
|
|
31
|
+
} catch {}
|
|
32
|
+
return open(fallbackCwd)
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function openAt(p: string): Database {
|
|
36
|
+
const db = new Database(p)
|
|
37
|
+
// busy_timeout DULU seperti sessions.db (audit #09 P1 §27): pola yang sama
|
|
38
|
+
// (journal_mode sebelum timeout aktif) gagal SQLITE_BUSY saat dua proses
|
|
39
|
+
// membuka bersamaan.
|
|
40
|
+
try {
|
|
41
|
+
db.exec(`PRAGMA busy_timeout=${LIMITS.SQLITE_BUSY_TIMEOUT_MS}`)
|
|
42
|
+
} catch {}
|
|
43
|
+
if (!initializedPaths.has(p)) {
|
|
44
|
+
// journal_size_limit + wal_autocheckpoint: WAL tidak tumbuh tak terbatas
|
|
45
|
+
db.exec(
|
|
46
|
+
`PRAGMA journal_mode=WAL; PRAGMA busy_timeout=${LIMITS.SQLITE_BUSY_TIMEOUT_MS}; PRAGMA synchronous=NORMAL; PRAGMA journal_size_limit=${LIMITS.SQLITE_WAL_SIZE_LIMIT_BYTES}; PRAGMA wal_autocheckpoint=${LIMITS.SQLITE_WAL_AUTOCHECKPOINT_PAGES};`,
|
|
47
|
+
)
|
|
48
|
+
initializedPaths.add(p)
|
|
49
|
+
}
|
|
50
|
+
db.exec(`
|
|
51
|
+
CREATE TABLE IF NOT EXISTS memory (id TEXT PRIMARY KEY, text TEXT, embedding BLOB, created_at INTEGER, category TEXT DEFAULT 'fact', tags TEXT, model TEXT, dim INTEGER, parent TEXT, access_count INTEGER DEFAULT 0);
|
|
52
|
+
CREATE INDEX IF NOT EXISTS idx_memory_created ON memory(created_at);
|
|
53
|
+
CREATE INDEX IF NOT EXISTS idx_memory_text ON memory(text);
|
|
54
|
+
`)
|
|
55
|
+
// P1.3: kolom model/dim untuk deteksi embedding-mismatch (best-effort migrasi)
|
|
56
|
+
// P2.2: kolom parent untuk chunk entri panjang (chunk berbagi parent id)
|
|
57
|
+
// P13 S2: kategori terstruktur untuk retrieval presisi
|
|
58
|
+
// P13 P1: access_count untuk bukti manfaat (berapa kali row ikut jadi hit)
|
|
59
|
+
try {
|
|
60
|
+
const cols = db.prepare(`PRAGMA table_info(memory)`).all() as { name: string }[]
|
|
61
|
+
if (!cols.some((c) => c.name === "model")) db.exec(`ALTER TABLE memory ADD COLUMN model TEXT`)
|
|
62
|
+
if (!cols.some((c) => c.name === "dim")) db.exec(`ALTER TABLE memory ADD COLUMN dim INTEGER`)
|
|
63
|
+
if (!cols.some((c) => c.name === "parent")) db.exec(`ALTER TABLE memory ADD COLUMN parent TEXT`)
|
|
64
|
+
if (!cols.some((c) => c.name === "category"))
|
|
65
|
+
db.exec(`ALTER TABLE memory ADD COLUMN category TEXT DEFAULT 'fact'`)
|
|
66
|
+
if (!cols.some((c) => c.name === "tags")) db.exec(`ALTER TABLE memory ADD COLUMN tags TEXT`)
|
|
67
|
+
if (!cols.some((c) => c.name === "access_count"))
|
|
68
|
+
db.exec(`ALTER TABLE memory ADD COLUMN access_count INTEGER DEFAULT 0`)
|
|
69
|
+
} catch {}
|
|
70
|
+
// P1.1: expression index + FTS5 untuk keyword pre-filter (10-50× vs instr scan)
|
|
71
|
+
try {
|
|
72
|
+
db.exec(`CREATE INDEX IF NOT EXISTS idx_memory_text_lower ON memory(lower(text));`)
|
|
73
|
+
} catch {}
|
|
74
|
+
try {
|
|
75
|
+
db.exec(`
|
|
76
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS memory_fts USING fts5(text, content='memory', content_rowid='rowid', tokenize='porter unicode61');
|
|
77
|
+
CREATE TRIGGER IF NOT EXISTS memory_ai AFTER INSERT ON memory BEGIN INSERT INTO memory_fts(rowid, text) VALUES (new.rowid, new.text); END;
|
|
78
|
+
CREATE TRIGGER IF NOT EXISTS memory_ad AFTER DELETE ON memory BEGIN INSERT INTO memory_fts(memory_fts, rowid, text) VALUES('delete', old.rowid, old.text); END;
|
|
79
|
+
CREATE TRIGGER IF NOT EXISTS memory_au AFTER UPDATE ON memory BEGIN INSERT INTO memory_fts(memory_fts, rowid, text) VALUES('delete', old.rowid, old.text); INSERT INTO memory_fts(rowid, text) VALUES (new.rowid, new.text); END;
|
|
80
|
+
`)
|
|
81
|
+
// populate FTS untuk DB lama yang sudah ada sebelum trigger
|
|
82
|
+
const cnt =
|
|
83
|
+
(db.prepare(`SELECT count(*) as c FROM memory_fts`).get() as { c: number } | null)?.c ?? 0
|
|
84
|
+
const total =
|
|
85
|
+
(db.prepare(`SELECT count(*) as c FROM memory`).get() as { c: number } | null)?.c ?? 0
|
|
86
|
+
if (cnt === 0 && total > 0) {
|
|
87
|
+
db.exec(`INSERT INTO memory_fts(memory_fts) VALUES('rebuild')`)
|
|
88
|
+
}
|
|
89
|
+
} catch {}
|
|
90
|
+
return db
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function toBlob(vec: number[]): Buffer {
|
|
94
|
+
// copy to avoid byteOffset alignment issues
|
|
95
|
+
const f32 = new Float32Array(vec)
|
|
96
|
+
return Buffer.from(f32.buffer, f32.byteOffset, f32.byteLength)
|
|
97
|
+
}
|
|
98
|
+
function fromBlob(blob: Buffer | Uint8Array): number[] {
|
|
99
|
+
const buf = blob instanceof Uint8Array ? Buffer.from(blob) : (blob as Buffer)
|
|
100
|
+
// ensure 4-byte alignment: copy if misaligned
|
|
101
|
+
if (buf.byteOffset % 4 !== 0) {
|
|
102
|
+
const copy = Buffer.alloc(buf.byteLength)
|
|
103
|
+
buf.copy(copy)
|
|
104
|
+
return Array.from(new Float32Array(copy.buffer, copy.byteOffset, copy.byteLength / 4))
|
|
105
|
+
}
|
|
106
|
+
return Array.from(new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4))
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function cosine(a: number[], b: number[]): number {
|
|
110
|
+
if (a.length !== b.length) return 0
|
|
111
|
+
let dot = 0,
|
|
112
|
+
na = 0,
|
|
113
|
+
nb = 0
|
|
114
|
+
for (let i = 0; i < a.length; i++) {
|
|
115
|
+
dot += a[i]! * b[i]!
|
|
116
|
+
na += a[i]! * a[i]!
|
|
117
|
+
nb += b[i]! * b[i]!
|
|
118
|
+
}
|
|
119
|
+
return dot / (Math.sqrt(na) * Math.sqrt(nb) + 1e-9)
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function keywordScore(query: string, text: string): number {
|
|
123
|
+
const q = query.toLowerCase().split(/\W+/).filter(Boolean)
|
|
124
|
+
const t = text.toLowerCase()
|
|
125
|
+
if (q.length === 0) return 0
|
|
126
|
+
let hits = 0
|
|
127
|
+
for (const w of q) if (t.includes(w)) hits++
|
|
128
|
+
return hits / q.length
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
async function fetchEmbeddingsOnce(
|
|
132
|
+
url: string,
|
|
133
|
+
headers: Record<string, string>,
|
|
134
|
+
body: string,
|
|
135
|
+
parentSignal?: AbortSignal,
|
|
136
|
+
attemptMs: number = LIMITS.EMBEDDING_TIMEOUT_MS,
|
|
137
|
+
): Promise<{ data?: { embedding: number[] }[] } | null> {
|
|
138
|
+
// P1.4: redirect manual max 2 hop, tiap hop dicek SSRF strict (fail-close,
|
|
139
|
+
// tanpa cache agar DNS rebinding tidak lolos via cache basi).
|
|
140
|
+
let current = url
|
|
141
|
+
for (let hop = 0; hop <= 2; hop++) {
|
|
142
|
+
try {
|
|
143
|
+
const hostname = new URL(current).hostname
|
|
144
|
+
if (await isPrivateHostWithDns(hostname, { strict: true, timeoutMs: 1000, noCache: true })) {
|
|
145
|
+
process.stderr.write(`[vector] private host rejected: ${hostname}\n`)
|
|
146
|
+
return null
|
|
147
|
+
}
|
|
148
|
+
} catch {
|
|
149
|
+
return null
|
|
150
|
+
}
|
|
151
|
+
let res: Response
|
|
152
|
+
try {
|
|
153
|
+
// Budget total (parent) di-race dengan timeout per-attempt: attempt
|
|
154
|
+
// lambat tetap dibunuh 3,5 dtk, TAPI seluruh matriks 3×2 tak boleh
|
|
155
|
+
// melewati budget total — tanpa ini setup RAG macet ~21 dtk.
|
|
156
|
+
const perAttempt = AbortSignal.timeout(attemptMs)
|
|
157
|
+
const attemptSignal = parentSignal ? AbortSignal.any([parentSignal, perAttempt]) : perAttempt
|
|
158
|
+
res = await fetch(current, {
|
|
159
|
+
method: "POST",
|
|
160
|
+
headers,
|
|
161
|
+
body,
|
|
162
|
+
redirect: "manual",
|
|
163
|
+
signal: attemptSignal,
|
|
164
|
+
})
|
|
165
|
+
} catch (e) {
|
|
166
|
+
// fallback keyword-only tetap berjalan, tapi jangan senyap total
|
|
167
|
+
process.stderr.write(`[warn] vector: embedding attempt failed: ${(e as Error).message}\n`)
|
|
168
|
+
return null
|
|
169
|
+
}
|
|
170
|
+
if (res.status >= 300 && res.status < 400) {
|
|
171
|
+
const loc = res.headers.get("location")
|
|
172
|
+
if (!loc || hop === 2) return null
|
|
173
|
+
try {
|
|
174
|
+
current = new URL(loc, current).toString()
|
|
175
|
+
const proto = new URL(current).protocol
|
|
176
|
+
if (proto !== "https:" && proto !== "http:") return null
|
|
177
|
+
continue
|
|
178
|
+
} catch {
|
|
179
|
+
return null
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
if (!res.ok) return null
|
|
183
|
+
try {
|
|
184
|
+
return (await res.json()) as { data?: { embedding: number[] }[] }
|
|
185
|
+
} catch {
|
|
186
|
+
return null
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
return null
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
// Diekspor agar bisa diuji (budget total vs per-attempt) tanpa DB —
|
|
193
|
+
// pemakaian produksi lewat addMemory/query yang memakai default.
|
|
194
|
+
export async function embedTexts(
|
|
195
|
+
baseUrl: string,
|
|
196
|
+
apiKey: string,
|
|
197
|
+
texts: string[],
|
|
198
|
+
model?: string,
|
|
199
|
+
timeouts?: { attemptMs?: number; totalMs?: number },
|
|
200
|
+
): Promise<number[][] | null> {
|
|
201
|
+
if (!apiKey || !baseUrl) return null
|
|
202
|
+
try {
|
|
203
|
+
const hostname = new URL(baseUrl).hostname
|
|
204
|
+
if (await isPrivateHostWithDns(hostname, { strict: true, timeoutMs: 1000, noCache: true })) {
|
|
205
|
+
process.stderr.write(`[vector] private host rejected: ${hostname}\n`)
|
|
206
|
+
return null
|
|
207
|
+
}
|
|
208
|
+
} catch {
|
|
209
|
+
return null
|
|
210
|
+
}
|
|
211
|
+
const urls = [
|
|
212
|
+
`${baseUrl.replace(/\/+$/, "")}/embeddings`,
|
|
213
|
+
`${baseUrl.replace(/\/+$/, "")}/v1/embeddings`,
|
|
214
|
+
]
|
|
215
|
+
const headersList = [
|
|
216
|
+
{ Authorization: `Bearer ${apiKey}`, "x-api-key": apiKey, "content-type": "application/json" },
|
|
217
|
+
{ Authorization: `Bearer ${apiKey}`, "content-type": "application/json" },
|
|
218
|
+
{ "x-api-key": apiKey, "content-type": "application/json" },
|
|
219
|
+
]
|
|
220
|
+
const body = JSON.stringify({
|
|
221
|
+
model: model ?? process.env.MINICODE_EMBED_MODEL ?? "text-embedding-3-small",
|
|
222
|
+
input: texts,
|
|
223
|
+
})
|
|
224
|
+
// Budget total untuk seluruh matriks attempt (lihat EMBEDDING_TOTAL_TIMEOUT_MS).
|
|
225
|
+
const budget = AbortSignal.timeout(timeouts?.totalMs ?? LIMITS.EMBEDDING_TOTAL_TIMEOUT_MS)
|
|
226
|
+
// Jangan tahan process hidup hanya karena timer budget ini.
|
|
227
|
+
try {
|
|
228
|
+
;(budget as unknown as { unref?: () => void }).unref?.()
|
|
229
|
+
} catch {}
|
|
230
|
+
for (const headers of headersList) {
|
|
231
|
+
for (const url of urls) {
|
|
232
|
+
if (budget.aborted) return null
|
|
233
|
+
const json = await fetchEmbeddingsOnce(
|
|
234
|
+
url,
|
|
235
|
+
headers as Record<string, string>,
|
|
236
|
+
body,
|
|
237
|
+
budget,
|
|
238
|
+
timeouts?.attemptMs,
|
|
239
|
+
)
|
|
240
|
+
if (json?.data && Array.isArray(json.data)) return json.data.map((d) => d.embedding)
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
return null
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// Retry singkat utk SQLITE_BUSY saat beberapa sub-agent menulis bersamaan.
|
|
247
|
+
// P0.2: async + Bun.sleep agar tidak block event-loop (sebelumnya Atomics.wait freeze 175ms).
|
|
248
|
+
async function withBusyRetry<T>(fn: () => T, attempts = 3): Promise<T> {
|
|
249
|
+
let last: unknown
|
|
250
|
+
for (let i = 0; i < attempts; i++) {
|
|
251
|
+
try {
|
|
252
|
+
return fn()
|
|
253
|
+
} catch (e) {
|
|
254
|
+
const msg = String((e as Error).message ?? e)
|
|
255
|
+
if (!msg.includes("SQLITE_BUSY") && !msg.includes("database is locked")) throw e
|
|
256
|
+
last = e
|
|
257
|
+
await Bun.sleep(25 * 2 ** i)
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
throw last
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
// Escape nilai untuk LIKE: % _ \ jadi literal (dipakai dengan ESCAPE '\').
|
|
264
|
+
function escapeLike(s: string): string {
|
|
265
|
+
return s.replace(/\\/g, "\\\\").replace(/%/g, "\\%").replace(/_/g, "\\_")
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
export async function deleteMemoryByQuery(query: string, cwd?: string): Promise<number> {
|
|
269
|
+
// Hapus dari DB lokal DAN global (bila berkasnya beda): searchHybrid
|
|
270
|
+
// scope=all menggabung keduanya, sehingga hapus satu sisi membuat "lupa"
|
|
271
|
+
// tak tuntas — data lama global tetap ditemukan (temuan audit 2026-09-16
|
|
272
|
+
// M4; tulisan baru selalu lokal sejak audit #10 P1, tapi data legacy
|
|
273
|
+
// global masih bisa ada). Global yang belum pernah ada dilewati agar
|
|
274
|
+
// forget tak membuat berkas DB kosong sebagai side-effect.
|
|
275
|
+
const pattern = escapeLike(query.toLowerCase())
|
|
276
|
+
const paths = [dbPath(cwd)]
|
|
277
|
+
try {
|
|
278
|
+
const { join } = await import("node:path")
|
|
279
|
+
const { homeDir } = await import("../lib/db-path.ts")
|
|
280
|
+
const { existsSync } = await import("node:fs")
|
|
281
|
+
const g = join(homeDir(), ".minicode", "vector.db")
|
|
282
|
+
if (g !== paths[0] && existsSync(g)) paths.push(g)
|
|
283
|
+
} catch {}
|
|
284
|
+
let total = 0
|
|
285
|
+
for (const p of paths) {
|
|
286
|
+
const db = openAt(p)
|
|
287
|
+
try {
|
|
288
|
+
// Hitung dulu: sqlite3_changes() ikut menghitung tulis trigger FTS,
|
|
289
|
+
// jadi info.changes DELETE bukan jumlah baris memory yang terhapus.
|
|
290
|
+
const before = db
|
|
291
|
+
.prepare(
|
|
292
|
+
"SELECT count(*) as c FROM memory WHERE lower(text) LIKE '%' || ? || '%' ESCAPE '\\'",
|
|
293
|
+
)
|
|
294
|
+
.get(pattern) as { c: number } | null
|
|
295
|
+
const n = before?.c ?? 0
|
|
296
|
+
if (n === 0) continue
|
|
297
|
+
await withBusyRetry(() =>
|
|
298
|
+
db
|
|
299
|
+
.prepare("DELETE FROM memory WHERE lower(text) LIKE '%' || ? || '%' ESCAPE '\\'")
|
|
300
|
+
.run(pattern),
|
|
301
|
+
)
|
|
302
|
+
total += n
|
|
303
|
+
} finally {
|
|
304
|
+
db.close()
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
return total
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
export async function clearAllMemory(cwd?: string): Promise<void> {
|
|
311
|
+
const db = open(cwd)
|
|
312
|
+
try {
|
|
313
|
+
await withBusyRetry(() => {
|
|
314
|
+
db.exec("DELETE FROM memory")
|
|
315
|
+
db.exec("VACUUM")
|
|
316
|
+
})
|
|
317
|
+
} finally {
|
|
318
|
+
db.close()
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
// P2.2: pecah teks panjang jadi chunk ber-overlap agar embedding mean-pool
|
|
323
|
+
// tidak menghilangkan detail catatan panjang. Tiap chunk satu baris DB.
|
|
324
|
+
export function splitMemoryChunks(text: string): string[] {
|
|
325
|
+
const size = LIMITS.MEMORY_CHUNK_CHARS
|
|
326
|
+
const overlap = LIMITS.MEMORY_CHUNK_OVERLAP
|
|
327
|
+
if (text.length <= size) return [text]
|
|
328
|
+
const out: string[] = []
|
|
329
|
+
let start = 0
|
|
330
|
+
while (start < text.length) {
|
|
331
|
+
out.push(text.slice(start, start + size))
|
|
332
|
+
if (start + size >= text.length) break
|
|
333
|
+
start += size - overlap
|
|
334
|
+
}
|
|
335
|
+
return out
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
export async function addMemory(
|
|
339
|
+
text: string,
|
|
340
|
+
opts: {
|
|
341
|
+
baseUrl?: string
|
|
342
|
+
apiKey?: string
|
|
343
|
+
cwd?: string
|
|
344
|
+
embeddingModel?: string
|
|
345
|
+
category?: string
|
|
346
|
+
tags?: string[]
|
|
347
|
+
} = {},
|
|
348
|
+
) {
|
|
349
|
+
const clean = scrubSecrets(text)
|
|
350
|
+
const chunks = splitMemoryChunks(clean)
|
|
351
|
+
// Audit #10 P1 (§10/§39): tulis SELALU lokal ke proyek. Tanpa ini,
|
|
352
|
+
// addMemory dari cwd tanpa `.minicode/` jatuh ke DB GLOBAL bersama
|
|
353
|
+
// (resolveDbPath) — memori repo jahat A terbaca di proyek B yang tak
|
|
354
|
+
// terkait. Buat direktori lokal dulu agar resolusi mendarat lokal; baca
|
|
355
|
+
// tetap boleh fallback global (kompatibilitas data lama).
|
|
356
|
+
try {
|
|
357
|
+
const { mkdirSync } = require("node:fs") as typeof import("node:fs")
|
|
358
|
+
const { resolve: resolvePath, join: joinPath } =
|
|
359
|
+
require("node:path") as typeof import("node:path")
|
|
360
|
+
mkdirSync(joinPath(resolvePath(opts.cwd ?? process.cwd()), ".minicode"), { recursive: true })
|
|
361
|
+
} catch {}
|
|
362
|
+
const embedModel =
|
|
363
|
+
opts.embeddingModel ?? process.env.MINICODE_EMBED_MODEL ?? "text-embedding-3-small"
|
|
364
|
+
// Satu panggilan batch untuk semua chunk — hemat round-trip embedding.
|
|
365
|
+
let vecs: number[][] | null = null
|
|
366
|
+
if (opts.baseUrl && opts.apiKey) {
|
|
367
|
+
vecs = await embedTexts(opts.baseUrl, opts.apiKey, chunks, embedModel)
|
|
368
|
+
}
|
|
369
|
+
const parentId = randomUUID()
|
|
370
|
+
const now = Date.now()
|
|
371
|
+
const db = open(opts.cwd)
|
|
372
|
+
// chmod 600 untuk vector.db (+ -wal/-shm) agar secret tidak world-readable
|
|
373
|
+
try {
|
|
374
|
+
const { chmodSync } = require("node:fs") as typeof import("node:fs")
|
|
375
|
+
const p = dbPath(opts.cwd)
|
|
376
|
+
for (const f of [p, `${p}-wal`, `${p}-shm`]) {
|
|
377
|
+
try {
|
|
378
|
+
chmodSync(f, 0o600)
|
|
379
|
+
} catch {}
|
|
380
|
+
}
|
|
381
|
+
} catch {}
|
|
382
|
+
try {
|
|
383
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
384
|
+
const v = vecs?.[i]
|
|
385
|
+
const embedding = v ? toBlob(v) : null
|
|
386
|
+
const id = i === 0 ? parentId : randomUUID()
|
|
387
|
+
const category = (opts.category as string) ?? "fact"
|
|
388
|
+
const tagsJson = opts.tags ? JSON.stringify(opts.tags) : null
|
|
389
|
+
await withBusyRetry(() =>
|
|
390
|
+
db
|
|
391
|
+
.prepare(
|
|
392
|
+
"INSERT INTO memory (id, text, embedding, created_at, model, dim, parent, category, tags, access_count) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 0)",
|
|
393
|
+
)
|
|
394
|
+
.run(
|
|
395
|
+
id,
|
|
396
|
+
chunks[i]!,
|
|
397
|
+
embedding,
|
|
398
|
+
now,
|
|
399
|
+
embedding ? embedModel : null,
|
|
400
|
+
v?.length ?? null,
|
|
401
|
+
parentId,
|
|
402
|
+
category,
|
|
403
|
+
tagsJson,
|
|
404
|
+
),
|
|
405
|
+
)
|
|
406
|
+
}
|
|
407
|
+
// P13 P1: prune TTL HIERARKIS per kategori (best-effort, jangan gagalkan write).
|
|
408
|
+
// Fakta/keputusan awet 180 hari, ringkasan 90, snippet 14 — snippet basi
|
|
409
|
+
// (mis. "test X lolos") menyesatkan bila dibiarkan selama fakta.
|
|
410
|
+
try {
|
|
411
|
+
const day = 24 * 60 * 60 * 1000
|
|
412
|
+
const cuts = [
|
|
413
|
+
{ cats: ["snippet"], ms: LIMITS.MEMORY_TTL_SNIPPET_DAYS * day },
|
|
414
|
+
{ cats: ["summary"], ms: LIMITS.MEMORY_TTL_SUMMARY_DAYS * day },
|
|
415
|
+
{ cats: ["fact", "decision", "preference"], ms: LIMITS.MEMORY_TTL_FACT_DAYS * day },
|
|
416
|
+
]
|
|
417
|
+
for (const c of cuts) {
|
|
418
|
+
if (c.ms <= 0) continue
|
|
419
|
+
const ph = c.cats.map(() => "?").join(",")
|
|
420
|
+
// Bucket fakta juga menampung NULL (DB lama pra-S2) — jangan abadi.
|
|
421
|
+
const nullCatch = c.cats.includes("fact") ? " OR category IS NULL" : ""
|
|
422
|
+
await withBusyRetry(() =>
|
|
423
|
+
db
|
|
424
|
+
.prepare(
|
|
425
|
+
`DELETE FROM memory WHERE (category IN (${ph})${nullCatch}) AND created_at < ?`,
|
|
426
|
+
)
|
|
427
|
+
.run(...c.cats, now - c.ms),
|
|
428
|
+
)
|
|
429
|
+
}
|
|
430
|
+
const cnt =
|
|
431
|
+
(db.prepare("SELECT count(*) as c FROM memory").get() as { c: number } | null)?.c ?? 0
|
|
432
|
+
if (cnt > LIMITS.MEMORY_MAX_ROWS) {
|
|
433
|
+
await withBusyRetry(() =>
|
|
434
|
+
db
|
|
435
|
+
.prepare(
|
|
436
|
+
`DELETE FROM memory WHERE id IN (SELECT id FROM memory ORDER BY created_at DESC LIMIT -1 OFFSET ${LIMITS.MEMORY_MAX_ROWS})`,
|
|
437
|
+
)
|
|
438
|
+
.run(),
|
|
439
|
+
)
|
|
440
|
+
}
|
|
441
|
+
if (cnt % 1000 === 0) {
|
|
442
|
+
try {
|
|
443
|
+
db.exec("VACUUM")
|
|
444
|
+
} catch {}
|
|
445
|
+
}
|
|
446
|
+
} catch (e) {
|
|
447
|
+
process.stderr.write(`[warn] vector: prune failed: ${(e as Error).message}\n`)
|
|
448
|
+
}
|
|
449
|
+
} finally {
|
|
450
|
+
db.close()
|
|
451
|
+
}
|
|
452
|
+
return parentId
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
type MemoryRow = {
|
|
456
|
+
text: string
|
|
457
|
+
embedding: Buffer | null
|
|
458
|
+
dim: number | null
|
|
459
|
+
model: string | null
|
|
460
|
+
created_at: number
|
|
461
|
+
category?: string | null
|
|
462
|
+
tags?: string | null
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
export interface MemoryHit {
|
|
466
|
+
text: string
|
|
467
|
+
score: number
|
|
468
|
+
createdAt: number
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
// Kemiripan teks Jaccard untuk MMR bila pasangan tanpa vektor: cegah 5 hits
|
|
472
|
+
// parafrase ("prefer bun over npm" ×3) menghabiskan budget prompt.
|
|
473
|
+
function textSim(a: string, b: string): number {
|
|
474
|
+
const ta = new Set(a.toLowerCase().split(/\W+/).filter(Boolean))
|
|
475
|
+
const tb = new Set(b.toLowerCase().split(/\W+/).filter(Boolean))
|
|
476
|
+
if (ta.size === 0 || tb.size === 0) return 0
|
|
477
|
+
let inter = 0
|
|
478
|
+
for (const w of ta) if (tb.has(w)) inter++
|
|
479
|
+
return inter / (ta.size + tb.size - inter)
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
// P2.1: buang near-duplikat (cosine > MEMORY_DEDUP_COSINE) + MMR λ untuk
|
|
483
|
+
// diversitas. Tanpa vektor hanya dedup teks eksak (Map) + MMR via textSim.
|
|
484
|
+
function mmrRerank(
|
|
485
|
+
cands: { text: string; score: number; vec: number[] | null }[],
|
|
486
|
+
topK: number,
|
|
487
|
+
): { text: string; score: number }[] {
|
|
488
|
+
const lambda = LIMITS.MEMORY_MMR_LAMBDA
|
|
489
|
+
// 1. near-dedup: kandidat skor-rendah yang nyaris identik dengan yang
|
|
490
|
+
// skor-lebih-tinggi dibuang (tak ada gunanya di prompt dua-duanya).
|
|
491
|
+
const kept: typeof cands = []
|
|
492
|
+
for (const c of cands) {
|
|
493
|
+
let dup = false
|
|
494
|
+
for (const k of kept) {
|
|
495
|
+
if (c.vec && k.vec) {
|
|
496
|
+
if (c.vec.length === k.vec.length && cosine(c.vec, k.vec) > LIMITS.MEMORY_DEDUP_COSINE) {
|
|
497
|
+
dup = true
|
|
498
|
+
break
|
|
499
|
+
}
|
|
500
|
+
} else if (c.text === k.text) {
|
|
501
|
+
dup = true
|
|
502
|
+
break
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
if (!dup) kept.push(c)
|
|
506
|
+
}
|
|
507
|
+
// 2. MMR: tiap putaran pilih kandidat dengan λ*relevansi − (1−λ)*maxSimTerpilih.
|
|
508
|
+
const selected: typeof cands = []
|
|
509
|
+
const rest = [...kept]
|
|
510
|
+
while (rest.length > 0 && selected.length < topK) {
|
|
511
|
+
let best = 0
|
|
512
|
+
let bestVal = -Infinity
|
|
513
|
+
for (let i = 0; i < rest.length; i++) {
|
|
514
|
+
const c = rest[i]!
|
|
515
|
+
let maxSim = 0
|
|
516
|
+
for (const s of selected) {
|
|
517
|
+
const sim =
|
|
518
|
+
c.vec && s.vec && c.vec.length === s.vec.length
|
|
519
|
+
? cosine(c.vec, s.vec)
|
|
520
|
+
: textSim(c.text, s.text)
|
|
521
|
+
if (sim > maxSim) maxSim = sim
|
|
522
|
+
}
|
|
523
|
+
const val = lambda * c.score - (1 - lambda) * maxSim
|
|
524
|
+
if (val > bestVal) {
|
|
525
|
+
bestVal = val
|
|
526
|
+
best = i
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
selected.push(rest.splice(best, 1)[0]!)
|
|
530
|
+
}
|
|
531
|
+
return selected.map(({ text, score }) => ({ text, score }))
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
// Peringatan dim-mismatch cukup sekali per dimensi agar tidak spam stderr tiap hit.
|
|
535
|
+
const dimMismatchWarned = new Set<string>()
|
|
536
|
+
|
|
537
|
+
// Bangun query FTS5 aman: tiap keyword di-quote ("...") agar karakter khusus
|
|
538
|
+
// FTS (*, :, ", -) tidak jadi operator. Return null bila tidak ada keyword valid.
|
|
539
|
+
function buildFtsQuery(query: string): string | null {
|
|
540
|
+
const kws = query.toLowerCase().split(/\W+/).filter(Boolean).slice(0, 5)
|
|
541
|
+
if (kws.length === 0) return null
|
|
542
|
+
return kws.map((k) => `"${k.replace(/"/g, '""')}"`).join(" OR ")
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
export async function searchHybrid(
|
|
546
|
+
query: string,
|
|
547
|
+
opts: {
|
|
548
|
+
baseUrl?: string
|
|
549
|
+
apiKey?: string
|
|
550
|
+
cwd?: string
|
|
551
|
+
topK?: number
|
|
552
|
+
embeddingModel?: string
|
|
553
|
+
scope?: "cwd" | "global" | "all"
|
|
554
|
+
// false = jangan sentuh DB (readonly/plan: retrieval tak boleh menulis
|
|
555
|
+
// access_count/WAL). Default true agar perilaku lama tidak berubah.
|
|
556
|
+
trackAccess?: boolean
|
|
557
|
+
} = {},
|
|
558
|
+
): Promise<MemoryHit[]> {
|
|
559
|
+
const scope = opts.scope ?? (process.env.MINICODE_MEMORY_SCOPE as string) ?? "cwd"
|
|
560
|
+
// helper untuk ambil rows dari satu DB path (dipakai untuk scope all)
|
|
561
|
+
const fetchRows = (db: ReturnType<typeof open>): MemoryRow[] => {
|
|
562
|
+
const keywords = query.toLowerCase().split(/\W+/).filter(Boolean).slice(0, 5)
|
|
563
|
+
const escKeywords = keywords.map((k) =>
|
|
564
|
+
k.replace(/\\/g, "\\\\").replace(/%/g, "\\%").replace(/_/g, "\\_"),
|
|
565
|
+
)
|
|
566
|
+
if (keywords.length > 0) {
|
|
567
|
+
const recentRows = db
|
|
568
|
+
.prepare(
|
|
569
|
+
`SELECT text, embedding, dim, model, created_at, category, tags FROM memory ORDER BY created_at DESC LIMIT ${LIMITS.VECTOR_RECENT_LIMIT}`,
|
|
570
|
+
)
|
|
571
|
+
.all() as MemoryRow[]
|
|
572
|
+
let keywordRows: MemoryRow[] = []
|
|
573
|
+
const ftsQ = buildFtsQuery(query)
|
|
574
|
+
if (ftsQ) {
|
|
575
|
+
try {
|
|
576
|
+
keywordRows = db
|
|
577
|
+
.prepare(
|
|
578
|
+
`SELECT m.text as text, m.embedding as embedding, m.dim as dim, m.model as model, m.created_at as created_at, m.category as category, m.tags as tags FROM memory_fts f JOIN memory m ON m.rowid = f.rowid WHERE memory_fts MATCH ? ORDER BY rank LIMIT ${LIMITS.VECTOR_KEYWORD_LIMIT}`,
|
|
579
|
+
)
|
|
580
|
+
.all(ftsQ) as MemoryRow[]
|
|
581
|
+
} catch {
|
|
582
|
+
keywordRows = []
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
if (keywordRows.length === 0) {
|
|
586
|
+
const likeClauses = keywords
|
|
587
|
+
.map(() => `lower(text) LIKE '%' || lower(?) || '%' ESCAPE '\\'`)
|
|
588
|
+
.join(" OR ")
|
|
589
|
+
keywordRows = db
|
|
590
|
+
.prepare(
|
|
591
|
+
`SELECT text, embedding, dim, model, created_at, category, tags FROM memory WHERE ${likeClauses} ORDER BY created_at DESC LIMIT ${LIMITS.VECTOR_KEYWORD_LIMIT}`,
|
|
592
|
+
)
|
|
593
|
+
.all(...escKeywords) as MemoryRow[]
|
|
594
|
+
}
|
|
595
|
+
const merged = new Map<string, MemoryRow>()
|
|
596
|
+
for (const r of [...recentRows, ...keywordRows])
|
|
597
|
+
if (!merged.has(r.text)) merged.set(r.text, r)
|
|
598
|
+
return [...merged.values()].slice(0, LIMITS.VECTOR_SEARCH_LIMIT)
|
|
599
|
+
} else {
|
|
600
|
+
return db
|
|
601
|
+
.prepare(
|
|
602
|
+
`SELECT text, embedding, dim, model, created_at, category, tags FROM memory ORDER BY created_at DESC LIMIT ${LIMITS.VECTOR_SEARCH_LIMIT}`,
|
|
603
|
+
)
|
|
604
|
+
.all() as MemoryRow[]
|
|
605
|
+
}
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
let rows: MemoryRow[]
|
|
609
|
+
const db = scope === "global" ? openGlobalDb(opts.cwd) : open(opts.cwd)
|
|
610
|
+
try {
|
|
611
|
+
rows = fetchRows(db)
|
|
612
|
+
if (scope === "all") {
|
|
613
|
+
try {
|
|
614
|
+
const { join } = await import("node:path")
|
|
615
|
+
const { homeDir } = await import("../lib/db-path.ts")
|
|
616
|
+
const { Database } = await import("bun:sqlite")
|
|
617
|
+
const globalPath = join(homeDir(), ".minicode", "vector.db")
|
|
618
|
+
const localPath = dbPath(opts.cwd)
|
|
619
|
+
if (globalPath !== localPath) {
|
|
620
|
+
const gdb = new Database(globalPath)
|
|
621
|
+
try {
|
|
622
|
+
const gRows = fetchRows(gdb as unknown as ReturnType<typeof open>)
|
|
623
|
+
const merged = new Map<string, MemoryRow>()
|
|
624
|
+
for (const r of [...rows, ...gRows]) if (!merged.has(r.text)) merged.set(r.text, r)
|
|
625
|
+
rows = [...merged.values()].slice(0, LIMITS.VECTOR_SEARCH_LIMIT)
|
|
626
|
+
} finally {
|
|
627
|
+
gdb.close()
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
} catch {}
|
|
631
|
+
}
|
|
632
|
+
} finally {
|
|
633
|
+
db.close()
|
|
634
|
+
}
|
|
635
|
+
if (rows.length === 0) return []
|
|
636
|
+
|
|
637
|
+
let queryVec: number[] | null = null
|
|
638
|
+
if (opts.baseUrl && opts.apiKey) {
|
|
639
|
+
const vecs = await embedTexts(opts.baseUrl, opts.apiKey, [query], opts.embeddingModel)
|
|
640
|
+
if (vecs?.[0]) queryVec = vecs[0]
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
const scored = rows.map((r) => {
|
|
644
|
+
let vec: number[] | null = null
|
|
645
|
+
let vecScore = 0
|
|
646
|
+
if (queryVec && r.embedding) {
|
|
647
|
+
vec = fromBlob(r.embedding)
|
|
648
|
+
if (vec.length !== queryVec.length) {
|
|
649
|
+
// P1.3: dim mismatch (ganti MINICODE_EMBED_MODEL) → vecScore 0 + warn sekali
|
|
650
|
+
const key = `${r.dim ?? vec.length}->${queryVec.length}`
|
|
651
|
+
if (!dimMismatchWarned.has(key)) {
|
|
652
|
+
dimMismatchWarned.add(key)
|
|
653
|
+
process.stderr.write(
|
|
654
|
+
`[warn] vector: dim mismatch (stored ${r.dim ?? vec.length} vs query ${queryVec.length}, model ${r.model ?? "?"}) — fallback keyword-only untuk hit ini\n`,
|
|
655
|
+
)
|
|
656
|
+
}
|
|
657
|
+
vec = null
|
|
658
|
+
} else {
|
|
659
|
+
vecScore = cosine(queryVec, vec)
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
const kw = keywordScore(query, r.text)
|
|
663
|
+
// hybrid 0.7 vector + 0.3 keyword, if no vector -> keyword only
|
|
664
|
+
let score = queryVec ? vecScore * 0.7 + kw * 0.3 : kw
|
|
665
|
+
// Recency decay (audit #14): koreksi kemarin harus mengalahkan fakta 170
|
|
666
|
+
// hari dengan skor hampir sama — TTL menghapus di ujung, tapi sebelum itu
|
|
667
|
+
// tak ada penurunan bertahap. Paruh-hidup 60 hari: 1 minggu ≈ 0.92×,
|
|
668
|
+
// 6 bulan ≈ 0.12× (jatuh di bawah ambang relevansi, efeknya sama dengan
|
|
669
|
+
// kadaluwarsa). Fakta fresh (age≈0) tidak berubah → test lama tetap hijau.
|
|
670
|
+
const ageDays = (Date.now() - r.created_at) / 86_400_000
|
|
671
|
+
score *= 0.5 ** (ageDays / 60)
|
|
672
|
+
// P13 S2 — boost decision/preference bila query menyentuh kata kuncinya.
|
|
673
|
+
// Regex kini dwibahasa (EN + ID): prompt Indonesia ("perbaiki",
|
|
674
|
+
// "keputusan", "suka") tidak boleh kehilangan boost yang dinikmati versi EN.
|
|
675
|
+
if (
|
|
676
|
+
(r as MemoryRow).category === "decision" &&
|
|
677
|
+
/fix|decide|decision|perbaiki|keputusan|selesaikan/i.test(query)
|
|
678
|
+
)
|
|
679
|
+
score += 0.1
|
|
680
|
+
if (
|
|
681
|
+
(r as MemoryRow).category === "preference" &&
|
|
682
|
+
/prefer|like|preference|suka|ingin/i.test(query)
|
|
683
|
+
)
|
|
684
|
+
score += 0.05
|
|
685
|
+
return { text: r.text, score, vec, createdAt: r.created_at }
|
|
686
|
+
})
|
|
687
|
+
const minScore = queryVec ? LIMITS.MEMORY_MIN_SCORE_HYBRID : LIMITS.MEMORY_MIN_SCORE_KEYWORD
|
|
688
|
+
const filtered = scored.filter((s) => s.score >= minScore)
|
|
689
|
+
filtered.sort((a, b) => b.score - a.score)
|
|
690
|
+
// P2.1: MMR di atas kandidat teratas agar prompt tidak dipenuhi parafrase sama.
|
|
691
|
+
const pool = filtered.slice(0, LIMITS.MEMORY_MMR_CANDIDATES)
|
|
692
|
+
const ranked = mmrRerank(pool, opts.topK ?? 5)
|
|
693
|
+
const createdByText = new Map(pool.map((p) => [p.text, p.createdAt] as const))
|
|
694
|
+
const hits = ranked.map((h) => ({ ...h, createdAt: createdByText.get(h.text) ?? Date.now() }))
|
|
695
|
+
// P13 P1 — bukti manfaat per row: hit yang dikembalikan menaikkan
|
|
696
|
+
// access_count (best-effort; gagal diam — jangan rusak retrieval).
|
|
697
|
+
// Hanya DB lokal yang disentuh; row global (scope all) tidak dihitung.
|
|
698
|
+
// Dilewati bila trackAccess === false (mode readonly/plan).
|
|
699
|
+
if (hits.length > 0 && opts.trackAccess !== false) {
|
|
700
|
+
try {
|
|
701
|
+
const db2 = open(opts.cwd)
|
|
702
|
+
try {
|
|
703
|
+
const ph = hits.map(() => "?").join(",")
|
|
704
|
+
db2
|
|
705
|
+
.prepare(
|
|
706
|
+
`UPDATE memory SET access_count = COALESCE(access_count, 0) + 1 WHERE text IN (${ph})`,
|
|
707
|
+
)
|
|
708
|
+
.run(...hits.map((h) => h.text))
|
|
709
|
+
} finally {
|
|
710
|
+
db2.close()
|
|
711
|
+
}
|
|
712
|
+
} catch {}
|
|
713
|
+
}
|
|
714
|
+
return hits
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
export interface MemoryStats {
|
|
718
|
+
rows: number
|
|
719
|
+
dbBytes: number
|
|
720
|
+
walBytes: number
|
|
721
|
+
shmBytes: number
|
|
722
|
+
models: { model: string; count: number }[]
|
|
723
|
+
dims: { dim: number; count: number }[]
|
|
724
|
+
categories: { category: string; count: number }[]
|
|
725
|
+
oldest: number | null
|
|
726
|
+
newest: number | null
|
|
727
|
+
}
|
|
728
|
+
|
|
729
|
+
// P2.4: observabilitas untuk `minicode memory status` — tanpa embedding, murni SQL.
|
|
730
|
+
export function getMemoryStats(cwd?: string): MemoryStats {
|
|
731
|
+
const db = open(cwd)
|
|
732
|
+
try {
|
|
733
|
+
const rows =
|
|
734
|
+
(db.prepare("SELECT count(*) as c FROM memory").get() as { c: number } | null)?.c ?? 0
|
|
735
|
+
const bounds = db
|
|
736
|
+
.prepare("SELECT min(created_at) as oldest, max(created_at) as newest FROM memory")
|
|
737
|
+
.get() as { oldest: number | null; newest: number | null } | null
|
|
738
|
+
const models = db
|
|
739
|
+
.prepare(
|
|
740
|
+
"SELECT coalesce(model, '(keyword-only)') as model, count(*) as count FROM memory GROUP BY model ORDER BY count DESC",
|
|
741
|
+
)
|
|
742
|
+
.all() as { model: string; count: number }[]
|
|
743
|
+
const dims = db
|
|
744
|
+
.prepare(
|
|
745
|
+
"SELECT coalesce(dim, 0) as dim, count(*) as count FROM memory GROUP BY dim ORDER BY count DESC",
|
|
746
|
+
)
|
|
747
|
+
.all() as { dim: number; count: number }[]
|
|
748
|
+
const categories = db
|
|
749
|
+
.prepare(
|
|
750
|
+
"SELECT coalesce(category, 'fact') as category, count(*) as count FROM memory GROUP BY category ORDER BY count DESC",
|
|
751
|
+
)
|
|
752
|
+
.all() as { category: string; count: number }[]
|
|
753
|
+
const p = dbPath(cwd)
|
|
754
|
+
const sizeOf = (f: string): number => {
|
|
755
|
+
try {
|
|
756
|
+
return (require("node:fs") as typeof import("node:fs")).statSync(f).size
|
|
757
|
+
} catch {
|
|
758
|
+
return 0
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
return {
|
|
762
|
+
rows,
|
|
763
|
+
dbBytes: sizeOf(p),
|
|
764
|
+
walBytes: sizeOf(`${p}-wal`),
|
|
765
|
+
shmBytes: sizeOf(`${p}-shm`),
|
|
766
|
+
models,
|
|
767
|
+
dims,
|
|
768
|
+
categories,
|
|
769
|
+
oldest: bounds?.oldest ?? null,
|
|
770
|
+
newest: bounds?.newest ?? null,
|
|
771
|
+
}
|
|
772
|
+
} finally {
|
|
773
|
+
db.close()
|
|
774
|
+
}
|
|
775
|
+
}
|