akm-cli 0.9.1-beta.1 → 0.9.1-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -1
- package/dist/cli/parse-args.js +7 -1
- package/dist/commands/env/child-env.js +14 -0
- package/dist/commands/improve/eval-cases.js +2 -0
- package/dist/commands/improve/memory/memory-improve.js +1 -0
- package/dist/commands/lint/index.js +5 -1
- package/dist/commands/sources/add-cli.js +8 -2
- package/dist/commands/sources/migration-help.js +12 -3
- package/dist/commands/sources/self-update.js +9 -1
- package/dist/core/adapter/adapters/agent-skills-adapter.js +32 -16
- package/dist/core/adapter/adapters/akm-lint.js +6 -2
- package/dist/core/adapter/adapters/akm-task-adapter.js +4 -2
- package/dist/core/adapter/adapters/dotenv-adapter.js +21 -0
- package/dist/core/asset/frontmatter.js +6 -1
- package/dist/core/common.js +81 -3
- package/dist/core/config/config-io.js +5 -45
- package/dist/core/config/schema/engines.js +14 -3
- package/dist/core/extra-params.js +11 -0
- package/dist/core/fs-txn.js +15 -2
- package/dist/core/json-schema.js +19 -2
- package/dist/core/paths.js +16 -2
- package/dist/core/redaction.js +22 -1
- package/dist/core/state-db.js +1 -0
- package/dist/core/write-source.js +26 -2
- package/dist/indexer/indexer.js +31 -6
- package/dist/indexer/search/db-search.js +17 -2
- package/dist/indexer/walk/walker.js +6 -1
- package/dist/integrations/agent/detect.js +13 -1
- package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +21 -0
- package/dist/integrations/lockfile.js +10 -0
- package/dist/llm/client.js +14 -19
- package/dist/llm/embedder.js +23 -3
- package/dist/llm/embedders/remote.js +27 -2
- package/dist/output/html-render.js +40 -1
- package/dist/runtime.js +23 -1
- package/dist/scripts/akm-migrate-node.js +303 -107
- package/dist/scripts/akm-migrate.js +303 -107
- package/dist/setup/setup.js +22 -7
- package/dist/sources/providers/git-install.js +25 -2
- package/dist/sources/snapshot-fetchers/content-extract.js +63 -1
- package/dist/storage/database.js +71 -12
- package/dist/storage/engines/sqlite-migrations.js +61 -2
- package/dist/storage/repositories/index-connection.js +11 -1
- package/dist/storage/repositories/index-meta-repository.js +11 -0
- package/dist/storage/repositories/index-schema.js +17 -2
- package/dist/storage/repositories/index-vec-repository.js +43 -5
- package/dist/storage/sqlite-pragmas.js +12 -1
- package/dist/tasks/runner.js +84 -7
- package/dist/tasks/scheduler-invocation.js +19 -0
- package/dist/tasks/schema.js +21 -1
- package/dist/text-import-hook.mjs +1 -1
- package/dist/workflows/exec/native-executor.js +8 -0
- package/dist/workflows/exec/step-work.js +10 -2
- package/dist/workflows/parser.js +26 -1
- package/package.json +1 -1
- package/schemas/akm-config.json +10 -5
- package/schemas/akm-workflow.json +7 -3
package/dist/setup/setup.js
CHANGED
|
@@ -496,17 +496,32 @@ export function buildSetupSteps(options) {
|
|
|
496
496
|
];
|
|
497
497
|
return { steps, outcome };
|
|
498
498
|
}
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
499
|
+
/**
|
|
500
|
+
* Resolve the stash directory, apply stash isolation, and THEN read the config.
|
|
501
|
+
*
|
|
502
|
+
* Order is the whole point. The pre-isolation read exists only to discover
|
|
503
|
+
* where the stash is; `applyStashIsolationToEnv` can repoint AKM_CONFIG_DIR,
|
|
504
|
+
* after which both the config contents and `getConfigPath()` differ. Reading
|
|
505
|
+
* the merge base before isolation meant an isolated run merged the wizard's
|
|
506
|
+
* answers onto the HOST config and reported the host path as the save target
|
|
507
|
+
* while writing somewhere else.
|
|
508
|
+
*/
|
|
509
|
+
function resolveIsolatedSetupConfig(opts) {
|
|
510
|
+
const preIsolationConfig = loadUserConfig();
|
|
511
|
+
// Resolve stash directory early so akmInit can run before any prompts.
|
|
512
|
+
const resolvedStashDir = opts?.dir
|
|
513
|
+
? path.resolve(opts.dir)
|
|
514
|
+
: (primaryBundlePath(preIsolationConfig) ?? getDefaultStashDir());
|
|
506
515
|
// Refuse explicit --dir /tmp/... before doing any work — protects the host
|
|
507
516
|
// config from being clobbered with a stashDir that the OS may reap.
|
|
508
517
|
assertSetupSandbox(resolvedStashDir, opts?.dir != null);
|
|
509
518
|
applyStashIsolationToEnv(resolvedStashDir, opts?.dir != null);
|
|
519
|
+
return { current: loadUserConfig(), configPath: getConfigPath(), resolvedStashDir };
|
|
520
|
+
}
|
|
521
|
+
export async function runSetupWizard(opts) {
|
|
522
|
+
assertSetupConfigPreflight();
|
|
523
|
+
p.intro("akm setup");
|
|
524
|
+
const { current, configPath, resolvedStashDir } = resolveIsolatedSetupConfig(opts);
|
|
510
525
|
// Quick connectivity check — skip network-dependent steps when offline
|
|
511
526
|
const online = await isOnline();
|
|
512
527
|
if (!online) {
|
|
@@ -90,6 +90,16 @@ export function assertNoIgnoredPathOverwrite(repoDir, targetRevision) {
|
|
|
90
90
|
throw new UsageError(`Git update would overwrite ignored local path ${path.join(repoDir, conflict)}; move or remove it before update.`);
|
|
91
91
|
}
|
|
92
92
|
}
|
|
93
|
+
/**
|
|
94
|
+
* Whether a requested ref is a full commit hash rather than a branch or tag.
|
|
95
|
+
*
|
|
96
|
+
* Only the unambiguous 40-hex (SHA-1) and 64-hex (SHA-256) forms count. An
|
|
97
|
+
* abbreviated hash is indistinguishable from a legal branch name, so it stays
|
|
98
|
+
* on the `--branch` path where git itself reports the mismatch.
|
|
99
|
+
*/
|
|
100
|
+
function isCommitSha(ref) {
|
|
101
|
+
return /^[0-9a-f]{40}$/i.test(ref) || /^[0-9a-f]{64}$/i.test(ref);
|
|
102
|
+
}
|
|
93
103
|
function normalizeRemoteUrl(value) {
|
|
94
104
|
return value
|
|
95
105
|
.trim()
|
|
@@ -199,10 +209,17 @@ async function doSyncGit(parsed, options) {
|
|
|
199
209
|
let installRoot;
|
|
200
210
|
let stashRoot;
|
|
201
211
|
try {
|
|
212
|
+
// `git clone --branch` accepts only a branch or tag name, never a raw commit
|
|
213
|
+
// hash — so pinning an install to a commit (`#<40-hex-sha>`, which
|
|
214
|
+
// parseGithubShorthand/parseGitUrl accept and validateGitRef allows) made the
|
|
215
|
+
// clone fail outright. A commit pin needs a full clone followed by a
|
|
216
|
+
// checkout of that revision; `--depth 1` cannot fetch an arbitrary commit
|
|
217
|
+
// either, so the read-only shallow optimization is skipped in that case.
|
|
218
|
+
const pinnedCommit = parsed.requestedRef !== undefined && isCommitSha(parsed.requestedRef);
|
|
202
219
|
const cloneArgs = ["clone"];
|
|
203
|
-
if (!options?.writable)
|
|
220
|
+
if (!options?.writable && !pinnedCommit)
|
|
204
221
|
cloneArgs.push("--depth", "1");
|
|
205
|
-
if (parsed.requestedRef) {
|
|
222
|
+
if (parsed.requestedRef && !pinnedCommit) {
|
|
206
223
|
cloneArgs.push("--branch", parsed.requestedRef);
|
|
207
224
|
}
|
|
208
225
|
cloneArgs.push(parsed.url, cloneDir);
|
|
@@ -210,6 +227,12 @@ async function doSyncGit(parsed, options) {
|
|
|
210
227
|
if (cloneResult.status !== 0) {
|
|
211
228
|
throw new Error(classifyCloneFailure(parsed.url, cloneResult.stderr, cloneResult.error));
|
|
212
229
|
}
|
|
230
|
+
if (pinnedCommit) {
|
|
231
|
+
const checkout = runGit(["-C", cloneDir, "checkout", "--detach", parsed.requestedRef], { timeout: 120_000 });
|
|
232
|
+
if (checkout.status !== 0) {
|
|
233
|
+
throw new Error(`Could not check out commit ${parsed.requestedRef} from ${parsed.url}: ${checkout.stderr.trim() || "unknown git error"}`);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
213
236
|
// R-011: `resolved.resolvedRevision` was resolved via a SEPARATE
|
|
214
237
|
// `git ls-remote` round-trip before this clone ran (resolveGitArtifact /
|
|
215
238
|
// resolveGithubArtifact in registry/resolve.ts) and was never checked
|
|
@@ -217,6 +217,30 @@ function markdownDestination(url) {
|
|
|
217
217
|
* the stack — which would otherwise abort an entire crawl.
|
|
218
218
|
*/
|
|
219
219
|
const MAX_NESTING_DEPTH = 2_000;
|
|
220
|
+
/**
|
|
221
|
+
* HTML5 void elements. They have no closing tag and are conventionally written
|
|
222
|
+
* WITHOUT a trailing slash, so a depth counter that only decrements on `</x>`
|
|
223
|
+
* or `<x/>` treats each one as a permanent +1. The counter then measured total
|
|
224
|
+
* void-element COUNT rather than nesting depth, and an ordinary page with more
|
|
225
|
+
* than MAX_NESTING_DEPTH images or line breaks was misjudged as pathologically
|
|
226
|
+
* nested and degraded to plain text.
|
|
227
|
+
*/
|
|
228
|
+
const VOID_ELEMENTS = new Set([
|
|
229
|
+
"area",
|
|
230
|
+
"base",
|
|
231
|
+
"br",
|
|
232
|
+
"col",
|
|
233
|
+
"embed",
|
|
234
|
+
"hr",
|
|
235
|
+
"img",
|
|
236
|
+
"input",
|
|
237
|
+
"link",
|
|
238
|
+
"meta",
|
|
239
|
+
"param",
|
|
240
|
+
"source",
|
|
241
|
+
"track",
|
|
242
|
+
"wbr",
|
|
243
|
+
]);
|
|
220
244
|
function exceedsNestingBudget(html) {
|
|
221
245
|
let depth = 0;
|
|
222
246
|
let max = 0;
|
|
@@ -226,6 +250,8 @@ function exceedsNestingBudget(html) {
|
|
|
226
250
|
const selfClosing = match[3] === "/";
|
|
227
251
|
if (selfClosing)
|
|
228
252
|
continue;
|
|
253
|
+
if (VOID_ELEMENTS.has(match[2].toLowerCase()))
|
|
254
|
+
continue;
|
|
229
255
|
if (closing)
|
|
230
256
|
depth = Math.max(0, depth - 1);
|
|
231
257
|
else {
|
|
@@ -463,9 +489,45 @@ function escapeResidualMarkup(markdown) {
|
|
|
463
489
|
// Only `<` that begins a tag-like construct; a bare `a < b` stays readable.
|
|
464
490
|
return markdown.replace(/<(?=[a-zA-Z/!?])/g, "<");
|
|
465
491
|
}
|
|
492
|
+
/**
|
|
493
|
+
* Apply {@link escapeResidualMarkup} everywhere EXCEPT inside fenced code
|
|
494
|
+
* blocks.
|
|
495
|
+
*
|
|
496
|
+
* Markup inside a fence is inert — a renderer shows it as text, it cannot
|
|
497
|
+
* execute — so the escape buys no safety there and actively corrupts content.
|
|
498
|
+
* Turndown emits code with entities already decoded, so a documentation page
|
|
499
|
+
* showing `<div>` in a `<pre><code>` block became a fence containing
|
|
500
|
+
* `<div>`, which this rewrote to `<div>`: a half-escaped, wrong-on-both-ends
|
|
501
|
+
* rendering of the example the page exists to show. Every HTML, XML and JSX
|
|
502
|
+
* snippet in a snapshot was affected.
|
|
503
|
+
*/
|
|
504
|
+
function escapeOutsideCodeFences(markdown) {
|
|
505
|
+
const lines = markdown.split("\n");
|
|
506
|
+
let inFence = false;
|
|
507
|
+
let fenceMarker = "";
|
|
508
|
+
for (let i = 0; i < lines.length; i++) {
|
|
509
|
+
const line = lines[i];
|
|
510
|
+
const fence = /^\s*(`{3,}|~{3,})/.exec(line);
|
|
511
|
+
if (fence) {
|
|
512
|
+
const marker = fence[1];
|
|
513
|
+
if (!inFence) {
|
|
514
|
+
inFence = true;
|
|
515
|
+
fenceMarker = marker[0];
|
|
516
|
+
}
|
|
517
|
+
else if (marker[0] === fenceMarker) {
|
|
518
|
+
inFence = false;
|
|
519
|
+
fenceMarker = "";
|
|
520
|
+
}
|
|
521
|
+
continue;
|
|
522
|
+
}
|
|
523
|
+
if (!inFence)
|
|
524
|
+
lines[i] = escapeResidualMarkup(line);
|
|
525
|
+
}
|
|
526
|
+
return lines.join("\n");
|
|
527
|
+
}
|
|
466
528
|
/** Escape residual markup, then normalize whitespace, into the final snapshot. */
|
|
467
529
|
function finalizeMarkdown(markdown) {
|
|
468
|
-
return
|
|
530
|
+
return escapeOutsideCodeFences(markdown)
|
|
469
531
|
.replace(/\r/g, "")
|
|
470
532
|
.replace(/[ \t]+\n/g, "\n")
|
|
471
533
|
.replace(/\n{3,}/g, "\n\n")
|
package/dist/storage/database.js
CHANGED
|
@@ -149,6 +149,48 @@ function loadBunSqlite() {
|
|
|
149
149
|
return bunSqliteModule;
|
|
150
150
|
}
|
|
151
151
|
let betterSqlite3Ctor;
|
|
152
|
+
/**
|
|
153
|
+
* The binding is absent or unbuildable — a toolchain/install problem.
|
|
154
|
+
*/
|
|
155
|
+
const MISSING_BINDING_REMEDY = "akm could not load 'better-sqlite3', the SQLite driver it needs on Node.js.\n" +
|
|
156
|
+
" • Reinstall akm with a working C/C++ build toolchain so its optional\n" +
|
|
157
|
+
" 'better-sqlite3' native binding builds (a global `npm i -g better-sqlite3`\n" +
|
|
158
|
+
" will NOT be resolved — Node loads it from akm's own node_modules).\n" +
|
|
159
|
+
" • Or run akm under Bun, which has a built-in SQLite driver and needs no native build.";
|
|
160
|
+
/**
|
|
161
|
+
* Recognize a binding built for a DIFFERENT Node ABI than the one now running,
|
|
162
|
+
* and answer with the one command that fixes it.
|
|
163
|
+
*
|
|
164
|
+
* This is the most likely failure a real user hits, and it is not a broken
|
|
165
|
+
* install: a native addon is compiled (or a prebuilt binary is selected) for
|
|
166
|
+
* the Node major present at `npm install` time. Upgrade Node afterwards and the
|
|
167
|
+
* same file no longer loads.
|
|
168
|
+
*
|
|
169
|
+
* It is matched rather than described because the symptom text varies and the
|
|
170
|
+
* previous wording only named ONE of them. The prebuilt-binary path — which is
|
|
171
|
+
* now the normal path, since better-sqlite3 is pinned to a version publishing a
|
|
172
|
+
* prebuild for every supported Node (see package.json → pinNotes) — reports
|
|
173
|
+
* `Module did not self-register`, saying nothing about versions at all. A
|
|
174
|
+
* from-source build reports the explicit `NODE_MODULE_VERSION` mismatch. Asking
|
|
175
|
+
* the user to decide which bullet applies is the step worth deleting.
|
|
176
|
+
*/
|
|
177
|
+
export function abiMismatchRemedy(message) {
|
|
178
|
+
const ABI_MISMATCH_SHAPES = [
|
|
179
|
+
"did not self-register", // prebuilt binary for another ABI
|
|
180
|
+
"NODE_MODULE_VERSION", // explicit mismatch, from-source build
|
|
181
|
+
"was compiled against a different", // same, older phrasing
|
|
182
|
+
"invalid ELF header", // binary for another platform/arch entirely
|
|
183
|
+
];
|
|
184
|
+
if (!ABI_MISMATCH_SHAPES.some((shape) => message.includes(shape)))
|
|
185
|
+
return undefined;
|
|
186
|
+
return ("akm could not load 'better-sqlite3': its native binding was built for a different\n" +
|
|
187
|
+
`Node.js version than the one now running (this Node is ABI ${process.versions.modules}).\n` +
|
|
188
|
+
"This is what happens when Node is upgraded after akm is installed. It is NOT a\n" +
|
|
189
|
+
"broken install, and reinstalling akm is not required.\n" +
|
|
190
|
+
" Fix: npm rebuild better-sqlite3 # in akm's install directory\n" +
|
|
191
|
+
" Or: npm install -g akm-cli # reinstall, rebuilding against this Node\n" +
|
|
192
|
+
" Or: run akm under Bun, whose built-in SQLite driver needs no native binding.");
|
|
193
|
+
}
|
|
152
194
|
function loadBetterSqlite3() {
|
|
153
195
|
if (!betterSqlite3Ctor) {
|
|
154
196
|
// Runtime-gated dynamic require: only reached when NOT on Bun, so Bun never
|
|
@@ -161,17 +203,13 @@ function loadBetterSqlite3() {
|
|
|
161
203
|
mod = nodeRequire("better-sqlite3");
|
|
162
204
|
}
|
|
163
205
|
catch (err) {
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
" 'better-sqlite3' native binding rebuilds (a global `npm i -g better-sqlite3`\n" +
|
|
172
|
-
" will NOT be resolved — Node loads it from akm's own node_modules).\n" +
|
|
173
|
-
" • Or run akm under Bun, which has a built-in SQLite driver and needs no native build.\n" +
|
|
174
|
-
` Underlying load error: ${err instanceof Error ? err.message : String(err)}`);
|
|
206
|
+
// An ABI mismatch does NOT arrive here — `require` succeeds and the
|
|
207
|
+
// failure lands at construction (see openNodeDatabase). This path is a
|
|
208
|
+
// genuinely absent or unresolvable module. `abiMismatchRemedy` is still
|
|
209
|
+
// consulted because a from-source build CAN fail at load with the
|
|
210
|
+
// explicit NODE_MODULE_VERSION message.
|
|
211
|
+
const raw = err instanceof Error ? err.message : String(err);
|
|
212
|
+
throw new Error(`${abiMismatchRemedy(raw) ?? MISSING_BINDING_REMEDY}\n Underlying load error: ${raw}`);
|
|
175
213
|
}
|
|
176
214
|
betterSqlite3Ctor = mod.default ?? mod;
|
|
177
215
|
}
|
|
@@ -188,7 +226,24 @@ function openNodeDatabase(path, opts) {
|
|
|
188
226
|
options.readonly = opts.readonly;
|
|
189
227
|
if (opts?.create === false)
|
|
190
228
|
options.fileMustExist = true;
|
|
191
|
-
|
|
229
|
+
// Construction, not `require`, is where an ABI mismatch surfaces.
|
|
230
|
+
// `require("better-sqlite3")` SUCCEEDS against a binding built for another
|
|
231
|
+
// Node ABI — the package resolves its `.node` file lazily — so the loader's
|
|
232
|
+
// catch never sees this error and cannot explain it. Verified against a real
|
|
233
|
+
// ABI-127 binding under Node 24 (ABI 137): `require()` returned a function
|
|
234
|
+
// and `new Database(...)` threw. Wrapping the require alone left the most
|
|
235
|
+
// likely real-world failure reported as a bare Node internals message.
|
|
236
|
+
let db;
|
|
237
|
+
try {
|
|
238
|
+
db = opts ? new BetterSqlite3(path, options) : new BetterSqlite3(path);
|
|
239
|
+
}
|
|
240
|
+
catch (err) {
|
|
241
|
+
const raw = err instanceof Error ? err.message : String(err);
|
|
242
|
+
const remedy = abiMismatchRemedy(raw);
|
|
243
|
+
if (!remedy)
|
|
244
|
+
throw err;
|
|
245
|
+
throw new Error(`${remedy}\n Underlying error: ${raw}`);
|
|
246
|
+
}
|
|
192
247
|
return {
|
|
193
248
|
prepare: db.prepare.bind(db),
|
|
194
249
|
exec: db.exec.bind(db),
|
|
@@ -196,6 +251,10 @@ function openNodeDatabase(path, opts) {
|
|
|
196
251
|
// bun:sqlite also provides db.run(). Normalize the latter at the provider
|
|
197
252
|
// boundary so callers and maintenance wrappers can rely on one contract.
|
|
198
253
|
run: (sql, ...params) => db.prepare(sql).run(...params),
|
|
254
|
+
// sqlite-vec's load(db) calls db.loadExtension(). Without forwarding it the
|
|
255
|
+
// extension could never load on Node, so the vector fast path was dead
|
|
256
|
+
// across the entire npm distribution even when sqlite-vec was installed.
|
|
257
|
+
loadExtension: db.loadExtension.bind(db),
|
|
199
258
|
transaction: db.transaction.bind(db),
|
|
200
259
|
get inTransaction() {
|
|
201
260
|
return db.inTransaction;
|
|
@@ -100,9 +100,68 @@ export function runMigrations(db, migrations, opts) {
|
|
|
100
100
|
if (applied.has(migration.id))
|
|
101
101
|
continue;
|
|
102
102
|
opts?.beforeMigration?.(migration);
|
|
103
|
-
db
|
|
103
|
+
withImmediateWriteLock(db, () => {
|
|
104
|
+
// Re-check under the write lock. `applied` is a snapshot taken before the
|
|
105
|
+
// loop, so two processes bootstrapping the same fresh DB concurrently
|
|
106
|
+
// (both see existed=false, both run with applyPending) could each decide
|
|
107
|
+
// to apply migration N. The first commits; the second must not re-run the
|
|
108
|
+
// DDL and must not hit a UNIQUE violation on the ledger insert.
|
|
109
|
+
const already = db.prepare("SELECT 1 FROM schema_migrations WHERE id = ?").get(migration.id);
|
|
110
|
+
if (already)
|
|
111
|
+
return;
|
|
104
112
|
db.exec(migration.up);
|
|
105
113
|
db.prepare("INSERT INTO schema_migrations (id) VALUES (?)").run(migration.id);
|
|
106
|
-
})
|
|
114
|
+
});
|
|
115
|
+
applied.add(migration.id);
|
|
107
116
|
}
|
|
108
117
|
}
|
|
118
|
+
/** Attempts to acquire the write lock before giving up to the caller. */
|
|
119
|
+
const IMMEDIATE_LOCK_MAX_ATTEMPTS = 5;
|
|
120
|
+
/**
|
|
121
|
+
* Run `fn` inside a `BEGIN IMMEDIATE` transaction.
|
|
122
|
+
*
|
|
123
|
+
* The write lock is taken up front rather than upgraded from a read lock, so a
|
|
124
|
+
* second process bootstrapping the same database WAITS for the first to commit
|
|
125
|
+
* instead of racing it. `db.transaction()` opens a DEFERRED transaction, which
|
|
126
|
+
* only takes the write lock on first write — leaving the read-then-write gap
|
|
127
|
+
* this guards.
|
|
128
|
+
*
|
|
129
|
+
* Deliberately local rather than reusing `withImmediateTransaction` from
|
|
130
|
+
* core/state-db: that module imports this one, so the dependency cannot be
|
|
131
|
+
* pointed the other way.
|
|
132
|
+
*/
|
|
133
|
+
function withImmediateWriteLock(db, fn) {
|
|
134
|
+
if (db.inTransaction) {
|
|
135
|
+
fn();
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
138
|
+
let lastBeginErr;
|
|
139
|
+
for (let attempt = 1; attempt <= IMMEDIATE_LOCK_MAX_ATTEMPTS; attempt++) {
|
|
140
|
+
try {
|
|
141
|
+
db.exec("BEGIN IMMEDIATE");
|
|
142
|
+
}
|
|
143
|
+
catch (err) {
|
|
144
|
+
// Busy despite busy_timeout (another writer holding it across the whole
|
|
145
|
+
// window). Retry a bounded number of times before surfacing.
|
|
146
|
+
lastBeginErr = err;
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
149
|
+
try {
|
|
150
|
+
fn();
|
|
151
|
+
db.exec("COMMIT");
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
catch (err) {
|
|
155
|
+
try {
|
|
156
|
+
db.exec("ROLLBACK");
|
|
157
|
+
}
|
|
158
|
+
catch {
|
|
159
|
+
// Already rolled back by SQLite (e.g. the statement aborted the txn).
|
|
160
|
+
}
|
|
161
|
+
throw err;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
throw lastBeginErr instanceof Error
|
|
165
|
+
? lastBeginErr
|
|
166
|
+
: new Error(`could not acquire the migration write lock after ${IMMEDIATE_LOCK_MAX_ATTEMPTS} attempts`);
|
|
167
|
+
}
|
|
@@ -16,6 +16,7 @@ import { classifyPathAccess, describeInaccessiblePath } from "../../core/path-ac
|
|
|
16
16
|
import { getDbPath } from "../../core/paths.js";
|
|
17
17
|
import { openDatabase } from "../database.js";
|
|
18
18
|
import { openManagedDatabase } from "../managed-db.js";
|
|
19
|
+
import { SQLITE_BUSY_TIMEOUT_MS } from "../sqlite-pragmas.js";
|
|
19
20
|
import { ensureSchema } from "./index-schema.js";
|
|
20
21
|
import { loadVecExtension, warnIfVecMissing } from "./index-vec-repository.js";
|
|
21
22
|
export function openIndexDatabase(dbPath, options) {
|
|
@@ -109,7 +110,16 @@ export function openReadonlyExistingDatabase(dbPath) {
|
|
|
109
110
|
assertIndexPathReadable(resolvedPath);
|
|
110
111
|
if (classifyPathAccess(resolvedPath).access === "absent")
|
|
111
112
|
return undefined;
|
|
112
|
-
|
|
113
|
+
const db = openDatabase(resolvedPath, { readonly: true, create: false });
|
|
114
|
+
// This opener bypasses openManagedDatabase/applyStandardPragmas by design (no
|
|
115
|
+
// journal or schema work on a read-only handle), but that also left
|
|
116
|
+
// busy_timeout at SQLite's default of 0. In WAL that is harmless — readers
|
|
117
|
+
// never block — but in the DELETE/TRUNCATE modes the network-FS fallback and
|
|
118
|
+
// AKM_SQLITE_JOURNAL_MODE can select, a concurrent writer makes every read
|
|
119
|
+
// fail instantly with SQLITE_BUSY. busy_timeout is legal on a read-only
|
|
120
|
+
// connection, so apply just that one.
|
|
121
|
+
db.exec(`PRAGMA busy_timeout = ${SQLITE_BUSY_TIMEOUT_MS}`);
|
|
122
|
+
return db;
|
|
113
123
|
}
|
|
114
124
|
export function closeDatabase(db) {
|
|
115
125
|
db.close();
|
|
@@ -17,6 +17,17 @@ export function getMeta(db, key) {
|
|
|
17
17
|
export function setMeta(db, key, value) {
|
|
18
18
|
db.prepare("INSERT OR REPLACE INTO index_meta (key, value) VALUES (?, ?)").run(key, value);
|
|
19
19
|
}
|
|
20
|
+
/**
|
|
21
|
+
* Remove a meta key entirely.
|
|
22
|
+
*
|
|
23
|
+
* Distinct from writing an empty string: absence is what callers test for
|
|
24
|
+
* (`getMeta(...) === undefined`), and it is what lets a value be re-derived —
|
|
25
|
+
* clearing `embeddingDim` after a model change is how the vec table gets
|
|
26
|
+
* rebuilt at the new width.
|
|
27
|
+
*/
|
|
28
|
+
export function deleteMeta(db, key) {
|
|
29
|
+
db.prepare("DELETE FROM index_meta WHERE key = ?").run(key);
|
|
30
|
+
}
|
|
20
31
|
// ── Per-directory index state ───────────────────────────────────────────────
|
|
21
32
|
export function getIndexDirState(db, dirPath) {
|
|
22
33
|
const row = db
|
|
@@ -533,8 +533,23 @@ function ensureBundleRefColumns(db) {
|
|
|
533
533
|
* fallback.
|
|
534
534
|
*/
|
|
535
535
|
function ensureUniqueItemRefIndex(db) {
|
|
536
|
-
|
|
537
|
-
|
|
536
|
+
// Probe before mutating. This ran unconditionally on EVERY open as two
|
|
537
|
+
// separate autocommit statements, so there was always a window in which the
|
|
538
|
+
// index did not exist — a concurrent open (registry-cache search, indexer,
|
|
539
|
+
// improve) could DROP between the other's DROP and CREATE and then fail with
|
|
540
|
+
// "index idx_entries_item_ref already exists", or serve a query with no index
|
|
541
|
+
// at all. A DB whose index is already UNIQUE needs no work.
|
|
542
|
+
const existing = db
|
|
543
|
+
.prepare("SELECT sql FROM sqlite_master WHERE type = 'index' AND name = 'idx_entries_item_ref'")
|
|
544
|
+
.get();
|
|
545
|
+
if (existing?.sql && /\bUNIQUE\b/i.test(existing.sql))
|
|
546
|
+
return;
|
|
547
|
+
// Pre-v19 (non-unique index) or absent: convert atomically so a racing open
|
|
548
|
+
// sees either the old index or the new one, never neither.
|
|
549
|
+
db.transaction(() => {
|
|
550
|
+
db.exec("DROP INDEX IF EXISTS idx_entries_item_ref");
|
|
551
|
+
db.exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_entries_item_ref ON entries(item_ref)");
|
|
552
|
+
})();
|
|
538
553
|
}
|
|
539
554
|
/**
|
|
540
555
|
* Returns true when a table exists in the current database.
|
|
@@ -22,10 +22,10 @@ export function loadVecExtension(db) {
|
|
|
22
22
|
try {
|
|
23
23
|
const esmRequire = createRequire(import.meta.url);
|
|
24
24
|
const sqliteVec = esmRequire("sqlite-vec");
|
|
25
|
-
// `db`
|
|
26
|
-
//
|
|
27
|
-
//
|
|
28
|
-
// so
|
|
25
|
+
// `db` is the storage boundary's handle. On Bun that IS the bun:sqlite
|
|
26
|
+
// handle; on Node it is a wrapper, which must forward `loadExtension` for
|
|
27
|
+
// this call to work at all (see openNodeDatabase in storage/database.ts —
|
|
28
|
+
// it did not, so vec could never load on the entire npm distribution).
|
|
29
29
|
sqliteVec.load(db);
|
|
30
30
|
vecStatus.set(db, true);
|
|
31
31
|
}
|
|
@@ -56,7 +56,45 @@ export function setVecFastPathReady(db, ready) {
|
|
|
56
56
|
* rather than silently returning partial fast-path results.
|
|
57
57
|
*/
|
|
58
58
|
export function isVecFastPathReady(db) {
|
|
59
|
-
|
|
59
|
+
if (getMeta(db, VEC_FAST_PATH_READY_META) === "0")
|
|
60
|
+
return false;
|
|
61
|
+
// The meta flag alone is not sufficient. An index built while sqlite-vec was
|
|
62
|
+
// unavailable wrote only BLOB rows, and because "unavailable" outcomes were
|
|
63
|
+
// not counted as failures the flag was still set to "1" against a table that
|
|
64
|
+
// is empty or absent. If sqlite-vec later becomes loadable — the user installs
|
|
65
|
+
// it, or the same index is opened under the other runtime — the fast path
|
|
66
|
+
// would then be trusted and return zero neighbours while the BLOB table holds
|
|
67
|
+
// every embedding. Indexes written by earlier versions still carry that stale
|
|
68
|
+
// flag, so the read path has to verify the table really exists.
|
|
69
|
+
return hasVecTable(db);
|
|
70
|
+
}
|
|
71
|
+
const vecTablePresent = new WeakMap();
|
|
72
|
+
/**
|
|
73
|
+
* Whether `entries_vec` exists on this connection, memoized per handle.
|
|
74
|
+
*
|
|
75
|
+
* openExistingDatabase loads the vec extension but deliberately does not run
|
|
76
|
+
* ensureSchema, so the table is not created on read paths — its absence is a
|
|
77
|
+
* normal state, not an error.
|
|
78
|
+
*/
|
|
79
|
+
function hasVecTable(db) {
|
|
80
|
+
// Only a POSITIVE result is memoized. The table cannot vanish from a live
|
|
81
|
+
// connection, but it CAN appear — ensureSchema creates it partway through an
|
|
82
|
+
// index run — so caching "absent" would pin a stale answer for the rest of
|
|
83
|
+
// the handle's life.
|
|
84
|
+
if (vecTablePresent.get(db) === true)
|
|
85
|
+
return true;
|
|
86
|
+
let present = false;
|
|
87
|
+
try {
|
|
88
|
+
present =
|
|
89
|
+
db.prepare("SELECT 1 FROM sqlite_master WHERE type IN ('table','view') AND name = 'entries_vec'").get() !==
|
|
90
|
+
undefined;
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
present = false;
|
|
94
|
+
}
|
|
95
|
+
if (present)
|
|
96
|
+
vecTablePresent.set(db, true);
|
|
97
|
+
return present;
|
|
60
98
|
}
|
|
61
99
|
/** Remove both vector representations for an entry whose embedding input changed. */
|
|
62
100
|
export function deleteEntryVectors(db, id) {
|
|
@@ -95,6 +95,17 @@ export function isNetworkFilesystem(fsType) {
|
|
|
95
95
|
return false;
|
|
96
96
|
return NETWORK_FS_MAGICS.has(fsType);
|
|
97
97
|
}
|
|
98
|
+
/** Options for {@link applyStandardPragmas}. */
|
|
99
|
+
/**
|
|
100
|
+
* How long a statement waits for a lock before failing with SQLITE_BUSY.
|
|
101
|
+
*
|
|
102
|
+
* Exported so read-only openers can apply it too. They cannot run the rest of
|
|
103
|
+
* the standard set (journal_mode and foreign_keys are write operations), but
|
|
104
|
+
* the default of 0 makes reads fail INSTANTLY under writer contention — which
|
|
105
|
+
* matters in the DELETE/TRUNCATE journal modes 0.9.1's network-filesystem
|
|
106
|
+
* fallback and `AKM_SQLITE_JOURNAL_MODE` can select, where readers do block.
|
|
107
|
+
*/
|
|
108
|
+
export const SQLITE_BUSY_TIMEOUT_MS = 30_000;
|
|
98
109
|
/**
|
|
99
110
|
* Apply AKM's standard opening PRAGMAs to `db`, in order:
|
|
100
111
|
* 1. `journal_mode` = the configured mode (with WAL→DELETE network-FS fallback)
|
|
@@ -128,7 +139,7 @@ export function applyStandardPragmas(db, opts = {}) {
|
|
|
128
139
|
// lock instead of failing immediately with SQLITE_BUSY. For the WAL default
|
|
129
140
|
// this is a no-op (WAL→WAL changes nothing), so byte-identical behaviour is
|
|
130
141
|
// preserved.
|
|
131
|
-
db.exec(
|
|
142
|
+
db.exec(`PRAGMA busy_timeout = ${SQLITE_BUSY_TIMEOUT_MS}`);
|
|
132
143
|
db.exec(`PRAGMA journal_mode = ${mode}`);
|
|
133
144
|
if (opts.foreignKeys !== false) {
|
|
134
145
|
db.exec("PRAGMA foreign_keys = ON");
|