akm-cli 0.9.0-beta.45 → 0.9.0-beta.47

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/dist/assets/wiki/ingest-workflow-template.md +17 -10
  2. package/dist/cli/shared.js +28 -0
  3. package/dist/cli.js +1 -2
  4. package/dist/commands/env/env-cli.js +16 -24
  5. package/dist/commands/env/secret-cli.js +12 -20
  6. package/dist/commands/graph/graph-cli.js +5 -13
  7. package/dist/commands/graph/graph.js +3 -3
  8. package/dist/commands/improve/consolidate/chunking.js +141 -0
  9. package/dist/commands/improve/consolidate/eligibility.js +64 -0
  10. package/dist/commands/improve/consolidate/merge.js +145 -0
  11. package/dist/commands/improve/consolidate/sanitize.js +231 -0
  12. package/dist/commands/improve/consolidate/types.js +4 -0
  13. package/dist/commands/improve/consolidate.js +20 -571
  14. package/dist/commands/improve/distill.js +5 -9
  15. package/dist/commands/improve/eligibility.js +434 -0
  16. package/dist/commands/improve/extract-cli.js +9 -1
  17. package/dist/commands/improve/extract.js +5 -19
  18. package/dist/commands/improve/improve-auto-accept.js +4 -8
  19. package/dist/commands/improve/improve-cli.js +35 -60
  20. package/dist/commands/improve/improve-result-file.js +5 -23
  21. package/dist/commands/improve/improve-session.js +58 -0
  22. package/dist/commands/improve/improve.js +107 -3606
  23. package/dist/commands/improve/locks.js +154 -0
  24. package/dist/commands/improve/loop-stages.js +1079 -0
  25. package/dist/commands/improve/preparation.js +1963 -0
  26. package/dist/commands/improve/recombine.js +6 -12
  27. package/dist/commands/improve/reflect.js +29 -34
  28. package/dist/commands/proposal/drain.js +25 -48
  29. package/dist/commands/proposal/proposal-cli.js +21 -31
  30. package/dist/commands/proposal/validators/proposals.js +3 -7
  31. package/dist/commands/read/curate.js +70 -14
  32. package/dist/commands/read/knowledge.js +2 -2
  33. package/dist/commands/sources/self-update.js +2 -2
  34. package/dist/commands/sources/stash-cli.js +9 -37
  35. package/dist/commands/tasks/tasks-cli.js +19 -27
  36. package/dist/commands/wiki-cli.js +21 -35
  37. package/dist/core/config/config.js +18 -2
  38. package/dist/core/events.js +3 -7
  39. package/dist/core/logs-db.js +6 -63
  40. package/dist/core/state/migrations.js +714 -0
  41. package/dist/core/state-db.js +28 -779
  42. package/dist/indexer/db/db.js +82 -216
  43. package/dist/indexer/indexer.js +11 -112
  44. package/dist/indexer/passes/dir-staleness.js +114 -0
  45. package/dist/indexer/search/search-source.js +10 -24
  46. package/dist/indexer/search/semantic-status.js +4 -0
  47. package/dist/integrations/agent/runner-dispatch.js +59 -0
  48. package/dist/llm/client.js +22 -11
  49. package/dist/llm/embedder.js +15 -0
  50. package/dist/llm/embedders/deterministic.js +66 -0
  51. package/dist/llm/graph-extract.js +28 -39
  52. package/dist/llm/memory-infer.js +34 -22
  53. package/dist/llm/metadata-enhance.js +35 -30
  54. package/dist/llm/structured-call.js +49 -0
  55. package/dist/output/shapes/passthrough.js +0 -1
  56. package/dist/registry/providers/skills-sh.js +21 -147
  57. package/dist/registry/providers/static-index.js +15 -157
  58. package/dist/registry/resolve.js +22 -9
  59. package/dist/scripts/migrate-storage.js +892 -1186
  60. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +214 -179
  61. package/dist/setup/setup.js +26 -5
  62. package/dist/sources/providers/filesystem.js +0 -1
  63. package/dist/sources/providers/git-install.js +206 -0
  64. package/dist/sources/providers/git-provider.js +234 -0
  65. package/dist/sources/providers/git-stash.js +248 -0
  66. package/dist/sources/providers/git.js +10 -671
  67. package/dist/sources/providers/npm.js +2 -6
  68. package/dist/sources/providers/sync-from-ref.js +9 -1
  69. package/dist/sources/providers/website.js +2 -3
  70. package/dist/sources/website-ingest.js +51 -9
  71. package/dist/sources/wiki-fetchers/registry.js +53 -0
  72. package/dist/sources/wiki-fetchers/youtube.js +185 -0
  73. package/dist/storage/database.js +45 -10
  74. package/dist/storage/managed-db.js +82 -0
  75. package/dist/storage/repositories/registry-cache.js +92 -0
  76. package/dist/tasks/runner.js +5 -13
  77. package/dist/workflows/runtime/runs.js +1 -117
  78. package/dist/workflows/runtime/workflow-asset-loader.js +125 -0
  79. package/package.json +5 -5
  80. package/dist/commands/db-cli.js +0 -23
  81. package/dist/indexer/db/db-backup.js +0 -376
@@ -36,18 +36,14 @@ class NpmSourceProvider {
36
36
  this.#config = config;
37
37
  this.name = config.name ?? config.url ?? "npm";
38
38
  }
39
- async init(_ctx) {
40
- // Resolution happens lazily in path(): until `sync()` runs there's no
41
- // reliable on-disk path. Init is the registration handshake.
42
- }
43
39
  path() {
44
40
  if (this.#config.path)
45
41
  return this.#config.path;
46
42
  throw new ConfigError(`npm source "${this.name}" has no resolved content path — run \`akm update\` to sync it before indexing.`);
47
43
  }
48
- async sync() {
44
+ async sync(options) {
49
45
  const ref = npmRefFromConfig(this.#config);
50
- await syncNpmRef(ref);
46
+ await syncNpmRef(ref, { force: options?.force });
51
47
  }
52
48
  }
53
49
  registerSourceProvider("npm", (config) => new NpmSourceProvider(config));
@@ -1,6 +1,13 @@
1
1
  // This Source Code Form is subject to the terms of the Mozilla Public
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ /**
5
+ * Unified install-ref dispatcher.
6
+ *
7
+ * Replaces the historical `installRegistryRef()` entry point. Given an
8
+ * unparsed install ref, this resolves the right syncable provider and
9
+ * invokes its `sync()` method.
10
+ */
4
11
  import { UsageError } from "../../core/errors.js";
5
12
  import { parseRegistryRef } from "../../registry/resolve.js";
6
13
  import { detectStashRoot } from "./provider-utils.js";
@@ -18,7 +25,8 @@ export async function syncFromRef(ref, options) {
18
25
  return syncRegistryGitRef(ref, options);
19
26
  }
20
27
  // Exhaustiveness — `parseRegistryRef` only emits the four sources above.
21
- throw new UsageError(`No syncable provider for ref: ${ref} (source=${parsed.source})`);
28
+ const _exhaustive = parsed;
29
+ throw new UsageError(`No syncable provider for ref: ${ref}`);
22
30
  }
23
31
  function syncLocalRef(parsed, options) {
24
32
  const stashRoot = detectStashRoot(parsed.sourcePath);
@@ -12,12 +12,11 @@ registerSourceProvider("website", (config) => {
12
12
  return {
13
13
  kind: "website",
14
14
  name,
15
- async init(_ctx) { },
16
15
  path() {
17
16
  return getWebsiteCachePaths(url).stashDir;
18
17
  },
19
- async sync() {
20
- await ensureWebsiteMirror(config, { requireStashDir: true });
18
+ async sync(options) {
19
+ await ensureWebsiteMirror(config, { requireStashDir: true, force: options?.force });
21
20
  },
22
21
  };
23
22
  });
@@ -4,11 +4,12 @@
4
4
  import { createHash } from "node:crypto";
5
5
  import fs from "node:fs";
6
6
  import path from "node:path";
7
- import { fetchWithRetry, ResponseTooLargeError, readBodyWithByteCap } from "../core/common.js";
7
+ import { fetchWithRetry, ResponseTooLargeError, readBodyWithByteCap, resolveStashDir } from "../core/common.js";
8
8
  import { ConfigError, UsageError } from "../core/errors.js";
9
9
  import { getRegistryIndexCacheDir } from "../core/paths.js";
10
10
  import { warn } from "../core/warn.js";
11
11
  import { isExpired, sanitizeString } from "./providers/provider-utils.js";
12
+ import { loadWikiSnapshotFetchers } from "./wiki-fetchers/registry.js";
12
13
  /** Refresh website snapshots every 12 hours to balance freshness with scraping load. */
13
14
  const CACHE_REFRESH_INTERVAL_MS = 12 * 60 * 60 * 1000;
14
15
  /** Allow up to 7 days of stale snapshots when refresh fails so search remains available during outages. */
@@ -30,6 +31,16 @@ const WEBSITE_PAGE_BYTE_CAP = 5 * 1024 * 1024;
30
31
  * whole crawl and return what we have when time runs out.
31
32
  */
32
33
  const WEBSITE_CRAWL_WALL_CLOCK_MS = 10 * 60 * 1000;
34
+ function resolveFetcherStashDir(explicitStashDir) {
35
+ if (explicitStashDir)
36
+ return explicitStashDir;
37
+ try {
38
+ return resolveStashDir({ readOnly: true });
39
+ }
40
+ catch {
41
+ return null;
42
+ }
43
+ }
33
44
  export function getWebsiteCachePaths(siteUrl) {
34
45
  const key = createHash("sha256").update(normalizeSiteUrl(siteUrl)).digest("hex").slice(0, 16);
35
46
  const rootDir = path.join(getRegistryIndexCacheDir(), `website-${key}`);
@@ -114,20 +125,51 @@ async function scrapeWebsiteToStash(startUrl, stashDir, options) {
114
125
  fs.writeFileSync(filePath, buildMarkdownSnapshot(page, slug), "utf8");
115
126
  }
116
127
  }
117
- export async function fetchWebsiteMarkdownSnapshot(rawUrl) {
128
+ export async function fetchWebsiteMarkdownSnapshot(rawUrl, options) {
118
129
  const normalizedUrl = validateWebsiteInputUrl(rawUrl);
130
+ const parsedUrl = new URL(normalizedUrl);
131
+ const stashDir = resolveFetcherStashDir(options?.stashDir);
132
+ const context = {
133
+ stashDir: stashDir ?? "",
134
+ timeoutMs: options?.timeoutMs ?? 15_000,
135
+ signal: options?.signal,
136
+ };
137
+ for (const fetcher of await loadWikiSnapshotFetchers(stashDir)) {
138
+ try {
139
+ if (!fetcher.matches(parsedUrl, context))
140
+ continue;
141
+ const snapshot = await fetcher.fetch(parsedUrl, context);
142
+ if (!snapshot)
143
+ continue;
144
+ return websiteMarkdownSnapshotFromResult(snapshot);
145
+ }
146
+ catch (error) {
147
+ warn("[akm] wiki-fetcher %s threw on %s: %s", fetcher.name, normalizedUrl, error instanceof Error ? error.message : String(error));
148
+ }
149
+ }
119
150
  const fetched = await fetchWebsitePage(normalizedUrl);
120
151
  if (!fetched) {
121
152
  throw new UsageError(`No content could be fetched from ${normalizedUrl}`);
122
153
  }
123
- const preferredName = deriveImportPath(fetched.page.url);
124
- const slug = preferredName.split("/").pop() ?? preferredName;
125
- return {
154
+ return websiteMarkdownSnapshotFromResult({
126
155
  url: fetched.page.url,
127
156
  title: fetched.page.title,
128
157
  markdown: fetched.page.markdown,
158
+ });
159
+ }
160
+ function websiteMarkdownSnapshotFromResult(snapshot) {
161
+ const preferredName = snapshot.preferredName ?? deriveImportPath(snapshot.url);
162
+ const slug = preferredName.split("/").pop() ?? preferredName;
163
+ return {
164
+ url: snapshot.url,
165
+ title: snapshot.title,
166
+ markdown: snapshot.markdown,
129
167
  preferredName,
130
- content: buildMarkdownSnapshot(fetched.page, slug || "website"),
168
+ content: buildMarkdownSnapshot({
169
+ url: snapshot.url,
170
+ title: snapshot.title,
171
+ markdown: snapshot.markdown,
172
+ }, slug || "website", snapshot.tags),
131
173
  };
132
174
  }
133
175
  async function crawlWebsite(startUrl, options) {
@@ -216,11 +258,12 @@ async function fetchWebsitePage(pageUrl) {
216
258
  links: [],
217
259
  };
218
260
  }
219
- function buildMarkdownSnapshot(page, slug) {
261
+ function buildMarkdownSnapshot(page, slug, tags) {
220
262
  const title = sanitizeString(page.title, 200) || slug;
221
263
  const description = sanitizeString(`Snapshot of ${page.url}`, 500);
222
264
  const host = sanitizeString(new URL(page.url).hostname, 120);
223
265
  const content = page.markdown.trim() || `Source: ${page.url}`;
266
+ const normalizedTags = Array.from(new Set(["website", host, ...(tags ?? [])]));
224
267
  return [
225
268
  "---",
226
269
  `name: ${JSON.stringify(slug)}`,
@@ -228,8 +271,7 @@ function buildMarkdownSnapshot(page, slug) {
228
271
  `sourceUrl: ${JSON.stringify(page.url)}`,
229
272
  `title: ${JSON.stringify(title)}`,
230
273
  "tags:",
231
- ` - ${JSON.stringify("website")}`,
232
- ` - ${JSON.stringify(host)}`,
274
+ ...normalizedTags.map((tag) => ` - ${JSON.stringify(tag)}`),
233
275
  "---",
234
276
  "",
235
277
  `# ${title}`,
@@ -0,0 +1,53 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ import fs from "node:fs";
5
+ import path from "node:path";
6
+ import { pathToFileURL } from "node:url";
7
+ import { warn } from "../../core/warn.js";
8
+ import youtubeFetcher from "./youtube.js";
9
+ const FETCHER_DIR = path.join("scripts", "wiki-fetchers");
10
+ const FETCHER_FILE_PATTERN = /\.(?:ts|js|mjs)$/i;
11
+ const BUILTIN_FETCHERS = [youtubeFetcher];
12
+ function isWikiSnapshotFetcher(value) {
13
+ if (!value || typeof value !== "object")
14
+ return false;
15
+ const candidate = value;
16
+ return (typeof candidate.name === "string" &&
17
+ typeof candidate.matches === "function" &&
18
+ typeof candidate.fetch === "function");
19
+ }
20
+ export async function loadWikiSnapshotFetchers(stashDir) {
21
+ const fetchers = [];
22
+ if (stashDir) {
23
+ const fetcherDir = path.join(stashDir, FETCHER_DIR);
24
+ try {
25
+ const entries = fs.readdirSync(fetcherDir).sort();
26
+ for (const entry of entries) {
27
+ if (!FETCHER_FILE_PATTERN.test(entry))
28
+ continue;
29
+ try {
30
+ const fileUrl = pathToFileURL(path.join(fetcherDir, entry)).toString();
31
+ const mod = await import(fileUrl);
32
+ if (isWikiSnapshotFetcher(mod.default)) {
33
+ fetchers.push(mod.default);
34
+ }
35
+ else {
36
+ warn("[akm] wiki-fetcher %s skipped: missing { name, matches, fetch }", entry);
37
+ }
38
+ }
39
+ catch (error) {
40
+ warn("[akm] wiki-fetcher %s failed to load: %s", entry, error instanceof Error ? error.message : String(error));
41
+ }
42
+ }
43
+ }
44
+ catch (error) {
45
+ const code = typeof error === "object" && error && "code" in error ? error.code : undefined;
46
+ if (code !== "ENOENT") {
47
+ warn("[akm] wiki-fetcher directory %s could not be read: %s", fetcherDir, error instanceof Error ? error.message : String(error));
48
+ }
49
+ // Missing directory means no custom fetchers.
50
+ }
51
+ }
52
+ return [...fetchers, ...BUILTIN_FETCHERS];
53
+ }
@@ -0,0 +1,185 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ import { fetchWithRetry } from "../../core/common.js";
5
+ const YOUTUBE_HOSTS = new Set(["www.youtube.com", "youtube.com", "m.youtube.com", "youtu.be"]);
6
+ const WATCH_HOST = "https://www.youtube.com/watch?v=";
7
+ function extractVideoId(url) {
8
+ if (!YOUTUBE_HOSTS.has(url.hostname))
9
+ return null;
10
+ if (url.hostname === "youtu.be") {
11
+ const id = url.pathname.replace(/^\/+/, "").split("/")[0]?.trim();
12
+ return id || null;
13
+ }
14
+ if (url.pathname === "/watch") {
15
+ const id = url.searchParams.get("v")?.trim();
16
+ return id || null;
17
+ }
18
+ if (url.pathname.startsWith("/shorts/")) {
19
+ const id = url.pathname.slice("/shorts/".length).split("/")[0]?.trim();
20
+ return id || null;
21
+ }
22
+ if (url.pathname.startsWith("/embed/")) {
23
+ const id = url.pathname.slice("/embed/".length).split("/")[0]?.trim();
24
+ return id || null;
25
+ }
26
+ return null;
27
+ }
28
+ function canonicalWatchUrl(videoId) {
29
+ return `${WATCH_HOST}${encodeURIComponent(videoId)}`;
30
+ }
31
+ function decodeEntities(value) {
32
+ const safeFromCodePoint = (codePoint, fallback) => {
33
+ if (!Number.isFinite(codePoint) || codePoint < 0 || codePoint > 0x10ffff)
34
+ return fallback;
35
+ try {
36
+ return String.fromCodePoint(codePoint);
37
+ }
38
+ catch {
39
+ return fallback;
40
+ }
41
+ };
42
+ return value
43
+ .replace(/&#x([0-9a-f]+);/gi, (m, hex) => safeFromCodePoint(Number.parseInt(hex, 16), m))
44
+ .replace(/&#([0-9]+);/g, (m, dec) => safeFromCodePoint(Number.parseInt(dec, 10), m))
45
+ .replace(/&quot;/g, '"')
46
+ .replace(/&#39;|&apos;/g, "'")
47
+ .replace(/&lt;/g, "<")
48
+ .replace(/&gt;/g, ">")
49
+ .replace(/&amp;/g, "&");
50
+ }
51
+ function stripTags(value) {
52
+ return value.replace(/<[^>]+>/g, "");
53
+ }
54
+ function extractJsonObjectAfter(marker, source) {
55
+ const markerIndex = source.indexOf(marker);
56
+ if (markerIndex === -1)
57
+ return null;
58
+ const start = source.indexOf("{", markerIndex + marker.length);
59
+ if (start === -1)
60
+ return null;
61
+ let depth = 0;
62
+ let inString = false;
63
+ let escaped = false;
64
+ for (let i = start; i < source.length; i += 1) {
65
+ const char = source[i];
66
+ if (inString) {
67
+ if (escaped) {
68
+ escaped = false;
69
+ }
70
+ else if (char === "\\") {
71
+ escaped = true;
72
+ }
73
+ else if (char === '"') {
74
+ inString = false;
75
+ }
76
+ continue;
77
+ }
78
+ if (char === '"') {
79
+ inString = true;
80
+ continue;
81
+ }
82
+ if (char === "{")
83
+ depth += 1;
84
+ if (char === "}") {
85
+ depth -= 1;
86
+ if (depth === 0)
87
+ return source.slice(start, i + 1);
88
+ }
89
+ }
90
+ return null;
91
+ }
92
+ function parsePlayerResponse(html) {
93
+ const jsonText = extractJsonObjectAfter("ytInitialPlayerResponse =", html);
94
+ if (!jsonText)
95
+ return null;
96
+ try {
97
+ return JSON.parse(jsonText);
98
+ }
99
+ catch {
100
+ return null;
101
+ }
102
+ }
103
+ function chooseCaptionTrack(tracks) {
104
+ const normalized = tracks
105
+ .map((track) => ({
106
+ baseUrl: typeof track.baseUrl === "string" ? track.baseUrl : "",
107
+ languageCode: typeof track.languageCode === "string" ? track.languageCode : "",
108
+ kind: typeof track.kind === "string" ? track.kind : "",
109
+ }))
110
+ .filter((track) => track.baseUrl);
111
+ if (normalized.length === 0)
112
+ return null;
113
+ const preferred = normalized.find((track) => track.languageCode === "en" && track.kind !== "asr") ??
114
+ normalized.find((track) => track.languageCode.startsWith("en") && track.kind !== "asr") ??
115
+ normalized.find((track) => track.languageCode.startsWith("en")) ??
116
+ normalized[0];
117
+ return preferred.baseUrl;
118
+ }
119
+ async function fetchText(url, timeoutMs, signal) {
120
+ const response = await fetchWithRetry(url, {
121
+ headers: {
122
+ Accept: "text/html,application/json,application/xml,text/xml,text/plain;q=0.9,*/*;q=0.1",
123
+ "User-Agent": "akm-cli youtube fetcher",
124
+ },
125
+ signal,
126
+ }, { timeout: timeoutMs, retries: 1 });
127
+ if (!response.ok)
128
+ return null;
129
+ return response.text();
130
+ }
131
+ function parseTranscript(xml) {
132
+ const texts = [...xml.matchAll(/<text\b[^>]*>([\s\S]*?)<\/text>/g)]
133
+ .map((match) => decodeEntities(stripTags(match[1] ?? "")).trim())
134
+ .filter(Boolean);
135
+ if (texts.length === 0)
136
+ return null;
137
+ return texts.join("\n");
138
+ }
139
+ const youtubeFetcher = {
140
+ name: "youtube-transcript",
141
+ matches(url) {
142
+ return extractVideoId(url) !== null;
143
+ },
144
+ async fetch(url, context) {
145
+ const videoId = extractVideoId(url);
146
+ if (!videoId)
147
+ return null;
148
+ const watchUrl = canonicalWatchUrl(videoId);
149
+ const watchHtml = await fetchText(watchUrl, context.timeoutMs, context.signal);
150
+ if (!watchHtml)
151
+ return null;
152
+ const playerResponse = parsePlayerResponse(watchHtml);
153
+ if (!playerResponse)
154
+ return null;
155
+ const title = typeof playerResponse.videoDetails?.title === "string" ? playerResponse.videoDetails.title.trim() : "";
156
+ const description = typeof playerResponse.videoDetails?.shortDescription === "string"
157
+ ? playerResponse.videoDetails.shortDescription.trim()
158
+ : "";
159
+ const sections = [];
160
+ let hasTranscript = false;
161
+ if (description) {
162
+ sections.push("## Description", "", description);
163
+ }
164
+ const tracks = playerResponse.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
165
+ const captionUrl = chooseCaptionTrack(tracks);
166
+ if (captionUrl) {
167
+ const transcriptXml = await fetchText(captionUrl, context.timeoutMs, context.signal);
168
+ const transcript = transcriptXml ? parseTranscript(transcriptXml) : null;
169
+ if (transcript) {
170
+ hasTranscript = true;
171
+ sections.push("## Transcript", "", transcript);
172
+ }
173
+ }
174
+ if (sections.length === 0)
175
+ return null;
176
+ return {
177
+ url: watchUrl,
178
+ title: title || `YouTube ${videoId}`,
179
+ markdown: sections.join("\n"),
180
+ preferredName: `youtube/${videoId}`,
181
+ tags: hasTranscript ? ["youtube", "video", "transcript"] : ["youtube", "video"],
182
+ };
183
+ },
184
+ };
185
+ export default youtubeFetcher;
@@ -35,18 +35,39 @@ const isBun = !!process.versions?.bun;
35
35
  // Bun built-in, unresolvable on Node) nor `better-sqlite3` (an optional native
36
36
  // dep, possibly absent under Bun) is statically imported.
37
37
  const nodeRequire = createRequire(import.meta.url);
38
+ // bun:sqlite — Bun built-in, no native build. Cannot run on Node.
39
+ const bunSqliteProvider = {
40
+ name: "bun:sqlite",
41
+ supported: () => isBun,
42
+ open: openBunDatabase,
43
+ };
44
+ // better-sqlite3 — native Node driver. Cannot run on Bun (oven-sh/bun#4290).
45
+ const nodeSqliteProvider = {
46
+ name: "better-sqlite3",
47
+ supported: () => !isBun,
48
+ open: openNodeDatabase,
49
+ };
38
50
  /**
39
- * Open a SQLite database handle at `path`, selecting the driver for the current
40
- * runtime. Returns a handle conforming to the structural {@link Database} type.
41
- *
42
- * The Node driver (`better-sqlite3`) is required lazily and ONLY when not on
43
- * Bun, so importing this module under Bun never touches `better-sqlite3`.
51
+ * Ordered provider registry. The factory selects the first supported provider.
52
+ * A future Postgres provider is appended here and only here. Both SQLite
53
+ * engines are kept as distinct providers (not collapsed) because no single
54
+ * SQLite driver runs on both Bun and Node today.
44
55
  */
45
- export function openDatabase(path, opts) {
46
- if (isBun) {
47
- return openBunDatabase(path, opts);
56
+ const PROVIDERS = [bunSqliteProvider, nodeSqliteProvider];
57
+ /** Select the provider for the current runtime. */
58
+ function selectProvider() {
59
+ const provider = PROVIDERS.find((p) => p.supported());
60
+ if (!provider) {
61
+ throw new Error(`No storage provider supports the current runtime (${isBun ? "Bun" : "Node"}).`);
48
62
  }
49
- return openNodeDatabase(path, opts);
63
+ return provider;
64
+ }
65
+ /**
66
+ * Open a SQLite database handle at `path` via the active {@link StorageProvider}.
67
+ * Returns a handle conforming to the structural {@link Database} type.
68
+ */
69
+ export function openDatabase(path, opts) {
70
+ return selectProvider().open(path, opts);
50
71
  }
51
72
  function openBunDatabase(path, opts) {
52
73
  const { Database: BunDatabase } = loadBunSqlite();
@@ -81,7 +102,21 @@ function loadBetterSqlite3() {
81
102
  if (!betterSqlite3Ctor) {
82
103
  // Runtime-gated dynamic require: only reached when NOT on Bun, so Bun never
83
104
  // resolves or loads the optional `better-sqlite3` native dependency.
84
- const mod = nodeRequire("better-sqlite3");
105
+ // `better-sqlite3` is an optionalDependency, so `npm i` can succeed without
106
+ // it (or with a native build that failed). Convert the raw MODULE_NOT_FOUND
107
+ // into an actionable message instead of a cryptic onboarding crash.
108
+ let mod;
109
+ try {
110
+ mod = nodeRequire("better-sqlite3");
111
+ }
112
+ catch (err) {
113
+ throw new Error("akm could not load 'better-sqlite3', the SQLite driver it needs on Node.js.\n" +
114
+ " • Reinstall akm with a working C/C++ build toolchain so its optional\n" +
115
+ " 'better-sqlite3' native binding rebuilds (a global `npm i -g better-sqlite3`\n" +
116
+ " will NOT be resolved — Node loads it from akm's own node_modules).\n" +
117
+ " • Or run akm under Bun, which has a built-in SQLite driver and needs no native build.\n" +
118
+ ` Underlying load error: ${err instanceof Error ? err.message : String(err)}`);
119
+ }
85
120
  betterSqlite3Ctor = mod.default ?? mod;
86
121
  }
87
122
  return betterSqlite3Ctor;
@@ -0,0 +1,82 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ /**
5
+ * Managed-database seam — the single home for the SQLite open/lifecycle recipe.
6
+ *
7
+ * Before this module, two idioms were copy-pasted across state.db / logs.db /
8
+ * workflow.db / index.db and their consumers:
9
+ *
10
+ * 1. The open recipe: `mkdir(dir) → openDatabase(path) → applyStandardPragmas
11
+ * → migrate`.
12
+ * 2. The borrow-or-own lifecycle: `const db = ctx?.db ?? open(); const owns =
13
+ * !ctx?.db; try { … } finally { if (owns) db.close(); }`.
14
+ *
15
+ * {@link openManagedDatabase} owns (1); {@link withManagedDb} owns (2). Each DB
16
+ * module supplies only a path + initializer and gets a `withXDb` loan helper
17
+ * (see `withIndexDb`, `withStateDb`, etc.) so callers never hand-roll the
18
+ * ownership flag or the finally/close again. This is also the one place to add
19
+ * busy-timeout tuning, integrity checks, or test-isolation injection.
20
+ */
21
+ import fs from "node:fs";
22
+ import path from "node:path";
23
+ import { openDatabase } from "./database.js";
24
+ import { applyStandardPragmas } from "./sqlite-pragmas.js";
25
+ /**
26
+ * Open a managed SQLite database: ensure the parent dir exists, open the handle,
27
+ * apply standard pragmas, then run the schema initializer. The single home for
28
+ * the open→pragmas→migrate recipe.
29
+ */
30
+ export function openManagedDatabase(spec) {
31
+ const dir = path.dirname(spec.path);
32
+ if (!fs.existsSync(dir)) {
33
+ fs.mkdirSync(dir, { recursive: true });
34
+ }
35
+ const db = openDatabase(spec.path);
36
+ applyStandardPragmas(db, spec.pragmas ?? { dataDir: dir });
37
+ spec.init?.(db);
38
+ return db;
39
+ }
40
+ /**
41
+ * Run `fn` against a managed database, owning its lifecycle.
42
+ *
43
+ * When `opts.borrowed` is supplied the caller already owns an open handle: it is
44
+ * passed straight through and NOT closed (borrow). Otherwise a fresh handle is
45
+ * opened via `open` and closed in a `finally` (own). This replaces the
46
+ * hand-rolled `ctx?.db ?? open()` + `ownsDb` flag + `finally`/close idiom — the
47
+ * ownership decision and the close live here, once.
48
+ *
49
+ * Synchronous by design: the DB consumers (telemetry writers, planners) finish
50
+ * all work within the tick, matching the inline blocks this replaces.
51
+ */
52
+ export function withManagedDb(open, fn, opts) {
53
+ if (opts?.borrowed) {
54
+ return fn(opts.borrowed);
55
+ }
56
+ const db = open();
57
+ try {
58
+ return fn(db);
59
+ }
60
+ finally {
61
+ db.close();
62
+ }
63
+ }
64
+ /**
65
+ * Async sibling of {@link withManagedDb}. Use this — NOT `withManagedDb` — when
66
+ * `fn` holds the handle across an `await`: the sync version closes in its
67
+ * `finally` before the awaited work resolves (use-after-close). Here the handle
68
+ * is closed only after `fn`'s promise settles. Borrowed handles pass straight
69
+ * through and are not closed, as in the sync version.
70
+ */
71
+ export async function withManagedDbAsync(open, fn, opts) {
72
+ if (opts?.borrowed) {
73
+ return fn(opts.borrowed);
74
+ }
75
+ const db = open();
76
+ try {
77
+ return await fn(db);
78
+ }
79
+ finally {
80
+ db.close();
81
+ }
82
+ }
@@ -0,0 +1,92 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ import { rethrowIfTestIsolationError } from "../../core/errors.js";
5
+ import { closeDatabase, getRegistryIndexCache, openIndexDatabase, upsertRegistryIndexCache } from "../../indexer/db/db.js";
6
+ /**
7
+ * RAII-style lifecycle helper for the registry cache DB. Opens the DB (treating
8
+ * a failed open exactly like the legacy fall-through: the bun-test isolation
9
+ * guard is re-thrown, any other failure yields `db = undefined`), runs `fn`,
10
+ * and guarantees the DB is closed in a `finally` after `fn` has fully settled
11
+ * (the await is required: the callbacks are async, and closing before they
12
+ * settle would tear the DB down mid-write).
13
+ */
14
+ export async function withRegistryCacheDb(fn) {
15
+ let db;
16
+ try {
17
+ db = openIndexDatabase();
18
+ }
19
+ catch (err) {
20
+ // Never mask the bun-test isolation guard as "DB unavailable".
21
+ rethrowIfTestIsolationError(err);
22
+ db = undefined;
23
+ }
24
+ try {
25
+ return await fn(db);
26
+ }
27
+ finally {
28
+ if (db) {
29
+ try {
30
+ closeDatabase(db);
31
+ }
32
+ catch {
33
+ /* ignore */
34
+ }
35
+ }
36
+ }
37
+ }
38
+ /**
39
+ * Shared registry index-cache fetch template. Opens the cache DB, returns a
40
+ * fresh cache hit when present, otherwise fetches live (writing the result back
41
+ * to the cache best-effort), and falls back to a stale cache row when the fetch
42
+ * fails. Behaviour-preserving extraction of the logic previously duplicated in
43
+ * `skills-sh.ts` (`fetchSkills`) and `static-index.ts` (`loadIndex`).
44
+ */
45
+ export async function fetchCachedJson(opts) {
46
+ const { cacheKey, ttlMs, parseCache, fetchFresh } = opts;
47
+ return withRegistryCacheDb(async (db) => {
48
+ // ── Step 1: Try DB cache (index.db) ─────────────────────────────────────
49
+ let dbCacheResult;
50
+ try {
51
+ if (db) {
52
+ dbCacheResult = getRegistryIndexCache(db, cacheKey, ttlMs);
53
+ }
54
+ }
55
+ catch (err) {
56
+ // Never mask the bun-test isolation guard as "DB unavailable" — see
57
+ // rethrowIfTestIsolationError in src/core/errors.ts. Without this, a
58
+ // leaky test silently gets a cold cache instead of the loud
59
+ // TEST_ISOLATION_MISSING failure the guard intends.
60
+ rethrowIfTestIsolationError(err);
61
+ // index.db read failed (pre-migration install or test env) — fall through
62
+ }
63
+ if (dbCacheResult) {
64
+ const cached = parseCache(dbCacheResult.indexJson, { stale: false });
65
+ if (cached !== undefined) {
66
+ return cached;
67
+ }
68
+ }
69
+ // ── Step 2: Fetch fresh ──────────────────────────────────────────────────
70
+ try {
71
+ const { value, cacheJson, cacheOpts } = await fetchFresh(db);
72
+ if (db) {
73
+ try {
74
+ upsertRegistryIndexCache(db, cacheKey, cacheJson, cacheOpts);
75
+ }
76
+ catch {
77
+ /* best-effort */
78
+ }
79
+ }
80
+ return value;
81
+ }
82
+ catch (err) {
83
+ // Fetch failed — use stale DB cache if available.
84
+ if (dbCacheResult) {
85
+ const stale = parseCache(dbCacheResult.indexJson, { stale: true });
86
+ if (stale !== undefined)
87
+ return stale;
88
+ }
89
+ throw err;
90
+ }
91
+ });
92
+ }