akm-cli 0.9.0-beta.5 → 0.9.0-beta.50

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (207) hide show
  1. package/CHANGELOG.md +709 -0
  2. package/dist/assets/profiles/default.json +9 -4
  3. package/dist/assets/profiles/frequent.json +1 -1
  4. package/dist/assets/profiles/memory-focus.json +1 -1
  5. package/dist/assets/profiles/quick.json +1 -1
  6. package/dist/assets/profiles/synthesize.json +15 -0
  7. package/dist/assets/profiles/thorough.json +1 -1
  8. package/dist/assets/prompts/consolidate-system.md +23 -0
  9. package/dist/assets/prompts/contradiction-judge.md +33 -0
  10. package/dist/assets/prompts/distill-knowledge-system.md +22 -0
  11. package/dist/assets/prompts/distill-lesson-system.md +36 -0
  12. package/dist/assets/prompts/extract-session.md +6 -2
  13. package/dist/assets/prompts/graph-extract-system.md +1 -0
  14. package/dist/assets/prompts/graph-extract-user-prompt.md +1 -1
  15. package/dist/assets/prompts/memory-infer-system.md +1 -0
  16. package/dist/assets/prompts/memory-infer-user.md +5 -0
  17. package/dist/assets/prompts/metadata-enhance-system.md +1 -0
  18. package/dist/assets/prompts/procedural-system.md +44 -0
  19. package/dist/assets/prompts/recombine-system.md +40 -0
  20. package/dist/assets/prompts/staleness-detect-system.md +6 -0
  21. package/dist/assets/prompts/validate-summary-judge.md +1 -0
  22. package/dist/assets/stash-skeleton/facts/conventions/assets/agent.md +38 -0
  23. package/dist/assets/stash-skeleton/facts/conventions/assets/command.md +38 -0
  24. package/dist/assets/stash-skeleton/facts/conventions/assets/fact.md +39 -0
  25. package/dist/assets/stash-skeleton/facts/conventions/assets/knowledge.md +40 -0
  26. package/dist/assets/stash-skeleton/facts/conventions/assets/lesson.md +43 -0
  27. package/dist/assets/stash-skeleton/facts/conventions/assets/memory.md +38 -0
  28. package/dist/assets/stash-skeleton/facts/conventions/assets/script.md +43 -0
  29. package/dist/assets/stash-skeleton/facts/conventions/assets/skill.md +40 -0
  30. package/dist/assets/stash-skeleton/facts/conventions/assets/workflow.md +43 -0
  31. package/dist/assets/templates/html/health.html +281 -111
  32. package/dist/assets/wiki/ingest-workflow-template.md +17 -10
  33. package/dist/cli/shared.js +28 -0
  34. package/dist/cli.js +15 -5
  35. package/dist/commands/agent/agent-dispatch.js +2 -2
  36. package/dist/commands/agent/agent-support.js +0 -7
  37. package/dist/commands/agent/contribute-cli.js +17 -4
  38. package/dist/commands/env/env-cli.js +16 -24
  39. package/dist/commands/env/secret-cli.js +12 -20
  40. package/dist/commands/feedback-cli.js +15 -6
  41. package/dist/commands/graph/graph-cli.js +5 -13
  42. package/dist/commands/graph/graph.js +76 -72
  43. package/dist/commands/health/checks.js +48 -0
  44. package/dist/commands/health/html-report.js +422 -80
  45. package/dist/commands/health.js +386 -9
  46. package/dist/commands/improve/calibration.js +161 -0
  47. package/dist/commands/improve/consolidate/chunking.js +141 -0
  48. package/dist/commands/improve/consolidate/eligibility.js +81 -0
  49. package/dist/commands/improve/consolidate/merge.js +145 -0
  50. package/dist/commands/improve/consolidate/sanitize.js +231 -0
  51. package/dist/commands/{lint.js → improve/consolidate/types.js} +1 -1
  52. package/dist/commands/improve/consolidate.js +635 -660
  53. package/dist/commands/improve/dedup.js +482 -0
  54. package/dist/commands/improve/distill.js +159 -69
  55. package/dist/commands/improve/eligibility.js +434 -0
  56. package/dist/commands/improve/encoding-salience.js +205 -0
  57. package/dist/commands/improve/extract-cli.js +124 -2
  58. package/dist/commands/improve/extract-prompt.js +39 -2
  59. package/dist/commands/improve/extract-watch.js +140 -0
  60. package/dist/commands/improve/extract.js +389 -40
  61. package/dist/commands/improve/feedback-valence.js +54 -0
  62. package/dist/commands/improve/homeostatic.js +467 -0
  63. package/dist/commands/improve/improve-auto-accept.js +109 -6
  64. package/dist/commands/improve/improve-cli.js +35 -60
  65. package/dist/commands/improve/improve-profiles.js +14 -0
  66. package/dist/commands/improve/improve-result-file.js +5 -23
  67. package/dist/commands/improve/improve-session.js +58 -0
  68. package/dist/commands/improve/improve.js +485 -2498
  69. package/dist/commands/improve/locks.js +154 -0
  70. package/dist/commands/improve/loop-stages.js +1083 -0
  71. package/dist/commands/improve/memory/memory-contradiction-detect.js +23 -28
  72. package/dist/commands/improve/outcome-loop.js +256 -0
  73. package/dist/commands/improve/preparation.js +1966 -0
  74. package/dist/commands/improve/proactive-maintenance.js +115 -0
  75. package/dist/commands/improve/procedural.js +418 -0
  76. package/dist/commands/improve/recombine.js +813 -0
  77. package/dist/commands/improve/reflect-noise.js +0 -0
  78. package/dist/commands/improve/reflect.js +183 -40
  79. package/dist/commands/improve/salience.js +438 -0
  80. package/dist/commands/improve/triage.js +93 -0
  81. package/dist/commands/lint/agent-linter.js +19 -24
  82. package/dist/commands/lint/base-linter.js +173 -60
  83. package/dist/commands/lint/command-linter.js +19 -24
  84. package/dist/commands/lint/env-key-rules.js +34 -1
  85. package/dist/commands/lint/fact-linter.js +39 -0
  86. package/dist/commands/lint/index.js +31 -13
  87. package/dist/commands/lint/memory-linter.js +1 -1
  88. package/dist/commands/lint/registry.js +7 -2
  89. package/dist/commands/lint/task-linter.js +3 -3
  90. package/dist/commands/lint/workflow-linter.js +26 -1
  91. package/dist/commands/proposal/drain-policies.js +5 -0
  92. package/dist/commands/proposal/drain.js +43 -50
  93. package/dist/commands/proposal/proposal-cli.js +21 -31
  94. package/dist/commands/proposal/proposal.js +5 -0
  95. package/dist/commands/proposal/propose.js +7 -2
  96. package/dist/commands/proposal/validators/proposal-quality-validators.js +9 -8
  97. package/dist/commands/proposal/validators/proposals.js +189 -63
  98. package/dist/commands/read/curate.js +414 -94
  99. package/dist/commands/read/knowledge.js +2 -2
  100. package/dist/commands/read/search-cli.js +7 -0
  101. package/dist/commands/read/search.js +1 -0
  102. package/dist/commands/read/show.js +67 -2
  103. package/dist/commands/sources/init.js +36 -9
  104. package/dist/commands/sources/installed-stashes.js +5 -1
  105. package/dist/commands/sources/schema-repair.js +13 -1
  106. package/dist/commands/sources/self-update.js +2 -2
  107. package/dist/commands/sources/stash-cli.js +28 -40
  108. package/dist/commands/sources/stash-skeleton.js +23 -8
  109. package/dist/commands/tasks/tasks-cli.js +19 -27
  110. package/dist/commands/tasks/tasks.js +1 -1
  111. package/dist/commands/wiki-cli.js +21 -35
  112. package/dist/core/asset/asset-registry.js +2 -0
  113. package/dist/core/asset/asset-spec.js +14 -0
  114. package/dist/core/asset/frontmatter.js +166 -167
  115. package/dist/core/asset/markdown.js +8 -0
  116. package/dist/core/authoring-rules.js +92 -0
  117. package/dist/core/common.js +0 -5
  118. package/dist/core/config/config-schema.js +340 -56
  119. package/dist/core/config/config-types.js +3 -3
  120. package/dist/core/config/config.js +28 -7
  121. package/dist/core/events.js +3 -7
  122. package/dist/core/improve-types.js +11 -8
  123. package/dist/core/logs-db.js +10 -66
  124. package/dist/core/parse.js +36 -16
  125. package/dist/core/paths.js +3 -0
  126. package/dist/core/standards/resolve-standards-context.js +87 -0
  127. package/dist/core/standards/resolve-stash-standards.js +99 -0
  128. package/dist/core/standards/resolve-type-conventions.js +66 -0
  129. package/dist/core/state/migrations.js +714 -0
  130. package/dist/core/state-db.js +525 -474
  131. package/dist/indexer/db/db.js +439 -247
  132. package/dist/indexer/db/graph-db.js +129 -86
  133. package/dist/indexer/ensure-index.js +152 -17
  134. package/dist/indexer/graph/graph-boost.js +51 -41
  135. package/dist/indexer/graph/graph-extraction.js +218 -4
  136. package/dist/indexer/index-writer-lock.js +99 -0
  137. package/dist/indexer/indexer.js +123 -221
  138. package/dist/indexer/passes/dir-staleness.js +114 -0
  139. package/dist/indexer/passes/memory-inference.js +10 -3
  140. package/dist/indexer/passes/staleness-detect.js +2 -5
  141. package/dist/indexer/search/db-search.js +15 -4
  142. package/dist/indexer/search/ranking-contributors.js +22 -0
  143. package/dist/indexer/search/ranking.js +4 -0
  144. package/dist/indexer/search/search-source.js +10 -24
  145. package/dist/indexer/search/semantic-status.js +4 -0
  146. package/dist/indexer/walk/matchers.js +9 -0
  147. package/dist/integrations/agent/config.js +6 -53
  148. package/dist/integrations/agent/index.js +2 -18
  149. package/dist/integrations/agent/prompts.js +74 -8
  150. package/dist/integrations/agent/runner-dispatch.js +59 -0
  151. package/dist/integrations/harnesses/claude/session-log.js +11 -1
  152. package/dist/integrations/harnesses/index.js +2 -3
  153. package/dist/integrations/harnesses/opencode/session-log.js +173 -3
  154. package/dist/integrations/harnesses/opencode-sdk/index.js +2 -2
  155. package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +0 -2
  156. package/dist/integrations/session-logs/index.js +16 -0
  157. package/dist/llm/client.js +45 -15
  158. package/dist/llm/embedder.js +42 -3
  159. package/dist/llm/embedders/deterministic.js +66 -0
  160. package/dist/llm/embedders/local.js +66 -2
  161. package/dist/llm/feature-gate.js +8 -4
  162. package/dist/llm/graph-extract.js +67 -44
  163. package/dist/llm/memory-infer.js +38 -30
  164. package/dist/llm/metadata-enhance.js +44 -31
  165. package/dist/llm/structured-call.js +49 -0
  166. package/dist/output/context.js +5 -5
  167. package/dist/output/renderers.js +73 -1
  168. package/dist/output/shapes/curate.js +14 -2
  169. package/dist/output/shapes/passthrough.js +0 -1
  170. package/dist/output/text/helpers.js +16 -1
  171. package/dist/registry/providers/skills-sh.js +21 -147
  172. package/dist/registry/providers/static-index.js +15 -157
  173. package/dist/registry/resolve.js +22 -9
  174. package/dist/runtime.js +25 -1
  175. package/dist/scripts/migrate-storage.js +2136 -1596
  176. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +682 -433
  177. package/dist/setup/setup.js +29 -8
  178. package/dist/sources/providers/filesystem.js +0 -1
  179. package/dist/sources/providers/git-install.js +206 -0
  180. package/dist/sources/providers/git-provider.js +234 -0
  181. package/dist/sources/providers/git-stash.js +248 -0
  182. package/dist/sources/providers/git.js +10 -661
  183. package/dist/sources/providers/npm.js +2 -6
  184. package/dist/sources/providers/sync-from-ref.js +9 -1
  185. package/dist/sources/providers/tar-utils.js +16 -8
  186. package/dist/sources/providers/website.js +2 -3
  187. package/dist/sources/website-ingest.js +51 -9
  188. package/dist/sources/wiki-fetchers/registry.js +53 -0
  189. package/dist/sources/wiki-fetchers/youtube.js +239 -0
  190. package/dist/storage/database.js +45 -10
  191. package/dist/storage/managed-db.js +82 -0
  192. package/dist/storage/repositories/registry-cache.js +92 -0
  193. package/dist/storage/sqlite-pragmas.js +146 -0
  194. package/dist/tasks/backends/cron.js +1 -1
  195. package/dist/tasks/backends/launchd.js +1 -1
  196. package/dist/tasks/backends/schtasks.js +1 -1
  197. package/dist/tasks/{resolveAkmBin.js → resolve-akm-bin.js} +2 -2
  198. package/dist/tasks/runner.js +5 -13
  199. package/dist/wiki/wiki.js +37 -0
  200. package/dist/workflows/db.js +3 -4
  201. package/dist/workflows/runtime/runs.js +1 -117
  202. package/dist/workflows/runtime/workflow-asset-loader.js +125 -0
  203. package/dist/workflows/validate-summary.js +2 -7
  204. package/docs/data-and-telemetry.md +1 -0
  205. package/package.json +9 -7
  206. package/dist/commands/db-cli.js +0 -23
  207. package/dist/indexer/db/db-backup.js +0 -376
@@ -36,18 +36,14 @@ class NpmSourceProvider {
36
36
  this.#config = config;
37
37
  this.name = config.name ?? config.url ?? "npm";
38
38
  }
39
- async init(_ctx) {
40
- // Resolution happens lazily in path(): until `sync()` runs there's no
41
- // reliable on-disk path. Init is the registration handshake.
42
- }
43
39
  path() {
44
40
  if (this.#config.path)
45
41
  return this.#config.path;
46
42
  throw new ConfigError(`npm source "${this.name}" has no resolved content path — run \`akm update\` to sync it before indexing.`);
47
43
  }
48
- async sync() {
44
+ async sync(options) {
49
45
  const ref = npmRefFromConfig(this.#config);
50
- await syncNpmRef(ref);
46
+ await syncNpmRef(ref, { force: options?.force });
51
47
  }
52
48
  }
53
49
  registerSourceProvider("npm", (config) => new NpmSourceProvider(config));
@@ -1,6 +1,13 @@
1
1
  // This Source Code Form is subject to the terms of the Mozilla Public
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ /**
5
+ * Unified install-ref dispatcher.
6
+ *
7
+ * Replaces the historical `installRegistryRef()` entry point. Given an
8
+ * unparsed install ref, this resolves the right syncable provider and
9
+ * invokes its `sync()` method.
10
+ */
4
11
  import { UsageError } from "../../core/errors.js";
5
12
  import { parseRegistryRef } from "../../registry/resolve.js";
6
13
  import { detectStashRoot } from "./provider-utils.js";
@@ -18,7 +25,8 @@ export async function syncFromRef(ref, options) {
18
25
  return syncRegistryGitRef(ref, options);
19
26
  }
20
27
  // Exhaustiveness — `parseRegistryRef` only emits the four sources above.
21
- throw new UsageError(`No syncable provider for ref: ${ref} (source=${parsed.source})`);
28
+ const _exhaustive = parsed;
29
+ throw new UsageError(`No syncable provider for ref: ${ref}`);
22
30
  }
23
31
  function syncLocalRef(parsed, options) {
24
32
  const stashRoot = detectStashRoot(parsed.sourcePath);
@@ -12,12 +12,12 @@
12
12
  * providers that fetch tarballs (currently `NpmSourceProvider` and the
13
13
  * registry index builder).
14
14
  */
15
- import { spawnSync } from "node:child_process";
16
15
  import { createHash } from "node:crypto";
17
16
  import fs from "node:fs";
18
17
  import path from "node:path";
19
18
  import { isWithin } from "../../core/common.js";
20
19
  import { warn } from "../../core/warn.js";
20
+ import { spawnSync } from "../../runtime.js";
21
21
  /**
22
22
  * Verify an archive's integrity against a known hash. Throws and removes
23
23
  * the archive when verification fails.
@@ -65,17 +65,25 @@ export function verifyArchiveIntegrity(archivePath, expected, source) {
65
65
  * tree for symlinks that would escape the destination.
66
66
  */
67
67
  export function extractTarGzSecure(archivePath, destinationDir) {
68
- const listResult = spawnSync("tar", ["tzf", archivePath], { encoding: "utf8" });
69
- if (listResult.status !== 0) {
70
- const err = listResult.stderr?.trim() || listResult.error?.message || "unknown error";
68
+ const listResult = spawnSync(["tar", "tzf", archivePath]);
69
+ if (!listResult.success) {
70
+ const err = listResult.stderr?.toString().trim() || "unknown error";
71
71
  throw new Error(`Failed to inspect archive ${archivePath}: ${err}`);
72
72
  }
73
- validateTarEntries(listResult.stdout);
73
+ validateTarEntries(listResult.stdout.toString());
74
74
  fs.rmSync(destinationDir, { recursive: true, force: true });
75
75
  fs.mkdirSync(destinationDir, { recursive: true });
76
- const extractResult = spawnSync("tar", ["xzf", archivePath, "--no-same-owner", "--strip-components=1", "-C", destinationDir], { encoding: "utf8" });
77
- if (extractResult.status !== 0) {
78
- const err = extractResult.stderr?.trim() || extractResult.error?.message || "unknown error";
76
+ const extractResult = spawnSync([
77
+ "tar",
78
+ "xzf",
79
+ archivePath,
80
+ "--no-same-owner",
81
+ "--strip-components=1",
82
+ "-C",
83
+ destinationDir,
84
+ ]);
85
+ if (!extractResult.success) {
86
+ const err = extractResult.stderr?.toString().trim() || "unknown error";
79
87
  throw new Error(`Failed to extract archive ${archivePath}: ${err}`);
80
88
  }
81
89
  // Post-extraction scan: verify all extracted files are within destinationDir
@@ -12,12 +12,11 @@ registerSourceProvider("website", (config) => {
12
12
  return {
13
13
  kind: "website",
14
14
  name,
15
- async init(_ctx) { },
16
15
  path() {
17
16
  return getWebsiteCachePaths(url).stashDir;
18
17
  },
19
- async sync() {
20
- await ensureWebsiteMirror(config, { requireStashDir: true });
18
+ async sync(options) {
19
+ await ensureWebsiteMirror(config, { requireStashDir: true, force: options?.force });
21
20
  },
22
21
  };
23
22
  });
@@ -4,11 +4,12 @@
4
4
  import { createHash } from "node:crypto";
5
5
  import fs from "node:fs";
6
6
  import path from "node:path";
7
- import { fetchWithRetry, ResponseTooLargeError, readBodyWithByteCap } from "../core/common.js";
7
+ import { fetchWithRetry, ResponseTooLargeError, readBodyWithByteCap, resolveStashDir } from "../core/common.js";
8
8
  import { ConfigError, UsageError } from "../core/errors.js";
9
9
  import { getRegistryIndexCacheDir } from "../core/paths.js";
10
10
  import { warn } from "../core/warn.js";
11
11
  import { isExpired, sanitizeString } from "./providers/provider-utils.js";
12
+ import { loadWikiSnapshotFetchers } from "./wiki-fetchers/registry.js";
12
13
  /** Refresh website snapshots every 12 hours to balance freshness with scraping load. */
13
14
  const CACHE_REFRESH_INTERVAL_MS = 12 * 60 * 60 * 1000;
14
15
  /** Allow up to 7 days of stale snapshots when refresh fails so search remains available during outages. */
@@ -30,6 +31,16 @@ const WEBSITE_PAGE_BYTE_CAP = 5 * 1024 * 1024;
30
31
  * whole crawl and return what we have when time runs out.
31
32
  */
32
33
  const WEBSITE_CRAWL_WALL_CLOCK_MS = 10 * 60 * 1000;
34
+ function resolveFetcherStashDir(explicitStashDir) {
35
+ if (explicitStashDir)
36
+ return explicitStashDir;
37
+ try {
38
+ return resolveStashDir({ readOnly: true });
39
+ }
40
+ catch {
41
+ return null;
42
+ }
43
+ }
33
44
  export function getWebsiteCachePaths(siteUrl) {
34
45
  const key = createHash("sha256").update(normalizeSiteUrl(siteUrl)).digest("hex").slice(0, 16);
35
46
  const rootDir = path.join(getRegistryIndexCacheDir(), `website-${key}`);
@@ -114,20 +125,51 @@ async function scrapeWebsiteToStash(startUrl, stashDir, options) {
114
125
  fs.writeFileSync(filePath, buildMarkdownSnapshot(page, slug), "utf8");
115
126
  }
116
127
  }
117
- export async function fetchWebsiteMarkdownSnapshot(rawUrl) {
128
+ export async function fetchWebsiteMarkdownSnapshot(rawUrl, options) {
118
129
  const normalizedUrl = validateWebsiteInputUrl(rawUrl);
130
+ const parsedUrl = new URL(normalizedUrl);
131
+ const stashDir = resolveFetcherStashDir(options?.stashDir);
132
+ const context = {
133
+ stashDir: stashDir ?? "",
134
+ timeoutMs: options?.timeoutMs ?? 15_000,
135
+ signal: options?.signal,
136
+ };
137
+ for (const fetcher of await loadWikiSnapshotFetchers(stashDir)) {
138
+ try {
139
+ if (!fetcher.matches(parsedUrl, context))
140
+ continue;
141
+ const snapshot = await fetcher.fetch(parsedUrl, context);
142
+ if (!snapshot)
143
+ continue;
144
+ return websiteMarkdownSnapshotFromResult(snapshot);
145
+ }
146
+ catch (error) {
147
+ warn("[akm] wiki-fetcher %s threw on %s: %s", fetcher.name, normalizedUrl, error instanceof Error ? error.message : String(error));
148
+ }
149
+ }
119
150
  const fetched = await fetchWebsitePage(normalizedUrl);
120
151
  if (!fetched) {
121
152
  throw new UsageError(`No content could be fetched from ${normalizedUrl}`);
122
153
  }
123
- const preferredName = deriveImportPath(fetched.page.url);
124
- const slug = preferredName.split("/").pop() ?? preferredName;
125
- return {
154
+ return websiteMarkdownSnapshotFromResult({
126
155
  url: fetched.page.url,
127
156
  title: fetched.page.title,
128
157
  markdown: fetched.page.markdown,
158
+ });
159
+ }
160
+ function websiteMarkdownSnapshotFromResult(snapshot) {
161
+ const preferredName = snapshot.preferredName ?? deriveImportPath(snapshot.url);
162
+ const slug = preferredName.split("/").pop() ?? preferredName;
163
+ return {
164
+ url: snapshot.url,
165
+ title: snapshot.title,
166
+ markdown: snapshot.markdown,
129
167
  preferredName,
130
- content: buildMarkdownSnapshot(fetched.page, slug || "website"),
168
+ content: buildMarkdownSnapshot({
169
+ url: snapshot.url,
170
+ title: snapshot.title,
171
+ markdown: snapshot.markdown,
172
+ }, slug || "website", snapshot.tags),
131
173
  };
132
174
  }
133
175
  async function crawlWebsite(startUrl, options) {
@@ -216,11 +258,12 @@ async function fetchWebsitePage(pageUrl) {
216
258
  links: [],
217
259
  };
218
260
  }
219
- function buildMarkdownSnapshot(page, slug) {
261
+ function buildMarkdownSnapshot(page, slug, tags) {
220
262
  const title = sanitizeString(page.title, 200) || slug;
221
263
  const description = sanitizeString(`Snapshot of ${page.url}`, 500);
222
264
  const host = sanitizeString(new URL(page.url).hostname, 120);
223
265
  const content = page.markdown.trim() || `Source: ${page.url}`;
266
+ const normalizedTags = Array.from(new Set(["website", host, ...(tags ?? [])]));
224
267
  return [
225
268
  "---",
226
269
  `name: ${JSON.stringify(slug)}`,
@@ -228,8 +271,7 @@ function buildMarkdownSnapshot(page, slug) {
228
271
  `sourceUrl: ${JSON.stringify(page.url)}`,
229
272
  `title: ${JSON.stringify(title)}`,
230
273
  "tags:",
231
- ` - ${JSON.stringify("website")}`,
232
- ` - ${JSON.stringify(host)}`,
274
+ ...normalizedTags.map((tag) => ` - ${JSON.stringify(tag)}`),
233
275
  "---",
234
276
  "",
235
277
  `# ${title}`,
@@ -0,0 +1,53 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ import fs from "node:fs";
5
+ import path from "node:path";
6
+ import { pathToFileURL } from "node:url";
7
+ import { warn } from "../../core/warn.js";
8
+ import youtubeFetcher from "./youtube.js";
9
+ const FETCHER_DIR = path.join("scripts", "wiki-fetchers");
10
+ const FETCHER_FILE_PATTERN = /\.(?:ts|js|mjs)$/i;
11
+ const BUILTIN_FETCHERS = [youtubeFetcher];
12
+ function isWikiSnapshotFetcher(value) {
13
+ if (!value || typeof value !== "object")
14
+ return false;
15
+ const candidate = value;
16
+ return (typeof candidate.name === "string" &&
17
+ typeof candidate.matches === "function" &&
18
+ typeof candidate.fetch === "function");
19
+ }
20
+ export async function loadWikiSnapshotFetchers(stashDir) {
21
+ const fetchers = [];
22
+ if (stashDir) {
23
+ const fetcherDir = path.join(stashDir, FETCHER_DIR);
24
+ try {
25
+ const entries = fs.readdirSync(fetcherDir).sort();
26
+ for (const entry of entries) {
27
+ if (!FETCHER_FILE_PATTERN.test(entry))
28
+ continue;
29
+ try {
30
+ const fileUrl = pathToFileURL(path.join(fetcherDir, entry)).toString();
31
+ const mod = await import(fileUrl);
32
+ if (isWikiSnapshotFetcher(mod.default)) {
33
+ fetchers.push(mod.default);
34
+ }
35
+ else {
36
+ warn("[akm] wiki-fetcher %s skipped: missing { name, matches, fetch }", entry);
37
+ }
38
+ }
39
+ catch (error) {
40
+ warn("[akm] wiki-fetcher %s failed to load: %s", entry, error instanceof Error ? error.message : String(error));
41
+ }
42
+ }
43
+ }
44
+ catch (error) {
45
+ const code = typeof error === "object" && error && "code" in error ? error.code : undefined;
46
+ if (code !== "ENOENT") {
47
+ warn("[akm] wiki-fetcher directory %s could not be read: %s", fetcherDir, error instanceof Error ? error.message : String(error));
48
+ }
49
+ // Missing directory means no custom fetchers.
50
+ }
51
+ }
52
+ return [...fetchers, ...BUILTIN_FETCHERS];
53
+ }
@@ -0,0 +1,239 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ import { fetchWithRetry } from "../../core/common.js";
5
+ const YOUTUBE_HOSTS = new Set(["www.youtube.com", "youtube.com", "m.youtube.com", "youtu.be"]);
6
+ const WATCH_HOST = "https://www.youtube.com/watch?v=";
7
+ // Captions: the WEB watch-page caption baseUrls are now Proof-of-Origin-Token
8
+ // gated (they carry `exp=xpe` and return an empty body without a `pot` token).
9
+ // The ANDROID InnerTube `player` client still returns UNGATED caption baseUrls
10
+ // that serve the transcript, so captions are sourced from there. See
11
+ // https://github.com/yt-dlp/yt-dlp/issues/13075.
12
+ const INNERTUBE_PLAYER_URL = "https://www.youtube.com/youtubei/v1/player";
13
+ // Long-stable public InnerTube key; only authorizes the player call (not auth).
14
+ const DEFAULT_INNERTUBE_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
15
+ const ANDROID_INNERTUBE_CLIENT = {
16
+ clientName: "ANDROID",
17
+ clientVersion: "20.10.38",
18
+ androidSdkVersion: 34,
19
+ hl: "en",
20
+ gl: "US",
21
+ };
22
+ function extractVideoId(url) {
23
+ if (!YOUTUBE_HOSTS.has(url.hostname))
24
+ return null;
25
+ if (url.hostname === "youtu.be") {
26
+ const id = url.pathname.replace(/^\/+/, "").split("/")[0]?.trim();
27
+ return id || null;
28
+ }
29
+ if (url.pathname === "/watch") {
30
+ const id = url.searchParams.get("v")?.trim();
31
+ return id || null;
32
+ }
33
+ if (url.pathname.startsWith("/shorts/")) {
34
+ const id = url.pathname.slice("/shorts/".length).split("/")[0]?.trim();
35
+ return id || null;
36
+ }
37
+ if (url.pathname.startsWith("/embed/")) {
38
+ const id = url.pathname.slice("/embed/".length).split("/")[0]?.trim();
39
+ return id || null;
40
+ }
41
+ return null;
42
+ }
43
+ function canonicalWatchUrl(videoId) {
44
+ return `${WATCH_HOST}${encodeURIComponent(videoId)}`;
45
+ }
46
+ function decodeEntities(value) {
47
+ const safeFromCodePoint = (codePoint, fallback) => {
48
+ if (!Number.isFinite(codePoint) || codePoint < 0 || codePoint > 0x10ffff)
49
+ return fallback;
50
+ try {
51
+ return String.fromCodePoint(codePoint);
52
+ }
53
+ catch {
54
+ return fallback;
55
+ }
56
+ };
57
+ return value
58
+ .replace(/&#x([0-9a-f]+);/gi, (m, hex) => safeFromCodePoint(Number.parseInt(hex, 16), m))
59
+ .replace(/&#([0-9]+);/g, (m, dec) => safeFromCodePoint(Number.parseInt(dec, 10), m))
60
+ .replace(/&quot;/g, '"')
61
+ .replace(/&#39;|&apos;/g, "'")
62
+ .replace(/&lt;/g, "<")
63
+ .replace(/&gt;/g, ">")
64
+ .replace(/&amp;/g, "&");
65
+ }
66
+ function stripTags(value) {
67
+ return value.replace(/<[^>]+>/g, "");
68
+ }
69
+ function extractJsonObjectAfter(marker, source) {
70
+ const markerIndex = source.indexOf(marker);
71
+ if (markerIndex === -1)
72
+ return null;
73
+ const start = source.indexOf("{", markerIndex + marker.length);
74
+ if (start === -1)
75
+ return null;
76
+ let depth = 0;
77
+ let inString = false;
78
+ let escaped = false;
79
+ for (let i = start; i < source.length; i += 1) {
80
+ const char = source[i];
81
+ if (inString) {
82
+ if (escaped) {
83
+ escaped = false;
84
+ }
85
+ else if (char === "\\") {
86
+ escaped = true;
87
+ }
88
+ else if (char === '"') {
89
+ inString = false;
90
+ }
91
+ continue;
92
+ }
93
+ if (char === '"') {
94
+ inString = true;
95
+ continue;
96
+ }
97
+ if (char === "{")
98
+ depth += 1;
99
+ if (char === "}") {
100
+ depth -= 1;
101
+ if (depth === 0)
102
+ return source.slice(start, i + 1);
103
+ }
104
+ }
105
+ return null;
106
+ }
107
+ function parsePlayerResponse(html) {
108
+ const jsonText = extractJsonObjectAfter("ytInitialPlayerResponse =", html);
109
+ if (!jsonText)
110
+ return null;
111
+ try {
112
+ return JSON.parse(jsonText);
113
+ }
114
+ catch {
115
+ return null;
116
+ }
117
+ }
118
+ function chooseCaptionTrack(tracks) {
119
+ const normalized = tracks
120
+ .map((track) => ({
121
+ baseUrl: typeof track.baseUrl === "string" ? track.baseUrl : "",
122
+ languageCode: typeof track.languageCode === "string" ? track.languageCode : "",
123
+ kind: typeof track.kind === "string" ? track.kind : "",
124
+ }))
125
+ .filter((track) => track.baseUrl);
126
+ if (normalized.length === 0)
127
+ return null;
128
+ const preferred = normalized.find((track) => track.languageCode === "en" && track.kind !== "asr") ??
129
+ normalized.find((track) => track.languageCode.startsWith("en") && track.kind !== "asr") ??
130
+ normalized.find((track) => track.languageCode.startsWith("en")) ??
131
+ normalized[0];
132
+ return preferred.baseUrl;
133
+ }
134
+ async function fetchText(url, timeoutMs, signal) {
135
+ const response = await fetchWithRetry(url, {
136
+ headers: {
137
+ Accept: "text/html,application/json,application/xml,text/xml,text/plain;q=0.9,*/*;q=0.1",
138
+ "User-Agent": "akm-cli youtube fetcher",
139
+ },
140
+ signal,
141
+ }, { timeout: timeoutMs, retries: 1 });
142
+ if (!response.ok)
143
+ return null;
144
+ return response.text();
145
+ }
146
+ function parseTranscript(xml) {
147
+ // Two timedtext shapes: legacy srv1 `<text>` cues, and the format-3
148
+ // `<timedtext format="3">` `<p>` cues that the ANDROID InnerTube caption
149
+ // URLs return. Strip any nested `<s>` word tags via stripTags.
150
+ const matches = [...xml.matchAll(/<text\b[^>]*>([\s\S]*?)<\/text>/g), ...xml.matchAll(/<p\b[^>]*>([\s\S]*?)<\/p>/g)];
151
+ const texts = matches.map((match) => decodeEntities(stripTags(match[1] ?? "")).trim()).filter(Boolean);
152
+ if (texts.length === 0)
153
+ return null;
154
+ return texts.join("\n");
155
+ }
156
+ /**
157
+ * Fetch caption tracks via the ANDROID InnerTube `player` endpoint, whose
158
+ * caption baseUrls are not POT-gated. Returns [] on any failure so the caller
159
+ * can fall back to the (usually empty) watch-page tracks and degrade to a
160
+ * description-only snapshot.
161
+ */
162
+ async function fetchInnertubeCaptionTracks(videoId, apiKey, timeoutMs, signal) {
163
+ try {
164
+ const response = await fetchWithRetry(`${INNERTUBE_PLAYER_URL}?key=${encodeURIComponent(apiKey)}`, {
165
+ method: "POST",
166
+ headers: {
167
+ "Content-Type": "application/json",
168
+ "User-Agent": "com.google.android.youtube/20.10.38 (Linux; U; Android 14) gzip",
169
+ },
170
+ body: JSON.stringify({ context: { client: ANDROID_INNERTUBE_CLIENT }, videoId }),
171
+ signal,
172
+ }, { timeout: timeoutMs, retries: 1 });
173
+ if (!response.ok)
174
+ return [];
175
+ const json = (await response.json());
176
+ return json.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
177
+ }
178
+ catch {
179
+ return [];
180
+ }
181
+ }
182
+ /** Extract the page's InnerTube API key, falling back to the stable default. */
183
+ function extractInnertubeKey(html) {
184
+ return html.match(/"INNERTUBE_API_KEY":"([^"]+)"/)?.[1] ?? DEFAULT_INNERTUBE_KEY;
185
+ }
186
+ const youtubeFetcher = {
187
+ name: "youtube-transcript",
188
+ matches(url) {
189
+ return extractVideoId(url) !== null;
190
+ },
191
+ async fetch(url, context) {
192
+ const videoId = extractVideoId(url);
193
+ if (!videoId)
194
+ return null;
195
+ const watchUrl = canonicalWatchUrl(videoId);
196
+ const watchHtml = await fetchText(watchUrl, context.timeoutMs, context.signal);
197
+ if (!watchHtml)
198
+ return null;
199
+ const playerResponse = parsePlayerResponse(watchHtml);
200
+ if (!playerResponse)
201
+ return null;
202
+ const title = typeof playerResponse.videoDetails?.title === "string" ? playerResponse.videoDetails.title.trim() : "";
203
+ const description = typeof playerResponse.videoDetails?.shortDescription === "string"
204
+ ? playerResponse.videoDetails.shortDescription.trim()
205
+ : "";
206
+ const sections = [];
207
+ let hasTranscript = false;
208
+ if (description) {
209
+ sections.push("## Description", "", description);
210
+ }
211
+ // Prefer ungated ANDROID InnerTube caption tracks; fall back to the
212
+ // (now usually empty) watch-page tracks so a description-only snapshot
213
+ // still works when InnerTube is unavailable.
214
+ const apiKey = extractInnertubeKey(watchHtml);
215
+ let tracks = await fetchInnertubeCaptionTracks(videoId, apiKey, context.timeoutMs, context.signal);
216
+ if (tracks.length === 0) {
217
+ tracks = playerResponse.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
218
+ }
219
+ const captionUrl = chooseCaptionTrack(tracks);
220
+ if (captionUrl) {
221
+ const transcriptXml = await fetchText(captionUrl, context.timeoutMs, context.signal);
222
+ const transcript = transcriptXml ? parseTranscript(transcriptXml) : null;
223
+ if (transcript) {
224
+ hasTranscript = true;
225
+ sections.push("## Transcript", "", transcript);
226
+ }
227
+ }
228
+ if (sections.length === 0)
229
+ return null;
230
+ return {
231
+ url: watchUrl,
232
+ title: title || `YouTube ${videoId}`,
233
+ markdown: sections.join("\n"),
234
+ preferredName: `youtube/${videoId}`,
235
+ tags: hasTranscript ? ["youtube", "video", "transcript"] : ["youtube", "video"],
236
+ };
237
+ },
238
+ };
239
+ export default youtubeFetcher;
@@ -35,18 +35,39 @@ const isBun = !!process.versions?.bun;
35
35
  // Bun built-in, unresolvable on Node) nor `better-sqlite3` (an optional native
36
36
  // dep, possibly absent under Bun) is statically imported.
37
37
  const nodeRequire = createRequire(import.meta.url);
38
+ // bun:sqlite — Bun built-in, no native build. Cannot run on Node.
39
+ const bunSqliteProvider = {
40
+ name: "bun:sqlite",
41
+ supported: () => isBun,
42
+ open: openBunDatabase,
43
+ };
44
+ // better-sqlite3 — native Node driver. Cannot run on Bun (oven-sh/bun#4290).
45
+ const nodeSqliteProvider = {
46
+ name: "better-sqlite3",
47
+ supported: () => !isBun,
48
+ open: openNodeDatabase,
49
+ };
38
50
  /**
39
- * Open a SQLite database handle at `path`, selecting the driver for the current
40
- * runtime. Returns a handle conforming to the structural {@link Database} type.
41
- *
42
- * The Node driver (`better-sqlite3`) is required lazily and ONLY when not on
43
- * Bun, so importing this module under Bun never touches `better-sqlite3`.
51
+ * Ordered provider registry. The factory selects the first supported provider.
52
+ * A future Postgres provider is appended here and only here. Both SQLite
53
+ * engines are kept as distinct providers (not collapsed) because no single
54
+ * SQLite driver runs on both Bun and Node today.
44
55
  */
45
- export function openDatabase(path, opts) {
46
- if (isBun) {
47
- return openBunDatabase(path, opts);
56
+ const PROVIDERS = [bunSqliteProvider, nodeSqliteProvider];
57
+ /** Select the provider for the current runtime. */
58
+ function selectProvider() {
59
+ const provider = PROVIDERS.find((p) => p.supported());
60
+ if (!provider) {
61
+ throw new Error(`No storage provider supports the current runtime (${isBun ? "Bun" : "Node"}).`);
48
62
  }
49
- return openNodeDatabase(path, opts);
63
+ return provider;
64
+ }
65
+ /**
66
+ * Open a SQLite database handle at `path` via the active {@link StorageProvider}.
67
+ * Returns a handle conforming to the structural {@link Database} type.
68
+ */
69
+ export function openDatabase(path, opts) {
70
+ return selectProvider().open(path, opts);
50
71
  }
51
72
  function openBunDatabase(path, opts) {
52
73
  const { Database: BunDatabase } = loadBunSqlite();
@@ -81,7 +102,21 @@ function loadBetterSqlite3() {
81
102
  if (!betterSqlite3Ctor) {
82
103
  // Runtime-gated dynamic require: only reached when NOT on Bun, so Bun never
83
104
  // resolves or loads the optional `better-sqlite3` native dependency.
84
- const mod = nodeRequire("better-sqlite3");
105
+ // `better-sqlite3` is an optionalDependency, so `npm i` can succeed without
106
+ // it (or with a native build that failed). Convert the raw MODULE_NOT_FOUND
107
+ // into an actionable message instead of a cryptic onboarding crash.
108
+ let mod;
109
+ try {
110
+ mod = nodeRequire("better-sqlite3");
111
+ }
112
+ catch (err) {
113
+ throw new Error("akm could not load 'better-sqlite3', the SQLite driver it needs on Node.js.\n" +
114
+ " • Reinstall akm with a working C/C++ build toolchain so its optional\n" +
115
+ " 'better-sqlite3' native binding rebuilds (a global `npm i -g better-sqlite3`\n" +
116
+ " will NOT be resolved — Node loads it from akm's own node_modules).\n" +
117
+ " • Or run akm under Bun, which has a built-in SQLite driver and needs no native build.\n" +
118
+ ` Underlying load error: ${err instanceof Error ? err.message : String(err)}`);
119
+ }
85
120
  betterSqlite3Ctor = mod.default ?? mod;
86
121
  }
87
122
  return betterSqlite3Ctor;
@@ -0,0 +1,82 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ /**
5
+ * Managed-database seam — the single home for the SQLite open/lifecycle recipe.
6
+ *
7
+ * Before this module, two idioms were copy-pasted across state.db / logs.db /
8
+ * workflow.db / index.db and their consumers:
9
+ *
10
+ * 1. The open recipe: `mkdir(dir) → openDatabase(path) → applyStandardPragmas
11
+ * → migrate`.
12
+ * 2. The borrow-or-own lifecycle: `const db = ctx?.db ?? open(); const owns =
13
+ * !ctx?.db; try { … } finally { if (owns) db.close(); }`.
14
+ *
15
+ * {@link openManagedDatabase} owns (1); {@link withManagedDb} owns (2). Each DB
16
+ * module supplies only a path + initializer and gets a `withXDb` loan helper
17
+ * (see `withIndexDb`, `withStateDb`, etc.) so callers never hand-roll the
18
+ * ownership flag or the finally/close again. This is also the one place to add
19
+ * busy-timeout tuning, integrity checks, or test-isolation injection.
20
+ */
21
+ import fs from "node:fs";
22
+ import path from "node:path";
23
+ import { openDatabase } from "./database.js";
24
+ import { applyStandardPragmas } from "./sqlite-pragmas.js";
25
+ /**
26
+ * Open a managed SQLite database: ensure the parent dir exists, open the handle,
27
+ * apply standard pragmas, then run the schema initializer. The single home for
28
+ * the open→pragmas→migrate recipe.
29
+ */
30
+ export function openManagedDatabase(spec) {
31
+ const dir = path.dirname(spec.path);
32
+ if (!fs.existsSync(dir)) {
33
+ fs.mkdirSync(dir, { recursive: true });
34
+ }
35
+ const db = openDatabase(spec.path);
36
+ applyStandardPragmas(db, spec.pragmas ?? { dataDir: dir });
37
+ spec.init?.(db);
38
+ return db;
39
+ }
40
+ /**
41
+ * Run `fn` against a managed database, owning its lifecycle.
42
+ *
43
+ * When `opts.borrowed` is supplied the caller already owns an open handle: it is
44
+ * passed straight through and NOT closed (borrow). Otherwise a fresh handle is
45
+ * opened via `open` and closed in a `finally` (own). This replaces the
46
+ * hand-rolled `ctx?.db ?? open()` + `ownsDb` flag + `finally`/close idiom — the
47
+ * ownership decision and the close live here, once.
48
+ *
49
+ * Synchronous by design: the DB consumers (telemetry writers, planners) finish
50
+ * all work within the tick, matching the inline blocks this replaces.
51
+ */
52
+ export function withManagedDb(open, fn, opts) {
53
+ if (opts?.borrowed) {
54
+ return fn(opts.borrowed);
55
+ }
56
+ const db = open();
57
+ try {
58
+ return fn(db);
59
+ }
60
+ finally {
61
+ db.close();
62
+ }
63
+ }
64
+ /**
65
+ * Async sibling of {@link withManagedDb}. Use this — NOT `withManagedDb` — when
66
+ * `fn` holds the handle across an `await`: the sync version closes in its
67
+ * `finally` before the awaited work resolves (use-after-close). Here the handle
68
+ * is closed only after `fn`'s promise settles. Borrowed handles pass straight
69
+ * through and are not closed, as in the sync version.
70
+ */
71
+ export async function withManagedDbAsync(open, fn, opts) {
72
+ if (opts?.borrowed) {
73
+ return fn(opts.borrowed);
74
+ }
75
+ const db = open();
76
+ try {
77
+ return await fn(db);
78
+ }
79
+ finally {
80
+ db.close();
81
+ }
82
+ }