akm-cli 0.9.0-beta.5 → 0.9.0-beta.51

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. package/CHANGELOG.md +711 -0
  2. package/README.md +12 -4
  3. package/dist/akm +38 -0
  4. package/dist/akm-migrate-storage +38 -0
  5. package/dist/assets/profiles/default.json +9 -4
  6. package/dist/assets/profiles/frequent.json +1 -1
  7. package/dist/assets/profiles/memory-focus.json +1 -1
  8. package/dist/assets/profiles/quick.json +1 -1
  9. package/dist/assets/profiles/synthesize.json +15 -0
  10. package/dist/assets/profiles/thorough.json +1 -1
  11. package/dist/assets/prompts/consolidate-system.md +23 -0
  12. package/dist/assets/prompts/contradiction-judge.md +33 -0
  13. package/dist/assets/prompts/distill-knowledge-system.md +22 -0
  14. package/dist/assets/prompts/distill-lesson-system.md +36 -0
  15. package/dist/assets/prompts/extract-session.md +6 -2
  16. package/dist/assets/prompts/graph-extract-system.md +1 -0
  17. package/dist/assets/prompts/graph-extract-user-prompt.md +1 -1
  18. package/dist/assets/prompts/memory-infer-system.md +1 -0
  19. package/dist/assets/prompts/memory-infer-user.md +5 -0
  20. package/dist/assets/prompts/metadata-enhance-system.md +1 -0
  21. package/dist/assets/prompts/procedural-system.md +44 -0
  22. package/dist/assets/prompts/recombine-system.md +40 -0
  23. package/dist/assets/prompts/staleness-detect-system.md +6 -0
  24. package/dist/assets/prompts/validate-summary-judge.md +1 -0
  25. package/dist/assets/stash-skeleton/facts/conventions/assets/agent.md +38 -0
  26. package/dist/assets/stash-skeleton/facts/conventions/assets/command.md +38 -0
  27. package/dist/assets/stash-skeleton/facts/conventions/assets/fact.md +39 -0
  28. package/dist/assets/stash-skeleton/facts/conventions/assets/knowledge.md +40 -0
  29. package/dist/assets/stash-skeleton/facts/conventions/assets/lesson.md +43 -0
  30. package/dist/assets/stash-skeleton/facts/conventions/assets/memory.md +38 -0
  31. package/dist/assets/stash-skeleton/facts/conventions/assets/script.md +43 -0
  32. package/dist/assets/stash-skeleton/facts/conventions/assets/skill.md +40 -0
  33. package/dist/assets/stash-skeleton/facts/conventions/assets/workflow.md +43 -0
  34. package/dist/assets/templates/html/health.html +281 -111
  35. package/dist/assets/wiki/ingest-workflow-template.md +38 -10
  36. package/dist/cli/parse-args.js +46 -1
  37. package/dist/cli/shared.js +28 -0
  38. package/dist/cli.js +27 -11
  39. package/dist/commands/agent/agent-dispatch.js +2 -2
  40. package/dist/commands/agent/agent-support.js +0 -7
  41. package/dist/commands/agent/contribute-cli.js +17 -4
  42. package/dist/commands/config-cli.js +18 -2
  43. package/dist/commands/env/child-env.js +47 -0
  44. package/dist/commands/env/env-cli.js +33 -26
  45. package/dist/commands/env/secret-cli.js +36 -22
  46. package/dist/commands/feedback-cli.js +15 -6
  47. package/dist/commands/graph/graph-cli.js +5 -13
  48. package/dist/commands/graph/graph.js +76 -72
  49. package/dist/commands/health/checks.js +49 -1
  50. package/dist/commands/health/html-report.js +422 -80
  51. package/dist/commands/health.js +386 -9
  52. package/dist/commands/improve/calibration.js +161 -0
  53. package/dist/commands/improve/consolidate/chunking.js +141 -0
  54. package/dist/commands/improve/consolidate/eligibility.js +81 -0
  55. package/dist/commands/improve/consolidate/merge.js +145 -0
  56. package/dist/commands/improve/consolidate/sanitize.js +231 -0
  57. package/dist/commands/{lint.js → improve/consolidate/types.js} +1 -1
  58. package/dist/commands/improve/consolidate.js +635 -660
  59. package/dist/commands/improve/dedup.js +482 -0
  60. package/dist/commands/improve/distill.js +159 -69
  61. package/dist/commands/improve/eligibility.js +434 -0
  62. package/dist/commands/improve/encoding-salience.js +205 -0
  63. package/dist/commands/improve/extract-cli.js +124 -2
  64. package/dist/commands/improve/extract-prompt.js +39 -2
  65. package/dist/commands/improve/extract-watch.js +140 -0
  66. package/dist/commands/improve/extract.js +389 -40
  67. package/dist/commands/improve/feedback-valence.js +54 -0
  68. package/dist/commands/improve/homeostatic.js +467 -0
  69. package/dist/commands/improve/improve-auto-accept.js +138 -7
  70. package/dist/commands/improve/improve-cli.js +36 -61
  71. package/dist/commands/improve/improve-profiles.js +14 -0
  72. package/dist/commands/improve/improve-result-file.js +14 -25
  73. package/dist/commands/improve/improve-session.js +58 -0
  74. package/dist/commands/improve/improve.js +485 -2498
  75. package/dist/commands/improve/locks.js +154 -0
  76. package/dist/commands/improve/loop-stages.js +1083 -0
  77. package/dist/commands/improve/memory/memory-contradiction-detect.js +23 -28
  78. package/dist/commands/improve/outcome-loop.js +256 -0
  79. package/dist/commands/improve/preparation.js +1966 -0
  80. package/dist/commands/improve/proactive-maintenance.js +115 -0
  81. package/dist/commands/improve/procedural.js +418 -0
  82. package/dist/commands/improve/recombine.js +850 -0
  83. package/dist/commands/improve/reflect-noise.js +0 -0
  84. package/dist/commands/improve/reflect.js +183 -40
  85. package/dist/commands/improve/salience.js +438 -0
  86. package/dist/commands/improve/triage.js +93 -0
  87. package/dist/commands/lint/agent-linter.js +19 -24
  88. package/dist/commands/lint/base-linter.js +173 -60
  89. package/dist/commands/lint/command-linter.js +19 -24
  90. package/dist/commands/lint/env-key-rules.js +38 -1
  91. package/dist/commands/lint/fact-linter.js +39 -0
  92. package/dist/commands/lint/index.js +31 -13
  93. package/dist/commands/lint/memory-linter.js +1 -1
  94. package/dist/commands/lint/registry.js +7 -2
  95. package/dist/commands/lint/task-linter.js +3 -3
  96. package/dist/commands/lint/workflow-linter.js +26 -1
  97. package/dist/commands/proposal/drain-policies.js +5 -0
  98. package/dist/commands/proposal/drain.js +43 -50
  99. package/dist/commands/proposal/proposal-cli.js +21 -31
  100. package/dist/commands/proposal/proposal.js +5 -0
  101. package/dist/commands/proposal/propose.js +7 -2
  102. package/dist/commands/proposal/validators/proposal-quality-validators.js +9 -8
  103. package/dist/commands/proposal/validators/proposals.js +189 -63
  104. package/dist/commands/read/curate.js +414 -94
  105. package/dist/commands/read/knowledge.js +6 -3
  106. package/dist/commands/read/search-cli.js +9 -4
  107. package/dist/commands/read/search.js +10 -6
  108. package/dist/commands/read/show.js +86 -7
  109. package/dist/commands/sources/init.js +49 -17
  110. package/dist/commands/sources/installed-stashes.js +11 -3
  111. package/dist/commands/sources/schema-repair.js +43 -45
  112. package/dist/commands/sources/self-update.js +2 -2
  113. package/dist/commands/sources/source-add.js +7 -3
  114. package/dist/commands/sources/stash-cli.js +28 -40
  115. package/dist/commands/sources/stash-skeleton.js +23 -8
  116. package/dist/commands/tasks/tasks-cli.js +19 -27
  117. package/dist/commands/tasks/tasks.js +39 -11
  118. package/dist/commands/wiki-cli.js +21 -35
  119. package/dist/core/asset/asset-registry.js +3 -1
  120. package/dist/core/asset/asset-spec.js +18 -2
  121. package/dist/core/asset/frontmatter.js +166 -167
  122. package/dist/core/asset/markdown.js +8 -0
  123. package/dist/core/authoring-rules.js +92 -0
  124. package/dist/core/common.js +0 -5
  125. package/dist/core/config/config-migration.js +12 -11
  126. package/dist/core/config/config-schema.js +340 -56
  127. package/dist/core/config/config-types.js +3 -3
  128. package/dist/core/config/config.js +28 -7
  129. package/dist/core/events.js +3 -7
  130. package/dist/core/improve-types.js +11 -8
  131. package/dist/core/logs-db.js +10 -66
  132. package/dist/core/parse.js +36 -16
  133. package/dist/core/paths.js +3 -0
  134. package/dist/core/standards/resolve-standards-context.js +87 -0
  135. package/dist/core/standards/resolve-stash-standards.js +99 -0
  136. package/dist/core/standards/resolve-type-conventions.js +66 -0
  137. package/dist/core/state/migrations.js +714 -0
  138. package/dist/core/state-db.js +525 -474
  139. package/dist/indexer/db/db.js +439 -247
  140. package/dist/indexer/db/graph-db.js +129 -86
  141. package/dist/indexer/ensure-index.js +152 -17
  142. package/dist/indexer/graph/graph-boost.js +51 -41
  143. package/dist/indexer/graph/graph-extraction.js +218 -4
  144. package/dist/indexer/index-writer-lock.js +99 -0
  145. package/dist/indexer/indexer.js +123 -221
  146. package/dist/indexer/passes/dir-staleness.js +114 -0
  147. package/dist/indexer/passes/memory-inference.js +13 -5
  148. package/dist/indexer/passes/staleness-detect.js +2 -5
  149. package/dist/indexer/search/db-search.js +19 -6
  150. package/dist/indexer/search/ranking-contributors.js +22 -0
  151. package/dist/indexer/search/ranking.js +4 -0
  152. package/dist/indexer/search/search-source.js +17 -18
  153. package/dist/indexer/search/semantic-status.js +4 -0
  154. package/dist/indexer/walk/matchers.js +9 -0
  155. package/dist/integrations/agent/config.js +6 -53
  156. package/dist/integrations/agent/index.js +2 -18
  157. package/dist/integrations/agent/prompts.js +75 -9
  158. package/dist/integrations/agent/runner-dispatch.js +59 -0
  159. package/dist/integrations/harnesses/claude/session-log.js +11 -1
  160. package/dist/integrations/harnesses/index.js +2 -3
  161. package/dist/integrations/harnesses/opencode/session-log.js +173 -3
  162. package/dist/integrations/harnesses/opencode-sdk/index.js +2 -2
  163. package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +0 -2
  164. package/dist/integrations/session-logs/index.js +16 -0
  165. package/dist/llm/client.js +45 -15
  166. package/dist/llm/embedder.js +42 -3
  167. package/dist/llm/embedders/deterministic.js +66 -0
  168. package/dist/llm/embedders/local.js +66 -2
  169. package/dist/llm/feature-gate.js +8 -4
  170. package/dist/llm/graph-extract.js +67 -44
  171. package/dist/llm/memory-infer-impl.js +138 -0
  172. package/dist/llm/memory-infer.js +1 -127
  173. package/dist/llm/metadata-enhance.js +44 -31
  174. package/dist/llm/structured-call.js +49 -0
  175. package/dist/migrate-storage-node.mjs +8 -0
  176. package/dist/output/context.js +5 -5
  177. package/dist/output/renderers.js +74 -2
  178. package/dist/output/shapes/curate.js +14 -2
  179. package/dist/output/shapes/passthrough.js +0 -1
  180. package/dist/output/text/helpers.js +16 -1
  181. package/dist/registry/providers/skills-sh.js +21 -147
  182. package/dist/registry/providers/static-index.js +15 -157
  183. package/dist/registry/resolve.js +22 -9
  184. package/dist/runtime.js +25 -1
  185. package/dist/scripts/migrate-storage.js +2617 -1961
  186. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +759 -510
  187. package/dist/setup/setup.js +29 -8
  188. package/dist/sources/include.js +6 -2
  189. package/dist/sources/providers/filesystem.js +0 -1
  190. package/dist/sources/providers/git-install.js +210 -0
  191. package/dist/sources/providers/git-provider.js +234 -0
  192. package/dist/sources/providers/git-stash.js +248 -0
  193. package/dist/sources/providers/git.js +10 -661
  194. package/dist/sources/providers/npm.js +2 -6
  195. package/dist/sources/providers/provider-utils.js +13 -7
  196. package/dist/sources/providers/sync-from-ref.js +9 -1
  197. package/dist/sources/providers/tar-utils.js +16 -8
  198. package/dist/sources/providers/website.js +9 -5
  199. package/dist/sources/website-ingest.js +187 -29
  200. package/dist/sources/wiki-fetchers/registry.js +53 -0
  201. package/dist/sources/wiki-fetchers/youtube.js +239 -0
  202. package/dist/storage/database.js +45 -10
  203. package/dist/storage/managed-db.js +82 -0
  204. package/dist/storage/repositories/registry-cache.js +92 -0
  205. package/dist/storage/sqlite-pragmas.js +146 -0
  206. package/dist/tasks/backends/cron.js +1 -1
  207. package/dist/tasks/backends/launchd.js +1 -1
  208. package/dist/tasks/backends/schtasks.js +1 -1
  209. package/dist/tasks/{resolveAkmBin.js → resolve-akm-bin.js} +2 -2
  210. package/dist/tasks/runner.js +5 -13
  211. package/dist/text-import-hook.mjs +0 -0
  212. package/dist/wiki/wiki.js +37 -0
  213. package/dist/workflows/db.js +3 -4
  214. package/dist/workflows/runtime/runs.js +1 -117
  215. package/dist/workflows/runtime/workflow-asset-loader.js +125 -0
  216. package/dist/workflows/validate-summary.js +2 -7
  217. package/docs/data-and-telemetry.md +3 -2
  218. package/docs/migration/release-notes/0.9.0.md +39 -0
  219. package/package.json +13 -11
  220. package/dist/commands/db-cli.js +0 -23
  221. package/dist/indexer/db/db-backup.js +0 -376
@@ -0,0 +1,239 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ import { fetchWithRetry } from "../../core/common.js";
5
+ const YOUTUBE_HOSTS = new Set(["www.youtube.com", "youtube.com", "m.youtube.com", "youtu.be"]);
6
+ const WATCH_HOST = "https://www.youtube.com/watch?v=";
7
+ // Captions: the WEB watch-page caption baseUrls are now Proof-of-Origin-Token
8
+ // gated (they carry `exp=xpe` and return an empty body without a `pot` token).
9
+ // The ANDROID InnerTube `player` client still returns UNGATED caption baseUrls
10
+ // that serve the transcript, so captions are sourced from there. See
11
+ // https://github.com/yt-dlp/yt-dlp/issues/13075.
12
+ const INNERTUBE_PLAYER_URL = "https://www.youtube.com/youtubei/v1/player";
13
+ // Long-stable public InnerTube key; only authorizes the player call (not auth).
14
+ const DEFAULT_INNERTUBE_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
15
+ const ANDROID_INNERTUBE_CLIENT = {
16
+ clientName: "ANDROID",
17
+ clientVersion: "20.10.38",
18
+ androidSdkVersion: 34,
19
+ hl: "en",
20
+ gl: "US",
21
+ };
22
+ function extractVideoId(url) {
23
+ if (!YOUTUBE_HOSTS.has(url.hostname))
24
+ return null;
25
+ if (url.hostname === "youtu.be") {
26
+ const id = url.pathname.replace(/^\/+/, "").split("/")[0]?.trim();
27
+ return id || null;
28
+ }
29
+ if (url.pathname === "/watch") {
30
+ const id = url.searchParams.get("v")?.trim();
31
+ return id || null;
32
+ }
33
+ if (url.pathname.startsWith("/shorts/")) {
34
+ const id = url.pathname.slice("/shorts/".length).split("/")[0]?.trim();
35
+ return id || null;
36
+ }
37
+ if (url.pathname.startsWith("/embed/")) {
38
+ const id = url.pathname.slice("/embed/".length).split("/")[0]?.trim();
39
+ return id || null;
40
+ }
41
+ return null;
42
+ }
43
+ function canonicalWatchUrl(videoId) {
44
+ return `${WATCH_HOST}${encodeURIComponent(videoId)}`;
45
+ }
46
+ function decodeEntities(value) {
47
+ const safeFromCodePoint = (codePoint, fallback) => {
48
+ if (!Number.isFinite(codePoint) || codePoint < 0 || codePoint > 0x10ffff)
49
+ return fallback;
50
+ try {
51
+ return String.fromCodePoint(codePoint);
52
+ }
53
+ catch {
54
+ return fallback;
55
+ }
56
+ };
57
+ return value
58
+ .replace(/&#x([0-9a-f]+);/gi, (m, hex) => safeFromCodePoint(Number.parseInt(hex, 16), m))
59
+ .replace(/&#([0-9]+);/g, (m, dec) => safeFromCodePoint(Number.parseInt(dec, 10), m))
60
+ .replace(/&quot;/g, '"')
61
+ .replace(/&#39;|&apos;/g, "'")
62
+ .replace(/&lt;/g, "<")
63
+ .replace(/&gt;/g, ">")
64
+ .replace(/&amp;/g, "&");
65
+ }
66
+ function stripTags(value) {
67
+ return value.replace(/<[^>]+>/g, "");
68
+ }
69
+ function extractJsonObjectAfter(marker, source) {
70
+ const markerIndex = source.indexOf(marker);
71
+ if (markerIndex === -1)
72
+ return null;
73
+ const start = source.indexOf("{", markerIndex + marker.length);
74
+ if (start === -1)
75
+ return null;
76
+ let depth = 0;
77
+ let inString = false;
78
+ let escaped = false;
79
+ for (let i = start; i < source.length; i += 1) {
80
+ const char = source[i];
81
+ if (inString) {
82
+ if (escaped) {
83
+ escaped = false;
84
+ }
85
+ else if (char === "\\") {
86
+ escaped = true;
87
+ }
88
+ else if (char === '"') {
89
+ inString = false;
90
+ }
91
+ continue;
92
+ }
93
+ if (char === '"') {
94
+ inString = true;
95
+ continue;
96
+ }
97
+ if (char === "{")
98
+ depth += 1;
99
+ if (char === "}") {
100
+ depth -= 1;
101
+ if (depth === 0)
102
+ return source.slice(start, i + 1);
103
+ }
104
+ }
105
+ return null;
106
+ }
107
+ function parsePlayerResponse(html) {
108
+ const jsonText = extractJsonObjectAfter("ytInitialPlayerResponse =", html);
109
+ if (!jsonText)
110
+ return null;
111
+ try {
112
+ return JSON.parse(jsonText);
113
+ }
114
+ catch {
115
+ return null;
116
+ }
117
+ }
118
+ function chooseCaptionTrack(tracks) {
119
+ const normalized = tracks
120
+ .map((track) => ({
121
+ baseUrl: typeof track.baseUrl === "string" ? track.baseUrl : "",
122
+ languageCode: typeof track.languageCode === "string" ? track.languageCode : "",
123
+ kind: typeof track.kind === "string" ? track.kind : "",
124
+ }))
125
+ .filter((track) => track.baseUrl);
126
+ if (normalized.length === 0)
127
+ return null;
128
+ const preferred = normalized.find((track) => track.languageCode === "en" && track.kind !== "asr") ??
129
+ normalized.find((track) => track.languageCode.startsWith("en") && track.kind !== "asr") ??
130
+ normalized.find((track) => track.languageCode.startsWith("en")) ??
131
+ normalized[0];
132
+ return preferred.baseUrl;
133
+ }
134
+ async function fetchText(url, timeoutMs, signal) {
135
+ const response = await fetchWithRetry(url, {
136
+ headers: {
137
+ Accept: "text/html,application/json,application/xml,text/xml,text/plain;q=0.9,*/*;q=0.1",
138
+ "User-Agent": "akm-cli youtube fetcher",
139
+ },
140
+ signal,
141
+ }, { timeout: timeoutMs, retries: 1 });
142
+ if (!response.ok)
143
+ return null;
144
+ return response.text();
145
+ }
146
+ function parseTranscript(xml) {
147
+ // Two timedtext shapes: legacy srv1 `<text>` cues, and the format-3
148
+ // `<timedtext format="3">` `<p>` cues that the ANDROID InnerTube caption
149
+ // URLs return. Strip any nested `<s>` word tags via stripTags.
150
+ const matches = [...xml.matchAll(/<text\b[^>]*>([\s\S]*?)<\/text>/g), ...xml.matchAll(/<p\b[^>]*>([\s\S]*?)<\/p>/g)];
151
+ const texts = matches.map((match) => decodeEntities(stripTags(match[1] ?? "")).trim()).filter(Boolean);
152
+ if (texts.length === 0)
153
+ return null;
154
+ return texts.join("\n");
155
+ }
156
+ /**
157
+ * Fetch caption tracks via the ANDROID InnerTube `player` endpoint, whose
158
+ * caption baseUrls are not POT-gated. Returns [] on any failure so the caller
159
+ * can fall back to the (usually empty) watch-page tracks and degrade to a
160
+ * description-only snapshot.
161
+ */
162
+ async function fetchInnertubeCaptionTracks(videoId, apiKey, timeoutMs, signal) {
163
+ try {
164
+ const response = await fetchWithRetry(`${INNERTUBE_PLAYER_URL}?key=${encodeURIComponent(apiKey)}`, {
165
+ method: "POST",
166
+ headers: {
167
+ "Content-Type": "application/json",
168
+ "User-Agent": "com.google.android.youtube/20.10.38 (Linux; U; Android 14) gzip",
169
+ },
170
+ body: JSON.stringify({ context: { client: ANDROID_INNERTUBE_CLIENT }, videoId }),
171
+ signal,
172
+ }, { timeout: timeoutMs, retries: 1 });
173
+ if (!response.ok)
174
+ return [];
175
+ const json = (await response.json());
176
+ return json.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
177
+ }
178
+ catch {
179
+ return [];
180
+ }
181
+ }
182
+ /** Extract the page's InnerTube API key, falling back to the stable default. */
183
+ function extractInnertubeKey(html) {
184
+ return html.match(/"INNERTUBE_API_KEY":"([^"]+)"/)?.[1] ?? DEFAULT_INNERTUBE_KEY;
185
+ }
186
+ const youtubeFetcher = {
187
+ name: "youtube-transcript",
188
+ matches(url) {
189
+ return extractVideoId(url) !== null;
190
+ },
191
+ async fetch(url, context) {
192
+ const videoId = extractVideoId(url);
193
+ if (!videoId)
194
+ return null;
195
+ const watchUrl = canonicalWatchUrl(videoId);
196
+ const watchHtml = await fetchText(watchUrl, context.timeoutMs, context.signal);
197
+ if (!watchHtml)
198
+ return null;
199
+ const playerResponse = parsePlayerResponse(watchHtml);
200
+ if (!playerResponse)
201
+ return null;
202
+ const title = typeof playerResponse.videoDetails?.title === "string" ? playerResponse.videoDetails.title.trim() : "";
203
+ const description = typeof playerResponse.videoDetails?.shortDescription === "string"
204
+ ? playerResponse.videoDetails.shortDescription.trim()
205
+ : "";
206
+ const sections = [];
207
+ let hasTranscript = false;
208
+ if (description) {
209
+ sections.push("## Description", "", description);
210
+ }
211
+ // Prefer ungated ANDROID InnerTube caption tracks; fall back to the
212
+ // (now usually empty) watch-page tracks so a description-only snapshot
213
+ // still works when InnerTube is unavailable.
214
+ const apiKey = extractInnertubeKey(watchHtml);
215
+ let tracks = await fetchInnertubeCaptionTracks(videoId, apiKey, context.timeoutMs, context.signal);
216
+ if (tracks.length === 0) {
217
+ tracks = playerResponse.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
218
+ }
219
+ const captionUrl = chooseCaptionTrack(tracks);
220
+ if (captionUrl) {
221
+ const transcriptXml = await fetchText(captionUrl, context.timeoutMs, context.signal);
222
+ const transcript = transcriptXml ? parseTranscript(transcriptXml) : null;
223
+ if (transcript) {
224
+ hasTranscript = true;
225
+ sections.push("## Transcript", "", transcript);
226
+ }
227
+ }
228
+ if (sections.length === 0)
229
+ return null;
230
+ return {
231
+ url: watchUrl,
232
+ title: title || `YouTube ${videoId}`,
233
+ markdown: sections.join("\n"),
234
+ preferredName: `youtube/${videoId}`,
235
+ tags: hasTranscript ? ["youtube", "video", "transcript"] : ["youtube", "video"],
236
+ };
237
+ },
238
+ };
239
+ export default youtubeFetcher;
@@ -35,18 +35,39 @@ const isBun = !!process.versions?.bun;
35
35
  // Bun built-in, unresolvable on Node) nor `better-sqlite3` (an optional native
36
36
  // dep, possibly absent under Bun) is statically imported.
37
37
  const nodeRequire = createRequire(import.meta.url);
38
+ // bun:sqlite — Bun built-in, no native build. Cannot run on Node.
39
+ const bunSqliteProvider = {
40
+ name: "bun:sqlite",
41
+ supported: () => isBun,
42
+ open: openBunDatabase,
43
+ };
44
+ // better-sqlite3 — native Node driver. Cannot run on Bun (oven-sh/bun#4290).
45
+ const nodeSqliteProvider = {
46
+ name: "better-sqlite3",
47
+ supported: () => !isBun,
48
+ open: openNodeDatabase,
49
+ };
38
50
  /**
39
- * Open a SQLite database handle at `path`, selecting the driver for the current
40
- * runtime. Returns a handle conforming to the structural {@link Database} type.
41
- *
42
- * The Node driver (`better-sqlite3`) is required lazily and ONLY when not on
43
- * Bun, so importing this module under Bun never touches `better-sqlite3`.
51
+ * Ordered provider registry. The factory selects the first supported provider.
52
+ * A future Postgres provider is appended here and only here. Both SQLite
53
+ * engines are kept as distinct providers (not collapsed) because no single
54
+ * SQLite driver runs on both Bun and Node today.
44
55
  */
45
- export function openDatabase(path, opts) {
46
- if (isBun) {
47
- return openBunDatabase(path, opts);
56
+ const PROVIDERS = [bunSqliteProvider, nodeSqliteProvider];
57
+ /** Select the provider for the current runtime. */
58
+ function selectProvider() {
59
+ const provider = PROVIDERS.find((p) => p.supported());
60
+ if (!provider) {
61
+ throw new Error(`No storage provider supports the current runtime (${isBun ? "Bun" : "Node"}).`);
48
62
  }
49
- return openNodeDatabase(path, opts);
63
+ return provider;
64
+ }
65
+ /**
66
+ * Open a SQLite database handle at `path` via the active {@link StorageProvider}.
67
+ * Returns a handle conforming to the structural {@link Database} type.
68
+ */
69
+ export function openDatabase(path, opts) {
70
+ return selectProvider().open(path, opts);
50
71
  }
51
72
  function openBunDatabase(path, opts) {
52
73
  const { Database: BunDatabase } = loadBunSqlite();
@@ -81,7 +102,21 @@ function loadBetterSqlite3() {
81
102
  if (!betterSqlite3Ctor) {
82
103
  // Runtime-gated dynamic require: only reached when NOT on Bun, so Bun never
83
104
  // resolves or loads the optional `better-sqlite3` native dependency.
84
- const mod = nodeRequire("better-sqlite3");
105
+ // `better-sqlite3` is an optionalDependency, so `npm i` can succeed without
106
+ // it (or with a native build that failed). Convert the raw MODULE_NOT_FOUND
107
+ // into an actionable message instead of a cryptic onboarding crash.
108
+ let mod;
109
+ try {
110
+ mod = nodeRequire("better-sqlite3");
111
+ }
112
+ catch (err) {
113
+ throw new Error("akm could not load 'better-sqlite3', the SQLite driver it needs on Node.js.\n" +
114
+ " • Reinstall akm with a working C/C++ build toolchain so its optional\n" +
115
+ " 'better-sqlite3' native binding rebuilds (a global `npm i -g better-sqlite3`\n" +
116
+ " will NOT be resolved — Node loads it from akm's own node_modules).\n" +
117
+ " • Or run akm under Bun, which has a built-in SQLite driver and needs no native build.\n" +
118
+ ` Underlying load error: ${err instanceof Error ? err.message : String(err)}`);
119
+ }
85
120
  betterSqlite3Ctor = mod.default ?? mod;
86
121
  }
87
122
  return betterSqlite3Ctor;
@@ -0,0 +1,82 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ /**
5
+ * Managed-database seam — the single home for the SQLite open/lifecycle recipe.
6
+ *
7
+ * Before this module, two idioms were copy-pasted across state.db / logs.db /
8
+ * workflow.db / index.db and their consumers:
9
+ *
10
+ * 1. The open recipe: `mkdir(dir) → openDatabase(path) → applyStandardPragmas
11
+ * → migrate`.
12
+ * 2. The borrow-or-own lifecycle: `const db = ctx?.db ?? open(); const owns =
13
+ * !ctx?.db; try { … } finally { if (owns) db.close(); }`.
14
+ *
15
+ * {@link openManagedDatabase} owns (1); {@link withManagedDb} owns (2). Each DB
16
+ * module supplies only a path + initializer and gets a `withXDb` loan helper
17
+ * (see `withIndexDb`, `withStateDb`, etc.) so callers never hand-roll the
18
+ * ownership flag or the finally/close again. This is also the one place to add
19
+ * busy-timeout tuning, integrity checks, or test-isolation injection.
20
+ */
21
+ import fs from "node:fs";
22
+ import path from "node:path";
23
+ import { openDatabase } from "./database.js";
24
+ import { applyStandardPragmas } from "./sqlite-pragmas.js";
25
+ /**
26
+ * Open a managed SQLite database: ensure the parent dir exists, open the handle,
27
+ * apply standard pragmas, then run the schema initializer. The single home for
28
+ * the open→pragmas→migrate recipe.
29
+ */
30
+ export function openManagedDatabase(spec) {
31
+ const dir = path.dirname(spec.path);
32
+ if (!fs.existsSync(dir)) {
33
+ fs.mkdirSync(dir, { recursive: true });
34
+ }
35
+ const db = openDatabase(spec.path);
36
+ applyStandardPragmas(db, spec.pragmas ?? { dataDir: dir });
37
+ spec.init?.(db);
38
+ return db;
39
+ }
40
+ /**
41
+ * Run `fn` against a managed database, owning its lifecycle.
42
+ *
43
+ * When `opts.borrowed` is supplied the caller already owns an open handle: it is
44
+ * passed straight through and NOT closed (borrow). Otherwise a fresh handle is
45
+ * opened via `open` and closed in a `finally` (own). This replaces the
46
+ * hand-rolled `ctx?.db ?? open()` + `ownsDb` flag + `finally`/close idiom — the
47
+ * ownership decision and the close live here, once.
48
+ *
49
+ * Synchronous by design: the DB consumers (telemetry writers, planners) finish
50
+ * all work within the tick, matching the inline blocks this replaces.
51
+ */
52
+ export function withManagedDb(open, fn, opts) {
53
+ if (opts?.borrowed) {
54
+ return fn(opts.borrowed);
55
+ }
56
+ const db = open();
57
+ try {
58
+ return fn(db);
59
+ }
60
+ finally {
61
+ db.close();
62
+ }
63
+ }
64
+ /**
65
+ * Async sibling of {@link withManagedDb}. Use this — NOT `withManagedDb` — when
66
+ * `fn` holds the handle across an `await`: the sync version closes in its
67
+ * `finally` before the awaited work resolves (use-after-close). Here the handle
68
+ * is closed only after `fn`'s promise settles. Borrowed handles pass straight
69
+ * through and are not closed, as in the sync version.
70
+ */
71
+ export async function withManagedDbAsync(open, fn, opts) {
72
+ if (opts?.borrowed) {
73
+ return fn(opts.borrowed);
74
+ }
75
+ const db = open();
76
+ try {
77
+ return await fn(db);
78
+ }
79
+ finally {
80
+ db.close();
81
+ }
82
+ }
@@ -0,0 +1,92 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ import { rethrowIfTestIsolationError } from "../../core/errors.js";
5
+ import { closeDatabase, getRegistryIndexCache, openIndexDatabase, upsertRegistryIndexCache } from "../../indexer/db/db.js";
6
+ /**
7
+ * RAII-style lifecycle helper for the registry cache DB. Opens the DB (treating
8
+ * a failed open exactly like the legacy fall-through: the bun-test isolation
9
+ * guard is re-thrown, any other failure yields `db = undefined`), runs `fn`,
10
+ * and guarantees the DB is closed in a `finally` after `fn` has fully settled
11
+ * (the await is required: the callbacks are async, and closing before they
12
+ * settle would tear the DB down mid-write).
13
+ */
14
+ export async function withRegistryCacheDb(fn) {
15
+ let db;
16
+ try {
17
+ db = openIndexDatabase();
18
+ }
19
+ catch (err) {
20
+ // Never mask the bun-test isolation guard as "DB unavailable".
21
+ rethrowIfTestIsolationError(err);
22
+ db = undefined;
23
+ }
24
+ try {
25
+ return await fn(db);
26
+ }
27
+ finally {
28
+ if (db) {
29
+ try {
30
+ closeDatabase(db);
31
+ }
32
+ catch {
33
+ /* ignore */
34
+ }
35
+ }
36
+ }
37
+ }
38
+ /**
39
+ * Shared registry index-cache fetch template. Opens the cache DB, returns a
40
+ * fresh cache hit when present, otherwise fetches live (writing the result back
41
+ * to the cache best-effort), and falls back to a stale cache row when the fetch
42
+ * fails. Behaviour-preserving extraction of the logic previously duplicated in
43
+ * `skills-sh.ts` (`fetchSkills`) and `static-index.ts` (`loadIndex`).
44
+ */
45
+ export async function fetchCachedJson(opts) {
46
+ const { cacheKey, ttlMs, parseCache, fetchFresh } = opts;
47
+ return withRegistryCacheDb(async (db) => {
48
+ // ── Step 1: Try DB cache (index.db) ─────────────────────────────────────
49
+ let dbCacheResult;
50
+ try {
51
+ if (db) {
52
+ dbCacheResult = getRegistryIndexCache(db, cacheKey, ttlMs);
53
+ }
54
+ }
55
+ catch (err) {
56
+ // Never mask the bun-test isolation guard as "DB unavailable" — see
57
+ // rethrowIfTestIsolationError in src/core/errors.ts. Without this, a
58
+ // leaky test silently gets a cold cache instead of the loud
59
+ // TEST_ISOLATION_MISSING failure the guard intends.
60
+ rethrowIfTestIsolationError(err);
61
+ // index.db read failed (pre-migration install or test env) — fall through
62
+ }
63
+ if (dbCacheResult) {
64
+ const cached = parseCache(dbCacheResult.indexJson, { stale: false });
65
+ if (cached !== undefined) {
66
+ return cached;
67
+ }
68
+ }
69
+ // ── Step 2: Fetch fresh ──────────────────────────────────────────────────
70
+ try {
71
+ const { value, cacheJson, cacheOpts } = await fetchFresh(db);
72
+ if (db) {
73
+ try {
74
+ upsertRegistryIndexCache(db, cacheKey, cacheJson, cacheOpts);
75
+ }
76
+ catch {
77
+ /* best-effort */
78
+ }
79
+ }
80
+ return value;
81
+ }
82
+ catch (err) {
83
+ // Fetch failed — use stale DB cache if available.
84
+ if (dbCacheResult) {
85
+ const stale = parseCache(dbCacheResult.indexJson, { stale: true });
86
+ if (stale !== undefined)
87
+ return stale;
88
+ }
89
+ throw err;
90
+ }
91
+ });
92
+ }
@@ -0,0 +1,146 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ /**
5
+ * Shared SQLite PRAGMA application + journal-mode resolution (#628).
6
+ *
7
+ * Every AKM SQLite opener applies the same opening PRAGMAs: a journal mode, a
8
+ * 30 s busy_timeout, and (for most) foreign_keys = ON. Historically each opener
9
+ * hard-coded `PRAGMA journal_mode = WAL`. WAL is impossible on network
10
+ * filesystems (NFS/SMB) because its `-shm` shared-memory wal-index cannot be
11
+ * mmap'd over a network mount, so AKM could not run with its data dir on a
12
+ * network share.
13
+ *
14
+ * This module centralises that PRAGMA block behind {@link applyStandardPragmas}
15
+ * and makes the journal mode configurable via the `AKM_SQLITE_JOURNAL_MODE`
16
+ * env var (WAL | DELETE | TRUNCATE; default WAL = unchanged behaviour). When the
17
+ * mode is the WAL default and the data directory is detected to live on a
18
+ * network filesystem, it auto-falls-back to DELETE with a one-line warning.
19
+ *
20
+ * Boundary note: this is a PLAIN module, not a runtime-boundary file. It only
21
+ * does pure string work plus `db.exec()` (allowed everywhere). The single
22
+ * runtime primitive it needs — a filesystem-type probe (statfs) — lives in
23
+ * src/runtime.ts and is injected here, keeping the network-FS classifier a
24
+ * pure, unit-testable function.
25
+ *
26
+ * @module storage/sqlite-pragmas
27
+ */
28
+ import { warn } from "../core/warn.js";
29
+ import { statfsType } from "../runtime.js";
30
+ const VALID_MODES = new Set(["WAL", "DELETE", "TRUNCATE"]);
31
+ // One-shot warning guards so a misconfigured env var or a network-FS fallback
32
+ // each emit AT MOST ONCE per process rather than on every db open.
33
+ let warnedInvalid = false;
34
+ let warnedNetworkFallback = false;
35
+ /**
36
+ * Resolve a raw `AKM_SQLITE_JOURNAL_MODE` value to a canonical {@link JournalMode}.
37
+ *
38
+ * PURE and unit-testable: the raw string is passed in (not read from
39
+ * `process.env` here). Trims + uppercases; an empty/undefined value yields the
40
+ * WAL default; a recognised value yields its canonical form; any other
41
+ * non-empty value warns once and falls back to WAL. Never throws.
42
+ */
43
+ export function resolveJournalMode(raw) {
44
+ if (raw === undefined)
45
+ return "WAL";
46
+ const normalized = raw.trim().toUpperCase();
47
+ if (normalized === "")
48
+ return "WAL";
49
+ if (VALID_MODES.has(normalized)) {
50
+ return normalized;
51
+ }
52
+ warnInvalidJournalModeOnce(raw);
53
+ return "WAL";
54
+ }
55
+ /**
56
+ * The single env-reading seam: resolve the configured journal mode from
57
+ * `process.env.AKM_SQLITE_JOURNAL_MODE`. Read at call time (per open) so tests
58
+ * that set the env per-case see the right value and we avoid stale-env flakes.
59
+ */
60
+ export function resolveConfiguredJournalMode() {
61
+ return resolveJournalMode(process.env.AKM_SQLITE_JOURNAL_MODE);
62
+ }
63
+ function warnInvalidJournalModeOnce(raw) {
64
+ if (warnedInvalid)
65
+ return;
66
+ warnedInvalid = true;
67
+ warn(`[akm] invalid AKM_SQLITE_JOURNAL_MODE=${JSON.stringify(raw)} — using WAL (valid: WAL, DELETE, TRUNCATE)`);
68
+ }
69
+ // Known Linux f_type magic numbers for network filesystems. node's statfs
70
+ // normalises `type` to this numeric f_type magic on all platforms.
71
+ const FS_MAGIC_NFS = 0x6969; // 26985 — NFS
72
+ const FS_MAGIC_SMB = 0x517b; // 20859 — older SMB_SUPER_MAGIC
73
+ const FS_MAGIC_CIFS = 0xff534d42; // 4283649346 — SMB/CIFS
74
+ const FS_MAGIC_SMB2 = 0xfe534d42; // 4267272514 — SMB2
75
+ const FS_MAGIC_FUSE = 0x65735546; // 1702057286 — FUSE (sshfs + many network FUSE mounts)
76
+ const NETWORK_FS_MAGICS = new Set([
77
+ FS_MAGIC_NFS,
78
+ FS_MAGIC_SMB,
79
+ FS_MAGIC_CIFS,
80
+ FS_MAGIC_SMB2,
81
+ // FUSE is a judgment call: it backs BOTH network mounts (sshfs) and local
82
+ // mounts (some encrypted/overlay FS). Treating it as network falls back to
83
+ // DELETE — conservative-but-safe (DELETE works everywhere; the only cost is
84
+ // losing WAL concurrency). An operator can always force WAL via the env var.
85
+ FS_MAGIC_FUSE,
86
+ ]);
87
+ /**
88
+ * PURE classifier: is `fsType` a known network-filesystem magic number?
89
+ * Returns false for `undefined` (probe failed/unsupported) and for local
90
+ * magics (ext4 0xEF53, btrfs, xfs, tmpfs, apfs, …). Unit-testable with
91
+ * injected magic numbers — no real mount required.
92
+ */
93
+ export function isNetworkFilesystem(fsType) {
94
+ if (fsType === undefined)
95
+ return false;
96
+ return NETWORK_FS_MAGICS.has(fsType);
97
+ }
98
+ /**
99
+ * Apply AKM's standard opening PRAGMAs to `db`, in order:
100
+ * 1. `journal_mode` = the configured mode (with WAL→DELETE network-FS fallback)
101
+ * 2. `busy_timeout = 30000`
102
+ * 3. `foreign_keys = ON` (unless `opts.foreignKeys === false`)
103
+ * 4. `synchronous = FULL` (only in a rollback-journal mode — DELETE/TRUNCATE)
104
+ *
105
+ * Returns the effective {@link JournalMode} so callers/tests can assert it.
106
+ *
107
+ * `synchronous = FULL` is set explicitly only in DELETE/TRUNCATE so durability
108
+ * intent is clear: rollback journals need FULL for crash-durability across
109
+ * power loss, whereas WAL is durable at NORMAL. SQLite's default synchronous is
110
+ * already FULL when unset, so this never changes WAL-default behaviour — the
111
+ * WAL path emits no `synchronous` pragma, exactly as before.
112
+ */
113
+ export function applyStandardPragmas(db, opts = {}) {
114
+ let mode = resolveConfiguredJournalMode();
115
+ // Network-FS fallback only fires for the WAL default and only when we have a
116
+ // directory to probe. An explicitly-requested DELETE/TRUNCATE is never
117
+ // overridden, and a failed/unsupported probe (undefined) keeps WAL.
118
+ if (mode === "WAL" && opts.dataDir) {
119
+ const probe = opts.fsTypeProbe ?? statfsType;
120
+ if (isNetworkFilesystem(probe(opts.dataDir))) {
121
+ mode = "DELETE";
122
+ warnNetworkFallbackOnce(opts.dataDir);
123
+ }
124
+ }
125
+ // PRAGMAs must run before any DDL or DML. busy_timeout is applied FIRST so a
126
+ // journal-mode change that must reclaim a leftover `-wal` file (WAL→DELETE on
127
+ // reopen of an unclean WAL db, e.g. after a crash) can wait out a transient
128
+ // lock instead of failing immediately with SQLITE_BUSY. For the WAL default
129
+ // this is a no-op (WAL→WAL changes nothing), so byte-identical behaviour is
130
+ // preserved.
131
+ db.exec("PRAGMA busy_timeout = 30000");
132
+ db.exec(`PRAGMA journal_mode = ${mode}`);
133
+ if (opts.foreignKeys !== false) {
134
+ db.exec("PRAGMA foreign_keys = ON");
135
+ }
136
+ if (mode !== "WAL") {
137
+ db.exec("PRAGMA synchronous = FULL");
138
+ }
139
+ return mode;
140
+ }
141
+ function warnNetworkFallbackOnce(dataDir) {
142
+ if (warnedNetworkFallback)
143
+ return;
144
+ warnedNetworkFallback = true;
145
+ warn(`[akm] network filesystem detected at ${dataDir} — WAL unsupported, using DELETE journal mode`);
146
+ }
@@ -29,7 +29,7 @@ import fs from "node:fs";
29
29
  import path from "node:path";
30
30
  import { ConfigError } from "../../core/errors.js";
31
31
  import { getTaskLogDir } from "../../core/paths.js";
32
- import { resolveAkmInvocation } from "../resolveAkmBin.js";
32
+ import { resolveAkmInvocation } from "../resolve-akm-bin.js";
33
33
  import { parseSchedule, translateToCron } from "../schedule.js";
34
34
  const BEGIN = (id) => `# akm:task ${id} BEGIN`;
35
35
  const END = (id) => `# akm:task ${id} END`;
@@ -27,7 +27,7 @@ import path from "node:path";
27
27
  import launchdTemplate from "../../assets/backends/launchd-template.xml" with { type: "text" };
28
28
  import { ConfigError } from "../../core/errors.js";
29
29
  import { getTaskLogDir } from "../../core/paths.js";
30
- import { resolveAkmInvocation } from "../resolveAkmBin.js";
30
+ import { resolveAkmInvocation } from "../resolve-akm-bin.js";
31
31
  import { parseSchedule, translateToLaunchd } from "../schedule.js";
32
32
  import { escapeXml, nodeExec, nodeFs } from "./exec-utils.js";
33
33
  export const LAUNCHD_LABEL_PREFIX = "com.akm.task.";
@@ -35,7 +35,7 @@ import path from "node:path";
35
35
  import schtasksTemplate from "../../assets/backends/schtasks-template.xml" with { type: "text" };
36
36
  import { ConfigError } from "../../core/errors.js";
37
37
  import { getTaskLogDir } from "../../core/paths.js";
38
- import { resolveAkmInvocation } from "../resolveAkmBin.js";
38
+ import { resolveAkmInvocation } from "../resolve-akm-bin.js";
39
39
  import { parseSchedule, translateToSchtasks } from "../schedule.js";
40
40
  import { escapeXml, nodeExec, nodeFs } from "./exec-utils.js";
41
41
  export const DEFAULT_FOLDER_PREFIX = "\\akm\\";
@@ -44,8 +44,8 @@ export function resolveAkmInvocation(options = {}) {
44
44
  /**
45
45
  * From the URL of a module inside `src/tasks/` figure out the CLI entry.
46
46
  *
47
- * • dev `…/src/tasks/resolveAkmBin.ts` → `…/src/cli.ts`
48
- * • build `…/dist/tasks/resolveAkmBin.js` → `…/dist/cli.js`
47
+ * • dev `…/src/tasks/resolve-akm-bin.ts` → `…/src/cli.ts`
48
+ * • build `…/dist/tasks/resolve-akm-bin.js` → `…/dist/cli.js`
49
49
  */
50
50
  function resolveCliEntry(moduleUrl) {
51
51
  let modulePath;