akm-cli 0.9.0-beta.45 → 0.9.0-beta.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/shared.js +28 -0
- package/dist/cli.js +1 -2
- package/dist/commands/env/env-cli.js +16 -24
- package/dist/commands/env/secret-cli.js +12 -20
- package/dist/commands/graph/graph-cli.js +5 -13
- package/dist/commands/graph/graph.js +3 -3
- package/dist/commands/improve/consolidate/chunking.js +141 -0
- package/dist/commands/improve/consolidate/eligibility.js +64 -0
- package/dist/commands/improve/consolidate/merge.js +145 -0
- package/dist/commands/improve/consolidate/sanitize.js +231 -0
- package/dist/commands/improve/consolidate/types.js +4 -0
- package/dist/commands/improve/consolidate.js +20 -571
- package/dist/commands/improve/distill.js +5 -9
- package/dist/commands/improve/eligibility.js +434 -0
- package/dist/commands/improve/extract-cli.js +9 -1
- package/dist/commands/improve/extract.js +5 -19
- package/dist/commands/improve/improve-auto-accept.js +4 -8
- package/dist/commands/improve/improve-cli.js +35 -60
- package/dist/commands/improve/improve-result-file.js +5 -23
- package/dist/commands/improve/improve-session.js +58 -0
- package/dist/commands/improve/improve.js +107 -3606
- package/dist/commands/improve/locks.js +154 -0
- package/dist/commands/improve/loop-stages.js +1079 -0
- package/dist/commands/improve/preparation.js +1963 -0
- package/dist/commands/improve/recombine.js +6 -12
- package/dist/commands/improve/reflect.js +29 -34
- package/dist/commands/proposal/drain.js +25 -48
- package/dist/commands/proposal/proposal-cli.js +21 -31
- package/dist/commands/proposal/validators/proposals.js +3 -7
- package/dist/commands/read/curate.js +70 -14
- package/dist/commands/read/knowledge.js +2 -2
- package/dist/commands/sources/self-update.js +2 -2
- package/dist/commands/sources/stash-cli.js +9 -37
- package/dist/commands/tasks/tasks-cli.js +19 -27
- package/dist/commands/wiki-cli.js +21 -35
- package/dist/core/config/config.js +18 -2
- package/dist/core/events.js +3 -7
- package/dist/core/logs-db.js +6 -63
- package/dist/core/state/migrations.js +714 -0
- package/dist/core/state-db.js +28 -779
- package/dist/indexer/db/db.js +82 -216
- package/dist/indexer/indexer.js +11 -112
- package/dist/indexer/passes/dir-staleness.js +114 -0
- package/dist/indexer/search/search-source.js +10 -24
- package/dist/integrations/agent/runner-dispatch.js +59 -0
- package/dist/llm/client.js +22 -11
- package/dist/llm/graph-extract.js +28 -39
- package/dist/llm/memory-infer.js +34 -22
- package/dist/llm/metadata-enhance.js +35 -30
- package/dist/llm/structured-call.js +49 -0
- package/dist/output/shapes/passthrough.js +0 -1
- package/dist/registry/providers/skills-sh.js +21 -147
- package/dist/registry/providers/static-index.js +15 -157
- package/dist/registry/resolve.js +22 -9
- package/dist/scripts/migrate-storage.js +892 -1186
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +214 -179
- package/dist/setup/setup.js +26 -5
- package/dist/sources/providers/filesystem.js +0 -1
- package/dist/sources/providers/git-install.js +206 -0
- package/dist/sources/providers/git-provider.js +234 -0
- package/dist/sources/providers/git-stash.js +248 -0
- package/dist/sources/providers/git.js +10 -671
- package/dist/sources/providers/npm.js +2 -6
- package/dist/sources/providers/sync-from-ref.js +9 -1
- package/dist/sources/providers/website.js +2 -3
- package/dist/sources/website-ingest.js +51 -9
- package/dist/sources/wiki-fetchers/registry.js +53 -0
- package/dist/sources/wiki-fetchers/youtube.js +181 -0
- package/dist/storage/database.js +45 -10
- package/dist/storage/managed-db.js +82 -0
- package/dist/storage/repositories/registry-cache.js +92 -0
- package/dist/tasks/runner.js +5 -13
- package/dist/workflows/runtime/runs.js +1 -117
- package/dist/workflows/runtime/workflow-asset-loader.js +125 -0
- package/package.json +5 -5
- package/dist/commands/db-cli.js +0 -23
- package/dist/indexer/db/db-backup.js +0 -376
|
@@ -36,18 +36,14 @@ class NpmSourceProvider {
|
|
|
36
36
|
this.#config = config;
|
|
37
37
|
this.name = config.name ?? config.url ?? "npm";
|
|
38
38
|
}
|
|
39
|
-
async init(_ctx) {
|
|
40
|
-
// Resolution happens lazily in path(): until `sync()` runs there's no
|
|
41
|
-
// reliable on-disk path. Init is the registration handshake.
|
|
42
|
-
}
|
|
43
39
|
path() {
|
|
44
40
|
if (this.#config.path)
|
|
45
41
|
return this.#config.path;
|
|
46
42
|
throw new ConfigError(`npm source "${this.name}" has no resolved content path — run \`akm update\` to sync it before indexing.`);
|
|
47
43
|
}
|
|
48
|
-
async sync() {
|
|
44
|
+
async sync(options) {
|
|
49
45
|
const ref = npmRefFromConfig(this.#config);
|
|
50
|
-
await syncNpmRef(ref);
|
|
46
|
+
await syncNpmRef(ref, { force: options?.force });
|
|
51
47
|
}
|
|
52
48
|
}
|
|
53
49
|
registerSourceProvider("npm", (config) => new NpmSourceProvider(config));
|
|
@@ -1,6 +1,13 @@
|
|
|
1
1
|
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
2
|
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
3
|
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/**
|
|
5
|
+
* Unified install-ref dispatcher.
|
|
6
|
+
*
|
|
7
|
+
* Replaces the historical `installRegistryRef()` entry point. Given an
|
|
8
|
+
* unparsed install ref, this resolves the right syncable provider and
|
|
9
|
+
* invokes its `sync()` method.
|
|
10
|
+
*/
|
|
4
11
|
import { UsageError } from "../../core/errors.js";
|
|
5
12
|
import { parseRegistryRef } from "../../registry/resolve.js";
|
|
6
13
|
import { detectStashRoot } from "./provider-utils.js";
|
|
@@ -18,7 +25,8 @@ export async function syncFromRef(ref, options) {
|
|
|
18
25
|
return syncRegistryGitRef(ref, options);
|
|
19
26
|
}
|
|
20
27
|
// Exhaustiveness — `parseRegistryRef` only emits the four sources above.
|
|
21
|
-
|
|
28
|
+
const _exhaustive = parsed;
|
|
29
|
+
throw new UsageError(`No syncable provider for ref: ${ref}`);
|
|
22
30
|
}
|
|
23
31
|
function syncLocalRef(parsed, options) {
|
|
24
32
|
const stashRoot = detectStashRoot(parsed.sourcePath);
|
|
@@ -12,12 +12,11 @@ registerSourceProvider("website", (config) => {
|
|
|
12
12
|
return {
|
|
13
13
|
kind: "website",
|
|
14
14
|
name,
|
|
15
|
-
async init(_ctx) { },
|
|
16
15
|
path() {
|
|
17
16
|
return getWebsiteCachePaths(url).stashDir;
|
|
18
17
|
},
|
|
19
|
-
async sync() {
|
|
20
|
-
await ensureWebsiteMirror(config, { requireStashDir: true });
|
|
18
|
+
async sync(options) {
|
|
19
|
+
await ensureWebsiteMirror(config, { requireStashDir: true, force: options?.force });
|
|
21
20
|
},
|
|
22
21
|
};
|
|
23
22
|
});
|
|
@@ -4,11 +4,12 @@
|
|
|
4
4
|
import { createHash } from "node:crypto";
|
|
5
5
|
import fs from "node:fs";
|
|
6
6
|
import path from "node:path";
|
|
7
|
-
import { fetchWithRetry, ResponseTooLargeError, readBodyWithByteCap } from "../core/common.js";
|
|
7
|
+
import { fetchWithRetry, ResponseTooLargeError, readBodyWithByteCap, resolveStashDir } from "../core/common.js";
|
|
8
8
|
import { ConfigError, UsageError } from "../core/errors.js";
|
|
9
9
|
import { getRegistryIndexCacheDir } from "../core/paths.js";
|
|
10
10
|
import { warn } from "../core/warn.js";
|
|
11
11
|
import { isExpired, sanitizeString } from "./providers/provider-utils.js";
|
|
12
|
+
import { loadWikiSnapshotFetchers } from "./wiki-fetchers/registry.js";
|
|
12
13
|
/** Refresh website snapshots every 12 hours to balance freshness with scraping load. */
|
|
13
14
|
const CACHE_REFRESH_INTERVAL_MS = 12 * 60 * 60 * 1000;
|
|
14
15
|
/** Allow up to 7 days of stale snapshots when refresh fails so search remains available during outages. */
|
|
@@ -30,6 +31,16 @@ const WEBSITE_PAGE_BYTE_CAP = 5 * 1024 * 1024;
|
|
|
30
31
|
* whole crawl and return what we have when time runs out.
|
|
31
32
|
*/
|
|
32
33
|
const WEBSITE_CRAWL_WALL_CLOCK_MS = 10 * 60 * 1000;
|
|
34
|
+
function resolveFetcherStashDir(explicitStashDir) {
|
|
35
|
+
if (explicitStashDir)
|
|
36
|
+
return explicitStashDir;
|
|
37
|
+
try {
|
|
38
|
+
return resolveStashDir({ readOnly: true });
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
33
44
|
export function getWebsiteCachePaths(siteUrl) {
|
|
34
45
|
const key = createHash("sha256").update(normalizeSiteUrl(siteUrl)).digest("hex").slice(0, 16);
|
|
35
46
|
const rootDir = path.join(getRegistryIndexCacheDir(), `website-${key}`);
|
|
@@ -114,20 +125,51 @@ async function scrapeWebsiteToStash(startUrl, stashDir, options) {
|
|
|
114
125
|
fs.writeFileSync(filePath, buildMarkdownSnapshot(page, slug), "utf8");
|
|
115
126
|
}
|
|
116
127
|
}
|
|
117
|
-
export async function fetchWebsiteMarkdownSnapshot(rawUrl) {
|
|
128
|
+
export async function fetchWebsiteMarkdownSnapshot(rawUrl, options) {
|
|
118
129
|
const normalizedUrl = validateWebsiteInputUrl(rawUrl);
|
|
130
|
+
const parsedUrl = new URL(normalizedUrl);
|
|
131
|
+
const stashDir = resolveFetcherStashDir(options?.stashDir);
|
|
132
|
+
const context = {
|
|
133
|
+
stashDir: stashDir ?? "",
|
|
134
|
+
timeoutMs: options?.timeoutMs ?? 15_000,
|
|
135
|
+
signal: options?.signal,
|
|
136
|
+
};
|
|
137
|
+
for (const fetcher of await loadWikiSnapshotFetchers(stashDir)) {
|
|
138
|
+
try {
|
|
139
|
+
if (!fetcher.matches(parsedUrl, context))
|
|
140
|
+
continue;
|
|
141
|
+
const snapshot = await fetcher.fetch(parsedUrl, context);
|
|
142
|
+
if (!snapshot)
|
|
143
|
+
continue;
|
|
144
|
+
return websiteMarkdownSnapshotFromResult(snapshot);
|
|
145
|
+
}
|
|
146
|
+
catch (error) {
|
|
147
|
+
warn("[akm] wiki-fetcher %s threw on %s: %s", fetcher.name, normalizedUrl, error instanceof Error ? error.message : String(error));
|
|
148
|
+
}
|
|
149
|
+
}
|
|
119
150
|
const fetched = await fetchWebsitePage(normalizedUrl);
|
|
120
151
|
if (!fetched) {
|
|
121
152
|
throw new UsageError(`No content could be fetched from ${normalizedUrl}`);
|
|
122
153
|
}
|
|
123
|
-
|
|
124
|
-
const slug = preferredName.split("/").pop() ?? preferredName;
|
|
125
|
-
return {
|
|
154
|
+
return websiteMarkdownSnapshotFromResult({
|
|
126
155
|
url: fetched.page.url,
|
|
127
156
|
title: fetched.page.title,
|
|
128
157
|
markdown: fetched.page.markdown,
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
function websiteMarkdownSnapshotFromResult(snapshot) {
|
|
161
|
+
const preferredName = snapshot.preferredName ?? deriveImportPath(snapshot.url);
|
|
162
|
+
const slug = preferredName.split("/").pop() ?? preferredName;
|
|
163
|
+
return {
|
|
164
|
+
url: snapshot.url,
|
|
165
|
+
title: snapshot.title,
|
|
166
|
+
markdown: snapshot.markdown,
|
|
129
167
|
preferredName,
|
|
130
|
-
content: buildMarkdownSnapshot(
|
|
168
|
+
content: buildMarkdownSnapshot({
|
|
169
|
+
url: snapshot.url,
|
|
170
|
+
title: snapshot.title,
|
|
171
|
+
markdown: snapshot.markdown,
|
|
172
|
+
}, slug || "website", snapshot.tags),
|
|
131
173
|
};
|
|
132
174
|
}
|
|
133
175
|
async function crawlWebsite(startUrl, options) {
|
|
@@ -216,11 +258,12 @@ async function fetchWebsitePage(pageUrl) {
|
|
|
216
258
|
links: [],
|
|
217
259
|
};
|
|
218
260
|
}
|
|
219
|
-
function buildMarkdownSnapshot(page, slug) {
|
|
261
|
+
function buildMarkdownSnapshot(page, slug, tags) {
|
|
220
262
|
const title = sanitizeString(page.title, 200) || slug;
|
|
221
263
|
const description = sanitizeString(`Snapshot of ${page.url}`, 500);
|
|
222
264
|
const host = sanitizeString(new URL(page.url).hostname, 120);
|
|
223
265
|
const content = page.markdown.trim() || `Source: ${page.url}`;
|
|
266
|
+
const normalizedTags = Array.from(new Set(["website", host, ...(tags ?? [])]));
|
|
224
267
|
return [
|
|
225
268
|
"---",
|
|
226
269
|
`name: ${JSON.stringify(slug)}`,
|
|
@@ -228,8 +271,7 @@ function buildMarkdownSnapshot(page, slug) {
|
|
|
228
271
|
`sourceUrl: ${JSON.stringify(page.url)}`,
|
|
229
272
|
`title: ${JSON.stringify(title)}`,
|
|
230
273
|
"tags:",
|
|
231
|
-
` - ${JSON.stringify(
|
|
232
|
-
` - ${JSON.stringify(host)}`,
|
|
274
|
+
...normalizedTags.map((tag) => ` - ${JSON.stringify(tag)}`),
|
|
233
275
|
"---",
|
|
234
276
|
"",
|
|
235
277
|
`# ${title}`,
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
import fs from "node:fs";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import { pathToFileURL } from "node:url";
|
|
7
|
+
import { warn } from "../../core/warn.js";
|
|
8
|
+
import youtubeFetcher from "./youtube.js";
|
|
9
|
+
const FETCHER_DIR = path.join("scripts", "wiki-fetchers");
|
|
10
|
+
const FETCHER_FILE_PATTERN = /\.(?:ts|js|mjs)$/i;
|
|
11
|
+
const BUILTIN_FETCHERS = [youtubeFetcher];
|
|
12
|
+
function isWikiSnapshotFetcher(value) {
|
|
13
|
+
if (!value || typeof value !== "object")
|
|
14
|
+
return false;
|
|
15
|
+
const candidate = value;
|
|
16
|
+
return (typeof candidate.name === "string" &&
|
|
17
|
+
typeof candidate.matches === "function" &&
|
|
18
|
+
typeof candidate.fetch === "function");
|
|
19
|
+
}
|
|
20
|
+
export async function loadWikiSnapshotFetchers(stashDir) {
|
|
21
|
+
const fetchers = [];
|
|
22
|
+
if (stashDir) {
|
|
23
|
+
const fetcherDir = path.join(stashDir, FETCHER_DIR);
|
|
24
|
+
try {
|
|
25
|
+
const entries = fs.readdirSync(fetcherDir).sort();
|
|
26
|
+
for (const entry of entries) {
|
|
27
|
+
if (!FETCHER_FILE_PATTERN.test(entry))
|
|
28
|
+
continue;
|
|
29
|
+
try {
|
|
30
|
+
const fileUrl = pathToFileURL(path.join(fetcherDir, entry)).toString();
|
|
31
|
+
const mod = await import(fileUrl);
|
|
32
|
+
if (isWikiSnapshotFetcher(mod.default)) {
|
|
33
|
+
fetchers.push(mod.default);
|
|
34
|
+
}
|
|
35
|
+
else {
|
|
36
|
+
warn("[akm] wiki-fetcher %s skipped: missing { name, matches, fetch }", entry);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
catch (error) {
|
|
40
|
+
warn("[akm] wiki-fetcher %s failed to load: %s", entry, error instanceof Error ? error.message : String(error));
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
catch (error) {
|
|
45
|
+
const code = typeof error === "object" && error && "code" in error ? error.code : undefined;
|
|
46
|
+
if (code !== "ENOENT") {
|
|
47
|
+
warn("[akm] wiki-fetcher directory %s could not be read: %s", fetcherDir, error instanceof Error ? error.message : String(error));
|
|
48
|
+
}
|
|
49
|
+
// Missing directory means no custom fetchers.
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
return [...fetchers, ...BUILTIN_FETCHERS];
|
|
53
|
+
}
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
import { fetchWithRetry } from "../../core/common.js";
|
|
5
|
+
const YOUTUBE_HOSTS = new Set(["www.youtube.com", "youtube.com", "m.youtube.com", "youtu.be"]);
|
|
6
|
+
const WATCH_HOST = "https://www.youtube.com/watch?v=";
|
|
7
|
+
function extractVideoId(url) {
|
|
8
|
+
if (!YOUTUBE_HOSTS.has(url.hostname))
|
|
9
|
+
return null;
|
|
10
|
+
if (url.hostname === "youtu.be") {
|
|
11
|
+
const id = url.pathname.replace(/^\/+/, "").split("/")[0]?.trim();
|
|
12
|
+
return id || null;
|
|
13
|
+
}
|
|
14
|
+
if (url.pathname === "/watch") {
|
|
15
|
+
const id = url.searchParams.get("v")?.trim();
|
|
16
|
+
return id || null;
|
|
17
|
+
}
|
|
18
|
+
if (url.pathname.startsWith("/shorts/")) {
|
|
19
|
+
const id = url.pathname.slice("/shorts/".length).split("/")[0]?.trim();
|
|
20
|
+
return id || null;
|
|
21
|
+
}
|
|
22
|
+
return null;
|
|
23
|
+
}
|
|
24
|
+
function canonicalWatchUrl(videoId) {
|
|
25
|
+
return `${WATCH_HOST}${encodeURIComponent(videoId)}`;
|
|
26
|
+
}
|
|
27
|
+
function decodeEntities(value) {
|
|
28
|
+
const safeFromCodePoint = (codePoint, fallback) => {
|
|
29
|
+
if (!Number.isFinite(codePoint) || codePoint < 0 || codePoint > 0x10ffff)
|
|
30
|
+
return fallback;
|
|
31
|
+
try {
|
|
32
|
+
return String.fromCodePoint(codePoint);
|
|
33
|
+
}
|
|
34
|
+
catch {
|
|
35
|
+
return fallback;
|
|
36
|
+
}
|
|
37
|
+
};
|
|
38
|
+
return value
|
|
39
|
+
.replace(/&#x([0-9a-f]+);/gi, (m, hex) => safeFromCodePoint(Number.parseInt(hex, 16), m))
|
|
40
|
+
.replace(/&#([0-9]+);/g, (m, dec) => safeFromCodePoint(Number.parseInt(dec, 10), m))
|
|
41
|
+
.replace(/"/g, '"')
|
|
42
|
+
.replace(/'|'/g, "'")
|
|
43
|
+
.replace(/</g, "<")
|
|
44
|
+
.replace(/>/g, ">")
|
|
45
|
+
.replace(/&/g, "&");
|
|
46
|
+
}
|
|
47
|
+
function stripTags(value) {
|
|
48
|
+
return value.replace(/<[^>]+>/g, "");
|
|
49
|
+
}
|
|
50
|
+
function extractJsonObjectAfter(marker, source) {
|
|
51
|
+
const markerIndex = source.indexOf(marker);
|
|
52
|
+
if (markerIndex === -1)
|
|
53
|
+
return null;
|
|
54
|
+
const start = source.indexOf("{", markerIndex + marker.length);
|
|
55
|
+
if (start === -1)
|
|
56
|
+
return null;
|
|
57
|
+
let depth = 0;
|
|
58
|
+
let inString = false;
|
|
59
|
+
let escaped = false;
|
|
60
|
+
for (let i = start; i < source.length; i += 1) {
|
|
61
|
+
const char = source[i];
|
|
62
|
+
if (inString) {
|
|
63
|
+
if (escaped) {
|
|
64
|
+
escaped = false;
|
|
65
|
+
}
|
|
66
|
+
else if (char === "\\") {
|
|
67
|
+
escaped = true;
|
|
68
|
+
}
|
|
69
|
+
else if (char === '"') {
|
|
70
|
+
inString = false;
|
|
71
|
+
}
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
if (char === '"') {
|
|
75
|
+
inString = true;
|
|
76
|
+
continue;
|
|
77
|
+
}
|
|
78
|
+
if (char === "{")
|
|
79
|
+
depth += 1;
|
|
80
|
+
if (char === "}") {
|
|
81
|
+
depth -= 1;
|
|
82
|
+
if (depth === 0)
|
|
83
|
+
return source.slice(start, i + 1);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return null;
|
|
87
|
+
}
|
|
88
|
+
function parsePlayerResponse(html) {
|
|
89
|
+
const jsonText = extractJsonObjectAfter("ytInitialPlayerResponse =", html);
|
|
90
|
+
if (!jsonText)
|
|
91
|
+
return null;
|
|
92
|
+
try {
|
|
93
|
+
return JSON.parse(jsonText);
|
|
94
|
+
}
|
|
95
|
+
catch {
|
|
96
|
+
return null;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
function chooseCaptionTrack(tracks) {
|
|
100
|
+
const normalized = tracks
|
|
101
|
+
.map((track) => ({
|
|
102
|
+
baseUrl: typeof track.baseUrl === "string" ? track.baseUrl : "",
|
|
103
|
+
languageCode: typeof track.languageCode === "string" ? track.languageCode : "",
|
|
104
|
+
kind: typeof track.kind === "string" ? track.kind : "",
|
|
105
|
+
}))
|
|
106
|
+
.filter((track) => track.baseUrl);
|
|
107
|
+
if (normalized.length === 0)
|
|
108
|
+
return null;
|
|
109
|
+
const preferred = normalized.find((track) => track.languageCode === "en" && track.kind !== "asr") ??
|
|
110
|
+
normalized.find((track) => track.languageCode.startsWith("en") && track.kind !== "asr") ??
|
|
111
|
+
normalized.find((track) => track.languageCode.startsWith("en")) ??
|
|
112
|
+
normalized[0];
|
|
113
|
+
return preferred.baseUrl;
|
|
114
|
+
}
|
|
115
|
+
async function fetchText(url, timeoutMs, signal) {
|
|
116
|
+
const response = await fetchWithRetry(url, {
|
|
117
|
+
headers: {
|
|
118
|
+
Accept: "text/html,application/json,application/xml,text/xml,text/plain;q=0.9,*/*;q=0.1",
|
|
119
|
+
"User-Agent": "akm-cli youtube fetcher",
|
|
120
|
+
},
|
|
121
|
+
signal,
|
|
122
|
+
}, { timeout: timeoutMs, retries: 1 });
|
|
123
|
+
if (!response.ok)
|
|
124
|
+
return null;
|
|
125
|
+
return response.text();
|
|
126
|
+
}
|
|
127
|
+
function parseTranscript(xml) {
|
|
128
|
+
const texts = [...xml.matchAll(/<text\b[^>]*>([\s\S]*?)<\/text>/g)]
|
|
129
|
+
.map((match) => decodeEntities(stripTags(match[1] ?? "")).trim())
|
|
130
|
+
.filter(Boolean);
|
|
131
|
+
if (texts.length === 0)
|
|
132
|
+
return null;
|
|
133
|
+
return texts.join("\n");
|
|
134
|
+
}
|
|
135
|
+
const youtubeFetcher = {
|
|
136
|
+
name: "youtube-transcript",
|
|
137
|
+
matches(url) {
|
|
138
|
+
return extractVideoId(url) !== null;
|
|
139
|
+
},
|
|
140
|
+
async fetch(url, context) {
|
|
141
|
+
const videoId = extractVideoId(url);
|
|
142
|
+
if (!videoId)
|
|
143
|
+
return null;
|
|
144
|
+
const watchUrl = canonicalWatchUrl(videoId);
|
|
145
|
+
const watchHtml = await fetchText(watchUrl, context.timeoutMs, context.signal);
|
|
146
|
+
if (!watchHtml)
|
|
147
|
+
return null;
|
|
148
|
+
const playerResponse = parsePlayerResponse(watchHtml);
|
|
149
|
+
if (!playerResponse)
|
|
150
|
+
return null;
|
|
151
|
+
const title = typeof playerResponse.videoDetails?.title === "string" ? playerResponse.videoDetails.title.trim() : "";
|
|
152
|
+
const description = typeof playerResponse.videoDetails?.shortDescription === "string"
|
|
153
|
+
? playerResponse.videoDetails.shortDescription.trim()
|
|
154
|
+
: "";
|
|
155
|
+
const sections = [];
|
|
156
|
+
let hasTranscript = false;
|
|
157
|
+
if (description) {
|
|
158
|
+
sections.push("## Description", "", description);
|
|
159
|
+
}
|
|
160
|
+
const tracks = playerResponse.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
|
|
161
|
+
const captionUrl = chooseCaptionTrack(tracks);
|
|
162
|
+
if (captionUrl) {
|
|
163
|
+
const transcriptXml = await fetchText(captionUrl, context.timeoutMs, context.signal);
|
|
164
|
+
const transcript = transcriptXml ? parseTranscript(transcriptXml) : null;
|
|
165
|
+
if (transcript) {
|
|
166
|
+
hasTranscript = true;
|
|
167
|
+
sections.push("## Transcript", "", transcript);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
if (sections.length === 0)
|
|
171
|
+
return null;
|
|
172
|
+
return {
|
|
173
|
+
url: watchUrl,
|
|
174
|
+
title: title || `YouTube ${videoId}`,
|
|
175
|
+
markdown: sections.join("\n"),
|
|
176
|
+
preferredName: `youtube/${videoId}`,
|
|
177
|
+
tags: hasTranscript ? ["youtube", "video", "transcript"] : ["youtube", "video"],
|
|
178
|
+
};
|
|
179
|
+
},
|
|
180
|
+
};
|
|
181
|
+
export default youtubeFetcher;
|
package/dist/storage/database.js
CHANGED
|
@@ -35,18 +35,39 @@ const isBun = !!process.versions?.bun;
|
|
|
35
35
|
// Bun built-in, unresolvable on Node) nor `better-sqlite3` (an optional native
|
|
36
36
|
// dep, possibly absent under Bun) is statically imported.
|
|
37
37
|
const nodeRequire = createRequire(import.meta.url);
|
|
38
|
+
// bun:sqlite — Bun built-in, no native build. Cannot run on Node.
|
|
39
|
+
const bunSqliteProvider = {
|
|
40
|
+
name: "bun:sqlite",
|
|
41
|
+
supported: () => isBun,
|
|
42
|
+
open: openBunDatabase,
|
|
43
|
+
};
|
|
44
|
+
// better-sqlite3 — native Node driver. Cannot run on Bun (oven-sh/bun#4290).
|
|
45
|
+
const nodeSqliteProvider = {
|
|
46
|
+
name: "better-sqlite3",
|
|
47
|
+
supported: () => !isBun,
|
|
48
|
+
open: openNodeDatabase,
|
|
49
|
+
};
|
|
38
50
|
/**
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
* Bun, so importing this module under Bun never touches `better-sqlite3`.
|
|
51
|
+
* Ordered provider registry. The factory selects the first supported provider.
|
|
52
|
+
* A future Postgres provider is appended here — and only here. Both SQLite
|
|
53
|
+
* engines are kept as distinct providers (not collapsed) because no single
|
|
54
|
+
* SQLite driver runs on both Bun and Node today.
|
|
44
55
|
*/
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
56
|
+
const PROVIDERS = [bunSqliteProvider, nodeSqliteProvider];
|
|
57
|
+
/** Select the provider for the current runtime. */
|
|
58
|
+
function selectProvider() {
|
|
59
|
+
const provider = PROVIDERS.find((p) => p.supported());
|
|
60
|
+
if (!provider) {
|
|
61
|
+
throw new Error(`No storage provider supports the current runtime (${isBun ? "Bun" : "Node"}).`);
|
|
48
62
|
}
|
|
49
|
-
return
|
|
63
|
+
return provider;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Open a SQLite database handle at `path` via the active {@link StorageProvider}.
|
|
67
|
+
* Returns a handle conforming to the structural {@link Database} type.
|
|
68
|
+
*/
|
|
69
|
+
export function openDatabase(path, opts) {
|
|
70
|
+
return selectProvider().open(path, opts);
|
|
50
71
|
}
|
|
51
72
|
function openBunDatabase(path, opts) {
|
|
52
73
|
const { Database: BunDatabase } = loadBunSqlite();
|
|
@@ -81,7 +102,21 @@ function loadBetterSqlite3() {
|
|
|
81
102
|
if (!betterSqlite3Ctor) {
|
|
82
103
|
// Runtime-gated dynamic require: only reached when NOT on Bun, so Bun never
|
|
83
104
|
// resolves or loads the optional `better-sqlite3` native dependency.
|
|
84
|
-
|
|
105
|
+
// `better-sqlite3` is an optionalDependency, so `npm i` can succeed without
|
|
106
|
+
// it (or with a native build that failed). Convert the raw MODULE_NOT_FOUND
|
|
107
|
+
// into an actionable message instead of a cryptic onboarding crash.
|
|
108
|
+
let mod;
|
|
109
|
+
try {
|
|
110
|
+
mod = nodeRequire("better-sqlite3");
|
|
111
|
+
}
|
|
112
|
+
catch (err) {
|
|
113
|
+
throw new Error("akm could not load 'better-sqlite3', the SQLite driver it needs on Node.js.\n" +
|
|
114
|
+
" • Reinstall akm with a working C/C++ build toolchain so its optional\n" +
|
|
115
|
+
" 'better-sqlite3' native binding rebuilds (a global `npm i -g better-sqlite3`\n" +
|
|
116
|
+
" will NOT be resolved — Node loads it from akm's own node_modules).\n" +
|
|
117
|
+
" • Or run akm under Bun, which has a built-in SQLite driver and needs no native build.\n" +
|
|
118
|
+
` Underlying load error: ${err instanceof Error ? err.message : String(err)}`);
|
|
119
|
+
}
|
|
85
120
|
betterSqlite3Ctor = mod.default ?? mod;
|
|
86
121
|
}
|
|
87
122
|
return betterSqlite3Ctor;
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/**
|
|
5
|
+
* Managed-database seam — the single home for the SQLite open/lifecycle recipe.
|
|
6
|
+
*
|
|
7
|
+
* Before this module, two idioms were copy-pasted across state.db / logs.db /
|
|
8
|
+
* workflow.db / index.db and their consumers:
|
|
9
|
+
*
|
|
10
|
+
* 1. The open recipe: `mkdir(dir) → openDatabase(path) → applyStandardPragmas
|
|
11
|
+
* → migrate`.
|
|
12
|
+
* 2. The borrow-or-own lifecycle: `const db = ctx?.db ?? open(); const owns =
|
|
13
|
+
* !ctx?.db; try { … } finally { if (owns) db.close(); }`.
|
|
14
|
+
*
|
|
15
|
+
* {@link openManagedDatabase} owns (1); {@link withManagedDb} owns (2). Each DB
|
|
16
|
+
* module supplies only a path + initializer and gets a `withXDb` loan helper
|
|
17
|
+
* (see `withIndexDb`, `withStateDb`, etc.) so callers never hand-roll the
|
|
18
|
+
* ownership flag or the finally/close again. This is also the one place to add
|
|
19
|
+
* busy-timeout tuning, integrity checks, or test-isolation injection.
|
|
20
|
+
*/
|
|
21
|
+
import fs from "node:fs";
|
|
22
|
+
import path from "node:path";
|
|
23
|
+
import { openDatabase } from "./database.js";
|
|
24
|
+
import { applyStandardPragmas } from "./sqlite-pragmas.js";
|
|
25
|
+
/**
|
|
26
|
+
* Open a managed SQLite database: ensure the parent dir exists, open the handle,
|
|
27
|
+
* apply standard pragmas, then run the schema initializer. The single home for
|
|
28
|
+
* the open→pragmas→migrate recipe.
|
|
29
|
+
*/
|
|
30
|
+
export function openManagedDatabase(spec) {
|
|
31
|
+
const dir = path.dirname(spec.path);
|
|
32
|
+
if (!fs.existsSync(dir)) {
|
|
33
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
34
|
+
}
|
|
35
|
+
const db = openDatabase(spec.path);
|
|
36
|
+
applyStandardPragmas(db, spec.pragmas ?? { dataDir: dir });
|
|
37
|
+
spec.init?.(db);
|
|
38
|
+
return db;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Run `fn` against a managed database, owning its lifecycle.
|
|
42
|
+
*
|
|
43
|
+
* When `opts.borrowed` is supplied the caller already owns an open handle: it is
|
|
44
|
+
* passed straight through and NOT closed (borrow). Otherwise a fresh handle is
|
|
45
|
+
* opened via `open` and closed in a `finally` (own). This replaces the
|
|
46
|
+
* hand-rolled `ctx?.db ?? open()` + `ownsDb` flag + `finally`/close idiom — the
|
|
47
|
+
* ownership decision and the close live here, once.
|
|
48
|
+
*
|
|
49
|
+
* Synchronous by design: the DB consumers (telemetry writers, planners) finish
|
|
50
|
+
* all work within the tick, matching the inline blocks this replaces.
|
|
51
|
+
*/
|
|
52
|
+
export function withManagedDb(open, fn, opts) {
|
|
53
|
+
if (opts?.borrowed) {
|
|
54
|
+
return fn(opts.borrowed);
|
|
55
|
+
}
|
|
56
|
+
const db = open();
|
|
57
|
+
try {
|
|
58
|
+
return fn(db);
|
|
59
|
+
}
|
|
60
|
+
finally {
|
|
61
|
+
db.close();
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Async sibling of {@link withManagedDb}. Use this — NOT `withManagedDb` — when
|
|
66
|
+
* `fn` holds the handle across an `await`: the sync version closes in its
|
|
67
|
+
* `finally` before the awaited work resolves (use-after-close). Here the handle
|
|
68
|
+
* is closed only after `fn`'s promise settles. Borrowed handles pass straight
|
|
69
|
+
* through and are not closed, as in the sync version.
|
|
70
|
+
*/
|
|
71
|
+
export async function withManagedDbAsync(open, fn, opts) {
|
|
72
|
+
if (opts?.borrowed) {
|
|
73
|
+
return fn(opts.borrowed);
|
|
74
|
+
}
|
|
75
|
+
const db = open();
|
|
76
|
+
try {
|
|
77
|
+
return await fn(db);
|
|
78
|
+
}
|
|
79
|
+
finally {
|
|
80
|
+
db.close();
|
|
81
|
+
}
|
|
82
|
+
}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
import { rethrowIfTestIsolationError } from "../../core/errors.js";
|
|
5
|
+
import { closeDatabase, getRegistryIndexCache, openIndexDatabase, upsertRegistryIndexCache } from "../../indexer/db/db.js";
|
|
6
|
+
/**
|
|
7
|
+
* RAII-style lifecycle helper for the registry cache DB. Opens the DB (treating
|
|
8
|
+
* a failed open exactly like the legacy fall-through: the bun-test isolation
|
|
9
|
+
* guard is re-thrown, any other failure yields `db = undefined`), runs `fn`,
|
|
10
|
+
* and guarantees the DB is closed in a `finally` after `fn` has fully settled
|
|
11
|
+
* (the await is required: the callbacks are async, and closing before they
|
|
12
|
+
* settle would tear the DB down mid-write).
|
|
13
|
+
*/
|
|
14
|
+
export async function withRegistryCacheDb(fn) {
|
|
15
|
+
let db;
|
|
16
|
+
try {
|
|
17
|
+
db = openIndexDatabase();
|
|
18
|
+
}
|
|
19
|
+
catch (err) {
|
|
20
|
+
// Never mask the bun-test isolation guard as "DB unavailable".
|
|
21
|
+
rethrowIfTestIsolationError(err);
|
|
22
|
+
db = undefined;
|
|
23
|
+
}
|
|
24
|
+
try {
|
|
25
|
+
return await fn(db);
|
|
26
|
+
}
|
|
27
|
+
finally {
|
|
28
|
+
if (db) {
|
|
29
|
+
try {
|
|
30
|
+
closeDatabase(db);
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
/* ignore */
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Shared registry index-cache fetch template. Opens the cache DB, returns a
|
|
40
|
+
* fresh cache hit when present, otherwise fetches live (writing the result back
|
|
41
|
+
* to the cache best-effort), and falls back to a stale cache row when the fetch
|
|
42
|
+
* fails. Behaviour-preserving extraction of the logic previously duplicated in
|
|
43
|
+
* `skills-sh.ts` (`fetchSkills`) and `static-index.ts` (`loadIndex`).
|
|
44
|
+
*/
|
|
45
|
+
export async function fetchCachedJson(opts) {
|
|
46
|
+
const { cacheKey, ttlMs, parseCache, fetchFresh } = opts;
|
|
47
|
+
return withRegistryCacheDb(async (db) => {
|
|
48
|
+
// ── Step 1: Try DB cache (index.db) ─────────────────────────────────────
|
|
49
|
+
let dbCacheResult;
|
|
50
|
+
try {
|
|
51
|
+
if (db) {
|
|
52
|
+
dbCacheResult = getRegistryIndexCache(db, cacheKey, ttlMs);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
catch (err) {
|
|
56
|
+
// Never mask the bun-test isolation guard as "DB unavailable" — see
|
|
57
|
+
// rethrowIfTestIsolationError in src/core/errors.ts. Without this, a
|
|
58
|
+
// leaky test silently gets a cold cache instead of the loud
|
|
59
|
+
// TEST_ISOLATION_MISSING failure the guard intends.
|
|
60
|
+
rethrowIfTestIsolationError(err);
|
|
61
|
+
// index.db read failed (pre-migration install or test env) — fall through
|
|
62
|
+
}
|
|
63
|
+
if (dbCacheResult) {
|
|
64
|
+
const cached = parseCache(dbCacheResult.indexJson, { stale: false });
|
|
65
|
+
if (cached !== undefined) {
|
|
66
|
+
return cached;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
// ── Step 2: Fetch fresh ──────────────────────────────────────────────────
|
|
70
|
+
try {
|
|
71
|
+
const { value, cacheJson, cacheOpts } = await fetchFresh(db);
|
|
72
|
+
if (db) {
|
|
73
|
+
try {
|
|
74
|
+
upsertRegistryIndexCache(db, cacheKey, cacheJson, cacheOpts);
|
|
75
|
+
}
|
|
76
|
+
catch {
|
|
77
|
+
/* best-effort */
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return value;
|
|
81
|
+
}
|
|
82
|
+
catch (err) {
|
|
83
|
+
// Fetch failed — use stale DB cache if available.
|
|
84
|
+
if (dbCacheResult) {
|
|
85
|
+
const stale = parseCache(dbCacheResult.indexJson, { stale: true });
|
|
86
|
+
if (stale !== undefined)
|
|
87
|
+
return stale;
|
|
88
|
+
}
|
|
89
|
+
throw err;
|
|
90
|
+
}
|
|
91
|
+
});
|
|
92
|
+
}
|