akm-cli 0.9.0-beta.5 → 0.9.0-beta.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +711 -0
- package/README.md +12 -4
- package/dist/akm +38 -0
- package/dist/akm-migrate-storage +38 -0
- package/dist/assets/profiles/default.json +9 -4
- package/dist/assets/profiles/frequent.json +1 -1
- package/dist/assets/profiles/memory-focus.json +1 -1
- package/dist/assets/profiles/quick.json +1 -1
- package/dist/assets/profiles/synthesize.json +15 -0
- package/dist/assets/profiles/thorough.json +1 -1
- package/dist/assets/prompts/consolidate-system.md +23 -0
- package/dist/assets/prompts/contradiction-judge.md +33 -0
- package/dist/assets/prompts/distill-knowledge-system.md +22 -0
- package/dist/assets/prompts/distill-lesson-system.md +36 -0
- package/dist/assets/prompts/extract-session.md +6 -2
- package/dist/assets/prompts/graph-extract-system.md +1 -0
- package/dist/assets/prompts/graph-extract-user-prompt.md +1 -1
- package/dist/assets/prompts/memory-infer-system.md +1 -0
- package/dist/assets/prompts/memory-infer-user.md +5 -0
- package/dist/assets/prompts/metadata-enhance-system.md +1 -0
- package/dist/assets/prompts/procedural-system.md +44 -0
- package/dist/assets/prompts/recombine-system.md +40 -0
- package/dist/assets/prompts/staleness-detect-system.md +6 -0
- package/dist/assets/prompts/validate-summary-judge.md +1 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/agent.md +38 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/command.md +38 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/fact.md +39 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/knowledge.md +40 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/lesson.md +43 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/memory.md +38 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/script.md +43 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/skill.md +40 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/workflow.md +43 -0
- package/dist/assets/templates/html/health.html +281 -111
- package/dist/assets/wiki/ingest-workflow-template.md +38 -10
- package/dist/cli/parse-args.js +46 -1
- package/dist/cli/shared.js +28 -0
- package/dist/cli.js +27 -11
- package/dist/commands/agent/agent-dispatch.js +2 -2
- package/dist/commands/agent/agent-support.js +0 -7
- package/dist/commands/agent/contribute-cli.js +17 -4
- package/dist/commands/config-cli.js +18 -2
- package/dist/commands/env/child-env.js +47 -0
- package/dist/commands/env/env-cli.js +33 -26
- package/dist/commands/env/secret-cli.js +36 -22
- package/dist/commands/feedback-cli.js +15 -6
- package/dist/commands/graph/graph-cli.js +5 -13
- package/dist/commands/graph/graph.js +76 -72
- package/dist/commands/health/checks.js +49 -1
- package/dist/commands/health/html-report.js +422 -80
- package/dist/commands/health.js +386 -9
- package/dist/commands/improve/calibration.js +161 -0
- package/dist/commands/improve/consolidate/chunking.js +141 -0
- package/dist/commands/improve/consolidate/eligibility.js +81 -0
- package/dist/commands/improve/consolidate/merge.js +145 -0
- package/dist/commands/improve/consolidate/sanitize.js +231 -0
- package/dist/commands/{lint.js → improve/consolidate/types.js} +1 -1
- package/dist/commands/improve/consolidate.js +635 -660
- package/dist/commands/improve/dedup.js +482 -0
- package/dist/commands/improve/distill.js +159 -69
- package/dist/commands/improve/eligibility.js +434 -0
- package/dist/commands/improve/encoding-salience.js +205 -0
- package/dist/commands/improve/extract-cli.js +124 -2
- package/dist/commands/improve/extract-prompt.js +39 -2
- package/dist/commands/improve/extract-watch.js +140 -0
- package/dist/commands/improve/extract.js +389 -40
- package/dist/commands/improve/feedback-valence.js +54 -0
- package/dist/commands/improve/homeostatic.js +467 -0
- package/dist/commands/improve/improve-auto-accept.js +138 -7
- package/dist/commands/improve/improve-cli.js +36 -61
- package/dist/commands/improve/improve-profiles.js +14 -0
- package/dist/commands/improve/improve-result-file.js +14 -25
- package/dist/commands/improve/improve-session.js +58 -0
- package/dist/commands/improve/improve.js +485 -2498
- package/dist/commands/improve/locks.js +154 -0
- package/dist/commands/improve/loop-stages.js +1083 -0
- package/dist/commands/improve/memory/memory-contradiction-detect.js +23 -28
- package/dist/commands/improve/outcome-loop.js +256 -0
- package/dist/commands/improve/preparation.js +1966 -0
- package/dist/commands/improve/proactive-maintenance.js +115 -0
- package/dist/commands/improve/procedural.js +418 -0
- package/dist/commands/improve/recombine.js +850 -0
- package/dist/commands/improve/reflect-noise.js +0 -0
- package/dist/commands/improve/reflect.js +183 -40
- package/dist/commands/improve/salience.js +438 -0
- package/dist/commands/improve/triage.js +93 -0
- package/dist/commands/lint/agent-linter.js +19 -24
- package/dist/commands/lint/base-linter.js +173 -60
- package/dist/commands/lint/command-linter.js +19 -24
- package/dist/commands/lint/env-key-rules.js +38 -1
- package/dist/commands/lint/fact-linter.js +39 -0
- package/dist/commands/lint/index.js +31 -13
- package/dist/commands/lint/memory-linter.js +1 -1
- package/dist/commands/lint/registry.js +7 -2
- package/dist/commands/lint/task-linter.js +3 -3
- package/dist/commands/lint/workflow-linter.js +26 -1
- package/dist/commands/proposal/drain-policies.js +5 -0
- package/dist/commands/proposal/drain.js +43 -50
- package/dist/commands/proposal/proposal-cli.js +21 -31
- package/dist/commands/proposal/proposal.js +5 -0
- package/dist/commands/proposal/propose.js +7 -2
- package/dist/commands/proposal/validators/proposal-quality-validators.js +9 -8
- package/dist/commands/proposal/validators/proposals.js +189 -63
- package/dist/commands/read/curate.js +414 -94
- package/dist/commands/read/knowledge.js +6 -3
- package/dist/commands/read/search-cli.js +9 -4
- package/dist/commands/read/search.js +10 -6
- package/dist/commands/read/show.js +86 -7
- package/dist/commands/sources/init.js +49 -17
- package/dist/commands/sources/installed-stashes.js +11 -3
- package/dist/commands/sources/schema-repair.js +43 -45
- package/dist/commands/sources/self-update.js +2 -2
- package/dist/commands/sources/source-add.js +7 -3
- package/dist/commands/sources/stash-cli.js +28 -40
- package/dist/commands/sources/stash-skeleton.js +23 -8
- package/dist/commands/tasks/tasks-cli.js +19 -27
- package/dist/commands/tasks/tasks.js +39 -11
- package/dist/commands/wiki-cli.js +21 -35
- package/dist/core/asset/asset-registry.js +3 -1
- package/dist/core/asset/asset-spec.js +18 -2
- package/dist/core/asset/frontmatter.js +166 -167
- package/dist/core/asset/markdown.js +8 -0
- package/dist/core/authoring-rules.js +92 -0
- package/dist/core/common.js +0 -5
- package/dist/core/config/config-migration.js +12 -11
- package/dist/core/config/config-schema.js +340 -56
- package/dist/core/config/config-types.js +3 -3
- package/dist/core/config/config.js +28 -7
- package/dist/core/events.js +3 -7
- package/dist/core/improve-types.js +11 -8
- package/dist/core/logs-db.js +10 -66
- package/dist/core/parse.js +36 -16
- package/dist/core/paths.js +3 -0
- package/dist/core/standards/resolve-standards-context.js +87 -0
- package/dist/core/standards/resolve-stash-standards.js +99 -0
- package/dist/core/standards/resolve-type-conventions.js +66 -0
- package/dist/core/state/migrations.js +714 -0
- package/dist/core/state-db.js +525 -474
- package/dist/indexer/db/db.js +439 -247
- package/dist/indexer/db/graph-db.js +129 -86
- package/dist/indexer/ensure-index.js +152 -17
- package/dist/indexer/graph/graph-boost.js +51 -41
- package/dist/indexer/graph/graph-extraction.js +218 -4
- package/dist/indexer/index-writer-lock.js +99 -0
- package/dist/indexer/indexer.js +123 -221
- package/dist/indexer/passes/dir-staleness.js +114 -0
- package/dist/indexer/passes/memory-inference.js +13 -5
- package/dist/indexer/passes/staleness-detect.js +2 -5
- package/dist/indexer/search/db-search.js +19 -6
- package/dist/indexer/search/ranking-contributors.js +22 -0
- package/dist/indexer/search/ranking.js +4 -0
- package/dist/indexer/search/search-source.js +17 -18
- package/dist/indexer/search/semantic-status.js +4 -0
- package/dist/indexer/walk/matchers.js +9 -0
- package/dist/integrations/agent/config.js +6 -53
- package/dist/integrations/agent/index.js +2 -18
- package/dist/integrations/agent/prompts.js +75 -9
- package/dist/integrations/agent/runner-dispatch.js +59 -0
- package/dist/integrations/harnesses/claude/session-log.js +11 -1
- package/dist/integrations/harnesses/index.js +2 -3
- package/dist/integrations/harnesses/opencode/session-log.js +173 -3
- package/dist/integrations/harnesses/opencode-sdk/index.js +2 -2
- package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +0 -2
- package/dist/integrations/session-logs/index.js +16 -0
- package/dist/llm/client.js +45 -15
- package/dist/llm/embedder.js +42 -3
- package/dist/llm/embedders/deterministic.js +66 -0
- package/dist/llm/embedders/local.js +66 -2
- package/dist/llm/feature-gate.js +8 -4
- package/dist/llm/graph-extract.js +67 -44
- package/dist/llm/memory-infer-impl.js +138 -0
- package/dist/llm/memory-infer.js +1 -127
- package/dist/llm/metadata-enhance.js +44 -31
- package/dist/llm/structured-call.js +49 -0
- package/dist/migrate-storage-node.mjs +8 -0
- package/dist/output/context.js +5 -5
- package/dist/output/renderers.js +74 -2
- package/dist/output/shapes/curate.js +14 -2
- package/dist/output/shapes/passthrough.js +0 -1
- package/dist/output/text/helpers.js +16 -1
- package/dist/registry/providers/skills-sh.js +21 -147
- package/dist/registry/providers/static-index.js +15 -157
- package/dist/registry/resolve.js +22 -9
- package/dist/runtime.js +25 -1
- package/dist/scripts/migrate-storage.js +2617 -1961
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +759 -510
- package/dist/setup/setup.js +29 -8
- package/dist/sources/include.js +6 -2
- package/dist/sources/providers/filesystem.js +0 -1
- package/dist/sources/providers/git-install.js +210 -0
- package/dist/sources/providers/git-provider.js +234 -0
- package/dist/sources/providers/git-stash.js +248 -0
- package/dist/sources/providers/git.js +10 -661
- package/dist/sources/providers/npm.js +2 -6
- package/dist/sources/providers/provider-utils.js +13 -7
- package/dist/sources/providers/sync-from-ref.js +9 -1
- package/dist/sources/providers/tar-utils.js +16 -8
- package/dist/sources/providers/website.js +9 -5
- package/dist/sources/website-ingest.js +187 -29
- package/dist/sources/wiki-fetchers/registry.js +53 -0
- package/dist/sources/wiki-fetchers/youtube.js +239 -0
- package/dist/storage/database.js +45 -10
- package/dist/storage/managed-db.js +82 -0
- package/dist/storage/repositories/registry-cache.js +92 -0
- package/dist/storage/sqlite-pragmas.js +146 -0
- package/dist/tasks/backends/cron.js +1 -1
- package/dist/tasks/backends/launchd.js +1 -1
- package/dist/tasks/backends/schtasks.js +1 -1
- package/dist/tasks/{resolveAkmBin.js → resolve-akm-bin.js} +2 -2
- package/dist/tasks/runner.js +5 -13
- package/dist/text-import-hook.mjs +0 -0
- package/dist/wiki/wiki.js +37 -0
- package/dist/workflows/db.js +3 -4
- package/dist/workflows/runtime/runs.js +1 -117
- package/dist/workflows/runtime/workflow-asset-loader.js +125 -0
- package/dist/workflows/validate-summary.js +2 -7
- package/docs/data-and-telemetry.md +3 -2
- package/docs/migration/release-notes/0.9.0.md +39 -0
- package/package.json +13 -11
- package/dist/commands/db-cli.js +0 -23
- package/dist/indexer/db/db-backup.js +0 -376
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
import { fetchWithRetry } from "../../core/common.js";
|
|
5
|
+
const YOUTUBE_HOSTS = new Set(["www.youtube.com", "youtube.com", "m.youtube.com", "youtu.be"]);
|
|
6
|
+
const WATCH_HOST = "https://www.youtube.com/watch?v=";
|
|
7
|
+
// Captions: the WEB watch-page caption baseUrls are now Proof-of-Origin-Token
|
|
8
|
+
// gated (they carry `exp=xpe` and return an empty body without a `pot` token).
|
|
9
|
+
// The ANDROID InnerTube `player` client still returns UNGATED caption baseUrls
|
|
10
|
+
// that serve the transcript, so captions are sourced from there. See
|
|
11
|
+
// https://github.com/yt-dlp/yt-dlp/issues/13075.
|
|
12
|
+
const INNERTUBE_PLAYER_URL = "https://www.youtube.com/youtubei/v1/player";
|
|
13
|
+
// Long-stable public InnerTube key; only authorizes the player call (not auth).
|
|
14
|
+
const DEFAULT_INNERTUBE_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
|
|
15
|
+
const ANDROID_INNERTUBE_CLIENT = {
|
|
16
|
+
clientName: "ANDROID",
|
|
17
|
+
clientVersion: "20.10.38",
|
|
18
|
+
androidSdkVersion: 34,
|
|
19
|
+
hl: "en",
|
|
20
|
+
gl: "US",
|
|
21
|
+
};
|
|
22
|
+
function extractVideoId(url) {
|
|
23
|
+
if (!YOUTUBE_HOSTS.has(url.hostname))
|
|
24
|
+
return null;
|
|
25
|
+
if (url.hostname === "youtu.be") {
|
|
26
|
+
const id = url.pathname.replace(/^\/+/, "").split("/")[0]?.trim();
|
|
27
|
+
return id || null;
|
|
28
|
+
}
|
|
29
|
+
if (url.pathname === "/watch") {
|
|
30
|
+
const id = url.searchParams.get("v")?.trim();
|
|
31
|
+
return id || null;
|
|
32
|
+
}
|
|
33
|
+
if (url.pathname.startsWith("/shorts/")) {
|
|
34
|
+
const id = url.pathname.slice("/shorts/".length).split("/")[0]?.trim();
|
|
35
|
+
return id || null;
|
|
36
|
+
}
|
|
37
|
+
if (url.pathname.startsWith("/embed/")) {
|
|
38
|
+
const id = url.pathname.slice("/embed/".length).split("/")[0]?.trim();
|
|
39
|
+
return id || null;
|
|
40
|
+
}
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
function canonicalWatchUrl(videoId) {
|
|
44
|
+
return `${WATCH_HOST}${encodeURIComponent(videoId)}`;
|
|
45
|
+
}
|
|
46
|
+
function decodeEntities(value) {
|
|
47
|
+
const safeFromCodePoint = (codePoint, fallback) => {
|
|
48
|
+
if (!Number.isFinite(codePoint) || codePoint < 0 || codePoint > 0x10ffff)
|
|
49
|
+
return fallback;
|
|
50
|
+
try {
|
|
51
|
+
return String.fromCodePoint(codePoint);
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
return fallback;
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
return value
|
|
58
|
+
.replace(/&#x([0-9a-f]+);/gi, (m, hex) => safeFromCodePoint(Number.parseInt(hex, 16), m))
|
|
59
|
+
.replace(/&#([0-9]+);/g, (m, dec) => safeFromCodePoint(Number.parseInt(dec, 10), m))
|
|
60
|
+
.replace(/"/g, '"')
|
|
61
|
+
.replace(/'|'/g, "'")
|
|
62
|
+
.replace(/</g, "<")
|
|
63
|
+
.replace(/>/g, ">")
|
|
64
|
+
.replace(/&/g, "&");
|
|
65
|
+
}
|
|
66
|
+
function stripTags(value) {
|
|
67
|
+
return value.replace(/<[^>]+>/g, "");
|
|
68
|
+
}
|
|
69
|
+
function extractJsonObjectAfter(marker, source) {
|
|
70
|
+
const markerIndex = source.indexOf(marker);
|
|
71
|
+
if (markerIndex === -1)
|
|
72
|
+
return null;
|
|
73
|
+
const start = source.indexOf("{", markerIndex + marker.length);
|
|
74
|
+
if (start === -1)
|
|
75
|
+
return null;
|
|
76
|
+
let depth = 0;
|
|
77
|
+
let inString = false;
|
|
78
|
+
let escaped = false;
|
|
79
|
+
for (let i = start; i < source.length; i += 1) {
|
|
80
|
+
const char = source[i];
|
|
81
|
+
if (inString) {
|
|
82
|
+
if (escaped) {
|
|
83
|
+
escaped = false;
|
|
84
|
+
}
|
|
85
|
+
else if (char === "\\") {
|
|
86
|
+
escaped = true;
|
|
87
|
+
}
|
|
88
|
+
else if (char === '"') {
|
|
89
|
+
inString = false;
|
|
90
|
+
}
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
if (char === '"') {
|
|
94
|
+
inString = true;
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
97
|
+
if (char === "{")
|
|
98
|
+
depth += 1;
|
|
99
|
+
if (char === "}") {
|
|
100
|
+
depth -= 1;
|
|
101
|
+
if (depth === 0)
|
|
102
|
+
return source.slice(start, i + 1);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
return null;
|
|
106
|
+
}
|
|
107
|
+
function parsePlayerResponse(html) {
|
|
108
|
+
const jsonText = extractJsonObjectAfter("ytInitialPlayerResponse =", html);
|
|
109
|
+
if (!jsonText)
|
|
110
|
+
return null;
|
|
111
|
+
try {
|
|
112
|
+
return JSON.parse(jsonText);
|
|
113
|
+
}
|
|
114
|
+
catch {
|
|
115
|
+
return null;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
function chooseCaptionTrack(tracks) {
|
|
119
|
+
const normalized = tracks
|
|
120
|
+
.map((track) => ({
|
|
121
|
+
baseUrl: typeof track.baseUrl === "string" ? track.baseUrl : "",
|
|
122
|
+
languageCode: typeof track.languageCode === "string" ? track.languageCode : "",
|
|
123
|
+
kind: typeof track.kind === "string" ? track.kind : "",
|
|
124
|
+
}))
|
|
125
|
+
.filter((track) => track.baseUrl);
|
|
126
|
+
if (normalized.length === 0)
|
|
127
|
+
return null;
|
|
128
|
+
const preferred = normalized.find((track) => track.languageCode === "en" && track.kind !== "asr") ??
|
|
129
|
+
normalized.find((track) => track.languageCode.startsWith("en") && track.kind !== "asr") ??
|
|
130
|
+
normalized.find((track) => track.languageCode.startsWith("en")) ??
|
|
131
|
+
normalized[0];
|
|
132
|
+
return preferred.baseUrl;
|
|
133
|
+
}
|
|
134
|
+
async function fetchText(url, timeoutMs, signal) {
|
|
135
|
+
const response = await fetchWithRetry(url, {
|
|
136
|
+
headers: {
|
|
137
|
+
Accept: "text/html,application/json,application/xml,text/xml,text/plain;q=0.9,*/*;q=0.1",
|
|
138
|
+
"User-Agent": "akm-cli youtube fetcher",
|
|
139
|
+
},
|
|
140
|
+
signal,
|
|
141
|
+
}, { timeout: timeoutMs, retries: 1 });
|
|
142
|
+
if (!response.ok)
|
|
143
|
+
return null;
|
|
144
|
+
return response.text();
|
|
145
|
+
}
|
|
146
|
+
function parseTranscript(xml) {
|
|
147
|
+
// Two timedtext shapes: legacy srv1 `<text>` cues, and the format-3
|
|
148
|
+
// `<timedtext format="3">` `<p>` cues that the ANDROID InnerTube caption
|
|
149
|
+
// URLs return. Strip any nested `<s>` word tags via stripTags.
|
|
150
|
+
const matches = [...xml.matchAll(/<text\b[^>]*>([\s\S]*?)<\/text>/g), ...xml.matchAll(/<p\b[^>]*>([\s\S]*?)<\/p>/g)];
|
|
151
|
+
const texts = matches.map((match) => decodeEntities(stripTags(match[1] ?? "")).trim()).filter(Boolean);
|
|
152
|
+
if (texts.length === 0)
|
|
153
|
+
return null;
|
|
154
|
+
return texts.join("\n");
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Fetch caption tracks via the ANDROID InnerTube `player` endpoint, whose
|
|
158
|
+
* caption baseUrls are not POT-gated. Returns [] on any failure so the caller
|
|
159
|
+
* can fall back to the (usually empty) watch-page tracks and degrade to a
|
|
160
|
+
* description-only snapshot.
|
|
161
|
+
*/
|
|
162
|
+
async function fetchInnertubeCaptionTracks(videoId, apiKey, timeoutMs, signal) {
|
|
163
|
+
try {
|
|
164
|
+
const response = await fetchWithRetry(`${INNERTUBE_PLAYER_URL}?key=${encodeURIComponent(apiKey)}`, {
|
|
165
|
+
method: "POST",
|
|
166
|
+
headers: {
|
|
167
|
+
"Content-Type": "application/json",
|
|
168
|
+
"User-Agent": "com.google.android.youtube/20.10.38 (Linux; U; Android 14) gzip",
|
|
169
|
+
},
|
|
170
|
+
body: JSON.stringify({ context: { client: ANDROID_INNERTUBE_CLIENT }, videoId }),
|
|
171
|
+
signal,
|
|
172
|
+
}, { timeout: timeoutMs, retries: 1 });
|
|
173
|
+
if (!response.ok)
|
|
174
|
+
return [];
|
|
175
|
+
const json = (await response.json());
|
|
176
|
+
return json.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
|
|
177
|
+
}
|
|
178
|
+
catch {
|
|
179
|
+
return [];
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
/** Extract the page's InnerTube API key, falling back to the stable default. */
|
|
183
|
+
function extractInnertubeKey(html) {
|
|
184
|
+
return html.match(/"INNERTUBE_API_KEY":"([^"]+)"/)?.[1] ?? DEFAULT_INNERTUBE_KEY;
|
|
185
|
+
}
|
|
186
|
+
const youtubeFetcher = {
|
|
187
|
+
name: "youtube-transcript",
|
|
188
|
+
matches(url) {
|
|
189
|
+
return extractVideoId(url) !== null;
|
|
190
|
+
},
|
|
191
|
+
async fetch(url, context) {
|
|
192
|
+
const videoId = extractVideoId(url);
|
|
193
|
+
if (!videoId)
|
|
194
|
+
return null;
|
|
195
|
+
const watchUrl = canonicalWatchUrl(videoId);
|
|
196
|
+
const watchHtml = await fetchText(watchUrl, context.timeoutMs, context.signal);
|
|
197
|
+
if (!watchHtml)
|
|
198
|
+
return null;
|
|
199
|
+
const playerResponse = parsePlayerResponse(watchHtml);
|
|
200
|
+
if (!playerResponse)
|
|
201
|
+
return null;
|
|
202
|
+
const title = typeof playerResponse.videoDetails?.title === "string" ? playerResponse.videoDetails.title.trim() : "";
|
|
203
|
+
const description = typeof playerResponse.videoDetails?.shortDescription === "string"
|
|
204
|
+
? playerResponse.videoDetails.shortDescription.trim()
|
|
205
|
+
: "";
|
|
206
|
+
const sections = [];
|
|
207
|
+
let hasTranscript = false;
|
|
208
|
+
if (description) {
|
|
209
|
+
sections.push("## Description", "", description);
|
|
210
|
+
}
|
|
211
|
+
// Prefer ungated ANDROID InnerTube caption tracks; fall back to the
|
|
212
|
+
// (now usually empty) watch-page tracks so a description-only snapshot
|
|
213
|
+
// still works when InnerTube is unavailable.
|
|
214
|
+
const apiKey = extractInnertubeKey(watchHtml);
|
|
215
|
+
let tracks = await fetchInnertubeCaptionTracks(videoId, apiKey, context.timeoutMs, context.signal);
|
|
216
|
+
if (tracks.length === 0) {
|
|
217
|
+
tracks = playerResponse.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
|
|
218
|
+
}
|
|
219
|
+
const captionUrl = chooseCaptionTrack(tracks);
|
|
220
|
+
if (captionUrl) {
|
|
221
|
+
const transcriptXml = await fetchText(captionUrl, context.timeoutMs, context.signal);
|
|
222
|
+
const transcript = transcriptXml ? parseTranscript(transcriptXml) : null;
|
|
223
|
+
if (transcript) {
|
|
224
|
+
hasTranscript = true;
|
|
225
|
+
sections.push("## Transcript", "", transcript);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
if (sections.length === 0)
|
|
229
|
+
return null;
|
|
230
|
+
return {
|
|
231
|
+
url: watchUrl,
|
|
232
|
+
title: title || `YouTube ${videoId}`,
|
|
233
|
+
markdown: sections.join("\n"),
|
|
234
|
+
preferredName: `youtube/${videoId}`,
|
|
235
|
+
tags: hasTranscript ? ["youtube", "video", "transcript"] : ["youtube", "video"],
|
|
236
|
+
};
|
|
237
|
+
},
|
|
238
|
+
};
|
|
239
|
+
export default youtubeFetcher;
|
package/dist/storage/database.js
CHANGED
|
@@ -35,18 +35,39 @@ const isBun = !!process.versions?.bun;
|
|
|
35
35
|
// Bun built-in, unresolvable on Node) nor `better-sqlite3` (an optional native
|
|
36
36
|
// dep, possibly absent under Bun) is statically imported.
|
|
37
37
|
const nodeRequire = createRequire(import.meta.url);
|
|
38
|
+
// bun:sqlite — Bun built-in, no native build. Cannot run on Node.
|
|
39
|
+
const bunSqliteProvider = {
|
|
40
|
+
name: "bun:sqlite",
|
|
41
|
+
supported: () => isBun,
|
|
42
|
+
open: openBunDatabase,
|
|
43
|
+
};
|
|
44
|
+
// better-sqlite3 — native Node driver. Cannot run on Bun (oven-sh/bun#4290).
|
|
45
|
+
const nodeSqliteProvider = {
|
|
46
|
+
name: "better-sqlite3",
|
|
47
|
+
supported: () => !isBun,
|
|
48
|
+
open: openNodeDatabase,
|
|
49
|
+
};
|
|
38
50
|
/**
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
* Bun, so importing this module under Bun never touches `better-sqlite3`.
|
|
51
|
+
* Ordered provider registry. The factory selects the first supported provider.
|
|
52
|
+
* A future Postgres provider is appended here — and only here. Both SQLite
|
|
53
|
+
* engines are kept as distinct providers (not collapsed) because no single
|
|
54
|
+
* SQLite driver runs on both Bun and Node today.
|
|
44
55
|
*/
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
56
|
+
const PROVIDERS = [bunSqliteProvider, nodeSqliteProvider];
|
|
57
|
+
/** Select the provider for the current runtime. */
|
|
58
|
+
function selectProvider() {
|
|
59
|
+
const provider = PROVIDERS.find((p) => p.supported());
|
|
60
|
+
if (!provider) {
|
|
61
|
+
throw new Error(`No storage provider supports the current runtime (${isBun ? "Bun" : "Node"}).`);
|
|
48
62
|
}
|
|
49
|
-
return
|
|
63
|
+
return provider;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Open a SQLite database handle at `path` via the active {@link StorageProvider}.
|
|
67
|
+
* Returns a handle conforming to the structural {@link Database} type.
|
|
68
|
+
*/
|
|
69
|
+
export function openDatabase(path, opts) {
|
|
70
|
+
return selectProvider().open(path, opts);
|
|
50
71
|
}
|
|
51
72
|
function openBunDatabase(path, opts) {
|
|
52
73
|
const { Database: BunDatabase } = loadBunSqlite();
|
|
@@ -81,7 +102,21 @@ function loadBetterSqlite3() {
|
|
|
81
102
|
if (!betterSqlite3Ctor) {
|
|
82
103
|
// Runtime-gated dynamic require: only reached when NOT on Bun, so Bun never
|
|
83
104
|
// resolves or loads the optional `better-sqlite3` native dependency.
|
|
84
|
-
|
|
105
|
+
// `better-sqlite3` is an optionalDependency, so `npm i` can succeed without
|
|
106
|
+
// it (or with a native build that failed). Convert the raw MODULE_NOT_FOUND
|
|
107
|
+
// into an actionable message instead of a cryptic onboarding crash.
|
|
108
|
+
let mod;
|
|
109
|
+
try {
|
|
110
|
+
mod = nodeRequire("better-sqlite3");
|
|
111
|
+
}
|
|
112
|
+
catch (err) {
|
|
113
|
+
throw new Error("akm could not load 'better-sqlite3', the SQLite driver it needs on Node.js.\n" +
|
|
114
|
+
" • Reinstall akm with a working C/C++ build toolchain so its optional\n" +
|
|
115
|
+
" 'better-sqlite3' native binding rebuilds (a global `npm i -g better-sqlite3`\n" +
|
|
116
|
+
" will NOT be resolved — Node loads it from akm's own node_modules).\n" +
|
|
117
|
+
" • Or run akm under Bun, which has a built-in SQLite driver and needs no native build.\n" +
|
|
118
|
+
` Underlying load error: ${err instanceof Error ? err.message : String(err)}`);
|
|
119
|
+
}
|
|
85
120
|
betterSqlite3Ctor = mod.default ?? mod;
|
|
86
121
|
}
|
|
87
122
|
return betterSqlite3Ctor;
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/**
|
|
5
|
+
* Managed-database seam — the single home for the SQLite open/lifecycle recipe.
|
|
6
|
+
*
|
|
7
|
+
* Before this module, two idioms were copy-pasted across state.db / logs.db /
|
|
8
|
+
* workflow.db / index.db and their consumers:
|
|
9
|
+
*
|
|
10
|
+
* 1. The open recipe: `mkdir(dir) → openDatabase(path) → applyStandardPragmas
|
|
11
|
+
* → migrate`.
|
|
12
|
+
* 2. The borrow-or-own lifecycle: `const db = ctx?.db ?? open(); const owns =
|
|
13
|
+
* !ctx?.db; try { … } finally { if (owns) db.close(); }`.
|
|
14
|
+
*
|
|
15
|
+
* {@link openManagedDatabase} owns (1); {@link withManagedDb} owns (2). Each DB
|
|
16
|
+
* module supplies only a path + initializer and gets a `withXDb` loan helper
|
|
17
|
+
* (see `withIndexDb`, `withStateDb`, etc.) so callers never hand-roll the
|
|
18
|
+
* ownership flag or the finally/close again. This is also the one place to add
|
|
19
|
+
* busy-timeout tuning, integrity checks, or test-isolation injection.
|
|
20
|
+
*/
|
|
21
|
+
import fs from "node:fs";
|
|
22
|
+
import path from "node:path";
|
|
23
|
+
import { openDatabase } from "./database.js";
|
|
24
|
+
import { applyStandardPragmas } from "./sqlite-pragmas.js";
|
|
25
|
+
/**
|
|
26
|
+
* Open a managed SQLite database: ensure the parent dir exists, open the handle,
|
|
27
|
+
* apply standard pragmas, then run the schema initializer. The single home for
|
|
28
|
+
* the open→pragmas→migrate recipe.
|
|
29
|
+
*/
|
|
30
|
+
export function openManagedDatabase(spec) {
|
|
31
|
+
const dir = path.dirname(spec.path);
|
|
32
|
+
if (!fs.existsSync(dir)) {
|
|
33
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
34
|
+
}
|
|
35
|
+
const db = openDatabase(spec.path);
|
|
36
|
+
applyStandardPragmas(db, spec.pragmas ?? { dataDir: dir });
|
|
37
|
+
spec.init?.(db);
|
|
38
|
+
return db;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Run `fn` against a managed database, owning its lifecycle.
|
|
42
|
+
*
|
|
43
|
+
* When `opts.borrowed` is supplied the caller already owns an open handle: it is
|
|
44
|
+
* passed straight through and NOT closed (borrow). Otherwise a fresh handle is
|
|
45
|
+
* opened via `open` and closed in a `finally` (own). This replaces the
|
|
46
|
+
* hand-rolled `ctx?.db ?? open()` + `ownsDb` flag + `finally`/close idiom — the
|
|
47
|
+
* ownership decision and the close live here, once.
|
|
48
|
+
*
|
|
49
|
+
* Synchronous by design: the DB consumers (telemetry writers, planners) finish
|
|
50
|
+
* all work within the tick, matching the inline blocks this replaces.
|
|
51
|
+
*/
|
|
52
|
+
export function withManagedDb(open, fn, opts) {
|
|
53
|
+
if (opts?.borrowed) {
|
|
54
|
+
return fn(opts.borrowed);
|
|
55
|
+
}
|
|
56
|
+
const db = open();
|
|
57
|
+
try {
|
|
58
|
+
return fn(db);
|
|
59
|
+
}
|
|
60
|
+
finally {
|
|
61
|
+
db.close();
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Async sibling of {@link withManagedDb}. Use this — NOT `withManagedDb` — when
|
|
66
|
+
* `fn` holds the handle across an `await`: the sync version closes in its
|
|
67
|
+
* `finally` before the awaited work resolves (use-after-close). Here the handle
|
|
68
|
+
* is closed only after `fn`'s promise settles. Borrowed handles pass straight
|
|
69
|
+
* through and are not closed, as in the sync version.
|
|
70
|
+
*/
|
|
71
|
+
export async function withManagedDbAsync(open, fn, opts) {
|
|
72
|
+
if (opts?.borrowed) {
|
|
73
|
+
return fn(opts.borrowed);
|
|
74
|
+
}
|
|
75
|
+
const db = open();
|
|
76
|
+
try {
|
|
77
|
+
return await fn(db);
|
|
78
|
+
}
|
|
79
|
+
finally {
|
|
80
|
+
db.close();
|
|
81
|
+
}
|
|
82
|
+
}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
import { rethrowIfTestIsolationError } from "../../core/errors.js";
|
|
5
|
+
import { closeDatabase, getRegistryIndexCache, openIndexDatabase, upsertRegistryIndexCache } from "../../indexer/db/db.js";
|
|
6
|
+
/**
|
|
7
|
+
* RAII-style lifecycle helper for the registry cache DB. Opens the DB (treating
|
|
8
|
+
* a failed open exactly like the legacy fall-through: the bun-test isolation
|
|
9
|
+
* guard is re-thrown, any other failure yields `db = undefined`), runs `fn`,
|
|
10
|
+
* and guarantees the DB is closed in a `finally` after `fn` has fully settled
|
|
11
|
+
* (the await is required: the callbacks are async, and closing before they
|
|
12
|
+
* settle would tear the DB down mid-write).
|
|
13
|
+
*/
|
|
14
|
+
export async function withRegistryCacheDb(fn) {
|
|
15
|
+
let db;
|
|
16
|
+
try {
|
|
17
|
+
db = openIndexDatabase();
|
|
18
|
+
}
|
|
19
|
+
catch (err) {
|
|
20
|
+
// Never mask the bun-test isolation guard as "DB unavailable".
|
|
21
|
+
rethrowIfTestIsolationError(err);
|
|
22
|
+
db = undefined;
|
|
23
|
+
}
|
|
24
|
+
try {
|
|
25
|
+
return await fn(db);
|
|
26
|
+
}
|
|
27
|
+
finally {
|
|
28
|
+
if (db) {
|
|
29
|
+
try {
|
|
30
|
+
closeDatabase(db);
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
/* ignore */
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Shared registry index-cache fetch template. Opens the cache DB, returns a
|
|
40
|
+
* fresh cache hit when present, otherwise fetches live (writing the result back
|
|
41
|
+
* to the cache best-effort), and falls back to a stale cache row when the fetch
|
|
42
|
+
* fails. Behaviour-preserving extraction of the logic previously duplicated in
|
|
43
|
+
* `skills-sh.ts` (`fetchSkills`) and `static-index.ts` (`loadIndex`).
|
|
44
|
+
*/
|
|
45
|
+
export async function fetchCachedJson(opts) {
|
|
46
|
+
const { cacheKey, ttlMs, parseCache, fetchFresh } = opts;
|
|
47
|
+
return withRegistryCacheDb(async (db) => {
|
|
48
|
+
// ── Step 1: Try DB cache (index.db) ─────────────────────────────────────
|
|
49
|
+
let dbCacheResult;
|
|
50
|
+
try {
|
|
51
|
+
if (db) {
|
|
52
|
+
dbCacheResult = getRegistryIndexCache(db, cacheKey, ttlMs);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
catch (err) {
|
|
56
|
+
// Never mask the bun-test isolation guard as "DB unavailable" — see
|
|
57
|
+
// rethrowIfTestIsolationError in src/core/errors.ts. Without this, a
|
|
58
|
+
// leaky test silently gets a cold cache instead of the loud
|
|
59
|
+
// TEST_ISOLATION_MISSING failure the guard intends.
|
|
60
|
+
rethrowIfTestIsolationError(err);
|
|
61
|
+
// index.db read failed (pre-migration install or test env) — fall through
|
|
62
|
+
}
|
|
63
|
+
if (dbCacheResult) {
|
|
64
|
+
const cached = parseCache(dbCacheResult.indexJson, { stale: false });
|
|
65
|
+
if (cached !== undefined) {
|
|
66
|
+
return cached;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
// ── Step 2: Fetch fresh ──────────────────────────────────────────────────
|
|
70
|
+
try {
|
|
71
|
+
const { value, cacheJson, cacheOpts } = await fetchFresh(db);
|
|
72
|
+
if (db) {
|
|
73
|
+
try {
|
|
74
|
+
upsertRegistryIndexCache(db, cacheKey, cacheJson, cacheOpts);
|
|
75
|
+
}
|
|
76
|
+
catch {
|
|
77
|
+
/* best-effort */
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return value;
|
|
81
|
+
}
|
|
82
|
+
catch (err) {
|
|
83
|
+
// Fetch failed — use stale DB cache if available.
|
|
84
|
+
if (dbCacheResult) {
|
|
85
|
+
const stale = parseCache(dbCacheResult.indexJson, { stale: true });
|
|
86
|
+
if (stale !== undefined)
|
|
87
|
+
return stale;
|
|
88
|
+
}
|
|
89
|
+
throw err;
|
|
90
|
+
}
|
|
91
|
+
});
|
|
92
|
+
}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/**
|
|
5
|
+
* Shared SQLite PRAGMA application + journal-mode resolution (#628).
|
|
6
|
+
*
|
|
7
|
+
* Every AKM SQLite opener applies the same opening PRAGMAs: a journal mode, a
|
|
8
|
+
* 30 s busy_timeout, and (for most) foreign_keys = ON. Historically each opener
|
|
9
|
+
* hard-coded `PRAGMA journal_mode = WAL`. WAL is impossible on network
|
|
10
|
+
* filesystems (NFS/SMB) because its `-shm` shared-memory wal-index cannot be
|
|
11
|
+
* mmap'd over a network mount, so AKM could not run with its data dir on a
|
|
12
|
+
* network share.
|
|
13
|
+
*
|
|
14
|
+
* This module centralises that PRAGMA block behind {@link applyStandardPragmas}
|
|
15
|
+
* and makes the journal mode configurable via the `AKM_SQLITE_JOURNAL_MODE`
|
|
16
|
+
* env var (WAL | DELETE | TRUNCATE; default WAL = unchanged behaviour). When the
|
|
17
|
+
* mode is the WAL default and the data directory is detected to live on a
|
|
18
|
+
* network filesystem, it auto-falls-back to DELETE with a one-line warning.
|
|
19
|
+
*
|
|
20
|
+
* Boundary note: this is a PLAIN module, not a runtime-boundary file. It only
|
|
21
|
+
* does pure string work plus `db.exec()` (allowed everywhere). The single
|
|
22
|
+
* runtime primitive it needs — a filesystem-type probe (statfs) — lives in
|
|
23
|
+
* src/runtime.ts and is injected here, keeping the network-FS classifier a
|
|
24
|
+
* pure, unit-testable function.
|
|
25
|
+
*
|
|
26
|
+
* @module storage/sqlite-pragmas
|
|
27
|
+
*/
|
|
28
|
+
import { warn } from "../core/warn.js";
|
|
29
|
+
import { statfsType } from "../runtime.js";
|
|
30
|
+
const VALID_MODES = new Set(["WAL", "DELETE", "TRUNCATE"]);
|
|
31
|
+
// One-shot warning guards so a misconfigured env var or a network-FS fallback
|
|
32
|
+
// each emit AT MOST ONCE per process rather than on every db open.
|
|
33
|
+
let warnedInvalid = false;
|
|
34
|
+
let warnedNetworkFallback = false;
|
|
35
|
+
/**
|
|
36
|
+
* Resolve a raw `AKM_SQLITE_JOURNAL_MODE` value to a canonical {@link JournalMode}.
|
|
37
|
+
*
|
|
38
|
+
* PURE and unit-testable: the raw string is passed in (not read from
|
|
39
|
+
* `process.env` here). Trims + uppercases; an empty/undefined value yields the
|
|
40
|
+
* WAL default; a recognised value yields its canonical form; any other
|
|
41
|
+
* non-empty value warns once and falls back to WAL. Never throws.
|
|
42
|
+
*/
|
|
43
|
+
export function resolveJournalMode(raw) {
|
|
44
|
+
if (raw === undefined)
|
|
45
|
+
return "WAL";
|
|
46
|
+
const normalized = raw.trim().toUpperCase();
|
|
47
|
+
if (normalized === "")
|
|
48
|
+
return "WAL";
|
|
49
|
+
if (VALID_MODES.has(normalized)) {
|
|
50
|
+
return normalized;
|
|
51
|
+
}
|
|
52
|
+
warnInvalidJournalModeOnce(raw);
|
|
53
|
+
return "WAL";
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* The single env-reading seam: resolve the configured journal mode from
|
|
57
|
+
* `process.env.AKM_SQLITE_JOURNAL_MODE`. Read at call time (per open) so tests
|
|
58
|
+
* that set the env per-case see the right value and we avoid stale-env flakes.
|
|
59
|
+
*/
|
|
60
|
+
export function resolveConfiguredJournalMode() {
|
|
61
|
+
return resolveJournalMode(process.env.AKM_SQLITE_JOURNAL_MODE);
|
|
62
|
+
}
|
|
63
|
+
function warnInvalidJournalModeOnce(raw) {
|
|
64
|
+
if (warnedInvalid)
|
|
65
|
+
return;
|
|
66
|
+
warnedInvalid = true;
|
|
67
|
+
warn(`[akm] invalid AKM_SQLITE_JOURNAL_MODE=${JSON.stringify(raw)} — using WAL (valid: WAL, DELETE, TRUNCATE)`);
|
|
68
|
+
}
|
|
69
|
+
// Known Linux f_type magic numbers for network filesystems. node's statfs
|
|
70
|
+
// normalises `type` to this numeric f_type magic on all platforms.
|
|
71
|
+
const FS_MAGIC_NFS = 0x6969; // 26985 — NFS
|
|
72
|
+
const FS_MAGIC_SMB = 0x517b; // 20859 — older SMB_SUPER_MAGIC
|
|
73
|
+
const FS_MAGIC_CIFS = 0xff534d42; // 4283649346 — SMB/CIFS
|
|
74
|
+
const FS_MAGIC_SMB2 = 0xfe534d42; // 4267272514 — SMB2
|
|
75
|
+
const FS_MAGIC_FUSE = 0x65735546; // 1702057286 — FUSE (sshfs + many network FUSE mounts)
|
|
76
|
+
const NETWORK_FS_MAGICS = new Set([
|
|
77
|
+
FS_MAGIC_NFS,
|
|
78
|
+
FS_MAGIC_SMB,
|
|
79
|
+
FS_MAGIC_CIFS,
|
|
80
|
+
FS_MAGIC_SMB2,
|
|
81
|
+
// FUSE is a judgment call: it backs BOTH network mounts (sshfs) and local
|
|
82
|
+
// mounts (some encrypted/overlay FS). Treating it as network falls back to
|
|
83
|
+
// DELETE — conservative-but-safe (DELETE works everywhere; the only cost is
|
|
84
|
+
// losing WAL concurrency). An operator can always force WAL via the env var.
|
|
85
|
+
FS_MAGIC_FUSE,
|
|
86
|
+
]);
|
|
87
|
+
/**
|
|
88
|
+
* PURE classifier: is `fsType` a known network-filesystem magic number?
|
|
89
|
+
* Returns false for `undefined` (probe failed/unsupported) and for local
|
|
90
|
+
* magics (ext4 0xEF53, btrfs, xfs, tmpfs, apfs, …). Unit-testable with
|
|
91
|
+
* injected magic numbers — no real mount required.
|
|
92
|
+
*/
|
|
93
|
+
export function isNetworkFilesystem(fsType) {
|
|
94
|
+
if (fsType === undefined)
|
|
95
|
+
return false;
|
|
96
|
+
return NETWORK_FS_MAGICS.has(fsType);
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Apply AKM's standard opening PRAGMAs to `db`, in order:
|
|
100
|
+
* 1. `journal_mode` = the configured mode (with WAL→DELETE network-FS fallback)
|
|
101
|
+
* 2. `busy_timeout = 30000`
|
|
102
|
+
* 3. `foreign_keys = ON` (unless `opts.foreignKeys === false`)
|
|
103
|
+
* 4. `synchronous = FULL` (only in a rollback-journal mode — DELETE/TRUNCATE)
|
|
104
|
+
*
|
|
105
|
+
* Returns the effective {@link JournalMode} so callers/tests can assert it.
|
|
106
|
+
*
|
|
107
|
+
* `synchronous = FULL` is set explicitly only in DELETE/TRUNCATE so durability
|
|
108
|
+
* intent is clear: rollback journals need FULL for crash-durability across
|
|
109
|
+
* power loss, whereas WAL is durable at NORMAL. SQLite's default synchronous is
|
|
110
|
+
* already FULL when unset, so this never changes WAL-default behaviour — the
|
|
111
|
+
* WAL path emits no `synchronous` pragma, exactly as before.
|
|
112
|
+
*/
|
|
113
|
+
export function applyStandardPragmas(db, opts = {}) {
|
|
114
|
+
let mode = resolveConfiguredJournalMode();
|
|
115
|
+
// Network-FS fallback only fires for the WAL default and only when we have a
|
|
116
|
+
// directory to probe. An explicitly-requested DELETE/TRUNCATE is never
|
|
117
|
+
// overridden, and a failed/unsupported probe (undefined) keeps WAL.
|
|
118
|
+
if (mode === "WAL" && opts.dataDir) {
|
|
119
|
+
const probe = opts.fsTypeProbe ?? statfsType;
|
|
120
|
+
if (isNetworkFilesystem(probe(opts.dataDir))) {
|
|
121
|
+
mode = "DELETE";
|
|
122
|
+
warnNetworkFallbackOnce(opts.dataDir);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
// PRAGMAs must run before any DDL or DML. busy_timeout is applied FIRST so a
|
|
126
|
+
// journal-mode change that must reclaim a leftover `-wal` file (WAL→DELETE on
|
|
127
|
+
// reopen of an unclean WAL db, e.g. after a crash) can wait out a transient
|
|
128
|
+
// lock instead of failing immediately with SQLITE_BUSY. For the WAL default
|
|
129
|
+
// this is a no-op (WAL→WAL changes nothing), so byte-identical behaviour is
|
|
130
|
+
// preserved.
|
|
131
|
+
db.exec("PRAGMA busy_timeout = 30000");
|
|
132
|
+
db.exec(`PRAGMA journal_mode = ${mode}`);
|
|
133
|
+
if (opts.foreignKeys !== false) {
|
|
134
|
+
db.exec("PRAGMA foreign_keys = ON");
|
|
135
|
+
}
|
|
136
|
+
if (mode !== "WAL") {
|
|
137
|
+
db.exec("PRAGMA synchronous = FULL");
|
|
138
|
+
}
|
|
139
|
+
return mode;
|
|
140
|
+
}
|
|
141
|
+
function warnNetworkFallbackOnce(dataDir) {
|
|
142
|
+
if (warnedNetworkFallback)
|
|
143
|
+
return;
|
|
144
|
+
warnedNetworkFallback = true;
|
|
145
|
+
warn(`[akm] network filesystem detected at ${dataDir} — WAL unsupported, using DELETE journal mode`);
|
|
146
|
+
}
|
|
@@ -29,7 +29,7 @@ import fs from "node:fs";
|
|
|
29
29
|
import path from "node:path";
|
|
30
30
|
import { ConfigError } from "../../core/errors.js";
|
|
31
31
|
import { getTaskLogDir } from "../../core/paths.js";
|
|
32
|
-
import { resolveAkmInvocation } from "../
|
|
32
|
+
import { resolveAkmInvocation } from "../resolve-akm-bin.js";
|
|
33
33
|
import { parseSchedule, translateToCron } from "../schedule.js";
|
|
34
34
|
const BEGIN = (id) => `# akm:task ${id} BEGIN`;
|
|
35
35
|
const END = (id) => `# akm:task ${id} END`;
|
|
@@ -27,7 +27,7 @@ import path from "node:path";
|
|
|
27
27
|
import launchdTemplate from "../../assets/backends/launchd-template.xml" with { type: "text" };
|
|
28
28
|
import { ConfigError } from "../../core/errors.js";
|
|
29
29
|
import { getTaskLogDir } from "../../core/paths.js";
|
|
30
|
-
import { resolveAkmInvocation } from "../
|
|
30
|
+
import { resolveAkmInvocation } from "../resolve-akm-bin.js";
|
|
31
31
|
import { parseSchedule, translateToLaunchd } from "../schedule.js";
|
|
32
32
|
import { escapeXml, nodeExec, nodeFs } from "./exec-utils.js";
|
|
33
33
|
export const LAUNCHD_LABEL_PREFIX = "com.akm.task.";
|
|
@@ -35,7 +35,7 @@ import path from "node:path";
|
|
|
35
35
|
import schtasksTemplate from "../../assets/backends/schtasks-template.xml" with { type: "text" };
|
|
36
36
|
import { ConfigError } from "../../core/errors.js";
|
|
37
37
|
import { getTaskLogDir } from "../../core/paths.js";
|
|
38
|
-
import { resolveAkmInvocation } from "../
|
|
38
|
+
import { resolveAkmInvocation } from "../resolve-akm-bin.js";
|
|
39
39
|
import { parseSchedule, translateToSchtasks } from "../schedule.js";
|
|
40
40
|
import { escapeXml, nodeExec, nodeFs } from "./exec-utils.js";
|
|
41
41
|
export const DEFAULT_FOLDER_PREFIX = "\\akm\\";
|
|
@@ -44,8 +44,8 @@ export function resolveAkmInvocation(options = {}) {
|
|
|
44
44
|
/**
|
|
45
45
|
* From the URL of a module inside `src/tasks/` figure out the CLI entry.
|
|
46
46
|
*
|
|
47
|
-
* • dev `…/src/tasks/
|
|
48
|
-
* • build `…/dist/tasks/
|
|
47
|
+
* • dev `…/src/tasks/resolve-akm-bin.ts` → `…/src/cli.ts`
|
|
48
|
+
* • build `…/dist/tasks/resolve-akm-bin.js` → `…/dist/cli.js`
|
|
49
49
|
*/
|
|
50
50
|
function resolveCliEntry(moduleUrl) {
|
|
51
51
|
let modulePath;
|