akm-cli 0.9.0-beta.6 → 0.9.0-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +663 -0
- package/README.md +12 -4
- package/dist/akm +38 -0
- package/dist/akm-migrate-storage +38 -0
- package/dist/assets/help/help-improve.md +9 -6
- package/dist/assets/hints/cli-hints-full.md +6 -5
- package/dist/assets/profiles/default.json +9 -4
- package/dist/assets/profiles/frequent.json +1 -1
- package/dist/assets/profiles/memory-focus.json +1 -1
- package/dist/assets/profiles/proactive-maintenance.json +25 -0
- package/dist/assets/profiles/quick.json +1 -1
- package/dist/assets/profiles/recombine-only.json +21 -0
- package/dist/assets/profiles/reflect-distill.json +30 -0
- package/dist/assets/profiles/synthesize.json +15 -0
- package/dist/assets/profiles/thorough.json +1 -1
- package/dist/assets/prompts/consolidate-system.md +23 -0
- package/dist/assets/prompts/contradiction-judge.md +33 -0
- package/dist/assets/prompts/distill-knowledge-system.md +22 -0
- package/dist/assets/prompts/distill-lesson-system.md +36 -0
- package/dist/assets/prompts/extract-session.md +11 -3
- package/dist/assets/prompts/graph-extract-system.md +1 -0
- package/dist/assets/prompts/graph-extract-user-prompt.md +1 -1
- package/dist/assets/prompts/memory-infer-system.md +1 -0
- package/dist/assets/prompts/memory-infer-user.md +5 -0
- package/dist/assets/prompts/metadata-enhance-system.md +1 -0
- package/dist/assets/prompts/procedural-system.md +44 -0
- package/dist/assets/prompts/recombine-system.md +40 -0
- package/dist/assets/prompts/staleness-detect-system.md +6 -0
- package/dist/assets/prompts/validate-summary-judge.md +1 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/agent.md +38 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/command.md +38 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/fact.md +39 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/knowledge.md +40 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/lesson.md +43 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/memory.md +38 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/script.md +43 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/skill.md +40 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/workflow.md +43 -0
- package/dist/assets/templates/html/health.html +281 -111
- package/dist/assets/wiki/ingest-workflow-template.md +45 -16
- package/dist/assets/wiki/schema-template.md +4 -4
- package/dist/cli/clack.js +56 -0
- package/dist/cli/config-migrate.js +7 -1
- package/dist/cli/confirm.js +1 -1
- package/dist/cli/parse-args.js +46 -1
- package/dist/cli/shared.js +28 -0
- package/dist/cli.js +25 -14
- package/dist/commands/agent/agent-dispatch.js +3 -2
- package/dist/commands/agent/agent-support.js +0 -7
- package/dist/commands/agent/contribute-cli.js +26 -7
- package/dist/commands/config-cli.js +26 -13
- package/dist/commands/env/child-env.js +47 -0
- package/dist/commands/env/env-cli.js +220 -227
- package/dist/commands/env/env.js +14 -67
- package/dist/commands/env/secret-cli.js +140 -138
- package/dist/commands/feedback-cli.js +153 -147
- package/dist/commands/graph/graph-cli.js +5 -13
- package/dist/commands/graph/graph.js +76 -72
- package/dist/commands/health/advisories.js +151 -0
- package/dist/commands/health/checks.js +103 -16
- package/dist/commands/health/html-report.js +447 -81
- package/dist/commands/health/improve-metrics.js +771 -0
- package/dist/commands/health/llm-usage.js +65 -0
- package/dist/commands/health/md-report.js +103 -0
- package/dist/commands/health/metrics.js +278 -0
- package/dist/commands/health/stash-exposure.js +46 -0
- package/dist/commands/health/surfaces.js +216 -0
- package/dist/commands/health/task-runs.js +135 -0
- package/dist/commands/health/types.js +26 -0
- package/dist/commands/health/windows.js +195 -0
- package/dist/commands/health.js +91 -1083
- package/dist/commands/improve/anti-collapse.js +170 -0
- package/dist/commands/improve/calibration.js +161 -0
- package/dist/commands/improve/collapse-detector.js +421 -0
- package/dist/commands/improve/consolidate/chunking.js +141 -0
- package/dist/commands/improve/consolidate/eligibility.js +64 -0
- package/dist/commands/improve/consolidate/merge.js +145 -0
- package/dist/commands/improve/consolidate/sanitize.js +231 -0
- package/dist/commands/{lint.js → improve/consolidate/types.js} +1 -1
- package/dist/commands/improve/consolidate.js +1313 -1278
- package/dist/commands/improve/dedup.js +482 -0
- package/dist/commands/improve/distill/content-repair.js +202 -0
- package/dist/commands/improve/distill/promote-memory.js +229 -0
- package/dist/commands/improve/distill/quality-gate.js +236 -0
- package/dist/commands/improve/distill-guards.js +127 -0
- package/dist/commands/improve/distill-promotion-policy.js +826 -167
- package/dist/commands/improve/distill.js +243 -599
- package/dist/commands/improve/eligibility.js +434 -0
- package/dist/commands/improve/encoding-salience.js +205 -0
- package/dist/commands/improve/extract-cli.js +179 -59
- package/dist/commands/improve/extract-prompt.js +55 -4
- package/dist/commands/improve/extract-watch.js +140 -0
- package/dist/commands/improve/extract.js +409 -43
- package/dist/commands/improve/feedback-valence.js +54 -0
- package/dist/commands/improve/hot-probation.js +45 -0
- package/dist/commands/improve/improve-auto-accept.js +160 -7
- package/dist/commands/improve/improve-cli.js +115 -73
- package/dist/commands/improve/improve-profiles.js +32 -8
- package/dist/commands/improve/improve-result-file.js +15 -25
- package/dist/commands/improve/improve-session.js +58 -0
- package/dist/commands/improve/improve.js +510 -2537
- package/dist/commands/improve/locks.js +154 -0
- package/dist/commands/improve/loop-stages.js +1100 -0
- package/dist/commands/improve/memory/memory-belief.js +14 -15
- package/dist/commands/improve/memory/memory-contradiction-detect.js +83 -60
- package/dist/commands/improve/memory/memory-improve.js +27 -27
- package/dist/commands/improve/outcome-loop.js +270 -0
- package/dist/commands/improve/preparation.js +2002 -0
- package/dist/commands/improve/proactive-maintenance.js +115 -0
- package/dist/commands/improve/procedural.js +398 -0
- package/dist/commands/improve/recombine.js +818 -0
- package/dist/commands/improve/reflect-noise.js +0 -0
- package/dist/commands/improve/reflect.js +212 -45
- package/dist/commands/improve/salience.js +455 -0
- package/dist/commands/improve/schema-similarity-gate.js +168 -0
- package/dist/commands/improve/shared.js +51 -0
- package/dist/commands/improve/triage.js +93 -0
- package/dist/commands/lint/agent-linter.js +19 -24
- package/dist/commands/lint/base-linter.js +173 -60
- package/dist/commands/lint/command-linter.js +19 -24
- package/dist/commands/lint/env-key-rules.js +38 -1
- package/dist/commands/lint/fact-linter.js +39 -0
- package/dist/commands/lint/index.js +31 -13
- package/dist/commands/lint/memory-linter.js +1 -1
- package/dist/commands/lint/registry.js +7 -2
- package/dist/commands/lint/task-linter.js +3 -3
- package/dist/commands/lint/workflow-linter.js +26 -1
- package/dist/commands/observability-cli.js +4 -4
- package/dist/commands/proposal/drain-policies.js +13 -4
- package/dist/commands/proposal/drain.js +45 -51
- package/dist/commands/proposal/legacy-import.js +115 -0
- package/dist/commands/proposal/proposal-cli.js +24 -34
- package/dist/commands/proposal/proposal.js +7 -1
- package/dist/commands/proposal/propose.js +8 -3
- package/dist/commands/proposal/repository.js +829 -0
- package/dist/commands/proposal/validators/proposal-quality-validators.js +9 -8
- package/dist/commands/proposal/validators/proposals.js +93 -882
- package/dist/commands/read/curate.js +419 -103
- package/dist/commands/read/knowledge.js +10 -3
- package/dist/commands/read/remember-cli.js +133 -138
- package/dist/commands/read/search-cli.js +15 -8
- package/dist/commands/read/search.js +22 -11
- package/dist/commands/read/show.js +106 -14
- package/dist/commands/registry-cli.js +76 -87
- package/dist/commands/remember.js +11 -12
- package/dist/commands/sources/add-cli.js +91 -95
- package/dist/commands/sources/history.js +1 -1
- package/dist/commands/sources/init.js +66 -18
- package/dist/commands/sources/installed-stashes.js +11 -3
- package/dist/commands/sources/schema-repair.js +44 -46
- package/dist/commands/sources/self-update.js +2 -2
- package/dist/commands/sources/source-add.js +7 -3
- package/dist/commands/sources/sources-cli.js +3 -3
- package/dist/commands/sources/stash-cli.js +29 -41
- package/dist/commands/sources/stash-skeleton.js +57 -8
- package/dist/commands/tasks/default-tasks.js +15 -2
- package/dist/commands/tasks/tasks-cli.js +20 -29
- package/dist/commands/tasks/tasks.js +39 -11
- package/dist/commands/wiki-cli.js +23 -38
- package/dist/commands/workflow-cli.js +15 -1
- package/dist/core/asset/asset-registry.js +3 -1
- package/dist/core/asset/asset-spec.js +21 -4
- package/dist/core/asset/frontmatter.js +188 -167
- package/dist/core/asset/markdown.js +8 -0
- package/dist/core/authoring-rules.js +92 -0
- package/dist/core/common.js +4 -23
- package/dist/core/concurrent.js +10 -1
- package/dist/core/config/config-io.js +10 -1
- package/dist/core/config/config-migration.js +18 -40
- package/dist/core/config/config-schema.js +389 -58
- package/dist/core/config/config-types.js +3 -3
- package/dist/core/config/config.js +67 -22
- package/dist/core/deep-merge.js +38 -0
- package/dist/core/errors.js +1 -0
- package/dist/core/eval/rank-metrics.js +113 -0
- package/dist/core/events.js +4 -7
- package/dist/core/improve-types.js +47 -8
- package/dist/core/logs-db.js +14 -75
- package/dist/core/parse.js +36 -16
- package/dist/core/paths.js +21 -18
- package/dist/core/standards/resolve-standards-context.js +87 -0
- package/dist/core/standards/resolve-stash-standards.js +99 -0
- package/dist/core/standards/resolve-type-conventions.js +66 -0
- package/dist/core/state/migrations.js +770 -0
- package/dist/core/state-db.js +142 -1091
- package/dist/core/structured.js +69 -0
- package/dist/core/time.js +53 -0
- package/dist/core/warn.js +21 -0
- package/dist/core/write-source.js +37 -0
- package/dist/indexer/db/db.js +356 -780
- package/dist/indexer/db/entry-mapper.js +41 -0
- package/dist/indexer/db/graph-db.js +129 -86
- package/dist/indexer/db/llm-cache.js +2 -2
- package/dist/indexer/db/schema.js +516 -0
- package/dist/indexer/ensure-index.js +103 -24
- package/dist/indexer/feedback/utility-policy.js +75 -0
- package/dist/indexer/graph/graph-boost.js +51 -41
- package/dist/indexer/graph/graph-extraction.js +207 -4
- package/dist/indexer/index-writer-lock.js +106 -0
- package/dist/indexer/index-written-assets.js +105 -0
- package/dist/indexer/indexer.js +291 -310
- package/dist/indexer/passes/dir-staleness.js +114 -0
- package/dist/indexer/passes/memory-inference.js +13 -5
- package/dist/indexer/passes/metadata.js +20 -0
- package/dist/indexer/read-preflight.js +23 -0
- package/dist/indexer/search/db-search.js +89 -13
- package/dist/indexer/search/fts-query.js +51 -0
- package/dist/indexer/search/ranking-contributors.js +95 -9
- package/dist/indexer/search/ranking.js +79 -3
- package/dist/indexer/search/search-fields.js +6 -0
- package/dist/indexer/search/search-source.js +32 -21
- package/dist/indexer/search/semantic-status.js +4 -0
- package/dist/indexer/walk/matchers.js +9 -0
- package/dist/indexer/walk/walker.js +21 -13
- package/dist/integrations/agent/builders.js +39 -13
- package/dist/integrations/agent/config.js +20 -59
- package/dist/integrations/agent/detect.js +9 -0
- package/dist/integrations/agent/index.js +3 -19
- package/dist/integrations/agent/model-aliases.js +7 -2
- package/dist/integrations/agent/profiles.js +7 -1
- package/dist/integrations/agent/prompts.js +75 -9
- package/dist/integrations/agent/runner-dispatch.js +59 -0
- package/dist/integrations/agent/runner.js +13 -9
- package/dist/integrations/agent/spawn.js +69 -67
- package/dist/integrations/harnesses/claude/agent-builder.js +1 -1
- package/dist/integrations/harnesses/claude/index.js +2 -0
- package/dist/integrations/harnesses/claude/session-log.js +11 -1
- package/dist/integrations/harnesses/index.js +2 -3
- package/dist/integrations/harnesses/opencode/agent-builder.js +1 -1
- package/dist/integrations/harnesses/opencode/index.js +2 -0
- package/dist/integrations/harnesses/opencode/session-log.js +173 -3
- package/dist/integrations/harnesses/opencode-sdk/index.js +2 -2
- package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +98 -17
- package/dist/integrations/harnesses/types.js +1 -0
- package/dist/integrations/session-logs/index.js +16 -0
- package/dist/llm/call-ai.js +2 -2
- package/dist/llm/client.js +57 -15
- package/dist/llm/embedder.js +67 -4
- package/dist/llm/embedders/cache.js +3 -1
- package/dist/llm/embedders/deterministic.js +66 -0
- package/dist/llm/embedders/local.js +73 -3
- package/dist/llm/feature-gate.js +16 -15
- package/dist/llm/graph-extract.js +67 -44
- package/dist/llm/memory-infer-impl.js +138 -0
- package/dist/llm/memory-infer.js +1 -127
- package/dist/llm/metadata-enhance.js +44 -31
- package/dist/llm/structured-call.js +49 -0
- package/dist/migrate-storage-node.mjs +8 -0
- package/dist/output/context.js +5 -5
- package/dist/output/renderers.js +85 -14
- package/dist/output/shapes/curate.js +14 -2
- package/dist/output/shapes/helpers.js +0 -3
- package/dist/output/shapes/passthrough.js +2 -1
- package/dist/output/text/helpers.js +29 -1
- package/dist/output/text/workflow.js +1 -0
- package/dist/registry/providers/skills-sh.js +21 -147
- package/dist/registry/providers/static-index.js +15 -157
- package/dist/registry/resolve.js +27 -9
- package/dist/runtime.js +25 -1
- package/dist/scripts/migrate-storage.js +2718 -2354
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +891 -597
- package/dist/setup/detect.js +9 -0
- package/dist/setup/legacy-config.js +106 -0
- package/dist/setup/prompt.js +57 -0
- package/dist/setup/providers.js +14 -0
- package/dist/setup/registry-stash-loader.js +12 -0
- package/dist/setup/semantic-assets.js +124 -0
- package/dist/setup/setup.js +52 -1614
- package/dist/setup/steps/connection.js +734 -0
- package/dist/setup/steps/output.js +31 -0
- package/dist/setup/steps/platforms.js +124 -0
- package/dist/setup/steps/semantic.js +27 -0
- package/dist/setup/steps/sources.js +222 -0
- package/dist/setup/steps/stashdir.js +42 -0
- package/dist/setup/steps/tasks.js +152 -0
- package/dist/sources/include.js +6 -2
- package/dist/sources/providers/filesystem.js +0 -1
- package/dist/sources/providers/git-install.js +210 -0
- package/dist/sources/providers/git-provider.js +234 -0
- package/dist/sources/providers/git-stash.js +248 -0
- package/dist/sources/providers/git.js +10 -661
- package/dist/sources/providers/npm.js +2 -6
- package/dist/sources/providers/provider-utils.js +13 -7
- package/dist/sources/providers/sync-from-ref.js +9 -1
- package/dist/sources/providers/tar-utils.js +16 -8
- package/dist/sources/providers/website.js +9 -5
- package/dist/sources/website-ingest.js +187 -29
- package/dist/sources/wiki-fetchers/registry.js +53 -0
- package/dist/sources/wiki-fetchers/youtube.js +239 -0
- package/dist/storage/database.js +45 -10
- package/dist/storage/managed-db.js +82 -0
- package/dist/storage/repositories/canaries-repository.js +107 -0
- package/dist/storage/repositories/consolidation-repository.js +38 -0
- package/dist/storage/repositories/embeddings-repository.js +72 -0
- package/dist/storage/repositories/events-repository.js +187 -0
- package/dist/storage/repositories/extract-sessions-repository.js +96 -0
- package/dist/storage/repositories/improve-runs-repository.js +146 -0
- package/dist/storage/repositories/index-db.js +14 -8
- package/dist/storage/repositories/proposals-repository.js +220 -0
- package/dist/storage/repositories/recombine-repository.js +213 -0
- package/dist/storage/repositories/registry-cache.js +93 -0
- package/dist/storage/repositories/registry-index-cache-repository.js +46 -0
- package/dist/storage/repositories/task-history-repository.js +93 -0
- package/dist/storage/sqlite-pragmas.js +146 -0
- package/dist/tasks/backends/cron.js +1 -1
- package/dist/tasks/backends/index.js +9 -0
- package/dist/tasks/backends/launchd.js +1 -1
- package/dist/tasks/backends/schtasks.js +1 -1
- package/dist/tasks/{resolveAkmBin.js → resolve-akm-bin.js} +2 -2
- package/dist/tasks/runner.js +15 -13
- package/dist/text-import-hook.mjs +0 -0
- package/dist/wiki/wiki.js +52 -11
- package/dist/workflows/cli.js +1 -0
- package/dist/workflows/db.js +3 -4
- package/dist/workflows/runtime/runs.js +43 -118
- package/dist/workflows/runtime/workflow-asset-loader.js +125 -0
- package/dist/workflows/validate-summary.js +2 -7
- package/docs/README.md +69 -18
- package/docs/data-and-telemetry.md +5 -4
- package/docs/migration/release-notes/0.7.0.md +1 -1
- package/docs/migration/release-notes/0.9.0.md +39 -0
- package/package.json +10 -10
- package/dist/assets/tasks/core/update-stashes.yml +0 -4
- package/dist/commands/db-cli.js +0 -23
- package/dist/indexer/db/db-backup.js +0 -376
- package/dist/indexer/passes/staleness-detect.js +0 -488
|
@@ -36,18 +36,14 @@ class NpmSourceProvider {
|
|
|
36
36
|
this.#config = config;
|
|
37
37
|
this.name = config.name ?? config.url ?? "npm";
|
|
38
38
|
}
|
|
39
|
-
async init(_ctx) {
|
|
40
|
-
// Resolution happens lazily in path(): until `sync()` runs there's no
|
|
41
|
-
// reliable on-disk path. Init is the registration handshake.
|
|
42
|
-
}
|
|
43
39
|
path() {
|
|
44
40
|
if (this.#config.path)
|
|
45
41
|
return this.#config.path;
|
|
46
42
|
throw new ConfigError(`npm source "${this.name}" has no resolved content path — run \`akm update\` to sync it before indexing.`);
|
|
47
43
|
}
|
|
48
|
-
async sync() {
|
|
44
|
+
async sync(options) {
|
|
49
45
|
const ref = npmRefFromConfig(this.#config);
|
|
50
|
-
await syncNpmRef(ref);
|
|
46
|
+
await syncNpmRef(ref, { force: options?.force });
|
|
51
47
|
}
|
|
52
48
|
}
|
|
53
49
|
registerSourceProvider("npm", (config) => new NpmSourceProvider(config));
|
|
@@ -91,14 +91,20 @@ export function copyDirectoryContents(sourceDir, destinationDir) {
|
|
|
91
91
|
continue;
|
|
92
92
|
const src = path.join(sourceDir, entry.name);
|
|
93
93
|
const dest = path.join(destinationDir, entry.name);
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
94
|
+
copyPathWithoutSymlinks(src, dest);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
function copyPathWithoutSymlinks(sourcePath, destinationPath) {
|
|
98
|
+
const stat = fs.lstatSync(sourcePath);
|
|
99
|
+
if (stat.isSymbolicLink())
|
|
100
|
+
return;
|
|
101
|
+
fs.mkdirSync(path.dirname(destinationPath), { recursive: true });
|
|
102
|
+
if (stat.isDirectory()) {
|
|
103
|
+
fs.mkdirSync(destinationPath, { recursive: true });
|
|
104
|
+
copyDirectoryContents(sourcePath, destinationPath);
|
|
105
|
+
return;
|
|
101
106
|
}
|
|
107
|
+
fs.copyFileSync(sourcePath, destinationPath);
|
|
102
108
|
}
|
|
103
109
|
export function isDirectory(target) {
|
|
104
110
|
try {
|
|
@@ -1,6 +1,13 @@
|
|
|
1
1
|
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
2
|
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
3
|
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/**
|
|
5
|
+
* Unified install-ref dispatcher.
|
|
6
|
+
*
|
|
7
|
+
* Replaces the historical `installRegistryRef()` entry point. Given an
|
|
8
|
+
* unparsed install ref, this resolves the right syncable provider and
|
|
9
|
+
* invokes its `sync()` method.
|
|
10
|
+
*/
|
|
4
11
|
import { UsageError } from "../../core/errors.js";
|
|
5
12
|
import { parseRegistryRef } from "../../registry/resolve.js";
|
|
6
13
|
import { detectStashRoot } from "./provider-utils.js";
|
|
@@ -18,7 +25,8 @@ export async function syncFromRef(ref, options) {
|
|
|
18
25
|
return syncRegistryGitRef(ref, options);
|
|
19
26
|
}
|
|
20
27
|
// Exhaustiveness — `parseRegistryRef` only emits the four sources above.
|
|
21
|
-
|
|
28
|
+
const _exhaustive = parsed;
|
|
29
|
+
throw new UsageError(`No syncable provider for ref: ${ref}`);
|
|
22
30
|
}
|
|
23
31
|
function syncLocalRef(parsed, options) {
|
|
24
32
|
const stashRoot = detectStashRoot(parsed.sourcePath);
|
|
@@ -12,12 +12,12 @@
|
|
|
12
12
|
* providers that fetch tarballs (currently `NpmSourceProvider` and the
|
|
13
13
|
* registry index builder).
|
|
14
14
|
*/
|
|
15
|
-
import { spawnSync } from "node:child_process";
|
|
16
15
|
import { createHash } from "node:crypto";
|
|
17
16
|
import fs from "node:fs";
|
|
18
17
|
import path from "node:path";
|
|
19
18
|
import { isWithin } from "../../core/common.js";
|
|
20
19
|
import { warn } from "../../core/warn.js";
|
|
20
|
+
import { spawnSync } from "../../runtime.js";
|
|
21
21
|
/**
|
|
22
22
|
* Verify an archive's integrity against a known hash. Throws and removes
|
|
23
23
|
* the archive when verification fails.
|
|
@@ -65,17 +65,25 @@ export function verifyArchiveIntegrity(archivePath, expected, source) {
|
|
|
65
65
|
* tree for symlinks that would escape the destination.
|
|
66
66
|
*/
|
|
67
67
|
export function extractTarGzSecure(archivePath, destinationDir) {
|
|
68
|
-
const listResult = spawnSync("tar",
|
|
69
|
-
if (listResult.
|
|
70
|
-
const err = listResult.stderr?.trim() ||
|
|
68
|
+
const listResult = spawnSync(["tar", "tzf", archivePath]);
|
|
69
|
+
if (!listResult.success) {
|
|
70
|
+
const err = listResult.stderr?.toString().trim() || "unknown error";
|
|
71
71
|
throw new Error(`Failed to inspect archive ${archivePath}: ${err}`);
|
|
72
72
|
}
|
|
73
|
-
validateTarEntries(listResult.stdout);
|
|
73
|
+
validateTarEntries(listResult.stdout.toString());
|
|
74
74
|
fs.rmSync(destinationDir, { recursive: true, force: true });
|
|
75
75
|
fs.mkdirSync(destinationDir, { recursive: true });
|
|
76
|
-
const extractResult = spawnSync(
|
|
77
|
-
|
|
78
|
-
|
|
76
|
+
const extractResult = spawnSync([
|
|
77
|
+
"tar",
|
|
78
|
+
"xzf",
|
|
79
|
+
archivePath,
|
|
80
|
+
"--no-same-owner",
|
|
81
|
+
"--strip-components=1",
|
|
82
|
+
"-C",
|
|
83
|
+
destinationDir,
|
|
84
|
+
]);
|
|
85
|
+
if (!extractResult.success) {
|
|
86
|
+
const err = extractResult.stderr?.toString().trim() || "unknown error";
|
|
79
87
|
throw new Error(`Failed to extract archive ${archivePath}: ${err}`);
|
|
80
88
|
}
|
|
81
89
|
// Post-extraction scan: verify all extracted files are within destinationDir
|
|
@@ -2,22 +2,26 @@
|
|
|
2
2
|
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
3
|
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
4
|
import { registerSourceProvider } from "../provider-factory.js";
|
|
5
|
-
import { ensureWebsiteMirror, getWebsiteCachePaths, validateWebsiteUrl } from "../website-ingest.js";
|
|
5
|
+
import { ensureWebsiteMirror, getWebsiteCachePaths, shouldAllowPrivateWebsiteUrlForTests, validateWebsiteUrl, } from "../website-ingest.js";
|
|
6
6
|
/**
|
|
7
7
|
* Website source provider — thin adapter over the shared website ingest module.
|
|
8
8
|
*/
|
|
9
9
|
registerSourceProvider("website", (config) => {
|
|
10
|
-
const
|
|
10
|
+
const allowPrivateHosts = shouldAllowPrivateWebsiteUrlForTests(config.url ?? "");
|
|
11
|
+
const url = validateWebsiteUrl(config.url ?? "", { allowPrivateHosts });
|
|
11
12
|
const name = config.name ?? "website";
|
|
12
13
|
return {
|
|
13
14
|
kind: "website",
|
|
14
15
|
name,
|
|
15
|
-
async init(_ctx) { },
|
|
16
16
|
path() {
|
|
17
17
|
return getWebsiteCachePaths(url).stashDir;
|
|
18
18
|
},
|
|
19
|
-
async sync() {
|
|
20
|
-
await ensureWebsiteMirror(config, {
|
|
19
|
+
async sync(options) {
|
|
20
|
+
await ensureWebsiteMirror(config, {
|
|
21
|
+
requireStashDir: true,
|
|
22
|
+
force: options?.force,
|
|
23
|
+
...(allowPrivateHosts ? { allowPrivateHosts: true } : {}),
|
|
24
|
+
});
|
|
21
25
|
},
|
|
22
26
|
};
|
|
23
27
|
});
|
|
@@ -3,12 +3,14 @@
|
|
|
3
3
|
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
4
|
import { createHash } from "node:crypto";
|
|
5
5
|
import fs from "node:fs";
|
|
6
|
+
import { isIP } from "node:net";
|
|
6
7
|
import path from "node:path";
|
|
7
|
-
import { fetchWithRetry, ResponseTooLargeError, readBodyWithByteCap } from "../core/common.js";
|
|
8
|
+
import { fetchWithRetry, ResponseTooLargeError, readBodyWithByteCap, resolveStashDir } from "../core/common.js";
|
|
8
9
|
import { ConfigError, UsageError } from "../core/errors.js";
|
|
9
10
|
import { getRegistryIndexCacheDir } from "../core/paths.js";
|
|
10
11
|
import { warn } from "../core/warn.js";
|
|
11
12
|
import { isExpired, sanitizeString } from "./providers/provider-utils.js";
|
|
13
|
+
import { loadWikiSnapshotFetchers } from "./wiki-fetchers/registry.js";
|
|
12
14
|
/** Refresh website snapshots every 12 hours to balance freshness with scraping load. */
|
|
13
15
|
const CACHE_REFRESH_INTERVAL_MS = 12 * 60 * 60 * 1000;
|
|
14
16
|
/** Allow up to 7 days of stale snapshots when refresh fails so search remains available during outages. */
|
|
@@ -30,6 +32,30 @@ const WEBSITE_PAGE_BYTE_CAP = 5 * 1024 * 1024;
|
|
|
30
32
|
* whole crawl and return what we have when time runs out.
|
|
31
33
|
*/
|
|
32
34
|
const WEBSITE_CRAWL_WALL_CLOCK_MS = 10 * 60 * 1000;
|
|
35
|
+
const WEBSITE_MAX_REDIRECTS = 8;
|
|
36
|
+
export function shouldAllowPrivateWebsiteHostsForTests() {
|
|
37
|
+
return process.env.BUN_TEST === "1" || process.env.NODE_ENV === "test";
|
|
38
|
+
}
|
|
39
|
+
export function shouldAllowPrivateWebsiteUrlForTests(rawUrl) {
|
|
40
|
+
if (!shouldAllowPrivateWebsiteHostsForTests())
|
|
41
|
+
return false;
|
|
42
|
+
try {
|
|
43
|
+
return isLoopbackWebsiteHostname(new URL(rawUrl).hostname.toLowerCase());
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
return false;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
function resolveFetcherStashDir(explicitStashDir) {
|
|
50
|
+
if (explicitStashDir)
|
|
51
|
+
return explicitStashDir;
|
|
52
|
+
try {
|
|
53
|
+
return resolveStashDir({ readOnly: true });
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
return null;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
33
59
|
export function getWebsiteCachePaths(siteUrl) {
|
|
34
60
|
const key = createHash("sha256").update(normalizeSiteUrl(siteUrl)).digest("hex").slice(0, 16);
|
|
35
61
|
const rootDir = path.join(getRegistryIndexCacheDir(), `website-${key}`);
|
|
@@ -41,7 +67,7 @@ export function getWebsiteCachePaths(siteUrl) {
|
|
|
41
67
|
}
|
|
42
68
|
export async function ensureWebsiteMirror(config, options) {
|
|
43
69
|
const rawUrl = config.url ?? "";
|
|
44
|
-
const normalizedUrl = validateWebsiteUrl(rawUrl);
|
|
70
|
+
const normalizedUrl = validateWebsiteUrl(rawUrl, { allowPrivateHosts: options?.allowPrivateHosts });
|
|
45
71
|
const cachePaths = getWebsiteCachePaths(normalizedUrl);
|
|
46
72
|
const requireStashDir = options?.requireStashDir === true;
|
|
47
73
|
const force = options?.force === true;
|
|
@@ -63,6 +89,7 @@ export async function ensureWebsiteMirror(config, options) {
|
|
|
63
89
|
await scrapeWebsiteToStash(normalizedUrl, cachePaths.stashDir, {
|
|
64
90
|
maxPages: coercePositiveInt(config.options?.maxPages, MAX_PAGES_DEFAULT),
|
|
65
91
|
maxDepth: coercePositiveInt(config.options?.maxDepth, MAX_DEPTH_DEFAULT),
|
|
92
|
+
allowPrivateHosts: options?.allowPrivateHosts,
|
|
66
93
|
});
|
|
67
94
|
fs.writeFileSync(cachePaths.manifestPath, `${JSON.stringify({ url: normalizedUrl, fetchedAt: new Date().toISOString() }, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
|
|
68
95
|
return cachePaths;
|
|
@@ -114,20 +141,51 @@ async function scrapeWebsiteToStash(startUrl, stashDir, options) {
|
|
|
114
141
|
fs.writeFileSync(filePath, buildMarkdownSnapshot(page, slug), "utf8");
|
|
115
142
|
}
|
|
116
143
|
}
|
|
117
|
-
export async function fetchWebsiteMarkdownSnapshot(rawUrl) {
|
|
118
|
-
const normalizedUrl = validateWebsiteInputUrl(rawUrl);
|
|
119
|
-
const
|
|
144
|
+
export async function fetchWebsiteMarkdownSnapshot(rawUrl, options) {
|
|
145
|
+
const normalizedUrl = validateWebsiteInputUrl(rawUrl, { allowPrivateHosts: options?.allowPrivateHosts });
|
|
146
|
+
const parsedUrl = new URL(normalizedUrl);
|
|
147
|
+
const stashDir = resolveFetcherStashDir(options?.stashDir);
|
|
148
|
+
const context = {
|
|
149
|
+
stashDir: stashDir ?? "",
|
|
150
|
+
timeoutMs: options?.timeoutMs ?? 15_000,
|
|
151
|
+
signal: options?.signal,
|
|
152
|
+
};
|
|
153
|
+
for (const fetcher of await loadWikiSnapshotFetchers(stashDir)) {
|
|
154
|
+
try {
|
|
155
|
+
if (!fetcher.matches(parsedUrl, context))
|
|
156
|
+
continue;
|
|
157
|
+
const snapshot = await fetcher.fetch(parsedUrl, context);
|
|
158
|
+
if (!snapshot)
|
|
159
|
+
continue;
|
|
160
|
+
return websiteMarkdownSnapshotFromResult(snapshot);
|
|
161
|
+
}
|
|
162
|
+
catch (error) {
|
|
163
|
+
warn("[akm] wiki-fetcher %s threw on %s: %s", fetcher.name, normalizedUrl, error instanceof Error ? error.message : String(error));
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
const fetched = await fetchWebsitePage(normalizedUrl, { allowPrivateHosts: options?.allowPrivateHosts });
|
|
120
167
|
if (!fetched) {
|
|
121
168
|
throw new UsageError(`No content could be fetched from ${normalizedUrl}`);
|
|
122
169
|
}
|
|
123
|
-
|
|
124
|
-
const slug = preferredName.split("/").pop() ?? preferredName;
|
|
125
|
-
return {
|
|
170
|
+
return websiteMarkdownSnapshotFromResult({
|
|
126
171
|
url: fetched.page.url,
|
|
127
172
|
title: fetched.page.title,
|
|
128
173
|
markdown: fetched.page.markdown,
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
function websiteMarkdownSnapshotFromResult(snapshot) {
|
|
177
|
+
const preferredName = snapshot.preferredName ?? deriveImportPath(snapshot.url);
|
|
178
|
+
const slug = preferredName.split("/").pop() ?? preferredName;
|
|
179
|
+
return {
|
|
180
|
+
url: snapshot.url,
|
|
181
|
+
title: snapshot.title,
|
|
182
|
+
markdown: snapshot.markdown,
|
|
129
183
|
preferredName,
|
|
130
|
-
content: buildMarkdownSnapshot(
|
|
184
|
+
content: buildMarkdownSnapshot({
|
|
185
|
+
url: snapshot.url,
|
|
186
|
+
title: snapshot.title,
|
|
187
|
+
markdown: snapshot.markdown,
|
|
188
|
+
}, slug || "website", snapshot.tags),
|
|
131
189
|
};
|
|
132
190
|
}
|
|
133
191
|
async function crawlWebsite(startUrl, options) {
|
|
@@ -147,7 +205,7 @@ async function crawlWebsite(startUrl, options) {
|
|
|
147
205
|
if (!normalized || visited.has(normalized))
|
|
148
206
|
continue;
|
|
149
207
|
visited.add(normalized);
|
|
150
|
-
const fetched = await fetchWebsitePage(normalized);
|
|
208
|
+
const fetched = await fetchWebsitePage(normalized, { allowPrivateHosts: options.allowPrivateHosts });
|
|
151
209
|
if (!fetched)
|
|
152
210
|
continue;
|
|
153
211
|
pages.push(fetched.page);
|
|
@@ -169,17 +227,8 @@ async function crawlWebsite(startUrl, options) {
|
|
|
169
227
|
}
|
|
170
228
|
return pages;
|
|
171
229
|
}
|
|
172
|
-
async function fetchWebsitePage(pageUrl) {
|
|
173
|
-
const
|
|
174
|
-
if (parsedUrl.hostname.endsWith(".invalid")) {
|
|
175
|
-
throw new Error(`Refusing to fetch reserved invalid hostname: ${parsedUrl.hostname}`);
|
|
176
|
-
}
|
|
177
|
-
const response = await fetchWithRetry(pageUrl, {
|
|
178
|
-
headers: {
|
|
179
|
-
Accept: "text/html, text/markdown, text/plain;q=0.9, application/xhtml+xml;q=0.8",
|
|
180
|
-
"User-Agent": "akm-cli website provider",
|
|
181
|
-
},
|
|
182
|
-
}, { timeout: 15_000, retries: 1 });
|
|
230
|
+
async function fetchWebsitePage(pageUrl, options) {
|
|
231
|
+
const response = await fetchWebsiteResponse(pageUrl, 0, options);
|
|
183
232
|
if (!response.ok) {
|
|
184
233
|
if (response.status === 404)
|
|
185
234
|
return null;
|
|
@@ -196,6 +245,7 @@ async function fetchWebsitePage(pageUrl) {
|
|
|
196
245
|
throw err;
|
|
197
246
|
}
|
|
198
247
|
const finalUrl = normalizeCrawlUrl(response.url || pageUrl) ?? pageUrl;
|
|
248
|
+
assertWebsiteRequestUrl(finalUrl, Error, options);
|
|
199
249
|
if (contentType.includes("text/html") || contentType.includes("application/xhtml+xml") || looksLikeMarkup(body)) {
|
|
200
250
|
const title = extractHtmlTitle(body) || new URL(finalUrl).hostname;
|
|
201
251
|
return {
|
|
@@ -216,11 +266,35 @@ async function fetchWebsitePage(pageUrl) {
|
|
|
216
266
|
links: [],
|
|
217
267
|
};
|
|
218
268
|
}
|
|
219
|
-
function
|
|
269
|
+
async function fetchWebsiteResponse(pageUrl, redirectCount = 0, options) {
|
|
270
|
+
assertWebsiteRequestUrl(pageUrl, Error, options);
|
|
271
|
+
const response = await fetchWithRetry(pageUrl, {
|
|
272
|
+
headers: {
|
|
273
|
+
Accept: "text/html, text/markdown, text/plain;q=0.9, application/xhtml+xml;q=0.8",
|
|
274
|
+
"User-Agent": "akm-cli website provider",
|
|
275
|
+
},
|
|
276
|
+
redirect: "manual",
|
|
277
|
+
}, { timeout: 15_000, retries: 1 });
|
|
278
|
+
if (response.status >= 300 && response.status < 400) {
|
|
279
|
+
if (redirectCount >= WEBSITE_MAX_REDIRECTS) {
|
|
280
|
+
throw new Error(`Too many redirects while fetching ${pageUrl}`);
|
|
281
|
+
}
|
|
282
|
+
const location = response.headers.get("location");
|
|
283
|
+
if (!location) {
|
|
284
|
+
throw new Error(`Redirect response from ${pageUrl} did not include a Location header`);
|
|
285
|
+
}
|
|
286
|
+
const nextUrl = new URL(location, pageUrl).toString();
|
|
287
|
+
assertWebsiteRequestUrl(nextUrl, Error, options);
|
|
288
|
+
return fetchWebsiteResponse(nextUrl, redirectCount + 1, options);
|
|
289
|
+
}
|
|
290
|
+
return response;
|
|
291
|
+
}
|
|
292
|
+
function buildMarkdownSnapshot(page, slug, tags) {
|
|
220
293
|
const title = sanitizeString(page.title, 200) || slug;
|
|
221
294
|
const description = sanitizeString(`Snapshot of ${page.url}`, 500);
|
|
222
295
|
const host = sanitizeString(new URL(page.url).hostname, 120);
|
|
223
296
|
const content = page.markdown.trim() || `Source: ${page.url}`;
|
|
297
|
+
const normalizedTags = Array.from(new Set(["website", host, ...(tags ?? [])]));
|
|
224
298
|
return [
|
|
225
299
|
"---",
|
|
226
300
|
`name: ${JSON.stringify(slug)}`,
|
|
@@ -228,8 +302,7 @@ function buildMarkdownSnapshot(page, slug) {
|
|
|
228
302
|
`sourceUrl: ${JSON.stringify(page.url)}`,
|
|
229
303
|
`title: ${JSON.stringify(title)}`,
|
|
230
304
|
"tags:",
|
|
231
|
-
` - ${JSON.stringify(
|
|
232
|
-
` - ${JSON.stringify(host)}`,
|
|
305
|
+
...normalizedTags.map((tag) => ` - ${JSON.stringify(tag)}`),
|
|
233
306
|
"---",
|
|
234
307
|
"",
|
|
235
308
|
`# ${title}`,
|
|
@@ -240,13 +313,13 @@ function buildMarkdownSnapshot(page, slug) {
|
|
|
240
313
|
"",
|
|
241
314
|
].join("\n");
|
|
242
315
|
}
|
|
243
|
-
export function validateWebsiteUrl(rawUrl) {
|
|
244
|
-
return validateWebsiteUrlWithError(rawUrl, ConfigError);
|
|
316
|
+
export function validateWebsiteUrl(rawUrl, options) {
|
|
317
|
+
return validateWebsiteUrlWithError(rawUrl, ConfigError, options);
|
|
245
318
|
}
|
|
246
|
-
export function validateWebsiteInputUrl(rawUrl) {
|
|
247
|
-
return validateWebsiteUrlWithError(rawUrl, UsageError);
|
|
319
|
+
export function validateWebsiteInputUrl(rawUrl, options) {
|
|
320
|
+
return validateWebsiteUrlWithError(rawUrl, UsageError, options);
|
|
248
321
|
}
|
|
249
|
-
function validateWebsiteUrlWithError(rawUrl, ErrorType) {
|
|
322
|
+
function validateWebsiteUrlWithError(rawUrl, ErrorType, options) {
|
|
250
323
|
if (!rawUrl) {
|
|
251
324
|
throw new ErrorType("Website provider requires a URL");
|
|
252
325
|
}
|
|
@@ -263,6 +336,7 @@ function validateWebsiteUrlWithError(rawUrl, ErrorType) {
|
|
|
263
336
|
if (parsed.username || parsed.password) {
|
|
264
337
|
throw new ErrorType("Website URL must not contain embedded credentials");
|
|
265
338
|
}
|
|
339
|
+
assertWebsiteRequestUrl(parsed.toString(), ErrorType, options);
|
|
266
340
|
parsed.hash = "";
|
|
267
341
|
return normalizeSiteUrl(parsed.toString());
|
|
268
342
|
}
|
|
@@ -461,6 +535,90 @@ function isAssetLikePath(pathname) {
|
|
|
461
535
|
function isSafeLinkUrl(url) {
|
|
462
536
|
return url.protocol === "http:" || url.protocol === "https:";
|
|
463
537
|
}
|
|
538
|
+
function assertWebsiteRequestUrl(rawUrl, ErrorType = Error, options) {
|
|
539
|
+
const parsedUrl = new URL(rawUrl);
|
|
540
|
+
const hostname = parsedUrl.hostname.toLowerCase();
|
|
541
|
+
if (hostname.endsWith(".invalid")) {
|
|
542
|
+
throw new ErrorType(`Refusing to fetch reserved invalid hostname: ${parsedUrl.hostname}`);
|
|
543
|
+
}
|
|
544
|
+
if (isForbiddenWebsiteHostname(hostname, options)) {
|
|
545
|
+
throw new ErrorType(`Refusing to fetch non-public website host: ${parsedUrl.hostname}`);
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
// WHATWG URL.hostname wraps IPv6 literals in brackets (e.g. "[::1]"), but
|
|
549
|
+
// node:net's isIP() only recognizes the bare address form and returns 0 for
|
|
550
|
+
// anything bracketed — silently skipping all IPv6 forbidden-host checks
|
|
551
|
+
// below for every hostname parsed off a URL. Strip the brackets before any
|
|
552
|
+
// isIP()/isForbiddenIpv6() call so those checks actually run.
|
|
553
|
+
function stripIpv6Brackets(hostname) {
|
|
554
|
+
return hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname;
|
|
555
|
+
}
|
|
556
|
+
function isForbiddenWebsiteHostname(hostname, options) {
|
|
557
|
+
if (options?.allowPrivateHosts === true)
|
|
558
|
+
return false;
|
|
559
|
+
if (hostname === "localhost" || hostname.endsWith(".localhost") || hostname === "metadata.google.internal") {
|
|
560
|
+
return true;
|
|
561
|
+
}
|
|
562
|
+
const bareHostname = stripIpv6Brackets(hostname);
|
|
563
|
+
const ipVersion = isIP(bareHostname);
|
|
564
|
+
if (ipVersion === 4)
|
|
565
|
+
return isForbiddenIpv4(bareHostname);
|
|
566
|
+
if (ipVersion === 6)
|
|
567
|
+
return isForbiddenIpv6(bareHostname);
|
|
568
|
+
return false;
|
|
569
|
+
}
|
|
570
|
+
function isLoopbackWebsiteHostname(hostname) {
|
|
571
|
+
if (hostname === "localhost" || hostname.endsWith(".localhost"))
|
|
572
|
+
return true;
|
|
573
|
+
const bareHostname = stripIpv6Brackets(hostname);
|
|
574
|
+
const ipVersion = isIP(bareHostname);
|
|
575
|
+
if (ipVersion === 4)
|
|
576
|
+
return bareHostname.startsWith("127.");
|
|
577
|
+
if (ipVersion === 6)
|
|
578
|
+
return bareHostname === "::1";
|
|
579
|
+
return false;
|
|
580
|
+
}
|
|
581
|
+
function isForbiddenIpv4(hostname) {
|
|
582
|
+
const parts = hostname.split(".").map((part) => Number.parseInt(part, 10));
|
|
583
|
+
if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255))
|
|
584
|
+
return true;
|
|
585
|
+
const [a, b] = parts;
|
|
586
|
+
return (a === 0 ||
|
|
587
|
+
a === 10 ||
|
|
588
|
+
a === 127 ||
|
|
589
|
+
(a === 169 && b === 254) ||
|
|
590
|
+
(a === 172 && b >= 16 && b <= 31) ||
|
|
591
|
+
(a === 192 && b === 168));
|
|
592
|
+
}
|
|
593
|
+
/**
|
|
594
|
+
* Extracts the embedded IPv4 address from an IPv4-mapped IPv6 literal
|
|
595
|
+
* (`::ffff:a.b.c.d` or its canonical hex form `::ffff:xxxx:yyyy`), or
|
|
596
|
+
* returns null if `hostname` isn't one.
|
|
597
|
+
*/
|
|
598
|
+
function extractIpv4MappedAddress(normalizedHostname) {
|
|
599
|
+
const match = normalizedHostname.match(/^::ffff:(?:(\d{1,3}(?:\.\d{1,3}){3})|([0-9a-f]{1,4}):([0-9a-f]{1,4}))$/);
|
|
600
|
+
if (!match)
|
|
601
|
+
return null;
|
|
602
|
+
if (match[1])
|
|
603
|
+
return match[1];
|
|
604
|
+
const high = Number.parseInt(match[2], 16);
|
|
605
|
+
const low = Number.parseInt(match[3], 16);
|
|
606
|
+
return `${(high >> 8) & 0xff}.${high & 0xff}.${(low >> 8) & 0xff}.${low & 0xff}`;
|
|
607
|
+
}
|
|
608
|
+
function isForbiddenIpv6(hostname) {
|
|
609
|
+
const normalized = hostname.toLowerCase();
|
|
610
|
+
const mappedIpv4 = extractIpv4MappedAddress(normalized);
|
|
611
|
+
if (mappedIpv4)
|
|
612
|
+
return isForbiddenIpv4(mappedIpv4);
|
|
613
|
+
return (normalized === "::" ||
|
|
614
|
+
normalized === "::1" ||
|
|
615
|
+
normalized.startsWith("fc") ||
|
|
616
|
+
normalized.startsWith("fd") ||
|
|
617
|
+
normalized.startsWith("fe8") ||
|
|
618
|
+
normalized.startsWith("fe9") ||
|
|
619
|
+
normalized.startsWith("fea") ||
|
|
620
|
+
normalized.startsWith("feb"));
|
|
621
|
+
}
|
|
464
622
|
function stripDangerousBlockTag(value, tagName) {
|
|
465
623
|
const pattern = new RegExp(`<${tagName}\\b[^>]*>[\\s\\S]*?<\\/${tagName}\\s*>`, "gi");
|
|
466
624
|
return value.replace(pattern, "");
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
import fs from "node:fs";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import { pathToFileURL } from "node:url";
|
|
7
|
+
import { warn } from "../../core/warn.js";
|
|
8
|
+
import youtubeFetcher from "./youtube.js";
|
|
9
|
+
const FETCHER_DIR = path.join("scripts", "wiki-fetchers");
|
|
10
|
+
const FETCHER_FILE_PATTERN = /\.(?:ts|js|mjs)$/i;
|
|
11
|
+
const BUILTIN_FETCHERS = [youtubeFetcher];
|
|
12
|
+
function isWikiSnapshotFetcher(value) {
|
|
13
|
+
if (!value || typeof value !== "object")
|
|
14
|
+
return false;
|
|
15
|
+
const candidate = value;
|
|
16
|
+
return (typeof candidate.name === "string" &&
|
|
17
|
+
typeof candidate.matches === "function" &&
|
|
18
|
+
typeof candidate.fetch === "function");
|
|
19
|
+
}
|
|
20
|
+
export async function loadWikiSnapshotFetchers(stashDir) {
|
|
21
|
+
const fetchers = [];
|
|
22
|
+
if (stashDir) {
|
|
23
|
+
const fetcherDir = path.join(stashDir, FETCHER_DIR);
|
|
24
|
+
try {
|
|
25
|
+
const entries = fs.readdirSync(fetcherDir).sort();
|
|
26
|
+
for (const entry of entries) {
|
|
27
|
+
if (!FETCHER_FILE_PATTERN.test(entry))
|
|
28
|
+
continue;
|
|
29
|
+
try {
|
|
30
|
+
const fileUrl = pathToFileURL(path.join(fetcherDir, entry)).toString();
|
|
31
|
+
const mod = await import(fileUrl);
|
|
32
|
+
if (isWikiSnapshotFetcher(mod.default)) {
|
|
33
|
+
fetchers.push(mod.default);
|
|
34
|
+
}
|
|
35
|
+
else {
|
|
36
|
+
warn("[akm] wiki-fetcher %s skipped: missing { name, matches, fetch }", entry);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
catch (error) {
|
|
40
|
+
warn("[akm] wiki-fetcher %s failed to load: %s", entry, error instanceof Error ? error.message : String(error));
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
catch (error) {
|
|
45
|
+
const code = typeof error === "object" && error && "code" in error ? error.code : undefined;
|
|
46
|
+
if (code !== "ENOENT") {
|
|
47
|
+
warn("[akm] wiki-fetcher directory %s could not be read: %s", fetcherDir, error instanceof Error ? error.message : String(error));
|
|
48
|
+
}
|
|
49
|
+
// Missing directory means no custom fetchers.
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
return [...fetchers, ...BUILTIN_FETCHERS];
|
|
53
|
+
}
|