akm-cli 0.6.0-rc1 → 0.6.0-rc2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/README.md +9 -9
- package/dist/cli.js +181 -111
- package/dist/{completions.js → commands/completions.js} +1 -1
- package/dist/{config-cli.js → commands/config-cli.js} +109 -11
- package/dist/{curate.js → commands/curate.js} +8 -3
- package/dist/{info.js → commands/info.js} +15 -9
- package/dist/{init.js → commands/init.js} +4 -4
- package/dist/{install-audit.js → commands/install-audit.js} +4 -7
- package/dist/{installed-stashes.js → commands/installed-stashes.js} +77 -31
- package/dist/{migration-help.js → commands/migration-help.js} +2 -2
- package/dist/{registry-search.js → commands/registry-search.js} +8 -6
- package/dist/{remember.js → commands/remember.js} +55 -49
- package/dist/{stash-search.js → commands/search.js} +28 -69
- package/dist/{self-update.js → commands/self-update.js} +3 -3
- package/dist/{stash-show.js → commands/show.js} +103 -78
- package/dist/{stash-add.js → commands/source-add.js} +42 -32
- package/dist/{stash-clone.js → commands/source-clone.js} +12 -10
- package/dist/{stash-source-manage.js → commands/source-manage.js} +24 -24
- package/dist/{vault.js → commands/vault.js} +43 -0
- package/dist/{stash-ref.js → core/asset-ref.js} +4 -4
- package/dist/{asset-registry.js → core/asset-registry.js} +1 -1
- package/dist/{asset-spec.js → core/asset-spec.js} +1 -1
- package/dist/{config.js → core/config.js} +79 -31
- package/dist/core/errors.js +90 -0
- package/dist/{frontmatter.js → core/frontmatter.js} +5 -3
- package/dist/core/write-source.js +280 -0
- package/dist/{db-search.js → indexer/db-search.js} +25 -19
- package/dist/{db.js → indexer/db.js} +70 -47
- package/dist/{file-context.js → indexer/file-context.js} +3 -3
- package/dist/{indexer.js → indexer/indexer.js} +123 -31
- package/dist/{manifest.js → indexer/manifest.js} +10 -10
- package/dist/{matchers.js → indexer/matchers.js} +3 -6
- package/dist/{metadata.js → indexer/metadata.js} +9 -5
- package/dist/{search-source.js → indexer/search-source.js} +52 -41
- package/dist/{semantic-status.js → indexer/semantic-status.js} +2 -2
- package/dist/{walker.js → indexer/walker.js} +1 -1
- package/dist/{lockfile.js → integrations/lockfile.js} +1 -1
- package/dist/{llm-client.js → llm/client.js} +1 -1
- package/dist/{embedders → llm/embedders}/local.js +2 -2
- package/dist/{embedders → llm/embedders}/remote.js +1 -1
- package/dist/{embedders → llm/embedders}/types.js +1 -1
- package/dist/{metadata-enhance.js → llm/metadata-enhance.js} +2 -2
- package/dist/{cli-hints.js → output/cli-hints.js} +1 -0
- package/dist/{output-context.js → output/context.js} +21 -3
- package/dist/{renderers.js → output/renderers.js} +9 -65
- package/dist/{output-shapes.js → output/shapes.js} +18 -4
- package/dist/{output-text.js → output/text.js} +1 -1
- package/dist/{registry-build-index.js → registry/build-index.js} +16 -7
- package/dist/{create-provider-registry.js → registry/create-provider-registry.js} +6 -2
- package/dist/registry/factory.js +33 -0
- package/dist/{origin-resolve.js → registry/origin-resolve.js} +1 -1
- package/dist/{providers → registry/providers}/index.js +1 -1
- package/dist/{providers → registry/providers}/skills-sh.js +59 -3
- package/dist/{providers → registry/providers}/static-index.js +80 -12
- package/dist/registry/providers/types.js +25 -0
- package/dist/{registry-resolve.js → registry/resolve.js} +3 -3
- package/dist/{detect.js → setup/detect.js} +0 -27
- package/dist/{ripgrep-install.js → setup/ripgrep-install.js} +1 -1
- package/dist/{ripgrep-resolve.js → setup/ripgrep-resolve.js} +2 -2
- package/dist/{setup.js → setup/setup.js} +16 -56
- package/dist/{stash-include.js → sources/include.js} +1 -1
- package/dist/sources/provider-factory.js +36 -0
- package/dist/sources/provider.js +21 -0
- package/dist/sources/providers/filesystem.js +35 -0
- package/dist/{stash-providers → sources/providers}/git.js +53 -64
- package/dist/{stash-providers → sources/providers}/index.js +3 -4
- package/dist/sources/providers/install-types.js +14 -0
- package/dist/{stash-providers → sources/providers}/npm.js +42 -41
- package/dist/{stash-providers → sources/providers}/provider-utils.js +3 -3
- package/dist/{stash-providers → sources/providers}/sync-from-ref.js +2 -2
- package/dist/{stash-providers → sources/providers}/tar-utils.js +11 -8
- package/dist/{stash-providers → sources/providers}/website.js +29 -65
- package/dist/{stash-resolve.js → sources/resolve.js} +8 -7
- package/dist/{wiki.js → wiki/wiki.js} +12 -11
- package/dist/{workflow-authoring.js → workflows/authoring.js} +37 -14
- package/dist/{workflow-cli.js → workflows/cli.js} +2 -1
- package/dist/{workflow-db.js → workflows/db.js} +1 -1
- package/dist/workflows/document-cache.js +20 -0
- package/dist/workflows/parser.js +379 -0
- package/dist/workflows/renderer.js +78 -0
- package/dist/{workflow-runs.js → workflows/runs.js} +72 -28
- package/dist/workflows/schema.js +11 -0
- package/dist/workflows/validator.js +48 -0
- package/docs/migration/release-notes/0.6.0.md +69 -23
- package/package.json +1 -1
- package/dist/errors.js +0 -45
- package/dist/llm.js +0 -16
- package/dist/registry-factory.js +0 -19
- package/dist/ripgrep.js +0 -2
- package/dist/stash-provider-factory.js +0 -35
- package/dist/stash-provider.js +0 -3
- package/dist/stash-providers/filesystem.js +0 -71
- package/dist/stash-providers/openviking.js +0 -348
- package/dist/stash-types.js +0 -1
- package/dist/workflow-markdown.js +0 -260
- /package/dist/{common.js → core/common.js} +0 -0
- /package/dist/{markdown.js → core/markdown.js} +0 -0
- /package/dist/{paths.js → core/paths.js} +0 -0
- /package/dist/{warn.js → core/warn.js} +0 -0
- /package/dist/{search-fields.js → indexer/search-fields.js} +0 -0
- /package/dist/{usage-events.js → indexer/usage-events.js} +0 -0
- /package/dist/{github.js → integrations/github.js} +0 -0
- /package/dist/{embedder.js → llm/embedder.js} +0 -0
- /package/dist/{embedders → llm/embedders}/cache.js +0 -0
- /package/dist/{registry-provider.js → registry/types.js} +0 -0
- /package/dist/{setup-steps.js → setup/steps.js} +0 -0
- /package/dist/{registry-types.js → sources/types.js} +0 -0
|
@@ -2,12 +2,12 @@ import { spawnSync } from "node:child_process";
|
|
|
2
2
|
import { createHash, randomBytes } from "node:crypto";
|
|
3
3
|
import fs from "node:fs";
|
|
4
4
|
import path from "node:path";
|
|
5
|
-
import { resolveStashDir } from "
|
|
6
|
-
import { loadConfig } from "
|
|
7
|
-
import { ConfigError, UsageError } from "
|
|
8
|
-
import { getRegistryCacheDir, getRegistryIndexCacheDir } from "
|
|
9
|
-
import { parseRegistryRef, resolveRegistryArtifact, validateGitRef, validateGitUrl } from "
|
|
10
|
-
import {
|
|
5
|
+
import { resolveStashDir } from "../../core/common";
|
|
6
|
+
import { loadConfig } from "../../core/config";
|
|
7
|
+
import { ConfigError, UsageError } from "../../core/errors";
|
|
8
|
+
import { getRegistryCacheDir, getRegistryIndexCacheDir } from "../../core/paths";
|
|
9
|
+
import { parseRegistryRef, resolveRegistryArtifact, validateGitRef, validateGitUrl } from "../../registry/resolve";
|
|
10
|
+
import { registerSourceProvider } from "../provider-factory";
|
|
11
11
|
import { applyAkmIncludeConfig, buildInstallCacheDir, copyDirectoryContents, detectStashRoot, isDirectory, isExpired, sanitizeString, } from "./provider-utils";
|
|
12
12
|
/** Cache TTL before refreshing the mirrored repo (12 hours). */
|
|
13
13
|
const CACHE_TTL_MS = 12 * 60 * 60 * 1000;
|
|
@@ -15,74 +15,63 @@ const CACHE_TTL_MS = 12 * 60 * 60 * 1000;
|
|
|
15
15
|
const CACHE_STALE_MS = 7 * 24 * 60 * 60 * 1000;
|
|
16
16
|
const GIT_STASH_TYPES = new Set(["git"]);
|
|
17
17
|
/**
|
|
18
|
-
* Git
|
|
19
|
-
*
|
|
20
|
-
*
|
|
18
|
+
* Git source provider — clones (and re-pulls) a remote repo into a local
|
|
19
|
+
* cache directory. Implements the v1 {@link SourceProvider} interface (spec
|
|
20
|
+
* §2.1, §2.5): `{ name, kind, init, path, sync }`.
|
|
21
|
+
*
|
|
22
|
+
* Reading is the indexer's job — this class doesn't implement `search` or
|
|
23
|
+
* `show`. The install-time helpers `syncRegistryGitRef` / `syncMirroredRepo`
|
|
24
|
+
* live below as standalone functions used by `akm add` / `akm update`.
|
|
21
25
|
*/
|
|
22
|
-
class
|
|
23
|
-
|
|
24
|
-
kind = "syncable";
|
|
26
|
+
class GitSourceProvider {
|
|
27
|
+
kind = "git";
|
|
25
28
|
name;
|
|
26
|
-
config;
|
|
29
|
+
#config;
|
|
30
|
+
#path = null;
|
|
27
31
|
constructor(config) {
|
|
28
|
-
this
|
|
32
|
+
this.#config = config;
|
|
29
33
|
this.name = config.name ?? "git";
|
|
30
34
|
}
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
35
|
+
async init(_ctx) {
|
|
36
|
+
// Resolve the on-disk content directory once. For configured git sources
|
|
37
|
+
// this is the cached working tree; for one-shot install refs it's the
|
|
38
|
+
// path the install pipeline materialised.
|
|
39
|
+
this.#path = resolveGitContentDir(this.#config);
|
|
40
|
+
}
|
|
41
|
+
path() {
|
|
42
|
+
if (this.#path == null) {
|
|
43
|
+
// Lazy resolution: providers are sometimes constructed without an
|
|
44
|
+
// explicit init() call (e.g. by legacy callers that just want the
|
|
45
|
+
// path). Resolve on demand and cache.
|
|
46
|
+
this.#path = resolveGitContentDir(this.#config);
|
|
47
|
+
}
|
|
48
|
+
return this.#path;
|
|
42
49
|
}
|
|
43
|
-
async sync(
|
|
50
|
+
async sync() {
|
|
44
51
|
// Two execution modes:
|
|
45
|
-
// 1. Long-lived configured
|
|
52
|
+
// 1. Long-lived configured source (config.url) — mirror into the
|
|
46
53
|
// registry-index cache and serve as a read-only working tree.
|
|
47
|
-
// 2. One-shot install ref (
|
|
48
|
-
//
|
|
49
|
-
if (typeof config.options?.ref === "string" && config.options.ref) {
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
return syncMirroredRepo(config, options);
|
|
53
|
-
}
|
|
54
|
-
getContentDir(config) {
|
|
55
|
-
if (config.path)
|
|
56
|
-
return config.path;
|
|
57
|
-
if (config.url) {
|
|
58
|
-
const repo = parseGitRepoUrl(config.url);
|
|
59
|
-
return getCachePaths(repo.canonicalUrl).repoDir;
|
|
60
|
-
}
|
|
61
|
-
throw new ConfigError("git stash entry must have either `path` or `url`");
|
|
62
|
-
}
|
|
63
|
-
async remove(config) {
|
|
64
|
-
if (config.path && isDirectory(config.path)) {
|
|
65
|
-
try {
|
|
66
|
-
fs.rmSync(path.dirname(config.path), { recursive: true, force: true });
|
|
67
|
-
}
|
|
68
|
-
catch {
|
|
69
|
-
/* best-effort */
|
|
70
|
-
}
|
|
71
|
-
}
|
|
72
|
-
else if (config.url) {
|
|
73
|
-
const repo = parseGitRepoUrl(config.url);
|
|
74
|
-
const paths = getCachePaths(repo.canonicalUrl);
|
|
75
|
-
try {
|
|
76
|
-
fs.rmSync(paths.rootDir, { recursive: true, force: true });
|
|
77
|
-
}
|
|
78
|
-
catch {
|
|
79
|
-
/* best-effort */
|
|
80
|
-
}
|
|
54
|
+
// 2. One-shot install ref (options.ref like "git:..." / "github:...") —
|
|
55
|
+
// delegate to the install-time pipeline.
|
|
56
|
+
if (typeof this.#config.options?.ref === "string" && this.#config.options.ref) {
|
|
57
|
+
await syncRegistryGitRef(String(this.#config.options.ref));
|
|
58
|
+
return;
|
|
81
59
|
}
|
|
60
|
+
await syncMirroredRepo(this.#config);
|
|
82
61
|
}
|
|
83
62
|
}
|
|
63
|
+
/** Resolve the on-disk content directory for a configured git source. */
|
|
64
|
+
function resolveGitContentDir(config) {
|
|
65
|
+
if (config.path)
|
|
66
|
+
return config.path;
|
|
67
|
+
if (config.url) {
|
|
68
|
+
const repo = parseGitRepoUrl(config.url);
|
|
69
|
+
return getCachePaths(repo.canonicalUrl).repoDir;
|
|
70
|
+
}
|
|
71
|
+
throw new ConfigError("git source entry must have either `path` or `url`");
|
|
72
|
+
}
|
|
84
73
|
// ── Self-register ───────────────────────────────────────────────────────────
|
|
85
|
-
|
|
74
|
+
registerSourceProvider("git", (config) => new GitSourceProvider(config));
|
|
86
75
|
// ── Cache management ────────────────────────────────────────────────────────
|
|
87
76
|
function getCachePaths(repoUrl) {
|
|
88
77
|
const key = createHash("sha256").update(repoUrl).digest("hex").slice(0, 16);
|
|
@@ -414,7 +403,7 @@ export function saveGitStash(name, message, writableOverride) {
|
|
|
414
403
|
let writable = false;
|
|
415
404
|
if (name) {
|
|
416
405
|
const config = loadConfig();
|
|
417
|
-
const stash = config.stashes
|
|
406
|
+
const stash = (config.sources ?? config.stashes ?? []).find((s) => s.name === name || s.url === name);
|
|
418
407
|
if (!stash)
|
|
419
408
|
throw new UsageError(`No git stash found with name "${name}"`);
|
|
420
409
|
if (!GIT_STASH_TYPES.has(stash.type)) {
|
|
@@ -476,4 +465,4 @@ export function saveGitStash(name, message, writableOverride) {
|
|
|
476
465
|
};
|
|
477
466
|
}
|
|
478
467
|
// ── Exports ─────────────────────────────────────────────────────────────────
|
|
479
|
-
export { ensureGitMirror,
|
|
468
|
+
export { ensureGitMirror, GitSourceProvider, getCachePaths, parseGitRepoUrl };
|
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Centralized
|
|
2
|
+
* Centralized source provider registration.
|
|
3
3
|
*
|
|
4
|
-
* Import this module (side-effect import) to register all built-in
|
|
4
|
+
* Import this module (side-effect import) to register all built-in source
|
|
5
5
|
* providers with the provider registry. This replaces the individual
|
|
6
|
-
* side-effect imports that were duplicated in
|
|
6
|
+
* side-effect imports that were duplicated in source-search.ts and source-show.ts.
|
|
7
7
|
*/
|
|
8
8
|
import "./filesystem";
|
|
9
9
|
import "./git";
|
|
10
10
|
import "./npm";
|
|
11
|
-
import "./openviking";
|
|
12
11
|
import "./website";
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Install-time types used by `syncFromRef` and the legacy install pipeline.
|
|
3
|
+
*
|
|
4
|
+
* Distinct from the v1 {@link SourceProvider} interface (which only deals
|
|
5
|
+
* with "configured sources" — entries already resolved into a directory).
|
|
6
|
+
* These types describe the resolution+lockfile step that runs when
|
|
7
|
+
* `akm add <install-ref>` materialises an upstream artifact into a local
|
|
8
|
+
* cache directory.
|
|
9
|
+
*
|
|
10
|
+
* They live here, outside `provider.ts`, so the v1 SourceProvider
|
|
11
|
+
* interface stays minimal (`{ name, kind, init, path, sync? }`) per the
|
|
12
|
+
* architecture spec §2.1.
|
|
13
|
+
*/
|
|
14
|
+
export {};
|
|
@@ -12,53 +12,46 @@
|
|
|
12
12
|
*/
|
|
13
13
|
import fs from "node:fs";
|
|
14
14
|
import path from "node:path";
|
|
15
|
-
import { ConfigError, UsageError } from "
|
|
16
|
-
import { getRegistryCacheDir } from "
|
|
17
|
-
import { parseRegistryRef, resolveRegistryArtifact } from "
|
|
18
|
-
import {
|
|
15
|
+
import { ConfigError, UsageError } from "../../core/errors";
|
|
16
|
+
import { getRegistryCacheDir } from "../../core/paths";
|
|
17
|
+
import { parseRegistryRef, resolveRegistryArtifact } from "../../registry/resolve";
|
|
18
|
+
import { registerSourceProvider } from "../provider-factory";
|
|
19
19
|
import { applyAkmIncludeConfig, buildInstallCacheDir, computeFileHash, detectStashRoot, downloadArchive, isDirectory, } from "./provider-utils";
|
|
20
20
|
import { extractTarGzSecure, verifyArchiveIntegrity } from "./tar-utils";
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
21
|
+
/**
|
|
22
|
+
* NPM source provider — fetches a tarball from the npm registry and extracts
|
|
23
|
+
* it into a local cache. Implements the v1 {@link SourceProvider} interface
|
|
24
|
+
* (spec §2.1): `{ name, kind, init, path, sync }`.
|
|
25
|
+
*
|
|
26
|
+
* The install-time pipeline (`syncNpmRef`) lives below as a standalone
|
|
27
|
+
* function used by `akm add` / `akm update` — that path produces a
|
|
28
|
+
* {@link SourceLockData} record for lockfile bookkeeping. The provider's own
|
|
29
|
+
* `sync()` is a void refresh (delegates to the install pipeline but discards
|
|
30
|
+
* the lock data, which is owned by `lockfile.ts`).
|
|
31
|
+
*/
|
|
32
|
+
class NpmSourceProvider {
|
|
33
|
+
kind = "npm";
|
|
24
34
|
name;
|
|
35
|
+
#config;
|
|
25
36
|
constructor(config) {
|
|
37
|
+
this.#config = config;
|
|
26
38
|
this.name = config.name ?? config.url ?? "npm";
|
|
27
39
|
}
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
40
|
+
async init(_ctx) {
|
|
41
|
+
// Resolution happens lazily in path(): until `sync()` runs there's no
|
|
42
|
+
// reliable on-disk path. Init is the registration handshake.
|
|
31
43
|
}
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
44
|
+
path() {
|
|
45
|
+
if (this.#config.path)
|
|
46
|
+
return this.#config.path;
|
|
47
|
+
throw new ConfigError(`npm source "${this.name}" has no resolved content path — run \`akm update\` to sync it before indexing.`);
|
|
35
48
|
}
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
async sync(config, options) {
|
|
40
|
-
const ref = npmRefFromConfig(config);
|
|
41
|
-
return syncNpmRef(ref, options);
|
|
42
|
-
}
|
|
43
|
-
getContentDir(config) {
|
|
44
|
-
if (config.path)
|
|
45
|
-
return config.path;
|
|
46
|
-
throw new ConfigError("npm stash entry missing resolved content path");
|
|
47
|
-
}
|
|
48
|
-
async remove(config) {
|
|
49
|
-
if (config.path && isDirectory(config.path)) {
|
|
50
|
-
// Remove the whole versioned cache dir if we know the parent layout.
|
|
51
|
-
const parent = path.dirname(config.path);
|
|
52
|
-
try {
|
|
53
|
-
fs.rmSync(parent, { recursive: true, force: true });
|
|
54
|
-
}
|
|
55
|
-
catch {
|
|
56
|
-
/* best-effort */
|
|
57
|
-
}
|
|
58
|
-
}
|
|
49
|
+
async sync() {
|
|
50
|
+
const ref = npmRefFromConfig(this.#config);
|
|
51
|
+
await syncNpmRef(ref);
|
|
59
52
|
}
|
|
60
53
|
}
|
|
61
|
-
|
|
54
|
+
registerSourceProvider("npm", (config) => new NpmSourceProvider(config));
|
|
62
55
|
function npmRefFromConfig(config) {
|
|
63
56
|
// Prefer an explicit ref-bearing field (set by akmAdd when persisting), else fall back
|
|
64
57
|
// to options or url so the provider stays usable from a hand-rolled config.
|
|
@@ -69,7 +62,7 @@ function npmRefFromConfig(config) {
|
|
|
69
62
|
return candidate.startsWith("npm:") ? candidate : `npm:${candidate}`;
|
|
70
63
|
}
|
|
71
64
|
/**
|
|
72
|
-
* Fetch and extract an npm tarball, returning a populated `
|
|
65
|
+
* Fetch and extract an npm tarball, returning a populated `SourceLockData`.
|
|
73
66
|
*
|
|
74
67
|
* Mirrors the historical `installRegistryRef()` path for npm sources:
|
|
75
68
|
* - resolve artifact URL + integrity from the npm registry
|
|
@@ -127,9 +120,17 @@ async function doSyncNpm(parsed, options) {
|
|
|
127
120
|
verifyArchiveIntegrity(archivePath, resolved.resolvedRevision, resolved.source);
|
|
128
121
|
integrity = await computeFileHash(archivePath);
|
|
129
122
|
extractTarGzSecure(archivePath, extractedDir);
|
|
130
|
-
|
|
123
|
+
const detectedProvisionalKitRoot = detectStashRoot(extractedDir);
|
|
124
|
+
if (!detectedProvisionalKitRoot) {
|
|
125
|
+
throw new UsageError(`Unable to detect a stash root in extracted npm package: ${resolved.ref}`);
|
|
126
|
+
}
|
|
127
|
+
provisionalKitRoot = detectedProvisionalKitRoot;
|
|
131
128
|
installRoot = applyAkmIncludeConfig(provisionalKitRoot, cacheDir, extractedDir) ?? provisionalKitRoot;
|
|
132
|
-
|
|
129
|
+
const detectedStashRoot = detectStashRoot(installRoot);
|
|
130
|
+
if (!detectedStashRoot) {
|
|
131
|
+
throw new UsageError(`Unable to detect a stash root after applying .akm-include configuration for npm package: ${resolved.ref}`);
|
|
132
|
+
}
|
|
133
|
+
stashRoot = detectedStashRoot;
|
|
133
134
|
}
|
|
134
135
|
catch (err) {
|
|
135
136
|
// Clean up so stale or partial extractions don't cause false cache hits.
|
|
@@ -156,4 +157,4 @@ async function doSyncNpm(parsed, options) {
|
|
|
156
157
|
syncedAt,
|
|
157
158
|
};
|
|
158
159
|
}
|
|
159
|
-
export {
|
|
160
|
+
export { NpmSourceProvider };
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import fs from "node:fs";
|
|
3
3
|
import path from "node:path";
|
|
4
|
-
import { TYPE_DIRS } from "
|
|
5
|
-
import { fetchWithRetry } from "
|
|
6
|
-
import { copyIncludedPaths, findNearestIncludeConfig } from "../
|
|
4
|
+
import { TYPE_DIRS } from "../../core/asset-spec";
|
|
5
|
+
import { fetchWithRetry } from "../../core/common";
|
|
6
|
+
import { copyIncludedPaths, findNearestIncludeConfig } from "../include";
|
|
7
7
|
const REGISTRY_STASH_DIR_NAMES = new Set(Object.values(TYPE_DIRS));
|
|
8
8
|
/** Strip terminal control characters from untrusted strings. */
|
|
9
9
|
export function sanitizeString(value, maxLength = 255) {
|
|
@@ -9,8 +9,8 @@
|
|
|
9
9
|
* `akmUpdate`) decide whether to run `auditInstallCandidate` on the
|
|
10
10
|
* synced `contentDir` because they own the `--trust` flag.
|
|
11
11
|
*/
|
|
12
|
-
import { UsageError } from "
|
|
13
|
-
import { parseRegistryRef } from "
|
|
12
|
+
import { UsageError } from "../../core/errors";
|
|
13
|
+
import { parseRegistryRef } from "../../registry/resolve";
|
|
14
14
|
import { detectStashRoot } from "./provider-utils";
|
|
15
15
|
export async function syncFromRef(ref, options) {
|
|
16
16
|
const parsed = parseRegistryRef(ref);
|
|
@@ -6,15 +6,15 @@
|
|
|
6
6
|
* and verify integrity hashes (SRI or hex shasum) before extraction.
|
|
7
7
|
*
|
|
8
8
|
* Extracted from `registry-install.ts` and shared by all syncable
|
|
9
|
-
* providers that fetch tarballs (currently `
|
|
9
|
+
* providers that fetch tarballs (currently `NpmSourceProvider` and the
|
|
10
10
|
* registry index builder).
|
|
11
11
|
*/
|
|
12
12
|
import { spawnSync } from "node:child_process";
|
|
13
13
|
import { createHash } from "node:crypto";
|
|
14
14
|
import fs from "node:fs";
|
|
15
15
|
import path from "node:path";
|
|
16
|
-
import { isWithin } from "
|
|
17
|
-
import { warn } from "
|
|
16
|
+
import { isWithin } from "../../core/common";
|
|
17
|
+
import { warn } from "../../core/warn";
|
|
18
18
|
/**
|
|
19
19
|
* Verify an archive's integrity against a known hash. Throws and removes
|
|
20
20
|
* the archive when verification fails.
|
|
@@ -103,11 +103,14 @@ function scanExtractedFiles(dir, root) {
|
|
|
103
103
|
throw new Error(`Post-extraction scan: symlink escapes destination directory: ${fullPath} -> ${target}`);
|
|
104
104
|
}
|
|
105
105
|
}
|
|
106
|
-
// Belt-and-suspenders:
|
|
107
|
-
// the destination root
|
|
108
|
-
//
|
|
109
|
-
if (!entry.isSymbolicLink()
|
|
110
|
-
|
|
106
|
+
// Belt-and-suspenders: check that the resolved path of regular entries
|
|
107
|
+
// stays within the destination root. This catches path traversal attempts
|
|
108
|
+
// via symlink TOCTOU, directory renames, or any other anomalies.
|
|
109
|
+
if (!entry.isSymbolicLink()) {
|
|
110
|
+
const resolved = path.resolve(fullPath);
|
|
111
|
+
if (!isWithin(resolved, root)) {
|
|
112
|
+
throw new Error(`Post-extraction scan: entry escapes destination directory: ${fullPath}`);
|
|
113
|
+
}
|
|
111
114
|
}
|
|
112
115
|
if (entry.isDirectory()) {
|
|
113
116
|
scanExtractedFiles(fullPath, root);
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import fs from "node:fs";
|
|
3
3
|
import path from "node:path";
|
|
4
|
-
import { fetchWithRetry, ResponseTooLargeError, readBodyWithByteCap } from "
|
|
5
|
-
import { ConfigError, UsageError } from "
|
|
6
|
-
import { getRegistryIndexCacheDir } from "
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
4
|
+
import { fetchWithRetry, ResponseTooLargeError, readBodyWithByteCap } from "../../core/common";
|
|
5
|
+
import { ConfigError, UsageError } from "../../core/errors";
|
|
6
|
+
import { getRegistryIndexCacheDir } from "../../core/paths";
|
|
7
|
+
import { warn } from "../../core/warn";
|
|
8
|
+
import { registerSourceProvider } from "../provider-factory";
|
|
9
|
+
import { isExpired, sanitizeString } from "./provider-utils";
|
|
9
10
|
/** Refresh website snapshots every 12 hours to balance freshness with scraping load. */
|
|
10
11
|
const CACHE_REFRESH_INTERVAL_MS = 12 * 60 * 60 * 1000;
|
|
11
12
|
/** Allow up to 7 days of stale snapshots when refresh fails so search remains available during outages. */
|
|
@@ -28,68 +29,34 @@ const WEBSITE_PAGE_BYTE_CAP = 5 * 1024 * 1024;
|
|
|
28
29
|
*/
|
|
29
30
|
const WEBSITE_CRAWL_WALL_CLOCK_MS = 10 * 60 * 1000;
|
|
30
31
|
/**
|
|
31
|
-
* Website
|
|
32
|
-
*
|
|
33
|
-
*
|
|
32
|
+
* Website source provider — scrapes pages into a local mirror so the FTS5
|
|
33
|
+
* indexer can walk them. Implements the v1 {@link SourceProvider} interface
|
|
34
|
+
* (spec §2.1): `{ name, kind, init, path, sync }`.
|
|
35
|
+
*
|
|
36
|
+
* Reading is the indexer's job — this class doesn't implement `search` or
|
|
37
|
+
* `show`.
|
|
34
38
|
*/
|
|
35
|
-
class
|
|
36
|
-
|
|
37
|
-
kind = "syncable";
|
|
39
|
+
class WebsiteSourceProvider {
|
|
40
|
+
kind = "website";
|
|
38
41
|
name;
|
|
39
|
-
config;
|
|
42
|
+
#config;
|
|
43
|
+
#url;
|
|
40
44
|
constructor(config) {
|
|
41
|
-
this
|
|
45
|
+
this.#config = config;
|
|
42
46
|
this.name = config.name ?? "website";
|
|
43
|
-
validateWebsiteUrl(config.url ?? "");
|
|
47
|
+
this.#url = validateWebsiteUrl(config.url ?? "");
|
|
44
48
|
}
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
return { hits: [] };
|
|
49
|
+
async init(_ctx) {
|
|
50
|
+
// URL validation already happens in the constructor; nothing else to do.
|
|
48
51
|
}
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
throw new Error("Website provider content is shown via local index");
|
|
52
|
+
path() {
|
|
53
|
+
return getCachePaths(this.#url).stashDir;
|
|
52
54
|
}
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
return false;
|
|
56
|
-
}
|
|
57
|
-
async sync(config, options) {
|
|
58
|
-
const cachePaths = await ensureWebsiteMirror(config, { requireStashDir: true, force: options?.force });
|
|
59
|
-
const syncedAt = (options?.now ?? new Date()).toISOString();
|
|
60
|
-
const url = config.url ?? "";
|
|
61
|
-
// #123 added "website" to the StashSource union, so we can use it directly.
|
|
62
|
-
return {
|
|
63
|
-
id: url,
|
|
64
|
-
source: "website",
|
|
65
|
-
ref: url,
|
|
66
|
-
artifactUrl: url,
|
|
67
|
-
contentDir: cachePaths.stashDir,
|
|
68
|
-
cacheDir: cachePaths.rootDir,
|
|
69
|
-
extractedDir: cachePaths.stashDir,
|
|
70
|
-
syncedAt,
|
|
71
|
-
};
|
|
72
|
-
}
|
|
73
|
-
getContentDir(config) {
|
|
74
|
-
const url = config.url ?? "";
|
|
75
|
-
return getCachePaths(url).stashDir;
|
|
76
|
-
}
|
|
77
|
-
async remove(config) {
|
|
78
|
-
const url = config.url;
|
|
79
|
-
if (!url)
|
|
80
|
-
return;
|
|
81
|
-
const paths = getCachePaths(url);
|
|
82
|
-
if (isDirectory(paths.rootDir)) {
|
|
83
|
-
try {
|
|
84
|
-
fs.rmSync(paths.rootDir, { recursive: true, force: true });
|
|
85
|
-
}
|
|
86
|
-
catch {
|
|
87
|
-
/* best-effort */
|
|
88
|
-
}
|
|
89
|
-
}
|
|
55
|
+
async sync() {
|
|
56
|
+
await ensureWebsiteMirror(this.#config, { requireStashDir: true });
|
|
90
57
|
}
|
|
91
58
|
}
|
|
92
|
-
|
|
59
|
+
registerSourceProvider("website", (config) => new WebsiteSourceProvider(config));
|
|
93
60
|
function getCachePaths(siteUrl) {
|
|
94
61
|
const key = createHash("sha256").update(normalizeSiteUrl(siteUrl)).digest("hex").slice(0, 16);
|
|
95
62
|
const rootDir = path.join(getRegistryIndexCacheDir(), `website-${key}`);
|
|
@@ -182,12 +149,9 @@ async function crawlWebsite(startUrl, options) {
|
|
|
182
149
|
const visited = new Set();
|
|
183
150
|
const pages = [];
|
|
184
151
|
const deadline = Date.now() + WEBSITE_CRAWL_WALL_CLOCK_MS;
|
|
185
|
-
let stoppedAtDeadline = false;
|
|
186
152
|
while (queue.length > 0 && pages.length < options.maxPages) {
|
|
187
|
-
if (Date.now() > deadline)
|
|
188
|
-
stoppedAtDeadline = true;
|
|
153
|
+
if (Date.now() > deadline)
|
|
189
154
|
break;
|
|
190
|
-
}
|
|
191
155
|
const next = queue.shift();
|
|
192
156
|
if (!next)
|
|
193
157
|
break;
|
|
@@ -212,8 +176,8 @@ async function crawlWebsite(startUrl, options) {
|
|
|
212
176
|
queue.push({ url: candidate, depth: next.depth + 1 });
|
|
213
177
|
}
|
|
214
178
|
}
|
|
215
|
-
if (
|
|
216
|
-
|
|
179
|
+
if (Date.now() > deadline) {
|
|
180
|
+
warn("[akm] website crawl stopped at the %ds wall-clock cap with %d/%d pages collected from %s.", WEBSITE_CRAWL_WALL_CLOCK_MS / 1000, pages.length, options.maxPages, startUrl);
|
|
217
181
|
}
|
|
218
182
|
return pages;
|
|
219
183
|
}
|
|
@@ -516,4 +480,4 @@ function safeCodePointToString(value) {
|
|
|
516
480
|
return undefined;
|
|
517
481
|
}
|
|
518
482
|
}
|
|
519
|
-
export { ensureWebsiteMirror, getCachePaths, validateWebsiteInputUrl, validateWebsiteUrl,
|
|
483
|
+
export { ensureWebsiteMirror, getCachePaths, validateWebsiteInputUrl, validateWebsiteUrl, WebsiteSourceProvider };
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
|
-
import { deriveCanonicalAssetNameFromStashRoot, isRelevantAssetFile, resolveAssetPathFromName, TYPE_DIRS, } from "
|
|
4
|
-
import { hasErrnoCode, isWithin } from "
|
|
5
|
-
import { NotFoundError, UsageError } from "
|
|
6
|
-
import { runMatchers } from "
|
|
7
|
-
import { walkStashFlat } from "
|
|
3
|
+
import { deriveCanonicalAssetNameFromStashRoot, isRelevantAssetFile, resolveAssetPathFromName, TYPE_DIRS, } from "../core/asset-spec";
|
|
4
|
+
import { hasErrnoCode, isWithin } from "../core/common";
|
|
5
|
+
import { NotFoundError, UsageError } from "../core/errors";
|
|
6
|
+
import { runMatchers } from "../indexer/file-context";
|
|
7
|
+
import { walkStashFlat } from "../indexer/walker";
|
|
8
8
|
/**
|
|
9
9
|
* Resolve an asset path from a stash directory, type, and name.
|
|
10
10
|
*/
|
|
@@ -50,7 +50,8 @@ function resolveInTypeDir(stashDir, typeDir, type, name) {
|
|
|
50
50
|
function resolveAndValidateTypeRoot(root, type, name) {
|
|
51
51
|
const rootStat = readTypeRootStat(root, type, name);
|
|
52
52
|
if (!rootStat.isDirectory()) {
|
|
53
|
-
throw new NotFoundError(`
|
|
53
|
+
throw new NotFoundError(`Asset directory for ${type} assets is not accessible — got a file where a directory was expected for ref: ${type}:${name}. ` +
|
|
54
|
+
"Run `akm index` to rebuild the index, or check your source configuration.", "ASSET_NOT_FOUND", "Run `akm list` to see your configured sources and verify the source path exists.");
|
|
54
55
|
}
|
|
55
56
|
return fs.realpathSync(root);
|
|
56
57
|
}
|
|
@@ -60,7 +61,7 @@ function readTypeRootStat(root, type, name) {
|
|
|
60
61
|
}
|
|
61
62
|
catch (error) {
|
|
62
63
|
if (hasErrnoCode(error, "ENOENT")) {
|
|
63
|
-
throw new NotFoundError(`
|
|
64
|
+
throw new NotFoundError(`Asset not found for ref: ${type}:${name}. No ${type} assets are present in the configured source.`, "ASSET_NOT_FOUND", "Run `akm list` to see your configured sources, or `akm index` to rebuild the search index.");
|
|
64
65
|
}
|
|
65
66
|
throw error;
|
|
66
67
|
}
|
|
@@ -15,13 +15,13 @@
|
|
|
15
15
|
import fs from "node:fs";
|
|
16
16
|
import path from "node:path";
|
|
17
17
|
import { parse as yamlParse } from "yaml";
|
|
18
|
-
import {
|
|
19
|
-
import {
|
|
20
|
-
import {
|
|
21
|
-
import {
|
|
22
|
-
import {
|
|
23
|
-
import {
|
|
24
|
-
import { buildIndexMd, buildLogMd, buildSchemaMd } from "
|
|
18
|
+
import { akmSearch } from "../commands/search";
|
|
19
|
+
import { isWithin } from "../core/common";
|
|
20
|
+
import { loadUserConfig, saveConfig } from "../core/config";
|
|
21
|
+
import { NotFoundError, UsageError } from "../core/errors";
|
|
22
|
+
import { parseFrontmatter, parseFrontmatterBlock } from "../core/frontmatter";
|
|
23
|
+
import { resolveSourceEntries } from "../indexer/search-source";
|
|
24
|
+
import { buildIndexMd, buildLogMd, buildSchemaMd } from "../templates/wiki-templates";
|
|
25
25
|
// ── Constants ───────────────────────────────────────────────────────────────
|
|
26
26
|
export const WIKIS_SUBDIR = "wikis";
|
|
27
27
|
export const SCHEMA_MD = "schema.md";
|
|
@@ -64,7 +64,7 @@ function wikiNotFoundMessage(name) {
|
|
|
64
64
|
return `Wiki not found: ${name}. Run \`akm wiki create ${name}\` to create it or \`akm wiki register ${name} <path-or-repo>\` to register an external wiki.`;
|
|
65
65
|
}
|
|
66
66
|
function registeredWikiSources(stashDir) {
|
|
67
|
-
return
|
|
67
|
+
return resolveSourceEntries(stashDir)
|
|
68
68
|
.filter((source) => typeof source.wikiName === "string")
|
|
69
69
|
.map((source) => ({
|
|
70
70
|
name: source.wikiName,
|
|
@@ -357,11 +357,12 @@ export function removeWiki(stashDir, name, options = {}) {
|
|
|
357
357
|
const wikiDir = resolved.path;
|
|
358
358
|
if (resolved.mode === "external") {
|
|
359
359
|
const config = loadUserConfig();
|
|
360
|
-
const
|
|
360
|
+
const filteredSources = (config.sources ?? config.stashes ?? []).filter((entry) => entry.wikiName !== name);
|
|
361
361
|
const installed = (config.installed ?? []).filter((entry) => entry.wikiName !== name);
|
|
362
362
|
saveConfig({
|
|
363
363
|
...config,
|
|
364
|
-
|
|
364
|
+
sources: filteredSources.length > 0 ? filteredSources : undefined,
|
|
365
|
+
stashes: undefined,
|
|
365
366
|
installed: installed.length > 0 ? installed : undefined,
|
|
366
367
|
});
|
|
367
368
|
return {
|
|
@@ -527,7 +528,7 @@ export async function searchInWiki(input) {
|
|
|
527
528
|
const rawDir = path.join(wikiDir, RAW_SUBDIR);
|
|
528
529
|
const filtered = [];
|
|
529
530
|
for (const hit of response.hits) {
|
|
530
|
-
// hits can be
|
|
531
|
+
// hits can be SourceSearchHit or RegistrySearchResultHit (union); filter
|
|
531
532
|
// by path inclusion. Registry hits have no path and are dropped.
|
|
532
533
|
if (hit.type === "registry")
|
|
533
534
|
continue;
|