akm-cli 0.10.261124 → 0.10.261125
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/dist/cli/usage.js +4 -0
- package/dist/cli.js +10 -0
- package/dist/commands/feedback-cli.js +12 -3
- package/dist/commands/health/semantic-search-check.js +12 -2
- package/dist/commands/health.js +8 -15
- package/dist/commands/lint/index.js +6 -6
- package/dist/commands/read/knowledge.js +8 -8
- package/dist/commands/read/related.js +3 -9
- package/dist/commands/read/show.js +17 -9
- package/dist/commands/sources/bundle-cli.js +1 -0
- package/dist/commands/sources/installed-stashes.js +36 -27
- package/dist/core/asset/resolve-ref.js +8 -0
- package/dist/core/common.js +21 -0
- package/dist/core/endpoint-url.js +4 -4
- package/dist/core/events.js +8 -2
- package/dist/core/file-lock.js +52 -1
- package/dist/core/state-db.js +47 -15
- package/dist/core/stdio-reader-gone.js +19 -0
- package/dist/core/write-source.js +2 -2
- package/dist/indexer/index-build.js +6 -3
- package/dist/indexer/index-readiness.js +30 -4
- package/dist/indexer/indexer.js +56 -8
- package/dist/indexer/search/db-search.js +7 -1
- package/dist/indexer/search/search-source.js +23 -25
- package/dist/indexer/usage/usage-events.js +13 -4
- package/dist/indexer/vector-file.js +3 -3
- package/dist/registry/origin-resolve.js +2 -3
- package/dist/scripts/akm-migrate-node.js +136 -88
- package/dist/scripts/akm-migrate.js +136 -88
- package/dist/setup/endpoint.js +1 -1
- package/dist/storage/database.js +29 -36
- package/dist/storage/repositories/index-connection.js +44 -41
- package/dist/storage/repositories/index-entries-repository.js +14 -7
- package/dist/storage/repositories/index-entry-schema.js +7 -5
- package/dist/storage/repositories/index-schema.js +23 -0
- package/dist/storage/sqlite-read-snapshot.js +2 -2
- package/dist/workspace/assets.js +24 -44
- package/dist/workspace/bundle-files.js +2 -2
- package/dist/workspace/documents.js +7 -12
- package/dist/workspace/graph-topics.js +249 -0
- package/dist/workspace/graph.js +68 -4
- package/docs/reference/workspace.md +21 -4
- package/package.json +1 -1
|
@@ -18,7 +18,7 @@ import path from "node:path";
|
|
|
18
18
|
import { detectAdapterId } from "../../core/adapter/detect-adapter.js";
|
|
19
19
|
import { isWithin, resolveStashDir } from "../../core/common.js";
|
|
20
20
|
import { bundleComponentConfig, getSources, loadConfig, resolveSecret } from "../../core/config/config.js";
|
|
21
|
-
import { AkmError, ConfigError, NotFoundError, UsageError } from "../../core/errors.js";
|
|
21
|
+
import { AkmError, ConfigError, NotFoundError, TransientError, UsageError } from "../../core/errors.js";
|
|
22
22
|
import { isPathAbsent } from "../../core/path-access.js";
|
|
23
23
|
import { getDbPath, getRegistryCacheDir } from "../../core/paths.js";
|
|
24
24
|
import { warn } from "../../core/warn.js";
|
|
@@ -408,8 +408,8 @@ function fastForwardCheckout(liveRepo, stagedRepo) {
|
|
|
408
408
|
* that) and detected adapters are not written to config.json — an update
|
|
409
409
|
* leaves config alone; the next `akm index` records them.
|
|
410
410
|
*/
|
|
411
|
-
function reindexAfterUpdate(stashDir) {
|
|
412
|
-
return
|
|
411
|
+
function reindexAfterUpdate(stashDir, lockMode) {
|
|
412
|
+
return withIndexBuildLock(undefined, (build) => build(updateIndexOptions(stashDir)), lockMode);
|
|
413
413
|
}
|
|
414
414
|
function updateIndexOptions(stashDir) {
|
|
415
415
|
return { stashDir, hydrateSources: false, persistDetectedAdapters: false };
|
|
@@ -420,7 +420,7 @@ function updateIndexOptions(stashDir) {
|
|
|
420
420
|
* When the index step fails after the publish, the bundle is updated and only its index is stale: say so, then
|
|
421
421
|
* rethrow the original error.
|
|
422
422
|
*/
|
|
423
|
-
function publishAndIndex(stashDir, publish) {
|
|
423
|
+
function publishAndIndex(stashDir, lockMode, publish) {
|
|
424
424
|
return withIndexBuildLock(undefined, (build) => withAssetMutationLease("source-update", async () => {
|
|
425
425
|
await publish();
|
|
426
426
|
try {
|
|
@@ -430,7 +430,7 @@ function publishAndIndex(stashDir, publish) {
|
|
|
430
430
|
warn("The bundle was updated but its search index was not refreshed; `akm index` (or the next scheduled build) will index it.");
|
|
431
431
|
throw error;
|
|
432
432
|
}
|
|
433
|
-
}));
|
|
433
|
+
}), lockMode);
|
|
434
434
|
}
|
|
435
435
|
function managedCredential(managed) {
|
|
436
436
|
return managed.credential ? resolveSecret(managed.credential, storeSecretResolver.resolveSecret) : undefined;
|
|
@@ -561,7 +561,7 @@ async function stageManagedUpdate(managed, force) {
|
|
|
561
561
|
}
|
|
562
562
|
}
|
|
563
563
|
/** Fetch a plain git source into staging, publish it, and reindex. */
|
|
564
|
-
async function syncGitPlainSource(config, gitSource, stashDir) {
|
|
564
|
+
async function syncGitPlainSource(config, gitSource, stashDir, lockMode) {
|
|
565
565
|
const id = gitSource.name ?? gitSource.url ?? "";
|
|
566
566
|
const ref = gitSource.url ?? "";
|
|
567
567
|
if (!gitSource.url)
|
|
@@ -585,7 +585,7 @@ async function syncGitPlainSource(config, gitSource, stashDir) {
|
|
|
585
585
|
? { credential: resolveSecret(gitSource.credential, storeSecretResolver.resolveSecret) }
|
|
586
586
|
: {}),
|
|
587
587
|
});
|
|
588
|
-
const index = await publishAndIndex(stashDir, () => {
|
|
588
|
+
const index = await publishAndIndex(stashDir, lockMode, () => {
|
|
589
589
|
if (liveCheckout)
|
|
590
590
|
fastForwardCheckout(livePaths.repoDir, stagedPaths.repoDir);
|
|
591
591
|
else
|
|
@@ -603,31 +603,31 @@ async function syncGitPlainSource(config, gitSource, stashDir) {
|
|
|
603
603
|
* keeps the previous mirror; a snapshot holds only knowledge pages, so there
|
|
604
604
|
* is no env file in it.
|
|
605
605
|
*/
|
|
606
|
-
async function syncWebsitePlainSource(websiteSource, stashDir) {
|
|
606
|
+
async function syncWebsitePlainSource(websiteSource, stashDir, lockMode) {
|
|
607
607
|
const id = websiteSource.name ?? websiteSource.url ?? "";
|
|
608
608
|
await createWebsiteProvider(websiteSource).sync({
|
|
609
609
|
force: true,
|
|
610
610
|
secrets: storeSecretResolver,
|
|
611
611
|
ensureWebsiteMirror,
|
|
612
612
|
});
|
|
613
|
-
const index = await reindexAfterUpdate(stashDir);
|
|
613
|
+
const index = await reindexAfterUpdate(stashDir, lockMode);
|
|
614
614
|
return { item: { id, kind: "website", ref: websiteSource.url ?? "" }, index };
|
|
615
615
|
}
|
|
616
616
|
/** Sync a git-mirrored (plain) source and return an UpdateResponse (single-target path). */
|
|
617
|
-
async function updateGitSource(config, stashDir, target, all, gitSource) {
|
|
618
|
-
const synced = await syncGitPlainSource(config, gitSource, stashDir);
|
|
617
|
+
async function updateGitSource(config, stashDir, target, all, gitSource, lockMode) {
|
|
618
|
+
const synced = await syncGitPlainSource(config, gitSource, stashDir, lockMode);
|
|
619
619
|
return buildUpdateResponse(stashDir, target, all, [], { plainSynced: [synced.item], index: synced.index });
|
|
620
620
|
}
|
|
621
621
|
/** Re-crawl a website (plain) source and return an UpdateResponse (single-target path). */
|
|
622
|
-
async function updateWebsiteSource(stashDir, target, all, websiteSource) {
|
|
623
|
-
const synced = await syncWebsitePlainSource(websiteSource, stashDir);
|
|
622
|
+
async function updateWebsiteSource(stashDir, target, all, websiteSource, lockMode) {
|
|
623
|
+
const synced = await syncWebsitePlainSource(websiteSource, stashDir, lockMode);
|
|
624
624
|
return buildUpdateResponse(stashDir, target, all, [], { plainSynced: [synced.item], index: synced.index });
|
|
625
625
|
}
|
|
626
626
|
/** Reconcile a filesystem source's current bytes without provider hydration. */
|
|
627
|
-
async function updateFilesystemSource(stashDir, target, all, filesystemSource) {
|
|
627
|
+
async function updateFilesystemSource(stashDir, target, all, filesystemSource, lockMode) {
|
|
628
628
|
const id = filesystemSource.name ?? filesystemSource.path ?? target;
|
|
629
629
|
const ref = filesystemSource.path ?? target;
|
|
630
|
-
const index = await
|
|
630
|
+
const index = await withIndexBuildLock(undefined, (build) => build({ stashDir, hydrateSources: false }), lockMode);
|
|
631
631
|
if (!index.scanComplete) {
|
|
632
632
|
return buildUpdateResponse(stashDir, target, all, [], {
|
|
633
633
|
skipped: [incompleteFilesystemOutcome(id)],
|
|
@@ -677,7 +677,7 @@ function managedInstallViewOfPlainNpm(config, npmSource) {
|
|
|
677
677
|
* Those directories are a re-fetchable cache (local and writable installs never
|
|
678
678
|
* take this path), so none of this prompts.
|
|
679
679
|
*/
|
|
680
|
-
async function updateManagedInstall(managed, force, stashDir) {
|
|
680
|
+
async function updateManagedInstall(managed, force, stashDir, lockMode) {
|
|
681
681
|
const unchanged = await unchangedManagedItem(managed, force);
|
|
682
682
|
if (unchanged)
|
|
683
683
|
return { item: unchanged };
|
|
@@ -710,7 +710,7 @@ async function updateManagedInstall(managed, force, stashDir) {
|
|
|
710
710
|
(managed.resolvedVersion ?? "") === (synced.resolvedVersion ?? "");
|
|
711
711
|
const index = nothingNew
|
|
712
712
|
? undefined
|
|
713
|
-
: await publishAndIndex(stashDir, async () => {
|
|
713
|
+
: await publishAndIndex(stashDir, lockMode, async () => {
|
|
714
714
|
staged.publish();
|
|
715
715
|
await upsertLockEntry({
|
|
716
716
|
id: managed.bundleKey,
|
|
@@ -805,13 +805,14 @@ export async function akmUpdate(input) {
|
|
|
805
805
|
const target = input?.target?.trim();
|
|
806
806
|
const all = input?.all === true;
|
|
807
807
|
const force = input?.force === true;
|
|
808
|
+
const lockMode = input?.skipIfLocked ? "try" : "wait";
|
|
808
809
|
const config = loadConfig();
|
|
809
810
|
const managedInstalls = listManagedInstalls(config);
|
|
810
811
|
if (target && !all) {
|
|
811
812
|
// Registry-managed install (lock-backed) — re-download from its locator.
|
|
812
813
|
const managed = resolveManagedTarget(config, target);
|
|
813
814
|
if (managed) {
|
|
814
|
-
const updated = await updateManagedInstall(managed, force, stashDir);
|
|
815
|
+
const updated = await updateManagedInstall(managed, force, stashDir, lockMode);
|
|
815
816
|
return buildUpdateResponse(stashDir, target, all, [updated.item], {
|
|
816
817
|
index: updated.index ?? readCurrentIndexSummary(),
|
|
817
818
|
});
|
|
@@ -842,7 +843,7 @@ export async function akmUpdate(input) {
|
|
|
842
843
|
return false;
|
|
843
844
|
});
|
|
844
845
|
if (gitMatch)
|
|
845
|
-
return updateGitSource(config, stashDir, target, all, gitMatch);
|
|
846
|
+
return updateGitSource(config, stashDir, target, all, gitMatch, lockMode);
|
|
846
847
|
const websiteMatch = stashes.find((s) => {
|
|
847
848
|
if (s.type !== "website")
|
|
848
849
|
return false;
|
|
@@ -855,7 +856,7 @@ export async function akmUpdate(input) {
|
|
|
855
856
|
return false;
|
|
856
857
|
});
|
|
857
858
|
if (websiteMatch)
|
|
858
|
-
return updateWebsiteSource(stashDir, target, all, websiteMatch);
|
|
859
|
+
return updateWebsiteSource(stashDir, target, all, websiteMatch, lockMode);
|
|
859
860
|
// Plain npm source (bundle without a lock) — sync via the registry
|
|
860
861
|
// pipeline and promote to a managed install (see
|
|
861
862
|
// managedInstallViewOfPlainNpm's doc comment for why npm can't stay
|
|
@@ -870,7 +871,7 @@ export async function akmUpdate(input) {
|
|
|
870
871
|
return false;
|
|
871
872
|
});
|
|
872
873
|
if (npmMatch) {
|
|
873
|
-
const updated = await updateManagedInstall(managedInstallViewOfPlainNpm(config, npmMatch), force, stashDir);
|
|
874
|
+
const updated = await updateManagedInstall(managedInstallViewOfPlainNpm(config, npmMatch), force, stashDir, lockMode);
|
|
874
875
|
return buildUpdateResponse(stashDir, target, all, [updated.item], {
|
|
875
876
|
index: updated.index ?? readCurrentIndexSummary(),
|
|
876
877
|
});
|
|
@@ -883,7 +884,7 @@ export async function akmUpdate(input) {
|
|
|
883
884
|
return resolvedPath !== undefined && source.path !== undefined && path.resolve(source.path) === resolvedPath;
|
|
884
885
|
});
|
|
885
886
|
if (filesystemMatch)
|
|
886
|
-
return updateFilesystemSource(stashDir, target, all, filesystemMatch);
|
|
887
|
+
return updateFilesystemSource(stashDir, target, all, filesystemMatch, lockMode);
|
|
887
888
|
}
|
|
888
889
|
const enabledManagedInstalls = all
|
|
889
890
|
? managedInstalls.filter((managed) => config.bundles?.[managed.bundleKey]?.enabled !== false)
|
|
@@ -895,13 +896,14 @@ export async function akmUpdate(input) {
|
|
|
895
896
|
let latestIndex;
|
|
896
897
|
for (const managed of selected) {
|
|
897
898
|
try {
|
|
898
|
-
const updated = await updateManagedInstall(managed, force, stashDir);
|
|
899
|
+
const updated = await updateManagedInstall(managed, force, stashDir, lockMode);
|
|
899
900
|
processed.push(updated.item);
|
|
900
901
|
latestIndex = updated.index ?? latestIndex;
|
|
901
902
|
}
|
|
902
903
|
catch (error) {
|
|
903
904
|
if (!all)
|
|
904
905
|
throw error;
|
|
906
|
+
rethrowIfSkipped(error, lockMode);
|
|
905
907
|
skipped.push(updateFailureOutcome(managed.installId, sourceKindForInstall(managed.source), error));
|
|
906
908
|
}
|
|
907
909
|
}
|
|
@@ -920,17 +922,17 @@ export async function akmUpdate(input) {
|
|
|
920
922
|
const id = plain.name ?? plain.path ?? plain.url ?? "";
|
|
921
923
|
try {
|
|
922
924
|
if (plain.type === "git") {
|
|
923
|
-
const updated = await syncGitPlainSource(config, plain, stashDir);
|
|
925
|
+
const updated = await syncGitPlainSource(config, plain, stashDir, lockMode);
|
|
924
926
|
plainSynced.push(updated.item);
|
|
925
927
|
latestIndex = updated.index;
|
|
926
928
|
}
|
|
927
929
|
else if (plain.type === "npm") {
|
|
928
|
-
const updated = await updateManagedInstall(managedInstallViewOfPlainNpm(config, plain), force, stashDir);
|
|
930
|
+
const updated = await updateManagedInstall(managedInstallViewOfPlainNpm(config, plain), force, stashDir, lockMode);
|
|
929
931
|
processed.push(updated.item);
|
|
930
932
|
latestIndex = updated.index ?? latestIndex;
|
|
931
933
|
}
|
|
932
934
|
else if (plain.type === "website") {
|
|
933
|
-
const updated = await syncWebsitePlainSource(plain, stashDir);
|
|
935
|
+
const updated = await syncWebsitePlainSource(plain, stashDir, lockMode);
|
|
934
936
|
plainSynced.push(updated.item);
|
|
935
937
|
latestIndex = updated.index;
|
|
936
938
|
}
|
|
@@ -939,6 +941,7 @@ export async function akmUpdate(input) {
|
|
|
939
941
|
}
|
|
940
942
|
}
|
|
941
943
|
catch (error) {
|
|
944
|
+
rethrowIfSkipped(error, lockMode);
|
|
942
945
|
skipped.push(updateFailureOutcome(id, plain.type, error));
|
|
943
946
|
}
|
|
944
947
|
}
|
|
@@ -950,7 +953,7 @@ export async function akmUpdate(input) {
|
|
|
950
953
|
// a genuinely missing filesystem root remains incomplete and is
|
|
951
954
|
// reported below without claiming reconciliation.
|
|
952
955
|
if (!latestIndex?.scanComplete)
|
|
953
|
-
latestIndex = await
|
|
956
|
+
latestIndex = await withIndexBuildLock(undefined, (build) => build({ stashDir, hydrateSources: false }), lockMode);
|
|
954
957
|
if (latestIndex.scanComplete) {
|
|
955
958
|
for (const source of filesystemSources) {
|
|
956
959
|
plainSynced.push({
|
|
@@ -967,6 +970,7 @@ export async function akmUpdate(input) {
|
|
|
967
970
|
}
|
|
968
971
|
}
|
|
969
972
|
catch (error) {
|
|
973
|
+
rethrowIfSkipped(error, lockMode);
|
|
970
974
|
for (const source of filesystemSources) {
|
|
971
975
|
skipped.push(updateFailureOutcome(source.name ?? source.path ?? "", "filesystem", error));
|
|
972
976
|
}
|
|
@@ -986,6 +990,11 @@ function sourceKindForInstall(source) {
|
|
|
986
990
|
return "git";
|
|
987
991
|
return source;
|
|
988
992
|
}
|
|
993
|
+
/** Under `--skip-if-locked` a held rebuild lock skips the whole update; it is not one source's failure. */
|
|
994
|
+
function rethrowIfSkipped(error, lockMode) {
|
|
995
|
+
if (lockMode === "try" && error instanceof TransientError && error.code === "INDEX_DB_CONTENDED")
|
|
996
|
+
throw error;
|
|
997
|
+
}
|
|
989
998
|
function updateFailureOutcome(id, kind, error) {
|
|
990
999
|
const code = error instanceof AkmError ? error.code : undefined;
|
|
991
1000
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -125,6 +125,14 @@ export function displayRefForConceptId(conceptId, bundleId, defaultBundleId) {
|
|
|
125
125
|
// Non-default bundle → the new fully-qualified `bundle//conceptId` grammar.
|
|
126
126
|
return `${bundleId}//${conceptId}`;
|
|
127
127
|
}
|
|
128
|
+
/**
|
|
129
|
+
* The bundle whose items display with the SHORT conceptId: the configured default bundle, else the bundle
|
|
130
|
+
* of the item's own source when that source is the default one. `show` and `feedback` both name an asset
|
|
131
|
+
* by `displayRef` under this rule.
|
|
132
|
+
*/
|
|
133
|
+
export function displayDefaultBundle(configDefault, source, bundleId) {
|
|
134
|
+
return configDefault ?? (source?.isDefault === true ? bundleId : undefined);
|
|
135
|
+
}
|
|
128
136
|
// ── Placement refs (write by placement) ──────────────────────────────────────
|
|
129
137
|
/**
|
|
130
138
|
* Split a conceptId that sits under a type's stash folder (`<stash-subdir>/<name>`)
|
package/dist/core/common.js
CHANGED
|
@@ -452,6 +452,27 @@ export function safeRealpath(p) {
|
|
|
452
452
|
}
|
|
453
453
|
}
|
|
454
454
|
}
|
|
455
|
+
/** Whether two paths name the same location, however either is spelled (symlinks, macOS `/tmp`). */
|
|
456
|
+
export function samePath(a, b) {
|
|
457
|
+
return safeRealpath(a) === safeRealpath(b);
|
|
458
|
+
}
|
|
459
|
+
/**
|
|
460
|
+
* `filePath` relative to `root`, when it sits inside it under either spelling of the root
|
|
461
|
+
* (a symlink, macOS `/tmp` and `/private/tmp`). Only the root's spelling is resolved: a link
|
|
462
|
+
* below the root stays a link, so callers that police it still see it.
|
|
463
|
+
*/
|
|
464
|
+
export function relativeToRoot(filePath, root) {
|
|
465
|
+
const resolved = path.resolve(filePath);
|
|
466
|
+
const base = path.resolve(root);
|
|
467
|
+
if (resolved.startsWith(base + path.sep))
|
|
468
|
+
return resolved.slice(base.length + 1);
|
|
469
|
+
const physicalRoot = safeRealpath(base);
|
|
470
|
+
for (let dir = path.dirname(resolved); dir !== path.dirname(dir); dir = path.dirname(dir)) {
|
|
471
|
+
if (safeRealpath(dir) === physicalRoot)
|
|
472
|
+
return path.relative(dir, resolved);
|
|
473
|
+
}
|
|
474
|
+
return undefined;
|
|
475
|
+
}
|
|
455
476
|
function normalizeFsPathForComparison(value) {
|
|
456
477
|
return process.platform === "win32" ? value.toLowerCase() : value;
|
|
457
478
|
}
|
|
@@ -10,8 +10,8 @@
|
|
|
10
10
|
* setup, the stored-key binding, akm's own runtime) reads it through this
|
|
11
11
|
* one module.
|
|
12
12
|
*/
|
|
13
|
-
/** The operation a complete endpoint URL may end with. */
|
|
14
|
-
const OPERATION_SUFFIX = /\/(?:chat\/completions|embeddings)
|
|
13
|
+
/** The operation a complete endpoint URL may end with, in any case. */
|
|
14
|
+
const OPERATION_SUFFIX = /\/(?:chat\/completions|completions|embeddings|models)$/i;
|
|
15
15
|
function parse(endpoint) {
|
|
16
16
|
try {
|
|
17
17
|
return new URL(endpoint);
|
|
@@ -20,7 +20,7 @@ function parse(endpoint) {
|
|
|
20
20
|
return undefined;
|
|
21
21
|
}
|
|
22
22
|
}
|
|
23
|
-
/** The pathname of the endpoint's base: no trailing slash, no `/chat/completions
|
|
23
|
+
/** The pathname of the endpoint's base: no trailing slash, no operation (`/chat/completions`, `/completions`, `/embeddings`, `/models`). */
|
|
24
24
|
function basePathname(url) {
|
|
25
25
|
return url.pathname.replace(/\/+$/, "").replace(OPERATION_SUFFIX, "");
|
|
26
26
|
}
|
|
@@ -31,7 +31,7 @@ function withPathname(url, pathname) {
|
|
|
31
31
|
// A bare host serializes with a "/" the endpoint never had.
|
|
32
32
|
return pathname === "" && url.search === "" ? text.replace(/\/$/, "") : text;
|
|
33
33
|
}
|
|
34
|
-
/** The base of an OpenAI-compatible endpoint: the URL without
|
|
34
|
+
/** The base of an OpenAI-compatible endpoint: the URL without an operation suffix or a trailing slash. */
|
|
35
35
|
export function openAiBaseUrl(endpoint) {
|
|
36
36
|
const url = parse(endpoint);
|
|
37
37
|
return url ? withPathname(url, basePathname(url)) : endpoint;
|
package/dist/core/events.js
CHANGED
|
@@ -26,7 +26,7 @@ import fs from "node:fs";
|
|
|
26
26
|
import { openDatabase } from "../storage/database.js";
|
|
27
27
|
import { insertEvent, readStateEvents } from "../storage/repositories/events-repository.js";
|
|
28
28
|
import { rethrowIfTestIsolationError } from "./errors.js";
|
|
29
|
-
import { getStateDbPath, openStateDatabase, withStateDb } from "./state-db.js";
|
|
29
|
+
import { classifyStateDbError, damagedStateDbError, getStateDbPath, isStateDbCorruption, openStateDatabase, withStateDb, } from "./state-db.js";
|
|
30
30
|
import { borrowScopedStateDb } from "./state-db-scope.js";
|
|
31
31
|
import { isReadOnlyFilesystemError } from "./system-error.js";
|
|
32
32
|
import { error } from "./warn.js";
|
|
@@ -132,6 +132,9 @@ export function readEvents(options = {}, ctx) {
|
|
|
132
132
|
catch (err) {
|
|
133
133
|
// Never mask the bun-test isolation guard as "no events".
|
|
134
134
|
rethrowIfTestIsolationError(err);
|
|
135
|
+
// A damaged state.db is not "no events".
|
|
136
|
+
if (isStateDbCorruption(err))
|
|
137
|
+
throw classifyStateDbError(err, dbPath);
|
|
135
138
|
// DB does not exist yet or cannot be opened — return empty result.
|
|
136
139
|
return { events: [], nextOffset: 0 };
|
|
137
140
|
}
|
|
@@ -159,9 +162,12 @@ export function readEvents(options = {}, ctx) {
|
|
|
159
162
|
}
|
|
160
163
|
catch (error) {
|
|
161
164
|
if (ctx?.readOnly && error instanceof Error && /no such table:/i.test(error.message)) {
|
|
165
|
+
// An empty or behind-on-migrations state.db has no events; a non-empty file with no schema at all is zero-filled.
|
|
166
|
+
if (fs.statSync(dbPath).size > 0 && !db.prepare("SELECT 1 FROM sqlite_master LIMIT 1").get())
|
|
167
|
+
throw damagedStateDbError(dbPath, "no schema in a non-empty file");
|
|
162
168
|
return { events: [], nextOffset: options.sinceOffset ?? 0 };
|
|
163
169
|
}
|
|
164
|
-
throw error;
|
|
170
|
+
throw classifyStateDbError(error, dbPath);
|
|
165
171
|
}
|
|
166
172
|
const filtered = rawEvents.filter((envelope) => {
|
|
167
173
|
if (options.runId !== undefined && envelope.metadata?.runId !== options.runId)
|
package/dist/core/file-lock.js
CHANGED
|
@@ -8,7 +8,11 @@
|
|
|
8
8
|
* makes atomic: exactly one creator wins, so no further mutex or barrier is
|
|
9
9
|
* needed around acquisition. The payload records the holder's PID (and the
|
|
10
10
|
* launcher PID under a scheduler) so a probe can tell a live holder from a
|
|
11
|
-
* dead one; only a verifiably dead holder is ever reclaimed.
|
|
11
|
+
* dead one; only a verifiably dead holder is ever reclaimed. A live PID is not
|
|
12
|
+
* proof of a live holder (a container entrypoint is PID 1 on every start, PIDs
|
|
13
|
+
* are reused after a reboot), so where the OS exposes it the payload also
|
|
14
|
+
* records the holder's boot id and process start time; a live PID with a
|
|
15
|
+
* different identity is a reused PID and the lock is stale. Reclaim moves the
|
|
12
16
|
* stale file aside, re-verifies it is the very file that was probed, and only
|
|
13
17
|
* then unlinks it, so two reclaimers cannot both "win" the same stale lock.
|
|
14
18
|
*
|
|
@@ -20,6 +24,8 @@
|
|
|
20
24
|
import { randomUUID } from "node:crypto";
|
|
21
25
|
import fs from "node:fs";
|
|
22
26
|
import { isProcessAlive, MAX_LOCK_METADATA_BYTES, readTextFileDescriptorWithLimit } from "./common.js";
|
|
27
|
+
/** Locks this process acquired and has not released, by path, so a lock naming our own PID that we do not hold is known to be left over. */
|
|
28
|
+
const acquiredByThisProcess = new Map();
|
|
23
29
|
function readLockSnapshot(lockPath) {
|
|
24
30
|
let fd;
|
|
25
31
|
try {
|
|
@@ -128,8 +134,35 @@ export function tryAcquireLockSync(lockPath, payload) {
|
|
|
128
134
|
// Another process reclaimed our still-empty lock and took the path: we lost the race.
|
|
129
135
|
if (snapshot.rawContent !== payload)
|
|
130
136
|
return undefined;
|
|
137
|
+
acquiredByThisProcess.set(lockPath, payload);
|
|
131
138
|
return { lockPath, ...snapshot };
|
|
132
139
|
}
|
|
140
|
+
/**
|
|
141
|
+
* What a reused PID cannot fake: the kernel boot id and the process start time
|
|
142
|
+
* (`/proc/<pid>/stat` field 22, in ticks since boot). Linux only; `undefined`
|
|
143
|
+
* elsewhere or when `/proc` is unreadable, which leaves the PID as the only evidence.
|
|
144
|
+
*/
|
|
145
|
+
export function processIdentity(pid) {
|
|
146
|
+
try {
|
|
147
|
+
const bootId = fs.readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim();
|
|
148
|
+
const stat = fs.readFileSync(`/proc/${pid}/stat`, "utf8");
|
|
149
|
+
// The command name (field 2) may contain spaces and parens; fields resume after the last ")".
|
|
150
|
+
const startTime = stat.slice(stat.lastIndexOf(")") + 2).split(" ")[19];
|
|
151
|
+
return bootId && startTime ? { bootId, startTime } : undefined;
|
|
152
|
+
}
|
|
153
|
+
catch {
|
|
154
|
+
return undefined;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
/** True when the lock records an identity and the live holder PID now has a different one. */
|
|
158
|
+
function holderPidReused(holderPid, recorded) {
|
|
159
|
+
if (typeof recorded.bootId !== "string" || typeof recorded.startTime !== "string")
|
|
160
|
+
return false;
|
|
161
|
+
const current = processIdentity(holderPid);
|
|
162
|
+
if (!current)
|
|
163
|
+
return false;
|
|
164
|
+
return current.bootId !== recorded.bootId || current.startTime !== recorded.startTime;
|
|
165
|
+
}
|
|
133
166
|
export function launcherPidFromEnv() {
|
|
134
167
|
const raw = process.env.AKM_LAUNCHER_PID;
|
|
135
168
|
if (!raw)
|
|
@@ -142,6 +175,7 @@ export function createLockPayload(metadata = {}) {
|
|
|
142
175
|
return JSON.stringify({
|
|
143
176
|
...metadata,
|
|
144
177
|
pid: process.pid,
|
|
178
|
+
...processIdentity(process.pid),
|
|
145
179
|
...(launcherPid !== undefined ? { launcherPid } : {}),
|
|
146
180
|
lockId: randomUUID(),
|
|
147
181
|
});
|
|
@@ -168,6 +202,12 @@ export function probeLock(lockPath) {
|
|
|
168
202
|
if (!isProcessAlive(holderPid)) {
|
|
169
203
|
return { state: "stale", reason: "pid_dead", holderPid, launcherPid, ageMs, rawContent, identity };
|
|
170
204
|
}
|
|
205
|
+
// A lock naming this process that it did not acquire is left over from an earlier process with the same PID; so is
|
|
206
|
+
// one whose recorded identity no longer matches the live holder.
|
|
207
|
+
const ownLeftover = holderPid === process.pid && acquiredByThisProcess.get(lockPath) !== rawContent;
|
|
208
|
+
if (ownLeftover || holderPidReused(holderPid, parseLockJson(rawContent))) {
|
|
209
|
+
return { state: "stale", reason: "pid_reused", holderPid, launcherPid, ageMs, rawContent, identity };
|
|
210
|
+
}
|
|
171
211
|
return { state: "held", holderPid, launcherPid, ageMs, rawContent, identity };
|
|
172
212
|
}
|
|
173
213
|
/**
|
|
@@ -243,6 +283,8 @@ export function reclaimStaleLock(lockPath, probe, options) {
|
|
|
243
283
|
/** Unlink the lock only while it is still the exact file this owner created. */
|
|
244
284
|
export function releaseLock(ownership) {
|
|
245
285
|
const { lockPath } = ownership;
|
|
286
|
+
if (acquiredByThisProcess.get(lockPath) === ownership.rawContent)
|
|
287
|
+
acquiredByThisProcess.delete(lockPath);
|
|
246
288
|
let current;
|
|
247
289
|
try {
|
|
248
290
|
current = readLockSnapshot(lockPath);
|
|
@@ -254,6 +296,15 @@ export function releaseLock(ownership) {
|
|
|
254
296
|
releaseLockRaw(lockPath);
|
|
255
297
|
}
|
|
256
298
|
}
|
|
299
|
+
function parseLockJson(content) {
|
|
300
|
+
try {
|
|
301
|
+
const parsed = JSON.parse(content);
|
|
302
|
+
return typeof parsed === "object" && parsed !== null ? parsed : {};
|
|
303
|
+
}
|
|
304
|
+
catch {
|
|
305
|
+
return {};
|
|
306
|
+
}
|
|
307
|
+
}
|
|
257
308
|
function extractLockIdentity(content) {
|
|
258
309
|
const trimmed = content.trim();
|
|
259
310
|
if (!trimmed)
|
package/dist/core/state-db.js
CHANGED
|
@@ -64,6 +64,7 @@ import { openManagedDatabase, withManagedDb } from "../storage/managed-db.js";
|
|
|
64
64
|
import { assertMigrationLedger, runMigrations, } from "../storage/sqlite-migrations.js";
|
|
65
65
|
import { applyReadonlyPragmas } from "../storage/sqlite-pragmas.js";
|
|
66
66
|
import { pkgVersion } from "../version.js";
|
|
67
|
+
import { ConfigError } from "./errors.js";
|
|
67
68
|
import { getDataDir } from "./paths.js";
|
|
68
69
|
import { getStateMigrationSafety, STATE_MIGRATIONS } from "./state/migrations.js";
|
|
69
70
|
import { warnOnce } from "./warn.js";
|
|
@@ -130,16 +131,42 @@ function migrateStateDatabase(db, dbPath) {
|
|
|
130
131
|
db.exec("VACUUM");
|
|
131
132
|
return backupPath ? { applied, backupPath } : { applied };
|
|
132
133
|
}
|
|
134
|
+
/** SQLite refused the file as a database, or found its pages damaged or missing: garbage, zero-filled, truncated. */
|
|
135
|
+
export function isStateDbCorruption(error) {
|
|
136
|
+
const code = error?.code;
|
|
137
|
+
if (code === "SQLITE_NOTADB" || code === "SQLITE_CORRUPT" || code === "SQLITE_IOERR_SHORT_READ")
|
|
138
|
+
return true;
|
|
139
|
+
return /not a database|disk image is malformed|SQLITE_CORRUPT/i.test(error instanceof Error ? error.message : String(error));
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* The one error every state.db reader raises for a damaged file. state.db holds
|
|
143
|
+
* what akm cannot regenerate, so a damaged file is never "no rows": it is a
|
|
144
|
+
* configuration error (exit 78) that names the file. Other errors pass through.
|
|
145
|
+
*/
|
|
146
|
+
export function classifyStateDbError(error, dbPath) {
|
|
147
|
+
if (error instanceof ConfigError || !isStateDbCorruption(error))
|
|
148
|
+
return error;
|
|
149
|
+
return damagedStateDbError(dbPath, error instanceof Error ? error.message : String(error));
|
|
150
|
+
}
|
|
151
|
+
export function damagedStateDbError(dbPath, detail) {
|
|
152
|
+
return new ConfigError(`state.db at ${dbPath} is damaged (${detail}).`, "INVALID_CONFIG_FILE", "state.db holds non-regenerable state. Back it up, then recover it with " +
|
|
153
|
+
'`sqlite3 -readonly state.db ".recover" | sqlite3 state.new.db`, or restore a copy; if nothing can be recovered, move it aside and the next akm command creates a new one.');
|
|
154
|
+
}
|
|
133
155
|
function openAndMigrate(dbPath) {
|
|
134
156
|
let report = { applied: [] };
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
157
|
+
try {
|
|
158
|
+
const db = openManagedDatabase({
|
|
159
|
+
path: dbPath,
|
|
160
|
+
pragmas: { dataDir: path.dirname(dbPath) },
|
|
161
|
+
init: (handle) => {
|
|
162
|
+
report = migrateStateDatabase(handle, dbPath);
|
|
163
|
+
},
|
|
164
|
+
});
|
|
165
|
+
return { db, ...report };
|
|
166
|
+
}
|
|
167
|
+
catch (error) {
|
|
168
|
+
throw classifyStateDbError(error, dbPath);
|
|
169
|
+
}
|
|
143
170
|
}
|
|
144
171
|
/**
|
|
145
172
|
* Open (and initialise / migrate) the state database on one connection — see
|
|
@@ -181,15 +208,20 @@ export function listPendingStateMigrations(dbPath = getStateDbPath()) {
|
|
|
181
208
|
*/
|
|
182
209
|
export function withReadonlyStateDb(fn, fallback) {
|
|
183
210
|
const dbPath = getStateDbPath();
|
|
184
|
-
if (!fs.existsSync(dbPath) || listPendingStateMigrations(dbPath).length > 0)
|
|
185
|
-
return fallback;
|
|
186
|
-
const db = openDatabase(dbPath, { readonly: true, create: false });
|
|
187
211
|
try {
|
|
188
|
-
|
|
189
|
-
|
|
212
|
+
if (!fs.existsSync(dbPath) || listPendingStateMigrations(dbPath).length > 0)
|
|
213
|
+
return fallback;
|
|
214
|
+
const db = openDatabase(dbPath, { readonly: true, create: false });
|
|
215
|
+
try {
|
|
216
|
+
applyReadonlyPragmas(db);
|
|
217
|
+
return fn(db);
|
|
218
|
+
}
|
|
219
|
+
finally {
|
|
220
|
+
db.close();
|
|
221
|
+
}
|
|
190
222
|
}
|
|
191
|
-
|
|
192
|
-
|
|
223
|
+
catch (error) {
|
|
224
|
+
throw classifyStateDbError(error, dbPath);
|
|
193
225
|
}
|
|
194
226
|
}
|
|
195
227
|
/**
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/**
|
|
5
|
+
* Tells a broken pipe on our own stdout/stderr (the reader closed) from the same errno on any other
|
|
6
|
+
* stream. Only the first is ignorable; the rest stay failures.
|
|
7
|
+
*/
|
|
8
|
+
const isBrokenPipe = (err) => err?.code === "EPIPE";
|
|
9
|
+
export function trackStdioReaderGone(streams) {
|
|
10
|
+
let gone = false;
|
|
11
|
+
for (const stream of streams) {
|
|
12
|
+
stream.on("error", (err) => {
|
|
13
|
+
if (!isBrokenPipe(err))
|
|
14
|
+
throw err;
|
|
15
|
+
gone = true;
|
|
16
|
+
});
|
|
17
|
+
}
|
|
18
|
+
return { explains: (err) => gone && isBrokenPipe(err) };
|
|
19
|
+
}
|
|
@@ -36,7 +36,7 @@ import { normalizeAkmMarkdown } from "./asset/akm-markdown.js";
|
|
|
36
36
|
import { assetPathForName, stashDirFor } from "./asset/asset-placement.js";
|
|
37
37
|
import { conceptIdFromTypeName, displayRef } from "./asset/resolve-ref.js";
|
|
38
38
|
import { deriveBundleId } from "./bundle-id.js";
|
|
39
|
-
import { existingFileMode, isWithin, resolveStashDir, writeFileAtomic } from "./common.js";
|
|
39
|
+
import { existingFileMode, isWithin, relativeToRoot, resolveStashDir, writeFileAtomic } from "./common.js";
|
|
40
40
|
import { bundleKeyForContentRoot, resolveActiveConfiguredSources, resolveConfiguredSources } from "./config/config.js";
|
|
41
41
|
import { ConfigError, UsageError } from "./errors.js";
|
|
42
42
|
import { sanitizeCommitMessage } from "./git-message.js";
|
|
@@ -386,7 +386,7 @@ export function assertWritePathsInsideSource(source, filePaths) {
|
|
|
386
386
|
const lexicalRoot = path.resolve(source.path);
|
|
387
387
|
const canonicalRoot = fs.realpathSync(lexicalRoot);
|
|
388
388
|
for (const filePath of filePaths) {
|
|
389
|
-
const relative =
|
|
389
|
+
const relative = relativeToRoot(filePath, lexicalRoot) ?? "..";
|
|
390
390
|
if (!relative || relative === ".." || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
|
|
391
391
|
throw new UsageError(`Write path resolves outside source "${source.name}".`, "PATH_ESCAPE_VIOLATION");
|
|
392
392
|
}
|
|
@@ -54,10 +54,13 @@ export async function buildIndex(options, mode) {
|
|
|
54
54
|
/**
|
|
55
55
|
* Take the rebuild lock (waiting for a running build, as `"wait"`), then run `run` with a `build` that builds under
|
|
56
56
|
* that lock; release it when `run` settles. A build cannot start without the lock, and `build` must not be called
|
|
57
|
-
* after `run` returns.
|
|
57
|
+
* after `run` returns. With `"try"` a live holder throws `INDEX_DB_CONTENDED` at once instead of being waited for.
|
|
58
58
|
*/
|
|
59
|
-
export async function withIndexBuildLock(signal, run) {
|
|
60
|
-
const lock = await acquireBuildLock(
|
|
59
|
+
export async function withIndexBuildLock(signal, run, mode = "wait") {
|
|
60
|
+
const lock = await acquireBuildLock(mode, signal);
|
|
61
|
+
if ("skipped" in lock) {
|
|
62
|
+
throw new TransientError(`Another akm process is building the index (${describeLockHolder(lock.skipped)}); not building.`, "INDEX_DB_CONTENDED");
|
|
63
|
+
}
|
|
61
64
|
try {
|
|
62
65
|
return await run(buildUnderLock);
|
|
63
66
|
}
|
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
* empty result and {@link noIndexWarning}; one whose build has not completed
|
|
19
19
|
* gets {@link incompleteIndexWarning} alongside its results.
|
|
20
20
|
*/
|
|
21
|
+
import { safeRealpath } from "../core/common.js";
|
|
21
22
|
import { ConfigError } from "../core/errors.js";
|
|
22
23
|
import { classifyPathAccess } from "../core/path-access.js";
|
|
23
24
|
import { getDbPath } from "../core/paths.js";
|
|
@@ -40,8 +41,14 @@ export function incompleteIndexWarning() {
|
|
|
40
41
|
: "The last index build for this library did not complete (it was interrupted or a source could not be scanned); " +
|
|
41
42
|
"results may be incomplete. Run `akm index`.";
|
|
42
43
|
}
|
|
44
|
+
/** Whether the index's `stashDir` / `stashDirs` meta names `library` (a realpath), however the stored value is spelled. */
|
|
45
|
+
function namesLibrary(db, library) {
|
|
46
|
+
const stored = [getMeta(db, "stashDir"), ...JSON.parse(getMeta(db, "stashDirs") ?? "[]")];
|
|
47
|
+
return stored.some((dir) => dir !== undefined && safeRealpath(dir) === library);
|
|
48
|
+
}
|
|
43
49
|
/**
|
|
44
|
-
* Open the index when it can serve `stashDir
|
|
50
|
+
* Open the index when it can serve `stashDir` (any library when omitted: the
|
|
51
|
+
* workspace reads every bundle's entries), or return `undefined`: no file,
|
|
45
52
|
* no schema, nothing persisted for this library, built for another library, or
|
|
46
53
|
* a file that cannot be read as an index (corrupt, mid-write). Never creates,
|
|
47
54
|
* migrates, deletes or waits. The caller closes `db`.
|
|
@@ -62,9 +69,7 @@ export function openUsableIndex(stashDir, options = {}) {
|
|
|
62
69
|
return undefined;
|
|
63
70
|
if (options.busyTimeoutMs !== undefined)
|
|
64
71
|
db.exec(`PRAGMA busy_timeout = ${Math.max(0, Math.floor(options.busyTimeoutMs))}`);
|
|
65
|
-
const servesLibrary = hasCurrentEntriesTable(db) &&
|
|
66
|
-
(getMeta(db, "stashDir") === stashDir ||
|
|
67
|
-
JSON.parse(getMeta(db, "stashDirs") ?? "[]").includes(stashDir));
|
|
72
|
+
const servesLibrary = hasCurrentEntriesTable(db) && (stashDir === undefined || namesLibrary(db, safeRealpath(stashDir)));
|
|
68
73
|
if (servesLibrary)
|
|
69
74
|
return { db, complete: Boolean(getMeta(db, "builtAt")) };
|
|
70
75
|
}
|
|
@@ -80,3 +85,24 @@ export function openUsableIndex(stashDir, options = {}) {
|
|
|
80
85
|
closeDatabase(db);
|
|
81
86
|
return undefined;
|
|
82
87
|
}
|
|
88
|
+
/**
|
|
89
|
+
* Run `read` against the usable index, or return `fallback` when there is none
|
|
90
|
+
* or a query fails on a damaged one: an index that cannot answer is no index.
|
|
91
|
+
* Permission and newer-layout errors (`ConfigError`) still raise.
|
|
92
|
+
*/
|
|
93
|
+
export function readUsableIndex(read, fallback) {
|
|
94
|
+
const usable = openUsableIndex(undefined, { readOnly: true });
|
|
95
|
+
if (!usable)
|
|
96
|
+
return fallback;
|
|
97
|
+
try {
|
|
98
|
+
return read(usable.db);
|
|
99
|
+
}
|
|
100
|
+
catch (error) {
|
|
101
|
+
if (error instanceof ConfigError)
|
|
102
|
+
throw error;
|
|
103
|
+
return fallback;
|
|
104
|
+
}
|
|
105
|
+
finally {
|
|
106
|
+
closeDatabase(usable.db);
|
|
107
|
+
}
|
|
108
|
+
}
|