akm-cli 0.6.0-rc1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/README.md +9 -9
- package/dist/cli.js +199 -114
- package/dist/{completions.js → commands/completions.js} +1 -1
- package/dist/{config-cli.js → commands/config-cli.js} +109 -11
- package/dist/{curate.js → commands/curate.js} +8 -3
- package/dist/{info.js → commands/info.js} +15 -9
- package/dist/{init.js → commands/init.js} +4 -4
- package/dist/{install-audit.js → commands/install-audit.js} +4 -7
- package/dist/{installed-stashes.js → commands/installed-stashes.js} +77 -31
- package/dist/{migration-help.js → commands/migration-help.js} +2 -2
- package/dist/{registry-search.js → commands/registry-search.js} +8 -6
- package/dist/{remember.js → commands/remember.js} +55 -49
- package/dist/{stash-search.js → commands/search.js} +28 -69
- package/dist/{self-update.js → commands/self-update.js} +69 -3
- package/dist/{stash-show.js → commands/show.js} +104 -84
- package/dist/{stash-add.js → commands/source-add.js} +42 -32
- package/dist/{stash-clone.js → commands/source-clone.js} +12 -10
- package/dist/{stash-source-manage.js → commands/source-manage.js} +24 -24
- package/dist/{vault.js → commands/vault.js} +43 -0
- package/dist/{stash-ref.js → core/asset-ref.js} +4 -4
- package/dist/{asset-registry.js → core/asset-registry.js} +1 -1
- package/dist/{asset-spec.js → core/asset-spec.js} +1 -1
- package/dist/{config.js → core/config.js} +133 -56
- package/dist/core/errors.js +90 -0
- package/dist/{frontmatter.js → core/frontmatter.js} +5 -3
- package/dist/core/write-source.js +280 -0
- package/dist/{db-search.js → indexer/db-search.js} +25 -19
- package/dist/{db.js → indexer/db.js} +79 -47
- package/dist/{file-context.js → indexer/file-context.js} +3 -3
- package/dist/{indexer.js → indexer/indexer.js} +132 -33
- package/dist/{manifest.js → indexer/manifest.js} +10 -10
- package/dist/{matchers.js → indexer/matchers.js} +3 -6
- package/dist/{metadata.js → indexer/metadata.js} +9 -5
- package/dist/{search-source.js → indexer/search-source.js} +52 -41
- package/dist/{semantic-status.js → indexer/semantic-status.js} +2 -2
- package/dist/{walker.js → indexer/walker.js} +1 -1
- package/dist/{lockfile.js → integrations/lockfile.js} +1 -1
- package/dist/{llm-client.js → llm/client.js} +1 -1
- package/dist/{embedders → llm/embedders}/local.js +2 -2
- package/dist/{embedders → llm/embedders}/remote.js +1 -1
- package/dist/{embedders → llm/embedders}/types.js +1 -1
- package/dist/{metadata-enhance.js → llm/metadata-enhance.js} +2 -2
- package/dist/{cli-hints.js → output/cli-hints.js} +3 -0
- package/dist/{output-context.js → output/context.js} +21 -3
- package/dist/{renderers.js → output/renderers.js} +9 -65
- package/dist/{output-shapes.js → output/shapes.js} +18 -4
- package/dist/{output-text.js → output/text.js} +2 -2
- package/dist/{registry-build-index.js → registry/build-index.js} +16 -7
- package/dist/{create-provider-registry.js → registry/create-provider-registry.js} +6 -2
- package/dist/registry/factory.js +33 -0
- package/dist/{origin-resolve.js → registry/origin-resolve.js} +1 -1
- package/dist/{providers → registry/providers}/index.js +1 -1
- package/dist/{providers → registry/providers}/skills-sh.js +59 -3
- package/dist/{providers → registry/providers}/static-index.js +80 -12
- package/dist/registry/providers/types.js +25 -0
- package/dist/{registry-resolve.js → registry/resolve.js} +3 -3
- package/dist/{detect.js → setup/detect.js} +0 -27
- package/dist/{ripgrep-install.js → setup/ripgrep-install.js} +1 -1
- package/dist/{ripgrep-resolve.js → setup/ripgrep-resolve.js} +2 -2
- package/dist/{setup.js → setup/setup.js} +16 -56
- package/dist/{stash-include.js → sources/include.js} +1 -1
- package/dist/sources/provider-factory.js +36 -0
- package/dist/sources/provider.js +21 -0
- package/dist/sources/providers/filesystem.js +35 -0
- package/dist/{stash-providers → sources/providers}/git.js +53 -64
- package/dist/{stash-providers → sources/providers}/index.js +3 -4
- package/dist/sources/providers/install-types.js +14 -0
- package/dist/{stash-providers → sources/providers}/npm.js +42 -41
- package/dist/{stash-providers → sources/providers}/provider-utils.js +3 -3
- package/dist/{stash-providers → sources/providers}/sync-from-ref.js +2 -2
- package/dist/{stash-providers → sources/providers}/tar-utils.js +11 -8
- package/dist/{stash-providers → sources/providers}/website.js +29 -65
- package/dist/{stash-resolve.js → sources/resolve.js} +8 -7
- package/dist/{wiki.js → wiki/wiki.js} +34 -18
- package/dist/{workflow-authoring.js → workflows/authoring.js} +37 -14
- package/dist/{workflow-cli.js → workflows/cli.js} +2 -1
- package/dist/{workflow-db.js → workflows/db.js} +1 -1
- package/dist/workflows/document-cache.js +20 -0
- package/dist/workflows/parser.js +379 -0
- package/dist/workflows/renderer.js +78 -0
- package/dist/{workflow-runs.js → workflows/runs.js} +72 -28
- package/dist/workflows/schema.js +11 -0
- package/dist/workflows/validator.js +48 -0
- package/docs/migration/release-notes/0.6.0.md +91 -23
- package/package.json +1 -1
- package/dist/errors.js +0 -45
- package/dist/llm.js +0 -16
- package/dist/registry-factory.js +0 -19
- package/dist/ripgrep.js +0 -2
- package/dist/stash-provider-factory.js +0 -35
- package/dist/stash-provider.js +0 -3
- package/dist/stash-providers/filesystem.js +0 -71
- package/dist/stash-providers/openviking.js +0 -348
- package/dist/stash-types.js +0 -1
- package/dist/workflow-markdown.js +0 -260
- /package/dist/{common.js → core/common.js} +0 -0
- /package/dist/{markdown.js → core/markdown.js} +0 -0
- /package/dist/{paths.js → core/paths.js} +0 -0
- /package/dist/{warn.js → core/warn.js} +0 -0
- /package/dist/{search-fields.js → indexer/search-fields.js} +0 -0
- /package/dist/{usage-events.js → indexer/usage-events.js} +0 -0
- /package/dist/{github.js → integrations/github.js} +0 -0
- /package/dist/{embedder.js → llm/embedder.js} +0 -0
- /package/dist/{embedders → llm/embedders}/cache.js +0 -0
- /package/dist/{registry-provider.js → registry/types.js} +0 -0
- /package/dist/{setup-steps.js → setup/steps.js} +0 -0
- /package/dist/{registry-types.js → sources/types.js} +0 -0
|
@@ -1,27 +1,28 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
|
-
import { isHttpUrl, resolveStashDir } from "
|
|
3
|
+
import { isHttpUrl, resolveStashDir, toErrorMessage } from "../core/common";
|
|
4
|
+
import { getDbPath } from "../core/paths";
|
|
5
|
+
import { warn } from "../core/warn";
|
|
6
|
+
import { takeWorkflowDocument } from "../workflows/document-cache";
|
|
4
7
|
import { closeDatabase, deleteEntriesByDir, deleteEntriesByStashDir, getEmbeddingCount, getEntriesByDir, getEntryCount, getMeta, isVecAvailable, openDatabase, rebuildFts, setMeta, upsertEmbedding, upsertEntry, upsertUtilityScore, warnIfVecMissing, } from "./db";
|
|
5
8
|
import { generateMetadataFlat, loadStashFile, shouldIndexStashFile } from "./metadata";
|
|
6
|
-
import { getDbPath } from "./paths";
|
|
7
9
|
import { buildSearchText } from "./search-fields";
|
|
8
10
|
import { classifySemanticFailure, clearSemanticStatus, deriveSemanticProviderFingerprint, writeSemanticStatus, } from "./semantic-status";
|
|
9
11
|
import { ensureUsageEventsSchema, purgeOldUsageEvents } from "./usage-events";
|
|
10
12
|
import { walkStashFlat } from "./walker";
|
|
11
|
-
import { warn } from "./warn";
|
|
12
13
|
// ── Indexer ──────────────────────────────────────────────────────────────────
|
|
13
14
|
export async function akmIndex(options) {
|
|
14
15
|
const stashDir = options?.stashDir || resolveStashDir();
|
|
15
16
|
const onProgress = options?.onProgress ?? (() => { });
|
|
16
17
|
// Load config and resolve all stash sources
|
|
17
|
-
const { loadConfig } = await import("
|
|
18
|
+
const { loadConfig } = await import("../core/config.js");
|
|
18
19
|
const config = loadConfig();
|
|
19
20
|
// Ensure git stash caches are extracted before resolving stash dirs,
|
|
20
21
|
// so their content directories exist on disk for the walker to discover.
|
|
21
|
-
const {
|
|
22
|
-
await
|
|
23
|
-
const
|
|
24
|
-
const
|
|
22
|
+
const { ensureSourceCaches, resolveSourceEntries } = await import("./search-source.js");
|
|
23
|
+
await ensureSourceCaches(config);
|
|
24
|
+
const allSourceEntries = resolveSourceEntries(stashDir, config);
|
|
25
|
+
const allSourceDirs = allSourceEntries.map((s) => s.path);
|
|
25
26
|
const t0 = Date.now();
|
|
26
27
|
// Open database — pass embedding dimension from config if available
|
|
27
28
|
const dbPath = getDbPath();
|
|
@@ -37,7 +38,7 @@ export async function akmIndex(options) {
|
|
|
37
38
|
phase: "summary",
|
|
38
39
|
message: buildIndexSummaryMessage({
|
|
39
40
|
mode: isIncremental ? "incremental" : "full",
|
|
40
|
-
|
|
41
|
+
sourcesCount: allSourceDirs.length,
|
|
41
42
|
semanticSearchMode: config.semanticSearchMode,
|
|
42
43
|
embeddingProvider: getEmbeddingProvider(config.embedding),
|
|
43
44
|
llmEnabled: !!config.llm,
|
|
@@ -67,7 +68,7 @@ export async function akmIndex(options) {
|
|
|
67
68
|
catch {
|
|
68
69
|
warn("index_meta stashDirs value is corrupt JSON — treating as empty");
|
|
69
70
|
}
|
|
70
|
-
const currentSet = new Set(
|
|
71
|
+
const currentSet = new Set(allSourceDirs);
|
|
71
72
|
for (const dir of prevStashDirs) {
|
|
72
73
|
if (!currentSet.has(dir)) {
|
|
73
74
|
deleteEntriesByStashDir(db, dir);
|
|
@@ -80,7 +81,7 @@ export async function akmIndex(options) {
|
|
|
80
81
|
// doFullDelete=true merges the wipe into the same transaction as the
|
|
81
82
|
// inserts so readers never see an empty database mid-rebuild.
|
|
82
83
|
const doFullDelete = options?.full || !isIncremental;
|
|
83
|
-
const { scannedDirs, skippedDirs, generatedCount, dirsNeedingLlm, warnings } = await indexEntries(db,
|
|
84
|
+
const { scannedDirs, skippedDirs, generatedCount, dirsNeedingLlm, warnings } = await indexEntries(db, allSourceEntries, isIncremental, builtAtMs, doFullDelete);
|
|
84
85
|
onProgress({
|
|
85
86
|
phase: "scan",
|
|
86
87
|
message: `Scanned ${scannedDirs} ${scannedDirs === 1 ? "directory" : "directories"} and skipped ${skippedDirs}.`,
|
|
@@ -130,7 +131,7 @@ export async function akmIndex(options) {
|
|
|
130
131
|
// are read-only caches, and regenerating their indexes would mutate
|
|
131
132
|
// cache content.
|
|
132
133
|
try {
|
|
133
|
-
const { regenerateAllWikiIndexes } = await import("
|
|
134
|
+
const { regenerateAllWikiIndexes } = await import("../wiki/wiki.js");
|
|
134
135
|
regenerateAllWikiIndexes(stashDir);
|
|
135
136
|
}
|
|
136
137
|
catch {
|
|
@@ -142,7 +143,7 @@ export async function akmIndex(options) {
|
|
|
142
143
|
// Update metadata
|
|
143
144
|
setMeta(db, "builtAt", new Date().toISOString());
|
|
144
145
|
setMeta(db, "stashDir", stashDir);
|
|
145
|
-
setMeta(db, "stashDirs", JSON.stringify(
|
|
146
|
+
setMeta(db, "stashDirs", JSON.stringify(allSourceDirs));
|
|
146
147
|
setMeta(db, "hasEmbeddings", embeddingResult.success ? "1" : "0");
|
|
147
148
|
const totalEntries = getEntryCount(db);
|
|
148
149
|
// Warn on every index run if using JS fallback with many entries
|
|
@@ -188,7 +189,7 @@ export async function akmIndex(options) {
|
|
|
188
189
|
}
|
|
189
190
|
}
|
|
190
191
|
// ── Extracted helpers for indexing ────────────────────────────────────────────
|
|
191
|
-
async function indexEntries(db,
|
|
192
|
+
async function indexEntries(db, allSourceEntries, isIncremental, builtAtMs, doFullDelete = false) {
|
|
192
193
|
let scannedDirs = 0;
|
|
193
194
|
let skippedDirs = 0;
|
|
194
195
|
let generatedCount = 0;
|
|
@@ -196,12 +197,12 @@ async function indexEntries(db, allStashSources, isIncremental, builtAtMs, doFul
|
|
|
196
197
|
const seenPaths = new Set();
|
|
197
198
|
const dirsNeedingLlm = [];
|
|
198
199
|
const dirRecords = [];
|
|
199
|
-
for (const
|
|
200
|
-
const currentStashDir =
|
|
200
|
+
for (const sourceAdded of allSourceEntries) {
|
|
201
|
+
const currentStashDir = sourceAdded.path;
|
|
201
202
|
const fileContexts = walkStashFlat(currentStashDir);
|
|
202
203
|
// Wiki-root stashes: all .md files are indexed as wiki pages under wikiName
|
|
203
|
-
if (
|
|
204
|
-
const wikiName =
|
|
204
|
+
if (sourceAdded.wikiName) {
|
|
205
|
+
const wikiName = sourceAdded.wikiName;
|
|
205
206
|
const wikiDirGroups = new Map();
|
|
206
207
|
for (const ctx of fileContexts) {
|
|
207
208
|
if (ctx.ext !== ".md")
|
|
@@ -353,7 +354,13 @@ async function indexEntries(db, allStashSources, isIncremental, builtAtMs, doFul
|
|
|
353
354
|
const entryKey = `${currentStashDir}:${entry.type}:${entry.name}`;
|
|
354
355
|
const searchText = buildSearchText(entry);
|
|
355
356
|
const entryWithSize = attachFileSize(entry, entryPath);
|
|
356
|
-
upsertEntry(db, entryKey, dirPath, entryPath, currentStashDir, entryWithSize, searchText);
|
|
357
|
+
const entryId = upsertEntry(db, entryKey, dirPath, entryPath, currentStashDir, entryWithSize, searchText);
|
|
358
|
+
if (entry.type === "workflow") {
|
|
359
|
+
const doc = takeWorkflowDocument(entry);
|
|
360
|
+
if (doc) {
|
|
361
|
+
upsertWorkflowDocument(db, entryId, doc, fs.readFileSync(entryPath));
|
|
362
|
+
}
|
|
363
|
+
}
|
|
357
364
|
}
|
|
358
365
|
// Collect dirs needing LLM enhancement during the first walk
|
|
359
366
|
if (stash.entries.some((e) => e.quality === "generated")) {
|
|
@@ -428,7 +435,7 @@ async function generateEmbeddingsForDb(db, config, onProgress) {
|
|
|
428
435
|
setMeta(db, "hasEmbeddings", "0");
|
|
429
436
|
}
|
|
430
437
|
try {
|
|
431
|
-
const { embedBatch } = await import("
|
|
438
|
+
const { embedBatch } = await import("../llm/embedder.js");
|
|
432
439
|
const allEntries = getAllEntriesForEmbedding(db);
|
|
433
440
|
if (allEntries.length === 0) {
|
|
434
441
|
onProgress({ phase: "embeddings", message: "Embeddings already up to date." });
|
|
@@ -486,10 +493,32 @@ function attachFileSize(entry, entryPath) {
|
|
|
486
493
|
return entry;
|
|
487
494
|
}
|
|
488
495
|
}
|
|
496
|
+
function upsertWorkflowDocument(db, entryId, doc, content) {
|
|
497
|
+
const sourceHash = computeSourceHash(content);
|
|
498
|
+
db.prepare(`INSERT INTO workflow_documents (entry_id, schema_version, document_json, source_path, source_hash, updated_at)
|
|
499
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
500
|
+
ON CONFLICT(entry_id) DO UPDATE SET
|
|
501
|
+
schema_version = excluded.schema_version,
|
|
502
|
+
document_json = excluded.document_json,
|
|
503
|
+
source_path = excluded.source_path,
|
|
504
|
+
source_hash = excluded.source_hash,
|
|
505
|
+
updated_at = excluded.updated_at`).run(entryId, doc.schemaVersion, JSON.stringify(doc), doc.source.path, sourceHash, new Date().toISOString());
|
|
506
|
+
}
|
|
507
|
+
function computeSourceHash(content) {
|
|
508
|
+
// Cheap, stable identity for the source markdown — used by future
|
|
509
|
+
// incremental fast-paths that skip re-validation when content is unchanged.
|
|
510
|
+
// Not security-sensitive; FNV-1a over the bytes is sufficient.
|
|
511
|
+
let hash = 0x811c9dc5;
|
|
512
|
+
for (let i = 0; i < content.length; i++) {
|
|
513
|
+
hash ^= content[i];
|
|
514
|
+
hash = Math.imul(hash, 0x01000193);
|
|
515
|
+
}
|
|
516
|
+
return (hash >>> 0).toString(16);
|
|
517
|
+
}
|
|
489
518
|
function buildIndexSummaryMessage(options) {
|
|
490
|
-
const stashSourceLabel = options.
|
|
519
|
+
const stashSourceLabel = options.sourcesCount === 1 ? "stash source" : "stash sources";
|
|
491
520
|
const semanticDetail = getSemanticSearchLabel(options.semanticSearchMode, options.embeddingProvider, options.vecAvailable);
|
|
492
|
-
return `Starting ${options.mode} index (${options.
|
|
521
|
+
return `Starting ${options.mode} index (${options.sourcesCount} ${stashSourceLabel}, semantic search: ${semanticDetail}, LLM: ${options.llmEnabled ? "enabled" : "disabled"}).`;
|
|
493
522
|
}
|
|
494
523
|
function getEmbeddingProvider(embedding) {
|
|
495
524
|
return isHttpUrl(embedding?.endpoint) ? "remote" : "local";
|
|
@@ -590,9 +619,8 @@ function isDirStale(dirPath, currentFiles, previousEntries, builtAtMs) {
|
|
|
590
619
|
return false;
|
|
591
620
|
}
|
|
592
621
|
async function enhanceStashWithLlm(llmConfig, stash, files, summary) {
|
|
593
|
-
const { enhanceMetadata } = await import("
|
|
622
|
+
const { enhanceMetadata } = await import("../llm/metadata-enhance");
|
|
594
623
|
const enhanced = [];
|
|
595
|
-
const seenSamples = new Set();
|
|
596
624
|
for (const entry of stash.entries) {
|
|
597
625
|
summary.attempted++;
|
|
598
626
|
try {
|
|
@@ -621,10 +649,11 @@ async function enhanceStashWithLlm(llmConfig, stash, files, summary) {
|
|
|
621
649
|
}
|
|
622
650
|
catch (err) {
|
|
623
651
|
enhanced.push(entry);
|
|
624
|
-
const msg =
|
|
625
|
-
|
|
652
|
+
const msg = toErrorMessage(err);
|
|
653
|
+
// failureSamples is bounded to 3 items, so a linear scan is cheaper
|
|
654
|
+
// than maintaining a parallel Set for membership checks (#177 review).
|
|
655
|
+
if (summary.failureSamples.length < 3 && !summary.failureSamples.includes(msg)) {
|
|
626
656
|
summary.failureSamples.push(msg);
|
|
627
|
-
seenSamples.add(msg);
|
|
628
657
|
}
|
|
629
658
|
}
|
|
630
659
|
}
|
|
@@ -667,10 +696,73 @@ export function matchEntryToFile(entryName, fileMap, files) {
|
|
|
667
696
|
// Fallback to first file, or null if no files are available
|
|
668
697
|
return files[0] || null;
|
|
669
698
|
}
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
699
|
+
/**
|
|
700
|
+
* Look up a single asset by ref. Spec §6.2 — `akm show` queries this and
|
|
701
|
+
* reads the file from disk. The index is the source of truth for which
|
|
702
|
+
* file corresponds to which ref; the indexer walks `provider.path()` for
|
|
703
|
+
* every configured source, so this query covers all source kinds.
|
|
704
|
+
*
|
|
705
|
+
* Match rules:
|
|
706
|
+
* - `ref.origin === undefined` → first match across all sources (primary
|
|
707
|
+
* source first, then in declared order — same priority as the indexer's
|
|
708
|
+
* write order).
|
|
709
|
+
* - `ref.origin === "local"` → primary source only (entry_key prefix is
|
|
710
|
+
* the primary stash dir).
|
|
711
|
+
* - `ref.origin === <name>` → restrict to the matching source name. We
|
|
712
|
+
* resolve the source's directory and match on `entry_key` prefix.
|
|
713
|
+
*
|
|
714
|
+
* Returns `null` when no row matches — callers translate that into a
|
|
715
|
+
* `NotFoundError` with their own messaging.
|
|
716
|
+
*/
|
|
717
|
+
export async function lookup(ref) {
|
|
718
|
+
const { loadConfig } = await import("../core/config.js");
|
|
719
|
+
const { resolveSourceEntries } = await import("./search-source.js");
|
|
720
|
+
const config = loadConfig();
|
|
721
|
+
const sources = resolveSourceEntries(undefined, config);
|
|
722
|
+
if (sources.length === 0)
|
|
723
|
+
return null;
|
|
724
|
+
const dbPath = getDbPath();
|
|
725
|
+
const db = openDatabase(dbPath);
|
|
726
|
+
try {
|
|
727
|
+
// entry_key shape: `${stashDir}:${type}:${name}`. Suffix-match on
|
|
728
|
+
// `:type:name` so we can scope by source dir as a prefix when origin is
|
|
729
|
+
// supplied. Use parameterised queries throughout — names may include
|
|
730
|
+
// user-supplied glob characters.
|
|
731
|
+
const escapeLike = (value) => value.replace(/\\/g, "\\\\").replace(/%/g, "\\%").replace(/_/g, "\\_");
|
|
732
|
+
const suffix = `:${ref.type}:${ref.name}`;
|
|
733
|
+
const escapedSuffix = escapeLike(suffix);
|
|
734
|
+
const candidateDirs = (() => {
|
|
735
|
+
if (!ref.origin)
|
|
736
|
+
return sources.map((s) => s.path);
|
|
737
|
+
if (ref.origin === "local")
|
|
738
|
+
return [sources[0].path];
|
|
739
|
+
const named = sources.find((s) => s.registryId === ref.origin);
|
|
740
|
+
return named ? [named.path] : [];
|
|
741
|
+
})();
|
|
742
|
+
if (candidateDirs.length === 0)
|
|
743
|
+
return null;
|
|
744
|
+
for (const dir of candidateDirs) {
|
|
745
|
+
const escapedDir = escapeLike(dir);
|
|
746
|
+
const row = db
|
|
747
|
+
.prepare("SELECT entry_key AS entryKey, file_path AS filePath, stash_dir AS stashDir, entry_type AS type FROM entries " +
|
|
748
|
+
"WHERE entry_key LIKE ? ESCAPE '\\' AND entry_type = ? LIMIT 1")
|
|
749
|
+
.get(`${escapedDir}${escapedSuffix}`, ref.type);
|
|
750
|
+
if (row) {
|
|
751
|
+
return {
|
|
752
|
+
entryKey: row.entryKey,
|
|
753
|
+
filePath: row.filePath,
|
|
754
|
+
stashDir: row.stashDir,
|
|
755
|
+
type: row.type,
|
|
756
|
+
name: ref.name,
|
|
757
|
+
};
|
|
758
|
+
}
|
|
759
|
+
}
|
|
760
|
+
return null;
|
|
761
|
+
}
|
|
762
|
+
finally {
|
|
763
|
+
closeDatabase(db);
|
|
764
|
+
}
|
|
765
|
+
}
|
|
674
766
|
// ── Utility score recomputation ──────────────────────────────────────────────
|
|
675
767
|
/** Retention window for usage events: events older than this are purged. */
|
|
676
768
|
const USAGE_EVENT_RETENTION_DAYS = 90;
|
|
@@ -680,8 +772,10 @@ const USAGE_EVENT_RETENTION_DAYS = 90;
|
|
|
680
772
|
* For each entry:
|
|
681
773
|
* - Count search appearances (event_type = 'search')
|
|
682
774
|
* - Count show events (event_type = 'show')
|
|
775
|
+
* - Count positive/negative feedback events
|
|
683
776
|
* - Compute select_rate = showCount / searchCount, clamped to [0, 1]
|
|
684
|
-
* -
|
|
777
|
+
* - Convert feedback counts into a positive-only feedback_rate
|
|
778
|
+
* - Update utility via EMA from the stronger of select_rate / feedback_rate
|
|
685
779
|
*
|
|
686
780
|
* Also purges usage_events older than 90 days and ensures the M-1
|
|
687
781
|
* usage_events table exists before querying.
|
|
@@ -711,6 +805,8 @@ export function recomputeUtilityScores(db) {
|
|
|
711
805
|
SELECT entry_id,
|
|
712
806
|
SUM(CASE WHEN event_type = 'search' THEN 1 ELSE 0 END) AS search_count,
|
|
713
807
|
SUM(CASE WHEN event_type = 'show' THEN 1 ELSE 0 END) AS show_count,
|
|
808
|
+
SUM(CASE WHEN event_type = 'feedback' AND signal = 'positive' THEN 1 ELSE 0 END) AS positive_feedback_count,
|
|
809
|
+
SUM(CASE WHEN event_type = 'feedback' AND signal = 'negative' THEN 1 ELSE 0 END) AS negative_feedback_count,
|
|
714
810
|
MAX(created_at) AS last_used_at
|
|
715
811
|
FROM usage_events
|
|
716
812
|
WHERE entry_id IS NOT NULL
|
|
@@ -729,8 +825,11 @@ export function recomputeUtilityScores(db) {
|
|
|
729
825
|
}
|
|
730
826
|
for (const row of usageRows) {
|
|
731
827
|
const selectRate = row.search_count > 0 ? Math.min(1, row.show_count / row.search_count) : 0;
|
|
828
|
+
const feedbackTotal = row.positive_feedback_count + row.negative_feedback_count;
|
|
829
|
+
const feedbackRate = feedbackTotal > 0 ? Math.max(0, row.positive_feedback_count - row.negative_feedback_count) / feedbackTotal : 0;
|
|
830
|
+
const effectiveRate = Math.max(selectRate, feedbackRate);
|
|
732
831
|
const prevUtility = existingScores.get(row.entry_id) ?? 0;
|
|
733
|
-
const utility = prevUtility * emaDecay +
|
|
832
|
+
const utility = prevUtility * emaDecay + effectiveRate * emaNew;
|
|
734
833
|
upsertUtilityScore(db, row.entry_id, {
|
|
735
834
|
utility,
|
|
736
835
|
showCount: row.show_count,
|
|
@@ -8,16 +8,16 @@
|
|
|
8
8
|
*/
|
|
9
9
|
import fs from "node:fs";
|
|
10
10
|
import path from "node:path";
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
11
|
+
import { makeAssetRef } from "../core/asset-ref";
|
|
12
|
+
import { deriveCanonicalAssetNameFromStashRoot } from "../core/asset-spec";
|
|
13
|
+
import { resolveStashDir } from "../core/common";
|
|
14
|
+
import { loadConfig } from "../core/config";
|
|
15
|
+
import { getDbPath } from "../core/paths";
|
|
16
|
+
import { warn } from "../core/warn";
|
|
14
17
|
import { closeDatabase, getAllEntries, getEntryCount, getMeta, openDatabase } from "./db";
|
|
15
18
|
import { generateMetadataFlat, loadStashFile } from "./metadata";
|
|
16
|
-
import {
|
|
17
|
-
import { resolveStashSources } from "./search-source";
|
|
18
|
-
import { makeAssetRef } from "./stash-ref";
|
|
19
|
+
import { resolveSourceEntries } from "./search-source";
|
|
19
20
|
import { walkStashFlat } from "./walker";
|
|
20
|
-
import { warn } from "./warn";
|
|
21
21
|
const MAX_DESCRIPTION_LENGTH = 80;
|
|
22
22
|
/**
|
|
23
23
|
* Truncate a description string to a maximum length, appending "..." if truncated.
|
|
@@ -99,9 +99,9 @@ function getManifestFromDb(stashDir, config, sources, type) {
|
|
|
99
99
|
* Get the manifest by walking the stash directory (fallback when no index).
|
|
100
100
|
*/
|
|
101
101
|
async function getManifestFromWalker(sources, type) {
|
|
102
|
-
const
|
|
102
|
+
const allSourceDirs = sources.map((s) => s.path);
|
|
103
103
|
const entries = [];
|
|
104
|
-
for (const currentStashDir of
|
|
104
|
+
for (const currentStashDir of allSourceDirs) {
|
|
105
105
|
const fileContexts = walkStashFlat(currentStashDir);
|
|
106
106
|
// Group by parent directory
|
|
107
107
|
const dirGroups = new Map();
|
|
@@ -154,7 +154,7 @@ export async function akmManifest(options) {
|
|
|
154
154
|
const stashDir = options?.stashDir ?? resolveStashDir();
|
|
155
155
|
const type = options?.type;
|
|
156
156
|
const config = loadConfig();
|
|
157
|
-
const sources =
|
|
157
|
+
const sources = resolveSourceEntries(stashDir, config);
|
|
158
158
|
// Fast path: try database
|
|
159
159
|
const dbEntries = getManifestFromDb(stashDir, config, sources, type);
|
|
160
160
|
if (dbEntries !== null) {
|
|
@@ -18,7 +18,8 @@
|
|
|
18
18
|
* - `wikiMatcher` (20) -- classifies any `.md` under `wikis/<name>/…` as
|
|
19
19
|
* `wiki`. Registered last so the later-wins tiebreaker beats agent at 20.
|
|
20
20
|
*/
|
|
21
|
-
import { SCRIPT_EXTENSIONS } from "
|
|
21
|
+
import { SCRIPT_EXTENSIONS } from "../core/asset-spec";
|
|
22
|
+
import { looksLikeWorkflow } from "../workflows/parser";
|
|
22
23
|
import { registerMatcher } from "./file-context";
|
|
23
24
|
// ── extensionMatcher (specificity: 3) ────────────────────────────────────────
|
|
24
25
|
/**
|
|
@@ -140,11 +141,7 @@ export function smartMdMatcher(ctx) {
|
|
|
140
141
|
if (ctx.ext !== ".md")
|
|
141
142
|
return null;
|
|
142
143
|
const body = ctx.content();
|
|
143
|
-
|
|
144
|
-
/^##\s+Step:\s+/m.test(body) &&
|
|
145
|
-
/^Step ID:\s+/m.test(body) &&
|
|
146
|
-
/^###\s+Instructions\s*$/m.test(body);
|
|
147
|
-
if (hasWorkflowSignals) {
|
|
144
|
+
if (looksLikeWorkflow(body)) {
|
|
148
145
|
return { type: "workflow", specificity: 19, renderer: "workflow-md" };
|
|
149
146
|
}
|
|
150
147
|
const fm = ctx.frontmatter();
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
|
-
import { deriveCanonicalAssetName, deriveCanonicalAssetNameFromStashRoot, isRelevantAssetFile } from "
|
|
4
|
-
import { isAssetType } from "
|
|
3
|
+
import { deriveCanonicalAssetName, deriveCanonicalAssetNameFromStashRoot, isRelevantAssetFile, } from "../core/asset-spec";
|
|
4
|
+
import { isAssetType } from "../core/common";
|
|
5
|
+
import { parseFrontmatter, toStringOrUndefined } from "../core/frontmatter";
|
|
6
|
+
import { warn } from "../core/warn";
|
|
5
7
|
import { buildFileContext, buildRenderContext, getRenderer, runMatchers } from "./file-context";
|
|
6
|
-
import { parseFrontmatter, toStringOrUndefined } from "./frontmatter";
|
|
7
|
-
import { warn } from "./warn";
|
|
8
8
|
// ── Load / Write ────────────────────────────────────────────────────────────
|
|
9
9
|
const STASH_FILENAME = ".stash.json";
|
|
10
10
|
export function stashFilePath(dirPath) {
|
|
@@ -568,7 +568,11 @@ export async function generateMetadataFlat(stashRoot, files) {
|
|
|
568
568
|
}
|
|
569
569
|
function buildMetadataSkipWarning(filePath, assetType, error) {
|
|
570
570
|
const detail = error instanceof Error ? error.message : String(error);
|
|
571
|
-
|
|
571
|
+
// Workflow errors are already multi-line `path:line — message` blocks; print
|
|
572
|
+
// them as-is so the author sees a flat list without a redundant prefix.
|
|
573
|
+
const warning = assetType === "workflow"
|
|
574
|
+
? `Skipped workflow ${filePath}:\n${detail}`
|
|
575
|
+
: `Skipped malformed ${assetType} asset at ${filePath}: ${detail}`;
|
|
572
576
|
warn(warning);
|
|
573
577
|
return warning;
|
|
574
578
|
}
|
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
|
-
import { resolveStashDir } from "
|
|
4
|
-
import { loadConfig } from "
|
|
5
|
-
import {
|
|
6
|
-
|
|
7
|
-
|
|
3
|
+
import { resolveStashDir } from "../core/common";
|
|
4
|
+
import { loadConfig } from "../core/config";
|
|
5
|
+
import { resolveSourceProviderFactory } from "../sources/provider-factory";
|
|
6
|
+
// Eager side-effect imports so all built-in source providers self-register
|
|
7
|
+
// before resolveEntryContentDir() runs.
|
|
8
|
+
import "../sources/providers/index";
|
|
9
|
+
import { warn } from "../core/warn";
|
|
10
|
+
import { ensureGitMirror, getCachePaths, parseGitRepoUrl } from "../sources/providers/git";
|
|
11
|
+
import { ensureWebsiteMirror } from "../sources/providers/website";
|
|
8
12
|
// Legacy "context-hub" / "github" type aliases are normalized to "git" at
|
|
9
13
|
// config-load time (see src/config.ts), so this set only contains the canonical
|
|
10
14
|
// type.
|
|
@@ -17,7 +21,7 @@ const GIT_STASH_TYPES = new Set(["git"]);
|
|
|
17
21
|
* 1. The primary stash directory (the entry marked `primary: true`, or the
|
|
18
22
|
* legacy top-level `stashDir`). Always emitted, even when the directory
|
|
19
23
|
* does not yet exist on disk, so callers can use it as the clone target.
|
|
20
|
-
* 2. Each entry in `config.stashes[]` (in declared order), excluding the
|
|
24
|
+
* 2. Each entry in `config.sources ?? config.stashes[]` (in declared order), excluding the
|
|
21
25
|
* one already emitted as the primary.
|
|
22
26
|
* 3. Each entry in `config.installed[]` (registry-managed stashes).
|
|
23
27
|
*
|
|
@@ -25,7 +29,7 @@ const GIT_STASH_TYPES = new Set(["git"]);
|
|
|
25
29
|
* for each provider kind. Disabled entries (`enabled: false`) and entries
|
|
26
30
|
* whose disk path doesn't exist are filtered after deduplication.
|
|
27
31
|
*/
|
|
28
|
-
export function
|
|
32
|
+
export function resolveSourceEntries(overrideStashDir, existingConfig) {
|
|
29
33
|
const stashDir = overrideStashDir ?? resolveStashDir();
|
|
30
34
|
const config = existingConfig ?? loadConfig();
|
|
31
35
|
const sources = [{ path: stashDir }];
|
|
@@ -49,7 +53,7 @@ export function resolveStashSources(overrideStashDir, existingConfig) {
|
|
|
49
53
|
// (1) + (2) Single pass over declared stashes — primary first if present,
|
|
50
54
|
// then the rest in declared order. The primary's directory is already
|
|
51
55
|
// injected as `sources[0]` above, so we only need to dedupe the source set.
|
|
52
|
-
const stashes = config.stashes ?? [];
|
|
56
|
+
const stashes = config.sources ?? config.stashes ?? [];
|
|
53
57
|
const primaryIdx = stashes.findIndex((entry) => entry.primary === true);
|
|
54
58
|
const ordered = [];
|
|
55
59
|
if (primaryIdx >= 0) {
|
|
@@ -78,43 +82,54 @@ export function resolveStashSources(overrideStashDir, existingConfig) {
|
|
|
78
82
|
}
|
|
79
83
|
/**
|
|
80
84
|
* Resolve the content directory the indexer should walk for a given config
|
|
81
|
-
* entry. Returns `undefined` if the entry has no walkable content
|
|
82
|
-
*
|
|
85
|
+
* entry. Returns `undefined` if the entry has no walkable content
|
|
86
|
+
* so the caller can skip it.
|
|
87
|
+
*
|
|
88
|
+
* Single source of truth: each provider owns its own path. We instantiate the
|
|
89
|
+
* registered {@link import("../sources/provider").SourceProvider} for the entry
|
|
90
|
+
* and call `provider.path()`. This replaces the old per-kind switch ladder
|
|
91
|
+
* (filesystem path / git cache / website cache) that lived here in 0.6.0 —
|
|
92
|
+
* see spec §10 step 4 and §7 "Removed from 0.6.0".
|
|
93
|
+
*
|
|
94
|
+
* The git case still does one extra step: the provider returns the cloned
|
|
95
|
+
* repo dir, but the indexer walks the `content/` subdirectory inside it.
|
|
96
|
+
* That convention is part of the akm content layout, not a provider concern,
|
|
97
|
+
* so it stays here.
|
|
83
98
|
*/
|
|
84
99
|
function resolveEntryContentDir(entry) {
|
|
85
|
-
|
|
86
|
-
|
|
100
|
+
const factory = resolveSourceProviderFactory(entry.type);
|
|
101
|
+
if (!factory)
|
|
102
|
+
return undefined;
|
|
103
|
+
let provider;
|
|
104
|
+
try {
|
|
105
|
+
provider = factory(entry);
|
|
87
106
|
}
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
const cachePaths = getCachePaths(repo.canonicalUrl);
|
|
92
|
-
// The content/ subdirectory inside the extracted repo is the actual
|
|
93
|
-
// stash root containing DOC.md / SKILL.md files that the walker indexes.
|
|
94
|
-
return path.join(cachePaths.repoDir, "content");
|
|
95
|
-
}
|
|
96
|
-
catch (err) {
|
|
97
|
-
warn(`Warning: failed to resolve git stash cache for "${entry.url}": ${err instanceof Error ? err.message : String(err)}`);
|
|
98
|
-
return undefined;
|
|
99
|
-
}
|
|
107
|
+
catch (err) {
|
|
108
|
+
warn(`Warning: failed to construct ${entry.type} source provider for "${entry.name ?? entry.url ?? entry.path}": ${err instanceof Error ? err.message : String(err)}`);
|
|
109
|
+
return undefined;
|
|
100
110
|
}
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
}
|
|
105
|
-
catch (err) {
|
|
106
|
-
warn(`Warning: failed to resolve website stash cache for "${entry.url}": ${err instanceof Error ? err.message : String(err)}`);
|
|
107
|
-
return undefined;
|
|
108
|
-
}
|
|
111
|
+
let dir;
|
|
112
|
+
try {
|
|
113
|
+
dir = provider.path();
|
|
109
114
|
}
|
|
110
|
-
|
|
111
|
-
|
|
115
|
+
catch (err) {
|
|
116
|
+
warn(`Warning: failed to resolve ${entry.type} source path for "${entry.name ?? entry.url ?? entry.path}": ${err instanceof Error ? err.message : String(err)}`);
|
|
117
|
+
return undefined;
|
|
118
|
+
}
|
|
119
|
+
// Git providers expose the cloned repo root as their path. The akm content
|
|
120
|
+
// layout puts indexable files under `<repo>/content/`, so the walker needs
|
|
121
|
+
// that subdirectory. This is a content-layout convention, not a provider
|
|
122
|
+
// capability — keep it here.
|
|
123
|
+
if (GIT_STASH_TYPES.has(entry.type)) {
|
|
124
|
+
return path.join(dir, "content");
|
|
125
|
+
}
|
|
126
|
+
return dir;
|
|
112
127
|
}
|
|
113
128
|
/**
|
|
114
129
|
* Convenience: returns just the directory paths, preserving priority order.
|
|
115
130
|
*/
|
|
116
131
|
export function resolveAllStashDirs(overrideStashDir) {
|
|
117
|
-
return
|
|
132
|
+
return resolveSourceEntries(overrideStashDir).map((s) => s.path);
|
|
118
133
|
}
|
|
119
134
|
/**
|
|
120
135
|
* Find which source a file path belongs to.
|
|
@@ -204,10 +219,10 @@ function isValidDirectory(dir) {
|
|
|
204
219
|
/**
|
|
205
220
|
* Ensure all cache-backed stash providers are refreshed so their cache
|
|
206
221
|
* directories exist on disk. Must be called (async) before
|
|
207
|
-
* `
|
|
222
|
+
* `resolveSourceEntries()` so the content directories pass the
|
|
208
223
|
* `isValidDirectory()` check.
|
|
209
224
|
*/
|
|
210
|
-
export async function
|
|
225
|
+
export async function ensureSourceCaches(config) {
|
|
211
226
|
const cfg = config ?? loadConfig();
|
|
212
227
|
for (const entry of cfg.stashes ?? []) {
|
|
213
228
|
if (!GIT_STASH_TYPES.has(entry.type) || !entry.url || entry.enabled === false)
|
|
@@ -232,7 +247,3 @@ export async function ensureStashCaches(config) {
|
|
|
232
247
|
}
|
|
233
248
|
}
|
|
234
249
|
}
|
|
235
|
-
/** @deprecated Use ensureStashCaches instead. */
|
|
236
|
-
export const ensureGitCaches = ensureStashCaches;
|
|
237
|
-
/** @deprecated Use ensureStashCaches instead. */
|
|
238
|
-
export const ensureContextHubCaches = ensureStashCaches;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
3
|
+
import { getCacheDir, getSemanticStatusPath } from "../core/paths";
|
|
4
|
+
import { DEFAULT_LOCAL_MODEL } from "../llm/embedder";
|
|
5
5
|
export function deriveSemanticProviderFingerprint(embedding) {
|
|
6
6
|
if (embedding?.endpoint) {
|
|
7
7
|
return `remote:${embedding.endpoint}|${embedding.model}|${embedding.dimension ?? "default"}`;
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
*/
|
|
8
8
|
import fs from "node:fs";
|
|
9
9
|
import path from "node:path";
|
|
10
|
-
import { isRelevantAssetFile } from "
|
|
10
|
+
import { isRelevantAssetFile } from "../core/asset-spec";
|
|
11
11
|
import { buildFileContext } from "./file-context";
|
|
12
12
|
const SKIP_DIRS = new Set([".git", "node_modules", "bin", ".cache"]);
|
|
13
13
|
/**
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
|
-
import { getConfigDir } from "
|
|
3
|
+
import { getConfigDir } from "../core/config";
|
|
4
4
|
// ── Paths ───────────────────────────────────────────────────────────────────
|
|
5
5
|
const LOCKFILE_NAME = "akm.lock";
|
|
6
6
|
const LEGACY_LOCKFILE_NAME = "stash.lock";
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
*
|
|
8
8
|
* `llm.ts` re-exports everything from this module for backward compatibility.
|
|
9
9
|
*/
|
|
10
|
-
import { fetchWithTimeout } from "
|
|
10
|
+
import { fetchWithTimeout } from "../core/common";
|
|
11
11
|
export async function chatCompletion(config, messages, options) {
|
|
12
12
|
const headers = { "Content-Type": "application/json" };
|
|
13
13
|
if (config.apiKey) {
|
|
@@ -7,8 +7,8 @@
|
|
|
7
7
|
* shared instance for the production code path.
|
|
8
8
|
*/
|
|
9
9
|
import path from "node:path";
|
|
10
|
-
import { getCacheDir } from "
|
|
11
|
-
import { warn } from "
|
|
10
|
+
import { getCacheDir } from "../../core/paths";
|
|
11
|
+
import { warn } from "../../core/warn";
|
|
12
12
|
/**
|
|
13
13
|
* Default local transformer model for embeddings.
|
|
14
14
|
* `bge-small-en-v1.5` scores higher on MTEB benchmarks than the previous
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Calls the configured `/embeddings` endpoint and L2-normalizes the returned
|
|
5
5
|
* vectors so the scoring pipeline's L2-to-cosine conversion is correct.
|
|
6
6
|
*/
|
|
7
|
-
import { fetchWithTimeout, isHttpUrl } from "
|
|
7
|
+
import { fetchWithTimeout, isHttpUrl } from "../../core/common";
|
|
8
8
|
const REMOTE_BATCH_SIZE = 100;
|
|
9
9
|
export class RemoteEmbedder {
|
|
10
10
|
config;
|
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Split out of `llm.ts` so the higher-level workflow (prompting the LLM to
|
|
5
5
|
* improve descriptions/tags/searchHints) lives separately from the low-level
|
|
6
|
-
* transport client in `
|
|
6
|
+
* transport client in `client.ts`.
|
|
7
7
|
*/
|
|
8
|
-
import { chatCompletion, parseJsonResponse } from "./
|
|
8
|
+
import { chatCompletion, parseJsonResponse } from "./client";
|
|
9
9
|
const SYSTEM_PROMPT = `You are a metadata generator for a developer asset registry. Given a script/skill/command/agent entry, generate improved metadata. Respond with ONLY valid JSON, no markdown fencing.`;
|
|
10
10
|
/**
|
|
11
11
|
* Use an LLM to enhance a stash entry's metadata: improve description,
|
|
@@ -121,9 +121,12 @@ akm remember --name release-retro < notes.md # Save multiline memory from stdi
|
|
|
121
121
|
akm import ./docs/auth-flow.md # Import a file as knowledge
|
|
122
122
|
akm import - --name scratch-notes < notes.md # Import stdin as a knowledge doc
|
|
123
123
|
akm workflow create ship-release # Create a workflow asset in the stash
|
|
124
|
+
akm workflow validate workflows/foo.md # Validate a workflow file or ref; lists every error
|
|
124
125
|
akm workflow next workflow:ship-release # Start or resume the next workflow step
|
|
125
126
|
akm feedback skill:code-review --positive # Record that an asset helped
|
|
126
127
|
akm feedback agent:reviewer --negative # Record that an asset missed the mark
|
|
128
|
+
akm feedback memory:deployment-notes --positive # Works for memories too
|
|
129
|
+
akm feedback vault:prod --positive # Records vault feedback without surfacing values
|
|
127
130
|
\`\`\`
|
|
128
131
|
|
|
129
132
|
Use \`akm feedback\` whenever an asset materially helps or fails so future search
|