akm-cli 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +12 -2
- package/dist/common.js +3 -0
- package/dist/config.js +1 -1
- package/dist/db.js +4 -0
- package/dist/embedder.js +48 -12
- package/dist/indexer.js +111 -6
- package/dist/renderers.js +1 -1
- package/dist/setup.js +151 -7
- package/dist/stash-providers/git.js +1 -1
- package/dist/stash-providers/openviking.js +1 -1
- package/package.json +2 -2
package/dist/cli.js
CHANGED
|
@@ -283,7 +283,13 @@ function formatPlain(command, result, detail) {
|
|
|
283
283
|
return out;
|
|
284
284
|
}
|
|
285
285
|
case "index": {
|
|
286
|
-
|
|
286
|
+
const indexResult = result;
|
|
287
|
+
let out = `Indexed ${indexResult.totalEntries ?? 0} entries from ${indexResult.directoriesScanned ?? 0} directories (mode: ${indexResult.mode ?? "unknown"})`;
|
|
288
|
+
const verification = indexResult.verification;
|
|
289
|
+
if (verification?.ok === false && verification.message) {
|
|
290
|
+
out += `\nVerification: ${String(verification.message)}`;
|
|
291
|
+
}
|
|
292
|
+
return out;
|
|
287
293
|
}
|
|
288
294
|
case "show": {
|
|
289
295
|
const lines = [];
|
|
@@ -481,10 +487,14 @@ const indexCommand = defineCommand({
|
|
|
481
487
|
meta: { name: "index", description: "Build search index (incremental by default; --full forces full reindex)" },
|
|
482
488
|
args: {
|
|
483
489
|
full: { type: "boolean", description: "Force full reindex", default: false },
|
|
490
|
+
verbose: { type: "boolean", description: "Print indexing summary and phase progress to stderr", default: false },
|
|
484
491
|
},
|
|
485
492
|
async run({ args }) {
|
|
486
493
|
await runWithJsonErrors(async () => {
|
|
487
|
-
const result = await akmIndex({
|
|
494
|
+
const result = await akmIndex({
|
|
495
|
+
full: args.full,
|
|
496
|
+
onProgress: args.verbose ? ({ message }) => console.error(`[index] ${message}`) : undefined,
|
|
497
|
+
});
|
|
488
498
|
output("index", result);
|
|
489
499
|
});
|
|
490
500
|
},
|
package/dist/common.js
CHANGED
|
@@ -5,6 +5,9 @@ import { ConfigError } from "./errors";
|
|
|
5
5
|
import { getConfigPath, getDefaultStashDir } from "./paths";
|
|
6
6
|
// ── Constants ───────────────────────────────────────────────────────────────
|
|
7
7
|
export const IS_WINDOWS = process.platform === "win32";
|
|
8
|
+
export function isHttpUrl(value) {
|
|
9
|
+
return !!value && /^https?:\/\//.test(value);
|
|
10
|
+
}
|
|
8
11
|
// ── Validators ──────────────────────────────────────────────────────────────
|
|
9
12
|
export function isAssetType(type) {
|
|
10
13
|
return Object.hasOwn(TYPE_DIRS, type);
|
package/dist/config.js
CHANGED
|
@@ -460,7 +460,7 @@ function parseRegistryConfigEntry(value) {
|
|
|
460
460
|
return undefined;
|
|
461
461
|
const obj = value;
|
|
462
462
|
const url = asNonEmptyString(obj.url);
|
|
463
|
-
if (!url
|
|
463
|
+
if (!url?.startsWith("http"))
|
|
464
464
|
return undefined;
|
|
465
465
|
const entry = { url };
|
|
466
466
|
const name = asNonEmptyString(obj.name);
|
package/dist/db.js
CHANGED
|
@@ -535,6 +535,10 @@ export function getEntryCount(db) {
|
|
|
535
535
|
const row = db.prepare("SELECT COUNT(*) AS cnt FROM entries").get();
|
|
536
536
|
return row.cnt;
|
|
537
537
|
}
|
|
538
|
+
export function getEmbeddingCount(db) {
|
|
539
|
+
const row = db.prepare("SELECT COUNT(*) AS cnt FROM embeddings").get();
|
|
540
|
+
return row.cnt;
|
|
541
|
+
}
|
|
538
542
|
export function getEntryById(db, id) {
|
|
539
543
|
const row = db.prepare("SELECT file_path, entry_json FROM entries WHERE id = ?").get(id);
|
|
540
544
|
if (!row)
|
package/dist/embedder.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { fetchWithTimeout } from "./common";
|
|
1
|
+
import { fetchWithTimeout, isHttpUrl } from "./common";
|
|
2
2
|
import { warn } from "./warn";
|
|
3
3
|
// ── Default local model ─────────────────────────────────────────────────────
|
|
4
4
|
/**
|
|
@@ -32,11 +32,11 @@ async function getLocalEmbedder(modelName) {
|
|
|
32
32
|
localEmbedderPromise = (async () => {
|
|
33
33
|
let pipeline;
|
|
34
34
|
try {
|
|
35
|
-
const mod = await import("@
|
|
35
|
+
const mod = await import("@huggingface/transformers");
|
|
36
36
|
pipeline = mod.pipeline;
|
|
37
37
|
}
|
|
38
38
|
catch {
|
|
39
|
-
throw new Error("Semantic search requires @
|
|
39
|
+
throw new Error("Semantic search requires @huggingface/transformers. Install it with: npm install @huggingface/transformers");
|
|
40
40
|
}
|
|
41
41
|
const pipelineFn = pipeline;
|
|
42
42
|
return pipelineFn("feature-extraction", resolvedModel);
|
|
@@ -103,7 +103,7 @@ async function embedRemote(text, config) {
|
|
|
103
103
|
// ── Helpers ──────────────────────────────────────────────────────────────────
|
|
104
104
|
/** Check whether an EmbeddingConnectionConfig has a valid remote endpoint. */
|
|
105
105
|
function hasRemoteEndpoint(config) {
|
|
106
|
-
return
|
|
106
|
+
return isHttpUrl(config.endpoint);
|
|
107
107
|
}
|
|
108
108
|
// ── LRU embedding cache ─────────────────────────────────────────────────────
|
|
109
109
|
// Caches query embeddings to avoid redundant computation for repeated queries.
|
|
@@ -133,7 +133,7 @@ export function clearEmbeddingCache() {
|
|
|
133
133
|
/**
|
|
134
134
|
* Generate an embedding for the given text.
|
|
135
135
|
* If embeddingConfig has a remote endpoint, uses the configured OpenAI-compatible endpoint.
|
|
136
|
-
* Otherwise falls back to local @
|
|
136
|
+
* Otherwise falls back to local @huggingface/transformers using the model from
|
|
137
137
|
* `embeddingConfig.localModel` or `DEFAULT_LOCAL_MODEL`.
|
|
138
138
|
*
|
|
139
139
|
* Results are cached in an LRU cache (max ~100 entries) keyed by query text
|
|
@@ -244,21 +244,57 @@ export function cosineSimilarity(a, b) {
|
|
|
244
244
|
return denom === 0 ? 0 : dot / denom;
|
|
245
245
|
}
|
|
246
246
|
// ── Availability check ──────────────────────────────────────────────────────
|
|
247
|
-
|
|
247
|
+
/**
|
|
248
|
+
* Check whether the `@huggingface/transformers` package can be imported.
|
|
249
|
+
* Returns `true` if it can, `false` otherwise.
|
|
250
|
+
*/
|
|
251
|
+
export async function isTransformersAvailable() {
|
|
252
|
+
try {
|
|
253
|
+
await import("@huggingface/transformers");
|
|
254
|
+
return true;
|
|
255
|
+
}
|
|
256
|
+
catch {
|
|
257
|
+
return false;
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
/**
|
|
261
|
+
* Check whether embedding is available with a detailed reason on failure.
|
|
262
|
+
*/
|
|
263
|
+
export async function checkEmbeddingAvailability(embeddingConfig) {
|
|
248
264
|
if (embeddingConfig && hasRemoteEndpoint(embeddingConfig)) {
|
|
249
265
|
try {
|
|
250
266
|
await embedRemote("test", embeddingConfig);
|
|
251
|
-
return true;
|
|
267
|
+
return { available: true };
|
|
252
268
|
}
|
|
253
|
-
catch {
|
|
254
|
-
return
|
|
269
|
+
catch (err) {
|
|
270
|
+
return {
|
|
271
|
+
available: false,
|
|
272
|
+
reason: "remote-unreachable",
|
|
273
|
+
message: err instanceof Error ? err.message : String(err),
|
|
274
|
+
};
|
|
255
275
|
}
|
|
256
276
|
}
|
|
277
|
+
// Check if the package is importable before attempting the model download.
|
|
278
|
+
if (!(await isTransformersAvailable())) {
|
|
279
|
+
return {
|
|
280
|
+
available: false,
|
|
281
|
+
reason: "missing-package",
|
|
282
|
+
message: "@huggingface/transformers is not installed.",
|
|
283
|
+
};
|
|
284
|
+
}
|
|
257
285
|
try {
|
|
258
286
|
await getLocalEmbedder(embeddingConfig?.localModel);
|
|
259
|
-
return true;
|
|
287
|
+
return { available: true };
|
|
260
288
|
}
|
|
261
|
-
catch {
|
|
262
|
-
return
|
|
289
|
+
catch (err) {
|
|
290
|
+
return {
|
|
291
|
+
available: false,
|
|
292
|
+
reason: "model-download-failed",
|
|
293
|
+
message: err instanceof Error ? err.message : String(err),
|
|
294
|
+
};
|
|
263
295
|
}
|
|
264
296
|
}
|
|
297
|
+
export async function isEmbeddingAvailable(embeddingConfig) {
|
|
298
|
+
const result = await checkEmbeddingAvailability(embeddingConfig);
|
|
299
|
+
return result.available;
|
|
300
|
+
}
|
package/dist/indexer.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
|
-
import { resolveStashDir } from "./common";
|
|
4
|
-
import { closeDatabase, deleteEntriesByDir, deleteEntriesByStashDir, getEntriesByDir, getEntryCount, getMeta, isVecAvailable, openDatabase, rebuildFts, setMeta, upsertEmbedding, upsertEntry, upsertUtilityScore, warnIfVecMissing, } from "./db";
|
|
3
|
+
import { isHttpUrl, resolveStashDir } from "./common";
|
|
4
|
+
import { closeDatabase, deleteEntriesByDir, deleteEntriesByStashDir, getEmbeddingCount, getEntriesByDir, getEntryCount, getMeta, isVecAvailable, openDatabase, rebuildFts, setMeta, upsertEmbedding, upsertEntry, upsertUtilityScore, warnIfVecMissing, } from "./db";
|
|
5
5
|
import { generateMetadataFlat, loadStashFile } from "./metadata";
|
|
6
6
|
import { getDbPath } from "./paths";
|
|
7
7
|
import { buildSearchText } from "./search-fields";
|
|
@@ -11,6 +11,7 @@ import { warn } from "./warn";
|
|
|
11
11
|
// ── Indexer ──────────────────────────────────────────────────────────────────
|
|
12
12
|
export async function akmIndex(options) {
|
|
13
13
|
const stashDir = options?.stashDir || resolveStashDir();
|
|
14
|
+
const onProgress = options?.onProgress ?? (() => { });
|
|
14
15
|
// Load config and resolve all stash sources
|
|
15
16
|
const { loadConfig } = await import("./config.js");
|
|
16
17
|
const config = loadConfig();
|
|
@@ -30,6 +31,17 @@ export async function akmIndex(options) {
|
|
|
30
31
|
const prevBuiltAt = getMeta(db, "builtAt");
|
|
31
32
|
const isIncremental = !options?.full && prevStashDir === stashDir && !!prevBuiltAt;
|
|
32
33
|
const builtAtMs = isIncremental && prevBuiltAt ? new Date(prevBuiltAt).getTime() : 0;
|
|
34
|
+
onProgress({
|
|
35
|
+
phase: "summary",
|
|
36
|
+
message: buildIndexSummaryMessage({
|
|
37
|
+
mode: isIncremental ? "incremental" : "full",
|
|
38
|
+
stashSources: allStashDirs.length,
|
|
39
|
+
semanticSearch: config.semanticSearch,
|
|
40
|
+
embeddingProvider: getEmbeddingProvider(config.embedding),
|
|
41
|
+
llmEnabled: !!config.llm,
|
|
42
|
+
vecAvailable: isVecAvailable(db),
|
|
43
|
+
}),
|
|
44
|
+
});
|
|
33
45
|
if (options?.full || !isIncremental) {
|
|
34
46
|
// The delete is now merged into the insert transaction inside
|
|
35
47
|
// indexEntries() so that a reader never sees an empty database between
|
|
@@ -67,17 +79,28 @@ export async function akmIndex(options) {
|
|
|
67
79
|
// inserts so readers never see an empty database mid-rebuild.
|
|
68
80
|
const doFullDelete = options?.full || !isIncremental;
|
|
69
81
|
const { scannedDirs, skippedDirs, generatedCount, dirsNeedingLlm } = await indexEntries(db, allStashDirs, isIncremental, builtAtMs, doFullDelete);
|
|
82
|
+
onProgress({
|
|
83
|
+
phase: "scan",
|
|
84
|
+
message: `Scanned ${scannedDirs} ${scannedDirs === 1 ? "directory" : "directories"} and skipped ${skippedDirs}.`,
|
|
85
|
+
});
|
|
70
86
|
const tWalkEnd = Date.now();
|
|
71
87
|
// Enhance entries with LLM if configured
|
|
72
88
|
await enhanceDirsWithLlm(db, config, dirsNeedingLlm);
|
|
89
|
+
onProgress({
|
|
90
|
+
phase: "llm",
|
|
91
|
+
message: config.llm
|
|
92
|
+
? `LLM enhancement reviewed ${dirsNeedingLlm.length} ${dirsNeedingLlm.length === 1 ? "directory" : "directories"}.`
|
|
93
|
+
: "LLM enhancement disabled.",
|
|
94
|
+
});
|
|
73
95
|
const tLlmEnd = Date.now();
|
|
74
96
|
// Rebuild FTS after all inserts
|
|
75
97
|
rebuildFts(db);
|
|
98
|
+
onProgress({ phase: "fts", message: "Rebuilt full-text search index." });
|
|
76
99
|
const tFtsEnd = Date.now();
|
|
77
100
|
// Recompute utility scores from usage_events after FTS rebuild
|
|
78
101
|
recomputeUtilityScores(db);
|
|
79
102
|
// Generate embeddings if semantic search is enabled
|
|
80
|
-
const hasEmbeddings = await generateEmbeddingsForDb(db, config);
|
|
103
|
+
const hasEmbeddings = await generateEmbeddingsForDb(db, config, onProgress);
|
|
81
104
|
const tEmbedEnd = Date.now();
|
|
82
105
|
// Update metadata
|
|
83
106
|
setMeta(db, "builtAt", new Date().toISOString());
|
|
@@ -88,6 +111,8 @@ export async function akmIndex(options) {
|
|
|
88
111
|
// Warn on every index run if using JS fallback with many entries
|
|
89
112
|
warnIfVecMissing(db);
|
|
90
113
|
const tEnd = Date.now();
|
|
114
|
+
const verification = verifyIndexState(db, config, totalEntries);
|
|
115
|
+
onProgress({ phase: "verify", message: verification.message });
|
|
91
116
|
return {
|
|
92
117
|
stashDir,
|
|
93
118
|
totalEntries,
|
|
@@ -96,6 +121,7 @@ export async function akmIndex(options) {
|
|
|
96
121
|
mode: isIncremental ? "incremental" : "full",
|
|
97
122
|
directoriesScanned: scannedDirs,
|
|
98
123
|
directoriesSkipped: skippedDirs,
|
|
124
|
+
verification,
|
|
99
125
|
timing: {
|
|
100
126
|
totalMs: tEnd - t0,
|
|
101
127
|
walkMs: tWalkEnd - tWalkStart,
|
|
@@ -256,14 +282,22 @@ async function enhanceDirsWithLlm(db, config, dirsNeedingLlm) {
|
|
|
256
282
|
})();
|
|
257
283
|
}
|
|
258
284
|
}
|
|
259
|
-
async function generateEmbeddingsForDb(db, config) {
|
|
260
|
-
if (!config.semanticSearch)
|
|
285
|
+
async function generateEmbeddingsForDb(db, config, onProgress) {
|
|
286
|
+
if (!config.semanticSearch) {
|
|
287
|
+
onProgress({ phase: "embeddings", message: "Semantic search disabled; skipping embeddings." });
|
|
261
288
|
return false;
|
|
289
|
+
}
|
|
262
290
|
try {
|
|
263
291
|
const { embedBatch } = await import("./embedder.js");
|
|
264
292
|
const allEntries = getAllEntriesForEmbedding(db);
|
|
265
|
-
if (allEntries.length === 0)
|
|
293
|
+
if (allEntries.length === 0) {
|
|
294
|
+
onProgress({ phase: "embeddings", message: "Embeddings already up to date." });
|
|
266
295
|
return true;
|
|
296
|
+
}
|
|
297
|
+
onProgress({
|
|
298
|
+
phase: "embeddings",
|
|
299
|
+
message: `Generating embeddings for ${allEntries.length} entr${allEntries.length === 1 ? "y" : "ies"}.`,
|
|
300
|
+
});
|
|
267
301
|
const texts = allEntries.map((e) => e.searchText);
|
|
268
302
|
const embeddings = await embedBatch(texts, config.embedding);
|
|
269
303
|
// Wrap all embedding upserts in a single transaction so partial
|
|
@@ -273,10 +307,18 @@ async function generateEmbeddingsForDb(db, config) {
|
|
|
273
307
|
upsertEmbedding(db, allEntries[i].id, embeddings[i]);
|
|
274
308
|
}
|
|
275
309
|
})();
|
|
310
|
+
onProgress({
|
|
311
|
+
phase: "embeddings",
|
|
312
|
+
message: `Stored ${embeddings.length} embedding${embeddings.length === 1 ? "" : "s"}.`,
|
|
313
|
+
});
|
|
276
314
|
return true;
|
|
277
315
|
}
|
|
278
316
|
catch (error) {
|
|
279
317
|
warn("Embedding generation failed, continuing without:", error instanceof Error ? error.message : String(error));
|
|
318
|
+
onProgress({
|
|
319
|
+
phase: "embeddings",
|
|
320
|
+
message: `Embedding generation failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
321
|
+
});
|
|
280
322
|
return false;
|
|
281
323
|
}
|
|
282
324
|
}
|
|
@@ -297,6 +339,69 @@ function attachFileSize(entry, entryPath) {
|
|
|
297
339
|
return entry;
|
|
298
340
|
}
|
|
299
341
|
}
|
|
342
|
+
function buildIndexSummaryMessage(options) {
|
|
343
|
+
const stashSourceLabel = options.stashSources === 1 ? "stash source" : "stash sources";
|
|
344
|
+
const semanticDetail = getSemanticSearchLabel(options.semanticSearch, options.embeddingProvider, options.vecAvailable);
|
|
345
|
+
return `Starting ${options.mode} index (${options.stashSources} ${stashSourceLabel}, semantic search: ${semanticDetail}, LLM: ${options.llmEnabled ? "enabled" : "disabled"}).`;
|
|
346
|
+
}
|
|
347
|
+
function getEmbeddingProvider(embedding) {
|
|
348
|
+
return isHttpUrl(embedding?.endpoint) ? "remote" : "local";
|
|
349
|
+
}
|
|
350
|
+
function getSemanticSearchLabel(semanticSearch, embeddingProvider, vecAvailable) {
|
|
351
|
+
if (!semanticSearch)
|
|
352
|
+
return "disabled";
|
|
353
|
+
return `${embeddingProvider} embeddings, ${vecAvailable ? "sqlite-vec" : "JS fallback"}`;
|
|
354
|
+
}
|
|
355
|
+
function verifyIndexState(db, config, totalEntries) {
|
|
356
|
+
const embeddingCount = getEmbeddingCount(db);
|
|
357
|
+
const vecAvailable = isVecAvailable(db);
|
|
358
|
+
const embeddingProvider = getEmbeddingProvider(config.embedding);
|
|
359
|
+
if (totalEntries === 0) {
|
|
360
|
+
return {
|
|
361
|
+
ok: true,
|
|
362
|
+
message: "Index ready. No assets were found yet.",
|
|
363
|
+
semanticSearchEnabled: config.semanticSearch,
|
|
364
|
+
embeddingProvider,
|
|
365
|
+
entryCount: totalEntries,
|
|
366
|
+
embeddingCount,
|
|
367
|
+
vecAvailable,
|
|
368
|
+
};
|
|
369
|
+
}
|
|
370
|
+
if (!config.semanticSearch) {
|
|
371
|
+
return {
|
|
372
|
+
ok: true,
|
|
373
|
+
message: "Keyword index ready. Semantic search is disabled.",
|
|
374
|
+
semanticSearchEnabled: false,
|
|
375
|
+
embeddingProvider,
|
|
376
|
+
entryCount: totalEntries,
|
|
377
|
+
embeddingCount,
|
|
378
|
+
vecAvailable,
|
|
379
|
+
};
|
|
380
|
+
}
|
|
381
|
+
if (embeddingCount >= totalEntries) {
|
|
382
|
+
return {
|
|
383
|
+
ok: true,
|
|
384
|
+
message: `Semantic search ready (${embeddingCount}/${totalEntries} embeddings, ${vecAvailable ? "sqlite-vec active" : "JS fallback active"}).`,
|
|
385
|
+
semanticSearchEnabled: true,
|
|
386
|
+
embeddingProvider,
|
|
387
|
+
entryCount: totalEntries,
|
|
388
|
+
embeddingCount,
|
|
389
|
+
vecAvailable,
|
|
390
|
+
};
|
|
391
|
+
}
|
|
392
|
+
return {
|
|
393
|
+
ok: false,
|
|
394
|
+
message: `Semantic search verification failed (${embeddingCount}/${totalEntries} embeddings available).`,
|
|
395
|
+
guidance: embeddingProvider === "remote"
|
|
396
|
+
? "Check your embedding endpoint and credentials, then retry `akm index --full --verbose`."
|
|
397
|
+
: "Retry `akm index --full --verbose`. If it still fails, confirm local model downloads are permitted and see docs/configuration.md for local embedding dependency setup.",
|
|
398
|
+
semanticSearchEnabled: true,
|
|
399
|
+
embeddingProvider,
|
|
400
|
+
entryCount: totalEntries,
|
|
401
|
+
embeddingCount,
|
|
402
|
+
vecAvailable,
|
|
403
|
+
};
|
|
404
|
+
}
|
|
300
405
|
function isDirStale(dirPath, currentFiles, previousEntries, builtAtMs) {
|
|
301
406
|
// Check if file set changed (additions or deletions)
|
|
302
407
|
const prevFileNames = new Set(previousEntries.map((ie) => ie.entry.filename).filter((e) => !!e));
|
package/dist/renderers.js
CHANGED
|
@@ -399,4 +399,4 @@ export function registerBuiltinRenderers() {
|
|
|
399
399
|
}
|
|
400
400
|
}
|
|
401
401
|
// ── Named exports for testing ────────────────────────────────────────────────
|
|
402
|
-
export {
|
|
402
|
+
export { agentMdRenderer, commandMdRenderer, INTERPRETER_MAP, knowledgeMdRenderer, memoryMdRenderer, SETUP_SIGNALS, scriptSourceRenderer, skillMdRenderer, };
|
package/dist/setup.js
CHANGED
|
@@ -6,8 +6,11 @@
|
|
|
6
6
|
* Collects all choices and writes config once at the end.
|
|
7
7
|
*/
|
|
8
8
|
import * as p from "@clack/prompts";
|
|
9
|
+
import { isHttpUrl } from "./common";
|
|
9
10
|
import { DEFAULT_CONFIG, getConfigPath, loadConfig, saveConfig } from "./config";
|
|
11
|
+
import { closeDatabase, isVecAvailable, openDatabase } from "./db";
|
|
10
12
|
import { detectAgentPlatforms, detectOllama, detectOpenViking } from "./detect";
|
|
13
|
+
import { checkEmbeddingAvailability, DEFAULT_LOCAL_MODEL, isTransformersAvailable } from "./embedder";
|
|
11
14
|
import { akmIndex } from "./indexer";
|
|
12
15
|
import { akmInit } from "./init";
|
|
13
16
|
import { getDefaultStashDir } from "./paths";
|
|
@@ -20,6 +23,11 @@ const RECOMMENDED_GITHUB_REPOS = [
|
|
|
20
23
|
hint: "community knowledge",
|
|
21
24
|
},
|
|
22
25
|
];
|
|
26
|
+
// Approximate first-download sizes used in the setup note.
|
|
27
|
+
// LOCAL_MODEL_APPROX_SIZE_MB tracks the default local model (DEFAULT_LOCAL_MODEL).
|
|
28
|
+
const LOCAL_MODEL_APPROX_SIZE_MB = 130;
|
|
29
|
+
// SQLITE_VEC_APPROX_SIZE_MB reflects the optional sqlite-vec install footprint.
|
|
30
|
+
const SQLITE_VEC_APPROX_SIZE_MB = 5;
|
|
23
31
|
// ── Helpers ─────────────────────────────────────────────────────────────────
|
|
24
32
|
function bail() {
|
|
25
33
|
p.cancel("Setup cancelled. No changes were saved.");
|
|
@@ -70,6 +78,112 @@ async function promptOrBack(fn) {
|
|
|
70
78
|
return null;
|
|
71
79
|
return result;
|
|
72
80
|
}
|
|
81
|
+
function isRemoteEmbeddingConfig(embedding) {
|
|
82
|
+
return isHttpUrl(embedding?.endpoint);
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* @internal Exported for testing only.
|
|
86
|
+
*/
|
|
87
|
+
export function describeSemanticSearchAssets(embedding) {
|
|
88
|
+
if (isRemoteEmbeddingConfig(embedding)) {
|
|
89
|
+
return [
|
|
90
|
+
`• Embedding endpoint: ${embedding?.provider ?? "custom"} / ${embedding?.model} (no local model download)`,
|
|
91
|
+
`• sqlite-vec acceleration: optional native extension (~${SQLITE_VEC_APPROX_SIZE_MB} MB when installed separately)`,
|
|
92
|
+
];
|
|
93
|
+
}
|
|
94
|
+
return [
|
|
95
|
+
`• Local embedding model: ${embedding?.localModel ?? DEFAULT_LOCAL_MODEL} (~${LOCAL_MODEL_APPROX_SIZE_MB} MB download on first use)`,
|
|
96
|
+
`• sqlite-vec acceleration: optional native extension (~${SQLITE_VEC_APPROX_SIZE_MB} MB when installed separately)`,
|
|
97
|
+
];
|
|
98
|
+
}
|
|
99
|
+
export async function stepSemanticSearch(current, embedding) {
|
|
100
|
+
const enabled = await prompt(() => p.confirm({
|
|
101
|
+
message: "Enable semantic search?",
|
|
102
|
+
initialValue: current.semanticSearch,
|
|
103
|
+
}));
|
|
104
|
+
if (!enabled) {
|
|
105
|
+
return { enabled: false, prepareAssets: false };
|
|
106
|
+
}
|
|
107
|
+
p.note(describeSemanticSearchAssets(embedding).join("\n"), "Semantic Search Assets");
|
|
108
|
+
const prepareAssets = await prompt(() => p.confirm({
|
|
109
|
+
message: isRemoteEmbeddingConfig(embedding)
|
|
110
|
+
? "Check the embedding endpoint and verify semantic search now?"
|
|
111
|
+
: "Download and verify semantic-search assets now?",
|
|
112
|
+
initialValue: true,
|
|
113
|
+
}));
|
|
114
|
+
return { enabled: true, prepareAssets };
|
|
115
|
+
}
|
|
116
|
+
async function prepareSemanticSearchAssets(config) {
|
|
117
|
+
const remote = isRemoteEmbeddingConfig(config.embedding);
|
|
118
|
+
// For local embeddings, ensure the required package is installed first.
|
|
119
|
+
if (!remote) {
|
|
120
|
+
if (!(await isTransformersAvailable())) {
|
|
121
|
+
const spin = p.spinner();
|
|
122
|
+
spin.start("Installing @huggingface/transformers...");
|
|
123
|
+
try {
|
|
124
|
+
const proc = Bun.spawn(["bun", "add", "@huggingface/transformers"], {
|
|
125
|
+
stdout: "pipe",
|
|
126
|
+
stderr: "pipe",
|
|
127
|
+
});
|
|
128
|
+
await proc.exited;
|
|
129
|
+
if (proc.exitCode !== 0) {
|
|
130
|
+
const stderr = await new Response(proc.stderr).text();
|
|
131
|
+
throw new Error(stderr || `exit code ${proc.exitCode}`);
|
|
132
|
+
}
|
|
133
|
+
spin.stop("@huggingface/transformers installed.");
|
|
134
|
+
}
|
|
135
|
+
catch (err) {
|
|
136
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
137
|
+
spin.stop("Could not install @huggingface/transformers.");
|
|
138
|
+
p.log.warn(`Automatic install failed: ${msg}\n` +
|
|
139
|
+
"Install it manually with: bun add @huggingface/transformers\n" +
|
|
140
|
+
"Then re-run `akm setup` or `akm index --full --verbose`.");
|
|
141
|
+
return false;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
const spin = p.spinner();
|
|
146
|
+
spin.start(remote
|
|
147
|
+
? "Checking remote embedding endpoint..."
|
|
148
|
+
: `Downloading local embedding model (${config.embedding?.localModel ?? DEFAULT_LOCAL_MODEL})...`);
|
|
149
|
+
const result = await checkEmbeddingAvailability(config.embedding);
|
|
150
|
+
if (!result.available) {
|
|
151
|
+
spin.stop("Semantic-search assets could not be prepared.");
|
|
152
|
+
if (result.reason === "remote-unreachable") {
|
|
153
|
+
p.log.warn("The remote embedding endpoint is not reachable. Check your endpoint and credentials, then retry `akm index --full --verbose`.");
|
|
154
|
+
}
|
|
155
|
+
else if (result.reason === "missing-package") {
|
|
156
|
+
p.log.warn("@huggingface/transformers is not installed. Install it with: bun add @huggingface/transformers\n" +
|
|
157
|
+
"Then re-run `akm setup` or `akm index --full --verbose`.");
|
|
158
|
+
}
|
|
159
|
+
else {
|
|
160
|
+
p.log.warn(`The local embedding model could not be downloaded: ${result.message}\n` +
|
|
161
|
+
"Retry `akm index --full --verbose` after confirming local model downloads are permitted.");
|
|
162
|
+
}
|
|
163
|
+
return false;
|
|
164
|
+
}
|
|
165
|
+
spin.stop(remote ? "Remote embedding endpoint is ready." : "Local embedding model downloaded and ready.");
|
|
166
|
+
let db;
|
|
167
|
+
try {
|
|
168
|
+
db = openDatabase();
|
|
169
|
+
if (isVecAvailable(db)) {
|
|
170
|
+
p.log.info("sqlite-vec is available for fast vector search.");
|
|
171
|
+
}
|
|
172
|
+
else {
|
|
173
|
+
p.log.info("sqlite-vec is not available. Semantic search will use the JS fallback until the optional extension is installed.");
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
catch (error) {
|
|
177
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
178
|
+
p.log.warn(`Could not open the local database or check for sqlite-vec. Semantic search will use the JS fallback. (${message})\n` +
|
|
179
|
+
"Check file permissions and available disk space in the cache directory, or run `akm index --full --verbose` to diagnose.");
|
|
180
|
+
}
|
|
181
|
+
finally {
|
|
182
|
+
if (db)
|
|
183
|
+
closeDatabase(db);
|
|
184
|
+
}
|
|
185
|
+
return true;
|
|
186
|
+
}
|
|
73
187
|
// ── Steps ───────────────────────────────────────────────────────────────────
|
|
74
188
|
async function stepStashDir(current) {
|
|
75
189
|
const defaultDir = current.stashDir ?? getDefaultStashDir();
|
|
@@ -431,14 +545,17 @@ export async function runSetupWizard() {
|
|
|
431
545
|
// Step 2: Ollama / Embedding / LLM
|
|
432
546
|
p.log.step("Step 2: Embedding & LLM");
|
|
433
547
|
const { embedding, llm } = await stepOllama(current);
|
|
434
|
-
// Step 3:
|
|
435
|
-
p.log.step("Step 3:
|
|
548
|
+
// Step 3: Semantic search assets
|
|
549
|
+
p.log.step("Step 3: Semantic Search");
|
|
550
|
+
const semanticSearch = await stepSemanticSearch(current, embedding);
|
|
551
|
+
// Step 4: Registries
|
|
552
|
+
p.log.step("Step 4: Registries");
|
|
436
553
|
const registries = await stepRegistries(current);
|
|
437
|
-
// Step
|
|
438
|
-
p.log.step("Step
|
|
554
|
+
// Step 5: Stash sources
|
|
555
|
+
p.log.step("Step 5: Stash Sources");
|
|
439
556
|
const stashes = await stepStashSources(current);
|
|
440
|
-
// Step
|
|
441
|
-
p.log.step("Step
|
|
557
|
+
// Step 6: Agent platform detection
|
|
558
|
+
p.log.step("Step 6: Agent Platform Detection");
|
|
442
559
|
const platformStashes = await stepAgentPlatforms(current);
|
|
443
560
|
// Merge platform stashes into main stashes list
|
|
444
561
|
const allStashes = [...stashes];
|
|
@@ -456,7 +573,7 @@ export async function runSetupWizard() {
|
|
|
456
573
|
registries,
|
|
457
574
|
stashes: allStashes.length > 0 ? allStashes : undefined,
|
|
458
575
|
// Preserve existing fields
|
|
459
|
-
semanticSearch:
|
|
576
|
+
semanticSearch: semanticSearch.enabled,
|
|
460
577
|
installed: current.installed,
|
|
461
578
|
output: current.output,
|
|
462
579
|
};
|
|
@@ -466,6 +583,7 @@ export async function runSetupWizard() {
|
|
|
466
583
|
`Stash directory: ${stashDir}`,
|
|
467
584
|
`Embedding: ${embedding ? `${embedding.provider ?? "remote"} / ${embedding.model}` : "built-in local"}`,
|
|
468
585
|
`LLM: ${llm ? `${llm.provider ?? "remote"} / ${llm.model}` : "disabled"}`,
|
|
586
|
+
`Semantic search: ${semanticSearch.enabled ? "enabled" : "disabled"}`,
|
|
469
587
|
`Registries: ${effectiveRegistries.filter((r) => r.enabled !== false).length} enabled`,
|
|
470
588
|
`Stash sources: ${allStashes.length}`,
|
|
471
589
|
].join("\n"), "Configuration Summary");
|
|
@@ -479,12 +597,38 @@ export async function runSetupWizard() {
|
|
|
479
597
|
saveConfig(newConfig);
|
|
480
598
|
// Initialize stash directory
|
|
481
599
|
await akmInit({ dir: stashDir });
|
|
600
|
+
if (semanticSearch.enabled) {
|
|
601
|
+
if (semanticSearch.prepareAssets) {
|
|
602
|
+
const ready = await prepareSemanticSearchAssets(newConfig);
|
|
603
|
+
if (!ready) {
|
|
604
|
+
// Asset preparation failed: disable semantic search and persist the update.
|
|
605
|
+
newConfig.semanticSearch = false;
|
|
606
|
+
saveConfig(newConfig);
|
|
607
|
+
p.log.warn("Semantic search has been disabled in the saved configuration. Re-run `akm setup` or `akm index --full --verbose` once the issue is resolved.");
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
else {
|
|
611
|
+
p.log.info("Semantic search will be enabled, but asset preparation was skipped. Run `akm index --full --verbose` later to verify it.");
|
|
612
|
+
}
|
|
613
|
+
}
|
|
482
614
|
// Build search index
|
|
615
|
+
p.log.info("Building search index...");
|
|
483
616
|
const spin = p.spinner();
|
|
484
617
|
spin.start("Building search index...");
|
|
485
618
|
try {
|
|
486
619
|
const indexResult = await akmIndex({ stashDir });
|
|
487
620
|
spin.stop(`Indexed ${indexResult.totalEntries} assets.`);
|
|
621
|
+
if (newConfig.semanticSearch) {
|
|
622
|
+
if (indexResult.verification.ok) {
|
|
623
|
+
p.log.success(indexResult.verification.message);
|
|
624
|
+
}
|
|
625
|
+
else {
|
|
626
|
+
p.log.warn(indexResult.verification.message);
|
|
627
|
+
if (indexResult.verification.guidance) {
|
|
628
|
+
p.log.info(indexResult.verification.guidance);
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
}
|
|
488
632
|
}
|
|
489
633
|
catch (err) {
|
|
490
634
|
spin.stop("Indexing failed — you can run `akm index` manually later.");
|
|
@@ -137,4 +137,4 @@ function parseGitRepoUrl(rawUrl) {
|
|
|
137
137
|
};
|
|
138
138
|
}
|
|
139
139
|
// ── Exports ─────────────────────────────────────────────────────────────────
|
|
140
|
-
export {
|
|
140
|
+
export { ensureGitMirror, GitStashProvider, getCachePaths, parseGitRepoUrl };
|
|
@@ -345,4 +345,4 @@ function inferTypeFromUri(uri) {
|
|
|
345
345
|
return OV_TYPE_MAP[firstSegment] ?? "knowledge";
|
|
346
346
|
}
|
|
347
347
|
// ── Exports for testing ─────────────────────────────────────────────────────
|
|
348
|
-
export { OpenVikingStashProvider,
|
|
348
|
+
export { OpenVikingStashProvider, parseOVSearchResponse, refToVikingUri };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "akm-cli",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.1",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "CLI tool to search, open, and run extension assets from an akm stash directory.",
|
|
6
6
|
"keywords": [
|
|
@@ -51,7 +51,6 @@
|
|
|
51
51
|
"typescript": "^5.9.3"
|
|
52
52
|
},
|
|
53
53
|
"optionalDependencies": {
|
|
54
|
-
"@xenova/transformers": "^2.17.0",
|
|
55
54
|
"sqlite-vec": "0.1.7-alpha.2"
|
|
56
55
|
},
|
|
57
56
|
"engines": {
|
|
@@ -59,6 +58,7 @@
|
|
|
59
58
|
},
|
|
60
59
|
"dependencies": {
|
|
61
60
|
"@clack/prompts": "^1.1.0",
|
|
61
|
+
"@huggingface/transformers": "^3.8.1",
|
|
62
62
|
"citty": "^0.2.1",
|
|
63
63
|
"yaml": "^2.8.2"
|
|
64
64
|
}
|