akm-cli 0.9.1-beta.1 → 0.9.1-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -1
- package/dist/cli/parse-args.js +7 -1
- package/dist/commands/env/child-env.js +14 -0
- package/dist/commands/health/advisories.js +5 -5
- package/dist/commands/health/html-report.js +2 -2
- package/dist/commands/health/metrics.js +38 -22
- package/dist/commands/health/report-view-model.js +1 -1
- package/dist/commands/improve/consolidate.js +61 -9
- package/dist/commands/improve/eval-cases.js +2 -0
- package/dist/commands/improve/memory/memory-improve.js +1 -0
- package/dist/commands/lint/base-linter.js +93 -20
- package/dist/commands/lint/index.js +5 -1
- package/dist/commands/sources/add-cli.js +8 -2
- package/dist/commands/sources/migration-help.js +12 -3
- package/dist/commands/sources/self-update.js +9 -1
- package/dist/core/adapter/adapters/agent-skills-adapter.js +32 -16
- package/dist/core/adapter/adapters/akm-lint.js +6 -2
- package/dist/core/adapter/adapters/akm-task-adapter.js +4 -2
- package/dist/core/adapter/adapters/dotenv-adapter.js +21 -0
- package/dist/core/asset/frontmatter.js +6 -1
- package/dist/core/common.js +81 -3
- package/dist/core/config/config-io.js +5 -45
- package/dist/core/config/schema/engines.js +14 -3
- package/dist/core/extra-params.js +11 -0
- package/dist/core/fs-txn.js +15 -2
- package/dist/core/json-schema.js +19 -2
- package/dist/core/paths.js +16 -2
- package/dist/core/redaction.js +22 -1
- package/dist/core/state-db.js +1 -0
- package/dist/core/write-source.js +26 -2
- package/dist/indexer/indexer.js +48 -9
- package/dist/indexer/search/db-search.js +17 -2
- package/dist/indexer/walk/walker.js +6 -1
- package/dist/integrations/agent/detect.js +13 -1
- package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +21 -0
- package/dist/integrations/lockfile.js +10 -0
- package/dist/llm/client.js +14 -19
- package/dist/llm/embedder.js +23 -3
- package/dist/llm/embedders/remote.js +27 -2
- package/dist/output/html-render.js +40 -1
- package/dist/runtime.js +23 -1
- package/dist/scripts/akm-migrate-node.js +303 -107
- package/dist/scripts/akm-migrate.js +303 -107
- package/dist/setup/setup.js +22 -7
- package/dist/sources/providers/git-install.js +25 -2
- package/dist/sources/snapshot-fetchers/content-extract.js +63 -1
- package/dist/storage/database.js +71 -12
- package/dist/storage/engines/sqlite-migrations.js +61 -2
- package/dist/storage/repositories/index-connection.js +11 -1
- package/dist/storage/repositories/index-meta-repository.js +11 -0
- package/dist/storage/repositories/index-schema.js +17 -2
- package/dist/storage/repositories/index-vec-repository.js +43 -5
- package/dist/storage/repositories/salience-repository.js +13 -12
- package/dist/storage/sqlite-pragmas.js +12 -1
- package/dist/tasks/runner.js +84 -7
- package/dist/tasks/scheduler-invocation.js +19 -0
- package/dist/tasks/schema.js +21 -1
- package/dist/text-import-hook.mjs +1 -1
- package/dist/workflows/exec/native-executor.js +8 -0
- package/dist/workflows/exec/step-work.js +10 -2
- package/dist/workflows/parser.js +26 -1
- package/package.json +1 -1
- package/schemas/akm-config.json +10 -5
- package/schemas/akm-workflow.json +7 -3
package/dist/indexer/indexer.js
CHANGED
|
@@ -19,7 +19,7 @@ import { closeDatabase, openExistingDatabase, openIndexDatabase } from "../stora
|
|
|
19
19
|
import { deleteEntriesByDirAndStash, deleteEntriesByDirExceptKeys, deleteEntriesByIds, deleteEntriesByStashDir, deleteUsageEventsByEntryIds, findEntryIdByRef, getAllEntries, getEmbeddableEntryCount, getEntryCount, getIndexedDirPathsByStashDir, getIndexedStashDirsByDir, relinkUsageEvents, upsertEntry, upsertWorkflowDocument, } from "../storage/repositories/index-entries-repository.js";
|
|
20
20
|
import { rebuildFts } from "../storage/repositories/index-fts-repository.js";
|
|
21
21
|
import { clearStaleCacheEntries } from "../storage/repositories/index-llm-cache-repository.js";
|
|
22
|
-
import { deleteIndexDirState, deleteIndexDirStatesByStashDir, getMeta, setMeta, upsertIndexDirState, } from "../storage/repositories/index-meta-repository.js";
|
|
22
|
+
import { deleteIndexDirState, deleteIndexDirStatesByStashDir, deleteMeta, getMeta, setMeta, upsertIndexDirState, } from "../storage/repositories/index-meta-repository.js";
|
|
23
23
|
import { upsertUtilityScore } from "../storage/repositories/index-utility-repository.js";
|
|
24
24
|
import { getAllEntriesForEmbedding, getEmbeddingCount, isVecAvailable, isVecFastPathReady, purgeEmbeddings, setVecFastPathReady, upsertEmbedding, warnIfVecMissing, } from "../storage/repositories/index-vec-repository.js";
|
|
25
25
|
import { takeWorkflowDocument } from "../workflows/runtime/document-cache.js";
|
|
@@ -158,7 +158,12 @@ async function runWalkPhase(ctx) {
|
|
|
158
158
|
async function runEmbeddingPhase(ctx) {
|
|
159
159
|
const { db, config, signal, onProgress } = ctx;
|
|
160
160
|
throwIfAborted(signal);
|
|
161
|
-
|
|
161
|
+
// Forward the signal. Without it generateEmbeddingsForDb's abort machinery was
|
|
162
|
+
// inert — its throwIfAborted checks and the signal it threads into embedBatch
|
|
163
|
+
// (which RemoteEmbedder passes to every fetch and LocalEmbedder honours between
|
|
164
|
+
// chunks) never saw a controller. Ctrl-C and the improve budget abort could not
|
|
165
|
+
// stop the embedding phase, the longest phase of an index run.
|
|
166
|
+
ctx.embeddingResult = await generateEmbeddingsForDb(db, config, onProgress, signal);
|
|
162
167
|
ctx.timing.tEmbedEnd = Date.now();
|
|
163
168
|
}
|
|
164
169
|
/**
|
|
@@ -1357,9 +1362,20 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal) {
|
|
|
1357
1362
|
const currentFingerprint = deriveSemanticProviderFingerprint(config.embedding);
|
|
1358
1363
|
const storedFingerprint = getMeta(db, "embeddingFingerprint");
|
|
1359
1364
|
if (storedFingerprint && storedFingerprint !== currentFingerprint) {
|
|
1360
|
-
// Model/provider changed → stored vectors are incompatible. Clear them
|
|
1361
|
-
//
|
|
1362
|
-
|
|
1365
|
+
// Model/provider changed → stored vectors are incompatible. Clear them;
|
|
1366
|
+
// re-embedded by this index run.
|
|
1367
|
+
//
|
|
1368
|
+
// The vec table goes too. "Same dimension, so keep the vec table" only held
|
|
1369
|
+
// for a same-width model swap: entries_vec is a vec0 virtual table declared
|
|
1370
|
+
// at a FIXED width, so after a dimension-changing model change every insert
|
|
1371
|
+
// failed against the old width, and ensureSchema's dim-change rebuild never
|
|
1372
|
+
// fired because it only runs for callers that pass an explicit
|
|
1373
|
+
// embeddingDim. The stale table survived `--full` — the exact remedy the
|
|
1374
|
+
// warning recommended. Clearing the stored dim lets the next ensureSchema
|
|
1375
|
+
// materialize it at the new width; until then the fast-path flag reads
|
|
1376
|
+
// false (no table) and search uses the complete BLOB table.
|
|
1377
|
+
purgeEmbeddings(db, { dropVecTable: true });
|
|
1378
|
+
deleteMeta(db, "embeddingDim");
|
|
1363
1379
|
}
|
|
1364
1380
|
try {
|
|
1365
1381
|
const { embedBatch } = await import("../llm/embedder.js");
|
|
@@ -1404,6 +1420,7 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal) {
|
|
|
1404
1420
|
let storedCount = 0;
|
|
1405
1421
|
let skippedCount = 0;
|
|
1406
1422
|
let vecFailedCount = 0;
|
|
1423
|
+
let vecUnavailableCount = 0;
|
|
1407
1424
|
db.transaction(() => {
|
|
1408
1425
|
for (let i = 0; i < allEntries.length; i++) {
|
|
1409
1426
|
const res = upsertEmbedding(db, allEntries[i].id, embeddings[i]);
|
|
@@ -1415,6 +1432,8 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal) {
|
|
|
1415
1432
|
}
|
|
1416
1433
|
if (res.vec === "failed")
|
|
1417
1434
|
vecFailedCount++;
|
|
1435
|
+
if (res.vec === "unavailable")
|
|
1436
|
+
vecUnavailableCount++;
|
|
1418
1437
|
}
|
|
1419
1438
|
})();
|
|
1420
1439
|
if (skippedCount > 0) {
|
|
@@ -1424,7 +1443,13 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal) {
|
|
|
1424
1443
|
// instead of inferring readiness from stored-BLOB counts. Any failure
|
|
1425
1444
|
// marks the fast path degraded, routing search to the JS-cosine fallback
|
|
1426
1445
|
// over the (complete) BLOB table — honest degradation, not a hard failure.
|
|
1427
|
-
|
|
1446
|
+
//
|
|
1447
|
+
// 'unavailable' has to degrade the flag too. It means no vec row was
|
|
1448
|
+
// written at all, so marking the fast path ready left a later open (a
|
|
1449
|
+
// different runtime, or sqlite-vec installed afterwards) trusting an
|
|
1450
|
+
// empty entries_vec and returning zero semantic hits against a fully
|
|
1451
|
+
// populated BLOB table.
|
|
1452
|
+
setVecFastPathReady(db, vecFailedCount === 0 && vecUnavailableCount === 0);
|
|
1428
1453
|
if (vecFailedCount > 0) {
|
|
1429
1454
|
warn(`[embed] ${vecFailedCount} sqlite-vec fast-path insert${vecFailedCount === 1 ? "" : "s"} failed — ` +
|
|
1430
1455
|
"semantic search will use the slower JS-cosine fallback over stored embeddings. " +
|
|
@@ -1799,7 +1824,12 @@ export function recomputeUtilityScores(db, stateDb) {
|
|
|
1799
1824
|
SUM(CASE WHEN u.event_type = 'show' THEN 1 ELSE 0 END) AS show_count,
|
|
1800
1825
|
SUM(CASE WHEN u.event_type = 'feedback' AND u.signal = 'positive' THEN 1 ELSE 0 END) AS positive_feedback_count,
|
|
1801
1826
|
SUM(CASE WHEN u.event_type = 'feedback' AND u.signal = 'negative' THEN 1 ELSE 0 END) AS negative_feedback_count,
|
|
1802
|
-
MAX(
|
|
1827
|
+
MAX(
|
|
1828
|
+
CASE
|
|
1829
|
+
WHEN u.event_type IN ('search', 'show', 'curate') THEN u.created_at
|
|
1830
|
+
ELSE NULL
|
|
1831
|
+
END
|
|
1832
|
+
) AS last_used_at
|
|
1803
1833
|
FROM usage_events u
|
|
1804
1834
|
WHERE u.entry_id IS NOT NULL
|
|
1805
1835
|
AND u.source = 'user'
|
|
@@ -1814,7 +1844,6 @@ export function recomputeUtilityScores(db, stateDb) {
|
|
|
1814
1844
|
for (const row of scoreRows) {
|
|
1815
1845
|
existingScores.set(row.entry_id, { utility: row.utility, lastUsedAt: row.last_used_at ?? undefined });
|
|
1816
1846
|
}
|
|
1817
|
-
const now = new Date().toISOString();
|
|
1818
1847
|
const entryIds = new Set([...existingScores.keys(), ...usageByEntry.keys()]);
|
|
1819
1848
|
for (const entryId of entryIds) {
|
|
1820
1849
|
const row = usageByEntry.get(entryId) ?? {
|
|
@@ -1832,7 +1861,17 @@ export function recomputeUtilityScores(db, stateDb) {
|
|
|
1832
1861
|
const existing = existingScores.get(row.entry_id);
|
|
1833
1862
|
const prevUtility = existing?.utility ?? 0;
|
|
1834
1863
|
const utility = prevUtility * emaDecay + effectiveRate * emaNew;
|
|
1835
|
-
|
|
1864
|
+
// `utility_scores.last_used_at` is consumed by salience as the timestamp of
|
|
1865
|
+
// the most-recent retrieval. Preserve that meaning by carrying the event's
|
|
1866
|
+
// timestamp through verbatim. The former `effectiveRate > 0.5 ? now : ...`
|
|
1867
|
+
// branch stamped every high-select-rate entry with the index run time,
|
|
1868
|
+
// making unrelated assets look simultaneously fresh and flattening the
|
|
1869
|
+
// recency component of retrieval salience.
|
|
1870
|
+
//
|
|
1871
|
+
// `usage_events` is the source of truth within its retention window. A
|
|
1872
|
+
// missing row therefore clears legacy/index-time stamps on the next index
|
|
1873
|
+
// pass; salience already treats an absent timestamp as long ago.
|
|
1874
|
+
const lastUsedAt = row.last_used_at ?? undefined;
|
|
1836
1875
|
upsertUtilityScore(db, row.entry_id, {
|
|
1837
1876
|
utility,
|
|
1838
1877
|
showCount: row.show_count,
|
|
@@ -86,6 +86,21 @@ export function shouldQueryPositiveFeedbackCounts(utilityDecayRaw) {
|
|
|
86
86
|
return boost > 1.0;
|
|
87
87
|
}
|
|
88
88
|
// ── Main search entrypoint ───────────────────────────────────────────────────
|
|
89
|
+
/**
|
|
90
|
+
* Whether an embedding provider is actually configured.
|
|
91
|
+
*
|
|
92
|
+
* A remote provider needs BOTH endpoint and model. A LOCAL provider needs
|
|
93
|
+
* neither — `embedding.localModel` selects a transformers model that runs in
|
|
94
|
+
* process. Checking only the remote pair told every local-provider user that
|
|
95
|
+
* "no embedding provider is configured" and pointed them at
|
|
96
|
+
* `akm config set embedding '{"endpoint":...}'`, which is the wrong remedy and
|
|
97
|
+
* hid the real diagnostic recorded in the semantic status.
|
|
98
|
+
*/
|
|
99
|
+
function hasConfiguredEmbeddingProvider(config) {
|
|
100
|
+
if (config.embedding?.localModel)
|
|
101
|
+
return true;
|
|
102
|
+
return Boolean(config.embedding?.endpoint && config.embedding?.model);
|
|
103
|
+
}
|
|
89
104
|
export async function searchLocal(input) {
|
|
90
105
|
const { query, searchType, limit, stashDir, sources, config } = input;
|
|
91
106
|
const filters = input.filters;
|
|
@@ -105,7 +120,7 @@ export async function searchLocal(input) {
|
|
|
105
120
|
if (rawStatus && rawStatus.providerFingerprint !== currentFingerprint) {
|
|
106
121
|
warnings.push("Embedding config changed. Run 'akm index --full' to rebuild the semantic index with the new provider.");
|
|
107
122
|
}
|
|
108
|
-
else if (!config
|
|
123
|
+
else if (!hasConfiguredEmbeddingProvider(config)) {
|
|
109
124
|
// #480: when semantic mode is `auto` but no embedding provider is
|
|
110
125
|
// configured (e.g. `akm setup --yes` ran without picking one), telling
|
|
111
126
|
// the user to "run akm setup" is misleading — they just did. Surface
|
|
@@ -120,7 +135,7 @@ export async function searchLocal(input) {
|
|
|
120
135
|
}
|
|
121
136
|
}
|
|
122
137
|
if (config.semanticSearchMode === "auto" && semanticStatus === "blocked") {
|
|
123
|
-
if (!config
|
|
138
|
+
if (!hasConfiguredEmbeddingProvider(config)) {
|
|
124
139
|
// F7/A2: same predicate as the `pending` branch above (#480) — a
|
|
125
140
|
// `blocked` status can outlive the provider config that produced it
|
|
126
141
|
// (e.g. the embedding config was later unset). This is not a fault;
|
|
@@ -123,7 +123,12 @@ function walkStashGit(stashRoot, options) {
|
|
|
123
123
|
for (const relFile of files) {
|
|
124
124
|
const absPath = path.join(stashRoot, relFile);
|
|
125
125
|
try {
|
|
126
|
-
|
|
126
|
+
// lstat, not stat: a tracked symlink must not be dereferenced. statSync
|
|
127
|
+
// follows the link, so a target outside stashRoot would be read and
|
|
128
|
+
// indexed. The manual walk below already skips symlinks for exactly this
|
|
129
|
+
// reason; the two walkers have to agree regardless of whether the stash
|
|
130
|
+
// happens to sit inside a git repo.
|
|
131
|
+
if (fs.lstatSync(absPath).isFile()) {
|
|
127
132
|
results.push(buildFileContext(stashRoot, absPath));
|
|
128
133
|
}
|
|
129
134
|
}
|
|
@@ -74,6 +74,16 @@ const DEFAULT_PATHEXT = ".COM;.EXE;.BAT;.CMD";
|
|
|
74
74
|
* on win32 or when the env supplies PATHEXT (which is also the seam tests use
|
|
75
75
|
* to cover Windows resolution from a POSIX runner).
|
|
76
76
|
*/
|
|
77
|
+
/**
|
|
78
|
+
* Script shims that `child_process.spawn` cannot execute directly on Windows.
|
|
79
|
+
*
|
|
80
|
+
* Node refuses to spawn `.bat`/`.cmd` without a shell (the CVE-2024-27980
|
|
81
|
+
* hardening), so a detection that returns one of these reports an agent as
|
|
82
|
+
* available that dispatch then fails to launch. They stay in the candidate
|
|
83
|
+
* list — a shim is better than reporting the CLI as missing — but they sort
|
|
84
|
+
* last, so a real `.exe` next to `claude.cmd` wins.
|
|
85
|
+
*/
|
|
86
|
+
const NON_SPAWNABLE_SUFFIXES = new Set([".cmd", ".bat"]);
|
|
77
87
|
function executableSuffixes(envSource) {
|
|
78
88
|
const pathext = envSource.PATHEXT ?? envSource.Pathext ?? envSource.pathext;
|
|
79
89
|
if (process.platform !== "win32" && !pathext)
|
|
@@ -83,7 +93,9 @@ function executableSuffixes(envSource) {
|
|
|
83
93
|
.map((ext) => ext.trim())
|
|
84
94
|
.filter(Boolean)
|
|
85
95
|
.map((ext) => (ext.startsWith(".") ? ext : `.${ext}`));
|
|
86
|
-
|
|
96
|
+
const directlySpawnable = exts.filter((ext) => !NON_SPAWNABLE_SUFFIXES.has(ext.toLowerCase()));
|
|
97
|
+
const shims = exts.filter((ext) => NON_SPAWNABLE_SUFFIXES.has(ext.toLowerCase()));
|
|
98
|
+
return ["", ...directlySpawnable, ...shims];
|
|
87
99
|
}
|
|
88
100
|
let detectOverrides;
|
|
89
101
|
/** TEST-ONLY. Swap the detection implementations; pass undefined to restore. */
|
|
@@ -118,6 +118,12 @@ let _testServer = null;
|
|
|
118
118
|
// Test seam replacing the real `createOpencode` import (see __setServerFactory).
|
|
119
119
|
let _serverFactory = null;
|
|
120
120
|
let _exitHookInstalled = false;
|
|
121
|
+
/**
|
|
122
|
+
* True once `process.on("exit")` has fired. Close paths consult it because an
|
|
123
|
+
* exit handler runs after the event loop has stopped: no timer scheduled there
|
|
124
|
+
* will ever fire, so any cleanup that needs to happen must happen inline.
|
|
125
|
+
*/
|
|
126
|
+
let _processExiting = false;
|
|
121
127
|
/**
|
|
122
128
|
* Test-only seam: inject a fake {@link SdkServer} so `runOpencodeSdk` can be
|
|
123
129
|
* exercised without the real `@opencode-ai/sdk` (which would spin up a server).
|
|
@@ -424,6 +430,20 @@ async function createManagedOpencode(options) {
|
|
|
424
430
|
}
|
|
425
431
|
if (childExited())
|
|
426
432
|
return;
|
|
433
|
+
if (_processExiting) {
|
|
434
|
+
// During `process.on("exit")` the event loop is finished, so a timer
|
|
435
|
+
// scheduled here would never fire and the SIGKILL escalation could not
|
|
436
|
+
// happen at all — a server child that ignores SIGTERM outlived akm.
|
|
437
|
+
// Exit handlers must be synchronous, so escalate immediately instead of
|
|
438
|
+
// waiting out a grace period we have no way to wait for.
|
|
439
|
+
try {
|
|
440
|
+
proc.kill("SIGKILL");
|
|
441
|
+
}
|
|
442
|
+
catch {
|
|
443
|
+
/* already dead */
|
|
444
|
+
}
|
|
445
|
+
return;
|
|
446
|
+
}
|
|
427
447
|
closeEscalation = setTimeout(() => {
|
|
428
448
|
closeEscalation = undefined;
|
|
429
449
|
if (childExited())
|
|
@@ -525,6 +545,7 @@ async function startServer(profile, sdkConfig, env, registryKey, startupSignal)
|
|
|
525
545
|
if (!_exitHookInstalled) {
|
|
526
546
|
_exitHookInstalled = true;
|
|
527
547
|
process.once("exit", () => {
|
|
548
|
+
_processExiting = true;
|
|
528
549
|
void closeServer();
|
|
529
550
|
});
|
|
530
551
|
}
|
|
@@ -126,6 +126,16 @@ function readLockfileOrThrow() {
|
|
|
126
126
|
if (!Array.isArray(parsed)) {
|
|
127
127
|
throw new ConfigError(`Refusing to modify lockfile ${lockfilePath}: existing content is not a JSON array. Fix or remove the file by hand before retrying — every existing lock entry would otherwise be lost.`, "INVALID_CONFIG_FILE");
|
|
128
128
|
}
|
|
129
|
+
// Refuse rather than filter. This is the WRITE path's read: everything it
|
|
130
|
+
// returns is what gets written back, so silently dropping entries that fail
|
|
131
|
+
// per-entry validation destroyed them on the next write — the same
|
|
132
|
+
// data-losing overwrite the two refusals above exist to prevent, just at
|
|
133
|
+
// entry granularity instead of file granularity.
|
|
134
|
+
const invalid = parsed.filter((entry) => !isValidLockfileEntry(entry));
|
|
135
|
+
if (invalid.length > 0) {
|
|
136
|
+
throw new ConfigError(`Refusing to modify lockfile ${lockfilePath}: ${invalid.length} existing entr${invalid.length === 1 ? "y is" : "ies are"} malformed. ` +
|
|
137
|
+
"Fix or remove the file by hand before retrying — those entries would otherwise be lost.", "INVALID_CONFIG_FILE");
|
|
138
|
+
}
|
|
129
139
|
return parsed.filter(isValidLockfileEntry);
|
|
130
140
|
}
|
|
131
141
|
/**
|
package/dist/llm/client.js
CHANGED
|
@@ -9,34 +9,22 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import { fetchWithTimeout, readBodyWithByteCap } from "../core/common.js";
|
|
11
11
|
import { resolveSecret } from "../core/config/config.js";
|
|
12
|
+
import { ENV_REFERENCE_PATTERN } from "../core/config/schema/primitives.js";
|
|
12
13
|
import { formatExtraParamsIssue, validateExtraParams } from "../core/extra-params.js";
|
|
13
14
|
import { parseJsonResponse } from "../core/parse.js";
|
|
14
|
-
import {
|
|
15
|
+
import { redactErrorBody, redactSensitiveText } from "../core/redaction.js";
|
|
15
16
|
import { warnVerbose } from "../core/warn.js";
|
|
16
17
|
import { DEFAULT_LLM_TIMEOUT_MS } from "../integrations/agent/config.js";
|
|
17
18
|
import { emitLlmUsage, extractUsageTokens, } from "./usage-telemetry.js";
|
|
18
|
-
/** Maximum length of an
|
|
19
|
+
/** Maximum length of an upstream response excerpt included in thrown errors. */
|
|
19
20
|
const ERROR_BODY_MAX_LEN = 200;
|
|
20
21
|
/** Stable OpenAI-compatible response-schema name used for every structured call. */
|
|
21
22
|
const JSON_SCHEMA_RESPONSE_NAME = "akm_response";
|
|
22
23
|
/**
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
* so that a verbose provider response cannot leak large amounts of context.
|
|
26
|
-
*
|
|
27
|
-
* The pattern set itself lives in {@link redactCredentialPatterns}
|
|
28
|
-
* (src/core/redaction.ts) so other output paths (e.g. task run logs) can
|
|
29
|
-
* reuse it without this function's length cap.
|
|
24
|
+
* Re-exported from src/core/redaction.ts, where it now lives so every HTTP
|
|
25
|
+
* transport can apply the same hardening — the embeddings client needs it too.
|
|
30
26
|
*/
|
|
31
|
-
export
|
|
32
|
-
if (!input)
|
|
33
|
-
return "";
|
|
34
|
-
let out = redactCredentialPatterns(input);
|
|
35
|
-
if (out.length > ERROR_BODY_MAX_LEN) {
|
|
36
|
-
out = `${out.slice(0, ERROR_BODY_MAX_LEN)}…`;
|
|
37
|
-
}
|
|
38
|
-
return out;
|
|
39
|
-
}
|
|
27
|
+
export { redactErrorBody } from "../core/redaction.js";
|
|
40
28
|
/**
|
|
41
29
|
* Detect a response body that is an HTML document rather than the expected
|
|
42
30
|
* JSON. LM Studio (and similar local providers) can serve their web UI on
|
|
@@ -223,7 +211,14 @@ async function chatCompletionAttempt(config, messages, options, timeoutMs) {
|
|
|
223
211
|
throw new Error(formatExtraParamsIssue("LLM extraParams", issue));
|
|
224
212
|
}
|
|
225
213
|
const headers = { "Content-Type": "application/json" };
|
|
226
|
-
|
|
214
|
+
// Resolve ONLY a whole-string env reference. Every live caller already hands
|
|
215
|
+
// us the materialized credential (materializeLlmConnection / materializeFrozenLlm
|
|
216
|
+
// resolve `$VAR` upstream, and engine config REQUIRES the symbolic form), so
|
|
217
|
+
// re-running the substitution over a literal key mangled any credential
|
|
218
|
+
// containing `$` — `sk-live$ecret` lost everything from the `$` onward, and
|
|
219
|
+
// the request failed with an opaque 401. The narrow check keeps the symbolic
|
|
220
|
+
// form working for any direct caller that still passes one.
|
|
221
|
+
const resolvedKey = ENV_REFERENCE_PATTERN.test(config.apiKey ?? "") ? resolveSecret(config.apiKey) : config.apiKey;
|
|
227
222
|
if (resolvedKey) {
|
|
228
223
|
headers.Authorization = `Bearer ${resolvedKey}`;
|
|
229
224
|
}
|
package/dist/llm/embedder.js
CHANGED
|
@@ -65,12 +65,32 @@ export async function embed(text, embeddingConfig, signal) {
|
|
|
65
65
|
const cached = getCachedEmbedding(key);
|
|
66
66
|
if (cached)
|
|
67
67
|
return cached;
|
|
68
|
-
const result =
|
|
69
|
-
? await new RemoteEmbedder(embeddingConfig).embed(text, signal)
|
|
70
|
-
: await getLocalEmbedder().embed(text, signal);
|
|
68
|
+
const result = await embedOnce(text, embeddingConfig, signal);
|
|
71
69
|
setCachedEmbedding(key, result);
|
|
72
70
|
return result;
|
|
73
71
|
}
|
|
72
|
+
/**
|
|
73
|
+
* Resolve a single embedding through the configured provider.
|
|
74
|
+
*
|
|
75
|
+
* The local branch must honour `localModel` exactly as {@link embedBatch}
|
|
76
|
+
* does. The singleton is constructed with no default model, so routing through
|
|
77
|
+
* `getLocalEmbedder().embed()` silently used DEFAULT_LOCAL_MODEL: queries were
|
|
78
|
+
* embedded with a different model than the index was built with. Nothing
|
|
79
|
+
* detected it, because the provider fingerprint keys on `localModel`, so no
|
|
80
|
+
* purge or "pending" status ever fired — a dimension mismatch made semantic
|
|
81
|
+
* ranking contribute nothing, and a same-dimension override silently produced
|
|
82
|
+
* meaningless cross-model scores.
|
|
83
|
+
*/
|
|
84
|
+
async function embedOnce(text, embeddingConfig, signal) {
|
|
85
|
+
if (embeddingConfig && hasRemoteEndpoint(embeddingConfig)) {
|
|
86
|
+
return new RemoteEmbedder(embeddingConfig).embed(text, signal);
|
|
87
|
+
}
|
|
88
|
+
const localModel = embeddingConfig?.localModel;
|
|
89
|
+
if (localModel) {
|
|
90
|
+
return getLocalEmbedder().embedWithModel(text, localModel);
|
|
91
|
+
}
|
|
92
|
+
return getLocalEmbedder().embed(text, signal);
|
|
93
|
+
}
|
|
74
94
|
/**
|
|
75
95
|
* Generate embeddings for multiple texts in batch.
|
|
76
96
|
* Uses the OpenAI-compatible batch API for remote endpoints (batches of 100).
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import { fetchWithTimeout, isHttpUrl, readBodyWithByteCap } from "../../core/common.js";
|
|
11
11
|
import { resolveSecret } from "../../core/config/config.js";
|
|
12
|
+
import { redactErrorBody, redactSensitiveText } from "../../core/redaction.js";
|
|
12
13
|
const DEFAULT_REMOTE_BATCH_SIZE = 100;
|
|
13
14
|
/** Cheap token estimator: 4 chars ≈ 1 token. Used in verbose logging and error messages. */
|
|
14
15
|
export function estimateTokenCount(text) {
|
|
@@ -54,7 +55,7 @@ export class RemoteEmbedder {
|
|
|
54
55
|
throw err;
|
|
55
56
|
return "";
|
|
56
57
|
});
|
|
57
|
-
throw new Error(`Embedding request failed (${response.status}): ${errBody}`);
|
|
58
|
+
throw new Error(`Embedding request failed (${response.status}): ${this.safeErrorBody(errBody)}`);
|
|
58
59
|
}
|
|
59
60
|
const json = JSON.parse(await readBodyWithByteCap(response, undefined, { bodyTimeoutMs: 30_000, signal }));
|
|
60
61
|
if (!json.data?.[0]?.embedding) {
|
|
@@ -94,7 +95,7 @@ export class RemoteEmbedder {
|
|
|
94
95
|
throw err;
|
|
95
96
|
return "";
|
|
96
97
|
});
|
|
97
|
-
throw new Error(`Embedding batch request failed (${response.status}): ${respBody}`);
|
|
98
|
+
throw new Error(`Embedding batch request failed (${response.status}): ${this.safeErrorBody(respBody)}`);
|
|
98
99
|
}
|
|
99
100
|
const json = JSON.parse(await readBodyWithByteCap(response, undefined, { bodyTimeoutMs: 30_000, signal }));
|
|
100
101
|
if (!json.data || json.data.length !== batch.length) {
|
|
@@ -119,6 +120,21 @@ export class RemoteEmbedder {
|
|
|
119
120
|
}
|
|
120
121
|
return headers;
|
|
121
122
|
}
|
|
123
|
+
/**
|
|
124
|
+
* Make a provider error body safe to embed in a thrown Error, matching the
|
|
125
|
+
* hardening llm/client.ts applies on the identical path: pattern-redact
|
|
126
|
+
* credential shapes, exact-scrub this connection's own key, and clip.
|
|
127
|
+
*
|
|
128
|
+
* These messages are durable — generateEmbeddingsForDb surfaces them as
|
|
129
|
+
* `embeddingResult.message`, which is written to semantic-status.json and
|
|
130
|
+
* replayed by `akm info` (including `--json`) until the next successful
|
|
131
|
+
* index, and printed on every vector-search attempt. Raw bodies reached that
|
|
132
|
+
* far unredacted and uncapped, at readBodyWithByteCap's 10 MB default.
|
|
133
|
+
*/
|
|
134
|
+
safeErrorBody(body) {
|
|
135
|
+
const resolvedKey = resolveSecret(this.config.apiKey);
|
|
136
|
+
return redactSensitiveText(redactErrorBody(body), resolvedKey ? [resolvedKey] : []);
|
|
137
|
+
}
|
|
122
138
|
}
|
|
123
139
|
/**
|
|
124
140
|
* L2-normalize a vector to unit length.
|
|
@@ -144,6 +160,15 @@ export function normalizeEmbeddingEndpoint(endpoint) {
|
|
|
144
160
|
if (normalizedPath.endsWith("/embeddings")) {
|
|
145
161
|
return parsed.toString();
|
|
146
162
|
}
|
|
163
|
+
// Ollama's NATIVE embedding route is `/api/embed` (and the older
|
|
164
|
+
// `/api/embeddings`). Appending "/embeddings" to it produced
|
|
165
|
+
// `/api/embed/embeddings`, a 404 — so pointing akm at the native endpoint,
|
|
166
|
+
// which is what its own options like ollamaOptions and contextLength are
|
|
167
|
+
// for, could never work. An explicit path that is already an embedding route
|
|
168
|
+
// is left alone.
|
|
169
|
+
if (normalizedPath.endsWith("/embed")) {
|
|
170
|
+
return parsed.toString();
|
|
171
|
+
}
|
|
147
172
|
parsed.pathname = normalizedPath ? `${normalizedPath}/embeddings` : "/embeddings";
|
|
148
173
|
return parsed.toString();
|
|
149
174
|
}
|
|
@@ -14,8 +14,28 @@
|
|
|
14
14
|
*/
|
|
15
15
|
import fs from "node:fs";
|
|
16
16
|
import path from "node:path";
|
|
17
|
+
import healthTemplate from "../assets/templates/html/health.html" with { type: "text" };
|
|
17
18
|
import { getDirname } from "../runtime.js";
|
|
18
19
|
const TEMPLATES_DIR = path.join(getDirname(import.meta.url), "../assets/templates/html");
|
|
20
|
+
/**
|
|
21
|
+
* Templates embedded at build time, keyed by command name.
|
|
22
|
+
*
|
|
23
|
+
* `bun build --compile` embeds only what is imported `with { type: "text" }`;
|
|
24
|
+
* a plain `readFileSync` from a path relative to `import.meta.url` resolves
|
|
25
|
+
* into the virtual `/$bunfs` tree and misses. `akm health --report --format
|
|
26
|
+
* html` therefore crashed with ENOENT (exit 70) on the standalone binary that
|
|
27
|
+
* the CLI's own install error and the CHANGELOG promote as the runtime-free
|
|
28
|
+
* option. The text import works on all three runtimes: natively on Bun, via
|
|
29
|
+
* scripts/node-runtime/text-import-hook.mjs on Node, and embedded in the
|
|
30
|
+
* compiled binary.
|
|
31
|
+
*/
|
|
32
|
+
const EMBEDDED_TEMPLATES = {
|
|
33
|
+
// bun-types declares `*.html` as an `HTMLBundle` (its HTML-bundler entrypoint
|
|
34
|
+
// feature), which is not what a `type: "text"` import yields — the value is
|
|
35
|
+
// the file's contents as a string on every runtime. The cast reconciles the
|
|
36
|
+
// ambient declaration with the actual import attribute.
|
|
37
|
+
health: healthTemplate,
|
|
38
|
+
};
|
|
19
39
|
/**
|
|
20
40
|
* Resolve the on-disk template path for a command's bespoke `<command>.html`.
|
|
21
41
|
* The command name is sanitized to a bare basename so a hostile command
|
|
@@ -37,9 +57,28 @@ const TOKEN_RE = /%%[A-Z_]+%%/g;
|
|
|
37
57
|
* matching the skill renderer's behaviour.
|
|
38
58
|
*/
|
|
39
59
|
export function renderHtml(templatePath, replacements) {
|
|
40
|
-
const html =
|
|
60
|
+
const html = readTemplate(templatePath);
|
|
41
61
|
return html.replace(TOKEN_RE, (token) => replacements[token] ?? token);
|
|
42
62
|
}
|
|
63
|
+
/**
|
|
64
|
+
* Read a template from disk, falling back to the embedded copy.
|
|
65
|
+
*
|
|
66
|
+
* Disk stays primary so an operator (or a test) editing
|
|
67
|
+
* `src/assets/templates/html/<name>.html` sees the change without a rebuild.
|
|
68
|
+
* The fallback covers the standalone binary, where the file does not exist on
|
|
69
|
+
* any real filesystem.
|
|
70
|
+
*/
|
|
71
|
+
function readTemplate(templatePath) {
|
|
72
|
+
try {
|
|
73
|
+
return fs.readFileSync(templatePath, "utf8");
|
|
74
|
+
}
|
|
75
|
+
catch (err) {
|
|
76
|
+
const embedded = EMBEDDED_TEMPLATES[path.basename(templatePath, ".html")];
|
|
77
|
+
if (embedded !== undefined)
|
|
78
|
+
return embedded;
|
|
79
|
+
throw err;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
43
82
|
/**
|
|
44
83
|
* Minimal HTML entity escaping for text interpolated into templates. Escapes
|
|
45
84
|
* the single quote as well as the double quote so escaped values are safe in
|
package/dist/runtime.js
CHANGED
|
@@ -78,8 +78,17 @@ function nodeSpawnAdapter(cmd, options) {
|
|
|
78
78
|
detached: options.detached,
|
|
79
79
|
stdio: [stdioFor(options.stdin), stdioFor(options.stdout), stdioFor(options.stderr)],
|
|
80
80
|
});
|
|
81
|
+
// Node's 'exit' fires (null, signal) when the child dies from a signal.
|
|
82
|
+
// Resolving `code ?? 0` reported that as SUCCESS, so an OOM-killed or
|
|
83
|
+
// segfaulted child came back exit 0 with partial stdout. Bun resolves
|
|
84
|
+
// 128 + signum for the same case; match it so both runtimes agree and
|
|
85
|
+
// callers that only check `exitCode !== 0` classify signal deaths correctly.
|
|
86
|
+
let signalCode = null;
|
|
81
87
|
const exited = new Promise((resolve, reject) => {
|
|
82
|
-
child.once("exit", (code) =>
|
|
88
|
+
child.once("exit", (code, signal) => {
|
|
89
|
+
signalCode = signal;
|
|
90
|
+
resolve(code ?? (signal ? 128 + signalNumber(signal) : 0));
|
|
91
|
+
});
|
|
83
92
|
child.once("error", reject);
|
|
84
93
|
});
|
|
85
94
|
return {
|
|
@@ -96,12 +105,25 @@ function nodeSpawnAdapter(cmd, options) {
|
|
|
96
105
|
get exitCode() {
|
|
97
106
|
return child.exitCode;
|
|
98
107
|
},
|
|
108
|
+
get signalCode() {
|
|
109
|
+
return signalCode ?? child.signalCode;
|
|
110
|
+
},
|
|
99
111
|
pid: child.pid,
|
|
100
112
|
kill(signal) {
|
|
101
113
|
child.kill(signal);
|
|
102
114
|
},
|
|
103
115
|
};
|
|
104
116
|
}
|
|
117
|
+
/**
|
|
118
|
+
* Map a signal name to its number so a signal death can be reported as the
|
|
119
|
+
* conventional 128 + signum exit status. Falls back to SIGKILL's 9 for a name
|
|
120
|
+
* this platform does not define, which still yields a non-zero status — the
|
|
121
|
+
* property that matters for classifying the run as failed.
|
|
122
|
+
*/
|
|
123
|
+
function signalNumber(signal) {
|
|
124
|
+
const { constants } = nodeRequire("node:os");
|
|
125
|
+
return constants.signals[signal] ?? 9;
|
|
126
|
+
}
|
|
105
127
|
// `node:stream`'s Writable.toWeb is available on Node >=17; referenced via the
|
|
106
128
|
// class to avoid a static import that Bun's typings may not expose.
|
|
107
129
|
function Writable_toWeb(w) {
|