akm-cli 0.9.0-rc.0 → 0.9.0-rc.13

package/dist/commands/improve/eligibility.js

@@ -3,16 +3,21 @@
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 import fs from "node:fs";
 import path from "node:path";
-import { makeAssetRef, parseAssetRef } from "../../core/asset/asset-ref.js";
 import { parseFrontmatter } from "../../core/asset/frontmatter.js";
-import { isAssetType } from "../../core/common.js";
+import { conceptIdFromTypeName, parseRefInput, resolveRef } from "../../core/asset/resolve-ref.js";
+import { loadConfig } from "../../core/config/config.js";
 import { NotFoundError, rethrowIfTestIsolationError, UsageError } from "../../core/errors.js";
 import { readEvents } from "../../core/events.js";
-import { closeDatabase, getAllEntries, getUtilityScoresByIds, openExistingDatabase } from "../../indexer/db/db.js";
+import { deriveInstallations } from "../../indexer/installations.js";
 import { getWritableStashDirs, resolveSourceEntries } from "../../indexer/search/search-source.js";
 import { resolveAssetPath } from "../../indexer/walk/path-resolver.js";
+import { closeDatabase, openExistingDatabase, openReadonlyExistingDatabase, } from "../../storage/repositories/index-connection.js";
+import { getAllEntries } from "../../storage/repositories/index-entries-repository.js";
+import { getUtilityScoresByIds } from "../../storage/repositories/index-utility-repository.js";
 import { isDistillRefusedInputType } from "./distill.js";
-import { isProfileFilteredForAllPasses } from "./improve-profiles.js";
+import { isStrategyFilteredForAllPasses } from "./improve-strategies.js";
+import { parseMemoryRef } from "./memory/derived-ref.js";
+import { improveStateReadRefs } from "./source-identity.js";
 // Eligibility / candidate-selection predicates for improve. Free functions
 // (no akmImprove closure state) extracted from improve.ts to shrink the
 // orchestrator and make candidate selection independently testable.
@@ -21,69 +26,122 @@ export function resolveImproveScope(scope) {
     if (!trimmed)
         return { mode: "all" };
     try {
-        parseAssetRef(trimmed);
+        parseRefInput(trimmed);
         return { mode: "ref", value: trimmed };
     }
-    catch {
-        if (!isAssetType(trimmed)) {
-            throw new UsageError(`Unknown asset type: "${trimmed}". Valid types: memory, knowledge, skill, lesson, workflow, agent, command, script, wiki, env, secret, task.\n` +
-                `If you passed --format to akm improve, that flag is not supported — use it with akm search or akm show instead.`, "INVALID_FLAG_VALUE");
+    catch (err) {
+        // A bare word with no ref separator is a `--scope <type>` filter attempt,
+        // not a ref. Any such word is accepted, including foreign/unknown type strings,
+        // which simply match zero entries downstream (a read-only query filter,
+        // not a data-acceptance gate, so no deny-list is needed here: an
+        // unrecognized type just matches nothing, since nothing is ever indexed
+        // with it). No test pinned the old "Unknown asset type" rejection for
+        // this case (chunk-1.5 anchors §A.5 — 0 hits).
+        //
+        // A colon-shaped value that still fails `parseRefInput` is a genuinely
+        // malformed ref attempt (bad name, path traversal, or a deny-listed
+        // deliberately-removed type like tool/vault, D1.5-6) — that must still
+        // surface as a real error, not be silently absorbed into a type filter
+        // that will never match anything.
+        if (!/[/:#]/.test(trimmed)) {
+            return { mode: "type", value: trimmed };
         }
-        return { mode: "type", value: trimmed };
+        const message = err instanceof Error ? err.message : String(err);
+        throw new UsageError(`Invalid --scope "${trimmed}": ${message}`, "INVALID_FLAG_VALUE");
     }
 }
 /**
- * Dedupe a list of eligible refs by `ref`, preserving first-seen order. Used to
- * merge the three eligibility sources (feedback-signal, P0-A high-retrieval,
- * Layer-2 proactive-maintenance) without admitting a ref into the loop twice.
+ * Dedupe eligible refs by their durable item ref when available, preserving
+ * the first candidate and its display ref. Used to
+ * merge the eligibility sources (feedback-signal, Layer-2 proactive-maintenance,
+ * high-salience) without admitting a ref into the loop twice.
  */
 export function dedupeRefs(refs) {
     const seen = new Set();
     const out = [];
     for (const r of refs) {
-        if (seen.has(r.ref))
+        const key = r.itemRef ?? r.ref;
+        if (seen.has(key))
             continue;
-        seen.add(r.ref);
+        seen.add(key);
         out.push(r);
     }
     return out;
 }
-export async function collectEligibleRefs(scope, stashDir, improveProfile) {
+export async function collectEligibleRefs(scope, stashDir, improveProfile, config) {
+    return collectEligibleRefsFromIndex(scope, stashDir, improveProfile, false, config);
+}
+/** Dry-run planner path: query an existing index without creating or mutating it. */
+export async function collectEligibleRefsReadOnly(scope, stashDir, improveProfile, config) {
+    return collectEligibleRefsFromIndex(scope, stashDir, improveProfile, true, config);
+}
+async function collectEligibleRefsFromIndex(scope, stashDir, improveProfile, readOnly, configOverride) {
+    const config = configOverride ?? loadConfig();
     if (scope.mode === "ref" && scope.value) {
-        const parsed = parseAssetRef(scope.value);
-        const writableDirs = new Set(getWritableStashDirs(stashDir).map((dir) => path.resolve(dir)));
-        const filePath = await findAssetFilePath(scope.value, stashDir, writableDirs);
-        if (!filePath) {
+        const writableDirs = new Set(getWritableStashDirs(stashDir, config).map((dir) => path.resolve(dir)));
+        let db;
+        try {
+            db = readOnly ? openReadonlyExistingDatabase() : openExistingDatabase();
+            if (!db) {
+                return { plannedRefs: [], memorySummary: { eligible: 0, derived: 0 }, strategyFilteredRefs: [] };
+            }
+            const entries = getAllEntries(db);
+            const entriesByItemRef = new Map(entries.flatMap((entry) => entry.bundleId && entry.conceptId ? [[`${entry.bundleId}//${entry.conceptId}`, entry]] : []));
+            const installations = deriveInstallations(resolveSourceEntries(stashDir, config));
+            const resolved = resolveRef(scope.value, {
+                defaultBundle: config.defaultBundle,
+                bundles: installations.map((installation) => ({
+                    id: installation.id,
+                    hasConcept: (conceptId) => entriesByItemRef.has(`${installation.id}//${conceptId}`),
+                })),
+            });
+            const indexed = entriesByItemRef.get(`${resolved.bundle}//${resolved.conceptId}`);
+            if (!indexed?.bundleId || !indexed.conceptId || !fs.existsSync(indexed.filePath)) {
+                if (await findAssetFilePath(scope.value, stashDir)) {
+                    return { plannedRefs: [], memorySummary: { eligible: 0, derived: 0 }, strategyFilteredRefs: [] };
+                }
+                throw new NotFoundError(`Asset not found in the selected writable source: ${scope.value}`, "ASSET_NOT_FOUND");
+            }
+            if (!isEntryInScope(indexed.stashDir, indexed.filePath, stashDir) ||
+                !isEntryInWritableSource(indexed.stashDir, indexed.filePath, writableDirs)) {
+                return { plannedRefs: [], memorySummary: { eligible: 0, derived: 0 }, strategyFilteredRefs: [] };
+            }
+            const itemRef = `${indexed.bundleId}//${indexed.conceptId}`;
             return {
-                plannedRefs: [],
-                memorySummary: { eligible: 0, derived: 0 },
-                profileFilteredRefs: [],
+                plannedRefs: [{ ref: indexed.conceptId, itemRef, reason: "scope-ref", filePath: indexed.filePath }],
+                memorySummary: {
+                    eligible: indexed.entry.type === "memory" ? 1 : 0,
+                    derived: indexed.entry.type === "memory" && indexed.entry.name.endsWith(".derived") ? 1 : 0,
+                },
+                strategyFilteredRefs: [],
             };
         }
-        return {
-            plannedRefs: [{ ref: scope.value, reason: "scope-ref", filePath }],
-            memorySummary: {
-                eligible: parsed.type === "memory" ? 1 : 0,
-                derived: parsed.type === "memory" && parsed.name.endsWith(".derived") ? 1 : 0,
-            },
-            profileFilteredRefs: [],
-        };
+        catch (error) {
+            if (error instanceof Error && /no such table:\s*entries/i.test(error.message)) {
+                return { plannedRefs: [], memorySummary: { eligible: 0, derived: 0 }, strategyFilteredRefs: [] };
+            }
+            throw error;
+        }
+        finally {
+            if (db)
+                closeDatabase(db);
+        }
     }
     let sources;
     try {
-        sources = resolveSourceEntries(stashDir);
+        sources = resolveSourceEntries(stashDir, config);
     }
     catch {
-        return { plannedRefs: [], memorySummary: { eligible: 0, derived: 0 }, profileFilteredRefs: [] };
+        return { plannedRefs: [], memorySummary: { eligible: 0, derived: 0 }, strategyFilteredRefs: [] };
     }
     if (sources.length === 0) {
-        return { plannedRefs: [], memorySummary: { eligible: 0, derived: 0 }, profileFilteredRefs: [] };
+        return { plannedRefs: [], memorySummary: { eligible: 0, derived: 0 }, strategyFilteredRefs: [] };
     }
     // Only operate on writable sources — never mutate read-only registry caches
     // or remote stashes that the user did not mark writable.
     let writableDirs;
     try {
-        writableDirs = getWritableStashDirs(stashDir);
+        writableDirs = getWritableStashDirs(stashDir, config);
     }
     catch {
         writableDirs = sources.slice(0, 1).map((s) => s.path); // fallback: primary only
@@ -91,7 +149,10 @@ export async function collectEligibleRefs(scope, stashDir, improveProfile) {
     const writableDirSet = new Set(writableDirs.map((d) => path.resolve(d)));
     let db;
     try {
-        db = openExistingDatabase();
+        db = readOnly ? openReadonlyExistingDatabase() : openExistingDatabase();
+        if (!db) {
+            return { plannedRefs: [], memorySummary: { eligible: 0, derived: 0 }, strategyFilteredRefs: [] };
+        }
         const entries = getAllEntries(db, scope.mode === "type" ? scope.value : undefined).filter((indexed) => {
             // First apply the existing stashDir-scope filter (no-op when stashDir is unset).
             if (!isEntryInScope(indexed.stashDir, indexed.filePath, stashDir))
@@ -100,11 +161,26 @@ export async function collectEligibleRefs(scope, stashDir, improveProfile) {
             return isEntryInWritableSource(indexed.stashDir, indexed.filePath, writableDirSet);
         });
         const planned = new Map();
-        const profileFiltered = new Map();
+        const strategyFiltered = new Map();
         let memoryEligible = 0;
         let memoryDerived = 0;
         for (const indexed of entries) {
-            const ref = makeAssetRef(indexed.entry.type, indexed.entry.name);
+            // Chunk-8 WI-8.5c: the candidate `ref` is the SHORT conceptId
+            // (`<stash-subdir>/<name>`, D-R2) — it now matches the disk lookup,
+            // xrefs, and `displayRef` output spelling. `.itemRef` below stays the
+            // fully-qualified durable key.
+            const ref = conceptIdFromTypeName(indexed.entry.type, indexed.entry.name);
+            try {
+                parseRefInput(ref);
+            }
+            catch (error) {
+                if (error instanceof UsageError || error instanceof NotFoundError)
+                    continue;
+                throw error;
+            }
+            // Derive the durable item_ref from indexed provenance without another query.
+            // A provenance-free row uses the short conceptId ref for improve state.
+            const itemRef = indexed.bundleId && indexed.conceptId ? `${indexed.bundleId}//${indexed.conceptId}` : undefined;
             const isDerived = indexed.entry.name.endsWith(".derived");
             // `.derived` memories are LLM-inferred and intentionally skip reflect
             // (see the synthetic `derived-memory-reflect-skipped` branch in the
@@ -113,19 +189,20 @@ export async function collectEligibleRefs(scope, stashDir, improveProfile) {
             // 2026-05-21: 11 derived refs re-planned every hour during idle periods.
             // The cleanup phase (analyzeMemoryCleanup) inspects derived memories
             // independently of `plannedRefs`, so dropping them here loses nothing.
-            if (!isDerived && !planned.has(ref) && !profileFiltered.has(ref)) {
-                // 2026-05-27: extend the .derived precedent to profile-incompatible
+            if (!isDerived && !planned.has(ref) && !strategyFiltered.has(ref)) {
+                // 2026-05-27: extend the .derived precedent to strategy-incompatible
                 // refs. If every per-ref pass (reflect + distill) on the active
-                // profile would refuse this ref, drop it from `plannedRefs`. The
-                // caller emits `improve_skipped { reason: profile_filtered_all_passes }`
+                // strategy would refuse this ref, drop it from `plannedRefs`. The
+                // caller emits `improve_skipped { reason: strategy_filtered_all_passes }`
                 // once `eventsCtx` is available so the audit trail is preserved in a
                 // single event per ref instead of 2× synthetic actions per run.
                 // Background: see /tmp/akm-health-investigations/planner-profile-metrics-deep-analysis.md
-                if (improveProfile && isProfileFilteredForAllPasses(ref, improveProfile)) {
-                    profileFiltered.set(ref, {
+                if (improveProfile && isStrategyFilteredForAllPasses(ref, improveProfile)) {
+                    strategyFiltered.set(ref, {
                         ref,
-                        reason: "profile_filtered_all_passes",
+                        reason: "strategy_filtered_all_passes",
                         filePath: indexed.filePath,
+                        itemRef,
                     });
                 }
                 else {
@@ -133,6 +210,7 @@ export async function collectEligibleRefs(scope, stashDir, improveProfile) {
                         ref,
                         reason: scope.mode === "type" ? "scope-type" : indexed.entry.type === "memory" ? "memory-cleanup" : "scope-type",
                         filePath: indexed.filePath,
+                        itemRef,
                     });
                 }
             }
@@ -145,14 +223,14 @@ export async function collectEligibleRefs(scope, stashDir, improveProfile) {
         return {
             plannedRefs: [...planned.values()],
             memorySummary: { eligible: memoryEligible, derived: memoryDerived },
-            profileFilteredRefs: [...profileFiltered.values()],
+            strategyFilteredRefs: [...strategyFiltered.values()],
         };
     }
     catch (error) {
-        // The bun-test isolation guard must never be downgraded to "empty plan".
+        // Empty-stash setup paths can open index.db before its schema exists.
         rethrowIfTestIsolationError(error);
-        if (error instanceof NotFoundError || error instanceof Error) {
-            return { plannedRefs: [], memorySummary: { eligible: 0, derived: 0 }, profileFilteredRefs: [] };
+        if (error instanceof Error && /no such table:\s*entries/i.test(error.message)) {
+            return { plannedRefs: [], memorySummary: { eligible: 0, derived: 0 }, strategyFilteredRefs: [] };
         }
         throw error;
     }
@@ -191,11 +269,16 @@ export function isEntryInWritableSource(entryStashDir, filePath, writableDirSet)
 export function memoryCleanupParentRef(scope, stashDir) {
     if (scope.mode !== "ref" || !scope.value)
         return undefined;
-    const parsed = parseAssetRef(scope.value);
+    const parsed = parseRefInput(scope.value);
     if (parsed.type !== "memory")
         return undefined;
+    // Non-derived parent scope: emit the canonical `memories/<name>` conceptId so
+    // it matches `resolveParentRef`'s output in analyzeMemoryCleanup's parentRef
+    // filter (Group-C item 2 — the reader and every comparison site flipped
+    // together; emitting the raw scope value would re-open the mismatch the
+    // chunk-8 history warns about).
     if (!parsed.name.endsWith(".derived"))
-        return scope.value;
+        return conceptIdFromTypeName(parsed.type, parsed.name);
     const sources = resolveSourceEntries(stashDir);
     for (const source of sources) {
         const candidate = path.join(source.path, "memories", `${parsed.name}.md`);
@@ -203,23 +286,21 @@ export function memoryCleanupParentRef(scope, stashDir) {
             continue;
         const raw = fs.readFileSync(candidate, "utf8");
         const fm = parseFrontmatter(raw).data;
-        const sourceRef = typeof fm.source === "string" ? fm.source : undefined;
-        if (sourceRef) {
-            try {
-                const parent = parseAssetRef(sourceRef.trim());
-                if (parent.type === "memory")
-                    return makeAssetRef(parent.type, parent.name);
-            }
-            catch { }
-        }
+        // The `source:` backref is normalized to the `memories/<name>` conceptId.
+        // That is exactly what
+        // analyzeMemoryCleanup's parentRef filter compares against — resolveParentRef
+        // emits the same conceptId — so the two sites stay in lockstep.
+        const parent = parseMemoryRef(typeof fm.source === "string" ? fm.source : undefined);
+        if (parent)
+            return parent;
     }
-    return makeAssetRef("memory", parsed.name.slice(0, -".derived".length));
+    return conceptIdFromTypeName("memory", parsed.name.slice(0, -".derived".length));
 }
 export function isLessonCandidate(ref) {
     // Only lesson assets need lesson-schema validation (description + when_to_use).
     // Memories have their own distill path via shouldDistillMemoryRef.
     // All other types go through reflect, not distill.
-    return parseAssetRef(ref).type === "lesson";
+    return parseRefInput(ref).type === "lesson";
 }
 /**
  * Planner-side check: should this ref enter the distill queue?
@@ -239,16 +320,16 @@ export function isLessonCandidate(ref) {
  * which was true *only* for `lesson:*` refs — exactly the set distill refuses.
  * The result: every hourly run re-queued the same lesson refs, the same skip
  * message returned, and no work was ever done. See
- * `tests/commands/improve-distill-planner-skip-lessons.test.ts`.
+ * `tests/integration/commands/improve-distill-planner-skip-lessons.test.ts`.
  */
 export function isDistillCandidateRef(ref, stashDir) {
-    const parsed = parseAssetRef(ref);
+    const parsed = parseRefInput(ref);
     if (isDistillRefusedInputType(parsed.type))
         return false;
     return shouldDistillMemoryRef(ref, stashDir);
 }
 export function shouldDistillMemoryRef(ref, stashDir) {
-    const parsed = parseAssetRef(ref);
+    const parsed = parseRefInput(ref);
     if (parsed.type !== "memory")
         return false;
     const sources = resolveSourceEntries(stashDir);
@@ -282,15 +363,16 @@ export function shouldDistillMemoryRef(ref, stashDir) {
  * metadata events are ignored so a stray `akm feedback <ref>` invocation
  * without a flag doesn't trigger downstream re-processing.
  */
-export function buildLatestFeedbackTsMap(refs, sinceIso) {
+export function buildLatestFeedbackTsMap(refs, sinceIso, itemRefByRef) {
     const out = new Map();
     if (refs.length === 0)
         return out;
-    const refSet = new Set(refs);
+    // Correlate item_ref-keyed events and conceptId-keyed direct invocations.
+    const refByDurableKey = new Map(refs.flatMap((ref) => improveStateReadRefs(ref, itemRefByRef?.get(ref)).map((key) => [key, ref])));
     const { events } = readEvents({ type: "feedback", since: sinceIso });
     for (const e of events) {
-        const ref = e.ref;
-        if (!ref || !refSet.has(ref))
+        const ref = e.ref ? refByDurableKey.get(e.ref) : undefined;
+        if (!ref)
             continue;
         const meta = e.metadata;
         const hasSignal = meta !== undefined && (typeof meta.signal === "string" || typeof meta.note === "string");
@@ -306,21 +388,22 @@ export function buildLatestFeedbackTsMap(refs, sinceIso) {
  * Latest proposal timestamp per input-ref, filtered by source ('reflect' or
  * 'distill'). Reads the corresponding `*_invoked` events from state.db —
  * these events are emitted at proposal creation time and carry the *input*
- * asset ref (memory:foo, skill:bar, etc.) directly. We use them rather than
+ * asset ref (`memories/foo`, `skills/bar`, etc.) directly. We use them rather than
  * `listProposals` because distill proposals are keyed by the derived
  * lesson/knowledge ref, not the source memory — joining back through the
  * payload would be fragile.
  */
-export function buildLatestProposalTsMap(refs, source) {
+export function buildLatestProposalTsMap(refs, source, itemRefByRef) {
     const out = new Map();
     if (refs.length === 0)
         return out;
-    const refSet = new Set(refs);
+    // Correlate item_ref-keyed events and conceptId-keyed direct invocations.
+    const refByDurableKey = new Map(refs.flatMap((ref) => improveStateReadRefs(ref, itemRefByRef?.get(ref)).map((key) => [key, ref])));
     const eventType = source === "reflect" ? "reflect_invoked" : "distill_invoked";
     const { events } = readEvents({ type: eventType });
     for (const e of events) {
-        const ref = e.ref;
-        if (!ref || !refSet.has(ref))
+        const ref = e.ref ? refByDurableKey.get(e.ref) : undefined;
+        if (!ref)
             continue;
         // For distill_invoked we only count attempts that produced (or attempted
         // to produce) a real proposal — config_disabled / parse-error outcomes
@@ -343,8 +426,9 @@ export function buildLatestProposalTsMap(refs, source) {
  * exists for this (ref, source) OR `latestFeedback[ref] > lastProposal[ref]`.
  *
  * Refs with no feedback signal at all are ineligible by definition — the
- * high-retrieval fallback path (see `noFeedbackCandidates` later in the
- * planner) handles never-touched-but-frequently-read assets separately.
+ * proactive-maintenance and high-salience fallback lanes (see
+ * `noFeedbackCandidates` later in the planner) handle never-rated assets
+ * separately.
  */
 export function isSignalDeltaEligible(ref, latestFeedback, lastProposal) {
     const fb = latestFeedback.get(ref);
@@ -385,7 +469,7 @@ export function shouldAnalyzeMemoryCleanup(scope, eligibleMemories, primaryStash
         return scope.value === "memory";
     if (!scope.value)
         return false;
-    return parseAssetRef(scope.value).type === "memory";
+    return parseRefInput(scope.value).type === "memory";
 }
 export function buildUtilityMap(refs) {
     const map = new Map();
@@ -398,7 +482,9 @@ export function buildUtilityMap(refs) {
         const allDbEntries = getAllEntries(db);
         const idToRef = new Map();
         for (const indexed of allDbEntries) {
-            const ref = makeAssetRef(indexed.entry.type, indexed.entry.name);
+            // Chunk-8 WI-8.5c: correlate on the SHORT conceptId to match the
+            // `ImproveEligibleRef.ref` set built in collectEligibleRefsFromIndex.
+            const ref = conceptIdFromTypeName(indexed.entry.type, indexed.entry.name);
             if (refSet.has(ref))
                 idToRef.set(indexed.id, ref);
         }
@@ -429,6 +515,6 @@ export async function findAssetFilePath(ref, stashDir, writableDirSet) {
         writableDirSet,
         directoryIndexNames: ["SKILL.md"],
         preserveDirectNameFallback: true,
-        honorOrigin: false,
+        honorOrigin: true,
     });
 }