akm-cli 0.9.0-beta.9 → 0.9.0-rc.1

package/dist/commands/read/knowledge.js

@@ -16,7 +16,8 @@ import { isHttpUrl, isWithin, tryReadStdinText } from "../../core/common.js";
 import { loadConfig } from "../../core/config/config.js";
 import { UsageError } from "../../core/errors.js";
 import { commitWriteTargetBoundary, formatRefForMessage, resolveWriteTarget, writeAssetToSource, } from "../../core/write-source.js";
-import { fetchWebsiteMarkdownSnapshot } from "../../sources/website-ingest.js";
+import { indexWrittenAssets } from "../../indexer/index-written-assets.js";
+import { fetchWebsiteMarkdownSnapshot, shouldAllowPrivateWebsiteUrlForTests } from "../../sources/website-ingest.js";
 const MAX_CAPTURED_ASSET_SLUG_LENGTH = 64;
 // ── Asset-name normalisation ─────────────────────────────────────────────────
 /**
@@ -101,10 +102,13 @@ export function readKnowledgeContent(source) {
  * URLs are fetched via `fetchWebsiteMarkdownSnapshot`; local sources delegate
  * to `readKnowledgeContent`.
  */
-export async function readKnowledgeInput(source) {
+export async function readKnowledgeInput(source, options) {
     if (!isHttpUrl(source))
         return readKnowledgeContent(source);
-    const snapshot = await fetchWebsiteMarkdownSnapshot(source);
+    const snapshot = await fetchWebsiteMarkdownSnapshot(source, {
+        stashDir: options?.stashDir,
+        allowPrivateHosts: options?.allowPrivateHosts ?? shouldAllowPrivateWebsiteUrlForTests(source),
+    });
     return { content: snapshot.content, preferredName: snapshot.preferredName };
 }
 // ── Asset writing ────────────────────────────────────────────────────────────
@@ -141,6 +145,9 @@ export async function writeMarkdownAsset(options) {
     // 0.9.0 (issue #507): single batch commit at the write boundary for git
     // targets. No-op for filesystem/primary-stash targets.
     commitWriteTargetBoundary(target, `Update ${formatRefForMessage(ref)}`);
+    // Write-path indexing: the asset is searchable immediately. Fail-open; reads
+    // no longer trigger reindexes, so keeping the index current is the writer's job.
+    await indexWrittenAssets(source.path, [result.path]);
     return {
         ref: result.ref,
         path: result.path,

package/dist/commands/read/remember-cli.js

@@ -1,8 +1,8 @@
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
-import { defineCommand } from "citty";
-import { output, parseAllFlagValues, runWithJsonErrors } from "../../cli/shared.js";
+import { getStringArg } from "../../cli/parse-args.js";
+import { defineJsonCommand, output, parseAllFlagValues } from "../../cli/shared.js";
 import { UsageError } from "../../core/errors.js";
 import { appendEvent } from "../../core/events.js";
 import { buildMemoryFrontmatter, parseDuration, readMemoryContent, resolveRememberContentArg, runAutoHeuristics, runLlmEnrich, } from "../remember.js";
@@ -26,7 +26,7 @@ async function fetchSimilarMemories(query, excludeRef) {
     }
 }
 // ── Command definition ────────────────────────────────────────────────────────
-export const rememberCommand = defineCommand({
+export const rememberCommand = defineJsonCommand({
     meta: {
         name: "remember",
         description: "Record a memory in the default stash",
@@ -60,7 +60,7 @@ export const rememberCommand = defineCommand({
         },
         expires: {
             type: "string",
-            description: "Expiry duration shorthand (e.g. 30d, 12h, 6m). Resolved to an ISO date.",
+            description: "Expiry duration shorthand — e.g. 30d, 12h, 5m (minutes), 3M (months). Resolved to an ISO date.",
         },
         source: {
             type: "string",
@@ -102,141 +102,42 @@ export const rememberCommand = defineCommand({
         },
     },
     async run({ args }) {
-        return runWithJsonErrors(async () => {
-            const body = readMemoryContent(resolveRememberContentArg(args.content));
-            // `--name` is a flat name; subdirectory placement is `--path`'s job.
-            assertFlatAssetName(args.name);
-            // Determine if the user has requested any structured metadata mode.
-            // Collect all --tag occurrences directly from process.argv because citty
-            // only exposes the last value for repeated string flags.
-            const rawTags = parseAllFlagValues("--tag");
-            // Collect scope flags. Scope alone counts as structured metadata so we
-            // emit frontmatter, but it does NOT trigger the "tags required" check —
-            // memory + scope (no tags) is a valid combination for multi-tenant use.
-            const scopeFields = {};
-            if (typeof args.user === "string" && args.user.trim())
-                scopeFields.user = args.user.trim();
-            if (typeof args.agent === "string" && args.agent.trim())
-                scopeFields.agent = args.agent.trim();
-            if (typeof args.run === "string" && args.run.trim())
-                scopeFields.run = args.run.trim();
-            if (typeof args.channel === "string" && args.channel.trim())
-                scopeFields.channel = args.channel.trim();
-            const hasScope = Object.keys(scopeFields).length > 0;
-            const hasTagRequiringArgs = rawTags.length > 0 || !!args.expires || !!args.source || !!args.description;
-            const hasStructuredArgs = hasTagRequiringArgs || hasScope || args.auto;
-            if (!hasStructuredArgs) {
-                // Phase 1B / Rec 7: even the zero-flag hot-path emits
-                // `captureMode: hot` + `beliefState: asserted` so user-supplied
-                // memories outrank background-derived ones during ranking.
-                const frontmatterBlock = buildMemoryFrontmatter({
-                    captureMode: "hot",
-                    beliefState: "asserted",
-                });
-                const contentWithFrontmatter = `${frontmatterBlock}\n${body}`;
-                // Derive the asset slug from the body (not the frontmatter block);
-                // otherwise inferAssetName would key off the leading `---` delimiter.
-                const result = await writeMarkdownAsset({
-                    type: "memory",
-                    content: contentWithFrontmatter,
-                    name: args.name,
-                    fallbackPrefix: "memory",
-                    preferredName: inferAssetName(body, "memory"),
-                    force: args.force,
-                    target: args.target,
-                    path: args.path,
-                });
-                appendEvent({
-                    eventType: "remember",
-                    ref: result.ref,
-                    metadata: { path: result.path, force: args.force === true },
-                });
-                if (args.showSimilar) {
-                    const similar = await fetchSimilarMemories(body.slice(0, 500), result.ref);
-                    output("remember", { ok: true, ...result, similar });
-                }
-                else {
-                    output("remember", { ok: true, ...result });
-                }
-                return;
-            }
-            // ── Accumulate metadata from all three modes ──────────────────────────
-            // Start with CLI args (Mode 1: always)
-            const tags = [...rawTags];
-            // --description is persisted as-is; LLM enrichment may fill it if absent.
-            let description = args.description || undefined;
-            let source = args.source;
-            let observed_at;
-            let expires;
-            let subjective;
-            // Resolve --expires to an ISO date string
-            if (args.expires) {
-                const durationMs = parseDuration(args.expires);
-                const expiresDate = new Date(Date.now() + durationMs);
-                expires = expiresDate.toISOString().slice(0, 10);
-            }
-            // Mode 2: --auto heuristics
-            if (args.auto) {
-                const auto = runAutoHeuristics(body);
-                for (const t of auto.tags) {
-                    if (!tags.includes(t))
-                        tags.push(t);
-                }
-                if (!source && auto.source)
-                    source = auto.source;
-                if (!observed_at && auto.observed_at)
-                    observed_at = auto.observed_at;
-                if (!subjective && auto.subjective)
-                    subjective = auto.subjective;
-            }
-            // Mode 3: --enrich LLM (fail-soft)
-            if (args.enrich) {
-                const enriched = await runLlmEnrich(body);
-                for (const t of enriched.tags) {
-                    if (!tags.includes(t))
-                        tags.push(t);
-                }
-                if (!description && enriched.description)
-                    description = enriched.description;
-                if (!observed_at && enriched.observed_at)
-                    observed_at = enriched.observed_at;
-            }
-            // ── Required-field check (before any write) ───────────────────────────
-            // Tags remain required when the user explicitly asked for tag-bearing
-            // metadata (--tag / --enrich / --description / --source / --expires).
-            // `--auto` alone is allowed even when its heuristics derive zero tags.
-            // Scope-only writes (`akm remember "..." --user u1`) also skip this
-            // check — scope is independent metadata and a memory with only scope is
-            // valid.
-            const missing = [];
-            if (hasTagRequiringArgs && tags.length === 0)
-                missing.push("tags");
-            if (missing.length > 0) {
-                throw new UsageError(`Memory is missing required frontmatter field(s): ${missing.join(", ")}. ` +
-                    "Provide them via --tag <value>, --auto (heuristics), or --enrich (LLM).");
-            }
-            // ── Build frontmatter and write ───────────────────────────────────────
-            // Phase 1B / Rec 7: the hot-path CLI write always marks the memory as
-            // `captureMode: hot` and `beliefState: asserted`. Ranking applies a
-            // hot-capture boost so user-supplied memories outrank otherwise-equal
-            // background-derived ones.
+        const body = readMemoryContent(resolveRememberContentArg(args.content));
+        // `--name` is a flat name; subdirectory placement is `--path`'s job.
+        assertFlatAssetName(args.name);
+        // Determine if the user has requested any structured metadata mode.
+        // Collect all --tag occurrences directly from process.argv because citty
+        // only exposes the last value for repeated string flags.
+        const rawTags = parseAllFlagValues("--tag");
+        // Collect scope flags. Scope alone counts as structured metadata so we
+        // emit frontmatter, but it does NOT trigger the "tags required" check —
+        // memory + scope (no tags) is a valid combination for multi-tenant use.
+        const scopeFields = {};
+        for (const k of ["user", "agent", "run", "channel"]) {
+            const v = getStringArg(args, k);
+            if (v)
+                scopeFields[k] = v;
+        }
+        const hasScope = Object.keys(scopeFields).length > 0;
+        const hasTagRequiringArgs = rawTags.length > 0 || !!args.expires || !!args.source || !!args.description;
+        const hasStructuredArgs = hasTagRequiringArgs || hasScope || args.auto;
+        if (!hasStructuredArgs) {
+            // Phase 1B / Rec 7: even the zero-flag hot-path emits
+            // `captureMode: hot` + `beliefState: asserted` so user-supplied
+            // memories outrank background-derived ones during ranking.
             const frontmatterBlock = buildMemoryFrontmatter({
-                description,
-                tags,
-                source,
-                observed_at,
-                expires,
-                subjective,
                 captureMode: "hot",
                 beliefState: "asserted",
-                ...(hasScope ? { scope: scopeFields } : {}),
             });
             const contentWithFrontmatter = `${frontmatterBlock}\n${body}`;
+            // Derive the asset slug from the body (not the frontmatter block);
+            // otherwise inferAssetName would key off the leading `---` delimiter.
             const result = await writeMarkdownAsset({
                 type: "memory",
                 content: contentWithFrontmatter,
                 name: args.name,
                 fallbackPrefix: "memory",
+                preferredName: inferAssetName(body, "memory"),
                 force: args.force,
                 target: args.target,
                 path: args.path,
@@ -244,22 +145,116 @@ export const rememberCommand = defineCommand({
             appendEvent({
                 eventType: "remember",
                 ref: result.ref,
-                metadata: {
-                    path: result.path,
-                    force: args.force === true,
-                    tagCount: tags.length,
-                    enriched: args.enrich === true,
-                    auto: args.auto === true,
-                    ...(hasScope ? { scope: scopeFields } : {}),
-                },
+                metadata: { path: result.path, force: args.force === true },
             });
             if (args.showSimilar) {
-                const similar = await fetchSimilarMemories((body ?? args.content ?? "").slice(0, 500), result.ref);
+                const similar = await fetchSimilarMemories(body.slice(0, 500), result.ref);
                 output("remember", { ok: true, ...result, similar });
             }
             else {
                 output("remember", { ok: true, ...result });
             }
+            return;
+        }
+        // ── Accumulate metadata from all three modes ──────────────────────────
+        // Start with CLI args (Mode 1: always)
+        const tags = [...rawTags];
+        // --description is persisted as-is; LLM enrichment may fill it if absent.
+        let description = args.description || undefined;
+        let source = args.source;
+        let observed_at;
+        let expires;
+        let subjective;
+        // Resolve --expires to an ISO date string
+        if (args.expires) {
+            const durationMs = parseDuration(args.expires);
+            const expiresDate = new Date(Date.now() + durationMs);
+            expires = expiresDate.toISOString().slice(0, 10);
+        }
+        // Mode 2: --auto heuristics
+        if (args.auto) {
+            const auto = runAutoHeuristics(body);
+            for (const t of auto.tags) {
+                if (!tags.includes(t))
+                    tags.push(t);
+            }
+            if (!source && auto.source)
+                source = auto.source;
+            if (!observed_at && auto.observed_at)
+                observed_at = auto.observed_at;
+            if (!subjective && auto.subjective)
+                subjective = auto.subjective;
+        }
+        // Mode 3: --enrich LLM (fail-soft)
+        if (args.enrich) {
+            const enriched = await runLlmEnrich(body);
+            for (const t of enriched.tags) {
+                if (!tags.includes(t))
+                    tags.push(t);
+            }
+            if (!description && enriched.description)
+                description = enriched.description;
+            if (!observed_at && enriched.observed_at)
+                observed_at = enriched.observed_at;
+        }
+        // ── Required-field check (before any write) ───────────────────────────
+        // Tags remain required when the user explicitly asked for tag-bearing
+        // metadata (--tag / --enrich / --description / --source / --expires).
+        // `--auto` alone is allowed even when its heuristics derive zero tags.
+        // Scope-only writes (`akm remember "..." --user u1`) also skip this
+        // check — scope is independent metadata and a memory with only scope is
+        // valid.
+        const missing = [];
+        if (hasTagRequiringArgs && tags.length === 0)
+            missing.push("tags");
+        if (missing.length > 0) {
+            throw new UsageError(`Memory is missing required frontmatter field(s): ${missing.join(", ")}. ` +
+                "Provide them via --tag <value>, --auto (heuristics), or --enrich (LLM).");
+        }
+        // ── Build frontmatter and write ───────────────────────────────────────
+        // Phase 1B / Rec 7: the hot-path CLI write always marks the memory as
+        // `captureMode: hot` and `beliefState: asserted`. Ranking applies a
+        // hot-capture boost so user-supplied memories outrank otherwise-equal
+        // background-derived ones.
+        const frontmatterBlock = buildMemoryFrontmatter({
+            description,
+            tags,
+            source,
+            observed_at,
+            expires,
+            subjective,
+            captureMode: "hot",
+            beliefState: "asserted",
+            ...(hasScope ? { scope: scopeFields } : {}),
+        });
+        const contentWithFrontmatter = `${frontmatterBlock}\n${body}`;
+        const result = await writeMarkdownAsset({
+            type: "memory",
+            content: contentWithFrontmatter,
+            name: args.name,
+            fallbackPrefix: "memory",
+            force: args.force,
+            target: args.target,
+            path: args.path,
+        });
+        appendEvent({
+            eventType: "remember",
+            ref: result.ref,
+            metadata: {
+                path: result.path,
+                force: args.force === true,
+                tagCount: tags.length,
+                enriched: args.enrich === true,
+                auto: args.auto === true,
+                ...(hasScope ? { scope: scopeFields } : {}),
+            },
         });
+        if (args.showSimilar) {
+            const similar = await fetchSimilarMemories((body ?? args.content ?? "").slice(0, 500), result.ref);
+            output("remember", { ok: true, ...result, similar });
+        }
+        else {
+            output("remember", { ok: true, ...result });
+        }
     },
 });

package/dist/commands/read/search-cli.js

@@ -21,13 +21,15 @@ import { getHyphenatedBoolean, getOutputMode, parseFlagValue } from "../../outpu
 import { akmCurate } from "./curate.js";
 import { akmSearch, parseBeliefFilterMode, parseScopeFilterFlags, parseSearchSource } from "./search.js";
 import { akmShowUnified } from "./show.js";
-// AKM_EVENT_SOURCE attributes a query to a `user` invocation or the internal
-// `improve` loop so the event log can distinguish them; any other value is
-// treated as unset.
+// AKM_EVENT_SOURCE attributes a query to a `user` invocation, the internal
+// `improve` loop, or the `task` runner so the event log can distinguish
+// genuine demand from machine traffic; any other value is treated as unset.
 function resolveEventSource() {
     const raw = process.env.AKM_EVENT_SOURCE;
     if (raw === "improve")
         return "improve";
+    if (raw === "task")
+        return "task";
     if (raw === "user")
         return "user";
     return undefined;
@@ -63,6 +65,11 @@ export const searchCommand = defineJsonCommand({
             description: "Disable the automatic project-context ranking boost (also disabled by AKM_DISABLE_PROJECT_CONTEXT=1).",
             default: false,
         },
+        "include-sessions": {
+            type: "boolean",
+            description: "Include session assets (excluded from default search results via config.search.defaultExcludeTypes).",
+            default: false,
+        },
     },
     async run({ args }) {
         const query = (args.query ?? "").trim();
@@ -79,10 +86,7 @@ export const searchCommand = defineJsonCommand({
         const includeProposed = args["include-proposed"] === true;
         const belief = parseBeliefFilterMode(typeof args.belief === "string" ? args.belief : undefined);
         const noProjectContext = getHyphenatedBoolean(args, "no-project-context");
-        // --no-project-context sets env so searchDatabase picks it up without
-        // threading the flag through the entire call stack.
-        if (noProjectContext)
-            process.env.AKM_DISABLE_PROJECT_CONTEXT = "1";
+        const includeSessions = args["include-sessions"];
         const result = await akmSearch({
             query,
             type,
@@ -91,6 +95,9 @@ export const searchCommand = defineJsonCommand({
             filters,
             includeProposed,
             belief,
+            includeSessions,
+            disableProjectContext: noProjectContext,
+            disableScopedUtility: noProjectContext,
             eventSource: resolveEventSource(),
         });
         output("search", result);
@@ -124,7 +131,7 @@ export const curateCommand = defineJsonCommand({
         const limitParsed = parsePositiveIntFlag(args.limit ?? undefined);
         const limit = limitParsed && limitParsed > 0 ? limitParsed : 4;
         const source = parseSearchSource(args.source ?? "stash");
-        const curated = await akmCurate({ query: args.query, type, limit, source });
+        const curated = await akmCurate({ query: args.query, type, limit, source, eventSource: resolveEventSource() });
         output("curate", curated);
     },
 });

package/dist/commands/read/search.js

@@ -16,14 +16,14 @@ import { rethrowIfTestIsolationError, UsageError } from "../../core/errors.js";
 import { appendEvent } from "../../core/events.js";
 import { isTransientStashPath } from "../../core/paths.js";
 import { bumpUtilityScoresBatch, getEntryIdByFilePath } from "../../indexer/db/db.js";
+import { resolveReadSources } from "../../indexer/read-preflight.js";
 import { searchLocal } from "../../indexer/search/db-search.js";
-import { resolveSourceEntries } from "../../indexer/search/search-source.js";
 import { getCurrentWorkflowScopeKey } from "../../workflows/authoring/scope-key.js";
 // Eagerly import source providers to trigger self-registration before the
 // indexer or path-resolution code runs.
 import "../../sources/providers/index.js";
 import { insertUsageEvent } from "../../indexer/usage/usage-events.js";
-import { withIndexDb } from "../../storage/repositories/index-db.js";
+import { TELEMETRY_BUSY_TIMEOUT_MS, withIndexDb } from "../../storage/repositories/index-db.js";
 import { searchRegistry } from "./registry-search.js";
 const DEFAULT_LIMIT = 20;
 export async function akmSearch(input) {
@@ -57,7 +57,7 @@ export async function akmSearch(input) {
     else {
         source = parsedSource;
     }
-    let allSources = resolveSourceEntries(undefined, config);
+    let allSources = resolveReadSources(undefined, config).sources;
     // When a named source was requested, narrow the sources list to just that entry.
     // `resolveSourceEntries` sets `registryId` to `entry.name` for each config source.
     if (namedSourceName !== undefined) {
@@ -108,6 +108,9 @@ export async function akmSearch(input) {
             // Without this, the index (which spans every configured source)
             // would leak hits from sources the caller did not request.
             restrictToSources: namedSourceName !== undefined,
+            includeExcludedTypes: input.includeSessions === true,
+            disableProjectContext: input.disableProjectContext === true,
+            disableScopedUtility: input.disableScopedUtility === true,
         });
     const registryResult = source === "stash" ? undefined : await searchRegistry(query, { limit, registries: config.registries });
     if (source === "stash") {
@@ -122,8 +125,9 @@ export async function akmSearch(input) {
             warnings: localResult?.warnings?.length ? localResult.warnings : undefined,
             timing: { totalMs: Date.now() - t0, rankMs: localResult?.rankMs, embedMs: localResult?.embedMs },
         };
-        if (!input.skipLogging)
-            logSearchEvent(query, response, localResult?.mode ?? "keyword", input.eventSource);
+        if (!input.skipLogging) {
+            logSearchEvent(query, response, localResult?.mode ?? "keyword", input.eventSource, input.disableScopedUtility === true);
+        }
         return response;
     }
     const registryHits = (registryResult?.hits ?? []).map((hit) => {
@@ -158,7 +162,7 @@ export async function akmSearch(input) {
             timing: { totalMs: Date.now() - t0 },
         };
         if (!input.skipLogging)
-            logSearchEvent(query, response, undefined, input.eventSource);
+            logSearchEvent(query, response, undefined, input.eventSource, input.disableScopedUtility === true);
         return response;
     }
     // source === "both"
@@ -176,7 +180,7 @@ export async function akmSearch(input) {
         timing: { totalMs: Date.now() - t0 },
     };
     if (!input.skipLogging)
-        logSearchEvent(query, response, undefined, input.eventSource);
+        logSearchEvent(query, response, undefined, input.eventSource, input.disableScopedUtility === true);
     return response;
 }
 /**
@@ -211,7 +215,7 @@ function resolveEntryIds(db, hits) {
  * Per-entry events are recorded only for stash hits because registry hits
  * have no local entry_id to reference.
  */
-function logSearchEvent(query, response, mode = "keyword", eventSource = "user") {
+function logSearchEvent(query, response, mode = "keyword", eventSource = "user", disableScopedUtility = false) {
     // Emit a structured event to events.jsonl so workflow-trace consumers
     // detect akm search invocations without relying on stdout scraping.
     const stashHits = response.hits.filter((h) => h.type !== "registry");
@@ -223,6 +227,9 @@ function logSearchEvent(query, response, mode = "keyword", eventSource = "user")
         metadata: { query, hitCount: stashHits.length, resultRefs: allResultRefs, mode },
     });
     try {
+        // Short busy timeout: telemetry must never stall the search result behind
+        // a background reindex holding the index.db write lock (30s default wait).
+        // Under contention these usage hints are skipped, not waited for.
         withIndexDb((db) => {
             const resolved = resolveEntryIds(db, stashHits.slice(0, 50));
             for (const { entryId, ref } of resolved) {
@@ -236,12 +243,16 @@ function logSearchEvent(query, response, mode = "keyword", eventSource = "user")
             }
             // Bump utility scores for all resolved entries (MemRL retrieval signal).
             // The indexer overwrites these at next reindex; bumps are temporary hints.
-            const resolvedIds = resolved.map((r) => r.entryId).filter((id) => id !== undefined);
+            // Gated to user-sourced events: pipeline searches (improve probes, task
+            // runner) must not feed the utility signal (meta-review 05 DRIFT-6 —
+            // the bump previously fired unconditionally, so even correctly-tagged
+            // machine traffic inflated utility).
+            const resolvedIds = eventSource === "user" ? resolved.map((r) => r.entryId).filter((id) => id !== undefined) : [];
             if (resolvedIds.length > 0) {
                 let scopeKey;
                 try {
                     const stashPath = response.stashDir;
-                    const disabled = process.env.AKM_DISABLE_SCOPED_UTILITY === "1" || (stashPath && isTransientStashPath(stashPath));
+                    const disabled = disableScopedUtility || (stashPath && isTransientStashPath(stashPath));
                     scopeKey = disabled ? undefined : getCurrentWorkflowScopeKey();
                 }
                 catch {
@@ -265,7 +276,7 @@ function logSearchEvent(query, response, mode = "keyword", eventSource = "user")
                 }),
                 source: eventSource,
             });
-        });
+        }, { busyTimeoutMs: TELEMETRY_BUSY_TIMEOUT_MS });
     }
     catch (err) {
         rethrowIfTestIsolationError(err);