@aiwayds/dsh-topics-memory 0.6.0

package/lib/paths.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Path resolution for the local bundle (ADR 0008): the Cache lives at
+ * `~/.dsh/topics/` (override with $DSH_TOPICS_HOME for tests), laid out as
+ * an OKF bundle:
+ *
+ *   <root>/topics/<slug>.md      concept documents
+ *   <root>/index.md              auto-generated bundle index
+ *   <root>/meta/observations.jsonl   undistilled observations (M2)
+ *   <root>/meta/injections.jsonl     injection log (ADR 0007)
+ *   <root>/meta/conflicts.json       rebase-conflicted topics (ADR 0003)
+ *
+ * @module paths
+ */
+export declare function resolveDshHome(): string;
+export declare function resolveBundleRoot(): string;
+/**
+ * Minimal fs surface, injectable so tests can hit the rename-race catch
+ * deterministically (a real-fs test cannot mutate the world between the
+ * guard checks and the rename). Defaults to the real node:fs functions —
+ * production callers never pass it.
+ */
+interface FsLike {
+    existsSync(path: string): boolean;
+    renameSync(from: string, to: string): void;
+}
+/**
+ * One-time migration from the pre-rename data location (`~/.dsh/llmwiki` to
+ * `~/.dsh/topics`, ADR 0013). Constraints:
+ * - one-time: fires only while the new directory is absent and the legacy one
+ *   exists; after the rename (or a fresh bundle on the new path) the check is
+ *   a no-op;
+ * - fail-open: any rename error falls back to the legacy path so the plugin
+ *   keeps serving the old bundle instead of booting an empty new one;
+ * - bypassed entirely by an explicit $DSH_TOPICS_HOME (test/CI isolation).
+ */
+export declare function migrateLegacyBundleRoot(next: string, legacy: string, fs?: FsLike): string;
+export declare function topicsDir(root: string): string;
+export declare function metaDir(root: string): string;
+export declare function topicFile(root: string, slug: string): string;
+export declare function indexFile(root: string): string;
+export declare function observationsFile(root: string): string;
+export declare function injectionsFile(root: string): string;
+export declare function conflictsFile(root: string): string;
+/** Sanitized host name recorded in `generated.by` for multi-machine provenance. */
+export declare function hostId(): string;
+/** The actor string stamped into `generated.by` for machine-written topics. */
+export declare function actorFor(): string;
+export {};

package/lib/paths.js

@@ -0,0 +1,84 @@
+/**
+ * Path resolution for the local bundle (ADR 0008): the Cache lives at
+ * `~/.dsh/topics/` (override with $DSH_TOPICS_HOME for tests), laid out as
+ * an OKF bundle:
+ *
+ *   <root>/topics/<slug>.md      concept documents
+ *   <root>/index.md              auto-generated bundle index
+ *   <root>/meta/observations.jsonl   undistilled observations (M2)
+ *   <root>/meta/injections.jsonl     injection log (ADR 0007)
+ *   <root>/meta/conflicts.json       rebase-conflicted topics (ADR 0003)
+ *
+ * @module paths
+ */
+import { existsSync, renameSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { hostname as osHostname } from 'node:os';
+import { join } from 'node:path';
+export function resolveDshHome() {
+    const env = process.env.DSH_HOME?.trim();
+    if (env !== undefined && env !== '')
+        return env;
+    return join(homedir(), '.dsh');
+}
+export function resolveBundleRoot() {
+    const env = process.env.DSH_TOPICS_HOME?.trim();
+    if (env !== undefined && env !== '')
+        return env;
+    const dshHome = resolveDshHome();
+    return migrateLegacyBundleRoot(join(dshHome, 'topics'), join(dshHome, 'llmwiki'));
+}
+/**
+ * One-time migration from the pre-rename data location (`~/.dsh/llmwiki` to
+ * `~/.dsh/topics`, ADR 0013). Constraints:
+ * - one-time: fires only while the new directory is absent and the legacy one
+ *   exists; after the rename (or a fresh bundle on the new path) the check is
+ *   a no-op;
+ * - fail-open: any rename error falls back to the legacy path so the plugin
+ *   keeps serving the old bundle instead of booting an empty new one;
+ * - bypassed entirely by an explicit $DSH_TOPICS_HOME (test/CI isolation).
+ */
+export function migrateLegacyBundleRoot(next, legacy, fs = { existsSync, renameSync }) {
+    if (fs.existsSync(next) || !fs.existsSync(legacy))
+        return next;
+    try {
+        fs.renameSync(legacy, next);
+        return next;
+    }
+    catch {
+        // A concurrent boot may have won the rename: re-check the live filesystem
+        // instead of trusting the stale pre-rename view — falling back to `legacy`
+        // after it was moved away would strand this session on a dead path (empty
+        // bundle, writes into an orphan directory nothing will ever read).
+        return fs.existsSync(next) ? next : legacy;
+    }
+}
+export function topicsDir(root) {
+    return join(root, 'topics');
+}
+export function metaDir(root) {
+    return join(root, 'meta');
+}
+export function topicFile(root, slug) {
+    return join(root, 'topics', `${slug}.md`);
+}
+export function indexFile(root) {
+    return join(root, 'index.md');
+}
+export function observationsFile(root) {
+    return join(metaDir(root), 'observations.jsonl');
+}
+export function injectionsFile(root) {
+    return join(metaDir(root), 'injections.jsonl');
+}
+export function conflictsFile(root) {
+    return join(metaDir(root), 'conflicts.json');
+}
+/** Sanitized host name recorded in `generated.by` for multi-machine provenance. */
+export function hostId() {
+    return osHostname().replace(/[^A-Za-z0-9._-]/g, '-').replace(/^-+|-+$/g, '') || 'unknown-host';
+}
+/** The actor string stamped into `generated.by` for machine-written topics. */
+export function actorFor() {
+    return `agent:dsh-topics-memory@${hostId()}`;
+}

package/lib/retrieval.d.ts

@@ -0,0 +1,61 @@
+/**
+ * Hot-path retrieval — LLM-free, millisecond-scale (ADR 0004).
+ *
+ * Lineage: pi-topic-memory's tokenizer + Dice matching, adapted to the Topic
+ * profile. Query tokens are matched against title/tags/slug/description/
+ * conclusion via containment (|query ∩ field| / |query|), so long topics are
+ * not penalized the way plain Dice dilutes them. Tag hits add a bounded
+ * boost, recent topics a small recency bonus, and the `depends` graph expands
+ * seeds to adjacent topics with geometric decay. Conflicted topics are
+ * demoted, never dropped (ADR 0003).
+ *
+ * @module retrieval
+ */
+import type { TopicMeta } from './store.ts';
+export interface RetrievableTopic extends TopicMeta {
+    /** `# Conclusion` section text (may be empty). */
+    conclusion: string;
+    /** Body-referenced topic slugs ([[wikilinks]] + markdown links) — graph edges. */
+    links?: string[];
+}
+export interface RetrievalConfig {
+    threshold: number;
+    topK: number;
+    tagBoost: number;
+    graphDepth: number;
+    recencyWindowDays: number;
+    now?: Date;
+    /** Slugs currently conflicted — demoted, not dropped. */
+    conflicts?: ReadonlySet<string>;
+}
+export declare const DEFAULT_RETRIEVAL: RetrievalConfig;
+export interface SearchHit {
+    slug: string;
+    score: number;
+    reasons: string[];
+    viaGraph: boolean;
+}
+export interface SearchOutcome {
+    hits: SearchHit[];
+    /** Below threshold but close — the tuning evidence for /topics stats (ADR 0007). */
+    nearMisses: SearchHit[];
+    rosterSize: number;
+}
+/**
+ * Tokenize for matching: latin/digit words (≥2 chars) plus CJK character
+ * bigrams (single char kept when the run has length 1).
+ */
+export declare function tokenize(text: string): string[];
+export interface ScoredTopic {
+    slug: string;
+    score: number;
+    reasons: string[];
+}
+/** Score one topic against the query tokens. */
+export declare function scoreTopic(queryTokens: ReadonlySet<string>, topic: RetrievableTopic, cfg: RetrievalConfig): ScoredTopic;
+/**
+ * Search the roster: score every topic, split hits from near-misses, then
+ * expand seeds through the `depends` graph (both directions, geometric decay,
+ * depth ≤ cfg.graphDepth). Expanded entries enter at the back of the hits.
+ */
+export declare function searchTopics(query: string, roster: readonly RetrievableTopic[], cfg?: Partial<RetrievalConfig>): SearchOutcome;

package/lib/retrieval.js

@@ -0,0 +1,178 @@
+/**
+ * Hot-path retrieval — LLM-free, millisecond-scale (ADR 0004).
+ *
+ * Lineage: pi-topic-memory's tokenizer + Dice matching, adapted to the Topic
+ * profile. Query tokens are matched against title/tags/slug/description/
+ * conclusion via containment (|query ∩ field| / |query|), so long topics are
+ * not penalized the way plain Dice dilutes them. Tag hits add a bounded
+ * boost, recent topics a small recency bonus, and the `depends` graph expands
+ * seeds to adjacent topics with geometric decay. Conflicted topics are
+ * demoted, never dropped (ADR 0003).
+ *
+ * @module retrieval
+ */
+import { dependsSlugs } from "./okf.js";
+export const DEFAULT_RETRIEVAL = {
+    threshold: 0.3,
+    topK: 4,
+    tagBoost: 0.15,
+    graphDepth: 2,
+    recencyWindowDays: 7,
+};
+const CJK_RUN = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]+/gu;
+const WORD_RUN = /[\p{Script=Latin}\p{Script=Greek}\p{Script=Cyrillic}\p{Nd}][\p{Script=Latin}\p{Script=Greek}\p{Script=Cyrillic}\p{Nd}+#.-]*/gu;
+/**
+ * Tokenize for matching: latin/digit words (≥2 chars) plus CJK character
+ * bigrams (single char kept when the run has length 1).
+ */
+export function tokenize(text) {
+    const out = new Set();
+    for (const w of text.toLowerCase().match(WORD_RUN) ?? []) {
+        if (w.length >= 2)
+            out.add(w);
+    }
+    for (const run of text.match(CJK_RUN) ?? []) {
+        if (run.length === 1) {
+            out.add(run);
+            continue;
+        }
+        for (let i = 0; i < run.length - 1; i += 1)
+            out.add(run.slice(i, i + 2));
+    }
+    return [...out];
+}
+function containment(query, field) {
+    if (query.size === 0 || field.size === 0)
+        return 0;
+    let hits = 0;
+    for (const t of query)
+        if (field.has(t))
+            hits += 1;
+    return hits / query.size;
+}
+function overlap(a, b) {
+    for (const t of a)
+        if (b.has(t))
+            return true;
+    return false;
+}
+/** Score one topic against the query tokens. */
+export function scoreTopic(queryTokens, topic, cfg) {
+    const title = new Set(tokenize(topic.title));
+    const slug = new Set(tokenize(topic.slug));
+    const tags = new Set(topic.tags.flatMap((t) => tokenize(t)));
+    const description = new Set(tokenize(topic.description ?? ''));
+    const conclusion = new Set(tokenize(topic.conclusion));
+    const reasons = [];
+    let score = 0;
+    const cTitle = containment(queryTokens, title);
+    if (cTitle > 0) {
+        score += 3 * cTitle;
+        reasons.push(`title:${cTitle.toFixed(2)}`);
+    }
+    const cTag = containment(queryTokens, new Set([...tags, ...slug]));
+    if (cTag > 0) {
+        score += 2 * cTag;
+        reasons.push(`tags:${cTag.toFixed(2)}`);
+    }
+    const cDesc = containment(queryTokens, description);
+    if (cDesc > 0) {
+        score += 1.2 * cDesc;
+        reasons.push(`description:${cDesc.toFixed(2)}`);
+    }
+    const cConc = containment(queryTokens, conclusion);
+    if (cConc > 0) {
+        score += 0.8 * cConc;
+        reasons.push(`conclusion:${cConc.toFixed(2)}`);
+    }
+    // Exact tag hit boost, additive and capped (pi-topic-memory lesson: add,
+    // never fold into the token set — dilution breaks the guard).
+    let tagHits = 0;
+    for (const t of tags) {
+        if (overlap(queryTokens, new Set([...tokenize(t)])))
+            tagHits += 1;
+    }
+    if (tagHits > 0) {
+        const boost = Math.min(cfg.tagBoost * tagHits, cfg.tagBoost * 3);
+        score += boost;
+        reasons.push(`tag-boost:+${boost.toFixed(2)}`);
+    }
+    // Recency is a tiebreaker among relevant topics, never a free pass: only
+    // fires when the query already matched some content above.
+    if (score > 0 && cfg.recencyWindowDays > 0) {
+        const generated = Date.parse(topic.generatedAt);
+        if (!Number.isNaN(generated)) {
+            const ageDays = ((cfg.now?.getTime() ?? Date.now()) - generated) / 86_400_000;
+            if (ageDays >= 0 && ageDays <= cfg.recencyWindowDays) {
+                score += 0.2;
+                reasons.push('recency');
+            }
+        }
+    }
+    if (cfg.conflicts?.has(topic.slug) === true) {
+        score *= 0.3;
+        reasons.push('conflicted-demoted');
+    }
+    return { slug: topic.slug, score: Math.round(score * 1000) / 1000, reasons };
+}
+/**
+ * Search the roster: score every topic, split hits from near-misses, then
+ * expand seeds through the `depends` graph (both directions, geometric decay,
+ * depth ≤ cfg.graphDepth). Expanded entries enter at the back of the hits.
+ */
+export function searchTopics(query, roster, cfg = {}) {
+    const c = { ...DEFAULT_RETRIEVAL, ...cfg };
+    const queryTokens = new Set(tokenize(query));
+    if (queryTokens.size === 0 || roster.length === 0) {
+        return { hits: [], nearMisses: [], rosterSize: roster.length };
+    }
+    const scored = roster.map((t) => scoreTopic(queryTokens, t, c)).sort((a, b) => b.score - a.score);
+    const direct = scored.filter((s) => s.score >= c.threshold).slice(0, c.topK);
+    const near = scored.filter((s) => s.score < c.threshold && s.score >= c.threshold * 0.5).slice(0, 8);
+    const hits = direct.map((s) => ({ ...s, viaGraph: false }));
+    // Graph expansion (ADR 0005 graph-shaped bundle): walk depends + body
+    // links, both directions, from the seeds; include up to topK+2 slots at
+    // decaying score.
+    if (c.graphDepth > 0 && hits.length > 0) {
+        const bySlug = new Map(roster.map((r) => [r.slug, r]));
+        const neighborsOf = (slug) => {
+            const topic = bySlug.get(slug);
+            if (topic === undefined)
+                return [];
+            const out = new Set(dependsSlugs({ depends: topic.depends }));
+            for (const l of topic.links ?? [])
+                out.add(l);
+            return [...out];
+        };
+        const dependents = new Map();
+        for (const r of roster) {
+            for (const dep of new Set([...dependsSlugs({ depends: r.depends }), ...(r.links ?? [])])) {
+                const list = dependents.get(dep) ?? [];
+                list.push(r.slug);
+                dependents.set(dep, list);
+            }
+        }
+        const seen = new Set(hits.map((h) => h.slug));
+        let frontier = hits.map((h) => ({ slug: h.slug, score: h.score }));
+        for (let depth = 1; depth <= c.graphDepth && frontier.length > 0; depth += 1) {
+            const next = [];
+            for (const node of frontier) {
+                const neighbors = new Set([...neighborsOf(node.slug), ...(dependents.get(node.slug) ?? [])]);
+                for (const neighbor of neighbors) {
+                    if (seen.has(neighbor))
+                        continue;
+                    seen.add(neighbor);
+                    const decayed = node.score * 0.5 ** depth;
+                    if (decayed >= c.threshold && hits.length < c.topK + 2) {
+                        const already = hits.find((h) => h.slug === neighbor);
+                        if (already === undefined)
+                            hits.push({ slug: neighbor, score: Math.round(decayed * 1000) / 1000, reasons: [`depends:d${depth}`], viaGraph: true });
+                        next.push({ slug: neighbor, score: decayed });
+                    }
+                }
+            }
+            frontier = next;
+        }
+    }
+    return { hits, nearMisses: near.map((s) => ({ ...s, viaGraph: false })), rosterSize: roster.length };
+}

package/lib/service.d.ts

@@ -0,0 +1,96 @@
+/**
+ * TopicsService — the facade shared by tools, commands, the injection seam,
+ * and the distill lane. Owns the roster cache (mtime-keyed, pi-topic-memory
+ * lesson #3: never re-parse unchanged files on the hot path).
+ *
+ * @module service
+ */
+import * as okf from './okf.ts';
+import { type RetrievableTopic, type SearchOutcome } from './retrieval.ts';
+import { type AssembleResult } from './digest.ts';
+import type { BundleStore, Observation, SaveResult } from './store.ts';
+import type { TopicsConfigValue } from './config.ts';
+import { type AggregateStats } from './ilog.ts';
+import type { Sync } from './sync.ts';
+export interface RetrieveOutcome {
+    outcome: SearchOutcome;
+    injection: AssembleResult;
+    /** Roster was empty or injection disabled — no record written for pure no-ops. */
+    recorded: boolean;
+}
+export declare class TopicsService {
+    private cache;
+    readonly store: BundleStore;
+    private readonly getConfig;
+    readonly sync?: Sync;
+    constructor(store: BundleStore, getConfig: () => TopicsConfigValue, sync?: Sync);
+    get cfg(): TopicsConfigValue;
+    get githubMode(): boolean;
+    /** All retrievable topics with conclusion text, mtime-cached. */
+    roster(): Promise<RetrievableTopic[]>;
+    /** Drop the roster cache (tests, bulk external edits). */
+    invalidate(): void;
+    /**
+     * Full retrieval round: search → assemble → log (ADR 0006/0007). The
+     * injection record is written even for zero-hit rounds — the hit-rate
+     * denominator must count every round (near-miss evidence).
+     */
+    retrieve(query: string, sessionId?: string): Promise<RetrieveOutcome>;
+    /**
+     * SYNCHRONOUS hot path for same-turn injection (the chancelu lesson):
+     * `agent/inbox/spliced` dispatches before prompt assembly and the context
+     * provider reads the assembled text synchronously — an async retrieval
+     * would always lose the race and inject nothing. Reads files with sync fs
+     * behind the mtime cache; the log record is written fire-and-forget.
+     *
+     * `dedup.exclude` (session-level injection dedup) filters hits AFTER
+     * retrieval and BEFORE assembly: excluded slugs are reported back as
+     * `deduped` (and logged as `record.deduped`), never packed. `included`
+     * mirrors what actually entered the context — only these may be marked
+     * as injected by the caller; budget-dropped slugs stay injectable later.
+     */
+    retrieveSync(query: string, sessionId?: string, dedup?: {
+        exclude?: ReadonlySet<string>;
+    }): {
+        text: string;
+        outcome: SearchOutcome;
+        deduped: string[];
+        included: string[];
+    };
+    /** Sync roster read (same mtime cache as roster()). */
+    rosterSync(): RetrievableTopic[];
+    private readDocSync;
+    private readDoc;
+    /** topic_save tool path: create or update with actor stamping. */
+    saveTopic(input: {
+        title: string;
+        description?: string;
+        tags?: string[];
+        depends?: string[];
+        openQuestions?: string[];
+        impact?: string[];
+        status?: okf.TopicStatus;
+        conclusion?: string;
+        recommendations?: string;
+        source?: 'model' | 'distill';
+        slug?: string;
+    }): Promise<SaveResult & {
+        slug: string;
+    }>;
+    observe(input: {
+        kind: Observation['kind'];
+        text: string;
+        sessionId?: string;
+        source?: 'model' | 'auto';
+    }): Promise<Observation>;
+    search(query: string, topK?: number): Promise<SearchOutcome>;
+    history(slug: string, limit?: number): Promise<{
+        entries: {
+            hash: string;
+            date: string;
+            message: string;
+            conclusion?: string;
+        }[];
+    }>;
+    stats(): Promise<AggregateStats>;
+}