@aiwayds/dsh-topics-memory 0.6.0

package/lib/service.js

@@ -0,0 +1,375 @@
+/**
+ * TopicsService — the facade shared by tools, commands, the injection seam,
+ * and the distill lane. Owns the roster cache (mtime-keyed, pi-topic-memory
+ * lesson #3: never re-parse unchanged files on the hot path).
+ *
+ * @module service
+ */
+import { readFile, stat } from 'node:fs/promises';
+import { readdirSync, readFileSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import * as okf from "./okf.js";
+import { searchTopics, tokenize } from "./retrieval.js";
+import { assembleInjection } from "./digest.js";
+import { aggregateStats, querySample } from "./ilog.js";
+import { fileHistory, fileAtRev } from "./git.js";
+export class TopicsService {
+    cache = new Map();
+    store;
+    getConfig;
+    sync;
+    constructor(store, getConfig, sync) {
+        this.store = store;
+        this.getConfig = getConfig;
+        this.sync = sync;
+    }
+    get cfg() {
+        return this.getConfig();
+    }
+    get githubMode() {
+        return this.cfg.repo !== '';
+    }
+    /** All retrievable topics with conclusion text, mtime-cached. */
+    async roster() {
+        let files;
+        try {
+            files = (await this.store.listTopics()).map((m) => `${m.slug}.md`);
+        }
+        catch {
+            return [];
+        }
+        const out = [];
+        for (const file of files) {
+            const path = join(this.store.topicsDir(), file);
+            let st;
+            try {
+                st = await stat(path);
+            }
+            catch {
+                continue;
+            }
+            const cached = this.cache.get(file);
+            let entry = cached;
+            if (cached === undefined || cached.mtimeMs !== st.mtimeMs || cached.size !== st.size) {
+                try {
+                    const doc = okf.parseTopicDoc(await readFile(path, 'utf8'));
+                    entry = { mtimeMs: st.mtimeMs, size: st.size, doc, slug: file.slice(0, -3) };
+                    this.cache.set(file, entry);
+                }
+                catch {
+                    this.cache.delete(file);
+                    continue;
+                }
+            }
+            if (entry === undefined)
+                continue;
+            const doc = entry.doc;
+            out.push({
+                slug: entry.slug,
+                title: doc.fm.title,
+                description: doc.fm.description,
+                status: doc.fm.status,
+                tags: doc.fm.tags,
+                depends: doc.fm.depends,
+                links: okf.bodyLinkSlugs(doc.body),
+                generatedAt: doc.fm.generated.at,
+                conclusion: okf.sectionOf(doc.body, okf.CONCLUSION_HEADING) ?? '',
+            });
+        }
+        return out;
+    }
+    /** Drop the roster cache (tests, bulk external edits). */
+    invalidate() {
+        this.cache.clear();
+    }
+    /**
+     * Full retrieval round: search → assemble → log (ADR 0006/0007). The
+     * injection record is written even for zero-hit rounds — the hit-rate
+     * denominator must count every round (near-miss evidence).
+     */
+    async retrieve(query, sessionId) {
+        const cfg = this.cfg;
+        const roster = await this.roster();
+        if (!cfg.autoInject || query.trim() === '' || roster.length === 0) {
+            return {
+                outcome: { hits: [], nearMisses: [], rosterSize: roster.length },
+                injection: { text: '', usedTokens: 0, included: [], dropped: [] },
+                recorded: false,
+            };
+        }
+        const conflicts = await this.store.getConflicts();
+        const outcome = searchTopics(query, roster, {
+            threshold: cfg.matchThreshold,
+            topK: cfg.topK,
+            tagBoost: cfg.tagBoost,
+            graphDepth: cfg.graphDepth,
+            recencyWindowDays: cfg.recencyWindowDays,
+            conflicts,
+        });
+        const bySlug = new Map(roster.map((r) => [r.slug, r]));
+        const entries = [];
+        for (const hit of outcome.hits) {
+            const topic = bySlug.get(hit.slug);
+            if (topic === undefined)
+                continue;
+            const doc = await this.readDoc(hit.slug);
+            if (doc !== undefined)
+                entries.push({ slug: hit.slug, doc, hit });
+        }
+        const injection = assembleInjection(entries, {
+            perTopicBudget: cfg.perTopicBudget,
+            totalBudget: cfg.totalBudget,
+        });
+        const record = {
+            at: new Date().toISOString(),
+            queryTokenCount: tokenize(query).length,
+            querySample: querySample(query),
+            rosterSize: outcome.rosterSize,
+            hits: outcome.hits.map((h) => ({ slug: h.slug, score: h.score, reasons: h.reasons, viaGraph: h.viaGraph })),
+            nearMisses: outcome.nearMisses.map((h) => ({ slug: h.slug, score: h.score })),
+            injected: injection.text !== '',
+            usedTokens: injection.usedTokens,
+        };
+        if (sessionId !== undefined)
+            record.sessionId = sessionId;
+        if (injection.text === '' && outcome.hits.length > 0)
+            record.why = 'below-budget-or-dropped';
+        if (inclusionDropped(injection))
+            record.dropped = injection.dropped;
+        await this.store.appendInjectionRecord(record);
+        return { outcome, injection, recorded: true };
+    }
+    /**
+     * SYNCHRONOUS hot path for same-turn injection (the chancelu lesson):
+     * `agent/inbox/spliced` dispatches before prompt assembly and the context
+     * provider reads the assembled text synchronously — an async retrieval
+     * would always lose the race and inject nothing. Reads files with sync fs
+     * behind the mtime cache; the log record is written fire-and-forget.
+     *
+     * `dedup.exclude` (session-level injection dedup) filters hits AFTER
+     * retrieval and BEFORE assembly: excluded slugs are reported back as
+     * `deduped` (and logged as `record.deduped`), never packed. `included`
+     * mirrors what actually entered the context — only these may be marked
+     * as injected by the caller; budget-dropped slugs stay injectable later.
+     */
+    retrieveSync(query, sessionId, dedup) {
+        const cfg = this.cfg;
+        const roster = this.rosterSync();
+        const empty = { hits: [], nearMisses: [], rosterSize: roster.length };
+        if (!cfg.autoInject || query.trim() === '' || roster.length === 0) {
+            return { text: '', outcome: empty, deduped: [], included: [] };
+        }
+        const conflicts = this.store.getConflictsSync();
+        const outcome = searchTopics(query, roster, {
+            threshold: cfg.matchThreshold,
+            topK: cfg.topK,
+            tagBoost: cfg.tagBoost,
+            graphDepth: cfg.graphDepth,
+            recencyWindowDays: cfg.recencyWindowDays,
+            conflicts,
+        });
+        const exclude = dedup?.exclude;
+        const deduped = exclude === undefined ? [] : outcome.hits.filter((h) => exclude.has(h.slug)).map((h) => h.slug);
+        const entries = [];
+        for (const hit of outcome.hits) {
+            if (exclude?.has(hit.slug))
+                continue;
+            const doc = this.readDocSync(hit.slug);
+            if (doc !== undefined)
+                entries.push({ slug: hit.slug, doc, hit });
+        }
+        const injection = assembleInjection(entries, {
+            perTopicBudget: cfg.perTopicBudget,
+            totalBudget: cfg.totalBudget,
+        });
+        const record = {
+            at: new Date().toISOString(),
+            queryTokenCount: tokenize(query).length,
+            querySample: querySample(query),
+            rosterSize: outcome.rosterSize,
+            hits: outcome.hits.map((h) => ({ slug: h.slug, score: h.score, reasons: h.reasons, viaGraph: h.viaGraph })),
+            nearMisses: outcome.nearMisses.map((h) => ({ slug: h.slug, score: h.score })),
+            injected: injection.text !== '',
+            usedTokens: injection.usedTokens,
+        };
+        if (sessionId !== undefined)
+            record.sessionId = sessionId;
+        if (deduped.length > 0)
+            record.deduped = deduped;
+        if (injection.text === '' && outcome.hits.length > 0) {
+            // All hits deduped is a different story than a budget squeeze — say so.
+            record.why = deduped.length === outcome.hits.length ? 'dedup' : 'below-budget-or-dropped';
+        }
+        if (injection.dropped.length > 0)
+            record.dropped = injection.dropped;
+        void this.store.appendInjectionRecord(record).catch(() => undefined);
+        return { text: injection.text, outcome, deduped, included: injection.included };
+    }
+    /** Sync roster read (same mtime cache as roster()). */
+    rosterSync() {
+        let files;
+        try {
+            files = readdirSync(this.store.topicsDir());
+        }
+        catch {
+            return [];
+        }
+        const out = [];
+        for (const file of files) {
+            if (!file.endsWith('.md') || file === 'index.md')
+                continue;
+            const path = join(this.store.topicsDir(), file);
+            let st;
+            try {
+                st = statSync(path);
+            }
+            catch {
+                continue;
+            }
+            const cached = this.cache.get(file);
+            let entry = cached;
+            if (cached === undefined || cached.mtimeMs !== st.mtimeMs || cached.size !== st.size) {
+                try {
+                    const doc = okf.parseTopicDoc(readFileSync(path, 'utf8'));
+                    entry = { mtimeMs: st.mtimeMs, size: st.size, doc, slug: file.slice(0, -3) };
+                    this.cache.set(file, entry);
+                }
+                catch {
+                    this.cache.delete(file);
+                    continue;
+                }
+            }
+            if (entry === undefined)
+                continue;
+            const doc = entry.doc;
+            out.push({
+                slug: entry.slug,
+                title: doc.fm.title,
+                description: doc.fm.description,
+                status: doc.fm.status,
+                tags: doc.fm.tags,
+                depends: doc.fm.depends,
+                links: okf.bodyLinkSlugs(doc.body),
+                generatedAt: doc.fm.generated.at,
+                conclusion: okf.sectionOf(doc.body, okf.CONCLUSION_HEADING) ?? '',
+            });
+        }
+        return out;
+    }
+    readDocSync(slug) {
+        try {
+            return okf.parseTopicDoc(readFileSync(join(this.store.topicsDir(), `${slug}.md`), 'utf8'));
+        }
+        catch {
+            return undefined;
+        }
+    }
+    async readDoc(slug) {
+        try {
+            return await this.store.readTopic(slug);
+        }
+        catch {
+            return undefined;
+        }
+    }
+    /** topic_save tool path: create or update with actor stamping. */
+    async saveTopic(input) {
+        const now = new Date().toISOString();
+        const existing = input.slug !== undefined ? await this.store.readTopic(input.slug) : undefined;
+        let body = existing?.body ?? '';
+        if (input.conclusion !== undefined && input.conclusion !== '') {
+            body = okf.setSection(body, okf.CONCLUSION_HEADING, input.conclusion);
+        }
+        else if (existing === undefined) {
+            body = okf.setSection(body, okf.CONCLUSION_HEADING, input.description ?? input.title);
+        }
+        if (input.recommendations !== undefined && input.recommendations !== '') {
+            body = okf.setSection(body, okf.RECOMMENDATIONS_HEADING, input.recommendations);
+        }
+        const baseSlug = input.slug ?? okf.slugify(input.title);
+        const slug = existing === undefined ? await this.store.uniqueSlug(baseSlug) : okf.slugify(input.slug ?? baseSlug);
+        // store.saveTopic stamps generated {by, at} itself (write-through actor).
+        const fm = {
+            type: 'Topic',
+            title: input.title,
+            tags: dedupeLower(input.tags ?? existing?.fm.tags ?? []),
+            depends: (input.depends ?? existing?.fm.depends ?? []).map(okf.slugToPath),
+            open_questions: input.openQuestions ?? existing?.fm.open_questions ?? [],
+            impact: input.impact ?? existing?.fm.impact ?? [],
+            status: input.status ?? existing?.fm.status ?? 'draft',
+            generated: existing?.fm.generated ?? { by: 'pending', at: now },
+        };
+        if (input.description !== undefined && input.description !== '')
+            fm.description = input.description;
+        if (existing !== undefined && existing.fm.verified !== undefined)
+            fm.verified = existing.fm.verified;
+        const result = await this.store.saveTopic({ slug, doc: { fm, body } }, { message: existing === undefined ? `topics(topic): create ${slug}` : `topics(topic): update ${slug}`, created: existing === undefined });
+        this.invalidate();
+        void this.sync?.schedulePush();
+        return { ...result, slug: result.slug };
+    }
+    async observe(input) {
+        return this.store.appendObservation({ kind: input.kind, source: input.source ?? 'model', text: input.text, sessionId: input.sessionId });
+    }
+    async search(query, topK) {
+        const cfg = this.cfg;
+        return searchTopics(query, await this.roster(), {
+            threshold: cfg.matchThreshold,
+            topK: topK ?? Math.max(cfg.topK, 8),
+            tagBoost: cfg.tagBoost,
+            graphDepth: cfg.graphDepth,
+            recencyWindowDays: cfg.recencyWindowDays,
+            conflicts: await this.store.getConflicts(),
+        });
+    }
+    async history(slug, limit = 20) {
+        if (!(await this.store.hasGit()))
+            return { entries: [] };
+        const path = `topics/${okf.slugify(slug)}.md`;
+        const log = await fileHistory(this.store.root, path, limit);
+        const entries = [];
+        for (const e of log) {
+            const raw = await fileAtRev(this.store.root, path, e.hash);
+            let conclusion;
+            if (raw !== undefined) {
+                try {
+                    conclusion = firstLine(okf.sectionOf(okf.parseTopicDoc(raw).body, okf.CONCLUSION_HEADING) ?? '');
+                }
+                catch {
+                    conclusion = undefined;
+                }
+            }
+            entries.push({ hash: e.hash.slice(0, 10), date: e.date, message: e.message, conclusion });
+        }
+        return { entries };
+    }
+    async stats() {
+        return aggregateStats(await this.store.readInjectionRecords());
+    }
+}
+function inclusionDropped(injection) {
+    return injection.dropped.length > 0;
+}
+function firstLine(text) {
+    return firstParagraphOf(text);
+}
+function firstParagraphOf(text) {
+    for (const line of text.split('\n')) {
+        const t = line.trim();
+        if (t !== '')
+            return t;
+    }
+    return '';
+}
+function dedupeLower(tags) {
+    const seen = new Set();
+    const out = [];
+    for (const t of tags) {
+        const key = t.trim().toLowerCase();
+        if (key === '' || seen.has(key))
+            continue;
+        seen.add(key);
+        out.push(key);
+    }
+    return out;
+}

package/lib/store.d.ts

@@ -0,0 +1,153 @@
+/**
+ * Bundle store — reads and writes the local OKF bundle (the Cache).
+ *
+ * All writes go through a serialized queue: one git commit per topic save
+ * (write-through, ADR 0003) plus a regenerated `index.md`. Observations and
+ * the injection log are JSONL sidecars under `meta/`; they are committed on
+ * the flush cadence (sync layer), not per append.
+ *
+ * @module store
+ */
+import * as okf from './okf.ts';
+import type { InjectionRecord } from './ilog.ts';
+export interface TopicMeta {
+    slug: string;
+    title: string;
+    description?: string;
+    status: okf.TopicStatus;
+    tags: string[];
+    depends: string[];
+    generatedAt: string;
+}
+export type ObservationKind = 'decision' | 'finding' | 'constraint' | 'question' | 'turn';
+export interface Observation {
+    id: string;
+    at: string;
+    kind: ObservationKind;
+    source: 'model' | 'auto';
+    text: string;
+    sessionId?: string;
+    distilled: boolean;
+    distilledInto?: string[];
+    /** Failed distill attempts: fed to the model but consumed by no op (GC counter). */
+    attempts?: number;
+}
+export interface SaveResult {
+    slug: string;
+    path: string;
+    committed: boolean;
+    created: boolean;
+}
+export declare class StoreError extends Error {
+    name: string;
+}
+export interface BundleStoreOptions {
+    /** Override the stamped actor (tests). Defaults to `agent:dsh-topics-memory@<host>`. */
+    actor?: string;
+    /** Disable git operations entirely (pure-filesystem tests). */
+    gitDisabled?: boolean;
+}
+export declare class BundleStore {
+    readonly root: string;
+    private readonly actor;
+    private readonly gitDisabled;
+    /** Serializes all write paths; reads are lock-free. */
+    private queue;
+    constructor(root: string, opts?: BundleStoreOptions);
+    /** Serialize an async write operation behind the store-wide queue. */
+    private enqueue;
+    topicsDir(): string;
+    metaDir(): string;
+    topicPath(slug: string): string;
+    private observationsPath;
+    private injectionsPath;
+    private conflictsPath;
+    private distillStatePath;
+    /** Last distill-lane run outcome (diagnostics for /topics status and e2e). */
+    writeDistillState(state: unknown): Promise<void>;
+    readDistillState(): Promise<Record<string, unknown> | undefined>;
+    /** Create the directory skeleton, git repo, and initial index (idempotent). */
+    ensure(): Promise<void>;
+    /** Repo initialized and not gitDisabled. */
+    hasGit(): Promise<boolean>;
+    private commit;
+    listTopics(): Promise<TopicMeta[]>;
+    /** Topics broken beyond parsing — surfaced by /topics status, never silently dropped. */
+    brokenTopics(): Promise<string[]>;
+    readTopic(slug: string): Promise<okf.TopicDoc | undefined>;
+    exists(slug: string): Promise<boolean>;
+    /** First free slug: `foo`, then `foo-2`, `foo-3`, … */
+    uniqueSlug(base: string): Promise<string>;
+    /**
+     * Write a topic (atomic), regenerate the index, and commit both — one
+     * commit per conclusion change (ADR 0003). `slug` in the doc's depends is
+     * NOT normalized here; callers pass bundle-relative paths.
+     */
+    saveTopic(input: {
+        slug: string;
+        doc: okf.TopicDoc;
+    }, opts: {
+        message: string;
+        actor?: string;
+        created?: boolean;
+    }): Promise<SaveResult>;
+    private regenerateIndex;
+    /**
+     * Reverse-reference index: slug → who references it and how (`depends`
+     * edges + body links). Regenerated on every topic write (ADR 0003
+     * write-through), so it never drifts from the files.
+     */
+    private computeBacklinks;
+    /** Read the generated backlinks index (empty map when absent). */
+    readBacklinks(): Promise<Record<string, {
+        slug: string;
+        via: 'depends' | 'link';
+    }[]>>;
+    appendObservation(input: {
+        kind: ObservationKind;
+        source: 'model' | 'auto';
+        text: string;
+        sessionId?: string;
+    }): Promise<Observation>;
+    allObservations(limit?: number): Promise<Observation[]>;
+    undistilledObservations(limit?: number): Promise<Observation[]>;
+    markDistilled(ids: readonly string[], intoSlugs: readonly string[]): Promise<number>;
+    /**
+     * GC policy for the distill lane: an observation fed to the model but left
+     * unconsumed by any op has failed one attempt; after OBSERVATION_MAX_ATTEMPTS
+     * failed attempts it is physically deleted. The user explicitly authorized
+     * deleting raw observations the lane demonstrably cannot process — the
+     * alternative is the same head haunting the backlog forever. Consumed
+     * observations leave the pool via markDistilled and never accrue attempts.
+     */
+    static readonly OBSERVATION_MAX_ATTEMPTS = 3;
+    /**
+     * Post-run GC bookkeeping: increment `attempts` for every observation in
+     * `fedIds` that no op in `consumedIds` accounts for, then delete those that
+     * reached OBSERVATION_MAX_ATTEMPTS. Returns the deletion count (the caller
+     * records it in the distill-state detail). Increments persist with the
+     * usual flush cadence; a deletion commits immediately — destroying data
+     * must be traceable in git.
+     */
+    recordUnconsumed(fedIds: readonly string[], consumedIds: readonly string[]): Promise<{
+        dropped: number;
+    }>;
+    appendInjectionRecord(record: unknown): Promise<void>;
+    private compactInjectionsIfNeeded;
+    readInjectionRecords(limit?: number): Promise<InjectionRecord[]>;
+    getConflicts(): Promise<Set<string>>;
+    /** Sync variant for the injection hot path. */
+    getConflictsSync(): Set<string>;
+    setConflicts(paths: readonly string[]): Promise<void>;
+    status(): Promise<{
+        root: string;
+        topicCount: number;
+        byStatus: Record<okf.TopicStatus, number>;
+        observationsPending: number;
+        observationsTotal: number;
+        conflicts: string[];
+        broken: string[];
+        git: boolean;
+        head?: string;
+    }>;
+}