ashlr-plugin 1.36.4

package/servers/_genome-discoveries.ts

@@ -0,0 +1,140 @@
+/**
+ * _genome-discoveries.ts — surface AI-synthesized discovery sections in
+ * ashlr__grep retrieval.
+ *
+ * Mirrors `_genome-commits.ts` but for `kind: "discovery"` sections produced
+ * by `scripts/genome-synthesizer.ts`. Reuses the keyword scorer exported by
+ * the commit module so we have a single source of truth for tag/title/summary
+ * scoring across all v2 section kinds.
+ *
+ * Discoveries are ADDITIVE — they extend the corpus, never replace it. The
+ * grep handler formats them with a distinct `[discovery <id> - <date>]`
+ * prefix and a `## Discoveries` header so users can tell synthesis insights
+ * from static code and from raw commit history.
+ */
+
+import { readFile } from "fs/promises";
+import { existsSync } from "fs";
+import {
+  type DiscoverySection,
+  type SectionMetaV2,
+  discoverySectionAbsPath,
+  loadManifestV2,
+} from "./_manifest-v2";
+import { scoreSectionMeta, tokenize } from "./_genome-commits";
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+export interface RetrievedDiscovery {
+  id: string;
+  shortId: string;
+  synthesizedAt: string;
+  summary: string;
+  evidence: DiscoverySection["evidence"];
+  sourceCommits: string[];
+  confidence: number;
+  score: number;
+}
+
+/**
+ * Retrieve up to `limit` discovery sections matching `query`.
+ *
+ * Returns an empty array when:
+ *   - no genome exists,
+ *   - the manifest has no discovery sections,
+ *   - no discovery section's keywords overlap the query.
+ *
+ * Never throws — best-effort retrieval, callers should fall through to
+ * static + commit results when this returns [].
+ */
+export async function retrieveDiscoverySections(
+  cwd: string,
+  query: string,
+  limit = 3,
+): Promise<RetrievedDiscovery[]> {
+  try {
+    const manifest = await loadManifestV2(cwd);
+    if (!manifest) return [];
+
+    const discoveryMetas = manifest.sections.filter((s) => s.kind === "discovery");
+    if (discoveryMetas.length === 0) return [];
+
+    const queryTerms = new Set(tokenize(query));
+    let scored: Array<{ meta: SectionMetaV2; score: number }>;
+
+    if (queryTerms.size === 0) {
+      // No meaningful query — surface the most recent discoveries.
+      scored = discoveryMetas.map((m) => ({ meta: m, score: 1 }));
+    } else {
+      scored = discoveryMetas
+        .map((m) => ({ meta: m, score: scoreSectionMeta(m, queryTerms) }))
+        .filter((s) => s.score > 0);
+    }
+
+    if (scored.length === 0) return [];
+
+    scored.sort((a, b) => {
+      if (b.score !== a.score) return b.score - a.score;
+      const ad = a.meta.lastUpdatedAt ?? a.meta.updatedAt;
+      const bd = b.meta.lastUpdatedAt ?? b.meta.updatedAt;
+      return bd > ad ? 1 : bd < ad ? -1 : 0;
+    });
+
+    const out: RetrievedDiscovery[] = [];
+    for (const { meta, score } of scored.slice(0, limit)) {
+      const id = meta.path.replace(/^discoveries\//, "").replace(/\.json$/, "");
+      const abs = discoverySectionAbsPath(cwd, id);
+      if (!existsSync(abs)) continue;
+      let payload: DiscoverySection;
+      try {
+        payload = JSON.parse(await readFile(abs, "utf-8")) as DiscoverySection;
+      } catch {
+        continue;
+      }
+      out.push({
+        id: payload.id,
+        shortId: payload.id.slice(0, 7),
+        synthesizedAt: payload.synthesizedAt,
+        summary: payload.summary,
+        evidence: payload.evidence,
+        sourceCommits: payload.sourceCommits,
+        confidence: payload.confidence,
+        score,
+      });
+    }
+    return out;
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Format a list of retrieved discovery sections as a markdown block suitable
+ * for prepending to grep output. Each discovery is clearly labeled with
+ * `[discovery <id> - <date>]` so users see it's AI-synthesized, not static
+ * documentation or raw commit history.
+ */
+export function formatDiscoveriesForPrompt(discoveries: RetrievedDiscovery[]): string {
+  if (discoveries.length === 0) return "";
+  const parts = discoveries.map((d) => {
+    const dateShort = d.synthesizedAt ? d.synthesizedAt.slice(0, 10) : "unknown";
+    const evidence =
+      d.evidence.length === 0
+        ? ""
+        : `\n_evidence: ${d.evidence
+            .slice(0, 6)
+            .map((e) => (e.lineRange ? `${e.path}:${e.lineRange[0]}-${e.lineRange[1]}` : e.path))
+            .join(", ")}${d.evidence.length > 6 ? ` (+${d.evidence.length - 6})` : ""}_\n`;
+    const commits =
+      d.sourceCommits.length === 0
+        ? ""
+        : `_from commits: ${d.sourceCommits.map((s) => s.slice(0, 7)).join(", ")}_\n`;
+    const conf = `_confidence: ${d.confidence.toFixed(2)}_\n`;
+    return `### [discovery ${d.shortId} - ${dateShort}]
+${d.summary}
+${evidence}${commits}${conf}`;
+  });
+  return `## Discoveries\n\n${parts.join("\n\n---\n\n")}`;
+}

package/servers/_genome-embed-populator.ts

@@ -0,0 +1,216 @@
+/**
+ * _genome-embed-populator.ts — Populate the embedding cache from genome sections.
+ *
+ * Walks the genome manifest on first grep per session and upserts one
+ * embedding per section into the shared ContextDb. The cache is then
+ * consulted on every `ashlr__grep` call (see grep-server.ts:236–293) to
+ * either prepend a high-similarity match or fall through to the genome
+ * retrieval / ripgrep path.
+ *
+ * Historical note: a prior version of this module shipped empty — the cache
+ * was a placebo with no rows. The populator now seeds it on every grep and
+ * grep-server consumes the rows; the marketing claim ("genome-aware
+ * retrieval saves 50–90% on relevant queries") is now load-bearing. Kill
+ * the entire pipeline with `ASHLR_GENOME_RETRIEVAL=off` to fall back to
+ * bare ripgrep for diagnostics.
+ *
+ * Cost model:
+ *   - First-call-after-manifest-change: O(sections) reads + embeds. For a
+ *     typical genome (~15 sections, each < 4KB) this is a few ms in BM25 mode.
+ *   - Steady state: O(1) watermark check, no disk reads.
+ *
+ * Watermark:
+ *   Per-project-hash manifest-mtime stored at ~/.ashlr/embed-watermark.json.
+ *   Re-population is triggered ONLY when the manifest mtime advances.
+ *   Section-level deltas re-embed any section whose section-text hash changed
+ *   since the last run.
+ *
+ * Non-goals:
+ *   - Cross-session invalidation via hashes of source files (handled by the
+ *     post-tool-use-embedding hook + embed-file-worker for code files).
+ *   - AST chunking of markdown genome sections — chunks are a code concern.
+ */
+
+import { existsSync, readFileSync, mkdirSync, renameSync, writeFileSync, statSync } from "fs";
+import { createHash } from "crypto";
+import { homedir } from "os";
+import { dirname, join } from "path";
+
+import { loadManifest, sectionPath } from "@ashlr/core-efficiency";
+import type { ContextDb } from "./_embedding-cache";
+import { embed, upsertCorpus } from "./_embedding-model";
+
+// ---------------------------------------------------------------------------
+// Watermark I/O
+// ---------------------------------------------------------------------------
+
+interface ProjectWatermark {
+  manifestMtime: number;
+  sections: Record<string, string>; // section path → content hash
+}
+
+interface WatermarkFile {
+  version: 1;
+  projects: Record<string, ProjectWatermark>;
+}
+
+function watermarkPath(home?: string): string {
+  return join(home ?? homedir(), ".ashlr", "embed-watermark.json");
+}
+
+function readWatermark(home?: string): WatermarkFile {
+  const path = watermarkPath(home);
+  if (!existsSync(path)) return { version: 1, projects: {} };
+  try {
+    const raw = readFileSync(path, "utf-8");
+    const parsed = JSON.parse(raw) as WatermarkFile;
+    if (parsed.version !== 1 || !parsed.projects) {
+      return { version: 1, projects: {} };
+    }
+    return parsed;
+  } catch {
+    return { version: 1, projects: {} };
+  }
+}
+
+function writeWatermark(wm: WatermarkFile, home?: string): void {
+  const path = watermarkPath(home);
+  const dir = dirname(path);
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+  const tmp = `${path}.${process.pid}.tmp`;
+  writeFileSync(tmp, JSON.stringify(wm), "utf-8");
+  renameSync(tmp, path);
+}
+
+function manifestMtime(genomeRoot: string): number {
+  try {
+    return statSync(join(genomeRoot, ".ashlrcode", "genome", "manifest.json")).mtimeMs;
+  } catch {
+    return 0;
+  }
+}
+
+function sha256(s: string): string {
+  return createHash("sha256").update(s).digest("hex").slice(0, 16);
+}
+
+// ---------------------------------------------------------------------------
+// In-process guard: avoid concurrent populate runs for the same project.
+// ---------------------------------------------------------------------------
+
+const inflight = new Map<string, Promise<PopulateStats>>();
+
+export interface PopulateStats {
+  /** Sections embedded (inserted or refreshed). */
+  embedded: number;
+  /** Sections that were up-to-date and skipped. */
+  skipped: number;
+  /** True when the manifest mtime was unchanged — the entire run is a no-op. */
+  unchanged: boolean;
+}
+
+export interface PopulateOptions {
+  /** ContextDb instance (injected so tests can use a fresh tmp home). */
+  ctxDb: ContextDb;
+  /** Override `~` so tests never touch the real watermark file. */
+  home?: string;
+  /** Already-computed stable 8-char project hash. */
+  projectHash: string;
+}
+
+/**
+ * Populate the embedding cache with every section in the genome manifest.
+ * Cheap + idempotent: the watermark ensures a no-op when nothing changed.
+ */
+export async function populateGenomeEmbeddings(
+  genomeRoot: string,
+  opts: PopulateOptions,
+): Promise<PopulateStats> {
+  const key = `${opts.projectHash}\x00${genomeRoot}`;
+  const running = inflight.get(key);
+  if (running) return running;
+
+  const run = populateInner(genomeRoot, opts).finally(() => {
+    if (inflight.get(key)) inflight.delete(key);
+  });
+  inflight.set(key, run);
+  return run;
+}
+
+async function populateInner(
+  genomeRoot: string,
+  { ctxDb, home, projectHash }: PopulateOptions,
+): Promise<PopulateStats> {
+  const mtime = manifestMtime(genomeRoot);
+  if (mtime === 0) return { embedded: 0, skipped: 0, unchanged: true };
+
+  const wm = readWatermark(home);
+  const prev: ProjectWatermark = wm.projects[projectHash] ?? {
+    manifestMtime: 0,
+    sections: {},
+  };
+
+  if (prev.manifestMtime === mtime) {
+    return { embedded: 0, skipped: 0, unchanged: true };
+  }
+
+  const manifest = await loadManifest(genomeRoot);
+  if (!manifest || manifest.sections.length === 0) {
+    // Manifest is unreadable or empty — advance the watermark so we don't
+    // re-read every call, but don't record a bogus section map.
+    wm.projects[projectHash] = { manifestMtime: mtime, sections: {} };
+    try { writeWatermark(wm, home); } catch { /* best-effort */ }
+    return { embedded: 0, skipped: 0, unchanged: false };
+  }
+
+  let embedded = 0;
+  let skipped = 0;
+  const nextSections: Record<string, string> = {};
+
+  for (const section of manifest.sections) {
+    let content: string;
+    try {
+      const full = sectionPath(genomeRoot, section.path);
+      content = readFileSync(full, "utf-8");
+    } catch {
+      continue; // missing or invalid path — skip this section
+    }
+
+    const hash = sha256(content);
+    nextSections[section.path] = hash;
+
+    if (prev.sections[section.path] === hash) {
+      skipped++;
+      continue;
+    }
+
+    // Sections are typically < 4KB of markdown; embed the title + summary + body
+    // so terms like the filename and tags also contribute to retrieval.
+    const text = `${section.title}\n${section.summary}\n${content}`;
+    try {
+      upsertCorpus(text);
+      const vec = await embed(text);
+      ctxDb.upsertEmbedding({
+        projectHash,
+        sectionPath: `genome:${section.path}`,
+        sectionText: content.slice(0, 2000),
+        embedding: vec,
+        embeddingDim: vec.length,
+        source: "genome",
+      });
+      embedded++;
+    } catch {
+      // Never let one bad section abort the whole populate pass.
+    }
+  }
+
+  wm.projects[projectHash] = { manifestMtime: mtime, sections: nextSections };
+  try { writeWatermark(wm, home); } catch { /* best-effort */ }
+
+  return { embedded, skipped, unchanged: false };
+}
+
+/** Test helper — drop the in-process in-flight map. */
+export function _clearInflightForTests(): void {
+  inflight.clear();
+}

package/servers/_genome-freshness.ts

@@ -0,0 +1,134 @@
+/**
+ * _genome-freshness.ts — render compact freshness badges for retrieved
+ * genome sections (Q2 prep, foundational work).
+ *
+ * The v2 manifest (`_manifest-v2.ts`) carries `lastUpdatedAt` on every
+ * section. Surfacing that age in grep output lets the human + the model
+ * see at a glance whether the retrieved context is current.
+ *
+ * Badge format (max ~15 chars):
+ *   <1h   → `[fresh]`
+ *   <24h  → `[fresh: Xh]`
+ *   <7d   → `[fresh: Xd]`
+ *   <30d  → `[stale: Xd]`
+ *   ≥30d  → `[stale: ≥30d]`
+ *   undef → `""`  (legacy v1 sections; render nothing — caller stays safe)
+ *
+ * Decoration is purely additive: legacy v1 sections still render unchanged
+ * because their entry in the freshness map is absent.
+ */
+
+import { loadManifestV2 } from "./_manifest-v2";
+
+// ---------------------------------------------------------------------------
+// Pure helper — no I/O, fully tested.
+// ---------------------------------------------------------------------------
+
+/**
+ * Format a freshness badge from an ISO timestamp.
+ *
+ * Returns "" when `lastUpdatedAt` is undefined OR fails to parse — callers
+ * can safely splice the result into any string template without guarding.
+ *
+ * Bucket boundaries are inclusive on the LOWER bound and exclusive on the
+ * UPPER bound. e.g. exactly 1h old → `[fresh: 1h]` (>= 1h, < 24h).
+ */
+export function formatFreshness(
+  lastUpdatedAt: string | undefined,
+  now: Date = new Date(),
+): string {
+  if (!lastUpdatedAt) return "";
+  const t = Date.parse(lastUpdatedAt);
+  if (!Number.isFinite(t)) return "";
+
+  const ageMs = now.getTime() - t;
+  if (ageMs < 0) {
+    // Future timestamp — treat as fresh-just-now rather than emitting
+    // misleading "stale" text or empty string.
+    return "[fresh]";
+  }
+
+  const HOUR = 60 * 60 * 1000;
+  const DAY = 24 * HOUR;
+
+  if (ageMs < HOUR) return "[fresh]";
+  if (ageMs < DAY) {
+    const hours = Math.floor(ageMs / HOUR);
+    return `[fresh: ${hours}h]`;
+  }
+  if (ageMs < 7 * DAY) {
+    const days = Math.floor(ageMs / DAY);
+    return `[fresh: ${days}d]`;
+  }
+  if (ageMs < 30 * DAY) {
+    const days = Math.floor(ageMs / DAY);
+    return `[stale: ${days}d]`;
+  }
+  return "[stale: >=30d]";
+}
+
+// ---------------------------------------------------------------------------
+// Manifest lookup
+// ---------------------------------------------------------------------------
+
+/**
+ * Build a (sectionPath → lastUpdatedAt) lookup from the v2 manifest at `cwd`.
+ *
+ * Returns an empty Map when:
+ *   - no genome exists,
+ *   - the manifest is unreadable,
+ *   - sections have no lastUpdatedAt (legacy v1 not yet re-saved).
+ *
+ * Never throws. Safe to call on every grep retrieval.
+ */
+export async function loadSectionFreshnessMap(
+  cwd: string,
+): Promise<Map<string, string>> {
+  const out = new Map<string, string>();
+  try {
+    const manifest = await loadManifestV2(cwd);
+    if (!manifest) return out;
+    for (const s of manifest.sections) {
+      if (s.lastUpdatedAt) out.set(s.path, s.lastUpdatedAt);
+    }
+  } catch {
+    // Best-effort — never break retrieval.
+  }
+  return out;
+}
+
+// ---------------------------------------------------------------------------
+// Output decoration
+// ---------------------------------------------------------------------------
+
+/**
+ * Inject freshness badges into core-efficiency's formatGenomeForPrompt
+ * output. Targets each `### <title> (<path>)` header — if the path is in
+ * `freshnessMap` we append ` <badge>`. Headers with no metadata are left
+ * untouched (legacy v1 safety net).
+ *
+ * Pure / synchronous. Idempotent — re-running on already-decorated output
+ * is a no-op because the regex matches only un-decorated headers.
+ */
+export function decorateGenomeOutputWithFreshness(
+  formatted: string,
+  freshnessMap: Map<string, string>,
+  now: Date = new Date(),
+): string {
+  if (formatted.length === 0 || freshnessMap.size === 0) return formatted;
+
+  // Match: `### <title> (<path>)` at start of a line. Path is greedy-by-
+  // necessity (titles can contain parens), so we anchor to the LAST `(...)`
+  // on the line by requiring the trailing `)` to be followed by end-of-line
+  // or whitespace — works for the core-efficiency formatter's exact output.
+  return formatted.replace(
+    /^### (.+?) \(([^()\n]+)\)(?= *$)/gm,
+    (whole, title: string, path: string) => {
+      const ts = freshnessMap.get(path);
+      if (!ts) return whole;
+      const badge = formatFreshness(ts, now);
+      if (!badge) return whole;
+      return `### ${title} (${path}) ${badge}`;
+    },
+  );
+}

package/servers/_genome-issues.ts

@@ -0,0 +1,171 @@
+/**
+ * _genome-issues.ts — surface closed-issue sections in ashlr__grep retrieval.
+ *
+ * Q3 sibling of _genome-prs.ts. Issue sections are written by
+ * scripts/genome-cloud-sync.ts when an `issue_closed` cloud delta arrives.
+ * The grep handler calls retrieveIssueSections() so `ashlr__grep "auth bug"`
+ * returns the recently-closed issues alongside code + PRs.
+ *
+ * Mirrors the PR retriever exactly — scans sections/issues/ directly rather
+ * than going through the v2 manifest (cloud-sync writes files out-of-band
+ * from manifest updates). Best-effort, fails-silent.
+ */
+
+import { readFile, readdir, stat } from "fs/promises";
+import { existsSync } from "fs";
+import { join } from "path";
+import { type IssueSection, issuesDir } from "./_manifest-v2";
+import { scoreSectionMeta, tokenize } from "./_genome-commits";
+import { formatFreshness } from "./_genome-freshness";
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+export interface RetrievedIssue {
+  id: string;
+  title: string;
+  closedAt: string;
+  author: string;
+  labels: string[];
+  summary: string;
+  url: string;
+  score: number;
+}
+
+/**
+ * Retrieve up to `limit` issue sections from `sections/issues/` matching `query`.
+ *
+ * @param cwd        genome root (the directory containing `.ashlrcode/`)
+ * @param query      keyword query — same scorer commit/discovery/PR use
+ * @param limit      max results (default 3)
+ * @param sinceDays  optional freshness filter — drop issues closed longer
+ *                   than N days ago. `undefined` disables the filter.
+ */
+export async function retrieveIssueSections(
+  cwd: string,
+  query: string,
+  limit = 3,
+  sinceDays?: number,
+): Promise<RetrievedIssue[]> {
+  try {
+    const dir = issuesDir(cwd);
+    if (!existsSync(dir)) return [];
+
+    let entries: string[];
+    try {
+      entries = await readdir(dir);
+    } catch {
+      return [];
+    }
+    const files = entries.filter((n) => n.endsWith(".json"));
+    if (files.length === 0) return [];
+
+    const cutoff =
+      typeof sinceDays === "number" && Number.isFinite(sinceDays) && sinceDays > 0
+        ? Date.now() - sinceDays * 24 * 60 * 60 * 1000
+        : null;
+
+    const loaded: Array<{ section: IssueSection; mtimeMs: number }> = [];
+    for (const file of files) {
+      const abs = join(dir, file);
+      try {
+        const [raw, st] = await Promise.all([readFile(abs, "utf-8"), stat(abs)]);
+        const section = JSON.parse(raw) as IssueSection;
+        if (!section || typeof section !== "object") continue;
+        loaded.push({ section, mtimeMs: st.mtimeMs });
+      } catch {
+        // Skip corrupt file.
+      }
+    }
+
+    const queryTerms = new Set(tokenize(query));
+
+    const scored = loaded
+      .map(({ section, mtimeMs }) => {
+        // Freshness filter — drop issues older than the cutoff.
+        if (cutoff !== null) {
+          const t = section.closedAt ? Date.parse(section.closedAt) : NaN;
+          if (Number.isFinite(t) && t < cutoff) {
+            return null;
+          }
+        }
+        const tags = [
+          "issue",
+          "bug",
+          section.id,
+          ...(section.author ? [section.author.toLowerCase()] : []),
+          ...section.labels.map((l) => l.toLowerCase()),
+        ];
+        const meta = {
+          path: `issues/${section.id}.json`,
+          title: section.title,
+          summary: section.summary,
+          tags,
+          tokens: Math.ceil((section.title.length + section.summary.length) / 4),
+          updatedAt: section.closedAt || new Date(mtimeMs).toISOString(),
+        };
+        let score: number;
+        if (queryTerms.size === 0) {
+          score = 1;
+        } else {
+          score = scoreSectionMeta(meta, queryTerms);
+        }
+        return { section, score, mtimeMs };
+      })
+      .filter(
+        (x): x is { section: IssueSection; score: number; mtimeMs: number } =>
+          x !== null && x.score > 0,
+      );
+
+    if (scored.length === 0) return [];
+
+    scored.sort((a, b) => {
+      if (b.score !== a.score) return b.score - a.score;
+      const ad = a.section.closedAt || new Date(a.mtimeMs).toISOString();
+      const bd = b.section.closedAt || new Date(b.mtimeMs).toISOString();
+      return bd > ad ? 1 : bd < ad ? -1 : 0;
+    });
+
+    return scored.slice(0, limit).map(({ section, score }) => ({
+      id: section.id,
+      title: section.title,
+      closedAt: section.closedAt,
+      author: section.author,
+      labels: section.labels,
+      summary: section.summary,
+      url: section.url,
+      score,
+    }));
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Format a list of retrieved issue sections as a markdown block. Each
+ * issue gets `[Issue #<n> - <date>]` so it reads distinctly from PRs +
+ * commits + code in the same response.
+ */
+export function formatIssuesForPrompt(
+  issues: RetrievedIssue[],
+  now: Date = new Date(),
+): string {
+  if (issues.length === 0) return "";
+  const parts = issues.map((i) => {
+    const dateShort = i.closedAt ? i.closedAt.slice(0, 10) : "unknown";
+    const badge = formatFreshness(i.closedAt || undefined, now);
+    const badgeFragment = badge ? ` · ${badge.slice(1, -1)}` : "";
+    const labels =
+      i.labels.length === 0
+        ? ""
+        : `\n_labels: ${i.labels.slice(0, 6).join(", ")}${
+            i.labels.length > 6 ? ` (+${i.labels.length - 6})` : ""
+          }_\n`;
+    const author = i.author ? `_author: ${i.author}_\n` : "";
+    const link = i.url ? `_${i.url}_\n` : "";
+    return `### [Issue #${i.id} - ${dateShort}${badgeFragment}] ${i.title}${labels}${author}${link}
+${i.summary}`;
+  });
+  return `## Issues\n\n${parts.join("\n\n---\n\n")}`;
+}