@hevmind/ask 0.1.0

package/src/digest/expand.ts

@@ -0,0 +1,24 @@
+import { tokenize } from '../search/chunk.ts';
+import type { GlossaryEntry } from './schema';
+
+export function expandQueryTerms(query: string, glossary: GlossaryEntry[], cap = 24): string[] {
+  const terms = new Set(tokenize(query));
+  if (!terms.size) return [];
+
+  for (const entry of glossary) {
+    if (terms.size >= cap) break;
+    const entryTerms = new Set([...tokenize(entry.term), ...entry.aliases.flatMap((alias) => tokenize(alias))]);
+    if (!intersects(terms, entryTerms)) continue;
+    for (const term of entryTerms) {
+      terms.add(term);
+      if (terms.size >= cap) break;
+    }
+  }
+
+  return [...terms];
+}
+
+function intersects(a: Set<string>, b: Set<string>): boolean {
+  for (const item of b) if (a.has(item)) return true;
+  return false;
+}

package/src/digest/facts.ts

@@ -0,0 +1,77 @@
+import { tokenize } from '../search/chunk.ts';
+import type { Fact } from './schema.ts';
+
+const FENCE_RE = /```[a-zA-Z0-9]*\n([\s\S]*?)```/g;
+const INLINE_CODE_RE = /`([^`\n]+)`/g;
+const FLAG_RE = /(?<![\w-])(--?[a-zA-Z][\w-]*)/g;
+const VERSION_RE = /\bv?\d+(?:\.\d+)+\b/g;
+const MODEL_ID_RE = /\b[a-z][a-z0-9]*(?:-[a-z0-9]+)*-\d[a-z0-9-]*\b/gi;
+const DOTTED_RE = /\b[a-z0-9-]+(?:\.[a-z0-9-]+)+\b/gi;
+
+const MAX_FACTS = 24;
+const MAX_LITERAL = 400;
+
+/**
+ * Extracts byte-verbatim literals (code, flags, identifiers, versions) from a
+ * section's raw markdown. These are the answer-critical strings the prose
+ * tokenizer destroys; they are carried into the digest node unchanged so the agent
+ * can quote them exactly without re-reading the source.
+ *
+ * Fully deterministic — the model never authors or edits a fact.
+ */
+export function extractFacts(chunkId: string, raw: string): Fact[] {
+  const seen = new Set<string>();
+  const facts: Fact[] = [];
+  const push = (kind: Fact['kind'], literal: string) => {
+    const value = literal.trim();
+    if (value.length < 2 || value.length > MAX_LITERAL || seen.has(value)) return;
+    seen.add(value);
+    facts.push({ kind, literal: value, chunkId });
+  };
+
+  // Fenced code blocks first, then strip them so inline scanners don't re-read them.
+  let rest = raw;
+  for (const match of raw.matchAll(FENCE_RE)) push('code', match[1]);
+  rest = rest.replace(FENCE_RE, ' ');
+
+  for (const match of rest.matchAll(INLINE_CODE_RE)) push('code', match[1]);
+  // Inline code is the richest source of exact tokens; scan the remainder for
+  // bare flags/identifiers/versions that weren't wrapped in backticks.
+  const bare = rest.replace(INLINE_CODE_RE, ' ');
+  for (const match of bare.matchAll(FLAG_RE)) push('flag', match[1]);
+  for (const match of bare.matchAll(MODEL_ID_RE)) push('value', match[0]);
+  for (const match of bare.matchAll(DOTTED_RE)) push('value', match[0]);
+  for (const match of bare.matchAll(VERSION_RE)) push('value', match[0]);
+
+  return facts.slice(0, MAX_FACTS);
+}
+
+/** Sections in these groups carry dense literals the agent should read verbatim. */
+const SOURCE_PRIMARY_GROUP_RE = /reference|api/i;
+
+export function classifyMode(group: string | undefined | null): 'agent-primary' | 'source-primary' {
+  return group && SOURCE_PRIMARY_GROUP_RE.test(group) ? 'source-primary' : 'agent-primary';
+}
+
+const STOPWORDS = new Set([
+  'the', 'and', 'for', 'with', 'that', 'this', 'from', 'into', 'are', 'was', 'has', 'have', 'its',
+  'use', 'used', 'using', 'can', 'will', 'when', 'where', 'how', 'what', 'which', 'each', 'all',
+  'one', 'two', 'per', 'via', 'not', 'but', 'you', 'your', 'they', 'them', 'then', 'than', 'over',
+]);
+
+/**
+ * Distinctive tokens for the render-time link-support check: tokens long enough
+ * to be meaningful, minus common words. A cited link survives only if the
+ * answer block that contains it shares one of these with the cited node.
+ */
+export function distinctiveTokens(text: string, cap = 40): string[] {
+  const out: string[] = [];
+  const seen = new Set<string>();
+  for (const token of tokenize(text)) {
+    if (token.length < 4 || STOPWORDS.has(token) || seen.has(token)) continue;
+    seen.add(token);
+    out.push(token);
+    if (out.length >= cap) break;
+  }
+  return out;
+}

package/src/digest/frontmatter.ts

@@ -0,0 +1,41 @@
+export interface FrontmatterDocument {
+  data: Record<string, unknown>;
+  body: string;
+}
+
+export function parseFrontmatter(src: string): FrontmatterDocument {
+  if (!src.startsWith('---')) return { data: {}, body: src };
+  const end = src.indexOf('\n---', 3);
+  if (end === -1) return { data: {}, body: src };
+
+  const raw = src.slice(3, end).trim();
+  const body = src.slice(end).replace(/^\n---\s*\r?\n?/, '');
+  return { data: parseFlatYaml(raw), body };
+}
+
+function parseFlatYaml(src: string): Record<string, unknown> {
+  const data: Record<string, unknown> = {};
+  for (const line of src.split(/\r?\n/)) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith('#')) continue;
+    const colon = trimmed.indexOf(':');
+    if (colon === -1) continue;
+    const key = trimmed.slice(0, colon).trim();
+    const raw = trimmed.slice(colon + 1).trim();
+    if (!key) continue;
+    data[key] = parseScalar(raw);
+  }
+  return data;
+}
+
+function parseScalar(value: string): unknown {
+  if (!value) return '';
+  if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
+    return value.slice(1, -1);
+  }
+  if (value === 'true') return true;
+  if (value === 'false') return false;
+  const numberValue = Number(value);
+  if (Number.isFinite(numberValue) && /^-?\d+(\.\d+)?$/.test(value)) return numberValue;
+  return value;
+}

package/src/digest/read.ts

@@ -0,0 +1,63 @@
+import type { GlossaryEntry, Digest, DigestNode } from './schema.ts';
+
+export interface SectionSummary {
+  id: string;
+  title: string;
+  heading: string | null;
+  group: string | null;
+  url: string;
+}
+
+export function listGlossary(digest: Digest): GlossaryEntry[] {
+  return digest.glossary;
+}
+
+export function getGlossaryEntry(digest: Digest, term: string): GlossaryEntry | null {
+  const needle = normalizeLookup(term);
+  if (!needle) return null;
+  return (
+    digest.glossary.find((entry) => {
+      if (normalizeLookup(entry.term) === needle) return true;
+      return entry.aliases.some((alias) => normalizeLookup(alias) === needle);
+    }) ?? null
+  );
+}
+
+export function listSectionSummaries(digest: Digest, group?: string | null): SectionSummary[] {
+  const wantedGroup = group ? normalizeLookup(group) : '';
+  return digest.nodes
+    .filter((node) => !wantedGroup || normalizeLookup(node.group ?? '') === wantedGroup)
+    .map(sectionSummary);
+}
+
+export function getSection(digest: Digest, id: string): DigestNode | null {
+  const needle = decodePathValue(id).trim();
+  if (!needle) return null;
+  return digest.nodes.find((node) => node.id === needle) ?? null;
+}
+
+export function getOverview(digest: Digest): { overview: string; context: string } {
+  return { overview: digest.overview, context: digest.context };
+}
+
+export function sectionSummary(node: DigestNode): SectionSummary {
+  return {
+    id: node.id,
+    title: node.title,
+    heading: node.heading,
+    group: node.group,
+    url: node.url,
+  };
+}
+
+export function decodePathValue(value: string): string {
+  try {
+    return decodeURIComponent(value);
+  } catch {
+    return value;
+  }
+}
+
+function normalizeLookup(value: string): string {
+  return decodePathValue(value).trim().toLowerCase();
+}

package/src/digest/schema.ts

@@ -0,0 +1,185 @@
+export interface GlossaryEntry {
+  term: string;
+  aliases: string[];
+  definition: string;
+}
+
+/** A byte-verbatim literal lifted from a source section (a flag, code span, value). */
+export interface Fact {
+  kind: 'flag' | 'code' | 'value' | 'default' | 'key';
+  /** Exact text as it appears in the source — never paraphrased by the model. */
+  literal: string;
+  /** The chunk id this literal was extracted from. */
+  chunkId: string;
+}
+
+/** Where a node's knowledge came from, and the human deep link for it. */
+export interface SourceRef {
+  /** Equals a Chunk.id. */
+  chunkId: string;
+  /** The rendered deep link, e.g. "/docs/concepts#kubernetes-autoscaling". */
+  url: string;
+  /** The github-slugger anchor, or null for a page-level (intro) section. */
+  anchor: string | null;
+}
+
+/**
+ * One distilled, source-grounded section of the docs — the agent's "shadow site"
+ * view of a single heading section. `id` equals the originating chunk id.
+ */
+export interface DigestNode {
+  id: string;
+  kind: 'section';
+  title: string;
+  heading: string | null;
+  group: string | null;
+  url: string;
+  /** Model-distilled prose. May paraphrase; exact strings live in `facts`. */
+  summary: string;
+  /** Deterministically extracted verbatim literals. */
+  facts: Fact[];
+  /** Provenance — for a section node, its own chunk. */
+  sources: SourceRef[];
+  /**
+   * 'source-primary' sections (reference/API) carry dense literals the agent
+   * should read verbatim rather than trust a paraphrase of.
+   */
+  mode: 'agent-primary' | 'source-primary';
+  /** Distinctive tokens used for the render-time link-support check. */
+  terms: string[];
+}
+
+/** Reserved for the deferred edge layer; ships empty in this version. */
+export interface DigestEdge {
+  rel: string;
+  from: string;
+  to: string;
+}
+
+export interface Digest {
+  version: 2;
+  generatedAt: string;
+  contentHash: string;
+  /** Compact prose orientation. Degradation fallback when `nodes` is empty. */
+  context: string;
+  glossary: GlossaryEntry[];
+  /** Deterministic high-level map injected into the agent's system prompt. */
+  overview: string;
+  /** Model-authored example questions the overlay offers on open. */
+  suggestions: string[];
+  nodes: DigestNode[];
+  edges: DigestEdge[];
+}
+
+export const EMPTY_DIGEST: Digest = {
+  version: 2,
+  generatedAt: '',
+  contentHash: '',
+  context: '',
+  glossary: [],
+  overview: '',
+  suggestions: [],
+  nodes: [],
+  edges: [],
+};
+
+const FACT_KINDS = new Set<Fact['kind']>(['flag', 'code', 'value', 'default', 'key']);
+const NODE_MODES = new Set<DigestNode['mode']>(['agent-primary', 'source-primary']);
+
+/**
+ * Coerces unknown JSON into a Digest. A v1 artifact (`{context,
+ * glossary}` with no `nodes`) degrades cleanly to an empty-node v2 digest, so the
+ * runtime falls back to keyword/legacy behavior rather than hard-failing.
+ */
+export function normalizeDigest(value: unknown): Digest {
+  if (!value || typeof value !== 'object') return EMPTY_DIGEST;
+  const maybe = value as Partial<Digest>;
+  const glossary = Array.isArray(maybe.glossary)
+    ? maybe.glossary
+        .map((entry) => normalizeGlossaryEntry(entry))
+        .filter((entry): entry is GlossaryEntry => entry !== null)
+    : [];
+  const nodes = Array.isArray(maybe.nodes)
+    ? maybe.nodes.map((node) => normalizeNode(node)).filter((node): node is DigestNode => node !== null)
+    : [];
+  const edges = Array.isArray(maybe.edges)
+    ? maybe.edges.map((edge) => normalizeEdge(edge)).filter((edge): edge is DigestEdge => edge !== null)
+    : [];
+
+  return {
+    version: 2,
+    generatedAt: typeof maybe.generatedAt === 'string' ? maybe.generatedAt : '',
+    contentHash: typeof maybe.contentHash === 'string' ? maybe.contentHash : '',
+    context: typeof maybe.context === 'string' ? maybe.context : '',
+    glossary,
+    overview: typeof maybe.overview === 'string' ? maybe.overview : '',
+    suggestions: Array.isArray(maybe.suggestions)
+      ? maybe.suggestions.filter((s): s is string => typeof s === 'string' && s.trim().length > 0)
+      : [],
+    nodes,
+    edges,
+  };
+}
+
+function normalizeGlossaryEntry(value: unknown): GlossaryEntry | null {
+  if (!value || typeof value !== 'object') return null;
+  const maybe = value as Partial<GlossaryEntry>;
+  if (typeof maybe.term !== 'string' || typeof maybe.definition !== 'string') return null;
+  return {
+    term: maybe.term,
+    aliases: Array.isArray(maybe.aliases) ? maybe.aliases.filter((alias): alias is string => typeof alias === 'string') : [],
+    definition: maybe.definition,
+  };
+}
+
+function normalizeNode(value: unknown): DigestNode | null {
+  if (!value || typeof value !== 'object') return null;
+  const maybe = value as Partial<DigestNode>;
+  if (typeof maybe.id !== 'string' || typeof maybe.url !== 'string') return null;
+  return {
+    id: maybe.id,
+    kind: 'section',
+    title: typeof maybe.title === 'string' ? maybe.title : maybe.id,
+    heading: typeof maybe.heading === 'string' ? maybe.heading : null,
+    group: typeof maybe.group === 'string' ? maybe.group : null,
+    url: maybe.url,
+    summary: typeof maybe.summary === 'string' ? maybe.summary : '',
+    facts: Array.isArray(maybe.facts)
+      ? maybe.facts.map((fact) => normalizeFact(fact)).filter((fact): fact is Fact => fact !== null)
+      : [],
+    sources: Array.isArray(maybe.sources)
+      ? maybe.sources.map((src) => normalizeSource(src)).filter((src): src is SourceRef => src !== null)
+      : [],
+    mode: maybe.mode && NODE_MODES.has(maybe.mode) ? maybe.mode : 'agent-primary',
+    terms: Array.isArray(maybe.terms) ? maybe.terms.filter((term): term is string => typeof term === 'string') : [],
+  };
+}
+
+function normalizeFact(value: unknown): Fact | null {
+  if (!value || typeof value !== 'object') return null;
+  const maybe = value as Partial<Fact>;
+  if (typeof maybe.literal !== 'string' || !maybe.literal) return null;
+  return {
+    kind: maybe.kind && FACT_KINDS.has(maybe.kind) ? maybe.kind : 'value',
+    literal: maybe.literal,
+    chunkId: typeof maybe.chunkId === 'string' ? maybe.chunkId : '',
+  };
+}
+
+function normalizeSource(value: unknown): SourceRef | null {
+  if (!value || typeof value !== 'object') return null;
+  const maybe = value as Partial<SourceRef>;
+  if (typeof maybe.chunkId !== 'string' || typeof maybe.url !== 'string') return null;
+  return {
+    chunkId: maybe.chunkId,
+    url: maybe.url,
+    anchor: typeof maybe.anchor === 'string' ? maybe.anchor : null,
+  };
+}
+
+function normalizeEdge(value: unknown): DigestEdge | null {
+  if (!value || typeof value !== 'object') return null;
+  const maybe = value as Partial<DigestEdge>;
+  if (typeof maybe.rel !== 'string' || typeof maybe.from !== 'string' || typeof maybe.to !== 'string') return null;
+  return { rel: maybe.rel, from: maybe.from, to: maybe.to };
+}

package/src/digest/verify.ts

@@ -0,0 +1,116 @@
+import { execFile } from 'node:child_process';
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { promisify } from 'node:util';
+import { buildCorpus } from './build.ts';
+import { extractFacts } from './facts.ts';
+import { normalizeDigest } from './schema.ts';
+
+const execFileAsync = promisify(execFile);
+
+export interface VerifyAnchorsOptions {
+  siteRoot: string;
+  collections: string[] | null;
+  basePath: string;
+  digestContentGlobs?: string[];
+  chunkHeadingDepth: number;
+  digestPath?: string;
+  buildCommand?: string;
+  distDir?: string;
+  skipBuild?: boolean;
+}
+
+export interface VerifyAnchorsResult {
+  checked: number;
+  missing: Array<{ url: string; file: string; anchorId: string }>;
+  /** Source literals that the committed digest dropped from an agent-primary node. */
+  dropped: Array<{ id: string; literal: string }>;
+  /** Section ids present in the corpus but absent from the committed digest. */
+  uncovered: string[];
+}
+
+export async function verifyAnchors(options: VerifyAnchorsOptions): Promise<VerifyAnchorsResult> {
+  if (!options.skipBuild) {
+    const command = options.buildCommand ?? 'pnpm build';
+    await execFileAsync('sh', ['-c', command], { cwd: options.siteRoot, maxBuffer: 1024 * 1024 * 8 });
+  }
+
+  const corpus = await buildCorpus(options);
+  const distDir = path.resolve(options.siteRoot, options.distDir ?? 'dist');
+  const anchored = corpus.chunks.filter((chunk) => chunk.anchorId);
+  const missing: VerifyAnchorsResult['missing'] = [];
+
+  for (const chunk of anchored) {
+    const files = htmlFilesForUrl(distDir, chunk.url);
+    const found = await findHtmlWithId(files, chunk.anchorId!);
+    if (!found) {
+      const file = files[0];
+      missing.push({ url: chunk.url, file, anchorId: chunk.anchorId! });
+    }
+  }
+
+  const { dropped, uncovered } = await verifyFidelity(options, corpus.chunks);
+  return { checked: anchored.length, missing, dropped, uncovered };
+}
+
+/**
+ * Literal-fidelity gate: every verbatim literal the builder would extract from a
+ * section must be recoverable from that section's committed node — unless the
+ * node is source-primary (the agent reads its raw text instead). Catches lossy
+ * distillation that would otherwise produce confident-but-wrong exact answers.
+ *
+ * Reads the committed artifact only; never calls the model.
+ */
+async function verifyFidelity(
+  options: VerifyAnchorsOptions,
+  chunks: Awaited<ReturnType<typeof buildCorpus>>['chunks'],
+): Promise<{ dropped: VerifyAnchorsResult['dropped']; uncovered: string[] }> {
+  const digestPath = path.resolve(options.siteRoot, options.digestPath ?? '.hev-ask/digest.json');
+  const digest = normalizeDigest(await readJson(digestPath));
+  if (!digest.nodes.length) return { dropped: [], uncovered: [] }; // v1 / degraded digest — nothing to check
+
+  const nodeById = new Map(digest.nodes.map((node) => [node.id, node]));
+  const dropped: VerifyAnchorsResult['dropped'] = [];
+  const uncovered: string[] = [];
+
+  for (const chunk of chunks) {
+    const node = nodeById.get(chunk.id);
+    if (!node) {
+      uncovered.push(chunk.id);
+      continue;
+    }
+    if (node.mode === 'source-primary') continue;
+    const carried = new Set(node.facts.map((fact) => fact.literal));
+    for (const fact of extractFacts(chunk.id, chunk.raw)) {
+      if (!carried.has(fact.literal)) dropped.push({ id: chunk.id, literal: fact.literal });
+    }
+  }
+
+  return { dropped, uncovered };
+}
+
+async function readJson(file: string): Promise<unknown> {
+  try {
+    return JSON.parse(await readFile(file, 'utf8'));
+  } catch {
+    return null;
+  }
+}
+
+async function findHtmlWithId(files: string[], id: string): Promise<boolean> {
+  for (const file of files) {
+    const html = await readFile(file, 'utf8').catch(() => '');
+    if (html && hasId(html, id)) return true;
+  }
+  return false;
+}
+
+function htmlFilesForUrl(distDir: string, url: string): string[] {
+  const pathname = url.split('#')[0].replace(/^\//, '').replace(/\/$/, '');
+  return [path.join(distDir, pathname, 'index.html'), path.join(distDir, 'client', pathname, 'index.html')];
+}
+
+function hasId(html: string, id: string): boolean {
+  const escaped = id.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+  return new RegExp(`\\sid=(["'])${escaped}\\1`).test(html);
+}