@hevmind/ask 0.1.0 → 0.2.0

package/src/digest/tree.ts

@@ -0,0 +1,259 @@
+import { readFileSync, readdirSync, statSync } from 'node:fs';
+import path from 'node:path';
+import { parseFrontmatter } from './frontmatter.ts';
+import { EMPTY_DIGEST, normalizeDigest, type Digest, type DigestNode, type Fact, type SourceRef } from './schema.ts';
+
+const META_FILE = '_meta.md';
+const GLOSSARY_DIR = '_glossary';
+const META_OVERVIEW = '\n## Overview\n\n';
+
+export interface DigestTreeFile {
+  path: string;
+  body: string;
+}
+
+export function readDigestArtifact(siteRoot: string, digestPath: string): Digest {
+  const resolved = path.resolve(siteRoot, digestPath);
+  try {
+    const stat = statSync(resolved);
+    if (stat.isDirectory()) {
+      try {
+        return readDigestTree(resolved);
+      } catch {
+        return readLegacyJson(path.join(resolved, 'digest.json'));
+      }
+    }
+    return readLegacyJson(resolved);
+  } catch {
+    try {
+      return readLegacyJson(path.join(resolved, 'digest.json'));
+    } catch {
+      return EMPTY_DIGEST;
+    }
+  }
+}
+
+export function readDigestTree(root: string): Digest {
+  const meta = parseFrontmatter(readFileSync(path.join(root, META_FILE), 'utf8'));
+  const { context, overview } = parseMetaBody(meta.body);
+  const digest: Digest = {
+    version: 2,
+    generatedAt: stringField(meta.data, 'generatedAt'),
+    contentHash: stringField(meta.data, 'contentHash'),
+    context,
+    glossary: [],
+    overview,
+    suggestions: stringArrayField(meta.data, 'suggestions'),
+    nodes: [],
+    edges: [],
+  };
+
+  for (const file of walk(root)) {
+    const rel = path.relative(root, file).replace(/\\/g, '/');
+    if (rel === META_FILE || !rel.endsWith('.md')) continue;
+    const parsed = parseFrontmatter(readFileSync(file, 'utf8'));
+    if (rel.startsWith(`${GLOSSARY_DIR}/`)) {
+      const term = stringField(parsed.data, 'term') || titleFromPath(rel);
+      if (!term) continue;
+      digest.glossary.push({
+        term,
+        aliases: stringArrayField(parsed.data, 'aliases'),
+        definition: parsed.body.trim(),
+      });
+      continue;
+    }
+    const node = nodeFromTreeFile(rel, parsed);
+    if (node) digest.nodes.push(node);
+  }
+
+  digest.glossary.sort((a, b) => a.term.localeCompare(b.term));
+  digest.nodes.sort((a, b) => a.id.localeCompare(b.id));
+  return normalizeDigest(digest);
+}
+
+export function digestTreeFiles(digest: Digest): DigestTreeFile[] {
+  const files: DigestTreeFile[] = [{ path: META_FILE, body: renderMetaFile(digest) }];
+  for (const entry of digest.glossary) {
+    files.push({
+      path: `${GLOSSARY_DIR}/${glossaryPath(entry.term)}.md`,
+      body: markdownWithFrontmatter(
+        [
+          ['term', entry.term],
+          ['aliases', entry.aliases],
+        ],
+        entry.definition,
+      ),
+    });
+  }
+  digest.nodes.forEach((node, order) => {
+    files.push({ path: `${nodePath(node)}.md`, body: renderNodeFile(node, order) });
+  });
+  return files.sort((a, b) => a.path.localeCompare(b.path));
+}
+
+function readLegacyJson(file: string): Digest {
+  return normalizeDigest(JSON.parse(readFileSync(file, 'utf8')));
+}
+
+function renderMetaFile(digest: Digest): string {
+  return markdownWithFrontmatter(
+    [
+      ['version', digest.version],
+      ['generatedAt', digest.generatedAt || new Date().toISOString()],
+      ['contentHash', digest.contentHash],
+      ['suggestions', digest.suggestions],
+    ],
+    `## Context\n\n${digest.context.trim()}${META_OVERVIEW}${digest.overview.trim()}`.trim(),
+  );
+}
+
+function renderNodeFile(node: DigestNode, order: number): string {
+  return markdownWithFrontmatter(
+    [
+      ['id', node.id],
+      ['title', node.title],
+      ['heading', node.heading],
+      ['group', node.group],
+      ['order', order],
+      ['url', node.url],
+      ['anchor', node.sources[0]?.anchor ?? null],
+      ['terms', node.terms],
+      ['hash', node.hash ?? null],
+      ['mode', node.mode],
+      ['facts', node.facts],
+      ['sources', node.sources],
+    ],
+    node.summary,
+  );
+}
+
+function markdownWithFrontmatter(fields: Array<[string, unknown]>, body: string): string {
+  const frontmatter = fields.map(([key, value]) => `${key}: ${formatFrontmatterValue(value)}`).join('\n');
+  return `---\n${frontmatter}\n---\n\n${body.trim()}\n`;
+}
+
+function formatFrontmatterValue(value: unknown): string {
+  if (value === null || value === undefined || value === '') return 'null';
+  if (typeof value === 'number') return String(value);
+  return JSON.stringify(value);
+}
+
+function nodePath(node: DigestNode): string {
+  return node.id.trim().replace(/#/g, '/').replace(/^\/+|\/+$/g, '');
+}
+
+function glossaryPath(term: string): string {
+  const slug = term
+    .toLowerCase()
+    .replace(/[^a-z0-9 _-]+/g, '')
+    .trim()
+    .replace(/\s+/g, '-');
+  return slug || 'term';
+}
+
+function nodeFromTreeFile(rel: string, parsed: ReturnType<typeof parseFrontmatter>): DigestNode | null {
+  const id = stringField(parsed.data, 'id') || rel.replace(/\.md$/, '').replace(/\//g, '#');
+  const url = stringField(parsed.data, 'url');
+  if (!id || !url) return null;
+  const anchor = nullableStringField(parsed.data, 'anchor');
+  const sources = sourceArrayField(parsed.data, 'sources');
+  return {
+    id,
+    kind: 'section',
+    title: stringField(parsed.data, 'title') || id,
+    heading: nullableStringField(parsed.data, 'heading'),
+    group: nullableStringField(parsed.data, 'group'),
+    url,
+    summary: firstParagraph(parsed.body) || stringField(parsed.data, 'summary'),
+    hash: stringField(parsed.data, 'hash') || undefined,
+    facts: factArrayField(parsed.data, 'facts'),
+    sources: sources.length ? sources : [{ chunkId: id, url, anchor }],
+    mode: stringField(parsed.data, 'mode') === 'source-primary' ? 'source-primary' : 'agent-primary',
+    terms: stringArrayField(parsed.data, 'terms'),
+  };
+}
+
+function parseMetaBody(body: string): { context: string; overview: string } {
+  let trimmed = body.trim();
+  if (trimmed.startsWith('## Context')) trimmed = trimmed.slice('## Context'.length).trim();
+  const at = trimmed.indexOf(META_OVERVIEW);
+  if (at >= 0) {
+    return { context: trimmed.slice(0, at).trim(), overview: trimmed.slice(at + META_OVERVIEW.length).trim() };
+  }
+  return { context: trimmed, overview: '' };
+}
+
+function walk(dir: string): string[] {
+  const out: string[] = [];
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    const file = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      if (entry.name === 'shards') continue;
+      out.push(...walk(file));
+    } else {
+      out.push(file);
+    }
+  }
+  return out;
+}
+
+function firstParagraph(body: string): string {
+  return body
+    .trim()
+    .split(/\n{2,}/)
+    .map((part) => part.trim())
+    .find(Boolean) ?? '';
+}
+
+function titleFromPath(value: string): string {
+  return path.basename(value, '.md').replace(/-/g, ' ');
+}
+
+function stringField(data: Record<string, unknown>, key: string): string {
+  const value = data[key];
+  return typeof value === 'string' ? value : '';
+}
+
+function nullableStringField(data: Record<string, unknown>, key: string): string | null {
+  return stringField(data, key) || null;
+}
+
+function stringArrayField(data: Record<string, unknown>, key: string): string[] {
+  const value = data[key];
+  if (Array.isArray(value)) return value.filter((item): item is string => typeof item === 'string' && item.length > 0);
+  return typeof value === 'string' && value ? [value] : [];
+}
+
+function factArrayField(data: Record<string, unknown>, key: string): Fact[] {
+  const value = data[key];
+  if (!Array.isArray(value)) return [];
+  return value
+    .map((item) => {
+      if (!item || typeof item !== 'object') return null;
+      const maybe = item as Partial<Fact>;
+      if (typeof maybe.literal !== 'string' || !maybe.literal) return null;
+      return {
+        kind: maybe.kind ?? 'value',
+        literal: maybe.literal,
+        chunkId: typeof maybe.chunkId === 'string' ? maybe.chunkId : '',
+      } satisfies Fact;
+    })
+    .filter((item): item is Fact => item !== null);
+}
+
+function sourceArrayField(data: Record<string, unknown>, key: string): SourceRef[] {
+  const value = data[key];
+  if (!Array.isArray(value)) return [];
+  return value
+    .map((item) => {
+      if (!item || typeof item !== 'object') return null;
+      const maybe = item as Partial<SourceRef>;
+      if (typeof maybe.chunkId !== 'string' || typeof maybe.url !== 'string') return null;
+      return {
+        chunkId: maybe.chunkId,
+        url: maybe.url,
+        anchor: typeof maybe.anchor === 'string' ? maybe.anchor : null,
+      } satisfies SourceRef;
+    })
+    .filter((item): item is SourceRef => item !== null);
+}

package/src/digest/verify.ts

@@ -4,7 +4,7 @@ import path from 'node:path';
 import { promisify } from 'node:util';
 import { buildCorpus } from './build.ts';
 import { extractFacts } from './facts.ts';
-import { normalizeDigest } from './schema.ts';
+import { readDigestArtifact } from './tree.ts';
 
 const execFileAsync = promisify(execFile);
 
@@ -65,8 +65,7 @@ async function verifyFidelity(
   options: VerifyAnchorsOptions,
   chunks: Awaited<ReturnType<typeof buildCorpus>>['chunks'],
 ): Promise<{ dropped: VerifyAnchorsResult['dropped']; uncovered: string[] }> {
-  const digestPath = path.resolve(options.siteRoot, options.digestPath ?? '.hev-ask/digest.json');
-  const digest = normalizeDigest(await readJson(digestPath));
+  const digest = readDigestArtifact(options.siteRoot, options.digestPath ?? '.hev-ask');
   if (!digest.nodes.length) return { dropped: [], uncovered: [] }; // v1 / degraded digest — nothing to check
 
   const nodeById = new Map(digest.nodes.map((node) => [node.id, node]));
@@ -89,14 +88,6 @@ async function verifyFidelity(
   return { dropped, uncovered };
 }
 
-async function readJson(file: string): Promise<unknown> {
-  try {
-    return JSON.parse(await readFile(file, 'utf8'));
-  } catch {
-    return null;
-  }
-}
-
 async function findHtmlWithId(files: string[], id: string): Promise<boolean> {
   for (const file of files) {
     const html = await readFile(file, 'utf8').catch(() => '');

package/src/endpoint.ts

@@ -9,7 +9,9 @@ import {
   listGlossary,
   listSectionSummaries,
 } from './digest/read.ts';
+import { digestTreeFiles } from './digest/tree.ts';
 import { makeTelemetry, telemetryFromEnv } from './observability';
+import { PROVIDERS, clientFor, resolveProviderName } from './providers';
 import { hashableChunkText } from './search/chunk';
 import { buildIndex, prefilter, type Candidate, type Chunk } from './search/index';
 import { runAgenticAnswerLoop, type AgenticEvent } from './search/loop';
@@ -34,8 +36,13 @@ function resolveEnv(locals: unknown): Record<string, string | undefined> {
   return { ...fromImportMeta, ...fromProcess, ...fromRuntime };
 }
 
+// `config.provider` is baked at build time; only the key is read per-request.
+const provider = resolveProviderName(config.provider);
+const providerEnvKey = PROVIDERS[provider].envKey;
+const llm = clientFor(provider, config.providerBaseUrl);
+
 function resolveApiKey(locals: unknown): string | undefined {
-  return resolveEnv(locals).ANTHROPIC_API_KEY;
+  return resolveEnv(locals)[providerEnvKey];
 }
 
 // PostHog LLM tracing for the answer loop. On Cloudflare, capture promises
@@ -45,17 +52,25 @@ function resolveTelemetry(locals: unknown) {
   const ctx = (locals as { runtime?: { ctx?: { waitUntil?: (promise: Promise<unknown>) => void } } })
     ?.runtime?.ctx;
   const waitUntil = ctx?.waitUntil ? (promise: Promise<unknown>) => ctx.waitUntil!(promise) : undefined;
-  return makeTelemetry(telemetryFromEnv(resolveEnv(locals), { waitUntil }));
+  return makeTelemetry(telemetryFromEnv(resolveEnv(locals), { waitUntil, provider }));
 }
 
 // The overlay fetches suggested questions from the base route. Sub-routes expose
 // keyless reads over the committed digest for CLI, MCP, and generated clients.
-export const GET: APIRoute = ({ params, request }) => {
+export const GET: APIRoute = async ({ params, request }) => {
   const resource = resourceSegments(params.resource);
   if (!resource.length) return json({ suggestions: digest.suggestions ?? [], model: config.model });
   return readResource(resource, request);
 };
 
+export const HEAD: APIRoute = ({ params }) => {
+  const resource = resourceSegments(params.resource);
+  if (resource.length === 1 && decodePathValue(resource[0]).trim() === 'archive') {
+    return new Response(null, { status: 200, headers: archiveHeaders() });
+  }
+  return notFound();
+};
+
 export const POST: APIRoute = async ({ request, locals, params }) => {
   if (resourceSegments(params.resource).length) return notFound();
 
@@ -95,7 +110,7 @@ export const POST: APIRoute = async ({ request, locals, params }) => {
       query,
       model: config.model,
       mode: 'keyword',
-      warning: 'AI search is unavailable because ANTHROPIC_API_KEY is not configured.',
+      warning: `AI search is unavailable because ${providerEnvKey} is not configured.`,
     });
   }
 
@@ -121,6 +136,8 @@ export const POST: APIRoute = async ({ request, locals, params }) => {
           chunks,
           digest,
           telemetry: resolveTelemetry(locals),
+          call: llm.call,
+          stream: llm.stream,
           config: {
             model: config.model,
             maxIterations: config.maxIterations,
@@ -154,10 +171,12 @@ export const POST: APIRoute = async ({ request, locals, params }) => {
   });
 };
 
-function readResource(resource: string[], request: Request): Response {
+async function readResource(resource: string[], request: Request): Promise<Response> {
   const [rawRoot, ...rest] = resource;
   const root = decodePathValue(rawRoot).trim();
 
+  if (root === 'archive' && rest.length === 0) return archiveResponse();
+
   if (root === 'glossary') {
     if (!rest.length) return json({ terms: listGlossary(digest) });
     const entry = getGlossaryEntry(digest, rest.join('/'));
@@ -178,6 +197,103 @@ function readResource(resource: string[], request: Request): Response {
   return notFound();
 }
 
+async function archiveResponse(): Promise<Response> {
+  const tar = writeTar(digestTreeFiles(digest));
+  const body = await gzip(tar);
+  return new Response(arrayBufferFor(body), { status: 200, headers: archiveHeaders() });
+}
+
+function archiveHeaders(): HeadersInit {
+  return {
+    'content-type': 'application/gzip',
+    'content-disposition': 'attachment; filename="hev-ask-digest.tar.gz"',
+    'cache-control': 'public, max-age=60',
+    'x-hev-ask-content-hash': digest.contentHash ?? '',
+  };
+}
+
+interface ArchiveFile {
+  path: string;
+  body: string;
+}
+
+function writeTar(files: ArchiveFile[]): Uint8Array {
+  const encoder = new TextEncoder();
+  const chunks: Uint8Array[] = [];
+  for (const file of files) {
+    const body = encoder.encode(file.body);
+    chunks.push(tarHeader(file.path, body.length));
+    chunks.push(body);
+    const padding = (512 - (body.length % 512)) % 512;
+    if (padding) chunks.push(new Uint8Array(padding));
+  }
+  chunks.push(new Uint8Array(1024));
+  const size = chunks.reduce((total, chunk) => total + chunk.length, 0);
+  const out = new Uint8Array(size);
+  let offset = 0;
+  for (const chunk of chunks) {
+    out.set(chunk, offset);
+    offset += chunk.length;
+  }
+  return out;
+}
+
+function tarHeader(filePath: string, size: number): Uint8Array {
+  const header = new Uint8Array(512);
+  const { name, prefix } = tarNameParts(filePath);
+  writeAscii(header, 0, 100, name);
+  writeOctal(header, 100, 8, 0o644);
+  writeOctal(header, 108, 8, 0);
+  writeOctal(header, 116, 8, 0);
+  writeOctal(header, 124, 12, size);
+  writeOctal(header, 136, 12, 0);
+  header.fill(0x20, 148, 156);
+  header[156] = '0'.charCodeAt(0);
+  writeAscii(header, 257, 6, 'ustar');
+  writeAscii(header, 263, 2, '00');
+  writeAscii(header, 345, 155, prefix);
+  let checksum = 0;
+  for (const byte of header) checksum += byte;
+  writeOctal(header, 148, 8, checksum);
+  return header;
+}
+
+function tarNameParts(filePath: string): { name: string; prefix: string } {
+  const normalized = filePath.replace(/^\/+/, '');
+  if (normalized.length <= 100) return { name: normalized, prefix: '' };
+  const minNameStart = Math.max(0, normalized.length - 100);
+  for (let i = minNameStart; i >= 0; i -= 1) {
+    if (normalized[i] !== '/') continue;
+    const prefix = normalized.slice(0, i);
+    const name = normalized.slice(i + 1);
+    if (prefix.length <= 155 && name.length <= 100) return { name, prefix };
+  }
+  throw new Error(`Digest archive path is too long for ustar: ${filePath}`);
+}
+
+function writeAscii(target: Uint8Array, offset: number, length: number, value: string): void {
+  const bytes = new TextEncoder().encode(value);
+  target.set(bytes.slice(0, length), offset);
+}
+
+function writeOctal(target: Uint8Array, offset: number, length: number, value: number): void {
+  const text = value.toString(8).padStart(length - 1, '0').slice(-(length - 1));
+  writeAscii(target, offset, length, text);
+}
+
+async function gzip(data: Uint8Array): Promise<Uint8Array> {
+  const Compression = (globalThis as { CompressionStream?: new (format: 'gzip') => TransformStream<Uint8Array, Uint8Array> }).CompressionStream;
+  if (!Compression) throw new Error('CompressionStream is unavailable in this runtime.');
+  const stream = new Blob([arrayBufferFor(data)]).stream().pipeThrough(new Compression('gzip'));
+  return new Uint8Array(await new Response(stream).arrayBuffer());
+}
+
+function arrayBufferFor(data: Uint8Array): ArrayBuffer {
+  const copy = new Uint8Array(data.byteLength);
+  copy.set(data);
+  return copy.buffer;
+}
+
 function resourceSegments(value: string | undefined): string[] {
   return (value ?? '')
     .split('/')

package/src/index.ts

@@ -1,2 +1,2 @@
 export { default } from './integration';
-export type { HevAskOptions } from './types';
+export type { HevAskOptions, ProviderName } from './types';

package/src/integration.ts

@@ -1,10 +1,9 @@
 import type { AstroIntegration } from 'astro';
 import { execFile } from 'node:child_process';
-import { readFileSync } from 'node:fs';
-import path from 'node:path';
 import { promisify } from 'node:util';
 import { fileURLToPath } from 'node:url';
-import { EMPTY_DIGEST, normalizeDigest } from './digest/schema';
+import { readDigestArtifact } from './digest/tree';
+import { PROVIDERS, resolveProviderName } from './providers';
 import type { HevAskOptions, ResolvedConfig } from './types';
 
 const CONFIG_VIRTUAL_ID = 'virtual:hev-ask/config';
@@ -16,10 +15,13 @@ const execFileAsync = promisify(execFile);
  * configuration plus the committed ask digest through virtual modules.
  */
 export default function hevAsk(options: HevAskOptions = {}): AstroIntegration {
+  const provider = resolveProviderName(options.provider);
   const config: ResolvedConfig = {
     collections: options.collections ?? null,
-    model: options.model ?? 'claude-haiku-4-5',
-    digestModel: options.digestModel ?? 'claude-opus-4-8',
+    provider,
+    providerBaseUrl: options.providerBaseUrl,
+    model: options.model ?? PROVIDERS[provider].defaultModel,
+    digestModel: options.digestModel ?? PROVIDERS[provider].defaultDigestModel,
     endpoint: options.endpoint ?? '/api/ask',
     basePath: options.basePath ?? '/docs/',
     maxResults: options.maxResults ?? 6,
@@ -28,7 +30,7 @@ export default function hevAsk(options: HevAskOptions = {}): AstroIntegration {
     chunkHeadingDepth: options.chunkHeadingDepth ?? 3,
     candidatePerSearch: options.candidatePerSearch ?? 8,
     perDocCap: options.perDocCap ?? 2,
-    digestPath: options.digestPath ?? '.hev-ask/digest.json',
+    digestPath: options.digestDir ?? options.digestPath ?? '.hev-ask',
     digestContentGlobs: options.digestContentGlobs,
   };
 
@@ -58,9 +60,10 @@ export default function hevAsk(options: HevAskOptions = {}): AstroIntegration {
       },
       'astro:build:start': async ({ logger }) => {
         if (!config.collections?.length) return;
-        const apiKey = process.env.ANTHROPIC_API_KEY;
+        const envKey = PROVIDERS[config.provider].envKey;
+        const apiKey = process.env[envKey];
         if (!apiKey) {
-          logger.warn(`ANTHROPIC_API_KEY is not set; using committed ${config.digestPath} if present.`);
+          logger.warn(`${envKey} is not set; using committed ${config.digestPath} if present.`);
           return;
         }
 
@@ -105,11 +108,7 @@ function virtualDigestPlugin(config: ResolvedConfig, siteRoot: string) {
 }
 
 function readDigest(siteRoot: string, digestPath: string) {
-  try {
-    return normalizeDigest(JSON.parse(readFileSync(path.resolve(siteRoot, digestPath), 'utf8')));
-  } catch {
-    return EMPTY_DIGEST;
-  }
+  return readDigestArtifact(siteRoot, digestPath);
 }
 
 async function runDigestBuild(siteRoot: string, config: ResolvedConfig): Promise<string> {
@@ -118,7 +117,7 @@ async function runDigestBuild(siteRoot: string, config: ResolvedConfig): Promise
     askBin,
     'digest',
     'build',
-    '--digest-path',
+    '--digest-dir',
     config.digestPath,
     '--base-path',
     config.basePath,
@@ -126,7 +125,10 @@ async function runDigestBuild(siteRoot: string, config: ResolvedConfig): Promise
     String(config.chunkHeadingDepth),
     '--digest-model',
     config.digestModel,
+    '--provider',
+    config.provider,
   ];
+  if (config.providerBaseUrl) args.push('--provider-url', config.providerBaseUrl);
   for (const collection of config.collections ?? []) args.push('--collection', collection);
   for (const glob of config.digestContentGlobs ?? []) args.push('--content-glob', glob);