@h1v35/hivex 0.1.0 → 0.2.1

package/src/retrieval/lexical.ts

@@ -3,15 +3,47 @@ import { z } from 'zod';
 import { HivexError } from '../errors.ts';
 
 const row = z.object({ id: z.string(), score: z.number() });
-export const searchTerms = (text: string) => [
-  ...new Set(
-    text
-      .toLowerCase()
-      .normalize('NFKC')
-      .match(/[\p{L}\p{N}]+/gu) ?? [],
-  ),
-];
-type Record = { id: string; title: string; content: string };
+export const searchTerms = (text: string) => {
+  const terms = text
+    .toLowerCase()
+    .normalize('NFKC')
+    .match(/[\p{L}\p{N}]+/gu);
+  if (terms === null) {
+    return [];
+  }
+  return [...new Set(terms)];
+};
+interface Record {
+  id: string;
+  title: string;
+  content: string;
+}
+interface RankOptions {
+  database: Database;
+  excludedId?: string | null;
+  limit: number;
+  terms: string[];
+}
+
+const rank = function rank({ database, excludedId = null, limit, terms }: RankOptions) {
+  const expression = terms.map((term) => `"${term.replaceAll('"', '""')}"`).join(' OR ');
+  if (!expression) {
+    return [];
+  }
+  return database
+    .prepare(
+      'SELECT id, bm25(sources) AS score FROM sources WHERE sources MATCH ? AND (? IS NULL OR id != ?) ORDER BY score, id LIMIT ?'
+    )
+    .all(expression, excludedId, excludedId, limit)
+    .map((value) => row.parse(value));
+};
+
+const prepareVocabulary = function prepareVocabulary(database: Database) {
+  database.run("CREATE VIRTUAL TABLE vocabulary USING fts5vocab(sources, 'row')");
+  database.run("CREATE VIRTUAL TABLE instances USING fts5vocab(sources, 'instance')");
+  database.run('CREATE TABLE document_terms AS SELECT DISTINCT doc, term FROM instances');
+  database.run('CREATE INDEX document_terms_by_doc ON document_terms(doc)');
+};
 
 export class LexicalIndex {
   private readonly db = new Database(':memory:');
@@ -23,10 +55,10 @@ export class LexicalIndex {
       this.db.run('PRAGMA page_size=4096');
       this.db.run('PRAGMA max_page_count=32768');
       this.db.run(
-        'CREATE VIRTUAL TABLE sources USING fts5(id UNINDEXED, title, content, tokenize=unicode61)',
+        'CREATE VIRTUAL TABLE sources USING fts5(id UNINDEXED, title, content, tokenize=unicode61)'
       );
       const insert = this.db.prepare(
-        'INSERT INTO sources(rowid, id, title, content) VALUES (?, ?, ?, ?)',
+        'INSERT INTO sources(rowid, id, title, content) VALUES (?, ?, ?, ?)'
       );
       this.db.transaction(() => {
         for (const [index, record] of records.entries()) {
@@ -41,44 +73,31 @@ export class LexicalIndex {
   }
 
   search(query: string, limit = 32) {
-    return this.rank(searchTerms(query), limit);
-  }
-
-  private rank(terms: string[], limit: number, excludedId: string | null = null) {
-    const expression = terms.map((term) => `"${term.replaceAll('"', '""')}"`).join(' OR ');
-    if (!expression) return [];
-    return this.db
-      .prepare(
-        'SELECT id, bm25(sources) AS score FROM sources WHERE sources MATCH ? AND (? IS NULL OR id != ?) ORDER BY score, id LIMIT ?',
-      )
-      .all(expression, excludedId, excludedId, limit)
-      .map((value) => row.parse(value));
-  }
-
-  private prepareVocabulary() {
-    if (this.vocabularyReady) return;
-    this.db.run("CREATE VIRTUAL TABLE vocabulary USING fts5vocab(sources, 'row')");
-    this.db.run("CREATE VIRTUAL TABLE instances USING fts5vocab(sources, 'instance')");
-    this.db.run('CREATE TABLE document_terms AS SELECT DISTINCT doc, term FROM instances');
-    this.db.run('CREATE INDEX document_terms_by_doc ON document_terms(doc)');
-    this.vocabularyReady = true;
+    return rank({ database: this.db, limit, terms: searchTerms(query) });
   }
 
   neighbors(id: string, limit: number) {
-    const doc = this.ids.get(id);
-    if (doc === undefined)
+    const documentId = this.ids.get(id);
+    if (documentId === undefined) {
       throw new HivexError({
         code: 'LEXICAL_SOURCE_NOT_FOUND',
         message: 'The source is not in this lexical index',
       });
-    this.prepareVocabulary();
+    }
+    if (!this.vocabularyReady) {
+      prepareVocabulary(this.db);
+      this.vocabularyReady = true;
+    }
     const terms = this.db
       .query<{ term: string }, [number]>(
-        'SELECT t.term FROM document_terms t JOIN vocabulary v ON v.term=t.term WHERE t.doc=? AND v.doc>=2 ORDER BY v.doc, t.term LIMIT 32',
+        'SELECT t.term FROM document_terms t JOIN vocabulary v ON v.term=t.term WHERE t.doc=? AND v.doc>=2 ORDER BY v.doc, t.term LIMIT 32'
       )
-      .all(doc)
-      .map((row) => row.term);
-    return { terms, matches: this.rank(terms, limit, id) };
+      .all(documentId)
+      .map((entry) => entry.term);
+    return {
+      matches: rank({ database: this.db, excludedId: id, limit, terms }),
+      terms,
+    };
   }
 
   [Symbol.dispose]() {
@@ -86,7 +105,11 @@ export class LexicalIndex {
   }
 }
 
-export function rankLexically(records: readonly Record[], query: string, limit = 32) {
+export const rankLexically = function rankLexically(
+  records: readonly Record[],
+  query: string,
+  limit = 32
+) {
   using index = new LexicalIndex(records);
   return index.search(query, limit);
-}
+};

package/src/review.ts

@@ -1,32 +1,30 @@
 import { readFileSync, statSync } from 'node:fs';
-import { resolve } from 'node:path';
+import pathModule from 'node:path';
 import { parseArgs } from 'node:util';
 import { z } from 'zod';
-import { loadProject, type Project } from './documents.ts';
-import { captureImplementation, type Implementation } from './implementation.ts';
-import {
-  citationSchema,
-  sourceEvidence,
-  suppliedCitation,
-  type SuppliedDocument,
-} from './knowledge-model.ts';
+import { loadProject } from './documents.ts';
+import { captureImplementation } from './implementation.ts';
+import { citationSchema, sourceEvidence, suppliedCitation } from './knowledge-model.ts';
 import { HivexError } from './errors.ts';
+import type { Project } from './documents.ts';
+import type { Implementation } from './implementation.ts';
+import type { SuppliedDocument } from './knowledge-model.ts';
 
 const codeCitation = z.object({
+  lineEnd: z.number().int().positive(),
+  lineStart: z.number().int().positive(),
   path: z.string().min(1),
   side: z.enum(['before', 'after']),
-  lineStart: z.number().int().positive(),
-  lineEnd: z.number().int().positive(),
 });
 export const reviewSchema = z.object({
   findings: z
     .array(
       z.object({
         assessment: z.enum(['conflict', 'exception', 'uncertain']),
-        explanation: z.string().min(1).max(4096),
-        documents: z.array(citationSchema).max(8),
         code: z.array(codeCitation).max(8),
-      }),
+        documents: z.array(citationSchema).max(8),
+        explanation: z.string().min(1).max(4096),
+      })
     )
     .max(12),
   uncertainties: z.array(z.string().min(1).max(2048)).max(24),
@@ -34,96 +32,126 @@ export const reviewSchema = z.object({
 export const reviewInstructions =
   'Assist the principal reviewer with the task and implementation diff. Discover possible conflicts without requiring suspicions. Explain how documentary rules, direct/indirect dependencies, conditions and exceptions apply. Findings may identify a conflict, a valid exception, or uncertainty; do not turn missing context into approval or reject the entire change. Cite the supplied Markdown ranges and before/after code lines supporting each finding. Distinguish a rule violated by the change from behavior merely seen in context. The reviewer must verify each finding. This is knowledge assistance, not general code review, lint, tests or implementation approval.';
 
-function codeEvidence(citation: z.infer<typeof codeCitation>, implementation: Implementation) {
+const codeEvidence = function codeEvidence(
+  citation: z.infer<typeof codeCitation>,
+  implementation: Implementation
+) {
   const file = implementation.files.find((entry) => entry.path === citation.path)?.[citation.side];
-  if (!file || citation.lineEnd < citation.lineStart) return null;
+  if (!file || citation.lineEnd < citation.lineStart) {
+    return null;
+  }
   const lines = file.lines.filter(
-    ([number]) => number >= citation.lineStart && number <= citation.lineEnd,
+    ([number]) => number >= citation.lineStart && number <= citation.lineEnd
   );
-  if (lines.length !== citation.lineEnd - citation.lineStart + 1) return null;
-  return { ...citation, version: file.version, text: lines.map(([, text]) => text).join('\n') };
-}
-export function materializeReview(
+  if (lines.length !== citation.lineEnd - citation.lineStart + 1) {
+    return null;
+  }
+  return {
+    ...citation,
+    text: lines.map(([, text]) => text).join('\n'),
+    version: file.version,
+  };
+};
+
+export const materializeReview = function materializeReview(
   project: Project,
   implementation: Implementation,
-  value: unknown,
-  supplied: SuppliedDocument[],
+  { supplied, value }: { supplied: SuppliedDocument[]; value: unknown }
 ) {
   const response = reviewSchema.parse(value);
   const findings = response.findings.map((finding) => {
     const documents = finding.documents
-      .map((citation) =>
-        suppliedCitation(citation, supplied) ? sourceEvidence(citation, project) : null,
-      )
+      .map((citation) => {
+        const isSupplied = suppliedCitation(citation, supplied);
+        return isSupplied ? sourceEvidence(citation, project) : null;
+      })
       .filter((entry) => entry !== null);
     const code = finding.code
       .map((citation) => codeEvidence(citation, implementation))
       .filter((entry) => entry !== null);
-    const referencesVerified =
+    const areReferencesVerified =
       documents.length > 0 &&
       code.length > 0 &&
       documents.length === finding.documents.length &&
       code.length === finding.code.length;
     return {
       ...finding,
-      assessment: referencesVerified ? finding.assessment : 'uncertain',
-      documents,
+      assessment: areReferencesVerified ? finding.assessment : 'uncertain',
       code,
-      referencesVerified,
+      documents,
+      referencesVerified: areReferencesVerified,
     };
   });
   return {
     findings,
-    uncertainties: response.uncertainties,
     invalidReferences: findings.some((finding) => !finding.referencesVerified),
+    uncertainties: response.uncertainties,
   };
-}
+};
 
 const bindingSchema = z.object({
-  baseCommit: z.string().regex(/^[a-f0-9]{40,64}$/),
-  implementation: z.string().regex(/^[a-f0-9]{64}$/),
-  documents: z.string().regex(/^[a-f0-9]{64}$/),
+  baseCommit: z.string().regex(/^[a-f\d]{40,64}$/u),
+  documents: z.string().regex(/^[a-f\d]{64}$/u),
+  implementation: z.string().regex(/^[a-f\d]{64}$/u),
 });
-export function reviewBinding(project: Project, implementation: Implementation) {
+export const reviewBinding = function reviewBinding(
+  project: Project,
+  implementation: Implementation
+) {
   return {
     baseCommit: implementation.baseCommit,
-    implementation: implementation.fingerprint,
     documents: project.snapshot,
+    implementation: implementation.fingerprint,
   };
-}
-export function reviewFreshness(root: string, binding: z.infer<typeof bindingSchema>) {
+};
+
+export const reviewFreshness = function reviewFreshness(
+  root: string,
+  binding: z.infer<typeof bindingSchema>
+) {
   const project = loadProject(root);
   const implementation = captureImplementation(root, binding.baseCommit);
-  const documentsChanged = project.snapshot !== binding.documents;
-  const implementationChanged = implementation.fingerprint !== binding.implementation;
+  const areDocumentsChanged = project.snapshot !== binding.documents;
+  const isImplementationChanged = implementation.fingerprint !== binding.implementation;
   return {
-    status: documentsChanged || implementationChanged ? 'stale' : 'current',
-    documentsChanged,
-    implementationChanged,
+    documentsChanged: areDocumentsChanged,
+    implementationChanged: isImplementationChanged,
+    status: areDocumentsChanged || isImplementationChanged ? 'stale' : 'current',
   };
-}
-export function checkReview(args: string[]) {
+};
+
+export const checkReview = function checkReview(reviewArguments: string[]) {
   const parsed = parseArgs({
-    args,
     allowPositionals: true,
+    args: reviewArguments,
+    options: { check: { type: 'string' }, root: { type: 'string' } },
     strict: true,
-    options: { root: { type: 'string' }, check: { type: 'string' } },
   });
-  if (parsed.positionals.length !== 1 || parsed.positionals[0] !== 'review' || !parsed.values.check)
+  if (
+    parsed.positionals.length !== 1 ||
+    parsed.positionals[0] !== 'review' ||
+    typeof parsed.values.check !== 'string' ||
+    parsed.values.check.length === 0
+  ) {
     throw new HivexError({
       code: 'INVALID_ARGUMENT',
       message: 'Use review --check <saved-report.json> [--root <project>].',
     });
+  }
   const root = parsed.values.root ?? process.cwd();
-  const path = resolve(root, parsed.values.check);
-  if (statSync(path).size > 1048576)
-    throw new HivexError({ code: 'INVALID_REVIEW', message: 'Saved review exceeds 1 MiB.' });
+  const reportPath = pathModule.resolve(root, parsed.values.check);
+  if (statSync(reportPath).size > 1_048_576) {
+    throw new HivexError({
+      code: 'INVALID_REVIEW',
+      message: 'Saved review exceeds 1 MiB.',
+    });
+  }
   const report = z
-    .object({ command: z.literal('review'), binding: bindingSchema })
-    .parse(JSON.parse(readFileSync(path, 'utf8')));
+    .object({ binding: bindingSchema, command: z.literal('review') })
+    .parse(JSON.parse(readFileSync(reportPath, 'utf-8')));
   return {
     command: 'review-check',
     ...reviewFreshness(root, report.binding),
     guidance: 'Current means the versions still match, not that the implementation is approved.',
   };
-}
+};

package/src/runtime.d.ts

@@ -0,0 +1,4 @@
+// Bun 1.4 implements Iterator.concat before TypeScript includes its declaration.
+interface IteratorConstructor {
+  concat: <T>(...iterables: Iterable<T>[]) => IteratorObject<T, undefined>;
+}

package/src/snapshot-command.ts

@@ -0,0 +1,105 @@
+import { parseArgs } from 'node:util';
+import { compareSerializedStrings } from './ordering.ts';
+import { HivexError } from './errors.ts';
+import { ingestionUnits } from './ingestion-units.ts';
+import { KnowledgeStore } from './knowledge-store.ts';
+import { readKnowledgeSnapshot, writeKnowledgeSnapshot } from './knowledge-snapshot.ts';
+import { loadProject } from './documents.ts';
+import type { Graph } from './knowledge-model.ts';
+import type { Project } from './documents.ts';
+
+const sourceVersion = function sourceVersion([document, version]: [string, string]) {
+  return { document, version };
+};
+
+const sourceVersions = function sourceVersions(project: Project, graph: Graph) {
+  const references = [
+    ...Object.entries(graph.documents).map(sourceVersion),
+    ...Object.values(graph.units),
+    ...graph.decisions,
+    ...graph.relationships.flatMap((edge) => edge.evidence),
+  ];
+  const current = new Set<string>();
+  const stale = new Set<string>();
+  const unavailable = new Set<string>();
+  for (const reference of references) {
+    const source = project.documents.find((document) => document.id === reference.document);
+    if (source === undefined) {
+      unavailable.add(reference.document);
+    } else if (source.hash === reference.version) {
+      current.add(reference.document);
+    } else {
+      stale.add(reference.document);
+    }
+  }
+  return {
+    current: [...current].filter((id) => !stale.has(id)).toSorted(compareSerializedStrings),
+    stale: [...stale].toSorted(compareSerializedStrings),
+    unavailable: [...unavailable].toSorted(compareSerializedStrings),
+  };
+};
+
+const snapshotReport = function snapshotReport(project: Project, graph: Graph, operation: string) {
+  const plan = ingestionUnits(project.currentDocuments);
+  const pending = plan.units.filter((unit) => {
+    const source = project.documents.find((document) => document.id === unit.document);
+    return graph.units[unit.id]?.version !== source?.hash;
+  });
+  const sources = sourceVersions(project, graph);
+  const warnings = [...graph.warnings, ...project.warnings, ...plan.warnings];
+  const isPartial = [
+    pending.length > 0,
+    warnings.length > 0,
+    sources.stale.length > 0,
+    sources.unavailable.length > 0,
+    [...graph.decisions, ...graph.relationships].some((entry) => entry.quality !== 'checked'),
+  ].includes(true);
+  return {
+    command: 'snapshot',
+    decisions: graph.decisions.length,
+    modelCalls: 0,
+    operation,
+    path: '.hivex/graph.json',
+    pendingUnits: pending.map((unit) => unit.id),
+    relationships: graph.relationships.length,
+    sources,
+    status: isPartial ? 'partial' : 'ready',
+    warnings,
+  };
+};
+
+export const snapshotCommand = function snapshotCommand(argumentsList: string[]) {
+  const { positionals, values } = parseArgs({
+    allowPositionals: true,
+    args: argumentsList,
+    options: { root: { type: 'string' } },
+    strict: true,
+  });
+  const [, operation] = positionals;
+  if (
+    positionals.length !== 2 ||
+    positionals[0] !== 'snapshot' ||
+    (operation !== 'export' && operation !== 'import')
+  ) {
+    throw new HivexError({
+      code: 'INVALID_ARGUMENT',
+      message: 'Use snapshot export | import [--root <project>]',
+    });
+  }
+  const project = loadProject(values.root ?? process.cwd());
+  const incoming = operation === 'import' ? readKnowledgeSnapshot(project.root) : null;
+  if (operation === 'import' && incoming === null) {
+    throw new HivexError({
+      code: 'SNAPSHOT_NOT_FOUND',
+      message: 'No .hivex/graph.json snapshot is available.',
+    });
+  }
+  using store = new KnowledgeStore(project.root, { update: true });
+  const graph = incoming ?? store.graph();
+  if (incoming === null) {
+    writeKnowledgeSnapshot(project.root, graph);
+  } else {
+    store.importGraph(incoming);
+  }
+  return snapshotReport(project, graph, operation);
+};