@h1v35/hivex 0.1.0

package/src/implementation.ts

@@ -0,0 +1,191 @@
+import { spawnSync } from 'node:child_process';
+import { lstatSync, readFileSync, readlinkSync, realpathSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { HivexError } from './errors.ts';
+import { digest } from './knowledge-model.ts';
+import { rawMarkdownLines, lineContent } from './markdown.ts';
+
+type Version = { version: string; lines: Array<[number, string]> };
+export type Implementation = {
+  baseCommit: string;
+  fingerprint: string;
+  diff: string;
+  files: Array<{ path: string; before: Version | null; after: Version | null }>;
+  warnings: string[];
+};
+const maxBytes = 256 * 1024;
+const maxFileBytes = 4 * 1024 * 1024;
+const protectedDirectories = new Set(['.git', '.hivex', 'node_modules', '.codex']);
+const decoder = new TextDecoder('utf-8', { fatal: true });
+
+function git(root: string, args: string[]) {
+  const result = spawnSync('git', ['--literal-pathspecs', ...args], {
+    cwd: root,
+    maxBuffer: maxFileBytes + 1,
+    timeout: 30000,
+    env: { ...process.env, LC_ALL: 'C', GIT_OPTIONAL_LOCKS: '0' },
+  });
+  if (result.error || result.status !== 0) {
+    const tooLarge = result.error && 'code' in result.error && result.error.code === 'ENOBUFS';
+    throw new HivexError({
+      code: tooLarge ? 'IMPLEMENTATION_TOO_LARGE' : 'GIT_COMMAND_FAILED',
+      message: result.error?.message ?? result.stderr.toString('utf8').trim().slice(0, 1024),
+    });
+  }
+  return result.stdout;
+}
+function text(root: string, args: string[]) {
+  return decoder.decode(git(root, args));
+}
+function checkSize(bytes: number, limit = maxBytes) {
+  if (bytes > limit)
+    throw new HivexError({
+      code: 'IMPLEMENTATION_TOO_LARGE',
+      message: `Implementation exceeds ${limit} bytes; split the change into coherent reviews.`,
+    });
+}
+function version(bytes: Buffer, label: string, warnings: string[]): Version | null {
+  checkSize(bytes.byteLength, maxFileBytes);
+  let content: string;
+  try {
+    content = decoder.decode(bytes);
+    if (content.includes('\0')) throw new Error('binary');
+  } catch {
+    warnings.push(
+      `Unsupported binary or invalid UTF-8 content: ${label} (${digest(bytes.toString('base64'))})`,
+    );
+    return null;
+  }
+  return {
+    version: digest(bytes),
+    lines: rawMarkdownLines(content).map((line, index) => [index + 1, lineContent(line)]),
+  };
+}
+function beforeVersion(root: string, base: string, path: string, warnings: string[]) {
+  const entry = text(root, ['ls-tree', '-z', base, '--', path])
+    .split('\0')
+    .find((row) => row.slice(row.indexOf('\t') + 1) === path);
+  if (!entry) return null;
+  const [mode, kind, object] = entry.split('\t')[0]!.split(' ');
+  if (kind !== 'blob' || !mode?.startsWith('100')) {
+    warnings.push(`Unsupported base file: ${path} (${mode} ${object})`);
+    return null;
+  }
+  return version(git(root, ['cat-file', 'blob', object!]), `before ${path}`, warnings);
+}
+function afterVersion(root: string, path: string, warnings: string[]) {
+  const absolute = resolve(root, path);
+  let stat;
+  try {
+    stat = lstatSync(absolute);
+  } catch (error) {
+    if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT')
+      return null;
+    throw error;
+  }
+  if (stat.isSymbolicLink()) {
+    warnings.push(`Unsupported working symlink: ${path} (${digest(readlinkSync(absolute))})`);
+    return null;
+  }
+  if (!stat.isFile() || !realpathSync(absolute).startsWith(root + '/')) {
+    warnings.push(`Unsupported working file: ${path}`);
+    return null;
+  }
+  checkSize(stat.size, maxFileBytes);
+  return version(readFileSync(absolute), `after ${path}`, warnings);
+}
+function patch(root: string, base: string, paths: string[]) {
+  if (!paths.length) return '';
+  return text(root, [
+    'diff',
+    '--no-ext-diff',
+    '--no-textconv',
+    '--no-renames',
+    '--no-color',
+    '--src-prefix=a/',
+    '--dst-prefix=b/',
+    '--unified=3',
+    base,
+    '--',
+    ...paths,
+  ]);
+}
+function fileContext(
+  root: string,
+  base: string,
+  file: Implementation['files'][number],
+  warnings: string[],
+) {
+  if (Buffer.byteLength(JSON.stringify(file)) <= 32768) return file;
+  const hunks = [
+    ...patch(root, base, [file.path]).matchAll(/^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@/gm),
+  ];
+  if (!hunks.length) return file;
+  const excerpt = (version: Version | null, offset: number) =>
+    version && {
+      ...version,
+      lines: version.lines.filter(([line]) =>
+        hunks.some(
+          (hunk) =>
+            line >= Number(hunk[offset]) &&
+            line < Number(hunk[offset]) + Number(hunk[offset + 1] ?? 1),
+        ),
+      ),
+    };
+  warnings.push(`Only changed ranges are supplied for ${file.path}; unchanged code is omitted.`);
+  return { ...file, before: excerpt(file.before, 1), after: excerpt(file.after, 3) };
+}
+
+export function captureImplementation(root: string, base: string): Implementation {
+  const actualRoot = realpathSync(resolve(root));
+  if (realpathSync(text(actualRoot, ['rev-parse', '--show-toplevel']).trim()) !== actualRoot)
+    throw new HivexError({ code: 'INVALID_ROOT', message: 'Review from the Git project root.' });
+  const baseCommit = text(actualRoot, [
+    'rev-parse',
+    '--verify',
+    '--end-of-options',
+    base + '^{commit}',
+  ]).trim();
+  const tracked = text(actualRoot, [
+    'diff',
+    '--no-ext-diff',
+    '--no-textconv',
+    '--no-renames',
+    '--name-only',
+    '-z',
+    baseCommit,
+    '--',
+  ]).split('\0');
+  const untracked = text(actualRoot, ['ls-files', '--others', '--exclude-standard', '-z']).split(
+    '\0',
+  );
+  const paths = [...new Set([...tracked, ...untracked])]
+    .filter((path) => path && !path.split('/').some((part) => protectedDirectories.has(part)))
+    .sort();
+  if (paths.length > 64)
+    throw new HivexError({
+      code: 'IMPLEMENTATION_TOO_LARGE',
+      message: 'Implementation exceeds 64 files; split the change into coherent reviews.',
+    });
+  const warnings: string[] = [];
+  const files = paths.map((path) =>
+    fileContext(
+      actualRoot,
+      baseCommit,
+      {
+        path,
+        before: beforeVersion(actualRoot, baseCommit, path, warnings),
+        after: afterVersion(actualRoot, path, warnings),
+      },
+      warnings,
+    ),
+  );
+  let diff = patch(actualRoot, baseCommit, paths);
+  diff += paths
+    .filter((path) => untracked.includes(path))
+    .map((path) => `\nNew untracked file: ${JSON.stringify(path)}\n`)
+    .join('');
+  const packet = { baseCommit, diff, files, warnings };
+  checkSize(Buffer.byteLength(JSON.stringify(packet)));
+  return { ...packet, fingerprint: digest(JSON.stringify(packet)) };
+}

package/src/ingestion-units.ts

@@ -0,0 +1,155 @@
+import type { Document } from './documents.ts';
+import { rawMarkdownLines } from './markdown.ts';
+import { digest } from './knowledge-model.ts';
+
+const MAX_BYTES = 8192;
+
+export type IngestionUnit = {
+  id: string;
+  document: string;
+  hash: string;
+  lineStart: number;
+  lineEnd: number;
+  text: string;
+};
+
+type SourceLine = {
+  number: number;
+  text: string;
+  bytes: number;
+  blank: boolean;
+  heading: boolean;
+  fence: { marker: string; length: number; closing: boolean } | null;
+};
+type Warning = { path: string; message: string };
+
+const contentOf = (text: string) => text.replace(/(?:\r\n|\r|\n)$/, '');
+function fenceOf(content: string) {
+  const match = /^ {0,3}(`{3,}|~{3,})(.*)$/.exec(content);
+  return match?.[1]
+    ? { marker: match[1].charAt(0), length: match[1].length, closing: !match[2]?.trim() }
+    : null;
+}
+const headingOf = (content: string) => /^\s{0,3}#{1,6}(?:\s|$)/.test(content);
+
+function sourceLines(text: string) {
+  return rawMarkdownLines(text)
+    .filter((line) => line !== '')
+    .map((line, index) => {
+      const content = contentOf(line);
+      return {
+        number: index + 1,
+        text: line,
+        bytes: Buffer.byteLength(line, 'utf8'),
+        blank: content.trim() === '',
+        heading: headingOf(content),
+        fence: fenceOf(content),
+      };
+    });
+}
+
+function blocksFor(document: Document, warnings: Warning[]) {
+  const blocks: SourceLine[][] = [];
+  let block: SourceLine[] = [];
+  let activeFence: SourceLine['fence'] = null;
+  const flush = () => {
+    if (block.length) blocks.push(block);
+    block = [];
+  };
+
+  for (const line of sourceLines(document.text)) {
+    const inFence = activeFence !== null;
+    const closingFence =
+      activeFence &&
+      line.fence?.closing &&
+      line.fence.marker === activeFence.marker &&
+      line.fence.length >= activeFence.length;
+    if (closingFence) activeFence = null;
+    else if (!activeFence) activeFence = line.fence;
+    if (line.bytes > MAX_BYTES) {
+      flush();
+      warnings.push({
+        path: document.path,
+        message: `Line ${line.number} is ${line.bytes} UTF-8 bytes, exceeding the ${MAX_BYTES}-byte limit; omitted as unread.`,
+      });
+      continue;
+    }
+    if (!inFence && line.heading) flush();
+    block.push(line);
+    if (!activeFence && (line.blank || closingFence)) flush();
+  }
+  flush();
+  return blocks;
+}
+
+function splitBlock(block: SourceLine[]) {
+  const pieces: SourceLine[][] = [];
+  let piece: SourceLine[] = [];
+  let bytes = 0;
+  for (const line of block) {
+    if (piece.length && bytes + line.bytes > MAX_BYTES) {
+      pieces.push(piece);
+      piece = [];
+      bytes = 0;
+    }
+    piece.push(line);
+    bytes += line.bytes;
+  }
+  if (piece.length) pieces.push(piece);
+  return pieces;
+}
+
+function packedBlocks(blocks: SourceLine[][]) {
+  const packed: SourceLine[][] = [];
+  let current: SourceLine[] = [];
+  let bytes = 0;
+  const flush = () => {
+    if (current.length) packed.push(current);
+    current = [];
+    bytes = 0;
+  };
+
+  for (const block of blocks.flatMap((item) => splitBlock(item))) {
+    const first = block.at(0);
+    if (!first) continue;
+    const blockBytes = block.reduce((total, line) => total + line.bytes, 0);
+    const last = current.at(-1);
+    if (
+      current.length &&
+      (bytes + blockBytes > MAX_BYTES || !last || last.number + 1 !== first.number)
+    )
+      flush();
+    current.push(...block);
+    bytes += blockBytes;
+  }
+  flush();
+  return packed;
+}
+
+function makeUnit(document: Document, lines: SourceLine[]): IngestionUnit {
+  const first = lines.at(0);
+  const last = lines.at(-1);
+  if (!first || !last) throw new Error('Cannot create an empty ingestion unit');
+  const text = lines.map((line) => line.text).join('');
+  return {
+    id: `${document.path}:${first.number}-${last.number}`,
+    document: document.id,
+    hash: digest(text),
+    lineStart: first.number,
+    lineEnd: last.number,
+    text,
+  };
+}
+
+export function ingestionUnits(documents: Document[]): {
+  units: IngestionUnit[];
+  warnings: Warning[];
+} {
+  const units: IngestionUnit[] = [];
+  const warnings: Warning[] = [];
+  for (const document of documents)
+    units.push(
+      ...packedBlocks(blocksFor(document, warnings)).map((lines) => makeUnit(document, lines)),
+    );
+  return { units, warnings };
+}

package/src/knowledge-maintenance.ts

@@ -0,0 +1,76 @@
+import { parseArgs } from 'node:util';
+import { HivexError } from './errors.ts';
+import { KnowledgeStore } from './knowledge-store.ts';
+
+const DEFAULT_KEEP_COMPLETED = 8;
+const DEFAULT_KEEP_CACHES = 64;
+
+function retention(value: string | undefined, name: string, fallback: number) {
+  if (value === undefined) return fallback;
+  const number = Number(value);
+  if (!Number.isInteger(number) || number < 0 || number > 4096)
+    throw new HivexError({
+      code: 'INVALID_ARGUMENT',
+      message: `${name} must be an integer between 0 and 4096`,
+    });
+  return number;
+}
+
+export function knowledgeMaintenance(args: string[]) {
+  const parsed = parseArgs({
+    args,
+    allowPositionals: true,
+    strict: true,
+    options: {
+      root: { type: 'string' },
+      'keep-completed': { type: 'string' },
+      'keep-caches': { type: 'string' },
+      'acknowledge-uncertain': { type: 'boolean' },
+    },
+  });
+  const command = parsed.positionals[0];
+  if (!command || parsed.positionals.length !== 1 || !['recover', 'prune'].includes(command))
+    throw new HivexError({
+      code: 'INVALID_ARGUMENT',
+      message:
+        'Use recover [--acknowledge-uncertain] or prune [--keep-completed <count>] [--keep-caches <count>]',
+    });
+  if (
+    command === 'recover' &&
+    (parsed.values['keep-completed'] !== undefined || parsed.values['keep-caches'] !== undefined)
+  )
+    throw new HivexError({
+      code: 'INVALID_ARGUMENT',
+      message: 'recover does not accept retention options',
+    });
+  if (command === 'prune' && parsed.values['acknowledge-uncertain'])
+    throw new HivexError({
+      code: 'INVALID_ARGUMENT',
+      message: 'prune does not accept --acknowledge-uncertain',
+    });
+
+  const root = parsed.values.root ?? process.cwd();
+  using store = new KnowledgeStore(root);
+  if (command === 'recover')
+    return {
+      command,
+      modelCalls: 0,
+      ...store.recover({
+        acknowledgeUncertain: parsed.values['acknowledge-uncertain'] ?? false,
+      }),
+    };
+
+  using _lease = store.updateLease();
+  return {
+    command,
+    modelCalls: 0,
+    ...store.prune({
+      keepCompleted: retention(
+        parsed.values['keep-completed'],
+        '--keep-completed',
+        DEFAULT_KEEP_COMPLETED,
+      ),
+      keepCaches: retention(parsed.values['keep-caches'], '--keep-caches', DEFAULT_KEEP_CACHES),
+    }),
+  };
+}