@h1v35/hivex 0.2.0 → 0.2.1

package/src/errors.ts

@@ -1,13 +1,15 @@
+interface HivexErrorOptions extends ErrorOptions {
+  code: string;
+  details?: Readonly<Record<string, unknown>>;
+  message: string;
+}
+
 export class HivexError extends Error {
   readonly code: string;
   readonly details: Readonly<Record<string, unknown>> | undefined;
 
-  constructor(options: {
-    code: string;
-    message: string;
-    details?: Readonly<Record<string, unknown>>;
-  }) {
-    super(options.message);
+  constructor(options: HivexErrorOptions) {
+    super(options.message, options);
     this.name = 'HivexError';
     this.code = options.code;
     this.details = options.details;

package/src/implementation.ts

@@ -1,101 +1,154 @@
 import { spawnSync } from 'node:child_process';
 import { lstatSync, readFileSync, readlinkSync, realpathSync } from 'node:fs';
-import { resolve } from 'node:path';
+import pathModule from 'node:path';
+import { compareSerializedStrings } from './ordering.ts';
 import { HivexError } from './errors.ts';
 import { digest } from './knowledge-model.ts';
 import { rawMarkdownLines, lineContent } from './markdown.ts';
 
-type Version = { version: string; lines: Array<[number, string]> };
-export type Implementation = {
+interface Version {
+  version: string;
+  lines: [number, string][];
+}
+export interface Implementation {
   baseCommit: string;
   fingerprint: string;
   diff: string;
-  files: Array<{ path: string; before: Version | null; after: Version | null }>;
+  files: { path: string; before: Version | null; after: Version | null }[];
   warnings: string[];
-};
+}
+interface GitContext {
+  base: string;
+  root: string;
+  warnings: string[];
+}
 const maxBytes = 256 * 1024;
 const maxFileBytes = 4 * 1024 * 1024;
 const protectedDirectories = new Set(['.git', '.hivex', 'node_modules', '.codex']);
+const gitOptionalLocks = 'GIT_OPTIONAL_LOCKS';
+const localeAll = 'LC_ALL';
 const decoder = new TextDecoder('utf-8', { fatal: true });
 
-function git(root: string, args: string[]) {
-  const result = spawnSync('git', ['--literal-pathspecs', ...args], {
+const isMissingFile = function isMissingFile(error: unknown) {
+  return error !== null && typeof error === 'object' && 'code' in error && error.code === 'ENOENT';
+};
+
+const git = function git(root: string, gitArguments: string[]) {
+  const executable = Bun.which('git') ?? 'git';
+  const result = spawnSync(executable, ['--literal-pathspecs', ...gitArguments], {
     cwd: root,
+    env: {
+      ...process.env,
+      [gitOptionalLocks]: '0',
+      [localeAll]: 'C',
+    },
     maxBuffer: maxFileBytes + 1,
-    timeout: 30000,
-    env: { ...process.env, LC_ALL: 'C', GIT_OPTIONAL_LOCKS: '0' },
+    timeout: 30_000,
   });
   if (result.error || result.status !== 0) {
-    const tooLarge = result.error && 'code' in result.error && result.error.code === 'ENOBUFS';
+    const isTooLarge =
+      result.error !== undefined && 'code' in result.error && result.error.code === 'ENOBUFS';
     throw new HivexError({
-      code: tooLarge ? 'IMPLEMENTATION_TOO_LARGE' : 'GIT_COMMAND_FAILED',
-      message: result.error?.message ?? result.stderr.toString('utf8').trim().slice(0, 1024),
+      code: isTooLarge ? 'IMPLEMENTATION_TOO_LARGE' : 'GIT_COMMAND_FAILED',
+      message: result.error?.message ?? result.stderr.toString('utf-8').trim().slice(0, 1024),
     });
   }
   return result.stdout;
-}
-function text(root: string, args: string[]) {
-  return decoder.decode(git(root, args));
-}
-function checkSize(bytes: number, limit = maxBytes) {
-  if (bytes > limit)
+};
+
+const text = function text(root: string, gitArguments: string[]) {
+  return decoder.decode(git(root, gitArguments));
+};
+
+const checkSize = function checkSize(bytes: number, limit = maxBytes) {
+  if (bytes > limit) {
     throw new HivexError({
       code: 'IMPLEMENTATION_TOO_LARGE',
       message: `Implementation exceeds ${limit} bytes; split the change into coherent reviews.`,
     });
-}
-function version(bytes: Buffer, label: string, warnings: string[]): Version | null {
+  }
+};
+
+const unsupportedVersion = function unsupportedVersion(
+  bytes: Buffer,
+  label: string,
+  warnings: string[]
+): null {
+  warnings.push(
+    `Unsupported binary or invalid UTF-8 content: ${label} (${digest(bytes.toBase64())})`
+  );
+  return null;
+};
+
+const version = function version(bytes: Buffer, label: string, warnings: string[]): Version | null {
   checkSize(bytes.byteLength, maxFileBytes);
   let content: string;
   try {
     content = decoder.decode(bytes);
-    if (content.includes('\0')) throw new Error('binary');
   } catch {
-    warnings.push(
-      `Unsupported binary or invalid UTF-8 content: ${label} (${digest(bytes.toString('base64'))})`,
-    );
-    return null;
+    return unsupportedVersion(bytes, label, warnings);
+  }
+  if (content.includes('\u{0}')) {
+    return unsupportedVersion(bytes, label, warnings);
   }
   return {
+    lines: rawMarkdownLines(content).map<[number, string]>((line, index) => {
+      const lineNumber = index + 1;
+      return [lineNumber, lineContent(line)];
+    }),
     version: digest(bytes),
-    lines: rawMarkdownLines(content).map((line, index) => [index + 1, lineContent(line)]),
   };
-}
-function beforeVersion(root: string, base: string, path: string, warnings: string[]) {
+};
+
+const beforeVersion = function beforeVersion(context: GitContext, path: string) {
+  const { base, root, warnings } = context;
   const entry = text(root, ['ls-tree', '-z', base, '--', path])
     .split('\0')
     .find((row) => row.slice(row.indexOf('\t') + 1) === path);
-  if (!entry) return null;
-  const [mode, kind, object] = entry.split('\t')[0]!.split(' ');
-  if (kind !== 'blob' || !mode?.startsWith('100')) {
+  if (entry === undefined) {
+    return null;
+  }
+  const header = entry.slice(0, entry.indexOf('\t'));
+  const [mode, kind, object] = header.split(' ', 3);
+  if (kind !== 'blob' || mode === undefined || object === undefined) {
     warnings.push(`Unsupported base file: ${path} (${mode} ${object})`);
     return null;
   }
-  return version(git(root, ['cat-file', 'blob', object!]), `before ${path}`, warnings);
-}
-function afterVersion(root: string, path: string, warnings: string[]) {
-  const absolute = resolve(root, path);
+  if (!mode.startsWith('100')) {
+    warnings.push(`Unsupported base file: ${path} (${mode} ${object})`);
+    return null;
+  }
+  return version(git(root, ['cat-file', 'blob', object]), `before ${path}`, warnings);
+};
+
+const afterVersion = function afterVersion(context: GitContext, path: string) {
+  const { root, warnings } = context;
+  const absolute = pathModule.resolve(root, path);
   let stat;
   try {
     stat = lstatSync(absolute);
   } catch (error) {
-    if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT')
+    if (isMissingFile(error)) {
       return null;
+    }
     throw error;
   }
   if (stat.isSymbolicLink()) {
     warnings.push(`Unsupported working symlink: ${path} (${digest(readlinkSync(absolute))})`);
     return null;
   }
-  if (!stat.isFile() || !realpathSync(absolute).startsWith(root + '/')) {
+  if (!stat.isFile() || !realpathSync(absolute).startsWith(`${root}/`)) {
     warnings.push(`Unsupported working file: ${path}`);
     return null;
   }
   checkSize(stat.size, maxFileBytes);
   return version(readFileSync(absolute), `after ${path}`, warnings);
-}
-function patch(root: string, base: string, paths: string[]) {
-  if (!paths.length) return '';
+};
+
+const patch = function patch(root: string, base: string, paths: string[]) {
+  if (!paths.length) {
+    return '';
+  }
   return text(root, [
     'diff',
     '--no-ext-diff',
@@ -109,42 +162,72 @@ function patch(root: string, base: string, paths: string[]) {
     '--',
     ...paths,
   ]);
-}
-function fileContext(
-  root: string,
-  base: string,
-  file: Implementation['files'][number],
-  warnings: string[],
+};
+
+const fileContext = function fileContext(
+  context: GitContext,
+  file: Implementation['files'][number]
 ) {
-  if (Buffer.byteLength(JSON.stringify(file)) <= 32768) return file;
-  const hunks = [
-    ...patch(root, base, [file.path]).matchAll(/^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@/gm),
-  ];
-  if (!hunks.length) return file;
-  const excerpt = (version: Version | null, offset: number) =>
-    version && {
-      ...version,
-      lines: version.lines.filter(([line]) =>
-        hunks.some(
-          (hunk) =>
-            line >= Number(hunk[offset]) &&
-            line < Number(hunk[offset]) + Number(hunk[offset + 1] ?? 1),
-        ),
-      ),
+  const { base, root, warnings } = context;
+  if (Buffer.byteLength(JSON.stringify(file)) <= 32_768) {
+    return file;
+  }
+  const hunks = patch(root, base, [file.path])
+    .matchAll(
+      /^@@ -(?<beforeStart>\d+)(?:,(?<beforeLength>\d+))? \+(?<afterStart>\d+)(?:,(?<afterLength>\d+))? @@/gmu
+    )
+    .toArray();
+  if (!hunks.length) {
+    return file;
+  }
+  const excerpt = function excerpt(
+    documentVersion: Version | null,
+    startGroup: 'beforeStart' | 'afterStart',
+    lengthGroup: 'beforeLength' | 'afterLength'
+  ) {
+    if (documentVersion === null) {
+      return null;
+    }
+    const isLineInHunk = function isLineInHunk([line]: [number, string]) {
+      return hunks.some((hunk) => {
+        const { groups } = hunk;
+        if (groups === undefined) {
+          return false;
+        }
+        const start = Number(groups[startGroup]);
+        const length = Number(groups[lengthGroup] ?? 1);
+        return line >= start && line < start + length;
+      });
     };
+    return {
+      lines: documentVersion.lines.filter(isLineInHunk),
+      version: documentVersion.version,
+    };
+  };
   warnings.push(`Only changed ranges are supplied for ${file.path}; unchanged code is omitted.`);
-  return { ...file, before: excerpt(file.before, 1), after: excerpt(file.after, 3) };
-}
+  return {
+    ...file,
+    after: excerpt(file.after, 'afterStart', 'afterLength'),
+    before: excerpt(file.before, 'beforeStart', 'beforeLength'),
+  };
+};
 
-export function captureImplementation(root: string, base: string): Implementation {
-  const actualRoot = realpathSync(resolve(root));
-  if (realpathSync(text(actualRoot, ['rev-parse', '--show-toplevel']).trim()) !== actualRoot)
-    throw new HivexError({ code: 'INVALID_ROOT', message: 'Review from the Git project root.' });
+export const captureImplementation = function captureImplementation(
+  root: string,
+  base: string
+): Implementation {
+  const actualRoot = realpathSync(pathModule.resolve(root));
+  if (realpathSync(text(actualRoot, ['rev-parse', '--show-toplevel']).trim()) !== actualRoot) {
+    throw new HivexError({
+      code: 'INVALID_ROOT',
+      message: 'Review from the Git project root.',
+    });
+  }
   const baseCommit = text(actualRoot, [
     'rev-parse',
     '--verify',
     '--end-of-options',
-    base + '^{commit}',
+    base.concat('^{commit}'),
   ]).trim();
   const tracked = text(actualRoot, [
     'diff',
@@ -157,29 +240,32 @@ export function captureImplementation(root: string, base: string): Implementatio
     '--',
   ]).split('\0');
   const untracked = text(actualRoot, ['ls-files', '--others', '--exclude-standard', '-z']).split(
-    '\0',
+    '\0'
   );
-  const paths = [...new Set([...tracked, ...untracked])]
-    .filter((path) => path && !path.split('/').some((part) => protectedDirectories.has(part)))
-    .sort();
-  if (paths.length > 64)
+  const changedPaths = new Set(Iterator.concat(tracked, untracked));
+  const paths = changedPaths
+    .values()
+    .filter((path) => {
+      const isOutsideProtectedDirectory = path
+        .split('/')
+        .every((part) => !protectedDirectories.has(part));
+      return path.length > 0 && isOutsideProtectedDirectory;
+    })
+    .toArray()
+    .toSorted(compareSerializedStrings);
+  if (paths.length > 64) {
     throw new HivexError({
       code: 'IMPLEMENTATION_TOO_LARGE',
       message: 'Implementation exceeds 64 files; split the change into coherent reviews.',
     });
+  }
   const warnings: string[] = [];
-  const files = paths.map((path) =>
-    fileContext(
-      actualRoot,
-      baseCommit,
-      {
-        path,
-        before: beforeVersion(actualRoot, baseCommit, path, warnings),
-        after: afterVersion(actualRoot, path, warnings),
-      },
-      warnings,
-    ),
-  );
+  const context = { base: baseCommit, root: actualRoot, warnings };
+  const files = paths.map((path) => {
+    const before = beforeVersion(context, path);
+    const after = afterVersion(context, path);
+    return fileContext(context, { after, before, path });
+  });
   let diff = patch(actualRoot, baseCommit, paths);
   diff += paths
     .filter((path) => untracked.includes(path))
@@ -187,5 +273,17 @@ export function captureImplementation(root: string, base: string): Implementatio
     .join('');
   const packet = { baseCommit, diff, files, warnings };
   checkSize(Buffer.byteLength(JSON.stringify(packet)));
-  return { ...packet, fingerprint: digest(JSON.stringify(packet)) };
-}
+  // Retain the nested field order of the original persisted fingerprint format.
+  const fields = [
+    'baseCommit',
+    'diff',
+    'files',
+    'warnings',
+    'path',
+    'before',
+    'after',
+    'version',
+    'lines',
+  ];
+  return { ...packet, fingerprint: digest(JSON.stringify(packet, fields)) };
+};

package/src/ingestion-units.ts

@@ -1,93 +1,125 @@
-import type { Document } from './documents.ts';
-import { rawMarkdownLines } from './markdown.ts';
 import { digest } from './knowledge-model.ts';
+import { rawMarkdownLines } from './markdown.ts';
+import type { Document } from './documents.ts';
 
-const MAX_BYTES = 8192;
+const maxBytes = 8192;
 
-export type IngestionUnit = {
-  id: string;
+export interface IngestionUnit {
   document: string;
   hash: string;
-  lineStart: number;
+  id: string;
   lineEnd: number;
+  lineStart: number;
   text: string;
-};
+}
 
-type SourceLine = {
-  number: number;
-  text: string;
-  bytes: number;
+interface SourceLine {
   blank: boolean;
-  heading: boolean;
+  bytes: number;
   fence: { marker: string; length: number; closing: boolean } | null;
-};
-type Warning = { path: string; message: string };
-
-const contentOf = (text: string) => text.replace(/(?:\r\n|\r|\n)$/, '');
-function fenceOf(content: string) {
-  const match = /^ {0,3}(`{3,}|~{3,})(.*)$/.exec(content);
-  return match?.[1]
-    ? { marker: match[1].charAt(0), length: match[1].length, closing: !match[2]?.trim() }
-    : null;
+  heading: boolean;
+  number: number;
+  text: string;
+}
+interface Warning {
+  message: string;
+  path: string;
 }
-const headingOf = (content: string) => /^\s{0,3}#{1,6}(?:\s|$)/.test(content);
 
-function sourceLines(text: string) {
+const contentOf = function contentOf(text: string) {
+  return text.replace(/(?:\r\n|\r|\n)$/u, '');
+};
+const fenceOf = function fenceOf(content: string) {
+  const indentationMatch = /^ */u.exec(content);
+  const indentation = indentationMatch?.[0].length ?? 0;
+  if (indentation > 3) {
+    return null;
+  }
+  const source = content.slice(indentation);
+  const marker = source.at(0);
+  if (marker !== '`' && marker !== '~') {
+    return null;
+  }
+  const markerMatch = /^(?:`+|~+)/u.exec(source);
+  const markerRun = markerMatch?.[0] ?? '';
+  if (markerRun.length < 3) {
+    return null;
+  }
+  return {
+    closing: source.slice(markerRun.length).trim() === '',
+    length: markerRun.length,
+    marker,
+  };
+};
+const isHeading = (content: string) => /^\s{0,3}#{1,6}(?:\s|$)/u.test(content);
+
+const sourceLines = function sourceLines(text: string) {
   return rawMarkdownLines(text)
     .filter((line) => line !== '')
     .map((line, index) => {
       const content = contentOf(line);
       return {
-        number: index + 1,
-        text: line,
-        bytes: Buffer.byteLength(line, 'utf8'),
         blank: content.trim() === '',
-        heading: headingOf(content),
+        bytes: Buffer.byteLength(line, 'utf-8'),
         fence: fenceOf(content),
+        heading: isHeading(content),
+        number: index + 1,
+        text: line,
       };
     });
-}
+};
 
-function blocksFor(document: Document, warnings: Warning[]) {
+const blocksFor = function blocksFor(document: Document, warnings: Warning[]) {
   const blocks: SourceLine[][] = [];
   let block: SourceLine[] = [];
   let activeFence: SourceLine['fence'] = null;
   const flush = () => {
-    if (block.length) blocks.push(block);
+    if (block.length > 0) {
+      blocks.push(block);
+    }
     block = [];
   };
 
   for (const line of sourceLines(document.text)) {
-    const inFence = activeFence !== null;
-    const closingFence =
-      activeFence &&
-      line.fence?.closing &&
+    const isInFence = activeFence !== null;
+    const isSameFence =
+      activeFence !== null &&
+      line.fence !== null &&
       line.fence.marker === activeFence.marker &&
       line.fence.length >= activeFence.length;
-    if (closingFence) activeFence = null;
-    else if (!activeFence) activeFence = line.fence;
-    if (line.bytes > MAX_BYTES) {
+    const isClosingFence = line.fence?.closing === true && isSameFence;
+    if (isClosingFence) {
+      activeFence = null;
+    } else {
+      activeFence ??= line.fence;
+    }
+    if (line.bytes > maxBytes) {
       flush();
       warnings.push({
+        message: `Line ${line.number} is ${line.bytes} UTF-8 bytes, exceeding the ${maxBytes}-byte limit; omitted as unread.`,
         path: document.path,
-        message: `Line ${line.number} is ${line.bytes} UTF-8 bytes, exceeding the ${MAX_BYTES}-byte limit; omitted as unread.`,
       });
       continue;
     }
-    if (!inFence && line.heading) flush();
+    if (!isInFence && line.heading) {
+      flush();
+    }
     block.push(line);
-    if (!activeFence && (line.blank || closingFence)) flush();
+    const shouldFlush = activeFence === null && (line.blank || isClosingFence);
+    if (shouldFlush) {
+      flush();
+    }
   }
   flush();
   return blocks;
-}
+};
 
-function splitBlock(block: SourceLine[]) {
+const splitBlock = function splitBlock(block: SourceLine[]) {
   const pieces: SourceLine[][] = [];
   let piece: SourceLine[] = [];
   let bytes = 0;
   for (const line of block) {
-    if (piece.length && bytes + line.bytes > MAX_BYTES) {
+    if (piece.length > 0 && bytes + line.bytes > maxBytes) {
       pieces.push(piece);
       piece = [];
       bytes = 0;
@@ -95,61 +127,72 @@ function splitBlock(block: SourceLine[]) {
     piece.push(line);
     bytes += line.bytes;
   }
-  if (piece.length) pieces.push(piece);
+  if (piece.length > 0) {
+    pieces.push(piece);
+  }
   return pieces;
-}
+};
 
-function packedBlocks(blocks: SourceLine[][]) {
+const packedBlocks = function packedBlocks(blocks: SourceLine[][]) {
   const packed: SourceLine[][] = [];
   let current: SourceLine[] = [];
   let bytes = 0;
   const flush = () => {
-    if (current.length) packed.push(current);
+    if (current.length > 0) {
+      packed.push(current);
+    }
     current = [];
     bytes = 0;
   };
 
-  for (const block of blocks.flatMap((item) => splitBlock(item))) {
+  const splitBlocks = blocks.flatMap((item) => splitBlock(item));
+  for (const block of splitBlocks) {
     const first = block.at(0);
-    if (!first) continue;
+    if (first === undefined) {
+      continue;
+    }
     const blockBytes = block.reduce((total, line) => total + line.bytes, 0);
     const last = current.at(-1);
-    if (
-      current.length &&
-      (bytes + blockBytes > MAX_BYTES || !last || last.number + 1 !== first.number)
-    )
+    const shouldFlush =
+      last === undefined || last.number + 1 !== first.number || bytes + blockBytes > maxBytes;
+    if (shouldFlush && current.length > 0) {
       flush();
+    }
     current.push(...block);
     bytes += blockBytes;
   }
   flush();
   return packed;
-}
+};
 
-function makeUnit(document: Document, lines: SourceLine[]): IngestionUnit {
+const makeUnit = function makeUnit(document: Document, lines: SourceLine[]): IngestionUnit {
   const first = lines.at(0);
   const last = lines.at(-1);
-  if (!first || !last) throw new Error('Cannot create an empty ingestion unit');
+  if (first === undefined || last === undefined) {
+    throw new Error('Cannot create an empty ingestion unit');
+  }
   const text = lines.map((line) => line.text).join('');
   return {
-    id: `${document.path}:${first.number}-${last.number}`,
     document: document.id,
     hash: digest(text),
-    lineStart: first.number,
+    id: `${document.path}:${first.number}-${last.number}`,
     lineEnd: last.number,
+    lineStart: first.number,
     text,
   };
-}
+};
 
-export function ingestionUnits(documents: Document[]): {
+export const ingestionUnits = function ingestionUnits(documents: Document[]): {
   units: IngestionUnit[];
   warnings: Warning[];
 } {
   const units: IngestionUnit[] = [];
   const warnings: Warning[] = [];
-  for (const document of documents)
-    units.push(
-      ...packedBlocks(blocksFor(document, warnings)).map((lines) => makeUnit(document, lines)),
+  for (const document of documents) {
+    const documentUnits = packedBlocks(blocksFor(document, warnings)).map((lines) =>
+      makeUnit(document, lines)
     );
+    units.push(...documentUnits);
+  }
   return { units, warnings };
-}
+};