@h1v35/hivex 0.2.0 → 0.2.2

package/src/knowledge-serialization.ts

@@ -0,0 +1,239 @@
+// Field order is part of the persisted v1 cache format. Keep model inputs stable
+// when source formatting changes; unknown future fields remain part of the input.
+const fieldOrder = {
+  citation: ['document', 'lineStart', 'lineEnd', 'version', 'historical', 'text'],
+  code: ['path', 'side', 'lineStart', 'lineEnd', 'version', 'text'],
+  context: [
+    'command',
+    'snapshot',
+    'documents',
+    'unavailableDocuments',
+    'unexpandedDecisions',
+    'decisions',
+    'relationships',
+    'pendingDocuments',
+    'warnings',
+  ],
+  decision: [
+    'id',
+    'document',
+    'text',
+    'kind',
+    'status',
+    'conditions',
+    'exceptions',
+    'reason',
+    'lineStart',
+    'lineEnd',
+    'version',
+    'batch',
+    'localId',
+    'quality',
+  ],
+  document: ['id', 'title', 'status', 'historical', 'version', 'lineCount', 'lines'],
+  extraction: [
+    'operation',
+    'targets',
+    'repairReason',
+    'units',
+    'documents',
+    'existing',
+    'previousRelationships',
+    'scope',
+    'extraction',
+  ],
+  file: ['path', 'before', 'after'],
+  finding: ['target', 'reason'],
+  implementation: ['baseCommit', 'diff', 'files', 'warnings', 'fingerprint'],
+  packet: [
+    'operation',
+    'implementation',
+    'task',
+    'context',
+    'documents',
+    'omittedUnits',
+    'warnings',
+  ],
+  queriedDecision: [
+    'historical',
+    'id',
+    'document',
+    'version',
+    'text',
+    'kind',
+    'status',
+    'quality',
+    'conditions',
+    'exceptions',
+    'reason',
+    'evidence',
+  ],
+  relationship: ['id', 'from', 'to', 'type', 'reason', 'evidence', 'batch', 'localId', 'quality'],
+  reviewFinding: ['assessment', 'explanation', 'documents', 'code'],
+  unit: ['id', 'document', 'hash', 'lineStart', 'lineEnd', 'text'],
+  version: ['version', 'lines'],
+  warning: ['path', 'message'],
+};
+
+const isRecord = function isRecord(value: unknown): value is Record<string, unknown> {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+};
+const hasFields = function hasFields(value: Record<string, unknown>, fields: string[]) {
+  return fields.every((field) => Object.hasOwn(value, field));
+};
+const orderRecord = function orderRecord(value: Record<string, unknown>, order: string[]) {
+  const existing = Object.keys(value);
+  const names = new Set(Iterator.concat(order, existing));
+  return Object.fromEntries(
+    names
+      .values()
+      .filter((name) => existing.includes(name))
+      .map<[string, unknown]>((name) => [name, value[name]])
+  );
+};
+
+const hasOrderPrefix = function hasOrderPrefix(value: Record<string, unknown>, fields: string[]) {
+  const keys = Object.keys(value);
+  return fields.every((field, index) => keys[index] === field);
+};
+
+// New graph records must retain the original live provenance order between
+// rounds. Decoding a stored graph has its own schema order.
+export const withLiveProvenance = function withLiveProvenance<T extends object>(
+  value: T,
+  kind: 'decision' | 'relationship'
+): T {
+  const record = { ...value };
+  const fields =
+    kind === 'decision'
+      ? [...fieldOrder.decision.slice(0, 10), 'localId', 'version', 'batch', 'quality']
+      : [...fieldOrder.relationship.slice(0, 6), 'localId', 'batch', 'quality'];
+  for (const field of fields) {
+    const descriptor = Object.getOwnPropertyDescriptor(record, field);
+    if (descriptor !== undefined) {
+      Reflect.deleteProperty(record, field);
+      Object.defineProperty(record, field, descriptor);
+    }
+  }
+  return record;
+};
+
+const sourceOrder = function sourceOrder(value: Record<string, unknown>) {
+  if (hasFields(value, ['kind', 'id'])) {
+    if (Object.hasOwn(value, 'historical')) {
+      return fieldOrder.queriedDecision;
+    }
+    // Retained packets may contain live graph records, whose provenance fields
+    // were appended in a different order from records decoded by the store.
+    return hasOrderPrefix(value, fieldOrder.decision.slice(0, 10)) ? null : fieldOrder.decision;
+  }
+  if (hasFields(value, ['from', 'to', 'type'])) {
+    return hasOrderPrefix(value, fieldOrder.relationship.slice(0, 6))
+      ? null
+      : fieldOrder.relationship;
+  }
+  if (hasFields(value, ['id', 'title'])) {
+    return fieldOrder.document;
+  }
+  if (hasFields(value, ['document', 'lineStart', 'lineEnd'])) {
+    return Object.hasOwn(value, 'id') ? fieldOrder.unit : fieldOrder.citation;
+  }
+  if (hasFields(value, ['path', 'side'])) {
+    return fieldOrder.code;
+  }
+  if (hasFields(value, ['version', 'lines'])) {
+    return fieldOrder.version;
+  }
+  if (hasFields(value, ['path', 'message'])) {
+    return fieldOrder.warning;
+  }
+  // No source shape matched. Null above preserves a recognized record's live order.
+  return [];
+};
+
+const packetOrder = function packetOrder(key: string, value: Record<string, unknown>) {
+  if (key === '' && typeof value.operation === 'string') {
+    const isExtraction = value.operation === 'extract' || value.operation === 'check';
+    return isExtraction ? fieldOrder.extraction : fieldOrder.packet;
+  }
+  if (key === 'context' && hasFields(value, ['command', 'snapshot'])) {
+    return fieldOrder.context;
+  }
+  if (hasFields(value, ['baseCommit', 'files'])) {
+    return fieldOrder.implementation;
+  }
+  if (hasFields(value, ['path', 'before', 'after'])) {
+    return fieldOrder.file;
+  }
+  const order = sourceOrder(value);
+  if (order === null || order.length > 0) {
+    return order;
+  }
+  return hasFields(value, ['target', 'reason']) ? fieldOrder.finding : null;
+};
+
+export const stringifyKnowledge = function stringifyKnowledge(packet: unknown) {
+  return JSON.stringify(packet, (key: string, value: unknown) => {
+    if (!isRecord(value)) {
+      return value;
+    }
+    const order = packetOrder(key, value);
+    return order === null ? value : orderRecord(value, order);
+  });
+};
+
+const schemaOrder = function schemaOrder(properties: Record<string, unknown>) {
+  if (hasFields(properties, ['kind', 'id'])) {
+    return fieldOrder.decision;
+  }
+  if (hasFields(properties, ['from', 'to', 'type'])) {
+    return fieldOrder.relationship;
+  }
+  if (hasFields(properties, ['document', 'lineStart', 'lineEnd'])) {
+    return fieldOrder.citation;
+  }
+  if (hasFields(properties, ['path', 'side'])) {
+    return fieldOrder.code;
+  }
+  if (hasFields(properties, ['assessment', 'explanation'])) {
+    return fieldOrder.reviewFinding;
+  }
+  if (hasFields(properties, ['target', 'reason'])) {
+    return fieldOrder.finding;
+  }
+  return null;
+};
+
+const schemaNode = function schemaNode(value: unknown): unknown {
+  if (Array.isArray(value)) {
+    return value.map(schemaNode);
+  }
+  if (!isRecord(value)) {
+    return value;
+  }
+  const result = Object.fromEntries(
+    Object.entries(value).map(([key, node]) => [key, schemaNode(node)])
+  );
+  if (!isRecord(result.properties)) {
+    return result;
+  }
+  const order = schemaOrder(result.properties);
+  if (order === null) {
+    return result;
+  }
+  result.properties = orderRecord(result.properties, order);
+  if (Array.isArray(result.required)) {
+    const { required } = result;
+    const names = new Set(Iterator.concat(order, required));
+    result.required = [...names].filter((name) => required.includes(name));
+  }
+  return result;
+};
+
+export const schemaForKnowledge = function schemaForKnowledge(schema: Record<string, unknown>) {
+  const result = schemaNode(schema);
+  if (!isRecord(result)) {
+    throw new Error('Expected a JSON schema object');
+  }
+  return result;
+};

package/src/knowledge-snapshot.ts

@@ -1,140 +1,163 @@
 import { lstatSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
-import { join } from 'node:path';
 import { randomUUID } from 'node:crypto';
+import path from 'node:path';
+import { compareSerializedStrings } from './ordering.ts';
 import { HivexError } from './errors.ts';
 import { isMarkdownPath } from './markdown.ts';
-import { emptyGraph, graphSchema, type Graph } from './knowledge-model.ts';
+import { emptyGraph, graphSchema } from './knowledge-model.ts';
+import type { Graph } from './knowledge-model.ts';
 
-const MAX_BYTES = 64 * 1024 * 1024;
+const maxBytes = 64 * 1024 * 1024;
 
-function portableDocument(id: string) {
-  return (
-    isMarkdownPath(id) &&
-    !id.startsWith('/') &&
-    !id.includes('\\') &&
-    !id.includes('\0') &&
-    !id.split('/').some((part) => ['..', '.', '.git', '.hivex', 'node_modules', ''].includes(part))
-  );
-}
+const isPortableDocument = function isPortableDocument(id: string) {
+  const isProjectRelative = !id.startsWith('/') && !id.includes('\\') && !id.includes('\0');
+  const hasSafeParts = id
+    .split('/')
+    .every((part) => !['..', '.', '.git', '.hivex', 'node_modules', ''].includes(part));
+  return isMarkdownPath(id) && isProjectRelative && hasSafeParts;
+};
 
-function validateGraph(graph: Graph) {
+const validateGraph = function validateGraph(graph: Graph) {
   const decisions = new Set(graph.decisions.map((entry) => entry.id));
   const relationships = new Set(graph.relationships.map((entry) => entry.id));
-  const invalid =
+  const hasInvalidDecision = graph.decisions.some((entry) => {
+    if (entry.quality !== 'checked') {
+      return false;
+    }
+    return entry.lineStart > entry.lineEnd;
+  });
+  const hasInvalidRelationship = graph.relationships.some((edge) => {
+    if (!decisions.has(edge.from) || !decisions.has(edge.to)) {
+      return true;
+    }
+    if (edge.evidence.length === 0) {
+      return true;
+    }
+    return edge.evidence.some((entry) => entry.lineStart > entry.lineEnd);
+  });
+  const isInvalid =
     decisions.size !== graph.decisions.length ||
     relationships.size !== graph.relationships.length ||
-    graph.decisions.some(
-      (entry) => entry.quality === 'checked' && entry.lineStart > entry.lineEnd,
-    ) ||
-    graph.relationships.some(
-      (edge) =>
-        !decisions.has(edge.from) ||
-        !decisions.has(edge.to) ||
-        !edge.evidence.length ||
-        edge.evidence.some((entry) => entry.lineStart > entry.lineEnd),
-    );
+    hasInvalidDecision ||
+    hasInvalidRelationship;
   const references = [
     ...Object.keys(graph.documents),
     ...Object.values(graph.units).map((unit) => unit.document),
     ...graph.decisions.map((entry) => entry.document),
     ...graph.relationships.flatMap((edge) => edge.evidence.map((entry) => entry.document)),
-    ...graph.warnings.flatMap((warning) =>
-      typeof warning === 'string' ? [] : warning.scope.map((entry) => entry.document),
-    ),
+    ...graph.warnings.flatMap((warning) => {
+      if (typeof warning === 'string') {
+        return [];
+      }
+      return warning.scope.map((entry) => entry.document);
+    }),
   ];
-  if (invalid || references.some((id) => !portableDocument(id)))
+  if (isInvalid || references.some((id) => !isPortableDocument(id))) {
     throw new HivexError({
       code: 'INVALID_SNAPSHOT',
       message:
         'Knowledge snapshot has invalid identities, relationships or project-relative sources.',
     });
+  }
   return graph;
-}
+};
 
-function snapshotPath(root: string) {
-  const directory = join(root, '.hivex');
-  const path = join(directory, 'graph.json');
-  for (const candidate of [directory, path]) {
+const snapshotPath = function snapshotPath(root: string) {
+  const directory = path.join(root, '.hivex');
+  const snapshotFile = path.join(directory, 'graph.json');
+  for (const candidate of [directory, snapshotFile]) {
     const stat = lstatSync(candidate, { throwIfNoEntry: false });
     if (
-      stat?.isSymbolicLink() ||
+      stat?.isSymbolicLink() === true ||
       (stat && !(candidate === directory ? stat.isDirectory() : stat.isFile()))
-    )
+    ) {
       throw new HivexError({
         code: 'INVALID_SNAPSHOT',
         message: 'Knowledge snapshot must use regular project-local files.',
       });
+    }
   }
-  return path;
-}
+  return snapshotFile;
+};
 
-export function readKnowledgeSnapshot(root: string): Graph | null {
-  const path = snapshotPath(root);
-  const stat = lstatSync(path, { throwIfNoEntry: false });
-  if (!stat) return null;
-  if (stat.size > MAX_BYTES)
+const parseSnapshot = function parseSnapshot(bytes: Uint8Array) {
+  if (bytes.length > maxBytes) {
+    throw new Error('Snapshot exceeds size limit');
+  }
+  const decoder = new TextDecoder('utf-8', { fatal: true });
+  const data: unknown = JSON.parse(decoder.decode(bytes));
+  return validateGraph(graphSchema.strict().parse(data));
+};
+
+export const readKnowledgeSnapshot = function readKnowledgeSnapshot(root: string): Graph | null {
+  const snapshotFile = snapshotPath(root);
+  const stat = lstatSync(snapshotFile, { throwIfNoEntry: false });
+  if (stat === undefined) {
+    return null;
+  }
+  if (stat.size > maxBytes) {
     throw new HivexError({
       code: 'INVALID_SNAPSHOT',
       message: 'Knowledge snapshot exceeds 64 MiB.',
     });
+  }
   try {
-    const bytes = readFileSync(path);
-    if (bytes.length > MAX_BYTES) throw new Error('Snapshot exceeds size limit');
-    const data: unknown = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes));
-    return validateGraph(graphSchema.strict().parse(data));
+    return parseSnapshot(readFileSync(snapshotFile));
   } catch {
     throw new HivexError({
       code: 'INVALID_SNAPSHOT',
       message: 'Knowledge snapshot is not a supported graph JSON document.',
     });
   }
-}
+};
 
-function ordered(value: unknown): unknown {
-  if (Array.isArray(value)) return value.map(ordered);
-  if (value && typeof value === 'object')
+const ordered = function ordered(value: unknown): unknown {
+  if (Array.isArray(value)) {
+    return value.map(ordered);
+  }
+  if (value !== null && typeof value === 'object') {
     return Object.fromEntries(
       Object.entries(value)
-        .sort(([a], [b]) => (a < b ? -1 : Number(a > b)))
-        .map(([key, entry]) => [key, ordered(entry)]),
+        .toSorted(([a], [b]) => compareSerializedStrings(a, b))
+        .map(([key, entry]) => [key, ordered(entry)])
     );
+  }
   return value;
-}
+};
 
-function byId(a: { id: string }, b: { id: string }) {
-  if (a.id < b.id) return -1;
-  return Number(a.id > b.id);
-}
+const byId = function byId(a: { id: string }, b: { id: string }) {
+  return compareSerializedStrings(a.id, b.id);
+};
 
-export function writeKnowledgeSnapshot(root: string, value: Graph) {
-  const path = snapshotPath(root);
+export const writeKnowledgeSnapshot = function writeKnowledgeSnapshot(root: string, value: Graph) {
+  const snapshotFile = snapshotPath(root);
   const graph = validateGraph(graphSchema.parse(value));
-  const text =
-    JSON.stringify(
-      ordered({
-        ...graph,
-        decisions: graph.decisions.toSorted(byId),
-        relationships: graph.relationships.toSorted(byId),
-      }),
-      null,
-      2,
-    ) + '\n';
-  if (Buffer.byteLength(text) > MAX_BYTES)
+  const text = `${JSON.stringify(
+    ordered({
+      ...graph,
+      decisions: graph.decisions.toSorted(byId),
+      relationships: graph.relationships.toSorted(byId),
+    }),
+    null,
+    2
+  )}\n`;
+  if (Buffer.byteLength(text) > maxBytes) {
     throw new HivexError({
       code: 'INVALID_SNAPSHOT',
       message: 'Knowledge snapshot exceeds 64 MiB.',
     });
-  mkdirSync(join(root, '.hivex'), { recursive: true, mode: 0o700 });
-  const temporary = join(root, '.hivex', `graph-${randomUUID()}.tmp`);
+  }
+  mkdirSync(path.join(root, '.hivex'), { mode: 0o700, recursive: true });
+  const temporary = path.join(root, '.hivex', `graph-${randomUUID()}.tmp`);
   try {
     writeFileSync(temporary, text, { flag: 'wx', mode: 0o600 });
-    renameSync(temporary, path);
+    renameSync(temporary, snapshotFile);
   } finally {
     rmSync(temporary, { force: true });
   }
-  return path;
-}
+  return snapshotFile;
+};
 
-export function sharedKnowledge(root: string) {
+export const sharedKnowledge = function sharedKnowledge(root: string) {
   return readKnowledgeSnapshot(root) ?? emptyGraph();
-}
+};