@nimiplatform/nimi-coding 0.3.0 → 0.4.0

package/cli/lib/authority/terms.mjs

@@ -0,0 +1,421 @@
+import { Buffer } from "node:buffer";
+import { lstat, readFile, realpath } from "node:fs/promises";
+import path from "node:path";
+
+import { compileAuthorityPath } from "./compile.mjs";
+import { compareText, makeDiagnostic, sortDiagnostics } from "./diagnostics.mjs";
+import { AuthorityInputError } from "./format.mjs";
+import { AuthorityReviewRefusal } from "./git-snapshot.mjs";
+import { resolveRepositoryCorpus } from "./repository-corpus.mjs";
+import { repositoryInventory } from "./repository-inventory.mjs";
+import { CODE_EXTENSIONS } from "./lexicon.mjs";
+import { classifyTrackedTextFiles, trackedTextInventory } from "./tracked-text.mjs";
+
+// Only used to keep "foo.ts" from being read as a dotted term; file selection
+// is decided by content, not by this list.
+const CODE_EXTENSION_SET = new Set(CODE_EXTENSIONS);
+const TERMS_FORMAT = "nimicoding.authority-terms/v1";
+const TERM_CLASS_ORDER = ["snake_case", "camelCase", "UPPER_SNAKE", "PascalCase", "dotted"];
+const TERM_EXPRESSIONS = [
+  ["snake_case", /[a-z][a-z0-9]*(?:_[a-z0-9]+)+/gu],
+  ["camelCase", /[a-z][a-z0-9]*(?:[A-Z][a-z0-9]*)+/gu],
+  ["UPPER_SNAKE", /[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+/gu],
+  ["PascalCase", /[A-Z][a-z0-9]+(?:[A-Z][a-z0-9]*)+/gu],
+  ["dotted", /[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)+/gu],
+];
+const AUTHORITY_FIELDS = ["title", "meaning", "statement", "condition", "failure"];
+const IDENTIFIER_CHARACTER = /[A-Za-z0-9_]/u;
+const PATH_TOKEN_CHARACTER = /[A-Za-z0-9_./-]/u;
+const UTF8 = new TextDecoder("utf-8", { fatal: true });
+
+export const AUTHORITY_TERM_GRAMMAR_HELP = [
+  "Authority terms use a closed lexical grammar over active title, meaning, statement, condition, and failure fields:",
+  "  snake_case: [a-z][a-z0-9]*(_[a-z0-9]+)+",
+  "  camelCase: [a-z][a-z0-9]*([A-Z][a-z0-9]*)+",
+  "  UPPER_SNAKE: [A-Z][A-Z0-9]*(_[A-Z0-9]+)+",
+  "  PascalCase: [A-Z][a-z0-9]+([A-Z][a-z0-9]*)+",
+  "  dotted: [A-Za-z0-9_-]+(\\.[A-Za-z0-9_-]+)+, excluding final segments that are code extensions.",
+  "Matches require ASCII identifier boundaries and are excluded from longer path-like tokens containing '/'.",
+].join("\n");
+
+function validBudget(value) {
+  return Number.isSafeInteger(value) && value > 0;
+}
+
+function baseDiagnostic(code, reason, repair, file = ".") {
+  return makeDiagnostic({
+    code,
+    file,
+    range: { start: { line: 1, column: 1 }, end: { line: 1, column: 1 } },
+    reason,
+    repair,
+  });
+}
+
+function budgetDiagnostic(kind, required, admitted) {
+  const option = kind === "active units" ? "max-units" : kind === "distinct terms" ? "max-terms" : "max-bytes";
+  return baseDiagnostic(
+    "AUTH_TERMS_BUDGET",
+    `complete authority term analysis requires ${required} ${kind} but ${option} is ${admitted}`,
+    "increase the explicit term-analysis budget; partial indexes and cross-references are forbidden",
+  );
+}
+
+function refused(diagnostics, { fileCount = 0 } = {}) {
+  return {
+    ok: false,
+    complete: false,
+    partial: false,
+    diagnostics: sortDiagnostics(diagnostics),
+    fileCount,
+    unitCount: 0,
+    termCount: 0,
+    scannedBytes: 0,
+    inputBytes: 0,
+    terms: null,
+  };
+}
+
+function inputDiagnostic(code, reason, file = ".") {
+  return baseDiagnostic(
+    code,
+    reason,
+    "provide one exact readable Git repository, admitted canonical corpus, and valid UTF-8 tracked code inputs; partial analysis is forbidden",
+    file,
+  );
+}
+
+function isWithin(root, candidate) {
+  const relative = path.relative(root, candidate);
+  return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative));
+}
+
+function portableRelative(repository, candidate) {
+  return path.relative(repository, candidate).split(path.sep).join(path.posix.sep);
+}
+
+async function resolveCorpus(repository, specPath) {
+  return resolveRepositoryCorpus(repository, specPath, "authority term");
+}
+
+function hasIdentifierBoundary(text, start, end) {
+  const preceding = start === 0 ? "" : text[start - 1];
+  const following = end === text.length ? "" : text[end];
+  return !IDENTIFIER_CHARACTER.test(preceding) && !IDENTIFIER_CHARACTER.test(following);
+}
+
+function insidePathToken(text, start, end) {
+  let tokenStart = start;
+  let tokenEnd = end;
+  while (tokenStart > 0 && PATH_TOKEN_CHARACTER.test(text[tokenStart - 1])) tokenStart -= 1;
+  while (tokenEnd < text.length && PATH_TOKEN_CHARACTER.test(text[tokenEnd])) tokenEnd += 1;
+  return text.slice(tokenStart, tokenEnd).includes("/");
+}
+
+function finalDottedSegment(term) {
+  return term.slice(term.lastIndexOf(".") + 1);
+}
+
+export function extractLexicalTerms(text) {
+  const terms = new Map();
+  for (const [termClass, expression] of TERM_EXPRESSIONS) {
+    for (const match of text.matchAll(expression)) {
+      const start = match.index;
+      const end = start + match[0].length;
+      if (!hasIdentifierBoundary(text, start, end) || insidePathToken(text, start, end)) continue;
+      if (termClass === "dotted" && CODE_EXTENSION_SET.has(finalDottedSegment(match[0]))) continue;
+      if (!terms.has(match[0])) terms.set(match[0], new Set());
+      terms.get(match[0]).add(termClass);
+    }
+  }
+  return [...terms.entries()]
+    .sort(([left], [right]) => compareText(left, right))
+    .map(([term, classes]) => ({ term, classes: TERM_CLASS_ORDER.filter((termClass) => classes.has(termClass)) }));
+}
+
+function unitFieldEntries(unit) {
+  const entries = [["title", unit.metadata.title]];
+  if (unit.kind === "definition") entries.push(["meaning", unit.semantic.meaning]);
+  else entries.push(
+    ["statement", unit.semantic.statement],
+    ["condition", unit.semantic.condition],
+    ["failure", unit.semantic.failure],
+  );
+  return entries;
+}
+
+function sourceContainer(compiled, unitIndex) {
+  const mapped = compiled.sourceMap.fields[`/units/${unitIndex}/metadata/title`];
+  return path.basename(mapped.file);
+}
+
+function extractCorpusTerms(compiled) {
+  const active = [];
+  const records = new Map();
+  let scannedBytes = 0;
+  compiled.ir.units.forEach((unit, unitIndex) => {
+    if (unit.lifecycle !== "active") return;
+    const container = sourceContainer(compiled, unitIndex);
+    const fields = new Map();
+    active.push({ unit, unitIndex });
+    for (const [field, text] of unitFieldEntries(unit)) {
+      scannedBytes += Buffer.byteLength(text, "utf8");
+      for (const extracted of extractLexicalTerms(text)) {
+        if (!fields.has(extracted.term)) fields.set(extracted.term, { classes: new Set(), fields: new Set() });
+        const occurrence = fields.get(extracted.term);
+        extracted.classes.forEach((termClass) => occurrence.classes.add(termClass));
+        occurrence.fields.add(field);
+      }
+    }
+    for (const [term, occurrence] of fields) {
+      if (!records.has(term)) records.set(term, { classes: new Set(), units: new Map(), containers: new Set() });
+      const record = records.get(term);
+      occurrence.classes.forEach((termClass) => record.classes.add(termClass));
+      record.units.set(unit.id, { fields: occurrence.fields, container });
+      record.containers.add(container);
+    }
+  });
+  return { active, records, scannedBytes };
+}
+
+function classCounts(records) {
+  const counts = Object.fromEntries(TERM_CLASS_ORDER.map((termClass) => [termClass, 0]));
+  for (const record of records.values()) {
+    for (const termClass of record.classes) counts[termClass] += 1;
+  }
+  return counts;
+}
+
+function indexEntries(records) {
+  return [...records.entries()].sort(([left], [right]) => compareText(left, right)).map(([term, record]) => {
+    const unitIds = [...record.units.keys()].sort(compareText);
+    return {
+      term,
+      classes: TERM_CLASS_ORDER.filter((termClass) => record.classes.has(termClass)),
+      unitCount: unitIds.length,
+      containers: [...record.containers].sort(compareText),
+      unitIds: unitIds.slice(0, 20),
+      ...(unitIds.length > 20 ? { truncatedUnitIds: true } : {}),
+    };
+  });
+}
+
+function exactDetail(term, termClasses, records, code, codeReferences, codeInventory) {
+  const record = records.get(term);
+  const units = record === undefined ? [] : [...record.units.entries()]
+    .sort(([left], [right]) => compareText(left, right))
+    .map(([unitId, occurrence]) => ({ unitId, fields: [...occurrence.fields].sort(compareText) }));
+  return {
+    format: TERMS_FORMAT,
+    complete: true,
+    mode: "detail",
+    term,
+    classes: termClasses,
+    unitCount: units.length,
+    units,
+    code,
+    ...(code ? { codeReferences, codeInventory } : {}),
+  };
+}
+
+function atOrBelow(file, root) {
+  return file === root || file.startsWith(`${root}/`);
+}
+
+function corpusFile(file, corpus) {
+  if (atOrBelow(file, ".nimi/spec")) return true;
+  return corpus.directory ? atOrBelow(file, corpus.relative) : file === corpus.relative;
+}
+
+async function codeFileBytes(repository, files) {
+  let total = 0n;
+  for (const file of files) {
+    let metadata;
+    try {
+      metadata = await lstat(path.join(repository, ...file.split("/")), { bigint: true });
+    } catch (error) {
+      throw new Error(`${file} is not readable: ${error.message}`);
+    }
+    if (metadata.isSymbolicLink() || !metadata.isFile()) throw new Error(`${file} must be one regular non-symlink file`);
+    total += metadata.size;
+  }
+  return total;
+}
+
+function exactLineReferences(file, text, term) {
+  const references = [];
+  let offset = 0;
+  let line = 1;
+  let lineCursor = 0;
+  let lastLine = 0;
+  while (offset <= text.length - term.length) {
+    const index = text.indexOf(term, offset);
+    if (index < 0) break;
+    while (lineCursor < index) {
+      if (text[lineCursor] === "\n") line += 1;
+      lineCursor += 1;
+    }
+    if (hasIdentifierBoundary(text, index, index + term.length) && line !== lastLine) {
+      references.push({ file, line });
+      lastLine = line;
+    }
+    offset = index + Math.max(term.length, 1);
+  }
+  return references;
+}
+
+async function scanCode(repository, files, term, authorityInputBytes, authorityScannedBytes, maxBytes) {
+  let statedBytes;
+  try {
+    statedBytes = await codeFileBytes(repository, files);
+  } catch (error) {
+    return { input: inputDiagnostic("AUTH_TERMS_CODE_INPUT_INVALID", error.message), budget: null };
+  }
+  const statedTotal = BigInt(authorityInputBytes) + statedBytes;
+  if (statedTotal > BigInt(maxBytes)) {
+    return { input: null, budget: budgetDiagnostic("UTF-8 input bytes", statedTotal.toString(), maxBytes) };
+  }
+
+  const texts = [];
+  let inputBytes = authorityInputBytes;
+  let scannedBytes = authorityScannedBytes;
+  for (const file of files) {
+    let bytes;
+    try {
+      bytes = await readFile(path.join(repository, ...file.split("/")));
+    } catch (error) {
+      return { input: inputDiagnostic("AUTH_TERMS_CODE_INPUT_INVALID", `${file} is not readable: ${error.message}`, file), budget: null };
+    }
+    inputBytes += bytes.length;
+    scannedBytes += bytes.length;
+    if (inputBytes > maxBytes) return { input: null, budget: budgetDiagnostic("UTF-8 input bytes", inputBytes, maxBytes) };
+    try {
+      texts.push({ file, text: UTF8.decode(bytes) });
+    } catch {
+      return { input: inputDiagnostic("AUTH_TERMS_CODE_INPUT_INVALID", `${file} is not valid UTF-8`, file), budget: null };
+    }
+  }
+  return {
+    input: null,
+    budget: null,
+    inputBytes,
+    scannedBytes,
+    references: texts.flatMap(({ file, text }) => exactLineReferences(file, text, term)),
+  };
+}
+
+export async function termsAuthorityRepository(repositoryPath, specPath, { term = null, code = false, maxUnits, maxTerms, maxBytes }) {
+  if (![maxUnits, maxTerms, maxBytes].every(validBudget)) {
+    return refused([baseDiagnostic(
+      "AUTH_TERMS_BUDGET",
+      "authority term-analysis budgets must be positive safe integers",
+      "provide explicit positive safe max-units, max-terms, and max-bytes budgets",
+    )]);
+  }
+  if (code && term === null) {
+    return refused([inputDiagnostic("AUTH_TERMS_CODE_TERM_REQUIRED", "authority code cross-reference requires one exact term")]);
+  }
+  let termClasses = null;
+  if (term !== null) {
+    if (typeof term !== "string" || term.length === 0 || /[\0\r\n]/u.test(term)) {
+      return refused([inputDiagnostic("AUTH_TERMS_TERM_INVALID", "authority term detail requires one non-empty single-line term")]);
+    }
+    const exact = extractLexicalTerms(term).find((entry) => entry.term === term);
+    if (exact === undefined) {
+      return refused([inputDiagnostic("AUTH_TERMS_TERM_INVALID", "authority term detail requires one exact term admitted by the closed lexical grammar")]);
+    }
+    termClasses = exact.classes;
+  }
+
+  let inventory;
+  try {
+    inventory = await repositoryInventory(repositoryPath);
+  } catch (error) {
+    if (error instanceof AuthorityReviewRefusal) {
+      return refused([inputDiagnostic("AUTH_TERMS_REPOSITORY_INVALID", error.message, error.file ?? ".")]);
+    }
+    throw error;
+  }
+
+  let corpus;
+  try {
+    corpus = await resolveCorpus(inventory.repository, specPath);
+  } catch (error) {
+    return refused([inputDiagnostic("AUTH_TERMS_CORPUS_INVALID", error.message)]);
+  }
+
+  let compiled;
+  try {
+    compiled = await compileAuthorityPath(corpus.path);
+  } catch (error) {
+    if (error instanceof AuthorityInputError) {
+      return refused([inputDiagnostic("AUTH_TERMS_CORPUS_INVALID", error.message)]);
+    }
+    throw error;
+  }
+  if (!compiled.ok) return refused(compiled.diagnostics, { fileCount: compiled.fileCount });
+
+  const extracted = extractCorpusTerms(compiled);
+  if (extracted.active.length > maxUnits) {
+    return refused([budgetDiagnostic("active units", extracted.active.length, maxUnits)], { fileCount: compiled.fileCount });
+  }
+  if (extracted.records.size > maxTerms) {
+    return refused([budgetDiagnostic("distinct terms", extracted.records.size, maxTerms)], { fileCount: compiled.fileCount });
+  }
+  if (!code && compiled.inputBytes > maxBytes) {
+    return refused([budgetDiagnostic("UTF-8 input bytes", compiled.inputBytes, maxBytes)], { fileCount: compiled.fileCount });
+  }
+
+  let scannedBytes = extracted.scannedBytes;
+  let inputBytes = compiled.inputBytes;
+  let codeReferences = [];
+  let codeInventory = null;
+  if (code) {
+    const classified = await classifyTrackedTextFiles(
+      inventory.repository,
+      inventory.files,
+      (file) => corpusFile(file, corpus) ? "authority-corpus" : null,
+    );
+    if (classified.error !== undefined) {
+      return refused([inputDiagnostic("AUTH_TERMS_CODE_INPUT_INVALID", classified.error)], { fileCount: compiled.fileCount });
+    }
+    const scanned = await scanCode(
+      inventory.repository,
+      classified.scanned,
+      term,
+      compiled.inputBytes,
+      extracted.scannedBytes,
+      maxBytes,
+    );
+    if (scanned.input !== null) return refused([scanned.input], { fileCount: compiled.fileCount });
+    if (scanned.budget !== null) return refused([scanned.budget], { fileCount: compiled.fileCount });
+    scannedBytes = scanned.scannedBytes;
+    inputBytes = scanned.inputBytes;
+    codeReferences = scanned.references;
+    codeInventory = trackedTextInventory(inventory.files.length, classified);
+  }
+
+  const counts = {
+    units: extracted.active.length,
+    distinctTerms: extracted.records.size,
+    classes: classCounts(extracted.records),
+  };
+  const terms = term === null
+    ? { format: TERMS_FORMAT, complete: true, mode: "index", counts, index: indexEntries(extracted.records) }
+    : exactDetail(term, termClasses, extracted.records, code, codeReferences, codeInventory);
+  return {
+    ok: true,
+    complete: true,
+    partial: false,
+    diagnostics: [],
+    fileCount: compiled.fileCount,
+    unitCount: extracted.active.length,
+    termCount: extracted.records.size,
+    scannedBytes,
+    inputBytes,
+    trackedPathSetDigest: inventory.trackedPathSetDigest,
+    terms,
+  };
+}
+
+export { AUTHORITY_FIELDS, TERMS_FORMAT, TERM_CLASS_ORDER };

package/cli/lib/authority/tracked-text.mjs

@@ -0,0 +1,70 @@
+import { Buffer } from "node:buffer";
+import { lstat, open } from "node:fs/promises";
+import path from "node:path";
+
+import { compareText } from "./diagnostics.mjs";
+
+const TEXT_PROBE_BYTES = 8192;
+
+export async function classifyTrackedTextFiles(repository, files, excludedReason = () => null) {
+  const scanned = [];
+  const excluded = [];
+  for (const file of files) {
+    const reason = excludedReason(file);
+    if (reason !== null) {
+      excluded.push({ file, reason });
+      continue;
+    }
+    const absolute = path.join(repository, ...file.split("/"));
+    let metadata;
+    try {
+      metadata = await lstat(absolute);
+    } catch (error) {
+      return { error: `${file} is not readable: ${error.message}` };
+    }
+    if (metadata.isSymbolicLink() || !metadata.isFile()) {
+      return { error: `${file} must be one regular non-symlink file` };
+    }
+    let probe;
+    try {
+      const handle = await open(absolute, "r");
+      try {
+        const buffer = Buffer.alloc(Math.min(TEXT_PROBE_BYTES, metadata.size));
+        const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
+        probe = buffer.subarray(0, bytesRead);
+      } finally {
+        await handle.close();
+      }
+    } catch (error) {
+      return { error: `${file} is not readable: ${error.message}` };
+    }
+    if (probe.includes(0)) {
+      excluded.push({ file, reason: "binary-content" });
+      continue;
+    }
+    try {
+      // Probe boundaries can split a multi-byte character. Streaming accepts
+      // that pending suffix without carrying decoder state into another file.
+      const truncated = probe.length === TEXT_PROBE_BYTES;
+      new TextDecoder("utf-8", { fatal: true }).decode(probe, { stream: truncated });
+    } catch {
+      excluded.push({ file, reason: "not-utf8" });
+      continue;
+    }
+    scanned.push(file);
+  }
+  return { scanned, excluded };
+}
+
+export function trackedTextInventory(fileCount, classified) {
+  const excludedByReason = {};
+  for (const entry of classified.excluded) {
+    excludedByReason[entry.reason] = (excludedByReason[entry.reason] ?? 0) + 1;
+  }
+  return {
+    trackedFiles: fileCount,
+    scannedFiles: classified.scanned.length,
+    excludedByReason,
+    excluded: [...classified.excluded].sort((left, right) => compareText(left.file, right.file)),
+  };
+}