llmnav 0.5.1

package/src/index.js

@@ -0,0 +1,115 @@
+export { AGENT_PROTOCOL, installAgentInstructions } from "./agents.js";
+export {
+  AGENT_OPERATION_SCHEMA_VERSION,
+  AGENT_TOOL_SCHEMA_VERSION,
+  executeAgentOperation,
+  getAgentToolDefinitions,
+} from "./agent-protocol.js";
+export {
+  buildPromptPrefixBundle,
+  isCompatiblePromptPrefixBundle,
+  loadPromptPrefixBundle,
+  PROMPT_BUNDLE_SCHEMA_VERSION,
+  renderPromptPrefixBundle,
+} from "./prompt-bundle.js";
+export { compareCardIndexes, describeAffectedBoundaries, describeAffectedCatalogs } from "./changes.js";
+export { loadConfig, validateConfig } from "./config.js";
+export { BOUNDARY_KINDS, detectBoundaries } from "./boundaries.js";
+export {
+  diagnosticsToEditor,
+  EDITOR_DIAGNOSTIC_SCHEMA_VERSION,
+  EDITOR_INTEGRATION_SCHEMA_VERSION,
+  getEditorIntegration,
+  renderEditorDiagnostics,
+} from "./editor.js";
+export { diagnosticsToSarif, SARIF_SCHEMA, SARIF_VERSION } from "./sarif.js";
+export { GRAPH_INPUT_SCHEMA_VERSION, loadGraphInputs, normalizeGraphInput } from "./graph-input.js";
+export {
+  buildRepositoryGraph,
+  buildRepositoryGraphIncremental,
+  GRAPH_SCHEMA_VERSION,
+  GRAPH_STATE_SCHEMA_VERSION,
+  compatibleGraphState,
+  isCompatibleRepositoryGraph,
+  renderGraphState,
+  renderGraphNode,
+  renderRepositoryGraph,
+  resolveGraphNode,
+} from "./graph.js";
+export {
+  buildSearchShards,
+  SEARCH_SHARD_ENCODING,
+  SEARCH_SHARD_SCHEMA_VERSION,
+} from "./search-shards.js";
+export {
+  buildContractFingerprints,
+  compareContractFingerprints,
+  CONTRACT_FINGERPRINT_SCHEMA_VERSION,
+} from "./contracts.js";
+export { findAttachedDeclaration, extractImports } from "./declaration.js";
+export { doctorProject } from "./doctor.js";
+export { evaluateProject } from "./evaluation.js";
+export { collectSourceFiles, findProjectRoot } from "./files.js";
+export { formatProject } from "./formatter.js";
+export {
+  buildArtifacts,
+  buildArtifactSet,
+  generateProject,
+  renderCompactCard,
+  renderSemanticCard,
+} from "./generator.js";
+export {
+  buildFileStateFromProject,
+  FILE_STATE_SCHEMA_VERSION,
+  renderFileState,
+  scanProjectIncremental,
+  SOURCE_INDEXER_VERSION,
+  usableFileState,
+} from "./incremental.js";
+export { initializeProject } from "./initializer.js";
+export {
+  buildInvertedIndex,
+  buildSearchDocument,
+  isCompatibleSearchIndex,
+  renderSearchIndex,
+  SEARCH_FIELD_ORDER,
+  SEARCH_FIELD_WEIGHTS,
+  SEARCH_INDEX_ENCODING,
+  SEARCH_INDEX_SCHEMA_VERSION,
+  searchCardSetHash,
+  searchDocumentHash,
+  verifySearchIndex,
+} from "./inverted-index.js";
+export {
+  canonicalizeSource,
+  cardToCanonicalObject,
+  formatLlmnavBlock,
+  parseLlmnavBlocks,
+} from "./parser.js";
+export { scanProject } from "./project.js";
+export { ensureActiveIds, loadRegistry, mergeActiveIds, renderRegistryRecords, resolveRegistryId } from "./registry.js";
+export {
+  buildContext,
+  createProjectSession,
+  loadSearchData,
+  queryIndex,
+  queryIndexLegacy,
+  queryPreparedIndex,
+  queryProject,
+  showProjectCard,
+  tokenize,
+} from "./search.js";
+export { normalizeSearchText, TOKENIZER_VERSION } from "./tokenizer.js";
+export {
+  acquireGenerationLock,
+  commitGeneratedCache,
+  recoverGenerationTransaction,
+  releaseGenerationLock,
+  removeWithRetry,
+  renameWithRetry,
+  TRANSACTION_ABORT_EXIT_CODE,
+  TRANSACTION_SCHEMA_VERSION,
+  withGenerationLock,
+} from "./transaction.js";
+export * from "./spec.js";
+export { countDiagnostics, diagnostic, validateProject } from "./validator.js";

package/src/initializer.js

@@ -0,0 +1,137 @@
+/* llmnav/1 module
+id=llmnav.project.initialize
+role=Create a repository's LLMNav configuration, registry, schemas, agent instructions, and initial index.
+owns=project bootstrap|configuration templates|initial generation
+excludes=package installation|automatic annotation
+search=llmnav init|repository bootstrap|agent setup
+rel=workflow>llmnav.agent.install
+rel=workflow>llmnav.index.generate
+stability=architecture
+*/
+
+import { copyFile, mkdir } from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { installAgentInstructions } from "./agents.js";
+import { generateProject } from "./generator.js";
+import { DEFAULT_CONFIG } from "./spec.js";
+import { assertNoSymlinkTraversal, atomicWrite, readJson, readText, stableStringify } from "./util.js";
+
+const PACKAGE_ROOT = fileURLToPath(new URL("..", import.meta.url));
+
+export async function initializeProject(root, options = {}) {
+  await assertNoSymlinkTraversal(root, path.join(root, ".llmnav"), ".llmnav");
+  await mkdir(path.join(root, ".llmnav", "eval"), { recursive: true });
+  await mkdir(path.join(root, ".llmnav", "schema"), { recursive: true });
+  const repositoryId = await inferRepositoryId(root);
+  const changed = [];
+
+  const config = structuredClone(DEFAULT_CONFIG);
+  config.repositoryId = repositoryId;
+  config.$schema = "./schema/config.schema.json";
+  await writeIfMissingOrForced(
+    path.join(root, ".llmnav", "config.json"),
+    stableStringify(config),
+    options.force,
+    changed,
+    ".llmnav/config.json",
+  );
+
+  await writeIfMissingOrForced(
+    path.join(root, ".llmnav", "lexicon.json"),
+    stableStringify({ version: 1, aliases: {} }),
+    false,
+    changed,
+    ".llmnav/lexicon.json",
+  );
+  await writeIfMissingOrForced(
+    path.join(root, ".llmnav", "ids.jsonl"),
+    "",
+    false,
+    changed,
+    ".llmnav/ids.jsonl",
+  );
+  await writeIfMissingOrForced(
+    path.join(root, ".llmnav", "order.lock"),
+    "",
+    false,
+    changed,
+    ".llmnav/order.lock",
+  );
+  await writeIfMissingOrForced(
+    path.join(root, ".llmnav", "eval", "queries.jsonl"),
+    '# One JSON object per line: {"query":"...","expected":["domain.feature.action"]}\n',
+    false,
+    changed,
+    ".llmnav/eval/queries.jsonl",
+  );
+  await writeIfMissingOrForced(
+    path.join(root, ".llmnav", ".gitignore"),
+    "tmp/\nstate/\n.transactions/\ngeneration-transaction.json\ngeneration.lock\ngeneration.lock.release-*\n*.tmp-*\n",
+    options.force,
+    changed,
+    ".llmnav/.gitignore",
+  );
+
+  const schemaSource = path.join(PACKAGE_ROOT, "schema", "config.schema.json");
+  const schemaTarget = path.join(root, ".llmnav", "schema", "config.schema.json");
+  if (options.force || (await readText(schemaTarget, null)) === null) {
+    await copyFile(schemaSource, schemaTarget);
+    changed.push(".llmnav/schema/config.schema.json");
+  }
+
+  changed.push(...(await installAgentInstructions(root, options.agents ?? ["agents"])));
+
+  if (options.packageScripts) {
+    if (await addPackageScripts(root)) changed.push("package.json");
+  }
+
+  const generated = await generateProject(root, { check: false });
+  if (!generated.ok) {
+    return { ok: false, changed: [...new Set(changed)].sort(), generated };
+  }
+  changed.push(...generated.changedFiles);
+  return { ok: true, changed: [...new Set(changed)].sort(), generated };
+}
+
+async function inferRepositoryId(root) {
+  const packageJson = await readJson(path.join(root, "package.json"), null);
+  const candidate = packageJson?.name ?? path.basename(root);
+  const normalized = String(candidate)
+    .replace(/^@[^/]+\//u, "")
+    .normalize("NFKC")
+    .toLowerCase()
+    .replace(/[^a-z0-9-]+/gu, "-")
+    .replace(/^-+|-+$/gu, "");
+  return normalized || "repository";
+}
+
+async function writeIfMissingOrForced(filePath, content, force, changed, displayPath) {
+  const existing = await readText(filePath, null);
+  if (existing !== null && !force) return;
+  if (existing === content) return;
+  await atomicWrite(filePath, content);
+  changed.push(displayPath);
+}
+
+async function addPackageScripts(root) {
+  const packagePath = path.join(root, "package.json");
+  const text = await readText(packagePath, null);
+  if (text === null) return false;
+  const parsed = JSON.parse(text);
+  parsed.scripts ??= {};
+  const desired = {
+    "llmnav:check": "llmnav check",
+    "llmnav:format": "llmnav format",
+    "llmnav:generate": "llmnav generate",
+    "llmnav:eval": "llmnav eval",
+  };
+  let changed = false;
+  for (const [name, command] of Object.entries(desired)) {
+    if (parsed.scripts[name] === command) continue;
+    parsed.scripts[name] = command;
+    changed = true;
+  }
+  if (changed) await atomicWrite(packagePath, `${JSON.stringify(parsed, null, 2)}\n`);
+  return changed;
+}

package/src/inverted-index.js

@@ -0,0 +1,350 @@
+/* llmnav/1 module
+id=llmnav.index.inverted
+role=Compile searchable card fields into a deterministic inverted index that can be updated one card at a time.
+owns=posting lists|card search documents|incremental token reuse
+excludes=semantic source syntax|query presentation
+search=inverted index|incremental search index|posting list
+rel=workflow>llmnav.search.tokenize
+rel=workflow>llmnav.index.generate
+stability=architecture
+*/
+
+import { compareText, sha256, stableJson, stableStringify } from "./util.js";
+import { normalizeSearchText, tokenize, TOKENIZER_VERSION } from "./tokenizer.js";
+
+export const SEARCH_INDEX_SCHEMA_VERSION = 2;
+export const SEARCH_INDEX_ENCODING = "compact-v1";
+
+export const SEARCH_FIELD_ORDER = Object.freeze([
+  "id",
+  "role",
+  "search",
+  "owns",
+  "excludes",
+  "invariant",
+  "effect",
+  "risk",
+  "symbol",
+  "path",
+  "signature",
+]);
+
+export const SEARCH_FIELD_WEIGHTS = Object.freeze({
+  id: 12,
+  role: 8,
+  search: 10,
+  owns: 7,
+  excludes: 2,
+  invariant: 6,
+  effect: 5,
+  risk: 4,
+  symbol: 5,
+  path: 3,
+  signature: 3,
+});
+
+export function buildInvertedIndex(index, previous = null) {
+  let previousUsable = isCompatibleSearchIndex(previous, index.repositoryId);
+  let postings = new Map();
+  let previousDocuments = new Map();
+  if (previousUsable) {
+    try {
+      ({ postings, documents: previousDocuments } = deserializeSearchIndex(previous));
+    } catch {
+      previousUsable = false;
+      postings = new Map();
+      previousDocuments = new Map();
+    }
+  }
+
+  const documents = new Map();
+  const currentById = new Map(index.cards.map((card) => [card.id, card]));
+  const changedIds = [];
+  const removedIds = [];
+  let reusedCards = 0;
+  let indexedCards = 0;
+
+  if (previousUsable) {
+    for (const id of [...previousDocuments.keys()].sort(compareText)) {
+      if (currentById.has(id)) continue;
+      removeDocumentFromPostings(postings, id, previousDocuments.get(id));
+      removedIds.push(id);
+    }
+  }
+
+  for (const card of [...index.cards].sort((left, right) => compareText(left.id, right.id))) {
+    const hash = searchDocumentHash(card);
+    const oldDocument = previousDocuments.get(card.id);
+    if (oldDocument?.hash === hash) {
+      documents.set(card.id, oldDocument);
+      reusedCards += 1;
+      continue;
+    }
+
+    if (oldDocument) removeDocumentFromPostings(postings, card.id, oldDocument);
+    const document = buildSearchDocument(card, hash);
+    addDocumentToPostings(postings, card.id, document);
+    documents.set(card.id, document);
+    changedIds.push(card.id);
+    indexedCards += 1;
+  }
+
+  const cardIds = [...documents.keys()].sort(compareText);
+  const tokens = [...postings.keys()].sort(compareText);
+  const cardIndex = new Map(cardIds.map((id, indexValue) => [id, indexValue]));
+  const serializedDocuments = cardIds.map((id) => {
+    const document = documents.get(id);
+    return [document.hash, document.phrases];
+  });
+  const serializedPostings = tokens.map((token) =>
+    [...(postings.get(token)?.entries() ?? [])]
+      .sort(([left], [right]) => compareText(left, right))
+      .map(([id, vector]) => [cardIndex.get(id), vector]),
+  );
+  const cardSetHash = searchCardSetHash(index.cards);
+  const searchIndex = {
+    schemaVersion: SEARCH_INDEX_SCHEMA_VERSION,
+    encoding: SEARCH_INDEX_ENCODING,
+    tokenizerVersion: TOKENIZER_VERSION,
+    repositoryId: index.repositoryId ?? "",
+    cardSetHash,
+    documentCount: index.cards.length,
+    fieldOrder: [...SEARCH_FIELD_ORDER],
+    cardIds,
+    tokens,
+    documents: serializedDocuments,
+    postings: serializedPostings,
+  };
+
+  return {
+    searchIndex,
+    stats: {
+      previousUsable,
+      totalCards: index.cards.length,
+      reusedCards,
+      indexedCards,
+      removedCards: removedIds.length,
+      changedIds: changedIds.sort(compareText),
+      removedIds: removedIds.sort(compareText),
+      tokenCount: tokens.length,
+    },
+  };
+}
+
+export function buildSearchDocument(card, hash = searchDocumentHash(card)) {
+  const fieldValues = {
+    id: card.id.replaceAll(".", " "),
+    role: card.role,
+    search: (card.search ?? []).join(" "),
+    owns: (card.owns ?? []).join(" "),
+    excludes: (card.excludes ?? []).join(" "),
+    invariant: (card.invariant ?? []).join(" "),
+    effect: (card.effect ?? []).join(" "),
+    risk: (card.risk ?? []).join(" "),
+    symbol: card.location?.symbol ?? "",
+    path: card.location?.path ?? "",
+    signature: card.location?.signature ?? "",
+  };
+
+  const termMap = new Map();
+  for (const [fieldIndex, field] of SEARCH_FIELD_ORDER.entries()) {
+    const counts = countTokens(tokenize(fieldValues[field]));
+    for (const [token, count] of counts) {
+      const sparse = termMap.get(token) ?? new Map();
+      sparse.set(fieldIndex, count);
+      termMap.set(token, sparse);
+    }
+  }
+
+  const terms = [...termMap.entries()]
+    .sort(([left], [right]) => compareText(left, right))
+    .map(([token, sparse]) => [token, serializeSparseVector(sparse)]);
+  const phrases = [card.role, ...(card.search ?? []), ...(card.invariant ?? []), ...(card.owns ?? [])]
+    .filter(Boolean)
+    .map(normalizeSearchText);
+
+  return {
+    hash,
+    phrases,
+    terms,
+    tokens: terms.map(([token]) => token),
+  };
+}
+
+export function searchDocumentHash(card) {
+  return sha256(
+    stableJson({
+      id: card.id,
+      role: card.role,
+      search: card.search ?? [],
+      owns: card.owns ?? [],
+      excludes: card.excludes ?? [],
+      invariant: card.invariant ?? [],
+      effect: card.effect ?? [],
+      risk: card.risk ?? [],
+      symbol: card.location?.symbol ?? null,
+      path: card.location?.path ?? null,
+      signature: card.location?.signature ?? null,
+    }),
+  );
+}
+
+export function searchCardSetHash(cards) {
+  return sha256(
+    [...cards]
+      .sort((left, right) => compareText(left.id, right.id))
+      .map((card) => `${card.id}:${searchDocumentHash(card)}`)
+      .join("\n"),
+  );
+}
+
+export function isCompatibleSearchIndex(searchIndex, repositoryId = undefined) {
+  return Boolean(
+    searchIndex &&
+      searchIndex.schemaVersion === SEARCH_INDEX_SCHEMA_VERSION &&
+      searchIndex.encoding === SEARCH_INDEX_ENCODING &&
+      searchIndex.tokenizerVersion === TOKENIZER_VERSION &&
+      Array.isArray(searchIndex.fieldOrder) &&
+      SEARCH_FIELD_ORDER.every((field, index) => searchIndex.fieldOrder[index] === field) &&
+      Array.isArray(searchIndex.cardIds) &&
+      isSortedUniqueStrings(searchIndex.cardIds) &&
+      Array.isArray(searchIndex.tokens) &&
+      isSortedUniqueStrings(searchIndex.tokens) &&
+      Array.isArray(searchIndex.documents) &&
+      searchIndex.documents.length === searchIndex.cardIds.length &&
+      Array.isArray(searchIndex.postings) &&
+      searchIndex.postings.length === searchIndex.tokens.length &&
+      (repositoryId === undefined || searchIndex.repositoryId === (repositoryId ?? "")),
+  );
+}
+
+export function verifySearchIndex(index, searchIndex) {
+  if (!isCompatibleSearchIndex(searchIndex, index.repositoryId)) return false;
+  if (searchIndex.documentCount !== index.cards.length) return false;
+  if (searchIndex.cardSetHash !== searchCardSetHash(index.cards)) return false;
+  const cards = new Map(index.cards.map((card) => [card.id, card]));
+  if (searchIndex.cardIds.length !== cards.size) return false;
+  for (const [cardIndex, id] of searchIndex.cardIds.entries()) {
+    const card = cards.get(id);
+    const document = searchIndex.documents[cardIndex];
+    if (!card || !Array.isArray(document) || document[0] !== searchDocumentHash(card)) return false;
+  }
+  try {
+    deserializeSearchIndex(searchIndex);
+  } catch {
+    return false;
+  }
+  return true;
+}
+
+export function renderSearchIndex(searchIndex) {
+  return stableStringify(searchIndex);
+}
+
+function countTokens(tokens) {
+  const counts = new Map();
+  for (const token of tokens) counts.set(token, (counts.get(token) ?? 0) + 1);
+  return counts;
+}
+
+function addDocumentToPostings(postings, id, document) {
+  for (const [token, vector] of document.terms ?? []) {
+    const tokenPostings = postings.get(token) ?? new Map();
+    tokenPostings.set(id, vector);
+    postings.set(token, tokenPostings);
+  }
+}
+
+function removeDocumentFromPostings(postings, id, document) {
+  for (const token of document?.tokens ?? []) {
+    const tokenPostings = postings.get(token);
+    if (!tokenPostings) continue;
+    tokenPostings.delete(id);
+    if (tokenPostings.size === 0) postings.delete(token);
+  }
+}
+
+function deserializeSearchIndex(searchIndex) {
+  const documents = new Map();
+  for (const [cardIndex, id] of searchIndex.cardIds.entries()) {
+    const serialized = searchIndex.documents[cardIndex];
+    if (
+      typeof id !== "string" ||
+      !Array.isArray(serialized) ||
+      serialized.length !== 2 ||
+      typeof serialized[0] !== "string" ||
+      !Array.isArray(serialized[1]) ||
+      !serialized[1].every((phrase) => typeof phrase === "string")
+    ) {
+      throw new Error("Malformed compact search document.");
+    }
+    documents.set(id, {
+      hash: serialized[0],
+      phrases: serialized[1],
+      tokens: [],
+    });
+  }
+
+  const postings = new Map();
+  for (const [tokenIndex, token] of searchIndex.tokens.entries()) {
+    if (typeof token !== "string") throw new Error("Malformed compact search token.");
+    const tokenPostings = new Map();
+    let previousCardIndex = -1;
+    for (const entry of searchIndex.postings[tokenIndex] ?? []) {
+      if (!Array.isArray(entry) || entry.length !== 2 || !Number.isInteger(entry[0]) || !Array.isArray(entry[1])) {
+        throw new Error("Malformed compact search posting.");
+      }
+      const cardIndex = entry[0];
+      const vector = entry[1];
+      if (cardIndex <= previousCardIndex || cardIndex < 0 || cardIndex >= searchIndex.cardIds.length) {
+        throw new Error("Compact search postings are not strictly ordered card ordinals.");
+      }
+      validateSparseVector(vector);
+      previousCardIndex = cardIndex;
+      const id = searchIndex.cardIds[cardIndex];
+      const document = documents.get(id);
+      if (!document) throw new Error("Compact search posting references an unknown card.");
+      tokenPostings.set(id, vector);
+      document.tokens.push(token);
+    }
+    if (tokenPostings.size > 0) postings.set(token, tokenPostings);
+  }
+  return { documents, postings };
+}
+
+function validateSparseVector(vector) {
+  if (vector.length === 0 || vector.length % 2 !== 0) throw new Error("Malformed compact search field vector.");
+  let previousFieldIndex = -1;
+  for (let index = 0; index < vector.length; index += 2) {
+    const fieldIndex = vector[index];
+    const count = vector[index + 1];
+    if (
+      !Number.isInteger(fieldIndex) ||
+      fieldIndex <= previousFieldIndex ||
+      fieldIndex < 0 ||
+      fieldIndex >= SEARCH_FIELD_ORDER.length ||
+      !Number.isInteger(count) ||
+      count <= 0
+    ) {
+      throw new Error("Malformed compact search field vector.");
+    }
+    previousFieldIndex = fieldIndex;
+  }
+}
+
+function isSortedUniqueStrings(values) {
+  let previous = null;
+  for (const value of values) {
+    if (typeof value !== "string" || (previous !== null && compareText(previous, value) >= 0)) return false;
+    previous = value;
+  }
+  return true;
+}
+
+function serializeSparseVector(sparse) {
+  const output = [];
+  for (const [fieldIndex, count] of [...sparse.entries()].sort(([left], [right]) => left - right)) {
+    output.push(fieldIndex, count);
+  }
+  return output;
+}