@henols/c64-re-tools 2.0.0-rc.9 → 2.0.1

package/src/knowledge/import.ts

@@ -7,7 +7,7 @@
 // another analyzer are never replaced: a finding that contradicts one is
 // reported as a conflict. Code against data is a contradiction; a different
 // name or a finer data type for the same kind of memory is not. One exception
-// (D58): where a run executed an instruction start, a code finding replaces
+//: where a run executed an instruction start, a code finding replaces
 // the data of another analyzer, never of the user, the LLM or a heuristic.
 
 import { createHash } from "node:crypto";
@@ -15,21 +15,20 @@ import type { DatabaseSync } from "node:sqlite";
 
 import { KnowledgeError, transaction } from "./database.ts";
 import {
+  analyzerNamed,
   REFERENCE_COLUMNS,
   referenceSuppressed,
   regionsOverlapping,
   suppressionsOverlapping,
   SYMBOL_COLUMNS,
   symbolAt,
-  symbolNamed,
-  symbolSuppressed,
   toReference,
   toSymbol,
   type ReferenceRow,
   type RegionRow,
   type SymbolRow,
 } from "./read.ts";
-import { CODE_REGION_TYPES, DEFAULT_SPACE, REFERENCE_KINDS, REGION_TYPES, SYMBOL_KINDS, type Origin, type ReferenceKind, type RegionType, type Space, type SymbolKind } from "./schema.ts";
+import { CODE_REGION_TYPES, DEFAULT_SPACE, POINTER_KIND, REFERENCE_KINDS, REGION_TYPES, SYMBOL_KINDS, type Origin, type ReferenceKind, type RegionType, type Space, type SymbolKind } from "./schema.ts";
 import { Change, checkAddress, checkSpace, hex, subtract, SYMBOL_NAME, type AddressRange, type Evidence } from "./write.ts";
 
 export type { AddressRange };
@@ -39,7 +38,7 @@ export type Analyzer = (typeof ANALYZERS)[number];
 
 export interface NormalizedFindings {
   analyzer: Analyzer;
-  /** The address space that the analyzed image belongs to (D53); the C64 when absent. */
+  /** The address space that the analyzed image belongs to; the C64 when absent. */
   space?: Space;
   /** The address ranges the analyzer examined; it makes claims only there. */
   coverage: AddressRange[];
@@ -49,13 +48,13 @@ export interface NormalizedFindings {
   regions: Array<{ start: number; end: number; type: RegionType }>;
   /** Owned and covered by their source address. */
   references: Array<{ from: number; to: number; kind: ReferenceKind }>;
-  /** True for an address where a coverage run executed an instruction start (D58). */
+  /** True for an address where a coverage run executed an instruction start. */
   executed?: (address: number) => boolean;
 }
 
 export interface ImportContext {
   /** The revision the caller read; the import is refused if knowledge changed since. */
-  expectedRevision?: number;
+  expectedRevision?: string;
   description?: string;
   /** The SHA-256 of the analyzed image, as lowercase hex. */
   inputHash?: string;
@@ -86,13 +85,16 @@ export interface ImportCounts {
   changed: number;
   retired: number;
   unchanged: number;
-  /** Findings at a place that a user/LLM removal blocks (D36); not written. */
+  /** Findings at a place that a user/LLM removal blocks; not written. */
   suppressed: number;
 }
 
 export interface ImportResult {
   /** The new revision, or null when nothing changed. */
-  revision: number | null;
+  revision: string | null;
+  /** The analyzer names. */
+  names: ImportCounts;
+  /** The analyzer's own rows from an older version that the run closed in symbols: only `retired` counts. */
   symbols: ImportCounts;
   regions: ImportCounts;
   references: ImportCounts;
@@ -163,6 +165,7 @@ export function validateFindings(findings: NormalizedFindings): AddressRange[] {
     checkAddress(reference.to, "reference target");
     if (!covering(reference.from, reference.from)) invalid(`the reference from ${hex(reference.from)} is outside the coverage.`);
     if (!(REFERENCE_KINDS as readonly string[]).includes(reference.kind)) invalid(`the reference kind ${reference.kind} is not known.`);
+    if (reference.kind === POINTER_KIND) invalid(`only an operand expression writes a pointer reference, not an analyzer.`);
   }
   return coverage;
 }
@@ -181,68 +184,73 @@ function ownReferences(db: DatabaseSync, analyzer: Analyzer, range: AddressRange
     .map((row) => toReference(row as Record<string, unknown>));
 }
 
-function importSymbols(db: DatabaseSync, change: Change, findings: NormalizedFindings, coverage: AddressRange[], result: ImportResult): void {
+function ownNames(db: DatabaseSync, analyzer: Analyzer, range: AddressRange, space: Space): SymbolRow[] {
+  return db
+    .prepare(`SELECT ${SYMBOL_COLUMNS} FROM current_analyzer_names WHERE space = ? AND origin = ? AND address BETWEEN ? AND ? ORDER BY address`)
+    .all(space, analyzer, range.start, range.end)
+    .map((row) => toSymbol(row as Record<string, unknown>));
+}
+
+/**
+ * The names of an analyzer are second names: they go to analyzer_names,
+ * one for each analyzer at an address, and never to symbols. A run replaces
+ * the analyzer's own names in its coverage, and closes the analyzer's old rows
+ * in symbols there (knowledge from an older version). A name that the user or the LLM
+ * gave the address already is not kept twice. A code or data kind that
+ * disagrees with the name of the address is a conflict.
+ */
+function importNames(db: DatabaseSync, change: Change, findings: NormalizedFindings, coverage: AddressRange[], result: ImportResult): void {
   const { analyzer } = findings;
   const space = findings.space ?? DEFAULT_SPACE;
+  for (const range of coverage) {
+    for (const own of ownSymbols(db, analyzer, range, space)) {
+      change.close("symbols", "space = ? AND address = ? AND origin = ?", space, own.address, analyzer);
+      result.symbols.retired++;
+    }
+  }
   const wanted = new Map(findings.symbols.map((symbol) => [symbol.address, symbol]));
   const settled = new Set<number>();
   const replaced = new Set<number>();
-  // Close first, so a name can move between addresses within one import.
   for (const range of coverage) {
-    for (const own of ownSymbols(db, analyzer, range, space)) {
+    for (const own of ownNames(db, analyzer, range, space)) {
       const finding = wanted.get(own.address);
-      if (finding === undefined) {
-        if (!findings.authoritative.symbols) continue;
-        change.close("symbols", "space = ? AND address = ?", space, own.address);
-        result.symbols.retired++;
-      } else if (finding.name === own.name && finding.kind === own.kind) {
+      // A seeded run gives back the name of the user or the LLM: the analyzer's own name there stays.
+      const echo = finding !== undefined && symbolAt(db, own.address, space)?.name === finding.name;
+      if (echo || (finding !== undefined && finding.name === own.name && finding.kind === own.kind)) {
         settled.add(own.address);
-        result.symbols.unchanged++;
-      } else {
-        change.close("symbols", "space = ? AND address = ?", space, own.address);
-        replaced.add(own.address);
+        result.names.unchanged++;
+        continue;
       }
+      change.close("analyzer_names", "space = ? AND address = ? AND origin = ?", space, own.address, analyzer);
+      if (finding === undefined) result.names.retired++;
+      else replaced.add(own.address);
     }
   }
   for (const finding of findings.symbols) {
-    if (settled.has(finding.address)) continue;
-    if (symbolSuppressed(db, finding.address, space)) {
-      result.symbols.suppressed++;
-      continue;
+    const current = symbolAt(db, finding.address, space);
+    if (current !== undefined && isCode(current.kind) !== isCode(finding.kind)) {
+      result.conflicts.push({ category: "symbol", address: finding.address, reason: "kind", current, finding: { name: finding.name, kind: finding.kind } });
     }
-    let current = symbolAt(db, finding.address, space);
-    if (current !== undefined && isCode(finding.kind) && !isCode(current.kind) && YIELDS_TO_EXECUTED_CODE.has(current.origin) && findings.executed?.(finding.address) === true) {
-      // A run executed this address: the data symbol of another analyzer gives way (D58).
-      change.close("symbols", "space = ? AND address = ?", space, current.address);
-      result.symbols.retired++;
-      current = undefined;
-    }
-    if (current !== undefined) {
-      // A semantic or other-analyzer row stays; only a code/data disagreement matters.
-      if (isCode(current.kind) !== isCode(finding.kind)) {
-        result.conflicts.push({ category: "symbol", address: finding.address, reason: "kind", current, finding: { name: finding.name, kind: finding.kind } });
-      }
+    if (settled.has(finding.address)) continue;
+    if (current?.name === finding.name) {
+      // The analyzer only gives back the name of the address.
+      if (replaced.has(finding.address)) result.names.retired++;
       continue;
     }
-    const owner = symbolNamed(db, finding.name, space);
-    if (owner?.origin === analyzer) {
-      // The analyzer moved its own name: its newer finding replaces its own older row.
-      change.close("symbols", "space = ? AND address = ?", space, owner.address);
-      result.symbols.retired++;
-    } else if (owner !== undefined) {
-      result.conflicts.push({ category: "symbol", address: finding.address, reason: "name-taken", current: owner, finding: { name: finding.name, kind: finding.kind } });
-      if (replaced.has(finding.address)) result.symbols.retired++;
-      continue;
+    // The analyzer moved its own name from an address outside the coverage.
+    for (const moved of analyzerNamed(db, finding.name, space).filter((row) => row.origin === analyzer)) {
+      change.close("analyzer_names", "space = ? AND address = ? AND origin = ?", space, moved.address, analyzer);
+      result.names.retired++;
     }
-    change.insertSymbol({ space, address: finding.address, name: finding.name, kind: finding.kind, origin: analyzer });
-    if (replaced.has(finding.address)) result.symbols.changed++;
-    else result.symbols.added++;
+    change.insertAnalyzerName({ space, address: finding.address, name: finding.name, kind: finding.kind, origin: analyzer });
+    if (replaced.has(finding.address)) result.names.changed++;
+    else result.names.added++;
   }
 }
 
-/** Analyzers whose regions a heuristic finding replaces where it lies (D44): user/LLM > heuristic > DXA/Ghidra. */
+/** Analyzers whose regions a heuristic finding replaces where it lies: user/LLM > heuristic > DXA/Ghidra. */
 const YIELDS_TO_HEURISTIC = new Set<Origin>(["dxa", "ghidra"]);
-/** Analyzers whose data gives way to a code finding over an executed instruction start (D58). */
+/** Analyzers whose data gives way to a code finding over an executed instruction start. */
 const YIELDS_TO_EXECUTED_CODE = new Set<Origin>(["dxa", "ghidra"]);
 
 /** True when an address in the range is an executed instruction start. */
@@ -264,7 +272,7 @@ function importRegions(db: DatabaseSync, change: Change, findings: NormalizedFin
       }
     }
   }
-  // Executed code is evidence about its bytes (D58): another analyzer's data region under a code finding gives way where a run executed an instruction start.
+  // Executed code is evidence about its bytes: another analyzer's data region under a code finding gives way where a run executed an instruction start.
   for (const region of findings.regions.filter((candidate) => isCode(candidate.type))) {
     for (const row of regionsOverlapping(db, region.start, region.end, space)) {
       if (row.origin === analyzer || !YIELDS_TO_EXECUTED_CODE.has(row.origin) || isCode(row.type) || row.type === "dual-use") continue;
@@ -275,7 +283,7 @@ function importRegions(db: DatabaseSync, change: Change, findings: NormalizedFin
     }
   }
   const conflictWith = (row: RegionRow, region: { start: number; end: number; type: RegionType }) => {
-    // dual-use bytes are code at one time and data at another: neither finding contradicts them (D42).
+    // dual-use bytes are code at one time and data at another: neither finding contradicts them.
     if (isCode(row.type) === isCode(region.type) || row.type === "dual-use") return;
     result.conflicts.push({
       category: "region",
@@ -381,10 +389,11 @@ export function importFindings(db: DatabaseSync, findings: NormalizedFindings, c
   const coverage = validateFindings(findings);
   return transaction(db, () => {
     const change = new Change(db, { origin: findings.analyzer, ...context }, `import-${findings.analyzer}`);
-    const result: ImportResult = { revision: null, symbols: counts(), regions: counts(), references: counts(), conflicts: [] };
-    importSymbols(db, change, findings, coverage, result);
+    const result: ImportResult = { revision: null, names: counts(), symbols: counts(), regions: counts(), references: counts(), conflicts: [] };
+    importNames(db, change, findings, coverage, result);
     importRegions(db, change, findings, coverage, result);
     importReferences(db, change, findings, coverage, result);
+    change.check();
     result.revision = change.revision;
     return result;
   });

package/src/knowledge/join.ts

@@ -0,0 +1,211 @@
+// Joins the batches of agents that worked in parallel, each on its own range
+// of addresses, into one batch for apply. The owner of an address is
+// the batch whose range holds it: its name, comment, operand and module win
+// there; at an address that no batch owns, the first batch in the order given
+// wins. A name that a batch gave to an address, and that the join replaced
+// with the name of another batch, is replaced in that batch's operand
+// expressions and macro args too. The join writes no knowledge.
+
+import { replaceIdentifiers } from "./expression.ts";
+import type { BatchOperation } from "./write.ts";
+
+export interface Batch {
+  /** The file the batch came from, for the report. */
+  file: string;
+  /** The range of addresses that the batch owns. */
+  owner?: { start: number; end: number };
+  ops: BatchOperation[];
+}
+
+/** One place where batches gave different values: the value that stays, and the batches whose value the join left out. */
+export interface JoinChoice {
+  what: "name" | "comment" | "operand" | "module" | "macro";
+  space: string;
+  /** The address, or for a macro its name. */
+  at: number | string;
+  kept: { file: string; value: string };
+  left: Array<{ file: string; value: string }>;
+}
+
+/** A clash that the join cannot decide: one name, or one module file, at two addresses. */
+export interface JoinClash {
+  what: "name" | "file";
+  value: string;
+  space: string;
+  places: Array<{ address: number; file: string }>;
+}
+
+export interface JoinResult {
+  ops: BatchOperation[];
+  choices: JoinChoice[];
+  clashes: JoinClash[];
+  /** Writes that were the same in more than one batch, and that the join wrote once. */
+  repeated: number;
+}
+
+type Op<K extends BatchOperation["op"]> = Extract<BatchOperation, { op: K }>;
+const spaceOf = (op: { space?: string }) => op.space ?? "c64";
+
+/** The value of a keyed write as text, to compare and report it. */
+function valueOf(op: BatchOperation): string {
+  switch (op.op) {
+    case "rename":
+      return op.kind === undefined ? op.name : `${op.name} (${op.kind})`;
+    case "comment":
+      return op.text;
+    case "operand":
+      return op.expression ?? `${op.select} ${op.symbol ?? ""}${op.offset === undefined || op.offset === 0 ? "" : op.offset > 0 ? `+${op.offset}` : op.offset}`;
+    case "module":
+      return op.title === undefined ? op.file : `${op.file} (${op.title})`;
+    case "macro":
+      return `${op.name}(${(op.params ?? []).join(", ")})`;
+    default:
+      return JSON.stringify(op);
+  }
+}
+
+export function joinBatches(batches: readonly Batch[]): JoinResult {
+  const owns = (batch: Batch, address: number) => batch.owner !== undefined && address >= batch.owner.start && address <= batch.owner.end;
+  const choices: JoinChoice[] = [];
+  const clashes: JoinClash[] = [];
+
+  /** For each key, the write that stays: the owner's last write, else the first batch's last write. */
+  function choose<K extends "rename" | "comment" | "operand" | "module" | "macro">(op: K, what: JoinChoice["what"], keyOf: (write: Op<K>) => string, at: (write: Op<K>) => number | string) {
+    const all = new Map<string, Array<{ batch: Batch; write: Op<K> }>>();
+    for (const batch of batches) {
+      for (const write of batch.ops) {
+        if (write.op !== op) continue;
+        const key = keyOf(write as Op<K>);
+        all.set(key, [...(all.get(key) ?? []), { batch, write: write as Op<K> }]);
+      }
+    }
+    const kept = new Map<string, { batch: Batch; write: Op<K> }>();
+    for (const [key, candidates] of all) {
+      const place = at(candidates[0]!.write);
+      const owner = typeof place === "number" ? candidates.filter((candidate) => owns(candidate.batch, place)) : [];
+      // A batch can write a place twice; its last write counts, as in apply.
+      const pick = (owner.length > 0 ? owner : candidates.filter((candidate) => candidate.batch === candidates[0]!.batch)).at(-1)!;
+      kept.set(key, pick);
+      const left = candidates.filter((candidate) => candidate.batch !== pick.batch && valueOf(candidate.write) !== valueOf(pick.write));
+      const distinct = [...new Map(left.map((candidate) => [`${candidate.batch.file}|${valueOf(candidate.write)}`, candidate])).values()];
+      if (distinct.length > 0) {
+        choices.push({
+          what,
+          space: spaceOf(pick.write),
+          at: place,
+          kept: { file: pick.batch.file, value: valueOf(pick.write) },
+          left: distinct.map((candidate) => ({ file: candidate.batch.file, value: valueOf(candidate.write) })),
+        });
+      }
+    }
+    return kept;
+  }
+
+  const names = choose("rename", "name", (write) => `${spaceOf(write)}|${write.address}`, (write) => write.address);
+  // Name clashes: one name for two addresses after the choice.
+  const byName = new Map<string, Array<{ address: number; file: string }>>();
+  for (const { batch, write } of names.values()) {
+    const key = `${spaceOf(write)}|${write.name}`;
+    byName.set(key, [...(byName.get(key) ?? []), { address: write.address, file: batch.file }]);
+  }
+  for (const [key, places] of byName) {
+    if (places.length > 1) clashes.push({ what: "name", value: key.split("|")[1]!, space: key.split("|")[0]!, places: places.sort((a, b) => a.address - b.address) });
+  }
+  // A name that a batch gave and the join replaced: each batch's names follow the join's names.
+  const finalName = new Map<string, string>();
+  for (const write of names.values()) finalName.set(`${spaceOf(write.write)}|${write.write.address}`, write.write.name);
+  const renamedIn = new Map<Batch, Map<string, string>>();
+  for (const batch of batches) {
+    const map = new Map<string, string>();
+    for (const write of batch.ops) {
+      if (write.op !== "rename") continue;
+      const now = finalName.get(`${spaceOf(write)}|${write.address}`);
+      if (now !== undefined && now !== write.name) map.set(`${spaceOf(write)}|${write.name}`, now);
+    }
+    renamedIn.set(batch, map);
+  }
+  const follow = (batch: Batch, space: string, text: string) => {
+    const map = renamedIn.get(batch)!;
+    if (map.size === 0) return text;
+    try {
+      return replaceIdentifiers(text, (name) => map.get(`${space}|${name}`) ?? name);
+    } catch {
+      // Text that is no expression stays as the batch wrote it; apply refuses it then.
+      return text;
+    }
+  };
+
+  const comments = choose("comment", "comment", (write) => `${spaceOf(write)}|${write.address}|${write.placement}`, (write) => write.address);
+  const operands = choose("operand", "operand", (write) => `${spaceOf(write)}|${write.address}`, (write) => write.address);
+  const modules = choose("module", "module", (write) => `${spaceOf(write)}|${write.start}`, (write) => write.start);
+  const macros = choose("macro", "macro", (write) => `${spaceOf(write)}|${write.name}`, (write) => write.name);
+  const byFile = new Map<string, Array<{ address: number; file: string }>>();
+  for (const { batch, write } of modules.values()) {
+    const key = `${spaceOf(write)}|${write.file}`;
+    byFile.set(key, [...(byFile.get(key) ?? []), { address: write.start, file: batch.file }]);
+  }
+  for (const [key, places] of byFile) {
+    if (places.length > 1) clashes.push({ what: "file", value: key.split("|")[1]!, space: key.split("|")[0]!, places: places.sort((a, b) => a.address - b.address) });
+  }
+
+  // The order of the joined batch: removals, the regions of other batches and then the owner's (a later classify wins
+  // where they overlap), the names, the comments, the modules, the references, the macros, then the writes that use names.
+  let repeated = 0;
+  const seen = new Set<string>();
+  const once = (write: BatchOperation) => {
+    const key = JSON.stringify(write);
+    if (seen.has(key)) {
+      repeated++;
+      return false;
+    }
+    seen.add(key);
+    return true;
+  };
+  const named = new Set([...names.values()].map(({ write }) => `${spaceOf(write)}|${write.address}`));
+  const removals: BatchOperation[] = [];
+  const foreignRegions: BatchOperation[] = [];
+  const ownRegions: BatchOperation[] = [];
+  const references: BatchOperation[] = [];
+  const rest: BatchOperation[] = [];
+  for (const batch of batches) {
+    for (const write of batch.ops) {
+      switch (write.op) {
+        case "remove-symbol":
+          // A name from any batch wins over a removal at the same address.
+          if (!named.has(`${spaceOf(write)}|${write.address}`) && once(write)) removals.push(write);
+          break;
+        case "classify":
+        case "unclassify":
+          if (once(write)) (owns(batch, write.start) ? ownRegions : foreignRegions).push(write);
+          break;
+        case "reference":
+        case "unreference":
+          if (once(write)) references.push(write);
+          break;
+        case "rename":
+        case "comment":
+        case "operand":
+        case "module":
+        case "macro":
+          break;
+        case "use":
+          if (once(write)) rest.push({ ...write, args: (write.args ?? []).map((arg) => follow(batch, spaceOf(write), arg)) });
+          break;
+        default:
+          if (once(write)) rest.push(write);
+      }
+    }
+  }
+  const kept = <T extends { batch: Batch; write: BatchOperation }>(map: Map<string, T>) => [...map.values()].map(({ write }) => write);
+  const operandWrites = [...operands.values()].map(({ batch, write }) => ({
+    ...write,
+    ...(write.symbol === undefined ? {} : { symbol: follow(batch, spaceOf(write), write.symbol) }),
+    ...(write.expression === undefined ? {} : { expression: follow(batch, spaceOf(write), write.expression) }),
+  }));
+  return {
+    ops: [...removals, ...foreignRegions, ...ownRegions, ...kept(names), ...kept(comments), ...kept(modules), ...references, ...kept(macros), ...operandWrites, ...rest],
+    choices,
+    clashes,
+    repeated,
+  };
+}