@azure-id/orc 1.8.1 → 1.8.2

package/bin/graph-resolve.js

@@ -38,7 +38,11 @@
 // WHAT IS STORED
 //
 //   callers target symbol id → the confident callers of it, as
-//           `[caller symbol id, line, state, ref]`.
+//           `[caller symbol id, line, state, kind, extra]`, where `kind` is
+//           0 a call · 1 a function passed by name · 2 a URL · 3 an inherited
+//           member (v1.8.2, schema 3). `extra` is the URL text for 2 and the
+//           alias name (`store` in `store.list()`) for a call through an
+//           instance alias; absent otherwise.
 //   maybe   target symbol id → how many AMBIGUOUS calls list this target, by
 //           CALLER FILE. Per file, not per call site: 5 MB on django/django
 //           instead of 45 MB, and the card only ever shows the total.
@@ -47,8 +51,10 @@ const fs = require("fs");
 const path = require("path");
 const G = require("./graph.js");
 
-const RESOLVE_SCHEMA = 2;
-const CONFIDENT = new Set(["LOCAL", "IMPORT", "UNIQUE"]);
+// 3 (v1.8.2 W1): ROUTE rows. A 1.8.1 cache is refused by the schema check and
+// recomputed on read until the next `update` rewrites it — no answer changes.
+const RESOLVE_SCHEMA = 3;
+const CONFIDENT = new Set(["LOCAL", "IMPORT", "UNIQUE", "ROUTE"]);
 const NAMES_PER_KEY = 5;
 
 function resolvedPath(claudeDir) {
@@ -57,6 +63,15 @@ function resolvedPath(claudeDir) {
 function namesPath(claudeDir) {
   return path.join(G.graphPaths(claudeDir).dir, "names.json");
 }
+// v1.8.2 W3 (D3) — the WIDE-FILE table: a file with many symbols → the few
+// worth naming when somebody is about to read the whole thing. It is its OWN
+// file, not a field on `names.json` or `files.json`, because the hint that
+// reads it is OFF by default and the two hints that are ON must not pay for it.
+function widePath(claudeDir) {
+  return path.join(G.graphPaths(claudeDir).dir, "wide.json");
+}
+const WIDE_MIN = 8;
+const WIDE_TOP = 6;
 
 // Every map here is keyed by data from the repository — a bare symbol name can
 // be `constructor`, `toString` or `__proto__`. A plain object would hand back
@@ -79,30 +94,62 @@ function readJson(file) {
 // Called by `graphUpdate` INSIDE the lock, after index.json is on disk. It
 // never throws into the update: a cache that cannot be written is a slower
 // graph, not a broken one.
-function build(claudeDir, root) {
+function build(claudeDir, root, index) {
   const t0 = Date.now();
   const Q = require("./graph-query.js");
   try {
     // `noCache` matters: the writer must COMPUTE, never read back the cache it
-    // is about to replace.
-    const model = Q.loadModel(claudeDir, root, { noCache: true });
+    // is about to replace. `index` (S2) is the in-memory index the update just
+    // wrote, so the file is parsed once per update, not twice.
+    const model = Q.loadModel(claudeDir, root, { noCache: true, index });
     if (!model) return { route: "skipped", reason: "no-index", ms: Date.now() - t0 };
 
     const callers = bare();
     const maybe = bare();
+    // S1 (v1.8.2 W8): the FORWARD half, for the sharded fast path. Every
+    // resolution below is already computed for the reverse map; this records
+    // the same answer facing the other way, so `ctx` can read it instead of
+    // resolving. UNRESOLVED is counted, never stored — it is most of the calls
+    // in a real repository and the card only prints its total.
+    const S = require("./graph-shard.js");
+    const forward = bare();
+    const ambiguous = (r, rel) => {
+      for (const cand of r.candidates || []) {
+        const m = own(maybe, cand) || (maybe[cand] = bare());
+        m[rel] = (own(m, rel) || 0) + 1;
+      }
+    };
     for (const [rel, f] of Object.entries(model.byFile)) {
       for (const s of f.symbols || []) {
         s.file = rel;
-        for (const c of s.calls || []) {
+        const calls = s.calls || [];
+        const fwd = { u: 0, r: [] };
+        const seenName = new Set();
+        for (let i = 0; i < calls.length; i++) {
+          const c = calls[i];
           const r = Q.resolveCall(model, rel, s, c);
-          if (r.target && CONFIDENT.has(r.state)) {
-            (own(callers, r.target) || (callers[r.target] = [])).push([s.id, c.line, r.state, c.ref ? 1 : 0]);
-          } else if (r.state === "AMBIGUOUS") {
-            for (const cand of r.candidates || []) {
-              const m = own(maybe, cand) || (maybe[cand] = bare());
-              m[rel] = (own(m, rel) || 0) + 1;
-            }
+          // `calleesOf` dedupes by call NAME and keeps the first, so the stored
+          // rows must be the ones that survive that same dedupe — otherwise a
+          // fast card would answer from a row the full card never reaches.
+          if (!seenName.has(c.name)) {
+            seenName.add(c.name);
+            const row = S.forwardRow(i, r);
+            if (row) fwd.r.push(row);
+            else fwd.u++;
           }
+          if (r.target && CONFIDENT.has(r.state)) {
+            const kind = c.ref ? 1 : r.inherited ? 3 : 0;
+            const row = [s.id, c.line, r.state, kind];
+            if (kind === 0 && c.alias) row.push(c.alias);
+            (own(callers, r.target) || (callers[r.target] = [])).push(row);
+          } else if (r.state === "AMBIGUOUS") ambiguous(r, rel);
+        }
+        if (fwd.r.length || fwd.u) forward[s.id] = fwd;
+        // W1: the URLs this symbol sends, resolved to route symbols.
+        for (const u of s.urls || []) {
+          const r = Q.resolveUrl(model, u);
+          if (r.state === "ROUTE") (own(callers, r.target) || (callers[r.target] = [])).push([s.id, u.line, "ROUTE", 2, `${u.method} ${u.url}`]);
+          else if (r.state === "AMBIGUOUS") ambiguous(r, rel);
         }
       }
     }
@@ -118,13 +165,24 @@ function build(claudeDir, root) {
     };
     G.atomicWrite(resolvedPath(claudeDir), JSON.stringify(out));
     writeNames(claudeDir, model, out.callers);
-    return { route: "full", ms: Date.now() - t0 };
+    writeWide(claudeDir, model, out.callers);
+    // D4 (v1.8.2 W7): the ranked map is built HERE, from the callers object
+    // this pass already holds. Building it anywhere else would mean walking
+    // every call a second time to learn the same thing.
+    const map = require("./graph-map.js").build(claudeDir, model, out.callers);
+    // S1: the shards come LAST. They are the only derived store a reader can do
+    // without entirely, so a failure here costs speed and nothing else.
+    const shards = S.build(claudeDir, model, out.callers, out.maybe, forward);
+    return { route: "full", map, shards, ms: Date.now() - t0 };
   } catch (e) {
     // A cache is an optimisation. Losing it must never lose the update — and a
     // HALF-written one must never be read, so it goes.
     try {
       fs.rmSync(resolvedPath(claudeDir), { force: true });
       fs.rmSync(namesPath(claudeDir), { force: true });
+      fs.rmSync(widePath(claudeDir), { force: true });
+      fs.rmSync(require("./graph-map.js").mapPath(claudeDir), { force: true });
+      fs.rmSync(require("./graph-shard.js").shardDir(claudeDir), { recursive: true, force: true });
     } catch (_) {}
     return { route: "failed", reason: String((e && e.message) || e), ms: Date.now() - t0 };
   }
@@ -165,6 +223,25 @@ function writeNames(claudeDir, model, callers) {
   G.atomicWrite(namesPath(claudeDir), JSON.stringify({ schema: RESOLVE_SCHEMA, generation: model.meta.generation, names: sortedObject(out) }));
 }
 
+// A file the extractor found `WIDE_MIN` or more symbols in → the `WIDE_TOP`
+// most important of them, with their ranges. Only wide files are listed: the
+// hint exists to turn a whole-file read into a range read the NEXT time, and a
+// file with five symbols is already a range.
+function writeWide(claudeDir, model, callers) {
+  const SIG = require("./graph-signals.js");
+  const out = bare();
+  for (const [rel, f] of Object.entries(model.byFile)) {
+    const syms = (f.symbols || []).filter((s) => s.kind !== "module");
+    if (syms.length < WIDE_MIN) continue;
+    out[rel] = syms
+      .map((s) => ({ s, imp: SIG.importance(model, s, (own(callers, s.id) || []).length) }))
+      .sort((a, b) => b.imp - a.imp || a.s.lines[0] - b.s.lines[0])
+      .slice(0, WIDE_TOP)
+      .map(({ s }) => [s.qname, s.lines[0], s.lines[1]]);
+  }
+  G.atomicWrite(widePath(claudeDir), JSON.stringify({ schema: RESOLVE_SCHEMA, generation: model.meta.generation, files: sortedObject(out) }));
+}
+
 // ── read ────────────────────────────────────────────────────────────────────
 // Used ONLY when the generation matches. Anything else returns null and the
 // reader computes, exactly as it did before this file existed.
@@ -175,4 +252,4 @@ function load(claudeDir, meta) {
   return r;
 }
 
-module.exports = { own, RESOLVE_SCHEMA, resolvedPath, namesPath, build, load };
+module.exports = { own, RESOLVE_SCHEMA, resolvedPath, namesPath, widePath, WIDE_MIN, WIDE_TOP, build, load };

package/bin/graph-shard.js

@@ -0,0 +1,325 @@
+"use strict";
+// ── orc graph — S1: SHARDED READS (v1.8.2 W8, DE-I) ─────────────────────────
+//
+// EW0 dropped sharding because the parse (235 ms) was smaller than the resolve
+// (437 ms). EW2 then removed the resolve half by storing the reverse edges, and
+// what is left on django is the parse: a one-symbol `ctx` opens a 23 MB
+// `index.json` and a 15 MB `resolved.json` to answer a question about a handful
+// of records.
+//
+// Measured on django/django (2,977 files, 44,572 symbols):
+//
+//     index.json       234 ms  ─┐
+//     resolved.json    101 ms   ├─ 373 ms, every `ctx <symbol>`
+//     names.json        37 ms  ─┘
+//
+//     files.json         3 ms  ─┐
+//     names.json        37 ms   ├─ ~42 ms for the same answer
+//     ~11 blobs          1 ms   │
+//     1-3 shards         1 ms  ─┘
+//
+// THE RULE THAT MAKES THIS SAFE: the fast path answers, byte for byte, what the
+// full model answers — or it does not answer at all. It never approximates.
+// `fastModel()` returns null the moment it cannot PROVE its answer is the same
+// one, and the caller loads the full model exactly as before. Every reason it
+// gives up is named in `reason`, so a fast path that quietly stopped being fast
+// is visible rather than invisible.
+//
+// The place it gives up most is target resolution. `names.json` is capped at
+// `NAMES_PER_KEY` rows per bare name, so a name with six definitions shows five
+// — which means "these are all of them" is only knowable BELOW the cap. Above
+// it, and for a query that resolves through a URL, a file path, a line number
+// or a fuzzy nearest-match, the full model is the only honest answer.
+//
+// WHAT IS STORED — `resolved/<ab>.json`, `<ab>` being the first two hex
+// characters of the file's blob, which is the shard layout `blobs/` already
+// uses:
+//
+//     callers  target symbol id -> the rows `resolved.json` holds, for targets
+//              whose FILE hashes into this shard
+//     maybe    target symbol id -> the AMBIGUOUS-caller counts, sharded with
+//              the callers they belong to. Without it a fast card would print
+//              no `maybe` line where the full card prints one, which is a
+//              DIFFERENT card, not a shorter one.
+//     calls    symbol id -> { u, r } for symbols whose OWN file hashes here.
+//              `r` is one row per RESOLVED call, positional against the
+//              symbol's `calls` array; `u` is how many of its calls resolved to
+//              nothing. UNRESOLVED rows are not stored — they are 72% of the
+//              calls in a real repository and the card only ever counts them.
+//     cands    the AMBIGUOUS candidate lists, INTERNED. A name that is ambiguous
+//              is ambiguous the same way everywhere it is called, so the same
+//              list was being written once per call site: 25.3 MB of django's
+//              shards, against 8.2 MB once each. An AMBIGUOUS row carries the
+//              INDEX into this table, never the list.
+
+const fs = require("fs");
+const path = require("path");
+const G = require("./graph.js");
+
+const SHARD_SCHEMA = 1;
+// The state words, as one byte each. The card prints words; the store does not
+// have to repeat them 400,000 times.
+const STATE_CODE = { LOCAL: 0, IMPORT: 1, UNIQUE: 2, ROUTE: 3, AMBIGUOUS: 4 };
+const CODE_STATE = ["LOCAL", "IMPORT", "UNIQUE", "ROUTE", "AMBIGUOUS"];
+
+const own = (o, k) => (o && Object.prototype.hasOwnProperty.call(o, k) ? o[k] : undefined);
+
+// Thrown when the fast path discovers, HALFWAY through a card, that it cannot
+// answer identically after all. The caller catches it and re-answers from the
+// full model — the same fallback every other decline takes, just later.
+class FastUnavailable extends Error {}
+
+function shardDir(claudeDir) {
+  return path.join(G.graphPaths(claudeDir).dir, "resolved");
+}
+function shardOf(blob) {
+  return String(blob || "").slice(0, 2) || "00";
+}
+function shardPath(claudeDir, ab) {
+  return path.join(shardDir(claudeDir), `${ab}.json`);
+}
+
+function readJson(file) {
+  try {
+    return JSON.parse(fs.readFileSync(file, "utf8"));
+  } catch (_) {
+    return null;
+  }
+}
+
+// ── build (called by `update`, inside the lock) ─────────────────────────────
+// `callers` is the object `graph-resolve.build` already produced. `calls` is
+// the forward half, collected in the SAME pass — every resolution is computed
+// once and written twice, never computed twice.
+function build(claudeDir, model, callers, maybe, calls) {
+  const t0 = Date.now();
+  const dir = shardDir(claudeDir);
+  try {
+    // A shard set from another generation must never be half-replaced. The
+    // whole directory goes, then the new one is written.
+    fs.rmSync(dir, { recursive: true, force: true });
+    fs.mkdirSync(dir, { recursive: true });
+    const buckets = new Map();
+    const bucket = (rel) => {
+      const ab = shardOf((model.byFile[rel] || {}).blob);
+      let b = buckets.get(ab);
+      if (!b) buckets.set(ab, (b = { callers: Object.create(null), maybe: Object.create(null), calls: Object.create(null), cands: [], candIdx: new Map() }));
+      return b;
+    };
+    for (const [target, rows] of Object.entries(callers)) {
+      const rel = String(target).split("#")[0];
+      if (model.byFile[rel]) bucket(rel).callers[target] = rows;
+    }
+    for (const [target, rows] of Object.entries(maybe)) {
+      const rel = String(target).split("#")[0];
+      if (model.byFile[rel]) bucket(rel).maybe[target] = rows;
+    }
+    for (const [id, rec] of Object.entries(calls)) {
+      const rel = String(id).split("#")[0];
+      if (!model.byFile[rel]) continue;
+      const b = bucket(rel);
+      // The interning happens HERE, per shard, because a shard is the unit a
+      // reader parses: a table shared across shards would mean reading a second
+      // file to render one row.
+      b.calls[id] = {
+        u: rec.u,
+        r: rec.r.map((row) => {
+          if (row[1] !== STATE_CODE.AMBIGUOUS) return row;
+          const key = (row[4] || []).join("\u0000");
+          let at = b.candIdx.get(key);
+          if (at === undefined) {
+            at = b.cands.length;
+            b.cands.push(row[4] || []);
+            b.candIdx.set(key, at);
+          }
+          return [row[0], row[1], null, 0, at];
+        }),
+      };
+    }
+    let bytes = 0;
+    for (const [ab, b] of buckets) {
+      const body = JSON.stringify({
+        schema: SHARD_SCHEMA,
+        engine: G.ENGINE,
+        generation: model.meta.generation,
+        callers: b.callers,
+        maybe: b.maybe,
+        cands: b.cands,
+        calls: b.calls,
+      });
+      bytes += body.length;
+      G.atomicWrite(shardPath(claudeDir, ab), body);
+    }
+    return { route: "full", shards: buckets.size, bytes, ms: Date.now() - t0 };
+  } catch (e) {
+    // A shard set that cannot be written is a slower `ctx`, never a wrong one.
+    try {
+      fs.rmSync(dir, { recursive: true, force: true });
+    } catch (_) {}
+    return { route: "failed", reason: String((e && e.message) || e), ms: Date.now() - t0 };
+  }
+}
+
+// One forward row: the call's INDEX in the symbol's own `calls` array, its
+// state, its target, and — only for AMBIGUOUS — its candidate list.
+function forwardRow(i, res) {
+  const code = STATE_CODE[res.state];
+  if (code === undefined) return null; // UNRESOLVED — counted, never stored
+  if (res.state === "AMBIGUOUS") return [i, code, null, 0, res.candidates || []];
+  return [i, code, res.target, res.inherited ? 1 : 0];
+}
+
+// ── read ────────────────────────────────────────────────────────────────────
+// A mini-model that answers `ctx <symbol>` exactly as the full one does, or
+// null with the reason it could not.
+const NAMES_PER_KEY = 5;
+const norm = (p) => String(p).split("\\").join("/").replace(/^\.\//, "");
+
+function fastModel(claudeDir, root, query) {
+  const p = G.graphPaths(claudeDir);
+  const meta = readJson(p.meta);
+  if (!meta || !meta.generation) return { model: null, reason: "no-meta" };
+  if (!fs.existsSync(shardDir(claudeDir))) return { model: null, reason: "no-shards" };
+
+  const q = norm(query);
+  // Everything the fast path cannot prove it would answer identically. Each of
+  // these needs a whole-repository walk that `names.json` cannot stand in for.
+  if (q.includes("/")) return { model: null, reason: "path-or-url" };
+  if (/:\d+$/.test(q)) return { model: null, reason: "file-line" };
+  if (!q || /\s/.test(q)) return { model: null, reason: "not-a-plain-name" };
+
+  const files = readJson(p.files);
+  if (!files) return { model: null, reason: "no-files" };
+  const namesFile = readJson(require("./graph-resolve.js").namesPath(claudeDir));
+  if (!namesFile || namesFile.generation !== meta.generation) return { model: null, reason: "names-stale" };
+
+  const bare = q.split(".").pop();
+  const rows = own(namesFile.names, bare);
+  if (!rows) return { model: null, reason: "name-unknown" };
+  // AT the cap, "these are all of them" stops being knowable. Below it, it is.
+  if (rows.length >= NAMES_PER_KEY) return { model: null, reason: "name-at-cap" };
+  const hits = rows.filter((r) => r[0] === q);
+  if (hits.length !== 1) return { model: null, reason: hits.length ? "ambiguous-qname" : "qname-not-exact" };
+
+  const rel = hits[0][2];
+  const rec = files[rel];
+  if (!rec) return { model: null, reason: "file-unknown" };
+
+  const loaded = Object.create(null);
+  const loadFile = (r) => {
+    if (own(loaded, r) !== undefined) return loaded[r];
+    const f = files[r];
+    // A path the index never held is not an error — the full model would not
+    // know it either, and both answer the same "no".
+    if (!f) return (loaded[r] = null);
+    const blob = readJson(path.join(p.blobs, f.blob.slice(0, 2), `${f.blob}.json`));
+    // A file the index DOES hold, whose blob will not read, is different: the
+    // full model has that file's symbols from `index.json` and would print a
+    // row this path would drop. Dropping a row is a different card, so this
+    // throws and the caller starts again with the full model.
+    if (!blob) throw new FastUnavailable("blob-unreadable:" + r);
+    for (const s of blob.symbols || []) s.file = r;
+    return (loaded[r] = { ...blob, blob: f.blob });
+  };
+
+  let home;
+  try {
+    home = loadFile(rel);
+  } catch (e) {
+    return { model: null, reason: e instanceof FastUnavailable ? e.message : "blob-missing" };
+  }
+  if (!home) return { model: null, reason: "blob-missing" };
+  const mine = (home.symbols || []).filter((s) => s.kind !== "module" && s.qname === q);
+  // Two symbols with the same qname in one file (a Go build tag, a Python
+  // redefinition) are a case the full model settles by width. It is rare and it
+  // is not worth a second implementation.
+  if (mine.length !== 1) return { model: null, reason: mine.length ? "duplicate-qname" : "not-in-blob" };
+  const sym = mine[0];
+
+  const shards = new Map();
+  const shardFor = (r) => {
+    const f = files[r];
+    if (!f) return null;
+    const ab = shardOf(f.blob);
+    if (shards.has(ab)) return shards.get(ab);
+    const s = readJson(shardPath(claudeDir, ab));
+    const ok = s && s.schema === SHARD_SCHEMA && s.engine === G.ENGINE && s.generation === meta.generation ? s : null;
+    shards.set(ab, ok);
+    return ok;
+  };
+  if (!shardFor(rel)) return { model: null, reason: "shard-stale" };
+
+  // `byFile` and `symbols` are LAZY: a blob is read the first time something
+  // asks about the file it holds, and never again. A card about one symbol on
+  // django touches about a dozen files, out of nearly three thousand.
+  const byFile = new Proxy(Object.create(null), {
+    get: (_, r) => (typeof r === "string" ? loadFile(r) || undefined : undefined),
+    has: (_, r) => typeof r === "string" && !!files[r],
+  });
+  const symbols = {
+    get(id) {
+      const f = loadFile(String(id).split("#")[0]);
+      if (!f) return undefined;
+      return (f.symbols || []).find((s) => s.id === id);
+    },
+    has(id) {
+      return this.get(id) !== undefined;
+    },
+    // Nothing on the fast path may walk every symbol — that is exactly the
+    // whole-repository question this path exists to avoid. A caller that needs
+    // it has already been sent to the full model.
+    values() {
+      throw new Error("graph-shard: the fast model has no symbol table to walk");
+    },
+  };
+
+  // The target's own shard holds its callers AND its ambiguous-caller counts.
+  // `ctx` asks for callers of the TARGET and of nothing else, so one shard is
+  // the whole reverse half of this card.
+  const s0 = shardFor(rel);
+  const callersCache = Object.create(null);
+  const maybeCache = Object.create(null);
+  for (const [k, v] of Object.entries(s0.callers)) callersCache[k] = v;
+  for (const [k, v] of Object.entries(s0.maybe || {})) maybeCache[k] = v;
+
+  return {
+    model: {
+      root,
+      meta,
+      fast: true,
+      notes: require("./graph-notes.js").noteIndex(claudeDir),
+      byFile,
+      fileSet: new Set(Object.keys(files)),
+      symbols,
+      byName: new Map(),
+      bindings: new Map(),
+      resolved: new Map(),
+      callersCache,
+      maybeCache,
+      generation: meta.generation,
+      suffix: new Map(),
+      dirs: null,
+      goModule: undefined,
+      callIndex: null,
+      routes: null,
+      urlIndex: null,
+      urlMemo: new Map(),
+      blobStates: new Map(),
+      // The forward half. `calleesOf` reads this instead of resolving, and a
+      // symbol whose shard is missing sends the whole read back to the full
+      // model rather than answering with fewer calls than it has.
+      callsFor(s) {
+        const sh = shardFor(String(s.id).split("#")[0]);
+        if (!sh) return null;
+        const rec = own(sh.calls, s.id) || { u: 0, r: [] };
+        // The candidate table travels WITH the rows, so the caller never has to
+        // know which shard a row came out of.
+        return { u: rec.u, r: rec.r, cands: sh.cands || [] };
+      },
+    },
+    sym,
+    reason: null,
+  };
+}
+
+module.exports = { SHARD_SCHEMA, STATE_CODE, CODE_STATE, FastUnavailable, shardDir, shardPath, shardOf, build, forwardRow, fastModel };