@ghostlygawd/codeweb 0.9.0

package/scripts/lib/shingles.mjs

@@ -0,0 +1,39 @@
+// codeweb shared token-shingling primitives — lifted verbatim out of overlap.mjs so the
+// body-similarity logic lives ONCE (dogfooding the anti-duplication mission). Used by overlap.mjs
+// (duplicate confirmation) and find-similar.mjs (reuse-at-write-time). Pure, no I/O.
+//
+// NOTE: this KW stop-list is overlap.mjs's, and is DELIBERATELY DIFFERENT from the extractor's
+// KEYWORDS set (extract-symbols.mjs) — the extractor's includes call-site noise like `print`/`super`
+// for edge filtering; this one is tuned for body-shingle similarity. Do not unify the two.
+
+export const KW = new Set(['if', 'for', 'while', 'switch', 'catch', 'return', 'function', 'typeof', 'await', 'new', 'const', 'let', 'var', 'async', 'else', 'try', 'finally', 'class', 'case', 'of', 'in', 'throw']);
+
+// Tokenize source: strip line/block comments and string/template literals (→ ` STR `), lowercase,
+// then keep identifiers and a fixed set of structural operators, dropping keywords.
+export const tokenize = (src) => src
+  .replace(/\/\/[^\n]*/g, ' ')
+  .replace(/\/\*[\s\S]*?\*\//g, ' ')
+  // Linear-time string matcher: each char is consumed exactly one way (an escape pair XOR a
+  // non-backslash non-quote), so there is no backtracking ambiguity. The previous
+  // `(?:\\.|(?!\1).)*` let the engine re-partition backslash runs and went EXPONENTIAL on
+  // unterminated-quote content — a lone apostrophe in a big real-world body hung the whole
+  // overlap stage (found by the TypeScript-src scale test).
+  .replace(/(['"`])(?:\\[\s\S]|(?!\1)[^\\])*\1/g, ' STR ')
+  .toLowerCase()
+  .match(/[a-z_$][\w$]*|[{}();=><!+\-*/%]/g)?.filter((t) => !KW.has(t)) || [];
+
+// K-gram shingle set of the tokenized source (default K=3 — must match overlap.mjs's K).
+export const shingles = (src, k = 3) => {
+  const t = tokenize(src);
+  const s = new Set();
+  for (let i = 0; i + k <= t.length; i++) s.add(t.slice(i, i + k).join(' '));
+  return s;
+};
+
+// Jaccard similarity of two shingle sets. Empty either side → 0.
+export const jaccard = (a, b) => {
+  if (!a.size || !b.size) return 0;
+  let i = 0;
+  for (const x of a) if (b.has(x)) i++;
+  return i / (a.size + b.size - i);
+};

package/scripts/lib/skeleton.mjs

@@ -0,0 +1,41 @@
+// codeweb structural-skeleton primitives (F6) — normalize a function body to a token stream where
+// every IDENTIFIER collapses to a single `ID` placeholder, every number to `NUM`, every string to
+// `STR`, while KEYWORDS and OPERATORS are preserved verbatim. Two functions that are the same up to
+// variable renaming therefore have identical skeletons (Type-2 clones) — which the lexical shingler
+// (lib/shingles.mjs, which KEEPS identifiers) cannot see. Pure, deterministic, zero-dependency.
+//
+// Intent locks (tests): rename-invariance (single placeholder, not positional ID0/ID1), literal
+// normalization, but operator + keyword SENSITIVITY (`+`≠`*`, `&&`≠`||`, `if`≠`while`) so it does not
+// over-collapse and call genuinely different logic a clone. Reuses jaccard from lib/shingles.mjs.
+
+const JS_KW = new Set(['if', 'else', 'for', 'while', 'do', 'switch', 'case', 'default', 'break', 'continue', 'return', 'function', 'class', 'new', 'typeof', 'instanceof', 'const', 'let', 'var', 'of', 'in', 'try', 'catch', 'finally', 'throw', 'await', 'async', 'yield', 'extends', 'super', 'this', 'null', 'true', 'false', 'void', 'delete', 'import', 'export', 'from', 'static', 'get', 'set']);
+const PY_KW = new Set(['if', 'elif', 'else', 'for', 'while', 'def', 'class', 'return', 'try', 'except', 'finally', 'raise', 'with', 'as', 'import', 'from', 'and', 'or', 'not', 'in', 'is', 'lambda', 'pass', 'yield', 'await', 'async', 'None', 'True', 'False', 'global', 'nonlocal', 'del', 'assert']);
+
+// One pass, priority-ordered: string/template literals, numbers, identifiers, multi-char operators,
+// then single chars. Comments are stripped first so their contents don't tokenize.
+const TOKEN_RE = /`(?:\\.|[^`\\])*`|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\d+\.?\d*|[A-Za-z_$][\w$]*|===|!==|>>>|<<=|>>=|\*\*|\.\.\.|&&|\|\||\?\?|\?\.|=>|==|!=|<=|>=|\+\+|--|\+=|-=|\*=|\/=|&&|[-+*/%=<>!&|^~?:;,.(){}\[\]]/g;
+
+export function skeletonTokens(src, lang = 'js') {
+  const kw = lang === 'py' ? PY_KW : JS_KW;
+  const stripped = lang === 'py'
+    ? (src || '').replace(/#[^\n]*/g, ' ')
+    : (src || '').replace(/\/\/[^\n]*/g, ' ').replace(/\/\*[\s\S]*?\*\//g, ' ');
+  const out = [];
+  for (const tok of stripped.match(TOKEN_RE) || []) {
+    const c = tok[0];
+    if (c === '`' || c === '"' || c === "'") out.push('STR');
+    else if (c >= '0' && c <= '9') out.push('NUM');
+    else if (/[A-Za-z_$]/.test(c)) out.push(kw.has(tok) ? tok : 'ID');
+    else out.push(tok);
+  }
+  return out;
+}
+
+export const skeleton = (src, lang = 'js') => skeletonTokens(src, lang).join(' ');
+
+export const structuralShingles = (src, k = 3, lang = 'js') => {
+  const t = skeletonTokens(src, lang);
+  const set = new Set();
+  for (let i = 0; i + k <= t.length; i++) set.add(t.slice(i, i + k).join(' '));
+  return set;
+};

package/scripts/lib/stats.mjs

@@ -0,0 +1,92 @@
+// stats — the local outcome ledger: what codeweb actually DID during real work, counted where
+// it happened. This is the value receipt ("3 regressions blocked before landing, 14 pre-edit
+// cards, 87 queries served") — evidence from real use with a real denominator, not a lab.
+//
+// STRICTLY LOCAL: written beside the graph (<workspace>/stats.json), never transmitted,
+// nothing identifying — counter names and integers only. CODEWEB_NO_STATS=1 disables all
+// writes. Every operation is fail-open (a ledger must never break the tool it measures);
+// concurrent writers are last-write-wins (counters may undercount under races — acceptable
+// for a receipt, documented here).
+
+import { readFileSync, writeFileSync, unlinkSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+
+const statsPathOf = (graphPath) => join(dirname(graphPath), 'stats.json');
+const monthNow = () => new Date().toISOString().slice(0, 7);
+
+/** Read the ledger beside a graph.json, or null. Never throws. */
+export function readStats(graphPath) {
+  try { return JSON.parse(readFileSync(statsPathOf(graphPath), 'utf8')); } catch { return null; }
+}
+
+/** Increment a counter in the current month's bucket. Silent no-op on opt-out or any error. */
+export function bump(graphPath, counter, n = 1) {
+  if (process.env.CODEWEB_NO_STATS === '1' || !graphPath || !counter) return;
+  try {
+    const s = readStats(graphPath) || { since: monthNow(), months: {} };
+    const m = monthNow();
+    if (!s.months[m]) s.months[m] = {};
+    s.months[m][counter] = (s.months[m][counter] || 0) + n;
+    writeFileSync(statsPathOf(graphPath), JSON.stringify(s));
+  } catch { /* a receipt must never break the tool */ }
+}
+
+const LABELS = [
+  ['briefInjected', (v) => `${v} session brief(s)`],
+  ['cardsDelivered', (v) => `${v} pre-edit card(s)`],
+  ['cardCallersFollowed', (v) => `${v} card-named caller(s) followed`],
+  ['postEditChecks', (v) => `${v} post-edit check(s)`],
+  ['regressionsFlagged', (v) => `${v} regression(s) flagged before landing`],
+  ['queriesServed', (v) => `${v} quer${v === 1 ? 'y' : 'ies'} served`],
+  ['autoRefreshes', (v) => `${v} auto-refresh(es)`],
+];
+
+// ---- pending card (docs/specs/card-correlation.md) ---------------------------------------------
+// The pre-edit hook records which caller files the card WARNED about; when a later edit touches
+// one, the ledger records that the advice steered real work. One pending card at a time (a new
+// card replaces it), 30-minute expiry, each file counts once. Same rules as the ledger it feeds:
+// local-only, fail-open, CODEWEB_NO_STATS=1 disables.
+const PENDING_TTL_MS = 30 * 60 * 1000;
+const pendingPathOf = (graphPath) => join(dirname(graphPath), 'pending-card.json');
+
+/** Record the card's named caller files as the active pending set (replaces any previous). */
+export function recordPendingCard(graphPath, symbol, files) {
+  if (process.env.CODEWEB_NO_STATS === '1' || !graphPath || !files || !files.length) return;
+  try { writeFileSync(pendingPathOf(graphPath), JSON.stringify({ t: Date.now(), symbol, files })); }
+  catch { /* receipt only */ }
+}
+
+/**
+ * An edit landed on `editedRel` — if the pending card named it (and the card is fresh, and the
+ * edit is not the card's own subject file), count it once and consume it. Never throws.
+ */
+export function correlateEdit(graphPath, editedRel) {
+  if (process.env.CODEWEB_NO_STATS === '1' || !graphPath || !editedRel) return;
+  try {
+    const p = pendingPathOf(graphPath);
+    const card = JSON.parse(readFileSync(p, 'utf8'));
+    if (Date.now() - card.t > PENDING_TTL_MS) { try { unlinkSync(p); } catch {} return; }
+    const ix = (card.files || []).indexOf(editedRel);
+    if (ix === -1) return;
+    card.files.splice(ix, 1);
+    bump(graphPath, 'cardCallersFollowed');
+    if (card.files.length) writeFileSync(p, JSON.stringify(card));
+    else { try { unlinkSync(p); } catch {} }
+  } catch { /* no pending card / unreadable — fine */ }
+}
+
+/** One line for a month's counters, or null when the bucket is empty. */
+export function monthLine(counters) {
+  if (!counters) return null;
+  const parts = LABELS.filter(([k]) => counters[k] > 0).map(([k, f]) => f(counters[k]));
+  for (const k of Object.keys(counters)) if (!LABELS.some(([l]) => l === k) && counters[k] > 0) parts.push(`${counters[k]} ${k}`);
+  return parts.length ? parts.join(' · ') : null;
+}
+
+/** Attach the current month's activity to a brief object (mutates + returns it). */
+export function attachActivity(brief, graphPath) {
+  const s = readStats(graphPath);
+  const m = monthNow();
+  if (s?.months?.[m] && monthLine(s.months[m])) brief.activity = { month: m, counters: s.months[m] };
+  return brief;
+}