claude-memory-lint 0.1.0

package/lib/detectors/pii.js

@@ -0,0 +1,511 @@
+'use strict';
+// PII detector: personal data and secrets that should never sit in a
+// long-lived agent memory file, checked against the ONE list this package
+// ships — `lib/pii-categories.json`.
+//
+// -- THE LIST LIVES IN ONE PLACE ---------------------------------------------
+//
+// This file declares no category. No PII pattern, no carve-out, no clinical
+// word appears here — all of it comes from `pii-categories.json`, compiled by
+// `compileCategories`. A category in two places becomes two categories that
+// drift, and the one that drifts is always the one the gate reads. If a
+// pattern is wrong, the fix is in the JSON, and the test breaks with it.
+//
+// -- THE THREE `gate` VALUES --------------------------------------------------
+//
+//   block    -> an occurrence exits 1 (nationalId, bankAccount, health, credential)
+//   warn     -> an occurrence is counted and reported, never exits 1 (minor,
+//               specialCategoryOther)
+//   doctrine -> not detectable by form; `patterns: []`, never enters code
+//               (falseAnonymity). The detector REFUSES a category that
+//               declares `doctrine` and still carries a pattern — that would
+//               be doctrine becoming a gate through the back door.
+//
+// -- MENTION IS NOT USE -------------------------------------------------------
+//
+// The category list's `globalExemptions` (backtick span, fenced block) are
+// NOT reimplemented here: they are exactly `maskCode` (lib/mask-code.js),
+// shared with the other detectors in this package. A note that teaches this
+// list needs to be able to show the phrase it catches without tripping the
+// gate on itself.
+//
+// -- DECLARED LOW RECALL ------------------------------------------------------
+//
+// `health` ships with `precision: closed-vocabulary` and a declared
+// `coverage: declared-low` right in the category record. It is the pattern
+// most likely to miss a real case: prose describing a condition in the
+// person's own words will not match a closed vocabulary, and tightening the
+// regex is not the fix — declaring the gap is. The test suite reflects this:
+// it proves what the list catches, and separately plants a case it does NOT
+// catch, instead of implying coverage that doesn't exist.
+//
+// -- WHAT THIS DETECTOR DOES NOT DO -------------------------------------------
+//
+// It does not fix or delete anything. A hit in someone's own memory archive
+// is a FINDING TO REPORT, not something this tool edits on its own — changing
+// someone else's recorded material is a decision for the person who owns it.
+//
+// Exit: 0 nothing blocking | 1 a `block`-gate category matched | 2 the
+// category list is unreadable (not checked — a detector with no list would
+// otherwise go green over an archive it never actually read)
+
+const fs = require('fs');
+const path = require('path');
+const { readCollection, INDEX_NAME } = require('../collector');
+const { locateArchive } = require('../locate-archive');
+const { maskCode } = require('../mask-code');
+
+const DEFAULT_CATEGORIES_FILE = path.join(__dirname, '..', 'pii-categories.json');
+const GATES = new Set(['block', 'warn', 'doctrine']);
+
+// Checksum validators a pattern can opt into via `"checksum": "<name>"` in
+// the JSON. This is generic math, not PII data — the LIST of which pattern
+// uses which checksum still lives entirely in pii-categories.json, per the
+// single-source-of-truth rule at the top of this file.
+const CHECKSUMS = {
+  // Standard Luhn (mod-10) check, used by every major card scheme. A random
+  // 13-19 digit platform ID (an ad account, a page ID) passes this by
+  // coincidence roughly 1 time in 10 — far better than the 10-in-10 false
+  // positive rate of "any run of 13-19 digits".
+  luhn(digits) {
+    let sum = 0;
+    let alt = false;
+    for (let i = digits.length - 1; i >= 0; i--) {
+      let d = digits.charCodeAt(i) - 48;
+      if (d < 0 || d > 9) return false;
+      if (alt) {
+        d *= 2;
+        if (d > 9) d -= 9;
+      }
+      sum += d;
+      alt = !alt;
+    }
+    return digits.length > 0 && sum % 10 === 0;
+  },
+};
+
+// --- pure: the list -> compiled categories ----------------------------------
+// Takes the ALREADY-PARSED object (disk reading and the CLI are the only
+// callers that touch fs). Throws with the reason if the list is broken — a
+// broken list must not silently become a green gate.
+function compileCategories(doc) {
+  if (!doc || !Array.isArray(doc.categories) || doc.categories.length === 0) {
+    throw new Error('category list has no non-empty `categories` array');
+  }
+  const compilePatterns = (list, catId) =>
+    (Array.isArray(list) ? list : []).map((p) => {
+      try {
+        if (p.checksum && !CHECKSUMS[p.checksum]) {
+          throw new Error(`unknown checksum "${p.checksum}"`);
+        }
+        return {
+          id: p.id,
+          description: p.description || null,
+          re: new RegExp(p.regex, 'i'),
+          checksum: p.checksum || null,
+        };
+      } catch (e) {
+        throw new Error(`category ${catId}: pattern "${p.id}" does not compile — ${e.message}`);
+      }
+    });
+  const compileExemptions = (list, catId) =>
+    (Array.isArray(list) ? list : []).map((e) => {
+      try {
+        return {
+          id: e.id,
+          description: e.description || null,
+          re: new RegExp(e.regex, 'i'),
+          // `appliesTo`: pattern ids this exemption is allowed to neutralize.
+          // Absent/empty means "every pattern in this category" (the old,
+          // coarser default) — set it to scope a carve-out meant for one
+          // pattern (e.g. "last four digits" only makes sense for a card
+          // number, never for a full IBAN in the same category).
+          appliesTo: Array.isArray(e.appliesTo) ? e.appliesTo : null,
+          // `nullifiedBy`: pattern ids that, if any of them ALSO matches in
+          // the same clause as the match this exemption would otherwise
+          // excuse, void the exemption. This is how an exemption whose
+          // own wording names the category ("special category", "what not
+          // to store") stops laundering a real disclosure sitting in the
+          // same clause — including when the disclosure IS the very pattern
+          // the exemption would apply to.
+          nullifiedBy: Array.isArray(e.nullifiedBy) ? e.nullifiedBy : null,
+        };
+      } catch (err) {
+        throw new Error(`category ${catId}: exemption "${e.id}" does not compile — ${err.message}`);
+      }
+    });
+  const globalExemptions = compileExemptions(doc.globalExemptions, '(global)');
+  const categories = doc.categories.map((c) => {
+    if (!c.id || !c.name) throw new Error(`category with no id/name: ${JSON.stringify(c).slice(0, 80)}`);
+    if (!GATES.has(c.gate)) throw new Error(`category ${c.id}: gate "${c.gate}" is not one of block|warn|doctrine`);
+    const patterns = Array.isArray(c.patterns) ? c.patterns : [];
+    if (c.gate === 'doctrine' && patterns.length) {
+      throw new Error(`category ${c.id}: gate "doctrine" carries ${patterns.length} pattern(s) — doctrine does not become a gate by form`);
+    }
+    return {
+      id: c.id,
+      name: c.name,
+      gate: c.gate,
+      precision: c.precision || null,
+      coverage: c.coverage || null,
+      // Why THIS category sits at THIS gate, so the report can say it
+      // instead of leaving a reader to guess why `minor` is `warn` and
+      // `health` is `block` when both are Art. 9 categories.
+      gateRationale: c.gateRationale || null,
+      carveOut: c.carveOut || null,
+      example: c.example || null,
+      // Whether pattern matching runs against the line AFTER `maskCode`
+      // blanks fenced/backtick spans (true, the old universal behavior), or
+      // against the raw line (false). "Mention is not use" is correct
+      // doctrine for prose categories a note might cite as an example of
+      // itself (health, minor, specialCategoryOther) — it is WRONG for a
+      // category whose match IS the leak: a credential or a national ID
+      // pasted inside a fenced log block is still a leaked credential or ID.
+      // Default true preserves prior behavior for every category that
+      // doesn't say otherwise.
+      maskCodeSpans: c.maskCodeSpans !== false,
+      patterns: compilePatterns(patterns, c.id),
+      exemptions: compileExemptions(c.exemptions, c.id),
+    };
+  });
+  return { categories, globalExemptions };
+}
+
+// Splits a line into clauses on `, ; : . ! ?` — the "same neighborhood"
+// the exemption check is fixed down to. A carve-out phrase and the pattern
+// it's supposed to excuse have to share a clause, not just a line: "Call
+// them back: national ID 120385-2399" is one LINE but two clauses, and the
+// phone-number carve-out in the first clause has no business excusing a
+// national ID number that appears after the colon.
+function splitClauses(line) {
+  const bounds = [0];
+  const seps = /[,;:.!?]/g;
+  let m;
+  while ((m = seps.exec(line))) bounds.push(m.index + 1);
+  bounds.push(line.length + 1);
+  const clauses = [];
+  for (let i = 0; i < bounds.length - 1; i++) clauses.push({ start: bounds[i], end: bounds[i + 1] - 1 });
+  return clauses;
+}
+
+function clauseIndexAt(clauses, pos) {
+  for (let i = 0; i < clauses.length; i++) {
+    if (pos >= clauses[i].start && pos < clauses[i].end) return i;
+  }
+  return clauses.length - 1;
+}
+
+// Does this exemption excuse THIS pattern match, in THIS line? Both halves of
+// that scoping live here: `appliesTo` scopes WHICH pattern a carve-out can
+// touch, and the clause check scopes WHERE in the line it has to appear.
+function exemptionCovers(exemption, patternId, line, clauses, matchClause) {
+  if (exemption.appliesTo && !exemption.appliesTo.includes(patternId)) return false;
+  const g = new RegExp(exemption.re.source, exemption.re.flags.includes('g') ? exemption.re.flags : `${exemption.re.flags}g`);
+  let m;
+  while ((m = g.exec(line))) {
+    if (clauseIndexAt(clauses, m.index) === matchClause) return true;
+    if (m.index === g.lastIndex) g.lastIndex += 1;
+  }
+  return false;
+}
+
+// --- pure: judge ONE line ----------------------------------------------------
+// `masked` is the line after `maskCode` (a backtick mention or fenced block
+// already turned to spaces); `raw` is the line as written. Which one a
+// category matches against is `c.maskCodeSpans` (see compileCategories) — a
+// credential or national ID pasted inside a fenced block is still a leak,
+// so those categories match against `raw`, not `masked`.
+//
+// A category's exemption no longer blankets the whole line for every
+// pattern: it has to (a) apply to the specific pattern that matched
+// (`appliesTo`) and (b) share a clause with that match (`exemptionCovers`).
+// "card ending in 6467" still passes — carve-out and match are one clause,
+// one pattern.
+function judgeLine(masked, raw, categories) {
+  const found = [];
+  for (const c of categories) {
+    if (!c.patterns.length) continue;
+    const line = c.maskCodeSpans ? masked : raw;
+    const clauses = splitClauses(line);
+
+    // First pass: where does each pattern first match, if at all? Needed
+    // twice over: to report the occurrence itself, and so a
+    // `nullifiedBy` exemption check can ask "did pattern X ALSO match in
+    // this clause" without re-running every pattern's regex per exemption.
+    const matchClauseByPattern = {};
+    for (const p of c.patterns) {
+      const idx = line.search(p.re);
+      if (idx === -1) continue;
+      if (p.checksum) {
+        const m = line.match(p.re);
+        const digits = m ? m[0].replace(/\D/g, '') : '';
+        if (!CHECKSUMS[p.checksum](digits)) continue;
+      }
+      matchClauseByPattern[p.id] = clauseIndexAt(clauses, idx);
+    }
+
+    for (const p of c.patterns) {
+      if (!(p.id in matchClauseByPattern)) continue;
+      const matchClause = matchClauseByPattern[p.id];
+      const exempt = c.exemptions.find((e) => {
+        if (!exemptionCovers(e, p.id, line, clauses, matchClause)) return false;
+        // An exemption whose text names the category ("special
+        // category", "what not to store") is voided when a listed pattern
+        // ALSO matches in this same clause — including p itself, so a
+        // clinical word can't excuse its own disclosure.
+        if (e.nullifiedBy && e.nullifiedBy.some((otherId) => matchClauseByPattern[otherId] === matchClause)) {
+          return false;
+        }
+        return true;
+      });
+      found.push({
+        category: c.id,
+        name: c.name,
+        gate: c.gate,
+        pattern: p.id,
+        exempt: Boolean(exempt),
+        exemption: exempt ? exempt.id : null,
+        excerpt: String(raw).trim().slice(0, 160),
+      });
+    }
+  }
+  // An exempt hit stays exempt: it doesn't count as an occurrence, only shows
+  // up in the control report — proof that the carve-out test measures a real
+  // exemption, not just absence of detection.
+  return found;
+}
+
+// --- pure: judge ONE text (a file, or one section of it) --------------------
+function judgeText(piece, categories) {
+  const text = String(piece.text || '');
+  const rawLines = text.split('\n');
+  const maskedLines = maskCode(text).split('\n');
+  const occurrences = [];
+  const exempted = [];
+  for (let i = 0; i < rawLines.length; i++) {
+    for (const f of judgeLine(maskedLines[i], rawLines[i], categories)) {
+      const rec = { ...f, line: i + 1 };
+      if (f.exempt) exempted.push(rec);
+      else occurrences.push(rec);
+    }
+  }
+  return { name: piece.name, section: piece.section || 'file', occurrences, exempted };
+}
+
+function auditPii(pieces, categories) {
+  return pieces.map((p) => judgeText(p, categories));
+}
+
+// --- pure: summary -----------------------------------------------------------
+function summarizePii(verdicts, categories) {
+  const byCategory = {};
+  for (const c of categories) {
+    byCategory[c.id] = {
+      gate: c.gate,
+      precision: c.precision,
+      gateRationale: c.gateRationale || null,
+      // `doctrine` never runs a pattern — see the CLI reporting below,
+      // which prints this instead of a count that would read as "checked,
+      // found nothing."
+      machineCheckable: c.gate !== 'doctrine',
+      occurrences: 0,
+      exempted: 0,
+      files: [],
+    };
+  }
+  let blocking = 0;
+  let warning = 0;
+  const filesWithHits = new Set();
+  for (const v of verdicts) {
+    for (const o of v.occurrences) {
+      const target = byCategory[o.category] ||
+        (byCategory[o.category] = {
+          gate: o.gate,
+          precision: null,
+          gateRationale: null,
+          machineCheckable: o.gate !== 'doctrine',
+          occurrences: 0,
+          exempted: 0,
+          files: [],
+        });
+      target.occurrences += 1;
+      if (!target.files.includes(v.name)) target.files.push(v.name);
+      if (o.gate === 'block') blocking += 1;
+      else if (o.gate === 'warn') warning += 1;
+      filesWithHits.add(v.name);
+    }
+    for (const o of v.exempted) {
+      const target = byCategory[o.category];
+      if (target) target.exempted += 1;
+    }
+  }
+  return {
+    pieces: verdicts.length,
+    filesWithHits: filesWithHits.size,
+    occurrences: blocking + warning,
+    blocking,
+    warning,
+    byCategory,
+  };
+}
+
+// --- I/O: only from here down does this module touch disk -------------------
+
+// Turns a collected archive into judgeable pieces: frontmatter and body enter
+// as SEPARATE pieces, so a report's line number is the section's own.
+//
+// The index file (`MEMORY.md`) is out of scope for FILE COUNTS (`total`
+// below stays the note count, matching every other detector's `filesRead`)
+// but it is judged here like any other piece: one index LINE (title +
+// suffix note, e.g. "- [c](c.md) - the client has cancer") is exactly where
+// a person free-typing a one-line summary can paste the same personal data
+// they were careful to keep out of the note body. The index is also the
+// single file every boot reads in full, so a false negative here is the
+// worst possible place for one. Nothing about the line is stripped before
+// judging it — including the `(file.md)` link target — because that target
+// is a local filename, not free prose, and stripping it would be a second
+// unstated carve-out; if a real one is ever needed it belongs declared in
+// `_meta.coverageScope`, not silently trimmed here.
+function collectPieces(dir) {
+  const pieces = [];
+  const exists = fs.existsSync(dir);
+  let total = 0;
+  if (exists) {
+    const r = readCollection(dir);
+    total = r.files.length;
+    for (const f of r.files) {
+      if (f.frontmatter && f.frontmatter.raw) {
+        pieces.push({ name: f.name, section: 'frontmatter', text: f.frontmatter.raw });
+      }
+      pieces.push({ name: f.name, section: 'body', text: f.body });
+    }
+    for (const entry of r.indexEntries) {
+      pieces.push({ name: INDEX_NAME, section: 'index', text: entry.raw });
+    }
+  }
+  return { pieces, exists, total };
+}
+
+module.exports = {
+  DEFAULT_CATEGORIES_FILE,
+  GATES,
+  compileCategories,
+  judgeLine,
+  judgeText,
+  auditPii,
+  summarizePii,
+  collectPieces,
+};
+
+// --- CLI ---------------------------------------------------------------
+// Usage: node lib/detectors/pii.js [--json]
+// Exit: 0 nothing in a `block` category | 1 a `block` category matched |
+// 2 archive not found, or the category list is unreadable (not checked).
+if (require.main === module) {
+  const { loadConfig } = require('../config');
+  const json = process.argv.includes('--json');
+  const { config } = loadConfig();
+  const listPath =
+    process.env.MEMORY_LINT_PII_CATEGORIES || config.pii.categoriesFile || DEFAULT_CATEGORIES_FILE;
+
+  const notChecked = (state, msg, extra = {}) => {
+    if (json) console.log(JSON.stringify({ state, categoriesFile: listPath, summary: null, ...extra }, null, 2));
+    else console.log(msg);
+    console.error(msg);
+    process.exit(2);
+  };
+
+  let categories = null;
+  let listVersion = null;
+  let coverageScope = null;
+  try {
+    const doc = JSON.parse(fs.readFileSync(listPath, 'utf8'));
+    const compiled = compileCategories(doc);
+    categories = compiled.categories;
+    listVersion = doc.version || null;
+    // The classes this list does NOT scan have to be stated, not just
+    // absent. `languageScope` already declares the language limit; this is
+    // the same declaration for CLASS of data — printed on every run so
+    // "PII: nothing in a blocking category" never reads as "archive is
+    // clean of personal data" when whole categories were never checked.
+    coverageScope = (doc._meta && Array.isArray(doc._meta.coverageScope)) ? doc._meta.coverageScope : null;
+  } catch (e) {
+    notChecked(
+      'unreadable_list',
+      `PII: NOT CHECKED - category list ${listPath} is unreadable or invalid: ${e.message}. ` +
+        'There is nothing to detect without the list, and a detector with no list would go green over an archive it never read.'
+    );
+  }
+
+  const { dir, source } = locateArchive();
+  const { pieces, exists, total } = collectPieces(dir);
+
+  if (!exists) {
+    notChecked(
+      'archive_absent',
+      `PII: NOT CHECKED - ${dir} not found (path from ${source}). Absence is not a clean scan: run this where the archive exists.`
+    );
+  }
+
+  const verdicts = auditPii(pieces, categories);
+  const summary = summarizePii(verdicts, categories);
+  const hits = verdicts.filter((v) => v.occurrences.length);
+  // `ok` used to cover an archive with a `warn`-gate occurrence
+  // sitting in it (specialCategoryOther, minor, directIdentifier, ...) —
+  // exit code 0 either way, since only `block` trips exit 1, but the STATE
+  // STRING must not say "ok" over something a human still has to look at.
+  const state = summary.blocking > 0 ? 'blocked' : summary.warning > 0 ? 'ok_with_warnings' : 'ok';
+
+  if (json) {
+    console.log(
+      JSON.stringify(
+        {
+          state,
+          categoriesFile: listPath,
+          listVersion,
+          coverageScope,
+          dir,
+          source,
+          filesRead: total,
+          summary,
+          findings: hits.map((v) => ({ name: v.name, section: v.section, occurrences: v.occurrences })),
+        },
+        null,
+        2
+      )
+    );
+  } else {
+    console.log(`archive: ${dir}  (${source}, ${total} file(s))`);
+    console.log(`categories: ${listPath}`);
+    console.log(
+      `pieces judged: ${summary.pieces}  occurrences: ${summary.occurrences} (block ${summary.blocking} / warn ${summary.warning})`
+    );
+    for (const [id, d] of Object.entries(summary.byCategory)) {
+      // `doctrine` (falseAnonymity) never runs a pattern — saying
+      // "0 occurrence(s)" reads as "checked, found nothing" for the one
+      // category this tool structurally cannot check.
+      if (!d.machineCheckable) {
+        console.log(`  ${id} [${d.gate}]: not machine-checkable — needs a human pass`);
+      } else {
+        console.log(`  ${id} [${d.gate}]: ${d.occurrences} occurrence(s), ${d.exempted} exempt(ed) by carve-out`);
+      }
+      if (d.gateRationale) console.log(`      gate: ${d.gateRationale}`);
+    }
+    if (coverageScope && coverageScope.length) {
+      console.log(`\nNOT scanned by this list (declared limit, not a clean result for these): ${coverageScope.join('; ')}`);
+    }
+    if (hits.length) {
+      console.log('');
+      for (const v of hits) {
+        for (const o of v.occurrences) {
+          console.log(`  ${v.name} (${v.section}):${o.line}  [${o.category}/${o.pattern}]`);
+          console.log(`     ... ${o.excerpt} ...`);
+        }
+      }
+      console.log('\nThis is a FINDING to report, not something this tool edits on its own.');
+    } else {
+      console.log('\nPII: nothing in a blocking category.');
+    }
+  }
+  process.exit(summary.blocking > 0 ? 1 : 0);
+}