@piecemaker-legal/piecemaker 1.0.0

package/server/piecemaker/vendor/websocket-server/scripts/presidio-gliner/eval/sweep.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+"""Run each configuration ONCE at a very low threshold, storing every prediction with
+its score. The threshold can then be swept offline at zero extra inference cost.
+
+Configurations vary: checkpoint, label descriptions, chunk size / overlap.
+"""
+import json, os, re, sys, time
+import torch
+from gliner2 import GLiNER2
+
+sys.path.insert(0, "/Users/tsardet/Documents/GitHub/PieceMaker_Claude_CLI/websocket-server/scripts")
+from scan_utils import normalize_entity_text
+
+S = os.path.dirname(os.path.abspath(__file__))
+SLICES = {n: open(os.path.join(S, f"{n}.md"), encoding="utf-8").read()
+          for n in ("slice_A", "slice_B", "slice_C")}
+
+MAPPING = {"person": "PERSON", "company": "ORGANIZATION",
+           "organization": "ORGANIZATION", "location": "LOCATION"}
+
+# --- label description variants -------------------------------------------------
+DESC_BASELINE = {   # what the pipeline ships today
+    "person":       "Names of individual human beings or natural persons",
+    "company":      "Business entities, commercial companies, corporations of any country",
+    "organization": "Non-profit, governmental, or other institutional entities",
+    "location":     "Geographic locations, addresses, cities, or countries",
+}
+DESC_SHARP = {      # spells out the exclusions the baseline leaves implicit
+    "person": ("Full name of a specific individual human being, such as Bernard Gilly or "
+               "Mrs Laurence Rodriguez. Never a job title, never a role, never an acronym, "
+               "never a gene or a product name"),
+    "company": ("Name of a specific named commercial company, such as Novartis or Sofinnova "
+                "Partners SAS. Never a generic word like company, group or shareholders"),
+    "organization": ("Name of a specific named institution, agency or regulator, such as FDA, "
+                     "EMA or Inserm. Never a generic body such as board of directors, "
+                     "committee or working group"),
+    "location": ("Name of a specific geographic place: a country, a city, a region or a postal "
+                 "address. Never a nationality adjective such as French or European, never an "
+                 "anatomical part"),
+}
+
+PII_LOCAL = os.path.join(S, "pii_ckpt")   # patched copy, see pii_dl.log
+
+CONFIGS = [
+    ("baseline",        "fastino/gliner2-multi-v1", DESC_BASELINE, 250, 50),
+    ("sharp-desc",      "fastino/gliner2-multi-v1", DESC_SHARP,    250, 50),
+    ("chunk180",        "fastino/gliner2-multi-v1", DESC_BASELINE, 180, 20),
+    ("chunk120",        "fastino/gliner2-multi-v1", DESC_BASELINE, 120, 15),
+    ("sharp+chunk180",  "fastino/gliner2-multi-v1", DESC_SHARP,    180, 20),
+    ("pii-ckpt",        PII_LOCAL,                  DESC_BASELINE, 250, 50),
+    ("pii-ckpt+sharp",  PII_LOCAL,                  DESC_SHARP,    250, 50),
+]
+
+FLOOR = 0.05  # ask for everything; threshold is applied offline
+
+
+def chunk(text, max_words, overlap):
+    spans = [(m.start(), m.end()) for m in re.finditer(r"\S+", text)]
+    step = max(1, max_words - overlap)
+    out = []
+    for i in range(0, len(spans), step):
+        g = spans[i:i + max_words]
+        out.append({"text": text[g[0][0]:g[-1][1]], "off": g[0][0]})
+        if i + max_words >= len(spans):
+            break
+    return out
+
+
+def run(model, text, descs, cs, ov):
+    chunks = chunk(text, cs, ov)
+    preds = {}
+    for i in range(0, len(chunks), 8):
+        batch = [c["text"] for c in chunks[i:i + 8]]
+        res = model.batch_extract_entities(batch, descs, batch_size=8, threshold=FLOOR,
+                                           include_confidence=True, include_spans=True)
+        for c, r in zip(chunks[i:i + 8], res):
+            for label, matches in r.get("entities", {}).items():
+                etype = MAPPING.get(label, label.upper())
+                for m in matches:
+                    key = (etype, normalize_entity_text(m["text"]))
+                    if not key[1]:
+                        continue
+                    preds[key] = max(preds.get(key, 0.0), m["confidence"])
+    return [{"type": t, "text": x, "score": s} for (t, x), s in preds.items()]
+
+
+def main():
+    torch.set_num_threads(6)
+    out = {}
+    loaded, model = None, None
+    for name, ckpt, descs, cs, ov in CONFIGS:
+        if ckpt != loaded:
+            print(f"[load] {ckpt}", flush=True)
+            try:
+                model = GLiNER2.from_pretrained(ckpt)
+            except Exception as exc:
+                print(f"  !! {name} ignore: {type(exc).__name__}: {exc}", flush=True)
+                loaded = None
+                continue
+            loaded = ckpt
+        out[name] = {"checkpoint": ckpt, "chunk": [cs, ov], "slices": {}}
+        for sname, text in SLICES.items():
+            t = time.perf_counter()
+            out[name]["slices"][sname] = run(model, text, descs, cs, ov)
+            print(f"  {name}/{sname}: {len(out[name]['slices'][sname])} entites "
+                  f"en {time.perf_counter()-t:.1f}s", flush=True)
+        with open(os.path.join(S, "sweep_results.json"), "w", encoding="utf-8") as fh:
+            json.dump(out, fh, ensure_ascii=False, indent=1)
+    with open(os.path.join(S, "sweep_results.json"), "w", encoding="utf-8") as fh:
+        json.dump(out, fh, ensure_ascii=False, indent=1)
+    print("WROTE sweep_results.json", flush=True)
+
+
+if __name__ == "__main__":
+    main()

package/server/piecemaker/vendor/websocket-server/scripts/presidio-gliner/eval/verify_substitution.cjs

@@ -0,0 +1,116 @@
+#!/usr/bin/env node
+/**
+ * Check the substitution rules against the criterion the plan set for them:
+ * "on GENSIGHT, zero substitution inside a word — exactly verifiable".
+ *
+ * Runs the REAL helpers exported by anonymization-server.cjs over a real document and a
+ * real sensitive_map, and counts how many replacements land inside another word. The old
+ * rule (`new RegExp(escapeRegex(x), 'gi')`, arbitrary order) is replayed on the same input
+ * so the fix is quantified rather than asserted.
+ *
+ *   node verify_substitution.cjs <document.md> <document_sensitive_map.json>
+ */
+const fs = require('fs');
+const path = require('path');
+
+const MOD = path.resolve(__dirname, '../../../lib/anonymization-server.cjs');
+const { buildEntityRegex, byDescendingEntityLength } = require(MOD);
+
+function escapeRegex(s) {
+    return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+const WORD = /[\p{L}\p{N}_]/u;
+
+/** A match is "inside a word" when a word character touches either edge in the source. */
+function countInWord(text, regex) {
+    let total = 0;
+    let inWord = 0;
+    const samples = [];
+    const re = new RegExp(regex.source, regex.flags.includes('g') ? regex.flags : regex.flags + 'g');
+    let m;
+    while ((m = re.exec(text)) !== null) {
+        if (m[0].length === 0) { re.lastIndex++; continue; }
+        total++;
+        const before = m.index > 0 ? text[m.index - 1] : '';
+        const after = text[m.index + m[0].length] || '';
+        if ((before && WORD.test(before)) || (after && WORD.test(after))) {
+            inWord++;
+            if (samples.length < 6) {
+                samples.push(text.slice(Math.max(0, m.index - 18), m.index + m[0].length + 18)
+                    .replace(/\s+/g, ' '));
+            }
+        }
+    }
+    return { total, inWord, samples };
+}
+
+function main() {
+    const [docPath, mapPath] = process.argv.slice(2);
+    const text = fs.readFileSync(docPath, 'utf8');
+    const payload = JSON.parse(fs.readFileSync(mapPath, 'utf8'));
+
+    // The map is keyed by entity type, each holding occurrences; substitution works from
+    // the distinct entity strings.
+    const entities = new Set();
+    for (const items of Object.values(payload.entities)) {
+        for (const e of items) if (e.text) entities.add(e.text);
+    }
+    const list = [...entities];
+    console.log(`document : ${path.basename(docPath)} — ${text.split(/\s+/).length} mots`);
+    console.log(`entités distinctes dans le sensitive_map : ${list.length}\n`);
+
+    // --- ancienne règle -----------------------------------------------------------
+    let oldTotal = 0, oldInWord = 0;
+    const worst = [];
+    for (const ent of list) {
+        const r = countInWord(text, new RegExp(escapeRegex(ent), 'gi'));
+        oldTotal += r.total;
+        oldInWord += r.inWord;
+        if (r.inWord > 0) worst.push({ ent, inWord: r.inWord, sample: r.samples[0] });
+    }
+
+    // --- règle actuelle -----------------------------------------------------------
+    let newTotal = 0, newInWord = 0, skipped = 0;
+    const offenders = [];
+    const neverMatched = [];   // détectée mais jamais substituée = fuite de PII réelle
+    const skippedList = [];
+    for (const ent of [...list].sort(byDescendingEntityLength(x => x))) {
+        const rx = buildEntityRegex(ent);
+        if (!rx) { skipped++; skippedList.push(ent); continue; }
+        const r = countInWord(text, rx);
+        newTotal += r.total;
+        newInWord += r.inWord;
+        if (r.total === 0) neverMatched.push(ent);
+        if (r.inWord > 0) offenders.push({ ent, inWord: r.inWord, samples: r.samples });
+    }
+
+    console.log(`ancienne règle  : ${oldTotal} substitutions, dont ${oldInWord} à l'intérieur d'un mot`);
+    console.log(`règle actuelle  : ${newTotal} substitutions, dont ${newInWord} à l'intérieur d'un mot`);
+    console.log(`entités refusées par la règle actuelle (trop ambiguës) : ${skipped}\n`);
+
+    worst.sort((a, b) => b.inWord - a.inWord);
+    console.log('pires entités sous l\'ancienne règle :');
+    for (const w of worst.slice(0, 8)) {
+        console.log(`  ${JSON.stringify(w.ent).padEnd(28)} ${String(w.inWord).padStart(5)} dans un mot   ex: …${w.sample}…`);
+    }
+
+    // Une entité détectée que la substitution ne retrouve jamais reste en clair dans le
+    // document remis au tiers : c'est la seule vraie fuite de PII possible ici.
+    console.log(`\nentités jamais retrouvées dans le texte (fuite de PII) : ${neverMatched.length}`);
+    for (const e of neverMatched.slice(0, 10)) console.log(`   ${JSON.stringify(e)}`);
+
+    console.log(`\nentités refusées comme trop ambiguës (${skipped}) :`);
+    console.log('   ' + skippedList.slice(0, 25).map(e => JSON.stringify(e)).join(' '));
+
+    console.log(`\nCRITERE «0 substitution à l'intérieur d'un mot» : ${newInWord === 0 ? 'TENU' : 'ECHOUE'}`);
+    if (newInWord !== 0) {
+        for (const o of offenders.slice(0, 10)) {
+            console.log(`  ${JSON.stringify(o.ent)} → ${o.inWord}`);
+            for (const s of o.samples.slice(0, 3)) console.log(`      …${s}…`);
+        }
+        process.exitCode = 1;
+    }
+}
+
+main();

package/server/piecemaker/vendor/websocket-server/scripts/presidio-gliner/eval/verify_whitespace.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+"""Check the whitespace normalisation against the criterion the plan set for it:
+"zero entity containing \\n or a double space in the sensitive_map", and the 1273 ->
+971 collapse of falsely-distinct entities.
+
+An entity carrying a hard line break was, before the fix, escaped literally into the
+substitution regex — so it matched only where that exact run of whitespace occurred, i.e.
+almost nowhere. Those entities were detected and then never redacted: the one real PII
+leak the audit found (277 entities on GENSIGHT).
+
+    python3 verify_whitespace.py <document_sensitive_map.json> [document.md]
+"""
+import json
+import re
+import sys
+import unicodedata
+
+NBSP = "   "
+ZERO_WIDTH = "​‌‍"
+
+
+def main():
+    payload = json.load(open(sys.argv[1], encoding="utf-8"))
+    ents = [(etype, e["text"]) for etype, items in payload["entities"].items() for e in items]
+
+    bad_nl = [(t, x) for t, x in ents if "\n" in x]
+    bad_dbl = [(t, x) for t, x in ents if re.search(r"[ \t]{2,}", x)]
+    bad_nbsp = [(t, x) for t, x in ents if any(c in x for c in NBSP)]
+    bad_zw = [(t, x) for t, x in ents if any(c in x for c in ZERO_WIDTH)]
+
+    print(f"occurrences dans le sensitive_map : {len(ents)}")
+    print(f"entités distinctes (texte)        : {len({x for _, x in ents})}\n")
+
+    print(f"contenant un saut de ligne \\n     : {len(bad_nl)}")
+    print(f"contenant un double espace        : {len(bad_dbl)}")
+    print(f"contenant une espace insécable    : {len(bad_nbsp)}")
+    print(f"contenant une espace de largeur 0 : {len(bad_zw)}")
+
+    for label, items in (("\\n", bad_nl), ("double espace", bad_dbl),
+                         ("insécable", bad_nbsp), ("largeur nulle", bad_zw)):
+        for t, x in items[:5]:
+            print(f"   [{label}] {t}: {x!r}")
+
+    # False distinctness: how many entities would be extra without normalisation.
+    raw = {x for _, x in ents}
+    norm = {re.sub(r"\s+", " ", unicodedata.normalize("NFKC", x)).strip() for x in raw}
+    print(f"\nentités distinctes brutes {len(raw)} -> normalisées {len(norm)} "
+          f"({len(raw) - len(norm)} faux distincts restants)")
+
+    total_bad = len(bad_nl) + len(bad_dbl) + len(bad_nbsp) + len(bad_zw)
+    print(f"\nCRITERE «0 entité avec \\n ou double espace» : {'TENU' if total_bad == 0 else 'ECHOUE'}")
+
+    if len(sys.argv) > 2:
+        text = open(sys.argv[2], encoding="utf-8").read()
+        # An entity that never occurs verbatim in the document would never be substituted
+        # by a literal regex — the leak the normalisation is meant to close.
+        missing = [x for x in raw if x and x not in text]
+        print(f"\nentités absentes du document tel quel (donc jamais substituées "
+              f"par une regex littérale) : {len(missing)}")
+        for x in missing[:8]:
+            print(f"   {x!r}")
+
+    sys.exit(0 if total_bad == 0 else 1)
+
+
+if __name__ == "__main__":
+    main()

package/server/piecemaker/vendor/websocket-server/scripts/presidio-gliner/eval/verify_worker.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+"""Drive the real scanner_worker.py through its stdin/stdout protocol, once with CoreML and
+once without, and diff the two sensitive_map.json files.
+
+The benchmarks measured the encoder inside a hand-built harness. This measures the thing
+that actually ships: the worker process, its presidio pipeline, its pattern recognizers, its
+span arbitration and its JSON output. If the CoreML path changed anything the user would
+see, the diff shows it here and nowhere else.
+
+    python3 verify_worker.py <fichier.md>
+"""
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+
+HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+WORKER = os.path.join(HERE, "scanner_worker.py")
+
+
+def run(md_file, out_dir, coreml: bool):
+    env = dict(os.environ, PIECEMAKER_COREML="1" if coreml else "0")
+    p = subprocess.Popen([sys.executable, WORKER], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+                         stderr=subprocess.PIPE, text=True, env=env, bufsize=1)
+
+    # The worker logs one line per detected entity. Reading stderr only at the end fills
+    # the 64 KB pipe on any large document and the worker blocks forever inside its own
+    # print — GENSIGHT deadlocked here for 25 minutes. Drain it continuously instead.
+    err_lines = []
+
+    def drain():
+        for line in p.stderr:
+            err_lines.append(line)
+
+    err_thread = threading.Thread(target=drain, daemon=True)
+    err_thread.start()
+
+    # The worker prints READY on stdout once the models are loaded.
+    t0 = time.perf_counter()
+    while True:
+        line = p.stdout.readline()
+        if not line:
+            raise RuntimeError("worker mort avant READY:\n" + "".join(err_lines)[-2000:])
+        if line.strip() == "READY":
+            break
+    ready_s = time.perf_counter() - t0
+
+    t0 = time.perf_counter()
+    p.stdin.write(json.dumps({"cmd": "scan", "md_file": md_file, "output_dir": out_dir}) + "\n")
+    p.stdin.flush()
+    resp = json.loads(p.stdout.readline())
+    scan_s = time.perf_counter() - t0
+
+    p.stdin.write(json.dumps({"cmd": "quit"}) + "\n")
+    p.stdin.flush()
+    p.wait(timeout=60)
+    err_thread.join(timeout=10)
+    err = "".join(err_lines)
+
+    if resp.get("status") != "ok":
+        raise RuntimeError(f"scan echoue: {resp}\n{err[-2000:]}")
+
+    active = "CoreML GPU encoder active" in err
+    fallback = [l for l in err.splitlines() if "torch CPU" in l or "falling back" in l]
+    return resp["json_path"], ready_s, scan_s, active, fallback
+
+
+def entities(path):
+    """`entities` is keyed by entity type, each holding a list of occurrences."""
+    payload = json.load(open(path, encoding="utf-8"))
+    return {(etype, e["text"], e["start"], e["end"])
+            for etype, items in payload["entities"].items() for e in items}
+
+
+def main():
+    md = sys.argv[1]
+    print(f"document: {os.path.basename(md)}\n")
+    results = {}
+    for label, coreml in (("torch", False), ("coreml", True)):
+        with tempfile.TemporaryDirectory() as d:
+            path, ready_s, scan_s, active, notes = run(md, d, coreml)
+            ents = entities(path)
+            results[label] = ents
+            flag = "ACTIF" if active else "inactif"
+            print(f"{label:<7} demarrage {ready_s:>6.1f}s · scan {scan_s:>6.1f}s · "
+                  f"CoreML {flag} · {len(ents)} entites")
+            for n in notes:
+                print(f"         {n.strip()}")
+
+    a, b = results["torch"], results["coreml"]
+    print(f"\nentites torch {len(a)} · coreml {len(b)} · communes {len(a & b)}")
+    if a == b:
+        print("\nIDENTIQUE — sortie du worker strictement inchangee")
+    else:
+        print(f"\nDIFFERENCES ({len(a ^ b)}):")
+        for x in sorted(a - b)[:20]:
+            print(f"  seulement torch : {x}")
+        for x in sorted(b - a)[:20]:
+            print(f"  seulement coreml: {x}")
+
+
+if __name__ == "__main__":
+    main()

package/server/piecemaker/vendor/websocket-server/scripts/presidio-gliner/model_config.py

@@ -0,0 +1,28 @@
+"""Sélection et chargement du checkpoint GLiNER local de PieceMaker.
+
+GLiNER2.5 utilise une architecture ``boundary`` que l'ancienne classe
+``GLiNER2`` (architecture ``span``) ne sait pas charger. ``AutoExtractor``
+choisit la bonne implémentation. Le checkpoint historique n'est conservé ici
+que pour que l'installateur puisse détecter qu'une migration est nécessaire :
+il n'est jamais sélectionné à l'exécution.
+"""
+
+import os
+
+
+PREFERRED_GLINER_MODEL = "fastino/gliner2.5-multi-v1"
+LEGACY_GLINER_MODELS = ("fastino/gliner2-multi-v1",)
+
+
+def is_model_cached(model_id: str) -> bool:
+    """Retourne vrai si la configuration et les poids sont déjà locaux."""
+    try:
+        from huggingface_hub import try_to_load_from_cache
+
+        for filename in ("config.json", "model.safetensors"):
+            cached_path = try_to_load_from_cache(repo_id=model_id, filename=filename)
+            if not isinstance(cached_path, str) or not os.path.exists(cached_path):
+                return False
+        return True
+    except Exception:  # noqa: BLE001 - le cache ne doit jamais bloquer un scan
+        return False