@jinn-network/core 0.1.0-canary.3afae198

package/dist/scrub/gliner-detector.js

@@ -0,0 +1,129 @@
+/**
+ * GLiNER ONNX ML PII detector (#1973 / design §5.1, §6.3).
+ *
+ * Local-only inference behind the {@link PiiDetector} seam. Default model is
+ * `urchade/gliner_multi_pii-v1` (Apache-2.0, PII-tuned, multilingual). ONNX
+ * weights load from the community mirror `onnx-community/gliner_multi_pii-v1`
+ * (same weights; Transformers.js-compatible layout).
+ *
+ * Benchmark note (locked Q3): `urchade/gliner_multi_pii-v1` is the pinned
+ * default. The smaller `knowledgator/gliner-pii-edge-v1.0` family remains an
+ * alternate via `captures.piiDetection.model` — operators should measure
+ * Fβ=2 + latency on their local corpus before switching.
+ *
+ * Download-on-first-run is intentional (same posture as the retired
+ * bert-base-NER path). Init failure is fail-closed via
+ * {@link maybeBuildPiiDetector}.
+ */
+/**
+ * Logical model id operators configure. ONNX weights for the default pin live
+ * under the onnx-community mirror (see {@link resolveOnnxModelId}).
+ */
+export const DEFAULT_GLINER_MODEL = 'urchade/gliner_multi_pii-v1';
+/** Alternate smaller pin — set via config `captures.piiDetection.model`. */
+export const ALTERNATE_GLINER_EDGE_MODEL = 'knowledgator/gliner-pii-edge-v1.0';
+/**
+ * Zero-shot labels pinned in config (#1974 will hash these into the policy
+ * hash). Keep the list stable and reviewable — do not derive from model
+ * metadata at runtime.
+ */
+export const DEFAULT_GLINER_PII_LABELS = [
+    'person',
+    'phone number',
+    'email address',
+    'street address',
+    'credit card number',
+    'social security number',
+    'organization',
+    'username',
+    'ip address',
+    'date of birth',
+];
+/** Map a logical / source model id onto an ONNX-capable Hugging Face repo. */
+export function resolveOnnxModelId(modelId) {
+    if (modelId === DEFAULT_GLINER_MODEL || modelId === 'urchade/gliner_multi_pii-v1') {
+        return 'onnx-community/gliner_multi_pii-v1';
+    }
+    // knowledgator edge/base variants often ship ONNX under the same id or an
+    // onnx-community mirror; pass through and let fromPretrained fail closed.
+    return modelId;
+}
+/**
+ * Production GLiNER PII detector. Loads ONNX weights via `@lmoe/gliner-onnx`
+ * (dynamic import so unit tests never pull the runtime). Call {@link init}
+ * before {@link detect}; {@link maybeBuildPiiDetector} owns that lifecycle.
+ */
+export class GlinerPiiDetector {
+    runtime;
+    initPromise;
+    model;
+    labels;
+    threshold;
+    constructor(opts = {}) {
+        this.model = opts.model ?? DEFAULT_GLINER_MODEL;
+        this.labels = opts.labels ?? DEFAULT_GLINER_PII_LABELS;
+        this.threshold = opts.threshold ?? 0.5;
+    }
+    /** Logical model id (hashed into the policy digest — #1974). */
+    get modelId() {
+        return this.model;
+    }
+    /** Zero-shot labels (hashed into the policy digest — #1974). */
+    get labelSet() {
+        return this.labels;
+    }
+    /** Eagerly load the model (daemon boot / first publish). */
+    async init() {
+        await this.ensureReady();
+    }
+    ensureReady() {
+        if (this.runtime)
+            return Promise.resolve();
+        if (!this.initPromise) {
+            this.initPromise = (async () => {
+                let mod;
+                try {
+                    mod = (await import('@lmoe/gliner-onnx'));
+                }
+                catch (err) {
+                    const message = err instanceof Error ? err.message : String(err);
+                    throw new Error(`GLiNER ONNX runtime unavailable (install @lmoe/gliner-onnx / onnxruntime-node): ${message}`);
+                }
+                if (!mod.GLiNER1ONNXRuntime?.fromPretrained) {
+                    throw new Error('GLiNER ONNX runtime loaded but GLiNER1ONNXRuntime.fromPretrained is missing');
+                }
+                const onnxId = resolveOnnxModelId(this.model);
+                this.runtime = await mod.GLiNER1ONNXRuntime.fromPretrained(onnxId);
+            })();
+        }
+        return this.initPromise;
+    }
+    async detect(text) {
+        if (!text.trim())
+            return [];
+        await this.ensureReady();
+        const runtime = this.runtime;
+        const entities = await runtime.extractEntities(text, this.labels, {
+            threshold: this.threshold,
+        });
+        const out = [];
+        const seen = new Set();
+        for (const e of entities) {
+            const t = e.text?.trim();
+            if (!t)
+                continue;
+            const key = `${e.label}:${e.start}:${e.end}:${t}`;
+            if (seen.has(key))
+                continue;
+            seen.add(key);
+            out.push({
+                label: e.label,
+                text: t,
+                start: e.start,
+                end: e.end,
+                score: e.score,
+            });
+        }
+        return out;
+    }
+}

package/dist/scrub/index.d.ts

@@ -0,0 +1,26 @@
+export * from './apply-dispositions.js';
+export * from './build.js';
+export * from './checksummed-instruments-detector.js';
+export * from './emit-scrub.js';
+export * from './eval/index.js';
+export * from './finding.js';
+export * from './git-identity-detector.js';
+export * from './gitleaks-detector.js';
+export * from './ip-address-detector.js';
+export * from './key-policy.js';
+export * from './known-identity-detector.js';
+export * from './layer2.js';
+export * from './ml-pii-stage.js';
+export * from './gliner-detector.js';
+export * from './pii-build.js';
+export * from './pipeline.js';
+export * from './plain-patterns-stage.js';
+export * from './policy.js';
+export * from './provenance.js';
+export * from './reject-classes-detector.js';
+export * from './reject-publish-error.js';
+export * from './review-queue.js';
+export * from './secretlint-stage.js';
+export * from './transformers-detector.js';
+export * from './types.js';
+export * from './url-credentials-detector.js';

package/dist/scrub/index.js

@@ -0,0 +1,26 @@
+export * from './apply-dispositions.js';
+export * from './build.js';
+export * from './checksummed-instruments-detector.js';
+export * from './emit-scrub.js';
+export * from './eval/index.js';
+export * from './finding.js';
+export * from './git-identity-detector.js';
+export * from './gitleaks-detector.js';
+export * from './ip-address-detector.js';
+export * from './key-policy.js';
+export * from './known-identity-detector.js';
+export * from './layer2.js';
+export * from './ml-pii-stage.js';
+export * from './gliner-detector.js';
+export * from './pii-build.js';
+export * from './pipeline.js';
+export * from './plain-patterns-stage.js';
+export * from './policy.js';
+export * from './provenance.js';
+export * from './reject-classes-detector.js';
+export * from './reject-publish-error.js';
+export * from './review-queue.js';
+export * from './secretlint-stage.js';
+export * from './transformers-detector.js';
+export * from './types.js';
+export * from './url-credentials-detector.js';

package/dist/scrub/ip-address-detector.d.ts

@@ -0,0 +1,12 @@
+/**
+ * D2 — IPv4 addresses with range classification (#1972 / design §3.2 / §6.2).
+ *
+ * - public → VERY_HIGH (redact)
+ * - private (RFC1918 / link-local) → MEDIUM (flag)
+ * - loopback / unspecified / broadcast / documentation → no finding (pass)
+ */
+import { type KeyPolicy } from './key-policy.js';
+import type { Detector } from './finding.js';
+export type IpRangeClass = 'public' | 'private' | 'loopback' | 'reserved';
+export declare function classifyIpv4(ip: string): IpRangeClass;
+export declare function ipAddressDetector(policy: KeyPolicy): Detector;

package/dist/scrub/ip-address-detector.js

@@ -0,0 +1,81 @@
+/**
+ * D2 — IPv4 addresses with range classification (#1972 / design §3.2 / §6.2).
+ *
+ * - public → VERY_HIGH (redact)
+ * - private (RFC1918 / link-local) → MEDIUM (flag)
+ * - loopback / unspecified / broadcast / documentation → no finding (pass)
+ */
+import { classifyKey } from './key-policy.js';
+const VERSION = '1.0.0';
+const IPV4_PATTERN = /\b(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\b/g;
+export function classifyIpv4(ip) {
+    const parts = ip.split('.').map((p) => Number(p));
+    if (parts.length !== 4 || parts.some((n) => !Number.isInteger(n) || n < 0 || n > 255)) {
+        return 'reserved';
+    }
+    const [a, b] = parts;
+    // Loopback 127.0.0.0/8
+    if (a === 127)
+        return 'loopback';
+    // Unspecified
+    if (a === 0)
+        return 'reserved';
+    // Broadcast
+    if (a === 255 && b === 255 && parts[2] === 255 && parts[3] === 255)
+        return 'reserved';
+    // Current network / this host on this network historically
+    // Link-local 169.254.0.0/16
+    if (a === 169 && b === 254)
+        return 'reserved';
+    // Multicast 224.0.0.0/4
+    if (a >= 224 && a <= 239)
+        return 'reserved';
+    // Reserved for future / class E
+    if (a >= 240)
+        return 'reserved';
+    // Documentation (TEST-NET)
+    if (a === 192 && b === 0 && parts[2] === 2)
+        return 'reserved';
+    if (a === 198 && b === 51 && parts[2] === 100)
+        return 'reserved';
+    if (a === 203 && b === 0 && parts[2] === 113)
+        return 'reserved';
+    // RFC1918 private
+    if (a === 10)
+        return 'private';
+    if (a === 172 && b >= 16 && b <= 31)
+        return 'private';
+    if (a === 192 && b === 168)
+        return 'private';
+    return 'public';
+}
+export function ipAddressDetector(policy) {
+    const meta = { name: 'ip-address', version: VERSION };
+    return {
+        ...meta,
+        detect(attributes) {
+            const findings = [];
+            for (const [key, value] of Object.entries(attributes)) {
+                if (typeof value !== 'string' || classifyKey(key, policy) !== 'content')
+                    continue;
+                const re = new RegExp(IPV4_PATTERN.source, IPV4_PATTERN.flags);
+                let m;
+                while ((m = re.exec(value)) !== null) {
+                    const ip = m[0];
+                    const range = classifyIpv4(ip);
+                    if (range === 'loopback' || range === 'reserved')
+                        continue;
+                    const confidence = range === 'public' ? 'VERY_HIGH' : 'MEDIUM';
+                    findings.push({
+                        class: 'D2',
+                        span: { key, start: m.index, end: m.index + ip.length },
+                        confidence,
+                        evidence: [`ipv4-${range}`],
+                        detector: meta,
+                    });
+                }
+            }
+            return findings;
+        },
+    };
+}

package/dist/scrub/key-policy.d.ts

@@ -0,0 +1,30 @@
+import type { ScrubStage } from './types.js';
+import type { Detector } from './finding.js';
+export interface KeyPolicy {
+    /** keys whose values are structural and safe to publish raw */
+    safe: string[];
+    /** keys to delete entirely (never published) — A5 structural drop */
+    drop: string[];
+    /**
+     * Machine-identity keys (D3 carrier): attempt-manifest `host`, hostname
+     * telemetry, and similar. Deleted at the key level (redact disposition) —
+     * not reject-publish.
+     */
+    machineIdentity?: string[];
+}
+export type KeyClass = 'safe' | 'content' | 'drop' | 'machine-identity';
+/**
+ * Classify an attribute key against the policy. Precedence:
+ * `drop` > `machine-identity` > `safe` > `content`. Fail-safe: a key matching
+ * both drop and anything else is dropped.
+ */
+export declare function classifyKey(key: string, policy: KeyPolicy): KeyClass;
+/**
+ * Key-policy detector: emits A5 drop-key findings for `drop`-classified keys,
+ * and D3 machine-identity findings for hostname/carrier keys.
+ */
+export declare function keyPolicyDetector(policy: KeyPolicy): Detector;
+/**
+ * Legacy ScrubStage wrapper around {@link keyPolicyDetector}.
+ */
+export declare function keyPolicyStage(policy: KeyPolicy): ScrubStage;

package/dist/scrub/key-policy.js

@@ -0,0 +1,72 @@
+import { applyDispositions } from './apply-dispositions.js';
+const VERSION = '0.3.0';
+/** Exact match, or prefix match when the pattern ends with `*`. */
+function matches(key, pattern) {
+    if (pattern.endsWith('*'))
+        return key.startsWith(pattern.slice(0, -1));
+    return key === pattern;
+}
+/**
+ * Classify an attribute key against the policy. Precedence:
+ * `drop` > `machine-identity` > `safe` > `content`. Fail-safe: a key matching
+ * both drop and anything else is dropped.
+ */
+export function classifyKey(key, policy) {
+    if (policy.drop.some((p) => matches(key, p)))
+        return 'drop';
+    if ((policy.machineIdentity ?? []).some((p) => matches(key, p)))
+        return 'machine-identity';
+    if (policy.safe.some((p) => matches(key, p)))
+        return 'safe';
+    return 'content';
+}
+/**
+ * Key-policy detector: emits A5 drop-key findings for `drop`-classified keys,
+ * and D3 machine-identity findings for hostname/carrier keys.
+ */
+export function keyPolicyDetector(policy) {
+    const meta = { name: 'key-policy', version: VERSION };
+    return {
+        ...meta,
+        detect(attributes) {
+            const findings = [];
+            for (const key of Object.keys(attributes)) {
+                const cls = classifyKey(key, policy);
+                if (cls === 'drop') {
+                    findings.push({
+                        class: 'A5',
+                        span: { key, start: 0, end: 0 },
+                        confidence: 'VERY_HIGH',
+                        evidence: ['drop-key'],
+                        detector: meta,
+                    });
+                }
+                else if (cls === 'machine-identity') {
+                    findings.push({
+                        class: 'D3',
+                        span: { key, start: 0, end: 0 },
+                        confidence: 'VERY_HIGH',
+                        evidence: ['machine-identity-key'],
+                        detector: meta,
+                    });
+                }
+            }
+            return findings;
+        },
+    };
+}
+/**
+ * Legacy ScrubStage wrapper around {@link keyPolicyDetector}.
+ */
+export function keyPolicyStage(policy) {
+    const detector = keyPolicyDetector(policy);
+    return {
+        name: detector.name,
+        version: detector.version,
+        scrub(attributes) {
+            const findings = detector.detect(attributes);
+            const applied = applyDispositions(attributes, findings);
+            return { attributes: applied.attributes, redactions: applied.redactions };
+        },
+    };
+}

package/dist/scrub/known-identity-detector.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Known-identity pack + non-address instance allowlist (#1971 / design §6.4).
+ *
+ * Locked Q1: stub ALL address-shaped values (no address pass-allowlist). The
+ * pack exact-matches operator self-PII only. The allowlist covers loopback /
+ * reserved IPs and repo-slug patterns — never wallets.
+ */
+import { type KeyPolicy } from './key-policy.js';
+import type { Detector } from './finding.js';
+import type { ScrubStage } from './types.js';
+/** Operator self-PII assembled locally. No addresses (Q1). */
+export interface KnownIdentityPack {
+    gitUserName?: string;
+    gitUserEmail?: string;
+    homeUsername?: string;
+    hostname?: string;
+    ghLogin?: string;
+}
+export type InstanceAllowlistKind = 'loopback-ip' | 'reserved-ip' | 'repo-slug';
+export interface InstanceAllowlistEntry {
+    /** Exact value matched in content (case-insensitive for slugs/IPs). */
+    value: string;
+    kind: InstanceAllowlistKind;
+    /** Auditable provenance note recorded in the redaction manifest. */
+    provenance: string;
+}
+export interface InstanceAllowlist {
+    entries: InstanceAllowlistEntry[];
+}
+export interface AssembledKnownIdentity {
+    pack: KnownIdentityPack;
+    allowlist: InstanceAllowlist;
+}
+export interface AssembleKnownIdentityOptions {
+    /** Injected pack (tests / daemon). Wins over env/homedir fillers. */
+    pack?: KnownIdentityPack;
+    /** Extra allowlist entries merged onto the built-in non-address defaults. */
+    allowlist?: InstanceAllowlist;
+    /** Optional env (daemon). Only used to fill pack gaps. */
+    env?: NodeJS.ProcessEnv;
+    /** Optional homedir provider — basename becomes homeUsername when unset. */
+    homedir?: () => string;
+    /** Optional hostname provider. */
+    hostname?: () => string;
+}
+/** Built-in non-address public values (Q1): loopback + common reserved. */
+export declare const DEFAULT_INSTANCE_ALLOWLIST: InstanceAllowlist;
+export declare function isAddressShaped(value: string): boolean;
+/**
+ * Assemble pack + allowlist from injected options, optionally filling gaps from
+ * env/homedir. Injectable pack is enough for this PR; daemon wiring stays thin.
+ */
+export declare function assembleKnownIdentity(opts?: AssembleKnownIdentityOptions): AssembledKnownIdentity;
+export interface KnownIdentityDetectorOptions {
+    pack?: KnownIdentityPack;
+    allowlist?: InstanceAllowlist;
+    /** Pre-assembled identity; when set, pack/allowlist options are ignored. */
+    assembled?: AssembledKnownIdentity;
+}
+/**
+ * Detector: pack → VERY_HIGH redact findings; allowlist → pass findings with
+ * auditable evidence; public IPs → D2 redact; private IPs → D2 flag band.
+ */
+export declare function knownIdentityDetector(policy: KeyPolicy, opts?: KnownIdentityDetectorOptions): Detector;
+/** Legacy ScrubStage wrapper for stage-chain tests. */
+export declare function knownIdentityStage(policy: KeyPolicy, opts?: KnownIdentityDetectorOptions): ScrubStage;