@jinn-network/core 0.1.0

package/dist/scrub/build.d.ts

@@ -0,0 +1,52 @@
+import { ScrubPipeline } from './pipeline.js';
+import { type KeyPolicy } from './key-policy.js';
+import { type PiiDetector } from './ml-pii-stage.js';
+/**
+ * Default key policy. `jinn.*` identity/chain attributes are structural and pass
+ * raw. The `drop` tier deletes the spec's stage-1 high-confidence keys outright
+ * (auth headers, cookies, env dumps — see `spec/2026-06-15-ts-trajectory-scrub-stack.md`):
+ * these never carry sellable content and are never safe to publish. Globs use the
+ * trailing-`*` prefix form `classifyKey` supports; the HTTP header keys are listed
+ * per request/response direction (a leading `*.header.…` glob is not matched, so
+ * we enumerate the concrete keys). Everything else is `content` and flows through
+ * the value-scrubbing stages.
+ */
+export declare const DEFAULT_KEY_POLICY: KeyPolicy;
+export interface BuildScrubPipelineOptions {
+    policy?: KeyPolicy;
+    /** When provided, the ML PII (GLiNER) stage is appended. */
+    piiDetector?: PiiDetector;
+}
+/**
+ * Assembles the seller-side scrub pipeline (cost-ascending): structural key
+ * policy → openredaction (structured PII) → secretlint + entropy (secrets) →
+ * optional GLiNER (ML PII). The GLiNER stage is added only when a detector is
+ * supplied, so the daemon degrades gracefully (regex + secrets still scrub) when
+ * the ML model is unavailable.
+ */
+export declare function buildScrubPipeline(opts?: BuildScrubPipelineOptions): ScrubPipeline;
+/**
+ * Seed-profile scrub pipeline (#1409). Seeds are public, transformed or
+ * otherwise human-curated content — not operator trace data — so the
+ * probabilistic stages (openredaction, secretlint's pass-2 entropy fallback,
+ * ML PII) that exist to catch unknown-shape PII/secrets in private traces are
+ * dropped: on prose they false-positive (trigger words, dated env-var slugs,
+ * long camelCase identifiers) and deface the corpus. The deterministic
+ * detectors stay:
+ * structural key policy, plain-patterns (emails, home paths, and — seed-only,
+ * #1415 — bare AWS access-key IDs and GCP `AIza…` API keys, deterministic
+ * prefix shapes secretlint pass-1 does not cover), and secretlint's pass-1
+ * preset rules (AWS secret-key assignments, GitHub / Slack / npm token
+ * shapes, GCP service-account JSON). Accepted residual: JWTs and unprefixed
+ * high-entropy blobs (trace profile catches them via the entropy fallback),
+ * plus every structured identifier or PII class detected only by the omitted
+ * openredaction stage, pass unredacted. The latter is a 570+ pattern surface;
+ * payment cards, phone numbers, SSNs, medical or health-plan identifiers,
+ * government identity documents, and financial account references are
+ * examples, not an exhaustive allowlist. This is acceptable only for public
+ * seed material that a curator has transformed and reviewed for those classes.
+ * The reduced stage list is inspectable locally through the pipeline's
+ * `components` surface. `TraceEnvelopeV0` does not publish that list, so a
+ * fetched envelope cannot by itself prove which scrub profile ran.
+ */
+export declare function buildSeedScrubPipeline(policy?: KeyPolicy): ScrubPipeline;

package/dist/scrub/build.js

@@ -0,0 +1,84 @@
+import { ScrubPipeline } from './pipeline.js';
+import { keyPolicyStage } from './key-policy.js';
+import { openredactionStage } from './openredaction-stage.js';
+import { plainPatternsStage } from './plain-patterns-stage.js';
+import { secretlintStage } from './secretlint-stage.js';
+import { mlPiiStage } from './ml-pii-stage.js';
+/**
+ * Default key policy. `jinn.*` identity/chain attributes are structural and pass
+ * raw. The `drop` tier deletes the spec's stage-1 high-confidence keys outright
+ * (auth headers, cookies, env dumps — see `spec/2026-06-15-ts-trajectory-scrub-stack.md`):
+ * these never carry sellable content and are never safe to publish. Globs use the
+ * trailing-`*` prefix form `classifyKey` supports; the HTTP header keys are listed
+ * per request/response direction (a leading `*.header.…` glob is not matched, so
+ * we enumerate the concrete keys). Everything else is `content` and flows through
+ * the value-scrubbing stages.
+ */
+export const DEFAULT_KEY_POLICY = {
+    safe: ['jinn.*'],
+    drop: [
+        'http.request.header.authorization',
+        'http.response.header.authorization',
+        'http.request.header.cookie',
+        'http.response.header.cookie',
+        'http.request.header.set-cookie',
+        'http.response.header.set-cookie',
+        'env.*',
+    ],
+};
+/**
+ * Assembles the seller-side scrub pipeline (cost-ascending): structural key
+ * policy → openredaction (structured PII) → secretlint + entropy (secrets) →
+ * optional GLiNER (ML PII). The GLiNER stage is added only when a detector is
+ * supplied, so the daemon degrades gracefully (regex + secrets still scrub) when
+ * the ML model is unavailable.
+ */
+export function buildScrubPipeline(opts = {}) {
+    const policy = opts.policy ?? DEFAULT_KEY_POLICY;
+    return new ScrubPipeline(assembleScrubStages(policy, opts.piiDetector));
+}
+function assembleScrubStages(policy, piiDetector) {
+    // plain-patterns runs after openredaction: deterministic email/home-path
+    // regexes closing gaps the probabilistic stage demonstrably has (#1330).
+    const stages = [
+        keyPolicyStage(policy),
+        openredactionStage(policy),
+        plainPatternsStage(policy),
+        secretlintStage(policy),
+    ];
+    if (piiDetector) {
+        stages.push(mlPiiStage(policy, piiDetector));
+    }
+    return stages;
+}
+/**
+ * Seed-profile scrub pipeline (#1409). Seeds are public, transformed or
+ * otherwise human-curated content — not operator trace data — so the
+ * probabilistic stages (openredaction, secretlint's pass-2 entropy fallback,
+ * ML PII) that exist to catch unknown-shape PII/secrets in private traces are
+ * dropped: on prose they false-positive (trigger words, dated env-var slugs,
+ * long camelCase identifiers) and deface the corpus. The deterministic
+ * detectors stay:
+ * structural key policy, plain-patterns (emails, home paths, and — seed-only,
+ * #1415 — bare AWS access-key IDs and GCP `AIza…` API keys, deterministic
+ * prefix shapes secretlint pass-1 does not cover), and secretlint's pass-1
+ * preset rules (AWS secret-key assignments, GitHub / Slack / npm token
+ * shapes, GCP service-account JSON). Accepted residual: JWTs and unprefixed
+ * high-entropy blobs (trace profile catches them via the entropy fallback),
+ * plus every structured identifier or PII class detected only by the omitted
+ * openredaction stage, pass unredacted. The latter is a 570+ pattern surface;
+ * payment cards, phone numbers, SSNs, medical or health-plan identifiers,
+ * government identity documents, and financial account references are
+ * examples, not an exhaustive allowlist. This is acceptable only for public
+ * seed material that a curator has transformed and reviewed for those classes.
+ * The reduced stage list is inspectable locally through the pipeline's
+ * `components` surface. `TraceEnvelopeV0` does not publish that list, so a
+ * fetched envelope cannot by itself prove which scrub profile ran.
+ */
+export function buildSeedScrubPipeline(policy = DEFAULT_KEY_POLICY) {
+    return new ScrubPipeline([
+        keyPolicyStage(policy),
+        plainPatternsStage(policy, { credentialIds: true }),
+        secretlintStage(policy, { entropyFallback: false }),
+    ]);
+}

package/dist/scrub/emit-scrub.d.ts

@@ -0,0 +1,41 @@
+import type { RedactionManifest, Span } from '../trajectory/schema.js';
+import type { ScrubPipeline } from './pipeline.js';
+/**
+ * Seller-side scrub applied at emit time, before a trajectory is signed and
+ * published. Runs every span's attributes (and each event's attributes) through
+ * the pipeline, rebuilds the prev-span hash chain over the scrubbed spans, and
+ * produces the redaction manifest (one entry per span listing the keys touched;
+ * `event.<key>` for event-attribute redactions).
+ */
+export declare function scrubSpansForEmit(spans: Span[], pipeline: ScrubPipeline, taskCid: string): Promise<{
+    spans: Span[];
+    redactionManifest: RedactionManifest;
+}>;
+/**
+ * Minimal structural shape of a capture span (jinn.capture-trajectory.v1): a
+ * flat attribute bag plus the keys already redacted at ingest. Captures have no
+ * events array and no in-run hash chain, so they cannot go through
+ * {@link scrubSpansForEmit}; this is their dedicated seller-side scrub.
+ */
+export interface ScrubbableCaptureSpan {
+    attributes: Record<string, unknown>;
+    redactedKeys: string[];
+}
+/**
+ * Seller-side scrub for capture spans, applied at publish time before the
+ * capture trajectory is uploaded. Runs each span's attributes through the
+ * pipeline, replacing them with the scrubbed values, and unions the pipeline's
+ * redactions into the span's existing `redactedKeys`. Unlike
+ * {@link scrubSpansForEmit} it does NOT touch events or inject a hash chain —
+ * capture spans carry neither. Generic over the span type so the concrete
+ * `SpanRow` (with its extra columns) is returned unchanged apart from
+ * `attributes` and `redactedKeys`.
+ */
+export declare function scrubCaptureSpans<T extends ScrubbableCaptureSpan>(spans: T[], pipeline: ScrubPipeline): Promise<T[]>;
+/**
+ * Unions two redaction manifests by spanId (dedup redactedKeys per span). Used
+ * when an ingestion-time manifest (e.g. a capture's stored redactedKeys) is
+ * combined with the emit-time pipeline manifest. Keeps the
+ * `totalRedactions === sum(redactedKeys)` invariant the schema enforces.
+ */
+export declare function mergeRedactionManifests(a: RedactionManifest, b: RedactionManifest): RedactionManifest;

package/dist/scrub/emit-scrub.js

@@ -0,0 +1,87 @@
+import { computeGenesisHash, computePrevSpanHash, } from '../trajectory/hash-chain.js';
+/**
+ * Seller-side scrub applied at emit time, before a trajectory is signed and
+ * published. Runs every span's attributes (and each event's attributes) through
+ * the pipeline, rebuilds the prev-span hash chain over the scrubbed spans, and
+ * produces the redaction manifest (one entry per span listing the keys touched;
+ * `event.<key>` for event-attribute redactions).
+ */
+export async function scrubSpansForEmit(spans, pipeline, taskCid) {
+    const out = [];
+    const manifestSpans = [];
+    let total = 0;
+    let prev = computeGenesisHash(taskCid);
+    for (const span of spans) {
+        const redactedKeys = new Set();
+        const attrResult = await pipeline.run(span.attributes ?? {});
+        for (const r of attrResult.redactions)
+            redactedKeys.add(r.key);
+        const events = [];
+        for (const event of span.events) {
+            if (event.attributes && Object.keys(event.attributes).length > 0) {
+                const evResult = await pipeline.run(event.attributes);
+                events.push({ ...event, attributes: evResult.attributes });
+                for (const r of evResult.redactions)
+                    redactedKeys.add(`event.${r.key}`);
+            }
+            else {
+                events.push(event);
+            }
+        }
+        const scrubbed = {
+            ...span,
+            attributes: { ...attrResult.attributes, 'jinn.prevSpanHash': prev },
+            events,
+        };
+        out.push(scrubbed);
+        prev = computePrevSpanHash(scrubbed);
+        if (redactedKeys.size > 0) {
+            const keys = [...redactedKeys];
+            manifestSpans.push({ spanId: span.spanId, redactedKeys: keys });
+            total += keys.length;
+        }
+    }
+    return { spans: out, redactionManifest: { spans: manifestSpans, totalRedactions: total } };
+}
+/**
+ * Seller-side scrub for capture spans, applied at publish time before the
+ * capture trajectory is uploaded. Runs each span's attributes through the
+ * pipeline, replacing them with the scrubbed values, and unions the pipeline's
+ * redactions into the span's existing `redactedKeys`. Unlike
+ * {@link scrubSpansForEmit} it does NOT touch events or inject a hash chain —
+ * capture spans carry neither. Generic over the span type so the concrete
+ * `SpanRow` (with its extra columns) is returned unchanged apart from
+ * `attributes` and `redactedKeys`.
+ */
+export async function scrubCaptureSpans(spans, pipeline) {
+    const out = [];
+    for (const span of spans) {
+        const result = await pipeline.run(span.attributes ?? {});
+        const redactedKeys = new Set(span.redactedKeys);
+        for (const r of result.redactions)
+            redactedKeys.add(r.key);
+        out.push({ ...span, attributes: result.attributes, redactedKeys: [...redactedKeys] });
+    }
+    return out;
+}
+/**
+ * Unions two redaction manifests by spanId (dedup redactedKeys per span). Used
+ * when an ingestion-time manifest (e.g. a capture's stored redactedKeys) is
+ * combined with the emit-time pipeline manifest. Keeps the
+ * `totalRedactions === sum(redactedKeys)` invariant the schema enforces.
+ */
+export function mergeRedactionManifests(a, b) {
+    const bySpan = new Map();
+    for (const m of [a, b]) {
+        for (const s of m.spans) {
+            const set = bySpan.get(s.spanId) ?? new Set();
+            for (const k of s.redactedKeys)
+                set.add(k);
+            bySpan.set(s.spanId, set);
+        }
+    }
+    const spans = [...bySpan]
+        .map(([spanId, keys]) => ({ spanId, redactedKeys: [...keys] }))
+        .filter((s) => s.redactedKeys.length > 0);
+    return { spans, totalRedactions: spans.reduce((n, s) => n + s.redactedKeys.length, 0) };
+}

package/dist/scrub/index.d.ts

@@ -0,0 +1,12 @@
+export * from './build.js';
+export * from './emit-scrub.js';
+export * from './key-policy.js';
+export * from './layer2.js';
+export * from './ml-pii-stage.js';
+export * from './openredaction-stage.js';
+export * from './pii-build.js';
+export * from './pipeline.js';
+export * from './plain-patterns-stage.js';
+export * from './secretlint-stage.js';
+export * from './transformers-detector.js';
+export * from './types.js';

package/dist/scrub/index.js

@@ -0,0 +1,12 @@
+export * from './build.js';
+export * from './emit-scrub.js';
+export * from './key-policy.js';
+export * from './layer2.js';
+export * from './ml-pii-stage.js';
+export * from './openredaction-stage.js';
+export * from './pii-build.js';
+export * from './pipeline.js';
+export * from './plain-patterns-stage.js';
+export * from './secretlint-stage.js';
+export * from './transformers-detector.js';
+export * from './types.js';

package/dist/scrub/key-policy.d.ts

@@ -0,0 +1,20 @@
+import type { ScrubStage } from './types.js';
+export interface KeyPolicy {
+    /** keys whose values are structural and safe to publish raw */
+    safe: string[];
+    /** keys to delete entirely (never published) */
+    drop: string[];
+}
+export type KeyClass = 'safe' | 'content' | 'drop';
+/**
+ * Classify an attribute key against the policy. `drop` wins over `safe` (a key
+ * matching both is dropped — fail safe). Unmatched keys are `content` and flow
+ * to the value-scrubbing stages.
+ */
+export declare function classifyKey(key: string, policy: KeyPolicy): KeyClass;
+/**
+ * Structural key-policy stage: deletes `drop` keys outright (recording a
+ * redaction) and passes `safe` + `content` keys through unchanged. Value-level
+ * scrubbing of `content` keys is handled by later stages.
+ */
+export declare function keyPolicyStage(policy: KeyPolicy): ScrubStage;

package/dist/scrub/key-policy.js

@@ -0,0 +1,42 @@
+const VERSION = '0.1.0';
+/** Exact match, or prefix match when the pattern ends with `*`. */
+function matches(key, pattern) {
+    if (pattern.endsWith('*'))
+        return key.startsWith(pattern.slice(0, -1));
+    return key === pattern;
+}
+/**
+ * Classify an attribute key against the policy. `drop` wins over `safe` (a key
+ * matching both is dropped — fail safe). Unmatched keys are `content` and flow
+ * to the value-scrubbing stages.
+ */
+export function classifyKey(key, policy) {
+    if (policy.drop.some((p) => matches(key, p)))
+        return 'drop';
+    if (policy.safe.some((p) => matches(key, p)))
+        return 'safe';
+    return 'content';
+}
+/**
+ * Structural key-policy stage: deletes `drop` keys outright (recording a
+ * redaction) and passes `safe` + `content` keys through unchanged. Value-level
+ * scrubbing of `content` keys is handled by later stages.
+ */
+export function keyPolicyStage(policy) {
+    return {
+        name: 'key-policy',
+        version: VERSION,
+        scrub(attributes) {
+            const out = {};
+            const redactions = [];
+            for (const [key, value] of Object.entries(attributes)) {
+                if (classifyKey(key, policy) === 'drop') {
+                    redactions.push({ key, stage: 'key-policy', kind: 'dropped-key' });
+                    continue;
+                }
+                out[key] = value;
+            }
+            return { attributes: out, redactions };
+        },
+    };
+}

package/dist/scrub/layer2.d.ts

@@ -0,0 +1,23 @@
+import { ScrubPipeline } from './pipeline.js';
+import { type KeyPolicy } from './key-policy.js';
+/**
+ * The layer-2 / public / derived-content scrub altitude
+ * (spec/2026-07-06-distillation-v1.md §10, D6; reconciled 2026-07-07 with the
+ * shipped #1409 seed profile).
+ *
+ * Secret-only: the structural key policy + deterministic plain-patterns
+ * (emails, home-dir paths, credential-ID prefixes — #1330/#1415) + the FULL
+ * secretlint stage (including its Pass-2 entropy secret-shape fallback). It
+ * deliberately DROPS openredaction and ML-PII — the stages whose PII
+ * shape-matching and trigger-word behaviour deface ordinary public prose
+ * (#1409).
+ *
+ * Mode note vs the seed profile (`buildSeedScrubPipeline`): seeds are
+ * REDACT-mode (the scrubbed output is published, so prose must survive →
+ * entropy fallback OFF there). This pipeline serves CHECK-mode consumers —
+ * `distillClusters` REJECTS a skill on any scrub change rather than publishing
+ * a redacted body — so a false positive costs one re-distill, never defaces
+ * published content, and the entropy fallback stays ON as the stricter net for
+ * a fresh secret surface (DR-2026-07-06 decision 3).
+ */
+export declare function buildLayer2ScrubPipeline(policy?: KeyPolicy): ScrubPipeline;

package/dist/scrub/layer2.js

@@ -0,0 +1,32 @@
+import { ScrubPipeline } from './pipeline.js';
+import { keyPolicyStage } from './key-policy.js';
+import { secretlintStage } from './secretlint-stage.js';
+import { plainPatternsStage } from './plain-patterns-stage.js';
+import { DEFAULT_KEY_POLICY } from './build.js';
+/**
+ * The layer-2 / public / derived-content scrub altitude
+ * (spec/2026-07-06-distillation-v1.md §10, D6; reconciled 2026-07-07 with the
+ * shipped #1409 seed profile).
+ *
+ * Secret-only: the structural key policy + deterministic plain-patterns
+ * (emails, home-dir paths, credential-ID prefixes — #1330/#1415) + the FULL
+ * secretlint stage (including its Pass-2 entropy secret-shape fallback). It
+ * deliberately DROPS openredaction and ML-PII — the stages whose PII
+ * shape-matching and trigger-word behaviour deface ordinary public prose
+ * (#1409).
+ *
+ * Mode note vs the seed profile (`buildSeedScrubPipeline`): seeds are
+ * REDACT-mode (the scrubbed output is published, so prose must survive →
+ * entropy fallback OFF there). This pipeline serves CHECK-mode consumers —
+ * `distillClusters` REJECTS a skill on any scrub change rather than publishing
+ * a redacted body — so a false positive costs one re-distill, never defaces
+ * published content, and the entropy fallback stays ON as the stricter net for
+ * a fresh secret surface (DR-2026-07-06 decision 3).
+ */
+export function buildLayer2ScrubPipeline(policy = DEFAULT_KEY_POLICY) {
+    return new ScrubPipeline([
+        keyPolicyStage(policy),
+        plainPatternsStage(policy, { credentialIds: true }),
+        secretlintStage(policy),
+    ]);
+}

package/dist/scrub/ml-pii-stage.d.ts

@@ -0,0 +1,24 @@
+import { type KeyPolicy } from './key-policy.js';
+import type { ScrubStage } from './types.js';
+/** A PII entity detected in text: a label (PER/ORG/LOC/…) and the matched span text. */
+export interface PiiEntity {
+    label: string;
+    text: string;
+}
+/**
+ * ML PII detector seam. The production implementation wraps a Transformers.js
+ * token-classification (NER) model; tests inject a deterministic fake. Kept
+ * behind an interface so the stage logic is unit-tested without loading a model.
+ */
+export interface PiiDetector {
+    detect(text: string): Promise<PiiEntity[]>;
+}
+/**
+ * ML PII stage. Runs the injected NER detector over each `content`-classified
+ * string value and replaces whole-word occurrences of each detected entity with
+ * `[PII:<label>]` (word-boundary matching so substrings aren't corrupted — e.g.
+ * detecting "London" leaves "Londoner" intact). `safe` and non-string values
+ * pass through untouched. The detector returns matched text rather than offsets
+ * because the Transformers.js NER pipeline does not surface character offsets.
+ */
+export declare function mlPiiStage(policy: KeyPolicy, detector: PiiDetector): ScrubStage;

package/dist/scrub/ml-pii-stage.js

@@ -0,0 +1,42 @@
+import { classifyKey } from './key-policy.js';
+const VERSION = '0.1.0';
+function escapeRegExp(s) {
+    return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+/**
+ * ML PII stage. Runs the injected NER detector over each `content`-classified
+ * string value and replaces whole-word occurrences of each detected entity with
+ * `[PII:<label>]` (word-boundary matching so substrings aren't corrupted — e.g.
+ * detecting "London" leaves "Londoner" intact). `safe` and non-string values
+ * pass through untouched. The detector returns matched text rather than offsets
+ * because the Transformers.js NER pipeline does not surface character offsets.
+ */
+export function mlPiiStage(policy, detector) {
+    return {
+        name: 'ml-pii',
+        version: VERSION,
+        async scrub(attributes) {
+            const out = {};
+            const redactions = [];
+            for (const [key, value] of Object.entries(attributes)) {
+                if (typeof value !== 'string' || classifyKey(key, policy) !== 'content') {
+                    out[key] = value;
+                    continue;
+                }
+                let text = value;
+                for (const ent of await detector.detect(value)) {
+                    if (!ent.text)
+                        continue;
+                    const re = new RegExp(`\\b${escapeRegExp(ent.text)}\\b`, 'g');
+                    const replaced = text.replace(re, `[PII:${ent.label}]`);
+                    if (replaced !== text) {
+                        redactions.push({ key, stage: 'ml-pii', kind: 'pii', detail: ent.label });
+                        text = replaced;
+                    }
+                }
+                out[key] = text;
+            }
+            return { attributes: out, redactions };
+        },
+    };
+}