@jinn-network/core 0.1.0-canary.3afae198
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/canonical-json.d.ts +1 -0
- package/dist/canonical-json.js +67 -0
- package/dist/captured-task.d.ts +140 -0
- package/dist/captured-task.js +122 -0
- package/dist/contribution-store.d.ts +97 -0
- package/dist/contribution-store.js +770 -0
- package/dist/corpus-read/acquire.d.ts +39 -0
- package/dist/corpus-read/acquire.js +212 -0
- package/dist/corpus-read/cache.d.ts +14 -0
- package/dist/corpus-read/cache.js +17 -0
- package/dist/corpus-read/capture-meta.d.ts +16 -0
- package/dist/corpus-read/capture-meta.js +18 -0
- package/dist/corpus-read/create-corpus.d.ts +2 -0
- package/dist/corpus-read/create-corpus.js +97 -0
- package/dist/corpus-read/fetch-artifact.d.ts +21 -0
- package/dist/corpus-read/fetch-artifact.js +32 -0
- package/dist/corpus-read/fetch.d.ts +12 -0
- package/dist/corpus-read/fetch.js +24 -0
- package/dist/corpus-read/http-discovery.d.ts +9 -0
- package/dist/corpus-read/http-discovery.js +128 -0
- package/dist/corpus-read/index.d.ts +10 -0
- package/dist/corpus-read/index.js +10 -0
- package/dist/corpus-read/ipfs.d.ts +14 -0
- package/dist/corpus-read/ipfs.js +81 -0
- package/dist/corpus-read/route-resolver.d.ts +16 -0
- package/dist/corpus-read/route-resolver.js +19 -0
- package/dist/corpus-read/types.d.ts +177 -0
- package/dist/corpus-read/types.js +42 -0
- package/dist/envelope.d.ts +134 -0
- package/dist/envelope.js +162 -0
- package/dist/evidence-adapter.d.ts +26 -0
- package/dist/evidence-adapter.js +321 -0
- package/dist/evidence-filesystem.d.ts +40 -0
- package/dist/evidence-filesystem.js +267 -0
- package/dist/evidence-index.d.ts +117 -0
- package/dist/evidence-index.js +1083 -0
- package/dist/evidence-store-lock.d.ts +2 -0
- package/dist/evidence-store-lock.js +208 -0
- package/dist/execution-envelope.d.ts +3142 -0
- package/dist/execution-envelope.js +203 -0
- package/dist/index.d.ts +12 -0
- package/dist/index.js +12 -0
- package/dist/manifest.d.ts +75 -0
- package/dist/manifest.js +184 -0
- package/dist/paired.d.ts +68 -0
- package/dist/paired.js +92 -0
- package/dist/scrub/apply-dispositions.d.ts +74 -0
- package/dist/scrub/apply-dispositions.js +309 -0
- package/dist/scrub/build.d.ts +75 -0
- package/dist/scrub/build.js +143 -0
- package/dist/scrub/checksummed-instruments-detector.d.ts +13 -0
- package/dist/scrub/checksummed-instruments-detector.js +119 -0
- package/dist/scrub/data/bip39-english.d.ts +2 -0
- package/dist/scrub/data/bip39-english.js +2051 -0
- package/dist/scrub/data/gitleaks-rules.d.ts +56 -0
- package/dist/scrub/data/gitleaks-rules.js +67 -0
- package/dist/scrub/emit-scrub.d.ts +52 -0
- package/dist/scrub/emit-scrub.js +122 -0
- package/dist/scrub/eval/findings-from-scrub.d.ts +18 -0
- package/dist/scrub/eval/findings-from-scrub.js +99 -0
- package/dist/scrub/eval/fixtures.d.ts +14 -0
- package/dist/scrub/eval/fixtures.js +159 -0
- package/dist/scrub/eval/index.d.ts +6 -0
- package/dist/scrub/eval/index.js +6 -0
- package/dist/scrub/eval/local-corpus.d.ts +15 -0
- package/dist/scrub/eval/local-corpus.js +29 -0
- package/dist/scrub/eval/metrics.d.ts +19 -0
- package/dist/scrub/eval/metrics.js +77 -0
- package/dist/scrub/eval/run-bench.d.ts +8 -0
- package/dist/scrub/eval/run-bench.js +125 -0
- package/dist/scrub/eval/types.d.ts +69 -0
- package/dist/scrub/eval/types.js +7 -0
- package/dist/scrub/finding.d.ts +38 -0
- package/dist/scrub/finding.js +7 -0
- package/dist/scrub/git-identity-detector.d.ts +16 -0
- package/dist/scrub/git-identity-detector.js +116 -0
- package/dist/scrub/gitleaks-detector.d.ts +13 -0
- package/dist/scrub/gitleaks-detector.js +45 -0
- package/dist/scrub/gliner-detector.d.ts +65 -0
- package/dist/scrub/gliner-detector.js +129 -0
- package/dist/scrub/index.d.ts +26 -0
- package/dist/scrub/index.js +26 -0
- package/dist/scrub/ip-address-detector.d.ts +12 -0
- package/dist/scrub/ip-address-detector.js +81 -0
- package/dist/scrub/key-policy.d.ts +30 -0
- package/dist/scrub/key-policy.js +72 -0
- package/dist/scrub/known-identity-detector.d.ts +66 -0
- package/dist/scrub/known-identity-detector.js +284 -0
- package/dist/scrub/layer2.d.ts +18 -0
- package/dist/scrub/layer2.js +28 -0
- package/dist/scrub/ml-pii-stage.d.ts +42 -0
- package/dist/scrub/ml-pii-stage.js +119 -0
- package/dist/scrub/pii-build.d.ts +47 -0
- package/dist/scrub/pii-build.js +67 -0
- package/dist/scrub/pipeline.d.ts +77 -0
- package/dist/scrub/pipeline.js +225 -0
- package/dist/scrub/plain-patterns-stage.d.ts +33 -0
- package/dist/scrub/plain-patterns-stage.js +88 -0
- package/dist/scrub/policy.d.ts +28 -0
- package/dist/scrub/policy.js +156 -0
- package/dist/scrub/provenance.d.ts +53 -0
- package/dist/scrub/provenance.js +83 -0
- package/dist/scrub/reject-classes-detector.d.ts +11 -0
- package/dist/scrub/reject-classes-detector.js +195 -0
- package/dist/scrub/reject-publish-error.d.ts +30 -0
- package/dist/scrub/reject-publish-error.js +47 -0
- package/dist/scrub/review-queue.d.ts +90 -0
- package/dist/scrub/review-queue.js +259 -0
- package/dist/scrub/secretlint-stage.d.ts +21 -0
- package/dist/scrub/secretlint-stage.js +264 -0
- package/dist/scrub/transformers-detector.d.ts +41 -0
- package/dist/scrub/transformers-detector.js +77 -0
- package/dist/scrub/types.d.ts +36 -0
- package/dist/scrub/types.js +1 -0
- package/dist/scrub/url-credentials-detector.d.ts +9 -0
- package/dist/scrub/url-credentials-detector.js +66 -0
- package/dist/session-provenance.d.ts +75 -0
- package/dist/session-provenance.js +20 -0
- package/dist/skill-artifact.d.ts +442 -0
- package/dist/skill-artifact.js +136 -0
- package/dist/trajectory/hash-chain.d.ts +18 -0
- package/dist/trajectory/hash-chain.js +47 -0
- package/dist/trajectory/index.d.ts +4 -0
- package/dist/trajectory/index.js +4 -0
- package/dist/trajectory/schema.d.ts +681 -0
- package/dist/trajectory/schema.js +119 -0
- package/dist/trajectory/transcript-parsers/aider-history.d.ts +42 -0
- package/dist/trajectory/transcript-parsers/aider-history.js +181 -0
- package/dist/trajectory/transcript-parsers/claude-code-jsonl.d.ts +37 -0
- package/dist/trajectory/transcript-parsers/claude-code-jsonl.js +142 -0
- package/dist/trajectory/transcript-parsers/codex-session.d.ts +42 -0
- package/dist/trajectory/transcript-parsers/codex-session.js +235 -0
- package/dist/trajectory/transcript-parsers/continue-devdata.d.ts +41 -0
- package/dist/trajectory/transcript-parsers/continue-devdata.js +190 -0
- package/dist/trajectory/transcript-parsers/cursor-sqlite.d.ts +38 -0
- package/dist/trajectory/transcript-parsers/cursor-sqlite.js +118 -0
- package/dist/trajectory/transcript-parsers/gemini-session.d.ts +35 -0
- package/dist/trajectory/transcript-parsers/gemini-session.js +137 -0
- package/dist/trajectory/transcript-parsers/index.d.ts +7 -0
- package/dist/trajectory/transcript-parsers/index.js +7 -0
- package/dist/trajectory/transcript-parsers/types.d.ts +101 -0
- package/dist/trajectory/transcript-parsers/types.js +32 -0
- package/dist/trajectory/transcript-to-spans/attrs.d.ts +4 -0
- package/dist/trajectory/transcript-to-spans/attrs.js +30 -0
- package/dist/trajectory/transcript-to-spans/claude-code-stream-json.d.ts +8 -0
- package/dist/trajectory/transcript-to-spans/claude-code-stream-json.js +162 -0
- package/dist/trajectory/transcript-to-spans/codex-exec-json.d.ts +8 -0
- package/dist/trajectory/transcript-to-spans/codex-exec-json.js +253 -0
- package/dist/trajectory/transcript-to-spans/index.d.ts +4 -0
- package/dist/trajectory/transcript-to-spans/index.js +4 -0
- package/dist/trajectory/transcript-to-spans/types.d.ts +18 -0
- package/dist/trajectory/transcript-to-spans/types.js +1 -0
- package/dist/window.d.ts +12 -0
- package/dist/window.js +5 -0
- package/package.json +72 -0
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/** Vendored gitleaks MIT rule subset — see pin.note (Q6). */
|
|
2
|
+
export declare const GITLEAKS_PACK: {
|
|
3
|
+
readonly pin: {
|
|
4
|
+
readonly source: "https://github.com/gitleaks/gitleaks";
|
|
5
|
+
readonly ref: "v8.28.0";
|
|
6
|
+
readonly date: "2026-07-22";
|
|
7
|
+
readonly license: "MIT";
|
|
8
|
+
readonly note: "Q6: pin-and-manually-refresh subset; no build-time sync. Regexes adapted to JS (no Go (?i)/(?-i:) verbs).";
|
|
9
|
+
};
|
|
10
|
+
readonly rules: readonly [{
|
|
11
|
+
readonly id: "github-pat";
|
|
12
|
+
readonly description: "GitHub Personal Access Token (classic).";
|
|
13
|
+
readonly regex: "\\bghp_[0-9a-zA-Z]{36}\\b";
|
|
14
|
+
}, {
|
|
15
|
+
readonly id: "github-fine-grained-pat";
|
|
16
|
+
readonly description: "GitHub Fine-Grained Personal Access Token.";
|
|
17
|
+
readonly regex: "\\bgithub_pat_[0-9a-zA-Z_]{82}\\b";
|
|
18
|
+
}, {
|
|
19
|
+
readonly id: "slack-bot-token";
|
|
20
|
+
readonly description: "Slack Bot token.";
|
|
21
|
+
readonly regex: "\\bxoxb-[0-9]{10,13}-[0-9]{10,13}[a-zA-Z0-9-]*";
|
|
22
|
+
}, {
|
|
23
|
+
readonly id: "npm-access-token";
|
|
24
|
+
readonly description: "npm access token.";
|
|
25
|
+
readonly regex: "\\bnpm_[A-Za-z0-9]{36}\\b";
|
|
26
|
+
}, {
|
|
27
|
+
readonly id: "stripe-access-token";
|
|
28
|
+
readonly description: "Stripe secret/restricted key.";
|
|
29
|
+
readonly regex: "\\b(?:sk|rk)_(?:test|live|prod)_[A-Za-z0-9]{10,99}\\b";
|
|
30
|
+
}, {
|
|
31
|
+
readonly id: "openai-api-key";
|
|
32
|
+
readonly description: "OpenAI API key (legacy sk-…T3BlbkFJ… shape).";
|
|
33
|
+
readonly regex: "\\bsk-[A-Za-z0-9]{20}T3BlbkFJ[A-Za-z0-9]{20}\\b";
|
|
34
|
+
}, {
|
|
35
|
+
readonly id: "anthropic-api-key";
|
|
36
|
+
readonly description: "Anthropic API key.";
|
|
37
|
+
readonly regex: "\\bsk-ant-api03-[A-Za-z0-9_-]{93}AA\\b";
|
|
38
|
+
}, {
|
|
39
|
+
readonly id: "aws-access-token";
|
|
40
|
+
readonly description: "AWS access key id.";
|
|
41
|
+
readonly regex: "\\b(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}\\b";
|
|
42
|
+
}, {
|
|
43
|
+
readonly id: "pypi-upload-token";
|
|
44
|
+
readonly description: "PyPI upload token.";
|
|
45
|
+
readonly regex: "\\bpypi-AgEIcHlwaS5vcmc[\\w-]{50,1000}";
|
|
46
|
+
}, {
|
|
47
|
+
readonly id: "hashicorp-tf-api-token";
|
|
48
|
+
readonly description: "HashiCorp Terraform API token (…atlasv1…).";
|
|
49
|
+
readonly regex: "\\b[a-z0-9]{14}\\.atlasv1\\.[A-Za-z0-9\\-_=]{60,70}\\b";
|
|
50
|
+
}, {
|
|
51
|
+
readonly id: "private-key";
|
|
52
|
+
readonly description: "PEM private key block.";
|
|
53
|
+
readonly regex: "-----BEGIN[ A-Z0-9_-]{0,100}PRIVATE KEY(?: BLOCK)?-----[\\s\\S-]{64,}?-----END[ A-Z0-9_-]{0,100}PRIVATE KEY(?: BLOCK)?-----";
|
|
54
|
+
}];
|
|
55
|
+
};
|
|
56
|
+
export type GitleaksPack = typeof GITLEAKS_PACK;
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/** Vendored gitleaks MIT rule subset — see pin.note (Q6). */
|
|
2
|
+
export const GITLEAKS_PACK = {
|
|
3
|
+
"pin": {
|
|
4
|
+
"source": "https://github.com/gitleaks/gitleaks",
|
|
5
|
+
"ref": "v8.28.0",
|
|
6
|
+
"date": "2026-07-22",
|
|
7
|
+
"license": "MIT",
|
|
8
|
+
"note": "Q6: pin-and-manually-refresh subset; no build-time sync. Regexes adapted to JS (no Go (?i)/(?-i:) verbs)."
|
|
9
|
+
},
|
|
10
|
+
"rules": [
|
|
11
|
+
{
|
|
12
|
+
"id": "github-pat",
|
|
13
|
+
"description": "GitHub Personal Access Token (classic).",
|
|
14
|
+
"regex": "\\bghp_[0-9a-zA-Z]{36}\\b"
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
"id": "github-fine-grained-pat",
|
|
18
|
+
"description": "GitHub Fine-Grained Personal Access Token.",
|
|
19
|
+
"regex": "\\bgithub_pat_[0-9a-zA-Z_]{82}\\b"
|
|
20
|
+
},
|
|
21
|
+
{
|
|
22
|
+
"id": "slack-bot-token",
|
|
23
|
+
"description": "Slack Bot token.",
|
|
24
|
+
"regex": "\\bxoxb-[0-9]{10,13}-[0-9]{10,13}[a-zA-Z0-9-]*"
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
"id": "npm-access-token",
|
|
28
|
+
"description": "npm access token.",
|
|
29
|
+
"regex": "\\bnpm_[A-Za-z0-9]{36}\\b"
|
|
30
|
+
},
|
|
31
|
+
{
|
|
32
|
+
"id": "stripe-access-token",
|
|
33
|
+
"description": "Stripe secret/restricted key.",
|
|
34
|
+
"regex": "\\b(?:sk|rk)_(?:test|live|prod)_[A-Za-z0-9]{10,99}\\b"
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
"id": "openai-api-key",
|
|
38
|
+
"description": "OpenAI API key (legacy sk-…T3BlbkFJ… shape).",
|
|
39
|
+
"regex": "\\bsk-[A-Za-z0-9]{20}T3BlbkFJ[A-Za-z0-9]{20}\\b"
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
"id": "anthropic-api-key",
|
|
43
|
+
"description": "Anthropic API key.",
|
|
44
|
+
"regex": "\\bsk-ant-api03-[A-Za-z0-9_-]{93}AA\\b"
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
"id": "aws-access-token",
|
|
48
|
+
"description": "AWS access key id.",
|
|
49
|
+
"regex": "\\b(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}\\b"
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
"id": "pypi-upload-token",
|
|
53
|
+
"description": "PyPI upload token.",
|
|
54
|
+
"regex": "\\bpypi-AgEIcHlwaS5vcmc[\\w-]{50,1000}"
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
"id": "hashicorp-tf-api-token",
|
|
58
|
+
"description": "HashiCorp Terraform API token (…atlasv1…).",
|
|
59
|
+
"regex": "\\b[a-z0-9]{14}\\.atlasv1\\.[A-Za-z0-9\\-_=]{60,70}\\b"
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
"id": "private-key",
|
|
63
|
+
"description": "PEM private key block.",
|
|
64
|
+
"regex": "-----BEGIN[ A-Z0-9_-]{0,100}PRIVATE KEY(?: BLOCK)?-----[\\s\\S-]{64,}?-----END[ A-Z0-9_-]{0,100}PRIVATE KEY(?: BLOCK)?-----"
|
|
65
|
+
}
|
|
66
|
+
]
|
|
67
|
+
};
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import type { RedactionManifest, Span } from '../trajectory/schema.js';
|
|
2
|
+
import type { ScrubPipeline } from './pipeline.js';
|
|
3
|
+
/**
|
|
4
|
+
* Seller-side scrub applied at emit time, before a trajectory is signed and
|
|
5
|
+
* published. Runs every span's attributes (and each event's attributes) through
|
|
6
|
+
* the pipeline, rebuilds the prev-span hash chain over the scrubbed spans, and
|
|
7
|
+
* produces the redaction manifest (one entry per span listing the keys touched;
|
|
8
|
+
* `event.<key>` for event-attribute redactions). Populates additive provenance
|
|
9
|
+
* fields (#1974): `schemaVersion`, optional `policyHash`, `perClassCounts`.
|
|
10
|
+
*/
|
|
11
|
+
export declare function scrubSpansForEmit(spans: Span[], pipeline: ScrubPipeline, taskCid: string): Promise<{
|
|
12
|
+
spans: Span[];
|
|
13
|
+
redactionManifest: RedactionManifest;
|
|
14
|
+
}>;
|
|
15
|
+
/**
|
|
16
|
+
* Minimal structural shape of a capture span (jinn.capture-trajectory.v1): a
|
|
17
|
+
* flat attribute bag plus the keys already redacted at ingest. Captures have no
|
|
18
|
+
* events array and no in-run hash chain, so they cannot go through
|
|
19
|
+
* {@link scrubSpansForEmit}; this is their dedicated seller-side scrub.
|
|
20
|
+
*/
|
|
21
|
+
export interface ScrubbableCaptureSpan {
|
|
22
|
+
attributes: Record<string, unknown>;
|
|
23
|
+
redactedKeys: string[];
|
|
24
|
+
}
|
|
25
|
+
export interface ScrubCaptureSpansOptions {
|
|
26
|
+
/**
|
|
27
|
+
* Optional mutable tally bag. When provided, applied disposition counts from
|
|
28
|
+
* each span are merged in (#1974) so callers can attach `perClassCounts` to
|
|
29
|
+
* a capture redaction manifest.
|
|
30
|
+
*/
|
|
31
|
+
perClassCounts?: Record<string, number>;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Seller-side scrub for capture spans, applied at publish time before the
|
|
35
|
+
* capture trajectory is uploaded. Runs each span's attributes through the
|
|
36
|
+
* pipeline, replacing them with the scrubbed values, and unions the pipeline's
|
|
37
|
+
* redactions into the span's existing `redactedKeys`. Unlike
|
|
38
|
+
* {@link scrubSpansForEmit} it does NOT touch events or inject a hash chain —
|
|
39
|
+
* capture spans carry neither. Generic over the span type so the concrete
|
|
40
|
+
* `SpanRow` (with its extra columns) is returned unchanged apart from
|
|
41
|
+
* `attributes` and `redactedKeys`.
|
|
42
|
+
*/
|
|
43
|
+
export declare function scrubCaptureSpans<T extends ScrubbableCaptureSpan>(spans: T[], pipeline: ScrubPipeline, opts?: ScrubCaptureSpansOptions): Promise<T[]>;
|
|
44
|
+
/**
|
|
45
|
+
* Unions two redaction manifests by spanId (dedup redactedKeys per span). Used
|
|
46
|
+
* when an ingestion-time manifest (e.g. a capture's stored redactedKeys) is
|
|
47
|
+
* combined with the emit-time pipeline manifest. Keeps the
|
|
48
|
+
* `totalRedactions === sum(redactedKeys)` invariant the schema enforces.
|
|
49
|
+
* Provenance fields (#1974): prefer the higher `schemaVersion`, take the first
|
|
50
|
+
* defined `policyHash`, and sum `perClassCounts`.
|
|
51
|
+
*/
|
|
52
|
+
export declare function mergeRedactionManifests(a: RedactionManifest, b: RedactionManifest): RedactionManifest;
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
import { computeGenesisHash, computePrevSpanHash, } from '../trajectory/hash-chain.js';
|
|
2
|
+
import { mergePerClassCounts } from './provenance.js';
|
|
3
|
+
/**
|
|
4
|
+
* Seller-side scrub applied at emit time, before a trajectory is signed and
|
|
5
|
+
* published. Runs every span's attributes (and each event's attributes) through
|
|
6
|
+
* the pipeline, rebuilds the prev-span hash chain over the scrubbed spans, and
|
|
7
|
+
* produces the redaction manifest (one entry per span listing the keys touched;
|
|
8
|
+
* `event.<key>` for event-attribute redactions). Populates additive provenance
|
|
9
|
+
* fields (#1974): `schemaVersion`, optional `policyHash`, `perClassCounts`.
|
|
10
|
+
*/
|
|
11
|
+
export async function scrubSpansForEmit(spans, pipeline, taskCid) {
|
|
12
|
+
const out = [];
|
|
13
|
+
const manifestSpans = [];
|
|
14
|
+
let total = 0;
|
|
15
|
+
let prev = computeGenesisHash(taskCid);
|
|
16
|
+
let perClassCounts = {};
|
|
17
|
+
for (const span of spans) {
|
|
18
|
+
const redactedKeys = new Set();
|
|
19
|
+
const attrResult = await pipeline.run(span.attributes ?? {});
|
|
20
|
+
for (const r of attrResult.redactions)
|
|
21
|
+
redactedKeys.add(r.key);
|
|
22
|
+
perClassCounts =
|
|
23
|
+
mergePerClassCounts(perClassCounts, attrResult.perClassCounts) ?? perClassCounts;
|
|
24
|
+
const events = [];
|
|
25
|
+
for (const event of span.events) {
|
|
26
|
+
if (event.attributes && Object.keys(event.attributes).length > 0) {
|
|
27
|
+
const evResult = await pipeline.run(event.attributes);
|
|
28
|
+
events.push({ ...event, attributes: evResult.attributes });
|
|
29
|
+
for (const r of evResult.redactions)
|
|
30
|
+
redactedKeys.add(`event.${r.key}`);
|
|
31
|
+
perClassCounts =
|
|
32
|
+
mergePerClassCounts(perClassCounts, evResult.perClassCounts) ?? perClassCounts;
|
|
33
|
+
}
|
|
34
|
+
else {
|
|
35
|
+
events.push(event);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
const scrubbed = {
|
|
39
|
+
...span,
|
|
40
|
+
attributes: { ...attrResult.attributes, 'jinn.prevSpanHash': prev },
|
|
41
|
+
events,
|
|
42
|
+
};
|
|
43
|
+
out.push(scrubbed);
|
|
44
|
+
prev = computePrevSpanHash(scrubbed);
|
|
45
|
+
if (redactedKeys.size > 0) {
|
|
46
|
+
const keys = [...redactedKeys];
|
|
47
|
+
manifestSpans.push({ spanId: span.spanId, redactedKeys: keys });
|
|
48
|
+
total += keys.length;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
return {
|
|
52
|
+
spans: out,
|
|
53
|
+
redactionManifest: {
|
|
54
|
+
spans: manifestSpans,
|
|
55
|
+
totalRedactions: total,
|
|
56
|
+
...pipeline.manifestProvenance(perClassCounts),
|
|
57
|
+
},
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Seller-side scrub for capture spans, applied at publish time before the
|
|
62
|
+
* capture trajectory is uploaded. Runs each span's attributes through the
|
|
63
|
+
* pipeline, replacing them with the scrubbed values, and unions the pipeline's
|
|
64
|
+
* redactions into the span's existing `redactedKeys`. Unlike
|
|
65
|
+
* {@link scrubSpansForEmit} it does NOT touch events or inject a hash chain —
|
|
66
|
+
* capture spans carry neither. Generic over the span type so the concrete
|
|
67
|
+
* `SpanRow` (with its extra columns) is returned unchanged apart from
|
|
68
|
+
* `attributes` and `redactedKeys`.
|
|
69
|
+
*/
|
|
70
|
+
export async function scrubCaptureSpans(spans, pipeline, opts = {}) {
|
|
71
|
+
const out = [];
|
|
72
|
+
for (const span of spans) {
|
|
73
|
+
const result = await pipeline.run(span.attributes ?? {});
|
|
74
|
+
const redactedKeys = new Set(span.redactedKeys);
|
|
75
|
+
for (const r of result.redactions)
|
|
76
|
+
redactedKeys.add(r.key);
|
|
77
|
+
if (opts.perClassCounts && result.perClassCounts) {
|
|
78
|
+
const merged = mergePerClassCounts(opts.perClassCounts, result.perClassCounts);
|
|
79
|
+
if (merged) {
|
|
80
|
+
for (const key of Object.keys(opts.perClassCounts))
|
|
81
|
+
delete opts.perClassCounts[key];
|
|
82
|
+
Object.assign(opts.perClassCounts, merged);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
out.push({ ...span, attributes: result.attributes, redactedKeys: [...redactedKeys] });
|
|
86
|
+
}
|
|
87
|
+
return out;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Unions two redaction manifests by spanId (dedup redactedKeys per span). Used
|
|
91
|
+
* when an ingestion-time manifest (e.g. a capture's stored redactedKeys) is
|
|
92
|
+
* combined with the emit-time pipeline manifest. Keeps the
|
|
93
|
+
* `totalRedactions === sum(redactedKeys)` invariant the schema enforces.
|
|
94
|
+
* Provenance fields (#1974): prefer the higher `schemaVersion`, take the first
|
|
95
|
+
* defined `policyHash`, and sum `perClassCounts`.
|
|
96
|
+
*/
|
|
97
|
+
export function mergeRedactionManifests(a, b) {
|
|
98
|
+
const bySpan = new Map();
|
|
99
|
+
for (const m of [a, b]) {
|
|
100
|
+
for (const s of m.spans) {
|
|
101
|
+
const set = bySpan.get(s.spanId) ?? new Set();
|
|
102
|
+
for (const k of s.redactedKeys)
|
|
103
|
+
set.add(k);
|
|
104
|
+
bySpan.set(s.spanId, set);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
const spans = [...bySpan]
|
|
108
|
+
.map(([spanId, keys]) => ({ spanId, redactedKeys: [...keys] }))
|
|
109
|
+
.filter((s) => s.redactedKeys.length > 0);
|
|
110
|
+
const perClassCounts = mergePerClassCounts(a.perClassCounts, b.perClassCounts);
|
|
111
|
+
const schemaVersion = a.schemaVersion !== undefined || b.schemaVersion !== undefined
|
|
112
|
+
? Math.max(a.schemaVersion ?? 0, b.schemaVersion ?? 0)
|
|
113
|
+
: undefined;
|
|
114
|
+
const policyHash = a.policyHash ?? b.policyHash;
|
|
115
|
+
return {
|
|
116
|
+
spans,
|
|
117
|
+
totalRedactions: spans.reduce((n, s) => n + s.redactedKeys.length, 0),
|
|
118
|
+
...(schemaVersion !== undefined ? { schemaVersion } : {}),
|
|
119
|
+
...(policyHash !== undefined ? { policyHash } : {}),
|
|
120
|
+
...(perClassCounts !== undefined ? { perClassCounts } : {}),
|
|
121
|
+
};
|
|
122
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Map current-pipeline redactions + before/after text into approximate span
|
|
3
|
+
* findings for the M0 baseline harness. Prefer exact original-span absence:
|
|
4
|
+
* if a labeled substring is gone from the scrubbed output, emit a finding
|
|
5
|
+
* covering that label. Extra stubs in output that do not match any label
|
|
6
|
+
* become FP via orphan stub detection.
|
|
7
|
+
*/
|
|
8
|
+
import type { EvalFixture, ScoredFinding, ScrubClass } from './types.js';
|
|
9
|
+
/** Map known redaction `detail` / stage hints → ScrubClass. */
|
|
10
|
+
export declare function classFromRedactionDetail(detail: string | undefined, stage: string): ScrubClass | undefined;
|
|
11
|
+
/**
|
|
12
|
+
* Derive predicted findings for a fixture from original text, scrubbed text,
|
|
13
|
+
* and redaction records. Uses label-survival for TP/FN and stub orphans for FP.
|
|
14
|
+
*/
|
|
15
|
+
export declare function findingsFromScrubResult(fixture: EvalFixture, scrubbedText: string, redactions: Array<{
|
|
16
|
+
stage: string;
|
|
17
|
+
detail?: string;
|
|
18
|
+
}>): ScoredFinding[];
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Map current-pipeline redactions + before/after text into approximate span
|
|
3
|
+
* findings for the M0 baseline harness. Prefer exact original-span absence:
|
|
4
|
+
* if a labeled substring is gone from the scrubbed output, emit a finding
|
|
5
|
+
* covering that label. Extra stubs in output that do not match any label
|
|
6
|
+
* become FP via orphan stub detection.
|
|
7
|
+
*/
|
|
8
|
+
/** Map known redaction `detail` / stage hints → ScrubClass. */
|
|
9
|
+
export function classFromRedactionDetail(detail, stage) {
|
|
10
|
+
if (!detail) {
|
|
11
|
+
if (stage === 'key-policy')
|
|
12
|
+
return 'A5';
|
|
13
|
+
return undefined;
|
|
14
|
+
}
|
|
15
|
+
const d = detail.toLowerCase();
|
|
16
|
+
// B2 before generic "name"/"email" — git-identity evidence is carrier-shaped.
|
|
17
|
+
if (d.includes('git-identity') || d === 'trailer-name' || d === 'user-name' || d === 'user-email') {
|
|
18
|
+
return 'B2';
|
|
19
|
+
}
|
|
20
|
+
if (d.includes('email'))
|
|
21
|
+
return 'B1';
|
|
22
|
+
if (d.includes('home-path') || d.includes('home_path'))
|
|
23
|
+
return 'D1';
|
|
24
|
+
if (d.includes('eth-address') || d.includes('ethereum') || d.includes('eth_addr'))
|
|
25
|
+
return 'C1';
|
|
26
|
+
if (d.includes('aws') || d.includes('gcp') || d.includes('github') || d.includes('slack') || d.includes('npm') || d.includes('jwt') || d.includes('ssh') || d.includes('secret') || d.includes('token')) {
|
|
27
|
+
return 'A1';
|
|
28
|
+
}
|
|
29
|
+
if (d.includes('high-entropy') || d.includes('entropy'))
|
|
30
|
+
return 'A2';
|
|
31
|
+
if (d.includes('person') || d.includes('per') || d.includes('name'))
|
|
32
|
+
return 'B3';
|
|
33
|
+
if (d.includes('phone'))
|
|
34
|
+
return 'B5';
|
|
35
|
+
if (d.includes('iban') || d.includes('credit') || d.includes('ssn') || d.includes('card') || d.includes('luhn'))
|
|
36
|
+
return 'B7';
|
|
37
|
+
if (d.includes('ipv4') || d === 'ip' || d.startsWith('ip-') || d.includes('loopback') || d.includes('reserved')) {
|
|
38
|
+
return 'D2';
|
|
39
|
+
}
|
|
40
|
+
if (d.includes('hostname') ||
|
|
41
|
+
d.includes('known-identity:hostname') ||
|
|
42
|
+
d.includes('machine-identity') ||
|
|
43
|
+
d.includes('host')) {
|
|
44
|
+
return 'D3';
|
|
45
|
+
}
|
|
46
|
+
if (d.includes('username') || d.includes('gh-login') || d.includes('home-username'))
|
|
47
|
+
return 'B4';
|
|
48
|
+
if (d.includes('git-user-name') || d.includes('known-identity:git-user-name'))
|
|
49
|
+
return 'B3';
|
|
50
|
+
if (d.includes('url-') || d.includes('userinfo') || d.includes('query-credential'))
|
|
51
|
+
return 'A3';
|
|
52
|
+
if (d.includes('private-key') || d.includes('bip39') || d.includes('mnemonic'))
|
|
53
|
+
return 'A4';
|
|
54
|
+
if (d.includes('env-block'))
|
|
55
|
+
return 'A5';
|
|
56
|
+
if (d.includes('gitleaks'))
|
|
57
|
+
return 'A1';
|
|
58
|
+
if (detail === 'EMAIL' || detail.includes('EMAIL'))
|
|
59
|
+
return 'B1';
|
|
60
|
+
if (detail.includes('ETHEREUM') || detail.includes('ETH'))
|
|
61
|
+
return 'C1';
|
|
62
|
+
if (detail.includes('PATH') || detail.includes('USERNAME'))
|
|
63
|
+
return 'D1';
|
|
64
|
+
return 'E1';
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Derive predicted findings for a fixture from original text, scrubbed text,
|
|
68
|
+
* and redaction records. Uses label-survival for TP/FN and stub orphans for FP.
|
|
69
|
+
*/
|
|
70
|
+
export function findingsFromScrubResult(fixture, scrubbedText, redactions) {
|
|
71
|
+
const findings = [];
|
|
72
|
+
const claimed = new Set();
|
|
73
|
+
for (let i = 0; i < fixture.labels.length; i += 1) {
|
|
74
|
+
const label = fixture.labels[i];
|
|
75
|
+
const original = fixture.text.slice(label.start, label.end);
|
|
76
|
+
if (!original)
|
|
77
|
+
continue;
|
|
78
|
+
// Survived → miss (no finding). Gone or replaced → predicted hit at label span.
|
|
79
|
+
if (!scrubbedText.includes(original)) {
|
|
80
|
+
findings.push({ class: label.class, start: label.start, end: label.end });
|
|
81
|
+
claimed.add(i);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
// Orphan redactions (more redaction records than claimed labels of matching class)
|
|
85
|
+
// approximate FPs when scrubbed diverges without a matching label.
|
|
86
|
+
if (scrubbedText !== fixture.text) {
|
|
87
|
+
const labeledClasses = new Set(fixture.labels.map((l) => l.class));
|
|
88
|
+
for (const r of redactions) {
|
|
89
|
+
const cls = classFromRedactionDetail(r.detail, r.stage);
|
|
90
|
+
if (!cls)
|
|
91
|
+
continue;
|
|
92
|
+
if (!labeledClasses.has(cls)) {
|
|
93
|
+
// Synthetic FP span at 0..0+1 so scoreClass counts it
|
|
94
|
+
findings.push({ class: cls, start: 0, end: Math.min(1, fixture.text.length || 1) });
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
return findings;
|
|
99
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { EvalFixture } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Public synthetic fixtures for CI (#1968 / #1972). No real PII — template shapes only.
|
|
4
|
+
* Targets mirror design §3.2; baseline may miss some classes until later PRs.
|
|
5
|
+
*/
|
|
6
|
+
export declare function syntheticFixtures(): EvalFixture[];
|
|
7
|
+
/**
|
|
8
|
+
* Corruption corpus: known-clean prose that must stay byte-identical.
|
|
9
|
+
* Includes #1784 residual samples that seed profile intentionally leaves alone,
|
|
10
|
+
* plus a short excerpt shape from distractor-operator-claims (no secrets).
|
|
11
|
+
* Checksummed cards/IBANs are labeled fixtures (B7), not corruption samples.
|
|
12
|
+
*/
|
|
13
|
+
export declare function corruptionFixtures(): EvalFixture[];
|
|
14
|
+
export declare function allCiFixtures(): EvalFixture[];
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
/** Helper: build a fixture with a single labeled span covering `needle` in `text`. */
|
|
2
|
+
function labeled(id, text, needle, cls, opts = {}) {
|
|
3
|
+
const start = text.indexOf(needle);
|
|
4
|
+
if (start < 0)
|
|
5
|
+
throw new Error(`needle not found in fixture ${id}`);
|
|
6
|
+
return {
|
|
7
|
+
id,
|
|
8
|
+
text,
|
|
9
|
+
labels: [{ class: cls, start, end: start + needle.length }],
|
|
10
|
+
profile: opts.profile ?? 'seed',
|
|
11
|
+
...opts,
|
|
12
|
+
};
|
|
13
|
+
}
|
|
14
|
+
/** Synthetic BIP-39 12-word run (wordlist words; not a funded wallet). */
|
|
15
|
+
const SYNTH_MNEMONIC = 'abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about';
|
|
16
|
+
/** Synthetic 64-hex private-key material (not a real key). */
|
|
17
|
+
const SYNTH_PRIVKEY = 'a'.repeat(64);
|
|
18
|
+
/**
|
|
19
|
+
* Public synthetic fixtures for CI (#1968 / #1972). No real PII — template shapes only.
|
|
20
|
+
* Targets mirror design §3.2; baseline may miss some classes until later PRs.
|
|
21
|
+
*/
|
|
22
|
+
export function syntheticFixtures() {
|
|
23
|
+
return [
|
|
24
|
+
labeled('B1-email', 'Contact the operator at alice.operator@example.com for access.', 'alice.operator@example.com', 'B1'),
|
|
25
|
+
labeled('D1-home-path', 'Working directory was /Users/synthuser/jinn-mono/client.', '/Users/synthuser', 'D1'),
|
|
26
|
+
labeled('C1-wallet', 'Router lives at 0x1111222233334444555566667777888899990000 on Base.', '0x1111222233334444555566667777888899990000', 'C1', { profile: 'seed' }),
|
|
27
|
+
labeled('A1-github-pat-shape', 'Export GITHUB_TOKEN=ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcd before running.', 'ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcd', 'A1', { profile: 'seed' }),
|
|
28
|
+
labeled('A1-aws-key-id', 'Using key AKIAIOSFODNN7EXAMPLE in the dry-run.', 'AKIAIOSFODNN7EXAMPLE', 'A1', { profile: 'seed' }),
|
|
29
|
+
labeled('A3-url-userinfo', 'Fetch https://user:s3cr3t@localhost/v2/manifests/latest then continue.', 'user:s3cr3t@', 'A3'),
|
|
30
|
+
labeled('A3-url-token-query', 'Webhook https://hooks.example/push?token=tok_synth_example_value_001&x=1 fired.', 'tok_synth_example_value_001', 'A3'),
|
|
31
|
+
labeled('A4-private-key-hex', `Loaded private key ${SYNTH_PRIVKEY} into the signer.`, SYNTH_PRIVKEY, 'A4'),
|
|
32
|
+
labeled('A4-bip39-mnemonic', `Recovery phrase: ${SYNTH_MNEMONIC}`, SYNTH_MNEMONIC, 'A4'),
|
|
33
|
+
labeled('A5-env-block', 'Tool dumped env:\nAPI_KEY=synth\nDATABASE_URL=postgres://x\nSECRET_TOKEN=yyy\nDone.', 'API_KEY=synth\nDATABASE_URL=postgres://x\nSECRET_TOKEN=yyy', 'A5'),
|
|
34
|
+
labeled('B7-card-luhn', 'Customer card: 4111111111111111.', '4111111111111111', 'B7'),
|
|
35
|
+
labeled('B7-iban-mod97', 'Wire to IBAN GB82WEST12345698765432 only.', 'GB82WEST12345698765432', 'B7'),
|
|
36
|
+
labeled('D2-public-ip', 'Peer connected from 8.8.8.8 during the probe.', '8.8.8.8', 'D2'),
|
|
37
|
+
// Hazard: bare UUID must NOT be treated as a secret (batch-2 FP class).
|
|
38
|
+
{
|
|
39
|
+
id: 'hazard-uuid-not-secret',
|
|
40
|
+
text: 'Task id a33f2ca5-e325-4749-b2b5-adaab30c5fc6 completed.',
|
|
41
|
+
labels: [],
|
|
42
|
+
profile: 'seed',
|
|
43
|
+
mustSurvive: true,
|
|
44
|
+
},
|
|
45
|
+
// Hazard: bare 40-hex git SHA must survive (no 0x).
|
|
46
|
+
{
|
|
47
|
+
id: 'hazard-git-sha-survive',
|
|
48
|
+
text: 'Pinned commit deadbeefdeadbeefdeadbeefdeadbeefdeadbeef in the lockfile.',
|
|
49
|
+
labels: [],
|
|
50
|
+
profile: 'seed',
|
|
51
|
+
mustSurvive: true,
|
|
52
|
+
},
|
|
53
|
+
// Hazard: tx hash 0x+64 must survive C1 (C2 is flag-only later).
|
|
54
|
+
{
|
|
55
|
+
id: 'hazard-tx-hash-survive',
|
|
56
|
+
text: 'Receipt 0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa confirmed.',
|
|
57
|
+
labels: [],
|
|
58
|
+
profile: 'seed',
|
|
59
|
+
mustSurvive: true,
|
|
60
|
+
},
|
|
61
|
+
// Hazard: env-var *reference* must not trip a token-name denylist.
|
|
62
|
+
{
|
|
63
|
+
id: 'hazard-env-ref-survive',
|
|
64
|
+
text: 'curl -H "Authorization: Bearer $GH_TOKEN" https://api.github.com/user',
|
|
65
|
+
labels: [],
|
|
66
|
+
profile: 'seed',
|
|
67
|
+
mustSurvive: true,
|
|
68
|
+
},
|
|
69
|
+
// Hazard: GIT_CONFIG tutorial blocks are env-shaped but not secret dumps (#2005).
|
|
70
|
+
{
|
|
71
|
+
id: 'hazard-git-config-tutorial-survive',
|
|
72
|
+
text: [
|
|
73
|
+
'export GIT_CONFIG_GLOBAL=/dev/null',
|
|
74
|
+
'export GIT_CONFIG_SYSTEM=/dev/null',
|
|
75
|
+
'export GIT_CONFIG_NOSYSTEM=1',
|
|
76
|
+
'export GIT_CONFIG_COUNT=2',
|
|
77
|
+
'export GIT_CONFIG_KEY_0=credential.helper',
|
|
78
|
+
'export GIT_CONFIG_VALUE_0=',
|
|
79
|
+
'export GIT_CONFIG_KEY_1=core.askPass',
|
|
80
|
+
'export GIT_CONFIG_VALUE_1=/attempt/askpass',
|
|
81
|
+
].join('\n'),
|
|
82
|
+
labels: [],
|
|
83
|
+
profile: 'seed',
|
|
84
|
+
mustSurvive: true,
|
|
85
|
+
},
|
|
86
|
+
// Hazard: loopback IP must survive (illustrative config knowledge).
|
|
87
|
+
{
|
|
88
|
+
id: 'hazard-loopback-ip-survive',
|
|
89
|
+
text: 'Listen on 127.0.0.1:7331 for the local dashboard.',
|
|
90
|
+
labels: [],
|
|
91
|
+
profile: 'seed',
|
|
92
|
+
mustSurvive: true,
|
|
93
|
+
},
|
|
94
|
+
// B2 carrier — git-identity detector (#1970) jointly covers the name;
|
|
95
|
+
// email on the same line remains B1 via plain-patterns.
|
|
96
|
+
labeled('B2-git-author', 'Author: Synth Operator <synth.operator@example.com>', 'Synth Operator', 'B2', { profile: 'seed' }),
|
|
97
|
+
// B4 self-handle — requires known-identity pack (#1971).
|
|
98
|
+
{
|
|
99
|
+
id: 'B4-self-handle',
|
|
100
|
+
text: 'Operator login synth_operator_handle pushed the branch.',
|
|
101
|
+
labels: [
|
|
102
|
+
{
|
|
103
|
+
class: 'B4',
|
|
104
|
+
start: 'Operator login '.length,
|
|
105
|
+
end: 'Operator login synth_operator_handle'.length,
|
|
106
|
+
},
|
|
107
|
+
],
|
|
108
|
+
profile: 'seed',
|
|
109
|
+
identityPack: { ghLogin: 'synth_operator_handle' },
|
|
110
|
+
},
|
|
111
|
+
// D3 self-hostname — requires known-identity pack (#1971).
|
|
112
|
+
{
|
|
113
|
+
id: 'D3-self-hostname',
|
|
114
|
+
text: 'attempt host=synth-laptop-01 completed.',
|
|
115
|
+
labels: [
|
|
116
|
+
{
|
|
117
|
+
class: 'D3',
|
|
118
|
+
start: 'attempt host='.length,
|
|
119
|
+
end: 'attempt host=synth-laptop-01'.length,
|
|
120
|
+
},
|
|
121
|
+
],
|
|
122
|
+
profile: 'seed',
|
|
123
|
+
identityPack: { hostname: 'synth-laptop-01' },
|
|
124
|
+
},
|
|
125
|
+
// Allowlist: loopback IP must survive (D2 pass, #1971 Q1).
|
|
126
|
+
{
|
|
127
|
+
id: 'allowlist-loopback-survive',
|
|
128
|
+
text: 'Bind the health check to 127.0.0.1:7331 before serving.',
|
|
129
|
+
labels: [],
|
|
130
|
+
mustSurvive: true,
|
|
131
|
+
profile: 'seed',
|
|
132
|
+
},
|
|
133
|
+
];
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Corruption corpus: known-clean prose that must stay byte-identical.
|
|
137
|
+
* Includes #1784 residual samples that seed profile intentionally leaves alone,
|
|
138
|
+
* plus a short excerpt shape from distractor-operator-claims (no secrets).
|
|
139
|
+
* Checksummed cards/IBANs are labeled fixtures (B7), not corruption samples.
|
|
140
|
+
*/
|
|
141
|
+
export function corruptionFixtures() {
|
|
142
|
+
const residuals = [
|
|
143
|
+
'Call the customer at +1 (415) 555-2671.',
|
|
144
|
+
'Reporter SSN on file: 123-45-6789.',
|
|
145
|
+
'Medical record MRN: MED123456.',
|
|
146
|
+
'The operator claims the flake is environmental, not a product defect.',
|
|
147
|
+
'Catalog number SKU-7781 maps to the distractor fixture.',
|
|
148
|
+
];
|
|
149
|
+
return residuals.map((text, i) => ({
|
|
150
|
+
id: `corruption-${i}`,
|
|
151
|
+
text,
|
|
152
|
+
labels: [],
|
|
153
|
+
mustSurvive: true,
|
|
154
|
+
profile: 'seed',
|
|
155
|
+
}));
|
|
156
|
+
}
|
|
157
|
+
export function allCiFixtures() {
|
|
158
|
+
return [...syntheticFixtures(), ...corruptionFixtures()];
|
|
159
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Operator-local labeled corpus loader (#1968).
|
|
3
|
+
*
|
|
4
|
+
* Reads a JSONL file of EvalFixture objects from an operator-local path
|
|
5
|
+
* (default `~/.jinn-client/local-corpus/scrub-eval.jsonl`). Never ships
|
|
6
|
+
* fixture text in the metrics artifact — only counts.
|
|
7
|
+
*
|
|
8
|
+
* The mining-batch{1,2}-pii-inventory.md files are human analysis (counts +
|
|
9
|
+
* redacted examples). Operators derive synthetic/local JSONL labels from them;
|
|
10
|
+
* this loader does not parse the markdown inventories (they intentionally
|
|
11
|
+
* omit raw spans).
|
|
12
|
+
*/
|
|
13
|
+
import type { EvalFixture } from './types.js';
|
|
14
|
+
export declare const DEFAULT_LOCAL_CORPUS_PATH: string;
|
|
15
|
+
export declare function loadLocalCorpus(path?: string): EvalFixture[];
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Operator-local labeled corpus loader (#1968).
|
|
3
|
+
*
|
|
4
|
+
* Reads a JSONL file of EvalFixture objects from an operator-local path
|
|
5
|
+
* (default `~/.jinn-client/local-corpus/scrub-eval.jsonl`). Never ships
|
|
6
|
+
* fixture text in the metrics artifact — only counts.
|
|
7
|
+
*
|
|
8
|
+
* The mining-batch{1,2}-pii-inventory.md files are human analysis (counts +
|
|
9
|
+
* redacted examples). Operators derive synthetic/local JSONL labels from them;
|
|
10
|
+
* this loader does not parse the markdown inventories (they intentionally
|
|
11
|
+
* omit raw spans).
|
|
12
|
+
*/
|
|
13
|
+
import { readFileSync, existsSync } from 'node:fs';
|
|
14
|
+
import { homedir } from 'node:os';
|
|
15
|
+
import { join } from 'node:path';
|
|
16
|
+
export const DEFAULT_LOCAL_CORPUS_PATH = join(homedir(), '.jinn-client', 'local-corpus', 'scrub-eval.jsonl');
|
|
17
|
+
export function loadLocalCorpus(path = DEFAULT_LOCAL_CORPUS_PATH) {
|
|
18
|
+
if (!existsSync(path))
|
|
19
|
+
return [];
|
|
20
|
+
const raw = readFileSync(path, 'utf8');
|
|
21
|
+
const fixtures = [];
|
|
22
|
+
for (const line of raw.split('\n')) {
|
|
23
|
+
const trimmed = line.trim();
|
|
24
|
+
if (!trimmed || trimmed.startsWith('#'))
|
|
25
|
+
continue;
|
|
26
|
+
fixtures.push(JSON.parse(trimmed));
|
|
27
|
+
}
|
|
28
|
+
return fixtures;
|
|
29
|
+
}
|