@jinn-network/core 0.1.0 → 0.1.1-canary.7b188db5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/corpus-read/ipfs.d.ts +10 -1
- package/dist/corpus-read/ipfs.js +14 -5
- package/dist/scrub/apply-dispositions.d.ts +74 -0
- package/dist/scrub/apply-dispositions.js +309 -0
- package/dist/scrub/build.d.ts +54 -31
- package/dist/scrub/build.js +114 -55
- package/dist/scrub/checksummed-instruments-detector.d.ts +13 -0
- package/dist/scrub/checksummed-instruments-detector.js +119 -0
- package/dist/scrub/data/bip39-english.d.ts +2 -0
- package/dist/scrub/data/bip39-english.js +2051 -0
- package/dist/scrub/data/gitleaks-rules.d.ts +56 -0
- package/dist/scrub/data/gitleaks-rules.js +67 -0
- package/dist/scrub/emit-scrub.d.ts +13 -2
- package/dist/scrub/emit-scrub.js +39 -4
- package/dist/scrub/eval/findings-from-scrub.d.ts +18 -0
- package/dist/scrub/eval/findings-from-scrub.js +99 -0
- package/dist/scrub/eval/fixtures.d.ts +14 -0
- package/dist/scrub/eval/fixtures.js +159 -0
- package/dist/scrub/eval/index.d.ts +6 -0
- package/dist/scrub/eval/index.js +6 -0
- package/dist/scrub/eval/local-corpus.d.ts +15 -0
- package/dist/scrub/eval/local-corpus.js +29 -0
- package/dist/scrub/eval/metrics.d.ts +19 -0
- package/dist/scrub/eval/metrics.js +77 -0
- package/dist/scrub/eval/run-bench.d.ts +8 -0
- package/dist/scrub/eval/run-bench.js +125 -0
- package/dist/scrub/eval/types.d.ts +69 -0
- package/dist/scrub/eval/types.js +7 -0
- package/dist/scrub/finding.d.ts +38 -0
- package/dist/scrub/finding.js +7 -0
- package/dist/scrub/git-identity-detector.d.ts +16 -0
- package/dist/scrub/git-identity-detector.js +116 -0
- package/dist/scrub/gitleaks-detector.d.ts +13 -0
- package/dist/scrub/gitleaks-detector.js +45 -0
- package/dist/scrub/gliner-detector.d.ts +65 -0
- package/dist/scrub/gliner-detector.js +129 -0
- package/dist/scrub/index.d.ts +15 -1
- package/dist/scrub/index.js +15 -1
- package/dist/scrub/ip-address-detector.d.ts +12 -0
- package/dist/scrub/ip-address-detector.js +81 -0
- package/dist/scrub/key-policy.d.ts +18 -8
- package/dist/scrub/key-policy.js +49 -19
- package/dist/scrub/known-identity-detector.d.ts +66 -0
- package/dist/scrub/known-identity-detector.js +284 -0
- package/dist/scrub/layer2.d.ts +13 -18
- package/dist/scrub/layer2.js +22 -26
- package/dist/scrub/ml-pii-stage.d.ts +28 -10
- package/dist/scrub/ml-pii-stage.js +100 -23
- package/dist/scrub/pii-build.d.ts +17 -8
- package/dist/scrub/pii-build.js +9 -9
- package/dist/scrub/pipeline.d.ts +63 -24
- package/dist/scrub/pipeline.js +180 -38
- package/dist/scrub/plain-patterns-stage.d.ts +21 -19
- package/dist/scrub/plain-patterns-stage.js +61 -57
- package/dist/scrub/policy.d.ts +28 -0
- package/dist/scrub/policy.js +156 -0
- package/dist/scrub/provenance.d.ts +53 -0
- package/dist/scrub/provenance.js +83 -0
- package/dist/scrub/reject-classes-detector.d.ts +11 -0
- package/dist/scrub/reject-classes-detector.js +195 -0
- package/dist/scrub/reject-publish-error.d.ts +30 -0
- package/dist/scrub/reject-publish-error.js +47 -0
- package/dist/scrub/review-queue.d.ts +90 -0
- package/dist/scrub/review-queue.js +259 -0
- package/dist/scrub/secretlint-stage.d.ts +11 -26
- package/dist/scrub/secretlint-stage.js +45 -30
- package/dist/scrub/transformers-detector.d.ts +15 -9
- package/dist/scrub/transformers-detector.js +20 -10
- package/dist/scrub/types.d.ts +11 -0
- package/dist/scrub/url-credentials-detector.d.ts +9 -0
- package/dist/scrub/url-credentials-detector.js +66 -0
- package/dist/trajectory/schema.d.ts +75 -0
- package/dist/trajectory/schema.js +9 -0
- package/package.json +8 -3
- package/dist/scrub/openredaction-stage.d.ts +0 -196
- package/dist/scrub/openredaction-stage.js +0 -270
package/dist/scrub/pii-build.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { GlinerPiiDetector, DEFAULT_GLINER_MODEL, } from './gliner-detector.js';
|
|
2
2
|
/** Thrown by the fail-closed detector on every scrub call when ML PII was enabled but the model never loaded. */
|
|
3
3
|
export class MlPiiUnavailableError extends Error {
|
|
4
4
|
constructor(cause) {
|
|
@@ -24,17 +24,17 @@ function failClosedDetector(cause) {
|
|
|
24
24
|
};
|
|
25
25
|
}
|
|
26
26
|
/**
|
|
27
|
-
* Builds the ML PII detector (
|
|
28
|
-
*
|
|
27
|
+
* Builds the ML PII detector (GLiNER ONNX, in-process) when enabled, warming
|
|
28
|
+
* up the model. Default factory is {@link GlinerPiiDetector}; tests inject a
|
|
29
|
+
* mock factory so CI never hits the network.
|
|
29
30
|
*
|
|
30
31
|
* Failure altitude is **publish-time, not boot-time** (per
|
|
31
32
|
* `spec/2026-06-15-ts-trajectory-scrub-stack.md`, §"Failure posture": "if any
|
|
32
33
|
* stage errors or the model fails to load, the trajectory is **not**
|
|
33
34
|
* published"). Three cases:
|
|
34
35
|
*
|
|
35
|
-
* - Disabled (
|
|
36
|
-
*
|
|
37
|
-
* no ML PII tier and no error. Exactly the legacy behaviour.
|
|
36
|
+
* - Disabled (`enabled: false`): returns `undefined` — deterministic inventory
|
|
37
|
+
* only, no ML PII tier and no error.
|
|
38
38
|
* - Enabled and the model loads: returns the real detector.
|
|
39
39
|
* - Enabled but the model fails to load: does NOT throw at construction. Returns
|
|
40
40
|
* a fail-closed detector that hard-throws on every scrub call, so each affected
|
|
@@ -45,15 +45,15 @@ function failClosedDetector(cause) {
|
|
|
45
45
|
*/
|
|
46
46
|
export async function maybeBuildPiiDetector(cfg, log = (m) => console.warn(m), factory = {
|
|
47
47
|
create(options) {
|
|
48
|
-
return new
|
|
48
|
+
return new GlinerPiiDetector(options);
|
|
49
49
|
},
|
|
50
50
|
}) {
|
|
51
51
|
if (!cfg.enabled)
|
|
52
52
|
return undefined;
|
|
53
53
|
try {
|
|
54
|
-
const detector = factory.create(cfg.model ? { model: cfg.model } : {});
|
|
54
|
+
const detector = factory.create(cfg.model ? { model: cfg.model } : { model: DEFAULT_GLINER_MODEL });
|
|
55
55
|
await detector.init();
|
|
56
|
-
log(
|
|
56
|
+
log(`[scrub] ML PII detection (GLiNER ONNX, model=${cfg.model ?? DEFAULT_GLINER_MODEL}) enabled.`);
|
|
57
57
|
return detector;
|
|
58
58
|
}
|
|
59
59
|
catch (err) {
|
package/dist/scrub/pipeline.d.ts
CHANGED
|
@@ -1,38 +1,77 @@
|
|
|
1
|
+
import { assertNoRejectPublish, shouldRejectPublish } from './apply-dispositions.js';
|
|
2
|
+
import type { Detector } from './finding.js';
|
|
3
|
+
import { type PolicyTable } from './policy.js';
|
|
4
|
+
import { type PolicyHashInput } from './provenance.js';
|
|
5
|
+
import { type ReviewQueueStore } from './review-queue.js';
|
|
1
6
|
import type { Attributes, ScrubResult, ScrubStage } from './types.js';
|
|
7
|
+
export { shouldRejectPublish, assertNoRejectPublish };
|
|
8
|
+
export interface ScrubPipelineOptions {
|
|
9
|
+
policy?: PolicyTable;
|
|
10
|
+
/** Check-mode: any non-pass disposition rejects (design §6.5). */
|
|
11
|
+
checkMode?: boolean;
|
|
12
|
+
/**
|
|
13
|
+
* Review-queue store for flag resolutions (#1973). Defaults to the
|
|
14
|
+
* operator-local `~/.jinn-client/scrub-review/queue.jsonl` store in
|
|
15
|
+
* redact-mode. Pass an in-memory store in tests.
|
|
16
|
+
*/
|
|
17
|
+
reviewStore?: ReviewQueueStore;
|
|
18
|
+
/**
|
|
19
|
+
* When true (default in redact-mode), unresolved flags enqueue + throw
|
|
20
|
+
* {@link UnresolvedFlagError}. Check-mode leaves this off.
|
|
21
|
+
*/
|
|
22
|
+
failClosedOnUnresolvedFlags?: boolean;
|
|
23
|
+
/**
|
|
24
|
+
* Inputs for the published policy hash (#1974). When omitted, emit helpers
|
|
25
|
+
* still bump `schemaVersion` but leave `policyHash` unset.
|
|
26
|
+
*/
|
|
27
|
+
provenance?: Omit<PolicyHashInput, 'policy' | 'detectors'>;
|
|
28
|
+
}
|
|
2
29
|
/**
|
|
3
|
-
* Runs
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* the
|
|
30
|
+
* Runs one detector inventory, then one disposition pass (#1969).
|
|
31
|
+
*
|
|
32
|
+
* Nested values (#1378): after the flat detect→dispose pass, surviving
|
|
33
|
+
* object/array values are walked and every string leaf is scrubbed through
|
|
34
|
+
* the same detect→dispose path, keyed by the TOP-LEVEL attribute key.
|
|
8
35
|
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
* to pass through every stage untouched and publish verbatim, leaking the
|
|
12
|
-
* exact strings the same pipeline redacted out of `tool.result`. `run()` now
|
|
13
|
-
* walks surviving object/array values after the flat pass and scrubs every
|
|
14
|
-
* string leaf through the same stage chain, keyed by the TOP-LEVEL attribute
|
|
15
|
-
* key so the key policy classifies leaves exactly like the flat value
|
|
16
|
-
* (`jinn.*` structural attributes stay raw; `drop` keys are already gone
|
|
17
|
-
* before the walk). Redactions from leaf runs carry the top-level key, so the
|
|
18
|
-
* redaction manifest and capture redaction diff report `tool.args` as the
|
|
19
|
-
* touched field.
|
|
36
|
+
* Constructor also accepts legacy {@link ScrubStage} arrays (stage.scrub chain)
|
|
37
|
+
* so existing unit tests and injectors keep working through the migration.
|
|
20
38
|
*/
|
|
21
39
|
export declare class ScrubPipeline {
|
|
40
|
+
private readonly policy;
|
|
41
|
+
private readonly checkMode;
|
|
42
|
+
private readonly detectors;
|
|
22
43
|
private readonly stages;
|
|
23
|
-
|
|
44
|
+
private readonly reviewStore;
|
|
45
|
+
private readonly failClosedOnUnresolvedFlags;
|
|
46
|
+
private readonly provenanceExtras;
|
|
47
|
+
constructor(detectorsOrStages: Array<Detector | ScrubStage>, opts?: ScrubPipelineOptions);
|
|
24
48
|
get components(): Array<{
|
|
25
49
|
name: string;
|
|
26
50
|
version: string;
|
|
27
51
|
}>;
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
private runStages;
|
|
52
|
+
/** Disposition policy table used by this pipeline (#1974). */
|
|
53
|
+
get policyTable(): PolicyTable;
|
|
31
54
|
/**
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
* null) and non-plain objects pass through unchanged.
|
|
55
|
+
* Additive provenance fields for a redaction manifest (#1974). Always
|
|
56
|
+
* includes `schemaVersion`; `policyHash` is present when provenance
|
|
57
|
+
* extras were supplied at construction.
|
|
36
58
|
*/
|
|
59
|
+
manifestProvenance(perClassCounts?: Record<string, number>): {
|
|
60
|
+
schemaVersion: number;
|
|
61
|
+
policyHash?: string;
|
|
62
|
+
perClassCounts?: Record<string, number>;
|
|
63
|
+
};
|
|
64
|
+
/**
|
|
65
|
+
* Reproducible policy hash, or undefined when provenance inputs were not
|
|
66
|
+
* configured (legacy stage-only pipelines).
|
|
67
|
+
*/
|
|
68
|
+
policyHash(): string | undefined;
|
|
69
|
+
run(attributes: Attributes): Promise<ScrubResult>;
|
|
70
|
+
private assertAndEnqueueUnresolvedFlags;
|
|
71
|
+
/** Legacy ScrubStage chain (tests / residual injectors). */
|
|
72
|
+
private runLegacyStages;
|
|
73
|
+
private runStagesFlat;
|
|
74
|
+
private scrubNestedLegacy;
|
|
75
|
+
private runDetectDispose;
|
|
37
76
|
private scrubNested;
|
|
38
77
|
}
|
package/dist/scrub/pipeline.js
CHANGED
|
@@ -1,43 +1,153 @@
|
|
|
1
|
+
import { applyDispositions, assertNoRejectPublish, shouldRejectPublish } from './apply-dispositions.js';
|
|
2
|
+
import { DEFAULT_POLICY } from './policy.js';
|
|
3
|
+
import { REDACTION_MANIFEST_SCHEMA_VERSION, computePolicyHash, mergePerClassCounts, } from './provenance.js';
|
|
4
|
+
import { UnresolvedFlagError, createReviewQueueStore, } from './review-queue.js';
|
|
5
|
+
export { shouldRejectPublish, assertNoRejectPublish };
|
|
6
|
+
function isDetector(item) {
|
|
7
|
+
return typeof item.detect === 'function';
|
|
8
|
+
}
|
|
1
9
|
/**
|
|
2
|
-
* Runs
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* the
|
|
10
|
+
* Runs one detector inventory, then one disposition pass (#1969).
|
|
11
|
+
*
|
|
12
|
+
* Nested values (#1378): after the flat detect→dispose pass, surviving
|
|
13
|
+
* object/array values are walked and every string leaf is scrubbed through
|
|
14
|
+
* the same detect→dispose path, keyed by the TOP-LEVEL attribute key.
|
|
7
15
|
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* to pass through every stage untouched and publish verbatim, leaking the
|
|
11
|
-
* exact strings the same pipeline redacted out of `tool.result`. `run()` now
|
|
12
|
-
* walks surviving object/array values after the flat pass and scrubs every
|
|
13
|
-
* string leaf through the same stage chain, keyed by the TOP-LEVEL attribute
|
|
14
|
-
* key so the key policy classifies leaves exactly like the flat value
|
|
15
|
-
* (`jinn.*` structural attributes stay raw; `drop` keys are already gone
|
|
16
|
-
* before the walk). Redactions from leaf runs carry the top-level key, so the
|
|
17
|
-
* redaction manifest and capture redaction diff report `tool.args` as the
|
|
18
|
-
* touched field.
|
|
16
|
+
* Constructor also accepts legacy {@link ScrubStage} arrays (stage.scrub chain)
|
|
17
|
+
* so existing unit tests and injectors keep working through the migration.
|
|
19
18
|
*/
|
|
20
19
|
export class ScrubPipeline {
|
|
20
|
+
policy;
|
|
21
|
+
checkMode;
|
|
22
|
+
detectors;
|
|
21
23
|
stages;
|
|
22
|
-
|
|
23
|
-
|
|
24
|
+
reviewStore;
|
|
25
|
+
failClosedOnUnresolvedFlags;
|
|
26
|
+
provenanceExtras;
|
|
27
|
+
constructor(detectorsOrStages, opts = {}) {
|
|
28
|
+
this.policy = opts.policy ?? DEFAULT_POLICY;
|
|
29
|
+
this.checkMode = opts.checkMode ?? false;
|
|
30
|
+
this.reviewStore = opts.reviewStore;
|
|
31
|
+
this.failClosedOnUnresolvedFlags =
|
|
32
|
+
opts.failClosedOnUnresolvedFlags ?? !this.checkMode;
|
|
33
|
+
this.provenanceExtras = opts.provenance;
|
|
34
|
+
if (detectorsOrStages.length > 0 && detectorsOrStages.every(isDetector)) {
|
|
35
|
+
this.detectors = detectorsOrStages;
|
|
36
|
+
this.stages = null;
|
|
37
|
+
}
|
|
38
|
+
else {
|
|
39
|
+
this.detectors = null;
|
|
40
|
+
this.stages = detectorsOrStages;
|
|
41
|
+
}
|
|
24
42
|
}
|
|
25
43
|
get components() {
|
|
26
|
-
|
|
44
|
+
const items = this.detectors ?? this.stages ?? [];
|
|
45
|
+
return items.map((d) => ({ name: d.name, version: d.version }));
|
|
46
|
+
}
|
|
47
|
+
/** Disposition policy table used by this pipeline (#1974). */
|
|
48
|
+
get policyTable() {
|
|
49
|
+
return this.policy;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Additive provenance fields for a redaction manifest (#1974). Always
|
|
53
|
+
* includes `schemaVersion`; `policyHash` is present when provenance
|
|
54
|
+
* extras were supplied at construction.
|
|
55
|
+
*/
|
|
56
|
+
manifestProvenance(perClassCounts) {
|
|
57
|
+
const out = { schemaVersion: REDACTION_MANIFEST_SCHEMA_VERSION };
|
|
58
|
+
const hash = this.policyHash();
|
|
59
|
+
if (hash)
|
|
60
|
+
out.policyHash = hash;
|
|
61
|
+
if (perClassCounts && Object.keys(perClassCounts).length > 0) {
|
|
62
|
+
out.perClassCounts = perClassCounts;
|
|
63
|
+
}
|
|
64
|
+
return out;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Reproducible policy hash, or undefined when provenance inputs were not
|
|
68
|
+
* configured (legacy stage-only pipelines).
|
|
69
|
+
*/
|
|
70
|
+
policyHash() {
|
|
71
|
+
if (!this.provenanceExtras)
|
|
72
|
+
return undefined;
|
|
73
|
+
return computePolicyHash({
|
|
74
|
+
policy: this.policy,
|
|
75
|
+
detectors: this.components,
|
|
76
|
+
modelId: this.provenanceExtras.modelId,
|
|
77
|
+
labels: this.provenanceExtras.labels,
|
|
78
|
+
allowlistDigest: this.provenanceExtras.allowlistDigest,
|
|
79
|
+
});
|
|
27
80
|
}
|
|
28
81
|
async run(attributes) {
|
|
29
|
-
|
|
82
|
+
if (this.stages) {
|
|
83
|
+
return this.runLegacyStages(attributes);
|
|
84
|
+
}
|
|
85
|
+
const flat = await this.runDetectDispose(attributes);
|
|
30
86
|
const out = {};
|
|
31
87
|
const redactions = [...flat.redactions];
|
|
88
|
+
const findings = [...flat.findings];
|
|
89
|
+
const unresolvedFlags = [...flat.unresolvedFlags];
|
|
90
|
+
let perClassCounts = { ...flat.perClassCounts };
|
|
91
|
+
let rejected = flat.rejected;
|
|
32
92
|
for (const [key, value] of Object.entries(flat.attributes)) {
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
93
|
+
out[key] =
|
|
94
|
+
typeof value === 'string'
|
|
95
|
+
? value
|
|
96
|
+
: await this.scrubNested(key, value, redactions, findings, unresolvedFlags, (nestedCounts) => {
|
|
97
|
+
perClassCounts =
|
|
98
|
+
mergePerClassCounts(perClassCounts, nestedCounts) ?? perClassCounts;
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
// Check-mode: trust the disposition `rejected` bit. Do not re-derive from
|
|
102
|
+
// raw redactions.length — allowlist-pass records are auditable passes.
|
|
103
|
+
if (this.checkMode && flat.rejected)
|
|
104
|
+
rejected = true;
|
|
105
|
+
const result = {
|
|
106
|
+
attributes: out,
|
|
107
|
+
redactions,
|
|
108
|
+
findings,
|
|
109
|
+
rejected,
|
|
110
|
+
unresolvedFlags,
|
|
111
|
+
perClassCounts,
|
|
112
|
+
};
|
|
113
|
+
// Publish altitude: reject-publish classes abort loudly with a class name.
|
|
114
|
+
// Check-mode distill maps redactions → rejection reasons without throwing.
|
|
115
|
+
if (!this.checkMode)
|
|
116
|
+
assertNoRejectPublish(result, this.policy);
|
|
117
|
+
if (!this.checkMode && this.failClosedOnUnresolvedFlags) {
|
|
118
|
+
this.assertAndEnqueueUnresolvedFlags(result.unresolvedFlags ?? [], attributes);
|
|
119
|
+
}
|
|
120
|
+
return result;
|
|
121
|
+
}
|
|
122
|
+
assertAndEnqueueUnresolvedFlags(unresolved, attributes) {
|
|
123
|
+
if (unresolved.length === 0)
|
|
124
|
+
return;
|
|
125
|
+
// Lazy default: operator-local queue so unattended publishes leave a
|
|
126
|
+
// reviewable trail. Tests inject `reviewStore` (memory) or disable
|
|
127
|
+
// fail-closed via `failClosedOnUnresolvedFlags: false`.
|
|
128
|
+
const store = this.reviewStore ?? createReviewQueueStore();
|
|
129
|
+
const enqueued = store.enqueue(unresolved.map((finding) => {
|
|
130
|
+
const value = attributes[finding.span.key];
|
|
131
|
+
const text = typeof value === 'string' ? value : '';
|
|
132
|
+
const snippet = text.slice(Math.max(0, finding.span.start - 40), Math.min(text.length, finding.span.end + 40));
|
|
133
|
+
return {
|
|
134
|
+
finding,
|
|
135
|
+
context: { attributeKey: finding.span.key, snippet },
|
|
136
|
+
};
|
|
137
|
+
}));
|
|
138
|
+
throw new UnresolvedFlagError(unresolved, enqueued.map((e) => e.id));
|
|
139
|
+
}
|
|
140
|
+
/** Legacy ScrubStage chain (tests / residual injectors). */
|
|
141
|
+
async runLegacyStages(attributes) {
|
|
142
|
+
const flat = await this.runStagesFlat(attributes);
|
|
143
|
+
const out = {};
|
|
144
|
+
const redactions = [...flat.redactions];
|
|
145
|
+
for (const [key, value] of Object.entries(flat.attributes)) {
|
|
146
|
+
out[key] = typeof value === 'string' ? value : await this.scrubNestedLegacy(key, value, redactions);
|
|
36
147
|
}
|
|
37
148
|
return { attributes: out, redactions };
|
|
38
149
|
}
|
|
39
|
-
|
|
40
|
-
async runStages(attributes) {
|
|
150
|
+
async runStagesFlat(attributes) {
|
|
41
151
|
let current = attributes;
|
|
42
152
|
const redactions = [];
|
|
43
153
|
for (const stage of this.stages) {
|
|
@@ -47,34 +157,66 @@ export class ScrubPipeline {
|
|
|
47
157
|
}
|
|
48
158
|
return { attributes: current, redactions };
|
|
49
159
|
}
|
|
50
|
-
|
|
51
|
-
* Recursively scrub the string leaves of a nested attribute value through
|
|
52
|
-
* the full stage chain, using the top-level attribute key for policy
|
|
53
|
-
* classification. Non-container, non-string values (numbers, booleans,
|
|
54
|
-
* null) and non-plain objects pass through unchanged.
|
|
55
|
-
*/
|
|
56
|
-
async scrubNested(key, value, redactions) {
|
|
160
|
+
async scrubNestedLegacy(key, value, redactions) {
|
|
57
161
|
if (typeof value === 'string') {
|
|
58
|
-
const result = await this.
|
|
162
|
+
const result = await this.runStagesFlat({ [key]: value });
|
|
59
163
|
redactions.push(...result.redactions);
|
|
60
164
|
const scrubbed = result.attributes[key];
|
|
61
|
-
// Stages only rewrite strings; a `drop`-classified key never reaches
|
|
62
|
-
// here (dropped in the flat pass), so non-string means untouched.
|
|
63
165
|
return typeof scrubbed === 'string' ? scrubbed : value;
|
|
64
166
|
}
|
|
65
167
|
if (Array.isArray(value)) {
|
|
66
168
|
const items = [];
|
|
67
169
|
for (const item of value)
|
|
68
|
-
items.push(await this.
|
|
170
|
+
items.push(await this.scrubNestedLegacy(key, item, redactions));
|
|
171
|
+
return items;
|
|
172
|
+
}
|
|
173
|
+
if (value !== null && typeof value === 'object') {
|
|
174
|
+
const proto = Object.getPrototypeOf(value);
|
|
175
|
+
if (proto !== Object.prototype && proto !== null)
|
|
176
|
+
return value;
|
|
177
|
+
const entries = {};
|
|
178
|
+
for (const [k, v] of Object.entries(value)) {
|
|
179
|
+
entries[k] = await this.scrubNestedLegacy(key, v, redactions);
|
|
180
|
+
}
|
|
181
|
+
return entries;
|
|
182
|
+
}
|
|
183
|
+
return value;
|
|
184
|
+
}
|
|
185
|
+
async runDetectDispose(attributes) {
|
|
186
|
+
const findings = [];
|
|
187
|
+
for (const detector of this.detectors) {
|
|
188
|
+
findings.push(...(await detector.detect(attributes)));
|
|
189
|
+
}
|
|
190
|
+
return applyDispositions(attributes, findings, {
|
|
191
|
+
policy: this.policy,
|
|
192
|
+
checkMode: this.checkMode,
|
|
193
|
+
reviewStore: this.reviewStore,
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
async scrubNested(key, value, redactions, findings, unresolvedFlags, onCounts) {
|
|
197
|
+
if (typeof value === 'string') {
|
|
198
|
+
const result = await this.runDetectDispose({ [key]: value });
|
|
199
|
+
redactions.push(...result.redactions);
|
|
200
|
+
findings.push(...result.findings);
|
|
201
|
+
unresolvedFlags.push(...result.unresolvedFlags);
|
|
202
|
+
onCounts(result.perClassCounts);
|
|
203
|
+
const scrubbed = result.attributes[key];
|
|
204
|
+
return typeof scrubbed === 'string' ? scrubbed : value;
|
|
205
|
+
}
|
|
206
|
+
if (Array.isArray(value)) {
|
|
207
|
+
const items = [];
|
|
208
|
+
for (const item of value) {
|
|
209
|
+
items.push(await this.scrubNested(key, item, redactions, findings, unresolvedFlags, onCounts));
|
|
210
|
+
}
|
|
69
211
|
return items;
|
|
70
212
|
}
|
|
71
213
|
if (value !== null && typeof value === 'object') {
|
|
72
214
|
const proto = Object.getPrototypeOf(value);
|
|
73
215
|
if (proto !== Object.prototype && proto !== null)
|
|
74
|
-
return value;
|
|
216
|
+
return value;
|
|
75
217
|
const entries = {};
|
|
76
218
|
for (const [k, v] of Object.entries(value)) {
|
|
77
|
-
entries[k] = await this.scrubNested(key, v, redactions);
|
|
219
|
+
entries[k] = await this.scrubNested(key, v, redactions, findings, unresolvedFlags, onCounts);
|
|
78
220
|
}
|
|
79
221
|
return entries;
|
|
80
222
|
}
|
|
@@ -1,31 +1,33 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* plain-patterns
|
|
3
|
-
* probabilistic
|
|
4
|
-
* the harness-layer capture path in #1310):
|
|
2
|
+
* plain-patterns detector — deterministic regex findings for shapes the
|
|
3
|
+
* probabilistic detectors miss (#1330 / #1415 / #1959 / #1969):
|
|
5
4
|
*
|
|
6
|
-
* -
|
|
7
|
-
*
|
|
8
|
-
* -
|
|
9
|
-
*
|
|
5
|
+
* - email (B1)
|
|
6
|
+
* - POSIX home-directory paths (D1)
|
|
7
|
+
* - AWS/GCP credential-ID prefixes (A1)
|
|
8
|
+
* - Ethereum-style wallet addresses `0x`+40 hex (C1)
|
|
10
9
|
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
* Deliberately broad: over-redacting an email or a username is cheap; leaking
|
|
14
|
-
* one is not.
|
|
15
|
-
*
|
|
16
|
-
* An opt-in third set (`credentialIds`, #1415) adds deterministic cloud
|
|
17
|
-
* credential-ID prefixes for the seed profile, which runs without the entropy
|
|
18
|
-
* fallback that covers these shapes in the trace profile.
|
|
10
|
+
* Emits findings only — disposition owns stubs. Wallet + credential-ID shapes
|
|
11
|
+
* are always registered in the shared inventory (#1969); policy decides.
|
|
19
12
|
*/
|
|
20
13
|
import { type KeyPolicy } from './key-policy.js';
|
|
14
|
+
import type { Detector } from './finding.js';
|
|
21
15
|
import type { ScrubStage } from './types.js';
|
|
22
16
|
export interface PlainPatternsOptions {
|
|
23
17
|
/**
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
* and must stay byte-identical. The seed profile (which runs without the
|
|
27
|
-
* fallback) turns this on.
|
|
18
|
+
* @deprecated (#1969) Credential IDs are always in the shared inventory.
|
|
19
|
+
* Kept so call sites that pass `{ credentialIds: true }` still type-check.
|
|
28
20
|
*/
|
|
29
21
|
credentialIds?: boolean;
|
|
22
|
+
/**
|
|
23
|
+
* @deprecated (#1969) Wallet addresses are always in the shared inventory.
|
|
24
|
+
* Kept so call sites that pass `{ walletAddresses: true }` still type-check.
|
|
25
|
+
*/
|
|
26
|
+
walletAddresses?: boolean;
|
|
30
27
|
}
|
|
28
|
+
export declare function plainPatternsDetector(policy: KeyPolicy, _opts?: PlainPatternsOptions): Detector;
|
|
29
|
+
/**
|
|
30
|
+
* Legacy ScrubStage wrapper: detect + apply default dispositions. New callers
|
|
31
|
+
* should use {@link plainPatternsDetector} via ScrubPipeline.
|
|
32
|
+
*/
|
|
31
33
|
export declare function plainPatternsStage(policy: KeyPolicy, opts?: PlainPatternsOptions): ScrubStage;
|
|
@@ -1,36 +1,25 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* plain-patterns
|
|
3
|
-
* probabilistic
|
|
4
|
-
* the harness-layer capture path in #1310):
|
|
2
|
+
* plain-patterns detector — deterministic regex findings for shapes the
|
|
3
|
+
* probabilistic detectors miss (#1330 / #1415 / #1959 / #1969):
|
|
5
4
|
*
|
|
6
|
-
* -
|
|
7
|
-
*
|
|
8
|
-
* -
|
|
9
|
-
*
|
|
5
|
+
* - email (B1)
|
|
6
|
+
* - POSIX home-directory paths (D1)
|
|
7
|
+
* - AWS/GCP credential-ID prefixes (A1)
|
|
8
|
+
* - Ethereum-style wallet addresses `0x`+40 hex (C1)
|
|
10
9
|
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
* Deliberately broad: over-redacting an email or a username is cheap; leaking
|
|
14
|
-
* one is not.
|
|
15
|
-
*
|
|
16
|
-
* An opt-in third set (`credentialIds`, #1415) adds deterministic cloud
|
|
17
|
-
* credential-ID prefixes for the seed profile, which runs without the entropy
|
|
18
|
-
* fallback that covers these shapes in the trace profile.
|
|
10
|
+
* Emits findings only — disposition owns stubs. Wallet + credential-ID shapes
|
|
11
|
+
* are always registered in the shared inventory (#1969); policy decides.
|
|
19
12
|
*/
|
|
13
|
+
import { applyDispositions } from './apply-dispositions.js';
|
|
20
14
|
import { classifyKey } from './key-policy.js';
|
|
21
|
-
const VERSION = '0.
|
|
15
|
+
const VERSION = '0.2.0';
|
|
22
16
|
/** Plain email shape. */
|
|
23
17
|
const EMAIL_PATTERN = /[A-Za-z0-9._%+-]+@[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])+)+/g;
|
|
24
18
|
/** POSIX home-dir path segment carrying a username. */
|
|
25
19
|
const HOME_PATH_PATTERN = /\/(?:Users|home)\/[^/\s"'`]+/g;
|
|
26
|
-
const PATTERNS = [
|
|
27
|
-
{ pattern: EMAIL_PATTERN, replacement: '[EMAIL]', detail: 'email', kind: 'pii' },
|
|
28
|
-
{ pattern: HOME_PATH_PATTERN, replacement: '/users/anon', detail: 'home-path', kind: 'pii' },
|
|
29
|
-
];
|
|
30
20
|
/**
|
|
31
21
|
* Bare AWS access-key ID — the fixed four-char prefixes + 16 key chars that
|
|
32
|
-
* secretlint's aws rule only scans for under `enableIDScanRule: true
|
|
33
|
-
* preset-recommend leaves off (#1415).
|
|
22
|
+
* secretlint's aws rule only scans for under `enableIDScanRule: true`.
|
|
34
23
|
*/
|
|
35
24
|
const AWS_KEY_ID_PATTERN = /\b(?:A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}\b/g;
|
|
36
25
|
/**
|
|
@@ -38,47 +27,62 @@ const AWS_KEY_ID_PATTERN = /\b(?:A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|
|
|
|
38
27
|
* charset includes `-`, which breaks word-boundary semantics at the end.
|
|
39
28
|
*/
|
|
40
29
|
const GCP_API_KEY_PATTERN = /\bAIza[0-9A-Za-z_-]{35}/g;
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
},
|
|
48
|
-
{
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
kind: 'secret',
|
|
53
|
-
},
|
|
30
|
+
/**
|
|
31
|
+
* Ethereum-style wallet address: `0x` + exactly 40 hex chars (#1959 / C1).
|
|
32
|
+
* Bare 40-hex git SHAs and `0x`+64 tx hashes must survive.
|
|
33
|
+
*/
|
|
34
|
+
const ETH_ADDRESS_PATTERN = /\b0x[a-fA-F0-9]{40}\b/g;
|
|
35
|
+
const RULES = [
|
|
36
|
+
{ pattern: EMAIL_PATTERN, class: 'B1', evidence: 'email' },
|
|
37
|
+
{ pattern: HOME_PATH_PATTERN, class: 'D1', evidence: 'home-path' },
|
|
38
|
+
{ pattern: AWS_KEY_ID_PATTERN, class: 'A1', evidence: 'aws-access-key-id' },
|
|
39
|
+
{ pattern: GCP_API_KEY_PATTERN, class: 'A1', evidence: 'gcp-api-key' },
|
|
40
|
+
{ pattern: ETH_ADDRESS_PATTERN, class: 'C1', evidence: 'eth-address' },
|
|
54
41
|
];
|
|
55
|
-
|
|
56
|
-
const
|
|
42
|
+
function collectMatches(text, key, rule, detector) {
|
|
43
|
+
const findings = [];
|
|
44
|
+
const re = new RegExp(rule.pattern.source, rule.pattern.flags.includes('g') ? rule.pattern.flags : `${rule.pattern.flags}g`);
|
|
45
|
+
let match;
|
|
46
|
+
while ((match = re.exec(text)) !== null) {
|
|
47
|
+
findings.push({
|
|
48
|
+
class: rule.class,
|
|
49
|
+
span: { key, start: match.index, end: match.index + match[0].length },
|
|
50
|
+
confidence: 'VERY_HIGH',
|
|
51
|
+
evidence: [rule.evidence],
|
|
52
|
+
detector,
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
return findings;
|
|
56
|
+
}
|
|
57
|
+
export function plainPatternsDetector(policy, _opts = {}) {
|
|
58
|
+
const meta = { name: 'plain-patterns', version: VERSION };
|
|
57
59
|
return {
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
const out = {};
|
|
62
|
-
const redactions = [];
|
|
60
|
+
...meta,
|
|
61
|
+
detect(attributes) {
|
|
62
|
+
const findings = [];
|
|
63
63
|
for (const [key, value] of Object.entries(attributes)) {
|
|
64
|
-
if (typeof value !== 'string' || classifyKey(key, policy) !== 'content')
|
|
65
|
-
out[key] = value;
|
|
64
|
+
if (typeof value !== 'string' || classifyKey(key, policy) !== 'content')
|
|
66
65
|
continue;
|
|
66
|
+
for (const rule of RULES) {
|
|
67
|
+
findings.push(...collectMatches(value, key, rule, meta));
|
|
67
68
|
}
|
|
68
|
-
let scrubbed = value;
|
|
69
|
-
for (const { pattern, replacement, detail, kind } of patterns) {
|
|
70
|
-
let hits = 0;
|
|
71
|
-
scrubbed = scrubbed.replace(pattern, () => {
|
|
72
|
-
hits += 1;
|
|
73
|
-
return replacement;
|
|
74
|
-
});
|
|
75
|
-
for (let i = 0; i < hits; i += 1) {
|
|
76
|
-
redactions.push({ key, stage: 'plain-patterns', kind, detail });
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
out[key] = scrubbed;
|
|
80
69
|
}
|
|
81
|
-
return
|
|
70
|
+
return findings;
|
|
71
|
+
},
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Legacy ScrubStage wrapper: detect + apply default dispositions. New callers
|
|
76
|
+
* should use {@link plainPatternsDetector} via ScrubPipeline.
|
|
77
|
+
*/
|
|
78
|
+
export function plainPatternsStage(policy, opts = {}) {
|
|
79
|
+
const detector = plainPatternsDetector(policy, opts);
|
|
80
|
+
return {
|
|
81
|
+
name: detector.name,
|
|
82
|
+
version: detector.version,
|
|
83
|
+
scrub(attributes) {
|
|
84
|
+
const findings = detector.detect(attributes);
|
|
85
|
+
return applyDispositions(attributes, findings);
|
|
82
86
|
},
|
|
83
87
|
};
|
|
84
88
|
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Versioned disposition policy (#1969 / design §6.5).
|
|
3
|
+
*
|
|
4
|
+
* disposition(class, band) -> redact | reject-publish | flag | pass
|
|
5
|
+
*
|
|
6
|
+
* Defaults follow §3.2's rightmost columns. C2 stays `pass` until the flag
|
|
7
|
+
* review surface exists (do not redact 0x+64 tx hashes).
|
|
8
|
+
*/
|
|
9
|
+
import type { Band, Finding, ScrubClass } from './finding.js';
|
|
10
|
+
export type Disposition = 'redact' | 'reject-publish' | 'flag' | 'pass';
|
|
11
|
+
export declare const POLICY_VERSION = "1.0.0";
|
|
12
|
+
/** Per-band dispositions for one class. Missing bands fall back to `pass`. */
|
|
13
|
+
export type ClassDispositionRow = Partial<Record<Band, Disposition>>;
|
|
14
|
+
export interface PolicyTable {
|
|
15
|
+
version: string;
|
|
16
|
+
dispositions: Partial<Record<ScrubClass, ClassDispositionRow>>;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Default policy. High-confidence deterministic classes redact; reject classes
|
|
20
|
+
* (A4/A5) reject-publish; A2 mid-band flags; C2 passes pending flag surface.
|
|
21
|
+
*/
|
|
22
|
+
export declare const DEFAULT_POLICY: PolicyTable;
|
|
23
|
+
export declare function resolveDisposition(scrubClass: ScrubClass, band: Band, policy?: PolicyTable): Disposition;
|
|
24
|
+
/**
|
|
25
|
+
* Check-mode mapping (design §6.5): any non-`pass` disposition → reject.
|
|
26
|
+
* One mapping line — not a second pipeline.
|
|
27
|
+
*/
|
|
28
|
+
export declare function checkModeRejects(findings: Finding[], policy?: PolicyTable): boolean;
|