@deliverd/sdk 0.25.1 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +104 -0
- package/dist/adapters/index.d.ts +4 -0
- package/dist/adapters/index.d.ts.map +1 -1
- package/dist/adapters/index.js +2 -0
- package/dist/adapters/mastra.d.ts +81 -0
- package/dist/adapters/mastra.d.ts.map +1 -0
- package/dist/adapters/mastra.js +54 -0
- package/dist/adapters/muse-spark.d.ts +79 -0
- package/dist/adapters/muse-spark.d.ts.map +1 -0
- package/dist/adapters/muse-spark.js +59 -0
- package/dist/adapters/shared.d.ts +2 -1
- package/dist/adapters/shared.d.ts.map +1 -1
- package/dist/data-screen.d.ts +233 -0
- package/dist/data-screen.d.ts.map +1 -0
- package/dist/data-screen.js +395 -0
- package/dist/decision-token.d.ts +102 -0
- package/dist/decision-token.d.ts.map +1 -0
- package/dist/decision-token.js +139 -0
- package/dist/dev.d.ts.map +1 -1
- package/dist/dev.js +57 -2
- package/dist/gates.d.ts +2 -0
- package/dist/gates.d.ts.map +1 -1
- package/dist/gates.js +3 -0
- package/dist/governance.d.ts +94 -0
- package/dist/governance.d.ts.map +1 -0
- package/dist/governance.js +166 -0
- package/dist/index.d.ts +31 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +35 -0
- package/dist/messages.d.ts.map +1 -1
- package/dist/messages.js +4 -0
- package/dist/released.d.ts +1 -1
- package/dist/released.d.ts.map +1 -1
- package/dist/released.js +3803 -0
- package/dist/screen.d.ts +85 -0
- package/dist/screen.d.ts.map +1 -0
- package/dist/screen.js +69 -0
- package/dist/tasks.d.ts +38 -1
- package/dist/tasks.d.ts.map +1 -1
- package/dist/tasks.js +67 -0
- package/dist/types.d.ts +656 -4
- package/dist/types.d.ts.map +1 -1
- package/package.json +2 -2
- package/src/adapters/index.ts +11 -0
- package/src/adapters/mastra.ts +118 -0
- package/src/adapters/muse-spark.ts +147 -0
- package/src/adapters/shared.ts +2 -1
- package/src/data-screen.ts +536 -0
- package/src/decision-token.ts +184 -0
- package/src/dev.ts +63 -2
- package/src/gates.ts +3 -0
- package/src/governance.ts +204 -0
- package/src/index.ts +60 -0
- package/src/messages.ts +4 -0
- package/src/released.ts +3804 -0
- package/src/screen.ts +144 -0
- package/src/tasks.ts +88 -0
- package/src/types.ts +646 -4
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Screening what an agent is about to send to an AI model.
|
|
3
|
+
*
|
|
4
|
+
* Before a prompt, a document or a retrieved record goes to a model, this
|
|
5
|
+
* finds the personal data, secrets and confidential terms in it, says how
|
|
6
|
+
* sensitive the whole is, and decides — against what the organisation has
|
|
7
|
+
* approved that model to receive — whether it may go as it is, go with the
|
|
8
|
+
* findings replaced by placeholders, go only once a person agrees, or not go.
|
|
9
|
+
*
|
|
10
|
+
* Three lines are held throughout:
|
|
11
|
+
*
|
|
12
|
+
* - **The text need not leave the caller.** Everything here is pure and has no
|
|
13
|
+
* imports, so the SDKs run the same detectors where the agent runs and send
|
|
14
|
+
* Deliverd only counts: "two email addresses, one IBAN". This file is copied
|
|
15
|
+
* verbatim into `packages/sdk/src/data-screen.ts` and into the open-source
|
|
16
|
+
* `packages/screen/src/data-screen.ts`, and a test fails when they differ.
|
|
17
|
+
* - **A finding never carries its value off the machine.** `ScreenFinding` has
|
|
18
|
+
* offsets into the caller's own text; `summarise` keeps only counts.
|
|
19
|
+
* - **Detection is a net, not a guarantee.** Patterns find structured
|
|
20
|
+
* identifiers well and names hardly at all; a name is personal data and
|
|
21
|
+
* nothing here claims to have found every one. Pseudonymised text is still
|
|
22
|
+
* personal data in law. What this buys is fewer identifiers sent, a record of
|
|
23
|
+
* what was, and a person asked when it matters — never a certificate that a
|
|
24
|
+
* prompt is clean.
|
|
25
|
+
*/
|
|
26
|
+
/** Ordered: the stricter of two classifications is the later one. */
|
|
27
|
+
export declare const SCREEN_CLASSIFICATIONS: readonly ["public", "internal", "confidential", "restricted"];
|
|
28
|
+
export type ScreenClassification = (typeof SCREEN_CLASSIFICATIONS)[number];
|
|
29
|
+
export declare const SCREEN_CATEGORIES: readonly ["personal", "financial", "special", "secret", "confidential"];
|
|
30
|
+
export type ScreenCategory = (typeof SCREEN_CATEGORIES)[number];
|
|
31
|
+
export declare const SCREEN_VERDICTS: readonly ["allow", "redact", "ask", "deny"];
|
|
32
|
+
export type ScreenVerdict = (typeof SCREEN_VERDICTS)[number];
|
|
33
|
+
/** What a policy may do with data the target is not approved for. */
|
|
34
|
+
export type ScreenAction = "redact" | "ask" | "deny";
|
|
35
|
+
export declare const DATA_KINDS: {
|
|
36
|
+
readonly email: {
|
|
37
|
+
readonly label: "Email address";
|
|
38
|
+
readonly category: "personal";
|
|
39
|
+
readonly placeholder: "EMAIL";
|
|
40
|
+
readonly redactable: true;
|
|
41
|
+
};
|
|
42
|
+
readonly phone: {
|
|
43
|
+
readonly label: "Phone number";
|
|
44
|
+
readonly category: "personal";
|
|
45
|
+
readonly placeholder: "PHONE";
|
|
46
|
+
readonly redactable: true;
|
|
47
|
+
};
|
|
48
|
+
readonly iban: {
|
|
49
|
+
readonly label: "IBAN";
|
|
50
|
+
readonly category: "financial";
|
|
51
|
+
readonly placeholder: "IBAN";
|
|
52
|
+
readonly redactable: true;
|
|
53
|
+
};
|
|
54
|
+
readonly payment_card: {
|
|
55
|
+
readonly label: "Payment card number";
|
|
56
|
+
readonly category: "financial";
|
|
57
|
+
readonly placeholder: "CARD";
|
|
58
|
+
readonly redactable: true;
|
|
59
|
+
};
|
|
60
|
+
readonly uk_national_insurance: {
|
|
61
|
+
readonly label: "UK National Insurance number";
|
|
62
|
+
readonly category: "personal";
|
|
63
|
+
readonly placeholder: "NINO";
|
|
64
|
+
readonly redactable: true;
|
|
65
|
+
};
|
|
66
|
+
readonly ie_pps: {
|
|
67
|
+
readonly label: "Irish PPS number";
|
|
68
|
+
readonly category: "personal";
|
|
69
|
+
readonly placeholder: "PPS";
|
|
70
|
+
readonly redactable: true;
|
|
71
|
+
};
|
|
72
|
+
readonly us_ssn: {
|
|
73
|
+
readonly label: "US Social Security number";
|
|
74
|
+
readonly category: "personal";
|
|
75
|
+
readonly placeholder: "SSN";
|
|
76
|
+
readonly redactable: true;
|
|
77
|
+
};
|
|
78
|
+
readonly ip_address: {
|
|
79
|
+
readonly label: "IP address";
|
|
80
|
+
readonly category: "personal";
|
|
81
|
+
readonly placeholder: "IP";
|
|
82
|
+
readonly redactable: true;
|
|
83
|
+
};
|
|
84
|
+
readonly date_of_birth: {
|
|
85
|
+
readonly label: "Date of birth";
|
|
86
|
+
readonly category: "personal";
|
|
87
|
+
readonly placeholder: "DOB";
|
|
88
|
+
readonly redactable: true;
|
|
89
|
+
};
|
|
90
|
+
/** Never found by a pattern here: only by a model detector the caller plugs in (`spans`). */
|
|
91
|
+
readonly person_name: {
|
|
92
|
+
readonly label: "Person's name";
|
|
93
|
+
readonly category: "personal";
|
|
94
|
+
readonly placeholder: "NAME";
|
|
95
|
+
readonly redactable: true;
|
|
96
|
+
};
|
|
97
|
+
readonly special_category: {
|
|
98
|
+
readonly label: "Special-category data";
|
|
99
|
+
readonly category: "special";
|
|
100
|
+
readonly placeholder: "SPECIAL";
|
|
101
|
+
readonly redactable: false;
|
|
102
|
+
};
|
|
103
|
+
readonly secret: {
|
|
104
|
+
readonly label: "Secret or credential";
|
|
105
|
+
readonly category: "secret";
|
|
106
|
+
readonly placeholder: "SECRET";
|
|
107
|
+
readonly redactable: true;
|
|
108
|
+
};
|
|
109
|
+
readonly term: {
|
|
110
|
+
readonly label: "Confidential term";
|
|
111
|
+
readonly category: "confidential";
|
|
112
|
+
readonly placeholder: "TERM";
|
|
113
|
+
readonly redactable: true;
|
|
114
|
+
};
|
|
115
|
+
};
|
|
116
|
+
export type DataKind = keyof typeof DATA_KINDS;
|
|
117
|
+
export declare const DATA_KIND_KEYS: DataKind[];
|
|
118
|
+
/**
|
|
119
|
+
* A word or phrase the organisation does not want sent: a client's name, a
|
|
120
|
+
* project's code name, a deal. Matched whole-word and case-insensitively.
|
|
121
|
+
*/
|
|
122
|
+
export interface ScreenTerm {
|
|
123
|
+
term: string;
|
|
124
|
+
/** Shown to people and used in the placeholder: `[CLIENT_1]`. */
|
|
125
|
+
label?: string;
|
|
126
|
+
classification?: "confidential" | "restricted";
|
|
127
|
+
}
|
|
128
|
+
export interface ScreenFinding {
|
|
129
|
+
kind: DataKind;
|
|
130
|
+
category: ScreenCategory;
|
|
131
|
+
classification: ScreenClassification;
|
|
132
|
+
/** Offsets into the screened text. The value itself is never copied. */
|
|
133
|
+
start: number;
|
|
134
|
+
end: number;
|
|
135
|
+
/** For a term, its label; otherwise the kind's. */
|
|
136
|
+
label: string;
|
|
137
|
+
redactable: boolean;
|
|
138
|
+
}
|
|
139
|
+
export type ScreenCounts = Partial<Record<DataKind, number>>;
|
|
140
|
+
/** Terms past this are ignored, so a pasted list cannot make every screen slow. */
|
|
141
|
+
export declare const MAX_SCREEN_TERMS = 500;
|
|
142
|
+
/**
|
|
143
|
+
* What another detector found — a named-entity model, a DLP service — handed
|
|
144
|
+
* in as offsets into the same text. Patterns cannot find names; a model can,
|
|
145
|
+
* and this is where its answer joins theirs.
|
|
146
|
+
*/
|
|
147
|
+
export interface DetectedSpan {
|
|
148
|
+
kind: DataKind;
|
|
149
|
+
start: number;
|
|
150
|
+
end: number;
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* A detector to run beside the patterns: a function from the text to its
|
|
154
|
+
* spans. The open-source `@deliverd/screen` package has adapters that turn a
|
|
155
|
+
* Hugging Face token classifier or a Presidio analyzer into one.
|
|
156
|
+
*/
|
|
157
|
+
export type SpanDetector = (text: string) => readonly DetectedSpan[] | Promise<readonly DetectedSpan[]>;
|
|
158
|
+
/**
|
|
159
|
+
* Find what is in a text. Overlapping matches keep the stronger kind, so each
|
|
160
|
+
* character belongs to at most one finding. `spans` are another detector's
|
|
161
|
+
* findings, held to the same rules; one outside the text or of no known kind
|
|
162
|
+
* is dropped.
|
|
163
|
+
*/
|
|
164
|
+
export declare function screenText(text: string, terms?: readonly ScreenTerm[], spans?: readonly DetectedSpan[]): ScreenFinding[];
|
|
165
|
+
/** Counts only: what goes over the wire when the text stays where it is. */
|
|
166
|
+
export declare function summarise(findings: readonly ScreenFinding[]): ScreenCounts;
|
|
167
|
+
export interface Redaction {
|
|
168
|
+
text: string;
|
|
169
|
+
/** Placeholder to original value. Stays with the caller; never sent anywhere. */
|
|
170
|
+
replacements: Record<string, string>;
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* Replace each redactable finding with a placeholder. The same value gets the
|
|
174
|
+
* same placeholder throughout, so a model can still tell that two mentions are
|
|
175
|
+
* one person, and `restore` can put the values back in its answer.
|
|
176
|
+
*/
|
|
177
|
+
export declare function redact(text: string, findings: readonly ScreenFinding[]): Redaction;
|
|
178
|
+
/** Put the original values back into text a model wrote using the placeholders. */
|
|
179
|
+
export declare function restore(text: string, replacements: Readonly<Record<string, string>>): string;
|
|
180
|
+
/** What the destination model is approved to receive: the AI register's record. */
|
|
181
|
+
export interface ScreenTarget {
|
|
182
|
+
name: string;
|
|
183
|
+
/** The highest classification approved; null when the record does not say. */
|
|
184
|
+
approvedClassification: ScreenClassification | null;
|
|
185
|
+
personalData: boolean;
|
|
186
|
+
active: boolean;
|
|
187
|
+
}
|
|
188
|
+
export interface ScreenPolicy {
|
|
189
|
+
/** Secrets and credentials. Default deny: a key sent to a model is a key disclosed. */
|
|
190
|
+
secrets: ScreenAction;
|
|
191
|
+
/** Redactable data the target is not approved for. Default redact. */
|
|
192
|
+
beyondApproval: ScreenAction;
|
|
193
|
+
/** Data that redaction cannot fix (special category) the target is not approved for. Default ask. */
|
|
194
|
+
notRedactable: "ask" | "deny";
|
|
195
|
+
/** What an unnamed target is taken to be approved for. Default internal, no personal data. */
|
|
196
|
+
unregistered: {
|
|
197
|
+
approvedClassification: ScreenClassification;
|
|
198
|
+
personalData: boolean;
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
export declare const DEFAULT_SCREEN_POLICY: ScreenPolicy;
|
|
202
|
+
export interface ScreenAssessment {
|
|
203
|
+
verdict: ScreenVerdict;
|
|
204
|
+
/** The classification of what was found; public when nothing was. */
|
|
205
|
+
classification: ScreenClassification;
|
|
206
|
+
personalData: boolean;
|
|
207
|
+
categories: ScreenCategory[];
|
|
208
|
+
/** The kinds behind the verdict: what redaction replaces, and what a person is asked about. */
|
|
209
|
+
offending: DataKind[];
|
|
210
|
+
/** One sentence each, for the agent and for the person asked. */
|
|
211
|
+
reasons: string[];
|
|
212
|
+
}
|
|
213
|
+
/** Reads a stored or sent policy, keeping a default for anything missing or unknown. */
|
|
214
|
+
export declare function readScreenPolicy(raw: unknown): ScreenPolicy;
|
|
215
|
+
/** "2 email addresses and 1 IBAN", for a reason or an approval's title. */
|
|
216
|
+
export declare function describeCounts(counts: ScreenCounts): string;
|
|
217
|
+
/**
|
|
218
|
+
* Decide from counts alone, so the same answer comes back whether the text was
|
|
219
|
+
* screened here or on the caller's machine.
|
|
220
|
+
*
|
|
221
|
+
* `termClassification` is the strictest classification among the terms found,
|
|
222
|
+
* which counts cannot carry; restricted when a caller says so.
|
|
223
|
+
*/
|
|
224
|
+
export declare function assessCounts(counts: ScreenCounts, target: ScreenTarget | null, policy?: ScreenPolicy, termClassification?: ScreenClassification): ScreenAssessment;
|
|
225
|
+
/**
|
|
226
|
+
* The text with the findings behind a verdict replaced: only those, so data
|
|
227
|
+
* the target is approved for still reaches it. What a person asked is shown,
|
|
228
|
+
* and what a `redact` verdict sends. Null for a deny, which sends nothing.
|
|
229
|
+
*/
|
|
230
|
+
export declare function redactFor(text: string, findings: readonly ScreenFinding[], assessment: Pick<ScreenAssessment, "verdict" | "offending">): Redaction | null;
|
|
231
|
+
/** The strictest classification among the terms found, for `assessCounts`. */
|
|
232
|
+
export declare function termClassificationOf(findings: readonly ScreenFinding[]): ScreenClassification;
|
|
233
|
+
//# sourceMappingURL=data-screen.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"data-screen.d.ts","sourceRoot":"","sources":["../src/data-screen.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,qEAAqE;AACrE,eAAO,MAAM,sBAAsB,+DAAgE,CAAC;AACpG,MAAM,MAAM,oBAAoB,GAAG,CAAC,OAAO,sBAAsB,CAAC,CAAC,MAAM,CAAC,CAAC;AAE3E,eAAO,MAAM,iBAAiB,yEAA0E,CAAC;AACzG,MAAM,MAAM,cAAc,GAAG,CAAC,OAAO,iBAAiB,CAAC,CAAC,MAAM,CAAC,CAAC;AAEhE,eAAO,MAAM,eAAe,6CAA8C,CAAC;AAC3E,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,eAAe,CAAC,CAAC,MAAM,CAAC,CAAC;AAE7D,qEAAqE;AACrE,MAAM,MAAM,YAAY,GAAG,QAAQ,GAAG,KAAK,GAAG,MAAM,CAAC;AAerD,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAUrB,6FAA6F;;;;;;;;;;;;;;;;;;;;;;;;;CAKlD,CAAC;AAE9C,MAAM,MAAM,QAAQ,GAAG,MAAM,OAAO,UAAU,CAAC;AAC/C,eAAO,MAAM,cAAc,EAA8B,QAAQ,EAAE,CAAC;AAapE;;;GAGG;AACH,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,iEAAiE;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,cAAc,GAAG,YAAY,CAAC;CAChD;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,QAAQ,CAAC;IACf,QAAQ,EAAE,cAAc,CAAC;IACzB,cAAc,EAAE,oBAAoB,CAAC;IACrC,wEAAwE;IACxE,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,mDAAmD;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;AAsI7D,mFAAmF;AACnF,eAAO,MAAM,gBAAgB,MAAM,CAAC;AAEpC;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,QAAQ,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;;;GAIG;AACH,MAAM,MAAM,YAAY,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,SAAS,YAAY,EAAE,GAAG,OAAO,CAAC,SAAS,YAAY,EAAE,CAAC,CAAC;AAExG;;;;;GAKG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,GAAE,SAAS,UAAU,EAAO,EAAE,KAAK,GAAE,SAAS,YAAY,EAAO,GAAG,aAAa,EAAE,CA+ChI;AAED,4EAA4E;AAC5E,wBAAgB,SAAS,CAAC,QAAQ,EAAE,SAAS,aAAa,EAAE,GAAG,YAAY,CAI1E;AAID,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,iFAAiF;IACjF,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACtC;AAOD;;;;GAIG;AACH,wBAAgB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,aAAa,EAAE,GAAG,SAAS,CAuBlF;AAED,mFAAmF;AACnF,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GAAG,MAAM,CAK5F;AAID,mFAAmF;AACnF,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,8EAA8E;IAC9E,sBAAsB,EAAE,oBAAoB,GAAG,IAAI,CAAC;IACpD,YAAY,EAAE,OAAO,CAAC;IACtB,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,YAAY;IAC3B,uFAAuF;IACvF,OAAO,EAAE,YAAY,CAAC;IACtB,sEAAsE;IACtE,cAAc,EAAE,YAAY,CAAC;IAC7B,qGAAqG;IACrG,aAAa,EAAE,KAAK,GAAG,MAAM,CAAC;IAC9B,8FAA8F;IAC9F,YAAY,EAAE;QAAE,sBAAsB,EAAE,oBAAoB,CAAC;QAAC,YAAY,EAAE,OAAO,CAAA;KAAE,CAAC;CACvF;AAED,eAAO,MAAM,qBAAqB,EAAE,YAKnC,CAAC;AAEF,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,aAAa,CAAC;IACvB,qEAAqE;IACrE,cAAc,EAAE,oBAAoB,CAAC;IACrC,YAAY,EAAE,OAAO,CAAC;IACtB,UAAU,EAAE,cAAc,EAAE,CAAC;IAC7B,+FAA+F;IAC/F,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,iEAAiE;IACjE,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAMD,wFAAwF;AACxF,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,OAAO,GAAG,YAAY,CAe3D;AAID,2EAA2E;AAC3E,wBAAgB,cAAc,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CAgB3D;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAC1B,MAAM,EAAE,YAAY,EACpB,MAAM,EAAE,YAAY,GAAG,IAAI,EAC3B,MAAM,GAAE,YAAoC,EAC5C,kBAAkB,GAAE,oBAAqC,GACxD,gBAAgB,CAiDlB;AAED;;;;GAIG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,aAAa,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,gBAAgB,EAAE,SAAS,GAAG,WAAW,CAAC,GAAG,SAAS,GAAG,IAAI,CAIzJ;AAED,8EAA8E;AAC9E,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,SAAS,aAAa,EAAE,GAAG,oBAAoB,CAE7F"}
|
|
@@ -0,0 +1,395 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Screening what an agent is about to send to an AI model.
|
|
3
|
+
*
|
|
4
|
+
* Before a prompt, a document or a retrieved record goes to a model, this
|
|
5
|
+
* finds the personal data, secrets and confidential terms in it, says how
|
|
6
|
+
* sensitive the whole is, and decides — against what the organisation has
|
|
7
|
+
* approved that model to receive — whether it may go as it is, go with the
|
|
8
|
+
* findings replaced by placeholders, go only once a person agrees, or not go.
|
|
9
|
+
*
|
|
10
|
+
* Three lines are held throughout:
|
|
11
|
+
*
|
|
12
|
+
* - **The text need not leave the caller.** Everything here is pure and has no
|
|
13
|
+
* imports, so the SDKs run the same detectors where the agent runs and send
|
|
14
|
+
* Deliverd only counts: "two email addresses, one IBAN". This file is copied
|
|
15
|
+
* verbatim into `packages/sdk/src/data-screen.ts` and into the open-source
|
|
16
|
+
* `packages/screen/src/data-screen.ts`, and a test fails when they differ.
|
|
17
|
+
* - **A finding never carries its value off the machine.** `ScreenFinding` has
|
|
18
|
+
* offsets into the caller's own text; `summarise` keeps only counts.
|
|
19
|
+
* - **Detection is a net, not a guarantee.** Patterns find structured
|
|
20
|
+
* identifiers well and names hardly at all; a name is personal data and
|
|
21
|
+
* nothing here claims to have found every one. Pseudonymised text is still
|
|
22
|
+
* personal data in law. What this buys is fewer identifiers sent, a record of
|
|
23
|
+
* what was, and a person asked when it matters — never a certificate that a
|
|
24
|
+
* prompt is clean.
|
|
25
|
+
*/
|
|
26
|
+
/** Ordered: the stricter of two classifications is the later one. */
|
|
27
|
+
export const SCREEN_CLASSIFICATIONS = ["public", "internal", "confidential", "restricted"];
|
|
28
|
+
export const SCREEN_CATEGORIES = ["personal", "financial", "special", "secret", "confidential"];
|
|
29
|
+
export const SCREEN_VERDICTS = ["allow", "redact", "ask", "deny"];
|
|
30
|
+
export const DATA_KINDS = {
|
|
31
|
+
email: { label: "Email address", category: "personal", placeholder: "EMAIL", redactable: true },
|
|
32
|
+
phone: { label: "Phone number", category: "personal", placeholder: "PHONE", redactable: true },
|
|
33
|
+
iban: { label: "IBAN", category: "financial", placeholder: "IBAN", redactable: true },
|
|
34
|
+
payment_card: { label: "Payment card number", category: "financial", placeholder: "CARD", redactable: true },
|
|
35
|
+
uk_national_insurance: { label: "UK National Insurance number", category: "personal", placeholder: "NINO", redactable: true },
|
|
36
|
+
ie_pps: { label: "Irish PPS number", category: "personal", placeholder: "PPS", redactable: true },
|
|
37
|
+
us_ssn: { label: "US Social Security number", category: "personal", placeholder: "SSN", redactable: true },
|
|
38
|
+
ip_address: { label: "IP address", category: "personal", placeholder: "IP", redactable: true },
|
|
39
|
+
date_of_birth: { label: "Date of birth", category: "personal", placeholder: "DOB", redactable: true },
|
|
40
|
+
/** Never found by a pattern here: only by a model detector the caller plugs in (`spans`). */
|
|
41
|
+
person_name: { label: "Person's name", category: "personal", placeholder: "NAME", redactable: true },
|
|
42
|
+
special_category: { label: "Special-category data", category: "special", placeholder: "SPECIAL", redactable: false },
|
|
43
|
+
secret: { label: "Secret or credential", category: "secret", placeholder: "SECRET", redactable: true },
|
|
44
|
+
term: { label: "Confidential term", category: "confidential", placeholder: "TERM", redactable: true },
|
|
45
|
+
};
|
|
46
|
+
export const DATA_KIND_KEYS = Object.keys(DATA_KINDS);
|
|
47
|
+
const CATEGORY_CLASSIFICATION = {
|
|
48
|
+
personal: "confidential",
|
|
49
|
+
financial: "confidential",
|
|
50
|
+
confidential: "confidential",
|
|
51
|
+
special: "restricted",
|
|
52
|
+
secret: "restricted",
|
|
53
|
+
};
|
|
54
|
+
/** Categories that are personal data, for a target approved for none. */
|
|
55
|
+
const PERSONAL_CATEGORIES = new Set(["personal", "financial", "special"]);
|
|
56
|
+
const digitsOf = (s) => s.replace(/\D/g, "");
|
|
57
|
+
function luhn(digits) {
|
|
58
|
+
let sum = 0;
|
|
59
|
+
let double = false;
|
|
60
|
+
for (let i = digits.length - 1; i >= 0; i--) {
|
|
61
|
+
let d = digits.charCodeAt(i) - 48;
|
|
62
|
+
if (double) {
|
|
63
|
+
d *= 2;
|
|
64
|
+
if (d > 9)
|
|
65
|
+
d -= 9;
|
|
66
|
+
}
|
|
67
|
+
sum += d;
|
|
68
|
+
double = !double;
|
|
69
|
+
}
|
|
70
|
+
return sum % 10 === 0;
|
|
71
|
+
}
|
|
72
|
+
function ibanValid(raw) {
|
|
73
|
+
const iban = raw.replace(/\s/g, "").toUpperCase();
|
|
74
|
+
if (iban.length < 15 || iban.length > 34)
|
|
75
|
+
return false;
|
|
76
|
+
const moved = iban.slice(4) + iban.slice(0, 4);
|
|
77
|
+
let remainder = 0;
|
|
78
|
+
for (const ch of moved) {
|
|
79
|
+
const code = ch.charCodeAt(0);
|
|
80
|
+
const value = code >= 65 && code <= 90 ? String(code - 55) : ch;
|
|
81
|
+
for (const digit of value)
|
|
82
|
+
remainder = (remainder * 10 + (digit.charCodeAt(0) - 48)) % 97;
|
|
83
|
+
}
|
|
84
|
+
return remainder === 1;
|
|
85
|
+
}
|
|
86
|
+
/** The PPS check character: weights 8 to 2 on the digits, 9 on a second letter, mod 23, 0 is W. */
|
|
87
|
+
function ppsValid(raw) {
|
|
88
|
+
const m = /^(\d{7})([A-W])([A-IW]?)$/.exec(raw.toUpperCase());
|
|
89
|
+
if (!m)
|
|
90
|
+
return false;
|
|
91
|
+
let sum = 0;
|
|
92
|
+
for (let i = 0; i < 7; i++)
|
|
93
|
+
sum += (m[1].charCodeAt(i) - 48) * (8 - i);
|
|
94
|
+
if (m[3])
|
|
95
|
+
sum += (m[3] === "W" ? 0 : m[3].charCodeAt(0) - 64) * 9;
|
|
96
|
+
const r = sum % 23;
|
|
97
|
+
return m[2] === (r === 0 ? "W" : String.fromCharCode(64 + r));
|
|
98
|
+
}
|
|
99
|
+
function ipv4Valid(raw) {
|
|
100
|
+
const parts = raw.split(".").map(Number);
|
|
101
|
+
if (parts.length !== 4 || parts.some((p) => p > 255))
|
|
102
|
+
return false;
|
|
103
|
+
// Loopback and the unspecified address identify nobody.
|
|
104
|
+
return parts[0] !== 127 && parts[0] !== 0;
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Credential patterns. The same set the publish scanner blocks a report for,
|
|
108
|
+
* because a key that must not be published must not be sent to a model either.
|
|
109
|
+
*/
|
|
110
|
+
const SECRET_PATTERNS = [
|
|
111
|
+
/\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g,
|
|
112
|
+
/-----BEGIN (?:RSA |EC |DSA |OPENSSH |PGP )?PRIVATE KEY(?: BLOCK)?-----/g,
|
|
113
|
+
/\bgh[pousr]_[A-Za-z0-9]{36,255}\b/g,
|
|
114
|
+
/\bxox[baprs]-[A-Za-z0-9-]{10,}\b/g,
|
|
115
|
+
/\b[sr]k_(?:live|test)_[A-Za-z0-9]{20,}\b/g,
|
|
116
|
+
/\bsk-ant-[A-Za-z0-9_-]{24,}\b/g,
|
|
117
|
+
/\bsk-(?:proj-)?[A-Za-z0-9_-]{32,}\b/g,
|
|
118
|
+
/\bAIza[0-9A-Za-z_-]{35}\b/g,
|
|
119
|
+
/\beyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g,
|
|
120
|
+
/\b(?:password|passwd|secret_key|api_key|apikey|auth_token)\s*[:=]\s*["'][^"']{8,}["']/gi,
|
|
121
|
+
/\b(?:dlv|dlvk)_[A-Za-z0-9]{24,}\b/g,
|
|
122
|
+
];
|
|
123
|
+
/**
|
|
124
|
+
* Words that say a passage is about special-category data (GDPR Article 9,
|
|
125
|
+
* and criminal-offence data under Article 10). A keyword is a weak signal, so
|
|
126
|
+
* it is never redacted away: it sends the passage to a person when the target
|
|
127
|
+
* is not approved for restricted personal data.
|
|
128
|
+
*/
|
|
129
|
+
const SPECIAL_PATTERN = /\b(?:diagnos(?:is|ed|es)|medical (?:record|history|condition)s?|mental health|prescri(?:bed|ption)s?|HIV|pregnan(?:t|cy)|disabilit(?:y|ies)|sexual orientation|religious belief|ethnic origin|racial origin|trade union member(?:ship)?|biometric|genetic data|criminal (?:record|conviction|offence)s?)\b/gi;
|
|
130
|
+
const DETECTORS = [
|
|
131
|
+
...SECRET_PATTERNS.map((pattern) => ({ kind: "secret", pattern })),
|
|
132
|
+
{ kind: "iban", pattern: /\b[A-Z]{2}\d{2}(?: ?[A-Z0-9]{4}){2,7}(?: ?[A-Z0-9]{1,4})?\b/g, valid: ibanValid },
|
|
133
|
+
{
|
|
134
|
+
kind: "payment_card",
|
|
135
|
+
pattern: /\b[2-6]\d{3}(?:[ -]?\d{4}){2}[ -]?\d{1,7}\b/g,
|
|
136
|
+
valid: (m) => {
|
|
137
|
+
const d = digitsOf(m);
|
|
138
|
+
return d.length >= 13 && d.length <= 19 && luhn(d);
|
|
139
|
+
},
|
|
140
|
+
},
|
|
141
|
+
{ kind: "us_ssn", pattern: /\b(?!000|666|9\d\d)\d{3}-(?!00)\d{2}-(?!0000)\d{4}\b/g },
|
|
142
|
+
{
|
|
143
|
+
kind: "uk_national_insurance",
|
|
144
|
+
pattern: /\b(?!BG|GB|NK|KN|TN|NT|ZZ)[A-CEGHJ-PR-TW-Z][A-CEGHJ-NPR-TW-Z] ?\d{2} ?\d{2} ?\d{2} ?[A-D]\b/g,
|
|
145
|
+
},
|
|
146
|
+
{ kind: "ie_pps", pattern: /\b\d{7}[A-W][A-IW]?\b/g, valid: ppsValid },
|
|
147
|
+
{ kind: "email", pattern: /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}\b/g },
|
|
148
|
+
{
|
|
149
|
+
kind: "date_of_birth",
|
|
150
|
+
pattern: /\b(?:date of birth|d\.?o\.?b\.?|born(?: on)?)\s*[:\-]?\s*(\d{1,2}[\/.\-]\d{1,2}[\/.\-]\d{2,4}|\d{4}-\d{2}-\d{2}|\d{1,2} (?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]* \d{4})/gi,
|
|
151
|
+
group: 1,
|
|
152
|
+
},
|
|
153
|
+
{
|
|
154
|
+
kind: "phone",
|
|
155
|
+
// An international number, or a national one starting with 0: a bare run
|
|
156
|
+
// of digits is far more often an amount, an id or a date.
|
|
157
|
+
pattern: /(?<![\w+])(?:\+\d{1,3}[ .-]?(?:\(0\)[ .-]?)?|\(?0)\d{1,4}\)?(?:[ .-]?\d{2,4}){2,4}(?![\w])/g,
|
|
158
|
+
valid: (m) => {
|
|
159
|
+
const d = digitsOf(m);
|
|
160
|
+
return d.length >= 9 && d.length <= 15;
|
|
161
|
+
},
|
|
162
|
+
},
|
|
163
|
+
{ kind: "ip_address", pattern: /\b(?:\d{1,3}\.){3}\d{1,3}\b/g, valid: ipv4Valid },
|
|
164
|
+
{ kind: "special_category", pattern: SPECIAL_PATTERN },
|
|
165
|
+
];
|
|
166
|
+
/** Earlier wins an overlap: a card number is not also a phone number. */
|
|
167
|
+
const PRIORITY = Object.fromEntries(["secret", "iban", "payment_card", "us_ssn", "uk_national_insurance", "ie_pps", "email", "date_of_birth", "phone", "ip_address", "term", "person_name", "special_category"].map((k, i) => [k, i]));
|
|
168
|
+
const escapeRegExp = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
169
|
+
/** Terms past this are ignored, so a pasted list cannot make every screen slow. */
|
|
170
|
+
export const MAX_SCREEN_TERMS = 500;
|
|
171
|
+
/**
|
|
172
|
+
* Find what is in a text. Overlapping matches keep the stronger kind, so each
|
|
173
|
+
* character belongs to at most one finding. `spans` are another detector's
|
|
174
|
+
* findings, held to the same rules; one outside the text or of no known kind
|
|
175
|
+
* is dropped.
|
|
176
|
+
*/
|
|
177
|
+
export function screenText(text, terms = [], spans = []) {
|
|
178
|
+
const raw = [];
|
|
179
|
+
const add = (kind, start, end, label, classification) => {
|
|
180
|
+
const spec = DATA_KINDS[kind];
|
|
181
|
+
raw.push({
|
|
182
|
+
kind,
|
|
183
|
+
category: spec.category,
|
|
184
|
+
classification: classification ?? CATEGORY_CLASSIFICATION[spec.category],
|
|
185
|
+
start,
|
|
186
|
+
end,
|
|
187
|
+
label: label ?? spec.label,
|
|
188
|
+
redactable: spec.redactable,
|
|
189
|
+
});
|
|
190
|
+
};
|
|
191
|
+
for (const d of DETECTORS) {
|
|
192
|
+
d.pattern.lastIndex = 0;
|
|
193
|
+
for (const m of text.matchAll(d.pattern)) {
|
|
194
|
+
const value = d.group ? m[d.group] : m[0];
|
|
195
|
+
if (!value || (d.valid && !d.valid(value)))
|
|
196
|
+
continue;
|
|
197
|
+
const start = d.group ? m.index + m[0].lastIndexOf(value) : m.index;
|
|
198
|
+
add(d.kind, start, start + value.length);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
for (const t of terms.slice(0, MAX_SCREEN_TERMS)) {
|
|
202
|
+
const term = t.term.trim();
|
|
203
|
+
if (term.length < 2)
|
|
204
|
+
continue;
|
|
205
|
+
const pattern = new RegExp(`(?<![\\p{L}\\p{N}])${escapeRegExp(term)}(?![\\p{L}\\p{N}])`, "giu");
|
|
206
|
+
for (const m of text.matchAll(pattern)) {
|
|
207
|
+
add("term", m.index, m.index + m[0].length, t.label?.trim() || DATA_KINDS.term.label, t.classification ?? "confidential");
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
for (const s of spans) {
|
|
211
|
+
if (!Object.hasOwn(DATA_KINDS, s.kind) || s.kind === "term")
|
|
212
|
+
continue;
|
|
213
|
+
if (!Number.isInteger(s.start) || !Number.isInteger(s.end) || s.start < 0 || s.end > text.length || s.end <= s.start)
|
|
214
|
+
continue;
|
|
215
|
+
add(s.kind, s.start, s.end);
|
|
216
|
+
}
|
|
217
|
+
raw.sort((a, b) => PRIORITY[a.kind] - PRIORITY[b.kind] || b.end - b.start - (a.end - a.start) || a.start - b.start);
|
|
218
|
+
const kept = [];
|
|
219
|
+
for (const f of raw) {
|
|
220
|
+
if (kept.some((k) => f.start < k.end && k.start < f.end))
|
|
221
|
+
continue;
|
|
222
|
+
kept.push(f);
|
|
223
|
+
}
|
|
224
|
+
return kept.sort((a, b) => a.start - b.start);
|
|
225
|
+
}
|
|
226
|
+
/** Counts only: what goes over the wire when the text stays where it is. */
|
|
227
|
+
export function summarise(findings) {
|
|
228
|
+
const counts = {};
|
|
229
|
+
for (const f of findings)
|
|
230
|
+
counts[f.kind] = (counts[f.kind] ?? 0) + 1;
|
|
231
|
+
return counts;
|
|
232
|
+
}
|
|
233
|
+
const placeholderWord = (f) => f.kind === "term" && f.label !== DATA_KINDS.term.label
|
|
234
|
+
? f.label.toUpperCase().replace(/[^A-Z0-9]+/g, "_").replace(/^_|_$/g, "").slice(0, 24) || "TERM"
|
|
235
|
+
: DATA_KINDS[f.kind].placeholder;
|
|
236
|
+
/**
|
|
237
|
+
* Replace each redactable finding with a placeholder. The same value gets the
|
|
238
|
+
* same placeholder throughout, so a model can still tell that two mentions are
|
|
239
|
+
* one person, and `restore` can put the values back in its answer.
|
|
240
|
+
*/
|
|
241
|
+
export function redact(text, findings) {
|
|
242
|
+
const replacements = {};
|
|
243
|
+
const byValue = new Map();
|
|
244
|
+
const counters = new Map();
|
|
245
|
+
let out = "";
|
|
246
|
+
let at = 0;
|
|
247
|
+
for (const f of [...findings].sort((a, b) => a.start - b.start)) {
|
|
248
|
+
if (!f.redactable || f.start < at)
|
|
249
|
+
continue;
|
|
250
|
+
const value = text.slice(f.start, f.end);
|
|
251
|
+
const key = `${f.kind}\u0000${value.toLowerCase()}`;
|
|
252
|
+
let placeholder = byValue.get(key);
|
|
253
|
+
if (!placeholder) {
|
|
254
|
+
const word = placeholderWord(f);
|
|
255
|
+
const n = (counters.get(word) ?? 0) + 1;
|
|
256
|
+
counters.set(word, n);
|
|
257
|
+
placeholder = `[${word}_${n}]`;
|
|
258
|
+
byValue.set(key, placeholder);
|
|
259
|
+
replacements[placeholder] = value;
|
|
260
|
+
}
|
|
261
|
+
out += text.slice(at, f.start) + placeholder;
|
|
262
|
+
at = f.end;
|
|
263
|
+
}
|
|
264
|
+
return { text: out + text.slice(at), replacements };
|
|
265
|
+
}
|
|
266
|
+
/** Put the original values back into text a model wrote using the placeholders. */
|
|
267
|
+
export function restore(text, replacements) {
|
|
268
|
+
const keys = Object.keys(replacements).sort((a, b) => b.length - a.length);
|
|
269
|
+
if (keys.length === 0)
|
|
270
|
+
return text;
|
|
271
|
+
const pattern = new RegExp(keys.map(escapeRegExp).join("|"), "g");
|
|
272
|
+
return text.replace(pattern, (p) => replacements[p] ?? p);
|
|
273
|
+
}
|
|
274
|
+
export const DEFAULT_SCREEN_POLICY = {
|
|
275
|
+
secrets: "deny",
|
|
276
|
+
beyondApproval: "redact",
|
|
277
|
+
notRedactable: "ask",
|
|
278
|
+
unregistered: { approvedClassification: "internal", personalData: false },
|
|
279
|
+
};
|
|
280
|
+
const rank = (c) => SCREEN_CLASSIFICATIONS.indexOf(c);
|
|
281
|
+
const VERDICT_RANK = { allow: 0, redact: 1, ask: 2, deny: 3 };
|
|
282
|
+
const stricter = (a, b) => (VERDICT_RANK[b] > VERDICT_RANK[a] ? b : a);
|
|
283
|
+
/** Reads a stored or sent policy, keeping a default for anything missing or unknown. */
|
|
284
|
+
export function readScreenPolicy(raw) {
|
|
285
|
+
const p = (raw && typeof raw === "object" ? raw : {});
|
|
286
|
+
const actions = ["redact", "ask", "deny"];
|
|
287
|
+
const pick = (v, allowed, fallback) => typeof v === "string" && allowed.includes(v) ? v : fallback;
|
|
288
|
+
const u = (p.unregistered && typeof p.unregistered === "object" ? p.unregistered : {});
|
|
289
|
+
return {
|
|
290
|
+
secrets: pick(p.secrets, actions, DEFAULT_SCREEN_POLICY.secrets),
|
|
291
|
+
beyondApproval: pick(p.beyondApproval, actions, DEFAULT_SCREEN_POLICY.beyondApproval),
|
|
292
|
+
notRedactable: pick(p.notRedactable, ["ask", "deny"], DEFAULT_SCREEN_POLICY.notRedactable),
|
|
293
|
+
unregistered: {
|
|
294
|
+
approvedClassification: pick(u.approvedClassification, SCREEN_CLASSIFICATIONS, DEFAULT_SCREEN_POLICY.unregistered.approvedClassification),
|
|
295
|
+
personalData: typeof u.personalData === "boolean" ? u.personalData : DEFAULT_SCREEN_POLICY.unregistered.personalData,
|
|
296
|
+
},
|
|
297
|
+
};
|
|
298
|
+
}
|
|
299
|
+
const plural = (n, label) => `${n} ${label.toLowerCase()}${n === 1 ? "" : "s"}`;
|
|
300
|
+
/** "2 email addresses and 1 IBAN", for a reason or an approval's title. */
|
|
301
|
+
export function describeCounts(counts) {
|
|
302
|
+
const parts = DATA_KIND_KEYS.filter((k) => (counts[k] ?? 0) > 0).map((k) => {
|
|
303
|
+
const n = counts[k];
|
|
304
|
+
const label = DATA_KINDS[k].label;
|
|
305
|
+
if (k === "email")
|
|
306
|
+
return `${n} email address${n === 1 ? "" : "es"}`;
|
|
307
|
+
if (k === "special_category")
|
|
308
|
+
return `${n} mention${n === 1 ? "" : "s"} of special-category data`;
|
|
309
|
+
if (k === "secret")
|
|
310
|
+
return `${n} secret${n === 1 ? "" : "s"} or credential${n === 1 ? "" : "s"}`;
|
|
311
|
+
if (k === "term")
|
|
312
|
+
return plural(n, "confidential term");
|
|
313
|
+
if (k === "person_name")
|
|
314
|
+
return `${n} ${n === 1 ? "person's name" : "people's names"}`;
|
|
315
|
+
if (k === "ip_address")
|
|
316
|
+
return `${n} IP address${n === 1 ? "" : "es"}`;
|
|
317
|
+
if (k === "date_of_birth")
|
|
318
|
+
return `${n} date${n === 1 ? "" : "s"} of birth`;
|
|
319
|
+
if (k === "ie_pps")
|
|
320
|
+
return `${n} Irish PPS number${n === 1 ? "" : "s"}`;
|
|
321
|
+
return /^[A-Z]{2,}/.test(label) ? `${n} ${label}${n === 1 ? "" : "s"}` : plural(n, label);
|
|
322
|
+
});
|
|
323
|
+
if (parts.length <= 1)
|
|
324
|
+
return parts[0] ?? "nothing";
|
|
325
|
+
return `${parts.slice(0, -1).join(", ")} and ${parts.at(-1)}`;
|
|
326
|
+
}
|
|
327
|
+
/**
|
|
328
|
+
* Decide from counts alone, so the same answer comes back whether the text was
|
|
329
|
+
* screened here or on the caller's machine.
|
|
330
|
+
*
|
|
331
|
+
* `termClassification` is the strictest classification among the terms found,
|
|
332
|
+
* which counts cannot carry; restricted when a caller says so.
|
|
333
|
+
*/
|
|
334
|
+
export function assessCounts(counts, target, policy = DEFAULT_SCREEN_POLICY, termClassification = "confidential") {
|
|
335
|
+
const present = DATA_KIND_KEYS.filter((k) => (counts[k] ?? 0) > 0);
|
|
336
|
+
const categories = SCREEN_CATEGORIES.filter((c) => present.some((k) => DATA_KINDS[k].category === c));
|
|
337
|
+
const classOf = (k) => (k === "term" ? termClassification : CATEGORY_CLASSIFICATION[DATA_KINDS[k].category]);
|
|
338
|
+
const classification = present.reduce((acc, k) => (rank(classOf(k)) > rank(acc) ? classOf(k) : acc), "public");
|
|
339
|
+
const personalData = categories.some((c) => PERSONAL_CATEGORIES.has(c));
|
|
340
|
+
const reasons = [];
|
|
341
|
+
const offending = [];
|
|
342
|
+
let verdict = "allow";
|
|
343
|
+
if (target && !target.active) {
|
|
344
|
+
return {
|
|
345
|
+
verdict: "deny",
|
|
346
|
+
classification,
|
|
347
|
+
personalData,
|
|
348
|
+
categories,
|
|
349
|
+
offending: present,
|
|
350
|
+
reasons: [`${target.name} is not an active system in the AI register, so nothing should be sent to it.`],
|
|
351
|
+
};
|
|
352
|
+
}
|
|
353
|
+
if (present.length === 0)
|
|
354
|
+
return { verdict, classification, personalData, categories, offending, reasons };
|
|
355
|
+
const name = target?.name ?? "an AI system that is not in the register";
|
|
356
|
+
const approved = target ? target.approvedClassification ?? policy.unregistered.approvedClassification : policy.unregistered.approvedClassification;
|
|
357
|
+
const personalApproved = target ? target.personalData : policy.unregistered.personalData;
|
|
358
|
+
const secrets = counts.secret ?? 0;
|
|
359
|
+
if (secrets > 0) {
|
|
360
|
+
verdict = stricter(verdict, policy.secrets);
|
|
361
|
+
offending.push("secret");
|
|
362
|
+
reasons.push(`It contains ${describeCounts({ secret: secrets })}, which should never be sent to a model.`);
|
|
363
|
+
}
|
|
364
|
+
for (const k of present) {
|
|
365
|
+
if (k === "secret")
|
|
366
|
+
continue;
|
|
367
|
+
const tooSensitive = rank(classOf(k)) > rank(approved);
|
|
368
|
+
const personalNotApproved = DATA_KINDS[k].category !== "confidential" && !personalApproved;
|
|
369
|
+
if (!tooSensitive && !personalNotApproved)
|
|
370
|
+
continue;
|
|
371
|
+
const action = DATA_KINDS[k].redactable ? policy.beyondApproval : policy.notRedactable;
|
|
372
|
+
verdict = stricter(verdict, action);
|
|
373
|
+
offending.push(k);
|
|
374
|
+
const why = personalNotApproved
|
|
375
|
+
? "which is not approved to receive personal data"
|
|
376
|
+
: `which is approved for ${approved} data at most`;
|
|
377
|
+
reasons.push(`It contains ${describeCounts({ [k]: counts[k] })}, and goes to ${name}, ${why}.`);
|
|
378
|
+
}
|
|
379
|
+
return { verdict, classification, personalData, categories, offending, reasons };
|
|
380
|
+
}
|
|
381
|
+
/**
|
|
382
|
+
* The text with the findings behind a verdict replaced: only those, so data
|
|
383
|
+
* the target is approved for still reaches it. What a person asked is shown,
|
|
384
|
+
* and what a `redact` verdict sends. Null for a deny, which sends nothing.
|
|
385
|
+
*/
|
|
386
|
+
export function redactFor(text, findings, assessment) {
|
|
387
|
+
if (assessment.verdict === "deny")
|
|
388
|
+
return null;
|
|
389
|
+
const offending = new Set(assessment.offending);
|
|
390
|
+
return redact(text, findings.filter((f) => offending.has(f.kind)));
|
|
391
|
+
}
|
|
392
|
+
/** The strictest classification among the terms found, for `assessCounts`. */
|
|
393
|
+
export function termClassificationOf(findings) {
|
|
394
|
+
return findings.some((f) => f.kind === "term" && f.classification === "restricted") ? "restricted" : "confidential";
|
|
395
|
+
}
|