@guhttps/veu 0.5.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -3
- package/dist/analyzer.d.ts +10 -0
- package/dist/analyzer.js +49 -1
- package/dist/anonymizer.d.ts +16 -4
- package/dist/anonymizer.js +127 -25
- package/dist/cache.d.ts +17 -0
- package/dist/cache.js +48 -0
- package/dist/data.d.ts +10 -0
- package/dist/data.js +130 -4
- package/dist/entities.d.ts +2 -0
- package/dist/entities.js +34 -0
- package/dist/gateway.d.ts +7 -4
- package/dist/gateway.js +71 -20
- package/dist/index.d.ts +7 -5
- package/dist/index.js +7 -5
- package/dist/patterns.js +129 -3
- package/dist/persons.d.ts +3 -0
- package/dist/persons.js +21 -3
- package/dist/validators.d.ts +34 -0
- package/dist/validators.js +346 -0
- package/dist/vault.d.ts +66 -7
- package/dist/vault.js +195 -27
- package/dist/veu.global.js +23 -20
- package/package.json +2 -2
package/dist/entities.d.ts
CHANGED
|
@@ -18,3 +18,5 @@ export interface Recognizer {
|
|
|
18
18
|
labels: ReadonlySet<string>;
|
|
19
19
|
find(text: string): Iterable<Candidate>;
|
|
20
20
|
}
|
|
21
|
+
/** Grupos de tipos para caixinhas de escolha (os mesmos do painel): cada tipo fica em exatamente um grupo. */
|
|
22
|
+
export { groups as GROUPS, type GroupData } from "./data.ts";
|
package/dist/entities.js
CHANGED
|
@@ -22,6 +22,38 @@ export const LABELS = {
|
|
|
22
22
|
IP: "Endereço IP",
|
|
23
23
|
PROCESSO: "Número de processo judicial (CNJ)",
|
|
24
24
|
IBAN: "IBAN (conta bancária internacional)",
|
|
25
|
+
NIF_PT: "NIF ou NIPC (Portugal)",
|
|
26
|
+
CC_PT: "Cartão de Cidadão (Portugal)",
|
|
27
|
+
NISS_PT: "Número da Segurança Social (Portugal)",
|
|
28
|
+
CUIT: "CUIT ou CUIL (Argentina)",
|
|
29
|
+
DNI: "DNI (Argentina, Peru)",
|
|
30
|
+
CURP: "CURP (México)",
|
|
31
|
+
RFC: "RFC (México)",
|
|
32
|
+
RUT: "RUT ou RUN (Chile, Uruguai)",
|
|
33
|
+
CEDULA: "Cédula de identidade (Uruguai, Equador, Colômbia, Venezuela)",
|
|
34
|
+
NIT: "NIT (Colômbia)",
|
|
35
|
+
RUC: "RUC (Peru, Equador, Paraguai)",
|
|
36
|
+
RIF: "RIF (Venezuela)",
|
|
37
|
+
NUIT: "NUIT (Moçambique)",
|
|
38
|
+
BI_AO: "Bilhete de Identidade (Angola)",
|
|
39
|
+
BI_MZ: "Bilhete de Identidade (Moçambique)",
|
|
40
|
+
SSN: "SSN ou ITIN (Estados Unidos)",
|
|
41
|
+
MBI: "Medicare (Estados Unidos)",
|
|
42
|
+
DNI_ES: "DNI ou NIE (Espanha)",
|
|
43
|
+
CODICE_FISCALE: "Codice fiscale (Itália)",
|
|
44
|
+
NIR: "Número da segurança social (França)",
|
|
45
|
+
STEUER_ID: "Steuer-ID (Alemanha)",
|
|
46
|
+
NINO: "National Insurance (Reino Unido)",
|
|
47
|
+
NHS: "NHS (Reino Unido)",
|
|
48
|
+
BSN: "BSN (Países Baixos)",
|
|
49
|
+
SIN: "SIN (Canadá)",
|
|
50
|
+
AADHAAR: "Aadhaar (Índia)",
|
|
51
|
+
PAN: "PAN (Índia)",
|
|
52
|
+
MRZ: "Zona de leitura de passaporte (MRZ)",
|
|
53
|
+
VIN: "Chassi (VIN)",
|
|
54
|
+
CRIPTO: "Carteira de criptomoeda (Bitcoin, Ethereum)",
|
|
55
|
+
PORTA: "Porta de rede",
|
|
56
|
+
SERVIDOR: "Nome de servidor interno",
|
|
25
57
|
CHAVE_API: "Chave de API ou token de serviço (OpenAI, AWS, GitHub...)",
|
|
26
58
|
TOKEN: "Token de acesso (JWT, Bearer)",
|
|
27
59
|
SENHA: "Senha",
|
|
@@ -32,3 +64,5 @@ export const LABELS = {
|
|
|
32
64
|
export const SECRET_LABELS = new Set([
|
|
33
65
|
"CHAVE_API", "TOKEN", "SENHA", "CHAVE_PRIVADA", "URL_CONEXAO", "SEGREDO",
|
|
34
66
|
]);
|
|
67
|
+
/** Grupos de tipos para caixinhas de escolha (os mesmos do painel): cada tipo fica em exatamente um grupo. */
|
|
68
|
+
export { groups as GROUPS } from "./data.js";
|
package/dist/gateway.d.ts
CHANGED
|
@@ -14,18 +14,21 @@ export declare class Masker {
|
|
|
14
14
|
jsonString(value: Json): Json;
|
|
15
15
|
obj(value: Json): Json;
|
|
16
16
|
}
|
|
17
|
-
export declare function restoreObj(value: Json, vault: Vault): Json;
|
|
18
|
-
|
|
17
|
+
export declare function restoreObj(value: Json, vault: Vault, tool?: boolean, toolName?: string): Json;
|
|
18
|
+
/** Argumentos de ferramenta em texto JSON: restaura os valores sem quebrar o JSON. */
|
|
19
|
+
export declare function restoreJsonString(value: string, vault: Vault, tool?: boolean, toolName?: string): string;
|
|
19
20
|
/** Um restaurador por campo de texto do stream (cada escolha, bloco ou argumento de ferramenta). */
|
|
20
21
|
export declare class StreamState {
|
|
21
22
|
readonly vault: Vault;
|
|
22
23
|
readonly meta: Record<string, Json>;
|
|
24
|
+
readonly toolNames: Map<string, string>;
|
|
23
25
|
private restorers;
|
|
24
26
|
constructor(vault: Vault);
|
|
25
|
-
|
|
27
|
+
/** tool: o texto é argumento de ferramenta (padrão: igual a jsonEscape). */
|
|
28
|
+
feed(key: Json[], text: string, jsonEscape?: boolean, tool?: boolean): string;
|
|
26
29
|
flush(key: Json[]): string;
|
|
27
30
|
pendingKeys(): Json[][];
|
|
28
|
-
restore(value: Json): Json;
|
|
31
|
+
restore(value: Json, toolName?: string): Json;
|
|
29
32
|
}
|
|
30
33
|
export declare abstract class Adapter {
|
|
31
34
|
abstract name: string;
|
package/dist/gateway.js
CHANGED
|
@@ -43,48 +43,70 @@ export class Masker {
|
|
|
43
43
|
return value;
|
|
44
44
|
}
|
|
45
45
|
}
|
|
46
|
-
|
|
46
|
+
// Onde ficam os argumentos de chamadas de ferramenta nas respostas (OpenAI, Responses, Anthropic, Gemini, Ollama).
|
|
47
|
+
const TOOL_KEYS = new Set(["arguments", "partial_json", "args", "functionCall", "function_call", "tool_calls"]);
|
|
48
|
+
const TOOL_TYPES = new Set(["tool_use", "server_tool_use", "function_call", "custom_tool_call", "local_shell_call",
|
|
49
|
+
"response.custom_tool_call_input.done", "response.function_call_arguments.done"]);
|
|
50
|
+
/** Restaura todos os textos de uma resposta JSON. tool: o valor é argumento de uma chamada de ferramenta. */
|
|
51
|
+
/** O nome da ferramenta de uma chamada: {name, arguments|input|args} em todas as APIs. */
|
|
52
|
+
function toolNameOf(value, inherited) {
|
|
53
|
+
if (value.type === "local_shell_call")
|
|
54
|
+
return "local_shell";
|
|
55
|
+
if (typeof value.name === "string" && ("arguments" in value || "input" in value || "args" in value))
|
|
56
|
+
return value.name;
|
|
57
|
+
return inherited;
|
|
58
|
+
}
|
|
59
|
+
export function restoreObj(value, vault, tool = false, toolName) {
|
|
47
60
|
if (typeof value === "string")
|
|
48
|
-
return vault.restore(value);
|
|
61
|
+
return vault.restore(value, false, { tool, toolName: tool ? toolName : undefined });
|
|
49
62
|
if (Array.isArray(value))
|
|
50
|
-
return value.map((item) => restoreObj(item, vault));
|
|
63
|
+
return value.map((item) => restoreObj(item, vault, tool, toolName));
|
|
51
64
|
if (isObject(value)) {
|
|
52
65
|
if (UNTOUCHED_TYPES.has(value.type))
|
|
53
66
|
return value;
|
|
67
|
+
const call = TOOL_TYPES.has(value.type);
|
|
68
|
+
const name = toolNameOf(value, toolName);
|
|
54
69
|
const restored = {};
|
|
55
70
|
for (const [key, item] of Object.entries(value)) {
|
|
71
|
+
const inTool = tool || TOOL_KEYS.has(key) || (call && (key === "input" || key === "arguments"));
|
|
56
72
|
if (UNTOUCHED_KEYS.has(key))
|
|
57
73
|
restored[key] = item;
|
|
58
74
|
else if ((key === "arguments" || key === "partial_json") && typeof item === "string")
|
|
59
|
-
restored[key] = restoreJsonString(item, vault);
|
|
75
|
+
restored[key] = restoreJsonString(item, vault, true, name);
|
|
60
76
|
else
|
|
61
|
-
restored[key] = restoreObj(item, vault);
|
|
77
|
+
restored[key] = restoreObj(item, vault, inTool, name);
|
|
62
78
|
}
|
|
63
79
|
return restored;
|
|
64
80
|
}
|
|
65
81
|
return value;
|
|
66
82
|
}
|
|
67
|
-
|
|
83
|
+
/** Argumentos de ferramenta em texto JSON: restaura os valores sem quebrar o JSON. */
|
|
84
|
+
export function restoreJsonString(value, vault, tool = true, toolName) {
|
|
85
|
+
let parsed;
|
|
68
86
|
try {
|
|
69
|
-
|
|
87
|
+
parsed = JSON.parse(value);
|
|
70
88
|
}
|
|
71
89
|
catch {
|
|
72
|
-
return vault.restore(value, true);
|
|
90
|
+
return vault.restore(value, true, { tool, toolName });
|
|
73
91
|
}
|
|
92
|
+
return JSON.stringify(restoreObj(parsed, vault, tool, toolName));
|
|
74
93
|
}
|
|
75
94
|
/** Um restaurador por campo de texto do stream (cada escolha, bloco ou argumento de ferramenta). */
|
|
76
95
|
export class StreamState {
|
|
77
96
|
vault;
|
|
78
97
|
meta = {};
|
|
98
|
+
// Nome da ferramenta de cada campo de argumentos do stream (chega antes dos argumentos, num evento à parte).
|
|
99
|
+
toolNames = new Map();
|
|
79
100
|
restorers = new Map();
|
|
80
101
|
constructor(vault) {
|
|
81
102
|
this.vault = vault;
|
|
82
103
|
}
|
|
83
|
-
|
|
104
|
+
/** tool: o texto é argumento de ferramenta (padrão: igual a jsonEscape). */
|
|
105
|
+
feed(key, text, jsonEscape = false, tool) {
|
|
84
106
|
const id = JSON.stringify(key);
|
|
85
107
|
let entry = this.restorers.get(id);
|
|
86
108
|
if (!entry) {
|
|
87
|
-
entry = { key, restorer: this.vault.restorer(jsonEscape) };
|
|
109
|
+
entry = { key, restorer: this.vault.restorer(jsonEscape, tool, this.toolNames.get(id)) };
|
|
88
110
|
this.restorers.set(id, entry);
|
|
89
111
|
}
|
|
90
112
|
return entry.restorer.feed(text);
|
|
@@ -98,8 +120,8 @@ export class StreamState {
|
|
|
98
120
|
pendingKeys() {
|
|
99
121
|
return [...this.restorers.values()].filter((entry) => entry.restorer.pending).map((entry) => entry.key);
|
|
100
122
|
}
|
|
101
|
-
restore(value) {
|
|
102
|
-
return restoreObj(value, this.vault);
|
|
123
|
+
restore(value, toolName) {
|
|
124
|
+
return restoreObj(value, this.vault, false, toolName);
|
|
103
125
|
}
|
|
104
126
|
}
|
|
105
127
|
export class Adapter {
|
|
@@ -175,8 +197,11 @@ export class OpenAIChatAdapter extends Adapter {
|
|
|
175
197
|
delta[field] = state.feed(["c", index, field], delta[field]);
|
|
176
198
|
}
|
|
177
199
|
for (const call of delta.tool_calls ?? []) {
|
|
200
|
+
const key = ["t", index, call?.index ?? 0];
|
|
201
|
+
if (typeof call?.function?.name === "string")
|
|
202
|
+
state.toolNames.set(JSON.stringify(key), call.function.name);
|
|
178
203
|
if (typeof call?.function?.arguments === "string") {
|
|
179
|
-
call.function.arguments = state.feed(
|
|
204
|
+
call.function.arguments = state.feed(key, call.function.arguments, true);
|
|
180
205
|
}
|
|
181
206
|
}
|
|
182
207
|
if (choice.finish_reason)
|
|
@@ -291,8 +316,13 @@ export class OpenAIResponsesAdapter extends Adapter {
|
|
|
291
316
|
item.content = maskContent(item.content, masker);
|
|
292
317
|
if (item.type === "function_call")
|
|
293
318
|
item.arguments = masker.jsonString(item.arguments);
|
|
294
|
-
else if (
|
|
295
|
-
item.output =
|
|
319
|
+
else if (["function_call_output", "custom_tool_call_output", "local_shell_call_output"].includes(item.type)) {
|
|
320
|
+
item.output = maskContent(item.output, masker);
|
|
321
|
+
}
|
|
322
|
+
else if (item.type === "custom_tool_call")
|
|
323
|
+
item.input = masker.text(item.input); // apply_patch do Codex
|
|
324
|
+
else if (item.type === "local_shell_call" && isObject(item.action))
|
|
325
|
+
item.action = masker.obj(item.action);
|
|
296
326
|
}
|
|
297
327
|
}
|
|
298
328
|
instruct(payload, instruction) {
|
|
@@ -303,6 +333,13 @@ export class OpenAIResponsesAdapter extends Adapter {
|
|
|
303
333
|
if (!isObject(data))
|
|
304
334
|
return [[name, data]];
|
|
305
335
|
const kind = data.type;
|
|
336
|
+
if (kind === "response.output_item.added" && isObject(data.item)) {
|
|
337
|
+
const name = data.item.type === "local_shell_call" ? "local_shell" : data.item.name;
|
|
338
|
+
if (typeof name === "string") {
|
|
339
|
+
for (const prefix of ["f", "c"])
|
|
340
|
+
state.toolNames.set(JSON.stringify([prefix, data.output_index ?? 0]), name);
|
|
341
|
+
}
|
|
342
|
+
}
|
|
306
343
|
if (kind === "response.output_text.delta") {
|
|
307
344
|
data.delta = state.feed(["o", data.output_index ?? 0, data.content_index ?? 0], data.delta ?? "");
|
|
308
345
|
return [[name, data]];
|
|
@@ -311,17 +348,24 @@ export class OpenAIResponsesAdapter extends Adapter {
|
|
|
311
348
|
data.delta = state.feed(["f", data.output_index ?? 0], data.delta ?? "", true);
|
|
312
349
|
return [[name, data]];
|
|
313
350
|
}
|
|
314
|
-
if (kind === "response.
|
|
351
|
+
if (kind === "response.custom_tool_call_input.delta") {
|
|
352
|
+
data.delta = state.feed(["c", data.output_index ?? 0], data.delta ?? "", false, true);
|
|
353
|
+
return [[name, data]];
|
|
354
|
+
}
|
|
355
|
+
if (kind === "response.output_text.done" || kind === "response.function_call_arguments.done"
|
|
356
|
+
|| kind === "response.custom_tool_call_input.done") {
|
|
315
357
|
const text = kind === "response.output_text.done";
|
|
316
|
-
const
|
|
317
|
-
const
|
|
358
|
+
const custom = kind === "response.custom_tool_call_input.done";
|
|
359
|
+
const key = text ? ["o", data.output_index ?? 0, data.content_index ?? 0] : [custom ? "c" : "f", data.output_index ?? 0];
|
|
360
|
+
const deltaKind = text ? "response.output_text.delta"
|
|
361
|
+
: custom ? "response.custom_tool_call_input.delta" : "response.function_call_arguments.delta";
|
|
318
362
|
const events = [];
|
|
319
363
|
const tail = state.flush(key);
|
|
320
364
|
if (tail) {
|
|
321
365
|
const extra = template(data, ["item_id", "output_index", "content_index"]);
|
|
322
366
|
events.push([name ? deltaKind : null, { type: deltaKind, ...extra, delta: tail }]);
|
|
323
367
|
}
|
|
324
|
-
events.push([name, state.restore(data)]);
|
|
368
|
+
events.push([name, state.restore(data, state.toolNames.get(JSON.stringify(key)))]);
|
|
325
369
|
return events;
|
|
326
370
|
}
|
|
327
371
|
if (typeof kind === "string" && kind.includes("reasoning"))
|
|
@@ -337,6 +381,9 @@ export class OpenAIResponsesAdapter extends Adapter {
|
|
|
337
381
|
if (key[0] === "o") {
|
|
338
382
|
events.push([null, { type: "response.output_text.delta", output_index: key[1], content_index: key[2], delta: tail }]);
|
|
339
383
|
}
|
|
384
|
+
else if (key[0] === "c") {
|
|
385
|
+
events.push([null, { type: "response.custom_tool_call_input.delta", output_index: key[1], delta: tail }]);
|
|
386
|
+
}
|
|
340
387
|
else {
|
|
341
388
|
events.push([null, { type: "response.function_call_arguments.delta", output_index: key[1], delta: tail }]);
|
|
342
389
|
}
|
|
@@ -395,8 +442,12 @@ export class AnthropicMessagesAdapter extends Adapter {
|
|
|
395
442
|
if (!isObject(data))
|
|
396
443
|
return [[name, data]];
|
|
397
444
|
const kind = data.type;
|
|
398
|
-
if (kind === "content_block_start")
|
|
445
|
+
if (kind === "content_block_start") {
|
|
446
|
+
if (typeof data.content_block?.name === "string") {
|
|
447
|
+
state.toolNames.set(JSON.stringify(["b", data.index ?? 0, "input_json_delta"]), data.content_block.name);
|
|
448
|
+
}
|
|
399
449
|
return [[name, state.restore(data)]];
|
|
450
|
+
}
|
|
400
451
|
if (kind === "content_block_delta") {
|
|
401
452
|
const index = data.index ?? 0;
|
|
402
453
|
const delta = data.delta ?? {};
|
package/dist/index.d.ts
CHANGED
|
@@ -5,16 +5,17 @@ import { StreamRestorer, Vault } from "./vault.ts";
|
|
|
5
5
|
export { Analyzer, resolveOverlaps, defaultRecognizers } from "./analyzer.ts";
|
|
6
6
|
export { AnonymizationResult, Anonymizer, maskValue, normalizeOperator, OPERATORS } from "./anonymizer.ts";
|
|
7
7
|
export type { Operator, Replacement } from "./anonymizer.ts";
|
|
8
|
-
export { LABELS, SECRET_LABELS } from "./entities.ts";
|
|
8
|
+
export { GROUPS, LABELS, SECRET_LABELS, type GroupData } from "./entities.ts";
|
|
9
9
|
export type { Candidate, Entity, Recognizer } from "./entities.ts";
|
|
10
10
|
export { AddressRecognizer } from "./addresses.ts";
|
|
11
11
|
export { PatternRecognizer, customPattern, DEFAULT_SPECS } from "./patterns.ts";
|
|
12
12
|
export { PersonRecognizer } from "./persons.ts";
|
|
13
13
|
export { SecretRecognizer, keyKind, looksRandom, secretPrefix } from "./secrets.ts";
|
|
14
|
-
export {
|
|
15
|
-
export
|
|
14
|
+
export { DetectionCache } from "./cache.ts";
|
|
15
|
+
export { StreamRestorer, Vault, linkContext, neutralizePlaceholders, normalizeValue, restoreAllowed, urlHost } from "./vault.ts";
|
|
16
|
+
export type { RestoreContext, RestorePolicy, VaultData, VaultEntry } from "./vault.ts";
|
|
16
17
|
export * as validators from "./validators.ts";
|
|
17
|
-
export declare const VERSION = "0.
|
|
18
|
+
export declare const VERSION = "0.6.0";
|
|
18
19
|
export declare const DETECTORS: readonly ["padroes", "nomes", "enderecos", "segredos"];
|
|
19
20
|
export type Detector = (typeof DETECTORS)[number];
|
|
20
21
|
/** Padrão próprio, como no [[deteccao.personalizados]] do veu.toml. */
|
|
@@ -71,7 +72,8 @@ export declare class Session {
|
|
|
71
72
|
/** Anonimiza todos os textos de um objeto ou lista (por exemplo, uma lista de mensagens). */
|
|
72
73
|
anonymizeData<T>(value: T): T;
|
|
73
74
|
restore(text: string): string;
|
|
74
|
-
|
|
75
|
+
/** tool: os dados são argumentos de uma chamada de ferramenta (segue a política do cofre). */
|
|
76
|
+
restoreData<T>(value: T, tool?: boolean): T;
|
|
75
77
|
restorer(): StreamRestorer;
|
|
76
78
|
/** Restaura um stream de pedaços de texto (por exemplo, a resposta de um modelo em streaming). */
|
|
77
79
|
restoreStream(chunks: AsyncIterable<string> | Iterable<string>): AsyncGenerator<string>;
|
package/dist/index.js
CHANGED
|
@@ -9,14 +9,15 @@ import { counter } from "./text.js";
|
|
|
9
9
|
import { StreamRestorer, Vault } from "./vault.js";
|
|
10
10
|
export { Analyzer, resolveOverlaps, defaultRecognizers } from "./analyzer.js";
|
|
11
11
|
export { AnonymizationResult, Anonymizer, maskValue, normalizeOperator, OPERATORS } from "./anonymizer.js";
|
|
12
|
-
export { LABELS, SECRET_LABELS } from "./entities.js";
|
|
12
|
+
export { GROUPS, LABELS, SECRET_LABELS } from "./entities.js";
|
|
13
13
|
export { AddressRecognizer } from "./addresses.js";
|
|
14
14
|
export { PatternRecognizer, customPattern, DEFAULT_SPECS } from "./patterns.js";
|
|
15
15
|
export { PersonRecognizer } from "./persons.js";
|
|
16
16
|
export { SecretRecognizer, keyKind, looksRandom, secretPrefix } from "./secrets.js";
|
|
17
|
-
export {
|
|
17
|
+
export { DetectionCache } from "./cache.js";
|
|
18
|
+
export { StreamRestorer, Vault, linkContext, neutralizePlaceholders, normalizeValue, restoreAllowed, urlHost } from "./vault.js";
|
|
18
19
|
export * as validators from "./validators.js";
|
|
19
|
-
export const VERSION = "0.
|
|
20
|
+
export const VERSION = "0.6.0";
|
|
20
21
|
export const DETECTORS = ["padroes", "nomes", "enderecos", "segredos"];
|
|
21
22
|
function buildRecognizers(options) {
|
|
22
23
|
const detectors = options.detectors ?? [...DETECTORS];
|
|
@@ -106,8 +107,9 @@ export class Session {
|
|
|
106
107
|
restore(text) {
|
|
107
108
|
return this.vault.restore(text);
|
|
108
109
|
}
|
|
109
|
-
|
|
110
|
-
|
|
110
|
+
/** tool: os dados são argumentos de uma chamada de ferramenta (segue a política do cofre). */
|
|
111
|
+
restoreData(value, tool = false) {
|
|
112
|
+
return mapStrings(value, (text) => this.vault.restore(text, false, { tool }));
|
|
111
113
|
}
|
|
112
114
|
restorer() {
|
|
113
115
|
return this.vault.restorer();
|
package/dist/patterns.js
CHANGED
|
@@ -14,7 +14,8 @@ export function pickScore(scores, isValid, hasContext) {
|
|
|
14
14
|
export function spec(label, pattern, scores, options = {}) {
|
|
15
15
|
return {
|
|
16
16
|
label,
|
|
17
|
-
|
|
17
|
+
// "d" guarda as posições dos grupos: com um grupo "valor", só ele é o dado (a porta em "10.0.0.5:8080").
|
|
18
|
+
regex: new RegExp(pattern, `g${options.flags ?? ""}u${pattern.includes("(?<valor>") ? "d" : ""}`),
|
|
18
19
|
scores,
|
|
19
20
|
validator: options.validator,
|
|
20
21
|
context: wordsRegex(options.context ?? []),
|
|
@@ -52,8 +53,54 @@ const CTX_ACCOUNT = ["conta", "conta corrente", "conta-corrente", "c/c", "cc", "
|
|
|
52
53
|
const CTX_PROCESS = ["processo", "autos", "proc", "proc.", "acao", "numero do processo"];
|
|
53
54
|
const CTX_EMAIL = ["email", "e-mail"];
|
|
54
55
|
const CTX_IBAN = ["iban", "swift", "bic", "transferencia internacional", "conta internacional"];
|
|
56
|
+
// Outros países (pt-PT e espanhol, sem acento: o contexto é comparado depois do fold).
|
|
57
|
+
const CTX_NIF_PT = ["nif", "nipc", "contribuinte", "n.o de contribuinte", "numero de identificacao fiscal",
|
|
58
|
+
"pessoa coletiva", "pessoa colectiva"];
|
|
59
|
+
const CTX_CC_PT = ["cartao de cidadao", "cc", "n.o do documento", "nic", "bilhete de identidade", "bi"];
|
|
60
|
+
const CTX_NISS_PT = ["niss", "seguranca social", "n.o de beneficiario"];
|
|
61
|
+
const CTX_CUIT = ["cuit", "cuil", "afip", "arca", "clave fiscal"];
|
|
62
|
+
const CTX_DNI = ["dni", "d.n.i", "documento nacional de identidad", "reniec"];
|
|
63
|
+
const CTX_MX = ["curp", "rfc", "homoclave", "sat", "registro federal de contribuyentes"];
|
|
64
|
+
const CTX_RUT = ["rut", "run", "rol unico", "rol unico tributario", "sii", "dgi"];
|
|
65
|
+
const CTX_CEDULA = ["cedula", "c.i", "ci", "c.c", "cedula de ciudadania", "cedula de identidad", "documento de identidad",
|
|
66
|
+
"carnet de identidad"];
|
|
67
|
+
const CTX_NIT = ["nit", "dian"];
|
|
68
|
+
const CTX_RUC = ["ruc", "sunat", "sri", "set", "registro unico de contribuyentes"];
|
|
69
|
+
const CTX_RIF = ["rif", "seniat"];
|
|
70
|
+
const CUIT_PREFIX = "(?:20|23|24|27|30|33|34|50|51|55)";
|
|
71
|
+
const CTX_NUIT = ["nuit", "autoridade tributaria", "numero unico de identificacao tributaria"];
|
|
72
|
+
const CTX_BI_AFRICA = ["bi", "bilhete de identidade", "nif", "agt", "nuit"];
|
|
73
|
+
const CTX_SSN = ["ssn", "social security", "soc sec", "itin", "taxpayer identification", "tin"];
|
|
74
|
+
const CTX_MBI = ["mbi", "medicare", "beneficiary"];
|
|
75
|
+
const CTX_DNI_ES = ["dni", "nie", "nif", "documento nacional de identidad"];
|
|
76
|
+
const CTX_CF = ["codice fiscale", "c.f", "cod. fisc", "tessera sanitaria"];
|
|
77
|
+
const CTX_NIR = ["nir", "insee", "securite sociale", "n° ss", "carte vitale"];
|
|
78
|
+
const CTX_STEUER = ["steuer-id", "steuerid", "steueridentifikationsnummer", "idnr", "identifikationsnummer"];
|
|
79
|
+
const CTX_NINO = ["national insurance", "ni number", "nino"];
|
|
80
|
+
const CTX_NHS = ["nhs", "nhs number"];
|
|
81
|
+
const CTX_BSN = ["bsn", "burgerservicenummer", "sofinummer"];
|
|
82
|
+
const CTX_SIN = ["sin", "social insurance number", "numero d'assurance sociale"];
|
|
83
|
+
const CTX_AADHAAR = ["aadhaar", "uidai", "uid number"];
|
|
84
|
+
const CTX_PAN = ["pan", "permanent account number", "income tax"];
|
|
85
|
+
const CTX_MRZ = ["passaporte", "passport", "mrz"];
|
|
86
|
+
const CTX_VIN = ["chassi", "chassis", "vin", "numero do chassi"];
|
|
87
|
+
const CTX_HOST = ["servidor", "server", "host", "hostname", "banco", "database", "conectar", "conexao", "endpoint", "porta",
|
|
88
|
+
"port", "ssh", "redis", "postgres", "mysql", "api"];
|
|
89
|
+
const CTX_SHELL = ["ssh", "scp", "sftp", "psql", "mysql", "telnet", "nc", "docker", "redis-cli", "mongosh"];
|
|
90
|
+
// Terminações de nomes que só existem dentro de uma rede (RFC 6762, RFC 8375 e os costumes de redes de empresas).
|
|
91
|
+
const INTERNAL_TLDS = "(?:local|localdomain|internal|interna|interno|lan|corp|intranet|home\\.arpa)";
|
|
92
|
+
const W = "\\p{L}\\p{N}_";
|
|
93
|
+
const CTX_CRYPTO = ["carteira", "wallet", "bitcoin", "btc", "ethereum", "eth", "cripto", "usdt"];
|
|
94
|
+
// 1ª linha da MRZ: "P<" + país + sobrenome<<nomes, completando com "<".
|
|
95
|
+
const mrzNameLine = (value) => value.slice(5).includes("<<") && /[A-Z]/.test(value.slice(5));
|
|
55
96
|
const phoneAdjust = (value, score) => (/[()\s.+-]/u.test(value) ? score : score - 0.3);
|
|
56
97
|
const isYear = (value) => /^(?:19|20)\d{2}$/.test(value);
|
|
98
|
+
// Documentos já mascarados pela metade ("324.327.***-**", "***.982.247-**"): os dígitos que sobram ainda
|
|
99
|
+
// identificam a pessoa. Só valem com pelo menos 3 dígitos à mostra.
|
|
100
|
+
const M = "[\\d*xX\\u2022]";
|
|
101
|
+
const MB = "(?<![\\p{L}\\p{N}_*])";
|
|
102
|
+
const ME = "(?![\\p{L}\\p{N}_*])";
|
|
103
|
+
const partiallyMasked = (value) => [...value].filter((char) => /\d/.test(char)).length >= 3 && /[*xX\u2022]/.test(value);
|
|
57
104
|
const MONTHS = "janeiro|fevereiro|mar[cç]o|abril|maio|junho|julho|agosto|setembro|outubro|novembro|dezembro";
|
|
58
105
|
export const DEFAULT_SPECS = [
|
|
59
106
|
// CPF
|
|
@@ -61,20 +108,28 @@ export const DEFAULT_SPECS = [
|
|
|
61
108
|
spec("CPF", `${B}\\d{9}-\\d{2}${E}`, S(0.9, 0.4, 1.0, 0.65), { validator: v.isValidCpf, context: CTX_CPF }),
|
|
62
109
|
spec("CPF", `${B}\\d{3}\\s\\d{3}\\s\\d{3}\\s\\d{2}${E}`, S(0.85, null, 0.95, 0.6), { validator: v.isValidCpf, context: CTX_CPF }),
|
|
63
110
|
spec("CPF", `${B}\\d{11}${E}`, S(0.75, null, 0.95, 0.6), { validator: v.isValidCpf, context: CTX_CPF }),
|
|
111
|
+
spec("CPF", `${MB}${M}{3}\\.${M}{3}\\.${M}{3}-${M}{2}${ME}`, S(0.85, null, 0.97), { validator: partiallyMasked, context: CTX_CPF, name: "mascarado" }),
|
|
64
112
|
// CNPJ (numérico e alfanumérico)
|
|
65
113
|
spec("CNPJ", `${B}[0-9A-Z]{2}\\.[0-9A-Z]{3}\\.[0-9A-Z]{3}/[0-9A-Z]{4}-\\d{2}${E}`, S(0.95, 0.5, 1.0, 0.7), { validator: v.isValidCnpj, context: CTX_CNPJ }),
|
|
66
114
|
spec("CNPJ", `${B}\\d{14}${E}`, S(0.75, null, 0.95, 0.6), { validator: v.isValidCnpj, context: CTX_CNPJ }),
|
|
115
|
+
spec("CNPJ", `${B}\\d{2}\\s\\d{3}\\s\\d{3}\\s\\d{4}\\s\\d{2}${E}`, S(0.85, null, 0.95, 0.6), { validator: v.isValidCnpj, context: CTX_CNPJ }),
|
|
116
|
+
spec("CNPJ", `${MB}${M}{2}\\.${M}{3}\\.${M}{3}/${M}{4}-${M}{2}${ME}`, S(0.85, null, 0.97), { validator: partiallyMasked, context: CTX_CNPJ, name: "mascarado" }),
|
|
67
117
|
spec("CNPJ", `${B}(?=[0-9A-Z]{12}\\d{2}${E})(?=[0-9]*[A-Z])[0-9A-Z]{12}\\d{2}${E}`, S(0.6, null, 0.95, 0.55), { validator: v.isValidCnpj, context: CTX_CNPJ }),
|
|
68
118
|
// Documentos que dependem de contexto
|
|
69
119
|
spec("PIS", `${B}\\d{3}\\.\\d{5}\\.\\d{2}-\\d${E}`, S(0.85, 0.45, 1.0, 0.75), { validator: v.isValidPis, context: CTX_PIS }),
|
|
70
120
|
spec("PIS", `${B}\\d{11}${E}`, S(null, null, 0.97, 0.65), { validator: v.isValidPis, context: CTX_PIS, before: 25 }),
|
|
71
121
|
spec("CNH", `${B}\\d{11}${E}`, S(null, null, 0.97, 0.75), { validator: v.isValidCnh, context: CTX_CNH, before: 30 }),
|
|
122
|
+
spec("CNH", `${B}\\d{10}-\\d${E}`, S(null, null, 0.97, 0.75), { validator: v.isValidCnh, context: CTX_CNH, before: 30 }),
|
|
123
|
+
spec("CNH", `${MB}${M}{11}${ME}`, S(null, null, 0.9), { validator: partiallyMasked, context: CTX_CNH, before: 30, name: "mascarado" }),
|
|
72
124
|
spec("TITULO_ELEITOR", `${B}\\d{4}\\s?\\d{4}\\s?\\d{4}${E}`, S(null, null, 0.95, 0.75), { validator: v.isValidTituloEleitor, context: CTX_TITULO, before: 30 }),
|
|
125
|
+
spec("TITULO_ELEITOR", `${MB}${M}{4}\\s?${M}{4}\\s?${M}{4}${ME}`, S(null, null, 0.9), { validator: partiallyMasked, context: CTX_TITULO, before: 30, name: "mascarado" }),
|
|
73
126
|
spec("RENAVAM", `${B}\\d{9,11}${E}`, S(null, null, 0.97, 0.75), { validator: v.isValidRenavam, context: CTX_RENAVAM, before: 25 }),
|
|
74
127
|
spec("CNS", `${B}[12789]\\d{2}\\s?\\d{4}\\s?\\d{4}\\s?\\d{4}${E}`, S(0.6, null, 0.97, 0.65), { validator: v.isValidCns, context: CTX_CNS }),
|
|
75
128
|
spec("RG", `${B}\\d{1,2}\\.\\d{3}\\.\\d{3}-[\\dXx]${E}`, S(0.6, null, 0.95), { context: CTX_RG }),
|
|
76
129
|
spec("RG", `${B}\\d{7,9}-?[\\dXx]?${E}`, S(null, null, 0.85), { context: CTX_RG, before: 20 }),
|
|
77
130
|
spec("RG", `${B}[A-Z]{2}-?\\d{1,2}\\.?\\d{3}\\.?\\d{3}${E}`, S(null, null, 0.85), { context: CTX_RG, before: 20 }),
|
|
131
|
+
spec("RG", `${B}\\d{1,2}\\s\\d{3}\\s\\d{3}\\s[\\dXx]${E}`, S(null, null, 0.85), { context: CTX_RG, before: 20 }),
|
|
132
|
+
spec("RG", `${MB}${M}{1,2}\\.?${M}{3}\\.?${M}{2,3}-?${M}${ME}`, S(null, null, 0.85), { validator: partiallyMasked, context: CTX_RG, before: 20, name: "mascarado" }),
|
|
78
133
|
spec("PASSAPORTE", `${B}[A-Z]{2}\\d{6}${E}`, S(null, null, 0.9), { context: CTX_PASSAPORTE, before: 30 }),
|
|
79
134
|
// Contatos
|
|
80
135
|
spec("EMAIL", `(?<![\\p{L}\\p{N}_.+-])[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\\.[A-Za-z0-9-]+)*\\.[A-Za-z]{2,}${E}`, S(0.95), { context: CTX_EMAIL }),
|
|
@@ -86,6 +141,7 @@ export const DEFAULT_SPECS = [
|
|
|
86
141
|
spec("CEP", `${B}\\d{5}-\\d{3}${E}`, S(0.75, null, 0.95), { context: CTX_CEP }),
|
|
87
142
|
spec("CEP", `${B}\\d{2}\\.\\d{3}-\\d{3}${E}`, S(0.8, null, 0.95), { context: CTX_CEP }),
|
|
88
143
|
spec("CEP", `${B}\\d{8}${E}`, S(null, null, 0.85), { context: CTX_CEP, before: 15 }),
|
|
144
|
+
spec("CEP", `${B}\\d{5}\\s\\d{3}${E}`, S(null, null, 0.85), { context: CTX_CEP, before: 15 }),
|
|
89
145
|
spec("PLACA", `${B}[A-Z]{3}-\\d{4}${E}`, S(0.8, null, 0.95), { context: CTX_PLATE }),
|
|
90
146
|
spec("PLACA", `${B}[A-Z]{3}\\d[A-Z]\\d{2}${E}`, S(0.75, null, 0.95), { context: CTX_PLATE }),
|
|
91
147
|
spec("PLACA", `${B}[A-Z]{3}\\d{4}${E}`, S(0.45, null, 0.9), { context: CTX_PLATE }),
|
|
@@ -103,6 +159,68 @@ export const DEFAULT_SPECS = [
|
|
|
103
159
|
spec("DATA_NASCIMENTO", `${B}\\d{1,2}\\s+de\\s+(?:${MONTHS})\\s+de\\s+\\d{4}${E}`, S(null, null, 0.85), { validator: v.isPlausibleBirthdate, context: CTX_BIRTH, flags: "i", before: 35, after: 0 }),
|
|
104
160
|
// Processos judiciais
|
|
105
161
|
spec("PROCESSO", `${B}\\d{7}-\\d{2}\\.\\d{4}\\.\\d\\.\\d{2}\\.\\d{4}${E}`, S(0.9, 0.6, 1.0, 0.75), { validator: v.isValidProcessoCnj, context: CTX_PROCESS }),
|
|
162
|
+
// Outros países. Três níveis de confiança:
|
|
163
|
+
// - formatos estruturados ou com a pontuação típica: sem contexto (CURP, RFC, CC, CUIT com hífens, RIF);
|
|
164
|
+
// - numéricos de 11 dígitos ou mais, que podem ser CPF: só com contexto, e aí vencem o CPF;
|
|
165
|
+
// - numéricos curtos, que qualquer número de 7 a 9 dígitos pode imitar: só com contexto.
|
|
166
|
+
// A versão do cartão são duas letras ("ZZ", "ZX"): só dígitos seria qualquer protocolo de 12 dígitos.
|
|
167
|
+
spec("CC_PT", `${B}\\d{8}[ -]?\\d[ -]?[A-Z]{2}[ -]?\\d${E}`, S(0.9, null, 0.97, 0.7), { validator: v.isValidCcPt, context: CTX_CC_PT }),
|
|
168
|
+
spec("NIF_PT", `${B}\\d{3}[ .]?\\d{3}[ .]?\\d{3}${E}`, S(null, null, 0.95), { validator: v.isValidNifPt, context: CTX_NIF_PT, before: 30 }),
|
|
169
|
+
spec("NISS_PT", `${B}[12]\\d{10}${E}`, S(null, null, 0.97), { validator: v.isValidNissPt, context: CTX_NISS_PT, before: 30 }),
|
|
170
|
+
spec("CUIT", `${B}${CUIT_PREFIX}-\\d{8}-\\d${E}`, S(0.9, null, 0.97, 0.75), { validator: v.isValidCuit, context: CTX_CUIT }),
|
|
171
|
+
spec("CUIT", `${B}${CUIT_PREFIX}\\d{9}${E}`, S(null, null, 0.96), { validator: v.isValidCuit, context: CTX_CUIT, before: 30 }),
|
|
172
|
+
spec("DNI", `${B}\\d{1,2}\\.?\\d{3}\\.?\\d{3}(?:-[0-9A-K])?${E}`, S(null, null, 0.85), { context: CTX_DNI, before: 20 }),
|
|
173
|
+
spec("CURP", `${B}[A-Z]{4}\\d{6}[HMX][A-Z]{5}[0-9A-Z]\\d${E}`, S(0.95, 0.6, 1.0, 0.8), { validator: v.isValidCurp, context: CTX_MX }),
|
|
174
|
+
spec("RFC", `(?<![\\p{L}\\p{N}_&])[A-ZÑ&]{3,4}\\d{6}[A-Z0-9]{3}${E}`, S(0.85, null, 0.97), { validator: v.isValidRfc, context: CTX_MX }),
|
|
175
|
+
// O RUT chileno e a cédula uruguaia formatados têm o desenho do RG: sem contexto, perdem para ele (0,55 < 0,6).
|
|
176
|
+
spec("RUT", `${B}\\d{1,2}\\.\\d{3}\\.\\d{3}-[\\dkK]${E}`, S(0.55, null, 0.97, 0.75), { validator: v.isValidRutCl, context: CTX_RUT }),
|
|
177
|
+
spec("RUT", `${B}\\d{7,8}-[\\dkK]${E}`, S(null, null, 0.95, 0.75), { validator: v.isValidRutCl, context: CTX_RUT, before: 25 }),
|
|
178
|
+
spec("RUT", `${B}\\d{12}${E}`, S(null, null, 0.95), { validator: v.isValidRutUy, context: CTX_RUT, before: 25 }),
|
|
179
|
+
spec("CEDULA", `${B}\\d\\.\\d{3}\\.\\d{3}-\\d${E}`, S(0.55, null, 0.97, 0.75), { validator: v.isValidCiUy, context: CTX_CEDULA }),
|
|
180
|
+
spec("CEDULA", `${B}\\d{10}${E}`, S(null, null, 0.95), { validator: v.isValidCiEc, context: CTX_CEDULA, before: 25 }),
|
|
181
|
+
spec("CEDULA", `${B}(?:\\d{1,3}(?:\\.\\d{3}){1,3}|\\d{6,10})${E}`, S(null, null, 0.85), { context: CTX_CEDULA, before: 20 }),
|
|
182
|
+
spec("NIT", `${B}\\d{3}\\.\\d{3}\\.\\d{3}-\\d${E}`, S(0.8, null, 0.97, 0.75), { validator: v.isValidNitCo, context: CTX_NIT }),
|
|
183
|
+
spec("NIT", `${B}\\d{8,15}-?\\d${E}`, S(null, null, 0.95, 0.75), { validator: v.isValidNitCo, context: CTX_NIT, before: 25 }),
|
|
184
|
+
spec("RUC", `${B}(?:\\d{11}|\\d{13}|\\d{5,8}-\\d)${E}`, S(null, null, 0.96, 0.75), { validator: v.isValidRuc, context: CTX_RUC, before: 25 }),
|
|
185
|
+
spec("RIF", `${B}[VEJPG]-?\\d{8}-?\\d${E}`, S(0.9, null, 0.97, 0.75), { validator: v.isValidRif, context: CTX_RIF }),
|
|
186
|
+
// África lusófona, EUA, Europa, Índia e identificadores transversais.
|
|
187
|
+
spec("NUIT", `${B}\\d{3}\\s?\\d{3}\\s?\\d{3}${E}`, S(null, null, 0.95), { validator: v.isValidNuit, context: CTX_NUIT, before: 30 }),
|
|
188
|
+
spec("BI_AO", `${B}\\d{9}[A-Z]{2}\\d{3}${E}`, S(0.85, null, 0.97), { context: CTX_BI_AFRICA }),
|
|
189
|
+
// 12 dígitos e uma letra também é o formato das datas de certificados X.509 ("261108123045Z"): só com contexto.
|
|
190
|
+
spec("BI_MZ", `${B}\\d{12}[A-Z]${E}`, S(null, null, 0.95), { context: CTX_BI_AFRICA, before: 30 }),
|
|
191
|
+
spec("SSN", `${B}\\d{3}-\\d{2}-\\d{4}${E}`, S(0.85, null, 0.97), { validator: v.isValidSsn, context: CTX_SSN }),
|
|
192
|
+
spec("SSN", `${B}\\d{3}\\s?\\d{2}\\s?\\d{4}${E}`, S(null, null, 0.95), { validator: v.isValidSsn, context: CTX_SSN, before: 30 }),
|
|
193
|
+
spec("MBI", `${B}[1-9][AC-HJKMNP-RT-Y][AC-HJKMNP-RT-Y\\d]\\d-?[AC-HJKMNP-RT-Y][AC-HJKMNP-RT-Y\\d]\\d-?[AC-HJKMNP-RT-Y]{2}\\d{2}${E}`, S(null, null, 0.95), { context: CTX_MBI, before: 30 }),
|
|
194
|
+
spec("DNI_ES", `${B}[XYZ]?\\d{7,8}-?[A-Z]${E}`, S(0.85, null, 0.97), { validator: v.isValidDniEs, context: CTX_DNI_ES }),
|
|
195
|
+
spec("CODICE_FISCALE", `${B}[A-Z]{6}[\\dLMNP-V]{2}[ABCDEHLMPRST][\\dLMNP-V]{2}[A-Z][\\dLMNP-V]{3}[A-Z]${E}`, S(0.95, 0.6, 1.0, 0.8), { validator: v.isValidCodiceFiscale, context: CTX_CF }),
|
|
196
|
+
// Sem os espaços, 15 dígitos começando com 1 ou 2 podem ser um protocolo: só com contexto.
|
|
197
|
+
spec("NIR", `${B}[12] \\d{2} \\d{2} (?:\\d{2}|2[AB]) \\d{3} \\d{3} \\d{2}${E}`, S(0.85, null, 0.97), { validator: v.isValidNir, context: CTX_NIR }),
|
|
198
|
+
spec("NIR", `${B}[12]\\d{4}(?:\\d{2}|2[AB])\\d{8}${E}`, S(null, null, 0.95), { validator: v.isValidNir, context: CTX_NIR, before: 30 }),
|
|
199
|
+
spec("STEUER_ID", `${B}\\d{2}\\s?\\d{3}\\s?\\d{3}\\s?\\d{3}${E}`, S(null, null, 0.95), { validator: v.isValidSteuerId, context: CTX_STEUER, before: 35 }),
|
|
200
|
+
spec("NINO", `${B}[A-CEGHJ-PR-TW-Z][A-CEGHJ-NPR-TW-Z]\\s?\\d{2}\\s?\\d{2}\\s?\\d{2}\\s?[A-D]${E}`, S(0.85, null, 0.97), { validator: v.isValidNino, context: CTX_NINO }),
|
|
201
|
+
spec("NINO", `${B}[A-CEGHJ-PR-TW-Z][A-CEGHJ-NPR-TW-Z]\\s?\\d{2}\\s?\\d{2}\\s?\\d{2}${E}`, S(null, null, 0.95), { validator: v.isValidNino, context: CTX_NINO, before: 30 }),
|
|
202
|
+
spec("NHS", `${B}\\d{3}\\s?\\d{3}\\s?\\d{4}${E}`, S(null, null, 0.95), { validator: v.isValidNhs, context: CTX_NHS, before: 30 }),
|
|
203
|
+
spec("BSN", `${B}\\d{9}${E}`, S(null, null, 0.95), { validator: v.isValidBsn, context: CTX_BSN, before: 30 }),
|
|
204
|
+
spec("SIN", `${B}\\d{3}[\\s-]?\\d{3}[\\s-]?\\d{3}${E}`, S(null, null, 0.95), { validator: v.isValidSin, context: CTX_SIN, before: 30 }),
|
|
205
|
+
spec("AADHAAR", `${B}[2-9]\\d{3}\\s?\\d{4}\\s?\\d{4}${E}`, S(null, null, 0.95), { validator: v.isValidAadhaar, context: CTX_AADHAAR, before: 30 }),
|
|
206
|
+
spec("PAN", `${B}[A-Z]{3}[PCHFATBLJGK][A-Z]\\d{4}[A-Z]${E}`, S(null, null, 0.95), { context: CTX_PAN, before: 30 }),
|
|
207
|
+
// MRZ de passaporte (TD3): a 1ª linha traz o nome; a 2ª, número, nascimento e validade com dígitos verificadores.
|
|
208
|
+
spec("MRZ", "(?<![A-Z0-9<])P[A-Z<][A-Z<]{3}[A-Z<]{39}(?![A-Z0-9<])", S(0.9, null, 0.97), { validator: mrzNameLine, context: CTX_MRZ }),
|
|
209
|
+
spec("MRZ", "(?<![A-Z0-9<])[A-Z0-9<]{9}\\d[A-Z<]{3}\\d{7}[MFX<]\\d{7}[A-Z0-9<]{14}[\\d<]\\d(?![A-Z0-9<])", S(0.95, null, 1.0), { validator: v.isValidMrzLine, context: CTX_MRZ }),
|
|
210
|
+
// Chassi: o dígito da posição 9 só é obrigatório nos EUA e na China; sem ele, só com "chassi" ou "VIN" por perto.
|
|
211
|
+
spec("VIN", `${B}[A-HJ-NPR-Z0-9]{17}${E}`, S(0.85, null, 0.97, 0.8), { validator: v.isValidVin, context: CTX_VIN, exclude: (value) => /^(?:\d+|[A-Z]+)$/.test(value) }),
|
|
212
|
+
spec("CRIPTO", `${B}(?:0x[0-9a-fA-F]{40}|[13][1-9A-HJ-NP-Za-km-z]{25,34}|(?:bc|tb)1[02-9ac-hj-np-z]{11,71})${E}`, S(0.9, null, 0.97), { validator: v.isValidCryptoAddress, context: CTX_CRYPTO }),
|
|
213
|
+
// Infraestrutura. Uma porta é só um número: vale colada a um IP ou a um servidor, depois de "porta"/"port" ou
|
|
214
|
+
// numa opção de comando (-p), nunca solta. O grupo "valor" é o que vira o marcador; o resto é só o contexto.
|
|
215
|
+
spec("PORTA", `${B}(?:\\d{1,3}\\.){3}\\d{1,3}:(?<valor>\\d{2,5})${E}`, S(0.9), { validator: v.isValidPort }),
|
|
216
|
+
spec("PORTA", `\\[[0-9A-Fa-f:.]{3,45}\\]:(?<valor>\\d{2,5})${E}`, S(0.9), { validator: v.isValidPort }),
|
|
217
|
+
spec("PORTA", `://(?:[^\\s/@]{1,200}@)?[A-Za-z0-9.\\-]{1,253}:(?<valor>\\d{2,5})${E}`, S(0.9), { validator: v.isValidPort }),
|
|
218
|
+
spec("PORTA", `(?<![${W}.@\\-])(?:localhost|(?:[A-Za-z0-9\\-]+\\.)+${INTERNAL_TLDS}):(?<valor>\\d{2,5})${E}`, S(0.9), { validator: v.isValidPort, flags: "i" }),
|
|
219
|
+
spec("PORTA", `(?<![${W}.@/\\-])(?=[A-Za-z0-9.\\-]*[A-Za-z])(?:[A-Za-z0-9\\-]+\\.)+[A-Za-z]{2,24}:(?<valor>\\d{2,5})${E}`, S(null, null, 0.85), { validator: v.isValidPort, context: CTX_HOST }),
|
|
220
|
+
spec("PORTA", `(?<![A-Za-z])(?:portas?|ports?|--port)["']?\\s*(?:[:=]\\s*|n[ºo°]\\.?\\s*)?(?<valor>\\d{2,5})${E}`, S(0.85), { validator: v.isValidPort, flags: "i" }),
|
|
221
|
+
spec("PORTA", `(?<![${W}\\-])-[pP]\\s+(?<valor>\\d{2,5})${E}`, S(null, null, 0.85), { validator: v.isValidPort, context: CTX_SHELL, before: 60 }),
|
|
222
|
+
spec("PORTA", `(?<![${W}\\-])-[pP]\\s+\\d{2,5}:(?<valor>\\d{2,5})${E}`, S(null, null, 0.85), { validator: v.isValidPort, context: CTX_SHELL, before: 60 }),
|
|
223
|
+
spec("SERVIDOR", `(?<![${W}.@\\-])(?:[A-Za-z0-9\\-]+\\.)+${INTERNAL_TLDS}(?![${W}\\-]|\\.[A-Za-z])`, S(0.85), { flags: "i" }),
|
|
106
224
|
];
|
|
107
225
|
const round3 = (value) => Math.round(value * 1000) / 1000;
|
|
108
226
|
export class PatternRecognizer {
|
|
@@ -114,10 +232,18 @@ export class PatternRecognizer {
|
|
|
114
232
|
this.labels = new Set(this.specs.map((item) => item.label));
|
|
115
233
|
}
|
|
116
234
|
*find(text) {
|
|
235
|
+
let folded = null;
|
|
117
236
|
for (const item of this.specs) {
|
|
237
|
+
// Padrões que só valem com contexto: sem nenhuma palavra de contexto no texto, nem procura os números.
|
|
238
|
+
if ((item.scores.valid ?? null) === null && (item.scores.invalid ?? null) === null && item.context) {
|
|
239
|
+
folded ??= fold(text);
|
|
240
|
+
if (!item.context.test(folded))
|
|
241
|
+
continue;
|
|
242
|
+
}
|
|
118
243
|
for (const match of text.matchAll(item.regex)) {
|
|
119
|
-
const
|
|
120
|
-
const
|
|
244
|
+
const span = match.indices?.groups?.valor;
|
|
245
|
+
const value = span ? text.slice(span[0], span[1]) : match[0];
|
|
246
|
+
const start = span ? span[0] : match.index;
|
|
121
247
|
if (item.exclude?.(value))
|
|
122
248
|
continue;
|
|
123
249
|
const isValid = item.validator ? item.validator(value) : true;
|
package/dist/persons.d.ts
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import type { Candidate, Recognizer } from "./entities.ts";
|
|
2
2
|
export declare const CONNECTORS: ReadonlySet<string>;
|
|
3
|
+
/** Ambíguos revistos à mão ("Rosa", "Luz"): não se propagam no anonimizador. */
|
|
3
4
|
export declare function ambiguousNames(): ReadonlySet<string>;
|
|
5
|
+
/** Ambíguos que o dicionário marca como palavras ("Costa", "Nascimento"): só se propagam com inicial maiúscula. */
|
|
6
|
+
export declare function capitalizedOnlyNames(): ReadonlySet<string>;
|
|
4
7
|
export declare class PersonRecognizer implements Recognizer {
|
|
5
8
|
name: string;
|
|
6
9
|
labels: ReadonlySet<string>;
|
package/dist/persons.js
CHANGED
|
@@ -34,6 +34,14 @@ const ORGANIZATION_WORDS = new Set([
|
|
|
34
34
|
]);
|
|
35
35
|
const STREET_BEFORE = new RegExp("(?<![\\p{L}\\p{N}_])(?:rua|r\\.|av\\.?|avenida|travessa|tv\\.|alameda|al\\.|praca|pca\\.|rodovia|rod\\.|estrada|" +
|
|
36
36
|
"largo|viela|beco|ladeira|vila|bairro|jardim|parque|condominio|edificio|residencial)(?:\\s+d[aeo]s?)?\\s*$", "u");
|
|
37
|
+
// Tipos de logradouro: uma sequência que começa com eles é endereço ("Avenida Paulista", "Quadra Norte").
|
|
38
|
+
const STREET_WORDS = new Set([
|
|
39
|
+
"rua", "avenida", "av", "travessa", "alameda", "praca", "rodovia", "estrada", "largo", "viela", "beco",
|
|
40
|
+
"ladeira", "quadra", "conjunto", "setor", "loteamento", "condominio", "residencial", "edificio", "chacara",
|
|
41
|
+
"sitio", "fazenda", "via", "vila", "jardim", "parque", "bairro",
|
|
42
|
+
]);
|
|
43
|
+
// "São Cristóvão", "Santa Maria", "Santo Antônio": lugares e santos, não pessoas.
|
|
44
|
+
const SAINTS = new Set(["sao", "santo", "santa"]);
|
|
37
45
|
const GREETINGS = [
|
|
38
46
|
"oi", "ola", "e ai", "eai", "fala", "bom dia", "boa tarde", "boa noite", "obrigado", "obrigada", "valeu",
|
|
39
47
|
"vlw", "tchau", "beijo", "bjs", "abraco", "querido", "querida", "meu nome e", "me chamo", "sou o", "sou a",
|
|
@@ -47,7 +55,9 @@ function lists() {
|
|
|
47
55
|
shared ??= {
|
|
48
56
|
firstNames: foldedSet(data.firstNames),
|
|
49
57
|
surnames: foldedSet(data.surnames),
|
|
50
|
-
ambiguous: foldedSet(data.ambiguous),
|
|
58
|
+
ambiguous: new Set([...foldedSet(data.ambiguous), ...foldedSet(data.ibgeAmbiguous)]),
|
|
59
|
+
curatedAmbiguous: foldedSet(data.ambiguous),
|
|
60
|
+
ibgeAmbiguous: foldedSet(data.ibgeAmbiguous),
|
|
51
61
|
stopwords: foldedSet(data.stopwords),
|
|
52
62
|
stopExact: new Set(data.stopwords.map(casefold)),
|
|
53
63
|
places: foldedSet(data.places),
|
|
@@ -55,8 +65,14 @@ function lists() {
|
|
|
55
65
|
};
|
|
56
66
|
return shared;
|
|
57
67
|
}
|
|
68
|
+
/** Ambíguos revistos à mão ("Rosa", "Luz"): não se propagam no anonimizador. */
|
|
58
69
|
export function ambiguousNames() {
|
|
59
|
-
return lists().
|
|
70
|
+
return lists().curatedAmbiguous;
|
|
71
|
+
}
|
|
72
|
+
/** Ambíguos que o dicionário marca como palavras ("Costa", "Nascimento"): só se propagam com inicial maiúscula. */
|
|
73
|
+
export function capitalizedOnlyNames() {
|
|
74
|
+
const { curatedAmbiguous, ibgeAmbiguous } = lists();
|
|
75
|
+
return new Set([...ibgeAmbiguous].filter((word) => !curatedAmbiguous.has(word)));
|
|
60
76
|
}
|
|
61
77
|
export class PersonRecognizer {
|
|
62
78
|
name = "nomes";
|
|
@@ -135,6 +151,8 @@ export class PersonRecognizer {
|
|
|
135
151
|
return null;
|
|
136
152
|
if (tokens.some((token) => ORGANIZATION_WORDS.has(fold(token[2]))))
|
|
137
153
|
return null;
|
|
154
|
+
if (tokens.length && (STREET_WORDS.has(fold(tokens[0][2])) || SAINTS.has(fold(tokens[0][2]))))
|
|
155
|
+
return null;
|
|
138
156
|
const firstIndex = tokens.findIndex((token) => this.firstNames.has(fold(token[2])));
|
|
139
157
|
if (firstIndex !== -1) {
|
|
140
158
|
tokens = tokens.slice(firstIndex);
|
|
@@ -190,7 +208,7 @@ export class PersonRecognizer {
|
|
|
190
208
|
score = context ? 0.8 : 0.6;
|
|
191
209
|
}
|
|
192
210
|
}
|
|
193
|
-
else if (context && (this.surnames.has(word) || this.strongContext(text, start))) {
|
|
211
|
+
else if (context && ((this.surnames.has(word) && !this.ambiguous.has(word)) || this.strongContext(text, start))) {
|
|
194
212
|
score = 0.6;
|
|
195
213
|
}
|
|
196
214
|
else {
|