@conarium-ai/core 0.2.54 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +103 -0
- package/LIMITATIONS.md +94 -1
- package/LIMITATIONS.tr.md +96 -1
- package/dist/custom_patterns.d.ts +13 -12
- package/dist/custom_patterns.d.ts.map +1 -1
- package/dist/custom_patterns.js +576 -14
- package/dist/custom_patterns.js.map +1 -1
- package/dist/governance.d.ts.map +1 -1
- package/dist/governance.js +37 -139
- package/dist/governance.js.map +1 -1
- package/dist/mask/belt.d.ts +36 -0
- package/dist/mask/belt.d.ts.map +1 -0
- package/dist/mask/belt.js +865 -0
- package/dist/mask/belt.js.map +1 -0
- package/dist/mask/core.d.ts +36 -0
- package/dist/mask/core.d.ts.map +1 -0
- package/dist/mask/core.js +119 -0
- package/dist/mask/core.js.map +1 -0
- package/dist/mask/detectors/bearer.d.ts +6 -0
- package/dist/mask/detectors/bearer.d.ts.map +1 -0
- package/dist/mask/detectors/bearer.js +30 -0
- package/dist/mask/detectors/bearer.js.map +1 -0
- package/dist/mask/detectors/connection.d.ts +7 -0
- package/dist/mask/detectors/connection.d.ts.map +1 -0
- package/dist/mask/detectors/connection.js +20 -0
- package/dist/mask/detectors/connection.js.map +1 -0
- package/dist/mask/detectors/custom.d.ts +12 -0
- package/dist/mask/detectors/custom.d.ts.map +1 -0
- package/dist/mask/detectors/custom.js +44 -0
- package/dist/mask/detectors/custom.js.map +1 -0
- package/dist/mask/detectors/email.d.ts +3 -0
- package/dist/mask/detectors/email.d.ts.map +1 -0
- package/dist/mask/detectors/email.js +41 -0
- package/dist/mask/detectors/email.js.map +1 -0
- package/dist/mask/detectors/embedded.d.ts +3 -0
- package/dist/mask/detectors/embedded.d.ts.map +1 -0
- package/dist/mask/detectors/embedded.js +73 -0
- package/dist/mask/detectors/embedded.js.map +1 -0
- package/dist/mask/detectors/iban.d.ts +3 -0
- package/dist/mask/detectors/iban.d.ts.map +1 -0
- package/dist/mask/detectors/iban.js +92 -0
- package/dist/mask/detectors/iban.js.map +1 -0
- package/dist/mask/detectors/ip.d.ts +3 -0
- package/dist/mask/detectors/ip.d.ts.map +1 -0
- package/dist/mask/detectors/ip.js +42 -0
- package/dist/mask/detectors/ip.js.map +1 -0
- package/dist/mask/detectors/labelled_secret.d.ts +6 -0
- package/dist/mask/detectors/labelled_secret.d.ts.map +1 -0
- package/dist/mask/detectors/labelled_secret.js +31 -0
- package/dist/mask/detectors/labelled_secret.js.map +1 -0
- package/dist/mask/detectors/mrz.d.ts +3 -0
- package/dist/mask/detectors/mrz.d.ts.map +1 -0
- package/dist/mask/detectors/mrz.js +42 -0
- package/dist/mask/detectors/mrz.js.map +1 -0
- package/dist/mask/detectors/name.d.ts +7 -0
- package/dist/mask/detectors/name.d.ts.map +1 -0
- package/dist/mask/detectors/name.js +117 -0
- package/dist/mask/detectors/name.js.map +1 -0
- package/dist/mask/detectors/numeric.d.ts +3 -0
- package/dist/mask/detectors/numeric.d.ts.map +1 -0
- package/dist/mask/detectors/numeric.js +240 -0
- package/dist/mask/detectors/numeric.js.map +1 -0
- package/dist/mask/detectors/secret.d.ts +6 -0
- package/dist/mask/detectors/secret.d.ts.map +1 -0
- package/dist/mask/detectors/secret.js +31 -0
- package/dist/mask/detectors/secret.js.map +1 -0
- package/dist/mask/detectors/whole_base64.d.ts +3 -0
- package/dist/mask/detectors/whole_base64.d.ts.map +1 -0
- package/dist/mask/detectors/whole_base64.js +28 -0
- package/dist/mask/detectors/whole_base64.js.map +1 -0
- package/dist/mask/legacy_0254/conn_linear.d.ts +26 -0
- package/dist/mask/legacy_0254/conn_linear.d.ts.map +1 -0
- package/dist/mask/legacy_0254/conn_linear.js +97 -0
- package/dist/mask/legacy_0254/conn_linear.js.map +1 -0
- package/dist/mask/legacy_0254/custom_patterns.d.ts +42 -0
- package/dist/mask/legacy_0254/custom_patterns.d.ts.map +1 -0
- package/dist/mask/legacy_0254/custom_patterns.js +164 -0
- package/dist/mask/legacy_0254/custom_patterns.js.map +1 -0
- package/dist/mask/legacy_0254/digit_pii.d.ts +46 -0
- package/dist/mask/legacy_0254/digit_pii.d.ts.map +1 -0
- package/dist/mask/legacy_0254/digit_pii.js +230 -0
- package/dist/mask/legacy_0254/digit_pii.js.map +1 -0
- package/dist/mask/legacy_0254/iban.d.ts +56 -0
- package/dist/mask/legacy_0254/iban.d.ts.map +1 -0
- package/dist/mask/legacy_0254/iban.js +171 -0
- package/dist/mask/legacy_0254/iban.js.map +1 -0
- package/dist/mask/legacy_0254/ip_detect.d.ts +28 -0
- package/dist/mask/legacy_0254/ip_detect.d.ts.map +1 -0
- package/dist/mask/legacy_0254/ip_detect.js +92 -0
- package/dist/mask/legacy_0254/ip_detect.js.map +1 -0
- package/dist/mask/legacy_0254/legacy_governance_mask.d.ts +20 -0
- package/dist/mask/legacy_0254/legacy_governance_mask.d.ts.map +1 -0
- package/dist/mask/legacy_0254/legacy_governance_mask.js +171 -0
- package/dist/mask/legacy_0254/legacy_governance_mask.js.map +1 -0
- package/dist/mask/legacy_0254/legacy_mask_text.d.ts +11 -0
- package/dist/mask/legacy_0254/legacy_mask_text.d.ts.map +1 -0
- package/dist/mask/legacy_0254/legacy_mask_text.js +46 -0
- package/dist/mask/legacy_0254/legacy_mask_text.js.map +1 -0
- package/dist/mask/legacy_0254/mrz.d.ts +7 -0
- package/dist/mask/legacy_0254/mrz.d.ts.map +1 -0
- package/dist/mask/legacy_0254/mrz.js +74 -0
- package/dist/mask/legacy_0254/mrz.js.map +1 -0
- package/dist/mask/legacy_0254/pii_normalize.d.ts +38 -0
- package/dist/mask/legacy_0254/pii_normalize.d.ts.map +1 -0
- package/dist/mask/legacy_0254/pii_normalize.js +96 -0
- package/dist/mask/legacy_0254/pii_normalize.js.map +1 -0
- package/dist/mask/legacy_0254/safe-object.d.ts +14 -0
- package/dist/mask/legacy_0254/safe-object.d.ts.map +1 -0
- package/dist/mask/legacy_0254/safe-object.js +27 -0
- package/dist/mask/legacy_0254/safe-object.js.map +1 -0
- package/dist/mask/legacy_0254/tckn.d.ts +10 -0
- package/dist/mask/legacy_0254/tckn.d.ts.map +1 -0
- package/dist/mask/legacy_0254/tckn.js +70 -0
- package/dist/mask/legacy_0254/tckn.js.map +1 -0
- package/dist/mask/legacy_0254/types.d.ts +207 -0
- package/dist/mask/legacy_0254/types.d.ts.map +1 -0
- package/dist/mask/legacy_0254/types.js +2 -0
- package/dist/mask/legacy_0254/types.js.map +1 -0
- package/dist/mask-text.d.ts +11 -1
- package/dist/mask-text.d.ts.map +1 -1
- package/dist/mask-text.js +189 -30
- package/dist/mask-text.js.map +1 -1
- package/docs/claims/reviews/0.3.0.md +72 -0
- package/docs/claims/reviews/0.3.1.md +27 -0
- package/package.json +3 -1
|
@@ -0,0 +1,865 @@
|
|
|
1
|
+
// Covers the source spans implied by a frozen 0.2.54 mask, or refuses the field.
|
|
2
|
+
import { maskSensitiveText } from './legacy_0254/legacy_mask_text.js';
|
|
3
|
+
import { legacyMaskPiiString, maskByColumnName, } from './legacy_0254/legacy_governance_mask.js';
|
|
4
|
+
import { columnMatches, CUSTOM_PATTERN_MAX_MATCHES, matchCustomPatternSource, } from '../custom_patterns.js';
|
|
5
|
+
import { resolveScanCharCap } from './legacy_0254/digit_pii.js';
|
|
6
|
+
const BUILTIN_TOKENS = ['[MASKED_SECRET]', '[MASKED_PII]'];
|
|
7
|
+
const PRIVATE_OPEN = '\uE000';
|
|
8
|
+
const PRIVATE_CLOSE = '\uE001';
|
|
9
|
+
const LABELS = [
|
|
10
|
+
'adı soyadı',
|
|
11
|
+
'adi soyadi',
|
|
12
|
+
'ad soyad',
|
|
13
|
+
'full name',
|
|
14
|
+
'müşteri',
|
|
15
|
+
'musteri',
|
|
16
|
+
'yetkili',
|
|
17
|
+
'temsilci',
|
|
18
|
+
'customer',
|
|
19
|
+
'contact',
|
|
20
|
+
'kişi',
|
|
21
|
+
'kisi',
|
|
22
|
+
'name',
|
|
23
|
+
];
|
|
24
|
+
const TITLES = [
|
|
25
|
+
'Sayın',
|
|
26
|
+
'Sayin',
|
|
27
|
+
'Bayan',
|
|
28
|
+
'Mrs.',
|
|
29
|
+
'Mr.',
|
|
30
|
+
'Ms.',
|
|
31
|
+
'Prof.',
|
|
32
|
+
'Dr.',
|
|
33
|
+
'Av.',
|
|
34
|
+
'Bay',
|
|
35
|
+
'Sn.',
|
|
36
|
+
];
|
|
37
|
+
class BudgetExceeded extends Error {
|
|
38
|
+
}
|
|
39
|
+
class Budget {
|
|
40
|
+
left;
|
|
41
|
+
constructor(limit) {
|
|
42
|
+
this.left = limit;
|
|
43
|
+
}
|
|
44
|
+
tick(n = 1) {
|
|
45
|
+
this.left -= n;
|
|
46
|
+
if (this.left < 0)
|
|
47
|
+
throw new BudgetExceeded();
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
let alignCalls = 0;
|
|
51
|
+
export function alignCallCount() {
|
|
52
|
+
return alignCalls;
|
|
53
|
+
}
|
|
54
|
+
export function resetAlignCalls() {
|
|
55
|
+
alignCalls = 0;
|
|
56
|
+
}
|
|
57
|
+
export function alignRegions(source, output, tokens, mode) {
|
|
58
|
+
const budget = new Budget(1_000_000_000);
|
|
59
|
+
try {
|
|
60
|
+
const aligned = mode === 'plain'
|
|
61
|
+
? alignOutput(source, output, tokens, budget)
|
|
62
|
+
: alignTrimmed(source, output, tokens, budget);
|
|
63
|
+
if (!aligned.ok)
|
|
64
|
+
return aligned;
|
|
65
|
+
return { ok: true, spans: unionOverlaps(aligned.spans) };
|
|
66
|
+
}
|
|
67
|
+
catch (err) {
|
|
68
|
+
if (err instanceof BudgetExceeded)
|
|
69
|
+
return { ok: false, reason: 'budget' };
|
|
70
|
+
return { ok: false, reason: 'exception' };
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
export function beltCover(input) {
|
|
74
|
+
try {
|
|
75
|
+
return cover(input);
|
|
76
|
+
}
|
|
77
|
+
catch (err) {
|
|
78
|
+
if (err instanceof BudgetExceeded)
|
|
79
|
+
return { ok: false, reason: 'budget' };
|
|
80
|
+
return { ok: false, reason: 'exception' };
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
function cover(input) {
|
|
84
|
+
if (hasPrivateUse(input.raw) || hasPrivateUse(input.normalized)) {
|
|
85
|
+
return { ok: false, reason: 'private-use' };
|
|
86
|
+
}
|
|
87
|
+
const policy = input.policy ?? {};
|
|
88
|
+
const capArg = input.scanCharCap ?? policy.scanCharCap;
|
|
89
|
+
const budget = new Budget(input.budget ?? 64 * (Math.max(input.normalized.length, 1) + input.raw.length + 1));
|
|
90
|
+
budget.tick(1);
|
|
91
|
+
if (input.pipeline === 'governance' && input.column) {
|
|
92
|
+
const named = maskByColumnName(input.column);
|
|
93
|
+
if (named)
|
|
94
|
+
return whole(named);
|
|
95
|
+
}
|
|
96
|
+
if (input.raw.length > resolveScanCharCap(capArg))
|
|
97
|
+
return whole('[MASKED_PII]');
|
|
98
|
+
const legacy = runLegacy(input, policy, capArg);
|
|
99
|
+
const tokens = tokenVocab(input.customPatterns);
|
|
100
|
+
const stepped = applyRules(legacy, input.customPatterns, input.column, tokens, budget);
|
|
101
|
+
if (!stepped.ok)
|
|
102
|
+
return stepped;
|
|
103
|
+
if (stepped.wholeField)
|
|
104
|
+
return whole(stepped.wholeField);
|
|
105
|
+
if (stepped.text === input.normalized)
|
|
106
|
+
return { ok: true, spans: [], added: 0 };
|
|
107
|
+
const aligned = alignTrimmed(input.normalized, stepped.text, tokens, budget);
|
|
108
|
+
if (!aligned.ok)
|
|
109
|
+
return aligned;
|
|
110
|
+
// Union first so splitAtLineBreaks walks each source character once.
|
|
111
|
+
// Overlapping widest ranges from repeated titles become one span; a later
|
|
112
|
+
// split still drops a label that sat inside a mask, then edges are trimmed.
|
|
113
|
+
const spans = trimSpanEdges(input.normalized, dropExactLabels(input.normalized, splitAtLineBreaks(input.normalized, unionOverlaps(trimSpanEdges(input.normalized, aligned.spans)))));
|
|
114
|
+
return { ok: true, spans, added: spans.length };
|
|
115
|
+
}
|
|
116
|
+
// Frozen 0.2.54 name shape treats a line break as a word gap, so a mask can
|
|
117
|
+
// end on the next line's label. Split only when a piece is that label; a
|
|
118
|
+
// title run with no label stays one range, so the mask count does not grow.
|
|
119
|
+
function splitAtLineBreaks(source, spans) {
|
|
120
|
+
const out = [];
|
|
121
|
+
for (let s = 0; s < spans.length; s++) {
|
|
122
|
+
const sp = spans[s];
|
|
123
|
+
const kept = [];
|
|
124
|
+
let start = sp.start;
|
|
125
|
+
let split = false;
|
|
126
|
+
for (let i = sp.start; i < sp.end; i++) {
|
|
127
|
+
const c = source.charCodeAt(i);
|
|
128
|
+
if (isLineBreakCode(c)) {
|
|
129
|
+
if (start < i) {
|
|
130
|
+
if (isExactLabel(source, start, i))
|
|
131
|
+
split = true;
|
|
132
|
+
else if (!isBlank(source, start, i))
|
|
133
|
+
kept.push({ start, end: i });
|
|
134
|
+
}
|
|
135
|
+
start = i + 1;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
if (start < sp.end) {
|
|
139
|
+
if (isExactLabel(source, start, sp.end))
|
|
140
|
+
split = true;
|
|
141
|
+
else if (!isBlank(source, start, sp.end))
|
|
142
|
+
kept.push({ start, end: sp.end });
|
|
143
|
+
}
|
|
144
|
+
if (!split) {
|
|
145
|
+
out.push(sp);
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
for (let k = 0; k < kept.length; k++)
|
|
149
|
+
out.push(kept[k]);
|
|
150
|
+
}
|
|
151
|
+
return out;
|
|
152
|
+
}
|
|
153
|
+
function dropExactLabels(source, spans) {
|
|
154
|
+
const kept = [];
|
|
155
|
+
for (let i = 0; i < spans.length; i++) {
|
|
156
|
+
const sp = spans[i];
|
|
157
|
+
if (isExactLabel(source, sp.start, sp.end))
|
|
158
|
+
continue;
|
|
159
|
+
kept.push(sp);
|
|
160
|
+
}
|
|
161
|
+
return kept;
|
|
162
|
+
}
|
|
163
|
+
function isLineBreakCode(c) {
|
|
164
|
+
return c === 10 || c === 13 || c === 0x2028 || c === 0x2029 || c === 0x85;
|
|
165
|
+
}
|
|
166
|
+
function isHorizWs(ch) {
|
|
167
|
+
const c = ch.charCodeAt(0);
|
|
168
|
+
if (isLineBreakCode(c))
|
|
169
|
+
return false;
|
|
170
|
+
return /\s/u.test(ch);
|
|
171
|
+
}
|
|
172
|
+
function followedByColon(source, at) {
|
|
173
|
+
let i = at;
|
|
174
|
+
while (i < source.length && isHorizWs(source[i]))
|
|
175
|
+
i++;
|
|
176
|
+
return source[i] === ':';
|
|
177
|
+
}
|
|
178
|
+
function trimmedEnds(source, start, end) {
|
|
179
|
+
let lo = start;
|
|
180
|
+
let hi = end;
|
|
181
|
+
while (lo < hi && isWs(source[lo]))
|
|
182
|
+
lo++;
|
|
183
|
+
while (hi > lo && isWs(source[hi - 1]))
|
|
184
|
+
hi--;
|
|
185
|
+
return [lo, hi];
|
|
186
|
+
}
|
|
187
|
+
function isBlank(source, start, end) {
|
|
188
|
+
const [lo, hi] = trimmedEnds(source, start, end);
|
|
189
|
+
return lo >= hi;
|
|
190
|
+
}
|
|
191
|
+
function isExactLabel(source, start, end) {
|
|
192
|
+
const [lo, hi] = trimmedEnds(source, start, end);
|
|
193
|
+
if (lo >= hi)
|
|
194
|
+
return false;
|
|
195
|
+
const lower = source.slice(lo, hi).toLowerCase();
|
|
196
|
+
let known = false;
|
|
197
|
+
for (let i = 0; i < LABELS.length; i++) {
|
|
198
|
+
if (LABELS[i] === lower) {
|
|
199
|
+
known = true;
|
|
200
|
+
break;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
if (!known)
|
|
204
|
+
return false;
|
|
205
|
+
return followedByColon(source, hi);
|
|
206
|
+
}
|
|
207
|
+
// Repeated titles fit many placements, so the widest range of each token
|
|
208
|
+
// overlaps the next. Gaps inside those ranges are built per token, and the
|
|
209
|
+
// work grows with the square of the mask count. Overlaps become one range.
|
|
210
|
+
function unionOverlaps(spans) {
|
|
211
|
+
if (spans.length < 2)
|
|
212
|
+
return spans;
|
|
213
|
+
let ordered = spans;
|
|
214
|
+
for (let i = 1; i < spans.length; i++) {
|
|
215
|
+
if (spans[i].start < spans[i - 1].start) {
|
|
216
|
+
ordered = spans.slice().sort((a, b) => a.start - b.start || a.end - b.end);
|
|
217
|
+
break;
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
for (let i = 1; i < ordered.length; i++) {
|
|
221
|
+
if (ordered[i].start < ordered[i - 1].end)
|
|
222
|
+
return mergeOverlaps(ordered);
|
|
223
|
+
}
|
|
224
|
+
return ordered === spans ? spans : ordered;
|
|
225
|
+
}
|
|
226
|
+
function mergeOverlaps(ordered) {
|
|
227
|
+
const merged = [];
|
|
228
|
+
for (let i = 0; i < ordered.length; i++) {
|
|
229
|
+
const sp = ordered[i];
|
|
230
|
+
if (sp.end <= sp.start)
|
|
231
|
+
continue;
|
|
232
|
+
const cur = merged[merged.length - 1];
|
|
233
|
+
if (cur && sp.start < cur.end) {
|
|
234
|
+
if (sp.end > cur.end)
|
|
235
|
+
cur.end = sp.end;
|
|
236
|
+
}
|
|
237
|
+
else {
|
|
238
|
+
merged.push({ start: sp.start, end: sp.end });
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
return merged;
|
|
242
|
+
}
|
|
243
|
+
// A space is not a value. A span that starts on one is glued to the word
|
|
244
|
+
// on its left, and apply then swallows that word.
|
|
245
|
+
function trimSpanEdges(source, spans) {
|
|
246
|
+
const kept = [];
|
|
247
|
+
for (let i = 0; i < spans.length; i++) {
|
|
248
|
+
let start = spans[i].start;
|
|
249
|
+
let end = spans[i].end;
|
|
250
|
+
while (start < end && isWs(source[start]))
|
|
251
|
+
start++;
|
|
252
|
+
while (end > start && isWs(source[end - 1]))
|
|
253
|
+
end--;
|
|
254
|
+
if (start < end)
|
|
255
|
+
kept.push({ start, end });
|
|
256
|
+
}
|
|
257
|
+
return kept;
|
|
258
|
+
}
|
|
259
|
+
function whole(token) {
|
|
260
|
+
return { ok: true, spans: [], wholeField: token, added: 1 };
|
|
261
|
+
}
|
|
262
|
+
function hasPrivateUse(text) {
|
|
263
|
+
return text.includes(PRIVATE_OPEN) || text.includes(PRIVATE_CLOSE);
|
|
264
|
+
}
|
|
265
|
+
function runLegacy(input, policy, capArg) {
|
|
266
|
+
if (input.pipeline === 'text') {
|
|
267
|
+
return maskSensitiveText(input.raw, {
|
|
268
|
+
scanCharCap: capArg,
|
|
269
|
+
detectors: policy.detectors,
|
|
270
|
+
customPatterns: [],
|
|
271
|
+
});
|
|
272
|
+
}
|
|
273
|
+
const opts = {
|
|
274
|
+
policy: { ...policy, scanCharCap: capArg },
|
|
275
|
+
customPatterns: [],
|
|
276
|
+
column: input.column,
|
|
277
|
+
};
|
|
278
|
+
return legacyMaskPiiString(input.raw, opts).masked;
|
|
279
|
+
}
|
|
280
|
+
function tokenVocab(rules) {
|
|
281
|
+
const seen = new Set(BUILTIN_TOKENS);
|
|
282
|
+
for (let i = 0; i < rules.length; i++)
|
|
283
|
+
seen.add(rules[i].label);
|
|
284
|
+
const out = [];
|
|
285
|
+
for (const token of seen)
|
|
286
|
+
out.push(token);
|
|
287
|
+
out.sort((a, b) => b.length - a.length);
|
|
288
|
+
return out;
|
|
289
|
+
}
|
|
290
|
+
function applyRules(text, rules, column, tokens, budget) {
|
|
291
|
+
let current = text;
|
|
292
|
+
let exact = [];
|
|
293
|
+
for (const rule of rules) {
|
|
294
|
+
budget.tick(1);
|
|
295
|
+
if (rule.columns.length > 0) {
|
|
296
|
+
if (!column || !columnMatches(rule.columns, column))
|
|
297
|
+
continue;
|
|
298
|
+
}
|
|
299
|
+
const matches = collectReplaceMatches(current, rule, budget);
|
|
300
|
+
if (matches.length === 0)
|
|
301
|
+
continue;
|
|
302
|
+
if (matches.length > CUSTOM_PATTERN_MAX_MATCHES) {
|
|
303
|
+
return { ok: true, text: rule.label, wholeField: rule.label };
|
|
304
|
+
}
|
|
305
|
+
if (chainHit(current, matches, tokens, exact, budget)) {
|
|
306
|
+
return { ok: false, reason: 'custom-chain' };
|
|
307
|
+
}
|
|
308
|
+
const tracked = applyTracked(current, matches, rule.label, exact);
|
|
309
|
+
current = tracked.text;
|
|
310
|
+
exact = tracked.exact;
|
|
311
|
+
}
|
|
312
|
+
return { ok: true, text: current };
|
|
313
|
+
}
|
|
314
|
+
function collectReplaceMatches(text, rule, budget) {
|
|
315
|
+
const found = ruleMatches(rule, text);
|
|
316
|
+
for (let i = 0; i < found.length; i++)
|
|
317
|
+
budget.tick(1);
|
|
318
|
+
return found;
|
|
319
|
+
}
|
|
320
|
+
function ruleMatches(rule, text) {
|
|
321
|
+
if (typeof rule.matches === 'function')
|
|
322
|
+
return rule.matches(text);
|
|
323
|
+
return matchCustomPatternSource(rule.re.source, text);
|
|
324
|
+
}
|
|
325
|
+
function chainHit(text, matches, tokens, exact, budget) {
|
|
326
|
+
const spans = tokenSpans(text, tokens, budget);
|
|
327
|
+
let i = 0;
|
|
328
|
+
let j = 0;
|
|
329
|
+
while (i < matches.length && j < spans.length) {
|
|
330
|
+
const match = matches[i];
|
|
331
|
+
const span = spans[j];
|
|
332
|
+
const rel = relate(match.start, match.end, span[0], span[1]);
|
|
333
|
+
if (rel === 'overlap')
|
|
334
|
+
return true;
|
|
335
|
+
if (rel === 'adjacent' && !isExact(span, exact))
|
|
336
|
+
return true;
|
|
337
|
+
if (match.end <= span[0])
|
|
338
|
+
i++;
|
|
339
|
+
else
|
|
340
|
+
j++;
|
|
341
|
+
}
|
|
342
|
+
return false;
|
|
343
|
+
}
|
|
344
|
+
function isExact(span, exact) {
|
|
345
|
+
return exact.some((item) => item[0] === span[0] && item[1] === span[1]);
|
|
346
|
+
}
|
|
347
|
+
function relate(a0, a1, b0, b1) {
|
|
348
|
+
if (a0 === a1) {
|
|
349
|
+
if (a0 > b0 && a0 < b1)
|
|
350
|
+
return 'overlap';
|
|
351
|
+
if (a0 === b0 || a0 === b1)
|
|
352
|
+
return 'adjacent';
|
|
353
|
+
return 'clear';
|
|
354
|
+
}
|
|
355
|
+
if (a0 < b1 && b0 < a1)
|
|
356
|
+
return 'overlap';
|
|
357
|
+
if (a1 === b0 || b1 === a0)
|
|
358
|
+
return 'adjacent';
|
|
359
|
+
return 'clear';
|
|
360
|
+
}
|
|
361
|
+
function tokenSpans(text, tokens, budget) {
|
|
362
|
+
const spans = [];
|
|
363
|
+
let i = 0;
|
|
364
|
+
while (i < text.length) {
|
|
365
|
+
budget.tick(1);
|
|
366
|
+
if (text.charCodeAt(i) !== 91) {
|
|
367
|
+
i++;
|
|
368
|
+
continue;
|
|
369
|
+
}
|
|
370
|
+
const tok = tokens.find((token) => token.length > 0 && text.startsWith(token, i));
|
|
371
|
+
if (tok) {
|
|
372
|
+
spans.push([i, i + tok.length]);
|
|
373
|
+
i += tok.length;
|
|
374
|
+
}
|
|
375
|
+
else {
|
|
376
|
+
i++;
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
return spans;
|
|
380
|
+
}
|
|
381
|
+
function applyTracked(text, matches, label, exact) {
|
|
382
|
+
let out = '';
|
|
383
|
+
let cursor = 0;
|
|
384
|
+
const inserted = [];
|
|
385
|
+
for (const match of matches) {
|
|
386
|
+
out += text.slice(cursor, match.start);
|
|
387
|
+
const at = out.length;
|
|
388
|
+
out += label;
|
|
389
|
+
inserted.push([at, at + label.length]);
|
|
390
|
+
cursor = match.end;
|
|
391
|
+
}
|
|
392
|
+
out += text.slice(cursor);
|
|
393
|
+
const mapped = [];
|
|
394
|
+
for (const range of exact) {
|
|
395
|
+
if (matches.some((match) => range[0] < match.end && match.start < range[1]))
|
|
396
|
+
continue;
|
|
397
|
+
mapped.push([mapPos(range[0], matches, label), mapPos(range[1], matches, label)]);
|
|
398
|
+
}
|
|
399
|
+
return { text: out, exact: inserted.concat(mapped) };
|
|
400
|
+
}
|
|
401
|
+
function mapPos(pos, matches, label) {
|
|
402
|
+
let delta = 0;
|
|
403
|
+
for (const match of matches) {
|
|
404
|
+
if (match.end <= pos)
|
|
405
|
+
delta += label.length - (match.end - match.start);
|
|
406
|
+
else
|
|
407
|
+
break;
|
|
408
|
+
}
|
|
409
|
+
return pos + delta;
|
|
410
|
+
}
|
|
411
|
+
const LABEL_EDGE = 48;
|
|
412
|
+
function alignTrimmed(source, output, tokens, budget) {
|
|
413
|
+
const bounds = trimmedBounds(source, output, tokens, budget);
|
|
414
|
+
if (!bounds)
|
|
415
|
+
return alignOutput(source, output, tokens, budget);
|
|
416
|
+
const aligned = alignOutput(source.slice(bounds.srcLo, bounds.srcHi), output.slice(bounds.outLo, bounds.outHi), tokens, budget);
|
|
417
|
+
if (!aligned.ok || bounds.srcLo === 0)
|
|
418
|
+
return aligned;
|
|
419
|
+
const spans = [];
|
|
420
|
+
for (let i = 0; i < aligned.spans.length; i++) {
|
|
421
|
+
const span = aligned.spans[i];
|
|
422
|
+
spans.push({ start: span.start + bounds.srcLo, end: span.end + bounds.srcLo });
|
|
423
|
+
}
|
|
424
|
+
return { ok: true, spans };
|
|
425
|
+
}
|
|
426
|
+
function trimmedBounds(source, output, tokens, budget) {
|
|
427
|
+
if (source === output)
|
|
428
|
+
return null;
|
|
429
|
+
let outLo = 0;
|
|
430
|
+
const head = Math.min(source.length, output.length);
|
|
431
|
+
while (outLo < head && source.charCodeAt(outLo) === output.charCodeAt(outLo)) {
|
|
432
|
+
budget.tick(1);
|
|
433
|
+
outLo++;
|
|
434
|
+
}
|
|
435
|
+
let suffix = 0;
|
|
436
|
+
while (suffix < source.length - outLo &&
|
|
437
|
+
suffix < output.length - outLo &&
|
|
438
|
+
source.charCodeAt(source.length - 1 - suffix) === output.charCodeAt(output.length - 1 - suffix)) {
|
|
439
|
+
budget.tick(1);
|
|
440
|
+
suffix++;
|
|
441
|
+
}
|
|
442
|
+
if (outLo === 0 && suffix === 0)
|
|
443
|
+
return null;
|
|
444
|
+
let outHi = output.length - suffix;
|
|
445
|
+
snapTokenEdges(output, tokens, (lo, hi) => {
|
|
446
|
+
outLo = lo;
|
|
447
|
+
outHi = hi;
|
|
448
|
+
}, outLo, outHi);
|
|
449
|
+
coverDiscardedTokens(output, tokens, budget, (lo, hi) => {
|
|
450
|
+
outLo = lo;
|
|
451
|
+
outHi = hi;
|
|
452
|
+
}, outLo, outHi);
|
|
453
|
+
outLo = pullLabelEdge(output, outLo, tokens);
|
|
454
|
+
outHi = pushLabelEdge(output, outHi, tokens);
|
|
455
|
+
snapTokenEdges(output, tokens, (lo, hi) => {
|
|
456
|
+
outLo = lo;
|
|
457
|
+
outHi = hi;
|
|
458
|
+
}, outLo, outHi);
|
|
459
|
+
coverDiscardedTokens(output, tokens, budget, (lo, hi) => {
|
|
460
|
+
outLo = lo;
|
|
461
|
+
outHi = hi;
|
|
462
|
+
}, outLo, outHi);
|
|
463
|
+
if (outLo < 0)
|
|
464
|
+
outLo = 0;
|
|
465
|
+
if (outHi > output.length)
|
|
466
|
+
outHi = output.length;
|
|
467
|
+
const suffixLen = output.length - outHi;
|
|
468
|
+
const srcLo = outLo;
|
|
469
|
+
const srcHi = source.length - suffixLen;
|
|
470
|
+
if (srcHi < srcLo || outHi < outLo || suffixLen < 0)
|
|
471
|
+
return null;
|
|
472
|
+
if (srcLo === 0 && suffixLen === 0)
|
|
473
|
+
return null;
|
|
474
|
+
return { srcLo, srcHi, outLo, outHi };
|
|
475
|
+
}
|
|
476
|
+
function snapTokenEdges(output, tokens, write, outLo, outHi) {
|
|
477
|
+
const loHit = tokenCovering(output, outLo, tokens);
|
|
478
|
+
if (loHit)
|
|
479
|
+
outLo = loHit.start;
|
|
480
|
+
const hiHit = tokenCovering(output, outHi, tokens);
|
|
481
|
+
if (hiHit)
|
|
482
|
+
outHi = hiHit.end;
|
|
483
|
+
write(outLo, outHi);
|
|
484
|
+
}
|
|
485
|
+
function coverDiscardedTokens(output, tokens, budget, write, outLo, outHi) {
|
|
486
|
+
const before = scanTokens(output, 0, outLo, tokens, budget);
|
|
487
|
+
if (before.length > 0)
|
|
488
|
+
outLo = before[0].start;
|
|
489
|
+
const after = scanTokens(output, outHi, output.length, tokens, budget);
|
|
490
|
+
if (after.length > 0)
|
|
491
|
+
outHi = after[after.length - 1].end;
|
|
492
|
+
write(outLo, outHi);
|
|
493
|
+
}
|
|
494
|
+
function tokenCovering(output, index, tokens) {
|
|
495
|
+
if (index <= 0 || index >= output.length)
|
|
496
|
+
return null;
|
|
497
|
+
let max = 1;
|
|
498
|
+
for (let t = 0; t < tokens.length; t++) {
|
|
499
|
+
if (tokens[t].length > max)
|
|
500
|
+
max = tokens[t].length;
|
|
501
|
+
}
|
|
502
|
+
const from = Math.max(0, index - max + 1);
|
|
503
|
+
for (let s = from; s < index; s++) {
|
|
504
|
+
if (output.charCodeAt(s) !== 91)
|
|
505
|
+
continue;
|
|
506
|
+
const text = tokens.find((token) => token.length > 0 && output.startsWith(token, s));
|
|
507
|
+
if (!text)
|
|
508
|
+
continue;
|
|
509
|
+
const end = s + text.length;
|
|
510
|
+
if (index < end)
|
|
511
|
+
return { start: s, end };
|
|
512
|
+
}
|
|
513
|
+
return null;
|
|
514
|
+
}
|
|
515
|
+
function scanTokens(output, from, to, tokens, budget) {
|
|
516
|
+
const found = [];
|
|
517
|
+
const end = Math.min(to, output.length);
|
|
518
|
+
let i = Math.max(0, from);
|
|
519
|
+
while (i < end) {
|
|
520
|
+
budget.tick(1);
|
|
521
|
+
if (output.charCodeAt(i) !== 91) {
|
|
522
|
+
i++;
|
|
523
|
+
continue;
|
|
524
|
+
}
|
|
525
|
+
const text = tokens.find((token) => token.length > 0 && output.startsWith(token, i));
|
|
526
|
+
if (!text) {
|
|
527
|
+
i++;
|
|
528
|
+
continue;
|
|
529
|
+
}
|
|
530
|
+
found.push({ start: i, end: i + text.length, text });
|
|
531
|
+
i += text.length;
|
|
532
|
+
}
|
|
533
|
+
return found;
|
|
534
|
+
}
|
|
535
|
+
function pullLabelEdge(output, outLo, tokens) {
|
|
536
|
+
const found = nextPiiToken(output, outLo, tokens);
|
|
537
|
+
if (!found)
|
|
538
|
+
return outLo;
|
|
539
|
+
const from = Math.max(0, found.start - LABEL_EDGE);
|
|
540
|
+
const at = markerIndex(output.slice(from, found.start), from);
|
|
541
|
+
// endMin sits on the canonical space before [MASKED_PII]. Leaving that
|
|
542
|
+
// space in the trimmed prefix drops it from the span.
|
|
543
|
+
if (at !== null && outLo > at)
|
|
544
|
+
return at;
|
|
545
|
+
return outLo;
|
|
546
|
+
}
|
|
547
|
+
function pushLabelEdge(output, outHi, tokens) {
|
|
548
|
+
const found = nextPiiToken(output, outHi, tokens);
|
|
549
|
+
if (!found || found.start === outHi)
|
|
550
|
+
return outHi;
|
|
551
|
+
const from = Math.max(0, found.start - LABEL_EDGE);
|
|
552
|
+
if (outHi <= from)
|
|
553
|
+
return outHi;
|
|
554
|
+
if (markerIndex(output.slice(from, found.start), from) === null)
|
|
555
|
+
return outHi;
|
|
556
|
+
if (outHi > from && outHi < found.start)
|
|
557
|
+
return found.start;
|
|
558
|
+
return outHi;
|
|
559
|
+
}
|
|
560
|
+
function nextPiiToken(output, from, tokens) {
|
|
561
|
+
const hits = scanTokens(output, from, Math.min(output.length, from + LABEL_EDGE), tokens, new Budget(1_000_000));
|
|
562
|
+
for (let i = 0; i < hits.length; i++) {
|
|
563
|
+
if (hits[i].text === '[MASKED_PII]')
|
|
564
|
+
return hits[i];
|
|
565
|
+
}
|
|
566
|
+
return null;
|
|
567
|
+
}
|
|
568
|
+
function markerIndex(tail, tailAt) {
|
|
569
|
+
if (labelParts(tail)) {
|
|
570
|
+
const head = tail.slice(0, -2);
|
|
571
|
+
return tailAt + head.length - labelLength(head);
|
|
572
|
+
}
|
|
573
|
+
if (titleParts(tail)) {
|
|
574
|
+
const head = tail.slice(0, -1);
|
|
575
|
+
return tailAt + head.length - titleLength(head);
|
|
576
|
+
}
|
|
577
|
+
return null;
|
|
578
|
+
}
|
|
579
|
+
function alignOutput(source, output, tokens, budget) {
|
|
580
|
+
alignCalls++;
|
|
581
|
+
const pieces = parsePieces(output, tokens, budget);
|
|
582
|
+
const literals = pieces.filter((piece) => piece.kind === 'lit');
|
|
583
|
+
const earliestEnd = new Array(literals.length).fill(0);
|
|
584
|
+
const latestStart = new Array(literals.length).fill(0);
|
|
585
|
+
const first = matchLiteral(source, 0, literals[0], budget);
|
|
586
|
+
if (!first || first.start !== 0)
|
|
587
|
+
return { ok: false, reason: 'unaligned' };
|
|
588
|
+
earliestEnd[0] = first.endMin;
|
|
589
|
+
latestStart[0] = 0;
|
|
590
|
+
const last = literals.length - 1;
|
|
591
|
+
const anchored = matchLiteralEnding(source, source.length, literals[last], budget);
|
|
592
|
+
if (!anchored)
|
|
593
|
+
return { ok: false, reason: 'unaligned' };
|
|
594
|
+
latestStart[last] = anchored.start;
|
|
595
|
+
earliestEnd[last] = source.length;
|
|
596
|
+
for (let i = last - 1; i >= 1; i--) {
|
|
597
|
+
const found = latestLiteral(source, literals[i], latestStart[i + 1], budget);
|
|
598
|
+
if (!found)
|
|
599
|
+
return { ok: false, reason: 'unaligned' };
|
|
600
|
+
latestStart[i] = found.start;
|
|
601
|
+
}
|
|
602
|
+
for (let i = 1; i < last; i++) {
|
|
603
|
+
const found = earliestLiteral(source, literals[i], earliestEnd[i - 1], budget);
|
|
604
|
+
if (!found)
|
|
605
|
+
return { ok: false, reason: 'unaligned' };
|
|
606
|
+
earliestEnd[i] = found.endMin;
|
|
607
|
+
}
|
|
608
|
+
if (last > 0 && earliestEnd[last - 1] > latestStart[last]) {
|
|
609
|
+
return { ok: false, reason: 'unaligned' };
|
|
610
|
+
}
|
|
611
|
+
const spans = [];
|
|
612
|
+
let lit = 0;
|
|
613
|
+
for (const piece of pieces) {
|
|
614
|
+
if (piece.kind === 'lit') {
|
|
615
|
+
lit++;
|
|
616
|
+
continue;
|
|
617
|
+
}
|
|
618
|
+
const start = earliestEnd[lit - 1];
|
|
619
|
+
const end = latestStart[lit];
|
|
620
|
+
if (end < start)
|
|
621
|
+
return { ok: false, reason: 'unaligned' };
|
|
622
|
+
if (end > start)
|
|
623
|
+
spans.push({ start, end });
|
|
624
|
+
}
|
|
625
|
+
return { ok: true, spans };
|
|
626
|
+
}
|
|
627
|
+
function parsePieces(output, tokens, budget) {
|
|
628
|
+
const raw = [];
|
|
629
|
+
let i = 0;
|
|
630
|
+
while (i < output.length) {
|
|
631
|
+
budget.tick(1);
|
|
632
|
+
const tok = tokens.find((token) => token.length > 0 && output.startsWith(token, i));
|
|
633
|
+
if (tok) {
|
|
634
|
+
raw.push({ kind: 'tok', text: tok });
|
|
635
|
+
i += tok.length;
|
|
636
|
+
}
|
|
637
|
+
else {
|
|
638
|
+
const prev = raw[raw.length - 1];
|
|
639
|
+
if (prev && prev.kind === 'lit')
|
|
640
|
+
prev.text += output[i];
|
|
641
|
+
else
|
|
642
|
+
raw.push({ kind: 'lit', text: output[i] });
|
|
643
|
+
i++;
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
if (raw.length === 0 || raw[0].kind === 'tok')
|
|
647
|
+
raw.unshift({ kind: 'lit', text: '' });
|
|
648
|
+
if (raw[raw.length - 1].kind === 'tok')
|
|
649
|
+
raw.push({ kind: 'lit', text: '' });
|
|
650
|
+
const spaced = [];
|
|
651
|
+
for (const piece of raw) {
|
|
652
|
+
if (piece.kind === 'tok' && spaced.length > 0 && spaced[spaced.length - 1].kind === 'tok') {
|
|
653
|
+
spaced.push({ kind: 'lit', text: '' });
|
|
654
|
+
}
|
|
655
|
+
spaced.push(piece);
|
|
656
|
+
}
|
|
657
|
+
const pieces = [];
|
|
658
|
+
for (let n = 0; n < spaced.length; n++) {
|
|
659
|
+
const piece = spaced[n];
|
|
660
|
+
if (piece.kind === 'tok') {
|
|
661
|
+
pieces.push(piece);
|
|
662
|
+
continue;
|
|
663
|
+
}
|
|
664
|
+
const next = spaced[n + 1];
|
|
665
|
+
const nextTok = next && next.kind === 'tok' ? next.text : null;
|
|
666
|
+
pieces.push({ kind: 'lit', text: piece.text, mode: literalMode(piece.text, nextTok) });
|
|
667
|
+
}
|
|
668
|
+
return pieces;
|
|
669
|
+
}
|
|
670
|
+
function literalMode(text, nextTok) {
|
|
671
|
+
if (nextTok !== '[MASKED_PII]')
|
|
672
|
+
return 'exact';
|
|
673
|
+
if (labelParts(text))
|
|
674
|
+
return 'label';
|
|
675
|
+
if (titleParts(text))
|
|
676
|
+
return 'title';
|
|
677
|
+
return 'exact';
|
|
678
|
+
}
|
|
679
|
+
function boundaryOk(text, index) {
|
|
680
|
+
if (index <= 0)
|
|
681
|
+
return true;
|
|
682
|
+
return !/[\p{L}\p{N}]/u.test(text[index - 1] ?? '');
|
|
683
|
+
}
|
|
684
|
+
function labelParts(literal) {
|
|
685
|
+
if (!literal.endsWith(': '))
|
|
686
|
+
return null;
|
|
687
|
+
const head = literal.slice(0, -2);
|
|
688
|
+
const lower = head.toLowerCase();
|
|
689
|
+
for (const label of LABELS) {
|
|
690
|
+
if (!lower.endsWith(label))
|
|
691
|
+
continue;
|
|
692
|
+
const at = head.length - label.length;
|
|
693
|
+
if (!boundaryOk(head, at))
|
|
694
|
+
continue;
|
|
695
|
+
return { rigid: head };
|
|
696
|
+
}
|
|
697
|
+
return null;
|
|
698
|
+
}
|
|
699
|
+
function titleParts(literal) {
|
|
700
|
+
if (!literal.endsWith(' ') || literal.endsWith(': '))
|
|
701
|
+
return null;
|
|
702
|
+
const head = literal.slice(0, -1);
|
|
703
|
+
for (const title of TITLES) {
|
|
704
|
+
if (!head.endsWith(title))
|
|
705
|
+
continue;
|
|
706
|
+
const at = head.length - title.length;
|
|
707
|
+
if (!boundaryOk(head, at))
|
|
708
|
+
continue;
|
|
709
|
+
return { rigid: head };
|
|
710
|
+
}
|
|
711
|
+
return null;
|
|
712
|
+
}
|
|
713
|
+
function isWs(ch) {
|
|
714
|
+
return /\s/u.test(ch);
|
|
715
|
+
}
|
|
716
|
+
function matchLiteral(source, at, lit, budget) {
|
|
717
|
+
budget.tick(1);
|
|
718
|
+
if (lit.mode === 'exact') {
|
|
719
|
+
if (lit.text.length === 0) {
|
|
720
|
+
if (at > source.length)
|
|
721
|
+
return null;
|
|
722
|
+
return { start: at, endMin: at, endMax: at };
|
|
723
|
+
}
|
|
724
|
+
if (!source.startsWith(lit.text, at))
|
|
725
|
+
return null;
|
|
726
|
+
const end = at + lit.text.length;
|
|
727
|
+
return { start: at, endMin: end, endMax: end };
|
|
728
|
+
}
|
|
729
|
+
if (lit.mode === 'label')
|
|
730
|
+
return matchLabel(source, at, lit.text);
|
|
731
|
+
return matchTitle(source, at, lit.text);
|
|
732
|
+
}
|
|
733
|
+
function matchLabel(source, at, literal) {
|
|
734
|
+
const parts = labelParts(literal);
|
|
735
|
+
if (!parts)
|
|
736
|
+
return null;
|
|
737
|
+
if (!source.startsWith(parts.rigid, at))
|
|
738
|
+
return null;
|
|
739
|
+
const labelAt = at + parts.rigid.length - labelLength(parts.rigid);
|
|
740
|
+
if (!markerBoundaryOk(source, labelAt, literal, 'label'))
|
|
741
|
+
return null;
|
|
742
|
+
let cursor = at + parts.rigid.length;
|
|
743
|
+
while (cursor < source.length && isWs(source[cursor]))
|
|
744
|
+
cursor++;
|
|
745
|
+
if (source[cursor] !== ':')
|
|
746
|
+
return null;
|
|
747
|
+
const after = cursor + 1;
|
|
748
|
+
let endMax = after;
|
|
749
|
+
while (endMax < source.length && isWs(source[endMax]))
|
|
750
|
+
endMax++;
|
|
751
|
+
return { start: at, endMin: after, endMax };
|
|
752
|
+
}
|
|
753
|
+
function labelLength(rigid) {
|
|
754
|
+
const lower = rigid.toLowerCase();
|
|
755
|
+
for (const label of LABELS) {
|
|
756
|
+
if (lower.endsWith(label))
|
|
757
|
+
return label.length;
|
|
758
|
+
}
|
|
759
|
+
return 0;
|
|
760
|
+
}
|
|
761
|
+
function matchTitle(source, at, literal) {
|
|
762
|
+
const parts = titleParts(literal);
|
|
763
|
+
if (!parts)
|
|
764
|
+
return null;
|
|
765
|
+
if (!source.startsWith(parts.rigid, at))
|
|
766
|
+
return null;
|
|
767
|
+
const titleAt = at + parts.rigid.length - titleLength(parts.rigid);
|
|
768
|
+
if (!markerBoundaryOk(source, titleAt, literal, 'title'))
|
|
769
|
+
return null;
|
|
770
|
+
const after = at + parts.rigid.length;
|
|
771
|
+
let endMax = after;
|
|
772
|
+
while (endMax < source.length && isWs(source[endMax]))
|
|
773
|
+
endMax++;
|
|
774
|
+
return { start: at, endMin: after, endMax };
|
|
775
|
+
}
|
|
776
|
+
function titleLength(rigid) {
|
|
777
|
+
for (const title of TITLES) {
|
|
778
|
+
if (rigid.endsWith(title))
|
|
779
|
+
return title.length;
|
|
780
|
+
}
|
|
781
|
+
return 0;
|
|
782
|
+
}
|
|
783
|
+
function markerBoundaryOk(source, at, literal, kind) {
|
|
784
|
+
if (gluedAfterToken(literal, kind))
|
|
785
|
+
return true;
|
|
786
|
+
return boundaryOk(source, at);
|
|
787
|
+
}
|
|
788
|
+
function gluedAfterToken(literal, kind) {
|
|
789
|
+
if (kind === 'label') {
|
|
790
|
+
const parts = labelParts(literal);
|
|
791
|
+
return parts !== null && labelLength(parts.rigid) === parts.rigid.length;
|
|
792
|
+
}
|
|
793
|
+
const parts = titleParts(literal);
|
|
794
|
+
return parts !== null && titleLength(parts.rigid) === parts.rigid.length;
|
|
795
|
+
}
|
|
796
|
+
function matchLiteralEnding(source, endAt, lit, budget) {
|
|
797
|
+
budget.tick(1);
|
|
798
|
+
if (lit.mode !== 'exact') {
|
|
799
|
+
for (let start = 0; start <= endAt; start++) {
|
|
800
|
+
budget.tick(1);
|
|
801
|
+
const match = matchLiteral(source, start, lit, budget);
|
|
802
|
+
if (!match)
|
|
803
|
+
continue;
|
|
804
|
+
if (match.endMin <= endAt && endAt <= match.endMax)
|
|
805
|
+
return { ...match, start };
|
|
806
|
+
}
|
|
807
|
+
return null;
|
|
808
|
+
}
|
|
809
|
+
if (lit.text.length === 0) {
|
|
810
|
+
if (endAt > source.length)
|
|
811
|
+
return null;
|
|
812
|
+
return { start: endAt, endMin: endAt, endMax: endAt };
|
|
813
|
+
}
|
|
814
|
+
const start = endAt - lit.text.length;
|
|
815
|
+
if (start < 0 || source.slice(start, endAt) !== lit.text)
|
|
816
|
+
return null;
|
|
817
|
+
return { start, endMin: endAt, endMax: endAt };
|
|
818
|
+
}
|
|
819
|
+
function earliestLiteral(source, lit, from, budget) {
|
|
820
|
+
if (lit.mode === 'exact' && lit.text.length === 0) {
|
|
821
|
+
if (from > source.length)
|
|
822
|
+
return null;
|
|
823
|
+
return { start: from, endMin: from, endMax: from };
|
|
824
|
+
}
|
|
825
|
+
if (lit.mode === 'exact') {
|
|
826
|
+
const at = source.indexOf(lit.text, from);
|
|
827
|
+
budget.tick(Math.max(1, (at < 0 ? source.length : at) - from + lit.text.length));
|
|
828
|
+
if (at < 0)
|
|
829
|
+
return null;
|
|
830
|
+
return { start: at, endMin: at + lit.text.length, endMax: at + lit.text.length };
|
|
831
|
+
}
|
|
832
|
+
const floor = Math.max(0, from);
|
|
833
|
+
for (let at = floor; at <= source.length; at++) {
|
|
834
|
+
budget.tick(1);
|
|
835
|
+
const match = matchLiteral(source, at, lit, budget);
|
|
836
|
+
if (match && match.start >= floor)
|
|
837
|
+
return match;
|
|
838
|
+
}
|
|
839
|
+
return null;
|
|
840
|
+
}
|
|
841
|
+
function latestLiteral(source, lit, endLimit, budget) {
|
|
842
|
+
if (lit.mode === 'exact' && lit.text.length === 0) {
|
|
843
|
+
const start = Math.min(endLimit, source.length);
|
|
844
|
+
return { start, endMin: start, endMax: start };
|
|
845
|
+
}
|
|
846
|
+
if (lit.mode === 'exact') {
|
|
847
|
+
const maxStart = endLimit - lit.text.length;
|
|
848
|
+
budget.tick(lit.text.length + 1);
|
|
849
|
+
if (maxStart < 0)
|
|
850
|
+
return null;
|
|
851
|
+
const at = source.lastIndexOf(lit.text, maxStart);
|
|
852
|
+
if (at < 0)
|
|
853
|
+
return null;
|
|
854
|
+
return { start: at, endMin: at + lit.text.length, endMax: at + lit.text.length };
|
|
855
|
+
}
|
|
856
|
+
for (let at = Math.min(endLimit, source.length); at >= 0; at--) {
|
|
857
|
+
budget.tick(1);
|
|
858
|
+
const match = matchLiteral(source, at, lit, budget);
|
|
859
|
+
if (!match || match.endMin > endLimit)
|
|
860
|
+
continue;
|
|
861
|
+
return match;
|
|
862
|
+
}
|
|
863
|
+
return null;
|
|
864
|
+
}
|
|
865
|
+
//# sourceMappingURL=belt.js.map
|