@hyperfixi/testing-framework 4.4.0 → 4.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +197 -5
- package/package.json +7 -7
- package/src/multilingual/README.md +9 -6
- package/src/multilingual/command-shapes.ts +7 -91
- package/src/multilingual/direct-path-shapes.ts +60 -26
- package/src/multilingual/english-leaks.test.ts +118 -6
- package/src/multilingual/english-leaks.ts +208 -43
- package/src/multilingual/hand-written-input.tasks.json +162 -0
- package/src/multilingual/hand-written-input.test.ts +62 -0
- package/src/multilingual/hand-written-input.ts +201 -0
- package/src/multilingual/value-matrix.ts +1 -1
- package/src/vocab/batch3-roundtrip.test.ts +2 -1
|
@@ -34,6 +34,11 @@ function keys(source: string, render: string, language: string): string[] {
|
|
|
34
34
|
return scanner.scan(render, info, language).map(findingKey).sort();
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
+
/** The same, of one kind (the spaced markers of tr and ja stand beside their cases). */
|
|
38
|
+
function keysOf(kind: string, source: string, render: string, language: string): string[] {
|
|
39
|
+
return keys(source, render, language).filter(k => k.startsWith(`${kind}:`));
|
|
40
|
+
}
|
|
41
|
+
|
|
37
42
|
function findings(source: string, render: string, language: string): LeakFinding[] {
|
|
38
43
|
const info = scanner.sourceInfo(source, 'program');
|
|
39
44
|
if (!info) throw new Error(`the engine rejects ${source}`);
|
|
@@ -102,7 +107,7 @@ describe('english-leaks scanner', () => {
|
|
|
102
107
|
expect(
|
|
103
108
|
keys(
|
|
104
109
|
'on click js(me) return new Date() end then put it into #a',
|
|
105
|
-
'al clic js (me) return new Date() fin entonces
|
|
110
|
+
'al clic js (me) return new Date() fin entonces ponerlo en #a',
|
|
106
111
|
'es'
|
|
107
112
|
)
|
|
108
113
|
).toEqual([]);
|
|
@@ -128,7 +133,7 @@ describe('english-leaks scanner', () => {
|
|
|
128
133
|
expect(
|
|
129
134
|
keys(
|
|
130
135
|
'on click put me into #a then js(me) return 1 end',
|
|
131
|
-
'al clic
|
|
136
|
+
'al clic ponerme en #a entonces js (me) return 1 fin',
|
|
132
137
|
'es'
|
|
133
138
|
)
|
|
134
139
|
).toEqual([]);
|
|
@@ -195,10 +200,117 @@ describe('english-leaks scanner', () => {
|
|
|
195
200
|
|
|
196
201
|
it('counts a nominative `me` beside a marker, in the languages where that is wrong', () => {
|
|
197
202
|
expect(keys('on click add .b to me', 'al clic agregar .b a yo', 'es')).toEqual(['case:me']);
|
|
198
|
-
expect(
|
|
203
|
+
expect(keysOf('case', 'on click add .b to me', 'tıklama da .b i ben e ekle', 'tr')).toEqual([
|
|
204
|
+
'case:me',
|
|
205
|
+
]);
|
|
199
206
|
// fr writes the stressed form; ja's particle attaches to any noun.
|
|
200
207
|
expect(keys('on click add .b to me', 'sur clic ajouter .b à moi', 'fr')).toEqual([]);
|
|
201
|
-
expect(
|
|
208
|
+
expect(keysOf('case', 'on click add .b to me', 'クリック で 自分 に .b を 追加', 'ja')).toEqual(
|
|
209
|
+
[]
|
|
210
|
+
);
|
|
211
|
+
// The oblique forms are right.
|
|
212
|
+
expect(keys('on click add .b to me', 'al clic agregar .b a mí', 'es')).toEqual([]);
|
|
213
|
+
expect(keys('on click add .b to me', 'при клик добавить .b ко мне', 'ru')).toEqual([]);
|
|
214
|
+
// it writes `me` after a marker: the profile's own word, not English's.
|
|
215
|
+
expect(keys('on click put "x" into me', 'su clic mettere "x" in me', 'it')).toEqual([]);
|
|
216
|
+
expect(keys('on click put "x" into me', 'su clic mettere "x" in io', 'it')).toEqual([
|
|
217
|
+
'case:me',
|
|
218
|
+
]);
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
it('counts a nominative `it` beside a marker, where the language has another form', () => {
|
|
222
|
+
const source = 'on click add .b to it';
|
|
223
|
+
expect(keys(source, 'при клик добавить .b к это', 'ru')).toEqual(['case:it']);
|
|
224
|
+
expect(keys(source, 'wenn klick hinzufügen .b zu es', 'de')).toEqual(['case:it']);
|
|
225
|
+
expect(keysOf('case', source, 'tıklama i üzerinde o e .b i ekle', 'tr')).toEqual(['case:it']);
|
|
226
|
+
expect(keys(source, 'при клик добавить .b к этому', 'ru')).toEqual([]);
|
|
227
|
+
// es `ello` and it `esso` are right after a marker.
|
|
228
|
+
expect(keys(source, 'al clic agregar .b a ello', 'es')).toEqual([]);
|
|
229
|
+
expect(keys(source, 'su clic aggiungere .b a esso', 'it')).toEqual([]);
|
|
230
|
+
// pt: only `em` and `de` contract with `ele` (nele, dele); `a ele` is right.
|
|
231
|
+
expect(keys('on click put "x" into it', 'ao clique colocar "x" em ele', 'pt')).toEqual([
|
|
232
|
+
'case:it',
|
|
233
|
+
]);
|
|
234
|
+
expect(keys(source, 'ao clique adicionar .b a ele', 'pt')).toEqual([]);
|
|
235
|
+
// The ru/uk/pl demonstrative's accusative is its nominative: right after a
|
|
236
|
+
// marker that takes the accusative (a direction), wrong after one that never does.
|
|
237
|
+
const into = 'on click put "x" into it';
|
|
238
|
+
expect(keys(into, 'при клик положить "x" в это', 'ru')).toEqual([]);
|
|
239
|
+
expect(keys(into, 'gdy kliknięcie umieść "x" w to', 'pl')).toEqual([]);
|
|
240
|
+
expect(keys(source, 'gdy kliknięcie dodaj .b do to', 'pl')).toEqual(['case:it']);
|
|
241
|
+
expect(keys(source, 'при клік додати .b до це', 'uk')).toEqual(['case:it']);
|
|
242
|
+
// tl `bago ito` (before this) is right; `sa ito` is `dito`.
|
|
243
|
+
expect(keys(source, 'kapag click idagdag .b sa ito', 'tl')).toEqual(['case:it']);
|
|
244
|
+
expect(keys('on click wait 1s', 'kapag click maghintay bago ito', 'tl')).toEqual([]);
|
|
245
|
+
});
|
|
246
|
+
|
|
247
|
+
it('does not count a pronoun that opens a parenthesized expression', () => {
|
|
248
|
+
const source = 'on click set $b to (it * 2)';
|
|
249
|
+
expect(keys(source, 'wenn klick setze $b auf (es * 2)', 'de')).toEqual([]);
|
|
250
|
+
expect(keys('on click set $b to it', 'wenn klick setze $b auf es', 'de')).toEqual(['case:it']);
|
|
251
|
+
});
|
|
252
|
+
|
|
253
|
+
it("counts a nominative pronoun as a verb's unmarked object, where that is wrong", () => {
|
|
254
|
+
expect(keys('on click measure me', 'wenn klick messen ich', 'de')).toEqual(['case:me-object']);
|
|
255
|
+
expect(keys('on click show me', 'al clic mostrar yo', 'es')).toEqual(['case:me-object']);
|
|
256
|
+
expect(keys('on click hide it', 'al clic ocultar ello', 'es')).toEqual(['case:it-object']);
|
|
257
|
+
expect(keys('on click hide it', 'quand clic cacher il', 'fr')).toEqual(['case:it-object']);
|
|
258
|
+
expect(keys('on click put me into #o', 'при клик положить я в #o', 'ru')).toEqual([
|
|
259
|
+
'case:me-object',
|
|
260
|
+
]);
|
|
261
|
+
// A verb its patterns write (de `verstecke`; the keyword is `verbergen`).
|
|
262
|
+
expect(keys('on click hide me', 'wenn klick verstecke ich', 'de')).toEqual(['case:me-object']);
|
|
263
|
+
// The right object forms, and the objects that are right in the nominative.
|
|
264
|
+
expect(keys('on click measure me', 'wenn klick messen mich', 'de')).toEqual([]);
|
|
265
|
+
expect(keys('on click hide it', 'wenn klick verstecke es', 'de')).toEqual([]);
|
|
266
|
+
expect(keys('on click hide it', 'при клик скрыть это', 'ru')).toEqual([]);
|
|
267
|
+
// fr writes the clitic before the infinitive (`m'afficher`, `me cacher`), and
|
|
268
|
+
// `me` there is its own word.
|
|
269
|
+
expect(keys('on click show me', 'quand clic afficher moi', 'fr')).toEqual(['case:me-object']);
|
|
270
|
+
expect(keys('on click show me', "quand clic m'afficher", 'fr')).toEqual([]);
|
|
271
|
+
expect(keys('on click hide me', 'quand clic me cacher', 'fr')).toEqual([]);
|
|
272
|
+
expect(keys('on click hide it', 'quand clic le cacher', 'fr')).toEqual([]);
|
|
273
|
+
// A condition is not an object: `if` is no verb.
|
|
274
|
+
expect(
|
|
275
|
+
keys('on click if it is empty hide me end', 'al clic si ello es vacío ocultar yo fin', 'es')
|
|
276
|
+
).toEqual(['case:me-object']);
|
|
277
|
+
// A word before the pronoun that is no verb at all.
|
|
278
|
+
expect(keys('on click put it into #o', 'al clic ello en #o', 'es')).toEqual([]);
|
|
279
|
+
});
|
|
280
|
+
|
|
281
|
+
it('counts a marker its language attaches, written apart (N5)', () => {
|
|
282
|
+
const source = 'on click put "x" into #out';
|
|
283
|
+
expect(keys(source, '클릭 할 때 "x" 을 #out 에 넣다', 'ko')).toEqual([
|
|
284
|
+
'spacing:에',
|
|
285
|
+
'spacing:을',
|
|
286
|
+
]);
|
|
287
|
+
expect(keys(source, 'クリック で "x" を #out に 置く', 'ja')).toEqual([
|
|
288
|
+
'spacing:で',
|
|
289
|
+
'spacing:に',
|
|
290
|
+
'spacing:を',
|
|
291
|
+
]);
|
|
292
|
+
expect(keys(source, 'ক্লিক তে "x" কে #out এ রাখুন', 'bn')).toEqual([
|
|
293
|
+
'spacing:এ',
|
|
294
|
+
'spacing:কে',
|
|
295
|
+
'spacing:তে',
|
|
296
|
+
]);
|
|
297
|
+
expect(keys(source, 'tıklama i üzerinde "x" i #out e koy', 'tr')).toEqual([
|
|
298
|
+
'spacing:e',
|
|
299
|
+
'spacing:i',
|
|
300
|
+
'spacing:i',
|
|
301
|
+
]);
|
|
302
|
+
// Attached, nothing; a postposition that is its own word is spaced.
|
|
303
|
+
expect(keys(source, '클릭 할 때 "x"를 #out에 넣다', 'ko')).toEqual([]);
|
|
304
|
+
expect(keys(source, 'クリックで "x"を #outに 置く', 'ja')).toEqual([]);
|
|
305
|
+
expect(keys(source, 'tıklamada "x"\'i #out\'a koy', 'tr')).toEqual([]);
|
|
306
|
+
expect(
|
|
307
|
+
keys('on click hide <p/> in #list', "tıklamada #list içindeki <p/>'yi gizle", 'tr')
|
|
308
|
+
).toEqual([]);
|
|
309
|
+
// A marker that opens a line has nothing to attach to.
|
|
310
|
+
expect(keys(source, 'クリックで "x"を\nに 置く', 'ja')).toEqual([]);
|
|
311
|
+
// Not in a language whose markers are words (es `a`), nor in qu's yet.
|
|
312
|
+
expect(keys(source, 'al clic poner "x" en #out', 'es')).toEqual([]);
|
|
313
|
+
expect(keys(source, 'maykama ñitiy "x" ta #out man churay', 'qu')).toEqual([]);
|
|
202
314
|
});
|
|
203
315
|
});
|
|
204
316
|
|
|
@@ -328,8 +440,8 @@ describe('M2 exit targets', () => {
|
|
|
328
440
|
|
|
329
441
|
it('allows no event and no pronoun case, and skips a lane with no render', () => {
|
|
330
442
|
const rates = leakRatesByLanguage(rows({ ru: ['event'], pl: ['case'], he: 'none' }), 'shapes');
|
|
331
|
-
expect(rates.get('ru')!.met).toEqual({ word: true, event: false, case: true });
|
|
332
|
-
expect(rates.get('pl')!.met).toEqual({ word: true, event: true, case: false });
|
|
443
|
+
expect(rates.get('ru')!.met).toEqual({ word: true, event: false, case: true, spacing: true });
|
|
444
|
+
expect(rates.get('pl')!.met).toEqual({ word: true, event: true, case: false, spacing: true });
|
|
333
445
|
expect(rates.has('he')).toBe(false);
|
|
334
446
|
});
|
|
335
447
|
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* passes all of them. Measured 2026-10-08, 18% of corpus renders and 24% of
|
|
8
8
|
* the command-shape scripts' renders hold at least one English word.
|
|
9
9
|
*
|
|
10
|
-
* This gate counts
|
|
10
|
+
* This gate counts four kinds of finding in a render of English `source`
|
|
11
11
|
* into a language:
|
|
12
12
|
* - `word` an English word the engine reads as grammar in the source (or a
|
|
13
13
|
* reference: `me`, `my`, `I`, `it`, …), written as English in the
|
|
@@ -15,8 +15,12 @@
|
|
|
15
15
|
* - `event` an event the source handles or sends, written in English where
|
|
16
16
|
* the language's lexicon has a word for it and the pair is not on
|
|
17
17
|
* the renderer's denylist (OPEN_ITEMS N2);
|
|
18
|
-
* - `case` the language's nominative `me` next to a role marker,
|
|
19
|
-
*
|
|
18
|
+
* - `case` the language's nominative `me` or `it` next to a role marker,
|
|
19
|
+
* or as a command's unmarked object, in the languages where that
|
|
20
|
+
* form is wrong (N3, see CASE_RULES);
|
|
21
|
+
* - `spacing` a marker the language attaches to its noun, written as its
|
|
22
|
+
* own word after one (ko `#out 에`, tr `#out e`; N5, see
|
|
23
|
+
* ATTACHED_MARKERS).
|
|
20
24
|
*
|
|
21
25
|
* ## What counts as English
|
|
22
26
|
* Derived, never a hand list. A source's grammar words are the tokens the
|
|
@@ -66,12 +70,17 @@ export type LeakHalf = keyof typeof LEAK_BASELINE_PATHS;
|
|
|
66
70
|
// Findings
|
|
67
71
|
// ---------------------------------------------------------------------------
|
|
68
72
|
|
|
69
|
-
export type LeakKind = 'word' | 'event' | 'case';
|
|
73
|
+
export type LeakKind = 'word' | 'event' | 'case' | 'spacing';
|
|
70
74
|
export type LeakContext = 'plain' | 'clause' | 'bracket' | 'call' | 'property';
|
|
75
|
+
export const KINDS: readonly LeakKind[] = ['word', 'event', 'case', 'spacing'];
|
|
71
76
|
|
|
72
77
|
export interface LeakFinding {
|
|
73
78
|
kind: LeakKind;
|
|
74
|
-
/**
|
|
79
|
+
/**
|
|
80
|
+
* The English word; for `event` the event name; for `case` the pronoun (`me`,
|
|
81
|
+
* `it`), with `-object` when it is a verb's unmarked object (`me-object`); for
|
|
82
|
+
* `spacing` the marker.
|
|
83
|
+
*/
|
|
75
84
|
word: string;
|
|
76
85
|
context: LeakContext;
|
|
77
86
|
/** Does the language have its own word for it? */
|
|
@@ -106,33 +115,125 @@ export const KEPT_ENGLISH: {
|
|
|
106
115
|
],
|
|
107
116
|
};
|
|
108
117
|
|
|
109
|
-
/** A finding as the baseline writes it: the word, or `event:<name>` / `case:<pronoun>`. */
|
|
118
|
+
/** A finding as the baseline writes it: the word, or `event:<name>` / `case:<pronoun>` / `spacing:<marker>`. */
|
|
110
119
|
export const findingKey = (f: Pick<LeakFinding, 'kind' | 'word'>): string =>
|
|
111
120
|
f.kind === 'word' ? f.word : `${f.kind}:${f.word}`;
|
|
112
121
|
|
|
122
|
+
/** Where a language's nominative `me` / `it` is the wrong form ({@link CASE_RULES}). */
|
|
123
|
+
export interface CaseRule {
|
|
124
|
+
/** `me` beside a role marker. */
|
|
125
|
+
me?: true;
|
|
126
|
+
/**
|
|
127
|
+
* `it` beside a role marker: any marker, or only these (pt contracts only
|
|
128
|
+
* `em`, `de`; the ru/uk/pl demonstrative's accusative is its nominative, so
|
|
129
|
+
* it is wrong only after a marker that never takes the accusative).
|
|
130
|
+
*/
|
|
131
|
+
it?: true | readonly string[];
|
|
132
|
+
/** `me` / `it` as a command's unmarked object, right after its verb. */
|
|
133
|
+
meObject?: true;
|
|
134
|
+
itObject?: true;
|
|
135
|
+
}
|
|
136
|
+
|
|
113
137
|
/**
|
|
114
|
-
*
|
|
115
|
-
*
|
|
116
|
-
*
|
|
117
|
-
*
|
|
118
|
-
*
|
|
119
|
-
*
|
|
120
|
-
*
|
|
138
|
+
* The languages whose nominative `me` or `it` is the wrong form where a render
|
|
139
|
+
* writes it (OPEN_ITEMS N3). A finding is `case:me` / `case:it` beside a role
|
|
140
|
+
* marker, `case:me-object` / `case:it-object` right after a command's verb with
|
|
141
|
+
* no marker. Wrong → right:
|
|
142
|
+
*
|
|
143
|
+
* ar إلى أنا → إليّ; إلى هو → إليه; ضع أنا → ضعني; ضع هو → ضعه
|
|
144
|
+
* bn আমি তে → আমাতে
|
|
145
|
+
* de zu ich → zu mir; zu es → dazu, zu ihm; messen ich → messen mich
|
|
146
|
+
* es a yo → a mí; mostrar yo → mostrarme; poner ello → ponerlo
|
|
147
|
+
* fr dans il → dans lui, y; mettre il → le mettre; cacher moi → me cacher
|
|
148
|
+
* he אל אני → אליי
|
|
149
|
+
* hi मैं में → मुझमें; यह में → इसमें
|
|
150
|
+
* it a io → a me; mostrare io → mostrarmi; mettere esso → metterlo
|
|
151
|
+
* pl do ja → do mnie; do to → do tego, z to → z tego; pokaż ja → pokaż mnie
|
|
152
|
+
* pt a eu → a mim; em ele → nele, de ele → dele; mostrar eu → mostrar-me;
|
|
153
|
+
* colocar ele → colocá-lo
|
|
154
|
+
* ru к я → ко мне; к это → к этому, из это → из этого; показать я → показать меня
|
|
155
|
+
* tl sa ako → sa akin; sa ito → dito, mula sa ito → mula rito
|
|
156
|
+
* tr ben e → bana; o e → ona
|
|
157
|
+
* uk до я → до мене; до це → до цього, з це → з цього; показати я → показати мене
|
|
158
|
+
*
|
|
159
|
+
* Right as written: fr `à moi` (after a preposition); es `ello`, it `esso` and pt `ele` after any
|
|
160
|
+
* other marker (`a ello`, `a esso`, `a ele`); ru `в это`, uk `в це`, pl `w to`
|
|
161
|
+
* and `на`/`na` with them, the accusative of a direction (a location wants
|
|
162
|
+
* `в этом`, which the text cannot tell apart: not counted); tl `bago ito`,
|
|
163
|
+
* `matapos ito`; the objects de `es`, ru `это`, uk `це`, pl `to`. A pronoun
|
|
164
|
+
* that opens a parenthesized expression (`auf (es * 2)`) is not the marker's.
|
|
165
|
+
* Not counted here: he `ב זה` and bn `এটি কে`, whose fault is
|
|
166
|
+
* the spacing (N5); `you`, which waits on an owner decision (de `dir` is a
|
|
167
|
+
* plausible variable name). ja, zh, id, ms, vi, th, ko, sw and qu do not
|
|
168
|
+
* inflect these pronouns for case.
|
|
121
169
|
*/
|
|
122
|
-
export const
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
'
|
|
132
|
-
'
|
|
133
|
-
'
|
|
134
|
-
'
|
|
135
|
-
|
|
170
|
+
export const CASE_RULES: Readonly<Record<string, CaseRule>> = {
|
|
171
|
+
ar: { me: true, it: true, meObject: true, itObject: true },
|
|
172
|
+
bn: { me: true },
|
|
173
|
+
de: { me: true, it: true, meObject: true },
|
|
174
|
+
es: { me: true, meObject: true, itObject: true },
|
|
175
|
+
fr: { it: true, meObject: true, itObject: true },
|
|
176
|
+
he: { me: true },
|
|
177
|
+
hi: { me: true, it: true },
|
|
178
|
+
it: { me: true, meObject: true, itObject: true },
|
|
179
|
+
pl: { me: true, it: ['do', 'z', 'ze', 'od'], meObject: true },
|
|
180
|
+
pt: { me: true, it: ['em', 'de'], meObject: true, itObject: true },
|
|
181
|
+
ru: { me: true, it: ['к', 'ко', 'из', 'от', 'с', 'со'], meObject: true },
|
|
182
|
+
tl: { me: true, it: ['sa', 'mula_sa'] },
|
|
183
|
+
tr: { me: true, it: true },
|
|
184
|
+
uk: { me: true, it: ['до', 'з', 'із', 'від'], meObject: true },
|
|
185
|
+
};
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* The markers each language attaches to the word before them, where a render
|
|
189
|
+
* that writes them apart reads as broken (OPEN_ITEMS N5, FUSED_FORMS.md §3.3):
|
|
190
|
+
* a `spacing:<marker>` finding is one of these written as its own word after
|
|
191
|
+
* another word (`#out 에` → `#out에`, `.active i` → `.active'i`). The case
|
|
192
|
+
* particles and suffixes only, which attach in the language's own writing; a
|
|
193
|
+
* postposition that is a word of its own (tr `içinde`, `arasında`, ko `사이`,
|
|
194
|
+
* `동안`, bn `থেকে`, `মধ্যে`) is spaced there and is not listed. tr's suffixes
|
|
195
|
+
* are every harmony form its reader takes. qu attaches its suffixes too, but
|
|
196
|
+
* its reader takes an attached one into the word before (`#outman` is one
|
|
197
|
+
* selector), so it waits on the reader (FUSED_FORMS.md Q2, Q3).
|
|
198
|
+
*/
|
|
199
|
+
export const ATTACHED_MARKERS: Readonly<Record<string, ReadonlySet<string>>> = {
|
|
200
|
+
bn: new Set(['কে', 'তে', 'এ']),
|
|
201
|
+
ja: new Set(['を', 'に', 'へ', 'で', 'の', 'から', 'として', 'の間で']),
|
|
202
|
+
ko: new Set(['을', '를', '에', '으로', '로', '의', '에서', '부터', '에게']),
|
|
203
|
+
tr: new Set(
|
|
204
|
+
[
|
|
205
|
+
'i ı u ü yi yı yu yü ni nı nu nü',
|
|
206
|
+
'e a ye ya ne na',
|
|
207
|
+
'de da te ta',
|
|
208
|
+
'den dan ten tan',
|
|
209
|
+
'in ın un ün nin nın nun nün',
|
|
210
|
+
'le la yle yla',
|
|
211
|
+
].flatMap(forms => forms.split(' '))
|
|
212
|
+
),
|
|
213
|
+
};
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* Commands whose next word is not their object: a handler or block head, or a
|
|
217
|
+
* condition (`if it is empty`, `while it …`).
|
|
218
|
+
*/
|
|
219
|
+
const NOT_OBJECT_HEADS: ReadonlySet<string> = new Set([
|
|
220
|
+
'on',
|
|
221
|
+
'if',
|
|
222
|
+
'unless',
|
|
223
|
+
'else',
|
|
224
|
+
'repeat',
|
|
225
|
+
'for',
|
|
226
|
+
'while',
|
|
227
|
+
'when',
|
|
228
|
+
'init',
|
|
229
|
+
'behavior',
|
|
230
|
+
'install',
|
|
231
|
+
'live',
|
|
232
|
+
'js',
|
|
233
|
+
'compound',
|
|
234
|
+
'eventsource',
|
|
235
|
+
'socket',
|
|
236
|
+
'worker',
|
|
136
237
|
]);
|
|
137
238
|
|
|
138
239
|
// ---------------------------------------------------------------------------
|
|
@@ -210,8 +311,13 @@ interface ProfileLike {
|
|
|
210
311
|
keywords?: Record<string, string>;
|
|
211
312
|
};
|
|
212
313
|
eventHandler?: Record<string, unknown>;
|
|
314
|
+
objectClitics?: { forms?: Record<string, string>; attach?: string };
|
|
213
315
|
}
|
|
214
316
|
type LexiconLike = Record<string, Record<string, KeywordTranslationLike> | undefined>;
|
|
317
|
+
interface PatternLike {
|
|
318
|
+
command: string;
|
|
319
|
+
template?: { tokens?: ReadonlyArray<{ type: string; value?: string; alternatives?: string[] }> };
|
|
320
|
+
}
|
|
215
321
|
|
|
216
322
|
const LEXICON_CATEGORIES = ['values', 'expressions', 'logical', 'temporal', 'attributes'] as const;
|
|
217
323
|
|
|
@@ -226,16 +332,20 @@ interface LanguageWords {
|
|
|
226
332
|
translated: Set<string>;
|
|
227
333
|
/** Events its lexicon names, minus the renderer's denylist for it. */
|
|
228
334
|
events: Set<string>;
|
|
229
|
-
/** Its `me`
|
|
335
|
+
/** Its `me` and `it` words, its role markers by side, and its command verbs (as word lists). */
|
|
230
336
|
me: string | undefined;
|
|
337
|
+
it: string | undefined;
|
|
231
338
|
markersBefore: Set<string>;
|
|
232
339
|
markersAfter: Set<string>;
|
|
340
|
+
verbs: string[][];
|
|
233
341
|
}
|
|
234
342
|
|
|
235
343
|
function languageWords(
|
|
236
344
|
profile: ProfileLike | undefined,
|
|
237
345
|
lexicon: LexiconLike | undefined,
|
|
238
|
-
denied: ReadonlySet<string> | undefined
|
|
346
|
+
denied: ReadonlySet<string> | undefined,
|
|
347
|
+
commands: readonly string[],
|
|
348
|
+
patterns: readonly PatternLike[]
|
|
239
349
|
): LanguageWords {
|
|
240
350
|
const native = new Set<string>();
|
|
241
351
|
const translated = new Set<string>();
|
|
@@ -266,6 +376,10 @@ function languageWords(
|
|
|
266
376
|
// Native form → the English reference it stands for (es `mi` → `me`).
|
|
267
377
|
for (const n of Object.keys(profile?.possessive?.keywords ?? {}))
|
|
268
378
|
for (const w of wordsOf(n)) native.add(w);
|
|
379
|
+
// A clitic written as its own word (fr `me cacher`) is the language's; one
|
|
380
|
+
// attached to its verb (es `mostrarme`) never stands alone.
|
|
381
|
+
if (profile?.objectClitics?.attach === 'before')
|
|
382
|
+
for (const clitic of Object.values(profile.objectClitics.forms ?? {})) native.add(clitic);
|
|
269
383
|
for (const w of wordsOf(profile?.possessive?.marker)) native.add(w);
|
|
270
384
|
const handler = JSON.stringify(profile?.eventHandler ?? {});
|
|
271
385
|
for (const m of handler.matchAll(
|
|
@@ -285,13 +399,29 @@ function languageWords(
|
|
|
285
399
|
const markersAfter = new Set<string>();
|
|
286
400
|
for (const t of Object.values(profile?.roleMarkers ?? {}))
|
|
287
401
|
for (const w of spellings(t)) (t.position === 'after' ? markersAfter : markersBefore).add(w);
|
|
402
|
+
// A command's verb: its keyword, and the word its own patterns open with
|
|
403
|
+
// (de's `hide` pattern writes `verstecke`, its keyword is `verbergen`).
|
|
404
|
+
const verbForms: string[] = [];
|
|
405
|
+
for (const command of commands) {
|
|
406
|
+
const t = profile?.keywords?.[command];
|
|
407
|
+
if (t && !NOT_OBJECT_HEADS.has(command))
|
|
408
|
+
verbForms.push(t.primary ?? '', ...(t.alternatives ?? []));
|
|
409
|
+
}
|
|
410
|
+
for (const pattern of patterns) {
|
|
411
|
+
const first = pattern.template?.tokens?.[0];
|
|
412
|
+
if (!NOT_OBJECT_HEADS.has(pattern.command) && first?.type === 'literal')
|
|
413
|
+
verbForms.push(first.value ?? '', ...(first.alternatives ?? []));
|
|
414
|
+
}
|
|
415
|
+
const verbs = [...new Set(verbForms.filter(Boolean))].map(wordsOf);
|
|
288
416
|
return {
|
|
289
417
|
native,
|
|
290
418
|
translated,
|
|
291
419
|
events,
|
|
292
420
|
me: profile?.references?.me,
|
|
421
|
+
it: profile?.references?.it,
|
|
293
422
|
markersBefore,
|
|
294
423
|
markersAfter,
|
|
424
|
+
verbs,
|
|
295
425
|
};
|
|
296
426
|
}
|
|
297
427
|
|
|
@@ -389,6 +519,8 @@ interface SemanticModule {
|
|
|
389
519
|
tryGetProfile(language: string): unknown;
|
|
390
520
|
getLexicon(language: string): unknown;
|
|
391
521
|
getEventLocalizationDenylist(): Record<string, ReadonlySet<string>>;
|
|
522
|
+
commandSchemas: Record<string, unknown>;
|
|
523
|
+
getPatternsForLanguage(language: string): PatternLike[];
|
|
392
524
|
}
|
|
393
525
|
|
|
394
526
|
/** Load the engine and semantic (ESM-only, so imported here) and return a scanner. */
|
|
@@ -397,6 +529,7 @@ export async function initLeakScanner(): Promise<LeakScanner> {
|
|
|
397
529
|
engine.register(...engine.everything);
|
|
398
530
|
const semantic = (await import('@lokascript/semantic')) as unknown as SemanticModule;
|
|
399
531
|
const denylist = semantic.getEventLocalizationDenylist();
|
|
532
|
+
const commands = Object.keys(semantic.commandSchemas);
|
|
400
533
|
const languages = new Map<string, LanguageWords>();
|
|
401
534
|
for (const language of FOREIGN_LANGUAGES) {
|
|
402
535
|
languages.set(
|
|
@@ -404,7 +537,9 @@ export async function initLeakScanner(): Promise<LeakScanner> {
|
|
|
404
537
|
languageWords(
|
|
405
538
|
semantic.tryGetProfile(language) as ProfileLike | undefined,
|
|
406
539
|
semantic.getLexicon(language) as LexiconLike | undefined,
|
|
407
|
-
denylist[language]
|
|
540
|
+
denylist[language],
|
|
541
|
+
commands,
|
|
542
|
+
semantic.getPatternsForLanguage(language)
|
|
408
543
|
)
|
|
409
544
|
);
|
|
410
545
|
}
|
|
@@ -532,14 +667,44 @@ export async function initLeakScanner(): Promise<LeakScanner> {
|
|
|
532
667
|
...(isKept(offset, context) ? { kept: true as const } : {}),
|
|
533
668
|
});
|
|
534
669
|
}
|
|
535
|
-
|
|
670
|
+
const rule = CASE_RULES[language];
|
|
671
|
+
if (rule) {
|
|
536
672
|
const tokens = words.map(w => w.word);
|
|
673
|
+
const afterVerb = (i: number): boolean =>
|
|
674
|
+
lang.verbs.some(
|
|
675
|
+
verb => i >= verb.length && verb.every((w, k) => tokens[i - verb.length + k] === w)
|
|
676
|
+
);
|
|
537
677
|
for (let i = 0; i < tokens.length; i++) {
|
|
538
|
-
|
|
678
|
+
const pronoun = tokens[i] === lang.me ? 'me' : tokens[i] === lang.it ? 'it' : undefined;
|
|
679
|
+
if (!pronoun || /[([]/.test(text[words[i]!.offset - 1] ?? '')) continue;
|
|
539
680
|
const before = tokens[i - 1];
|
|
540
681
|
const after = tokens[i + 1];
|
|
541
|
-
|
|
542
|
-
|
|
682
|
+
const marker =
|
|
683
|
+
before && lang.markersBefore.has(before)
|
|
684
|
+
? before
|
|
685
|
+
: after && lang.markersAfter.has(after)
|
|
686
|
+
? after
|
|
687
|
+
: undefined;
|
|
688
|
+
const wrong = rule[pronoun];
|
|
689
|
+
const wrongBeside =
|
|
690
|
+
marker !== undefined && (wrong === true || (!!wrong && wrong.includes(marker)));
|
|
691
|
+
const wrongObject = !!rule[`${pronoun}Object`] && afterVerb(i);
|
|
692
|
+
if (wrongBeside || wrongObject)
|
|
693
|
+
findings.push({
|
|
694
|
+
kind: 'case',
|
|
695
|
+
word: wrongObject ? `${pronoun}-object` : pronoun,
|
|
696
|
+
context: 'plain',
|
|
697
|
+
hasWord: true,
|
|
698
|
+
});
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
const attached = ATTACHED_MARKERS[language];
|
|
702
|
+
if (attached) {
|
|
703
|
+
for (const { word, offset } of words) {
|
|
704
|
+
if (!attached.has(word)) continue;
|
|
705
|
+
// After a word on its line, strings included (`"x" 을`): the render, not the code.
|
|
706
|
+
const before = /(\S)[^\S\n]+$/.exec(render.slice(0, offset));
|
|
707
|
+
if (before) findings.push({ kind: 'spacing', word, context: 'plain', hasWord: true });
|
|
543
708
|
}
|
|
544
709
|
}
|
|
545
710
|
return findings;
|
|
@@ -597,6 +762,7 @@ export function leakBaselineFrom(results: LeakResults, description: string): Lea
|
|
|
597
762
|
word: { renders: 0, findings: 0 },
|
|
598
763
|
event: { renders: 0, findings: 0 },
|
|
599
764
|
case: { renders: 0, findings: 0 },
|
|
765
|
+
spacing: { renders: 0, findings: 0 },
|
|
600
766
|
},
|
|
601
767
|
entries: {},
|
|
602
768
|
unrendered: {},
|
|
@@ -613,7 +779,7 @@ export function leakBaselineFrom(results: LeakResults, description: string): Lea
|
|
|
613
779
|
if (!findings.length) continue;
|
|
614
780
|
doc.leaky++;
|
|
615
781
|
doc.findings += findings.length;
|
|
616
|
-
for (const kind of
|
|
782
|
+
for (const kind of KINDS) {
|
|
617
783
|
const n = findings.filter(f => f.kind === kind).length;
|
|
618
784
|
doc.kinds[kind].findings += n;
|
|
619
785
|
if (n) doc.kinds[kind].renders++;
|
|
@@ -711,15 +877,14 @@ export function loadLeakBaseline(half: LeakHalf): LeakBaseline {
|
|
|
711
877
|
* reader reads one, and an average hid he (35% of corpus renders with an English
|
|
712
878
|
* word where the others sit near 18%). Words count only English no recorded
|
|
713
879
|
* decision keeps ({@link KEPT_ENGLISH}). Events are already net of the denylist,
|
|
714
|
-
* whose entries carry reasons.
|
|
880
|
+
* whose entries carry reasons. Spacing (N5) joined on 2026-10-11, with none
|
|
881
|
+
* allowed, as case did (FUSED_FORMS.md Q3).
|
|
715
882
|
*/
|
|
716
883
|
export const LEAK_TARGETS: Readonly<Record<LeakHalf, Readonly<Record<LeakKind, number>>>> = {
|
|
717
|
-
corpus: { word: 0.05, event: 0, case: 0 },
|
|
718
|
-
shapes: { word: 0.1, event: 0, case: 0 },
|
|
884
|
+
corpus: { word: 0.05, event: 0, case: 0, spacing: 0 },
|
|
885
|
+
shapes: { word: 0.1, event: 0, case: 0, spacing: 0 },
|
|
719
886
|
};
|
|
720
887
|
|
|
721
|
-
const KINDS: readonly LeakKind[] = ['word', 'event', 'case'];
|
|
722
|
-
|
|
723
888
|
export interface LanguageLeakRates {
|
|
724
889
|
renders: number;
|
|
725
890
|
/** The share of the language's renders with a finding of each kind. */
|
|
@@ -737,7 +902,7 @@ export function leakRatesByLanguage(
|
|
|
737
902
|
for (const byLanguage of results.values()) {
|
|
738
903
|
for (const [language, findings] of byLanguage) {
|
|
739
904
|
if (!findings) continue;
|
|
740
|
-
const row = counts.get(language) ?? { renders: 0, word: 0, event: 0, case: 0 };
|
|
905
|
+
const row = counts.get(language) ?? { renders: 0, word: 0, event: 0, case: 0, spacing: 0 };
|
|
741
906
|
row.renders++;
|
|
742
907
|
for (const kind of KINDS) if (findings.some(f => f.kind === kind && !f.kept)) row[kind]++;
|
|
743
908
|
counts.set(language, row);
|
|
@@ -745,8 +910,8 @@ export function leakRatesByLanguage(
|
|
|
745
910
|
}
|
|
746
911
|
const out = new Map<string, LanguageLeakRates>();
|
|
747
912
|
for (const [language, row] of counts) {
|
|
748
|
-
const rate = { word: 0, event: 0, case: 0 };
|
|
749
|
-
const met = { word: true, event: true, case: true };
|
|
913
|
+
const rate = { word: 0, event: 0, case: 0, spacing: 0 };
|
|
914
|
+
const met = { word: true, event: true, case: true, spacing: true };
|
|
750
915
|
for (const kind of KINDS) {
|
|
751
916
|
rate[kind] = row[kind] / row.renders;
|
|
752
917
|
met[kind] = rate[kind] <= LEAK_TARGETS[half][kind];
|