@startsimpli/integrations 0.2.19 → 0.2.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@startsimpli/integrations",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.20",
|
|
4
4
|
"description": "Provider-agnostic types, API clients, hooks, and UI primitives for email + calendar integrations across StartSimpli apps.",
|
|
5
5
|
"main": "./src/index.ts",
|
|
6
6
|
"types": "./src/index.ts",
|
|
@@ -0,0 +1,428 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* THE CLIENT CORPUS — ingestion, term extraction, memory, isolation
|
|
3
|
+
* (bd startsim-ppfl, RED).
|
|
4
|
+
*
|
|
5
|
+
* The bead asks for four things and this file is all four:
|
|
6
|
+
*
|
|
7
|
+
* - client documents ingest, and terminology is extracted with the right
|
|
8
|
+
* candidates surfaced FOR HUMAN CONFIRMATION — "an unconfirmed
|
|
9
|
+
* auto-extracted term must not silently become authoritative";
|
|
10
|
+
* - an approved translation pair becomes a memory entry and is hit on a
|
|
11
|
+
* matching later segment;
|
|
12
|
+
* - an accepted human edit becomes a new entry;
|
|
13
|
+
* - one client's corpus is unreachable from another client's tenant.
|
|
14
|
+
*
|
|
15
|
+
* THE ISOLATION CASES ARE THE POINT OF THE LANE. `startsim-hopl` sells "one
|
|
16
|
+
* client's material never appears against another's", and a glossary term is
|
|
17
|
+
* the sharpest possible violation of it: short, memorable, and it would surface
|
|
18
|
+
* INSIDE ANOTHER CLIENT'S TRANSLATED DOCUMENT, in their own language, looking
|
|
19
|
+
* entirely legitimate.
|
|
20
|
+
*
|
|
21
|
+
* Every case here runs against records deliberately mixed across two orgs —
|
|
22
|
+
* the same posture `corpus-scope.test.ts` already takes, which assumes the query
|
|
23
|
+
* layer DOES NOT hold the line and proves the output is safe anyway.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import { describe, expect, it } from 'vitest';
|
|
27
|
+
|
|
28
|
+
import { FIXTURE_ORG, FIXTURE_OTHER_ORG } from '../corpus/fixtures';
|
|
29
|
+
import {
|
|
30
|
+
TRANSLATION_TYPE_KEYS,
|
|
31
|
+
applyTermVerdict,
|
|
32
|
+
exactMemoryHit,
|
|
33
|
+
extractTermCandidates,
|
|
34
|
+
fuzzyMemoryMatches,
|
|
35
|
+
glossaryForSegment,
|
|
36
|
+
ingestForTranslation,
|
|
37
|
+
memoryUnitFromReviewerEdit,
|
|
38
|
+
normalizeForMemory,
|
|
39
|
+
readAllGlossaryTerms,
|
|
40
|
+
scopedMemory,
|
|
41
|
+
scopedTerms,
|
|
42
|
+
type GlossaryTermRecord,
|
|
43
|
+
type MemoryUnitRecord,
|
|
44
|
+
type TranslationCorpusApi,
|
|
45
|
+
} from '../corpus/translation';
|
|
46
|
+
import type { CorpusPage, CorpusScope, SourceDocument } from '../corpus/types';
|
|
47
|
+
|
|
48
|
+
const TANDEM: CorpusScope = { orgId: FIXTURE_ORG };
|
|
49
|
+
const NORTHWIND: CorpusScope = { orgId: FIXTURE_OTHER_ORG };
|
|
50
|
+
|
|
51
|
+
const EN_AR = { sourceLocale: 'en', targetLocale: 'ar' };
|
|
52
|
+
|
|
53
|
+
function doc(
|
|
54
|
+
id: string,
|
|
55
|
+
body: string,
|
|
56
|
+
scope: CorpusScope = TANDEM,
|
|
57
|
+
title = 'Client document',
|
|
58
|
+
): SourceDocument {
|
|
59
|
+
return {
|
|
60
|
+
id,
|
|
61
|
+
title,
|
|
62
|
+
body,
|
|
63
|
+
sections: [],
|
|
64
|
+
occurredAt: '2026-03-01T00:00:00.000Z',
|
|
65
|
+
source: 'upload',
|
|
66
|
+
sourceLabel: 'Upload',
|
|
67
|
+
sourceRef: id,
|
|
68
|
+
participants: [],
|
|
69
|
+
scope,
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function term(
|
|
74
|
+
over: Partial<GlossaryTermRecord> & Pick<GlossaryTermRecord, 'sourceTerm'>,
|
|
75
|
+
): GlossaryTermRecord {
|
|
76
|
+
return {
|
|
77
|
+
id: `term-${over.sourceTerm}`,
|
|
78
|
+
sourceLocale: 'en',
|
|
79
|
+
targetLocale: 'ar',
|
|
80
|
+
targetTerm: 'اتفاقية التسوية',
|
|
81
|
+
status: 'approved',
|
|
82
|
+
occurrences: 3,
|
|
83
|
+
documentIds: ['doc-1'],
|
|
84
|
+
scope: TANDEM,
|
|
85
|
+
...over,
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function unit(over: Partial<MemoryUnitRecord> & Pick<MemoryUnitRecord, 'sourceText'>): MemoryUnitRecord {
|
|
90
|
+
return {
|
|
91
|
+
id: `unit-${over.sourceText.slice(0, 12)}`,
|
|
92
|
+
sourceLocale: 'en',
|
|
93
|
+
targetLocale: 'ar',
|
|
94
|
+
targetText: 'نص تجريبي',
|
|
95
|
+
sourceHash: over.sourceText.trim().toLowerCase(),
|
|
96
|
+
origin: 'reviewer_edit',
|
|
97
|
+
approved: true,
|
|
98
|
+
scope: TANDEM,
|
|
99
|
+
...over,
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// ---------------------------------------------------------------------------
|
|
104
|
+
|
|
105
|
+
describe('ingestion', () => {
|
|
106
|
+
it('writes rows of the translation types and NOTHING else', () => {
|
|
107
|
+
// A connector that quietly invents a sixth entity_type is a schema change
|
|
108
|
+
// nobody reviewed. Generic envelope, declared types, no surprises.
|
|
109
|
+
const writes = ingestForTranslation(doc('doc-1', 'The settlement agreement is enclosed.'), {
|
|
110
|
+
scope: TANDEM,
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
const types = new Set(writes.map((w) => w.entity_type));
|
|
114
|
+
for (const type of types) {
|
|
115
|
+
expect(Object.values(TRANSLATION_TYPE_KEYS)).toContain(type);
|
|
116
|
+
}
|
|
117
|
+
expect(writes.length).toBeGreaterThan(0);
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
it('carries an external_id on every row, so a re-ingest converges instead of duplicating', () => {
|
|
121
|
+
const writes = ingestForTranslation(doc('doc-1', 'The settlement agreement is enclosed.'), {
|
|
122
|
+
scope: TANDEM,
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
for (const write of writes) expect(write.external_id).toBeTruthy();
|
|
126
|
+
expect(new Set(writes.map((w) => w.external_id)).size).toBe(writes.length);
|
|
127
|
+
});
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
describe('term extraction proposes, it does not decide', () => {
|
|
131
|
+
// Planted terms repeat; the noise does not. That is the whole signal, and it
|
|
132
|
+
// is why `minOccurrences` is a parameter rather than a constant.
|
|
133
|
+
const documents = [
|
|
134
|
+
doc(
|
|
135
|
+
'doc-1',
|
|
136
|
+
[
|
|
137
|
+
'The settlement agreement was signed on Tuesday.',
|
|
138
|
+
'A settlement agreement binds both parties.',
|
|
139
|
+
'Northwind Logistics accepted the settlement agreement.',
|
|
140
|
+
'Force majeure applies to Northwind Logistics only.',
|
|
141
|
+
'The force majeure clause was invoked.',
|
|
142
|
+
'Somebody left a blue umbrella in the lobby.',
|
|
143
|
+
].join(' '),
|
|
144
|
+
),
|
|
145
|
+
doc('doc-2', 'Northwind Logistics confirmed the force majeure position.'),
|
|
146
|
+
];
|
|
147
|
+
|
|
148
|
+
const options = { scope: TANDEM, ...EN_AR, minOccurrences: 2 };
|
|
149
|
+
|
|
150
|
+
it('yields the planted terms', () => {
|
|
151
|
+
const candidates = extractTermCandidates(documents, options);
|
|
152
|
+
const sources = candidates.map((c) => c.sourceTerm.toLowerCase());
|
|
153
|
+
|
|
154
|
+
expect(sources).toContain('settlement agreement');
|
|
155
|
+
expect(sources).toContain('force majeure');
|
|
156
|
+
expect(sources).toContain('northwind logistics');
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
it('does NOT yield the planted noise', () => {
|
|
160
|
+
const candidates = extractTermCandidates(documents, options);
|
|
161
|
+
const sources = candidates.map((c) => c.sourceTerm.toLowerCase());
|
|
162
|
+
|
|
163
|
+
// Occurs once, and is not terminology however memorable it is.
|
|
164
|
+
expect(sources).not.toContain('blue umbrella');
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
it('marks every extracted term a CANDIDATE, never approved, never binding', () => {
|
|
168
|
+
// The rule the bead puts in capitals. A glossary that fills itself is a
|
|
169
|
+
// glossary the client never agreed to — and adherence is measured against it.
|
|
170
|
+
const candidates = extractTermCandidates(documents, options);
|
|
171
|
+
|
|
172
|
+
expect(candidates.length).toBeGreaterThan(0);
|
|
173
|
+
for (const candidate of candidates) {
|
|
174
|
+
expect(candidate.status).toBe('candidate');
|
|
175
|
+
expect(candidate.targetTerm).toBe('');
|
|
176
|
+
}
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
it('never proposes a term a human already rejected', () => {
|
|
180
|
+
const rejected = term({ sourceTerm: 'force majeure', status: 'rejected' });
|
|
181
|
+
|
|
182
|
+
const candidates = extractTermCandidates(documents, { ...options, known: [rejected] });
|
|
183
|
+
|
|
184
|
+
expect(candidates.map((c) => c.sourceTerm.toLowerCase())).not.toContain('force majeure');
|
|
185
|
+
});
|
|
186
|
+
|
|
187
|
+
it('promotes a term only through an explicit verdict', () => {
|
|
188
|
+
const candidate = term({ sourceTerm: 'settlement agreement', status: 'candidate', targetTerm: '' });
|
|
189
|
+
|
|
190
|
+
const approved = applyTermVerdict(candidate, {
|
|
191
|
+
status: 'approved',
|
|
192
|
+
targetTerm: 'اتفاقية التسوية',
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
expect(approved.status).toBe('approved');
|
|
196
|
+
expect(approved.targetTerm).toBe('اتفاقية التسوية');
|
|
197
|
+
});
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
describe('glossary lookup', () => {
|
|
201
|
+
const present = term({ sourceTerm: 'settlement agreement' });
|
|
202
|
+
const absent = term({ sourceTerm: 'force majeure', id: 'term-fm' });
|
|
203
|
+
const candidate = term({
|
|
204
|
+
sourceTerm: 'settlement agreement',
|
|
205
|
+
id: 'term-candidate',
|
|
206
|
+
status: 'candidate',
|
|
207
|
+
targetTerm: '',
|
|
208
|
+
});
|
|
209
|
+
|
|
210
|
+
it('returns only the terms whose source form OCCURS in the segment', () => {
|
|
211
|
+
// The prompt scales with the text, not with the glossary.
|
|
212
|
+
const got = glossaryForSegment('Please review the settlement agreement.', [present, absent]);
|
|
213
|
+
|
|
214
|
+
expect(got.map((t) => t.id)).toEqual([present.id]);
|
|
215
|
+
});
|
|
216
|
+
|
|
217
|
+
it('never returns an unconfirmed candidate, even when it occurs', () => {
|
|
218
|
+
const got = glossaryForSegment('Please review the settlement agreement.', [candidate]);
|
|
219
|
+
|
|
220
|
+
expect(got).toEqual([]);
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
it('matches case-insensitively without matching a different word', () => {
|
|
224
|
+
expect(
|
|
225
|
+
glossaryForSegment('The SETTLEMENT AGREEMENT is enclosed.', [present]).map((t) => t.id),
|
|
226
|
+
).toEqual([present.id]);
|
|
227
|
+
expect(glossaryForSegment('The settlement is enclosed.', [present])).toEqual([]);
|
|
228
|
+
});
|
|
229
|
+
});
|
|
230
|
+
|
|
231
|
+
describe('translation memory', () => {
|
|
232
|
+
const stored = unit({
|
|
233
|
+
sourceText: 'The quarterly report is ready for your review.',
|
|
234
|
+
targetText: 'نص تجريبي: التقرير ربع السنوي جاهز للمراجعة.',
|
|
235
|
+
});
|
|
236
|
+
|
|
237
|
+
it('hits exactly after normalising whitespace and case', () => {
|
|
238
|
+
const hit = exactMemoryHit(
|
|
239
|
+
' the QUARTERLY report is ready for your review. ',
|
|
240
|
+
[stored],
|
|
241
|
+
{ scope: TANDEM, ...EN_AR },
|
|
242
|
+
);
|
|
243
|
+
|
|
244
|
+
expect(hit?.id).toBe(stored.id);
|
|
245
|
+
expect(hit?.targetText).toBe(stored.targetText);
|
|
246
|
+
});
|
|
247
|
+
|
|
248
|
+
it('does NOT hit after a material change', () => {
|
|
249
|
+
// A different number is a different sentence. Normalisation must not eat it.
|
|
250
|
+
const hit = exactMemoryHit(
|
|
251
|
+
'The monthly report is ready for your review.',
|
|
252
|
+
[stored],
|
|
253
|
+
{ scope: TANDEM, ...EN_AR },
|
|
254
|
+
);
|
|
255
|
+
|
|
256
|
+
expect(hit).toBeNull();
|
|
257
|
+
});
|
|
258
|
+
|
|
259
|
+
it('normalises whitespace and case but preserves the material content', () => {
|
|
260
|
+
expect(normalizeForMemory(' The Report ')).toBe(normalizeForMemory('the report'));
|
|
261
|
+
expect(normalizeForMemory('12 seats')).not.toBe(normalizeForMemory('13 seats'));
|
|
262
|
+
});
|
|
263
|
+
|
|
264
|
+
it('never serves an unapproved unit', () => {
|
|
265
|
+
const draft = unit({ sourceText: stored.sourceText, id: 'draft', approved: false });
|
|
266
|
+
|
|
267
|
+
expect(exactMemoryHit(stored.sourceText, [draft], { scope: TANDEM, ...EN_AR })).toBeNull();
|
|
268
|
+
});
|
|
269
|
+
|
|
270
|
+
it('does not hit across a language pair', () => {
|
|
271
|
+
const zh = unit({ sourceText: stored.sourceText, id: 'zh', targetLocale: 'zh' });
|
|
272
|
+
|
|
273
|
+
expect(
|
|
274
|
+
exactMemoryHit(stored.sourceText, [zh], { scope: TANDEM, sourceLocale: 'en', targetLocale: 'ar' }),
|
|
275
|
+
).toBeNull();
|
|
276
|
+
});
|
|
277
|
+
|
|
278
|
+
it('returns fuzzy neighbours ordered by score and honours the limit', () => {
|
|
279
|
+
const near = unit({ sourceText: 'The quarterly report is ready for review.', id: 'near' });
|
|
280
|
+
const mid = unit({ sourceText: 'The quarterly report will be ready soon.', id: 'mid' });
|
|
281
|
+
const far = unit({ sourceText: 'Parking allocation for the London office.', id: 'far' });
|
|
282
|
+
|
|
283
|
+
const matches = fuzzyMemoryMatches(stored.sourceText, [far, mid, near], {
|
|
284
|
+
scope: TANDEM,
|
|
285
|
+
...EN_AR,
|
|
286
|
+
limit: 2,
|
|
287
|
+
});
|
|
288
|
+
|
|
289
|
+
expect(matches).toHaveLength(2);
|
|
290
|
+
expect(matches[0]!.unit.id).toBe('near');
|
|
291
|
+
expect(matches[0]!.score).toBeGreaterThan(matches[1]!.score);
|
|
292
|
+
expect(matches.map((m) => m.unit.id)).not.toContain('far');
|
|
293
|
+
});
|
|
294
|
+
});
|
|
295
|
+
|
|
296
|
+
describe('a reviewer edit becomes memory', () => {
|
|
297
|
+
it('creates an approved unit that a later identical segment hits exactly', () => {
|
|
298
|
+
// THE SINGLE TEST THAT PROVES "it learns from your material". Without it the
|
|
299
|
+
// reviewer corrects the same term forever, which is the upkeep trap that
|
|
300
|
+
// makes people abandon tools.
|
|
301
|
+
const source = 'Please review the settlement agreement before Friday.';
|
|
302
|
+
const edited = 'نص تجريبي مصحَّح: اتفاقية التسوية.';
|
|
303
|
+
|
|
304
|
+
const created = memoryUnitFromReviewerEdit(
|
|
305
|
+
{ sourceText: source, editedTargetText: edited, ...EN_AR },
|
|
306
|
+
TANDEM,
|
|
307
|
+
);
|
|
308
|
+
|
|
309
|
+
expect(created.origin).toBe('reviewer_edit');
|
|
310
|
+
expect(created.approved).toBe(true);
|
|
311
|
+
|
|
312
|
+
const hit = exactMemoryHit(source, [created], { scope: TANDEM, ...EN_AR });
|
|
313
|
+
// BYTE-IDENTICAL. A reviewer who fixed it once must never see it wrong again.
|
|
314
|
+
expect(hit?.targetText).toBe(edited);
|
|
315
|
+
});
|
|
316
|
+
});
|
|
317
|
+
|
|
318
|
+
describe('isolation between clients', () => {
|
|
319
|
+
const mine = term({ sourceTerm: 'settlement agreement', id: 'mine', scope: TANDEM });
|
|
320
|
+
const theirs = term({ sourceTerm: 'settlement agreement', id: 'theirs', scope: NORTHWIND });
|
|
321
|
+
const myUnit = unit({ sourceText: 'Shared sentence.', id: 'my-unit', scope: TANDEM });
|
|
322
|
+
const theirUnit = unit({ sourceText: 'Shared sentence.', id: 'their-unit', scope: NORTHWIND });
|
|
323
|
+
|
|
324
|
+
it('does not return another tenant’s glossary term, and SAYS it dropped one', () => {
|
|
325
|
+
const read = scopedTerms([mine, theirs], TANDEM);
|
|
326
|
+
|
|
327
|
+
expect(read.kept.map((t) => t.id)).toEqual(['mine']);
|
|
328
|
+
// Non-zero and surfaced — a silent zero here is how a leak stays invisible.
|
|
329
|
+
expect(read.droppedOutOfScope).toBe(1);
|
|
330
|
+
});
|
|
331
|
+
|
|
332
|
+
it('does not return another tenant’s memory unit, and SAYS it dropped one', () => {
|
|
333
|
+
const read = scopedMemory([myUnit, theirUnit], TANDEM);
|
|
334
|
+
|
|
335
|
+
expect(read.kept.map((u) => u.id)).toEqual(['my-unit']);
|
|
336
|
+
expect(read.droppedOutOfScope).toBe(1);
|
|
337
|
+
});
|
|
338
|
+
|
|
339
|
+
it('never serves a foreign memory unit as an exact hit', () => {
|
|
340
|
+
// The worst case in the lane: a fluent, confident, correctly-formatted
|
|
341
|
+
// translation sourced from another customer's approved wording.
|
|
342
|
+
const hit = exactMemoryHit('Shared sentence.', [theirUnit], { scope: TANDEM, ...EN_AR });
|
|
343
|
+
|
|
344
|
+
expect(hit).toBeNull();
|
|
345
|
+
});
|
|
346
|
+
|
|
347
|
+
it('never offers a foreign unit as a fuzzy example', () => {
|
|
348
|
+
const matches = fuzzyMemoryMatches('Shared sentence.', [theirUnit], {
|
|
349
|
+
scope: TANDEM,
|
|
350
|
+
...EN_AR,
|
|
351
|
+
limit: 5,
|
|
352
|
+
});
|
|
353
|
+
|
|
354
|
+
expect(matches).toEqual([]);
|
|
355
|
+
});
|
|
356
|
+
|
|
357
|
+
it('returns NOTHING for a principal carrying no org claim', () => {
|
|
358
|
+
// A missing claim must read as "nothing", never as "unscoped, so everything".
|
|
359
|
+
const read = scopedTerms([mine, theirs], { orgId: '' });
|
|
360
|
+
|
|
361
|
+
expect(read.kept).toEqual([]);
|
|
362
|
+
expect(read.droppedOutOfScope).toBe(2);
|
|
363
|
+
});
|
|
364
|
+
|
|
365
|
+
/**
|
|
366
|
+
* The RLS half of bd startsim-ppfl item 7 is NOT in this repository.
|
|
367
|
+
*
|
|
368
|
+
* It belongs to `tenant-starter`, a separate Django project with its own
|
|
369
|
+
* runner (`DJANGO_DEBUG=true .venv/bin/python manage.py test`, sqlite, its own
|
|
370
|
+
* venv — NOT the main-backend Docker stack). There is no `tenant-starter`
|
|
371
|
+
* checkout in this worktree, so the test cannot be written here, and writing
|
|
372
|
+
* a TypeScript imitation of it would be worse than leaving it undone: it would
|
|
373
|
+
* look like coverage of a database guarantee it does not touch.
|
|
374
|
+
*
|
|
375
|
+
* `startsim-bfem` already records this as UNVERIFIED, owner Quinn.
|
|
376
|
+
*/
|
|
377
|
+
it.todo(
|
|
378
|
+
'reading outside org_scope() returns nothing rather than everything — tenant-starter, not this repo',
|
|
379
|
+
);
|
|
380
|
+
});
|
|
381
|
+
|
|
382
|
+
describe('paging', () => {
|
|
383
|
+
/** Three pages of two. A reader that stops at the first sees a third of the glossary. */
|
|
384
|
+
function pagedApi(): TranslationCorpusApi {
|
|
385
|
+
const all = [
|
|
386
|
+
term({ sourceTerm: 'alpha', id: 't1' }),
|
|
387
|
+
term({ sourceTerm: 'bravo', id: 't2' }),
|
|
388
|
+
term({ sourceTerm: 'charlie', id: 't3' }),
|
|
389
|
+
term({ sourceTerm: 'delta', id: 't4' }),
|
|
390
|
+
term({ sourceTerm: 'echo', id: 't5' }),
|
|
391
|
+
term({ sourceTerm: 'foxtrot', id: 't6' }),
|
|
392
|
+
];
|
|
393
|
+
const PAGE = 2;
|
|
394
|
+
return {
|
|
395
|
+
async listGlossaryTerms({ page = 1 }): Promise<CorpusPage<GlossaryTermRecord>> {
|
|
396
|
+
const start = (page - 1) * PAGE;
|
|
397
|
+
const results = all.slice(start, start + PAGE);
|
|
398
|
+
return {
|
|
399
|
+
count: all.length,
|
|
400
|
+
// The server ignores `pageSize`, exactly as the live tenant does.
|
|
401
|
+
next: start + PAGE < all.length ? `?page=${page + 1}` : null,
|
|
402
|
+
previous: page > 1 ? `?page=${page - 1}` : null,
|
|
403
|
+
results,
|
|
404
|
+
};
|
|
405
|
+
},
|
|
406
|
+
async listMemoryUnits(): Promise<CorpusPage<MemoryUnitRecord>> {
|
|
407
|
+
return { count: 0, next: null, previous: null, results: [] };
|
|
408
|
+
},
|
|
409
|
+
};
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
it('reads the glossary PAST the first page', async () => {
|
|
413
|
+
const read = await readAllGlossaryTerms(pagedApi(), TANDEM);
|
|
414
|
+
|
|
415
|
+
expect(read.kept).toHaveLength(6);
|
|
416
|
+
expect(read.kept.map((t) => t.id)).toEqual(['t1', 't2', 't3', 't4', 't5', 't6']);
|
|
417
|
+
});
|
|
418
|
+
|
|
419
|
+
it('scope-filters every page, not just the first', async () => {
|
|
420
|
+
const read = await readAllGlossaryTerms(pagedApi(), NORTHWIND);
|
|
421
|
+
|
|
422
|
+
// Every fixture term belongs to tandem, so a northwind principal gets none
|
|
423
|
+
// — and is TOLD that six rows were dropped rather than shown an empty
|
|
424
|
+
// glossary that looks like a clean read.
|
|
425
|
+
expect(read.kept).toEqual([]);
|
|
426
|
+
expect(read.droppedOutOfScope).toBe(6);
|
|
427
|
+
});
|
|
428
|
+
});
|
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* THE CLIENT CORPUS FOR TRANSLATION — glossary and translation memory
|
|
3
|
+
* (bd startsim-ppfl RED / bd startsim-364c GREEN).
|
|
4
|
+
*
|
|
5
|
+
* CONTRACT AND STUBS ONLY. Every function throws `NotImplementedError`, so the
|
|
6
|
+
* tests in `../__tests__/translation-corpus.test.ts` fail on missing BEHAVIOUR
|
|
7
|
+
* rather than on a missing module.
|
|
8
|
+
*
|
|
9
|
+
* ── WHY THIS LIVES IN THE CORPUS PACKAGE ───────────────────────────────────
|
|
10
|
+
* `startsim-3o4e` decides the whole mechanism is glossary + translation-memory
|
|
11
|
+
* injection — there is no fine-tune. So "the client's own stuff" is a CORPUS
|
|
12
|
+
* question, and this package already owns the three hard parts of that:
|
|
13
|
+
*
|
|
14
|
+
* - the generic `TenantEntity` envelope (`./entities`), so a glossary term
|
|
15
|
+
* and a memory unit are rows of a new `entity_type` and NOT new models
|
|
16
|
+
* (feedback_brutally_generic);
|
|
17
|
+
* - the scope guard (`./retrieval`), which already partitions a candidate set
|
|
18
|
+
* and SURFACES the drop count rather than swallowing it — which is exactly
|
|
19
|
+
* the shape isolation needs;
|
|
20
|
+
* - a Porter stemmer and a Jaccard (`./text`), which term extraction and
|
|
21
|
+
* fuzzy memory both want and neither should reimplement.
|
|
22
|
+
*
|
|
23
|
+
* ── THE ONE RULE THE BEAD PUTS IN CAPITALS ─────────────────────────────────
|
|
24
|
+
* AN UNCONFIRMED AUTO-EXTRACTED TERM MUST NOT SILENTLY BECOME AUTHORITATIVE.
|
|
25
|
+
* Extraction proposes; a human disposes. `status: 'candidate'` is never
|
|
26
|
+
* binding, never injected into a prompt, and never counted in an adherence
|
|
27
|
+
* score — because a glossary that fills itself is a glossary the client did not
|
|
28
|
+
* agree to, and `startsim-r9q8` measures adherence AGAINST it.
|
|
29
|
+
*
|
|
30
|
+
* ── ISOLATION IS THE POINT, NOT A FEATURE ──────────────────────────────────
|
|
31
|
+
* `startsim-hopl` sells "one client's material never appears against
|
|
32
|
+
* another's". A glossary term is the sharpest possible violation of that: it is
|
|
33
|
+
* short, memorable, and it would appear IN ANOTHER CLIENT'S TRANSLATED
|
|
34
|
+
* DOCUMENT. So every read here is scope-partitioned and every drop is counted.
|
|
35
|
+
*/
|
|
36
|
+
|
|
37
|
+
import type { CorpusPage, CorpusScope, SourceDocument } from './types';
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* The `entity_type` keys translation adds. Defaults, not a closed set — the
|
|
41
|
+
* same convention as `DEFAULT_TYPE_KEYS` in `./entities`.
|
|
42
|
+
*/
|
|
43
|
+
export interface TranslationTypeKeys {
|
|
44
|
+
glossaryTerm: string;
|
|
45
|
+
memoryUnit: string;
|
|
46
|
+
/** The client's ingested source document, reusing the corpus document type. */
|
|
47
|
+
sourceDocument: string;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export const TRANSLATION_TYPE_KEYS: TranslationTypeKeys = {
|
|
51
|
+
glossaryTerm: 'glossary_term',
|
|
52
|
+
memoryUnit: 'translation_memory_unit',
|
|
53
|
+
sourceDocument: 'source_document',
|
|
54
|
+
};
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Where a term is in its life.
|
|
58
|
+
*
|
|
59
|
+
* `candidate` — proposed by extraction, NOT authoritative, NOT binding.
|
|
60
|
+
* `approved` — a human ruled on it. Only these bind.
|
|
61
|
+
* `rejected` — a human ruled against it. Kept so extraction stops re-proposing.
|
|
62
|
+
*/
|
|
63
|
+
export type TermStatus = 'candidate' | 'approved' | 'rejected';
|
|
64
|
+
|
|
65
|
+
export interface GlossaryTermRecord {
|
|
66
|
+
id: string;
|
|
67
|
+
sourceLocale: string;
|
|
68
|
+
targetLocale: string;
|
|
69
|
+
sourceTerm: string;
|
|
70
|
+
/** Empty while a candidate — nobody has said what it should be yet. */
|
|
71
|
+
targetTerm: string;
|
|
72
|
+
status: TermStatus;
|
|
73
|
+
/** Brand and entity names: `sourceTerm` must survive verbatim. */
|
|
74
|
+
doNotTranslate?: boolean;
|
|
75
|
+
/** How often extraction saw it. Evidence for a human, never a promotion rule. */
|
|
76
|
+
occurrences: number;
|
|
77
|
+
/** Which document(s) it came from, so a reviewer can check the claim. */
|
|
78
|
+
documentIds: string[];
|
|
79
|
+
scope: CorpusScope;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** Where a memory unit came from. A reviewer edit is the one that matters. */
|
|
83
|
+
export type MemoryOrigin = 'machine' | 'reviewer_edit' | 'imported';
|
|
84
|
+
|
|
85
|
+
export interface MemoryUnitRecord {
|
|
86
|
+
id: string;
|
|
87
|
+
sourceLocale: string;
|
|
88
|
+
targetLocale: string;
|
|
89
|
+
sourceText: string;
|
|
90
|
+
targetText: string;
|
|
91
|
+
/**
|
|
92
|
+
* The NORMALISED source, which is what an exact match compares. Stored rather
|
|
93
|
+
* than recomputed so a normalisation change is visible as a migration instead
|
|
94
|
+
* of silently repartitioning the memory.
|
|
95
|
+
*/
|
|
96
|
+
sourceHash: string;
|
|
97
|
+
origin: MemoryOrigin;
|
|
98
|
+
/** Only an approved unit is served. A machine draft is not memory. */
|
|
99
|
+
approved: boolean;
|
|
100
|
+
scope: CorpusScope;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** A row as the tenant's generic upsert takes it. */
|
|
104
|
+
export interface TenantEntityWrite {
|
|
105
|
+
entity_type: string;
|
|
106
|
+
external_id: string;
|
|
107
|
+
name: string;
|
|
108
|
+
data: Record<string, unknown>;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export class NotImplementedError extends Error {
|
|
112
|
+
constructor(what: string) {
|
|
113
|
+
super(`${what} is not implemented (bd startsim-ppfl is RED; bd startsim-364c implements it)`);
|
|
114
|
+
this.name = 'NotImplementedError';
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// ---------------------------------------------------------------------------
|
|
119
|
+
// ingestion
|
|
120
|
+
// ---------------------------------------------------------------------------
|
|
121
|
+
|
|
122
|
+
export interface IngestOptions {
|
|
123
|
+
scope: CorpusScope;
|
|
124
|
+
typeKeys?: TranslationTypeKeys;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* A client document, as rows.
|
|
129
|
+
*
|
|
130
|
+
* Returns WRITES rather than performing them, so the test can assert what would
|
|
131
|
+
* be written without a tenant — and so ingestion stays pure, like every other
|
|
132
|
+
* mapper in this package.
|
|
133
|
+
*/
|
|
134
|
+
export function ingestForTranslation(
|
|
135
|
+
_document: SourceDocument,
|
|
136
|
+
_options: IngestOptions,
|
|
137
|
+
): TenantEntityWrite[] {
|
|
138
|
+
throw new NotImplementedError('ingestForTranslation');
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
// ---------------------------------------------------------------------------
|
|
142
|
+
// term extraction — proposes, never decides
|
|
143
|
+
// ---------------------------------------------------------------------------
|
|
144
|
+
|
|
145
|
+
export interface ExtractTermsOptions {
|
|
146
|
+
scope: CorpusScope;
|
|
147
|
+
sourceLocale: string;
|
|
148
|
+
targetLocale: string;
|
|
149
|
+
/** Terms already ruled on, so a rejected term is not proposed again. */
|
|
150
|
+
known?: readonly GlossaryTermRecord[];
|
|
151
|
+
/** How many times a phrase must occur before it is worth a human's attention. */
|
|
152
|
+
minOccurrences?: number;
|
|
153
|
+
maxCandidates?: number;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* Propose glossary candidates from the client's own documents.
|
|
158
|
+
*
|
|
159
|
+
* EVERY RETURNED TERM IS `status: 'candidate'`. There is no code path in this
|
|
160
|
+
* module that produces an `approved` term without a human verdict.
|
|
161
|
+
*/
|
|
162
|
+
export function extractTermCandidates(
|
|
163
|
+
_documents: readonly SourceDocument[],
|
|
164
|
+
_options: ExtractTermsOptions,
|
|
165
|
+
): GlossaryTermRecord[] {
|
|
166
|
+
throw new NotImplementedError('extractTermCandidates');
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/** Record a human's verdict. The ONLY route from `candidate` to `approved`. */
|
|
170
|
+
export function applyTermVerdict(
|
|
171
|
+
_term: GlossaryTermRecord,
|
|
172
|
+
_verdict: { status: 'approved' | 'rejected'; targetTerm?: string; doNotTranslate?: boolean },
|
|
173
|
+
): GlossaryTermRecord {
|
|
174
|
+
throw new NotImplementedError('applyTermVerdict');
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// ---------------------------------------------------------------------------
|
|
178
|
+
// glossary lookup — scales with the text, not with the glossary
|
|
179
|
+
// ---------------------------------------------------------------------------
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* The BINDING terms whose source form occurs in `segmentText`.
|
|
183
|
+
*
|
|
184
|
+
* Two filters, both load-bearing: occurrence (so the prompt scales with the
|
|
185
|
+
* page, not the glossary) and status (so a candidate never reaches a model).
|
|
186
|
+
*/
|
|
187
|
+
export function glossaryForSegment(
|
|
188
|
+
_segmentText: string,
|
|
189
|
+
_terms: readonly GlossaryTermRecord[],
|
|
190
|
+
): GlossaryTermRecord[] {
|
|
191
|
+
throw new NotImplementedError('glossaryForSegment');
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// ---------------------------------------------------------------------------
|
|
195
|
+
// translation memory
|
|
196
|
+
// ---------------------------------------------------------------------------
|
|
197
|
+
|
|
198
|
+
/**
|
|
199
|
+
* The comparable form of a source string: whitespace collapsed, case folded,
|
|
200
|
+
* bidi isolates stripped. A MATERIAL change — a different word, a different
|
|
201
|
+
* number — must survive normalisation and defeat the match.
|
|
202
|
+
*/
|
|
203
|
+
export function normalizeForMemory(_text: string): string {
|
|
204
|
+
throw new NotImplementedError('normalizeForMemory');
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
export interface MemoryLookupOptions {
|
|
208
|
+
scope: CorpusScope;
|
|
209
|
+
sourceLocale: string;
|
|
210
|
+
targetLocale: string;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/** An exact hit, or null. Costs nothing and makes no provider call. */
|
|
214
|
+
export function exactMemoryHit(
|
|
215
|
+
_segmentText: string,
|
|
216
|
+
_units: readonly MemoryUnitRecord[],
|
|
217
|
+
_options: MemoryLookupOptions,
|
|
218
|
+
): MemoryUnitRecord | null {
|
|
219
|
+
throw new NotImplementedError('exactMemoryHit');
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
export interface FuzzyMatch {
|
|
223
|
+
unit: MemoryUnitRecord;
|
|
224
|
+
/** 0..1. Higher is closer. */
|
|
225
|
+
score: number;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
export interface FuzzyOptions extends MemoryLookupOptions {
|
|
229
|
+
limit?: number;
|
|
230
|
+
minScore?: number;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/** Neighbours, best first, capped. Offered to the model as EXAMPLES, never as answers. */
|
|
234
|
+
export function fuzzyMemoryMatches(
|
|
235
|
+
_segmentText: string,
|
|
236
|
+
_units: readonly MemoryUnitRecord[],
|
|
237
|
+
_options: FuzzyOptions,
|
|
238
|
+
): FuzzyMatch[] {
|
|
239
|
+
throw new NotImplementedError('fuzzyMemoryMatches');
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/**
|
|
243
|
+
* A reviewer's accepted edit, as a memory unit.
|
|
244
|
+
*
|
|
245
|
+
* `startsim-witd`: without this "the reviewer corrects the same term forever,
|
|
246
|
+
* which is exactly the upkeep trap that makes people abandon tools."
|
|
247
|
+
*/
|
|
248
|
+
export function memoryUnitFromReviewerEdit(
|
|
249
|
+
_edit: {
|
|
250
|
+
sourceText: string;
|
|
251
|
+
editedTargetText: string;
|
|
252
|
+
sourceLocale: string;
|
|
253
|
+
targetLocale: string;
|
|
254
|
+
},
|
|
255
|
+
_scope: CorpusScope,
|
|
256
|
+
): MemoryUnitRecord {
|
|
257
|
+
throw new NotImplementedError('memoryUnitFromReviewerEdit');
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
// ---------------------------------------------------------------------------
|
|
261
|
+
// isolation
|
|
262
|
+
// ---------------------------------------------------------------------------
|
|
263
|
+
|
|
264
|
+
export interface ScopedRead<T> {
|
|
265
|
+
kept: T[];
|
|
266
|
+
/** NON-ZERO IS A BUG UPSTREAM AND SOMEBODY NEEDS TO SEE IT. Never swallowed. */
|
|
267
|
+
droppedOutOfScope: number;
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* Partition glossary terms by the asking principal's scope.
|
|
272
|
+
*
|
|
273
|
+
* A THIRD check, after the tenant API's `org_id` filter and Postgres RLS, for
|
|
274
|
+
* the reason `./retrieval` already states: if any layer above hands back a row
|
|
275
|
+
* it should not have, the OUTPUT must still not contain it.
|
|
276
|
+
*/
|
|
277
|
+
export function scopedTerms(
|
|
278
|
+
_terms: readonly GlossaryTermRecord[],
|
|
279
|
+
_scope: CorpusScope,
|
|
280
|
+
): ScopedRead<GlossaryTermRecord> {
|
|
281
|
+
throw new NotImplementedError('scopedTerms');
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
export function scopedMemory(
|
|
285
|
+
_units: readonly MemoryUnitRecord[],
|
|
286
|
+
_scope: CorpusScope,
|
|
287
|
+
): ScopedRead<MemoryUnitRecord> {
|
|
288
|
+
throw new NotImplementedError('scopedMemory');
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
// ---------------------------------------------------------------------------
|
|
292
|
+
// paging
|
|
293
|
+
// ---------------------------------------------------------------------------
|
|
294
|
+
|
|
295
|
+
/**
|
|
296
|
+
* The read side of the corpus, paged.
|
|
297
|
+
*
|
|
298
|
+
* PAST THE FIRST PAGE, for the reason `./extract` documents from the wire: the
|
|
299
|
+
* tenant's `EntityViewSet` pages with DRF's default and ignores `page_size`, so
|
|
300
|
+
* a caller that reads one page and calls it the glossary silently stops seeing
|
|
301
|
+
* terms at whatever `PAGE_SIZE` happens to be. A glossary that quietly truncates
|
|
302
|
+
* is an adherence score that is quietly wrong.
|
|
303
|
+
*/
|
|
304
|
+
export interface TranslationCorpusApi {
|
|
305
|
+
listGlossaryTerms(query: {
|
|
306
|
+
page?: number;
|
|
307
|
+
pageSize?: number;
|
|
308
|
+
}): Promise<CorpusPage<GlossaryTermRecord>>;
|
|
309
|
+
listMemoryUnits(query: {
|
|
310
|
+
page?: number;
|
|
311
|
+
pageSize?: number;
|
|
312
|
+
}): Promise<CorpusPage<MemoryUnitRecord>>;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
export function readAllGlossaryTerms(
|
|
316
|
+
_api: TranslationCorpusApi,
|
|
317
|
+
_scope: CorpusScope,
|
|
318
|
+
_options?: { maxPages?: number },
|
|
319
|
+
): Promise<ScopedRead<GlossaryTermRecord>> {
|
|
320
|
+
throw new NotImplementedError('readAllGlossaryTerms');
|
|
321
|
+
}
|