hybridtm 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +233 -0
- package/README.md +116 -0
- package/dist/batchimporter.d.ts +22 -0
- package/dist/batchimporter.js +203 -0
- package/dist/batchimporter.js.map +1 -0
- package/dist/constants.d.ts +15 -0
- package/dist/constants.js +16 -0
- package/dist/constants.js.map +1 -0
- package/dist/debug-progress.d.ts +12 -0
- package/dist/debug-progress.js +49 -0
- package/dist/debug-progress.js.map +1 -0
- package/dist/hybridtm.d.ts +66 -0
- package/dist/hybridtm.js +912 -0
- package/dist/hybridtm.js.map +1 -0
- package/dist/hybridtmFactory.d.ts +43 -0
- package/dist/hybridtmFactory.js +165 -0
- package/dist/hybridtmFactory.js.map +1 -0
- package/dist/importOptions.d.ts +21 -0
- package/dist/importOptions.js +24 -0
- package/dist/importOptions.js.map +1 -0
- package/dist/index.d.ts +25 -0
- package/dist/index.js +22 -0
- package/dist/index.js.map +1 -0
- package/dist/langEntry.d.ts +64 -0
- package/dist/langEntry.js +13 -0
- package/dist/langEntry.js.map +1 -0
- package/dist/match.d.ts +22 -0
- package/dist/match.js +38 -0
- package/dist/match.js.map +1 -0
- package/dist/matchQuality.d.ts +16 -0
- package/dist/matchQuality.js +83 -0
- package/dist/matchQuality.js.map +1 -0
- package/dist/ngramIndex.d.ts +31 -0
- package/dist/ngramIndex.js +132 -0
- package/dist/ngramIndex.js.map +1 -0
- package/dist/ngrams.d.ts +21 -0
- package/dist/ngrams.js +73 -0
- package/dist/ngrams.js.map +1 -0
- package/dist/pendingEntry.d.ts +24 -0
- package/dist/pendingEntry.js +13 -0
- package/dist/pendingEntry.js.map +1 -0
- package/dist/progressProvider.d.ts +17 -0
- package/dist/progressProvider.js +14 -0
- package/dist/progressProvider.js.map +1 -0
- package/dist/searchFilters.d.ts +24 -0
- package/dist/searchFilters.js +13 -0
- package/dist/searchFilters.js.map +1 -0
- package/dist/simplexlifftest.d.ts +1 -0
- package/dist/simplexlifftest.js +23 -0
- package/dist/simplexlifftest.js.map +1 -0
- package/dist/test-import.d.ts +12 -0
- package/dist/test-import.js +101 -0
- package/dist/test-import.js.map +1 -0
- package/dist/test-progress-fix.d.ts +1 -0
- package/dist/test-progress-fix.js +32 -0
- package/dist/test-progress-fix.js.map +1 -0
- package/dist/test-progress.d.ts +12 -0
- package/dist/test-progress.js +32 -0
- package/dist/test-progress.js.map +1 -0
- package/dist/tests/vectorSimilarityComparison.d.ts +12 -0
- package/dist/tests/vectorSimilarityComparison.js +153 -0
- package/dist/tests/vectorSimilarityComparison.js.map +1 -0
- package/dist/tmxhandler.d.ts +61 -0
- package/dist/tmxhandler.js +347 -0
- package/dist/tmxhandler.js.map +1 -0
- package/dist/tmxreader.d.ts +25 -0
- package/dist/tmxreader.js +51 -0
- package/dist/tmxreader.js.map +1 -0
- package/dist/tmxtest.d.ts +16 -0
- package/dist/tmxtest.js +316 -0
- package/dist/tmxtest.js.map +1 -0
- package/dist/utils.d.ts +17 -0
- package/dist/utils.js +49 -0
- package/dist/utils.js.map +1 -0
- package/dist/vectorSimilarityTester.d.ts +56 -0
- package/dist/vectorSimilarityTester.js +164 -0
- package/dist/vectorSimilarityTester.js.map +1 -0
- package/dist/xliffhandler.d.ts +67 -0
- package/dist/xliffhandler.js +522 -0
- package/dist/xliffhandler.js.map +1 -0
- package/dist/xliffreader.d.ts +25 -0
- package/dist/xliffreader.js +50 -0
- package/dist/xliffreader.js.map +1 -0
- package/dist/xlifftest.d.ts +54 -0
- package/dist/xlifftest.js +245 -0
- package/dist/xlifftest.js.map +1 -0
- package/package.json +33 -0
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
export declare class XLIFFTest {
|
|
2
|
+
private static readonly NAME;
|
|
3
|
+
private static readonly DB_PATH;
|
|
4
|
+
private static readonly SAMPLE_XLIFF;
|
|
5
|
+
runTest(): Promise<void>;
|
|
6
|
+
private resetEnvironment;
|
|
7
|
+
private verifyEntry;
|
|
8
|
+
private expectTranslationMatch;
|
|
9
|
+
private expectConcordanceMatch;
|
|
10
|
+
private verifyMetadataRoundtrip;
|
|
11
|
+
private assertMetadataEquals;
|
|
12
|
+
}
|
|
13
|
+
/*******************************************************************************
|
|
14
|
+
|
|
15
|
+
Follow the steps to create a translation project from your DITA map
|
|
16
|
+
|
|
17
|
+
I need to edit the name or description of a translation memory
|
|
18
|
+
|
|
19
|
+
How do I add more target languages to my project?
|
|
20
|
+
|
|
21
|
+
What should I do after updating my DITA content?
|
|
22
|
+
|
|
23
|
+
Export a memory as a TMX file to save the translation data
|
|
24
|
+
|
|
25
|
+
Associate existing memories with the new project to reuse translations
|
|
26
|
+
|
|
27
|
+
I want to permanently remove a project from the system
|
|
28
|
+
|
|
29
|
+
Select a name and location for the TMX file containing memory data
|
|
30
|
+
|
|
31
|
+
Import a TMX file into a memory from the command line
|
|
32
|
+
|
|
33
|
+
Fluenta recovers exact matches from translations stored in the previous cycle
|
|
34
|
+
|
|
35
|
+
You don't need to pay again for translation of content that didn't change
|
|
36
|
+
|
|
37
|
+
A company specialized in providing translation and localization services
|
|
38
|
+
|
|
39
|
+
Before using the Command Line Interface you must start Fluenta from the GUI at least once
|
|
40
|
+
|
|
41
|
+
Create a JSON file containing the data required for creating the memory
|
|
42
|
+
|
|
43
|
+
An XLIFF file is generated for each selected target language
|
|
44
|
+
|
|
45
|
+
Check your email and register the evaluation key using the instructions
|
|
46
|
+
|
|
47
|
+
Additional configurations can be added by the user as required
|
|
48
|
+
|
|
49
|
+
Use the buttons in the dialog to update the configuration file
|
|
50
|
+
|
|
51
|
+
The configuration file is automatically updated as you add or remove elements
|
|
52
|
+
|
|
53
|
+
Avoid using folders that could be modified by other programs while you work
|
|
54
|
+
*/
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
import { rmSync } from 'node:fs';
|
|
2
|
+
import { HybridTM } from './hybridtm.js';
|
|
3
|
+
import { HybridTMFactory } from './hybridtmFactory.js';
|
|
4
|
+
import { Utils } from './utils.js';
|
|
5
|
+
export class XLIFFTest {
|
|
6
|
+
static NAME = 'XLIFF Test';
|
|
7
|
+
static DB_PATH = 'my-xliff.lancedb';
|
|
8
|
+
static SAMPLE_XLIFF = './samples/fluenta_es.ditamap.xlf';
|
|
9
|
+
async runTest() {
|
|
10
|
+
try {
|
|
11
|
+
this.resetEnvironment();
|
|
12
|
+
const tm = HybridTMFactory.createInstance(XLIFFTest.NAME, XLIFFTest.DB_PATH, HybridTM.SPEED_MODEL);
|
|
13
|
+
console.log('Created new HybridTM instance with name: ' + XLIFFTest.NAME);
|
|
14
|
+
await tm.importXLIFF(XLIFFTest.SAMPLE_XLIFF)
|
|
15
|
+
.then(() => {
|
|
16
|
+
console.log('XLIFF file imported successfully.');
|
|
17
|
+
})
|
|
18
|
+
.catch(err => {
|
|
19
|
+
console.error('Failed to import XLIFF file:', err);
|
|
20
|
+
});
|
|
21
|
+
await this.verifyEntry(tm, '45', '4', 'en', 1, 1);
|
|
22
|
+
await this.verifyEntry(tm, '45', '4', 'es', 1, 1);
|
|
23
|
+
await this.expectTranslationMatch(tm, 'Fluenta recovers exact matches from translations stored in the previous cycle.', 'en', 'es', 90, 'Fluenta recovers exact matches from translations stored in the previous cycle.', 'Fluenta recupera coincidencias exactas de las traducciones almacenadas en el ciclo anterior.', 'exact match');
|
|
24
|
+
await this.expectTranslationMatch(tm, 'Fluenta automatically recovers exact matches from the previous translation cycle.', 'en', 'es', 75, 'Fluenta recovers exact matches from translations stored in the previous cycle.', 'Fluenta recupera coincidencias exactas de las traducciones almacenadas en el ciclo anterior.', 'fuzzy paraphrase');
|
|
25
|
+
await this.expectTranslationMatch(tm, 'You do not need to pay again for content that stayed the same.', 'en', 'es', 65, 'You don\'t need to pay again for translation of content that didn\'t change.', 'No necesita pagar de nuevo por la traducción del contenido que no cambió.', 'semantic reuse reminder');
|
|
26
|
+
await this.expectConcordanceMatch(tm, 'translation memory for reuse', 'en', 'es', 'Archive the translation memory for reuse.', 'Archive la memoria de traducción para su reutilización.', 'concordance reuse unit');
|
|
27
|
+
await this.verifyMetadataRoundtrip(tm);
|
|
28
|
+
/*
|
|
29
|
+
const array: Map<string, XMLElement>[] = await tm.concordanceSearch('XLIFF file', 'en', 5);
|
|
30
|
+
console.log('Concordance search results:' + array.length + ' entries found.');
|
|
31
|
+
for (const map of array) {
|
|
32
|
+
for (const [lang, element] of map) {
|
|
33
|
+
console.log('Language: ' + lang + '\nElement: ' + element.toString());
|
|
34
|
+
}
|
|
35
|
+
console.log('-------------------------\n');
|
|
36
|
+
}
|
|
37
|
+
*/
|
|
38
|
+
await tm.close();
|
|
39
|
+
}
|
|
40
|
+
catch (err) {
|
|
41
|
+
console.error('Error stack:', err instanceof Error ? err.stack : String(err));
|
|
42
|
+
throw err;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
resetEnvironment() {
|
|
46
|
+
try {
|
|
47
|
+
HybridTMFactory.removeInstance(XLIFFTest.NAME);
|
|
48
|
+
}
|
|
49
|
+
catch (err) {
|
|
50
|
+
// ignore missing instance
|
|
51
|
+
}
|
|
52
|
+
try {
|
|
53
|
+
rmSync(XLIFFTest.DB_PATH, { recursive: true, force: true });
|
|
54
|
+
}
|
|
55
|
+
catch (err) {
|
|
56
|
+
// ignore file system cleanup issues
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
async verifyEntry(tm, fileId, unitId, lang, segmentIndex, segmentCount) {
|
|
60
|
+
const entry = await tm.getLangEntry(fileId, unitId, lang, segmentIndex);
|
|
61
|
+
if (!entry) {
|
|
62
|
+
throw new Error('Entry not found for ' + fileId + ':' + unitId + ':' + segmentIndex + ':' + lang);
|
|
63
|
+
}
|
|
64
|
+
if (entry.id !== fileId + ':' + unitId + ':' + segmentIndex + ':' + lang) {
|
|
65
|
+
throw new Error('Unexpected entry id ' + entry.id);
|
|
66
|
+
}
|
|
67
|
+
if (entry.segmentIndex !== segmentIndex) {
|
|
68
|
+
throw new Error('Segment index mismatch for ' + entry.id);
|
|
69
|
+
}
|
|
70
|
+
if (entry.segmentCount !== segmentCount) {
|
|
71
|
+
throw new Error('Segment count mismatch for ' + entry.id);
|
|
72
|
+
}
|
|
73
|
+
if (!entry.metadata) {
|
|
74
|
+
throw new Error('Metadata missing for ' + entry.id);
|
|
75
|
+
}
|
|
76
|
+
if (!entry.vector || entry.vector.length === 0) {
|
|
77
|
+
throw new Error('Vector missing for ' + entry.id);
|
|
78
|
+
}
|
|
79
|
+
console.log('Verified entry ' + entry.id + ' with vector size ' + entry.vector.length);
|
|
80
|
+
}
|
|
81
|
+
async expectTranslationMatch(tm, query, srcLang, tgtLang, similarity, expectedSourceSnippet, expectedTargetSnippet, label) {
|
|
82
|
+
const matches = await tm.semanticTranslationSearch(query, srcLang, tgtLang, similarity);
|
|
83
|
+
if (matches.length === 0) {
|
|
84
|
+
throw new Error('No matches found for "' + label + '" query.');
|
|
85
|
+
}
|
|
86
|
+
const best = matches[0];
|
|
87
|
+
const sourceXml = best.source.toString();
|
|
88
|
+
const targetXml = best.target.toString();
|
|
89
|
+
if (!sourceXml.includes(expectedSourceSnippet)) {
|
|
90
|
+
throw new Error('Unexpected source match for "' + label + '": ' + sourceXml);
|
|
91
|
+
}
|
|
92
|
+
if (!targetXml.includes(expectedTargetSnippet)) {
|
|
93
|
+
throw new Error('Unexpected target match for "' + label + '": ' + targetXml);
|
|
94
|
+
}
|
|
95
|
+
console.log('[' + label + '] matches=' + matches.length + ' semantic=' + best.semantic + ' fuzzy=' + best.fuzzy + ' hybrid=' + best.hybridScore());
|
|
96
|
+
}
|
|
97
|
+
async expectConcordanceMatch(tm, fragment, srcLang, tgtLang, expectedSourceSnippet, expectedTargetSnippet, label) {
|
|
98
|
+
const results = await tm.concordanceSearch(fragment, srcLang, 5);
|
|
99
|
+
if (results.length === 0) {
|
|
100
|
+
throw new Error('No concordance matches found for "' + label + '" fragment.');
|
|
101
|
+
}
|
|
102
|
+
const best = results[0];
|
|
103
|
+
const sourceElement = best.get(srcLang);
|
|
104
|
+
const targetElement = best.get(tgtLang);
|
|
105
|
+
if (!sourceElement || !sourceElement.toString().includes(expectedSourceSnippet)) {
|
|
106
|
+
throw new Error('Unexpected concordance source for "' + label + '".');
|
|
107
|
+
}
|
|
108
|
+
if (!targetElement || !targetElement.toString().includes(expectedTargetSnippet)) {
|
|
109
|
+
throw new Error('Unexpected concordance target for "' + label + '".');
|
|
110
|
+
}
|
|
111
|
+
console.log('[' + label + '] concordance languages=' + Array.from(best.keys()).join(', '));
|
|
112
|
+
}
|
|
113
|
+
async verifyMetadataRoundtrip(tm) {
|
|
114
|
+
const fileId = 'xliff-metadata-roundtrip.xlf';
|
|
115
|
+
const unitId = 'XLIFF-ROUNDTRIP-1';
|
|
116
|
+
const lang = 'es';
|
|
117
|
+
const segmentIndex = 1;
|
|
118
|
+
const metadata = {
|
|
119
|
+
state: 'reviewed',
|
|
120
|
+
quality: 88,
|
|
121
|
+
creationDate: '2024-11-05T09:15:00Z',
|
|
122
|
+
changeDate: '2024-12-20T17:45:00Z',
|
|
123
|
+
creationTool: 'XLIFF Test Harness',
|
|
124
|
+
creationToolVersion: '1.2.3',
|
|
125
|
+
context: 'XLIFF metadata context',
|
|
126
|
+
properties: {
|
|
127
|
+
release: 'Winter',
|
|
128
|
+
locale: 'es-ES'
|
|
129
|
+
},
|
|
130
|
+
segment: {
|
|
131
|
+
provider: 'xliff-roundtrip',
|
|
132
|
+
fileHash: 'hash-xliff-001',
|
|
133
|
+
fileId: 'roundtrip-file',
|
|
134
|
+
unitId,
|
|
135
|
+
segmentId: '1',
|
|
136
|
+
segmentIndex,
|
|
137
|
+
segmentCount: 1,
|
|
138
|
+
segmentKey: unitId + '-1'
|
|
139
|
+
}
|
|
140
|
+
};
|
|
141
|
+
const entryElement = '<seg>XLIFF metadata roundtrip target</seg>';
|
|
142
|
+
await tm.storeLangEntry(fileId, entryElement, unitId, lang, 'XLIFF metadata roundtrip target', Utils.buildXMLElement(entryElement), undefined, segmentIndex, 1, metadata);
|
|
143
|
+
const fetched = await tm.getLangEntry(fileId, unitId, lang, segmentIndex);
|
|
144
|
+
if (!fetched) {
|
|
145
|
+
throw new Error('XLIFF metadata roundtrip entry missing for ' + unitId);
|
|
146
|
+
}
|
|
147
|
+
this.assertMetadataEquals(fetched.metadata, metadata);
|
|
148
|
+
console.log('[XLIFF metadata roundtrip] state=' + fetched.metadata.state + ' quality=' + fetched.metadata.quality + ' properties=' + JSON.stringify(fetched.metadata.properties));
|
|
149
|
+
}
|
|
150
|
+
assertMetadataEquals(actual, expected) {
|
|
151
|
+
const fail = (field, actualValue, expectedValue) => {
|
|
152
|
+
throw new Error('XLIFF metadata roundtrip mismatch for ' + field + ': expected ' + JSON.stringify(expectedValue) + ' but received ' + JSON.stringify(actualValue));
|
|
153
|
+
};
|
|
154
|
+
const scalarFields = [
|
|
155
|
+
'state',
|
|
156
|
+
'quality',
|
|
157
|
+
'creationDate',
|
|
158
|
+
'changeDate',
|
|
159
|
+
'creationTool',
|
|
160
|
+
'creationToolVersion',
|
|
161
|
+
'context'
|
|
162
|
+
];
|
|
163
|
+
scalarFields.forEach((field) => {
|
|
164
|
+
if ((actual[field] ?? null) !== (expected[field] ?? null)) {
|
|
165
|
+
fail(field, actual[field], expected[field]);
|
|
166
|
+
}
|
|
167
|
+
});
|
|
168
|
+
const normalizeObject = (value) => {
|
|
169
|
+
if (!value) {
|
|
170
|
+
return {};
|
|
171
|
+
}
|
|
172
|
+
const sortedKeys = Object.keys(value).sort();
|
|
173
|
+
return sortedKeys.reduce((acc, key) => {
|
|
174
|
+
acc[key] = value[key];
|
|
175
|
+
return acc;
|
|
176
|
+
}, {});
|
|
177
|
+
};
|
|
178
|
+
const actualProperties = normalizeObject(actual.properties);
|
|
179
|
+
const expectedProperties = normalizeObject(expected.properties);
|
|
180
|
+
if (JSON.stringify(actualProperties) !== JSON.stringify(expectedProperties)) {
|
|
181
|
+
fail('properties', actualProperties, expectedProperties);
|
|
182
|
+
}
|
|
183
|
+
const actualSegment = normalizeObject(actual.segment);
|
|
184
|
+
const expectedSegment = normalizeObject(expected.segment);
|
|
185
|
+
if (JSON.stringify(actualSegment) !== JSON.stringify(expectedSegment)) {
|
|
186
|
+
fail('segment', actualSegment, expectedSegment);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
async function main() {
|
|
191
|
+
const test = new XLIFFTest();
|
|
192
|
+
try {
|
|
193
|
+
await test.runTest();
|
|
194
|
+
console.log('All done!');
|
|
195
|
+
process.exit(0);
|
|
196
|
+
}
|
|
197
|
+
catch (error) {
|
|
198
|
+
console.error('Test execution failed:', error);
|
|
199
|
+
process.exit(1);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
main();
|
|
203
|
+
/*******************************************************************************
|
|
204
|
+
|
|
205
|
+
Follow the steps to create a translation project from your DITA map
|
|
206
|
+
|
|
207
|
+
I need to edit the name or description of a translation memory
|
|
208
|
+
|
|
209
|
+
How do I add more target languages to my project?
|
|
210
|
+
|
|
211
|
+
What should I do after updating my DITA content?
|
|
212
|
+
|
|
213
|
+
Export a memory as a TMX file to save the translation data
|
|
214
|
+
|
|
215
|
+
Associate existing memories with the new project to reuse translations
|
|
216
|
+
|
|
217
|
+
I want to permanently remove a project from the system
|
|
218
|
+
|
|
219
|
+
Select a name and location for the TMX file containing memory data
|
|
220
|
+
|
|
221
|
+
Import a TMX file into a memory from the command line
|
|
222
|
+
|
|
223
|
+
Fluenta recovers exact matches from translations stored in the previous cycle
|
|
224
|
+
|
|
225
|
+
You don't need to pay again for translation of content that didn't change
|
|
226
|
+
|
|
227
|
+
A company specialized in providing translation and localization services
|
|
228
|
+
|
|
229
|
+
Before using the Command Line Interface you must start Fluenta from the GUI at least once
|
|
230
|
+
|
|
231
|
+
Create a JSON file containing the data required for creating the memory
|
|
232
|
+
|
|
233
|
+
An XLIFF file is generated for each selected target language
|
|
234
|
+
|
|
235
|
+
Check your email and register the evaluation key using the instructions
|
|
236
|
+
|
|
237
|
+
Additional configurations can be added by the user as required
|
|
238
|
+
|
|
239
|
+
Use the buttons in the dialog to update the configuration file
|
|
240
|
+
|
|
241
|
+
The configuration file is automatically updated as you add or remove elements
|
|
242
|
+
|
|
243
|
+
Avoid using folders that could be modified by other programs while you work
|
|
244
|
+
*/
|
|
245
|
+
//# sourceMappingURL=xlifftest.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"xlifftest.js","sourceRoot":"","sources":["../ts/xlifftest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAEvD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAGnC,MAAM,OAAO,SAAS;IAEV,MAAM,CAAU,IAAI,GAAW,YAAY,CAAC;IAC5C,MAAM,CAAU,OAAO,GAAW,kBAAkB,CAAC;IACrD,MAAM,CAAU,YAAY,GAAW,kCAAkC,CAAC;IAElF,KAAK,CAAC,OAAO;QACT,IAAI,CAAC;YACD,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACxB,MAAM,EAAE,GAAa,eAAe,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,EAAE,SAAS,CAAC,OAAO,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;YAC7G,OAAO,CAAC,GAAG,CAAC,2CAA2C,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;YAC1E,MAAM,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,YAAY,CAAC;iBACvC,IAAI,CAAC,GAAG,EAAE;gBACP,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;YACrD,CAAC,CAAC;iBACD,KAAK,CAAC,GAAG,CAAC,EAAE;gBACT,OAAO,CAAC,KAAK,CAAC,8BAA8B,EAAE,GAAG,CAAC,CAAC;YACvD,CAAC,CAAC,CAAC;YAEP,MAAM,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAClD,MAAM,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAElD,MAAM,IAAI,CAAC,sBAAsB,CAC7B,EAAE,EACF,gFAAgF,EAChF,IAAI,EACJ,IAAI,EACJ,EAAE,EACF,gFAAgF,EAChF,8FAA8F,EAC9F,aAAa,CAAC,CAAC;YAEnB,MAAM,IAAI,CAAC,sBAAsB,CAC7B,EAAE,EACF,mFAAmF,EACnF,IAAI,EACJ,IAAI,EACJ,EAAE,EACF,gFAAgF,EAChF,8FAA8F,EAC9F,kBAAkB,CAAC,CAAC;YAExB,MAAM,IAAI,CAAC,sBAAsB,CAC7B,EAAE,EACF,gEAAgE,EAChE,IAAI,EACJ,IAAI,EACJ,EAAE,EACF,8EAA8E,EAC9E,2EAA2E,EAC3E,yBAAyB,CAAC,CAAC;YAE/B,MAAM,IAAI,CAAC,sBAAsB,CAC7B,EAAE,EACF,8BAA8B,EAC9B,IAAI,EACJ,IAAI,EACJ,2CAA2C,EAC3C,yDAAyD,EACzD,wBAAwB,CAAC,CAAC;YAC9B,MAAM,IAAI,CAAC,uBAAuB,CAAC,EAAE,CAAC,CAAC;YACvC;;;;;;;;;cASE;YACF,MAAM,EAAG,CAAC,KAAK,EAAE,CAAC;QACtB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9E,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAEO,gBAAgB;QACpB,IAAI,CAAC;YACD,eAAe,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACnD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,0BAA0B;QAC9B,CAAC;QACD,IAAI,CAAC;YACD,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAChE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,oCAAoC;QACxC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,EAAY,EAAE,MAAc,EAAE,MAAc,EAAE,IAAY,EAAE,YAAoB,EAAE,YAAoB;QAC5H,MAAM,KAAK,GAAqB,MAAM,EAAE,CAAC,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;QAC1F,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,sBAAsB,GAAG,MAAM,GAAG,GAAG,GAAG,MAAM,GAAG,GAAG,GAAG,YAAY,GAAG,GAAG,GAAG,IAAI,CAAC,CAAC;QACtG,CAAC;QACD,IAAI,KAAK,CAAC,EAAE,KAAK,MAAM,GAAG,GAAG,GAAG,MAAM,GAAG,GAAG,GAAG,YAAY,GAAG,GAAG,GAAG,IAAI,EAAE,CAAC;YACvE,MAAM,IAAI,KAAK,CAAC,sBAAsB,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,KAAK,CAAC,YAAY,KAAK,YAAY,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,KAAK,CAAC,YAAY,KAAK,YAAY,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,uBAAuB,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC;QACxD,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7C,MAAM,IAAI,KAAK,CAAC,qBAAqB,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,KAAK,CAAC,EAAE,GAAG,oBAAoB,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC3F,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAChC,EAAY,EACZ,KAAa,EACb,OAAe,EACf,OAAe,EACf,UAAkB,EAClB,qBAA6B,EAC7B,qBAA6B,EAC7B,KAAa;QAEb,MAAM,OAAO,GAAY,MAAM,EAAE,CAAC,yBAAyB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;QACjG,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,wBAAwB,GAAG,KAAK,GAAG,UAAU,CAAC,CAAC;QACnE,CAAC;QACD,MAAM,IAAI,GAAU,OAAO,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,SAAS,GAAW,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;QACjD,MAAM,SAAS,GAAW,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;QACjD,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,CAAC;YAC7C,MAAM,IAAI,KAAK,CAAC,+BAA+B,GAAG,KAAK,GAAG,KAAK,GAAG,SAAS,CAAC,CAAC;QACjF,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,CAAC;YAC7C,MAAM,IAAI,KAAK,CAAC,+BAA+B,GAAG,KAAK,GAAG,KAAK,GAAG,SAAS,CAAC,CAAC;QACjF,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,KAAK,GAAG,YAAY,GAAG,OAAO,CAAC,MAAM,GAAG,YAAY,GAAG,IAAI,CAAC,QAAQ,GAAG,SAAS,GAAG,IAAI,CAAC,KAAK,GAAG,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;IACvJ,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAChC,EAAY,EACZ,QAAgB,EAChB,OAAe,EACf,OAAe,EACf,qBAA6B,EAC7B,qBAA6B,EAC7B,KAAa;QAEb,MAAM,OAAO,GAAmC,MAAM,EAAE,CAAC,iBAAiB,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QACjG,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,oCAAoC,GAAG,KAAK,GAAG,aAAa,CAAC,CAAC;QAClF,CAAC;QACD,MAAM,IAAI,GAAiC,OAAO,CAAC,CAAC,CAAC,CAAC;QACtD,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACxC,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACxC,IAAI,CAAC,aAAa,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,CAAC;YAC9E,MAAM,IAAI,KAAK,CAAC,qCAAqC,GAAG,KAAK,GAAG,IAAI,CAAC,CAAC;QAC1E,CAAC;QACD,IAAI,CAAC,aAAa,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,CAAC;YAC9E,MAAM,IAAI,KAAK,CAAC,qCAAqC,GAAG,KAAK,GAAG,IAAI,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,KAAK,GAAG,0BAA0B,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAC/F,CAAC;IAEO,KAAK,CAAC,uBAAuB,CAAC,EAAY;QAC9C,MAAM,MAAM,GAAW,8BAA8B,CAAC;QACtD,MAAM,MAAM,GAAW,mBAAmB,CAAC;QAC3C,MAAM,IAAI,GAAW,IAAI,CAAC;QAC1B,MAAM,YAAY,GAAW,CAAC,CAAC;QAC/B,MAAM,QAAQ,GAAkB;YAC5B,KAAK,EAAE,UAAU;YACjB,OAAO,EAAE,EAAE;YACX,YAAY,EAAE,sBAAsB;YACpC,UAAU,EAAE,sBAAsB;YAClC,YAAY,EAAE,oBAAoB;YAClC,mBAAmB,EAAE,OAAO;YAC5B,OAAO,EAAE,wBAAwB;YACjC,UAAU,EAAE;gBACR,OAAO,EAAE,QAAQ;gBACjB,MAAM,EAAE,OAAO;aAClB;YACD,OAAO,EAAE;gBACL,QAAQ,EAAE,iBAAiB;gBAC3B,QAAQ,EAAE,gBAAgB;gBAC1B,MAAM,EAAE,gBAAgB;gBACxB,MAAM;gBACN,SAAS,EAAE,GAAG;gBACd,YAAY;gBACZ,YAAY,EAAE,CAAC;gBACf,UAAU,EAAE,MAAM,GAAG,IAAI;aAC5B;SACJ,CAAC;QAEF,MAAM,YAAY,GAAW,4CAA4C,CAAC;QAC1E,MAAM,EAAE,CAAC,cAAc,CACnB,MAAM,EACN,YAAY,EACZ,MAAM,EACN,IAAI,EACJ,iCAAiC,EACjC,KAAK,CAAC,eAAe,CAAC,YAAY,CAAC,EACnC,SAAS,EACT,YAAY,EACZ,CAAC,EACD,QAAQ,CACX,CAAC;QAEF,MAAM,OAAO,GAAqB,MAAM,EAAE,CAAC,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;QAC5F,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CAAC,6CAA6C,GAAG,MAAM,CAAC,CAAC;QAC5E,CAAC;QAED,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACtD,OAAO,CAAC,GAAG,CAAC,mCAAmC,GAAG,OAAO,CAAC,QAAQ,CAAC,KAAK,GAAG,WAAW,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,GAAG,cAAc,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC;IACtL,CAAC;IAEO,oBAAoB,CAAC,MAAqB,EAAE,QAAuB;QACvE,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,WAAoB,EAAE,aAAsB,EAAS,EAAE;YAChF,MAAM,IAAI,KAAK,CAAC,wCAAwC,GAAG,KAAK,GAAG,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,GAAG,gBAAgB,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC;QACvK,CAAC,CAAC;QAEF,MAAM,YAAY,GAA4B;YAC1C,OAAO;YACP,SAAS;YACT,cAAc;YACd,YAAY;YACZ,cAAc;YACd,qBAAqB;YACrB,SAAS;SACZ,CAAC;QACF,YAAY,CAAC,OAAO,CAAC,CAAC,KAA0B,EAAE,EAAE;YAChD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC;gBACxD,IAAI,CAAC,KAAe,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;YAC1D,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,MAAM,eAAe,GAAG,CAAC,KAA0C,EAA2B,EAAE;YAC5F,IAAI,CAAC,KAAK,EAAE,CAAC;gBACT,OAAO,EAAE,CAAC;YACd,CAAC;YACD,MAAM,UAAU,GAAa,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;YACvD,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,GAA4B,EAAE,GAAW,EAAE,EAAE;gBACnE,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;gBACtB,OAAO,GAAG,CAAC;YACf,CAAC,EAAE,EAAE,CAAC,CAAC;QACX,CAAC,CAAC;QAEF,MAAM,gBAAgB,GAA4B,eAAe,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACrF,MAAM,kBAAkB,GAA4B,eAAe,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QACzF,IAAI,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC,EAAE,CAAC;YAC1E,IAAI,CAAC,YAAY,EAAE,gBAAgB,EAAE,kBAAkB,CAAC,CAAC;QAC7D,CAAC;QAED,MAAM,aAAa,GAA4B,eAAe,CAAC,MAAM,CAAC,OAA8C,CAAC,CAAC;QACtH,MAAM,eAAe,GAA4B,eAAe,CAAC,QAAQ,CAAC,OAA8C,CAAC,CAAC;QAC1H,IAAI,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,EAAE,CAAC;YACpE,IAAI,CAAC,SAAS,EAAE,aAAa,EAAE,eAAe,CAAC,CAAC;QACpD,CAAC;IACL,CAAC;;AAGL,KAAK,UAAU,IAAI;IACf,MAAM,IAAI,GAAc,IAAI,SAAS,EAAE,CAAC;IACxC,IAAI,CAAC;QACD,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACzB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC,CAAC;QAC/C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;AACL,CAAC;AAED,IAAI,EAAE,CAAC;AAIP;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAyCE"}
|
package/package.json
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "hybridtm",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "HybridTM is a Translation Memory (TM) engine that supports ngrams-based and vector-based matching.",
|
|
5
|
+
"main": "dist/index.js",
|
|
6
|
+
"type": "module",
|
|
7
|
+
"types": "dist/index.d.ts",
|
|
8
|
+
"files": [
|
|
9
|
+
"/dist"
|
|
10
|
+
],
|
|
11
|
+
"scripts": {
|
|
12
|
+
"build": "tsc"
|
|
13
|
+
},
|
|
14
|
+
"author": {
|
|
15
|
+
"name": "Rodolfo M. Raya",
|
|
16
|
+
"url": "https://maxprograms.com",
|
|
17
|
+
"email": "rmraya@maxprograms.com"
|
|
18
|
+
},
|
|
19
|
+
"keywords": [],
|
|
20
|
+
"license": "EPL-1.0",
|
|
21
|
+
"dependencies": {
|
|
22
|
+
"@lancedb/lancedb": "^0.23.0",
|
|
23
|
+
"@xenova/transformers": "^2.17.2",
|
|
24
|
+
"apache-arrow": "^18.1.0",
|
|
25
|
+
"typesbcp47": "^1.15.0",
|
|
26
|
+
"typesxml": "^1.16.0"
|
|
27
|
+
},
|
|
28
|
+
"devDependencies": {
|
|
29
|
+
"@types/node": "^24.10.4",
|
|
30
|
+
"ts-node": "^10.9.2",
|
|
31
|
+
"typescript": "^5.9.3"
|
|
32
|
+
}
|
|
33
|
+
}
|