hybridtm 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -10
- package/dist/backupHandler.d.ts +60 -0
- package/dist/backupHandler.js +335 -0
- package/dist/backupHandler.js.map +1 -0
- package/dist/backupReader.d.ts +26 -0
- package/dist/backupReader.js +55 -0
- package/dist/backupReader.js.map +1 -0
- package/dist/batchImporter.js +3 -2
- package/dist/batchImporter.js.map +1 -1
- package/dist/cli/backupCommand.d.ts +13 -0
- package/dist/cli/backupCommand.js +47 -0
- package/dist/cli/backupCommand.js.map +1 -0
- package/dist/cli/cliUtils.js +10 -9
- package/dist/cli/cliUtils.js.map +1 -1
- package/dist/cli/createCommand.js +5 -2
- package/dist/cli/createCommand.js.map +1 -1
- package/dist/cli/hybridtmCli.js +15 -5
- package/dist/cli/hybridtmCli.js.map +1 -1
- package/dist/cli/importCommand.js +1 -3
- package/dist/cli/importCommand.js.map +1 -1
- package/dist/cli/matchCommand.js +131 -15
- package/dist/cli/matchCommand.js.map +1 -1
- package/dist/cli/registryCommands.js +30 -6
- package/dist/cli/registryCommands.js.map +1 -1
- package/dist/cli/restoreCommand.d.ts +13 -0
- package/dist/cli/restoreCommand.js +98 -0
- package/dist/cli/restoreCommand.js.map +1 -0
- package/dist/hybridtm.d.ts +10 -3
- package/dist/hybridtm.js +267 -72
- package/dist/hybridtm.js.map +1 -1
- package/dist/importOptions.d.ts +0 -1
- package/dist/importOptions.js +0 -1
- package/dist/importOptions.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/tmxHandler.d.ts +1 -0
- package/dist/tmxHandler.js +8 -2
- package/dist/tmxHandler.js.map +1 -1
- package/dist/utils.d.ts +2 -0
- package/dist/utils.js +10 -0
- package/dist/utils.js.map +1 -1
- package/dist/xliffHandler.d.ts +0 -1
- package/dist/xliffHandler.js +29 -25
- package/dist/xliffHandler.js.map +1 -1
- package/package.json +6 -6
package/dist/hybridtm.js
CHANGED
|
@@ -9,13 +9,15 @@
|
|
|
9
9
|
* Contributors:
|
|
10
10
|
* Maxprograms - initial API and implementation
|
|
11
11
|
*******************************************************************************/
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
12
|
+
import { pipeline } from '@huggingface/transformers';
|
|
13
|
+
import { connect, Index } from '@lancedb/lancedb';
|
|
14
14
|
import { Field, FixedSizeList, Float32, Int32, Schema, Utf8 } from 'apache-arrow';
|
|
15
|
-
import { unlinkSync } from 'node:fs';
|
|
15
|
+
import { createWriteStream, existsSync, unlinkSync } from 'node:fs';
|
|
16
16
|
import { tmpdir } from "node:os";
|
|
17
17
|
import { join } from 'node:path';
|
|
18
18
|
import { TMReader } from 'sdltm';
|
|
19
|
+
import { XMLAttribute, XMLElement } from 'typesxml';
|
|
20
|
+
import { BackupReader } from './backupReader.js';
|
|
19
21
|
import { BatchImporter } from './batchImporter.js';
|
|
20
22
|
import { resolveImportOptions } from './importOptions.js';
|
|
21
23
|
import { Match } from './match.js';
|
|
@@ -25,18 +27,19 @@ import { Utils } from './utils.js';
|
|
|
25
27
|
import { XLIFFReader } from './xliffReader.js';
|
|
26
28
|
export class HybridTM {
|
|
27
29
|
// OPTIMIZED MODELS
|
|
28
|
-
static
|
|
29
|
-
static
|
|
30
|
-
static
|
|
30
|
+
static COMPACT_MODEL = 'Xenova/multilingual-e5-small'; // 384-dim, smallest footprint
|
|
31
|
+
static STANDARD_MODEL = 'Xenova/paraphrase-multilingual-MiniLM-L12-v2'; // 384-dim, same footprint as compact
|
|
32
|
+
static LARGE_MODEL = 'onnx-community/gte-multilingual-base'; // 768-dim, largest footprint
|
|
31
33
|
name;
|
|
32
34
|
db = null;
|
|
33
35
|
table = null;
|
|
34
36
|
dbPath = '';
|
|
35
37
|
embedder = null;
|
|
38
|
+
embedderPromise = null;
|
|
36
39
|
modelName = '';
|
|
37
40
|
initialized = false;
|
|
38
41
|
initializationPromise = null;
|
|
39
|
-
constructor(name, filePath, modelName = HybridTM.
|
|
42
|
+
constructor(name, filePath, modelName = HybridTM.LARGE_MODEL) {
|
|
40
43
|
this.name = name;
|
|
41
44
|
this.dbPath = filePath;
|
|
42
45
|
this.modelName = modelName;
|
|
@@ -117,6 +120,10 @@ export class HybridTM {
|
|
|
117
120
|
]);
|
|
118
121
|
// Create table with the schema
|
|
119
122
|
this.table = await this.db.createEmptyTable('langEntry', schema);
|
|
123
|
+
// "id" is looked up by exact match constantly (storeLangEntry/storeBatchEntries
|
|
124
|
+
// dedup, entryExists, getLangEntry, deleteLangEntry, findTargetEntry); without a
|
|
125
|
+
// scalar index each lookup is a full table scan.
|
|
126
|
+
await this.table.createIndex('id', { config: Index.btree(), waitTimeoutSeconds: 30 });
|
|
120
127
|
}
|
|
121
128
|
else {
|
|
122
129
|
this.table = await this.db.openTable('langEntry');
|
|
@@ -127,11 +134,21 @@ export class HybridTM {
|
|
|
127
134
|
throw err;
|
|
128
135
|
}
|
|
129
136
|
}
|
|
130
|
-
|
|
137
|
+
initializeEmbedder() {
|
|
138
|
+
if (!this.embedderPromise) {
|
|
139
|
+
this.embedderPromise = this.loadEmbedder();
|
|
140
|
+
}
|
|
141
|
+
return this.embedderPromise;
|
|
142
|
+
}
|
|
143
|
+
async loadEmbedder() {
|
|
131
144
|
try {
|
|
132
|
-
|
|
145
|
+
// dtype must be explicit: some models ship an unquantized model.onnx that
|
|
146
|
+
// requires an external model.onnx_data file Transformers.js does not reliably
|
|
147
|
+
// fetch/mount; quantized variants are single-file and avoid it.
|
|
148
|
+
this.embedder = await pipeline('feature-extraction', this.modelName, { dtype: 'q8' });
|
|
133
149
|
}
|
|
134
150
|
catch (err) {
|
|
151
|
+
this.embedderPromise = null;
|
|
135
152
|
console.error('Error initializing embedder:', err);
|
|
136
153
|
throw err;
|
|
137
154
|
}
|
|
@@ -408,9 +425,21 @@ export class HybridTM {
|
|
|
408
425
|
}
|
|
409
426
|
async close() {
|
|
410
427
|
try {
|
|
428
|
+
if (this.table) {
|
|
429
|
+
this.table.close();
|
|
430
|
+
this.table = null;
|
|
431
|
+
}
|
|
411
432
|
if (this.db) {
|
|
412
|
-
|
|
433
|
+
this.db.close();
|
|
434
|
+
this.db = null;
|
|
435
|
+
}
|
|
436
|
+
if (this.embedder) {
|
|
437
|
+
await this.embedder.dispose();
|
|
438
|
+
this.embedder = null;
|
|
413
439
|
}
|
|
440
|
+
this.embedderPromise = null;
|
|
441
|
+
this.initialized = false;
|
|
442
|
+
this.initializationPromise = null;
|
|
414
443
|
}
|
|
415
444
|
catch (err) {
|
|
416
445
|
console.error('Error closing database:', err);
|
|
@@ -423,9 +452,12 @@ export class HybridTM {
|
|
|
423
452
|
// Enhanced concordance search: finds text fragments and returns all language variants for matching units
|
|
424
453
|
try {
|
|
425
454
|
const table = await this.ensureTable();
|
|
426
|
-
const
|
|
427
|
-
|
|
428
|
-
|
|
455
|
+
const normalizedLang = Utils.normalizeLanguage(language);
|
|
456
|
+
if (!normalizedLang) {
|
|
457
|
+
throw new Error('Invalid language code "' + language + '"');
|
|
458
|
+
}
|
|
459
|
+
const escapedFragment = Utils.replaceQuotes(textFragment);
|
|
460
|
+
const whereFragment = 'language = ' + '\'' + normalizedLang + '\'' + ' AND contains(pureText, ' + '\'' + escapedFragment + '\')';
|
|
429
461
|
const fragmentEntries = this.hydrateEntries(await table
|
|
430
462
|
.query()
|
|
431
463
|
.where(whereFragment)
|
|
@@ -452,7 +484,7 @@ export class HybridTM {
|
|
|
452
484
|
const result = [];
|
|
453
485
|
for (const descriptor of segmentDescriptors.values()) {
|
|
454
486
|
const segmentPrefix = descriptor.fileId + ':' + descriptor.unitId + ':' + descriptor.segmentIndex + ':';
|
|
455
|
-
const sanitizedPrefix =
|
|
487
|
+
const sanitizedPrefix = Utils.replaceQuotes(segmentPrefix);
|
|
456
488
|
const segmentVariants = this.hydrateEntries(await table
|
|
457
489
|
.query()
|
|
458
490
|
.where('starts_with(id, ' + '\'' + sanitizedPrefix + '\')')
|
|
@@ -488,10 +520,14 @@ export class HybridTM {
|
|
|
488
520
|
async semanticSearch(queryText, language, limit = 10, filters) {
|
|
489
521
|
try {
|
|
490
522
|
const table = await this.ensureTable();
|
|
523
|
+
const normalizedLang = Utils.normalizeLanguage(language);
|
|
524
|
+
if (!normalizedLang) {
|
|
525
|
+
throw new Error('Invalid language code "' + language + '"');
|
|
526
|
+
}
|
|
491
527
|
const queryEmbedding = await this.generateEmbedding(queryText);
|
|
492
528
|
const results = this.hydrateEntries(await table
|
|
493
529
|
.vectorSearch(queryEmbedding)
|
|
494
|
-
.where('language = ' + '\'' +
|
|
530
|
+
.where('language = ' + '\'' + normalizedLang + '\'')
|
|
495
531
|
.limit(limit)
|
|
496
532
|
.toArray());
|
|
497
533
|
const filtered = filters
|
|
@@ -507,12 +543,24 @@ export class HybridTM {
|
|
|
507
543
|
async semanticTranslationSearch(searchStr, srcLang, tgtLang, similarity, limit = 100, filters) {
|
|
508
544
|
try {
|
|
509
545
|
const table = await this.ensureTable();
|
|
546
|
+
const normalizedSrc = Utils.normalizeLanguage(srcLang);
|
|
547
|
+
if (!normalizedSrc) {
|
|
548
|
+
throw new Error('Invalid language code "' + srcLang + '"');
|
|
549
|
+
}
|
|
550
|
+
const normalizedTgt = Utils.normalizeLanguage(tgtLang);
|
|
551
|
+
if (!normalizedTgt) {
|
|
552
|
+
throw new Error('Invalid language code "' + tgtLang + '"');
|
|
553
|
+
}
|
|
510
554
|
// Generate embedding for the search string
|
|
511
555
|
const queryEmbedding = await this.generateEmbedding(searchStr);
|
|
512
|
-
//
|
|
556
|
+
// Fetch a candidate pool well beyond the requested output size: hybrid re-ranking
|
|
557
|
+
// below can reorder results relative to raw vector distance, so a tight candidate
|
|
558
|
+
// pool can silently exclude the true best match before re-ranking ever sees it.
|
|
559
|
+
const candidateLimit = Math.max(limit * 5, 50);
|
|
513
560
|
const sourceEntries = this.hydrateEntries(await table
|
|
514
561
|
.vectorSearch(queryEmbedding)
|
|
515
|
-
.where('language = ' + '\'' +
|
|
562
|
+
.where('language = ' + '\'' + normalizedSrc + '\'')
|
|
563
|
+
.limit(candidateLimit)
|
|
516
564
|
.toArray());
|
|
517
565
|
const rankedMatches = [];
|
|
518
566
|
const sourceCriteria = filters?.source;
|
|
@@ -536,7 +584,7 @@ export class HybridTM {
|
|
|
536
584
|
const hybridScore = Math.round((semanticScore + fuzzyScore) / 2);
|
|
537
585
|
// Only include matches that meet the minimum similarity threshold
|
|
538
586
|
if (hybridScore >= similarity) {
|
|
539
|
-
const targetEntry = await this.findTargetEntry(table, sourceEntry,
|
|
587
|
+
const targetEntry = await this.findTargetEntry(table, sourceEntry, normalizedTgt, targetCriteria);
|
|
540
588
|
if (!targetEntry) {
|
|
541
589
|
continue;
|
|
542
590
|
}
|
|
@@ -560,7 +608,7 @@ export class HybridTM {
|
|
|
560
608
|
}
|
|
561
609
|
catch (err) {
|
|
562
610
|
console.error('Error performing semantic search with quality:', err);
|
|
563
|
-
|
|
611
|
+
throw err;
|
|
564
612
|
}
|
|
565
613
|
}
|
|
566
614
|
async findTargetEntry(table, sourceEntry, tgtLang, filters) {
|
|
@@ -578,10 +626,9 @@ export class HybridTM {
|
|
|
578
626
|
}
|
|
579
627
|
const unitPrefix = sourceEntry.fileId + ':' + sourceEntry.unitId + ':';
|
|
580
628
|
const sanitizedPrefix = Utils.replaceQuotes(unitPrefix);
|
|
581
|
-
const sanitizedLang = Utils.replaceQuotes(tgtLang);
|
|
582
629
|
const unitMatches = this.hydrateEntries(await table
|
|
583
630
|
.query()
|
|
584
|
-
.where('starts_with(id, ' + '\'' + sanitizedPrefix + '\') AND language = ' + '\'' +
|
|
631
|
+
.where('starts_with(id, ' + '\'' + sanitizedPrefix + '\') AND language = ' + '\'' + tgtLang + '\'')
|
|
585
632
|
.limit(50)
|
|
586
633
|
.toArray());
|
|
587
634
|
if (unitMatches.length === 0) {
|
|
@@ -720,40 +767,51 @@ export class HybridTM {
|
|
|
720
767
|
async storeLangEntry(fileId, original, unitId, lang, pureText, element, embeddings, segmentIndex = 0, segmentCount = 1, metadata) {
|
|
721
768
|
try {
|
|
722
769
|
const table = await this.ensureTable();
|
|
723
|
-
const
|
|
724
|
-
|
|
770
|
+
const normalizedLang = Utils.normalizeLanguage(lang);
|
|
771
|
+
if (!normalizedLang) {
|
|
772
|
+
throw new Error('Invalid language code "' + lang + '"');
|
|
773
|
+
}
|
|
725
774
|
const safeSegmentIndex = typeof segmentIndex === 'number' ? segmentIndex : 0;
|
|
726
775
|
const safeSegmentCount = typeof segmentCount === 'number' && segmentCount > 0 ? segmentCount : 1;
|
|
727
|
-
const entryId = this.buildEntryId(
|
|
776
|
+
const entryId = this.buildEntryId(fileId, unitId, safeSegmentIndex, normalizedLang);
|
|
728
777
|
const existingEntries = this.hydrateEntries(await table
|
|
729
778
|
.query()
|
|
730
|
-
.where('id = ' + '\'' + entryId + '\'')
|
|
779
|
+
.where('id = ' + '\'' + Utils.replaceQuotes(entryId) + '\'')
|
|
731
780
|
.toArray());
|
|
781
|
+
let existingVector;
|
|
732
782
|
if (existingEntries.length > 0) {
|
|
733
783
|
const existingEntry = existingEntries[0];
|
|
734
784
|
const contentChanged = (existingEntry.pureText !== pureText ||
|
|
735
785
|
existingEntry.element !== element.toString() ||
|
|
736
786
|
existingEntry.original !== original);
|
|
737
|
-
|
|
787
|
+
const metadataChanged = JSON.stringify(this.flattenMetadata(existingEntry.metadata))
|
|
788
|
+
!== JSON.stringify(this.flattenMetadata(metadata));
|
|
789
|
+
if (!contentChanged && !metadataChanged) {
|
|
738
790
|
return;
|
|
739
791
|
}
|
|
792
|
+
if (!contentChanged) {
|
|
793
|
+
existingVector = existingEntry.vector;
|
|
794
|
+
}
|
|
740
795
|
}
|
|
741
796
|
// Generate embeddings if not provided
|
|
742
797
|
let vectorEmbeddings;
|
|
743
798
|
if (embeddings && embeddings.length > 0) {
|
|
744
799
|
vectorEmbeddings = embeddings;
|
|
745
800
|
}
|
|
801
|
+
else if (existingVector && existingVector.length > 0) {
|
|
802
|
+
vectorEmbeddings = existingVector;
|
|
803
|
+
}
|
|
746
804
|
else {
|
|
747
805
|
vectorEmbeddings = await this.generateEmbedding(pureText);
|
|
748
806
|
}
|
|
749
807
|
const entry = {
|
|
750
808
|
id: entryId,
|
|
751
|
-
language:
|
|
809
|
+
language: normalizedLang,
|
|
752
810
|
pureText: pureText,
|
|
753
811
|
element: element.toString(),
|
|
754
|
-
fileId:
|
|
812
|
+
fileId: fileId,
|
|
755
813
|
original: original,
|
|
756
|
-
unitId:
|
|
814
|
+
unitId: unitId,
|
|
757
815
|
vector: vectorEmbeddings,
|
|
758
816
|
segmentIndex: safeSegmentIndex,
|
|
759
817
|
segmentCount: safeSegmentCount,
|
|
@@ -761,7 +819,7 @@ export class HybridTM {
|
|
|
761
819
|
};
|
|
762
820
|
// Delete existing entry first (LanceDB upsert approach)
|
|
763
821
|
try {
|
|
764
|
-
await table.delete('id = ' + '\'' + entryId + '\'');
|
|
822
|
+
await table.delete('id = ' + '\'' + Utils.replaceQuotes(entryId) + '\'');
|
|
765
823
|
}
|
|
766
824
|
catch (deleteErr) {
|
|
767
825
|
// Entry might not exist, which is fine
|
|
@@ -781,20 +839,22 @@ export class HybridTM {
|
|
|
781
839
|
const entryIds = [];
|
|
782
840
|
// Generate embeddings one by one and build LangEntry objects
|
|
783
841
|
for (const entry of entries) {
|
|
784
|
-
const
|
|
785
|
-
|
|
842
|
+
const normalizedLang = Utils.normalizeLanguage(entry.language);
|
|
843
|
+
if (!normalizedLang) {
|
|
844
|
+
throw new Error('Invalid language code "' + entry.language + '"');
|
|
845
|
+
}
|
|
786
846
|
const vectorEmbeddings = await this.generateEmbedding(entry.pureText);
|
|
787
847
|
const safeSegmentIndex = typeof entry.segmentIndex === 'number' ? entry.segmentIndex : 0;
|
|
788
848
|
const safeSegmentCount = typeof entry.segmentCount === 'number' && entry.segmentCount > 0 ? entry.segmentCount : 1;
|
|
789
|
-
const entryId = this.buildEntryId(
|
|
849
|
+
const entryId = this.buildEntryId(entry.fileId, entry.unitId, safeSegmentIndex, normalizedLang);
|
|
790
850
|
const langEntry = {
|
|
791
851
|
id: entryId,
|
|
792
|
-
language:
|
|
852
|
+
language: normalizedLang,
|
|
793
853
|
pureText: entry.pureText,
|
|
794
854
|
element: entry.element.toString(),
|
|
795
|
-
fileId:
|
|
855
|
+
fileId: entry.fileId,
|
|
796
856
|
original: entry.original,
|
|
797
|
-
unitId:
|
|
857
|
+
unitId: entry.unitId,
|
|
798
858
|
vector: vectorEmbeddings,
|
|
799
859
|
segmentIndex: safeSegmentIndex,
|
|
800
860
|
segmentCount: safeSegmentCount,
|
|
@@ -806,7 +866,7 @@ export class HybridTM {
|
|
|
806
866
|
// Delete any existing entries with these IDs to prevent duplicates
|
|
807
867
|
// This ensures that if the same file is imported twice, entries are overwritten
|
|
808
868
|
if (entryIds.length > 0) {
|
|
809
|
-
const idsFilter = entryIds.map(id => '\'' + id + '\'').join(',');
|
|
869
|
+
const idsFilter = entryIds.map(id => '\'' + Utils.replaceQuotes(id) + '\'').join(',');
|
|
810
870
|
try {
|
|
811
871
|
await table.delete('id IN (' + idsFilter + ')');
|
|
812
872
|
}
|
|
@@ -825,18 +885,15 @@ export class HybridTM {
|
|
|
825
885
|
}
|
|
826
886
|
async entryExists(fileId, unitId, lang, segmentIndex = 0) {
|
|
827
887
|
try {
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
throw new Error('Failed to initialize database table');
|
|
888
|
+
const table = await this.ensureTable();
|
|
889
|
+
const normalizedLang = Utils.normalizeLanguage(lang);
|
|
890
|
+
if (!normalizedLang) {
|
|
891
|
+
throw new Error('Invalid language code "' + lang + '"');
|
|
833
892
|
}
|
|
834
|
-
const
|
|
835
|
-
const
|
|
836
|
-
const entryId = this.buildEntryId(sanitizedFileId, sanitizedUnitId, segmentIndex, lang);
|
|
837
|
-
const existingEntries = this.hydrateEntries(await this.table
|
|
893
|
+
const entryId = this.buildEntryId(fileId, unitId, segmentIndex, normalizedLang);
|
|
894
|
+
const existingEntries = this.hydrateEntries(await table
|
|
838
895
|
.query()
|
|
839
|
-
.where('id = ' + '\'' + entryId + '\'')
|
|
896
|
+
.where('id = ' + '\'' + Utils.replaceQuotes(entryId) + '\'')
|
|
840
897
|
.toArray());
|
|
841
898
|
return existingEntries.length > 0;
|
|
842
899
|
}
|
|
@@ -847,18 +904,15 @@ export class HybridTM {
|
|
|
847
904
|
}
|
|
848
905
|
async getLangEntry(fileId, unitId, lang, segmentIndex = 0) {
|
|
849
906
|
try {
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
throw new Error('Failed to initialize database table');
|
|
907
|
+
const table = await this.ensureTable();
|
|
908
|
+
const normalizedLang = Utils.normalizeLanguage(lang);
|
|
909
|
+
if (!normalizedLang) {
|
|
910
|
+
throw new Error('Invalid language code "' + lang + '"');
|
|
855
911
|
}
|
|
856
|
-
const
|
|
857
|
-
const
|
|
858
|
-
const entryId = this.buildEntryId(sanitizedFileId, sanitizedUnitId, segmentIndex, lang);
|
|
859
|
-
const existingEntries = this.hydrateEntries(await this.table
|
|
912
|
+
const entryId = this.buildEntryId(fileId, unitId, segmentIndex, normalizedLang);
|
|
913
|
+
const existingEntries = this.hydrateEntries(await table
|
|
860
914
|
.query()
|
|
861
|
-
.where('id = ' + '\'' + entryId + '\'')
|
|
915
|
+
.where('id = ' + '\'' + Utils.replaceQuotes(entryId) + '\'')
|
|
862
916
|
.toArray());
|
|
863
917
|
return existingEntries.length > 0 ? existingEntries[0] : null;
|
|
864
918
|
}
|
|
@@ -869,22 +923,19 @@ export class HybridTM {
|
|
|
869
923
|
}
|
|
870
924
|
async deleteLangEntry(fileId, unitId, lang, segmentIndex = 0) {
|
|
871
925
|
try {
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
throw new Error('Failed to initialize database table');
|
|
926
|
+
const table = await this.ensureTable();
|
|
927
|
+
const normalizedLang = Utils.normalizeLanguage(lang);
|
|
928
|
+
if (!normalizedLang) {
|
|
929
|
+
throw new Error('Invalid language code "' + lang + '"');
|
|
877
930
|
}
|
|
878
|
-
const
|
|
879
|
-
const sanitizedUnitId = Utils.replaceQuotes(unitId);
|
|
880
|
-
const entryId = this.buildEntryId(sanitizedFileId, sanitizedUnitId, segmentIndex, lang);
|
|
931
|
+
const entryId = this.buildEntryId(fileId, unitId, segmentIndex, normalizedLang);
|
|
881
932
|
// Check if entry exists first
|
|
882
933
|
const exists = await this.entryExists(fileId, unitId, lang, segmentIndex);
|
|
883
934
|
if (!exists) {
|
|
884
935
|
return false;
|
|
885
936
|
}
|
|
886
937
|
// Delete the entry
|
|
887
|
-
await
|
|
938
|
+
await table.delete('id = \'' + Utils.replaceQuotes(entryId) + '\'');
|
|
888
939
|
return true;
|
|
889
940
|
}
|
|
890
941
|
catch (err) {
|
|
@@ -923,13 +974,157 @@ export class HybridTM {
|
|
|
923
974
|
const productName = packageJson.default.productName;
|
|
924
975
|
const version = packageJson.default.version;
|
|
925
976
|
const tmReader = new TMReader({ 'productName': productName, 'version': version });
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
977
|
+
try {
|
|
978
|
+
const data = await tmReader.convert(filePath, tempFilePath);
|
|
979
|
+
if (data.status !== 'Success') {
|
|
980
|
+
throw new Error('SDL TM conversion failed for file ' + filePath);
|
|
981
|
+
}
|
|
982
|
+
return await this.importTMX(tempFilePath, options);
|
|
983
|
+
}
|
|
984
|
+
finally {
|
|
985
|
+
if (existsSync(tempFilePath)) {
|
|
986
|
+
unlinkSync(tempFilePath);
|
|
987
|
+
}
|
|
988
|
+
}
|
|
989
|
+
}
|
|
990
|
+
async restore(filePath) {
|
|
991
|
+
const reader = new BackupReader(filePath);
|
|
992
|
+
await reader.parse();
|
|
993
|
+
const importer = new BatchImporter(this, reader.getTempFilePath(), reader.getEntryCount());
|
|
994
|
+
await importer.import();
|
|
995
|
+
return reader.getEntryCount();
|
|
996
|
+
}
|
|
997
|
+
async backup(outputPath) {
|
|
998
|
+
try {
|
|
999
|
+
const table = await this.ensureTable();
|
|
1000
|
+
const schema = await table.schema();
|
|
1001
|
+
const columns = schema.fields
|
|
1002
|
+
.map((field) => field.name)
|
|
1003
|
+
.filter((name) => name !== 'vector');
|
|
1004
|
+
const writeStream = createWriteStream(outputPath, { encoding: 'utf8' });
|
|
1005
|
+
const completionPromise = new Promise((resolve, reject) => {
|
|
1006
|
+
writeStream.once('finish', resolve);
|
|
1007
|
+
writeStream.once('error', reject);
|
|
1008
|
+
});
|
|
1009
|
+
const rootElement = new XMLElement('backup');
|
|
1010
|
+
rootElement.setAttribute(new XMLAttribute('name', this.name));
|
|
1011
|
+
rootElement.setAttribute(new XMLAttribute('model', this.modelName));
|
|
1012
|
+
rootElement.setAttribute(new XMLAttribute('date', new Date().toISOString()));
|
|
1013
|
+
writeStream.write('<?xml version="1.0" encoding="UTF-8"?>\n');
|
|
1014
|
+
writeStream.write(rootElement.getHead() + '\n');
|
|
1015
|
+
let entryCount = 0;
|
|
1016
|
+
for await (const batch of table.query().select(columns)) {
|
|
1017
|
+
for (const row of batch) {
|
|
1018
|
+
const entry = this.hydrateEntry(row);
|
|
1019
|
+
writeStream.write(this.buildBackupEntryElement(entry).toString() + '\n');
|
|
1020
|
+
entryCount++;
|
|
1021
|
+
}
|
|
1022
|
+
}
|
|
1023
|
+
writeStream.write(rootElement.getTail() + '\n');
|
|
1024
|
+
writeStream.end();
|
|
1025
|
+
await completionPromise;
|
|
1026
|
+
return entryCount;
|
|
1027
|
+
}
|
|
1028
|
+
catch (err) {
|
|
1029
|
+
console.error('Error creating backup:', err);
|
|
1030
|
+
throw err;
|
|
1031
|
+
}
|
|
1032
|
+
}
|
|
1033
|
+
buildBackupEntryElement(entry) {
|
|
1034
|
+
const entryElement = new XMLElement('entry');
|
|
1035
|
+
entryElement.setAttribute(new XMLAttribute('id', entry.id));
|
|
1036
|
+
entryElement.setAttribute(new XMLAttribute('language', entry.language));
|
|
1037
|
+
entryElement.setAttribute(new XMLAttribute('fileId', entry.fileId));
|
|
1038
|
+
entryElement.setAttribute(new XMLAttribute('original', entry.original));
|
|
1039
|
+
entryElement.setAttribute(new XMLAttribute('unitId', entry.unitId));
|
|
1040
|
+
entryElement.setAttribute(new XMLAttribute('segmentIndex', entry.segmentIndex.toString()));
|
|
1041
|
+
entryElement.setAttribute(new XMLAttribute('segmentCount', entry.segmentCount.toString()));
|
|
1042
|
+
const pureTextElement = new XMLElement('pureText');
|
|
1043
|
+
pureTextElement.addString(entry.pureText);
|
|
1044
|
+
entryElement.addElement(pureTextElement);
|
|
1045
|
+
const elementWrapper = new XMLElement('element');
|
|
1046
|
+
elementWrapper.addElement(Utils.buildXMLElement(entry.element));
|
|
1047
|
+
entryElement.addElement(elementWrapper);
|
|
1048
|
+
const metadataElement = this.buildBackupMetadataElement(entry.metadata);
|
|
1049
|
+
if (metadataElement) {
|
|
1050
|
+
entryElement.addElement(metadataElement);
|
|
1051
|
+
}
|
|
1052
|
+
return entryElement;
|
|
1053
|
+
}
|
|
1054
|
+
buildBackupMetadataElement(metadata) {
|
|
1055
|
+
if (!metadata || Object.keys(metadata).length === 0) {
|
|
1056
|
+
return undefined;
|
|
1057
|
+
}
|
|
1058
|
+
const metadataElement = new XMLElement('metadata');
|
|
1059
|
+
this.appendTextElement(metadataElement, 'state', metadata.state);
|
|
1060
|
+
this.appendTextElement(metadataElement, 'subState', metadata.subState);
|
|
1061
|
+
this.appendTextElement(metadataElement, 'quality', typeof metadata.quality === 'number' ? metadata.quality.toString() : undefined);
|
|
1062
|
+
this.appendTextElement(metadataElement, 'creationDate', metadata.creationDate);
|
|
1063
|
+
this.appendTextElement(metadataElement, 'creationId', metadata.creationId);
|
|
1064
|
+
this.appendTextElement(metadataElement, 'changeDate', metadata.changeDate);
|
|
1065
|
+
this.appendTextElement(metadataElement, 'changeId', metadata.changeId);
|
|
1066
|
+
this.appendTextElement(metadataElement, 'creationTool', metadata.creationTool);
|
|
1067
|
+
this.appendTextElement(metadataElement, 'creationToolVersion', metadata.creationToolVersion);
|
|
1068
|
+
this.appendTextElement(metadataElement, 'context', metadata.context);
|
|
1069
|
+
this.appendTextElement(metadataElement, 'usageCount', typeof metadata.usageCount === 'number' ? metadata.usageCount.toString() : undefined);
|
|
1070
|
+
this.appendTextElement(metadataElement, 'lastUsageDate', metadata.lastUsageDate);
|
|
1071
|
+
if (metadata.notes && metadata.notes.length > 0) {
|
|
1072
|
+
const notesElement = new XMLElement('notes');
|
|
1073
|
+
metadata.notes.forEach((note) => {
|
|
1074
|
+
const noteElement = new XMLElement('note');
|
|
1075
|
+
noteElement.addString(note);
|
|
1076
|
+
notesElement.addElement(noteElement);
|
|
1077
|
+
});
|
|
1078
|
+
metadataElement.addElement(notesElement);
|
|
1079
|
+
}
|
|
1080
|
+
if (metadata.properties && Object.keys(metadata.properties).length > 0) {
|
|
1081
|
+
const propertiesElement = new XMLElement('properties');
|
|
1082
|
+
Object.entries(metadata.properties).forEach(([key, value]) => {
|
|
1083
|
+
const propertyElement = new XMLElement('property');
|
|
1084
|
+
propertyElement.setAttribute(new XMLAttribute('name', key));
|
|
1085
|
+
propertyElement.addString(value);
|
|
1086
|
+
propertiesElement.addElement(propertyElement);
|
|
1087
|
+
});
|
|
1088
|
+
metadataElement.addElement(propertiesElement);
|
|
1089
|
+
}
|
|
1090
|
+
if (metadata.segment) {
|
|
1091
|
+
const segment = metadata.segment;
|
|
1092
|
+
const segmentElement = new XMLElement('segment');
|
|
1093
|
+
if (segment.provider) {
|
|
1094
|
+
segmentElement.setAttribute(new XMLAttribute('provider', segment.provider));
|
|
1095
|
+
}
|
|
1096
|
+
if (segment.fileHash) {
|
|
1097
|
+
segmentElement.setAttribute(new XMLAttribute('fileHash', segment.fileHash));
|
|
1098
|
+
}
|
|
1099
|
+
if (segment.fileId) {
|
|
1100
|
+
segmentElement.setAttribute(new XMLAttribute('fileId', segment.fileId));
|
|
1101
|
+
}
|
|
1102
|
+
if (segment.unitId) {
|
|
1103
|
+
segmentElement.setAttribute(new XMLAttribute('unitId', segment.unitId));
|
|
1104
|
+
}
|
|
1105
|
+
if (segment.segmentId) {
|
|
1106
|
+
segmentElement.setAttribute(new XMLAttribute('segmentId', segment.segmentId));
|
|
1107
|
+
}
|
|
1108
|
+
if (segment.segmentKey) {
|
|
1109
|
+
segmentElement.setAttribute(new XMLAttribute('segmentKey', segment.segmentKey));
|
|
1110
|
+
}
|
|
1111
|
+
if (typeof segment.segmentIndex === 'number') {
|
|
1112
|
+
segmentElement.setAttribute(new XMLAttribute('segmentIndex', segment.segmentIndex.toString()));
|
|
1113
|
+
}
|
|
1114
|
+
if (typeof segment.segmentCount === 'number') {
|
|
1115
|
+
segmentElement.setAttribute(new XMLAttribute('segmentCount', segment.segmentCount.toString()));
|
|
1116
|
+
}
|
|
1117
|
+
metadataElement.addElement(segmentElement);
|
|
1118
|
+
}
|
|
1119
|
+
return metadataElement;
|
|
1120
|
+
}
|
|
1121
|
+
appendTextElement(parent, name, value) {
|
|
1122
|
+
if (!value) {
|
|
1123
|
+
return;
|
|
929
1124
|
}
|
|
930
|
-
const
|
|
931
|
-
|
|
932
|
-
|
|
1125
|
+
const child = new XMLElement(name);
|
|
1126
|
+
child.addString(value);
|
|
1127
|
+
parent.addElement(child);
|
|
933
1128
|
}
|
|
934
1129
|
}
|
|
935
1130
|
//# sourceMappingURL=hybridtm.js.map
|