hybridtm 0.7.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/README.md +33 -10
  2. package/dist/backupHandler.d.ts +60 -0
  3. package/dist/backupHandler.js +335 -0
  4. package/dist/backupHandler.js.map +1 -0
  5. package/dist/backupReader.d.ts +26 -0
  6. package/dist/backupReader.js +55 -0
  7. package/dist/backupReader.js.map +1 -0
  8. package/dist/batchImporter.js +3 -2
  9. package/dist/batchImporter.js.map +1 -1
  10. package/dist/cli/backupCommand.d.ts +13 -0
  11. package/dist/cli/backupCommand.js +47 -0
  12. package/dist/cli/backupCommand.js.map +1 -0
  13. package/dist/cli/cliUtils.js +10 -9
  14. package/dist/cli/cliUtils.js.map +1 -1
  15. package/dist/cli/createCommand.js +5 -2
  16. package/dist/cli/createCommand.js.map +1 -1
  17. package/dist/cli/hybridtmCli.js +15 -5
  18. package/dist/cli/hybridtmCli.js.map +1 -1
  19. package/dist/cli/importCommand.js +1 -3
  20. package/dist/cli/importCommand.js.map +1 -1
  21. package/dist/cli/matchCommand.js +131 -15
  22. package/dist/cli/matchCommand.js.map +1 -1
  23. package/dist/cli/registryCommands.js +30 -6
  24. package/dist/cli/registryCommands.js.map +1 -1
  25. package/dist/cli/restoreCommand.d.ts +13 -0
  26. package/dist/cli/restoreCommand.js +98 -0
  27. package/dist/cli/restoreCommand.js.map +1 -0
  28. package/dist/hybridtm.d.ts +10 -3
  29. package/dist/hybridtm.js +267 -72
  30. package/dist/hybridtm.js.map +1 -1
  31. package/dist/importOptions.d.ts +0 -1
  32. package/dist/importOptions.js +0 -1
  33. package/dist/importOptions.js.map +1 -1
  34. package/dist/index.d.ts +2 -0
  35. package/dist/index.js +2 -0
  36. package/dist/index.js.map +1 -1
  37. package/dist/tmxHandler.d.ts +1 -0
  38. package/dist/tmxHandler.js +8 -2
  39. package/dist/tmxHandler.js.map +1 -1
  40. package/dist/utils.d.ts +2 -0
  41. package/dist/utils.js +10 -0
  42. package/dist/utils.js.map +1 -1
  43. package/dist/xliffHandler.d.ts +0 -1
  44. package/dist/xliffHandler.js +29 -25
  45. package/dist/xliffHandler.js.map +1 -1
  46. package/package.json +6 -6
package/dist/hybridtm.js CHANGED
@@ -9,13 +9,15 @@
9
9
  * Contributors:
10
10
  * Maxprograms - initial API and implementation
11
11
  *******************************************************************************/
12
- import { connect } from '@lancedb/lancedb';
13
- import { pipeline } from '@xenova/transformers';
12
+ import { pipeline } from '@huggingface/transformers';
13
+ import { connect, Index } from '@lancedb/lancedb';
14
14
  import { Field, FixedSizeList, Float32, Int32, Schema, Utf8 } from 'apache-arrow';
15
- import { unlinkSync } from 'node:fs';
15
+ import { createWriteStream, existsSync, unlinkSync } from 'node:fs';
16
16
  import { tmpdir } from "node:os";
17
17
  import { join } from 'node:path';
18
18
  import { TMReader } from 'sdltm';
19
+ import { XMLAttribute, XMLElement } from 'typesxml';
20
+ import { BackupReader } from './backupReader.js';
19
21
  import { BatchImporter } from './batchImporter.js';
20
22
  import { resolveImportOptions } from './importOptions.js';
21
23
  import { Match } from './match.js';
@@ -25,18 +27,19 @@ import { Utils } from './utils.js';
25
27
  import { XLIFFReader } from './xliffReader.js';
26
28
  export class HybridTM {
27
29
  // OPTIMIZED MODELS
28
- static SPEED_MODEL = 'Xenova/bge-small-en-v1.5'; // 384-dim, optimized for real-time
29
- static QUALITY_MODEL = 'Xenova/LaBSE'; // 768-dim, optimized for accuracy
30
- static RESOURCE_MODEL = 'Xenova/multilingual-e5-small'; // 384-dim, optimized for modest hardware
30
+ static COMPACT_MODEL = 'Xenova/multilingual-e5-small'; // 384-dim, smallest footprint
31
+ static STANDARD_MODEL = 'Xenova/paraphrase-multilingual-MiniLM-L12-v2'; // 384-dim, same footprint as compact
32
+ static LARGE_MODEL = 'onnx-community/gte-multilingual-base'; // 768-dim, largest footprint
31
33
  name;
32
34
  db = null;
33
35
  table = null;
34
36
  dbPath = '';
35
37
  embedder = null;
38
+ embedderPromise = null;
36
39
  modelName = '';
37
40
  initialized = false;
38
41
  initializationPromise = null;
39
- constructor(name, filePath, modelName = HybridTM.QUALITY_MODEL) {
42
+ constructor(name, filePath, modelName = HybridTM.LARGE_MODEL) {
40
43
  this.name = name;
41
44
  this.dbPath = filePath;
42
45
  this.modelName = modelName;
@@ -117,6 +120,10 @@ export class HybridTM {
117
120
  ]);
118
121
  // Create table with the schema
119
122
  this.table = await this.db.createEmptyTable('langEntry', schema);
123
+ // "id" is looked up by exact match constantly (storeLangEntry/storeBatchEntries
124
+ // dedup, entryExists, getLangEntry, deleteLangEntry, findTargetEntry); without a
125
+ // scalar index each lookup is a full table scan.
126
+ await this.table.createIndex('id', { config: Index.btree(), waitTimeoutSeconds: 30 });
120
127
  }
121
128
  else {
122
129
  this.table = await this.db.openTable('langEntry');
@@ -127,11 +134,21 @@ export class HybridTM {
127
134
  throw err;
128
135
  }
129
136
  }
130
- async initializeEmbedder() {
137
+ initializeEmbedder() {
138
+ if (!this.embedderPromise) {
139
+ this.embedderPromise = this.loadEmbedder();
140
+ }
141
+ return this.embedderPromise;
142
+ }
143
+ async loadEmbedder() {
131
144
  try {
132
- this.embedder = await pipeline('feature-extraction', this.modelName);
145
+ // dtype must be explicit: some models ship an unquantized model.onnx that
146
+ // requires an external model.onnx_data file Transformers.js does not reliably
147
+ // fetch/mount; quantized variants are single-file and avoid it.
148
+ this.embedder = await pipeline('feature-extraction', this.modelName, { dtype: 'q8' });
133
149
  }
134
150
  catch (err) {
151
+ this.embedderPromise = null;
135
152
  console.error('Error initializing embedder:', err);
136
153
  throw err;
137
154
  }
@@ -408,9 +425,21 @@ export class HybridTM {
408
425
  }
409
426
  async close() {
410
427
  try {
428
+ if (this.table) {
429
+ this.table.close();
430
+ this.table = null;
431
+ }
411
432
  if (this.db) {
412
- // LanceDB connections are automatically managed
433
+ this.db.close();
434
+ this.db = null;
435
+ }
436
+ if (this.embedder) {
437
+ await this.embedder.dispose();
438
+ this.embedder = null;
413
439
  }
440
+ this.embedderPromise = null;
441
+ this.initialized = false;
442
+ this.initializationPromise = null;
414
443
  }
415
444
  catch (err) {
416
445
  console.error('Error closing database:', err);
@@ -423,9 +452,12 @@ export class HybridTM {
423
452
  // Enhanced concordance search: finds text fragments and returns all language variants for matching units
424
453
  try {
425
454
  const table = await this.ensureTable();
426
- const escapeLiteral = (value) => value.replaceAll("'", "''");
427
- const escapedFragment = escapeLiteral(textFragment);
428
- const whereFragment = 'language = ' + '\'' + language + '\'' + ' AND contains(pureText, ' + '\'' + escapedFragment + '\')';
455
+ const normalizedLang = Utils.normalizeLanguage(language);
456
+ if (!normalizedLang) {
457
+ throw new Error('Invalid language code "' + language + '"');
458
+ }
459
+ const escapedFragment = Utils.replaceQuotes(textFragment);
460
+ const whereFragment = 'language = ' + '\'' + normalizedLang + '\'' + ' AND contains(pureText, ' + '\'' + escapedFragment + '\')';
429
461
  const fragmentEntries = this.hydrateEntries(await table
430
462
  .query()
431
463
  .where(whereFragment)
@@ -452,7 +484,7 @@ export class HybridTM {
452
484
  const result = [];
453
485
  for (const descriptor of segmentDescriptors.values()) {
454
486
  const segmentPrefix = descriptor.fileId + ':' + descriptor.unitId + ':' + descriptor.segmentIndex + ':';
455
- const sanitizedPrefix = escapeLiteral(segmentPrefix);
487
+ const sanitizedPrefix = Utils.replaceQuotes(segmentPrefix);
456
488
  const segmentVariants = this.hydrateEntries(await table
457
489
  .query()
458
490
  .where('starts_with(id, ' + '\'' + sanitizedPrefix + '\')')
@@ -488,10 +520,14 @@ export class HybridTM {
488
520
  async semanticSearch(queryText, language, limit = 10, filters) {
489
521
  try {
490
522
  const table = await this.ensureTable();
523
+ const normalizedLang = Utils.normalizeLanguage(language);
524
+ if (!normalizedLang) {
525
+ throw new Error('Invalid language code "' + language + '"');
526
+ }
491
527
  const queryEmbedding = await this.generateEmbedding(queryText);
492
528
  const results = this.hydrateEntries(await table
493
529
  .vectorSearch(queryEmbedding)
494
- .where('language = ' + '\'' + language + '\'')
530
+ .where('language = ' + '\'' + normalizedLang + '\'')
495
531
  .limit(limit)
496
532
  .toArray());
497
533
  const filtered = filters
@@ -507,12 +543,24 @@ export class HybridTM {
507
543
  async semanticTranslationSearch(searchStr, srcLang, tgtLang, similarity, limit = 100, filters) {
508
544
  try {
509
545
  const table = await this.ensureTable();
546
+ const normalizedSrc = Utils.normalizeLanguage(srcLang);
547
+ if (!normalizedSrc) {
548
+ throw new Error('Invalid language code "' + srcLang + '"');
549
+ }
550
+ const normalizedTgt = Utils.normalizeLanguage(tgtLang);
551
+ if (!normalizedTgt) {
552
+ throw new Error('Invalid language code "' + tgtLang + '"');
553
+ }
510
554
  // Generate embedding for the search string
511
555
  const queryEmbedding = await this.generateEmbedding(searchStr);
512
- // Get all entries for the source language
556
+ // Fetch a candidate pool well beyond the requested output size: hybrid re-ranking
557
+ // below can reorder results relative to raw vector distance, so a tight candidate
558
+ // pool can silently exclude the true best match before re-ranking ever sees it.
559
+ const candidateLimit = Math.max(limit * 5, 50);
513
560
  const sourceEntries = this.hydrateEntries(await table
514
561
  .vectorSearch(queryEmbedding)
515
- .where('language = ' + '\'' + srcLang + '\'')
562
+ .where('language = ' + '\'' + normalizedSrc + '\'')
563
+ .limit(candidateLimit)
516
564
  .toArray());
517
565
  const rankedMatches = [];
518
566
  const sourceCriteria = filters?.source;
@@ -536,7 +584,7 @@ export class HybridTM {
536
584
  const hybridScore = Math.round((semanticScore + fuzzyScore) / 2);
537
585
  // Only include matches that meet the minimum similarity threshold
538
586
  if (hybridScore >= similarity) {
539
- const targetEntry = await this.findTargetEntry(table, sourceEntry, tgtLang, targetCriteria);
587
+ const targetEntry = await this.findTargetEntry(table, sourceEntry, normalizedTgt, targetCriteria);
540
588
  if (!targetEntry) {
541
589
  continue;
542
590
  }
@@ -560,7 +608,7 @@ export class HybridTM {
560
608
  }
561
609
  catch (err) {
562
610
  console.error('Error performing semantic search with quality:', err);
563
- return [];
611
+ throw err;
564
612
  }
565
613
  }
566
614
  async findTargetEntry(table, sourceEntry, tgtLang, filters) {
@@ -578,10 +626,9 @@ export class HybridTM {
578
626
  }
579
627
  const unitPrefix = sourceEntry.fileId + ':' + sourceEntry.unitId + ':';
580
628
  const sanitizedPrefix = Utils.replaceQuotes(unitPrefix);
581
- const sanitizedLang = Utils.replaceQuotes(tgtLang);
582
629
  const unitMatches = this.hydrateEntries(await table
583
630
  .query()
584
- .where('starts_with(id, ' + '\'' + sanitizedPrefix + '\') AND language = ' + '\'' + sanitizedLang + '\'')
631
+ .where('starts_with(id, ' + '\'' + sanitizedPrefix + '\') AND language = ' + '\'' + tgtLang + '\'')
585
632
  .limit(50)
586
633
  .toArray());
587
634
  if (unitMatches.length === 0) {
@@ -720,40 +767,51 @@ export class HybridTM {
720
767
  async storeLangEntry(fileId, original, unitId, lang, pureText, element, embeddings, segmentIndex = 0, segmentCount = 1, metadata) {
721
768
  try {
722
769
  const table = await this.ensureTable();
723
- const sanitizedFileId = Utils.replaceQuotes(fileId);
724
- const sanitizedUnitId = Utils.replaceQuotes(unitId);
770
+ const normalizedLang = Utils.normalizeLanguage(lang);
771
+ if (!normalizedLang) {
772
+ throw new Error('Invalid language code "' + lang + '"');
773
+ }
725
774
  const safeSegmentIndex = typeof segmentIndex === 'number' ? segmentIndex : 0;
726
775
  const safeSegmentCount = typeof segmentCount === 'number' && segmentCount > 0 ? segmentCount : 1;
727
- const entryId = this.buildEntryId(sanitizedFileId, sanitizedUnitId, safeSegmentIndex, lang);
776
+ const entryId = this.buildEntryId(fileId, unitId, safeSegmentIndex, normalizedLang);
728
777
  const existingEntries = this.hydrateEntries(await table
729
778
  .query()
730
- .where('id = ' + '\'' + entryId + '\'')
779
+ .where('id = ' + '\'' + Utils.replaceQuotes(entryId) + '\'')
731
780
  .toArray());
781
+ let existingVector;
732
782
  if (existingEntries.length > 0) {
733
783
  const existingEntry = existingEntries[0];
734
784
  const contentChanged = (existingEntry.pureText !== pureText ||
735
785
  existingEntry.element !== element.toString() ||
736
786
  existingEntry.original !== original);
737
- if (!contentChanged) {
787
+ const metadataChanged = JSON.stringify(this.flattenMetadata(existingEntry.metadata))
788
+ !== JSON.stringify(this.flattenMetadata(metadata));
789
+ if (!contentChanged && !metadataChanged) {
738
790
  return;
739
791
  }
792
+ if (!contentChanged) {
793
+ existingVector = existingEntry.vector;
794
+ }
740
795
  }
741
796
  // Generate embeddings if not provided
742
797
  let vectorEmbeddings;
743
798
  if (embeddings && embeddings.length > 0) {
744
799
  vectorEmbeddings = embeddings;
745
800
  }
801
+ else if (existingVector && existingVector.length > 0) {
802
+ vectorEmbeddings = existingVector;
803
+ }
746
804
  else {
747
805
  vectorEmbeddings = await this.generateEmbedding(pureText);
748
806
  }
749
807
  const entry = {
750
808
  id: entryId,
751
- language: lang,
809
+ language: normalizedLang,
752
810
  pureText: pureText,
753
811
  element: element.toString(),
754
- fileId: sanitizedFileId,
812
+ fileId: fileId,
755
813
  original: original,
756
- unitId: sanitizedUnitId,
814
+ unitId: unitId,
757
815
  vector: vectorEmbeddings,
758
816
  segmentIndex: safeSegmentIndex,
759
817
  segmentCount: safeSegmentCount,
@@ -761,7 +819,7 @@ export class HybridTM {
761
819
  };
762
820
  // Delete existing entry first (LanceDB upsert approach)
763
821
  try {
764
- await table.delete('id = ' + '\'' + entryId + '\'');
822
+ await table.delete('id = ' + '\'' + Utils.replaceQuotes(entryId) + '\'');
765
823
  }
766
824
  catch (deleteErr) {
767
825
  // Entry might not exist, which is fine
@@ -781,20 +839,22 @@ export class HybridTM {
781
839
  const entryIds = [];
782
840
  // Generate embeddings one by one and build LangEntry objects
783
841
  for (const entry of entries) {
784
- const sanitizedFileId = Utils.replaceQuotes(entry.fileId);
785
- const sanitizedUnitId = Utils.replaceQuotes(entry.unitId);
842
+ const normalizedLang = Utils.normalizeLanguage(entry.language);
843
+ if (!normalizedLang) {
844
+ throw new Error('Invalid language code "' + entry.language + '"');
845
+ }
786
846
  const vectorEmbeddings = await this.generateEmbedding(entry.pureText);
787
847
  const safeSegmentIndex = typeof entry.segmentIndex === 'number' ? entry.segmentIndex : 0;
788
848
  const safeSegmentCount = typeof entry.segmentCount === 'number' && entry.segmentCount > 0 ? entry.segmentCount : 1;
789
- const entryId = this.buildEntryId(sanitizedFileId, sanitizedUnitId, safeSegmentIndex, entry.language);
849
+ const entryId = this.buildEntryId(entry.fileId, entry.unitId, safeSegmentIndex, normalizedLang);
790
850
  const langEntry = {
791
851
  id: entryId,
792
- language: entry.language,
852
+ language: normalizedLang,
793
853
  pureText: entry.pureText,
794
854
  element: entry.element.toString(),
795
- fileId: sanitizedFileId,
855
+ fileId: entry.fileId,
796
856
  original: entry.original,
797
- unitId: sanitizedUnitId,
857
+ unitId: entry.unitId,
798
858
  vector: vectorEmbeddings,
799
859
  segmentIndex: safeSegmentIndex,
800
860
  segmentCount: safeSegmentCount,
@@ -806,7 +866,7 @@ export class HybridTM {
806
866
  // Delete any existing entries with these IDs to prevent duplicates
807
867
  // This ensures that if the same file is imported twice, entries are overwritten
808
868
  if (entryIds.length > 0) {
809
- const idsFilter = entryIds.map(id => '\'' + id + '\'').join(',');
869
+ const idsFilter = entryIds.map(id => '\'' + Utils.replaceQuotes(id) + '\'').join(',');
810
870
  try {
811
871
  await table.delete('id IN (' + idsFilter + ')');
812
872
  }
@@ -825,18 +885,15 @@ export class HybridTM {
825
885
  }
826
886
  async entryExists(fileId, unitId, lang, segmentIndex = 0) {
827
887
  try {
828
- if (!this.table) {
829
- await this.initializeDatabase();
830
- }
831
- if (!this.table) {
832
- throw new Error('Failed to initialize database table');
888
+ const table = await this.ensureTable();
889
+ const normalizedLang = Utils.normalizeLanguage(lang);
890
+ if (!normalizedLang) {
891
+ throw new Error('Invalid language code "' + lang + '"');
833
892
  }
834
- const sanitizedFileId = Utils.replaceQuotes(fileId);
835
- const sanitizedUnitId = Utils.replaceQuotes(unitId);
836
- const entryId = this.buildEntryId(sanitizedFileId, sanitizedUnitId, segmentIndex, lang);
837
- const existingEntries = this.hydrateEntries(await this.table
893
+ const entryId = this.buildEntryId(fileId, unitId, segmentIndex, normalizedLang);
894
+ const existingEntries = this.hydrateEntries(await table
838
895
  .query()
839
- .where('id = ' + '\'' + entryId + '\'')
896
+ .where('id = ' + '\'' + Utils.replaceQuotes(entryId) + '\'')
840
897
  .toArray());
841
898
  return existingEntries.length > 0;
842
899
  }
@@ -847,18 +904,15 @@ export class HybridTM {
847
904
  }
848
905
  async getLangEntry(fileId, unitId, lang, segmentIndex = 0) {
849
906
  try {
850
- if (!this.table) {
851
- await this.initializeDatabase();
852
- }
853
- if (!this.table) {
854
- throw new Error('Failed to initialize database table');
907
+ const table = await this.ensureTable();
908
+ const normalizedLang = Utils.normalizeLanguage(lang);
909
+ if (!normalizedLang) {
910
+ throw new Error('Invalid language code "' + lang + '"');
855
911
  }
856
- const sanitizedFileId = Utils.replaceQuotes(fileId);
857
- const sanitizedUnitId = Utils.replaceQuotes(unitId);
858
- const entryId = this.buildEntryId(sanitizedFileId, sanitizedUnitId, segmentIndex, lang);
859
- const existingEntries = this.hydrateEntries(await this.table
912
+ const entryId = this.buildEntryId(fileId, unitId, segmentIndex, normalizedLang);
913
+ const existingEntries = this.hydrateEntries(await table
860
914
  .query()
861
- .where('id = ' + '\'' + entryId + '\'')
915
+ .where('id = ' + '\'' + Utils.replaceQuotes(entryId) + '\'')
862
916
  .toArray());
863
917
  return existingEntries.length > 0 ? existingEntries[0] : null;
864
918
  }
@@ -869,22 +923,19 @@ export class HybridTM {
869
923
  }
870
924
  async deleteLangEntry(fileId, unitId, lang, segmentIndex = 0) {
871
925
  try {
872
- if (!this.table) {
873
- await this.initializeDatabase();
874
- }
875
- if (!this.table) {
876
- throw new Error('Failed to initialize database table');
926
+ const table = await this.ensureTable();
927
+ const normalizedLang = Utils.normalizeLanguage(lang);
928
+ if (!normalizedLang) {
929
+ throw new Error('Invalid language code "' + lang + '"');
877
930
  }
878
- const sanitizedFileId = Utils.replaceQuotes(fileId);
879
- const sanitizedUnitId = Utils.replaceQuotes(unitId);
880
- const entryId = this.buildEntryId(sanitizedFileId, sanitizedUnitId, segmentIndex, lang);
931
+ const entryId = this.buildEntryId(fileId, unitId, segmentIndex, normalizedLang);
881
932
  // Check if entry exists first
882
933
  const exists = await this.entryExists(fileId, unitId, lang, segmentIndex);
883
934
  if (!exists) {
884
935
  return false;
885
936
  }
886
937
  // Delete the entry
887
- await this.table.delete('id = \'' + entryId + '\'');
938
+ await table.delete('id = \'' + Utils.replaceQuotes(entryId) + '\'');
888
939
  return true;
889
940
  }
890
941
  catch (err) {
@@ -923,13 +974,157 @@ export class HybridTM {
923
974
  const productName = packageJson.default.productName;
924
975
  const version = packageJson.default.version;
925
976
  const tmReader = new TMReader({ 'productName': productName, 'version': version });
926
- const data = await tmReader.convert(filePath, tempFilePath);
927
- if (data.status !== 'Success') {
928
- throw new Error('SDL TM conversion failed for file ' + filePath);
977
+ try {
978
+ const data = await tmReader.convert(filePath, tempFilePath);
979
+ if (data.status !== 'Success') {
980
+ throw new Error('SDL TM conversion failed for file ' + filePath);
981
+ }
982
+ return await this.importTMX(tempFilePath, options);
983
+ }
984
+ finally {
985
+ if (existsSync(tempFilePath)) {
986
+ unlinkSync(tempFilePath);
987
+ }
988
+ }
989
+ }
990
+ async restore(filePath) {
991
+ const reader = new BackupReader(filePath);
992
+ await reader.parse();
993
+ const importer = new BatchImporter(this, reader.getTempFilePath(), reader.getEntryCount());
994
+ await importer.import();
995
+ return reader.getEntryCount();
996
+ }
997
+ async backup(outputPath) {
998
+ try {
999
+ const table = await this.ensureTable();
1000
+ const schema = await table.schema();
1001
+ const columns = schema.fields
1002
+ .map((field) => field.name)
1003
+ .filter((name) => name !== 'vector');
1004
+ const writeStream = createWriteStream(outputPath, { encoding: 'utf8' });
1005
+ const completionPromise = new Promise((resolve, reject) => {
1006
+ writeStream.once('finish', resolve);
1007
+ writeStream.once('error', reject);
1008
+ });
1009
+ const rootElement = new XMLElement('backup');
1010
+ rootElement.setAttribute(new XMLAttribute('name', this.name));
1011
+ rootElement.setAttribute(new XMLAttribute('model', this.modelName));
1012
+ rootElement.setAttribute(new XMLAttribute('date', new Date().toISOString()));
1013
+ writeStream.write('<?xml version="1.0" encoding="UTF-8"?>\n');
1014
+ writeStream.write(rootElement.getHead() + '\n');
1015
+ let entryCount = 0;
1016
+ for await (const batch of table.query().select(columns)) {
1017
+ for (const row of batch) {
1018
+ const entry = this.hydrateEntry(row);
1019
+ writeStream.write(this.buildBackupEntryElement(entry).toString() + '\n');
1020
+ entryCount++;
1021
+ }
1022
+ }
1023
+ writeStream.write(rootElement.getTail() + '\n');
1024
+ writeStream.end();
1025
+ await completionPromise;
1026
+ return entryCount;
1027
+ }
1028
+ catch (err) {
1029
+ console.error('Error creating backup:', err);
1030
+ throw err;
1031
+ }
1032
+ }
1033
+ buildBackupEntryElement(entry) {
1034
+ const entryElement = new XMLElement('entry');
1035
+ entryElement.setAttribute(new XMLAttribute('id', entry.id));
1036
+ entryElement.setAttribute(new XMLAttribute('language', entry.language));
1037
+ entryElement.setAttribute(new XMLAttribute('fileId', entry.fileId));
1038
+ entryElement.setAttribute(new XMLAttribute('original', entry.original));
1039
+ entryElement.setAttribute(new XMLAttribute('unitId', entry.unitId));
1040
+ entryElement.setAttribute(new XMLAttribute('segmentIndex', entry.segmentIndex.toString()));
1041
+ entryElement.setAttribute(new XMLAttribute('segmentCount', entry.segmentCount.toString()));
1042
+ const pureTextElement = new XMLElement('pureText');
1043
+ pureTextElement.addString(entry.pureText);
1044
+ entryElement.addElement(pureTextElement);
1045
+ const elementWrapper = new XMLElement('element');
1046
+ elementWrapper.addElement(Utils.buildXMLElement(entry.element));
1047
+ entryElement.addElement(elementWrapper);
1048
+ const metadataElement = this.buildBackupMetadataElement(entry.metadata);
1049
+ if (metadataElement) {
1050
+ entryElement.addElement(metadataElement);
1051
+ }
1052
+ return entryElement;
1053
+ }
1054
+ buildBackupMetadataElement(metadata) {
1055
+ if (!metadata || Object.keys(metadata).length === 0) {
1056
+ return undefined;
1057
+ }
1058
+ const metadataElement = new XMLElement('metadata');
1059
+ this.appendTextElement(metadataElement, 'state', metadata.state);
1060
+ this.appendTextElement(metadataElement, 'subState', metadata.subState);
1061
+ this.appendTextElement(metadataElement, 'quality', typeof metadata.quality === 'number' ? metadata.quality.toString() : undefined);
1062
+ this.appendTextElement(metadataElement, 'creationDate', metadata.creationDate);
1063
+ this.appendTextElement(metadataElement, 'creationId', metadata.creationId);
1064
+ this.appendTextElement(metadataElement, 'changeDate', metadata.changeDate);
1065
+ this.appendTextElement(metadataElement, 'changeId', metadata.changeId);
1066
+ this.appendTextElement(metadataElement, 'creationTool', metadata.creationTool);
1067
+ this.appendTextElement(metadataElement, 'creationToolVersion', metadata.creationToolVersion);
1068
+ this.appendTextElement(metadataElement, 'context', metadata.context);
1069
+ this.appendTextElement(metadataElement, 'usageCount', typeof metadata.usageCount === 'number' ? metadata.usageCount.toString() : undefined);
1070
+ this.appendTextElement(metadataElement, 'lastUsageDate', metadata.lastUsageDate);
1071
+ if (metadata.notes && metadata.notes.length > 0) {
1072
+ const notesElement = new XMLElement('notes');
1073
+ metadata.notes.forEach((note) => {
1074
+ const noteElement = new XMLElement('note');
1075
+ noteElement.addString(note);
1076
+ notesElement.addElement(noteElement);
1077
+ });
1078
+ metadataElement.addElement(notesElement);
1079
+ }
1080
+ if (metadata.properties && Object.keys(metadata.properties).length > 0) {
1081
+ const propertiesElement = new XMLElement('properties');
1082
+ Object.entries(metadata.properties).forEach(([key, value]) => {
1083
+ const propertyElement = new XMLElement('property');
1084
+ propertyElement.setAttribute(new XMLAttribute('name', key));
1085
+ propertyElement.addString(value);
1086
+ propertiesElement.addElement(propertyElement);
1087
+ });
1088
+ metadataElement.addElement(propertiesElement);
1089
+ }
1090
+ if (metadata.segment) {
1091
+ const segment = metadata.segment;
1092
+ const segmentElement = new XMLElement('segment');
1093
+ if (segment.provider) {
1094
+ segmentElement.setAttribute(new XMLAttribute('provider', segment.provider));
1095
+ }
1096
+ if (segment.fileHash) {
1097
+ segmentElement.setAttribute(new XMLAttribute('fileHash', segment.fileHash));
1098
+ }
1099
+ if (segment.fileId) {
1100
+ segmentElement.setAttribute(new XMLAttribute('fileId', segment.fileId));
1101
+ }
1102
+ if (segment.unitId) {
1103
+ segmentElement.setAttribute(new XMLAttribute('unitId', segment.unitId));
1104
+ }
1105
+ if (segment.segmentId) {
1106
+ segmentElement.setAttribute(new XMLAttribute('segmentId', segment.segmentId));
1107
+ }
1108
+ if (segment.segmentKey) {
1109
+ segmentElement.setAttribute(new XMLAttribute('segmentKey', segment.segmentKey));
1110
+ }
1111
+ if (typeof segment.segmentIndex === 'number') {
1112
+ segmentElement.setAttribute(new XMLAttribute('segmentIndex', segment.segmentIndex.toString()));
1113
+ }
1114
+ if (typeof segment.segmentCount === 'number') {
1115
+ segmentElement.setAttribute(new XMLAttribute('segmentCount', segment.segmentCount.toString()));
1116
+ }
1117
+ metadataElement.addElement(segmentElement);
1118
+ }
1119
+ return metadataElement;
1120
+ }
1121
+ appendTextElement(parent, name, value) {
1122
+ if (!value) {
1123
+ return;
929
1124
  }
930
- const count = await this.importTMX(tempFilePath, options);
931
- unlinkSync(tempFilePath);
932
- return count;
1125
+ const child = new XMLElement(name);
1126
+ child.addString(value);
1127
+ parent.addElement(child);
933
1128
  }
934
1129
  }
935
1130
  //# sourceMappingURL=hybridtm.js.map