@danielsimonjr/memoryjs 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (300) hide show
  1. package/README.md +385 -113
  2. package/README.md.backup-1768084780988 +266 -0
  3. package/dist/index.cjs +17364 -0
  4. package/dist/index.cjs.map +1 -0
  5. package/dist/index.d.cts +12371 -0
  6. package/dist/index.d.ts +12367 -11
  7. package/dist/index.js +17102 -19
  8. package/dist/index.js.map +1 -1
  9. package/dist/workers/levenshteinWorker.cjs +102 -0
  10. package/dist/workers/levenshteinWorker.cjs.map +1 -0
  11. package/dist/workers/levenshteinWorker.js +57 -91
  12. package/dist/workers/levenshteinWorker.js.map +1 -1
  13. package/package.json +12 -6
  14. package/dist/core/EntityManager.d.ts +0 -268
  15. package/dist/core/EntityManager.d.ts.map +0 -1
  16. package/dist/core/EntityManager.js +0 -512
  17. package/dist/core/EntityManager.js.map +0 -1
  18. package/dist/core/GraphEventEmitter.d.ts +0 -202
  19. package/dist/core/GraphEventEmitter.d.ts.map +0 -1
  20. package/dist/core/GraphEventEmitter.js +0 -347
  21. package/dist/core/GraphEventEmitter.js.map +0 -1
  22. package/dist/core/GraphStorage.d.ts +0 -395
  23. package/dist/core/GraphStorage.d.ts.map +0 -1
  24. package/dist/core/GraphStorage.js +0 -786
  25. package/dist/core/GraphStorage.js.map +0 -1
  26. package/dist/core/GraphTraversal.d.ts +0 -141
  27. package/dist/core/GraphTraversal.d.ts.map +0 -1
  28. package/dist/core/GraphTraversal.js +0 -574
  29. package/dist/core/GraphTraversal.js.map +0 -1
  30. package/dist/core/HierarchyManager.d.ts +0 -111
  31. package/dist/core/HierarchyManager.d.ts.map +0 -1
  32. package/dist/core/HierarchyManager.js +0 -225
  33. package/dist/core/HierarchyManager.js.map +0 -1
  34. package/dist/core/ManagerContext.d.ts +0 -76
  35. package/dist/core/ManagerContext.d.ts.map +0 -1
  36. package/dist/core/ManagerContext.js +0 -129
  37. package/dist/core/ManagerContext.js.map +0 -1
  38. package/dist/core/ObservationManager.d.ts +0 -85
  39. package/dist/core/ObservationManager.d.ts.map +0 -1
  40. package/dist/core/ObservationManager.js +0 -124
  41. package/dist/core/ObservationManager.js.map +0 -1
  42. package/dist/core/RelationManager.d.ts +0 -131
  43. package/dist/core/RelationManager.d.ts.map +0 -1
  44. package/dist/core/RelationManager.js +0 -212
  45. package/dist/core/RelationManager.js.map +0 -1
  46. package/dist/core/SQLiteStorage.d.ts +0 -354
  47. package/dist/core/SQLiteStorage.d.ts.map +0 -1
  48. package/dist/core/SQLiteStorage.js +0 -919
  49. package/dist/core/SQLiteStorage.js.map +0 -1
  50. package/dist/core/StorageFactory.d.ts +0 -45
  51. package/dist/core/StorageFactory.d.ts.map +0 -1
  52. package/dist/core/StorageFactory.js +0 -65
  53. package/dist/core/StorageFactory.js.map +0 -1
  54. package/dist/core/TransactionManager.d.ts +0 -464
  55. package/dist/core/TransactionManager.d.ts.map +0 -1
  56. package/dist/core/TransactionManager.js +0 -869
  57. package/dist/core/TransactionManager.js.map +0 -1
  58. package/dist/core/index.d.ts +0 -17
  59. package/dist/core/index.d.ts.map +0 -1
  60. package/dist/core/index.js +0 -20
  61. package/dist/core/index.js.map +0 -1
  62. package/dist/features/AnalyticsManager.d.ts +0 -44
  63. package/dist/features/AnalyticsManager.d.ts.map +0 -1
  64. package/dist/features/AnalyticsManager.js +0 -224
  65. package/dist/features/AnalyticsManager.js.map +0 -1
  66. package/dist/features/ArchiveManager.d.ts +0 -133
  67. package/dist/features/ArchiveManager.d.ts.map +0 -1
  68. package/dist/features/ArchiveManager.js +0 -282
  69. package/dist/features/ArchiveManager.js.map +0 -1
  70. package/dist/features/CompressionManager.d.ts +0 -119
  71. package/dist/features/CompressionManager.d.ts.map +0 -1
  72. package/dist/features/CompressionManager.js +0 -470
  73. package/dist/features/CompressionManager.js.map +0 -1
  74. package/dist/features/IOManager.d.ts +0 -225
  75. package/dist/features/IOManager.d.ts.map +0 -1
  76. package/dist/features/IOManager.js +0 -1093
  77. package/dist/features/IOManager.js.map +0 -1
  78. package/dist/features/KeywordExtractor.d.ts +0 -61
  79. package/dist/features/KeywordExtractor.d.ts.map +0 -1
  80. package/dist/features/KeywordExtractor.js +0 -127
  81. package/dist/features/KeywordExtractor.js.map +0 -1
  82. package/dist/features/ObservationNormalizer.d.ts +0 -90
  83. package/dist/features/ObservationNormalizer.d.ts.map +0 -1
  84. package/dist/features/ObservationNormalizer.js +0 -194
  85. package/dist/features/ObservationNormalizer.js.map +0 -1
  86. package/dist/features/StreamingExporter.d.ts +0 -128
  87. package/dist/features/StreamingExporter.d.ts.map +0 -1
  88. package/dist/features/StreamingExporter.js +0 -212
  89. package/dist/features/StreamingExporter.js.map +0 -1
  90. package/dist/features/TagManager.d.ts +0 -147
  91. package/dist/features/TagManager.d.ts.map +0 -1
  92. package/dist/features/TagManager.js +0 -211
  93. package/dist/features/TagManager.js.map +0 -1
  94. package/dist/features/index.d.ts +0 -14
  95. package/dist/features/index.d.ts.map +0 -1
  96. package/dist/features/index.js +0 -15
  97. package/dist/features/index.js.map +0 -1
  98. package/dist/index.d.ts.map +0 -1
  99. package/dist/search/BM25Search.d.ts +0 -148
  100. package/dist/search/BM25Search.d.ts.map +0 -1
  101. package/dist/search/BM25Search.js +0 -340
  102. package/dist/search/BM25Search.js.map +0 -1
  103. package/dist/search/BasicSearch.d.ts +0 -51
  104. package/dist/search/BasicSearch.d.ts.map +0 -1
  105. package/dist/search/BasicSearch.js +0 -138
  106. package/dist/search/BasicSearch.js.map +0 -1
  107. package/dist/search/BooleanSearch.d.ts +0 -98
  108. package/dist/search/BooleanSearch.d.ts.map +0 -1
  109. package/dist/search/BooleanSearch.js +0 -431
  110. package/dist/search/BooleanSearch.js.map +0 -1
  111. package/dist/search/EarlyTerminationManager.d.ts +0 -140
  112. package/dist/search/EarlyTerminationManager.d.ts.map +0 -1
  113. package/dist/search/EarlyTerminationManager.js +0 -280
  114. package/dist/search/EarlyTerminationManager.js.map +0 -1
  115. package/dist/search/EmbeddingCache.d.ts +0 -175
  116. package/dist/search/EmbeddingCache.d.ts.map +0 -1
  117. package/dist/search/EmbeddingCache.js +0 -247
  118. package/dist/search/EmbeddingCache.js.map +0 -1
  119. package/dist/search/EmbeddingService.d.ts +0 -277
  120. package/dist/search/EmbeddingService.d.ts.map +0 -1
  121. package/dist/search/EmbeddingService.js +0 -531
  122. package/dist/search/EmbeddingService.js.map +0 -1
  123. package/dist/search/FuzzySearch.d.ts +0 -118
  124. package/dist/search/FuzzySearch.d.ts.map +0 -1
  125. package/dist/search/FuzzySearch.js +0 -313
  126. package/dist/search/FuzzySearch.js.map +0 -1
  127. package/dist/search/HybridScorer.d.ts +0 -181
  128. package/dist/search/HybridScorer.d.ts.map +0 -1
  129. package/dist/search/HybridScorer.js +0 -258
  130. package/dist/search/HybridScorer.js.map +0 -1
  131. package/dist/search/HybridSearchManager.d.ts +0 -80
  132. package/dist/search/HybridSearchManager.d.ts.map +0 -1
  133. package/dist/search/HybridSearchManager.js +0 -188
  134. package/dist/search/HybridSearchManager.js.map +0 -1
  135. package/dist/search/IncrementalIndexer.d.ts +0 -201
  136. package/dist/search/IncrementalIndexer.d.ts.map +0 -1
  137. package/dist/search/IncrementalIndexer.js +0 -343
  138. package/dist/search/IncrementalIndexer.js.map +0 -1
  139. package/dist/search/OptimizedInvertedIndex.d.ts +0 -163
  140. package/dist/search/OptimizedInvertedIndex.d.ts.map +0 -1
  141. package/dist/search/OptimizedInvertedIndex.js +0 -359
  142. package/dist/search/OptimizedInvertedIndex.js.map +0 -1
  143. package/dist/search/ParallelSearchExecutor.d.ts +0 -172
  144. package/dist/search/ParallelSearchExecutor.d.ts.map +0 -1
  145. package/dist/search/ParallelSearchExecutor.js +0 -310
  146. package/dist/search/ParallelSearchExecutor.js.map +0 -1
  147. package/dist/search/QuantizedVectorStore.d.ts +0 -171
  148. package/dist/search/QuantizedVectorStore.d.ts.map +0 -1
  149. package/dist/search/QuantizedVectorStore.js +0 -308
  150. package/dist/search/QuantizedVectorStore.js.map +0 -1
  151. package/dist/search/QueryAnalyzer.d.ts +0 -76
  152. package/dist/search/QueryAnalyzer.d.ts.map +0 -1
  153. package/dist/search/QueryAnalyzer.js +0 -228
  154. package/dist/search/QueryAnalyzer.js.map +0 -1
  155. package/dist/search/QueryCostEstimator.d.ts +0 -244
  156. package/dist/search/QueryCostEstimator.d.ts.map +0 -1
  157. package/dist/search/QueryCostEstimator.js +0 -653
  158. package/dist/search/QueryCostEstimator.js.map +0 -1
  159. package/dist/search/QueryPlanCache.d.ts +0 -220
  160. package/dist/search/QueryPlanCache.d.ts.map +0 -1
  161. package/dist/search/QueryPlanCache.js +0 -380
  162. package/dist/search/QueryPlanCache.js.map +0 -1
  163. package/dist/search/QueryPlanner.d.ts +0 -58
  164. package/dist/search/QueryPlanner.d.ts.map +0 -1
  165. package/dist/search/QueryPlanner.js +0 -138
  166. package/dist/search/QueryPlanner.js.map +0 -1
  167. package/dist/search/RankedSearch.d.ts +0 -71
  168. package/dist/search/RankedSearch.d.ts.map +0 -1
  169. package/dist/search/RankedSearch.js +0 -239
  170. package/dist/search/RankedSearch.js.map +0 -1
  171. package/dist/search/ReflectionManager.d.ts +0 -120
  172. package/dist/search/ReflectionManager.d.ts.map +0 -1
  173. package/dist/search/ReflectionManager.js +0 -232
  174. package/dist/search/ReflectionManager.js.map +0 -1
  175. package/dist/search/SavedSearchManager.d.ts +0 -79
  176. package/dist/search/SavedSearchManager.d.ts.map +0 -1
  177. package/dist/search/SavedSearchManager.js +0 -147
  178. package/dist/search/SavedSearchManager.js.map +0 -1
  179. package/dist/search/SearchFilterChain.d.ts +0 -120
  180. package/dist/search/SearchFilterChain.d.ts.map +0 -1
  181. package/dist/search/SearchFilterChain.js +0 -186
  182. package/dist/search/SearchFilterChain.js.map +0 -1
  183. package/dist/search/SearchManager.d.ts +0 -326
  184. package/dist/search/SearchManager.d.ts.map +0 -1
  185. package/dist/search/SearchManager.js +0 -454
  186. package/dist/search/SearchManager.js.map +0 -1
  187. package/dist/search/SearchSuggestions.d.ts +0 -27
  188. package/dist/search/SearchSuggestions.d.ts.map +0 -1
  189. package/dist/search/SearchSuggestions.js +0 -58
  190. package/dist/search/SearchSuggestions.js.map +0 -1
  191. package/dist/search/SemanticSearch.d.ts +0 -149
  192. package/dist/search/SemanticSearch.d.ts.map +0 -1
  193. package/dist/search/SemanticSearch.js +0 -324
  194. package/dist/search/SemanticSearch.js.map +0 -1
  195. package/dist/search/SymbolicSearch.d.ts +0 -61
  196. package/dist/search/SymbolicSearch.d.ts.map +0 -1
  197. package/dist/search/SymbolicSearch.js +0 -164
  198. package/dist/search/SymbolicSearch.js.map +0 -1
  199. package/dist/search/TFIDFEventSync.d.ts +0 -85
  200. package/dist/search/TFIDFEventSync.d.ts.map +0 -1
  201. package/dist/search/TFIDFEventSync.js +0 -134
  202. package/dist/search/TFIDFEventSync.js.map +0 -1
  203. package/dist/search/TFIDFIndexManager.d.ts +0 -151
  204. package/dist/search/TFIDFIndexManager.d.ts.map +0 -1
  205. package/dist/search/TFIDFIndexManager.js +0 -433
  206. package/dist/search/TFIDFIndexManager.js.map +0 -1
  207. package/dist/search/VectorStore.d.ts +0 -235
  208. package/dist/search/VectorStore.d.ts.map +0 -1
  209. package/dist/search/VectorStore.js +0 -312
  210. package/dist/search/VectorStore.js.map +0 -1
  211. package/dist/search/index.d.ts +0 -35
  212. package/dist/search/index.d.ts.map +0 -1
  213. package/dist/search/index.js +0 -53
  214. package/dist/search/index.js.map +0 -1
  215. package/dist/types/index.d.ts +0 -13
  216. package/dist/types/index.d.ts.map +0 -1
  217. package/dist/types/index.js +0 -13
  218. package/dist/types/index.js.map +0 -1
  219. package/dist/types/types.d.ts +0 -1811
  220. package/dist/types/types.d.ts.map +0 -1
  221. package/dist/types/types.js +0 -10
  222. package/dist/types/types.js.map +0 -1
  223. package/dist/utils/BatchProcessor.d.ts +0 -271
  224. package/dist/utils/BatchProcessor.d.ts.map +0 -1
  225. package/dist/utils/BatchProcessor.js +0 -377
  226. package/dist/utils/BatchProcessor.js.map +0 -1
  227. package/dist/utils/MemoryMonitor.d.ts +0 -176
  228. package/dist/utils/MemoryMonitor.d.ts.map +0 -1
  229. package/dist/utils/MemoryMonitor.js +0 -306
  230. package/dist/utils/MemoryMonitor.js.map +0 -1
  231. package/dist/utils/WorkerPoolManager.d.ts +0 -233
  232. package/dist/utils/WorkerPoolManager.d.ts.map +0 -1
  233. package/dist/utils/WorkerPoolManager.js +0 -421
  234. package/dist/utils/WorkerPoolManager.js.map +0 -1
  235. package/dist/utils/compressedCache.d.ts +0 -221
  236. package/dist/utils/compressedCache.d.ts.map +0 -1
  237. package/dist/utils/compressedCache.js +0 -349
  238. package/dist/utils/compressedCache.js.map +0 -1
  239. package/dist/utils/compressionUtil.d.ts +0 -214
  240. package/dist/utils/compressionUtil.d.ts.map +0 -1
  241. package/dist/utils/compressionUtil.js +0 -248
  242. package/dist/utils/compressionUtil.js.map +0 -1
  243. package/dist/utils/constants.d.ts +0 -245
  244. package/dist/utils/constants.d.ts.map +0 -1
  245. package/dist/utils/constants.js +0 -253
  246. package/dist/utils/constants.js.map +0 -1
  247. package/dist/utils/entityUtils.d.ts +0 -379
  248. package/dist/utils/entityUtils.d.ts.map +0 -1
  249. package/dist/utils/entityUtils.js +0 -649
  250. package/dist/utils/entityUtils.js.map +0 -1
  251. package/dist/utils/errors.d.ts +0 -95
  252. package/dist/utils/errors.d.ts.map +0 -1
  253. package/dist/utils/errors.js +0 -146
  254. package/dist/utils/errors.js.map +0 -1
  255. package/dist/utils/formatters.d.ts +0 -145
  256. package/dist/utils/formatters.d.ts.map +0 -1
  257. package/dist/utils/formatters.js +0 -133
  258. package/dist/utils/formatters.js.map +0 -1
  259. package/dist/utils/index.d.ts +0 -26
  260. package/dist/utils/index.d.ts.map +0 -1
  261. package/dist/utils/index.js +0 -88
  262. package/dist/utils/index.js.map +0 -1
  263. package/dist/utils/indexes.d.ts +0 -270
  264. package/dist/utils/indexes.d.ts.map +0 -1
  265. package/dist/utils/indexes.js +0 -527
  266. package/dist/utils/indexes.js.map +0 -1
  267. package/dist/utils/logger.d.ts +0 -31
  268. package/dist/utils/logger.d.ts.map +0 -1
  269. package/dist/utils/logger.js +0 -41
  270. package/dist/utils/logger.js.map +0 -1
  271. package/dist/utils/operationUtils.d.ts +0 -124
  272. package/dist/utils/operationUtils.d.ts.map +0 -1
  273. package/dist/utils/operationUtils.js +0 -176
  274. package/dist/utils/operationUtils.js.map +0 -1
  275. package/dist/utils/parallelUtils.d.ts +0 -76
  276. package/dist/utils/parallelUtils.d.ts.map +0 -1
  277. package/dist/utils/parallelUtils.js +0 -192
  278. package/dist/utils/parallelUtils.js.map +0 -1
  279. package/dist/utils/schemas.d.ts +0 -556
  280. package/dist/utils/schemas.d.ts.map +0 -1
  281. package/dist/utils/schemas.js +0 -485
  282. package/dist/utils/schemas.js.map +0 -1
  283. package/dist/utils/searchAlgorithms.d.ts +0 -99
  284. package/dist/utils/searchAlgorithms.d.ts.map +0 -1
  285. package/dist/utils/searchAlgorithms.js +0 -168
  286. package/dist/utils/searchAlgorithms.js.map +0 -1
  287. package/dist/utils/searchCache.d.ts +0 -108
  288. package/dist/utils/searchCache.d.ts.map +0 -1
  289. package/dist/utils/searchCache.js +0 -210
  290. package/dist/utils/searchCache.js.map +0 -1
  291. package/dist/utils/taskScheduler.d.ts +0 -294
  292. package/dist/utils/taskScheduler.d.ts.map +0 -1
  293. package/dist/utils/taskScheduler.js +0 -487
  294. package/dist/utils/taskScheduler.js.map +0 -1
  295. package/dist/workers/index.d.ts +0 -12
  296. package/dist/workers/index.d.ts.map +0 -1
  297. package/dist/workers/index.js +0 -10
  298. package/dist/workers/index.js.map +0 -1
  299. package/dist/workers/levenshteinWorker.d.ts +0 -60
  300. package/dist/workers/levenshteinWorker.d.ts.map +0 -1
@@ -1,1093 +0,0 @@
1
- /**
2
- * IO Manager
3
- *
4
- * Unified manager for import, export, and backup operations.
5
- * Consolidates BackupManager, ExportManager, and ImportManager (Sprint 11.4).
6
- *
7
- * @module features/IOManager
8
- */
9
- import { promises as fs } from 'fs';
10
- import { dirname, join } from 'path';
11
- import { FileOperationError } from '../utils/errors.js';
12
- import { compress, decompress, hasBrotliExtension, COMPRESSION_CONFIG, STREAMING_CONFIG, checkCancellation, createProgressReporter, createProgress, validateFilePath, sanitizeObject, escapeCsvFormula, } from '../utils/index.js';
13
- import { StreamingExporter } from './StreamingExporter.js';
14
- // ============================================================
15
- // IO MANAGER CLASS
16
- // ============================================================
17
- /**
18
- * Unified manager for import, export, and backup operations.
19
- *
20
- * Combines functionality from:
21
- * - ExportManager: Graph export to various formats
22
- * - ImportManager: Graph import from various formats
23
- * - BackupManager: Point-in-time backup and restore
24
- */
25
- export class IOManager {
26
- storage;
27
- backupDir;
28
- constructor(storage) {
29
- this.storage = storage;
30
- const filePath = this.storage.getFilePath();
31
- const dir = dirname(filePath);
32
- this.backupDir = join(dir, '.backups');
33
- }
34
- // ============================================================
35
- // EXPORT OPERATIONS
36
- // ============================================================
37
- /**
38
- * Export graph to specified format.
39
- *
40
- * @param graph - Knowledge graph to export
41
- * @param format - Export format
42
- * @returns Formatted export string
43
- */
44
- exportGraph(graph, format) {
45
- switch (format) {
46
- case 'json':
47
- return this.exportAsJson(graph);
48
- case 'csv':
49
- return this.exportAsCsv(graph);
50
- case 'graphml':
51
- return this.exportAsGraphML(graph);
52
- case 'gexf':
53
- return this.exportAsGEXF(graph);
54
- case 'dot':
55
- return this.exportAsDOT(graph);
56
- case 'markdown':
57
- return this.exportAsMarkdown(graph);
58
- case 'mermaid':
59
- return this.exportAsMermaid(graph);
60
- default:
61
- throw new Error(`Unsupported export format: ${format}`);
62
- }
63
- }
64
- /**
65
- * Export graph with optional brotli compression.
66
- *
67
- * Compression is applied when:
68
- * - `options.compress` is explicitly set to `true`
69
- * - The exported content exceeds 100KB (auto-compress threshold)
70
- *
71
- * Compressed content is returned as base64-encoded string.
72
- * Uncompressed content is returned as UTF-8 string.
73
- *
74
- * @param graph - Knowledge graph to export
75
- * @param format - Export format
76
- * @param options - Export options including compression settings
77
- * @returns Export result with content and compression metadata
78
- *
79
- * @example
80
- * ```typescript
81
- * // Export with explicit compression
82
- * const result = await manager.exportGraphWithCompression(graph, 'json', {
83
- * compress: true,
84
- * compressionQuality: 11
85
- * });
86
- *
87
- * // Export with auto-compression for large graphs
88
- * const result = await manager.exportGraphWithCompression(graph, 'json');
89
- * // Compresses automatically if content > 100KB
90
- * ```
91
- */
92
- async exportGraphWithCompression(graph, format, options) {
93
- // Check if streaming should be used
94
- const shouldStream = options?.streaming ||
95
- (options?.outputPath && graph.entities.length >= STREAMING_CONFIG.STREAMING_THRESHOLD);
96
- if (shouldStream && options?.outputPath) {
97
- return this.streamExport(format, graph, options);
98
- }
99
- // Generate export content using existing method
100
- const content = this.exportGraph(graph, format);
101
- const originalSize = Buffer.byteLength(content, 'utf-8');
102
- // Determine if compression should be applied
103
- const shouldCompress = options?.compress === true ||
104
- (options?.compress !== false &&
105
- originalSize > COMPRESSION_CONFIG.AUTO_COMPRESS_EXPORT_SIZE);
106
- if (shouldCompress) {
107
- const quality = options?.compressionQuality ?? COMPRESSION_CONFIG.BROTLI_QUALITY_BATCH;
108
- const compressionResult = await compress(content, {
109
- quality,
110
- mode: 'text',
111
- });
112
- return {
113
- format,
114
- content: compressionResult.compressed.toString('base64'),
115
- entityCount: graph.entities.length,
116
- relationCount: graph.relations.length,
117
- compressed: true,
118
- encoding: 'base64',
119
- originalSize,
120
- compressedSize: compressionResult.compressedSize,
121
- compressionRatio: compressionResult.ratio,
122
- };
123
- }
124
- // Return uncompressed content
125
- return {
126
- format,
127
- content,
128
- entityCount: graph.entities.length,
129
- relationCount: graph.relations.length,
130
- compressed: false,
131
- encoding: 'utf-8',
132
- originalSize,
133
- compressedSize: originalSize,
134
- compressionRatio: 1,
135
- };
136
- }
137
- /**
138
- * Stream export to a file for large graphs.
139
- *
140
- * Uses StreamingExporter to write entities and relations incrementally
141
- * to avoid loading the entire export content into memory.
142
- *
143
- * @param format - Export format
144
- * @param graph - Knowledge graph to export
145
- * @param options - Export options with required outputPath
146
- * @returns Export result with streaming metadata
147
- * @private
148
- */
149
- async streamExport(format, graph, options) {
150
- // Validate path to prevent path traversal attacks (defense in depth)
151
- const validatedOutputPath = validateFilePath(options.outputPath);
152
- const exporter = new StreamingExporter(validatedOutputPath);
153
- let result;
154
- switch (format) {
155
- case 'json':
156
- // Use JSONL format for streaming (line-delimited JSON)
157
- result = await exporter.streamJSONL(graph);
158
- break;
159
- case 'csv':
160
- result = await exporter.streamCSV(graph);
161
- break;
162
- default:
163
- // Fallback to in-memory export for unsupported streaming formats
164
- const content = this.exportGraph(graph, format);
165
- await fs.writeFile(validatedOutputPath, content);
166
- result = {
167
- bytesWritten: Buffer.byteLength(content, 'utf-8'),
168
- entitiesWritten: graph.entities.length,
169
- relationsWritten: graph.relations.length,
170
- durationMs: 0,
171
- };
172
- }
173
- return {
174
- format,
175
- content: `Streamed to ${validatedOutputPath}`,
176
- entityCount: result.entitiesWritten,
177
- relationCount: result.relationsWritten,
178
- compressed: false,
179
- encoding: 'utf-8',
180
- originalSize: result.bytesWritten,
181
- compressedSize: result.bytesWritten,
182
- compressionRatio: 1,
183
- streamed: true,
184
- outputPath: validatedOutputPath,
185
- };
186
- }
187
- exportAsJson(graph) {
188
- return JSON.stringify(graph, null, 2);
189
- }
190
- exportAsCsv(graph) {
191
- const lines = [];
192
- const escapeCsvField = (field) => {
193
- if (field === undefined || field === null)
194
- return '';
195
- // First protect against CSV formula injection
196
- let str = escapeCsvFormula(String(field));
197
- // Then handle CSV special characters
198
- if (str.includes(',') || str.includes('"') || str.includes('\n')) {
199
- return `"${str.replace(/"/g, '""')}"`;
200
- }
201
- return str;
202
- };
203
- lines.push('# ENTITIES');
204
- lines.push('name,entityType,observations,createdAt,lastModified,tags,importance');
205
- for (const entity of graph.entities) {
206
- const observationsStr = entity.observations.join('; ');
207
- const tagsStr = entity.tags ? entity.tags.join('; ') : '';
208
- const importanceStr = entity.importance !== undefined ? String(entity.importance) : '';
209
- lines.push([
210
- escapeCsvField(entity.name),
211
- escapeCsvField(entity.entityType),
212
- escapeCsvField(observationsStr),
213
- escapeCsvField(entity.createdAt),
214
- escapeCsvField(entity.lastModified),
215
- escapeCsvField(tagsStr),
216
- escapeCsvField(importanceStr),
217
- ].join(','));
218
- }
219
- lines.push('');
220
- lines.push('# RELATIONS');
221
- lines.push('from,to,relationType,createdAt,lastModified');
222
- for (const relation of graph.relations) {
223
- lines.push([
224
- escapeCsvField(relation.from),
225
- escapeCsvField(relation.to),
226
- escapeCsvField(relation.relationType),
227
- escapeCsvField(relation.createdAt),
228
- escapeCsvField(relation.lastModified),
229
- ].join(','));
230
- }
231
- return lines.join('\n');
232
- }
233
- exportAsGraphML(graph) {
234
- const lines = [];
235
- const escapeXml = (str) => {
236
- if (str === undefined || str === null)
237
- return '';
238
- return String(str)
239
- .replace(/&/g, '&')
240
- .replace(/</g, '&lt;')
241
- .replace(/>/g, '&gt;')
242
- .replace(/"/g, '&quot;')
243
- .replace(/'/g, '&apos;');
244
- };
245
- lines.push('<?xml version="1.0" encoding="UTF-8"?>');
246
- lines.push('<graphml xmlns="http://graphml.graphdrawing.org/xmlns">');
247
- lines.push(' <key id="d0" for="node" attr.name="entityType" attr.type="string"/>');
248
- lines.push(' <key id="d1" for="node" attr.name="observations" attr.type="string"/>');
249
- lines.push(' <key id="d2" for="node" attr.name="createdAt" attr.type="string"/>');
250
- lines.push(' <key id="d3" for="node" attr.name="lastModified" attr.type="string"/>');
251
- lines.push(' <key id="d4" for="node" attr.name="tags" attr.type="string"/>');
252
- lines.push(' <key id="d5" for="node" attr.name="importance" attr.type="double"/>');
253
- lines.push(' <key id="e0" for="edge" attr.name="relationType" attr.type="string"/>');
254
- lines.push(' <key id="e1" for="edge" attr.name="createdAt" attr.type="string"/>');
255
- lines.push(' <key id="e2" for="edge" attr.name="lastModified" attr.type="string"/>');
256
- lines.push(' <graph id="G" edgedefault="directed">');
257
- for (const entity of graph.entities) {
258
- const nodeId = escapeXml(entity.name);
259
- lines.push(` <node id="${nodeId}">`);
260
- lines.push(` <data key="d0">${escapeXml(entity.entityType)}</data>`);
261
- lines.push(` <data key="d1">${escapeXml(entity.observations.join('; '))}</data>`);
262
- if (entity.createdAt)
263
- lines.push(` <data key="d2">${escapeXml(entity.createdAt)}</data>`);
264
- if (entity.lastModified)
265
- lines.push(` <data key="d3">${escapeXml(entity.lastModified)}</data>`);
266
- if (entity.tags?.length)
267
- lines.push(` <data key="d4">${escapeXml(entity.tags.join('; '))}</data>`);
268
- if (entity.importance !== undefined)
269
- lines.push(` <data key="d5">${entity.importance}</data>`);
270
- lines.push(' </node>');
271
- }
272
- let edgeId = 0;
273
- for (const relation of graph.relations) {
274
- const sourceId = escapeXml(relation.from);
275
- const targetId = escapeXml(relation.to);
276
- lines.push(` <edge id="e${edgeId}" source="${sourceId}" target="${targetId}">`);
277
- lines.push(` <data key="e0">${escapeXml(relation.relationType)}</data>`);
278
- if (relation.createdAt)
279
- lines.push(` <data key="e1">${escapeXml(relation.createdAt)}</data>`);
280
- if (relation.lastModified)
281
- lines.push(` <data key="e2">${escapeXml(relation.lastModified)}</data>`);
282
- lines.push(' </edge>');
283
- edgeId++;
284
- }
285
- lines.push(' </graph>');
286
- lines.push('</graphml>');
287
- return lines.join('\n');
288
- }
289
- exportAsGEXF(graph) {
290
- const lines = [];
291
- const escapeXml = (str) => {
292
- if (str === undefined || str === null)
293
- return '';
294
- return String(str)
295
- .replace(/&/g, '&amp;')
296
- .replace(/</g, '&lt;')
297
- .replace(/>/g, '&gt;')
298
- .replace(/"/g, '&quot;')
299
- .replace(/'/g, '&apos;');
300
- };
301
- lines.push('<?xml version="1.0" encoding="UTF-8"?>');
302
- lines.push('<gexf xmlns="http://www.gexf.net/1.2draft" version="1.2">');
303
- lines.push(' <meta>');
304
- lines.push(' <creator>Memory MCP Server</creator>');
305
- lines.push(' </meta>');
306
- lines.push(' <graph mode="static" defaultedgetype="directed">');
307
- lines.push(' <attributes class="node">');
308
- lines.push(' <attribute id="0" title="entityType" type="string"/>');
309
- lines.push(' <attribute id="1" title="observations" type="string"/>');
310
- lines.push(' </attributes>');
311
- lines.push(' <nodes>');
312
- for (const entity of graph.entities) {
313
- const nodeId = escapeXml(entity.name);
314
- lines.push(` <node id="${nodeId}" label="${nodeId}">`);
315
- lines.push(' <attvalues>');
316
- lines.push(` <attvalue for="0" value="${escapeXml(entity.entityType)}"/>`);
317
- lines.push(` <attvalue for="1" value="${escapeXml(entity.observations.join('; '))}"/>`);
318
- lines.push(' </attvalues>');
319
- lines.push(' </node>');
320
- }
321
- lines.push(' </nodes>');
322
- lines.push(' <edges>');
323
- let edgeId = 0;
324
- for (const relation of graph.relations) {
325
- const sourceId = escapeXml(relation.from);
326
- const targetId = escapeXml(relation.to);
327
- const label = escapeXml(relation.relationType);
328
- lines.push(` <edge id="${edgeId}" source="${sourceId}" target="${targetId}" label="${label}"/>`);
329
- edgeId++;
330
- }
331
- lines.push(' </edges>');
332
- lines.push(' </graph>');
333
- lines.push('</gexf>');
334
- return lines.join('\n');
335
- }
336
- exportAsDOT(graph) {
337
- const lines = [];
338
- const escapeDot = (str) => {
339
- return '"' + str.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n') + '"';
340
- };
341
- lines.push('digraph KnowledgeGraph {');
342
- lines.push(' rankdir=LR;');
343
- lines.push(' node [shape=box, style=rounded];');
344
- lines.push('');
345
- for (const entity of graph.entities) {
346
- const nodeId = escapeDot(entity.name);
347
- const label = [`${entity.name}`, `Type: ${entity.entityType}`];
348
- if (entity.tags?.length)
349
- label.push(`Tags: ${entity.tags.join(', ')}`);
350
- const labelStr = escapeDot(label.join('\\n'));
351
- lines.push(` ${nodeId} [label=${labelStr}];`);
352
- }
353
- lines.push('');
354
- for (const relation of graph.relations) {
355
- const fromId = escapeDot(relation.from);
356
- const toId = escapeDot(relation.to);
357
- const label = escapeDot(relation.relationType);
358
- lines.push(` ${fromId} -> ${toId} [label=${label}];`);
359
- }
360
- lines.push('}');
361
- return lines.join('\n');
362
- }
363
- exportAsMarkdown(graph) {
364
- const lines = [];
365
- lines.push('# Knowledge Graph Export');
366
- lines.push('');
367
- lines.push(`**Exported:** ${new Date().toISOString()}`);
368
- lines.push(`**Entities:** ${graph.entities.length}`);
369
- lines.push(`**Relations:** ${graph.relations.length}`);
370
- lines.push('');
371
- lines.push('## Entities');
372
- lines.push('');
373
- for (const entity of graph.entities) {
374
- lines.push(`### ${entity.name}`);
375
- lines.push('');
376
- lines.push(`- **Type:** ${entity.entityType}`);
377
- if (entity.tags?.length)
378
- lines.push(`- **Tags:** ${entity.tags.map(t => `\`${t}\``).join(', ')}`);
379
- if (entity.importance !== undefined)
380
- lines.push(`- **Importance:** ${entity.importance}/10`);
381
- if (entity.observations.length > 0) {
382
- lines.push('');
383
- lines.push('**Observations:**');
384
- for (const obs of entity.observations) {
385
- lines.push(`- ${obs}`);
386
- }
387
- }
388
- lines.push('');
389
- }
390
- if (graph.relations.length > 0) {
391
- lines.push('## Relations');
392
- lines.push('');
393
- for (const relation of graph.relations) {
394
- lines.push(`- **${relation.from}** → *${relation.relationType}* → **${relation.to}**`);
395
- }
396
- lines.push('');
397
- }
398
- return lines.join('\n');
399
- }
400
- exportAsMermaid(graph) {
401
- const lines = [];
402
- const sanitizeId = (str) => str.replace(/[^a-zA-Z0-9_]/g, '_');
403
- const escapeLabel = (str) => str.replace(/"/g, '#quot;');
404
- lines.push('graph LR');
405
- lines.push(' %% Knowledge Graph');
406
- lines.push('');
407
- const nodeIds = new Map();
408
- for (const entity of graph.entities) {
409
- nodeIds.set(entity.name, sanitizeId(entity.name));
410
- }
411
- for (const entity of graph.entities) {
412
- const nodeId = nodeIds.get(entity.name);
413
- const labelParts = [entity.name, `Type: ${entity.entityType}`];
414
- if (entity.tags?.length)
415
- labelParts.push(`Tags: ${entity.tags.join(', ')}`);
416
- const label = escapeLabel(labelParts.join('<br/>'));
417
- lines.push(` ${nodeId}["${label}"]`);
418
- }
419
- lines.push('');
420
- for (const relation of graph.relations) {
421
- const fromId = nodeIds.get(relation.from);
422
- const toId = nodeIds.get(relation.to);
423
- if (fromId && toId) {
424
- const label = escapeLabel(relation.relationType);
425
- lines.push(` ${fromId} -->|"${label}"| ${toId}`);
426
- }
427
- }
428
- return lines.join('\n');
429
- }
430
- // ============================================================
431
- // IMPORT OPERATIONS
432
- // ============================================================
433
- /**
434
- * Import graph from formatted data.
435
- *
436
- * Phase 9B: Supports progress tracking and cancellation via LongRunningOperationOptions.
437
- *
438
- * @param format - Import format
439
- * @param data - Import data string
440
- * @param mergeStrategy - How to handle conflicts
441
- * @param dryRun - If true, preview changes without applying
442
- * @param options - Optional progress/cancellation options (Phase 9B)
443
- * @returns Import result with statistics
444
- * @throws {OperationCancelledError} If operation is cancelled via signal (Phase 9B)
445
- */
446
- async importGraph(format, data, mergeStrategy = 'skip', dryRun = false, options) {
447
- // Check for early cancellation
448
- checkCancellation(options?.signal, 'importGraph');
449
- // Setup progress reporter
450
- const reportProgress = createProgressReporter(options?.onProgress);
451
- reportProgress?.(createProgress(0, 100, 'importGraph'));
452
- let importedGraph;
453
- try {
454
- // Parsing phase (0-20% progress)
455
- reportProgress?.(createProgress(5, 100, 'parsing data'));
456
- checkCancellation(options?.signal, 'importGraph');
457
- switch (format) {
458
- case 'json':
459
- importedGraph = this.parseJsonImport(data);
460
- break;
461
- case 'csv':
462
- importedGraph = this.parseCsvImport(data);
463
- break;
464
- case 'graphml':
465
- importedGraph = this.parseGraphMLImport(data);
466
- break;
467
- default:
468
- throw new Error(`Unsupported import format: ${format}`);
469
- }
470
- reportProgress?.(createProgress(20, 100, 'parsing complete'));
471
- }
472
- catch (error) {
473
- return {
474
- entitiesAdded: 0,
475
- entitiesSkipped: 0,
476
- entitiesUpdated: 0,
477
- relationsAdded: 0,
478
- relationsSkipped: 0,
479
- errors: [`Failed to parse ${format} data: ${error instanceof Error ? error.message : String(error)}`],
480
- };
481
- }
482
- // Merging phase (20-100% progress)
483
- return await this.mergeImportedGraph(importedGraph, mergeStrategy, dryRun, options);
484
- }
485
- parseJsonImport(data) {
486
- // Security: Limit input size to prevent DoS (10MB max)
487
- const MAX_IMPORT_SIZE = 10 * 1024 * 1024;
488
- if (data.length > MAX_IMPORT_SIZE) {
489
- throw new FileOperationError(`JSON import data exceeds maximum size of ${MAX_IMPORT_SIZE / (1024 * 1024)}MB`, 'json-import');
490
- }
491
- const parsed = JSON.parse(data);
492
- if (!parsed.entities || !Array.isArray(parsed.entities)) {
493
- throw new Error('Invalid JSON: missing or invalid entities array');
494
- }
495
- if (!parsed.relations || !Array.isArray(parsed.relations)) {
496
- throw new Error('Invalid JSON: missing or invalid relations array');
497
- }
498
- // Security: Limit maximum number of entities/relations
499
- const MAX_ITEMS = 100000;
500
- if (parsed.entities.length > MAX_ITEMS) {
501
- throw new FileOperationError(`JSON import exceeds maximum entity count of ${MAX_ITEMS}`, 'json-import');
502
- }
503
- if (parsed.relations.length > MAX_ITEMS) {
504
- throw new FileOperationError(`JSON import exceeds maximum relation count of ${MAX_ITEMS}`, 'json-import');
505
- }
506
- return {
507
- entities: parsed.entities,
508
- relations: parsed.relations,
509
- };
510
- }
511
- parseCsvImport(data) {
512
- // Security: Limit input size to prevent DoS (10MB max)
513
- const MAX_IMPORT_SIZE = 10 * 1024 * 1024;
514
- if (data.length > MAX_IMPORT_SIZE) {
515
- throw new FileOperationError(`CSV import data exceeds maximum size of ${MAX_IMPORT_SIZE / (1024 * 1024)}MB`, 'csv-import');
516
- }
517
- // Security: Limit maximum number of entities/relations
518
- const MAX_ITEMS = 100000;
519
- const lines = data
520
- .split('\n')
521
- .map(line => line.trim())
522
- .filter(line => line);
523
- const entities = [];
524
- const relations = [];
525
- let section = null;
526
- let headerParsed = false;
527
- const parseCsvLine = (line) => {
528
- const fields = [];
529
- let current = '';
530
- let inQuotes = false;
531
- for (let i = 0; i < line.length; i++) {
532
- const char = line[i];
533
- if (char === '"') {
534
- if (inQuotes && line[i + 1] === '"') {
535
- current += '"';
536
- i++;
537
- }
538
- else {
539
- inQuotes = !inQuotes;
540
- }
541
- }
542
- else if (char === ',' && !inQuotes) {
543
- fields.push(current);
544
- current = '';
545
- }
546
- else {
547
- current += char;
548
- }
549
- }
550
- fields.push(current);
551
- return fields;
552
- };
553
- for (const line of lines) {
554
- if (line.startsWith('# ENTITIES')) {
555
- section = 'entities';
556
- headerParsed = false;
557
- continue;
558
- }
559
- else if (line.startsWith('# RELATIONS')) {
560
- section = 'relations';
561
- headerParsed = false;
562
- continue;
563
- }
564
- if (line.startsWith('#'))
565
- continue;
566
- if (section === 'entities') {
567
- if (!headerParsed) {
568
- headerParsed = true;
569
- continue;
570
- }
571
- const fields = parseCsvLine(line);
572
- if (fields.length >= 2) {
573
- // Security: Check entity limit
574
- if (entities.length >= MAX_ITEMS) {
575
- throw new FileOperationError(`CSV import exceeds maximum entity count of ${MAX_ITEMS}`, 'csv-import');
576
- }
577
- const entity = {
578
- name: fields[0],
579
- entityType: fields[1],
580
- observations: fields[2]
581
- ? fields[2]
582
- .split(';')
583
- .map(s => s.trim())
584
- .filter(s => s)
585
- : [],
586
- createdAt: fields[3] || undefined,
587
- lastModified: fields[4] || undefined,
588
- tags: fields[5]
589
- ? fields[5]
590
- .split(';')
591
- .map(s => s.trim().toLowerCase())
592
- .filter(s => s)
593
- : undefined,
594
- importance: fields[6] ? parseFloat(fields[6]) : undefined,
595
- };
596
- entities.push(entity);
597
- }
598
- }
599
- else if (section === 'relations') {
600
- if (!headerParsed) {
601
- headerParsed = true;
602
- continue;
603
- }
604
- const fields = parseCsvLine(line);
605
- if (fields.length >= 3) {
606
- // Security: Check relation limit
607
- if (relations.length >= MAX_ITEMS) {
608
- throw new FileOperationError(`CSV import exceeds maximum relation count of ${MAX_ITEMS}`, 'csv-import');
609
- }
610
- const relation = {
611
- from: fields[0],
612
- to: fields[1],
613
- relationType: fields[2],
614
- createdAt: fields[3] || undefined,
615
- lastModified: fields[4] || undefined,
616
- };
617
- relations.push(relation);
618
- }
619
- }
620
- }
621
- return { entities, relations };
622
- }
623
- parseGraphMLImport(data) {
624
- const entities = [];
625
- const relations = [];
626
- // Security: Limit input size to prevent ReDoS attacks (10MB max)
627
- const MAX_IMPORT_SIZE = 10 * 1024 * 1024;
628
- if (data.length > MAX_IMPORT_SIZE) {
629
- throw new FileOperationError(`GraphML import data exceeds maximum size of ${MAX_IMPORT_SIZE / (1024 * 1024)}MB`, 'graphml-import');
630
- }
631
- // Security: Limit maximum number of entities/relations to prevent infinite loops
632
- const MAX_ITEMS = 100000;
633
- let nodeCount = 0;
634
- let relationCount = 0;
635
- // Use non-greedy patterns with character class restrictions
636
- const nodeRegex = /<node\s+id="([^"]+)"[^>]*>([\s\S]*?)<\/node>/g;
637
- let nodeMatch;
638
- while ((nodeMatch = nodeRegex.exec(data)) !== null) {
639
- // Security: Limit iterations to prevent ReDoS
640
- if (++nodeCount > MAX_ITEMS) {
641
- throw new FileOperationError(`GraphML import exceeds maximum entity count of ${MAX_ITEMS}`, 'graphml-import');
642
- }
643
- const nodeId = nodeMatch[1];
644
- const nodeContent = nodeMatch[2];
645
- const getDataValue = (key) => {
646
- const dataRegex = new RegExp(`<data\\s+key="${key}">([^<]*)<\/data>`);
647
- const match = dataRegex.exec(nodeContent);
648
- return match ? match[1] : undefined;
649
- };
650
- const entity = {
651
- name: nodeId,
652
- entityType: getDataValue('d0') || getDataValue('entityType') || 'unknown',
653
- observations: (getDataValue('d1') || getDataValue('observations') || '')
654
- .split(';')
655
- .map(s => s.trim())
656
- .filter(s => s),
657
- createdAt: getDataValue('d2') || getDataValue('createdAt'),
658
- lastModified: getDataValue('d3') || getDataValue('lastModified'),
659
- tags: (getDataValue('d4') || getDataValue('tags') || '')
660
- .split(';')
661
- .map(s => s.trim().toLowerCase())
662
- .filter(s => s),
663
- importance: getDataValue('d5') || getDataValue('importance') ? parseFloat(getDataValue('d5') || getDataValue('importance') || '0') : undefined,
664
- };
665
- entities.push(entity);
666
- }
667
- const edgeRegex = /<edge\s+[^>]*source="([^"]+)"\s+target="([^"]+)"[^>]*>([\s\S]*?)<\/edge>/g;
668
- let edgeMatch;
669
- while ((edgeMatch = edgeRegex.exec(data)) !== null) {
670
- // Security: Limit iterations to prevent ReDoS
671
- if (++relationCount > MAX_ITEMS) {
672
- throw new FileOperationError(`GraphML import exceeds maximum relation count of ${MAX_ITEMS}`, 'graphml-import');
673
- }
674
- const source = edgeMatch[1];
675
- const target = edgeMatch[2];
676
- const edgeContent = edgeMatch[3];
677
- const getDataValue = (key) => {
678
- const dataRegex = new RegExp(`<data\\s+key="${key}">([^<]*)<\/data>`);
679
- const match = dataRegex.exec(edgeContent);
680
- return match ? match[1] : undefined;
681
- };
682
- const relation = {
683
- from: source,
684
- to: target,
685
- relationType: getDataValue('e0') || getDataValue('relationType') || 'related_to',
686
- createdAt: getDataValue('e1') || getDataValue('createdAt'),
687
- lastModified: getDataValue('e2') || getDataValue('lastModified'),
688
- };
689
- relations.push(relation);
690
- }
691
- return { entities, relations };
692
- }
693
- async mergeImportedGraph(importedGraph, mergeStrategy, dryRun, options) {
694
- // Check for cancellation
695
- checkCancellation(options?.signal, 'importGraph');
696
- // Setup progress reporter (we're at 20% from parsing, need to go to 100%)
697
- const reportProgress = createProgressReporter(options?.onProgress);
698
- const existingGraph = await this.storage.getGraphForMutation();
699
- const result = {
700
- entitiesAdded: 0,
701
- entitiesSkipped: 0,
702
- entitiesUpdated: 0,
703
- relationsAdded: 0,
704
- relationsSkipped: 0,
705
- errors: [],
706
- };
707
- const existingEntitiesMap = new Map();
708
- for (const entity of existingGraph.entities) {
709
- existingEntitiesMap.set(entity.name, entity);
710
- }
711
- const existingRelationsSet = new Set();
712
- for (const relation of existingGraph.relations) {
713
- existingRelationsSet.add(`${relation.from}|${relation.to}|${relation.relationType}`);
714
- }
715
- // Process entities (20-60% progress)
716
- const totalEntities = importedGraph.entities.length;
717
- const totalRelations = importedGraph.relations.length;
718
- let processedEntities = 0;
719
- for (const importedEntity of importedGraph.entities) {
720
- // Check for cancellation periodically
721
- checkCancellation(options?.signal, 'importGraph');
722
- const existing = existingEntitiesMap.get(importedEntity.name);
723
- if (!existing) {
724
- result.entitiesAdded++;
725
- if (!dryRun) {
726
- existingGraph.entities.push(importedEntity);
727
- existingEntitiesMap.set(importedEntity.name, importedEntity);
728
- }
729
- }
730
- else {
731
- switch (mergeStrategy) {
732
- case 'replace':
733
- result.entitiesUpdated++;
734
- if (!dryRun) {
735
- // Sanitize imported entity to prevent prototype pollution
736
- Object.assign(existing, sanitizeObject(importedEntity));
737
- }
738
- break;
739
- case 'skip':
740
- result.entitiesSkipped++;
741
- break;
742
- case 'merge':
743
- result.entitiesUpdated++;
744
- if (!dryRun) {
745
- existing.observations = [
746
- ...new Set([...existing.observations, ...importedEntity.observations]),
747
- ];
748
- if (importedEntity.tags) {
749
- existing.tags = existing.tags || [];
750
- existing.tags = [...new Set([...existing.tags, ...importedEntity.tags])];
751
- }
752
- if (importedEntity.importance !== undefined) {
753
- existing.importance = importedEntity.importance;
754
- }
755
- existing.lastModified = new Date().toISOString();
756
- }
757
- break;
758
- case 'fail':
759
- result.errors.push(`Entity "${importedEntity.name}" already exists`);
760
- break;
761
- }
762
- }
763
- processedEntities++;
764
- // Map entity progress (0-100%) to overall progress (20-60%)
765
- const entityProgress = totalEntities > 0 ? Math.round(20 + (processedEntities / totalEntities) * 40) : 60;
766
- reportProgress?.(createProgress(entityProgress, 100, 'importing entities'));
767
- }
768
- reportProgress?.(createProgress(60, 100, 'importing relations'));
769
- // Process relations (60-95% progress)
770
- let processedRelations = 0;
771
- for (const importedRelation of importedGraph.relations) {
772
- // Check for cancellation periodically
773
- checkCancellation(options?.signal, 'importGraph');
774
- const relationKey = `${importedRelation.from}|${importedRelation.to}|${importedRelation.relationType}`;
775
- if (!existingEntitiesMap.has(importedRelation.from)) {
776
- result.errors.push(`Relation source entity "${importedRelation.from}" does not exist`);
777
- processedRelations++;
778
- continue;
779
- }
780
- if (!existingEntitiesMap.has(importedRelation.to)) {
781
- result.errors.push(`Relation target entity "${importedRelation.to}" does not exist`);
782
- processedRelations++;
783
- continue;
784
- }
785
- if (!existingRelationsSet.has(relationKey)) {
786
- result.relationsAdded++;
787
- if (!dryRun) {
788
- existingGraph.relations.push(importedRelation);
789
- existingRelationsSet.add(relationKey);
790
- }
791
- }
792
- else {
793
- if (mergeStrategy === 'fail') {
794
- result.errors.push(`Relation "${relationKey}" already exists`);
795
- }
796
- else {
797
- result.relationsSkipped++;
798
- }
799
- }
800
- processedRelations++;
801
- // Map relation progress (0-100%) to overall progress (60-95%)
802
- const relationProgress = totalRelations > 0 ? Math.round(60 + (processedRelations / totalRelations) * 35) : 95;
803
- reportProgress?.(createProgress(relationProgress, 100, 'importing relations'));
804
- }
805
- // Check for cancellation before final save
806
- checkCancellation(options?.signal, 'importGraph');
807
- reportProgress?.(createProgress(95, 100, 'saving graph'));
808
- if (!dryRun && (mergeStrategy !== 'fail' || result.errors.length === 0)) {
809
- await this.storage.saveGraph(existingGraph);
810
- }
811
- // Report completion
812
- reportProgress?.(createProgress(100, 100, 'importGraph'));
813
- return result;
814
- }
815
- // ============================================================
816
- // BACKUP OPERATIONS
817
- // ============================================================
818
- /**
819
- * Ensure backup directory exists.
820
- */
821
- async ensureBackupDir() {
822
- try {
823
- await fs.mkdir(this.backupDir, { recursive: true });
824
- }
825
- catch (error) {
826
- throw new FileOperationError('create backup directory', this.backupDir, error);
827
- }
828
- }
829
- /**
830
- * Generate backup file name with timestamp.
831
- * @param compressed - Whether the backup will be compressed (affects extension)
832
- */
833
- generateBackupFileName(compressed = true) {
834
- const now = new Date();
835
- const timestamp = now.toISOString()
836
- .replace(/:/g, '-')
837
- .replace(/\./g, '-')
838
- .replace('T', '_')
839
- .replace('Z', '');
840
- const extension = compressed ? '.jsonl.br' : '.jsonl';
841
- return `backup_${timestamp}${extension}`;
842
- }
843
- /**
844
- * Create a backup of the current knowledge graph.
845
- *
846
- * By default, backups are compressed with brotli for 50-70% space reduction.
847
- * Use `options.compress = false` to create uncompressed backups.
848
- *
849
- * @param options - Backup options (compress, description) or legacy description string
850
- * @returns Promise resolving to BackupResult with compression statistics
851
- *
852
- * @example
853
- * ```typescript
854
- * // Compressed backup (default)
855
- * const result = await manager.createBackup({ description: 'Pre-migration backup' });
856
- * console.log(`Compressed from ${result.originalSize} to ${result.compressedSize} bytes`);
857
- *
858
- * // Uncompressed backup
859
- * const result = await manager.createBackup({ compress: false });
860
- * ```
861
- */
862
- async createBackup(options) {
863
- await this.ensureBackupDir();
864
- // Handle legacy string argument (backward compatibility)
865
- const opts = typeof options === 'string'
866
- ? { description: options, compress: COMPRESSION_CONFIG.AUTO_COMPRESS_BACKUP }
867
- : { compress: COMPRESSION_CONFIG.AUTO_COMPRESS_BACKUP, ...options };
868
- const shouldCompress = opts.compress ?? COMPRESSION_CONFIG.AUTO_COMPRESS_BACKUP;
869
- const graph = await this.storage.loadGraph();
870
- const timestamp = new Date().toISOString();
871
- const fileName = this.generateBackupFileName(shouldCompress);
872
- const backupPath = join(this.backupDir, fileName);
873
- try {
874
- const originalPath = this.storage.getFilePath();
875
- let fileContent;
876
- try {
877
- fileContent = await fs.readFile(originalPath, 'utf-8');
878
- }
879
- catch {
880
- // If file doesn't exist, generate content from graph
881
- const lines = [
882
- ...graph.entities.map(e => JSON.stringify({ type: 'entity', ...e })),
883
- ...graph.relations.map(r => JSON.stringify({ type: 'relation', ...r })),
884
- ];
885
- fileContent = lines.join('\n');
886
- }
887
- const originalSize = Buffer.byteLength(fileContent, 'utf-8');
888
- let compressedSize = originalSize;
889
- let compressionRatio = 1;
890
- if (shouldCompress) {
891
- // Compress with maximum quality for backups (archive quality)
892
- const compressionResult = await compress(fileContent, {
893
- quality: COMPRESSION_CONFIG.BROTLI_QUALITY_ARCHIVE,
894
- mode: 'text',
895
- });
896
- await fs.writeFile(backupPath, compressionResult.compressed);
897
- compressedSize = compressionResult.compressedSize;
898
- compressionRatio = compressionResult.ratio;
899
- }
900
- else {
901
- // Write uncompressed backup
902
- await fs.writeFile(backupPath, fileContent);
903
- }
904
- const stats = await fs.stat(backupPath);
905
- const metadata = {
906
- timestamp,
907
- entityCount: graph.entities.length,
908
- relationCount: graph.relations.length,
909
- fileSize: stats.size,
910
- description: opts.description,
911
- compressed: shouldCompress,
912
- originalSize,
913
- compressionRatio: shouldCompress ? compressionRatio : undefined,
914
- compressionFormat: shouldCompress ? 'brotli' : 'none',
915
- };
916
- const metadataPath = `${backupPath}.meta.json`;
917
- await fs.writeFile(metadataPath, JSON.stringify(metadata, null, 2));
918
- return {
919
- path: backupPath,
920
- timestamp,
921
- entityCount: graph.entities.length,
922
- relationCount: graph.relations.length,
923
- compressed: shouldCompress,
924
- originalSize,
925
- compressedSize,
926
- compressionRatio,
927
- description: opts.description,
928
- };
929
- }
930
- catch (error) {
931
- throw new FileOperationError('create backup', backupPath, error);
932
- }
933
- }
934
- /**
935
- * List all available backups, sorted by timestamp (newest first).
936
- *
937
- * Detects both compressed (.jsonl.br) and uncompressed (.jsonl) backups.
938
- *
939
- * @returns Promise resolving to array of backup information with compression details
940
- */
941
- async listBackups() {
942
- try {
943
- try {
944
- await fs.access(this.backupDir);
945
- }
946
- catch {
947
- return [];
948
- }
949
- const files = await fs.readdir(this.backupDir);
950
- // Match both .jsonl and .jsonl.br backup files, exclude metadata files
951
- const backupFiles = files.filter(f => f.startsWith('backup_') &&
952
- (f.endsWith('.jsonl') || f.endsWith('.jsonl.br')) &&
953
- !f.endsWith('.meta.json'));
954
- const backups = [];
955
- for (const fileName of backupFiles) {
956
- const filePath = join(this.backupDir, fileName);
957
- const isCompressed = hasBrotliExtension(fileName);
958
- // Try to read metadata file (handles both .jsonl.meta.json and .jsonl.br.meta.json)
959
- const metadataPath = `${filePath}.meta.json`;
960
- try {
961
- const [metadataContent, stats] = await Promise.all([
962
- fs.readFile(metadataPath, 'utf-8'),
963
- fs.stat(filePath),
964
- ]);
965
- const metadata = JSON.parse(metadataContent);
966
- // Ensure compression fields are present (backward compatibility)
967
- if (metadata.compressed === undefined) {
968
- metadata.compressed = isCompressed;
969
- }
970
- if (metadata.compressionFormat === undefined) {
971
- metadata.compressionFormat = isCompressed ? 'brotli' : 'none';
972
- }
973
- backups.push({
974
- fileName,
975
- filePath,
976
- metadata,
977
- compressed: isCompressed,
978
- size: stats.size,
979
- });
980
- }
981
- catch {
982
- // Skip backups without valid metadata
983
- continue;
984
- }
985
- }
986
- backups.sort((a, b) => new Date(b.metadata.timestamp).getTime() - new Date(a.metadata.timestamp).getTime());
987
- return backups;
988
- }
989
- catch (error) {
990
- throw new FileOperationError('list backups', this.backupDir, error);
991
- }
992
- }
993
- /**
994
- * Restore the knowledge graph from a backup file.
995
- *
996
- * Automatically detects and decompresses brotli-compressed backups (.br extension).
997
- * Maintains backward compatibility with uncompressed backups.
998
- *
999
- * @param backupPath - Path to the backup file to restore from
1000
- * @returns Promise resolving to RestoreResult with restoration details
1001
- *
1002
- * @example
1003
- * ```typescript
1004
- * // Restore from compressed backup
1005
- * const result = await manager.restoreFromBackup('/path/to/backup.jsonl.br');
1006
- * console.log(`Restored ${result.entityCount} entities from compressed backup`);
1007
- *
1008
- * // Restore from uncompressed backup (legacy)
1009
- * const result = await manager.restoreFromBackup('/path/to/backup.jsonl');
1010
- * ```
1011
- */
1012
- async restoreFromBackup(backupPath) {
1013
- try {
1014
- await fs.access(backupPath);
1015
- const isCompressed = hasBrotliExtension(backupPath);
1016
- const backupBuffer = await fs.readFile(backupPath);
1017
- let backupContent;
1018
- if (isCompressed) {
1019
- // Decompress the backup
1020
- const decompressedBuffer = await decompress(backupBuffer);
1021
- backupContent = decompressedBuffer.toString('utf-8');
1022
- }
1023
- else {
1024
- // Read as plain text
1025
- backupContent = backupBuffer.toString('utf-8');
1026
- }
1027
- const mainPath = this.storage.getFilePath();
1028
- await fs.writeFile(mainPath, backupContent);
1029
- this.storage.clearCache();
1030
- // Load the restored graph to get counts
1031
- const graph = await this.storage.loadGraph();
1032
- return {
1033
- entityCount: graph.entities.length,
1034
- relationCount: graph.relations.length,
1035
- restoredFrom: backupPath,
1036
- wasCompressed: isCompressed,
1037
- };
1038
- }
1039
- catch (error) {
1040
- throw new FileOperationError('restore from backup', backupPath, error);
1041
- }
1042
- }
1043
- /**
1044
- * Delete a specific backup file.
1045
- *
1046
- * @param backupPath - Path to the backup file to delete
1047
- */
1048
- async deleteBackup(backupPath) {
1049
- try {
1050
- await fs.unlink(backupPath);
1051
- try {
1052
- await fs.unlink(`${backupPath}.meta.json`);
1053
- }
1054
- catch {
1055
- // Metadata file doesn't exist - that's ok
1056
- }
1057
- }
1058
- catch (error) {
1059
- throw new FileOperationError('delete backup', backupPath, error);
1060
- }
1061
- }
1062
- /**
1063
- * Clean old backups, keeping only the most recent N backups.
1064
- *
1065
- * @param keepCount - Number of recent backups to keep (default: 10)
1066
- * @returns Promise resolving to number of backups deleted
1067
- */
1068
- async cleanOldBackups(keepCount = 10) {
1069
- const backups = await this.listBackups();
1070
- if (backups.length <= keepCount) {
1071
- return 0;
1072
- }
1073
- const backupsToDelete = backups.slice(keepCount);
1074
- let deletedCount = 0;
1075
- for (const backup of backupsToDelete) {
1076
- try {
1077
- await this.deleteBackup(backup.filePath);
1078
- deletedCount++;
1079
- }
1080
- catch {
1081
- continue;
1082
- }
1083
- }
1084
- return deletedCount;
1085
- }
1086
- /**
1087
- * Get the path to the backup directory.
1088
- */
1089
- getBackupDir() {
1090
- return this.backupDir;
1091
- }
1092
- }
1093
- //# sourceMappingURL=IOManager.js.map