@monoes/memory 1.0.19 → 1.0.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/hnsw-distance.d.ts +31 -0
- package/dist/hnsw-distance.d.ts.map +1 -0
- package/dist/hnsw-distance.js +106 -0
- package/dist/hnsw-distance.js.map +1 -0
- package/dist/hnsw-graph-ops.d.ts +29 -0
- package/dist/hnsw-graph-ops.d.ts.map +1 -0
- package/dist/hnsw-graph-ops.js +96 -0
- package/dist/hnsw-graph-ops.js.map +1 -0
- package/dist/hnsw-heap.d.ts +45 -0
- package/dist/hnsw-heap.d.ts.map +1 -0
- package/dist/hnsw-heap.js +155 -0
- package/dist/hnsw-heap.js.map +1 -0
- package/dist/hnsw-index.d.ts +18 -43
- package/dist/hnsw-index.d.ts.map +1 -1
- package/dist/hnsw-index.js +21 -463
- package/dist/hnsw-index.js.map +1 -1
- package/dist/hnsw-quantizer.d.ts +28 -0
- package/dist/hnsw-quantizer.d.ts.map +1 -0
- package/dist/hnsw-quantizer.js +100 -0
- package/dist/hnsw-quantizer.js.map +1 -0
- package/dist/hnsw-serialize.d.ts +39 -0
- package/dist/hnsw-serialize.d.ts.map +1 -0
- package/dist/hnsw-serialize.js +54 -0
- package/dist/hnsw-serialize.js.map +1 -0
- package/dist/migration-loaders.d.ts +38 -0
- package/dist/migration-loaders.d.ts.map +1 -0
- package/dist/migration-loaders.js +269 -0
- package/dist/migration-loaders.js.map +1 -0
- package/dist/migration-transform.d.ts +21 -0
- package/dist/migration-transform.d.ts.map +1 -0
- package/dist/migration-transform.js +124 -0
- package/dist/migration-transform.js.map +1 -0
- package/dist/migration.d.ts +38 -22
- package/dist/migration.d.ts.map +1 -1
- package/dist/migration.js +35 -348
- package/dist/migration.js.map +1 -1
- package/dist/sql-backend-ann.d.ts +46 -0
- package/dist/sql-backend-ann.d.ts.map +1 -0
- package/dist/sql-backend-ann.js +120 -0
- package/dist/sql-backend-ann.js.map +1 -0
- package/dist/sql-backend-introspect.d.ts +17 -0
- package/dist/sql-backend-introspect.d.ts.map +1 -0
- package/dist/sql-backend-introspect.js +114 -0
- package/dist/sql-backend-introspect.js.map +1 -0
- package/dist/sql-backend-reads.d.ts +53 -0
- package/dist/sql-backend-reads.d.ts.map +1 -0
- package/dist/sql-backend-reads.js +295 -0
- package/dist/sql-backend-reads.js.map +1 -0
- package/dist/sql-backend-writes.d.ts +64 -0
- package/dist/sql-backend-writes.d.ts.map +1 -0
- package/dist/sql-backend-writes.js +270 -0
- package/dist/sql-backend-writes.js.map +1 -0
- package/dist/sql-backend.d.ts +39 -96
- package/dist/sql-backend.d.ts.map +1 -1
- package/dist/sql-backend.js +49 -706
- package/dist/sql-backend.js.map +1 -1
- package/dist/types-backend.d.ts +187 -0
- package/dist/types-backend.d.ts.map +1 -0
- package/dist/types-backend.js +8 -0
- package/dist/types-backend.js.map +1 -0
- package/dist/types-core.d.ts +180 -0
- package/dist/types-core.d.ts.map +1 -0
- package/dist/types-core.js +8 -0
- package/dist/types-core.js.map +1 -0
- package/dist/types-misc.d.ts +172 -0
- package/dist/types-misc.d.ts.map +1 -0
- package/dist/types-misc.js +56 -0
- package/dist/types-misc.js.map +1 -0
- package/dist/types.d.ts +8 -516
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +5 -49
- package/dist/types.js.map +1 -1
- package/package.json +2 -2
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Batch processing and legacy-entry transformation for MemoryMigrator.
|
|
3
|
+
* File-size sweep: split out of migration.ts. Mixed into
|
|
4
|
+
* MemoryMigrator.prototype at the bottom of migration.ts.
|
|
5
|
+
*
|
|
6
|
+
* @module v1/memory/migration-transform
|
|
7
|
+
*/
|
|
8
|
+
import { createDefaultEntry, } from './types.js';
|
|
9
|
+
export const migrationTransformMethods = {
|
|
10
|
+
async processBatch(batch) {
|
|
11
|
+
for (const legacyEntry of batch) {
|
|
12
|
+
try {
|
|
13
|
+
// Validate if enabled
|
|
14
|
+
if (this.config.validateData) {
|
|
15
|
+
const validation = this.validateEntry(legacyEntry);
|
|
16
|
+
if (!validation.valid) {
|
|
17
|
+
if (this.config.continueOnError) {
|
|
18
|
+
this.addError(legacyEntry.key || 'unknown', validation.reason || 'Validation failed', 'VALIDATION_ERROR', false);
|
|
19
|
+
this.progress.skipped++;
|
|
20
|
+
continue;
|
|
21
|
+
}
|
|
22
|
+
else {
|
|
23
|
+
throw new Error(validation.reason);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
// Transform to new format
|
|
28
|
+
const newEntry = await this.transformEntry(legacyEntry);
|
|
29
|
+
// Store in target
|
|
30
|
+
await this.target.store(newEntry);
|
|
31
|
+
this.progress.migrated++;
|
|
32
|
+
}
|
|
33
|
+
catch (error) {
|
|
34
|
+
if (this.config.continueOnError) {
|
|
35
|
+
this.addError(legacyEntry.key || 'unknown', error.message, 'STORE_ERROR', true);
|
|
36
|
+
this.progress.failed++;
|
|
37
|
+
}
|
|
38
|
+
else {
|
|
39
|
+
throw error;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
},
|
|
44
|
+
async transformEntry(legacy) {
|
|
45
|
+
// Map namespace if configured
|
|
46
|
+
let namespace = legacy.namespace || 'default';
|
|
47
|
+
if (this.config.namespaceMapping?.[namespace]) {
|
|
48
|
+
namespace = this.config.namespaceMapping[namespace];
|
|
49
|
+
}
|
|
50
|
+
// Determine content
|
|
51
|
+
const content = typeof legacy.value === 'string' ? legacy.value : JSON.stringify(legacy.value);
|
|
52
|
+
// Map type if configured
|
|
53
|
+
let type = 'semantic';
|
|
54
|
+
if (legacy.metadata?.type && typeof legacy.metadata.type === 'string') {
|
|
55
|
+
if (this.config.typeMapping?.[legacy.metadata.type]) {
|
|
56
|
+
type = this.config.typeMapping[legacy.metadata.type];
|
|
57
|
+
}
|
|
58
|
+
else if (this.isValidMemoryType(legacy.metadata.type)) {
|
|
59
|
+
type = legacy.metadata.type;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
// Parse timestamps
|
|
63
|
+
const createdAt = this.parseTimestamp(legacy.createdAt || legacy.created_at || legacy.timestamp);
|
|
64
|
+
const updatedAt = this.parseTimestamp(legacy.updatedAt || legacy.updated_at || legacy.timestamp);
|
|
65
|
+
const input = {
|
|
66
|
+
key: legacy.key,
|
|
67
|
+
content,
|
|
68
|
+
type,
|
|
69
|
+
namespace,
|
|
70
|
+
tags: legacy.tags || [],
|
|
71
|
+
metadata: {
|
|
72
|
+
...legacy.metadata,
|
|
73
|
+
migrated: true,
|
|
74
|
+
migrationSource: this.config.source,
|
|
75
|
+
migrationTimestamp: Date.now(),
|
|
76
|
+
originalValue: legacy.value,
|
|
77
|
+
},
|
|
78
|
+
};
|
|
79
|
+
const entry = createDefaultEntry(input);
|
|
80
|
+
entry.createdAt = createdAt;
|
|
81
|
+
entry.updatedAt = updatedAt;
|
|
82
|
+
// Generate embedding if configured
|
|
83
|
+
if (this.config.generateEmbeddings && this.embeddingGenerator) {
|
|
84
|
+
try {
|
|
85
|
+
entry.embedding = await this.embeddingGenerator(content);
|
|
86
|
+
}
|
|
87
|
+
catch (error) {
|
|
88
|
+
// Log but don't fail
|
|
89
|
+
this.emit('migration:warning', {
|
|
90
|
+
message: `Failed to generate embedding for ${legacy.key}: ${error.message}`,
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return entry;
|
|
95
|
+
},
|
|
96
|
+
validateEntry(entry) {
|
|
97
|
+
if (!entry.key || typeof entry.key !== 'string') {
|
|
98
|
+
return { valid: false, reason: 'Missing or invalid key' };
|
|
99
|
+
}
|
|
100
|
+
if (entry.value === undefined) {
|
|
101
|
+
return { valid: false, reason: 'Missing value' };
|
|
102
|
+
}
|
|
103
|
+
if (entry.key.length > 500) {
|
|
104
|
+
return { valid: false, reason: 'Key too long (max 500 chars)' };
|
|
105
|
+
}
|
|
106
|
+
return { valid: true };
|
|
107
|
+
},
|
|
108
|
+
isValidMemoryType(type) {
|
|
109
|
+
// 'procedural' kept here for legacy migration compatibility only —
|
|
110
|
+
// it is no longer a valid MemoryType (procedural memory removed).
|
|
111
|
+
return ['episodic', 'semantic', 'procedural', 'working', 'cache'].includes(type);
|
|
112
|
+
},
|
|
113
|
+
parseTimestamp(value) {
|
|
114
|
+
if (!value)
|
|
115
|
+
return Date.now();
|
|
116
|
+
if (typeof value === 'number') {
|
|
117
|
+
// Handle both milliseconds and seconds
|
|
118
|
+
return value > 1e12 ? value : value * 1000;
|
|
119
|
+
}
|
|
120
|
+
const parsed = Date.parse(value);
|
|
121
|
+
return Number.isNaN(parsed) ? Date.now() : parsed;
|
|
122
|
+
},
|
|
123
|
+
};
|
|
124
|
+
//# sourceMappingURL=migration-transform.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"migration-transform.js","sourceRoot":"","sources":["../src/migration-transform.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAIH,OAAO,EACL,kBAAkB,GAInB,MAAM,YAAY,CAAC;AAEpB,MAAM,CAAC,MAAM,yBAAyB,GAAG;IACvC,KAAK,CAAC,YAAY,CAAuB,KAAoB;QAC3D,KAAK,MAAM,WAAW,IAAI,KAAK,EAAE,CAAC;YAChC,IAAI,CAAC;gBACH,sBAAsB;gBACtB,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;oBAC7B,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;oBACnD,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;wBACtB,IAAI,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;4BAChC,IAAI,CAAC,QAAQ,CACX,WAAW,CAAC,GAAG,IAAI,SAAS,EAC5B,UAAU,CAAC,MAAM,IAAI,mBAAmB,EACxC,kBAAkB,EAClB,KAAK,CACN,CAAC;4BACF,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;4BACxB,SAAS;wBACX,CAAC;6BAAM,CAAC;4BACN,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;wBACrC,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,0BAA0B;gBAC1B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;gBAExD,kBAAkB;gBAClB,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBAClC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;YAC3B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;oBAChC,IAAI,CAAC,QAAQ,CACX,WAAW,CAAC,GAAG,IAAI,SAAS,EAC3B,KAAe,CAAC,OAAO,EACxB,aAAa,EACb,IAAI,CACL,CAAC;oBACF,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;gBACzB,CAAC;qBAAM,CAAC;oBACN,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,cAAc,CAAuB,MAAmB;QAC5D,8BAA8B;QAC9B,IAAI,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,SAAS,CAAC;QAC9C,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9C,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QACtD,CAAC;QAED,oBAAoB;QACpB,MAAM,OAAO,GAAG,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAE/F,yBAAyB;QACzB,IAAI,IAAI,GAAe,UAAU,CAAC;QAClC,IAAI,MAAM,CAAC,QAAQ,EAAE,IAAI,IAAI,OAAO,MAAM,CAAC,QAAQ,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACtE,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBACpD,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACvD,CAAC;iBAAM,IAAI,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxD,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAkB,CAAC;YAC5C,CAAC;QACH,CAAC;QAED,mBAAmB;QACnB,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CACnC,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,SAAS,CAC1D,CAAC;QACF,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,CACnC,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,SAAS,CAC1D,CAAC;QAEF,MAAM,KAAK,GAAqB;YAC9B,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,OAAO;YACP,IAAI;YACJ,SAAS;YACT,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,EAAE;YACvB,QAAQ,EAAE;gBACR,GAAG,MAAM,CAAC,QAAQ;gBAClB,QAAQ,EAAE,IAAI;gBACd,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;gBACnC,kBAAkB,EAAE,IAAI,CAAC,GAAG,EAAE;gBAC9B,aAAa,EAAE,MAAM,CAAC,KAAK;aAC5B;SACF,CAAC;QAEF,MAAM,KAAK,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;QACxC,KAAK,CAAC,SAAS,GAAG,SAAS,CAAC;QAC5B,KAAK,CAAC,SAAS,GAAG,SAAS,CAAC;QAE5B,mCAAmC;QACnC,IAAI,IAAI,CAAC,MAAM,CAAC,kBAAkB,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC9D,IAAI,CAAC;gBACH,KAAK,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;YAC3D,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,qBAAqB;gBACrB,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE;oBAC7B,OAAO,EAAE,oCAAoC,MAAM,CAAC,GAAG,KAAM,KAAe,CAAC,OAAO,EAAE;iBACvF,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED,aAAa,CAAuB,KAAkB;QACpD,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,OAAO,KAAK,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;YAChD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,wBAAwB,EAAE,CAAC;QAC5D,CAAC;QAED,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC9B,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC;QACnD,CAAC;QAED,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YAC3B,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,8BAA8B,EAAE,CAAC;QAClE,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IACzB,CAAC;IAED,iBAAiB,CAAuB,IAAY;QAClD,mEAAmE;QACnE,kEAAkE;QAClE,OAAO,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IACnF,CAAC;IAED,cAAc,CAAuB,KAAkC;QACrE,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC,GAAG,EAAE,CAAC;QAE9B,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,uCAAuC;YACvC,OAAO,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC;QAC7C,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACjC,OAAO,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IACpD,CAAC;CACF,CAAC"}
|
package/dist/migration.d.ts
CHANGED
|
@@ -5,11 +5,24 @@
|
|
|
5
5
|
* to the unified memory system (any IMemoryBackend — SQLite via
|
|
6
6
|
* better-sqlite3/sql.js as of 2026-07).
|
|
7
7
|
*
|
|
8
|
+
* File-size sweep: the class's method BODIES are grouped into sibling
|
|
9
|
+
* modules — legacy source loaders (SQLite/Markdown/JSON/swarm/distributed) in
|
|
10
|
+
* migration-loaders.ts, and batch processing + legacy-entry transformation in
|
|
11
|
+
* migration-transform.ts — mixed onto MemoryMigrator.prototype below. Each
|
|
12
|
+
* stays a real method here (same signature, same visibility) so callers and
|
|
13
|
+
* TypeScript see no difference; the body is just
|
|
14
|
+
* `return theSiblingImpl.call(this, ...args)`. Fields those bodies read/write
|
|
15
|
+
* moved from `private` to `protected` (compile-time only; no runtime change)
|
|
16
|
+
* — TypeScript allows a standalone function typed `this: MemoryMigrator` to
|
|
17
|
+
* reach `protected` members but not `private` ones, so `protected` is the
|
|
18
|
+
* minimum visibility the split needs.
|
|
19
|
+
*
|
|
8
20
|
* @module v1/memory/migration
|
|
9
21
|
*/
|
|
10
22
|
import { EventEmitter } from 'node:events';
|
|
11
|
-
import type
|
|
12
|
-
import {
|
|
23
|
+
import { type LegacyEntry } from './migration-loaders.js';
|
|
24
|
+
import type { EmbeddingGenerator, IMemoryBackend, MemoryEntry, MigrationConfig, MigrationProgress, MigrationResult, MigrationSource } from './types.js';
|
|
25
|
+
export type { LegacyEntry };
|
|
13
26
|
/**
|
|
14
27
|
* Memory Migration Manager
|
|
15
28
|
*
|
|
@@ -22,10 +35,10 @@ import { type EmbeddingGenerator, type MigrationConfig, type MigrationProgress,
|
|
|
22
35
|
* - DistributedMemory instances
|
|
23
36
|
*/
|
|
24
37
|
export declare class MemoryMigrator extends EventEmitter {
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
38
|
+
protected config: MigrationConfig;
|
|
39
|
+
protected target: IMemoryBackend;
|
|
40
|
+
protected embeddingGenerator?: EmbeddingGenerator;
|
|
41
|
+
protected progress: MigrationProgress;
|
|
29
42
|
constructor(target: IMemoryBackend, config: Partial<MigrationConfig>, embeddingGenerator?: EmbeddingGenerator);
|
|
30
43
|
/**
|
|
31
44
|
* Run the migration
|
|
@@ -35,25 +48,28 @@ export declare class MemoryMigrator extends EventEmitter {
|
|
|
35
48
|
* Get current migration progress
|
|
36
49
|
*/
|
|
37
50
|
getProgress(): MigrationProgress;
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
51
|
+
protected loadFromSource(): Promise<LegacyEntry[]>;
|
|
52
|
+
protected loadFromSQLite(): Promise<LegacyEntry[]>;
|
|
53
|
+
protected safeJsonParse<T>(value: string | undefined, fallback: T): T;
|
|
54
|
+
protected loadFromMarkdown(): Promise<LegacyEntry[]>;
|
|
55
|
+
protected loadFromJSON(filePathOverride?: string): Promise<LegacyEntry[]>;
|
|
56
|
+
protected loadFromMemoryManager(): Promise<LegacyEntry[]>;
|
|
57
|
+
protected loadFromSwarmMemory(): Promise<LegacyEntry[]>;
|
|
58
|
+
protected loadFromDistributedMemory(): Promise<LegacyEntry[]>;
|
|
59
|
+
protected walkDirectory(dir: string, extension: string): Promise<string[]>;
|
|
60
|
+
protected parseMarkdownEntry(filePath: string, content: string, basePath: string): LegacyEntry | null;
|
|
61
|
+
protected processBatch(batch: LegacyEntry[]): Promise<void>;
|
|
62
|
+
protected transformEntry(legacy: LegacyEntry): Promise<MemoryEntry>;
|
|
48
63
|
private initializeProgress;
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
64
|
+
protected validateEntry(entry: LegacyEntry): {
|
|
65
|
+
valid: boolean;
|
|
66
|
+
reason?: string;
|
|
67
|
+
};
|
|
68
|
+
protected addError(entryId: string, message: string, code: string, recoverable: boolean): void;
|
|
69
|
+
protected parseTimestamp(value: string | number | undefined): number;
|
|
70
|
+
protected isValidMemoryType(type: string): boolean;
|
|
53
71
|
private estimateTimeRemaining;
|
|
54
72
|
private generateSummary;
|
|
55
|
-
private walkDirectory;
|
|
56
|
-
private parseMarkdownEntry;
|
|
57
73
|
}
|
|
58
74
|
/**
|
|
59
75
|
* Convenience function to create a migrator
|
package/dist/migration.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"migration.d.ts","sourceRoot":"","sources":["../src/migration.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"migration.d.ts","sourceRoot":"","sources":["../src/migration.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,KAAK,WAAW,EAA0B,MAAM,wBAAwB,CAAC;AAElF,OAAO,KAAK,EACV,kBAAkB,EAClB,cAAc,EACd,WAAW,EACX,eAAe,EAEf,iBAAiB,EACjB,eAAe,EACf,eAAe,EAChB,MAAM,YAAY,CAAC;AAEpB,YAAY,EAAE,WAAW,EAAE,CAAC;AAY5B;;;;;;;;;;GAUG;AACH,qBAAa,cAAe,SAAQ,YAAY;IAC9C,SAAS,CAAC,MAAM,EAAE,eAAe,CAAC;IAClC,SAAS,CAAC,MAAM,EAAE,cAAc,CAAC;IACjC,SAAS,CAAC,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAClD,SAAS,CAAC,QAAQ,EAAE,iBAAiB,CAAC;IAEtC,YACE,MAAM,EAAE,cAAc,EACtB,MAAM,EAAE,OAAO,CAAC,eAAe,CAAC,EAChC,kBAAkB,CAAC,EAAE,kBAAkB,EAOxC;IAED;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,eAAe,CAAC,CAuDxC;IAED;;OAEG;IACH,WAAW,IAAI,iBAAiB,CAE/B;IAID,UAAgB,cAAc,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,CAEvD;IAED,UAAgB,cAAc,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,CAEvD;IAED,SAAS,CAAC,aAAa,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,EAAE,QAAQ,EAAE,CAAC,GAAG,CAAC,CAEpE;IAED,UAAgB,gBAAgB,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,CAEzD;IAED,UAAgB,YAAY,CAAC,gBAAgB,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAE9E;IAED,UAAgB,qBAAqB,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,CAE9D;IAED,UAAgB,mBAAmB,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,CAE5D;IAED,UAAgB,yBAAyB,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,CAElE;IAED,UAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAE/E;IAED,SAAS,CAAC,kBAAkB,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,GACf,WAAW,GAAG,IAAI,CAEpB;IAID,UAAgB,YAAY,CAAC,KAAK,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAEhE;IAED,UAAgB,cAAc,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CAExE;IAID,OAAO,CAAC,kBAAkB;IAc1B,SAAS,CAAC,aAAa,CAAC,KAAK,EAAE,WAAW,GAAG;QAAE,KAAK,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAE/E;IAED,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,GAAG,IAAI,CAS7F;IAED,SAAS,CAAC,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,GAAG,MAAM,CAEnE;IAED,SAAS,CAAC,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAEjD;IAED,OAAO,CAAC,qBAAqB;IAU7B,OAAO,CAAC,eAAe;CA4BxB;AAED;;GAEG;AACH,wBAAgB,cAAc,CAC5B,MAAM,EAAE,cAAc,EACtB,MAAM,EAAE,eAAe,EACvB,UAAU,EAAE,MAAM,EAClB,OAAO,GAAE,OAAO,CAAC,eAAe,CAAM,EACtC,kBAAkB,CAAC,EAAE,kBAAkB,GACtC,cAAc,CAEhB;AAED;;GAEG;AACH,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,cAAc,EACtB,OAAO,EAAE,KAAK,CAAC;IAAE,MAAM,EAAE,eAAe,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,EACzD,OAAO,GAAE,OAAO,CAAC,eAAe,CAAM,EACtC,kBAAkB,CAAC,EAAE,kBAAkB,GACtC,OAAO,CAAC,eAAe,EAAE,CAAC,CAU5B;eAEc,cAAc"}
|
package/dist/migration.js
CHANGED
|
@@ -5,12 +5,23 @@
|
|
|
5
5
|
* to the unified memory system (any IMemoryBackend — SQLite via
|
|
6
6
|
* better-sqlite3/sql.js as of 2026-07).
|
|
7
7
|
*
|
|
8
|
+
* File-size sweep: the class's method BODIES are grouped into sibling
|
|
9
|
+
* modules — legacy source loaders (SQLite/Markdown/JSON/swarm/distributed) in
|
|
10
|
+
* migration-loaders.ts, and batch processing + legacy-entry transformation in
|
|
11
|
+
* migration-transform.ts — mixed onto MemoryMigrator.prototype below. Each
|
|
12
|
+
* stays a real method here (same signature, same visibility) so callers and
|
|
13
|
+
* TypeScript see no difference; the body is just
|
|
14
|
+
* `return theSiblingImpl.call(this, ...args)`. Fields those bodies read/write
|
|
15
|
+
* moved from `private` to `protected` (compile-time only; no runtime change)
|
|
16
|
+
* — TypeScript allows a standalone function typed `this: MemoryMigrator` to
|
|
17
|
+
* reach `protected` members but not `private` ones, so `protected` is the
|
|
18
|
+
* minimum visibility the split needs.
|
|
19
|
+
*
|
|
8
20
|
* @module v1/memory/migration
|
|
9
21
|
*/
|
|
10
22
|
import { EventEmitter } from 'node:events';
|
|
11
|
-
import {
|
|
12
|
-
import
|
|
13
|
-
import { createDefaultEntry, } from './types.js';
|
|
23
|
+
import { migrationLoaderMethods } from './migration-loaders.js';
|
|
24
|
+
import { migrationTransformMethods } from './migration-transform.js';
|
|
14
25
|
/**
|
|
15
26
|
* Default migration configuration
|
|
16
27
|
*/
|
|
@@ -93,277 +104,43 @@ export class MemoryMigrator extends EventEmitter {
|
|
|
93
104
|
getProgress() {
|
|
94
105
|
return { ...this.progress };
|
|
95
106
|
}
|
|
96
|
-
// ===== Source Loaders =====
|
|
107
|
+
// ===== Source Loaders (bodies live in migration-loaders.ts) =====
|
|
97
108
|
async loadFromSource() {
|
|
98
|
-
|
|
99
|
-
case 'sqlite':
|
|
100
|
-
return this.loadFromSQLite();
|
|
101
|
-
case 'markdown':
|
|
102
|
-
return this.loadFromMarkdown();
|
|
103
|
-
case 'json':
|
|
104
|
-
return this.loadFromJSON();
|
|
105
|
-
case 'memory-manager':
|
|
106
|
-
return this.loadFromMemoryManager();
|
|
107
|
-
case 'monoswarm-memory':
|
|
108
|
-
return this.loadFromSwarmMemory();
|
|
109
|
-
case 'distributed-memory':
|
|
110
|
-
return this.loadFromDistributedMemory();
|
|
111
|
-
default:
|
|
112
|
-
throw new Error(`Unknown migration source: ${this.config.source}`);
|
|
113
|
-
}
|
|
109
|
+
return migrationLoaderMethods.loadFromSource.call(this);
|
|
114
110
|
}
|
|
115
111
|
async loadFromSQLite() {
|
|
116
|
-
|
|
117
|
-
// A .json-suffixed "sqlite" source is actually an exported JSON dump —
|
|
118
|
-
// route it through the real JSON loader instead of opening it as a DB.
|
|
119
|
-
if (dbPath.endsWith('.json')) {
|
|
120
|
-
return this.loadFromJSON();
|
|
121
|
-
}
|
|
122
|
-
let Database;
|
|
123
|
-
try {
|
|
124
|
-
const mod = await import('better-sqlite3');
|
|
125
|
-
Database = mod.default;
|
|
126
|
-
}
|
|
127
|
-
catch (error) {
|
|
128
|
-
throw new Error(`Cannot read SQLite source "${dbPath}": better-sqlite3 is not available (${error.message})`);
|
|
129
|
-
}
|
|
130
|
-
let db;
|
|
131
|
-
try {
|
|
132
|
-
db = new Database(dbPath, { readonly: true, fileMustExist: true });
|
|
133
|
-
}
|
|
134
|
-
catch (error) {
|
|
135
|
-
throw new Error(`Failed to open SQLite database "${dbPath}": ${error.message}`);
|
|
136
|
-
}
|
|
137
|
-
try {
|
|
138
|
-
const tables = db
|
|
139
|
-
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='memory_entries'")
|
|
140
|
-
.all();
|
|
141
|
-
if (tables.length === 0) {
|
|
142
|
-
throw new Error(`SQLite source "${dbPath}" has no "memory_entries" table — not a recognized memory backend database`);
|
|
143
|
-
}
|
|
144
|
-
const rows = db.prepare('SELECT * FROM memory_entries').all();
|
|
145
|
-
return rows.map((row) => ({
|
|
146
|
-
id: row.id,
|
|
147
|
-
key: row.key,
|
|
148
|
-
value: row.content,
|
|
149
|
-
namespace: row.namespace,
|
|
150
|
-
tags: this.safeJsonParse(row.tags, []),
|
|
151
|
-
metadata: this.safeJsonParse(row.metadata, {}),
|
|
152
|
-
createdAt: row.created_at,
|
|
153
|
-
updatedAt: row.updated_at,
|
|
154
|
-
}));
|
|
155
|
-
}
|
|
156
|
-
finally {
|
|
157
|
-
db.close();
|
|
158
|
-
}
|
|
112
|
+
return migrationLoaderMethods.loadFromSQLite.call(this);
|
|
159
113
|
}
|
|
160
114
|
safeJsonParse(value, fallback) {
|
|
161
|
-
|
|
162
|
-
return fallback;
|
|
163
|
-
try {
|
|
164
|
-
return JSON.parse(value);
|
|
165
|
-
}
|
|
166
|
-
catch {
|
|
167
|
-
return fallback;
|
|
168
|
-
}
|
|
115
|
+
return migrationLoaderMethods.safeJsonParse.call(this, value, fallback);
|
|
169
116
|
}
|
|
170
117
|
async loadFromMarkdown() {
|
|
171
|
-
|
|
172
|
-
const basePath = this.config.sourcePath;
|
|
173
|
-
try {
|
|
174
|
-
const files = await this.walkDirectory(basePath, '.md');
|
|
175
|
-
for (const filePath of files) {
|
|
176
|
-
try {
|
|
177
|
-
const content = await fs.readFile(filePath, 'utf-8');
|
|
178
|
-
const entry = this.parseMarkdownEntry(filePath, content, basePath);
|
|
179
|
-
if (entry) {
|
|
180
|
-
entries.push(entry);
|
|
181
|
-
}
|
|
182
|
-
}
|
|
183
|
-
catch (error) {
|
|
184
|
-
this.addError(filePath, error.message, 'PARSE_ERROR', true);
|
|
185
|
-
}
|
|
186
|
-
}
|
|
187
|
-
return entries;
|
|
188
|
-
}
|
|
189
|
-
catch (error) {
|
|
190
|
-
throw new Error(`Failed to load Markdown: ${error.message}`);
|
|
191
|
-
}
|
|
118
|
+
return migrationLoaderMethods.loadFromMarkdown.call(this);
|
|
192
119
|
}
|
|
193
120
|
async loadFromJSON(filePathOverride) {
|
|
194
|
-
|
|
195
|
-
try {
|
|
196
|
-
const content = await fs.readFile(filePath, 'utf-8');
|
|
197
|
-
const data = JSON.parse(content);
|
|
198
|
-
// Handle different JSON formats
|
|
199
|
-
if (Array.isArray(data)) {
|
|
200
|
-
return data;
|
|
201
|
-
}
|
|
202
|
-
else if (data.entries) {
|
|
203
|
-
return data.entries;
|
|
204
|
-
}
|
|
205
|
-
else if (typeof data === 'object') {
|
|
206
|
-
// Assume it's a namespace -> entries map
|
|
207
|
-
const entries = [];
|
|
208
|
-
for (const [namespace, namespaceEntries] of Object.entries(data)) {
|
|
209
|
-
if (Array.isArray(namespaceEntries)) {
|
|
210
|
-
for (const entry of namespaceEntries) {
|
|
211
|
-
entries.push({ ...entry, namespace });
|
|
212
|
-
}
|
|
213
|
-
}
|
|
214
|
-
}
|
|
215
|
-
return entries;
|
|
216
|
-
}
|
|
217
|
-
return [];
|
|
218
|
-
}
|
|
219
|
-
catch (error) {
|
|
220
|
-
throw new Error(`Failed to load JSON: ${error.message}`);
|
|
221
|
-
}
|
|
121
|
+
return migrationLoaderMethods.loadFromJSON.call(this, filePathOverride);
|
|
222
122
|
}
|
|
223
123
|
async loadFromMemoryManager() {
|
|
224
|
-
|
|
225
|
-
// For now, try to load from common paths
|
|
226
|
-
const possiblePaths = ['./memory/memory-store.json', './.swarm/memory.db', './memory.json'];
|
|
227
|
-
for (const p of possiblePaths) {
|
|
228
|
-
try {
|
|
229
|
-
const fullPath = path.resolve(this.config.sourcePath, p);
|
|
230
|
-
await fs.access(fullPath);
|
|
231
|
-
// #85: read the verified fullPath, not this.config.sourcePath — the
|
|
232
|
-
// latter is the search directory, so readFile() on it throws EISDIR,
|
|
233
|
-
// the catch swallowed it, and the migration silently returned [].
|
|
234
|
-
// `await` is required: a bare `return promise` inside try/catch does
|
|
235
|
-
// not route rejections through the catch below.
|
|
236
|
-
const stat = await fs.stat(fullPath);
|
|
237
|
-
if (!stat.isFile())
|
|
238
|
-
continue;
|
|
239
|
-
return await this.loadFromJSON(fullPath);
|
|
240
|
-
}
|
|
241
|
-
catch { }
|
|
242
|
-
}
|
|
243
|
-
return [];
|
|
124
|
+
return migrationLoaderMethods.loadFromMemoryManager.call(this);
|
|
244
125
|
}
|
|
245
126
|
async loadFromSwarmMemory() {
|
|
246
|
-
|
|
247
|
-
const entries = [];
|
|
248
|
-
const basePath = this.config.sourcePath;
|
|
249
|
-
try {
|
|
250
|
-
// Check for swarm memory directory structure
|
|
251
|
-
const partitionsPath = path.join(basePath, '.swarm', 'memory');
|
|
252
|
-
const files = await this.walkDirectory(partitionsPath, '.json');
|
|
253
|
-
for (const filePath of files) {
|
|
254
|
-
try {
|
|
255
|
-
const content = await fs.readFile(filePath, 'utf-8');
|
|
256
|
-
const data = JSON.parse(content);
|
|
257
|
-
// Extract namespace from file path
|
|
258
|
-
const relativePath = path.relative(partitionsPath, filePath);
|
|
259
|
-
const namespace = path.dirname(relativePath).replace(/\\/g, '/');
|
|
260
|
-
if (Array.isArray(data)) {
|
|
261
|
-
entries.push(...data.map((e) => ({ ...e, namespace })));
|
|
262
|
-
}
|
|
263
|
-
else if (data.entries) {
|
|
264
|
-
entries.push(...data.entries.map((e) => ({ ...e, namespace })));
|
|
265
|
-
}
|
|
266
|
-
}
|
|
267
|
-
catch (error) {
|
|
268
|
-
this.addError(filePath, error.message, 'PARSE_ERROR', true);
|
|
269
|
-
}
|
|
270
|
-
}
|
|
271
|
-
return entries;
|
|
272
|
-
}
|
|
273
|
-
catch (_error) {
|
|
274
|
-
return [];
|
|
275
|
-
}
|
|
127
|
+
return migrationLoaderMethods.loadFromSwarmMemory.call(this);
|
|
276
128
|
}
|
|
277
129
|
async loadFromDistributedMemory() {
|
|
278
|
-
|
|
279
|
-
return this.loadFromSwarmMemory(); // Similar structure
|
|
130
|
+
return migrationLoaderMethods.loadFromDistributedMemory.call(this);
|
|
280
131
|
}
|
|
281
|
-
|
|
132
|
+
async walkDirectory(dir, extension) {
|
|
133
|
+
return migrationLoaderMethods.walkDirectory.call(this, dir, extension);
|
|
134
|
+
}
|
|
135
|
+
parseMarkdownEntry(filePath, content, basePath) {
|
|
136
|
+
return migrationLoaderMethods.parseMarkdownEntry.call(this, filePath, content, basePath);
|
|
137
|
+
}
|
|
138
|
+
// ===== Batch Processing (bodies live in migration-transform.ts) =====
|
|
282
139
|
async processBatch(batch) {
|
|
283
|
-
|
|
284
|
-
try {
|
|
285
|
-
// Validate if enabled
|
|
286
|
-
if (this.config.validateData) {
|
|
287
|
-
const validation = this.validateEntry(legacyEntry);
|
|
288
|
-
if (!validation.valid) {
|
|
289
|
-
if (this.config.continueOnError) {
|
|
290
|
-
this.addError(legacyEntry.key || 'unknown', validation.reason || 'Validation failed', 'VALIDATION_ERROR', false);
|
|
291
|
-
this.progress.skipped++;
|
|
292
|
-
continue;
|
|
293
|
-
}
|
|
294
|
-
else {
|
|
295
|
-
throw new Error(validation.reason);
|
|
296
|
-
}
|
|
297
|
-
}
|
|
298
|
-
}
|
|
299
|
-
// Transform to new format
|
|
300
|
-
const newEntry = await this.transformEntry(legacyEntry);
|
|
301
|
-
// Store in target
|
|
302
|
-
await this.target.store(newEntry);
|
|
303
|
-
this.progress.migrated++;
|
|
304
|
-
}
|
|
305
|
-
catch (error) {
|
|
306
|
-
if (this.config.continueOnError) {
|
|
307
|
-
this.addError(legacyEntry.key || 'unknown', error.message, 'STORE_ERROR', true);
|
|
308
|
-
this.progress.failed++;
|
|
309
|
-
}
|
|
310
|
-
else {
|
|
311
|
-
throw error;
|
|
312
|
-
}
|
|
313
|
-
}
|
|
314
|
-
}
|
|
140
|
+
return migrationTransformMethods.processBatch.call(this, batch);
|
|
315
141
|
}
|
|
316
142
|
async transformEntry(legacy) {
|
|
317
|
-
|
|
318
|
-
let namespace = legacy.namespace || 'default';
|
|
319
|
-
if (this.config.namespaceMapping?.[namespace]) {
|
|
320
|
-
namespace = this.config.namespaceMapping[namespace];
|
|
321
|
-
}
|
|
322
|
-
// Determine content
|
|
323
|
-
const content = typeof legacy.value === 'string' ? legacy.value : JSON.stringify(legacy.value);
|
|
324
|
-
// Map type if configured
|
|
325
|
-
let type = 'semantic';
|
|
326
|
-
if (legacy.metadata?.type && typeof legacy.metadata.type === 'string') {
|
|
327
|
-
if (this.config.typeMapping?.[legacy.metadata.type]) {
|
|
328
|
-
type = this.config.typeMapping[legacy.metadata.type];
|
|
329
|
-
}
|
|
330
|
-
else if (this.isValidMemoryType(legacy.metadata.type)) {
|
|
331
|
-
type = legacy.metadata.type;
|
|
332
|
-
}
|
|
333
|
-
}
|
|
334
|
-
// Parse timestamps
|
|
335
|
-
const createdAt = this.parseTimestamp(legacy.createdAt || legacy.created_at || legacy.timestamp);
|
|
336
|
-
const updatedAt = this.parseTimestamp(legacy.updatedAt || legacy.updated_at || legacy.timestamp);
|
|
337
|
-
const input = {
|
|
338
|
-
key: legacy.key,
|
|
339
|
-
content,
|
|
340
|
-
type,
|
|
341
|
-
namespace,
|
|
342
|
-
tags: legacy.tags || [],
|
|
343
|
-
metadata: {
|
|
344
|
-
...legacy.metadata,
|
|
345
|
-
migrated: true,
|
|
346
|
-
migrationSource: this.config.source,
|
|
347
|
-
migrationTimestamp: Date.now(),
|
|
348
|
-
originalValue: legacy.value,
|
|
349
|
-
},
|
|
350
|
-
};
|
|
351
|
-
const entry = createDefaultEntry(input);
|
|
352
|
-
entry.createdAt = createdAt;
|
|
353
|
-
entry.updatedAt = updatedAt;
|
|
354
|
-
// Generate embedding if configured
|
|
355
|
-
if (this.config.generateEmbeddings && this.embeddingGenerator) {
|
|
356
|
-
try {
|
|
357
|
-
entry.embedding = await this.embeddingGenerator(content);
|
|
358
|
-
}
|
|
359
|
-
catch (error) {
|
|
360
|
-
// Log but don't fail
|
|
361
|
-
this.emit('migration:warning', {
|
|
362
|
-
message: `Failed to generate embedding for ${legacy.key}: ${error.message}`,
|
|
363
|
-
});
|
|
364
|
-
}
|
|
365
|
-
}
|
|
366
|
-
return entry;
|
|
143
|
+
return migrationTransformMethods.transformEntry.call(this, legacy);
|
|
367
144
|
}
|
|
368
145
|
// ===== Helper Methods =====
|
|
369
146
|
initializeProgress() {
|
|
@@ -380,16 +157,7 @@ export class MemoryMigrator extends EventEmitter {
|
|
|
380
157
|
};
|
|
381
158
|
}
|
|
382
159
|
validateEntry(entry) {
|
|
383
|
-
|
|
384
|
-
return { valid: false, reason: 'Missing or invalid key' };
|
|
385
|
-
}
|
|
386
|
-
if (entry.value === undefined) {
|
|
387
|
-
return { valid: false, reason: 'Missing value' };
|
|
388
|
-
}
|
|
389
|
-
if (entry.key.length > 500) {
|
|
390
|
-
return { valid: false, reason: 'Key too long (max 500 chars)' };
|
|
391
|
-
}
|
|
392
|
-
return { valid: true };
|
|
160
|
+
return migrationTransformMethods.validateEntry.call(this, entry);
|
|
393
161
|
}
|
|
394
162
|
addError(entryId, message, code, recoverable) {
|
|
395
163
|
const error = {
|
|
@@ -402,19 +170,10 @@ export class MemoryMigrator extends EventEmitter {
|
|
|
402
170
|
this.emit('migration:error', error);
|
|
403
171
|
}
|
|
404
172
|
parseTimestamp(value) {
|
|
405
|
-
|
|
406
|
-
return Date.now();
|
|
407
|
-
if (typeof value === 'number') {
|
|
408
|
-
// Handle both milliseconds and seconds
|
|
409
|
-
return value > 1e12 ? value : value * 1000;
|
|
410
|
-
}
|
|
411
|
-
const parsed = Date.parse(value);
|
|
412
|
-
return Number.isNaN(parsed) ? Date.now() : parsed;
|
|
173
|
+
return migrationTransformMethods.parseTimestamp.call(this, value);
|
|
413
174
|
}
|
|
414
175
|
isValidMemoryType(type) {
|
|
415
|
-
|
|
416
|
-
// it is no longer a valid MemoryType (procedural memory removed).
|
|
417
|
-
return ['episodic', 'semantic', 'procedural', 'working', 'cache'].includes(type);
|
|
176
|
+
return migrationTransformMethods.isValidMemoryType.call(this, type);
|
|
418
177
|
}
|
|
419
178
|
estimateTimeRemaining(startTime, completed, total) {
|
|
420
179
|
if (completed === 0)
|
|
@@ -445,78 +204,6 @@ export class MemoryMigrator extends EventEmitter {
|
|
|
445
204
|
}
|
|
446
205
|
return summary;
|
|
447
206
|
}
|
|
448
|
-
async walkDirectory(dir, extension) {
|
|
449
|
-
const files = [];
|
|
450
|
-
try {
|
|
451
|
-
const entries = await fs.readdir(dir, { withFileTypes: true });
|
|
452
|
-
for (const entry of entries) {
|
|
453
|
-
const fullPath = path.join(dir, entry.name);
|
|
454
|
-
if (entry.isDirectory()) {
|
|
455
|
-
const subFiles = await this.walkDirectory(fullPath, extension);
|
|
456
|
-
files.push(...subFiles);
|
|
457
|
-
}
|
|
458
|
-
else if (entry.isFile() && entry.name.endsWith(extension)) {
|
|
459
|
-
files.push(fullPath);
|
|
460
|
-
}
|
|
461
|
-
}
|
|
462
|
-
}
|
|
463
|
-
catch (_error) {
|
|
464
|
-
// Directory doesn't exist or isn't readable
|
|
465
|
-
}
|
|
466
|
-
return files;
|
|
467
|
-
}
|
|
468
|
-
parseMarkdownEntry(filePath, content, basePath) {
|
|
469
|
-
// Extract frontmatter if present
|
|
470
|
-
const frontmatterMatch = content.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
|
|
471
|
-
const metadata = {};
|
|
472
|
-
let body = content;
|
|
473
|
-
if (frontmatterMatch) {
|
|
474
|
-
try {
|
|
475
|
-
// Simple YAML-like parsing
|
|
476
|
-
const frontmatter = frontmatterMatch[1];
|
|
477
|
-
for (const line of frontmatter.split('\n')) {
|
|
478
|
-
const colonIndex = line.indexOf(':');
|
|
479
|
-
if (colonIndex > 0) {
|
|
480
|
-
const key = line.substring(0, colonIndex).trim();
|
|
481
|
-
let value = line.substring(colonIndex + 1).trim();
|
|
482
|
-
// Parse common types
|
|
483
|
-
if (value === 'true')
|
|
484
|
-
value = true;
|
|
485
|
-
else if (value === 'false')
|
|
486
|
-
value = false;
|
|
487
|
-
else if (typeof value === 'string' && /^\d+$/.test(value))
|
|
488
|
-
value = parseInt(value, 10);
|
|
489
|
-
else if (typeof value === 'string' && value.startsWith('[') && value.endsWith(']')) {
|
|
490
|
-
try {
|
|
491
|
-
value = JSON.parse(value.replace(/'/g, '"'));
|
|
492
|
-
}
|
|
493
|
-
catch {
|
|
494
|
-
// Keep as string
|
|
495
|
-
}
|
|
496
|
-
}
|
|
497
|
-
metadata[key] = value;
|
|
498
|
-
}
|
|
499
|
-
}
|
|
500
|
-
body = frontmatterMatch[2];
|
|
501
|
-
}
|
|
502
|
-
catch {
|
|
503
|
-
// Failed to parse frontmatter, use whole content
|
|
504
|
-
}
|
|
505
|
-
}
|
|
506
|
-
// Derive key from file path
|
|
507
|
-
const relativePath = path.relative(basePath, filePath);
|
|
508
|
-
const key = relativePath.replace(/\\/g, '/').replace(/\.md$/, '').replace(/\//g, ':');
|
|
509
|
-
// Derive namespace from directory structure
|
|
510
|
-
const namespace = path.dirname(relativePath).replace(/\\/g, '/') || 'default';
|
|
511
|
-
return {
|
|
512
|
-
key,
|
|
513
|
-
value: body.trim(),
|
|
514
|
-
namespace,
|
|
515
|
-
tags: Array.isArray(metadata.tags) ? metadata.tags : [],
|
|
516
|
-
metadata,
|
|
517
|
-
timestamp: Date.now(),
|
|
518
|
-
};
|
|
519
|
-
}
|
|
520
207
|
}
|
|
521
208
|
/**
|
|
522
209
|
* Convenience function to create a migrator
|