@framers/agentos 0.1.101 → 0.1.102
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -0
- package/dist/memory/config.d.ts +39 -0
- package/dist/memory/config.d.ts.map +1 -1
- package/dist/memory/config.js.map +1 -1
- package/dist/memory/consolidation/ConsolidationLoop.d.ts +177 -0
- package/dist/memory/consolidation/ConsolidationLoop.d.ts.map +1 -0
- package/dist/memory/consolidation/ConsolidationLoop.js +517 -0
- package/dist/memory/consolidation/ConsolidationLoop.js.map +1 -0
- package/dist/memory/consolidation/ConsolidationPipeline.d.ts.map +1 -1
- package/dist/memory/consolidation/ConsolidationPipeline.js +7 -0
- package/dist/memory/consolidation/ConsolidationPipeline.js.map +1 -1
- package/dist/memory/consolidation/index.d.ts +8 -0
- package/dist/memory/consolidation/index.d.ts.map +1 -0
- package/dist/memory/consolidation/index.js +7 -0
- package/dist/memory/consolidation/index.js.map +1 -0
- package/dist/memory/decay/DecayModel.d.ts +33 -0
- package/dist/memory/decay/DecayModel.d.ts.map +1 -1
- package/dist/memory/decay/DecayModel.js +31 -0
- package/dist/memory/decay/DecayModel.js.map +1 -1
- package/dist/memory/facade/Memory.d.ts +228 -0
- package/dist/memory/facade/Memory.d.ts.map +1 -0
- package/dist/memory/facade/Memory.js +823 -0
- package/dist/memory/facade/Memory.js.map +1 -0
- package/dist/memory/facade/index.d.ts +13 -0
- package/dist/memory/facade/index.d.ts.map +1 -0
- package/dist/memory/facade/index.js +11 -0
- package/dist/memory/facade/index.js.map +1 -0
- package/dist/memory/facade/types.d.ts +606 -0
- package/dist/memory/facade/types.d.ts.map +1 -0
- package/dist/memory/facade/types.js +11 -0
- package/dist/memory/facade/types.js.map +1 -0
- package/dist/memory/feedback/RetrievalFeedbackSignal.d.ts +132 -0
- package/dist/memory/feedback/RetrievalFeedbackSignal.d.ts.map +1 -0
- package/dist/memory/feedback/RetrievalFeedbackSignal.js +178 -0
- package/dist/memory/feedback/RetrievalFeedbackSignal.js.map +1 -0
- package/dist/memory/feedback/index.d.ts +13 -0
- package/dist/memory/feedback/index.d.ts.map +1 -0
- package/dist/memory/feedback/index.js +12 -0
- package/dist/memory/feedback/index.js.map +1 -0
- package/dist/memory/index.d.ts +22 -0
- package/dist/memory/index.d.ts.map +1 -1
- package/dist/memory/index.js +24 -0
- package/dist/memory/index.js.map +1 -1
- package/dist/memory/ingestion/ChunkingEngine.d.ts +143 -0
- package/dist/memory/ingestion/ChunkingEngine.d.ts.map +1 -0
- package/dist/memory/ingestion/ChunkingEngine.js +508 -0
- package/dist/memory/ingestion/ChunkingEngine.js.map +1 -0
- package/dist/memory/ingestion/DoclingLoader.d.ts +44 -0
- package/dist/memory/ingestion/DoclingLoader.d.ts.map +1 -0
- package/dist/memory/ingestion/DoclingLoader.js +228 -0
- package/dist/memory/ingestion/DoclingLoader.js.map +1 -0
- package/dist/memory/ingestion/DocxLoader.d.ts +37 -0
- package/dist/memory/ingestion/DocxLoader.d.ts.map +1 -0
- package/dist/memory/ingestion/DocxLoader.js +111 -0
- package/dist/memory/ingestion/DocxLoader.js.map +1 -0
- package/dist/memory/ingestion/FolderScanner.d.ts +116 -0
- package/dist/memory/ingestion/FolderScanner.d.ts.map +1 -0
- package/dist/memory/ingestion/FolderScanner.js +127 -0
- package/dist/memory/ingestion/FolderScanner.js.map +1 -0
- package/dist/memory/ingestion/HtmlLoader.d.ts +49 -0
- package/dist/memory/ingestion/HtmlLoader.d.ts.map +1 -0
- package/dist/memory/ingestion/HtmlLoader.js +202 -0
- package/dist/memory/ingestion/HtmlLoader.js.map +1 -0
- package/dist/memory/ingestion/IDocumentLoader.d.ts +63 -0
- package/dist/memory/ingestion/IDocumentLoader.d.ts.map +1 -0
- package/dist/memory/ingestion/IDocumentLoader.js +11 -0
- package/dist/memory/ingestion/IDocumentLoader.js.map +1 -0
- package/dist/memory/ingestion/LoaderRegistry.d.ts +140 -0
- package/dist/memory/ingestion/LoaderRegistry.d.ts.map +1 -0
- package/dist/memory/ingestion/LoaderRegistry.js +229 -0
- package/dist/memory/ingestion/LoaderRegistry.js.map +1 -0
- package/dist/memory/ingestion/MarkdownLoader.d.ts +50 -0
- package/dist/memory/ingestion/MarkdownLoader.d.ts.map +1 -0
- package/dist/memory/ingestion/MarkdownLoader.js +169 -0
- package/dist/memory/ingestion/MarkdownLoader.js.map +1 -0
- package/dist/memory/ingestion/MultimodalAggregator.d.ts +88 -0
- package/dist/memory/ingestion/MultimodalAggregator.d.ts.map +1 -0
- package/dist/memory/ingestion/MultimodalAggregator.js +96 -0
- package/dist/memory/ingestion/MultimodalAggregator.js.map +1 -0
- package/dist/memory/ingestion/OcrPdfLoader.d.ts +41 -0
- package/dist/memory/ingestion/OcrPdfLoader.d.ts.map +1 -0
- package/dist/memory/ingestion/OcrPdfLoader.js +149 -0
- package/dist/memory/ingestion/OcrPdfLoader.js.map +1 -0
- package/dist/memory/ingestion/PdfLoader.d.ts +78 -0
- package/dist/memory/ingestion/PdfLoader.d.ts.map +1 -0
- package/dist/memory/ingestion/PdfLoader.js +179 -0
- package/dist/memory/ingestion/PdfLoader.js.map +1 -0
- package/dist/memory/ingestion/TextLoader.d.ts +66 -0
- package/dist/memory/ingestion/TextLoader.d.ts.map +1 -0
- package/dist/memory/ingestion/TextLoader.js +207 -0
- package/dist/memory/ingestion/TextLoader.js.map +1 -0
- package/dist/memory/ingestion/UrlLoader.d.ts +95 -0
- package/dist/memory/ingestion/UrlLoader.d.ts.map +1 -0
- package/dist/memory/ingestion/UrlLoader.js +174 -0
- package/dist/memory/ingestion/UrlLoader.js.map +1 -0
- package/dist/memory/io/ChatGptImporter.d.ts +85 -0
- package/dist/memory/io/ChatGptImporter.d.ts.map +1 -0
- package/dist/memory/io/ChatGptImporter.js +231 -0
- package/dist/memory/io/ChatGptImporter.js.map +1 -0
- package/dist/memory/io/JsonExporter.d.ts +67 -0
- package/dist/memory/io/JsonExporter.d.ts.map +1 -0
- package/dist/memory/io/JsonExporter.js +132 -0
- package/dist/memory/io/JsonExporter.js.map +1 -0
- package/dist/memory/io/JsonImporter.d.ts +84 -0
- package/dist/memory/io/JsonImporter.d.ts.map +1 -0
- package/dist/memory/io/JsonImporter.js +234 -0
- package/dist/memory/io/JsonImporter.js.map +1 -0
- package/dist/memory/io/MarkdownExporter.d.ts +95 -0
- package/dist/memory/io/MarkdownExporter.d.ts.map +1 -0
- package/dist/memory/io/MarkdownExporter.js +130 -0
- package/dist/memory/io/MarkdownExporter.js.map +1 -0
- package/dist/memory/io/MarkdownImporter.d.ts +84 -0
- package/dist/memory/io/MarkdownImporter.d.ts.map +1 -0
- package/dist/memory/io/MarkdownImporter.js +166 -0
- package/dist/memory/io/MarkdownImporter.js.map +1 -0
- package/dist/memory/io/ObsidianExporter.d.ts +80 -0
- package/dist/memory/io/ObsidianExporter.d.ts.map +1 -0
- package/dist/memory/io/ObsidianExporter.js +127 -0
- package/dist/memory/io/ObsidianExporter.js.map +1 -0
- package/dist/memory/io/ObsidianImporter.d.ts +93 -0
- package/dist/memory/io/ObsidianImporter.d.ts.map +1 -0
- package/dist/memory/io/ObsidianImporter.js +221 -0
- package/dist/memory/io/ObsidianImporter.js.map +1 -0
- package/dist/memory/io/SqliteExporter.d.ts +47 -0
- package/dist/memory/io/SqliteExporter.d.ts.map +1 -0
- package/dist/memory/io/SqliteExporter.js +56 -0
- package/dist/memory/io/SqliteExporter.js.map +1 -0
- package/dist/memory/io/SqliteImporter.d.ts +82 -0
- package/dist/memory/io/SqliteImporter.d.ts.map +1 -0
- package/dist/memory/io/SqliteImporter.js +232 -0
- package/dist/memory/io/SqliteImporter.js.map +1 -0
- package/dist/memory/io/index.d.ts +31 -0
- package/dist/memory/io/index.d.ts.map +1 -0
- package/dist/memory/io/index.js +31 -0
- package/dist/memory/io/index.js.map +1 -0
- package/dist/memory/store/SqliteBrain.d.ts +125 -0
- package/dist/memory/store/SqliteBrain.d.ts.map +1 -0
- package/dist/memory/store/SqliteBrain.js +407 -0
- package/dist/memory/store/SqliteBrain.js.map +1 -0
- package/dist/memory/store/SqliteKnowledgeGraph.d.ts +259 -0
- package/dist/memory/store/SqliteKnowledgeGraph.d.ts.map +1 -0
- package/dist/memory/store/SqliteKnowledgeGraph.js +1062 -0
- package/dist/memory/store/SqliteKnowledgeGraph.js.map +1 -0
- package/dist/memory/store/SqliteMemoryGraph.d.ts +251 -0
- package/dist/memory/store/SqliteMemoryGraph.d.ts.map +1 -0
- package/dist/memory/store/SqliteMemoryGraph.js +637 -0
- package/dist/memory/store/SqliteMemoryGraph.js.map +1 -0
- package/dist/memory/tools/MemoryAddTool.d.ts +98 -0
- package/dist/memory/tools/MemoryAddTool.d.ts.map +1 -0
- package/dist/memory/tools/MemoryAddTool.js +131 -0
- package/dist/memory/tools/MemoryAddTool.js.map +1 -0
- package/dist/memory/tools/MemoryDeleteTool.d.ts +83 -0
- package/dist/memory/tools/MemoryDeleteTool.d.ts.map +1 -0
- package/dist/memory/tools/MemoryDeleteTool.js +96 -0
- package/dist/memory/tools/MemoryDeleteTool.js.map +1 -0
- package/dist/memory/tools/MemoryMergeTool.d.ts +95 -0
- package/dist/memory/tools/MemoryMergeTool.d.ts.map +1 -0
- package/dist/memory/tools/MemoryMergeTool.js +164 -0
- package/dist/memory/tools/MemoryMergeTool.js.map +1 -0
- package/dist/memory/tools/MemoryReflectTool.d.ts +86 -0
- package/dist/memory/tools/MemoryReflectTool.d.ts.map +1 -0
- package/dist/memory/tools/MemoryReflectTool.js +102 -0
- package/dist/memory/tools/MemoryReflectTool.js.map +1 -0
- package/dist/memory/tools/MemorySearchTool.d.ts +117 -0
- package/dist/memory/tools/MemorySearchTool.d.ts.map +1 -0
- package/dist/memory/tools/MemorySearchTool.js +162 -0
- package/dist/memory/tools/MemorySearchTool.js.map +1 -0
- package/dist/memory/tools/MemoryUpdateTool.d.ts +92 -0
- package/dist/memory/tools/MemoryUpdateTool.d.ts.map +1 -0
- package/dist/memory/tools/MemoryUpdateTool.js +125 -0
- package/dist/memory/tools/MemoryUpdateTool.js.map +1 -0
- package/dist/memory/tools/index.d.ts +32 -0
- package/dist/memory/tools/index.d.ts.map +1 -0
- package/dist/memory/tools/index.js +26 -0
- package/dist/memory/tools/index.js.map +1 -0
- package/package.json +6 -1
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Obsidian vault importer for AgentOS memory brain.
|
|
3
|
+
*
|
|
4
|
+
* Extends `MarkdownImporter` with Obsidian-specific parsing:
|
|
5
|
+
*
|
|
6
|
+
* 1. **`[[wikilinks]]`** — each `[[Target Note]]` (or `[[Target|Alias]]`) in
|
|
7
|
+
* a note's body is parsed. For each wikilink, the importer looks up (or
|
|
8
|
+
* creates) a `knowledge_nodes` entry for the target label and then creates
|
|
9
|
+
* a `knowledge_edges` row of type `'related_to'` linking the source trace
|
|
10
|
+
* node to the target node.
|
|
11
|
+
*
|
|
12
|
+
* 2. **`#tags`** — inline hashtags are extracted from the body and merged
|
|
13
|
+
* into the trace's `tags` JSON column (in addition to any front-matter tags).
|
|
14
|
+
*
|
|
15
|
+
* 3. **`![[image.png]]`** — embedded-image syntax is detected and a warning
|
|
16
|
+
* is logged. Embedded images are not imported in the current version.
|
|
17
|
+
*
|
|
18
|
+
* @module memory/io/ObsidianImporter
|
|
19
|
+
*/
|
|
20
|
+
import crypto from 'node:crypto';
|
|
21
|
+
import { v4 as uuidv4 } from 'uuid';
|
|
22
|
+
import { MarkdownImporter } from './MarkdownImporter.js';
|
|
23
|
+
// ---------------------------------------------------------------------------
|
|
24
|
+
// Regex constants
|
|
25
|
+
// ---------------------------------------------------------------------------
|
|
26
|
+
/**
|
|
27
|
+
* Matches Obsidian wikilinks: `[[Target]]` or `[[Target|Alias]]`.
|
|
28
|
+
* Capture group 1 is the target note name; alias (if any) is ignored.
|
|
29
|
+
* Does NOT match embedded images (`![[...]]`) — those use a separate pattern.
|
|
30
|
+
*/
|
|
31
|
+
const WIKILINK_RE = /(?<!!)\[\[([^\]|]+)(?:\|[^\]]+)?\]\]/g;
|
|
32
|
+
/**
|
|
33
|
+
* Matches Obsidian embedded images: `![[image.png]]`.
|
|
34
|
+
* Used to emit a warning — embedded images are not yet supported for import.
|
|
35
|
+
*/
|
|
36
|
+
const EMBED_RE = /!\[\[[^\]]+\]\]/g;
|
|
37
|
+
/**
|
|
38
|
+
* Matches inline hashtags: `#tagName` (not preceded by `[` or `#`).
|
|
39
|
+
* Only captures the tag name (group 1) without the leading `#`.
|
|
40
|
+
*/
|
|
41
|
+
const HASHTAG_RE = /(?<![[\#])#([\w-]+)/g;
|
|
42
|
+
// ---------------------------------------------------------------------------
|
|
43
|
+
// ObsidianImporter
|
|
44
|
+
// ---------------------------------------------------------------------------
|
|
45
|
+
/**
|
|
46
|
+
* Imports an Obsidian vault (directory of Markdown files) into a `SqliteBrain`.
|
|
47
|
+
*
|
|
48
|
+
* **Usage:**
|
|
49
|
+
* ```ts
|
|
50
|
+
* const importer = new ObsidianImporter(brain);
|
|
51
|
+
* const result = await importer.import('/path/to/obsidian-vault');
|
|
52
|
+
* ```
|
|
53
|
+
*/
|
|
54
|
+
export class ObsidianImporter extends MarkdownImporter {
|
|
55
|
+
/**
|
|
56
|
+
* @param brain - The target `SqliteBrain` to import into.
|
|
57
|
+
*/
|
|
58
|
+
constructor(brain) {
|
|
59
|
+
super(brain);
|
|
60
|
+
}
|
|
61
|
+
// -------------------------------------------------------------------------
|
|
62
|
+
// Overridden hook
|
|
63
|
+
// -------------------------------------------------------------------------
|
|
64
|
+
/**
|
|
65
|
+
* Post-process a successfully imported Markdown file:
|
|
66
|
+
*
|
|
67
|
+
* 1. Warn about any embedded images (`![[...]]`).
|
|
68
|
+
* 2. Extract inline `#hashtags` and merge them into the trace's tag list.
|
|
69
|
+
* 3. Parse `[[wikilinks]]` and create `knowledge_edges` entries.
|
|
70
|
+
*
|
|
71
|
+
* @param _filePath - Absolute path of the source file (unused here).
|
|
72
|
+
* @param _frontmatter - Parsed front-matter data.
|
|
73
|
+
* @param body - Markdown body (content after front-matter).
|
|
74
|
+
* @param result - Mutable `ImportResult` accumulator.
|
|
75
|
+
* @param traceId - The ID of the just-inserted trace.
|
|
76
|
+
*/
|
|
77
|
+
async postProcess(_filePath, _frontmatter, body, result, traceId) {
|
|
78
|
+
// ---- 1. Warn about embedded images ----
|
|
79
|
+
const embedMatches = body.match(EMBED_RE);
|
|
80
|
+
if (embedMatches && embedMatches.length > 0) {
|
|
81
|
+
console.warn(`[ObsidianImporter] Embedded images are not yet supported in import. ` +
|
|
82
|
+
`Found ${embedMatches.length} embed(s) in trace ${traceId}.`);
|
|
83
|
+
}
|
|
84
|
+
// ---- 2. Extract inline hashtags and persist to trace ----
|
|
85
|
+
const inlineTags = [];
|
|
86
|
+
let tagMatch;
|
|
87
|
+
HASHTAG_RE.lastIndex = 0;
|
|
88
|
+
while ((tagMatch = HASHTAG_RE.exec(body)) !== null) {
|
|
89
|
+
if (tagMatch[1])
|
|
90
|
+
inlineTags.push(tagMatch[1]);
|
|
91
|
+
}
|
|
92
|
+
if (inlineTags.length > 0) {
|
|
93
|
+
this._mergeTagsIntoTrace(traceId, inlineTags, result);
|
|
94
|
+
}
|
|
95
|
+
// ---- 3. Parse wikilinks and create knowledge_edges ----
|
|
96
|
+
const wikiTargets = [];
|
|
97
|
+
let wikilinkMatch;
|
|
98
|
+
WIKILINK_RE.lastIndex = 0;
|
|
99
|
+
while ((wikilinkMatch = WIKILINK_RE.exec(body)) !== null) {
|
|
100
|
+
if (wikilinkMatch[1])
|
|
101
|
+
wikiTargets.push(wikilinkMatch[1].trim());
|
|
102
|
+
}
|
|
103
|
+
for (const target of wikiTargets) {
|
|
104
|
+
this._upsertWikiEdge(traceId, target, result);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
// -------------------------------------------------------------------------
|
|
108
|
+
// Private helpers
|
|
109
|
+
// -------------------------------------------------------------------------
|
|
110
|
+
/**
|
|
111
|
+
* Merge a list of inline hashtag names into a trace's `tags` JSON column.
|
|
112
|
+
*
|
|
113
|
+
* Reads the current tags array, deduplicates, and writes back.
|
|
114
|
+
*
|
|
115
|
+
* @param traceId - ID of the trace to update.
|
|
116
|
+
* @param newTags - Hashtag names to add (without the leading `#`).
|
|
117
|
+
* @param result - Mutable result accumulator (errors recorded here).
|
|
118
|
+
*/
|
|
119
|
+
_mergeTagsIntoTrace(traceId, newTags, result) {
|
|
120
|
+
try {
|
|
121
|
+
const db = this.brain.db;
|
|
122
|
+
const row = db
|
|
123
|
+
.prepare('SELECT tags FROM memory_traces WHERE id = ?')
|
|
124
|
+
.get(traceId);
|
|
125
|
+
if (!row)
|
|
126
|
+
return;
|
|
127
|
+
let existing = [];
|
|
128
|
+
try {
|
|
129
|
+
existing = JSON.parse(row.tags);
|
|
130
|
+
}
|
|
131
|
+
catch {
|
|
132
|
+
existing = [];
|
|
133
|
+
}
|
|
134
|
+
const merged = Array.from(new Set([...existing, ...newTags]));
|
|
135
|
+
db.prepare('UPDATE memory_traces SET tags = ? WHERE id = ?').run(JSON.stringify(merged), traceId);
|
|
136
|
+
}
|
|
137
|
+
catch (err) {
|
|
138
|
+
result.errors.push(`Tag merge error for trace ${traceId}: ${String(err)}`);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Ensure `knowledge_nodes` entries exist for both the source trace and the
|
|
143
|
+
* target label, then create a `knowledge_edges` row (type `'related_to'`)
|
|
144
|
+
* linking them.
|
|
145
|
+
*
|
|
146
|
+
* Because `knowledge_edges.source_id` has a FK reference to
|
|
147
|
+
* `knowledge_nodes(id)`, we first upsert a node for the source trace (using
|
|
148
|
+
* the trace content as the label) before creating the edge. This lets
|
|
149
|
+
* Obsidian's graph view visualise which note links to which concept.
|
|
150
|
+
*
|
|
151
|
+
* Both node upserts and the edge insert use `INSERT OR IGNORE` so repeated
|
|
152
|
+
* imports don't create duplicates.
|
|
153
|
+
*
|
|
154
|
+
* @param sourceTraceId - The memory trace ID that contains the wikilink.
|
|
155
|
+
* @param targetLabel - The label of the linked note (wikilink target).
|
|
156
|
+
* @param result - Mutable result accumulator.
|
|
157
|
+
*/
|
|
158
|
+
_upsertWikiEdge(sourceTraceId, targetLabel, result) {
|
|
159
|
+
try {
|
|
160
|
+
// Access brain.db through the protected field inherited from MarkdownImporter.
|
|
161
|
+
const db = this.brain.db;
|
|
162
|
+
// ---- Upsert source knowledge node for the trace ----
|
|
163
|
+
// We use the trace ID itself as the node label so the graph stays navigable.
|
|
164
|
+
const sourceLabel = `trace:${sourceTraceId}`;
|
|
165
|
+
const sourceHash = crypto
|
|
166
|
+
.createHash('sha256')
|
|
167
|
+
.update(`wiki-source::${sourceTraceId}`)
|
|
168
|
+
.digest('hex');
|
|
169
|
+
let sourceNodeId;
|
|
170
|
+
const existingSource = db
|
|
171
|
+
.prepare(`SELECT id FROM knowledge_nodes WHERE label = ? LIMIT 1`)
|
|
172
|
+
.get(sourceLabel);
|
|
173
|
+
if (existingSource) {
|
|
174
|
+
sourceNodeId = existingSource.id;
|
|
175
|
+
}
|
|
176
|
+
else {
|
|
177
|
+
sourceNodeId = `kn_${uuidv4()}`;
|
|
178
|
+
db.prepare(`INSERT OR IGNORE INTO knowledge_nodes
|
|
179
|
+
(id, type, label, properties, embedding, confidence, source, created_at)
|
|
180
|
+
VALUES (?, 'trace', ?, ?, NULL, 1.0, '{}', ?)`).run(sourceNodeId, sourceLabel, JSON.stringify({ import_hash: sourceHash, trace_id: sourceTraceId }), Date.now());
|
|
181
|
+
}
|
|
182
|
+
// ---- Upsert target knowledge node for the wikilink label ----
|
|
183
|
+
const targetHash = crypto
|
|
184
|
+
.createHash('sha256')
|
|
185
|
+
.update(`wiki::${targetLabel}`)
|
|
186
|
+
.digest('hex');
|
|
187
|
+
let targetNodeId;
|
|
188
|
+
const existingTarget = db
|
|
189
|
+
.prepare(`SELECT id FROM knowledge_nodes WHERE label = ? LIMIT 1`)
|
|
190
|
+
.get(targetLabel);
|
|
191
|
+
if (existingTarget) {
|
|
192
|
+
targetNodeId = existingTarget.id;
|
|
193
|
+
}
|
|
194
|
+
else {
|
|
195
|
+
targetNodeId = `kn_${uuidv4()}`;
|
|
196
|
+
db.prepare(`INSERT OR IGNORE INTO knowledge_nodes
|
|
197
|
+
(id, type, label, properties, embedding, confidence, source, created_at)
|
|
198
|
+
VALUES (?, 'concept', ?, ?, NULL, 1.0, '{}', ?)`).run(targetNodeId, targetLabel, JSON.stringify({ import_hash: targetHash, obsidian_wikilink: true }), Date.now());
|
|
199
|
+
}
|
|
200
|
+
// ---- Create the directed edge: source node → target node ----
|
|
201
|
+
const edgeHash = crypto
|
|
202
|
+
.createHash('sha256')
|
|
203
|
+
.update(`${sourceNodeId}::${targetNodeId}::related_to`)
|
|
204
|
+
.digest('hex');
|
|
205
|
+
// Check for existing edge before insert (extra safety beyond OR IGNORE).
|
|
206
|
+
const existingEdge = db
|
|
207
|
+
.prepare(`SELECT id FROM knowledge_edges
|
|
208
|
+
WHERE json_extract(metadata, '$.import_hash') = ? LIMIT 1`)
|
|
209
|
+
.get(edgeHash);
|
|
210
|
+
if (!existingEdge) {
|
|
211
|
+
db.prepare(`INSERT OR IGNORE INTO knowledge_edges
|
|
212
|
+
(id, source_id, target_id, type, weight, bidirectional, metadata, created_at)
|
|
213
|
+
VALUES (?, ?, ?, 'related_to', 1.0, 0, ?, ?)`).run(`ke_${uuidv4()}`, sourceNodeId, targetNodeId, JSON.stringify({ import_hash: edgeHash, source: 'obsidian_wikilink', trace_id: sourceTraceId }), Date.now());
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
catch (err) {
|
|
217
|
+
result.errors.push(`Wikilink edge error (${sourceTraceId} → "${targetLabel}"): ${String(err)}`);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
//# sourceMappingURL=ObsidianImporter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ObsidianImporter.js","sourceRoot":"","sources":["../../../src/memory/io/ObsidianImporter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AAGpC,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAEzD,8EAA8E;AAC9E,kBAAkB;AAClB,8EAA8E;AAE9E;;;;GAIG;AACH,MAAM,WAAW,GAAG,uCAAuC,CAAC;AAE5D;;;GAGG;AACH,MAAM,QAAQ,GAAG,kBAAkB,CAAC;AAEpC;;;GAGG;AACH,MAAM,UAAU,GAAG,sBAAsB,CAAC;AAmB1C,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E;;;;;;;;GAQG;AACH,MAAM,OAAO,gBAAiB,SAAQ,gBAAgB;IACpD;;OAEG;IACH,YAAY,KAAkB;QAC5B,KAAK,CAAC,KAAK,CAAC,CAAC;IACf,CAAC;IAED,4EAA4E;IAC5E,kBAAkB;IAClB,4EAA4E;IAE5E;;;;;;;;;;;;OAYG;IACgB,KAAK,CAAC,WAAW,CAClC,SAAiB,EACjB,YAA8B,EAC9B,IAAY,EACZ,MAAoB,EACpB,OAAe;QAEf,0CAA0C;QAC1C,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC1C,IAAI,YAAY,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5C,OAAO,CAAC,IAAI,CACV,sEAAsE;gBACpE,SAAS,YAAY,CAAC,MAAM,sBAAsB,OAAO,GAAG,CAC/D,CAAC;QACJ,CAAC;QAED,4DAA4D;QAC5D,MAAM,UAAU,GAAa,EAAE,CAAC;QAChC,IAAI,QAAgC,CAAC;QACrC,UAAU,CAAC,SAAS,GAAG,CAAC,CAAC;QACzB,OAAO,CAAC,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACnD,IAAI,QAAQ,CAAC,CAAC,CAAC;gBAAE,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QAChD,CAAC;QAED,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;QACxD,CAAC;QAED,0DAA0D;QAC1D,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,IAAI,aAAqC,CAAC;QAC1C,WAAW,CAAC,SAAS,GAAG,CAAC,CAAC;QAC1B,OAAO,CAAC,aAAa,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACzD,IAAI,aAAa,CAAC,CAAC,CAAC;gBAAE,WAAW,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAClE,CAAC;QAED,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE,CAAC;YACjC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED,4EAA4E;IAC5E,kBAAkB;IAClB,4EAA4E;IAE5E;;;;;;;;OAQG;IACK,mBAAmB,CACzB,OAAe,EACf,OAAiB,EACjB,MAAoB;QAEpB,IAAI,CAAC;YACH,MAAM,EAAE,GAAI,IAA0C,CAAC,KAAK,CAAC,EAAE,CAAC;YAEhE,MAAM,GAAG,GAAG,EAAE;iBACX,OAAO,CAA6B,6CAA6C,CAAC;iBAClF,GAAG,CAAC,OAAO,CAAC,CAAC;YAEhB,IAAI,CAAC,GAAG;gBAAE,OAAO;YAEjB,IAAI,QAAQ,GAAa,EAAE,CAAC;YAC5B,IAAI,CAAC;gBACH,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAa,CAAC;YAC9C,CAAC;YAAC,MAAM,CAAC;gBACP,QAAQ,GAAG,EAAE,CAAC;YAChB,CAAC;YAED,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YAE9D,EAAE,CAAC,OAAO,CAAC,gDAAgD,CAAC,CAAC,GAAG,CAC9D,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EACtB,OAAO,CACR,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,OAAO,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACK,eAAe,CACrB,aAAqB,EACrB,WAAmB,EACnB,MAAoB;QAEpB,IAAI,CAAC;YACH,+EAA+E;YAC/E,MAAM,EAAE,GAAI,IAA0C,CAAC,KAAK,CAAC,EAAE,CAAC;YAEhE,uDAAuD;YACvD,6EAA6E;YAC7E,MAAM,WAAW,GAAG,SAAS,aAAa,EAAE,CAAC;YAC7C,MAAM,UAAU,GAAG,MAAM;iBACtB,UAAU,CAAC,QAAQ,CAAC;iBACpB,MAAM,CAAC,gBAAgB,aAAa,EAAE,CAAC;iBACvC,MAAM,CAAC,KAAK,CAAC,CAAC;YAEjB,IAAI,YAAoB,CAAC;YACzB,MAAM,cAAc,GAAG,EAAE;iBACtB,OAAO,CACN,wDAAwD,CACzD;iBACA,GAAG,CAAC,WAAW,CAAC,CAAC;YAEpB,IAAI,cAAc,EAAE,CAAC;gBACnB,YAAY,GAAG,cAAc,CAAC,EAAE,CAAC;YACnC,CAAC;iBAAM,CAAC;gBACN,YAAY,GAAG,MAAM,MAAM,EAAE,EAAE,CAAC;gBAChC,EAAE,CAAC,OAAO,CACR;;yDAE+C,CAChD,CAAC,GAAG,CACH,YAAY,EACZ,WAAW,EACX,IAAI,CAAC,SAAS,CAAC,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,aAAa,EAAE,CAAC,EACpE,IAAI,CAAC,GAAG,EAAE,CACX,CAAC;YACJ,CAAC;YAED,gEAAgE;YAChE,MAAM,UAAU,GAAG,MAAM;iBACtB,UAAU,CAAC,QAAQ,CAAC;iBACpB,MAAM,CAAC,SAAS,WAAW,EAAE,CAAC;iBAC9B,MAAM,CAAC,KAAK,CAAC,CAAC;YAEjB,IAAI,YAAoB,CAAC;YACzB,MAAM,cAAc,GAAG,EAAE;iBACtB,OAAO,CACN,wDAAwD,CACzD;iBACA,GAAG,CAAC,WAAW,CAAC,CAAC;YAEpB,IAAI,cAAc,EAAE,CAAC;gBACnB,YAAY,GAAG,cAAc,CAAC,EAAE,CAAC;YACnC,CAAC;iBAAM,CAAC;gBACN,YAAY,GAAG,MAAM,MAAM,EAAE,EAAE,CAAC;gBAChC,EAAE,CAAC,OAAO,CACR;;2DAEiD,CAClD,CAAC,GAAG,CACH,YAAY,EACZ,WAAW,EACX,IAAI,CAAC,SAAS,CAAC,EAAE,WAAW,EAAE,UAAU,EAAE,iBAAiB,EAAE,IAAI,EAAE,CAAC,EACpE,IAAI,CAAC,GAAG,EAAE,CACX,CAAC;YACJ,CAAC;YAED,gEAAgE;YAChE,MAAM,QAAQ,GAAG,MAAM;iBACpB,UAAU,CAAC,QAAQ,CAAC;iBACpB,MAAM,CAAC,GAAG,YAAY,KAAK,YAAY,cAAc,CAAC;iBACtD,MAAM,CAAC,KAAK,CAAC,CAAC;YAEjB,yEAAyE;YACzE,MAAM,YAAY,GAAG,EAAE;iBACpB,OAAO,CACN;qEAC2D,CAC5D;iBACA,GAAG,CAAC,QAAQ,CAAC,CAAC;YAEjB,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,EAAE,CAAC,OAAO,CACR;;wDAE8C,CAC/C,CAAC,GAAG,CACH,MAAM,MAAM,EAAE,EAAE,EAChB,YAAY,EACZ,YAAY,EACZ,IAAI,CAAC,SAAS,CAAC,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,EAAE,mBAAmB,EAAE,QAAQ,EAAE,aAAa,EAAE,CAAC,EAC/F,IAAI,CAAC,GAAG,EAAE,CACX,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,MAAM,CAAC,IAAI,CAChB,wBAAwB,aAAa,OAAO,WAAW,OAAO,MAAM,CAAC,GAAG,CAAC,EAAE,CAC5E,CAAC;QACJ,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview SQLite exporter for AgentOS memory brain.
|
|
3
|
+
*
|
|
4
|
+
* Provides a full-fidelity backup of the `SqliteBrain` SQLite file by copying
|
|
5
|
+
* the database file to a specified output path. This is the highest-fidelity
|
|
6
|
+
* export format — it preserves all tables, indexes, and metadata exactly.
|
|
7
|
+
*
|
|
8
|
+
* Because `better-sqlite3` keeps the file open in WAL mode, we use the
|
|
9
|
+
* built-in `VACUUM INTO` SQL command (SQLite 3.27+) which atomically creates
|
|
10
|
+
* a clean, fully checkpointed copy without any WAL sidecar file.
|
|
11
|
+
*
|
|
12
|
+
* @module memory/io/SqliteExporter
|
|
13
|
+
*/
|
|
14
|
+
import type { ExportOptions } from '../facade/types.js';
|
|
15
|
+
import type { SqliteBrain } from '../store/SqliteBrain.js';
|
|
16
|
+
/**
|
|
17
|
+
* Exports a `SqliteBrain` as a portable SQLite file.
|
|
18
|
+
*
|
|
19
|
+
* **Usage:**
|
|
20
|
+
* ```ts
|
|
21
|
+
* const exporter = new SqliteExporter(brain);
|
|
22
|
+
* await exporter.export('/path/to/backup.sqlite');
|
|
23
|
+
* ```
|
|
24
|
+
*/
|
|
25
|
+
export declare class SqliteExporter {
|
|
26
|
+
private readonly brain;
|
|
27
|
+
/**
|
|
28
|
+
* @param brain - The `SqliteBrain` instance to export.
|
|
29
|
+
*/
|
|
30
|
+
constructor(brain: SqliteBrain);
|
|
31
|
+
/**
|
|
32
|
+
* Copy the brain database to `outputPath`.
|
|
33
|
+
*
|
|
34
|
+
* Uses `VACUUM INTO` which:
|
|
35
|
+
* - Checkpoints all WAL frames into the output file.
|
|
36
|
+
* - Creates a compact, defragmented copy (no `-wal` or `-shm` sidecar).
|
|
37
|
+
* - Is safe to run while the database is open and being written to.
|
|
38
|
+
*
|
|
39
|
+
* The parent directory of `outputPath` must already exist.
|
|
40
|
+
*
|
|
41
|
+
* @param outputPath - Absolute path for the SQLite backup file.
|
|
42
|
+
* @param _options - Export options (unused — SQLite export always includes
|
|
43
|
+
* all data including embeddings).
|
|
44
|
+
*/
|
|
45
|
+
export(outputPath: string, _options?: ExportOptions): Promise<void>;
|
|
46
|
+
}
|
|
47
|
+
//# sourceMappingURL=SqliteExporter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SqliteExporter.d.ts","sourceRoot":"","sources":["../../../src/memory/io/SqliteExporter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAM3D;;;;;;;;GAQG;AACH,qBAAa,cAAc;IAIb,OAAO,CAAC,QAAQ,CAAC,KAAK;IAHlC;;OAEG;gBAC0B,KAAK,EAAE,WAAW;IAM/C;;;;;;;;;;;;;OAaG;IACG,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;CAK1E"}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview SQLite exporter for AgentOS memory brain.
|
|
3
|
+
*
|
|
4
|
+
* Provides a full-fidelity backup of the `SqliteBrain` SQLite file by copying
|
|
5
|
+
* the database file to a specified output path. This is the highest-fidelity
|
|
6
|
+
* export format — it preserves all tables, indexes, and metadata exactly.
|
|
7
|
+
*
|
|
8
|
+
* Because `better-sqlite3` keeps the file open in WAL mode, we use the
|
|
9
|
+
* built-in `VACUUM INTO` SQL command (SQLite 3.27+) which atomically creates
|
|
10
|
+
* a clean, fully checkpointed copy without any WAL sidecar file.
|
|
11
|
+
*
|
|
12
|
+
* @module memory/io/SqliteExporter
|
|
13
|
+
*/
|
|
14
|
+
// ---------------------------------------------------------------------------
|
|
15
|
+
// SqliteExporter
|
|
16
|
+
// ---------------------------------------------------------------------------
|
|
17
|
+
/**
|
|
18
|
+
* Exports a `SqliteBrain` as a portable SQLite file.
|
|
19
|
+
*
|
|
20
|
+
* **Usage:**
|
|
21
|
+
* ```ts
|
|
22
|
+
* const exporter = new SqliteExporter(brain);
|
|
23
|
+
* await exporter.export('/path/to/backup.sqlite');
|
|
24
|
+
* ```
|
|
25
|
+
*/
|
|
26
|
+
export class SqliteExporter {
|
|
27
|
+
/**
|
|
28
|
+
* @param brain - The `SqliteBrain` instance to export.
|
|
29
|
+
*/
|
|
30
|
+
constructor(brain) {
|
|
31
|
+
this.brain = brain;
|
|
32
|
+
}
|
|
33
|
+
// -------------------------------------------------------------------------
|
|
34
|
+
// Public API
|
|
35
|
+
// -------------------------------------------------------------------------
|
|
36
|
+
/**
|
|
37
|
+
* Copy the brain database to `outputPath`.
|
|
38
|
+
*
|
|
39
|
+
* Uses `VACUUM INTO` which:
|
|
40
|
+
* - Checkpoints all WAL frames into the output file.
|
|
41
|
+
* - Creates a compact, defragmented copy (no `-wal` or `-shm` sidecar).
|
|
42
|
+
* - Is safe to run while the database is open and being written to.
|
|
43
|
+
*
|
|
44
|
+
* The parent directory of `outputPath` must already exist.
|
|
45
|
+
*
|
|
46
|
+
* @param outputPath - Absolute path for the SQLite backup file.
|
|
47
|
+
* @param _options - Export options (unused — SQLite export always includes
|
|
48
|
+
* all data including embeddings).
|
|
49
|
+
*/
|
|
50
|
+
async export(outputPath, _options) {
|
|
51
|
+
// VACUUM INTO creates a compact, defragmented copy of the live database.
|
|
52
|
+
// It is an atomic operation from SQLite's perspective.
|
|
53
|
+
this.brain.db.exec(`VACUUM INTO '${outputPath.replace(/'/g, "''")}'`);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
//# sourceMappingURL=SqliteExporter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SqliteExporter.js","sourceRoot":"","sources":["../../../src/memory/io/SqliteExporter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAKH,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E;AAE9E;;;;;;;;GAQG;AACH,MAAM,OAAO,cAAc;IACzB;;OAEG;IACH,YAA6B,KAAkB;QAAlB,UAAK,GAAL,KAAK,CAAa;IAAG,CAAC;IAEnD,4EAA4E;IAC5E,aAAa;IACb,4EAA4E;IAE5E;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,MAAM,CAAC,UAAkB,EAAE,QAAwB;QACvD,yEAAyE;QACzE,uDAAuD;QACvD,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,gBAAgB,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IACxE,CAAC;CACF"}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview SQLite importer for AgentOS memory brain.
|
|
3
|
+
*
|
|
4
|
+
* Opens a source SQLite file (exported by `SqliteExporter` or any compatible
|
|
5
|
+
* AgentOS brain) as a separate `better-sqlite3` connection, reads all data
|
|
6
|
+
* tables, and merges them into the target `SqliteBrain`.
|
|
7
|
+
*
|
|
8
|
+
* ## Merge strategy
|
|
9
|
+
* - **memory_traces**: deduplicated by SHA-256 of `content`.
|
|
10
|
+
* - If a trace with the same hash already exists in the target:
|
|
11
|
+
* - Keep the newer `created_at` / `last_accessed` timestamp.
|
|
12
|
+
* - Merge `tags` arrays (union, dedup).
|
|
13
|
+
* - New traces are inserted wholesale.
|
|
14
|
+
* - **knowledge_nodes**: deduplicated by `label` + `type`.
|
|
15
|
+
* - New nodes are inserted; existing nodes are left unchanged.
|
|
16
|
+
* - **knowledge_edges**: deduplicated by `source_id` + `target_id` + `type`.
|
|
17
|
+
* - New edges are inserted; existing edges are left unchanged.
|
|
18
|
+
*
|
|
19
|
+
* @module memory/io/SqliteImporter
|
|
20
|
+
*/
|
|
21
|
+
import type { ImportResult } from '../facade/types.js';
|
|
22
|
+
import type { SqliteBrain } from '../store/SqliteBrain.js';
|
|
23
|
+
/**
|
|
24
|
+
* Merges a source SQLite brain file into a target `SqliteBrain`.
|
|
25
|
+
*
|
|
26
|
+
* **Usage:**
|
|
27
|
+
* ```ts
|
|
28
|
+
* const importer = new SqliteImporter(targetBrain);
|
|
29
|
+
* const result = await importer.import('/path/to/source.sqlite');
|
|
30
|
+
* ```
|
|
31
|
+
*/
|
|
32
|
+
export declare class SqliteImporter {
|
|
33
|
+
private readonly brain;
|
|
34
|
+
/**
|
|
35
|
+
* @param brain - The target `SqliteBrain` to merge data into.
|
|
36
|
+
*/
|
|
37
|
+
constructor(brain: SqliteBrain);
|
|
38
|
+
/**
|
|
39
|
+
* Open `sourcePath` as a read-only SQLite connection, read all tables, and
|
|
40
|
+
* merge their contents into the target brain.
|
|
41
|
+
*
|
|
42
|
+
* The source connection is closed when this method returns (even on error).
|
|
43
|
+
*
|
|
44
|
+
* @param sourcePath - Absolute path to the source `.sqlite` file to import.
|
|
45
|
+
* @returns `ImportResult` with counts of imported, skipped, and errored items.
|
|
46
|
+
*/
|
|
47
|
+
import(sourcePath: string): Promise<ImportResult>;
|
|
48
|
+
/**
|
|
49
|
+
* SHA-256 of an arbitrary string (hex output).
|
|
50
|
+
*/
|
|
51
|
+
private _sha256;
|
|
52
|
+
/**
|
|
53
|
+
* Merge `memory_traces` from source into target.
|
|
54
|
+
*
|
|
55
|
+
* Dedup key: SHA-256 of `content`.
|
|
56
|
+
* Conflict resolution: keep newer timestamp, union tags.
|
|
57
|
+
*
|
|
58
|
+
* @param src - Open source `better-sqlite3` database.
|
|
59
|
+
* @param result - Mutable result accumulator.
|
|
60
|
+
*/
|
|
61
|
+
private _mergeTraces;
|
|
62
|
+
/**
|
|
63
|
+
* Merge `knowledge_nodes` from source into target.
|
|
64
|
+
*
|
|
65
|
+
* Dedup key: SHA-256 of `label` + `type`.
|
|
66
|
+
*
|
|
67
|
+
* @param src - Open source database.
|
|
68
|
+
* @param result - Mutable result accumulator.
|
|
69
|
+
*/
|
|
70
|
+
private _mergeNodes;
|
|
71
|
+
/**
|
|
72
|
+
* Merge `knowledge_edges` from source into target.
|
|
73
|
+
*
|
|
74
|
+
* Dedup key: SHA-256 of `source_id` + `target_id` + `type`.
|
|
75
|
+
* Edges whose referenced nodes don't exist in the target are skipped.
|
|
76
|
+
*
|
|
77
|
+
* @param src - Open source database.
|
|
78
|
+
* @param result - Mutable result accumulator.
|
|
79
|
+
*/
|
|
80
|
+
private _mergeEdges;
|
|
81
|
+
}
|
|
82
|
+
//# sourceMappingURL=SqliteImporter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SqliteImporter.d.ts","sourceRoot":"","sources":["../../../src/memory/io/SqliteImporter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAKH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAgD3D;;;;;;;;GAQG;AACH,qBAAa,cAAc;IAIb,OAAO,CAAC,QAAQ,CAAC,KAAK;IAHlC;;OAEG;gBAC0B,KAAK,EAAE,WAAW;IAM/C;;;;;;;;OAQG;IACG,MAAM,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IA8BvD;;OAEG;IACH,OAAO,CAAC,OAAO;IAIf;;;;;;;;OAQG;IACH,OAAO,CAAC,YAAY;IA8EpB;;;;;;;OAOG;IACH,OAAO,CAAC,WAAW;IA4CnB;;;;;;;;OAQG;IACH,OAAO,CAAC,WAAW;CAmDpB"}
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview SQLite importer for AgentOS memory brain.
|
|
3
|
+
*
|
|
4
|
+
* Opens a source SQLite file (exported by `SqliteExporter` or any compatible
|
|
5
|
+
* AgentOS brain) as a separate `better-sqlite3` connection, reads all data
|
|
6
|
+
* tables, and merges them into the target `SqliteBrain`.
|
|
7
|
+
*
|
|
8
|
+
* ## Merge strategy
|
|
9
|
+
* - **memory_traces**: deduplicated by SHA-256 of `content`.
|
|
10
|
+
* - If a trace with the same hash already exists in the target:
|
|
11
|
+
* - Keep the newer `created_at` / `last_accessed` timestamp.
|
|
12
|
+
* - Merge `tags` arrays (union, dedup).
|
|
13
|
+
* - New traces are inserted wholesale.
|
|
14
|
+
* - **knowledge_nodes**: deduplicated by `label` + `type`.
|
|
15
|
+
* - New nodes are inserted; existing nodes are left unchanged.
|
|
16
|
+
* - **knowledge_edges**: deduplicated by `source_id` + `target_id` + `type`.
|
|
17
|
+
* - New edges are inserted; existing edges are left unchanged.
|
|
18
|
+
*
|
|
19
|
+
* @module memory/io/SqliteImporter
|
|
20
|
+
*/
|
|
21
|
+
import Database from 'better-sqlite3';
|
|
22
|
+
import crypto from 'node:crypto';
|
|
23
|
+
import { v4 as uuidv4 } from 'uuid';
|
|
24
|
+
// ---------------------------------------------------------------------------
|
|
25
|
+
// SqliteImporter
|
|
26
|
+
// ---------------------------------------------------------------------------
|
|
27
|
+
/**
|
|
28
|
+
* Merges a source SQLite brain file into a target `SqliteBrain`.
|
|
29
|
+
*
|
|
30
|
+
* **Usage:**
|
|
31
|
+
* ```ts
|
|
32
|
+
* const importer = new SqliteImporter(targetBrain);
|
|
33
|
+
* const result = await importer.import('/path/to/source.sqlite');
|
|
34
|
+
* ```
|
|
35
|
+
*/
|
|
36
|
+
export class SqliteImporter {
|
|
37
|
+
/**
|
|
38
|
+
* @param brain - The target `SqliteBrain` to merge data into.
|
|
39
|
+
*/
|
|
40
|
+
constructor(brain) {
|
|
41
|
+
this.brain = brain;
|
|
42
|
+
}
|
|
43
|
+
// -------------------------------------------------------------------------
|
|
44
|
+
// Public API
|
|
45
|
+
// -------------------------------------------------------------------------
|
|
46
|
+
/**
|
|
47
|
+
* Open `sourcePath` as a read-only SQLite connection, read all tables, and
|
|
48
|
+
* merge their contents into the target brain.
|
|
49
|
+
*
|
|
50
|
+
* The source connection is closed when this method returns (even on error).
|
|
51
|
+
*
|
|
52
|
+
* @param sourcePath - Absolute path to the source `.sqlite` file to import.
|
|
53
|
+
* @returns `ImportResult` with counts of imported, skipped, and errored items.
|
|
54
|
+
*/
|
|
55
|
+
async import(sourcePath) {
|
|
56
|
+
const result = { imported: 0, skipped: 0, errors: [] };
|
|
57
|
+
// Open the source file read-only so we cannot accidentally corrupt it.
|
|
58
|
+
let sourceDb;
|
|
59
|
+
try {
|
|
60
|
+
sourceDb = new Database(sourcePath, { readonly: true });
|
|
61
|
+
}
|
|
62
|
+
catch (err) {
|
|
63
|
+
result.errors.push(`Cannot open source SQLite: ${String(err)}`);
|
|
64
|
+
return result;
|
|
65
|
+
}
|
|
66
|
+
try {
|
|
67
|
+
// Run the whole merge in a single transaction on the target brain.
|
|
68
|
+
this.brain.db.transaction(() => {
|
|
69
|
+
this._mergeTraces(sourceDb, result);
|
|
70
|
+
this._mergeNodes(sourceDb, result);
|
|
71
|
+
this._mergeEdges(sourceDb, result);
|
|
72
|
+
})();
|
|
73
|
+
}
|
|
74
|
+
finally {
|
|
75
|
+
sourceDb.close();
|
|
76
|
+
}
|
|
77
|
+
return result;
|
|
78
|
+
}
|
|
79
|
+
// -------------------------------------------------------------------------
|
|
80
|
+
// Private helpers
|
|
81
|
+
// -------------------------------------------------------------------------
|
|
82
|
+
/**
|
|
83
|
+
* SHA-256 of an arbitrary string (hex output).
|
|
84
|
+
*/
|
|
85
|
+
_sha256(s) {
|
|
86
|
+
return crypto.createHash('sha256').update(s, 'utf8').digest('hex');
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Merge `memory_traces` from source into target.
|
|
90
|
+
*
|
|
91
|
+
* Dedup key: SHA-256 of `content`.
|
|
92
|
+
* Conflict resolution: keep newer timestamp, union tags.
|
|
93
|
+
*
|
|
94
|
+
* @param src - Open source `better-sqlite3` database.
|
|
95
|
+
* @param result - Mutable result accumulator.
|
|
96
|
+
*/
|
|
97
|
+
_mergeTraces(src, result) {
|
|
98
|
+
let sourceRows;
|
|
99
|
+
try {
|
|
100
|
+
sourceRows = src.prepare('SELECT * FROM memory_traces').all();
|
|
101
|
+
}
|
|
102
|
+
catch {
|
|
103
|
+
// Table might not exist in an incompatible source.
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
const checkStmt = this.brain.db.prepare(`SELECT id, created_at, tags
|
|
107
|
+
FROM memory_traces
|
|
108
|
+
WHERE json_extract(metadata, '$.import_hash') = ?
|
|
109
|
+
OR content = ?
|
|
110
|
+
LIMIT 1`);
|
|
111
|
+
const insertStmt = this.brain.db.prepare(`INSERT INTO memory_traces
|
|
112
|
+
(id, type, scope, content, embedding, strength, created_at, last_accessed,
|
|
113
|
+
retrieval_count, tags, emotions, metadata, deleted)
|
|
114
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
115
|
+
const updateTimestampStmt = this.brain.db.prepare(`UPDATE memory_traces SET created_at = ?, tags = ? WHERE id = ?`);
|
|
116
|
+
for (const row of sourceRows) {
|
|
117
|
+
try {
|
|
118
|
+
const hash = this._sha256(row.content);
|
|
119
|
+
const existing = checkStmt.get(hash, row.content);
|
|
120
|
+
if (existing) {
|
|
121
|
+
// Keep the newer timestamp and union the tags.
|
|
122
|
+
const newerAt = Math.max(existing.created_at, row.created_at);
|
|
123
|
+
let existingTags = [];
|
|
124
|
+
try {
|
|
125
|
+
existingTags = JSON.parse(existing.tags);
|
|
126
|
+
}
|
|
127
|
+
catch { /* ignore */ }
|
|
128
|
+
let sourceTags = [];
|
|
129
|
+
try {
|
|
130
|
+
sourceTags = JSON.parse(row.tags);
|
|
131
|
+
}
|
|
132
|
+
catch { /* ignore */ }
|
|
133
|
+
const merged = Array.from(new Set([...existingTags, ...sourceTags]));
|
|
134
|
+
updateTimestampStmt.run(newerAt, JSON.stringify(merged), existing.id);
|
|
135
|
+
result.skipped++;
|
|
136
|
+
continue;
|
|
137
|
+
}
|
|
138
|
+
// New trace — enrich metadata with import_hash.
|
|
139
|
+
let meta = {};
|
|
140
|
+
try {
|
|
141
|
+
meta = JSON.parse(row.metadata);
|
|
142
|
+
}
|
|
143
|
+
catch { /* ignore */ }
|
|
144
|
+
meta['import_hash'] = hash;
|
|
145
|
+
insertStmt.run(row.id ?? `mt_${uuidv4()}`, row.type ?? 'episodic', row.scope ?? 'user', row.content, row.embedding ?? null, row.strength ?? 1.0, row.created_at ?? Date.now(), row.last_accessed ?? null, row.retrieval_count ?? 0, row.tags ?? '[]', row.emotions ?? '{}', JSON.stringify(meta), row.deleted ?? 0);
|
|
146
|
+
result.imported++;
|
|
147
|
+
}
|
|
148
|
+
catch (err) {
|
|
149
|
+
result.errors.push(`Trace merge error: ${String(err)}`);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* Merge `knowledge_nodes` from source into target.
|
|
155
|
+
*
|
|
156
|
+
* Dedup key: SHA-256 of `label` + `type`.
|
|
157
|
+
*
|
|
158
|
+
* @param src - Open source database.
|
|
159
|
+
* @param result - Mutable result accumulator.
|
|
160
|
+
*/
|
|
161
|
+
_mergeNodes(src, result) {
|
|
162
|
+
let sourceRows;
|
|
163
|
+
try {
|
|
164
|
+
sourceRows = src.prepare('SELECT * FROM knowledge_nodes').all();
|
|
165
|
+
}
|
|
166
|
+
catch {
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
const checkStmt = this.brain.db.prepare(`SELECT id FROM knowledge_nodes WHERE label = ? AND type = ? LIMIT 1`);
|
|
170
|
+
const insertStmt = this.brain.db.prepare(`INSERT OR IGNORE INTO knowledge_nodes
|
|
171
|
+
(id, type, label, properties, embedding, confidence, source, created_at)
|
|
172
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
173
|
+
for (const row of sourceRows) {
|
|
174
|
+
try {
|
|
175
|
+
const existing = checkStmt.get(row.label ?? '', row.type ?? '');
|
|
176
|
+
if (existing) {
|
|
177
|
+
result.skipped++;
|
|
178
|
+
continue;
|
|
179
|
+
}
|
|
180
|
+
insertStmt.run(row.id ?? `kn_${uuidv4()}`, row.type ?? 'concept', row.label ?? '', row.properties ?? '{}', row.embedding ?? null, row.confidence ?? 1.0, row.source ?? '{}', row.created_at ?? Date.now());
|
|
181
|
+
result.imported++;
|
|
182
|
+
}
|
|
183
|
+
catch (err) {
|
|
184
|
+
result.errors.push(`Node merge error: ${String(err)}`);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* Merge `knowledge_edges` from source into target.
|
|
190
|
+
*
|
|
191
|
+
* Dedup key: SHA-256 of `source_id` + `target_id` + `type`.
|
|
192
|
+
* Edges whose referenced nodes don't exist in the target are skipped.
|
|
193
|
+
*
|
|
194
|
+
* @param src - Open source database.
|
|
195
|
+
* @param result - Mutable result accumulator.
|
|
196
|
+
*/
|
|
197
|
+
_mergeEdges(src, result) {
|
|
198
|
+
let sourceRows;
|
|
199
|
+
try {
|
|
200
|
+
sourceRows = src.prepare('SELECT * FROM knowledge_edges').all();
|
|
201
|
+
}
|
|
202
|
+
catch {
|
|
203
|
+
return;
|
|
204
|
+
}
|
|
205
|
+
const checkStmt = this.brain.db.prepare(`SELECT id FROM knowledge_edges
|
|
206
|
+
WHERE source_id = ? AND target_id = ? AND type = ?
|
|
207
|
+
LIMIT 1`);
|
|
208
|
+
const insertStmt = this.brain.db.prepare(`INSERT OR IGNORE INTO knowledge_edges
|
|
209
|
+
(id, source_id, target_id, type, weight, bidirectional, metadata, created_at)
|
|
210
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
211
|
+
for (const row of sourceRows) {
|
|
212
|
+
try {
|
|
213
|
+
if (!row.source_id || !row.target_id) {
|
|
214
|
+
result.skipped++;
|
|
215
|
+
continue;
|
|
216
|
+
}
|
|
217
|
+
const existing = checkStmt.get(row.source_id, row.target_id, row.type ?? '');
|
|
218
|
+
if (existing) {
|
|
219
|
+
result.skipped++;
|
|
220
|
+
continue;
|
|
221
|
+
}
|
|
222
|
+
insertStmt.run(row.id ?? `ke_${uuidv4()}`, row.source_id, row.target_id, row.type ?? 'related_to', row.weight ?? 1.0, row.bidirectional ?? 0, row.metadata ?? '{}', row.created_at ?? Date.now());
|
|
223
|
+
result.imported++;
|
|
224
|
+
}
|
|
225
|
+
catch (err) {
|
|
226
|
+
// FK constraint: target node not in this brain — expected for partial imports.
|
|
227
|
+
result.errors.push(`Edge merge error: ${String(err)}`);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
//# sourceMappingURL=SqliteImporter.js.map
|