@lanonasis/recall-forge 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claw/skills/SKILL.md +347 -0
- package/CHANGELOG.md +162 -0
- package/LICENSE +21 -0
- package/README.md +302 -0
- package/SETUP.md +190 -0
- package/dist/cli-common.d.ts +25 -0
- package/dist/cli-common.js +338 -0
- package/dist/cli-memory.d.ts +6 -0
- package/dist/cli-memory.js +146 -0
- package/dist/cli.d.ts +7 -0
- package/dist/cli.js +135 -0
- package/dist/client.d.ts +116 -0
- package/dist/client.js +643 -0
- package/dist/config.d.ts +41 -0
- package/dist/config.js +125 -0
- package/dist/enrichment/capture-filter.d.ts +4 -0
- package/dist/enrichment/capture-filter.js +44 -0
- package/dist/enrichment/prompt-safety.d.ts +13 -0
- package/dist/enrichment/prompt-safety.js +83 -0
- package/dist/enrichment/tag-extractor.d.ts +1 -0
- package/dist/enrichment/tag-extractor.js +47 -0
- package/dist/enrichment/type-detector.d.ts +2 -0
- package/dist/enrichment/type-detector.js +95 -0
- package/dist/extraction/cli-extract.d.ts +8 -0
- package/dist/extraction/cli-extract.js +66 -0
- package/dist/extraction/format-adapters.d.ts +8 -0
- package/dist/extraction/format-adapters.js +268 -0
- package/dist/extraction/index.d.ts +7 -0
- package/dist/extraction/index.js +7 -0
- package/dist/extraction/jsonl-extractor.d.ts +32 -0
- package/dist/extraction/jsonl-extractor.js +207 -0
- package/dist/extraction/markdown-extractor.d.ts +23 -0
- package/dist/extraction/markdown-extractor.js +228 -0
- package/dist/extraction/secret-redactor.d.ts +7 -0
- package/dist/extraction/secret-redactor.js +112 -0
- package/dist/extraction/sqlite-extractor.d.ts +15 -0
- package/dist/extraction/sqlite-extractor.js +245 -0
- package/dist/extraction/types.d.ts +50 -0
- package/dist/extraction/types.js +1 -0
- package/dist/hooks/capture.d.ts +23 -0
- package/dist/hooks/capture.js +162 -0
- package/dist/hooks/context-engine.d.ts +4 -0
- package/dist/hooks/context-engine.js +54 -0
- package/dist/hooks/local-fallback.d.ts +5 -0
- package/dist/hooks/local-fallback.js +31 -0
- package/dist/hooks/recall.d.ts +21 -0
- package/dist/hooks/recall.js +123 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.js +103 -0
- package/dist/plugin-sdk-stub.d.ts +53 -0
- package/dist/plugin-sdk-stub.js +3 -0
- package/dist/privacy/privacy-guard.d.ts +33 -0
- package/dist/privacy/privacy-guard.js +130 -0
- package/dist/privacy/privacy-log.d.ts +6 -0
- package/dist/privacy/privacy-log.js +44 -0
- package/dist/tools/memory-forget.d.ts +3 -0
- package/dist/tools/memory-forget.js +109 -0
- package/dist/tools/memory-get.d.ts +3 -0
- package/dist/tools/memory-get.js +46 -0
- package/dist/tools/memory-search.d.ts +4 -0
- package/dist/tools/memory-search.js +95 -0
- package/dist/tools/memory-store.d.ts +5 -0
- package/dist/tools/memory-store.js +199 -0
- package/openclaw.plugin.json +315 -0
- package/package.json +90 -0
- package/setup/agents-memory.md +63 -0
- package/setup/heartbeat-memory.md +53 -0
- package/setup/install.sh +179 -0
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
// Document-mode extraction for Markdown files
|
|
2
|
+
// Splits by heading sections, emits each as an ExtractionRecord
|
|
3
|
+
import { promises as fs } from "fs";
|
|
4
|
+
import { basename } from "path";
|
|
5
|
+
import { createHash } from "crypto";
|
|
6
|
+
import { shouldCapture } from "../enrichment/capture-filter.js";
|
|
7
|
+
import { detectMemoryType } from "../enrichment/type-detector.js";
|
|
8
|
+
import { extractTags } from "../enrichment/tag-extractor.js";
|
|
9
|
+
import { looksLikePromptInjection } from "../enrichment/prompt-safety.js";
|
|
10
|
+
import { redactSecrets } from "./secret-redactor.js";
|
|
11
|
+
/**
|
|
12
|
+
* Split a markdown file into heading-delimited sections.
|
|
13
|
+
* Each section includes the heading text, level (1-6), body content, and starting line number.
|
|
14
|
+
* Content before the first heading is emitted as a section with heading = filename.
|
|
15
|
+
*/
|
|
16
|
+
function splitMarkdownSections(content, filename) {
|
|
17
|
+
const lines = content.split("\n");
|
|
18
|
+
const sections = [];
|
|
19
|
+
const headingPattern = /^(#{1,6})\s+(.+)$/;
|
|
20
|
+
let currentHeading = filename.replace(/\.md$/i, "");
|
|
21
|
+
let currentLevel = 0;
|
|
22
|
+
let currentLines = [];
|
|
23
|
+
let currentStart = 1;
|
|
24
|
+
for (let i = 0; i < lines.length; i++) {
|
|
25
|
+
const match = headingPattern.exec(lines[i]);
|
|
26
|
+
if (match) {
|
|
27
|
+
// Flush previous section
|
|
28
|
+
const body = currentLines.join("\n").trim();
|
|
29
|
+
if (body.length > 0) {
|
|
30
|
+
sections.push({
|
|
31
|
+
heading: currentHeading,
|
|
32
|
+
level: currentLevel,
|
|
33
|
+
body,
|
|
34
|
+
lineNumber: currentStart,
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
currentHeading = match[2].trim();
|
|
38
|
+
currentLevel = match[1].length;
|
|
39
|
+
currentLines = [];
|
|
40
|
+
currentStart = i + 1;
|
|
41
|
+
}
|
|
42
|
+
else {
|
|
43
|
+
currentLines.push(lines[i]);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
// Flush final section
|
|
47
|
+
const body = currentLines.join("\n").trim();
|
|
48
|
+
if (body.length > 0) {
|
|
49
|
+
sections.push({
|
|
50
|
+
heading: currentHeading,
|
|
51
|
+
level: currentLevel,
|
|
52
|
+
body,
|
|
53
|
+
lineNumber: currentStart,
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
return sections;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Convert markdown sections into ExtractionRecords.
|
|
60
|
+
* Each section becomes one record with:
|
|
61
|
+
* - text: "{heading}\n\n{body}" (heading provides context)
|
|
62
|
+
* - role: "user" (markdown docs are considered user-authored)
|
|
63
|
+
* - sourceFormat: "markdown"
|
|
64
|
+
*/
|
|
65
|
+
function sectionsToRecords(sections) {
|
|
66
|
+
return sections.map((s) => ({
|
|
67
|
+
text: s.level > 0 ? `${s.heading}\n\n${s.body}` : s.body,
|
|
68
|
+
role: "user",
|
|
69
|
+
sourceFormat: "markdown",
|
|
70
|
+
lineNumber: s.lineNumber,
|
|
71
|
+
}));
|
|
72
|
+
}
|
|
73
|
+
function idempotencyKey(filePath, lineNumber, textPrefix) {
|
|
74
|
+
return createHash("sha256")
|
|
75
|
+
.update(`${filePath}:${lineNumber}:${textPrefix}`)
|
|
76
|
+
.digest("hex")
|
|
77
|
+
.slice(0, 32);
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Detect whether a file is markdown based on extension.
|
|
81
|
+
* Used by the CLI to route to extractMarkdown() vs extractJsonl().
|
|
82
|
+
*/
|
|
83
|
+
export function isMarkdownFile(filePath) {
|
|
84
|
+
return /\.(md|markdown|mdx)$/i.test(filePath);
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Extract memories from a Markdown document.
|
|
88
|
+
*
|
|
89
|
+
* Pipeline per section:
|
|
90
|
+
* 1. Split file by headings → MarkdownSection[]
|
|
91
|
+
* 2. Convert to ExtractionRecord[]
|
|
92
|
+
* 3. Role filter (default: user only — all sections are "user")
|
|
93
|
+
* 4. redactSecrets()
|
|
94
|
+
* 5. shouldCapture() — filter noise
|
|
95
|
+
* 6. looksLikePromptInjection() — safety check
|
|
96
|
+
* 7. detectMemoryType() + extractTags() — enrichment
|
|
97
|
+
* 8. Vector dedup via searchMemories()
|
|
98
|
+
* 9. createMemory() with idempotency key
|
|
99
|
+
* 10. Optional local markdown fallback
|
|
100
|
+
*/
|
|
101
|
+
export async function extractMarkdown(options, deps) {
|
|
102
|
+
const startTime = Date.now();
|
|
103
|
+
const stats = {
|
|
104
|
+
linesRead: 0,
|
|
105
|
+
linesParsed: 0,
|
|
106
|
+
linesSkipped: 0,
|
|
107
|
+
recordsExtracted: 0,
|
|
108
|
+
recordsFiltered: 0,
|
|
109
|
+
recordsDeduped: 0,
|
|
110
|
+
recordsStored: 0,
|
|
111
|
+
secretsRedacted: 0,
|
|
112
|
+
markdownWritten: 0,
|
|
113
|
+
errors: 0,
|
|
114
|
+
durationMs: 0,
|
|
115
|
+
};
|
|
116
|
+
const { filePath, channel = "md-extract", dedup = true, dedupThreshold = 0.92, localFallback = false, dryRun = false, limit, strict = false, roles = ["user"], } = options;
|
|
117
|
+
// Read the entire file
|
|
118
|
+
let content;
|
|
119
|
+
try {
|
|
120
|
+
content = await fs.readFile(filePath, "utf-8");
|
|
121
|
+
}
|
|
122
|
+
catch (err) {
|
|
123
|
+
deps.logger.warn(`Failed to read ${filePath}: ${err instanceof Error ? err.message : "unknown"}`);
|
|
124
|
+
stats.errors++;
|
|
125
|
+
stats.durationMs = Date.now() - startTime;
|
|
126
|
+
return stats;
|
|
127
|
+
}
|
|
128
|
+
const lines = content.split("\n");
|
|
129
|
+
stats.linesRead = lines.length;
|
|
130
|
+
deps.logger.info(`Format detected: markdown`);
|
|
131
|
+
// Split into sections
|
|
132
|
+
const filename = basename(filePath);
|
|
133
|
+
const sections = splitMarkdownSections(content, filename);
|
|
134
|
+
const records = sectionsToRecords(sections);
|
|
135
|
+
stats.linesParsed = stats.linesRead; // all lines are "parsed" in document mode
|
|
136
|
+
stats.recordsExtracted = records.length;
|
|
137
|
+
let totalProcessed = 0;
|
|
138
|
+
for (const record of records) {
|
|
139
|
+
if (limit && totalProcessed >= limit)
|
|
140
|
+
break;
|
|
141
|
+
// Role filter
|
|
142
|
+
if (roles.length > 0 && !roles.includes(record.role))
|
|
143
|
+
continue;
|
|
144
|
+
// Step 1: Redact secrets
|
|
145
|
+
const redaction = redactSecrets(record.text);
|
|
146
|
+
stats.secretsRedacted += redaction.secretsFound;
|
|
147
|
+
const cleanText = redaction.text;
|
|
148
|
+
// Step 2: Capture filter
|
|
149
|
+
if (!shouldCapture(cleanText, { strict })) {
|
|
150
|
+
stats.recordsFiltered++;
|
|
151
|
+
continue;
|
|
152
|
+
}
|
|
153
|
+
// Step 3: Prompt injection check
|
|
154
|
+
if (looksLikePromptInjection(cleanText)) {
|
|
155
|
+
stats.recordsFiltered++;
|
|
156
|
+
continue;
|
|
157
|
+
}
|
|
158
|
+
// Step 4: Enrichment
|
|
159
|
+
const memoryType = detectMemoryType(cleanText, filename);
|
|
160
|
+
const tags = [
|
|
161
|
+
...extractTags(cleanText, filename),
|
|
162
|
+
"md-extract",
|
|
163
|
+
filename.replace(/\.md$/i, "").toLowerCase(),
|
|
164
|
+
];
|
|
165
|
+
// Step 5: Vector dedup
|
|
166
|
+
if (dedup && !dryRun) {
|
|
167
|
+
try {
|
|
168
|
+
const existing = await deps.client.searchMemories({
|
|
169
|
+
query: cleanText.slice(0, 500),
|
|
170
|
+
threshold: dedupThreshold,
|
|
171
|
+
limit: 1,
|
|
172
|
+
});
|
|
173
|
+
if (existing && existing.length > 0) {
|
|
174
|
+
stats.recordsDeduped++;
|
|
175
|
+
continue;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
catch (err) {
|
|
179
|
+
deps.logger.warn(`Dedup check failed: ${err instanceof Error ? err.message : "unknown"}`);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
// Step 6: Store
|
|
183
|
+
if (!dryRun) {
|
|
184
|
+
const title = cleanText.slice(0, 80).replace(/\s+/g, " ").trim();
|
|
185
|
+
const params = {
|
|
186
|
+
title,
|
|
187
|
+
content: cleanText,
|
|
188
|
+
type: memoryType,
|
|
189
|
+
tags,
|
|
190
|
+
metadata: {
|
|
191
|
+
agent_id: deps.config.agentId,
|
|
192
|
+
source: "markdown",
|
|
193
|
+
channel,
|
|
194
|
+
line_number: record.lineNumber,
|
|
195
|
+
source_file: filename,
|
|
196
|
+
captured_at: new Date().toISOString(),
|
|
197
|
+
secrets_redacted: redaction.secretsFound,
|
|
198
|
+
},
|
|
199
|
+
idempotency_key: idempotencyKey(filePath, record.lineNumber, cleanText.slice(0, 200)),
|
|
200
|
+
};
|
|
201
|
+
try {
|
|
202
|
+
await deps.client.createMemory(params);
|
|
203
|
+
stats.recordsStored++;
|
|
204
|
+
}
|
|
205
|
+
catch (err) {
|
|
206
|
+
stats.errors++;
|
|
207
|
+
deps.logger.warn(`Store failed at line ${record.lineNumber}: ${err instanceof Error ? err.message : "unknown"}`);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
else {
|
|
211
|
+
stats.recordsStored++;
|
|
212
|
+
}
|
|
213
|
+
// Step 7: Local markdown fallback
|
|
214
|
+
if (localFallback && deps.fallback) {
|
|
215
|
+
try {
|
|
216
|
+
const title = cleanText.slice(0, 80).replace(/\s+/g, " ").trim();
|
|
217
|
+
await deps.fallback.writeMemory(title, cleanText);
|
|
218
|
+
stats.markdownWritten++;
|
|
219
|
+
}
|
|
220
|
+
catch {
|
|
221
|
+
// Non-fatal
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
totalProcessed++;
|
|
225
|
+
}
|
|
226
|
+
stats.durationMs = Date.now() - startTime;
|
|
227
|
+
return stats;
|
|
228
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { RedactionResult } from "./types.js";
|
|
2
|
+
export declare function redactSecrets(text: string, options?: {
|
|
3
|
+
redactPII?: boolean;
|
|
4
|
+
}): RedactionResult;
|
|
5
|
+
export declare function containsSecrets(text: string, options?: {
|
|
6
|
+
redactPII?: boolean;
|
|
7
|
+
}): boolean;
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
// Secret redaction for JSONL extraction
|
|
2
|
+
// Detects and redacts API keys, tokens, credentials, and PII
|
|
3
|
+
const SECRET_PATTERNS = [
|
|
4
|
+
{
|
|
5
|
+
name: "lanonasis-api-key",
|
|
6
|
+
pattern: /\blano_[A-Za-z0-9_-]{20,}\b/g,
|
|
7
|
+
},
|
|
8
|
+
{
|
|
9
|
+
name: "openai-api-key",
|
|
10
|
+
pattern: /\bsk-[A-Za-z0-9]{20,}\b/g,
|
|
11
|
+
},
|
|
12
|
+
{
|
|
13
|
+
name: "anthropic-api-key",
|
|
14
|
+
pattern: /\bsk-ant-[A-Za-z0-9_-]{20,}\b/g,
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
name: "generic-api-key",
|
|
18
|
+
pattern: /\b(api[_-]?key|apikey)\s*[:=]\s*["']?[A-Za-z0-9_-]{16,}["']?/gi,
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
name: "bearer-token",
|
|
22
|
+
pattern: /\bBearer\s+[A-Za-z0-9._~+\/=-]{20,}\b/gi,
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
name: "jwt-token",
|
|
26
|
+
pattern: /\beyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g,
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
name: "aws-access-key",
|
|
30
|
+
pattern: /\bAKIA[A-Z0-9]{16}\b/g,
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
name: "aws-secret-key",
|
|
34
|
+
pattern: /\baws[_-]?secret[_-]?access[_-]?key\s*[:=]\s*["']?[A-Za-z0-9/+=]{40}["']?/gi,
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
name: "database-url",
|
|
38
|
+
pattern: /\b(postgres|postgresql|mongodb|mysql|redis):\/\/[^"'\s)]+/gi,
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
name: "supabase-key",
|
|
42
|
+
pattern: /\b(eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+|[A-Za-z0-9_-]{40,})\b/g,
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
name: "secret-key",
|
|
46
|
+
pattern: /\b(secret[_-]?key|private[_-]?key)\s*[:=]\s*["']?[A-Za-z0-9_-]{16,}["']?/gi,
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
name: "password",
|
|
50
|
+
pattern: /\b(password|passwd|pwd)\s*[:=]\s*["']?[^\s"']{8,}["']?/gi,
|
|
51
|
+
},
|
|
52
|
+
];
|
|
53
|
+
const PII_PATTERNS = [
|
|
54
|
+
{
|
|
55
|
+
name: "email",
|
|
56
|
+
pattern: /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g,
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
name: "phone",
|
|
60
|
+
pattern: /\+?[\d\s()-]{10,}\b/g,
|
|
61
|
+
},
|
|
62
|
+
{
|
|
63
|
+
name: "credit-card",
|
|
64
|
+
pattern: /\b\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}\b/g,
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
name: "ssn",
|
|
68
|
+
pattern: /\b\d{3}[\s-]?\d{2}[\s-]?\d{4}\b/g,
|
|
69
|
+
},
|
|
70
|
+
];
|
|
71
|
+
export function redactSecrets(text, options = {}) {
|
|
72
|
+
let result = text;
|
|
73
|
+
const secretsFound = [];
|
|
74
|
+
const { redactPII = true } = options;
|
|
75
|
+
for (const { name, pattern } of SECRET_PATTERNS) {
|
|
76
|
+
const matches = result.match(pattern);
|
|
77
|
+
if (matches && matches.length > 0) {
|
|
78
|
+
secretsFound.push(name);
|
|
79
|
+
result = result.replace(pattern, `[REDACTED:${name}]`);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
if (redactPII) {
|
|
83
|
+
for (const { name, pattern } of PII_PATTERNS) {
|
|
84
|
+
const matches = result.match(pattern);
|
|
85
|
+
if (matches && matches.length > 0) {
|
|
86
|
+
secretsFound.push(name);
|
|
87
|
+
result = result.replace(pattern, `[REDACTED:${name}]`);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return {
|
|
92
|
+
text: result,
|
|
93
|
+
secretsFound: secretsFound.length,
|
|
94
|
+
types: secretsFound,
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
export function containsSecrets(text, options = {}) {
|
|
98
|
+
const { redactPII = true } = options;
|
|
99
|
+
for (const { pattern } of SECRET_PATTERNS) {
|
|
100
|
+
if (pattern.test(text)) {
|
|
101
|
+
return true;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
if (redactPII) {
|
|
105
|
+
for (const { pattern } of PII_PATTERNS) {
|
|
106
|
+
if (pattern.test(text)) {
|
|
107
|
+
return true;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return false;
|
|
112
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { ExtractionOptions, ExtractionStats } from "./types.js";
|
|
2
|
+
import type { ExtractionDeps } from "./jsonl-extractor.js";
|
|
3
|
+
/**
|
|
4
|
+
* Detect whether a file is a SQLite database based on extension.
|
|
5
|
+
*/
|
|
6
|
+
export declare function isSqliteFile(filePath: string): boolean;
|
|
7
|
+
/**
|
|
8
|
+
* Extract memories from an OpenClaw SQLite memory database.
|
|
9
|
+
*
|
|
10
|
+
* Reads the `chunks` table and processes each chunk through the standard pipeline:
|
|
11
|
+
* redact → filter → enrich → dedup → store
|
|
12
|
+
*
|
|
13
|
+
* Also reads `meta` for the embedding model info which is logged and tagged.
|
|
14
|
+
*/
|
|
15
|
+
export declare function extractSqlite(options: ExtractionOptions, deps: ExtractionDeps): Promise<ExtractionStats>;
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
// Document-mode extraction for OpenClaw SQLite memory databases
|
|
2
|
+
// Reads chunks table, emits each chunk as an ExtractionRecord
|
|
3
|
+
import { createHash } from "crypto";
|
|
4
|
+
import { basename } from "path";
|
|
5
|
+
import { shouldCapture } from "../enrichment/capture-filter.js";
|
|
6
|
+
import { detectMemoryType } from "../enrichment/type-detector.js";
|
|
7
|
+
import { extractTags } from "../enrichment/tag-extractor.js";
|
|
8
|
+
import { looksLikePromptInjection } from "../enrichment/prompt-safety.js";
|
|
9
|
+
import { redactSecrets } from "./secret-redactor.js";
|
|
10
|
+
/**
|
|
11
|
+
* Detect whether a file is a SQLite database based on extension.
|
|
12
|
+
*/
|
|
13
|
+
export function isSqliteFile(filePath) {
|
|
14
|
+
return /\.(sqlite3?|db)$/i.test(filePath);
|
|
15
|
+
}
|
|
16
|
+
function idempotencyKey(dbPath, chunkId) {
|
|
17
|
+
return createHash("sha256")
|
|
18
|
+
.update(`sqlite:${dbPath}:${chunkId}`)
|
|
19
|
+
.digest("hex")
|
|
20
|
+
.slice(0, 32);
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Open a SQLite database using available runtime bindings.
|
|
24
|
+
*
|
|
25
|
+
* Priority:
|
|
26
|
+
* 1. bun:sqlite — available when running under Bun (OpenClaw's runtime)
|
|
27
|
+
* 2. node:sqlite — available in Node.js >= 22.5.0 (experimental)
|
|
28
|
+
* 3. Throws with a clear message if neither is available
|
|
29
|
+
*
|
|
30
|
+
* No child_process or shell commands are used.
|
|
31
|
+
*/
|
|
32
|
+
async function openDatabase(dbPath) {
|
|
33
|
+
// 1. Try Bun's built-in SQLite (primary path — OpenClaw runs on Bun)
|
|
34
|
+
try {
|
|
35
|
+
// @ts-ignore — bun:sqlite is a Bun-specific module, not in @types
|
|
36
|
+
const { Database } = await import("bun:sqlite");
|
|
37
|
+
const db = new Database(dbPath, { readonly: true });
|
|
38
|
+
return {
|
|
39
|
+
all(sql) {
|
|
40
|
+
return db.query(sql).all();
|
|
41
|
+
},
|
|
42
|
+
get(sql) {
|
|
43
|
+
return db.query(sql).get();
|
|
44
|
+
},
|
|
45
|
+
close() {
|
|
46
|
+
db.close();
|
|
47
|
+
},
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
catch {
|
|
51
|
+
// bun:sqlite unavailable — try Node.js built-in (Node >= 22.5.0)
|
|
52
|
+
}
|
|
53
|
+
// 2. Try Node's built-in sqlite module (Node 22.5+ experimental)
|
|
54
|
+
try {
|
|
55
|
+
// @ts-ignore — node:sqlite is experimental and not in stable @types/node
|
|
56
|
+
const { DatabaseSync } = await import("node:sqlite");
|
|
57
|
+
const db = new DatabaseSync(dbPath, { open: true });
|
|
58
|
+
return {
|
|
59
|
+
all(sql) {
|
|
60
|
+
const stmt = db.prepare(sql);
|
|
61
|
+
return stmt.all();
|
|
62
|
+
},
|
|
63
|
+
get(sql) {
|
|
64
|
+
const stmt = db.prepare(sql);
|
|
65
|
+
return stmt.get();
|
|
66
|
+
},
|
|
67
|
+
close() {
|
|
68
|
+
db.close();
|
|
69
|
+
},
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
catch {
|
|
73
|
+
// node:sqlite unavailable
|
|
74
|
+
}
|
|
75
|
+
throw new Error(`SQLite extraction requires Bun (bun:sqlite) or Node.js >= 22.5 (node:sqlite). ` +
|
|
76
|
+
`Neither was available in the current runtime. Run this command under Bun or upgrade Node.`);
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Extract memories from an OpenClaw SQLite memory database.
|
|
80
|
+
*
|
|
81
|
+
* Reads the `chunks` table and processes each chunk through the standard pipeline:
|
|
82
|
+
* redact → filter → enrich → dedup → store
|
|
83
|
+
*
|
|
84
|
+
* Also reads `meta` for the embedding model info which is logged and tagged.
|
|
85
|
+
*/
|
|
86
|
+
export async function extractSqlite(options, deps) {
|
|
87
|
+
const startTime = Date.now();
|
|
88
|
+
const stats = {
|
|
89
|
+
linesRead: 0,
|
|
90
|
+
linesParsed: 0,
|
|
91
|
+
linesSkipped: 0,
|
|
92
|
+
recordsExtracted: 0,
|
|
93
|
+
recordsFiltered: 0,
|
|
94
|
+
recordsDeduped: 0,
|
|
95
|
+
recordsStored: 0,
|
|
96
|
+
secretsRedacted: 0,
|
|
97
|
+
markdownWritten: 0,
|
|
98
|
+
errors: 0,
|
|
99
|
+
durationMs: 0,
|
|
100
|
+
};
|
|
101
|
+
const { filePath, channel = "sqlite-extract", dedup = true, dedupThreshold = 0.92, localFallback = false, dryRun = false, limit, strict = false, } = options;
|
|
102
|
+
const dbName = basename(filePath);
|
|
103
|
+
let db;
|
|
104
|
+
try {
|
|
105
|
+
db = await openDatabase(filePath);
|
|
106
|
+
}
|
|
107
|
+
catch (err) {
|
|
108
|
+
deps.logger.warn(`Failed to open ${filePath}: ${err instanceof Error ? err.message : "unknown"}`);
|
|
109
|
+
stats.errors++;
|
|
110
|
+
stats.durationMs = Date.now() - startTime;
|
|
111
|
+
return stats;
|
|
112
|
+
}
|
|
113
|
+
deps.logger.info(`Format detected: sqlite (${dbName})`);
|
|
114
|
+
// Read embedding model from meta if available
|
|
115
|
+
let embeddingModel = "unknown";
|
|
116
|
+
try {
|
|
117
|
+
const meta = db.get("SELECT value FROM meta WHERE key = 'memory_index_meta_v1'");
|
|
118
|
+
if (meta && typeof meta.value === "string") {
|
|
119
|
+
try {
|
|
120
|
+
const parsed = JSON.parse(meta.value);
|
|
121
|
+
embeddingModel = parsed.model ?? "unknown";
|
|
122
|
+
deps.logger.info(`Embedding model: ${embeddingModel} via ${parsed.provider ?? "unknown"}`);
|
|
123
|
+
}
|
|
124
|
+
catch { /* non-fatal */ }
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
catch { /* meta table might not exist */ }
|
|
128
|
+
// Read all chunks
|
|
129
|
+
let chunks;
|
|
130
|
+
try {
|
|
131
|
+
chunks = db.all("SELECT id, path, source, start_line, end_line, model, text, updated_at FROM chunks ORDER BY updated_at ASC");
|
|
132
|
+
}
|
|
133
|
+
catch (err) {
|
|
134
|
+
deps.logger.warn(`Failed to read chunks: ${err instanceof Error ? err.message : "unknown"}`);
|
|
135
|
+
db.close();
|
|
136
|
+
stats.errors++;
|
|
137
|
+
stats.durationMs = Date.now() - startTime;
|
|
138
|
+
return stats;
|
|
139
|
+
}
|
|
140
|
+
stats.linesRead = chunks.length;
|
|
141
|
+
stats.linesParsed = chunks.length;
|
|
142
|
+
stats.recordsExtracted = chunks.length;
|
|
143
|
+
let totalProcessed = 0;
|
|
144
|
+
for (const chunk of chunks) {
|
|
145
|
+
if (limit && totalProcessed >= limit)
|
|
146
|
+
break;
|
|
147
|
+
if (!chunk.text || chunk.text.trim().length === 0) {
|
|
148
|
+
stats.linesSkipped++;
|
|
149
|
+
continue;
|
|
150
|
+
}
|
|
151
|
+
// Step 1: Redact secrets
|
|
152
|
+
const redaction = redactSecrets(chunk.text);
|
|
153
|
+
stats.secretsRedacted += redaction.secretsFound;
|
|
154
|
+
const cleanText = redaction.text;
|
|
155
|
+
// Step 2: Capture filter
|
|
156
|
+
if (!shouldCapture(cleanText, { strict })) {
|
|
157
|
+
stats.recordsFiltered++;
|
|
158
|
+
continue;
|
|
159
|
+
}
|
|
160
|
+
// Step 3: Prompt injection check
|
|
161
|
+
if (looksLikePromptInjection(cleanText)) {
|
|
162
|
+
stats.recordsFiltered++;
|
|
163
|
+
continue;
|
|
164
|
+
}
|
|
165
|
+
// Step 4: Enrichment
|
|
166
|
+
const sourceFile = chunk.path || dbName;
|
|
167
|
+
const memoryType = detectMemoryType(cleanText, sourceFile);
|
|
168
|
+
const tags = [
|
|
169
|
+
...extractTags(cleanText, sourceFile),
|
|
170
|
+
"sqlite-extract",
|
|
171
|
+
dbName.replace(/\.sqlite3?$/i, ""),
|
|
172
|
+
];
|
|
173
|
+
if (chunk.source)
|
|
174
|
+
tags.push(chunk.source);
|
|
175
|
+
// Step 5: Vector dedup
|
|
176
|
+
if (dedup && !dryRun) {
|
|
177
|
+
try {
|
|
178
|
+
const existing = await deps.client.searchMemories({
|
|
179
|
+
query: cleanText.slice(0, 500),
|
|
180
|
+
threshold: dedupThreshold,
|
|
181
|
+
limit: 1,
|
|
182
|
+
});
|
|
183
|
+
if (existing && existing.length > 0) {
|
|
184
|
+
stats.recordsDeduped++;
|
|
185
|
+
continue;
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
catch (err) {
|
|
189
|
+
deps.logger.warn(`Dedup check failed: ${err instanceof Error ? err.message : "unknown"}`);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
// Step 6: Store
|
|
193
|
+
if (!dryRun) {
|
|
194
|
+
const title = cleanText.slice(0, 80).replace(/\s+/g, " ").trim();
|
|
195
|
+
const params = {
|
|
196
|
+
title,
|
|
197
|
+
content: cleanText,
|
|
198
|
+
type: memoryType,
|
|
199
|
+
tags,
|
|
200
|
+
metadata: {
|
|
201
|
+
agent_id: deps.config.agentId,
|
|
202
|
+
source: "sqlite",
|
|
203
|
+
channel,
|
|
204
|
+
source_file: sourceFile,
|
|
205
|
+
chunk_id: chunk.id,
|
|
206
|
+
start_line: chunk.start_line,
|
|
207
|
+
end_line: chunk.end_line,
|
|
208
|
+
embedding_model: chunk.model || embeddingModel,
|
|
209
|
+
captured_at: chunk.updated_at
|
|
210
|
+
? new Date(chunk.updated_at * 1000).toISOString()
|
|
211
|
+
: new Date().toISOString(),
|
|
212
|
+
secrets_redacted: redaction.secretsFound,
|
|
213
|
+
},
|
|
214
|
+
idempotency_key: idempotencyKey(filePath, chunk.id),
|
|
215
|
+
};
|
|
216
|
+
try {
|
|
217
|
+
await deps.client.createMemory(params);
|
|
218
|
+
stats.recordsStored++;
|
|
219
|
+
}
|
|
220
|
+
catch (err) {
|
|
221
|
+
stats.errors++;
|
|
222
|
+
deps.logger.warn(`Store failed for chunk ${chunk.id.slice(0, 8)}: ${err instanceof Error ? err.message : "unknown"}`);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
else {
|
|
226
|
+
stats.recordsStored++;
|
|
227
|
+
}
|
|
228
|
+
// Step 7: Local markdown fallback
|
|
229
|
+
if (localFallback && deps.fallback) {
|
|
230
|
+
try {
|
|
231
|
+
const title = cleanText.slice(0, 80).replace(/\s+/g, " ").trim();
|
|
232
|
+
await deps.fallback.writeMemory(title, cleanText);
|
|
233
|
+
stats.markdownWritten++;
|
|
234
|
+
}
|
|
235
|
+
catch { /* non-fatal */ }
|
|
236
|
+
}
|
|
237
|
+
totalProcessed++;
|
|
238
|
+
if (totalProcessed % 100 === 0) {
|
|
239
|
+
deps.logger.info(`Progress: ${totalProcessed}/${chunks.length} chunks processed, ${stats.recordsStored} stored`);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
db.close();
|
|
243
|
+
stats.durationMs = Date.now() - startTime;
|
|
244
|
+
return stats;
|
|
245
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/** A single meaningful record extracted from one JSONL line */
|
|
2
|
+
export interface ExtractionRecord {
|
|
3
|
+
text: string;
|
|
4
|
+
role: "user" | "assistant" | "system" | "unknown";
|
|
5
|
+
sourceFormat: string;
|
|
6
|
+
lineNumber: number;
|
|
7
|
+
timestamp?: string;
|
|
8
|
+
}
|
|
9
|
+
/** Result of the redaction pass */
|
|
10
|
+
export interface RedactionResult {
|
|
11
|
+
text: string;
|
|
12
|
+
secretsFound: number;
|
|
13
|
+
types: string[];
|
|
14
|
+
}
|
|
15
|
+
/** Stats reported after extraction completes */
|
|
16
|
+
export interface ExtractionStats {
|
|
17
|
+
linesRead: number;
|
|
18
|
+
linesParsed: number;
|
|
19
|
+
linesSkipped: number;
|
|
20
|
+
recordsExtracted: number;
|
|
21
|
+
recordsFiltered: number;
|
|
22
|
+
recordsDeduped: number;
|
|
23
|
+
recordsStored: number;
|
|
24
|
+
secretsRedacted: number;
|
|
25
|
+
markdownWritten: number;
|
|
26
|
+
errors: number;
|
|
27
|
+
durationMs: number;
|
|
28
|
+
}
|
|
29
|
+
/** Options for the extraction function */
|
|
30
|
+
export interface ExtractionOptions {
|
|
31
|
+
filePath: string;
|
|
32
|
+
format?: "claude-code" | "openclaw-cache" | "openclaw-session" | "codex" | "generic" | "markdown" | "sqlite";
|
|
33
|
+
channel?: string;
|
|
34
|
+
dedup?: boolean;
|
|
35
|
+
dedupThreshold?: number;
|
|
36
|
+
localFallback?: boolean;
|
|
37
|
+
batchSize?: number;
|
|
38
|
+
dryRun?: boolean;
|
|
39
|
+
limit?: number;
|
|
40
|
+
strict?: boolean;
|
|
41
|
+
roles?: string[];
|
|
42
|
+
/** Enable PII detection alongside secret redaction (default: true) */
|
|
43
|
+
redactPII?: boolean;
|
|
44
|
+
}
|
|
45
|
+
/** A format adapter converts a parsed JSON line into ExtractionRecords */
|
|
46
|
+
export interface FormatAdapter {
|
|
47
|
+
name: string;
|
|
48
|
+
detect(sample: Record<string, unknown>): boolean;
|
|
49
|
+
extract(line: Record<string, unknown>, lineNumber: number): ExtractionRecord[];
|
|
50
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { LanonasisClient } from "../client.js";
|
|
2
|
+
import type { LanonasisConfig } from "../config.js";
|
|
3
|
+
import type { LocalFallbackWriter } from "./local-fallback.js";
|
|
4
|
+
import type { PrivacyGuard } from "../privacy/privacy-guard.js";
|
|
5
|
+
import type { PrivacyLogWriter } from "../privacy/privacy-log.js";
|
|
6
|
+
export declare function createCaptureHook(client: LanonasisClient, cfg: LanonasisConfig, logger: {
|
|
7
|
+
info(msg: string): void;
|
|
8
|
+
warn(msg: string): void;
|
|
9
|
+
}, fallback: LocalFallbackWriter, guard?: PrivacyGuard, privacyLog?: PrivacyLogWriter): (event: {
|
|
10
|
+
messages: unknown[];
|
|
11
|
+
success: boolean;
|
|
12
|
+
error?: string;
|
|
13
|
+
}) => Promise<void>;
|
|
14
|
+
export declare function createCompactionCaptureHook(client: LanonasisClient, cfg: LanonasisConfig, logger: {
|
|
15
|
+
info(msg: string): void;
|
|
16
|
+
warn(msg: string): void;
|
|
17
|
+
}, guard?: PrivacyGuard): (event: {
|
|
18
|
+
messageCount: number;
|
|
19
|
+
messages?: unknown[];
|
|
20
|
+
sessionFile?: string;
|
|
21
|
+
tokenCount?: number;
|
|
22
|
+
compactingCount?: number;
|
|
23
|
+
}) => Promise<void>;
|