@lanonasis/recall-forge 1.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/.claw/skills/SKILL.md +347 -0
  2. package/CHANGELOG.md +162 -0
  3. package/LICENSE +21 -0
  4. package/README.md +302 -0
  5. package/SETUP.md +190 -0
  6. package/dist/cli-common.d.ts +25 -0
  7. package/dist/cli-common.js +338 -0
  8. package/dist/cli-memory.d.ts +6 -0
  9. package/dist/cli-memory.js +146 -0
  10. package/dist/cli.d.ts +7 -0
  11. package/dist/cli.js +135 -0
  12. package/dist/client.d.ts +116 -0
  13. package/dist/client.js +643 -0
  14. package/dist/config.d.ts +41 -0
  15. package/dist/config.js +125 -0
  16. package/dist/enrichment/capture-filter.d.ts +4 -0
  17. package/dist/enrichment/capture-filter.js +44 -0
  18. package/dist/enrichment/prompt-safety.d.ts +13 -0
  19. package/dist/enrichment/prompt-safety.js +83 -0
  20. package/dist/enrichment/tag-extractor.d.ts +1 -0
  21. package/dist/enrichment/tag-extractor.js +47 -0
  22. package/dist/enrichment/type-detector.d.ts +2 -0
  23. package/dist/enrichment/type-detector.js +95 -0
  24. package/dist/extraction/cli-extract.d.ts +8 -0
  25. package/dist/extraction/cli-extract.js +66 -0
  26. package/dist/extraction/format-adapters.d.ts +8 -0
  27. package/dist/extraction/format-adapters.js +268 -0
  28. package/dist/extraction/index.d.ts +7 -0
  29. package/dist/extraction/index.js +7 -0
  30. package/dist/extraction/jsonl-extractor.d.ts +32 -0
  31. package/dist/extraction/jsonl-extractor.js +207 -0
  32. package/dist/extraction/markdown-extractor.d.ts +23 -0
  33. package/dist/extraction/markdown-extractor.js +228 -0
  34. package/dist/extraction/secret-redactor.d.ts +7 -0
  35. package/dist/extraction/secret-redactor.js +112 -0
  36. package/dist/extraction/sqlite-extractor.d.ts +15 -0
  37. package/dist/extraction/sqlite-extractor.js +245 -0
  38. package/dist/extraction/types.d.ts +50 -0
  39. package/dist/extraction/types.js +1 -0
  40. package/dist/hooks/capture.d.ts +23 -0
  41. package/dist/hooks/capture.js +162 -0
  42. package/dist/hooks/context-engine.d.ts +4 -0
  43. package/dist/hooks/context-engine.js +54 -0
  44. package/dist/hooks/local-fallback.d.ts +5 -0
  45. package/dist/hooks/local-fallback.js +31 -0
  46. package/dist/hooks/recall.d.ts +21 -0
  47. package/dist/hooks/recall.js +123 -0
  48. package/dist/index.d.ts +3 -0
  49. package/dist/index.js +103 -0
  50. package/dist/plugin-sdk-stub.d.ts +53 -0
  51. package/dist/plugin-sdk-stub.js +3 -0
  52. package/dist/privacy/privacy-guard.d.ts +33 -0
  53. package/dist/privacy/privacy-guard.js +130 -0
  54. package/dist/privacy/privacy-log.d.ts +6 -0
  55. package/dist/privacy/privacy-log.js +44 -0
  56. package/dist/tools/memory-forget.d.ts +3 -0
  57. package/dist/tools/memory-forget.js +109 -0
  58. package/dist/tools/memory-get.d.ts +3 -0
  59. package/dist/tools/memory-get.js +46 -0
  60. package/dist/tools/memory-search.d.ts +4 -0
  61. package/dist/tools/memory-search.js +95 -0
  62. package/dist/tools/memory-store.d.ts +5 -0
  63. package/dist/tools/memory-store.js +199 -0
  64. package/openclaw.plugin.json +315 -0
  65. package/package.json +90 -0
  66. package/setup/agents-memory.md +63 -0
  67. package/setup/heartbeat-memory.md +53 -0
  68. package/setup/install.sh +179 -0
@@ -0,0 +1,228 @@
1
+ // Document-mode extraction for Markdown files
2
+ // Splits by heading sections, emits each as an ExtractionRecord
3
+ import { promises as fs } from "fs";
4
+ import { basename } from "path";
5
+ import { createHash } from "crypto";
6
+ import { shouldCapture } from "../enrichment/capture-filter.js";
7
+ import { detectMemoryType } from "../enrichment/type-detector.js";
8
+ import { extractTags } from "../enrichment/tag-extractor.js";
9
+ import { looksLikePromptInjection } from "../enrichment/prompt-safety.js";
10
+ import { redactSecrets } from "./secret-redactor.js";
11
+ /**
12
+ * Split a markdown file into heading-delimited sections.
13
+ * Each section includes the heading text, level (1-6), body content, and starting line number.
14
+ * Content before the first heading is emitted as a section with heading = filename.
15
+ */
16
+ function splitMarkdownSections(content, filename) {
17
+ const lines = content.split("\n");
18
+ const sections = [];
19
+ const headingPattern = /^(#{1,6})\s+(.+)$/;
20
+ let currentHeading = filename.replace(/\.md$/i, "");
21
+ let currentLevel = 0;
22
+ let currentLines = [];
23
+ let currentStart = 1;
24
+ for (let i = 0; i < lines.length; i++) {
25
+ const match = headingPattern.exec(lines[i]);
26
+ if (match) {
27
+ // Flush previous section
28
+ const body = currentLines.join("\n").trim();
29
+ if (body.length > 0) {
30
+ sections.push({
31
+ heading: currentHeading,
32
+ level: currentLevel,
33
+ body,
34
+ lineNumber: currentStart,
35
+ });
36
+ }
37
+ currentHeading = match[2].trim();
38
+ currentLevel = match[1].length;
39
+ currentLines = [];
40
+ currentStart = i + 1;
41
+ }
42
+ else {
43
+ currentLines.push(lines[i]);
44
+ }
45
+ }
46
+ // Flush final section
47
+ const body = currentLines.join("\n").trim();
48
+ if (body.length > 0) {
49
+ sections.push({
50
+ heading: currentHeading,
51
+ level: currentLevel,
52
+ body,
53
+ lineNumber: currentStart,
54
+ });
55
+ }
56
+ return sections;
57
+ }
58
+ /**
59
+ * Convert markdown sections into ExtractionRecords.
60
+ * Each section becomes one record with:
61
+ * - text: "{heading}\n\n{body}" (heading provides context)
62
+ * - role: "user" (markdown docs are considered user-authored)
63
+ * - sourceFormat: "markdown"
64
+ */
65
+ function sectionsToRecords(sections) {
66
+ return sections.map((s) => ({
67
+ text: s.level > 0 ? `${s.heading}\n\n${s.body}` : s.body,
68
+ role: "user",
69
+ sourceFormat: "markdown",
70
+ lineNumber: s.lineNumber,
71
+ }));
72
+ }
73
+ function idempotencyKey(filePath, lineNumber, textPrefix) {
74
+ return createHash("sha256")
75
+ .update(`${filePath}:${lineNumber}:${textPrefix}`)
76
+ .digest("hex")
77
+ .slice(0, 32);
78
+ }
79
+ /**
80
+ * Detect whether a file is markdown based on extension.
81
+ * Used by the CLI to route to extractMarkdown() vs extractJsonl().
82
+ */
83
+ export function isMarkdownFile(filePath) {
84
+ return /\.(md|markdown|mdx)$/i.test(filePath);
85
+ }
86
+ /**
87
+ * Extract memories from a Markdown document.
88
+ *
89
+ * Pipeline per section:
90
+ * 1. Split file by headings → MarkdownSection[]
91
+ * 2. Convert to ExtractionRecord[]
92
+ * 3. Role filter (default: user only — all sections are "user")
93
+ * 4. redactSecrets()
94
+ * 5. shouldCapture() — filter noise
95
+ * 6. looksLikePromptInjection() — safety check
96
+ * 7. detectMemoryType() + extractTags() — enrichment
97
+ * 8. Vector dedup via searchMemories()
98
+ * 9. createMemory() with idempotency key
99
+ * 10. Optional local markdown fallback
100
+ */
101
+ export async function extractMarkdown(options, deps) {
102
+ const startTime = Date.now();
103
+ const stats = {
104
+ linesRead: 0,
105
+ linesParsed: 0,
106
+ linesSkipped: 0,
107
+ recordsExtracted: 0,
108
+ recordsFiltered: 0,
109
+ recordsDeduped: 0,
110
+ recordsStored: 0,
111
+ secretsRedacted: 0,
112
+ markdownWritten: 0,
113
+ errors: 0,
114
+ durationMs: 0,
115
+ };
116
+ const { filePath, channel = "md-extract", dedup = true, dedupThreshold = 0.92, localFallback = false, dryRun = false, limit, strict = false, roles = ["user"], } = options;
117
+ // Read the entire file
118
+ let content;
119
+ try {
120
+ content = await fs.readFile(filePath, "utf-8");
121
+ }
122
+ catch (err) {
123
+ deps.logger.warn(`Failed to read ${filePath}: ${err instanceof Error ? err.message : "unknown"}`);
124
+ stats.errors++;
125
+ stats.durationMs = Date.now() - startTime;
126
+ return stats;
127
+ }
128
+ const lines = content.split("\n");
129
+ stats.linesRead = lines.length;
130
+ deps.logger.info(`Format detected: markdown`);
131
+ // Split into sections
132
+ const filename = basename(filePath);
133
+ const sections = splitMarkdownSections(content, filename);
134
+ const records = sectionsToRecords(sections);
135
+ stats.linesParsed = stats.linesRead; // all lines are "parsed" in document mode
136
+ stats.recordsExtracted = records.length;
137
+ let totalProcessed = 0;
138
+ for (const record of records) {
139
+ if (limit && totalProcessed >= limit)
140
+ break;
141
+ // Role filter
142
+ if (roles.length > 0 && !roles.includes(record.role))
143
+ continue;
144
+ // Step 1: Redact secrets
145
+ const redaction = redactSecrets(record.text);
146
+ stats.secretsRedacted += redaction.secretsFound;
147
+ const cleanText = redaction.text;
148
+ // Step 2: Capture filter
149
+ if (!shouldCapture(cleanText, { strict })) {
150
+ stats.recordsFiltered++;
151
+ continue;
152
+ }
153
+ // Step 3: Prompt injection check
154
+ if (looksLikePromptInjection(cleanText)) {
155
+ stats.recordsFiltered++;
156
+ continue;
157
+ }
158
+ // Step 4: Enrichment
159
+ const memoryType = detectMemoryType(cleanText, filename);
160
+ const tags = [
161
+ ...extractTags(cleanText, filename),
162
+ "md-extract",
163
+ filename.replace(/\.md$/i, "").toLowerCase(),
164
+ ];
165
+ // Step 5: Vector dedup
166
+ if (dedup && !dryRun) {
167
+ try {
168
+ const existing = await deps.client.searchMemories({
169
+ query: cleanText.slice(0, 500),
170
+ threshold: dedupThreshold,
171
+ limit: 1,
172
+ });
173
+ if (existing && existing.length > 0) {
174
+ stats.recordsDeduped++;
175
+ continue;
176
+ }
177
+ }
178
+ catch (err) {
179
+ deps.logger.warn(`Dedup check failed: ${err instanceof Error ? err.message : "unknown"}`);
180
+ }
181
+ }
182
+ // Step 6: Store
183
+ if (!dryRun) {
184
+ const title = cleanText.slice(0, 80).replace(/\s+/g, " ").trim();
185
+ const params = {
186
+ title,
187
+ content: cleanText,
188
+ type: memoryType,
189
+ tags,
190
+ metadata: {
191
+ agent_id: deps.config.agentId,
192
+ source: "markdown",
193
+ channel,
194
+ line_number: record.lineNumber,
195
+ source_file: filename,
196
+ captured_at: new Date().toISOString(),
197
+ secrets_redacted: redaction.secretsFound,
198
+ },
199
+ idempotency_key: idempotencyKey(filePath, record.lineNumber, cleanText.slice(0, 200)),
200
+ };
201
+ try {
202
+ await deps.client.createMemory(params);
203
+ stats.recordsStored++;
204
+ }
205
+ catch (err) {
206
+ stats.errors++;
207
+ deps.logger.warn(`Store failed at line ${record.lineNumber}: ${err instanceof Error ? err.message : "unknown"}`);
208
+ }
209
+ }
210
+ else {
211
+ stats.recordsStored++;
212
+ }
213
+ // Step 7: Local markdown fallback
214
+ if (localFallback && deps.fallback) {
215
+ try {
216
+ const title = cleanText.slice(0, 80).replace(/\s+/g, " ").trim();
217
+ await deps.fallback.writeMemory(title, cleanText);
218
+ stats.markdownWritten++;
219
+ }
220
+ catch {
221
+ // Non-fatal
222
+ }
223
+ }
224
+ totalProcessed++;
225
+ }
226
+ stats.durationMs = Date.now() - startTime;
227
+ return stats;
228
+ }
@@ -0,0 +1,7 @@
1
+ import type { RedactionResult } from "./types.js";
2
+ export declare function redactSecrets(text: string, options?: {
3
+ redactPII?: boolean;
4
+ }): RedactionResult;
5
+ export declare function containsSecrets(text: string, options?: {
6
+ redactPII?: boolean;
7
+ }): boolean;
@@ -0,0 +1,112 @@
1
+ // Secret redaction for JSONL extraction
2
+ // Detects and redacts API keys, tokens, credentials, and PII
3
+ const SECRET_PATTERNS = [
4
+ {
5
+ name: "lanonasis-api-key",
6
+ pattern: /\blano_[A-Za-z0-9_-]{20,}\b/g,
7
+ },
8
+ {
9
+ name: "openai-api-key",
10
+ pattern: /\bsk-[A-Za-z0-9]{20,}\b/g,
11
+ },
12
+ {
13
+ name: "anthropic-api-key",
14
+ pattern: /\bsk-ant-[A-Za-z0-9_-]{20,}\b/g,
15
+ },
16
+ {
17
+ name: "generic-api-key",
18
+ pattern: /\b(api[_-]?key|apikey)\s*[:=]\s*["']?[A-Za-z0-9_-]{16,}["']?/gi,
19
+ },
20
+ {
21
+ name: "bearer-token",
22
+ pattern: /\bBearer\s+[A-Za-z0-9._~+\/=-]{20,}\b/gi,
23
+ },
24
+ {
25
+ name: "jwt-token",
26
+ pattern: /\beyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g,
27
+ },
28
+ {
29
+ name: "aws-access-key",
30
+ pattern: /\bAKIA[A-Z0-9]{16}\b/g,
31
+ },
32
+ {
33
+ name: "aws-secret-key",
34
+ pattern: /\baws[_-]?secret[_-]?access[_-]?key\s*[:=]\s*["']?[A-Za-z0-9/+=]{40}["']?/gi,
35
+ },
36
+ {
37
+ name: "database-url",
38
+ pattern: /\b(postgres|postgresql|mongodb|mysql|redis):\/\/[^"'\s)]+/gi,
39
+ },
40
+ {
41
+ name: "supabase-key",
42
+ pattern: /\b(eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+|[A-Za-z0-9_-]{40,})\b/g,
43
+ },
44
+ {
45
+ name: "secret-key",
46
+ pattern: /\b(secret[_-]?key|private[_-]?key)\s*[:=]\s*["']?[A-Za-z0-9_-]{16,}["']?/gi,
47
+ },
48
+ {
49
+ name: "password",
50
+ pattern: /\b(password|passwd|pwd)\s*[:=]\s*["']?[^\s"']{8,}["']?/gi,
51
+ },
52
+ ];
53
+ const PII_PATTERNS = [
54
+ {
55
+ name: "email",
56
+ pattern: /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g,
57
+ },
58
+ {
59
+ name: "phone",
60
+ pattern: /\+?[\d\s()-]{10,}\b/g,
61
+ },
62
+ {
63
+ name: "credit-card",
64
+ pattern: /\b\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}\b/g,
65
+ },
66
+ {
67
+ name: "ssn",
68
+ pattern: /\b\d{3}[\s-]?\d{2}[\s-]?\d{4}\b/g,
69
+ },
70
+ ];
71
+ export function redactSecrets(text, options = {}) {
72
+ let result = text;
73
+ const secretsFound = [];
74
+ const { redactPII = true } = options;
75
+ for (const { name, pattern } of SECRET_PATTERNS) {
76
+ const matches = result.match(pattern);
77
+ if (matches && matches.length > 0) {
78
+ secretsFound.push(name);
79
+ result = result.replace(pattern, `[REDACTED:${name}]`);
80
+ }
81
+ }
82
+ if (redactPII) {
83
+ for (const { name, pattern } of PII_PATTERNS) {
84
+ const matches = result.match(pattern);
85
+ if (matches && matches.length > 0) {
86
+ secretsFound.push(name);
87
+ result = result.replace(pattern, `[REDACTED:${name}]`);
88
+ }
89
+ }
90
+ }
91
+ return {
92
+ text: result,
93
+ secretsFound: secretsFound.length,
94
+ types: secretsFound,
95
+ };
96
+ }
97
+ export function containsSecrets(text, options = {}) {
98
+ const { redactPII = true } = options;
99
+ for (const { pattern } of SECRET_PATTERNS) {
100
+ if (pattern.test(text)) {
101
+ return true;
102
+ }
103
+ }
104
+ if (redactPII) {
105
+ for (const { pattern } of PII_PATTERNS) {
106
+ if (pattern.test(text)) {
107
+ return true;
108
+ }
109
+ }
110
+ }
111
+ return false;
112
+ }
@@ -0,0 +1,15 @@
1
+ import type { ExtractionOptions, ExtractionStats } from "./types.js";
2
+ import type { ExtractionDeps } from "./jsonl-extractor.js";
3
+ /**
4
+ * Detect whether a file is a SQLite database based on extension.
5
+ */
6
+ export declare function isSqliteFile(filePath: string): boolean;
7
+ /**
8
+ * Extract memories from an OpenClaw SQLite memory database.
9
+ *
10
+ * Reads the `chunks` table and processes each chunk through the standard pipeline:
11
+ * redact → filter → enrich → dedup → store
12
+ *
13
+ * Also reads `meta` for the embedding model info which is logged and tagged.
14
+ */
15
+ export declare function extractSqlite(options: ExtractionOptions, deps: ExtractionDeps): Promise<ExtractionStats>;
@@ -0,0 +1,245 @@
1
+ // Document-mode extraction for OpenClaw SQLite memory databases
2
+ // Reads chunks table, emits each chunk as an ExtractionRecord
3
+ import { createHash } from "crypto";
4
+ import { basename } from "path";
5
+ import { shouldCapture } from "../enrichment/capture-filter.js";
6
+ import { detectMemoryType } from "../enrichment/type-detector.js";
7
+ import { extractTags } from "../enrichment/tag-extractor.js";
8
+ import { looksLikePromptInjection } from "../enrichment/prompt-safety.js";
9
+ import { redactSecrets } from "./secret-redactor.js";
10
+ /**
11
+ * Detect whether a file is a SQLite database based on extension.
12
+ */
13
+ export function isSqliteFile(filePath) {
14
+ return /\.(sqlite3?|db)$/i.test(filePath);
15
+ }
16
+ function idempotencyKey(dbPath, chunkId) {
17
+ return createHash("sha256")
18
+ .update(`sqlite:${dbPath}:${chunkId}`)
19
+ .digest("hex")
20
+ .slice(0, 32);
21
+ }
22
+ /**
23
+ * Open a SQLite database using available runtime bindings.
24
+ *
25
+ * Priority:
26
+ * 1. bun:sqlite — available when running under Bun (OpenClaw's runtime)
27
+ * 2. node:sqlite — available in Node.js >= 22.5.0 (experimental)
28
+ * 3. Throws with a clear message if neither is available
29
+ *
30
+ * No child_process or shell commands are used.
31
+ */
32
+ async function openDatabase(dbPath) {
33
+ // 1. Try Bun's built-in SQLite (primary path — OpenClaw runs on Bun)
34
+ try {
35
+ // @ts-ignore — bun:sqlite is a Bun-specific module, not in @types
36
+ const { Database } = await import("bun:sqlite");
37
+ const db = new Database(dbPath, { readonly: true });
38
+ return {
39
+ all(sql) {
40
+ return db.query(sql).all();
41
+ },
42
+ get(sql) {
43
+ return db.query(sql).get();
44
+ },
45
+ close() {
46
+ db.close();
47
+ },
48
+ };
49
+ }
50
+ catch {
51
+ // bun:sqlite unavailable — try Node.js built-in (Node >= 22.5.0)
52
+ }
53
+ // 2. Try Node's built-in sqlite module (Node 22.5+ experimental)
54
+ try {
55
+ // @ts-ignore — node:sqlite is experimental and not in stable @types/node
56
+ const { DatabaseSync } = await import("node:sqlite");
57
+ const db = new DatabaseSync(dbPath, { open: true });
58
+ return {
59
+ all(sql) {
60
+ const stmt = db.prepare(sql);
61
+ return stmt.all();
62
+ },
63
+ get(sql) {
64
+ const stmt = db.prepare(sql);
65
+ return stmt.get();
66
+ },
67
+ close() {
68
+ db.close();
69
+ },
70
+ };
71
+ }
72
+ catch {
73
+ // node:sqlite unavailable
74
+ }
75
+ throw new Error(`SQLite extraction requires Bun (bun:sqlite) or Node.js >= 22.5 (node:sqlite). ` +
76
+ `Neither was available in the current runtime. Run this command under Bun or upgrade Node.`);
77
+ }
78
+ /**
79
+ * Extract memories from an OpenClaw SQLite memory database.
80
+ *
81
+ * Reads the `chunks` table and processes each chunk through the standard pipeline:
82
+ * redact → filter → enrich → dedup → store
83
+ *
84
+ * Also reads `meta` for the embedding model info which is logged and tagged.
85
+ */
86
+ export async function extractSqlite(options, deps) {
87
+ const startTime = Date.now();
88
+ const stats = {
89
+ linesRead: 0,
90
+ linesParsed: 0,
91
+ linesSkipped: 0,
92
+ recordsExtracted: 0,
93
+ recordsFiltered: 0,
94
+ recordsDeduped: 0,
95
+ recordsStored: 0,
96
+ secretsRedacted: 0,
97
+ markdownWritten: 0,
98
+ errors: 0,
99
+ durationMs: 0,
100
+ };
101
+ const { filePath, channel = "sqlite-extract", dedup = true, dedupThreshold = 0.92, localFallback = false, dryRun = false, limit, strict = false, } = options;
102
+ const dbName = basename(filePath);
103
+ let db;
104
+ try {
105
+ db = await openDatabase(filePath);
106
+ }
107
+ catch (err) {
108
+ deps.logger.warn(`Failed to open ${filePath}: ${err instanceof Error ? err.message : "unknown"}`);
109
+ stats.errors++;
110
+ stats.durationMs = Date.now() - startTime;
111
+ return stats;
112
+ }
113
+ deps.logger.info(`Format detected: sqlite (${dbName})`);
114
+ // Read embedding model from meta if available
115
+ let embeddingModel = "unknown";
116
+ try {
117
+ const meta = db.get("SELECT value FROM meta WHERE key = 'memory_index_meta_v1'");
118
+ if (meta && typeof meta.value === "string") {
119
+ try {
120
+ const parsed = JSON.parse(meta.value);
121
+ embeddingModel = parsed.model ?? "unknown";
122
+ deps.logger.info(`Embedding model: ${embeddingModel} via ${parsed.provider ?? "unknown"}`);
123
+ }
124
+ catch { /* non-fatal */ }
125
+ }
126
+ }
127
+ catch { /* meta table might not exist */ }
128
+ // Read all chunks
129
+ let chunks;
130
+ try {
131
+ chunks = db.all("SELECT id, path, source, start_line, end_line, model, text, updated_at FROM chunks ORDER BY updated_at ASC");
132
+ }
133
+ catch (err) {
134
+ deps.logger.warn(`Failed to read chunks: ${err instanceof Error ? err.message : "unknown"}`);
135
+ db.close();
136
+ stats.errors++;
137
+ stats.durationMs = Date.now() - startTime;
138
+ return stats;
139
+ }
140
+ stats.linesRead = chunks.length;
141
+ stats.linesParsed = chunks.length;
142
+ stats.recordsExtracted = chunks.length;
143
+ let totalProcessed = 0;
144
+ for (const chunk of chunks) {
145
+ if (limit && totalProcessed >= limit)
146
+ break;
147
+ if (!chunk.text || chunk.text.trim().length === 0) {
148
+ stats.linesSkipped++;
149
+ continue;
150
+ }
151
+ // Step 1: Redact secrets
152
+ const redaction = redactSecrets(chunk.text);
153
+ stats.secretsRedacted += redaction.secretsFound;
154
+ const cleanText = redaction.text;
155
+ // Step 2: Capture filter
156
+ if (!shouldCapture(cleanText, { strict })) {
157
+ stats.recordsFiltered++;
158
+ continue;
159
+ }
160
+ // Step 3: Prompt injection check
161
+ if (looksLikePromptInjection(cleanText)) {
162
+ stats.recordsFiltered++;
163
+ continue;
164
+ }
165
+ // Step 4: Enrichment
166
+ const sourceFile = chunk.path || dbName;
167
+ const memoryType = detectMemoryType(cleanText, sourceFile);
168
+ const tags = [
169
+ ...extractTags(cleanText, sourceFile),
170
+ "sqlite-extract",
171
+ dbName.replace(/\.sqlite3?$/i, ""),
172
+ ];
173
+ if (chunk.source)
174
+ tags.push(chunk.source);
175
+ // Step 5: Vector dedup
176
+ if (dedup && !dryRun) {
177
+ try {
178
+ const existing = await deps.client.searchMemories({
179
+ query: cleanText.slice(0, 500),
180
+ threshold: dedupThreshold,
181
+ limit: 1,
182
+ });
183
+ if (existing && existing.length > 0) {
184
+ stats.recordsDeduped++;
185
+ continue;
186
+ }
187
+ }
188
+ catch (err) {
189
+ deps.logger.warn(`Dedup check failed: ${err instanceof Error ? err.message : "unknown"}`);
190
+ }
191
+ }
192
+ // Step 6: Store
193
+ if (!dryRun) {
194
+ const title = cleanText.slice(0, 80).replace(/\s+/g, " ").trim();
195
+ const params = {
196
+ title,
197
+ content: cleanText,
198
+ type: memoryType,
199
+ tags,
200
+ metadata: {
201
+ agent_id: deps.config.agentId,
202
+ source: "sqlite",
203
+ channel,
204
+ source_file: sourceFile,
205
+ chunk_id: chunk.id,
206
+ start_line: chunk.start_line,
207
+ end_line: chunk.end_line,
208
+ embedding_model: chunk.model || embeddingModel,
209
+ captured_at: chunk.updated_at
210
+ ? new Date(chunk.updated_at * 1000).toISOString()
211
+ : new Date().toISOString(),
212
+ secrets_redacted: redaction.secretsFound,
213
+ },
214
+ idempotency_key: idempotencyKey(filePath, chunk.id),
215
+ };
216
+ try {
217
+ await deps.client.createMemory(params);
218
+ stats.recordsStored++;
219
+ }
220
+ catch (err) {
221
+ stats.errors++;
222
+ deps.logger.warn(`Store failed for chunk ${chunk.id.slice(0, 8)}: ${err instanceof Error ? err.message : "unknown"}`);
223
+ }
224
+ }
225
+ else {
226
+ stats.recordsStored++;
227
+ }
228
+ // Step 7: Local markdown fallback
229
+ if (localFallback && deps.fallback) {
230
+ try {
231
+ const title = cleanText.slice(0, 80).replace(/\s+/g, " ").trim();
232
+ await deps.fallback.writeMemory(title, cleanText);
233
+ stats.markdownWritten++;
234
+ }
235
+ catch { /* non-fatal */ }
236
+ }
237
+ totalProcessed++;
238
+ if (totalProcessed % 100 === 0) {
239
+ deps.logger.info(`Progress: ${totalProcessed}/${chunks.length} chunks processed, ${stats.recordsStored} stored`);
240
+ }
241
+ }
242
+ db.close();
243
+ stats.durationMs = Date.now() - startTime;
244
+ return stats;
245
+ }
@@ -0,0 +1,50 @@
1
+ /** A single meaningful record extracted from one JSONL line */
2
+ export interface ExtractionRecord {
3
+ text: string;
4
+ role: "user" | "assistant" | "system" | "unknown";
5
+ sourceFormat: string;
6
+ lineNumber: number;
7
+ timestamp?: string;
8
+ }
9
+ /** Result of the redaction pass */
10
+ export interface RedactionResult {
11
+ text: string;
12
+ secretsFound: number;
13
+ types: string[];
14
+ }
15
+ /** Stats reported after extraction completes */
16
+ export interface ExtractionStats {
17
+ linesRead: number;
18
+ linesParsed: number;
19
+ linesSkipped: number;
20
+ recordsExtracted: number;
21
+ recordsFiltered: number;
22
+ recordsDeduped: number;
23
+ recordsStored: number;
24
+ secretsRedacted: number;
25
+ markdownWritten: number;
26
+ errors: number;
27
+ durationMs: number;
28
+ }
29
+ /** Options for the extraction function */
30
+ export interface ExtractionOptions {
31
+ filePath: string;
32
+ format?: "claude-code" | "openclaw-cache" | "openclaw-session" | "codex" | "generic" | "markdown" | "sqlite";
33
+ channel?: string;
34
+ dedup?: boolean;
35
+ dedupThreshold?: number;
36
+ localFallback?: boolean;
37
+ batchSize?: number;
38
+ dryRun?: boolean;
39
+ limit?: number;
40
+ strict?: boolean;
41
+ roles?: string[];
42
+ /** Enable PII detection alongside secret redaction (default: true) */
43
+ redactPII?: boolean;
44
+ }
45
+ /** A format adapter converts a parsed JSON line into ExtractionRecords */
46
+ export interface FormatAdapter {
47
+ name: string;
48
+ detect(sample: Record<string, unknown>): boolean;
49
+ extract(line: Record<string, unknown>, lineNumber: number): ExtractionRecord[];
50
+ }
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,23 @@
1
+ import type { LanonasisClient } from "../client.js";
2
+ import type { LanonasisConfig } from "../config.js";
3
+ import type { LocalFallbackWriter } from "./local-fallback.js";
4
+ import type { PrivacyGuard } from "../privacy/privacy-guard.js";
5
+ import type { PrivacyLogWriter } from "../privacy/privacy-log.js";
6
+ export declare function createCaptureHook(client: LanonasisClient, cfg: LanonasisConfig, logger: {
7
+ info(msg: string): void;
8
+ warn(msg: string): void;
9
+ }, fallback: LocalFallbackWriter, guard?: PrivacyGuard, privacyLog?: PrivacyLogWriter): (event: {
10
+ messages: unknown[];
11
+ success: boolean;
12
+ error?: string;
13
+ }) => Promise<void>;
14
+ export declare function createCompactionCaptureHook(client: LanonasisClient, cfg: LanonasisConfig, logger: {
15
+ info(msg: string): void;
16
+ warn(msg: string): void;
17
+ }, guard?: PrivacyGuard): (event: {
18
+ messageCount: number;
19
+ messages?: unknown[];
20
+ sessionFile?: string;
21
+ tokenCount?: number;
22
+ compactingCount?: number;
23
+ }) => Promise<void>;