@lanonasis/recall-forge 1.1.1

package/dist/extraction/format-adapters.js

@@ -0,0 +1,268 @@
+// Pluggable JSONL format parsers
+// Each adapter detects and extracts content from a specific JSONL format
+// Shared: extract text from Claude-style message content (string or content blocks)
+function extractMessageText(msg) {
+    const texts = [];
+    const content = msg.content;
+    if (typeof content === "string") {
+        texts.push(content);
+    }
+    else if (Array.isArray(content)) {
+        for (const block of content) {
+            const b = block;
+            if (b.type === "text" && typeof b.text === "string") {
+                texts.push(b.text);
+            }
+        }
+    }
+    return texts;
+}
+const TEXT_BLOCK_TYPES = new Set(["text", "input_text", "output_text", "markdown"]);
+function extractOpenClawSessionText(value) {
+    if (typeof value === "string") {
+        return value.length > 0 ? [value] : [];
+    }
+    if (Array.isArray(value)) {
+        return value.flatMap((entry) => extractOpenClawSessionText(entry));
+    }
+    if (!value || typeof value !== "object") {
+        return [];
+    }
+    const record = value;
+    const texts = [];
+    const blockType = typeof record.type === "string" ? record.type : undefined;
+    if (typeof record.text === "string" && (!blockType || TEXT_BLOCK_TYPES.has(blockType))) {
+        texts.push(record.text);
+    }
+    if ("content" in record) {
+        texts.push(...extractOpenClawSessionText(record.content));
+    }
+    if ("message" in record && typeof record.message !== "object") {
+        texts.push(...extractOpenClawSessionText(record.message));
+    }
+    if (typeof record.body === "string") {
+        texts.push(record.body);
+    }
+    if (typeof record.prompt === "string") {
+        texts.push(record.prompt);
+    }
+    return [...new Set(texts.filter((text) => text.length > 0))];
+}
+/**
+ * Claude Code session JSONL format
+ * Lines have: { role: "user"|"assistant"|"system", content: string|ContentBlock[] }
+ */
+const claudeCodeAdapter = {
+    name: "claude-code",
+    detect(sample) {
+        return (typeof sample.role === "string" &&
+            ["user", "assistant", "system"].includes(sample.role) &&
+            ("content" in sample || "type" in sample));
+    },
+    extract(line, lineNumber) {
+        const role = line.role;
+        const normalizedRole = role === "user" || role === "assistant" || role === "system"
+            ? role
+            : "unknown";
+        const texts = extractMessageText(line);
+        const timestamp = (line.timestamp ?? line.created_at);
+        return texts
+            .filter((t) => t.length > 0)
+            .map((text) => ({
+            text,
+            role: normalizedRole,
+            sourceFormat: "claude-code",
+            lineNumber,
+            timestamp,
+        }));
+    },
+};
+/**
+ * OpenClaw cache-trace JSONL format
+ * Lines may have request/response payloads, tool outputs, etc.
+ */
+const openclawCacheAdapter = {
+    name: "openclaw-cache",
+    detect(sample) {
+        return ("cache_key" in sample ||
+            "trace_id" in sample ||
+            ("request" in sample && "response" in sample) ||
+            ("tool" in sample && "output" in sample));
+    },
+    extract(line, lineNumber) {
+        const records = [];
+        const timestamp = (line.timestamp ?? line.ts ?? line.created_at);
+        // Extract from request body
+        if (line.request && typeof line.request === "object") {
+            const req = line.request;
+            const texts = extractMessageText(req);
+            if (typeof req.body === "string")
+                texts.push(req.body);
+            if (typeof req.prompt === "string")
+                texts.push(req.prompt);
+            for (const text of texts.filter((t) => t.length > 0)) {
+                records.push({
+                    text,
+                    role: "user",
+                    sourceFormat: "openclaw-cache",
+                    lineNumber,
+                    timestamp,
+                });
+            }
+        }
+        // Extract from response body
+        if (line.response && typeof line.response === "object") {
+            const res = line.response;
+            const texts = extractMessageText(res);
+            if (typeof res.body === "string")
+                texts.push(res.body);
+            if (typeof res.result === "string")
+                texts.push(res.result);
+            for (const text of texts.filter((t) => t.length > 0)) {
+                records.push({
+                    text,
+                    role: "assistant",
+                    sourceFormat: "openclaw-cache",
+                    lineNumber,
+                    timestamp,
+                });
+            }
+        }
+        // Extract from tool output
+        if (typeof line.output === "string" && line.output.length > 0) {
+            records.push({
+                text: line.output,
+                role: "assistant",
+                sourceFormat: "openclaw-cache",
+                lineNumber,
+                timestamp,
+            });
+        }
+        // Extract from top-level content
+        if (typeof line.content === "string" && line.content.length > 0) {
+            records.push({
+                text: line.content,
+                role: "unknown",
+                sourceFormat: "openclaw-cache",
+                lineNumber,
+                timestamp,
+            });
+        }
+        return records;
+    },
+};
+/**
+ * OpenClaw session JSONL format
+ * Lines have: { type: "message", message: { role, content[] } }
+ */
+const openclawSessionAdapter = {
+    name: "openclaw-session",
+    detect(sample) {
+        // Session header line (first line of OpenClaw session files)
+        if (sample.type === "session" && ("cwd" in sample || "id" in sample))
+            return true;
+        // Standard message line
+        if (sample.type !== "message" || !sample.message || typeof sample.message !== "object") {
+            return false;
+        }
+        const message = sample.message;
+        return typeof message.role === "string" && ("content" in message || "text" in message);
+    },
+    extract(line, lineNumber) {
+        // Skip non-message lines (session header, model_change, thinking_level_change, custom, etc.)
+        if (line.type !== "message" || !line.message || typeof line.message !== "object") {
+            return [];
+        }
+        const message = line.message;
+        const role = message.role;
+        const normalizedRole = role === "user" || role === "assistant" || role === "system"
+            ? role
+            : "unknown";
+        const timestamp = (line.timestamp ?? line.ts ?? line.created_at ?? message.timestamp);
+        const texts = extractOpenClawSessionText(message.content ?? message.text ?? message);
+        return texts.map((text) => ({
+            text,
+            role: normalizedRole,
+            sourceFormat: "openclaw-session",
+            lineNumber,
+            timestamp,
+        }));
+    },
+};
+/**
+ * Codex session JSONL format
+ * Lines have: { type: "message", sender: "user"|"agent", content: string }
+ */
+const codexAdapter = {
+    name: "codex",
+    detect(sample) {
+        return ((sample.type === "message" || sample.type === "rollout") &&
+            ("sender" in sample || "agent" in sample));
+    },
+    extract(line, lineNumber) {
+        const records = [];
+        const timestamp = (line.timestamp ?? line.ts);
+        const sender = (line.sender ?? line.agent ?? "unknown");
+        const role = sender === "user" ? "user" : sender === "agent" ? "assistant" : "unknown";
+        if (typeof line.content === "string" && line.content.length > 0) {
+            records.push({ text: line.content, role: role, sourceFormat: "codex", lineNumber, timestamp });
+        }
+        if (typeof line.message === "string" && line.message.length > 0) {
+            records.push({ text: line.message, role: role, sourceFormat: "codex", lineNumber, timestamp });
+        }
+        return records;
+    },
+};
+/**
+ * Generic fallback — tries to find text in any JSON structure
+ */
+const genericAdapter = {
+    name: "generic",
+    detect() {
+        return true; // Always matches as fallback
+    },
+    extract(line, lineNumber) {
+        const records = [];
+        const timestamp = (line.timestamp ?? line.ts ?? line.created_at);
+        // Try common text field names
+        const textFields = ["content", "text", "message", "body", "prompt", "output", "result"];
+        for (const field of textFields) {
+            const val = line[field];
+            if (typeof val === "string" && val.length > 0) {
+                records.push({
+                    text: val,
+                    role: "unknown",
+                    sourceFormat: "generic",
+                    lineNumber,
+                    timestamp,
+                });
+            }
+        }
+        return records;
+    },
+};
+/** All adapters in detection priority order */
+export const FORMAT_ADAPTERS = [
+    claudeCodeAdapter,
+    openclawCacheAdapter,
+    openclawSessionAdapter,
+    codexAdapter,
+    genericAdapter,
+];
+/**
+ * Auto-detect the format adapter for a JSONL file
+ * Uses the first successfully parsed line as a sample
+ */
+export function detectFormat(sample, forceFormat) {
+    if (forceFormat) {
+        const forced = FORMAT_ADAPTERS.find((a) => a.name === forceFormat);
+        if (forced)
+            return forced;
+    }
+    for (const adapter of FORMAT_ADAPTERS) {
+        if (adapter.detect(sample))
+            return adapter;
+    }
+    // Should never reach here — generic always matches
+    return genericAdapter;
+}

package/dist/extraction/index.d.ts

@@ -0,0 +1,7 @@
+export { extractJsonl, formatStats } from "./jsonl-extractor.js";
+export { extractMarkdown, isMarkdownFile } from "./markdown-extractor.js";
+export { extractSqlite, isSqliteFile } from "./sqlite-extractor.js";
+export { redactSecrets, containsSecrets } from "./secret-redactor.js";
+export { detectFormat, FORMAT_ADAPTERS } from "./format-adapters.js";
+export { registerExtractCli } from "./cli-extract.js";
+export type { ExtractionOptions, ExtractionStats, ExtractionRecord, RedactionResult, FormatAdapter, } from "./types.js";

package/dist/extraction/index.js

@@ -0,0 +1,7 @@
+// Extraction pipeline — barrel export
+export { extractJsonl, formatStats } from "./jsonl-extractor.js";
+export { extractMarkdown, isMarkdownFile } from "./markdown-extractor.js";
+export { extractSqlite, isSqliteFile } from "./sqlite-extractor.js";
+export { redactSecrets, containsSecrets } from "./secret-redactor.js";
+export { detectFormat, FORMAT_ADAPTERS } from "./format-adapters.js";
+export { registerExtractCli } from "./cli-extract.js";

package/dist/extraction/jsonl-extractor.d.ts

@@ -0,0 +1,32 @@
+import type { LanonasisClient } from "../client.js";
+import type { LanonasisConfig } from "../config.js";
+import type { LocalFallbackWriter } from "../hooks/local-fallback.js";
+import type { ExtractionOptions, ExtractionStats } from "./types.js";
+export interface ExtractionDeps {
+    client: LanonasisClient;
+    config: LanonasisConfig;
+    logger: {
+        info(msg: string): void;
+        warn(msg: string): void;
+    };
+    fallback?: LocalFallbackWriter;
+}
+/**
+ * Extract memories from a JSONL file with secret redaction
+ *
+ * Pipeline per record:
+ * 1. Parse JSONL line → format adapter → ExtractionRecord
+ * 2. Role filter (default: user only)
+ * 3. redactSecrets() — strips API keys, tokens, credentials
+ * 4. shouldCapture() — filters noise
+ * 5. looksLikePromptInjection() — safety check
+ * 6. detectMemoryType() + extractTags() — enrichment
+ * 7. vector dedup via searchMemories() — skip duplicates
+ * 8. createMemory() with idempotency key — store
+ * 9. Optional local markdown fallback
+ */
+export declare function extractJsonl(options: ExtractionOptions, deps: ExtractionDeps): Promise<ExtractionStats>;
+/**
+ * Format extraction stats as a human-readable report
+ */
+export declare function formatStats(stats: ExtractionStats, dryRun: boolean): string;

package/dist/extraction/jsonl-extractor.js

@@ -0,0 +1,207 @@
+// Core streaming JSONL extraction engine
+// Reads any JSONL source, redacts secrets, enriches, deduplicates, stores as memories
+import { createReadStream } from "fs";
+import { createInterface } from "readline";
+import { createHash } from "crypto";
+import { shouldCapture } from "../enrichment/capture-filter.js";
+import { detectMemoryType } from "../enrichment/type-detector.js";
+import { extractTags } from "../enrichment/tag-extractor.js";
+import { looksLikePromptInjection } from "../enrichment/prompt-safety.js";
+import { redactSecrets } from "./secret-redactor.js";
+import { detectFormat } from "./format-adapters.js";
+function idempotencyKey(filePath, lineNumber, textPrefix) {
+    return createHash("sha256")
+        .update(`${filePath}:${lineNumber}:${textPrefix}`)
+        .digest("hex")
+        .slice(0, 32);
+}
+/**
+ * Extract memories from a JSONL file with secret redaction
+ *
+ * Pipeline per record:
+ * 1. Parse JSONL line → format adapter → ExtractionRecord
+ * 2. Role filter (default: user only)
+ * 3. redactSecrets() — strips API keys, tokens, credentials
+ * 4. shouldCapture() — filters noise
+ * 5. looksLikePromptInjection() — safety check
+ * 6. detectMemoryType() + extractTags() — enrichment
+ * 7. vector dedup via searchMemories() — skip duplicates
+ * 8. createMemory() with idempotency key — store
+ * 9. Optional local markdown fallback
+ */
+export async function extractJsonl(options, deps) {
+    const startTime = Date.now();
+    const stats = {
+        linesRead: 0,
+        linesParsed: 0,
+        linesSkipped: 0,
+        recordsExtracted: 0,
+        recordsFiltered: 0,
+        recordsDeduped: 0,
+        recordsStored: 0,
+        secretsRedacted: 0,
+        markdownWritten: 0,
+        errors: 0,
+        durationMs: 0,
+    };
+    const { filePath, format, channel = "jsonl-extract", dedup = true, dedupThreshold = 0.92, localFallback = false, dryRun = false, limit, strict = false, roles = ["user"], } = options;
+    let adapter = null;
+    let totalProcessed = 0;
+    // Stream the file line by line
+    const fileStream = createReadStream(filePath, { encoding: "utf-8" });
+    const rl = createInterface({ input: fileStream, crlfDelay: Infinity });
+    for await (const line of rl) {
+        stats.linesRead++;
+        // Check limit
+        if (limit && totalProcessed >= limit)
+            break;
+        // Skip empty lines
+        const trimmed = line.trim();
+        if (!trimmed) {
+            stats.linesSkipped++;
+            continue;
+        }
+        // Parse JSON
+        let parsed;
+        try {
+            parsed = JSON.parse(trimmed);
+        }
+        catch {
+            stats.linesSkipped++;
+            stats.errors++;
+            continue;
+        }
+        stats.linesParsed++;
+        // Auto-detect format on first valid line
+        if (!adapter) {
+            adapter = detectFormat(parsed, format);
+            deps.logger.info(`Format detected: ${adapter.name}`);
+        }
+        // Extract records from this line
+        const records = adapter.extract(parsed, stats.linesRead);
+        stats.recordsExtracted += records.length;
+        // Process each record through the pipeline
+        for (const record of records) {
+            if (limit && totalProcessed >= limit)
+                break;
+            // Role filter
+            if (roles.length > 0 && !roles.includes(record.role))
+                continue;
+            // Step 1: Redact secrets FIRST — before any other processing
+            const redaction = redactSecrets(record.text);
+            stats.secretsRedacted += redaction.secretsFound;
+            const cleanText = redaction.text;
+            // Step 2: Capture filter
+            if (!shouldCapture(cleanText, { strict })) {
+                stats.recordsFiltered++;
+                continue;
+            }
+            // Step 3: Prompt injection check
+            if (looksLikePromptInjection(cleanText)) {
+                stats.recordsFiltered++;
+                continue;
+            }
+            // Step 4: Enrichment
+            const memoryType = detectMemoryType(cleanText);
+            const tags = [
+                ...extractTags(cleanText),
+                "jsonl-extract",
+                record.sourceFormat,
+            ];
+            // Step 5: Vector dedup (if enabled and not dry run)
+            if (dedup && !dryRun) {
+                try {
+                    const existing = await deps.client.searchMemories({
+                        query: cleanText.slice(0, 500),
+                        threshold: dedupThreshold,
+                        limit: 1,
+                    });
+                    if (existing && existing.length > 0) {
+                        stats.recordsDeduped++;
+                        continue;
+                    }
+                }
+                catch (err) {
+                    // Dedup failure is non-fatal — proceed to store
+                    deps.logger.warn(`Dedup check failed: ${err instanceof Error ? err.message : "unknown"}`);
+                }
+            }
+            // Step 6: Store
+            if (!dryRun) {
+                const title = cleanText.slice(0, 80).replace(/\s+/g, " ").trim();
+                const params = {
+                    title,
+                    content: cleanText,
+                    type: memoryType,
+                    tags,
+                    metadata: {
+                        agent_id: deps.config.agentId,
+                        source: record.sourceFormat,
+                        channel,
+                        line_number: record.lineNumber,
+                        captured_at: record.timestamp ?? new Date().toISOString(),
+                        secrets_redacted: redaction.secretsFound,
+                    },
+                    idempotency_key: idempotencyKey(filePath, record.lineNumber, cleanText.slice(0, 200)),
+                };
+                try {
+                    await deps.client.createMemory(params);
+                    stats.recordsStored++;
+                }
+                catch (err) {
+                    stats.errors++;
+                    deps.logger.warn(`Store failed at line ${record.lineNumber}: ${err instanceof Error ? err.message : "unknown"}`);
+                }
+            }
+            else {
+                // Dry run — count as "would store"
+                stats.recordsStored++;
+            }
+            // Step 7: Local markdown fallback
+            if (localFallback && deps.fallback) {
+                try {
+                    const title = cleanText.slice(0, 80).replace(/\s+/g, " ").trim();
+                    await deps.fallback.writeMemory(title, cleanText);
+                    stats.markdownWritten++;
+                }
+                catch {
+                    // Non-fatal
+                }
+            }
+            totalProcessed++;
+            // Progress indicator every 500 records
+            if (totalProcessed % 500 === 0) {
+                deps.logger.info(`Progress: ${totalProcessed} records processed, ${stats.recordsStored} stored, ${stats.secretsRedacted} secrets redacted`);
+            }
+        }
+    }
+    stats.durationMs = Date.now() - startTime;
+    return stats;
+}
+/**
+ * Format extraction stats as a human-readable report
+ */
+export function formatStats(stats, dryRun) {
+    const lines = [
+        "",
+        dryRun ? "=== DRY RUN REPORT ===" : "=== EXTRACTION REPORT ===",
+        "",
+        `Lines read:        ${stats.linesRead}`,
+        `Lines parsed:      ${stats.linesParsed}`,
+        `Lines skipped:     ${stats.linesSkipped}`,
+        "",
+        `Records extracted: ${stats.recordsExtracted}`,
+        `Records filtered:  ${stats.recordsFiltered} (noise/injection)`,
+        `Records deduped:   ${stats.recordsDeduped} (already in memory)`,
+        `Records stored:    ${stats.recordsStored}${dryRun ? " (would store)" : ""}`,
+        "",
+        `Secrets redacted:  ${stats.secretsRedacted}`,
+        `Errors:            ${stats.errors}`,
+        "",
+        `Duration:          ${(stats.durationMs / 1000).toFixed(1)}s`,
+    ];
+    if (stats.markdownWritten > 0) {
+        lines.push(`Markdown written:  ${stats.markdownWritten}`);
+    }
+    return lines.join("\n");
+}

package/dist/extraction/markdown-extractor.d.ts

@@ -0,0 +1,23 @@
+import type { ExtractionOptions, ExtractionStats } from "./types.js";
+import type { ExtractionDeps } from "./jsonl-extractor.js";
+/**
+ * Detect whether a file is markdown based on extension.
+ * Used by the CLI to route to extractMarkdown() vs extractJsonl().
+ */
+export declare function isMarkdownFile(filePath: string): boolean;
+/**
+ * Extract memories from a Markdown document.
+ *
+ * Pipeline per section:
+ * 1. Split file by headings → MarkdownSection[]
+ * 2. Convert to ExtractionRecord[]
+ * 3. Role filter (default: user only — all sections are "user")
+ * 4. redactSecrets()
+ * 5. shouldCapture() — filter noise
+ * 6. looksLikePromptInjection() — safety check
+ * 7. detectMemoryType() + extractTags() — enrichment
+ * 8. Vector dedup via searchMemories()
+ * 9. createMemory() with idempotency key
+ * 10. Optional local markdown fallback
+ */
+export declare function extractMarkdown(options: ExtractionOptions, deps: ExtractionDeps): Promise<ExtractionStats>;