metainsight-context-engine 0.0.1

package/dist/local-memory-sync.js

@@ -0,0 +1,1126 @@
+/**
+ * Local Memory Sync — Upload MEMORY.md, daily logs, images, and documents to cloud
+ *
+ * This module provides functionality to sync local memory files and assets
+ * (images + documents) to the cloud vector store.
+ *
+ * Local file locations (workspace root = ~/.openclaw/workspace/):
+ *   - Long-term memory : ~/.openclaw/workspace/MEMORY.md
+ *   - Short-term memory: ~/.openclaw/workspace/memory/YYYY-MM-DD.md
+ *
+ * COS storage layout — Multi-agent (2 top-level directories):
+ *
+ *   openclaw-{agentId}/
+ *   ├── workspace/            ← MEMORY.md + memory/*.md (indexed by CI)
+ *   │   ├── MEMORY.md         ← long-term memory (MEMORY.md)
+ *   │   └── memory/           ← daily logs (YYYY-MM-DD.md)
+ *   └── asset/                ← images + documents preserving directory structure
+ *       ├── screenshot.png
+ *       ├── report.pdf
+ *       ├── images/
+ *       │   └── photo.jpg
+ *       └── .codebuddy/
+ *           └── diagrams/
+ *               └── arch.png
+ *
+ * COS storage layout — Legacy (cosPrefix = "memory/"):
+ *   - Long-term memory : memory/memory.md
+ *   - Short-term memory: memory/memory/YYYY-MM-DD.md
+ *   - Assets           : asset/{relPath}
+ *
+ * Asset extraction:
+ *   When syncing memory files, asset links (Markdown `![](path)` for images
+ *   and `[](path)` for documents) are detected. If the referenced file exists
+ *   on the local filesystem and has a supported extension, it is uploaded to
+ *   the COS `asset/` directory. Image files get `category: image` metadata,
+ *   document files get `category: document` metadata.
+ *
+ *   Supported extensions are configurable via `syncFileExtensions` config.
+ *   Default: common image extensions (.png, .jpg, etc.) + document extensions
+ *   (.pdf, .doc, .docx, .xls, .xlsx, .ppt, .pptx, .txt, .csv, .md, .rtf).
+ *
+ * Key design decisions:
+ *   - Each file is uploaded with a **deterministic docId** derived from its
+ *     relative path. This means re-uploading the same file overwrites the
+ *     previous version instead of creating duplicates.
+ *   - Content is hashed (SHA-256) and compared against a local cache to skip
+ *     unchanged files (avoiding unnecessary COS PUT calls).
+ *   - Asset uploads use a separate hash cache to avoid re-uploading unchanged files.
+ *   - The sync runs in the background (non-blocking) and logs progress.
+ *   - All operations are gated behind the `localMemorySync` config flag.
+ *
+ * Architecture:
+ *   local-memory-sync.ts  ←  engine.ts (afterTurn / bootstrap)
+ *        ↓
+ *   cos-operations.ts (upload)
+ */
+import crypto from 'node:crypto';
+import fs from 'node:fs/promises';
+import fsSync from 'node:fs';
+import path from 'node:path';
+/** In-memory mirror of the on-disk cache — loaded lazily on first access. */
+let cacheData = null;
+/** Resolved absolute path to the cache file (set by `initHashCache`). */
+let cacheFilePath = null;
+/** Whether the in-memory cache is dirty and needs flushing. */
+let cacheDirty = false;
+/**
+ * Resolve the cache file path from the runtime environment.
+ * Follows the same stateDir convention as the rest of the plugin.
+ */
+function resolveCacheFilePath() {
+    if (cacheFilePath) {
+        return cacheFilePath;
+    }
+    const homeDir = process.env.HOME ?? process.env.USERPROFILE ?? '';
+    const stateDir = process.env.OPENCLAW_STATE_DIR?.trim()
+        || (homeDir ? `${homeDir}/.openclaw` : '');
+    cacheFilePath = stateDir
+        ? path.join(stateDir, '.sync-hash-cache.json')
+        : path.join(process.cwd(), '.sync-hash-cache.json');
+    return cacheFilePath;
+}
+/**
+ * Load the persistent hash cache from disk (no-op if already loaded).
+ * If the file does not exist or is corrupted, starts with an empty cache.
+ */
+function loadHashCache() {
+    if (cacheData) {
+        return cacheData;
+    }
+    const filePath = resolveCacheFilePath();
+    try {
+        const raw = fsSync.readFileSync(filePath, 'utf-8');
+        const parsed = JSON.parse(raw);
+        cacheData = {
+            memory: (parsed.memory && typeof parsed.memory === 'object') ? parsed.memory : {},
+            asset: (parsed.asset && typeof parsed.asset === 'object') ? parsed.asset : {},
+        };
+    }
+    catch {
+        // File missing or corrupted — start fresh
+        cacheData = { memory: {}, asset: {} };
+    }
+    return cacheData;
+}
+/**
+ * Flush the in-memory hash cache to disk.  Only writes if dirty.
+ * Called after a sync pass completes (not after every individual upload).
+ */
+export async function flushHashCache() {
+    if (!cacheDirty || !cacheData) {
+        return;
+    }
+    const filePath = resolveCacheFilePath();
+    try {
+        await fs.mkdir(path.dirname(filePath), { recursive: true });
+        await fs.writeFile(filePath, JSON.stringify(cacheData, null, 2), 'utf-8');
+        cacheDirty = false;
+    }
+    catch {
+        // Best-effort — cache will still work in-memory for this session
+    }
+}
+// --- convenience accessors wrapping the persistent cache ---
+const hashCache = {
+    get(key) {
+        return loadHashCache().memory[key];
+    },
+    set(key, value) {
+        loadHashCache().memory[key] = value;
+        cacheDirty = true;
+    },
+    clear() {
+        loadHashCache().memory = {};
+        cacheDirty = true;
+    },
+};
+function computeHash(content) {
+    return crypto.createHash('sha256').update(content).digest('hex').slice(0, 16);
+}
+function hasChanged(filePath, content) {
+    const newHash = computeHash(content);
+    const oldHash = hashCache.get(filePath);
+    if (oldHash === newHash) {
+        return false;
+    }
+    hashCache.set(filePath, newHash);
+    return true;
+}
+/**
+ * Clear both in-memory and on-disk hash caches (useful for testing
+ * or forced re-sync).  Pass `persistOnly = true` to only wipe the
+ * disk file without clearing the in-memory state.
+ */
+export async function clearSyncHashCache() {
+    cacheData = { memory: {}, asset: {} };
+    cacheDirty = true;
+    await flushHashCache();
+}
+// ============================================================================
+// File discovery
+// ============================================================================
+/**
+ * Resolve the workspace directory from a session file path.
+ *
+ * The workspace directory is the root where memory files (MEMORY.md,
+ * memory/*.md) and workspace identity files (AGENTS.md, SOUL.md, etc.) live.
+ *
+ * Resolution order:
+ *   1. Well-known path: `~/.openclaw/workspace/` (gateway default workspace)
+ *   2. Walk up from `sessionFile` to find a directory with workspace markers
+ *   3. Fallback: CWD
+ */
+function resolveWorkspaceDir(sessionFile) {
+    // ---- Priority 1: Check the well-known gateway workspace path ----
+    // The openclaw gateway always uses `~/.openclaw/workspace/` as its workspace.
+    // This is the most common case and avoids issues with sessionFile path resolution.
+    const homeDir = process.env.HOME ?? process.env.USERPROFILE ?? '';
+    if (homeDir) {
+        const wellKnownWorkspace = path.join(homeDir, '.openclaw', 'workspace');
+        if (fsSync.existsSync(wellKnownWorkspace)) {
+            // Verify it looks like a real workspace (has at least one workspace marker)
+            const hasMarker = fsSync.existsSync(path.join(wellKnownWorkspace, 'MEMORY.md'))
+                || fsSync.existsSync(path.join(wellKnownWorkspace, 'memory'))
+                || fsSync.existsSync(path.join(wellKnownWorkspace, 'AGENTS.md'))
+                || fsSync.existsSync(path.join(wellKnownWorkspace, 'SOUL.md'))
+                || fsSync.existsSync(path.join(wellKnownWorkspace, '.git'));
+            if (hasMarker) {
+                return wellKnownWorkspace;
+            }
+        }
+    }
+    // ---- Priority 2: Walk up from sessionFile to find workspace ----
+    // Session files are under .openclaw/sessions/ in the workspace or state dir.
+    let dir = path.dirname(sessionFile);
+    for (let i = 0; i < 10; i += 1) {
+        // Check common workspace markers
+        if (fsSync.existsSync(path.join(dir, 'MEMORY.md'))
+            || fsSync.existsSync(path.join(dir, 'memory'))
+            || fsSync.existsSync(path.join(dir, 'AGENTS.md'))
+            || fsSync.existsSync(path.join(dir, '.codebuddy'))
+            || fsSync.existsSync(path.join(dir, 'package.json'))
+            || fsSync.existsSync(path.join(dir, '.git'))) {
+            return dir;
+        }
+        const parent = path.dirname(dir);
+        if (parent === dir) {
+            break;
+        }
+        dir = parent;
+    }
+    // ---- Priority 3: Fallback to CWD ----
+    return process.cwd();
+}
+/**
+ * Workspace files that are part of the AI's identity and behavioral system.
+ * These files live at the root of the workspace directory.
+ *
+ * @deprecated These files are no longer synced to the cloud.
+ * Kept for reference only.
+ */
+const WORKSPACE_SYNC_FILES = [
+    'AGENTS.md',
+    'SOUL.md',
+    'TOOLS.md',
+    'IDENTITY.md',
+    'BOOTSTRAP.md',
+    'HEARTBEAT.md',
+    'USER.md',
+];
+/**
+ * Discover all local memory files in a workspace.
+ *
+ * Scans for:
+ *   1. MEMORY.md / memory.md (long-term memory)
+ *   2. memory/*.md (daily logs / short-term memory)
+ *   3. .codebuddy/MEMORY.md (IDE-level long-term memory)
+ *   4. .codebuddy/memory/*.md (IDE-level daily logs)
+ *
+ * NOTE: Workspace identity files (AGENTS.md, SOUL.md, etc.) are no longer
+ * discovered or synced to the cloud.
+ */
+async function discoverMemoryFiles(workspaceDir, config) {
+    const files = [];
+    if (config.syncLongTermMemory) {
+        // Root-level MEMORY.md (or memory.md — pick the first that exists).
+        //
+        // IMPORTANT: On case-insensitive filesystems (macOS APFS default),
+        // `existsSync('memory.md')` returns true even when the actual file
+        // on disk is named `MEMORY.md`. To avoid uploading the same file
+        // twice under two different COS keys (MEMORY.md vs memory.md),
+        // we stop after finding the first match.
+        for (const name of ['MEMORY.md', 'memory.md']) {
+            const absPath = path.join(workspaceDir, name);
+            if (fsSync.existsSync(absPath)) {
+                files.push({ absPath, relPath: name, category: 'long-term' });
+                break; // Only take the first match to avoid duplicates on case-insensitive FS
+            }
+        }
+        // .codebuddy/MEMORY.md
+        const cbMemory = path.join(workspaceDir, '.codebuddy', 'MEMORY.md');
+        if (fsSync.existsSync(cbMemory)) {
+            files.push({ absPath: cbMemory, relPath: '.codebuddy/MEMORY.md', category: 'long-term' });
+        }
+    }
+    if (config.syncDailyLogs) {
+        // memory/*.md directory
+        const memoryDir = path.join(workspaceDir, 'memory');
+        await scanMarkdownDir(memoryDir, workspaceDir, 'daily-log', files);
+        // .codebuddy/memory/*.md directory
+        const cbMemoryDir = path.join(workspaceDir, '.codebuddy', 'memory');
+        await scanMarkdownDir(cbMemoryDir, workspaceDir, 'daily-log', files);
+    }
+    return files;
+}
+async function scanMarkdownDir(dir, workspaceDir, category, files) {
+    if (!fsSync.existsSync(dir)) {
+        return;
+    }
+    try {
+        const entries = await fs.readdir(dir, { withFileTypes: true });
+        for (const entry of entries) {
+            if (entry.isFile() && entry.name.endsWith('.md')) {
+                const absPath = path.join(dir, entry.name);
+                const relPath = path.relative(workspaceDir, absPath);
+                files.push({ absPath, relPath, category });
+            }
+        }
+    }
+    catch {
+        // Directory may not be accessible
+    }
+}
+// ============================================================================
+// Core sync logic
+// ============================================================================
+/**
+ * Human-readable label for sync file categories.
+ */
+function categoryLabel(category) {
+    switch (category) {
+        case 'long-term': {
+            return 'Long-term Memory (MEMORY.md)';
+        }
+        case 'daily-log': {
+            return 'Daily Log (Short-term Memory)';
+        }
+    }
+}
+/**
+ * Generate a deterministic docId from a file path.
+ *
+ * Uses the relative path to create a stable ID that allows overwriting
+ * the same file on re-upload (no duplicates).
+ *
+ * IMPORTANT: The returned docId does **not** include a `.md` extension
+ * because `cos-operations.ts upload()` always appends `.md` to the
+ * final COS key. Including `.md` here would produce double extensions
+ * like `memory.md.md`.
+ *
+ * For daily-log (short-term memory) files under
+ * `~/.openclaw/workspace/memory/`, a `memory/` directory prefix
+ * is prepended so they end up in `{cosPrefix}memory/` on COS:
+ *   relPath "memory/2026-03-15.md" → docId "memory/2026-03-15"
+ *   → COS key: "memory/memory/2026-03-15.md"
+ *
+ * For long-term memory (`~/.openclaw/workspace/MEMORY.md`)
+ * and other files, a flat docId is used (no slashes):
+ *   relPath "MEMORY.md"  → docId "MEMORY"
+ *   relPath "AGENTS.md"  → docId "AGENTS"
+ *   → COS key: "{cosPrefix}MEMORY.md", "{agentPrefix}AGENTS.md"
+ */
+function filePathToDocId(relPath, category) {
+    // Strip the .md extension first — cos-operations.ts upload() always
+    // appends ".md" to the final COS key, so keeping it here would produce
+    // double extensions like "memory.md.md".
+    const withoutExt = relPath.replace(/\.md$/i, '');
+    const sanitized = withoutExt
+        .replace(/[^a-zA-Z0-9._/-]/g, '-')
+        .replace(/-+/g, '-')
+        .replace(/^-|-$/g, '');
+    // For long-term memory, normalize to uppercase "MEMORY" to prevent
+    // case-insensitive filesystem ghosts (macOS APFS) from creating
+    // duplicate COS keys (MEMORY.md vs memory.md).
+    if (category === 'long-term') {
+        const flat = sanitized.replace(/\//g, '-');
+        // "memory" → "MEMORY", ".codebuddy-MEMORY" stays unchanged
+        if (flat.toLowerCase() === 'memory') {
+            return 'MEMORY';
+        }
+        // .codebuddy/MEMORY → ".codebuddy-MEMORY" (already correct)
+        return flat;
+    }
+    // For daily-log files, preserve the "memory/" directory structure
+    // so they are stored under {cosPrefix}memory/ on COS.
+    // e.g. relPath "memory/2026-03-15.md"
+    //   → withoutExt "memory/2026-03-15"
+    //   → basename "2026-03-15"
+    //   → docId "memory/2026-03-15"
+    //   → COS key: "memory/memory/2026-03-15.md"
+    if (category === 'daily-log') {
+        const basename = sanitized.split('/').pop() ?? sanitized;
+        return `memory/${basename}`;
+    }
+    // For other categories, use a flat sanitized ID (no slashes)
+    return sanitized.replace(/\//g, '-');
+}
+/**
+ * Sync a single local memory file to the cloud by its absolute path.
+ *
+ * This is a targeted entry point called by the `after_tool_call` hook
+ * when the AI writes to a memory file (MEMORY.md or memory/*.md).
+ * It re-uses the same deterministic docId and hash-change detection
+ * as the full sync, so re-uploading an unchanged file is a no-op.
+ *
+ * @param ops                CosOperations instance.
+ * @param absPath            Absolute path to the memory file.
+ * @param logger             Logger instance.
+ * @param allowedExtensions  Optional set of allowed file extensions for asset sync.
+ * @returns true if the file was uploaded, false if skipped/failed.
+ */
+export async function syncSingleMemoryFileToCloud(ops, absPath, logger, allowedExtensions) {
+    try {
+        const content = await fs.readFile(absPath, 'utf-8');
+        if (content.trim().length === 0) {
+            logger.info(`local-memory-sync: single-file skip (empty) — ${absPath}`);
+            return false;
+        }
+        if (!hasChanged(absPath, content)) {
+            logger.info(`local-memory-sync: single-file skip (unchanged) — ${absPath}`);
+            return false;
+        }
+        // Determine category from the filename / path
+        const basename = path.basename(absPath);
+        const basenameUpper = basename.toUpperCase();
+        const isLongTerm = basenameUpper === 'MEMORY.MD';
+        // Skip workspace identity files — they are no longer synced to cloud
+        const isWorkspaceFile = WORKSPACE_SYNC_FILES
+            .map((f) => f.toUpperCase())
+            .includes(basenameUpper);
+        if (isWorkspaceFile) {
+            logger.info(`local-memory-sync: single-file skip (workspace file no longer synced) — ${absPath}`);
+            return false;
+        }
+        const category = isLongTerm
+            ? 'long-term'
+            : 'daily-log';
+        // Build a stable relative-ish path for docId generation
+        // Try to extract a meaningful relative path from common patterns:
+        //   /path/to/workspace/MEMORY.md → MEMORY.md
+        //   /path/to/workspace/memory/2026-03-15.md → memory/2026-03-15.md
+        //   /path/to/.codebuddy/MEMORY.md → .codebuddy/MEMORY.md
+        //   /path/to/.codebuddy/memory/2026-03-15.md → .codebuddy/memory/2026-03-15.md
+        const relPath = extractMemoryRelPath(absPath);
+        const docId = filePathToDocId(relPath, category);
+        const enrichedContent = [
+            `# Local Memory: ${relPath}`,
+            `**Type**: ${categoryLabel(category)}`,
+            `**Source**: ${relPath}`,
+            `**Synced**: ${new Date().toISOString()}`,
+            '',
+            '---',
+            '',
+            content,
+        ].join('\n');
+        await ops.upload(enrichedContent, {
+            category: 'memory',
+            docId,
+            metadata: {
+                source: 'local-memory-sync',
+                type: category,
+                filePath: relPath,
+                timestamp: Date.now(),
+                contentHash: computeHash(content),
+                trigger: 'after_tool_call',
+            },
+        });
+        logger.info(`local-memory-sync: single-file uploaded — ${relPath} (${category})`);
+        // Extract and upload assets (images + documents) referenced in this memory file
+        try {
+            const workspaceDir = path.dirname(absPath);
+            const assetResult = await extractAndUploadAssets(ops, [{ absPath, content }], workspaceDir, logger, allowedExtensions);
+            if (assetResult.uploaded > 0 || assetResult.failed > 0) {
+                logger.info(`local-memory-sync: single-file asset sync — uploaded=${assetResult.uploaded}, `
+                    + `skipped=${assetResult.skipped}, failed=${assetResult.failed}`);
+            }
+        }
+        catch (assetErr) {
+            logger.warn(`local-memory-sync: single-file asset extraction failed — ${absPath}: ${String(assetErr)}`);
+        }
+        // Persist hash cache to disk so restarts don't re-upload unchanged files.
+        await flushHashCache();
+        return true;
+    }
+    catch (err) {
+        logger.warn(`local-memory-sync: single-file upload failed — ${absPath}: ${String(err)}`);
+        return false;
+    }
+}
+/**
+ * Extract a meaningful relative path from an absolute memory file path.
+ *
+ * Recognized patterns:
+ *   /path/to/workspace/MEMORY.md → MEMORY.md
+ *   /path/to/workspace/memory.md → memory.md
+ *   /path/to/workspace/memory/2026-03-15.md → memory/2026-03-15.md
+ *   /path/to/.codebuddy/MEMORY.md → .codebuddy/MEMORY.md
+ *   /path/to/.codebuddy/memory/2026-03-15.md → .codebuddy/memory/2026-03-15.md
+ *   /path/to/workspace/AGENTS.md → AGENTS.md
+ *   /path/to/workspace/SOUL.md → SOUL.md
+ *   /path/to/workspace/TOOLS.md → TOOLS.md
+ */
+function extractMemoryRelPath(absPath) {
+    const normalized = absPath.replace(/\\/g, '/');
+    // Try .codebuddy/memory/* pattern
+    const cbMemoryMatch = normalized.match(/\.codebuddy\/memory\/[^/]+\.md$/);
+    if (cbMemoryMatch) {
+        return cbMemoryMatch[0];
+    }
+    // Try .codebuddy/MEMORY.md pattern
+    const cbLongTermMatch = normalized.match(/\.codebuddy\/MEMORY\.md$/i);
+    if (cbLongTermMatch) {
+        return cbLongTermMatch[0];
+    }
+    // Try memory/* pattern (daily logs)
+    const memoryDirMatch = normalized.match(/memory\/[^/]+\.md$/);
+    if (memoryDirMatch) {
+        return memoryDirMatch[0];
+    }
+    // Try MEMORY.md / memory.md at the end
+    const rootMemoryMatch = normalized.match(/(?:MEMORY|memory)\.md$/);
+    if (rootMemoryMatch) {
+        return rootMemoryMatch[0];
+    }
+    // Try workspace identity files (AGENTS.md, SOUL.md, TOOLS.md, etc.)
+    const basename = path.basename(absPath);
+    if (WORKSPACE_SYNC_FILES.includes(basename)) {
+        return basename;
+    }
+    // Fallback: use the basename
+    return basename;
+}
+/**
+ * Check whether an absolute file path refers to a syncable workspace file.
+ *
+ * Recognized patterns:
+ *   - MEMORY.md, memory.md (long-term memory)
+ *   - memory/*.md (daily log files)
+ *   - .codebuddy/MEMORY.md, .codebuddy/memory/*.md (IDE-level)
+ *
+ * NOTE: Workspace identity files (AGENTS.md, SOUL.md, etc.) are no longer
+ * considered syncable and will return false.
+ */
+export function isMemoryFilePath(absPath) {
+    const normalized = absPath.replace(/\\/g, '/').toLowerCase();
+    // MEMORY.md or memory.md anywhere in the path
+    if (normalized.endsWith('/memory.md')) {
+        return true;
+    }
+    // memory/<something>.md — daily log files
+    if (/\/memory\/[^/]+\.md$/.test(normalized)) {
+        return true;
+    }
+    // .codebuddy/MEMORY.md
+    if (normalized.endsWith('/.codebuddy/memory.md')) {
+        return true;
+    }
+    // .codebuddy/memory/*.md
+    if (/\/\.codebuddy\/memory\/[^/]+\.md$/.test(normalized)) {
+        return true;
+    }
+    // NOTE: Workspace identity files (AGENTS.md, SOUL.md, etc.) are no longer
+    // considered syncable memory files.
+    return false;
+}
+/**
+ * Sync all local memory files and config to the cloud.
+ *
+ * This is the main entry point called by the engine during bootstrap
+ * or afterTurn when `localMemorySync.enabled` is true.
+ *
+ * @param ops                CosOperations instance.
+ * @param sessionFile        Session file path (used to resolve workspace dir).
+ * @param config             Sync configuration flags.
+ * @param logger             Logger instance.
+ * @param allowedExtensions  Optional set of allowed file extensions for asset sync.
+ */
+export async function syncLocalMemoryToCloud(ops, sessionFile, config, logger, allowedExtensions) {
+    const result = {
+        uploaded: 0,
+        skipped: 0,
+        failed: 0,
+        details: [],
+    };
+    if (!config.enabled) {
+        return result;
+    }
+    const workspaceDir = resolveWorkspaceDir(sessionFile);
+    logger.info(`local-memory-sync: workspace=${workspaceDir}`);
+    // ---- Step 1: Discover and upload all syncable files ----
+    // discoverMemoryFiles respects individual config flags internally
+    // (syncLongTermMemory, syncDailyLogs).
+    const memoryFiles = await discoverMemoryFiles(workspaceDir, config);
+    logger.info(`local-memory-sync: found ${memoryFiles.length} syncable file(s) in ${workspaceDir}`
+        + ` (longTerm=${config.syncLongTermMemory}, dailyLogs=${config.syncDailyLogs})`);
+    for (const file of memoryFiles) {
+        try {
+            const content = await fs.readFile(file.absPath, 'utf-8');
+            if (content.trim().length === 0) {
+                result.details.push({ path: file.relPath, status: 'skipped', reason: 'empty' });
+                result.skipped += 1;
+                logger.info(`local-memory-sync: skip (empty) — ${file.relPath}`);
+                continue;
+            }
+            if (!hasChanged(file.absPath, content)) {
+                result.details.push({ path: file.relPath, status: 'skipped', reason: 'unchanged' });
+                result.skipped += 1;
+                continue;
+            }
+            const docId = filePathToDocId(file.relPath, file.category);
+            const enrichedContent = [
+                `# Local Memory: ${file.relPath}`,
+                `**Type**: ${categoryLabel(file.category)}`,
+                `**Source**: ${file.relPath}`,
+                `**Synced**: ${new Date().toISOString()}`,
+                '',
+                '---',
+                '',
+                content,
+            ].join('\n');
+            await ops.upload(enrichedContent, {
+                category: 'memory',
+                docId,
+                metadata: {
+                    source: 'local-memory-sync',
+                    type: file.category,
+                    filePath: file.relPath,
+                    timestamp: Date.now(),
+                    contentHash: computeHash(content),
+                },
+            });
+            const targetPath = `{cosPrefix}${docId}.md`;
+            result.details.push({ path: file.relPath, status: 'uploaded' });
+            result.uploaded += 1;
+            logger.info(`local-memory-sync: uploaded — ${file.relPath} [${file.category}] → ${targetPath}`);
+        }
+        catch (err) {
+            result.details.push({
+                path: file.relPath,
+                status: 'failed',
+                reason: String(err),
+            });
+            result.failed += 1;
+            logger.warn(`local-memory-sync: upload failed — ${file.relPath}: ${String(err)}`);
+        }
+    }
+    logger.info(`local-memory-sync: done — uploaded=${result.uploaded}, skipped=${result.skipped}, failed=${result.failed}`);
+    // ---- Step 3: Extract and upload assets (images + documents) referenced in memory files ----
+    // Scan all successfully synced memory file contents for asset links,
+    // check if those assets exist locally, and upload them to COS.
+    try {
+        const allContents = [];
+        for (const file of memoryFiles) {
+            try {
+                const content = await fs.readFile(file.absPath, 'utf-8');
+                allContents.push({ absPath: file.absPath, content });
+            }
+            catch {
+                // Already handled above — skip silently
+            }
+        }
+        const assetResult = await extractAndUploadAssets(ops, allContents, workspaceDir, logger, allowedExtensions);
+        if (assetResult.uploaded > 0 || assetResult.failed > 0) {
+            logger.info(`local-memory-sync: asset sync — uploaded=${assetResult.uploaded}, `
+                + `skipped=${assetResult.skipped}, failed=${assetResult.failed}`);
+        }
+    }
+    catch (err) {
+        logger.warn(`local-memory-sync: asset extraction/upload failed: ${String(err)}`);
+    }
+    // Persist hash cache to disk so restarts don't re-upload unchanged files.
+    await flushHashCache();
+    return result;
+}
+// ============================================================================
+// Image extraction and upload
+// ============================================================================
+/**
+ * Regex to match image/file links in Markdown content.
+ *
+ * Supported formats:
+ *   - `![alt](path)` — standard Markdown image
+ *   - `![alt](path "title")` — with optional title
+ *   - `[alt](path)` — standard Markdown link (for PDFs, docs, etc.)
+ *
+ * Only matches local file paths (not http/https URLs). Paths can be:
+ *   - Absolute: `/path/to/image.png`
+ *   - Relative: `./images/photo.jpg`, `../assets/icon.svg`
+ *   - Home-relative: `~/Pictures/screenshot.png`
+ *
+ * Does NOT match:
+ *   - HTTP URLs: `![](https://example.com/image.png)`
+ *   - Data URIs: `![](data:image/png;base64,...)`
+ */
+const MARKDOWN_IMAGE_RE = /!\[([^\]]*)\]\(([^)]+)\)/g;
+/**
+ * Regex to match standard Markdown links `[alt](path)` — used to extract
+ * linked local files (e.g. PDFs, documents) that are not wrapped in `![]()`.
+ */
+const MARKDOWN_LINK_RE = /(?<!!)\[([^\]]*)\]\(([^)]+)\)/g;
+/**
+ * Regex to match HTML `<img>` and `<video>` tags with a `src` attribute.
+ *
+ * Captures the `src` value from tags like:
+ *   - `<img src="path/to/image.png">`
+ *   - `<video src="path/to/video.mp4">`
+ *   - Handles both single and double quotes, or unquoted src values.
+ */
+const HTML_MEDIA_SRC_RE = /<(?:img|video)\s[^>]*?\bsrc\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/gi;
+/**
+ * Regex to match bare image URLs in Markdown content (not inside `![]()`).
+ *
+ * Matches HTTP(S) URLs that end with a known image extension (before any query
+ * string or fragment). These are "bare" URLs — plain text links not wrapped in
+ * standard Markdown image syntax.
+ *
+ * The extraction logic will attempt to find a locally downloaded copy of the
+ * image in `~/Downloads/` by matching the filename from the URL path.
+ */
+const BARE_IMAGE_URL_RE = /(?<!\]\()https?:\/\/[^\s)]+\.(?:png|jpe?g|gif|bmp|webp|svg|ico|tiff?|avif|heic|heif)(?=[?\s#]|$)/gi;
+/**
+ * Regex to match bare local file paths in plain text.
+ *
+ * Matches absolute or home-relative paths ending with a known media/document extension:
+ *   - `/root/.openclaw/workspace/pictures/10/1.jpg`
+ *   - `~/Pictures/screenshot.png`
+ *   - `~Downloads/report.pdf`  (common typo: `~` without `/`)
+ *   - `../../.openclaw/workspace/pictures/10/1.jpg`
+ *
+ * Does NOT match paths already captured by Markdown or HTML syntax (those are
+ * handled by dedicated regexes above).
+ *
+ * The `~` prefix is treated as home-directory shorthand, with or without a
+ * separating `/` (e.g. both `~/Downloads/x.pdf` and `~Downloads/x.pdf`).
+ */
+const BARE_LOCAL_PATH_RE = /(?:^|[\s,;])(~[^\s,;:*?"<>|/\\]+\/[^\s,;:*?"<>|]+\.(?:png|jpe?g|gif|bmp|webp|svg|ico|tiff?|avif|heic|heif|pdf|doc|docx|xls|xlsx|ppt|pptx|txt|csv|md|rtf|mp4|mov|avi|mkv)|[~.]?(?:\.\/|\.\.\/|\/)[^\s,;:*?"<>|]+\.(?:png|jpe?g|gif|bmp|webp|svg|ico|tiff?|avif|heic|heif|pdf|doc|docx|xls|xlsx|ppt|pptx|txt|csv|md|rtf|mp4|mov|avi|mkv))(?=[\s,;]|$)/gim;
+/**
+ * Common image file extensions (lowercase).
+ */
+const IMAGE_EXTENSIONS = new Set([
+    '.png', '.jpg', '.jpeg', '.gif', '.bmp', '.webp',
+    '.svg', '.ico', '.tiff', '.tif', '.avif', '.heic', '.heif',
+]);
+/**
+ * Default file extensions for asset sync (images + documents).
+ *
+ * Users can override this via `syncFileExtensions` config.
+ */
+const DEFAULT_SYNC_FILE_EXTENSIONS = [
+    // Images
+    '.png', '.jpg', '.jpeg', '.gif', '.bmp', '.webp',
+    '.svg', '.ico', '.tiff', '.tif', '.avif', '.heic', '.heif',
+    // Documents
+    '.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx',
+    '.txt', '.csv', '.md', '.rtf',
+];
+/**
+ * MIME type mapping for common image extensions.
+ */
+const MIME_TYPES = {
+    '.png': 'image/png',
+    '.jpg': 'image/jpeg',
+    '.jpeg': 'image/jpeg',
+    '.gif': 'image/gif',
+    '.bmp': 'image/bmp',
+    '.webp': 'image/webp',
+    '.svg': 'image/svg+xml',
+    '.ico': 'image/x-icon',
+    '.tiff': 'image/tiff',
+    '.tif': 'image/tiff',
+    '.avif': 'image/avif',
+    '.heic': 'image/heic',
+    '.heif': 'image/heif',
+    // Documents
+    '.pdf': 'application/pdf',
+    '.doc': 'application/msword',
+    '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+    '.xls': 'application/vnd.ms-excel',
+    '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+    '.ppt': 'application/vnd.ms-powerpoint',
+    '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+    '.txt': 'text/plain',
+    '.csv': 'text/csv',
+    '.md': 'text/markdown',
+    '.rtf': 'application/rtf',
+};
+/** Hash cache accessor for assets — backed by the persistent cache's `asset` namespace. */
+const imageHashCache = {
+    get(key) {
+        return loadHashCache().asset[key];
+    },
+    set(key, value) {
+        loadHashCache().asset[key] = value;
+        cacheDirty = true;
+    },
+};
+/**
+ * Extract image/media links from Markdown content.
+ *
+ * Returns an array of raw file paths found in the content. Supports five formats:
+ *   1. Standard Markdown image syntax: `![alt](path)` — local file paths only
+ *   2. Standard Markdown link syntax: `[alt](path)` — local files (PDFs, docs, etc.)
+ *   3. HTML media tags: `<img src="path">`, `<video src="path">`
+ *   4. Bare image URLs: `https://example.com/.../image.jpg` — resolved to a
+ *      locally downloaded copy in `~/Downloads/` by matching the filename.
+ *   5. Bare local file paths in plain text: `/root/pics/1.jpg`, `../../pics/1.jpg`
+ *
+ * HTTP(S) URLs inside `![alt](url)` or `[alt](url)` are skipped (remote links
+ * in Markdown syntax are expected to stay remote). Bare URLs in plain text are
+ * checked against `~/Downloads/{filename}` to discover locally downloaded copies.
+ *
+ * @param content  Markdown text to scan for image links.
+ * @returns Array of local file path strings referenced in the content.
+ */
+export function extractImageLinks(content) {
+    const links = [];
+    const seenPaths = new Set();
+    let match;
+    // --- Pass 1: Standard Markdown image syntax ![alt](path) ---
+    MARKDOWN_IMAGE_RE.lastIndex = 0;
+    while ((match = MARKDOWN_IMAGE_RE.exec(content)) !== null) {
+        let rawPath = match[2].trim();
+        // Strip optional title: ![alt](path "title") → path
+        const titleMatch = rawPath.match(/^(.+?)\s+"[^"]*"$/);
+        if (titleMatch) {
+            rawPath = titleMatch[1].trim();
+        }
+        // Skip HTTP(S) URLs
+        if (/^https?:\/\//i.test(rawPath)) {
+            continue;
+        }
+        // Skip data URIs
+        if (rawPath.startsWith('data:')) {
+            continue;
+        }
+        // Skip empty paths
+        if (rawPath.length === 0) {
+            continue;
+        }
+        if (!seenPaths.has(rawPath)) {
+            seenPaths.add(rawPath);
+            links.push(rawPath);
+        }
+    }
+    // --- Pass 2: Bare image URLs → resolve to ~/Downloads/{filename} ---
+    const homeDir = process.env.HOME ?? process.env.USERPROFILE ?? '';
+    if (homeDir) {
+        BARE_IMAGE_URL_RE.lastIndex = 0;
+        while ((match = BARE_IMAGE_URL_RE.exec(content)) !== null) {
+            const rawUrl = match[0];
+            // Extract filename from the URL path (strip query/fragment)
+            let urlPath;
+            try {
+                urlPath = new URL(rawUrl).pathname;
+            }
+            catch {
+                // Malformed URL — skip
+                continue;
+            }
+            const filename = path.basename(urlPath);
+            if (!filename || filename === '/') {
+                continue;
+            }
+            // Check if the file exists in ~/Downloads/
+            const localPath = path.join(homeDir, 'Downloads', filename);
+            if (fsSync.existsSync(localPath) && isImageFile(localPath)) {
+                if (!seenPaths.has(localPath)) {
+                    seenPaths.add(localPath);
+                    links.push(localPath);
+                }
+            }
+        }
+    }
+    // --- Pass 3: Standard Markdown link syntax [alt](path) — local files only ---
+    MARKDOWN_LINK_RE.lastIndex = 0;
+    while ((match = MARKDOWN_LINK_RE.exec(content)) !== null) {
+        let rawPath = match[2].trim();
+        // Strip optional title: [alt](path "title") → path
+        const titleMatch = rawPath.match(/^(.+?)\s+"[^"]*"$/);
+        if (titleMatch) {
+            rawPath = titleMatch[1].trim();
+        }
+        // Skip HTTP(S) URLs and data URIs
+        if (/^https?:\/\//i.test(rawPath) || rawPath.startsWith('data:')) {
+            continue;
+        }
+        // Skip empty paths and anchors-only
+        if (rawPath.length === 0 || rawPath.startsWith('#')) {
+            continue;
+        }
+        if (!seenPaths.has(rawPath)) {
+            seenPaths.add(rawPath);
+            links.push(rawPath);
+        }
+    }
+    // --- Pass 4: HTML <img> and <video> tags with src attribute ---
+    HTML_MEDIA_SRC_RE.lastIndex = 0;
+    while ((match = HTML_MEDIA_SRC_RE.exec(content)) !== null) {
+        // Capture group 1 = double-quoted, 2 = single-quoted, 3 = unquoted
+        const rawPath = (match[1] ?? match[2] ?? match[3] ?? '').trim();
+        // Skip HTTP(S) URLs, data URIs, and empty values
+        if (/^https?:\/\//i.test(rawPath) || rawPath.startsWith('data:') || rawPath.length === 0) {
+            continue;
+        }
+        if (!seenPaths.has(rawPath)) {
+            seenPaths.add(rawPath);
+            links.push(rawPath);
+        }
+    }
+    // --- Pass 5: Bare local file paths in plain text ---
+    BARE_LOCAL_PATH_RE.lastIndex = 0;
+    while ((match = BARE_LOCAL_PATH_RE.exec(content)) !== null) {
+        const rawPath = match[1].trim();
+        // Skip paths already captured by earlier passes
+        if (seenPaths.has(rawPath) || rawPath.length === 0) {
+            continue;
+        }
+        seenPaths.add(rawPath);
+        links.push(rawPath);
+    }
+    return links;
+}
+/**
+ * Resolve an image path to an absolute file path.
+ *
+ * Handles:
+ *   - Absolute paths: returned as-is
+ *   - Home-relative (`~/...`): expanded using HOME env var
+ *   - Relative paths: resolved relative to the markdown file's directory
+ *
+ * @param imagePath    Raw image path from Markdown.
+ * @param mdFilePath   Absolute path of the Markdown file containing the link.
+ * @param workspaceDir Workspace root directory (fallback for resolution).
+ * @returns Resolved absolute path.
+ */
+function resolveImagePath(imagePath, mdFilePath, workspaceDir) {
+    // Absolute path
+    if (path.isAbsolute(imagePath)) {
+        return imagePath;
+    }
+    // Home-relative path: ~/path or ~Dir/path (common typo missing the /)
+    if (imagePath.startsWith('~')) {
+        const homeDir = process.env.HOME ?? process.env.USERPROFILE ?? '';
+        if (homeDir) {
+            if (imagePath.startsWith('~/') || imagePath.startsWith('~\\')) {
+                // Standard: ~/Downloads/file.pdf → $HOME/Downloads/file.pdf
+                return path.join(homeDir, imagePath.slice(2));
+            }
+            // Typo variant: ~Downloads/file.pdf → $HOME/Downloads/file.pdf
+            return path.join(homeDir, imagePath.slice(1));
+        }
+    }
+    // Relative path — resolve from the markdown file's directory first,
+    // then fallback to workspace dir
+    const mdDir = path.dirname(mdFilePath);
+    const resolvedFromMd = path.resolve(mdDir, imagePath);
+    if (fsSync.existsSync(resolvedFromMd)) {
+        return resolvedFromMd;
+    }
+    // Fallback: resolve from workspace root
+    return path.resolve(workspaceDir, imagePath);
+}
+/**
+ * Check if a file path points to a supported image file.
+ */
+function isImageFile(filePath) {
+    const ext = path.extname(filePath).toLowerCase();
+    return IMAGE_EXTENSIONS.has(ext);
+}
+/**
+ * Check if a file path points to a supported asset file (image or document).
+ *
+ * @param filePath       Path to check.
+ * @param allowedExts    Set of allowed extensions. If not provided, uses IMAGE_EXTENSIONS (backward compat).
+ */
+function isAssetFile(filePath, allowedExts) {
+    const ext = path.extname(filePath).toLowerCase();
+    return allowedExts ? allowedExts.has(ext) : IMAGE_EXTENSIONS.has(ext);
+}
+/**
+ * Compute a hash for binary content (used for asset dedup).
+ */
+function computeAssetHash(buffer) {
+    return crypto.createHash('sha256').update(buffer).digest('hex').slice(0, 16);
+}
+/**
+ * Check whether an asset file has changed since the last upload.
+ */
+function assetHasChanged(absPath, buffer) {
+    const newHash = computeAssetHash(buffer);
+    const oldHash = imageHashCache.get(absPath);
+    if (oldHash === newHash) {
+        return false;
+    }
+    imageHashCache.set(absPath, newHash);
+    return true;
+}
+/**
+ * Build the COS key for an image file in the **asset/** directory,
+ * using the absolute path directly to preserve full directory structure.
+ *
+ * Layout: `openclaw-{agentId}/asset/{absolutePath}`
+ *   e.g. `/Users/shawn/Downloads/test.jpg` → `asset/Users/shawn/Downloads/test.jpg`
+ *   e.g. `/tmp/pic.png` → `asset/tmp/pic.png`
+ *   e.g. `/Users/shawn/Downloads/CI控制台指南.pdf` → `asset/Users/shawn/Downloads/CI控制台指南.pdf`
+ * For legacy setups (no agentId): `asset/{absolutePath}`
+ *
+ * COS (S3-compatible) keys support UTF-8, so Unicode characters (Chinese,
+ * Japanese, etc.) are preserved. Only characters that are unsafe for COS
+ * object keys or shell handling are replaced: control chars, `*`, `?`, `"`,
+ * `<`, `>`, `|`, `#`, `%`, `{`, `}`, `^`, `` ` ``, `[`, `]`.
+ *
+ * @param ops             CosOperations instance (for agent prefix).
+ * @param resolvedPath    Absolute path of the image file.
+ * @returns Full COS key string.
+ */
+function buildAssetCosKey(ops, resolvedPath) {
+    const agentPrefix = ops.getAgentPrefix();
+    // Use the absolute path directly, stripping the leading separator.
+    const normalResolved = path.resolve(resolvedPath);
+    const stripped = normalResolved.startsWith(path.sep)
+        ? normalResolved.slice(path.sep.length)
+        : normalResolved;
+    // Sanitise each path segment (keep `/` separators intact).
+    // Preserve Unicode (CJK, etc.) — only strip COS-unsafe / shell-unsafe chars
+    // and control characters (U+0000–U+001F, U+007F).
+    const safePath = stripped
+        .split(path.sep)
+        .map(seg => seg
+        // eslint-disable-next-line no-control-regex
+        .replace(/[\x00-\x1f\x7f*?"<>|#%{}^`[\]]/g, '-')
+        .replace(/-+/g, '-')
+        .replace(/^-|-$/g, ''))
+        .join('/');
+    return `${agentPrefix}asset/${safePath}`;
+}
+/**
+ * Extract asset links (images + documents) from memory file contents and upload them to COS.
+ *
+ * For each memory file content:
+ *   1. Extract `![alt](path)` image links and `[alt](path)` document links
+ *   2. Resolve the path to an absolute local file path
+ *   3. Check if the file exists and has a supported extension (configurable)
+ *   4. Check if the file has changed (hash-based dedup)
+ *   5. Upload to `{agentPrefix}asset/{relPath}` preserving directory structure
+ *   6. Image files use `x-cos-meta-category: image`, document files use `document`
+ *
+ * @param ops              CosOperations instance for uploading.
+ * @param fileContents     Array of { absPath, content } for each memory file.
+ * @param workspaceDir     Workspace root for resolving relative paths.
+ * @param logger           Logger for progress/error reporting.
+ * @param allowedExtensions  Optional set of allowed file extensions. Defaults to DEFAULT_SYNC_FILE_EXTENSIONS.
+ * @returns Summary of asset upload results.
+ */
+export async function extractAndUploadAssets(ops, fileContents, workspaceDir, logger, allowedExtensions) {
+    const result = {
+        uploaded: 0,
+        skipped: 0,
+        failed: 0,
+        details: [],
+    };
+    // Build the effective allowed-extension set
+    const allowedExts = allowedExtensions ?? new Set(DEFAULT_SYNC_FILE_EXTENSIONS);
+    // Collect all unique asset paths across all files
+    const seenPaths = new Set();
+    for (const { absPath: mdPath, content } of fileContents) {
+        const assetLinks = extractImageLinks(content);
+        for (const rawLink of assetLinks) {
+            const resolvedPath = resolveImagePath(rawLink, mdPath, workspaceDir);
+            // Skip duplicates (same file referenced in multiple files)
+            if (seenPaths.has(resolvedPath)) {
+                continue;
+            }
+            seenPaths.add(resolvedPath);
+            try {
+                // Check if file exists
+                if (!fsSync.existsSync(resolvedPath)) {
+                    logger.info(`local-memory-sync: asset not found — ${rawLink} (resolved: ${resolvedPath})`);
+                    result.details.push({
+                        localPath: resolvedPath,
+                        status: 'skipped',
+                        reason: 'file not found',
+                    });
+                    result.skipped += 1;
+                    continue;
+                }
+                // Check if it's a supported asset file (image or document)
+                if (!isAssetFile(resolvedPath, allowedExts)) {
+                    logger.info(`local-memory-sync: not a supported asset — ${resolvedPath}`);
+                    result.details.push({
+                        localPath: resolvedPath,
+                        status: 'skipped',
+                        reason: 'not a supported asset format',
+                    });
+                    result.skipped += 1;
+                    continue;
+                }
+                // Read the file
+                const buffer = await fs.readFile(resolvedPath);
+                // Check if file has changed (hash-based dedup)
+                if (!assetHasChanged(resolvedPath, buffer)) {
+                    result.details.push({
+                        localPath: resolvedPath,
+                        status: 'skipped',
+                        reason: 'unchanged',
+                    });
+                    result.skipped += 1;
+                    continue;
+                }
+                // Build COS key — assets go to asset/ preserving directory structure
+                const cosKey = buildAssetCosKey(ops, resolvedPath);
+                // Determine MIME type
+                const ext = path.extname(resolvedPath).toLowerCase();
+                const contentType = MIME_TYPES[ext] ?? 'application/octet-stream';
+                // Determine category: image or document
+                const assetCategory = isImageFile(resolvedPath) ? 'image' : 'document';
+                // Upload to COS (sole destination: asset/ with directory structure).
+                // Paths may contain non-ASCII characters (e.g. Chinese); HTTP custom
+                // headers require ASCII values, so we URI-encode path values.
+                const assetMetadata = {
+                    source: 'local-memory-sync',
+                    type: `asset-${assetCategory}`,
+                    category: assetCategory,
+                    originalPath: encodeURIComponent(rawLink),
+                    resolvedPath: encodeURIComponent(resolvedPath),
+                    timestamp: Date.now(),
+                    contentHash: computeAssetHash(buffer),
+                };
+                await ops.uploadBinary(buffer, cosKey, contentType, assetMetadata);
+                result.details.push({
+                    localPath: resolvedPath,
+                    cosKey,
+                    status: 'uploaded',
+                });
+                result.uploaded += 1;
+                logger.info(`local-memory-sync: asset uploaded — ${rawLink} → ${cosKey} [${assetCategory}]`);
+            }
+            catch (err) {
+                result.details.push({
+                    localPath: resolvedPath,
+                    status: 'failed',
+                    reason: String(err),
+                });
+                result.failed += 1;
+                const errMsg = err instanceof Error ? err.message : JSON.stringify(err, null, 2);
+                logger.warn(`local-memory-sync: asset upload failed — ${resolvedPath}: ${errMsg}`);
+            }
+        }
+    }
+    return result;
+}
+/**
+ * @deprecated Use `extractAndUploadAssets` instead. Kept for backward compatibility.
+ */
+export const extractAndUploadImages = extractAndUploadAssets;
+//# sourceMappingURL=local-memory-sync.js.map