@psiclawops/hypermem 0.9.1 → 0.9.3

package/dist/obsidian-exporter.js

@@ -0,0 +1,274 @@
+/**
+ * obsidian-exporter.ts
+ *
+ * Exports hypermem facts and wiki pages to an Obsidian vault.
+ * Complements obsidian-watcher.ts (import direction).
+ *
+ * Output format:
+ *   - Wiki pages → one .md file per topic, with frontmatter + [[wikilinks]]
+ *   - Facts → grouped by domain into .md files, frontmatter-tagged
+ *   - Index note → _hypermem-index.md with TOC and stats
+ *
+ * Safe to run repeatedly — existing files are overwritten only if content changed.
+ */
+import { mkdirSync, writeFileSync, existsSync, readFileSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { createHash } from 'node:crypto';
+// ─── Helpers ────────────────────────────────────────────────────────────────
+function hashContent(content) {
+    return createHash('sha256').update(content).digest('hex').slice(0, 16);
+}
+function safeFilename(name) {
+    return name
+        .replace(/[\\/:*?"<>|]/g, '-')
+        .replace(/\s+/g, ' ')
+        .trim()
+        .slice(0, 200);
+}
+function formatTimestamp(unixMs) {
+    return new Date(unixMs).toISOString().split('T')[0];
+}
+function yamlString(value) {
+    if (/[:#\[\]{},&*?|<>=!%@`]/.test(value) || value.includes('\n')) {
+        return `"${value.replace(/"/g, '\\"')}"`;
+    }
+    return value;
+}
+function buildFrontmatter(fields) {
+    const lines = ['---'];
+    for (const [key, value] of Object.entries(fields)) {
+        if (value === null || value === undefined)
+            continue;
+        if (Array.isArray(value)) {
+            if (value.length === 0)
+                continue;
+            lines.push(`${key}:`);
+            for (const item of value)
+                lines.push(`  - ${yamlString(String(item))}`);
+        }
+        else {
+            lines.push(`${key}: ${yamlString(String(value))}`);
+        }
+    }
+    lines.push('---', '');
+    return lines.join('\n');
+}
+function writeIfChanged(filePath, content, skipUnchanged, result) {
+    if (skipUnchanged && existsSync(filePath)) {
+        const existing = readFileSync(filePath, 'utf8');
+        if (existing === content) {
+            result.skipped++;
+            return;
+        }
+    }
+    writeFileSync(filePath, content, 'utf8');
+    result.written++;
+    result.files.push(filePath);
+}
+// ─── Wiki page export ────────────────────────────────────────────────────────
+function exportWikiPages(libraryDb, outputDir, agentFilter, tagPrefix, skipUnchanged, result) {
+    const wikiDir = join(outputDir, 'wiki');
+    mkdirSync(wikiDir, { recursive: true });
+    let rows;
+    try {
+        const sql = agentFilter
+            ? `SELECT * FROM knowledge WHERE agent_id = ? AND superseded_at IS NULL AND domain = 'topic-synthesis' ORDER BY updated_at DESC`
+            : `SELECT * FROM knowledge WHERE superseded_at IS NULL AND domain = 'topic-synthesis' ORDER BY updated_at DESC`;
+        rows = (agentFilter
+            ? libraryDb.prepare(sql).all(agentFilter)
+            : libraryDb.prepare(sql).all());
+    }
+    catch {
+        result.errors.push('wiki: failed to query knowledge table');
+        return [];
+    }
+    const topicFiles = [];
+    for (const row of rows) {
+        try {
+            const rawTags = row.tags ? JSON.parse(row.tags) : [];
+            const tags = [tagPrefix, `${tagPrefix}/wiki`, ...rawTags];
+            // Resolve cross-links from knowledge_links table
+            let linkRows = [];
+            try {
+                linkRows = libraryDb.prepare(`
+          SELECT kl.from_id, kl.to_id, kl.link_type, k.key as to_key
+          FROM knowledge_links kl
+          JOIN knowledge k ON k.id = kl.to_id
+          WHERE kl.from_id = ? AND kl.from_type = 'knowledge' AND kl.to_type = 'knowledge'
+        `).all(row.id);
+            }
+            catch {
+                // knowledge_links may not exist yet — non-fatal
+            }
+            const wikilinks = linkRows
+                .map(l => `[[${safeFilename(l.to_key)}]]`)
+                .join(' ');
+            const frontmatter = buildFrontmatter({
+                title: row.key,
+                tags,
+                agent: row.agent_id,
+                topic: row.topic || row.key,
+                domain: row.domain,
+                'hypermem-type': 'wiki-page',
+                'last-synthesized': formatTimestamp(row.updated_at),
+                'created-at': formatTimestamp(row.created_at),
+            });
+            let body = row.content.trim();
+            if (linkRows.length > 0) {
+                body += `\n\n## Related Topics\n\n${wikilinks}`;
+            }
+            const note = `${frontmatter}${body}\n`;
+            const fileName = `${safeFilename(row.key)}.md`;
+            const filePath = join(wikiDir, fileName);
+            writeIfChanged(filePath, note, skipUnchanged, result);
+            topicFiles.push(fileName);
+        }
+        catch (err) {
+            result.errors.push(`wiki: failed to export topic '${row.key}': ${String(err)}`);
+        }
+    }
+    return topicFiles;
+}
+// ─── Facts export ────────────────────────────────────────────────────────────
+function exportFacts(mainDb, outputDir, agentFilter, tagPrefix, skipUnchanged, result) {
+    const factsDir = join(outputDir, 'facts');
+    mkdirSync(factsDir, { recursive: true });
+    let rows;
+    try {
+        const sql = agentFilter
+            ? `SELECT * FROM facts WHERE agent_id = ? ORDER BY domain, updated_at DESC`
+            : `SELECT * FROM facts ORDER BY agent_id, domain, updated_at DESC`;
+        rows = (agentFilter
+            ? mainDb.prepare(sql).all(agentFilter)
+            : mainDb.prepare(sql).all());
+    }
+    catch {
+        result.errors.push('facts: failed to query facts table');
+        return [];
+    }
+    // Group by agent + domain
+    const groups = new Map();
+    for (const row of rows) {
+        const domain = row.domain || 'general';
+        const key = `${row.agent_id}__${domain}`;
+        if (!groups.has(key))
+            groups.set(key, []);
+        groups.get(key).push(row);
+    }
+    const factFiles = [];
+    for (const [groupKey, facts] of groups) {
+        try {
+            const [agentId, domain] = groupKey.split('__');
+            const rawTags = [tagPrefix, `${tagPrefix}/facts`, `agent/${agentId}`, `domain/${domain}`];
+            const frontmatter = buildFrontmatter({
+                title: `${agentId} — ${domain} facts`,
+                tags: rawTags,
+                agent: agentId,
+                domain,
+                'hypermem-type': 'facts',
+                'fact-count': facts.length,
+                'last-updated': formatTimestamp(Math.max(...facts.map(f => f.updated_at))),
+            });
+            const lines = [`# ${agentId} — ${domain}\n`];
+            for (const fact of facts) {
+                const confidence = Math.round(fact.confidence * 100);
+                const date = formatTimestamp(fact.updated_at);
+                const factTags = fact.tags
+                    ? JSON.parse(fact.tags).map(t => `#${t}`).join(' ')
+                    : '';
+                lines.push(`- ${fact.content}  `);
+                lines.push(`  *confidence: ${confidence}% · ${date}${factTags ? ' · ' + factTags : ''}*\n`);
+            }
+            const note = `${frontmatter}${lines.join('\n')}\n`;
+            const fileName = `${safeFilename(agentId)}-${safeFilename(domain)}.md`;
+            const filePath = join(factsDir, fileName);
+            writeIfChanged(filePath, note, skipUnchanged, result);
+            factFiles.push(fileName);
+        }
+        catch (err) {
+            result.errors.push(`facts: failed to export group '${groupKey}': ${String(err)}`);
+        }
+    }
+    return factFiles;
+}
+// ─── Index note ───────────────────────────────────────────────────────────────
+function exportIndex(outputDir, wikiFiles, factFiles, tagPrefix, skipUnchanged, result) {
+    const now = new Date().toISOString();
+    const frontmatter = buildFrontmatter({
+        title: 'hypermem export index',
+        tags: [tagPrefix, `${tagPrefix}/index`],
+        'hypermem-type': 'index',
+        'exported-at': now,
+        'wiki-pages': wikiFiles.length,
+        'fact-files': factFiles.length,
+    });
+    const lines = [
+        '# hypermem export index\n',
+        `> exported at ${now}\n`,
+    ];
+    if (wikiFiles.length > 0) {
+        lines.push('## wiki pages\n');
+        for (const f of wikiFiles) {
+            const name = f.replace('.md', '');
+            lines.push(`- [[wiki/${name}]]`);
+        }
+        lines.push('');
+    }
+    if (factFiles.length > 0) {
+        lines.push('## fact collections\n');
+        for (const f of factFiles) {
+            const name = f.replace('.md', '');
+            lines.push(`- [[facts/${name}]]`);
+        }
+        lines.push('');
+    }
+    const note = `${frontmatter}${lines.join('\n')}\n`;
+    const filePath = join(outputDir, '_hypermem-index.md');
+    writeIfChanged(filePath, note, skipUnchanged, result);
+}
+// ─── Main export function ────────────────────────────────────────────────────
+/**
+ * Export hypermem memories to an Obsidian vault.
+ *
+ * @param mainDb - hypermem main DB (facts, episodes)
+ * @param libraryDb - hypermem library DB (knowledge, wiki pages)
+ * @param config - export config
+ * @returns ObsidianExportResult with counts and file list
+ *
+ * @example
+ * const result = await exportToVault(mainDb, libraryDb, {
+ *   vaultPath: '/Users/me/obsidian/my-vault',
+ *   agentId: 'main',
+ *   tagPrefix: 'hypermem',
+ * });
+ * console.log(`Wrote ${result.written} files to ${result.outputDir}`);
+ */
+export function exportToVault(mainDb, libraryDb, config) {
+    const vaultFolder = config.vaultFolder ?? 'hypermem';
+    const tagPrefix = config.tagPrefix ?? 'hypermem';
+    const includeFacts = config.includeFacts ?? true;
+    const includeWikiPages = config.includeWikiPages ?? true;
+    const includeIndex = config.includeIndex ?? true;
+    const skipUnchanged = config.skipUnchanged ?? true;
+    const agentFilter = config.agentId ?? null;
+    const outputDir = resolve(config.vaultPath, vaultFolder);
+    mkdirSync(outputDir, { recursive: true });
+    const result = {
+        written: 0,
+        skipped: 0,
+        errors: [],
+        outputDir,
+        files: [],
+    };
+    const wikiFiles = includeWikiPages
+        ? exportWikiPages(libraryDb, outputDir, agentFilter, tagPrefix, skipUnchanged, result)
+        : [];
+    const factFiles = includeFacts
+        ? exportFacts(mainDb, outputDir, agentFilter, tagPrefix, skipUnchanged, result)
+        : [];
+    if (includeIndex) {
+        exportIndex(outputDir, wikiFiles, factFiles, tagPrefix, skipUnchanged, result);
+    }
+    return result;
+}
+//# sourceMappingURL=obsidian-exporter.js.map

package/dist/obsidian-watcher.d.ts

@@ -0,0 +1,147 @@
+/**
+ * hypermem Obsidian Compatibility Layer
+ *
+ * Watch a user-configured Obsidian vault directory and import markdown
+ * notes into the hypermem fact/doc-chunk pipeline.
+ *
+ * Obsidian-specific parsing:
+ *   - YAML frontmatter (--- blocks): extracted as metadata, tags, aliases
+ *   - [[Wikilinks]]: resolved to cross-references, stored as fact relationships
+ *   - [[Wikilink|alias]]: aliased links normalized
+ *   - ![[Embedded files]]: stripped (non-text embeds skipped)
+ *   - #tags: extracted and stored as fact tags
+ *   - Backlinks: tracked via wikilink resolution
+ *
+ * Design:
+ *   - Uses fs.watch for low-dependency file watching (no chokidar)
+ *   - Idempotent: tracks file hashes, skips unchanged files
+ *   - Respects .obsidian/app.json excludedFolders if present
+ *   - Sanitizes secrets before ingest (reuses secret-scanner)
+ *   - Does NOT ingest .obsidian/ config files, templates, or attachments
+ *
+ * Config (via HyperMemConfig.obsidian):
+ *   vaultPath:        Absolute path to Obsidian vault directory
+ *   enabled:          Master switch (default: false)
+ *   watchInterval:    Polling interval ms for fs.watch fallback (default: 30000)
+ *   collection:       Doc-chunk collection name (default: 'obsidian/vault')
+ *   excludeFolders:   Additional folders to skip (merged with .obsidian excludedFolders)
+ *   agentId:          Agent to scope imported facts to (default: plugin agentId)
+ *   importTags:       Import #tags as fact tags (default: true)
+ *   importFrontmatter: Import frontmatter fields as facts (default: true)
+ *   staleDays:        Re-import files not seen in N days (default: 7)
+ */
+import { type DocChunk } from './doc-chunker.js';
+export interface ObsidianConfig {
+    /** Absolute path to the Obsidian vault directory */
+    vaultPath: string;
+    /** Master switch — vault is not watched unless true */
+    enabled: boolean;
+    /** Polling interval ms for change detection (default: 30000) */
+    watchInterval?: number;
+    /** Collection name for doc-chunk store (default: 'obsidian/vault') */
+    collection?: string;
+    /** Additional folders to exclude from import */
+    excludeFolders?: string[];
+    /** Agent ID to scope imported facts to */
+    agentId?: string;
+    /** Import #tags as fact tags (default: true) */
+    importTags?: boolean;
+    /** Import frontmatter key/value pairs as facts (default: true) */
+    importFrontmatter?: boolean;
+    /** Re-import files not seen in N days (default: 7) */
+    staleDays?: number;
+}
+export interface ObsidianNote {
+    /** Relative path from vault root */
+    relativePath: string;
+    /** Note title (filename without extension, or frontmatter title) */
+    title: string;
+    /** Raw markdown content (post-frontmatter strip) */
+    content: string;
+    /** Parsed frontmatter fields */
+    frontmatter: Record<string, unknown>;
+    /** Extracted #tags (from content and frontmatter) */
+    tags: string[];
+    /** Wikilinks found in this note: [[target]] → target title */
+    wikilinks: ObsidianWikiLink[];
+    /** SHA-256 of original file content */
+    contentHash: string;
+    /** File mtime */
+    modifiedAt: Date;
+}
+export interface ObsidianWikiLink {
+    /** The target page title (normalized) */
+    target: string;
+    /** Alias if present: [[target|alias]] */
+    alias?: string;
+    /** Whether this is an embed: ![[target]] */
+    isEmbed: boolean;
+}
+export interface ObsidianImportResult {
+    imported: number;
+    skipped: number;
+    failed: number;
+    chunks: DocChunk[];
+    notes: ObsidianNote[];
+    wikilinks: Map<string, ObsidianWikiLink[]>;
+}
+/**
+ * Extract YAML frontmatter from a markdown file.
+ * Returns { frontmatter, body } where frontmatter is a key/value map
+ * and body is the content after the closing ---.
+ */
+export declare function parseFrontmatter(content: string): {
+    frontmatter: Record<string, unknown>;
+    body: string;
+};
+/**
+ * Extract all [[wikilinks]] from markdown content.
+ * Handles:
+ *   [[Page Name]]          → { target: 'Page Name' }
+ *   [[Page Name|Alias]]    → { target: 'Page Name', alias: 'Alias' }
+ *   ![[Embedded File.png]] → { target: 'Embedded File.png', isEmbed: true }
+ */
+export declare function extractWikilinks(content: string): ObsidianWikiLink[];
+/**
+ * Extract #tags from markdown content (not inside code blocks).
+ * Also merges frontmatter tags array if present.
+ */
+export declare function extractTags(content: string, frontmatter: Record<string, unknown>): string[];
+/**
+ * Clean Obsidian markdown for chunking:
+ * - Convert [[wikilinks]] to plain text (preserve the label)
+ * - Strip ![[embeds]] entirely
+ * - Strip Obsidian-specific syntax that confuses the chunker
+ */
+export declare function cleanObsidianMarkdown(content: string): string;
+/**
+ * Parse a single Obsidian markdown file into an ObsidianNote.
+ * Returns null if the file cannot be read or fails secret scanning.
+ */
+export declare function parseObsidianNote(filePath: string, vaultRoot: string): ObsidianNote | null;
+/**
+ * Import all notes from an Obsidian vault into hypermem doc chunks.
+ *
+ * This is the main entry point for one-shot import. The watcher calls
+ * this incrementally on file change events.
+ *
+ * @param config   ObsidianConfig from HyperMemConfig
+ * @param seenHashes  Map of relativePath → contentHash for skip-unchanged logic
+ * @returns Import result with chunks ready for doc-chunk-store insertion
+ */
+export declare function importVault(config: ObsidianConfig, seenHashes?: Map<string, string>): ObsidianImportResult;
+export type VaultChangeCallback = (result: ObsidianImportResult) => void | Promise<void>;
+/**
+ * Watch an Obsidian vault for changes and trigger imports incrementally.
+ *
+ * Uses Node's built-in fs.watch (no chokidar dependency).
+ * Falls back to polling if watch events are unreliable on the platform.
+ *
+ * @param config    ObsidianConfig
+ * @param onChange  Callback receiving incremental import results on each change
+ * @returns stop()  Call to unwatch the vault
+ */
+export declare function watchVault(config: ObsidianConfig, onChange: VaultChangeCallback): {
+    stop: () => void;
+};
+//# sourceMappingURL=obsidian-watcher.d.ts.map

package/dist/obsidian-watcher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"obsidian-watcher.d.ts","sourceRoot":"","sources":["../src/obsidian-watcher.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAIH,OAAO,EAA8B,KAAK,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAK7E,MAAM,WAAW,cAAc;IAC7B,oDAAoD;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,uDAAuD;IACvD,OAAO,EAAE,OAAO,CAAC;IACjB,gEAAgE;IAChE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,sEAAsE;IACtE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gDAAgD;IAChD,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,0CAA0C;IAC1C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,gDAAgD;IAChD,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,kEAAkE;IAClE,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,sDAAsD;IACtD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,YAAY;IAC3B,oCAAoC;IACpC,YAAY,EAAE,MAAM,CAAC;IACrB,oEAAoE;IACpE,KAAK,EAAE,MAAM,CAAC;IACd,oDAAoD;IACpD,OAAO,EAAE,MAAM,CAAC;IAChB,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,qDAAqD;IACrD,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,8DAA8D;IAC9D,SAAS,EAAE,gBAAgB,EAAE,CAAC;IAC9B,uCAAuC;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,iBAAiB;IACjB,UAAU,EAAE,IAAI,CAAC;CAClB;AAED,MAAM,WAAW,gBAAgB;IAC/B,yCAAyC;IACzC,MAAM,EAAE,MAAM,CAAC;IACf,yCAAyC;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,4CAA4C;IAC5C,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,QAAQ,EAAE,CAAC;IACnB,KAAK,EAAE,YAAY,EAAE,CAAC;IACtB,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,gBAAgB,EAAE,CAAC,CAAC;CAC5C;AAID;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG;IACjD,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,IAAI,EAAE,MAAM,CAAC;CACd,CA8CA;AAID;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,gBAAgB,EAAE,CAsBpE;AAID;;;GAGG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,EAAE,CAqB3F;AAID;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAW7D;AA4ED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,YAAY,GAAG,IAAI,CA2C1F;AAID;;;;;;;;;GASG;AACH,wBAAgB,WAAW,CACzB,MAAM,EAAE,cAAc,EACtB,UAAU,GAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAa,GAC1C,oBAAoB,CA+EtB;AAID,MAAM,MAAM,mBAAmB,GAAG,CAAC,MAAM,EAAE,oBAAoB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAEzF;;;;;;;;;GASG;AACH,wBAAgB,UAAU,CACxB,MAAM,EAAE,cAAc,EACtB,QAAQ,EAAE,mBAAmB,GAC5B;IAAE,IAAI,EAAE,MAAM,IAAI,CAAA;CAAE,CAmDtB"}