@psiclawops/hypermem 0.5.0 → 0.5.1

package/dist/doc-chunker.js

@@ -0,0 +1,324 @@
+/**
+ * hypermem Document Chunker
+ *
+ * Splits markdown documents into semantically coherent chunks for L3 indexing.
+ *
+ * Design principles:
+ * - Chunk by logical section (## / ###), NOT by token count
+ * - Each chunk is a self-contained policy/operational unit
+ * - Preserve section hierarchy for context assembly
+ * - Track source file hash for atomic re-indexing
+ * - Idempotent: same source produces same chunks (deterministic IDs)
+ *
+ * Collections (as defined in ACA offload spec):
+ *   governance/policy   — POLICY.md, shared-fleet
+ *   governance/charter  — CHARTER.md, per-tier (council/director)
+ *   governance/comms    — COMMS.md, shared-fleet
+ *   operations/agents   — AGENTS.md, per-tier
+ *   operations/tools    — TOOLS.md, per-agent
+ *   memory/decisions    — MEMORY.md, per-agent
+ *   memory/daily        — memory/YYYY-MM-DD.md, per-agent
+ *   identity/soul       — SOUL.md, per-agent (always-loaded kernel, but still indexed)
+ *   identity/job        — JOB.md, per-agent (demand-loaded during deliberation)
+ */
+import { createHash } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+// ─── Core chunker ───────────────────────────────────────────────
+/**
+ * Hash a string with SHA-256.
+ */
+export function hashContent(content) {
+    return createHash('sha256').update(content).digest('hex');
+}
+/**
+ * Estimate token count from character length.
+ * Rough heuristic: 1 token ≈ 4 chars for English prose.
+ */
+function estimateTokens(text) {
+    return Math.ceil(text.length / 4);
+}
+function parseMarkdownSections(markdown) {
+    const lines = markdown.split('\n');
+    const sections = [];
+    let currentLevel = 0;
+    let currentHeading = '';
+    let currentRaw = '';
+    let contentLines = [];
+    function flush() {
+        if (currentHeading || contentLines.length > 0) {
+            sections.push({
+                heading: currentHeading,
+                level: currentLevel,
+                content: contentLines.join('\n').trim(),
+                rawHeading: currentRaw,
+            });
+        }
+    }
+    for (const line of lines) {
+        const h3Match = line.match(/^### (.+)$/);
+        const h2Match = line.match(/^## (.+)$/);
+        const h1Match = line.match(/^# (.+)$/);
+        if (h3Match || h2Match || h1Match) {
+            flush();
+            currentRaw = line;
+            if (h3Match) {
+                currentLevel = 3;
+                currentHeading = h3Match[1].trim();
+            }
+            else if (h2Match) {
+                currentLevel = 2;
+                currentHeading = h2Match[1].trim();
+            }
+            else if (h1Match) {
+                currentLevel = 1;
+                currentHeading = h1Match[1].trim();
+            }
+            contentLines = [];
+        }
+        else {
+            contentLines.push(line);
+        }
+    }
+    flush();
+    return sections;
+}
+/**
+ * Build a hierarchical section path from the section stack.
+ * e.g., ["POLICY.md", "§3 Escalation", "Triggers"] → "POLICY.md > §3 Escalation > Triggers"
+ */
+function buildSectionPath(stack) {
+    return stack.filter(Boolean).join(' > ');
+}
+/**
+ * Generate a deterministic chunk ID from its identifying properties.
+ * Includes sourcePath to prevent collisions when different files have
+ * identical content and section structure (e.g., POLICY.md in two workspaces).
+ */
+function chunkId(collection, sectionPath, sourceHash, sourcePath) {
+    const key = `${collection}::${sourcePath}::${sectionPath}::${sourceHash}`;
+    return createHash('sha256').update(key).digest('hex').slice(0, 16);
+}
+/**
+ * Chunk a markdown document into semantic sections.
+ *
+ * Approach:
+ * - Level 1 (#) headings become top-level section anchors
+ * - Level 2 (##) headings become primary chunks
+ * - Level 3 (###) headings become sub-chunks under their parent
+ * - Content before the first heading becomes a "preamble" chunk
+ * - Empty sections (heading only, no content) are skipped unless minContentLen=0
+ *
+ * For documents with deeply nested content, we group level-3 sections under
+ * their parent level-2 section. This keeps related policy sections together.
+ */
+export function chunkMarkdown(content, opts) {
+    const minLen = opts.minContentLen ?? 50;
+    const sourceHash = hashContent(content);
+    const sections = parseMarkdownSections(content);
+    const chunks = [];
+    // Track current section hierarchy for path building
+    let h1Heading = '';
+    let h2Heading = '';
+    const h2ContentLines = [];
+    let h2Active = false;
+    function flushH2() {
+        if (!h2Active)
+            return;
+        const body = h2ContentLines.join('\n').trim();
+        if (body.length < minLen && !opts.includeParentContext) {
+            h2Active = false;
+            h2ContentLines.length = 0;
+            return;
+        }
+        const sectionPath = buildSectionPath([h1Heading, h2Heading].filter(Boolean));
+        const chunkContent = [
+            h1Heading ? `# ${h1Heading}` : '',
+            `## ${h2Heading}`,
+            '',
+            body,
+        ].filter(l => l !== '' || body).join('\n').trim();
+        if (chunkContent.length >= minLen) {
+            chunks.push({
+                id: chunkId(opts.collection, sectionPath, sourceHash, opts.sourcePath),
+                collection: opts.collection,
+                sectionPath,
+                depth: 2,
+                content: chunkContent,
+                tokenEstimate: estimateTokens(chunkContent),
+                sourceHash,
+                sourcePath: opts.sourcePath,
+                scope: opts.scope,
+                tier: opts.tier,
+                agentId: opts.agentId,
+                parentPath: h1Heading || undefined,
+            });
+        }
+        h2Active = false;
+        h2ContentLines.length = 0;
+    }
+    // Preamble: text before first heading
+    let preamble = '';
+    for (const section of sections) {
+        if (section.level === 1) {
+            flushH2();
+            h1Heading = section.heading;
+            h2Heading = '';
+            // Level-1 headings with body content become a preamble chunk
+            if (section.content.length >= minLen) {
+                const sectionPath = buildSectionPath([h1Heading]);
+                const chunkContent = [`# ${h1Heading}`, '', section.content].join('\n').trim();
+                chunks.push({
+                    id: chunkId(opts.collection, sectionPath + '::intro', sourceHash, opts.sourcePath),
+                    collection: opts.collection,
+                    sectionPath: sectionPath + ' (intro)',
+                    depth: 1,
+                    content: chunkContent,
+                    tokenEstimate: estimateTokens(chunkContent),
+                    sourceHash,
+                    sourcePath: opts.sourcePath,
+                    scope: opts.scope,
+                    tier: opts.tier,
+                    agentId: opts.agentId,
+                });
+            }
+        }
+        else if (section.level === 2) {
+            flushH2();
+            h2Heading = section.heading;
+            h2Active = true;
+            // Start accumulating: begin with this section's own content
+            h2ContentLines.length = 0;
+            if (section.content)
+                h2ContentLines.push(section.content);
+        }
+        else if (section.level === 3) {
+            // Append h3 sub-sections into the current h2 chunk
+            if (h2Active) {
+                h2ContentLines.push(`\n### ${section.heading}`);
+                if (section.content)
+                    h2ContentLines.push(section.content);
+            }
+            else {
+                // h3 without a parent h2 — emit as standalone
+                const sectionPath = buildSectionPath([h1Heading, section.heading].filter(Boolean));
+                const chunkContent = [
+                    h1Heading ? `# ${h1Heading}` : '',
+                    `### ${section.heading}`,
+                    '',
+                    section.content,
+                ].filter(l => l !== '' || section.content).join('\n').trim();
+                if (chunkContent.length >= minLen) {
+                    chunks.push({
+                        id: chunkId(opts.collection, sectionPath, sourceHash, opts.sourcePath),
+                        collection: opts.collection,
+                        sectionPath,
+                        depth: 3,
+                        content: chunkContent,
+                        tokenEstimate: estimateTokens(chunkContent),
+                        sourceHash,
+                        sourcePath: opts.sourcePath,
+                        scope: opts.scope,
+                        tier: opts.tier,
+                        agentId: opts.agentId,
+                        parentPath: h1Heading || undefined,
+                    });
+                }
+            }
+        }
+        else if (section.level === 0) {
+            // Pre-heading content
+            preamble = section.content.trim();
+        }
+    }
+    // Flush any remaining h2
+    flushH2();
+    // Emit preamble if substantial
+    if (preamble.length >= minLen) {
+        chunks.unshift({
+            id: chunkId(opts.collection, '(preamble)', sourceHash, opts.sourcePath),
+            collection: opts.collection,
+            sectionPath: '(preamble)',
+            depth: 0,
+            content: preamble,
+            tokenEstimate: estimateTokens(preamble),
+            sourceHash,
+            sourcePath: opts.sourcePath,
+            scope: opts.scope,
+            tier: opts.tier,
+            agentId: opts.agentId,
+        });
+    }
+    return chunks;
+}
+/**
+ * Chunk a file from disk.
+ */
+export function chunkFile(filePath, opts) {
+    const content = readFileSync(filePath, 'utf-8');
+    return chunkMarkdown(content, { ...opts, sourcePath: filePath });
+}
+export const ACA_COLLECTIONS = {
+    'POLICY.md': {
+        collection: 'governance/policy',
+        scope: 'shared-fleet',
+        description: 'Governance policy: escalation triggers, decision states, council procedures, naming rules',
+    },
+    'CHARTER.md': {
+        collection: 'governance/charter',
+        scope: 'per-tier',
+        description: 'Org charter: mission, director structure, boundaries, escalation, work queue',
+    },
+    'COMMS.md': {
+        collection: 'governance/comms',
+        scope: 'shared-fleet',
+        description: 'Communications protocol: inter-agent tiers, delegation, platform formatting',
+    },
+    'AGENTS.md': {
+        collection: 'operations/agents',
+        scope: 'per-tier',
+        description: 'Agent operational guide: boot sequence, identity, memory, messaging, group chats',
+    },
+    'TOOLS.md': {
+        collection: 'operations/tools',
+        scope: 'per-agent',
+        description: 'Tool and runtime configuration: workspace path, model, key paths, quick commands',
+    },
+    'SOUL.md': {
+        collection: 'identity/soul',
+        scope: 'per-agent',
+        description: 'Agent soul: core principles, personality, tone, continuity',
+    },
+    'JOB.md': {
+        collection: 'identity/job',
+        scope: 'per-agent',
+        description: 'Job performance criteria: duties, response contract, council mode, output discipline',
+    },
+    'MOTIVATIONS.md': {
+        collection: 'identity/motivations',
+        scope: 'per-agent',
+        description: 'Agent motivations: drives, fears, tensions that shape perspective',
+    },
+    'MEMORY.md': {
+        collection: 'memory/decisions',
+        scope: 'per-agent',
+        description: 'Long-term curated memory: key decisions, lessons, context',
+    },
+};
+/**
+ * Infer the collection definition for a file based on its name.
+ * Returns undefined if the file is not a known ACA file.
+ */
+export function inferCollection(fileName, agentId) {
+    // Strip path, get just the filename
+    const base = fileName.split('/').pop() ?? fileName;
+    // Daily memory files: memory/YYYY-MM-DD.md
+    if (/^\d{4}-\d{2}-\d{2}\.md$/.test(base)) {
+        return {
+            collection: 'memory/daily',
+            scope: 'per-agent',
+            description: `Daily memory log for ${agentId || 'agent'}`,
+        };
+    }
+    return ACA_COLLECTIONS[base];
+}
+//# sourceMappingURL=doc-chunker.js.map

package/dist/dreaming-promoter.d.ts

@@ -0,0 +1,86 @@
+/**
+ * dreaming-promoter.ts
+ *
+ * hypermem-native dreaming promotion pass.
+ *
+ * Unlike the stock memory-core dreaming feature (which appends raw content to
+ * MEMORY.md), this promoter generates pointer-format entries that match the
+ * council's MEMORY.md convention:
+ *
+ *   - **{domain} — {title}:** {summary}
+ *     → `memory_search("{query}")`
+ *
+ * Scoring uses confidence, decay, recency, and domain cluster weight.
+ * Dedup prevents re-promoting topics already covered by existing pointers.
+ *
+ * Dry-run mode returns what would be written without modifying any files.
+ */
+import type { DatabaseSync } from 'node:sqlite';
+export interface DreamerConfig {
+    /** Enable the promotion pass. Default: false */
+    enabled: boolean;
+    /** Minimum composite score for promotion. Default: 0.70 */
+    minScore: number;
+    /** Minimum confidence threshold (pre-scoring). Default: 0.70 */
+    minConfidence: number;
+    /** Max new pointer entries to write per agent per run. Default: 5 */
+    maxPromotionsPerRun: number;
+    /** How often to run: every N indexer ticks. Default: 12 (~1hr at 5min interval) */
+    tickInterval: number;
+    /** Preview what would be promoted without writing. Default: false */
+    dryRun: boolean;
+    /** Recency half-life in days (score decays to 0.5 at this age). Default: 14 */
+    recencyHalfLifeDays: number;
+    /** Max age in days to consider a fact. Default: 90 */
+    maxAgeDays: number;
+}
+export declare const DEFAULT_DREAMER_CONFIG: DreamerConfig;
+export interface FactCandidate {
+    id: number;
+    agentId: string;
+    domain: string;
+    content: string;
+    confidence: number;
+    decayScore: number;
+    ageDays: number;
+    score: number;
+}
+export interface PromotionEntry {
+    factId: number;
+    domain: string;
+    pointer: string;
+    title: string;
+    summary: string;
+    query: string;
+    score: number;
+    dryRun: boolean;
+}
+export interface DreamerResult {
+    agentId: string;
+    candidates: number;
+    promoted: number;
+    skippedDuplicate: number;
+    skippedThreshold: number;
+    entries: PromotionEntry[];
+    memoryPath: string | null;
+    dryRun: boolean;
+}
+/**
+ * Resolve the workspace directory for an agent.
+ * Council agents live at ~/.openclaw/workspace-council/{agentId}/
+ * Other agents at ~/.openclaw/workspace/{agentId}/
+ */
+export declare function resolveAgentWorkspacePath(agentId: string): Promise<string | null>;
+/**
+ * Run the dreaming promotion pass for a single agent.
+ *
+ * Reads qualified facts from library.db, scores them, deduplicates against
+ * existing MEMORY.md pointers, and writes new pointer entries.
+ */
+export declare function runDreamingPromoter(agentId: string, libraryDb: DatabaseSync, config?: Partial<DreamerConfig>): Promise<DreamerResult>;
+/**
+ * Run the dreaming promotion pass for all agents in a fleet.
+ * Called from the BackgroundIndexer on every N ticks.
+ */
+export declare function runDreamingPassForFleet(agentIds: string[], libraryDb: DatabaseSync, config?: Partial<DreamerConfig>): Promise<DreamerResult[]>;
+//# sourceMappingURL=dreaming-promoter.d.ts.map

package/dist/dreaming-promoter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dreaming-promoter.d.ts","sourceRoot":"","sources":["../src/dreaming-promoter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAKH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAIhD,MAAM,WAAW,aAAa;IAC5B,gDAAgD;IAChD,OAAO,EAAE,OAAO,CAAC;IACjB,2DAA2D;IAC3D,QAAQ,EAAE,MAAM,CAAC;IACjB,gEAAgE;IAChE,aAAa,EAAE,MAAM,CAAC;IACtB,qEAAqE;IACrE,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mFAAmF;IACnF,YAAY,EAAE,MAAM,CAAC;IACrB,qEAAqE;IACrE,MAAM,EAAE,OAAO,CAAC;IAChB,+EAA+E;IAC/E,mBAAmB,EAAE,MAAM,CAAC;IAC5B,sDAAsD;IACtD,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,eAAO,MAAM,sBAAsB,EAAE,aASpC,CAAC;AAIF,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,MAAM,EAAE,OAAO,CAAC;CACjB;AAID;;;;GAIG;AACH,wBAAsB,yBAAyB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAgBvF;AA2ND;;;;;GAKG;AACH,wBAAsB,mBAAmB,CACvC,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,YAAY,EACvB,MAAM,GAAE,OAAO,CAAC,aAAa,CAAM,GAClC,OAAO,CAAC,aAAa,CAAC,CAgJxB;AAED;;;GAGG;AACH,wBAAsB,uBAAuB,CAC3C,QAAQ,EAAE,MAAM,EAAE,EAClB,SAAS,EAAE,YAAY,EACvB,MAAM,GAAE,OAAO,CAAC,aAAa,CAAM,GAClC,OAAO,CAAC,aAAa,EAAE,CAAC,CAa1B"}