@psiclawops/hypermem 0.1.0 → 0.5.1

package/dist/open-domain.js

@@ -0,0 +1,125 @@
+/**
+ * open-domain.ts — Open-domain query detection and FTS5 retrieval
+ *
+ * LoCoMo benchmark open-domain questions are broad, exploratory, and have no
+ * topical anchor. They span the full conversation history and require content
+ * that may have been filtered out by the quality gate (isQualityFact). The
+ * fix: detect open-domain queries and run a separate FTS5 search against raw
+ * messages_fts, bypassing the quality filter entirely.
+ *
+ * Detection heuristics (conservative — false positives add noise):
+ *   - Short query with no named entities (no TitleCase tokens)
+ *   - Broad interrogative patterns (what did, how did, tell me about, etc.)
+ *   - No temporal signals (those go to the temporal retrieval path)
+ *   - No specific identifiers (URLs, IDs, ticket numbers, version strings)
+ *
+ * Retrieval: MessageStore.searchMessages() against messages_fts — covers all
+ * raw message history regardless of quality gate.
+ */
+// ── Open-domain signal patterns ───────────────────────────────────────────
+const BROAD_INTERROGATIVE = /\b(what did|what does|what has|what was|what were|what is|how did|how does|how has|tell me about|describe|explain|summarize|overview|recap|what do you know about|what have|who is|who was|who did)\b/i;
+const SPECIFIC_ANCHOR = /\b([A-Z][a-z]{2,}(?:\s+[A-Z][a-z]{2,})+|v\d+\.\d+|#\d{2,}|https?:\/\/|[A-Z]{2,}-\d+)\b/;
+const TEMPORAL_SIGNALS = /\b(before|after|when|last\s+\w+|yesterday|today|recently|between|since|until|ago|this\s+week|this\s+month|in\s+(january|february|march|april|may|june|july|august|september|october|november|december))\b/i;
+/**
+ * Returns true if the query looks like an open-domain question:
+ * broad, exploratory, no specific anchors, no temporal signals.
+ */
+export function isOpenDomainQuery(query) {
+    if (!query || query.trim().length < 8)
+        return false;
+    // Has temporal signals → temporal path handles it
+    if (TEMPORAL_SIGNALS.test(query))
+        return false;
+    // Has specific named entity / version / ticket anchor → not open-domain
+    if (SPECIFIC_ANCHOR.test(query))
+        return false;
+    // Must match a broad interrogative pattern
+    if (!BROAD_INTERROGATIVE.test(query))
+        return false;
+    // Sanity: query should not be too long (long queries are usually specific)
+    const wordCount = query.trim().split(/\s+/).length;
+    if (wordCount > 20)
+        return false;
+    return true;
+}
+// ── FTS5 query builder ────────────────────────────────────────────────────
+/**
+ * Build a FTS5 MATCH query from a broad question.
+ * Strips stop words, question words, and punctuation.
+ * Returns up to 6 prefix-matched terms joined with OR.
+ */
+export function buildOpenDomainFtsQuery(query) {
+    const STOP_WORDS = new Set([
+        'what', 'did', 'does', 'has', 'was', 'were', 'is', 'are', 'how',
+        'tell', 'me', 'about', 'describe', 'explain', 'summarize', 'overview',
+        'recap', 'who', 'do', 'you', 'know', 'have', 'the', 'a', 'an', 'of',
+        'in', 'on', 'at', 'to', 'for', 'and', 'or', 'but', 'with', 'from',
+    ]);
+    const terms = query
+        .toLowerCase()
+        .replace(/[^a-z0-9\s]/g, ' ')
+        .split(/\s+/)
+        .filter(w => w.length >= 3 && !STOP_WORDS.has(w))
+        .slice(0, 6)
+        .map(w => `${w}*`);
+    if (terms.length === 0)
+        return null;
+    return terms.join(' OR ');
+}
+/**
+ * Search raw message history via FTS5 for open-domain queries.
+ * Returns up to `limit` matching messages, deduplicated against existing context.
+ *
+ * @param db — agent messages DB (contains messages_fts)
+ * @param query — the user's query
+ * @param existingContent — already-assembled context (for dedup)
+ * @param limit — max results (default 10)
+ */
+export function searchOpenDomain(db, query, existingContent, limit = 10) {
+    const ftsQuery = buildOpenDomainFtsQuery(query);
+    if (!ftsQuery)
+        return [];
+    try {
+        const rows = db.prepare(`
+      WITH fts_matches AS (
+        SELECT rowid, rank
+        FROM messages_fts
+        WHERE messages_fts MATCH ?
+        ORDER BY rank
+        LIMIT ?
+      )
+      SELECT
+        m.role,
+        m.text_content AS content,
+        m.created_at AS createdAt
+      FROM messages m
+      JOIN fts_matches ON m.id = fts_matches.rowid
+      WHERE m.text_content IS NOT NULL
+        AND m.text_content != ''
+        AND m.is_heartbeat = 0
+      ORDER BY fts_matches.rank
+    `).all(ftsQuery, limit * 2);
+        // Deduplicate against existing context and filter short content
+        const seen = new Set();
+        const results = [];
+        for (const row of rows) {
+            if (!row.content || row.content.trim().length < 20)
+                continue;
+            const fingerprint = row.content.slice(0, 80);
+            if (seen.has(fingerprint))
+                continue;
+            if (existingContent.includes(fingerprint))
+                continue;
+            seen.add(fingerprint);
+            results.push(row);
+            if (results.length >= limit)
+                break;
+        }
+        return results;
+    }
+    catch {
+        // FTS query may fail on special characters — degrade silently
+        return [];
+    }
+}
+//# sourceMappingURL=open-domain.js.map

package/dist/preference-store.d.ts

@@ -1,5 +1,5 @@
 /**
- * HyperMem Preference Store
+ * hypermem Preference Store
  *
  * Behavioral patterns observed about people, systems, and workflows.
  * Lives in the central library DB.

package/dist/preference-store.js

@@ -1,5 +1,5 @@
 /**
- * HyperMem Preference Store
+ * hypermem Preference Store
  *
  * Behavioral patterns observed about people, systems, and workflows.
  * Lives in the central library DB.

package/dist/preservation-gate.d.ts

@@ -1,5 +1,5 @@
 /**
- * HyperMem Preservation Gate
+ * hypermem Preservation Gate
  *
  * Verifies that a proposed compaction summary preserves the semantic
  * content of its source messages by measuring geometric fidelity in

package/dist/preservation-gate.js

@@ -1,5 +1,5 @@
 /**
- * HyperMem Preservation Gate
+ * hypermem Preservation Gate
  *
  * Verifies that a proposed compaction summary preserves the semantic
  * content of its source messages by measuring geometric fidelity in

package/dist/proactive-pass.d.ts

@@ -0,0 +1,63 @@
+/**
+ * hypermem Proactive Passes
+ *
+ * Background maintenance passes that run between indexer ticks to keep
+ * message storage lean. Two passes:
+ *
+ *   1. Noise Sweep — deletes low/zero-signal messages outside the recent
+ *      window (heartbeats, acks, empty strings, control tokens).
+ *
+ *   2. Tool Decay — truncates oversized tool_results outside the recent
+ *      window in-place, preserving JSON structure but collapsing large
+ *      content blobs into a byte-count placeholder.
+ *
+ * Both passes are:
+ *   - Synchronous (DatabaseSync, no async)
+ *   - Wrapped in transactions (atomic)
+ *   - Best-effort: catch all errors, log, and return a zero-change result
+ *
+ * Ported and adapted from ClawText proactive-pass.ts.
+ * hypermem schema differences vs ClawText:
+ *   - No content_type column — we classify on the fly via classifyContentType()
+ *   - No external payload store — we truncate content inline in tool_results JSON
+ *   - No ClawText-specific dependencies (payload-store, tool-tracker, etc.)
+ */
+import type { DatabaseSync } from 'node:sqlite';
+export interface NoiseSweepResult {
+    messagesDeleted: number;
+    passType: 'noise_sweep';
+}
+export interface ToolDecayResult {
+    messagesUpdated: number;
+    bytesFreed: number;
+    passType: 'tool_decay';
+}
+/**
+ * Delete noise and heartbeat messages outside the recent window.
+ *
+ * "Outside the recent window" means message_index < maxIndex - recentWindowSize.
+ * Messages inside the window are never deleted, even if they are noise —
+ * the model may still reference them in the current turn.
+ *
+ * Deletions are wrapped in a single transaction. The FTS5 trigger handles
+ * index cleanup automatically (msg_fts_ad fires on DELETE).
+ */
+export declare function runNoiseSweep(db: DatabaseSync, conversationId: number, recentWindowSize?: number): NoiseSweepResult;
+/**
+ * Truncate oversized tool_results outside the recent window.
+ *
+ * Strategy:
+ *   1. Find messages whose tool_results JSON string is > 2000 chars total,
+ *      outside the recent window.
+ *   2. Parse the JSON array.
+ *   3. For each result entry where the `content` field exceeds 500 chars,
+ *      replace `content` with `[tool result truncated — N bytes]`.
+ *   4. Re-serialize and write back.
+ *
+ * The JSON structure is preserved (array of result objects). Only the
+ * oversized `content` values are collapsed.
+ *
+ * Mutations are committed in a single transaction.
+ */
+export declare function runToolDecay(db: DatabaseSync, conversationId: number, recentWindowSize?: number): ToolDecayResult;
+//# sourceMappingURL=proactive-pass.d.ts.map

package/dist/proactive-pass.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"proactive-pass.d.ts","sourceRoot":"","sources":["../src/proactive-pass.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAKhD,MAAM,WAAW,gBAAgB;IAC/B,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,eAAe;IAC9B,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,YAAY,CAAC;CACxB;AA6CD;;;;;;;;;GASG;AACH,wBAAgB,aAAa,CAC3B,EAAE,EAAE,YAAY,EAChB,cAAc,EAAE,MAAM,EACtB,gBAAgB,GAAE,MAAW,GAC5B,gBAAgB,CA2ElB;AAID;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,YAAY,CAC1B,EAAE,EAAE,YAAY,EAChB,cAAc,EAAE,MAAM,EACtB,gBAAgB,GAAE,MAAW,GAC5B,eAAe,CAiGjB"}

package/dist/proactive-pass.js

@@ -0,0 +1,239 @@
+/**
+ * hypermem Proactive Passes
+ *
+ * Background maintenance passes that run between indexer ticks to keep
+ * message storage lean. Two passes:
+ *
+ *   1. Noise Sweep — deletes low/zero-signal messages outside the recent
+ *      window (heartbeats, acks, empty strings, control tokens).
+ *
+ *   2. Tool Decay — truncates oversized tool_results outside the recent
+ *      window in-place, preserving JSON structure but collapsing large
+ *      content blobs into a byte-count placeholder.
+ *
+ * Both passes are:
+ *   - Synchronous (DatabaseSync, no async)
+ *   - Wrapped in transactions (atomic)
+ *   - Best-effort: catch all errors, log, and return a zero-change result
+ *
+ * Ported and adapted from ClawText proactive-pass.ts.
+ * hypermem schema differences vs ClawText:
+ *   - No content_type column — we classify on the fly via classifyContentType()
+ *   - No external payload store — we truncate content inline in tool_results JSON
+ *   - No ClawText-specific dependencies (payload-store, tool-tracker, etc.)
+ */
+import { classifyContentType } from './content-type-classifier.js';
+// ─── Internal helpers ────────────────────────────────────────────
+/**
+ * Resolve the safe window to a finite positive integer.
+ * Mirrors the ClawText resolveSafeWindow() guard.
+ */
+function resolveSafeWindow(recentWindowSize) {
+    if (Number.isFinite(recentWindowSize) && recentWindowSize > 0) {
+        return Math.floor(recentWindowSize);
+    }
+    return 20;
+}
+/**
+ * Get the maximum message_index for a conversation.
+ * Returns -1 if no messages exist.
+ */
+function getMaxMessageIndex(db, conversationId) {
+    const row = db
+        .prepare('SELECT COALESCE(MAX(message_index), -1) AS max_index FROM messages WHERE conversation_id = ?')
+        .get(conversationId);
+    return typeof row.max_index === 'number' ? row.max_index : -1;
+}
+/**
+ * Decide if a message is noise based on content + is_heartbeat flag.
+ *
+ * A message is noise when:
+ *   - is_heartbeat = 1  (explicit heartbeat marker), OR
+ *   - text_content is NULL or empty (≤3 chars after trimming), OR
+ *   - classifyContentType() returns 'noise' or 'ack'
+ *
+ * We call the classifier rather than duplicating its patterns here.
+ */
+function isNoiseMessage(textContent, isHeartbeat) {
+    if (isHeartbeat === 1)
+        return true;
+    if (textContent === null || textContent.trim().length <= 3)
+        return true;
+    const { type } = classifyContentType(textContent);
+    return type === 'noise' || type === 'ack';
+}
+// ─── Noise Sweep ─────────────────────────────────────────────────
+/**
+ * Delete noise and heartbeat messages outside the recent window.
+ *
+ * "Outside the recent window" means message_index < maxIndex - recentWindowSize.
+ * Messages inside the window are never deleted, even if they are noise —
+ * the model may still reference them in the current turn.
+ *
+ * Deletions are wrapped in a single transaction. The FTS5 trigger handles
+ * index cleanup automatically (msg_fts_ad fires on DELETE).
+ */
+export function runNoiseSweep(db, conversationId, recentWindowSize = 20) {
+    const ZERO = { messagesDeleted: 0, passType: 'noise_sweep' };
+    try {
+        const safeWindow = resolveSafeWindow(recentWindowSize);
+        const maxIndex = getMaxMessageIndex(db, conversationId);
+        if (maxIndex < 0)
+            return ZERO;
+        // Messages with message_index strictly below this value are eligible.
+        const cutoff = maxIndex - safeWindow;
+        if (cutoff <= 0)
+            return ZERO; // Not enough history yet
+        // Fetch all candidate messages outside the recent window.
+        // Exclude messages whose content lives entirely in tool_results — those
+        // are tool result rows handled by runToolDecay(), not noise sweep.
+        // We deliberately avoid a content-based WHERE clause for the classifier
+        // because SQLite can't use the index for JS classification logic;
+        // it's cheaper to fetch a small batch and classify in JS.
+        const candidates = db
+            .prepare(`
+        SELECT id, text_content, is_heartbeat
+        FROM messages
+        WHERE conversation_id = ?
+          AND message_index < ?
+          AND (tool_results IS NULL OR tool_results = '')
+      `)
+            .all(conversationId, cutoff);
+        if (candidates.length === 0)
+            return ZERO;
+        // Filter to noise messages
+        const toDelete = candidates.filter(row => isNoiseMessage(row.text_content, row.is_heartbeat));
+        if (toDelete.length === 0)
+            return ZERO;
+        const ids = toDelete.map(r => r.id);
+        // Delete in a transaction; use chunked IN clauses to avoid
+        // SQLite's SQLITE_LIMIT_VARIABLE_NUMBER (default 999).
+        let totalDeleted = 0;
+        const CHUNK = 500;
+        db.prepare('BEGIN').run();
+        try {
+            for (let i = 0; i < ids.length; i += CHUNK) {
+                const chunk = ids.slice(i, i + CHUNK);
+                const placeholders = chunk.map(() => '?').join(', ');
+                const result = db
+                    .prepare(`DELETE FROM messages WHERE id IN (${placeholders})`)
+                    .run(...chunk);
+                totalDeleted += typeof result.changes === 'number' ? result.changes : chunk.length;
+            }
+            db.prepare('COMMIT').run();
+        }
+        catch (innerErr) {
+            db.prepare('ROLLBACK').run();
+            throw innerErr;
+        }
+        return { messagesDeleted: totalDeleted, passType: 'noise_sweep' };
+    }
+    catch (err) {
+        console.warn(`[proactive-pass] Noise sweep failed for conversation ${conversationId}: ${err instanceof Error ? err.message : String(err)}`);
+        return ZERO;
+    }
+}
+// ─── Tool Decay ──────────────────────────────────────────────────
+/**
+ * Truncate oversized tool_results outside the recent window.
+ *
+ * Strategy:
+ *   1. Find messages whose tool_results JSON string is > 2000 chars total,
+ *      outside the recent window.
+ *   2. Parse the JSON array.
+ *   3. For each result entry where the `content` field exceeds 500 chars,
+ *      replace `content` with `[tool result truncated — N bytes]`.
+ *   4. Re-serialize and write back.
+ *
+ * The JSON structure is preserved (array of result objects). Only the
+ * oversized `content` values are collapsed.
+ *
+ * Mutations are committed in a single transaction.
+ */
+export function runToolDecay(db, conversationId, recentWindowSize = 40) {
+    const ZERO = { messagesUpdated: 0, bytesFreed: 0, passType: 'tool_decay' };
+    try {
+        const safeWindow = resolveSafeWindow(recentWindowSize);
+        const maxIndex = getMaxMessageIndex(db, conversationId);
+        if (maxIndex < 0)
+            return ZERO;
+        const cutoff = maxIndex - safeWindow;
+        if (cutoff <= 0)
+            return ZERO;
+        // Fetch messages with large tool_results outside the recent window.
+        const candidates = db
+            .prepare(`
+        SELECT id, tool_results
+        FROM messages
+        WHERE conversation_id = ?
+          AND message_index < ?
+          AND tool_results IS NOT NULL
+          AND length(tool_results) > 2000
+      `)
+            .all(conversationId, cutoff);
+        if (candidates.length === 0)
+            return ZERO;
+        // Build the update list by processing each candidate.
+        const updates = [];
+        for (const row of candidates) {
+            let parsed;
+            try {
+                parsed = JSON.parse(row.tool_results);
+            }
+            catch {
+                // Corrupt JSON — skip this row
+                continue;
+            }
+            if (!Array.isArray(parsed))
+                continue;
+            let changed = false;
+            const newResults = parsed.map(entry => {
+                const content = entry.content;
+                if (typeof content === 'string' && content.length > 500) {
+                    const originalBytes = Buffer.byteLength(content, 'utf8');
+                    changed = true;
+                    return {
+                        ...entry,
+                        content: `[tool result truncated — ${originalBytes} bytes]`,
+                    };
+                }
+                return entry;
+            });
+            if (!changed)
+                continue;
+            const newJson = JSON.stringify(newResults);
+            const savedBytes = row.tool_results.length - newJson.length;
+            if (savedBytes > 0) {
+                updates.push({ id: row.id, newJson, savedBytes });
+            }
+        }
+        if (updates.length === 0)
+            return ZERO;
+        let totalUpdated = 0;
+        let totalBytesFreed = 0;
+        db.prepare('BEGIN').run();
+        try {
+            const stmt = db.prepare('UPDATE messages SET tool_results = ? WHERE id = ?');
+            for (const { id, newJson, savedBytes } of updates) {
+                stmt.run(newJson, id);
+                totalUpdated++;
+                totalBytesFreed += savedBytes;
+            }
+            db.prepare('COMMIT').run();
+        }
+        catch (innerErr) {
+            db.prepare('ROLLBACK').run();
+            throw innerErr;
+        }
+        return {
+            messagesUpdated: totalUpdated,
+            bytesFreed: totalBytesFreed,
+            passType: 'tool_decay',
+        };
+    }
+    catch (err) {
+        console.warn(`[proactive-pass] Tool decay failed for conversation ${conversationId}: ${err instanceof Error ? err.message : String(err)}`);
+        return ZERO;
+    }
+}
+//# sourceMappingURL=proactive-pass.js.map

package/dist/profiles.d.ts

@@ -0,0 +1,44 @@
+/**
+ * hypermem configuration profiles
+ *
+ * Pre-built configs for common deployment patterns. Pass to createHyperMem()
+ * directly or use as a base for custom configs via mergeProfile().
+ *
+ * Profiles:
+ *   minimal  — 64k context, single agent, low resource usage
+ *   standard — 128k context, fleet default, balanced
+ *   rich     — 200k+ context, multi-agent, full feature set
+ */
+import type { HyperMemConfig } from './types.js';
+export declare const lightProfile: HyperMemConfig;
+export declare const standardProfile: HyperMemConfig;
+export declare const fullProfile: HyperMemConfig;
+export type ProfileName = 'light' | 'standard' | 'full';
+export declare const minimalProfile: HyperMemConfig;
+export declare const extendedProfile: HyperMemConfig;
+export declare const richProfile: HyperMemConfig;
+export declare const PROFILES: Record<ProfileName, HyperMemConfig>;
+/**
+ * Load a named profile.
+ *
+ * @example
+ * const config = getProfile('light');
+ * const hm = createHyperMem(config);
+ */
+export declare function getProfile(name: ProfileName | 'extended'): HyperMemConfig;
+/**
+ * Merge a partial config on top of a named profile.
+ * Deep-merges compositor and indexer; top-level fields are replaced.
+ *
+ * @example
+ * const config = mergeProfile('light', {
+ *   cache: { keyPrefix: 'myapp:' },
+ *   compositor: { outputProfile: 'standard' },  // upgrade tier
+ * });
+ */
+export declare function mergeProfile(name: ProfileName | 'extended', overrides: DeepPartial<HyperMemConfig>): HyperMemConfig;
+type DeepPartial<T> = {
+    [K in keyof T]?: T[K] extends object ? DeepPartial<T[K]> : T[K];
+};
+export {};
+//# sourceMappingURL=profiles.d.ts.map

package/dist/profiles.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"profiles.d.ts","sourceRoot":"","sources":["../src/profiles.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAyE,MAAM,YAAY,CAAC;AAiExH,eAAO,MAAM,YAAY,EAAE,cAa1B,CAAC;AA6CF,eAAO,MAAM,eAAe,EAAE,cAO7B,CAAC;AA8CF,eAAO,MAAM,WAAW,EAAE,cAWzB,CAAC;AAMF,MAAM,MAAM,WAAW,GAAG,OAAO,GAAG,UAAU,GAAG,MAAM,CAAC;AAGxD,eAAO,MAAM,cAAc,gBAAe,CAAC;AAC3C,eAAO,MAAM,eAAe,gBAAc,CAAC;AAC3C,eAAO,MAAM,WAAW,gBAAc,CAAC;AAEvC,eAAO,MAAM,QAAQ,EAAE,MAAM,CAAC,WAAW,EAAE,cAAc,CAIxD,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,WAAW,GAAG,UAAU,GAAG,cAAc,CAIzE;AAED;;;;;;;;;GASG;AACH,wBAAgB,YAAY,CAC1B,IAAI,EAAE,WAAW,GAAG,UAAU,EAC9B,SAAS,EAAE,WAAW,CAAC,cAAc,CAAC,GACrC,cAAc,CAUhB;AAMD,KAAK,WAAW,CAAC,CAAC,IAAI;KACnB,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;CAChE,CAAC"}