hippo-memory 1.69.0 → 1.71.0

package/dist/store/goals.js

@@ -0,0 +1,260 @@
+// src/store/goals.ts
+import { randomUUID } from 'node:crypto';
+import { withWriteScope } from '../db/index.js';
+import { onHandle } from './open.js';
+const MAX_ACTIVE_GOAL_DEPTH = 3;
+export function rowToGoal(row) {
+    return {
+        id: row.id,
+        sessionId: row.session_id,
+        tenantId: row.tenant_id,
+        goalName: row.goal_name,
+        level: row.level,
+        parentGoalId: row.parent_goal_id ?? undefined,
+        status: row.status,
+        successCondition: row.success_condition ?? undefined,
+        retrievalPolicyId: row.retrieval_policy_id ?? undefined,
+        createdAt: row.created_at,
+        completedAt: row.completed_at ?? undefined,
+        outcomeScore: row.outcome_score ?? undefined,
+    };
+}
+export function pushGoal(hippoRoot, opts) {
+    return onHandle(hippoRoot, (db) => {
+        return pushGoalWithDb(db, opts);
+    });
+}
+/** Depth-cap enforcer for pushGoalWithDb and resumeGoal: suspends the oldest `overflow` goals; **caller MUST be inside a `BEGIN IMMEDIATE` transaction**.
+ * @internal Internal goal-stack invariant. Subject to change. */
+function enforceDepthCapWithinTx(db, tenantId, sessionId) {
+    // SAFETY: the row comes from the SELECT above, which projects exactly one
+    // column, `c`, as a COUNT(*).
+    const activeCount = db.prepare(`
+    SELECT COUNT(*) AS c
+    FROM goal_stack
+    WHERE tenant_id = ? AND session_id = ? AND status = 'active'
+  `).get(tenantId, sessionId).c;
+    if (activeCount >= MAX_ACTIVE_GOAL_DEPTH) {
+        const overflow = activeCount - MAX_ACTIVE_GOAL_DEPTH + 1;
+        db.prepare(`
+      UPDATE goal_stack
+      SET status = 'suspended'
+      WHERE id IN (
+        SELECT id FROM goal_stack
+        WHERE tenant_id = ? AND session_id = ? AND status = 'active'
+        ORDER BY created_at ASC
+        LIMIT ?
+      )
+    `).run(tenantId, sessionId, overflow);
+    }
+}
+export function pushGoalWithDb(db, opts) {
+    const id = `g_${randomUUID().replace(/-/g, '').slice(0, 16)}`;
+    const createdAt = new Date().toISOString();
+    let policyId = null;
+    withWriteScope(db, 'push_goal', () => {
+        // Depth cap: count active for (tenant, session); suspend oldest if at cap.
+        enforceDepthCapWithinTx(db, opts.tenantId, opts.sessionId);
+        assertParentInSession(db, opts);
+        // Parent goal_stack row first (FK target).
+        db.prepare(`
+      INSERT INTO goal_stack
+        (id, session_id, tenant_id, goal_name, level, parent_goal_id, status,
+         success_condition, retrieval_policy_id, created_at)
+      VALUES (?, ?, ?, ?, ?, ?, 'active', ?, NULL, ?)
+    `).run(id, opts.sessionId, opts.tenantId, opts.goalName, opts.level ?? 0, opts.parentGoalId ?? null, opts.successCondition ?? null, createdAt);
+        // Optional policy row, then point goal_stack.retrieval_policy_id at it.
+        if (opts.policy) {
+            policyId = insertRetrievalPolicy(db, id, opts.policy);
+        }
+    });
+    return {
+        id,
+        sessionId: opts.sessionId,
+        tenantId: opts.tenantId,
+        goalName: opts.goalName,
+        level: opts.level ?? 0,
+        parentGoalId: opts.parentGoalId,
+        status: 'active',
+        successCondition: opts.successCondition,
+        retrievalPolicyId: policyId ?? undefined,
+        createdAt,
+    };
+}
+function assertParentInSession(db, opts) {
+    if (!opts.parentGoalId)
+        return;
+    // SAFETY: the row comes from the SELECT above, which projects exactly
+    // the tenant_id and session_id columns of goal_stack.
+    const parent = db.prepare(`SELECT tenant_id, session_id FROM goal_stack WHERE id = ?`).get(opts.parentGoalId);
+    if (!parent) {
+        throw new Error(`parent goal not found: ${opts.parentGoalId}`);
+    }
+    if (parent.tenant_id !== opts.tenantId || parent.session_id !== opts.sessionId) {
+        throw new Error(`parent goal ${opts.parentGoalId} belongs to a different (tenant, session)`);
+    }
+}
+/** Insert the policy row, then point goal_stack.retrieval_policy_id at it; returns the new policy id. */
+function insertRetrievalPolicy(db, goalId, policy) {
+    const policyId = `rp_${randomUUID().replace(/-/g, '').slice(0, 16)}`;
+    db.prepare(`
+    INSERT INTO retrieval_policy
+      (id, goal_id, policy_type, weight_schema_fit, weight_recency, weight_outcome, error_priority)
+    VALUES (?, ?, ?, ?, ?, ?, ?)
+  `).run(policyId, goalId, policy.policyType, policy.weightSchemaFit ?? 1.0, policy.weightRecency ?? 1.0, policy.weightOutcome ?? 1.0, policy.errorPriority ?? 1.0);
+    db.prepare(`UPDATE goal_stack SET retrieval_policy_id = ? WHERE id = ?`).run(policyId, goalId);
+    return policyId;
+}
+export function getActiveGoals(hippoRoot, opts) {
+    return onHandle(hippoRoot, (db) => {
+        return getActiveGoalsWithDb(db, opts);
+    });
+}
+/** The active goals and their policies on one handle, so a goal and its policy never disagree. */
+export function activeGoalsWithPolicies(hippoRoot, opts) {
+    return onHandle(hippoRoot, (db) => {
+        const goals = getActiveGoalsWithDb(db, opts);
+        return { goals, policies: loadGoalPolicies(db, goals) };
+    });
+}
+/** Every goal of a (tenant, session), whatever its status, oldest first. */
+export function getSessionGoals(hippoRoot, opts) {
+    return onHandle(hippoRoot, (db) => {
+        // SAFETY: rows come from the SELECT below, which projects exactly GoalRow's columns.
+        const rows = db.prepare(`
+      SELECT id, session_id, tenant_id, goal_name, level, parent_goal_id, status,
+             success_condition, retrieval_policy_id, created_at, completed_at, outcome_score
+      FROM goal_stack
+      WHERE tenant_id = ? AND session_id = ?
+      ORDER BY created_at ASC
+    `).all(opts.tenantId, opts.sessionId);
+        return rows.map(rowToGoal);
+    });
+}
+function getActiveGoalsWithDb(db, opts) {
+    // SAFETY: rows come from the SELECT above, which projects exactly
+    // GoalRow's columns (in the same order goal_stack defines them).
+    const rows = db.prepare(`
+    SELECT id, session_id, tenant_id, goal_name, level, parent_goal_id, status,
+           success_condition, retrieval_policy_id, created_at, completed_at, outcome_score
+    FROM goal_stack
+    WHERE tenant_id = ? AND session_id = ? AND status = 'active'
+    ORDER BY created_at ASC
+  `).all(opts.tenantId, opts.sessionId);
+    return rows.map(rowToGoal);
+}
+// Per-goal retrieval_policy multipliers compose onto the base goal-tag boost and are hard-capped at MAX_FINAL_MULTIPLIER (3.0x) before scoring.
+function loadGoalPolicies(db, active) {
+    const policiesByGoalId = new Map();
+    for (const g of active) {
+        if (!g.retrievalPolicyId)
+            continue;
+        // SAFETY: the row comes from the SELECT above, which projects exactly
+        // these seven retrieval_policy columns.
+        const row = db.prepare(`
+      SELECT id, goal_id, policy_type, weight_schema_fit, weight_recency, weight_outcome, error_priority
+      FROM retrieval_policy WHERE id = ?
+    `).get(g.retrievalPolicyId);
+        if (row) {
+            policiesByGoalId.set(g.id, {
+                id: row.id,
+                goalId: row.goal_id,
+                policyType: row.policy_type,
+                weightSchemaFit: row.weight_schema_fit,
+                weightRecency: row.weight_recency,
+                weightOutcome: row.weight_outcome,
+                errorPriority: row.error_priority,
+            });
+        }
+    }
+    return policiesByGoalId;
+}
+/** The rows whose memory lives in this store: goal_recall_log.memory_id references memories, so a global row would fail
+ *  the insert. A global result keeps its boost; it only earns no log row, so no outcome propagation. */
+export function localGoalRecallRows(db, rows) {
+    if (rows.length === 0)
+        return [];
+    const ids = [...new Set(rows.map((r) => r.memoryId))];
+    // SAFETY: the SELECT projects exactly one column, `id`, from memories.
+    const local = db.prepare(`SELECT id FROM memories WHERE id IN (${ids.map(() => '?').join(',')})`).all(...ids);
+    const localIds = new Set(local.map((r) => r.id));
+    return rows.filter((r) => localIds.has(r.memoryId));
+}
+/** INSERT OR IGNORE: UNIQUE(memory_id, goal_id) makes a re-recall during the same goal life a no-op for outcome attribution. */
+export function writeGoalRecallLog(db, rows) {
+    if (rows.length === 0)
+        return;
+    const insertLog = db.prepare(`
+    INSERT OR IGNORE INTO goal_recall_log
+      (goal_id, memory_id, tenant_id, session_id, recalled_at, score)
+    VALUES (?, ?, ?, ?, ?, ?)
+  `);
+    for (const row of rows) {
+        insertLog.run(row.goalId, row.memoryId, row.tenantId, row.sessionId, row.recalledAt, row.score);
+    }
+}
+const POSITIVE_OUTCOME_THRESHOLD = 0.7;
+const NEGATIVE_OUTCOME_THRESHOLD = 0.3;
+const STRENGTH_BOOST = 1.10;
+const STRENGTH_DECAY = 0.85;
+export function completeGoal(hippoRoot, goalId, opts) {
+    return onHandle(hippoRoot, (db) => {
+        const completedAt = new Date().toISOString();
+        const score = opts.outcomeScore ?? null;
+        withWriteScope(db, 'complete_goal', () => {
+            // SAFETY: the row comes from the SELECT above, which projects exactly
+            // the created_at and status columns of goal_stack.
+            const goalRow = db.prepare(`SELECT created_at, status FROM goal_stack WHERE id = ?`).get(goalId);
+            if (!goalRow)
+                return;
+            if (goalRow.status === 'completed') {
+                // Already completed -- second call is a no-op for idempotency.
+                return;
+            }
+            db.prepare(`
+        UPDATE goal_stack
+        SET status = 'completed', completed_at = ?, outcome_score = ?
+        WHERE id = ?
+      `).run(completedAt, score, goalId);
+            if (score !== null && !opts.noPropagate) {
+                let multiplier = 1;
+                if (score >= POSITIVE_OUTCOME_THRESHOLD)
+                    multiplier = STRENGTH_BOOST;
+                else if (score < NEGATIVE_OUTCOME_THRESHOLD)
+                    multiplier = STRENGTH_DECAY;
+                if (multiplier !== 1) {
+                    // Lifespan window: only recalls during this goal's active life; UNIQUE(memory_id, goal_id) gives one adjustment per pair.
+                    db.prepare(`
+            UPDATE memories
+            SET strength = MIN(1.0, MAX(0.0, strength * ?))
+            WHERE id IN (
+              SELECT memory_id FROM goal_recall_log
+              WHERE goal_id = ?
+                AND recalled_at >= ?
+                AND recalled_at <= ?
+            )
+          `).run(multiplier, goalId, goalRow.created_at, completedAt);
+                }
+            }
+        });
+    });
+}
+export function suspendGoal(hippoRoot, goalId) {
+    onHandle(hippoRoot, (db) => {
+        db.prepare(`UPDATE goal_stack SET status = 'suspended' WHERE id = ? AND status = 'active'`).run(goalId);
+    });
+}
+export function resumeGoal(hippoRoot, goalId) {
+    return onHandle(hippoRoot, (db) => {
+        withWriteScope(db, 'resume_goal', () => {
+            // SAFETY: the row comes from the SELECT above, which projects exactly
+            // the session_id, tenant_id, and status columns of goal_stack.
+            const row = db.prepare(`SELECT session_id, tenant_id, status FROM goal_stack WHERE id = ?`).get(goalId);
+            if (!row || row.status !== 'suspended')
+                return;
+            enforceDepthCapWithinTx(db, row.tenant_id, row.session_id);
+            db.prepare(`UPDATE goal_stack SET status = 'active' WHERE id = ?`).run(goalId);
+        });
+    });
+}
+//# sourceMappingURL=goals.js.map

package/dist/store/graph-queue.d.ts

@@ -0,0 +1,13 @@
+/** Fail-soft producer hook: mark a tenant dirty for graph re-extraction by enqueuing its consolidated mirror memory. NEVER throws: a failed dirty signal
+ * must not abort an object write. Call POST-COMMIT from the graph-source save/close mutations; a null memoryId (forgotten mirror) is a no-op. */
+export declare function markGraphDirty(hippoRoot: string, tenantId: string, memoryId: string | null): void;
+/** Mark every pending item for a tenant with `id <= maxId` processed in one UPDATE (status/processed_at only, so the source guard trigger is not involved).
+ * The watermark excludes items enqueued after the drain snapshot. Returns the count marked. */
+export declare function markPendingProcessedUpTo(hippoRoot: string, tenantId: string, maxId: number): number;
+/** Dirty tenants awaiting graph re-extraction, each with the MAX pending queue id at read time (a watermark); host-wide, as sleep is cross-tenant.
+ * The drain marks only items at or below it processed, so items enqueued DURING the rebuild stay pending (no lost update). */
+export declare function loadPendingExtractionTenants(hippoRoot: string): {
+    tenantId: string;
+    maxPendingId: number;
+}[];
+//# sourceMappingURL=graph-queue.d.ts.map

package/dist/store/graph-queue.js

@@ -0,0 +1,70 @@
+// The graph extraction queue: the consolidated memories that wait for a graph rebuild, and the watermark the sleep drain marks them by.
+import { onHandle } from './open.js';
+import { assertTenantId } from './tenant.js';
+import { errorMessage, log } from '../util/log.js';
+import { rowToQueueItem, QUEUE_COLS } from './graph-rows.js';
+import { resolveConsolidatedSource } from './graph-writes.js';
+/** Enqueue a consolidated memory for later graph extraction; rejects a raw / missing / cross-tenant memory (the DB trigger is the backstop). */
+function enqueueExtraction(hippoRoot, tenantId, memoryId) {
+    assertTenantId('enqueueExtraction', tenantId);
+    const now = new Date().toISOString();
+    return onHandle(hippoRoot, (db) => {
+        // enqueue is memory-keyed (no source object), so a missing memory still throws (correct -
+        // you cannot enqueue a forgotten mirror); memoryId is non-null here by construction.
+        const { sourceKind } = resolveConsolidatedSource(db, tenantId, memoryId, null, 'enqueueExtraction');
+        const result = db.prepare(`
+      INSERT INTO graph_extraction_queue(tenant_id, memory_id, kind, status, enqueued_at, processed_at)
+      VALUES (?, ?, ?, 'pending', ?, NULL)
+    `).run(tenantId, memoryId, sourceKind, now);
+        const id = Number(result.lastInsertRowid ?? 0);
+        // SAFETY: row's shape matches the columns named in QUEUE_COLS above.
+        const row = db.prepare(`SELECT ${QUEUE_COLS} FROM graph_extraction_queue WHERE id = ?`).get(id);
+        if (!row)
+            throw new Error('enqueueExtraction: failed to reload queue item');
+        return rowToQueueItem(row);
+    });
+}
+/** Fail-soft producer hook: mark a tenant dirty for graph re-extraction by enqueuing its consolidated mirror memory. NEVER throws: a failed dirty signal
+ * must not abort an object write. Call POST-COMMIT from the graph-source save/close mutations; a null memoryId (forgotten mirror) is a no-op. */
+export function markGraphDirty(hippoRoot, tenantId, memoryId) {
+    if (!memoryId)
+        return;
+    try {
+        enqueueExtraction(hippoRoot, tenantId, memoryId);
+    }
+    catch (err) {
+        // Logged (warn) so a SYSTEMATIC enqueue failure surfaces to operators, but
+        // swallowed so the already-committed object write is never rolled back.
+        log.warn(`markGraphDirty: enqueue failed for tenant=${tenantId} memory=${memoryId}: ${errorMessage(err)}`);
+    }
+}
+/** Mark every pending item for a tenant with `id <= maxId` processed in one UPDATE (status/processed_at only, so the source guard trigger is not involved).
+ * The watermark excludes items enqueued after the drain snapshot. Returns the count marked. */
+export function markPendingProcessedUpTo(hippoRoot, tenantId, maxId) {
+    assertTenantId('markPendingProcessedUpTo', tenantId);
+    const now = new Date().toISOString();
+    return onHandle(hippoRoot, (db) => {
+        const res = db.prepare(`
+      UPDATE graph_extraction_queue
+      SET status = 'processed', processed_at = ?
+      WHERE tenant_id = ? AND status = 'pending' AND id <= ?
+    `).run(now, tenantId, maxId);
+        return Number(res.changes ?? 0);
+    });
+}
+/** Dirty tenants awaiting graph re-extraction, each with the MAX pending queue id at read time (a watermark); host-wide, as sleep is cross-tenant.
+ * The drain marks only items at or below it processed, so items enqueued DURING the rebuild stay pending (no lost update). */
+export function loadPendingExtractionTenants(hippoRoot) {
+    return onHandle(hippoRoot, (db) => {
+        // SAFETY: rows' shape matches the two aliased columns (tenant_id, max_id)
+        // named in the SELECT above.
+        const rows = db.prepare(`
+      SELECT tenant_id AS tenant_id, MAX(id) AS max_id
+      FROM graph_extraction_queue
+      WHERE status = 'pending'
+      GROUP BY tenant_id
+    `).all();
+        return rows.map((r) => ({ tenantId: r.tenant_id, maxPendingId: Number(r.max_id) }));
+    });
+}
+//# sourceMappingURL=graph-queue.js.map

package/dist/store/graph-reads.d.ts

@@ -0,0 +1,41 @@
+import { type DatabaseSyncLike } from '../db/index.js';
+import { type EntityType, type Entity, type Relation, type StoredGraph } from './graph-rows.js';
+/** Entities with an exact `name`, bounded by `limit` in SQL with a deterministic order, so graph-view focus finds `--entity NAME` directly
+ * without materializing every same-name row. */
+export declare function loadEntitiesByName(hippoRoot: string, tenantId: string, name: string, opts?: {
+    limit?: number;
+}, txDb?: DatabaseSyncLike): Entity[];
+export declare function loadEntities(hippoRoot: string, tenantId: string, opts?: {
+    entityType?: EntityType;
+    limit?: number;
+}, txDb?: DatabaseSyncLike): Entity[];
+export declare function loadRelations(hippoRoot: string, tenantId: string, opts?: {
+    fromEntityId?: number;
+    limit?: number;
+}, txDb?: DatabaseSyncLike): Relation[];
+/** Chunk size for IN-list queries: well under SQLite's 999-bound-variable default
+ *  (leaves headroom for the tenant_id param + the doubled list in neighbour lookups). */
+export declare const IN_LIST_CHUNK = 400;
+/** Load entities by primary id (the BFS-reached rows whose `memory_id` maps back to a recall result). Tenant-scoped, read-only. */
+export declare function loadEntitiesByIds(hippoRoot: string, tenantId: string, ids: number[], txDb?: DatabaseSyncLike): Entity[];
+/** All relations touching ANY of `entityIds` in EITHER direction: the per-hop neighbour query for multi-hop traversal, one query for the whole frontier
+ * (no N+1). `limit` caps rows and must be a non-negative integer (raw `LIMIT ?` rejects a fraction). */
+export declare function loadNeighborRelations(hippoRoot: string, tenantId: string, entityIds: number[], opts?: {
+    limit?: number;
+}, txDb?: DatabaseSyncLike): Relation[];
+/** Relations with BOTH endpoints in `entityIds` (edges AMONG the set), for the graph-view focus subgraph; `LIMIT` caps only intra-union edges,
+ * so no out-of-union row evicts a valid one. The caller bounds `entityIds` (<= view limit), so a single query is safe. */
+export declare function loadRelationsAmong(hippoRoot: string, tenantId: string, entityIds: number[], opts?: {
+    limit?: number;
+}, txDb?: DatabaseSyncLike): Relation[];
+/** Map consolidated source memory ids to their graph entities: the SEED step of multi-hop recall. Tenant-scoped, read-only; chunks the IN-list under the
+ * SQLite variable cap. */
+export declare function loadEntitiesByMemoryId(hippoRoot: string, tenantId: string, memoryIds: string[]): Entity[];
+/** The stored rows a rebuild diffs against, read on the caller's handle so the diff and the apply can share one write lock. */
+export declare function storedGraphOn(db: DatabaseSyncLike, tenantId: string, memoryIds: readonly string[]): StoredGraph;
+/** storedGraphOn on its own connection, outside any write lock. */
+export declare function loadStoredGraph(hippoRoot: string, tenantId: string, memoryIds: readonly string[]): StoredGraph;
+/** Run `fn` in ONE read transaction (a single WAL snapshot) so every `load*` read via `txDb` sees a consistent view
+ * even if a rebuild commits concurrently (it clears and reinserts entities, so separate reads could mix old entity ids with new relation ids). */
+export declare function withGraphReadSnapshot<T>(hippoRoot: string, fn: (txDb: DatabaseSyncLike) => T): T;
+//# sourceMappingURL=graph-reads.d.ts.map