hippo-memory 1.70.0 → 1.71.0

package/dist/sharing/shared.js

@@ -1,563 +0,0 @@
-/**
- * Cross-agent shared memory for Hippo.
- * Global store is shared across all projects.
- * Resolution: $HIPPO_HOME > $XDG_DATA_HOME/hippo > ~/.hippo/
- * Local .hippo/ stores are per-project.
- */
-import { BadRequestError, NotFoundError } from '../core/api-errors.js';
-import { compareScoresDesc } from '../core/compare.js';
-import * as fs from 'fs';
-import * as path from 'path';
-import { generateId, COMPACTION_MEMORY_TAG } from '../core/memory.js';
-import { AGENT_MEMORY_SOURCE_PREFIX, AGENT_MEMORY_TAGS } from '../core/agent-memory-tools.js';
-import { initStore } from '../store/open.js';
-import { writeEntry } from '../store/entry-writes.js';
-import { loadAllEntries, readEntry } from '../store/entry-reads.js';
-import { loadSearchEntries, loadRecallSearchEntries, recallScopeFilter } from '../store/search-rows.js';
-import { tallySources } from '../store/candidates.js';
-import { isPersonalScope, passesScopeFilterForRecall, passesCliRecallScopeFilter } from '../store/recall-scope.js';
-import { search } from '../search/bm25-search.js';
-import { hybridSearch } from '../search/hybrid.js';
-import { fitBudget } from '../search/finalize.js';
-import { DEFAULT_LOCAL_BUMP, DEFAULT_RECALL_BUDGET } from '../core/search-types.js';
-import { evalNow } from '../core/ablation.js';
-import { fallbackOrigin, classifyOriginProject, resolveGlobalRootDir, resolveProjectIdentity } from '../core/project-identity.js';
-import { isSharedStore } from '../core/config.js';
-import { detectSecret } from '../util/secret-detect.js';
-import { isQuarantineScope } from '../trust/quarantine.js';
-import { RejectedValueError } from '../store/rejection.js';
-import { embedMemory, embedAll } from '../store/embeddings/index.js';
-import { duplicateKey, storedTextKeys } from '../util/same-text.js';
-import { isReusable } from '../core/memory-quality.js';
-import { errorMessage, log } from '../util/log.js';
-import { appendAuditEvent } from '../store/audit.js';
-// The rows are already copied; a failed background embed only delays vectors, so it warns instead of throwing.
-function logEmbedAllFailure(caller, err) {
-    log.warn(`${caller}: background embed failed (${errorMessage(err)}); run 'hippo embed' to backfill`);
-}
-/**
- * Returns the path to the global Hippo store.
- * Resolution order: $HIPPO_HOME > $XDG_DATA_HOME/hippo > ~/.hippo/
- */
-export function getGlobalRoot() {
-    // Single source of truth lives in project-identity.ts (leaf) so db.ts
-    // migrations can resolve the same path without a shared.ts import cycle.
-    return resolveGlobalRootDir();
-}
-/**
- * Ensure the global store exists.
- */
-export function initGlobal() {
-    const globalRoot = getGlobalRoot();
-    if (!fs.existsSync(globalRoot)) {
-        initStore(globalRoot);
-    }
-    else {
-        // Ensure subdirectories exist in case partially initialized
-        initStore(globalRoot);
-    }
-}
-/** The copy's write hook: the `promote` row when asked for, then the caller's own hook, both in the copy's transaction. */
-function promoteHook(sourceId, auditAs, afterWrite) {
-    if (!auditAs)
-        return afterWrite;
-    return (db, globalId) => {
-        appendAuditEvent(db, { tenantId: auditAs.tenantId, actor: auditAs.actor, op: 'promote', targetId: globalId, metadata: { sourceId } });
-        afterWrite?.(db, globalId);
-    };
-}
-/** Copies a local memory to the global store and returns the copy, which gets a new `g_` id so it cannot collide. */
-export function promoteToGlobal(localRoot, id, opts) {
-    const entry = readEntry(localRoot, id, opts?.tenantId);
-    if (!entry)
-        throw new NotFoundError(`Memory not found: ${id}`);
-    // Same quarantine veto as shareMemory; a promoted copy would have no quarantine record to review.
-    if (isQuarantineScope(entry.scope)) {
-        throw new BadRequestError(`Refusing to promote ${id}: it is quarantined pending review. Approve it first via 'hippo quarantine approve ${id}'.`);
-    }
-    if (isPersonalScope(entry.scope ?? null)) {
-        throw new BadRequestError(`Refusing to promote ${id}: it is a personal memory and stays with its owner on this server.`);
-    }
-    // Secret producer veto: promote is a producer path to the global store
-    // exactly like shareMemory - same hard rule.
-    const promoteSecret = detectSecret(entry);
-    if (promoteSecret.flagged) {
-        throw new BadRequestError(`Refusing to promote ${id} to the global store: content matches secret material (${promoteSecret.reason}). ` +
-            `Secrets stay in their owning project's store.`);
-    }
-    initGlobal();
-    const globalRoot = getGlobalRoot();
-    // A project store's NULL row gets its folder back; a shared store's folder is no caller's project, so NULL stays and the label names no path.
-    const globalEntry = {
-        ...entry,
-        id: generateId('g'),
-        source: isSharedStore(localRoot) ? `shared::${new Date().toISOString()}` : `promoted:${localRoot}`,
-        origin_project: entry.origin_project ?? fallbackOrigin(localRoot),
-    };
-    writeEntry(globalRoot, globalEntry, { actor: opts?.actor, afterWrite: promoteHook(id, opts?.auditAs, opts?.afterWrite) });
-    // Fire-and-forget: embedMemory gates on availability and never rejects.
-    void embedMemory(globalRoot, globalEntry);
-    return globalEntry;
-}
-/**
- * Search across both local and global stores, merging results.
- * Local results are boosted by 1.2x to prefer project-specific context.
- * Returns results sorted by adjusted score, within combined token budget.
- */
-export function searchBoth(query, localRoot, globalRoot, options = {}) {
-    const { budget = DEFAULT_RECALL_BUDGET, now = evalNow(), minResults, tenantId } = options;
-    const effectiveMin = minResults ?? 1;
-    const localEntries = fs.existsSync(localRoot) ? loadSearchEntries(localRoot, query, undefined, tenantId) : [];
-    const globalEntries = fs.existsSync(globalRoot) ? loadSearchEntries(globalRoot, query, undefined, tenantId) : [];
-    if (localEntries.length === 0 && globalEntries.length === 0)
-        return [];
-    // Search each store with full budget, then blend
-    const localResults = search(query, localEntries, { budget, now, minResults });
-    const globalResults = search(query, globalEntries, { budget, now, minResults });
-    const deduped = dedupeByContent(tagLocalAndGlobal(localResults, globalResults));
-    // PLAIN stable score sort on purpose: both inputs are deterministically ordered,
-    // and an exact tie keeps the LOCAL result ahead of the global one (concat order).
-    deduped.sort((a, b) => compareScoresDesc(a.score, b.score));
-    // Apply combined token budget (guarantee at least minResults items)
-    const results = [];
-    let usedTokens = 0;
-    for (let i = 0; i < deduped.length; i++) {
-        if (results.length >= effectiveMin && usedTokens + deduped[i].tokens > budget)
-            continue;
-        usedTokens += deduped[i].tokens;
-        results.push(deduped[i]);
-    }
-    return results;
-}
-function tagLocalAndGlobal(localResults, globalResults) {
-    // Tag global results. Local memories get a configurable priority bump.
-    return [
-        ...localResults.map((r) => ({
-            ...r,
-            isGlobal: false,
-            score: r.score * DEFAULT_LOCAL_BUMP,
-            breakdown: r.breakdown
-                ? { ...r.breakdown, sourceBump: DEFAULT_LOCAL_BUMP, final: r.breakdown.final * DEFAULT_LOCAL_BUMP }
-                : undefined,
-        })),
-        ...globalResults.map((r) => ({ ...r, isGlobal: true })),
-    ];
-}
-function dedupeByContent(tagged) {
-    // Remove duplicates by content (local/global IDs differ after promote/share)
-    const seen = new Set();
-    return tagged.filter((r) => {
-        const key = duplicateKey(r.entry.content);
-        if (seen.has(key))
-            return false;
-        seen.add(key);
-        return true;
-    });
-}
-/**
- * Hybrid search across both local and global stores, using embeddings when available.
- * Async version of searchBoth that calls hybridSearch instead of search.
- */
-export async function searchBothHybrid(query, localRoot, globalRoot, options = {}) {
-    const { includeSuperseded, asOf, tenantId, entryFilter, recallScope } = options;
-    // When an admission filter is active, lift the per-store candidate cap
-    // (default 200): excluded rows matching the query could otherwise fill the
-    // window before any admitted row is even loaded. 5000 is bounded so a common
-    // term on a large store cannot stall an interactive call by ranking every match.
-    const searchWindow = entryFilter ? 5000 : undefined;
-    // Recall mode: push the scope predicate into SQL exactly like
-    // api.recall (loadRecallSearchEntries), so quarantine/private rows never
-    // enter the candidate set, never shadow admitted duplicates in the dedupe
-    // pass, and never consume budget. The JS post-filter below is the
-    // regex-only `<source>:private:*` half plus defense-in-depth on exact
-    // match, mirroring api.ts's recall load.
-    const loadEntries = (root) => {
-        if (!fs.existsSync(root))
-            return [];
-        return recallScope
-            ? loadRecallSearchEntries(root, query, searchWindow, tenantId, recallScope.requested, recallScope.additive ? 'additive' : 'exact', Boolean(includeSuperseded) || Boolean(asOf), undefined, recallScope.ownScope)
-            : loadSearchEntries(root, query, searchWindow, tenantId);
-    };
-    const passesScope = (e) => !recallScope || (recallScope.additive
-        ? passesCliRecallScopeFilter(e.scope ?? null, recallScope.requested) || passesScopeFilterForRecall(e.scope ?? null, undefined, recallScope.ownScope)
-        : passesScopeFilterForRecall(e.scope ?? null, recallScope.requested, recallScope.ownScope));
-    const admit = (e) => passesScope(e) && (!entryFilter || entryFilter(e));
-    const localEntries = loadEntries(localRoot).filter(admit);
-    const globalEntries = loadEntries(globalRoot).filter(admit);
-    // The vector arm loads under the same SQL rules as loadEntries, then the same JS admission.
-    const vectorCandidates = {
-        tenantId,
-        scope: recallScope ? recallScopeFilter(recallScope.requested, recallScope.additive ? 'additive' : 'exact', recallScope.ownScope) : undefined,
-        includeSuperseded: !recallScope || Boolean(includeSuperseded) || Boolean(asOf),
-        admit,
-    };
-    return rankBothStores(query, { local: localRoot, global: globalRoot }, { local: localEntries, global: globalEntries }, vectorCandidates, options);
-}
-/** Hybrid ranking of rows already loaded from each store: the local bump, one copy per text, then the shared budget. */
-export async function rankBothStores(query, roots, entries, vectorCandidates, options = {}) {
-    const { budget = DEFAULT_RECALL_BUDGET, now = evalNow(), embeddingWeight, explain, mmr, mmrLambda, localBump = DEFAULT_LOCAL_BUMP, minResults, cost, scope, includeSuperseded, asOf, summaryDeboost, summaryFreshness } = options;
-    if (entries.local.length === 0 && entries.global.length === 0)
-        return [];
-    const shared = { budget, now, embeddingWeight, explain, mmr, mmrLambda, minResults, cost, scope, includeSuperseded, asOf, summaryDeboost, summaryFreshness, vectorCandidates };
-    const localResults = await hybridSearch(query, entries.local, { ...shared, hippoRoot: roots.local });
-    const globalResults = await hybridSearch(query, entries.global, { ...shared, hippoRoot: roots.global });
-    // Tag global results. Local memories get a configurable priority bump.
-    const tagged = [
-        ...localResults.map((r) => ({
-            ...r,
-            isGlobal: false,
-            score: r.score * localBump,
-            breakdown: r.breakdown
-                ? { ...r.breakdown, sourceBump: localBump, final: r.breakdown.final * localBump }
-                : undefined,
-        })),
-        ...globalResults.map((r) => ({ ...r, isGlobal: true })),
-    ];
-    // Remove duplicates by content (local/global IDs differ after promote/share)
-    const seen = new Set();
-    const deduped = tagged.filter((r) => {
-        const key = duplicateKey(r.entry.content);
-        if (seen.has(key))
-            return false;
-        seen.add(key);
-        return true;
-    });
-    // PLAIN stable score sort on purpose -- see searchBoth above;
-    // same rationale (deterministic inputs + stability; local-first on ties).
-    deduped.sort((a, b) => compareScoresDesc(a.score, b.score));
-    return fitBudget(deduped, budget, minResults ?? 1, cost);
-}
-// ---------------------------------------------------------------------------
-// Multi-agent shared memory
-// ---------------------------------------------------------------------------
-/** Tags that indicate project-specific memories (poor transfer candidates) */
-const PROJECT_SPECIFIC_TAGS = new Set([
-    'file-path', 'config', 'deploy', 'cron', 'url', 'auth',
-    'field-names', 'column-names', 'api-key', 'endpoint',
-]);
-/** Tags that indicate transferable memories (good transfer candidates) */
-const TRANSFERABLE_TAGS = new Set([
-    'error', 'platform', 'windows', 'encoding', 'python', 'shell',
-    'powershell', 'quant', 'backtest', 'pattern', 'rule', 'gotcha',
-    'sub-agent', 'review', 'best-practice',
-]);
-/** Tags whose rows only a hand-run share or promote may copy to the global store; derived rows inherit them. */
-export const NEVER_AUTO_SHARE_TAGS = new Set([
-    'git-learned',
-    'session-digest',
-    ...AGENT_MEMORY_TAGS,
-]);
-export function neverAutoShareTags(sources) {
-    return [...NEVER_AUTO_SHARE_TAGS].filter((tag) => sources.some((s) => s.tags.includes(tag)));
-}
-/** Tags whose rows sleep keeps as written: never merged, never sent to LLM extraction. Conflict detection keeps its own list. */
-export const NO_MERGE_TAGS = new Set([
-    'extracted',
-    'session-digest',
-    COMPACTION_MEMORY_TAG,
-    ...AGENT_MEMORY_TAGS,
-]);
-/**
- * Estimate how well a memory would transfer to other projects.
- * Returns 0..1 where >0.5 = good candidate for sharing.
- */
-export function transferScore(entry) {
-    let score = 0.5; // neutral default
-    // Boost for transferable tags
-    const transferableCount = entry.tags.filter((t) => TRANSFERABLE_TAGS.has(t)).length;
-    score += transferableCount * 0.1;
-    // Penalize for project-specific tags
-    const specificCount = entry.tags.filter((t) => PROJECT_SPECIFIC_TAGS.has(t)).length;
-    score -= specificCount * 0.15;
-    // High-retrieval memories are more likely to be universally useful
-    if (entry.retrieval_count >= 3)
-        score += 0.1;
-    // Pinned memories are important to their owner
-    if (entry.pinned)
-        score += 0.1;
-    // Error-tagged memories often encode universal lessons
-    if (entry.emotional_valence === 'negative' || entry.emotional_valence === 'critical')
-        score += 0.05;
-    return Math.min(1, Math.max(0, score));
-}
-/**
- * Share a memory to the global store with attribution.
- * Enriches the source field with project path and timestamp.
- * Returns the new global entry, or null if transfer score is too low.
- */
-export function shareMemory(localRoot, id, options = {}) {
-    // tenantId is OPTIONAL for backward compat — autoShare's internal call at
-    // :370 passes only `{ force: true }` in a single-tenant context. MCP/REST
-    // hosts MUST pass tenantId so a Bearer for tenant A cannot share tenant
-    // B's memory to global. readEntry returns null on cross-tenant lookups
-    // when tenantId is provided.
-    const entry = readEntry(localRoot, id, options.tenantId);
-    if (!entry)
-        throw new NotFoundError(`Memory not found: ${id}`);
-    assertShareable(entry, id);
-    const score = transferScore(entry);
-    if (score < 0.3 && !options.force)
-        return null;
-    initGlobal();
-    const globalRoot = getGlobalRoot();
-    // The label keeps the folder name for a user-global row; a NULL row gets `shared::`, which originFromSource reads as no project.
-    const fallbackName = path.basename(path.resolve(localRoot, '..'));
-    const originName = entry.origin_project ?? fallbackOrigin(localRoot);
-    const label = originName === '' ? fallbackName : (originName ?? '');
-    const globalEntry = {
-        ...entry,
-        id: generateId('g'),
-        source: `shared:${label}:${new Date().toISOString()}`,
-        origin_project: originName,
-    };
-    writeEntry(globalRoot, globalEntry);
-    // Single-row producer: embed here unless the caller opts out. autoShare
-    // sets skipEmbed so it can batch its whole run through one embedAll() at
-    // the end instead of N serialized full-index rewrites (embedMemory rewrites
-    // the whole index JSON per call).
-    if (!options.skipEmbed) {
-        void embedMemory(globalRoot, globalEntry);
-    }
-    return globalEntry;
-}
-function assertShareable(entry, id) {
-    // Secret producer veto: secrets never go to the global store, not even
-    // with --force. Explicit and loud - a silent null would read as "low
-    // transfer score" and invite retries.
-    const secret = detectSecret(entry);
-    if (secret.flagged) {
-        throw new BadRequestError(`Refusing to share ${id} to the global store: content matches secret material (${secret.reason}). ` +
-            `Secrets stay in their owning project's store.`);
-    }
-    // A quarantined row is unreviewed input, not a lesson; sharing it would spread poison globally.
-    if (isQuarantineScope(entry.scope)) {
-        throw new BadRequestError(`Refusing to share ${id}: it is quarantined pending review. Approve it first via 'hippo quarantine approve ${id}'.`);
-    }
-    if (isPersonalScope(entry.scope ?? null)) {
-        throw new BadRequestError(`Refusing to share ${id}: it is a personal memory and stays with its owner on this server.`);
-    }
-}
-/**
- * List all projects that have contributed memories to the global store.
- * Parses the source field for 'shared:<project>:' or 'promoted:<path>' patterns.
- *
- * `tenantId` is optional. When provided, the global entries
- * are filtered to that tenant before aggregation — matches every other
- * read path's default-safe behaviour. When undefined, host-wide (back-compat
- * for legacy callers like CLI standalone + dashboard internal use). Operators
- * who genuinely want cross-tenant peer discovery can pass undefined or
- * use direct SQL.
- */
-export function listPeers(globalRoot, tenantId) {
-    const root = globalRoot ?? getGlobalRoot();
-    if (!fs.existsSync(root))
-        return [];
-    // Tenant-scoped by default when tenantId provided. Host-wide when
-    // undefined (preserves back-compat).
-    const tallies = tallySources(root, tenantId).sort((a, b) => (a.first < b.first ? -1 : a.first > b.first ? 1 : 0));
-    const peerMap = new Map();
-    for (const tally of tallies) {
-        let project = 'unknown';
-        if (tally.source.startsWith('shared:')) {
-            const parts = tally.source.split(':');
-            project = parts[1] || 'unknown';
-        }
-        else if (tally.source.startsWith('promoted:')) {
-            const promotedPath = tally.source.slice('promoted:'.length);
-            project = path.basename(path.resolve(promotedPath, '..'));
-        }
-        else if (tally.source === 'cli-global') {
-            project = 'global-cli';
-        }
-        const existing = peerMap.get(project);
-        if (!existing) {
-            peerMap.set(project, { count: tally.count, latest: tally.latest });
-        }
-        else {
-            existing.count += tally.count;
-            if (tally.latest > existing.latest)
-                existing.latest = tally.latest;
-        }
-    }
-    return Array.from(peerMap.entries())
-        .map(([project, data]) => ({ project, ...data }))
-        .sort((a, b) => b.count - a.count);
-}
-function isAutoShareCandidate(entry, globalContentSet, minScore, stats) {
-    // shareMemory refuses quarantined and personal rows; filtering here keeps sleep from aborting on one.
-    if (isQuarantineScope(entry.scope ?? null) || isPersonalScope(entry.scope ?? null) || !isReusable(entry))
-        return false;
-    // Before the score: these rows describe one project only, and a git seed's 'error' tag clears the bar.
-    if (entry.tags.some((t) => NEVER_AUTO_SHARE_TAGS.has(t)) || entry.source.startsWith(AGENT_MEMORY_SOURCE_PREFIX)) {
-        if (stats)
-            stats.neverAutoShareSkipped = (stats.neverAutoShareSkipped ?? 0) + 1;
-        return false;
-    }
-    const score = transferScore(entry);
-    if (score < minScore)
-        return false;
-    // Skip if already shared (same text apart from spacing)
-    if (globalContentSet.has(duplicateKey(entry.content)))
-        return false;
-    // Secret producer veto: secret rows never auto-share, regardless of
-    // transfer score. (shareMemory would throw; filtering here keeps the
-    // sleep pipeline fail-safe.) Checked LAST so the stats counter
-    // only counts rows the veto actually withheld — a row failing the score
-    // or dedupe gates was never going to share, secret or not.
-    if (detectSecret(entry).flagged) {
-        if (stats)
-            stats.secretSkipped++;
-        return false;
-    }
-    return true;
-}
-// Rejection containment (sync/promote/share copy paths must not let ONE rejected
-// candidate kill the batch): shareMemory -> writeEntry hits the LIVE guard
-// against the GLOBAL store's tombstones. A matching candidate throws
-// RejectedValueError, which (uncaught) would abort this whole loop and,
-// via api.ts's sleep pipeline, the entire autoShare sleep phase. Mirrors
-// syncGlobalToLocal's per-item catch just above in this file. writeEntry's
-// own catch already writes the reject_refusal audit before rethrowing,
-// so do not double-audit here, just count and continue.
-function shareCandidates(localRoot, candidates, stats) {
-    const shared = [];
-    let rejectedSkipped = 0;
-    for (const entry of candidates) {
-        try {
-            // skipEmbed: batching invariant, this is a batch producer, so it embeds
-            // once via embedAll() below rather than once per row inside shareMemory.
-            const result = shareMemory(localRoot, entry.id, { force: true, skipEmbed: true });
-            if (result)
-                shared.push(result);
-        }
-        catch (err) {
-            if (err instanceof RejectedValueError) {
-                rejectedSkipped++;
-                if (stats)
-                    stats.rejectedSkipped = (stats.rejectedSkipped ?? 0) + 1;
-                continue;
-            }
-            throw err;
-        }
-    }
-    if (rejectedSkipped > 0) {
-        log.warn(`autoShare: skipped ${rejectedSkipped} candidate(s) refused by the global store's rejection tombstone`);
-    }
-    return shared;
-}
-/**
- * Auto-share: local memories with high transfer scores, not already global, no NEVER_AUTO_SHARE_TAGS tag.
- * Returns the list of shared entries.
- *
- * `options.tenantId` is opt-in. When provided, the LOCAL-entries read is
- * scoped to that tenant. When undefined, the local read is host-wide (current
- * behaviour). The GLOBAL-entries read is always unioned — the global root IS
- * the cross-tenant aggregate by design. `api.sleep` passes no tenantId
- * because `sleep` is host-wide by intent.
- *
- * `options.stats` is an opt-in out-param. When provided,
- * `stats.secretSkipped` is incremented once per row that passed every OTHER
- * admission gate (transfer score, not-already-global) and was withheld SOLELY
- * by the secret veto — i.e. it counts shares actually prevented, not secret
- * rows merely present. Filled identically under `dryRun`.
- *
- * `stats.rejectedSkipped` (optional) is incremented once per candidate
- * refused by the GLOBAL store's rejection tombstone (RejectedValueError from
- * shareMemory -> writeEntry). Unlike secretSkipped, this can only be
- * detected by attempting the write — `dryRun` returns candidates before the
- * write loop runs, so `rejectedSkipped` stays at its initial value under
- * `dryRun` (candidates that WOULD be refused are not distinguished in the
- * dry-run preview).
- */
-export function autoShare(localRoot, options = {}) {
-    const { minScore = 0.6, dryRun = false } = options;
-    const localEntries = loadAllEntries(localRoot, options.tenantId);
-    initGlobal();
-    const globalRoot = getGlobalRoot();
-    // Host-wide read. The global store IS the union across all tenants;
-    // per-tenant filtering on the global root would defeat the purpose.
-    const globalEntries = loadAllEntries(globalRoot);
-    // Build set of global content hashes to avoid duplicates
-    const globalContentSet = storedTextKeys(globalEntries);
-    const candidates = localEntries.filter((entry) => isAutoShareCandidate(entry, globalContentSet, minScore, options.stats));
-    if (dryRun)
-        return candidates;
-    const shared = shareCandidates(localRoot, candidates, options.stats);
-    if (shared.length > 0) {
-        void embedAll(globalRoot).catch((err) => logEmbedAllFailure('autoShare', err));
-    }
-    return shared;
-}
-/**
- * Copy all global memories into the local store.
- * Skips entries that already exist locally, by ID or by text (promote and share copy under a new ID).
- * Returns the count of newly copied entries.
- */
-export function syncGlobalToLocal(localRoot, globalRoot, opts = {}) {
-    if (isSharedStore(localRoot)) {
-        throw new BadRequestError(`Refusing to sync into ${localRoot}: a shared store takes no copies of a personal global store, whose rows would reach every member.`);
-    }
-    if (!fs.existsSync(globalRoot))
-        return 0;
-    // Host-wide read. syncGlobalToLocal copies the global union into a
-    // tenant-scoped local store; writeEntry on each row carries the tenant if
-    // the local-root context provides one.
-    const globalEntries = loadAllEntries(globalRoot);
-    const textKey = (e) => `${e.tenantId}\n${e.content}`;
-    const localEntries = loadAllEntries(localRoot);
-    const localIds = new Set(localEntries.map((e) => e.id));
-    const localText = new Set(localEntries.map(textKey));
-    // Syncing down must not re-import what ambient context
-    // excludes - other-project rows are skipped by default and secret rows
-    // are never copied. origin_project is preserved on the copy (writeEntry
-    // only stamps when the field is missing).
-    const currentProject = resolveProjectIdentity(path.dirname(path.resolve(localRoot)));
-    let count = 0;
-    // A locally rejected value must not come back through sync down: caught per item, printed as one line.
-    let rejected = 0;
-    for (const entry of globalEntries) {
-        // Skip if already present by ID
-        if (localIds.has(entry.id))
-            continue;
-        // Only the global store's user pass sets an imported note's row aside, so a copy would outlive the note.
-        if (entry.source.startsWith(AGENT_MEMORY_SOURCE_PREFIX))
-            continue;
-        if (localText.has(textKey(entry)))
-            continue;
-        if (detectSecret(entry).flagged)
-            continue;
-        if (!opts.includeCrossProject && classifyOriginProject(entry.origin_project, currentProject) === 'cross-project')
-            continue;
-        if (!writeUnlessRejected(localRoot, entry)) {
-            rejected++;
-            continue;
-        }
-        localText.add(textKey(entry));
-        count++;
-    }
-    finishSyncDown(localRoot, count, rejected);
-    return count;
-}
-/** False when the local store rejects the value; any other failure still throws. */
-function writeUnlessRejected(localRoot, entry) {
-    try {
-        writeEntry(localRoot, entry);
-        return true;
-    }
-    catch (err) {
-        if (err instanceof RejectedValueError)
-            return false;
-        throw err;
-    }
-}
-function finishSyncDown(localRoot, count, rejected) {
-    if (rejected > 0) {
-        log.warn(`syncGlobalToLocal: skipped ${rejected} rejected value(s) (run \`hippo unreject\` on the local store to allow).`);
-    }
-    // Batch producer: one embedAll() on the destination rather than an
-    // embedMemory() per copied row (same batching invariant as autoShare).
-    if (count > 0) {
-        void embedAll(localRoot).catch((err) => logEmbedAllFailure('syncGlobalToLocal', err));
-    }
-}
-//# sourceMappingURL=shared.js.map

package/dist/store/vector-writes.d.ts

@@ -1,13 +0,0 @@
-import { type DatabaseSyncLike } from '../db/index.js';
-import type { MemoryEntry } from '../core/memory.js';
-import type { PhysicsParticle } from '../core/physics.js';
-import type { VectorBackfillQuery, VectorWrite, VectorWriteResult } from './port.js';
-export declare function entriesWithoutVectorAt(db: DatabaseSyncLike, query: VectorBackfillQuery): MemoryEntry[];
-export declare function writeVectorsAt(db: DatabaseSyncLike, write: VectorWrite): VectorWriteResult;
-/** Every particle stored under `hippoRoot`. */
-export declare function loadStoredParticles(hippoRoot: string): PhysicsParticle[];
-/** Writes `particles` over their stored state under `hippoRoot`, as one batch. */
-export declare function saveStoredParticles(hippoRoot: string, particles: PhysicsParticle[]): void;
-/** Replaces every particle under `hippoRoot` with a fresh one per entry that has an embedding; returns how many. */
-export declare function resetStoredParticles(hippoRoot: string, entries: MemoryEntry[], embeddingIndex: Record<string, number[]>): number;
-//# sourceMappingURL=vector-writes.d.ts.map

package/dist/store/vector-writes.js

@@ -1,89 +0,0 @@
-// hippo.db's half of the VectorWrites store group.
-import { closeHippoDb, openHippoDb, setMeta, withWriteScope } from '../db/index.js';
-import { embeddingIndexStateOn, replacesIndex } from './embeddings/index.js';
-import { float32ToBuffer, loadPhysicsState, resetAllPhysicsState, savePhysicsState } from '../db/physics-state.js';
-import { EMBEDDING_MODEL_META_KEY, upsertVectors } from '../db/vector-store.js';
-import { chunked } from './entry-reads.js';
-import { MEMORY_SELECT_COLUMNS, rowToEntry } from './rows.js';
-const MAX_BACKFILL_PAGE = 500;
-export function entriesWithoutVectorAt(db, query) {
-    if (!Number.isInteger(query.limit))
-        throw new RangeError('limit must be an integer');
-    const limit = Math.max(1, Math.min(query.limit, MAX_BACKFILL_PAGE));
-    const tenantClause = query.tenantId === undefined ? '' : ' AND tenant_id = ?';
-    const tenantArgs = query.tenantId === undefined ? [] : [query.tenantId];
-    // SAFETY: selects exactly MEMORY_SELECT_COLUMNS, matching MemoryRow's field set.
-    const rows = db.prepare(`SELECT ${MEMORY_SELECT_COLUMNS} FROM memories WHERE id > ?${tenantClause}
-       AND NOT EXISTS (SELECT 1 FROM memory_vectors v WHERE v.memory_id = memories.id AND v.model = ?)
-     ORDER BY id LIMIT ?`).all(query.afterId ?? '', ...tenantArgs, query.model, limit);
-    return rows.map(rowToEntry);
-}
-function ownedIds(db, ids, tenantId) {
-    const owned = new Set();
-    for (const chunk of chunked([...new Set(ids)])) {
-        // SAFETY: the SELECT names one column, id.
-        const rows = db.prepare(`SELECT id FROM memories WHERE +tenant_id = ? AND id IN (${chunk.map(() => '?').join(', ')})`).all(tenantId, ...chunk);
-        for (const row of rows)
-            owned.add(row.id);
-    }
-    return owned;
-}
-// savePhysicsState opens its own transaction and overwrites, so the first particle for a memory is inserted here instead.
-function insertNewParticles(db, rows) {
-    const stmt = db.prepare(`
-    INSERT INTO memory_physics (memory_id, position_blob, velocity_blob, mass, charge, temperature, last_simulation)
-    VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(memory_id) DO NOTHING
-  `);
-    for (const { memoryId, particle: p } of rows) {
-        if (p)
-            stmt.run(memoryId, float32ToBuffer(p.position), float32ToBuffer(p.velocity), p.mass, p.charge, p.temperature, p.lastSimulation);
-    }
-}
-export function writeVectorsAt(db, write) {
-    return withWriteScope(db, 'write_vectors', () => {
-        const replaces = replacesIndex(embeddingIndexStateOn(db), write.model);
-        if (replaces && !write.replaceIndex)
-            return { written: 0, modelMismatch: true };
-        const owned = ownedIds(db, write.rows.map((r) => r.memoryId), write.tenantId);
-        const rows = write.rows.filter((r) => owned.has(r.memoryId) && r.vector.length > 0 && r.vector.every(Number.isFinite));
-        if (rows.length === 0)
-            return { written: 0, modelMismatch: false };
-        if (replaces)
-            db.exec('DELETE FROM memory_vectors; DELETE FROM memory_physics;');
-        const written = upsertVectors(db, rows.map((r) => [r.memoryId, r.vector]), write.model);
-        insertNewParticles(db, rows);
-        setMeta(db, EMBEDDING_MODEL_META_KEY, write.model);
-        return { written, modelMismatch: false };
-    });
-}
-/** Every particle stored under `hippoRoot`. */
-export function loadStoredParticles(hippoRoot) {
-    const db = openHippoDb(hippoRoot);
-    try {
-        return Array.from(loadPhysicsState(db).values());
-    }
-    finally {
-        closeHippoDb(db);
-    }
-}
-/** Writes `particles` over their stored state under `hippoRoot`, as one batch. */
-export function saveStoredParticles(hippoRoot, particles) {
-    const db = openHippoDb(hippoRoot);
-    try {
-        savePhysicsState(db, particles);
-    }
-    finally {
-        closeHippoDb(db);
-    }
-}
-/** Replaces every particle under `hippoRoot` with a fresh one per entry that has an embedding; returns how many. */
-export function resetStoredParticles(hippoRoot, entries, embeddingIndex) {
-    const db = openHippoDb(hippoRoot);
-    try {
-        return resetAllPhysicsState(db, entries, embeddingIndex);
-    }
-    finally {
-        closeHippoDb(db);
-    }
-}
-//# sourceMappingURL=vector-writes.js.map