hippo-memory 1.71.0 → 1.72.0

package/dist/server/server-detect.js

@@ -1,6 +1,7 @@
 import { existsSync, readFileSync, writeFileSync, unlinkSync, renameSync } from 'node:fs';
 import { join } from 'node:path';
 import { envHealthProbeMs } from '../util/env.js';
+import { errorCode, errorFields, errorMessage, log } from '../util/log.js';
 // Pidfile sits directly inside hippoRoot (the `.hippo` directory itself, as for openHippoDb), i.e. `${hippoRoot}/server.pid`.
 const PIDFILE = 'server.pid';
 /** How long detectServer waits for the `/health` probe before treating the pidfile as stale; short because it fires only when a pidfile exists with a live
@@ -11,6 +12,15 @@ const HEALTH_PROBE_TIMEOUT_MS = 300;
 const HEALTH_BODY_MAX_BYTES = 64 * 1024;
 /** Loopback hosts a recorded pidfile url may point at (serve() only binds these); any other host is malformed or forged and is not probed. */
 const PIDFILE_LOOPBACK_HOSTS = new Set(['127.0.0.1', '::1', 'localhost']);
+/** The pidfile names a live process on loopback that did not answer /health in time: most likely a busy server, so a caller must neither
+ * bypass it with a second writer nor start another server beside it. */
+export class ServerUnresponsiveError extends Error {
+    constructor(info, probeMs, options) {
+        super(`a hippo server (pid ${info.pid}) is recorded at ${info.url} but did not answer /health within ${probeMs} ms; it may be busy. ` +
+            'Retry shortly, raise HIPPO_HEALTH_PROBE_MS, or stop that server.', options);
+        this.name = 'ServerUnresponsiveError';
+    }
+}
 /** False when the pidfile's pid is dead or its url could not be one serve() wrote. */
 function isLiveLoopbackTarget(info) {
     // Signal 0 throws if the pid is dead or owned by another user we cannot signal; either way, treat as stale.
@@ -56,14 +66,23 @@ async function readCappedBody(body) {
     raw += decoder.decode();
     return raw;
 }
-/** True when /health on info.url reports the pidfile's started_at; unlinks the pidfile on any definitive mismatch. */
+/** True when fetch failed because nothing listens on the port (the error or its cause, or any address tried, is ECONNREFUSED). */
+function isConnectionRefused(err) {
+    if (errorCode(err) === 'ECONNREFUSED')
+        return true;
+    const cause = err instanceof Error ? err.cause : undefined;
+    if (errorCode(cause) === 'ECONNREFUSED')
+        return true;
+    return cause instanceof AggregateError && cause.errors.length > 0 && cause.errors.every((e) => errorCode(e) === 'ECONNREFUSED');
+}
+/** True when /health on info.url reports the pidfile's started_at; unlinks the pidfile on any definitive mismatch and throws ServerUnresponsiveError on a
+ * probe timeout. */
 async function healthMatchesPidfile(hippoRoot, info) {
-    // Confirm the answering process is this server by matching /health `started_at` against the pidfile; refusal, non-200 or a malformed body unlink the
-    // pidfile as stale. A probe timeout is ambiguous (a live but busy server can miss the 300ms window), so it returns null WITHOUT unlinking.
+    // Refusal, non-200 or a malformed body unlink the pidfile as stale. A timeout proves neither a dead server nor this one, so it keeps the pidfile and throws.
+    // Any other error keeps it and warns.
+    const probeMs = envHealthProbeMs() ?? HEALTH_PROBE_TIMEOUT_MS;
     try {
-        const res = await fetch(`${info.url}/health`, {
-            signal: AbortSignal.timeout(envHealthProbeMs() ?? HEALTH_PROBE_TIMEOUT_MS),
-        });
+        const res = await fetch(`${info.url}/health`, { signal: AbortSignal.timeout(probeMs) });
         if (!res.ok || !res.body) {
             removePidfile(hippoRoot);
             return false;
@@ -81,16 +100,21 @@ async function healthMatchesPidfile(hippoRoot, info) {
         return true;
     }
     catch (err) {
-        // A timeout is ambiguous (the server may be busy), so keep the pidfile; any other failure (refused, malformed body) is definitive: unlink as stale.
-        // SAFETY: err's shape is unknown (catch clause); reading an optional .name property structurally is safe regardless of the actual type.
-        if (err?.name !== 'TimeoutError') {
+        if (err instanceof Error && err.name === 'TimeoutError')
+            throw new ServerUnresponsiveError(info, probeMs, { cause: err });
+        // A refusal or a non-JSON body proves no hippo server answers there; any other failure may be a live server, so the file stays.
+        if (isConnectionRefused(err) || err instanceof SyntaxError) {
             removePidfile(hippoRoot);
         }
+        else {
+            log.warn(`server pidfile ${join(hippoRoot, PIDFILE)} kept: the /health probe failed (${errorMessage(err)})`, errorFields(err));
+        }
         return false;
     }
 }
 /** Returns the pidfile's ServerInfo if a live hippo server answers on the recorded url, else null (best-effort unlinking of missing/stale/malformed pidfiles).
- * `process.kill(pid, 0)` rules out dead pids; GET /health must then return the pidfile's `started_at` (pids get reused). Probes only if the pid is live. */
+ * `process.kill(pid, 0)` rules out dead pids; GET /health must then return the pidfile's `started_at` (pids get reused). Probes only if the pid is live.
+ * Throws ServerUnresponsiveError when that probe times out. */
 export async function detectServer(hippoRoot) {
     const path = join(hippoRoot, PIDFILE);
     if (!existsSync(path))
@@ -99,7 +123,8 @@ export async function detectServer(hippoRoot) {
     try {
         info = JSON.parse(readFileSync(path, 'utf8'));
     }
-    catch {
+    catch (err) {
+        log.warn(`server pidfile ${path} is unreadable (${errorMessage(err)}); removing it`);
         removePidfile(hippoRoot);
         return null;
     }

package/dist/server/sleep-child.js

@@ -1,19 +1,38 @@
-// Entry of the process that runs one POST /v1/sleep; sleep-offload.ts starts it and nothing imports it.
-// The sleep module alone: the api barrel loads far more than one sleep needs, and every sleep pays for the start.
+// Entry of the process that runs one POST /v1/sleep or MCP auto-sleep; sleep-offload.ts starts it and nothing imports it.
+// The sleep modules alone: the api barrel loads far more than one sleep needs, and every sleep pays for the start.
 import { sleep } from '../api/sleep.js';
-import { isStoreBusy, runWithRequestStores, SERVER_DB_WAIT_MS } from '../db/index.js';
-import { errorMessage } from '../util/log.js';
+import { consolidate } from '../consolidate/sleep.js';
+import { runWithRequestStores, SERVER_DB_WAIT_MS } from '../db/index.js';
+import { isStoreBusy } from '../store/sqlite/busy.js';
+import { errorFields, errorMessage, log } from '../util/log.js';
 function failed(err) {
     return { ok: false, busy: isStoreBusy(err), message: errorMessage(err), stack: err instanceof Error ? err.stack : undefined };
 }
+/** Resolves to the exit code once the channel has taken the reply, or failed to. */
+function sendReply(reply) {
+    return new Promise((resolve) => {
+        process.send?.(reply, (sendErr) => {
+            if (sendErr)
+                log.error(`sleep child: sending the reply failed: ${errorMessage(sendErr)}`, errorFields(sendErr));
+            resolve(sendErr ? 1 : 0);
+        });
+    });
+}
 process.once('message', (message) => {
     // SAFETY: sleep-offload.ts is the only writer on this channel and sends exactly one SleepJob.
     const job = message;
-    const ctx = { hippoRoot: job.hippoRoot, tenantId: job.tenantId, actor: job.actor };
+    const work = () => job.kind === 'consolidate'
+        ? consolidate(job.hippoRoot)
+        : sleep({ hippoRoot: job.hippoRoot, tenantId: job.tenantId, actor: job.actor }, job.opts);
     // The server's own request scope and lock wait, so a busy store answers as it did when sleep ran in the server.
-    runWithRequestStores(() => sleep(ctx, job.opts), { busyWaitMs: SERVER_DB_WAIT_MS })
+    runWithRequestStores(work, { busyWaitMs: SERVER_DB_WAIT_MS })
         .then((result) => ({ ok: true, result }), failed)
         // Exit only once the reply is written, or the parent could see the process end first.
-        .then((reply) => process.send?.(reply, () => process.exit(0)));
+        .then(sendReply)
+        .catch((err) => {
+        log.error(`sleep child: the reply could not be sent: ${errorMessage(err)}`, errorFields(err));
+        return 1;
+    })
+        .then((code) => process.exit(code));
 });
 //# sourceMappingURL=sleep-child.js.map

package/dist/server/sleep-offload.d.ts

@@ -1,14 +1,18 @@
 import { type Actor, type Context, type SleepOpts, type SleepResult } from '../api/index.js';
-/** What the child needs to rebuild the request's context; the store handle itself cannot cross a process boundary. */
-export interface SleepJob {
+/** What the child needs to rebuild the request's context (the store handle itself cannot cross a process boundary), or the store an auto-sleep consolidates. */
+export type SleepJob = {
+    readonly kind: 'sleep';
     readonly hippoRoot: string;
     readonly tenantId: string;
     readonly actor: Actor;
     readonly opts: SleepOpts;
-}
-export type SleepReply = {
+} | {
+    readonly kind: 'consolidate';
+    readonly hippoRoot: string;
+};
+export type SleepReply<R = SleepResult> = {
     readonly ok: true;
-    readonly result: SleepResult;
+    readonly result: R;
 } | {
     readonly ok: false;
     readonly busy: boolean;
@@ -17,4 +21,6 @@ export type SleepReply = {
 };
 /** `sleep` for the HTTP route: same result and errors, run outside the serving process and stopped at the deadline (10 minutes, or HIPPO_SLEEP_TIMEOUT_MS). */
 export declare function sleepInChild(ctx: Context, opts: SleepOpts): Promise<SleepResult>;
+/** The MCP auto-sleep's consolidation, in the same queue and child as POST /v1/sleep; auto-sleep only starts on a hippo.db store. */
+export declare function consolidateInChild(hippoRoot: string): Promise<void>;
 //# sourceMappingURL=sleep-offload.d.ts.map

package/dist/server/sleep-offload.js

@@ -1,8 +1,8 @@
-// POST /v1/sleep runs its consolidation in a child process, so the server keeps answering and a stuck run can be stopped.
+// POST /v1/sleep and the MCP auto-sleep run their consolidation in a child process, so the server keeps answering and a stuck run can be stopped.
 import { spawn } from 'node:child_process';
 import { fileURLToPath } from 'node:url';
 import { sleep } from '../api/index.js';
-import { StoreBusyError } from '../db/index.js';
+import { StoreBusyError } from '../store/port.js';
 import { envSleepTimeoutMs } from '../util/env.js';
 import { HttpError } from '../util/http-util.js';
 const DEFAULT_SLEEP_DEADLINE_MS = 600_000;
@@ -32,7 +32,7 @@ function runInChild(job) {
         // SIGKILL because the child is inside synchronous SQLite work and cannot run a handler; the OS drops its write lock.
         const timer = setTimeout(() => { timedOut = true; child.kill('SIGKILL'); }, deadlineMs);
         child.once('message', (message) => {
-            // SAFETY: sleep-child.ts is the only writer on this channel and sends exactly one SleepReply.
+            // SAFETY: sleep-child.ts is the only writer on this channel and sends exactly one SleepReply for the job it was given.
             reply = message;
         });
         child.once('error', (err) => { clearTimeout(timer); reject(err); });
@@ -54,20 +54,26 @@ let turn = Promise.resolve();
 const MAX_SLEEPS_IN_LINE = 2;
 const SLEEP_RETRY_AFTER_SEC = 30;
 let inLine = 0;
-/** `sleep` for the HTTP route: same result and errors, run outside the serving process and stopped at the deadline (10 minutes, or HIPPO_SLEEP_TIMEOUT_MS). */
-export function sleepInChild(ctx, opts) {
-    // The child reopens the store by path, which only SQLite has; any other store keeps sleep's own refusal.
-    if (ctx.store && ctx.store.kind !== 'sqlite')
-        return sleep(ctx, opts);
+function inTurn(job) {
     if (inLine >= MAX_SLEEPS_IN_LINE) {
         return Promise.reject(new HttpError(503, 'a sleep is already running and another is waiting; retry when one has finished', SLEEP_RETRY_AFTER_SEC));
     }
     inLine += 1;
-    const job = { hippoRoot: ctx.hippoRoot, tenantId: ctx.tenantId, actor: ctx.actor, opts };
-    // One sleep at a time, as on the server's single thread: a second request waits its turn and does not fight the first for the write lock.
+    // One sleep at a time, as on the server's single thread: a second one waits its turn and does not fight the first for the write lock.
     const run = turn.then(() => runInChild(job));
-    // A failed sleep reaches its own request through `run`; the queue only needs to know the turn is over.
+    // A failed sleep reaches its own caller through `run`; the queue only needs to know the turn is over.
     turn = run.then(() => undefined, () => undefined).finally(() => { inLine -= 1; });
     return run;
 }
+/** `sleep` for the HTTP route: same result and errors, run outside the serving process and stopped at the deadline (10 minutes, or HIPPO_SLEEP_TIMEOUT_MS). */
+export function sleepInChild(ctx, opts) {
+    // The child reopens the store by path, which only SQLite has; any other store keeps sleep's own refusal.
+    if (ctx.store && ctx.store.kind !== 'sqlite')
+        return sleep(ctx, opts);
+    return inTurn({ kind: 'sleep', hippoRoot: ctx.hippoRoot, tenantId: ctx.tenantId, actor: ctx.actor, opts });
+}
+/** The MCP auto-sleep's consolidation, in the same queue and child as POST /v1/sleep; auto-sleep only starts on a hippo.db store. */
+export async function consolidateInChild(hippoRoot) {
+    await inTurn({ kind: 'consolidate', hippoRoot });
+}
 //# sourceMappingURL=sleep-offload.js.map

package/dist/server/types.d.ts

@@ -88,6 +88,10 @@ export interface RouteRequest {
     opts: ResolvedServeOpts;
     query: URLSearchParams;
 }
+/** A request past the dispatch gate, carrying the caller the gate authenticated. */
+export interface AuthedRouteRequest extends RouteRequest {
+    ctx: Context;
+}
 /** A route's store status: a `storeReady` group, a `sqliteOnly` reason, or neither while it waits for a group; never both. */
 type StoreStatus = {
     storeReady?: StoreGroup;
@@ -96,20 +100,28 @@ type StoreStatus = {
     sqliteOnly: string;
     storeReady?: never;
 };
-/** One /v1 route: an exact path, a matchPath pattern, or a regex, each paired with one method's handler. Under another store it runs only with its
- * store's `storeReady` group; `loop: 'off'` runs its SQLite work on a worker thread, so the handler opens no hippo.db on the server thread. */
-export type Route = {
-    method: string;
-    loop?: 'off';
-} & StoreStatus & ({
+/** What the dispatch gate proves before a handler runs: any caller, an admin, a host admin on loopback, or nothing, for a route
+ * whose handler checks the sender's signature itself. */
+export type RouteAuth = 'caller' | 'admin' | 'host-admin-local' | 'webhook-signature';
+export type RouteEndpoint<R> = {
     path: string;
-    handler: (r: RouteRequest) => Promise<void>;
+    handler: (r: R) => Promise<void>;
 } | {
     pattern: string;
-    handler: (r: RouteRequest, params: Record<string, string>) => Promise<void>;
+    handler: (r: R, params: Record<string, string>) => Promise<void>;
 } | {
     regex: RegExp;
-    handler: (r: RouteRequest, match: RegExpMatchArray) => Promise<void>;
-});
+    handler: (r: R, match: RegExpMatchArray) => Promise<void>;
+};
+/** One /v1 route: an exact path, a matchPath pattern, or a regex, each paired with one method's handler. Under another store it runs only with its
+ * store's `storeReady` group; `loop: 'off'` runs its SQLite work on a worker thread, so the handler opens no hippo.db on the server thread. */
+export type Route = {
+    method: string;
+    loop?: 'off';
+} & StoreStatus & (({
+    auth: Exclude<RouteAuth, 'webhook-signature'>;
+} & RouteEndpoint<AuthedRouteRequest>) | ({
+    auth: 'webhook-signature';
+} & RouteEndpoint<RouteRequest>));
 export {};
 //# sourceMappingURL=types.d.ts.map

package/dist/server/validation.d.ts

@@ -3,10 +3,9 @@ import type { Context, RememberOpts } from '../api/index.js';
 import { type JsonValue } from '../util/json.js';
 export declare function isSetMember<T extends string>(set: ReadonlySet<T>, value: string): value is T;
 export declare const MAX_SHORT_FIELD_LEN = 4096;
-export declare const MAX_LIST_LIMIT = 1000;
 export declare function parseListLimit(limitRaw: string | null, defaultLimit?: number, maxLimit?: number): number;
 /** `_authed` is proof the caller passed auth, so a full-size body is read only for
- * an authenticated caller; the key mint reads first under its own small cap. */
+ * an authenticated caller; the key mint reads under its own small cap. */
 export declare function parseJsonBody(req: IncomingMessage, _authed: Context): Promise<Record<string, JsonValue>>;
 export declare function parseJsonObjectText(raw: string): Record<string, JsonValue>;
 export declare function getString(obj: Record<string, JsonValue>, key: string): string | undefined;

package/dist/server/validation.js

@@ -1,4 +1,5 @@
-import { HttpError, MAX_ID_LEN, readBody } from '../util/http-util.js';
+import { HttpError, readBody } from '../util/http-util.js';
+import { DEFAULT_LIST_LIMIT, MAX_ID_LEN, MAX_LIST_LIMIT, resolveListLimit } from '../util/limits.js';
 import { isJsonString, isJsonObject } from '../util/json.js';
 // Runtime membership check for a `ReadonlySet<T>` of string-literal members: Set<T>.has gives no narrowing, so this is the one place the `as T` assertion lives
 // and callers narrow via the `value is T` return.
@@ -8,20 +9,14 @@ export function isSetMember(set, value) {
 }
 // Cap for short free-text HTTP fields (names, text, context, change summaries) on the object routes.
 export const MAX_SHORT_FIELD_LEN = 4096;
-// Number.isInteger, not isFinite: SQLite `LIMIT ?` rejects "1.5" with a 500.
-// Shared by every first-class-object list route so the guard cannot drift.
-export const MAX_LIST_LIMIT = 1000;
-export function parseListLimit(limitRaw, defaultLimit = 100, maxLimit = MAX_LIST_LIMIT) {
-    if (limitRaw === null)
-        return defaultLimit;
-    const limit = Number(limitRaw);
-    if (!Number.isInteger(limit) || limit <= 0 || limit > maxLimit) {
+export function parseListLimit(limitRaw, defaultLimit = DEFAULT_LIST_LIMIT, maxLimit = MAX_LIST_LIMIT) {
+    const limit = resolveListLimit(limitRaw === null ? undefined : Number(limitRaw), defaultLimit, maxLimit);
+    if (limit === undefined)
         throw new HttpError(400, `limit must be a positive integer <= ${maxLimit}`);
-    }
     return limit;
 }
 /** `_authed` is proof the caller passed auth, so a full-size body is read only for
- * an authenticated caller; the key mint reads first under its own small cap. */
+ * an authenticated caller; the key mint reads under its own small cap. */
 export async function parseJsonBody(req, _authed) {
     return parseJsonObjectText(await readBody(req));
 }

package/dist/server.d.ts

@@ -13,8 +13,9 @@ export type { JsonValue } from './util/json.js';
 export { captureSessionTexts, type SessionCaptureRequest, type SessionCaptureResult } from './capture/session-texts.js';
 export { hasGroup, sqliteStore, type AmbientCandidateRequest, type ContextReads, type EntryRemoval, type EntryTarget, type EntryWrite, type EntryWrites, type HippoStore, type KeyAudit, type KeyListQuery, type KeyMint, type KeyRevoke, type KeyWrites, type OutcomeWrite, type RawArchive, type RecallSearchArgs, type RecallWrites, type SelfKeyMint, type StoreGroup, type StoreGroups, type SupersedeWrite, type VectorReads, type VectorBackfillQuery, type VectorRowWrite, type VectorWrite, type VectorWriteResult, type VectorWrites, } from './store/index.js';
 export { entryAfterOutcome } from './core/memory.js';
-export { rejectionDigest, RejectedValueError } from './store/rejection.js';
-export { ownScopeTouches } from './store/recall-scope.js';
+export { rejectionDigest } from './store/rejection.js';
+export { RejectedValueError } from './core/api-errors.js';
+export { ownScopeTouches } from './core/recall-scope.js';
 export { BadRequestError, ConflictError } from './core/api-errors.js';
 export type { HippoDbContext, StoreReply } from './api/types.js';
 export type { ApiKeyListItem, ApiKeyListRow, ApiKeyRecord, ListApiKeysOpts, NewApiKey } from './store/auth.js';
@@ -37,11 +38,11 @@ export type { AmbientTallies } from './core/ambient.js';
 export type { TokenUse } from './store/token-ledger.js';
 export type { EmbeddingIndexState } from './store/vector-index.js';
 export type { PhysicsParticle } from './core/physics.js';
-export { StoreBusyError } from './db/index.js';
+export { StoreBusyError } from './store/port.js';
 export { decodeVector, EMBEDDING_MODEL_META_KEY, encodeVector, rankVectorRows, type VectorMatch, type VectorRow } from './db/vector-store.js';
 export { bufferToFloat32, float32ToBuffer } from './db/physics-state.js';
 export { replacesIndex } from './store/vector-index.js';
-export { passesScopeFilterForRecall, RECALL_DEFAULT_DENY_SCOPES } from './store/recall-scope.js';
+export { passesScopeFilterForRecall, RECALL_DEFAULT_DENY_SCOPES } from './core/recall-scope.js';
 export { SECRET_TAGS } from './util/secret-detect.js';
 export { tallyAmbientEntries } from './core/ambient.js';
 export { ftsTermParts, rarestFtsQuery } from './core/prompt-recall.js';

package/dist/server.js

@@ -19,17 +19,18 @@ export { captureSessionTexts } from './capture/session-texts.js';
 export { hasGroup, sqliteStore, } from './store/index.js';
 // An add-on store's entry writes apply an outcome, guard tombstones and check reach exactly as hippo.db does.
 export { entryAfterOutcome } from './core/memory.js';
-export { rejectionDigest, RejectedValueError } from './store/rejection.js';
-export { ownScopeTouches } from './store/recall-scope.js';
+export { rejectionDigest } from './store/rejection.js';
+export { RejectedValueError } from './core/api-errors.js';
+export { ownScopeTouches } from './core/recall-scope.js';
 export { BadRequestError, ConflictError } from './core/api-errors.js';
-export { StoreBusyError } from './db/index.js';
+export { StoreBusyError } from './store/port.js';
 // An add-on store encodes, decodes and ranks vectors and particles with hippo.db's own code, and drops the index by its rule,
 // so both stores keep the same bytes and return the same ids in the same order.
 export { decodeVector, EMBEDDING_MODEL_META_KEY, encodeVector, rankVectorRows } from './db/vector-store.js';
 export { bufferToFloat32, float32ToBuffer } from './db/physics-state.js';
 export { replacesIndex } from './store/vector-index.js';
 // An add-on's ContextReads applies hippo.db's scope, secret, tally and rarest-term rules with the same code.
-export { passesScopeFilterForRecall, RECALL_DEFAULT_DENY_SCOPES } from './store/recall-scope.js';
+export { passesScopeFilterForRecall, RECALL_DEFAULT_DENY_SCOPES } from './core/recall-scope.js';
 export { SECRET_TAGS } from './util/secret-detect.js';
 export { tallyAmbientEntries } from './core/ambient.js';
 export { ftsTermParts, rarestFtsQuery } from './core/prompt-recall.js';

package/dist/sharing/global-store.js

@@ -5,12 +5,12 @@ import { generateId } from '../core/memory.js';
 import { initStore } from '../store/open.js';
 import { writeEntry } from '../store/entry-writes.js';
 import { readEntry } from '../store/entry-reads.js';
-import { isPersonalScope } from '../store/recall-scope.js';
+import { isPersonalScope } from '../core/recall-scope.js';
 import { fallbackOrigin, resolveGlobalRootDir } from '../core/project-identity.js';
 import { isSharedStore } from '../core/config.js';
 import { detectSecret } from '../util/secret-detect.js';
 import { isQuarantineScope } from '../trust/quarantine.js';
-import { embedMemory } from '../store/embeddings/index.js';
+import { embedMemory } from '../search/embed-memories.js';
 import { appendAuditEvent } from '../store/audit.js';
 /** Returns the path to the global Hippo store.
  *  Resolution order: $HIPPO_HOME > $XDG_DATA_HOME/hippo > ~/.hippo/ */

package/dist/sharing/global-sync.js

@@ -3,70 +3,63 @@ import { BadRequestError } from '../core/api-errors.js';
 import * as fs from 'fs';
 import * as path from 'path';
 import { AGENT_MEMORY_SOURCE_PREFIX } from '../core/agent-memory-tools.js';
-import { writeEntry } from '../store/entry-writes.js';
-import { loadAllEntries } from '../store/entry-reads.js';
+import { writeEntriesSkippingRejected } from '../store/entry-writes.js';
+import { loadEntriesByIds, loadEntryTextKeys } from '../store/entry-reads.js';
+import { chunked } from '../util/chunked.js';
+import { withRequestStoresSync } from '../db/request-stores.js';
 import { classifyOriginProject, resolveProjectIdentity } from '../core/project-identity.js';
 import { isSharedStore } from '../core/config.js';
 import { detectSecret } from '../util/secret-detect.js';
-import { RejectedValueError } from '../store/rejection.js';
-import { embedAll } from '../store/embeddings/index.js';
+import { embedAll } from '../search/embed-memories.js';
 import { log } from '../util/log.js';
 import { logEmbedAllFailure } from './search-both.js';
 /** Copy all global memories into the local store, skipping entries that already exist locally by ID or text (promote and share copy under a new ID).
  *  Returns the count of newly copied entries. */
 export function syncGlobalToLocal(localRoot, globalRoot, opts = {}) {
     if (isSharedStore(localRoot)) {
-        throw new BadRequestError(`Refusing to sync into ${localRoot}: a shared store takes no copies of a personal global store, whose rows would reach every member.`);
+        throw new BadRequestError(`Refusing to sync into ${localRoot}: a shared store takes no copies of a personal global store, ` +
+            `whose rows would reach every member.`);
     }
     if (!fs.existsSync(globalRoot))
         return 0;
-    // Host-wide read: the global union is copied into a tenant-scoped local store, and writeEntry carries the tenant if the local-root context has one.
-    const globalEntries = loadAllEntries(globalRoot);
-    const textKey = (e) => `${e.tenantId}\n${e.content}`;
-    const localEntries = loadAllEntries(localRoot);
-    const localIds = new Set(localEntries.map((e) => e.id));
-    const localText = new Set(localEntries.map(textKey));
+    return withRequestStoresSync(() => copyMissingRows(localRoot, globalRoot, opts.includeCrossProject === true));
+}
+const textKey = (e) => `${e.tenantId}\n${e.content}`;
+/** One open per store and one transaction: id, tenant and text decide which rows to read whole, and the copies commit together. */
+function copyMissingRows(localRoot, globalRoot, includeCrossProject) {
+    // Host-wide read: the global union is copied into a tenant-scoped local store, and each copy keeps its own tenant.
+    const localKeys = loadEntryTextKeys(localRoot);
+    const localIds = new Set(localKeys.map((k) => k.id));
+    const localText = new Set(localKeys.map(textKey));
+    // Only the global store's user pass sets an imported note's row aside, so a copy would outlive the note.
+    const candidates = loadEntryTextKeys(globalRoot).filter((k) => !localIds.has(k.id) && !k.source.startsWith(AGENT_MEMORY_SOURCE_PREFIX) && !localText.has(textKey(k)));
+    if (candidates.length === 0)
+        return 0;
+    const byId = new Map(chunked(candidates.map((k) => k.id)).flatMap((ids) => loadEntriesByIds(globalRoot, ids)).map((e) => [e.id, e]));
     // Syncing down must not re-import what ambient context excludes: other-project rows are skipped by default and secret rows never copied.
-    // origin_project is preserved on the copy (writeEntry stamps it only when missing).
+    // origin_project is preserved on the copy (the write stamps it only when missing).
     const currentProject = resolveProjectIdentity(path.dirname(path.resolve(localRoot)));
-    let count = 0;
-    // A locally rejected value must not come back through sync down: caught per item, printed as one line.
+    // A locally rejected value must not come back through sync down: skipped per item, printed as one line.
     let rejected = 0;
-    for (const entry of globalEntries) {
-        // Skip if already present by ID
-        if (localIds.has(entry.id))
-            continue;
-        // Only the global store's user pass sets an imported note's row aside, so a copy would outlive the note.
-        if (entry.source.startsWith(AGENT_MEMORY_SOURCE_PREFIX))
-            continue;
-        if (localText.has(textKey(entry)))
-            continue;
-        if (detectSecret(entry).flagged)
-            continue;
-        if (!opts.includeCrossProject && classifyOriginProject(entry.origin_project, currentProject) === 'cross-project')
-            continue;
-        if (!writeUnlessRejected(localRoot, entry)) {
-            rejected++;
-            continue;
+    const count = writeEntriesSkippingRejected(localRoot, (put) => {
+        for (const { id } of candidates) {
+            const entry = byId.get(id);
+            // A row the global store dropped between the two reads, or a text an earlier copy in this batch already brought down.
+            if (!entry || localText.has(textKey(entry)))
+                continue;
+            if (detectSecret(entry).flagged)
+                continue;
+            if (!includeCrossProject && classifyOriginProject(entry.origin_project, currentProject) === 'cross-project')
+                continue;
+            if (put(entry))
+                localText.add(textKey(entry));
+            else
+                rejected++;
         }
-        localText.add(textKey(entry));
-        count++;
-    }
+    });
     finishSyncDown(localRoot, count, rejected);
     return count;
 }
-/** False when the local store rejects the value; any other failure still throws. */
-function writeUnlessRejected(localRoot, entry) {
-    try {
-        writeEntry(localRoot, entry);
-        return true;
-    }
-    catch (err) {
-        if (err instanceof RejectedValueError)
-            return false;
-        throw err;
-    }
-}
 function finishSyncDown(localRoot, count, rejected) {
     if (rejected > 0) {
         log.warn(`syncGlobalToLocal: skipped ${rejected} rejected value(s) (run \`hippo unreject\` on the local store to allow).`);

package/dist/sharing/project-merge.d.ts

@@ -1,4 +1,5 @@
-import { type DatabaseSyncLike } from '../db/index.js';
+import { type ProjectFold } from '../agent-memories/project-folds.js';
+import type { ProjectTagReads, ProjectTagStore } from '../store/project-tags.js';
 export interface ProjectSummary {
     /** '' is user-global, null is unknown; neither can be merged. */
     readonly origin: string | null;
@@ -18,10 +19,6 @@ export interface MergeResult {
     readonly compactions: number;
     readonly backup: string | null;
 }
-export interface ProjectFold {
-    readonly from: string;
-    readonly into: string;
-}
 /** An old name whose session folders now resolve to several projects: no fold can say whose rows are whose. */
 export interface ProjectCollision {
     readonly name: string;
@@ -45,28 +42,24 @@ export interface RepairResult {
     readonly backup: string | null;
 }
 /** Live rows per project name, newest write first, with how many of its imported notes are copies held elsewhere. */
-export declare function listProjects(db: DatabaseSyncLike, tenantId: string): ProjectSummary[];
+export declare function listProjects(store: ProjectTagReads): ProjectSummary[];
 /** Refuses user-global and unknown: they are not projects, and folding them would leak or hide every row. */
 export declare function validateMergeNames(from: string, into: string): string | null;
 /** Folds project `from` into `into` for one tenant in one transaction; a dry run rolls back and writes nothing, mirrors included. */
-export declare function mergeProjects(db: DatabaseSyncLike, hippoRoot: string, opts: {
-    tenantId: string;
+export declare function mergeProjects(store: ProjectTagStore, opts: {
     from: string;
     into: string;
     dryRun: boolean;
 }): MergeResult;
-/** Names folded, directly or through others, into one of `names`; the sync moves imports still filed under them. */
-export declare function namesFoldedInto(db: DatabaseSyncLike, tenantId: string, names: readonly string[]): string[];
 /** Reads only, so doctor and a dry run take no write lock; merged rows are planned before any fold, so a few may re-tag differently once folds apply. */
-export declare function planProjectRepair(db: DatabaseSyncLike, hippoRoot: string, tenantId: string, globalFolds?: boolean): Omit<RepairResult, 'backup'>;
+export declare function planProjectRepair(store: ProjectTagReads, hippoRoot: string, globalFolds?: boolean): Omit<RepairResult, 'backup'>;
 /** Sets aside stray imports, folds the names the resolver now maps elsewhere, then
  * re-tags sleep's user-global merges by their parents; a dry run only plans. */
-export declare function repairProjects(db: DatabaseSyncLike, hippoRoot: string, opts: {
-    tenantId: string;
+export declare function repairProjects(store: ProjectTagStore, opts: {
     dryRun: boolean;
     globalFolds?: boolean;
 }): RepairResult;
 /** Sleep runs repair once per store, so an upgrade needs no command. Global-store folds stay
  *  with `hippo projects repair`: their evidence is compaction folders, blind to a same-named repo that never compacted. */
-export declare function repairOnceOnSleep(db: DatabaseSyncLike, hippoRoot: string, tenantId: string): RepairResult | null;
+export declare function repairOnceOnSleep(store: ProjectTagStore): RepairResult | null;
 //# sourceMappingURL=project-merge.d.ts.map