hippo-memory 1.61.0 → 1.63.0

package/dist/server.js

@@ -61,7 +61,6 @@ function isAddressInfo(a) {
 // HTTP /health response uses this; reading package.json synchronously here
 // would couple the daemon to its on-disk install path, which we want to
 // avoid for tests that mkdtemp a hippoRoot.
-// v1.3.1: source from src/version.ts so /health no longer reports stale 0.39.0.
 const VERSION = PACKAGE_VERSION;
 const LOOPBACK_HOSTS = new Set(['127.0.0.1', '::1', 'localhost']);
 /** The /v1 routes in dispatch order; the first entry whose method and path match handles the request. */
@@ -159,7 +158,7 @@ async function dispatchV1Route(r, method, path) {
     return false;
 }
 async function handleRequest(req, res, opts, startedAt, streamSlots, limiter) {
-    // v1.6.4: pre-decode raw-URL slash check. Catches `%2F` / `%2f` before
+    // Pre-decode raw-URL slash check. Catches `%2F` / `%2f` before
     // Node's URL parser collapses them and they slip past the route table.
     rejectEncodedSlash(req.url ?? '/');
     const { method, path, query } = parseRequest(req);
@@ -215,126 +214,93 @@ function sendHealth(req, res, startedAt) {
         sendJson(res, 200, { ok: true });
     }
 }
-/**
- * Boot the HTTP daemon on host:port and write the pidfile under hippoRoot.
- *
- * Refuses non-loopback hosts at boot (Footgun #3 from the A1 plan) unless
- * HIPPO_REQUIRE_AUTH=1 is set. The A5 v2 auth middleware (buildContextWithAuth /
- * requireAuth) has shipped and every route checks it except GET /health
- * (public by design for platform health checks) and the two connector
- * webhooks in PUBLIC_ROUTES, which are HMAC-gated by their own signing
- * secrets and 404 when those secrets are unset. But the loopback
- * no-auth fallback inside buildContextWithAuth still admits unauthenticated
- * requests from a loopback remote address (unless they carry Forwarded,
- * X-Forwarded-For/-Host/-Proto, X-Real-IP, Cf-Connecting-Ip or True-Client-Ip, which mark a same-host proxy and get
- * a 401 like any keyless remote request), so binding to a non-loopback host
- * is only safe once that fallback is disabled with HIPPO_REQUIRE_AUTH=1,
- * which forces every request (loopback or not) through Bearer-token
- * validation. Without that env var set, a non-loopback bind would expose the
- * DB to the network with no auth, so we fail fast instead.
- *
- * Use port: 0 in tests to bind to an ephemeral port and read the actual
- * port back via server.address() after listen.
- */
-export async function serve(opts) {
-    const host = opts.host ?? '127.0.0.1';
-    const requestedPort = opts.port ?? Number(envPort() ?? 6789);
+function assertBindable(host) {
     if (!LOOPBACK_HOSTS.has(host) && !envRequireAuth()) {
         throw new Error(`Refusing to bind hippo serve to non-loopback host '${host}' without auth. ` +
             `Set HIPPO_REQUIRE_AUTH=1 to bind non-loopback; every request then requires ` +
             `a valid API key. Bind to 127.0.0.1 / ::1 / localhost otherwise.`);
     }
-    // H3: refuse to start if a live hippo server already serves this hippoRoot.
+}
+async function assertNoLiveServer(hippoRoot) {
+    // Refuse to start if a live hippo server already serves this hippoRoot.
     // detectServer probes the recorded /health — a stale pidfile is unlinked and
     // ignored, but a live peer means a concurrent `hippo serve` would race for
     // the port and clobber the pidfile.
-    const existing = await detectServer(opts.hippoRoot);
+    const existing = await detectServer(hippoRoot);
     if (existing) {
         throw new Error(`hippo serve: already running on port ${existing.port} (pid ${existing.pid}). ` +
             `Stop that server before starting another on the same hippoRoot.`);
     }
-    // The server's start time. Single source of truth: it is returned by every
-    // GET /health response and (below) written into the pidfile, so detectServer
-    // can match the two and prove a pid-reusing impostor is not the real server.
-    const startedAt = new Date().toISOString();
-    // E3: per-IP rate limiter for /v1/* and /mcp*. Built here (not at module scope) so
+}
+function bootRateLimiter() {
+    // Per-IP rate limiter for /v1/* and /mcp*. Built here (not at module scope) so
     // HIPPO_V1_RPS is read at boot, matching HIPPO_PORT above and letting a test
     // set the rate before serve(). A non-positive or non-finite value disables
     // limiting (the opt-out knob).
     const v1Rps = Number(envV1Rps() ?? 20);
-    const limiter = Number.isFinite(v1Rps) && v1Rps > 0
+    return Number.isFinite(v1Rps) && v1Rps > 0
         ? createRateLimiter({ ratePerSec: v1Rps, burst: v1Rps * 2, idleEvictMs: 60000, maxKeys: 10000 })
         : undefined;
-    // Open /mcp/stream count per client key, so the cap is per server rather than per process.
-    const streamSlots = new Map();
+}
+function createStoreHolder(hippoRoot) {
     // Handlers open and close their own connections; while this one is held, none of those closes is SQLite's last,
     // which checkpoints and deletes the WAL. It opens only once the store exists, so serving never creates one.
     let heldDb;
     let stopHolding = false;
-    const holdStore = () => {
-        if (heldDb || stopHolding || !existsSync(getHippoDbPath(opts.hippoRoot)))
+    const hold = () => {
+        if (heldDb || stopHolding || !existsSync(getHippoDbPath(hippoRoot)))
             return;
         try {
-            heldDb = openHippoDb(opts.hippoRoot);
+            heldDb = openHippoDb(hippoRoot);
         }
         catch (err) {
             stopHolding = true;
             log.warn(`serve: could not hold a store connection; requests still work, only slower: ${err instanceof Error ? err.message : String(err)}`);
         }
     };
-    const inflight = new Set();
-    const server = createServer((req, res) => {
-        res.once('finish', holdStore);
-        inflight.add(res);
-        res.once('close', () => inflight.delete(res));
-        const requestId = resolveRequestId(req.headers['x-request-id']);
-        requestIds.set(req, requestId);
-        res.setHeader('X-Request-Id', requestId);
-        withBusyWait(SERVER_DB_WAIT_MS, () => handleRequest(req, res, opts, startedAt, streamSlots, limiter)).catch((err) => {
-            const mapped = replyFor(err);
-            logRequestFailure(req, err, requestId, mapped.status);
-            if (res.headersSent) {
-                try {
-                    res.end();
-                }
-                catch { /* socket already gone */ }
-                return;
-            }
-            if (isSqliteBusy(err))
-                res.setHeader('Retry-After', '1');
-            if (mapped.status === 500) {
-                // The id lets an operator find the logged cause without the client seeing internal text.
-                sendJson(res, 500, { error: mapped.message, requestId });
-                return;
-            }
-            // RecallContractError keeps the shared {error} shape and adds `code` so clients branch without parsing prose.
-            if (err instanceof RecallContractError) {
-                sendJson(res, 400, { error: err.message, code: err.code });
-                return;
-            }
-            sendError(res, mapped.status, mapped.message);
-            // M3: readBody hit the 1 MB cap mid-stream, so drop the socket rather than drain unbounded bytes.
-            if (err instanceof BodyTooLargeError)
-                req.destroy();
-        });
-    });
-    // T3b capture (v1.26.2): tests/server-concurrency.test.ts's ECONNRESET flake
-    // traced to a chunk-boundary reuse race — a kept-alive socket idled through
-    // a prior response chunk gets closed by the server's default 5s
-    // keepAliveTimeout just as a client reuses it for the next request. Raising
-    // both timeouts shrinks that idle-close/reuse window ~13x. Keep
-    // headersTimeout ABOVE the EFFECTIVE keep-alive expiry, which is
-    // keepAliveTimeout + keepAliveTimeoutBuffer (the buffer defaults to
-    // 1,000ms on Node 22.19+/24.6+ — verified 1,000 on node 24.13, so the
-    // effective expiry here is 66s; codex review caught that a 66s
-    // headersTimeout would sit exactly ON that boundary and recreate the
-    // race). The headers timer also runs while a kept-alive socket waits for
-    // its next request, so a value at or below the effective expiry would
-    // itself close idle reused sockets, and Node would not flag it (no error
-    // or warning at listen time — verified empirically).
+    const release = () => {
+        stopHolding = true;
+        if (heldDb)
+            closeHippoDb(heldDb);
+        heldDb = undefined;
+    };
+    return { hold, release };
+}
+function replyWithFailure(req, res, err, requestId) {
+    const mapped = replyFor(err);
+    logRequestFailure(req, err, requestId, mapped.status);
+    if (res.headersSent) {
+        try {
+            res.end();
+        }
+        catch { /* socket already gone */ }
+        return;
+    }
+    if (isSqliteBusy(err))
+        res.setHeader('Retry-After', '1');
+    if (mapped.status === 500) {
+        // The id lets an operator find the logged cause without the client seeing internal text.
+        sendJson(res, 500, { error: mapped.message, requestId });
+        return;
+    }
+    // RecallContractError keeps the shared {error} shape and adds `code` so clients branch without parsing prose.
+    if (err instanceof RecallContractError) {
+        sendJson(res, 400, { error: err.message, code: err.code });
+        return;
+    }
+    sendError(res, mapped.status, mapped.message);
+    // readBody hit the 1 MB cap mid-stream, so drop the socket rather than drain unbounded bytes.
+    if (err instanceof BodyTooLargeError)
+        req.destroy();
+}
+function setKeepAliveTimeouts(server) {
+    // The default 5s keepAliveTimeout closes idle sockets just as clients reuse them (ECONNRESET).
+    // headersTimeout must stay ABOVE keepAliveTimeout + keepAliveTimeoutBuffer (1s), or it closes idle reused sockets itself.
     server.keepAliveTimeout = 65_000;
     server.headersTimeout = 70_000;
-    await new Promise((resolve, reject) => {
+}
+function listenOn(server, port, host) {
+    return new Promise((resolve, reject) => {
         const onError = (err) => {
             server.removeListener('listening', onListening);
             reject(err);
@@ -345,17 +311,85 @@ export async function serve(opts) {
         };
         server.once('error', onError);
         server.once('listening', onListening);
-        server.listen(requestedPort, host);
+        server.listen(port, host);
     });
+}
+function installSignalHandlers(stop) {
+    let shuttingDown = false;
+    const gracefulShutdown = async (signal) => {
+        if (shuttingDown)
+            return;
+        shuttingDown = true;
+        log.warn(`received ${signal}, shutting down`);
+        try {
+            await stop();
+            process.exit(0);
+        }
+        catch (err) {
+            log.error(`error during stop: ${err instanceof Error ? err.message : String(err)}`, errorFields(err));
+            process.exit(1);
+        }
+    };
+    process.once('SIGTERM', () => { void gracefulShutdown('SIGTERM'); });
+    process.once('SIGINT', () => { void gracefulShutdown('SIGINT'); });
+}
+/**
+ * Boot the HTTP daemon on host:port and write the pidfile under hippoRoot.
+ *
+ * Refuses non-loopback hosts at boot unless
+ * HIPPO_REQUIRE_AUTH=1 is set. The auth middleware (buildContextWithAuth /
+ * requireAuth) has shipped and every route checks it except GET /health
+ * (public by design for platform health checks) and the two connector
+ * webhooks in PUBLIC_ROUTES, which are HMAC-gated by their own signing
+ * secrets and 404 when those secrets are unset. But the loopback
+ * no-auth fallback inside buildContextWithAuth still admits unauthenticated
+ * requests from a loopback remote address (unless they carry Forwarded,
+ * X-Forwarded-For/-Host/-Proto, X-Real-IP, Cf-Connecting-Ip or True-Client-Ip, which mark a same-host proxy and get
+ * a 401 like any keyless remote request), so binding to a non-loopback host
+ * is only safe once that fallback is disabled with HIPPO_REQUIRE_AUTH=1,
+ * which forces every request (loopback or not) through Bearer-token
+ * validation. Without that env var set, a non-loopback bind would expose the
+ * DB to the network with no auth, so we fail fast instead.
+ *
+ * Use port: 0 in tests to bind to an ephemeral port and read the actual
+ * port back via server.address() after listen.
+ */
+export async function serve(opts) {
+    const host = opts.host ?? '127.0.0.1';
+    const requestedPort = opts.port ?? Number(envPort() ?? 6789);
+    assertBindable(host);
+    await assertNoLiveServer(opts.hippoRoot);
+    // The server's start time. Single source of truth: it is returned by every
+    // GET /health response and (below) written into the pidfile, so detectServer
+    // can match the two and prove a pid-reusing impostor is not the real server.
+    const startedAt = new Date().toISOString();
+    const limiter = bootRateLimiter();
+    // Open /mcp/stream count per client key, so the cap is per server rather than per process.
+    const streamSlots = new Map();
+    const store = createStoreHolder(opts.hippoRoot);
+    const inflight = new Set();
+    const server = createServer((req, res) => {
+        res.once('finish', store.hold);
+        inflight.add(res);
+        res.once('close', () => inflight.delete(res));
+        const requestId = resolveRequestId(req.headers['x-request-id']);
+        requestIds.set(req, requestId);
+        res.setHeader('X-Request-Id', requestId);
+        withBusyWait(SERVER_DB_WAIT_MS, () => handleRequest(req, res, opts, startedAt, streamSlots, limiter)).catch((err) => {
+            replyWithFailure(req, res, err, requestId);
+        });
+    });
+    setKeepAliveTimeouts(server);
+    await listenOn(server, requestedPort, host);
     const address = server.address();
     if (!isAddressInfo(address)) {
         throw new Error('server.address() returned unexpected shape');
     }
     const addressInfo = address;
     const actualPort = addressInfo.port;
-    const url = `http://${host}:${actualPort}`;
+    const url = `http://${host.includes(':') ? `[${host}]` : host}:${actualPort}`;
     writePidfile(opts.hippoRoot, { port: actualPort, url, startedAt });
-    holdStore();
+    store.hold();
     let stopping = false;
     const stop = async () => {
         if (stopping)
@@ -363,33 +397,13 @@ export async function serve(opts) {
         stopping = true;
         // Remove the pidfile only if it still names this server. A newer server
         // may have started on this hippoRoot and rewritten the pidfile; an
-        // unconditional unlink here would orphan it. (v0.37.0 server-hardening.)
+        // unconditional unlink here would orphan it.
         removePidfileIfOwned(opts.hippoRoot, { pid: process.pid, startedAt });
         await drainAndClose(server, inflight, opts.shutdownDrainMs ?? 5000);
-        stopHolding = true;
-        if (heldDb)
-            closeHippoDb(heldDb);
-        heldDb = undefined;
+        store.release();
     };
-    if (opts.handleSignals) {
-        let shuttingDown = false;
-        const gracefulShutdown = async (signal) => {
-            if (shuttingDown)
-                return;
-            shuttingDown = true;
-            log.warn(`received ${signal}, shutting down`);
-            try {
-                await stop();
-                process.exit(0);
-            }
-            catch (err) {
-                log.error(`error during stop: ${err instanceof Error ? err.message : String(err)}`, errorFields(err));
-                process.exit(1);
-            }
-        };
-        process.once('SIGTERM', () => { void gracefulShutdown('SIGTERM'); });
-        process.once('SIGINT', () => { void gracefulShutdown('SIGINT'); });
-    }
+    if (opts.handleSignals)
+        installSignalHandlers(stop);
     return { port: actualPort, url, stop, server };
 }
 //# sourceMappingURL=server.js.map

package/dist/shared.d.ts

@@ -5,7 +5,8 @@
  * Local .hippo/ stores are per-project.
  */
 import { MemoryEntry } from './memory.js';
-import type { SearchResult, ResultCost } from './search/types.js';
+import { type SearchResult, type ResultCost } from './search/types.js';
+import type { HybridVectorCandidates } from './search/vector.js';
 import type { DatabaseSyncLike } from './db.js';
 /**
  * Returns the path to the global Hippo store.
@@ -55,10 +56,10 @@ export interface HybridSearchOptions extends SearchOptions {
     asOf?: string;
     /** Budget cost per result, spent the same way in each store and in the merged list. */
     cost?: ResultCost;
-    /** v0.30 / E4 — propagated to underlying hybridSearch calls.
+    /** Propagated to underlying hybridSearch calls.
      *  Per-call > env HIPPO_SUMMARY_DEBOOST > 0.85 default. */
     summaryDeboost?: number;
-    /** v0.30 / E4 — propagated. Default true (1.05 boost if rebuilt within 7d). */
+    /** Propagated. Default true (1.05 boost if rebuilt within 7d). */
     summaryFreshness?: boolean;
     /** v39 memory scope isolation: optional admission predicate applied to the
      *  loaded candidate entries of BOTH stores BEFORE ranking, cross-store
@@ -66,7 +67,7 @@ export interface HybridSearchOptions extends SearchOptions {
      *  its admitted duplicate in the dedupe pass, or saturate the budget.
      *  Default undefined = unchanged behavior (recall paths never set it). */
     entryFilter?: (entry: MemoryEntry) => boolean;
-    /** v1.25.0 — recall-mode scope filter, consumed by `searchBothHybrid` only.
+    /** Recall-mode scope filter, consumed by `searchBothHybrid` only.
      *  ABSENT (undefined) is the only unfiltered mode: both stores load via
      *  `loadSearchEntries` unchanged (background pipelines / eval callers).
      *  PRESENT switches the internal loads to `loadRecallSearchEntries` (SQL
@@ -93,6 +94,14 @@ export interface HybridSearchOptions extends SearchOptions {
  * Async version of searchBoth that calls hybridSearch instead of search.
  */
 export declare function searchBothHybrid(query: string, localRoot: string, globalRoot: string, options?: HybridSearchOptions): Promise<SearchResult[]>;
+/** Hybrid ranking of rows already loaded from each store: the local bump, one copy per text, then the shared budget. */
+export declare function rankBothStores(query: string, roots: {
+    local: string;
+    global: string;
+}, entries: {
+    local: MemoryEntry[];
+    global: MemoryEntry[];
+}, vectorCandidates: HybridVectorCandidates, options?: HybridSearchOptions): Promise<SearchResult[]>;
 /** Tags whose rows only a hand-run share or promote may copy to the global store; derived rows inherit them. */
 export declare const NEVER_AUTO_SHARE_TAGS: ReadonlySet<string>;
 export declare function neverAutoShareTags(sources: readonly MemoryEntry[]): string[];
@@ -117,7 +126,7 @@ export declare function shareMemory(localRoot: string, id: string, options?: {
  * List all projects that have contributed memories to the global store.
  * Parses the source field for 'shared:<project>:' or 'promoted:<path>' patterns.
  *
- * D4 v1.12.10: `tenantId` is now optional. When provided, the global entries
+ * `tenantId` is optional. When provided, the global entries
  * are filtered to that tenant before aggregation — matches every other
  * read path's default-safe behaviour. When undefined, host-wide (back-compat
  * for legacy callers like CLI standalone + dashboard internal use). Operators
@@ -129,25 +138,28 @@ export declare function listPeers(globalRoot?: string, tenantId?: string): Array
     count: number;
     latest: string;
 }>;
+type AutoShareStats = {
+    secretSkipped: number;
+    rejectedSkipped?: number;
+    neverAutoShareSkipped?: number;
+};
 /**
  * Auto-share: local memories with high transfer scores, not already global, no NEVER_AUTO_SHARE_TAGS tag.
  * Returns the list of shared entries.
  *
- * L9: `options.tenantId` is opt-in. When provided, the LOCAL-entries read is
+ * `options.tenantId` is opt-in. When provided, the LOCAL-entries read is
  * scoped to that tenant. When undefined, the local read is host-wide (current
  * behaviour). The GLOBAL-entries read is always unioned — the global root IS
- * the cross-tenant aggregate by design. The only intentional unscoped
- * internal caller as of v1.12.1 is `api.sleep` (`src/api.ts:2041`), which
- * passes options without tenantId because `sleep` is host-wide by intent;
- * see `src/api.ts:2073-2077` for the cross-tenant dedup rationale.
+ * the cross-tenant aggregate by design. `api.sleep` passes no tenantId
+ * because `sleep` is host-wide by intent.
  *
- * v1.25.0: `options.stats` is an opt-in out-param. When provided,
+ * `options.stats` is an opt-in out-param. When provided,
  * `stats.secretSkipped` is incremented once per row that passed every OTHER
  * admission gate (transfer score, not-already-global) and was withheld SOLELY
  * by the secret veto — i.e. it counts shares actually prevented, not secret
  * rows merely present. Filled identically under `dryRun`.
  *
- * AT1: `stats.rejectedSkipped` (optional) is incremented once per candidate
+ * `stats.rejectedSkipped` (optional) is incremented once per candidate
  * refused by the GLOBAL store's rejection tombstone (RejectedValueError from
  * shareMemory -> writeEntry). Unlike secretSkipped, this can only be
  * detected by attempting the write — `dryRun` returns candidates before the
@@ -159,11 +171,7 @@ export declare function autoShare(localRoot: string, options?: {
     minScore?: number;
     dryRun?: boolean;
     tenantId?: string;
-    stats?: {
-        secretSkipped: number;
-        rejectedSkipped?: number;
-        neverAutoShareSkipped?: number;
-    };
+    stats?: AutoShareStats;
 }): MemoryEntry[];
 /**
  * Copy all global memories into the local store.
@@ -173,4 +181,5 @@ export declare function autoShare(localRoot: string, options?: {
 export declare function syncGlobalToLocal(localRoot: string, globalRoot: string, opts?: {
     includeCrossProject?: boolean;
 }): number;
+export {};
 //# sourceMappingURL=shared.d.ts.map