@remnic/core 1.1.30 → 9.3.515

package/src/access-http.ts

@@ -5,6 +5,7 @@ import { existsSync } from "node:fs";
 import { readFile } from "node:fs/promises";
 import path from "node:path";
 import { fileURLToPath, URL } from "node:url";
+import { gunzipSync } from "node:zlib";
 import { log } from "./logger.js";
 import { EngramAccessInputError, type EngramAccessService } from "./access-service.js";
 import { EngramMcpServer } from "./access-mcp.js";
@@ -24,6 +25,7 @@ import {
   type GraphEvent,
 } from "./graph-events.js";
 import { expandTildePath } from "./utils/path.js";
+import { projectTagProjectId } from "./coding/coding-namespace.js";
 
 export interface EngramAccessHttpServerOptions {
   service: EngramAccessService;
@@ -136,6 +138,36 @@ function summarizeHttpRequest(req: IncomingMessage): string {
   }
 }
 
+function parseStrictIntegerQuery(
+  raw: string | null,
+  field: string,
+  defaultValue: number,
+  minValue: number,
+): number {
+  if (raw === null) return defaultValue;
+  if (!/^(?:0|[1-9]\d*)$/.test(raw)) {
+    throw new HttpError(400, `${field} must be an integer`, `invalid_${field}`);
+  }
+  const value = Number(raw);
+  if (!Number.isInteger(value) || value < minValue) {
+    throw new HttpError(400, `${field} must be an integer >= ${minValue}`, `invalid_${field}`);
+  }
+  return value;
+}
+
+function parseMemorySort(raw: string | null): "updated_desc" | "updated_asc" | "created_desc" | "created_asc" | undefined {
+  if (raw === null) return undefined;
+  if (
+    raw === "updated_desc" ||
+    raw === "updated_asc" ||
+    raw === "created_desc" ||
+    raw === "created_asc"
+  ) {
+    return raw;
+  }
+  throw new HttpError(400, "sort must be one of updated_desc|updated_asc|created_desc|created_asc", "invalid_sort");
+}
+
 /**
  * Decode a `:peerId` URL path segment, converting malformed percent-encoded
  * input (e.g., `%E0%A4%A`) into a 400 client error rather than letting
@@ -149,6 +181,21 @@ function decodePeerIdSegment(raw: string): string {
   }
 }
 
+function codingContextFromProjectTag(projectTag: string): {
+  projectId: string;
+  branch: string | null;
+  rootPath: string;
+  defaultBranch: string | null;
+} {
+  const projectId = projectTagProjectId(projectTag);
+  return {
+    projectId,
+    branch: null,
+    rootPath: projectId,
+    defaultBranch: null,
+  };
+}
+
 export class EngramAccessHttpServer {
   private readonly service: EngramAccessService;
   private readonly host: string;
@@ -321,14 +368,15 @@ export class EngramAccessHttpServer {
   }
 
   /** Cache for per-request identity resolution (avoids double adapter resolution) */
-  private identityCache = new WeakMap<IncomingMessage, { principal?: string; namespace?: string }>();
+  private identityCache = new WeakMap<IncomingMessage, { principal?: string; namespace?: string; sessionKey?: string }>();
 
-  /** Resolve principal and namespace from request headers and adapter identity */
-  private resolveRequestIdentity(req: IncomingMessage): { principal?: string; namespace?: string } {
+  /** Resolve principal, namespace, and session key from request headers and adapter identity */
+  private resolveRequestIdentity(req: IncomingMessage): { principal?: string; namespace?: string; sessionKey?: string } {
     const cached = this.identityCache.get(req);
     if (cached) return cached;
     let principal: string | undefined;
     let namespace: string | undefined;
+    let sessionKey: string | undefined;
 
     // Explicit header override takes priority for principal
     if (this.trustPrincipalHeader) {
@@ -342,20 +390,22 @@ export class EngramAccessHttpServer {
       }
     }
 
-    // Try adapter-based identity resolution for both principal and namespace
+    if (!principal) {
+      principal = this.authenticatedPrincipal;
+    }
+
+    // Try adapter-based identity resolution for namespace and, only when no
+    // server principal is configured, an adapter-owned default principal.
     const adapterIdentity = this.resolveAdapterIdentity(req);
     if (adapterIdentity) {
       if (!principal) {
         principal = adapterIdentity.principal;
       }
       namespace = adapterIdentity.namespace;
+      sessionKey = adapterIdentity.sessionKey;
     }
 
-    if (!principal) {
-      principal = this.authenticatedPrincipal;
-    }
-
-    const result = { principal, namespace };
+    const result = { principal, namespace, sessionKey };
     this.identityCache.set(req, result);
     return result;
   }
@@ -561,9 +611,17 @@ export class EngramAccessHttpServer {
     // doctor` (PR 8) surfaces the attached context.
     if (req.method === "POST" && pathname === "/engram/v1/coding-context") {
       const body = await this.readValidatedBody(req, "setCodingContext");
+      const codingContext =
+        body.codingContext !== undefined
+          ? body.codingContext
+          : typeof body.projectTag === "string"
+            ? codingContextFromProjectTag(body.projectTag)
+            : (() => {
+                throw new EngramAccessInputError("codingContext or projectTag is required");
+              })();
       this.service.setCodingContext({
         sessionKey: body.sessionKey,
-        codingContext: body.codingContext,
+        codingContext,
       });
       this.respondJson(res, 200, { ok: true });
       return;
@@ -601,6 +659,7 @@ export class EngramAccessHttpServer {
         namespace: this.resolveNamespace(req, body.namespace),
         principal: this.resolveRequestPrincipal(req),
         mode: body.mode,
+        passphrase: body.passphrase,
       });
       this.recordWriteRateLimitHit();
       this.respondJson(res, 200, result);
@@ -645,6 +704,42 @@ export class EngramAccessHttpServer {
       return;
     }
 
+    if (
+      req.method === "GET" &&
+      (pathname === "/engram/v1/offline-sync/snapshot-stream" ||
+        pathname === "/remnic/v1/offline-sync/snapshot-stream")
+    ) {
+      const includeTranscriptsRaw = parsed.searchParams.get("include_transcripts");
+      const includeContentRaw = parsed.searchParams.get("content");
+      if (
+        includeTranscriptsRaw !== null &&
+        includeTranscriptsRaw !== "true" &&
+        includeTranscriptsRaw !== "false"
+      ) {
+        throw new EngramAccessInputError(
+          `include_transcripts must be one of: true, false (got: ${includeTranscriptsRaw})`,
+        );
+      }
+      if (
+        includeContentRaw !== null &&
+        includeContentRaw !== "false"
+      ) {
+        throw new EngramAccessInputError("snapshot-stream content must be false");
+      }
+      const namespaceParam = parsed.searchParams.get("namespace");
+      const result = await this.service.offlineSyncSnapshotStream({
+        namespace: this.resolveNamespace(
+          req,
+          namespaceParam && namespaceParam.length > 0 ? namespaceParam : undefined,
+        ),
+        principal: this.resolveRequestPrincipal(req),
+        includeTranscripts: includeTranscriptsRaw !== "false",
+        includeContent: false,
+      });
+      await this.respondOfflineSnapshotStream(res, result);
+      return;
+    }
+
     if (
       req.method === "POST" &&
       (pathname === "/engram/v1/offline-sync/snapshot" || pathname === "/remnic/v1/offline-sync/snapshot")
@@ -929,6 +1024,7 @@ export class EngramAccessHttpServer {
       const response = await this.service.lcmSearch({
         query: body.query,
         sessionKey: body.sessionKey,
+        sessionPrefix: body.sessionPrefix,
         namespace: this.resolveNamespace(req, body.namespace),
         authenticatedPrincipal: this.resolveRequestPrincipal(req),
         limit: body.limit,
@@ -1035,15 +1131,9 @@ export class EngramAccessHttpServer {
     }
 
     if (req.method === "GET" && pathname === "/engram/v1/memories") {
-      const limitRaw = parseInt(parsed.searchParams.get("limit") ?? "50", 10);
-      const offsetRaw = parseInt(parsed.searchParams.get("offset") ?? "0", 10);
-      const sortParam = parsed.searchParams.get("sort") ?? undefined;
-      const sort = sortParam === "updated_desc"
-        || sortParam === "updated_asc"
-        || sortParam === "created_desc"
-        || sortParam === "created_asc"
-        ? sortParam
-        : undefined;
+      const limit = parseStrictIntegerQuery(parsed.searchParams.get("limit"), "limit", 50, 1);
+      const offset = parseStrictIntegerQuery(parsed.searchParams.get("offset"), "offset", 0, 0);
+      const sort = parseMemorySort(parsed.searchParams.get("sort"));
       const response = await this.service.memoryBrowse({
         query: parsed.searchParams.get("q") ?? undefined,
         status: parsed.searchParams.get("status") ?? undefined,
@@ -1051,8 +1141,8 @@ export class EngramAccessHttpServer {
         namespace: parsed.searchParams.get("namespace") ?? undefined,
         authenticatedPrincipal: this.resolveRequestPrincipal(req),
         sort,
-        limit: Number.isFinite(limitRaw) ? limitRaw : 50,
-        offset: Number.isFinite(offsetRaw) ? offsetRaw : 0,
+        limit,
+        offset,
       });
       this.respondJson(res, 200, response);
       return;
@@ -1071,21 +1161,20 @@ export class EngramAccessHttpServer {
     if (req.method === "GET" && timelineMatch) {
       const memoryId = decodeURIComponent(timelineMatch[1] ?? "");
       const namespace = parsed.searchParams.get("namespace") ?? undefined;
-      const limitRaw = parseInt(parsed.searchParams.get("limit") ?? "200", 10);
-      const limit = Number.isFinite(limitRaw) ? limitRaw : 200;
+      const limit = parseStrictIntegerQuery(parsed.searchParams.get("limit"), "limit", 200, 1);
       const response = await this.service.memoryTimeline(memoryId, namespace, limit, this.resolveRequestPrincipal(req));
       this.respondJson(res, response.found ? 200 : 404, response);
       return;
     }
 
     if (req.method === "GET" && pathname === "/engram/v1/entities") {
-      const limitRaw = parseInt(parsed.searchParams.get("limit") ?? "50", 10);
-      const offsetRaw = parseInt(parsed.searchParams.get("offset") ?? "0", 10);
+      const limit = parseStrictIntegerQuery(parsed.searchParams.get("limit"), "limit", 50, 1);
+      const offset = parseStrictIntegerQuery(parsed.searchParams.get("offset"), "offset", 0, 0);
       const response = await this.service.entityList({
         namespace: parsed.searchParams.get("namespace") ?? undefined,
         query: parsed.searchParams.get("q") ?? undefined,
-        limit: Number.isFinite(limitRaw) ? limitRaw : 50,
-        offset: Number.isFinite(offsetRaw) ? offsetRaw : 0,
+        limit,
+        offset,
       });
       this.respondJson(res, 200, response);
       return;
@@ -1153,16 +1242,16 @@ export class EngramAccessHttpServer {
     }
 
     if (req.method === "GET" && pathname === "/engram/v1/trust-zones/records") {
-      const limitRaw = parseInt(parsed.searchParams.get("limit") ?? "25", 10);
-      const offsetRaw = parseInt(parsed.searchParams.get("offset") ?? "0", 10);
+      const limit = parseStrictIntegerQuery(parsed.searchParams.get("limit"), "limit", 25, 1);
+      const offset = parseStrictIntegerQuery(parsed.searchParams.get("offset"), "offset", 0, 0);
       const response = await this.service.trustZoneBrowse({
         query: parsed.searchParams.get("q") ?? undefined,
         zone: parseTrustZoneFilter(parsed.searchParams.get("zone")),
         kind: parseTrustZoneKindFilter(parsed.searchParams.get("kind")),
         sourceClass: parseTrustZoneSourceClassFilter(parsed.searchParams.get("sourceClass")),
         namespace: parsed.searchParams.get("namespace") ?? undefined,
-        limit: Number.isFinite(limitRaw) ? limitRaw : 25,
-        offset: Number.isFinite(offsetRaw) ? offsetRaw : 0,
+        limit,
+        offset,
       }, this.resolveRequestPrincipal(req));
       this.respondJson(res, 200, response);
       return;
@@ -1308,7 +1397,7 @@ export class EngramAccessHttpServer {
         return;
       }
       const namespace = parsed.searchParams.get("namespace") ?? undefined;
-      const limitRaw = parseInt(parsed.searchParams.get("limit") ?? "50", 10);
+      const limit = parseStrictIntegerQuery(parsed.searchParams.get("limit"), "limit", 50, 1);
       const {
         isDefaultReviewNamespace,
         listPairs,
@@ -1323,7 +1412,7 @@ export class EngramAccessHttpServer {
         filter: rawFilter as "all" | "unresolved" | "contradicts" | "independent" | "duplicates" | "needs-user",
         namespace: reviewNamespace,
         includeUnscopedForNamespace,
-        limit: Number.isFinite(limitRaw) ? limitRaw : 50,
+        limit,
       });
       this.respondJson(res, 200, result);
       return;
@@ -1865,8 +1954,11 @@ export class EngramAccessHttpServer {
       return typeof raw === "string" ? raw.trim() : undefined;
     })();
     const mcpCorrelationId = correlationIdStore.getStore() ?? randomUUID();
+    const requestIdentity = this.resolveRequestIdentity(req);
     const response = await this.mcpServer.handleRequest(request, {
-      principalOverride: this.resolveRequestPrincipal(req),
+      principalOverride: requestIdentity.principal,
+      namespaceOverride: requestIdentity.namespace,
+      sessionKeyOverride: requestIdentity.sessionKey,
       sessionId,
       correlationId: mcpCorrelationId,
     });
@@ -1905,6 +1997,62 @@ export class EngramAccessHttpServer {
     res.end(body);
   }
 
+  private async respondOfflineSnapshotStream(
+    res: ServerResponse,
+    snapshot: Awaited<ReturnType<EngramAccessService["offlineSyncSnapshotStream"]>>,
+  ): Promise<void> {
+    res.statusCode = 200;
+    res.setHeader("content-type", "application/x-ndjson; charset=utf-8");
+    res.setHeader("cache-control", "no-store");
+    const cid = correlationIdStore.getStore();
+    if (cid) {
+      res.setHeader("x-request-id", cid);
+    }
+    const waitForDrainOrClose = async (): Promise<boolean> => new Promise((resolve, reject) => {
+      const cleanup = () => {
+        res.off("drain", onDrain);
+        res.off("close", onClose);
+        res.off("error", onError);
+      };
+      const onDrain = () => {
+        cleanup();
+        resolve(true);
+      };
+      const onClose = () => {
+        cleanup();
+        resolve(false);
+      };
+      const onError = (error: Error) => {
+        cleanup();
+        reject(error);
+      };
+      res.once("drain", onDrain);
+      res.once("close", onClose);
+      res.once("error", onError);
+    });
+    const writeLine = async (payload: unknown): Promise<boolean> => {
+      if (res.destroyed || res.writableEnded) return false;
+      if (res.write(`${JSON.stringify(payload)}\n`)) return true;
+      if (res.destroyed || res.writableEnded) return false;
+      return waitForDrainOrClose();
+    };
+    if (!await writeLine({
+      type: "snapshot",
+      namespace: snapshot.namespace,
+      format: snapshot.format,
+      schemaVersion: snapshot.schemaVersion,
+      createdAt: snapshot.createdAt,
+      sourceId: snapshot.sourceId,
+      includeTranscripts: snapshot.includeTranscripts,
+    })) return;
+    for await (const file of snapshot.files) {
+      if (!await writeLine({ type: "file", file })) return;
+    }
+    if (!res.destroyed && !res.writableEnded) {
+      res.end();
+    }
+  }
+
   private respondBinary(
     res: ServerResponse,
     status: number,
@@ -1963,6 +2111,10 @@ export class EngramAccessHttpServer {
     req: IncomingMessage,
     maxBodyBytes = this.maxBodyBytes,
   ): Promise<Record<string, unknown>> {
+    const encoding = (this.readOptionalHeader(req, "content-encoding") ?? "identity").toLowerCase();
+    if (encoding !== "identity" && encoding !== "gzip") {
+      throw new HttpError(415, "unsupported_content_encoding", "unsupported_content_encoding");
+    }
     const chunks: Buffer[] = [];
     let total = 0;
     for await (const chunk of req) {
@@ -1974,7 +2126,21 @@ export class EngramAccessHttpServer {
       chunks.push(buffer);
     }
     if (chunks.length === 0) return {};
-    const raw = Buffer.concat(chunks).toString("utf-8").trim();
+    let body = Buffer.concat(chunks, total);
+    if (encoding === "gzip") {
+      try {
+        body = gunzipSync(body, { maxOutputLength: maxBodyBytes });
+      } catch (error) {
+        if ((error as NodeJS.ErrnoException).code === "ERR_BUFFER_TOO_LARGE") {
+          throw new HttpError(413, "request_body_too_large", "request_body_too_large");
+        }
+        throw new HttpError(400, "invalid_gzip_body", "invalid_gzip_body");
+      }
+      if (body.byteLength > maxBodyBytes) {
+        throw new HttpError(413, "request_body_too_large", "request_body_too_large");
+      }
+    }
+    const raw = body.toString("utf-8").trim();
     if (raw.length === 0) return {};
     let parsed: unknown;
     try {

package/src/access-idempotency.ts

@@ -1,6 +1,6 @@
 import { mkdir, open, readFile, rename, stat, unlink, utimes, writeFile } from "node:fs/promises";
 import path from "node:path";
-import { createHash } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
 
 type AccessIdempotencyEntry = {
   recordedAt: string;
@@ -13,6 +13,8 @@ type AccessIdempotencyTestHooks = {
   lockTimeoutMs?: number;
   staleLockMs?: number;
   lockHeartbeatMs?: number;
+  beforeLockOwnerWrite?: (lockPath: string, ownerToken: string) => Promise<void> | void;
+  beforeStaleLockUnlink?: () => Promise<void> | void;
 };
 
 let testHooks: AccessIdempotencyTestHooks | null = null;
@@ -171,8 +173,17 @@ export class AccessIdempotencyStore {
     const startedAt = Date.now();
 
     while (true) {
+      const ownerToken = `${process.pid}:${randomUUID()}`;
       try {
         const handle = await open(lockPath, "wx");
+        try {
+          await testHooks?.beforeLockOwnerWrite?.(lockPath, ownerToken);
+          await handle.writeFile(ownerToken, "utf-8");
+        } catch (error) {
+          await handle.close().catch(() => undefined);
+          await unlink(lockPath).catch(() => undefined);
+          throw error;
+        }
         let heartbeat: NodeJS.Timeout | null = null;
         if (lockHeartbeatMs > 0) {
           heartbeat = setInterval(() => {
@@ -187,14 +198,24 @@ export class AccessIdempotencyStore {
             clearInterval(heartbeat);
           }
           await handle.close().catch(() => undefined);
-          await unlink(lockPath).catch(() => undefined);
+          await unlinkLockIfOwner(lockPath, ownerToken, staleLockMs);
         }
       } catch (error) {
         if (!isAlreadyExistsError(error)) throw error;
         try {
           const lockStat = await stat(lockPath);
+          const lockOwner = await readLockOwner(lockPath);
           if (Date.now() - lockStat.mtimeMs > staleLockMs) {
-            await unlink(lockPath).catch(() => undefined);
+            await testHooks?.beforeStaleLockUnlink?.();
+            const removed = await unlinkStaleLockIfUnchanged({
+              lockPath,
+              observedOwner: lockOwner,
+              observedMtimeMs: lockStat.mtimeMs,
+              staleLockMs,
+            });
+            if (!removed) {
+              await sleep(10);
+            }
             continue;
           }
         } catch {
@@ -227,6 +248,118 @@ function isAlreadyExistsError(error: unknown): boolean {
   return typeof error === "object" && error !== null && "code" in error && error.code === "EEXIST";
 }
 
+async function readLockOwner(lockPath: string): Promise<string | null> {
+  try {
+    return await readFile(lockPath, "utf-8");
+  } catch {
+    return null;
+  }
+}
+
+async function unlinkLockIfOwner(lockPath: string, ownerToken: string, staleLockMs: number): Promise<void> {
+  const reclaimHandle = await openStaleReclaimLock(`${lockPath}.reclaim`, staleLockMs);
+  if (!reclaimHandle) return;
+  try {
+    await reclaimHandle.writeFile(`${process.pid}:${Date.now()}`, "utf-8");
+    await unlinkLockIfOwnerWhileReclaimHeld(lockPath, ownerToken);
+  } finally {
+    await reclaimHandle.close().catch(() => undefined);
+    await unlink(`${lockPath}.reclaim`).catch(() => undefined);
+  }
+}
+
+async function unlinkLockIfOwnerWhileReclaimHeld(lockPath: string, ownerToken: string): Promise<void> {
+  let handle: Awaited<ReturnType<typeof open>>;
+  try {
+    handle = await open(lockPath, "r");
+  } catch {
+    return;
+  }
+  try {
+    const openedStat = await handle.stat();
+    const currentOwner = await handle.readFile("utf-8");
+    if (currentOwner !== ownerToken) return;
+    let currentStat: Awaited<ReturnType<typeof stat>>;
+    try {
+      currentStat = await stat(lockPath);
+    } catch {
+      return;
+    }
+    if (!isSameFileIdentity(openedStat, currentStat)) return;
+  } finally {
+    await handle.close().catch(() => undefined);
+  }
+  await unlink(lockPath).catch(() => undefined);
+}
+
+function isSameFileIdentity(left: Awaited<ReturnType<typeof stat>>, right: Awaited<ReturnType<typeof stat>>): boolean {
+  return left.dev === right.dev && left.ino === right.ino;
+}
+
+async function unlinkStaleLockIfUnchanged(options: {
+  lockPath: string;
+  observedOwner: string | null;
+  observedMtimeMs: number;
+  staleLockMs: number;
+}): Promise<boolean> {
+  const reclaimPath = `${options.lockPath}.reclaim`;
+  const reclaimHandle = await openStaleReclaimLock(reclaimPath, options.staleLockMs);
+  if (!reclaimHandle) return false;
+  try {
+    await reclaimHandle.writeFile(`${process.pid}:${Date.now()}`, "utf-8");
+    return await unlinkStaleLockIfStillOwned(options);
+  } finally {
+    await reclaimHandle.close().catch(() => undefined);
+    await unlink(reclaimPath).catch(() => undefined);
+  }
+}
+
+async function openStaleReclaimLock(
+  reclaimPath: string,
+  staleLockMs: number,
+): Promise<Awaited<ReturnType<typeof open>> | null> {
+  try {
+    return await open(reclaimPath, "wx");
+  } catch (error) {
+    if (!isAlreadyExistsError(error)) throw error;
+  }
+
+  try {
+    const reclaimStat = await stat(reclaimPath);
+    if (Date.now() - reclaimStat.mtimeMs <= staleLockMs) return null;
+    await unlink(reclaimPath);
+  } catch {
+    return null;
+  }
+
+  try {
+    return await open(reclaimPath, "wx");
+  } catch (error) {
+    if (isAlreadyExistsError(error)) return null;
+    throw error;
+  }
+}
+
+async function unlinkStaleLockIfStillOwned(options: {
+  lockPath: string;
+  observedOwner: string | null;
+  observedMtimeMs: number;
+  staleLockMs: number;
+}): Promise<boolean> {
+  let currentStat: Awaited<ReturnType<typeof stat>>;
+  try {
+    currentStat = await stat(options.lockPath);
+  } catch {
+    return true;
+  }
+  if (currentStat.mtimeMs !== options.observedMtimeMs) return false;
+  if (Date.now() - currentStat.mtimeMs <= options.staleLockMs) return false;
+  const currentOwner = await readLockOwner(options.lockPath);
+  if (currentOwner !== options.observedOwner) return false;
+  await unlink(options.lockPath);
+  return true;
+}
+
 function sleep(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }