chainlesschain 0.166.85 → 0.166.87

package/src/lib/context-memory-kernel/memory-query-index.js

@@ -0,0 +1,278 @@
+import { createHash } from "node:crypto";
+
+export const MEMORY_QUERY_INDEX_SCHEMA = "chainlesschain.memory-query-index/v1";
+const CURSOR_SCHEMA = "chainlesschain.memory-list-cursor/v1";
+
+export function queryDigest(bytes) {
+  return `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
+}
+
+export function compareMemoryRows(left, right) {
+  return (
+    right.importance - left.importance ||
+    right.updatedAt.localeCompare(left.updatedAt, "en") ||
+    left.memoryId.localeCompare(right.memoryId, "en")
+  );
+}
+
+// No content, evidence, or audit is copied into the derived index. The manifest
+// binds these bytes to the immutable authority shard from which they were made.
+export function encodeQueryIndex(state) {
+  const rows = Object.values(state.records)
+    .map((record) => ({
+      memoryId: record.memoryId,
+      category: record.category,
+      scope: record.scope,
+      scopeId: record.scopeId ?? null,
+      state: record.state,
+      importance: record.importance,
+      updatedAt: record.updatedAt,
+      tags: record.tags,
+      allowedSinks: record.allowedSinks,
+    }))
+    .sort(compareMemoryRows);
+  return Buffer.from(
+    JSON.stringify({
+      schema: MEMORY_QUERY_INDEX_SCHEMA,
+      shardDigest: state.digest,
+      rows,
+    }),
+  );
+}
+
+function scopeKey(scope, scopeId) {
+  return JSON.stringify([scope, scopeId ?? null]);
+}
+
+export function compileQueryIndex(bytes, descriptor) {
+  const value = JSON.parse(bytes.toString("utf8"));
+  if (
+    value.schema !== MEMORY_QUERY_INDEX_SCHEMA ||
+    value.shardDigest !== descriptor.digest ||
+    !Array.isArray(value.rows) ||
+    value.rows.length !== descriptor.recordCount
+  ) {
+    throw new Error("query index does not match authority shard");
+  }
+  const fields = ["category", "scope", "state", "tag", "sink"];
+  const postings = Object.fromEntries(
+    fields.map((field) => [field, new Map()]),
+  );
+  const add = (field, key, index) => {
+    const map = postings[field];
+    if (!map.has(key)) map.set(key, []);
+    map.get(key).push(index);
+  };
+  value.rows.forEach((row, index) => {
+    add("category", row.category, index);
+    add("scope", scopeKey(row.scope, row.scopeId), index);
+    add("state", row.state, index);
+    for (const tag of row.tags) add("tag", tag, index);
+    for (const sink of row.allowedSinks) add("sink", sink, index);
+  });
+  return { rows: value.rows, postings };
+}
+
+export function normalizeListQuery(options = {}) {
+  const result = { includeTombstones: options.includeTombstones === true };
+  if (options.category) result.category = String(options.category);
+  if (options.sink) result.sink = String(options.sink);
+  if (options.scopeAdmissions !== undefined) {
+    if (!Array.isArray(options.scopeAdmissions))
+      throw new TypeError("scopeAdmissions must be an array");
+    result.scopeAdmissions = options.scopeAdmissions.map(
+      ({ scope, scopeId }) => ({
+        scope,
+        scopeId: scopeId ?? null,
+      }),
+    );
+  }
+  if (options.tags !== undefined) {
+    if (!Array.isArray(options.tags))
+      throw new TypeError("tags must be an array");
+    result.tags = [...new Set(options.tags.map(String))].sort();
+  }
+  if (options.states !== undefined) {
+    if (!Array.isArray(options.states))
+      throw new TypeError("states must be an array");
+    result.states = [...new Set(options.states.map(String))].sort();
+  }
+  return result;
+}
+
+export function matchesListQuery(row, query) {
+  return (
+    (query.includeTombstones || !["deleted", "purged"].includes(row.state)) &&
+    (!query.category || row.category === query.category) &&
+    (!query.states || query.states.includes(row.state)) &&
+    (!query.scopeAdmissions ||
+      query.scopeAdmissions.some(
+        (scope) =>
+          scope.scope === row.scope &&
+          (scope.scopeId ?? null) === (row.scopeId ?? null),
+      )) &&
+    (!query.sink ||
+      row.allowedSinks.includes("*") ||
+      row.allowedSinks.includes(query.sink)) &&
+    (!query.tags?.length || row.tags.some((tag) => query.tags.includes(tag)))
+  );
+}
+
+// Each posting is already in the public list order. Pick the narrowest posting
+// before applying the remaining predicates; no full-record scan or top-k before
+// permission filtering. Recall uses these gates but retains kernel ranking.
+export function* selectQueryRows(index, query, after = null) {
+  const candidates = [];
+  const union = (field, keys) => {
+    const lists = keys
+      .map((key) => index.postings[field].get(key))
+      .filter(Boolean);
+    return lists.length < 2
+      ? lists[0] || []
+      : [...new Set(lists.flat())].sort((a, b) => a - b);
+  };
+  if (query.category)
+    candidates.push(index.postings.category.get(query.category) || []);
+  if (query.scopeAdmissions)
+    candidates.push(
+      union(
+        "scope",
+        query.scopeAdmissions.map((scope) =>
+          scopeKey(scope.scope, scope.scopeId),
+        ),
+      ),
+    );
+  if (query.states) candidates.push(union("state", query.states));
+  if (query.tags?.length) candidates.push(union("tag", query.tags));
+  if (query.sink) candidates.push(union("sink", ["*", query.sink]));
+  candidates.sort((a, b) => a.length - b.length);
+  const positions = candidates[0];
+  const length = positions ? positions.length : index.rows.length;
+  const rowAt = (offset) => index.rows[positions ? positions[offset] : offset];
+  let start = 0;
+  if (after) {
+    let end = length;
+    while (start < end) {
+      const middle = Math.floor((start + end) / 2);
+      if (compareMemoryRows(rowAt(middle), after) <= 0) start = middle + 1;
+      else end = middle;
+    }
+  }
+  for (let offset = start; offset < length; offset++) {
+    const row = rowAt(offset);
+    if (matchesListQuery(row, query)) yield row;
+  }
+}
+
+export function* mergeQueryRows(heads) {
+  const heap = [];
+  const less = (left, right) => compareMemoryRows(left.row, right.row) < 0;
+  const push = (head) => {
+    let at = heap.length;
+    heap.push(head);
+    while (at > 0) {
+      const parent = Math.floor((at - 1) / 2);
+      if (!less(head, heap[parent])) break;
+      heap[at] = heap[parent];
+      at = parent;
+    }
+    heap[at] = head;
+  };
+  for (const head of heads) push(head);
+  while (heap.length) {
+    const first = heap[0];
+    const last = heap.pop();
+    if (heap.length) {
+      let at = 0;
+      while (at * 2 + 1 < heap.length) {
+        let child = at * 2 + 1;
+        if (child + 1 < heap.length && less(heap[child + 1], heap[child]))
+          child++;
+        if (!less(heap[child], last)) break;
+        heap[at] = heap[child];
+        at = child;
+      }
+      heap[at] = last;
+    }
+    yield { bucket: first.bucket, row: first.row };
+    const next = first.iterator.next();
+    if (!next.done) push({ ...first, row: next.value });
+  }
+}
+
+export function cursorError(reason) {
+  return Object.assign(new Error(`Memory list cursor is invalid: ${reason}`), {
+    code: "CONTEXT_MEMORY_CURSOR_INVALID",
+  });
+}
+
+export function listCursorBinding(filePath, generation, query) {
+  return queryDigest(
+    JSON.stringify({
+      filePath,
+      generation,
+      query,
+      order: "importance-desc,updatedAt-desc,memoryId-en-asc/v1",
+    }),
+  );
+}
+
+export function decodeListCursor(cursor, binding) {
+  if (cursor == null) return null;
+  if (
+    typeof cursor !== "string" ||
+    cursor.length > 4096 ||
+    !/^[\w-]+$/u.test(cursor)
+  )
+    throw cursorError("encoding");
+  let decoded;
+  try {
+    decoded = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8"));
+  } catch {
+    throw cursorError("encoding");
+  }
+  if (
+    !decoded ||
+    decoded.schema !== CURSOR_SCHEMA ||
+    decoded.binding !== binding ||
+    !decoded.after ||
+    typeof decoded.after.memoryId !== "string" ||
+    typeof decoded.after.updatedAt !== "string" ||
+    !Number.isFinite(decoded.after.importance) ||
+    decoded.digest !==
+      queryDigest(
+        JSON.stringify({
+          schema: decoded.schema,
+          binding: decoded.binding,
+          after: decoded.after,
+        }),
+      )
+  ) {
+    throw cursorError("generation, filters, or checksum changed");
+  }
+  return decoded.after;
+}
+
+export function encodeListCursor(binding, row) {
+  const value = {
+    schema: CURSOR_SCHEMA,
+    binding,
+    after: {
+      memoryId: row.memoryId,
+      updatedAt: row.updatedAt,
+      importance: row.importance,
+    },
+  };
+  return Buffer.from(
+    JSON.stringify({ ...value, digest: queryDigest(JSON.stringify(value)) }),
+  ).toString("base64url");
+}
+
+export function pageLimit(value, fallback = 20) {
+  const number = value == null ? fallback : Number(value);
+  if (!Number.isSafeInteger(number) || number < 1 || number > 10000)
+    throw new RangeError(
+      "memory page limit must be an integer from 1 to 10000",
+    );
+  return number;
+}

package/src/lib/context-memory-kernel/memory-service.js

@@ -50,7 +50,8 @@ function normalizedScope(scope, scopeId) {
     return { scope: value };
   }
   const normalizedId = safeIdentifier(scopeId, "");
-  if (!normalizedId) throw new TypeError(`scopeId is required for ${value} scope`);
+  if (!normalizedId)
+    throw new TypeError(`scopeId is required for ${value} scope`);
   return { scope: value, scopeId: normalizedId };
 }
 
@@ -87,7 +88,10 @@ function legacyProposal(entry, now) {
       {
         store: "cli-sqlite-memory-entries",
         id: safeIdentifier(entry?.id, memoryId),
-        digest: canonicalDigest(entry, "chainlesschain.cli-legacy-memory-row/v1"),
+        digest: canonicalDigest(
+          entry,
+          "chainlesschain.cli-legacy-memory-row/v1",
+        ),
       },
     ],
     confidence: 0.7,
@@ -206,7 +210,11 @@ export class CliCanonicalMemoryService {
         scopeId: "local-user",
         category: safeIdentifier(options.category, "general"),
         content,
-        provenance: { source: "cli-shadow", actor: "local-user", observedAt: now },
+        provenance: {
+          source: "cli-shadow",
+          actor: "local-user",
+          observedAt: now,
+        },
         evidenceRefs: [{ store: "cli-command", id: "shadow-memory-add" }],
         confidence: 0.7,
         importance: normalizedImportance(options.importance),
@@ -316,7 +324,9 @@ export class CliCanonicalMemoryService {
         const proposal = legacyProposal(entry, now);
         const existing = await this.runtime.memoryPort.read(proposal.memoryId);
         if (existing) {
-          const projected = createMemoryCandidate(proposal, { clock: this.now });
+          const projected = createMemoryCandidate(proposal, {
+            clock: this.now,
+          });
           if (existing.digest !== projected.digest) {
             const error = new Error(
               `legacy memory ${entry?.id || "unknown"} conflicts with canonical authority`,
@@ -358,7 +368,9 @@ export class CliCanonicalMemoryService {
         const proposal = legacyScopedProposal(entry, now);
         const existing = await this.runtime.memoryPort.read(proposal.memoryId);
         if (existing) {
-          const projected = createMemoryCandidate(proposal, { clock: this.now });
+          const projected = createMemoryCandidate(proposal, {
+            clock: this.now,
+          });
           if (existing.digest !== projected.digest) {
             const error = new Error(
               `legacy scoped memory ${entry?.id || "unknown"} conflicts with canonical authority`,
@@ -391,14 +403,29 @@ export class CliCanonicalMemoryService {
     return result;
   }
 
-  async list({ limit = 20, category } = {}) {
-    const records = await this.runtime.memoryPort.listRecords();
+  async list({ limit = 20, category, scopeAdmissions, sink, tags } = {}) {
+    const records = await this.runtime.memoryPort.listRecords({
+      limit: Math.max(1, Number(limit) || 20),
+      category,
+      scopeAdmissions,
+      sink,
+      tags,
+    });
     return records
       .filter((record) => !category || record.category === category)
       .slice(0, Math.max(1, Number(limit) || 20))
       .map((record) => publicEntry(record));
   }
 
+  async listPage(options = {}) {
+    const page = await this.runtime.memoryPort.listPage(options);
+    return {
+      entries: page.records.map((record) => publicEntry(record)),
+      nextCursor: page.nextCursor,
+      storeRevision: page.storeRevision,
+    };
+  }
+
   async add(content, options = {}) {
     const now = new Date(Number(this.now())).toISOString();
     const evidenceId = `add-${canonicalDigest(
@@ -533,8 +560,7 @@ export class CliCanonicalMemoryService {
       .filter(
         ({ record }) =>
           (!options.category || record.category === options.category) &&
-          (tags.length === 0 ||
-            record.tags.some((tag) => tags.includes(tag))),
+          (tags.length === 0 || record.tags.some((tag) => tags.includes(tag))),
       )
       .slice(0, limit)
       .map(({ record, relevance }) => publicScopedEntry(record, relevance));
@@ -558,11 +584,19 @@ export class CliCanonicalMemoryService {
   }
 
   async delete(idOrPrefix) {
-    const records = await this.runtime.memoryPort.listRecords();
-    const exact = records.find((record) => record.memoryId === idOrPrefix);
-    const candidates = exact
-      ? [exact]
-      : records.filter((record) => record.memoryId.startsWith(idOrPrefix));
+    const exact = await this.runtime.memoryPort.read(idOrPrefix);
+    let candidates;
+    if (exact && !["deleted", "purged"].includes(exact.state)) {
+      candidates = [exact];
+    } else {
+      const records = await this.runtime.memoryPort.listRecords();
+      const listedExact = records.find(
+        (record) => record.memoryId === idOrPrefix,
+      );
+      candidates = listedExact
+        ? [listedExact]
+        : records.filter((record) => record.memoryId.startsWith(idOrPrefix));
+    }
     if (candidates.length === 0) return null;
     if (candidates.length > 1) {
       const error = new Error(`memory prefix is ambiguous: ${idOrPrefix}`);
@@ -601,7 +635,9 @@ export class CliCanonicalMemoryService {
           const existing = await this.runtime.memoryPort.read(record.memoryId);
           if (existing) {
             if (existing.digest !== record.digest) {
-              const error = new Error(`canonical memory conflict: ${record.memoryId}`);
+              const error = new Error(
+                `canonical memory conflict: ${record.memoryId}`,
+              );
               error.code = "CONTEXT_MEMORY_IMPORT_CONFLICT";
               throw error;
             }
@@ -620,18 +656,24 @@ export class CliCanonicalMemoryService {
             recordDigest: record.digest,
             at: new Date(Number(this.now())).toISOString(),
           };
-          event.digest = canonicalDigest(event, "chainlesschain.memory-event/v1");
+          event.digest = canonicalDigest(
+            event,
+            "chainlesschain.memory-event/v1",
+          );
           const committed = await this.runtime.memoryPort.commit(
             { record, event },
             0,
           );
-          if (!committed.ok) throw new Error(`memory import raced: ${record.memoryId}`);
+          if (!committed.ok)
+            throw new Error(`memory import raced: ${record.memoryId}`);
         } else {
           const proposal = legacyProposal(
             entry,
             new Date(Number(this.now())).toISOString(),
           );
-          const existing = await this.runtime.memoryPort.read(proposal.memoryId);
+          const existing = await this.runtime.memoryPort.read(
+            proposal.memoryId,
+          );
           if (existing) {
             result.existing += 1;
             continue;

package/src/lib/context-memory-kernel/runtime.js

@@ -1,5 +1,5 @@
 import { ContextMemoryKernel } from "@chainlesschain/context-memory-kernel";
-import { DurableJsonMemoryPort } from "./durable-memory-port.js";
+import { SegmentedMemoryPort } from "./segmented-memory-port.js";
 import { JsonlSessionContextPort } from "./jsonl-session-context-port.js";
 import { CliLegacyMemoryPrivacyPurgePort } from "./privacy-purge-port.js";
 import {
@@ -8,7 +8,8 @@ import {
 } from "./authority.js";
 
 export function createCliContextMemoryRuntime(options = {}) {
-  const scopeKey = options.scopeKey ||
+  const scopeKey =
+    options.scopeKey ||
     (options.sessionId ? `cli:session:${options.sessionId}` : "cli:memory");
   const decision = resolveCliContextMemoryCutover({
     env: options.env,
@@ -20,7 +21,8 @@ export function createCliContextMemoryRuntime(options = {}) {
   });
   const memoryPort =
     options.memoryPort ||
-    new DurableJsonMemoryPort({
+    new SegmentedMemoryPort({
+      readOnly: !decision.canonical,
       ...(options.memoryFilePath ? { filePath: options.memoryFilePath } : {}),
     });
   const sessionPort =