chainlesschain 0.166.86 → 0.166.87

package/src/lib/context-memory-kernel/memory-query-index.js

@@ -0,0 +1,278 @@
+import { createHash } from "node:crypto";
+
+export const MEMORY_QUERY_INDEX_SCHEMA = "chainlesschain.memory-query-index/v1";
+const CURSOR_SCHEMA = "chainlesschain.memory-list-cursor/v1";
+
+export function queryDigest(bytes) {
+  return `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
+}
+
+export function compareMemoryRows(left, right) {
+  return (
+    right.importance - left.importance ||
+    right.updatedAt.localeCompare(left.updatedAt, "en") ||
+    left.memoryId.localeCompare(right.memoryId, "en")
+  );
+}
+
+// No content, evidence, or audit is copied into the derived index. The manifest
+// binds these bytes to the immutable authority shard from which they were made.
+export function encodeQueryIndex(state) {
+  const rows = Object.values(state.records)
+    .map((record) => ({
+      memoryId: record.memoryId,
+      category: record.category,
+      scope: record.scope,
+      scopeId: record.scopeId ?? null,
+      state: record.state,
+      importance: record.importance,
+      updatedAt: record.updatedAt,
+      tags: record.tags,
+      allowedSinks: record.allowedSinks,
+    }))
+    .sort(compareMemoryRows);
+  return Buffer.from(
+    JSON.stringify({
+      schema: MEMORY_QUERY_INDEX_SCHEMA,
+      shardDigest: state.digest,
+      rows,
+    }),
+  );
+}
+
+function scopeKey(scope, scopeId) {
+  return JSON.stringify([scope, scopeId ?? null]);
+}
+
+export function compileQueryIndex(bytes, descriptor) {
+  const value = JSON.parse(bytes.toString("utf8"));
+  if (
+    value.schema !== MEMORY_QUERY_INDEX_SCHEMA ||
+    value.shardDigest !== descriptor.digest ||
+    !Array.isArray(value.rows) ||
+    value.rows.length !== descriptor.recordCount
+  ) {
+    throw new Error("query index does not match authority shard");
+  }
+  const fields = ["category", "scope", "state", "tag", "sink"];
+  const postings = Object.fromEntries(
+    fields.map((field) => [field, new Map()]),
+  );
+  const add = (field, key, index) => {
+    const map = postings[field];
+    if (!map.has(key)) map.set(key, []);
+    map.get(key).push(index);
+  };
+  value.rows.forEach((row, index) => {
+    add("category", row.category, index);
+    add("scope", scopeKey(row.scope, row.scopeId), index);
+    add("state", row.state, index);
+    for (const tag of row.tags) add("tag", tag, index);
+    for (const sink of row.allowedSinks) add("sink", sink, index);
+  });
+  return { rows: value.rows, postings };
+}
+
+export function normalizeListQuery(options = {}) {
+  const result = { includeTombstones: options.includeTombstones === true };
+  if (options.category) result.category = String(options.category);
+  if (options.sink) result.sink = String(options.sink);
+  if (options.scopeAdmissions !== undefined) {
+    if (!Array.isArray(options.scopeAdmissions))
+      throw new TypeError("scopeAdmissions must be an array");
+    result.scopeAdmissions = options.scopeAdmissions.map(
+      ({ scope, scopeId }) => ({
+        scope,
+        scopeId: scopeId ?? null,
+      }),
+    );
+  }
+  if (options.tags !== undefined) {
+    if (!Array.isArray(options.tags))
+      throw new TypeError("tags must be an array");
+    result.tags = [...new Set(options.tags.map(String))].sort();
+  }
+  if (options.states !== undefined) {
+    if (!Array.isArray(options.states))
+      throw new TypeError("states must be an array");
+    result.states = [...new Set(options.states.map(String))].sort();
+  }
+  return result;
+}
+
+export function matchesListQuery(row, query) {
+  return (
+    (query.includeTombstones || !["deleted", "purged"].includes(row.state)) &&
+    (!query.category || row.category === query.category) &&
+    (!query.states || query.states.includes(row.state)) &&
+    (!query.scopeAdmissions ||
+      query.scopeAdmissions.some(
+        (scope) =>
+          scope.scope === row.scope &&
+          (scope.scopeId ?? null) === (row.scopeId ?? null),
+      )) &&
+    (!query.sink ||
+      row.allowedSinks.includes("*") ||
+      row.allowedSinks.includes(query.sink)) &&
+    (!query.tags?.length || row.tags.some((tag) => query.tags.includes(tag)))
+  );
+}
+
+// Each posting is already in the public list order. Pick the narrowest posting
+// before applying the remaining predicates; no full-record scan or top-k before
+// permission filtering. Recall uses these gates but retains kernel ranking.
+export function* selectQueryRows(index, query, after = null) {
+  const candidates = [];
+  const union = (field, keys) => {
+    const lists = keys
+      .map((key) => index.postings[field].get(key))
+      .filter(Boolean);
+    return lists.length < 2
+      ? lists[0] || []
+      : [...new Set(lists.flat())].sort((a, b) => a - b);
+  };
+  if (query.category)
+    candidates.push(index.postings.category.get(query.category) || []);
+  if (query.scopeAdmissions)
+    candidates.push(
+      union(
+        "scope",
+        query.scopeAdmissions.map((scope) =>
+          scopeKey(scope.scope, scope.scopeId),
+        ),
+      ),
+    );
+  if (query.states) candidates.push(union("state", query.states));
+  if (query.tags?.length) candidates.push(union("tag", query.tags));
+  if (query.sink) candidates.push(union("sink", ["*", query.sink]));
+  candidates.sort((a, b) => a.length - b.length);
+  const positions = candidates[0];
+  const length = positions ? positions.length : index.rows.length;
+  const rowAt = (offset) => index.rows[positions ? positions[offset] : offset];
+  let start = 0;
+  if (after) {
+    let end = length;
+    while (start < end) {
+      const middle = Math.floor((start + end) / 2);
+      if (compareMemoryRows(rowAt(middle), after) <= 0) start = middle + 1;
+      else end = middle;
+    }
+  }
+  for (let offset = start; offset < length; offset++) {
+    const row = rowAt(offset);
+    if (matchesListQuery(row, query)) yield row;
+  }
+}
+
+export function* mergeQueryRows(heads) {
+  const heap = [];
+  const less = (left, right) => compareMemoryRows(left.row, right.row) < 0;
+  const push = (head) => {
+    let at = heap.length;
+    heap.push(head);
+    while (at > 0) {
+      const parent = Math.floor((at - 1) / 2);
+      if (!less(head, heap[parent])) break;
+      heap[at] = heap[parent];
+      at = parent;
+    }
+    heap[at] = head;
+  };
+  for (const head of heads) push(head);
+  while (heap.length) {
+    const first = heap[0];
+    const last = heap.pop();
+    if (heap.length) {
+      let at = 0;
+      while (at * 2 + 1 < heap.length) {
+        let child = at * 2 + 1;
+        if (child + 1 < heap.length && less(heap[child + 1], heap[child]))
+          child++;
+        if (!less(heap[child], last)) break;
+        heap[at] = heap[child];
+        at = child;
+      }
+      heap[at] = last;
+    }
+    yield { bucket: first.bucket, row: first.row };
+    const next = first.iterator.next();
+    if (!next.done) push({ ...first, row: next.value });
+  }
+}
+
+export function cursorError(reason) {
+  return Object.assign(new Error(`Memory list cursor is invalid: ${reason}`), {
+    code: "CONTEXT_MEMORY_CURSOR_INVALID",
+  });
+}
+
+export function listCursorBinding(filePath, generation, query) {
+  return queryDigest(
+    JSON.stringify({
+      filePath,
+      generation,
+      query,
+      order: "importance-desc,updatedAt-desc,memoryId-en-asc/v1",
+    }),
+  );
+}
+
+export function decodeListCursor(cursor, binding) {
+  if (cursor == null) return null;
+  if (
+    typeof cursor !== "string" ||
+    cursor.length > 4096 ||
+    !/^[\w-]+$/u.test(cursor)
+  )
+    throw cursorError("encoding");
+  let decoded;
+  try {
+    decoded = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8"));
+  } catch {
+    throw cursorError("encoding");
+  }
+  if (
+    !decoded ||
+    decoded.schema !== CURSOR_SCHEMA ||
+    decoded.binding !== binding ||
+    !decoded.after ||
+    typeof decoded.after.memoryId !== "string" ||
+    typeof decoded.after.updatedAt !== "string" ||
+    !Number.isFinite(decoded.after.importance) ||
+    decoded.digest !==
+      queryDigest(
+        JSON.stringify({
+          schema: decoded.schema,
+          binding: decoded.binding,
+          after: decoded.after,
+        }),
+      )
+  ) {
+    throw cursorError("generation, filters, or checksum changed");
+  }
+  return decoded.after;
+}
+
+export function encodeListCursor(binding, row) {
+  const value = {
+    schema: CURSOR_SCHEMA,
+    binding,
+    after: {
+      memoryId: row.memoryId,
+      updatedAt: row.updatedAt,
+      importance: row.importance,
+    },
+  };
+  return Buffer.from(
+    JSON.stringify({ ...value, digest: queryDigest(JSON.stringify(value)) }),
+  ).toString("base64url");
+}
+
+export function pageLimit(value, fallback = 20) {
+  const number = value == null ? fallback : Number(value);
+  if (!Number.isSafeInteger(number) || number < 1 || number > 10000)
+    throw new RangeError(
+      "memory page limit must be an integer from 1 to 10000",
+    );
+  return number;
+}

package/src/lib/context-memory-kernel/memory-service.js

@@ -403,14 +403,29 @@ export class CliCanonicalMemoryService {
     return result;
   }
 
-  async list({ limit = 20, category } = {}) {
-    const records = await this.runtime.memoryPort.listRecords();
+  async list({ limit = 20, category, scopeAdmissions, sink, tags } = {}) {
+    const records = await this.runtime.memoryPort.listRecords({
+      limit: Math.max(1, Number(limit) || 20),
+      category,
+      scopeAdmissions,
+      sink,
+      tags,
+    });
     return records
       .filter((record) => !category || record.category === category)
       .slice(0, Math.max(1, Number(limit) || 20))
       .map((record) => publicEntry(record));
   }
 
+  async listPage(options = {}) {
+    const page = await this.runtime.memoryPort.listPage(options);
+    return {
+      entries: page.records.map((record) => publicEntry(record)),
+      nextCursor: page.nextCursor,
+      storeRevision: page.storeRevision,
+    };
+  }
+
   async add(content, options = {}) {
     const now = new Date(Number(this.now())).toISOString();
     const evidenceId = `add-${canonicalDigest(

package/src/lib/context-memory-kernel/segmented-memory-port.js

@@ -26,6 +26,20 @@ import {
   stateDigest,
   syncDirectory,
 } from "./durable-memory-port.js";
+import {
+  compareMemoryRows,
+  compileQueryIndex,
+  decodeListCursor,
+  encodeListCursor,
+  encodeQueryIndex,
+  listCursorBinding,
+  matchesListQuery,
+  mergeQueryRows,
+  normalizeListQuery,
+  pageLimit,
+  queryDigest,
+  selectQueryRows,
+} from "./memory-query-index.js";
 
 export const SEGMENTED_STORE_SCHEMA =
   "chainlesschain.cli-context-memory-store/v2";
@@ -147,6 +161,7 @@ export class SegmentedMemoryPort extends DurableJsonMemoryPort {
     this.readOnly = Boolean(readOnly);
     this.filePath = resolve(this.filePath);
     this.shardDirectory = `${this.filePath}.shards`;
+    this._queryIndexes = new Map();
   }
 
   _locked(operation) {
@@ -194,6 +209,13 @@ export class SegmentedMemoryPort extends DurableJsonMemoryPort {
     );
   }
 
+  _queryIndexPath(bucket, descriptor) {
+    return join(
+      this.shardDirectory,
+      `query-${bucket}-${descriptor.queryDigest.slice(7)}.json`,
+    );
+  }
+
   _validateManifest(input) {
     const invalid = () => {
       throw corruptStore(this.filePath, "segmented manifest is invalid");
@@ -219,13 +241,23 @@ export class SegmentedMemoryPort extends DurableJsonMemoryPort {
       if (
         !BUCKET.test(bucket) ||
         !entry ||
-        Object.keys(entry).sort().join() !==
-          "bytes,digest,eventCount,recordCount" ||
+        ![
+          "bytes,digest,eventCount,recordCount",
+          "bytes,digest,eventCount,queryBytes,queryDigest,recordCount",
+        ].includes(Object.keys(entry).sort().join()) ||
         !HEX.test(entry.digest)
       )
         invalid();
       for (const key of ["bytes", "eventCount", "recordCount"])
         if (!Number.isSafeInteger(entry[key]) || entry[key] < 0) invalid();
+      if (
+        entry.queryDigest !== undefined &&
+        (!HEX.test(entry.queryDigest) ||
+          !Number.isSafeInteger(entry.queryBytes) ||
+          entry.queryBytes < 1 ||
+          entry.queryBytes > this.maxStoreBytes)
+      )
+        invalid();
       if (entry.bytes > this.maxStoreBytes)
         throw limit("shard exceeds configured byte limit");
       bytes += entry.bytes;
@@ -241,6 +273,15 @@ export class SegmentedMemoryPort extends DurableJsonMemoryPort {
       invalid();
     if (bytes > this.maxTotalBytes || events > this.maxEvents)
       throw limit("aggregate capacity exceeded");
+    if (
+      bytes +
+        Object.values(input.shards).reduce(
+          (sum, entry) => sum + (entry.queryBytes || 0),
+          0,
+        ) >
+      this.maxTotalBytes
+    )
+      throw limit("authority and query indexes exceed aggregate capacity");
     return input;
   }
 
@@ -339,9 +380,12 @@ export class SegmentedMemoryPort extends DurableJsonMemoryPort {
   _collect(manifest) {
     this._checkDirectories();
     const live = new Set(
-      Object.entries(manifest.shards).map(
-        ([bucket, entry]) => `${bucket}-${entry.digest.slice(7)}.json`,
-      ),
+      Object.entries(manifest.shards).flatMap(([bucket, entry]) => [
+        `${bucket}-${entry.digest.slice(7)}.json`,
+        ...(entry.queryDigest
+          ? [`query-${bucket}-${entry.queryDigest.slice(7)}.json`]
+          : []),
+      ]),
     );
     // All readers hold the same authority lock. No reader can still be using
     // a superseded shard here. Unknown files are never treated as our garbage.
@@ -355,6 +399,8 @@ export class SegmentedMemoryPort extends DurableJsonMemoryPort {
           throw limit("shard directory inventory exceeds 4096 entries");
         const owned =
           /^[a-f0-9]{2}-[a-f0-9]{64}\.json$/u.test(entry.name) ||
+          /^query-[a-f0-9]{2}-[a-f0-9]{64}\.json$/u.test(entry.name) ||
+          /^query-rebuild-[a-f0-9-]{36}\.tmp$/u.test(entry.name) ||
           /^manifest-[a-f0-9-]{36}\.tmp$/u.test(entry.name);
         if (owned && !live.has(entry.name)) {
           const file = join(this.shardDirectory, entry.name);
@@ -388,7 +434,12 @@ export class SegmentedMemoryPort extends DurableJsonMemoryPort {
     for (const [bucket, state] of changed) {
       state.digest = stateDigest(state);
       const bytes = `${JSON.stringify(state)}\n`;
-      const size = Buffer.byteLength(bytes);
+      const unchanged = manifest.shards[bucket]?.digest === state.digest;
+      // Canonical digests do not encode JSON property order. During a derived
+      // index upgrade retain the original immutable authority bytes verbatim.
+      const size = unchanged
+        ? manifest.shards[bucket].bytes
+        : Buffer.byteLength(bytes);
       if (size > this.maxStoreBytes)
         throw limit("shard exceeds configured byte limit");
       pendingBytes += size - (next.shards[bucket]?.bytes || 0);
@@ -402,9 +453,19 @@ export class SegmentedMemoryPort extends DurableJsonMemoryPort {
         eventCount: state.events.length,
         recordCount: Object.keys(state.records).length,
       };
+      const queryBytes = encodeQueryIndex(state);
+      if (queryBytes.length > this.maxStoreBytes)
+        throw limit("query index exceeds configured byte limit");
+      next.shards[bucket].queryDigest = queryDigest(queryBytes);
+      next.shards[bucket].queryBytes = queryBytes.length;
+      if (!unchanged)
+        pending.push({
+          file: this._shardPath(bucket, next.shards[bucket]),
+          bytes,
+        });
       pending.push({
-        file: this._shardPath(bucket, next.shards[bucket]),
-        bytes,
+        file: this._queryIndexPath(bucket, next.shards[bucket]),
+        bytes: queryBytes,
       });
     }
     next.totalBytes = Object.values(next.shards).reduce(
@@ -557,7 +618,214 @@ export class SegmentedMemoryPort extends DurableJsonMemoryPort {
       ),
     );
   }
-  async query() {
+
+  _ensureQueryIndexes(manifest) {
+    if (manifest.schema === STORE_SCHEMA || this.readOnly) return manifest;
+    const missing = Object.keys(manifest.shards).filter(
+      (bucket) => !manifest.shards[bucket].queryDigest,
+    );
+    if (!missing.length) return manifest;
+    // Upgrade older v2 snapshots without changing records, revisions or audit.
+    // Old strict readers reject these extended descriptors instead of writing
+    // a second authority. Shadow readers derive indexes only in memory.
+    return this._publish(
+      manifest,
+      new Map(
+        missing.map((bucket) => [bucket, this._readShard(manifest, bucket)]),
+      ),
+    );
+  }
+
+  _readQueryIndex(manifest, bucket) {
+    const descriptor = manifest.shards[bucket];
+    let bytes;
+    if (descriptor.queryDigest) {
+      const file = this._queryIndexPath(bucket, descriptor);
+      this._checkDirectories();
+      if (existsSync(file)) {
+        // Unsafe filesystem objects are authority errors, never repair targets.
+        regular(file);
+        try {
+          bytes = readBoundedState(file, descriptor.queryBytes, true);
+        } catch (error) {
+          if (error.code !== "CONTEXT_MEMORY_STORE_CORRUPT") throw error;
+        }
+      }
+      if (
+        !bytes ||
+        bytes.length !== descriptor.queryBytes ||
+        queryDigest(bytes) !== descriptor.queryDigest
+      ) {
+        bytes = encodeQueryIndex(this._readShard(manifest, bucket));
+        if (
+          bytes.length !== descriptor.queryBytes ||
+          queryDigest(bytes) !== descriptor.queryDigest
+        )
+          throw corruptStore(
+            file,
+            "query index cannot be reproduced from authority",
+          );
+        if (!this.readOnly) {
+          const temporary = join(
+            this.shardDirectory,
+            `query-rebuild-${randomUUID()}.tmp`,
+          );
+          const fd = openSync(temporary, "wx", 0o600);
+          try {
+            writeFileSync(fd, bytes);
+            fsyncSync(fd);
+          } finally {
+            closeSync(fd);
+          }
+          this._checkDirectories();
+          if (existsSync(file)) regular(file);
+          renameSync(temporary, file);
+          syncDirectory(this.shardDirectory);
+        }
+      }
+    } else {
+      bytes = encodeQueryIndex(this._readShard(manifest, bucket));
+    }
+    const digest = descriptor.queryDigest || queryDigest(bytes);
+    const cached = this._queryIndexes.get(bucket);
+    if (cached?.digest === digest) return cached.index;
+    let index;
+    try {
+      index = compileQueryIndex(bytes, descriptor);
+    } catch (cause) {
+      throw corruptStore(
+        this.filePath,
+        "query index structure is invalid",
+        cause,
+      );
+    }
+    this._queryIndexes.set(bucket, { digest, index });
+    return index;
+  }
+
+  async _indexedRecords(
+    options = {},
+    { limit: count = Infinity, cursor = null } = {},
+  ) {
+    const query = normalizeListQuery(options);
+    const captured = await this._locked(() => {
+      const manifest = this._ensureQueryIndexes(this._loadManifest());
+      const binding = listCursorBinding(
+        this.filePath,
+        manifest.digest || queryDigest(JSON.stringify(manifest)),
+        query,
+      );
+      const after = decodeListCursor(cursor, binding);
+      if (manifest.schema === STORE_SCHEMA) {
+        const matches = Object.values(manifest.records)
+          .filter(
+            (row) =>
+              matchesListQuery(row, query) &&
+              (!after || compareMemoryRows(row, after) > 0),
+          )
+          .sort(compareMemoryRows);
+        const records = matches.slice(0, count);
+        return {
+          legacy: records,
+          storeRevision: manifest.storeRevision,
+          nextCursor:
+            matches.length > count
+              ? encodeListCursor(binding, records.at(-1))
+              : null,
+        };
+      }
+      // Merge each sorted posting lazily. Only the requested page is captured;
+      // every candidate has already passed scope, sink and lifecycle filters.
+      const heads = [];
+      for (const bucket of Object.keys(manifest.shards)) {
+        const iterator = selectQueryRows(
+          this._readQueryIndex(manifest, bucket),
+          query,
+          after,
+        );
+        const next = iterator.next();
+        if (!next.done) heads.push({ bucket, iterator, row: next.value });
+      }
+      const selected = [];
+      for (const row of mergeQueryRows(heads)) {
+        selected.push(row);
+        if (selected.length > count) break;
+      }
+      const more = selected.length > count;
+      if (more) selected.pop();
+      const shards = new Map();
+      for (const { bucket } of selected)
+        if (!shards.has(bucket))
+          shards.set(bucket, this._readShardBytes(manifest, bucket));
+      return {
+        manifest,
+        selected,
+        shards,
+        storeRevision: manifest.storeRevision,
+        nextCursor: more
+          ? encodeListCursor(binding, selected.at(-1).row)
+          : null,
+      };
+    });
+    if (captured.legacy)
+      return {
+        records: captured.legacy,
+        nextCursor: captured.nextCursor,
+        storeRevision: captured.storeRevision,
+      };
+    const decoded = new Map(
+      [...captured.shards].map(([bucket, bytes]) => [
+        bucket,
+        this._decodeShard(captured.manifest, bucket, bytes),
+      ]),
+    );
+    const records = captured.selected.map(({ bucket, row }) => {
+      const record = own(decoded.get(bucket).records, row.memoryId);
+      if (
+        !record ||
+        !matchesListQuery(record, query) ||
+        compareMemoryRows(record, row) !== 0
+      )
+        throw corruptStore(
+          this.filePath,
+          "query index result disagrees with authority",
+        );
+      return record;
+    });
+    return {
+      records,
+      nextCursor: captured.nextCursor,
+      storeRevision: captured.storeRevision,
+    };
+  }
+
+  async listRecords(options = {}) {
+    const count =
+      options.limit == null
+        ? Infinity
+        : Math.max(1, Number(options.limit) || 20);
+    return (await this._indexedRecords(options, { limit: count })).records;
+  }
+
+  async listPage(options = {}) {
+    return this._indexedRecords(options, {
+      limit: pageLimit(options.limit),
+      cursor: options.cursor,
+    });
+  }
+
+  async query(request) {
+    if (request?.scopeAdmissions || request?.sink) {
+      // Do not push recall's limit before lexical/semantic ranking in the
+      // kernel. Push only gates that cannot discard an admissible result.
+      return (
+        await this._indexedRecords({
+          scopeAdmissions: request.scopeAdmissions,
+          sink: request.sink,
+          states: ["active", "reinforced"],
+        })
+      ).records;
+    }
     const snapshot = await this._locked(() => {
       const manifest = this._loadManifest();
       if (manifest.schema === STORE_SCHEMA) return { legacy: manifest };