@lorekit/cli 1.33.4 → 1.35.0

package/README.md

@@ -381,7 +381,10 @@ lorekit lint --json           # { total, offline, remote } structured findings
 
 Rules: **empty-value** (blank/whitespace-only body), **short-value** (a non-empty
 body below a small length threshold), **untrimmed-value** (real content with
-surrounding whitespace), **empty-key** (blank key), and **malformed-scope** (e.g.
+surrounding whitespace), **empty-key** (blank key), **volatile-key** (the key
+carries a per-sighting identifier — a run of 6+ digits such as a GitHub comment
+id, or a `pr<n>` / `issue<n>` segment — so it never collides, never dedups, and
+freezes `seen_count` at 1), and **malformed-scope** (e.g.
 a single `:` where `::` is expected). `lint` **exits non-zero (1) when any issue
 is found**, so it is usable as a CI gate (`lorekit lint || exit 1`); a clean run —
 or one where only a store is unavailable — exits 0. The pure rule predicates live

package/bin/lorekit.mjs

@@ -78,8 +78,9 @@ ${c.bold('Commands')}
     (resolve) hierarchy and mark, per key, which scope's memory WINS and which are
               shadowed — the real hook-resolution order. --json, --scope <s>.
   lint        Flag low-quality memories (empty/short/untrimmed value, empty key,
-              malformed scope) across the applicable scopes and both stores. Exits
-              non-zero when issues are found (CI gate). --json, --scope <s>.
+              volatile key, malformed scope) across the applicable scopes and
+              both stores. Exits non-zero when issues are found (CI gate).
+              --json, --scope <s>.
   dedupe      Find likely-duplicate memories via a zero-dep word-overlap HEURISTIC
               (Jaccard >= threshold, not semantic), grouped into clusters per
               store. --json, --scope <s>, --threshold <0..1>.
@@ -495,10 +496,12 @@ ${c.bold('Usage')}
 
 Checks every memory for the current directory's scopes (project/branch/repo/
 global), across both stores, against a small set of quality rules: empty or
-whitespace-only value, suspiciously short value, untrimmed value, empty key, and
-malformed scope (e.g. a single \`:\` where \`::\` is expected). Each finding names
-the rule it violated. Exits NON-ZERO when any issue is found, so it works as a CI
-gate; a clean run — or one where only a store is unavailable — exits 0.
+whitespace-only value, suspiciously short value, untrimmed value, empty key, a
+volatile per-sighting identifier in the key (a run of 6+ digits, or a \`pr<n>\` /
+\`issue<n>\` reference), and malformed scope (e.g. a single \`:\` where \`::\` is
+expected). Each finding names the rule it violated. Exits NON-ZERO when any issue
+is found, so it works as a CI gate; a clean run — or one where only a store is
+unavailable — exits 0.
 
 ${c.bold('Options')}
   -d, --dir <path>        Target project root (default: current directory)
@@ -633,6 +636,8 @@ const KNOWN_FLAGS = [
   'value', 'tags', 'source-agent', 'trigger', 'kind', 'host', 'ttl-days', 'clear-ttl', 'org', 'remote', 'local',
   'link', 'base', 'q', 'owner', 'range', 'view', 'archived',
   'origin-repo', 'origin-branch', 'origin-commit', 'origin-pr', 'no-origin',
+  // Scale-aware survey flags
+  'all', 'max', 'since', 'until', 'key-prefix',
 ];
 
 // Commands that write to disk / talk to the network on a human's behalf. These
@@ -657,7 +662,7 @@ async function main() {
   const argv = process.argv.slice(2);
   const args = parseArgs(argv, {
     aliases: { d: 'dir', e: 'endpoint', t: 'token', y: 'yes', h: 'help', v: 'version' },
-    booleans: ['yes', 'force', 'deep', 'apply', 'help', 'version', 'global', 'project', 'no-hooks', 'no-origin', 'json', 'remote', 'local', 'link', 'archived', 'clear-ttl', 'telemetry'],
+    booleans: ['yes', 'force', 'deep', 'apply', 'help', 'version', 'global', 'project', 'no-hooks', 'no-origin', 'json', 'remote', 'local', 'link', 'archived', 'clear-ttl', 'telemetry', 'all'],
     known: KNOWN_FLAGS,
   });
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lorekit/cli",
-  "version": "1.33.4",
+  "version": "1.35.0",
   "description": "Install the LoreKit shared-memory skill and run health checks for the LoreKit MCP server.",
   "license": "MIT",
   "repository": {

package/skill/lorekit-groom/SKILL.md

@@ -89,8 +89,8 @@ The short version is six phases; the first four are pure analysis.
    `lorekit scopes` (store-wide inventory: every scope with its lesson count)
    show where the mass is. Pick the noisiest scope to start.
 2. **Lint** — `lorekit lint --json` flags structurally bad lessons (empty /
-   whitespace / suspiciously short / untrimmed values, empty keys, malformed
-   scopes). These are the cheapest wins: fix or drop them first.
+   whitespace / suspiciously short / untrimmed values, empty keys, volatile
+   keys, malformed scopes). These are the cheapest wins: fix or drop them first.
 3. **Dedupe** — `lorekit dedupe --json` clusters near-duplicate lessons. Start at
    a high `--threshold` (e.g. `0.85`) for confident duplicates, then lower it to
    surface looser paraphrases. Use `lorekit show <scope::key>` to read each
@@ -117,8 +117,9 @@ archiving and deleting always go through `memory.delete` / `memory.archive`.
 |-----|-----|---------|
 | Count lessons per scope/store | `lorekit stats [--scope <s>]` | CLI (read) |
 | Inventory every scope + lesson count | `lorekit scopes` | CLI (read) |
-| Find low-quality lessons | `lorekit lint --json` | CLI (read) |
-| Find near-duplicate clusters | `lorekit dedupe --json [--threshold <n>]` | CLI (read) |
+| Find low-quality lessons | `lorekit lint --json [--since <date>] [--until <date>] [--max <n>]` | CLI (read) |
+| Find near-duplicate clusters | `lorekit dedupe --json [--threshold <n>] [--key-prefix <p>] [--since <date>] [--max <n>]` | CLI (read) |
+| List all lessons across large scopes | `lorekit list --all [--max <n>] [--since <date>] [--until <date>]` | CLI (read) |
 | Read one lesson in full | `lorekit show <scope::key> [--json]` | CLI (read) |
 | Compare offline vs remote | `lorekit diff` | CLI (read) |
 | Write the merged/consolidated lesson | `memory.write` (or `lorekit write`) | MCP / CLI |
@@ -126,6 +127,37 @@ archiving and deleting always go through `memory.delete` / `memory.archive`.
 | Archive a lesson (reversible) | `memory.archive` (or `memory.delete`) | **MCP only** |
 | Hard-delete a lesson (permanent) | `memory.delete { force: true }` | **MCP only** |
 
+### Survey flags for large scopes
+
+When a scope holds more lessons than a single page (> 50 for `list`, > 100 for
+`lint`/`dedupe`), use these flags to survey the full population or narrow it:
+
+| Flag | Commands | Effect |
+|------|----------|--------|
+| `--all` | `list` | Drain all pages (default for `lint`/`dedupe`) |
+| `--max <n>` | `list --all`, `lint`, `dedupe` | Hard cap on entries surveyed (default 5000) |
+| `--since <iso-date>` | `list --all`, `lint`, `dedupe` | Lower date bound (`created_since`) |
+| `--until <iso-date>` | `list --all`, `lint`, `dedupe` | Upper date bound (`created_until`) |
+| `--key-prefix <p>` | `dedupe` | Narrow deduplication to keys starting with prefix |
+
+`dedupe` applies an internal population cap of 2000 for memory safety. When the
+cap is reached, the output includes a warning with narrowing suggestions — use
+`--key-prefix` or `--since` to reduce the population.
+
+### MCP cursor paging
+
+`memory.list` and `memory.search` now support pagination. When more results
+exist than fit in one response, the response includes `hasMore: true` and a
+`nextCursor` string. Pass `nextCursor` as the `cursor` argument on the next
+call to retrieve the following page. Omit `cursor` to start from the first page.
+
+```json
+{ "scope": "global", "limit": 100, "cursor": "<nextCursor from previous response>" }
+```
+
+Use this to drain an arbitrarily large scope in an agent loop rather than
+relying on a truncated first page.
+
 > **`dedupe` is a heuristic, not a semantic judge.** It clusters on Jaccard
 > word-token overlap, so it can both miss reworded duplicates *and* group
 > coincidental ones. Treat every cluster as a candidate to read and decide on,

package/skill/lorekit-groom/rules/grooming-pass.md

@@ -23,9 +23,21 @@ lorekit scopes           # every scope in the store + its lesson count
 how you notice a `branch::…` scope you had forgotten about, or a scope with
 hundreds of lessons that dwarfs the rest. Neither command reports a
 last-activity date — the inventory is counts only — so judge staleness from the
-lessons themselves once you narrow in (`lorekit list --scope <scope>` tags each
-one with its `updated` date). Read both, then pick the noisiest scope as the
-target for this pass and narrow to it with `--scope <scope>` from here on.
+lessons themselves once you narrow in. Read both, then pick the noisiest scope as
+the target for this pass and narrow to it with `--scope <scope>` from here on.
+
+**Large scopes:** when a scope has more lessons than a single page, use:
+
+```bash
+lorekit list --all --scope <scope>           # drain all pages; tags each lesson with its updated date
+lorekit list --all --scope <scope> --max 500 # cap at 500 to get a representative sample
+lorekit list --all --scope <scope> --since 2024-01-01  # only lessons created since that date
+```
+
+The remote store is paginated (default 50 entries per `list` call, 100 per
+`lint`/`dedupe` call). `--all` drains every page; `--max` applies a hard cap
+(default 5000). `lint` and `dedupe` default to full-scope survey — add `--max`
+or `--since` when the population is large enough to slow things down.
 
 ## Phase 2 — Lint (read-only)
 
@@ -38,6 +50,11 @@ Findings are structural, not semantic — each names its rule:
 - **empty-value / short-value / untrimmed-value** — the lesson carries little or
   no signal, or has stray leading/trailing whitespace.
 - **empty-key** — no key to address it by.
+- **volatile-key** — the key carries a per-sighting identifier (a run of 6+
+  digits such as a GitHub comment id, or a `pr<n>` / `issue<n>` segment), so it
+  never collides with a later write, never dedups, and leaves `seen_count`
+  frozen at 1. Re-key it onto the structural pattern and move the identifier
+  into the body.
 - **malformed-scope** — the scope string is invalid.
 
 These are the cheapest wins and the least controversial, so clear them first.
@@ -51,6 +68,9 @@ makes it a clean CI gate — a passing `lint` is your Phase 6 proof.
 
 ```bash
 lorekit dedupe --json --scope <scope> --threshold 0.85
+# For large scopes, narrow the population:
+lorekit dedupe --json --scope <scope> --threshold 0.85 --key-prefix "debug-" --max 1000
+lorekit dedupe --json --scope <scope> --threshold 0.85 --since 2024-01-01
 ```
 
 Each cluster is a set of lessons whose values overlap heavily by word tokens.
@@ -59,6 +79,11 @@ Start high (`0.85`) to see the confident duplicates, then re-run lower (`0.75`,
 clusters are coincidental overlaps rather than true duplicates, so read more
 carefully.
 
+`dedupe` surveys the full scope by default. When the population exceeds 2000
+entries it stops and prints a narrowing warning — use `--key-prefix` to focus
+on a key namespace, or `--since` to limit the date range. The `--max` flag sets
+a lower cap (default 5000, internal safety cap at 2000).
+
 For every cluster you intend to act on, read the members in full first:
 
 ```bash

package/src/dedupe.mjs

@@ -15,18 +15,30 @@ import { resolveProjectRoot, readLorekitJson } from './config.mjs';
 import { deriveScope } from './scope.mjs';
 import { resolveDenies } from './control.mjs';
 import { resolveStores, remoteUnavailableReason } from './stores.mjs';
-import { scopeList, gather, clusterDuplicates } from './lessons-view.mjs';
+import { scopeList, gather, gatherStream, clusterDuplicates, clusterDuplicatesBlocked, DEFAULT_MAX } from './lessons-view.mjs';
 import { log, heading, status, c } from './util.mjs';
 
 const DEFAULT_THRESHOLD = 0.8;
+// Maximum entries to accumulate before the token-blocking index becomes
+// memory-prohibitive. Beyond this the user must narrow via --key-prefix /
+// --since / --max.
+const DEDUPE_POP_CAP = 2000;
 
-// Parse `--threshold` into a number in [0, 1]; anything unparseable or out of
+// Smallest threshold the blocked clusterer accepts. `clusterDuplicatesBlocked`
+// is provably equivalent to the oracle `clusterDuplicates` only for
+// threshold > 0 (at 0 the oracle clusters even zero-overlap pairs, which the
+// token-blocking sweep never generates). Any positive value below the smallest
+// possible Jaccard behaves identically to 0+ while preserving that invariant,
+// so we floor to a tiny epsilon rather than accept a literal 0.
+const MIN_THRESHOLD = Number.EPSILON;
+
+// Parse `--threshold` into a number in (0, 1]; anything unparseable or out of
 // range falls back to the default (never a crash on bad input). Pure-ish helper.
 export function parseThreshold(raw) {
   if (raw === undefined || raw === true) return DEFAULT_THRESHOLD;
   const n = Number(raw);
   if (!Number.isFinite(n)) return DEFAULT_THRESHOLD;
-  return Math.min(1, Math.max(0, n));
+  return Math.min(1, Math.max(MIN_THRESHOLD, n));
 }
 
 // Read `dedupe.threshold` from .lorekit.json (non-throwing). Returns the
@@ -78,19 +90,93 @@ export async function dedupe(args) {
   // Deny-wins section suppression, identical to the other read commands.
   const { localDenied, remoteDenied } = resolveDenies(root, { env });
 
-  const buildSection = (flat) => ({
-    available: true,
-    clusters: clusterDuplicates(flat.entries, threshold),
-    errored: flat.errored,
-  });
+  // `dedupe` defaults to full-scope survey. --max, --since, --key-prefix narrow
+  // the population. Population cap: 2000. Past it stop accumulating, warn, and
+  // surface a narrowing hint. Use token-blocking (`clusterDuplicatesBlocked`)
+  // for the one super-linear operation.
+  const surveyMax = args.max !== undefined ? Number(args.max) : DEFAULT_MAX;
+  const surveySince = args.since || undefined;
+  const surveyUntil = args.until || undefined;
+  const surveyKeyPrefix = args['key-prefix'] || undefined;
+
+  // Stream-accumulate entries up to DEDUPE_POP_CAP and note when capped.
+  async function streamAccumulate(store) {
+    const accumulated = [];
+    const errored = [];
+    let popCapped = false;
+
+    await gatherStream(store, scopes, {
+      max: surveyMax,
+      since: surveySince,
+      until: surveyUntil,
+      keyPrefix: surveyKeyPrefix,
+      onPage: ({ scope, entries }) => {
+        if (popCapped) return;
+        for (const e of entries) {
+          if (accumulated.length >= DEDUPE_POP_CAP) {
+            popCapped = true;
+            break;
+          }
+          accumulated.push({ ...e, scope: e.scope ?? scope });
+        }
+      },
+    });
+
+    return { entries: accumulated, errored, popCapped };
+  }
+
+  const buildSection = async (store, local) => {
+    if (local) {
+      // Local store is already exhaustive; gather everything, then apply the
+      // SAME narrowing the remote path gets server-side (the local store's
+      // `list()` honours only scope/tags, so it can't narrow itself). Filtering
+      // here — before the population cap — is what makes `--key-prefix`/
+      // `--since`/`--until`/`--max` real offline instead of silent no-ops.
+      const flat = flatten(await gather(store, scopes));
+      let entries = flat.entries;
+      if (surveyKeyPrefix) {
+        entries = entries.filter(
+          (e) => typeof e.key === 'string' && e.key.startsWith(surveyKeyPrefix),
+        );
+      }
+      // `created_at` is compared as an ISO string; both bounds mirror the REST
+      // handler — inclusive `since`, exclusive `until` (the `[since, until)`
+      // window). An entry with no `created` timestamp is kept (never dropped by
+      // a bound it can't be judged against).
+      if (surveySince) entries = entries.filter((e) => !e.created || e.created >= surveySince);
+      if (surveyUntil) entries = entries.filter((e) => !e.created || e.created < surveyUntil);
+      // Silent `--max` cap first, then the memory-safety population cap that
+      // drives the "partial results" warning — mirroring the remote path where
+      // gatherStream's `max` and the DEDUPE_POP_CAP are distinct.
+      if (entries.length > surveyMax) entries = entries.slice(0, surveyMax);
+      let popCapped = false;
+      if (entries.length > DEDUPE_POP_CAP) {
+        entries = entries.slice(0, DEDUPE_POP_CAP);
+        popCapped = true;
+      }
+      return {
+        available: true,
+        clusters: clusterDuplicatesBlocked(entries, threshold),
+        errored: flat.errored,
+        popCapped,
+      };
+    }
+    const { entries, errored, popCapped } = await streamAccumulate(store);
+    return {
+      available: true,
+      clusters: clusterDuplicatesBlocked(entries, threshold),
+      errored,
+      popCapped,
+    };
+  };
 
   const offlineSection = localDenied
     ? { available: false, reason: `disabled by deny constraint (${localDenied.source})` }
-    : buildSection(flatten(await gather(local, scopes)));
+    : await buildSection(local, true);
 
   const remoteAvailable = !remoteDenied && remote.usable();
   const remoteSection = remoteAvailable
-    ? buildSection(flatten(await gather(remote, scopes)))
+    ? await buildSection(remote, false)
     : {
         available: false,
         reason: remoteDenied
@@ -109,6 +195,13 @@ export async function dedupe(args) {
     log(`  scopes:  ${scopes.join('  →  ')}`);
     log(`  ${c.dim(`heuristic: Jaccard word-token overlap >= ${threshold} (not semantic)`)}`);
 
+    if (offlineSection.available && offlineSection.popCapped) {
+      log(`  ${c.yellow('!')} population cap (${DEDUPE_POP_CAP}) reached for Offline — results are partial. Narrow with --key-prefix, --since, or --max.`);
+    }
+    if (remoteSection.available && remoteSection.popCapped) {
+      log(`  ${c.yellow('!')} population cap (${DEDUPE_POP_CAP}) reached for Remote — results are partial. Narrow with --key-prefix, --since, or --max.`);
+    }
+
     renderDedupeSection({ title: 'Offline' }, offlineSection);
     renderDedupeSection(
       { title: 'Remote', subtitle: remoteAvailable ? connection.endpoint : undefined },

package/src/lessons-view.mjs

@@ -65,6 +65,7 @@ export function normalizeEntry(e = {}) {
     key: e.key ?? null,
     value: e.value == null ? '' : String(e.value),
     updated: e.updated ?? e.updated_at ?? null,
+    created: e.created ?? e.created_at ?? null,
     tags,
     kind: e.kind ?? inferred.kind ?? null,
     host: e.host ?? inferred.host ?? null,
@@ -271,6 +272,41 @@ export const LINT_RULES = {
     return v.trim() && v !== v.trim() ? 'value has leading/trailing whitespace' : null;
   },
   'empty-key': (e) => (String(e.key ?? '').trim() ? null : 'key is empty or whitespace-only'),
+  // A key carrying a per-sighting identifier (a comment id, a PR/issue number) is
+  // unique forever, so it never collides, so the upsert never dedups it, so
+  // `seen_count` stays frozen at 1 and the memory can never reach a recurrence
+  // threshold — a write-only record. Detection is deliberately conservative:
+  //   • a run of 6+ digits (a GitHub comment id is ~10; `sha256`, `oauth2`,
+  //     `wcag22`, and semantic versions are all shorter runs);
+  //   • a `pr<n>` / `issue<n>` reference — the number joined by nothing, `-`, or
+  //     `_` — delimited by `:`, `-`, `_`, `/`, or a string boundary, so mid-word
+  //     digits (`oauth2`) never match.
+  // `volatileKeyAllow` is an embedder/test knob mirroring `short-value`'s
+  // `minValueLen` precedent — a list of substrings that exempt a key. There is
+  // no config key and no per-entry marker.
+  'volatile-key': (e, { volatileKeyAllow = [] } = {}) => {
+    const key = String(e.key ?? '');
+    if (!key.trim()) return null; // an empty key is `empty-key`'s to report.
+    // Tolerate a bare string as well as a list, so a caller passing
+    // `{ volatileKeyAllow: 'lorekit-231' }` does not silently iterate characters.
+    const allowList = Array.isArray(volatileKeyAllow) ? volatileKeyAllow : [volatileKeyAllow];
+    for (const allow of allowList) {
+      if (allow && key.includes(String(allow))) return null;
+    }
+    const digitRun = key.match(/\d{6,}/);
+    if (digitRun) {
+      return `key contains a volatile per-sighting identifier: '${digitRun[0]}' (a run of ${digitRun[0].length} digits)`;
+    }
+    // Boundary-anchored rather than split-then-match: splitting on `-` would
+    // separate `pr` from `231` and `pr-231` would slip through. The reference
+    // must start at a boundary (`:`, `-`, `_`, `/`, or the string start) and end
+    // at one, so `oauth2`/`sha256`/`wcag22` still never match.
+    const reference = key.match(/(?:^|[:\-_/])((?:pr|issue)[-_]?\d+)(?=$|[:\-_/])/i);
+    if (reference) {
+      return `key contains a volatile per-sighting identifier: '${reference[1]}' (a pr/issue number segment)`;
+    }
+    return null;
+  },
   'malformed-scope': (e) => {
     const reason = scopeIssue(e.scope);
     return reason ? `malformed scope: ${reason}` : null;
@@ -398,6 +434,8 @@ export function clusterDuplicates(entries = [], threshold = 0.8) {
 // `store.list({scope})` contract. Returns ordered per-scope groups plus a total
 // — a per-scope read failure is captured on the group, never thrown, so one bad
 // scope can't abort the listing. `store` may be a local or remote store.
+// Single-page only (the existing default behaviour). For full-scope traversal
+// use `gatherStream` below.
 export async function gather(store, scopes, filters = {}) {
   // Parse the taxonomy filters into value sets. `filters` is passed to the store
   // too (the remote narrows server-side); we ALSO post-filter the normalized
@@ -433,6 +471,208 @@ export async function gather(store, scopes, filters = {}) {
   return { groups, total };
 }
 
+// Default maximum entries to survey in a full-scope traversal. High enough to
+// cover almost every real scope; callers can raise or lower via `--max`.
+export const DEFAULT_MAX = 5000;
+
+// Page size used by `gatherStream` for remote stores.
+const STREAM_PAGE_LIMIT = 100;
+
+// Stream every page of entries across `scopes` from `store`, invoking
+// `onPage({ scope, entries })` once per page as entries arrive. Designed for
+// linear consumers (lint, stats, dedupe) that process page-by-page without
+// accumulating all rows in memory.
+//
+// Options:
+//   max        — hard cap on total surveyed entries (default: DEFAULT_MAX). When
+//                reached the walk stops and `capped` is set in the result.
+//   since      — ISO date/timestamp lower bound, forwarded as `created_since`.
+//   until      — ISO date/timestamp upper bound, forwarded as `created_until`.
+//   keyPrefix  — key prefix filter, forwarded as `key_prefix`.
+//   onPage     — callback invoked with `{ scope, entries }` per page.
+//
+// Returns `{ surveyed, total, capped, byScope }` where:
+//   surveyed  — total entries delivered to `onPage`.
+//   total     — exact aggregate from `listScopes()` when available (remote),
+//               else equal to `surveyed`.
+//   capped    — true when `max` stopped the walk before all pages were read.
+//   byScope   — `[{ scope, count, error }]` per scope (error=null on success).
+//
+// A per-scope read failure is captured on `byScope`, never thrown — mirrors
+// `gather()`'s resilience contract. `LocalStore.list` returns everything in one
+// page (no `nextCursor`), so the loop naturally terminates after one iteration
+// offline and adds no overhead.
+export async function gatherStream(store, scopes, {
+  max = DEFAULT_MAX,
+  since,
+  until,
+  keyPrefix,
+  onPage,
+} = {}) {
+  const byScope = [];
+  let surveyed = 0;
+  let capped = false;
+
+  // Attempt to get exact aggregate counts for the remote store via listScopes().
+  // Only available on stores that implement listScopes(); degrades gracefully.
+  let scopeCountMap = null;
+  if (typeof store.listScopes === 'function') {
+    try {
+      const ls = await store.listScopes();
+      if (ls && ls.ok && Array.isArray(ls.scopes)) {
+        scopeCountMap = new Map(ls.scopes.map((s) => [s.scope, s.count]));
+      }
+    } catch { /* best-effort */ }
+  }
+
+  for (const scope of scopes) {
+    if (capped) {
+      byScope.push({ scope, count: 0, error: null });
+      continue;
+    }
+    let cursor = undefined;
+    let scopeCount = 0;
+    let scopeError = null;
+
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+      const remaining = max - surveyed;
+      if (remaining <= 0) {
+        capped = true;
+        break;
+      }
+      const pageLimit = Math.min(STREAM_PAGE_LIMIT, remaining);
+      let res;
+      try {
+        res = await store.list({
+          scope,
+          limit: pageLimit,
+          cursor,
+          ...(since ? { created_since: since } : {}),
+          ...(until ? { created_until: until } : {}),
+          ...(keyPrefix ? { key_prefix: keyPrefix } : {}),
+        });
+      } catch (e) {
+        res = { ok: false, networkError: (e && e.message) || 'error' };
+      }
+      if (!res || res.ok === false) {
+        scopeError = describeError(res);
+        break;
+      }
+      const entries = (res.entries || []).map(normalizeEntry);
+      if (entries.length > 0) {
+        if (onPage) onPage({ scope, entries });
+        scopeCount += entries.length;
+        surveyed += entries.length;
+      }
+      // hasMore absent (local store) or false → done with this scope.
+      if (!res.hasMore || !res.nextCursor) break;
+      cursor = res.nextCursor;
+      // Safety: stop if we hit the cap after this page.
+      if (surveyed >= max) {
+        capped = true;
+        break;
+      }
+    }
+    byScope.push({ scope, count: scopeCount, error: scopeError });
+  }
+
+  // Derive total: use the aggregate when available (remote), else surveyed.
+  let total = surveyed;
+  if (scopeCountMap) {
+    total = 0;
+    for (const scope of scopes) {
+      total += scopeCountMap.get(scope) ?? 0;
+    }
+  }
+
+  return { surveyed, total, capped, byScope };
+}
+
+// Token-blocking near-duplicate clustering — a performance-bounded alternative
+// to `clusterDuplicates`'s O(n²) all-pairs sweep. Uses an inverted index
+// (token → indices) to generate only the candidate pairs that share at least
+// one token, then applies the SAME Jaccard `>= threshold` + union-find logic as
+// `clusterDuplicates`. The resulting clusters are provably identical:
+//
+//   IF similarity(a, b) >= threshold > 0 THEN a and b share at least one token,
+//   so they WILL appear as a candidate pair in the inverted-index sweep.
+//
+// Keeping `clusterDuplicates` intact means equivalence is a testable property
+// (AC-7) rather than a rewrite claim. Pure.
+export function clusterDuplicatesBlocked(entries = [], threshold = 0.8) {
+  const items = entries.map((e, i) => ({
+    i,
+    scope: e.scope ?? null,
+    key: e.key ?? null,
+    tokens: tokenize(e.value),
+  }));
+
+  // Build inverted index: token → [item indices]
+  /** @type {Map<string, number[]>} */
+  const invertedIndex = new Map();
+  for (const item of items) {
+    for (const token of item.tokens) {
+      if (!invertedIndex.has(token)) invertedIndex.set(token, []);
+      invertedIndex.get(token).push(item.i);
+    }
+  }
+
+  // Generate candidate pairs: any two items sharing at least one token.
+  // Use a Set of encoded pair keys to avoid duplicates.
+  const candidatePairs = new Set();
+  for (const indices of invertedIndex.values()) {
+    for (let x = 0; x < indices.length; x += 1) {
+      for (let y = x + 1; y < indices.length; y += 1) {
+        const a = Math.min(indices[x], indices[y]);
+        const b = Math.max(indices[x], indices[y]);
+        candidatePairs.add(`${a}:${b}`);
+      }
+    }
+  }
+
+  // Apply the SAME union-find as clusterDuplicates over candidate pairs only.
+  const parent = items.map((_, i) => i);
+  const find = (x) => {
+    while (parent[x] !== x) {
+      parent[x] = parent[parent[x]];
+      x = parent[x];
+    }
+    return x;
+  };
+  const pairs = [];
+  for (const pairKey of candidatePairs) {
+    const [a, b] = pairKey.split(':').map(Number);
+    const sim = similarity(items[a].tokens, items[b].tokens);
+    if (sim >= threshold) {
+      pairs.push({ a, b, sim });
+      parent[find(a)] = find(b);
+    }
+  }
+
+  // Assemble clusters — identical logic to clusterDuplicates.
+  const byRoot = new Map();
+  for (const it of items) {
+    const r = find(it.i);
+    if (!byRoot.has(r)) byRoot.set(r, []);
+    byRoot.get(r).push(it);
+  }
+  const clusters = [];
+  for (const members of byRoot.values()) {
+    if (members.length < 2) continue;
+    const idx = new Set(members.map((m) => m.i));
+    const sims = pairs.filter((p) => idx.has(p.a) && idx.has(p.b)).map((p) => p.sim);
+    clusters.push({
+      members: members.map((m) => ({ scope: m.scope, key: m.key })),
+      size: members.length,
+      minSimilarity: sims.length ? Math.min(...sims) : threshold,
+      maxSimilarity: sims.length ? Math.max(...sims) : threshold,
+    });
+  }
+  clusters.sort((x, y) => y.size - x.size);
+  return clusters;
+}
+
 // Render one section (Offline or Remote). `section` is either
 //   { available:false, reason }                         → a graceful note, or
 //   { available:true, groups, total }                   → grouped lessons.

package/src/lint.mjs

@@ -1,8 +1,8 @@
 // `lorekit lint` — flag low-quality lessons across the applicable scopes and
 // both stores. Each finding names the rule it violated (empty/whitespace value,
-// suspiciously short value, untrimmed value, empty key, malformed scope). The
-// rules are pure predicates in `lessons-view.mjs` (`LINT_RULES` / `lintEntry`),
-// each independently unit-tested.
+// suspiciously short value, untrimmed value, empty key, volatile key, malformed
+// scope). The rules are pure predicates in `lessons-view.mjs` (`LINT_RULES` /
+// `lintEntry`), each independently unit-tested.
 //
 // Exit convention: `lint` exits NON-ZERO (1) when any finding exists, so it is
 // usable as a CI gate (`lorekit lint || fail`); a clean run — or a run where the
@@ -15,7 +15,7 @@ import { resolveProjectRoot } from './config.mjs';
 import { deriveScope } from './scope.mjs';
 import { resolveDenies } from './control.mjs';
 import { resolveStores, remoteUnavailableReason } from './stores.mjs';
-import { scopeList, gather, lintGroups } from './lessons-view.mjs';
+import { scopeList, gather, gatherStream, lintGroups, DEFAULT_MAX } from './lessons-view.mjs';
 import { log, heading, status, c } from './util.mjs';
 
 export async function lint(args) {
@@ -36,9 +36,47 @@ export async function lint(args) {
   // Deny-wins section suppression, identical to the other read commands.
   const { localDenied, remoteDenied } = resolveDenies(root, { env });
 
-  const offlineResult = localDenied ? { groups: [], total: 0 } : lintGroups(await gather(local, scopes));
+  // `lint` defaults to full-scope survey (--all). Use gatherStream for remote
+  // to drain all pages; gather (single-page) still serves local store which is
+  // already exhaustive. --max, --since, --until are forwarded when provided.
+  const surveyMax = args.max !== undefined ? Number(args.max) : DEFAULT_MAX;
+  const surveySince = args.since || undefined;
+  const surveyUntil = args.until || undefined;
+
+  let offlineResult;
+  if (localDenied) {
+    offlineResult = { groups: [], total: 0 };
+  } else {
+    offlineResult = lintGroups(await gather(local, scopes));
+  }
+
   const remoteAvailable = !remoteDenied && remote.usable();
-  const remoteResult = remoteAvailable ? lintGroups(await gather(remote, scopes)) : { groups: [], total: 0 };
+  let remoteResult;
+  if (!remoteAvailable) {
+    remoteResult = { groups: [], total: 0 };
+  } else {
+    // Stream all pages, accumulating entries per scope for linting.
+    const accumulated = new Map();
+    for (const scope of scopes) accumulated.set(scope, []);
+    await gatherStream(remote, scopes, {
+      max: surveyMax,
+      since: surveySince,
+      until: surveyUntil,
+      onPage: ({ scope, entries }) => {
+        const arr = accumulated.get(scope);
+        if (arr) for (const e of entries) arr.push(e);
+      },
+    });
+    // Build a gather()-shaped result so lintGroups can consume it unchanged.
+    const groups = [];
+    let total = 0;
+    for (const scope of scopes) {
+      const entries = accumulated.get(scope) || [];
+      total += entries.length;
+      groups.push({ scope, entries, error: null });
+    }
+    remoteResult = lintGroups({ groups, total });
+  }
 
   const offlineSection = localDenied
     ? { available: false, reason: `disabled by deny constraint (${localDenied.source})` }

package/src/list.mjs

@@ -13,7 +13,7 @@ import { resolveProjectRoot } from './config.mjs';
 import { deriveScope } from './scope.mjs';
 import { resolveDenies } from './control.mjs';
 import { resolveStores, remoteUnavailableReason } from './stores.mjs';
-import { scopeList, gather, renderSection } from './lessons-view.mjs';
+import { scopeList, gather, gatherStream, renderSection, DEFAULT_MAX } from './lessons-view.mjs';
 import { resolveAppBase, mostSpecificScope } from './deeplink-pure.mjs';
 import { emitLink } from './link.mjs';
 import { log, heading, c } from './util.mjs';
@@ -69,9 +69,54 @@ export async function list(args) {
   // agent-facing control model enforces, honored here in the human read view.
   const { localDenied, remoteDenied } = resolveDenies(root, { env });
 
+  // --all: drain all pages from the remote store (for large scopes). Default
+  // is single-page per scope (what `list` has always shown). The local store is
+  // always exhaustive regardless of --all.
+  const surveyAll = Boolean(args.all);
+  const surveyMax = args.max !== undefined ? Number(args.max) : DEFAULT_MAX;
+  const surveySince = args.since || undefined;
+  const surveyUntil = args.until || undefined;
+
+  // Taxonomy `keep` predicate mirroring `gather()`'s: `gatherStream` neither
+  // forwards `kind`/`host` to the store nor post-filters, so a `--all` drain
+  // must apply the same narrowing here or `list --all --kind X` would ignore X.
+  const wanted = (v) =>
+    v == null ? null : new Set(String(v).split(',').map((s) => s.trim()).filter(Boolean));
+  const kindSet = wanted(filters.kind);
+  const hostSet = wanted(filters.host);
+  const keep = (e) =>
+    (!kindSet || (e.kind != null && kindSet.has(e.kind))) &&
+    (!hostSet || (e.host != null && hostSet.has(e.host)));
+
   const offline = localDenied ? { groups: [], total: 0 } : await gather(local, scopes, filters);
   const remoteAvailable = !remoteDenied && remote.usable();
-  const remoteResult = remoteAvailable ? await gather(remote, scopes, filters) : { groups: [], total: 0 };
+  let remoteResult;
+  if (!remoteAvailable) {
+    remoteResult = { groups: [], total: 0 };
+  } else if (surveyAll) {
+    // Full drain: accumulate all pages into a groups-shaped result.
+    const accumulated = new Map();
+    for (const scope of scopes) accumulated.set(scope, []);
+    await gatherStream(remote, scopes, {
+      max: surveyMax,
+      since: surveySince,
+      until: surveyUntil,
+      onPage: ({ scope, entries }) => {
+        const arr = accumulated.get(scope);
+        if (arr) for (const e of entries) if (keep(e)) arr.push(e);
+      },
+    });
+    const groups = [];
+    let total = 0;
+    for (const scope of scopes) {
+      const entries = accumulated.get(scope) || [];
+      total += entries.length;
+      groups.push({ scope, entries, error: null });
+    }
+    remoteResult = { groups, total };
+  } else {
+    remoteResult = await gather(remote, scopes, filters);
+  }
 
   const offlineSection = localDenied
     ? { available: false, reason: `disabled by deny constraint (${localDenied.source})` }

package/src/mcp-server.mjs

@@ -100,12 +100,31 @@ export const MEMORY_TOOL_DEFS = [
   {
     name: 'memory.list',
     description: 'List memories for a scope',
-    inputSchema: { type: 'object', required: ['scope'] },
+    inputSchema: {
+      type: 'object',
+      required: ['scope'],
+      properties: {
+        scope: { type: 'string' },
+        tags: { type: 'array', items: { type: 'string' } },
+        limit: { type: 'integer', minimum: 1, maximum: 100, default: 50 },
+        cursor: { type: 'string', description: 'Opaque cursor from a previous response\'s nextCursor. Omit to start from the first page.' },
+      },
+    },
   },
   {
     name: 'memory.search',
     description: 'Keyword search across memories',
-    inputSchema: { type: 'object', required: ['q'] },
+    inputSchema: {
+      type: 'object',
+      required: ['q'],
+      properties: {
+        q: { type: 'string' },
+        scopes: { type: 'array', items: { type: 'string' } },
+        tags: { type: 'array', items: { type: 'string' } },
+        limit: { type: 'integer', minimum: 1, maximum: 100, default: 20 },
+        cursor: { type: 'string', description: 'Opaque cursor from a previous response\'s nextCursor. Omit to start from the first page.' },
+      },
+    },
   },
   {
     name: 'memory.delete',

package/src/stats.mjs

@@ -20,7 +20,7 @@ import { resolveProjectRoot } from './config.mjs';
 import { deriveScope } from './scope.mjs';
 import { resolveDenies } from './control.mjs';
 import { resolveStores, remoteUnavailableReason } from './stores.mjs';
-import { scopeList, gather, tallyGroups } from './lessons-view.mjs';
+import { scopeList, gather, tallyGroups, summarizeScopeInventory, filterScopeInventory } from './lessons-view.mjs';
 import { log, heading, status, c } from './util.mjs';
 
 export async function stats(args) {
@@ -44,9 +44,32 @@ export async function stats(args) {
 
   const offlineTally = localDenied ? { perScope: [], total: 0 } : tallyGroups(await gather(local, scopes));
   const remoteAvailable = !remoteDenied && remote.usable();
-  const remoteTally = remoteAvailable
-    ? tallyGroups(await gather(remote, scopes))
-    : { perScope: [], total: 0 };
+
+  // Remote counts come from the GET /memories/scopes Postgres aggregate
+  // (RemoteStore.listScopes), which is exact at any scale — never row-draining.
+  // This is the fix for `stats`=50 vs `scopes`=487: the old code used
+  // gather() which returned only the first page from the remote store.
+  let remoteTally;
+  if (!remoteAvailable) {
+    remoteTally = { perScope: [], total: 0 };
+  } else {
+    const scopesRes = await remote.listScopes();
+    if (scopesRes && scopesRes.ok && Array.isArray(scopesRes.scopes)) {
+      // Filter to only the applicable scopes; total the filtered set.
+      const filtered = filterScopeInventory(scopesRes.scopes, null).filter(
+        (s) => scopes.includes(s.scope),
+      );
+      const perScope = scopes.map((scope) => {
+        const match = filtered.find((s) => s.scope === scope);
+        return { scope, count: match ? match.count : 0, error: null };
+      });
+      const total = perScope.reduce((n, s) => n + s.count, 0);
+      remoteTally = { perScope, total };
+    } else {
+      // Fallback when listScopes() is unavailable/errors — single-page gather.
+      remoteTally = tallyGroups(await gather(remote, scopes));
+    }
+  }
 
   const offlineSection = localDenied
     ? { available: false, reason: `disabled by deny constraint (${localDenied.source})` }

package/src/store/remote.mjs

@@ -58,19 +58,29 @@ class RemoteStore {
 
   // ── Memory operations → REST ──────────────────────────────────────────────
 
-  async list({ scope, tags, kind, host, limit } = {}) {
+  async list({ scope, tags, kind, host, limit, cursor, created_since, created_until, key_prefix } = {}) {
     const p = new URLSearchParams();
     if (scope) p.set('scope', scope);
     if (tags?.length) p.set('tags', Array.isArray(tags) ? tags.join(',') : tags);
     if (kind) p.set('kind', Array.isArray(kind) ? kind.join(',') : kind);
     if (host) p.set('host', Array.isArray(host) ? host.join(',') : host);
     if (limit) p.set('limit', String(limit));
+    if (cursor) p.set('cursor', cursor);
+    if (created_since) p.set('created_since', created_since);
+    if (created_until) p.set('created_until', created_until);
+    if (key_prefix) p.set('key_prefix', key_prefix);
     const res = await this._rest(`/memories?${p}`);
     if (!res.ok) return { ok: false, error: res.error, networkError: res.networkError };
-    return { ok: true, entries: res.data?.entries ?? [] };
+    const data = res.data ?? {};
+    return {
+      ok: true,
+      entries: data.entries ?? [],
+      hasMore: data.hasMore ?? false,
+      nextCursor: data.nextCursor ?? null,
+    };
   }
 
-  async search({ q, scopes, tags } = {}) {
+  async search({ q, scopes, tags, limit, cursor } = {}) {
     // A list of terms collapses into ONE `websearch` query joined by `OR`, so a
     // multi-term failure lookup is a single round-trip (the server FTS ORs them
     // and stems each). `failureQuery` distils terms to `[a-z0-9]+` tokens, so no
@@ -80,9 +90,17 @@ class RemoteStore {
     if (query) body.q = query;
     if (scopes?.length) body.scopes = scopes;
     if (tags?.length) body.tags = tags;
+    if (limit) body.limit = limit;
+    if (cursor) body.cursor = cursor;
     const res = await this._rest('/memories/search', { method: 'POST', body });
     if (!res.ok) return { ok: false, error: res.error, networkError: res.networkError };
-    return { ok: true, entries: res.data?.entries ?? [] };
+    const data = res.data ?? {};
+    return {
+      ok: true,
+      entries: data.entries ?? [],
+      hasMore: data.hasMore ?? false,
+      nextCursor: data.nextCursor ?? null,
+    };
   }
 
   async read({ scope, key } = {}) {