@lorekit/cli 1.39.2 → 1.40.0

package/README.md

@@ -103,7 +103,8 @@ from `.gitignore`, negate it with `!.mcp.json`, or `git add -f .mcp.json` once);
 `install --mcp-json` **warns when the file it wrote is still git-ignored**, so a
 fresh web clone silently missing the config is not a mystery. Once `.mcp.json` is
 tracked, only run `install --mcp-json` in that repo — a plain `install --project`
-would embed a live token in the now-committed file. See the
+would embed a live token in the now-committed file (and the command warns before
+it does). See the
 [Claude Code on the web guide](https://lorekit.io/docs/claude-code-web).
 
 In a TTY it prompts for the scope (and for `--endpoint` / `--token` if missing).

package/bin/lorekit.mjs

@@ -542,18 +542,27 @@ HEURISTIC — Jaccard overlap of lowercased word tokens, not a semantic/embeddin
 measure — so it surfaces candidates for a human to review, and can both miss
 paraphrases and group coincidental overlaps. Tune the cutoff with --threshold.
 
+Pass --cluster-by-key <regex> to cluster by KEY shape instead of value overlap:
+entries whose keys share the same first capture group (or full match) form one
+family. This catches coordinate-key debt — e.g. many pr{N}-{commentId} rows for
+one review comment — that the value heuristic misses when the values differ.
+Key-shape mode has no similarity cutoff, so --threshold and --cluster-by-key are
+mutually exclusive: passing both is a usage error rather than a silent ignore.
+
 ${c.bold('Options')}
-  -d, --dir <path>        Target project root (default: current directory)
-      --scope <scope>     Restrict to a single scope (default: all applicable)
-      --threshold <0..1>  Similarity cutoff to cluster a pair (default: 0.8)
-      --json              Machine-readable output (clusters + similarity signal)
-  -e, --endpoint <url>    Remote endpoint override (else .mcp.json / LOREKIT_MCP_URL)
-  -t, --token <token>     Remote token override (else .mcp.json / LOREKIT_TOKEN)
-      --store <path>      Local project-tier store directory (default: .lorekit)
+  -d, --dir <path>          Target project root (default: current directory)
+      --scope <scope>       Restrict to a single scope (default: all applicable)
+      --threshold <0..1>    Similarity cutoff to cluster a pair (default: 0.8)
+      --cluster-by-key <re> Cluster by shared key capture instead of value overlap
+      --json                Machine-readable output (clusters + signal)
+  -e, --endpoint <url>      Remote endpoint override (else .mcp.json / LOREKIT_MCP_URL)
+  -t, --token <token>       Remote token override (else .mcp.json / LOREKIT_TOKEN)
+      --store <path>        Local project-tier store directory (default: .lorekit)
 
 ${c.bold('Examples')}
   npx @lorekit/cli dedupe
   npx @lorekit/cli dedupe --threshold 0.6 --json
+  npx @lorekit/cli dedupe --cluster-by-key "(pr\\d+-\\d+)" --json
 `,
   link: `${c.bold('lorekit link')} — print a shareable dashboard deep-link URL ${c.dim('(alias: url)')}
 
@@ -652,7 +661,7 @@ const KNOWN_FLAGS = [
   'link', 'base', 'q', 'owner', 'range', 'view', 'archived',
   'origin-repo', 'origin-branch', 'origin-commit', 'origin-pr', 'no-origin',
   // Scale-aware survey flags
-  'all', 'max', 'since', 'until', 'key-prefix',
+  'all', 'max', 'since', 'until', 'key-prefix', 'cluster-by-key',
 ];
 
 // Commands that write to disk / talk to the network on a human's behalf. These

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lorekit/cli",
-  "version": "1.39.2",
+  "version": "1.40.0",
   "description": "Install the LoreKit shared-memory skill and run health checks for the LoreKit MCP server.",
   "license": "MIT",
   "repository": {

package/src/config.mjs

@@ -591,7 +591,12 @@ export function tokenKind(token) {
 // avoid a circular import with mcp.mjs.
 //
 // A source that STORES a token wins outright (project beats global — closest
-// scope), but a TOKENLESS source must NOT shadow a later source that has one.
+// scope), and it brings its OWN endpoint: a token authenticates one endpoint,
+// so the two must travel together. A TOKENLESS source therefore never shadows a
+// later source that has a token — not its token AND not its endpoint; its
+// endpoint is only remembered as a fallback, used solely when NO source stores a
+// token. (So "closest-scope-first" governs which token wins; the endpoint simply
+// follows that token.)
 // That shadowing is exactly what `install --global --mcp-json` created: it
 // writes a committable, token-free project .mcp.json (auth via ${LOREKIT_TOKEN})
 // AND the real token into ~/.claude.json. Returning early on the project entry

package/src/dedupe.mjs

@@ -15,13 +15,18 @@ import { resolveProjectRoot, readLorekitJson } from './config.mjs';
 import { deriveScope } from './scope.mjs';
 import { resolveDenies } from './control.mjs';
 import { resolveStores, remoteUnavailableReason } from './stores.mjs';
-import { scopeList, gather, gatherStream, clusterDuplicates, clusterDuplicatesBlocked, DEFAULT_MAX } from './lessons-view.mjs';
-import { log, heading, status, c } from './util.mjs';
+import { scopeList, gather, gatherStream, clusterDuplicates, clusterDuplicatesBlocked, clusterByKeyPattern, compileKeyPattern, DEFAULT_MAX } from './lessons-view.mjs';
+import { log, heading, status, err, c } from './util.mjs';
 
 const DEFAULT_THRESHOLD = 0.8;
-// Maximum entries to accumulate before the token-blocking index becomes
-// memory-prohibitive. Beyond this the user must narrow via --key-prefix /
-// --since / --max.
+// Maximum entries to accumulate before the survey becomes memory-prohibitive.
+// It bounds the accumulated entry list itself, so it applies in BOTH modes —
+// key-shape clustering is O(n) and needs no blocking index, but it still holds
+// the whole population in memory, and an unbounded remote drain is the risk the
+// cap exists for. In value mode it additionally bounds the token-blocking index,
+// which is the super-linear part. Beyond the cap the results are genuinely
+// partial in either mode, and the user must narrow via --key-prefix / --since /
+// --max.
 const DEDUPE_POP_CAP = 2000;
 
 // Smallest threshold the blocked clusterer accepts. `clusterDuplicatesBlocked`
@@ -77,6 +82,34 @@ export async function dedupe(args) {
     args.threshold !== undefined
       ? parseThreshold(args.threshold)
       : (repoThreshold(root) ?? DEFAULT_THRESHOLD);
+
+  // `--cluster-by-key <regex>` switches from value-overlap clustering to KEY-shape
+  // clustering: entries whose keys share the same first capture group (or full
+  // match) of the regex are grouped as a duplicate family — catches coordinate-key
+  // debt (e.g. `bucket::pr{N}-{commentId}::slug`) that the Jaccard heuristic misses
+  // when the values differ. A bare flag (no value) or an unparseable regex is a
+  // usage error: report it and exit non-zero rather than silently surveying by value.
+  const clusterByKeyRaw = args['cluster-by-key'];
+  const byKeyMode = clusterByKeyRaw !== undefined;
+  const keyPattern = byKeyMode ? compileKeyPattern(clusterByKeyRaw) : null;
+  if (byKeyMode && !keyPattern) {
+    err(
+      clusterByKeyRaw === true
+        ? '--cluster-by-key needs a regex value, e.g. --cluster-by-key "(pr\\d+-\\d+)"'
+        : `--cluster-by-key: invalid regex ${JSON.stringify(String(clusterByKeyRaw))}`,
+    );
+    return { exitCode: 1 };
+  }
+  // Key-shape mode has no similarity cutoff, so an explicit `--threshold` would
+  // be silently ignored — the same silent fallback the bad-regex branch above
+  // refuses. Refuse it too. Only the EXPLICIT flag errors: a repo-level
+  // `dedupe.threshold` in .lorekit.json is a value-mode default, not a request,
+  // so it must not break a key-mode run.
+  if (byKeyMode && args.threshold !== undefined) {
+    err('--threshold is not used with --cluster-by-key (key-shape clustering has no similarity cutoff); drop one of them');
+    return { exitCode: 1 };
+  }
+
   const scopeInfo = deriveScope(root);
   // Default to every applicable scope; `--scope <s>` narrows to one.
   const scopes = args.scope && typeof args.scope === 'string' ? [args.scope] : scopeList(scopeInfo);
@@ -156,7 +189,9 @@ export async function dedupe(args) {
       }
       return {
         available: true,
-        clusters: clusterDuplicatesBlocked(entries, threshold),
+        clusters: byKeyMode
+          ? clusterByKeyPattern(entries, keyPattern)
+          : clusterDuplicatesBlocked(entries, threshold),
         errored: flat.errored,
         popCapped,
       };
@@ -164,7 +199,9 @@ export async function dedupe(args) {
     const { entries, errored, popCapped } = await streamAccumulate(store);
     return {
       available: true,
-      clusters: clusterDuplicatesBlocked(entries, threshold),
+      clusters: byKeyMode
+        ? clusterByKeyPattern(entries, keyPattern)
+        : clusterDuplicatesBlocked(entries, threshold),
       errored,
       popCapped,
     };
@@ -188,12 +225,16 @@ export async function dedupe(args) {
   const remoteClusters = remoteSection.available ? remoteSection.clusters.length : 0;
 
   if (args.json) {
-    log(JSON.stringify(buildJson({ root, scopes, threshold, offlineSection, remoteSection }), null, 2));
+    log(JSON.stringify(buildJson({ root, scopes, threshold, byKeyMode, keyPattern, offlineSection, remoteSection }), null, 2));
   } else {
     heading('LoreKit dedupe');
     log(`  project: ${c.dim(root)}`);
     log(`  scopes:  ${scopes.join('  →  ')}`);
-    log(`  ${c.dim(`heuristic: Jaccard word-token overlap >= ${threshold} (not semantic)`)}`);
+    log(
+      byKeyMode
+        ? `  ${c.dim(`key-shape: clustering by shared key capture of /${keyPattern.source}/`)}`
+        : `  ${c.dim(`heuristic: Jaccard word-token overlap >= ${threshold} (not semantic)`)}`,
+    );
 
     if (offlineSection.available && offlineSection.popCapped) {
       log(`  ${c.yellow('!')} population cap (${DEDUPE_POP_CAP}) reached for Offline — results are partial. Narrow with --key-prefix, --since, or --max.`);
@@ -211,7 +252,13 @@ export async function dedupe(args) {
     log('');
     const total = offlineClusters + remoteClusters;
     if (total === 0) {
-      log(`  ${c.green('✓')} no likely-duplicate clusters at this threshold`);
+      // "at this threshold" is only true in value mode — key-shape mode has no
+      // cutoff, so name the pattern that found nothing instead.
+      log(
+        byKeyMode
+          ? `  ${c.green('✓')} no key-shape clusters for /${keyPattern.source}/`
+          : `  ${c.green('✓')} no likely-duplicate clusters at this threshold`,
+      );
     } else {
       const plural = total === 1 ? '' : 's';
       log(`  ${c.yellow('!')} ${total} duplicate cluster${plural} found`);
@@ -220,11 +267,14 @@ export async function dedupe(args) {
   }
 
   // Bounded, non-PII telemetry extras — counts + a boolean, never a scope
-  // string, key, path, or token.
+  // string, key, path, or token. `threshold` is emitted only in value mode,
+  // where it is the cutoff actually applied; the key-mode pattern is user text
+  // and is never emitted, only the bounded mode name.
   return {
     exitCode: 0,
     'lorekit.cli.dedupe.scope_count': scopes.length,
-    'lorekit.cli.dedupe.threshold': threshold,
+    'lorekit.cli.dedupe.mode': byKeyMode ? 'key' : 'value',
+    ...(byKeyMode ? {} : { 'lorekit.cli.dedupe.threshold': threshold }),
     'lorekit.cli.dedupe.offline_clusters': offlineClusters,
     'lorekit.cli.dedupe.remote_clusters': remoteClusters,
     'lorekit.cli.dedupe.remote_available': remoteAvailable,
@@ -255,25 +305,35 @@ function renderDedupeSection(header, section) {
   let n = 0;
   for (const cluster of section.clusters) {
     n += 1;
-    const range =
-      cluster.minSimilarity === cluster.maxSimilarity
-        ? cluster.minSimilarity.toFixed(2)
-        : `${cluster.minSimilarity.toFixed(2)}–${cluster.maxSimilarity.toFixed(2)}`;
-    log(`  ${c.yellow('•')} cluster ${n} ${c.dim(`(${cluster.size} memories, similarity ${range})`)}`);
+    // Key-shape clusters carry a `keyGroup`; value-overlap clusters carry a
+    // similarity range. Render whichever signal the cluster has.
+    let signal;
+    if (cluster.keyGroup !== undefined) {
+      signal = `${cluster.size} memories, key-group "${cluster.keyGroup}"`;
+    } else {
+      const range =
+        cluster.minSimilarity === cluster.maxSimilarity
+          ? cluster.minSimilarity.toFixed(2)
+          : `${cluster.minSimilarity.toFixed(2)}–${cluster.maxSimilarity.toFixed(2)}`;
+      signal = `${cluster.size} memories, similarity ${range}`;
+    }
+    log(`  ${c.yellow('•')} cluster ${n} ${c.dim(`(${signal})`)}`);
     for (const m of cluster.members) {
       log(`    ${c.cyan('-')} ${m.scope}::${m.key}`);
     }
   }
 }
 
-// The `--json` payload: `{ root, scopes, threshold, offline, remote }` — each
-// store a `{ available, clusters: [{ members, size, minSimilarity,
-// maxSimilarity }], errored }` record (or an unavailable note).
-function buildJson({ root, scopes, threshold, offlineSection, remoteSection }) {
+// The `--json` payload: `{ root, scopes, mode, threshold|keyPattern, offline,
+// remote }`. In value mode (`mode: "value"`) each cluster carries `minSimilarity`
+// / `maxSimilarity`; in key-shape mode (`mode: "key"`) each carries `keyGroup`.
+// Each store is a `{ available, clusters, errored }` record (or an unavailable note).
+function buildJson({ root, scopes, threshold, byKeyMode, keyPattern, offlineSection, remoteSection }) {
   return {
     root,
     scopes,
-    threshold,
+    mode: byKeyMode ? 'key' : 'value',
+    ...(byKeyMode ? { keyPattern: keyPattern.source } : { threshold }),
     offline: sectionJson(offlineSection),
     remote: sectionJson(remoteSection),
   };

package/src/install.mjs

@@ -23,6 +23,8 @@ import {
   homeDir,
   mcpConfigPath,
   readJsonIfExists,
+  readLorekitServer,
+  isWebMcpServerEntry,
 } from './config.mjs';
 import { buildRemoteUrl, splitEndpoint } from './mcp.mjs';
 import { deriveScope } from './scope.mjs';
@@ -364,6 +366,21 @@ export async function install(args) {
   let file = null;
   let existed = false;
   if (!scopeWriteOwnedByWeb) {
+    // A plain project install embeds the token in .mcp.json. If that file
+    // currently holds the committable web form (written by an earlier
+    // `--mcp-json`), this write replaces it with an embedded secret — in the
+    // very file the docs tell you to commit. Warn before clobbering it;
+    // `isWebMcpServerEntry` recognises the shape we are about to overwrite.
+    if (scope === 'project' && token) {
+      const prior = readLorekitServer(root);
+      if (prior && isWebMcpServerEntry(prior.server)) {
+        status(
+          'warn',
+          '.mcp.json',
+          `replacing the committable web entry with an embedded token — do not commit this file, or re-run with --mcp-json to keep the \${${WEB_TOKEN_ENV_VAR}} form`,
+        );
+      }
+    }
     ({ file, existed } = upsertMcpServer(root, remoteUrl, scope));
   }
 
@@ -527,7 +544,7 @@ export async function install(args) {
       'warn',
       'token',
       `not stored — the committable .mcp.json resolves \${${WEB_TOKEN_ENV_VAR}} at runtime; set it as an environment secret${
-        token ? ' (any --token you passed is not persisted)' : ''
+        plan.action === 'flag' ? ' (the token you supplied is not persisted)' : ''
       }`,
     );
   } else if (kind === 'none') {

package/src/lessons-view.mjs

@@ -430,6 +430,57 @@ export function clusterDuplicates(entries = [], threshold = 0.8) {
   return clusters;
 }
 
+// Compile a `--cluster-by-key` pattern into a stateless RegExp, or null on bad
+// input (empty / non-string / unparseable) — never throws, mirroring
+// `parseThreshold`'s "never crash on bad input" contract. The source is always a
+// STRING, so `new RegExp(raw)` carries no flags and is stateless by construction
+// — flag stripping is only needed on the RegExp branch of `clusterByKeyPattern`,
+// which can be handed a caller-built `/…/g`. Pure.
+export function compileKeyPattern(raw) {
+  if (typeof raw !== 'string' || raw.length === 0) return null;
+  try {
+    return new RegExp(raw);
+  } catch {
+    return null;
+  }
+}
+
+// Cluster entries by a shared KEY capture rather than value overlap. Two entries
+// cluster iff their keys yield the SAME first capture group of `pattern` (or the
+// same full match when the pattern has no capture group). This catches
+// coordinate-key duplicate FAMILIES — many `bucket::pr{N}-{commentId}::slug` rows
+// recorded for one review comment, whose slugs (and thus values) differ enough
+// that `clusterDuplicates`'s Jaccard sweep never links them. Returns only 2+
+// clusters, each `{ members: [{ scope, key }], size, keyGroup }`, largest first
+// (ties broken by keyGroup for stable output). A key that doesn't match the
+// pattern is left unclustered. Pure — O(n), no threshold.
+export function clusterByKeyPattern(entries = [], pattern) {
+  const re =
+    pattern instanceof RegExp
+      ? new RegExp(pattern.source, pattern.flags.replace(/[gy]/g, ''))
+      : compileKeyPattern(pattern);
+  if (!re) return [];
+  const byGroup = new Map();
+  for (const e of entries) {
+    const key = e.key == null ? '' : String(e.key);
+    re.lastIndex = 0;
+    const m = re.exec(key);
+    if (!m) continue;
+    const group = m[1] ?? m[0];
+    if (!byGroup.has(group)) byGroup.set(group, []);
+    byGroup.get(group).push({ scope: e.scope ?? null, key });
+  }
+  const clusters = [];
+  for (const [keyGroup, members] of byGroup) {
+    if (members.length < 2) continue;
+    clusters.push({ members, size: members.length, keyGroup });
+  }
+  clusters.sort(
+    (x, y) => y.size - x.size || String(x.keyGroup).localeCompare(String(y.keyGroup)),
+  );
+  return clusters;
+}
+
 // Collect a store's non-archived entries for each scope, via the common
 // `store.list({scope})` contract. Returns ordered per-scope groups plus a total
 // — a per-scope read failure is captured on the group, never thrown, so one bad