@soleri/core 9.3.1 → 9.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (205) hide show
  1. package/dist/brain/intelligence.d.ts +5 -0
  2. package/dist/brain/intelligence.d.ts.map +1 -1
  3. package/dist/brain/intelligence.js +115 -26
  4. package/dist/brain/intelligence.js.map +1 -1
  5. package/dist/brain/learning-radar.d.ts +3 -3
  6. package/dist/brain/learning-radar.d.ts.map +1 -1
  7. package/dist/brain/learning-radar.js +8 -4
  8. package/dist/brain/learning-radar.js.map +1 -1
  9. package/dist/control/intent-router.d.ts +2 -2
  10. package/dist/control/intent-router.d.ts.map +1 -1
  11. package/dist/control/intent-router.js +35 -1
  12. package/dist/control/intent-router.js.map +1 -1
  13. package/dist/control/types.d.ts +10 -2
  14. package/dist/control/types.d.ts.map +1 -1
  15. package/dist/curator/curator.d.ts +4 -0
  16. package/dist/curator/curator.d.ts.map +1 -1
  17. package/dist/curator/curator.js +23 -1
  18. package/dist/curator/curator.js.map +1 -1
  19. package/dist/curator/schema.d.ts +1 -1
  20. package/dist/curator/schema.d.ts.map +1 -1
  21. package/dist/curator/schema.js +8 -0
  22. package/dist/curator/schema.js.map +1 -1
  23. package/dist/domain-packs/types.d.ts +6 -0
  24. package/dist/domain-packs/types.d.ts.map +1 -1
  25. package/dist/domain-packs/types.js +1 -0
  26. package/dist/domain-packs/types.js.map +1 -1
  27. package/dist/engine/module-manifest.js +3 -3
  28. package/dist/engine/module-manifest.js.map +1 -1
  29. package/dist/engine/register-engine.d.ts +9 -0
  30. package/dist/engine/register-engine.d.ts.map +1 -1
  31. package/dist/engine/register-engine.js +59 -1
  32. package/dist/engine/register-engine.js.map +1 -1
  33. package/dist/facades/types.d.ts +5 -1
  34. package/dist/facades/types.d.ts.map +1 -1
  35. package/dist/facades/types.js.map +1 -1
  36. package/dist/hooks/candidate-scorer.d.ts +28 -0
  37. package/dist/hooks/candidate-scorer.d.ts.map +1 -0
  38. package/dist/hooks/candidate-scorer.js +20 -0
  39. package/dist/hooks/candidate-scorer.js.map +1 -0
  40. package/dist/hooks/index.d.ts +2 -0
  41. package/dist/hooks/index.d.ts.map +1 -0
  42. package/dist/hooks/index.js +2 -0
  43. package/dist/hooks/index.js.map +1 -0
  44. package/dist/index.d.ts +4 -1
  45. package/dist/index.d.ts.map +1 -1
  46. package/dist/index.js +3 -0
  47. package/dist/index.js.map +1 -1
  48. package/dist/operator/operator-context-store.d.ts +54 -0
  49. package/dist/operator/operator-context-store.d.ts.map +1 -0
  50. package/dist/operator/operator-context-store.js +434 -0
  51. package/dist/operator/operator-context-store.js.map +1 -0
  52. package/dist/operator/operator-context-types.d.ts +101 -0
  53. package/dist/operator/operator-context-types.d.ts.map +1 -0
  54. package/dist/operator/operator-context-types.js +27 -0
  55. package/dist/operator/operator-context-types.js.map +1 -0
  56. package/dist/packs/index.d.ts +2 -2
  57. package/dist/packs/index.d.ts.map +1 -1
  58. package/dist/packs/index.js +1 -1
  59. package/dist/packs/index.js.map +1 -1
  60. package/dist/packs/lockfile.d.ts +3 -0
  61. package/dist/packs/lockfile.d.ts.map +1 -1
  62. package/dist/packs/lockfile.js.map +1 -1
  63. package/dist/packs/types.d.ts +8 -2
  64. package/dist/packs/types.d.ts.map +1 -1
  65. package/dist/packs/types.js +6 -0
  66. package/dist/packs/types.js.map +1 -1
  67. package/dist/planning/plan-lifecycle.d.ts +12 -1
  68. package/dist/planning/plan-lifecycle.d.ts.map +1 -1
  69. package/dist/planning/plan-lifecycle.js +54 -16
  70. package/dist/planning/plan-lifecycle.js.map +1 -1
  71. package/dist/planning/planner-types.d.ts +6 -0
  72. package/dist/planning/planner-types.d.ts.map +1 -1
  73. package/dist/planning/planner.d.ts +21 -1
  74. package/dist/planning/planner.d.ts.map +1 -1
  75. package/dist/planning/planner.js +62 -3
  76. package/dist/planning/planner.js.map +1 -1
  77. package/dist/planning/task-complexity-assessor.d.ts.map +1 -1
  78. package/dist/planning/task-complexity-assessor.js.map +1 -1
  79. package/dist/plugins/types.d.ts +18 -18
  80. package/dist/runtime/admin-ops.d.ts +1 -1
  81. package/dist/runtime/admin-ops.d.ts.map +1 -1
  82. package/dist/runtime/admin-ops.js +100 -3
  83. package/dist/runtime/admin-ops.js.map +1 -1
  84. package/dist/runtime/admin-setup-ops.d.ts.map +1 -1
  85. package/dist/runtime/admin-setup-ops.js +19 -9
  86. package/dist/runtime/admin-setup-ops.js.map +1 -1
  87. package/dist/runtime/capture-ops.d.ts.map +1 -1
  88. package/dist/runtime/capture-ops.js +35 -7
  89. package/dist/runtime/capture-ops.js.map +1 -1
  90. package/dist/runtime/facades/brain-facade.d.ts.map +1 -1
  91. package/dist/runtime/facades/brain-facade.js +4 -2
  92. package/dist/runtime/facades/brain-facade.js.map +1 -1
  93. package/dist/runtime/facades/control-facade.d.ts.map +1 -1
  94. package/dist/runtime/facades/control-facade.js +8 -2
  95. package/dist/runtime/facades/control-facade.js.map +1 -1
  96. package/dist/runtime/facades/curator-facade.d.ts.map +1 -1
  97. package/dist/runtime/facades/curator-facade.js +13 -0
  98. package/dist/runtime/facades/curator-facade.js.map +1 -1
  99. package/dist/runtime/facades/memory-facade.d.ts.map +1 -1
  100. package/dist/runtime/facades/memory-facade.js +10 -12
  101. package/dist/runtime/facades/memory-facade.js.map +1 -1
  102. package/dist/runtime/facades/orchestrate-facade.d.ts.map +1 -1
  103. package/dist/runtime/facades/orchestrate-facade.js +36 -1
  104. package/dist/runtime/facades/orchestrate-facade.js.map +1 -1
  105. package/dist/runtime/facades/plan-facade.d.ts.map +1 -1
  106. package/dist/runtime/facades/plan-facade.js +20 -4
  107. package/dist/runtime/facades/plan-facade.js.map +1 -1
  108. package/dist/runtime/orchestrate-ops.d.ts.map +1 -1
  109. package/dist/runtime/orchestrate-ops.js +71 -4
  110. package/dist/runtime/orchestrate-ops.js.map +1 -1
  111. package/dist/runtime/plan-feedback-helper.d.ts +21 -0
  112. package/dist/runtime/plan-feedback-helper.d.ts.map +1 -0
  113. package/dist/runtime/plan-feedback-helper.js +52 -0
  114. package/dist/runtime/plan-feedback-helper.js.map +1 -0
  115. package/dist/runtime/planning-extra-ops.d.ts.map +1 -1
  116. package/dist/runtime/planning-extra-ops.js +73 -34
  117. package/dist/runtime/planning-extra-ops.js.map +1 -1
  118. package/dist/runtime/session-briefing.d.ts.map +1 -1
  119. package/dist/runtime/session-briefing.js +9 -1
  120. package/dist/runtime/session-briefing.js.map +1 -1
  121. package/dist/runtime/types.d.ts +3 -0
  122. package/dist/runtime/types.d.ts.map +1 -1
  123. package/dist/skills/sync-skills.d.ts.map +1 -1
  124. package/dist/skills/sync-skills.js +13 -7
  125. package/dist/skills/sync-skills.js.map +1 -1
  126. package/package.json +1 -1
  127. package/src/brain/brain-intelligence.test.ts +30 -0
  128. package/src/brain/brain.ts +120 -46
  129. package/src/brain/extraction-quality.test.ts +323 -0
  130. package/src/brain/intelligence.ts +175 -64
  131. package/src/brain/learning-radar.ts +8 -5
  132. package/src/brain/second-brain-features.test.ts +1 -1
  133. package/src/chat/agent-loop.ts +1 -1
  134. package/src/chat/notifications.ts +4 -0
  135. package/src/control/intent-router.test.ts +73 -3
  136. package/src/control/intent-router.ts +48 -9
  137. package/src/control/types.ts +13 -2
  138. package/src/curator/curator.test.ts +92 -0
  139. package/src/curator/curator.ts +162 -18
  140. package/src/curator/schema.ts +8 -0
  141. package/src/domain-packs/types.ts +8 -0
  142. package/src/engine/module-manifest.test.ts +8 -2
  143. package/src/engine/module-manifest.ts +3 -3
  144. package/src/engine/register-engine.test.ts +73 -1
  145. package/src/engine/register-engine.ts +61 -1
  146. package/src/facades/types.ts +5 -0
  147. package/src/hooks/candidate-scorer.test.ts +76 -0
  148. package/src/hooks/candidate-scorer.ts +39 -0
  149. package/src/hooks/index.ts +6 -0
  150. package/src/index.ts +24 -0
  151. package/src/llm/llm-client.ts +1 -0
  152. package/src/operator/operator-context-store.test.ts +698 -0
  153. package/src/operator/operator-context-store.ts +569 -0
  154. package/src/operator/operator-context-types.ts +139 -0
  155. package/src/packs/index.ts +3 -1
  156. package/src/packs/lockfile.ts +3 -0
  157. package/src/packs/types.ts +9 -0
  158. package/src/persistence/sqlite-provider.ts +1 -0
  159. package/src/planning/github-projection.ts +48 -44
  160. package/src/planning/plan-lifecycle.ts +93 -22
  161. package/src/planning/planner-types.ts +6 -0
  162. package/src/planning/planner.ts +74 -4
  163. package/src/planning/task-complexity-assessor.test.ts +6 -2
  164. package/src/planning/task-complexity-assessor.ts +1 -4
  165. package/src/queue/pipeline-runner.ts +4 -0
  166. package/src/runtime/admin-ops.test.ts +139 -6
  167. package/src/runtime/admin-ops.ts +104 -3
  168. package/src/runtime/admin-setup-ops.ts +30 -10
  169. package/src/runtime/capture-ops.test.ts +84 -0
  170. package/src/runtime/capture-ops.ts +35 -7
  171. package/src/runtime/curator-extra-ops.test.ts +7 -0
  172. package/src/runtime/curator-extra-ops.ts +10 -1
  173. package/src/runtime/facades/admin-facade.test.ts +1 -1
  174. package/src/runtime/facades/brain-facade.ts +6 -3
  175. package/src/runtime/facades/control-facade.ts +10 -2
  176. package/src/runtime/facades/curator-facade.test.ts +7 -0
  177. package/src/runtime/facades/curator-facade.ts +18 -0
  178. package/src/runtime/facades/memory-facade.test.ts +14 -12
  179. package/src/runtime/facades/memory-facade.ts +197 -12
  180. package/src/runtime/facades/orchestrate-facade.ts +33 -1
  181. package/src/runtime/facades/plan-facade.test.ts +213 -0
  182. package/src/runtime/facades/plan-facade.ts +23 -4
  183. package/src/runtime/orchestrate-ops.test.ts +202 -2
  184. package/src/runtime/orchestrate-ops.ts +88 -7
  185. package/src/runtime/plan-feedback-helper.test.ts +173 -0
  186. package/src/runtime/plan-feedback-helper.ts +63 -0
  187. package/src/runtime/planning-extra-ops.test.ts +43 -1
  188. package/src/runtime/planning-extra-ops.ts +96 -33
  189. package/src/runtime/runtime.test.ts +50 -2
  190. package/src/runtime/runtime.ts +117 -89
  191. package/src/runtime/session-briefing.test.ts +1 -0
  192. package/src/runtime/session-briefing.ts +10 -1
  193. package/src/runtime/shutdown-registry.test.ts +151 -0
  194. package/src/runtime/shutdown-registry.ts +85 -0
  195. package/src/runtime/types.ts +7 -1
  196. package/src/skills/sync-skills.ts +14 -7
  197. package/src/transport/http-server.ts +50 -3
  198. package/src/transport/ws-server.ts +8 -0
  199. package/src/vault/linking.test.ts +12 -0
  200. package/src/vault/linking.ts +90 -44
  201. package/src/vault/vault-maintenance.ts +11 -18
  202. package/src/vault/vault-memories.ts +21 -13
  203. package/src/vault/vault-schema.ts +21 -0
  204. package/src/vault/vault.ts +8 -3
  205. package/vitest.config.ts +1 -0
@@ -647,4 +647,96 @@ describe('Curator', () => {
647
647
  expect(result.metrics.tagHealth).toBeLessThan(1);
648
648
  });
649
649
  });
650
+
651
+ // ─── Duplicate Dismissal ──────────────────────────────────────
652
+
653
+ describe('Duplicate Dismissal', () => {
654
+ it('dismissDuplicate stores a pair and detectDuplicates filters it out', () => {
655
+ // Add two similar entries that will be flagged as duplicates
656
+ vault.add(
657
+ makeEntry({
658
+ id: 'dup-a',
659
+ title: 'React hook cleanup pattern',
660
+ description:
661
+ 'Always clean up useEffect hooks with return cleanup function to prevent memory leaks and stale closures.',
662
+ }),
663
+ );
664
+ vault.add(
665
+ makeEntry({
666
+ id: 'dup-b',
667
+ title: 'React hook cleanup pattern for effects',
668
+ description:
669
+ 'Clean up useEffect hooks by returning a cleanup function to avoid memory leaks and stale closures.',
670
+ }),
671
+ );
672
+
673
+ // Verify they show up as duplicates
674
+ const before = curator.detectDuplicates();
675
+ const dupBefore = before.find(
676
+ (d) => d.entryId === 'dup-a' && d.matches.some((m) => m.entryId === 'dup-b'),
677
+ );
678
+ expect(dupBefore).toBeDefined();
679
+
680
+ // Dismiss the pair
681
+ const result = curator.dismissDuplicate('dup-a', 'dup-b', 'intentionally distinct');
682
+ expect(result.dismissed).toBe(true);
683
+
684
+ // Verify they no longer show up
685
+ const after = curator.detectDuplicates();
686
+ const dupAfter = after.find(
687
+ (d) => d.entryId === 'dup-a' && d.matches.some((m) => m.entryId === 'dup-b'),
688
+ );
689
+ expect(dupAfter).toBeUndefined();
690
+ });
691
+
692
+ it('dismissDuplicate works with swapped ID order', () => {
693
+ vault.add(
694
+ makeEntry({
695
+ id: 'sw-a',
696
+ title: 'Semantic color tokens for error states',
697
+ description: 'Use semantic tokens like text-error and bg-error-subtle for error UI.',
698
+ }),
699
+ );
700
+ vault.add(
701
+ makeEntry({
702
+ id: 'sw-b',
703
+ title: 'Semantic color tokens for error UI',
704
+ description:
705
+ 'Always prefer semantic tokens text-error and bg-error-subtle over raw colors.',
706
+ }),
707
+ );
708
+
709
+ // Dismiss with b,a order
710
+ curator.dismissDuplicate('sw-b', 'sw-a');
711
+
712
+ // Should still be filtered (stored as sorted pair)
713
+ const results = curator.detectDuplicates('sw-a');
714
+ const match = results.find((r) => r.matches.some((m) => m.entryId === 'sw-b'));
715
+ expect(match).toBeUndefined();
716
+ });
717
+
718
+ it('dismissDuplicate is idempotent', () => {
719
+ vault.add(
720
+ makeEntry({
721
+ id: 'idem-a',
722
+ title: 'Pattern A idempotent test',
723
+ description: 'First entry for idempotency test of duplicate dismissal.',
724
+ }),
725
+ );
726
+ vault.add(
727
+ makeEntry({
728
+ id: 'idem-b',
729
+ title: 'Pattern A idempotent test duplicate',
730
+ description: 'Second entry for idempotency test of duplicate dismissal.',
731
+ }),
732
+ );
733
+
734
+ const first = curator.dismissDuplicate('idem-a', 'idem-b');
735
+ expect(first.dismissed).toBe(true);
736
+
737
+ // Second call should be a no-op (INSERT OR IGNORE)
738
+ const second = curator.dismissDuplicate('idem-a', 'idem-b');
739
+ expect(second.dismissed).toBe(false);
740
+ });
741
+ });
650
742
  });
@@ -17,14 +17,17 @@ import type {
17
17
  } from './types.js';
18
18
 
19
19
  import {
20
- detectDuplicates as detectDuplicatesPure,
21
20
  DEFAULT_DUPLICATE_THRESHOLD,
21
+ MERGE_SUGGESTION_THRESHOLD,
22
+ buildVocabulary,
23
+ entryToText,
22
24
  } from './duplicate-detector.js';
23
25
  import {
24
26
  findContradictions,
25
27
  DEFAULT_CONTRADICTION_THRESHOLD,
26
28
  type ContradictionCandidate,
27
29
  } from './contradiction-detector.js';
30
+ import { tokenize, calculateTfIdf, cosineSimilarity } from '../text/similarity.js';
28
31
  import {
29
32
  normalizeTag as normalizeTagPure,
30
33
  normalizeAndDedup,
@@ -40,6 +43,7 @@ import { enrichEntryMetadata } from './metadata-enricher.js';
40
43
  // ─── Constants ──────────────────────────────────────────────────────
41
44
 
42
45
  const DEFAULT_STALE_DAYS = 90;
46
+ const DEFAULT_BATCH_SIZE = 100;
43
47
 
44
48
  // ─── Curator Class ──────────────────────────────────────────────────
45
49
 
@@ -151,7 +155,117 @@ export class Curator {
151
155
  // ─── Duplicates (delegates to duplicate-detector) ─────────────
152
156
 
153
157
  detectDuplicates(entryId?: string, threshold?: number): DuplicateDetectionResult[] {
154
- return detectDuplicatesPure(this.vault.list({ limit: 100000 }), entryId, threshold);
158
+ const effectiveThreshold = threshold ?? DEFAULT_DUPLICATE_THRESHOLD;
159
+ const dismissed = this.getDismissedPairs();
160
+
161
+ // --- Phase 1: Content-hash exact duplicates (O(n) via GROUP BY) ---
162
+ const exactDupes = this.provider.all<{ content_hash: string; ids: string }>(
163
+ `SELECT content_hash, GROUP_CONCAT(id) as ids FROM entries
164
+ WHERE content_hash IS NOT NULL
165
+ GROUP BY content_hash HAVING COUNT(*) > 1`,
166
+ );
167
+
168
+ const results: DuplicateDetectionResult[] = [];
169
+ const seenPairs = new Set<string>();
170
+
171
+ for (const { ids } of exactDupes) {
172
+ const idList = ids.split(',');
173
+ if (entryId && !idList.includes(entryId)) continue;
174
+ const targets = entryId ? [entryId] : idList;
175
+ for (const targetId of targets) {
176
+ const matches: DuplicateCandidate[] = [];
177
+ for (const otherId of idList) {
178
+ if (otherId === targetId) continue;
179
+ const pairKey = [targetId, otherId].sort().join('::');
180
+ if (dismissed.has(pairKey) || seenPairs.has(pairKey)) continue;
181
+ seenPairs.add(pairKey);
182
+ const other = this.vault.get(otherId);
183
+ if (!other) continue;
184
+ matches.push({
185
+ entryId: otherId,
186
+ title: other.title,
187
+ similarity: 1.0,
188
+ suggestMerge: true,
189
+ });
190
+ }
191
+ if (matches.length > 0) {
192
+ results.push({ entryId: targetId, matches, scannedCount: idList.length - 1 });
193
+ }
194
+ }
195
+ }
196
+
197
+ // --- Phase 2: FTS5 fuzzy candidate matching ---
198
+ // For each entry, use FTS5 MATCH to find top candidates, then TF-IDF cosine similarity.
199
+ // Complexity is O(n * k) where k = FTS5 candidate limit (10), not O(n^2).
200
+ const exactDupeEntryIds = new Set(results.map((r) => r.entryId));
201
+ const targetEntries = entryId
202
+ ? ([this.vault.get(entryId)].filter(Boolean) as IntelligenceEntry[])
203
+ : this.listBatched();
204
+
205
+ // Build vocabulary from all entries (batched)
206
+ const allEntries = entryId ? this.listBatched() : targetEntries;
207
+ const vocabulary = buildVocabulary(allEntries);
208
+
209
+ for (const entry of targetEntries) {
210
+ // Skip entries already fully handled by exact-hash matches
211
+ if (exactDupeEntryIds.has(entry.id)) continue;
212
+
213
+ // FTS5 candidate retrieval: find top-10 similar entries
214
+ let candidates: IntelligenceEntry[];
215
+ try {
216
+ candidates = this.vault
217
+ .search(entry.title, { domain: entry.domain, limit: 10 })
218
+ .map((r) => r.entry)
219
+ .filter((c) => c.id !== entry.id);
220
+ } catch {
221
+ candidates = [];
222
+ }
223
+ if (candidates.length === 0) continue;
224
+
225
+ const entryVec = calculateTfIdf(tokenize(entryToText(entry)), vocabulary);
226
+ const matches: DuplicateCandidate[] = [];
227
+
228
+ for (const candidate of candidates) {
229
+ // Skip cross-domain pairs
230
+ if (entry.domain !== candidate.domain) continue;
231
+ const pairKey = [entry.id, candidate.id].sort().join('::');
232
+ if (dismissed.has(pairKey)) continue;
233
+
234
+ const candidateVec = calculateTfIdf(tokenize(entryToText(candidate)), vocabulary);
235
+ const similarity = cosineSimilarity(entryVec, candidateVec);
236
+ if (similarity >= effectiveThreshold) {
237
+ matches.push({
238
+ entryId: candidate.id,
239
+ title: candidate.title,
240
+ similarity,
241
+ suggestMerge: similarity >= MERGE_SUGGESTION_THRESHOLD,
242
+ });
243
+ }
244
+ }
245
+
246
+ if (matches.length > 0) {
247
+ matches.sort((a, b) => b.similarity - a.similarity);
248
+ results.push({ entryId: entry.id, matches, scannedCount: candidates.length });
249
+ }
250
+ }
251
+
252
+ return results;
253
+ }
254
+
255
+ dismissDuplicate(entryIdA: string, entryIdB: string, reason?: string): { dismissed: boolean } {
256
+ const [a, b] = [entryIdA, entryIdB].sort();
257
+ const result = this.provider.run(
258
+ 'INSERT OR IGNORE INTO curator_duplicate_dismissals (entry_id_a, entry_id_b, reason) VALUES (?, ?, ?)',
259
+ [a, b, reason ?? 'reviewed — not duplicate'],
260
+ );
261
+ return { dismissed: result.changes > 0 };
262
+ }
263
+
264
+ private getDismissedPairs(): Set<string> {
265
+ const rows = this.provider.all<{ entry_id_a: string; entry_id_b: string }>(
266
+ 'SELECT entry_id_a, entry_id_b FROM curator_duplicate_dismissals',
267
+ );
268
+ return new Set(rows.map((r) => `${r.entry_id_a}::${r.entry_id_b}`));
155
269
  }
156
270
 
157
271
  // ─── Contradictions (delegates to contradiction-detector) ─────
@@ -159,9 +273,11 @@ export class Curator {
159
273
  detectContradictions(threshold?: number): Contradiction[] {
160
274
  const searchFn = (title: string) =>
161
275
  this.vault.search(title, { type: 'pattern', limit: 20 }).map((r) => r.entry);
162
- return this.persistContradictions(
163
- findContradictions(this.vault.list({ limit: 100000 }), threshold, searchFn),
164
- );
276
+ // Load only anti-patterns and patterns (bounded by type), not the entire vault
277
+ const antipatterns = this.vault.list({ type: 'anti-pattern', limit: 10000 });
278
+ const patterns = this.vault.list({ type: 'pattern', limit: 10000 });
279
+ const entries = [...antipatterns, ...patterns];
280
+ return this.persistContradictions(findContradictions(entries, threshold, searchFn));
165
281
  }
166
282
 
167
283
  getContradictions(status?: ContradictionStatus): Contradiction[] {
@@ -190,10 +306,12 @@ export class Curator {
190
306
  ): Promise<{ contradictions: Contradiction[]; method: 'tfidf-only' }> {
191
307
  const searchFn = (title: string) =>
192
308
  this.vault.search(title, { type: 'pattern', limit: 20 }).map((r) => r.entry);
309
+ // Load only anti-patterns and patterns (bounded by type), not the entire vault
310
+ const antipatterns = this.vault.list({ type: 'anti-pattern', limit: 10000 });
311
+ const patterns = this.vault.list({ type: 'pattern', limit: 10000 });
312
+ const entries = [...antipatterns, ...patterns];
193
313
  return {
194
- contradictions: this.persistContradictions(
195
- findContradictions(this.vault.list({ limit: 100000 }), threshold, searchFn),
196
- ),
314
+ contradictions: this.persistContradictions(findContradictions(entries, threshold, searchFn)),
197
315
  method: 'tfidf-only',
198
316
  };
199
317
  }
@@ -221,19 +339,28 @@ export class Curator {
221
339
 
222
340
  groomAll(): GroomAllResult {
223
341
  const start = Date.now();
224
- const entries = this.vault.list({ limit: 100000 });
225
342
  let tagsNormalized = 0,
226
- staleCount = 0;
227
- for (const entry of entries) {
228
- const result = this.groomEntry(entry.id);
229
- if (result) {
230
- tagsNormalized += result.tagsNormalized.filter((t) => t.wasAliased).length;
231
- if (result.stale) staleCount++;
343
+ staleCount = 0,
344
+ totalEntries = 0;
345
+ // Batch pagination — process entries in batches instead of loading all at once
346
+ let offset = 0;
347
+ while (true) {
348
+ const batch = this.vault.list({ limit: DEFAULT_BATCH_SIZE, offset });
349
+ if (batch.length === 0) break;
350
+ totalEntries += batch.length;
351
+ for (const entry of batch) {
352
+ const result = this.groomEntry(entry.id);
353
+ if (result) {
354
+ tagsNormalized += result.tagsNormalized.filter((t) => t.wasAliased).length;
355
+ if (result.stale) staleCount++;
356
+ }
232
357
  }
358
+ if (batch.length < DEFAULT_BATCH_SIZE) break;
359
+ offset += DEFAULT_BATCH_SIZE;
233
360
  }
234
361
  return {
235
- totalEntries: entries.length,
236
- groomedCount: entries.length,
362
+ totalEntries,
363
+ groomedCount: totalEntries,
237
364
  tagsNormalized,
238
365
  staleCount,
239
366
  durationMs: Date.now() - start,
@@ -315,7 +442,8 @@ export class Curator {
315
442
  // ─── Health Audit (delegates to health-audit) ─────────────────
316
443
 
317
444
  healthAudit(): HealthAuditResult {
318
- const entries = this.vault.list({ limit: 100000 });
445
+ // Load entries in batches instead of all at once
446
+ const entries = this.listBatched();
319
447
  const dataProvider: HealthDataProvider = {
320
448
  getStaleCount: (threshold) =>
321
449
  (
@@ -452,6 +580,22 @@ export class Curator {
452
580
 
453
581
  // ─── Private Helpers ──────────────────────────────────────────
454
582
 
583
+ /**
584
+ * Load all vault entries using batched pagination instead of a single 100k query.
585
+ */
586
+ private listBatched(batchSize: number = DEFAULT_BATCH_SIZE): IntelligenceEntry[] {
587
+ const all: IntelligenceEntry[] = [];
588
+ let offset = 0;
589
+ while (true) {
590
+ const batch = this.vault.list({ limit: batchSize, offset });
591
+ if (batch.length === 0) break;
592
+ all.push(...batch);
593
+ if (batch.length < batchSize) break;
594
+ offset += batchSize;
595
+ }
596
+ return all;
597
+ }
598
+
455
599
  private persistContradictions(candidates: ContradictionCandidate[]): Contradiction[] {
456
600
  const detected: Contradiction[] = [];
457
601
  for (const c of candidates) {
@@ -56,6 +56,14 @@ export const CURATOR_SCHEMA = `
56
56
  resolved_at INTEGER,
57
57
  UNIQUE(pattern_id, antipattern_id)
58
58
  );
59
+ CREATE TABLE IF NOT EXISTS curator_duplicate_dismissals (
60
+ entry_id_a TEXT NOT NULL,
61
+ entry_id_b TEXT NOT NULL,
62
+ dismissed_at INTEGER NOT NULL DEFAULT (unixepoch()),
63
+ reason TEXT,
64
+ PRIMARY KEY (entry_id_a, entry_id_b)
65
+ );
66
+
59
67
  CREATE INDEX IF NOT EXISTS idx_curator_state_status ON curator_entry_state(status);
60
68
  CREATE INDEX IF NOT EXISTS idx_curator_changelog_entry ON curator_changelog(entry_id);
61
69
  `;
@@ -67,12 +67,17 @@ export interface PackSkillDefinition {
67
67
  // DomainPack — the main interface
68
68
  // ---------------------------------------------------------------------------
69
69
 
70
+ /** Pack tier: determines visibility, licensing, and install behavior */
71
+ export type DomainPackTier = 'default' | 'community' | 'premium';
72
+
70
73
  /** The contract every domain pack must implement. */
71
74
  export interface DomainPack {
72
75
  /** Unique pack name (e.g., 'design', 'security-intelligence') */
73
76
  name: string;
74
77
  /** Semver version */
75
78
  version: string;
79
+ /** Tier: 'default' (ships with engine), 'community' (free, npm), 'premium' (unlocked today, gated later) */
80
+ tier?: DomainPackTier;
76
81
  /** Domains this pack claims. Ops inject into these domain facades. */
77
82
  domains: string[];
78
83
  /** Custom operations with real logic — injected into claimed domain facades. */
@@ -115,6 +120,8 @@ export interface DomainPackRef {
115
120
  package: string;
116
121
  /** Optional version constraint */
117
122
  version?: string;
123
+ /** Pack tier (inherited from pack if not set) */
124
+ tier?: DomainPackTier;
118
125
  }
119
126
 
120
127
  // ---------------------------------------------------------------------------
@@ -148,6 +155,7 @@ const packSkillSchema = z.object({
148
155
  const domainPackSchema = z.object({
149
156
  name: z.string().min(1),
150
157
  version: z.string().min(1),
158
+ tier: z.enum(['default', 'community', 'premium']).optional(),
151
159
  domains: z.array(z.string().min(1)).min(1),
152
160
  ops: z.array(
153
161
  z.object({
@@ -49,7 +49,7 @@ describe('ENGINE_MODULE_MANIFEST', () => {
49
49
  expect(entry.description.length).toBeGreaterThan(0);
50
50
  expect(Array.isArray(entry.keyOps)).toBe(true);
51
51
  expect(entry.keyOps.length).toBeGreaterThan(0);
52
- expect(entry.keyOps.length).toBeLessThanOrEqual(4);
52
+ expect(entry.keyOps.length).toBeLessThanOrEqual(5);
53
53
  }
54
54
  });
55
55
 
@@ -69,7 +69,13 @@ describe('ENGINE_MODULE_MANIFEST', () => {
69
69
 
70
70
  it('plan module has expected keyOps', () => {
71
71
  const plan = ENGINE_MODULE_MANIFEST.find((m) => m.suffix === 'plan')!;
72
- expect(plan.keyOps).toEqual(['create_plan', 'approve_plan', 'plan_split', 'plan_reconcile']);
72
+ expect(plan.keyOps).toEqual([
73
+ 'create_plan',
74
+ 'approve_plan',
75
+ 'plan_split',
76
+ 'plan_reconcile',
77
+ 'plan_close_stale',
78
+ ]);
73
79
  });
74
80
 
75
81
  it('conditional field is optional and boolean when present', () => {
@@ -41,7 +41,7 @@ export const ENGINE_MODULE_MANIFEST: ModuleManifestEntry[] = [
41
41
  {
42
42
  suffix: 'plan',
43
43
  description: 'Plan lifecycle — create, approve, execute, reconcile, complete, grading.',
44
- keyOps: ['create_plan', 'approve_plan', 'plan_split', 'plan_reconcile'],
44
+ keyOps: ['create_plan', 'approve_plan', 'plan_split', 'plan_reconcile', 'plan_close_stale'],
45
45
  intentSignals: {
46
46
  'plan this': 'create_plan',
47
47
  'break this down': 'create_plan',
@@ -56,7 +56,7 @@ export const ENGINE_MODULE_MANIFEST: ModuleManifestEntry[] = [
56
56
  keyOps: ['recommend', 'strengths', 'feedback'],
57
57
  intentSignals: {
58
58
  'what works': 'recommend',
59
- 'recommendations': 'recommend',
59
+ recommendations: 'recommend',
60
60
  'pattern strengths': 'strengths',
61
61
  'give feedback': 'feedback',
62
62
  },
@@ -87,7 +87,7 @@ export const ENGINE_MODULE_MANIFEST: ModuleManifestEntry[] = [
87
87
  {
88
88
  suffix: 'curator',
89
89
  description: 'Quality — duplicate detection, contradictions, grooming, health audit.',
90
- keyOps: ['curator_groom', 'curator_status', 'curator_health'],
90
+ keyOps: ['curator_groom', 'curator_status', 'curator_health', 'curator_dismiss_duplicate'],
91
91
  intentSignals: {
92
92
  'clean up vault': 'curator_groom',
93
93
  'find duplicates': 'curator_groom',
@@ -15,7 +15,7 @@
15
15
  import { describe, it, expect, beforeAll, afterAll } from 'vitest';
16
16
  import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
17
17
  import { createAgentRuntime } from '../runtime/runtime.js';
18
- import { registerEngine, ENGINE_MODULES } from './register-engine.js';
18
+ import { registerEngine, ENGINE_MODULES, INTERNAL_OPS } from './register-engine.js';
19
19
  import { ENGINE_MODULE_MANIFEST } from './module-manifest.js';
20
20
  import type { AgentRuntime } from '../runtime/types.js';
21
21
  import type { OpDefinition } from '../facades/types.js';
@@ -233,3 +233,75 @@ describe('ENGINE_MODULES descriptions match manifest', () => {
233
233
  }
234
234
  });
235
235
  });
236
+
237
+ describe('registerEngine — op visibility', () => {
238
+ it('INTERNAL_OPS set contains expected ops', () => {
239
+ // Spot-check known internal ops
240
+ expect(INTERNAL_OPS.has('admin_create_token')).toBe(true);
241
+ expect(INTERNAL_OPS.has('vault_bulk_add')).toBe(true);
242
+ expect(INTERNAL_OPS.has('plan_auto_reconcile')).toBe(true);
243
+ expect(INTERNAL_OPS.has('telemetry_errors')).toBe(true);
244
+ // User-facing ops should NOT be in the set
245
+ expect(INTERNAL_OPS.has('admin_health')).toBe(false);
246
+ expect(INTERNAL_OPS.has('search_intelligent')).toBe(false);
247
+ expect(INTERNAL_OPS.has('create_plan')).toBe(false);
248
+ });
249
+
250
+ it('INTERNAL_OPS has at least 25 entries', () => {
251
+ expect(INTERNAL_OPS.size).toBeGreaterThanOrEqual(25);
252
+ });
253
+
254
+ it('ops without visibility field default to user (backward compat)', () => {
255
+ const server = makeServer();
256
+ const userOp: OpDefinition = {
257
+ name: 'my_visible_op',
258
+ description: 'Visible op',
259
+ auth: 'read',
260
+ handler: async () => 'ok',
261
+ };
262
+ const result = registerEngine(server, runtime, {
263
+ agentId: 'vis',
264
+ domainPacks: [{ name: 'test', facades: [{ name: 'test', ops: [userOp] }] }],
265
+ });
266
+ expect(result.tools).toContain('vis_test');
267
+ });
268
+
269
+ it('ops with visibility: internal are excluded from MCP tool description but remain callable', () => {
270
+ const server = makeServer();
271
+ const visibleOp: OpDefinition = {
272
+ name: 'public_op',
273
+ description: 'Public op',
274
+ auth: 'read',
275
+ handler: async () => 'visible',
276
+ };
277
+ const internalOp: OpDefinition = {
278
+ name: 'secret_op',
279
+ description: 'Internal op',
280
+ auth: 'admin',
281
+ visibility: 'internal',
282
+ handler: async () => 'hidden',
283
+ };
284
+ // Register both ops under a pack facade
285
+ registerEngine(server, runtime, {
286
+ agentId: 'vt',
287
+ domainPacks: [{ name: 'test', facades: [{ name: 'check', ops: [visibleOp, internalOp] }] }],
288
+ });
289
+ // We can't easily inspect the MCP schema description string, but we verify
290
+ // that registration succeeds with mixed visibility ops
291
+ expect(true).toBe(true);
292
+ });
293
+
294
+ it('every INTERNAL_OPS entry corresponds to a real op in some facade', () => {
295
+ // Collect all op names across all engine modules
296
+ const allOpNames = new Set<string>();
297
+ for (const mod of ENGINE_MODULES) {
298
+ const ops = mod.createOps(runtime);
299
+ for (const op of ops) {
300
+ allOpNames.add(op.name);
301
+ }
302
+ }
303
+ for (const internalOp of INTERNAL_OPS) {
304
+ expect(allOpNames.has(internalOp)).toBe(true);
305
+ }
306
+ });
307
+ });
@@ -295,11 +295,70 @@ export function registerEngine(
295
295
  return { tools: registeredTools, totalOps, registerTool };
296
296
  }
297
297
 
298
+ // ─── Op Visibility ───────────────────────────────────────────────────
299
+
300
+ /**
301
+ * Ops classified as internal — hidden from MCP tool descriptions but still
302
+ * callable programmatically. Centralized here for easy auditing.
303
+ *
304
+ * Criteria: ops that are infrastructure plumbing, token/account management,
305
+ * bulk operations, or auto-* automation that users never call directly.
306
+ */
307
+ /** @internal Exported for testing — do not use outside engine */
308
+ export const INTERNAL_OPS = new Set([
309
+ // Admin: token management
310
+ 'admin_create_token',
311
+ 'admin_revoke_token',
312
+ 'admin_list_tokens',
313
+ // Admin: account management
314
+ 'admin_add_account',
315
+ 'admin_remove_account',
316
+ 'admin_rotate_account',
317
+ 'admin_list_accounts',
318
+ 'admin_account_status',
319
+ // Admin: infrastructure debug
320
+ 'admin_vault_size',
321
+ 'admin_vault_analytics',
322
+ 'admin_search_insights',
323
+ 'admin_env',
324
+ 'admin_gc',
325
+ 'admin_export_config',
326
+ 'admin_key_pool_status',
327
+ 'admin_persistence_info',
328
+ // Admin: flags
329
+ 'admin_list_flags',
330
+ 'admin_get_flag',
331
+ 'admin_set_flag',
332
+ // Admin: setup internals
333
+ 'admin_inject_claude_md',
334
+ // Admin: telemetry
335
+ 'admin_telemetry',
336
+ 'admin_telemetry_recent',
337
+ 'admin_telemetry_reset',
338
+ 'telemetry_errors',
339
+ 'telemetry_slow_ops',
340
+ // Vault: bulk operations
341
+ 'vault_bulk_add',
342
+ 'vault_bulk_remove',
343
+ // Plan: automation
344
+ 'plan_auto_reconcile',
345
+ 'plan_auto_improve',
346
+ ]);
347
+
348
+ /** Resolve effective visibility: explicit field wins, then INTERNAL_OPS set, else 'user'. */
349
+ function resolveVisibility(op: OpDefinition): 'user' | 'internal' {
350
+ if (op.visibility) return op.visibility;
351
+ return INTERNAL_OPS.has(op.name) ? 'internal' : 'user';
352
+ }
353
+
298
354
  // ─── Tool Registration (No Factory) ──────────────────────────────────
299
355
 
300
356
  /**
301
357
  * Register a single grouped tool with op dispatch.
302
358
  * This is the replacement for registerFacade() — same behavior, no FacadeConfig type.
359
+ *
360
+ * Internal ops (visibility: 'internal') are excluded from the MCP tool description
361
+ * but remain dispatchable — they just don't show up in the op list Claude sees.
303
362
  */
304
363
  function registerModuleTool(
305
364
  server: McpServer,
@@ -308,7 +367,8 @@ function registerModuleTool(
308
367
  ops: OpDefinition[],
309
368
  authPolicy?: () => AuthPolicy,
310
369
  ): void {
311
- const opNames = ops.map((o) => o.name);
370
+ const userOps = ops.filter((o) => resolveVisibility(o) !== 'internal');
371
+ const opNames = userOps.map((o) => o.name);
312
372
  const opMap = new Map(ops.map((o) => [o.name, o]));
313
373
 
314
374
  server.tool(
@@ -25,6 +25,9 @@ export const AUTH_LEVEL_RANK: Record<AuthLevel, number> = {
25
25
  admin: 2,
26
26
  };
27
27
 
28
+ /** Op visibility — controls whether an op is exposed via MCP tool registration */
29
+ export type OpVisibility = 'user' | 'internal';
30
+
28
31
  /** Operation definition within a facade */
29
32
  export interface OpDefinition {
30
33
  name: string;
@@ -34,6 +37,8 @@ export interface OpDefinition {
34
37
  schema?: z.ZodType;
35
38
  /** Promote to a first-class MCP tool with full schema discovery. */
36
39
  hot?: boolean;
40
+ /** Controls MCP exposure: 'user' (default) = listed in tool, 'internal' = hidden from MCP but callable programmatically. */
41
+ visibility?: OpVisibility;
37
42
  }
38
43
 
39
44
  /** Facade configuration — one MCP tool */