@soleri/core 9.3.1 → 9.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/brain/intelligence.d.ts +5 -0
- package/dist/brain/intelligence.d.ts.map +1 -1
- package/dist/brain/intelligence.js +115 -26
- package/dist/brain/intelligence.js.map +1 -1
- package/dist/brain/learning-radar.d.ts +3 -3
- package/dist/brain/learning-radar.d.ts.map +1 -1
- package/dist/brain/learning-radar.js +8 -4
- package/dist/brain/learning-radar.js.map +1 -1
- package/dist/control/intent-router.d.ts +2 -2
- package/dist/control/intent-router.d.ts.map +1 -1
- package/dist/control/intent-router.js +35 -1
- package/dist/control/intent-router.js.map +1 -1
- package/dist/control/types.d.ts +10 -2
- package/dist/control/types.d.ts.map +1 -1
- package/dist/curator/curator.d.ts +4 -0
- package/dist/curator/curator.d.ts.map +1 -1
- package/dist/curator/curator.js +23 -1
- package/dist/curator/curator.js.map +1 -1
- package/dist/curator/schema.d.ts +1 -1
- package/dist/curator/schema.d.ts.map +1 -1
- package/dist/curator/schema.js +8 -0
- package/dist/curator/schema.js.map +1 -1
- package/dist/domain-packs/types.d.ts +6 -0
- package/dist/domain-packs/types.d.ts.map +1 -1
- package/dist/domain-packs/types.js +1 -0
- package/dist/domain-packs/types.js.map +1 -1
- package/dist/engine/module-manifest.js +3 -3
- package/dist/engine/module-manifest.js.map +1 -1
- package/dist/engine/register-engine.d.ts +9 -0
- package/dist/engine/register-engine.d.ts.map +1 -1
- package/dist/engine/register-engine.js +59 -1
- package/dist/engine/register-engine.js.map +1 -1
- package/dist/facades/types.d.ts +5 -1
- package/dist/facades/types.d.ts.map +1 -1
- package/dist/facades/types.js.map +1 -1
- package/dist/hooks/candidate-scorer.d.ts +28 -0
- package/dist/hooks/candidate-scorer.d.ts.map +1 -0
- package/dist/hooks/candidate-scorer.js +20 -0
- package/dist/hooks/candidate-scorer.js.map +1 -0
- package/dist/hooks/index.d.ts +2 -0
- package/dist/hooks/index.d.ts.map +1 -0
- package/dist/hooks/index.js +2 -0
- package/dist/hooks/index.js.map +1 -0
- package/dist/index.d.ts +4 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/operator/operator-context-store.d.ts +54 -0
- package/dist/operator/operator-context-store.d.ts.map +1 -0
- package/dist/operator/operator-context-store.js +434 -0
- package/dist/operator/operator-context-store.js.map +1 -0
- package/dist/operator/operator-context-types.d.ts +101 -0
- package/dist/operator/operator-context-types.d.ts.map +1 -0
- package/dist/operator/operator-context-types.js +27 -0
- package/dist/operator/operator-context-types.js.map +1 -0
- package/dist/packs/index.d.ts +2 -2
- package/dist/packs/index.d.ts.map +1 -1
- package/dist/packs/index.js +1 -1
- package/dist/packs/index.js.map +1 -1
- package/dist/packs/lockfile.d.ts +3 -0
- package/dist/packs/lockfile.d.ts.map +1 -1
- package/dist/packs/lockfile.js.map +1 -1
- package/dist/packs/types.d.ts +8 -2
- package/dist/packs/types.d.ts.map +1 -1
- package/dist/packs/types.js +6 -0
- package/dist/packs/types.js.map +1 -1
- package/dist/planning/plan-lifecycle.d.ts +12 -1
- package/dist/planning/plan-lifecycle.d.ts.map +1 -1
- package/dist/planning/plan-lifecycle.js +54 -16
- package/dist/planning/plan-lifecycle.js.map +1 -1
- package/dist/planning/planner-types.d.ts +6 -0
- package/dist/planning/planner-types.d.ts.map +1 -1
- package/dist/planning/planner.d.ts +21 -1
- package/dist/planning/planner.d.ts.map +1 -1
- package/dist/planning/planner.js +62 -3
- package/dist/planning/planner.js.map +1 -1
- package/dist/planning/task-complexity-assessor.d.ts.map +1 -1
- package/dist/planning/task-complexity-assessor.js.map +1 -1
- package/dist/plugins/types.d.ts +18 -18
- package/dist/runtime/admin-ops.d.ts +1 -1
- package/dist/runtime/admin-ops.d.ts.map +1 -1
- package/dist/runtime/admin-ops.js +100 -3
- package/dist/runtime/admin-ops.js.map +1 -1
- package/dist/runtime/admin-setup-ops.d.ts.map +1 -1
- package/dist/runtime/admin-setup-ops.js +19 -9
- package/dist/runtime/admin-setup-ops.js.map +1 -1
- package/dist/runtime/capture-ops.d.ts.map +1 -1
- package/dist/runtime/capture-ops.js +35 -7
- package/dist/runtime/capture-ops.js.map +1 -1
- package/dist/runtime/facades/brain-facade.d.ts.map +1 -1
- package/dist/runtime/facades/brain-facade.js +4 -2
- package/dist/runtime/facades/brain-facade.js.map +1 -1
- package/dist/runtime/facades/control-facade.d.ts.map +1 -1
- package/dist/runtime/facades/control-facade.js +8 -2
- package/dist/runtime/facades/control-facade.js.map +1 -1
- package/dist/runtime/facades/curator-facade.d.ts.map +1 -1
- package/dist/runtime/facades/curator-facade.js +13 -0
- package/dist/runtime/facades/curator-facade.js.map +1 -1
- package/dist/runtime/facades/memory-facade.d.ts.map +1 -1
- package/dist/runtime/facades/memory-facade.js +10 -12
- package/dist/runtime/facades/memory-facade.js.map +1 -1
- package/dist/runtime/facades/orchestrate-facade.d.ts.map +1 -1
- package/dist/runtime/facades/orchestrate-facade.js +36 -1
- package/dist/runtime/facades/orchestrate-facade.js.map +1 -1
- package/dist/runtime/facades/plan-facade.d.ts.map +1 -1
- package/dist/runtime/facades/plan-facade.js +20 -4
- package/dist/runtime/facades/plan-facade.js.map +1 -1
- package/dist/runtime/orchestrate-ops.d.ts.map +1 -1
- package/dist/runtime/orchestrate-ops.js +71 -4
- package/dist/runtime/orchestrate-ops.js.map +1 -1
- package/dist/runtime/plan-feedback-helper.d.ts +21 -0
- package/dist/runtime/plan-feedback-helper.d.ts.map +1 -0
- package/dist/runtime/plan-feedback-helper.js +52 -0
- package/dist/runtime/plan-feedback-helper.js.map +1 -0
- package/dist/runtime/planning-extra-ops.d.ts.map +1 -1
- package/dist/runtime/planning-extra-ops.js +73 -34
- package/dist/runtime/planning-extra-ops.js.map +1 -1
- package/dist/runtime/session-briefing.d.ts.map +1 -1
- package/dist/runtime/session-briefing.js +9 -1
- package/dist/runtime/session-briefing.js.map +1 -1
- package/dist/runtime/types.d.ts +3 -0
- package/dist/runtime/types.d.ts.map +1 -1
- package/dist/skills/sync-skills.d.ts.map +1 -1
- package/dist/skills/sync-skills.js +13 -7
- package/dist/skills/sync-skills.js.map +1 -1
- package/package.json +1 -1
- package/src/brain/brain-intelligence.test.ts +30 -0
- package/src/brain/brain.ts +120 -46
- package/src/brain/extraction-quality.test.ts +323 -0
- package/src/brain/intelligence.ts +175 -64
- package/src/brain/learning-radar.ts +8 -5
- package/src/brain/second-brain-features.test.ts +1 -1
- package/src/chat/agent-loop.ts +1 -1
- package/src/chat/notifications.ts +4 -0
- package/src/control/intent-router.test.ts +73 -3
- package/src/control/intent-router.ts +48 -9
- package/src/control/types.ts +13 -2
- package/src/curator/curator.test.ts +92 -0
- package/src/curator/curator.ts +162 -18
- package/src/curator/schema.ts +8 -0
- package/src/domain-packs/types.ts +8 -0
- package/src/engine/module-manifest.test.ts +8 -2
- package/src/engine/module-manifest.ts +3 -3
- package/src/engine/register-engine.test.ts +73 -1
- package/src/engine/register-engine.ts +61 -1
- package/src/facades/types.ts +5 -0
- package/src/hooks/candidate-scorer.test.ts +76 -0
- package/src/hooks/candidate-scorer.ts +39 -0
- package/src/hooks/index.ts +6 -0
- package/src/index.ts +24 -0
- package/src/llm/llm-client.ts +1 -0
- package/src/operator/operator-context-store.test.ts +698 -0
- package/src/operator/operator-context-store.ts +569 -0
- package/src/operator/operator-context-types.ts +139 -0
- package/src/packs/index.ts +3 -1
- package/src/packs/lockfile.ts +3 -0
- package/src/packs/types.ts +9 -0
- package/src/persistence/sqlite-provider.ts +1 -0
- package/src/planning/github-projection.ts +48 -44
- package/src/planning/plan-lifecycle.ts +93 -22
- package/src/planning/planner-types.ts +6 -0
- package/src/planning/planner.ts +74 -4
- package/src/planning/task-complexity-assessor.test.ts +6 -2
- package/src/planning/task-complexity-assessor.ts +1 -4
- package/src/queue/pipeline-runner.ts +4 -0
- package/src/runtime/admin-ops.test.ts +139 -6
- package/src/runtime/admin-ops.ts +104 -3
- package/src/runtime/admin-setup-ops.ts +30 -10
- package/src/runtime/capture-ops.test.ts +84 -0
- package/src/runtime/capture-ops.ts +35 -7
- package/src/runtime/curator-extra-ops.test.ts +7 -0
- package/src/runtime/curator-extra-ops.ts +10 -1
- package/src/runtime/facades/admin-facade.test.ts +1 -1
- package/src/runtime/facades/brain-facade.ts +6 -3
- package/src/runtime/facades/control-facade.ts +10 -2
- package/src/runtime/facades/curator-facade.test.ts +7 -0
- package/src/runtime/facades/curator-facade.ts +18 -0
- package/src/runtime/facades/memory-facade.test.ts +14 -12
- package/src/runtime/facades/memory-facade.ts +197 -12
- package/src/runtime/facades/orchestrate-facade.ts +33 -1
- package/src/runtime/facades/plan-facade.test.ts +213 -0
- package/src/runtime/facades/plan-facade.ts +23 -4
- package/src/runtime/orchestrate-ops.test.ts +202 -2
- package/src/runtime/orchestrate-ops.ts +88 -7
- package/src/runtime/plan-feedback-helper.test.ts +173 -0
- package/src/runtime/plan-feedback-helper.ts +63 -0
- package/src/runtime/planning-extra-ops.test.ts +43 -1
- package/src/runtime/planning-extra-ops.ts +96 -33
- package/src/runtime/runtime.test.ts +50 -2
- package/src/runtime/runtime.ts +117 -89
- package/src/runtime/session-briefing.test.ts +1 -0
- package/src/runtime/session-briefing.ts +10 -1
- package/src/runtime/shutdown-registry.test.ts +151 -0
- package/src/runtime/shutdown-registry.ts +85 -0
- package/src/runtime/types.ts +7 -1
- package/src/skills/sync-skills.ts +14 -7
- package/src/transport/http-server.ts +50 -3
- package/src/transport/ws-server.ts +8 -0
- package/src/vault/linking.test.ts +12 -0
- package/src/vault/linking.ts +90 -44
- package/src/vault/vault-maintenance.ts +11 -18
- package/src/vault/vault-memories.ts +21 -13
- package/src/vault/vault-schema.ts +21 -0
- package/src/vault/vault.ts +8 -3
- package/vitest.config.ts +1 -0
|
@@ -647,4 +647,96 @@ describe('Curator', () => {
|
|
|
647
647
|
expect(result.metrics.tagHealth).toBeLessThan(1);
|
|
648
648
|
});
|
|
649
649
|
});
|
|
650
|
+
|
|
651
|
+
// ─── Duplicate Dismissal ──────────────────────────────────────
|
|
652
|
+
|
|
653
|
+
describe('Duplicate Dismissal', () => {
|
|
654
|
+
it('dismissDuplicate stores a pair and detectDuplicates filters it out', () => {
|
|
655
|
+
// Add two similar entries that will be flagged as duplicates
|
|
656
|
+
vault.add(
|
|
657
|
+
makeEntry({
|
|
658
|
+
id: 'dup-a',
|
|
659
|
+
title: 'React hook cleanup pattern',
|
|
660
|
+
description:
|
|
661
|
+
'Always clean up useEffect hooks with return cleanup function to prevent memory leaks and stale closures.',
|
|
662
|
+
}),
|
|
663
|
+
);
|
|
664
|
+
vault.add(
|
|
665
|
+
makeEntry({
|
|
666
|
+
id: 'dup-b',
|
|
667
|
+
title: 'React hook cleanup pattern for effects',
|
|
668
|
+
description:
|
|
669
|
+
'Clean up useEffect hooks by returning a cleanup function to avoid memory leaks and stale closures.',
|
|
670
|
+
}),
|
|
671
|
+
);
|
|
672
|
+
|
|
673
|
+
// Verify they show up as duplicates
|
|
674
|
+
const before = curator.detectDuplicates();
|
|
675
|
+
const dupBefore = before.find(
|
|
676
|
+
(d) => d.entryId === 'dup-a' && d.matches.some((m) => m.entryId === 'dup-b'),
|
|
677
|
+
);
|
|
678
|
+
expect(dupBefore).toBeDefined();
|
|
679
|
+
|
|
680
|
+
// Dismiss the pair
|
|
681
|
+
const result = curator.dismissDuplicate('dup-a', 'dup-b', 'intentionally distinct');
|
|
682
|
+
expect(result.dismissed).toBe(true);
|
|
683
|
+
|
|
684
|
+
// Verify they no longer show up
|
|
685
|
+
const after = curator.detectDuplicates();
|
|
686
|
+
const dupAfter = after.find(
|
|
687
|
+
(d) => d.entryId === 'dup-a' && d.matches.some((m) => m.entryId === 'dup-b'),
|
|
688
|
+
);
|
|
689
|
+
expect(dupAfter).toBeUndefined();
|
|
690
|
+
});
|
|
691
|
+
|
|
692
|
+
it('dismissDuplicate works with swapped ID order', () => {
|
|
693
|
+
vault.add(
|
|
694
|
+
makeEntry({
|
|
695
|
+
id: 'sw-a',
|
|
696
|
+
title: 'Semantic color tokens for error states',
|
|
697
|
+
description: 'Use semantic tokens like text-error and bg-error-subtle for error UI.',
|
|
698
|
+
}),
|
|
699
|
+
);
|
|
700
|
+
vault.add(
|
|
701
|
+
makeEntry({
|
|
702
|
+
id: 'sw-b',
|
|
703
|
+
title: 'Semantic color tokens for error UI',
|
|
704
|
+
description:
|
|
705
|
+
'Always prefer semantic tokens text-error and bg-error-subtle over raw colors.',
|
|
706
|
+
}),
|
|
707
|
+
);
|
|
708
|
+
|
|
709
|
+
// Dismiss with b,a order
|
|
710
|
+
curator.dismissDuplicate('sw-b', 'sw-a');
|
|
711
|
+
|
|
712
|
+
// Should still be filtered (stored as sorted pair)
|
|
713
|
+
const results = curator.detectDuplicates('sw-a');
|
|
714
|
+
const match = results.find((r) => r.matches.some((m) => m.entryId === 'sw-b'));
|
|
715
|
+
expect(match).toBeUndefined();
|
|
716
|
+
});
|
|
717
|
+
|
|
718
|
+
it('dismissDuplicate is idempotent', () => {
|
|
719
|
+
vault.add(
|
|
720
|
+
makeEntry({
|
|
721
|
+
id: 'idem-a',
|
|
722
|
+
title: 'Pattern A idempotent test',
|
|
723
|
+
description: 'First entry for idempotency test of duplicate dismissal.',
|
|
724
|
+
}),
|
|
725
|
+
);
|
|
726
|
+
vault.add(
|
|
727
|
+
makeEntry({
|
|
728
|
+
id: 'idem-b',
|
|
729
|
+
title: 'Pattern A idempotent test duplicate',
|
|
730
|
+
description: 'Second entry for idempotency test of duplicate dismissal.',
|
|
731
|
+
}),
|
|
732
|
+
);
|
|
733
|
+
|
|
734
|
+
const first = curator.dismissDuplicate('idem-a', 'idem-b');
|
|
735
|
+
expect(first.dismissed).toBe(true);
|
|
736
|
+
|
|
737
|
+
// Second call should be a no-op (INSERT OR IGNORE)
|
|
738
|
+
const second = curator.dismissDuplicate('idem-a', 'idem-b');
|
|
739
|
+
expect(second.dismissed).toBe(false);
|
|
740
|
+
});
|
|
741
|
+
});
|
|
650
742
|
});
|
package/src/curator/curator.ts
CHANGED
|
@@ -17,14 +17,17 @@ import type {
|
|
|
17
17
|
} from './types.js';
|
|
18
18
|
|
|
19
19
|
import {
|
|
20
|
-
detectDuplicates as detectDuplicatesPure,
|
|
21
20
|
DEFAULT_DUPLICATE_THRESHOLD,
|
|
21
|
+
MERGE_SUGGESTION_THRESHOLD,
|
|
22
|
+
buildVocabulary,
|
|
23
|
+
entryToText,
|
|
22
24
|
} from './duplicate-detector.js';
|
|
23
25
|
import {
|
|
24
26
|
findContradictions,
|
|
25
27
|
DEFAULT_CONTRADICTION_THRESHOLD,
|
|
26
28
|
type ContradictionCandidate,
|
|
27
29
|
} from './contradiction-detector.js';
|
|
30
|
+
import { tokenize, calculateTfIdf, cosineSimilarity } from '../text/similarity.js';
|
|
28
31
|
import {
|
|
29
32
|
normalizeTag as normalizeTagPure,
|
|
30
33
|
normalizeAndDedup,
|
|
@@ -40,6 +43,7 @@ import { enrichEntryMetadata } from './metadata-enricher.js';
|
|
|
40
43
|
// ─── Constants ──────────────────────────────────────────────────────
|
|
41
44
|
|
|
42
45
|
const DEFAULT_STALE_DAYS = 90;
|
|
46
|
+
const DEFAULT_BATCH_SIZE = 100;
|
|
43
47
|
|
|
44
48
|
// ─── Curator Class ──────────────────────────────────────────────────
|
|
45
49
|
|
|
@@ -151,7 +155,117 @@ export class Curator {
|
|
|
151
155
|
// ─── Duplicates (delegates to duplicate-detector) ─────────────
|
|
152
156
|
|
|
153
157
|
detectDuplicates(entryId?: string, threshold?: number): DuplicateDetectionResult[] {
|
|
154
|
-
|
|
158
|
+
const effectiveThreshold = threshold ?? DEFAULT_DUPLICATE_THRESHOLD;
|
|
159
|
+
const dismissed = this.getDismissedPairs();
|
|
160
|
+
|
|
161
|
+
// --- Phase 1: Content-hash exact duplicates (O(n) via GROUP BY) ---
|
|
162
|
+
const exactDupes = this.provider.all<{ content_hash: string; ids: string }>(
|
|
163
|
+
`SELECT content_hash, GROUP_CONCAT(id) as ids FROM entries
|
|
164
|
+
WHERE content_hash IS NOT NULL
|
|
165
|
+
GROUP BY content_hash HAVING COUNT(*) > 1`,
|
|
166
|
+
);
|
|
167
|
+
|
|
168
|
+
const results: DuplicateDetectionResult[] = [];
|
|
169
|
+
const seenPairs = new Set<string>();
|
|
170
|
+
|
|
171
|
+
for (const { ids } of exactDupes) {
|
|
172
|
+
const idList = ids.split(',');
|
|
173
|
+
if (entryId && !idList.includes(entryId)) continue;
|
|
174
|
+
const targets = entryId ? [entryId] : idList;
|
|
175
|
+
for (const targetId of targets) {
|
|
176
|
+
const matches: DuplicateCandidate[] = [];
|
|
177
|
+
for (const otherId of idList) {
|
|
178
|
+
if (otherId === targetId) continue;
|
|
179
|
+
const pairKey = [targetId, otherId].sort().join('::');
|
|
180
|
+
if (dismissed.has(pairKey) || seenPairs.has(pairKey)) continue;
|
|
181
|
+
seenPairs.add(pairKey);
|
|
182
|
+
const other = this.vault.get(otherId);
|
|
183
|
+
if (!other) continue;
|
|
184
|
+
matches.push({
|
|
185
|
+
entryId: otherId,
|
|
186
|
+
title: other.title,
|
|
187
|
+
similarity: 1.0,
|
|
188
|
+
suggestMerge: true,
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
if (matches.length > 0) {
|
|
192
|
+
results.push({ entryId: targetId, matches, scannedCount: idList.length - 1 });
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
// --- Phase 2: FTS5 fuzzy candidate matching ---
|
|
198
|
+
// For each entry, use FTS5 MATCH to find top candidates, then TF-IDF cosine similarity.
|
|
199
|
+
// Complexity is O(n * k) where k = FTS5 candidate limit (10), not O(n^2).
|
|
200
|
+
const exactDupeEntryIds = new Set(results.map((r) => r.entryId));
|
|
201
|
+
const targetEntries = entryId
|
|
202
|
+
? ([this.vault.get(entryId)].filter(Boolean) as IntelligenceEntry[])
|
|
203
|
+
: this.listBatched();
|
|
204
|
+
|
|
205
|
+
// Build vocabulary from all entries (batched)
|
|
206
|
+
const allEntries = entryId ? this.listBatched() : targetEntries;
|
|
207
|
+
const vocabulary = buildVocabulary(allEntries);
|
|
208
|
+
|
|
209
|
+
for (const entry of targetEntries) {
|
|
210
|
+
// Skip entries already fully handled by exact-hash matches
|
|
211
|
+
if (exactDupeEntryIds.has(entry.id)) continue;
|
|
212
|
+
|
|
213
|
+
// FTS5 candidate retrieval: find top-10 similar entries
|
|
214
|
+
let candidates: IntelligenceEntry[];
|
|
215
|
+
try {
|
|
216
|
+
candidates = this.vault
|
|
217
|
+
.search(entry.title, { domain: entry.domain, limit: 10 })
|
|
218
|
+
.map((r) => r.entry)
|
|
219
|
+
.filter((c) => c.id !== entry.id);
|
|
220
|
+
} catch {
|
|
221
|
+
candidates = [];
|
|
222
|
+
}
|
|
223
|
+
if (candidates.length === 0) continue;
|
|
224
|
+
|
|
225
|
+
const entryVec = calculateTfIdf(tokenize(entryToText(entry)), vocabulary);
|
|
226
|
+
const matches: DuplicateCandidate[] = [];
|
|
227
|
+
|
|
228
|
+
for (const candidate of candidates) {
|
|
229
|
+
// Skip cross-domain pairs
|
|
230
|
+
if (entry.domain !== candidate.domain) continue;
|
|
231
|
+
const pairKey = [entry.id, candidate.id].sort().join('::');
|
|
232
|
+
if (dismissed.has(pairKey)) continue;
|
|
233
|
+
|
|
234
|
+
const candidateVec = calculateTfIdf(tokenize(entryToText(candidate)), vocabulary);
|
|
235
|
+
const similarity = cosineSimilarity(entryVec, candidateVec);
|
|
236
|
+
if (similarity >= effectiveThreshold) {
|
|
237
|
+
matches.push({
|
|
238
|
+
entryId: candidate.id,
|
|
239
|
+
title: candidate.title,
|
|
240
|
+
similarity,
|
|
241
|
+
suggestMerge: similarity >= MERGE_SUGGESTION_THRESHOLD,
|
|
242
|
+
});
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
if (matches.length > 0) {
|
|
247
|
+
matches.sort((a, b) => b.similarity - a.similarity);
|
|
248
|
+
results.push({ entryId: entry.id, matches, scannedCount: candidates.length });
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
return results;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
dismissDuplicate(entryIdA: string, entryIdB: string, reason?: string): { dismissed: boolean } {
|
|
256
|
+
const [a, b] = [entryIdA, entryIdB].sort();
|
|
257
|
+
const result = this.provider.run(
|
|
258
|
+
'INSERT OR IGNORE INTO curator_duplicate_dismissals (entry_id_a, entry_id_b, reason) VALUES (?, ?, ?)',
|
|
259
|
+
[a, b, reason ?? 'reviewed — not duplicate'],
|
|
260
|
+
);
|
|
261
|
+
return { dismissed: result.changes > 0 };
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
private getDismissedPairs(): Set<string> {
|
|
265
|
+
const rows = this.provider.all<{ entry_id_a: string; entry_id_b: string }>(
|
|
266
|
+
'SELECT entry_id_a, entry_id_b FROM curator_duplicate_dismissals',
|
|
267
|
+
);
|
|
268
|
+
return new Set(rows.map((r) => `${r.entry_id_a}::${r.entry_id_b}`));
|
|
155
269
|
}
|
|
156
270
|
|
|
157
271
|
// ─── Contradictions (delegates to contradiction-detector) ─────
|
|
@@ -159,9 +273,11 @@ export class Curator {
|
|
|
159
273
|
detectContradictions(threshold?: number): Contradiction[] {
|
|
160
274
|
const searchFn = (title: string) =>
|
|
161
275
|
this.vault.search(title, { type: 'pattern', limit: 20 }).map((r) => r.entry);
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
);
|
|
276
|
+
// Load only anti-patterns and patterns (bounded by type), not the entire vault
|
|
277
|
+
const antipatterns = this.vault.list({ type: 'anti-pattern', limit: 10000 });
|
|
278
|
+
const patterns = this.vault.list({ type: 'pattern', limit: 10000 });
|
|
279
|
+
const entries = [...antipatterns, ...patterns];
|
|
280
|
+
return this.persistContradictions(findContradictions(entries, threshold, searchFn));
|
|
165
281
|
}
|
|
166
282
|
|
|
167
283
|
getContradictions(status?: ContradictionStatus): Contradiction[] {
|
|
@@ -190,10 +306,12 @@ export class Curator {
|
|
|
190
306
|
): Promise<{ contradictions: Contradiction[]; method: 'tfidf-only' }> {
|
|
191
307
|
const searchFn = (title: string) =>
|
|
192
308
|
this.vault.search(title, { type: 'pattern', limit: 20 }).map((r) => r.entry);
|
|
309
|
+
// Load only anti-patterns and patterns (bounded by type), not the entire vault
|
|
310
|
+
const antipatterns = this.vault.list({ type: 'anti-pattern', limit: 10000 });
|
|
311
|
+
const patterns = this.vault.list({ type: 'pattern', limit: 10000 });
|
|
312
|
+
const entries = [...antipatterns, ...patterns];
|
|
193
313
|
return {
|
|
194
|
-
contradictions: this.persistContradictions(
|
|
195
|
-
findContradictions(this.vault.list({ limit: 100000 }), threshold, searchFn),
|
|
196
|
-
),
|
|
314
|
+
contradictions: this.persistContradictions(findContradictions(entries, threshold, searchFn)),
|
|
197
315
|
method: 'tfidf-only',
|
|
198
316
|
};
|
|
199
317
|
}
|
|
@@ -221,19 +339,28 @@ export class Curator {
|
|
|
221
339
|
|
|
222
340
|
groomAll(): GroomAllResult {
|
|
223
341
|
const start = Date.now();
|
|
224
|
-
const entries = this.vault.list({ limit: 100000 });
|
|
225
342
|
let tagsNormalized = 0,
|
|
226
|
-
staleCount = 0
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
343
|
+
staleCount = 0,
|
|
344
|
+
totalEntries = 0;
|
|
345
|
+
// Batch pagination — process entries in batches instead of loading all at once
|
|
346
|
+
let offset = 0;
|
|
347
|
+
while (true) {
|
|
348
|
+
const batch = this.vault.list({ limit: DEFAULT_BATCH_SIZE, offset });
|
|
349
|
+
if (batch.length === 0) break;
|
|
350
|
+
totalEntries += batch.length;
|
|
351
|
+
for (const entry of batch) {
|
|
352
|
+
const result = this.groomEntry(entry.id);
|
|
353
|
+
if (result) {
|
|
354
|
+
tagsNormalized += result.tagsNormalized.filter((t) => t.wasAliased).length;
|
|
355
|
+
if (result.stale) staleCount++;
|
|
356
|
+
}
|
|
232
357
|
}
|
|
358
|
+
if (batch.length < DEFAULT_BATCH_SIZE) break;
|
|
359
|
+
offset += DEFAULT_BATCH_SIZE;
|
|
233
360
|
}
|
|
234
361
|
return {
|
|
235
|
-
totalEntries
|
|
236
|
-
groomedCount:
|
|
362
|
+
totalEntries,
|
|
363
|
+
groomedCount: totalEntries,
|
|
237
364
|
tagsNormalized,
|
|
238
365
|
staleCount,
|
|
239
366
|
durationMs: Date.now() - start,
|
|
@@ -315,7 +442,8 @@ export class Curator {
|
|
|
315
442
|
// ─── Health Audit (delegates to health-audit) ─────────────────
|
|
316
443
|
|
|
317
444
|
healthAudit(): HealthAuditResult {
|
|
318
|
-
|
|
445
|
+
// Load entries in batches instead of all at once
|
|
446
|
+
const entries = this.listBatched();
|
|
319
447
|
const dataProvider: HealthDataProvider = {
|
|
320
448
|
getStaleCount: (threshold) =>
|
|
321
449
|
(
|
|
@@ -452,6 +580,22 @@ export class Curator {
|
|
|
452
580
|
|
|
453
581
|
// ─── Private Helpers ──────────────────────────────────────────
|
|
454
582
|
|
|
583
|
+
/**
|
|
584
|
+
* Load all vault entries using batched pagination instead of a single 100k query.
|
|
585
|
+
*/
|
|
586
|
+
private listBatched(batchSize: number = DEFAULT_BATCH_SIZE): IntelligenceEntry[] {
|
|
587
|
+
const all: IntelligenceEntry[] = [];
|
|
588
|
+
let offset = 0;
|
|
589
|
+
while (true) {
|
|
590
|
+
const batch = this.vault.list({ limit: batchSize, offset });
|
|
591
|
+
if (batch.length === 0) break;
|
|
592
|
+
all.push(...batch);
|
|
593
|
+
if (batch.length < batchSize) break;
|
|
594
|
+
offset += batchSize;
|
|
595
|
+
}
|
|
596
|
+
return all;
|
|
597
|
+
}
|
|
598
|
+
|
|
455
599
|
private persistContradictions(candidates: ContradictionCandidate[]): Contradiction[] {
|
|
456
600
|
const detected: Contradiction[] = [];
|
|
457
601
|
for (const c of candidates) {
|
package/src/curator/schema.ts
CHANGED
|
@@ -56,6 +56,14 @@ export const CURATOR_SCHEMA = `
|
|
|
56
56
|
resolved_at INTEGER,
|
|
57
57
|
UNIQUE(pattern_id, antipattern_id)
|
|
58
58
|
);
|
|
59
|
+
CREATE TABLE IF NOT EXISTS curator_duplicate_dismissals (
|
|
60
|
+
entry_id_a TEXT NOT NULL,
|
|
61
|
+
entry_id_b TEXT NOT NULL,
|
|
62
|
+
dismissed_at INTEGER NOT NULL DEFAULT (unixepoch()),
|
|
63
|
+
reason TEXT,
|
|
64
|
+
PRIMARY KEY (entry_id_a, entry_id_b)
|
|
65
|
+
);
|
|
66
|
+
|
|
59
67
|
CREATE INDEX IF NOT EXISTS idx_curator_state_status ON curator_entry_state(status);
|
|
60
68
|
CREATE INDEX IF NOT EXISTS idx_curator_changelog_entry ON curator_changelog(entry_id);
|
|
61
69
|
`;
|
|
@@ -67,12 +67,17 @@ export interface PackSkillDefinition {
|
|
|
67
67
|
// DomainPack — the main interface
|
|
68
68
|
// ---------------------------------------------------------------------------
|
|
69
69
|
|
|
70
|
+
/** Pack tier: determines visibility, licensing, and install behavior */
|
|
71
|
+
export type DomainPackTier = 'default' | 'community' | 'premium';
|
|
72
|
+
|
|
70
73
|
/** The contract every domain pack must implement. */
|
|
71
74
|
export interface DomainPack {
|
|
72
75
|
/** Unique pack name (e.g., 'design', 'security-intelligence') */
|
|
73
76
|
name: string;
|
|
74
77
|
/** Semver version */
|
|
75
78
|
version: string;
|
|
79
|
+
/** Tier: 'default' (ships with engine), 'community' (free, npm), 'premium' (unlocked today, gated later) */
|
|
80
|
+
tier?: DomainPackTier;
|
|
76
81
|
/** Domains this pack claims. Ops inject into these domain facades. */
|
|
77
82
|
domains: string[];
|
|
78
83
|
/** Custom operations with real logic — injected into claimed domain facades. */
|
|
@@ -115,6 +120,8 @@ export interface DomainPackRef {
|
|
|
115
120
|
package: string;
|
|
116
121
|
/** Optional version constraint */
|
|
117
122
|
version?: string;
|
|
123
|
+
/** Pack tier (inherited from pack if not set) */
|
|
124
|
+
tier?: DomainPackTier;
|
|
118
125
|
}
|
|
119
126
|
|
|
120
127
|
// ---------------------------------------------------------------------------
|
|
@@ -148,6 +155,7 @@ const packSkillSchema = z.object({
|
|
|
148
155
|
const domainPackSchema = z.object({
|
|
149
156
|
name: z.string().min(1),
|
|
150
157
|
version: z.string().min(1),
|
|
158
|
+
tier: z.enum(['default', 'community', 'premium']).optional(),
|
|
151
159
|
domains: z.array(z.string().min(1)).min(1),
|
|
152
160
|
ops: z.array(
|
|
153
161
|
z.object({
|
|
@@ -49,7 +49,7 @@ describe('ENGINE_MODULE_MANIFEST', () => {
|
|
|
49
49
|
expect(entry.description.length).toBeGreaterThan(0);
|
|
50
50
|
expect(Array.isArray(entry.keyOps)).toBe(true);
|
|
51
51
|
expect(entry.keyOps.length).toBeGreaterThan(0);
|
|
52
|
-
expect(entry.keyOps.length).toBeLessThanOrEqual(
|
|
52
|
+
expect(entry.keyOps.length).toBeLessThanOrEqual(5);
|
|
53
53
|
}
|
|
54
54
|
});
|
|
55
55
|
|
|
@@ -69,7 +69,13 @@ describe('ENGINE_MODULE_MANIFEST', () => {
|
|
|
69
69
|
|
|
70
70
|
it('plan module has expected keyOps', () => {
|
|
71
71
|
const plan = ENGINE_MODULE_MANIFEST.find((m) => m.suffix === 'plan')!;
|
|
72
|
-
expect(plan.keyOps).toEqual([
|
|
72
|
+
expect(plan.keyOps).toEqual([
|
|
73
|
+
'create_plan',
|
|
74
|
+
'approve_plan',
|
|
75
|
+
'plan_split',
|
|
76
|
+
'plan_reconcile',
|
|
77
|
+
'plan_close_stale',
|
|
78
|
+
]);
|
|
73
79
|
});
|
|
74
80
|
|
|
75
81
|
it('conditional field is optional and boolean when present', () => {
|
|
@@ -41,7 +41,7 @@ export const ENGINE_MODULE_MANIFEST: ModuleManifestEntry[] = [
|
|
|
41
41
|
{
|
|
42
42
|
suffix: 'plan',
|
|
43
43
|
description: 'Plan lifecycle — create, approve, execute, reconcile, complete, grading.',
|
|
44
|
-
keyOps: ['create_plan', 'approve_plan', 'plan_split', 'plan_reconcile'],
|
|
44
|
+
keyOps: ['create_plan', 'approve_plan', 'plan_split', 'plan_reconcile', 'plan_close_stale'],
|
|
45
45
|
intentSignals: {
|
|
46
46
|
'plan this': 'create_plan',
|
|
47
47
|
'break this down': 'create_plan',
|
|
@@ -56,7 +56,7 @@ export const ENGINE_MODULE_MANIFEST: ModuleManifestEntry[] = [
|
|
|
56
56
|
keyOps: ['recommend', 'strengths', 'feedback'],
|
|
57
57
|
intentSignals: {
|
|
58
58
|
'what works': 'recommend',
|
|
59
|
-
|
|
59
|
+
recommendations: 'recommend',
|
|
60
60
|
'pattern strengths': 'strengths',
|
|
61
61
|
'give feedback': 'feedback',
|
|
62
62
|
},
|
|
@@ -87,7 +87,7 @@ export const ENGINE_MODULE_MANIFEST: ModuleManifestEntry[] = [
|
|
|
87
87
|
{
|
|
88
88
|
suffix: 'curator',
|
|
89
89
|
description: 'Quality — duplicate detection, contradictions, grooming, health audit.',
|
|
90
|
-
keyOps: ['curator_groom', 'curator_status', 'curator_health'],
|
|
90
|
+
keyOps: ['curator_groom', 'curator_status', 'curator_health', 'curator_dismiss_duplicate'],
|
|
91
91
|
intentSignals: {
|
|
92
92
|
'clean up vault': 'curator_groom',
|
|
93
93
|
'find duplicates': 'curator_groom',
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
|
16
16
|
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
17
17
|
import { createAgentRuntime } from '../runtime/runtime.js';
|
|
18
|
-
import { registerEngine, ENGINE_MODULES } from './register-engine.js';
|
|
18
|
+
import { registerEngine, ENGINE_MODULES, INTERNAL_OPS } from './register-engine.js';
|
|
19
19
|
import { ENGINE_MODULE_MANIFEST } from './module-manifest.js';
|
|
20
20
|
import type { AgentRuntime } from '../runtime/types.js';
|
|
21
21
|
import type { OpDefinition } from '../facades/types.js';
|
|
@@ -233,3 +233,75 @@ describe('ENGINE_MODULES descriptions match manifest', () => {
|
|
|
233
233
|
}
|
|
234
234
|
});
|
|
235
235
|
});
|
|
236
|
+
|
|
237
|
+
describe('registerEngine — op visibility', () => {
|
|
238
|
+
it('INTERNAL_OPS set contains expected ops', () => {
|
|
239
|
+
// Spot-check known internal ops
|
|
240
|
+
expect(INTERNAL_OPS.has('admin_create_token')).toBe(true);
|
|
241
|
+
expect(INTERNAL_OPS.has('vault_bulk_add')).toBe(true);
|
|
242
|
+
expect(INTERNAL_OPS.has('plan_auto_reconcile')).toBe(true);
|
|
243
|
+
expect(INTERNAL_OPS.has('telemetry_errors')).toBe(true);
|
|
244
|
+
// User-facing ops should NOT be in the set
|
|
245
|
+
expect(INTERNAL_OPS.has('admin_health')).toBe(false);
|
|
246
|
+
expect(INTERNAL_OPS.has('search_intelligent')).toBe(false);
|
|
247
|
+
expect(INTERNAL_OPS.has('create_plan')).toBe(false);
|
|
248
|
+
});
|
|
249
|
+
|
|
250
|
+
it('INTERNAL_OPS has at least 25 entries', () => {
|
|
251
|
+
expect(INTERNAL_OPS.size).toBeGreaterThanOrEqual(25);
|
|
252
|
+
});
|
|
253
|
+
|
|
254
|
+
it('ops without visibility field default to user (backward compat)', () => {
|
|
255
|
+
const server = makeServer();
|
|
256
|
+
const userOp: OpDefinition = {
|
|
257
|
+
name: 'my_visible_op',
|
|
258
|
+
description: 'Visible op',
|
|
259
|
+
auth: 'read',
|
|
260
|
+
handler: async () => 'ok',
|
|
261
|
+
};
|
|
262
|
+
const result = registerEngine(server, runtime, {
|
|
263
|
+
agentId: 'vis',
|
|
264
|
+
domainPacks: [{ name: 'test', facades: [{ name: 'test', ops: [userOp] }] }],
|
|
265
|
+
});
|
|
266
|
+
expect(result.tools).toContain('vis_test');
|
|
267
|
+
});
|
|
268
|
+
|
|
269
|
+
it('ops with visibility: internal are excluded from MCP tool description but remain callable', () => {
|
|
270
|
+
const server = makeServer();
|
|
271
|
+
const visibleOp: OpDefinition = {
|
|
272
|
+
name: 'public_op',
|
|
273
|
+
description: 'Public op',
|
|
274
|
+
auth: 'read',
|
|
275
|
+
handler: async () => 'visible',
|
|
276
|
+
};
|
|
277
|
+
const internalOp: OpDefinition = {
|
|
278
|
+
name: 'secret_op',
|
|
279
|
+
description: 'Internal op',
|
|
280
|
+
auth: 'admin',
|
|
281
|
+
visibility: 'internal',
|
|
282
|
+
handler: async () => 'hidden',
|
|
283
|
+
};
|
|
284
|
+
// Register both ops under a pack facade
|
|
285
|
+
registerEngine(server, runtime, {
|
|
286
|
+
agentId: 'vt',
|
|
287
|
+
domainPacks: [{ name: 'test', facades: [{ name: 'check', ops: [visibleOp, internalOp] }] }],
|
|
288
|
+
});
|
|
289
|
+
// We can't easily inspect the MCP schema description string, but we verify
|
|
290
|
+
// that registration succeeds with mixed visibility ops
|
|
291
|
+
expect(true).toBe(true);
|
|
292
|
+
});
|
|
293
|
+
|
|
294
|
+
it('every INTERNAL_OPS entry corresponds to a real op in some facade', () => {
|
|
295
|
+
// Collect all op names across all engine modules
|
|
296
|
+
const allOpNames = new Set<string>();
|
|
297
|
+
for (const mod of ENGINE_MODULES) {
|
|
298
|
+
const ops = mod.createOps(runtime);
|
|
299
|
+
for (const op of ops) {
|
|
300
|
+
allOpNames.add(op.name);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
for (const internalOp of INTERNAL_OPS) {
|
|
304
|
+
expect(allOpNames.has(internalOp)).toBe(true);
|
|
305
|
+
}
|
|
306
|
+
});
|
|
307
|
+
});
|
|
@@ -295,11 +295,70 @@ export function registerEngine(
|
|
|
295
295
|
return { tools: registeredTools, totalOps, registerTool };
|
|
296
296
|
}
|
|
297
297
|
|
|
298
|
+
// ─── Op Visibility ───────────────────────────────────────────────────
|
|
299
|
+
|
|
300
|
+
/**
|
|
301
|
+
* Ops classified as internal — hidden from MCP tool descriptions but still
|
|
302
|
+
* callable programmatically. Centralized here for easy auditing.
|
|
303
|
+
*
|
|
304
|
+
* Criteria: ops that are infrastructure plumbing, token/account management,
|
|
305
|
+
* bulk operations, or auto-* automation that users never call directly.
|
|
306
|
+
*/
|
|
307
|
+
/** @internal Exported for testing — do not use outside engine */
|
|
308
|
+
export const INTERNAL_OPS = new Set([
|
|
309
|
+
// Admin: token management
|
|
310
|
+
'admin_create_token',
|
|
311
|
+
'admin_revoke_token',
|
|
312
|
+
'admin_list_tokens',
|
|
313
|
+
// Admin: account management
|
|
314
|
+
'admin_add_account',
|
|
315
|
+
'admin_remove_account',
|
|
316
|
+
'admin_rotate_account',
|
|
317
|
+
'admin_list_accounts',
|
|
318
|
+
'admin_account_status',
|
|
319
|
+
// Admin: infrastructure debug
|
|
320
|
+
'admin_vault_size',
|
|
321
|
+
'admin_vault_analytics',
|
|
322
|
+
'admin_search_insights',
|
|
323
|
+
'admin_env',
|
|
324
|
+
'admin_gc',
|
|
325
|
+
'admin_export_config',
|
|
326
|
+
'admin_key_pool_status',
|
|
327
|
+
'admin_persistence_info',
|
|
328
|
+
// Admin: flags
|
|
329
|
+
'admin_list_flags',
|
|
330
|
+
'admin_get_flag',
|
|
331
|
+
'admin_set_flag',
|
|
332
|
+
// Admin: setup internals
|
|
333
|
+
'admin_inject_claude_md',
|
|
334
|
+
// Admin: telemetry
|
|
335
|
+
'admin_telemetry',
|
|
336
|
+
'admin_telemetry_recent',
|
|
337
|
+
'admin_telemetry_reset',
|
|
338
|
+
'telemetry_errors',
|
|
339
|
+
'telemetry_slow_ops',
|
|
340
|
+
// Vault: bulk operations
|
|
341
|
+
'vault_bulk_add',
|
|
342
|
+
'vault_bulk_remove',
|
|
343
|
+
// Plan: automation
|
|
344
|
+
'plan_auto_reconcile',
|
|
345
|
+
'plan_auto_improve',
|
|
346
|
+
]);
|
|
347
|
+
|
|
348
|
+
/** Resolve effective visibility: explicit field wins, then INTERNAL_OPS set, else 'user'. */
|
|
349
|
+
function resolveVisibility(op: OpDefinition): 'user' | 'internal' {
|
|
350
|
+
if (op.visibility) return op.visibility;
|
|
351
|
+
return INTERNAL_OPS.has(op.name) ? 'internal' : 'user';
|
|
352
|
+
}
|
|
353
|
+
|
|
298
354
|
// ─── Tool Registration (No Factory) ──────────────────────────────────
|
|
299
355
|
|
|
300
356
|
/**
|
|
301
357
|
* Register a single grouped tool with op dispatch.
|
|
302
358
|
* This is the replacement for registerFacade() — same behavior, no FacadeConfig type.
|
|
359
|
+
*
|
|
360
|
+
* Internal ops (visibility: 'internal') are excluded from the MCP tool description
|
|
361
|
+
* but remain dispatchable — they just don't show up in the op list Claude sees.
|
|
303
362
|
*/
|
|
304
363
|
function registerModuleTool(
|
|
305
364
|
server: McpServer,
|
|
@@ -308,7 +367,8 @@ function registerModuleTool(
|
|
|
308
367
|
ops: OpDefinition[],
|
|
309
368
|
authPolicy?: () => AuthPolicy,
|
|
310
369
|
): void {
|
|
311
|
-
const
|
|
370
|
+
const userOps = ops.filter((o) => resolveVisibility(o) !== 'internal');
|
|
371
|
+
const opNames = userOps.map((o) => o.name);
|
|
312
372
|
const opMap = new Map(ops.map((o) => [o.name, o]));
|
|
313
373
|
|
|
314
374
|
server.tool(
|
package/src/facades/types.ts
CHANGED
|
@@ -25,6 +25,9 @@ export const AUTH_LEVEL_RANK: Record<AuthLevel, number> = {
|
|
|
25
25
|
admin: 2,
|
|
26
26
|
};
|
|
27
27
|
|
|
28
|
+
/** Op visibility — controls whether an op is exposed via MCP tool registration */
|
|
29
|
+
export type OpVisibility = 'user' | 'internal';
|
|
30
|
+
|
|
28
31
|
/** Operation definition within a facade */
|
|
29
32
|
export interface OpDefinition {
|
|
30
33
|
name: string;
|
|
@@ -34,6 +37,8 @@ export interface OpDefinition {
|
|
|
34
37
|
schema?: z.ZodType;
|
|
35
38
|
/** Promote to a first-class MCP tool with full schema discovery. */
|
|
36
39
|
hot?: boolean;
|
|
40
|
+
/** Controls MCP exposure: 'user' (default) = listed in tool, 'internal' = hidden from MCP but callable programmatically. */
|
|
41
|
+
visibility?: OpVisibility;
|
|
37
42
|
}
|
|
38
43
|
|
|
39
44
|
/** Facade configuration — one MCP tool */
|