@soleri/core 9.4.0 → 9.6.0

package/src/curator/curator.ts

@@ -5,6 +5,7 @@ import type {
   CuratorStatus,
   TagNormalizationResult,
   CanonicalTag,
+  DuplicateCandidate,
   DuplicateDetectionResult,
   Contradiction,
   ContradictionStatus,
@@ -17,14 +18,17 @@ import type {
 } from './types.js';
 
 import {
-  detectDuplicates as detectDuplicatesPure,
   DEFAULT_DUPLICATE_THRESHOLD,
+  MERGE_SUGGESTION_THRESHOLD,
+  buildVocabulary,
+  entryToText,
 } from './duplicate-detector.js';
 import {
   findContradictions,
   DEFAULT_CONTRADICTION_THRESHOLD,
   type ContradictionCandidate,
 } from './contradiction-detector.js';
+import { tokenize, calculateTfIdf, cosineSimilarity } from '../text/similarity.js';
 import {
   normalizeTag as normalizeTagPure,
   normalizeAndDedup,
@@ -40,6 +44,7 @@ import { enrichEntryMetadata } from './metadata-enricher.js';
 // ─── Constants ──────────────────────────────────────────────────────
 
 const DEFAULT_STALE_DAYS = 90;
+const DEFAULT_BATCH_SIZE = 100;
 
 // ─── Curator Class ──────────────────────────────────────────────────
 
@@ -151,19 +156,101 @@ export class Curator {
   // ─── Duplicates (delegates to duplicate-detector) ─────────────
 
   detectDuplicates(entryId?: string, threshold?: number): DuplicateDetectionResult[] {
-    const results = detectDuplicatesPure(this.vault.list({ limit: 100000 }), entryId, threshold);
-    // Filter out dismissed pairs
+    const effectiveThreshold = threshold ?? DEFAULT_DUPLICATE_THRESHOLD;
     const dismissed = this.getDismissedPairs();
-    if (dismissed.size === 0) return results;
-    return results
-      .map((r) => ({
-        ...r,
-        matches: r.matches.filter((m) => {
-          const key = [r.entryId, m.entryId].sort().join('::');
-          return !dismissed.has(key);
-        }),
-      }))
-      .filter((r) => r.matches.length > 0);
+
+    // --- Phase 1: Content-hash exact duplicates (O(n) via GROUP BY) ---
+    const exactDupes = this.provider.all<{ content_hash: string; ids: string }>(
+      `SELECT content_hash, GROUP_CONCAT(id) as ids FROM entries
+       WHERE content_hash IS NOT NULL
+       GROUP BY content_hash HAVING COUNT(*) > 1`,
+    );
+
+    const results: DuplicateDetectionResult[] = [];
+    const seenPairs = new Set<string>();
+
+    for (const { ids } of exactDupes) {
+      const idList = ids.split(',');
+      if (entryId && !idList.includes(entryId)) continue;
+      const targets = entryId ? [entryId] : idList;
+      for (const targetId of targets) {
+        const matches: DuplicateCandidate[] = [];
+        for (const otherId of idList) {
+          if (otherId === targetId) continue;
+          const pairKey = [targetId, otherId].sort().join('::');
+          if (dismissed.has(pairKey) || seenPairs.has(pairKey)) continue;
+          seenPairs.add(pairKey);
+          const other = this.vault.get(otherId);
+          if (!other) continue;
+          matches.push({
+            entryId: otherId,
+            title: other.title,
+            similarity: 1.0,
+            suggestMerge: true,
+          });
+        }
+        if (matches.length > 0) {
+          results.push({ entryId: targetId, matches, scannedCount: idList.length - 1 });
+        }
+      }
+    }
+
+    // --- Phase 2: FTS5 fuzzy candidate matching ---
+    // For each entry, use FTS5 MATCH to find top candidates, then TF-IDF cosine similarity.
+    // Complexity is O(n * k) where k = FTS5 candidate limit (10), not O(n^2).
+    const exactDupeEntryIds = new Set(results.map((r) => r.entryId));
+    const targetEntries = entryId
+      ? ([this.vault.get(entryId)].filter(Boolean) as IntelligenceEntry[])
+      : this.listBatched();
+
+    // Build vocabulary from all entries (batched)
+    const allEntries = entryId ? this.listBatched() : targetEntries;
+    const vocabulary = buildVocabulary(allEntries);
+
+    for (const entry of targetEntries) {
+      // Skip entries already fully handled by exact-hash matches
+      if (exactDupeEntryIds.has(entry.id)) continue;
+
+      // FTS5 candidate retrieval: find top-10 similar entries
+      let candidates: IntelligenceEntry[];
+      try {
+        candidates = this.vault
+          .search(entry.title, { domain: entry.domain, limit: 10 })
+          .map((r) => r.entry)
+          .filter((c) => c.id !== entry.id);
+      } catch {
+        candidates = [];
+      }
+      if (candidates.length === 0) continue;
+
+      const entryVec = calculateTfIdf(tokenize(entryToText(entry)), vocabulary);
+      const matches: DuplicateCandidate[] = [];
+
+      for (const candidate of candidates) {
+        // Skip cross-domain pairs
+        if (entry.domain !== candidate.domain) continue;
+        const pairKey = [entry.id, candidate.id].sort().join('::');
+        if (dismissed.has(pairKey)) continue;
+
+        const candidateVec = calculateTfIdf(tokenize(entryToText(candidate)), vocabulary);
+        const similarity = cosineSimilarity(entryVec, candidateVec);
+        if (similarity >= effectiveThreshold) {
+          matches.push({
+            entryId: candidate.id,
+            title: candidate.title,
+            similarity,
+            suggestMerge: similarity >= MERGE_SUGGESTION_THRESHOLD,
+          });
+        }
+      }
+
+      if (matches.length > 0) {
+        matches.sort((a, b) => b.similarity - a.similarity);
+        results.push({ entryId: entry.id, matches, scannedCount: candidates.length });
+      }
+    }
+
+    return results;
   }
 
   dismissDuplicate(entryIdA: string, entryIdB: string, reason?: string): { dismissed: boolean } {
@@ -187,9 +274,11 @@ export class Curator {
   detectContradictions(threshold?: number): Contradiction[] {
     const searchFn = (title: string) =>
       this.vault.search(title, { type: 'pattern', limit: 20 }).map((r) => r.entry);
-    return this.persistContradictions(
-      findContradictions(this.vault.list({ limit: 100000 }), threshold, searchFn),
-    );
+    // Load only anti-patterns and patterns (bounded by type), not the entire vault
+    const antipatterns = this.vault.list({ type: 'anti-pattern', limit: 10000 });
+    const patterns = this.vault.list({ type: 'pattern', limit: 10000 });
+    const entries = [...antipatterns, ...patterns];
+    return this.persistContradictions(findContradictions(entries, threshold, searchFn));
   }
 
   getContradictions(status?: ContradictionStatus): Contradiction[] {
@@ -218,10 +307,12 @@ export class Curator {
   ): Promise<{ contradictions: Contradiction[]; method: 'tfidf-only' }> {
     const searchFn = (title: string) =>
       this.vault.search(title, { type: 'pattern', limit: 20 }).map((r) => r.entry);
+    // Load only anti-patterns and patterns (bounded by type), not the entire vault
+    const antipatterns = this.vault.list({ type: 'anti-pattern', limit: 10000 });
+    const patterns = this.vault.list({ type: 'pattern', limit: 10000 });
+    const entries = [...antipatterns, ...patterns];
     return {
-      contradictions: this.persistContradictions(
-        findContradictions(this.vault.list({ limit: 100000 }), threshold, searchFn),
-      ),
+      contradictions: this.persistContradictions(findContradictions(entries, threshold, searchFn)),
       method: 'tfidf-only',
     };
   }
@@ -249,19 +340,28 @@ export class Curator {
 
   groomAll(): GroomAllResult {
     const start = Date.now();
-    const entries = this.vault.list({ limit: 100000 });
     let tagsNormalized = 0,
-      staleCount = 0;
-    for (const entry of entries) {
-      const result = this.groomEntry(entry.id);
-      if (result) {
-        tagsNormalized += result.tagsNormalized.filter((t) => t.wasAliased).length;
-        if (result.stale) staleCount++;
+      staleCount = 0,
+      totalEntries = 0;
+    // Batch pagination — process entries in batches instead of loading all at once
+    let offset = 0;
+    while (true) {
+      const batch = this.vault.list({ limit: DEFAULT_BATCH_SIZE, offset });
+      if (batch.length === 0) break;
+      totalEntries += batch.length;
+      for (const entry of batch) {
+        const result = this.groomEntry(entry.id);
+        if (result) {
+          tagsNormalized += result.tagsNormalized.filter((t) => t.wasAliased).length;
+          if (result.stale) staleCount++;
+        }
       }
+      if (batch.length < DEFAULT_BATCH_SIZE) break;
+      offset += DEFAULT_BATCH_SIZE;
     }
     return {
-      totalEntries: entries.length,
-      groomedCount: entries.length,
+      totalEntries,
+      groomedCount: totalEntries,
       tagsNormalized,
       staleCount,
       durationMs: Date.now() - start,
@@ -343,7 +443,8 @@ export class Curator {
   // ─── Health Audit (delegates to health-audit) ─────────────────
 
   healthAudit(): HealthAuditResult {
-    const entries = this.vault.list({ limit: 100000 });
+    // Load entries in batches instead of all at once
+    const entries = this.listBatched();
     const dataProvider: HealthDataProvider = {
       getStaleCount: (threshold) =>
         (
@@ -480,6 +581,22 @@ export class Curator {
 
   // ─── Private Helpers ──────────────────────────────────────────
 
+  /**
+   * Load all vault entries using batched pagination instead of a single 100k query.
+   */
+  private listBatched(batchSize: number = DEFAULT_BATCH_SIZE): IntelligenceEntry[] {
+    const all: IntelligenceEntry[] = [];
+    let offset = 0;
+    while (true) {
+      const batch = this.vault.list({ limit: batchSize, offset });
+      if (batch.length === 0) break;
+      all.push(...batch);
+      if (batch.length < batchSize) break;
+      offset += batchSize;
+    }
+    return all;
+  }
+
   private persistContradictions(candidates: ContradictionCandidate[]): Contradiction[] {
     const detected: Contradiction[] = [];
     for (const c of candidates) {

package/src/hooks/candidate-scorer.test.ts

@@ -0,0 +1,76 @@
+import { describe, it, expect } from 'vitest';
+import { scoreCandidateForConversion } from './candidate-scorer.js';
+import type { CandidateDimensions } from './candidate-scorer.js';
+
+describe('scoreCandidateForConversion', () => {
+  it('should identify strong candidate (4/4 HIGH)', () => {
+    const dimensions: CandidateDimensions = {
+      frequency: 'HIGH',
+      eventCorrelation: 'HIGH',
+      determinism: 'HIGH',
+      autonomy: 'HIGH',
+    };
+    const result = scoreCandidateForConversion(dimensions);
+    expect(result.highCount).toBe(4);
+    expect(result.candidate).toBe(true);
+  });
+
+  it('should identify borderline candidate (3/4 HIGH)', () => {
+    const dimensions: CandidateDimensions = {
+      frequency: 'HIGH',
+      eventCorrelation: 'HIGH',
+      determinism: 'HIGH',
+      autonomy: 'LOW',
+    };
+    const result = scoreCandidateForConversion(dimensions);
+    expect(result.highCount).toBe(3);
+    expect(result.candidate).toBe(true);
+  });
+
+  it('should reject non-candidate (2/4 HIGH)', () => {
+    const dimensions: CandidateDimensions = {
+      frequency: 'HIGH',
+      eventCorrelation: 'LOW',
+      determinism: 'HIGH',
+      autonomy: 'LOW',
+    };
+    const result = scoreCandidateForConversion(dimensions);
+    expect(result.highCount).toBe(2);
+    expect(result.candidate).toBe(false);
+  });
+
+  it('should reject weak candidate (1/4 HIGH)', () => {
+    const dimensions: CandidateDimensions = {
+      frequency: 'LOW',
+      eventCorrelation: 'LOW',
+      determinism: 'LOW',
+      autonomy: 'HIGH',
+    };
+    const result = scoreCandidateForConversion(dimensions);
+    expect(result.highCount).toBe(1);
+    expect(result.candidate).toBe(false);
+  });
+
+  it('should reject zero score (0/4 HIGH)', () => {
+    const dimensions: CandidateDimensions = {
+      frequency: 'LOW',
+      eventCorrelation: 'LOW',
+      determinism: 'LOW',
+      autonomy: 'LOW',
+    };
+    const result = scoreCandidateForConversion(dimensions);
+    expect(result.highCount).toBe(0);
+    expect(result.candidate).toBe(false);
+  });
+
+  it('should preserve dimension values in result', () => {
+    const dimensions: CandidateDimensions = {
+      frequency: 'HIGH',
+      eventCorrelation: 'LOW',
+      determinism: 'HIGH',
+      autonomy: 'HIGH',
+    };
+    const result = scoreCandidateForConversion(dimensions);
+    expect(result.dimensions).toEqual(dimensions);
+  });
+});

package/src/hooks/candidate-scorer.ts

@@ -0,0 +1,39 @@
+/**
+ * Candidate scoring rubric for skill-to-hook conversion.
+ * 4-dimension scoring: frequency, eventCorrelation, determinism, autonomy.
+ * Threshold: >= 3 HIGH dimensions = candidate for conversion.
+ */
+
+export type DimensionLevel = 'HIGH' | 'LOW';
+
+export interface CandidateDimensions {
+  /** 3+ manual calls per session for same event type → HIGH */
+  frequency: DimensionLevel;
+  /** Skill consistently triggers on a recognizable hook event → HIGH */
+  eventCorrelation: DimensionLevel;
+  /** Skill produces consistent, non-exploratory guidance → HIGH */
+  determinism: DimensionLevel;
+  /** Skill requires no interactive user decisions mid-execution → HIGH */
+  autonomy: DimensionLevel;
+}
+
+export interface CandidateScore {
+  dimensions: CandidateDimensions;
+  highCount: number;
+  candidate: boolean;
+}
+
+/**
+ * Score a skill for hook conversion candidacy.
+ * @param dimensions - The 4 scored dimensions (each HIGH or LOW)
+ * @returns Score with candidate boolean (true if >= 3 HIGH)
+ */
+export function scoreCandidateForConversion(dimensions: CandidateDimensions): CandidateScore {
+  const values = Object.values(dimensions);
+  const highCount = values.filter((v) => v === 'HIGH').length;
+  return {
+    dimensions,
+    highCount,
+    candidate: highCount >= 3,
+  };
+}

package/src/index.ts

@@ -1,3 +1,34 @@
+// ─── Adapters ────────────────────────────────────────────────────────
+export { RuntimeAdapterRegistry } from './adapters/registry.js';
+export { ClaudeCodeRuntimeAdapter } from './adapters/claude-code-adapter.js';
+export type {
+  RuntimeAdapter,
+  AdapterExecutionContext,
+  AdapterExecutionResult,
+  AdapterTokenUsage,
+  AdapterSessionState,
+  AdapterSessionCodec,
+  AdapterEnvironmentTestResult,
+} from './adapters/types.js';
+
+// ─── Subagent ────────────────────────────────────────────────────────
+export { SubagentDispatcher } from './subagent/dispatcher.js';
+export { TaskCheckout } from './subagent/task-checkout.js';
+export { WorkspaceResolver } from './subagent/workspace-resolver.js';
+export { ConcurrencyManager } from './subagent/concurrency-manager.js';
+export { OrphanReaper } from './subagent/orphan-reaper.js';
+export { aggregate as aggregateResults } from './subagent/result-aggregator.js';
+export type {
+  SubagentTask,
+  SubagentStatus,
+  SubagentResult,
+  DispatchOptions,
+  AggregatedResult,
+  ClaimInfo,
+  WorktreeInfo,
+  TrackedProcess,
+} from './subagent/types.js';
+
 // ─── Paths ──────────────────────────────────────────────────────────
 export {
   SOLERI_HOME,
@@ -551,6 +582,8 @@ export { createSemanticFacades } from './runtime/facades/index.js';
 export { createDomainFacade, createDomainFacades } from './runtime/domain-ops.js';
 export { FeatureFlags } from './runtime/feature-flags.js';
 export type { FlagDefinition } from './runtime/feature-flags.js';
+export { ShutdownRegistry } from './runtime/shutdown-registry.js';
+export type { ShutdownCallback } from './runtime/shutdown-registry.js';
 export type { AgentRuntimeConfig, AgentRuntime } from './runtime/types.js';
 export {
   deprecationWarning,
@@ -615,7 +648,13 @@ export { computeContentHash } from './vault/content-hash.js';
 export type { HashableEntry } from './vault/content-hash.js';
 
 // ─── Knowledge Packs ────────────────────────────────────────────────────
-export { PackInstaller, packManifestSchema } from './packs/index.js';
+export {
+  PackInstaller,
+  PackLifecycleManager,
+  packManifestSchema,
+  VALID_TRANSITIONS,
+} from './packs/index.js';
+export type { PackState, PackTransition } from './packs/index.js';
 export { PackLockfile, inferPackType } from './packs/index.js';
 export { resolvePack, checkNpmVersion, checkVersionCompat } from './packs/index.js';
 export type {

package/src/llm/llm-client.ts

@@ -237,6 +237,7 @@ export class LLMClient {
           temperature: options.temperature ?? 0.3,
           max_completion_tokens: options.maxTokens ?? 500,
         }),
+        signal: AbortSignal.timeout(60_000),
       });
 
       if (response.headers) {

package/src/packs/index.ts

@@ -5,17 +5,21 @@
 export {
   packManifestSchema,
   PACK_TIERS,
+  VALID_TRANSITIONS,
   type PackManifest,
   type PackTier as ManifestPackTier,
   type PackStatus,
+  type PackState,
+  type PackTransition,
   type InstalledPack,
   type InstallResult,
   type ValidateResult,
 } from './types.js';
 
 export { PackInstaller } from './pack-installer.js';
+export { PackLifecycleManager } from './pack-lifecycle.js';
 
-export { PackLockfile, inferPackType } from './lockfile.js';
+export { PackLockfile, inferPackType, LOCKFILE_VERSION } from './lockfile.js';
 export type { LockEntry, PackType, PackSource, PackTier, LockfileData } from './lockfile.js';
 
 export { resolvePack, checkNpmVersion, checkVersionCompat } from './resolver.js';

package/src/packs/lockfile.ts

@@ -41,15 +41,25 @@ export interface LockEntry {
   soleriRange?: string;
   /** Pack tier: default (ships with engine), community (free), premium (unlocked today) */
   tier?: PackTier;
+  /** Current pack lifecycle state */
+  state?: string;
+  /** Most recent lifecycle transition */
+  lastTransition?: { from: string; to: string; timestamp: string; reason?: string };
+  /** ISO timestamp when the pack was disabled */
+  disabledAt?: string;
+  /** Error details if the pack is in an error state */
+  errorMessage?: string;
 }
 
 export type PackType = 'hooks' | 'skills' | 'knowledge' | 'domain' | 'bundle';
 export type PackSource = 'built-in' | 'local' | 'npm';
 export type PackTier = 'default' | 'community' | 'premium';
 
+export const LOCKFILE_VERSION = 2;
+
 export interface LockfileData {
   /** Lockfile format version */
-  version: 1;
+  version: number;
   /** Map of packId → lock entry */
   packs: Record<string, LockEntry>;
 }
@@ -141,18 +151,73 @@ export class PackLockfile {
     return 'sha256-' + createHash('sha256').update(content).digest('hex');
   }
 
+  /**
+   * Update lifecycle fields for a pack entry and save.
+   */
+  updateLifecycle(
+    packId: string,
+    state: string,
+    transition?: { from: string; to: string; reason?: string },
+  ): void {
+    const entry = this.data.packs[packId];
+    if (!entry) return;
+
+    entry.state = state;
+
+    if (transition) {
+      entry.lastTransition = {
+        from: transition.from,
+        to: transition.to,
+        timestamp: new Date().toISOString(),
+        reason: transition.reason,
+      };
+    }
+
+    if (state === 'disabled') {
+      entry.disabledAt = new Date().toISOString();
+    } else {
+      delete entry.disabledAt;
+    }
+
+    if (state === 'error') {
+      // errorMessage is set externally if needed; keep existing value
+    } else {
+      delete entry.errorMessage;
+    }
+
+    this.dirty = true;
+    this.save();
+  }
+
   private load(): LockfileData {
     if (!existsSync(this.filePath)) {
-      return { version: 1, packs: {} };
+      return { version: LOCKFILE_VERSION, packs: {} };
     }
     try {
       const raw = JSON.parse(readFileSync(this.filePath, 'utf-8'));
-      if (raw.version === 1 && typeof raw.packs === 'object') {
+      if (typeof raw.packs !== 'object') {
+        return { version: LOCKFILE_VERSION, packs: {} };
+      }
+
+      // Migrate v1 → v2: add lifecycle fields
+      if (raw.version === 1) {
+        for (const entry of Object.values(raw.packs) as LockEntry[]) {
+          if (!entry.state) {
+            entry.state = 'ready';
+          }
+        }
+        raw.version = LOCKFILE_VERSION;
+        // Mark dirty so the migrated data gets persisted on next save
+        this.dirty = true;
+      }
+
+      if (raw.version === LOCKFILE_VERSION) {
         return raw as LockfileData;
       }
-      return { version: 1, packs: {} };
+
+      return { version: LOCKFILE_VERSION, packs: {} };
     } catch {
-      return { version: 1, packs: {} };
+      return { version: LOCKFILE_VERSION, packs: {} };
     }
   }
 }