@soleri/core 9.14.4 → 9.16.7

package/src/vault/tag-normalizer.ts

@@ -0,0 +1,188 @@
+/**
+ * Tag Normalizer — canonical tag taxonomy enforcement.
+ *
+ * Maps raw tags to nearest canonical via Levenshtein edit-distance.
+ * Strips noise words (version strings, single generic words).
+ * Respects metadata tag prefixes (e.g. 'source:').
+ *
+ * Three modes:
+ *   - 'enforce': must match within edit-distance 3, else drop (return null)
+ *   - 'suggest': map to nearest canonical within edit-distance 2 (passthrough if no match)
+ *   - 'off':     no normalization — return tag as-is
+ */
+
+// ─── Noise filter ────────────────────────────────────────────────────────────
+
+/**
+ * Version-string pattern: v1.2, v10, v1.2.3, etc.
+ */
+const VERSION_PATTERN = /^v\d+(\.\d+)*/i;
+
+/**
+ * Generic single words that add no signal.
+ */
+const NOISE_WORDS = new Set([
+  'one',
+  'via',
+  'new',
+  'full',
+  'actual',
+  'raw',
+  'the',
+  'and',
+  'for',
+  'with',
+  'this',
+  'that',
+  'from',
+  'into',
+]);
+
+function isNoisy(tag: string): boolean {
+  if (VERSION_PATTERN.test(tag)) return true;
+  if (NOISE_WORDS.has(tag.toLowerCase())) return true;
+  return false;
+}
+
+// ─── Levenshtein edit distance ───────────────────────────────────────────────
+
+/**
+ * Compute Levenshtein edit distance between two strings.
+ * O(m*n) time, O(n) space using two-row DP.
+ */
+export function computeEditDistance(a: string, b: string): number {
+  if (a === b) return 0;
+  if (a.length === 0) return b.length;
+  if (b.length === 0) return a.length;
+
+  let prev = Array.from({ length: b.length + 1 }, (_, i) => i);
+  let curr = Array.from<number>({ length: b.length + 1 });
+
+  for (let i = 1; i <= a.length; i++) {
+    curr[0] = i;
+    for (let j = 1; j <= b.length; j++) {
+      const cost = a[i - 1] === b[j - 1] ? 0 : 1;
+      curr[j] = Math.min(
+        curr[j - 1] + 1, // insertion
+        prev[j] + 1, // deletion
+        prev[j - 1] + cost, // substitution
+      );
+    }
+    [prev, curr] = [curr, prev];
+  }
+
+  return prev[b.length];
+}
+
+// ─── Metadata tag check ──────────────────────────────────────────────────────
+
+/**
+ * Returns true if the tag starts with any of the given prefixes.
+ * Metadata tags (e.g. 'source:article') are exempt from canonical normalization.
+ */
+export function isMetadataTag(tag: string, prefixes: string[]): boolean {
+  return prefixes.some((prefix) => tag.startsWith(prefix));
+}
+
+// ─── Single tag normalization ────────────────────────────────────────────────
+
+/**
+ * Normalize a single tag against a canonical list.
+ *
+ * @param tag       - Raw tag to normalize
+ * @param canonical - Canonical tag list to map against
+ * @param mode      - Constraint mode: 'enforce' | 'suggest' | 'off'
+ * @returns Normalized tag string, or null if the tag should be dropped.
+ */
+export function normalizeTag(
+  tag: string,
+  canonical: string[],
+  mode: 'enforce' | 'suggest' | 'off',
+): string | null {
+  if (mode === 'off') return tag;
+
+  const lower = tag.toLowerCase().trim();
+
+  // Always drop noise words
+  if (isNoisy(lower)) return null;
+
+  // Derive lowercase canonical for matching; preserve original casing for return
+  const canonicalLower = canonical.map((x) => x.toLowerCase());
+
+  // Exact match in canonical list — return canonical form (original casing)
+  const exactIdx = canonicalLower.indexOf(lower);
+  if (exactIdx !== -1) return canonical[exactIdx];
+
+  if (canonical.length === 0) {
+    // No canonical list configured — pass through in suggest, drop in enforce
+    return mode === 'enforce' ? null : lower;
+  }
+
+  // Find nearest canonical by edit distance
+  let bestMatch: string | null = null;
+  let bestDist = Infinity;
+
+  for (let i = 0; i < canonicalLower.length; i++) {
+    const dist = computeEditDistance(lower, canonicalLower[i]);
+    if (dist < bestDist) {
+      bestDist = dist;
+      bestMatch = canonical[i];
+    }
+  }
+
+  const threshold = mode === 'enforce' ? 3 : 2;
+
+  if (bestDist <= threshold && bestMatch !== null) {
+    return bestMatch;
+  }
+
+  // No close match found
+  if (mode === 'enforce') {
+    return null; // drop the tag
+  }
+
+  // 'suggest' mode — keep original tag unchanged (passthrough)
+  return lower;
+}
+
+// ─── Batch tag normalization ─────────────────────────────────────────────────
+
+/**
+ * Normalize a batch of tags against a canonical list.
+ * Filters out nulls (dropped tags). Deduplicates the result.
+ *
+ * @param tags              - Raw tags
+ * @param canonical         - Canonical tag list
+ * @param mode              - Constraint mode
+ * @param metadataPrefixes  - Tags with these prefixes bypass normalization
+ */
+export function normalizeTags(
+  tags: string[],
+  canonical: string[],
+  mode: 'enforce' | 'suggest' | 'off',
+  metadataPrefixes: string[] = ['source:'],
+): string[] {
+  if (mode === 'off') return tags;
+
+  const seen = new Set<string>();
+  const result: string[] = [];
+
+  for (const tag of tags) {
+    // Metadata tags bypass canonical normalization but are still kept
+    if (isMetadataTag(tag, metadataPrefixes)) {
+      if (!seen.has(tag)) {
+        seen.add(tag);
+        result.push(tag);
+      }
+      continue;
+    }
+
+    const normalized = normalizeTag(tag, canonical, mode);
+    if (normalized !== null && !seen.has(normalized)) {
+      seen.add(normalized);
+      result.push(normalized);
+    }
+  }
+
+  return result;
+}

package/src/vault/vault-connect.test.ts

@@ -127,7 +127,7 @@ describe('VaultManager — named connections', () => {
     const results = mgr.search('shared entry');
     const sameResults = results.filter((r) => r.entry.id === 'same-id');
     expect(sameResults.length).toBe(1);
-    // Score should reflect agent weight (1.0) not external (0.7)
+    // Dedup keeps only one result — agent tier wins, score is positive
     expect(sameResults[0].score).toBeGreaterThan(0);
   });
 

package/src/vault/vault-entries.ts

@@ -526,9 +526,11 @@ export function rowToSearchResult(row: Record<string, unknown>): SearchResult {
 
 /** Build FTS5 query from natural language: terms joined with OR for broad matching. */
 export function buildFtsQuery(query: string): string {
+  // Split on whitespace AND punctuation (hyphens, underscores, dots, slashes)
+  // so that "smoke-test-entry" and "snake_case" are treated as multi-term queries.
   const terms = query
     .toLowerCase()
-    .split(/\s+/)
+    .split(/[\s\-_./\\]+/)
     .filter((t) => t.length >= 2)
     .map((t) => t.replace(/[^a-z0-9]/g, ''))
     .filter(Boolean);

package/src/vault/vault.test.ts

@@ -98,13 +98,14 @@ describe('Vault', () => {
 
     it('should find entries matching query', () => {
       const results = vault.search('validation input');
-      expect(results.length).toBeGreaterThan(0);
+      expect(results).toHaveLength(1);
       expect(results[0].entry.id).toBe('search-1');
     });
 
     it('should return scores with results', () => {
       const results = vault.search('caching');
       expect(results[0].score).toBeGreaterThan(0);
+      expect(results[0].entry.id).toBe('search-2');
     });
 
     it('should filter by domain', () => {
@@ -121,6 +122,20 @@ describe('Vault', () => {
       const results = vault.search('xyznonexistent');
       expect(results).toEqual([]);
     });
+
+    it('should find entries by hyphenated query (smoke-test-entry style)', () => {
+      vault.seed([
+        makeEntry({
+          id: 'hyphen-test-1',
+          title: 'smoke-test-entry',
+          description: 'Hyphenated title entry for FTS regression.',
+          domain: 'testing',
+        }),
+      ]);
+      const results = vault.search('smoke-test-entry');
+      expect(results.length).toBeGreaterThan(0);
+      expect(results[0].entry.id).toBe('hyphen-test-1');
+    });
   });
 
   describe('get', () => {
@@ -326,7 +341,7 @@ describe('Vault', () => {
         toolsUsed: [],
       });
       expect(memory.type).toBe('session');
-      expect(memory.createdAt).toBeGreaterThan(0);
+      expect(typeof memory.createdAt).toBe('number');
     });
 
     it('should capture preference memories', () => {
@@ -376,7 +391,7 @@ describe('Vault', () => {
 
     it('should find memories matching query', () => {
       const results = vault.searchMemories('parameterized queries');
-      expect(results.length).toBeGreaterThan(0);
+      expect(results).toHaveLength(1);
       expect(results[0].summary).toContain('parameterized');
     });
 
@@ -578,7 +593,7 @@ describe('Vault', () => {
 
       // Before archive: should appear in search
       const before = v.search('searchable');
-      expect(before.length).toBeGreaterThan(0);
+      expect(before).toHaveLength(1);
 
       v.archive({ olderThanDays: 90 });
 
@@ -643,7 +658,7 @@ describe('Vault', () => {
       v.restore('search-restore');
 
       const results = v.search('unique findable');
-      expect(results.length).toBeGreaterThan(0);
+      expect(results).toHaveLength(1);
       v.close();
     });
 
@@ -726,9 +741,9 @@ describe('Vault', () => {
         },
       ]);
       const stats = vault.contentHashStats();
-      expect(stats.total).toBeGreaterThanOrEqual(2);
-      expect(stats.hashed).toBe(stats.total);
-      expect(stats.uniqueHashes).toBe(stats.total);
+      expect(stats.total).toBe(2);
+      expect(stats.hashed).toBe(2);
+      expect(stats.uniqueHashes).toBe(2);
     });
 
     it('backfill hashes existing entries on re-initialize', () => {

package/dist/embeddings/index.d.ts

@@ -1,5 +0,0 @@
-export type { EmbeddingProvider, EmbeddingResult, EmbeddingConfig, StoredVector, EmbeddingStats, } from './types.js';
-export { OpenAIEmbeddingProvider } from './openai-provider.js';
-export { EmbeddingPipeline } from './pipeline.js';
-export type { BatchEmbedOptions, BatchEmbedResult } from './pipeline.js';
-//# sourceMappingURL=index.d.ts.map

package/dist/embeddings/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/embeddings/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,iBAAiB,EACjB,eAAe,EACf,eAAe,EACf,YAAY,EACZ,cAAc,GACf,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,YAAY,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC"}

package/dist/embeddings/index.js

@@ -1,3 +0,0 @@
-export { OpenAIEmbeddingProvider } from './openai-provider.js';
-export { EmbeddingPipeline } from './pipeline.js';
-//# sourceMappingURL=index.js.map

package/dist/embeddings/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/embeddings/index.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC"}