@soleri/core 9.14.4 → 9.15.0

package/src/vault/tag-normalizer.test.ts

@@ -0,0 +1,214 @@
+import { describe, it, expect } from 'vitest';
+import {
+  computeEditDistance,
+  normalizeTag,
+  normalizeTags,
+  isMetadataTag,
+} from './tag-normalizer.js';
+
+// ─── computeEditDistance ────────────────────────────────────────────────────
+
+describe('computeEditDistance', () => {
+  it('returns 0 for identical strings', () => {
+    expect(computeEditDistance('workflow', 'workflow')).toBe(0);
+  });
+
+  it('returns length of b for empty a', () => {
+    expect(computeEditDistance('', 'abc')).toBe(3);
+  });
+
+  it('returns length of a for empty b', () => {
+    expect(computeEditDistance('abc', '')).toBe(3);
+  });
+
+  it('single insertion: workflow → workflows', () => {
+    expect(computeEditDistance('workflow', 'workflows')).toBe(1);
+  });
+
+  it('single deletion: testing → testin', () => {
+    expect(computeEditDistance('testing', 'testin')).toBe(1);
+  });
+
+  it('single substitution: arch → arcs', () => {
+    expect(computeEditDistance('arch', 'arcs')).toBe(1);
+  });
+
+  it("distance 1: architcture → architecture (single insertion - missing 'e')", () => {
+    expect(computeEditDistance('architcture', 'architecture')).toBe(1);
+  });
+
+  it('distance 2: archtecrure → architecture (two edits)', () => {
+    expect(computeEditDistance('archtecrure', 'architecture')).toBe(2);
+  });
+
+  it('large distance: typescript → javascript', () => {
+    expect(computeEditDistance('typescript', 'javascript')).toBeGreaterThan(3);
+  });
+
+  it('is symmetric', () => {
+    expect(computeEditDistance('foo', 'bar')).toBe(computeEditDistance('bar', 'foo'));
+  });
+});
+
+// ─── isMetadataTag ──────────────────────────────────────────────────────────
+
+describe('isMetadataTag', () => {
+  it('returns true when tag matches a prefix', () => {
+    expect(isMetadataTag('source:article', ['source:'])).toBe(true);
+  });
+
+  it('returns true for exact prefix match', () => {
+    expect(isMetadataTag('source:ingested', ['source:'])).toBe(true);
+  });
+
+  it('returns false when no prefix matches', () => {
+    expect(isMetadataTag('typescript', ['source:'])).toBe(false);
+  });
+
+  it('returns false with empty prefix list', () => {
+    expect(isMetadataTag('source:article', [])).toBe(false);
+  });
+
+  it('supports multiple prefixes', () => {
+    expect(isMetadataTag('meta:foo', ['source:', 'meta:'])).toBe(true);
+  });
+});
+
+// ─── normalizeTag ───────────────────────────────────────────────────────────
+
+const CANONICAL = ['architecture', 'typescript', 'workflow', 'testing', 'performance'];
+
+describe('normalizeTag — mode: off', () => {
+  it('returns tag as-is regardless of canonical list', () => {
+    expect(normalizeTag('workflows', CANONICAL, 'off')).toBe('workflows');
+  });
+
+  it('passes through noise words in off mode', () => {
+    expect(normalizeTag('new', CANONICAL, 'off')).toBe('new');
+  });
+});
+
+describe('normalizeTag — noise stripping', () => {
+  it('drops version strings (v1.2)', () => {
+    expect(normalizeTag('v1.2', CANONICAL, 'suggest')).toBeNull();
+  });
+
+  it('drops version strings (v10)', () => {
+    expect(normalizeTag('v10', CANONICAL, 'suggest')).toBeNull();
+  });
+
+  it('drops generic noise word: new', () => {
+    expect(normalizeTag('new', CANONICAL, 'suggest')).toBeNull();
+  });
+
+  it('drops generic noise word: via', () => {
+    expect(normalizeTag('via', CANONICAL, 'suggest')).toBeNull();
+  });
+
+  it('drops generic noise word: raw', () => {
+    expect(normalizeTag('raw', CANONICAL, 'enforce')).toBeNull();
+  });
+});
+
+describe('normalizeTag — mode: suggest', () => {
+  it('returns canonical for exact match', () => {
+    expect(normalizeTag('typescript', CANONICAL, 'suggest')).toBe('typescript');
+  });
+
+  it('maps within edit-distance 2 to canonical', () => {
+    // 'workflows' is distance 1 from 'workflow'
+    expect(normalizeTag('workflows', CANONICAL, 'suggest')).toBe('workflow');
+  });
+
+  it('lowercases tag before matching', () => {
+    expect(normalizeTag('TypeScript', CANONICAL, 'suggest')).toBe('typescript');
+  });
+
+  it('passes through unknown tag with no close canonical (suggest passthrough)', () => {
+    const result = normalizeTag('gamification', CANONICAL, 'suggest');
+    // No match within distance 2 — passthrough
+    expect(result).toBe('gamification');
+  });
+
+  it('returns null for noise even in suggest mode', () => {
+    expect(normalizeTag('one', CANONICAL, 'suggest')).toBeNull();
+  });
+});
+
+describe('normalizeTag — mode: enforce', () => {
+  it('returns canonical for exact match', () => {
+    expect(normalizeTag('testing', CANONICAL, 'enforce')).toBe('testing');
+  });
+
+  it('maps within edit-distance 3 to canonical', () => {
+    // 'archtecrure' is 2 away from 'architecture'
+    expect(normalizeTag('archtecrure', CANONICAL, 'enforce')).toBe('architecture');
+  });
+
+  it('returns null for tag with no match within distance 3', () => {
+    // 'gamification' is far from all CANONICAL entries
+    expect(normalizeTag('gamification', CANONICAL, 'enforce')).toBeNull();
+  });
+
+  it('returns null for noise words', () => {
+    expect(normalizeTag('full', CANONICAL, 'enforce')).toBeNull();
+  });
+});
+
+describe('normalizeTag — empty canonical list', () => {
+  it('suggest mode: passes through non-noise tags', () => {
+    expect(normalizeTag('react', [], 'suggest')).toBe('react');
+  });
+
+  it('enforce mode: drops all tags (no canonical to match)', () => {
+    expect(normalizeTag('react', [], 'enforce')).toBeNull();
+  });
+});
+
+// ─── normalizeTags ──────────────────────────────────────────────────────────
+
+describe('normalizeTags', () => {
+  it('deduplicates tags that map to the same canonical', () => {
+    // Both 'workflows' and 'workflow' normalize to 'workflow'
+    const result = normalizeTags(['workflows', 'workflow'], CANONICAL, 'suggest');
+    expect(result).toEqual(['workflow']);
+  });
+
+  it('preserves metadata tags unchanged', () => {
+    const result = normalizeTags(['source:article', 'typescript'], CANONICAL, 'enforce', [
+      'source:',
+    ]);
+    expect(result).toContain('source:article');
+    expect(result).toContain('typescript');
+  });
+
+  it('metadata tags bypass canonical normalization in enforce mode', () => {
+    // 'source:mytype' does not match any canonical — but it should be kept
+    const result = normalizeTags(['source:mytype'], CANONICAL, 'enforce', ['source:']);
+    expect(result).toEqual(['source:mytype']);
+  });
+
+  it('returns empty array when all tags are noise', () => {
+    const result = normalizeTags(['new', 'via', 'raw', 'v1.2'], CANONICAL, 'suggest');
+    expect(result).toEqual([]);
+  });
+
+  it('in off mode, returns tags unchanged', () => {
+    const input = ['new', 'workflows', 'v1.2'];
+    const result = normalizeTags(input, CANONICAL, 'off');
+    expect(result).toEqual(input);
+  });
+
+  it('batch normalizes a mixed tag list', () => {
+    const result = normalizeTags(
+      ['TypeScript', 'workflows', 'new', 'source:article'],
+      CANONICAL,
+      'suggest',
+      ['source:'],
+    );
+    expect(result).toContain('typescript');
+    expect(result).toContain('workflow');
+    expect(result).toContain('source:article');
+    expect(result).not.toContain('new');
+  });
+});

package/src/vault/tag-normalizer.ts

@@ -0,0 +1,188 @@
+/**
+ * Tag Normalizer — canonical tag taxonomy enforcement.
+ *
+ * Maps raw tags to nearest canonical via Levenshtein edit-distance.
+ * Strips noise words (version strings, single generic words).
+ * Respects metadata tag prefixes (e.g. 'source:').
+ *
+ * Three modes:
+ *   - 'enforce': must match within edit-distance 3, else drop (return null)
+ *   - 'suggest': map to nearest canonical within edit-distance 2 (passthrough if no match)
+ *   - 'off':     no normalization — return tag as-is
+ */
+
+// ─── Noise filter ────────────────────────────────────────────────────────────
+
+/**
+ * Version-string pattern: v1.2, v10, v1.2.3, etc.
+ */
+const VERSION_PATTERN = /^v\d+(\.\d+)*/i;
+
+/**
+ * Generic single words that add no signal.
+ */
+const NOISE_WORDS = new Set([
+  'one',
+  'via',
+  'new',
+  'full',
+  'actual',
+  'raw',
+  'the',
+  'and',
+  'for',
+  'with',
+  'this',
+  'that',
+  'from',
+  'into',
+]);
+
+function isNoisy(tag: string): boolean {
+  if (VERSION_PATTERN.test(tag)) return true;
+  if (NOISE_WORDS.has(tag.toLowerCase())) return true;
+  return false;
+}
+
+// ─── Levenshtein edit distance ───────────────────────────────────────────────
+
+/**
+ * Compute Levenshtein edit distance between two strings.
+ * O(m*n) time, O(n) space using two-row DP.
+ */
+export function computeEditDistance(a: string, b: string): number {
+  if (a === b) return 0;
+  if (a.length === 0) return b.length;
+  if (b.length === 0) return a.length;
+
+  let prev = Array.from({ length: b.length + 1 }, (_, i) => i);
+  let curr = Array.from<number>({ length: b.length + 1 });
+
+  for (let i = 1; i <= a.length; i++) {
+    curr[0] = i;
+    for (let j = 1; j <= b.length; j++) {
+      const cost = a[i - 1] === b[j - 1] ? 0 : 1;
+      curr[j] = Math.min(
+        curr[j - 1] + 1, // insertion
+        prev[j] + 1, // deletion
+        prev[j - 1] + cost, // substitution
+      );
+    }
+    [prev, curr] = [curr, prev];
+  }
+
+  return prev[b.length];
+}
+
+// ─── Metadata tag check ──────────────────────────────────────────────────────
+
+/**
+ * Returns true if the tag starts with any of the given prefixes.
+ * Metadata tags (e.g. 'source:article') are exempt from canonical normalization.
+ */
+export function isMetadataTag(tag: string, prefixes: string[]): boolean {
+  return prefixes.some((prefix) => tag.startsWith(prefix));
+}
+
+// ─── Single tag normalization ────────────────────────────────────────────────
+
+/**
+ * Normalize a single tag against a canonical list.
+ *
+ * @param tag       - Raw tag to normalize
+ * @param canonical - Canonical tag list to map against
+ * @param mode      - Constraint mode: 'enforce' | 'suggest' | 'off'
+ * @returns Normalized tag string, or null if the tag should be dropped.
+ */
+export function normalizeTag(
+  tag: string,
+  canonical: string[],
+  mode: 'enforce' | 'suggest' | 'off',
+): string | null {
+  if (mode === 'off') return tag;
+
+  const lower = tag.toLowerCase().trim();
+
+  // Always drop noise words
+  if (isNoisy(lower)) return null;
+
+  // Derive lowercase canonical for matching; preserve original casing for return
+  const canonicalLower = canonical.map((x) => x.toLowerCase());
+
+  // Exact match in canonical list — return canonical form (original casing)
+  const exactIdx = canonicalLower.indexOf(lower);
+  if (exactIdx !== -1) return canonical[exactIdx];
+
+  if (canonical.length === 0) {
+    // No canonical list configured — pass through in suggest, drop in enforce
+    return mode === 'enforce' ? null : lower;
+  }
+
+  // Find nearest canonical by edit distance
+  let bestMatch: string | null = null;
+  let bestDist = Infinity;
+
+  for (let i = 0; i < canonicalLower.length; i++) {
+    const dist = computeEditDistance(lower, canonicalLower[i]);
+    if (dist < bestDist) {
+      bestDist = dist;
+      bestMatch = canonical[i];
+    }
+  }
+
+  const threshold = mode === 'enforce' ? 3 : 2;
+
+  if (bestDist <= threshold && bestMatch !== null) {
+    return bestMatch;
+  }
+
+  // No close match found
+  if (mode === 'enforce') {
+    return null; // drop the tag
+  }
+
+  // 'suggest' mode — keep original tag unchanged (passthrough)
+  return lower;
+}
+
+// ─── Batch tag normalization ─────────────────────────────────────────────────
+
+/**
+ * Normalize a batch of tags against a canonical list.
+ * Filters out nulls (dropped tags). Deduplicates the result.
+ *
+ * @param tags              - Raw tags
+ * @param canonical         - Canonical tag list
+ * @param mode              - Constraint mode
+ * @param metadataPrefixes  - Tags with these prefixes bypass normalization
+ */
+export function normalizeTags(
+  tags: string[],
+  canonical: string[],
+  mode: 'enforce' | 'suggest' | 'off',
+  metadataPrefixes: string[] = ['source:'],
+): string[] {
+  if (mode === 'off') return tags;
+
+  const seen = new Set<string>();
+  const result: string[] = [];
+
+  for (const tag of tags) {
+    // Metadata tags bypass canonical normalization but are still kept
+    if (isMetadataTag(tag, metadataPrefixes)) {
+      if (!seen.has(tag)) {
+        seen.add(tag);
+        result.push(tag);
+      }
+      continue;
+    }
+
+    const normalized = normalizeTag(tag, canonical, mode);
+    if (normalized !== null && !seen.has(normalized)) {
+      seen.add(normalized);
+      result.push(normalized);
+    }
+  }
+
+  return result;
+}

package/dist/embeddings/index.d.ts

@@ -1,5 +0,0 @@
-export type { EmbeddingProvider, EmbeddingResult, EmbeddingConfig, StoredVector, EmbeddingStats, } from './types.js';
-export { OpenAIEmbeddingProvider } from './openai-provider.js';
-export { EmbeddingPipeline } from './pipeline.js';
-export type { BatchEmbedOptions, BatchEmbedResult } from './pipeline.js';
-//# sourceMappingURL=index.d.ts.map

package/dist/embeddings/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/embeddings/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,iBAAiB,EACjB,eAAe,EACf,eAAe,EACf,YAAY,EACZ,cAAc,GACf,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,YAAY,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC"}

package/dist/embeddings/index.js

@@ -1,3 +0,0 @@
-export { OpenAIEmbeddingProvider } from './openai-provider.js';
-export { EmbeddingPipeline } from './pipeline.js';
-//# sourceMappingURL=index.js.map

package/dist/embeddings/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/embeddings/index.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC"}