@od-oneapp/ai-platform 0.1.0

package/src/grounding/sources/index.ts

@@ -0,0 +1,943 @@
+/**
+ * Source Management
+ *
+ * Utilities for managing, validating, deduplicating, and filtering grounding sources.
+ * Provides a registry for source lifecycle management and helper functions for
+ * common source operations used across citation, attribution, and verification.
+ *
+ * @module @od-oneapp/ai-platform/grounding/sources
+ *
+ * @example
+ * ```typescript
+ * import {
+ *   createSourceRegistry,
+ *   validateSource,
+ *   deduplicateSources,
+ *   filterByScore,
+ * } from '@od-oneapp/ai-platform/grounding/sources';
+ *
+ * // Create a registry
+ * const registry = createSourceRegistry();
+ * registry.add({ id: 'doc-1', content: 'Climate data...', score: 0.9 });
+ *
+ * // Validate sources
+ * const { valid, errors } = validateSources(sources);
+ *
+ * // Filter and deduplicate
+ * const unique = deduplicateSources(sources, { strategy: 'content' });
+ * const topSources = filterByScore(unique, { minScore: 0.7, limit: 5 });
+ * ```
+ */
+
+import type { GroundingSource } from '../types';
+
+// -----------------------------------------------------------------------------
+// Types
+// -----------------------------------------------------------------------------
+
+/** Extended source with provenance metadata */
+export interface SourceWithProvenance extends GroundingSource {
+  /** When the source was added */
+  addedAt?: Date;
+  /** When the source was last accessed */
+  lastAccessedAt?: Date;
+  /** Number of times this source was used */
+  useCount?: number;
+  /** Reliability score (0-1, separate from relevance) */
+  reliability?: number;
+  /** Source type classification */
+  sourceType?: 'document' | 'webpage' | 'database' | 'api' | 'user' | 'unknown';
+}
+
+/** Validation result for a single source */
+export interface SourceValidationResult {
+  valid: boolean;
+  sourceId: string;
+  errors: string[];
+  warnings: string[];
+}
+
+/** Validation options */
+export interface ValidationOptions {
+  /** Require URL field */
+  requireUrl?: boolean;
+  /** Require title field */
+  requireTitle?: boolean;
+  /** Minimum content length */
+  minContentLength?: number;
+  /** Maximum content length */
+  maxContentLength?: number;
+  /** Require score field */
+  requireScore?: boolean;
+}
+
+/** Deduplication strategy */
+export type DeduplicationStrategy = 'id' | 'url' | 'content' | 'hybrid';
+
+/** Deduplication options */
+export interface DeduplicationOptions {
+  /** Strategy for detecting duplicates */
+  strategy?: DeduplicationStrategy;
+  /** Similarity threshold for content-based dedup (0-1) */
+  similarityThreshold?: number;
+  /** Keep first or highest-scored duplicate */
+  keepStrategy?: 'first' | 'highest-score';
+}
+
+/** Filter options */
+export interface FilterOptions {
+  /** Minimum relevance score */
+  minScore?: number;
+  /** Maximum relevance score */
+  maxScore?: number;
+  /** Minimum reliability score */
+  minReliability?: number;
+  /** Limit number of results */
+  limit?: number;
+  /** Required metadata fields */
+  requiredMetadata?: string[];
+  /** Source types to include */
+  sourceTypes?: SourceWithProvenance['sourceType'][];
+}
+
+/** Source registry interface */
+export interface SourceRegistry {
+  /** Add a source to the registry */
+  add(source: GroundingSource): void;
+  /** Add multiple sources */
+  addMany(sources: GroundingSource[]): void;
+  /** Get a source by ID */
+  get(id: string): SourceWithProvenance | undefined;
+  /** Get all sources */
+  getAll(): SourceWithProvenance[];
+  /** Remove a source by ID */
+  remove(id: string): boolean;
+  /** Clear all sources */
+  clear(): void;
+  /** Check if source exists */
+  has(id: string): boolean;
+  /** Get source count */
+  size(): number;
+  /** Search sources by content similarity */
+  search(query: string, limit?: number): SourceWithProvenance[];
+  /** Record a source usage */
+  recordUsage(id: string): void;
+}
+
+/** Registry configuration */
+export interface SourceRegistryConfig {
+  /** Maximum sources to store */
+  maxSources?: number;
+  /** Auto-deduplicate on add */
+  autoDeduplicate?: boolean;
+  /** Deduplication strategy */
+  deduplicationStrategy?: DeduplicationStrategy;
+}
+
+// -----------------------------------------------------------------------------
+// Source Registry Implementation
+// -----------------------------------------------------------------------------
+
+/**
+ * Creates a source registry for managing grounding sources.
+ *
+ * @example
+ * ```typescript
+ * const registry = createSourceRegistry({ maxSources: 1000 });
+ *
+ * registry.add({
+ *   id: 'wiki-1',
+ *   content: 'Climate change refers to...',
+ *   url: 'https://example.com/climate',
+ *   score: 0.85
+ * });
+ *
+ * const source = registry.get('wiki-1');
+ * console.log(source?.useCount); // 0
+ *
+ * registry.recordUsage('wiki-1');
+ * console.log(registry.get('wiki-1')?.useCount); // 1
+ * ```
+ */
+export function createSourceRegistry(config: SourceRegistryConfig = {}): SourceRegistry {
+  const { maxSources = 10000, autoDeduplicate = false, deduplicationStrategy = 'id' } = config;
+
+  const sources = new Map<string, SourceWithProvenance>();
+
+  function add(source: GroundingSource): void {
+    if (sources.size >= maxSources) {
+      // Evict the oldest entry by insertion order (Map preserves insertion order).
+      const oldestId = sources.keys().next().value;
+      if (oldestId) sources.delete(oldestId);
+    }
+
+    // Check for duplicates if auto-deduplicate is enabled
+    if (autoDeduplicate) {
+      const isDuplicate = checkDuplicate(
+        source,
+        Array.from(sources.values()),
+        deduplicationStrategy,
+      );
+      if (isDuplicate) return;
+    }
+
+    const enhanced: SourceWithProvenance = {
+      ...source,
+      addedAt: new Date(),
+      lastAccessedAt: new Date(),
+      useCount: 0,
+      sourceType: inferSourceType(source),
+    };
+
+    sources.set(source.id, enhanced);
+  }
+
+  function addMany(newSources: GroundingSource[]): void {
+    for (const source of newSources) {
+      add(source);
+    }
+  }
+
+  function get(id: string): SourceWithProvenance | undefined {
+    const source = sources.get(id);
+    if (source) {
+      source.lastAccessedAt = new Date();
+    }
+    return source;
+  }
+
+  function getAll(): SourceWithProvenance[] {
+    return Array.from(sources.values());
+  }
+
+  function remove(id: string): boolean {
+    return sources.delete(id);
+  }
+
+  function clear(): void {
+    sources.clear();
+  }
+
+  function has(id: string): boolean {
+    return sources.has(id);
+  }
+
+  function size(): number {
+    return sources.size;
+  }
+
+  function search(query: string, limit = 10): SourceWithProvenance[] {
+    const results: Array<{ source: SourceWithProvenance; similarity: number }> = [];
+
+    for (const source of sources.values()) {
+      const similarity = computeTextSimilarity(query, source.content);
+      if (similarity > 0.1) {
+        results.push({ source, similarity });
+      }
+    }
+
+    return results
+      .sort((a, b) => b.similarity - a.similarity)
+      .slice(0, limit)
+      .map(r => r.source);
+  }
+
+  function recordUsage(id: string): void {
+    const source = sources.get(id);
+    if (source) {
+      source.useCount = (source.useCount ?? 0) + 1;
+      source.lastAccessedAt = new Date();
+    }
+  }
+
+  return {
+    add,
+    addMany,
+    get,
+    getAll,
+    remove,
+    clear,
+    has,
+    size,
+    search,
+    recordUsage,
+  };
+}
+
+// -----------------------------------------------------------------------------
+// Validation Functions
+// -----------------------------------------------------------------------------
+
+/**
+ * Validate a single grounding source.
+ *
+ * @example
+ * ```typescript
+ * const result = validateSource(source, { requireUrl: true, minContentLength: 50 });
+ * if (!result.valid) {
+ *   console.error('Validation errors:', result.errors);
+ * }
+ * ```
+ */
+export function validateSource(
+  source: GroundingSource,
+  options: ValidationOptions = {},
+): SourceValidationResult {
+  const {
+    requireUrl = false,
+    requireTitle = false,
+    minContentLength = 1,
+    maxContentLength = 100000,
+    requireScore = false,
+  } = options;
+
+  const errors: string[] = [];
+  const warnings: string[] = [];
+
+  // Required fields
+  if (!source.id || typeof source.id !== 'string') {
+    errors.push('Source must have a valid string id');
+  }
+
+  if (!source.content || typeof source.content !== 'string') {
+    errors.push('Source must have valid string content');
+  } else {
+    if (source.content.length < minContentLength) {
+      errors.push(`Content must be at least ${minContentLength} characters`);
+    }
+    if (source.content.length > maxContentLength) {
+      errors.push(`Content must not exceed ${maxContentLength} characters`);
+    }
+  }
+
+  // Optional required fields
+  if (requireUrl && !source.url) {
+    errors.push('URL is required');
+  }
+
+  if (requireTitle && !source.title) {
+    errors.push('Title is required');
+  }
+
+  if (requireScore && (source.score === undefined || source.score === null)) {
+    errors.push('Score is required');
+  }
+
+  // Score range validation
+  if (source.score !== undefined && (source.score < 0 || source.score > 1)) {
+    warnings.push('score should be between 0 and 1');
+  }
+
+  // URL format validation
+  if (source.url && !isValidUrl(source.url)) {
+    warnings.push('URL appears to be invalid');
+  }
+
+  return {
+    valid: errors.length === 0,
+    sourceId: source.id ?? 'unknown',
+    errors,
+    warnings,
+  };
+}
+
+/**
+ * Validate multiple sources.
+ *
+ * @example
+ * ```typescript
+ * const { valid, invalid, results } = validateSources(sources);
+ * console.log(`${valid.length} valid, ${invalid.length} invalid`);
+ * ```
+ */
+export function validateSources(
+  sources: GroundingSource[],
+  options: ValidationOptions = {},
+): {
+  valid: GroundingSource[];
+  invalid: GroundingSource[];
+  results: SourceValidationResult[];
+} {
+  const results = sources.map(s => validateSource(s, options));
+  const valid: GroundingSource[] = [];
+  const invalid: GroundingSource[] = [];
+
+  for (let i = 0; i < sources.length; i++) {
+    const result = results[i];
+    const source = sources[i];
+    if (!result || !source) continue;
+    if (result.valid) {
+      valid.push(source);
+    } else {
+      invalid.push(source);
+    }
+  }
+
+  return { valid, invalid, results };
+}
+
+// -----------------------------------------------------------------------------
+// Deduplication Functions
+// -----------------------------------------------------------------------------
+
+/**
+ * Remove duplicate sources based on the specified strategy.
+ *
+ * @example
+ * ```typescript
+ * // By ID (exact match)
+ * const unique = deduplicateSources(sources, { strategy: 'id' });
+ *
+ * // By content similarity
+ * const unique = deduplicateSources(sources, {
+ *   strategy: 'content',
+ *   similarityThreshold: 0.9,
+ *   keepStrategy: 'highest-score'
+ * });
+ * ```
+ */
+export function deduplicateSources(
+  sources: GroundingSource[],
+  options: DeduplicationOptions = {},
+): GroundingSource[] {
+  const { strategy = 'id', similarityThreshold = 0.9, keepStrategy = 'first' } = options;
+
+  if (sources.length === 0) return [];
+
+  switch (strategy) {
+    case 'id':
+      return deduplicateById(sources, keepStrategy);
+
+    case 'url':
+      return deduplicateByUrl(sources, keepStrategy);
+
+    case 'content':
+      return deduplicateByContent(sources, similarityThreshold, keepStrategy);
+
+    case 'hybrid':
+      // First by URL, then by content
+      const byUrl = deduplicateByUrl(sources, keepStrategy);
+      return deduplicateByContent(byUrl, similarityThreshold, keepStrategy);
+
+    default:
+      return sources;
+  }
+}
+
+function deduplicateById(
+  sources: GroundingSource[],
+  keepStrategy: 'first' | 'highest-score',
+): GroundingSource[] {
+  const seen = new Map<string, GroundingSource>();
+
+  for (const source of sources) {
+    const existing = seen.get(source.id);
+    if (!existing) {
+      seen.set(source.id, source);
+    } else if (keepStrategy === 'highest-score') {
+      if ((source.score ?? 0) > (existing.score ?? 0)) {
+        seen.set(source.id, source);
+      }
+    }
+  }
+
+  return Array.from(seen.values());
+}
+
+function deduplicateByUrl(
+  sources: GroundingSource[],
+  keepStrategy: 'first' | 'highest-score',
+): GroundingSource[] {
+  const seen = new Map<string, GroundingSource>();
+  const noUrl: GroundingSource[] = [];
+
+  for (const source of sources) {
+    if (!source.url) {
+      noUrl.push(source);
+      continue;
+    }
+
+    const normalizedUrl = normalizeUrl(source.url);
+    const existing = seen.get(normalizedUrl);
+
+    if (!existing) {
+      seen.set(normalizedUrl, source);
+    } else if (keepStrategy === 'highest-score') {
+      if ((source.score ?? 0) > (existing.score ?? 0)) {
+        seen.set(normalizedUrl, source);
+      }
+    }
+  }
+
+  return [...Array.from(seen.values()), ...noUrl];
+}
+
+function deduplicateByContent(
+  sources: GroundingSource[],
+  threshold: number,
+  keepStrategy: 'first' | 'highest-score',
+): GroundingSource[] {
+  const result: GroundingSource[] = [];
+
+  for (const source of sources) {
+    let isDuplicate = false;
+    let duplicateIndex = -1;
+
+    for (let i = 0; i < result.length; i++) {
+      const existing = result[i];
+      if (!existing) continue;
+      const similarity = computeTextSimilarity(source.content, existing.content);
+      if (similarity >= threshold) {
+        isDuplicate = true;
+        duplicateIndex = i;
+        break;
+      }
+    }
+
+    if (!isDuplicate) {
+      result.push(source);
+    } else if (keepStrategy === 'highest-score' && duplicateIndex >= 0) {
+      const current = result[duplicateIndex];
+      if (current && (source.score ?? 0) > (current.score ?? 0)) {
+        result[duplicateIndex] = source;
+      }
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Merge duplicate sources, combining their metadata.
+ *
+ * @example
+ * ```typescript
+ * const merged = mergeSources(sources, { strategy: 'url' });
+ * ```
+ */
+export function mergeSources(
+  sources: GroundingSource[],
+  options: { strategy?: DeduplicationStrategy } = {},
+): GroundingSource[] {
+  const { strategy = 'id' } = options;
+  const groups = new Map<string, GroundingSource[]>();
+
+  for (const source of sources) {
+    let key: string;
+    switch (strategy) {
+      case 'url':
+        key = source.url ? normalizeUrl(source.url) : source.id;
+        break;
+      case 'content':
+        key = hashContent(source.content);
+        break;
+      case 'hybrid': {
+        const base = source.url ? normalizeUrl(source.url) : source.id;
+        key = `${base}:${hashContent(source.content)}`;
+        break;
+      }
+      case 'id':
+      default:
+        key = source.id;
+    }
+
+    const group = groups.get(key) ?? [];
+    group.push(source);
+    groups.set(key, group);
+  }
+
+  return Array.from(groups.values())
+    .filter(group => group.length > 0)
+    .map(group => {
+      const first = group[0] as GroundingSource;
+      if (group.length === 1) return first;
+
+      // Merge group - highest score, combined metadata
+      const merged = group.reduce((acc, s) => {
+        return {
+          ...acc,
+          score: Math.max(acc.score ?? 0, s.score ?? 0),
+          metadata: { ...acc.metadata, ...s.metadata },
+        };
+      }, first);
+
+      return merged;
+    });
+}
+
+// -----------------------------------------------------------------------------
+// Filtering Functions
+// -----------------------------------------------------------------------------
+
+/**
+ * Filter sources by relevance score.
+ *
+ * @example
+ * ```typescript
+ * const topSources = filterByScore(sources, { minScore: 0.7, limit: 5 });
+ * ```
+ */
+export function filterByScore(
+  sources: GroundingSource[],
+  options: { minScore?: number; maxScore?: number; limit?: number } = {},
+): GroundingSource[] {
+  const { minScore = 0, maxScore = 1, limit } = options;
+
+  let filtered = sources.filter(s => {
+    const score = s.score ?? 0;
+    return score >= minScore && score <= maxScore;
+  });
+
+  // Sort by score descending
+  filtered.sort((a, b) => (b.score ?? 0) - (a.score ?? 0));
+
+  if (limit && limit > 0) {
+    filtered = filtered.slice(0, limit);
+  }
+
+  return filtered;
+}
+
+/**
+ * Filter sources by metadata fields.
+ *
+ * @example
+ * ```typescript
+ * const filtered = filterByMetadata(sources, {
+ *   requiredFields: ['author', 'date'],
+ *   matchValues: { category: 'science' }
+ * });
+ * ```
+ */
+export function filterByMetadata(
+  sources: GroundingSource[],
+  options: {
+    requiredFields?: string[];
+    matchValues?: Record<string, unknown>;
+  } = {},
+): GroundingSource[] {
+  const { requiredFields = [], matchValues = {} } = options;
+
+  return sources.filter(source => {
+    const metadata = source.metadata ?? {};
+
+    // Check required fields
+    for (const field of requiredFields) {
+      if (!(field in metadata)) return false;
+    }
+
+    // Check value matches
+    for (const [key, value] of Object.entries(matchValues)) {
+      if (metadata[key] !== value) return false;
+    }
+
+    return true;
+  });
+}
+
+/**
+ * Select top N sources, optionally with diversity.
+ *
+ * @example
+ * ```typescript
+ * const top = selectTopSources(sources, 5, { diversify: true });
+ * ```
+ */
+export function selectTopSources(
+  sources: GroundingSource[],
+  limit: number,
+  options: { diversify?: boolean; diversityThreshold?: number } = {},
+): GroundingSource[] {
+  const { diversify = false, diversityThreshold = 0.7 } = options;
+
+  // Sort by score
+  const sorted = [...sources].sort((a, b) => (b.score ?? 0) - (a.score ?? 0));
+
+  if (!diversify) {
+    return sorted.slice(0, limit);
+  }
+
+  // Maximal marginal relevance-style selection
+  const selected: GroundingSource[] = [];
+
+  for (const candidate of sorted) {
+    if (selected.length >= limit) break;
+
+    // Check if too similar to already selected
+    const tooSimilar = selected.some(
+      s => computeTextSimilarity(candidate.content, s.content) > diversityThreshold,
+    );
+
+    if (!tooSimilar) {
+      selected.push(candidate);
+    }
+  }
+
+  return selected;
+}
+
+/**
+ * Group sources by a metadata field or custom function.
+ *
+ * @example
+ * ```typescript
+ * const byCategory = groupSources(sources, s => s.metadata?.category ?? 'other');
+ * ```
+ */
+export function groupSources<K extends string>(
+  sources: GroundingSource[],
+  keyFn: (source: GroundingSource) => K,
+): Map<K, GroundingSource[]> {
+  const groups = new Map<K, GroundingSource[]>();
+
+  for (const source of sources) {
+    const key = keyFn(source);
+    const group = groups.get(key) ?? [];
+    group.push(source);
+    groups.set(key, group);
+  }
+
+  return groups;
+}
+
+// -----------------------------------------------------------------------------
+// Formatting Functions
+// -----------------------------------------------------------------------------
+
+/**
+ * Format sources as a bibliography string.
+ *
+ * @example
+ * ```typescript
+ * const bib = formatAsBibliography(sources, { style: 'apa' });
+ * console.log(bib);
+ * // [1] Author. Title. URL
+ * // [2] Author. Title. URL
+ * ```
+ */
+export function formatAsBibliography(
+  sources: GroundingSource[],
+  options: { style?: 'simple' | 'apa' | 'numbered' } = {},
+): string {
+  const { style = 'numbered' } = options;
+
+  return sources
+    .map((source, index) => {
+      const title = source.title ?? 'Untitled';
+      const url = source.url ?? '';
+
+      switch (style) {
+        case 'apa':
+          const author = (source.metadata?.author as string) ?? 'Unknown';
+          const year = (source.metadata?.year as string) ?? 'n.d.';
+          return `${author} (${year}). ${title}. ${url}`;
+
+        case 'simple':
+          return `${title}${url ? ` - ${url}` : ''}`;
+
+        case 'numbered':
+        default: {
+          const author = source.metadata?.author;
+          const authorPrefix =
+            typeof author === 'string' && author.trim().length > 0 ? `${author.trim()}. ` : '';
+          return `[${index + 1}] ${authorPrefix}${title}${url ? `. ${url}` : ''}`;
+        }
+      }
+    })
+    .join('\n');
+}
+
+/**
+ * Format sources as inline citations.
+ *
+ * @example
+ * ```typescript
+ * const citations = formatAsInlineCitations(sources);
+ * // Returns: "[1], [2], [3]"
+ * ```
+ */
+export function formatAsInlineCitations(
+  sources: GroundingSource[],
+  options: { separator?: string; format?: 'numbered' | 'id' | 'title' } = {},
+): string {
+  const { separator = ', ', format = 'numbered' } = options;
+
+  return sources
+    .map((source, index) => {
+      switch (format) {
+        case 'id':
+          return `[${source.id}]`;
+        case 'title':
+          return `[${source.title ?? source.id}]`;
+        case 'numbered':
+        default:
+          return `[${index + 1}]`;
+      }
+    })
+    .join(separator);
+}
+
+/**
+ * Convert sources to context string for LLM prompts.
+ *
+ * @example
+ * ```typescript
+ * const context = sourcesToContext(sources, { includeMetadata: true });
+ * ```
+ */
+export function sourcesToContext(
+  sources: GroundingSource[],
+  options: {
+    includeMetadata?: boolean;
+    includeScore?: boolean;
+    maxLength?: number;
+    separator?: string;
+  } = {},
+): string {
+  const {
+    includeMetadata = false,
+    includeScore = false,
+    maxLength = 50000,
+    separator = '\n\n---\n\n',
+  } = options;
+
+  let totalLength = 0;
+  const parts: string[] = [];
+
+  for (const source of sources) {
+    let part = '';
+
+    if (source.title) {
+      part += `# ${source.title}\n`;
+    }
+
+    if (includeScore && source.score !== undefined) {
+      part += `Relevance: ${(source.score * 100).toFixed(0)}%\n`;
+    }
+
+    if (source.url) {
+      part += `Source: ${source.url}\n`;
+    }
+
+    if (includeMetadata && source.metadata) {
+      const meta = Object.entries(source.metadata)
+        .map(([k, v]) => `${k}: ${v}`)
+        .join(', ');
+      part += `Metadata: ${meta}\n`;
+    }
+
+    part += `\n${source.content}`;
+
+    if (totalLength + part.length > maxLength) {
+      // Truncate this source
+      const remaining = maxLength - totalLength;
+      if (remaining > 100) {
+        parts.push(`${part.slice(0, remaining)}...`);
+      }
+      break;
+    }
+
+    parts.push(part);
+    totalLength += part.length + separator.length;
+  }
+
+  return parts.join(separator);
+}
+
+// -----------------------------------------------------------------------------
+// Utility Functions
+// -----------------------------------------------------------------------------
+
+function computeTextSimilarity(text1: string, text2: string): number {
+  const words1 = new Set(tokenize(text1));
+  const words2 = new Set(tokenize(text2));
+
+  if (words1.size === 0 || words2.size === 0) return 0;
+
+  let intersection = 0;
+  for (const word of words1) {
+    if (words2.has(word)) intersection++;
+  }
+
+  const union = words1.size + words2.size - intersection;
+  return union > 0 ? intersection / union : 0;
+}
+
+function tokenize(text: string): string[] {
+  return text
+    .toLowerCase()
+    .replace(/[^\w\s]/g, ' ')
+    .split(/\s+/)
+    .filter(w => w.length > 2);
+}
+
+function isValidUrl(url: string): boolean {
+  try {
+    new URL(url);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function normalizeUrl(url: string): string {
+  try {
+    const parsed = new URL(url);
+    // Remove trailing slash, lowercase host
+    return `${parsed.protocol}//${parsed.host.toLowerCase()}${parsed.pathname.replace(/\/$/, '')}`;
+  } catch {
+    return url.toLowerCase();
+  }
+}
+
+function hashContent(content: string): string {
+  let hash = 5381;
+  for (let i = 0; i < content.length; i++) {
+    const code = content.charCodeAt(i);
+    hash = (hash * 33) ^ code;
+  }
+  return `content-${hash >>> 0}`;
+}
+
+function inferSourceType(source: GroundingSource): SourceWithProvenance['sourceType'] {
+  if (!source.url) return 'unknown';
+
+  const url = source.url.toLowerCase();
+
+  if (url.includes('api.') || url.includes('/api/')) return 'api';
+  if (url.includes('.pdf') || url.includes('/doc')) return 'document';
+  if (url.includes('database') || url.includes('db.')) return 'database';
+  if (url.startsWith('http')) return 'webpage';
+
+  return 'unknown';
+}
+
+function checkDuplicate(
+  source: GroundingSource,
+  existing: GroundingSource[],
+  strategy: DeduplicationStrategy,
+): boolean {
+  switch (strategy) {
+    case 'id':
+      return existing.some(e => e.id === source.id);
+    case 'url':
+      if (!source.url) return false;
+      const normalizedUrl = normalizeUrl(source.url);
+      return existing.some(e => e.url && normalizeUrl(e.url) === normalizedUrl);
+    case 'content':
+      return existing.some(e => computeTextSimilarity(source.content, e.content) > 0.9);
+    case 'hybrid': {
+      const idMatch = existing.some(e => e.id === source.id);
+      const sourceUrl = source.url;
+      const urlMatch = sourceUrl
+        ? existing.some(e => e.url && normalizeUrl(e.url) === normalizeUrl(sourceUrl))
+        : false;
+      const contentMatch = existing.some(
+        e => computeTextSimilarity(source.content, e.content) > 0.9,
+      );
+      return idMatch || urlMatch || contentMatch;
+    }
+    default:
+      return false;
+  }
+}