@highway1/core 0.1.39 → 0.1.41

package/src/discovery/capability-matcher.ts

@@ -0,0 +1,167 @@
+/**
+ * Capability Matcher
+ *
+ * Matches semantic queries against agent capabilities
+ */
+
+import type { Capability } from './agent-card-types.js';
+import type { SemanticQuery } from './search-index.js';
+
+/**
+ * Capability Matcher
+ */
+export class CapabilityMatcher {
+  /**
+   * Match a query against a capability
+   * Returns a score between 0 and 1
+   */
+  match(query: SemanticQuery, capability: Capability): number {
+    let score = 0;
+    let weights = 0;
+
+    // Exact capability ID match
+    if (query.capability && capability.id === query.capability) {
+      return 1.0;
+    }
+
+    // Fuzzy capability name match
+    if (query.capability) {
+      const nameScore = this.fuzzyMatch(query.capability, capability.name);
+      score += nameScore * 0.4;
+      weights += 0.4;
+    }
+
+    // Text keyword match
+    if (query.text) {
+      const keywords = this.extractKeywords(query.text);
+      const keywordScore = this.matchKeywords(keywords, capability);
+      score += keywordScore * 0.4;
+      weights += 0.4;
+    }
+
+    // Type hierarchy match
+    if (query.capability && capability['@type']) {
+      const typeScore = this.matchesType(query.capability, capability['@type']) ? 0.6 : 0;
+      score += typeScore * 0.2;
+      weights += 0.2;
+    }
+
+    return weights > 0 ? score / weights : 0;
+  }
+
+  /**
+   * Extract keywords from natural language text
+   */
+  extractKeywords(text: string): string[] {
+    // Remove common stop words
+    const stopWords = new Set([
+      'a', 'an', 'the', 'is', 'are', 'was', 'were', 'be', 'been',
+      'to', 'from', 'in', 'on', 'at', 'by', 'for', 'with', 'about',
+      'can', 'could', 'should', 'would', 'will', 'do', 'does', 'did',
+      'i', 'you', 'he', 'she', 'it', 'we', 'they', 'me', 'him', 'her',
+    ]);
+
+    return text
+      .toLowerCase()
+      .split(/\W+/)
+      .filter(word => word.length > 2 && !stopWords.has(word));
+  }
+
+  /**
+   * Match keywords against capability
+   */
+  matchKeywords(keywords: string[], capability: Capability): number {
+    if (keywords.length === 0) return 0;
+
+    const capText = `${capability.name} ${capability.description}`.toLowerCase();
+    const matches = keywords.filter(keyword => capText.includes(keyword));
+
+    return matches.length / keywords.length;
+  }
+
+  /**
+   * Check if query matches capability type hierarchy
+   */
+  matchesType(query: string, type: string): boolean {
+    const queryLower = query.toLowerCase();
+    const typeLower = type.toLowerCase();
+
+    // Direct match
+    if (typeLower.includes(queryLower)) return true;
+
+    // Common type mappings
+    const typeMap: Record<string, string[]> = {
+      translate: ['translation', 'translationservice'],
+      review: ['codereview', 'reviewservice'],
+      analyze: ['analysis', 'dataanalysis'],
+      generate: ['generation', 'textgeneration', 'imagegeneration'],
+      search: ['searchservice', 'query'],
+      compute: ['computation', 'computationservice'],
+      store: ['storage', 'storageservice'],
+      message: ['messaging', 'messagingservice'],
+      auth: ['authentication', 'authenticationservice'],
+    };
+
+    for (const [key, values] of Object.entries(typeMap)) {
+      if (queryLower.includes(key) && values.some(v => typeLower.includes(v))) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  /**
+   * Fuzzy string matching using Levenshtein distance
+   */
+  private fuzzyMatch(query: string, target: string): number {
+    const queryLower = query.toLowerCase();
+    const targetLower = target.toLowerCase();
+
+    // Exact match
+    if (queryLower === targetLower) return 1.0;
+
+    // Substring match
+    if (targetLower.includes(queryLower)) return 0.8;
+    if (queryLower.includes(targetLower)) return 0.7;
+
+    // Levenshtein distance
+    const distance = this.levenshteinDistance(queryLower, targetLower);
+    const maxLen = Math.max(queryLower.length, targetLower.length);
+    const similarity = 1 - distance / maxLen;
+
+    // Only return positive scores for reasonable similarity
+    return similarity > 0.5 ? similarity * 0.6 : 0;
+  }
+
+  /**
+   * Calculate Levenshtein distance between two strings
+   */
+  private levenshteinDistance(a: string, b: string): number {
+    const matrix: number[][] = [];
+
+    for (let i = 0; i <= b.length; i++) {
+      matrix[i] = [i];
+    }
+
+    for (let j = 0; j <= a.length; j++) {
+      matrix[0][j] = j;
+    }
+
+    for (let i = 1; i <= b.length; i++) {
+      for (let j = 1; j <= a.length; j++) {
+        if (b.charAt(i - 1) === a.charAt(j - 1)) {
+          matrix[i][j] = matrix[i - 1][j - 1];
+        } else {
+          matrix[i][j] = Math.min(
+            matrix[i - 1][j - 1] + 1, // substitution
+            matrix[i][j - 1] + 1,     // insertion
+            matrix[i - 1][j] + 1      // deletion
+          );
+        }
+      }
+    }
+
+    return matrix[b.length][a.length];
+  }
+}

package/src/discovery/dht.ts

@@ -0,0 +1,285 @@
+import type { Libp2p } from 'libp2p';
+import type { AgentCard } from './agent-card.js';
+import { createLogger } from '../utils/logger.js';
+import { DiscoveryError } from '../utils/errors.js';
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string';
+import { toString as uint8ArrayToString } from 'uint8arrays/to-string';
+import { encodeForDHT, decodeFromCBOR } from './agent-card-encoder.js';
+import { createSemanticSearch } from './semantic-search.js';
+import type { SemanticQuery } from './search-index.js';
+
+const logger = createLogger('dht');
+
+export interface ResolvedDID {
+  peerId: string;
+  multiaddrs: string[];
+}
+
+export interface DHTOperations {
+  publishAgentCard: (card: AgentCard) => Promise<void>;
+  queryAgentCard: (did: string) => Promise<AgentCard | null>;
+  queryByCapability: (capability: string) => Promise<AgentCard[]>;
+  searchSemantic: (query: SemanticQuery) => Promise<AgentCard[]>;
+  resolveDID: (did: string) => Promise<ResolvedDID | null>;
+  queryRelayPeers: () => Promise<string[]>;
+}
+
+/** Extract a DHT value from a get() event, handling both VALUE and PEER_RESPONSE */
+function extractValue(event: any): Uint8Array | null {
+  if (event.name === 'VALUE' && event.value) return event.value;
+  if (event.name === 'PEER_RESPONSE' && event.value) return event.value;
+  return null;
+}
+
+/** Normalize capability name for use as a DHT key segment */
+function capKey(cap: string): string {
+  return cap.toLowerCase().replace(/[^a-z0-9_-]/g, '_');
+}
+
+/** Read a DID list stored at a DHT key (newline-separated) */
+async function readDIDList(dht: any, key: Uint8Array): Promise<string[]> {
+  try {
+    for await (const event of dht.get(key, { signal: AbortSignal.timeout(30000) })) {
+      const raw = extractValue(event);
+      if (raw) {
+        const text = uint8ArrayToString(raw);
+        return text.split('\n').filter(Boolean);
+      }
+    }
+  } catch {
+    // key not found or timeout
+  }
+  return [];
+}
+
+/** Write a DID list to a DHT key (newline-separated), deduplicating */
+async function writeDIDList(dht: any, key: Uint8Array, dids: string[]): Promise<void> {
+  const value = uint8ArrayFromString([...new Set(dids)].join('\n'));
+  // Retry up to 3 times with 30s timeout each
+  for (let attempt = 1; attempt <= 3; attempt++) {
+    try {
+      for await (const _ of dht.put(key, value, { signal: AbortSignal.timeout(30000) })) { /* consume */ }
+      return; // Success
+    } catch (e: any) {
+      if (e?.name === 'AbortError' && attempt < 3) {
+        logger.debug(`DHT put timeout, retrying (${attempt}/3)...`);
+        continue;
+      }
+      if (e?.name !== 'AbortError') throw e;
+    }
+  }
+}
+
+/**
+ * Create DHT operations for a libp2p node
+ */
+export function createDHTOperations(libp2p: Libp2p): DHTOperations {
+  const operations: DHTOperations = {} as DHTOperations;
+  const searchEngine = createSemanticSearch(operations);
+
+  return Object.assign(operations, {
+    publishAgentCard: async (card: AgentCard) => {
+      try {
+        const dht = (libp2p as any).services?.dht;
+        if (!dht) throw new DiscoveryError('DHT service not available');
+
+        // 1. Store Agent Card under /clawiverse/agent/<did>
+        const agentKey = uint8ArrayFromString(`/clawiverse/agent/${card.did}`);
+        // Retry up to 3 times with 30s timeout each
+        for (let attempt = 1; attempt <= 3; attempt++) {
+          try {
+            for await (const _ of dht.put(agentKey, encodeForDHT(card), { signal: AbortSignal.timeout(30000) })) { /* consume */ }
+            break; // Success
+          } catch (e: any) {
+            if (e?.name === 'AbortError' && attempt < 3) {
+              logger.debug(`DHT put agent card timeout, retrying (${attempt}/3)...`);
+              continue;
+            }
+            logger.warn('DHT put agent card failed (non-fatal)', { error: (e as Error).message });
+            break;
+          }
+        }
+
+        // 2. For each capability, append this DID to /clawiverse/cap/<capability>
+        // Skip capability indexing in slow networks - rely on local index and semantic search instead
+        const caps: string[] = (card.capabilities ?? []).flatMap((c: any) => {
+          if (typeof c === 'string') return [c];
+          return [c.name, c.id].filter(Boolean);
+        });
+
+        // Always index under the special "all" key so name/description search works
+        caps.push('__all__');
+
+        // Keep indexing lightweight, but preserve relay discovery path.
+        const importantCaps = ['__all__'];
+        if (caps.some((cap) => capKey(cap) === 'relay')) {
+          importantCaps.push('relay');
+        }
+        await Promise.all(importantCaps.map(async (cap) => {
+          const capKeyStr = `/clawiverse/cap/${capKey(cap)}`;
+          const capDHTKey = uint8ArrayFromString(capKeyStr);
+          try {
+            const existing = await readDIDList(dht, capDHTKey);
+            if (!existing.includes(card.did)) {
+              await writeDIDList(dht, capDHTKey, [...existing, card.did]);
+              logger.debug('Indexed capability in DHT', { cap: capKey(cap), did: card.did });
+            }
+          } catch (e: any) {
+            logger.warn('Failed to index capability (non-fatal)', { cap: capKey(cap), error: (e as Error).message });
+          }
+        }));
+
+        // 3. Index locally for fast in-process search
+        searchEngine.indexAgentCard(card);
+
+        logger.info('Published Agent Card to DHT', { did: card.did });
+      } catch (error) {
+        // DHT publish failure is non-fatal - local index is still available
+        logger.warn('Failed to publish Agent Card to DHT (non-fatal)', { error: (error as Error).message });
+        // Still index locally even if DHT fails
+        searchEngine.indexAgentCard(card);
+      }
+    },
+
+    queryAgentCard: async (did: string) => {
+      try {
+        const dht = (libp2p as any).services?.dht;
+        if (!dht) throw new DiscoveryError('DHT service not available');
+
+        const key = uint8ArrayFromString(`/clawiverse/agent/${did}`);
+        for await (const event of dht.get(key)) {
+          const raw = extractValue(event);
+          if (raw) {
+            const card = decodeFromCBOR(raw);
+            searchEngine.indexAgentCard(card);
+            logger.debug('Found Agent Card in DHT', { did });
+            return card;
+          }
+        }
+
+        logger.debug('Agent Card not found in DHT', { did });
+        return null;
+      } catch (error) {
+        logger.warn('Failed to query Agent Card', { did, error });
+        return null;
+      }
+    },
+
+    queryByCapability: async (capability: string) => {
+      try {
+        const dht = (libp2p as any).services?.dht;
+
+        // 1. Check local index first
+        const local = searchEngine.getAllIndexedCards().filter(card =>
+          card.capabilities.some((cap: any) => {
+            const name = typeof cap === 'string' ? cap : cap.name;
+            return name?.toLowerCase().includes(capability.toLowerCase());
+          })
+        );
+        if (local.length > 0) return local;
+
+        // 2. Fall back to DHT capability index
+        if (!dht) return [];
+
+        const capDHTKey = uint8ArrayFromString(`/clawiverse/cap/${capKey(capability)}`);
+        const dids = await readDIDList(dht, capDHTKey);
+        logger.debug('DHT capability index', { capability, dids });
+
+        // Fetch each Agent Card in parallel
+        const cards = await Promise.all(
+          dids.map(async (did) => {
+            const key = uint8ArrayFromString(`/clawiverse/agent/${did}`);
+            try {
+              for await (const event of dht.get(key)) {
+                const raw = extractValue(event);
+                if (raw) {
+                  const card = decodeFromCBOR(raw);
+                  searchEngine.indexAgentCard(card);
+                  return card;
+                }
+              }
+            } catch { /* skip unreachable */ }
+            return null;
+          })
+        );
+
+        return cards.filter((c): c is AgentCard => c !== null);
+      } catch (error) {
+        throw new DiscoveryError('Failed to query by capability', error);
+      }
+    },
+
+    searchSemantic: async (query: SemanticQuery) => {
+      try {
+        // Always pull from DHT __all__ index to discover remote nodes
+        const dht = (libp2p as any).services?.dht;
+        if (dht) {
+          const allKey = uint8ArrayFromString('/clawiverse/cap/__all__');
+          const dids = await readDIDList(dht, allKey);
+          logger.debug('DHT __all__ index', { count: dids.length });
+
+          await Promise.all(
+            dids.map(async (did) => {
+              const key = uint8ArrayFromString(`/clawiverse/agent/${did}`);
+              try {
+                for await (const event of dht.get(key)) {
+                  const raw = extractValue(event);
+                  if (raw) {
+                    searchEngine.indexAgentCard(decodeFromCBOR(raw));
+                    break;
+                  }
+                }
+              } catch { /* skip */ }
+            })
+          );
+        }
+
+        return searchEngine.search(query);
+      } catch (error) {
+        throw new DiscoveryError('Failed to perform semantic search', error);
+      }
+    },
+
+    queryRelayPeers: async (): Promise<string[]> => {
+      const dht = (libp2p as any).services?.dht;
+      if (!dht) return [];
+      const capDHTKey = uint8ArrayFromString('/clawiverse/cap/relay');
+      const dids = await readDIDList(dht, capDHTKey);
+      const addrs: string[] = [];
+      await Promise.all(dids.map(async (did) => {
+        const card = await operations.queryAgentCard(did);
+        if (card?.endpoints) {
+          addrs.push(...card.endpoints.filter((e: string) => !e.includes('/p2p-circuit/')));
+        }
+      }));
+      return addrs;
+    },
+
+    resolveDID: async (did: string) => {
+      try {
+        const dht = (libp2p as any).services?.dht;
+        if (!dht) throw new DiscoveryError('DHT service not available');
+
+        const key = uint8ArrayFromString(`/clawiverse/agent/${did}`);
+        for await (const event of dht.get(key)) {
+          const raw = extractValue(event);
+          if (raw) {
+            const card = decodeFromCBOR(raw);
+            if (card.peerId) {
+              logger.debug('Resolved DID to peer', { did, peerId: card.peerId });
+              return { peerId: card.peerId, multiaddrs: card.endpoints || [] };
+            }
+            logger.warn('Agent Card found but has no peerId', { did });
+            return null;
+          }
+        }
+
+        logger.debug('DID not found in DHT', { did });
+        return null;
+      } catch (error) {
+        logger.warn('Failed to resolve DID', { did, error });
+        return null;
+      }
+    },
+  });
+}

package/src/discovery/index.ts

@@ -0,0 +1,3 @@
+export * from './agent-card.js';
+export * from './dht.js';
+export * from './bootstrap.js';

package/src/discovery/search-index.ts

@@ -0,0 +1,247 @@
+/**
+ * Search Index for Agent Discovery
+ *
+ * Maintains a local index of discovered agents for fast semantic search
+ */
+
+import lunr from 'lunr';
+import Fuse from 'fuse.js';
+import type { AgentCard } from './agent-card-types.js';
+import { createLogger } from '../utils/logger.js';
+
+const logger = createLogger('search-index');
+
+/**
+ * Semantic Query Interface
+ */
+export interface SemanticQuery {
+  text?: string;              // Natural language: "translate Japanese"
+  capability?: string;        // Structured: "translate"
+  filters?: {
+    language?: string;
+    minTrustScore?: number;
+    maxCost?: number;
+    tags?: string[];
+  };
+  limit?: number;
+}
+
+/**
+ * Search Result with Score
+ */
+export interface SearchResult {
+  card: AgentCard;
+  score: number;
+}
+
+/**
+ * Search Index Implementation
+ */
+export class SearchIndex {
+  private cards = new Map<string, AgentCard>();
+  private lunrIndex?: lunr.Index;
+  private fuse?: Fuse<AgentCard>;
+  private needsRebuild = false;
+
+  /**
+   * Add or update an Agent Card in the index
+   */
+  indexAgentCard(card: AgentCard): void {
+    this.cards.set(card.did, card);
+    this.needsRebuild = true;
+    logger.debug('Indexed Agent Card', { did: card.did, capabilities: card.capabilities.length });
+  }
+
+  /**
+   * Remove an Agent Card from the index
+   */
+  removeAgentCard(did: string): void {
+    this.cards.delete(did);
+    this.needsRebuild = true;
+    logger.debug('Removed Agent Card from index', { did });
+  }
+
+  /**
+   * Search for agents matching a query
+   */
+  search(query: SemanticQuery): SearchResult[] {
+    if (this.needsRebuild) {
+      this.rebuild();
+    }
+
+    let results: SearchResult[] = [];
+
+    // Text search using Lunr
+    if (query.text && this.lunrIndex) {
+      results = this.searchByText(query.text);
+    }
+    // Capability search using Fuse
+    else if (query.capability && this.fuse) {
+      results = this.searchByCapability(query.capability);
+    }
+    // No query - return all
+    else {
+      results = Array.from(this.cards.values()).map(card => ({
+        card,
+        score: 1.0,
+      }));
+    }
+
+    // Apply filters
+    if (query.filters) {
+      results = this.applyFilters(results, query.filters);
+    }
+
+    // Sort by score (descending)
+    results.sort((a, b) => b.score - a.score);
+
+    // Apply limit
+    if (query.limit) {
+      results = results.slice(0, query.limit);
+    }
+
+    logger.debug('Search completed', { query, results: results.length });
+    return results;
+  }
+
+  /**
+   * Get all indexed cards
+   */
+  getAllCards(): AgentCard[] {
+    return Array.from(this.cards.values());
+  }
+
+  /**
+   * Clear the index
+   */
+  clear(): void {
+    this.cards.clear();
+    this.lunrIndex = undefined;
+    this.fuse = undefined;
+    this.needsRebuild = false;
+    logger.info('Search index cleared');
+  }
+
+  /**
+   * Get index size
+   */
+  size(): number {
+    return this.cards.size;
+  }
+
+  /**
+   * Rebuild search indexes
+   */
+  private rebuild(): void {
+    logger.info('Rebuilding search indexes', { cards: this.cards.size });
+
+    const cards = Array.from(this.cards.values());
+
+    // Build Lunr index for full-text search
+    this.lunrIndex = lunr(function (this: lunr.Builder) {
+      this.ref('did');
+      this.field('name', { boost: 10 });
+      this.field('description', { boost: 5 });
+      this.field('capabilities');
+
+      for (const card of cards) {
+        this.add({
+          did: card.did,
+          name: card.name,
+          description: card.description,
+          capabilities: card.capabilities
+            .map((cap: any) => `${cap.name} ${cap.description}`)
+            .join(' '),
+        });
+      }
+    });
+
+    // Build Fuse index for fuzzy matching
+    this.fuse = new Fuse(Array.from(this.cards.values()), {
+      keys: [
+        { name: 'name', weight: 0.3 },
+        { name: 'description', weight: 0.2 },
+        { name: 'capabilities.name', weight: 0.3 },
+        { name: 'capabilities.description', weight: 0.2 },
+      ],
+      threshold: 0.4,
+      includeScore: true,
+    });
+
+    this.needsRebuild = false;
+    logger.info('Search indexes rebuilt');
+  }
+
+  /**
+   * Search by text using Lunr
+   */
+  private searchByText(text: string): SearchResult[] {
+    if (!this.lunrIndex) return [];
+
+    const lunrResults = this.lunrIndex.search(text);
+    return lunrResults.map(result => ({
+      card: this.cards.get(result.ref)!,
+      score: result.score,
+    }));
+  }
+
+  /**
+   * Search by capability using Fuse
+   */
+  private searchByCapability(capability: string): SearchResult[] {
+    if (!this.fuse) return [];
+
+    const fuseResults = this.fuse.search(capability);
+    return fuseResults.map(result => ({
+      card: result.item,
+      score: 1 - (result.score || 0), // Fuse score is distance, convert to similarity
+    }));
+  }
+
+  /**
+   * Apply filters to search results
+   */
+  private applyFilters(
+    results: SearchResult[],
+    filters: NonNullable<SemanticQuery['filters']>
+  ): SearchResult[] {
+    return results.filter(result => {
+      const { card } = result;
+
+      // Trust score filter
+      if (filters.minTrustScore !== undefined && card.trust) {
+        const overallTrust =
+          card.trust.interactionScore * 0.4 +
+          Math.min(card.trust.endorsements / 10, 1) * 0.2 +
+          card.trust.completionRate * 0.2 +
+          card.trust.uptime * 0.2;
+
+        if (overallTrust < filters.minTrustScore) {
+          return false;
+        }
+      }
+
+      // Language filter (check capability metadata)
+      if (filters.language) {
+        const hasLanguage = card.capabilities.some(
+          cap => {
+            const metadata = cap.metadata as Record<string, any> | undefined;
+            if (!metadata) return false;
+            return metadata.language === filters.language ||
+                   (Array.isArray(metadata.languages) && metadata.languages.includes(filters.language));
+          }
+        );
+        if (!hasLanguage) return false;
+      }
+
+      // Tags filter
+      if (filters.tags && filters.tags.length > 0) {
+        const cardTags = Array.isArray(card.metadata?.tags) ? (card.metadata.tags as string[]) : [];
+        const hasAllTags = filters.tags.every(tag => cardTags.includes(tag));
+        if (!hasAllTags) return false;
+      }
+
+      return true;
+    });
+  }
+}