@highway1/core 0.1.43 → 0.1.45

package/src/discovery/capability-matcher.ts

@@ -0,0 +1,167 @@
+/**
+ * Capability Matcher
+ *
+ * Matches semantic queries against agent capabilities
+ */
+
+import type { Capability } from './agent-card-types.js';
+import type { SemanticQuery } from './search-index.js';
+
+/**
+ * Capability Matcher
+ */
+export class CapabilityMatcher {
+  /**
+   * Match a query against a capability
+   * Returns a score between 0 and 1
+   */
+  match(query: SemanticQuery, capability: Capability): number {
+    let score = 0;
+    let weights = 0;
+
+    // Exact capability ID match
+    if (query.capability && capability.id === query.capability) {
+      return 1.0;
+    }
+
+    // Fuzzy capability name match
+    if (query.capability) {
+      const nameScore = this.fuzzyMatch(query.capability, capability.name);
+      score += nameScore * 0.4;
+      weights += 0.4;
+    }
+
+    // Text keyword match
+    if (query.text) {
+      const keywords = this.extractKeywords(query.text);
+      const keywordScore = this.matchKeywords(keywords, capability);
+      score += keywordScore * 0.4;
+      weights += 0.4;
+    }
+
+    // Type hierarchy match
+    if (query.capability && capability['@type']) {
+      const typeScore = this.matchesType(query.capability, capability['@type']) ? 0.6 : 0;
+      score += typeScore * 0.2;
+      weights += 0.2;
+    }
+
+    return weights > 0 ? score / weights : 0;
+  }
+
+  /**
+   * Extract keywords from natural language text
+   */
+  extractKeywords(text: string): string[] {
+    // Remove common stop words
+    const stopWords = new Set([
+      'a', 'an', 'the', 'is', 'are', 'was', 'were', 'be', 'been',
+      'to', 'from', 'in', 'on', 'at', 'by', 'for', 'with', 'about',
+      'can', 'could', 'should', 'would', 'will', 'do', 'does', 'did',
+      'i', 'you', 'he', 'she', 'it', 'we', 'they', 'me', 'him', 'her',
+    ]);
+
+    return text
+      .toLowerCase()
+      .split(/\W+/)
+      .filter(word => word.length > 2 && !stopWords.has(word));
+  }
+
+  /**
+   * Match keywords against capability
+   */
+  matchKeywords(keywords: string[], capability: Capability): number {
+    if (keywords.length === 0) return 0;
+
+    const capText = `${capability.name} ${capability.description}`.toLowerCase();
+    const matches = keywords.filter(keyword => capText.includes(keyword));
+
+    return matches.length / keywords.length;
+  }
+
+  /**
+   * Check if query matches capability type hierarchy
+   */
+  matchesType(query: string, type: string): boolean {
+    const queryLower = query.toLowerCase();
+    const typeLower = type.toLowerCase();
+
+    // Direct match
+    if (typeLower.includes(queryLower)) return true;
+
+    // Common type mappings
+    const typeMap: Record<string, string[]> = {
+      translate: ['translation', 'translationservice'],
+      review: ['codereview', 'reviewservice'],
+      analyze: ['analysis', 'dataanalysis'],
+      generate: ['generation', 'textgeneration', 'imagegeneration'],
+      search: ['searchservice', 'query'],
+      compute: ['computation', 'computationservice'],
+      store: ['storage', 'storageservice'],
+      message: ['messaging', 'messagingservice'],
+      auth: ['authentication', 'authenticationservice'],
+    };
+
+    for (const [key, values] of Object.entries(typeMap)) {
+      if (queryLower.includes(key) && values.some(v => typeLower.includes(v))) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  /**
+   * Fuzzy string matching using Levenshtein distance
+   */
+  private fuzzyMatch(query: string, target: string): number {
+    const queryLower = query.toLowerCase();
+    const targetLower = target.toLowerCase();
+
+    // Exact match
+    if (queryLower === targetLower) return 1.0;
+
+    // Substring match
+    if (targetLower.includes(queryLower)) return 0.8;
+    if (queryLower.includes(targetLower)) return 0.7;
+
+    // Levenshtein distance
+    const distance = this.levenshteinDistance(queryLower, targetLower);
+    const maxLen = Math.max(queryLower.length, targetLower.length);
+    const similarity = 1 - distance / maxLen;
+
+    // Only return positive scores for reasonable similarity
+    return similarity > 0.5 ? similarity * 0.6 : 0;
+  }
+
+  /**
+   * Calculate Levenshtein distance between two strings
+   */
+  private levenshteinDistance(a: string, b: string): number {
+    const matrix: number[][] = [];
+
+    for (let i = 0; i <= b.length; i++) {
+      matrix[i] = [i];
+    }
+
+    for (let j = 0; j <= a.length; j++) {
+      matrix[0][j] = j;
+    }
+
+    for (let i = 1; i <= b.length; i++) {
+      for (let j = 1; j <= a.length; j++) {
+        if (b.charAt(i - 1) === a.charAt(j - 1)) {
+          matrix[i][j] = matrix[i - 1][j - 1];
+        } else {
+          matrix[i][j] = Math.min(
+            matrix[i - 1][j - 1] + 1, // substitution
+            matrix[i][j - 1] + 1,     // insertion
+            matrix[i - 1][j] + 1      // deletion
+          );
+        }
+      }
+    }
+
+    return matrix[b.length][a.length];
+  }
+}

package/src/discovery/dht.ts

@@ -0,0 +1,310 @@
+import type { Libp2p } from 'libp2p';
+import type { AgentCard } from './agent-card.js';
+import { createLogger } from '../utils/logger.js';
+import { DiscoveryError } from '../utils/errors.js';
+import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string';
+import { toString as uint8ArrayToString } from 'uint8arrays/to-string';
+import { encodeForDHT, decodeFromCBOR } from './agent-card-encoder.js';
+import { createSemanticSearch } from './semantic-search.js';
+import type { SemanticQuery } from './search-index.js';
+
+const logger = createLogger('dht');
+
+export interface ResolvedDID {
+  peerId: string;
+  multiaddrs: string[];
+}
+
+interface CachedPeerInfo {
+  peerId: string;
+  multiaddrs: string[];
+  timestamp: number;
+}
+
+const peerCache = new Map<string, CachedPeerInfo>();
+const CACHE_TTL = 5 * 60 * 1000; // 5 minutes
+
+export interface DHTOperations {
+  publishAgentCard: (card: AgentCard) => Promise<void>;
+  queryAgentCard: (did: string) => Promise<AgentCard | null>;
+  queryByCapability: (capability: string) => Promise<AgentCard[]>;
+  searchSemantic: (query: SemanticQuery) => Promise<AgentCard[]>;
+  resolveDID: (did: string) => Promise<ResolvedDID | null>;
+  queryRelayPeers: () => Promise<string[]>;
+}
+
+/** Extract a DHT value from a get() event, handling both VALUE and PEER_RESPONSE */
+function extractValue(event: any): Uint8Array | null {
+  if (event.name === 'VALUE' && event.value) return event.value;
+  if (event.name === 'PEER_RESPONSE' && event.value) return event.value;
+  return null;
+}
+
+/** Normalize capability name for use as a DHT key segment */
+function capKey(cap: string): string {
+  return cap.toLowerCase().replace(/[^a-z0-9_-]/g, '_');
+}
+
+/** Read a DID list stored at a DHT key (newline-separated) */
+async function readDIDList(dht: any, key: Uint8Array): Promise<string[]> {
+  try {
+    for await (const event of dht.get(key, { signal: AbortSignal.timeout(30000) })) {
+      const raw = extractValue(event);
+      if (raw) {
+        const text = uint8ArrayToString(raw);
+        return text.split('\n').filter(Boolean);
+      }
+    }
+  } catch {
+    // key not found or timeout
+  }
+  return [];
+}
+
+/** Write a DID list to a DHT key (newline-separated), deduplicating */
+async function writeDIDList(dht: any, key: Uint8Array, dids: string[]): Promise<void> {
+  const value = uint8ArrayFromString([...new Set(dids)].join('\n'));
+  // Retry up to 3 times with 30s timeout each
+  for (let attempt = 1; attempt <= 3; attempt++) {
+    try {
+      for await (const _ of dht.put(key, value, { signal: AbortSignal.timeout(30000) })) { /* consume */ }
+      return; // Success
+    } catch (e: any) {
+      if (e?.name === 'AbortError' && attempt < 3) {
+        logger.debug(`DHT put timeout, retrying (${attempt}/3)...`);
+        continue;
+      }
+      if (e?.name !== 'AbortError') throw e;
+    }
+  }
+}
+
+/**
+ * Create DHT operations for a libp2p node
+ */
+export function createDHTOperations(libp2p: Libp2p): DHTOperations {
+  const operations: DHTOperations = {} as DHTOperations;
+  const searchEngine = createSemanticSearch(operations);
+
+  return Object.assign(operations, {
+    publishAgentCard: async (card: AgentCard) => {
+      try {
+        const dht = (libp2p as any).services?.dht;
+        if (!dht) throw new DiscoveryError('DHT service not available');
+
+        // 1. Store Agent Card under /clawiverse/agent/<did>
+        const agentKey = uint8ArrayFromString(`/clawiverse/agent/${card.did}`);
+        // Retry up to 3 times with 30s timeout each
+        for (let attempt = 1; attempt <= 3; attempt++) {
+          try {
+            for await (const _ of dht.put(agentKey, encodeForDHT(card), { signal: AbortSignal.timeout(30000) })) { /* consume */ }
+            break; // Success
+          } catch (e: any) {
+            if (e?.name === 'AbortError' && attempt < 3) {
+              logger.debug(`DHT put agent card timeout, retrying (${attempt}/3)...`);
+              continue;
+            }
+            logger.warn('DHT put agent card failed (non-fatal)', { error: (e as Error).message });
+            break;
+          }
+        }
+
+        // 2. For each capability, append this DID to /clawiverse/cap/<capability>
+        // Skip capability indexing in slow networks - rely on local index and semantic search instead
+        const caps: string[] = (card.capabilities ?? []).flatMap((c: any) => {
+          if (typeof c === 'string') return [c];
+          return [c.name, c.id].filter((v): v is string => typeof v === 'string' && v.length > 0);
+        });
+
+        // Always index under the special "all" key so name/description search works
+        caps.push('__all__');
+
+        // Keep indexing lightweight, but preserve relay discovery path.
+        const importantCaps = ['__all__'];
+        if (caps.some((cap) => capKey(cap) === 'relay')) {
+          importantCaps.push('relay');
+        }
+        await Promise.all(importantCaps.map(async (cap) => {
+          const capKeyStr = `/clawiverse/cap/${capKey(cap)}`;
+          const capDHTKey = uint8ArrayFromString(capKeyStr);
+          try {
+            const existing = await readDIDList(dht, capDHTKey);
+            if (!existing.includes(card.did)) {
+              await writeDIDList(dht, capDHTKey, [...existing, card.did]);
+              logger.debug('Indexed capability in DHT', { cap: capKey(cap), did: card.did });
+            }
+          } catch (e: any) {
+            logger.warn('Failed to index capability (non-fatal)', { cap: capKey(cap), error: (e as Error).message });
+          }
+        }));
+
+        // 3. Index locally for fast in-process search
+        searchEngine.indexAgentCard(card);
+
+        logger.info('Published Agent Card to DHT', { did: card.did });
+      } catch (error) {
+        // DHT publish failure is non-fatal - local index is still available
+        logger.warn('Failed to publish Agent Card to DHT (non-fatal)', { error: (error as Error).message });
+        // Still index locally even if DHT fails
+        searchEngine.indexAgentCard(card);
+      }
+    },
+
+    queryAgentCard: async (did: string) => {
+      try {
+        const dht = (libp2p as any).services?.dht;
+        if (!dht) throw new DiscoveryError('DHT service not available');
+
+        const key = uint8ArrayFromString(`/clawiverse/agent/${did}`);
+        for await (const event of dht.get(key)) {
+          const raw = extractValue(event);
+          if (raw) {
+            const card = decodeFromCBOR(raw);
+            searchEngine.indexAgentCard(card);
+            logger.debug('Found Agent Card in DHT', { did });
+            return card;
+          }
+        }
+
+        logger.debug('Agent Card not found in DHT', { did });
+        return null;
+      } catch (error) {
+        logger.warn('Failed to query Agent Card', { did, error });
+        return null;
+      }
+    },
+
+    queryByCapability: async (capability: string) => {
+      try {
+        const dht = (libp2p as any).services?.dht;
+
+        // 1. Check local index first
+        const local = searchEngine.getAllIndexedCards().filter(card =>
+          card.capabilities.some((cap: any) => {
+            const name = typeof cap === 'string' ? cap : cap.name;
+            return name?.toLowerCase().includes(capability.toLowerCase());
+          })
+        );
+        if (local.length > 0) return local;
+
+        // 2. Fall back to DHT capability index
+        if (!dht) return [];
+
+        const capDHTKey = uint8ArrayFromString(`/clawiverse/cap/${capKey(capability)}`);
+        const dids = await readDIDList(dht, capDHTKey);
+        logger.debug('DHT capability index', { capability, dids });
+
+        // Fetch each Agent Card in parallel
+        const cards = await Promise.all(
+          dids.map(async (did) => {
+            const key = uint8ArrayFromString(`/clawiverse/agent/${did}`);
+            try {
+              for await (const event of dht.get(key)) {
+                const raw = extractValue(event);
+                if (raw) {
+                  const card = decodeFromCBOR(raw);
+                  searchEngine.indexAgentCard(card);
+                  return card;
+                }
+              }
+            } catch { /* skip unreachable */ }
+            return null;
+          })
+        );
+
+        return cards.filter((c): c is AgentCard => c !== null);
+      } catch (error) {
+        throw new DiscoveryError('Failed to query by capability', error);
+      }
+    },
+
+    searchSemantic: async (query: SemanticQuery) => {
+      try {
+        // Always pull from DHT __all__ index to discover remote nodes
+        const dht = (libp2p as any).services?.dht;
+        if (dht) {
+          const allKey = uint8ArrayFromString('/clawiverse/cap/__all__');
+          const dids = await readDIDList(dht, allKey);
+          logger.debug('DHT __all__ index', { count: dids.length });
+
+          await Promise.all(
+            dids.map(async (did) => {
+              const key = uint8ArrayFromString(`/clawiverse/agent/${did}`);
+              try {
+                for await (const event of dht.get(key)) {
+                  const raw = extractValue(event);
+                  if (raw) {
+                    searchEngine.indexAgentCard(decodeFromCBOR(raw));
+                    break;
+                  }
+                }
+              } catch { /* skip */ }
+            })
+          );
+        }
+
+        return searchEngine.search(query);
+      } catch (error) {
+        throw new DiscoveryError('Failed to perform semantic search', error);
+      }
+    },
+
+    queryRelayPeers: async (): Promise<string[]> => {
+      const dht = (libp2p as any).services?.dht;
+      if (!dht) return [];
+      const capDHTKey = uint8ArrayFromString('/clawiverse/cap/relay');
+      const dids = await readDIDList(dht, capDHTKey);
+      const addrs: string[] = [];
+      await Promise.all(dids.map(async (did) => {
+        const card = await operations.queryAgentCard(did);
+        if (card?.endpoints) {
+          addrs.push(...card.endpoints.filter((e: string) => !e.includes('/p2p-circuit/')));
+        }
+      }));
+      return addrs;
+    },
+
+    resolveDID: async (did: string) => {
+      try {
+        // Check cache first
+        const cached = peerCache.get(did);
+        if (cached && Date.now() - cached.timestamp < CACHE_TTL) {
+          logger.debug('Using cached peer info', { did });
+          return { peerId: cached.peerId, multiaddrs: cached.multiaddrs };
+        }
+
+        const dht = (libp2p as any).services?.dht;
+        if (!dht) throw new DiscoveryError('DHT service not available');
+
+        const key = uint8ArrayFromString(`/clawiverse/agent/${did}`);
+        for await (const event of dht.get(key)) {
+          const raw = extractValue(event);
+          if (raw) {
+            const card = decodeFromCBOR(raw);
+            if (card.peerId) {
+              logger.debug('Resolved DID to peer', { did, peerId: card.peerId });
+              const result = { peerId: card.peerId, multiaddrs: card.endpoints || [] };
+
+              // Cache the result
+              peerCache.set(did, {
+                peerId: result.peerId,
+                multiaddrs: result.multiaddrs,
+                timestamp: Date.now()
+              });
+
+              return result;
+            }
+            logger.warn('Agent Card found but has no peerId', { did });
+            return null;
+          }
+        }
+
+        logger.debug('DID not found in DHT', { did });
+        return null;
+      } catch (error) {
+        logger.warn('Failed to resolve DID', { did, error });
+        return null;
+      }
+    },
+  });
+}

package/src/discovery/index.ts

@@ -0,0 +1,3 @@
+export * from './agent-card.js';
+export * from './dht.js';
+export * from './bootstrap.js';