ex-brain 0.1.0 → 0.2.0

package/src/ai/entity-link.ts

@@ -0,0 +1,195 @@
+import type { ResolvedLLM } from "../settings";
+import { callLLM, resolveApiKey, isLLMConfigured } from "./llm-client";
+import { jsonrepair } from "jsonrepair";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export type EntityType = "person" | "company" | "project" | "organization" | "event" | "other";
+
+export type RelationType = 
+  | "founder_of"
+  | "works_at"
+  | "leader_of"
+  | "collaborates_with"
+  | "competes_with"
+  | "acquired"
+  | "part_of"
+  | "invested_in"
+  | "mentioned_in"
+  | "related_to";
+
+export interface EntityRef {
+  name: string;
+  type: EntityType;
+}
+
+export interface EntityRelation {
+  type: "relation";
+  from: EntityRef;
+  to: EntityRef;
+  /** Semantic relation type. */
+  relation: RelationType;
+  /** The original sentence mentioning this relationship. */
+  context: string;
+  /** Confidence score 0.0 - 1.0. */
+  confidence: number;
+}
+
+export type ExtractionResult = EntityRelation[];
+
+// ---------------------------------------------------------------------------
+// Entity type mapping for slug prefix
+// ---------------------------------------------------------------------------
+
+const TYPE_PREFIX: Record<EntityType, string> = {
+  person: "people",
+  company: "companies",
+  project: "projects",
+  organization: "organizations",
+  event: "events",
+  other: "entities",
+};
+
+/**
+ * Convert an entity name to a slug: "Ali Partovi" → "ali-partovi"
+ */
+export function entityToSlug(name: string, type: EntityType): string {
+  const prefix = TYPE_PREFIX[type] ?? "entities";
+  const slugPart = name
+    .toLowerCase()
+    .replace(/[^a-z0-9\u4e00-\u9fff]+/g, "-")
+    .replace(/^-+|-+$/g, "");
+  return `${prefix}/${slugPart || "untitled"}`;
+}
+
+// ---------------------------------------------------------------------------
+// LLM extraction
+// ---------------------------------------------------------------------------
+
+const RELATION_TYPES = [
+  "founder_of", "works_at", "leader_of", 
+  "collaborates_with", "competes_with", "acquired", 
+  "part_of", "invested_in", "mentioned_in", "related_to"
+].join(", ");
+
+/**
+ * Use the configured LLM to extract entity relationships from text.
+ * Returns a list of relations with relation type, confidence, and context.
+ * Filters out relations with confidence below the threshold (default: 0.7).
+ */
+export async function extractRelations(
+  content: string,
+  llm: ResolvedLLM,
+  options?: {
+    /** Minimum confidence threshold (0-1). Relations below this are filtered out. Default: 0.7 */
+    confidenceThreshold?: number;
+  },
+): Promise<ExtractionResult> {
+  const trimmed = content.trim();
+  if (!trimmed) return [];
+
+  // Truncate for API efficiency: first 4000 + last 1000 chars
+  let context: string;
+  if (trimmed.length <= 5000) {
+    context = trimmed;
+  } else {
+    context = trimmed.slice(0, 4000) + "\n\n...\n\n" + trimmed.slice(-1000);
+  }
+
+  if (!isLLMConfigured(llm)) return [];
+
+  const systemPrompt =
+    "You are a knowledge graph extraction assistant. " +
+    "Identify relationships between named entities. " +
+    "For each relationship, provide: from entity, to entity, relation type, confidence score, and exact context sentence. " +
+    `Allowed relation types: ${RELATION_TYPES}. ` +
+    "Output ONLY a JSON array. Schema: " +
+    '{ "type": "relation", "from": {"name": "...", "type": "..."}, ' +
+    '"to": {"name": "...", "type": "..."}, "relation": "...", "context": "...", "confidence": 0.9 }. ' +
+    "Output ONLY the JSON array. /no_think";
+
+  const resp = await callLLM(llm, `Extract relationships from:\n\n${context}`, 1024, systemPrompt);
+  if (!resp) return [];
+
+  // Extract JSON array from response
+  const match = resp.match(/\[[\s\S]*\]/);
+  if (!match) return [];
+
+  try {
+    // Use jsonrepair to fix common LLM JSON issues (unterminated strings, etc.)
+    const repaired = jsonrepair(match[0]);
+    const parsed = JSON.parse(repaired) as unknown[];
+    const relations: ExtractionResult = [];
+
+    for (const item of parsed) {
+      if (typeof item !== "object" || item === null) continue;
+      const r = item as Record<string, unknown>;
+      if (r.type !== "relation") continue;
+
+      const fromRef = parseEntityRef(r.from);
+      const toRef = parseEntityRef(r.to);
+      const relation = String(r.relation || "related_to");
+      const contextStr = typeof r.context === "string" ? r.context.trim() : "";
+      const confidence = typeof r.confidence === "number" ? r.confidence : 0.8;
+
+      if (!fromRef || !toRef || !contextStr) continue;
+
+      relations.push({
+        type: "relation",
+        from: fromRef,
+        to: toRef,
+        relation: normalizeRelationType(relation),
+        context: contextStr,
+        confidence,
+      });
+    }
+
+    // Filter by confidence threshold (default 0.7)
+    const threshold = options?.confidenceThreshold ?? 0.7;
+    return relations.filter((r) => r.confidence >= threshold);
+  } catch (error) {
+    const msg = error instanceof Error ? error.message : String(error);
+    console.warn(`[ebrain] Entity extraction error: ${msg}`);
+    return [];
+  }
+}
+
+function parseEntityRef(val: unknown): EntityRef | null {
+  if (typeof val !== "object" || val === null) return null;
+  const obj = val as Record<string, unknown>;
+  const name = typeof obj.name === "string" ? obj.name.trim() : "";
+  const rawType = typeof obj.type === "string" ? obj.type : "other";
+  if (!name) return null;
+  return { name, type: normalizeEntityType(rawType) };
+}
+
+function normalizeEntityType(raw: string): EntityType {
+  const lower = raw.toLowerCase().trim();
+  if (lower.includes("person") || lower.includes("people")) return "person";
+  if (lower.includes("company") || lower.includes("corp") || lower.includes("business")) return "company";
+  if (lower.includes("project")) return "project";
+  if (lower.includes("organization") || lower.includes("org") || lower.includes("ngo")) return "organization";
+  if (lower.includes("event")) return "event";
+  return "other";
+}
+
+export function normalizeRelationType(raw: string): RelationType {
+  const lower = raw.toLowerCase().trim().replace(/-/g, "_");
+  const validTypes = RELATION_TYPES.split(", ");
+  if (validTypes.includes(lower as RelationType)) return lower as RelationType;
+  // Fallbacks
+  if (lower.includes("founder") || lower.includes("create")) return "founder_of";
+  if (lower.includes("work") || lower.includes("join")) return "works_at";
+  if (lower.includes("lead") || lower.includes("head") || lower.includes("manage")) return "leader_of";
+  if (lower.includes("collabor") || lower.includes("partner")) return "collaborates_with";
+  if (lower.includes("compet")) return "competes_with";
+  if (lower.includes("acquir") || lower.includes("buy")) return "acquired";
+  if (lower.includes("invest")) return "invested_in";
+  if (lower.includes("part") || lower.includes("belong")) return "part_of";
+  if (lower.includes("mention") || lower.includes("refer")) return "mentioned_in";
+  return "related_to";
+}
+
+

package/src/ai/hash-embed.ts

@@ -0,0 +1,30 @@
+import type { EmbeddingConfig, EmbeddingFunction } from "seekdb";
+
+const DIM = 384;
+
+function hashToVector(text: string): number[] {
+  const v = new Array<number>(DIM).fill(0);
+  for (let i = 0; i < text.length; i += 1) {
+    const j = i % DIM;
+    v[j] = (v[j]! + text.charCodeAt(i) * (i + 1)) / 1e6;
+  }
+  const norm = Math.sqrt(v.reduce((s, x) => s + x * x, 0)) || 1;
+  return v.map((x) => x / norm);
+}
+
+/**
+ * 零依赖、确定性的伪向量，满足 seekdb 集合对「文档 → 向量」的硬性要求；
+ * 不替代真实语义模型，仅用于嵌入模式本地可跑通全文 + 近似检索管线。
+ */
+export class LocalHashEmbeddingFunction implements EmbeddingFunction {
+  readonly name = "ebrain-local-hash";
+  readonly dimension = DIM;
+
+  async generate(texts: string[]): Promise<number[][]> {
+    return texts.map((t) => hashToVector(t));
+  }
+
+  getConfig(): EmbeddingConfig {
+    return { dimension: DIM };
+  }
+}

package/src/ai/llm-client.ts

@@ -0,0 +1,291 @@
+/**
+ * Unified LLM Client Module
+ * 
+ * Provides centralized LLM calling functionality with:
+ * - Retry mechanism (exponential backoff, max 3 retries)
+ * - Error classification (APIError, TimeoutError, RateLimitError)
+ * - Timeout control
+ * - Unified API key resolution
+ */
+
+import type { ResolvedLLM } from "../settings";
+
+// ---------------------------------------------------------------------------
+// Error Classes
+// ---------------------------------------------------------------------------
+
+export class LLMError extends Error {
+  constructor(
+    message: string,
+    public readonly code: string,
+    public readonly statusCode?: number,
+    public readonly retryable: boolean = false,
+  ) {
+    super(message);
+    this.name = "LLMError";
+  }
+}
+
+export class APIError extends LLMError {
+  constructor(message: string, statusCode?: number) {
+    super(message, "API_ERROR", statusCode, false);
+    this.name = "APIError";
+  }
+}
+
+export class TimeoutError extends LLMError {
+  constructor(message: string = "LLM request timed out") {
+    super(message, "TIMEOUT_ERROR", undefined, true);
+    this.name = "TimeoutError";
+  }
+}
+
+export class RateLimitError extends LLMError {
+  constructor(message: string = "Rate limit exceeded", retryAfter?: number) {
+    super(message, "RATE_LIMIT_ERROR", 429, true);
+    this.name = "RateLimitError";
+    this.retryAfter = retryAfter;
+  }
+  readonly retryAfter?: number;
+}
+
+// ---------------------------------------------------------------------------
+// Configuration
+// ---------------------------------------------------------------------------
+
+export interface LLMClientConfig {
+  /** Maximum number of retry attempts (default: 3) */
+  maxRetries?: number;
+  /** Base delay for exponential backoff in ms (default: 1000) */
+  baseDelay?: number;
+  /** Maximum delay cap in ms (default: 10000) */
+  maxDelay?: number;
+  /** Request timeout in ms (default: 60000) */
+  timeout?: number;
+}
+
+const DEFAULT_CONFIG: Required<LLMClientConfig> = {
+  maxRetries: 3,
+  baseDelay: 1000,
+  maxDelay: 10000,
+  timeout: 60000,
+};
+
+// ---------------------------------------------------------------------------
+// API Key Resolution
+// ---------------------------------------------------------------------------
+
+/**
+ * Resolve API key from LLM configuration.
+ * Checks direct apiKey first, then falls back to environment variable.
+ */
+export function resolveApiKey(llm: ResolvedLLM): string {
+  if (llm.apiKey) return llm.apiKey;
+  if (llm.apiKeyEnv) return process.env[llm.apiKeyEnv] ?? "";
+  return "";
+}
+
+/**
+ * Check if LLM is properly configured with an API key.
+ */
+export function isLLMConfigured(llm: ResolvedLLM): boolean {
+  return !!resolveApiKey(llm);
+}
+
+// ---------------------------------------------------------------------------
+// LLM Call with Retry
+// ---------------------------------------------------------------------------
+
+/**
+ * Call LLM with unified fetch, retry mechanism, error handling, and timeout.
+ * 
+ * @param llm - Resolved LLM configuration
+ * @param prompt - Prompt to send to the LLM
+ * @param maxTokens - Maximum tokens in response
+ * @param systemPrompt - Optional system prompt (default provided)
+ * @param config - Optional client configuration
+ * @returns Raw response text from LLM, or empty string on failure
+ */
+export async function callLLM(
+  llm: ResolvedLLM,
+  prompt: string,
+  maxTokens: number,
+  systemPrompt: string = "You are a helpful assistant. Always output valid JSON.",
+  config: LLMClientConfig = {},
+): Promise<string> {
+  const apiKey = resolveApiKey(llm);
+  if (!apiKey) {
+    return "";
+  }
+
+  const cfg = { ...DEFAULT_CONFIG, ...config };
+  const url = llm.baseURL.endsWith("/") 
+    ? llm.baseURL + "chat/completions" 
+    : llm.baseURL + "/chat/completions";
+
+  const body = {
+    model: llm.model,
+    messages: [
+      { role: "system", content: systemPrompt },
+      { role: "user", content: prompt },
+    ],
+    temperature: 0.1,
+    max_tokens: maxTokens,
+    enable_thinking: false,
+  };
+
+  let lastError: LLMError | null = null;
+
+  for (let attempt = 0; attempt <= cfg.maxRetries; attempt++) {
+    try {
+      const response = await callWithTimeout(
+        fetch(url, {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${apiKey}`,
+          },
+          body: JSON.stringify(body),
+        }),
+        cfg.timeout,
+      );
+
+      if (!response.ok) {
+        const text = await response.text().catch(() => "");
+        lastError = classifyError(response.status, text, response.statusText);
+        
+        // Don't retry for non-retryable errors
+        if (!lastError.retryable || attempt === cfg.maxRetries) {
+          console.warn(`[llm-client] LLM call failed after ${attempt + 1} attempt(s): ${lastError.message}`);
+          return "";
+        }
+        
+        const delay = calculateBackoff(attempt, cfg.baseDelay, cfg.maxDelay, (lastError as RateLimitError).retryAfter);
+        console.warn(`[llm-client] Retrying after ${delay}ms (attempt ${attempt + 1}/${cfg.maxRetries})`);
+        await sleep(delay);
+        continue;
+      }
+
+      const data = await response.json() as { choices?: Array<{ message?: { content?: string } }> };
+      return data.choices?.[0]?.message?.content?.trim() ?? "";
+      
+    } catch (error) {
+      // Classify the error
+      if (error instanceof TimeoutError) {
+        lastError = error;
+      } else if (error instanceof LLMError) {
+        lastError = error;
+      } else {
+        // Unknown error - wrap it
+        const msg = error instanceof Error ? error.message : String(error);
+        lastError = new APIError(`Unexpected error: ${msg}`);
+      }
+
+      // Don't retry if we've exhausted attempts
+      if (attempt === cfg.maxRetries) {
+        console.warn(`[llm-client] LLM call failed after ${attempt + 1} attempt(s): ${lastError.message}`);
+        return "";
+      }
+
+      // Check if error is retryable
+      if (!lastError.retryable) {
+        console.warn(`[llm-client] Non-retryable error: ${lastError.message}`);
+        return "";
+      }
+
+      const delay = calculateBackoff(attempt, cfg.baseDelay, cfg.maxDelay);
+      console.warn(`[llm-client] Retrying after ${delay}ms (attempt ${attempt + 1}/${cfg.maxRetries}): ${lastError.message}`);
+      await sleep(delay);
+    }
+  }
+
+  return "";
+}
+
+/**
+ * Classify HTTP error into appropriate error type.
+ */
+function classifyError(status: number, responseText: string, statusText: string): LLMError {
+  const truncatedText = responseText.slice(0, 200);
+  
+  switch (status) {
+    case 429:
+      // Try to extract retry-after from response
+      const retryAfterMatch = responseText.match(/retry[- ]?after["']?\s*[:=]\s*(\d+)/i);
+      const retryAfter = retryAfterMatch?.[1] ? parseInt(retryAfterMatch[1], 10) : undefined;
+      return new RateLimitError(`Rate limited: ${statusText} - ${truncatedText}`, retryAfter);
+    
+    case 408:
+    case 504:
+      return new TimeoutError(`Request timeout: ${statusText}`);
+    
+    case 500:
+    case 502:
+    case 503:
+      return new APIError(`Server error (${status}): ${truncatedText}`, status);
+    
+    default:
+      if (status >= 500) {
+        return new APIError(`Server error (${status}): ${truncatedText}`, status);
+      }
+      if (status >= 400) {
+        return new APIError(`Client error (${status}): ${truncatedText}`, status);
+      }
+      return new APIError(`HTTP error (${status}): ${truncatedText}`, status);
+  }
+}
+
+/**
+ * Calculate exponential backoff delay with jitter.
+ */
+function calculateBackoff(
+  attempt: number,
+  baseDelay: number,
+  maxDelay: number,
+  retryAfter?: number,
+): number {
+  // If server specified retry-after, use that
+  if (retryAfter && retryAfter > 0) {
+    return Math.min(retryAfter * 1000, maxDelay);
+  }
+  
+  // Exponential backoff: baseDelay * 2^attempt
+  const exponentialDelay = baseDelay * Math.pow(2, attempt);
+  
+  // Add jitter (±25%)
+  const jitter = exponentialDelay * 0.25 * (Math.random() * 2 - 1);
+  
+  return Math.min(Math.round(exponentialDelay + jitter), maxDelay);
+}
+
+/**
+ * Sleep for specified milliseconds.
+ */
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Wrap fetch with timeout using Promise.race.
+ */
+async function callWithTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
+  let timeoutId: NodeJS.Timeout;
+  
+  const timeoutPromise = new Promise<never>((_, reject) => {
+    timeoutId = setTimeout(() => {
+      reject(new TimeoutError(`Request timed out after ${timeoutMs}ms`));
+    }, timeoutMs);
+  });
+
+  try {
+    return await Promise.race([promise, timeoutPromise]);
+  } finally {
+    clearTimeout(timeoutId!);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Re-export settings type for convenience
+// ---------------------------------------------------------------------------
+
+export type { ResolvedLLM } from "../settings";