@andreprado/agentkit 0.1.0-alpha.5 → 0.1.0-alpha.6

package/src/runtime/knowledge/config.ts

@@ -0,0 +1,135 @@
+import type {
+  AgentConfig,
+  AgentKnowledgeConfig,
+  AgentKnowledgeEmbeddingProvider,
+  AgentKnowledgeSource,
+} from "../../index";
+
+export type ResolvedKnowledgeSource =
+  | {
+      kind: "file";
+      value: string;
+      title: string | null;
+    }
+  | {
+      kind: "url";
+      value: string;
+      title: string | null;
+    };
+
+export type ResolvedKnowledgeConfig = {
+  enabled: boolean;
+  sources: ResolvedKnowledgeSource[];
+  embedding: {
+    provider: AgentKnowledgeEmbeddingProvider;
+    model: string | null;
+    dimensions: number | null;
+    secret: string | null;
+  };
+  retrieval: {
+    topK: number;
+    hybrid: boolean;
+  };
+};
+
+const DEFAULT_TOP_K = 8;
+const DEFAULT_TEST_EMBEDDING_DIMENSIONS = 32;
+const DEFAULT_OPENAI_EMBEDDING_MODEL = "text-embedding-3-small";
+const DEFAULT_OPENAI_EMBEDDING_DIMENSIONS = 1536;
+
+export function resolveKnowledgeConfig(config: Pick<AgentConfig, "knowledge">): ResolvedKnowledgeConfig {
+  const knowledge = config.knowledge;
+
+  if (!knowledge) {
+    return {
+      enabled: false,
+      sources: [],
+      embedding: {
+        provider: "none",
+        model: null,
+        dimensions: null,
+        secret: null,
+      },
+      retrieval: {
+        topK: DEFAULT_TOP_K,
+        hybrid: false,
+      },
+    };
+  }
+
+  const provider = knowledge.embedding?.provider ?? "none";
+  const embedding = resolveEmbeddingConfig(provider, knowledge);
+  const hybrid = knowledge.retrieval?.hybrid ?? provider !== "none";
+
+  return {
+    enabled: true,
+    sources: (knowledge.sources ?? []).map(resolveKnowledgeSource),
+    embedding,
+    retrieval: {
+      topK: knowledge.retrieval?.topK ?? DEFAULT_TOP_K,
+      hybrid,
+    },
+  };
+}
+
+function resolveEmbeddingConfig(
+  provider: AgentKnowledgeEmbeddingProvider,
+  knowledge: AgentKnowledgeConfig,
+): ResolvedKnowledgeConfig["embedding"] {
+  if (provider === "openai") {
+    return {
+      provider,
+      model: knowledge.embedding?.model ?? DEFAULT_OPENAI_EMBEDDING_MODEL,
+      dimensions: knowledge.embedding?.dimensions ?? DEFAULT_OPENAI_EMBEDDING_DIMENSIONS,
+      secret: knowledge.embedding?.secret ?? "OPENAI_API_KEY",
+    };
+  }
+
+  if (provider === "test") {
+    return {
+      provider,
+      model: knowledge.embedding?.model ?? "fake",
+      dimensions: knowledge.embedding?.dimensions ?? DEFAULT_TEST_EMBEDDING_DIMENSIONS,
+      secret: knowledge.embedding?.secret ?? null,
+    };
+  }
+
+  return {
+    provider: "none",
+    model: knowledge.embedding?.model ?? null,
+    dimensions: knowledge.embedding?.dimensions ?? null,
+    secret: knowledge.embedding?.secret ?? null,
+  };
+}
+
+function resolveKnowledgeSource(source: AgentKnowledgeSource): ResolvedKnowledgeSource {
+  if (typeof source === "string") {
+    if (/^https?:\/\//i.test(source)) {
+      return {
+        kind: "url",
+        value: source,
+        title: null,
+      };
+    }
+
+    return {
+      kind: "file",
+      value: source,
+      title: null,
+    };
+  }
+
+  if ("url" in source) {
+    return {
+      kind: "url",
+      value: source.url,
+      title: source.title ?? null,
+    };
+  }
+
+  return {
+    kind: "file",
+    value: source.path,
+    title: source.title ?? null,
+  };
+}

package/src/runtime/knowledge/embeddings.ts

@@ -0,0 +1,133 @@
+import { createHash } from "node:crypto";
+
+import type { ResolvedKnowledgeConfig } from "./config";
+import { AgentKitError } from "../errors";
+
+export type EmbeddingProvider = {
+  embed(input: string): Promise<number[]>;
+};
+
+export type CreateEmbeddingProviderOptions = {
+  config: ResolvedKnowledgeConfig["embedding"];
+  env?: Record<string, string | undefined>;
+  fetch?: typeof fetch;
+};
+
+export function createEmbeddingProvider(options: CreateEmbeddingProviderOptions): EmbeddingProvider | null {
+  if (options.config.provider === "none") {
+    return null;
+  }
+
+  if (options.config.provider === "test") {
+    return {
+      async embed(input) {
+        return fakeEmbedding(input, options.config.dimensions ?? 32);
+      },
+    };
+  }
+
+  if (options.config.provider === "openai") {
+    return {
+      async embed(input) {
+        return openAiEmbedding(input, options);
+      },
+    };
+  }
+
+  throw new AgentKitError("knowledge_embedding_provider_unsupported", "Unsupported knowledge embedding provider.");
+}
+
+export function fakeEmbedding(input: string, dimensions: number): number[] {
+  const values = Array.from({ length: dimensions }, () => 0);
+  const tokens = input.toLowerCase().match(/[\p{L}\p{N}_-]+/gu) ?? [];
+
+  for (const token of tokens) {
+    const hash = createHash("sha256").update(token).digest();
+    const index = hash[0] % dimensions;
+    const sign = hash[1] % 2 === 0 ? 1 : -1;
+    const weight = 1 + (hash[2] % 7) / 10;
+    values[index] += sign * weight;
+  }
+
+  return normalizeVector(values);
+}
+
+export function cosineSimilarity(left: number[], right: number[]): number {
+  if (left.length !== right.length || left.length === 0) {
+    return 0;
+  }
+
+  let dot = 0;
+  let leftNorm = 0;
+  let rightNorm = 0;
+
+  for (let index = 0; index < left.length; index += 1) {
+    dot += left[index] * right[index];
+    leftNorm += left[index] * left[index];
+    rightNorm += right[index] * right[index];
+  }
+
+  if (leftNorm === 0 || rightNorm === 0) {
+    return 0;
+  }
+
+  return dot / (Math.sqrt(leftNorm) * Math.sqrt(rightNorm));
+}
+
+async function openAiEmbedding(input: string, options: CreateEmbeddingProviderOptions): Promise<number[]> {
+  const secretName = options.config.secret ?? "OPENAI_API_KEY";
+  const apiKey = options.env?.[secretName] ?? process.env[secretName];
+
+  if (!apiKey) {
+    throw new AgentKitError(
+      "knowledge_embedding_secret_missing",
+      `Knowledge embeddings require missing secret "${secretName}". Set it in .env or the process environment.`,
+    );
+  }
+
+  const fetchImpl = options.fetch ?? fetch;
+  const response = await fetchImpl("https://api.openai.com/v1/embeddings", {
+    method: "POST",
+    headers: {
+      "content-type": "application/json",
+      authorization: `Bearer ${apiKey}`,
+    },
+    body: JSON.stringify({
+      model: options.config.model ?? "text-embedding-3-small",
+      input,
+      encoding_format: "float",
+      ...(options.config.dimensions ? { dimensions: options.config.dimensions } : {}),
+    }),
+  });
+
+  if (!response.ok) {
+    const text = await response.text().catch(() => "");
+    throw new AgentKitError(
+      "knowledge_embedding_failed",
+      `OpenAI embedding request failed with HTTP ${response.status}.${text ? ` ${truncate(text, 240)}` : ""}`,
+    );
+  }
+
+  const payload = await response.json();
+  const embedding = payload?.data?.[0]?.embedding;
+
+  if (!Array.isArray(embedding) || embedding.some((value) => typeof value !== "number")) {
+    throw new AgentKitError("knowledge_embedding_failed", "OpenAI embedding response did not contain a numeric vector.");
+  }
+
+  return embedding;
+}
+
+function normalizeVector(values: number[]): number[] {
+  const norm = Math.sqrt(values.reduce((sum, value) => sum + value * value, 0));
+
+  if (norm === 0) {
+    return values;
+  }
+
+  return values.map((value) => Number((value / norm).toFixed(8)));
+}
+
+function truncate(value: string, maxLength: number): string {
+  return value.length <= maxLength ? value : `${value.slice(0, maxLength - 3)}...`;
+}