@andreprado/agentkit 0.1.0-alpha.5 → 0.1.0-alpha.7

package/src/runtime/knowledge/chunk.ts

@@ -0,0 +1,333 @@
+import { basename, extname } from "node:path";
+
+export type KnowledgeChunkMetadata = {
+  sourcePath: string;
+  title: string;
+  section: string | null;
+  locator: string | null;
+};
+
+export type KnowledgeChunk = {
+  ordinal: number;
+  content: string;
+  tokenCount: number;
+  metadata: KnowledgeChunkMetadata;
+};
+
+export type ChunkKnowledgeSourceInput = {
+  sourcePath: string;
+  title?: string | null;
+  content: string;
+};
+
+const DEFAULT_MAX_WORDS = 720;
+const DEFAULT_OVERLAP_WORDS = 80;
+
+export function chunkKnowledgeSource(input: ChunkKnowledgeSourceInput): KnowledgeChunk[] {
+  const extension = extname(input.sourcePath).toLowerCase();
+
+  if (extension === ".csv") {
+    return chunkCsv(input);
+  }
+
+  if (extension === ".md" || extension === ".markdown") {
+    return chunkMarkdown(input);
+  }
+
+  return chunkText(input);
+}
+
+function chunkMarkdown(input: ChunkKnowledgeSourceInput): KnowledgeChunk[] {
+  const title = input.title ?? inferTitle(input.sourcePath, input.content);
+  const sections = markdownSections(input.content, title);
+  const chunks: KnowledgeChunk[] = [];
+
+  for (const section of sections) {
+    const contextualContent = [section.heading, section.body].filter(Boolean).join("\n\n").trim();
+    chunks.push(
+      ...splitContentIntoChunks({
+        sourcePath: input.sourcePath,
+        title,
+        section: section.heading || null,
+        locator: section.heading ? `section:${section.heading}` : null,
+        content: contextualContent,
+        startingOrdinal: chunks.length,
+      }),
+    );
+  }
+
+  return ensureAtLeastOneChunk(chunks, input, title);
+}
+
+function chunkText(input: ChunkKnowledgeSourceInput): KnowledgeChunk[] {
+  const title = input.title ?? inferTitle(input.sourcePath, input.content);
+  const paragraphs = input.content
+    .split(/\n{2,}/)
+    .map((paragraph) => paragraph.trim())
+    .filter(Boolean);
+  const content = paragraphs.length > 0 ? paragraphs.join("\n\n") : input.content.trim();
+
+  return ensureAtLeastOneChunk(
+    splitContentIntoChunks({
+      sourcePath: input.sourcePath,
+      title,
+      section: null,
+      locator: null,
+      content,
+      startingOrdinal: 0,
+    }),
+    input,
+    title,
+  );
+}
+
+function chunkCsv(input: ChunkKnowledgeSourceInput): KnowledgeChunk[] {
+  const title = input.title ?? inferTitle(input.sourcePath, input.content);
+  const rows = parseCsv(input.content);
+
+  if (rows.length === 0) {
+    return ensureAtLeastOneChunk([], input, title);
+  }
+
+  const [headers, ...records] = rows;
+
+  if (headers.length === 0 || records.length === 0) {
+    return chunkText(input);
+  }
+
+  const chunks: KnowledgeChunk[] = [];
+
+  for (const [index, record] of records.entries()) {
+    const content = headers
+      .map((header, columnIndex) => {
+        const name = header.trim() || `column_${columnIndex + 1}`;
+        const value = record[columnIndex]?.trim() ?? "";
+        return `${name}: ${value}`;
+      })
+      .join("\n")
+      .trim();
+
+    if (!content) {
+      continue;
+    }
+
+    chunks.push({
+      ordinal: chunks.length,
+      content,
+      tokenCount: estimateTokenCount(content),
+      metadata: {
+        sourcePath: input.sourcePath,
+        title,
+        section: null,
+        locator: `row:${index + 2}`,
+      },
+    });
+  }
+
+  return ensureAtLeastOneChunk(chunks, input, title);
+}
+
+function markdownSections(content: string, fallbackTitle: string): Array<{ heading: string | null; body: string }> {
+  const sections: Array<{ heading: string | null; body: string[] }> = [];
+  let current: { heading: string | null; body: string[] } = { heading: null, body: [] };
+  const headingStack: string[] = [];
+
+  for (const line of content.split(/\r?\n/)) {
+    const heading = line.match(/^(#{1,6})\s+(.+?)\s*#*\s*$/);
+
+    if (heading) {
+      if (current.body.join("\n").trim()) {
+        sections.push(current);
+      }
+
+      const level = heading[1].length;
+      headingStack.splice(level - 1);
+      headingStack[level - 1] = heading[2].trim();
+      current = {
+        heading: headingStack.filter(Boolean).join(" > "),
+        body: [],
+      };
+      continue;
+    }
+
+    current.body.push(line);
+  }
+
+  if (current.body.join("\n").trim()) {
+    sections.push(current);
+  }
+
+  if (sections.length === 0) {
+    return [{ heading: fallbackTitle, body: content.trim() }];
+  }
+
+  return sections.map((section) => ({
+    heading: section.heading,
+    body: section.body.join("\n").trim(),
+  }));
+}
+
+function splitContentIntoChunks(input: {
+  sourcePath: string;
+  title: string;
+  section: string | null;
+  locator: string | null;
+  content: string;
+  startingOrdinal: number;
+}): KnowledgeChunk[] {
+  const words = wordsOf(input.content);
+
+  if (words.length <= DEFAULT_MAX_WORDS) {
+    const content = input.content.trim();
+
+    if (!content) {
+      return [];
+    }
+
+    return [
+      {
+        ordinal: input.startingOrdinal,
+        content,
+        tokenCount: estimateTokenCount(content),
+        metadata: {
+          sourcePath: input.sourcePath,
+          title: input.title,
+          section: input.section,
+          locator: input.locator,
+        },
+      },
+    ];
+  }
+
+  const chunks: KnowledgeChunk[] = [];
+  let start = 0;
+
+  while (start < words.length) {
+    const end = Math.min(start + DEFAULT_MAX_WORDS, words.length);
+    const content = words.slice(start, end).join(" ");
+
+    chunks.push({
+      ordinal: input.startingOrdinal + chunks.length,
+      content,
+      tokenCount: estimateTokenCount(content),
+      metadata: {
+        sourcePath: input.sourcePath,
+        title: input.title,
+        section: input.section,
+        locator: input.locator ? `${input.locator}:part:${chunks.length + 1}` : `part:${chunks.length + 1}`,
+      },
+    });
+
+    if (end === words.length) {
+      break;
+    }
+
+    start = Math.max(end - DEFAULT_OVERLAP_WORDS, start + 1);
+  }
+
+  return chunks;
+}
+
+function ensureAtLeastOneChunk(
+  chunks: KnowledgeChunk[],
+  input: ChunkKnowledgeSourceInput,
+  title: string,
+): KnowledgeChunk[] {
+  if (chunks.length > 0) {
+    return chunks.map((chunk, index) => ({ ...chunk, ordinal: index }));
+  }
+
+  const content = input.content.trim();
+
+  if (!content) {
+    return [];
+  }
+
+  return [
+    {
+      ordinal: 0,
+      content,
+      tokenCount: estimateTokenCount(content),
+      metadata: {
+        sourcePath: input.sourcePath,
+        title,
+        section: null,
+        locator: null,
+      },
+    },
+  ];
+}
+
+function parseCsv(content: string): string[][] {
+  const rows: string[][] = [];
+  let row: string[] = [];
+  let field = "";
+  let quoted = false;
+
+  for (let index = 0; index < content.length; index += 1) {
+    const char = content[index];
+    const next = content[index + 1];
+
+    if (quoted) {
+      if (char === '"' && next === '"') {
+        field += '"';
+        index += 1;
+        continue;
+      }
+
+      if (char === '"') {
+        quoted = false;
+        continue;
+      }
+
+      field += char;
+      continue;
+    }
+
+    if (char === '"') {
+      quoted = true;
+      continue;
+    }
+
+    if (char === ",") {
+      row.push(field);
+      field = "";
+      continue;
+    }
+
+    if (char === "\n") {
+      row.push(field.replace(/\r$/, ""));
+      rows.push(row);
+      row = [];
+      field = "";
+      continue;
+    }
+
+    field += char;
+  }
+
+  row.push(field.replace(/\r$/, ""));
+  if (row.some((value) => value.length > 0)) {
+    rows.push(row);
+  }
+
+  return rows;
+}
+
+function inferTitle(sourcePath: string, content: string): string {
+  const markdownTitle = content.match(/^#\s+(.+)$/m)?.[1]?.trim();
+
+  if (markdownTitle) {
+    return markdownTitle;
+  }
+
+  return basename(sourcePath).replace(/\.[^.]+$/, "") || sourcePath;
+}
+
+function wordsOf(content: string): string[] {
+  return content.replace(/\s+/g, " ").trim().split(" ").filter(Boolean);
+}
+
+export function estimateTokenCount(content: string): number {
+  return Math.max(1, wordsOf(content).length);
+}

package/src/runtime/knowledge/config.ts

@@ -0,0 +1,135 @@
+import type {
+  AgentConfig,
+  AgentKnowledgeConfig,
+  AgentKnowledgeEmbeddingProvider,
+  AgentKnowledgeSource,
+} from "../../index";
+
+export type ResolvedKnowledgeSource =
+  | {
+      kind: "file";
+      value: string;
+      title: string | null;
+    }
+  | {
+      kind: "url";
+      value: string;
+      title: string | null;
+    };
+
+export type ResolvedKnowledgeConfig = {
+  enabled: boolean;
+  sources: ResolvedKnowledgeSource[];
+  embedding: {
+    provider: AgentKnowledgeEmbeddingProvider;
+    model: string | null;
+    dimensions: number | null;
+    secret: string | null;
+  };
+  retrieval: {
+    topK: number;
+    hybrid: boolean;
+  };
+};
+
+const DEFAULT_TOP_K = 8;
+const DEFAULT_TEST_EMBEDDING_DIMENSIONS = 32;
+const DEFAULT_OPENAI_EMBEDDING_MODEL = "text-embedding-3-small";
+const DEFAULT_OPENAI_EMBEDDING_DIMENSIONS = 1536;
+
+export function resolveKnowledgeConfig(config: Pick<AgentConfig, "knowledge">): ResolvedKnowledgeConfig {
+  const knowledge = config.knowledge;
+
+  if (!knowledge) {
+    return {
+      enabled: false,
+      sources: [],
+      embedding: {
+        provider: "none",
+        model: null,
+        dimensions: null,
+        secret: null,
+      },
+      retrieval: {
+        topK: DEFAULT_TOP_K,
+        hybrid: false,
+      },
+    };
+  }
+
+  const provider = knowledge.embedding?.provider ?? "none";
+  const embedding = resolveEmbeddingConfig(provider, knowledge);
+  const hybrid = knowledge.retrieval?.hybrid ?? provider !== "none";
+
+  return {
+    enabled: true,
+    sources: (knowledge.sources ?? []).map(resolveKnowledgeSource),
+    embedding,
+    retrieval: {
+      topK: knowledge.retrieval?.topK ?? DEFAULT_TOP_K,
+      hybrid,
+    },
+  };
+}
+
+function resolveEmbeddingConfig(
+  provider: AgentKnowledgeEmbeddingProvider,
+  knowledge: AgentKnowledgeConfig,
+): ResolvedKnowledgeConfig["embedding"] {
+  if (provider === "openai") {
+    return {
+      provider,
+      model: knowledge.embedding?.model ?? DEFAULT_OPENAI_EMBEDDING_MODEL,
+      dimensions: knowledge.embedding?.dimensions ?? DEFAULT_OPENAI_EMBEDDING_DIMENSIONS,
+      secret: knowledge.embedding?.secret ?? "OPENAI_API_KEY",
+    };
+  }
+
+  if (provider === "test") {
+    return {
+      provider,
+      model: knowledge.embedding?.model ?? "fake",
+      dimensions: knowledge.embedding?.dimensions ?? DEFAULT_TEST_EMBEDDING_DIMENSIONS,
+      secret: knowledge.embedding?.secret ?? null,
+    };
+  }
+
+  return {
+    provider: "none",
+    model: knowledge.embedding?.model ?? null,
+    dimensions: knowledge.embedding?.dimensions ?? null,
+    secret: knowledge.embedding?.secret ?? null,
+  };
+}
+
+function resolveKnowledgeSource(source: AgentKnowledgeSource): ResolvedKnowledgeSource {
+  if (typeof source === "string") {
+    if (/^https?:\/\//i.test(source)) {
+      return {
+        kind: "url",
+        value: source,
+        title: null,
+      };
+    }
+
+    return {
+      kind: "file",
+      value: source,
+      title: null,
+    };
+  }
+
+  if ("url" in source) {
+    return {
+      kind: "url",
+      value: source.url,
+      title: source.title ?? null,
+    };
+  }
+
+  return {
+    kind: "file",
+    value: source.path,
+    title: source.title ?? null,
+  };
+}

package/src/runtime/knowledge/embeddings.ts

@@ -0,0 +1,133 @@
+import { createHash } from "node:crypto";
+
+import type { ResolvedKnowledgeConfig } from "./config";
+import { AgentKitError } from "../errors";
+
+export type EmbeddingProvider = {
+  embed(input: string): Promise<number[]>;
+};
+
+export type CreateEmbeddingProviderOptions = {
+  config: ResolvedKnowledgeConfig["embedding"];
+  env?: Record<string, string | undefined>;
+  fetch?: typeof fetch;
+};
+
+export function createEmbeddingProvider(options: CreateEmbeddingProviderOptions): EmbeddingProvider | null {
+  if (options.config.provider === "none") {
+    return null;
+  }
+
+  if (options.config.provider === "test") {
+    return {
+      async embed(input) {
+        return fakeEmbedding(input, options.config.dimensions ?? 32);
+      },
+    };
+  }
+
+  if (options.config.provider === "openai") {
+    return {
+      async embed(input) {
+        return openAiEmbedding(input, options);
+      },
+    };
+  }
+
+  throw new AgentKitError("knowledge_embedding_provider_unsupported", "Unsupported knowledge embedding provider.");
+}
+
+export function fakeEmbedding(input: string, dimensions: number): number[] {
+  const values = Array.from({ length: dimensions }, () => 0);
+  const tokens = input.toLowerCase().match(/[\p{L}\p{N}_-]+/gu) ?? [];
+
+  for (const token of tokens) {
+    const hash = createHash("sha256").update(token).digest();
+    const index = hash[0] % dimensions;
+    const sign = hash[1] % 2 === 0 ? 1 : -1;
+    const weight = 1 + (hash[2] % 7) / 10;
+    values[index] += sign * weight;
+  }
+
+  return normalizeVector(values);
+}
+
+export function cosineSimilarity(left: number[], right: number[]): number {
+  if (left.length !== right.length || left.length === 0) {
+    return 0;
+  }
+
+  let dot = 0;
+  let leftNorm = 0;
+  let rightNorm = 0;
+
+  for (let index = 0; index < left.length; index += 1) {
+    dot += left[index] * right[index];
+    leftNorm += left[index] * left[index];
+    rightNorm += right[index] * right[index];
+  }
+
+  if (leftNorm === 0 || rightNorm === 0) {
+    return 0;
+  }
+
+  return dot / (Math.sqrt(leftNorm) * Math.sqrt(rightNorm));
+}
+
+async function openAiEmbedding(input: string, options: CreateEmbeddingProviderOptions): Promise<number[]> {
+  const secretName = options.config.secret ?? "OPENAI_API_KEY";
+  const apiKey = options.env?.[secretName] ?? process.env[secretName];
+
+  if (!apiKey) {
+    throw new AgentKitError(
+      "knowledge_embedding_secret_missing",
+      `Knowledge embeddings require missing secret "${secretName}". Set it in .env or the process environment.`,
+    );
+  }
+
+  const fetchImpl = options.fetch ?? fetch;
+  const response = await fetchImpl("https://api.openai.com/v1/embeddings", {
+    method: "POST",
+    headers: {
+      "content-type": "application/json",
+      authorization: `Bearer ${apiKey}`,
+    },
+    body: JSON.stringify({
+      model: options.config.model ?? "text-embedding-3-small",
+      input,
+      encoding_format: "float",
+      ...(options.config.dimensions ? { dimensions: options.config.dimensions } : {}),
+    }),
+  });
+
+  if (!response.ok) {
+    const text = await response.text().catch(() => "");
+    throw new AgentKitError(
+      "knowledge_embedding_failed",
+      `OpenAI embedding request failed with HTTP ${response.status}.${text ? ` ${truncate(text, 240)}` : ""}`,
+    );
+  }
+
+  const payload = await response.json();
+  const embedding = payload?.data?.[0]?.embedding;
+
+  if (!Array.isArray(embedding) || embedding.some((value) => typeof value !== "number")) {
+    throw new AgentKitError("knowledge_embedding_failed", "OpenAI embedding response did not contain a numeric vector.");
+  }
+
+  return embedding;
+}
+
+function normalizeVector(values: number[]): number[] {
+  const norm = Math.sqrt(values.reduce((sum, value) => sum + value * value, 0));
+
+  if (norm === 0) {
+    return values;
+  }
+
+  return values.map((value) => Number((value / norm).toFixed(8)));
+}
+
+function truncate(value: string, maxLength: number): string {
+  return value.length <= maxLength ? value : `${value.slice(0, maxLength - 3)}...`;
+}