@andreprado/agentkit 0.1.0-alpha.10

package/src/runtime/inspect.ts

@@ -0,0 +1,203 @@
+import { relative } from "node:path";
+
+import type { AgentChannel, AgentProvider, AgentRuntime, AccessMode } from "../index";
+import type { LoadedAgentCapsule } from "./config";
+import { loadAgentCapsule } from "./config";
+import { loadCapsuleEnv } from "./env";
+import { resolveKnowledgeConfig } from "./knowledge/config";
+import { KNOWLEDGE_SEARCH_TOOL_NAME } from "./knowledge/tool";
+
+export type SecretState = "set" | "missing";
+
+export type AgentInspectState = {
+  agent: string;
+  runtime: AgentRuntime;
+  provider: AgentProvider;
+  prompt: string;
+  tools: string[];
+  channels: AgentInspectChannel[];
+  knowledge: {
+    enabled: boolean;
+    sources: string[];
+    embedding: {
+      provider: string;
+      model: string | null;
+      dimensions: number | null;
+    };
+    retrieval: {
+      topK: number;
+      hybrid: boolean;
+    };
+    internalTool: string | null;
+  };
+  access: {
+    mode: AccessMode;
+  };
+  storage: string | null;
+  database: AgentInspectDatabase;
+  runtimeContract?: {
+    local: {
+      environment: "local";
+      target: "bun";
+      database: "sqlite" | "none";
+    };
+    hosted: {
+      environment: "hosted";
+      target: "cloudflare";
+      database: "turso" | "none";
+      providerSupported: boolean;
+    };
+    aliases: ["ctx.db", "ctx.database", "ctx.storage.sql"];
+  };
+  secrets: Record<string, SecretState>;
+  managedSecrets: string[];
+  userSecrets: string[];
+};
+
+export type AgentInspectDatabase = {
+  local: {
+    driver: "sqlite";
+    path: string | null;
+    schema: string | null;
+    migrations: string | null;
+  };
+  hosted: {
+    driver: "none" | "turso";
+    provisioning: "none" | "agentkit-managed";
+    schema: string | null;
+    migrations: string | null;
+  };
+};
+
+export type AgentInspectChannel = {
+  name: string;
+  type: AgentChannel["type"];
+  provider: AgentChannel["provider"];
+  secrets: string[];
+  buffer?: AgentChannel["buffer"];
+};
+
+export async function inspectAgentCapsule(
+  cwd = process.cwd(),
+  env: Record<string, string | undefined> = process.env,
+): Promise<AgentInspectState> {
+  const capsule = await loadAgentCapsule(cwd);
+  return buildInspectState(capsule, await loadCapsuleEnv(capsule.root, env));
+}
+
+export function buildInspectState(
+  capsule: LoadedAgentCapsule,
+  env: Record<string, string | undefined> = process.env,
+): AgentInspectState {
+  const declaredSecrets = new Set(capsule.config.secrets);
+  const knowledge = resolveKnowledgeConfig(capsule.config);
+
+  for (const tool of capsule.config.tools ?? []) {
+    for (const secret of tool.secrets ?? []) {
+      declaredSecrets.add(secret);
+    }
+  }
+
+  for (const channel of capsule.config.channels ?? []) {
+    for (const secret of channel.secrets) {
+      declaredSecrets.add(secret);
+    }
+  }
+
+  if (knowledge.embedding.secret) {
+    declaredSecrets.add(knowledge.embedding.secret);
+  }
+
+  const managedSecrets = managedCloudSecrets(capsule);
+  const userSecrets = Array.from(declaredSecrets).sort();
+  const secrets: Record<string, SecretState> = {};
+
+  for (const secret of userSecrets) {
+    secrets[secret] = env[secret] ? "set" : "missing";
+  }
+
+  const database = inspectDatabase(capsule);
+
+  return {
+    agent: capsule.config.name,
+    runtime: capsule.config.runtime,
+    provider: capsule.config.provider,
+    prompt: toCapsulePath(capsule.root, capsule.instructionsPath),
+    tools: (capsule.config.tools ?? []).map((tool) => tool.name),
+    channels: (capsule.config.channels ?? []).map((channel) => ({
+      name: channel.name,
+      type: channel.type,
+      provider: channel.provider,
+      secrets: [...channel.secrets],
+      ...(channel.buffer ? { buffer: channel.buffer } : {}),
+    })),
+    knowledge: {
+      enabled: knowledge.enabled,
+      sources: knowledge.sources.map((source) => source.value),
+      embedding: {
+        provider: knowledge.embedding.provider,
+        model: knowledge.embedding.model,
+        dimensions: knowledge.embedding.dimensions,
+      },
+      retrieval: knowledge.retrieval,
+      internalTool: knowledge.enabled ? KNOWLEDGE_SEARCH_TOOL_NAME : null,
+    },
+    access: capsule.config.access,
+    storage: capsule.storagePath ? toCapsulePath(capsule.root, capsule.storagePath) : null,
+    database,
+    runtimeContract: {
+      local: {
+        environment: "local",
+        target: "bun",
+        database: capsule.storagePath ? "sqlite" : "none",
+      },
+      hosted: {
+        environment: "hosted",
+        target: "cloudflare",
+        database: database.hosted.driver === "turso" ? "turso" : "none",
+        providerSupported: true,
+      },
+      aliases: ["ctx.db", "ctx.database", "ctx.storage.sql"],
+    },
+    secrets,
+    managedSecrets,
+    userSecrets,
+  };
+}
+
+function toCapsulePath(root: string, path: string): string {
+  const localPath = relative(root, path);
+  return localPath.length > 0 ? localPath : ".";
+}
+
+function inspectDatabase(capsule: LoadedAgentCapsule): AgentInspectDatabase {
+  const database = capsule.config.storage.database;
+  const schema = database?.driver === "turso" && database.schema ? database.schema : null;
+  const migrations = database?.driver === "turso" ? database.migrations ?? "migrations" : null;
+
+  return {
+    local: {
+      driver: "sqlite",
+      path: capsule.storagePath ? toCapsulePath(capsule.root, capsule.storagePath) : null,
+      schema,
+      migrations,
+    },
+    hosted: {
+      driver: database?.driver === "turso" ? "turso" : "none",
+      provisioning: database?.driver === "turso" ? "agentkit-managed" : "none",
+      schema,
+      migrations,
+    },
+  };
+}
+
+function managedCloudSecrets(capsule: LoadedAgentCapsule): string[] {
+  const secrets = new Set<string>();
+
+  if (capsule.config.storage.database?.driver === "turso") {
+    secrets.add("TURSO_AUTH_TOKEN");
+    secrets.add("TURSO_DATABASE_URL");
+  }
+
+  return Array.from(secrets).sort();
+}

package/src/runtime/knowledge/chunk.ts

@@ -0,0 +1,333 @@
+import { basename, extname } from "node:path";
+
+export type KnowledgeChunkMetadata = {
+  sourcePath: string;
+  title: string;
+  section: string | null;
+  locator: string | null;
+};
+
+export type KnowledgeChunk = {
+  ordinal: number;
+  content: string;
+  tokenCount: number;
+  metadata: KnowledgeChunkMetadata;
+};
+
+export type ChunkKnowledgeSourceInput = {
+  sourcePath: string;
+  title?: string | null;
+  content: string;
+};
+
+const DEFAULT_MAX_WORDS = 720;
+const DEFAULT_OVERLAP_WORDS = 80;
+
+export function chunkKnowledgeSource(input: ChunkKnowledgeSourceInput): KnowledgeChunk[] {
+  const extension = extname(input.sourcePath).toLowerCase();
+
+  if (extension === ".csv") {
+    return chunkCsv(input);
+  }
+
+  if (extension === ".md" || extension === ".markdown") {
+    return chunkMarkdown(input);
+  }
+
+  return chunkText(input);
+}
+
+function chunkMarkdown(input: ChunkKnowledgeSourceInput): KnowledgeChunk[] {
+  const title = input.title ?? inferTitle(input.sourcePath, input.content);
+  const sections = markdownSections(input.content, title);
+  const chunks: KnowledgeChunk[] = [];
+
+  for (const section of sections) {
+    const contextualContent = [section.heading, section.body].filter(Boolean).join("\n\n").trim();
+    chunks.push(
+      ...splitContentIntoChunks({
+        sourcePath: input.sourcePath,
+        title,
+        section: section.heading || null,
+        locator: section.heading ? `section:${section.heading}` : null,
+        content: contextualContent,
+        startingOrdinal: chunks.length,
+      }),
+    );
+  }
+
+  return ensureAtLeastOneChunk(chunks, input, title);
+}
+
+function chunkText(input: ChunkKnowledgeSourceInput): KnowledgeChunk[] {
+  const title = input.title ?? inferTitle(input.sourcePath, input.content);
+  const paragraphs = input.content
+    .split(/\n{2,}/)
+    .map((paragraph) => paragraph.trim())
+    .filter(Boolean);
+  const content = paragraphs.length > 0 ? paragraphs.join("\n\n") : input.content.trim();
+
+  return ensureAtLeastOneChunk(
+    splitContentIntoChunks({
+      sourcePath: input.sourcePath,
+      title,
+      section: null,
+      locator: null,
+      content,
+      startingOrdinal: 0,
+    }),
+    input,
+    title,
+  );
+}
+
+function chunkCsv(input: ChunkKnowledgeSourceInput): KnowledgeChunk[] {
+  const title = input.title ?? inferTitle(input.sourcePath, input.content);
+  const rows = parseCsv(input.content);
+
+  if (rows.length === 0) {
+    return ensureAtLeastOneChunk([], input, title);
+  }
+
+  const [headers, ...records] = rows;
+
+  if (headers.length === 0 || records.length === 0) {
+    return chunkText(input);
+  }
+
+  const chunks: KnowledgeChunk[] = [];
+
+  for (const [index, record] of records.entries()) {
+    const content = headers
+      .map((header, columnIndex) => {
+        const name = header.trim() || `column_${columnIndex + 1}`;
+        const value = record[columnIndex]?.trim() ?? "";
+        return `${name}: ${value}`;
+      })
+      .join("\n")
+      .trim();
+
+    if (!content) {
+      continue;
+    }
+
+    chunks.push({
+      ordinal: chunks.length,
+      content,
+      tokenCount: estimateTokenCount(content),
+      metadata: {
+        sourcePath: input.sourcePath,
+        title,
+        section: null,
+        locator: `row:${index + 2}`,
+      },
+    });
+  }
+
+  return ensureAtLeastOneChunk(chunks, input, title);
+}
+
+function markdownSections(content: string, fallbackTitle: string): Array<{ heading: string | null; body: string }> {
+  const sections: Array<{ heading: string | null; body: string[] }> = [];
+  let current: { heading: string | null; body: string[] } = { heading: null, body: [] };
+  const headingStack: string[] = [];
+
+  for (const line of content.split(/\r?\n/)) {
+    const heading = line.match(/^(#{1,6})\s+(.+?)\s*#*\s*$/);
+
+    if (heading) {
+      if (current.body.join("\n").trim()) {
+        sections.push(current);
+      }
+
+      const level = heading[1].length;
+      headingStack.splice(level - 1);
+      headingStack[level - 1] = heading[2].trim();
+      current = {
+        heading: headingStack.filter(Boolean).join(" > "),
+        body: [],
+      };
+      continue;
+    }
+
+    current.body.push(line);
+  }
+
+  if (current.body.join("\n").trim()) {
+    sections.push(current);
+  }
+
+  if (sections.length === 0) {
+    return [{ heading: fallbackTitle, body: content.trim() }];
+  }
+
+  return sections.map((section) => ({
+    heading: section.heading,
+    body: section.body.join("\n").trim(),
+  }));
+}
+
+function splitContentIntoChunks(input: {
+  sourcePath: string;
+  title: string;
+  section: string | null;
+  locator: string | null;
+  content: string;
+  startingOrdinal: number;
+}): KnowledgeChunk[] {
+  const words = wordsOf(input.content);
+
+  if (words.length <= DEFAULT_MAX_WORDS) {
+    const content = input.content.trim();
+
+    if (!content) {
+      return [];
+    }
+
+    return [
+      {
+        ordinal: input.startingOrdinal,
+        content,
+        tokenCount: estimateTokenCount(content),
+        metadata: {
+          sourcePath: input.sourcePath,
+          title: input.title,
+          section: input.section,
+          locator: input.locator,
+        },
+      },
+    ];
+  }
+
+  const chunks: KnowledgeChunk[] = [];
+  let start = 0;
+
+  while (start < words.length) {
+    const end = Math.min(start + DEFAULT_MAX_WORDS, words.length);
+    const content = words.slice(start, end).join(" ");
+
+    chunks.push({
+      ordinal: input.startingOrdinal + chunks.length,
+      content,
+      tokenCount: estimateTokenCount(content),
+      metadata: {
+        sourcePath: input.sourcePath,
+        title: input.title,
+        section: input.section,
+        locator: input.locator ? `${input.locator}:part:${chunks.length + 1}` : `part:${chunks.length + 1}`,
+      },
+    });
+
+    if (end === words.length) {
+      break;
+    }
+
+    start = Math.max(end - DEFAULT_OVERLAP_WORDS, start + 1);
+  }
+
+  return chunks;
+}
+
+function ensureAtLeastOneChunk(
+  chunks: KnowledgeChunk[],
+  input: ChunkKnowledgeSourceInput,
+  title: string,
+): KnowledgeChunk[] {
+  if (chunks.length > 0) {
+    return chunks.map((chunk, index) => ({ ...chunk, ordinal: index }));
+  }
+
+  const content = input.content.trim();
+
+  if (!content) {
+    return [];
+  }
+
+  return [
+    {
+      ordinal: 0,
+      content,
+      tokenCount: estimateTokenCount(content),
+      metadata: {
+        sourcePath: input.sourcePath,
+        title,
+        section: null,
+        locator: null,
+      },
+    },
+  ];
+}
+
+function parseCsv(content: string): string[][] {
+  const rows: string[][] = [];
+  let row: string[] = [];
+  let field = "";
+  let quoted = false;
+
+  for (let index = 0; index < content.length; index += 1) {
+    const char = content[index];
+    const next = content[index + 1];
+
+    if (quoted) {
+      if (char === '"' && next === '"') {
+        field += '"';
+        index += 1;
+        continue;
+      }
+
+      if (char === '"') {
+        quoted = false;
+        continue;
+      }
+
+      field += char;
+      continue;
+    }
+
+    if (char === '"') {
+      quoted = true;
+      continue;
+    }
+
+    if (char === ",") {
+      row.push(field);
+      field = "";
+      continue;
+    }
+
+    if (char === "\n") {
+      row.push(field.replace(/\r$/, ""));
+      rows.push(row);
+      row = [];
+      field = "";
+      continue;
+    }
+
+    field += char;
+  }
+
+  row.push(field.replace(/\r$/, ""));
+  if (row.some((value) => value.length > 0)) {
+    rows.push(row);
+  }
+
+  return rows;
+}
+
+function inferTitle(sourcePath: string, content: string): string {
+  const markdownTitle = content.match(/^#\s+(.+)$/m)?.[1]?.trim();
+
+  if (markdownTitle) {
+    return markdownTitle;
+  }
+
+  return basename(sourcePath).replace(/\.[^.]+$/, "") || sourcePath;
+}
+
+function wordsOf(content: string): string[] {
+  return content.replace(/\s+/g, " ").trim().split(" ").filter(Boolean);
+}
+
+export function estimateTokenCount(content: string): number {
+  return Math.max(1, wordsOf(content).length);
+}

package/src/runtime/knowledge/config.ts

@@ -0,0 +1,135 @@
+import type {
+  AgentConfig,
+  AgentKnowledgeConfig,
+  AgentKnowledgeEmbeddingProvider,
+  AgentKnowledgeSource,
+} from "../../index";
+
+export type ResolvedKnowledgeSource =
+  | {
+      kind: "file";
+      value: string;
+      title: string | null;
+    }
+  | {
+      kind: "url";
+      value: string;
+      title: string | null;
+    };
+
+export type ResolvedKnowledgeConfig = {
+  enabled: boolean;
+  sources: ResolvedKnowledgeSource[];
+  embedding: {
+    provider: AgentKnowledgeEmbeddingProvider;
+    model: string | null;
+    dimensions: number | null;
+    secret: string | null;
+  };
+  retrieval: {
+    topK: number;
+    hybrid: boolean;
+  };
+};
+
+const DEFAULT_TOP_K = 8;
+const DEFAULT_TEST_EMBEDDING_DIMENSIONS = 32;
+const DEFAULT_OPENAI_EMBEDDING_MODEL = "text-embedding-3-small";
+const DEFAULT_OPENAI_EMBEDDING_DIMENSIONS = 1536;
+
+export function resolveKnowledgeConfig(config: Pick<AgentConfig, "knowledge">): ResolvedKnowledgeConfig {
+  const knowledge = config.knowledge;
+
+  if (!knowledge) {
+    return {
+      enabled: false,
+      sources: [],
+      embedding: {
+        provider: "none",
+        model: null,
+        dimensions: null,
+        secret: null,
+      },
+      retrieval: {
+        topK: DEFAULT_TOP_K,
+        hybrid: false,
+      },
+    };
+  }
+
+  const provider = knowledge.embedding?.provider ?? "none";
+  const embedding = resolveEmbeddingConfig(provider, knowledge);
+  const hybrid = knowledge.retrieval?.hybrid ?? provider !== "none";
+
+  return {
+    enabled: true,
+    sources: (knowledge.sources ?? []).map(resolveKnowledgeSource),
+    embedding,
+    retrieval: {
+      topK: knowledge.retrieval?.topK ?? DEFAULT_TOP_K,
+      hybrid,
+    },
+  };
+}
+
+function resolveEmbeddingConfig(
+  provider: AgentKnowledgeEmbeddingProvider,
+  knowledge: AgentKnowledgeConfig,
+): ResolvedKnowledgeConfig["embedding"] {
+  if (provider === "openai") {
+    return {
+      provider,
+      model: knowledge.embedding?.model ?? DEFAULT_OPENAI_EMBEDDING_MODEL,
+      dimensions: knowledge.embedding?.dimensions ?? DEFAULT_OPENAI_EMBEDDING_DIMENSIONS,
+      secret: knowledge.embedding?.secret ?? "OPENAI_API_KEY",
+    };
+  }
+
+  if (provider === "test") {
+    return {
+      provider,
+      model: knowledge.embedding?.model ?? "fake",
+      dimensions: knowledge.embedding?.dimensions ?? DEFAULT_TEST_EMBEDDING_DIMENSIONS,
+      secret: knowledge.embedding?.secret ?? null,
+    };
+  }
+
+  return {
+    provider: "none",
+    model: knowledge.embedding?.model ?? null,
+    dimensions: knowledge.embedding?.dimensions ?? null,
+    secret: knowledge.embedding?.secret ?? null,
+  };
+}
+
+function resolveKnowledgeSource(source: AgentKnowledgeSource): ResolvedKnowledgeSource {
+  if (typeof source === "string") {
+    if (/^https?:\/\//i.test(source)) {
+      return {
+        kind: "url",
+        value: source,
+        title: null,
+      };
+    }
+
+    return {
+      kind: "file",
+      value: source,
+      title: null,
+    };
+  }
+
+  if ("url" in source) {
+    return {
+      kind: "url",
+      value: source.url,
+      title: source.title ?? null,
+    };
+  }
+
+  return {
+    kind: "file",
+    value: source.path,
+    title: source.title ?? null,
+  };
+}