@andreprado/agentkit 0.1.0-alpha.5 → 0.1.0-alpha.7

package/src/runtime/knowledge/ingest.ts

@@ -0,0 +1,521 @@
+import { createHash } from "node:crypto";
+import { readFile, stat } from "node:fs/promises";
+import { extname, isAbsolute, relative, resolve } from "node:path";
+
+import type { DatabaseRunner, DatabaseRow } from "../../index";
+import { openPreparedCapsuleStore, createLocalDatabaseRunner } from "../database";
+import { loadCapsuleEnv } from "../env";
+import { AgentKitError } from "../errors";
+import type { LoadedAgentCapsule } from "../config";
+import { loadAgentCapsule } from "../config";
+import { chunkKnowledgeSource } from "./chunk";
+import { resolveKnowledgeConfig, type ResolvedKnowledgeSource } from "./config";
+import { createEmbeddingProvider } from "./embeddings";
+import {
+  localKnowledgeVectorDatabaseExists,
+  rebuildLocalKnowledgeVectorIndex,
+  type LocalKnowledgeVectorStatus,
+} from "./vector";
+
+export type KnowledgeIngestStatus = "indexed" | "skipped" | "failed";
+
+export type KnowledgeIngestResult = {
+  source: string;
+  status: KnowledgeIngestStatus;
+  chunksCreated: number;
+  message: string;
+};
+
+export type KnowledgeIngestSummary = {
+  databasePath: string;
+  indexed: number;
+  skipped: number;
+  failed: number;
+  chunksCreated: number;
+  vector: LocalKnowledgeVectorStatus;
+  results: KnowledgeIngestResult[];
+};
+
+export type KnowledgeSourceSummary = {
+  source: string;
+  kind: string;
+  title: string | null;
+  status: string;
+  chunkCount: number;
+  updatedAt: string;
+  indexedAt: string | null;
+  errorMessage: string | null;
+};
+
+const SUPPORTED_EXTENSIONS = new Set([".md", ".markdown", ".txt", ".csv"]);
+
+export async function addKnowledgeSourceFromCwd(cwd: string, source: string): Promise<KnowledgeIngestSummary> {
+  const capsule = await loadAgentCapsule(cwd);
+  return ingestKnowledgeSources(capsule, [resolveFileSource(source)]);
+}
+
+export async function syncKnowledgeSourcesFromCwd(cwd: string): Promise<KnowledgeIngestSummary> {
+  const capsule = await loadAgentCapsule(cwd);
+  const knowledge = resolveKnowledgeConfig(capsule.config);
+
+  if (knowledge.sources.length === 0) {
+    throw new AgentKitError(
+      "knowledge_not_configured",
+      "No knowledge.sources are configured in agentkit.config.ts. Add sources or run `agentkit knowledge add <path>`.",
+    );
+  }
+
+  return ingestKnowledgeSources(capsule, knowledge.sources);
+}
+
+export async function syncConfiguredKnowledgeSources(capsule: LoadedAgentCapsule): Promise<KnowledgeIngestSummary | null> {
+  const knowledge = resolveKnowledgeConfig(capsule.config);
+
+  if (!knowledge.enabled || knowledge.sources.length === 0) {
+    return null;
+  }
+
+  const summary = await ingestKnowledgeSources(capsule, knowledge.sources);
+
+  if (summary.failed > 0) {
+    const failures = summary.results
+      .filter((result) => result.status === "failed")
+      .map((result) => `${result.source}: ${result.message}`)
+      .join("; ");
+    throw new AgentKitError(
+      "knowledge_sync_failed",
+      `Configured Knowledge sync failed before local run. ${failures}`,
+    );
+  }
+
+  return summary;
+}
+
+export async function listKnowledgeSourcesFromCwd(cwd: string): Promise<{
+  databasePath: string;
+  sources: KnowledgeSourceSummary[];
+}> {
+  const capsule = await loadAgentCapsule(cwd);
+  const { store, databasePath } = await openPreparedCapsuleStore(capsule);
+
+  try {
+    const db = createLocalDatabaseRunner(store);
+    const result = await db.query(`
+      SELECT
+        source_key,
+        kind,
+        title,
+        status,
+        chunk_count,
+        updated_at,
+        indexed_at,
+        error_message
+      FROM agentkit_knowledge_sources
+      ORDER BY updated_at DESC, source_key ASC
+    `);
+
+    return {
+      databasePath,
+      sources: result.rows.map(knowledgeSourceSummaryFromRow),
+    };
+  } finally {
+    store.close();
+  }
+}
+
+async function ingestKnowledgeSources(
+  capsule: LoadedAgentCapsule,
+  sources: ResolvedKnowledgeSource[],
+): Promise<KnowledgeIngestSummary> {
+  const { store, databasePath } = await openPreparedCapsuleStore(capsule);
+  const db = createLocalDatabaseRunner(store);
+  const env = await loadCapsuleEnv(capsule.root);
+  const knowledge = resolveKnowledgeConfig(capsule.config);
+  const embeddingProvider = createEmbeddingProvider({
+    config: knowledge.embedding,
+    env,
+  });
+  const results: KnowledgeIngestResult[] = [];
+
+  try {
+    for (const source of sources) {
+      try {
+        results.push(await ingestKnowledgeSource(capsule, db, source, embeddingProvider));
+      } catch (error) {
+        results.push({
+          source: source.value,
+          status: "failed",
+          chunksCreated: 0,
+          message: error instanceof Error ? error.message : String(error),
+        });
+      }
+    }
+
+    await checkpointLocalKnowledgeDatabase(db);
+  } finally {
+    store.close();
+  }
+
+  const vector = await syncLocalKnowledgeVectorIndex({
+    databasePath,
+    embeddingProviderEnabled: Boolean(embeddingProvider),
+    dimensions: knowledge.embedding.dimensions,
+    shouldRebuild: results.some((result) => result.status === "indexed"),
+  });
+
+  return {
+    databasePath,
+    indexed: results.filter((result) => result.status === "indexed").length,
+    skipped: results.filter((result) => result.status === "skipped").length,
+    failed: results.filter((result) => result.status === "failed").length,
+    chunksCreated: results.reduce((sum, result) => sum + result.chunksCreated, 0),
+    vector,
+    results,
+  };
+}
+
+async function checkpointLocalKnowledgeDatabase(db: DatabaseRunner): Promise<void> {
+  await db.execute("PRAGMA wal_checkpoint(TRUNCATE)").catch(() => undefined);
+}
+
+async function syncLocalKnowledgeVectorIndex(options: {
+  databasePath: string;
+  embeddingProviderEnabled: boolean;
+  dimensions: number | null;
+  shouldRebuild: boolean;
+}): Promise<LocalKnowledgeVectorStatus> {
+  if (!options.embeddingProviderEnabled) {
+    return localKnowledgeVectorDisabled();
+  }
+
+  const hasVectorDatabase = await localKnowledgeVectorDatabaseExists(options.databasePath);
+
+  if (!options.shouldRebuild && hasVectorDatabase) {
+    return {
+      enabled: true,
+      provider: "libsql",
+      chunksIndexed: 0,
+      message: "Local libSQL vector database is already up to date.",
+    };
+  }
+
+  return rebuildLocalKnowledgeVectorIndex({
+    databasePath: options.databasePath,
+    dimensions: options.dimensions,
+  });
+}
+
+function localKnowledgeVectorDisabled(): LocalKnowledgeVectorStatus {
+  return {
+    enabled: false,
+    provider: "none",
+    chunksIndexed: 0,
+    message: "Knowledge embeddings are disabled, so no local vector index was created.",
+  };
+}
+
+async function sourceEmbeddingsAreCurrent(
+  db: DatabaseRunner,
+  sourceId: string,
+  knowledge: ReturnType<typeof resolveKnowledgeConfig>,
+  embeddingProviderEnabled: boolean,
+): Promise<boolean> {
+  const result = await db.query(
+    `
+    SELECT
+      COUNT(*) AS chunk_count,
+      SUM(
+        CASE
+          WHEN ? = 0
+            AND embedding_json IS NULL
+            AND embedding_provider IS NULL
+            AND embedding_model IS NULL
+            AND embedding_dimensions IS NULL
+          THEN 1
+          WHEN ? = 1
+            AND embedding_json IS NOT NULL
+            AND embedding_provider = ?
+            AND embedding_model = ?
+            AND embedding_dimensions = ?
+          THEN 1
+          ELSE 0
+        END
+      ) AS current_count
+    FROM agentkit_knowledge_chunks
+    WHERE source_id = ?
+  `,
+    [
+      embeddingProviderEnabled ? 1 : 0,
+      embeddingProviderEnabled ? 1 : 0,
+      knowledge.embedding.provider,
+      knowledge.embedding.model,
+      knowledge.embedding.dimensions,
+      sourceId,
+    ],
+  );
+  const row = result.rows[0];
+  const chunkCount = Number(row?.chunk_count ?? 0);
+  const currentCount = Number(row?.current_count ?? 0);
+
+  return chunkCount === currentCount;
+}
+
+async function ingestKnowledgeSource(
+  capsule: LoadedAgentCapsule,
+  db: DatabaseRunner,
+  source: ResolvedKnowledgeSource,
+  embeddingProvider: ReturnType<typeof createEmbeddingProvider>,
+): Promise<KnowledgeIngestResult> {
+  if (source.kind === "url") {
+    throw new AgentKitError(
+      "knowledge_source_unsupported",
+      "URL knowledge sources are planned for hosted ingestion. Local Knowledge currently supports .md, .txt, and .csv files.",
+    );
+  }
+
+  const file = await readKnowledgeFile(capsule.root, source.value);
+  const contentSha256 = sha256(file.content);
+  const existing = await findExistingSource(db, file.sourceKey);
+  const resolvedKnowledge = resolveKnowledgeConfig(capsule.config);
+  const sourceId = typeof existing?.id === "string" ? existing.id : stableId("ksrc", file.sourceKey);
+
+  if (
+    existing?.content_sha256 === contentSha256 &&
+    existing.status === "indexed" &&
+    (await sourceEmbeddingsAreCurrent(db, sourceId, resolvedKnowledge, Boolean(embeddingProvider)))
+  ) {
+    return {
+      source: file.sourceKey,
+      status: "skipped",
+      chunksCreated: 0,
+      message: "unchanged",
+    };
+  }
+
+  const chunks = chunkKnowledgeSource({
+    sourcePath: file.sourceKey,
+    title: source.title,
+    content: file.content,
+  });
+  const embeddedChunks: Array<{
+    chunk: (typeof chunks)[number];
+    embedding: number[] | null;
+    contentSha256: string;
+    id: string;
+  }> = [];
+
+  for (const chunk of chunks) {
+    const embedding = embeddingProvider ? await embeddingProvider.embed(chunk.content) : null;
+    embeddedChunks.push({
+      chunk,
+      embedding,
+      contentSha256: sha256(chunk.content),
+      id: stableId("kchk", `${sourceId}:${chunk.ordinal}:${sha256(chunk.content)}`),
+    });
+  }
+
+  const now = new Date().toISOString();
+
+  await db.transaction(async (tx) => {
+    await tx.execute(
+      `
+      INSERT INTO agentkit_knowledge_sources (
+        id,
+        source_key,
+        kind,
+        path,
+        url,
+        title,
+        content_sha256,
+        status,
+        chunk_count,
+        metadata_json,
+        created_at,
+        updated_at,
+        indexed_at,
+        error_message
+      )
+      VALUES (?, ?, ?, ?, ?, ?, ?, 'indexed', ?, '{}', ?, ?, ?, NULL)
+      ON CONFLICT(source_key) DO UPDATE SET
+        kind = excluded.kind,
+        path = excluded.path,
+        url = excluded.url,
+        title = excluded.title,
+        content_sha256 = excluded.content_sha256,
+        status = 'indexed',
+        chunk_count = excluded.chunk_count,
+        updated_at = excluded.updated_at,
+        indexed_at = excluded.indexed_at,
+        error_message = NULL
+    `,
+      [
+        sourceId,
+        file.sourceKey,
+        "file",
+        file.sourceKey,
+        null,
+        source.title ?? embeddedChunks[0]?.chunk.metadata.title ?? null,
+        contentSha256,
+        embeddedChunks.length,
+        now,
+        now,
+        now,
+      ],
+    );
+    await tx.execute("DELETE FROM agentkit_knowledge_fts WHERE source_id = ?", [sourceId]);
+    await tx.execute("DELETE FROM agentkit_knowledge_chunks WHERE source_id = ?", [sourceId]);
+
+    for (const item of embeddedChunks) {
+      const embeddingJson = item.embedding ? JSON.stringify(item.embedding) : null;
+      const embeddingHash = embeddingJson ? sha256(embeddingJson) : null;
+
+      await tx.execute(
+        `
+        INSERT INTO agentkit_knowledge_chunks (
+          id,
+          source_id,
+          ordinal,
+          content,
+          token_count,
+          title,
+          section,
+          locator,
+          content_sha256,
+          embedding_json,
+          embedding_hash,
+          embedding_provider,
+          embedding_model,
+          embedding_dimensions,
+          metadata_json,
+          created_at
+        )
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `,
+        [
+          item.id,
+          sourceId,
+          item.chunk.ordinal,
+          item.chunk.content,
+          item.chunk.tokenCount,
+          item.chunk.metadata.title,
+          item.chunk.metadata.section,
+          item.chunk.metadata.locator,
+          item.contentSha256,
+          embeddingJson,
+          embeddingHash,
+          item.embedding ? resolvedKnowledge.embedding.provider : null,
+          item.embedding ? resolvedKnowledge.embedding.model : null,
+          item.embedding ? item.embedding.length : null,
+          JSON.stringify(item.chunk.metadata),
+          now,
+        ],
+      );
+      await tx.execute(
+        `
+        INSERT INTO agentkit_knowledge_fts (chunk_id, source_id, title, section, content)
+        VALUES (?, ?, ?, ?, ?)
+      `,
+        [
+          item.id,
+          sourceId,
+          item.chunk.metadata.title,
+          item.chunk.metadata.section,
+          item.chunk.content,
+        ],
+      );
+    }
+  });
+
+  return {
+    source: file.sourceKey,
+    status: "indexed",
+    chunksCreated: embeddedChunks.length,
+    message: `${embeddedChunks.length} chunks indexed`,
+  };
+}
+
+function resolveFileSource(value: string): ResolvedKnowledgeSource {
+  if (/^https?:\/\//i.test(value)) {
+    return {
+      kind: "url",
+      value,
+      title: null,
+    };
+  }
+
+  return {
+    kind: "file",
+    value,
+    title: null,
+  };
+}
+
+async function readKnowledgeFile(root: string, inputPath: string): Promise<{ sourceKey: string; content: string }> {
+  const absolutePath = resolve(root, inputPath);
+  const sourceKey = capsuleRelativePath(root, absolutePath);
+  const extension = extname(sourceKey).toLowerCase();
+
+  if (!SUPPORTED_EXTENSIONS.has(extension)) {
+    throw new AgentKitError(
+      "knowledge_source_unsupported",
+      `Knowledge source "${sourceKey}" is not supported yet. Use .md, .txt, or .csv.`,
+    );
+  }
+
+  const stats = await stat(absolutePath).catch((error) => {
+    throw new AgentKitError("knowledge_source_missing", `Knowledge source "${sourceKey}" was not found.`, {
+      cause: error,
+    });
+  });
+
+  if (!stats.isFile()) {
+    throw new AgentKitError("knowledge_source_invalid", `Knowledge source "${sourceKey}" must be a file.`);
+  }
+
+  return {
+    sourceKey,
+    content: await readFile(absolutePath, "utf8"),
+  };
+}
+
+function capsuleRelativePath(root: string, absolutePath: string): string {
+  const relativePath = relative(root, absolutePath);
+
+  if (!relativePath || relativePath.startsWith("..") || isAbsolute(relativePath)) {
+    throw new AgentKitError(
+      "knowledge_source_outside_capsule",
+      "Knowledge source paths must stay inside the Agent Capsule.",
+    );
+  }
+
+  return relativePath.split("\\").join("/");
+}
+
+async function findExistingSource(db: DatabaseRunner, sourceKey: string): Promise<DatabaseRow | undefined> {
+  const result = await db.query(
+    "SELECT id, content_sha256, status FROM agentkit_knowledge_sources WHERE source_key = ?",
+    [sourceKey],
+  );
+  return result.rows[0];
+}
+
+function knowledgeSourceSummaryFromRow(row: DatabaseRow): KnowledgeSourceSummary {
+  return {
+    source: String(row.source_key),
+    kind: String(row.kind),
+    title: typeof row.title === "string" ? row.title : null,
+    status: String(row.status),
+    chunkCount: Number(row.chunk_count),
+    updatedAt: String(row.updated_at),
+    indexedAt: typeof row.indexed_at === "string" ? row.indexed_at : null,
+    errorMessage: typeof row.error_message === "string" ? row.error_message : null,
+  };
+}
+
+function stableId(prefix: string, value: string): string {
+  return `${prefix}_${sha256(value).slice(0, 24)}`;
+}
+
+function sha256(value: string): string {
+  return createHash("sha256").update(value).digest("hex");
+}

package/src/runtime/knowledge/prompt-policy.ts

@@ -0,0 +1,30 @@
+import type { AgentConfig } from "../../index";
+import { resolveKnowledgeConfig } from "./config";
+import { KNOWLEDGE_SEARCH_TOOL_NAME } from "./tool";
+
+export function knowledgePromptPolicy(config: Pick<AgentConfig, "knowledge">): string | null {
+  const knowledge = resolveKnowledgeConfig(config);
+
+  if (!knowledge.enabled) {
+    return null;
+  }
+
+  return `Knowledge rules:
+- Use ${KNOWLEDGE_SEARCH_TOOL_NAME} before answering business-specific factual questions about prices, policies, availability, procedures, requirements, documents, or source-specific details.
+- If Knowledge has no relevant result, say that the Knowledge sources do not contain the answer and ask for the missing detail or escalation path.
+- Do not invent prices, policies, availability, legal, clinical, or source-specific facts.
+- Use Knowledge results to answer naturally. Do not expose raw search JSON, scores, chunk IDs, or internal tool details to the user.`;
+}
+
+export function appendKnowledgePromptPolicy(instructions: string, config: Pick<AgentConfig, "knowledge">): string {
+  const policy = knowledgePromptPolicy(config);
+
+  if (!policy) {
+    return instructions;
+  }
+
+  return `${instructions.trimEnd()}
+
+${policy}
+`;
+}