@andreprado/agentkit 0.1.0-alpha.6 → 0.1.0-alpha.7

package/src/storage/sqlite.ts

@@ -42,12 +42,30 @@ export type StoredToolCall = {
   visibility: ToolCallVisibility;
   input: unknown;
   output: unknown;
+  rendered: unknown;
   status: ToolCallStatus;
   startedAt: string;
   completedAt: string | null;
   errorMessage: string | null;
 };
 
+export type StoredRun = {
+  id: string;
+  conversationId: string | null;
+  provider: string;
+  model: string;
+  status: RunStatus;
+  usage: {
+    inputTokens: number | null;
+    outputTokens: number | null;
+    totalTokens: number | null;
+  };
+  startedAt: string;
+  completedAt: string | null;
+  errorCode: string | null;
+  errorMessage: string | null;
+};
+
 export type PersistedChatRun = {
   conversationId: string;
   runId: string;
@@ -158,6 +176,13 @@ const MIGRATIONS = [
     id: KNOWLEDGE_MIGRATION_ID,
     sql: KNOWLEDGE_SCHEMA_SQL,
   },
+  {
+    id: "0007_add_tool_call_rendered",
+    sql: `
+      ALTER TABLE tool_calls
+        ADD COLUMN rendered_json TEXT;
+    `,
+  },
 ] as const;
 
 export async function openCapsuleStore(capsule: LoadedAgentCapsule): Promise<SqliteAgentKitStore> {
@@ -462,6 +487,7 @@ export class SqliteAgentKitStore {
     toolCallId: string;
     status: ToolCallStatus;
     output?: unknown;
+    rendered?: unknown;
     completedAt?: string;
     errorMessage?: string;
     secretValues?: string[];
@@ -474,6 +500,7 @@ export class SqliteAgentKitStore {
         SET
           status = $status,
           output_json = $outputJson,
+          rendered_json = $renderedJson,
           completed_at = $completedAt,
           error_message = $errorMessage
         WHERE id = $toolCallId
@@ -483,6 +510,7 @@ export class SqliteAgentKitStore {
           $toolCallId: input.toolCallId,
           $status: input.status,
           $outputJson: input.output === undefined ? null : serializeJson(input.output, input.secretValues),
+          $renderedJson: input.rendered === undefined ? null : serializeJson(input.rendered, input.secretValues),
           $completedAt: input.completedAt ?? new Date().toISOString(),
           $errorMessage: redactSecrets(input.errorMessage ?? null, input.secretValues),
         });
@@ -555,6 +583,35 @@ export class SqliteAgentKitStore {
     });
   }
 
+  listRunsForConversation(conversationId: string): StoredRun[] {
+    return this.sqlite(() => {
+      const rows = this.db
+        .prepare(
+          `
+          SELECT
+            id,
+            conversation_id,
+            provider,
+            model,
+            status,
+            input_tokens,
+            output_tokens,
+            total_tokens,
+            started_at,
+            completed_at,
+            error_code,
+            error_message
+          FROM runs
+          WHERE conversation_id = $conversationId
+          ORDER BY started_at ASC
+        `,
+        )
+        .all({ $conversationId: conversationId }) as SqliteRow[];
+
+      return rows.map(storedRunFromRow);
+    });
+  }
+
   listToolCallsForRun(runId: string): StoredToolCall[] {
     return this.sqlite(() => {
       const rows = this.db
@@ -568,6 +625,7 @@ export class SqliteAgentKitStore {
             visibility,
             input_json,
             output_json,
+            rendered_json,
             status,
             started_at,
             completed_at,
@@ -583,6 +641,54 @@ export class SqliteAgentKitStore {
     });
   }
 
+  applyApplicationMigrations(migrations: Array<{ id: string; sql: string }>): { applied: string[]; skipped: string[] } {
+    return this.sqlite(() => {
+      if (migrations.length === 0) {
+        return { applied: [], skipped: [] };
+      }
+
+      this.db.exec(`
+        CREATE TABLE IF NOT EXISTS agentkit_app_migrations (
+          id TEXT PRIMARY KEY,
+          applied_at TEXT NOT NULL
+        )
+      `);
+
+      const applied = new Set(
+        (
+          this.db.prepare("SELECT id FROM agentkit_app_migrations").all() as Array<{
+            id: string;
+          }>
+        ).map((row) => row.id),
+      );
+      const result = {
+        applied: [] as string[],
+        skipped: [] as string[],
+      };
+
+      for (const migration of migrations) {
+        if (applied.has(migration.id)) {
+          result.skipped.push(migration.id);
+          continue;
+        }
+
+        const statements = splitSqlStatements(migration.sql);
+        this.transaction(() => {
+          for (const statement of statements) {
+            this.db.exec(statement);
+          }
+
+          this.db
+            .prepare("INSERT INTO agentkit_app_migrations (id, applied_at) VALUES ($id, $appliedAt)")
+            .run({ $id: migration.id, $appliedAt: new Date().toISOString() });
+        });
+        result.applied.push(migration.id);
+      }
+
+      return result;
+    });
+  }
+
   applyApplicationSchema(source: string): void {
     this.sqlite(() => {
       const statements = splitSqlStatements(source);
@@ -790,6 +896,31 @@ function storedMessageFromRow(row: SqliteRow): StoredMessage {
   };
 }
 
+function storedRunFromRow(row: SqliteRow): StoredRun {
+  const status = String(row.status);
+
+  if (status !== "running" && status !== "completed" && status !== "failed") {
+    throw new AgentKitError("storage_error", `Invalid stored run status "${status}".`);
+  }
+
+  return {
+    id: String(row.id),
+    conversationId: row.conversation_id === null ? null : String(row.conversation_id),
+    provider: String(row.provider),
+    model: String(row.model),
+    status,
+    usage: {
+      inputTokens: nullableNumber(row.input_tokens),
+      outputTokens: nullableNumber(row.output_tokens),
+      totalTokens: nullableNumber(row.total_tokens),
+    },
+    startedAt: String(row.started_at),
+    completedAt: row.completed_at === null ? null : String(row.completed_at),
+    errorCode: row.error_code === null ? null : String(row.error_code),
+    errorMessage: row.error_message === null ? null : String(row.error_message),
+  };
+}
+
 function storedToolCallFromRow(row: SqliteRow): StoredToolCall {
   const status = String(row.status);
   const visibility = String(row.visibility);
@@ -810,6 +941,7 @@ function storedToolCallFromRow(row: SqliteRow): StoredToolCall {
     visibility,
     input: parseStoredJson(row.input_json),
     output: row.output_json === null ? undefined : parseStoredJson(row.output_json),
+    rendered: row.rendered_json === null ? undefined : parseStoredJson(row.rendered_json),
     status,
     startedAt: String(row.started_at),
     completedAt: row.completed_at === null ? null : String(row.completed_at),
@@ -817,6 +949,10 @@ function storedToolCallFromRow(row: SqliteRow): StoredToolCall {
   };
 }
 
+function nullableNumber(value: unknown): number | null {
+  return value === null || value === undefined ? null : Number(value);
+}
+
 function parseStoredJson(value: unknown): unknown {
   if (typeof value !== "string") {
     return value;

package/src/templates/blank.ts

@@ -148,10 +148,12 @@ When the owner opens this folder in Codex, Claude Code, or another coding agent
 Start building immediately:
 
 - Start with \`skills/agentkit-capsule/SKILL.md\`, then use \`npm run agentkit -- docs llms\` as the docs router.
+- Create or update \`AGENT_SPEC.md\` from the owner's request with \`npm run agentkit -- spec init --brief "<owner request>"\`; the owner should not fill this file by hand before work starts.
 - Infer the first useful version from the owner's request.
 - Edit \`prompts/instructions.md\` for the agent behavior.
 - Edit \`agentkit.config.ts\` for provider, tools, secrets, access, and storage.
 - Add TypeScript tools under \`tools/\` when the requested agent needs actions or external data.
+- Add \`sync.ts\`, \`seed.sql\`, and ordered \`migrations/*.sql\` when the requested agent depends on external catalogs or production-shaped data changes.
 - Do not wait for a wizard or recipe. AgentKit provides the scaffold and contract; you decide the implementation from the owner's brief.
 - Ask follow-up questions only when missing information blocks a safe local implementation.
 - State assumptions in the final response.
@@ -162,6 +164,9 @@ Start building immediately:
 - \`npm run dev\`: run the local Agent Capsule runtime.
 - \`npm run chat -- --message "hello"\`: send one local chat message.
 - \`npm run eval\`: run agent evals.
+- \`npm run agentkit -- spec check\`: verify the local agent implementation contract exists.
+- \`npm run agentkit -- eval from-conversation <conversation-id>\`: turn a real conversation into a regression eval.
+- \`npm run agentkit -- conversations trace <conversation-id>\`: inspect messages, runs, tool calls, inputs, outputs, rendered output, and final responses.
 - \`npm run agentkit -- tool <name> --input fixtures/input.json\`: run one registered tool directly.
 - \`npm run agentkit -- db migrate\`: apply internal migrations and \`schema.sql\` locally.
 - \`npm run agentkit -- db reset --yes\`: recreate the local SQLite database and reapply schema.
@@ -212,7 +217,7 @@ export const myTool = defineTool({
 
 Use \`ctx.db\` as the canonical helper. \`ctx.database\` and \`ctx.storage.sql\` are aliases. Use \`ctx.db.batch([...])\` for atomic writes; local tools can also use \`ctx.db.transaction(async (tx) => ...)\`. Do not import local database drivers or Node-only APIs in tools. AgentKit owns local and hosted database routing.
 
-\`schema.sql\` is an idempotent bootstrap file in v1. Use \`CREATE TABLE IF NOT EXISTS\`, \`CREATE INDEX IF NOT EXISTS\`, and safe additive changes. AgentKit does not run destructive schema changes or ordered \`migrations/*.sql\` automatically yet.
+\`schema.sql\` is an idempotent bootstrap file. Use \`CREATE TABLE IF NOT EXISTS\`, \`CREATE INDEX IF NOT EXISTS\`, and safe additive changes. Use ordered \`migrations/*.sql\` for production-shaped schema evolution; \`npm run agentkit -- db migrate\` applies unapplied local migrations before \`schema.sql\`.
 
 ## Hosted Deploy
 
@@ -252,8 +257,10 @@ Example owner request:
 Turn the request into a working local capsule:
 
 - Update \`prompts/instructions.md\` with domain-specific behavior, boundaries, intake questions, and escalation rules.
+- Create or update \`AGENT_SPEC.md\` with \`npm run agentkit -- spec init --brief "<owner request>"\`. The owner gives the general idea; the coding agent turns it into the structured contract.
 - Update \`agentkit.config.ts\` when tools, secrets, provider, or access rules change.
 - Add TypeScript tools under \`tools/\` for real actions or external data.
+- Use \`npm run agentkit -- sync init\` when the agent needs catalog sync, fixture seed data, or ordered migrations.
 - Keep the first version runnable with \`test/fake\` unless the owner explicitly asks for a real provider.
 - Do not use a wizard or recipe. Build the capsule directly from the scaffold, the AgentKit contract, and the owner's brief.
 - Make practical assumptions and list them in your final response.

package/src/templates/dentista.ts

@@ -857,6 +857,8 @@ Esta é uma AgentKit Agent Capsule para a Clara, atendente em português de um c
 
 A Clara conversa com clientes, coleta nome, email e telefone, consulta disponibilidade e agenda consultas em slots de 30 minutos. Ela também consulta e altera o próprio horário do cliente usando email e telefone como verificação mínima.
 
+Quando o dono pedir mudanças em linguagem natural, crie ou atualize \`AGENT_SPEC.md\` com \`npm run agentkit -- spec init --brief "<pedido do dono>"\`. O dono dá a ideia geral; o agente de código transforma isso no contrato estruturado.
+
 ## Regras de Produto
 
 - Persona: Clara, atendente humana, simpática e objetiva.
@@ -885,6 +887,9 @@ A Clara conversa com clientes, coleta nome, email e telefone, consulta disponibi
 - \`npm run agentkit -- tool consultar_consulta --input '{"email":"ana@example.com","telefone":"11999999999"}'\`: consultar uma consulta.
 - \`npm run agentkit -- tool alterar_consulta --input '{"email":"ana@example.com","telefone":"11999999999","novaData":"2099-01-06","novoHorario":"10:30","confirmadoPeloCliente":true}'\`: alterar uma consulta confirmada.
 - \`npm run eval\`: rodar evals.
+- \`npm run agentkit -- spec check\`: validar o contrato local da implementação.
+- \`npm run agentkit -- eval from-conversation <conversation-id>\`: transformar uma conversa real em eval de regressão.
+- \`npm run agentkit -- conversations trace <conversation-id>\`: inspecionar mensagens, runs, ferramentas, inputs, outputs, renderização e resposta final.
 - \`npm run dev\`: rodar a runtime local.
 - \`npm run agentkit -- inspect\`: imprimir o estado da cápsula.
 - \`npm run agentkit -- docs llms\`: imprimir o roteador leve da documentação AgentKit.
@@ -900,7 +905,8 @@ A Clara conversa com clientes, coleta nome, email e telefone, consulta disponibi
 ## Regras de Implementação
 
 - Use \`ctx.db\` nas ferramentas. Não importe drivers SQLite, Turso ou Node-only APIs.
-- Mantenha \`schema.sql\` idempotente com \`CREATE TABLE IF NOT EXISTS\` e \`CREATE INDEX IF NOT EXISTS\`.
+- Mantenha \`schema.sql\` idempotente com \`CREATE TABLE IF NOT EXISTS\` e \`CREATE INDEX IF NOT EXISTS\`. Use \`migrations/*.sql\` ordenadas para evolução de schema com cara de produção.
+- Use \`npm run agentkit -- sync init\` quando a agente depender de catálogo externo, fixture local ou seed padronizado.
 - Mantenha segredos em \`.env\`, nunca em arquivos versionados.
 - Não espere wizard ou recipe. AgentKit fornece o scaffold e o contrato; implemente diretamente conforme o brief do dono.
 - Para trocar para um provider real, o dono deve escolher OpenRouter, OpenAI, Anthropic ou outro provider suportado. Depois edite \`agentkit.config.ts\`, atualize \`.env.schema\` e configure secrets locais/hosted.
@@ -933,6 +939,13 @@ npm run chat -- --message "Oi, quero marcar uma consulta."
 
 A cápsula começa com \`test/fake\`. Isso valida caminhos determinísticos, mas não prova qualidade de conversa natural.
 
+Se o dono pedir uma mudança grande, crie ou atualize o contrato de implementação:
+
+\`\`\`sh
+npm run agentkit -- spec init --brief "<pedido do dono>"
+npm run agentkit -- spec check
+\`\`\`
+
 ## Ferramentas
 
 - \`listar_horarios_disponiveis\`: recebe \`data\` em \`YYYY-MM-DD\` e retorna slots livres.

package/src/templates/skills/agentkit-build-agent/SKILL.md

@@ -10,12 +10,14 @@ Use this when the owner asks for an agent in plain language.
 ## Workflow
 
 1. Read `agentkit.config.ts`, `prompts/instructions.md`, `schema.sql`, `evals/`, and existing `tools/`.
-2. Infer the first useful local version from the owner's brief.
-3. Edit `prompts/instructions.md` for behavior, boundaries, intake questions, escalation rules, and tool-use policy.
-4. Add tools only when the agent needs action, live data, authorization-sensitive data, or durable writes.
-5. Add database tables to `schema.sql` when the agent owns records.
-6. Add or update evals for the main flow.
-7. Keep the capsule runnable on `test/fake` unless the owner has chosen a real provider.
+2. If `AGENT_SPEC.md` does not exist, create it from the owner's plain-language request with `npm run agentkit -- spec init --brief "<owner request>"`. If it exists, update it directly before changing behavior.
+3. Infer the first useful local version from the owner's brief and the spec. Do not ask the owner to fill a form.
+4. Edit `prompts/instructions.md` for behavior, boundaries, intake questions, escalation rules, and tool-use policy.
+5. Add tools only when the agent needs action, live data, authorization-sensitive data, or durable writes.
+6. Add database tables to `schema.sql` or ordered `migrations/*.sql` when the agent owns records.
+7. Add `sync.ts` and `seed.sql` with `npm run agentkit -- sync init` when the agent depends on external catalogs or recurring imports.
+8. Add or update evals for the main flow. Prefer multi-turn `turns` evals for real conversations.
+9. Keep the capsule runnable on `test/fake` unless the owner has chosen a real provider.
 
 ## Templates
 
@@ -35,6 +37,7 @@ npm run typecheck
 npm run agentkit -- inspect
 npm run chat -- --message "hello"
 npm run eval
+npm run agentkit -- spec check
 ```
 
 If a tool was added:
@@ -46,4 +49,3 @@ npm run agentkit -- tool <tool_name> --input '<json>'
 ## Final Response
 
 Summarize the files changed, assumptions made, verification results, and whether the behavior was tested with `test/fake` or a real provider selected by the owner.
-

package/src/templates/skills/agentkit-database/SKILL.md

@@ -9,12 +9,14 @@ Use this when a capsule owns durable application records.
 
 ## Rules
 
-- Put agent-owned tables and indexes in `schema.sql`.
+- Put the first idempotent bootstrap schema in `schema.sql`.
+- For production-shaped changes, prefer ordered `migrations/*.sql` files such as `migrations/0001_initial.sql`.
 - Keep `schema.sql` idempotent with `CREATE TABLE IF NOT EXISTS`, `CREATE INDEX IF NOT EXISTS`, and safe additive changes.
 - Keep deploy-ready capsules on `storage.driver: "agentkit"`.
 - Use `ctx.db` inside tools. `ctx.database` and `ctx.storage.sql` are aliases.
 - Do not import SQLite, Turso, or other database drivers from tools.
 - Do not edit `.agentkit/agentkit.db` by hand.
+- For external catalogs, run `npm run agentkit -- sync init`, then implement `sync.ts` and keep local fixtures in `seed.sql`.
 
 ## Templates
 
@@ -26,6 +28,8 @@ Use this when a capsule owns durable application records.
 ```sh
 npm run agentkit -- db migrate
 npm run agentkit -- db seed --file seed.sql
+npm run agentkit -- sync init
+npm run agentkit -- sync run
 npm run agentkit -- db shell
 npm run agentkit -- db reset --yes
 ```
@@ -39,4 +43,3 @@ npm run agentkit -- tool <tool_name> --input '<json>'
 ```
 
 Hosted deploy applies AgentKit-managed storage internally. The user should not create hosted databases or buckets by hand.
-

package/src/templates/skills/agentkit-evals/SKILL.md

@@ -11,14 +11,43 @@ Use evals after chat works and before claiming behavior is stable.
 
 1. Create or edit `evals/<name>.eval.ts`.
 2. Keep assertions small and deterministic.
-3. Use `persisted_tool_call` for tool behavior stored in local SQLite.
-4. Do not put secrets or real client PII in evals.
-5. For tools that write externally, delete, charge money, send email, or call real customer systems, branch on `ctx.runtime.environment === "eval"` inside the registered tool.
+3. Use `turns` for full conversation flows, such as user asks, agent calls a tool, then the answer follows the required format.
+4. Use `persisted_tool_call` for tool behavior stored in local SQLite.
+5. Convert real failures into regression tests with `npm run agentkit -- eval from-conversation <conversation-id>`.
+6. Do not put secrets or real client PII in evals.
+7. For tools that write externally, delete, charge money, send email, or call real customer systems, branch on `ctx.runtime.environment === "eval"` inside the registered tool.
+
+## Multi-turn Example
+
+```ts
+export default {
+  name: "buyer under budget",
+  turns: [
+    {
+      input: "I want a house up to 600k near Pinheiros.",
+      expect: {
+        persisted_tool_call: {
+          name: "buscar_imoveis",
+          status: "completed",
+          input: { maxPrice: 600000 },
+        },
+      },
+    },
+    {
+      input: "Show me the best two.",
+      expect: {
+        contains: ["R$", "Pinheiros"],
+      },
+    },
+  ],
+};
+```
 
 ## Templates
 
 - `templates/smoke.eval.md`
 - `templates/tool-call.eval.md`
+- `templates/multi-turn.eval.md`
 - `templates/no-leak.eval.md`
 
 ## Verification

package/src/templates/skills/agentkit-evals/templates/multi-turn.eval.md

@@ -0,0 +1,22 @@
+```ts
+export default {
+  name: "main conversation flow",
+  turns: [
+    {
+      input: "I need help finding an option under my budget.",
+      expect: {
+        contains: "budget",
+      },
+    },
+    {
+      input: "Show me the best match.",
+      expect: {
+        persisted_tool_call: {
+          name: "replace_with_tool_name",
+          status: "completed",
+        },
+      },
+    },
+  ],
+};
+```

package/src/templates/support.ts

@@ -191,10 +191,12 @@ When the owner opens this folder in Codex, Claude Code, or another coding agent
 Start building immediately:
 
 - Start with \`skills/agentkit-capsule/SKILL.md\`, then use \`npm run agentkit -- docs llms\` as the docs router.
+- Create or update \`AGENT_SPEC.md\` from the owner's request with \`npm run agentkit -- spec init --brief "<owner request>"\`; the owner should not fill this file by hand before work starts.
 - Infer the first useful version from the owner's request.
 - Edit \`prompts/instructions.md\` for support behavior.
 - Edit \`agentkit.config.ts\` for provider, tools, secrets, access, and storage.
 - Add or replace TypeScript tools under \`tools/\` when the requested support agent needs actions or external data.
+- Add \`sync.ts\`, \`seed.sql\`, and ordered \`migrations/*.sql\` when the support agent depends on external catalogs or production-shaped data changes.
 - Do not wait for a wizard or recipe. AgentKit provides the scaffold and contract; you decide the implementation from the owner's brief.
 - Ask follow-up questions only when missing information blocks a safe local implementation.
 - State assumptions in the final response.
@@ -205,6 +207,9 @@ Start building immediately:
 - \`npm run chat -- --message "hello"\`: send one local chat message.
 - \`npm run agentkit -- tool lookup_order --input '{"orderId":"A100"}'\`: test the example tool directly.
 - \`npm run eval\`: run agent evals.
+- \`npm run agentkit -- spec check\`: verify the local agent implementation contract exists.
+- \`npm run agentkit -- eval from-conversation <conversation-id>\`: turn a real conversation into a regression eval.
+- \`npm run agentkit -- conversations trace <conversation-id>\`: inspect messages, runs, tool calls, inputs, outputs, rendered output, and final responses.
 - \`npm run dev\`: run the local Agent Capsule runtime.
 - \`npm run agentkit -- inspect\`: print machine-readable capsule state.
 - \`printf %s "$VALUE" | npm run agentkit -- env set <NAME> --stdin\`: write a local secret value to ignored \`.env\` without putting it in shell history.
@@ -259,8 +264,10 @@ Example owner request:
 Turn the request into a working local capsule:
 
 - Update \`prompts/instructions.md\` with domain-specific behavior, boundaries, intake questions, and escalation rules.
+- Create or update \`AGENT_SPEC.md\` with \`npm run agentkit -- spec init --brief "<owner request>"\`. The owner gives the general idea; the coding agent turns it into the structured contract.
 - Update \`agentkit.config.ts\` when tools, secrets, provider, or access rules change.
 - Add, replace, or remove TypeScript tools under \`tools/\` for real actions or external data.
+- Use \`npm run agentkit -- sync init\` when the agent needs catalog sync, fixture seed data, or ordered migrations.
 - Keep the first version runnable with \`test/fake\` unless the owner explicitly asks for a real provider.
 - Do not use a wizard or recipe. Build the capsule directly from the scaffold, the AgentKit contract, and the owner's brief.
 - Make practical assumptions and list them in your final response.