@deepagents/text2sql 0.12.0 → 0.13.0

package/dist/index.js

@@ -392,6 +392,7 @@ import z2 from "zod";
 import { toState } from "@deepagents/agent";
 
 // packages/context/dist/index.js
+import { mergeWith } from "lodash-es";
 import { encode } from "gpt-tokenizer";
 import { generateId } from "ai";
 import pluralize from "pluralize";
@@ -603,7 +604,7 @@ function message(content) {
   } : content;
   return {
     id: message2.id,
-    name: "message",
+    name: message2.role,
     data: "content",
     type: "message",
     persist: true,
@@ -625,6 +626,22 @@ function assistantText(content, options) {
     parts: [{ type: "text", text: content }]
   });
 }
+var LAZY_ID = Symbol("lazy-id");
+function isLazyFragment(fragment2) {
+  return LAZY_ID in fragment2;
+}
+function lastAssistantMessage(content) {
+  return {
+    name: "assistant",
+    type: "message",
+    persist: true,
+    data: "content",
+    [LAZY_ID]: {
+      type: "last-assistant",
+      content
+    }
+  };
+}
 var ContextRenderer = class {
   options;
   constructor(options = {}) {
@@ -951,6 +968,8 @@ var ContextEngine = class {
   #branch = null;
   #chatData = null;
   #initialized = false;
+  /** Initial metadata to merge on first initialization */
+  #initialMetadata;
   constructor(options) {
     if (!options.chatId) {
       throw new Error("chatId is required");
@@ -962,6 +981,7 @@ var ContextEngine = class {
     this.#chatId = options.chatId;
     this.#userId = options.userId;
     this.#branchName = "main";
+    this.#initialMetadata = options.metadata;
   }
   /**
    * Initialize the chat and branch if they don't exist.
@@ -974,6 +994,15 @@ var ContextEngine = class {
       id: this.#chatId,
       userId: this.#userId
     });
+    if (this.#initialMetadata) {
+      this.#chatData = await this.#store.updateChat(this.#chatId, {
+        metadata: {
+          ...this.#chatData.metadata,
+          ...this.#initialMetadata
+        }
+      });
+      this.#initialMetadata = void 0;
+    }
     this.#branch = await this.#store.getActiveBranch(this.#chatId);
     this.#initialized = true;
   }
@@ -1121,6 +1150,12 @@ var ContextEngine = class {
     if (this.#pendingMessages.length === 0) {
       return;
     }
+    for (let i = 0; i < this.#pendingMessages.length; i++) {
+      const fragment2 = this.#pendingMessages[i];
+      if (isLazyFragment(fragment2)) {
+        this.#pendingMessages[i] = await this.#resolveLazyFragment(fragment2);
+      }
+    }
     let parentId = this.#branch.headMessageId;
     const now = Date.now();
     for (const fragment2 of this.#pendingMessages) {
@@ -1140,6 +1175,39 @@ var ContextEngine = class {
     this.#branch.headMessageId = parentId;
     this.#pendingMessages = [];
   }
+  /**
+   * Resolve a lazy fragment by finding the appropriate ID.
+   */
+  async #resolveLazyFragment(fragment2) {
+    const lazy = fragment2[LAZY_ID];
+    if (lazy.type === "last-assistant") {
+      const lastId = await this.#getLastAssistantId();
+      return assistantText(lazy.content, { id: lastId ?? crypto.randomUUID() });
+    }
+    throw new Error(`Unknown lazy fragment type: ${lazy.type}`);
+  }
+  /**
+   * Find the most recent assistant message ID (pending or persisted).
+   */
+  async #getLastAssistantId() {
+    for (let i = this.#pendingMessages.length - 1; i >= 0; i--) {
+      const msg = this.#pendingMessages[i];
+      if (msg.name === "assistant" && !isLazyFragment(msg)) {
+        return msg.id;
+      }
+    }
+    if (this.#branch?.headMessageId) {
+      const chain = await this.#store.getMessageChain(
+        this.#branch.headMessageId
+      );
+      for (let i = chain.length - 1; i >= 0; i--) {
+        if (chain[i].name === "assistant") {
+          return chain[i].id;
+        }
+      }
+    }
+    return void 0;
+  }
   /**
    * Estimate token count and cost for the full context.
    *
@@ -1408,6 +1476,36 @@ var ContextEngine = class {
     }
     this.#chatData = await this.#store.updateChat(this.#chatId, storeUpdates);
   }
+  /**
+   * Track token usage for the current chat.
+   * Accumulates usage metrics in chat.metadata.usage.
+   *
+   * @param usage - Token usage from AI SDK (LanguageModelUsage)
+   *
+   * @example
+   * ```ts
+   * // In onFinish callback
+   * const usage = await result.totalUsage;
+   * await context.trackUsage(usage);
+   * ```
+   */
+  async trackUsage(usage) {
+    await this.#ensureInitialized();
+    const freshChatData = await this.#store.getChat(this.#chatId);
+    const currentUsage = freshChatData?.metadata?.usage ?? {};
+    const updatedUsage = mergeWith(
+      {},
+      currentUsage,
+      usage,
+      (a, b) => typeof a === "number" || typeof b === "number" ? (a ?? 0) + (b ?? 0) : void 0
+    );
+    this.#chatData = await this.#store.updateChat(this.#chatId, {
+      metadata: {
+        ...freshChatData?.metadata,
+        usage: updatedUsage
+      }
+    });
+  }
   /**
    * Consolidate context fragments (no-op for now).
    *
@@ -1419,6 +1517,35 @@ var ContextEngine = class {
   consolidate() {
     return void 0;
   }
+  /**
+   * Extract skill path mappings from available_skills fragments.
+   * Returns array of { host, sandbox } for mounting in sandbox filesystem.
+   *
+   * Reads the original `paths` configuration stored in fragment metadata
+   * by the skills() fragment helper.
+   *
+   * @example
+   * ```ts
+   * const context = new ContextEngine({ store, chatId, userId })
+   *   .set(skills({ paths: [{ host: './skills', sandbox: '/skills' }] }));
+   *
+   * const mounts = context.getSkillMounts();
+   * // [{ host: './skills', sandbox: '/skills' }]
+   * ```
+   */
+  getSkillMounts() {
+    const mounts = [];
+    for (const fragment2 of this.#fragments) {
+      if (fragment2.name === "available_skills" && fragment2.metadata && Array.isArray(fragment2.metadata.paths)) {
+        for (const mapping of fragment2.metadata.paths) {
+          if (typeof mapping === "object" && mapping !== null && typeof mapping.host === "string" && typeof mapping.sandbox === "string") {
+            mounts.push({ host: mapping.host, sandbox: mapping.sandbox });
+          }
+        }
+      }
+    }
+    return mounts;
+  }
   /**
    * Inspect the full context state for debugging.
    * Returns a JSON-serializable object with context information.
@@ -1658,9 +1785,10 @@ var errorRecoveryGuardrail = {
         "My response format was invalid. Let me try again with a properly formatted response."
       );
     }
+    console.dir({ part }, { depth: null });
     return logAndFail(
       "Unknown error",
-      `An error occurred: ${errorText.slice(0, 100)}. Let me try a different approach.`
+      `An error occurred: ${errorText}. Let me try a different approach.`
     );
   }
 };
@@ -1845,12 +1973,20 @@ var SqliteContextStore = class extends ContextStore {
   }
   async listChats(options) {
     const params = [];
-    let whereClause = "";
+    const whereClauses = [];
     let limitClause = "";
     if (options?.userId) {
-      whereClause = "WHERE c.userId = ?";
+      whereClauses.push("c.userId = ?");
       params.push(options.userId);
     }
+    if (options?.metadata) {
+      whereClauses.push(`json_extract(c.metadata, '$.' || ?) = ?`);
+      params.push(options.metadata.key);
+      params.push(
+        typeof options.metadata.value === "boolean" ? options.metadata.value ? 1 : 0 : options.metadata.value
+      );
+    }
+    const whereClause = whereClauses.length > 0 ? `WHERE ${whereClauses.join(" AND ")}` : "";
     if (options?.limit !== void 0) {
       limitClause = " LIMIT ?";
       params.push(options.limit);
@@ -1864,6 +2000,7 @@ var SqliteContextStore = class extends ContextStore {
           c.id,
           c.userId,
           c.title,
+          c.metadata,
           c.createdAt,
           c.updatedAt,
           COUNT(DISTINCT m.id) as messageCount,
@@ -1879,6 +2016,7 @@ var SqliteContextStore = class extends ContextStore {
       id: row.id,
       userId: row.userId,
       title: row.title ?? void 0,
+      metadata: row.metadata ? JSON.parse(row.metadata) : void 0,
       messageCount: row.messageCount,
       branchCount: row.branchCount,
       createdAt: row.createdAt,
@@ -1910,9 +2048,16 @@ var SqliteContextStore = class extends ContextStore {
   async addMessage(message2) {
     this.#db.prepare(
       `INSERT INTO messages (id, chatId, parentId, name, type, data, createdAt)
-         VALUES (?, ?, ?, ?, ?, ?, ?)
+         VALUES (
+           ?1,
+           ?2,
+           CASE WHEN ?3 = ?1 THEN (SELECT parentId FROM messages WHERE id = ?1) ELSE ?3 END,
+           ?4,
+           ?5,
+           ?6,
+           ?7
+         )
          ON CONFLICT(id) DO UPDATE SET
-           parentId = excluded.parentId,
            name = excluded.name,
            type = excluded.type,
            data = excluded.data`
@@ -2387,13 +2532,9 @@ var Agent = class _Agent {
               writer.write({ type: "finish" });
               return;
             }
-            writer.write({
-              type: "text-delta",
-              id: generateId2(),
-              delta: ` ${failureFeedback}`
-            });
+            writeText(writer, failureFeedback);
             const selfCorrectionText = accumulatedText + " " + failureFeedback;
-            context.set(assistantText(selfCorrectionText));
+            context.set(lastAssistantMessage(selfCorrectionText));
             await context.save();
             currentResult = await this.#createRawStream(
               contextVariables,
@@ -2484,10 +2625,10 @@ var repairToolCall = async ({
   if (NoSuchToolError.isInstance(error)) {
     return null;
   }
-  const tool3 = tools3[toolCall.toolName];
+  const tool4 = tools3[toolCall.toolName];
   const { output } = await generateText({
     model: groq("openai/gpt-oss-20b"),
-    output: Output.object({ schema: tool3.inputSchema }),
+    output: Output.object({ schema: tool4.inputSchema }),
     prompt: [
       `The model tried to call the tool "${toolCall.toolName}" with the following inputs:`,
       JSON.stringify(toolCall.input),
@@ -2498,6 +2639,22 @@ var repairToolCall = async ({
   });
   return { ...toolCall, input: JSON.stringify(output) };
 };
+function writeText(writer, text) {
+  const feedbackPartId = generateId2();
+  writer.write({
+    id: feedbackPartId,
+    type: "text-start"
+  });
+  writer.write({
+    id: feedbackPartId,
+    type: "text-delta",
+    delta: ` ${text}`
+  });
+  writer.write({
+    id: feedbackPartId,
+    type: "text-end"
+  });
+}
 
 // packages/text2sql/src/lib/agents/explainer.agent.ts
 import { groq as groq2 } from "@ai-sdk/groq";
@@ -2602,10 +2759,12 @@ var fragments = [
 var developer_agent_default = { tools, fragments };
 
 // packages/text2sql/src/lib/agents/result-tools.ts
+import "ai";
 import { createBashTool as createBashTool2 } from "bash-tool";
 import chalk3 from "chalk";
 import {
   Bash,
+  InMemoryFs,
   MountableFs,
   OverlayFs,
   ReadWriteFs,
@@ -2613,6 +2772,7 @@ import {
 } from "just-bash";
 import * as fs from "node:fs/promises";
 import * as path from "node:path";
+import { v7 } from "uuid";
 function createCommand(name, subcommands) {
   const usageLines = Object.entries(subcommands).map(([, def]) => `  ${name} ${def.usage.padEnd(30)} ${def.description}`).join("\n");
   return defineCommand2(name, async (args, ctx) => {
@@ -2663,12 +2823,20 @@ function createSqlCommand(adapter) {
         try {
           const rows = await adapter.execute(query);
           const rowsArray = Array.isArray(rows) ? rows : [];
-          const filePath = `/results/${crypto.randomUUID()}.json`;
-          await ctx.fs.writeFile(filePath, JSON.stringify(rowsArray, null, 2));
+          const content = JSON.stringify(rowsArray, null, 2);
+          const filename = `${v7()}.json`;
+          const isolatedPath = `/results/${filename}`;
+          const sharedPath = `/artifacts/${filename}`;
+          await Promise.all([
+            ctx.fs.writeFile(isolatedPath, content),
+            // Current turn's isolated copy
+            ctx.fs.writeFile(sharedPath, content)
+            // Shared copy for cross-turn access
+          ]);
           const columns = rowsArray.length > 0 ? Object.keys(rowsArray[0]) : [];
           return {
             stdout: [
-              filePath,
+              `results stored in ${sharedPath}`,
               `columns: ${columns.join(", ") || "(none)"}`,
               `rows: ${rowsArray.length}`
             ].join("\n") + "\n",
@@ -2714,14 +2882,24 @@ function createSqlCommand(adapter) {
   });
 }
 async function createResultTools(options) {
-  const { adapter, chatId, messageId } = options;
+  const { adapter, chatId, messageId, skillMounts = [] } = options;
   const sqlCommand = createSqlCommand(adapter);
-  const chatDir = path.join(process.cwd(), "artifacts", chatId);
+  const root = process.env.TEXT2SQL_FS_ROOT || process.cwd();
+  const chatDir = path.join(root, "artifacts", chatId);
   const resultsDir = path.join(chatDir, messageId, "results");
   await fs.mkdir(resultsDir, { recursive: true });
+  const fsMounts = skillMounts.map(({ host, sandbox: sandbox2 }) => ({
+    mountPoint: sandbox2,
+    filesystem: new OverlayFs({
+      root: host,
+      mountPoint: "/",
+      readOnly: true
+    })
+  }));
   const filesystem = new MountableFs({
-    base: new OverlayFs({ root: process.cwd() }),
+    base: new InMemoryFs(),
     mounts: [
+      ...fsMounts,
       {
         mountPoint: "/results",
         filesystem: new ReadWriteFs({ root: resultsDir })
@@ -2739,10 +2917,6 @@ async function createResultTools(options) {
   const { bash, sandbox } = await createBashTool2({
     sandbox: bashInstance,
     destination: "/",
-    uploadDirectory: {
-      source: process.cwd(),
-      include: "packages/text2sql/src/skills/**/*.md"
-    },
     onBeforeBashCall: ({ command }) => {
       console.log(chalk3.cyan(`[onBeforeBashCall]: ${command}`));
       return { command };
@@ -2765,7 +2939,9 @@ import {
   NoContentGeneratedError,
   NoObjectGeneratedError,
   NoOutputGeneratedError,
-  TypeValidationError
+  TypeValidationError,
+  defaultSettingsMiddleware,
+  wrapLanguageModel
 } from "ai";
 import { Console } from "node:console";
 import { createWriteStream } from "node:fs";
@@ -2777,6 +2953,7 @@ var logger = new Console({
   stderr: createWriteStream("./sql-agent-error.log", { flags: "a" }),
   inspectOptions: { depth: null }
 });
+var RETRY_TEMPERATURES = [0, 0.2, 0.3];
 function extractSql(output) {
   const match = output.match(/```sql\n?([\s\S]*?)```/);
   return match ? match[1].trim() : output.trim();
@@ -2830,8 +3007,14 @@ async function toSql(options) {
       } else {
         context.set(user(options.input));
       }
+      const temperature = RETRY_TEMPERATURES[attemptNumber - 1] ?? RETRY_TEMPERATURES[RETRY_TEMPERATURES.length - 1];
+      const baseModel = options.model ?? groq3("openai/gpt-oss-20b");
+      const model = wrapLanguageModel({
+        model: baseModel,
+        middleware: defaultSettingsMiddleware({ settings: { temperature } })
+      });
       const sqlOutput = structuredOutput({
-        model: options.model ?? groq3("openai/gpt-oss-20b"),
+        model,
         context,
         schema: z3.union([
           z3.object({
@@ -2971,12 +3154,12 @@ var suggestionsAgent = agent2({
 });
 
 // packages/text2sql/src/lib/agents/text2sql.agent.ts
-import { tool as tool2 } from "ai";
+import { tool as tool3 } from "ai";
 import z5 from "zod";
 import { toState as toState2 } from "@deepagents/agent";
 import { scratchpad_tool } from "@deepagents/toolbox";
 var tools2 = {
-  validate_query: tool2({
+  validate_query: tool3({
     description: `Validate SQL query syntax before execution. Use this to check if your SQL is valid before running db_query. This helps catch errors early and allows you to correct the query if needed.`,
     inputSchema: z5.object({
       sql: z5.string().describe("The SQL query to validate.")
@@ -2990,7 +3173,7 @@ var tools2 = {
       return "Query is valid.";
     }
   }),
-  db_query: tool2({
+  db_query: tool3({
     description: `Internal tool to fetch data from the store's database. Write a SQL query to retrieve the information needed to answer the user's question. The results will be returned as data that you can then present to the user in natural language.`,
     inputSchema: z5.object({
       reasoning: z5.string().describe(
@@ -3327,6 +3510,10 @@ function reasoningFramework() {
       principle({
         title: "Inhibit your response",
         description: "Only take an action after all the above reasoning is completed. Once you've taken an action, you cannot take it back."
+      }),
+      principle({
+        title: "Continuous self-monitoring",
+        description: "Constantly evaluate your own reasoning process for any gaps, biases, or errors. Apply the above principles iteratively as needed."
       })
     )
   ];
@@ -3540,7 +3727,31 @@ Action: Ask user: "Top by what metric\u2014total revenue, number of orders, or m
         "Maintain consistency with previous query structure when possible."
       ],
       notes: "If reference is ambiguous, ask which previous result or entity the user means."
-    })
+    }),
+    fragment(
+      "Bash tool usage",
+      workflow({
+        task: "Query execution",
+        steps: [
+          'Execute SQL through bash tool: sql run "SELECT ..."',
+          "Read the output: file path, column names, and row count.",
+          "Use column names to construct jq filters: cat <path> | jq '.[] | {col1, col2}'",
+          "For large results, slice first: cat <path> | jq '.[:10]'"
+        ]
+      }),
+      hint(
+        `You cannot access sql through a tool, it'll fail so the proper way to access it is through the bash tool using "sql run" and "sql validate" commands.`
+      ),
+      hint(
+        "The sql command outputs: file path, column names (comma-separated), and row count. Use column names to construct precise jq queries."
+      ),
+      hint(
+        'This is virtual bash environment and "sql" commands proxy to the database hence you cannot access sql files directly.'
+      ),
+      hint(
+        "If a query fails, the sql command returns an error message in stderr."
+      )
+    )
   ];
   if (date === "strict") {
     baseTeachings.push(
@@ -3573,12 +3784,9 @@ var Text2Sql = class {
   #config;
   constructor(config) {
     this.#config = {
+      teachingsOptions: config.teachingsOptions,
       adapter: config.adapter,
-      store: config.store,
-      instructions: [
-        ...guidelines(config.teachingsOptions),
-        ...config.instructions ?? []
-      ],
+      context: config.context,
       tools: config.tools ?? {},
       model: config.model,
       introspection: new JsonCache(
@@ -3592,7 +3800,7 @@ var Text2Sql = class {
       input,
       adapter: this.#config.adapter,
       schemaFragments,
-      instructions: this.#config.instructions,
+      instructions: [],
       model: this.#config.model
     });
     return result.sql;
@@ -3652,45 +3860,24 @@ var Text2Sql = class {
       })
     ];
   }
-  async chat(messages, params) {
-    const schemaFragments = await this.index();
-    const context = new ContextEngine({
-      store: this.#config.store,
-      chatId: params.chatId,
-      userId: params.userId
-    }).set(
-      ...schemaFragments,
-      ...this.#buildRenderingInstructions(),
-      fragment(
-        "Bash tool usage",
-        workflow({
-          task: "Query execution",
-          steps: [
-            'Execute SQL through bash tool: sql run "SELECT ..."',
-            "Read the output: file path, column names, and row count.",
-            "Use column names to construct jq filters: cat <path> | jq '.[] | {col1, col2}'",
-            "For large results, slice first: cat <path> | jq '.[:10]'"
-          ]
-        }),
-        hint(
-          "The sql command outputs: file path, column names (comma-separated), and row count. Use column names to construct precise jq queries."
-        ),
-        hint(
-          "If a query fails, the sql command returns an error message in stderr."
-        )
-      ),
-      ...this.#config.instructions
+  async chat(messages) {
+    const context = this.#config.context(
+      ...guidelines(this.#config.teachingsOptions),
+      ...await this.index(),
+      ...this.#buildRenderingInstructions()
     );
     const userMsg = messages.at(-1);
     if (userMsg) {
-      context.set(user(userMsg));
+      context.set(message(userMsg));
       await context.save();
     }
     const messageId = userMsg?.id ?? generateId3();
+    const skillMounts = context.getSkillMounts();
     const { bash } = await createResultTools({
       adapter: this.#config.adapter,
-      chatId: params.chatId,
-      messageId
+      chatId: context.chatId,
+      messageId,
+      skillMounts
     });
     const chatAgent = agent({
       name: "text2sql",
@@ -3710,31 +3897,25 @@ var Text2Sql = class {
       sendFinish: true,
       sendReasoning: true,
       sendSources: true,
+      originalMessages: messages,
       generateMessageId: generateId3,
       onFinish: async ({ responseMessage }) => {
         context.set(assistant(responseMessage));
         await context.save();
+        await context.trackUsage(await result.totalUsage);
       }
     });
   }
-  /**
-   * Developer chat interface - power-user mode for SQL generation.
-   * Uses db_query tool for direct SQL execution (LLM writes SQL).
-   */
-  async developer(messages, params) {
-    const schemaFragments = await this.index();
-    const context = new ContextEngine({
-      store: this.#config.store,
-      chatId: params.chatId,
-      userId: params.userId
-    }).set(
+  async developer(messages) {
+    const context = this.#config.context(
+      ...guidelines(this.#config.teachingsOptions),
       ...developer_agent_default.fragments,
-      ...this.#config.instructions,
-      ...schemaFragments
+      ...await this.index(),
+      ...this.#buildRenderingInstructions()
     );
     const userMsg = messages.at(-1);
     if (userMsg) {
-      context.set(user(userMsg));
+      context.set(message(userMsg));
       await context.save();
     }
     const developerAgent = agent({
@@ -3756,6 +3937,7 @@ var Text2Sql = class {
       onFinish: async ({ responseMessage }) => {
         context.set(assistant(responseMessage));
         await context.save();
+        await context.trackUsage(await result.totalUsage);
       }
     });
   }