@deepagents/context 0.8.1 → 0.10.0

package/dist/index.js

@@ -1,3 +1,6 @@
+// packages/context/src/index.ts
+import { generateId } from "ai";
+
 // packages/context/src/lib/context.ts
 function isFragment(data) {
   return typeof data === "object" && data !== null && "name" in data && "data" in data && typeof data.name === "string";
@@ -119,7 +122,8 @@ var ModelsRegistry = class {
         context: model.limit.context,
         output: model.limit.output,
         exceedsContext: tokens > model.limit.context
-      }
+      },
+      fragments: []
     };
   }
 };
@@ -130,6 +134,43 @@ function getModelsRegistry() {
   }
   return _registry;
 }
+async function estimate(modelId, renderer, ...fragments) {
+  const registry = getModelsRegistry();
+  await registry.load();
+  const input = renderer.render(fragments);
+  const model = registry.get(modelId);
+  if (!model) {
+    throw new Error(
+      `Model "${modelId}" not found. Call load() first or check model ID.`
+    );
+  }
+  const tokenizer = registry.getTokenizer(modelId);
+  const totalTokens = tokenizer.count(input);
+  const totalCost = totalTokens / 1e6 * model.cost.input;
+  const fragmentEstimates = fragments.map((fragment2) => {
+    const rendered = renderer.render([fragment2]);
+    const tokens = tokenizer.count(rendered);
+    const cost = tokens / 1e6 * model.cost.input;
+    return {
+      id: fragment2.id,
+      name: fragment2.name,
+      tokens,
+      cost
+    };
+  });
+  return {
+    model: model.id,
+    provider: model.provider,
+    tokens: totalTokens,
+    cost: totalCost,
+    limits: {
+      context: model.limit.context,
+      output: model.limit.output,
+      exceedsContext: totalTokens > model.limit.context
+    },
+    fragments: fragmentEstimates
+  };
+}
 
 // packages/context/src/lib/renderers/abstract.renderer.ts
 import pluralize from "pluralize";
@@ -789,12 +830,13 @@ var ContextStore = class {
 import { DatabaseSync } from "node:sqlite";
 var STORE_DDL = `
 -- Chats table
+-- createdAt/updatedAt: DEFAULT for insert, inline SET for updates
 CREATE TABLE IF NOT EXISTS chats (
   id TEXT PRIMARY KEY,
   title TEXT,
   metadata TEXT,
-  createdAt INTEGER NOT NULL,
-  updatedAt INTEGER NOT NULL
+  createdAt INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
+  updatedAt INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000)
 );
 
 CREATE INDEX IF NOT EXISTS idx_chats_updatedAt ON chats(updatedAt);
@@ -807,7 +849,6 @@ CREATE TABLE IF NOT EXISTS messages (
   name TEXT NOT NULL,
   type TEXT,
   data TEXT NOT NULL,
-  persist INTEGER NOT NULL DEFAULT 1,
   createdAt INTEGER NOT NULL,
   FOREIGN KEY (chatId) REFERENCES chats(id) ON DELETE CASCADE,
   FOREIGN KEY (parentId) REFERENCES messages(id)
@@ -846,13 +887,13 @@ CREATE TABLE IF NOT EXISTS checkpoints (
 CREATE INDEX IF NOT EXISTS idx_checkpoints_chatId ON checkpoints(chatId);
 
 -- FTS5 virtual table for full-text search
--- Using external content mode (content='') so we manage sync manually
+-- messageId/chatId/name are UNINDEXED (stored but not searchable, used for filtering/joining)
+-- Only 'content' is indexed for full-text search
 CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
-  messageId,
-  chatId,
-  name,
+  messageId UNINDEXED,
+  chatId UNINDEXED,
+  name UNINDEXED,
   content,
-  content='',
   tokenize='porter unicode61'
 );
 `;
@@ -869,16 +910,33 @@ var SqliteContextStore = class extends ContextStore {
   // ==========================================================================
   async createChat(chat) {
     this.#db.prepare(
-      `INSERT INTO chats (id, title, metadata, createdAt, updatedAt)
-         VALUES (?, ?, ?, ?, ?)`
+      `INSERT INTO chats (id, title, metadata)
+         VALUES (?, ?, ?)`
     ).run(
       chat.id,
       chat.title ?? null,
-      chat.metadata ? JSON.stringify(chat.metadata) : null,
-      chat.createdAt,
-      chat.updatedAt
+      chat.metadata ? JSON.stringify(chat.metadata) : null
     );
   }
+  async upsertChat(chat) {
+    const row = this.#db.prepare(
+      `INSERT INTO chats (id, title, metadata)
+         VALUES (?, ?, ?)
+         ON CONFLICT(id) DO UPDATE SET id = excluded.id
+         RETURNING *`
+    ).get(
+      chat.id,
+      chat.title ?? null,
+      chat.metadata ? JSON.stringify(chat.metadata) : null
+    );
+    return {
+      id: row.id,
+      title: row.title ?? void 0,
+      metadata: row.metadata ? JSON.parse(row.metadata) : void 0,
+      createdAt: row.createdAt,
+      updatedAt: row.updatedAt
+    };
+  }
   async getChat(chatId) {
     const row = this.#db.prepare("SELECT * FROM chats WHERE id = ?").get(chatId);
     if (!row) {
@@ -893,7 +951,7 @@ var SqliteContextStore = class extends ContextStore {
     };
   }
   async updateChat(chatId, updates) {
-    const setClauses = [];
+    const setClauses = ["updatedAt = strftime('%s', 'now') * 1000"];
     const params = [];
     if (updates.title !== void 0) {
       setClauses.push("title = ?");
@@ -903,15 +961,17 @@ var SqliteContextStore = class extends ContextStore {
       setClauses.push("metadata = ?");
       params.push(JSON.stringify(updates.metadata));
     }
-    if (updates.updatedAt !== void 0) {
-      setClauses.push("updatedAt = ?");
-      params.push(updates.updatedAt);
-    }
-    if (setClauses.length === 0) {
-      return;
-    }
     params.push(chatId);
-    this.#db.prepare(`UPDATE chats SET ${setClauses.join(", ")} WHERE id = ?`).run(...params);
+    const row = this.#db.prepare(
+      `UPDATE chats SET ${setClauses.join(", ")} WHERE id = ? RETURNING *`
+    ).get(...params);
+    return {
+      id: row.id,
+      title: row.title ?? void 0,
+      metadata: row.metadata ? JSON.parse(row.metadata) : void 0,
+      createdAt: row.createdAt,
+      updatedAt: row.updatedAt
+    };
   }
   async listChats() {
     const rows = this.#db.prepare(
@@ -940,25 +1000,30 @@ var SqliteContextStore = class extends ContextStore {
   // ==========================================================================
   // Message Operations (Graph Nodes)
   // ==========================================================================
-  async addMessage(message) {
+  async addMessage(message2) {
     this.#db.prepare(
-      `INSERT INTO messages (id, chatId, parentId, name, type, data, persist, createdAt)
-         VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
+      `INSERT INTO messages (id, chatId, parentId, name, type, data, createdAt)
+         VALUES (?, ?, ?, ?, ?, ?, ?)
+         ON CONFLICT(id) DO UPDATE SET
+           parentId = excluded.parentId,
+           name = excluded.name,
+           type = excluded.type,
+           data = excluded.data`
     ).run(
-      message.id,
-      message.chatId,
-      message.parentId,
-      message.name,
-      message.type ?? null,
-      JSON.stringify(message.data),
-      message.persist ? 1 : 0,
-      message.createdAt
+      message2.id,
+      message2.chatId,
+      message2.parentId,
+      message2.name,
+      message2.type ?? null,
+      JSON.stringify(message2.data),
+      message2.createdAt
     );
-    const content = typeof message.data === "string" ? message.data : JSON.stringify(message.data);
+    const content = typeof message2.data === "string" ? message2.data : JSON.stringify(message2.data);
+    this.#db.prepare(`DELETE FROM messages_fts WHERE messageId = ?`).run(message2.id);
     this.#db.prepare(
       `INSERT INTO messages_fts(messageId, chatId, name, content)
          VALUES (?, ?, ?, ?)`
-    ).run(message.id, message.chatId, message.name, content);
+    ).run(message2.id, message2.chatId, message2.name, content);
   }
   async getMessage(messageId) {
     const row = this.#db.prepare("SELECT * FROM messages WHERE id = ?").get(messageId);
@@ -972,7 +1037,6 @@ var SqliteContextStore = class extends ContextStore {
       name: row.name,
       type: row.type ?? void 0,
       data: JSON.parse(row.data),
-      persist: row.persist === 1,
       createdAt: row.createdAt
     };
   }
@@ -994,7 +1058,6 @@ var SqliteContextStore = class extends ContextStore {
       name: row.name,
       type: row.type ?? void 0,
       data: JSON.parse(row.data),
-      persist: row.persist === 1,
       createdAt: row.createdAt
     }));
   }
@@ -1155,7 +1218,6 @@ var SqliteContextStore = class extends ContextStore {
         m.name,
         m.type,
         m.data,
-        m.persist,
         m.createdAt,
         fts.rank,
         snippet(messages_fts, 3, '<mark>', '</mark>', '...', 32) as snippet
@@ -1181,7 +1243,6 @@ var SqliteContextStore = class extends ContextStore {
         name: row.name,
         type: row.type ?? void 0,
         data: JSON.parse(row.data),
-        persist: row.persist === 1,
         createdAt: row.createdAt
       },
       rank: row.rank,
@@ -1305,7 +1366,7 @@ function visualizeGraph(data) {
 // packages/context/src/index.ts
 var ContextEngine = class {
   /** Non-message fragments (role, hints, etc.) - not persisted in graph */
-  #contextFragments = [];
+  #fragments = [];
   /** Pending message fragments to be added to graph */
   #pendingMessages = [];
   #store;
@@ -1329,18 +1390,7 @@ var ContextEngine = class {
     if (this.#initialized) {
       return;
     }
-    const existingChat = await this.#store.getChat(this.#chatId);
-    if (existingChat) {
-      this.#chatData = existingChat;
-    } else {
-      const now = Date.now();
-      this.#chatData = {
-        id: this.#chatId,
-        createdAt: now,
-        updatedAt: now
-      };
-      await this.#store.createChat(this.#chatData);
-    }
+    this.#chatData = await this.#store.upsertChat({ id: this.#chatId });
     const existingBranch = await this.#store.getBranch(
       this.#chatId,
       this.#branchName
@@ -1360,6 +1410,41 @@ var ContextEngine = class {
     }
     this.#initialized = true;
   }
+  /**
+   * Create a new branch from a specific message.
+   * Shared logic between rewind() and btw().
+   */
+  async #createBranchFrom(messageId, switchTo) {
+    const branches = await this.#store.listBranches(this.#chatId);
+    const samePrefix = branches.filter(
+      (b) => b.name === this.#branchName || b.name.startsWith(`${this.#branchName}-v`)
+    );
+    const newBranchName = `${this.#branchName}-v${samePrefix.length + 1}`;
+    const newBranch = {
+      id: crypto.randomUUID(),
+      chatId: this.#chatId,
+      name: newBranchName,
+      headMessageId: messageId,
+      isActive: false,
+      createdAt: Date.now()
+    };
+    await this.#store.createBranch(newBranch);
+    if (switchTo) {
+      await this.#store.setActiveBranch(this.#chatId, newBranch.id);
+      this.#branch = { ...newBranch, isActive: true };
+      this.#branchName = newBranchName;
+      this.#pendingMessages = [];
+    }
+    const chain = await this.#store.getMessageChain(messageId);
+    return {
+      id: newBranch.id,
+      name: newBranch.name,
+      headMessageId: newBranch.headMessageId,
+      isActive: switchTo,
+      messageCount: chain.length,
+      createdAt: newBranch.createdAt
+    };
+  }
   /**
    * Get the current chat ID.
    */
@@ -1399,7 +1484,7 @@ var ContextEngine = class {
       if (isMessageFragment(fragment2)) {
         this.#pendingMessages.push(fragment2);
       } else {
-        this.#contextFragments.push(fragment2);
+        this.#fragments.push(fragment2);
       }
     }
     return this;
@@ -1409,7 +1494,7 @@ var ContextEngine = class {
    * @internal Use resolve() instead for public API.
    */
   render(renderer) {
-    return renderer.render(this.#contextFragments);
+    return renderer.render(this.#fragments);
   }
   /**
    * Resolve context into AI SDK-ready format.
@@ -1428,26 +1513,21 @@ var ContextEngine = class {
    * await generateText({ system: systemPrompt, messages });
    * ```
    */
-  async resolve(options = {}) {
+  async resolve(options) {
     await this.#ensureInitialized();
-    const renderer = options.renderer ?? new XmlRenderer();
-    const systemPrompt = renderer.render(this.#contextFragments);
-    const persistedMessages = [];
+    const systemPrompt = options.renderer.render(this.#fragments);
+    const messages = [];
     if (this.#branch?.headMessageId) {
       const chain = await this.#store.getMessageChain(
         this.#branch.headMessageId
       );
-      persistedMessages.push(...chain);
+      for (const msg of chain) {
+        messages.push(message(msg.data).codec?.decode());
+      }
     }
-    const messages = persistedMessages.map((msg) => ({
-      role: msg.name,
-      content: String(msg.data)
-    }));
     for (const fragment2 of this.#pendingMessages) {
-      messages.push({
-        role: fragment2.name,
-        content: String(fragment2.data)
-      });
+      const decoded = fragment2.codec.decode();
+      messages.push(decoded);
     }
     return { systemPrompt, messages };
   }
@@ -1479,8 +1559,7 @@ var ContextEngine = class {
         parentId,
         name: fragment2.name,
         type: fragment2.type,
-        data: fragment2.data,
-        persist: fragment2.persist ?? true,
+        data: fragment2.codec.encode(),
         createdAt: now
       };
       await this.#store.addMessage(messageData);
@@ -1488,25 +1567,85 @@ var ContextEngine = class {
     }
     await this.#store.updateBranchHead(this.#branch.id, parentId);
     this.#branch.headMessageId = parentId;
-    await this.#store.updateChat(this.#chatId, { updatedAt: now });
-    if (this.#chatData) {
-      this.#chatData.updatedAt = now;
-    }
     this.#pendingMessages = [];
   }
   /**
-   * Estimate token count and cost for the current context.
+   * Estimate token count and cost for the full context.
+   *
+   * Includes:
+   * - System prompt fragments (role, hints, etc.)
+   * - Persisted chat messages (from store)
+   * - Pending messages (not yet saved)
    *
    * @param modelId - Model ID (e.g., "openai:gpt-4o", "anthropic:claude-3-5-sonnet")
    * @param options - Optional settings
-   * @returns Estimate result with token counts and costs
+   * @returns Estimate result with token counts, costs, and per-fragment breakdown
    */
   async estimate(modelId, options = {}) {
+    await this.#ensureInitialized();
     const renderer = options.renderer ?? new XmlRenderer();
-    const renderedContext = this.render(renderer);
     const registry = getModelsRegistry();
     await registry.load();
-    return registry.estimate(modelId, renderedContext);
+    const model = registry.get(modelId);
+    if (!model) {
+      throw new Error(
+        `Model "${modelId}" not found. Call load() first or check model ID.`
+      );
+    }
+    const tokenizer = registry.getTokenizer(modelId);
+    const fragmentEstimates = [];
+    for (const fragment2 of this.#fragments) {
+      const rendered = renderer.render([fragment2]);
+      const tokens = tokenizer.count(rendered);
+      const cost = tokens / 1e6 * model.cost.input;
+      fragmentEstimates.push({
+        id: fragment2.id,
+        name: fragment2.name,
+        tokens,
+        cost
+      });
+    }
+    if (this.#branch?.headMessageId) {
+      const chain = await this.#store.getMessageChain(
+        this.#branch.headMessageId
+      );
+      for (const msg of chain) {
+        const content = String(msg.data);
+        const tokens = tokenizer.count(content);
+        const cost = tokens / 1e6 * model.cost.input;
+        fragmentEstimates.push({
+          name: msg.name,
+          id: msg.id,
+          tokens,
+          cost
+        });
+      }
+    }
+    for (const fragment2 of this.#pendingMessages) {
+      const content = String(fragment2.data);
+      const tokens = tokenizer.count(content);
+      const cost = tokens / 1e6 * model.cost.input;
+      fragmentEstimates.push({
+        name: fragment2.name,
+        id: fragment2.id,
+        tokens,
+        cost
+      });
+    }
+    const totalTokens = fragmentEstimates.reduce((sum, f) => sum + f.tokens, 0);
+    const totalCost = fragmentEstimates.reduce((sum, f) => sum + f.cost, 0);
+    return {
+      model: model.id,
+      provider: model.provider,
+      tokens: totalTokens,
+      cost: totalCost,
+      limits: {
+        context: model.limit.context,
+        output: model.limit.output,
+        exceedsContext: totalTokens > model.limit.context
+      },
+      fragments: fragmentEstimates
+    };
   }
   /**
    * Rewind to a specific message by ID.
@@ -1533,37 +1672,14 @@ var ContextEngine = class {
    */
   async rewind(messageId) {
     await this.#ensureInitialized();
-    const message = await this.#store.getMessage(messageId);
-    if (!message) {
+    const message2 = await this.#store.getMessage(messageId);
+    if (!message2) {
       throw new Error(`Message "${messageId}" not found`);
     }
-    if (message.chatId !== this.#chatId) {
+    if (message2.chatId !== this.#chatId) {
       throw new Error(`Message "${messageId}" belongs to a different chat`);
     }
-    const branches = await this.#store.listBranches(this.#chatId);
-    const newBranchName = `${this.#branchName}-v${branches.length + 1}`;
-    const newBranch = {
-      id: crypto.randomUUID(),
-      chatId: this.#chatId,
-      name: newBranchName,
-      headMessageId: messageId,
-      isActive: false,
-      createdAt: Date.now()
-    };
-    await this.#store.createBranch(newBranch);
-    await this.#store.setActiveBranch(this.#chatId, newBranch.id);
-    this.#branch = { ...newBranch, isActive: true };
-    this.#branchName = newBranchName;
-    this.#pendingMessages = [];
-    const chain = await this.#store.getMessageChain(messageId);
-    return {
-      id: newBranch.id,
-      name: newBranch.name,
-      headMessageId: newBranch.headMessageId,
-      isActive: true,
-      messageCount: chain.length,
-      createdAt: newBranch.createdAt
-    };
+    return this.#createBranchFrom(messageId, true);
   }
   /**
    * Create a checkpoint at the current position.
@@ -1656,6 +1772,44 @@ var ContextEngine = class {
     this.#branchName = name;
     this.#pendingMessages = [];
   }
+  /**
+   * Create a parallel branch from the current position ("by the way").
+   *
+   * Use this when you want to fork the conversation without leaving
+   * the current branch. Common use case: user wants to ask another
+   * question while waiting for the model to respond.
+   *
+   * Unlike rewind(), this method:
+   * - Uses the current HEAD (no messageId needed)
+   * - Does NOT switch to the new branch
+   * - Keeps pending messages intact
+   *
+   * @returns The new branch info (does not switch to it)
+   * @throws Error if no messages exist in the conversation
+   *
+   * @example
+   * ```ts
+   * // User asked a question, model is generating...
+   * context.set(user('What is the weather?'));
+   * await context.save();
+   *
+   * // User wants to ask something else without waiting
+   * const newBranch = await context.btw();
+   * // newBranch = { name: 'main-v2', ... }
+   *
+   * // Later, switch to the new branch and add the question
+   * await context.switchBranch(newBranch.name);
+   * context.set(user('Also, what time is it?'));
+   * await context.save();
+   * ```
+   */
+  async btw() {
+    await this.#ensureInitialized();
+    if (!this.#branch?.headMessageId) {
+      throw new Error("Cannot create btw branch: no messages in conversation");
+    }
+    return this.#createBranchFrom(this.#branch.headMessageId, false);
+  }
   /**
    * Update metadata for the current chat.
    *
@@ -1671,10 +1825,7 @@ var ContextEngine = class {
    */
   async updateChat(updates) {
     await this.#ensureInitialized();
-    const now = Date.now();
-    const storeUpdates = {
-      updatedAt: now
-    };
+    const storeUpdates = {};
     if (updates.title !== void 0) {
       storeUpdates.title = updates.title;
     }
@@ -1684,16 +1835,7 @@ var ContextEngine = class {
         ...updates.metadata
       };
     }
-    await this.#store.updateChat(this.#chatId, storeUpdates);
-    if (this.#chatData) {
-      if (storeUpdates.title !== void 0) {
-        this.#chatData.title = storeUpdates.title;
-      }
-      if (storeUpdates.metadata !== void 0) {
-        this.#chatData.metadata = storeUpdates.metadata;
-      }
-      this.#chatData.updatedAt = now;
-    }
+    this.#chatData = await this.#store.updateChat(this.#chatId, storeUpdates);
   }
   /**
    * Consolidate context fragments (no-op for now).
@@ -1706,6 +1848,54 @@ var ContextEngine = class {
   consolidate() {
     return void 0;
   }
+  /**
+   * Inspect the full context state for debugging.
+   * Returns a comprehensive JSON-serializable object with all context information.
+   *
+   * @param options - Inspection options (modelId and renderer required)
+   * @returns Complete inspection data including estimates, rendered output, fragments, and graph
+   *
+   * @example
+   * ```ts
+   * const inspection = await context.inspect({
+   *   modelId: 'openai:gpt-4o',
+   *   renderer: new XmlRenderer(),
+   * });
+   * console.log(JSON.stringify(inspection, null, 2));
+   *
+   * // Or write to file for analysis
+   * await fs.writeFile('context-debug.json', JSON.stringify(inspection, null, 2));
+   * ```
+   */
+  async inspect(options) {
+    await this.#ensureInitialized();
+    const { renderer } = options;
+    const estimateResult = await this.estimate(options.modelId, { renderer });
+    const rendered = renderer.render(this.#fragments);
+    const persistedMessages = [];
+    if (this.#branch?.headMessageId) {
+      const chain = await this.#store.getMessageChain(
+        this.#branch.headMessageId
+      );
+      persistedMessages.push(...chain);
+    }
+    const graph = await this.#store.getGraph(this.#chatId);
+    return {
+      estimate: estimateResult,
+      rendered,
+      fragments: {
+        context: [...this.#fragments],
+        pending: [...this.#pendingMessages],
+        persisted: persistedMessages
+      },
+      graph,
+      meta: {
+        chatId: this.#chatId,
+        branch: this.#branchName,
+        timestamp: Date.now()
+      }
+    };
+  }
 };
 function hint(text) {
   return {
@@ -1725,24 +1915,75 @@ function role(content) {
     data: content
   };
 }
-function user(content, options) {
+function user(content) {
+  const message2 = typeof content === "string" ? {
+    id: generateId(),
+    role: "user",
+    parts: [{ type: "text", text: content }]
+  } : content;
   return {
-    id: options?.id ?? crypto.randomUUID(),
+    id: message2.id,
     name: "user",
-    data: content,
+    data: "content",
     type: "message",
-    persist: true
+    persist: true,
+    codec: {
+      decode() {
+        return message2;
+      },
+      encode() {
+        return message2;
+      }
+    }
   };
 }
-function assistant(content, options) {
+function assistant(message2) {
   return {
-    id: options?.id ?? crypto.randomUUID(),
+    id: message2.id,
     name: "assistant",
-    data: content,
+    data: "content",
     type: "message",
-    persist: true
+    persist: true,
+    codec: {
+      decode() {
+        return message2;
+      },
+      encode() {
+        return message2;
+      }
+    }
   };
 }
+function message(content) {
+  const message2 = typeof content === "string" ? {
+    id: generateId(),
+    role: "user",
+    parts: [{ type: "text", text: content }]
+  } : content;
+  return {
+    id: message2.id,
+    name: "message",
+    data: "content",
+    type: "message",
+    persist: true,
+    codec: {
+      decode() {
+        return message2;
+      },
+      encode() {
+        return message2;
+      }
+    }
+  };
+}
+function assistantText(content, options) {
+  const id = options?.id ?? crypto.randomUUID();
+  return assistant({
+    id,
+    role: "assistant",
+    parts: [{ type: "text", text: content }]
+  });
+}
 export {
   ContextEngine,
   ContextStore,
@@ -1754,11 +1995,14 @@ export {
   ToonRenderer,
   XmlRenderer,
   assistant,
+  assistantText,
   defaultTokenizer,
+  estimate,
   fragment,
   getModelsRegistry,
   hint,
   isMessageFragment,
+  message,
   role,
   user,
   visualizeGraph