@tyvm/knowhow 0.0.144 → 0.0.145

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. package/bin/knowhow.js +22 -20
  2. package/package.json +1 -1
  3. package/scripts/migrate-usage-json.js +147 -0
  4. package/src/agents/base/base.ts +65 -10
  5. package/src/agents/tools/patch.ts +84 -2
  6. package/src/agents/tools/textSearch.ts +98 -3
  7. package/src/chat/modules/AgentModule.ts +129 -6
  8. package/src/chat/modules/SessionsModule.ts +6 -0
  9. package/src/chat/types.ts +14 -0
  10. package/src/chat.ts +11 -0
  11. package/src/cli.ts +2 -0
  12. package/src/clients/anthropic.ts +56 -1
  13. package/src/clients/contextLimits.ts +40 -0
  14. package/src/clients/gemini.ts +59 -5
  15. package/src/clients/http.ts +85 -4
  16. package/src/clients/index.ts +4 -0
  17. package/src/clients/openai.ts +179 -31
  18. package/src/clients/pricing/fireworks.ts +50 -1
  19. package/src/clients/pricing/index.ts +1 -1
  20. package/src/clients/pricing/models.ts +3 -1
  21. package/src/clients/pricing/openai.ts +2 -2
  22. package/src/clients/pricing/qwen.ts +80 -0
  23. package/src/clients/qwen.ts +16 -0
  24. package/src/clients/types.ts +69 -1
  25. package/src/clients/xai.ts +73 -4
  26. package/src/commands/agent.ts +20 -1
  27. package/src/index.ts +9 -0
  28. package/src/processors/Base64ImageDetector.ts +46 -7
  29. package/src/processors/GemmaToolProcessor.ts +163 -0
  30. package/src/processors/index.ts +1 -0
  31. package/src/services/AgentSyncFs.ts +37 -0
  32. package/src/services/SessionManager.ts +14 -0
  33. package/src/services/Tools.ts +3 -0
  34. package/src/services/modules/computerUse.ts +216 -0
  35. package/src/services/modules/index.ts +47 -9
  36. package/src/services/modules/types.ts +23 -1
  37. package/src/types.ts +3 -0
  38. package/tests/clients/contextLimits.test.ts +4 -0
  39. package/tests/clients/openai.images.test.ts +105 -0
  40. package/tests/clients/pricing.test.ts +6 -0
  41. package/tests/compressor/usageBase64.json +159 -0
  42. package/tests/compressor/usageBase64.test.ts +175 -0
  43. package/tests/patching/fixtures/transcript-mangles/agent-sync-fs-nested-catch.json +9 -0
  44. package/tests/patching/fixtures/transcript-mangles/automation-functions-crossed.json +9 -0
  45. package/tests/patching/fixtures/transcript-mangles/fruit-ninja-confirmed-retirement.json +8 -0
  46. package/tests/patching/patchFileOutput.test.ts +112 -1
  47. package/tests/patching/transcriptMangleRegression.test.ts +125 -0
  48. package/tests/processors/Base64ImageDetector.test.ts +36 -1
  49. package/ts_build/package.json +1 -1
  50. package/ts_build/src/agents/base/base.d.ts +8 -1
  51. package/ts_build/src/agents/base/base.js +37 -3
  52. package/ts_build/src/agents/base/base.js.map +1 -1
  53. package/ts_build/src/agents/tools/patch.d.ts +1 -0
  54. package/ts_build/src/agents/tools/patch.js +35 -2
  55. package/ts_build/src/agents/tools/patch.js.map +1 -1
  56. package/ts_build/src/agents/tools/textSearch.js +56 -2
  57. package/ts_build/src/agents/tools/textSearch.js.map +1 -1
  58. package/ts_build/src/chat/modules/AgentModule.d.ts +14 -1
  59. package/ts_build/src/chat/modules/AgentModule.js +73 -5
  60. package/ts_build/src/chat/modules/AgentModule.js.map +1 -1
  61. package/ts_build/src/chat/modules/SessionsModule.js +4 -0
  62. package/ts_build/src/chat/modules/SessionsModule.js.map +1 -1
  63. package/ts_build/src/chat/types.d.ts +11 -0
  64. package/ts_build/src/chat.js +7 -0
  65. package/ts_build/src/chat.js.map +1 -1
  66. package/ts_build/src/cli.d.ts +2 -1
  67. package/ts_build/src/cli.js +1 -0
  68. package/ts_build/src/cli.js.map +1 -1
  69. package/ts_build/src/clients/anthropic.js +28 -1
  70. package/ts_build/src/clients/anthropic.js.map +1 -1
  71. package/ts_build/src/clients/contextLimits.js +25 -0
  72. package/ts_build/src/clients/contextLimits.js.map +1 -1
  73. package/ts_build/src/clients/gemini.js +48 -4
  74. package/ts_build/src/clients/gemini.js.map +1 -1
  75. package/ts_build/src/clients/http.d.ts +2 -0
  76. package/ts_build/src/clients/http.js +66 -9
  77. package/ts_build/src/clients/http.js.map +1 -1
  78. package/ts_build/src/clients/index.d.ts +1 -0
  79. package/ts_build/src/clients/index.js +4 -0
  80. package/ts_build/src/clients/index.js.map +1 -1
  81. package/ts_build/src/clients/openai.d.ts +6 -1
  82. package/ts_build/src/clients/openai.js +131 -33
  83. package/ts_build/src/clients/openai.js.map +1 -1
  84. package/ts_build/src/clients/pricing/fireworks.d.ts +4 -1
  85. package/ts_build/src/clients/pricing/fireworks.js +33 -1
  86. package/ts_build/src/clients/pricing/fireworks.js.map +1 -1
  87. package/ts_build/src/clients/pricing/index.d.ts +1 -1
  88. package/ts_build/src/clients/pricing/index.js +6 -2
  89. package/ts_build/src/clients/pricing/index.js.map +1 -1
  90. package/ts_build/src/clients/pricing/models.d.ts +2 -1
  91. package/ts_build/src/clients/pricing/models.js +4 -1
  92. package/ts_build/src/clients/pricing/models.js.map +1 -1
  93. package/ts_build/src/clients/pricing/openai.js +2 -2
  94. package/ts_build/src/clients/pricing/openai.js.map +1 -1
  95. package/ts_build/src/clients/pricing/qwen.d.ts +5 -0
  96. package/ts_build/src/clients/pricing/qwen.js +30 -0
  97. package/ts_build/src/clients/pricing/qwen.js.map +1 -0
  98. package/ts_build/src/clients/qwen.d.ts +4 -0
  99. package/ts_build/src/clients/qwen.js +15 -0
  100. package/ts_build/src/clients/qwen.js.map +1 -0
  101. package/ts_build/src/clients/types.d.ts +12 -0
  102. package/ts_build/src/clients/xai.d.ts +1 -0
  103. package/ts_build/src/clients/xai.js +56 -9
  104. package/ts_build/src/clients/xai.js.map +1 -1
  105. package/ts_build/src/commands/agent.js +10 -1
  106. package/ts_build/src/commands/agent.js.map +1 -1
  107. package/ts_build/src/index.d.ts +2 -0
  108. package/ts_build/src/index.js +1 -0
  109. package/ts_build/src/index.js.map +1 -1
  110. package/ts_build/src/processors/Base64ImageDetector.js +22 -3
  111. package/ts_build/src/processors/Base64ImageDetector.js.map +1 -1
  112. package/ts_build/src/processors/GemmaToolProcessor.d.ts +14 -0
  113. package/ts_build/src/processors/GemmaToolProcessor.js +97 -0
  114. package/ts_build/src/processors/GemmaToolProcessor.js.map +1 -0
  115. package/ts_build/src/processors/index.d.ts +1 -0
  116. package/ts_build/src/processors/index.js +3 -1
  117. package/ts_build/src/processors/index.js.map +1 -1
  118. package/ts_build/src/services/AgentSyncFs.js +17 -0
  119. package/ts_build/src/services/AgentSyncFs.js.map +1 -1
  120. package/ts_build/src/services/SessionManager.js +12 -0
  121. package/ts_build/src/services/SessionManager.js.map +1 -1
  122. package/ts_build/src/services/Tools.d.ts +2 -0
  123. package/ts_build/src/services/Tools.js.map +1 -1
  124. package/ts_build/src/services/modules/computerUse.d.ts +163 -0
  125. package/ts_build/src/services/modules/computerUse.js +3 -0
  126. package/ts_build/src/services/modules/computerUse.js.map +1 -0
  127. package/ts_build/src/services/modules/index.d.ts +1 -0
  128. package/ts_build/src/services/modules/index.js +23 -6
  129. package/ts_build/src/services/modules/index.js.map +1 -1
  130. package/ts_build/src/services/modules/types.d.ts +4 -0
  131. package/ts_build/src/types.d.ts +3 -0
  132. package/ts_build/src/types.js.map +1 -1
  133. package/ts_build/src/workers/tools/index.d.ts +1 -1
  134. package/ts_build/tests/clients/contextLimits.test.js +2 -0
  135. package/ts_build/tests/clients/contextLimits.test.js.map +1 -1
  136. package/ts_build/tests/clients/openai.images.test.d.ts +1 -0
  137. package/ts_build/tests/clients/openai.images.test.js +94 -0
  138. package/ts_build/tests/clients/openai.images.test.js.map +1 -0
  139. package/ts_build/tests/clients/pricing.test.js +4 -0
  140. package/ts_build/tests/clients/pricing.test.js.map +1 -1
  141. package/ts_build/tests/compressor/usageBase64.test.d.ts +1 -0
  142. package/ts_build/tests/compressor/usageBase64.test.js +109 -0
  143. package/ts_build/tests/compressor/usageBase64.test.js.map +1 -0
  144. package/ts_build/tests/patching/patchFileOutput.test.js +82 -0
  145. package/ts_build/tests/patching/patchFileOutput.test.js.map +1 -1
  146. package/ts_build/tests/patching/transcriptMangleRegression.test.d.ts +1 -0
  147. package/ts_build/tests/patching/transcriptMangleRegression.test.js +104 -0
  148. package/ts_build/tests/patching/transcriptMangleRegression.test.js.map +1 -0
  149. package/ts_build/tests/processors/Base64ImageDetector.test.js +24 -0
  150. package/ts_build/tests/processors/Base64ImageDetector.test.js.map +1 -1
  151. package/x-home.png +0 -0
@@ -0,0 +1,16 @@
1
+ import { HttpClient } from "./http";
2
+ import { QwenTextPricing } from "./pricing/qwen";
3
+
4
+ /**
5
+ * QWEN Cloud client — OpenAI-compatible API
6
+ * https://dashscope-intl.aliyuncs.com/compatible-mode/v1
7
+ * Alibaba Cloud's Qwen model family via the international DashScope endpoint.
8
+ * Set env var QWEN_CLOUD_API_KEY to enable.
9
+ */
10
+ export class GenericQwenClient extends HttpClient {
11
+ constructor(apiKey = process.env.QWEN_CLOUD_API_KEY) {
12
+ super("https://dashscope-intl.aliyuncs.com/compatible-mode/v1");
13
+ if (apiKey) this.setJwt(apiKey);
14
+ this.setPrices(QwenTextPricing);
15
+ }
16
+ }
@@ -8,7 +8,10 @@ export type ModelModality =
8
8
 
9
9
  export type MessageContent =
10
10
  | { type: "text"; text: string }
11
- | { type: "image_url"; image_url: { url: string } }
11
+ | {
12
+ type: "image_url";
13
+ image_url: { url: string; detail?: "auto" | "low" | "high" };
14
+ }
12
15
  | { type: "audio_url"; audio_url: { url: string } }
13
16
  | { type: "video_url"; video_url: { url: string } };
14
17
 
@@ -23,6 +26,50 @@ export interface Message {
23
26
 
24
27
  export interface OutputMessage extends Message {
25
28
  content?: string | null;
29
+ /**
30
+ * The model's reasoning/thinking content, when the provider exposes it.
31
+ * - OpenAI Responses API: a human-readable summary of the model's reasoning
32
+ * (populated when reasoning_summary is requested).
33
+ */
34
+ reasoning_summary?: string;
35
+ /**
36
+ * OpenAI Responses API only (internal): the raw reasoning output items
37
+ * (carrying encrypted_content) captured from a stateless response. These are
38
+ * re-injected into the next request's input to preserve the reasoning chain.
39
+ * Not intended for display.
40
+ *
41
+ * @deprecated Prefer the provider-agnostic `_reasoning_details` slot. This is
42
+ * retained for backward compatibility with the OpenAI client only.
43
+ */
44
+ _reasoning_items?: any[];
45
+ /**
46
+ * Provider-agnostic (internal): an opaque, provider-tagged carrier for the
47
+ * model's reasoning artifact that must be round-tripped verbatim on the next
48
+ * tool-use turn to preserve reasoning continuity. Each provider stashes its
49
+ * own native artifact here and re-injects it only when the `provider` tag
50
+ * matches, so provider-specific payload never leaks into the generic
51
+ * (OpenAI-format) message shape.
52
+ *
53
+ * - openai / xai: Responses API `reasoning` output items (encrypted_content)
54
+ * - anthropic: `thinking` / `redacted_thinking` blocks (with signature/data)
55
+ * - gemini: `thoughtSignature` strings keyed by tool-call id (+ thought parts)
56
+ *
57
+ * Not intended for display.
58
+ */
59
+ _reasoning_details?: ReasoningDetails;
60
+ }
61
+
62
+ /**
63
+ * Opaque, provider-tagged reasoning artifact. The `items` are provider-native
64
+ * and only meaningful to the client whose id matches `provider`; any other
65
+ * client must ignore them (they are never mapped into a foreign provider's
66
+ * request payload).
67
+ */
68
+ export interface ReasoningDetails {
69
+ /** The provider that produced these items, e.g. "openai" | "xai" | "anthropic" | "gemini". */
70
+ provider: string;
71
+ /** Provider-native reasoning items to round-trip verbatim on the next turn. */
72
+ items: any[];
26
73
  }
27
74
 
28
75
  export interface ToolProp {
@@ -30,6 +77,7 @@ export interface ToolProp {
30
77
  description?: string;
31
78
  properties?: { [key: string]: ToolProp };
32
79
  items?: ToolProp;
80
+ enum?: any[];
33
81
  }
34
82
 
35
83
  export interface Tool {
@@ -114,6 +162,20 @@ export interface CompletionOptions extends RetryOptions {
114
162
  * Defaults to false (no summary displayed).
115
163
  */
116
164
  reasoning_summary?: boolean;
165
+ /**
166
+ * OpenAI Responses API only: when true, OpenAI stores the response server-side
167
+ * so subsequent turns can reference it via `previous_response_id` instead of
168
+ * resending the full reasoning/message history. Defaults to false (stateless).
169
+ * When false, the client automatically requests `reasoning.encrypted_content`
170
+ * and threads the reasoning items back in the next request's input.
171
+ */
172
+ store?: boolean;
173
+ /**
174
+ * OpenAI Responses API only: the id of a previously-stored response to continue
175
+ * from. Only meaningful when `store` was true on the prior request. When set,
176
+ * the model keeps its prior chain-of-thought instead of re-deriving it.
177
+ */
178
+ previous_response_id?: string;
117
179
  /**
118
180
  * When true, hints to the client that this task is long-running and it should
119
181
  * use a long-TTL cache where available.
@@ -165,6 +227,12 @@ export interface CompletionResponse {
165
227
  model: string;
166
228
  usage: TokenUsage | undefined;
167
229
  usd_cost?: number;
230
+ /**
231
+ * OpenAI Responses API only: the id of this response. When `store` was true,
232
+ * this can be passed as `previous_response_id` on the next request to continue
233
+ * the reasoning chain without resending history.
234
+ */
235
+ response_id?: string;
168
236
  }
169
237
 
170
238
  /** A single chunk yielded by a streaming completion. */
@@ -116,6 +116,20 @@ export class GenericXAIClient implements GenericClient {
116
116
  * Used for grok-4.20 reasoning/non-reasoning and multi-agent models.
117
117
  * Translates Chat Completions message format to Responses API format.
118
118
  */
119
+ /**
120
+ * Resolve the xai-native reasoning items (Responses API `reasoning` items
121
+ * with encrypted_content) to re-inject for an assistant message, reading the
122
+ * provider-agnostic `_reasoning_details` slot only when it was produced by
123
+ * this provider.
124
+ */
125
+ private getReasoningItems(msg: any): any[] {
126
+ const details = msg?._reasoning_details;
127
+ if (details && details.provider === "xai" && Array.isArray(details.items)) {
128
+ return details.items;
129
+ }
130
+ return [];
131
+ }
132
+
119
133
  async createChatResponse(
120
134
  options: CompletionOptions
121
135
  ): Promise<CompletionResponse> {
@@ -133,22 +147,46 @@ export class GenericXAIClient implements GenericClient {
133
147
  .trim() || undefined;
134
148
 
135
149
  // Convert chat messages to Responses API input items
150
+ // Responses API hard limit: 10,485,760 chars per function_call_output.output
151
+ const MAX_TOOL_OUTPUT = 10_485_000;
152
+ const truncateToolOutput = (s: string) =>
153
+ s.length > MAX_TOOL_OUTPUT
154
+ ? s.slice(0, MAX_TOOL_OUTPUT) + "\n\n[TRUNCATED: output exceeded 10MB API limit]"
155
+ : s;
136
156
  const input: any[] = nonSystemMessages.map((msg) => {
137
157
  if (msg.role === "tool") {
138
158
  return {
139
159
  type: "function_call_output",
140
160
  call_id: msg.tool_call_id,
141
- output: typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content),
161
+ output: truncateToolOutput(typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content)),
142
162
  };
143
163
  }
144
164
  if (msg.role === "assistant" && msg.tool_calls?.length) {
145
- return msg.tool_calls.map((tc) => ({
165
+ // Re-inject any captured reasoning items (encrypted_content) BEFORE the
166
+ // function_call items so the model keeps its chain-of-thought instead of
167
+ // re-deriving it from scratch each turn.
168
+ const reasoningItems = this.getReasoningItems(msg);
169
+ return [
170
+ ...reasoningItems,
171
+ ...msg.tool_calls.map((tc) => ({
146
172
  type: "function_call",
147
173
  id: tc.id.startsWith("fc") ? tc.id : `fc_${tc.id}`,
148
174
  call_id: tc.id,
149
175
  name: tc.function.name,
150
176
  arguments: tc.function.arguments,
151
- }));
177
+ })),
178
+ ];
179
+ }
180
+ if (msg.role === "assistant" && this.getReasoningItems(msg).length) {
181
+ // assistant message (no tool calls) that carried reasoning items →
182
+ // re-inject reasoning items, then the visible message content.
183
+ return [
184
+ ...this.getReasoningItems(msg),
185
+ {
186
+ role: msg.role,
187
+ content: typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content),
188
+ },
189
+ ];
152
190
  }
153
191
  return {
154
192
  role: msg.role,
@@ -182,7 +220,18 @@ export class GenericXAIClient implements GenericClient {
182
220
  ...(options.max_tokens && { max_output_tokens: Math.max(options.max_tokens, 16_000) }),
183
221
  ...(reasoningEffort && { reasoning: { effort: reasoningEffort } }),
184
222
  ...(tools?.length && { tools, tool_choice: "auto" }),
185
- store: false,
223
+ // Reasoning persistence (same construct as OpenAI's Responses API):
224
+ // - store === true → xAI persists the response; next turn passes
225
+ // previous_response_id to continue the reasoning chain server-side.
226
+ // - store !== true → stateless; request encrypted reasoning content so
227
+ // we can thread the reasoning items back into the next request ourselves.
228
+ store: options.store === true,
229
+ ...(options.previous_response_id && {
230
+ previous_response_id: options.previous_response_id,
231
+ }),
232
+ ...(options.store !== true && reasoningEffort && {
233
+ include: ["reasoning.encrypted_content"],
234
+ }),
186
235
  };
187
236
 
188
237
  const response = await fetch("https://api.x.ai/v1/responses", {
@@ -218,6 +267,10 @@ export class GenericXAIClient implements GenericClient {
218
267
  // Collect text content and tool calls from output items
219
268
  let textContent: string | null = null;
220
269
  const toolCalls: any[] = [];
270
+ // Reasoning items (with encrypted_content) to thread back into the next
271
+ // turn, plus a human-readable summary of the model's thinking (if present).
272
+ const reasoningItems: any[] = [];
273
+ let reasoningSummary: string | null = null;
221
274
 
222
275
  for (const item of data.output ?? []) {
223
276
  if (item.type === "message") {
@@ -226,6 +279,17 @@ export class GenericXAIClient implements GenericClient {
226
279
  textContent = (textContent ?? "") + part.text;
227
280
  }
228
281
  }
282
+ } else if (item.type === "reasoning") {
283
+ // Keep the raw reasoning item so it can be re-injected next turn
284
+ // (carries encrypted_content when include was requested).
285
+ reasoningItems.push(item);
286
+ const summaryParts = item.summary ?? [];
287
+ for (const part of summaryParts) {
288
+ const text = typeof part === "string" ? part : part?.text;
289
+ if (text) {
290
+ reasoningSummary = (reasoningSummary ?? "") + text;
291
+ }
292
+ }
229
293
  } else if (item.type === "function_call") {
230
294
  toolCalls.push({
231
295
  id: item.call_id,
@@ -245,12 +309,17 @@ export class GenericXAIClient implements GenericClient {
245
309
  role: "assistant",
246
310
  content: textContent,
247
311
  ...(toolCalls.length > 0 && { tool_calls: toolCalls }),
312
+ ...(reasoningSummary && { reasoning_summary: reasoningSummary }),
313
+ ...(reasoningItems.length > 0 && {
314
+ _reasoning_details: { provider: "xai", items: reasoningItems },
315
+ }),
248
316
  },
249
317
  },
250
318
  ],
251
319
  model: options.model,
252
320
  usage,
253
321
  usd_cost: usdCost,
322
+ response_id: data.id,
254
323
  };
255
324
  }
256
325
 
@@ -55,6 +55,10 @@ export function addAgentCommand(program: Command, getChatService: () => any): vo
55
55
  )
56
56
  .option("--model <model>", "Specific model for the provider")
57
57
  .option("--agent-name <name>", "Which agent to use", "Patcher")
58
+ .option(
59
+ "--reasoning-effort <effort>",
60
+ "Reasoning effort: none, low, medium, or high. Use 'none' to disable thinking on models that support it (e.g. for faster computer-use)."
61
+ )
58
62
  .option(
59
63
  "--max-time-limit <minutes>",
60
64
  "Time limit for agent execution (minutes)",
@@ -116,17 +120,29 @@ export function addAgentCommand(program: Command, getChatService: () => any): vo
116
120
  options.taskId,
117
121
  options.messageId
118
122
  );
123
+ // Restore the exact model/provider/reasoning the original run used
124
+ // (falls back to agent defaults when not persisted / remote-only).
125
+ const savedSettings = await agentModule.loadTaskSettings(
126
+ options.taskId,
127
+ options.messageId
128
+ );
119
129
  const resumeInput =
120
130
  options.input || "Please continue from where you left off.";
121
131
 
122
132
  await agentModule.initialize(chatService);
123
133
  const { taskCompleted: resumed } =
124
134
  await agentModule.resumeFromMessages({
125
- agentName: options.agentName || "Patcher",
135
+ agentName:
136
+ options.agentName || savedSettings.agentName || "Patcher",
126
137
  input: resumeInput,
127
138
  threads,
128
139
  messageId: options.messageId,
129
140
  taskId: options.taskId,
141
+ // CLI-provided model/provider still win over persisted ones.
142
+ model: options.model || savedSettings.model,
143
+ provider: options.provider || savedSettings.provider,
144
+ reasoningEffort: savedSettings.reasoningEffort,
145
+ summarizeReasoning: savedSettings.summarizeReasoning,
130
146
  });
131
147
  await resumed;
132
148
  return;
@@ -208,6 +224,9 @@ export function addAgentCommand(program: Command, getChatService: () => any): vo
208
224
  input,
209
225
  maxTimeLimit: parseInt(options.maxTimeLimit, 10),
210
226
  maxSpendLimit: parseFloat(options.maxSpendLimit),
227
+ ...(options.reasoningEffort
228
+ ? { reasoningEffort: options.reasoningEffort }
229
+ : {}),
211
230
  ...(behaviorSystemPrompt ? { systemPrompt: behaviorSystemPrompt } : {}),
212
231
  ...(behaviorModel ? { model: behaviorModel } : {}),
213
232
  run: true,
package/src/index.ts CHANGED
@@ -51,6 +51,14 @@ export {
51
51
  } from "./generate";
52
52
 
53
53
  export * as clients from "./clients";
54
+ export type {
55
+ Tool,
56
+ ToolProp,
57
+ ToolCall,
58
+ Message,
59
+ CompletionOptions,
60
+ CompletionResponse,
61
+ } from "./clients/types";
54
62
  export * as agents from "./agents";
55
63
  export * as services from "./services";
56
64
  export * as embeddings from "./embeddings";
@@ -61,6 +69,7 @@ export * as ai from "./ai";
61
69
  // Export module system types for external modules
62
70
  export * from "./services/modules/types";
63
71
  export { ModulesService } from "./services/modules";
72
+ export * from "./services/modules/computerUse";
64
73
  // Export conversion types for external modules (e.g. knowhow-module-pdf)
65
74
  export * from "./services/conversion/types";
66
75
  export { ConversionService } from "./services/conversion/ConversionService";
@@ -257,11 +257,26 @@ export class Base64ImageProcessor {
257
257
  try {
258
258
  // Try to parse as JSON
259
259
  const parsed = JSON.parse(message.content);
260
-
261
- // Check if it's an image_url object
262
- if (parsed.type === "image_url" && parsed.image_url?.url) {
263
- // Convert the tool message content from JSON string to an array with the image
264
- message.content = [parsed];
260
+
261
+ // Normalize the parsed value into an array of candidate parts so we can
262
+ // handle BOTH shapes a tool might return:
263
+ // 1. a single image_url object: {type:"image_url", image_url:{url}}
264
+ // 2. an array of parts: [{type:"image_url", image_url:{url}}, {type:"text", ...}]
265
+ // (Computer-use screenshot tools and loadWebpage return the array form,
266
+ // which the single-object check above previously missed — leaving the
267
+ // model to see a raw base64 JSON string instead of an actual image.)
268
+ const candidates = Array.isArray(parsed) ? parsed : [parsed];
269
+ const isImagePart = (p: any) =>
270
+ p && p.type === "image_url" && p.image_url?.url;
271
+ const isTextPart = (p: any) =>
272
+ p && p.type === "text" && typeof p.text === "string";
273
+
274
+ // Only rewrite the content if it actually contains at least one image
275
+ // part; otherwise leave the original tool text untouched.
276
+ if (candidates.some(isImagePart)) {
277
+ message.content = candidates.filter(
278
+ (p) => isImagePart(p) || isTextPart(p)
279
+ );
265
280
  }
266
281
  } catch (e) {
267
282
  // Not JSON, check if it's a plain base64 string (only if still a string)
@@ -414,10 +429,34 @@ export class Base64ImageProcessor {
414
429
 
415
430
  // Read the file as base64
416
431
  const imageBuffer = fs.readFileSync(filePath);
432
+
433
+ // Detect actual MIME type from magic bytes (overrides extension-based detection)
434
+ // This handles cases where a file is saved with the wrong extension (e.g. JPEG saved as .png)
435
+ let detectedMimeType = mimeType;
436
+ if (imageBuffer.length >= 4) {
437
+ // JPEG: starts with FF D8 FF
438
+ if (imageBuffer[0] === 0xff && imageBuffer[1] === 0xd8 && imageBuffer[2] === 0xff) {
439
+ detectedMimeType = "image/jpeg";
440
+ }
441
+ // PNG: starts with 89 50 4E 47 0D 0A 1A 0A
442
+ else if (imageBuffer[0] === 0x89 && imageBuffer[1] === 0x50 && imageBuffer[2] === 0x4e && imageBuffer[3] === 0x47) {
443
+ detectedMimeType = "image/png";
444
+ }
445
+ // GIF: starts with 47 49 46 38 (GIF8)
446
+ else if (imageBuffer[0] === 0x47 && imageBuffer[1] === 0x49 && imageBuffer[2] === 0x46 && imageBuffer[3] === 0x38) {
447
+ detectedMimeType = "image/gif";
448
+ }
449
+ // WebP: starts with 52 49 46 46 (RIFF) ... 57 45 42 50 (WEBP) at offset 8
450
+ else if (imageBuffer.length >= 12 && imageBuffer[0] === 0x52 && imageBuffer[1] === 0x49 && imageBuffer[2] === 0x46 && imageBuffer[3] === 0x46 &&
451
+ imageBuffer[8] === 0x57 && imageBuffer[9] === 0x45 && imageBuffer[10] === 0x42 && imageBuffer[11] === 0x50) {
452
+ detectedMimeType = "image/webp";
453
+ }
454
+ }
455
+
417
456
  const base64Data = imageBuffer.toString("base64");
418
457
 
419
- // Create data URL
420
- const dataUrl = `data:${mimeType};base64,${base64Data}`;
458
+ // Create data URL using the actual detected MIME type
459
+ const dataUrl = `data:${detectedMimeType};base64,${base64Data}`;
421
460
 
422
461
  // Return in a format that indicates this is an image
423
462
  // The Base64ImageDetector will convert this to proper image content
@@ -0,0 +1,163 @@
1
+ import { Message, ToolCall } from "../clients/types";
2
+ import { MessageProcessorFunction } from "../services/MessageProcessor";
3
+
4
+ /**
5
+ * Gemma 4 Tool Call Processor
6
+ *
7
+ * Gemma 4 uses a non-standard tool-call format with its own string-quoting token:
8
+ *
9
+ * <|tool_call>call:toolName{key:<|"|>string value<|"|>,numKey:42,boolKey:true}<tool_call|>
10
+ *
11
+ * Strings are delimited by <|"|> instead of standard JSON quotes.
12
+ * Numbers and booleans are written bare (no quotes).
13
+ * Multiple tool calls may appear in a single assistant message.
14
+ *
15
+ * Reference: https://ai.google.dev/gemma/docs/capabilities/text/function-calling-gemma4
16
+ */
17
+ export class GemmaToolProcessor {
18
+ private static instance: GemmaToolProcessor;
19
+
20
+ static getInstance(): GemmaToolProcessor {
21
+ if (!GemmaToolProcessor.instance) {
22
+ GemmaToolProcessor.instance = new GemmaToolProcessor();
23
+ }
24
+ return GemmaToolProcessor.instance;
25
+ }
26
+
27
+ // ── Detection ─────────────────────────────────────────────────────────────
28
+
29
+ /** Returns true when the content looks like it contains Gemma tool-call tokens. */
30
+ private isGemmaFormat(content: string): boolean {
31
+ return content.includes("<|tool_call>") && content.includes("<tool_call|>");
32
+ }
33
+
34
+ // ── Parsing ───────────────────────────────────────────────────────────────
35
+
36
+ /**
37
+ * Cast a raw argument value string to a JS primitive.
38
+ * Mirrors the Python `cast()` helper from the official docs.
39
+ */
40
+ private castValue(raw: string): string | number | boolean {
41
+ const trimmed = raw.trim();
42
+
43
+ // Boolean
44
+ const lower = trimmed.toLowerCase();
45
+ if (lower === "true") return true;
46
+ if (lower === "false") return false;
47
+
48
+ // Integer
49
+ if (/^-?\d+$/.test(trimmed)) return parseInt(trimmed, 10);
50
+
51
+ // Float
52
+ if (/^-?\d+\.\d*$/.test(trimmed)) return parseFloat(trimmed);
53
+
54
+ // Strip surrounding single/double quotes that the model might emit
55
+ return trimmed.replace(/^['"]|['"]$/g, "");
56
+ }
57
+
58
+ /**
59
+ * Parse the argument body of a single Gemma tool call.
60
+ *
61
+ * The body looks like:
62
+ * key1:<|"|>string value<|"|>,key2:42,key3:true
63
+ *
64
+ * The regex (from the official docs) handles both quoted strings and bare values:
65
+ * (\w+):(?:<|"|>(.*?)<|"|>|([^,}]*))
66
+ */
67
+ private parseArgs(argsBody: string): Record<string, string | number | boolean> {
68
+ const result: Record<string, string | number | boolean> = {};
69
+
70
+ // Regex from the official Google Gemma 4 function-calling docs
71
+ const argPattern = /(\w+):(?:<\|"\|>(.*?)<\|"\|>|([^,}]*))/gs;
72
+ let m: RegExpExecArray | null;
73
+
74
+ while ((m = argPattern.exec(argsBody)) !== null) {
75
+ const key = m[1];
76
+ // v1: string inside <|"|>…<|"|> v2: bare value
77
+ const rawValue = m[2] !== undefined ? m[2] : m[3] ?? "";
78
+ result[key] = m[2] !== undefined ? rawValue : this.castValue(rawValue);
79
+ }
80
+
81
+ return result;
82
+ }
83
+
84
+ /**
85
+ * Extract all Gemma tool calls from an assistant message string.
86
+ * Returns an array of { toolName, arguments } objects.
87
+ */
88
+ private extractGemmaToolCalls(
89
+ content: string
90
+ ): { toolName: string; arguments: string }[] {
91
+ const results: { toolName: string; arguments: string }[] = [];
92
+
93
+ // Outer pattern: <|tool_call>call:name{args}<tool_call|>
94
+ // Using DOTALL-equivalent workaround: [\s\S] for multi-line args
95
+ const outerPattern = /<\|tool_call>call:(\w+)\{([\s\S]*?)\}<tool_call\|>/g;
96
+ let match: RegExpExecArray | null;
97
+
98
+ while ((match = outerPattern.exec(content)) !== null) {
99
+ const toolName = match[1];
100
+ const argsBody = match[2];
101
+
102
+ const parsed = this.parseArgs(argsBody);
103
+ results.push({
104
+ toolName,
105
+ arguments: JSON.stringify(parsed),
106
+ });
107
+ }
108
+
109
+ return results;
110
+ }
111
+
112
+ // ── Conversion ────────────────────────────────────────────────────────────
113
+
114
+ private generateToolCallId(): string {
115
+ return `gemma_call_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
116
+ }
117
+
118
+ private toToolCalls(
119
+ extracted: { toolName: string; arguments: string }[]
120
+ ): ToolCall[] {
121
+ return extracted.map(({ toolName, arguments: args }) => ({
122
+ id: this.generateToolCallId(),
123
+ type: "function" as const,
124
+ function: {
125
+ name: toolName,
126
+ arguments: args,
127
+ },
128
+ }));
129
+ }
130
+
131
+ // ── Message processing ────────────────────────────────────────────────────
132
+
133
+ private processMessage(message: Message): void {
134
+ if (message.role !== "assistant") return;
135
+ if (typeof message.content !== "string") return;
136
+ if (message?.tool_calls?.length) return;
137
+
138
+ if (!this.isGemmaFormat(message.content)) return;
139
+
140
+ const extracted = this.extractGemmaToolCalls(message.content);
141
+ if (extracted.length === 0) return;
142
+
143
+ const toolCalls = this.toToolCalls(extracted);
144
+
145
+ if (message.tool_calls) {
146
+ message.tool_calls.push(...toolCalls);
147
+ } else {
148
+ message.tool_calls = toolCalls;
149
+ }
150
+ }
151
+
152
+ // ── Public API ────────────────────────────────────────────────────────────
153
+
154
+ createProcessor(): MessageProcessorFunction {
155
+ return (_originalMessages: Message[], modifiedMessages: Message[]): void => {
156
+ for (const message of modifiedMessages) {
157
+ this.processMessage(message);
158
+ }
159
+ };
160
+ }
161
+ }
162
+
163
+ export const globalGemmaToolProcessor = new GemmaToolProcessor();
@@ -10,4 +10,5 @@ export {
10
10
  export { ToolResponseCache } from "./ToolResponseCache";
11
11
  export { XmlToolCallProcessor } from "./XmlToolCallProcessor";
12
12
  export { HarmonyToolProcessor } from "./HarmonyToolProcessor";
13
+ export { GemmaToolProcessor } from "./GemmaToolProcessor";
13
14
  export { MinimalToolsMessageProcessor } from "./MinimalToolsMessageProcessor";
@@ -133,6 +133,11 @@ export class AgentSyncFs {
133
133
  * that call) to usage.json. This is intended for debugging cache-hit
134
134
  * issues, where comparing the message chains and usage values between
135
135
  * consecutive calls can reveal when a cache prefix was invalidated.
136
+ *
137
+ * To prevent exponential file growth, if the new entry's message thread
138
+ * fully contains the previous entry's thread as a prefix, the previous
139
+ * thread is replaced with { PREV_CACHE_HIT: true } so only the new
140
+ * messages (the delta) are stored alongside the new usage data.
136
141
  */
137
142
  private async appendUsageEntry(entry: any): Promise<void> {
138
143
  if (!this.taskPath) return;
@@ -148,6 +153,27 @@ export class AgentSyncFs {
148
153
  // File doesn't exist or is invalid, start fresh
149
154
  }
150
155
 
156
+ // If the new entry's messages are a superset of the last entry's
157
+ // messages (i.e. the last thread is a prefix of the new thread),
158
+ // collapse the last entry's messages to { PREV_CACHE_HIT: true }
159
+ // and store only the new delta — preventing exponential growth.
160
+ if (entries.length > 0) {
161
+ const lastEntry = entries[entries.length - 1];
162
+ const prevMessages: any[] = lastEntry?.messages;
163
+ const newMessages: any[] = entry?.messages;
164
+ if (
165
+ Array.isArray(prevMessages) &&
166
+ Array.isArray(newMessages) &&
167
+ prevMessages.length > 0 &&
168
+ newMessages.length >= prevMessages.length &&
169
+ JSON.stringify(newMessages.slice(0, prevMessages.length)) ===
170
+ JSON.stringify(prevMessages)
171
+ ) {
172
+ // Replace the previous full thread with a compact sentinel
173
+ lastEntry.messages = [{ PREV_CACHE_HIT: true }];
174
+ }
175
+ }
176
+
151
177
  entries.push(entry);
152
178
  await fs.writeFile(usagePath, JSON.stringify(entries, null, 2), "utf8");
153
179
  } catch (error) {
@@ -180,6 +206,17 @@ export class AgentSyncFs {
180
206
  metadata.totalCostUsd = agent.getTotalCostUsd();
181
207
  metadata.tokenUsage = agent.getTokenUsage();
182
208
  metadata.agentName = agent.name;
209
+ // Record the model/provider actually in use at this sync so task
210
+ // history reflects the real model (including mid-run fallbacks), not
211
+ // just the named agent.
212
+ metadata.model = agent.getModel();
213
+ metadata.provider = agent.getProvider();
214
+ // Persist reasoning settings so a resume can restore the exact
215
+ // reasoning effort / summarize behavior the run was using. (The
216
+ // encrypted reasoning items + summaries themselves ride along on the
217
+ // thread messages via `_reasoning_details` / `reasoning_summary`.)
218
+ metadata.reasoningEffort = agent.getReasoningEffort?.();
219
+ metadata.summarizeReasoning = agent.getSummarizeReasoning?.();
183
220
  metadata.inProgress = inProgress;
184
221
  metadata.lastUpdate = new Date().toISOString();
185
222
 
@@ -65,6 +65,10 @@ export class SessionManager {
65
65
  threads,
66
66
  currentThread: 0,
67
67
  lastUpdated: Date.now(),
68
+ model: taskInfo.model,
69
+ provider: taskInfo.provider,
70
+ reasoningEffort: taskInfo.reasoningEffort,
71
+ summarizeReasoning: taskInfo.summarizeReasoning,
68
72
  };
69
73
 
70
74
  fs.writeFileSync(sessionPath, JSON.stringify(session, null, 2));
@@ -101,6 +105,16 @@ export class SessionManager {
101
105
  session.knowhowMessageId = taskInfo.knowhowMessageId;
102
106
  session.knowhowTaskId = taskInfo.knowhowTaskId;
103
107
  session.chatSessionId = taskInfo.chatSessionId;
108
+
109
+ // Keep the persisted run configuration in sync (captures mid-run
110
+ // model/provider fallbacks) so a resume restores the exact setup.
111
+ if (taskInfo.model !== undefined) session.model = taskInfo.model;
112
+ if (taskInfo.provider !== undefined)
113
+ session.provider = taskInfo.provider;
114
+ if (taskInfo.reasoningEffort !== undefined)
115
+ session.reasoningEffort = taskInfo.reasoningEffort;
116
+ if (taskInfo.summarizeReasoning !== undefined)
117
+ session.summarizeReasoning = taskInfo.summarizeReasoning;
104
118
  }
105
119
 
106
120
  fs.writeFileSync(sessionPath, JSON.stringify(session, null, 2));
@@ -34,6 +34,8 @@ export interface ToolCallContext {
34
34
  caller?: any;
35
35
  /** The task id of the calling agent, if known. */
36
36
  taskId?: string;
37
+ /** The task id of the parent agent that spawned the calling agent, if any. */
38
+ parentTaskId?: string;
37
39
  /** Arbitrary additional per-call context. */
38
40
  [key: string]: any;
39
41
  }
@@ -46,6 +48,7 @@ export interface ToolContext {
46
48
  Plugins?: PluginService;
47
49
  Mcp?: McpService;
48
50
  Behaviors?: BehaviorsService;
51
+ ComputerUse?: import("./modules/computerUse").ComputerUseService;
49
52
  metadata?: { [key: string]: any };
50
53
  }
51
54