@trops/dash-core 0.1.691 → 0.1.692

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -75371,6 +75371,20 @@ function mcpToolToAnthropic(tool) {
75371
75371
  };
75372
75372
  }
75373
75373
 
75374
+ // A text-only result stays a plain string; with images it's a block list
75375
+ // (no empty text block — the API rejects those).
75376
+ function toolResultContent(r) {
75377
+ if (!Array.isArray(r.images) || !r.images.length) return r.text;
75378
+ const blocks = r.text ? [{ type: "text", text: r.text }] : [];
75379
+ for (const img of r.images) {
75380
+ blocks.push({
75381
+ type: "image",
75382
+ source: { type: "base64", media_type: img.mimeType, data: img.data },
75383
+ });
75384
+ }
75385
+ return blocks;
75386
+ }
75387
+
75374
75388
  const anthropicAdapter$1 = {
75375
75389
  id: "anthropic",
75376
75390
 
@@ -75430,7 +75444,8 @@ const anthropicAdapter$1 = {
75430
75444
 
75431
75445
  /**
75432
75446
  * Anthropic takes tool results as a single user message of tool_result blocks.
75433
- * @param {Array<{id: string, name: string, text: string, isError: boolean}>} results
75447
+ * Images ride inside the tool_result as base64 image blocks (CAP-001).
75448
+ * @param {Array<{id: string, name: string, text: string, images?: Array<{data: string, mimeType: string}>, isError: boolean}>} results
75434
75449
  */
75435
75450
  formatToolResults(results) {
75436
75451
  return [
@@ -75439,7 +75454,7 @@ const anthropicAdapter$1 = {
75439
75454
  content: results.map((r) => ({
75440
75455
  type: "tool_result",
75441
75456
  tool_use_id: r.id,
75442
- content: r.text,
75457
+ content: toolResultContent(r),
75443
75458
  is_error: !!r.isError,
75444
75459
  })),
75445
75460
  },
@@ -75561,20 +75576,202 @@ const openAICompatibleAdapter$1 = {
75561
75576
 
75562
75577
  /**
75563
75578
  * OpenAI takes each tool result as its own `role: "tool"` message keyed by
75564
- * tool_call_id.
75565
- * @param {Array<{id: string, name: string, text: string, isError: boolean}>} results
75579
+ * tool_call_id. Tool messages can only carry text, so images (CAP-001)
75580
+ * follow in one user message, each group labelled with the call it came from.
75581
+ * @param {Array<{id: string, name: string, text: string, images?: Array<{data: string, mimeType: string}>, isError: boolean}>} results
75566
75582
  */
75567
75583
  formatToolResults(results) {
75568
- return results.map((r) => ({
75569
- role: "tool",
75570
- tool_call_id: r.id,
75571
- content: r.isError ? `Error: ${r.text}` : r.text,
75572
- }));
75584
+ const messages = results.map((r) => {
75585
+ const hasImages = Array.isArray(r.images) && r.images.length > 0;
75586
+ const text =
75587
+ r.text ||
75588
+ (hasImages
75589
+ ? `Returned ${r.images.length} image(s), attached below.`
75590
+ : "");
75591
+ return {
75592
+ role: "tool",
75593
+ tool_call_id: r.id,
75594
+ content: r.isError ? `Error: ${text}` : text,
75595
+ };
75596
+ });
75597
+ const parts = [];
75598
+ for (const r of results) {
75599
+ if (!Array.isArray(r.images) || !r.images.length) continue;
75600
+ parts.push({
75601
+ type: "text",
75602
+ text: `Images returned by ${r.name} (call ${r.id}):`,
75603
+ });
75604
+ for (const img of r.images) {
75605
+ parts.push({
75606
+ type: "image_url",
75607
+ image_url: { url: `data:${img.mimeType};base64,${img.data}` },
75608
+ });
75609
+ }
75610
+ }
75611
+ if (parts.length) messages.push({ role: "user", content: parts });
75612
+ return messages;
75573
75613
  },
75574
75614
  };
75575
75615
 
75576
75616
  var openAICompatibleAdapter_1 = openAICompatibleAdapter$1;
75577
75617
 
75618
+ /**
75619
+ * toolImages.js
75620
+ *
75621
+ * Image content in bot tool results (bot-capabilities CAP-001). A tool result
75622
+ * is { text, images?, isError } where images are { data (base64), mimeType }.
75623
+ * These helpers decide which images go to the model, describe them for the
75624
+ * Activity feed, and keep image data out of stored sessions. Pure — no
75625
+ * Electron.
75626
+ */
75627
+
75628
+ const SUPPORTED_TYPES = new Set([
75629
+ "image/png",
75630
+ "image/jpeg",
75631
+ "image/gif",
75632
+ "image/webp",
75633
+ ]);
75634
+ const MAX_IMAGES = 5;
75635
+ // Anthropic's per-image limit; the strictest of the bot providers.
75636
+ const MAX_IMAGE_BYTES = 5 * 1024 * 1024;
75637
+
75638
+ function byteLength(base64) {
75639
+ const s = String(base64 || "");
75640
+ const padding = s.endsWith("==") ? 2 : s.endsWith("=") ? 1 : 0;
75641
+ return Math.max(0, Math.floor((s.length * 3) / 4) - padding);
75642
+ }
75643
+
75644
+ function formatSize(bytes) {
75645
+ if (bytes >= 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
75646
+ return `${Math.max(1, Math.round(bytes / 1024))} KB`;
75647
+ }
75648
+
75649
+ /**
75650
+ * Pick the images from MCP content blocks that can go to the model.
75651
+ * @param {Array} blocks MCP content blocks
75652
+ * @returns {{ images: Array<{data: string, mimeType: string}>, notes: string[] }}
75653
+ */
75654
+ function collectImages$1(blocks) {
75655
+ const images = [];
75656
+ const notes = [];
75657
+ let leftOut = 0;
75658
+ for (const b of Array.isArray(blocks) ? blocks : []) {
75659
+ if (!b || b.type !== "image") continue;
75660
+ if (!b.data) {
75661
+ notes.push("An image was left out: it had no data.");
75662
+ continue;
75663
+ }
75664
+ if (!SUPPORTED_TYPES.has(b.mimeType)) {
75665
+ notes.push(
75666
+ `An image was left out: ${b.mimeType || "unknown type"} isn't supported (PNG, JPEG, GIF, or WebP only).`,
75667
+ );
75668
+ continue;
75669
+ }
75670
+ const bytes = byteLength(b.data);
75671
+ if (bytes > MAX_IMAGE_BYTES) {
75672
+ notes.push(
75673
+ `An image was left out: too large (${formatSize(bytes)}; limit ${formatSize(MAX_IMAGE_BYTES)}).`,
75674
+ );
75675
+ continue;
75676
+ }
75677
+ if (images.length >= MAX_IMAGES) {
75678
+ leftOut++;
75679
+ continue;
75680
+ }
75681
+ images.push({ data: b.data, mimeType: b.mimeType });
75682
+ }
75683
+ if (leftOut) {
75684
+ notes.push(
75685
+ `${leftOut} more image${leftOut === 1 ? "" : "s"} left out (limit ${MAX_IMAGES} per result).`,
75686
+ );
75687
+ }
75688
+ return { images, notes };
75689
+ }
75690
+
75691
+ /** "[image: image/png, 240 KB]" */
75692
+ function imagePlaceholder$1(image) {
75693
+ return `[image: ${image.mimeType}, ${formatSize(byteLength(image.data))}]`;
75694
+ }
75695
+
75696
+ /** Text for the Activity feed: the result text plus a line per image/note. */
75697
+ function describeResult$1({ text, images, notes }) {
75698
+ const lines = [];
75699
+ if (text) lines.push(text);
75700
+ for (const img of images || []) lines.push(imagePlaceholder$1(img));
75701
+ for (const n of notes || []) lines.push(n);
75702
+ return lines.join("\n");
75703
+ }
75704
+
75705
+ // One content part → its storage form (images become placeholder text).
75706
+ function stripPart(part) {
75707
+ if (!part || typeof part !== "object") return part;
75708
+ // Anthropic: { type: "image", source: { type: "base64", media_type, data } }
75709
+ if (part.type === "image" && part.source && part.source.data) {
75710
+ return {
75711
+ type: "text",
75712
+ text: imagePlaceholder$1({
75713
+ data: part.source.data,
75714
+ mimeType: part.source.media_type,
75715
+ }),
75716
+ };
75717
+ }
75718
+ // OpenAI: { type: "image_url", image_url: { url: "data:<mime>;base64,<data>" } }
75719
+ if (part.type === "image_url" && part.image_url) {
75720
+ const m = /^data:([^;]+);base64,(.*)$/.exec(part.image_url.url || "");
75721
+ if (m) {
75722
+ return {
75723
+ type: "text",
75724
+ text: imagePlaceholder$1({ mimeType: m[1], data: m[2] }),
75725
+ };
75726
+ }
75727
+ }
75728
+ if (Array.isArray(part.content)) {
75729
+ return { ...part, content: part.content.map(stripPart) };
75730
+ }
75731
+ return part;
75732
+ }
75733
+
75734
+ /**
75735
+ * A copy of a message history with image data replaced by placeholders, for
75736
+ * saving as the bot's session. The live history is not modified.
75737
+ */
75738
+ function stripImagesForStorage$1(messages) {
75739
+ return (Array.isArray(messages) ? messages : []).map((msg) =>
75740
+ msg && Array.isArray(msg.content)
75741
+ ? { ...msg, content: msg.content.map(stripPart) }
75742
+ : msg,
75743
+ );
75744
+ }
75745
+
75746
+ /**
75747
+ * A plain message when the model API refused the images, else null.
75748
+ * @param {Error} err
75749
+ * @param {string} model
75750
+ */
75751
+ function imageRejectionMessage$1(err, model) {
75752
+ const msg = String((err && err.message) || "");
75753
+ if (!/image/i.test(msg)) return null;
75754
+ if (
75755
+ !/not supported|unsupported|does not support|doesn't support|invalid/i.test(
75756
+ msg,
75757
+ )
75758
+ ) {
75759
+ return null;
75760
+ }
75761
+ return `This model can't read images (${model || "unknown model"}). Choose a model that supports images in the bot's Settings.`;
75762
+ }
75763
+
75764
+ var toolImages = {
75765
+ SUPPORTED_TYPES,
75766
+ MAX_IMAGES,
75767
+ MAX_IMAGE_BYTES,
75768
+ collectImages: collectImages$1,
75769
+ imagePlaceholder: imagePlaceholder$1,
75770
+ describeResult: describeResult$1,
75771
+ stripImagesForStorage: stripImagesForStorage$1,
75772
+ imageRejectionMessage: imageRejectionMessage$1,
75773
+ };
75774
+
75578
75775
  /**
75579
75776
  * toolLoopEngine.js
75580
75777
  *
@@ -75597,6 +75794,11 @@ var openAICompatibleAdapter_1 = openAICompatibleAdapter$1;
75597
75794
  const { createEventStream: createEventStream$1 } = eventStream;
75598
75795
  const anthropicAdapter = anthropicAdapter_1;
75599
75796
  const openAICompatibleAdapter = openAICompatibleAdapter_1;
75797
+ const {
75798
+ describeResult,
75799
+ stripImagesForStorage,
75800
+ imageRejectionMessage,
75801
+ } = toolImages;
75600
75802
 
75601
75803
  const ADAPTERS = {
75602
75804
  anthropic: anthropicAdapter,
@@ -75620,7 +75822,7 @@ function resolveAdapter(ctx) {
75620
75822
  return adapter;
75621
75823
  }
75622
75824
 
75623
- async function runLoop(ctx, stream) {
75825
+ async function runLoop(ctx, stream, state) {
75624
75826
  const adapter = resolveAdapter(ctx);
75625
75827
  const maxTurns = ctx.maxTurns || DEFAULT_MAX_TURNS;
75626
75828
  const providerTools = adapter.toProviderTools(ctx.tools || []);
@@ -75633,11 +75835,18 @@ async function runLoop(ctx, stream) {
75633
75835
  if (ctx.prompt) messages.push({ role: "user", content: ctx.prompt });
75634
75836
 
75635
75837
  const aborted = () => ctx.signal && ctx.signal.aborted;
75838
+ // The stored session keeps placeholders, not image data (CAP-001); the live
75839
+ // history keeps the images so later turns of this run can still see them.
75840
+ const pushSession = () =>
75841
+ stream.push({
75842
+ type: "session",
75843
+ session: { messages: stripImagesForStorage(messages) },
75844
+ });
75636
75845
 
75637
75846
  let turn = 0;
75638
75847
  while (turn <= maxTurns) {
75639
75848
  if (aborted()) {
75640
- stream.push({ type: "session", session: { messages } });
75849
+ pushSession();
75641
75850
  return;
75642
75851
  }
75643
75852
 
@@ -75657,7 +75866,7 @@ async function runLoop(ctx, stream) {
75657
75866
 
75658
75867
  const toolCalls = result.toolCalls || [];
75659
75868
  if (!toolCalls.length || result.stopReason === "end_turn") {
75660
- stream.push({ type: "session", session: { messages } });
75869
+ pushSession();
75661
75870
  stream.push({
75662
75871
  type: "done",
75663
75872
  stopReason: result.stopReason || "end_turn",
@@ -75671,7 +75880,7 @@ async function runLoop(ctx, stream) {
75671
75880
  const results = [];
75672
75881
  for (const call of toolCalls) {
75673
75882
  if (aborted()) {
75674
- stream.push({ type: "session", session: { messages } });
75883
+ pushSession();
75675
75884
  return;
75676
75885
  }
75677
75886
 
@@ -75683,6 +75892,7 @@ async function runLoop(ctx, stream) {
75683
75892
  });
75684
75893
 
75685
75894
  let text = "";
75895
+ let images = null;
75686
75896
  let isError = false;
75687
75897
 
75688
75898
  // Every tool call routes through the gate before executing — the engine
@@ -75696,6 +75906,10 @@ async function runLoop(ctx, stream) {
75696
75906
  const r = await ctx.executeTool(call.name, call.input);
75697
75907
  text = (r && r.text) || "";
75698
75908
  isError = !!(r && r.isError);
75909
+ if (r && Array.isArray(r.images) && r.images.length) {
75910
+ images = r.images;
75911
+ state.sentImages = true;
75912
+ }
75699
75913
  } catch (err) {
75700
75914
  text = `Error: ${err.message}`;
75701
75915
  isError = true;
@@ -75706,10 +75920,15 @@ async function runLoop(ctx, stream) {
75706
75920
  type: "tool_result",
75707
75921
  id: call.id,
75708
75922
  name: call.name,
75709
- output: text,
75923
+ // Activity shows a placeholder per image, never the image data.
75924
+ output: images ? describeResult({ text, images }) : text,
75710
75925
  isError,
75711
75926
  });
75712
- results.push({ id: call.id, name: call.name, text, isError });
75927
+ results.push(
75928
+ images
75929
+ ? { id: call.id, name: call.name, text, images, isError }
75930
+ : { id: call.id, name: call.name, text, isError },
75931
+ );
75713
75932
  }
75714
75933
 
75715
75934
  for (const m of adapter.formatToolResults(results)) messages.push(m);
@@ -75729,14 +75948,19 @@ const toolLoopEngine$1 = {
75729
75948
 
75730
75949
  run(ctx) {
75731
75950
  const stream = createEventStream$1();
75951
+ const state = { sentImages: false };
75732
75952
  (async () => {
75733
75953
  try {
75734
- await runLoop(ctx, stream);
75954
+ await runLoop(ctx, stream, state);
75735
75955
  } catch (err) {
75956
+ // A model that can't read images gets a plain explanation.
75957
+ const imageMessage = state.sentImages
75958
+ ? imageRejectionMessage(err, ctx.model)
75959
+ : null;
75736
75960
  stream.push({
75737
75961
  type: "error",
75738
- message: err.message || "Engine error",
75739
- code: err.code || "ENGINE_ERROR",
75962
+ message: imageMessage || err.message || "Engine error",
75963
+ code: imageMessage ? "MODEL_NO_IMAGES" : err.code || "ENGINE_ERROR",
75740
75964
  });
75741
75965
  } finally {
75742
75966
  stream.end();
@@ -75839,10 +76063,20 @@ function buildBotMcpServer$1(ctx, sdk, z) {
75839
76063
  jsonSchemaToZodShape(t.inputSchema, z),
75840
76064
  async (args) => {
75841
76065
  const r = await ctx.executeTool(t.name, args);
75842
- return {
75843
- content: [{ type: "text", text: (r && r.text) || "" }],
75844
- isError: !!(r && r.isError),
75845
- };
76066
+ const text = (r && r.text) || "";
76067
+ const images = r && Array.isArray(r.images) ? r.images : [];
76068
+ // Images go back as MCP image blocks so the agent sees them (CAP-001).
76069
+ const content = images.length
76070
+ ? [
76071
+ ...(text ? [{ type: "text", text }] : []),
76072
+ ...images.map((img) => ({
76073
+ type: "image",
76074
+ data: img.data,
76075
+ mimeType: img.mimeType,
76076
+ })),
76077
+ ]
76078
+ : [{ type: "text", text }];
76079
+ return { content, isError: !!(r && r.isError) };
75846
76080
  },
75847
76081
  ),
75848
76082
  );
@@ -75882,6 +76116,7 @@ var agentToolBridge = {
75882
76116
 
75883
76117
  const { createEventStream } = eventStream;
75884
76118
  const { BOT_MCP_SERVER, buildBotMcpServer } = agentToolBridge;
76119
+ const { imagePlaceholder } = toolImages;
75885
76120
 
75886
76121
  // The SDK names bridged tools "mcp__<server>__<tool>".
75887
76122
  const BRIDGED_PREFIX = `mcp__${BOT_MCP_SERVER}__`;
@@ -75947,9 +76182,21 @@ function* _fromUser(message) {
75947
76182
  let text = "";
75948
76183
  if (typeof b.content === "string") text = b.content;
75949
76184
  else if (Array.isArray(b.content)) {
76185
+ // Images show as placeholders in Activity, never as data (CAP-001).
75950
76186
  text = b.content
75951
- .filter((c) => c && c.type === "text")
75952
- .map((c) => c.text)
76187
+ .map((c) =>
76188
+ c && c.type === "text"
76189
+ ? c.text
76190
+ : c && c.type === "image"
76191
+ ? // API shape { source: { data, media_type } } or MCP { data, mimeType }
76192
+ imagePlaceholder({
76193
+ data: (c.source && c.source.data) || c.data || "",
76194
+ mimeType:
76195
+ (c.source && c.source.media_type) || c.mimeType || "image",
76196
+ })
76197
+ : null,
76198
+ )
76199
+ .filter((t) => t)
75953
76200
  .join("\n");
75954
76201
  }
75955
76202
  yield {
@@ -77495,7 +77742,8 @@ var EventDispatcher_1 = { EventDispatcher: EventDispatcher$1, DEFAULT_COOLDOWN_M
77495
77742
  * mcpResult.js
77496
77743
  *
77497
77744
  * Normalizes an mcpController.callTool(...) return value into the
77498
- * { text, isError } shape the tool-loop engine's executeTool contract expects.
77745
+ * { text, images?, isError } shape the engines' executeTool contract expects
77746
+ * (`images` only when the tool returned some — see toolImages.js).
77499
77747
  * Mirrors the extraction in electron/controller/llmController.js so bot tool
77500
77748
  * results read the same as the AI Assistant's. Pure — no Electron.
77501
77749
  *
@@ -77505,9 +77753,11 @@ var EventDispatcher_1 = { EventDispatcher: EventDispatcher$1, DEFAULT_COOLDOWN_M
77505
77753
  * { success: true, result: <string|other> } → string or JSON fallback
77506
77754
  */
77507
77755
 
77756
+ const { collectImages } = toolImages;
77757
+
77508
77758
  /**
77509
77759
  * @param {object} mcpResult
77510
- * @returns {{ text: string, isError: boolean }}
77760
+ * @returns {{ text: string, images?: Array<{data: string, mimeType: string}>, isError: boolean }}
77511
77761
  */
77512
77762
  function normalizeMcpResult$1(mcpResult) {
77513
77763
  if (!mcpResult) return { text: "No result returned.", isError: true };
@@ -77520,13 +77770,17 @@ function normalizeMcpResult$1(mcpResult) {
77520
77770
  return { text: "No result returned.", isError: false };
77521
77771
  }
77522
77772
 
77523
- // MCP standard: { content: [{ type: "text", text }] }
77773
+ // MCP standard: { content: [{ type: "text", text } | { type: "image", data, mimeType }] }
77524
77774
  if (result.content && Array.isArray(result.content)) {
77525
- const text = result.content
77775
+ const textParts = result.content
77526
77776
  .filter((c) => c && c.type === "text")
77527
- .map((c) => c.text)
77528
- .join("\n");
77529
- return { text, isError: !!result.isError };
77777
+ .map((c) => c.text);
77778
+ // Images go to the model (CAP-001); ones it can't take leave a note.
77779
+ const { images, notes } = collectImages(result.content);
77780
+ const text = [...textParts, ...notes].join("\n");
77781
+ return images.length
77782
+ ? { text, images, isError: !!result.isError }
77783
+ : { text, isError: !!result.isError };
77530
77784
  }
77531
77785
 
77532
77786
  if (typeof result === "string") return { text: result, isError: false };