clanka 0.5.2 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/dist/Acp.d.ts.map +1 -1
  2. package/dist/Acp.js +195 -16
  3. package/dist/Acp.js.map +1 -1
  4. package/dist/AcpImage.test.d.ts +2 -0
  5. package/dist/AcpImage.test.d.ts.map +1 -0
  6. package/dist/AcpImage.test.js +470 -0
  7. package/dist/AcpImage.test.js.map +1 -0
  8. package/dist/Agent.d.ts +9 -2
  9. package/dist/Agent.d.ts.map +1 -1
  10. package/dist/Agent.js +43 -6
  11. package/dist/Agent.js.map +1 -1
  12. package/dist/AgentExecutor.d.ts +27 -5
  13. package/dist/AgentExecutor.d.ts.map +1 -1
  14. package/dist/AgentExecutor.js +29 -4
  15. package/dist/AgentExecutor.js.map +1 -1
  16. package/dist/AgentExecutorImage.test.d.ts +2 -0
  17. package/dist/AgentExecutorImage.test.d.ts.map +1 -0
  18. package/dist/AgentExecutorImage.test.js +63 -0
  19. package/dist/AgentExecutorImage.test.js.map +1 -0
  20. package/dist/AgentImage.test.d.ts +2 -0
  21. package/dist/AgentImage.test.d.ts.map +1 -0
  22. package/dist/AgentImage.test.js +176 -0
  23. package/dist/AgentImage.test.js.map +1 -0
  24. package/dist/AgentTools.d.ts +25 -5
  25. package/dist/AgentTools.d.ts.map +1 -1
  26. package/dist/AgentTools.js +40 -4
  27. package/dist/AgentTools.js.map +1 -1
  28. package/dist/AgentToolsImage.test.d.ts +2 -0
  29. package/dist/AgentToolsImage.test.d.ts.map +1 -0
  30. package/dist/AgentToolsImage.test.js +158 -0
  31. package/dist/AgentToolsImage.test.js.map +1 -0
  32. package/dist/Codex.js +6 -1
  33. package/dist/Codex.js.map +1 -1
  34. package/dist/Compaction.d.ts.map +1 -1
  35. package/dist/Compaction.js +16 -1
  36. package/dist/Compaction.js.map +1 -1
  37. package/dist/CompactionImage.test.d.ts +2 -0
  38. package/dist/CompactionImage.test.d.ts.map +1 -0
  39. package/dist/CompactionImage.test.js +105 -0
  40. package/dist/CompactionImage.test.js.map +1 -0
  41. package/dist/Copilot.d.ts.map +1 -1
  42. package/dist/Copilot.js +5 -1
  43. package/dist/Copilot.js.map +1 -1
  44. package/dist/Image.d.ts +192 -0
  45. package/dist/Image.d.ts.map +1 -0
  46. package/dist/Image.js +586 -0
  47. package/dist/Image.js.map +1 -0
  48. package/dist/Image.test.d.ts +2 -0
  49. package/dist/Image.test.d.ts.map +1 -0
  50. package/dist/Image.test.js +142 -0
  51. package/dist/Image.test.js.map +1 -0
  52. package/dist/fixtures/TestImages.d.ts +73 -0
  53. package/dist/fixtures/TestImages.d.ts.map +1 -0
  54. package/dist/fixtures/TestImages.js +263 -0
  55. package/dist/fixtures/TestImages.js.map +1 -0
  56. package/dist/index.d.ts +4 -0
  57. package/dist/index.d.ts.map +1 -1
  58. package/dist/index.js +4 -0
  59. package/dist/index.js.map +1 -1
  60. package/package.json +2 -1
  61. package/src/Acp.ts +274 -21
  62. package/src/AcpImage.test.ts +676 -0
  63. package/src/Agent.ts +69 -7
  64. package/src/AgentExecutor.ts +51 -3
  65. package/src/AgentExecutorImage.test.ts +103 -0
  66. package/src/AgentImage.test.ts +295 -0
  67. package/src/AgentTools.ts +78 -7
  68. package/src/AgentToolsImage.test.ts +302 -0
  69. package/src/Codex.ts +6 -1
  70. package/src/Compaction.ts +16 -1
  71. package/src/CompactionImage.test.ts +142 -0
  72. package/src/Copilot.ts +5 -1
  73. package/src/Image.test.ts +220 -0
  74. package/src/Image.ts +704 -0
  75. package/src/fixtures/TestImages.ts +287 -0
  76. package/src/index.ts +5 -0
package/src/Agent.ts CHANGED
@@ -12,9 +12,12 @@ import type * as HttpClient from "effect/unstable/http/HttpClient"
12
12
  import type {
13
13
  CurrentDirectory,
14
14
  DirectoryChanger,
15
+ ImageAttacher,
16
+ ImageAttachment,
15
17
  SubagentExecutor,
16
18
  TaskCompleter,
17
19
  } from "./AgentTools.ts"
20
+ import * as Image from "./Image.ts"
18
21
  import type * as FileSystem from "effect/FileSystem"
19
22
  import * as Prompt from "effect/unstable/ai/Prompt"
20
23
  import * as Effect from "effect/Effect"
@@ -197,6 +200,13 @@ ${content}
197
200
  // `inputTokens.total` from the most recent finish part; undefined until
198
201
  // the first response and after every compaction.
199
202
  let lastContextTokens: number | undefined = undefined
203
+ // Images stay in the Prompt; this only decides whether they are sent.
204
+ // Starts false for models known not to take images, and flips to false
205
+ // after a provider rejects image input once.
206
+ let sendImages = modelConfig.supportsImages !== false
207
+ // Images attached by `readFile` during the current script, spliced onto
208
+ // the Prompt as a user message once the tool result has been recorded.
209
+ const pendingImages: Array<ImageAttachment> = []
200
210
 
201
211
  const output = yield* Queue.make<Output, AgentFinished | AiError.AiError>()
202
212
  let inputTokens = 0
@@ -386,6 +396,10 @@ ${content}
386
396
  Effect.sync(() => {
387
397
  finalSummary = Option.some(summary)
388
398
  }),
399
+ onImage: (image) =>
400
+ Effect.sync(() => {
401
+ pendingImages.push(image)
402
+ }),
389
403
  }),
390
404
  Stream.mkString,
391
405
  )
@@ -454,12 +468,21 @@ ${content}
454
468
 
455
469
  // oxlint-disable-next-line typescript/no-explicit-any
456
470
  let response = Array.empty<Response.StreamPart<any>>()
471
+ const discardAttempt = () => {
472
+ response = []
473
+ pendingImages.length = 0
474
+ }
457
475
  let reasoningStarted = false
458
476
  let hadReasoningDelta = false
459
477
  let hadToolCall = false
460
478
  const runModel = pipe(
461
479
  Stream.suspend(() =>
462
- ai.streamText({ prompt: prompt.current, toolkit: singleTool }),
480
+ ai.streamText({
481
+ prompt: sendImages
482
+ ? prompt.current
483
+ : Image.stripImages(prompt.current),
484
+ toolkit: singleTool,
485
+ }),
463
486
  ),
464
487
  Stream.takeUntil((part) => {
465
488
  if (
@@ -555,7 +578,7 @@ ${content}
555
578
  }),
556
579
  }),
557
580
  })
558
- response = []
581
+ discardAttempt()
559
582
  return true
560
583
  }
561
584
  // Never retry a context-length error as is: the overflow
@@ -575,21 +598,37 @@ ${content}
575
598
  }
576
599
  }
577
600
  }
578
- response = []
601
+ discardAttempt()
579
602
  return err.isRetryable
580
603
  },
581
604
  schedule: retryPolicy,
582
605
  }),
583
606
  Effect.catchTag("TimeoutError", Effect.die),
584
607
  )
608
+ // Unsupported images: drop them from the request and retry once.
609
+ // The images stay in the Prompt with an omitted note in their place.
610
+ const attemptWithImageFallback = attempt.pipe(
611
+ Effect.catchIf(
612
+ (err): err is AiError.AiError =>
613
+ sendImages &&
614
+ Image.hasImages(prompt.current) &&
615
+ Image.isUnsupportedImageError(err),
616
+ (err) => {
617
+ sendImages = false
618
+ discardAttempt()
619
+ maybeSend({ agentId, part: new ErrorRetry({ error: err }) })
620
+ return attempt
621
+ },
622
+ ),
623
+ )
585
624
  // Overflow: compact once, retry the attempt once, then fail.
586
- yield* attempt.pipe(
625
+ yield* attemptWithImageFallback.pipe(
587
626
  Effect.catchIf(
588
627
  (err): err is AiError.AiError =>
589
628
  Compaction.isContextLengthError(err),
590
629
  (err) =>
591
630
  Effect.gen(function* () {
592
- response = []
631
+ discardAttempt()
593
632
  const compacted = yield* compactWithEvents(
594
633
  "overflow",
595
634
  Compaction.compact({
@@ -605,7 +644,9 @@ ${content}
605
644
  Effect.catchTag("TimeoutError", Effect.die),
606
645
  ),
607
646
  )
608
- return Option.isSome(compacted) ? yield* attempt : yield* err
647
+ return Option.isSome(compacted)
648
+ ? yield* attemptWithImageFallback
649
+ : yield* err
609
650
  }),
610
651
  ),
611
652
  )
@@ -613,6 +654,16 @@ ${content}
613
654
  prompt,
614
655
  Prompt.concat(Prompt.fromResponseParts(response)),
615
656
  )
657
+ if (pendingImages.length > 0) {
658
+ // After the tool result, so call / result pairing stays intact.
659
+ MutableRef.update(
660
+ prompt,
661
+ Prompt.concat(
662
+ Prompt.fromMessages([Image.userMessage(pendingImages)]),
663
+ ),
664
+ )
665
+ pendingImages.length = 0
666
+ }
616
667
  if (conversationMode && !hadToolCall && pendingMessages.size === 0) {
617
668
  finalSummary = Option.some(responseToSummary(response))
618
669
  }
@@ -793,7 +844,11 @@ export const layerLocal = <Toolkit extends Toolkit.Any = never>(options: {
793
844
  Toolkit extends Toolkit.Toolkit<infer T>
794
845
  ? Tool.HandlersFor<T> | Tool.HandlerServices<T[keyof T]>
795
846
  : never,
796
- CurrentDirectory | DirectoryChanger | SubagentExecutor | TaskCompleter
847
+ | CurrentDirectory
848
+ | DirectoryChanger
849
+ | SubagentExecutor
850
+ | TaskCompleter
851
+ | ImageAttacher
797
852
  >
798
853
  > => layer.pipe(Layer.provide(AgentExecutor.layerLocal(options)))
799
854
 
@@ -870,6 +925,13 @@ export class AgentModelConfig extends Context.Reference<{
870
925
  effect: Effect.Effect<A, E, R>,
871
926
  ) => Effect.Effect<A, E, R>)
872
927
  | undefined
928
+ /**
929
+ * `false` for models known not to take image input: image parts are
930
+ * stripped before every request. Leave unset when unknown; the Agent then
931
+ * sends images and retries once without them if the provider rejects
932
+ * them.
933
+ */
934
+ readonly supportsImages?: boolean | undefined
873
935
  }>("clanka/Agent/SystemPromptTransform", {
874
936
  defaultValue: () => ({}),
875
937
  }) {
@@ -11,9 +11,12 @@ import {
11
11
  AgentToolsWithSearch,
12
12
  CurrentDirectory,
13
13
  DirectoryChanger,
14
+ ImageAttacher,
15
+ type ImageAttachment,
14
16
  SubagentExecutor,
15
17
  TaskCompleter,
16
18
  } from "./AgentTools.ts"
19
+ import * as Image from "./Image.ts"
17
20
  import { ToolkitRenderer } from "./ToolkitRenderer.ts"
18
21
  import * as AgentSkills from "./AgentSkills.ts"
19
22
  import type * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"
@@ -56,6 +59,10 @@ export class AgentExecutor extends Context.Service<
56
59
  readonly script: string
57
60
  readonly onTaskComplete: (summary: string) => Effect.Effect<void>
58
61
  readonly onSubagent: (message: string) => Effect.Effect<string>
62
+ /**
63
+ * Receives every image `readFile` attached while the script ran.
64
+ */
65
+ readonly onImage: (image: ImageAttachment) => Effect.Effect<void>
59
66
  }): Stream.Stream<string>
60
67
  executeUnsafe<Tool extends keyof typeof AgentTools.tools>(options: {
61
68
  readonly tool: Tool
@@ -101,7 +108,11 @@ export const makeLocal = Effect.fnUntraced(function* <
101
108
  Toolkit extends Toolkit.Toolkit<infer T>
102
109
  ? Tool.HandlersFor<T> | Tool.HandlerServices<T[keyof T]>
103
110
  : never,
104
- CurrentDirectory | DirectoryChanger | SubagentExecutor | TaskCompleter
111
+ | CurrentDirectory
112
+ | DirectoryChanger
113
+ | SubagentExecutor
114
+ | TaskCompleter
115
+ | ImageAttacher
105
116
  >
106
117
  > {
107
118
  const fs = yield* FileSystem.FileSystem
@@ -145,6 +156,7 @@ export const makeLocal = Effect.fnUntraced(function* <
145
156
  readonly script: string
146
157
  readonly onTaskComplete: (summary: string) => Effect.Effect<void>
147
158
  readonly onSubagent: (message: string) => Effect.Effect<string>
159
+ readonly onImage: (image: ImageAttachment) => Effect.Effect<void>
148
160
  }) {
149
161
  const output = yield* Queue.unbounded<string, Cause.Done>()
150
162
  const console = yield* makeConsole(output)
@@ -155,6 +167,7 @@ export const makeLocal = Effect.fnUntraced(function* <
155
167
  Context.add(TaskCompleter, opts.onTaskComplete),
156
168
  Context.add(DirectoryChanger, changeDirectory),
157
169
  Context.add(SubagentExecutor, opts.onSubagent),
170
+ Context.add(ImageAttacher, opts.onImage),
158
171
  Context.add(Console.Console, console),
159
172
  Context.add(References.CurrentLogAnnotations, {}),
160
173
  Option.isSome(search)
@@ -254,6 +267,8 @@ export const makeLocal = Effect.fnUntraced(function* <
254
267
  Context.add(CurrentDirectory, currentDirectory),
255
268
  Context.add(DirectoryChanger, changeDirectory),
256
269
  Context.add(SubagentExecutor, () => Effect.succeed("")),
270
+ // Direct tool calls have no model turn to attach images to.
271
+ Context.add(ImageAttacher, () => Effect.void),
257
272
  Context.add(References.CurrentLogAnnotations, {}),
258
273
  Option.isSome(search)
259
274
  ? Context.add(SemanticSearch, search.value)
@@ -305,6 +320,13 @@ export const makeRpc = Effect.gen(function* () {
305
320
  Effect.forkIn(scope),
306
321
  )
307
322
  }
323
+ case "Image": {
324
+ return opts.onImage({
325
+ fileName: part.fileName,
326
+ mediaType: part.mediaType,
327
+ data: part.data,
328
+ })
329
+ }
308
330
  }
309
331
  }),
310
332
  Stream.orDie,
@@ -337,7 +359,11 @@ export const layerLocal = <Toolkit extends Toolkit.Any = never>(options: {
337
359
  Toolkit extends Toolkit.Toolkit<infer T>
338
360
  ? Tool.HandlersFor<T> | Tool.HandlerServices<T[keyof T]>
339
361
  : never,
340
- CurrentDirectory | DirectoryChanger | SubagentExecutor | TaskCompleter
362
+ | CurrentDirectory
363
+ | DirectoryChanger
364
+ | SubagentExecutor
365
+ | TaskCompleter
366
+ | ImageAttacher
341
367
  >
342
368
  > =>
343
369
  Layer.effect(AgentExecutor, makeLocal(options)).pipe(
@@ -377,7 +403,11 @@ export const layerRpcServer = <Toolkit extends Toolkit.Any = never>(options: {
377
403
  Toolkit extends Toolkit.Toolkit<infer T>
378
404
  ? Tool.HandlersFor<T> | Tool.HandlerServices<T[keyof T]>
379
405
  : never,
380
- CurrentDirectory | DirectoryChanger | SubagentExecutor | TaskCompleter
406
+ | CurrentDirectory
407
+ | DirectoryChanger
408
+ | SubagentExecutor
409
+ | TaskCompleter
410
+ | ImageAttacher
381
411
  >
382
412
  > =>
383
413
  RpcServer.layer(Rpcs, {
@@ -433,6 +463,14 @@ export const layerRpcServer = <Toolkit extends Toolkit.Any = never>(options: {
433
463
  })
434
464
  })
435
465
  },
466
+ onImage(image) {
467
+ return Queue.offer(queue, {
468
+ _tag: "Image",
469
+ fileName: image.fileName,
470
+ mediaType: image.mediaType,
471
+ data: image.data,
472
+ })
473
+ },
436
474
  }),
437
475
  Stream.runForEachArray((parts) => {
438
476
  for (const part of parts) {
@@ -443,6 +481,11 @@ export const layerRpcServer = <Toolkit extends Toolkit.Any = never>(options: {
443
481
  }
444
482
  return Effect.void
445
483
  }),
484
+ Effect.onExit((exit) =>
485
+ Exit.isSuccess(exit)
486
+ ? Queue.end(queue)
487
+ : Queue.failCause(queue, exit.cause),
488
+ ),
446
489
  Effect.forkScoped,
447
490
  )
448
491
 
@@ -466,6 +509,11 @@ export const ExecuteOutput = Schema.TaggedUnion({
466
509
  Text: { text: Schema.String },
467
510
  TaskComplete: { summary: Schema.String },
468
511
  Subagent: { id: Schema.Finite, prompt: Schema.String },
512
+ Image: {
513
+ fileName: Schema.String,
514
+ mediaType: Image.ImageMediaType,
515
+ data: Schema.Uint8Array,
516
+ },
469
517
  })
470
518
 
471
519
  /**
@@ -0,0 +1,103 @@
1
+ import { assert, describe, it } from "@effect/vitest"
2
+ import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer"
3
+ import * as NodeServices from "@effect/platform-node/NodeServices"
4
+ import * as Effect from "effect/Effect"
5
+ import * as Encoding from "effect/Encoding"
6
+ import * as FileSystem from "effect/FileSystem"
7
+ import * as Layer from "effect/Layer"
8
+ import * as Path from "effect/Path"
9
+ import * as Stream from "effect/Stream"
10
+ import * as HttpServer from "effect/unstable/http/HttpServer"
11
+ import * as RpcClient from "effect/unstable/rpc/RpcClient"
12
+ import * as RpcSerialization from "effect/unstable/rpc/RpcSerialization"
13
+ import * as RpcServer from "effect/unstable/rpc/RpcServer"
14
+ import * as AgentExecutor from "./AgentExecutor.ts"
15
+ import type { ImageAttachment } from "./AgentTools.ts"
16
+ import { equalBytes, supportedImages } from "./fixtures/TestImages.ts"
17
+
18
+ // Real HTTP transport and NDJSON serialization, not RpcTest's no-codec path.
19
+ const withRpc = <A, E, R>(
20
+ f: (
21
+ executor: AgentExecutor.AgentExecutor["Service"],
22
+ ) => Effect.Effect<A, E, R>,
23
+ ) =>
24
+ Effect.gen(function* () {
25
+ const fs = yield* FileSystem.FileSystem
26
+ const path = yield* Path.Path
27
+ const directory = yield* fs.makeTempDirectoryScoped()
28
+ for (const [name, data] of fixtures) {
29
+ yield* fs.writeFile(path.join(directory, name), data)
30
+ }
31
+ const { protocol, httpEffect } =
32
+ yield* RpcServer.makeProtocolWithHttpEffect()
33
+ yield* Layer.build(
34
+ AgentExecutor.layerRpcServer({ directory }).pipe(
35
+ Layer.provide(Layer.succeed(RpcServer.Protocol, protocol)),
36
+ ),
37
+ )
38
+ yield* HttpServer.serveEffect(httpEffect)
39
+ return yield* Effect.gen(function* () {
40
+ const executor = yield* AgentExecutor.AgentExecutor
41
+ return yield* f(executor)
42
+ }).pipe(
43
+ Effect.provide(
44
+ AgentExecutor.layerRpc.pipe(
45
+ Layer.provide(RpcClient.layerProtocolHttp({ url: "" })),
46
+ ),
47
+ ),
48
+ )
49
+ }).pipe(
50
+ Effect.provide(
51
+ Layer.mergeAll(
52
+ RpcSerialization.layerNdjson,
53
+ NodeHttpServer.layerTest,
54
+ NodeServices.layer,
55
+ ),
56
+ ),
57
+ Effect.scoped,
58
+ )
59
+
60
+ const fixtures = supportedImages.map(
61
+ ({ fileName, data, mediaType }) => [fileName, data, mediaType] as const,
62
+ )
63
+
64
+ describe("RPC image executor", () => {
65
+ it.live(
66
+ "delivers supported images and markers, then completes the RPC stream",
67
+ () =>
68
+ withRpc((executor) =>
69
+ Effect.gen(function* () {
70
+ const images: Array<ImageAttachment> = []
71
+ const output = yield* executor
72
+ .execute({
73
+ script: fixtures
74
+ .map(
75
+ ([name]) =>
76
+ "console.log(await readFile({ path: " +
77
+ JSON.stringify(name) +
78
+ " }))",
79
+ )
80
+ .join(";"),
81
+ onImage: (image) =>
82
+ Effect.sync(() => {
83
+ images.push(image)
84
+ }),
85
+ onTaskComplete: () => Effect.void,
86
+ onSubagent: () => Effect.succeed(""),
87
+ })
88
+ .pipe(Stream.runCollect, Effect.timeout("5 seconds"))
89
+ assert.strictEqual(images.length, fixtures.length)
90
+ for (const [index, [name, bytes, mediaType]] of fixtures.entries()) {
91
+ const image = images[index]!
92
+ assert.strictEqual(image.fileName, name)
93
+ assert.strictEqual(image.mediaType, mediaType)
94
+ assert.instanceOf(image.data, Uint8Array)
95
+ assert.isTrue(equalBytes(image.data, bytes))
96
+ assert.include(output.join(""), "Image attached: " + name)
97
+ assert.notInclude(output.join(""), Encoding.encodeBase64(bytes))
98
+ }
99
+ assert.notInclude(output.join(""), "data:image/")
100
+ }),
101
+ ),
102
+ )
103
+ })
@@ -0,0 +1,295 @@
1
+ import { assert, describe, it } from "@effect/vitest"
2
+ import * as Duration from "effect/Duration"
3
+ import * as Deferred from "effect/Deferred"
4
+ import * as Effect from "effect/Effect"
5
+ import * as Exit from "effect/Exit"
6
+ import * as Layer from "effect/Layer"
7
+ import * as Option from "effect/Option"
8
+ import * as Stream from "effect/Stream"
9
+ import * as AiError from "effect/unstable/ai/AiError"
10
+ import * as LanguageModel from "effect/unstable/ai/LanguageModel"
11
+ import * as Model from "effect/unstable/ai/Model"
12
+ import type * as Prompt from "effect/unstable/ai/Prompt"
13
+ import type * as Response from "effect/unstable/ai/Response"
14
+ import * as Agent from "./Agent.ts"
15
+ import * as AgentExecutor from "./AgentExecutor.ts"
16
+ import { tinyPng } from "./fixtures/TestImages.ts"
17
+
18
+ const capabilities = new AgentExecutor.Capabilities({
19
+ toolsDts: "",
20
+ agentsMd: Option.none(),
21
+ supportsSearch: false,
22
+ skills: [],
23
+ })
24
+
25
+ const makeExecutor = (
26
+ execute: AgentExecutor.AgentExecutor["Service"]["execute"] = () =>
27
+ Stream.empty,
28
+ ) =>
29
+ AgentExecutor.AgentExecutor.of({
30
+ capabilities: Effect.succeed(capabilities),
31
+ execute,
32
+ executeUnsafe: () => Effect.die("executeUnsafe not implemented"),
33
+ })
34
+
35
+ interface RecordedCall {
36
+ readonly prompt: Prompt.Prompt
37
+ }
38
+
39
+ const runAgent = (options: {
40
+ readonly prompt: Prompt.RawInput
41
+ readonly executor?: AgentExecutor.AgentExecutor["Service"] | undefined
42
+ readonly modelConfig?: typeof Agent.AgentModelConfig.Service | undefined
43
+ readonly respond: (
44
+ call: RecordedCall,
45
+ index: number,
46
+ ) => Stream.Stream<Response.StreamPartEncoded, AiError.AiError>
47
+ }) =>
48
+ Effect.gen(function* () {
49
+ const calls: Array<RecordedCall> = []
50
+ const languageModel = yield* LanguageModel.make({
51
+ generateText: () => Effect.succeed([]),
52
+ streamText: (providerOptions) => {
53
+ const call = { prompt: providerOptions.prompt }
54
+ const index = calls.push(call) - 1
55
+ return options.respond(call, index)
56
+ },
57
+ })
58
+ const modelLayer = Layer.mergeAll(
59
+ Layer.succeed(LanguageModel.LanguageModel, languageModel),
60
+ Layer.succeed(Model.ProviderName, "test-provider"),
61
+ Layer.succeed(Model.ModelName, "test-model"),
62
+ )
63
+ const agent = yield* Agent.make.pipe(
64
+ Effect.provideService(
65
+ AgentExecutor.AgentExecutor,
66
+ options.executor ?? makeExecutor(),
67
+ ),
68
+ )
69
+ const exit = yield* agent.send({ prompt: options.prompt }).pipe(
70
+ Effect.flatMap((stream) =>
71
+ stream.pipe(
72
+ Stream.runDrain,
73
+ Effect.as(""),
74
+ Effect.catchTag("AgentFinished", (finished) =>
75
+ Effect.succeed(finished.summary),
76
+ ),
77
+ ),
78
+ ),
79
+ Effect.provide(
80
+ Layer.mergeAll(
81
+ modelLayer,
82
+ Agent.ConversationMode.layer(true),
83
+ Agent.layerSubagentModel(modelLayer),
84
+ Agent.AgentModelConfig.layer(options.modelConfig ?? {}),
85
+ Layer.succeed(Agent.TurnTimeout, Duration.minutes(5)),
86
+ ),
87
+ ),
88
+ Effect.exit,
89
+ )
90
+ return { exit, calls, history: agent.history.current }
91
+ }).pipe(Effect.scoped)
92
+
93
+ const text = (content: string): Array<Response.StreamPartEncoded> => [
94
+ { type: "text-start", id: "t" },
95
+ { type: "text-delta", id: "t", delta: content },
96
+ { type: "text-end", id: "t" },
97
+ ]
98
+
99
+ const toolCall = (id: string, script: string): Response.StreamPartEncoded => ({
100
+ type: "tool-call",
101
+ id,
102
+ name: "execute",
103
+ params: { script },
104
+ })
105
+
106
+ /** A user turn with a caption and a screenshot, as ACP ingest would build. */
107
+ const imagePrompt: Prompt.RawInput = [
108
+ {
109
+ role: "user",
110
+ content: [
111
+ { type: "text", text: "what is in this screenshot?" },
112
+ {
113
+ type: "file",
114
+ mediaType: "image/png",
115
+ fileName: "shot.png",
116
+ data: tinyPng,
117
+ },
118
+ ],
119
+ },
120
+ ]
121
+
122
+ const fileParts = (prompt: Prompt.Prompt): Array<Prompt.FilePart> =>
123
+ prompt.content.flatMap((message) =>
124
+ message.role === "user"
125
+ ? message.content.filter((part) => part.type === "file")
126
+ : [],
127
+ )
128
+
129
+ const userText = (prompt: Prompt.Prompt): string =>
130
+ prompt.content
131
+ .flatMap((message) =>
132
+ message.role === "user"
133
+ ? message.content.flatMap((part) =>
134
+ part.type === "text" ? [part.text] : [],
135
+ )
136
+ : [],
137
+ )
138
+ .join("\n")
139
+
140
+ const omittedNote = /\[image: shot\.png omitted\]/
141
+
142
+ /** Mirrors the OpenAI rejection for image input on a text-only model. */
143
+ const unsupportedImageError = AiError.make({
144
+ module: "OpenAiLanguageModel",
145
+ method: "streamText",
146
+ reason: new AiError.InvalidRequestError({
147
+ description:
148
+ "Invalid content type. image_url is only supported by certain models. (POST https://api.openai.com/v1/responses) [code: invalid_value]",
149
+ }),
150
+ })
151
+
152
+ const unrelatedError = AiError.make({
153
+ module: "OpenAiLanguageModel",
154
+ method: "streamText",
155
+ reason: new AiError.InvalidRequestError({
156
+ description:
157
+ "Unsupported parameter: 'temperature' is not supported with this model. (POST https://api.openai.com/v1/responses)",
158
+ }),
159
+ })
160
+
161
+ describe("Agent images", () => {
162
+ it.effect(
163
+ "strips images and leaves an omitted note for a model known not to take images",
164
+ () =>
165
+ Effect.gen(function* () {
166
+ const { exit, calls } = yield* runAgent({
167
+ prompt: imagePrompt,
168
+ modelConfig: { supportsImages: false },
169
+ respond: () => Stream.fromIterable(text("I cannot see images.")),
170
+ })
171
+ assert.isTrue(Exit.isSuccess(exit))
172
+ assert.strictEqual(calls.length, 1)
173
+ assert.strictEqual(fileParts(calls[0]!.prompt).length, 0)
174
+ const prompt = userText(calls[0]!.prompt)
175
+ assert.include(prompt, "what is in this screenshot?")
176
+ assert.match(prompt, omittedNote)
177
+ }),
178
+ )
179
+
180
+ it.effect(
181
+ "retries without images and discards attachments from the failed attempt",
182
+ () =>
183
+ Effect.gen(function* () {
184
+ const attached = yield* Deferred.make<void>()
185
+ let executions = 0
186
+ const executor = makeExecutor(({ onImage }) =>
187
+ Stream.fromEffect(
188
+ Effect.gen(function* () {
189
+ executions++
190
+ yield* onImage({
191
+ data: tinyPng,
192
+ mediaType: "image/png",
193
+ fileName: executions === 1 ? "discarded.png" : "shot.png",
194
+ })
195
+ yield* Deferred.succeed(attached, undefined)
196
+ return "Image attached"
197
+ }),
198
+ ),
199
+ )
200
+ const { exit, calls, history } = yield* runAgent({
201
+ prompt: imagePrompt,
202
+ executor,
203
+ respond: (_, index) =>
204
+ index === 0
205
+ ? Stream.make(
206
+ toolCall("discarded", 'await readFile({ path: "shot.png" })'),
207
+ ).pipe(
208
+ // Advance past the tool-call chunk so its handler starts.
209
+ Stream.concat(
210
+ Stream.make({ type: "text-start", id: "failed" } as const),
211
+ ),
212
+ Stream.concat(
213
+ Stream.fromEffect(
214
+ Deferred.await(attached).pipe(
215
+ Effect.andThen(Effect.fail(unsupportedImageError)),
216
+ ),
217
+ ),
218
+ ),
219
+ )
220
+ : index === 1
221
+ ? Stream.make(
222
+ toolCall("retried", 'await readFile({ path: "shot.png" })'),
223
+ )
224
+ : Stream.fromIterable(text("Text only, but I tried.")),
225
+ })
226
+ assert.isTrue(Exit.isSuccess(exit))
227
+ assert.strictEqual(calls.length, 3)
228
+ assert.strictEqual(executions, 2)
229
+ assert.deepStrictEqual(
230
+ fileParts(history).map((part) => part.fileName),
231
+ ["shot.png", "shot.png"],
232
+ )
233
+ assert.strictEqual(fileParts(calls[0]!.prompt).length, 1)
234
+ assert.strictEqual(fileParts(calls[1]!.prompt).length, 0)
235
+ const retried = userText(calls[1]!.prompt)
236
+ assert.include(retried, "what is in this screenshot?")
237
+ assert.match(retried, omittedNote)
238
+ }),
239
+ )
240
+
241
+ it.effect("does not retry when the provider rejects it a second time", () =>
242
+ Effect.gen(function* () {
243
+ const { exit, calls } = yield* runAgent({
244
+ prompt: imagePrompt,
245
+ respond: () => Stream.fail(unsupportedImageError),
246
+ })
247
+ assert.isTrue(Exit.isFailure(exit))
248
+ assert.strictEqual(calls.length, 2)
249
+ }),
250
+ )
251
+
252
+ it.effect(
253
+ "does not treat other invalid-request errors as image rejections",
254
+ () =>
255
+ Effect.gen(function* () {
256
+ const { exit, calls } = yield* runAgent({
257
+ prompt: imagePrompt,
258
+ respond: () => Stream.fail(unrelatedError),
259
+ })
260
+ assert.isTrue(Exit.isFailure(exit))
261
+ assert.strictEqual(calls.length, 1)
262
+ }),
263
+ )
264
+
265
+ it.effect("does not copy image bytes into a delegate prompt", () =>
266
+ Effect.gen(function* () {
267
+ const executor = makeExecutor(({ onSubagent }) =>
268
+ Stream.fromEffect(onSubagent("child task: describe the layout")),
269
+ )
270
+ const { exit, calls } = yield* runAgent({
271
+ prompt: imagePrompt,
272
+ executor,
273
+ respond: (call, index) => {
274
+ if (index === 0) {
275
+ return Stream.make(
276
+ toolCall("call-1", 'await delegate("describe the layout")'),
277
+ )
278
+ }
279
+ const isChild = userText(call.prompt).includes("child task")
280
+ return Stream.fromIterable(
281
+ text(isChild ? "child done" : "parent done"),
282
+ )
283
+ },
284
+ })
285
+ assert.isTrue(Exit.isSuccess(exit))
286
+ const child = calls.find((call) =>
287
+ userText(call.prompt).includes("child task"),
288
+ )
289
+ assert.isDefined(child, "the delegate call reached the model")
290
+ assert.strictEqual(fileParts(child!.prompt).length, 0)
291
+ // The parent still has its image.
292
+ assert.strictEqual(fileParts(calls[0]!.prompt).length, 1)
293
+ }),
294
+ )
295
+ })