clanka 0.5.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Acp.d.ts.map +1 -1
- package/dist/Acp.js +195 -16
- package/dist/Acp.js.map +1 -1
- package/dist/AcpImage.test.d.ts +2 -0
- package/dist/AcpImage.test.d.ts.map +1 -0
- package/dist/AcpImage.test.js +470 -0
- package/dist/AcpImage.test.js.map +1 -0
- package/dist/Agent.d.ts +9 -2
- package/dist/Agent.d.ts.map +1 -1
- package/dist/Agent.js +43 -6
- package/dist/Agent.js.map +1 -1
- package/dist/AgentExecutor.d.ts +27 -5
- package/dist/AgentExecutor.d.ts.map +1 -1
- package/dist/AgentExecutor.js +29 -4
- package/dist/AgentExecutor.js.map +1 -1
- package/dist/AgentExecutorImage.test.d.ts +2 -0
- package/dist/AgentExecutorImage.test.d.ts.map +1 -0
- package/dist/AgentExecutorImage.test.js +63 -0
- package/dist/AgentExecutorImage.test.js.map +1 -0
- package/dist/AgentImage.test.d.ts +2 -0
- package/dist/AgentImage.test.d.ts.map +1 -0
- package/dist/AgentImage.test.js +176 -0
- package/dist/AgentImage.test.js.map +1 -0
- package/dist/AgentTools.d.ts +25 -5
- package/dist/AgentTools.d.ts.map +1 -1
- package/dist/AgentTools.js +40 -4
- package/dist/AgentTools.js.map +1 -1
- package/dist/AgentToolsImage.test.d.ts +2 -0
- package/dist/AgentToolsImage.test.d.ts.map +1 -0
- package/dist/AgentToolsImage.test.js +158 -0
- package/dist/AgentToolsImage.test.js.map +1 -0
- package/dist/Codex.js +6 -1
- package/dist/Codex.js.map +1 -1
- package/dist/Compaction.d.ts.map +1 -1
- package/dist/Compaction.js +16 -1
- package/dist/Compaction.js.map +1 -1
- package/dist/CompactionImage.test.d.ts +2 -0
- package/dist/CompactionImage.test.d.ts.map +1 -0
- package/dist/CompactionImage.test.js +105 -0
- package/dist/CompactionImage.test.js.map +1 -0
- package/dist/Copilot.d.ts.map +1 -1
- package/dist/Copilot.js +5 -1
- package/dist/Copilot.js.map +1 -1
- package/dist/Image.d.ts +192 -0
- package/dist/Image.d.ts.map +1 -0
- package/dist/Image.js +586 -0
- package/dist/Image.js.map +1 -0
- package/dist/Image.test.d.ts +2 -0
- package/dist/Image.test.d.ts.map +1 -0
- package/dist/Image.test.js +142 -0
- package/dist/Image.test.js.map +1 -0
- package/dist/fixtures/TestImages.d.ts +73 -0
- package/dist/fixtures/TestImages.d.ts.map +1 -0
- package/dist/fixtures/TestImages.js +263 -0
- package/dist/fixtures/TestImages.js.map +1 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -0
- package/dist/index.js.map +1 -1
- package/package.json +2 -1
- package/src/Acp.ts +274 -21
- package/src/AcpImage.test.ts +676 -0
- package/src/Agent.ts +69 -7
- package/src/AgentExecutor.ts +51 -3
- package/src/AgentExecutorImage.test.ts +103 -0
- package/src/AgentImage.test.ts +295 -0
- package/src/AgentTools.ts +78 -7
- package/src/AgentToolsImage.test.ts +302 -0
- package/src/Codex.ts +6 -1
- package/src/Compaction.ts +16 -1
- package/src/CompactionImage.test.ts +142 -0
- package/src/Copilot.ts +5 -1
- package/src/Image.test.ts +220 -0
- package/src/Image.ts +704 -0
- package/src/fixtures/TestImages.ts +287 -0
- package/src/index.ts +5 -0
package/src/Agent.ts
CHANGED
|
@@ -12,9 +12,12 @@ import type * as HttpClient from "effect/unstable/http/HttpClient"
|
|
|
12
12
|
import type {
|
|
13
13
|
CurrentDirectory,
|
|
14
14
|
DirectoryChanger,
|
|
15
|
+
ImageAttacher,
|
|
16
|
+
ImageAttachment,
|
|
15
17
|
SubagentExecutor,
|
|
16
18
|
TaskCompleter,
|
|
17
19
|
} from "./AgentTools.ts"
|
|
20
|
+
import * as Image from "./Image.ts"
|
|
18
21
|
import type * as FileSystem from "effect/FileSystem"
|
|
19
22
|
import * as Prompt from "effect/unstable/ai/Prompt"
|
|
20
23
|
import * as Effect from "effect/Effect"
|
|
@@ -197,6 +200,13 @@ ${content}
|
|
|
197
200
|
// `inputTokens.total` from the most recent finish part; undefined until
|
|
198
201
|
// the first response and after every compaction.
|
|
199
202
|
let lastContextTokens: number | undefined = undefined
|
|
203
|
+
// Images stay in the Prompt; this only decides whether they are sent.
|
|
204
|
+
// Starts false for models known not to take images, and flips to false
|
|
205
|
+
// after a provider rejects image input once.
|
|
206
|
+
let sendImages = modelConfig.supportsImages !== false
|
|
207
|
+
// Images attached by `readFile` during the current script, spliced onto
|
|
208
|
+
// the Prompt as a user message once the tool result has been recorded.
|
|
209
|
+
const pendingImages: Array<ImageAttachment> = []
|
|
200
210
|
|
|
201
211
|
const output = yield* Queue.make<Output, AgentFinished | AiError.AiError>()
|
|
202
212
|
let inputTokens = 0
|
|
@@ -386,6 +396,10 @@ ${content}
|
|
|
386
396
|
Effect.sync(() => {
|
|
387
397
|
finalSummary = Option.some(summary)
|
|
388
398
|
}),
|
|
399
|
+
onImage: (image) =>
|
|
400
|
+
Effect.sync(() => {
|
|
401
|
+
pendingImages.push(image)
|
|
402
|
+
}),
|
|
389
403
|
}),
|
|
390
404
|
Stream.mkString,
|
|
391
405
|
)
|
|
@@ -454,12 +468,21 @@ ${content}
|
|
|
454
468
|
|
|
455
469
|
// oxlint-disable-next-line typescript/no-explicit-any
|
|
456
470
|
let response = Array.empty<Response.StreamPart<any>>()
|
|
471
|
+
const discardAttempt = () => {
|
|
472
|
+
response = []
|
|
473
|
+
pendingImages.length = 0
|
|
474
|
+
}
|
|
457
475
|
let reasoningStarted = false
|
|
458
476
|
let hadReasoningDelta = false
|
|
459
477
|
let hadToolCall = false
|
|
460
478
|
const runModel = pipe(
|
|
461
479
|
Stream.suspend(() =>
|
|
462
|
-
ai.streamText({
|
|
480
|
+
ai.streamText({
|
|
481
|
+
prompt: sendImages
|
|
482
|
+
? prompt.current
|
|
483
|
+
: Image.stripImages(prompt.current),
|
|
484
|
+
toolkit: singleTool,
|
|
485
|
+
}),
|
|
463
486
|
),
|
|
464
487
|
Stream.takeUntil((part) => {
|
|
465
488
|
if (
|
|
@@ -555,7 +578,7 @@ ${content}
|
|
|
555
578
|
}),
|
|
556
579
|
}),
|
|
557
580
|
})
|
|
558
|
-
|
|
581
|
+
discardAttempt()
|
|
559
582
|
return true
|
|
560
583
|
}
|
|
561
584
|
// Never retry a context-length error as is: the overflow
|
|
@@ -575,21 +598,37 @@ ${content}
|
|
|
575
598
|
}
|
|
576
599
|
}
|
|
577
600
|
}
|
|
578
|
-
|
|
601
|
+
discardAttempt()
|
|
579
602
|
return err.isRetryable
|
|
580
603
|
},
|
|
581
604
|
schedule: retryPolicy,
|
|
582
605
|
}),
|
|
583
606
|
Effect.catchTag("TimeoutError", Effect.die),
|
|
584
607
|
)
|
|
608
|
+
// Unsupported images: drop them from the request and retry once.
|
|
609
|
+
// The images stay in the Prompt with an omitted note in their place.
|
|
610
|
+
const attemptWithImageFallback = attempt.pipe(
|
|
611
|
+
Effect.catchIf(
|
|
612
|
+
(err): err is AiError.AiError =>
|
|
613
|
+
sendImages &&
|
|
614
|
+
Image.hasImages(prompt.current) &&
|
|
615
|
+
Image.isUnsupportedImageError(err),
|
|
616
|
+
(err) => {
|
|
617
|
+
sendImages = false
|
|
618
|
+
discardAttempt()
|
|
619
|
+
maybeSend({ agentId, part: new ErrorRetry({ error: err }) })
|
|
620
|
+
return attempt
|
|
621
|
+
},
|
|
622
|
+
),
|
|
623
|
+
)
|
|
585
624
|
// Overflow: compact once, retry the attempt once, then fail.
|
|
586
|
-
yield*
|
|
625
|
+
yield* attemptWithImageFallback.pipe(
|
|
587
626
|
Effect.catchIf(
|
|
588
627
|
(err): err is AiError.AiError =>
|
|
589
628
|
Compaction.isContextLengthError(err),
|
|
590
629
|
(err) =>
|
|
591
630
|
Effect.gen(function* () {
|
|
592
|
-
|
|
631
|
+
discardAttempt()
|
|
593
632
|
const compacted = yield* compactWithEvents(
|
|
594
633
|
"overflow",
|
|
595
634
|
Compaction.compact({
|
|
@@ -605,7 +644,9 @@ ${content}
|
|
|
605
644
|
Effect.catchTag("TimeoutError", Effect.die),
|
|
606
645
|
),
|
|
607
646
|
)
|
|
608
|
-
return Option.isSome(compacted)
|
|
647
|
+
return Option.isSome(compacted)
|
|
648
|
+
? yield* attemptWithImageFallback
|
|
649
|
+
: yield* err
|
|
609
650
|
}),
|
|
610
651
|
),
|
|
611
652
|
)
|
|
@@ -613,6 +654,16 @@ ${content}
|
|
|
613
654
|
prompt,
|
|
614
655
|
Prompt.concat(Prompt.fromResponseParts(response)),
|
|
615
656
|
)
|
|
657
|
+
if (pendingImages.length > 0) {
|
|
658
|
+
// After the tool result, so call / result pairing stays intact.
|
|
659
|
+
MutableRef.update(
|
|
660
|
+
prompt,
|
|
661
|
+
Prompt.concat(
|
|
662
|
+
Prompt.fromMessages([Image.userMessage(pendingImages)]),
|
|
663
|
+
),
|
|
664
|
+
)
|
|
665
|
+
pendingImages.length = 0
|
|
666
|
+
}
|
|
616
667
|
if (conversationMode && !hadToolCall && pendingMessages.size === 0) {
|
|
617
668
|
finalSummary = Option.some(responseToSummary(response))
|
|
618
669
|
}
|
|
@@ -793,7 +844,11 @@ export const layerLocal = <Toolkit extends Toolkit.Any = never>(options: {
|
|
|
793
844
|
Toolkit extends Toolkit.Toolkit<infer T>
|
|
794
845
|
? Tool.HandlersFor<T> | Tool.HandlerServices<T[keyof T]>
|
|
795
846
|
: never,
|
|
796
|
-
|
|
847
|
+
| CurrentDirectory
|
|
848
|
+
| DirectoryChanger
|
|
849
|
+
| SubagentExecutor
|
|
850
|
+
| TaskCompleter
|
|
851
|
+
| ImageAttacher
|
|
797
852
|
>
|
|
798
853
|
> => layer.pipe(Layer.provide(AgentExecutor.layerLocal(options)))
|
|
799
854
|
|
|
@@ -870,6 +925,13 @@ export class AgentModelConfig extends Context.Reference<{
|
|
|
870
925
|
effect: Effect.Effect<A, E, R>,
|
|
871
926
|
) => Effect.Effect<A, E, R>)
|
|
872
927
|
| undefined
|
|
928
|
+
/**
|
|
929
|
+
* `false` for models known not to take image input: image parts are
|
|
930
|
+
* stripped before every request. Leave unset when unknown; the Agent then
|
|
931
|
+
* sends images and retries once without them if the provider rejects
|
|
932
|
+
* them.
|
|
933
|
+
*/
|
|
934
|
+
readonly supportsImages?: boolean | undefined
|
|
873
935
|
}>("clanka/Agent/SystemPromptTransform", {
|
|
874
936
|
defaultValue: () => ({}),
|
|
875
937
|
}) {
|
package/src/AgentExecutor.ts
CHANGED
|
@@ -11,9 +11,12 @@ import {
|
|
|
11
11
|
AgentToolsWithSearch,
|
|
12
12
|
CurrentDirectory,
|
|
13
13
|
DirectoryChanger,
|
|
14
|
+
ImageAttacher,
|
|
15
|
+
type ImageAttachment,
|
|
14
16
|
SubagentExecutor,
|
|
15
17
|
TaskCompleter,
|
|
16
18
|
} from "./AgentTools.ts"
|
|
19
|
+
import * as Image from "./Image.ts"
|
|
17
20
|
import { ToolkitRenderer } from "./ToolkitRenderer.ts"
|
|
18
21
|
import * as AgentSkills from "./AgentSkills.ts"
|
|
19
22
|
import type * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"
|
|
@@ -56,6 +59,10 @@ export class AgentExecutor extends Context.Service<
|
|
|
56
59
|
readonly script: string
|
|
57
60
|
readonly onTaskComplete: (summary: string) => Effect.Effect<void>
|
|
58
61
|
readonly onSubagent: (message: string) => Effect.Effect<string>
|
|
62
|
+
/**
|
|
63
|
+
* Receives every image `readFile` attached while the script ran.
|
|
64
|
+
*/
|
|
65
|
+
readonly onImage: (image: ImageAttachment) => Effect.Effect<void>
|
|
59
66
|
}): Stream.Stream<string>
|
|
60
67
|
executeUnsafe<Tool extends keyof typeof AgentTools.tools>(options: {
|
|
61
68
|
readonly tool: Tool
|
|
@@ -101,7 +108,11 @@ export const makeLocal = Effect.fnUntraced(function* <
|
|
|
101
108
|
Toolkit extends Toolkit.Toolkit<infer T>
|
|
102
109
|
? Tool.HandlersFor<T> | Tool.HandlerServices<T[keyof T]>
|
|
103
110
|
: never,
|
|
104
|
-
|
|
111
|
+
| CurrentDirectory
|
|
112
|
+
| DirectoryChanger
|
|
113
|
+
| SubagentExecutor
|
|
114
|
+
| TaskCompleter
|
|
115
|
+
| ImageAttacher
|
|
105
116
|
>
|
|
106
117
|
> {
|
|
107
118
|
const fs = yield* FileSystem.FileSystem
|
|
@@ -145,6 +156,7 @@ export const makeLocal = Effect.fnUntraced(function* <
|
|
|
145
156
|
readonly script: string
|
|
146
157
|
readonly onTaskComplete: (summary: string) => Effect.Effect<void>
|
|
147
158
|
readonly onSubagent: (message: string) => Effect.Effect<string>
|
|
159
|
+
readonly onImage: (image: ImageAttachment) => Effect.Effect<void>
|
|
148
160
|
}) {
|
|
149
161
|
const output = yield* Queue.unbounded<string, Cause.Done>()
|
|
150
162
|
const console = yield* makeConsole(output)
|
|
@@ -155,6 +167,7 @@ export const makeLocal = Effect.fnUntraced(function* <
|
|
|
155
167
|
Context.add(TaskCompleter, opts.onTaskComplete),
|
|
156
168
|
Context.add(DirectoryChanger, changeDirectory),
|
|
157
169
|
Context.add(SubagentExecutor, opts.onSubagent),
|
|
170
|
+
Context.add(ImageAttacher, opts.onImage),
|
|
158
171
|
Context.add(Console.Console, console),
|
|
159
172
|
Context.add(References.CurrentLogAnnotations, {}),
|
|
160
173
|
Option.isSome(search)
|
|
@@ -254,6 +267,8 @@ export const makeLocal = Effect.fnUntraced(function* <
|
|
|
254
267
|
Context.add(CurrentDirectory, currentDirectory),
|
|
255
268
|
Context.add(DirectoryChanger, changeDirectory),
|
|
256
269
|
Context.add(SubagentExecutor, () => Effect.succeed("")),
|
|
270
|
+
// Direct tool calls have no model turn to attach images to.
|
|
271
|
+
Context.add(ImageAttacher, () => Effect.void),
|
|
257
272
|
Context.add(References.CurrentLogAnnotations, {}),
|
|
258
273
|
Option.isSome(search)
|
|
259
274
|
? Context.add(SemanticSearch, search.value)
|
|
@@ -305,6 +320,13 @@ export const makeRpc = Effect.gen(function* () {
|
|
|
305
320
|
Effect.forkIn(scope),
|
|
306
321
|
)
|
|
307
322
|
}
|
|
323
|
+
case "Image": {
|
|
324
|
+
return opts.onImage({
|
|
325
|
+
fileName: part.fileName,
|
|
326
|
+
mediaType: part.mediaType,
|
|
327
|
+
data: part.data,
|
|
328
|
+
})
|
|
329
|
+
}
|
|
308
330
|
}
|
|
309
331
|
}),
|
|
310
332
|
Stream.orDie,
|
|
@@ -337,7 +359,11 @@ export const layerLocal = <Toolkit extends Toolkit.Any = never>(options: {
|
|
|
337
359
|
Toolkit extends Toolkit.Toolkit<infer T>
|
|
338
360
|
? Tool.HandlersFor<T> | Tool.HandlerServices<T[keyof T]>
|
|
339
361
|
: never,
|
|
340
|
-
|
|
362
|
+
| CurrentDirectory
|
|
363
|
+
| DirectoryChanger
|
|
364
|
+
| SubagentExecutor
|
|
365
|
+
| TaskCompleter
|
|
366
|
+
| ImageAttacher
|
|
341
367
|
>
|
|
342
368
|
> =>
|
|
343
369
|
Layer.effect(AgentExecutor, makeLocal(options)).pipe(
|
|
@@ -377,7 +403,11 @@ export const layerRpcServer = <Toolkit extends Toolkit.Any = never>(options: {
|
|
|
377
403
|
Toolkit extends Toolkit.Toolkit<infer T>
|
|
378
404
|
? Tool.HandlersFor<T> | Tool.HandlerServices<T[keyof T]>
|
|
379
405
|
: never,
|
|
380
|
-
|
|
406
|
+
| CurrentDirectory
|
|
407
|
+
| DirectoryChanger
|
|
408
|
+
| SubagentExecutor
|
|
409
|
+
| TaskCompleter
|
|
410
|
+
| ImageAttacher
|
|
381
411
|
>
|
|
382
412
|
> =>
|
|
383
413
|
RpcServer.layer(Rpcs, {
|
|
@@ -433,6 +463,14 @@ export const layerRpcServer = <Toolkit extends Toolkit.Any = never>(options: {
|
|
|
433
463
|
})
|
|
434
464
|
})
|
|
435
465
|
},
|
|
466
|
+
onImage(image) {
|
|
467
|
+
return Queue.offer(queue, {
|
|
468
|
+
_tag: "Image",
|
|
469
|
+
fileName: image.fileName,
|
|
470
|
+
mediaType: image.mediaType,
|
|
471
|
+
data: image.data,
|
|
472
|
+
})
|
|
473
|
+
},
|
|
436
474
|
}),
|
|
437
475
|
Stream.runForEachArray((parts) => {
|
|
438
476
|
for (const part of parts) {
|
|
@@ -443,6 +481,11 @@ export const layerRpcServer = <Toolkit extends Toolkit.Any = never>(options: {
|
|
|
443
481
|
}
|
|
444
482
|
return Effect.void
|
|
445
483
|
}),
|
|
484
|
+
Effect.onExit((exit) =>
|
|
485
|
+
Exit.isSuccess(exit)
|
|
486
|
+
? Queue.end(queue)
|
|
487
|
+
: Queue.failCause(queue, exit.cause),
|
|
488
|
+
),
|
|
446
489
|
Effect.forkScoped,
|
|
447
490
|
)
|
|
448
491
|
|
|
@@ -466,6 +509,11 @@ export const ExecuteOutput = Schema.TaggedUnion({
|
|
|
466
509
|
Text: { text: Schema.String },
|
|
467
510
|
TaskComplete: { summary: Schema.String },
|
|
468
511
|
Subagent: { id: Schema.Finite, prompt: Schema.String },
|
|
512
|
+
Image: {
|
|
513
|
+
fileName: Schema.String,
|
|
514
|
+
mediaType: Image.ImageMediaType,
|
|
515
|
+
data: Schema.Uint8Array,
|
|
516
|
+
},
|
|
469
517
|
})
|
|
470
518
|
|
|
471
519
|
/**
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import { assert, describe, it } from "@effect/vitest"
|
|
2
|
+
import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer"
|
|
3
|
+
import * as NodeServices from "@effect/platform-node/NodeServices"
|
|
4
|
+
import * as Effect from "effect/Effect"
|
|
5
|
+
import * as Encoding from "effect/Encoding"
|
|
6
|
+
import * as FileSystem from "effect/FileSystem"
|
|
7
|
+
import * as Layer from "effect/Layer"
|
|
8
|
+
import * as Path from "effect/Path"
|
|
9
|
+
import * as Stream from "effect/Stream"
|
|
10
|
+
import * as HttpServer from "effect/unstable/http/HttpServer"
|
|
11
|
+
import * as RpcClient from "effect/unstable/rpc/RpcClient"
|
|
12
|
+
import * as RpcSerialization from "effect/unstable/rpc/RpcSerialization"
|
|
13
|
+
import * as RpcServer from "effect/unstable/rpc/RpcServer"
|
|
14
|
+
import * as AgentExecutor from "./AgentExecutor.ts"
|
|
15
|
+
import type { ImageAttachment } from "./AgentTools.ts"
|
|
16
|
+
import { equalBytes, supportedImages } from "./fixtures/TestImages.ts"
|
|
17
|
+
|
|
18
|
+
// Real HTTP transport and NDJSON serialization, not RpcTest's no-codec path.
|
|
19
|
+
const withRpc = <A, E, R>(
|
|
20
|
+
f: (
|
|
21
|
+
executor: AgentExecutor.AgentExecutor["Service"],
|
|
22
|
+
) => Effect.Effect<A, E, R>,
|
|
23
|
+
) =>
|
|
24
|
+
Effect.gen(function* () {
|
|
25
|
+
const fs = yield* FileSystem.FileSystem
|
|
26
|
+
const path = yield* Path.Path
|
|
27
|
+
const directory = yield* fs.makeTempDirectoryScoped()
|
|
28
|
+
for (const [name, data] of fixtures) {
|
|
29
|
+
yield* fs.writeFile(path.join(directory, name), data)
|
|
30
|
+
}
|
|
31
|
+
const { protocol, httpEffect } =
|
|
32
|
+
yield* RpcServer.makeProtocolWithHttpEffect()
|
|
33
|
+
yield* Layer.build(
|
|
34
|
+
AgentExecutor.layerRpcServer({ directory }).pipe(
|
|
35
|
+
Layer.provide(Layer.succeed(RpcServer.Protocol, protocol)),
|
|
36
|
+
),
|
|
37
|
+
)
|
|
38
|
+
yield* HttpServer.serveEffect(httpEffect)
|
|
39
|
+
return yield* Effect.gen(function* () {
|
|
40
|
+
const executor = yield* AgentExecutor.AgentExecutor
|
|
41
|
+
return yield* f(executor)
|
|
42
|
+
}).pipe(
|
|
43
|
+
Effect.provide(
|
|
44
|
+
AgentExecutor.layerRpc.pipe(
|
|
45
|
+
Layer.provide(RpcClient.layerProtocolHttp({ url: "" })),
|
|
46
|
+
),
|
|
47
|
+
),
|
|
48
|
+
)
|
|
49
|
+
}).pipe(
|
|
50
|
+
Effect.provide(
|
|
51
|
+
Layer.mergeAll(
|
|
52
|
+
RpcSerialization.layerNdjson,
|
|
53
|
+
NodeHttpServer.layerTest,
|
|
54
|
+
NodeServices.layer,
|
|
55
|
+
),
|
|
56
|
+
),
|
|
57
|
+
Effect.scoped,
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
const fixtures = supportedImages.map(
|
|
61
|
+
({ fileName, data, mediaType }) => [fileName, data, mediaType] as const,
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
describe("RPC image executor", () => {
|
|
65
|
+
it.live(
|
|
66
|
+
"delivers supported images and markers, then completes the RPC stream",
|
|
67
|
+
() =>
|
|
68
|
+
withRpc((executor) =>
|
|
69
|
+
Effect.gen(function* () {
|
|
70
|
+
const images: Array<ImageAttachment> = []
|
|
71
|
+
const output = yield* executor
|
|
72
|
+
.execute({
|
|
73
|
+
script: fixtures
|
|
74
|
+
.map(
|
|
75
|
+
([name]) =>
|
|
76
|
+
"console.log(await readFile({ path: " +
|
|
77
|
+
JSON.stringify(name) +
|
|
78
|
+
" }))",
|
|
79
|
+
)
|
|
80
|
+
.join(";"),
|
|
81
|
+
onImage: (image) =>
|
|
82
|
+
Effect.sync(() => {
|
|
83
|
+
images.push(image)
|
|
84
|
+
}),
|
|
85
|
+
onTaskComplete: () => Effect.void,
|
|
86
|
+
onSubagent: () => Effect.succeed(""),
|
|
87
|
+
})
|
|
88
|
+
.pipe(Stream.runCollect, Effect.timeout("5 seconds"))
|
|
89
|
+
assert.strictEqual(images.length, fixtures.length)
|
|
90
|
+
for (const [index, [name, bytes, mediaType]] of fixtures.entries()) {
|
|
91
|
+
const image = images[index]!
|
|
92
|
+
assert.strictEqual(image.fileName, name)
|
|
93
|
+
assert.strictEqual(image.mediaType, mediaType)
|
|
94
|
+
assert.instanceOf(image.data, Uint8Array)
|
|
95
|
+
assert.isTrue(equalBytes(image.data, bytes))
|
|
96
|
+
assert.include(output.join(""), "Image attached: " + name)
|
|
97
|
+
assert.notInclude(output.join(""), Encoding.encodeBase64(bytes))
|
|
98
|
+
}
|
|
99
|
+
assert.notInclude(output.join(""), "data:image/")
|
|
100
|
+
}),
|
|
101
|
+
),
|
|
102
|
+
)
|
|
103
|
+
})
|
|
@@ -0,0 +1,295 @@
|
|
|
1
|
+
import { assert, describe, it } from "@effect/vitest"
|
|
2
|
+
import * as Duration from "effect/Duration"
|
|
3
|
+
import * as Deferred from "effect/Deferred"
|
|
4
|
+
import * as Effect from "effect/Effect"
|
|
5
|
+
import * as Exit from "effect/Exit"
|
|
6
|
+
import * as Layer from "effect/Layer"
|
|
7
|
+
import * as Option from "effect/Option"
|
|
8
|
+
import * as Stream from "effect/Stream"
|
|
9
|
+
import * as AiError from "effect/unstable/ai/AiError"
|
|
10
|
+
import * as LanguageModel from "effect/unstable/ai/LanguageModel"
|
|
11
|
+
import * as Model from "effect/unstable/ai/Model"
|
|
12
|
+
import type * as Prompt from "effect/unstable/ai/Prompt"
|
|
13
|
+
import type * as Response from "effect/unstable/ai/Response"
|
|
14
|
+
import * as Agent from "./Agent.ts"
|
|
15
|
+
import * as AgentExecutor from "./AgentExecutor.ts"
|
|
16
|
+
import { tinyPng } from "./fixtures/TestImages.ts"
|
|
17
|
+
|
|
18
|
+
const capabilities = new AgentExecutor.Capabilities({
|
|
19
|
+
toolsDts: "",
|
|
20
|
+
agentsMd: Option.none(),
|
|
21
|
+
supportsSearch: false,
|
|
22
|
+
skills: [],
|
|
23
|
+
})
|
|
24
|
+
|
|
25
|
+
const makeExecutor = (
|
|
26
|
+
execute: AgentExecutor.AgentExecutor["Service"]["execute"] = () =>
|
|
27
|
+
Stream.empty,
|
|
28
|
+
) =>
|
|
29
|
+
AgentExecutor.AgentExecutor.of({
|
|
30
|
+
capabilities: Effect.succeed(capabilities),
|
|
31
|
+
execute,
|
|
32
|
+
executeUnsafe: () => Effect.die("executeUnsafe not implemented"),
|
|
33
|
+
})
|
|
34
|
+
|
|
35
|
+
interface RecordedCall {
|
|
36
|
+
readonly prompt: Prompt.Prompt
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const runAgent = (options: {
|
|
40
|
+
readonly prompt: Prompt.RawInput
|
|
41
|
+
readonly executor?: AgentExecutor.AgentExecutor["Service"] | undefined
|
|
42
|
+
readonly modelConfig?: typeof Agent.AgentModelConfig.Service | undefined
|
|
43
|
+
readonly respond: (
|
|
44
|
+
call: RecordedCall,
|
|
45
|
+
index: number,
|
|
46
|
+
) => Stream.Stream<Response.StreamPartEncoded, AiError.AiError>
|
|
47
|
+
}) =>
|
|
48
|
+
Effect.gen(function* () {
|
|
49
|
+
const calls: Array<RecordedCall> = []
|
|
50
|
+
const languageModel = yield* LanguageModel.make({
|
|
51
|
+
generateText: () => Effect.succeed([]),
|
|
52
|
+
streamText: (providerOptions) => {
|
|
53
|
+
const call = { prompt: providerOptions.prompt }
|
|
54
|
+
const index = calls.push(call) - 1
|
|
55
|
+
return options.respond(call, index)
|
|
56
|
+
},
|
|
57
|
+
})
|
|
58
|
+
const modelLayer = Layer.mergeAll(
|
|
59
|
+
Layer.succeed(LanguageModel.LanguageModel, languageModel),
|
|
60
|
+
Layer.succeed(Model.ProviderName, "test-provider"),
|
|
61
|
+
Layer.succeed(Model.ModelName, "test-model"),
|
|
62
|
+
)
|
|
63
|
+
const agent = yield* Agent.make.pipe(
|
|
64
|
+
Effect.provideService(
|
|
65
|
+
AgentExecutor.AgentExecutor,
|
|
66
|
+
options.executor ?? makeExecutor(),
|
|
67
|
+
),
|
|
68
|
+
)
|
|
69
|
+
const exit = yield* agent.send({ prompt: options.prompt }).pipe(
|
|
70
|
+
Effect.flatMap((stream) =>
|
|
71
|
+
stream.pipe(
|
|
72
|
+
Stream.runDrain,
|
|
73
|
+
Effect.as(""),
|
|
74
|
+
Effect.catchTag("AgentFinished", (finished) =>
|
|
75
|
+
Effect.succeed(finished.summary),
|
|
76
|
+
),
|
|
77
|
+
),
|
|
78
|
+
),
|
|
79
|
+
Effect.provide(
|
|
80
|
+
Layer.mergeAll(
|
|
81
|
+
modelLayer,
|
|
82
|
+
Agent.ConversationMode.layer(true),
|
|
83
|
+
Agent.layerSubagentModel(modelLayer),
|
|
84
|
+
Agent.AgentModelConfig.layer(options.modelConfig ?? {}),
|
|
85
|
+
Layer.succeed(Agent.TurnTimeout, Duration.minutes(5)),
|
|
86
|
+
),
|
|
87
|
+
),
|
|
88
|
+
Effect.exit,
|
|
89
|
+
)
|
|
90
|
+
return { exit, calls, history: agent.history.current }
|
|
91
|
+
}).pipe(Effect.scoped)
|
|
92
|
+
|
|
93
|
+
const text = (content: string): Array<Response.StreamPartEncoded> => [
|
|
94
|
+
{ type: "text-start", id: "t" },
|
|
95
|
+
{ type: "text-delta", id: "t", delta: content },
|
|
96
|
+
{ type: "text-end", id: "t" },
|
|
97
|
+
]
|
|
98
|
+
|
|
99
|
+
const toolCall = (id: string, script: string): Response.StreamPartEncoded => ({
|
|
100
|
+
type: "tool-call",
|
|
101
|
+
id,
|
|
102
|
+
name: "execute",
|
|
103
|
+
params: { script },
|
|
104
|
+
})
|
|
105
|
+
|
|
106
|
+
/** A user turn with a caption and a screenshot, as ACP ingest would build. */
|
|
107
|
+
const imagePrompt: Prompt.RawInput = [
|
|
108
|
+
{
|
|
109
|
+
role: "user",
|
|
110
|
+
content: [
|
|
111
|
+
{ type: "text", text: "what is in this screenshot?" },
|
|
112
|
+
{
|
|
113
|
+
type: "file",
|
|
114
|
+
mediaType: "image/png",
|
|
115
|
+
fileName: "shot.png",
|
|
116
|
+
data: tinyPng,
|
|
117
|
+
},
|
|
118
|
+
],
|
|
119
|
+
},
|
|
120
|
+
]
|
|
121
|
+
|
|
122
|
+
const fileParts = (prompt: Prompt.Prompt): Array<Prompt.FilePart> =>
|
|
123
|
+
prompt.content.flatMap((message) =>
|
|
124
|
+
message.role === "user"
|
|
125
|
+
? message.content.filter((part) => part.type === "file")
|
|
126
|
+
: [],
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
const userText = (prompt: Prompt.Prompt): string =>
|
|
130
|
+
prompt.content
|
|
131
|
+
.flatMap((message) =>
|
|
132
|
+
message.role === "user"
|
|
133
|
+
? message.content.flatMap((part) =>
|
|
134
|
+
part.type === "text" ? [part.text] : [],
|
|
135
|
+
)
|
|
136
|
+
: [],
|
|
137
|
+
)
|
|
138
|
+
.join("\n")
|
|
139
|
+
|
|
140
|
+
const omittedNote = /\[image: shot\.png omitted\]/
|
|
141
|
+
|
|
142
|
+
/** Mirrors the OpenAI rejection for image input on a text-only model. */
|
|
143
|
+
const unsupportedImageError = AiError.make({
|
|
144
|
+
module: "OpenAiLanguageModel",
|
|
145
|
+
method: "streamText",
|
|
146
|
+
reason: new AiError.InvalidRequestError({
|
|
147
|
+
description:
|
|
148
|
+
"Invalid content type. image_url is only supported by certain models. (POST https://api.openai.com/v1/responses) [code: invalid_value]",
|
|
149
|
+
}),
|
|
150
|
+
})
|
|
151
|
+
|
|
152
|
+
const unrelatedError = AiError.make({
|
|
153
|
+
module: "OpenAiLanguageModel",
|
|
154
|
+
method: "streamText",
|
|
155
|
+
reason: new AiError.InvalidRequestError({
|
|
156
|
+
description:
|
|
157
|
+
"Unsupported parameter: 'temperature' is not supported with this model. (POST https://api.openai.com/v1/responses)",
|
|
158
|
+
}),
|
|
159
|
+
})
|
|
160
|
+
|
|
161
|
+
describe("Agent images", () => {
|
|
162
|
+
it.effect(
|
|
163
|
+
"strips images and leaves an omitted note for a model known not to take images",
|
|
164
|
+
() =>
|
|
165
|
+
Effect.gen(function* () {
|
|
166
|
+
const { exit, calls } = yield* runAgent({
|
|
167
|
+
prompt: imagePrompt,
|
|
168
|
+
modelConfig: { supportsImages: false },
|
|
169
|
+
respond: () => Stream.fromIterable(text("I cannot see images.")),
|
|
170
|
+
})
|
|
171
|
+
assert.isTrue(Exit.isSuccess(exit))
|
|
172
|
+
assert.strictEqual(calls.length, 1)
|
|
173
|
+
assert.strictEqual(fileParts(calls[0]!.prompt).length, 0)
|
|
174
|
+
const prompt = userText(calls[0]!.prompt)
|
|
175
|
+
assert.include(prompt, "what is in this screenshot?")
|
|
176
|
+
assert.match(prompt, omittedNote)
|
|
177
|
+
}),
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
it.effect(
|
|
181
|
+
"retries without images and discards attachments from the failed attempt",
|
|
182
|
+
() =>
|
|
183
|
+
Effect.gen(function* () {
|
|
184
|
+
const attached = yield* Deferred.make<void>()
|
|
185
|
+
let executions = 0
|
|
186
|
+
const executor = makeExecutor(({ onImage }) =>
|
|
187
|
+
Stream.fromEffect(
|
|
188
|
+
Effect.gen(function* () {
|
|
189
|
+
executions++
|
|
190
|
+
yield* onImage({
|
|
191
|
+
data: tinyPng,
|
|
192
|
+
mediaType: "image/png",
|
|
193
|
+
fileName: executions === 1 ? "discarded.png" : "shot.png",
|
|
194
|
+
})
|
|
195
|
+
yield* Deferred.succeed(attached, undefined)
|
|
196
|
+
return "Image attached"
|
|
197
|
+
}),
|
|
198
|
+
),
|
|
199
|
+
)
|
|
200
|
+
const { exit, calls, history } = yield* runAgent({
|
|
201
|
+
prompt: imagePrompt,
|
|
202
|
+
executor,
|
|
203
|
+
respond: (_, index) =>
|
|
204
|
+
index === 0
|
|
205
|
+
? Stream.make(
|
|
206
|
+
toolCall("discarded", 'await readFile({ path: "shot.png" })'),
|
|
207
|
+
).pipe(
|
|
208
|
+
// Advance past the tool-call chunk so its handler starts.
|
|
209
|
+
Stream.concat(
|
|
210
|
+
Stream.make({ type: "text-start", id: "failed" } as const),
|
|
211
|
+
),
|
|
212
|
+
Stream.concat(
|
|
213
|
+
Stream.fromEffect(
|
|
214
|
+
Deferred.await(attached).pipe(
|
|
215
|
+
Effect.andThen(Effect.fail(unsupportedImageError)),
|
|
216
|
+
),
|
|
217
|
+
),
|
|
218
|
+
),
|
|
219
|
+
)
|
|
220
|
+
: index === 1
|
|
221
|
+
? Stream.make(
|
|
222
|
+
toolCall("retried", 'await readFile({ path: "shot.png" })'),
|
|
223
|
+
)
|
|
224
|
+
: Stream.fromIterable(text("Text only, but I tried.")),
|
|
225
|
+
})
|
|
226
|
+
assert.isTrue(Exit.isSuccess(exit))
|
|
227
|
+
assert.strictEqual(calls.length, 3)
|
|
228
|
+
assert.strictEqual(executions, 2)
|
|
229
|
+
assert.deepStrictEqual(
|
|
230
|
+
fileParts(history).map((part) => part.fileName),
|
|
231
|
+
["shot.png", "shot.png"],
|
|
232
|
+
)
|
|
233
|
+
assert.strictEqual(fileParts(calls[0]!.prompt).length, 1)
|
|
234
|
+
assert.strictEqual(fileParts(calls[1]!.prompt).length, 0)
|
|
235
|
+
const retried = userText(calls[1]!.prompt)
|
|
236
|
+
assert.include(retried, "what is in this screenshot?")
|
|
237
|
+
assert.match(retried, omittedNote)
|
|
238
|
+
}),
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
it.effect("does not retry when the provider rejects it a second time", () =>
|
|
242
|
+
Effect.gen(function* () {
|
|
243
|
+
const { exit, calls } = yield* runAgent({
|
|
244
|
+
prompt: imagePrompt,
|
|
245
|
+
respond: () => Stream.fail(unsupportedImageError),
|
|
246
|
+
})
|
|
247
|
+
assert.isTrue(Exit.isFailure(exit))
|
|
248
|
+
assert.strictEqual(calls.length, 2)
|
|
249
|
+
}),
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
it.effect(
|
|
253
|
+
"does not treat other invalid-request errors as image rejections",
|
|
254
|
+
() =>
|
|
255
|
+
Effect.gen(function* () {
|
|
256
|
+
const { exit, calls } = yield* runAgent({
|
|
257
|
+
prompt: imagePrompt,
|
|
258
|
+
respond: () => Stream.fail(unrelatedError),
|
|
259
|
+
})
|
|
260
|
+
assert.isTrue(Exit.isFailure(exit))
|
|
261
|
+
assert.strictEqual(calls.length, 1)
|
|
262
|
+
}),
|
|
263
|
+
)
|
|
264
|
+
|
|
265
|
+
it.effect("does not copy image bytes into a delegate prompt", () =>
|
|
266
|
+
Effect.gen(function* () {
|
|
267
|
+
const executor = makeExecutor(({ onSubagent }) =>
|
|
268
|
+
Stream.fromEffect(onSubagent("child task: describe the layout")),
|
|
269
|
+
)
|
|
270
|
+
const { exit, calls } = yield* runAgent({
|
|
271
|
+
prompt: imagePrompt,
|
|
272
|
+
executor,
|
|
273
|
+
respond: (call, index) => {
|
|
274
|
+
if (index === 0) {
|
|
275
|
+
return Stream.make(
|
|
276
|
+
toolCall("call-1", 'await delegate("describe the layout")'),
|
|
277
|
+
)
|
|
278
|
+
}
|
|
279
|
+
const isChild = userText(call.prompt).includes("child task")
|
|
280
|
+
return Stream.fromIterable(
|
|
281
|
+
text(isChild ? "child done" : "parent done"),
|
|
282
|
+
)
|
|
283
|
+
},
|
|
284
|
+
})
|
|
285
|
+
assert.isTrue(Exit.isSuccess(exit))
|
|
286
|
+
const child = calls.find((call) =>
|
|
287
|
+
userText(call.prompt).includes("child task"),
|
|
288
|
+
)
|
|
289
|
+
assert.isDefined(child, "the delegate call reached the model")
|
|
290
|
+
assert.strictEqual(fileParts(child!.prompt).length, 0)
|
|
291
|
+
// The parent still has its image.
|
|
292
|
+
assert.strictEqual(fileParts(calls[0]!.prompt).length, 1)
|
|
293
|
+
}),
|
|
294
|
+
)
|
|
295
|
+
})
|