clanka 0.5.2 → 0.6.0

package/src/AgentTools.ts

@@ -21,7 +21,9 @@ import { pipe } from "effect/Function"
 import * as Array from "effect/Array"
 import * as Data from "effect/Data"
 import * as Layer from "effect/Layer"
+import * as Option from "effect/Option"
 import * as SemanticSearch from "./SemanticSearch/Service.ts"
+import * as Image from "./Image.ts"
 
 /**
  * @since 1.0.0
@@ -59,6 +61,28 @@ export class SubagentExecutor extends Context.Service<
   (prompt: string) => Effect.Effect<string>
 >()("clanka/AgentTools/SubagentExecutor") {}
 
+/**
+ * An image read by `readFile`, handed to the Agent so it can be spliced onto
+ * the Prompt for the next model turn.
+ *
+ * @since 1.0.0
+ * @category Models
+ */
+export interface ImageAttachment {
+  readonly fileName: string
+  readonly mediaType: Image.ImageMediaType
+  readonly data: Uint8Array
+}
+
+/**
+ * @since 1.0.0
+ * @category Context
+ */
+export class ImageAttacher extends Context.Service<
+  ImageAttacher,
+  (image: ImageAttachment) => Effect.Effect<void>
+>()("clanka/AgentTools/ImageAttacher") {}
+
 /**
  * @since 1.0.0
  * @category Context
@@ -68,6 +92,7 @@ export const makeContextNoop = (cwd?: string) =>
     Context.add(CurrentDirectory, cwd ?? "/"),
     Context.add(DirectoryChanger, () => Effect.die("Not implemented")),
     Context.add(TaskCompleter, () => Effect.void),
+    Context.add(ImageAttacher, () => Effect.void),
   )
 
 class TodoItem extends Schema.Opaque<TodoItem>()(
@@ -92,14 +117,14 @@ export const AgentTools = Toolkit.make(
   }),
   Tool.make("readFile", {
     description:
-      "Read a file and optionally filter the lines to return. Returns null if the file doesn't exist.",
+      "Read a file and optionally filter the lines to return. Returns null if the file doesn't exist. For png/jpeg/gif/webp files the image itself is attached to your next turn and a short marker is returned instead of the bytes; line ranges are not allowed on images.",
     parameters: Schema.Struct({
       path: Schema.String,
       startLine: Schema.optional(Schema.Number),
       endLine: Schema.optional(Schema.Number),
     }),
     success: Schema.NullOr(Schema.String),
-    dependencies: [CurrentDirectory],
+    dependencies: [CurrentDirectory, ImageAttacher],
   }),
   Tool.make("rg", {
     description: "Search for a pattern in files using ripgrep",
@@ -300,6 +325,51 @@ export const AgentToolHandlersNoDeps = AgentToolsWithSearch.toLayer(
       )
     }, Effect.scoped)
 
+    /**
+     * Image branch of `readFile`: the bytes go to the `ImageAttacher` for
+     * the next model turn and the script only sees a marker.
+     */
+    const readImage = Effect.fnUntraced(function* (
+      path: string,
+      extensionType: Image.ImageMediaType,
+      options: {
+        readonly startLine?: number | undefined
+        readonly endLine?: number | undefined
+      },
+    ) {
+      if (options.startLine !== undefined || options.endLine !== undefined) {
+        return yield* Effect.die(
+          new Error(
+            `startLine / endLine are not supported for images: ${path}`,
+          ),
+        )
+      }
+      const bytes = yield* Image.readBoundedFile(fs, path).pipe(
+        Effect.catchReason("PlatformError", "NotFound", () =>
+          Effect.succeed(null),
+        ),
+        Effect.orDie,
+      )
+      if (bytes === null) return null
+      // The extension picks the branch; the magic bytes pick the real type.
+      const mediaType = Option.getOrElse(
+        Image.mediaTypeFromBytes(bytes),
+        () => extensionType,
+      )
+      const prepared = yield* Image.prepare({ data: bytes, mediaType }).pipe(
+        Effect.mapError((error) => new Error(`${error.message} (${path})`)),
+        Effect.orDie,
+      )
+      const fileName = pathService.basename(path)
+      const attach = yield* ImageAttacher
+      yield* attach({
+        fileName,
+        mediaType: prepared.mediaType,
+        data: prepared.data,
+      })
+      return `Image attached: ${fileName}`
+    })
+
     return AgentToolsWithSearch.of({
       changeDirectory: Effect.fn("AgentTools.changeDirectory")(
         function* (directory) {
@@ -315,11 +385,12 @@ export const AgentToolHandlersNoDeps = AgentToolsWithSearch.toLayer(
           Effect.annotateLogs(options),
         )
         const cwd = yield* CurrentDirectory
-        let stream = pipe(
-          fs.stream(pathService.resolve(cwd, options.path)),
-          Stream.decodeText,
-          Stream.splitLines,
-        )
+        const path = pathService.resolve(cwd, options.path)
+        const imageType = Image.mediaTypeFromPath(path)
+        if (Option.isSome(imageType)) {
+          return yield* readImage(path, imageType.value, options)
+        }
+        let stream = pipe(fs.stream(path), Stream.decodeText, Stream.splitLines)
         if (options.startLine) {
           stream = Stream.drop(stream, options.startLine - 1)
         }

package/src/AgentToolsImage.test.ts

@@ -0,0 +1,302 @@
+import * as Photon from "@silvia-odwyer/photon-node"
+import { assert, describe, it } from "@effect/vitest"
+import { vi } from "vitest"
+import * as NodeHttpClient from "@effect/platform-node/NodeHttpClient"
+import * as NodeServices from "@effect/platform-node/NodeServices"
+import * as Duration from "effect/Duration"
+import * as Effect from "effect/Effect"
+import * as Exit from "effect/Exit"
+import * as FileSystem from "effect/FileSystem"
+import * as Layer from "effect/Layer"
+import * as Path from "effect/Path"
+import * as Stream from "effect/Stream"
+import * as LanguageModel from "effect/unstable/ai/LanguageModel"
+import * as Model from "effect/unstable/ai/Model"
+import type * as Prompt from "effect/unstable/ai/Prompt"
+import type * as Response from "effect/unstable/ai/Response"
+import * as Agent from "./Agent.ts"
+import * as AgentExecutor from "./AgentExecutor.ts"
+import type * as AgentOutput from "./AgentOutput.ts"
+import {
+  bytesOf,
+  dimensions,
+  encodePng,
+  equalBytes,
+  tinyPng,
+} from "./fixtures/TestImages.ts"
+
+const svgText =
+  '<svg xmlns="http://www.w3.org/2000/svg"><rect width="1" height="1"/></svg>'
+
+/**
+ * A temp project directory holding the image fixtures, with a real local
+ * executor rooted in it.
+ */
+const withProject = <A, E, R>(f: (project: string) => Effect.Effect<A, E, R>) =>
+  Effect.gen(function* () {
+    const fs = yield* FileSystem.FileSystem
+    const path = yield* Path.Path
+    const project = yield* fs.makeTempDirectoryScoped()
+    yield* fs.writeFile(path.join(project, "shot.png"), tinyPng)
+    yield* fs.writeFile(
+      path.join(project, "wide.png"),
+      encodePng({ width: 2500, height: 100 }),
+    )
+    yield* fs.writeFileString(path.join(project, "logo.svg"), svgText)
+    yield* fs.writeFile(
+      path.join(project, "broken.png"),
+      new TextEncoder().encode("this is not a png".repeat(20)),
+    )
+    return yield* f(project).pipe(
+      Effect.provide(
+        AgentExecutor.layerLocal({ directory: project }).pipe(
+          Layer.provide(NodeHttpClient.layerUndici),
+        ),
+      ),
+    )
+  }).pipe(Effect.provide(NodeServices.layer), Effect.scoped)
+
+const readFile = (params: {
+  readonly path: string
+  readonly startLine?: number
+  readonly endLine?: number
+}) =>
+  Effect.gen(function* () {
+    const executor = yield* AgentExecutor.AgentExecutor
+    return yield* executor.executeUnsafe({ tool: "readFile", params })
+  })
+
+describe("readFile images", () => {
+  it.effect("rejects startLine / endLine on an image", () =>
+    withProject(() =>
+      Effect.gen(function* () {
+        const withStart = yield* Effect.exit(
+          readFile({ path: "shot.png", startLine: 1 }),
+        )
+        assert.isTrue(Exit.isFailure(withStart))
+        const withEnd = yield* Effect.exit(
+          readFile({ path: "shot.png", endLine: 5 }),
+        )
+        assert.isTrue(Exit.isFailure(withEnd))
+      }),
+    ),
+  )
+
+  it.effect(
+    "rejects undecodable and over-ceiling images without decoding them",
+    () =>
+      withProject((project) =>
+        Effect.gen(function* () {
+          const fs = yield* FileSystem.FileSystem
+          const path = yield* Path.Path
+          // Sparse file one byte over the shared 20 MiB input ceiling.
+          const huge = path.join(project, "huge.png")
+          yield* fs.writeFile(huge, tinyPng)
+          yield* fs.truncate(huge, 20 * 1024 * 1024 + 1)
+
+          const decoder = vi.spyOn(Photon.PhotonImage, "new_from_byteslice")
+          try {
+            const broken = yield* Effect.exit(readFile({ path: "broken.png" }))
+            assert.isTrue(Exit.isFailure(broken))
+            const oversized = yield* Effect.exit(readFile({ path: "huge.png" }))
+            assert.isTrue(Exit.isFailure(oversized))
+            assert.strictEqual(
+              decoder.mock.calls.length,
+              0,
+              "neither garbage nor an over-ceiling file may reach the decoder",
+            )
+          } finally {
+            decoder.mockRestore()
+          }
+        }),
+      ),
+  )
+})
+
+// =============================================================================
+// Injection into the next model turn, through the real Agent loop
+// =============================================================================
+
+interface RecordedCall {
+  readonly prompt: Prompt.Prompt
+}
+
+const runAgent = (options: {
+  readonly respond: (
+    call: RecordedCall,
+    index: number,
+  ) => Stream.Stream<Response.StreamPartEncoded>
+}) =>
+  Effect.gen(function* () {
+    const calls: Array<RecordedCall> = []
+    const outputs: Array<AgentOutput.Output> = []
+    const languageModel = yield* LanguageModel.make({
+      generateText: () => Effect.succeed([]),
+      streamText: (providerOptions) => {
+        const call = { prompt: providerOptions.prompt }
+        const index = calls.push(call) - 1
+        return options.respond(call, index)
+      },
+    })
+    const modelLayer = Layer.mergeAll(
+      Layer.succeed(LanguageModel.LanguageModel, languageModel),
+      Layer.succeed(Model.ProviderName, "test-provider"),
+      Layer.succeed(Model.ModelName, "test-model"),
+    )
+    const agent = yield* Agent.make
+    const exit = yield* agent.send({ prompt: "read the screenshot" }).pipe(
+      Effect.flatMap((stream) =>
+        stream.pipe(
+          Stream.runForEach((output) =>
+            Effect.sync(() => {
+              outputs.push(output)
+            }),
+          ),
+          Effect.as(""),
+          Effect.catchTag("AgentFinished", (finished) =>
+            Effect.succeed(finished.summary),
+          ),
+        ),
+      ),
+      Effect.provide(
+        Layer.mergeAll(
+          modelLayer,
+          Agent.ConversationMode.layer(true),
+          Agent.layerSubagentModel(modelLayer),
+          Layer.succeed(Agent.TurnTimeout, Duration.minutes(5)),
+        ),
+      ),
+      Effect.exit,
+    )
+    return { exit, calls, outputs }
+  }).pipe(Effect.scoped)
+
+const toolCall = (id: string, script: string): Response.StreamPartEncoded => ({
+  type: "tool-call",
+  id,
+  name: "execute",
+  params: { script },
+})
+
+const text = (content: string): Array<Response.StreamPartEncoded> => [
+  { type: "text-start", id: "t" },
+  { type: "text-delta", id: "t", delta: content },
+  { type: "text-end", id: "t" },
+]
+
+const toolResultText = (message: Prompt.Message): string | undefined => {
+  if (message.role !== "tool") return undefined
+  const part = message.content.find((p) => p.type === "tool-result")
+  return part && typeof part.result === "string" ? part.result : undefined
+}
+
+const fileParts = (message: Prompt.Message): Array<Prompt.FilePart> =>
+  message.role === "user"
+    ? message.content.filter((part) => part.type === "file")
+    : []
+
+describe("readFile image injection", () => {
+  it.effect("injects an image as a file part after the execute result", () =>
+    withProject(() =>
+      Effect.gen(function* () {
+        const { exit, calls, outputs } = yield* runAgent({
+          respond: (_, index) =>
+            index === 0
+              ? Stream.make(
+                  toolCall(
+                    "call-1",
+                    'console.log(await readFile({ path: "shot.png" }))',
+                  ),
+                )
+              : Stream.fromIterable(text("It is a red rectangle.")),
+        })
+        assert.isTrue(Exit.isSuccess(exit))
+        assert.strictEqual(calls.length, 2)
+
+        const messages = calls[1]!.prompt.content
+        const resultIndex = messages.findIndex((m) =>
+          toolResultText(m)?.includes("Image attached: shot.png"),
+        )
+        assert.notStrictEqual(resultIndex, -1, "script saw the marker")
+
+        const image = messages
+          .slice(resultIndex + 1)
+          .flatMap(fileParts)
+          .find((part) => part.mediaType === "image/png")
+        assert.isDefined(image, "a file part follows the tool result")
+        assert.strictEqual(
+          calls[1]!.prompt.content.flatMap(fileParts).length,
+          1,
+        )
+        assert.strictEqual(image!.fileName, "shot.png")
+        assert.isTrue(equalBytes(bytesOf(image!.data), tinyPng))
+
+        // The first model call had no image: injection happens after the
+        // script runs, not before.
+        assert.strictEqual(
+          calls[0]!.prompt.content.flatMap(fileParts).length,
+          0,
+        )
+
+        // The script output shown to the client is the marker, not bytes.
+        const scriptOutput = outputs.find(
+          (o): o is AgentOutput.ScriptOutput => o._tag === "ScriptOutput",
+        )
+        assert.isDefined(scriptOutput)
+        assert.include(scriptOutput!.output, "Image attached: shot.png")
+        assert.isBelow(scriptOutput!.output.length, 200)
+      }),
+    ),
+  )
+
+  it.effect("applies the image limits to readFile images", () =>
+    withProject(() =>
+      Effect.gen(function* () {
+        const { exit, calls } = yield* runAgent({
+          respond: (_, index) =>
+            index === 0
+              ? Stream.make(
+                  toolCall(
+                    "call-1",
+                    'console.log(await readFile({ path: "wide.png" }))',
+                  ),
+                )
+              : Stream.fromIterable(text("done")),
+        })
+        assert.isTrue(Exit.isSuccess(exit))
+        const images = calls[1]!.prompt.content.flatMap(fileParts)
+        assert.strictEqual(images.length, 1)
+        const size = dimensions(bytesOf(images[0]!.data))
+        assert.isAtMost(size.width, 2000)
+        assert.isAtMost(size.height, 2000)
+      }),
+    ),
+  )
+
+  it.effect("reads SVG as text without injecting an image", () =>
+    withProject(() =>
+      Effect.gen(function* () {
+        const { calls } = yield* runAgent({
+          respond: (_, index) =>
+            index === 0
+              ? Stream.make(
+                  toolCall(
+                    "call-1",
+                    'console.log(await readFile({ path: "logo.svg" }))',
+                  ),
+                )
+              : Stream.fromIterable(text("done")),
+        })
+        assert.strictEqual(calls.length, 2)
+        assert.strictEqual(
+          calls[1]!.prompt.content.flatMap(fileParts).length,
+          0,
+        )
+        const result = calls[1]!.prompt.content
+          .map(toolResultText)
+          .find((r) => r !== undefined)
+        assert.include(result, "<svg")
+      }),
+    ),
+  )
+})

package/src/Codex.ts

@@ -67,7 +67,11 @@ const layerModel = (
   OpenAiLanguageModel.layer({
     model,
     config: {
-      ...Struct.omit(options ?? {}, ["reasoning", "systemPromptTransform"]),
+      ...Struct.omit(options ?? {}, [
+        "reasoning",
+        "systemPromptTransform",
+        "supportsImages",
+      ]),
       store: false,
       reasoning: {
         effort: "medium",
@@ -82,6 +86,7 @@ const layerModel = (
           OpenAiLanguageModel.withConfigOverride(effect, {
             instructions: system,
           }),
+        supportsImages: options?.supportsImages,
       }),
       // No Compaction.SummarizerTransform: Codex rejects max_output_tokens,
       // so compaction summaries have no configured output-token cap.

package/src/Compaction.ts

@@ -23,6 +23,7 @@ import * as AiError from "effect/unstable/ai/AiError"
 import * as LanguageModel from "effect/unstable/ai/LanguageModel"
 import * as Prompt from "effect/unstable/ai/Prompt"
 import type * as AgentOutput from "./AgentOutput.ts"
+import * as Image from "./Image.ts"
 
 // =============================================================================
 // Configuration
@@ -531,8 +532,22 @@ const renderMessage = (message: Prompt.Message): string => {
     case "system":
       return ""
     case "user": {
+      // Image bytes never reach the summarizer: stub them in place.
       const text = message.content
-        .flatMap((part) => (part.type === "text" ? [part.text] : []))
+        .flatMap((part) => {
+          switch (part.type) {
+            case "text":
+              return [part.text]
+            case "file":
+              return [
+                part.mediaType.startsWith("image/")
+                  ? Image.omittedText(part.fileName)
+                  : `[file: ${part.fileName ?? part.mediaType} omitted]`,
+              ]
+            default:
+              return []
+          }
+        })
         .join("\n")
       return `[user]\n${text}`
     }

package/src/CompactionImage.test.ts

@@ -0,0 +1,142 @@
+import { assert, describe, it } from "@effect/vitest"
+import * as Effect from "effect/Effect"
+import * as Encoding from "effect/Encoding"
+import * as Layer from "effect/Layer"
+import * as Option from "effect/Option"
+import * as Stream from "effect/Stream"
+import * as LanguageModel from "effect/unstable/ai/LanguageModel"
+import * as Prompt from "effect/unstable/ai/Prompt"
+import * as Compaction from "./Compaction.ts"
+import { tinyPng } from "./fixtures/TestImages.ts"
+
+const base64 = Encoding.encodeBase64(tinyPng)
+
+const user = (text: string) =>
+  Prompt.makeMessage("user", {
+    content: [Prompt.makePart("text", { text })],
+  })
+
+const userWithImage = (text: string, fileName: string, data: Uint8Array) =>
+  Prompt.makeMessage("user", {
+    content: [
+      Prompt.makePart("text", { text }),
+      Prompt.makePart("file", { mediaType: "image/png", fileName, data }),
+    ],
+  })
+
+const assistantText = (text: string) =>
+  Prompt.makeMessage("assistant", {
+    content: [Prompt.makePart("text", { text })],
+  })
+
+const system = Prompt.makeMessage("system", { content: "You are a test." })
+
+const promptJson = (prompt: Prompt.Prompt): string =>
+  JSON.stringify(prompt.content)
+
+const hasFileParts = (prompt: Prompt.Prompt): boolean =>
+  prompt.content.some(
+    (message) =>
+      message.role === "user" &&
+      message.content.some((part) => part.type === "file"),
+  )
+
+describe("Compaction.compact with images", () => {
+  it.effect("never sends image parts to the summarizer", () =>
+    Effect.gen(function* () {
+      const summarizerPrompts: Array<Prompt.Prompt> = []
+      const languageModel = yield* LanguageModel.make({
+        generateText: () => Effect.succeed([]),
+        streamText: (options) => {
+          summarizerPrompts.push(options.prompt)
+          return Stream.fromIterable([
+            { type: "text-start", id: "s" },
+            { type: "text-delta", id: "s", delta: "SUMMARY" },
+            { type: "text-end", id: "s" },
+          ])
+        },
+      })
+      // Old image turn to summarize, then a fat text tail so the cut lands
+      // after the image.
+      const filler = "tail ".repeat(3_000)
+      const history = Prompt.fromMessages([
+        system,
+        userWithImage("look at this", "shot.png", tinyPng),
+        Prompt.makeMessage("user", {
+          content: [
+            Prompt.makePart("file", { mediaType: "image/png", data: base64 }),
+          ],
+        }),
+        assistantText("It is a red rectangle."),
+        user("now the next thing"),
+        assistantText(filler),
+        user("and again"),
+        assistantText(filler),
+      ])
+
+      const result = yield* Compaction.compact({
+        prompt: history,
+        reason: "threshold",
+      }).pipe(
+        Effect.provide(
+          Layer.mergeAll(
+            Layer.succeed(LanguageModel.LanguageModel, languageModel),
+            Compaction.CompactionConfig.layer({ keepRecentTokens: 2_000 }),
+          ),
+        ),
+      )
+      const compacted = Option.getOrThrow(result)
+      assert.strictEqual(summarizerPrompts.length, 1)
+      const sent = promptJson(summarizerPrompts[0]!)
+      assert.notInclude(sent, base64)
+      assert.isFalse(hasFileParts(summarizerPrompts[0]!))
+      assert.include(sent, "[image: shot.png omitted]")
+      assert.include(sent, "[image: unnamed omitted]")
+      assert.include(sent, "look at this")
+
+      // The rewritten prompt carries the summary and no stale image bytes
+      // from the summarized side.
+      const rewritten = promptJson(compacted.prompt)
+      assert.include(rewritten, "SUMMARY")
+      assert.notInclude(rewritten, base64)
+    }),
+  )
+
+  it.effect("keeps image parts in the kept tail", () =>
+    Effect.gen(function* () {
+      const languageModel = yield* LanguageModel.make({
+        generateText: () => Effect.succeed([]),
+        streamText: () =>
+          Stream.fromIterable([
+            { type: "text-start", id: "s" },
+            { type: "text-delta", id: "s", delta: "SUMMARY" },
+            { type: "text-end", id: "s" },
+          ]),
+      })
+      const filler = "old ".repeat(3_000)
+      const history = Prompt.fromMessages([
+        system,
+        user("start"),
+        assistantText(filler),
+        user("more"),
+        assistantText(filler),
+        // Recent image turn that must survive compaction verbatim.
+        userWithImage("latest screenshot", "latest.png", tinyPng),
+      ])
+      const result = yield* Compaction.compact({
+        prompt: history,
+        reason: "threshold",
+      }).pipe(
+        Effect.provide(
+          Layer.mergeAll(
+            Layer.succeed(LanguageModel.LanguageModel, languageModel),
+            Compaction.CompactionConfig.layer({ keepRecentTokens: 500 }),
+          ),
+        ),
+      )
+      const compacted = Option.getOrThrow(result)
+      assert.isTrue(hasFileParts(compacted.prompt))
+      assert.include(promptJson(compacted.prompt), "latest screenshot")
+    }),
+  )
+})

package/src/Copilot.ts

@@ -38,10 +38,14 @@ export const model = (
     Layer.mergeAll(
       OpenAiLanguageModel.layer({
         model,
-        config: Struct.omit(options ?? {}, ["systemPromptTransform"]),
+        config: Struct.omit(options ?? {}, [
+          "systemPromptTransform",
+          "supportsImages",
+        ]),
       }),
       AgentModelConfig.layer({
         systemPromptTransform: options?.systemPromptTransform,
+        supportsImages: options?.supportsImages,
       }),
       // Cap compaction summaries; Copilot honours max_output_tokens.
       Layer.succeed(Compaction.SummarizerTransform, (effect) =>