@neutrome/lilsdk 0.4.0 → 0.4.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@neutrome/lilsdk",
-  "version": "0.4.0",
+  "version": "0.4.2",
   "type": "module",
   "exports": {
     ".": "./src/index.ts",
@@ -13,7 +13,7 @@
     "./managed": "./src/managed/index.ts"
   },
   "dependencies": {
-    "@neutrome/lil-engine": "0.4.0"
+    "@neutrome/lil-engine": "0.4.1"
   },
   "devDependencies": {
     "@types/node": "^25.9.3",

package/src/managed/attachment-to-text.ts

@@ -0,0 +1,176 @@
+import {
+  contentText,
+  insertAfter,
+  isAttachmentOpcode,
+  messages,
+  programAttachments,
+  type Program,
+  type ProgramAttachment,
+} from "@neutrome/lil-engine";
+import {
+  appendToolInteraction,
+  prependSystemPrompt,
+} from "../synthetic/index.ts";
+import type { Executor, ExecutorContext, ExecutorInput } from "../types.ts";
+
+const toolName = "attachment_to_text";
+const defaultPrompt =
+  "Read the attached file carefully. Return a detailed factual description of its content for another assistant to use. Include all relevant visible text, structure, and details. Do not discuss this instruction.";
+
+export type AttachmentReaderRule = {
+  mimeTypes: readonly string[];
+  executor: ExecutorInput;
+};
+
+export type AttachmentToTextOptions = {
+  systemPrompt?: string;
+  cacheKey?: (attachment: ProgramAttachment, ctx: ExecutorContext) => string;
+};
+
+export function createAttachmentToTextExecutor(
+  inner: ExecutorInput,
+  rules: readonly AttachmentReaderRule[],
+  options: AttachmentToTextOptions = {},
+): Executor {
+  if (rules.length === 0)
+    throw new Error("attachment-to-text requires at least one reader rule");
+
+  const prompt = options.systemPrompt ?? defaultPrompt;
+  return {
+    async execute(request, ctx) {
+      return ctx.invoke(
+        inner,
+        await prepareRequest(request, ctx, rules, prompt, options),
+      );
+    },
+    async *stream(request, ctx) {
+      yield* ctx.invokeStream(
+        inner,
+        await prepareRequest(request, ctx, rules, prompt, options),
+      );
+    },
+  };
+}
+
+async function prepareRequest(
+  request: Program,
+  ctx: ExecutorContext,
+  rules: readonly AttachmentReaderRule[],
+  prompt: string,
+  options: AttachmentToTextOptions,
+): Promise<Program> {
+  const attachments = programAttachments(request);
+  if (attachments.length === 0) return request;
+  const latestUserStart = [...attachments]
+    .filter((attachment) => attachment.message.role === "user")
+    .at(-1)?.message.start;
+  let result = stripAttachments(request);
+  const sourceMessages = messages(request);
+
+  for (const attachment of [...attachments].reverse()) {
+    const rule = rules.find((candidate) =>
+      candidate.mimeTypes.some((mime) =>
+        matchesMime(mime, attachment.mimeType),
+      ),
+    );
+    if (!rule) {
+      if (attachment.message.start === latestUserStart) {
+        throw new Error(`No attachment reader matches ${attachment.mimeType}`);
+      }
+      throw new Error(
+        `Missing cached attachment description for historical ${attachment.mimeType}`,
+      );
+    }
+    const key = await attachmentCacheKey(attachment, ctx, options);
+    let description = await ctx.cache.get(key);
+    if (!description) {
+      if (attachment.message.start !== latestUserStart) {
+        throw new Error(
+          `Missing cached attachment description for historical ${attachment.mimeType}`,
+        );
+      }
+      const readerRequest = readerProgram(request, attachment, prompt);
+      description = contentText(await ctx.invoke(rule.executor, readerRequest));
+      if (!description.trim())
+        throw new Error(
+          `Attachment reader returned no text for ${attachment.mimeType}`,
+        );
+      await ctx.cache.put(key, description);
+    }
+    const interaction = {
+      callId: `attachment_${key.slice(-24)}`,
+      name: toolName,
+      args: {
+        mimeType: attachment.mimeType,
+        ...(attachment.name ? { name: attachment.name } : {}),
+        context:
+          "The user attached this file. Its binary media was replaced with the reconstructed text result below.",
+      },
+      result: `Reconstructed content for the file the user attached. The original binary media is not present in this context.\n\n${description}`,
+    };
+    const messageIndex = sourceMessages.findIndex(
+      (message) => message.start === attachment.message.start,
+    );
+    const targetMessage = messages(result)[messageIndex];
+    if (!targetMessage)
+      throw new Error("Attachment message was removed from context");
+    const synthetic = appendToolInteraction(
+      { code: [], buffers: [] },
+      interaction,
+    );
+    result = insertAfter(result, targetMessage.end, synthetic.code);
+  }
+  return result;
+}
+
+function readerProgram(
+  request: Program,
+  attachment: ProgramAttachment,
+  prompt: string,
+): Program {
+  return prependSystemPrompt(
+    {
+      code: request.code.slice(
+        attachment.message.start,
+        attachment.message.end + 1,
+      ),
+      buffers: request.buffers.map((buffer) => buffer.slice()),
+    },
+    prompt,
+  );
+}
+
+function stripAttachments(request: Program): Program {
+  return {
+    code: request.code.filter(
+      (instruction) => !isAttachmentOpcode(instruction.opcode),
+    ),
+    buffers: request.buffers.map((buffer) => buffer.slice()),
+  };
+}
+
+async function attachmentCacheKey(
+  attachment: ProgramAttachment,
+  ctx: ExecutorContext,
+  options: AttachmentToTextOptions,
+): Promise<string> {
+  if (options.cacheKey)
+    return `attachment-to-text:${options.cacheKey(attachment, ctx)}`;
+  const bytes = new TextEncoder().encode(
+    JSON.stringify({
+      mimeType: attachment.mimeType,
+      url: attachment.url,
+      name: attachment.name,
+    }),
+  );
+  const hash = await crypto.subtle.digest("SHA-256", bytes);
+  return `attachment-to-text:${Array.from(new Uint8Array(hash), (value) => value.toString(16).padStart(2, "0")).join("")}`;
+}
+
+function matchesMime(pattern: string, mimeType: string): boolean {
+  return (
+    pattern === "*" ||
+    pattern === mimeType ||
+    (pattern.endsWith("/*") && mimeType.startsWith(pattern.slice(0, -1)))
+  );
+}

package/src/managed/index.ts

@@ -1,5 +1,10 @@
 export { createCapabilitiesExecutor } from "./capabilities.ts";
 export type { CapabilitiesExecutorOptions } from "./capabilities.ts";
+export { createAttachmentToTextExecutor } from "./attachment-to-text.ts";
+export type {
+  AttachmentReaderRule,
+  AttachmentToTextOptions,
+} from "./attachment-to-text.ts";
 
 export {
   appendInternalDraft,

package/test/lilsdk-ts.test.ts

@@ -7,6 +7,8 @@ import {
   getModel,
   parseChatCompletionsRequest,
   parseChatCompletionsStreamChunk,
+  programAttachments,
+  viewProgram,
   setModel,
   Opcode,
   type Program,
@@ -36,12 +38,64 @@ import {
   createTwoPassExecutor,
   INTERNAL_DRAFT_TOOL_NAME,
 } from "../src/managed/two-pass.ts";
+import { createAttachmentToTextExecutor } from "../src/managed/attachment-to-text.ts";
 import type { Executor, ExecutorContext, OutputSink } from "../src/types.ts";
 
 const encoder = new TextEncoder();
 const decoder = new TextDecoder();
 
 describe("@neutrome/lilsdk", () => {
+  it("describes the latest attachment and removes it before the inner executor", async () => {
+    const request = parseChatCompletionsRequest(
+      encoder.encode(
+        JSON.stringify({
+          messages: [
+            {
+              role: "user",
+              content: [
+                { type: "text", text: "What is this?" },
+                {
+                  type: "image_url",
+                  image_url: { url: "data:image/png;base64,aGVsbG8=" },
+                },
+              ],
+            },
+          ],
+        }),
+      ),
+    );
+    let innerRequest: Program | undefined;
+    const executor = createAttachmentToTextExecutor(
+      {
+        async execute(value) {
+          innerRequest = value;
+          return value;
+        },
+        async *stream(value) {
+          yield value;
+        },
+      },
+      [
+        {
+          mimeTypes: ["*"],
+          executor: {
+            async execute() {
+              return appendAssistantMessage(
+                { code: [], buffers: [] },
+                "A tiny image.",
+              );
+            },
+            async *stream() {},
+          },
+        },
+      ],
+    );
+    await executor.execute(request, buildExecutorContext());
+    expect(programAttachments(innerRequest!)).toHaveLength(0);
+    expect(
+      viewProgram(innerRequest!).messages.at(-1)?.toolResult?.text,
+    ).toContain("The original binary media is not present in this context.");
+  });
   it("provides clone-safe structural primitives", () => {
     const needle = {
       opcode: Opcode.MSG_START,