@neutrome/lilsdk 0.3.5

package/test/lilsdk-ts.test.ts

@@ -0,0 +1,660 @@
+import {
+  contentText,
+  emitChatCompletionsRequest,
+  emitChatCompletionsStreamChunk,
+  getModel,
+  parseChatCompletionsRequest,
+  parseChatCompletionsStreamChunk,
+  Opcode,
+  type Program,
+} from "@neutrome/lil-engine";
+import { describe, expect, it } from "vitest";
+import {
+  all,
+  any,
+  find,
+  has,
+  indexOf,
+  map,
+  reduce,
+} from "../src/primitives/index.ts";
+import {
+  appendAssistantMessage,
+  appendToolInteraction,
+} from "../src/synthetic/index.ts";
+import {
+  observeExecutionStream,
+  streamTextResponse,
+  writeReasoning,
+} from "../src/stream/index.ts";
+import { createGoalExecutor, fallback, retry } from "../src/loops/index.ts";
+import {
+  appendInternalDraft,
+  createTwoPassExecutor,
+  invokeExecutor,
+  INTERNAL_DRAFT_TOOL_NAME,
+  streamExecutor,
+} from "../src/managed/twoPassExecutor.ts";
+import type { Executor, ExecutorContext, OutputSink } from "../src/types.ts";
+
+const encoder = new TextEncoder();
+const decoder = new TextDecoder();
+
+describe("@neutrome/lilsdk", () => {
+  it("provides clone-safe structural primitives", () => {
+    const needle = {
+      opcode: Opcode.MSG_START,
+      value: { kind: "none" as const },
+    };
+    const program: Program = {
+      code: [
+        needle,
+        { opcode: Opcode.TXT_CHUNK, value: { kind: "string", value: "hello" } },
+      ],
+      buffers: [new Uint8Array([1, 2, 3])],
+    };
+
+    expect(indexOf(program, needle)).toBe(0);
+    expect(has(program, needle)).toBe(true);
+    expect(find(program, needle)).toEqual(needle);
+    expect(reduce(program, (count) => count + 1, 0)).toBe(2);
+    expect(
+      all(program, (instruction) => instruction.opcode !== Opcode.MSG_END),
+    ).toBe(true);
+    expect(
+      any(program, (instruction) => instruction.opcode === Opcode.TXT_CHUNK),
+    ).toBe(true);
+
+    const mapped = map(program, (instruction) => instruction);
+    mapped.buffers[0]![0] = 9;
+    expect(program.buffers[0]![0]).toBe(1);
+  });
+
+  it("observes tool-call streams", async () => {
+    const observed = await observeExecutionStream(
+      (async function* () {
+        yield parseChatCompletionsStreamChunk(
+          encoder.encode(
+            JSON.stringify({
+              id: "tool-stream",
+              object: "chat.completion.chunk",
+              model: "smart",
+              choices: [{ index: 0, delta: { role: "assistant" } }],
+            }),
+          ),
+        );
+        yield parseChatCompletionsStreamChunk(
+          encoder.encode(
+            JSON.stringify({
+              id: "tool-stream",
+              object: "chat.completion.chunk",
+              model: "smart",
+              choices: [
+                {
+                  index: 0,
+                  delta: {
+                    tool_calls: [
+                      {
+                        index: 0,
+                        id: "call_lookup",
+                        type: "function",
+                        function: { name: "lookup", arguments: '{"q":"kyiv"}' },
+                      },
+                    ],
+                  },
+                },
+              ],
+            }),
+          ),
+        );
+        yield parseChatCompletionsStreamChunk(
+          encoder.encode(
+            JSON.stringify({
+              id: "tool-stream",
+              object: "chat.completion.chunk",
+              model: "smart",
+              choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }],
+            }),
+          ),
+        );
+      })(),
+    );
+
+    expect(observed.mode).toBe("tool");
+    if (observed.mode !== "tool") {
+      throw new Error("expected tool mode");
+    }
+
+    const parsed = observed.chunks.map((chunk) =>
+      JSON.parse(decoder.decode(emitChatCompletionsStreamChunk(chunk))),
+    );
+    expect(parsed.some((chunk) => chunk.choices?.[0]?.delta?.tool_calls)).toBe(
+      true,
+    );
+  });
+
+  it("appends internal drafts with fixed SDK metadata and empty args", () => {
+    const request = parseChatCompletionsRequest(
+      encoder.encode(
+        JSON.stringify({
+          model: "virtual-model",
+          messages: [{ role: "user", content: "hello" }],
+        }),
+      ),
+    );
+
+    const updated = appendInternalDraft(request, "draft answer");
+    const emitted = JSON.parse(
+      decoder.decode(emitChatCompletionsRequest(updated)),
+    );
+
+    expect(emitted.messages[1].tool_calls[0].function.name).toBe(
+      INTERNAL_DRAFT_TOOL_NAME,
+    );
+    expect(emitted.messages[1].tool_calls[0].function.arguments).toBe("{}");
+    expect(emitted.messages[2].content).toBe("draft answer");
+  });
+
+  it("appends generic synthetic tool interactions", () => {
+    const request = parseChatCompletionsRequest(
+      encoder.encode(
+        JSON.stringify({
+          model: "virtual-model",
+          messages: [{ role: "user", content: "hello" }],
+        }),
+      ),
+    );
+
+    const updated = appendToolInteraction(request, {
+      callId: "call_lookup",
+      name: "lookup",
+      args: { id: "42" },
+      result: { status: "active" },
+    });
+    const emitted = JSON.parse(
+      decoder.decode(emitChatCompletionsRequest(updated)),
+    );
+
+    expect(emitted.messages[1].tool_calls[0].function).toEqual({
+      name: "lookup",
+      arguments: '{"id":"42"}',
+    });
+    expect(emitted.messages[2]).toEqual({
+      role: "tool",
+      tool_call_id: "call_lookup",
+      content: '{"status":"active"}',
+    });
+  });
+
+  it("builds two-pass executors from model and settings resolvers", async () => {
+    const request = parseChatCompletionsRequest(
+      encoder.encode(
+        JSON.stringify({
+          model: "virtual-model",
+          messages: [{ role: "user", content: "hello" }],
+        }),
+      ),
+    );
+    const seen: Program[] = [];
+    const ctx = buildExecutorContext({
+      async invoke(program) {
+        seen.push(program);
+        if (seen.length === 1) {
+          return appendAssistantMessage(
+            { code: [], buffers: [] },
+            "draft answer",
+          );
+        }
+        return appendAssistantMessage(
+          { code: [], buffers: [] },
+          "final answer",
+        );
+      },
+      async *invokeStream() {
+        throw new Error("streaming path is not used in this test");
+      },
+    });
+
+    const executor = createTwoPassExecutor({
+      draftModel: "smart-model",
+      finalModel: "base-model",
+      resolveFinalSettings: () => ({
+        reasoningLevel: "high",
+        systemPrompt: "system prompt",
+      }),
+    });
+
+    const result = await executor.execute(request, ctx);
+
+    expect(seen.map((program) => getModel(program))).toEqual([
+      "smart-model",
+      "base-model",
+    ]);
+    expect(contentText(result)).toBe("final answer");
+
+    const finalRequest = JSON.parse(
+      decoder.decode(emitChatCompletionsRequest(seen[1]!)),
+    );
+    expect(finalRequest.messages[0]).toEqual({
+      role: "system",
+      content: "system prompt",
+    });
+    expect(finalRequest.reasoning_effort).toBe("high");
+    expect(finalRequest.messages[2].tool_calls[0].function).toEqual({
+      name: INTERNAL_DRAFT_TOOL_NAME,
+      arguments: "{}",
+    });
+    expect(finalRequest.messages[3]).toEqual({
+      role: "tool",
+      tool_call_id: "knowledge_0",
+      content: "draft answer",
+    });
+  });
+
+  it("replaces draft system prompt and moves existing system prompt to first user message", async () => {
+    const request = parseChatCompletionsRequest(
+      encoder.encode(
+        JSON.stringify({
+          model: "virtual-model",
+          messages: [
+            { role: "system", content: "original system" },
+            { role: "user", content: "hello" },
+          ],
+        }),
+      ),
+    );
+    const seen: Program[] = [];
+    const ctx = buildExecutorContext({
+      async invoke(program) {
+        seen.push(program);
+        return appendAssistantMessage(
+          { code: [], buffers: [] },
+          seen.length === 1 ? "draft" : "final",
+        );
+      },
+      async *invokeStream() {
+        throw new Error("streaming path is not used in this test");
+      },
+    });
+
+    const executor = createTwoPassExecutor({
+      draftModel: "smart-model",
+      finalModel: "base-model",
+      resolveDraftSettings: () => ({ systemPrompt: "draft system" }),
+    });
+
+    await executor.execute(request, ctx);
+
+    const draftRequest = JSON.parse(
+      decoder.decode(emitChatCompletionsRequest(seen[0]!)),
+    );
+    expect(draftRequest.messages).toEqual([
+      { role: "system", content: "draft system" },
+      { role: "user", content: "original system" },
+      { role: "user", content: "hello" },
+    ]);
+  });
+
+  it("invokes and streams string or custom executors", async () => {
+    const calls: Array<{ model: string; streaming: boolean }> = [];
+    const ctx = buildExecutorContext({
+      async invoke(program) {
+        calls.push({ model: getModel(program) ?? "", streaming: false });
+        return appendAssistantMessage(
+          { code: [], buffers: [] },
+          "model answer",
+        );
+      },
+      async *invokeStream(program) {
+        calls.push({ model: getModel(program) ?? "", streaming: true });
+        yield parseChatCompletionsStreamChunk(
+          encoder.encode(
+            JSON.stringify({
+              id: "stream-response",
+              model: "base-model",
+              choices: [{ index: 0, delta: { content: "hello" } }],
+            }),
+          ),
+        );
+      },
+    });
+
+    const request = parseChatCompletionsRequest(
+      encoder.encode(
+        JSON.stringify({
+          model: "virtual-model",
+          messages: [{ role: "user", content: "hello" }],
+        }),
+      ),
+    );
+    const first = await invokeExecutor(ctx, request, "smart-model");
+    const chunks: Program[] = [];
+    for await (const chunk of streamExecutor(ctx, request, "base-model")) {
+      chunks.push(chunk);
+    }
+
+    expect(calls).toEqual([
+      { model: "smart-model", streaming: false },
+      { model: "base-model", streaming: true },
+    ]);
+    expect(contentText(first)).toBe("model answer");
+    expect(
+      decoder.decode(emitChatCompletionsStreamChunk(chunks[0]!)),
+    ).toContain("hello");
+    expect(getModel(request)).toBe("virtual-model");
+
+    const customExecutor = executorFromExecute(async () =>
+      appendAssistantMessage({ code: [], buffers: [] }, "custom answer"),
+    );
+    const custom = await invokeExecutor(ctx, request, customExecutor);
+    expect(contentText(custom)).toBe("custom answer");
+
+    const twoPass = createTwoPassExecutor({
+      draftModel: customExecutor,
+      finalModel: customExecutor,
+    });
+    expect(contentText(await twoPass.execute(request, ctx))).toBe(
+      "custom answer",
+    );
+  });
+
+  it("can skip the draft pass and limits the final context length", async () => {
+    const request = parseChatCompletionsRequest(
+      encoder.encode(
+        JSON.stringify({
+          model: "virtual-model",
+          messages: [{ role: "user", content: "request" }],
+        }),
+      ),
+    );
+    const seen: Program[] = [];
+    const ctx = buildExecutorContext({
+      async invoke(program) {
+        seen.push(program);
+        return appendAssistantMessage(
+          { code: [], buffers: [] },
+          seen.length === 1 ? "a very long draft" : "final",
+        );
+      },
+      async *invokeStream() {
+        throw new Error("streaming path is not used in this test");
+      },
+    });
+
+    const executor = createTwoPassExecutor({
+      draftModel: "smart-model",
+      finalModel: "base-model",
+      maxTotalContextLength: 10,
+      resolveDraftSettings: () => ({ systemPrompt: "draft" }),
+    });
+    await executor.execute(request, ctx);
+
+    const finalRequest = JSON.parse(
+      decoder.decode(emitChatCompletionsRequest(seen[1]!)),
+    );
+    expect(finalRequest.messages.at(-1)?.content).toBe("a v");
+
+    let receivedRequest: Program | undefined;
+    let receivedContext: ExecutorContext | undefined;
+    const directFinal = createTwoPassExecutor({
+      draftModel: "unused",
+      finalModel: "base-model",
+      resolveDraftSettings: (incomingRequest, incomingContext) => {
+        receivedRequest = incomingRequest;
+        receivedContext = incomingContext;
+        return null;
+      },
+    });
+    await directFinal.execute(request, ctx);
+    expect(getModel(seen.at(-1)!)).toBe("base-model");
+    expect(receivedRequest).toBe(request);
+    expect(receivedContext).toBe(ctx);
+
+    const draftOnly = createTwoPassExecutor({
+      draftModel: "smart-model",
+      finalModel: "unused",
+      resolveFinalSettings: () => null,
+    });
+    await draftOnly.execute(request, ctx);
+    expect(getModel(seen.at(-1)!)).toBe("smart-model");
+  });
+
+  it("writes reasoning helpers to a sink", async () => {
+    const emitted: string[] = [];
+    const sink: OutputSink = {
+      write(chunk) {
+        emitted.push(decoder.decode(emitChatCompletionsStreamChunk(chunk)));
+      },
+      close() {
+        emitted.push("[DONE]");
+      },
+    };
+
+    await writeReasoning(sink, "thinking");
+
+    expect(JSON.parse(emitted[0]!).choices[0].delta.reasoning_content).toBe(
+      "thinking",
+    );
+  });
+
+  it("retries executor failures", async () => {
+    let calls = 0;
+    const executor = retry(
+      executorFromExecute(async () => {
+        calls += 1;
+        if (calls === 1) {
+          throw new Error("transient");
+        }
+        return appendAssistantMessage({ code: [], buffers: [] }, "ok");
+      }),
+    );
+
+    const result = await executor.execute(
+      { code: [], buffers: [] },
+      buildExecutorContext(),
+    );
+
+    expect(calls).toBe(2);
+    expect(contentText(result)).toBe("ok");
+  });
+
+  it("falls back to the next executor after failure", async () => {
+    const executor = fallback([
+      executorFromExecute(async () => {
+        throw new Error("primary failed");
+      }),
+      executorFromExecute(async () =>
+        appendAssistantMessage({ code: [], buffers: [] }, "fallback"),
+      ),
+    ]);
+
+    const result = await executor.execute(
+      { code: [], buffers: [] },
+      buildExecutorContext(),
+    );
+
+    expect(contentText(result)).toBe("fallback");
+  });
+
+  it("does not retry a stream after exposing a partial chunk", async () => {
+    let retryCalls = 0;
+    const executor = retry(
+      streamingExecutor(async function* () {
+        retryCalls += 1;
+        yield appendAssistantMessage({ code: [], buffers: [] }, "partial");
+        throw new Error("stream interrupted");
+      }),
+      { attempts: 2 },
+    );
+    const context = buildExecutorContext();
+    const stream = executor
+      .stream({ code: [], buffers: [] }, context)
+      [Symbol.asyncIterator]();
+
+    await expect(stream.next()).resolves.toMatchObject({ done: false });
+    await expect(stream.next()).rejects.toThrow("stream interrupted");
+    expect(retryCalls).toBe(1);
+  });
+
+  it("does not fall back after exposing a partial stream chunk", async () => {
+    let fallbackCalls = 0;
+    const executor = fallback([
+      streamingExecutor(async function* () {
+        yield appendAssistantMessage({ code: [], buffers: [] }, "partial");
+        throw new Error("stream interrupted");
+      }),
+      streamingExecutor(async function* () {
+        fallbackCalls += 1;
+        yield appendAssistantMessage({ code: [], buffers: [] }, "fallback");
+      }),
+    ]);
+    const stream = executor
+      .stream({ code: [], buffers: [] }, buildExecutorContext())
+      [Symbol.asyncIterator]();
+
+    await expect(stream.next()).resolves.toMatchObject({ done: false });
+    await expect(stream.next()).rejects.toThrow("stream interrupted");
+    expect(fallbackCalls).toBe(0);
+  });
+
+  it("runs goal loops until the review is satisfied", async () => {
+    let attempts = 0;
+    const executor = createGoalExecutor({
+      draft: executorFromExecute(async () => {
+        attempts += 1;
+        return appendAssistantMessage(
+          { code: [], buffers: [] },
+          `answer ${attempts}`,
+        );
+      }),
+      review: executorFromExecute(async () =>
+        appendAssistantMessage(
+          { code: [], buffers: [] },
+          attempts > 1 ? "pass" : "retry",
+        ),
+      ),
+      satisfied(review) {
+        return contentText(review) === "pass";
+      },
+      refine(request) {
+        return request;
+      },
+    });
+
+    const result = await executor.execute(
+      { code: [], buffers: [] },
+      buildExecutorContext(),
+    );
+
+    expect(attempts).toBe(2);
+    expect(contentText(result)).toBe("answer 2");
+  });
+
+  it("streams draft and review passes through each goal iteration", async () => {
+    let attempts = 0;
+    const executor = createGoalExecutor({
+      draft: streamingExecutor(async function* () {
+        attempts += 1;
+        yield appendAssistantMessage(
+          { code: [], buffers: [] },
+          `answer ${attempts}`,
+        );
+      }),
+      review: streamingExecutor(async function* () {
+        yield appendAssistantMessage(
+          { code: [], buffers: [] },
+          attempts > 1 ? "pass" : "retry",
+        );
+      }),
+      satisfied(review) {
+        return contentText(review) === "pass";
+      },
+      refine(request) {
+        return request;
+      },
+    });
+
+    const chunks: Program[] = [];
+    for await (const chunk of executor.stream(
+      { code: [], buffers: [] },
+      buildExecutorContext(),
+    )) {
+      chunks.push(chunk);
+    }
+
+    expect(chunks.map(contentText)).toEqual([
+      "answer 1",
+      "retry",
+      "answer 2",
+      "pass",
+    ]);
+  });
+
+  it("uses streamed text as the next goal stage's response", async () => {
+    let reviewedAnswer = "";
+    const executor = createGoalExecutor({
+      draft: streamingExecutor(async function* () {
+        yield* streamTextResponse("answer");
+      }),
+      review: streamingExecutor(async function* (request) {
+        reviewedAnswer = contentText(request);
+        yield* streamTextResponse("pass");
+      }),
+      satisfied(review) {
+        return contentText(review) === "pass";
+      },
+      refine(request) {
+        return request;
+      },
+    });
+
+    for await (const _chunk of executor.stream(
+      { code: [], buffers: [] },
+      buildExecutorContext(),
+    )) {
+      // Consume the entire goal stream.
+    }
+
+    expect(reviewedAnswer).toBe("answer");
+  });
+});
+
+function buildExecutorContext(
+  impl: Pick<ExecutorContext, "invoke" | "invokeStream"> = {
+    async invoke(program) {
+      return program;
+    },
+    async *invokeStream(program) {
+      yield program;
+    },
+  },
+): ExecutorContext {
+  return {
+    requestId: "req_test",
+    executionId: "exec_test",
+    invoke: impl.invoke,
+    invokeStream: impl.invokeStream,
+    observe: () => {},
+    signal: new AbortController().signal,
+  };
+}
+
+function executorFromExecute(execute: Executor["execute"]): Executor {
+  return {
+    execute,
+    async *stream(request, ctx) {
+      yield await execute(request, ctx);
+    },
+  };
+}
+
+function streamingExecutor(stream: Executor["stream"]): Executor {
+  return {
+    async execute(request, ctx) {
+      let result = request;
+      for await (const chunk of stream(request, ctx)) result = chunk;
+      return result;
+    },
+    stream,
+  };
+}