@neutrome/lilsdk 0.3.5

package/test/tools.test.ts

@@ -0,0 +1,452 @@
+import { describe, expect, it } from "vitest";
+import {
+  callData,
+  createProgram,
+  Opcode,
+  type Instruction,
+  type Program,
+} from "@neutrome/lil-engine";
+import { createTargetExecutor } from "../src/managed/target-executor.ts";
+import { connectTools, ToolArgumentsError } from "../src/tools.ts";
+import type { ExecutionTarget, ExecutorContext, Tool } from "../src/types.ts";
+
+function buildToolCallResponse(
+  calls: Array<{ id: string; name: string; args: string }>,
+): Program {
+  const code: Instruction[] = [
+    { opcode: Opcode.MSG_START, value: { kind: "none" } },
+    { opcode: Opcode.ROLE_AST, value: { kind: "none" } },
+  ];
+  for (const call of calls) {
+    code.push(
+      { opcode: Opcode.CALL_START, value: { kind: "string", value: call.id } },
+      { opcode: Opcode.CALL_NAME, value: { kind: "string", value: call.name } },
+      {
+        opcode: Opcode.CALL_ARGS,
+        value: { kind: "json", value: new TextEncoder().encode(call.args) },
+      },
+      { opcode: Opcode.CALL_END, value: { kind: "none" } },
+    );
+  }
+  code.push(
+    {
+      opcode: Opcode.RESP_DONE,
+      value: { kind: "string", value: "tool_calls" },
+    },
+    { opcode: Opcode.MSG_END, value: { kind: "none" } },
+  );
+  return createProgram({ code });
+}
+
+function buildTextResponse(text: string): Program {
+  return createProgram({
+    code: [
+      { opcode: Opcode.MSG_START, value: { kind: "none" } },
+      { opcode: Opcode.ROLE_AST, value: { kind: "none" } },
+      { opcode: Opcode.TXT_CHUNK, value: { kind: "string", value: text } },
+      { opcode: Opcode.RESP_DONE, value: { kind: "string", value: "stop" } },
+      { opcode: Opcode.MSG_END, value: { kind: "none" } },
+    ],
+  });
+}
+
+const calculatorTool: Tool = {
+  name: "calculator",
+  description: "Evaluate a math expression",
+  schema: {
+    type: "object",
+    properties: { expr: { type: "string" } },
+    required: ["expr"],
+  },
+  systemPromptFragment: "You have a calculator tool. Use it for math.",
+  async execute(args) {
+    return String(eval((args as { expr: string }).expr));
+  },
+};
+
+describe("connectTools", () => {
+  it("passes through when LLM returns no tool calls", async () => {
+    const target: ExecutionTarget = { kind: "provider", model: "test-model" };
+    const executor = connectTools(
+      [calculatorTool],
+      createTargetExecutor(target),
+    );
+
+    const request = createProgram({
+      code: [
+        {
+          opcode: Opcode.SET_MODEL,
+          value: { kind: "string", value: "test-model" },
+        },
+      ],
+    });
+
+    const ctx = buildCtx(
+      {
+        async invoke() {
+          return buildTextResponse("Hello!");
+        },
+        async *invokeStream() {
+          yield buildTextResponse("Hello!");
+        },
+      },
+      target,
+    );
+    const result = await executor.execute(request, ctx);
+
+    const hasText = result.code.some(
+      (i) =>
+        i.opcode === Opcode.TXT_CHUNK &&
+        i.value.kind === "string" &&
+        i.value.value === "Hello!",
+    );
+    expect(hasText).toBe(true);
+  });
+
+  it("executes connected tool and loops until text response", async () => {
+    const target: ExecutionTarget = { kind: "provider", model: "test-model" };
+    const executor = connectTools(
+      [calculatorTool],
+      createTargetExecutor(target),
+    );
+
+    let callCount = 0;
+    const request = createProgram({
+      code: [
+        {
+          opcode: Opcode.SET_MODEL,
+          value: { kind: "string", value: "test-model" },
+        },
+      ],
+    });
+
+    const ctx = buildCtx(
+      {
+        async invoke() {
+          callCount += 1;
+          if (callCount === 1) {
+            return buildToolCallResponse([
+              { id: "call_1", name: "calculator", args: '{"expr":"2+2"}' },
+            ]);
+          }
+          return buildTextResponse("The answer is 4");
+        },
+        async *invokeStream() {
+          yield buildTextResponse("The answer is 4");
+        },
+      },
+      target,
+    );
+    const result = await executor.execute(request, ctx);
+
+    expect(callCount).toBe(2);
+    const hasAnswer = result.code.some(
+      (i) =>
+        i.opcode === Opcode.TXT_CHUNK &&
+        i.value.kind === "string" &&
+        i.value.value === "The answer is 4",
+    );
+    expect(hasAnswer).toBe(true);
+  });
+
+  it("rejects malformed connected-tool arguments", async () => {
+    const target: ExecutionTarget = { kind: "provider", model: "test-model" };
+    const executor = connectTools(
+      [calculatorTool],
+      createTargetExecutor(target),
+    );
+    const ctx = buildCtx(
+      {
+        async invoke() {
+          return buildToolCallResponse([
+            { id: "call_1", name: "calculator", args: "{not json}" },
+          ]);
+        },
+        async *invokeStream() {
+          return;
+        },
+      },
+      target,
+    );
+
+    await expect(executor.execute(createProgram(), ctx)).rejects.toBeInstanceOf(
+      ToolArgumentsError,
+    );
+  });
+
+  it("handles multiple tool calls in single response (parallel execution)", async () => {
+    const execOrder: string[] = [];
+    const multiTool: Tool = {
+      name: "lookup",
+      description: "Look up a value",
+      schema: { type: "object", properties: { key: { type: "string" } } },
+      async execute(args) {
+        const key = (args as { key: string }).key;
+        execOrder.push(key);
+        return `value_of_${key}`;
+      },
+    };
+
+    const target: ExecutionTarget = { kind: "provider", model: "test-model" };
+    const executor = connectTools([multiTool], createTargetExecutor(target));
+
+    let callCount = 0;
+    const request = createProgram();
+    const ctx = buildCtx(
+      {
+        async invoke() {
+          callCount += 1;
+          if (callCount === 1) {
+            return buildToolCallResponse([
+              { id: "call_a", name: "lookup", args: '{"key":"alpha"}' },
+              { id: "call_b", name: "lookup", args: '{"key":"beta"}' },
+            ]);
+          }
+          return buildTextResponse("Done");
+        },
+        async *invokeStream() {
+          yield buildTextResponse("Done");
+        },
+      },
+      target,
+    );
+    const result = await executor.execute(request, ctx);
+
+    expect(callCount).toBe(2);
+    expect(execOrder).toContain("alpha");
+    expect(execOrder).toContain("beta");
+  });
+
+  it("returns immediately when outer (non-connected) tool calls are present", async () => {
+    const target: ExecutionTarget = { kind: "provider", model: "test-model" };
+    const executor = connectTools(
+      [calculatorTool],
+      createTargetExecutor(target),
+    );
+
+    const request = createProgram();
+    const ctx = buildCtx(
+      {
+        async invoke() {
+          return buildToolCallResponse([
+            { id: "call_1", name: "calculator", args: '{"expr":"1+1"}' },
+            { id: "call_2", name: "external_tool", args: "{}" },
+          ]);
+        },
+        async *invokeStream() {
+          return;
+        },
+      },
+      target,
+    );
+    let executions = 0;
+    const tool: Tool = {
+      ...calculatorTool,
+      async execute(args) {
+        executions += 1;
+        return calculatorTool.execute(args, ctx);
+      },
+    };
+    const guardedExecutor = connectTools([tool], createTargetExecutor(target));
+    const result = await guardedExecutor.execute(request, ctx);
+
+    const calls = callData(result);
+    expect(calls).toHaveLength(2);
+    expect(calls.some((c) => c.name === "external_tool")).toBe(true);
+    expect(executions).toBe(0);
+  });
+
+  it("respects maxIterations", async () => {
+    const target: ExecutionTarget = { kind: "provider", model: "test-model" };
+    const executor = connectTools(
+      [calculatorTool],
+      createTargetExecutor(target),
+      { maxIterations: 2 },
+    );
+
+    let callCount = 0;
+    const request = createProgram();
+    const ctx = buildCtx(
+      {
+        async invoke() {
+          callCount += 1;
+          return buildToolCallResponse([
+            {
+              id: `call_${callCount}`,
+              name: "calculator",
+              args: '{"expr":"1"}',
+            },
+          ]);
+        },
+        async *invokeStream() {
+          yield buildTextResponse("final");
+        },
+      },
+      target,
+    );
+    await executor.execute(request, ctx);
+
+    // 2 iterations in the loop + 1 final invoke
+    expect(callCount).toBe(3);
+  });
+
+  it("streams final response after tool loop completes", async () => {
+    const target: ExecutionTarget = { kind: "provider", model: "test-model" };
+    const executor = connectTools(
+      [calculatorTool],
+      createTargetExecutor(target),
+    );
+
+    let callCount = 0;
+    const request = createProgram();
+    const ctx = buildCtx(
+      {
+        async invoke() {
+          callCount += 1;
+          if (callCount === 1) {
+            return buildToolCallResponse([
+              { id: "call_1", name: "calculator", args: '{"expr":"3+3"}' },
+            ]);
+          }
+          return buildTextResponse("6");
+        },
+        async *invokeStream() {
+          yield createProgram({
+            code: [{ opcode: Opcode.STREAM_START, value: { kind: "none" } }],
+          });
+          yield createProgram({
+            code: [
+              {
+                opcode: Opcode.STREAM_DELTA,
+                value: { kind: "string", value: "6" },
+              },
+            ],
+          });
+          yield createProgram({
+            code: [
+              {
+                opcode: Opcode.RESP_DONE,
+                value: { kind: "string", value: "stop" },
+              },
+              { opcode: Opcode.STREAM_END, value: { kind: "none" } },
+            ],
+          });
+        },
+      },
+      target,
+    );
+
+    const chunks: Program[] = [];
+    for await (const chunk of executor.stream(request, ctx)) {
+      chunks.push(chunk);
+    }
+
+    expect(chunks.length).toBeGreaterThanOrEqual(2);
+    const deltas = chunks.filter((c) =>
+      c.code.some((i) => i.opcode === Opcode.STREAM_DELTA),
+    );
+    expect(deltas.length).toBeGreaterThan(0);
+  });
+
+  it("assembles split stream tool deltas before executing a connected tool", async () => {
+    const target: ExecutionTarget = { kind: "provider", model: "test-model" };
+    const calls: unknown[] = [];
+    const tool: Tool = {
+      ...calculatorTool,
+      async execute(args) {
+        calls.push(args);
+        return "4";
+      },
+    };
+    const executor = connectTools([tool], createTargetExecutor(target));
+    let invocation = 0;
+    const ctx = buildCtx(
+      {
+        async invoke() {
+          return buildTextResponse("unused");
+        },
+        async *invokeStream() {
+          invocation += 1;
+          if (invocation === 1) {
+            yield streamToolDelta({ index: 0, id: "call_1" });
+            yield streamToolDelta({ index: 0, name: "calculator" });
+            yield streamToolDelta({ index: 0, arguments: '{"expr":"2+2"}' });
+            yield createProgram({
+              code: [
+                {
+                  opcode: Opcode.RESP_DONE,
+                  value: { kind: "string", value: "tool_calls" },
+                },
+                { opcode: Opcode.STREAM_END, value: { kind: "none" } },
+              ],
+            });
+            return;
+          }
+          yield createProgram({
+            code: [
+              {
+                opcode: Opcode.STREAM_DELTA,
+                value: { kind: "string", value: "4" },
+              },
+              {
+                opcode: Opcode.RESP_DONE,
+                value: { kind: "string", value: "stop" },
+              },
+              { opcode: Opcode.STREAM_END, value: { kind: "none" } },
+            ],
+          });
+        },
+      },
+      target,
+    );
+
+    const chunks: Program[] = [];
+    for await (const chunk of executor.stream(createProgram(), ctx))
+      chunks.push(chunk);
+
+    expect(calls).toEqual([{ expr: "2+2" }]);
+    expect(
+      chunks.some((chunk) =>
+        chunk.code.some((item) => item.opcode === Opcode.STREAM_DELTA),
+      ),
+    ).toBe(true);
+    expect(
+      chunks.some((chunk) =>
+        chunk.code.some((item) => item.opcode === Opcode.STREAM_TOOL_DELTA),
+      ),
+    ).toBe(false);
+  });
+});
+
+function streamToolDelta(delta: Record<string, unknown>): Program {
+  return createProgram({
+    code: [
+      {
+        opcode: Opcode.STREAM_TOOL_DELTA,
+        value: {
+          kind: "json",
+          value: new TextEncoder().encode(JSON.stringify(delta)),
+        },
+      },
+    ],
+  });
+}
+
+function buildCtx(
+  impl: Pick<ExecutorContext, "invoke" | "invokeStream">,
+  target: ExecutionTarget,
+): ExecutorContext {
+  const signal = new AbortController().signal;
+  return {
+    requestId: "req_test",
+    executionId: "exec_test",
+    async invoke(request: Program, options) {
+      return impl.invoke(request, { target: options?.target ?? target });
+    },
+    async *invokeStream(request: Program, options) {
+      yield* impl.invokeStream(request, { target: options?.target ?? target });
+    },
+    observe: () => {},
+    signal,
+  };
+}

package/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ESNext",
+    "moduleResolution": "Bundler",
+    "lib": ["ES2022", "DOM", "DOM.Iterable", "WebWorker"],
+    "strict": true,
+    "allowImportingTsExtensions": true,
+    "allowArbitraryExtensions": true,
+    "rewriteRelativeImportExtensions": true,
+    "noUncheckedIndexedAccess": true,
+    "exactOptionalPropertyTypes": true,
+    "skipLibCheck": true,
+    "declaration": true,
+    "sourceMap": true,
+    "types": ["node", "vitest/globals"],
+    "outDir": "dist"
+  },
+  "include": ["src/**/*.ts", "test/**/*.ts"],
+  "exclude": ["dist", "node_modules"]
+}

package/vitest.config.ts

@@ -0,0 +1,3 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({});