@neutrome/lilsdk 0.3.5

package/src/managed/twoPassExecutor.ts

@@ -0,0 +1,200 @@
+import {
+  callData,
+  contentText,
+  deltaText,
+  finishReason,
+  hasToolDelta,
+  lastMessageRole,
+  setModel,
+  type Program,
+} from "@neutrome/lil-engine";
+import { streamReasoningDelta } from "../output.ts";
+import type { Executor, ExecutorContext, InvokeOptions } from "../types.ts";
+import {
+  appendInternalDraft,
+  buildDraftRequest,
+  buildFinalRequest,
+  INTERNAL_DRAFT_CALL_ID,
+  INTERNAL_DRAFT_TOOL_NAME,
+  validateMaxContextLength,
+  type InternalDraft,
+  type TwoPassRequestOptions,
+  type TwoPassSettings,
+  type TwoPassSettingsResolver,
+} from "./two-pass-request.ts";
+
+export type {
+  InternalDraft,
+  TwoPassSettings,
+  TwoPassSettingsResolver,
+} from "./two-pass-request.ts";
+export {
+  appendInternalDraft,
+  INTERNAL_DRAFT_CALL_ID,
+  INTERNAL_DRAFT_TOOL_NAME,
+};
+
+export type ModelExecutor = string | Executor;
+
+export type TwoPassExecutorOptions = TwoPassRequestOptions & {
+  draftModel: ModelExecutor;
+  finalModel: ModelExecutor;
+  resolveDraftSettings?: TwoPassSettingsResolver;
+  resolveFinalSettings?: TwoPassSettingsResolver;
+  reasoningIntro?: string;
+  reasoningSeparator?: string;
+};
+
+export function invokeExecutor(
+  ctx: ExecutorContext,
+  request: Program,
+  executor: ModelExecutor,
+  options?: InvokeOptions,
+): Promise<Program> {
+  return typeof executor === "string"
+    ? ctx.invoke(setModel(request, executor), options)
+    : executor.execute(request, ctx);
+}
+
+export async function* streamExecutor(
+  ctx: ExecutorContext,
+  request: Program,
+  executor: ModelExecutor,
+  options?: InvokeOptions,
+): AsyncIterable<Program> {
+  if (typeof executor === "string") {
+    yield* ctx.invokeStream(setModel(request, executor), options);
+    return;
+  }
+  yield* executor.stream(request, ctx);
+}
+
+export function createTwoPassExecutor(
+  options: TwoPassExecutorOptions,
+): Executor {
+  const reasoningIntro = options.reasoningIntro ?? "Let me think...\n\n";
+  const reasoningSeparator = options.reasoningSeparator ?? "\n\n";
+  validateMaxContextLength(options.maxTotalContextLength);
+
+  return {
+    async execute(request, ctx) {
+      const settings = await resolveSettings(options, request, ctx);
+      if (!settings.draft) {
+        return invokeFinal(options, request, ctx, settings.final!);
+      }
+      const draftRequest = buildDraftRequest(request, settings.draft);
+      if (!settings.final || lastMessageRole(request) === "tool") {
+        return invokeExecutor(ctx, draftRequest, options.draftModel);
+      }
+
+      const draftResponse = await invokeExecutor(
+        ctx,
+        draftRequest,
+        options.draftModel,
+      );
+      if (callData(draftResponse).length > 0) return draftResponse;
+
+      return invokeFinal(
+        options,
+        request,
+        ctx,
+        settings.final,
+        contentText(draftResponse),
+      );
+    },
+
+    async *stream(request, ctx) {
+      const settings = await resolveSettings(options, request, ctx);
+      if (!settings.draft) {
+        yield* streamFinal(options, request, ctx, settings.final!);
+        return;
+      }
+      const draftRequest = buildDraftRequest(request, settings.draft);
+      if (!settings.final || lastMessageRole(request) === "tool") {
+        yield* streamExecutor(ctx, draftRequest, options.draftModel);
+        return;
+      }
+
+      let transcript = "";
+      let emittedReasoning = false;
+      let toolMode = false;
+      for await (const chunk of streamExecutor(
+        ctx,
+        draftRequest,
+        options.draftModel,
+      )) {
+        if (ctx.signal.aborted) return;
+        if (toolMode) {
+          yield chunk;
+          continue;
+        }
+        if (hasToolDelta(chunk) || finishReason(chunk) === "tool_calls") {
+          toolMode = true;
+          yield chunk;
+          continue;
+        }
+        const text = deltaText(chunk);
+        if (!text) continue;
+        transcript += text;
+        if (!emittedReasoning) {
+          emittedReasoning = true;
+          yield streamReasoningDelta(reasoningIntro);
+        }
+        yield streamReasoningDelta(text);
+      }
+
+      if (toolMode || ctx.signal.aborted) return;
+      if (emittedReasoning && reasoningSeparator) {
+        yield streamReasoningDelta(reasoningSeparator);
+      }
+      yield* streamFinal(options, request, ctx, settings.final, transcript);
+    },
+  };
+}
+
+async function resolveSettings(
+  options: TwoPassExecutorOptions,
+  request: Program,
+  ctx: ExecutorContext,
+): Promise<{ draft: TwoPassSettings | null; final: TwoPassSettings | null }> {
+  const [draft, final] = await Promise.all([
+    options.resolveDraftSettings
+      ? options.resolveDraftSettings(request, ctx)
+      : {},
+    options.resolveFinalSettings
+      ? options.resolveFinalSettings(request, ctx)
+      : {},
+  ]);
+  if (!draft && !final) {
+    throw new Error("Two-pass executor requires at least one enabled pass");
+  }
+  return { draft, final };
+}
+
+function invokeFinal(
+  options: TwoPassExecutorOptions,
+  request: Program,
+  ctx: ExecutorContext,
+  settings: TwoPassSettings,
+  draft = "",
+): Promise<Program> {
+  return invokeExecutor(
+    ctx,
+    buildFinalRequest(options, request, settings, draft),
+    options.finalModel,
+  );
+}
+
+async function* streamFinal(
+  options: TwoPassExecutorOptions,
+  request: Program,
+  ctx: ExecutorContext,
+  settings: TwoPassSettings,
+  draft = "",
+): AsyncIterable<Program> {
+  yield* streamExecutor(
+    ctx,
+    buildFinalRequest(options, request, settings, draft),
+    options.finalModel,
+  );
+}

package/src/observe.ts

@@ -0,0 +1,95 @@
+import {
+  cloneProgram,
+  deltaText,
+  finishReason,
+  hasToolDelta,
+  type Program,
+} from "@neutrome/lil-engine";
+
+export type StreamObservationHooks = {
+  onTextStart?: () => void;
+  onTextChunk?: (text: string, chunk: Program) => void;
+};
+
+export type ObservedExecution =
+  | {
+      mode: "text";
+      transcript: string;
+      emittedText: boolean;
+    }
+  | {
+      mode: "tool";
+      transcript: string;
+      emittedText: boolean;
+      chunks: Program[];
+    };
+
+export async function observeExecutionStream(
+  source: AsyncIterable<Program>,
+  hooks: StreamObservationHooks = {},
+): Promise<ObservedExecution> {
+  const pending: Program[] = [];
+  const passthrough: Program[] = [];
+  let transcript = "";
+  let mode: "pending" | "text" | "tool" = "pending";
+  let emittedText = false;
+
+  for await (const chunk of source) {
+    const text = deltaText(chunk);
+    const toolRequested =
+      hasToolDelta(chunk) || finishReason(chunk) === "tool_calls";
+
+    if (mode === "pending") {
+      const clone = cloneProgram(chunk);
+      pending.push(clone);
+
+      if (toolRequested) {
+        mode = "tool";
+        passthrough.push(...pending.splice(0));
+        continue;
+      }
+
+      if (!text) {
+        continue;
+      }
+
+      mode = "text";
+      emittedText = true;
+      hooks.onTextStart?.();
+      for (const pendingChunk of pending.splice(0)) {
+        const pendingText = deltaText(pendingChunk);
+        if (pendingText) {
+          transcript += pendingText;
+          hooks.onTextChunk?.(pendingText, pendingChunk);
+        }
+      }
+      continue;
+    }
+
+    if (mode === "tool") {
+      passthrough.push(cloneProgram(chunk));
+      continue;
+    }
+
+    if (toolRequested) {
+      mode = "tool";
+      passthrough.push(cloneProgram(chunk));
+      continue;
+    }
+
+    if (text) {
+      if (!emittedText) {
+        emittedText = true;
+        hooks.onTextStart?.();
+      }
+      transcript += text;
+      hooks.onTextChunk?.(text, chunk);
+    }
+  }
+
+  if (mode === "tool") {
+    return { mode: "tool", transcript, emittedText, chunks: passthrough };
+  }
+
+  return { mode: "text", transcript, emittedText };
+}

package/src/output.ts

@@ -0,0 +1,31 @@
+import {
+  createProgram,
+  Opcode,
+  streamEnd,
+  streamStart,
+  streamTextDelta,
+  type Program,
+} from "@neutrome/lil-engine";
+import type { OutputSink } from "./types.ts";
+
+export async function writeReasoning(
+  sink: OutputSink,
+  text: string,
+): Promise<void> {
+  await sink.write(streamReasoningDelta(text));
+}
+
+export function streamTextResponse(text: string): Program[] {
+  return [streamStart(), streamTextDelta(text), streamEnd()];
+}
+
+export function streamReasoningDelta(text: string): Program {
+  return createProgram({
+    code: [
+      {
+        opcode: Opcode.STREAM_THINK_DELTA,
+        value: { kind: "string", value: text },
+      },
+    ],
+  });
+}

package/src/primitives/index.ts

@@ -0,0 +1,154 @@
+import {
+  cloneInstruction,
+  type Instruction,
+  type Program,
+} from "@neutrome/lil-engine";
+
+export function indexOf(program: Program, needle: Instruction): number {
+  return program.code.findIndex((instruction) =>
+    instructionEquals(instruction, needle),
+  );
+}
+
+export function has(program: Program, needle: Instruction): boolean {
+  return indexOf(program, needle) >= 0;
+}
+
+export function find(
+  program: Program,
+  needle: Instruction,
+): Instruction | undefined {
+  const i = indexOf(program, needle);
+  return i >= 0 ? cloneInstruction(program.code[i]!) : undefined;
+}
+
+export function walk(
+  program: Program,
+  callback: (instruction: Instruction, index: number, program: Program) => void,
+): void {
+  program.code.forEach((instruction, index) =>
+    callback(instruction, index, program),
+  );
+}
+
+export function map(
+  program: Program,
+  callback: (
+    instruction: Instruction,
+    index: number,
+    program: Program,
+  ) => Instruction,
+): Program {
+  return {
+    code: program.code.map((instruction, index) =>
+      cloneInstruction(callback(instruction, index, program)),
+    ),
+    buffers: cloneBuffers(program.buffers),
+  };
+}
+
+export function filter(
+  program: Program,
+  callback: (
+    instruction: Instruction,
+    index: number,
+    program: Program,
+  ) => boolean,
+): Program {
+  return {
+    code: program.code
+      .filter((instruction, index) => callback(instruction, index, program))
+      .map(cloneInstruction),
+    buffers: cloneBuffers(program.buffers),
+  };
+}
+
+export function reduce<T>(
+  program: Program,
+  callback: (
+    accumulator: T,
+    instruction: Instruction,
+    index: number,
+    program: Program,
+  ) => T,
+  initialValue: T,
+): T {
+  let accumulator = initialValue;
+  program.code.forEach((instruction, index) => {
+    accumulator = callback(accumulator, instruction, index, program);
+  });
+  return accumulator;
+}
+
+export function all(
+  program: Program,
+  callback: (
+    instruction: Instruction,
+    index: number,
+    program: Program,
+  ) => boolean,
+): boolean {
+  for (const [index, instruction] of program.code.entries()) {
+    if (!callback(instruction, index, program)) return false;
+  }
+  return true;
+}
+
+export function any(
+  program: Program,
+  callback: (
+    instruction: Instruction,
+    index: number,
+    program: Program,
+  ) => boolean,
+): boolean {
+  for (const [index, instruction] of program.code.entries()) {
+    if (callback(instruction, index, program)) return true;
+  }
+  return false;
+}
+
+function instructionEquals(left: Instruction, right: Instruction): boolean {
+  if (left.opcode !== right.opcode || left.value.kind !== right.value.kind) {
+    return false;
+  }
+
+  switch (left.value.kind) {
+    case "none":
+      return true;
+    case "string":
+    case "float":
+    case "int":
+    case "ref":
+      return (
+        right.value.kind === left.value.kind &&
+        left.value.value === right.value.value
+      );
+    case "json":
+      return (
+        right.value.kind === "json" &&
+        bytesEqual(left.value.value, right.value.value)
+      );
+    case "key_string":
+      return (
+        right.value.kind === "key_string" &&
+        left.value.key === right.value.key &&
+        left.value.value === right.value.value
+      );
+    case "key_json":
+      return (
+        right.value.kind === "key_json" &&
+        left.value.key === right.value.key &&
+        bytesEqual(left.value.value, right.value.value)
+      );
+  }
+}
+
+function bytesEqual(left: Uint8Array, right: Uint8Array): boolean {
+  if (left.byteLength !== right.byteLength) return false;
+  return left.every((value, index) => value === right[index]);
+}
+
+function cloneBuffers(buffers: Uint8Array[]): Uint8Array[] {
+  return buffers.map((buffer) => buffer.slice());
+}

package/src/stream/index.ts

@@ -0,0 +1,8 @@
+export type { ObservedExecution, StreamObservationHooks } from "../observe.ts";
+
+export { observeExecutionStream } from "../observe.ts";
+export {
+  streamReasoningDelta,
+  streamTextResponse,
+  writeReasoning,
+} from "../output.ts";

package/src/synthetic/index.ts

@@ -0,0 +1,134 @@
+import {
+  addTool,
+  appendInstructions,
+  createProgram,
+  insertBefore,
+  messages,
+  Opcode,
+  setModel,
+  type Instruction,
+  type Opcode as OpcodeValue,
+  type Program,
+} from "@neutrome/lil-engine";
+
+const encoder = new TextEncoder();
+
+export type TextMessageRole = "system" | "user" | "assistant";
+
+export type ToolInteraction = {
+  callId: string;
+  name: string;
+  args?: unknown;
+  result: unknown;
+};
+
+export function createModelProgram(model: string): Program {
+  return setModel(createProgram(), model);
+}
+
+export function makeMessage(role: TextMessageRole, text: string): Program {
+  return createProgram({ code: buildMessage(role, text) });
+}
+
+export function prependSystemPrompt(program: Program, text: string): Program {
+  const systemMessage = buildMessage("system", text);
+  const spans = messages(program);
+  if (spans.length === 0) {
+    return appendInstructions(program, systemMessage);
+  }
+  const firstSpan = spans[0];
+  if (!firstSpan) return appendInstructions(program, systemMessage);
+  return insertBefore(program, firstSpan.start, systemMessage);
+}
+
+export function appendUserMessage(program: Program, text: string): Program {
+  return appendInstructions(program, buildMessage("user", text));
+}
+
+export function appendAssistantMessage(
+  program: Program,
+  text: string,
+): Program {
+  return appendInstructions(program, buildMessage("assistant", text));
+}
+
+export function makeToolCall(
+  name: string,
+  args: unknown = {},
+  callId: string = crypto.randomUUID(),
+): Program {
+  return createProgram({
+    code: [
+      { opcode: Opcode.MSG_START, value: { kind: "none" } },
+      { opcode: Opcode.ROLE_AST, value: { kind: "none" } },
+      { opcode: Opcode.CALL_START, value: { kind: "string", value: callId } },
+      { opcode: Opcode.CALL_NAME, value: { kind: "string", value: name } },
+      {
+        opcode: Opcode.CALL_ARGS,
+        value: { kind: "json", value: encodeJson(args) },
+      },
+      { opcode: Opcode.CALL_END, value: { kind: "none" } },
+      { opcode: Opcode.MSG_END, value: { kind: "none" } },
+    ],
+  });
+}
+
+export function makeToolResponse(callId: string, result: unknown): Program {
+  return createProgram({ code: buildToolResponse(callId, result) });
+}
+
+export function appendToolInteraction(
+  program: Program,
+  interaction: ToolInteraction,
+): Program {
+  return appendInstructions(program, [
+    ...makeToolCall(
+      interaction.name,
+      interaction.args ?? {},
+      interaction.callId,
+    ).code,
+    ...buildToolResponse(interaction.callId, interaction.result),
+  ]);
+}
+
+function buildMessage(role: TextMessageRole, text: string): Instruction[] {
+  return [
+    { opcode: Opcode.MSG_START, value: { kind: "none" } },
+    { opcode: roleOpcode(role), value: { kind: "none" } },
+    { opcode: Opcode.TXT_CHUNK, value: { kind: "string", value: text } },
+    { opcode: Opcode.MSG_END, value: { kind: "none" } },
+  ];
+}
+
+function buildToolResponse(callId: string, result: unknown): Instruction[] {
+  return [
+    { opcode: Opcode.MSG_START, value: { kind: "none" } },
+    { opcode: Opcode.ROLE_TOOL, value: { kind: "none" } },
+    { opcode: Opcode.RESULT_START, value: { kind: "string", value: callId } },
+    {
+      opcode: Opcode.RESULT_DATA,
+      value: { kind: "string", value: stringifyResult(result) },
+    },
+    { opcode: Opcode.RESULT_END, value: { kind: "none" } },
+    { opcode: Opcode.MSG_END, value: { kind: "none" } },
+  ];
+}
+
+function roleOpcode(role: TextMessageRole): OpcodeValue {
+  switch (role) {
+    case "system":
+      return Opcode.ROLE_SYS;
+    case "user":
+      return Opcode.ROLE_USR;
+    case "assistant":
+      return Opcode.ROLE_AST;
+  }
+}
+
+function encodeJson(value: unknown): Uint8Array {
+  return encoder.encode(JSON.stringify(value));
+}
+
+function stringifyResult(result: unknown): string {
+  return typeof result === "string" ? result : JSON.stringify(result);
+}