@octomil/browser 1.2.0 → 1.4.0

package/dist/index.cjs

@@ -34646,7 +34646,51 @@ function createTransformersJsLocalResponsesRuntime(options = {}) {
       return toResponse(request, config.model, generatedText);
     },
     async *stream(request) {
-      const response = await this.create(request);
+      const generator = await getGenerator(config);
+      const messages = buildMessages(request, config.maxInputChars);
+      const generationInput = renderGenerationInput(generator, messages);
+      const pending = [];
+      let wake = null;
+      const push = (item) => {
+        pending.push(item);
+        if (wake) {
+          wake();
+          wake = null;
+        }
+      };
+      const pull = async () => {
+        while (pending.length === 0) {
+          await new Promise((r) => {
+            wake = r;
+          });
+        }
+        return pending.shift();
+      };
+      const genPromise = generator(generationInput, {
+        max_new_tokens: request.maxOutputTokens ?? config.maxNewTokens,
+        temperature: request.temperature ?? config.temperature,
+        top_p: request.topP ?? config.topP,
+        repetition_penalty: config.repetitionPenalty,
+        do_sample: (request.temperature ?? config.temperature) > 0,
+        return_full_text: false,
+        callback_function: (text) => {
+          if (typeof text === "string" && text.length > 0) {
+            push({ kind: "token", text });
+          }
+        }
+      }).then(() => push({ kind: "done" })).catch((err) => push({ kind: "error", error: err }));
+      let fullText = "";
+      while (true) {
+        const item = await pull();
+        if (item.kind === "error") {
+          throw item.error instanceof Error ? item.error : new Error(String(item.error));
+        }
+        if (item.kind === "done") break;
+        fullText += item.text;
+        yield { type: "text_delta", delta: item.text };
+      }
+      await genPromise;
+      const response = toResponse(request, config.model, fullText);
       const firstOutput = response.output[0];
       if (firstOutput?.type === "tool_call") {
         yield {
@@ -34656,8 +34700,6 @@ function createTransformersJsLocalResponsesRuntime(options = {}) {
           name: firstOutput.toolCall?.name,
           argumentsDelta: firstOutput.toolCall?.arguments
         };
-      } else if (firstOutput?.type === "text" && firstOutput.text) {
-        yield { type: "text_delta", delta: firstOutput.text };
       }
       yield { type: "done", response };
     }
@@ -34702,17 +34744,14 @@ async function runLocalGeneration(request, config) {
   return extractGeneratedText(generation);
 }
 async function getGenerator(config) {
-  const key = JSON.stringify([
-    config.runtimeModel,
-    resolveDevice(config.device),
-    config.dtype
-  ]);
+  const device = await resolveDevice(config.device);
+  const key = JSON.stringify([config.runtimeModel, device, config.dtype]);
   let pending = pipelineCache.get(key);
   if (!pending) {
     pending = (async () => {
       const { pipeline: pipeline3 } = await importTransformers(config);
       return pipeline3("text-generation", config.runtimeModel, {
-        device: resolveDevice(config.device),
+        device,
         dtype: config.dtype
       });
     })();
@@ -34732,11 +34771,65 @@ async function importTransformers(config) {
   }
   return transformers;
 }
-function resolveDevice(device) {
-  if (device === "webgpu" || device === "wasm") {
-    return device;
+var resolvedDeviceCache = null;
+async function resolveDevice(device) {
+  if (device === "webgpu" || device === "wasm") return device;
+  if (resolvedDeviceCache) return resolvedDeviceCache;
+  const result = await probeWebGPU();
+  resolvedDeviceCache = result;
+  return result;
+}
+async function probeWebGPU() {
+  try {
+    if (typeof navigator === "undefined" || !("gpu" in navigator)) return "wasm";
+    const gpu = navigator.gpu;
+    const adapter = await gpu.requestAdapter();
+    if (!adapter) return "wasm";
+    const device = await adapter.requestDevice();
+    const module2 = device.createShaderModule({
+      code: `@group(0) @binding(0) var<storage, read_write> out: array<f32>;
+@compute @workgroup_size(1)
+fn main() { out[0] = 42.0; }`
+    });
+    const storageBuffer = device.createBuffer({
+      size: 4,
+      usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC
+    });
+    const readBuffer = device.createBuffer({
+      size: 4,
+      usage: GPUBufferUsage.MAP_READ | GPUBufferUsage.COPY_DST
+    });
+    const bindGroupLayout = device.createBindGroupLayout({
+      entries: [{ binding: 0, visibility: GPUShaderStage.COMPUTE, buffer: { type: "storage" } }]
+    });
+    const pipelineLayout = device.createPipelineLayout({ bindGroupLayouts: [bindGroupLayout] });
+    const pipeline3 = device.createComputePipeline({
+      layout: pipelineLayout,
+      compute: { module: module2, entryPoint: "main" }
+    });
+    const bindGroup = device.createBindGroup({
+      layout: bindGroupLayout,
+      entries: [{ binding: 0, resource: { buffer: storageBuffer } }]
+    });
+    const encoder = device.createCommandEncoder();
+    const pass = encoder.beginComputePass();
+    pass.setPipeline(pipeline3);
+    pass.setBindGroup(0, bindGroup);
+    pass.dispatchWorkgroups(1);
+    pass.end();
+    encoder.copyBufferToBuffer(storageBuffer, 0, readBuffer, 0, 4);
+    device.queue.submit([encoder.finish()]);
+    await readBuffer.mapAsync(GPUMapMode.READ);
+    const data = new Float32Array(readBuffer.getMappedRange());
+    const value = data[0];
+    readBuffer.unmap();
+    storageBuffer.destroy();
+    readBuffer.destroy();
+    device.destroy();
+    return value === 42 ? "webgpu" : "wasm";
+  } catch {
+    return "wasm";
   }
-  return typeof navigator !== "undefined" && "gpu" in navigator ? "webgpu" : "wasm";
 }
 function renderGenerationInput(generator, messages) {
   const applyChatTemplate = generator.tokenizer?.apply_chat_template;