@workglow/ai-provider 0.0.126 → 0.1.0

package/dist/provider-llamacpp/runtime.js

@@ -73,6 +73,33 @@ async function getOrLoadModel(model) {
   llamaCppModels.set(modelPath, loadedModel);
   return loadedModel;
 }
+function llamaCppSeedPromptSpread(provider_config) {
+  return provider_config.seed !== undefined ? { seed: provider_config.seed } : {};
+}
+function detectQwenChatWrapperVariation(model) {
+  const candidates = [
+    model.model_id,
+    model.title,
+    model.description,
+    model.provider_config.model_url,
+    model.provider_config.model_path
+  ].filter((value) => typeof value === "string" && value.length > 0).map((value) => value.toLowerCase());
+  if (candidates.some((value) => /\bqwen(?:[\s._-]?|)3(?:[\s._-]?|)5\b|\bqwen(?:[\s._-]?|)3\.5\b/.test(value))) {
+    return "3.5";
+  }
+  if (candidates.some((value) => /\bqwen(?:[\s._-]?|)3\b/.test(value))) {
+    return "3";
+  }
+  return;
+}
+function llamaCppChatSessionConstructorSpread(model) {
+  const variation = detectQwenChatWrapperVariation(model);
+  if (!variation) {
+    return {};
+  }
+  const { QwenChatWrapper } = getLlamaCppSdk();
+  return { chatWrapper: new QwenChatWrapper({ variation }) };
+}
 async function getOrCreateTextContext(model) {
   const modelPath = getActualModelPath(model);
   const cached = llamaCppTextContexts.get(modelPath);
@@ -255,18 +282,7 @@ var LlamaCpp_ModelSearch = async (input, _model, _onProgress, signal) => {
 };
 
 // src/provider-llamacpp/common/LlamaCpp_CountTokens.ts
-import { getLogger } from "@workglow/util/worker";
 var LlamaCpp_CountTokens = async (input, model, onProgress, signal) => {
-  if (Array.isArray(input.text)) {
-    getLogger().warn("LlamaCpp_CountTokens: array input received; processing sequentially (no native batch support)");
-    const texts = input.text;
-    const counts = [];
-    for (const item of texts) {
-      const r = await LlamaCpp_CountTokens({ ...input, text: item }, model, onProgress, signal);
-      counts.push(r.count);
-    }
-    return { count: counts };
-  }
   const loadedModel = await getOrLoadModel(model);
   const tokens = loadedModel.tokenizer(input.text);
   return { count: tokens.length };
@@ -347,11 +363,15 @@ var LlamaCpp_StructuredGeneration = async (input, model, update_progress, signal
   const grammar = await llama.createGrammarForJsonSchema(input.outputSchema);
   const sequence = context.getSequence();
   const { LlamaChatSession } = getLlamaCppSdk();
-  const session = new LlamaChatSession({ contextSequence: sequence });
+  const session = new LlamaChatSession({
+    contextSequence: sequence,
+    ...llamaCppChatSessionConstructorSpread(model)
+  });
   try {
     const text = await session.prompt(input.prompt, {
       signal,
       grammar,
+      ...llamaCppSeedPromptSpread(model.provider_config),
       ...input.temperature !== undefined && { temperature: input.temperature },
       ...input.maxTokens !== undefined && { maxTokens: input.maxTokens }
     });
@@ -364,6 +384,7 @@ var LlamaCpp_StructuredGeneration = async (input, model, update_progress, signal
     update_progress(100, "Structured generation complete");
     return { object };
   } finally {
+    session.dispose({ disposeSequence: false });
     sequence.dispose();
   }
 };
@@ -376,7 +397,10 @@ var LlamaCpp_StructuredGeneration_Stream = async function* (input, model, signal
   const grammar = await llama.createGrammarForJsonSchema(input.outputSchema);
   const sequence = context.getSequence();
   const { LlamaChatSession } = getLlamaCppSdk();
-  const session = new LlamaChatSession({ contextSequence: sequence });
+  const session = new LlamaChatSession({
+    contextSequence: sequence,
+    ...llamaCppChatSessionConstructorSpread(model)
+  });
   const queue = [];
   let isComplete = false;
   let completionError;
@@ -389,6 +413,7 @@ var LlamaCpp_StructuredGeneration_Stream = async function* (input, model, signal
   const promptPromise = session.prompt(input.prompt, {
     signal,
     grammar,
+    ...llamaCppSeedPromptSpread(model.provider_config),
     onTextChunk: (chunk) => {
       queue.push(chunk);
       notifyWaiter();
@@ -429,6 +454,7 @@ var LlamaCpp_StructuredGeneration_Stream = async function* (input, model, signal
     }
   } finally {
     await promptPromise.catch(() => {});
+    session.dispose({ disposeSequence: false });
     sequence.dispose();
   }
   if (completionError) {
@@ -462,18 +488,7 @@ var LlamaCpp_TextEmbedding = async (input, model, update_progress, _signal) => {
 };
 
 // src/provider-llamacpp/common/LlamaCpp_TextGeneration.ts
-import { getLogger as getLogger2 } from "@workglow/util/worker";
 var LlamaCpp_TextGeneration = async (input, model, update_progress, signal) => {
-  if (Array.isArray(input.prompt)) {
-    getLogger2().warn("LlamaCpp_TextGeneration: array input received; processing sequentially (no native batch support)");
-    const prompts = input.prompt;
-    const results = [];
-    for (const item of prompts) {
-      const r = await LlamaCpp_TextGeneration({ ...input, prompt: item }, model, update_progress, signal);
-      results.push(r.text);
-    }
-    return { text: results };
-  }
   if (!model)
     throw new Error("Model config is required for TextGenerationTask.");
   const { LlamaChatSession } = await loadSdk();
@@ -481,10 +496,14 @@ var LlamaCpp_TextGeneration = async (input, model, update_progress, signal) => {
   const context = await getOrCreateTextContext(model);
   update_progress(10, "Generating text");
   const sequence = context.getSequence();
-  const session = new LlamaChatSession({ contextSequence: sequence });
+  const session = new LlamaChatSession({
+    contextSequence: sequence,
+    ...llamaCppChatSessionConstructorSpread(model)
+  });
   try {
     const text = await session.prompt(input.prompt, {
       signal,
+      ...llamaCppSeedPromptSpread(model.provider_config),
       ...input.temperature !== undefined && { temperature: input.temperature },
       ...input.maxTokens !== undefined && { maxTokens: input.maxTokens },
       ...input.topP !== undefined && { topP: input.topP }
@@ -492,6 +511,7 @@ var LlamaCpp_TextGeneration = async (input, model, update_progress, signal) => {
     update_progress(100, "Text generation complete");
     return { text };
   } finally {
+    session.dispose({ disposeSequence: false });
     sequence.dispose();
   }
 };
@@ -501,35 +521,29 @@ var LlamaCpp_TextGeneration_Stream = async function* (input, model, signal) {
   const { LlamaChatSession } = await loadSdk();
   const context = await getOrCreateTextContext(model);
   const sequence = context.getSequence();
-  const session = new LlamaChatSession({ contextSequence: sequence });
+  const session = new LlamaChatSession({
+    contextSequence: sequence,
+    ...llamaCppChatSessionConstructorSpread(model)
+  });
   try {
     yield* streamFromSession((onTextChunk) => {
       return session.prompt(input.prompt, {
         signal,
         onTextChunk,
+        ...llamaCppSeedPromptSpread(model.provider_config),
         ...input.temperature !== undefined && { temperature: input.temperature },
         ...input.maxTokens !== undefined && { maxTokens: input.maxTokens },
         ...input.topP !== undefined && { topP: input.topP }
       });
     }, signal);
   } finally {
+    session.dispose({ disposeSequence: false });
     sequence.dispose();
   }
 };
 
 // src/provider-llamacpp/common/LlamaCpp_TextRewriter.ts
-import { getLogger as getLogger3 } from "@workglow/util/worker";
 var LlamaCpp_TextRewriter = async (input, model, update_progress, signal) => {
-  if (Array.isArray(input.text)) {
-    getLogger3().warn("LlamaCpp_TextRewriter: array input received; processing sequentially (no native batch support)");
-    const texts = input.text;
-    const results = [];
-    for (const item of texts) {
-      const r = await LlamaCpp_TextRewriter({ ...input, text: item }, model, update_progress, signal);
-      results.push(r.text);
-    }
-    return { text: results };
-  }
   if (!model)
     throw new Error("Model config is required for TextRewriterTask.");
   const { LlamaChatSession } = await loadSdk();
@@ -539,13 +553,18 @@ var LlamaCpp_TextRewriter = async (input, model, update_progress, signal) => {
   const sequence = context.getSequence();
   const session = new LlamaChatSession({
     contextSequence: sequence,
+    ...llamaCppChatSessionConstructorSpread(model),
     systemPrompt: input.prompt
   });
   try {
-    const text = await session.prompt(input.text, { signal });
+    const text = await session.prompt(input.text, {
+      signal,
+      ...llamaCppSeedPromptSpread(model.provider_config)
+    });
     update_progress(100, "Text rewriting complete");
     return { text };
   } finally {
+    session.dispose({ disposeSequence: false });
     sequence.dispose();
   }
 };
@@ -557,30 +576,25 @@ var LlamaCpp_TextRewriter_Stream = async function* (input, model, signal) {
   const sequence = context.getSequence();
   const session = new LlamaChatSession({
     contextSequence: sequence,
+    ...llamaCppChatSessionConstructorSpread(model),
     systemPrompt: input.prompt
   });
   try {
     yield* streamFromSession((onTextChunk) => {
-      return session.prompt(input.text, { signal, onTextChunk });
+      return session.prompt(input.text, {
+        signal,
+        onTextChunk,
+        ...llamaCppSeedPromptSpread(model.provider_config)
+      });
     }, signal);
   } finally {
+    session.dispose({ disposeSequence: false });
     sequence.dispose();
   }
 };
 
 // src/provider-llamacpp/common/LlamaCpp_TextSummary.ts
-import { getLogger as getLogger4 } from "@workglow/util/worker";
 var LlamaCpp_TextSummary = async (input, model, update_progress, signal) => {
-  if (Array.isArray(input.text)) {
-    getLogger4().warn("LlamaCpp_TextSummary: array input received; processing sequentially (no native batch support)");
-    const texts = input.text;
-    const results = [];
-    for (const item of texts) {
-      const r = await LlamaCpp_TextSummary({ ...input, text: item }, model, update_progress, signal);
-      results.push(r.text);
-    }
-    return { text: results };
-  }
   if (!model)
     throw new Error("Model config is required for TextSummaryTask.");
   const { LlamaChatSession } = await loadSdk();
@@ -590,13 +604,18 @@ var LlamaCpp_TextSummary = async (input, model, update_progress, signal) => {
   const sequence = context.getSequence();
   const session = new LlamaChatSession({
     contextSequence: sequence,
+    ...llamaCppChatSessionConstructorSpread(model),
     systemPrompt: "Summarize the following text concisely, preserving the key points."
   });
   try {
-    const text = await session.prompt(input.text, { signal });
+    const text = await session.prompt(input.text, {
+      signal,
+      ...llamaCppSeedPromptSpread(model.provider_config)
+    });
     update_progress(100, "Summarization complete");
     return { text };
   } finally {
+    session.dispose({ disposeSequence: false });
     sequence.dispose();
   }
 };
@@ -608,186 +627,21 @@ var LlamaCpp_TextSummary_Stream = async function* (input, model, signal) {
   const sequence = context.getSequence();
   const session = new LlamaChatSession({
     contextSequence: sequence,
+    ...llamaCppChatSessionConstructorSpread(model),
     systemPrompt: "Summarize the following text concisely, preserving the key points."
   });
   try {
     yield* streamFromSession((onTextChunk) => {
-      return session.prompt(input.text, { signal, onTextChunk });
-    }, signal);
-  } finally {
-    sequence.dispose();
-  }
-};
-
-// src/provider-llamacpp/common/LlamaCpp_ToolCalling.ts
-import { filterValidToolCalls } from "@workglow/ai/worker";
-import { getLogger as getLogger5 } from "@workglow/util/worker";
-function buildLlamaCppPrompt(input) {
-  const inputMessages = input.messages;
-  if (!inputMessages || inputMessages.length === 0) {
-    return Array.isArray(input.prompt) ? input.prompt.join(`
-`) : input.prompt;
-  }
-  const parts = [];
-  for (const msg of inputMessages) {
-    if (msg.role === "user") {
-      parts.push(`User: ${msg.content}`);
-    } else if (msg.role === "assistant" && Array.isArray(msg.content)) {
-      const text = msg.content.filter((b) => b.type === "text").map((b) => b.text).join("");
-      if (text)
-        parts.push(`Assistant: ${text}`);
-    } else if (msg.role === "tool" && Array.isArray(msg.content)) {
-      for (const block of msg.content) {
-        parts.push(`Tool Result: ${block.content}`);
-      }
-    }
-  }
-  return parts.join(`
-
-`);
-}
-function buildLlamaCppFunctions(tools, capturedCalls) {
-  const { defineChatSessionFunction } = getLlamaCppSdk();
-  const functions = {};
-  for (const tool of tools) {
-    const toolName = tool.name;
-    functions[toolName] = defineChatSessionFunction({
-      description: tool.description,
-      params: tool.inputSchema,
-      handler(params) {
-        capturedCalls.push({ name: toolName, input: params ?? {} });
-        return "OK";
-      }
-    });
-  }
-  return functions;
-}
-var LlamaCpp_ToolCalling = async (input, model, update_progress, signal) => {
-  if (Array.isArray(input.prompt)) {
-    getLogger5().warn("LlamaCpp_ToolCalling: array input received; processing sequentially (no native batch support)");
-    const prompts = input.prompt;
-    const texts = [];
-    const toolCallsList = [];
-    for (const item of prompts) {
-      const r = await LlamaCpp_ToolCalling({ ...input, prompt: item }, model, update_progress, signal);
-      texts.push(r.text);
-      toolCallsList.push(r.toolCalls);
-    }
-    return { text: texts, toolCalls: toolCallsList };
-  }
-  if (!model)
-    throw new Error("Model config is required for ToolCallingTask.");
-  await loadSdk();
-  update_progress(0, "Loading model");
-  const context = await getOrCreateTextContext(model);
-  const capturedCalls = [];
-  const functions = input.toolChoice === "none" ? undefined : buildLlamaCppFunctions(input.tools, capturedCalls);
-  update_progress(10, "Running tool calling");
-  const sequence = context.getSequence();
-  const { LlamaChatSession } = getLlamaCppSdk();
-  const promptText = buildLlamaCppPrompt(input);
-  const session = new LlamaChatSession({
-    contextSequence: sequence,
-    ...input.systemPrompt && { systemPrompt: input.systemPrompt }
-  });
-  try {
-    const text = await session.prompt(promptText, {
-      signal,
-      ...functions && { functions },
-      ...input.temperature !== undefined && { temperature: input.temperature },
-      ...input.maxTokens !== undefined && { maxTokens: input.maxTokens }
-    });
-    const toolCalls = [];
-    capturedCalls.forEach((call, index) => {
-      const id = `call_${index}`;
-      toolCalls.push({ id, name: call.name, input: call.input });
-    });
-    update_progress(100, "Tool calling complete");
-    return { text, toolCalls: filterValidToolCalls(toolCalls, input.tools) };
-  } finally {
-    sequence.dispose();
-  }
-};
-var LlamaCpp_ToolCalling_Stream = async function* (input, model, signal) {
-  if (!model)
-    throw new Error("Model config is required for ToolCallingTask.");
-  await loadSdk();
-  const context = await getOrCreateTextContext(model);
-  const capturedCalls = [];
-  const functions = input.toolChoice === "none" ? undefined : buildLlamaCppFunctions(input.tools, capturedCalls);
-  const sequence = context.getSequence();
-  const { LlamaChatSession } = getLlamaCppSdk();
-  const promptText = buildLlamaCppPrompt(input);
-  const session = new LlamaChatSession({
-    contextSequence: sequence,
-    ...input.systemPrompt && { systemPrompt: input.systemPrompt }
-  });
-  const queue = [];
-  let isComplete = false;
-  let completionError;
-  let resolveWait = null;
-  const notifyWaiter = () => {
-    resolveWait?.();
-    resolveWait = null;
-  };
-  let accumulatedText = "";
-  const promptPromise = session.prompt(promptText, {
-    signal,
-    ...functions && { functions },
-    onTextChunk: (chunk) => {
-      queue.push(chunk);
-      notifyWaiter();
-    },
-    ...input.temperature !== undefined && { temperature: input.temperature },
-    ...input.maxTokens !== undefined && { maxTokens: input.maxTokens }
-  }).then(() => {
-    isComplete = true;
-    notifyWaiter();
-  }).catch((err) => {
-    completionError = err;
-    isComplete = true;
-    notifyWaiter();
-  });
-  try {
-    while (true) {
-      while (queue.length > 0) {
-        const chunk = queue.shift();
-        accumulatedText += chunk;
-        yield { type: "text-delta", port: "text", textDelta: chunk };
-      }
-      if (isComplete)
-        break;
-      await new Promise((r) => {
-        resolveWait = r;
+      return session.prompt(input.text, {
+        signal,
+        onTextChunk,
+        ...llamaCppSeedPromptSpread(model.provider_config)
       });
-    }
-    while (queue.length > 0) {
-      const chunk = queue.shift();
-      accumulatedText += chunk;
-      yield { type: "text-delta", port: "text", textDelta: chunk };
-    }
+    }, signal);
   } finally {
-    await promptPromise.catch(() => {});
+    session.dispose({ disposeSequence: false });
     sequence.dispose();
   }
-  if (completionError) {
-    if (!signal.aborted)
-      throw completionError;
-    return;
-  }
-  const toolCalls = [];
-  capturedCalls.forEach((call, index) => {
-    const id = `call_${index}`;
-    toolCalls.push({ id, name: call.name, input: call.input });
-  });
-  const validToolCalls = filterValidToolCalls(toolCalls, input.tools);
-  if (validToolCalls.length > 0) {
-    yield { type: "object-delta", port: "toolCalls", objectDelta: [...validToolCalls] };
-  }
-  yield {
-    type: "finish",
-    data: { text: accumulatedText, toolCalls: validToolCalls }
-  };
 };
 
 // src/provider-llamacpp/common/LlamaCpp_Unload.ts
@@ -828,7 +682,6 @@ var LLAMACPP_TASKS = {
   TextEmbeddingTask: LlamaCpp_TextEmbedding,
   TextRewriterTask: LlamaCpp_TextRewriter,
   TextSummaryTask: LlamaCpp_TextSummary,
-  ToolCallingTask: LlamaCpp_ToolCalling,
   StructuredGenerationTask: LlamaCpp_StructuredGeneration,
   ModelSearchTask: LlamaCpp_ModelSearch
 };
@@ -836,7 +689,6 @@ var LLAMACPP_STREAM_TASKS = {
   TextGenerationTask: LlamaCpp_TextGeneration_Stream,
   TextRewriterTask: LlamaCpp_TextRewriter_Stream,
   TextSummaryTask: LlamaCpp_TextSummary_Stream,
-  ToolCallingTask: LlamaCpp_ToolCalling_Stream,
   StructuredGenerationTask: LlamaCpp_StructuredGeneration_Stream
 };
 var LLAMACPP_REACTIVE_TASKS = {
@@ -861,7 +713,6 @@ class LlamaCppQueuedProvider extends QueuedAiProvider {
     "TextEmbeddingTask",
     "TextRewriterTask",
     "TextSummaryTask",
-    "ToolCallingTask",
     "ModelSearchTask"
   ];
   constructor(tasks, streamTasks, reactiveTasks) {
@@ -874,7 +725,7 @@ async function registerLlamaCppInline(options) {
   await new LlamaCppQueuedProvider(LLAMACPP_TASKS, LLAMACPP_STREAM_TASKS, LLAMACPP_REACTIVE_TASKS).register(options ?? {});
 }
 // src/provider-llamacpp/registerLlamaCppWorker.ts
-import { getLogger as getLogger6, globalServiceRegistry, WORKER_SERVER } from "@workglow/util/worker";
+import { getLogger, globalServiceRegistry, WORKER_SERVER } from "@workglow/util/worker";
 
 // src/provider-llamacpp/LlamaCppProvider.ts
 import {
@@ -894,7 +745,6 @@ class LlamaCppProvider extends AiProvider {
     "TextEmbeddingTask",
     "TextRewriterTask",
     "TextSummaryTask",
-    "ToolCallingTask",
     "ModelSearchTask"
   ];
   constructor(tasks, streamTasks, reactiveTasks) {
@@ -907,7 +757,7 @@ async function registerLlamaCppWorker() {
   const workerServer = globalServiceRegistry.get(WORKER_SERVER);
   new LlamaCppProvider(LLAMACPP_TASKS, LLAMACPP_STREAM_TASKS, LLAMACPP_REACTIVE_TASKS).registerOnWorkerServer(workerServer);
   workerServer.sendReady();
-  getLogger6().info("LlamaCpp worker job run functions registered");
+  getLogger().info("LlamaCpp worker job run functions registered");
 }
 export {
   streamFromSession,
@@ -916,8 +766,10 @@ export {
   registerLlamaCppInline,
   loadSdk,
   llamaCppTextContexts,
+  llamaCppSeedPromptSpread,
   llamaCppModels,
   llamaCppEmbeddingContexts,
+  llamaCppChatSessionConstructorSpread,
   getOrLoadModel,
   getOrCreateTextContext,
   getOrCreateEmbeddingContext,
@@ -928,4 +780,4 @@ export {
   disposeLlamaCppResources
 };
 
-//# debugId=DFC4AD999DD570D964756E2164756E21
+//# debugId=EB49E8246FF54A0364756E2164756E21