@workglow/ai-provider 0.0.121 → 0.0.123

package/dist/provider-llamacpp/runtime.js

@@ -0,0 +1,929 @@
+var __defProp = Object.defineProperty;
+var __returnValue = (v) => v;
+function __exportSetter(name, newValue) {
+  this[name] = __returnValue.bind(null, newValue);
+}
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, {
+      get: all[name],
+      enumerable: true,
+      configurable: true,
+      set: __exportSetter.bind(all, name)
+    });
+};
+var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
+var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
+  get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
+}) : x)(function(x) {
+  if (typeof require !== "undefined")
+    return require.apply(this, arguments);
+  throw Error('Dynamic require of "' + x + '" is not supported');
+});
+
+// src/provider-llamacpp/common/LlamaCpp_Runtime.ts
+var _sdk;
+async function loadSdk() {
+  if (!_sdk) {
+    try {
+      _sdk = await import("node-llama-cpp");
+    } catch {
+      throw new Error("node-llama-cpp is required for LOCAL_LLAMACPP tasks. Install it with: bun add node-llama-cpp");
+    }
+  }
+  return _sdk;
+}
+function getLlamaCppSdk() {
+  if (!_sdk) {
+    throw new Error("LlamaCpp SDK not loaded; call loadSdk() first");
+  }
+  return _sdk;
+}
+var llamaInstance;
+var llamaCppModels = new Map;
+var llamaCppTextContexts = new Map;
+var llamaCppEmbeddingContexts = new Map;
+var resolvedPaths = new Map;
+async function getLlamaInstance() {
+  if (!llamaInstance) {
+    const { getLlama } = await loadSdk();
+    llamaInstance = await getLlama();
+  }
+  return llamaInstance;
+}
+function getConfigKey(model) {
+  return model.provider_config.model_url ?? model.provider_config.model_path;
+}
+function getActualModelPath(model) {
+  const key = getConfigKey(model);
+  const resolved = resolvedPaths.get(key);
+  return resolved ?? model.provider_config.model_path;
+}
+async function getOrLoadModel(model) {
+  const modelPath = getActualModelPath(model);
+  const cached = llamaCppModels.get(modelPath);
+  if (cached)
+    return cached;
+  const llama = await getLlamaInstance();
+  const config = model.provider_config;
+  const loadedModel = await llama.loadModel({
+    modelPath,
+    ...config.gpu_layers !== undefined && { gpuLayers: config.gpu_layers }
+  });
+  llamaCppModels.set(modelPath, loadedModel);
+  return loadedModel;
+}
+async function getOrCreateTextContext(model) {
+  const modelPath = getActualModelPath(model);
+  const cached = llamaCppTextContexts.get(modelPath);
+  if (cached)
+    return cached;
+  const loadedModel = await getOrLoadModel(model);
+  const config = model.provider_config;
+  const context = await loadedModel.createContext({
+    ...config.context_size && { contextSize: config.context_size },
+    ...config.flash_attention !== undefined && { flashAttention: config.flash_attention }
+  });
+  llamaCppTextContexts.set(modelPath, context);
+  return context;
+}
+async function getOrCreateEmbeddingContext(model) {
+  const modelPath = getActualModelPath(model);
+  const cached = llamaCppEmbeddingContexts.get(modelPath);
+  if (cached)
+    return cached;
+  const loadedModel = await getOrLoadModel(model);
+  const context = await loadedModel.createEmbeddingContext();
+  llamaCppEmbeddingContexts.set(modelPath, context);
+  return context;
+}
+async function* streamFromSession(promptFn, signal) {
+  const queue = [];
+  let isComplete = false;
+  let completionError;
+  let resolveWait = null;
+  const notifyWaiter = () => {
+    resolveWait?.();
+    resolveWait = null;
+  };
+  const promptPromise = promptFn((chunk) => {
+    queue.push(chunk);
+    notifyWaiter();
+  }).then(() => {
+    isComplete = true;
+    notifyWaiter();
+  }).catch((err) => {
+    completionError = err;
+    isComplete = true;
+    notifyWaiter();
+  });
+  try {
+    while (true) {
+      while (queue.length > 0) {
+        yield { type: "text-delta", port: "text", textDelta: queue.shift() };
+      }
+      if (isComplete)
+        break;
+      await new Promise((r) => {
+        resolveWait = r;
+      });
+    }
+    while (queue.length > 0) {
+      yield { type: "text-delta", port: "text", textDelta: queue.shift() };
+    }
+  } finally {
+    await promptPromise.catch(() => {});
+  }
+  if (completionError) {
+    if (signal.aborted)
+      return;
+    throw completionError;
+  }
+  yield { type: "finish", data: {} };
+}
+async function disposeLlamaCppResources() {
+  const disposeAll = async (map) => {
+    for (const resource of map.values()) {
+      await resource.dispose().catch(() => {});
+    }
+    map.clear();
+  };
+  await disposeAll(llamaCppTextContexts);
+  await disposeAll(llamaCppEmbeddingContexts);
+  await disposeAll(llamaCppModels);
+  if (llamaInstance) {
+    await llamaInstance.dispose?.().catch(() => {});
+    llamaInstance = undefined;
+  }
+  resolvedPaths.clear();
+}
+// src/common/PipelineTaskMapping.ts
+var TASK_TO_PIPELINES = {
+  TextEmbeddingTask: ["feature-extraction"],
+  TextGenerationTask: ["text-generation"],
+  TextSummaryTask: ["sentence-similarity", "summarization"],
+  TextTranslationTask: ["translation"],
+  TextClassificationTask: ["text-classification", "zero-shot-classification"],
+  TextQuestionAnswerTask: ["question-answering"],
+  TextFillMaskTask: ["fill-mask"],
+  TextLanguageDetectionTask: ["text-classification"],
+  TextNamedEntityRecognitionTask: ["token-classification"],
+  TokenClassificationTask: ["token-classification"],
+  ImageClassificationTask: ["image-classification", "zero-shot-image-classification"],
+  ImageEmbeddingTask: ["image-feature-extraction"],
+  ImageSegmentationTask: ["image-segmentation"],
+  ImageToImageTask: ["image-to-image"],
+  ImageToTextTask: ["image-to-text"],
+  ObjectDetectionTask: ["object-detection", "zero-shot-object-detection"],
+  DepthEstimationTask: ["depth-estimation"],
+  AudioClassificationTask: ["audio-classification"],
+  SpeechRecognitionTask: ["automatic-speech-recognition"]
+};
+function pipelineToTaskTypes(pipeline) {
+  return Object.entries(TASK_TO_PIPELINES).filter(([, pipelines]) => pipelines.includes(pipeline)).map(([task]) => task);
+}
+
+// src/common/HfModelSearch.ts
+var HF_API_BASE = "https://huggingface.co/api";
+function formatDownloads(n) {
+  if (n >= 1e6)
+    return `${(n / 1e6).toFixed(1)}M`;
+  if (n >= 1000)
+    return `${(n / 1000).toFixed(1)}k`;
+  return String(n);
+}
+function mapHfProviderConfig(entry, provider) {
+  switch (provider) {
+    case "HF_TRANSFORMERS_ONNX":
+      return {
+        model_path: entry.id,
+        ...entry.pipeline_tag ? { pipeline: entry.pipeline_tag } : {}
+      };
+    case "LOCAL_LLAMACPP":
+      return { model_path: entry.id };
+    default:
+      return { model_name: entry.id };
+  }
+}
+function mapHfModelResult(entry, provider) {
+  const badges = [entry.pipeline_tag, entry.library_name].filter(Boolean).join(" | ");
+  return {
+    id: entry.id,
+    label: `${entry.id}${badges ? `  ${badges}` : ""}`,
+    description: `${formatDownloads(entry.downloads)} downloads`,
+    record: {
+      model_id: entry.id,
+      provider,
+      title: entry.id.split("/").pop() ?? entry.id,
+      description: [entry.pipeline_tag, `${formatDownloads(entry.downloads)} downloads`].filter(Boolean).join(" — "),
+      tasks: entry.pipeline_tag ? pipelineToTaskTypes(entry.pipeline_tag) : [],
+      provider_config: mapHfProviderConfig(entry, provider),
+      metadata: {}
+    },
+    raw: entry
+  };
+}
+async function searchHfModels(query, extraParams, expandFields, signal) {
+  const params = new URLSearchParams({
+    search: query,
+    limit: "500",
+    sort: "downloads",
+    direction: "-1",
+    ...extraParams
+  });
+  params.append("expand[]", "pipeline_tag");
+  if (expandFields) {
+    for (const field of expandFields) {
+      params.append("expand[]", field);
+    }
+  }
+  const res = await fetch(`${HF_API_BASE}/models?${params}`, { signal });
+  if (!res.ok)
+    throw new Error(`HuggingFace API returned ${res.status}`);
+  return res.json();
+}
+
+// src/provider-llamacpp/common/LlamaCpp_Constants.ts
+var LOCAL_LLAMACPP = "LOCAL_LLAMACPP";
+var LLAMACPP_DEFAULT_MODELS_DIR = "./models";
+
+// src/provider-llamacpp/common/LlamaCpp_ModelSearch.ts
+var LlamaCpp_ModelSearch = async (input, _model, _onProgress, signal) => {
+  const entries = await searchHfModels(input.query, { filter: "gguf" }, undefined, signal);
+  const results = entries.map((entry) => mapHfModelResult(entry, LOCAL_LLAMACPP));
+  return { results };
+};
+
+// src/provider-llamacpp/common/LlamaCpp_CountTokens.ts
+import { getLogger } from "@workglow/util/worker";
+var LlamaCpp_CountTokens = async (input, model, onProgress, signal) => {
+  if (Array.isArray(input.text)) {
+    getLogger().warn("LlamaCpp_CountTokens: array input received; processing sequentially (no native batch support)");
+    const texts = input.text;
+    const counts = [];
+    for (const item of texts) {
+      const r = await LlamaCpp_CountTokens({ ...input, text: item }, model, onProgress, signal);
+      counts.push(r.count);
+    }
+    return { count: counts };
+  }
+  const loadedModel = await getOrLoadModel(model);
+  const tokens = loadedModel.tokenizer(input.text);
+  return { count: tokens.length };
+};
+var LlamaCpp_CountTokens_Reactive = async (input, _output, model) => {
+  return LlamaCpp_CountTokens(input, model, () => {}, new AbortController().signal);
+};
+
+// src/provider-llamacpp/common/LlamaCpp_Download.ts
+var LlamaCpp_Download = async (input, model, update_progress, _signal) => {
+  if (!model)
+    throw new Error("Model config is required for DownloadModelTask.");
+  const { createModelDownloader } = await loadSdk();
+  const config = model.provider_config;
+  const modelUri = config.model_url ?? config.model_path;
+  const dirPath = config.models_dir ?? LLAMACPP_DEFAULT_MODELS_DIR;
+  update_progress(0, "Creating model downloader");
+  const downloader = await createModelDownloader({ modelUri, dirPath });
+  const progressInterval = setInterval(() => {
+    const total = downloader.totalSize;
+    const downloaded = downloader.downloadedSize;
+    if (total && total > 0 && downloaded !== undefined) {
+      const pct = Math.min(99, Math.round(downloaded / total * 100));
+      update_progress(pct, "Downloading model", { file: modelUri, progress: pct });
+    }
+  }, 500);
+  let modelPath;
+  try {
+    modelPath = await downloader.download();
+  } finally {
+    clearInterval(progressInterval);
+  }
+  resolvedPaths.set(getConfigKey(model), modelPath);
+  update_progress(100, "Model downloaded", { file: modelUri, progress: 100 });
+  return { model: input.model };
+};
+
+// src/provider-llamacpp/common/LlamaCpp_ModelInfo.ts
+var LlamaCpp_ModelInfo = async (input, model) => {
+  if (!model)
+    throw new Error("Model config is required for ModelInfoTask.");
+  const modelPath = getActualModelPath(model);
+  const is_loaded = llamaCppModels.has(modelPath);
+  let is_cached = is_loaded;
+  let file_sizes = null;
+  try {
+    const fs = await import("node:fs/promises");
+    const stat = await fs.stat(modelPath);
+    is_cached = true;
+    file_sizes = { model: stat.size };
+  } catch {
+    if (resolvedPaths.has(getConfigKey(model))) {
+      is_cached = true;
+    }
+  }
+  return {
+    model: input.model,
+    is_local: true,
+    is_remote: false,
+    supports_browser: false,
+    supports_node: true,
+    is_cached,
+    is_loaded,
+    file_sizes
+  };
+};
+
+// src/provider-llamacpp/common/LlamaCpp_StructuredGeneration.ts
+import { parsePartialJson } from "@workglow/util/worker";
+var LlamaCpp_StructuredGeneration = async (input, model, update_progress, signal) => {
+  if (!model)
+    throw new Error("Model config is required for StructuredGenerationTask.");
+  await loadSdk();
+  update_progress(0, "Loading model");
+  const llama = await getLlamaInstance();
+  const context = await getOrCreateTextContext(model);
+  update_progress(10, "Running structured generation");
+  const grammar = await llama.createGrammarForJsonSchema(input.outputSchema);
+  const sequence = context.getSequence();
+  const { LlamaChatSession } = getLlamaCppSdk();
+  const session = new LlamaChatSession({ contextSequence: sequence });
+  try {
+    const text = await session.prompt(input.prompt, {
+      signal,
+      grammar,
+      ...input.temperature !== undefined && { temperature: input.temperature },
+      ...input.maxTokens !== undefined && { maxTokens: input.maxTokens }
+    });
+    let object;
+    try {
+      object = JSON.parse(text);
+    } catch {
+      object = {};
+    }
+    update_progress(100, "Structured generation complete");
+    return { object };
+  } finally {
+    sequence.dispose();
+  }
+};
+var LlamaCpp_StructuredGeneration_Stream = async function* (input, model, signal) {
+  if (!model)
+    throw new Error("Model config is required for StructuredGenerationTask.");
+  await loadSdk();
+  const llama = await getLlamaInstance();
+  const context = await getOrCreateTextContext(model);
+  const grammar = await llama.createGrammarForJsonSchema(input.outputSchema);
+  const sequence = context.getSequence();
+  const { LlamaChatSession } = getLlamaCppSdk();
+  const session = new LlamaChatSession({ contextSequence: sequence });
+  const queue = [];
+  let isComplete = false;
+  let completionError;
+  let resolveWait = null;
+  const notifyWaiter = () => {
+    resolveWait?.();
+    resolveWait = null;
+  };
+  let accumulatedText = "";
+  const promptPromise = session.prompt(input.prompt, {
+    signal,
+    grammar,
+    onTextChunk: (chunk) => {
+      queue.push(chunk);
+      notifyWaiter();
+    },
+    ...input.temperature !== undefined && { temperature: input.temperature },
+    ...input.maxTokens !== undefined && { maxTokens: input.maxTokens }
+  }).then(() => {
+    isComplete = true;
+    notifyWaiter();
+  }).catch((err) => {
+    completionError = err;
+    isComplete = true;
+    notifyWaiter();
+  });
+  try {
+    while (true) {
+      while (queue.length > 0) {
+        const chunk = queue.shift();
+        accumulatedText += chunk;
+        const partial = parsePartialJson(accumulatedText);
+        if (partial !== undefined) {
+          yield {
+            type: "object-delta",
+            port: "object",
+            objectDelta: partial
+          };
+        }
+      }
+      if (isComplete)
+        break;
+      await new Promise((r) => {
+        resolveWait = r;
+      });
+    }
+    while (queue.length > 0) {
+      const chunk = queue.shift();
+      accumulatedText += chunk;
+    }
+  } finally {
+    await promptPromise.catch(() => {});
+    sequence.dispose();
+  }
+  if (completionError) {
+    if (signal.aborted)
+      return;
+    throw completionError;
+  }
+  let finalObject;
+  try {
+    finalObject = JSON.parse(accumulatedText);
+  } catch {
+    finalObject = parsePartialJson(accumulatedText) ?? {};
+  }
+  yield { type: "finish", data: { object: finalObject } };
+};
+
+// src/provider-llamacpp/common/LlamaCpp_TextEmbedding.ts
+var LlamaCpp_TextEmbedding = async (input, model, update_progress, _signal) => {
+  if (!model)
+    throw new Error("Model config is required for TextEmbeddingTask.");
+  update_progress(0, "Loading embedding model");
+  const context = await getOrCreateEmbeddingContext(model);
+  const texts = Array.isArray(input.text) ? input.text : [input.text];
+  update_progress(10, "Computing embeddings");
+  const embeddings = await Promise.all(texts.map((text) => context.getEmbeddingFor(text).then((e) => new Float32Array(e.vector))));
+  update_progress(100, "Embeddings complete");
+  if (Array.isArray(input.text)) {
+    return { vector: embeddings };
+  }
+  return { vector: embeddings[0] };
+};
+
+// src/provider-llamacpp/common/LlamaCpp_TextGeneration.ts
+import { getLogger as getLogger2 } from "@workglow/util/worker";
+var LlamaCpp_TextGeneration = async (input, model, update_progress, signal) => {
+  if (Array.isArray(input.prompt)) {
+    getLogger2().warn("LlamaCpp_TextGeneration: array input received; processing sequentially (no native batch support)");
+    const prompts = input.prompt;
+    const results = [];
+    for (const item of prompts) {
+      const r = await LlamaCpp_TextGeneration({ ...input, prompt: item }, model, update_progress, signal);
+      results.push(r.text);
+    }
+    return { text: results };
+  }
+  if (!model)
+    throw new Error("Model config is required for TextGenerationTask.");
+  const { LlamaChatSession } = await loadSdk();
+  update_progress(0, "Loading model");
+  const context = await getOrCreateTextContext(model);
+  update_progress(10, "Generating text");
+  const sequence = context.getSequence();
+  const session = new LlamaChatSession({ contextSequence: sequence });
+  try {
+    const text = await session.prompt(input.prompt, {
+      signal,
+      ...input.temperature !== undefined && { temperature: input.temperature },
+      ...input.maxTokens !== undefined && { maxTokens: input.maxTokens },
+      ...input.topP !== undefined && { topP: input.topP }
+    });
+    update_progress(100, "Text generation complete");
+    return { text };
+  } finally {
+    sequence.dispose();
+  }
+};
+var LlamaCpp_TextGeneration_Stream = async function* (input, model, signal) {
+  if (!model)
+    throw new Error("Model config is required for TextGenerationTask.");
+  const { LlamaChatSession } = await loadSdk();
+  const context = await getOrCreateTextContext(model);
+  const sequence = context.getSequence();
+  const session = new LlamaChatSession({ contextSequence: sequence });
+  try {
+    yield* streamFromSession((onTextChunk) => {
+      return session.prompt(input.prompt, {
+        signal,
+        onTextChunk,
+        ...input.temperature !== undefined && { temperature: input.temperature },
+        ...input.maxTokens !== undefined && { maxTokens: input.maxTokens },
+        ...input.topP !== undefined && { topP: input.topP }
+      });
+    }, signal);
+  } finally {
+    sequence.dispose();
+  }
+};
+
+// src/provider-llamacpp/common/LlamaCpp_TextRewriter.ts
+import { getLogger as getLogger3 } from "@workglow/util/worker";
+var LlamaCpp_TextRewriter = async (input, model, update_progress, signal) => {
+  if (Array.isArray(input.text)) {
+    getLogger3().warn("LlamaCpp_TextRewriter: array input received; processing sequentially (no native batch support)");
+    const texts = input.text;
+    const results = [];
+    for (const item of texts) {
+      const r = await LlamaCpp_TextRewriter({ ...input, text: item }, model, update_progress, signal);
+      results.push(r.text);
+    }
+    return { text: results };
+  }
+  if (!model)
+    throw new Error("Model config is required for TextRewriterTask.");
+  const { LlamaChatSession } = await loadSdk();
+  update_progress(0, "Loading model");
+  const context = await getOrCreateTextContext(model);
+  update_progress(10, "Rewriting text");
+  const sequence = context.getSequence();
+  const session = new LlamaChatSession({
+    contextSequence: sequence,
+    systemPrompt: input.prompt
+  });
+  try {
+    const text = await session.prompt(input.text, { signal });
+    update_progress(100, "Text rewriting complete");
+    return { text };
+  } finally {
+    sequence.dispose();
+  }
+};
+var LlamaCpp_TextRewriter_Stream = async function* (input, model, signal) {
+  if (!model)
+    throw new Error("Model config is required for TextRewriterTask.");
+  const { LlamaChatSession } = await loadSdk();
+  const context = await getOrCreateTextContext(model);
+  const sequence = context.getSequence();
+  const session = new LlamaChatSession({
+    contextSequence: sequence,
+    systemPrompt: input.prompt
+  });
+  try {
+    yield* streamFromSession((onTextChunk) => {
+      return session.prompt(input.text, { signal, onTextChunk });
+    }, signal);
+  } finally {
+    sequence.dispose();
+  }
+};
+
+// src/provider-llamacpp/common/LlamaCpp_TextSummary.ts
+import { getLogger as getLogger4 } from "@workglow/util/worker";
+var LlamaCpp_TextSummary = async (input, model, update_progress, signal) => {
+  if (Array.isArray(input.text)) {
+    getLogger4().warn("LlamaCpp_TextSummary: array input received; processing sequentially (no native batch support)");
+    const texts = input.text;
+    const results = [];
+    for (const item of texts) {
+      const r = await LlamaCpp_TextSummary({ ...input, text: item }, model, update_progress, signal);
+      results.push(r.text);
+    }
+    return { text: results };
+  }
+  if (!model)
+    throw new Error("Model config is required for TextSummaryTask.");
+  const { LlamaChatSession } = await loadSdk();
+  update_progress(0, "Loading model");
+  const context = await getOrCreateTextContext(model);
+  update_progress(10, "Summarizing text");
+  const sequence = context.getSequence();
+  const session = new LlamaChatSession({
+    contextSequence: sequence,
+    systemPrompt: "Summarize the following text concisely, preserving the key points."
+  });
+  try {
+    const text = await session.prompt(input.text, { signal });
+    update_progress(100, "Summarization complete");
+    return { text };
+  } finally {
+    sequence.dispose();
+  }
+};
+var LlamaCpp_TextSummary_Stream = async function* (input, model, signal) {
+  if (!model)
+    throw new Error("Model config is required for TextSummaryTask.");
+  const { LlamaChatSession } = await loadSdk();
+  const context = await getOrCreateTextContext(model);
+  const sequence = context.getSequence();
+  const session = new LlamaChatSession({
+    contextSequence: sequence,
+    systemPrompt: "Summarize the following text concisely, preserving the key points."
+  });
+  try {
+    yield* streamFromSession((onTextChunk) => {
+      return session.prompt(input.text, { signal, onTextChunk });
+    }, signal);
+  } finally {
+    sequence.dispose();
+  }
+};
+
+// src/provider-llamacpp/common/LlamaCpp_ToolCalling.ts
+import { filterValidToolCalls } from "@workglow/ai/worker";
+import { getLogger as getLogger5 } from "@workglow/util/worker";
+function buildLlamaCppPrompt(input) {
+  const inputMessages = input.messages;
+  if (!inputMessages || inputMessages.length === 0) {
+    return Array.isArray(input.prompt) ? input.prompt.join(`
+`) : input.prompt;
+  }
+  const parts = [];
+  for (const msg of inputMessages) {
+    if (msg.role === "user") {
+      parts.push(`User: ${msg.content}`);
+    } else if (msg.role === "assistant" && Array.isArray(msg.content)) {
+      const text = msg.content.filter((b) => b.type === "text").map((b) => b.text).join("");
+      if (text)
+        parts.push(`Assistant: ${text}`);
+    } else if (msg.role === "tool" && Array.isArray(msg.content)) {
+      for (const block of msg.content) {
+        parts.push(`Tool Result: ${block.content}`);
+      }
+    }
+  }
+  return parts.join(`
+
+`);
+}
+function buildLlamaCppFunctions(tools, capturedCalls) {
+  const { defineChatSessionFunction } = getLlamaCppSdk();
+  const functions = {};
+  for (const tool of tools) {
+    const toolName = tool.name;
+    functions[toolName] = defineChatSessionFunction({
+      description: tool.description,
+      params: tool.inputSchema,
+      handler(params) {
+        capturedCalls.push({ name: toolName, input: params ?? {} });
+        return "OK";
+      }
+    });
+  }
+  return functions;
+}
+var LlamaCpp_ToolCalling = async (input, model, update_progress, signal) => {
+  if (Array.isArray(input.prompt)) {
+    getLogger5().warn("LlamaCpp_ToolCalling: array input received; processing sequentially (no native batch support)");
+    const prompts = input.prompt;
+    const texts = [];
+    const toolCallsList = [];
+    for (const item of prompts) {
+      const r = await LlamaCpp_ToolCalling({ ...input, prompt: item }, model, update_progress, signal);
+      texts.push(r.text);
+      toolCallsList.push(r.toolCalls);
+    }
+    return { text: texts, toolCalls: toolCallsList };
+  }
+  if (!model)
+    throw new Error("Model config is required for ToolCallingTask.");
+  await loadSdk();
+  update_progress(0, "Loading model");
+  const context = await getOrCreateTextContext(model);
+  const capturedCalls = [];
+  const functions = input.toolChoice === "none" ? undefined : buildLlamaCppFunctions(input.tools, capturedCalls);
+  update_progress(10, "Running tool calling");
+  const sequence = context.getSequence();
+  const { LlamaChatSession } = getLlamaCppSdk();
+  const promptText = buildLlamaCppPrompt(input);
+  const session = new LlamaChatSession({
+    contextSequence: sequence,
+    ...input.systemPrompt && { systemPrompt: input.systemPrompt }
+  });
+  try {
+    const text = await session.prompt(promptText, {
+      signal,
+      ...functions && { functions },
+      ...input.temperature !== undefined && { temperature: input.temperature },
+      ...input.maxTokens !== undefined && { maxTokens: input.maxTokens }
+    });
+    const toolCalls = [];
+    capturedCalls.forEach((call, index) => {
+      const id = `call_${index}`;
+      toolCalls.push({ id, name: call.name, input: call.input });
+    });
+    update_progress(100, "Tool calling complete");
+    return { text, toolCalls: filterValidToolCalls(toolCalls, input.tools) };
+  } finally {
+    sequence.dispose();
+  }
+};
+var LlamaCpp_ToolCalling_Stream = async function* (input, model, signal) {
+  if (!model)
+    throw new Error("Model config is required for ToolCallingTask.");
+  await loadSdk();
+  const context = await getOrCreateTextContext(model);
+  const capturedCalls = [];
+  const functions = input.toolChoice === "none" ? undefined : buildLlamaCppFunctions(input.tools, capturedCalls);
+  const sequence = context.getSequence();
+  const { LlamaChatSession } = getLlamaCppSdk();
+  const promptText = buildLlamaCppPrompt(input);
+  const session = new LlamaChatSession({
+    contextSequence: sequence,
+    ...input.systemPrompt && { systemPrompt: input.systemPrompt }
+  });
+  const queue = [];
+  let isComplete = false;
+  let completionError;
+  let resolveWait = null;
+  const notifyWaiter = () => {
+    resolveWait?.();
+    resolveWait = null;
+  };
+  let accumulatedText = "";
+  const promptPromise = session.prompt(promptText, {
+    signal,
+    ...functions && { functions },
+    onTextChunk: (chunk) => {
+      queue.push(chunk);
+      notifyWaiter();
+    },
+    ...input.temperature !== undefined && { temperature: input.temperature },
+    ...input.maxTokens !== undefined && { maxTokens: input.maxTokens }
+  }).then(() => {
+    isComplete = true;
+    notifyWaiter();
+  }).catch((err) => {
+    completionError = err;
+    isComplete = true;
+    notifyWaiter();
+  });
+  try {
+    while (true) {
+      while (queue.length > 0) {
+        const chunk = queue.shift();
+        accumulatedText += chunk;
+        yield { type: "text-delta", port: "text", textDelta: chunk };
+      }
+      if (isComplete)
+        break;
+      await new Promise((r) => {
+        resolveWait = r;
+      });
+    }
+    while (queue.length > 0) {
+      const chunk = queue.shift();
+      accumulatedText += chunk;
+      yield { type: "text-delta", port: "text", textDelta: chunk };
+    }
+  } finally {
+    await promptPromise.catch(() => {});
+    sequence.dispose();
+  }
+  if (completionError) {
+    if (!signal.aborted)
+      throw completionError;
+    return;
+  }
+  const toolCalls = [];
+  capturedCalls.forEach((call, index) => {
+    const id = `call_${index}`;
+    toolCalls.push({ id, name: call.name, input: call.input });
+  });
+  const validToolCalls = filterValidToolCalls(toolCalls, input.tools);
+  if (validToolCalls.length > 0) {
+    yield { type: "object-delta", port: "toolCalls", objectDelta: [...validToolCalls] };
+  }
+  yield {
+    type: "finish",
+    data: { text: accumulatedText, toolCalls: validToolCalls }
+  };
+};
+
+// src/provider-llamacpp/common/LlamaCpp_Unload.ts
+var LlamaCpp_Unload = async (input, model, update_progress, _signal) => {
+  if (!model)
+    throw new Error("Model config is required for UnloadModelTask.");
+  const modelPath = getActualModelPath(model);
+  const ctx = llamaCppTextContexts.get(modelPath);
+  if (ctx) {
+    await ctx.dispose();
+    llamaCppTextContexts.delete(modelPath);
+    update_progress(33, "Text context disposed");
+  }
+  const embCtx = llamaCppEmbeddingContexts.get(modelPath);
+  if (embCtx) {
+    await embCtx.dispose();
+    llamaCppEmbeddingContexts.delete(modelPath);
+    update_progress(66, "Embedding context disposed");
+  }
+  const cachedModel = llamaCppModels.get(modelPath);
+  if (cachedModel) {
+    await cachedModel.dispose();
+    llamaCppModels.delete(modelPath);
+    update_progress(100, "Model unloaded from memory");
+  } else {
+    update_progress(100, "Model was not loaded");
+  }
+  return { model: input.model };
+};
+
+// src/provider-llamacpp/common/LlamaCpp_JobRunFns.ts
+var LLAMACPP_TASKS = {
+  DownloadModelTask: LlamaCpp_Download,
+  UnloadModelTask: LlamaCpp_Unload,
+  ModelInfoTask: LlamaCpp_ModelInfo,
+  CountTokensTask: LlamaCpp_CountTokens,
+  TextGenerationTask: LlamaCpp_TextGeneration,
+  TextEmbeddingTask: LlamaCpp_TextEmbedding,
+  TextRewriterTask: LlamaCpp_TextRewriter,
+  TextSummaryTask: LlamaCpp_TextSummary,
+  ToolCallingTask: LlamaCpp_ToolCalling,
+  StructuredGenerationTask: LlamaCpp_StructuredGeneration,
+  ModelSearchTask: LlamaCpp_ModelSearch
+};
+var LLAMACPP_STREAM_TASKS = {
+  TextGenerationTask: LlamaCpp_TextGeneration_Stream,
+  TextRewriterTask: LlamaCpp_TextRewriter_Stream,
+  TextSummaryTask: LlamaCpp_TextSummary_Stream,
+  ToolCallingTask: LlamaCpp_ToolCalling_Stream,
+  StructuredGenerationTask: LlamaCpp_StructuredGeneration_Stream
+};
+var LLAMACPP_REACTIVE_TASKS = {
+  CountTokensTask: LlamaCpp_CountTokens_Reactive
+};
+
+// src/provider-llamacpp/LlamaCppQueuedProvider.ts
+import {
+  QueuedAiProvider
+} from "@workglow/ai";
+class LlamaCppQueuedProvider extends QueuedAiProvider {
+  name = LOCAL_LLAMACPP;
+  isLocal = true;
+  supportsBrowser = false;
+  taskTypes = [
+    "DownloadModelTask",
+    "UnloadModelTask",
+    "ModelInfoTask",
+    "CountTokensTask",
+    "TextGenerationTask",
+    "TextEmbeddingTask",
+    "TextRewriterTask",
+    "TextSummaryTask",
+    "ToolCallingTask",
+    "ModelSearchTask"
+  ];
+  constructor(tasks, streamTasks, reactiveTasks) {
+    super(tasks, streamTasks, reactiveTasks);
+  }
+}
+
+// src/provider-llamacpp/registerLlamaCppInline.ts
+async function registerLlamaCppInline(options) {
+  await new LlamaCppQueuedProvider(LLAMACPP_TASKS, LLAMACPP_STREAM_TASKS, LLAMACPP_REACTIVE_TASKS).register(options ?? {});
+}
+// src/provider-llamacpp/registerLlamaCppWorker.ts
+import { getLogger as getLogger6, globalServiceRegistry, WORKER_SERVER } from "@workglow/util/worker";
+
+// src/provider-llamacpp/LlamaCppProvider.ts
+import {
+  AiProvider
+} from "@workglow/ai/worker";
+class LlamaCppProvider extends AiProvider {
+  name = LOCAL_LLAMACPP;
+  isLocal = true;
+  supportsBrowser = false;
+  taskTypes = [
+    "DownloadModelTask",
+    "UnloadModelTask",
+    "ModelInfoTask",
+    "CountTokensTask",
+    "TextGenerationTask",
+    "TextEmbeddingTask",
+    "TextRewriterTask",
+    "TextSummaryTask",
+    "ToolCallingTask",
+    "ModelSearchTask"
+  ];
+  constructor(tasks, streamTasks, reactiveTasks) {
+    super(tasks, streamTasks, reactiveTasks);
+  }
+}
+
+// src/provider-llamacpp/registerLlamaCppWorker.ts
+async function registerLlamaCppWorker() {
+  const workerServer = globalServiceRegistry.get(WORKER_SERVER);
+  new LlamaCppProvider(LLAMACPP_TASKS, LLAMACPP_STREAM_TASKS, LLAMACPP_REACTIVE_TASKS).registerOnWorkerServer(workerServer);
+  workerServer.sendReady();
+  getLogger6().info("LlamaCpp worker job run functions registered");
+}
+export {
+  streamFromSession,
+  resolvedPaths,
+  registerLlamaCppWorker,
+  registerLlamaCppInline,
+  loadSdk,
+  llamaCppTextContexts,
+  llamaCppModels,
+  llamaCppEmbeddingContexts,
+  getOrLoadModel,
+  getOrCreateTextContext,
+  getOrCreateEmbeddingContext,
+  getLlamaInstance,
+  getLlamaCppSdk,
+  getConfigKey,
+  getActualModelPath,
+  disposeLlamaCppResources
+};
+
+//# debugId=B6040A734F2E794D64756E2164756E21