@nemesis-oss/ollama-sdk 1.13.0 → 1.15.0

package/dist/index.js

@@ -2815,12 +2815,29 @@ var ModelsClient = class {
    * await client.models.createModelFromGguf('my-model', '/path/to/model.gguf', {
    *   template: '{{ .Prompt }}',
    * });
+   *
+   * // Speculative decoding: primary + draft shards + tuning:
+   * await client.models.createModelFromGguf('my-model', '/path/to/model.gguf', {
+   *   draftGguf: '/path/to/draft.gguf',
+   *   draftQuantize: 'q8_0',
+   *   parameters: { draft_num_predict: 8 },
+   * });
    * ```
    *
+   * Speculative-decoding models assemble the same way: pass the draft
+   * network's GGUF(s) as `draftGguf` (uploaded into `draft_files`), optionally
+   * set `draft_quantize` to compress the draft weights independently of the
+   * primary model, and set `parameters.draft_num_predict` to tune how many
+   * tokens the draft network predicts per step (Ollama defaults separate
+   * draft models to 4; `0` disables drafting; models with embedded MTP
+   * tensors require setting it explicitly).
+   *
    * @param model Name for the new model (e.g. `'my-model:latest'`).
    * @param gguf Path(s) to GGUF file(s) on the local filesystem.
    * @param opts Everything else `/api/create` accepts (`template`, `system`,
-   *   `parameters`, `license`, `quantize`, …) except `files` and `from`.
+   *   `parameters`, `license`, `quantize`, `draftGguf`, …) except `files`,
+   *   `draft_files` (use `draftGguf` for path-based drafts; an explicit digest
+   *   map is still accepted and merged), `from`, and `stream`.
    */
   async createModelFromGguf(model, gguf, opts) {
     const paths = typeof gguf === "string" ? [gguf] : gguf;
@@ -2834,14 +2851,29 @@ var ModelsClient = class {
       const uploaded = await this.createBlobFromFile(path, { signal: opts?.signal });
       files[uploaded.fileName] = uploaded.digest;
     }
+    const draftOpt = opts?.draftGguf;
+    const draftPaths = typeof draftOpt === "string" ? [draftOpt] : draftOpt ?? [];
     const {
       model: _model,
       files: _files,
       from: _from,
       stream: _stream,
+      draftGguf: _draftGguf,
+      draft_files: _draftFiles,
       ...rest
     } = opts ?? {};
-    return this.create({ ...rest, model, files, stream: false });
+    const draftFiles = { ..._draftFiles ?? {} };
+    for (const path of draftPaths) {
+      const uploaded = await this.createBlobFromFile(path, { signal: opts?.signal });
+      draftFiles[uploaded.fileName] = uploaded.digest;
+    }
+    return this.create({
+      ...rest,
+      model,
+      files,
+      ...Object.keys(draftFiles).length > 0 ? { draft_files: draftFiles } : {},
+      stream: false
+    });
   }
 };
 
@@ -7716,6 +7748,111 @@ var Agent = class {
   }
 };
 
+// src/draft-benchmark.ts
+var DEFAULT_DRAFT_TOKENS = 4;
+var DEFAULT_SAMPLES = 5;
+var DEFAULT_NUM_PREDICT = 256;
+var DEFAULT_SEED = 42;
+var DEFAULT_GAIN_THRESHOLD = 1.05;
+function median(values) {
+  const sorted = [...values].sort((a, b) => a - b);
+  const mid = Math.floor(sorted.length / 2);
+  return sorted.length % 2 === 1 ? sorted[mid] ?? 0 : ((sorted[mid - 1] ?? 0) + (sorted[mid] ?? 0)) / 2;
+}
+function roundTo(value, decimals) {
+  const factor = 10 ** decimals;
+  return Math.round(value * factor) / factor;
+}
+function requirePositiveInteger(name, value) {
+  if (!Number.isInteger(value) || value < 1) {
+    throw new RangeError(
+      `compareDraftPerformance: ${name} must be an integer >= 1 (got ${value}).`
+    );
+  }
+}
+async function runSide(client, options, draftNumPredict, samples, numPredict, seed) {
+  const samplingOptions = {
+    num_predict: numPredict,
+    temperature: 0,
+    seed,
+    draft_num_predict: draftNumPredict
+  };
+  if (options.warmup ?? true) {
+    await client.generate({
+      model: options.model,
+      prompt: options.prompt,
+      options: { ...samplingOptions, num_predict: 1 },
+      stream: false,
+      signal: options.signal
+    });
+  }
+  const tokensPerSecond = [];
+  const generationMs = [];
+  for (let i = 0; i < samples; i++) {
+    options.signal?.throwIfAborted();
+    const response = await client.generate({
+      model: options.model,
+      prompt: options.prompt,
+      options: samplingOptions,
+      stream: false,
+      signal: options.signal
+    });
+    const telemetry = formatTelemetry(response);
+    if (Number.isFinite(telemetry.tokensPerSecond) && telemetry.tokensPerSecond > 0) {
+      tokensPerSecond.push(telemetry.tokensPerSecond);
+      generationMs.push(telemetry.generationMs);
+    }
+  }
+  if (tokensPerSecond.length === 0) {
+    throw new OllamaClientError(
+      `compareDraftPerformance: every draft_num_predict=${draftNumPredict} sample returned a non-positive tokens/second. The model name is likely wrong, or the server returned empty generations.`,
+      { code: "draft_benchmark_no_valid_samples" }
+    );
+  }
+  return {
+    draftNumPredict,
+    medianTokensPerSecond: median(tokensPerSecond),
+    sampleTokensPerSecond: tokensPerSecond,
+    medianEvalDurationMs: median(generationMs)
+  };
+}
+async function compareDraftPerformance(client, options) {
+  const draftNumPredict = options.draftNumPredict ?? DEFAULT_DRAFT_TOKENS;
+  requirePositiveInteger("draftNumPredict", draftNumPredict);
+  const samples = options.samples ?? DEFAULT_SAMPLES;
+  requirePositiveInteger("samples", samples);
+  const numPredict = options.numPredict ?? DEFAULT_NUM_PREDICT;
+  requirePositiveInteger("numPredict", numPredict);
+  const seed = options.seed ?? DEFAULT_SEED;
+  if (!Number.isInteger(seed)) {
+    throw new RangeError(`compareDraftPerformance: seed must be an integer (got ${seed}).`);
+  }
+  const gainThreshold = options.gainThreshold ?? DEFAULT_GAIN_THRESHOLD;
+  if (!Number.isFinite(gainThreshold) || gainThreshold <= 0) {
+    throw new RangeError(
+      `compareDraftPerformance: gainThreshold must be a positive finite number (got ${gainThreshold}).`
+    );
+  }
+  options.signal?.throwIfAborted();
+  const baseline = await runSide(client, options, 0, samples, numPredict, seed);
+  const speculative = await runSide(client, options, draftNumPredict, samples, numPredict, seed);
+  const baselineTokensPerSec = roundTo(baseline.medianTokensPerSecond, 1);
+  const speculativeTokensPerSec = roundTo(speculative.medianTokensPerSecond, 1);
+  const speedupMultiplier = baselineTokensPerSec > 0 ? roundTo(speculativeTokensPerSec / baselineTokensPerSec, 2) : 0;
+  return {
+    model: options.model,
+    prompt: options.prompt,
+    draftNumPredict,
+    baselineTokensPerSec,
+    speculativeTokensPerSec,
+    speedupMultiplier,
+    effectiveGain: speedupMultiplier > gainThreshold,
+    gainThreshold,
+    baseline,
+    speculative
+  };
+}
+
 // src/affinity.ts
 var DEFAULT_CONCURRENT_MODELS = 1;
 var DEFAULT_PER_MODEL_CONCURRENCY = 1;
@@ -7877,6 +8014,6 @@ function estimateSystemOneRequestBytes(request) {
   return new TextEncoder().encode(JSON.stringify(request)).byteLength;
 }
 
-export { ATTR_GEN_AI_OPERATION_NAME, ATTR_GEN_AI_REQUEST_MODEL, ATTR_GEN_AI_RESPONSE_MODEL, ATTR_GEN_AI_SYSTEM, ATTR_GEN_AI_TOOL_CALL_ID, ATTR_GEN_AI_TOOL_NAME, ATTR_GEN_AI_USAGE_INPUT_TOKENS, ATTR_GEN_AI_USAGE_OUTPUT_TOKENS, ATTR_HTTP_REQUEST_METHOD, ATTR_HTTP_RESPONSE_STATUS_CODE, ATTR_OLLAMA_AGENT_ITERATION, ATTR_OLLAMA_AGENT_MAX_ITERATIONS, ATTR_OLLAMA_ENDPOINT_ATTEMPT, ATTR_OLLAMA_ENDPOINT_NAME, ATTR_SERVER_ADDRESS, ATTR_SERVER_PORT, ATTR_URL_FULL, Agent, AnthropicCompatClient, AnthropicMessagesStream, ConversationSession, DEFAULT_BACKOFF, DEFAULT_BASE_URL, DEFAULT_CONTEXT_WARNING_THRESHOLD, DEFAULT_EMBED_BATCH_SIZE, DEFAULT_EMBED_CONCURRENCY, DEFAULT_FAILOVER_CODES, DEFAULT_FETCH_BACKOFF_CONFIG, DEFAULT_RETRY_CONFIG, DEFAULT_TIMEOUT_MS, EndpointRegistry, GEN_AI_SYSTEM_OLLAMA, HttpClient, IMAGE_TOKEN_ESTIMATE, KEEP_ALIVE_INDEFINITE, KEEP_ALIVE_UNLOAD, KNOWN_DONE_REASONS, MAX_SYSTEM_ONE_IMAGES_BYTES, MAX_SYSTEM_ONE_REQUEST_BYTES, MESSAGE_OVERHEAD_TOKENS, McpBridge, ModelAffinityRouter, ModelAffinityScheduler, ModelsClient, NANOS_PER_MS, NANOS_PER_SECOND, NOOP_LOGGER, OLLAMA_CLOUD_BASE_URL, OLLAMA_FALLBACK_CONTEXT_LENGTH, OllamaAbortError, OllamaAgentMaxIterationsError, OllamaAgentMaxToolCallsError, OllamaAgentToolLoopError, OllamaAuthError, OllamaBadGatewayError, OllamaClient, OllamaClientError, OllamaGenericClientError, OllamaIncompatibleModelError, OllamaMcpError, OllamaModelRoutingError, OllamaNetworkError, OllamaNotFoundError, OllamaOverloadedError, OllamaQuotaExceededError, OllamaRateLimitError, OllamaRequestTooLargeError, OllamaRequestValidationError, OllamaResponseValidationError, OllamaServerError, OllamaServerVersionUnknownError, OllamaSkillInvalidError, OllamaSkillNotFoundError, OllamaStream, OllamaStreamError, OllamaTimeoutError, OllamaToolTimeoutError, OllamaToolValidationError, OllamaUnsupportedCapabilityError, OpenAIChatCompletionStream, OpenAICompatClient, OpenAICompletionStream, OpenAIResponsesStream, OpenAIResponsesStreamError, QuotaManager, RETRYABLE_STATUS_CODES, ResponsesModule, ToolCallAccumulator, ToolRegistry, WEB_FETCH_TOOL_NAME, WEB_SEARCH_TOOL_NAME, addSpanEvent, applySkill, batchEmbed, calculateBackoff, canonicalToolCallSignature, checkChatContext, checkEndpointHealth, checkGenerateContext, compactConversationHistory, composeMiddleware, contextWarningMessage, createConsoleLogger, createDecision, createModelAffinityScheduler, createOllamaCloudFreeTierQuota, createTimeoutSignal, createWebFetchTool, createWebSearchTool, createWebTools, defineTool, detectModelCapabilities, embedBatchOverflowMessage, encodeImage, estimateChatRequestTokens, estimateGenerateRequestTokens, estimateMessageTokens, estimateSystemOneRequestBytes, estimateTokens, extractContextLength as extractNativeContextLength, extractParameterNumCtx, extractUsage, fetchWithBackoff, findOversizedEmbedInputs, findRunningModelContextLength, formatTelemetry, formatWebFetchResult, formatWebSearchResult, imageStringNeedsResolution, inferRuntimeMode, isKeepAliveSugar, isKnownDoneReason, isModelResident, isSameToolCall, listAllMcpTools, listAvailableModels, loadMcpTools, mapError, mergeToolCall, mergeToolCallArgumentsString, mergeToolCallArrays, normalizeChatStream, normalizeGenerateStream, normalizeKeepAlive, normalizeProgressStream, parseFrontmatter, parseNdjsonStream, parseSseStream, parseStructuredOutput, registerMcpTools, resolveContextLength, resolveCredentialEndpoints, resolveImageInput, sanitizeHistoryForNextTurn, setSpanError, toDataStream, toResponse, toTextStream, withRetry, withSpan, zodToJsonSchema };
+export { ATTR_GEN_AI_OPERATION_NAME, ATTR_GEN_AI_REQUEST_MODEL, ATTR_GEN_AI_RESPONSE_MODEL, ATTR_GEN_AI_SYSTEM, ATTR_GEN_AI_TOOL_CALL_ID, ATTR_GEN_AI_TOOL_NAME, ATTR_GEN_AI_USAGE_INPUT_TOKENS, ATTR_GEN_AI_USAGE_OUTPUT_TOKENS, ATTR_HTTP_REQUEST_METHOD, ATTR_HTTP_RESPONSE_STATUS_CODE, ATTR_OLLAMA_AGENT_ITERATION, ATTR_OLLAMA_AGENT_MAX_ITERATIONS, ATTR_OLLAMA_ENDPOINT_ATTEMPT, ATTR_OLLAMA_ENDPOINT_NAME, ATTR_SERVER_ADDRESS, ATTR_SERVER_PORT, ATTR_URL_FULL, Agent, AnthropicCompatClient, AnthropicMessagesStream, ConversationSession, DEFAULT_BACKOFF, DEFAULT_BASE_URL, DEFAULT_CONTEXT_WARNING_THRESHOLD, DEFAULT_EMBED_BATCH_SIZE, DEFAULT_EMBED_CONCURRENCY, DEFAULT_FAILOVER_CODES, DEFAULT_FETCH_BACKOFF_CONFIG, DEFAULT_RETRY_CONFIG, DEFAULT_TIMEOUT_MS, EndpointRegistry, GEN_AI_SYSTEM_OLLAMA, HttpClient, IMAGE_TOKEN_ESTIMATE, KEEP_ALIVE_INDEFINITE, KEEP_ALIVE_UNLOAD, KNOWN_DONE_REASONS, MAX_SYSTEM_ONE_IMAGES_BYTES, MAX_SYSTEM_ONE_REQUEST_BYTES, MESSAGE_OVERHEAD_TOKENS, McpBridge, ModelAffinityRouter, ModelAffinityScheduler, ModelsClient, NANOS_PER_MS, NANOS_PER_SECOND, NOOP_LOGGER, OLLAMA_CLOUD_BASE_URL, OLLAMA_FALLBACK_CONTEXT_LENGTH, OllamaAbortError, OllamaAgentMaxIterationsError, OllamaAgentMaxToolCallsError, OllamaAgentToolLoopError, OllamaAuthError, OllamaBadGatewayError, OllamaClient, OllamaClientError, OllamaGenericClientError, OllamaIncompatibleModelError, OllamaMcpError, OllamaModelRoutingError, OllamaNetworkError, OllamaNotFoundError, OllamaOverloadedError, OllamaQuotaExceededError, OllamaRateLimitError, OllamaRequestTooLargeError, OllamaRequestValidationError, OllamaResponseValidationError, OllamaServerError, OllamaServerVersionUnknownError, OllamaSkillInvalidError, OllamaSkillNotFoundError, OllamaStream, OllamaStreamError, OllamaTimeoutError, OllamaToolTimeoutError, OllamaToolValidationError, OllamaUnsupportedCapabilityError, OpenAIChatCompletionStream, OpenAICompatClient, OpenAICompletionStream, OpenAIResponsesStream, OpenAIResponsesStreamError, QuotaManager, RETRYABLE_STATUS_CODES, ResponsesModule, ToolCallAccumulator, ToolRegistry, WEB_FETCH_TOOL_NAME, WEB_SEARCH_TOOL_NAME, addSpanEvent, applySkill, batchEmbed, calculateBackoff, canonicalToolCallSignature, checkChatContext, checkEndpointHealth, checkGenerateContext, compactConversationHistory, compareDraftPerformance, composeMiddleware, contextWarningMessage, createConsoleLogger, createDecision, createModelAffinityScheduler, createOllamaCloudFreeTierQuota, createTimeoutSignal, createWebFetchTool, createWebSearchTool, createWebTools, defineTool, detectModelCapabilities, embedBatchOverflowMessage, encodeImage, estimateChatRequestTokens, estimateGenerateRequestTokens, estimateMessageTokens, estimateSystemOneRequestBytes, estimateTokens, extractContextLength as extractNativeContextLength, extractParameterNumCtx, extractUsage, fetchWithBackoff, findOversizedEmbedInputs, findRunningModelContextLength, formatTelemetry, formatWebFetchResult, formatWebSearchResult, imageStringNeedsResolution, inferRuntimeMode, isKeepAliveSugar, isKnownDoneReason, isModelResident, isSameToolCall, listAllMcpTools, listAvailableModels, loadMcpTools, mapError, mergeToolCall, mergeToolCallArgumentsString, mergeToolCallArrays, normalizeChatStream, normalizeGenerateStream, normalizeKeepAlive, normalizeProgressStream, parseFrontmatter, parseNdjsonStream, parseSseStream, parseStructuredOutput, registerMcpTools, resolveContextLength, resolveCredentialEndpoints, resolveImageInput, sanitizeHistoryForNextTurn, setSpanError, toDataStream, toResponse, toTextStream, withRetry, withSpan, zodToJsonSchema };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map