@promptbook/ollama 0.103.0-67 → 0.103.0-68

package/esm/typings/src/execution/PromptResult.d.ts

@@ -8,7 +8,7 @@ import type { Usage } from './Usage';
  *
  * @see https://github.com/webgptorg/promptbook#prompt-result
  */
-export type PromptResult = CompletionPromptResult | ChatPromptResult | EmbeddingPromptResult;
+export type PromptResult = CompletionPromptResult | ChatPromptResult | ImagePromptResult | EmbeddingPromptResult;
 /**
  * Completion prompt result
  *
@@ -22,6 +22,12 @@ export type CompletionPromptResult = CommonPromptResult;
  * Note: [🚉] This is fully serializable as JSON
  */
 export type ChatPromptResult = CommonPromptResult & {};
+/**
+ * Image prompt result
+ *
+ * Note: [🚉] This is fully serializable as JSON
+ */
+export type ImagePromptResult = CommonPromptResult;
 /**
  * Embedding prompt  result
  *

package/esm/typings/src/llm-providers/ollama/OllamaExecutionTools.d.ts

@@ -38,6 +38,10 @@ export declare class OllamaExecutionTools extends OpenAiCompatibleExecutionTools
      * Default model for completion variant.
      */
     protected getDefaultEmbeddingModel(): AvailableModel;
+    /**
+     * Default model for image generation variant.
+     */
+    protected getDefaultImageGenerationModel(): AvailableModel;
 }
 /**
  * TODO: [🛄] Some way how to re-wrap the errors from `OpenAiCompatibleExecutionTools`

package/esm/typings/src/llm-providers/openai/OpenAiCompatibleExecutionTools.d.ts

@@ -1,7 +1,7 @@
 import OpenAI from 'openai';
 import type { AvailableModel } from '../../execution/AvailableModel';
 import type { LlmExecutionTools } from '../../execution/LlmExecutionTools';
-import type { ChatPromptResult, CompletionPromptResult, EmbeddingPromptResult } from '../../execution/PromptResult';
+import type { ChatPromptResult, CompletionPromptResult, EmbeddingPromptResult, ImagePromptResult } from '../../execution/PromptResult';
 import type { Usage } from '../../execution/Usage';
 import type { Prompt } from '../../types/Prompt';
 import type { string_markdown, string_markdown_text, string_model_name, string_title } from '../../types/typeAliases';
@@ -63,6 +63,14 @@ export declare abstract class OpenAiCompatibleExecutionTools implements LlmExecu
      * Internal method that handles parameter retry for embedding model calls
      */
     private callEmbeddingModelWithRetry;
+    /**
+     * Calls OpenAI compatible API to use a image generation model
+     */
+    callImageGenerationModel(prompt: Pick<Prompt, 'content' | 'parameters' | 'modelRequirements'>): Promise<ImagePromptResult>;
+    /**
+     * Internal method that handles parameter retry for image generation model calls
+     */
+    private callImageGenerationModelWithRetry;
     /**
      * Get the model that should be used as default
      */
@@ -89,6 +97,10 @@ export declare abstract class OpenAiCompatibleExecutionTools implements LlmExecu
      * Default model for completion variant.
      */
     protected abstract getDefaultEmbeddingModel(): AvailableModel;
+    /**
+     * Default model for image generation variant.
+     */
+    protected abstract getDefaultImageGenerationModel(): AvailableModel;
     /**
      * Makes a request with retry logic for network errors like ECONNRESET
      */

package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts

@@ -35,4 +35,8 @@ export declare class OpenAiExecutionTools extends OpenAiCompatibleExecutionTools
      * Default model for completion variant.
      */
     protected getDefaultEmbeddingModel(): AvailableModel;
+    /**
+     * Default model for image generation variant.
+     */
+    protected getDefaultImageGenerationModel(): AvailableModel;
 }

package/esm/typings/src/llm-providers/openai/createOpenAiCompatibleExecutionTools.d.ts

@@ -1,15 +1,11 @@
 import type { AvailableModel } from '../../execution/AvailableModel';
 import type { LlmExecutionTools } from '../../execution/LlmExecutionTools';
 import type { Usage } from '../../execution/Usage';
-import type { string_markdown } from '../../types/typeAliases';
-import type { string_markdown_text } from '../../types/typeAliases';
-import type { string_model_name } from '../../types/typeAliases';
-import type { string_title } from '../../types/typeAliases';
+import type { string_markdown, string_markdown_text, string_model_name, string_title } from '../../types/typeAliases';
 import { RemoteLlmExecutionTools } from '../remote/RemoteLlmExecutionTools';
 import { computeOpenAiUsage } from './computeOpenAiUsage';
 import { OpenAiCompatibleExecutionTools } from './OpenAiCompatibleExecutionTools';
-import type { OpenAiCompatibleExecutionToolsNonProxiedOptions } from './OpenAiCompatibleExecutionToolsOptions';
-import type { OpenAiCompatibleExecutionToolsOptions } from './OpenAiCompatibleExecutionToolsOptions';
+import type { OpenAiCompatibleExecutionToolsNonProxiedOptions, OpenAiCompatibleExecutionToolsOptions } from './OpenAiCompatibleExecutionToolsOptions';
 /**
  * Execution Tools for calling OpenAI compatible API
  *
@@ -67,6 +63,10 @@ export declare class HardcodedOpenAiCompatibleExecutionTools extends OpenAiCompa
      * Default model for completion variant.
      */
     protected getDefaultEmbeddingModel(): AvailableModel;
+    /**
+     * Default model for image generation variant.
+     */
+    protected getDefaultImageGenerationModel(): AvailableModel;
 }
 /**
  * TODO: [🦺] Is there some way how to put `packageName` and `className` on top and function definition on bottom?

package/esm/typings/src/types/ModelRequirements.d.ts

@@ -7,7 +7,7 @@ import type { number_model_temperature, number_seed, string_model_name, string_s
  * Note: [🚉] This is fully serializable as JSON
  * @see https://github.com/webgptorg/promptbook#model-requirements
  */
-export type ModelRequirements = CompletionModelRequirements | ChatModelRequirements | EmbeddingModelRequirements;
+export type ModelRequirements = CompletionModelRequirements | ChatModelRequirements | ImageGenerationModelRequirements | EmbeddingModelRequirements;
 /**
  * Model requirements for the completion variant
  *
@@ -34,6 +34,17 @@ export type ChatModelRequirements = CommonModelRequirements & {
      */
     readonly systemMessage?: string_system_message;
 };
+/**
+ * Model requirements for the image generation variant
+ *
+ * Note: [🚉] This is fully serializable as JSON
+ */
+export type ImageGenerationModelRequirements = CommonModelRequirements & {
+    /**
+     * Image generation model variant
+     */
+    modelVariant: 'IMAGE_GENERATION';
+};
 /**
  * Model requirements for the embedding variant
  *
@@ -59,6 +70,7 @@ export type CommonModelRequirements = {
      * There are 3 variants:
      * - **COMPLETION** - model that takes prompt and writes the rest of the text
      * - **CHAT** - model that takes prompt and previous messages and returns response
+     * - **IMAGE_GENERATION** - model that takes prompt and returns image
      * - **EMBEDDING** - model that takes prompt and returns embedding
      * <- [🤖]
      */

package/esm/typings/src/types/ModelVariant.d.ts

@@ -12,4 +12,4 @@ export type ModelVariant = TupleToUnion<typeof MODEL_VARIANTS>;
  * @see {@link ModelVariant}
  * @public exported from `@promptbook/core`
  */
-export declare const MODEL_VARIANTS: readonly ["COMPLETION", "CHAT", "EMBEDDING"];
+export declare const MODEL_VARIANTS: readonly ["COMPLETION", "CHAT", "IMAGE_GENERATION", "EMBEDDING"];

package/esm/typings/src/types/Prompt.d.ts

@@ -4,6 +4,7 @@ import type { Expectations } from '../pipeline/PipelineJson/Expectations';
 import type { ChatModelRequirements } from './ModelRequirements';
 import type { CompletionModelRequirements } from './ModelRequirements';
 import type { EmbeddingModelRequirements } from './ModelRequirements';
+import type { ImageGenerationModelRequirements } from './ModelRequirements';
 import type { ModelRequirements } from './ModelRequirements';
 import type { Parameters } from './typeAliases';
 import type { string_pipeline_url_with_task_hash } from './typeAliases';
@@ -17,7 +18,7 @@ import type { string_title } from './typeAliases';
  * Note: [🚉] This is fully serializable as JSON
  * @see https://github.com/webgptorg/promptbook#prompt
  */
-export type Prompt = CompletionPrompt | ChatPrompt | EmbeddingPrompt;
+export type Prompt = CompletionPrompt | ChatPrompt | ImagePrompt | EmbeddingPrompt;
 /**
  * Completion prompt
  *
@@ -44,6 +45,17 @@ export type ChatPrompt = CommonPrompt & {
      */
     thread?: ChatMessage[];
 };
+/**
+ * Image prompt
+ *
+ * Note: [🚉] This is fully serializable as JSON
+ */
+export type ImagePrompt = CommonPrompt & {
+    /**
+     * Requirements for image generation model
+     */
+    modelRequirements: ImageGenerationModelRequirements;
+};
 /**
  * Embedding prompt
  *

package/esm/typings/src/version.d.ts

@@ -15,7 +15,7 @@ export declare const BOOK_LANGUAGE_VERSION: string_semantic_version;
 export declare const PROMPTBOOK_ENGINE_VERSION: string_promptbook_version;
 /**
  * Represents the version string of the Promptbook engine.
- * It follows semantic versioning (e.g., `0.103.0-66`).
+ * It follows semantic versioning (e.g., `0.103.0-67`).
  *
  * @generated
  */

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@promptbook/ollama",
-    "version": "0.103.0-67",
+    "version": "0.103.0-68",
     "description": "Promptbook: Turn your company's scattered knowledge into AI ready books",
     "private": false,
     "sideEffects": false,
@@ -94,7 +94,7 @@
     "module": "./esm/index.es.js",
     "typings": "./esm/typings/src/_packages/ollama.index.d.ts",
     "peerDependencies": {
-        "@promptbook/core": "0.103.0-67"
+        "@promptbook/core": "0.103.0-68"
     },
     "dependencies": {
         "bottleneck": "2.19.5",

package/umd/index.umd.js

@@ -25,7 +25,7 @@
      * @generated
      * @see https://github.com/webgptorg/promptbook
      */
-    const PROMPTBOOK_ENGINE_VERSION = '0.103.0-67';
+    const PROMPTBOOK_ENGINE_VERSION = '0.103.0-68';
     /**
      * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
      * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -2174,12 +2174,18 @@
                 },
             },
             /**/
-            /*/
-              {
-                  modelTitle: 'dall-e-3',
-                  modelName: 'dall-e-3',
-              },
-              /**/
+            /**/
+            {
+                modelVariant: 'IMAGE_GENERATION',
+                modelTitle: 'dall-e-3',
+                modelName: 'dall-e-3',
+                modelDescription: 'DALL·E 3 is the latest version of the DALL·E art generation model. It understands significantly more nuance and detail than our previous systems, allowing you to easily translate your ideas into exceptionally accurate images.',
+                pricing: {
+                    prompt: 0,
+                    output: 0.04,
+                },
+            },
+            /**/
             /*/
               {
                   modelTitle: 'whisper-1',
@@ -2198,12 +2204,18 @@
                 },
             },
             /**/
-            /*/
-          {
-              modelTitle: 'dall-e-2',
-              modelName: 'dall-e-2',
-          },
-          /**/
+            /**/
+            {
+                modelVariant: 'IMAGE_GENERATION',
+                modelTitle: 'dall-e-2',
+                modelName: 'dall-e-2',
+                modelDescription: 'DALL·E 2 is an AI system that can create realistic images and art from a description in natural language.',
+                pricing: {
+                    prompt: 0,
+                    output: 0.02,
+                },
+            },
+            /**/
             /**/
             {
                 modelVariant: 'CHAT',
@@ -3414,6 +3426,151 @@
                 return this.callEmbeddingModelWithRetry(prompt, modifiedModelRequirements, attemptStack, retriedUnsupportedParameters);
             }
         }
+        /**
+         * Calls OpenAI compatible API to use a image generation model
+         */
+        async callImageGenerationModel(prompt) {
+            // Deep clone prompt and modelRequirements to avoid mutation across calls
+            const clonedPrompt = JSON.parse(JSON.stringify(prompt));
+            const retriedUnsupportedParameters = new Set();
+            return this.callImageGenerationModelWithRetry(clonedPrompt, clonedPrompt.modelRequirements, [], retriedUnsupportedParameters);
+        }
+        /**
+         * Internal method that handles parameter retry for image generation model calls
+         */
+        async callImageGenerationModelWithRetry(prompt, currentModelRequirements, attemptStack = [], retriedUnsupportedParameters = new Set()) {
+            var _a, _b;
+            if (this.options.isVerbose) {
+                console.info(`🎨 ${this.title} callImageGenerationModel call`, { prompt, currentModelRequirements });
+            }
+            const { content, parameters } = prompt;
+            const client = await this.getClient();
+            // TODO: [☂] Use here more modelRequirements
+            if (currentModelRequirements.modelVariant !== 'IMAGE_GENERATION') {
+                throw new PipelineExecutionError('Use callImageGenerationModel only for IMAGE_GENERATION variant');
+            }
+            const modelName = currentModelRequirements.modelName || this.getDefaultImageGenerationModel().modelName;
+            const modelSettings = {
+                model: modelName,
+                // size: currentModelRequirements.size,
+                // quality: currentModelRequirements.quality,
+                // style: currentModelRequirements.style,
+            };
+            const rawPromptContent = templateParameters(content, { ...parameters, modelName });
+            const rawRequest = {
+                ...modelSettings,
+                prompt: rawPromptContent,
+                user: (_a = this.options.userId) === null || _a === void 0 ? void 0 : _a.toString(),
+                response_format: 'url', // TODO: [🧠] Maybe allow b64_json
+            };
+            const start = $getCurrentDate();
+            if (this.options.isVerbose) {
+                console.info(colors__default["default"].bgWhite('rawRequest'), JSON.stringify(rawRequest, null, 4));
+            }
+            try {
+                const rawResponse = await this.limiter
+                    .schedule(() => this.makeRequestWithNetworkRetry(() => client.images.generate(rawRequest)))
+                    .catch((error) => {
+                    assertsError(error);
+                    if (this.options.isVerbose) {
+                        console.info(colors__default["default"].bgRed('error'), error);
+                    }
+                    throw error;
+                });
+                if (this.options.isVerbose) {
+                    console.info(colors__default["default"].bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4));
+                }
+                const complete = $getCurrentDate();
+                if (!rawResponse.data[0]) {
+                    throw new PipelineExecutionError(`No choises from ${this.title}`);
+                }
+                if (rawResponse.data.length > 1) {
+                    throw new PipelineExecutionError(`More than one choise from ${this.title}`);
+                }
+                const resultContent = rawResponse.data[0].url;
+                const modelInfo = this.HARDCODED_MODELS.find((model) => model.modelName === modelName);
+                const price = ((_b = modelInfo === null || modelInfo === void 0 ? void 0 : modelInfo.pricing) === null || _b === void 0 ? void 0 : _b.output) ? uncertainNumber(modelInfo.pricing.output) : uncertainNumber();
+                return exportJson({
+                    name: 'promptResult',
+                    message: `Result of \`OpenAiCompatibleExecutionTools.callImageGenerationModel\``,
+                    order: [],
+                    value: {
+                        content: resultContent,
+                        modelName: modelName,
+                        timing: {
+                            start,
+                            complete,
+                        },
+                        usage: {
+                            price,
+                            input: {
+                                tokensCount: uncertainNumber(0),
+                                ...computeUsageCounts(rawPromptContent),
+                            },
+                            output: {
+                                tokensCount: uncertainNumber(0),
+                                ...computeUsageCounts(''),
+                            },
+                        },
+                        rawPromptContent,
+                        rawRequest,
+                        rawResponse,
+                    },
+                });
+            }
+            catch (error) {
+                assertsError(error);
+                if (!isUnsupportedParameterError(error)) {
+                    if (attemptStack.length > 0) {
+                        throw new PipelineExecutionError(`All attempts failed. Attempt history:\n` +
+                            attemptStack
+                                .map((a, i) => `  ${i + 1}. Model: ${a.modelName}` +
+                                (a.unsupportedParameter ? `, Stripped: ${a.unsupportedParameter}` : '') +
+                                `, Error: ${a.errorMessage}` +
+                                (a.stripped ? ' (stripped and retried)' : ''))
+                                .join('\n') +
+                            `\nFinal error: ${error.message}`);
+                    }
+                    throw error;
+                }
+                const unsupportedParameter = parseUnsupportedParameterError(error.message);
+                if (!unsupportedParameter) {
+                    if (this.options.isVerbose) {
+                        console.warn(colors__default["default"].bgYellow('Warning'), 'Could not parse unsupported parameter from error:', error.message);
+                    }
+                    throw error;
+                }
+                const retryKey = `${modelName}-${unsupportedParameter}`;
+                if (retriedUnsupportedParameters.has(retryKey)) {
+                    attemptStack.push({
+                        modelName,
+                        unsupportedParameter,
+                        errorMessage: error.message,
+                        stripped: true,
+                    });
+                    throw new PipelineExecutionError(`All attempts failed. Attempt history:\n` +
+                        attemptStack
+                            .map((a, i) => `  ${i + 1}. Model: ${a.modelName}` +
+                            (a.unsupportedParameter ? `, Stripped: ${a.unsupportedParameter}` : '') +
+                            `, Error: ${a.errorMessage}` +
+                            (a.stripped ? ' (stripped and retried)' : ''))
+                            .join('\n') +
+                        `\nFinal error: ${error.message}`);
+                }
+                retriedUnsupportedParameters.add(retryKey);
+                if (this.options.isVerbose) {
+                    console.warn(colors__default["default"].bgYellow('Warning'), `Removing unsupported parameter '${unsupportedParameter}' for model '${modelName}' and retrying request`);
+                }
+                attemptStack.push({
+                    modelName,
+                    unsupportedParameter,
+                    errorMessage: error.message,
+                    stripped: true,
+                });
+                const modifiedModelRequirements = removeUnsupportedModelRequirement(currentModelRequirements, unsupportedParameter);
+                return this.callImageGenerationModelWithRetry(prompt, modifiedModelRequirements, attemptStack, retriedUnsupportedParameters);
+            }
+        }
         // <- Note: [🤖] callXxxModel
         /**
          * Get the model that should be used as default
@@ -3844,6 +4001,13 @@
             return this.getDefaultModel('text-embedding-3-large'); // <- TODO: [🧠] Pick the best default model
             // <- TODO: [🛄]
         }
+        /**
+         * Default model for image generation variant.
+         */
+        getDefaultImageGenerationModel() {
+            return this.getDefaultModel('!!!'); // <- TODO: [🧠] Pick the best default model
+            // <- TODO: [🛄]
+        }
     }
     /**
      * TODO: [🛄] Some way how to re-wrap the errors from `OpenAiCompatibleExecutionTools`