@saltcorn/large-language-model 0.8.4 → 0.8.6

package/generate.js

@@ -78,6 +78,23 @@ const getEmbedding = async (config, opts) => {
   }
 };
 
+const getImageGeneration = async (config, opts) => {
+  switch (config.backend) {
+    case "OpenAI":
+      return await getImageGenOpenAICompatible(
+        {
+          imageEndpoint: "https://api.openai.com/v1/images/generations",
+          bearer: opts?.api_key || opts?.bearer || config.api_key,
+          model: opts?.model || config.model,
+          responses_api: config.responses_api,
+        },
+        opts
+      );
+    default:
+      throw new Error("Image generation not implemented for this backend");
+  }
+};
+
 const getCompletion = async (config, opts) => {
   switch (config.backend) {
     case "OpenAI":
@@ -318,6 +335,50 @@ const getCompletionOpenAICompatible = async (
 
 const emptyToUndefined = (xs) => (xs.length ? xs : undefined);
 
+const getImageGenOpenAICompatible = async (
+  config,
+  {
+    prompt,
+    model,
+    debugResult,
+    size,
+    quality,
+    n,
+    output_format,
+    response_format,
+  }
+) => {
+  const { imageEndpoint, bearer, apiKey, image_model } = config;
+  const headers = {
+    "Content-Type": "application/json",
+    Accept: "application/json",
+  };
+  if (bearer) headers.Authorization = "Bearer " + bearer;
+  if (apiKey) headers["api-key"] = apiKey;
+  const body = {
+    //prompt: "How are you?",
+    model: model || image_model || "gpt-image-1",
+    prompt,
+    size: size || "1024x1024",
+    n: n || 1,
+  };
+  if (quality) body.quality = quality;
+  if (output_format) body.output_format = output_format;
+  if (response_format) body.response_format = response_format;
+  if (n) body.n = n;
+  if (debugResult) console.log("OpenAI image request", imageEndpoint, body);
+
+  const rawResponse = await fetch(imageEndpoint, {
+    method: "POST",
+    headers,
+    body: JSON.stringify(body),
+  });
+  const results = await rawResponse.json();
+  if (debugResult) console.log("OpenAI image response", results);
+  if (results.error) throw new Error(`OpenAI error: ${results.error.message}`);
+  return results?.data?.[0];
+};
+
 const getEmbeddingOpenAICompatible = async (
   config,
   { prompt, model, debugResult }
@@ -518,4 +579,4 @@ const getEmbeddingGoogleVertex = async (config, opts, oauth2Client) => {
   return embeddings;
 };
 
-module.exports = { getCompletion, getEmbedding };
+module.exports = { getCompletion, getEmbedding, getImageGeneration };

package/index.js

@@ -1,10 +1,16 @@
 const Workflow = require("@saltcorn/data/models/workflow");
 const Form = require("@saltcorn/data/models/form");
+const File = require("@saltcorn/data/models/file");
+const User = require("@saltcorn/data/models/user");
 const FieldRepeat = require("@saltcorn/data/models/fieldrepeat");
 const Plugin = require("@saltcorn/data/models/plugin");
 const { domReady } = require("@saltcorn/markup/tags");
 const db = require("@saltcorn/data/db");
-const { getCompletion, getEmbedding } = require("./generate");
+const {
+  getCompletion,
+  getEmbedding,
+  getImageGeneration,
+} = require("./generate");
 const { OPENAI_MODELS } = require("./constants.js");
 const { eval_expression } = require("@saltcorn/data/models/expression");
 const { interpolate } = require("@saltcorn/data/utils");
@@ -126,7 +132,17 @@ ${domReady(`
                   ],
                 },
               },
-                  {
+              {
+                name: "image_model",
+                label: "Image model", //gpt-3.5-turbo
+                type: "String",
+                required: true,
+                showIf: { backend: "OpenAI" },
+                attributes: {
+                  options: ["gpt-image-1", "dall-e-2", "dall-e-3"],
+                },
+              },
+              {
                 name: "client_id",
                 label: "Client ID",
                 sublabel: "OAuth2 client ID from your Google Cloud account",
@@ -219,7 +235,7 @@ ${domReady(`
                 showIf: { backend: "Google Vertex AI" },
                 default: "us-central1",
               },
-          
+
               {
                 name: "bearer_auth",
                 label: "Bearer Auth",
@@ -318,6 +334,15 @@ const functions = (config) => {
       description: "Generate text with GPT",
       arguments: [{ name: "prompt", type: "String" }],
     },
+    llm_image_generate: {
+      run: async (prompt, opts) => {
+        const result = await getImageGeneration(config, { prompt, ...opts });
+        return result;
+      },
+      isAsync: true,
+      description: "Generate image",
+      arguments: [{ name: "prompt", type: "String" }],
+    },
     llm_embedding: {
       run: async (prompt, opts) => {
         const result = await getEmbedding(config, { prompt, ...opts });
@@ -567,6 +592,139 @@ module.exports = {
         else await table.updateRow(upd, row[table.pk_name]);
       },
     },
+    llm_generate_image: {
+      description: "Generate image with AI based on a text prompt",
+      requireRow: true,
+      configFields: async ({ table, mode }) => {
+        const roleOptions = (await User.get_roles()).map((r) => ({
+          value: r.id,
+          label: r.role,
+        }));
+        const commonFields = [
+          {
+            label: "Minimum role to access",
+            name: "min_role",
+            input_type: "select",
+            sublabel: "User must have this role or higher access image file",
+            options: roleOptions,
+          },
+        ];
+        if (mode === "workflow") {
+          return [
+            {
+              name: "prompt_template",
+              label: "Prompt",
+              sublabel:
+                "Prompt text. Use interpolations {{ }} to access variables in the context",
+              type: "String",
+              fieldview: "textarea",
+              required: true,
+            },
+            {
+              name: "answer_field",
+              label: "Answer variable",
+              sublabel:
+                "Set the generated image filename to this context variable",
+              type: "String",
+              required: true,
+            },
+            {
+              name: "model",
+              label: "Model",
+              sublabel: "Override default model name",
+              type: "String",
+            },
+            ...commonFields,
+          ];
+        } else if (table) {
+          const textFields = table.fields
+            .filter((f) => f.type?.sql_name === "text")
+            .map((f) => f.name);
+          const fileFields = table.fields
+            .filter((f) => f.type === "File")
+            .map((f) => f.name);
+
+          return [
+            {
+              name: "prompt_field",
+              label: "Prompt field",
+              sublabel: "Field with the text of the prompt",
+              type: "String",
+              required: true,
+              attributes: { options: [...textFields, "Formula"] },
+            },
+            {
+              name: "prompt_formula",
+              label: "Prompt formula",
+              type: "String",
+              showIf: { prompt_field: "Formula" },
+            },
+            {
+              name: "answer_field",
+              label: "Answer field",
+              sublabel: "Output field will be set to the generated image file",
+              type: "String",
+              required: true,
+              attributes: { options: fileFields },
+            },
+            ...commonFields,
+          ];
+        }
+      },
+      run: async ({
+        row,
+        table,
+        user,
+        mode,
+        configuration: {
+          prompt_field,
+          prompt_formula,
+          prompt_template,
+          answer_field,
+          min_role,
+          model,
+        },
+      }) => {
+        let prompt;
+        if (mode === "workflow")
+          prompt = interpolate(prompt_template, row, user);
+        else if (prompt_field === "Formula" || mode === "workflow")
+          prompt = eval_expression(
+            prompt_formula,
+            row,
+            user,
+            "llm_generate prompt formula"
+          );
+        else prompt = row[prompt_field];
+        const opts = { debugResult: true }; // response_format: "b64_json" };
+
+        if (model) opts.model = model;
+        let history = [];
+
+        const ans = await getImageGeneration(config, {
+          prompt,
+          ...opts,
+        });
+        const upd = {};
+
+        if (ans.url) {
+          //fetch url
+        } else if (ans.b64_json) {
+          const imgContents = Buffer.from(ans.b64_json, "base64");
+          const file = await File.from_contents(
+            "generated.png",
+            "image/png",
+            imgContents,
+            user?.id,
+            min_role || 1
+          );
+          upd[answer_field] = file.path_to_serve;
+        }
+        if (mode === "workflow") return upd;
+        else await table.updateRow(upd, row[table.pk_name]);
+      },
+    },
+
     llm_generate_json: {
       description:
         "Generate JSON with AI based on a text prompt. You must sppecify the JSON fields in the configuration.",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@saltcorn/large-language-model",
-  "version": "0.8.4",
+  "version": "0.8.6",
   "description": "Large language models and functionality for Saltcorn",
   "main": "index.js",
   "dependencies": {