@saltcorn/large-language-model 0.2.0 → 0.3.0

package/generate.js

@@ -1,8 +1,48 @@
-const axios = require("axios");
+const fetch = require("node-fetch");
 const util = require("util");
 const exec = util.promisify(require("child_process").exec);
 const db = require("@saltcorn/data/db");
 
+const { features, getState } = require("@saltcorn/data/db/state");
+let ollamaMod;
+if (features.esm_plugins) ollamaMod = require("ollama");
+
+const getEmbedding = async (config, opts) => {
+  switch (config.backend) {
+    case "OpenAI":
+      return await getEmbeddingOpenAICompatible(
+        {
+          embeddingsEndpoint: "https://api.openai.com/v1/embeddings",
+          bearer: config.api_key,
+          embed_model: config.embed_model,
+        },
+        opts
+      );
+    case "OpenAI-compatible API":
+      return await getEmbeddingOpenAICompatible(
+        {
+          embeddingsEndpoint: config.embed_endpoint,
+          bearer: config.api_key,
+          embed_model: config.model,
+        },
+        opts
+      );
+    case "Local Ollama":
+      if (!ollamaMod) throw new Error("Not implemented for this backend");
+
+      const { Ollama } = ollamaMod;
+      const ollama = new Ollama();
+      const olres = await ollama.embeddings({
+        model: opts?.model || config.model,
+        prompt: opts.prompt,
+      });
+      //console.log("embedding response ", olres);
+      return olres.embedding;
+    default:
+      throw new Error("Not implemented for this backend");
+  }
+};
+
 const getCompletion = async (config, opts) => {
   switch (config.backend) {
     case "OpenAI":
@@ -23,6 +63,18 @@ const getCompletion = async (config, opts) => {
         },
         opts
       );
+    case "Local Ollama":
+      if (!ollamaMod) throw new Error("Not implemented for this backend");
+
+      const { Ollama } = ollamaMod;
+
+      const ollama = new Ollama();
+      const olres = await ollama.generate({
+        model: config.model,
+        prompt: opts.prompt,
+      });
+      //console.log("the response ", olres);
+      return olres.response;
     case "Local llama.cpp":
       //TODO only check if unsafe plugins not allowed
       const isRoot = db.getTenantSchema() === db.connectObj.default_schema;
@@ -52,12 +104,10 @@ const getCompletionOpenAICompatible = async (
 ) => {
   const headers = {
     "Content-Type": "application/json",
+    Accept: "application/json",
   };
   if (bearer) headers.Authorization = "Bearer " + bearer;
-  const client = axios.create({
-    headers,
-  });
-  const params = {
+  const body = {
     //prompt: "How are you?",
     model,
     messages: [
@@ -69,9 +119,38 @@ const getCompletionOpenAICompatible = async (
     ],
     temperature: temperature || 0.7,
   };
+  const rawResponse = await fetch(chatCompleteEndpoint, {
+    method: "POST",
+    headers,
+    body: JSON.stringify(body),
+  });
+  const results = await rawResponse.json();
+  console.log(results);
 
-  const results = await client.post(chatCompleteEndpoint, params);
-  return results?.data?.choices?.[0]?.message?.content;
+  return results?.choices?.[0]?.message?.content;
 };
 
-module.exports = { getCompletion };
+const getEmbeddingOpenAICompatible = async (config, { prompt, model }) => {
+  const { embeddingsEndpoint, bearer, embed_model } = config;
+  const headers = {
+    "Content-Type": "application/json",
+    Accept: "application/json",
+  };
+  if (bearer) headers.Authorization = "Bearer " + bearer;
+  const body = {
+    //prompt: "How are you?",
+    model: model || embed_model || "text-embedding-3-small",
+    input: prompt,
+  };
+  console.log({ body, config });
+  const rawResponse = await fetch(embeddingsEndpoint, {
+    method: "POST",
+    headers,
+    body: JSON.stringify(body),
+  });
+  const results = await rawResponse.json();
+  console.log(results);
+
+  return results?.data?.[0]?.embedding;
+};
+module.exports = { getCompletion, getEmbedding };

package/index.js

@@ -1,6 +1,6 @@
 const Workflow = require("@saltcorn/data/models/workflow");
 const Form = require("@saltcorn/data/models/form");
-const { getCompletion } = require("./generate");
+const { getCompletion, getEmbedding } = require("./generate");
 const db = require("@saltcorn/data/db");
 
 const configuration_workflow = () =>
@@ -21,6 +21,7 @@ const configuration_workflow = () =>
                   options: [
                     "OpenAI",
                     "OpenAI-compatible API",
+                    "Local Ollama",
                     ...(isRoot ? ["Local llama.cpp"] : []),
                   ],
                 },
@@ -62,6 +63,20 @@ const configuration_workflow = () =>
                   ],
                 },
               },
+              {
+                name: "embed_model",
+                label: "Embedding model", //gpt-3.5-turbo
+                type: "String",
+                required: true,
+                showIf: { backend: "OpenAI" },
+                attributes: {
+                  options: [
+                    "text-embedding-3-small",
+                    "text-embedding-3-large",
+                    "text-embedding-ada-002",
+                  ],
+                },
+              },
               {
                 name: "bearer_auth",
                 label: "Bearer Auth",
@@ -73,7 +88,7 @@ const configuration_workflow = () =>
                 name: "model",
                 label: "Model",
                 type: "String",
-                showIf: { backend: "OpenAI-compatible API" },
+                showIf: { backend: ["OpenAI-compatible API", "Local Ollama"] },
               },
               {
                 name: "endpoint",
@@ -82,6 +97,13 @@ const configuration_workflow = () =>
                 sublabel: "Example: http://localhost:8080/v1/chat/completions",
                 showIf: { backend: "OpenAI-compatible API" },
               },
+              {
+                name: "embed_endpoint",
+                label: "Embedding endpoint",
+                type: "String",
+                sublabel: "Example: http://localhost:8080/v1/embeddings",
+                showIf: { backend: "OpenAI-compatible API" },
+              },
             ],
           });
         },
@@ -97,6 +119,14 @@ const functions = (config) => ({
     description: "Generate text with GPT",
     arguments: [{ name: "prompt", type: "String" }],
   },
+  llm_embedding: {
+    run: async (prompt, opts) => {
+      return await getEmbedding(config, { prompt, ...opts });
+    },
+    isAsync: true,
+    description: "Get vector embedding",
+    arguments: [{ name: "prompt", type: "String" }],
+  },
 });
 
 module.exports = {

package/package.json

@@ -1,12 +1,13 @@
 {
   "name": "@saltcorn/large-language-model",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Large language models and functionality for Saltcorn",
   "main": "index.js",
   "dependencies": {
     "@saltcorn/data": "^0.9.0",
-    "axios": "0.16.2",
-    "underscore": "1.13.6"
+    "node-fetch": "2.6.9",
+    "underscore": "1.13.6",
+    "ollama": "0.5.0"
   },
   "author": "Tom Nielsen",
   "license": "MIT",