@saltcorn/large-language-model 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Saltcorn
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,2 @@
+# large-language-model
+LLM models and functionality for Saltcorn

package/generate.js

@@ -0,0 +1,77 @@
+const axios = require("axios");
+const util = require("util");
+const exec = util.promisify(require("child_process").exec);
+const db = require("@saltcorn/data/db");
+
+const getCompletion = async (config, opts) => {
+  switch (config.backend) {
+    case "OpenAI":
+      return await getCompletionOpenAICompatible(
+        {
+          chatCompleteEndpoint: "https://api.openai.com/v1/chat/completions",
+          bearer: config.api_key,
+          model: config.model,
+        },
+        opts
+      );
+    case "OpenAI-compatible API":
+      return await getCompletionOpenAICompatible(
+        {
+          chatCompleteEndpoint: config.endpoint,
+          bearer: config.bearer,
+          model: config.model,
+        },
+
+        opts
+      );
+    case "Local llama.cpp":
+      const isRoot = db.getTenantSchema() === db.connectObj.default_schema;
+      if (!isRoot)
+        throw new Error(
+          "llama.cpp inference is not permitted on subdomain tenants"
+        );
+      let hyperStr = "";
+      if (opts.temperature) hyperStr += ` --temp ${opts.temperature}`;
+      let nstr = "";
+      if (opts.ntokens) nstr = `-n ${opts.ntokens}`;
+      console.log("running llama with prompt: ", opts.prompt);
+
+      const { stdout, stderr } = await exec(
+        `./main -m ${config.model_path} -p "${opts.prompt}" ${nstr}${hyperStr}`,
+        { cwd: config.llama_dir }
+      );
+      return stdout;
+    default:
+      break;
+  }
+};
+
+const getCompletionOpenAICompatible = async (
+  { chatCompleteEndpoint, bearer, model },
+  { systemPrompt, prompt, temperature }
+) => {
+  const headers = {
+    "Content-Type": "application/json",
+  };
+  if (bearer) headers.Authorization = "Bearer " + bearer;
+  const client = axios.create({
+    headers,
+  });
+  const params = {
+    //prompt: "How are you?",
+    model,
+    messages: [
+      {
+        role: "system",
+        content: systemPrompt || "You are a helpful assistant.",
+      },
+      { role: "user", content: prompt },
+    ],
+    temperature: temperature || 0.7,
+  };
+
+  const results = await client.post(chatCompleteEndpoint, params);
+  return results?.data?.choices?.[0]?.message?.content;
+};
+
+module.exports = { getCompletion };

package/index.js

@@ -0,0 +1,105 @@
+const Workflow = require("@saltcorn/data/models/workflow");
+const Form = require("@saltcorn/data/models/form");
+const { getCompletion } = require("./generate");
+const db = require("@saltcorn/data/db");
+
+const configuration_workflow = () =>
+  new Workflow({
+    steps: [
+      {
+        name: "API key",
+        form: async (context) => {
+          const isRoot = db.getTenantSchema() === db.connectObj.default_schema;
+          return new Form({
+            fields: [
+              {
+                name: "backend",
+                label: "Inference backend",
+                type: "String",
+                required: true,
+                attributes: {
+                  options: [
+                    "OpenAI",
+                    "OpenAI-compatible API",
+                    ...(isRoot ? ["Local llama.cpp"] : []),
+                  ],
+                },
+              },
+              {
+                name: "api_key",
+                label: "API key",
+                sublabel: "From your OpenAI account",
+                type: "String",
+                required: true,
+                showIf: { backend: "OpenAI" },
+              },
+              {
+                name: "llama_dir",
+                label: "llama.cpp directory",
+                type: "String",
+                required: true,
+                showIf: { backend: "Local llama.cpp" },
+              },
+              {
+                name: "model_path",
+                label: "Model path",
+                type: "String",
+                required: true,
+                showIf: { backend: "Local llama.cpp" },
+              },
+              {
+                name: "model",
+                label: "Model", //gpt-3.5-turbo
+                type: "String",
+                required: true,
+                showIf: { backend: "OpenAI" },
+                attributes: {
+                  options: [
+                    "gpt-3.5-turbo",
+                    "gpt-3.5-turbo-16k",
+                    "gpt-4",
+                    "gpt-4-32k",
+                  ],
+                },
+              },
+              {
+                name: "bearer_auth",
+                label: "Bearer Auth",
+                sublabel: "HTTP Header authorization with bearer token",
+                type: "String",
+                showIf: { backend: "OpenAI-compatible API" },
+              },
+              {
+                name: "model",
+                label: "Model",
+                type: "String",
+                showIf: { backend: "OpenAI-compatible API" },
+              },
+              {
+                name: "endpoint",
+                label: "Chat completions endpoint",
+                type: "String",
+                sublabel: "Example: http://localhost:8080/v1/chat/completions",
+                showIf: { backend: "OpenAI-compatible API" },
+              },
+            ],
+          });
+        },
+      },
+    ],
+  });
+const functions = (config) => ({
+  llm_generate: {
+    run: async (prompt, opts) => {
+      return await getCompletion(config, { prompt, ...opts });
+    },
+    isAsync: true,
+    description: "Generate text with GPT",
+    arguments: [{ name: "prompt", type: "String" }],
+  },
+});
+module.exports = {
+  sc_plugin_api_version: 1,
+  configuration_workflow,
+  functions,
+};

package/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@saltcorn/large-language-model",
+  "version": "0.1.0",
+  "description": "Large language models and functionality for Saltcorn",
+  "main": "index.js",
+  "dependencies": {
+    "@saltcorn/data": "^0.9.0",
+    "axios": "0.16.2",
+    "underscore": "1.13.6"
+  },
+  "author": "Tom Nielsen",
+  "license": "MIT",
+  "repository": "github:saltcorn/large-language-model",
+  "eslintConfig": {
+    "extends": "eslint:recommended",
+    "parserOptions": {
+      "ecmaVersion": 2020
+    },
+    "env": {
+      "node": true,
+      "es6": true
+    },
+    "rules": {
+      "no-unused-vars": "off",
+      "no-case-declarations": "off",
+      "no-empty": "warn",
+      "no-fallthrough": "warn"
+    }
+  }
+}