@budibase/worker 3.9.4 → 3.10.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "3.9.4",
+  "version": "3.10.0",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -120,5 +120,5 @@
       }
     }
   },
-  "gitHead": "e357c4e0cbfb4e0376cb8b692bf725263a5f7dc3"
+  "gitHead": "35c16326f7215671d405cd811b28e78f5be70d07"
 }

package/src/api/controllers/global/configs.ts

@@ -9,6 +9,7 @@ import {
   events,
   objectStore,
   tenancy,
+  BadRequestError,
 } from "@budibase/backend-core"
 import { checkAnyUserExists } from "../../../utilities/users"
 import {
@@ -31,14 +32,12 @@ import {
   OIDCConfigs,
   OIDCLogosConfig,
   PASSWORD_REPLACEMENT,
-  QuotaUsageType,
   SaveConfigRequest,
   SaveConfigResponse,
   SettingsBrandingConfig,
   SettingsInnerConfig,
   SSOConfig,
   SSOConfigType,
-  StaticQuotaName,
   UploadConfigFileResponse,
   UserCtx,
 } from "@budibase/types"
@@ -52,7 +51,6 @@ const getEventFns = async (config: Config, existing?: Config) => {
       fns.push(events.email.SMTPCreated)
     } else if (isAIConfig(config)) {
       fns.push(() => events.ai.AIConfigCreated)
-      fns.push(() => pro.quotas.addCustomAIConfig())
     } else if (isGoogleConfig(config)) {
       fns.push(() => events.auth.SSOCreated(ConfigType.GOOGLE))
       if (config.config.activated) {
@@ -91,12 +89,6 @@ const getEventFns = async (config: Config, existing?: Config) => {
       fns.push(events.email.SMTPUpdated)
     } else if (isAIConfig(config)) {
       fns.push(() => events.ai.AIConfigUpdated)
-      if (
-        Object.keys(existing.config).length > Object.keys(config.config).length
-      ) {
-        fns.push(() => pro.quotas.removeCustomAIConfig())
-      }
-      fns.push(() => pro.quotas.addCustomAIConfig())
     } else if (isGoogleConfig(config)) {
       fns.push(() => events.auth.SSOUpdated(ConfigType.GOOGLE))
       if (!existing.config.activated && config.config.activated) {
@@ -219,14 +211,30 @@ async function verifyOIDCConfig(config: OIDCConfigs) {
   await verifySSOConfig(ConfigType.OIDC, config.configs[0])
 }
 
-export async function verifyAIConfig(
-  configToSave: AIInnerConfig,
-  existingConfig: AIConfig
+export async function processAIConfig(
+  newConfig: AIInnerConfig,
+  existingConfig: AIInnerConfig
 ) {
   // ensure that the redacted API keys are not overwritten in the DB
-  for (const uuid in existingConfig.config) {
-    if (configToSave[uuid]?.apiKey === PASSWORD_REPLACEMENT) {
-      configToSave[uuid].apiKey = existingConfig.config[uuid].apiKey
+  for (const key in existingConfig) {
+    if (newConfig[key]?.apiKey === PASSWORD_REPLACEMENT) {
+      newConfig[key].apiKey = existingConfig[key].apiKey
+    }
+  }
+
+  let numBudibaseAI = 0
+  for (const config of Object.values(newConfig)) {
+    if (config.provider === "BudibaseAI") {
+      numBudibaseAI++
+      if (numBudibaseAI > 1) {
+        throw new BadRequestError("Only one Budibase AI provider is allowed")
+      }
+    } else {
+      if (!config.apiKey) {
+        throw new BadRequestError(
+          `API key is required for provider ${config.provider}`
+        )
+      }
     }
   }
 }
@@ -246,7 +254,6 @@ export async function save(
   }
 
   try {
-    // verify the configuration
     switch (type) {
       case ConfigType.SMTP:
         await email.verifyConfig(config)
@@ -262,7 +269,7 @@ export async function save(
         break
       case ConfigType.AI:
         if (existingConfig) {
-          await verifyAIConfig(config, existingConfig)
+          await processAIConfig(config, existingConfig.config)
         }
         break
     }
@@ -354,7 +361,7 @@ export async function find(ctx: UserCtx<void, FindConfigResponse>) {
     if (scopedConfig) {
       await handleConfigType(type, scopedConfig)
     } else if (type === ConfigType.AI) {
-      scopedConfig = { config: {} } as AIConfig
+      scopedConfig = { type: ConfigType.AI, config: {} }
       await handleAIConfig(scopedConfig)
     } else {
       // If no config found and not AI type, just return an empty body
@@ -560,13 +567,6 @@ export async function destroy(ctx: UserCtx<void, DeleteConfigResponse>) {
   try {
     await db.remove(id, rev)
     await cache.destroy(cache.CacheKey.CHECKLIST)
-    if (id === configs.generateConfigID(ConfigType.AI)) {
-      await pro.quotas.set(
-        StaticQuotaName.AI_CUSTOM_CONFIGS,
-        QuotaUsageType.STATIC,
-        0
-      )
-    }
     ctx.body = { message: "Config deleted successfully" }
   } catch (err: any) {
     ctx.throw(err.status, err)

package/src/api/controllers/global/self.ts

@@ -120,7 +120,6 @@ export async function getSelf(ctx: UserCtx<void, GetGlobalSelfResponse>) {
     ? {
         provider: llmConfig.provider,
         model: llmConfig.model,
-        measureUsage: llmConfig.measureUsage,
       }
     : undefined
 

package/src/api/controllers/global/tests/configs.spec.ts

@@ -1,6 +1,21 @@
-import { verifyAIConfig } from "../configs"
-import { TestConfiguration, structures } from "../../../../tests"
-import { AIInnerConfig } from "@budibase/types"
+import { TestConfiguration } from "../../../../tests"
+import { AIConfig, ConfigType } from "@budibase/types"
+import { configs, context } from "@budibase/backend-core"
+
+const BASE_CONFIG: AIConfig = {
+  type: ConfigType.AI,
+  config: {
+    ai: {
+      provider: "OpenAI",
+      isDefault: false,
+      name: "Test",
+      active: true,
+      defaultModel: "gpt4",
+      apiKey: "myapikey",
+      baseUrl: "https://api.example.com",
+    },
+  },
+}
 
 describe("Global configs controller", () => {
   const config = new TestConfiguration()
@@ -13,40 +28,70 @@ describe("Global configs controller", () => {
     await config.afterAll()
   })
 
-  it("Should strip secrets when pulling AI config", async () => {
-    const data = structures.configs.ai()
-    await config.api.configs.saveConfig(data)
+  it("should strip secrets when pulling AI config", async () => {
+    await config.api.configs.saveConfig(BASE_CONFIG)
     const response = await config.api.configs.getAIConfig()
-    expect(response.body.config).toEqual({
-      ai: {
-        active: true,
-        apiKey: "--secret-value--",
-        baseUrl: "https://api.example.com",
-        defaultModel: "gpt4",
-        isDefault: false,
-        name: "Test",
-        provider: "OpenAI",
-      },
+    expect(response.config.ai.apiKey).toEqual("--secret-value--")
+  })
+
+  it("should not update existing secrets when updating an existing AI Config", async () => {
+    await config.api.configs.saveConfig(BASE_CONFIG)
+
+    const savedConfig = await config.api.configs.getAIConfig()
+    delete savedConfig._id
+    delete savedConfig._rev
+    delete savedConfig.createdAt
+    delete savedConfig.updatedAt
+
+    await config.api.configs.saveConfig(savedConfig)
+
+    await context.doInTenant(config.tenantId, async () => {
+      const aiConfig = await configs.getAIConfig()
+      expect(aiConfig!.config.ai.apiKey).toEqual(BASE_CONFIG.config.ai.apiKey)
     })
   })
 
-  it("Should not update existing secrets when updating an existing AI Config", async () => {
-    const data = structures.configs.ai()
-    await config.api.configs.saveConfig(data)
-
-    const newConfig: AIInnerConfig = {
-      ai: {
-        provider: "OpenAI",
-        isDefault: true,
-        apiKey: "--secret-value--",
-        name: "MyConfig",
-        active: true,
-        defaultModel: "gpt4",
+  it("should allow BudibaseAI to save without an apiKey", async () => {
+    await config.api.configs.saveConfig({
+      type: ConfigType.AI,
+      config: {
+        ai: {
+          name: "Budibase AI",
+          active: true,
+          provider: "BudibaseAI",
+          isDefault: true,
+        },
       },
-    }
+    })
+
+    const aiConfig = await config.api.configs.getAIConfig()
+    expect(aiConfig.config.ai).toEqual({
+      name: "Budibase AI",
+      provider: "BudibaseAI",
+      active: true,
+      isDefault: true,
+    })
+  })
 
-    await verifyAIConfig(newConfig, data)
-    // should be unchanged
-    expect(newConfig.ai.apiKey).toEqual("myapikey")
+  it("should not allow OpenAI to save without an apiKey", async () => {
+    await config.api.configs.saveConfig(
+      {
+        type: ConfigType.AI,
+        config: {
+          ai: {
+            name: "OpenAI",
+            active: true,
+            provider: "OpenAI",
+            isDefault: true,
+          },
+        },
+      },
+      {
+        status: 400,
+        body: {
+          message: /API key is required for provider OpenAI/,
+        },
+      }
+    )
   })
 })

package/src/api/controllers/global/users.ts

@@ -16,6 +16,7 @@ import {
   DeleteInviteUsersRequest,
   DeleteInviteUsersResponse,
   DeleteUserResponse,
+  ErrorCode,
   FetchUsersResponse,
   FindUserResponse,
   GetUserInvitesResponse,
@@ -42,7 +43,6 @@ import {
 import {
   users,
   cache,
-  ErrorCode,
   events,
   platform,
   tenancy,

package/src/api/index.ts

@@ -130,6 +130,7 @@ const router: Router = new Router()
 
 router
   .use(middleware.errorHandling)
+  .use(middleware.featureFlagCookie)
   .use(
     compress({
       threshold: 2048,

package/src/api/routes/global/configs.ts

@@ -68,16 +68,16 @@ function scimValidation() {
 function aiValidation() {
   // prettier-ignore
   return Joi.object().pattern(
-     Joi.string(),
-     Joi.object({
-       provider: Joi.string().required(),
-       isDefault: Joi.boolean().required(),
-       name: Joi.string().required(),
-       active: Joi.boolean().required(),
-       baseUrl: Joi.string().optional().allow("", null),
-       apiKey: Joi.string().required(),
-       defaultModel: Joi.string().optional(),
-     }).required()
+    Joi.string(),
+    Joi.object({
+      provider: Joi.string().required(),
+      isDefault: Joi.boolean().required(),
+      name: Joi.string().required(),
+      active: Joi.boolean().required(),
+      baseUrl: Joi.string().optional().allow("", null),
+      apiKey: Joi.string().optional(),
+      defaultModel: Joi.string().optional(),
+    }).required()
   )
 }
 

package/src/api/routes/global/tests/configs.spec.ts

@@ -28,10 +28,7 @@ describe("configs", () => {
       _rev,
     }
     const res = await config.api.configs.saveConfig(data)
-    return {
-      ...data,
-      ...res.body,
-    }
+    return { ...data, ...res }
   }
 
   const saveSettingsConfig = async (

package/src/tests/api/configs.ts

@@ -1,3 +1,8 @@
+import {
+  AIConfig,
+  SaveConfigRequest,
+  SaveConfigResponse,
+} from "@budibase/types"
 import { TestAPI } from "./base"
 
 export class ConfigAPI extends TestAPI {
@@ -17,21 +22,44 @@ export class ConfigAPI extends TestAPI {
       .expect("Content-Type", /json/)
   }
 
-  getAIConfig = () => {
-    return this.request
+  getAIConfig = async () => {
+    const resp = await this.request
       .get(`/api/global/configs/ai`)
       .set(this.config.defaultHeaders())
       .expect(200)
       .expect("Content-Type", /json/)
+    return resp.body as AIConfig
   }
 
-  saveConfig = (data: any) => {
-    return this.request
+  saveConfig = async (
+    data: SaveConfigRequest,
+    expectations?: { status?: number; body?: Record<string, string | RegExp> }
+  ) => {
+    const { status = 200, body } = expectations || {}
+
+    const resp = await this.request
       .post(`/api/global/configs`)
       .send(data)
       .set(this.config.defaultHeaders())
-      .expect(200)
       .expect("Content-Type", /json/)
+
+    if (resp.status !== status) {
+      throw new Error(
+        `Expected status ${status}, got ${resp.status}: ${resp.text}`
+      )
+    }
+
+    if (body) {
+      for (const [key, value] of Object.entries(body)) {
+        if (typeof value === "string") {
+          expect(resp.body[key]).toEqual(value)
+        } else if (value instanceof RegExp) {
+          expect(resp.body[key]).toMatch(value)
+        }
+      }
+    }
+
+    return resp.body as SaveConfigResponse
   }
 
   OIDCCallback = (configId: string, preAuthRes: any) => {

package/src/tests/structures/configs.ts

@@ -5,7 +5,6 @@ import {
   SMTPConfig,
   GoogleConfig,
   OIDCConfig,
-  AIConfig,
 } from "@budibase/types"
 
 export function oidc(conf?: any): OIDCConfig {
@@ -82,20 +81,3 @@ export function settings(conf?: any): SettingsConfig {
     },
   }
 }
-
-export function ai(): AIConfig {
-  return {
-    type: ConfigType.AI,
-    config: {
-      ai: {
-        provider: "OpenAI",
-        isDefault: false,
-        name: "Test",
-        active: true,
-        defaultModel: "gpt4",
-        apiKey: "myapikey",
-        baseUrl: "https://api.example.com",
-      },
-    },
-  }
-}