@budibase/server 1.0.105-alpha.4 → 1.0.105-alpha.40

package/src/integrations/index.ts

@@ -11,6 +11,7 @@ const arangodb = require("./arangodb")
 const rest = require("./rest")
 const googlesheets = require("./googlesheets")
 const firebase = require("./firebase")
+const redis = require("./redis")
 const { SourceNames } = require("../definitions/datasource")
 const environment = require("../environment")
 
@@ -26,6 +27,8 @@ const DEFINITIONS = {
   [SourceNames.MYSQL]: mysql.schema,
   [SourceNames.ARANGODB]: arangodb.schema,
   [SourceNames.REST]: rest.schema,
+  [SourceNames.FIREBASE]: firebase.schema,
+  [SourceNames.REDIS]: redis.schema,
 }
 
 const INTEGRATIONS = {
@@ -42,6 +45,7 @@ const INTEGRATIONS = {
   [SourceNames.REST]: rest.integration,
   [SourceNames.FIREBASE]: firebase.integration,
   [SourceNames.GOOGLE_SHEETS]: googlesheets.integration,
+  [SourceNames.REDIS]: redis.integration,
 }
 
 // optionally add oracle integration if the oracle binary can be installed

package/src/integrations/redis.ts

@@ -0,0 +1,152 @@
+import {
+  DatasourceFieldTypes,
+  Integration,
+  QueryTypes,
+} from "../definitions/datasource"
+import Redis from "ioredis"
+
+module RedisModule {
+  interface RedisConfig {
+    host: string
+    port: number
+    username: string
+    password?: string
+  }
+
+  const SCHEMA: Integration = {
+    docs: "https://redis.io/docs/",
+    description: "",
+    friendlyName: "Redis",
+    datasource: {
+      host: {
+        type: "string",
+        required: true,
+        default: "localhost",
+      },
+      port: {
+        type: "number",
+        required: true,
+        default: 6379,
+      },
+      username: {
+        type: "string",
+        required: false,
+      },
+      password: {
+        type: "password",
+        required: false,
+      },
+    },
+    query: {
+      create: {
+        type: QueryTypes.FIELDS,
+        fields: {
+          key: {
+            type: DatasourceFieldTypes.STRING,
+            required: true,
+          },
+          value: {
+            type: DatasourceFieldTypes.STRING,
+            required: true,
+          },
+          ttl: {
+            type: DatasourceFieldTypes.NUMBER,
+          },
+        },
+      },
+      read: {
+        readable: true,
+        type: QueryTypes.FIELDS,
+        fields: {
+          key: {
+            type: DatasourceFieldTypes.STRING,
+            required: true,
+          },
+        },
+      },
+      delete: {
+        type: QueryTypes.FIELDS,
+        fields: {
+          key: {
+            type: DatasourceFieldTypes.STRING,
+            required: true,
+          },
+        },
+      },
+      command: {
+        readable: true,
+        displayName: "Redis Command",
+        type: QueryTypes.JSON,
+      },
+    },
+  }
+
+  class RedisIntegration {
+    private readonly config: RedisConfig
+    private client: any
+
+    constructor(config: RedisConfig) {
+      this.config = config
+      this.client = new Redis({
+        host: this.config.host,
+        port: this.config.port,
+        username: this.config.username,
+        password: this.config.password,
+      })
+    }
+
+    async disconnect() {
+      this.client.disconnect()
+    }
+
+    async redisContext(query: Function) {
+      try {
+        return await query()
+      } catch (err) {
+        throw new Error(`Redis error: ${err}`)
+      } finally {
+        this.disconnect()
+      }
+    }
+
+    async create(query: { key: string; value: string; ttl: number }) {
+      return this.redisContext(async () => {
+        const response = await this.client.set(query.key, query.value)
+        if (query.ttl) {
+          await this.client.expire(query.key, query.ttl)
+        }
+        return response
+      })
+    }
+
+    async read(query: { key: string }) {
+      return this.redisContext(async () => {
+        const response = await this.client.get(query.key)
+        return response
+      })
+    }
+
+    async delete(query: { key: string }) {
+      return this.redisContext(async () => {
+        const response = await this.client.del(query.key)
+        return response
+      })
+    }
+
+    async command(query: { json: string }) {
+      return this.redisContext(async () => {
+        const commands = query.json.trim().split(" ")
+        const pipeline = this.client.pipeline([commands])
+        const result = await pipeline.exec()
+        return {
+          response: result[0][1],
+        }
+      })
+    }
+  }
+
+  module.exports = {
+    schema: SCHEMA,
+    integration: RedisIntegration,
+  }
+}

package/src/integrations/tests/redis.spec.js

@@ -0,0 +1,60 @@
+const Redis = require("ioredis-mock")
+const RedisIntegration = require("../redis")
+
+class TestConfiguration {
+  constructor(config = {}) {
+    this.integration = new RedisIntegration.integration(config) 
+    this.redis = new Redis({
+      data: {
+        test: 'test',
+        result: "1"
+      },
+    })
+    this.integration.client = this.redis
+  }
+}
+
+describe("Redis Integration", () => {
+  let config 
+
+  beforeEach(() => {
+    config = new TestConfiguration()
+  })
+
+  it("calls the create method with the correct params", async () => {
+    const body = {
+      key: "key",
+      value: "value"
+    }
+    const response = await config.integration.create(body)
+    expect(await config.redis.get("key")).toEqual("value")
+  })
+
+  it("calls the read method with the correct params", async () => {
+    const body = {
+      key: "test"
+    }
+    const response = await config.integration.read(body)
+    expect(response).toEqual("test")
+  })
+
+  it("calls the delete method with the correct params", async () => {
+    const body = {
+      key: "test"
+    }
+    await config.integration.delete(body)
+    expect(await config.redis.get(body.key)).toEqual(null)
+  })
+
+  it("calls the command method with the correct params", async () => {
+    const body = {
+      json: "KEYS *"
+    }
+
+    // ioredis-mock doesn't support pipelines
+    config.integration.client.pipeline = jest.fn(() => ({ exec: jest.fn(() => [[]]) }))
+
+    await config.integration.command(body)
+    expect(config.integration.client.pipeline).toHaveBeenCalledWith([["KEYS", "*"]])
+  })
+})

package/src/middleware/{authorized.js → authorized.ts}

@@ -1,8 +1,8 @@
-const {
+import {
   getUserRoleHierarchy,
   getRequiredResourceRole,
   BUILTIN_ROLE_IDS,
-} = require("@budibase/backend-core/roles")
+} from "@budibase/backend-core/roles"
 const {
   PermissionTypes,
   PermissionLevels,
@@ -13,7 +13,7 @@ const { isWebhookEndpoint } = require("./utils")
 const { buildCsrfMiddleware } = require("@budibase/backend-core/auth")
 const { getAppId } = require("@budibase/backend-core/context")
 
-function hasResource(ctx) {
+function hasResource(ctx: any) {
   return ctx.resourceId != null
 }
 
@@ -25,7 +25,12 @@ const csrf = buildCsrfMiddleware()
  * - Builders can access all resources.
  * - Otherwise the user must have the required role.
  */
-const checkAuthorized = async (ctx, resourceRoles, permType, permLevel) => {
+const checkAuthorized = async (
+  ctx: any,
+  resourceRoles: any,
+  permType: any,
+  permLevel: any
+) => {
   // check if this is a builder api and the user is not a builder
   const isBuilder = ctx.user && ctx.user.builder && ctx.user.builder.global
   const isBuilderApi = permType === PermissionTypes.BUILDER
@@ -40,10 +45,10 @@ const checkAuthorized = async (ctx, resourceRoles, permType, permLevel) => {
 }
 
 const checkAuthorizedResource = async (
-  ctx,
-  resourceRoles,
-  permType,
-  permLevel
+  ctx: any,
+  resourceRoles: any,
+  permType: any,
+  permLevel: any
 ) => {
   // get the user's roles
   const roleId = ctx.roleId || BUILTIN_ROLE_IDS.PUBLIC
@@ -54,7 +59,9 @@ const checkAuthorizedResource = async (
   // check if the user has the required role
   if (resourceRoles.length > 0) {
     // deny access if the user doesn't have the required resource role
-    const found = userRoles.find(role => resourceRoles.indexOf(role._id) !== -1)
+    const found = userRoles.find(
+      (role: any) => resourceRoles.indexOf(role._id) !== -1
+    )
     if (!found) {
       ctx.throw(403, permError)
     }
@@ -64,9 +71,8 @@ const checkAuthorizedResource = async (
   }
 }
 
-module.exports =
-  (permType, permLevel = null, opts = { schema: false }) =>
-  async (ctx, next) => {
+export = (permType: any, permLevel: any = null, opts = { schema: false }) =>
+  async (ctx: any, next: any) => {
     // webhooks don't need authentication, each webhook unique
     // also internal requests (between services) don't need authorized
     if (isWebhookEndpoint(ctx) || ctx.internal) {
@@ -82,8 +88,8 @@ module.exports =
     await builderMiddleware(ctx, permType)
 
     // get the resource roles
-    let resourceRoles = [],
-      otherLevelRoles
+    let resourceRoles: any = []
+    let otherLevelRoles: any = []
     const otherLevel =
       permLevel === PermissionLevels.READ
         ? PermissionLevels.WRITE

package/src/middleware/currentapp.js

@@ -4,7 +4,7 @@ const {
   getCookie,
   clearCookie,
 } = require("@budibase/backend-core/utils")
-const { Cookies } = require("@budibase/backend-core/constants")
+const { Cookies, Headers } = require("@budibase/backend-core/constants")
 const { getRole } = require("@budibase/backend-core/roles")
 const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const { generateUserMetadataID, isDevAppID } = require("../db/utils")
@@ -63,6 +63,21 @@ module.exports = async (ctx, next) => {
     appId = requestAppId
     // retrieving global user gets the right role
     roleId = globalUser.roleId || roleId
+
+    // Allow builders to specify their role via a header
+    const isBuilder =
+      globalUser && globalUser.builder && globalUser.builder.global
+    const isDevApp = appId && isDevAppID(appId)
+    const roleHeader = ctx.request && ctx.request.headers[Headers.PREVIEW_ROLE]
+    if (isBuilder && isDevApp && roleHeader) {
+      // Ensure the role is valid by ensuring a definition exists
+      try {
+        await getRole(roleHeader)
+        roleId = roleHeader
+      } catch (error) {
+        // Swallow error and do nothing
+      }
+    }
   }
 
   // nothing more to do

package/src/migrations/functions/developerQuota.ts

@@ -0,0 +1,15 @@
+const { createUserBuildersView } = require("@budibase/backend-core/db")
+import * as syncDevelopers from "./usageQuotas/syncDevelopers"
+
+/**
+ * Date:
+ * March 2022
+ *
+ * Description:
+ * Create the builder users view and sync the developer count
+ */
+
+export const run = async (db: any) => {
+  await createUserBuildersView(db)
+  await syncDevelopers.run()
+}

package/src/migrations/functions/publishedAppsQuota.ts

@@ -0,0 +1,13 @@
+import * as syncPublishedApps from "./usageQuotas/syncPublishedApps"
+
+/**
+ * Date:
+ * March 2022
+ *
+ * Description:
+ * Sync the published apps count
+ */
+
+export const run = async (db: any) => {
+  await syncPublishedApps.run()
+}

package/src/migrations/functions/tests/quotas1.spec.js

@@ -1,4 +1,3 @@
-const env = require("../../../environment")
 const TestConfig = require("../../../tests/utilities/TestConfiguration")
 
 const syncApps = jest.fn()
@@ -14,7 +13,6 @@ describe("run", () => {
 
   beforeEach(async () => {
     await config.init()
-    env._set("USE_QUOTAS", 1)
   })
 
   afterAll(config.end)  

package/src/migrations/functions/usageQuotas/index.ts

@@ -1,8 +1,3 @@
-const { useQuotas } = require("../../../utilities/usageQuota")
-
 export const runQuotaMigration = async (migration: Function) => {
-  if (!useQuotas()) {
-    return
-  }
   await migration()
 }

package/src/migrations/functions/usageQuotas/syncApps.ts

@@ -1,9 +1,8 @@
-import { getGlobalDB, getTenantId } from "@budibase/backend-core/tenancy"
+import { getTenantId } from "@budibase/backend-core/tenancy"
 import { getAllApps } from "@budibase/backend-core/db"
-import { getUsageQuotaDoc } from "../../../utilities/usageQuota"
+import { quotas, QuotaUsageType, StaticQuotaName } from "@budibase/pro"
 
 export const run = async () => {
-  const db = getGlobalDB()
   // get app count
   // @ts-ignore
   const devApps = await getAllApps({ dev: true })
@@ -12,7 +11,5 @@ export const run = async () => {
   // sync app count
   const tenantId = getTenantId()
   console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`)
-  const usageDoc = await getUsageQuotaDoc(db)
-  usageDoc.usageQuota.apps = appCount
-  await db.put(usageDoc)
+  await quotas.setUsage(appCount, StaticQuotaName.APPS, QuotaUsageType.STATIC)
 }

package/src/migrations/functions/usageQuotas/syncDevelopers.ts

@@ -0,0 +1,19 @@
+import { getTenantId } from "@budibase/backend-core/tenancy"
+import { utils } from "@budibase/backend-core"
+import { quotas, QuotaUsageType, StaticQuotaName } from "@budibase/pro"
+
+export const run = async () => {
+  // get developer count
+  const developerCount = await utils.getBuildersCount()
+
+  // sync developer count
+  const tenantId = getTenantId()
+  console.log(
+    `[Tenant: ${tenantId}] Syncing developer count: ${developerCount}`
+  )
+  await quotas.setUsage(
+    developerCount,
+    StaticQuotaName.DEVELOPERS,
+    QuotaUsageType.STATIC
+  )
+}

package/src/migrations/functions/usageQuotas/syncPublishedApps.ts

@@ -0,0 +1,21 @@
+import { getTenantId } from "@budibase/backend-core/tenancy"
+import { getAllApps } from "@budibase/backend-core/db"
+import { quotas, QuotaUsageType, StaticQuotaName } from "@budibase/pro"
+
+export const run = async () => {
+  // get app count
+  const opts: any = { dev: false }
+  const prodApps = await getAllApps(opts)
+  const prodAppCount = prodApps ? prodApps.length : 0
+
+  // sync app count
+  const tenantId = getTenantId()
+  console.log(
+    `[Tenant: ${tenantId}] Syncing published app count: ${prodAppCount}`
+  )
+  await quotas.setUsage(
+    prodAppCount,
+    StaticQuotaName.PUBLISHED_APPS,
+    QuotaUsageType.STATIC
+  )
+}

package/src/migrations/functions/usageQuotas/syncRows.ts

@@ -1,10 +1,9 @@
-import { getGlobalDB, getTenantId } from "@budibase/backend-core/tenancy"
+import { getTenantId } from "@budibase/backend-core/tenancy"
 import { getAllApps } from "@budibase/backend-core/db"
-import { getUsageQuotaDoc } from "../../../utilities/usageQuota"
 import { getUniqueRows } from "../../../utilities/usageQuota/rows"
+import { quotas, QuotaUsageType, StaticQuotaName } from "@budibase/pro"
 
 export const run = async () => {
-  const db = getGlobalDB()
   // get all rows in all apps
   // @ts-ignore
   const allApps = await getAllApps({ all: true })
@@ -16,7 +15,5 @@ export const run = async () => {
   // sync row count
   const tenantId = getTenantId()
   console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`)
-  const usageDoc = await getUsageQuotaDoc(db)
-  usageDoc.usageQuota.rows = rowCount
-  await db.put(usageDoc)
+  await quotas.setUsage(rowCount, StaticQuotaName.ROWS, QuotaUsageType.STATIC)
 }

package/src/migrations/functions/usageQuotas/tests/syncApps.spec.ts

@@ -0,0 +1,32 @@
+import TestConfig from "../../../../tests/utilities/TestConfiguration"
+import * as syncApps from "../syncApps"
+import { quotas, QuotaUsageType, StaticQuotaName } from "@budibase/pro"
+
+describe("syncApps", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+  })
+
+  afterAll(config.end)
+
+  it("runs successfully", async () => {
+    // create the usage quota doc and mock usages
+    await quotas.getQuotaUsage()
+    await quotas.setUsage(3, StaticQuotaName.APPS, QuotaUsageType.STATIC)
+
+    let usageDoc = await quotas.getQuotaUsage()
+    expect(usageDoc.usageQuota.apps).toEqual(3)
+
+    // create an extra app to test the migration
+    await config.createApp("quota-test")
+
+    // migrate
+    await syncApps.run()
+
+    // assert the migration worked
+    usageDoc = await quotas.getQuotaUsage()
+    expect(usageDoc.usageQuota.apps).toEqual(2)
+  })
+})

package/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts

@@ -0,0 +1,38 @@
+import TestConfig from "../../../../tests/utilities/TestConfiguration"
+import * as syncRows from "../syncRows"
+import { quotas, QuotaUsageType, StaticQuotaName } from "@budibase/pro"
+
+describe("syncRows", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+  })
+
+  afterAll(config.end)
+
+  it("runs successfully", async () => {
+    // create the usage quota doc and mock usages
+    await quotas.getQuotaUsage()
+    await quotas.setUsage(300, StaticQuotaName.ROWS, QuotaUsageType.STATIC)
+
+    let usageDoc = await quotas.getQuotaUsage()
+    expect(usageDoc.usageQuota.rows).toEqual(300)
+
+    // app 1
+    await config.createTable()
+    await config.createRow()
+    // app 2
+    await config.createApp("second-app")
+    await config.createTable()
+    await config.createRow()
+    await config.createRow()
+
+    // migrate
+    await syncRows.run()
+
+    // assert the migration worked
+    usageDoc = await quotas.getQuotaUsage()
+    expect(usageDoc.usageQuota.rows).toEqual(3)
+  })
+})

package/src/migrations/index.ts

@@ -8,6 +8,8 @@ const {
 import * as userEmailViewCasing from "./functions/userEmailViewCasing"
 import * as quota1 from "./functions/quotas1"
 import * as appUrls from "./functions/appUrls"
+import * as developerQuota from "./functions/developerQuota"
+import * as publishedAppsQuota from "./functions/publishedAppsQuota"
 
 export interface Migration {
   type: string
@@ -27,7 +29,7 @@ export interface Migration {
  */
 export interface MigrationOptions {
   tenantIds?: string[]
-  forced?: {
+  force?: {
     [type: string]: string[]
   }
 }
@@ -49,6 +51,16 @@ export const MIGRATIONS: Migration[] = [
     opts: { all: true },
     fn: appUrls.run,
   },
+  {
+    type: MIGRATION_TYPES.GLOBAL,
+    name: "developer_quota",
+    fn: developerQuota.run,
+  },
+  {
+    type: MIGRATION_TYPES.GLOBAL,
+    name: "published_apps_quota",
+    fn: publishedAppsQuota.run,
+  },
 ]
 
 export const migrate = async (options?: MigrationOptions) => {

package/src/module.d.ts

@@ -1,3 +1,7 @@
 declare module "@budibase/backend-core"
 declare module "@budibase/backend-core/tenancy"
 declare module "@budibase/backend-core/db"
+declare module "@budibase/backend-core/context"
+declare module "@budibase/backend-core/cache"
+declare module "@budibase/backend-core/permissions"
+declare module "@budibase/backend-core/roles"