@budibase/backend-core 2.30.2 → 2.30.3

package/src/features/tests/features.spec.ts

@@ -1,86 +1,230 @@
-import { defaultFlags, fetch, get, Flags } from "../"
+import { IdentityContext, IdentityType, UserCtx } from "@budibase/types"
+import { Flag, FlagSet, FlagValues, init } from "../"
 import { context } from "../.."
-import env from "../../environment"
+import environment, { withEnv } from "../../environment"
+import nodeFetch from "node-fetch"
+import nock from "nock"
 
-async function withFlags<T>(flags: string, f: () => T): Promise<T> {
-  const oldFlags = env.TENANT_FEATURE_FLAGS
-  env._set("TENANT_FEATURE_FLAGS", flags)
-  try {
-    return await f()
-  } finally {
-    env._set("TENANT_FEATURE_FLAGS", oldFlags)
-  }
+const schema = {
+  TEST_BOOLEAN: Flag.boolean(false),
+  TEST_STRING: Flag.string("default value"),
+  TEST_NUMBER: Flag.number(0),
+}
+const flags = new FlagSet(schema)
+
+interface TestCase {
+  it: string
+  identity?: Partial<IdentityContext>
+  environmentFlags?: string
+  posthogFlags?: PostHogFlags
+  licenseFlags?: Array<string>
+  expected?: Partial<FlagValues<typeof schema>>
+  errorMessage?: string | RegExp
+}
+
+interface PostHogFlags {
+  featureFlags?: Record<string, boolean>
+  featureFlagPayloads?: Record<string, string>
+}
+
+function mockPosthogFlags(flags: PostHogFlags) {
+  nock("https://us.i.posthog.com")
+    .post("/decide/?v=3", body => {
+      return body.token === "test" && body.distinct_id === "us_1234"
+    })
+    .reply(200, flags)
+    .persist()
 }
 
 describe("feature flags", () => {
-  interface TestCase {
-    tenant: string
-    flags: string
-    expected: Partial<Flags>
-  }
+  beforeEach(() => {
+    nock.cleanAll()
+  })
 
   it.each<TestCase>([
     {
-      tenant: "tenant1",
-      flags: "tenant1:ONBOARDING_TOUR",
-      expected: { ONBOARDING_TOUR: true },
+      it: "should should find a simple boolean flag in the environment",
+      environmentFlags: "default:TEST_BOOLEAN",
+      expected: { TEST_BOOLEAN: true },
+    },
+    {
+      it: "should should find a simple netgative boolean flag in the environment",
+      environmentFlags: "default:!TEST_BOOLEAN",
+      expected: { TEST_BOOLEAN: false },
+    },
+    {
+      it: "should should match stars in the environment",
+      environmentFlags: "*:TEST_BOOLEAN",
+      expected: { TEST_BOOLEAN: true },
+    },
+    {
+      it: "should not match a different tenant's flags",
+      environmentFlags: "otherTenant:TEST_BOOLEAN",
+      expected: { TEST_BOOLEAN: false },
+    },
+    {
+      it: "should return the defaults when no flags are set",
+      expected: flags.defaults(),
+    },
+    {
+      it: "should fail when an environment flag is not recognised",
+      environmentFlags: "default:TEST_BOOLEAN,default:FOO",
+      errorMessage: "Feature: FOO is not an allowed option",
+    },
+    {
+      it: "should be able to read boolean flags from PostHog",
+      posthogFlags: {
+        featureFlags: { TEST_BOOLEAN: true },
+      },
+      expected: { TEST_BOOLEAN: true },
+    },
+    {
+      it: "should be able to read string flags from PostHog",
+      posthogFlags: {
+        featureFlags: { TEST_STRING: true },
+        featureFlagPayloads: { TEST_STRING: "test" },
+      },
+      expected: { TEST_STRING: "test" },
     },
     {
-      tenant: "tenant1",
-      flags: "tenant1:!ONBOARDING_TOUR",
-      expected: { ONBOARDING_TOUR: false },
+      it: "should be able to read numeric flags from PostHog",
+      posthogFlags: {
+        featureFlags: { TEST_NUMBER: true },
+        featureFlagPayloads: { TEST_NUMBER: "123" },
+      },
+      expected: { TEST_NUMBER: 123 },
     },
     {
-      tenant: "tenant1",
-      flags: "*:ONBOARDING_TOUR",
-      expected: { ONBOARDING_TOUR: true },
+      it: "should not be able to override a negative environment flag from PostHog",
+      environmentFlags: "default:!TEST_BOOLEAN",
+      posthogFlags: {
+        featureFlags: { TEST_BOOLEAN: true },
+      },
+      expected: { TEST_BOOLEAN: false },
     },
     {
-      tenant: "tenant1",
-      flags: "tenant2:ONBOARDING_TOUR",
-      expected: { ONBOARDING_TOUR: false },
+      it: "should not be able to override a positive environment flag from PostHog",
+      environmentFlags: "default:TEST_BOOLEAN",
+      posthogFlags: {
+        featureFlags: {
+          TEST_BOOLEAN: false,
+        },
+      },
+      expected: { TEST_BOOLEAN: true },
     },
     {
-      tenant: "tenant1",
-      flags: "",
-      expected: defaultFlags(),
+      it: "should be able to set boolean flags through the license",
+      licenseFlags: ["TEST_BOOLEAN"],
+      expected: { TEST_BOOLEAN: true },
+    },
+    {
+      it: "should not be able to override a negative environment flag from license",
+      environmentFlags: "default:!TEST_BOOLEAN",
+      licenseFlags: ["TEST_BOOLEAN"],
+      expected: { TEST_BOOLEAN: false },
+    },
+    {
+      it: "should not error on unrecognised PostHog flag",
+      posthogFlags: {
+        featureFlags: { UNDEFINED: true },
+      },
+      expected: flags.defaults(),
+    },
+    {
+      it: "should not error on unrecognised license flag",
+      licenseFlags: ["UNDEFINED"],
+      expected: flags.defaults(),
     },
   ])(
-    'should find flags $expected for $tenant with string "$flags"',
-    ({ tenant, flags, expected }) =>
-      context.doInTenant(tenant, () =>
-        withFlags(flags, async () => {
-          const flags = await fetch()
-          expect(flags).toMatchObject(expected)
+    "$it",
+    async ({
+      identity,
+      environmentFlags,
+      posthogFlags,
+      licenseFlags,
+      expected,
+      errorMessage,
+    }) => {
+      const env: Partial<typeof environment> = {
+        TENANT_FEATURE_FLAGS: environmentFlags,
+      }
 
-          for (const [key, expectedValue] of Object.entries(expected)) {
-            const value = await get(key as keyof Flags)
-            expect(value).toBe(expectedValue)
-          }
+      if (posthogFlags) {
+        mockPosthogFlags(posthogFlags)
+        env.POSTHOG_TOKEN = "test"
+        env.POSTHOG_API_HOST = "https://us.i.posthog.com"
+        env.POSTHOG_PERSONAL_TOKEN = "test"
+      }
+
+      const ctx = { user: { license: { features: licenseFlags || [] } } }
+
+      await withEnv(env, async () => {
+        // We need to pass in node-fetch here otherwise nock won't get used
+        // because posthog-node uses axios under the hood.
+        init({
+          fetch: (url, opts) => {
+            return nodeFetch(url, opts)
+          },
         })
-      )
-  )
 
-  interface FailedTestCase {
-    tenant: string
-    flags: string
-    expected: string | RegExp
-  }
+        const fullIdentity: IdentityContext = {
+          _id: "us_1234",
+          tenantId: "default",
+          type: IdentityType.USER,
+          email: "test@example.com",
+          firstName: "Test",
+          lastName: "User",
+          ...identity,
+        }
 
-  it.each<FailedTestCase>([
-    {
-      tenant: "tenant1",
-      flags: "tenant1:ONBOARDING_TOUR,tenant1:FOO",
-      expected: "Feature: FOO is not an allowed option",
-    },
-  ])(
-    "should fail with message \"$expected\" for $tenant with string '$flags'",
-    async ({ tenant, flags, expected }) => {
-      context.doInTenant(tenant, () =>
-        withFlags(flags, async () => {
-          await expect(fetch()).rejects.toThrow(expected)
+        await context.doInIdentityContext(fullIdentity, async () => {
+          if (errorMessage) {
+            await expect(flags.fetch(ctx as UserCtx)).rejects.toThrow(
+              errorMessage
+            )
+          } else if (expected) {
+            const values = await flags.fetch(ctx as UserCtx)
+            expect(values).toMatchObject(expected)
+
+            for (const [key, expectedValue] of Object.entries(expected)) {
+              const value = await flags.get(
+                key as keyof typeof schema,
+                ctx as UserCtx
+              )
+              expect(value).toBe(expectedValue)
+            }
+          } else {
+            throw new Error("No expected value")
+          }
         })
-      )
+      })
     }
   )
+
+  it("should not error if PostHog is down", async () => {
+    const identity: IdentityContext = {
+      _id: "us_1234",
+      tenantId: "default",
+      type: IdentityType.USER,
+      email: "test@example.com",
+      firstName: "Test",
+      lastName: "User",
+    }
+
+    nock("https://us.i.posthog.com")
+      .post("/decide/?v=3", body => {
+        return body.token === "test" && body.distinct_id === "us_1234"
+      })
+      .reply(503)
+      .persist()
+
+    await withEnv(
+      { POSTHOG_TOKEN: "test", POSTHOG_API_HOST: "https://us.i.posthog.com" },
+      async () => {
+        await context.doInIdentityContext(identity, async () => {
+          await flags.fetch()
+        })
+      }
+    )
+  })
 })

package/src/index.ts

@@ -27,7 +27,7 @@ export * as locks from "./redis/redlockImpl"
 export * as utils from "./utils"
 export * as errors from "./errors"
 export * as timers from "./timers"
-export { default as env } from "./environment"
+export { default as env, withEnv, setEnv } from "./environment"
 export * as blacklist from "./blacklist"
 export * as docUpdates from "./docUpdates"
 export * from "./utils/Duration"

package/src/middleware/passport/sso/tests/oidc.spec.ts

@@ -1,4 +1,4 @@
-import { generator, mocks, structures } from "../../../../../tests"
+import { generator, structures } from "../../../../../tests"
 import {
   JwtClaims,
   OIDCInnerConfig,
@@ -7,6 +7,7 @@ import {
 } from "@budibase/types"
 import * as _sso from "../sso"
 import * as oidc from "../oidc"
+import nock from "nock"
 
 jest.mock("@techpass/passport-openidconnect")
 const mockStrategy = require("@techpass/passport-openidconnect").Strategy
@@ -22,16 +23,9 @@ describe("oidc", () => {
   const oidcConfig: OIDCInnerConfig = structures.sso.oidcConfig()
   const wellKnownConfig = structures.sso.oidcWellKnownConfig()
 
-  function mockRetrieveWellKnownConfig() {
-    // mock the request to retrieve the oidc configuration
-    mocks.fetch.mockReturnValue({
-      ok: true,
-      json: () => wellKnownConfig,
-    })
-  }
-
   beforeEach(() => {
-    mockRetrieveWellKnownConfig()
+    nock.cleanAll()
+    nock(oidcConfig.configUrl).get("/").reply(200, wellKnownConfig)
   })
 
   describe("strategyFactory", () => {
@@ -42,8 +36,6 @@ describe("oidc", () => {
       )
       await oidc.strategyFactory(strategyConfiguration, mockSaveUser)
 
-      expect(mocks.fetch).toHaveBeenCalledWith(oidcConfig.configUrl)
-
       const expectedOptions = {
         issuer: wellKnownConfig.issuer,
         authorizationURL: wellKnownConfig.authorization_endpoint,

package/src/middleware/passport/sso/tests/sso.spec.ts

@@ -1,10 +1,11 @@
-import { structures, mocks } from "../../../../../tests"
+import { structures } from "../../../../../tests"
 import { testEnv } from "../../../../../tests/extra"
 import { SSOAuthDetails, User } from "@budibase/types"
 
 import { HTTPError } from "../../../../errors"
 import * as sso from "../sso"
 import * as context from "../../../../context"
+import nock from "nock"
 
 const mockDone = jest.fn()
 const mockSaveUser = jest.fn()
@@ -23,6 +24,7 @@ describe("sso", () => {
     beforeEach(() => {
       jest.clearAllMocks()
       testEnv.singleTenant()
+      nock.cleanAll()
     })
 
     describe("validation", () => {
@@ -51,15 +53,6 @@ describe("sso", () => {
       })
     })
 
-    function mockGetProfilePicture() {
-      mocks.fetch.mockReturnValueOnce(
-        Promise.resolve({
-          status: 200,
-          headers: { get: () => "image/" },
-        })
-      )
-    }
-
     describe("when the user doesn't exist", () => {
       let user: User
       let details: SSOAuthDetails
@@ -68,7 +61,10 @@ describe("sso", () => {
         users.getById.mockImplementationOnce(() => {
           throw new HTTPError("", 404)
         })
-        mockGetProfilePicture()
+
+        nock("http://example.com").get("/").reply(200, undefined, {
+          "Content-Type": "image/png",
+        })
 
         user = structures.users.user()
         delete user._rev
@@ -131,7 +127,9 @@ describe("sso", () => {
         existingUser = structures.users.user()
         existingUser._id = structures.uuid()
         details = structures.sso.authDetails(existingUser)
-        mockGetProfilePicture()
+        nock("http://example.com").get("/").reply(200, undefined, {
+          "Content-Type": "image/png",
+        })
       })
 
       describe("exists by email", () => {

package/src/plugin/tests/validation.spec.ts

@@ -1,12 +1,129 @@
 import { validate } from "../utils"
 import fetch from "node-fetch"
 import { PluginType } from "@budibase/types"
+import nock from "nock"
 
-const repoUrl =
-  "https://raw.githubusercontent.com/Budibase/budibase-skeleton/master"
-const automationLink = `${repoUrl}/automation/schema.json.hbs`
-const componentLink = `${repoUrl}/component/schema.json.hbs`
-const datasourceLink = `${repoUrl}/datasource/schema.json.hbs`
+const automationLink = `http://example.com/automation/schema.json`
+const componentLink = `http://example.com/component/schema.json`
+const datasourceLink = `http://example.com/datasource/schema.json`
+
+function mockDatasourceSchema() {
+  nock("http://example.com")
+    .get("/datasource/schema.json")
+    .reply(200, {
+      type: "datasource",
+      metadata: {},
+      schema: {
+        docs: "https://docs.budibase.com",
+        friendlyName: "Basic HTTP",
+        type: "API",
+        description: "Performs a basic HTTP calls to a URL",
+        datasource: {
+          url: {
+            type: "string",
+            required: true,
+          },
+          cookie: {
+            type: "string",
+            required: false,
+          },
+        },
+        query: {
+          create: {
+            type: "json",
+          },
+          read: {
+            type: "fields",
+            fields: {
+              queryString: {
+                display: "Query string",
+                type: "string",
+                required: false,
+              },
+            },
+          },
+          update: {
+            type: "json",
+          },
+          delete: {
+            type: "fields",
+            fields: {
+              id: {
+                type: "string",
+                required: true,
+              },
+            },
+          },
+        },
+      },
+    })
+}
+
+function mockAutomationSchema() {
+  nock("http://example.com")
+    .get("/automation/schema.json")
+    .reply(200, {
+      type: "automation",
+      metadata: {},
+      schema: {
+        name: "{{ name }}",
+        tagline: "{{ description }}",
+        icon: "Actions",
+        description: "{{ description }}",
+        type: "action",
+        stepId: "{{ name }}",
+        inputs: {
+          text: "",
+        },
+        schema: {
+          inputs: {
+            properties: {
+              text: {
+                type: "string",
+                title: "Log",
+              },
+            },
+            required: ["text"],
+          },
+          outputs: {
+            properties: {
+              success: {
+                type: "boolean",
+                description: "Whether the action was successful",
+              },
+              message: {
+                type: "string",
+                description: "What was output",
+              },
+            },
+            required: ["success", "message"],
+          },
+        },
+      },
+    })
+}
+
+function mockComponentSchema() {
+  nock("http://example.com")
+    .get("/component/schema.json")
+    .reply(200, {
+      type: "component",
+      metadata: {},
+      schema: {
+        name: "{{ name }}",
+        friendlyName: "{{ name }}",
+        description: "{{ description }}",
+        icon: "Text",
+        settings: [
+          {
+            type: "text",
+            key: "text",
+            label: "Text",
+          },
+        ],
+      },
+    })
+}
 
 async function getSchema(link: string) {
   const response = await fetch(link)
@@ -31,53 +148,62 @@ async function runTest(opts: { link?: string; schema?: any }) {
   return error
 }
 
-describe("it should be able to validate an automation schema", () => {
-  it("should return automation skeleton schema is valid", async () => {
-    const error = await runTest({ link: automationLink })
-    expect(error).toBeUndefined()
+describe("plugin validation", () => {
+  beforeEach(() => {
+    nock.cleanAll()
+    mockAutomationSchema()
+    mockComponentSchema()
+    mockDatasourceSchema()
   })
 
-  it("should fail given invalid automation schema", async () => {
-    const error = await runTest({
-      schema: {
-        type: PluginType.AUTOMATION,
-        schema: {},
-      },
+  describe("it should be able to validate an automation schema", () => {
+    it("should return automation skeleton schema is valid", async () => {
+      const error = await runTest({ link: automationLink })
+      expect(error).toBeUndefined()
     })
-    expect(error).toBeDefined()
-  })
-})
 
-describe("it should be able to validate a component schema", () => {
-  it("should return component skeleton schema is valid", async () => {
-    const error = await runTest({ link: componentLink })
-    expect(error).toBeUndefined()
+    it("should fail given invalid automation schema", async () => {
+      const error = await runTest({
+        schema: {
+          type: PluginType.AUTOMATION,
+          schema: {},
+        },
+      })
+      expect(error).toBeDefined()
+    })
   })
 
-  it("should fail given invalid component schema", async () => {
-    const error = await runTest({
-      schema: {
-        type: PluginType.COMPONENT,
-        schema: {},
-      },
+  describe("it should be able to validate a component schema", () => {
+    it("should return component skeleton schema is valid", async () => {
+      const error = await runTest({ link: componentLink })
+      expect(error).toBeUndefined()
     })
-    expect(error).toBeDefined()
-  })
-})
 
-describe("it should be able to validate a datasource schema", () => {
-  it("should return datasource skeleton schema is valid", async () => {
-    const error = await runTest({ link: datasourceLink })
-    expect(error).toBeUndefined()
+    it("should fail given invalid component schema", async () => {
+      const error = await runTest({
+        schema: {
+          type: PluginType.COMPONENT,
+          schema: {},
+        },
+      })
+      expect(error).toBeDefined()
+    })
   })
 
-  it("should fail given invalid datasource schema", async () => {
-    const error = await runTest({
-      schema: {
-        type: PluginType.DATASOURCE,
-        schema: {},
-      },
+  describe("it should be able to validate a datasource schema", () => {
+    it("should return datasource skeleton schema is valid", async () => {
+      const error = await runTest({ link: datasourceLink })
+      expect(error).toBeUndefined()
+    })
+
+    it("should fail given invalid datasource schema", async () => {
+      const error = await runTest({
+        schema: {
+          type: PluginType.DATASOURCE,
+          schema: {},
+        },
+      })
+      expect(error).toBeDefined()
     })
-    expect(error).toBeDefined()
   })
 })

package/src/redis/redis.ts

@@ -111,6 +111,10 @@ function init(selectDb = DEFAULT_SELECT_DB) {
   CLIENTS[selectDb] = client
 }
 
+export function closeAll() {
+  Object.values(CLIENTS).forEach(client => client.disconnect())
+}
+
 function waitForConnection(selectDb: number = DEFAULT_SELECT_DB) {
   return new Promise(resolve => {
     if (pickClient(selectDb) == null) {

package/src/redis/tests/redis.spec.ts

@@ -1,6 +1,6 @@
 import { GenericContainer, StartedTestContainer } from "testcontainers"
 import { generator, structures } from "../../../tests"
-import RedisWrapper from "../redis"
+import RedisWrapper, { closeAll } from "../redis"
 import { env } from "../.."
 import { randomUUID } from "crypto"
 
@@ -23,7 +23,10 @@ describe("redis", () => {
     env._set("REDIS_PASSWORD", 0)
   })
 
-  afterAll(() => container?.stop())
+  afterAll(() => {
+    container?.stop()
+    closeAll()
+  })
 
   beforeEach(async () => {
     redis = new RedisWrapper(structures.db.id())

package/src/security/auth.ts

@@ -1,6 +1,6 @@
 import env from "../environment"
 
-export const PASSWORD_MIN_LENGTH = +(env.PASSWORD_MIN_LENGTH || 8)
+export const PASSWORD_MIN_LENGTH = +(env.PASSWORD_MIN_LENGTH || 12)
 export const PASSWORD_MAX_LENGTH = +(env.PASSWORD_MAX_LENGTH || 512)
 
 export function validatePassword(