@budibase/server 0.0.32 → 0.0.999-alpha.31

package/src/api/controllers/query/import/sources/tests/openapi3/openapi3.spec.js

@@ -0,0 +1,240 @@
+const fs = require("fs")
+const path = require("path")
+
+const { OpenAPI3 } = require("../../openapi3")
+
+const getData = (file, extension) => {
+  return fs.readFileSync(
+    path.join(__dirname, `./data/${file}/${file}.${extension}`),
+    "utf8"
+  )
+}
+
+describe("OpenAPI3 Import", () => {
+  let openapi3
+
+  beforeEach(() => {
+    openapi3 = new OpenAPI3()
+  })
+
+  it("validates unsupported data", async () => {
+    let data
+    let supported
+
+    // non json / yaml
+    data = "curl http://example.com"
+    supported = await openapi3.isSupported(data)
+    expect(supported).toBe(false)
+
+    // Empty
+    data = ""
+    supported = await openapi3.isSupported(data)
+    expect(supported).toBe(false)
+  })
+
+  const runTests = async (filename, test, assertions) => {
+    for (let extension of ["json", "yaml"]) {
+      await test(filename, extension, assertions)
+    }
+  }
+
+  const testImportInfo = async (file, extension) => {
+    await openapi3.isSupported(getData(file, extension))
+    const info = await openapi3.getInfo()
+    expect(info.name).toBe("Swagger Petstore - OpenAPI 3.0")
+  }
+
+  it("returns import info", async () => {
+    await runTests("petstore", testImportInfo)
+  })
+
+  describe("Returns queries", () => {
+    const indexQueries = queries => {
+      return queries.reduce((acc, query) => {
+        acc[query.name] = query
+        return acc
+      }, {})
+    }
+
+    const getQueries = async (file, extension) => {
+      await openapi3.isSupported(getData(file, extension))
+      const queries = await openapi3.getQueries()
+      expect(queries.length).toBe(6)
+      return indexQueries(queries)
+    }
+
+    const testVerb = async (file, extension, assertions) => {
+      const queries = await getQueries(file, extension)
+      for (let [operationId, method] of Object.entries(assertions)) {
+        expect(queries[operationId].queryVerb).toBe(method)
+      }
+    }
+
+    it("populates verb", async () => {
+      const assertions = {
+        createEntity: "create",
+        getEntities: "read",
+        getEntity: "read",
+        updateEntity: "update",
+        patchEntity: "patch",
+        deleteEntity: "delete",
+      }
+      await runTests("crud", testVerb, assertions)
+    })
+
+    const testPath = async (file, extension, assertions) => {
+      const queries = await getQueries(file, extension)
+      for (let [operationId, urlPath] of Object.entries(assertions)) {
+        expect(queries[operationId].fields.path).toBe(urlPath)
+      }
+    }
+
+    it("populates path", async () => {
+      const assertions = {
+        createEntity: "http://example.com/entities",
+        getEntities: "http://example.com/entities",
+        getEntity: "http://example.com/entities/{{entityId}}",
+        updateEntity: "http://example.com/entities/{{entityId}}",
+        patchEntity: "http://example.com/entities/{{entityId}}",
+        deleteEntity: "http://example.com/entities/{{entityId}}",
+      }
+      await runTests("crud", testPath, assertions)
+    })
+
+    const testHeaders = async (file, extension, assertions) => {
+      const queries = await getQueries(file, extension)
+      for (let [operationId, headers] of Object.entries(assertions)) {
+        expect(queries[operationId].fields.headers).toStrictEqual(headers)
+      }
+    }
+
+    const contentTypeHeader = {
+      "Content-Type": "application/json",
+    }
+
+    it("populates headers", async () => {
+      const assertions = {
+        createEntity: {
+          ...contentTypeHeader,
+        },
+        getEntities: {},
+        getEntity: {},
+        updateEntity: {
+          ...contentTypeHeader,
+        },
+        patchEntity: {
+          ...contentTypeHeader,
+        },
+        deleteEntity: {
+          "x-api-key": "{{x-api-key}}",
+        },
+      }
+
+      await runTests("crud", testHeaders, assertions)
+    })
+
+    const testQuery = async (file, extension, assertions) => {
+      const queries = await getQueries(file, extension)
+      for (let [operationId, queryString] of Object.entries(assertions)) {
+        expect(queries[operationId].fields.queryString).toStrictEqual(
+          queryString
+        )
+      }
+    }
+
+    it("populates query", async () => {
+      const assertions = {
+        createEntity: "",
+        getEntities: "page={{page}}&size={{size}}",
+        getEntity: "",
+        updateEntity: "",
+        patchEntity: "",
+        deleteEntity: "",
+      }
+      await runTests("crud", testQuery, assertions)
+    })
+
+    const testParameters = async (file, extension, assertions) => {
+      const queries = await getQueries(file, extension)
+      for (let [operationId, parameters] of Object.entries(assertions)) {
+        expect(queries[operationId].parameters).toStrictEqual(parameters)
+      }
+    }
+
+    it("populates parameters", async () => {
+      const assertions = {
+        createEntity: [],
+        getEntities: [
+          {
+            name: "page",
+            default: "",
+          },
+          {
+            name: "size",
+            default: "",
+          },
+        ],
+        getEntity: [
+          {
+            name: "entityId",
+            default: "",
+          },
+        ],
+        updateEntity: [
+          {
+            name: "entityId",
+            default: "",
+          },
+        ],
+        patchEntity: [
+          {
+            name: "entityId",
+            default: "",
+          },
+        ],
+        deleteEntity: [
+          {
+            name: "entityId",
+            default: "",
+          },
+          {
+            name: "x-api-key",
+            default: "",
+          },
+        ],
+      }
+      await runTests("crud", testParameters, assertions)
+    })
+
+    const testBody = async (file, extension, assertions) => {
+      const queries = await getQueries(file, extension)
+      for (let [operationId, body] of Object.entries(assertions)) {
+        expect(queries[operationId].fields.requestBody).toStrictEqual(
+          JSON.stringify(body, null, 2)
+        )
+      }
+    }
+    it("populates body", async () => {
+      const assertions = {
+        createEntity: {
+          name: "name",
+          type: "type",
+        },
+        getEntities: undefined,
+        getEntity: undefined,
+        updateEntity: {
+          id: 1,
+          name: "name",
+          type: "type",
+        },
+        patchEntity: {
+          id: 1,
+          name: "name",
+          type: "type",
+        },
+        deleteEntity: undefined,
+      }
+      await runTests("crud", testBody, assertions)
+    })
+  })
+})

package/src/api/controllers/query/import/tests/index.spec.js

@@ -0,0 +1,162 @@
+const TestConfig = require("../../../../../tests/utilities/TestConfiguration")
+const { RestImporter } = require("../index")
+const fs = require("fs")
+const path = require('path')
+const { events} = require("@budibase/backend-core")
+
+const getData = (file) => {
+  return fs.readFileSync(path.join(__dirname, `../sources/tests/${file}`), "utf8")
+}
+
+// openapi2 (swagger)
+const oapi2CrudJson = getData("openapi2/data/crud/crud.json")
+const oapi2CrudYaml = getData("openapi2/data/crud/crud.json")
+const oapi2PetstoreJson = getData("openapi2/data/petstore/petstore.json")
+const oapi2PetstoreYaml = getData("openapi2/data/petstore/petstore.json")
+
+// openapi3 
+const oapi3CrudJson = getData("openapi3/data/crud/crud.json")
+const oapi3CrudYaml = getData("openapi3/data/crud/crud.json")
+const oapi3PetstoreJson = getData("openapi3/data/petstore/petstore.json")
+const oapi3PetstoreYaml = getData("openapi3/data/petstore/petstore.json")
+
+// curl
+const curl = getData("curl/data/post.txt")
+
+const datasets = {
+  // openapi2 (swagger)
+  oapi2CrudJson,
+  oapi2CrudYaml,
+  oapi2PetstoreJson,
+  oapi2PetstoreYaml,
+  // openapi3
+  oapi3CrudJson,
+  oapi3CrudYaml,
+  oapi3PetstoreJson,
+  oapi3PetstoreYaml,
+  // curl
+  curl
+}
+
+describe("Rest Importer", () => {
+  const config = new TestConfig(false)
+
+  beforeAll(async () => {
+    await config.init()
+  })
+
+  let restImporter 
+
+  const init = async (data) => {
+    restImporter = new RestImporter(data)
+    await restImporter.init()
+  }
+
+  const runTest = async (test, assertions) => {
+    await config.doInContext(config.appId, async () => {
+      for (let [key, data] of Object.entries(datasets)) {
+        await test(key, data, assertions)
+      }
+    })
+  }
+
+  const testGetInfo = async (key, data, assertions) => {
+    await init(data)
+    const info = await restImporter.getInfo()
+    expect(info.name).toBe(assertions[key].name)
+  }
+
+  it("gets info", async () => {
+    const assertions = {
+      // openapi2 (swagger)
+      "oapi2CrudJson" : {
+        name: "CRUD",
+      },
+      "oapi2CrudYaml" : {
+        name: "CRUD",
+      },
+      "oapi2PetstoreJson" : {
+        name: "Swagger Petstore",
+      },
+      "oapi2PetstoreYaml" :{
+        name: "Swagger Petstore",
+      },
+      // openapi3
+      "oapi3CrudJson" : {
+        name: "CRUD",
+      },
+      "oapi3CrudYaml" : {
+        name: "CRUD",
+      },
+      "oapi3PetstoreJson" : {
+        name: "Swagger Petstore - OpenAPI 3.0",
+      },
+      "oapi3PetstoreYaml" :{
+        name: "Swagger Petstore - OpenAPI 3.0",
+      },
+      // curl
+      "curl": {
+        name: "example.com",
+      }
+    }
+    await runTest(testGetInfo, assertions)
+  })
+
+  const testImportQueries = async (key, data, assertions) => {
+    await init(data)
+    const datasource = await config.createDatasource()
+    const importResult = await restImporter.importQueries(datasource._id)
+    expect(importResult.errorQueries.length).toBe(0)
+    expect(importResult.queries.length).toBe(assertions[key].count)
+    expect(events.query.imported).toBeCalledTimes(1)
+    expect(events.query.imported).toBeCalledWith(datasource, assertions[key].source, assertions[key].count)
+    jest.clearAllMocks()
+  }
+
+  it("imports queries", async () => {
+    // simple sanity assertions that the whole dataset 
+    // makes it through the importer
+    const assertions = {
+      // openapi2 (swagger)
+      "oapi2CrudJson" : {
+        count: 6,
+        source: "openapi2.0",
+      },
+      "oapi2CrudYaml" :{
+        count: 6,
+        source: "openapi2.0"
+      },
+      "oapi2PetstoreJson" : {
+        count: 20,
+        source: "openapi2.0"
+      },
+      "oapi2PetstoreYaml" :{
+        count: 20,
+        source: "openapi2.0"
+      },
+      // openapi3
+      "oapi3CrudJson" : {
+        count: 6,
+        source: "openapi3.0"
+      },
+      "oapi3CrudYaml" :{
+        count: 6,
+        source: "openapi3.0"
+      },
+      "oapi3PetstoreJson" : {
+        count: 19,
+        source: "openapi3.0"
+      },
+      "oapi3PetstoreYaml" :{
+        count: 19,
+        source: "openapi3.0"
+      },
+      // curl
+      "curl": {
+        count: 1,
+        source: "curl"
+      }
+    }
+    await runTest(testImportQueries, assertions)
+  })
+})

package/src/api/controllers/query/index.ts

@@ -0,0 +1,307 @@
+import { generateQueryID, getQueryParams, isProdAppID } from "../../../db/utils"
+import { BaseQueryVerbs, FieldTypes } from "../../../constants"
+import { Thread, ThreadType } from "../../../threads"
+import { save as saveDatasource } from "../datasource"
+import { RestImporter } from "./import"
+import { invalidateDynamicVariables } from "../../../threads/utils"
+import env from "../../../environment"
+import { quotas } from "@budibase/pro"
+import { events, context, utils, constants } from "@budibase/backend-core"
+import sdk from "../../../sdk"
+import { QueryEvent } from "../../../threads/definitions"
+
+const Runner = new Thread(ThreadType.QUERY, {
+  timeoutMs: env.QUERY_THREAD_TIMEOUT || 10000,
+})
+
+// simple function to append "readable" to all read queries
+function enrichQueries(input: any) {
+  const wasArray = Array.isArray(input)
+  const queries = wasArray ? input : [input]
+  for (let query of queries) {
+    if (query.queryVerb === BaseQueryVerbs.READ) {
+      query.readable = true
+    }
+  }
+  return wasArray ? queries : queries[0]
+}
+
+export async function fetch(ctx: any) {
+  const db = context.getAppDB()
+
+  const body = await db.allDocs(
+    getQueryParams(null, {
+      include_docs: true,
+    })
+  )
+
+  ctx.body = enrichQueries(body.rows.map((row: any) => row.doc))
+}
+
+const _import = async (ctx: any) => {
+  const body = ctx.request.body
+  const data = body.data
+
+  const importer = new RestImporter(data)
+  await importer.init()
+
+  let datasourceId
+  if (!body.datasourceId) {
+    // construct new datasource
+    const info: any = await importer.getInfo()
+    let datasource = {
+      type: "datasource",
+      source: "REST",
+      config: {
+        url: info.url,
+        defaultHeaders: [],
+        rejectUnauthorized: true,
+      },
+      name: info.name,
+    }
+    // save the datasource
+    const datasourceCtx = { ...ctx }
+    datasourceCtx.request.body.datasource = datasource
+    await saveDatasource(datasourceCtx)
+    datasourceId = datasourceCtx.body.datasource._id
+  } else {
+    // use existing datasource
+    datasourceId = body.datasourceId
+  }
+
+  const importResult = await importer.importQueries(datasourceId)
+
+  ctx.body = {
+    ...importResult,
+    datasourceId,
+  }
+  ctx.status = 200
+}
+export { _import as import }
+
+export async function save(ctx: any) {
+  const db = context.getAppDB()
+  const query = ctx.request.body
+
+  const datasource = await sdk.datasources.get(query.datasourceId)
+
+  let eventFn
+  if (!query._id) {
+    query._id = generateQueryID(query.datasourceId)
+    eventFn = () => events.query.created(datasource, query)
+  } else {
+    eventFn = () => events.query.updated(datasource, query)
+  }
+
+  const response = await db.put(query)
+  await eventFn()
+  query._rev = response.rev
+
+  ctx.body = query
+  ctx.message = `Query ${query.name} saved successfully.`
+}
+
+export async function find(ctx: any) {
+  const db = context.getAppDB()
+  const query = enrichQueries(await db.get(ctx.params.queryId))
+  // remove properties that could be dangerous in real app
+  if (isProdAppID(ctx.appId)) {
+    delete query.fields
+    delete query.parameters
+  }
+  ctx.body = query
+}
+
+//Required to discern between OIDC OAuth config entries
+function getOAuthConfigCookieId(ctx: any) {
+  if (ctx.user.providerType === constants.Config.OIDC) {
+    return utils.getCookie(ctx, constants.Cookie.OIDC_CONFIG)
+  }
+}
+
+function getAuthConfig(ctx: any) {
+  const authCookie = utils.getCookie(ctx, constants.Cookie.Auth)
+  let authConfigCtx: any = {}
+  authConfigCtx["configId"] = getOAuthConfigCookieId(ctx)
+  authConfigCtx["sessionId"] = authCookie ? authCookie.sessionId : null
+  return authConfigCtx
+}
+
+export async function preview(ctx: any) {
+  const { datasource, envVars } = await sdk.datasources.getWithEnvVars(
+    ctx.request.body.datasourceId
+  )
+  const query = ctx.request.body
+  // preview may not have a queryId as it hasn't been saved, but if it does
+  // this stops dynamic variables from calling the same query
+  const { fields, parameters, queryVerb, transformer, queryId } = query
+
+  const authConfigCtx: any = getAuthConfig(ctx)
+
+  try {
+    const inputs: QueryEvent = {
+      appId: ctx.appId,
+      datasource,
+      queryVerb,
+      fields,
+      parameters,
+      transformer,
+      queryId,
+      // have to pass down to the thread runner - can't put into context now
+      environmentVariables: envVars,
+      ctx: {
+        user: ctx.user,
+        auth: { ...authConfigCtx },
+      },
+    }
+    const runFn = () => Runner.run(inputs)
+
+    const { rows, keys, info, extra } = await quotas.addQuery(runFn, {
+      datasourceId: datasource._id,
+    })
+    const schemaFields: any = {}
+    if (rows?.length > 0) {
+      for (let key of [...new Set(keys)] as string[]) {
+        const field = rows[0][key]
+        let type = typeof field,
+          fieldType = FieldTypes.STRING
+        if (field)
+          switch (type) {
+            case "boolean":
+              schemaFields[key] = FieldTypes.BOOLEAN
+              break
+            case "object":
+              if (field instanceof Date) {
+                fieldType = FieldTypes.DATETIME
+              } else if (Array.isArray(field)) {
+                fieldType = FieldTypes.ARRAY
+              } else {
+                fieldType = FieldTypes.JSON
+              }
+              break
+            case "number":
+              fieldType = FieldTypes.NUMBER
+              break
+          }
+        schemaFields[key] = fieldType
+      }
+    }
+    // remove configuration before sending event
+    delete datasource.config
+    await events.query.previewed(datasource, query)
+    ctx.body = {
+      rows,
+      schemaFields,
+      info,
+      extra,
+    }
+  } catch (err) {
+    ctx.throw(400, err)
+  }
+}
+
+async function execute(
+  ctx: any,
+  opts: any = { rowsOnly: false, isAutomation: false }
+) {
+  const db = context.getAppDB()
+
+  const query = await db.get(ctx.params.queryId)
+  const { datasource, envVars } = await sdk.datasources.getWithEnvVars(
+    query.datasourceId
+  )
+
+  let authConfigCtx: any = {}
+  if (!opts.isAutomation) {
+    authConfigCtx = getAuthConfig(ctx)
+  }
+  const enrichedParameters = ctx.request.body.parameters || {}
+  // make sure parameters are fully enriched with defaults
+  if (query && query.parameters) {
+    for (let parameter of query.parameters) {
+      if (!enrichedParameters[parameter.name]) {
+        enrichedParameters[parameter.name] = parameter.default
+      }
+    }
+  }
+
+  // call the relevant CRUD method on the integration class
+  try {
+    const inputs: QueryEvent = {
+      appId: ctx.appId,
+      datasource,
+      queryVerb: query.queryVerb,
+      fields: query.fields,
+      pagination: ctx.request.body.pagination,
+      parameters: enrichedParameters,
+      transformer: query.transformer,
+      queryId: ctx.params.queryId,
+      // have to pass down to the thread runner - can't put into context now
+      environmentVariables: envVars,
+      ctx: {
+        user: ctx.user,
+        auth: { ...authConfigCtx },
+      },
+    }
+    const runFn = () => Runner.run(inputs)
+
+    const { rows, pagination, extra } = await quotas.addQuery(runFn, {
+      datasourceId: datasource._id,
+    })
+    // remove the raw from execution incase transformer being used to hide data
+    if (extra?.raw) {
+      delete extra.raw
+    }
+    if (opts && opts.rowsOnly) {
+      ctx.body = rows
+    } else {
+      ctx.body = { data: rows, pagination, ...extra }
+    }
+  } catch (err) {
+    ctx.throw(400, err)
+  }
+}
+
+export async function executeV1(ctx: any) {
+  return execute(ctx, { rowsOnly: true, isAutomation: false })
+}
+
+export async function executeV2(
+  ctx: any,
+  { isAutomation }: { isAutomation?: boolean } = {}
+) {
+  return execute(ctx, { rowsOnly: false, isAutomation })
+}
+
+const removeDynamicVariables = async (queryId: any) => {
+  const db = context.getAppDB()
+  const query = await db.get(queryId)
+  const datasource = await sdk.datasources.get(query.datasourceId)
+  const dynamicVariables = datasource.config?.dynamicVariables as any[]
+
+  if (dynamicVariables) {
+    // delete dynamic variables from the datasource
+    datasource.config!.dynamicVariables = dynamicVariables!.filter(
+      (dv: any) => dv.queryId !== queryId
+    )
+    await db.put(datasource)
+
+    // invalidate the deleted variables
+    const variablesToDelete = dynamicVariables!.filter(
+      (dv: any) => dv.queryId === queryId
+    )
+    await invalidateDynamicVariables(variablesToDelete)
+  }
+}
+
+export async function destroy(ctx: any) {
+  const db = context.getAppDB()
+  const queryId = ctx.params.queryId
+  await removeDynamicVariables(queryId)
+  const query = await db.get(queryId)
+  const datasource = await sdk.datasources.get(query.datasourceId)
+  await db.remove(ctx.params.queryId, ctx.params.revId)
+  ctx.message = `Query deleted.`
+  ctx.status = 200
+  await events.query.deleted(datasource, query)
+}