@budibase/server 1.2.37 → 1.2.39-alpha.1

package/src/definitions/datasource.ts

@@ -1,214 +1,13 @@
-import { Row, Table, Base } from "./common"
-
-export enum Operation {
-  CREATE = "CREATE",
-  READ = "READ",
-  UPDATE = "UPDATE",
-  DELETE = "DELETE",
-  BULK_CREATE = "BULK_CREATE",
-  CREATE_TABLE = "CREATE_TABLE",
-  UPDATE_TABLE = "UPDATE_TABLE",
-  DELETE_TABLE = "DELETE_TABLE",
-}
-
-export enum SortDirection {
-  ASCENDING = "ASCENDING",
-  DESCENDING = "DESCENDING",
-}
-
-export enum QueryTypes {
-  SQL = "sql",
-  JSON = "json",
-  FIELDS = "fields",
-}
-
-export enum DatasourceFieldTypes {
-  STRING = "string",
-  LONGFORM = "longForm",
-  BOOLEAN = "boolean",
-  NUMBER = "number",
-  PASSWORD = "password",
-  LIST = "list",
-  OBJECT = "object",
-  JSON = "json",
-  FILE = "file",
-}
-
-export enum SourceNames {
-  POSTGRES = "POSTGRES",
-  DYNAMODB = "DYNAMODB",
-  MONGODB = "MONGODB",
-  ELASTICSEARCH = "ELASTICSEARCH",
-  COUCHDB = "COUCHDB",
-  SQL_SERVER = "SQL_SERVER",
-  S3 = "S3",
-  AIRTABLE = "AIRTABLE",
-  MYSQL = "MYSQL",
-  ARANGODB = "ARANGODB",
-  REST = "REST",
-  ORACLE = "ORACLE",
-  GOOGLE_SHEETS = "GOOGLE_SHEETS",
-  FIRESTORE = "FIRESTORE",
-  REDIS = "REDIS",
-  SNOWFLAKE = "SNOWFLAKE",
-}
-
-export enum IncludeRelationships {
-  INCLUDE = 1,
-  EXCLUDE = 0,
-}
-
-export enum FilterTypes {
-  STRING = "string",
-  FUZZY = "fuzzy",
-  RANGE = "range",
-  EQUAL = "equal",
-  NOT_EQUAL = "notEqual",
-  EMPTY = "empty",
-  NOT_EMPTY = "notEmpty",
-  ONE_OF = "oneOf",
-}
-
-export interface QueryDefinition {
-  type: QueryTypes
-  displayName?: string
-  readable?: boolean
-  customisable?: boolean
-  fields?: object
-  urlDisplay?: boolean
-}
-
-export interface ExtraQueryConfig {
-  [key: string]: {
-    displayName: string
-    type: string
-    required: boolean
-    data?: object
-  }
-}
-
-export interface Integration {
-  docs: string
-  plus?: boolean
-  auth?: { type: string }
-  relationships?: boolean
-  description: string
-  friendlyName: string
-  type?: string
-  datasource: {}
-  query: {
-    [key: string]: QueryDefinition
-  }
-  extra?: ExtraQueryConfig
-}
-
-export interface SearchFilters {
-  allOr?: boolean
-  string?: {
-    [key: string]: string
-  }
-  fuzzy?: {
-    [key: string]: string
-  }
-  range?: {
-    [key: string]: {
-      high: number | string
-      low: number | string
-    }
-  }
-  equal?: {
-    [key: string]: any
-  }
-  notEqual?: {
-    [key: string]: any
-  }
-  empty?: {
-    [key: string]: any
-  }
-  notEmpty?: {
-    [key: string]: any
-  }
-  oneOf?: {
-    [key: string]: any[]
-  }
-  contains?: {
-    [key: string]: any
-  }
-}
-
-export interface SortJson {
-  [key: string]: SortDirection
-}
-
-export interface PaginationJson {
-  limit: number
-  page?: string | number
-}
-
-export interface RenameColumn {
-  old: string
-  updated: string
-}
-
-export interface RelationshipsJson {
-  through?: string
-  from?: string
-  to?: string
-  fromPrimary?: string
-  toPrimary?: string
-  tableName: string
-  column: string
-}
-
-export interface QueryJson {
-  endpoint: {
-    datasourceId: string
-    entityId: string
-    operation: Operation
-    schema?: string
-  }
-  resource: {
-    fields: string[]
-  }
-  filters?: SearchFilters
-  sort?: SortJson
-  paginate?: PaginationJson
-  body?: Row | Row[]
-  table?: Table
-  meta?: {
-    table?: Table
-    tables?: Record<string, Table>
-    renamed: RenameColumn
-  }
-  extra?: {
-    idFilter?: SearchFilters
-  }
-  relationships?: RelationshipsJson[]
-}
-
-export interface SqlQuery {
-  sql: string
-  bindings?: string[]
-}
+/********************************************
+ * This file contains structures which are  *
+ * internal to the server and don't need to *
+ * be exposed for use by other services.    *
+ ********************************************/
 
 export interface QueryOptions {
   disableReturning?: boolean
 }
 
-export interface Datasource extends Base {
-  type: string
-  name: string
-  source: SourceNames
-  // the config is defined by the schema
-  config: {
-    [key: string]: string | number | boolean
-  }
-  plus: boolean
-  entities?: {
-    [key: string]: Table
-  }
-}
-
 export enum AuthType {
   BASIC = "basic",
   BEARER = "bearer",
@@ -230,25 +29,6 @@ export interface BearerAuthConfig {
   token: string
 }
 
-export interface QueryParameter {
-  name: string
-  default: string
-}
-
-export interface RestQueryFields {
-  path: string
-  queryString?: string
-  headers: { [key: string]: any }
-  disabledHeaders: { [key: string]: any }
-  requestBody: any
-  bodyType: string
-  json: object
-  method: string
-  authConfigId: string
-  pagination: PaginationConfig | null
-  paginationValues: PaginationValues | null
-}
-
 export interface RestConfig {
   url: string
   defaultHeaders: {
@@ -266,28 +46,3 @@ export interface RestConfig {
     }
   ]
 }
-
-export interface PaginationConfig {
-  type: string
-  location: string
-  pageParam: string
-  sizeParam: string | null
-  responseParam: string | null
-}
-
-export interface PaginationValues {
-  page: string | number | null
-  limit: number | null
-}
-
-export interface Query {
-  _id?: string
-  datasourceId: string
-  name: string
-  parameters: QueryParameter[]
-  fields: RestQueryFields | any
-  transformer: string | null
-  schema: any
-  readable: boolean
-  queryVerb: string
-}

package/src/integrations/airtable.ts

@@ -1,9 +1,9 @@
 import {
   Integration,
-  DatasourceFieldTypes,
-  QueryTypes,
-} from "../definitions/datasource"
-import { IntegrationBase } from "./base/IntegrationBase"
+  DatasourceFieldType,
+  QueryType,
+  IntegrationBase,
+} from "@budibase/types"
 
 module AirtableModule {
   const Airtable = require("airtable")
@@ -21,61 +21,61 @@ module AirtableModule {
     type: "Spreadsheet",
     datasource: {
       apiKey: {
-        type: DatasourceFieldTypes.PASSWORD,
+        type: DatasourceFieldType.PASSWORD,
         default: "enter api key",
         required: true,
       },
       base: {
-        type: DatasourceFieldTypes.STRING,
+        type: DatasourceFieldType.STRING,
         default: "mybase",
         required: true,
       },
     },
     query: {
       create: {
-        type: QueryTypes.FIELDS,
+        type: QueryType.FIELDS,
         customisable: true,
         fields: {
           table: {
-            type: DatasourceFieldTypes.STRING,
+            type: DatasourceFieldType.STRING,
             required: true,
           },
         },
       },
       read: {
-        type: QueryTypes.FIELDS,
+        type: QueryType.FIELDS,
         fields: {
           table: {
-            type: DatasourceFieldTypes.STRING,
+            type: DatasourceFieldType.STRING,
             required: true,
           },
           view: {
-            type: DatasourceFieldTypes.STRING,
+            type: DatasourceFieldType.STRING,
             required: true,
           },
           numRecords: {
-            type: DatasourceFieldTypes.NUMBER,
+            type: DatasourceFieldType.NUMBER,
             default: 10,
           },
         },
       },
       update: {
-        type: QueryTypes.FIELDS,
+        type: QueryType.FIELDS,
         customisable: true,
         fields: {
           id: {
             display: "Record ID",
-            type: DatasourceFieldTypes.STRING,
+            type: DatasourceFieldType.STRING,
             required: true,
           },
           table: {
-            type: DatasourceFieldTypes.STRING,
+            type: DatasourceFieldType.STRING,
             required: true,
           },
         },
       },
       delete: {
-        type: QueryTypes.JSON,
+        type: QueryType.JSON,
       },
     },
   }

package/src/integrations/arangodb.ts

@@ -1,9 +1,9 @@
 import {
   Integration,
-  DatasourceFieldTypes,
-  QueryTypes,
-} from "../definitions/datasource"
-import { IntegrationBase } from "./base/IntegrationBase"
+  DatasourceFieldType,
+  QueryType,
+  IntegrationBase,
+} from "@budibase/types"
 
 module ArangoModule {
   const { Database, aql } = require("arangojs")
@@ -24,35 +24,35 @@ module ArangoModule {
       "ArangoDB is a scalable open-source multi-model database natively supporting graph, document and search. All supported data models & access patterns can be combined in queries allowing for maximal flexibility. ",
     datasource: {
       url: {
-        type: DatasourceFieldTypes.STRING,
+        type: DatasourceFieldType.STRING,
         default: "http://localhost:8529",
         required: true,
       },
       username: {
-        type: DatasourceFieldTypes.STRING,
+        type: DatasourceFieldType.STRING,
         default: "root",
         required: true,
       },
       password: {
-        type: DatasourceFieldTypes.PASSWORD,
+        type: DatasourceFieldType.PASSWORD,
         required: true,
       },
       databaseName: {
-        type: DatasourceFieldTypes.STRING,
+        type: DatasourceFieldType.STRING,
         default: "_system",
         required: true,
       },
       collection: {
-        type: DatasourceFieldTypes.STRING,
+        type: DatasourceFieldType.STRING,
         required: true,
       },
     },
     query: {
       read: {
-        type: QueryTypes.SQL,
+        type: QueryType.SQL,
       },
       create: {
-        type: QueryTypes.JSON,
+        type: QueryType.JSON,
       },
     },
   }

package/src/integrations/base/query.ts

@@ -0,0 +1,16 @@
+import { QueryJson, Datasource } from "@budibase/types"
+const { integrations } = require("../index")
+
+export async function makeExternalQuery(
+  datasource: Datasource,
+  json: QueryJson
+) {
+  const Integration = integrations[datasource.source]
+  // query is the opinionated function
+  if (Integration.prototype.query) {
+    const integration = new Integration(datasource.config)
+    return integration.query(json)
+  } else {
+    throw "Datasource does not support query."
+  }
+}

package/src/integrations/base/sql.ts

@@ -2,14 +2,15 @@ import { Knex, knex } from "knex"
 import {
   Operation,
   QueryJson,
-  QueryOptions,
   RelationshipsJson,
   SearchFilters,
   SortDirection,
-} from "../../definitions/datasource"
-import { isIsoDateString, SqlClients } from "../utils"
+} from "@budibase/types"
+import { QueryOptions } from "../../definitions/datasource"
+import { isIsoDateString, SqlClient } from "../utils"
 import SqlTableQueryBuilder from "./sqlTable"
 import environment from "../../environment"
+import { removeKeyNumbering } from "./utils"
 
 const envLimit = environment.SQL_MAX_ROWS
   ? parseInt(environment.SQL_MAX_ROWS)
@@ -27,14 +28,14 @@ function likeKey(client: string, key: string): string {
   }
   let start: string, end: string
   switch (client) {
-    case SqlClients.MY_SQL:
+    case SqlClient.MY_SQL:
       start = end = "`"
       break
-    case SqlClients.ORACLE:
-    case SqlClients.POSTGRES:
+    case SqlClient.ORACLE:
+    case SqlClient.POSTGRES:
       start = end = '"'
       break
-    case SqlClients.MS_SQL:
+    case SqlClient.MS_SQL:
       start = "["
       end = "]"
       break
@@ -102,7 +103,7 @@ function generateSelectStatement(
     if (
       columnName &&
       schema?.[columnName] &&
-      knex.client.config.client === SqlClients.POSTGRES
+      knex.client.config.client === SqlClient.POSTGRES
     ) {
       const externalType = schema[columnName].externalType
       if (externalType?.includes("money")) {
@@ -133,12 +134,13 @@ class InternalBuilder {
       fn: (key: string, value: any) => void
     ) {
       for (let [key, value] of Object.entries(structure)) {
-        const isRelationshipField = key.includes(".")
+        const updatedKey = removeKeyNumbering(key)
+        const isRelationshipField = updatedKey.includes(".")
         if (!opts.relationship && !isRelationshipField) {
-          fn(`${opts.tableName}.${key}`, value)
+          fn(`${opts.tableName}.${updatedKey}`, value)
         }
         if (opts.relationship && isRelationshipField) {
-          fn(key, value)
+          fn(updatedKey, value)
         }
       }
     }
@@ -146,7 +148,7 @@ class InternalBuilder {
     const like = (key: string, value: any) => {
       const fnc = allOr ? "orWhere" : "where"
       // postgres supports ilike, nothing else does
-      if (this.client === SqlClients.POSTGRES) {
+      if (this.client === SqlClient.POSTGRES) {
         query = query[fnc](key, "ilike", `%${value}%`)
       } else {
         const rawFnc = `${fnc}Raw`
@@ -173,7 +175,7 @@ class InternalBuilder {
       iterate(filters.string, (key, value) => {
         const fnc = allOr ? "orWhere" : "where"
         // postgres supports ilike, nothing else does
-        if (this.client === SqlClients.POSTGRES) {
+        if (this.client === SqlClient.POSTGRES) {
           query = query[fnc](key, "ilike", `${value}%`)
         } else {
           const rawFnc = `${fnc}Raw`
@@ -229,7 +231,7 @@ class InternalBuilder {
     if (filters.contains) {
       const fnc = allOr ? "orWhere" : "where"
       const rawFnc = `${fnc}Raw`
-      if (this.client === SqlClients.POSTGRES) {
+      if (this.client === SqlClient.POSTGRES) {
         iterate(filters.contains, (key: string, value: any) => {
           const fieldNames = key.split(/\./g)
           const tableName = fieldNames[0]
@@ -242,7 +244,7 @@ class InternalBuilder {
             `"${tableName}"."${columnName}"::jsonb @> '[${value}]'`
           )
         })
-      } else if (this.client === SqlClients.MY_SQL) {
+      } else if (this.client === SqlClient.MY_SQL) {
         iterate(filters.contains, (key: string, value: any) => {
           if (typeof value === "string") {
             value = `"${value}"`
@@ -265,7 +267,7 @@ class InternalBuilder {
         const direction = value === SortDirection.ASCENDING ? "asc" : "desc"
         query = query.orderBy(`${table?.name}.${key}`, direction)
       }
-    } else if (this.client === SqlClients.MS_SQL && paginate?.limit) {
+    } else if (this.client === SqlClient.MS_SQL && paginate?.limit) {
       // @ts-ignore
       query = query.orderBy(`${table?.name}.${table?.primary[0]}`)
     }
@@ -414,7 +416,7 @@ class InternalBuilder {
       [tableName]: query,
     }).select(selectStatement)
     // have to add after as well (this breaks MS-SQL)
-    if (this.client !== SqlClients.MS_SQL) {
+    if (this.client !== SqlClient.MS_SQL) {
       preQuery = this.addSorting(preQuery, json)
     }
     // handle joins
@@ -565,9 +567,9 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
     // same as delete, manage returning
     if (operation === Operation.CREATE || operation === Operation.UPDATE) {
       let id
-      if (sqlClient === SqlClients.MS_SQL) {
+      if (sqlClient === SqlClient.MS_SQL) {
         id = results?.[0].id
-      } else if (sqlClient === SqlClients.MY_SQL) {
+      } else if (sqlClient === SqlClient.MY_SQL) {
         id = results?.insertId
       }
       row = processFn(
@@ -582,4 +584,3 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
 }
 
 export default SqlQueryBuilder
-module.exports = SqlQueryBuilder

package/src/integrations/base/sqlTable.ts

@@ -1,10 +1,5 @@
 import { Knex, knex } from "knex"
-import { Table } from "../../definitions/common"
-import {
-  Operation,
-  QueryJson,
-  RenameColumn,
-} from "../../definitions/datasource"
+import { Operation, QueryJson, RenameColumn, Table } from "@budibase/types"
 import { breakExternalTableId } from "../utils"
 import SchemaBuilder = Knex.SchemaBuilder
 import CreateTableBuilder = Knex.CreateTableBuilder

package/src/integrations/base/utils.ts

@@ -1,22 +1,12 @@
-import { QueryJson } from "../../definitions/datasource"
-import { Datasource } from "../../definitions/common"
+const QUERY_START_REGEX = /\d[0-9]*:/g
 
-module DatasourceUtils {
-  const { integrations } = require("../index")
-
-  export async function makeExternalQuery(
-    datasource: Datasource,
-    json: QueryJson
-  ) {
-    const Integration = integrations[datasource.source]
-    // query is the opinionated function
-    if (Integration.prototype.query) {
-      const integration = new Integration(datasource.config)
-      return integration.query(json)
-    } else {
-      throw "Datasource does not support query."
-    }
+export function removeKeyNumbering(key: any): string {
+  if (typeof key === "string" && key.match(QUERY_START_REGEX) != null) {
+    const parts = key.split(":")
+    // remove the number
+    parts.shift()
+    return parts.join(":")
+  } else {
+    return key
   }
-
-  module.exports.makeExternalQuery = makeExternalQuery
 }

package/src/integrations/couchdb.ts

@@ -1,9 +1,9 @@
 import {
   Integration,
-  DatasourceFieldTypes,
-  QueryTypes,
-} from "../definitions/datasource"
-import { IntegrationBase } from "./base/IntegrationBase"
+  DatasourceFieldType,
+  QueryType,
+  IntegrationBase,
+} from "@budibase/types"
 
 module CouchDBModule {
   const PouchDB = require("pouchdb")
@@ -21,30 +21,30 @@ module CouchDBModule {
       "Apache CouchDB is an open-source document-oriented NoSQL database, implemented in Erlang.",
     datasource: {
       url: {
-        type: DatasourceFieldTypes.STRING,
+        type: DatasourceFieldType.STRING,
         required: true,
         default: "http://localhost:5984",
       },
       database: {
-        type: DatasourceFieldTypes.STRING,
+        type: DatasourceFieldType.STRING,
         required: true,
       },
     },
     query: {
       create: {
-        type: QueryTypes.JSON,
+        type: QueryType.JSON,
       },
       read: {
-        type: QueryTypes.JSON,
+        type: QueryType.JSON,
       },
       update: {
-        type: QueryTypes.JSON,
+        type: QueryType.JSON,
       },
       delete: {
-        type: QueryTypes.FIELDS,
+        type: QueryType.FIELDS,
         fields: {
           id: {
-            type: DatasourceFieldTypes.STRING,
+            type: DatasourceFieldType.STRING,
             required: true,
           },
         },