@budibase/server 0.9.184 → 0.9.185-alpha.11

package/dist/integrations/base/sqlTable.js

@@ -0,0 +1,143 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const knex_1 = require("knex");
+const datasource_1 = require("../../definitions/datasource");
+const utils_1 = require("../utils");
+const { FieldTypes, RelationshipTypes } = require("../../constants");
+function generateSchema(schema, table, tables, oldTable = null) {
+    let primaryKey = table && table.primary ? table.primary[0] : null;
+    const columns = Object.values(table.schema);
+    // all columns in a junction table will be meta
+    let metaCols = columns.filter(col => col.meta);
+    let isJunction = metaCols.length === columns.length;
+    // can't change primary once its set for now
+    if (primaryKey && !oldTable && !isJunction) {
+        schema.increments(primaryKey).primary();
+    }
+    else if (!oldTable && isJunction) {
+        schema.primary(metaCols.map(col => col.name));
+    }
+    // check if any columns need added
+    const foreignKeys = Object.values(table.schema).map(col => col.foreignKey);
+    for (let [key, column] of Object.entries(table.schema)) {
+        // skip things that are already correct
+        const oldColumn = oldTable ? oldTable.schema[key] : null;
+        if ((oldColumn && oldColumn.type) || (primaryKey === key && !isJunction)) {
+            continue;
+        }
+        switch (column.type) {
+            case FieldTypes.STRING:
+            case FieldTypes.OPTIONS:
+            case FieldTypes.LONGFORM:
+                schema.string(key);
+                break;
+            case FieldTypes.NUMBER:
+                // if meta is specified then this is a junction table entry
+                if (column.meta && column.meta.toKey && column.meta.toTable) {
+                    const { toKey, toTable } = column.meta;
+                    schema.integer(key).unsigned();
+                    schema.foreign(key).references(`${toTable}.${toKey}`);
+                }
+                else if (foreignKeys.indexOf(key) === -1) {
+                    schema.float(key);
+                }
+                break;
+            case FieldTypes.BOOLEAN:
+                schema.boolean(key);
+                break;
+            case FieldTypes.DATETIME:
+                schema.datetime(key);
+                break;
+            case FieldTypes.ARRAY:
+                schema.json(key);
+                break;
+            case FieldTypes.LINK:
+                // this side of the relationship doesn't need any SQL work
+                if (column.relationshipType !== RelationshipTypes.MANY_TO_ONE &&
+                    column.relationshipType !== RelationshipTypes.MANY_TO_MANY) {
+                    if (!column.foreignKey || !column.tableId) {
+                        throw "Invalid relationship schema";
+                    }
+                    const { tableName } = utils_1.breakExternalTableId(column.tableId);
+                    // @ts-ignore
+                    const relatedTable = tables[tableName];
+                    if (!relatedTable) {
+                        throw "Referenced table doesn't exist";
+                    }
+                    schema.integer(column.foreignKey).unsigned();
+                    schema
+                        .foreign(column.foreignKey)
+                        .references(`${tableName}.${relatedTable.primary[0]}`);
+                }
+                break;
+        }
+    }
+    // need to check if any columns have been deleted
+    if (oldTable) {
+        const deletedColumns = Object.entries(oldTable.schema)
+            .filter(([key, schema]) => schema.type !== FieldTypes.LINK && table.schema[key] == null)
+            .map(([key]) => key);
+        deletedColumns.forEach(key => {
+            if (oldTable.constrained && oldTable.constrained.indexOf(key) !== -1) {
+                schema.dropForeign(key);
+            }
+            schema.dropColumn(key);
+        });
+    }
+    return schema;
+}
+function buildCreateTable(knex, table, tables) {
+    return knex.schema.createTable(table.name, schema => {
+        generateSchema(schema, table, tables);
+    });
+}
+function buildUpdateTable(knex, table, tables, oldTable) {
+    return knex.schema.alterTable(table.name, schema => {
+        generateSchema(schema, table, tables, oldTable);
+    });
+}
+function buildDeleteTable(knex, table) {
+    return knex.schema.dropTable(table.name);
+}
+class SqlTableQueryBuilder {
+    // pass through client to get flavour of SQL
+    constructor(client) {
+        this.sqlClient = client;
+    }
+    getSqlClient() {
+        return this.sqlClient;
+    }
+    /**
+     * @param json the input JSON structure from which an SQL query will be built.
+     * @return {string} the operation that was found in the JSON.
+     */
+    _operation(json) {
+        return json.endpoint.operation;
+    }
+    _tableQuery(json) {
+        const client = knex_1.knex({ client: this.sqlClient });
+        let query;
+        if (!json.table || !json.meta || !json.meta.tables) {
+            throw "Cannot execute without table being specified";
+        }
+        switch (this._operation(json)) {
+            case datasource_1.Operation.CREATE_TABLE:
+                query = buildCreateTable(client, json.table, json.meta.tables);
+                break;
+            case datasource_1.Operation.UPDATE_TABLE:
+                if (!json.meta || !json.meta.table) {
+                    throw "Must specify old table for update";
+                }
+                query = buildUpdateTable(client, json.table, json.meta.tables, json.meta.table);
+                break;
+            case datasource_1.Operation.DELETE_TABLE:
+                query = buildDeleteTable(client, json.table);
+                break;
+            default:
+                throw "Table operation is of unknown type";
+        }
+        return query.toSQL();
+    }
+}
+exports.default = SqlTableQueryBuilder;
+module.exports = SqlTableQueryBuilder;

package/dist/integrations/base/utils.js

@@ -0,0 +1,30 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+var DatasourceUtils;
+(function (DatasourceUtils) {
+    const { integrations } = require("../index");
+    function makeExternalQuery(datasource, json) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const Integration = integrations[datasource.source];
+            // query is the opinionated function
+            if (Integration.prototype.query) {
+                const integration = new Integration(datasource.config);
+                return integration.query(json);
+            }
+            else {
+                throw "Datasource does not support query.";
+            }
+        });
+    }
+    DatasourceUtils.makeExternalQuery = makeExternalQuery;
+    module.exports.makeExternalQuery = makeExternalQuery;
+})(DatasourceUtils || (DatasourceUtils = {}));

package/dist/integrations/microsoftSqlServer.js

@@ -17,6 +17,7 @@ var MSSQLModule;
     const Sql = require("./base/sql");
     const SCHEMA = {
         docs: "https://github.com/tediousjs/node-mssql",
+        plus: true,
         description: "Microsoft SQL Server is a relational database management system developed by Microsoft. ",
         friendlyName: "MS SQL Server",
         datasource: {
@@ -62,10 +63,22 @@ var MSSQLModule;
             },
         },
     };
-    function internalQuery(client, query) {
+    function internalQuery(client, query, operation = undefined) {
         return __awaiter(this, void 0, void 0, function* () {
+            const request = client.request();
             try {
-                return yield client.query(query.sql, query.bindings || {});
+                if (Array.isArray(query.bindings)) {
+                    let count = 0;
+                    for (let binding of query.bindings) {
+                        request.input(`p${count++}`, binding);
+                    }
+                }
+                // this is a hack to get the inserted ID back,
+                //  no way to do this with Knex nicely
+                const sql = operation === datasource_1.Operation.CREATE
+                    ? `${query.sql}; SELECT SCOPE_IDENTITY() AS id;`
+                    : query.sql;
+                return yield request.query(sql);
             }
             catch (err) {
                 // @ts-ignore
@@ -75,21 +88,56 @@ var MSSQLModule;
     }
     class SqlServerIntegration extends Sql {
         constructor(config) {
-            super("mssql");
+            super(utils_1.SqlClients.MS_SQL);
+            this.tables = {};
+            this.schemaErrors = {};
+            this.MASTER_TABLES = [
+                "spt_fallback_db",
+                "spt_fallback_dev",
+                "spt_fallback_usg",
+                "spt_monitor",
+                "MSreplication_options",
+            ];
+            this.TABLES_SQL = "SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE='BASE TABLE'";
             this.config = config;
             const clientCfg = Object.assign(Object.assign({}, this.config), { options: {
                     encrypt: this.config.encrypt,
+                    enableArithAbort: true,
                 } });
             delete clientCfg.encrypt;
             if (!this.pool) {
                 this.pool = new sqlServer.ConnectionPool(clientCfg);
             }
         }
+        getDefinitionSQL(tableName) {
+            return `select *
+              from INFORMATION_SCHEMA.COLUMNS
+              where TABLE_NAME='${tableName}'`;
+        }
+        getConstraintsSQL(tableName) {
+            return `SELECT * FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS AS TC 
+              INNER JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE AS KU
+                ON TC.CONSTRAINT_TYPE = 'PRIMARY KEY' 
+                AND TC.CONSTRAINT_NAME = KU.CONSTRAINT_NAME 
+                AND KU.table_name='${tableName}'
+              ORDER BY 
+                KU.TABLE_NAME,
+                KU.ORDINAL_POSITION;`;
+        }
+        getAutoColumnsSQL(tableName) {
+            return `SELECT 
+              COLUMNPROPERTY(OBJECT_ID(TABLE_SCHEMA+'.'+TABLE_NAME),COLUMN_NAME,'IsComputed') 
+                AS IS_COMPUTED,
+              COLUMNPROPERTY(object_id(TABLE_SCHEMA+'.'+TABLE_NAME), COLUMN_NAME, 'IsIdentity')
+                AS IS_IDENTITY,
+              *
+              FROM INFORMATION_SCHEMA.COLUMNS
+              WHERE TABLE_NAME='${tableName}'`;
+        }
         connect() {
             return __awaiter(this, void 0, void 0, function* () {
                 try {
-                    const client = yield this.pool.connect();
-                    this.client = client.request();
+                    this.client = yield this.pool.connect();
                 }
                 catch (err) {
                     // @ts-ignore
@@ -97,6 +145,65 @@ var MSSQLModule;
                 }
             });
         }
+        runSQL(sql) {
+            return __awaiter(this, void 0, void 0, function* () {
+                return (yield internalQuery(this.client, utils_1.getSqlQuery(sql))).recordset;
+            });
+        }
+        /**
+         * Fetches the tables from the sql server database and assigns them to the datasource.
+         * @param {*} datasourceId - datasourceId to fetch
+         * @param entities - the tables that are to be built
+         */
+        buildSchema(datasourceId, entities) {
+            return __awaiter(this, void 0, void 0, function* () {
+                yield this.connect();
+                let tableNames = yield this.runSQL(this.TABLES_SQL);
+                if (tableNames == null || !Array.isArray(tableNames)) {
+                    throw "Unable to get list of tables in database";
+                }
+                tableNames = tableNames
+                    .map((record) => record.TABLE_NAME)
+                    .filter((name) => this.MASTER_TABLES.indexOf(name) === -1);
+                const tables = {};
+                for (let tableName of tableNames) {
+                    // get the column definition (type)
+                    const definition = yield this.runSQL(this.getDefinitionSQL(tableName));
+                    // find primary key constraints
+                    const constraints = yield this.runSQL(this.getConstraintsSQL(tableName));
+                    // find the computed and identity columns (auto columns)
+                    const columns = yield this.runSQL(this.getAutoColumnsSQL(tableName));
+                    const primaryKeys = constraints
+                        .filter((constraint) => constraint.CONSTRAINT_TYPE === "PRIMARY KEY")
+                        .map((constraint) => constraint.COLUMN_NAME);
+                    const autoColumns = columns
+                        .filter((col) => col.IS_COMPUTED || col.IS_IDENTITY)
+                        .map((col) => col.COLUMN_NAME);
+                    let schema = {};
+                    for (let def of definition) {
+                        const name = def.COLUMN_NAME;
+                        if (typeof name !== "string") {
+                            continue;
+                        }
+                        const type = utils_1.convertSqlType(def.DATA_TYPE);
+                        schema[name] = {
+                            autocolumn: !!autoColumns.find((col) => col === name),
+                            name: name,
+                            type,
+                        };
+                    }
+                    tables[tableName] = {
+                        _id: utils_1.buildExternalTableId(datasourceId, tableName),
+                        primary: primaryKeys,
+                        name: tableName,
+                        schema,
+                    };
+                }
+                const final = utils_1.finaliseExternalTables(tables, entities);
+                this.tables = final.tables;
+                this.schemaErrors = final.errors;
+            });
+        }
         read(query) {
             return __awaiter(this, void 0, void 0, function* () {
                 yield this.connect();
@@ -127,10 +234,11 @@ var MSSQLModule;
         }
         query(json) {
             return __awaiter(this, void 0, void 0, function* () {
-                const operation = this._operation(json).toLowerCase();
-                const input = this._query(json);
-                const response = yield internalQuery(this.client, input);
-                return response.recordset ? response.recordset : [{ [operation]: true }];
+                yield this.connect();
+                const operation = this._operation(json);
+                const queryFn = (query, op) => internalQuery(this.client, query, op);
+                const processFn = (result) => result.recordset ? result.recordset : [{ [operation]: true }];
+                return this.queryWithReturning(json, queryFn, processFn);
             });
         }
     }

package/dist/integrations/mysql.js

@@ -15,31 +15,6 @@ var MySQLModule;
 (function (MySQLModule) {
     const mysql = require("mysql2");
     const Sql = require("./base/sql");
-    const { buildExternalTableId, convertType, copyExistingPropsOver, } = require("./utils");
-    const { FieldTypes } = require("../constants");
-    const TYPE_MAP = {
-        text: FieldTypes.LONGFORM,
-        blob: FieldTypes.LONGFORM,
-        enum: FieldTypes.STRING,
-        varchar: FieldTypes.STRING,
-        float: FieldTypes.NUMBER,
-        int: FieldTypes.NUMBER,
-        numeric: FieldTypes.NUMBER,
-        bigint: FieldTypes.NUMBER,
-        mediumint: FieldTypes.NUMBER,
-        decimal: FieldTypes.NUMBER,
-        dec: FieldTypes.NUMBER,
-        double: FieldTypes.NUMBER,
-        real: FieldTypes.NUMBER,
-        fixed: FieldTypes.NUMBER,
-        smallint: FieldTypes.NUMBER,
-        timestamp: FieldTypes.DATETIME,
-        date: FieldTypes.DATETIME,
-        datetime: FieldTypes.DATETIME,
-        time: FieldTypes.DATETIME,
-        tinyint: FieldTypes.BOOLEAN,
-        json: datasource_1.DatasourceFieldTypes.JSON,
-    };
     const SCHEMA = {
         docs: "https://github.com/mysqljs/mysql",
         plus: true,
@@ -111,7 +86,9 @@ var MySQLModule;
     }
     class MySQLIntegration extends Sql {
         constructor(config) {
-            super("mysql");
+            super(utils_1.SqlClients.MY_SQL);
+            this.tables = {};
+            this.schemaErrors = {};
             this.config = config;
             if (config.ssl && Object.keys(config.ssl).length === 0) {
                 delete config.ssl;
@@ -144,26 +121,23 @@ var MySQLModule;
                         schema[columnName] = {
                             name: columnName,
                             autocolumn: isAuto,
-                            type: convertType(column.Type, TYPE_MAP),
+                            type: utils_1.convertSqlType(column.Type),
                             constraints,
                         };
                     }
-                    // for now just default to first column
-                    if (primaryKeys.length === 0) {
-                        primaryKeys.push(descResp[0].Field);
-                    }
                     if (!tables[tableName]) {
                         tables[tableName] = {
-                            _id: buildExternalTableId(datasourceId, tableName),
+                            _id: utils_1.buildExternalTableId(datasourceId, tableName),
                             primary: primaryKeys,
                             name: tableName,
                             schema,
                         };
                     }
-                    copyExistingPropsOver(tableName, tables, entities);
                 }
                 this.client.end();
-                this.tables = tables;
+                const final = utils_1.finaliseExternalTables(tables, entities);
+                this.tables = final.tables;
+                this.schemaErrors = final.errors;
             });
         }
         create(query) {
@@ -173,7 +147,9 @@ var MySQLModule;
             });
         }
         read(query) {
-            return internalQuery(this.client, utils_1.getSqlQuery(query));
+            return __awaiter(this, void 0, void 0, function* () {
+                return internalQuery(this.client, utils_1.getSqlQuery(query));
+            });
         }
         update(query) {
             return __awaiter(this, void 0, void 0, function* () {
@@ -187,59 +163,13 @@ var MySQLModule;
                 return results.length ? results : [{ deleted: true }];
             });
         }
-        getReturningRow(json) {
-            return __awaiter(this, void 0, void 0, function* () {
-                if (!json.extra || !json.extra.idFilter) {
-                    return {};
-                }
-                const input = this._query({
-                    endpoint: Object.assign(Object.assign({}, json.endpoint), { operation: datasource_1.Operation.READ }),
-                    fields: [],
-                    filters: json.extra.idFilter,
-                    paginate: {
-                        limit: 1,
-                    },
-                });
-                return internalQuery(this.client, input, false);
-            });
-        }
-        // when creating if an ID has been inserted need to make sure
-        // the id filter is enriched with it before trying to retrieve the row
-        checkLookupKeys(results, json) {
-            var _a, _b;
-            if (!(results === null || results === void 0 ? void 0 : results.insertId) || !((_a = json.meta) === null || _a === void 0 ? void 0 : _a.table) || !json.meta.table.primary) {
-                return json;
-            }
-            const primaryKey = (_b = json.meta.table.primary) === null || _b === void 0 ? void 0 : _b[0];
-            json.extra = {
-                idFilter: {
-                    equal: {
-                        [primaryKey]: results.insertId,
-                    },
-                },
-            };
-            return json;
-        }
         query(json) {
             return __awaiter(this, void 0, void 0, function* () {
-                const operation = this._operation(json);
                 this.client.connect();
-                const input = this._query(json, { disableReturning: true });
-                let row;
-                // need to manage returning, a feature mySQL can't do
-                if (operation === operation.DELETE) {
-                    row = this.getReturningRow(json);
-                }
-                const results = yield internalQuery(this.client, input, false);
-                // same as delete, manage returning
-                if (operation === datasource_1.Operation.CREATE || operation === datasource_1.Operation.UPDATE) {
-                    row = this.getReturningRow(this.checkLookupKeys(results, json));
-                }
+                const queryFn = (query) => internalQuery(this.client, query, false);
+                const output = yield this.queryWithReturning(json, queryFn);
                 this.client.end();
-                if (operation !== datasource_1.Operation.READ) {
-                    return row;
-                }
-                return results.length ? results : [{ [operation.toLowerCase()]: true }];
+                return output;
             });
         }
     }

package/dist/integrations/postgres.js

@@ -15,8 +15,6 @@ var PostgresModule;
 (function (PostgresModule) {
     const { Pool } = require("pg");
     const Sql = require("./base/sql");
-    const { FieldTypes } = require("../constants");
-    const { buildExternalTableId, convertType, copyExistingPropsOver, } = require("./utils");
     const { escapeDangerousCharacters } = require("../utilities");
     const JSON_REGEX = /'{.*}'::json/s;
     const SCHEMA = {
@@ -50,6 +48,11 @@ var PostgresModule;
                 default: "root",
                 required: true,
             },
+            schema: {
+                type: datasource_1.DatasourceFieldTypes.STRING,
+                default: "public",
+                required: true,
+            },
             ssl: {
                 type: datasource_1.DatasourceFieldTypes.BOOLEAN,
                 default: false,
@@ -81,21 +84,6 @@ var PostgresModule;
             },
         },
     };
-    const TYPE_MAP = {
-        text: FieldTypes.LONGFORM,
-        varchar: FieldTypes.STRING,
-        integer: FieldTypes.NUMBER,
-        bigint: FieldTypes.NUMBER,
-        decimal: FieldTypes.NUMBER,
-        smallint: FieldTypes.NUMBER,
-        real: FieldTypes.NUMBER,
-        "double precision": FieldTypes.NUMBER,
-        timestamp: FieldTypes.DATETIME,
-        time: FieldTypes.DATETIME,
-        boolean: FieldTypes.BOOLEAN,
-        json: FieldTypes.JSON,
-        date: FieldTypes.DATETIME,
-    };
     function internalQuery(client, query) {
         return __awaiter(this, void 0, void 0, function* () {
             // need to handle a specific issue with json data types in postgres,
@@ -120,8 +108,9 @@ var PostgresModule;
     }
     class PostgresIntegration extends Sql {
         constructor(config) {
-            super("pg");
-            this.COLUMNS_SQL = "select * from information_schema.columns where not table_schema = 'information_schema' and not table_schema = 'pg_catalog'";
+            super(utils_1.SqlClients.POSTGRES);
+            this.tables = {};
+            this.schemaErrors = {};
             this.PRIMARY_KEYS_SQL = `
     select tc.table_schema, tc.table_name, kc.column_name as primary_key 
     from information_schema.table_constraints tc
@@ -142,6 +131,16 @@ var PostgresModule;
                 this.pool = new Pool(newConfig);
             }
             this.client = this.pool;
+            this.setSchema();
+        }
+        setSchema() {
+            if (!this.config.schema) {
+                this.config.schema = "public";
+            }
+            this.client.on("connect", (client) => {
+                client.query(`SET search_path TO ${this.config.schema}`);
+            });
+            this.COLUMNS_SQL = `select * from information_schema.columns where table_schema = '${this.config.schema}'`;
         }
         /**
          * Fetches the tables from the postgres table and assigns them to the datasource.
@@ -176,13 +175,13 @@ var PostgresModule;
                     // table key doesn't exist yet
                     if (!tables[tableName] || !tables[tableName].schema) {
                         tables[tableName] = {
-                            _id: buildExternalTableId(datasourceId, tableName),
-                            primary: tableKeys[tableName] || ["id"],
+                            _id: utils_1.buildExternalTableId(datasourceId, tableName),
+                            primary: tableKeys[tableName] || [],
                             name: tableName,
                             schema: {},
                         };
                     }
-                    const type = convertType(column.data_type, TYPE_MAP);
+                    const type = utils_1.convertSqlType(column.data_type);
                     const identity = !!(column.identity_generation ||
                         column.identity_start ||
                         column.identity_increment);
@@ -196,10 +195,9 @@ var PostgresModule;
                         type,
                     };
                 }
-                for (let tableName of Object.keys(tables)) {
-                    copyExistingPropsOver(tableName, tables, entities);
-                }
-                this.tables = tables;
+                const final = utils_1.finaliseExternalTables(tables, entities);
+                this.tables = final.tables;
+                this.schemaErrors = final.errors;
             });
         }
         create(query) {
@@ -230,8 +228,17 @@ var PostgresModule;
             return __awaiter(this, void 0, void 0, function* () {
                 const operation = this._operation(json).toLowerCase();
                 const input = this._query(json);
-                const response = yield internalQuery(this.client, input);
-                return response.rows.length ? response.rows : [{ [operation]: true }];
+                if (Array.isArray(input)) {
+                    const responses = [];
+                    for (let query of input) {
+                        responses.push(yield internalQuery(this.client, query));
+                    }
+                    return responses;
+                }
+                else {
+                    const response = yield internalQuery(this.client, input);
+                    return response.rows.length ? response.rows : [{ [operation]: true }];
+                }
             });
         }
     }