@budibase/server 2.0.39 → 2.0.40-alpha.1

package/dist/utilities/csvParser.js

@@ -13,6 +13,7 @@ const { FieldTypes } = require("../constants");
 const VALIDATORS = {
     [FieldTypes.STRING]: () => true,
     [FieldTypes.OPTIONS]: () => true,
+    [FieldTypes.BARCODEQR]: () => true,
     [FieldTypes.NUMBER]: attribute => {
         // allow not to be present
         if (!attribute) {

package/dist/utilities/fileSystem/index.js

@@ -12,14 +12,11 @@ const { budibaseTempDir } = require("../budibaseDir");
 const fs = require("fs");
 const { join } = require("path");
 const uuid = require("uuid/v4");
-const { doWithDB, dangerousGetDB, closeDB, } = require("@budibase/backend-core/db");
 const { ObjectStoreBuckets } = require("../../constants");
-const { upload, retrieve, retrieveToTmp, streamUpload, deleteFolder, downloadTarball, downloadTarballDirect, deleteFiles, } = require("./utilities");
+const { upload, retrieve, retrieveToTmp, deleteFolder, downloadTarball, downloadTarballDirect, deleteFiles, } = require("./utilities");
 const { updateClientLibrary } = require("./clientLibrary");
 const { checkSlashesInUrl } = require("../");
 const env = require("../../environment");
-const { USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX, TABLE_ROW_PREFIX, } = require("../../db/utils");
-const MemoryStream = require("memorystream");
 const { getAppId } = require("@budibase/backend-core/context");
 const tar = require("tar");
 const fetch = require("node-fetch");
@@ -75,21 +72,6 @@ exports.checkDevelopmentEnvironment = () => {
         process.exit(-1);
     }
 };
-/**
- * This function manages temporary template files which are stored by Koa.
- * @param {Object} template The template object retrieved from the Koa context object.
- * @returns {Object} Returns an fs read stream which can be loaded into the database.
- */
-exports.getTemplateStream = (template) => __awaiter(void 0, void 0, void 0, function* () {
-    if (template.file) {
-        return fs.createReadStream(template.file.path);
-    }
-    else {
-        const [type, name] = template.key.split("/");
-        const tmpPath = yield exports.downloadTemplate(type, name);
-        return fs.createReadStream(join(tmpPath, name, "db", "dump.txt"));
-    }
-});
 /**
  * Used to retrieve a handlebars file from the system which will be used as a template.
  * This is allowable as the template handlebars files should be static and identical across
@@ -111,83 +93,9 @@ exports.apiFileReturn = contents => {
     fs.writeFileSync(path, contents);
     return fs.createReadStream(path);
 };
-exports.defineFilter = excludeRows => {
-    const ids = [USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX];
-    if (excludeRows) {
-        ids.push(TABLE_ROW_PREFIX);
-    }
-    return doc => !ids.map(key => doc._id.includes(key)).reduce((prev, curr) => prev || curr);
+exports.streamFile = path => {
+    return fs.createReadStream(path);
 };
-/**
- * Local utility to back up the database state for an app, excluding global user
- * data or user relationships.
- * @param {string} appId The app to backup
- * @param {object} config Config to send to export DB
- * @param {boolean} excludeRows Flag to state whether the export should include data.
- * @returns {*} either a string or a stream of the backup
- */
-const backupAppData = (appId, config, excludeRows) => __awaiter(void 0, void 0, void 0, function* () {
-    return yield exports.exportDB(appId, Object.assign(Object.assign({}, config), { filter: exports.defineFilter(excludeRows) }));
-});
-/**
- * Takes a copy of the database state for an app to the object store.
- * @param {string} appId The ID of the app which is to be backed up.
- * @param {string} backupName The name of the backup located in the object store.
- * @return {*} a readable stream to the completed backup file
- */
-exports.performBackup = (appId, backupName) => __awaiter(void 0, void 0, void 0, function* () {
-    return yield backupAppData(appId, { exportName: backupName });
-});
-/**
- * Streams a backup of the database state for an app
- * @param {string} appId The ID of the app which is to be backed up.
- * @param {boolean} excludeRows Flag to state whether the export should include data.
- * @returns {*} a readable stream of the backup which is written in real time
- */
-exports.streamBackup = (appId, excludeRows) => __awaiter(void 0, void 0, void 0, function* () {
-    return yield backupAppData(appId, { stream: true }, excludeRows);
-});
-/**
- * Exports a DB to either file or a variable (memory).
- * @param {string} dbName the DB which is to be exported.
- * @param {string} exportName optional - provide a filename to write the backup to a file
- * @param {boolean} stream optional - whether to perform a full backup
- * @param {function} filter optional - a filter function to clear out any un-wanted docs.
- * @return {*} either a readable stream or a string
- */
-exports.exportDB = (dbName, { stream, filter, exportName } = {}) => __awaiter(void 0, void 0, void 0, function* () {
-    // streaming a DB dump is a bit more complicated, can't close DB
-    if (stream) {
-        const db = dangerousGetDB(dbName);
-        const memStream = new MemoryStream();
-        memStream.on("end", () => __awaiter(void 0, void 0, void 0, function* () {
-            yield closeDB(db);
-        }));
-        db.dump(memStream, { filter });
-        return memStream;
-    }
-    return doWithDB(dbName, (db) => __awaiter(void 0, void 0, void 0, function* () {
-        // Write the dump to file if required
-        if (exportName) {
-            const path = join(budibaseTempDir(), exportName);
-            const writeStream = fs.createWriteStream(path);
-            yield db.dump(writeStream, { filter });
-            // Upload the dump to the object store if self hosted
-            if (env.SELF_HOSTED) {
-                yield streamUpload(ObjectStoreBuckets.BACKUPS, join(dbName, exportName), fs.createReadStream(path));
-            }
-            return fs.createReadStream(path);
-        }
-        // Stringify the dump in memory if required
-        const memStream = new MemoryStream();
-        let appString = "";
-        memStream.on("data", chunk => {
-            appString += chunk.toString();
-        });
-        yield db.dump(memStream, { filter });
-        return appString;
-    }));
-});
 /**
  * Writes the provided contents to a temporary file, which can be used briefly.
  * @param {string} fileContents contents which will be written to a temp file.

package/dist/utilities/fileSystem/utilities.js

@@ -1,5 +1,5 @@
 "use strict";
-const { ObjectStore, makeSureBucketExists, upload, deleteFiles, streamUpload, retrieve, retrieveToTmp, deleteFolder, uploadDirectory, downloadTarball, downloadTarballDirect, } = require("@budibase/backend-core/objectStore");
+const { ObjectStore, makeSureBucketExists, upload, deleteFiles, streamUpload, retrieve, retrieveToTmp, retrieveDirectory, deleteFolder, uploadDirectory, downloadTarball, downloadTarballDirect, } = require("@budibase/backend-core/objectStore");
 /***********************************
  *              NOTE               *
  * This file purely exists so that *
@@ -14,6 +14,7 @@ exports.upload = upload;
 exports.streamUpload = streamUpload;
 exports.retrieve = retrieve;
 exports.retrieveToTmp = retrieveToTmp;
+exports.retrieveDirectory = retrieveDirectory;
 exports.deleteFolder = deleteFolder;
 exports.uploadDirectory = uploadDirectory;
 exports.downloadTarball = downloadTarball;

package/dist/utilities/rowProcessor/index.js

@@ -52,6 +52,11 @@ const TYPE_TRANSFORM_MAP = {
         [null]: "",
         [undefined]: undefined,
     },
+    [FieldTypes.BARCODEQR]: {
+        "": "",
+        [null]: "",
+        [undefined]: undefined,
+    },
     [FieldTypes.FORMULA]: {
         "": "",
         [null]: "",

package/dist/utilities/usageQuota/rows.js

@@ -9,7 +9,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 const { getRowParams, USER_METDATA_PREFIX } = require("../../db/utils");
-const { isDevAppID, getDevelopmentAppID, doWithDB, } = require("@budibase/backend-core/db");
+const { isDevAppID, getDevelopmentAppID, getProdAppID, doWithDB, } = require("@budibase/backend-core/db");
 const ROW_EXCLUSIONS = [USER_METDATA_PREFIX];
 const getAppPairs = appIds => {
     // collect the app ids into dev / prod pairs
@@ -53,7 +53,7 @@ const getAppRows = (appId) => __awaiter(void 0, void 0, void 0, function* () {
  * Rows duplicates may exist across apps due to data import so they are not filtered out.
  */
 exports.getUniqueRows = (appIds) => __awaiter(void 0, void 0, void 0, function* () {
-    let uniqueRows = [];
+    let uniqueRows = [], rowsByApp = {};
     const pairs = getAppPairs(appIds);
     for (let pair of Object.values(pairs)) {
         let appRows = [];
@@ -73,7 +73,9 @@ exports.getUniqueRows = (appIds) => __awaiter(void 0, void 0, void 0, function*
         // this can't be done on all rows because app import results in
         // duplicate row ids across apps
         // the array pre-concat is important to avoid stack overflow
-        uniqueRows = uniqueRows.concat([...new Set(appRows)]);
+        const prodId = getProdAppID(pair.devId || pair.prodId);
+        rowsByApp[prodId] = [...new Set(appRows)];
+        uniqueRows = uniqueRows.concat(rowsByApp[prodId]);
     }
-    return uniqueRows;
+    return { rows: uniqueRows, appRows: rowsByApp };
 });

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.0.39",
+  "version": "2.0.40-alpha.1",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -77,23 +77,23 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "^2.0.39",
-    "@budibase/client": "^2.0.39",
-    "@budibase/pro": "2.0.38",
-    "@budibase/string-templates": "^2.0.39",
-    "@budibase/types": "^2.0.39",
+    "@budibase/backend-core": "2.0.40-alpha.1",
+    "@budibase/client": "2.0.40-alpha.1",
+    "@budibase/pro": "2.0.40-alpha.0",
+    "@budibase/string-templates": "2.0.40-alpha.1",
+    "@budibase/types": "2.0.40-alpha.1",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
     "@google-cloud/firestore": "5.0.2",
-    "@koa/router": "8.0.0",
+    "@koa/router": "8.0.8",
     "@sendgrid/mail": "7.1.1",
     "@sentry/node": "6.17.7",
     "airtable": "0.10.1",
     "arangojs": "7.2.0",
     "aws-sdk": "2.1030.0",
     "bcryptjs": "2.4.3",
-    "bull": "4.8.5",
+    "bull": "4.10.1",
     "chmodr": "1.2.0",
     "chokidar": "3.5.3",
     "csvtojson": "2.0.10",
@@ -112,7 +112,7 @@
     "js-yaml": "4.1.0",
     "jsonschema": "1.4.0",
     "knex": "0.95.15",
-    "koa": "2.7.0",
+    "koa": "2.13.4",
     "koa-body": "4.2.0",
     "koa-compress": "4.0.1",
     "koa-connect": "2.1.0",
@@ -123,7 +123,7 @@
     "koa2-ratelimit": "1.1.1",
     "lodash": "4.17.21",
     "memorystream": "0.3.1",
-    "mongodb": "3.6.3",
+    "mongodb": "4.9",
     "mssql": "6.2.3",
     "mysql2": "2.3.3",
     "node-fetch": "2.6.7",
@@ -146,7 +146,7 @@
     "to-json-schema": "0.2.5",
     "uuid": "3.3.2",
     "validate.js": "0.13.1",
-    "vm2": "3.9.6",
+    "vm2": "3.9.11",
     "worker-farm": "1.7.0",
     "xml2js": "0.4.23",
     "yargs": "13.2.4",
@@ -159,17 +159,17 @@
     "@jest/test-sequencer": "24.9.0",
     "@types/apidoc": "0.50.0",
     "@types/bson": "4.2.0",
-    "@types/bull": "3.15.8",
     "@types/global-agent": "2.1.1",
     "@types/google-spreadsheet": "3.1.5",
+    "@types/ioredis": "4.28.10",
     "@types/jest": "27.5.1",
     "@types/koa": "2.13.4",
-    "@types/koa__router": "8.0.0",
+    "@types/koa__router": "8.0.11",
     "@types/lodash": "4.14.180",
-    "@types/mongodb": "3.6.3",
     "@types/node": "14.18.20",
     "@types/node-fetch": "2.6.1",
     "@types/oracledb": "5.2.2",
+    "@types/pouchdb": "6.4.0",
     "@types/redis": "4.0.11",
     "@typescript-eslint/parser": "5.12.0",
     "apidoc": "0.50.4",
@@ -199,5 +199,5 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "dce0d52b0ad22c3214c348154a7090f872a82ecf"
+  "gitHead": "d7ab0afac13397f1dcc34fe581a270287c17c177"
 }

package/specs/openapi.json

@@ -783,6 +783,7 @@
                       "type": "string",
                       "enum": [
                         "string",
+                        "barcodeqr",
                         "longform",
                         "options",
                         "number",
@@ -986,6 +987,7 @@
                           "type": "string",
                           "enum": [
                             "string",
+                            "barcodeqr",
                             "longform",
                             "options",
                             "number",
@@ -1200,6 +1202,7 @@
                             "type": "string",
                             "enum": [
                               "string",
+                              "barcodeqr",
                               "longform",
                               "options",
                               "number",

package/specs/openapi.yaml

@@ -579,6 +579,7 @@ components:
                     type: string
                     enum:
                       - string
+                      - barcodeqr
                       - longform
                       - options
                       - number
@@ -741,6 +742,7 @@ components:
                         type: string
                         enum:
                           - string
+                          - barcodeqr
                           - longform
                           - options
                           - number
@@ -910,6 +912,7 @@ components:
                           type: string
                           enum:
                             - string
+                            - barcodeqr
                             - longform
                             - options
                             - number

package/src/api/controllers/application.ts

@@ -5,11 +5,7 @@ import {
   createRoutingView,
   createAllSearchIndex,
 } from "../../db/views/staticViews"
-import {
-  getTemplateStream,
-  createApp,
-  deleteApp,
-} from "../../utilities/fileSystem"
+import { createApp, deleteApp } from "../../utilities/fileSystem"
 import {
   generateAppID,
   getLayoutParams,
@@ -50,6 +46,7 @@ import { errors, events, migrations } from "@budibase/backend-core"
 import { App, Layout, Screen, MigrationType } from "@budibase/types"
 import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts"
 import { enrichPluginURLs } from "../../utilities/plugins"
+import sdk from "../../sdk"
 
 const URL_REGEX_SLASH = /\/|\\/g
 
@@ -153,11 +150,7 @@ async function createInstance(template: any) {
       throw "Error loading database dump from memory."
     }
   } else if (template && template.useTemplate === "true") {
-    /* istanbul ignore next */
-    const { ok } = await db.load(await getTemplateStream(template))
-    if (!ok) {
-      throw "Error loading database dump from template."
-    }
+    await sdk.backups.importApp(appId, db, template)
   } else {
     // create the users table
     await db.put(USERS_TABLE_SCHEMA)
@@ -359,7 +352,7 @@ const appPostCreate = async (ctx: any, app: App) => {
   await creationEvents(ctx.request, app)
   // app import & template creation
   if (ctx.request.body.useTemplate === "true") {
-    const rows = await getUniqueRows([app.appId])
+    const { rows } = await getUniqueRows([app.appId])
     const rowCount = rows ? rows.length : 0
     if (rowCount) {
       try {
@@ -493,7 +486,7 @@ const destroyApp = async (ctx: any) => {
 }
 
 const preDestroyApp = async (ctx: any) => {
-  const rows = await getUniqueRows([ctx.params.appId])
+  const { rows } = await getUniqueRows([ctx.params.appId])
   ctx.rowCount = rows.length
 }
 

package/src/api/controllers/{backup.js → backup.ts}

@@ -1,15 +1,15 @@
-const { streamBackup } = require("../../utilities/fileSystem")
-const { events, context } = require("@budibase/backend-core")
-const { DocumentType } = require("../../db/utils")
-const { isQsTrue } = require("../../utilities")
+import sdk from "../../sdk"
+import { events, context } from "@budibase/backend-core"
+import { DocumentType } from "../../db/utils"
+import { isQsTrue } from "../../utilities"
 
-exports.exportAppDump = async function (ctx) {
+export async function exportAppDump(ctx: any) {
   let { appId, excludeRows } = ctx.query
   const appName = decodeURI(ctx.query.appname)
   excludeRows = isQsTrue(excludeRows)
-  const backupIdentifier = `${appName}-export-${new Date().getTime()}.txt`
+  const backupIdentifier = `${appName}-export-${new Date().getTime()}.tar.gz`
   ctx.attachment(backupIdentifier)
-  ctx.body = await streamBackup(appId, excludeRows)
+  ctx.body = await sdk.backups.streamExportApp(appId, excludeRows)
 
   await context.doInAppContext(appId, async () => {
     const appDb = context.getAppDB()

package/src/api/controllers/cloud.js

@@ -1,58 +1,59 @@
 const env = require("../../environment")
 const { getAllApps, getGlobalDBName } = require("@budibase/backend-core/db")
-const {
-  exportDB,
-  sendTempFile,
-  readFileSync,
-} = require("../../utilities/fileSystem")
-const { stringToReadStream } = require("../../utilities")
 const { getGlobalDB } = require("@budibase/backend-core/tenancy")
-const { create } = require("./application")
+const { streamFile } = require("../../utilities/fileSystem")
+const { stringToReadStream } = require("../../utilities")
 const { getDocParams, DocumentType, isDevAppID } = require("../../db/utils")
+const { create } = require("./application")
+const { join } = require("path")
+const sdk = require("../../sdk")
 
-async function createApp(appName, appImport) {
+async function createApp(appName, appDirectory) {
   const ctx = {
     request: {
       body: {
-        templateString: appImport,
+        useTemplate: true,
         name: appName,
       },
+      files: {
+        templateFile: {
+          path: appDirectory,
+        },
+      },
     },
   }
   return create(ctx)
 }
 
+async function getAllDocType(db, docType) {
+  const response = await db.allDocs(
+    getDocParams(docType, null, {
+      include_docs: true,
+    })
+  )
+  return response.rows.map(row => row.doc)
+}
+
 exports.exportApps = async ctx => {
   if (env.SELF_HOSTED || !env.MULTI_TENANCY) {
     ctx.throw(400, "Exporting only allowed in multi-tenant cloud environments.")
   }
   const apps = await getAllApps({ all: true })
-  const globalDBString = await exportDB(getGlobalDBName(), {
+  const globalDBString = await sdk.backups.exportDB(getGlobalDBName(), {
     filter: doc => !doc._id.startsWith(DocumentType.USER),
   })
-  let allDBs = {
-    global: globalDBString,
-  }
-  for (let app of apps) {
-    const appId = app.appId || app._id
-    // only export the dev apps as they will be the latest, the user can republish the apps
-    // in their self hosted environment
-    if (isDevAppID(appId)) {
-      allDBs[app.name] = await exportDB(appId)
-    }
-  }
-  const filename = `cloud-export-${new Date().getTime()}.txt`
-  ctx.attachment(filename)
-  ctx.body = sendTempFile(JSON.stringify(allDBs))
-}
-
-async function getAllDocType(db, docType) {
-  const response = await db.allDocs(
-    getDocParams(docType, null, {
-      include_docs: true,
-    })
+  // only export the dev apps as they will be the latest, the user can republish the apps
+  // in their self-hosted environment
+  let appMetadata = apps
+    .filter(app => isDevAppID(app.appId || app._id))
+    .map(app => ({ appId: app.appId || app._id, name: app.name }))
+  const tmpPath = await sdk.backups.exportMultipleApps(
+    appMetadata,
+    globalDBString
   )
-  return response.rows.map(row => row.doc)
+  const filename = `cloud-export-${new Date().getTime()}.tar.gz`
+  ctx.attachment(filename)
+  ctx.body = streamFile(tmpPath)
 }
 
 async function hasBeenImported() {
@@ -80,17 +81,20 @@ exports.importApps = async ctx => {
       "Import file is required and environment must be fresh to import apps."
     )
   }
-  const importFile = ctx.request.files.importFile
-  const importString = readFileSync(importFile.path)
-  const dbs = JSON.parse(importString)
-  const globalDbImport = dbs.global
-  // remove from the list of apps
-  delete dbs.global
+  if (ctx.request.files.importFile.type !== "application/gzip") {
+    ctx.throw(400, "Import file must be a gzipped tarball.")
+  }
+
+  // initially get all the app databases out of the tarball
+  const tmpPath = sdk.backups.untarFile(ctx.request.files.importFile)
+  const globalDbImport = sdk.backups.getGlobalDBFile(tmpPath)
+  const appNames = sdk.backups.getListOfAppsInMulti(tmpPath)
+
   const globalDb = getGlobalDB()
   // load the global db first
   await globalDb.load(stringToReadStream(globalDbImport))
-  for (let [appName, appImport] of Object.entries(dbs)) {
-    await createApp(appName, appImport)
+  for (let appName of appNames) {
+    await createApp(appName, join(tmpPath, appName))
   }
 
   // if there are any users make sure to remove them

package/src/api/controllers/deploy/index.ts

@@ -1,24 +1,25 @@
 import Deployment from "./Deployment"
 import {
-  Replication,
-  getProdAppID,
   getDevelopmentAppID,
+  getProdAppID,
+  Replication,
 } from "@budibase/backend-core/db"
 import { DocumentType, getAutomationParams } from "../../../db/utils"
 import {
+  clearMetadata,
   disableAllCrons,
   enableCronTrigger,
-  clearMetadata,
 } from "../../../automations/utils"
 import { app as appCache } from "@budibase/backend-core/cache"
 import {
-  getAppId,
   getAppDB,
-  getProdAppDB,
+  getAppId,
   getDevAppDB,
+  getProdAppDB,
 } from "@budibase/backend-core/context"
-import { quotas } from "@budibase/pro"
 import { events } from "@budibase/backend-core"
+import { backups } from "@budibase/pro"
+import { AppBackupTrigger } from "@budibase/types"
 
 // the max time we can wait for an invalidation to complete before considering it failed
 const MAX_PENDING_TIME_MS = 30 * 60000
@@ -99,13 +100,24 @@ async function initDeployedApp(prodAppId: any) {
   console.log("Enabled cron triggers for deployed app..")
 }
 
-async function deployApp(deployment: any) {
+async function deployApp(deployment: any, userId: string) {
   let replication
   try {
     const appId = getAppId()
     const devAppId = getDevelopmentAppID(appId)
     const productionAppId = getProdAppID(appId)
 
+    // don't try this if feature isn't allowed, will error
+    if (await backups.isEnabled()) {
+      // trigger backup initially
+      await backups.triggerAppBackup(
+        productionAppId,
+        AppBackupTrigger.PUBLISH,
+        {
+          createdBy: userId,
+        }
+      )
+    }
     const config: any = {
       source: devAppId,
       target: productionAppId,
@@ -206,7 +218,7 @@ const _deployApp = async function (ctx: any) {
 
   console.log("Deploying app...")
 
-  let app = await deployApp(deployment)
+  let app = await deployApp(deployment, ctx.user._id)
 
   await events.app.published(app)
   ctx.body = deployment

package/src/api/controllers/dev.js

@@ -103,7 +103,7 @@ exports.revert = async ctx => {
     target: appId,
   })
   try {
-    if (!env.isTest()) {
+    if (env.COUCH_DB_URL) {
       // in-memory db stalls on rollback
       await replication.rollback()
     }

package/src/api/controllers/query/index.ts

@@ -56,6 +56,7 @@ const _import = async (ctx: any) => {
       config: {
         url: info.url,
         defaultHeaders: [],
+        rejectUnauthorized: true,
       },
       name: info.name,
     }
@@ -153,7 +154,10 @@ export async function preview(ctx: any) {
           auth: { ...authConfigCtx },
         },
       })
-    const { rows, keys, info, extra } = await quotas.addQuery(runFn)
+
+    const { rows, keys, info, extra } = await quotas.addQuery(runFn, {
+      datasourceId: datasource._id,
+    })
     const schemaFields: any = {}
     if (rows?.length > 0) {
       for (let key of [...new Set(keys)] as string[]) {
@@ -234,7 +238,9 @@ async function execute(
         },
       })
 
-    const { rows, pagination, extra } = await quotas.addQuery(runFn)
+    const { rows, pagination, extra } = await quotas.addQuery(runFn, {
+      datasourceId: datasource._id,
+    })
     // remove the raw from execution incase transformer being used to hide data
     if (extra?.raw) {
       delete extra.raw