@budibase/server 1.0.45 → 1.0.46-alpha.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. package/builder/assets/{index.39e01193.css → index.0df4f7ab.css} +1 -1
  2. package/builder/assets/{index.04ee727d.js → index.6eae2fa0.js} +229 -227
  3. package/builder/assets/{vendor.f9faf6ab.js → vendor.10e2ad1b.js} +12 -12
  4. package/builder/index.html +3 -3
  5. package/coverage/clover.xml +1470 -1285
  6. package/coverage/coverage-final.json +90 -84
  7. package/coverage/lcov-report/api/controllers/analytics.js.html +1 -1
  8. package/coverage/lcov-report/api/controllers/apikeys.js.html +1 -1
  9. package/coverage/lcov-report/api/controllers/application.js.html +90 -90
  10. package/coverage/lcov-report/api/controllers/auth.js.html +1 -1
  11. package/coverage/lcov-report/api/controllers/automation.js.html +21 -21
  12. package/coverage/lcov-report/api/controllers/backup.js.html +1 -1
  13. package/coverage/lcov-report/api/controllers/cloud.js.html +1 -1
  14. package/coverage/lcov-report/api/controllers/component.js.html +1 -1
  15. package/coverage/lcov-report/api/controllers/datasource.js.html +16 -16
  16. package/coverage/lcov-report/api/controllers/dev.js.html +1 -1
  17. package/coverage/lcov-report/api/controllers/hosting.js.html +1 -1
  18. package/coverage/lcov-report/api/controllers/index.html +1 -1
  19. package/coverage/lcov-report/api/controllers/integration.js.html +1 -1
  20. package/coverage/lcov-report/api/controllers/layout.js.html +6 -6
  21. package/coverage/lcov-report/api/controllers/metadata.js.html +1 -1
  22. package/coverage/lcov-report/api/controllers/permission.js.html +15 -15
  23. package/coverage/lcov-report/api/controllers/query/index.html +1 -1
  24. package/coverage/lcov-report/api/controllers/query/index.js.html +19 -19
  25. package/coverage/lcov-report/api/controllers/query/validation.js.html +6 -6
  26. package/coverage/lcov-report/api/controllers/role.js.html +9 -9
  27. package/coverage/lcov-report/api/controllers/routing.js.html +1 -1
  28. package/coverage/lcov-report/api/controllers/row/external.js.html +16 -16
  29. package/coverage/lcov-report/api/controllers/row/index.html +15 -15
  30. package/coverage/lcov-report/api/controllers/row/index.js.html +26 -26
  31. package/coverage/lcov-report/api/controllers/row/internal.js.html +44 -44
  32. package/coverage/lcov-report/api/controllers/row/internalSearch.js.html +8 -8
  33. package/coverage/lcov-report/api/controllers/row/utils.js.html +53 -38
  34. package/coverage/lcov-report/api/controllers/screen.js.html +7 -7
  35. package/coverage/lcov-report/api/controllers/script.js.html +4 -4
  36. package/coverage/lcov-report/api/controllers/table/external.js.html +12 -12
  37. package/coverage/lcov-report/api/controllers/table/index.html +23 -23
  38. package/coverage/lcov-report/api/controllers/table/index.js.html +57 -39
  39. package/coverage/lcov-report/api/controllers/table/internal.js.html +40 -34
  40. package/coverage/lcov-report/api/controllers/table/utils.js.html +86 -62
  41. package/coverage/lcov-report/api/controllers/templates.js.html +1 -1
  42. package/coverage/lcov-report/api/controllers/user.js.html +20 -20
  43. package/coverage/lcov-report/api/controllers/view/exporters.js.html +4 -4
  44. package/coverage/lcov-report/api/controllers/view/index.html +1 -1
  45. package/coverage/lcov-report/api/controllers/view/index.js.html +14 -14
  46. package/coverage/lcov-report/api/controllers/view/utils.js.html +11 -11
  47. package/coverage/lcov-report/api/controllers/view/viewBuilder.js.html +7 -7
  48. package/coverage/lcov-report/api/controllers/webhook.js.html +15 -15
  49. package/coverage/lcov-report/api/index.html +1 -1
  50. package/coverage/lcov-report/api/index.js.html +1 -1
  51. package/coverage/lcov-report/api/routes/analytics.js.html +1 -1
  52. package/coverage/lcov-report/api/routes/apikeys.js.html +1 -1
  53. package/coverage/lcov-report/api/routes/application.js.html +1 -1
  54. package/coverage/lcov-report/api/routes/auth.js.html +1 -1
  55. package/coverage/lcov-report/api/routes/automation.js.html +1 -1
  56. package/coverage/lcov-report/api/routes/backup.js.html +1 -1
  57. package/coverage/lcov-report/api/routes/cloud.js.html +1 -1
  58. package/coverage/lcov-report/api/routes/component.js.html +1 -1
  59. package/coverage/lcov-report/api/routes/datasource.js.html +1 -1
  60. package/coverage/lcov-report/api/routes/deploy.js.html +1 -1
  61. package/coverage/lcov-report/api/routes/dev.js.html +1 -1
  62. package/coverage/lcov-report/api/routes/hosting.js.html +1 -1
  63. package/coverage/lcov-report/api/routes/index.html +1 -1
  64. package/coverage/lcov-report/api/routes/index.js.html +1 -1
  65. package/coverage/lcov-report/api/routes/integration.js.html +1 -1
  66. package/coverage/lcov-report/api/routes/layout.js.html +1 -1
  67. package/coverage/lcov-report/api/routes/metadata.js.html +1 -1
  68. package/coverage/lcov-report/api/routes/permission.js.html +1 -1
  69. package/coverage/lcov-report/api/routes/query.js.html +1 -1
  70. package/coverage/lcov-report/api/routes/role.js.html +1 -1
  71. package/coverage/lcov-report/api/routes/routing.js.html +1 -1
  72. package/coverage/lcov-report/api/routes/row.js.html +1 -1
  73. package/coverage/lcov-report/api/routes/screen.js.html +1 -1
  74. package/coverage/lcov-report/api/routes/script.js.html +1 -1
  75. package/coverage/lcov-report/api/routes/static.js.html +1 -1
  76. package/coverage/lcov-report/api/routes/table.js.html +1 -1
  77. package/coverage/lcov-report/api/routes/templates.js.html +1 -1
  78. package/coverage/lcov-report/api/routes/user.js.html +1 -1
  79. package/coverage/lcov-report/api/routes/view.js.html +1 -1
  80. package/coverage/lcov-report/api/routes/webhook.js.html +1 -1
  81. package/coverage/lcov-report/automations/actions.js.html +24 -24
  82. package/coverage/lcov-report/automations/automationUtils.js.html +5 -5
  83. package/coverage/lcov-report/automations/bullboard.js.html +16 -16
  84. package/coverage/lcov-report/automations/index.html +1 -1
  85. package/coverage/lcov-report/automations/index.js.html +1 -1
  86. package/coverage/lcov-report/automations/steps/bash.js.html +6 -6
  87. package/coverage/lcov-report/automations/steps/createRow.js.html +13 -19
  88. package/coverage/lcov-report/automations/steps/delay.js.html +4 -4
  89. package/coverage/lcov-report/automations/steps/deleteRow.js.html +11 -20
  90. package/coverage/lcov-report/automations/steps/discord.js.html +7 -7
  91. package/coverage/lcov-report/automations/steps/executeQuery.js.html +6 -6
  92. package/coverage/lcov-report/automations/steps/executeScript.js.html +6 -6
  93. package/coverage/lcov-report/automations/steps/filter.js.html +7 -7
  94. package/coverage/lcov-report/automations/steps/index.html +13 -13
  95. package/coverage/lcov-report/automations/steps/integromat.js.html +5 -5
  96. package/coverage/lcov-report/automations/steps/outgoingWebhook.js.html +8 -8
  97. package/coverage/lcov-report/automations/steps/queryRows.js.html +10 -10
  98. package/coverage/lcov-report/automations/steps/sendSmtpEmail.js.html +5 -5
  99. package/coverage/lcov-report/automations/steps/serverLog.js.html +3 -3
  100. package/coverage/lcov-report/automations/steps/slack.js.html +5 -5
  101. package/coverage/lcov-report/automations/steps/updateRow.js.html +6 -6
  102. package/coverage/lcov-report/automations/steps/utils.js.html +3 -3
  103. package/coverage/lcov-report/automations/steps/zapier.js.html +5 -5
  104. package/coverage/lcov-report/automations/triggerInfo/app.js.html +2 -2
  105. package/coverage/lcov-report/automations/triggerInfo/cron.js.html +2 -2
  106. package/coverage/lcov-report/automations/triggerInfo/index.html +1 -1
  107. package/coverage/lcov-report/automations/triggerInfo/index.js.html +8 -8
  108. package/coverage/lcov-report/automations/triggerInfo/rowDeleted.js.html +2 -2
  109. package/coverage/lcov-report/automations/triggerInfo/rowSaved.js.html +2 -2
  110. package/coverage/lcov-report/automations/triggerInfo/rowUpdated.js.html +2 -2
  111. package/coverage/lcov-report/automations/triggerInfo/webhook.js.html +2 -2
  112. package/coverage/lcov-report/automations/triggers.js.html +18 -18
  113. package/coverage/lcov-report/automations/utils.js.html +32 -32
  114. package/coverage/lcov-report/constants/index.html +1 -1
  115. package/coverage/lcov-report/constants/index.js.html +22 -22
  116. package/coverage/lcov-report/constants/layouts.js.html +5 -5
  117. package/coverage/lcov-report/constants/screens.js.html +4 -4
  118. package/coverage/lcov-report/db/client.js.html +18 -18
  119. package/coverage/lcov-report/db/inMemoryView.js.html +7 -7
  120. package/coverage/lcov-report/db/index.html +15 -15
  121. package/coverage/lcov-report/db/index.js.html +3 -3
  122. package/coverage/lcov-report/db/linkedRows/LinkController.js.html +18 -18
  123. package/coverage/lcov-report/db/linkedRows/index.html +1 -1
  124. package/coverage/lcov-report/db/linkedRows/index.js.html +31 -31
  125. package/coverage/lcov-report/db/linkedRows/linkUtils.js.html +15 -15
  126. package/coverage/lcov-report/db/newid.js.html +4 -4
  127. package/coverage/lcov-report/db/utils.js.html +83 -83
  128. package/coverage/lcov-report/events/AutomationEmitter.js.html +1 -1
  129. package/coverage/lcov-report/events/index.html +1 -1
  130. package/coverage/lcov-report/events/index.js.html +5 -5
  131. package/coverage/lcov-report/events/utils.js.html +3 -3
  132. package/coverage/lcov-report/index.html +74 -44
  133. package/coverage/lcov-report/integrations/tests/TestConfiguration.js.html +1 -1
  134. package/coverage/lcov-report/integrations/tests/index.html +1 -1
  135. package/coverage/lcov-report/middleware/appInfo.js.html +1 -1
  136. package/coverage/lcov-report/middleware/authorized.js.html +1 -1
  137. package/coverage/lcov-report/middleware/builder.js.html +1 -1
  138. package/coverage/lcov-report/middleware/currentapp.js.html +33 -6
  139. package/coverage/lcov-report/middleware/index.html +25 -25
  140. package/coverage/lcov-report/middleware/joi-validator.js.html +3 -3
  141. package/coverage/lcov-report/middleware/resourceId.js.html +1 -1
  142. package/coverage/lcov-report/middleware/selfhost.js.html +1 -1
  143. package/coverage/lcov-report/middleware/usageQuota.js.html +275 -41
  144. package/coverage/lcov-report/middleware/utils.js.html +1 -1
  145. package/coverage/lcov-report/migrations/usageQuotas/index.html +141 -0
  146. package/coverage/lcov-report/migrations/usageQuotas/index.js.html +152 -0
  147. package/coverage/lcov-report/migrations/usageQuotas/syncApps.js.html +134 -0
  148. package/coverage/lcov-report/migrations/usageQuotas/syncRows.js.html +143 -0
  149. package/coverage/lcov-report/threads/automation.js.html +1 -1
  150. package/coverage/lcov-report/threads/index.html +1 -1
  151. package/coverage/lcov-report/threads/index.js.html +10 -10
  152. package/coverage/lcov-report/threads/query.js.html +1 -1
  153. package/coverage/lcov-report/threads/utils.js.html +14 -14
  154. package/coverage/lcov-report/utilities/bcrypt.js.html +1 -1
  155. package/coverage/lcov-report/utilities/budibaseDir.js.html +7 -7
  156. package/coverage/lcov-report/utilities/centralPath.js.html +4 -4
  157. package/coverage/lcov-report/utilities/csvParser.js.html +6 -6
  158. package/coverage/lcov-report/utilities/global.js.html +14 -14
  159. package/coverage/lcov-report/utilities/index.html +1 -1
  160. package/coverage/lcov-report/utilities/index.js.html +25 -25
  161. package/coverage/lcov-report/utilities/queue/inMemoryQueue.js.html +8 -8
  162. package/coverage/lcov-report/utilities/queue/index.html +1 -1
  163. package/coverage/lcov-report/utilities/routing/index.html +1 -1
  164. package/coverage/lcov-report/utilities/routing/index.js.html +1 -1
  165. package/coverage/lcov-report/utilities/rowProcessor/index.html +1 -1
  166. package/coverage/lcov-report/utilities/rowProcessor/index.js.html +53 -53
  167. package/coverage/lcov-report/utilities/rowProcessor/utils.js.html +10 -10
  168. package/coverage/lcov-report/utilities/scriptRunner.js.html +5 -5
  169. package/coverage/lcov-report/utilities/security.js.html +8 -8
  170. package/coverage/lcov-report/utilities/statusCodes.js.html +1 -1
  171. package/coverage/lcov-report/utilities/usageQuota/index.html +141 -0
  172. package/coverage/lcov-report/utilities/usageQuota/index.js.html +356 -0
  173. package/coverage/lcov-report/utilities/usageQuota/rows.js.html +302 -0
  174. package/coverage/lcov-report/utilities/usageQuota/usageQuoteReset.js.html +134 -0
  175. package/coverage/lcov-report/utilities/users.js.html +5 -5
  176. package/coverage/lcov-report/utilities/workerRequests.js.html +31 -31
  177. package/coverage/lcov.info +2010 -1698
  178. package/dist/api/controllers/row/utils.js +12 -6
  179. package/dist/api/controllers/table/index.js +10 -4
  180. package/dist/api/controllers/table/internal.js +2 -0
  181. package/dist/api/controllers/table/utils.js +9 -2
  182. package/dist/automations/steps/createRow.js +2 -4
  183. package/dist/automations/steps/deleteRow.js +1 -4
  184. package/dist/environment.js +1 -0
  185. package/dist/integrations/base/sql.js +13 -6
  186. package/dist/integrations/utils.js +53 -8
  187. package/dist/middleware/currentapp.js +9 -0
  188. package/dist/middleware/usageQuota.js +82 -13
  189. package/dist/migrations/usageQuotas/index.js +24 -0
  190. package/dist/migrations/usageQuotas/syncApps.js +26 -0
  191. package/dist/migrations/usageQuotas/syncRows.js +29 -0
  192. package/dist/package.json +4 -4
  193. package/dist/tsconfig.tsbuildinfo +1 -1
  194. package/dist/utilities/{usageQuota.js → usageQuota/index.js} +44 -31
  195. package/dist/utilities/usageQuota/rows.js +77 -0
  196. package/dist/utilities/usageQuota/usageQuoteReset.js +16 -0
  197. package/package.json +5 -5
  198. package/src/api/controllers/row/utils.js +11 -6
  199. package/src/api/controllers/table/index.js +15 -9
  200. package/src/api/controllers/table/internal.js +2 -0
  201. package/src/api/controllers/table/utils.js +9 -1
  202. package/src/automations/steps/createRow.js +2 -4
  203. package/src/automations/steps/deleteRow.js +1 -4
  204. package/src/environment.js +1 -0
  205. package/src/integrations/base/sql.ts +14 -7
  206. package/src/integrations/tests/sql.spec.js +16 -0
  207. package/src/integrations/utils.ts +57 -10
  208. package/src/middleware/currentapp.js +9 -0
  209. package/src/middleware/tests/usageQuota.spec.js +18 -24
  210. package/src/middleware/usageQuota.js +93 -15
  211. package/src/migrations/tests/usageQuotas/index.spec.js +27 -0
  212. package/src/migrations/tests/usageQuotas/syncApps.spec.js +37 -0
  213. package/src/migrations/tests/usageQuotas/syncRows.spec.js +43 -0
  214. package/src/migrations/usageQuotas/index.js +24 -0
  215. package/src/migrations/usageQuotas/syncApps.js +18 -0
  216. package/src/migrations/usageQuotas/syncRows.js +21 -0
  217. package/src/utilities/tests/usageQuota/usageQuota.spec.js +72 -0
  218. package/src/utilities/usageQuota/index.js +92 -0
  219. package/src/utilities/usageQuota/rows.js +74 -0
  220. package/src/utilities/usageQuota/usageQuoteReset.js +18 -0
  221. package/src/utilities/usageQuota.js +0 -73
@@ -8,12 +8,24 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
8
8
  step((generator = generator.apply(thisArg, _arguments || [])).next());
9
9
  });
10
10
  };
11
- const env = require("../environment");
12
- const { getGlobalDB } = require("@budibase/backend-core/tenancy");
11
+ const env = require("../../environment");
12
+ const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy");
13
13
  const { StaticDatabases, generateNewUsageQuotaDoc, } = require("@budibase/backend-core/db");
14
- function getNewQuotaReset() {
15
- return Date.now() + 2592000000;
16
- }
14
+ exports.useQuotas = () => {
15
+ // check if quotas are enabled
16
+ if (env.USE_QUOTAS) {
17
+ // check if there are any tenants without limits
18
+ if (env.EXCLUDE_QUOTAS_TENANTS) {
19
+ const excludedTenants = env.EXCLUDE_QUOTAS_TENANTS.replace(/\s/g, "").split(",");
20
+ const tenantId = getTenantId();
21
+ if (excludedTenants.includes(tenantId)) {
22
+ return false;
23
+ }
24
+ }
25
+ return true;
26
+ }
27
+ return false;
28
+ };
17
29
  exports.Properties = {
18
30
  ROW: "rows",
19
31
  UPLOAD: "storage",
@@ -21,21 +33,21 @@ exports.Properties = {
21
33
  USER: "users",
22
34
  AUTOMATION: "automationRuns",
23
35
  APPS: "apps",
24
- EMAILS: "emails",
36
+ EMAILS: "emails", // doesn't work yet
25
37
  };
26
- function getUsageQuotaDoc(db) {
27
- return __awaiter(this, void 0, void 0, function* () {
28
- let quota;
29
- try {
30
- quota = yield db.get(StaticDatabases.PLATFORM_INFO.docs.usageQuota);
31
- }
32
- catch (err) {
33
- // doc doesn't exist. Create it
34
- quota = yield db.post(generateNewUsageQuotaDoc());
35
- }
36
- return quota;
37
- });
38
- }
38
+ exports.getUsageQuotaDoc = (db) => __awaiter(void 0, void 0, void 0, function* () {
39
+ let quota;
40
+ try {
41
+ quota = yield db.get(StaticDatabases.GLOBAL.docs.usageQuota);
42
+ }
43
+ catch (err) {
44
+ // doc doesn't exist. Create it
45
+ quota = generateNewUsageQuotaDoc();
46
+ const response = yield db.put(quota);
47
+ quota._rev = response.rev;
48
+ }
49
+ return quota;
50
+ });
39
51
  /**
40
52
  * Given a specified tenantId this will add to the usage object for the specified property.
41
53
  * @param {string} property The property which is to be added to (within the nested usageQuota object).
@@ -43,27 +55,28 @@ function getUsageQuotaDoc(db) {
43
55
  * @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
44
56
  * also been reset after this call.
45
57
  */
46
- exports.update = (property, usage) => __awaiter(void 0, void 0, void 0, function* () {
47
- if (!env.USE_QUOTAS) {
58
+ exports.update = (property, usage, opts = { dryRun: false }) => __awaiter(void 0, void 0, void 0, function* () {
59
+ if (!exports.useQuotas()) {
48
60
  return;
49
61
  }
50
62
  try {
51
63
  const db = getGlobalDB();
52
- const quota = yield getUsageQuotaDoc(db);
53
- // Check if the quota needs reset
54
- if (Date.now() >= quota.quotaReset) {
55
- quota.quotaReset = getNewQuotaReset();
56
- for (let prop of Object.keys(quota.usageQuota)) {
57
- quota.usageQuota[prop] = 0;
58
- }
59
- }
64
+ const quota = yield exports.getUsageQuotaDoc(db);
60
65
  // increment the quota
61
66
  quota.usageQuota[property] += usage;
62
- if (quota.usageQuota[property] > quota.usageLimits[property]) {
67
+ if (quota.usageQuota[property] > quota.usageLimits[property] &&
68
+ usage > 0 // allow for decrementing usage when the quota is already exceeded
69
+ ) {
63
70
  throw new Error(`You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`);
64
71
  }
72
+ if (quota.usageQuota[property] < 0) {
73
+ // never go negative if the quota has previously been exceeded
74
+ quota.usageQuota[property] = 0;
75
+ }
65
76
  // update the usage quotas
66
- yield db.put(quota);
77
+ if (!opts.dryRun) {
78
+ yield db.put(quota);
79
+ }
67
80
  }
68
81
  catch (err) {
69
82
  console.error(`Error updating usage quotas for ${property}`, err);
@@ -0,0 +1,77 @@
1
+ "use strict";
2
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
+ return new (P || (P = Promise))(function (resolve, reject) {
5
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
9
+ });
10
+ };
11
+ const { getRowParams, USER_METDATA_PREFIX } = require("../../db/utils");
12
+ const CouchDB = require("../../db");
13
+ const { isDevAppID, getDevelopmentAppID } = require("@budibase/backend-core/db");
14
+ const ROW_EXCLUSIONS = [USER_METDATA_PREFIX];
15
+ const getAppPairs = appIds => {
16
+ // collect the app ids into dev / prod pairs
17
+ // keyed by the dev app id
18
+ const pairs = {};
19
+ for (let appId of appIds) {
20
+ const devId = getDevelopmentAppID(appId);
21
+ if (!pairs[devId]) {
22
+ pairs[devId] = {};
23
+ }
24
+ if (isDevAppID(appId)) {
25
+ pairs[devId].devId = appId;
26
+ }
27
+ else {
28
+ pairs[devId].prodId = appId;
29
+ }
30
+ }
31
+ return pairs;
32
+ };
33
+ const getAppRows = (appId) => __awaiter(void 0, void 0, void 0, function* () {
34
+ const appDb = new CouchDB(appId);
35
+ const response = yield appDb.allDocs(getRowParams(null, null, {
36
+ include_docs: false,
37
+ }));
38
+ return response.rows
39
+ .map(r => r.id)
40
+ .filter(id => {
41
+ for (let exclusion of ROW_EXCLUSIONS) {
42
+ if (id.startsWith(exclusion)) {
43
+ return false;
44
+ }
45
+ }
46
+ return true;
47
+ });
48
+ });
49
+ /**
50
+ * Return a set of all rows in the given app ids.
51
+ * The returned rows will be unique on a per dev/prod app basis.
52
+ * Rows duplicates may exist across apps due to data import so they are not filtered out.
53
+ */
54
+ exports.getUniqueRows = (appIds) => __awaiter(void 0, void 0, void 0, function* () {
55
+ let uniqueRows = [];
56
+ const pairs = getAppPairs(appIds);
57
+ for (let pair of Object.values(pairs)) {
58
+ let appRows = [];
59
+ for (let appId of [pair.devId, pair.prodId]) {
60
+ if (!appId) {
61
+ continue;
62
+ }
63
+ try {
64
+ appRows.push(yield getAppRows(appId));
65
+ }
66
+ catch (e) {
67
+ console.error(e);
68
+ // don't error out if we can't count the app rows, just continue
69
+ }
70
+ }
71
+ // ensure uniqueness on a per app pair basis
72
+ // this can't be done on all rows because app import results in
73
+ // duplicate row ids across apps
74
+ uniqueRows = uniqueRows.concat(...new Set(appRows));
75
+ }
76
+ return uniqueRows;
77
+ });
@@ -0,0 +1,16 @@
1
+ "use strict";
2
+ // UNUSED CODE
3
+ // Preserved for future use
4
+ /* eslint-disable no-unused-vars */
5
+ function getNewQuotaReset() {
6
+ return Date.now() + 2592000000;
7
+ }
8
+ function resetQuotasIfRequired(quota) {
9
+ // Check if the quota needs reset
10
+ if (Date.now() >= quota.quotaReset) {
11
+ quota.quotaReset = getNewQuotaReset();
12
+ for (let prop of Object.keys(quota.usageQuota)) {
13
+ quota.usageQuota[prop] = 0;
14
+ }
15
+ }
16
+ }
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@budibase/server",
3
3
  "email": "hi@budibase.com",
4
- "version": "1.0.45",
4
+ "version": "1.0.46-alpha.3",
5
5
  "description": "Budibase Web Server",
6
6
  "main": "src/index.ts",
7
7
  "repository": {
@@ -70,9 +70,9 @@
70
70
  "license": "GPL-3.0",
71
71
  "dependencies": {
72
72
  "@apidevtools/swagger-parser": "^10.0.3",
73
- "@budibase/backend-core": "^1.0.45",
74
- "@budibase/client": "^1.0.45",
75
- "@budibase/string-templates": "^1.0.45",
73
+ "@budibase/backend-core": "^1.0.46-alpha.3",
74
+ "@budibase/client": "^1.0.46-alpha.3",
75
+ "@budibase/string-templates": "^1.0.46-alpha.3",
76
76
  "@bull-board/api": "^3.7.0",
77
77
  "@bull-board/koa": "^3.7.0",
78
78
  "@elastic/elasticsearch": "7.10.0",
@@ -165,5 +165,5 @@
165
165
  "optionalDependencies": {
166
166
  "oracledb": "^5.3.0"
167
167
  },
168
- "gitHead": "38122eec327e124055adcbb89c456756c5e822d1"
168
+ "gitHead": "b10f29e52b94b0c2c0c61a7a1ad7669fcd37d473"
169
169
  }
@@ -61,12 +61,17 @@ exports.validate = async ({ appId, tableId, row, table }) => {
61
61
  let res
62
62
 
63
63
  // Validate.js doesn't seem to handle array
64
- if (type === FieldTypes.ARRAY && row[fieldName] && row[fieldName].length) {
65
- row[fieldName].map(val => {
66
- if (!constraints.inclusion.includes(val)) {
67
- errors[fieldName] = "Field not in list"
68
- }
69
- })
64
+ if (type === FieldTypes.ARRAY && row[fieldName]) {
65
+ if (row[fieldName].length) {
66
+ row[fieldName].map(val => {
67
+ if (!constraints.inclusion.includes(val)) {
68
+ errors[fieldName] = "Field not in list"
69
+ }
70
+ })
71
+ } else if (constraints.presence && row[fieldName].length === 0) {
72
+ // non required MultiSelect creates an empty array, which should not throw errors
73
+ errors[fieldName] = [`${fieldName} is required`]
74
+ }
70
75
  } else if (type === FieldTypes.JSON && typeof row[fieldName] === "string") {
71
76
  // this should only happen if there is an error
72
77
  try {
@@ -2,7 +2,7 @@ const CouchDB = require("../../../db")
2
2
  const internal = require("./internal")
3
3
  const external = require("./external")
4
4
  const csvParser = require("../../../utilities/csvParser")
5
- const { isExternalTable } = require("../../../integrations/utils")
5
+ const { isExternalTable, isSQL } = require("../../../integrations/utils")
6
6
  const {
7
7
  getTableParams,
8
8
  getDatasourceParams,
@@ -32,8 +32,8 @@ exports.fetch = async function (ctx) {
32
32
  })
33
33
  )
34
34
 
35
- const internal = internalTables.rows.map(row => ({
36
- ...row.doc,
35
+ const internal = internalTables.rows.map(tableDoc => ({
36
+ ...tableDoc.doc,
37
37
  type: "internal",
38
38
  sourceId: BudibaseInternalDB._id,
39
39
  }))
@@ -44,12 +44,18 @@ exports.fetch = async function (ctx) {
44
44
  })
45
45
  )
46
46
 
47
- const external = externalTables.rows.flatMap(row => {
48
- return Object.values(row.doc.entities || {}).map(entity => ({
49
- ...entity,
50
- type: "external",
51
- sourceId: row.doc._id,
52
- }))
47
+ const external = externalTables.rows.flatMap(tableDoc => {
48
+ let entities = tableDoc.doc.entities
49
+ if (entities) {
50
+ return Object.values(entities).map(entity => ({
51
+ ...entity,
52
+ type: "external",
53
+ sourceId: tableDoc.doc._id,
54
+ sql: isSQL(tableDoc.doc),
55
+ }))
56
+ } else {
57
+ return []
58
+ }
53
59
  })
54
60
 
55
61
  ctx.body = [...internal, ...external]
@@ -8,6 +8,7 @@ const {
8
8
  getTable,
9
9
  handleDataImport,
10
10
  } = require("./utils")
11
+ const usageQuota = require("../../../utilities/usageQuota")
11
12
 
12
13
  exports.save = async function (ctx) {
13
14
  const appId = ctx.appId
@@ -119,6 +120,7 @@ exports.destroy = async function (ctx) {
119
120
  })
120
121
  )
121
122
  await db.bulkDocs(rows.rows.map(row => ({ ...row.doc, _deleted: true })))
123
+ await usageQuota.update(usageQuota.Properties.ROW, -rows.rows.length)
122
124
 
123
125
  // update linked rows
124
126
  await linkRows.updateLinks({
@@ -12,9 +12,11 @@ const { USERS_TABLE_SCHEMA, SwitchableTypes } = require("../../../constants")
12
12
  const {
13
13
  isExternalTable,
14
14
  breakExternalTableId,
15
+ isSQL,
15
16
  } = require("../../../integrations/utils")
16
17
  const { getViews, saveView } = require("../view/utils")
17
18
  const viewTemplate = require("../view/viewBuilder")
19
+ const usageQuota = require("../../../utilities/usageQuota")
18
20
 
19
21
  exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
20
22
  let updatedRows = []
@@ -111,7 +113,11 @@ exports.handleDataImport = async (appId, user, table, dataImport) => {
111
113
  finalData.push(row)
112
114
  }
113
115
 
116
+ await usageQuota.update(usageQuota.Properties.ROW, finalData.length, {
117
+ dryRun: true,
118
+ })
114
119
  await db.bulkDocs(finalData)
120
+ await usageQuota.update(usageQuota.Properties.ROW, finalData.length)
115
121
  let response = await db.put(table)
116
122
  table._rev = response._rev
117
123
  return table
@@ -242,7 +248,9 @@ exports.getTable = async (appId, tableId) => {
242
248
  const db = new CouchDB(appId)
243
249
  if (isExternalTable(tableId)) {
244
250
  let { datasourceId, tableName } = breakExternalTableId(tableId)
245
- return exports.getExternalTable(appId, datasourceId, tableName)
251
+ const datasource = await db.get(datasourceId)
252
+ const table = await exports.getExternalTable(appId, datasourceId, tableName)
253
+ return { ...table, sql: isSQL(datasource) }
246
254
  } else {
247
255
  return db.get(tableId)
248
256
  }
@@ -1,6 +1,5 @@
1
1
  const rowController = require("../../api/controllers/row")
2
2
  const automationUtils = require("../automationUtils")
3
- const env = require("../../environment")
4
3
  const usage = require("../../utilities/usageQuota")
5
4
  const { buildCtx } = require("./utils")
6
5
 
@@ -83,10 +82,9 @@ exports.run = async function ({ inputs, appId, emitter }) {
83
82
  inputs.row.tableId,
84
83
  inputs.row
85
84
  )
86
- if (env.USE_QUOTAS) {
87
- await usage.update(usage.Properties.ROW, 1)
88
- }
85
+ await usage.update(usage.Properties.ROW, 1, { dryRun: true })
89
86
  await rowController.save(ctx)
87
+ await usage.update(usage.Properties.ROW, 1)
90
88
  return {
91
89
  row: inputs.row,
92
90
  response: ctx.body,
@@ -1,5 +1,4 @@
1
1
  const rowController = require("../../api/controllers/row")
2
- const env = require("../../environment")
3
2
  const usage = require("../../utilities/usageQuota")
4
3
  const { buildCtx } = require("./utils")
5
4
  const automationUtils = require("../automationUtils")
@@ -74,9 +73,7 @@ exports.run = async function ({ inputs, appId, emitter }) {
74
73
  })
75
74
 
76
75
  try {
77
- if (env.isProd()) {
78
- await usage.update(usage.Properties.ROW, -1)
79
- }
76
+ await usage.update(usage.Properties.ROW, -1)
80
77
  await rowController.destroy(ctx)
81
78
  return {
82
79
  response: ctx.body,
@@ -38,6 +38,7 @@ module.exports = {
38
38
  MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
39
39
  MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
40
40
  USE_QUOTAS: process.env.USE_QUOTAS,
41
+ EXCLUDE_QUOTAS_TENANTS: process.env.EXCLUDE_QUOTAS_TENANTS,
41
42
  REDIS_URL: process.env.REDIS_URL,
42
43
  REDIS_PASSWORD: process.env.REDIS_PASSWORD,
43
44
  INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
@@ -72,16 +72,22 @@ class InternalBuilder {
72
72
 
73
73
  // right now we only do filters on the specific table being queried
74
74
  addFilters(
75
- tableName: string,
76
75
  query: KnexQuery,
77
- filters: SearchFilters | undefined
76
+ filters: SearchFilters | undefined,
77
+ opts: { relationship?: boolean; tableName?: string }
78
78
  ): KnexQuery {
79
79
  function iterate(
80
80
  structure: { [key: string]: any },
81
81
  fn: (key: string, value: any) => void
82
82
  ) {
83
83
  for (let [key, value] of Object.entries(structure)) {
84
- fn(`${tableName}.${key}`, value)
84
+ const isRelationshipField = key.includes(".")
85
+ if (!opts.relationship && !isRelationshipField) {
86
+ fn(`${opts.tableName}.${key}`, value)
87
+ }
88
+ if (opts.relationship && isRelationshipField) {
89
+ fn(key, value)
90
+ }
85
91
  }
86
92
  }
87
93
  if (!filters) {
@@ -272,7 +278,7 @@ class InternalBuilder {
272
278
  if (foundOffset) {
273
279
  query = query.offset(foundOffset)
274
280
  }
275
- query = this.addFilters(tableName, query, filters)
281
+ query = this.addFilters(query, filters, { tableName })
276
282
  // add sorting to pre-query
277
283
  query = this.addSorting(query, json)
278
284
  // @ts-ignore
@@ -285,20 +291,21 @@ class InternalBuilder {
285
291
  preQuery = this.addSorting(preQuery, json)
286
292
  }
287
293
  // handle joins
288
- return this.addRelationships(
294
+ query = this.addRelationships(
289
295
  knex,
290
296
  preQuery,
291
297
  selectStatement,
292
298
  tableName,
293
299
  relationships
294
300
  )
301
+ return this.addFilters(query, filters, { relationship: true })
295
302
  }
296
303
 
297
304
  update(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
298
305
  const { endpoint, body, filters } = json
299
306
  let query: KnexQuery = knex(endpoint.entityId)
300
307
  const parsedBody = parseBody(body)
301
- query = this.addFilters(endpoint.entityId, query, filters)
308
+ query = this.addFilters(query, filters, { tableName: endpoint.entityId })
302
309
  // mysql can't use returning
303
310
  if (opts.disableReturning) {
304
311
  return query.update(parsedBody)
@@ -310,7 +317,7 @@ class InternalBuilder {
310
317
  delete(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
311
318
  const { endpoint, filters } = json
312
319
  let query: KnexQuery = knex(endpoint.entityId)
313
- query = this.addFilters(endpoint.entityId, query, filters)
320
+ query = this.addFilters(query, filters, { tableName: endpoint.entityId })
314
321
  // mysql can't use returning
315
322
  if (opts.disableReturning) {
316
323
  return query.delete()
@@ -119,6 +119,22 @@ describe("SQL query builder", () => {
119
119
  })
120
120
  })
121
121
 
122
+ it("should allow filtering on a related field", () => {
123
+ const query = sql._query(generateReadJson({
124
+ filters: {
125
+ equal: {
126
+ age: 10,
127
+ "task.name": "task 1",
128
+ },
129
+ },
130
+ }))
131
+ // order of bindings changes because relationship filters occur outside inner query
132
+ expect(query).toEqual({
133
+ bindings: [10, limit, "task 1"],
134
+ sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" = $1 limit $2) as "${TABLE_NAME}" where "task"."name" = $3`
135
+ })
136
+ })
137
+
122
138
  it("should test an create statement", () => {
123
139
  const query = sql._query(generateCreateJson(TABLE_NAME, {
124
140
  name: "Michael",
@@ -1,6 +1,5 @@
1
- import { SqlQuery } from "../definitions/datasource"
1
+ import { SourceNames, SqlQuery } from "../definitions/datasource"
2
2
  import { Datasource, Table } from "../definitions/common"
3
- import { SourceNames } from "../definitions/datasource"
4
3
  import { DocumentTypes, SEPARATOR } from "../db/utils"
5
4
  import { FieldTypes, BuildSchemaErrors, InvalidColumns } from "../constants"
6
5
 
@@ -127,7 +126,12 @@ export function isSQL(datasource: Datasource): boolean {
127
126
  if (!datasource || !datasource.source) {
128
127
  return false
129
128
  }
130
- const SQL = [SourceNames.POSTGRES, SourceNames.SQL_SERVER, SourceNames.MYSQL]
129
+ const SQL = [
130
+ SourceNames.POSTGRES,
131
+ SourceNames.SQL_SERVER,
132
+ SourceNames.MYSQL,
133
+ SourceNames.ORACLE,
134
+ ]
131
135
  return SQL.indexOf(datasource.source) !== -1
132
136
  }
133
137
 
@@ -139,11 +143,46 @@ export function isIsoDateString(str: string) {
139
143
  return d.toISOString() === str
140
144
  }
141
145
 
142
- // add the existing relationships from the entities if they exist, to prevent them from being overridden
146
+ /**
147
+ * This function will determine whether a column is a relationship and whether it
148
+ * is currently valid. The reason for the validity check is that tables can be deleted
149
+ * outside of Budibase control and if this is the case it will break Budibase relationships.
150
+ * The tableIds is a list passed down from the main finalise tables function, which is
151
+ * based on the tables that have just been fetched. This will only really be used on subsequent
152
+ * fetches to the first one - if the user is periodically refreshing Budibase knowledge of tables.
153
+ * @param column The column to check, to see if it is a valid relationship.
154
+ * @param tableIds The IDs of the tables which currently exist.
155
+ */
156
+ function shouldCopyRelationship(column: { type: string, tableId?: string }, tableIds: [string]) {
157
+ return column.type === FieldTypes.LINK && column.tableId && tableIds.includes(column.tableId)
158
+ }
159
+
160
+ /**
161
+ * Similar function to the shouldCopyRelationship function, but instead this looks for options and boolean
162
+ * types. It is possible to switch a string -> options and a number -> boolean (and vice versus) need to make
163
+ * sure that these get copied over when tables are fetched. Also checks whether they are still valid, if a
164
+ * column has changed type in the external database then copying it over may not be possible.
165
+ * @param column The column to check for options or boolean type.
166
+ * @param fetchedColumn The fetched column to check for the type in the external database.
167
+ */
168
+ function shouldCopySpecialColumn(column: { type: string }, fetchedColumn: { type: string } | undefined) {
169
+ return column.type === FieldTypes.OPTIONS ||
170
+ ((!fetchedColumn || fetchedColumn.type === FieldTypes.NUMBER) && column.type === FieldTypes.BOOLEAN)
171
+ }
172
+
173
+ /**
174
+ * Looks for columns which need to be copied over into the new table definitions, like relationships
175
+ * and options types.
176
+ * @param tableName The name of the table which is being checked.
177
+ * @param table The specific table which is being checked.
178
+ * @param entities All the tables that existed before - the old table definitions.
179
+ * @param tableIds The IDs of the tables which exist now, to check if anything has been removed.
180
+ */
143
181
  function copyExistingPropsOver(
144
182
  tableName: string,
145
183
  table: Table,
146
- entities: { [key: string]: any }
184
+ entities: { [key: string]: any },
185
+ tableIds: [string]
147
186
  ) {
148
187
  if (entities && entities[tableName]) {
149
188
  if (entities[tableName].primaryDisplay) {
@@ -154,11 +193,10 @@ function copyExistingPropsOver(
154
193
  if (!existingTableSchema.hasOwnProperty(key)) {
155
194
  continue
156
195
  }
196
+ const column = existingTableSchema[key]
157
197
  if (
158
- existingTableSchema[key].type === FieldTypes.LINK ||
159
- existingTableSchema[key].type === FieldTypes.OPTIONS ||
160
- ((!table.schema[key] || table.schema[key].type === FieldTypes.NUMBER) &&
161
- existingTableSchema[key].type === FieldTypes.BOOLEAN)
198
+ shouldCopyRelationship(column, tableIds) ||
199
+ shouldCopySpecialColumn(column, table.schema[key])
162
200
  ) {
163
201
  table.schema[key] = existingTableSchema[key]
164
202
  }
@@ -167,6 +205,13 @@ function copyExistingPropsOver(
167
205
  return table
168
206
  }
169
207
 
208
+ /**
209
+ * Look through the final table definitions to see if anything needs to be
210
+ * copied over from the old and if any errors have occurred mark them so
211
+ * that the user can be made aware.
212
+ * @param tables The list of tables that have been retrieved from the external database.
213
+ * @param entities The old list of tables, if there was any to look for definitions in.
214
+ */
170
215
  export function finaliseExternalTables(
171
216
  tables: { [key: string]: any },
172
217
  entities: { [key: string]: any }
@@ -174,6 +219,8 @@ export function finaliseExternalTables(
174
219
  const invalidColumns = Object.values(InvalidColumns)
175
220
  let finalTables: { [key: string]: any } = {}
176
221
  const errors: { [key: string]: string } = {}
222
+ // @ts-ignore
223
+ const tableIds: [string] = Object.values(tables).map(table => table._id)
177
224
  for (let [name, table] of Object.entries(tables)) {
178
225
  const schemaFields = Object.keys(table.schema)
179
226
  // make sure every table has a key
@@ -185,7 +232,7 @@ export function finaliseExternalTables(
185
232
  continue
186
233
  }
187
234
  // make sure all previous props have been added back
188
- finalTables[name] = copyExistingPropsOver(name, table, entities)
235
+ finalTables[name] = copyExistingPropsOver(name, table, entities, tableIds)
189
236
  }
190
237
  // sort the tables by name
191
238
  finalTables = Object.entries(finalTables)
@@ -47,6 +47,15 @@ module.exports = async (ctx, next) => {
47
47
  (!ctx.user || !ctx.user.builder || !ctx.user.builder.global)
48
48
  ) {
49
49
  clearCookie(ctx, Cookies.CurrentApp)
50
+ // have to set the return url on the server side as client side is not available
51
+ setCookie(ctx, ctx.url, Cookies.RETURN_URL, {
52
+ // don't sign so the browser can easily read
53
+ sign: false,
54
+ // use the request domain to match how ui handles the return url cookie.
55
+ // it's important we don't use the shared domain here as the builder
56
+ // can't delete from it without awareness of the domain.
57
+ requestDomain: true,
58
+ })
50
59
  return ctx.redirect("/")
51
60
  }
52
61