@budibase/server 1.3.11 → 1.3.12-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (250) hide show
  1. package/__mocks__/mongodb.ts +1 -1
  2. package/builder/assets/{index.e50bda9f.css → index.7886c551.css} +2 -2
  3. package/builder/assets/index.a0a1e7ff.js +994 -0
  4. package/builder/index.html +2 -4
  5. package/coverage/clover.xml +295 -253
  6. package/coverage/coverage-final.json +17 -17
  7. package/coverage/lcov-report/api/controllers/apikeys.js.html +1 -1
  8. package/coverage/lcov-report/api/controllers/auth.js.html +1 -1
  9. package/coverage/lcov-report/api/controllers/automation.js.html +1 -1
  10. package/coverage/lcov-report/api/controllers/backup.js.html +1 -1
  11. package/coverage/lcov-report/api/controllers/cloud.js.html +1 -1
  12. package/coverage/lcov-report/api/controllers/component.js.html +82 -13
  13. package/coverage/lcov-report/api/controllers/datasource.js.html +20 -20
  14. package/coverage/lcov-report/api/controllers/dev.js.html +1 -1
  15. package/coverage/lcov-report/api/controllers/index.html +30 -30
  16. package/coverage/lcov-report/api/controllers/integration.js.html +6 -6
  17. package/coverage/lcov-report/api/controllers/layout.js.html +1 -1
  18. package/coverage/lcov-report/api/controllers/metadata.js.html +1 -1
  19. package/coverage/lcov-report/api/controllers/migrations.js.html +1 -1
  20. package/coverage/lcov-report/api/controllers/permission.js.html +1 -1
  21. package/coverage/lcov-report/api/controllers/query/index.html +1 -1
  22. package/coverage/lcov-report/api/controllers/query/validation.js.html +1 -1
  23. package/coverage/lcov-report/api/controllers/role.js.html +1 -1
  24. package/coverage/lcov-report/api/controllers/routing.js.html +1 -1
  25. package/coverage/lcov-report/api/controllers/row/external.js.html +1 -1
  26. package/coverage/lcov-report/api/controllers/row/index.html +1 -1
  27. package/coverage/lcov-report/api/controllers/row/internal.js.html +1 -1
  28. package/coverage/lcov-report/api/controllers/row/internalSearch.js.html +1 -1
  29. package/coverage/lcov-report/api/controllers/row/staticFormula.js.html +1 -1
  30. package/coverage/lcov-report/api/controllers/row/utils.js.html +1 -1
  31. package/coverage/lcov-report/api/controllers/screen.js.html +204 -12
  32. package/coverage/lcov-report/api/controllers/script.js.html +1 -1
  33. package/coverage/lcov-report/api/controllers/table/bulkFormula.js.html +1 -1
  34. package/coverage/lcov-report/api/controllers/table/external.js.html +1 -1
  35. package/coverage/lcov-report/api/controllers/table/index.html +1 -1
  36. package/coverage/lcov-report/api/controllers/table/index.js.html +1 -1
  37. package/coverage/lcov-report/api/controllers/templates.js.html +1 -1
  38. package/coverage/lcov-report/api/controllers/user.js.html +1 -1
  39. package/coverage/lcov-report/api/controllers/view/exporters.js.html +1 -1
  40. package/coverage/lcov-report/api/controllers/view/index.html +1 -1
  41. package/coverage/lcov-report/api/controllers/view/index.js.html +1 -1
  42. package/coverage/lcov-report/api/controllers/view/utils.js.html +1 -1
  43. package/coverage/lcov-report/api/controllers/view/viewBuilder.js.html +1 -1
  44. package/coverage/lcov-report/api/controllers/webhook.js.html +1 -1
  45. package/coverage/lcov-report/api/index.html +1 -1
  46. package/coverage/lcov-report/api/index.js.html +12 -12
  47. package/coverage/lcov-report/api/routes/analytics.js.html +1 -1
  48. package/coverage/lcov-report/api/routes/apikeys.js.html +1 -1
  49. package/coverage/lcov-report/api/routes/auth.js.html +1 -1
  50. package/coverage/lcov-report/api/routes/automation.js.html +1 -1
  51. package/coverage/lcov-report/api/routes/backup.js.html +1 -1
  52. package/coverage/lcov-report/api/routes/cloud.js.html +1 -1
  53. package/coverage/lcov-report/api/routes/component.js.html +1 -1
  54. package/coverage/lcov-report/api/routes/datasource.js.html +1 -1
  55. package/coverage/lcov-report/api/routes/deploy.js.html +1 -1
  56. package/coverage/lcov-report/api/routes/dev.js.html +1 -1
  57. package/coverage/lcov-report/api/routes/index.html +1 -1
  58. package/coverage/lcov-report/api/routes/integration.js.html +1 -1
  59. package/coverage/lcov-report/api/routes/layout.js.html +1 -1
  60. package/coverage/lcov-report/api/routes/metadata.js.html +1 -1
  61. package/coverage/lcov-report/api/routes/migrations.js.html +1 -1
  62. package/coverage/lcov-report/api/routes/permission.js.html +1 -1
  63. package/coverage/lcov-report/api/routes/query.js.html +1 -1
  64. package/coverage/lcov-report/api/routes/role.js.html +1 -1
  65. package/coverage/lcov-report/api/routes/routing.js.html +1 -1
  66. package/coverage/lcov-report/api/routes/screen.js.html +1 -1
  67. package/coverage/lcov-report/api/routes/script.js.html +1 -1
  68. package/coverage/lcov-report/api/routes/table.js.html +1 -1
  69. package/coverage/lcov-report/api/routes/templates.js.html +1 -1
  70. package/coverage/lcov-report/api/routes/user.js.html +1 -1
  71. package/coverage/lcov-report/api/routes/utils/index.html +1 -1
  72. package/coverage/lcov-report/api/routes/utils/validators.js.html +1 -1
  73. package/coverage/lcov-report/api/routes/view.js.html +1 -1
  74. package/coverage/lcov-report/api/routes/webhook.js.html +1 -1
  75. package/coverage/lcov-report/automations/actions.js.html +1 -1
  76. package/coverage/lcov-report/automations/automationUtils.js.html +1 -1
  77. package/coverage/lcov-report/automations/bullboard.js.html +1 -1
  78. package/coverage/lcov-report/automations/index.html +1 -1
  79. package/coverage/lcov-report/automations/index.js.html +1 -1
  80. package/coverage/lcov-report/automations/steps/bash.js.html +1 -1
  81. package/coverage/lcov-report/automations/steps/delay.js.html +1 -1
  82. package/coverage/lcov-report/automations/steps/discord.js.html +1 -1
  83. package/coverage/lcov-report/automations/steps/executeQuery.js.html +1 -1
  84. package/coverage/lcov-report/automations/steps/executeScript.js.html +1 -1
  85. package/coverage/lcov-report/automations/steps/filter.js.html +1 -1
  86. package/coverage/lcov-report/automations/steps/index.html +1 -1
  87. package/coverage/lcov-report/automations/steps/integromat.js.html +1 -1
  88. package/coverage/lcov-report/automations/steps/loop.js.html +1 -1
  89. package/coverage/lcov-report/automations/steps/outgoingWebhook.js.html +1 -1
  90. package/coverage/lcov-report/automations/steps/queryRows.js.html +1 -1
  91. package/coverage/lcov-report/automations/steps/sendSmtpEmail.js.html +1 -1
  92. package/coverage/lcov-report/automations/steps/serverLog.js.html +1 -1
  93. package/coverage/lcov-report/automations/steps/slack.js.html +1 -1
  94. package/coverage/lcov-report/automations/steps/updateRow.js.html +1 -1
  95. package/coverage/lcov-report/automations/steps/utils.js.html +1 -1
  96. package/coverage/lcov-report/automations/steps/zapier.js.html +1 -1
  97. package/coverage/lcov-report/automations/triggerInfo/app.js.html +1 -1
  98. package/coverage/lcov-report/automations/triggerInfo/cron.js.html +1 -1
  99. package/coverage/lcov-report/automations/triggerInfo/index.html +1 -1
  100. package/coverage/lcov-report/automations/triggerInfo/index.js.html +1 -1
  101. package/coverage/lcov-report/automations/triggerInfo/rowDeleted.js.html +1 -1
  102. package/coverage/lcov-report/automations/triggerInfo/rowSaved.js.html +1 -1
  103. package/coverage/lcov-report/automations/triggerInfo/rowUpdated.js.html +1 -1
  104. package/coverage/lcov-report/automations/triggerInfo/webhook.js.html +1 -1
  105. package/coverage/lcov-report/automations/triggers.js.html +1 -1
  106. package/coverage/lcov-report/constants/index.html +1 -1
  107. package/coverage/lcov-report/constants/index.js.html +1 -1
  108. package/coverage/lcov-report/constants/layouts.js.html +1 -1
  109. package/coverage/lcov-report/constants/screens.js.html +1 -1
  110. package/coverage/lcov-report/db/inMemoryView.js.html +1 -1
  111. package/coverage/lcov-report/db/index.html +18 -18
  112. package/coverage/lcov-report/db/index.js.html +1 -1
  113. package/coverage/lcov-report/db/linkedRows/LinkController.js.html +1 -1
  114. package/coverage/lcov-report/db/linkedRows/index.html +1 -1
  115. package/coverage/lcov-report/db/linkedRows/index.js.html +1 -1
  116. package/coverage/lcov-report/db/linkedRows/linkUtils.js.html +1 -1
  117. package/coverage/lcov-report/db/newid.js.html +2 -2
  118. package/coverage/lcov-report/db/utils.js.html +56 -20
  119. package/coverage/lcov-report/events/AutomationEmitter.js.html +1 -1
  120. package/coverage/lcov-report/events/index.html +1 -1
  121. package/coverage/lcov-report/events/index.js.html +1 -1
  122. package/coverage/lcov-report/events/utils.js.html +1 -1
  123. package/coverage/lcov-report/index.html +28 -28
  124. package/coverage/lcov-report/integrations/tests/TestConfiguration.js.html +1 -1
  125. package/coverage/lcov-report/integrations/tests/index.html +1 -1
  126. package/coverage/lcov-report/middleware/appInfo.js.html +1 -1
  127. package/coverage/lcov-report/middleware/builder.js.html +12 -12
  128. package/coverage/lcov-report/middleware/currentapp.js.html +33 -33
  129. package/coverage/lcov-report/middleware/index.html +1 -1
  130. package/coverage/lcov-report/middleware/joi-validator.js.html +1 -1
  131. package/coverage/lcov-report/middleware/publicApi.js.html +1 -1
  132. package/coverage/lcov-report/middleware/resourceId.js.html +7 -7
  133. package/coverage/lcov-report/middleware/selfhost.js.html +1 -1
  134. package/coverage/lcov-report/middleware/utils.js.html +2 -2
  135. package/coverage/lcov-report/utilities/budibaseDir.js.html +3 -3
  136. package/coverage/lcov-report/utilities/centralPath.js.html +4 -4
  137. package/coverage/lcov-report/utilities/csvParser.js.html +1 -1
  138. package/coverage/lcov-report/utilities/global.js.html +16 -16
  139. package/coverage/lcov-report/utilities/index.html +1 -1
  140. package/coverage/lcov-report/utilities/index.js.html +23 -23
  141. package/coverage/lcov-report/utilities/queue/inMemoryQueue.js.html +2 -2
  142. package/coverage/lcov-report/utilities/queue/index.html +1 -1
  143. package/coverage/lcov-report/utilities/routing/index.html +1 -1
  144. package/coverage/lcov-report/utilities/routing/index.js.html +1 -1
  145. package/coverage/lcov-report/utilities/rowProcessor/index.html +1 -1
  146. package/coverage/lcov-report/utilities/rowProcessor/index.js.html +1 -1
  147. package/coverage/lcov-report/utilities/rowProcessor/utils.js.html +1 -1
  148. package/coverage/lcov-report/utilities/scriptRunner.js.html +1 -1
  149. package/coverage/lcov-report/utilities/security.js.html +2 -2
  150. package/coverage/lcov-report/utilities/statusCodes.js.html +1 -1
  151. package/coverage/lcov-report/utilities/usageQuota/index.html +1 -1
  152. package/coverage/lcov-report/utilities/usageQuota/rows.js.html +1 -1
  153. package/coverage/lcov-report/utilities/usageQuota/usageQuoteReset.js.html +1 -1
  154. package/coverage/lcov-report/utilities/users.js.html +1 -1
  155. package/coverage/lcov-report/utilities/workerRequests.js.html +1 -1
  156. package/coverage/lcov.info +532 -444
  157. package/dist/api/controllers/application.js +5 -4
  158. package/dist/api/controllers/component.js +18 -1
  159. package/dist/api/controllers/datasource.js +4 -4
  160. package/dist/api/controllers/integration.js +4 -4
  161. package/dist/api/controllers/plugin.js +119 -0
  162. package/dist/api/controllers/screen.js +51 -2
  163. package/dist/api/controllers/static/index.js +1 -0
  164. package/dist/api/controllers/static/templates/BudibaseApp.svelte +10 -0
  165. package/dist/api/controllers/static/templates/preview.hbs +3 -1
  166. package/dist/api/routes/index.js +2 -0
  167. package/dist/api/routes/plugin.js +38 -0
  168. package/dist/app.js +9 -0
  169. package/dist/db/utils.js +10 -1
  170. package/dist/environment.js +1 -0
  171. package/dist/integrations/airtable.js +126 -129
  172. package/dist/integrations/arangodb.js +86 -89
  173. package/dist/integrations/base/query.js +2 -2
  174. package/dist/integrations/couchdb.js +80 -83
  175. package/dist/integrations/dynamodb.js +161 -164
  176. package/dist/integrations/elasticsearch.js +135 -138
  177. package/dist/integrations/firebase.js +154 -157
  178. package/dist/integrations/googlesheets.js +323 -327
  179. package/dist/integrations/index.js +93 -51
  180. package/dist/integrations/microsoftSqlServer.js +240 -243
  181. package/dist/integrations/mongodb.js +239 -242
  182. package/dist/integrations/mysql.js +215 -219
  183. package/dist/integrations/oracle.js +318 -322
  184. package/dist/integrations/postgres.js +262 -265
  185. package/dist/integrations/redis.js +134 -139
  186. package/dist/integrations/rest.js +278 -281
  187. package/dist/integrations/s3.js +61 -64
  188. package/dist/integrations/snowflake.js +83 -86
  189. package/dist/package.json +8 -6
  190. package/dist/threads/query.js +2 -2
  191. package/dist/tsconfig.build.tsbuildinfo +1 -1
  192. package/dist/utilities/fileSystem/index.js +65 -0
  193. package/dist/watch.js +77 -0
  194. package/package.json +9 -7
  195. package/scripts/dev/manage.js +1 -0
  196. package/src/api/controllers/application.ts +1 -1
  197. package/src/api/controllers/component.js +25 -2
  198. package/src/api/controllers/datasource.js +4 -4
  199. package/src/api/controllers/integration.js +4 -4
  200. package/src/api/controllers/plugin.ts +110 -0
  201. package/src/api/controllers/screen.js +65 -1
  202. package/src/api/controllers/static/index.ts +1 -0
  203. package/src/api/controllers/static/templates/BudibaseApp.svelte +10 -0
  204. package/src/api/controllers/static/templates/preview.hbs +3 -1
  205. package/src/api/routes/index.ts +2 -0
  206. package/src/api/routes/plugin.ts +13 -0
  207. package/src/api/routes/tests/query.spec.js +8 -25
  208. package/src/app.ts +12 -0
  209. package/src/db/utils.js +12 -0
  210. package/src/environment.js +1 -0
  211. package/src/integrations/airtable.ts +116 -118
  212. package/src/integrations/arangodb.ts +87 -89
  213. package/src/integrations/base/query.ts +2 -2
  214. package/src/integrations/couchdb.ts +83 -89
  215. package/src/integrations/dynamodb.ts +173 -175
  216. package/src/integrations/elasticsearch.ts +120 -122
  217. package/src/integrations/firebase.ts +157 -159
  218. package/src/integrations/googlesheets.ts +328 -338
  219. package/src/integrations/index.ts +59 -21
  220. package/src/integrations/microsoftSqlServer.ts +249 -251
  221. package/src/integrations/mongodb.ts +261 -258
  222. package/src/integrations/mysql.ts +229 -234
  223. package/src/integrations/oracle.ts +373 -380
  224. package/src/integrations/postgres.ts +264 -268
  225. package/src/integrations/redis.ts +128 -132
  226. package/src/integrations/rest.ts +310 -315
  227. package/src/integrations/s3.ts +68 -71
  228. package/src/integrations/snowflake.ts +80 -82
  229. package/src/integrations/tests/{airtable.spec.js → airtable.spec.ts} +21 -18
  230. package/src/integrations/tests/{arangodb.spec.js → arangodb.spec.ts} +12 -9
  231. package/src/integrations/tests/{couchdb.spec.js → couchdb.spec.ts} +33 -27
  232. package/src/integrations/tests/{dynamodb.spec.js → dynamodb.spec.ts} +45 -41
  233. package/src/integrations/tests/{elasticsearch.spec.js → elasticsearch.spec.ts} +22 -21
  234. package/src/integrations/tests/{firebase.spec.js → firebase.spec.ts} +33 -28
  235. package/src/integrations/tests/{microsoftSqlServer.spec.js → microsoftSqlServer.spec.ts} +12 -11
  236. package/src/integrations/tests/{mongo.spec.js → mongo.spec.ts} +46 -35
  237. package/src/integrations/tests/{mysql.spec.js → mysql.spec.ts} +17 -15
  238. package/src/integrations/tests/{oracle.spec.js → oracle.spec.ts} +19 -17
  239. package/src/integrations/tests/{postgres.spec.js → postgres.spec.ts} +19 -17
  240. package/src/integrations/tests/{redis.spec.js → redis.spec.ts} +27 -18
  241. package/src/integrations/tests/{rest.spec.js → rest.spec.ts} +19 -11
  242. package/src/integrations/tests/{s3.spec.js → s3.spec.ts} +10 -8
  243. package/src/integrations/tests/{sql.spec.js → sql.spec.ts} +228 -177
  244. package/src/module.d.ts +1 -0
  245. package/src/threads/query.ts +2 -2
  246. package/src/utilities/fileSystem/index.js +69 -0
  247. package/src/watch.ts +38 -0
  248. package/builder/assets/index.504401fd.js +0 -875
  249. package/builder/assets/vendor.5e1fe644.css +0 -1
  250. package/builder/assets/vendor.67dbd8fa.js +0 -248
@@ -16,12 +16,16 @@ const { doWithDB, dangerousGetDB, closeDB, } = require("@budibase/backend-core/d
16
16
  const { ObjectStoreBuckets } = require("../../constants");
17
17
  const { upload, retrieve, retrieveToTmp, streamUpload, deleteFolder, downloadTarball, deleteFiles, } = require("./utilities");
18
18
  const { updateClientLibrary } = require("./clientLibrary");
19
+ const { checkSlashesInUrl } = require("../");
19
20
  const env = require("../../environment");
20
21
  const { USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX, TABLE_ROW_PREFIX, } = require("../../db/utils");
21
22
  const MemoryStream = require("memorystream");
22
23
  const { getAppId } = require("@budibase/backend-core/context");
24
+ const tar = require("tar");
25
+ const fetch = require("node-fetch");
23
26
  const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..");
24
27
  const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules");
28
+ const DATASOURCE_PATH = join(budibaseTempDir(), "datasource");
25
29
  /**
26
30
  * The single stack system (Cloud and Builder) should not make use of the file system where possible,
27
31
  * this file handles all of the file access for the system with the intention of limiting it all to one
@@ -87,6 +91,12 @@ exports.getTemplateStream = (template) => __awaiter(void 0, void 0, void 0, func
87
91
  exports.loadHandlebarsFile = path => {
88
92
  return fs.readFileSync(path, "utf8");
89
93
  };
94
+ /**
95
+ * Same as above just with a different name.
96
+ */
97
+ exports.loadJSFile = (directory, name) => {
98
+ return fs.readFileSync(join(directory, name), "utf8");
99
+ };
90
100
  /**
91
101
  * When return a file from the API need to write the file to the system temporarily so we
92
102
  * can create a read stream to send.
@@ -285,6 +295,61 @@ exports.cleanup = appIds => {
285
295
  }
286
296
  }
287
297
  };
298
+ exports.extractPluginTarball = (file) => __awaiter(void 0, void 0, void 0, function* () {
299
+ if (!file.name.endsWith(".tar.gz")) {
300
+ throw new Error("Plugin must be compressed into a gzipped tarball.");
301
+ }
302
+ const path = join(budibaseTempDir(), file.name.split(".tar.gz")[0]);
303
+ // remove old tmp directories automatically - don't combine
304
+ if (fs.existsSync(path)) {
305
+ fs.rmSync(path, { recursive: true, force: true });
306
+ }
307
+ fs.mkdirSync(path);
308
+ yield tar.extract({
309
+ file: file.path,
310
+ C: path,
311
+ });
312
+ let metadata = {};
313
+ try {
314
+ const pkg = fs.readFileSync(join(path, "package.json"), "utf8");
315
+ const schema = fs.readFileSync(join(path, "schema.json"), "utf8");
316
+ metadata.schema = JSON.parse(schema);
317
+ metadata.package = JSON.parse(pkg);
318
+ }
319
+ catch (err) {
320
+ throw new Error("Unable to process schema.json/package.json in plugin.");
321
+ }
322
+ return { metadata, directory: path };
323
+ });
324
+ exports.getDatasourcePlugin = (name, url, hash) => __awaiter(void 0, void 0, void 0, function* () {
325
+ if (!fs.existsSync(DATASOURCE_PATH)) {
326
+ fs.mkdirSync(DATASOURCE_PATH);
327
+ }
328
+ const filename = join(DATASOURCE_PATH, name);
329
+ const metadataName = `${filename}.bbmetadata`;
330
+ if (fs.existsSync(filename)) {
331
+ const currentHash = fs.readFileSync(metadataName, "utf8");
332
+ // if hash is the same return the file, otherwise remove it and re-download
333
+ if (currentHash === hash) {
334
+ return require(filename);
335
+ }
336
+ else {
337
+ console.log(`Updating plugin: ${name}`);
338
+ fs.unlinkSync(filename);
339
+ }
340
+ }
341
+ const fullUrl = checkSlashesInUrl(`${env.MINIO_URL}/${ObjectStoreBuckets.PLUGINS}/${url}`);
342
+ const response = yield fetch(fullUrl);
343
+ if (response.status === 200) {
344
+ const content = yield response.text();
345
+ fs.writeFileSync(filename, content);
346
+ fs.writeFileSync(metadataName, hash);
347
+ require(filename);
348
+ }
349
+ else {
350
+ throw new Error(`Unable to retrieve plugin - reason: ${yield response.text()}`);
351
+ }
352
+ });
288
353
  /**
289
354
  * Full function definition for below can be found in the utilities.
290
355
  */
package/dist/watch.js ADDED
@@ -0,0 +1,77 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || function (mod) {
19
+ if (mod && mod.__esModule) return mod;
20
+ var result = {};
21
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
+ __setModuleDefault(result, mod);
23
+ return result;
24
+ };
25
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
26
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
27
+ return new (P || (P = Promise))(function (resolve, reject) {
28
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
29
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
30
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
31
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
32
+ });
33
+ };
34
+ var __importDefault = (this && this.__importDefault) || function (mod) {
35
+ return (mod && mod.__esModule) ? mod : { "default": mod };
36
+ };
37
+ Object.defineProperty(exports, "__esModule", { value: true });
38
+ exports.watch = void 0;
39
+ const path_1 = __importDefault(require("path"));
40
+ const env = __importStar(require("./environment"));
41
+ const chokidar_1 = __importDefault(require("chokidar"));
42
+ const fs_1 = __importDefault(require("fs"));
43
+ const backend_core_1 = require("@budibase/backend-core");
44
+ const constants_1 = require("@budibase/backend-core/constants");
45
+ const plugin_1 = require("./api/controllers/plugin");
46
+ function watch() {
47
+ const watchPath = path_1.default.join(env.PLUGINS_DIR, "./**/*.tar.gz");
48
+ chokidar_1.default
49
+ .watch(watchPath, {
50
+ ignored: "**/node_modules",
51
+ awaitWriteFinish: {
52
+ pollInterval: 100,
53
+ stabilityThreshold: 250,
54
+ },
55
+ usePolling: true,
56
+ interval: 250,
57
+ })
58
+ .on("all", (event, path) => __awaiter(this, void 0, void 0, function* () {
59
+ // Sanity checks
60
+ if (!(path === null || path === void 0 ? void 0 : path.endsWith(".tar.gz")) || !fs_1.default.existsSync(path)) {
61
+ return;
62
+ }
63
+ yield backend_core_1.tenancy.doInTenant(constants_1.DEFAULT_TENANT_ID, () => __awaiter(this, void 0, void 0, function* () {
64
+ try {
65
+ const split = path.split("/");
66
+ const name = split[split.length - 1];
67
+ console.log("Importing plugin:", path);
68
+ yield (0, plugin_1.processPlugin)({ name, path });
69
+ }
70
+ catch (err) {
71
+ const message = (err === null || err === void 0 ? void 0 : err.message) ? err === null || err === void 0 ? void 0 : err.message : err;
72
+ console.error("Failed to import plugin:", message);
73
+ }
74
+ }));
75
+ }));
76
+ }
77
+ exports.watch = watch;
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@budibase/server",
3
3
  "email": "hi@budibase.com",
4
- "version": "1.3.11",
4
+ "version": "1.3.12-alpha.2",
5
5
  "description": "Budibase Web Server",
6
6
  "main": "src/index.ts",
7
7
  "repository": {
@@ -77,11 +77,11 @@
77
77
  "license": "GPL-3.0",
78
78
  "dependencies": {
79
79
  "@apidevtools/swagger-parser": "10.0.3",
80
- "@budibase/backend-core": "^1.3.11",
81
- "@budibase/client": "^1.3.11",
82
- "@budibase/pro": "1.3.10",
83
- "@budibase/string-templates": "^1.3.11",
84
- "@budibase/types": "^1.3.11",
80
+ "@budibase/backend-core": "1.3.12-alpha.2",
81
+ "@budibase/client": "1.3.12-alpha.2",
82
+ "@budibase/pro": "1.3.12-alpha.1",
83
+ "@budibase/string-templates": "1.3.12-alpha.2",
84
+ "@budibase/types": "1.3.12-alpha.2",
85
85
  "@bull-board/api": "3.7.0",
86
86
  "@bull-board/koa": "3.9.4",
87
87
  "@elastic/elasticsearch": "7.10.0",
@@ -95,6 +95,7 @@
95
95
  "bcryptjs": "2.4.3",
96
96
  "bull": "4.8.5",
97
97
  "chmodr": "1.2.0",
98
+ "chokidar": "^3.5.3",
98
99
  "csvtojson": "2.0.10",
99
100
  "curlconverter": "3.21.0",
100
101
  "dotenv": "8.2.0",
@@ -140,6 +141,7 @@
140
141
  "snowflake-promise": "^4.5.0",
141
142
  "svelte": "3.49.0",
142
143
  "swagger-parser": "10.0.3",
144
+ "tar": "^6.1.11",
143
145
  "to-json-schema": "0.2.5",
144
146
  "uuid": "3.3.2",
145
147
  "validate.js": "0.13.1",
@@ -196,5 +198,5 @@
196
198
  "optionalDependencies": {
197
199
  "oracledb": "5.3.0"
198
200
  },
199
- "gitHead": "4011bfe1995a18043e71ff95e1f52ee73166a8a8"
201
+ "gitHead": "76870217376710fa1a8bf554fdfcc5f446dbde6c"
200
202
  }
@@ -58,6 +58,7 @@ async function init() {
58
58
  DEPLOYMENT_ENVIRONMENT: "development",
59
59
  BB_ADMIN_USER_EMAIL: "",
60
60
  BB_ADMIN_USER_PASSWORD: "",
61
+ PLUGINS_DIR: "",
61
62
  }
62
63
  let envFile = ""
63
64
  Object.keys(envFileJson).forEach(key => {
@@ -576,7 +576,7 @@ export const sync = async (ctx: any, next: any) => {
576
576
  }
577
577
  }
578
578
 
579
- const updateAppPackage = async (appPackage: any, appId: any) => {
579
+ export const updateAppPackage = async (appPackage: any, appId: any) => {
580
580
  return context.doInAppContext(appId, async () => {
581
581
  const db = context.getAppDB()
582
582
  const application = await db.get(DocumentType.APP_METADATA)
@@ -1,6 +1,8 @@
1
- const { DocumentType } = require("../../db/utils")
1
+ const { DocumentType, getPluginParams } = require("../../db/utils")
2
2
  const { getComponentLibraryManifest } = require("../../utilities/fileSystem")
3
3
  const { getAppDB } = require("@budibase/backend-core/context")
4
+ const { getGlobalDB } = require("@budibase/backend-core/tenancy")
5
+ const env = require("../../environment")
4
6
 
5
7
  exports.fetchAppComponentDefinitions = async function (ctx) {
6
8
  try {
@@ -10,7 +12,6 @@ exports.fetchAppComponentDefinitions = async function (ctx) {
10
12
  let componentManifests = await Promise.all(
11
13
  app.componentLibraries.map(async library => {
12
14
  let manifest = await getComponentLibraryManifest(library)
13
-
14
15
  return {
15
16
  manifest,
16
17
  library,
@@ -31,6 +32,28 @@ exports.fetchAppComponentDefinitions = async function (ctx) {
31
32
  }
32
33
  }
33
34
  }
35
+
36
+ // for now custom components only supported in self-host
37
+ if (env.SELF_HOSTED) {
38
+ // Add custom components
39
+ const globalDB = getGlobalDB()
40
+ const response = await globalDB.allDocs(
41
+ getPluginParams(null, {
42
+ include_docs: true,
43
+ })
44
+ )
45
+ response.rows
46
+ .map(row => row.doc)
47
+ .filter(plugin => plugin.schema.type === "component")
48
+ .forEach(plugin => {
49
+ const fullComponentName = `plugin/${plugin.name}`
50
+ definitions[fullComponentName] = {
51
+ component: fullComponentName,
52
+ ...plugin.schema.schema,
53
+ }
54
+ })
55
+ }
56
+
34
57
  ctx.body = definitions
35
58
  } catch (err) {
36
59
  console.error(`component-definitions=failed`, err)
@@ -7,7 +7,7 @@ const {
7
7
  getTableParams,
8
8
  } = require("../../db/utils")
9
9
  const { BuildSchemaErrors, InvalidColumns } = require("../../constants")
10
- const { integrations } = require("../../integrations")
10
+ const { getIntegration } = require("../../integrations")
11
11
  const { getDatasourceAndQuery } = require("./row/utils")
12
12
  const { invalidateDynamicVariables } = require("../../threads/utils")
13
13
  const { getAppDB } = require("@budibase/backend-core/context")
@@ -114,7 +114,7 @@ exports.update = async function (ctx) {
114
114
 
115
115
  // Drain connection pools when configuration is changed
116
116
  if (datasource.source) {
117
- const source = integrations[datasource.source]
117
+ const source = await getIntegration(datasource.source)
118
118
  if (source && source.pool) {
119
119
  await source.pool.end()
120
120
  }
@@ -149,7 +149,7 @@ exports.save = async function (ctx) {
149
149
 
150
150
  // Drain connection pools when configuration is changed
151
151
  if (datasource.source) {
152
- const source = integrations[datasource.source]
152
+ const source = await getIntegration(datasource.source)
153
153
  if (source && source.pool) {
154
154
  await source.pool.end()
155
155
  }
@@ -218,7 +218,7 @@ function updateError(error, newError, tables) {
218
218
  }
219
219
 
220
220
  const buildSchemaHelper = async datasource => {
221
- const Connector = integrations[datasource.source]
221
+ const Connector = await getIntegration(datasource.source)
222
222
 
223
223
  // Connect to the DB and build the schema
224
224
  const connector = new Connector(datasource.config)
@@ -1,12 +1,11 @@
1
- const { cloneDeep } = require("lodash")
2
- const { definitions } = require("../../integrations")
1
+ const { getDefinitions } = require("../../integrations")
3
2
  const { SourceName } = require("@budibase/types")
4
3
  const googlesheets = require("../../integrations/googlesheets")
5
4
  const { featureFlags } = require("@budibase/backend-core")
6
5
 
7
6
  exports.fetch = async function (ctx) {
8
7
  ctx.status = 200
9
- const defs = cloneDeep(definitions)
8
+ const defs = await getDefinitions()
10
9
 
11
10
  // for google sheets integration google verification
12
11
  if (featureFlags.isEnabled(featureFlags.FeatureFlag.GOOGLE_SHEETS)) {
@@ -17,6 +16,7 @@ exports.fetch = async function (ctx) {
17
16
  }
18
17
 
19
18
  exports.find = async function (ctx) {
19
+ const defs = await getDefinitions()
20
20
  ctx.status = 200
21
- ctx.body = definitions[ctx.params.type]
21
+ ctx.body = defs[ctx.params.type]
22
22
  }
@@ -0,0 +1,110 @@
1
+ import { ObjectStoreBuckets } from "../../constants"
2
+ import { extractPluginTarball, loadJSFile } from "../../utilities/fileSystem"
3
+ import { getGlobalDB } from "@budibase/backend-core/tenancy"
4
+ import { generatePluginID, getPluginParams } from "../../db/utils"
5
+ import { uploadDirectory } from "@budibase/backend-core/objectStore"
6
+ import { PluginType, FileType } from "@budibase/types"
7
+ import env from "../../environment"
8
+
9
+ export async function getPlugins(type?: PluginType) {
10
+ const db = getGlobalDB()
11
+ const response = await db.allDocs(
12
+ getPluginParams(null, {
13
+ include_docs: true,
14
+ })
15
+ )
16
+ const plugins = response.rows.map((row: any) => row.doc)
17
+ if (type) {
18
+ return plugins.filter((plugin: any) => plugin.schema?.type === type)
19
+ } else {
20
+ return plugins
21
+ }
22
+ }
23
+
24
+ export async function upload(ctx: any) {
25
+ const plugins: FileType[] =
26
+ ctx.request.files.file.length > 1
27
+ ? Array.from(ctx.request.files.file)
28
+ : [ctx.request.files.file]
29
+ try {
30
+ let docs = []
31
+ // can do single or multiple plugins
32
+ for (let plugin of plugins) {
33
+ const doc = await processPlugin(plugin)
34
+ docs.push(doc)
35
+ }
36
+ ctx.body = {
37
+ message: "Plugin(s) uploaded successfully",
38
+ plugins: docs,
39
+ }
40
+ } catch (err: any) {
41
+ const errMsg = err?.message ? err?.message : err
42
+ ctx.throw(400, `Failed to import plugin: ${errMsg}`)
43
+ }
44
+ }
45
+
46
+ export async function fetch(ctx: any) {
47
+ ctx.body = await getPlugins()
48
+ }
49
+
50
+ export async function destroy(ctx: any) {}
51
+
52
+ export async function processPlugin(plugin: FileType) {
53
+ if (!env.SELF_HOSTED) {
54
+ throw new Error("Plugins not supported outside of self-host.")
55
+ }
56
+ const db = getGlobalDB()
57
+ const { metadata, directory } = await extractPluginTarball(plugin)
58
+ const version = metadata.package.version,
59
+ name = metadata.package.name,
60
+ description = metadata.package.description
61
+
62
+ // first open the tarball into tmp directory
63
+ const bucketPath = `${name}/`
64
+ const files = await uploadDirectory(
65
+ ObjectStoreBuckets.PLUGINS,
66
+ directory,
67
+ bucketPath
68
+ )
69
+ const jsFile = files.find((file: any) => file.name.endsWith(".js"))
70
+ if (!jsFile) {
71
+ throw new Error(`Plugin missing .js file.`)
72
+ }
73
+ // validate the JS for a datasource
74
+ if (metadata.schema.type === PluginType.DATASOURCE) {
75
+ const js = loadJSFile(directory, jsFile.name)
76
+ // TODO: this isn't safe - but we need full node environment
77
+ // in future we should do this in a thread for safety
78
+ try {
79
+ eval(js)
80
+ } catch (err: any) {
81
+ const message = err?.message ? err.message : JSON.stringify(err)
82
+ throw new Error(`JS invalid: ${message}`)
83
+ }
84
+ }
85
+ const jsFileName = jsFile.name
86
+ const pluginId = generatePluginID(name)
87
+
88
+ // overwrite existing docs entirely if they exist
89
+ let rev
90
+ try {
91
+ const existing = await db.get(pluginId)
92
+ rev = existing._rev
93
+ } catch (err) {
94
+ rev = undefined
95
+ }
96
+ const doc = {
97
+ _id: pluginId,
98
+ _rev: rev,
99
+ ...metadata,
100
+ name,
101
+ version,
102
+ description,
103
+ jsUrl: `${bucketPath}${jsFileName}`,
104
+ }
105
+ const response = await db.put(doc)
106
+ return {
107
+ ...doc,
108
+ _rev: response.rev,
109
+ }
110
+ }
@@ -1,7 +1,14 @@
1
- const { getScreenParams, generateScreenID } = require("../../db/utils")
1
+ const {
2
+ getScreenParams,
3
+ generateScreenID,
4
+ getPluginParams,
5
+ DocumentType,
6
+ } = require("../../db/utils")
2
7
  const { AccessController } = require("@budibase/backend-core/roles")
3
8
  const { getAppDB } = require("@budibase/backend-core/context")
4
9
  const { events } = require("@budibase/backend-core")
10
+ const { getGlobalDB } = require("@budibase/backend-core/tenancy")
11
+ const { updateAppPackage } = require("./application")
5
12
 
6
13
  exports.fetch = async ctx => {
7
14
  const db = getAppDB()
@@ -32,6 +39,47 @@ exports.save = async ctx => {
32
39
 
33
40
  const response = await db.put(screen)
34
41
 
42
+ // Find any custom components being used
43
+ let pluginNames = []
44
+ let pluginAdded = false
45
+ findPlugins(screen.props, pluginNames)
46
+ if (pluginNames.length) {
47
+ const globalDB = getGlobalDB()
48
+ const pluginsResponse = await globalDB.allDocs(
49
+ getPluginParams(null, {
50
+ include_docs: true,
51
+ })
52
+ )
53
+ const requiredPlugins = pluginsResponse.rows
54
+ .map(row => row.doc)
55
+ .filter(plugin => {
56
+ return (
57
+ plugin.schema.type === "component" &&
58
+ pluginNames.includes(`plugin/${plugin.name}`)
59
+ )
60
+ })
61
+
62
+ // Update the app metadata
63
+ const application = await db.get(DocumentType.APP_METADATA)
64
+ let usedPlugins = application.usedPlugins || []
65
+
66
+ requiredPlugins.forEach(plugin => {
67
+ if (!usedPlugins.find(x => x._id === plugin._id)) {
68
+ pluginAdded = true
69
+ usedPlugins.push({
70
+ _id: plugin._id,
71
+ name: plugin.name,
72
+ version: plugin.version,
73
+ jsUrl: plugin.jsUrl,
74
+ })
75
+ }
76
+ })
77
+
78
+ if (pluginAdded) {
79
+ await updateAppPackage({ usedPlugins }, ctx.appId)
80
+ }
81
+ }
82
+
35
83
  if (eventFn) {
36
84
  await eventFn(screen)
37
85
  }
@@ -40,6 +88,7 @@ exports.save = async ctx => {
40
88
  ...screen,
41
89
  _id: response.id,
42
90
  _rev: response.rev,
91
+ pluginAdded,
43
92
  }
44
93
  }
45
94
 
@@ -56,3 +105,18 @@ exports.destroy = async ctx => {
56
105
  }
57
106
  ctx.status = 200
58
107
  }
108
+
109
+ const findPlugins = (component, foundPlugins) => {
110
+ if (!component) {
111
+ return
112
+ }
113
+ if (component._component.startsWith("plugin")) {
114
+ if (!foundPlugins.includes(component._component)) {
115
+ foundPlugins.push(component._component)
116
+ }
117
+ }
118
+ if (!component._children || !component._children.length) {
119
+ return
120
+ }
121
+ component._children.forEach(child => findPlugins(child, foundPlugins))
122
+ }
@@ -113,6 +113,7 @@ export const serveApp = async function (ctx: any) {
113
113
  production: env.isProd(),
114
114
  appId,
115
115
  clientLibPath: clientLibraryPath(appId, appInfo.version, ctx),
116
+ usedPlugins: appInfo.usedPlugins,
116
117
  })
117
118
 
118
119
  const appHbs = loadHandlebarsFile(`${__dirname}/templates/app.hbs`)
@@ -3,6 +3,7 @@
3
3
  export let favicon = ""
4
4
 
5
5
  export let clientLibPath
6
+ export let usedPlugins
6
7
  </script>
7
8
 
8
9
  <svelte:head>
@@ -83,6 +84,15 @@
83
84
  </script>
84
85
  <script type="application/javascript" src={clientLibPath}>
85
86
  </script>
87
+ <!-- Custom components need inserted after the core client library -->
88
+ <!-- But before loadBudibase is called -->
89
+ {#if usedPlugins?.length}
90
+ {#each usedPlugins as plugin}
91
+ <script
92
+ type="application/javascript"
93
+ src={`/plugins/${plugin.jsUrl}`}></script>
94
+ {/each}
95
+ {/if}
86
96
  <script type="application/javascript">
87
97
  if (window.loadBudibase) {
88
98
  window.loadBudibase()
@@ -66,7 +66,8 @@
66
66
  customTheme,
67
67
  previewDevice,
68
68
  navigation,
69
- hiddenComponentIds
69
+ hiddenComponentIds,
70
+ usedPlugins
70
71
  } = parsed
71
72
 
72
73
  // Set some flags so the app knows we're in the builder
@@ -81,6 +82,7 @@
81
82
  window["##BUDIBASE_PREVIEW_DEVICE##"] = previewDevice
82
83
  window["##BUDIBASE_PREVIEW_NAVIGATION##"] = navigation
83
84
  window["##BUDIBASE_HIDDEN_COMPONENT_IDS##"] = hiddenComponentIds
85
+ window["##BUDIBASE_USED_PLUGINS##"] = usedPlugins
84
86
 
85
87
  // Initialise app
86
88
  try {
@@ -24,6 +24,7 @@ import metadataRoutes from "./metadata"
24
24
  import devRoutes from "./dev"
25
25
  import cloudRoutes from "./cloud"
26
26
  import migrationRoutes from "./migrations"
27
+ import pluginRoutes from "./plugin"
27
28
 
28
29
  export { default as staticRoutes } from "./static"
29
30
  export { default as publicRoutes } from "./public"
@@ -57,4 +58,5 @@ export const mainRoutes = [
57
58
  tableRoutes,
58
59
  rowRoutes,
59
60
  migrationRoutes,
61
+ pluginRoutes,
60
62
  ]
@@ -0,0 +1,13 @@
1
+ import Router from "@koa/router"
2
+ import * as controller from "../controllers/plugin"
3
+ import authorized from "../../middleware/authorized"
4
+ import { BUILDER } from "@budibase/backend-core/permissions"
5
+
6
+ const router = new Router()
7
+
8
+ router
9
+ .post("/api/plugin/upload", authorized(BUILDER), controller.upload)
10
+ .get("/api/plugin", authorized(BUILDER), controller.fetch)
11
+ .delete("/api/plugin/:pluginId", authorized(BUILDER), controller.destroy)
12
+
13
+ export default router
@@ -231,20 +231,6 @@ describe("/queries", () => {
231
231
  url: `/api/queries/preview`,
232
232
  })
233
233
  })
234
-
235
- it("should fail with invalid integration type", async () => {
236
- const { datasource } = await createInvalidIntegration()
237
- await request
238
- .post(`/api/queries/preview`)
239
- .send({
240
- datasourceId: datasource._id,
241
- parameters: {},
242
- fields: {},
243
- queryVerb: "read",
244
- })
245
- .set(config.defaultHeaders())
246
- .expect(400)
247
- })
248
234
  })
249
235
 
250
236
  describe("execute", () => {
@@ -261,17 +247,14 @@ describe("/queries", () => {
261
247
  })
262
248
 
263
249
  it("should fail with invalid integration type", async () => {
264
- const { query, datasource } = await createInvalidIntegration()
265
- await request
266
- .post(`/api/queries/${query._id}`)
267
- .send({
268
- datasourceId: datasource._id,
269
- parameters: {},
270
- fields: {},
271
- queryVerb: "read",
272
- })
273
- .set(config.defaultHeaders())
274
- .expect(400)
250
+ let error
251
+ try {
252
+ await createInvalidIntegration()
253
+ } catch (err) {
254
+ error = err
255
+ }
256
+ expect(error).toBeDefined()
257
+ expect(error.message).toBe("No datasource implementation found.")
275
258
  })
276
259
  })
277
260