@budibase/server 1.3.13 → 1.3.15-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (283) hide show
  1. package/__mocks__/mongodb.ts +1 -1
  2. package/builder/assets/{index.e50bda9f.css → index.ecdac070.css} +2 -2
  3. package/builder/assets/index.ed9ae6a1.js +995 -0
  4. package/builder/index.html +2 -4
  5. package/coverage/clover.xml +377 -313
  6. package/coverage/coverage-final.json +22 -22
  7. package/coverage/lcov-report/api/controllers/apikeys.js.html +1 -1
  8. package/coverage/lcov-report/api/controllers/auth.js.html +1 -1
  9. package/coverage/lcov-report/api/controllers/automation.js.html +1 -1
  10. package/coverage/lcov-report/api/controllers/backup.js.html +8 -5
  11. package/coverage/lcov-report/api/controllers/cloud.js.html +1 -1
  12. package/coverage/lcov-report/api/controllers/component.js.html +82 -13
  13. package/coverage/lcov-report/api/controllers/datasource.js.html +20 -20
  14. package/coverage/lcov-report/api/controllers/dev.js.html +1 -1
  15. package/coverage/lcov-report/api/controllers/index.html +32 -32
  16. package/coverage/lcov-report/api/controllers/integration.js.html +6 -6
  17. package/coverage/lcov-report/api/controllers/layout.js.html +1 -1
  18. package/coverage/lcov-report/api/controllers/metadata.js.html +1 -1
  19. package/coverage/lcov-report/api/controllers/migrations.js.html +1 -1
  20. package/coverage/lcov-report/api/controllers/permission.js.html +1 -1
  21. package/coverage/lcov-report/api/controllers/query/index.html +1 -1
  22. package/coverage/lcov-report/api/controllers/query/validation.js.html +1 -1
  23. package/coverage/lcov-report/api/controllers/role.js.html +1 -1
  24. package/coverage/lcov-report/api/controllers/routing.js.html +1 -1
  25. package/coverage/lcov-report/api/controllers/row/external.js.html +1 -1
  26. package/coverage/lcov-report/api/controllers/row/index.html +1 -1
  27. package/coverage/lcov-report/api/controllers/row/internal.js.html +1 -1
  28. package/coverage/lcov-report/api/controllers/row/internalSearch.js.html +1 -1
  29. package/coverage/lcov-report/api/controllers/row/staticFormula.js.html +1 -1
  30. package/coverage/lcov-report/api/controllers/row/utils.js.html +1 -1
  31. package/coverage/lcov-report/api/controllers/screen.js.html +204 -12
  32. package/coverage/lcov-report/api/controllers/script.js.html +1 -1
  33. package/coverage/lcov-report/api/controllers/table/bulkFormula.js.html +1 -1
  34. package/coverage/lcov-report/api/controllers/table/external.js.html +1 -1
  35. package/coverage/lcov-report/api/controllers/table/index.html +1 -1
  36. package/coverage/lcov-report/api/controllers/table/index.js.html +1 -1
  37. package/coverage/lcov-report/api/controllers/templates.js.html +1 -1
  38. package/coverage/lcov-report/api/controllers/user.js.html +1 -1
  39. package/coverage/lcov-report/api/controllers/view/exporters.js.html +1 -1
  40. package/coverage/lcov-report/api/controllers/view/index.html +1 -1
  41. package/coverage/lcov-report/api/controllers/view/index.js.html +1 -1
  42. package/coverage/lcov-report/api/controllers/view/utils.js.html +1 -1
  43. package/coverage/lcov-report/api/controllers/view/viewBuilder.js.html +1 -1
  44. package/coverage/lcov-report/api/controllers/webhook.js.html +1 -1
  45. package/coverage/lcov-report/api/index.html +1 -1
  46. package/coverage/lcov-report/api/index.js.html +12 -12
  47. package/coverage/lcov-report/api/routes/analytics.js.html +1 -1
  48. package/coverage/lcov-report/api/routes/apikeys.js.html +1 -1
  49. package/coverage/lcov-report/api/routes/auth.js.html +1 -1
  50. package/coverage/lcov-report/api/routes/automation.js.html +1 -1
  51. package/coverage/lcov-report/api/routes/backup.js.html +1 -1
  52. package/coverage/lcov-report/api/routes/cloud.js.html +1 -1
  53. package/coverage/lcov-report/api/routes/component.js.html +1 -1
  54. package/coverage/lcov-report/api/routes/datasource.js.html +1 -1
  55. package/coverage/lcov-report/api/routes/deploy.js.html +1 -1
  56. package/coverage/lcov-report/api/routes/dev.js.html +1 -1
  57. package/coverage/lcov-report/api/routes/index.html +1 -1
  58. package/coverage/lcov-report/api/routes/integration.js.html +1 -1
  59. package/coverage/lcov-report/api/routes/layout.js.html +1 -1
  60. package/coverage/lcov-report/api/routes/metadata.js.html +1 -1
  61. package/coverage/lcov-report/api/routes/migrations.js.html +1 -1
  62. package/coverage/lcov-report/api/routes/permission.js.html +1 -1
  63. package/coverage/lcov-report/api/routes/query.js.html +1 -1
  64. package/coverage/lcov-report/api/routes/role.js.html +1 -1
  65. package/coverage/lcov-report/api/routes/routing.js.html +1 -1
  66. package/coverage/lcov-report/api/routes/screen.js.html +1 -1
  67. package/coverage/lcov-report/api/routes/script.js.html +1 -1
  68. package/coverage/lcov-report/api/routes/table.js.html +1 -1
  69. package/coverage/lcov-report/api/routes/templates.js.html +1 -1
  70. package/coverage/lcov-report/api/routes/user.js.html +1 -1
  71. package/coverage/lcov-report/api/routes/utils/index.html +1 -1
  72. package/coverage/lcov-report/api/routes/utils/validators.js.html +1 -1
  73. package/coverage/lcov-report/api/routes/view.js.html +1 -1
  74. package/coverage/lcov-report/api/routes/webhook.js.html +1 -1
  75. package/coverage/lcov-report/automations/actions.js.html +1 -1
  76. package/coverage/lcov-report/automations/automationUtils.js.html +1 -1
  77. package/coverage/lcov-report/automations/bullboard.js.html +1 -1
  78. package/coverage/lcov-report/automations/index.html +24 -24
  79. package/coverage/lcov-report/automations/index.js.html +18 -9
  80. package/coverage/lcov-report/automations/steps/bash.js.html +1 -1
  81. package/coverage/lcov-report/automations/steps/delay.js.html +1 -1
  82. package/coverage/lcov-report/automations/steps/discord.js.html +1 -1
  83. package/coverage/lcov-report/automations/steps/executeQuery.js.html +1 -1
  84. package/coverage/lcov-report/automations/steps/executeScript.js.html +1 -1
  85. package/coverage/lcov-report/automations/steps/filter.js.html +1 -1
  86. package/coverage/lcov-report/automations/steps/index.html +1 -1
  87. package/coverage/lcov-report/automations/steps/integromat.js.html +1 -1
  88. package/coverage/lcov-report/automations/steps/loop.js.html +1 -1
  89. package/coverage/lcov-report/automations/steps/outgoingWebhook.js.html +1 -1
  90. package/coverage/lcov-report/automations/steps/queryRows.js.html +1 -1
  91. package/coverage/lcov-report/automations/steps/sendSmtpEmail.js.html +1 -1
  92. package/coverage/lcov-report/automations/steps/serverLog.js.html +1 -1
  93. package/coverage/lcov-report/automations/steps/slack.js.html +1 -1
  94. package/coverage/lcov-report/automations/steps/updateRow.js.html +1 -1
  95. package/coverage/lcov-report/automations/steps/utils.js.html +1 -1
  96. package/coverage/lcov-report/automations/steps/zapier.js.html +1 -1
  97. package/coverage/lcov-report/automations/triggerInfo/app.js.html +1 -1
  98. package/coverage/lcov-report/automations/triggerInfo/cron.js.html +1 -1
  99. package/coverage/lcov-report/automations/triggerInfo/index.html +1 -1
  100. package/coverage/lcov-report/automations/triggerInfo/index.js.html +1 -1
  101. package/coverage/lcov-report/automations/triggerInfo/rowDeleted.js.html +1 -1
  102. package/coverage/lcov-report/automations/triggerInfo/rowSaved.js.html +1 -1
  103. package/coverage/lcov-report/automations/triggerInfo/rowUpdated.js.html +1 -1
  104. package/coverage/lcov-report/automations/triggerInfo/webhook.js.html +1 -1
  105. package/coverage/lcov-report/automations/triggers.js.html +126 -24
  106. package/coverage/lcov-report/constants/index.html +1 -1
  107. package/coverage/lcov-report/constants/index.js.html +1 -1
  108. package/coverage/lcov-report/constants/layouts.js.html +1 -1
  109. package/coverage/lcov-report/constants/screens.js.html +1 -1
  110. package/coverage/lcov-report/db/inMemoryView.js.html +1 -1
  111. package/coverage/lcov-report/db/index.html +18 -18
  112. package/coverage/lcov-report/db/index.js.html +1 -1
  113. package/coverage/lcov-report/db/linkedRows/LinkController.js.html +1 -1
  114. package/coverage/lcov-report/db/linkedRows/index.html +1 -1
  115. package/coverage/lcov-report/db/linkedRows/index.js.html +1 -1
  116. package/coverage/lcov-report/db/linkedRows/linkUtils.js.html +1 -1
  117. package/coverage/lcov-report/db/newid.js.html +2 -2
  118. package/coverage/lcov-report/db/utils.js.html +56 -20
  119. package/coverage/lcov-report/events/AutomationEmitter.js.html +1 -1
  120. package/coverage/lcov-report/events/index.html +1 -1
  121. package/coverage/lcov-report/events/index.js.html +1 -1
  122. package/coverage/lcov-report/events/utils.js.html +1 -1
  123. package/coverage/lcov-report/index.html +48 -48
  124. package/coverage/lcov-report/integrations/tests/TestConfiguration.js.html +1 -1
  125. package/coverage/lcov-report/integrations/tests/index.html +1 -1
  126. package/coverage/lcov-report/middleware/appInfo.js.html +1 -1
  127. package/coverage/lcov-report/middleware/builder.js.html +12 -12
  128. package/coverage/lcov-report/middleware/currentapp.js.html +33 -33
  129. package/coverage/lcov-report/middleware/index.html +1 -1
  130. package/coverage/lcov-report/middleware/joi-validator.js.html +1 -1
  131. package/coverage/lcov-report/middleware/publicApi.js.html +1 -1
  132. package/coverage/lcov-report/middleware/resourceId.js.html +7 -7
  133. package/coverage/lcov-report/middleware/selfhost.js.html +1 -1
  134. package/coverage/lcov-report/middleware/utils.js.html +2 -2
  135. package/coverage/lcov-report/utilities/budibaseDir.js.html +3 -3
  136. package/coverage/lcov-report/utilities/centralPath.js.html +4 -4
  137. package/coverage/lcov-report/utilities/csvParser.js.html +1 -1
  138. package/coverage/lcov-report/utilities/global.js.html +16 -16
  139. package/coverage/lcov-report/utilities/index.html +19 -19
  140. package/coverage/lcov-report/utilities/index.js.html +66 -42
  141. package/coverage/lcov-report/utilities/queue/inMemoryQueue.js.html +2 -2
  142. package/coverage/lcov-report/utilities/queue/index.html +1 -1
  143. package/coverage/lcov-report/utilities/routing/index.html +1 -1
  144. package/coverage/lcov-report/utilities/routing/index.js.html +1 -1
  145. package/coverage/lcov-report/utilities/rowProcessor/index.html +5 -5
  146. package/coverage/lcov-report/utilities/rowProcessor/index.js.html +1 -1
  147. package/coverage/lcov-report/utilities/rowProcessor/utils.js.html +4 -4
  148. package/coverage/lcov-report/utilities/scriptRunner.js.html +1 -1
  149. package/coverage/lcov-report/utilities/security.js.html +2 -2
  150. package/coverage/lcov-report/utilities/statusCodes.js.html +1 -1
  151. package/coverage/lcov-report/utilities/usageQuota/index.html +1 -1
  152. package/coverage/lcov-report/utilities/usageQuota/rows.js.html +6 -3
  153. package/coverage/lcov-report/utilities/usageQuota/usageQuoteReset.js.html +1 -1
  154. package/coverage/lcov-report/utilities/users.js.html +1 -1
  155. package/coverage/lcov-report/utilities/workerRequests.js.html +1 -1
  156. package/coverage/lcov.info +692 -564
  157. package/dist/api/controllers/application.js +6 -9
  158. package/dist/api/controllers/backup.js +2 -1
  159. package/dist/api/controllers/component.js +18 -1
  160. package/dist/api/controllers/datasource.js +4 -4
  161. package/dist/api/controllers/deploy/index.js +17 -2
  162. package/dist/api/controllers/integration.js +4 -4
  163. package/dist/api/controllers/plugin/file.js +24 -0
  164. package/dist/api/controllers/plugin/github.js +69 -0
  165. package/dist/api/controllers/plugin/index.js +189 -0
  166. package/dist/api/controllers/plugin/npm.js +58 -0
  167. package/dist/api/controllers/plugin/uploaders.js +11 -0
  168. package/dist/api/controllers/plugin/url.js +24 -0
  169. package/dist/api/controllers/plugin/utils.js +26 -0
  170. package/dist/api/controllers/row/ExternalRequest.js +1 -1
  171. package/dist/api/controllers/screen.js +51 -2
  172. package/dist/api/controllers/static/index.js +1 -0
  173. package/dist/api/controllers/static/templates/BudibaseApp.svelte +10 -0
  174. package/dist/api/controllers/static/templates/preview.hbs +3 -1
  175. package/dist/api/routes/index.js +2 -0
  176. package/dist/api/routes/plugin.js +39 -0
  177. package/dist/app.js +9 -0
  178. package/dist/automations/index.js +10 -5
  179. package/dist/automations/triggers.js +39 -5
  180. package/dist/automations/utils.js +16 -7
  181. package/dist/db/utils.js +10 -1
  182. package/dist/environment.js +1 -0
  183. package/dist/integrations/airtable.js +126 -129
  184. package/dist/integrations/arangodb.js +86 -89
  185. package/dist/integrations/base/query.js +2 -2
  186. package/dist/integrations/couchdb.js +80 -83
  187. package/dist/integrations/dynamodb.js +161 -164
  188. package/dist/integrations/elasticsearch.js +135 -138
  189. package/dist/integrations/firebase.js +154 -157
  190. package/dist/integrations/googlesheets.js +323 -327
  191. package/dist/integrations/index.js +93 -51
  192. package/dist/integrations/microsoftSqlServer.js +240 -243
  193. package/dist/integrations/mongodb.js +239 -242
  194. package/dist/integrations/mysql.js +215 -219
  195. package/dist/integrations/oracle.js +318 -322
  196. package/dist/integrations/postgres.js +262 -265
  197. package/dist/integrations/redis.js +134 -139
  198. package/dist/integrations/rest.js +293 -281
  199. package/dist/integrations/s3.js +61 -64
  200. package/dist/integrations/snowflake.js +83 -86
  201. package/dist/package.json +8 -6
  202. package/dist/threads/automation.js +6 -0
  203. package/dist/threads/query.js +2 -2
  204. package/dist/tsconfig.build.tsbuildinfo +1 -1
  205. package/dist/utilities/fileSystem/index.js +116 -7
  206. package/dist/utilities/index.js +8 -0
  207. package/dist/utilities/rowProcessor/utils.js +1 -1
  208. package/dist/utilities/usageQuota/rows.js +2 -1
  209. package/dist/watch.js +77 -0
  210. package/package.json +9 -7
  211. package/scripts/dev/manage.js +1 -0
  212. package/src/api/controllers/application.ts +2 -6
  213. package/src/api/controllers/backup.js +2 -1
  214. package/src/api/controllers/component.js +25 -2
  215. package/src/api/controllers/datasource.js +4 -4
  216. package/src/api/controllers/deploy/index.ts +17 -4
  217. package/src/api/controllers/integration.js +4 -4
  218. package/src/api/controllers/plugin/file.ts +15 -0
  219. package/src/api/controllers/plugin/github.ts +75 -0
  220. package/src/api/controllers/plugin/index.ts +202 -0
  221. package/src/api/controllers/plugin/npm.ts +56 -0
  222. package/src/api/controllers/plugin/uploaders.ts +4 -0
  223. package/src/api/controllers/plugin/url.ts +12 -0
  224. package/src/api/controllers/plugin/utils.ts +19 -0
  225. package/src/api/controllers/row/ExternalRequest.ts +1 -1
  226. package/src/api/controllers/screen.js +65 -1
  227. package/src/api/controllers/static/index.ts +1 -0
  228. package/src/api/controllers/static/templates/BudibaseApp.svelte +10 -0
  229. package/src/api/controllers/static/templates/preview.hbs +3 -1
  230. package/src/api/routes/index.ts +2 -0
  231. package/src/api/routes/plugin.ts +14 -0
  232. package/src/api/routes/tests/query.spec.js +11 -28
  233. package/src/app.ts +12 -0
  234. package/src/automations/index.js +6 -3
  235. package/src/automations/triggers.js +44 -10
  236. package/src/automations/utils.ts +21 -9
  237. package/src/db/utils.js +12 -0
  238. package/src/definitions/datasource.ts +1 -0
  239. package/src/environment.js +1 -0
  240. package/src/integrations/airtable.ts +116 -118
  241. package/src/integrations/arangodb.ts +87 -89
  242. package/src/integrations/base/query.ts +2 -2
  243. package/src/integrations/couchdb.ts +83 -89
  244. package/src/integrations/dynamodb.ts +173 -175
  245. package/src/integrations/elasticsearch.ts +120 -122
  246. package/src/integrations/firebase.ts +157 -159
  247. package/src/integrations/googlesheets.ts +328 -338
  248. package/src/integrations/index.ts +59 -21
  249. package/src/integrations/microsoftSqlServer.ts +249 -251
  250. package/src/integrations/mongodb.ts +261 -258
  251. package/src/integrations/mysql.ts +229 -234
  252. package/src/integrations/oracle.ts +373 -380
  253. package/src/integrations/postgres.ts +264 -268
  254. package/src/integrations/redis.ts +128 -132
  255. package/src/integrations/rest.ts +325 -317
  256. package/src/integrations/s3.ts +68 -71
  257. package/src/integrations/snowflake.ts +80 -82
  258. package/src/integrations/tests/{airtable.spec.js → airtable.spec.ts} +21 -18
  259. package/src/integrations/tests/{arangodb.spec.js → arangodb.spec.ts} +12 -9
  260. package/src/integrations/tests/{couchdb.spec.js → couchdb.spec.ts} +33 -27
  261. package/src/integrations/tests/{dynamodb.spec.js → dynamodb.spec.ts} +45 -41
  262. package/src/integrations/tests/{elasticsearch.spec.js → elasticsearch.spec.ts} +22 -21
  263. package/src/integrations/tests/{firebase.spec.js → firebase.spec.ts} +33 -28
  264. package/src/integrations/tests/{microsoftSqlServer.spec.js → microsoftSqlServer.spec.ts} +12 -11
  265. package/src/integrations/tests/{mongo.spec.js → mongo.spec.ts} +46 -35
  266. package/src/integrations/tests/{mysql.spec.js → mysql.spec.ts} +17 -15
  267. package/src/integrations/tests/{oracle.spec.js → oracle.spec.ts} +19 -17
  268. package/src/integrations/tests/{postgres.spec.js → postgres.spec.ts} +19 -17
  269. package/src/integrations/tests/{redis.spec.js → redis.spec.ts} +27 -18
  270. package/src/integrations/tests/{rest.spec.js → rest.spec.ts} +57 -14
  271. package/src/integrations/tests/{s3.spec.js → s3.spec.ts} +10 -8
  272. package/src/integrations/tests/{sql.spec.js → sql.spec.ts} +228 -177
  273. package/src/module.d.ts +2 -0
  274. package/src/threads/automation.ts +6 -0
  275. package/src/threads/query.ts +2 -2
  276. package/src/utilities/fileSystem/index.js +135 -6
  277. package/src/utilities/index.js +8 -0
  278. package/src/utilities/rowProcessor/utils.js +1 -1
  279. package/src/utilities/usageQuota/rows.js +2 -1
  280. package/src/watch.ts +38 -0
  281. package/builder/assets/index.fb22f106.js +0 -875
  282. package/builder/assets/vendor.5e1fe644.css +0 -1
  283. package/builder/assets/vendor.67dbd8fa.js +0 -248
@@ -14,14 +14,18 @@ const { join } = require("path");
14
14
  const uuid = require("uuid/v4");
15
15
  const { doWithDB, dangerousGetDB, closeDB, } = require("@budibase/backend-core/db");
16
16
  const { ObjectStoreBuckets } = require("../../constants");
17
- const { upload, retrieve, retrieveToTmp, streamUpload, deleteFolder, downloadTarball, deleteFiles, } = require("./utilities");
17
+ const { upload, retrieve, retrieveToTmp, streamUpload, deleteFolder, downloadTarball, downloadTarballDirect, deleteFiles, } = require("./utilities");
18
18
  const { updateClientLibrary } = require("./clientLibrary");
19
+ const { checkSlashesInUrl } = require("../");
19
20
  const env = require("../../environment");
20
21
  const { USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX, TABLE_ROW_PREFIX, } = require("../../db/utils");
21
22
  const MemoryStream = require("memorystream");
22
23
  const { getAppId } = require("@budibase/backend-core/context");
24
+ const tar = require("tar");
25
+ const fetch = require("node-fetch");
23
26
  const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..");
24
27
  const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules");
28
+ const DATASOURCE_PATH = join(budibaseTempDir(), "datasource");
25
29
  /**
26
30
  * The single stack system (Cloud and Builder) should not make use of the file system where possible,
27
31
  * this file handles all of the file access for the system with the intention of limiting it all to one
@@ -87,6 +91,12 @@ exports.getTemplateStream = (template) => __awaiter(void 0, void 0, void 0, func
87
91
  exports.loadHandlebarsFile = path => {
88
92
  return fs.readFileSync(path, "utf8");
89
93
  };
94
+ /**
95
+ * Same as above just with a different name.
96
+ */
97
+ exports.loadJSFile = (directory, name) => {
98
+ return fs.readFileSync(join(directory, name), "utf8");
99
+ };
90
100
  /**
91
101
  * When return a file from the API need to write the file to the system temporarily so we
92
102
  * can create a read stream to send.
@@ -110,11 +120,11 @@ exports.defineFilter = excludeRows => {
110
120
  * data or user relationships.
111
121
  * @param {string} appId The app to backup
112
122
  * @param {object} config Config to send to export DB
113
- * @param {boolean} includeRows Flag to state whether the export should include data.
123
+ * @param {boolean} excludeRows Flag to state whether the export should include data.
114
124
  * @returns {*} either a string or a stream of the backup
115
125
  */
116
- const backupAppData = (appId, config, includeRows) => __awaiter(void 0, void 0, void 0, function* () {
117
- return yield exports.exportDB(appId, Object.assign(Object.assign({}, config), { filter: exports.defineFilter(includeRows) }));
126
+ const backupAppData = (appId, config, excludeRows) => __awaiter(void 0, void 0, void 0, function* () {
127
+ return yield exports.exportDB(appId, Object.assign(Object.assign({}, config), { filter: exports.defineFilter(excludeRows) }));
118
128
  });
119
129
  /**
120
130
  * Takes a copy of the database state for an app to the object store.
@@ -128,11 +138,11 @@ exports.performBackup = (appId, backupName) => __awaiter(void 0, void 0, void 0,
128
138
  /**
129
139
  * Streams a backup of the database state for an app
130
140
  * @param {string} appId The ID of the app which is to be backed up.
131
- * @param {boolean} includeRows Flag to state whether the export should include data.
141
+ * @param {boolean} excludeRows Flag to state whether the export should include data.
132
142
  * @returns {*} a readable stream of the backup which is written in real time
133
143
  */
134
- exports.streamBackup = (appId, includeRows) => __awaiter(void 0, void 0, void 0, function* () {
135
- return yield backupAppData(appId, { stream: true }, includeRows);
144
+ exports.streamBackup = (appId, excludeRows) => __awaiter(void 0, void 0, void 0, function* () {
145
+ return yield backupAppData(appId, { stream: true }, excludeRows);
136
146
  });
137
147
  /**
138
148
  * Exports a DB to either file or a variable (memory).
@@ -285,6 +295,104 @@ exports.cleanup = appIds => {
285
295
  }
286
296
  }
287
297
  };
298
+ const createTempFolder = item => {
299
+ const path = join(budibaseTempDir(), item);
300
+ try {
301
+ // remove old tmp directories automatically - don't combine
302
+ if (fs.existsSync(path)) {
303
+ fs.rmSync(path, { recursive: true, force: true });
304
+ }
305
+ fs.mkdirSync(path);
306
+ }
307
+ catch (err) {
308
+ throw new Error(`Path cannot be created: ${err.message}`);
309
+ }
310
+ return path;
311
+ };
312
+ exports.createTempFolder = createTempFolder;
313
+ const extractTarball = (fromFilePath, toPath) => __awaiter(void 0, void 0, void 0, function* () {
314
+ yield tar.extract({
315
+ file: fromFilePath,
316
+ C: toPath,
317
+ });
318
+ });
319
+ exports.extractTarball = extractTarball;
320
+ const getPluginMetadata = (path) => __awaiter(void 0, void 0, void 0, function* () {
321
+ let metadata = {};
322
+ try {
323
+ const pkg = fs.readFileSync(join(path, "package.json"), "utf8");
324
+ const schema = fs.readFileSync(join(path, "schema.json"), "utf8");
325
+ metadata.schema = JSON.parse(schema);
326
+ metadata.package = JSON.parse(pkg);
327
+ if (!metadata.package.name ||
328
+ !metadata.package.version ||
329
+ !metadata.package.description) {
330
+ throw new Error("package.json is missing one of 'name', 'version' or 'description'.");
331
+ }
332
+ }
333
+ catch (err) {
334
+ throw new Error(`Unable to process schema.json/package.json in plugin. ${err.message}`);
335
+ }
336
+ return { metadata, directory: path };
337
+ });
338
+ exports.getPluginMetadata = getPluginMetadata;
339
+ exports.getDatasourcePlugin = (name, url, hash) => __awaiter(void 0, void 0, void 0, function* () {
340
+ if (!fs.existsSync(DATASOURCE_PATH)) {
341
+ fs.mkdirSync(DATASOURCE_PATH);
342
+ }
343
+ const filename = join(DATASOURCE_PATH, name);
344
+ const metadataName = `${filename}.bbmetadata`;
345
+ if (fs.existsSync(filename)) {
346
+ const currentHash = fs.readFileSync(metadataName, "utf8");
347
+ // if hash is the same return the file, otherwise remove it and re-download
348
+ if (currentHash === hash) {
349
+ return require(filename);
350
+ }
351
+ else {
352
+ console.log(`Updating plugin: ${name}`);
353
+ fs.unlinkSync(filename);
354
+ }
355
+ }
356
+ const fullUrl = checkSlashesInUrl(`${env.MINIO_URL}/${ObjectStoreBuckets.PLUGINS}/${url}`);
357
+ const response = yield fetch(fullUrl);
358
+ if (response.status === 200) {
359
+ const content = yield response.text();
360
+ fs.writeFileSync(filename, content);
361
+ fs.writeFileSync(metadataName, hash);
362
+ require(filename);
363
+ }
364
+ else {
365
+ throw new Error(`Unable to retrieve plugin - reason: ${yield response.text()}`);
366
+ }
367
+ });
368
+ /**
369
+ * Find for a file recursively from start path applying filter, return first match
370
+ */
371
+ exports.findFileRec = (startPath, filter) => {
372
+ if (!fs.existsSync(startPath)) {
373
+ return;
374
+ }
375
+ const files = fs.readdirSync(startPath);
376
+ for (let i = 0, len = files.length; i < len; i++) {
377
+ const filename = join(startPath, files[i]);
378
+ const stat = fs.lstatSync(filename);
379
+ if (stat.isDirectory()) {
380
+ return exports.findFileRec(filename, filter);
381
+ }
382
+ else if (filename.endsWith(filter)) {
383
+ return filename;
384
+ }
385
+ }
386
+ };
387
+ /**
388
+ * Remove a folder which is not empty from the file system
389
+ */
390
+ exports.deleteFolderFileSystem = path => {
391
+ if (!fs.existsSync(path)) {
392
+ return;
393
+ }
394
+ fs.rmSync(path, { recursive: true, force: true });
395
+ };
288
396
  /**
289
397
  * Full function definition for below can be found in the utilities.
290
398
  */
@@ -292,5 +400,6 @@ exports.upload = upload;
292
400
  exports.retrieve = retrieve;
293
401
  exports.retrieveToTmp = retrieveToTmp;
294
402
  exports.deleteFiles = deleteFiles;
403
+ exports.downloadTarballDirect = downloadTarballDirect;
295
404
  exports.TOP_LEVEL_PATH = TOP_LEVEL_PATH;
296
405
  exports.NODE_MODULES_PATH = NODE_MODULES_PATH;
@@ -150,3 +150,11 @@ exports.convertBookmark = bookmark => {
150
150
  }
151
151
  return bookmark;
152
152
  };
153
+ exports.isQsTrue = param => {
154
+ if (typeof param === "string") {
155
+ return param.toLowerCase() === "true";
156
+ }
157
+ else {
158
+ return param === true;
159
+ }
160
+ };
@@ -64,7 +64,7 @@ exports.processDates = (table, rows) => {
64
64
  if (schema.type !== FieldTypes.DATETIME) {
65
65
  continue;
66
66
  }
67
- if (!schema.ignoreTimezones) {
67
+ if (!schema.timeOnly && !schema.ignoreTimezones) {
68
68
  datesWithTZ.push(column);
69
69
  }
70
70
  }
@@ -72,7 +72,8 @@ exports.getUniqueRows = (appIds) => __awaiter(void 0, void 0, void 0, function*
72
72
  // ensure uniqueness on a per app pair basis
73
73
  // this can't be done on all rows because app import results in
74
74
  // duplicate row ids across apps
75
- uniqueRows = uniqueRows.concat(...new Set(appRows));
75
+ // the array pre-concat is important to avoid stack overflow
76
+ uniqueRows = uniqueRows.concat([...new Set(appRows)]);
76
77
  }
77
78
  return uniqueRows;
78
79
  });
package/dist/watch.js ADDED
@@ -0,0 +1,77 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || function (mod) {
19
+ if (mod && mod.__esModule) return mod;
20
+ var result = {};
21
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
+ __setModuleDefault(result, mod);
23
+ return result;
24
+ };
25
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
26
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
27
+ return new (P || (P = Promise))(function (resolve, reject) {
28
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
29
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
30
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
31
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
32
+ });
33
+ };
34
+ var __importDefault = (this && this.__importDefault) || function (mod) {
35
+ return (mod && mod.__esModule) ? mod : { "default": mod };
36
+ };
37
+ Object.defineProperty(exports, "__esModule", { value: true });
38
+ exports.watch = void 0;
39
+ const path_1 = __importDefault(require("path"));
40
+ const env = __importStar(require("./environment"));
41
+ const chokidar_1 = __importDefault(require("chokidar"));
42
+ const fs_1 = __importDefault(require("fs"));
43
+ const backend_core_1 = require("@budibase/backend-core");
44
+ const constants_1 = require("@budibase/backend-core/constants");
45
+ const plugin_1 = require("./api/controllers/plugin");
46
+ function watch() {
47
+ const watchPath = path_1.default.join(env.PLUGINS_DIR, "./**/*.tar.gz");
48
+ chokidar_1.default
49
+ .watch(watchPath, {
50
+ ignored: "**/node_modules",
51
+ awaitWriteFinish: {
52
+ pollInterval: 100,
53
+ stabilityThreshold: 250,
54
+ },
55
+ usePolling: true,
56
+ interval: 250,
57
+ })
58
+ .on("all", (event, path) => __awaiter(this, void 0, void 0, function* () {
59
+ // Sanity checks
60
+ if (!(path === null || path === void 0 ? void 0 : path.endsWith(".tar.gz")) || !fs_1.default.existsSync(path)) {
61
+ return;
62
+ }
63
+ yield backend_core_1.tenancy.doInTenant(constants_1.DEFAULT_TENANT_ID, () => __awaiter(this, void 0, void 0, function* () {
64
+ try {
65
+ const split = path.split("/");
66
+ const name = split[split.length - 1];
67
+ console.log("Importing plugin:", path);
68
+ yield (0, plugin_1.processPlugin)({ name, path });
69
+ }
70
+ catch (err) {
71
+ const message = (err === null || err === void 0 ? void 0 : err.message) ? err === null || err === void 0 ? void 0 : err.message : err;
72
+ console.error("Failed to import plugin:", message);
73
+ }
74
+ }));
75
+ }));
76
+ }
77
+ exports.watch = watch;
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@budibase/server",
3
3
  "email": "hi@budibase.com",
4
- "version": "1.3.13",
4
+ "version": "1.3.15-alpha.1",
5
5
  "description": "Budibase Web Server",
6
6
  "main": "src/index.ts",
7
7
  "repository": {
@@ -77,11 +77,11 @@
77
77
  "license": "GPL-3.0",
78
78
  "dependencies": {
79
79
  "@apidevtools/swagger-parser": "10.0.3",
80
- "@budibase/backend-core": "^1.3.13",
81
- "@budibase/client": "^1.3.13",
82
- "@budibase/pro": "1.3.12",
83
- "@budibase/string-templates": "^1.3.13",
84
- "@budibase/types": "^1.3.13",
80
+ "@budibase/backend-core": "1.3.15-alpha.1",
81
+ "@budibase/client": "1.3.15-alpha.1",
82
+ "@budibase/pro": "1.3.15-alpha.0",
83
+ "@budibase/string-templates": "1.3.15-alpha.1",
84
+ "@budibase/types": "1.3.15-alpha.1",
85
85
  "@bull-board/api": "3.7.0",
86
86
  "@bull-board/koa": "3.9.4",
87
87
  "@elastic/elasticsearch": "7.10.0",
@@ -95,6 +95,7 @@
95
95
  "bcryptjs": "2.4.3",
96
96
  "bull": "4.8.5",
97
97
  "chmodr": "1.2.0",
98
+ "chokidar": "3.5.3",
98
99
  "csvtojson": "2.0.10",
99
100
  "curlconverter": "3.21.0",
100
101
  "dotenv": "8.2.0",
@@ -140,6 +141,7 @@
140
141
  "snowflake-promise": "^4.5.0",
141
142
  "svelte": "3.49.0",
142
143
  "swagger-parser": "10.0.3",
144
+ "tar": "6.1.11",
143
145
  "to-json-schema": "0.2.5",
144
146
  "uuid": "3.3.2",
145
147
  "validate.js": "0.13.1",
@@ -196,5 +198,5 @@
196
198
  "optionalDependencies": {
197
199
  "oracledb": "5.3.0"
198
200
  },
199
- "gitHead": "272a2ec7c7f2fabf74153fc8573ce6745134cdde"
201
+ "gitHead": "a92a8ed882cde62db5903d495112e202173defc7"
200
202
  }
@@ -58,6 +58,7 @@ async function init() {
58
58
  DEPLOYMENT_ENVIRONMENT: "development",
59
59
  BB_ADMIN_USER_EMAIL: "",
60
60
  BB_ADMIN_USER_PASSWORD: "",
61
+ PLUGINS_DIR: "",
61
62
  }
62
63
  let envFile = ""
63
64
  Object.keys(envFileJson).forEach(key => {
@@ -553,11 +553,7 @@ export const sync = async (ctx: any, next: any) => {
553
553
  })
554
554
  let error
555
555
  try {
556
- await replication.replicate({
557
- filter: function (doc: any) {
558
- return doc._id !== DocumentType.APP_METADATA
559
- },
560
- })
556
+ await replication.replicate(replication.appReplicateOpts())
561
557
  } catch (err) {
562
558
  error = err
563
559
  } finally {
@@ -576,7 +572,7 @@ export const sync = async (ctx: any, next: any) => {
576
572
  }
577
573
  }
578
574
 
579
- const updateAppPackage = async (appPackage: any, appId: any) => {
575
+ export const updateAppPackage = async (appPackage: any, appId: any) => {
580
576
  return context.doInAppContext(appId, async () => {
581
577
  const db = context.getAppDB()
582
578
  const application = await db.get(DocumentType.APP_METADATA)
@@ -1,11 +1,12 @@
1
1
  const { streamBackup } = require("../../utilities/fileSystem")
2
2
  const { events, context } = require("@budibase/backend-core")
3
3
  const { DocumentType } = require("../../db/utils")
4
+ const { isQsTrue } = require("../../utilities")
4
5
 
5
6
  exports.exportAppDump = async function (ctx) {
6
7
  let { appId, excludeRows } = ctx.query
7
8
  const appName = decodeURI(ctx.query.appname)
8
- excludeRows = excludeRows === "true"
9
+ excludeRows = isQsTrue(excludeRows)
9
10
  const backupIdentifier = `${appName}-export-${new Date().getTime()}.txt`
10
11
  ctx.attachment(backupIdentifier)
11
12
  ctx.body = await streamBackup(appId, excludeRows)
@@ -1,6 +1,8 @@
1
- const { DocumentType } = require("../../db/utils")
1
+ const { DocumentType, getPluginParams } = require("../../db/utils")
2
2
  const { getComponentLibraryManifest } = require("../../utilities/fileSystem")
3
3
  const { getAppDB } = require("@budibase/backend-core/context")
4
+ const { getGlobalDB } = require("@budibase/backend-core/tenancy")
5
+ const env = require("../../environment")
4
6
 
5
7
  exports.fetchAppComponentDefinitions = async function (ctx) {
6
8
  try {
@@ -10,7 +12,6 @@ exports.fetchAppComponentDefinitions = async function (ctx) {
10
12
  let componentManifests = await Promise.all(
11
13
  app.componentLibraries.map(async library => {
12
14
  let manifest = await getComponentLibraryManifest(library)
13
-
14
15
  return {
15
16
  manifest,
16
17
  library,
@@ -31,6 +32,28 @@ exports.fetchAppComponentDefinitions = async function (ctx) {
31
32
  }
32
33
  }
33
34
  }
35
+
36
+ // for now custom components only supported in self-host
37
+ if (env.SELF_HOSTED) {
38
+ // Add custom components
39
+ const globalDB = getGlobalDB()
40
+ const response = await globalDB.allDocs(
41
+ getPluginParams(null, {
42
+ include_docs: true,
43
+ })
44
+ )
45
+ response.rows
46
+ .map(row => row.doc)
47
+ .filter(plugin => plugin.schema.type === "component")
48
+ .forEach(plugin => {
49
+ const fullComponentName = `plugin/${plugin.name}`
50
+ definitions[fullComponentName] = {
51
+ component: fullComponentName,
52
+ ...plugin.schema.schema,
53
+ }
54
+ })
55
+ }
56
+
34
57
  ctx.body = definitions
35
58
  } catch (err) {
36
59
  console.error(`component-definitions=failed`, err)
@@ -7,7 +7,7 @@ const {
7
7
  getTableParams,
8
8
  } = require("../../db/utils")
9
9
  const { BuildSchemaErrors, InvalidColumns } = require("../../constants")
10
- const { integrations } = require("../../integrations")
10
+ const { getIntegration } = require("../../integrations")
11
11
  const { getDatasourceAndQuery } = require("./row/utils")
12
12
  const { invalidateDynamicVariables } = require("../../threads/utils")
13
13
  const { getAppDB } = require("@budibase/backend-core/context")
@@ -114,7 +114,7 @@ exports.update = async function (ctx) {
114
114
 
115
115
  // Drain connection pools when configuration is changed
116
116
  if (datasource.source) {
117
- const source = integrations[datasource.source]
117
+ const source = await getIntegration(datasource.source)
118
118
  if (source && source.pool) {
119
119
  await source.pool.end()
120
120
  }
@@ -149,7 +149,7 @@ exports.save = async function (ctx) {
149
149
 
150
150
  // Drain connection pools when configuration is changed
151
151
  if (datasource.source) {
152
- const source = integrations[datasource.source]
152
+ const source = await getIntegration(datasource.source)
153
153
  if (source && source.pool) {
154
154
  await source.pool.end()
155
155
  }
@@ -218,7 +218,7 @@ function updateError(error, newError, tables) {
218
218
  }
219
219
 
220
220
  const buildSchemaHelper = async datasource => {
221
- const Connector = integrations[datasource.source]
221
+ const Connector = await getIntegration(datasource.source)
222
222
 
223
223
  // Connect to the DB and build the schema
224
224
  const connector = new Connector(datasource.config)
@@ -15,6 +15,7 @@ import {
15
15
  getAppId,
16
16
  getAppDB,
17
17
  getProdAppDB,
18
+ getDevAppDB,
18
19
  } from "@budibase/backend-core/context"
19
20
  import { quotas } from "@budibase/pro"
20
21
  import { events } from "@budibase/backend-core"
@@ -110,17 +111,29 @@ async function deployApp(deployment: any) {
110
111
  target: productionAppId,
111
112
  }
112
113
  replication = new Replication(config)
113
-
114
+ const devDb = getDevAppDB()
115
+ console.log("Compacting development DB")
116
+ await devDb.compact()
114
117
  console.log("Replication object created")
115
- await replication.replicate()
118
+ await replication.replicate(replication.appReplicateOpts())
116
119
  console.log("replication complete.. replacing app meta doc")
120
+ // app metadata is excluded as it is likely to be in conflict
121
+ // replicate the app metadata document manually
117
122
  const db = getProdAppDB()
118
- const appDoc = await db.get(DocumentType.APP_METADATA)
123
+ const appDoc = await devDb.get(DocumentType.APP_METADATA)
124
+ try {
125
+ const prodAppDoc = await db.get(DocumentType.APP_METADATA)
126
+ appDoc._rev = prodAppDoc._rev
127
+ } catch (err) {
128
+ delete appDoc._rev
129
+ }
119
130
 
131
+ // switch to production app ID
120
132
  deployment.appUrl = appDoc.url
121
-
122
133
  appDoc.appId = productionAppId
123
134
  appDoc.instance._id = productionAppId
135
+ // remove automation errors if they exist
136
+ delete appDoc.automationErrors
124
137
  await db.put(appDoc)
125
138
  await appCache.invalidateAppMetadata(productionAppId)
126
139
  console.log("New app doc written successfully.")
@@ -1,12 +1,11 @@
1
- const { cloneDeep } = require("lodash")
2
- const { definitions } = require("../../integrations")
1
+ const { getDefinitions } = require("../../integrations")
3
2
  const { SourceName } = require("@budibase/types")
4
3
  const googlesheets = require("../../integrations/googlesheets")
5
4
  const { featureFlags } = require("@budibase/backend-core")
6
5
 
7
6
  exports.fetch = async function (ctx) {
8
7
  ctx.status = 200
9
- const defs = cloneDeep(definitions)
8
+ const defs = await getDefinitions()
10
9
 
11
10
  // for google sheets integration google verification
12
11
  if (featureFlags.isEnabled(featureFlags.FeatureFlag.GOOGLE_SHEETS)) {
@@ -17,6 +16,7 @@ exports.fetch = async function (ctx) {
17
16
  }
18
17
 
19
18
  exports.find = async function (ctx) {
19
+ const defs = await getDefinitions()
20
20
  ctx.status = 200
21
- ctx.body = definitions[ctx.params.type]
21
+ ctx.body = defs[ctx.params.type]
22
22
  }
@@ -0,0 +1,15 @@
1
+ import {
2
+ createTempFolder,
3
+ getPluginMetadata,
4
+ extractTarball,
5
+ } from "../../../utilities/fileSystem"
6
+
7
+ export async function fileUpload(file: { name: string; path: string }) {
8
+ if (!file.name.endsWith(".tar.gz")) {
9
+ throw new Error("Plugin must be compressed into a gzipped tarball.")
10
+ }
11
+ const path = createTempFolder(file.name.split(".tar.gz")[0])
12
+ await extractTarball(file.path, path)
13
+
14
+ return await getPluginMetadata(path)
15
+ }
@@ -0,0 +1,75 @@
1
+ import { getPluginMetadata } from "../../../utilities/fileSystem"
2
+ import fetch from "node-fetch"
3
+ import { downloadUnzipTarball } from "./utils"
4
+
5
+ export async function request(
6
+ url: string,
7
+ headers: { [key: string]: string },
8
+ err: string
9
+ ) {
10
+ const response = await fetch(url, { headers })
11
+ if (response.status >= 300) {
12
+ const respErr = await response.text()
13
+ throw new Error(`Error: ${err} - ${respErr}`)
14
+ }
15
+ return response.json()
16
+ }
17
+
18
+ export async function githubUpload(url: string, name = "", token = "") {
19
+ let githubUrl = url
20
+
21
+ if (!githubUrl.includes("https://github.com/")) {
22
+ throw new Error("The plugin origin must be from Github")
23
+ }
24
+
25
+ if (url.includes(".git")) {
26
+ githubUrl = url.replace(".git", "")
27
+ }
28
+
29
+ const githubApiUrl = githubUrl.replace(
30
+ "https://github.com/",
31
+ "https://api.github.com/repos/"
32
+ )
33
+ const headers: any = token ? { Authorization: `Bearer ${token}` } : {}
34
+ const pluginDetails = await request(
35
+ githubApiUrl,
36
+ headers,
37
+ "Repository not found"
38
+ )
39
+ const pluginName = pluginDetails.name || name
40
+ const pluginLatestReleaseUrl = pluginDetails?.["releases_url"]
41
+ ? pluginDetails?.["releases_url"].replace("{/id}", "/latest")
42
+ : undefined
43
+ if (!pluginLatestReleaseUrl) {
44
+ throw new Error("Github release not found")
45
+ }
46
+
47
+ const pluginReleaseDetails = await request(
48
+ pluginLatestReleaseUrl,
49
+ headers,
50
+ "Github latest release not found"
51
+ )
52
+ const pluginReleaseTarballAsset = pluginReleaseDetails?.assets?.find(
53
+ (x: any) => x?.["content_type"] === "application/gzip"
54
+ )
55
+ const pluginLastReleaseTarballUrl =
56
+ pluginReleaseTarballAsset?.["browser_download_url"]
57
+ if (!pluginLastReleaseTarballUrl) {
58
+ throw new Error("Github latest release url not found")
59
+ }
60
+
61
+ try {
62
+ const path = await downloadUnzipTarball(
63
+ pluginLastReleaseTarballUrl,
64
+ pluginName,
65
+ headers
66
+ )
67
+ return await getPluginMetadata(path)
68
+ } catch (err: any) {
69
+ let errMsg = err?.message || err
70
+ if (errMsg === "unexpected response Not Found") {
71
+ errMsg = "Github release tarball not found"
72
+ }
73
+ throw new Error(errMsg)
74
+ }
75
+ }