@budibase/backend-core 3.2.0 → 3.2.1

package/dist/tests/core/utilities/testContainerUtils.js

@@ -1,11 +1,22 @@
 "use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getContainerByImage = getContainerByImage;
 exports.getContainerById = getContainerById;
 exports.getExposedV4Ports = getExposedV4Ports;
 exports.getExposedV4Port = getExposedV4Port;
 exports.setupEnv = setupEnv;
+exports.startContainer = startContainer;
 const child_process_1 = require("child_process");
+const lodash_1 = require("lodash");
 const IPV4_PORT_REGEX = new RegExp(`0\\.0\\.0\\.0:(\\d+)->(\\d+)/tcp`, "g");
 function getTestcontainers() {
     // We use --format json to make sure the output is nice and machine-readable,
@@ -76,4 +87,56 @@ function setupEnv(...envs) {
         }
     }
 }
+function startContainer(container) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const imageName = container.imageName.string;
+        let key = imageName;
+        if (imageName.includes("@sha256")) {
+            key = imageName.split("@")[0];
+        }
+        key = key.replace(/\//g, "-").replace(/:/g, "-");
+        container = container
+            .withReuse()
+            .withLabels({ "com.budibase": "true" })
+            .withName(`${key}_testcontainer`);
+        let startedContainer = undefined;
+        let lastError = undefined;
+        for (let i = 0; i < 10; i++) {
+            try {
+                // container.start() is not an idempotent operation, calling `start`
+                // modifies the internal state of a GenericContainer instance such that
+                // the hash it uses to determine reuse changes. We need to clone the
+                // container before calling start to ensure that we're using the same
+                // reuse hash every time.
+                const containerCopy = (0, lodash_1.cloneDeep)(container);
+                startedContainer = yield containerCopy.start();
+                lastError = undefined;
+                break;
+            }
+            catch (e) {
+                lastError = e;
+                yield new Promise(resolve => setTimeout(resolve, 1000));
+            }
+        }
+        if (!startedContainer) {
+            if (lastError) {
+                throw lastError;
+            }
+            throw new Error(`failed to start container: ${imageName}`);
+        }
+        const info = getContainerById(startedContainer.getId());
+        if (!info) {
+            throw new Error("Container not found");
+        }
+        // Some Docker runtimes, when you expose a port, will bind it to both
+        // 127.0.0.1 and ::1, so ipv4 and ipv6. The port spaces of ipv4 and ipv6
+        // addresses are not shared, and testcontainers will sometimes give you back
+        // the ipv6 port. There's no way to know that this has happened, and if you
+        // try to then connect to `localhost:port` you may attempt to bind to the v4
+        // address which could be unbound or even an entirely different container. For
+        // that reason, we don't use testcontainers' `getExposedPort` function,
+        // preferring instead our own method that guaranteed v4 ports.
+        return getExposedV4Ports(info);
+    });
+}
 //# sourceMappingURL=testContainerUtils.js.map

package/dist/tests/core/utilities/testContainerUtils.js.map

@@ -1 +1 @@
-{"version":3,"file":"testContainerUtils.js","sourceRoot":"","sources":["../../../../tests/core/utilities/testContainerUtils.ts"],"names":[],"mappings":";;AAqCA,kDAUC;AAED,4CAEC;AAOD,8CAMC;AAED,4CAEC;AAED,4BAqCC;AA3GD,iDAAwC;AAExC,MAAM,eAAe,GAAG,IAAI,MAAM,CAAC,kCAAkC,EAAE,GAAG,CAAC,CAAA;AAmB3E,SAAS,iBAAiB;IACxB,6EAA6E;IAC7E,6EAA6E;IAC7E,gCAAgC;IAChC,OAAO,IAAA,wBAAQ,EAAC,oCAAoC,CAAC;SAClD,QAAQ,EAAE;SACV,KAAK,CAAC,IAAI,CAAC;SACX,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;SACzB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAkB,CAAC;SACxC,MAAM,CACL,CAAC,CAAC,EAAE,CACF,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAAC;QAC5C,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CACzC,CAAA;AACL,CAAC;AAED,SAAgB,mBAAmB,CAAC,KAAa;IAC/C,MAAM,UAAU,GAAG,iBAAiB,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAA;IAC7E,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,IAAI,YAAY,GAAG,mDAAmD,KAAK,OAAO,CAAA;QAClF,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,YAAY,IAAI,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAA;QACpD,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,CAAA;IAC/B,CAAC;IACD,OAAO,UAAU,CAAC,CAAC,CAAC,CAAA;AACtB,CAAC;AAED,SAAgB,gBAAgB,CAAC,EAAU;IACzC,OAAO,iBAAiB,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAA;AACnD,CAAC;AAOD,SAAgB,iBAAiB,CAAC,SAAwB;IACxD,IAAI,KAAK,GAAW,EAAE,CAAA;IACtB,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;QAC9D,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IACzE,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAgB,gBAAgB,CAAC,SAAwB,EAAE,IAAY;;IACrE,OAAO,MAAA,iBAAiB,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,IAAI,CAAC,0CAAE,IAAI,CAAA;AAC3E,CAAC;AAED,SAAgB,QAAQ,CAAC,GAAG,IAAW;IACrC,yEAAyE;IACzE,8DAA8D;IAC9D,MAAM,KAAK,GAAG,mBAAmB,CAAC,kBAAkB,CAAC,CAAA;IACrD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAA;IAChD,CAAC;IAED,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;IAC/C,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAA;IAC3C,CAAC;IAED,MAAM,YAAY,GAAG,gBAAgB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;IAClD,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAA;IAC/C,CAAC;IAED,MAAM,KAAK,GAAG,mBAAmB,CAAC,aAAa,CAAC,CAAA;IAEhD,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;IAC/C,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAA;IACzC,CAAC;IAED,MAAM,OAAO,GAAG;QACd,EAAE,GAAG,EAAE,eAAe,EAAE,KAAK,EAAE,GAAG,SAAS,EAAE,EAAE;QAC/C,EAAE,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,oBAAoB,SAAS,EAAE,EAAE;QAC/D,EAAE,GAAG,EAAE,kBAAkB,EAAE,KAAK,EAAE,oBAAoB,YAAY,EAAE,EAAE;QACtE,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,oBAAoB,SAAS,EAAE,EAAE;KAC7D,CAAA;IAED,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;QACpD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,CAAA;QACpC,CAAC;IACH,CAAC;AACH,CAAC"}
+{"version":3,"file":"testContainerUtils.js","sourceRoot":"","sources":["../../../../tests/core/utilities/testContainerUtils.ts"],"names":[],"mappings":";;;;;;;;;;;AAuCA,kDAUC;AAED,4CAEC;AAOD,8CAMC;AAED,4CAEC;AAED,4BAqCC;AAED,wCAqDC;AApKD,iDAAwC;AACxC,mCAAkC;AAGlC,MAAM,eAAe,GAAG,IAAI,MAAM,CAAC,kCAAkC,EAAE,GAAG,CAAC,CAAA;AAmB3E,SAAS,iBAAiB;IACxB,6EAA6E;IAC7E,6EAA6E;IAC7E,gCAAgC;IAChC,OAAO,IAAA,wBAAQ,EAAC,oCAAoC,CAAC;SAClD,QAAQ,EAAE;SACV,KAAK,CAAC,IAAI,CAAC;SACX,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;SACzB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAkB,CAAC;SACxC,MAAM,CACL,CAAC,CAAC,EAAE,CACF,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAAC;QAC5C,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CACzC,CAAA;AACL,CAAC;AAED,SAAgB,mBAAmB,CAAC,KAAa;IAC/C,MAAM,UAAU,GAAG,iBAAiB,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAA;IAC7E,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,IAAI,YAAY,GAAG,mDAAmD,KAAK,OAAO,CAAA;QAClF,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,YAAY,IAAI,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAA;QACpD,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,CAAA;IAC/B,CAAC;IACD,OAAO,UAAU,CAAC,CAAC,CAAC,CAAA;AACtB,CAAC;AAED,SAAgB,gBAAgB,CAAC,EAAU;IACzC,OAAO,iBAAiB,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAA;AACnD,CAAC;AAOD,SAAgB,iBAAiB,CAAC,SAAwB;IACxD,IAAI,KAAK,GAAW,EAAE,CAAA;IACtB,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;QAC9D,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IACzE,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAgB,gBAAgB,CAAC,SAAwB,EAAE,IAAY;;IACrE,OAAO,MAAA,iBAAiB,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,IAAI,CAAC,0CAAE,IAAI,CAAA;AAC3E,CAAC;AAED,SAAgB,QAAQ,CAAC,GAAG,IAAW;IACrC,yEAAyE;IACzE,8DAA8D;IAC9D,MAAM,KAAK,GAAG,mBAAmB,CAAC,kBAAkB,CAAC,CAAA;IACrD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAA;IAChD,CAAC;IAED,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;IAC/C,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAA;IAC3C,CAAC;IAED,MAAM,YAAY,GAAG,gBAAgB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;IAClD,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAA;IAC/C,CAAC;IAED,MAAM,KAAK,GAAG,mBAAmB,CAAC,aAAa,CAAC,CAAA;IAEhD,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;IAC/C,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAA;IACzC,CAAC;IAED,MAAM,OAAO,GAAG;QACd,EAAE,GAAG,EAAE,eAAe,EAAE,KAAK,EAAE,GAAG,SAAS,EAAE,EAAE;QAC/C,EAAE,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,oBAAoB,SAAS,EAAE,EAAE;QAC/D,EAAE,GAAG,EAAE,kBAAkB,EAAE,KAAK,EAAE,oBAAoB,YAAY,EAAE,EAAE;QACtE,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,oBAAoB,SAAS,EAAE,EAAE;KAC7D,CAAA;IAED,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;QACpD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,CAAA;QACpC,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAsB,cAAc,CAAC,SAA2B;;QAC9D,MAAM,SAAS,GAAI,SAAiB,CAAC,SAAS,CAAC,MAAgB,CAAA;QAC/D,IAAI,GAAG,GAAW,SAAS,CAAA;QAC3B,IAAI,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YAClC,GAAG,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;QAC/B,CAAC;QACD,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;QAEhD,SAAS,GAAG,SAAS;aAClB,SAAS,EAAE;aACX,UAAU,CAAC,EAAE,cAAc,EAAE,MAAM,EAAE,CAAC;aACtC,QAAQ,CAAC,GAAG,GAAG,gBAAgB,CAAC,CAAA;QAEnC,IAAI,gBAAgB,GAAqC,SAAS,CAAA;QAClE,IAAI,SAAS,GAAG,SAAS,CAAA;QACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,IAAI,CAAC;gBACH,oEAAoE;gBACpE,uEAAuE;gBACvE,oEAAoE;gBACpE,qEAAqE;gBACrE,yBAAyB;gBACzB,MAAM,aAAa,GAAG,IAAA,kBAAS,EAAC,SAAS,CAAC,CAAA;gBAC1C,gBAAgB,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,CAAA;gBAC9C,SAAS,GAAG,SAAS,CAAA;gBACrB,MAAK;YACP,CAAC;YAAC,OAAO,CAAM,EAAE,CAAC;gBAChB,SAAS,GAAG,CAAC,CAAA;gBACb,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAA;YACzD,CAAC;QACH,CAAC;QAED,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACtB,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,SAAS,CAAA;YACjB,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,8BAA8B,SAAS,EAAE,CAAC,CAAA;QAC5D,CAAC;QAED,MAAM,IAAI,GAAG,gBAAgB,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC,CAAA;QACvD,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAA;QACxC,CAAC;QAED,qEAAqE;QACrE,wEAAwE;QACxE,4EAA4E;QAC5E,2EAA2E;QAC3E,4EAA4E;QAC5E,8EAA8E;QAC9E,uEAAuE;QACvE,8DAA8D;QAC9D,OAAO,iBAAiB,CAAC,IAAI,CAAC,CAAA;IAChC,CAAC;CAAA"}

package/dist/tests/core/utilities/utils/index.d.ts

@@ -1 +1,2 @@
 export * as time from "./time";
+export * as queue from "./queue";

package/dist/tests/core/utilities/utils/index.js

@@ -23,6 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.time = void 0;
+exports.queue = exports.time = void 0;
 exports.time = __importStar(require("./time"));
+exports.queue = __importStar(require("./queue"));
 //# sourceMappingURL=index.js.map

package/dist/tests/core/utilities/utils/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../tests/core/utilities/utils/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,+CAA8B"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../tests/core/utilities/utils/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,+CAA8B;AAC9B,iDAAgC"}

package/dist/tests/core/utilities/utils/queue.d.ts

@@ -0,0 +1,3 @@
+import { Queue } from "bull";
+export declare function useRealQueues(): Promise<void>;
+export declare function processMessages(queue: Queue): Promise<void>;

package/dist/tests/core/utilities/utils/queue.js

@@ -0,0 +1,37 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.useRealQueues = useRealQueues;
+exports.processMessages = processMessages;
+const testcontainers_1 = require("testcontainers");
+const testContainerUtils_1 = require("../testContainerUtils");
+function useRealQueues() {
+    return __awaiter(this, void 0, void 0, function* () {
+        var _a;
+        const ports = yield (0, testContainerUtils_1.startContainer)(new testcontainers_1.GenericContainer("redis")
+            .withExposedPorts(6379)
+            .withWaitStrategy(testcontainers_1.Wait.forSuccessfulCommand(`redis-cli`).withStartupTimeout(10000)));
+        const port = (_a = ports.find(x => x.container === 6379)) === null || _a === void 0 ? void 0 : _a.host;
+        if (!port) {
+            throw new Error("Redis port not found");
+        }
+        process.env.BULL_TEST_REDIS_PORT = port.toString();
+    });
+}
+function processMessages(queue) {
+    return __awaiter(this, void 0, void 0, function* () {
+        do {
+            yield queue.whenCurrentJobsFinished();
+        } while (yield queue.count());
+        yield queue.whenCurrentJobsFinished();
+    });
+}
+//# sourceMappingURL=queue.js.map

package/dist/tests/core/utilities/utils/queue.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"queue.js","sourceRoot":"","sources":["../../../../../tests/core/utilities/utils/queue.ts"],"names":[],"mappings":";;;;;;;;;;;AAIA,sCAcC;AAED,0CAMC;AAzBD,mDAAuD;AACvD,8DAAsD;AAEtD,SAAsB,aAAa;;;QACjC,MAAM,KAAK,GAAG,MAAM,IAAA,mCAAc,EAChC,IAAI,iCAAgB,CAAC,OAAO,CAAC;aAC1B,gBAAgB,CAAC,IAAI,CAAC;aACtB,gBAAgB,CACf,qBAAI,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,kBAAkB,CAAC,KAAK,CAAC,CACjE,CACJ,CAAA;QAED,MAAM,IAAI,GAAG,MAAA,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,IAAI,CAAC,0CAAE,IAAI,CAAA;QACxD,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAA;QACzC,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAA;IACpD,CAAC;CAAA;AAED,SAAsB,eAAe,CAAC,KAAY;;QAChD,GAAG,CAAC;YACF,MAAM,KAAK,CAAC,uBAAuB,EAAE,CAAA;QACvC,CAAC,QAAQ,MAAM,KAAK,CAAC,KAAK,EAAE,EAAC;QAE7B,MAAM,KAAK,CAAC,uBAAuB,EAAE,CAAA;IACvC,CAAC;CAAA"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "3.2.0",
+  "version": "3.2.1",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/index.js",
   "types": "dist/src/index.d.ts",
@@ -23,8 +23,8 @@
   "dependencies": {
     "@budibase/nano": "10.1.5",
     "@budibase/pouchdb-replication-stream": "1.2.11",
-    "@budibase/shared-core": "3.2.0",
-    "@budibase/types": "3.2.0",
+    "@budibase/shared-core": "3.2.1",
+    "@budibase/types": "3.2.1",
     "aws-cloudfront-sign": "3.0.2",
     "aws-sdk": "2.1030.0",
     "bcrypt": "5.1.0",
@@ -95,5 +95,5 @@
       }
     }
   },
-  "gitHead": "03c8ffc9fe27435014a76bb4becd81278ad1ed9c"
+  "gitHead": "46f5cd0a5d1c053ce787de66825dce0c03d5e559"
 }

package/src/cache/tests/docWritethrough.spec.ts

@@ -1,7 +1,12 @@
 import tk from "timekeeper"
 
 import _ from "lodash"
-import { DBTestConfiguration, generator, structures } from "../../../tests"
+import {
+  DBTestConfiguration,
+  generator,
+  structures,
+  utils,
+} from "../../../tests"
 import { getDB } from "../../db"
 
 import {
@@ -10,15 +15,14 @@ import {
   init,
 } from "../docWritethrough"
 
-import InMemoryQueue from "../../queue/inMemoryQueue"
-
 const initialTime = Date.now()
 
 async function waitForQueueCompletion() {
-  const queue: InMemoryQueue = DocWritethroughProcessor.queue as never
-  await queue.waitForCompletion()
+  await utils.queue.processMessages(DocWritethroughProcessor.queue)
 }
 
+beforeAll(() => utils.queue.useRealQueues())
+
 describe("docWritethrough", () => {
   beforeAll(() => {
     init()
@@ -67,7 +71,7 @@ describe("docWritethrough", () => {
         const patch3 = generatePatchObject(3)
         await docWritethrough.patch(patch3)
 
-        expect(await db.get(documentId)).toEqual({
+        expect(await db.tryGet(documentId)).toEqual({
           _id: documentId,
           ...patch1,
           ...patch2,
@@ -92,7 +96,7 @@ describe("docWritethrough", () => {
 
         await waitForQueueCompletion()
 
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining({
             _id: documentId,
             ...patch1,
@@ -117,7 +121,7 @@ describe("docWritethrough", () => {
         await waitForQueueCompletion()
 
         expect(date1).not.toEqual(date2)
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining({
             createdAt: date1.toISOString(),
             updatedAt: date2.toISOString(),
@@ -135,7 +139,7 @@ describe("docWritethrough", () => {
         await docWritethrough.patch(patch2)
 
         const keyToOverride = _.sample(Object.keys(patch1))!
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining({
             [keyToOverride]: patch1[keyToOverride],
           })
@@ -150,7 +154,7 @@ describe("docWritethrough", () => {
         await docWritethrough.patch(patch3)
         await waitForQueueCompletion()
 
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining({
             ...patch1,
             ...patch2,
@@ -180,14 +184,14 @@ describe("docWritethrough", () => {
         await secondDocWritethrough.patch(doc2Patch2)
         await waitForQueueCompletion()
 
-        expect(await db.get(docWritethrough.docId)).toEqual(
+        expect(await db.tryGet(docWritethrough.docId)).toEqual(
           expect.objectContaining({
             ...doc1Patch,
             ...doc1Patch2,
           })
         )
 
-        expect(await db.get(secondDocWritethrough.docId)).toEqual(
+        expect(await db.tryGet(secondDocWritethrough.docId)).toEqual(
           expect.objectContaining({
             ...doc2Patch,
             ...doc2Patch2,
@@ -203,7 +207,7 @@ describe("docWritethrough", () => {
         await docWritethrough.patch(initialPatch)
         await waitForQueueCompletion()
 
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining(initialPatch)
         )
 
@@ -214,10 +218,10 @@ describe("docWritethrough", () => {
         await docWritethrough.patch(extraPatch)
         await waitForQueueCompletion()
 
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining(extraPatch)
         )
-        expect(await db.get(documentId)).not.toEqual(
+        expect(await db.tryGet(documentId)).not.toEqual(
           expect.objectContaining(initialPatch)
         )
       })
@@ -242,7 +246,7 @@ describe("docWritethrough", () => {
         expect(queueMessageSpy).toHaveBeenCalledTimes(5)
 
         await waitForQueueCompletion()
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining(patches)
         )
 
@@ -250,7 +254,7 @@ describe("docWritethrough", () => {
         expect(queueMessageSpy).toHaveBeenCalledTimes(45)
 
         await waitForQueueCompletion()
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining(patches)
         )
 
@@ -258,20 +262,18 @@ describe("docWritethrough", () => {
         expect(queueMessageSpy).toHaveBeenCalledTimes(55)
 
         await waitForQueueCompletion()
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining(patches)
         )
       })
     })
 
-    // This is not yet supported
-    // eslint-disable-next-line jest/no-disabled-tests
-    it.skip("patches will execute in order", async () => {
+    it("patches will execute in order", async () => {
       let incrementalValue = 0
       const keyToOverride = generator.word()
       async function incrementalPatches(count: number) {
         for (let i = 0; i < count; i++) {
-          await docWritethrough.patch({ [keyToOverride]: incrementalValue++ })
+          await docWritethrough.patch({ [keyToOverride]: ++incrementalValue })
         }
       }
 
@@ -279,13 +281,13 @@ describe("docWritethrough", () => {
         await incrementalPatches(5)
 
         await waitForQueueCompletion()
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining({ [keyToOverride]: 5 })
         )
 
         await incrementalPatches(40)
         await waitForQueueCompletion()
-        expect(await db.get(documentId)).toEqual(
+        expect(await db.tryGet(documentId)).toEqual(
           expect.objectContaining({ [keyToOverride]: 45 })
         )
       })

package/src/queue/inMemoryQueue.ts

@@ -1,5 +1,5 @@
 import events from "events"
-import { newid, timeout } from "../utils"
+import { newid } from "../utils"
 import { Queue, QueueOptions, JobOptions } from "./queue"
 
 interface JobMessage {
@@ -141,7 +141,7 @@ class InMemoryQueue implements Partial<Queue> {
     } else {
       pushMessage()
     }
-    return {} as any
+    return { id: jobId } as any
   }
 
   /**
@@ -184,16 +184,6 @@ class InMemoryQueue implements Partial<Queue> {
     // do nothing
     return this as any
   }
-
-  async waitForCompletion() {
-    do {
-      await timeout(50)
-    } while (this.hasRunningJobs())
-  }
-
-  hasRunningJobs() {
-    return this._addCount > this._runCount
-  }
 }
 
 export default InMemoryQueue

package/src/queue/queue.ts

@@ -15,7 +15,7 @@ const QUEUE_LOCK_MS = Duration.fromMinutes(5).toMs()
 const QUEUE_LOCK_RENEW_INTERNAL_MS = Duration.fromSeconds(30).toMs()
 // cleanup the queue every 60 seconds
 const CLEANUP_PERIOD_MS = Duration.fromSeconds(60).toMs()
-let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = []
+let QUEUES: BullQueue.Queue[] = []
 let cleanupInterval: NodeJS.Timeout
 
 async function cleanup() {
@@ -45,11 +45,18 @@ export function createQueue<T>(
   if (opts.jobOptions) {
     queueConfig.defaultJobOptions = opts.jobOptions
   }
-  let queue: any
+  let queue: BullQueue.Queue<T>
   if (!env.isTest()) {
     queue = new BullQueue(jobQueue, queueConfig)
+  } else if (
+    process.env.BULL_TEST_REDIS_PORT &&
+    !isNaN(+process.env.BULL_TEST_REDIS_PORT)
+  ) {
+    queue = new BullQueue(jobQueue, {
+      redis: { host: "localhost", port: +process.env.BULL_TEST_REDIS_PORT },
+    })
   } else {
-    queue = new InMemoryQueue(jobQueue, queueConfig)
+    queue = new InMemoryQueue(jobQueue, queueConfig) as any
   }
   addListeners(queue, jobQueue, opts?.removeStalledCb)
   QUEUES.push(queue)

package/tests/core/utilities/index.ts

@@ -4,3 +4,4 @@ export { generator } from "./structures"
 export * as testContainerUtils from "./testContainerUtils"
 export * as utils from "./utils"
 export * from "./jestUtils"
+export * as queue from "./queue"

package/tests/core/utilities/queue.ts

@@ -0,0 +1,9 @@
+import { Queue } from "bull"
+
+export async function processMessages(queue: Queue) {
+  do {
+    await queue.whenCurrentJobsFinished()
+  } while (await queue.count())
+
+  await queue.whenCurrentJobsFinished()
+}

package/tests/core/utilities/testContainerUtils.ts

@@ -1,4 +1,6 @@
 import { execSync } from "child_process"
+import { cloneDeep } from "lodash"
+import { GenericContainer, StartedTestContainer } from "testcontainers"
 
 const IPV4_PORT_REGEX = new RegExp(`0\\.0\\.0\\.0:(\\d+)->(\\d+)/tcp`, "g")
 
@@ -106,3 +108,58 @@ export function setupEnv(...envs: any[]) {
     }
   }
 }
+
+export async function startContainer(container: GenericContainer) {
+  const imageName = (container as any).imageName.string as string
+  let key: string = imageName
+  if (imageName.includes("@sha256")) {
+    key = imageName.split("@")[0]
+  }
+  key = key.replace(/\//g, "-").replace(/:/g, "-")
+
+  container = container
+    .withReuse()
+    .withLabels({ "com.budibase": "true" })
+    .withName(`${key}_testcontainer`)
+
+  let startedContainer: StartedTestContainer | undefined = undefined
+  let lastError = undefined
+  for (let i = 0; i < 10; i++) {
+    try {
+      // container.start() is not an idempotent operation, calling `start`
+      // modifies the internal state of a GenericContainer instance such that
+      // the hash it uses to determine reuse changes. We need to clone the
+      // container before calling start to ensure that we're using the same
+      // reuse hash every time.
+      const containerCopy = cloneDeep(container)
+      startedContainer = await containerCopy.start()
+      lastError = undefined
+      break
+    } catch (e: any) {
+      lastError = e
+      await new Promise(resolve => setTimeout(resolve, 1000))
+    }
+  }
+
+  if (!startedContainer) {
+    if (lastError) {
+      throw lastError
+    }
+    throw new Error(`failed to start container: ${imageName}`)
+  }
+
+  const info = getContainerById(startedContainer.getId())
+  if (!info) {
+    throw new Error("Container not found")
+  }
+
+  // Some Docker runtimes, when you expose a port, will bind it to both
+  // 127.0.0.1 and ::1, so ipv4 and ipv6. The port spaces of ipv4 and ipv6
+  // addresses are not shared, and testcontainers will sometimes give you back
+  // the ipv6 port. There's no way to know that this has happened, and if you
+  // try to then connect to `localhost:port` you may attempt to bind to the v4
+  // address which could be unbound or even an entirely different container. For
+  // that reason, we don't use testcontainers' `getExposedPort` function,
+  // preferring instead our own method that guaranteed v4 ports.
+  return getExposedV4Ports(info)
+}

package/tests/core/utilities/utils/index.ts

@@ -1 +1,2 @@
 export * as time from "./time"
+export * as queue from "./queue"

package/tests/core/utilities/utils/queue.ts

@@ -0,0 +1,27 @@
+import { Queue } from "bull"
+import { GenericContainer, Wait } from "testcontainers"
+import { startContainer } from "../testContainerUtils"
+
+export async function useRealQueues() {
+  const ports = await startContainer(
+    new GenericContainer("redis")
+      .withExposedPorts(6379)
+      .withWaitStrategy(
+        Wait.forSuccessfulCommand(`redis-cli`).withStartupTimeout(10000)
+      )
+  )
+
+  const port = ports.find(x => x.container === 6379)?.host
+  if (!port) {
+    throw new Error("Redis port not found")
+  }
+  process.env.BULL_TEST_REDIS_PORT = port.toString()
+}
+
+export async function processMessages(queue: Queue) {
+  do {
+    await queue.whenCurrentJobsFinished()
+  } while (await queue.count())
+
+  await queue.whenCurrentJobsFinished()
+}