@budibase/backend-core 2.21.4 → 2.21.5

package/dist/index.js

@@ -19623,11 +19623,11 @@ var require_nacl_fast = __commonJS({
     "use strict";
     (function(nacl) {
       "use strict";
-      var gf = function(init9) {
+      var gf = function(init10) {
         var i, r = new Float64Array(16);
-        if (init9)
-          for (i = 0; i < init9.length; i++)
-            r[i] = init9[i];
+        if (init10)
+          for (i = 0; i < init10.length; i++)
+            r[i] = init10[i];
         return r;
       };
       var randombytes = function() {
@@ -54160,7 +54160,7 @@ __export(src_exports, {
   featureFlags: () => features_exports,
   features: () => installation_exports2,
   getPublicError: () => getPublicError,
-  init: () => init8,
+  init: () => init9,
   installation: () => installation_exports,
   locks: () => redlockImpl_exports,
   logging: () => logging_exports,
@@ -54579,6 +54579,7 @@ var DocumentType = /* @__PURE__ */ ((DocumentType2) => {
   DocumentType2["AUTOMATION_METADATA"] = "meta_au";
   DocumentType2["AUDIT_LOG"] = "al";
   DocumentType2["APP_MIGRATION_METADATA"] = "_design/migrations";
+  DocumentType2["SCIM_LOG"] = "scimlog";
   return DocumentType2;
 })(DocumentType || {});
 var InternalTable = /* @__PURE__ */ ((InternalTable2) => {
@@ -54682,6 +54683,9 @@ var StaticDatabases = {
   },
   AUDIT_LOGS: {
     name: "audit-logs"
+  },
+  SCIM_LOGS: {
+    name: "scim-logs"
   }
 };
 var APP_PREFIX = prefixed("app" /* APP */);
@@ -55062,6 +55066,7 @@ __export(cache_exports, {
   app: () => appMetadata_exports,
   bustCache: () => bustCache,
   destroy: () => destroy,
+  docWritethrough: () => docWritethrough_exports,
   generic: () => generic_exports,
   get: () => get2,
   invite: () => invite_exports,
@@ -55113,6 +55118,7 @@ __export(context_exports, {
   getPlatformURL: () => getPlatformURL,
   getProdAppDB: () => getProdAppDB,
   getProdAppId: () => getProdAppId,
+  getScimDBName: () => getScimDBName,
   getTenantIDFromAppID: () => getTenantIDFromAppID,
   getTenantId: () => getTenantId,
   identity: () => identity_exports,
@@ -55318,7 +55324,8 @@ var environment = {
     process.env[key] = value;
     environment[key] = value;
   },
-  ROLLING_LOG_MAX_SIZE: process.env.ROLLING_LOG_MAX_SIZE || "10M"
+  ROLLING_LOG_MAX_SIZE: process.env.ROLLING_LOG_MAX_SIZE || "10M",
+  DISABLE_SCIM_CALLS: process.env.DISABLE_SCIM_CALLS
 };
 for (let [key, value] of Object.entries(environment)) {
   if (value === "0") {
@@ -55585,9 +55592,12 @@ var DDInstrumentedDatabase = class {
   get name() {
     return this.db.name;
   }
-  exists() {
+  exists(docId) {
     return import_dd_trace.default.trace("db.exists", (span) => {
-      span?.addTags({ db_name: this.name });
+      span?.addTags({ db_name: this.name, doc_id: docId });
+      if (docId) {
+        return this.db.exists(docId);
+      }
       return this.db.exists();
     });
   }
@@ -55715,7 +55725,13 @@ var DatabaseImpl = class _DatabaseImpl {
     const couchInfo = getCouchInfo();
     _DatabaseImpl.nano = buildNano(couchInfo);
   }
-  async exists() {
+  exists(docId) {
+    if (docId === void 0) {
+      return this.dbExists();
+    }
+    return this.docExists(docId);
+  }
+  async dbExists() {
     const response = await directCouchUrlCall({
       url: `${this.couchInfo.url}/${this.name}`,
       method: "HEAD",
@@ -55723,6 +55739,14 @@ var DatabaseImpl = class _DatabaseImpl {
     });
     return response.status === 200;
   }
+  async docExists(id) {
+    try {
+      await this.performCall((db) => () => db.head(id));
+      return true;
+    } catch {
+      return false;
+    }
+  }
   nano() {
     return this.instanceNano || _DatabaseImpl.nano;
   }
@@ -55946,6 +55970,16 @@ function getAuditLogDBName(tenantId) {
     return `${tenantId}${SEPARATOR}${StaticDatabases.AUDIT_LOGS.name}`;
   }
 }
+function getScimDBName(tenantId) {
+  if (!tenantId) {
+    tenantId = getTenantId();
+  }
+  if (tenantId === DEFAULT_TENANT_ID) {
+    return StaticDatabases.SCIM_LOGS.name;
+  } else {
+    return `${tenantId}${SEPARATOR}${StaticDatabases.SCIM_LOGS.name}`;
+  }
+}
 function baseGlobalDBName(tenantId) {
   if (!tenantId || tenantId === DEFAULT_TENANT_ID) {
     return StaticDatabases.GLOBAL.name;
@@ -56182,6 +56216,7 @@ var init_exports = {};
 __export(init_exports, {
   getAppClient: () => getAppClient,
   getCacheClient: () => getCacheClient,
+  getDocWritethroughClient: () => getDocWritethroughClient,
   getInviteClient: () => getInviteClient,
   getLockClient: () => getLockClient,
   getPasswordResetClient: () => getPasswordResetClient,
@@ -56227,6 +56262,7 @@ var Databases = /* @__PURE__ */ ((Databases2) => {
   Databases2["LOCKS"] = "locks";
   Databases2["SOCKET_IO"] = "socket_io";
   Databases2["BPM_EVENTS"] = "bpmEvents";
+  Databases2["DOC_WRITE_THROUGH"] = "docWriteThrough";
   return Databases2;
 })(Databases || {});
 var SelectableDatabase = /* @__PURE__ */ ((SelectableDatabase2) => {
@@ -58755,6 +58791,10 @@ var RedisWrapper = class {
     const db = this._db;
     await this.getClient().del(addDbPrefix(db, key));
   }
+  async bulkDelete(keys2) {
+    const db = this._db;
+    await this.getClient().del(keys2.map((key) => addDbPrefix(db, key)));
+  }
   async clear() {
     let items = await this.scan();
     await Promise.all(items.map((obj) => this.delete(obj.key)));
@@ -58788,6 +58828,7 @@ var lockClient;
 var socketClient;
 var inviteClient;
 var passwordResetClient;
+var docWritethroughClient;
 async function init3() {
   userClient = await new redis_default("users" /* USER_CACHE */).init();
   sessionClient = await new redis_default("session" /* SESSIONS */).init();
@@ -58801,6 +58842,9 @@ async function init3() {
     "socket_io" /* SOCKET_IO */,
     1 /* SOCKET_IO */
   ).init();
+  docWritethroughClient = await new redis_default(
+    "docWriteThrough" /* DOC_WRITE_THROUGH */
+  ).init();
 }
 async function shutdown() {
   if (userClient)
@@ -58879,6 +58923,12 @@ async function getPasswordResetClient() {
   }
   return passwordResetClient;
 }
+async function getDocWritethroughClient() {
+  if (!writethroughClient) {
+    await init3();
+  }
+  return writethroughClient;
+}
 
 // src/cache/base/index.ts
 function generateTenantKey(key) {
@@ -58896,6 +58946,16 @@ var BaseCache = class {
     const client = await this.getClient();
     return client.keys(pattern);
   }
+  async exists(key, opts = { useTenancy: true }) {
+    key = opts.useTenancy ? generateTenantKey(key) : key;
+    const client = await this.getClient();
+    return client.exists(key);
+  }
+  async scan(key, opts = { useTenancy: true }) {
+    key = opts.useTenancy ? generateTenantKey(key) : key;
+    const client = await this.getClient();
+    return client.scan(key);
+  }
   /**
    * Read only from the cache.
    */
@@ -58904,6 +58964,14 @@ var BaseCache = class {
     const client = await this.getClient();
     return client.get(key);
   }
+  /**
+   * Read only from the cache.
+   */
+  async bulkGet(keys2, opts = { useTenancy: true }) {
+    keys2 = opts.useTenancy ? keys2.map((key) => generateTenantKey(key)) : keys2;
+    const client = await this.getClient();
+    return client.bulkGet(keys2);
+  }
   /**
    * Write to the cache.
    */
@@ -58912,6 +58980,19 @@ var BaseCache = class {
     const client = await this.getClient();
     await client.store(key, value, ttl);
   }
+  /**
+   * Bulk write to the cache.
+   */
+  async bulkStore(data, ttl = null, opts = { useTenancy: true }) {
+    if (opts.useTenancy) {
+      data = Object.entries(data).reduce((acc, [key, value]) => {
+        acc[generateTenantKey(key)] = value;
+        return acc;
+      }, {});
+    }
+    const client = await this.getClient();
+    await client.bulkStore(data, ttl);
+  }
   /**
    * Remove from cache.
    */
@@ -58920,10 +59001,18 @@ var BaseCache = class {
     const client = await this.getClient();
     return client.delete(key);
   }
+  /**
+   * Remove from cache.
+   */
+  async bulkDelete(keys2, opts = { useTenancy: true }) {
+    keys2 = opts.useTenancy ? keys2.map((key) => generateTenantKey(key)) : keys2;
+    const client = await this.getClient();
+    return client.bulkDelete(keys2);
+  }
   /**
    * Read from the cache. Write to the cache if not exists.
    */
-  async withCache(key, ttl, fetchFn, opts = { useTenancy: true }) {
+  async withCache(key, ttl = null, fetchFn, opts = { useTenancy: true }) {
     const cachedValue = await this.get(key, opts);
     if (cachedValue) {
       return cachedValue;
@@ -58946,6 +59035,14 @@ var BaseCache = class {
       throw err;
     }
   }
+  /**
+   * Delete the entry if the provided value matches the stored one.
+   */
+  async deleteIfValue(key, value, opts = { useTenancy: true }) {
+    key = opts.useTenancy ? generateTenantKey(key) : key;
+    const client = await this.getClient();
+    await client.deleteIfValue(key, value);
+  }
 };
 
 // src/cache/generic.ts
@@ -60795,19 +60892,23 @@ var LoggingProcessor = class {
 // src/queue/index.ts
 var queue_exports = {};
 __export(queue_exports, {
+  JobOptions: () => import_bull2.JobOptions,
   JobQueue: () => JobQueue,
+  Queue: () => import_bull2.Queue,
+  QueueOptions: () => import_bull2.QueueOptions,
   createQueue: () => createQueue,
   shutdown: () => shutdown2
 });
 
 // src/queue/inMemoryQueue.ts
 var import_events2 = __toESM(require("events"));
-function newJob(queue, message) {
+function newJob(queue, message, opts) {
   return {
+    id: newid(),
     timestamp: Date.now(),
     queue,
     data: message,
-    opts: {}
+    opts
   };
 }
 var InMemoryQueue = class {
@@ -60824,6 +60925,7 @@ var InMemoryQueue = class {
     this._emitter = new import_events2.default.EventEmitter();
     this._runCount = 0;
     this._addCount = 0;
+    this._queuedJobIds = /* @__PURE__ */ new Set();
   }
   /**
    * Same callback API as Bull, each callback passed to this will consume messages as they are
@@ -60834,21 +60936,37 @@ var InMemoryQueue = class {
    * note this is incredibly limited compared to Bull as in reality the Job would contain
    * a lot more information about the queue and current status of Bull cluster.
    */
-  process(func) {
+  async process(func) {
     this._emitter.on("message", async () => {
       if (this._messages.length <= 0) {
         return;
       }
       let msg = this._messages.shift();
       let resp = func(msg);
+      async function retryFunc(fnc) {
+        try {
+          await fnc;
+        } catch (e) {
+          await new Promise((r) => setTimeout(() => r(), 50));
+          await retryFunc(func(msg));
+        }
+      }
       if (resp.then != null) {
-        await resp;
+        try {
+          await retryFunc(resp);
+        } catch (e) {
+          console.error(e);
+        }
       }
       this._runCount++;
+      const jobId = msg?.opts?.jobId?.toString();
+      if (jobId && msg?.opts?.removeOnComplete) {
+        this._queuedJobIds.delete(jobId);
+      }
     });
   }
   async isReady() {
-    return true;
+    return this;
   }
   // simply puts a message to the queue and emits to the queue for processing
   /**
@@ -60860,35 +60978,51 @@ var InMemoryQueue = class {
    * @param repeat serves no purpose for the import queue.
    */
   // eslint-disable-next-line no-unused-vars
-  add(msg, repeat) {
-    if (typeof msg !== "object") {
+  async add(data, opts) {
+    const jobId = opts?.jobId?.toString();
+    if (jobId && this._queuedJobIds.has(jobId)) {
+      console.log(`Ignoring already queued job ${jobId}`);
+      return;
+    }
+    if (typeof data !== "object") {
       throw "Queue only supports carrying JSON.";
     }
-    this._messages.push(newJob(this._name, msg));
-    this._addCount++;
-    this._emitter.emit("message");
+    if (jobId) {
+      this._queuedJobIds.add(jobId);
+    }
+    const pushMessage = () => {
+      this._messages.push(newJob(this._name, data, opts));
+      this._addCount++;
+      this._emitter.emit("message");
+    };
+    const delay = opts?.delay;
+    if (delay) {
+      setTimeout(pushMessage, delay);
+    } else {
+      pushMessage();
+    }
+    return {};
   }
   /**
    * replicating the close function from bull, which waits for jobs to finish.
    */
   async close() {
-    return [];
   }
   /**
    * This removes a cron which has been implemented, this is part of Bull API.
    * @param cronJobId The cron which is to be removed.
    */
-  removeRepeatableByKey(cronJobId) {
+  async removeRepeatableByKey(cronJobId) {
     console.log(cronJobId);
   }
   /**
    * Implemented for tests
    */
-  getRepeatableJobs() {
+  async getRepeatableJobs() {
     return [];
   }
   // eslint-disable-next-line no-unused-vars
-  removeJobs(pattern) {
+  async removeJobs(pattern) {
   }
   /**
    * Implemented for tests
@@ -60897,7 +61031,7 @@ var InMemoryQueue = class {
     return [];
   }
   async getJob() {
-    return {};
+    return null;
   }
   on() {
     return this;
@@ -60905,7 +61039,10 @@ var InMemoryQueue = class {
   async waitForCompletion() {
     do {
       await timeout(50);
-    } while (this._addCount < this._runCount);
+    } while (this.hasRunningJobs());
+  }
+  hasRunningJobs() {
+    return this._addCount > this._runCount;
   }
 };
 var inMemoryQueue_default = InMemoryQueue;
@@ -60920,6 +61057,7 @@ var JobQueue = /* @__PURE__ */ ((JobQueue2) => {
   JobQueue2["AUDIT_LOG"] = "auditLogQueue";
   JobQueue2["SYSTEM_EVENT_QUEUE"] = "systemEventQueue";
   JobQueue2["APP_MIGRATION"] = "appMigration";
+  JobQueue2["DOC_WRITETHROUGH_QUEUE"] = "docWritethroughQueue";
   return JobQueue2;
 })(JobQueue || {});
 
@@ -60971,7 +61109,8 @@ var EventTypeMap = {
   ["appBackupQueue" /* APP_BACKUP */]: "app-backup-event" /* APP_BACKUP_EVENT */,
   ["auditLogQueue" /* AUDIT_LOG */]: "audit-log-event" /* AUDIT_LOG_EVENT */,
   ["systemEventQueue" /* SYSTEM_EVENT_QUEUE */]: "system-event" /* SYSTEM_EVENT */,
-  ["appMigration" /* APP_MIGRATION */]: "app-migration" /* APP_MIGRATION */
+  ["appMigration" /* APP_MIGRATION */]: "app-migration" /* APP_MIGRATION */,
+  ["docWritethroughQueue" /* DOC_WRITETHROUGH_QUEUE */]: "doc-writethrough" /* DOC_WRITETHROUGH */
 };
 function logging(queue, jobQueue) {
   const eventType = EventTypeMap[jobQueue];
@@ -61042,6 +61181,7 @@ function logging(queue, jobQueue) {
 }
 
 // src/queue/queue.ts
+var import_bull2 = require("bull");
 var QUEUE_LOCK_MS = Duration.fromMinutes(5).toMs();
 var QUEUE_LOCK_RENEW_INTERNAL_MS = Duration.fromSeconds(30).toMs();
 var CLEANUP_PERIOD_MS = Duration.fromSeconds(60).toMs();
@@ -63994,6 +64134,83 @@ async function invalidateCode(code) {
   await client.delete(code);
 }
 
+// src/cache/docWritethrough.ts
+var docWritethrough_exports = {};
+__export(docWritethrough_exports, {
+  DocWritethrough: () => DocWritethrough,
+  docWritethroughProcessorQueue: () => docWritethroughProcessorQueue,
+  getProcessor: () => getProcessor,
+  init: () => init7
+});
+var PERSIST_MAX_ATTEMPTS = 100;
+var processor;
+var docWritethroughProcessorQueue = createQueue(
+  "docWritethroughQueue" /* DOC_WRITETHROUGH_QUEUE */,
+  {
+    jobOptions: {
+      attempts: PERSIST_MAX_ATTEMPTS
+    }
+  }
+);
+var DocWritethroughProcessor = class {
+  init() {
+    docWritethroughProcessorQueue.process(async (message) => {
+      try {
+        await this.persistToDb(message.data);
+      } catch (err) {
+        if (err.status === 409) {
+          throw new Error(
+            `Conflict persisting message ${message.id}. Attempt ${message.attemptsMade}`
+          );
+        }
+        throw err;
+      }
+    });
+    return this;
+  }
+  async persistToDb({
+    dbName,
+    docId,
+    data
+  }) {
+    const db = getDB(dbName);
+    let doc;
+    try {
+      doc = await db.get(docId);
+    } catch {
+      doc = { _id: docId };
+    }
+    doc = { ...doc, ...data };
+    await db.put(doc);
+  }
+};
+var DocWritethrough = class {
+  constructor(db, docId) {
+    this.db = db;
+    this._docId = docId;
+  }
+  get docId() {
+    return this._docId;
+  }
+  async patch(data) {
+    await docWritethroughProcessorQueue.add({
+      dbName: this.db.name,
+      docId: this.docId,
+      data
+    });
+  }
+};
+function init7() {
+  processor = new DocWritethroughProcessor().init();
+  return processor;
+}
+function getProcessor() {
+  if (!processor) {
+    return init7();
+  }
+  return processor;
+}
+
 // src/configs/configs.ts
 function generateConfigID(type) {
   return `${"config" /* CONFIG */}${SEPARATOR}${type}`;
@@ -65763,7 +65980,7 @@ async function isBlacklisted(address) {
 // src/docUpdates/index.ts
 var docUpdates_exports = {};
 __export(docUpdates_exports, {
-  init: () => init7
+  init: () => init8
 });
 
 // src/events/documentId.ts
@@ -65802,10 +66019,10 @@ var DocumentUpdateProcessor = class {
     if (!tenantId || !docId) {
       return;
     }
-    for (let { events: events2, processor } of this.processors) {
+    for (let { events: events2, processor: processor2 } of this.processors) {
       if (events2.includes(event)) {
         await doInTenant(tenantId, async () => {
-          await processor({
+          await processor2({
             id: docId,
             tenantId
           });
@@ -65821,7 +66038,7 @@ var DocumentUpdateProcessor = class {
 // src/docUpdates/index.ts
 var processingPromise;
 var documentProcessor;
-function init7(processors2) {
+function init8(processors2) {
   if (!asyncEventQueue) {
     init5();
   }
@@ -65846,7 +66063,7 @@ var tenancy = {
   ...tenancy_exports,
   ...context_exports
 };
-var init8 = (opts = {}) => {
+var init9 = (opts = {}) => {
   init(opts.db);
 };
 // Annotate the CommonJS export names for ESM import in node: