@backstage/plugin-scaffolder-backend 1.19.3-next.2 → 1.21.0-next.0

package/dist/cjs/{router-03a1f408.cjs.js → router-842a762b.cjs.js}

@@ -27,11 +27,11 @@ var gerrit = require('@backstage/plugin-scaffolder-backend-module-gerrit');
 var gitlab = require('@backstage/plugin-scaffolder-backend-module-gitlab');
 var uuid = require('uuid');
 var ObservableImpl = require('zen-observable');
+var lodash = require('lodash');
 var PQueue = require('p-queue');
 var winston = require('winston');
 var nunjucks = require('nunjucks');
 var stream = require('stream');
-var lodash = require('lodash');
 var pluginPermissionNode = require('@backstage/plugin-permission-node');
 var promClient = require('prom-client');
 var pluginPermissionCommon = require('@backstage/plugin-permission-common');
@@ -1319,6 +1319,9 @@ const createBuiltinActions = (options) => {
     gitlab.createPublishGitlabMergeRequestAction({
       integrations
     }),
+    gitlab.createGitlabRepoPushAction({
+      integrations
+    }),
     bitbucket.createPublishBitbucketAction({
       integrations,
       config
@@ -1376,6 +1379,9 @@ const createBuiltinActions = (options) => {
     github.createGithubAutolinksAction({
       integrations,
       githubCredentialsProvider
+    }),
+    bitbucket.createBitbucketPipelinesRunAction({
+      integrations
     })
   ];
   return actions;
@@ -1413,6 +1419,36 @@ class TemplateActionRegistry {
   }
 }
 
+const trimEventsTillLastRecovery = (events) => {
+  const recoveredEventInd = events.slice().reverse().findIndex((event) => event.type === "recovered");
+  if (recoveredEventInd >= 0) {
+    const ind = events.length - recoveredEventInd - 1;
+    const { recoverStrategy } = events[ind].body;
+    if (recoverStrategy === "startOver") {
+      return {
+        events: recoveredEventInd === 0 ? [] : events.slice(ind)
+      };
+    }
+  }
+  return { events };
+};
+
+const intervalFromNowTill = (timeoutS, knex) => {
+  let heartbeatInterval = knex.raw(`? - interval '${timeoutS} seconds'`, [
+    knex.fn.now()
+  ]);
+  if (knex.client.config.client.includes("mysql")) {
+    heartbeatInterval = knex.raw(
+      `date_sub(now(), interval ${timeoutS} second)`
+    );
+  } else if (knex.client.config.client.includes("sqlite3")) {
+    heartbeatInterval = knex.raw(`datetime('now', ?)`, [
+      `-${timeoutS} seconds`
+    ]);
+  }
+  return heartbeatInterval;
+};
+
 var __defProp$3 = Object.defineProperty;
 var __defNormalProp$3 = (obj, key, value) => key in obj ? __defProp$3(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
 var __publicField$3 = (obj, key, value) => {
@@ -1449,6 +1485,28 @@ class DatabaseTaskStore {
     await this.runMigrations(database, client);
     return new DatabaseTaskStore(client);
   }
+  isRecoverableTask(spec) {
+    var _a, _b;
+    return ["startOver"].includes(
+      (_b = (_a = spec.EXPERIMENTAL_recovery) == null ? void 0 : _a.EXPERIMENTAL_strategy) != null ? _b : "none"
+    );
+  }
+  parseSpec({ spec, id }) {
+    try {
+      return JSON.parse(spec);
+    } catch (error) {
+      throw new Error(`Failed to parse spec of task '${id}', ${error}`);
+    }
+  }
+  parseTaskSecrets(taskRow) {
+    try {
+      return taskRow.secrets ? JSON.parse(taskRow.secrets) : void 0;
+    } catch (error) {
+      throw new Error(
+        `Failed to parse secrets of task '${taskRow.id}', ${error}`
+      );
+    }
+  }
   static async getClient(database) {
     if (isPluginDatabaseManager(database)) {
       return database.getClient();
@@ -1533,30 +1591,26 @@ class DatabaseTaskStore {
       if (!task) {
         return void 0;
       }
+      const spec = this.parseSpec(task);
       const updateCount = await tx("tasks").where({ id: task.id, status: "open" }).update({
         status: "processing",
         last_heartbeat_at: this.db.fn.now(),
-        // remove the secrets when moving to processing state.
-        secrets: null
+        // remove the secrets for non-recoverable tasks when moving to processing state.
+        secrets: this.isRecoverableTask(spec) ? task.secrets : null
       });
       if (updateCount < 1) {
         return void 0;
       }
-      try {
-        const spec = JSON.parse(task.spec);
-        const secrets = task.secrets ? JSON.parse(task.secrets) : void 0;
-        return {
-          id: task.id,
-          spec,
-          status: "processing",
-          lastHeartbeatAt: task.last_heartbeat_at,
-          createdAt: task.created_at,
-          createdBy: (_a = task.created_by) != null ? _a : void 0,
-          secrets
-        };
-      } catch (error) {
-        throw new Error(`Failed to parse spec of task '${task.id}', ${error}`);
-      }
+      const secrets = this.parseTaskSecrets(task);
+      return {
+        id: task.id,
+        spec,
+        status: "processing",
+        lastHeartbeatAt: task.last_heartbeat_at,
+        createdAt: task.created_at,
+        createdBy: (_a = task.created_by) != null ? _a : void 0,
+        secrets
+      };
     });
   }
   async heartbeatTask(taskId) {
@@ -1569,20 +1623,10 @@ class DatabaseTaskStore {
   }
   async listStaleTasks(options) {
     const { timeoutS } = options;
-    let heartbeatInterval = this.db.raw(`? - interval '${timeoutS} seconds'`, [
-      this.db.fn.now()
-    ]);
-    if (this.db.client.config.client.includes("mysql")) {
-      heartbeatInterval = this.db.raw(
-        `date_sub(now(), interval ${timeoutS} second)`
-      );
-    } else if (this.db.client.config.client.includes("sqlite3")) {
-      heartbeatInterval = this.db.raw(`datetime('now', ?)`, [
-        `-${timeoutS} seconds`
-      ]);
-    }
+    const heartbeatInterval = intervalFromNowTill(timeoutS, this.db);
     const rawRows = await this.db("tasks").where("status", "processing").andWhere("last_heartbeat_at", "<=", heartbeatInterval);
     const tasks = rawRows.map((row) => ({
+      recovery: JSON.parse(row.spec).EXPERIMENTAL_recovery,
       taskId: row.id
     }));
     return { tasks };
@@ -1603,7 +1647,8 @@ class DatabaseTaskStore {
       }).limit(1).select();
       const updateTask = async (criteria) => {
         const updateCount = await tx("tasks").where(criteria).update({
-          status
+          status,
+          secrets: null
         });
         if (updateCount !== 1) {
           throw new errors.ConflictError(
@@ -1673,7 +1718,7 @@ class DatabaseTaskStore {
         );
       }
     });
-    return { events };
+    return trimEventsTillLastRecovery(events);
   }
   async shutdownTask(options) {
     const { taskId } = options;
@@ -1712,7 +1757,72 @@ class DatabaseTaskStore {
       body: serializedBody
     });
   }
+  async recoverTasks(options) {
+    const taskIdsToRecover = [];
+    const timeoutS = luxon.Duration.fromObject(options.timeout).as("seconds");
+    await this.db.transaction(async (tx) => {
+      var _a, _b;
+      const heartbeatInterval = intervalFromNowTill(timeoutS, this.db);
+      const result = await tx("tasks").where("status", "processing").andWhere("last_heartbeat_at", "<=", heartbeatInterval).update(
+        {
+          status: "open",
+          last_heartbeat_at: this.db.fn.now()
+        },
+        ["id", "spec"]
+      );
+      taskIdsToRecover.push(...result.map((i) => i.id));
+      for (const { id, spec } of result) {
+        const taskSpec = JSON.parse(spec);
+        await this.db("task_events").insert({
+          task_id: id,
+          event_type: "recovered",
+          body: JSON.stringify({
+            recoverStrategy: (_b = (_a = taskSpec.EXPERIMENTAL_recovery) == null ? void 0 : _a.EXPERIMENTAL_strategy) != null ? _b : "none"
+          })
+        });
+      }
+    });
+    return { ids: taskIdsToRecover };
+  }
+}
+
+function isTruthy(value) {
+  return lodash.isArray(value) ? value.length > 0 : !!value;
+}
+function generateExampleOutput(schema) {
+  var _a, _b;
+  const { examples } = schema;
+  if (examples && Array.isArray(examples)) {
+    return examples[0];
+  }
+  if (schema.type === "object") {
+    return Object.fromEntries(
+      Object.entries((_a = schema.properties) != null ? _a : {}).map(([key, value]) => [
+        key,
+        generateExampleOutput(value)
+      ])
+    );
+  } else if (schema.type === "array") {
+    const [firstSchema] = (_b = [schema.items]) == null ? void 0 : _b.flat();
+    if (firstSchema) {
+      return [generateExampleOutput(firstSchema)];
+    }
+    return [];
+  } else if (schema.type === "string") {
+    return "<example>";
+  } else if (schema.type === "number") {
+    return 0;
+  } else if (schema.type === "boolean") {
+    return false;
+  }
+  return "<unknown>";
 }
+const readDuration$1 = (config$1, key, defaultValue) => {
+  if (config$1 == null ? void 0 : config$1.has(key)) {
+    return config.readDurationFromConfig(config$1, { key });
+  }
+  return defaultValue;
+};
 
 var __defProp$2 = Object.defineProperty;
 var __defNormalProp$2 = (obj, key, value) => key in obj ? __defProp$2(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
@@ -1797,9 +1907,10 @@ function defer() {
   return { promise, resolve };
 }
 class StorageTaskBroker {
-  constructor(storage, logger) {
+  constructor(storage, logger, config) {
     this.storage = storage;
     this.logger = logger;
+    this.config = config;
     __publicField$2(this, "deferredDispatch", defer());
   }
   async list(options) {
@@ -1832,6 +1943,26 @@ class StorageTaskBroker {
       }
     });
   }
+  async recoverTasks() {
+    var _a, _b, _c, _d;
+    const enabled = (_a = this.config && this.config.getOptionalBoolean(
+      "scaffolder.EXPERIMENTAL_recoverTasks"
+    )) != null ? _a : false;
+    if (enabled) {
+      const defaultTimeout = { seconds: 30 };
+      const timeout = readDuration$1(
+        this.config,
+        "scaffolder.EXPERIMENTAL_recoverTasksTimeout",
+        defaultTimeout
+      );
+      const { ids: recoveredTaskIds } = (_d = await ((_c = (_b = this.storage).recoverTasks) == null ? void 0 : _c.call(_b, {
+        timeout
+      }))) != null ? _d : { ids: [] };
+      if (recoveredTaskIds.length > 0) {
+        this.signalDispatch();
+      }
+    }
+  }
   /**
    * {@inheritdoc TaskBroker.claim}
    */
@@ -1939,38 +2070,6 @@ class StorageTaskBroker {
   }
 }
 
-function isTruthy(value) {
-  return lodash.isArray(value) ? value.length > 0 : !!value;
-}
-function generateExampleOutput(schema) {
-  var _a, _b;
-  const { examples } = schema;
-  if (examples && Array.isArray(examples)) {
-    return examples[0];
-  }
-  if (schema.type === "object") {
-    return Object.fromEntries(
-      Object.entries((_a = schema.properties) != null ? _a : {}).map(([key, value]) => [
-        key,
-        generateExampleOutput(value)
-      ])
-    );
-  } else if (schema.type === "array") {
-    const [firstSchema] = (_b = [schema.items]) == null ? void 0 : _b.flat();
-    if (firstSchema) {
-      return [generateExampleOutput(firstSchema)];
-    }
-    return [];
-  } else if (schema.type === "string") {
-    return "<example>";
-  } else if (schema.type === "number") {
-    return 0;
-  } else if (schema.type === "boolean") {
-    return false;
-  }
-  return "<unknown>";
-}
-
 function createCounterMetric(config) {
   let metric = promClient.register.getSingleMetric(config.name);
   if (!metric) {
@@ -2504,6 +2603,10 @@ class TaskWorker {
   constructor(options) {
     this.options = options;
     __publicField(this, "taskQueue");
+    __publicField(this, "logger");
+    __publicField(this, "stopWorkers");
+    this.stopWorkers = false;
+    this.logger = options.logger;
     this.taskQueue = new PQueue__default["default"]({
       concurrency: options.concurrentTasksLimit
     });
@@ -2537,15 +2640,34 @@ class TaskWorker {
       permissions
     });
   }
+  async recoverTasks() {
+    var _a, _b, _c;
+    try {
+      await ((_b = (_a = this.options.taskBroker).recoverTasks) == null ? void 0 : _b.call(_a));
+    } catch (err) {
+      (_c = this.logger) == null ? void 0 : _c.error(errors.stringifyError(err));
+    }
+  }
   start() {
     (async () => {
-      for (; ; ) {
+      while (!this.stopWorkers) {
+        await new Promise((resolve) => setTimeout(resolve, 1e4));
+        await this.recoverTasks();
+      }
+    })();
+    (async () => {
+      while (!this.stopWorkers) {
         await this.onReadyToClaimTask();
-        const task = await this.options.taskBroker.claim();
-        this.taskQueue.add(() => this.runOneTask(task));
+        if (!this.stopWorkers) {
+          const task = await this.options.taskBroker.claim();
+          void this.taskQueue.add(() => this.runOneTask(task));
+        }
       }
     })();
   }
+  stop() {
+    this.stopWorkers = true;
+  }
   onReadyToClaimTask() {
     if (this.taskQueue.pending < this.options.concurrentTasksLimit) {
       return Promise.resolve();
@@ -2806,7 +2928,7 @@ async function createRouter(options) {
   let taskBroker;
   if (!options.taskBroker) {
     const databaseTaskStore = await DatabaseTaskStore.create({ database });
-    taskBroker = new StorageTaskBroker(databaseTaskStore, logger);
+    taskBroker = new StorageTaskBroker(databaseTaskStore, logger, config);
     if (scheduler && databaseTaskStore.listStaleTasks) {
       await scheduler.scheduleTask({
         id: "close_stale_tasks",
@@ -2863,7 +2985,16 @@ async function createRouter(options) {
     additionalTemplateGlobals
   });
   actionsToRegister.forEach((action) => actionRegistry.register(action));
-  workers.forEach((worker) => worker.start());
+  const launchWorkers = () => workers.forEach((worker) => worker.start());
+  const shutdownWorkers = () => {
+    workers.forEach((worker) => worker.stop());
+  };
+  if (options.lifecycle) {
+    options.lifecycle.addStartupHook(launchWorkers);
+    options.lifecycle.addShutdownHook(shutdownWorkers);
+  } else {
+    launchWorkers();
+  }
   const dryRunner = createDryRunner({
     actionRegistry,
     integrations,
@@ -2977,6 +3108,7 @@ async function createRouter(options) {
           name: (_b2 = step.name) != null ? _b2 : step.action
         };
       }),
+      EXPERIMENTAL_recovery: template.spec.EXPERIMENTAL_recovery,
       output: (_b = template.spec.output) != null ? _b : {},
       parameters: values,
       user: {
@@ -3208,4 +3340,4 @@ exports.createRouter = createRouter;
 exports.createWaitAction = createWaitAction;
 exports.scaffolderActionRules = scaffolderActionRules;
 exports.scaffolderTemplateRules = scaffolderTemplateRules;
-//# sourceMappingURL=router-03a1f408.cjs.js.map
+//# sourceMappingURL=router-842a762b.cjs.js.map