@backstage/plugin-scaffolder-backend 1.20.0 → 1.21.0-next.1

package/dist/cjs/{router-e667d04e.cjs.js → router-43c4dd8a.cjs.js}

@@ -27,11 +27,11 @@ var gerrit = require('@backstage/plugin-scaffolder-backend-module-gerrit');
 var gitlab = require('@backstage/plugin-scaffolder-backend-module-gitlab');
 var uuid = require('uuid');
 var ObservableImpl = require('zen-observable');
+var lodash = require('lodash');
 var PQueue = require('p-queue');
 var winston = require('winston');
 var nunjucks = require('nunjucks');
 var stream = require('stream');
-var lodash = require('lodash');
 var pluginPermissionNode = require('@backstage/plugin-permission-node');
 var promClient = require('prom-client');
 var pluginPermissionCommon = require('@backstage/plugin-permission-common');
@@ -640,6 +640,11 @@ function createFetchPlainAction(options) {
             title: "Target Path",
             description: "Target path within the working directory to download the contents to.",
             type: "string"
+          },
+          token: {
+            title: "Token",
+            description: "An optional token to use for authentication when reading the resources.",
+            type: "string"
           }
         }
       }
@@ -655,7 +660,8 @@ function createFetchPlainAction(options) {
         integrations,
         baseUrl: (_b = ctx.templateInfo) == null ? void 0 : _b.baseUrl,
         fetchUrl: ctx.input.url,
-        outputPath
+        outputPath,
+        token: ctx.input.token
       });
     }
   });
@@ -700,6 +706,11 @@ function createFetchPlainFileAction(options) {
             title: "Target Path",
             description: "Target path within the working directory to download the file as.",
             type: "string"
+          },
+          token: {
+            title: "Token",
+            description: "An optional token to use for authentication when reading the resources.",
+            type: "string"
           }
         }
       }
@@ -717,12 +728,22 @@ function createFetchPlainFileAction(options) {
         integrations,
         baseUrl: (_a = ctx.templateInfo) == null ? void 0 : _a.baseUrl,
         fetchUrl: ctx.input.url,
-        outputPath
+        outputPath,
+        token: ctx.input.token
       });
     }
   });
 }
 
+function isNoNodeSnapshotOptionProvided() {
+  var _a;
+  return ((_a = process.env.NODE_OPTIONS) == null ? void 0 : _a.includes("--no-node-snapshot")) || process.argv.includes("--no-node-snapshot");
+}
+function getMajorNodeVersion() {
+  const version = process.versions.node;
+  return parseInt(version.split(".")[0], 10);
+}
+
 const mkScript = (nunjucksSource) => `
 const { render, renderCompat } = (() => {
   const module = {};
@@ -733,6 +754,7 @@ const { render, renderCompat } = (() => {
 
   const env = module.exports.configure({
     autoescape: false,
+    ...JSON.parse(nunjucksConfigs),
     tags: {
       variableStart: '\${{',
       variableEnd: '}}',
@@ -741,6 +763,7 @@ const { render, renderCompat } = (() => {
 
   const compatEnv = module.exports.configure({
     autoescape: false,
+    ...JSON.parse(nunjucksConfigs),
     tags: {
       variableStart: '{{',
       variableEnd: '}}',
@@ -793,8 +816,16 @@ class SecureTemplater {
     const {
       cookiecutterCompat,
       templateFilters = {},
-      templateGlobals = {}
+      templateGlobals = {},
+      nunjucksConfigs = {}
     } = options;
+    const nodeVersion = getMajorNodeVersion();
+    if (nodeVersion >= 20 && !isNoNodeSnapshotOptionProvided()) {
+      throw new Error(
+        `When using Node.js version 20 or newer, the scaffolder backend plugin requires that it be started with the --no-node-snapshot option. 
+        Please make sure that you have NODE_OPTIONS=--no-node-snapshot in your environment.`
+      );
+    }
     const isolate = new isolatedVm.Isolate({ memoryLimit: 128 });
     const context = await isolate.createContext();
     const contextGlobal = context.global;
@@ -808,6 +839,7 @@ class SecureTemplater {
     const nunjucksScript = await isolate.compileScript(
       mkScript(nunjucksSource)
     );
+    await contextGlobal.set("nunjucksConfigs", JSON.stringify(nunjucksConfigs));
     const availableFilters = Object.keys(templateFilters);
     await contextGlobal.set(
       "availableTemplateFilters",
@@ -884,7 +916,7 @@ const createDefaultFilters = ({
 
 const examples$2 = [
   {
-    description: "Downloads a skelaton directory that lives alongside the template file and fill it out with values.",
+    description: "Downloads a skeleton directory that lives alongside the template file and fill it out with values.",
     example: yaml__default["default"].stringify({
       steps: [
         {
@@ -969,6 +1001,11 @@ function createFetchTemplateAction(options) {
             title: "Replace files",
             description: "If set, replace files in targetPath instead of skipping existing ones.",
             type: "boolean"
+          },
+          token: {
+            title: "Token",
+            description: "An optional token to use for authentication when reading the resources.",
+            type: "string"
           }
         }
       }
@@ -1020,7 +1057,8 @@ function createFetchTemplateAction(options) {
         integrations,
         baseUrl: (_b = ctx.templateInfo) == null ? void 0 : _b.baseUrl,
         fetchUrl: ctx.input.url,
-        outputPath: templateDir
+        outputPath: templateDir,
+        token: ctx.input.token
       });
       ctx.logger.info("Listing files and directories in template");
       const allEntriesInTemplate = await globby__default["default"](`**/*`, {
@@ -1053,7 +1091,11 @@ function createFetchTemplateAction(options) {
           ...defaultTemplateFilters,
           ...additionalTemplateFilters
         },
-        templateGlobals: additionalTemplateGlobals
+        templateGlobals: additionalTemplateGlobals,
+        nunjucksConfigs: {
+          trimBlocks: ctx.input.trimBlocks,
+          lstripBlocks: ctx.input.lstripBlocks
+        }
       });
       for (const location of allEntriesInTemplate) {
         let renderContents;
@@ -1419,6 +1461,36 @@ class TemplateActionRegistry {
   }
 }
 
+const trimEventsTillLastRecovery = (events) => {
+  const recoveredEventInd = events.slice().reverse().findIndex((event) => event.type === "recovered");
+  if (recoveredEventInd >= 0) {
+    const ind = events.length - recoveredEventInd - 1;
+    const { recoverStrategy } = events[ind].body;
+    if (recoverStrategy === "startOver") {
+      return {
+        events: recoveredEventInd === 0 ? [] : events.slice(ind)
+      };
+    }
+  }
+  return { events };
+};
+
+const intervalFromNowTill = (timeoutS, knex) => {
+  let heartbeatInterval = knex.raw(`? - interval '${timeoutS} seconds'`, [
+    knex.fn.now()
+  ]);
+  if (knex.client.config.client.includes("mysql")) {
+    heartbeatInterval = knex.raw(
+      `date_sub(now(), interval ${timeoutS} second)`
+    );
+  } else if (knex.client.config.client.includes("sqlite3")) {
+    heartbeatInterval = knex.raw(`datetime('now', ?)`, [
+      `-${timeoutS} seconds`
+    ]);
+  }
+  return heartbeatInterval;
+};
+
 var __defProp$3 = Object.defineProperty;
 var __defNormalProp$3 = (obj, key, value) => key in obj ? __defProp$3(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
 var __publicField$3 = (obj, key, value) => {
@@ -1455,6 +1527,28 @@ class DatabaseTaskStore {
     await this.runMigrations(database, client);
     return new DatabaseTaskStore(client);
   }
+  isRecoverableTask(spec) {
+    var _a, _b;
+    return ["startOver"].includes(
+      (_b = (_a = spec.EXPERIMENTAL_recovery) == null ? void 0 : _a.EXPERIMENTAL_strategy) != null ? _b : "none"
+    );
+  }
+  parseSpec({ spec, id }) {
+    try {
+      return JSON.parse(spec);
+    } catch (error) {
+      throw new Error(`Failed to parse spec of task '${id}', ${error}`);
+    }
+  }
+  parseTaskSecrets(taskRow) {
+    try {
+      return taskRow.secrets ? JSON.parse(taskRow.secrets) : void 0;
+    } catch (error) {
+      throw new Error(
+        `Failed to parse secrets of task '${taskRow.id}', ${error}`
+      );
+    }
+  }
   static async getClient(database) {
     if (isPluginDatabaseManager(database)) {
       return database.getClient();
@@ -1539,30 +1633,26 @@ class DatabaseTaskStore {
       if (!task) {
         return void 0;
       }
+      const spec = this.parseSpec(task);
       const updateCount = await tx("tasks").where({ id: task.id, status: "open" }).update({
         status: "processing",
         last_heartbeat_at: this.db.fn.now(),
-        // remove the secrets when moving to processing state.
-        secrets: null
+        // remove the secrets for non-recoverable tasks when moving to processing state.
+        secrets: this.isRecoverableTask(spec) ? task.secrets : null
       });
       if (updateCount < 1) {
         return void 0;
       }
-      try {
-        const spec = JSON.parse(task.spec);
-        const secrets = task.secrets ? JSON.parse(task.secrets) : void 0;
-        return {
-          id: task.id,
-          spec,
-          status: "processing",
-          lastHeartbeatAt: task.last_heartbeat_at,
-          createdAt: task.created_at,
-          createdBy: (_a = task.created_by) != null ? _a : void 0,
-          secrets
-        };
-      } catch (error) {
-        throw new Error(`Failed to parse spec of task '${task.id}', ${error}`);
-      }
+      const secrets = this.parseTaskSecrets(task);
+      return {
+        id: task.id,
+        spec,
+        status: "processing",
+        lastHeartbeatAt: task.last_heartbeat_at,
+        createdAt: task.created_at,
+        createdBy: (_a = task.created_by) != null ? _a : void 0,
+        secrets
+      };
     });
   }
   async heartbeatTask(taskId) {
@@ -1575,20 +1665,10 @@ class DatabaseTaskStore {
   }
   async listStaleTasks(options) {
     const { timeoutS } = options;
-    let heartbeatInterval = this.db.raw(`? - interval '${timeoutS} seconds'`, [
-      this.db.fn.now()
-    ]);
-    if (this.db.client.config.client.includes("mysql")) {
-      heartbeatInterval = this.db.raw(
-        `date_sub(now(), interval ${timeoutS} second)`
-      );
-    } else if (this.db.client.config.client.includes("sqlite3")) {
-      heartbeatInterval = this.db.raw(`datetime('now', ?)`, [
-        `-${timeoutS} seconds`
-      ]);
-    }
+    const heartbeatInterval = intervalFromNowTill(timeoutS, this.db);
     const rawRows = await this.db("tasks").where("status", "processing").andWhere("last_heartbeat_at", "<=", heartbeatInterval);
     const tasks = rawRows.map((row) => ({
+      recovery: JSON.parse(row.spec).EXPERIMENTAL_recovery,
       taskId: row.id
     }));
     return { tasks };
@@ -1609,7 +1689,8 @@ class DatabaseTaskStore {
       }).limit(1).select();
       const updateTask = async (criteria) => {
         const updateCount = await tx("tasks").where(criteria).update({
-          status
+          status,
+          secrets: null
         });
         if (updateCount !== 1) {
           throw new errors.ConflictError(
@@ -1679,7 +1760,7 @@ class DatabaseTaskStore {
         );
       }
     });
-    return { events };
+    return trimEventsTillLastRecovery(events);
   }
   async shutdownTask(options) {
     const { taskId } = options;
@@ -1718,7 +1799,72 @@ class DatabaseTaskStore {
       body: serializedBody
     });
   }
+  async recoverTasks(options) {
+    const taskIdsToRecover = [];
+    const timeoutS = luxon.Duration.fromObject(options.timeout).as("seconds");
+    await this.db.transaction(async (tx) => {
+      var _a, _b;
+      const heartbeatInterval = intervalFromNowTill(timeoutS, this.db);
+      const result = await tx("tasks").where("status", "processing").andWhere("last_heartbeat_at", "<=", heartbeatInterval).update(
+        {
+          status: "open",
+          last_heartbeat_at: this.db.fn.now()
+        },
+        ["id", "spec"]
+      );
+      taskIdsToRecover.push(...result.map((i) => i.id));
+      for (const { id, spec } of result) {
+        const taskSpec = JSON.parse(spec);
+        await this.db("task_events").insert({
+          task_id: id,
+          event_type: "recovered",
+          body: JSON.stringify({
+            recoverStrategy: (_b = (_a = taskSpec.EXPERIMENTAL_recovery) == null ? void 0 : _a.EXPERIMENTAL_strategy) != null ? _b : "none"
+          })
+        });
+      }
+    });
+    return { ids: taskIdsToRecover };
+  }
+}
+
+function isTruthy(value) {
+  return lodash.isArray(value) ? value.length > 0 : !!value;
+}
+function generateExampleOutput(schema) {
+  var _a, _b;
+  const { examples } = schema;
+  if (examples && Array.isArray(examples)) {
+    return examples[0];
+  }
+  if (schema.type === "object") {
+    return Object.fromEntries(
+      Object.entries((_a = schema.properties) != null ? _a : {}).map(([key, value]) => [
+        key,
+        generateExampleOutput(value)
+      ])
+    );
+  } else if (schema.type === "array") {
+    const [firstSchema] = (_b = [schema.items]) == null ? void 0 : _b.flat();
+    if (firstSchema) {
+      return [generateExampleOutput(firstSchema)];
+    }
+    return [];
+  } else if (schema.type === "string") {
+    return "<example>";
+  } else if (schema.type === "number") {
+    return 0;
+  } else if (schema.type === "boolean") {
+    return false;
+  }
+  return "<unknown>";
 }
+const readDuration$1 = (config$1, key, defaultValue) => {
+  if (config$1 == null ? void 0 : config$1.has(key)) {
+    return config.readDurationFromConfig(config$1, { key });
+  }
+  return defaultValue;
+};
 
 var __defProp$2 = Object.defineProperty;
 var __defNormalProp$2 = (obj, key, value) => key in obj ? __defProp$2(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
@@ -1803,9 +1949,10 @@ function defer() {
   return { promise, resolve };
 }
 class StorageTaskBroker {
-  constructor(storage, logger) {
+  constructor(storage, logger, config) {
     this.storage = storage;
     this.logger = logger;
+    this.config = config;
     __publicField$2(this, "deferredDispatch", defer());
   }
   async list(options) {
@@ -1838,6 +1985,26 @@ class StorageTaskBroker {
       }
     });
   }
+  async recoverTasks() {
+    var _a, _b, _c, _d;
+    const enabled = (_a = this.config && this.config.getOptionalBoolean(
+      "scaffolder.EXPERIMENTAL_recoverTasks"
+    )) != null ? _a : false;
+    if (enabled) {
+      const defaultTimeout = { seconds: 30 };
+      const timeout = readDuration$1(
+        this.config,
+        "scaffolder.EXPERIMENTAL_recoverTasksTimeout",
+        defaultTimeout
+      );
+      const { ids: recoveredTaskIds } = (_d = await ((_c = (_b = this.storage).recoverTasks) == null ? void 0 : _c.call(_b, {
+        timeout
+      }))) != null ? _d : { ids: [] };
+      if (recoveredTaskIds.length > 0) {
+        this.signalDispatch();
+      }
+    }
+  }
   /**
    * {@inheritdoc TaskBroker.claim}
    */
@@ -1945,38 +2112,6 @@ class StorageTaskBroker {
   }
 }
 
-function isTruthy(value) {
-  return lodash.isArray(value) ? value.length > 0 : !!value;
-}
-function generateExampleOutput(schema) {
-  var _a, _b;
-  const { examples } = schema;
-  if (examples && Array.isArray(examples)) {
-    return examples[0];
-  }
-  if (schema.type === "object") {
-    return Object.fromEntries(
-      Object.entries((_a = schema.properties) != null ? _a : {}).map(([key, value]) => [
-        key,
-        generateExampleOutput(value)
-      ])
-    );
-  } else if (schema.type === "array") {
-    const [firstSchema] = (_b = [schema.items]) == null ? void 0 : _b.flat();
-    if (firstSchema) {
-      return [generateExampleOutput(firstSchema)];
-    }
-    return [];
-  } else if (schema.type === "string") {
-    return "<example>";
-  } else if (schema.type === "number") {
-    return 0;
-  } else if (schema.type === "boolean") {
-    return false;
-  }
-  return "<unknown>";
-}
-
 function createCounterMetric(config) {
   let metric = promClient.register.getSingleMetric(config.name);
   if (!metric) {
@@ -2510,6 +2645,10 @@ class TaskWorker {
   constructor(options) {
     this.options = options;
     __publicField(this, "taskQueue");
+    __publicField(this, "logger");
+    __publicField(this, "stopWorkers");
+    this.stopWorkers = false;
+    this.logger = options.logger;
     this.taskQueue = new PQueue__default["default"]({
       concurrency: options.concurrentTasksLimit
     });
@@ -2543,15 +2682,34 @@ class TaskWorker {
       permissions
     });
   }
+  async recoverTasks() {
+    var _a, _b, _c;
+    try {
+      await ((_b = (_a = this.options.taskBroker).recoverTasks) == null ? void 0 : _b.call(_a));
+    } catch (err) {
+      (_c = this.logger) == null ? void 0 : _c.error(errors.stringifyError(err));
+    }
+  }
   start() {
     (async () => {
-      for (; ; ) {
+      while (!this.stopWorkers) {
+        await new Promise((resolve) => setTimeout(resolve, 1e4));
+        await this.recoverTasks();
+      }
+    })();
+    (async () => {
+      while (!this.stopWorkers) {
         await this.onReadyToClaimTask();
-        const task = await this.options.taskBroker.claim();
-        this.taskQueue.add(() => this.runOneTask(task));
+        if (!this.stopWorkers) {
+          const task = await this.options.taskBroker.claim();
+          void this.taskQueue.add(() => this.runOneTask(task));
+        }
       }
     })();
   }
+  stop() {
+    this.stopWorkers = true;
+  }
   onReadyToClaimTask() {
     if (this.taskQueue.pending < this.options.concurrentTasksLimit) {
       return Promise.resolve();
@@ -2812,7 +2970,7 @@ async function createRouter(options) {
   let taskBroker;
   if (!options.taskBroker) {
     const databaseTaskStore = await DatabaseTaskStore.create({ database });
-    taskBroker = new StorageTaskBroker(databaseTaskStore, logger);
+    taskBroker = new StorageTaskBroker(databaseTaskStore, logger, config);
     if (scheduler && databaseTaskStore.listStaleTasks) {
       await scheduler.scheduleTask({
         id: "close_stale_tasks",
@@ -2869,7 +3027,16 @@ async function createRouter(options) {
     additionalTemplateGlobals
   });
   actionsToRegister.forEach((action) => actionRegistry.register(action));
-  workers.forEach((worker) => worker.start());
+  const launchWorkers = () => workers.forEach((worker) => worker.start());
+  const shutdownWorkers = () => {
+    workers.forEach((worker) => worker.stop());
+  };
+  if (options.lifecycle) {
+    options.lifecycle.addStartupHook(launchWorkers);
+    options.lifecycle.addShutdownHook(shutdownWorkers);
+  } else {
+    launchWorkers();
+  }
   const dryRunner = createDryRunner({
     actionRegistry,
     integrations,
@@ -2983,6 +3150,7 @@ async function createRouter(options) {
           name: (_b2 = step.name) != null ? _b2 : step.action
         };
       }),
+      EXPERIMENTAL_recovery: template.spec.EXPERIMENTAL_recovery,
       output: (_b = template.spec.output) != null ? _b : {},
       parameters: values,
       user: {
@@ -3214,4 +3382,4 @@ exports.createRouter = createRouter;
 exports.createWaitAction = createWaitAction;
 exports.scaffolderActionRules = scaffolderActionRules;
 exports.scaffolderTemplateRules = scaffolderTemplateRules;
-//# sourceMappingURL=router-e667d04e.cjs.js.map
+//# sourceMappingURL=router-43c4dd8a.cjs.js.map