@backstage/plugin-scaffolder-backend 0.15.4 → 0.15.8

package/CHANGELOG.md

@@ -1,5 +1,48 @@
 # @backstage/plugin-scaffolder-backend
 
+## 0.15.8
+
+### Patch Changes
+
+- 42c618abf6: Use `resolveSafeChildPath` in the `fetchContents` function to forbid reading files outside the base directory when a template is registered from a `file:` location.
+- 18083d1821: Introduce the new `scaffolder.backstage.io/v1beta3` template kind with nunjucks support 🥋
+- Updated dependencies
+  - @backstage/integration@0.6.8
+  - @backstage/plugin-catalog-backend@0.17.0
+
+## 0.15.7
+
+### Patch Changes
+
+- ca3086a7ad: Fixed a bug where the `catalog:register` action would not return any entity when running towards recent versions of the catalog.
+- Updated dependencies
+  - @backstage/catalog-model@0.9.4
+  - @backstage/backend-common@0.9.6
+  - @backstage/catalog-client@0.5.0
+  - @backstage/integration@0.6.7
+
+## 0.15.6
+
+### Patch Changes
+
+- Updated dependencies
+  - @backstage/integration@0.6.5
+  - @backstage/catalog-client@0.4.0
+  - @backstage/catalog-model@0.9.3
+  - @backstage/backend-common@0.9.4
+  - @backstage/config@0.1.10
+
+## 0.15.5
+
+### Patch Changes
+
+- 618143c3c7: Action needed: If you are using the templates located at https://github.com/backstage/backstage/tree/master/ in your Backstage app directly using the URL via the `app-config.yaml`, you should copy over the templates inside your org and import from there. The templates have now been moved to https://github.com/backstage/software-templates. See https://github.com/backstage/backstage/issues/6415 for explanation.
+- cfade02127: Change hardcoded branch `master` to \$defaultBranch in GitLab provider
+- 96fef17a18: Upgrade git-parse-url to v11.6.0
+- Updated dependencies
+  - @backstage/backend-common@0.9.3
+  - @backstage/integration@0.6.4
+
 ## 0.15.4
 
 ### Patch Changes

package/dist/index.cjs.js

@@ -27,9 +27,11 @@ var Router = require('express-promise-router');
 var jsonschema = require('jsonschema');
 var uuid = require('uuid');
 var luxon = require('luxon');
+var os = require('os');
 var Handlebars = require('handlebars');
 var winston = require('winston');
-var os = require('os');
+var pluginCatalogBackend = require('@backstage/plugin-catalog-backend');
+var pluginScaffolderCommon = require('@backstage/plugin-scaffolder-common');
 
 function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
 
@@ -54,6 +56,7 @@ function _interopNamespace(e) {
 }
 
 var fs__default = /*#__PURE__*/_interopDefaultLegacy(fs);
+var path__namespace = /*#__PURE__*/_interopNamespace(path);
 var path__default = /*#__PURE__*/_interopDefaultLegacy(path);
 var yaml__namespace = /*#__PURE__*/_interopNamespace(yaml);
 var globby__default = /*#__PURE__*/_interopDefaultLegacy(globby);
@@ -61,9 +64,9 @@ var nunjucks__default = /*#__PURE__*/_interopDefaultLegacy(nunjucks);
 var fetch__default = /*#__PURE__*/_interopDefaultLegacy(fetch);
 var express__default = /*#__PURE__*/_interopDefaultLegacy(express);
 var Router__default = /*#__PURE__*/_interopDefaultLegacy(Router);
+var os__default = /*#__PURE__*/_interopDefaultLegacy(os);
 var Handlebars__namespace = /*#__PURE__*/_interopNamespace(Handlebars);
 var winston__namespace = /*#__PURE__*/_interopNamespace(winston);
-var os__default = /*#__PURE__*/_interopDefaultLegacy(os);
 
 const createTemplateAction = (templateAction) => {
   return templateAction;
@@ -108,6 +111,7 @@ function createCatalogRegisterAction(options) {
       }
     },
     async handler(ctx) {
+      var _a;
       const {input} = ctx;
       let catalogInfoUrl;
       if ("catalogInfoUrl" in input) {
@@ -124,15 +128,21 @@ function createCatalogRegisterAction(options) {
         });
       }
       ctx.logger.info(`Registering ${catalogInfoUrl} in the catalog`);
+      await catalogClient.addLocation({
+        type: "url",
+        target: catalogInfoUrl
+      }, ctx.token ? {token: ctx.token} : {});
       const result = await catalogClient.addLocation({
+        dryRun: true,
         type: "url",
         target: catalogInfoUrl
       }, ctx.token ? {token: ctx.token} : {});
-      if (result.entities.length >= 1) {
-        const {kind, name, namespace} = catalogModel.getEntityName(result.entities[0]);
-        ctx.output("entityRef", `${kind}:${namespace}/${name}`);
-        ctx.output("catalogInfoUrl", catalogInfoUrl);
+      if (result.entities.length > 0) {
+        const {entities} = result;
+        const entity = (_a = entities.find((e) => !e.metadata.name.startsWith("generated-"))) != null ? _a : entities[0];
+        ctx.output("entityRef", catalogModel.stringifyEntityRef(entity));
       }
+      ctx.output("catalogInfoUrl", catalogInfoUrl);
     }
   });
 }
@@ -176,12 +186,16 @@ function createDebugLogAction() {
           listWorkspace: {
             title: "List all files in the workspace, if true.",
             type: "boolean"
+          },
+          extra: {
+            title: "Extra info"
           }
         }
       }
     },
     async handler(ctx) {
       var _a, _b;
+      ctx.logger.info(JSON.stringify(ctx.input, null, 2));
       if ((_a = ctx.input) == null ? void 0 : _a.message) {
         ctx.logStream.write(ctx.input.message);
       }
@@ -220,10 +234,7 @@ async function fetchContents({
   }
   if (!fetchUrlIsAbsolute && (baseUrl == null ? void 0 : baseUrl.startsWith("file://"))) {
     const basePath = baseUrl.slice("file://".length);
-    if (path.isAbsolute(fetchUrl)) {
-      throw new errors.InputError(`Fetch URL may not be absolute for file locations, ${fetchUrl}`);
-    }
-    const srcDir = path.resolve(basePath, "..", fetchUrl);
+    const srcDir = backendCommon.resolveSafeChildPath(path__namespace.dirname(basePath), fetchUrl);
     await fs__default['default'].copy(srcDir, outputPath);
   } else {
     let readUrl;
@@ -1462,7 +1473,7 @@ function createPublishGitlabAction(options) {
         visibility: repoVisibility
       });
       const remoteUrl = http_url_to_repo.replace(/\.git$/, "");
-      const repoContentsUrl = `${remoteUrl}/-/blob/master`;
+      const repoContentsUrl = `${remoteUrl}/-/blob/${defaultBranch}`;
       const gitAuthorInfo = {
         name: config.getOptionalString("scaffolder.defaultAuthor.name"),
         email: config.getOptionalString("scaffolder.defaultAuthor.email")
@@ -2028,39 +2039,90 @@ class StorageTaskBroker {
   }
 }
 
+class TaskWorker {
+  constructor(options) {
+    this.options = options;
+  }
+  start() {
+    (async () => {
+      for (; ; ) {
+        const task = await this.options.taskBroker.claim();
+        await this.runOneTask(task);
+      }
+    })();
+  }
+  async runOneTask(task) {
+    try {
+      const {output} = task.spec.apiVersion === "scaffolder.backstage.io/v1beta3" ? await this.options.runners.workflowRunner.execute(task) : await this.options.runners.legacyWorkflowRunner.execute(task);
+      await task.complete("completed", {output});
+    } catch (error) {
+      await task.complete("failed", {
+        error: {name: error.name, message: error.message}
+      });
+    }
+  }
+}
+
+async function getWorkingDirectory(config, logger) {
+  if (!config.has("backend.workingDirectory")) {
+    return os__default['default'].tmpdir();
+  }
+  const workingDirectory = config.getString("backend.workingDirectory");
+  try {
+    await fs__default['default'].access(workingDirectory, fs__default['default'].constants.F_OK | fs__default['default'].constants.W_OK);
+    logger.info(`using working directory: ${workingDirectory}`);
+  } catch (err) {
+    logger.error(`working directory ${workingDirectory} ${err.code === "ENOENT" ? "does not exist" : "is not writable"}`);
+    throw err;
+  }
+  return workingDirectory;
+}
+function getEntityBaseUrl(entity) {
+  var _a, _b;
+  let location = (_a = entity.metadata.annotations) == null ? void 0 : _a[catalogModel.SOURCE_LOCATION_ANNOTATION];
+  if (!location) {
+    location = (_b = entity.metadata.annotations) == null ? void 0 : _b[catalogModel.LOCATION_ANNOTATION];
+  }
+  if (!location) {
+    return void 0;
+  }
+  const {type, target} = catalogModel.parseLocationReference(location);
+  if (type === "url") {
+    return target;
+  } else if (type === "file") {
+    return `file://${target}`;
+  }
+  return void 0;
+}
+
 function isTruthy(value) {
   return lodash.isArray(value) ? value.length > 0 : !!value;
 }
 
-class TaskWorker {
+const isValidTaskSpec$1 = (taskSpec) => taskSpec.apiVersion === "backstage.io/v1beta2";
+class LegacyWorkflowRunner {
   constructor(options) {
     this.options = options;
     this.handlebars = Handlebars__namespace.create();
     this.handlebars.registerHelper("parseRepoUrl", (repoUrl) => {
-      return JSON.stringify(parseRepoUrl(repoUrl, options.integrations));
+      return JSON.stringify(parseRepoUrl(repoUrl, this.options.integrations));
     });
     this.handlebars.registerHelper("projectSlug", (repoUrl) => {
-      const {owner, repo} = parseRepoUrl(repoUrl, options.integrations);
+      const {owner, repo} = parseRepoUrl(repoUrl, this.options.integrations);
       return `${owner}/${repo}`;
     });
     this.handlebars.registerHelper("json", (obj) => JSON.stringify(obj));
     this.handlebars.registerHelper("not", (value) => !isTruthy(value));
     this.handlebars.registerHelper("eq", (a, b) => a === b);
   }
-  start() {
-    (async () => {
-      for (; ; ) {
-        const task = await this.options.taskBroker.claim();
-        await this.runOneTask(task);
-      }
-    })();
-  }
-  async runOneTask(task) {
+  async execute(task) {
     var _a, _b;
-    let workspacePath = void 0;
+    if (!isValidTaskSpec$1(task.spec)) {
+      throw new errors.InputError(`Task spec is not a valid v1beta2 task spec`);
+    }
+    const {actionRegistry} = this.options;
+    const workspacePath = path__default['default'].join(this.options.workingDirectory, await task.getWorkspaceName());
     try {
-      const {actionRegistry} = this.options;
-      workspacePath = path__default['default'].join(this.options.workingDirectory, await task.getWorkspaceName());
       await fs__default['default'].ensureDir(workspacePath);
       await task.emitLog(`Starting up task with ${task.spec.steps.length} steps`);
       const templateCtx = {parameters: task.spec.values, steps: {}};
@@ -2202,11 +2264,7 @@ class TaskWorker {
         }
         return value;
       });
-      await task.complete("completed", {output});
-    } catch (error) {
-      await task.complete("failed", {
-        error: {name: error.name, message: error.message}
-      });
+      return {output};
     } finally {
       if (workspacePath) {
         await fs__default['default'].remove(workspacePath);
@@ -2215,40 +2273,159 @@ class TaskWorker {
   }
 }
 
-async function getWorkingDirectory(config, logger) {
-  if (!config.has("backend.workingDirectory")) {
-    return os__default['default'].tmpdir();
-  }
-  const workingDirectory = config.getString("backend.workingDirectory");
-  try {
-    await fs__default['default'].access(workingDirectory, fs__default['default'].constants.F_OK | fs__default['default'].constants.W_OK);
-    logger.info(`using working directory: ${workingDirectory}`);
-  } catch (err) {
-    logger.error(`working directory ${workingDirectory} ${err.code === "ENOENT" ? "does not exist" : "is not writable"}`);
-    throw err;
+const isValidTaskSpec = (taskSpec) => {
+  return taskSpec.apiVersion === "scaffolder.backstage.io/v1beta3";
+};
+const createStepLogger = ({task, step}) => {
+  const metadata = {stepId: step.id};
+  const taskLogger = winston__namespace.createLogger({
+    level: process.env.LOG_LEVEL || "info",
+    format: winston__namespace.format.combine(winston__namespace.format.colorize(), winston__namespace.format.timestamp(), winston__namespace.format.simple()),
+    defaultMeta: {}
+  });
+  const streamLogger = new stream.PassThrough();
+  streamLogger.on("data", async (data) => {
+    const message = data.toString().trim();
+    if ((message == null ? void 0 : message.length) > 1) {
+      await task.emitLog(message, metadata);
+    }
+  });
+  taskLogger.add(new winston__namespace.transports.Stream({stream: streamLogger}));
+  return {taskLogger, streamLogger};
+};
+class DefaultWorkflowRunner {
+  constructor(options) {
+    this.options = options;
+    this.nunjucksOptions = {
+      autoescape: false,
+      tags: {
+        variableStart: "${{",
+        variableEnd: "}}"
+      }
+    };
+    this.nunjucks = nunjucks__default['default'].configure(this.nunjucksOptions);
+    this.nunjucks.addFilter("parseRepoUrl", (repoUrl) => {
+      return parseRepoUrl(repoUrl, this.options.integrations);
+    });
+    this.nunjucks.addFilter("projectSlug", (repoUrl) => {
+      const {owner, repo} = parseRepoUrl(repoUrl, this.options.integrations);
+      return `${owner}/${repo}`;
+    });
   }
-  return workingDirectory;
-}
-function getEntityBaseUrl(entity) {
-  var _a, _b;
-  let location = (_a = entity.metadata.annotations) == null ? void 0 : _a[catalogModel.SOURCE_LOCATION_ANNOTATION];
-  if (!location) {
-    location = (_b = entity.metadata.annotations) == null ? void 0 : _b[catalogModel.LOCATION_ANNOTATION];
+  isSingleTemplateString(input) {
+    const {parser, nodes} = require("nunjucks");
+    const parsed = parser.parse(input, {}, this.nunjucksOptions);
+    return parsed.children.length === 1 && !(parsed.children[0] instanceof nodes.TemplateData);
   }
-  if (!location) {
-    return void 0;
+  render(input, context) {
+    return JSON.parse(JSON.stringify(input), (_key, value) => {
+      try {
+        if (typeof value === "string") {
+          try {
+            if (this.isSingleTemplateString(value)) {
+              const wrappedDumped = value.replace(/\${{(.+)}}/g, "${{ ( $1 ) | dump }}");
+              const templated2 = this.nunjucks.renderString(wrappedDumped, context);
+              if (templated2 === "") {
+                return void 0;
+              }
+              return JSON.parse(templated2);
+            }
+          } catch (ex) {
+            this.options.logger.error(`Failed to parse template string: ${value} with error ${ex.message}`);
+          }
+          const templated = this.nunjucks.renderString(value, context);
+          if (templated === "") {
+            return void 0;
+          }
+          return templated;
+        }
+      } catch {
+        return value;
+      }
+      return value;
+    });
   }
-  const {type, target} = catalogModel.parseLocationReference(location);
-  if (type === "url") {
-    return target;
-  } else if (type === "file") {
-    return `file://${target}`;
+  async execute(task) {
+    var _a, _b;
+    if (!isValidTaskSpec(task.spec)) {
+      throw new errors.InputError("Wrong template version executed with the workflow engine");
+    }
+    const workspacePath = path__default['default'].join(this.options.workingDirectory, await task.getWorkspaceName());
+    try {
+      await fs__default['default'].ensureDir(workspacePath);
+      await task.emitLog(`Starting up task with ${task.spec.steps.length} steps`);
+      const context = {
+        parameters: task.spec.parameters,
+        steps: {}
+      };
+      for (const step of task.spec.steps) {
+        try {
+          if (step.if) {
+            const ifResult = await this.render(step.if, context);
+            if (!isTruthy(ifResult)) {
+              await task.emitLog(`Skipping step ${step.id} because it's if condition was false`, {stepId: step.id, status: "skipped"});
+              continue;
+            }
+          }
+          await task.emitLog(`Beginning step ${step.name}`, {
+            stepId: step.id,
+            status: "processing"
+          });
+          const action = this.options.actionRegistry.get(step.action);
+          const {taskLogger, streamLogger} = createStepLogger({task, step});
+          const input = (_a = step.input && this.render(step.input, context)) != null ? _a : {};
+          if ((_b = action.schema) == null ? void 0 : _b.input) {
+            const validateResult = jsonschema.validate(input, action.schema.input);
+            if (!validateResult.valid) {
+              const errors$1 = validateResult.errors.join(", ");
+              throw new errors.InputError(`Invalid input passed to action ${action.id}, ${errors$1}`);
+            }
+          }
+          const tmpDirs = new Array();
+          const stepOutput = {};
+          await action.handler({
+            baseUrl: task.spec.baseUrl,
+            input,
+            logger: taskLogger,
+            logStream: streamLogger,
+            workspacePath,
+            createTemporaryDirectory: async () => {
+              const tmpDir = await fs__default['default'].mkdtemp(`${workspacePath}_step-${step.id}-`);
+              tmpDirs.push(tmpDir);
+              return tmpDir;
+            },
+            output(name, value) {
+              stepOutput[name] = value;
+            }
+          });
+          for (const tmpDir of tmpDirs) {
+            await fs__default['default'].remove(tmpDir);
+          }
+          context.steps[step.id] = {output: stepOutput};
+          await task.emitLog(`Finished step ${step.name}`, {
+            stepId: step.id,
+            status: "completed"
+          });
+        } catch (err) {
+          await task.emitLog(String(err.stack), {
+            stepId: step.id,
+            status: "failed"
+          });
+          throw err;
+        }
+      }
+      const output = this.render(task.spec.output, context);
+      return {output};
+    } finally {
+      if (workspacePath) {
+        await fs__default['default'].remove(workspacePath);
+      }
+    }
   }
-  return void 0;
 }
 
-function isBeta2Template(entity) {
-  return entity.apiVersion === "backstage.io/v1beta2";
+function isSupportedTemplate(entity) {
+  return entity.apiVersion === "backstage.io/v1beta2" || entity.apiVersion === "scaffolder.backstage.io/v1beta3";
 }
 async function createRouter(options) {
   const router = Router__default['default']();
@@ -2270,14 +2447,26 @@ async function createRouter(options) {
   const databaseTaskStore = await DatabaseTaskStore.create(await database.getClient());
   const taskBroker = new StorageTaskBroker(databaseTaskStore, logger);
   const actionRegistry = new TemplateActionRegistry();
+  const legacyWorkflowRunner = new LegacyWorkflowRunner({
+    logger,
+    actionRegistry,
+    integrations,
+    workingDirectory
+  });
+  const workflowRunner = new DefaultWorkflowRunner({
+    actionRegistry,
+    integrations,
+    logger,
+    workingDirectory
+  });
   const workers = [];
   for (let i = 0; i < (taskWorkers || 1); i++) {
     const worker = new TaskWorker({
-      logger,
       taskBroker,
-      actionRegistry,
-      workingDirectory,
-      integrations
+      runners: {
+        legacyWorkflowRunner,
+        workflowRunner
+      }
     });
     workers.push(worker);
   }
@@ -2302,7 +2491,7 @@ async function createRouter(options) {
     const template = await entityClient.findTemplate(name, {
       token: getBearerToken(req.headers.authorization)
     });
-    if (isBeta2Template(template)) {
+    if (isSupportedTemplate(template)) {
       const parameters = [(_a = template.spec.parameters) != null ? _a : []].flat();
       res.json({
         title: (_b = template.metadata.title) != null ? _b : template.metadata.name,
@@ -2327,7 +2516,7 @@ async function createRouter(options) {
     });
     res.json(actionsList);
   }).post("/v2/tasks", async (req, res) => {
-    var _a, _b;
+    var _a, _b, _c;
     const templateName = req.body.templateName;
     const values = req.body.values;
     const token = getBearerToken(req.headers.authorization);
@@ -2335,7 +2524,7 @@ async function createRouter(options) {
       token
     });
     let taskSpec;
-    if (isBeta2Template(template)) {
+    if (isSupportedTemplate(template)) {
       for (const parameters of [(_a = template.spec.parameters) != null ? _a : []].flat()) {
         const result2 = jsonschema.validate(values, parameters);
         if (!result2.valid) {
@@ -2344,7 +2533,8 @@ async function createRouter(options) {
         }
       }
       const baseUrl = getEntityBaseUrl(template);
-      taskSpec = {
+      taskSpec = template.apiVersion === "backstage.io/v1beta2" ? {
+        apiVersion: template.apiVersion,
         baseUrl,
         values,
         steps: template.spec.steps.map((step, index) => {
@@ -2356,6 +2546,19 @@ async function createRouter(options) {
           };
         }),
         output: (_b = template.spec.output) != null ? _b : {}
+      } : {
+        apiVersion: template.apiVersion,
+        baseUrl,
+        parameters: values,
+        steps: template.spec.steps.map((step, index) => {
+          var _a2, _b2;
+          return {
+            ...step,
+            id: (_a2 = step.id) != null ? _a2 : `step-${index + 1}`,
+            name: (_b2 = step.name) != null ? _b2 : step.action
+          };
+        }),
+        output: (_c = template.spec.output) != null ? _c : {}
       };
     } else {
       throw new errors.InputError(`Unsupported apiVersion field in schema entity, ${template.apiVersion}`);
@@ -2414,6 +2617,54 @@ function getBearerToken(header) {
   return (_a = header == null ? void 0 : header.match(/Bearer\s+(\S+)/i)) == null ? void 0 : _a[1];
 }
 
+class ScaffolderEntitiesProcessor {
+  constructor() {
+    this.validators = [
+      catalogModel.entityKindSchemaValidator(pluginScaffolderCommon.templateEntityV1beta3Schema)
+    ];
+  }
+  async validateEntityKind(entity) {
+    for (const validator of this.validators) {
+      if (validator(entity)) {
+        return true;
+      }
+    }
+    return false;
+  }
+  async postProcessEntity(entity, _location, emit) {
+    const selfRef = catalogModel.getEntityName(entity);
+    if (entity.apiVersion === "scaffolder.backstage.io/v1beta3" && entity.kind === "Template") {
+      const template = entity;
+      const target = template.spec.owner;
+      if (target) {
+        const targetRef = catalogModel.parseEntityRef(target, {
+          defaultKind: "Group",
+          defaultNamespace: selfRef.namespace
+        });
+        emit(pluginCatalogBackend.results.relation({
+          source: selfRef,
+          type: catalogModel.RELATION_OWNED_BY,
+          target: {
+            kind: targetRef.kind,
+            namespace: targetRef.namespace,
+            name: targetRef.name
+          }
+        }));
+        emit(pluginCatalogBackend.results.relation({
+          source: {
+            kind: targetRef.kind,
+            namespace: targetRef.namespace,
+            name: targetRef.name
+          },
+          type: catalogModel.RELATION_OWNER_OF,
+          target: selfRef
+        }));
+      }
+    }
+    return entity;
+  }
+}
+
 Object.defineProperty(exports, 'createFetchCookiecutterAction', {
   enumerable: true,
   get: function () {
@@ -2422,6 +2673,7 @@ Object.defineProperty(exports, 'createFetchCookiecutterAction', {
 });
 exports.CatalogEntityClient = CatalogEntityClient;
 exports.OctokitProvider = OctokitProvider;
+exports.ScaffolderEntitiesProcessor = ScaffolderEntitiesProcessor;
 exports.TemplateActionRegistry = TemplateActionRegistry;
 exports.createBuiltinActions = createBuiltinActions;
 exports.createCatalogRegisterAction = createCatalogRegisterAction;