@backstage/plugin-scaffolder-backend 0.15.11 → 0.15.12

package/CHANGELOG.md

@@ -1,5 +1,55 @@
 # @backstage/plugin-scaffolder-backend
 
+## 0.15.12
+
+### Patch Changes
+
+- 9990df8a1f: Expose some classes and interfaces public so TaskWorkers can run externally from the scaffolder API.
+- b45a34fb15: Adds a new endpoint for consuming logs from the Scaffolder that uses long polling instead of Server Sent Events.
+
+  This is useful if Backstage is accessed from an environment that doesn't support SSE correctly, which happens in combination with certain enterprise HTTP Proxy servers.
+
+  It is intended to switch the endpoint globally for the whole instance.
+  If you want to use it, you can provide a reconfigured API to the `scaffolderApiRef`:
+
+  ```tsx
+  // packages/app/src/apis.ts
+
+  // ...
+  import {
+    scaffolderApiRef,
+    ScaffolderClient,
+  } from '@backstage/plugin-scaffolder';
+
+  export const apis: AnyApiFactory[] = [
+    // ...
+
+    createApiFactory({
+      api: scaffolderApiRef,
+      deps: {
+        discoveryApi: discoveryApiRef,
+        identityApi: identityApiRef,
+        scmIntegrationsApi: scmIntegrationsApiRef,
+      },
+      factory: ({ discoveryApi, identityApi, scmIntegrationsApi }) =>
+        new ScaffolderClient({
+          discoveryApi,
+          identityApi,
+          scmIntegrationsApi,
+          // use long polling instead of an eventsource
+          useLongPollingLogs: true,
+        }),
+    }),
+  ];
+  ```
+
+- a794c341ca: Fix a bug where only file mode 775 is considered an executable
+- Updated dependencies
+  - @backstage/backend-common@0.9.9
+  - @backstage/catalog-client@0.5.1
+  - @backstage/plugin-catalog-backend@0.17.3
+  - @backstage/plugin-scaffolder-backend-module-cookiecutter@0.1.4
+
 ## 0.15.11
 
 ### Patch Changes

package/dist/index.cjs.js

@@ -22,14 +22,14 @@ var lodash = require('lodash');
 var octokitPluginCreatePullRequest = require('octokit-plugin-create-pull-request');
 var node = require('@gitbeaker/node');
 var webhooks = require('@octokit/webhooks');
-var express = require('express');
-var Router = require('express-promise-router');
-var jsonschema = require('jsonschema');
 var uuid = require('uuid');
 var luxon = require('luxon');
-var os = require('os');
 var Handlebars = require('handlebars');
 var winston = require('winston');
+var jsonschema = require('jsonschema');
+var express = require('express');
+var Router = require('express-promise-router');
+var os = require('os');
 var pluginCatalogBackend = require('@backstage/plugin-catalog-backend');
 var pluginScaffolderCommon = require('@backstage/plugin-scaffolder-common');
 
@@ -62,11 +62,11 @@ var yaml__namespace = /*#__PURE__*/_interopNamespace(yaml);
 var globby__default = /*#__PURE__*/_interopDefaultLegacy(globby);
 var nunjucks__default = /*#__PURE__*/_interopDefaultLegacy(nunjucks);
 var fetch__default = /*#__PURE__*/_interopDefaultLegacy(fetch);
+var Handlebars__namespace = /*#__PURE__*/_interopNamespace(Handlebars);
+var winston__namespace = /*#__PURE__*/_interopNamespace(winston);
 var express__default = /*#__PURE__*/_interopDefaultLegacy(express);
 var Router__default = /*#__PURE__*/_interopDefaultLegacy(Router);
 var os__default = /*#__PURE__*/_interopDefaultLegacy(os);
-var Handlebars__namespace = /*#__PURE__*/_interopNamespace(Handlebars);
-var winston__namespace = /*#__PURE__*/_interopNamespace(winston);
 
 const createTemplateAction = (templateAction) => {
   return templateAction;
@@ -714,6 +714,11 @@ const parseRepoUrl = (repoUrl, integrations) => {
   }
   return {host, owner, repo, organization, workspace, project};
 };
+const isExecutable = (fileMode) => {
+  const executeBitMask = 73;
+  const res = fileMode & executeBitMask;
+  return res > 0;
+};
 
 function createPublishAzureAction(options) {
   const {integrations, config} = options;
@@ -1395,8 +1400,7 @@ const createPublishGithubPullRequestAction = ({
         const absPath = path__default['default'].resolve(fileRoot, filePath);
         const base64EncodedContent = fs__default['default'].readFileSync(absPath).toString("base64");
         const fileStat = fs__default['default'].statSync(absPath);
-        const isExecutable = fileStat.mode === 33277;
-        const githubTreeItemMode = isExecutable ? "100755" : "100644";
+        const githubTreeItemMode = isExecutable(fileStat.mode) ? "100755" : "100644";
         const encoding = "base64";
         return {
           encoding,
@@ -1747,38 +1751,16 @@ class TemplateActionRegistry {
   }
 }
 
-class CatalogEntityClient {
-  constructor(catalogClient) {
-    this.catalogClient = catalogClient;
-  }
-  async findTemplate(templateName, options) {
-    const {items: templates} = await this.catalogClient.getEntities({
-      filter: {
-        kind: "template",
-        "metadata.name": templateName
-      }
-    }, options);
-    if (templates.length !== 1) {
-      if (templates.length > 1) {
-        throw new errors.ConflictError("Templates lookup resulted in multiple matches");
-      } else {
-        throw new errors.NotFoundError("Template not found");
-      }
-    }
-    return templates[0];
-  }
-}
-
 const migrationsDir = backendCommon.resolvePackagePath("@backstage/plugin-scaffolder-backend", "migrations");
 class DatabaseTaskStore {
-  constructor(db) {
-    this.db = db;
-  }
-  static async create(knex) {
-    await knex.migrate.latest({
+  static async create(options) {
+    await options.database.migrate.latest({
       directory: migrationsDir
     });
-    return new DatabaseTaskStore(knex);
+    return new DatabaseTaskStore(options);
+  }
+  constructor(options) {
+    this.db = options.database;
   }
   async getTask(taskId) {
     const [result] = await this.db("tasks").where({id: taskId}).select();
@@ -1934,7 +1916,7 @@ class DatabaseTaskStore {
   }
 }
 
-class TaskAgent {
+class TaskManager {
   constructor(state, storage, logger) {
     this.state = state;
     this.storage = storage;
@@ -1942,7 +1924,7 @@ class TaskAgent {
     this.isDone = false;
   }
   static create(state, storage, logger) {
-    const agent = new TaskAgent(state, storage, logger);
+    const agent = new TaskManager(state, storage, logger);
     agent.startTimeout();
     return agent;
   }
@@ -2008,7 +1990,7 @@ class StorageTaskBroker {
     for (; ; ) {
       const pendingTask = await this.storage.claimTask();
       if (pendingTask) {
-        return TaskAgent.create({
+        return TaskManager.create({
           taskId: pendingTask.id,
           spec: pendingTask.spec,
           secrets: pendingTask.secrets
@@ -2050,7 +2032,7 @@ class StorageTaskBroker {
         await new Promise((resolve) => setTimeout(resolve, 1e3));
       }
     })();
-    return unsubscribe;
+    return {unsubscribe};
   }
   async vacuumTasks(timeoutS) {
     const {tasks} = await this.storage.listStaleTasks(timeoutS);
@@ -2077,70 +2059,12 @@ class StorageTaskBroker {
   }
 }
 
-class TaskWorker {
-  constructor(options) {
-    this.options = options;
-  }
-  start() {
-    (async () => {
-      for (; ; ) {
-        const task = await this.options.taskBroker.claim();
-        await this.runOneTask(task);
-      }
-    })();
-  }
-  async runOneTask(task) {
-    try {
-      const {output} = task.spec.apiVersion === "scaffolder.backstage.io/v1beta3" ? await this.options.runners.workflowRunner.execute(task) : await this.options.runners.legacyWorkflowRunner.execute(task);
-      await task.complete("completed", {output});
-    } catch (error) {
-      errors.assertError(error);
-      await task.complete("failed", {
-        error: {name: error.name, message: error.message}
-      });
-    }
-  }
-}
-
-async function getWorkingDirectory(config, logger) {
-  if (!config.has("backend.workingDirectory")) {
-    return os__default['default'].tmpdir();
-  }
-  const workingDirectory = config.getString("backend.workingDirectory");
-  try {
-    await fs__default['default'].access(workingDirectory, fs__default['default'].constants.F_OK | fs__default['default'].constants.W_OK);
-    logger.info(`using working directory: ${workingDirectory}`);
-  } catch (err) {
-    errors.assertError(err);
-    logger.error(`working directory ${workingDirectory} ${err.code === "ENOENT" ? "does not exist" : "is not writable"}`);
-    throw err;
-  }
-  return workingDirectory;
-}
-function getEntityBaseUrl(entity) {
-  var _a, _b;
-  let location = (_a = entity.metadata.annotations) == null ? void 0 : _a[catalogModel.SOURCE_LOCATION_ANNOTATION];
-  if (!location) {
-    location = (_b = entity.metadata.annotations) == null ? void 0 : _b[catalogModel.LOCATION_ANNOTATION];
-  }
-  if (!location) {
-    return void 0;
-  }
-  const {type, target} = catalogModel.parseLocationReference(location);
-  if (type === "url") {
-    return target;
-  } else if (type === "file") {
-    return `file://${target}`;
-  }
-  return void 0;
-}
-
 function isTruthy(value) {
   return lodash.isArray(value) ? value.length > 0 : !!value;
 }
 
 const isValidTaskSpec$1 = (taskSpec) => taskSpec.apiVersion === "backstage.io/v1beta2";
-class LegacyWorkflowRunner {
+class HandlebarsWorkflowRunner {
   constructor(options) {
     this.options = options;
     this.handlebars = Handlebars__namespace.create();
@@ -2316,7 +2240,10 @@ class LegacyWorkflowRunner {
 const isValidTaskSpec = (taskSpec) => {
   return taskSpec.apiVersion === "scaffolder.backstage.io/v1beta3";
 };
-const createStepLogger = ({task, step}) => {
+const createStepLogger = ({
+  task,
+  step
+}) => {
   const metadata = {stepId: step.id};
   const taskLogger = winston__namespace.createLogger({
     level: process.env.LOG_LEVEL || "info",
@@ -2333,7 +2260,7 @@ const createStepLogger = ({task, step}) => {
   taskLogger.add(new winston__namespace.transports.Stream({stream: streamLogger}));
   return {taskLogger, streamLogger};
 };
-class DefaultWorkflowRunner {
+class NunjucksWorkflowRunner {
   constructor(options) {
     this.options = options;
     this.nunjucksOptions = {
@@ -2464,6 +2391,111 @@ class DefaultWorkflowRunner {
   }
 }
 
+class TaskWorker {
+  constructor(options) {
+    this.options = options;
+  }
+  static async create(options) {
+    const {
+      taskBroker,
+      logger,
+      actionRegistry,
+      integrations,
+      workingDirectory
+    } = options;
+    const legacyWorkflowRunner = new HandlebarsWorkflowRunner({
+      logger,
+      actionRegistry,
+      integrations,
+      workingDirectory
+    });
+    const workflowRunner = new NunjucksWorkflowRunner({
+      actionRegistry,
+      integrations,
+      logger,
+      workingDirectory
+    });
+    return new TaskWorker({
+      taskBroker,
+      runners: {legacyWorkflowRunner, workflowRunner}
+    });
+  }
+  start() {
+    (async () => {
+      for (; ; ) {
+        const task = await this.options.taskBroker.claim();
+        await this.runOneTask(task);
+      }
+    })();
+  }
+  async runOneTask(task) {
+    try {
+      const {output} = task.spec.apiVersion === "scaffolder.backstage.io/v1beta3" ? await this.options.runners.workflowRunner.execute(task) : await this.options.runners.legacyWorkflowRunner.execute(task);
+      await task.complete("completed", {output});
+    } catch (error) {
+      errors.assertError(error);
+      await task.complete("failed", {
+        error: {name: error.name, message: error.message}
+      });
+    }
+  }
+}
+
+class CatalogEntityClient {
+  constructor(catalogClient) {
+    this.catalogClient = catalogClient;
+  }
+  async findTemplate(templateName, options) {
+    const {items: templates} = await this.catalogClient.getEntities({
+      filter: {
+        kind: "template",
+        "metadata.name": templateName
+      }
+    }, options);
+    if (templates.length !== 1) {
+      if (templates.length > 1) {
+        throw new errors.ConflictError("Templates lookup resulted in multiple matches");
+      } else {
+        throw new errors.NotFoundError("Template not found");
+      }
+    }
+    return templates[0];
+  }
+}
+
+async function getWorkingDirectory(config, logger) {
+  if (!config.has("backend.workingDirectory")) {
+    return os__default['default'].tmpdir();
+  }
+  const workingDirectory = config.getString("backend.workingDirectory");
+  try {
+    await fs__default['default'].access(workingDirectory, fs__default['default'].constants.F_OK | fs__default['default'].constants.W_OK);
+    logger.info(`using working directory: ${workingDirectory}`);
+  } catch (err) {
+    errors.assertError(err);
+    logger.error(`working directory ${workingDirectory} ${err.code === "ENOENT" ? "does not exist" : "is not writable"}`);
+    throw err;
+  }
+  return workingDirectory;
+}
+function getEntityBaseUrl(entity) {
+  var _a, _b;
+  let location = (_a = entity.metadata.annotations) == null ? void 0 : _a[catalogModel.SOURCE_LOCATION_ANNOTATION];
+  if (!location) {
+    location = (_b = entity.metadata.annotations) == null ? void 0 : _b[catalogModel.LOCATION_ANNOTATION];
+  }
+  if (!location) {
+    return void 0;
+  }
+  const {type, target} = catalogModel.parseLocationReference(location);
+  if (type === "url") {
+    return target;
+  } else if (type === "file") {
+    return `file://${target}`;
+  }
+  return void 0;
+}
+
 function isSupportedTemplate(entity) {
   return entity.apiVersion === "backstage.io/v1beta2" || entity.apiVersion === "scaffolder.backstage.io/v1beta3";
 }
@@ -2484,29 +2516,24 @@ async function createRouter(options) {
   const workingDirectory = await getWorkingDirectory(config, logger);
   const entityClient = new CatalogEntityClient(catalogClient);
   const integrations = integration.ScmIntegrations.fromConfig(config);
-  const databaseTaskStore = await DatabaseTaskStore.create(await database.getClient());
-  const taskBroker = new StorageTaskBroker(databaseTaskStore, logger);
+  let taskBroker;
+  if (!options.taskBroker) {
+    const databaseTaskStore = await DatabaseTaskStore.create({
+      database: await database.getClient()
+    });
+    taskBroker = new StorageTaskBroker(databaseTaskStore, logger);
+  } else {
+    taskBroker = options.taskBroker;
+  }
   const actionRegistry = new TemplateActionRegistry();
-  const legacyWorkflowRunner = new LegacyWorkflowRunner({
-    logger,
-    actionRegistry,
-    integrations,
-    workingDirectory
-  });
-  const workflowRunner = new DefaultWorkflowRunner({
-    actionRegistry,
-    integrations,
-    logger,
-    workingDirectory
-  });
   const workers = [];
   for (let i = 0; i < (taskWorkers || 1); i++) {
-    const worker = new TaskWorker({
+    const worker = await TaskWorker.create({
       taskBroker,
-      runners: {
-        legacyWorkflowRunner,
-        workflowRunner
-      }
+      actionRegistry,
+      integrations,
+      logger,
+      workingDirectory
     });
     workers.push(worker);
   }
@@ -2617,14 +2644,15 @@ async function createRouter(options) {
     res.status(200).json(task);
   }).get("/v2/tasks/:taskId/eventstream", async (req, res) => {
     const {taskId} = req.params;
-    const after = Number(req.query.after) || void 0;
+    const after = req.query.after !== void 0 ? Number(req.query.after) : void 0;
     logger.debug(`Event stream observing taskId '${taskId}' opened`);
     res.writeHead(200, {
       Connection: "keep-alive",
       "Cache-Control": "no-cache",
       "Content-Type": "text/event-stream"
     });
-    const unsubscribe = taskBroker.observe({taskId, after}, (error, {events}) => {
+    const {unsubscribe} = taskBroker.observe({taskId, after}, (error, {events}) => {
+      var _a;
       if (error) {
         logger.error(`Received error from event stream when observing taskId '${taskId}', ${error}`);
       }
@@ -2638,7 +2666,7 @@ data: ${JSON.stringify(event)}
           shouldUnsubscribe = true;
         }
       }
-      res.flush();
+      (_a = res.flush) == null ? void 0 : _a.call(res);
       if (shouldUnsubscribe)
         unsubscribe();
     });
@@ -2646,6 +2674,27 @@ data: ${JSON.stringify(event)}
       unsubscribe();
       logger.debug(`Event stream observing taskId '${taskId}' closed`);
     });
+  }).get("/v2/tasks/:taskId/events", async (req, res) => {
+    const {taskId} = req.params;
+    const after = Number(req.query.after) || void 0;
+    let unsubscribe = () => {
+    };
+    const timeout = setTimeout(() => {
+      unsubscribe();
+      res.json([]);
+    }, 3e4);
+    ({unsubscribe} = taskBroker.observe({taskId, after}, (error, {events}) => {
+      clearTimeout(timeout);
+      unsubscribe();
+      if (error) {
+        logger.error(`Received error from log when observing taskId '${taskId}', ${error}`);
+      }
+      res.json(events);
+    }));
+    req.on("close", () => {
+      unsubscribe();
+      clearTimeout(timeout);
+    });
   });
   const app = express__default['default']();
   app.set("logger", logger);
@@ -2712,8 +2761,11 @@ Object.defineProperty(exports, 'createFetchCookiecutterAction', {
   }
 });
 exports.CatalogEntityClient = CatalogEntityClient;
+exports.DatabaseTaskStore = DatabaseTaskStore;
 exports.OctokitProvider = OctokitProvider;
 exports.ScaffolderEntitiesProcessor = ScaffolderEntitiesProcessor;
+exports.TaskManager = TaskManager;
+exports.TaskWorker = TaskWorker;
 exports.TemplateActionRegistry = TemplateActionRegistry;
 exports.createBuiltinActions = createBuiltinActions;
 exports.createCatalogRegisterAction = createCatalogRegisterAction;