@backstage/plugin-scaffolder-backend 1.6.0 → 1.7.0-next.0

package/CHANGELOG.md

@@ -1,5 +1,31 @@
 # @backstage/plugin-scaffolder-backend
 
+## 1.7.0-next.0
+
+### Minor Changes
+
+- 253453fa14: Added a new property called `additionalTemplateGlobals` which allows you to add global functions to the scaffolder nunjucks templates.
+- 304305dd20: Add `allowAutoMerge` option for `publish:github` action
+- 694bfe2d61: Add functionality to shutdown scaffolder tasks if they are stale
+
+### Patch Changes
+
+- b681275e69: Ignore .git directories in Template Editor, increase upload limit for dry-runs to 10MB.
+- Updated dependencies
+  - @backstage/catalog-model@1.1.2-next.0
+  - @backstage/backend-plugin-api@0.1.3-next.0
+  - @backstage/plugin-catalog-backend@1.4.1-next.0
+  - @backstage/catalog-client@1.1.1-next.0
+  - @backstage/plugin-catalog-node@1.1.1-next.0
+  - @backstage/plugin-scaffolder-common@1.2.1-next.0
+  - @backstage/backend-common@0.15.2-next.0
+  - @backstage/backend-tasks@0.3.6-next.0
+  - @backstage/plugin-auth-node@0.2.6-next.0
+  - @backstage/config@1.0.3-next.0
+  - @backstage/errors@1.1.2-next.0
+  - @backstage/integration@1.3.2-next.0
+  - @backstage/types@1.0.0
+
 ## 1.6.0
 
 ### Minor Changes

package/alpha/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-scaffolder-backend",
-  "version": "1.6.0",
+  "version": "1.7.0-next.0",
   "main": "../dist/index.cjs.js",
   "types": "../dist/index.alpha.d.ts"
 }

package/dist/index.alpha.d.ts

@@ -24,6 +24,7 @@ import { Logger } from 'winston';
 import { Observable } from '@backstage/types';
 import { Octokit } from 'octokit';
 import { PluginDatabaseManager } from '@backstage/backend-common';
+import { PluginTaskScheduler } from '@backstage/backend-tasks';
 import { Schema } from 'jsonschema';
 import { ScmIntegrationRegistry } from '@backstage/integration';
 import { ScmIntegrations } from '@backstage/integration';
@@ -106,6 +107,7 @@ export declare interface CreateBuiltInActionsOptions {
      * Template Manifests and also template skeleton files when using `fetch:template`.
      */
     additionalTemplateFilters?: Record<string, TemplateFilter>;
+    additionalTemplateGlobals?: Record<string, TemplateGlobal>;
 }
 
 /**
@@ -172,6 +174,7 @@ export declare function createFetchTemplateAction(options: {
     reader: UrlReader;
     integrations: ScmIntegrations;
     additionalTemplateFilters?: Record<string, TemplateFilter>;
+    additionalTemplateGlobals?: Record<string, TemplateGlobal>;
 }): TemplateAction<    {
 url: string;
 targetPath?: string | undefined;
@@ -287,6 +290,7 @@ gitAuthorEmail?: string | undefined;
 allowRebaseMerge?: boolean | undefined;
 allowSquashMerge?: boolean | undefined;
 allowMergeCommit?: boolean | undefined;
+allowAutoMerge?: boolean | undefined;
 requireCodeOwnerReviews?: boolean | undefined;
 requiredStatusCheckContexts?: string[] | undefined;
 repoVisibility?: "internal" | "private" | "public" | undefined;
@@ -484,6 +488,7 @@ gitAuthorEmail?: string | undefined;
 allowRebaseMerge?: boolean | undefined;
 allowSquashMerge?: boolean | undefined;
 allowMergeCommit?: boolean | undefined;
+allowAutoMerge?: boolean | undefined;
 sourcePath?: string | undefined;
 requireCodeOwnerReviews?: boolean | undefined;
 requiredStatusCheckContexts?: string[] | undefined;
@@ -586,6 +591,7 @@ export declare type CreateWorkerOptions = {
     workingDirectory: string;
     logger: Logger;
     additionalTemplateFilters?: Record<string, TemplateFilter>;
+    additionalTemplateGlobals?: Record<string, TemplateGlobal>;
 };
 
 /**
@@ -650,6 +656,7 @@ export declare class DatabaseTaskStore implements TaskStore {
     listEvents(options: TaskStoreListEventsOptions): Promise<{
         events: SerializedTaskEvent[];
     }>;
+    shutdownTask({ taskId }: TaskStoreShutDownTaskOptions): Promise<void>;
 }
 
 /**
@@ -703,10 +710,12 @@ export declare interface RouterOptions {
     reader: UrlReader;
     database: PluginDatabaseManager;
     catalogClient: CatalogApi;
+    scheduler?: PluginTaskScheduler;
     actions?: TemplateAction<any>[];
     taskWorkers?: number;
     taskBroker?: TaskBroker;
     additionalTemplateFilters?: Record<string, TemplateFilter>;
+    additionalTemplateGlobals?: Record<string, TemplateGlobal>;
     identity?: IdentityApi;
 }
 
@@ -751,6 +760,7 @@ export declare type ScaffolderPluginOptions = {
     taskWorkers?: number;
     taskBroker?: TaskBroker;
     additionalTemplateFilters?: Record<string, TemplateFilter>;
+    additionalTemplateGlobals?: Record<string, TemplateGlobal>;
 };
 
 /**
@@ -927,6 +937,7 @@ export declare interface TaskStore {
     listEvents({ taskId, after, }: TaskStoreListEventsOptions): Promise<{
         events: SerializedTaskEvent[];
     }>;
+    shutdownTask?({ taskId }: TaskStoreShutDownTaskOptions): Promise<void>;
 }
 
 /**
@@ -967,6 +978,15 @@ export declare type TaskStoreListEventsOptions = {
     after?: number | undefined;
 };
 
+/**
+ * TaskStoreShutDownTaskOptions
+ *
+ * @public
+ */
+export declare type TaskStoreShutDownTaskOptions = {
+    taskId: string;
+};
+
 /**
  * TaskWorker
  *
@@ -1006,4 +1026,7 @@ export declare class TemplateActionRegistry {
 /** @public */
 export declare type TemplateFilter = (...args: JsonValue[]) => JsonValue | undefined;
 
+/** @public */
+export declare type TemplateGlobal = ((...args: JsonValue[]) => JsonValue | undefined) | JsonValue;
+
 export { }

package/dist/index.beta.d.ts

@@ -24,6 +24,7 @@ import { Logger } from 'winston';
 import { Observable } from '@backstage/types';
 import { Octokit } from 'octokit';
 import { PluginDatabaseManager } from '@backstage/backend-common';
+import { PluginTaskScheduler } from '@backstage/backend-tasks';
 import { Schema } from 'jsonschema';
 import { ScmIntegrationRegistry } from '@backstage/integration';
 import { ScmIntegrations } from '@backstage/integration';
@@ -106,6 +107,7 @@ export declare interface CreateBuiltInActionsOptions {
      * Template Manifests and also template skeleton files when using `fetch:template`.
      */
     additionalTemplateFilters?: Record<string, TemplateFilter>;
+    additionalTemplateGlobals?: Record<string, TemplateGlobal>;
 }
 
 /**
@@ -172,6 +174,7 @@ export declare function createFetchTemplateAction(options: {
     reader: UrlReader;
     integrations: ScmIntegrations;
     additionalTemplateFilters?: Record<string, TemplateFilter>;
+    additionalTemplateGlobals?: Record<string, TemplateGlobal>;
 }): TemplateAction<    {
 url: string;
 targetPath?: string | undefined;
@@ -287,6 +290,7 @@ gitAuthorEmail?: string | undefined;
 allowRebaseMerge?: boolean | undefined;
 allowSquashMerge?: boolean | undefined;
 allowMergeCommit?: boolean | undefined;
+allowAutoMerge?: boolean | undefined;
 requireCodeOwnerReviews?: boolean | undefined;
 requiredStatusCheckContexts?: string[] | undefined;
 repoVisibility?: "internal" | "private" | "public" | undefined;
@@ -484,6 +488,7 @@ gitAuthorEmail?: string | undefined;
 allowRebaseMerge?: boolean | undefined;
 allowSquashMerge?: boolean | undefined;
 allowMergeCommit?: boolean | undefined;
+allowAutoMerge?: boolean | undefined;
 sourcePath?: string | undefined;
 requireCodeOwnerReviews?: boolean | undefined;
 requiredStatusCheckContexts?: string[] | undefined;
@@ -586,6 +591,7 @@ export declare type CreateWorkerOptions = {
     workingDirectory: string;
     logger: Logger;
     additionalTemplateFilters?: Record<string, TemplateFilter>;
+    additionalTemplateGlobals?: Record<string, TemplateGlobal>;
 };
 
 /**
@@ -650,6 +656,7 @@ export declare class DatabaseTaskStore implements TaskStore {
     listEvents(options: TaskStoreListEventsOptions): Promise<{
         events: SerializedTaskEvent[];
     }>;
+    shutdownTask({ taskId }: TaskStoreShutDownTaskOptions): Promise<void>;
 }
 
 /**
@@ -703,10 +710,12 @@ export declare interface RouterOptions {
     reader: UrlReader;
     database: PluginDatabaseManager;
     catalogClient: CatalogApi;
+    scheduler?: PluginTaskScheduler;
     actions?: TemplateAction<any>[];
     taskWorkers?: number;
     taskBroker?: TaskBroker;
     additionalTemplateFilters?: Record<string, TemplateFilter>;
+    additionalTemplateGlobals?: Record<string, TemplateGlobal>;
     identity?: IdentityApi;
 }
 
@@ -910,6 +919,7 @@ export declare interface TaskStore {
     listEvents({ taskId, after, }: TaskStoreListEventsOptions): Promise<{
         events: SerializedTaskEvent[];
     }>;
+    shutdownTask?({ taskId }: TaskStoreShutDownTaskOptions): Promise<void>;
 }
 
 /**
@@ -950,6 +960,15 @@ export declare type TaskStoreListEventsOptions = {
     after?: number | undefined;
 };
 
+/**
+ * TaskStoreShutDownTaskOptions
+ *
+ * @public
+ */
+export declare type TaskStoreShutDownTaskOptions = {
+    taskId: string;
+};
+
 /**
  * TaskWorker
  *
@@ -989,4 +1008,7 @@ export declare class TemplateActionRegistry {
 /** @public */
 export declare type TemplateFilter = (...args: JsonValue[]) => JsonValue | undefined;
 
+/** @public */
+export declare type TemplateGlobal = ((...args: JsonValue[]) => JsonValue | undefined) | JsonValue;
+
 export { }

package/dist/index.cjs.js

@@ -403,6 +403,16 @@ const { render, renderCompat } = (() => {
     }
   }
 
+  if (typeof additionalTemplateGlobals !== 'undefined') {
+    for (const [globalName, global] of Object.entries(additionalTemplateGlobals)) {
+      if (typeof global === 'function') {
+        env.addGlobal(globalName, (...args) => JSON.parse(global(...args)));
+      } else {
+        env.addGlobal(globalName, JSON.parse(global));
+      }
+    }
+  }
+
   let uninstallCompat = undefined;
 
   function render(str, values) {
@@ -435,7 +445,12 @@ const { render, renderCompat } = (() => {
 `;
 class SecureTemplater {
   static async loadRenderer(options = {}) {
-    const { parseRepoUrl, cookiecutterCompat, additionalTemplateFilters } = options;
+    const {
+      parseRepoUrl,
+      cookiecutterCompat,
+      additionalTemplateFilters,
+      additionalTemplateGlobals
+    } = options;
     const sandbox = {};
     if (parseRepoUrl) {
       sandbox.parseRepoUrl = (url) => JSON.stringify(parseRepoUrl(url));
@@ -448,6 +463,19 @@ class SecureTemplater {
         ])
       );
     }
+    if (additionalTemplateGlobals) {
+      sandbox.additionalTemplateGlobals = Object.fromEntries(
+        Object.entries(additionalTemplateGlobals).filter(([_, global]) => !!global).map(([globalName, global]) => {
+          if (typeof global === "function") {
+            return [
+              globalName,
+              (...args) => JSON.stringify(global(...args))
+            ];
+          }
+          return [globalName, JSON.stringify(global)];
+        })
+      );
+    }
     const vm = new vm2.VM({ sandbox });
     const nunjucksSource = await fs__default["default"].readFile(
       backendCommon.resolvePackagePath(
@@ -473,7 +501,12 @@ class SecureTemplater {
 }
 
 function createFetchTemplateAction(options) {
-  const { reader, integrations, additionalTemplateFilters } = options;
+  const {
+    reader,
+    integrations,
+    additionalTemplateFilters,
+    additionalTemplateGlobals
+  } = options;
   return createTemplateAction({
     id: "fetch:template",
     description: "Downloads a skeleton, templates variables into file and directory names and content, and places the result in the workspace, or optionally in a subdirectory specified by the 'targetPath' input option.",
@@ -606,7 +639,8 @@ function createFetchTemplateAction(options) {
       );
       const renderTemplate = await SecureTemplater.loadRenderer({
         cookiecutterCompat: ctx.input.cookiecutterCompat,
-        additionalTemplateFilters
+        additionalTemplateFilters,
+        additionalTemplateGlobals
       });
       for (const location of allEntriesInTemplate) {
         let renderContents;
@@ -1039,7 +1073,7 @@ async function getOctokitOptions(options) {
     previews: ["nebula-preview"]
   };
 }
-async function createGithubRepoWithCollaboratorsAndTopics(client, repo, owner, repoVisibility, description, homepage, deleteBranchOnMerge, allowMergeCommit, allowSquashMerge, allowRebaseMerge, access, collaborators, topics, logger) {
+async function createGithubRepoWithCollaboratorsAndTopics(client, repo, owner, repoVisibility, description, homepage, deleteBranchOnMerge, allowMergeCommit, allowSquashMerge, allowRebaseMerge, allowAutoMerge, access, collaborators, topics, logger) {
   const user = await client.rest.users.getByUsername({
     username: owner
   });
@@ -1053,6 +1087,7 @@ async function createGithubRepoWithCollaboratorsAndTopics(client, repo, owner, r
     allow_merge_commit: allowMergeCommit,
     allow_squash_merge: allowSquashMerge,
     allow_rebase_merge: allowRebaseMerge,
+    allow_auto_merge: allowAutoMerge,
     homepage
   }) : client.rest.repos.createForAuthenticatedUser({
     name: repo,
@@ -1062,6 +1097,7 @@ async function createGithubRepoWithCollaboratorsAndTopics(client, repo, owner, r
     allow_merge_commit: allowMergeCommit,
     allow_squash_merge: allowSquashMerge,
     allow_rebase_merge: allowRebaseMerge,
+    allow_auto_merge: allowAutoMerge,
     homepage
   });
   let newRepo;
@@ -1390,6 +1426,11 @@ const allowRebaseMerge = {
   type: "boolean",
   description: `Allow rebase merges. The default value is 'true'`
 };
+const allowAutoMerge = {
+  title: "Allow Auto Merges",
+  type: "boolean",
+  description: `Allow individual PRs to merge automatically when all merge requirements are met. The default value is 'false'`
+};
 const collaborators = {
   title: "Collaborators",
   description: "Provide additional users or teams with permissions",
@@ -1484,6 +1525,7 @@ function createGithubRepoCreateAction(options) {
           allowMergeCommit: allowMergeCommit,
           allowSquashMerge: allowSquashMerge,
           allowRebaseMerge: allowRebaseMerge,
+          allowAutoMerge: allowAutoMerge,
           collaborators: collaborators,
           token: token,
           topics: topics
@@ -1508,6 +1550,7 @@ function createGithubRepoCreateAction(options) {
         allowMergeCommit = true,
         allowSquashMerge = true,
         allowRebaseMerge = true,
+        allowAutoMerge = false,
         collaborators,
         topics,
         token: providedToken
@@ -1534,6 +1577,7 @@ function createGithubRepoCreateAction(options) {
         allowMergeCommit,
         allowSquashMerge,
         allowRebaseMerge,
+        allowAutoMerge,
         access,
         collaborators,
         topics,
@@ -2841,6 +2885,7 @@ function createPublishGithubAction(options) {
           allowMergeCommit: allowMergeCommit,
           allowSquashMerge: allowSquashMerge,
           allowRebaseMerge: allowRebaseMerge,
+          allowAutoMerge: allowAutoMerge,
           sourcePath: sourcePath,
           collaborators: collaborators,
           token: token,
@@ -2874,6 +2919,7 @@ function createPublishGithubAction(options) {
         allowMergeCommit = true,
         allowSquashMerge = true,
         allowRebaseMerge = true,
+        allowAutoMerge = false,
         collaborators,
         topics,
         token: providedToken
@@ -2900,6 +2946,7 @@ function createPublishGithubAction(options) {
         allowMergeCommit,
         allowSquashMerge,
         allowRebaseMerge,
+        allowAutoMerge,
         access,
         collaborators,
         topics,
@@ -3551,7 +3598,8 @@ const createBuiltinActions = (options) => {
     integrations,
     catalogClient,
     config,
-    additionalTemplateFilters
+    additionalTemplateFilters,
+    additionalTemplateGlobals
   } = options;
   const githubCredentialsProvider = integration.DefaultGithubCredentialsProvider.fromIntegrations(integrations);
   const actions = [
@@ -3562,7 +3610,8 @@ const createBuiltinActions = (options) => {
     createFetchTemplateAction({
       integrations,
       reader,
-      additionalTemplateFilters
+      additionalTemplateFilters,
+      additionalTemplateGlobals
     }),
     createPublishGerritAction({
       integrations,
@@ -3805,8 +3854,7 @@ class DatabaseTaskStore {
     const rawRows = await this.db("tasks").where("status", "processing").andWhere(
       "last_heartbeat_at",
       "<=",
-      this.db.client.config.client.includes("sqlite3") ? this.db.raw(`datetime('now', ?)`, [`-${timeoutS} seconds`]) : this.db.raw(`dateadd('second', ?, ?)`, [
-        `-${timeoutS}`,
+      this.db.client.config.client.includes("sqlite3") ? this.db.raw(`datetime('now', ?)`, [`-${timeoutS} seconds`]) : this.db.raw(`? - interval '${timeoutS} seconds'`, [
         this.db.fn.now()
       ])
     );
@@ -3891,6 +3939,33 @@ class DatabaseTaskStore {
     });
     return { events };
   }
+  async shutdownTask({ taskId }) {
+    const message = `This task was marked as stale as it exceeded its timeout`;
+    const statusStepEvents = (await this.listEvents({ taskId })).events.filter(
+      ({ body }) => body == null ? void 0 : body.stepId
+    );
+    const completedSteps = statusStepEvents.filter(
+      ({ body: { status } }) => status === "failed" || status === "completed"
+    ).map((step) => step.body.stepId);
+    const hungProcessingSteps = statusStepEvents.filter(({ body: { status } }) => status === "processing").map((event) => event.body.stepId).filter((step) => !completedSteps.includes(step));
+    for (const step of hungProcessingSteps) {
+      await this.emitLogEvent({
+        taskId,
+        body: {
+          message,
+          stepId: step,
+          status: "failed"
+        }
+      });
+    }
+    await this.completeTask({
+      taskId,
+      status: "failed",
+      eventBody: {
+        message
+      }
+    });
+  }
 }
 
 class TaskManager {
@@ -4181,7 +4256,8 @@ class NunjucksWorkflowRunner {
       parseRepoUrl(url) {
         return parseRepoUrl(url, integrations);
       },
-      additionalTemplateFilters: this.options.additionalTemplateFilters
+      additionalTemplateFilters: this.options.additionalTemplateFilters,
+      additionalTemplateGlobals: this.options.additionalTemplateGlobals
     });
     try {
       await fs__default["default"].ensureDir(workspacePath);
@@ -4332,14 +4408,16 @@ class TaskWorker {
       actionRegistry,
       integrations,
       workingDirectory,
-      additionalTemplateFilters
+      additionalTemplateFilters,
+      additionalTemplateGlobals
     } = options;
     const workflowRunner = new NunjucksWorkflowRunner({
       actionRegistry,
       integrations,
       logger,
       workingDirectory,
-      additionalTemplateFilters
+      additionalTemplateFilters,
+      additionalTemplateGlobals
     });
     return new TaskWorker({
       taskBroker,
@@ -4563,7 +4641,7 @@ function buildDefaultIdentityClient({
 }
 async function createRouter(options) {
   const router = Router__default["default"]();
-  router.use(express__default["default"].json());
+  router.use(express__default["default"].json({ limit: "10MB" }));
   const {
     logger: parentLogger,
     config,
@@ -4572,7 +4650,9 @@ async function createRouter(options) {
     catalogClient,
     actions,
     taskWorkers,
-    additionalTemplateFilters
+    scheduler,
+    additionalTemplateFilters,
+    additionalTemplateGlobals
   } = options;
   const logger = parentLogger.child({ plugin: "scaffolder" });
   const identity = options.identity || buildDefaultIdentityClient({ logger });
@@ -4582,6 +4662,22 @@ async function createRouter(options) {
   if (!options.taskBroker) {
     const databaseTaskStore = await DatabaseTaskStore.create({ database });
     taskBroker = new StorageTaskBroker(databaseTaskStore, logger);
+    if (scheduler && databaseTaskStore.listStaleTasks) {
+      await scheduler.scheduleTask({
+        id: "close_stale_tasks",
+        frequency: { cron: "*/5 * * * *" },
+        timeout: { minutes: 15 },
+        fn: async () => {
+          const { tasks } = await databaseTaskStore.listStaleTasks({
+            timeoutS: 86400
+          });
+          for (const task of tasks) {
+            await databaseTaskStore.shutdownTask(task);
+            logger.info(`Successfully closed stale task ${task.taskId}`);
+          }
+        }
+      });
+    }
   } else {
     taskBroker = options.taskBroker;
   }
@@ -4594,7 +4690,8 @@ async function createRouter(options) {
       integrations,
       logger,
       workingDirectory,
-      additionalTemplateFilters
+      additionalTemplateFilters,
+      additionalTemplateGlobals
     });
     workers.push(worker);
   }
@@ -4603,7 +4700,8 @@ async function createRouter(options) {
     catalogClient,
     reader,
     config,
-    additionalTemplateFilters
+    additionalTemplateFilters,
+    additionalTemplateGlobals
   });
   actionsToRegister.forEach((action) => actionRegistry.register(action));
   workers.forEach((worker) => worker.start());
@@ -4612,7 +4710,8 @@ async function createRouter(options) {
     integrations,
     logger,
     workingDirectory,
-    additionalTemplateFilters
+    additionalTemplateFilters,
+    additionalTemplateGlobals
   });
   router.get(
     "/v2/templates/:namespace/:kind/:name/parameter-schema",
@@ -5005,7 +5104,12 @@ const scaffolderPlugin = backendPluginApi.createBackendPlugin({
         httpRouter,
         catalogClient
       }) {
-        const { additionalTemplateFilters, taskBroker, taskWorkers } = options;
+        const {
+          additionalTemplateFilters,
+          taskBroker,
+          taskWorkers,
+          additionalTemplateGlobals
+        } = options;
         const log = backendPluginApi.loggerToWinstonLogger(logger);
         const actions = options.actions || [
           ...actionsExtensions.actions,
@@ -5014,7 +5118,8 @@ const scaffolderPlugin = backendPluginApi.createBackendPlugin({
             catalogClient,
             reader,
             config,
-            additionalTemplateFilters
+            additionalTemplateFilters,
+            additionalTemplateGlobals
           })
         ];
         const actionIds = actions.map((action) => action.id).join(", ");
@@ -5030,7 +5135,8 @@ const scaffolderPlugin = backendPluginApi.createBackendPlugin({
           actions,
           taskBroker,
           taskWorkers,
-          additionalTemplateFilters
+          additionalTemplateFilters,
+          additionalTemplateGlobals
         });
         httpRouter.use(router);
       }