@backstage/plugin-scaffolder-backend 1.8.1-next.0 → 1.9.0-next.2

package/CHANGELOG.md

@@ -1,5 +1,65 @@
 # @backstage/plugin-scaffolder-backend
 
+## 1.9.0-next.2
+
+### Minor Changes
+
+- b32005e98a: Deprecated the `taskWorkers` option in RouterOptions in favor of `concurrentTasksLimit` which sets the limit of concurrent tasks in a single TaskWorker
+
+  TaskWorker can now run multiple (defaults to 10) tasks concurrently using the `concurrentTasksLimit` option available in both `RouterOptions` and `CreateWorkerOptions`.
+
+  To use the option to create a TaskWorker:
+
+  ```diff
+  const worker = await TaskWorker.create({
+      taskBroker,
+      actionRegistry,
+      integrations,
+      logger,
+      workingDirectory,
+      additionalTemplateFilters,
+  +   concurrentTasksLimit: 10 // (1 to Infinity)
+  });
+  ```
+
+### Patch Changes
+
+- 884d749b14: Refactored to use `coreServices` from `@backstage/backend-plugin-api`.
+- Updated dependencies
+  - @backstage/plugin-catalog-backend@1.6.0-next.2
+  - @backstage/plugin-catalog-node@1.3.0-next.2
+  - @backstage/backend-common@0.17.0-next.2
+  - @backstage/backend-plugin-api@0.2.0-next.2
+  - @backstage/backend-tasks@0.4.0-next.2
+  - @backstage/plugin-auth-node@0.2.8-next.2
+  - @backstage/catalog-client@1.2.0-next.1
+  - @backstage/catalog-model@1.1.4-next.1
+  - @backstage/config@1.0.5-next.1
+  - @backstage/errors@1.1.4-next.1
+  - @backstage/integration@1.4.1-next.1
+  - @backstage/types@1.0.2-next.1
+  - @backstage/plugin-scaffolder-common@1.2.3-next.1
+
+## 1.8.1-next.1
+
+### Patch Changes
+
+- c3fa90e184: Updated dependency `zen-observable` to `^0.10.0`.
+- Updated dependencies
+  - @backstage/backend-common@0.17.0-next.1
+  - @backstage/plugin-catalog-backend@1.6.0-next.1
+  - @backstage/backend-tasks@0.4.0-next.1
+  - @backstage/types@1.0.2-next.1
+  - @backstage/backend-plugin-api@0.1.5-next.1
+  - @backstage/plugin-auth-node@0.2.8-next.1
+  - @backstage/plugin-catalog-node@1.2.2-next.1
+  - @backstage/config@1.0.5-next.1
+  - @backstage/integration@1.4.1-next.1
+  - @backstage/catalog-client@1.2.0-next.1
+  - @backstage/catalog-model@1.1.4-next.1
+  - @backstage/errors@1.1.4-next.1
+  - @backstage/plugin-scaffolder-common@1.2.3-next.1
+
 ## 1.8.1-next.0
 
 ### Patch Changes

package/alpha/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backstage/plugin-scaffolder-backend",
-  "version": "1.8.1-next.0",
+  "version": "1.9.0-next.2",
   "main": "../dist/index.cjs.js",
   "types": "../dist/index.alpha.d.ts"
 }

package/dist/index.alpha.d.ts

@@ -580,7 +580,7 @@ branchName: string;
 sourcePath?: string | undefined;
 targetPath?: string | undefined;
 token?: string | undefined;
-commitAction?: "update" | "create" | "delete" | undefined;
+commitAction?: "update" | "delete" | "create" | undefined;
 /** @deprecated projectID passed as query parameters in the repoUrl */
 projectid?: string | undefined;
 removeSourceBranch?: boolean | undefined;
@@ -611,6 +611,19 @@ export declare type CreateWorkerOptions = {
     workingDirectory: string;
     logger: Logger;
     additionalTemplateFilters?: Record<string, TemplateFilter>;
+    /**
+     * The number of tasks that can be executed at the same time by the worker
+     * @defaultValue 10
+     * @example
+     * ```
+     * {
+     *   concurrentTasksLimit: 1,
+     *   // OR
+     *   concurrentTasksLimit: Infinity
+     * }
+     * ```
+     */
+    concurrentTasksLimit?: number;
     additionalTemplateGlobals?: Record<string, TemplateGlobal>;
 };
 
@@ -732,7 +745,16 @@ export declare interface RouterOptions {
     catalogClient: CatalogApi;
     scheduler?: PluginTaskScheduler;
     actions?: TemplateAction<any>[];
+    /**
+     * @deprecated taskWorkers is deprecated in favor of concurrentTasksLimit option with a single TaskWorker
+     * @defaultValue 1
+     */
     taskWorkers?: number;
+    /**
+     * Sets the number of concurrent tasks that can be run at any given time on the TaskWorker
+     * @defaultValue 10
+     */
+    concurrentTasksLimit?: number;
     taskBroker?: TaskBroker;
     additionalTemplateFilters?: Record<string, TemplateFilter>;
     additionalTemplateGlobals?: Record<string, TemplateGlobal>;
@@ -1015,8 +1037,10 @@ export declare type TaskStoreShutDownTaskOptions = {
 export declare class TaskWorker {
     private readonly options;
     private constructor();
+    private taskQueue;
     static create(options: CreateWorkerOptions): Promise<TaskWorker>;
     start(): void;
+    protected onReadyToClaimTask(): Promise<void>;
     runOneTask(task: TaskContext): Promise<void>;
 }
 

package/dist/index.beta.d.ts

@@ -580,7 +580,7 @@ branchName: string;
 sourcePath?: string | undefined;
 targetPath?: string | undefined;
 token?: string | undefined;
-commitAction?: "update" | "create" | "delete" | undefined;
+commitAction?: "update" | "delete" | "create" | undefined;
 /** @deprecated projectID passed as query parameters in the repoUrl */
 projectid?: string | undefined;
 removeSourceBranch?: boolean | undefined;
@@ -611,6 +611,19 @@ export declare type CreateWorkerOptions = {
     workingDirectory: string;
     logger: Logger;
     additionalTemplateFilters?: Record<string, TemplateFilter>;
+    /**
+     * The number of tasks that can be executed at the same time by the worker
+     * @defaultValue 10
+     * @example
+     * ```
+     * {
+     *   concurrentTasksLimit: 1,
+     *   // OR
+     *   concurrentTasksLimit: Infinity
+     * }
+     * ```
+     */
+    concurrentTasksLimit?: number;
     additionalTemplateGlobals?: Record<string, TemplateGlobal>;
 };
 
@@ -732,7 +745,16 @@ export declare interface RouterOptions {
     catalogClient: CatalogApi;
     scheduler?: PluginTaskScheduler;
     actions?: TemplateAction<any>[];
+    /**
+     * @deprecated taskWorkers is deprecated in favor of concurrentTasksLimit option with a single TaskWorker
+     * @defaultValue 1
+     */
     taskWorkers?: number;
+    /**
+     * Sets the number of concurrent tasks that can be run at any given time on the TaskWorker
+     * @defaultValue 10
+     */
+    concurrentTasksLimit?: number;
     taskBroker?: TaskBroker;
     additionalTemplateFilters?: Record<string, TemplateFilter>;
     additionalTemplateGlobals?: Record<string, TemplateGlobal>;
@@ -997,8 +1019,10 @@ export declare type TaskStoreShutDownTaskOptions = {
 export declare class TaskWorker {
     private readonly options;
     private constructor();
+    private taskQueue;
     static create(options: CreateWorkerOptions): Promise<TaskWorker>;
     start(): void;
+    protected onReadyToClaimTask(): Promise<void>;
     runOneTask(task: TaskContext): Promise<void>;
 }
 

package/dist/index.cjs.js

@@ -26,6 +26,7 @@ var node = require('@gitbeaker/node');
 var uuid = require('uuid');
 var luxon = require('luxon');
 var ObservableImpl = require('zen-observable');
+var PQueue = require('p-queue');
 var winston = require('winston');
 var nunjucks = require('nunjucks');
 var lodash = require('lodash');
@@ -69,6 +70,7 @@ var fetch__default = /*#__PURE__*/_interopDefaultLegacy(fetch);
 var crypto__default = /*#__PURE__*/_interopDefaultLegacy(crypto);
 var limiterFactory__default = /*#__PURE__*/_interopDefaultLegacy(limiterFactory);
 var ObservableImpl__default = /*#__PURE__*/_interopDefaultLegacy(ObservableImpl);
+var PQueue__default = /*#__PURE__*/_interopDefaultLegacy(PQueue);
 var winston__namespace = /*#__PURE__*/_interopNamespace(winston);
 var nunjucks__default = /*#__PURE__*/_interopDefaultLegacy(nunjucks);
 var express__default = /*#__PURE__*/_interopDefaultLegacy(express);
@@ -4585,6 +4587,9 @@ function scaffoldingTracker() {
 class TaskWorker {
   constructor(options) {
     this.options = options;
+    this.taskQueue = new PQueue__default["default"]({
+      concurrency: this.options.concurrentTasksLimit
+    });
   }
   static async create(options) {
     const {
@@ -4594,6 +4599,7 @@ class TaskWorker {
       integrations,
       workingDirectory,
       additionalTemplateFilters,
+      concurrentTasksLimit = 10,
       additionalTemplateGlobals
     } = options;
     const workflowRunner = new NunjucksWorkflowRunner({
@@ -4606,17 +4612,29 @@ class TaskWorker {
     });
     return new TaskWorker({
       taskBroker,
-      runners: { workflowRunner }
+      runners: { workflowRunner },
+      concurrentTasksLimit
     });
   }
   start() {
     (async () => {
       for (; ; ) {
+        await this.onReadyToClaimTask();
         const task = await this.options.taskBroker.claim();
-        await this.runOneTask(task);
+        this.taskQueue.add(() => this.runOneTask(task));
       }
     })();
   }
+  onReadyToClaimTask() {
+    if (this.taskQueue.pending < this.options.concurrentTasksLimit) {
+      return Promise.resolve();
+    }
+    return new Promise((resolve) => {
+      this.taskQueue.once("next", () => {
+        resolve();
+      });
+    });
+  }
   async runOneTask(task) {
     try {
       if (task.spec.apiVersion !== "scaffolder.backstage.io/v1beta3") {
@@ -4835,6 +4853,7 @@ async function createRouter(options) {
     catalogClient,
     actions,
     taskWorkers,
+    concurrentTasksLimit,
     scheduler,
     additionalTemplateFilters,
     additionalTemplateGlobals
@@ -4868,7 +4887,7 @@ async function createRouter(options) {
   }
   const actionRegistry = new TemplateActionRegistry();
   const workers = [];
-  for (let i = 0; i < (taskWorkers || 3); i++) {
+  for (let i = 0; i < (taskWorkers || 1); i++) {
     const worker = await TaskWorker.create({
       taskBroker,
       actionRegistry,
@@ -4876,7 +4895,8 @@ async function createRouter(options) {
       logger,
       workingDirectory,
       additionalTemplateFilters,
-      additionalTemplateGlobals
+      additionalTemplateGlobals,
+      concurrentTasksLimit
     });
     workers.push(worker);
   }
@@ -5273,12 +5293,12 @@ const scaffolderPlugin = backendPluginApi.createBackendPlugin({
     );
     env.registerInit({
       deps: {
-        logger: backendPluginApi.loggerServiceRef,
-        config: backendPluginApi.configServiceRef,
-        reader: backendPluginApi.urlReaderServiceRef,
-        permissions: backendPluginApi.permissionsServiceRef,
-        database: backendPluginApi.databaseServiceRef,
-        httpRouter: backendPluginApi.httpRouterServiceRef,
+        logger: backendPluginApi.coreServices.logger,
+        config: backendPluginApi.coreServices.config,
+        reader: backendPluginApi.coreServices.urlReader,
+        permissions: backendPluginApi.coreServices.permissions,
+        database: backendPluginApi.coreServices.database,
+        httpRouter: backendPluginApi.coreServices.httpRouter,
         catalogClient: pluginCatalogNode.catalogServiceRef
       },
       async init({