@backstage/plugin-scaffolder-backend-module-gcp 0.2.26 → 0.3.0-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,30 @@
|
|
|
1
1
|
# @backstage/plugin-scaffolder-backend-module-gcp
|
|
2
2
|
|
|
3
|
+
## 0.3.0-next.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- Updated dependencies
|
|
8
|
+
- @backstage/backend-plugin-api@1.12.0-next.1
|
|
9
|
+
- @backstage/errors@1.4.0-next.0
|
|
10
|
+
- @backstage/integration@2.1.4-next.0
|
|
11
|
+
- @backstage/plugin-scaffolder-node@1.0.0-next.2
|
|
12
|
+
- @backstage/config@1.3.11-next.0
|
|
13
|
+
|
|
14
|
+
## 0.3.0-next.0
|
|
15
|
+
|
|
16
|
+
### Minor Changes
|
|
17
|
+
|
|
18
|
+
- 44d5ba2: **BREAKING**: The backend's Google Cloud credentials now require the `storage.objects.list` permission on the configured task recovery bucket, in addition to the existing object permissions. Grant this permission before upgrading.
|
|
19
|
+
|
|
20
|
+
Fixed task recovery workspace saves failing with Google Cloud Storage rate limit errors when template steps finish in quick succession. Each snapshot is saved to a separate object, and recovery continues to support snapshots from older versions.
|
|
21
|
+
|
|
22
|
+
### Patch Changes
|
|
23
|
+
|
|
24
|
+
- Updated dependencies
|
|
25
|
+
- @backstage/backend-plugin-api@1.11.1-next.0
|
|
26
|
+
- @backstage/plugin-scaffolder-node@0.13.9-next.0
|
|
27
|
+
|
|
3
28
|
## 0.2.26
|
|
4
29
|
|
|
5
30
|
### Patch Changes
|
package/README.md
CHANGED
|
@@ -2,4 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
The GCP bucket module for [@backstage/plugin-scaffolder-backend](https://www.npmjs.com/package/@backstage/plugin-scaffolder-backend).
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
When used as the `gcpBucket` task recovery workspace provider, the module saves
|
|
6
|
+
each workspace snapshot as a separate object under the task's prefix in the
|
|
7
|
+
configured bucket. This avoids the per-object write rate limit when template
|
|
8
|
+
steps finish in quick succession. Recovery restores the latest snapshot, including
|
|
9
|
+
snapshots saved by older versions of the provider.
|
|
10
|
+
|
|
11
|
+
Older snapshots are deleted after a successful save. If that cleanup fails, the
|
|
12
|
+
module logs a warning and keeps the saved workspace available for recovery.
|
|
13
|
+
Task cleanup removes all snapshots for the task. The backend's Google Cloud
|
|
14
|
+
credentials need permissions to list, create, read, and delete objects in the
|
|
15
|
+
bucket.
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
var errors = require('@backstage/errors');
|
|
4
4
|
var getRawBody = require('raw-body');
|
|
5
5
|
var storage = require('@google-cloud/storage');
|
|
6
|
+
var node_crypto = require('node:crypto');
|
|
6
7
|
var alpha = require('@backstage/plugin-scaffolder-node/alpha');
|
|
7
8
|
|
|
8
9
|
function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e : { default: e }; }
|
|
@@ -22,6 +23,9 @@ class GcpBucketWorkspaceProvider {
|
|
|
22
23
|
this.config = config;
|
|
23
24
|
}
|
|
24
25
|
async cleanWorkspace(options) {
|
|
26
|
+
await this.storage.bucket(this.getGcpBucketName()).deleteFiles({
|
|
27
|
+
prefix: `${options.taskId}/`
|
|
28
|
+
});
|
|
25
29
|
const file = this.storage.bucket(this.getGcpBucketName()).file(options.taskId);
|
|
26
30
|
const result = await file.exists();
|
|
27
31
|
if (result[0]) {
|
|
@@ -29,8 +33,20 @@ class GcpBucketWorkspaceProvider {
|
|
|
29
33
|
}
|
|
30
34
|
}
|
|
31
35
|
async serializeWorkspace(options) {
|
|
32
|
-
const fileCloud = this.storage.bucket(this.getGcpBucketName()).file(options.taskId);
|
|
33
36
|
const { contents: workspace } = await alpha.serializeWorkspace(options);
|
|
37
|
+
const bucket = this.storage.bucket(this.getGcpBucketName());
|
|
38
|
+
const prefix = `${options.taskId}/`;
|
|
39
|
+
const [files] = await bucket.getFiles({ prefix });
|
|
40
|
+
const sequence = files.reduce(
|
|
41
|
+
(latest, file) => Math.max(
|
|
42
|
+
latest,
|
|
43
|
+
Number(file.name.slice(prefix.length).split("-")[0]) || 0
|
|
44
|
+
),
|
|
45
|
+
0
|
|
46
|
+
) + 1;
|
|
47
|
+
const fileCloud = bucket.file(
|
|
48
|
+
`${prefix}${String(sequence).padStart(16, "0")}-${node_crypto.randomUUID()}.tar`
|
|
49
|
+
);
|
|
34
50
|
try {
|
|
35
51
|
await fileCloud.save(workspace, {
|
|
36
52
|
contentType: "application/x-tar"
|
|
@@ -44,9 +60,26 @@ class GcpBucketWorkspaceProvider {
|
|
|
44
60
|
this.logger.info(
|
|
45
61
|
`Workspace for task ${options.taskId} has been serialized.`
|
|
46
62
|
);
|
|
63
|
+
try {
|
|
64
|
+
await Promise.all(
|
|
65
|
+
files.map((file) => file.delete({ ignoreNotFound: true }))
|
|
66
|
+
);
|
|
67
|
+
} catch (error) {
|
|
68
|
+
this.logger.warn(
|
|
69
|
+
`Failed to delete old workspace snapshots for task '${options.taskId}'`,
|
|
70
|
+
error
|
|
71
|
+
);
|
|
72
|
+
}
|
|
47
73
|
}
|
|
48
74
|
async rehydrateWorkspace(options) {
|
|
49
75
|
const bucket = this.storage.bucket(this.getGcpBucketName());
|
|
76
|
+
const [files] = await bucket.getFiles({ prefix: `${options.taskId}/` });
|
|
77
|
+
const latest = files.sort((a, b) => b.name.localeCompare(a.name))[0];
|
|
78
|
+
if (latest) {
|
|
79
|
+
const workspace = await getRawBody__default.default(latest.createReadStream());
|
|
80
|
+
await alpha.restoreWorkspace({ path: options.targetPath, buffer: workspace });
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
50
83
|
const file = bucket.file(options.taskId);
|
|
51
84
|
const result = await file.exists();
|
|
52
85
|
if (result[0]) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"GcpBucketWorkspaceProvider.cjs.js","sources":["../../src/providers/GcpBucketWorkspaceProvider.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Config } from '@backstage/config';\nimport { ForwardedError } from '@backstage/errors';\nimport { WorkspaceProvider } from '@backstage/plugin-scaffolder-node/alpha';\n\nimport getRawBody from 'raw-body';\nimport { Storage } from '@google-cloud/storage';\nimport { LoggerService } from '@backstage/backend-plugin-api';\n\nimport {\n serializeWorkspace,\n restoreWorkspace,\n} from '@backstage/plugin-scaffolder-node/alpha';\n\nexport class GcpBucketWorkspaceProvider implements WorkspaceProvider {\n private readonly storage: Storage;\n private readonly logger: LoggerService;\n private readonly config?: Config;\n\n static create(logger: LoggerService, config?: Config) {\n return new GcpBucketWorkspaceProvider(new Storage(), logger, config);\n }\n\n private constructor(\n storage: Storage,\n logger: LoggerService,\n config?: Config,\n ) {\n this.storage = storage;\n this.logger = logger;\n this.config = config;\n }\n\n public async cleanWorkspace(options: { taskId: string }): Promise<void> {\n const file = this.storage\n .bucket(this.getGcpBucketName())\n .file(options.taskId);\n\n const result = await file.exists();\n if (result[0]) {\n await file.delete();\n }\n }\n\n public async serializeWorkspace(options: {\n path: string;\n taskId: string;\n }): Promise<void> {\n const
|
|
1
|
+
{"version":3,"file":"GcpBucketWorkspaceProvider.cjs.js","sources":["../../src/providers/GcpBucketWorkspaceProvider.ts"],"sourcesContent":["/*\n * Copyright 2021 The Backstage Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { Config } from '@backstage/config';\nimport { ForwardedError } from '@backstage/errors';\nimport { WorkspaceProvider } from '@backstage/plugin-scaffolder-node/alpha';\n\nimport getRawBody from 'raw-body';\nimport { Storage } from '@google-cloud/storage';\nimport { LoggerService } from '@backstage/backend-plugin-api';\nimport { randomUUID } from 'node:crypto';\n\nimport {\n serializeWorkspace,\n restoreWorkspace,\n} from '@backstage/plugin-scaffolder-node/alpha';\n\nexport class GcpBucketWorkspaceProvider implements WorkspaceProvider {\n private readonly storage: Storage;\n private readonly logger: LoggerService;\n private readonly config?: Config;\n\n static create(logger: LoggerService, config?: Config) {\n return new GcpBucketWorkspaceProvider(new Storage(), logger, config);\n }\n\n private constructor(\n storage: Storage,\n logger: LoggerService,\n config?: Config,\n ) {\n this.storage = storage;\n this.logger = logger;\n this.config = config;\n }\n\n public async cleanWorkspace(options: { taskId: string }): Promise<void> {\n await this.storage.bucket(this.getGcpBucketName()).deleteFiles({\n prefix: `${options.taskId}/`,\n });\n\n // Also remove snapshots written by older versions of the provider.\n const file = this.storage\n .bucket(this.getGcpBucketName())\n .file(options.taskId);\n\n const result = await file.exists();\n if (result[0]) {\n await file.delete();\n }\n }\n\n public async serializeWorkspace(options: {\n path: string;\n taskId: string;\n }): Promise<void> {\n const { contents: workspace } = await serializeWorkspace(options);\n const bucket = this.storage.bucket(this.getGcpBucketName());\n const prefix = `${options.taskId}/`;\n const [files] = await bucket.getFiles({ prefix });\n // Derive the sequence from stored snapshots so ordering survives restarts\n // and does not depend on clock resolution or clock skew between workers.\n const sequence =\n files.reduce(\n (latest, file) =>\n Math.max(\n latest,\n Number(file.name.slice(prefix.length).split('-')[0]) || 0,\n ),\n 0,\n ) + 1;\n const fileCloud = bucket.file(\n `${prefix}${String(sequence).padStart(16, '0')}-${randomUUID()}.tar`,\n );\n try {\n await fileCloud.save(workspace, {\n contentType: 'application/x-tar',\n });\n } catch (error) {\n throw new ForwardedError(\n `Failed to upload workspace for task '${options.taskId}' to GCS`,\n error,\n );\n }\n this.logger.info(\n `Workspace for task ${options.taskId} has been serialized.`,\n );\n\n // Only remove snapshots that existed before this upload. Cleanup failure\n // must not invalidate a successfully saved workspace.\n try {\n await Promise.all(\n files.map(file => file.delete({ ignoreNotFound: true })),\n );\n } catch (error) {\n this.logger.warn(\n `Failed to delete old workspace snapshots for task '${options.taskId}'`,\n error,\n );\n }\n }\n\n public async rehydrateWorkspace(options: {\n taskId: string;\n targetPath: string;\n }): Promise<void> {\n const bucket = this.storage.bucket(this.getGcpBucketName());\n const [files] = await bucket.getFiles({ prefix: `${options.taskId}/` });\n const latest = files.sort((a, b) => b.name.localeCompare(a.name))[0];\n if (latest) {\n const workspace = await getRawBody(latest.createReadStream());\n await restoreWorkspace({ path: options.targetPath, buffer: workspace });\n return;\n }\n\n // Tasks started before upgrading may still have a single legacy snapshot.\n const file = bucket.file(options.taskId);\n const result = await file.exists();\n if (result[0]) {\n const workspace = await getRawBody(file.createReadStream());\n await restoreWorkspace({ path: options.targetPath, buffer: workspace });\n }\n }\n\n private getGcpBucketName(): string {\n // New config path with fallback to old experimental flag\n const bucketName =\n this.config?.getOptionalString(\n 'scaffolder.taskRecovery.gcsBucket.name',\n ) ??\n this.config?.getOptionalString(\n 'scaffolder.EXPERIMENTAL_workspaceSerializationGcpBucketName',\n );\n if (!bucketName) {\n throw new Error(\n `Missing GCS bucket configuration. Set scaffolder.taskRecovery.gcsBucket.name in app-config.yaml`,\n );\n }\n return bucketName;\n }\n}\n"],"names":["Storage","serializeWorkspace","randomUUID","ForwardedError","getRawBody","restoreWorkspace"],"mappings":";;;;;;;;;;;;AA8BO,MAAM,0BAAA,CAAwD;AAAA,EAClD,OAAA;AAAA,EACA,MAAA;AAAA,EACA,MAAA;AAAA,EAEjB,OAAO,MAAA,CAAO,MAAA,EAAuB,MAAA,EAAiB;AACpD,IAAA,OAAO,IAAI,0BAAA,CAA2B,IAAIA,eAAA,EAAQ,EAAG,QAAQ,MAAM,CAAA;AAAA,EACrE;AAAA,EAEQ,WAAA,CACN,OAAA,EACA,MAAA,EACA,MAAA,EACA;AACA,IAAA,IAAA,CAAK,OAAA,GAAU,OAAA;AACf,IAAA,IAAA,CAAK,MAAA,GAAS,MAAA;AACd,IAAA,IAAA,CAAK,MAAA,GAAS,MAAA;AAAA,EAChB;AAAA,EAEA,MAAa,eAAe,OAAA,EAA4C;AACtE,IAAA,MAAM,KAAK,OAAA,CAAQ,MAAA,CAAO,KAAK,gBAAA,EAAkB,EAAE,WAAA,CAAY;AAAA,MAC7D,MAAA,EAAQ,CAAA,EAAG,OAAA,CAAQ,MAAM,CAAA,CAAA;AAAA,KAC1B,CAAA;AAGD,IAAA,MAAM,IAAA,GAAO,IAAA,CAAK,OAAA,CACf,MAAA,CAAO,IAAA,CAAK,kBAAkB,CAAA,CAC9B,IAAA,CAAK,OAAA,CAAQ,MAAM,CAAA;AAEtB,IAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,MAAA,EAAO;AACjC,IAAA,IAAI,MAAA,CAAO,CAAC,CAAA,EAAG;AACb,MAAA,MAAM,KAAK,MAAA,EAAO;AAAA,IACpB;AAAA,EACF;AAAA,EAEA,MAAa,mBAAmB,OAAA,EAGd;AAChB,IAAA,MAAM,EAAE,QAAA,EAAU,SAAA,EAAU,GAAI,MAAMC,yBAAmB,OAAO,CAAA;AAChE,IAAA,MAAM,SAAS,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,IAAA,CAAK,kBAAkB,CAAA;AAC1D,IAAA,MAAM,MAAA,GAAS,CAAA,EAAG,OAAA,CAAQ,MAAM,CAAA,CAAA,CAAA;AAChC,IAAA,MAAM,CAAC,KAAK,CAAA,GAAI,MAAM,OAAO,QAAA,CAAS,EAAE,QAAQ,CAAA;AAGhD,IAAA,MAAM,WACJ,KAAA,CAAM,MAAA;AAAA,MACJ,CAAC,MAAA,EAAQ,IAAA,KACP,IAAA,CAAK,GAAA;AAAA,QACH,MAAA;AAAA,QACA,MAAA,CAAO,IAAA,CAAK,IAAA,CAAK,KAAA,CAAM,MAAA,CAAO,MAAM,CAAA,CAAE,KAAA,CAAM,GAAG,CAAA,CAAE,CAAC,CAAC,CAAA,IAAK;AAAA,OAC1D;AAAA,MACF;AAAA,KACF,GAAI,CAAA;AACN,IAAA,MAAM,YAAY,MAAA,CAAO,IAAA;AAAA,MACvB,CAAA,EAAG,MAAM,CAAA,EAAG,MAAA,CAAO,QAAQ,CAAA,CAAE,QAAA,CAAS,EAAA,EAAI,GAAG,CAAC,CAAA,CAAA,EAAIC,sBAAA,EAAY,CAAA,IAAA;AAAA,KAChE;AACA,IAAA,IAAI;AACF,MAAA,MAAM,SAAA,CAAU,KAAK,SAAA,EAAW;AAAA,QAC9B,WAAA,EAAa;AAAA,OACd,CAAA;AAAA,IACH,SAAS,KAAA,EAAO;AACd,MAAA,MAAM,IAAIC,qBAAA;AAAA,QACR,CAAA,qCAAA,EAAwC,QAAQ,MAAM,CAAA,QAAA,CAAA;AAAA,QACtD;AAAA,OACF;AAAA,IACF;AACA,IAAA,IAAA,CAAK,MAAA,CAAO,IAAA;AAAA,MACV,CAAA,mBAAA,EAAsB,QAAQ,MAAM,CAAA,qBAAA;AAAA,KACtC;AAIA,IAAA,IAAI;AACF,MAAA,MAAM,OAAA,CAAQ,GAAA;AAAA,QACZ,KAAA,CAAM,IAAI,CAAA,IAAA,KAAQ,IAAA,CAAK,OAAO,EAAE,cAAA,EAAgB,IAAA,EAAM,CAAC;AAAA,OACzD;AAAA,IACF,SAAS,KAAA,EAAO;AACd,MAAA,IAAA,CAAK,MAAA,CAAO,IAAA;AAAA,QACV,CAAA,mDAAA,EAAsD,QAAQ,MAAM,CAAA,CAAA,CAAA;AAAA,QACpE;AAAA,OACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAa,mBAAmB,OAAA,EAGd;AAChB,IAAA,MAAM,SAAS,IAAA,CAAK,OAAA,CAAQ,MAAA,CAAO,IAAA,CAAK,kBAAkB,CAAA;AAC1D,IAAA,MAAM,CAAC,KAAK,CAAA,GAAI,MAAM,MAAA,CAAO,QAAA,CAAS,EAAE,MAAA,EAAQ,CAAA,EAAG,OAAA,CAAQ,MAAM,CAAA,CAAA,CAAA,EAAK,CAAA;AACtE,IAAA,MAAM,MAAA,GAAS,KAAA,CAAM,IAAA,CAAK,CAAC,CAAA,EAAG,CAAA,KAAM,CAAA,CAAE,IAAA,CAAK,aAAA,CAAc,CAAA,CAAE,IAAI,CAAC,EAAE,CAAC,CAAA;AACnE,IAAA,IAAI,MAAA,EAAQ;AACV,MAAA,MAAM,SAAA,GAAY,MAAMC,2BAAA,CAAW,MAAA,CAAO,kBAAkB,CAAA;AAC5D,MAAA,MAAMC,uBAAiB,EAAE,IAAA,EAAM,QAAQ,UAAA,EAAY,MAAA,EAAQ,WAAW,CAAA;AACtE,MAAA;AAAA,IACF;AAGA,IAAA,MAAM,IAAA,GAAO,MAAA,CAAO,IAAA,CAAK,OAAA,CAAQ,MAAM,CAAA;AACvC,IAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,MAAA,EAAO;AACjC,IAAA,IAAI,MAAA,CAAO,CAAC,CAAA,EAAG;AACb,MAAA,MAAM,SAAA,GAAY,MAAMD,2BAAA,CAAW,IAAA,CAAK,kBAAkB,CAAA;AAC1D,MAAA,MAAMC,uBAAiB,EAAE,IAAA,EAAM,QAAQ,UAAA,EAAY,MAAA,EAAQ,WAAW,CAAA;AAAA,IACxE;AAAA,EACF;AAAA,EAEQ,gBAAA,GAA2B;AAEjC,IAAA,MAAM,UAAA,GACJ,KAAK,MAAA,EAAQ,iBAAA;AAAA,MACX;AAAA,KACF,IACA,KAAK,MAAA,EAAQ,iBAAA;AAAA,MACX;AAAA,KACF;AACF,IAAA,IAAI,CAAC,UAAA,EAAY;AACf,MAAA,MAAM,IAAI,KAAA;AAAA,QACR,CAAA,+FAAA;AAAA,OACF;AAAA,IACF;AACA,IAAA,OAAO,UAAA;AAAA,EACT;AACF;;"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@backstage/plugin-scaffolder-backend-module-gcp",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0-next.1",
|
|
4
4
|
"description": "The GCP Bucket module for @backstage/plugin-scaffolder-backend",
|
|
5
5
|
"backstage": {
|
|
6
6
|
"role": "backend-plugin-module",
|
|
@@ -51,18 +51,18 @@
|
|
|
51
51
|
"test": "backstage-cli package test"
|
|
52
52
|
},
|
|
53
53
|
"dependencies": {
|
|
54
|
-
"@backstage/backend-plugin-api": "
|
|
55
|
-
"@backstage/config": "
|
|
56
|
-
"@backstage/errors": "
|
|
57
|
-
"@backstage/integration": "
|
|
58
|
-
"@backstage/plugin-scaffolder-node": "
|
|
54
|
+
"@backstage/backend-plugin-api": "1.12.0-next.1",
|
|
55
|
+
"@backstage/config": "1.3.11-next.0",
|
|
56
|
+
"@backstage/errors": "1.4.0-next.0",
|
|
57
|
+
"@backstage/integration": "2.1.4-next.0",
|
|
58
|
+
"@backstage/plugin-scaffolder-node": "1.0.0-next.2",
|
|
59
59
|
"@google-cloud/storage": "^7.0.0",
|
|
60
60
|
"raw-body": "^2.4.1"
|
|
61
61
|
},
|
|
62
62
|
"devDependencies": {
|
|
63
|
-
"@backstage/backend-test-utils": "
|
|
64
|
-
"@backstage/cli": "
|
|
65
|
-
"@backstage/plugin-scaffolder-node-test-utils": "
|
|
63
|
+
"@backstage/backend-test-utils": "1.12.0-next.2",
|
|
64
|
+
"@backstage/cli": "0.36.8-next.2",
|
|
65
|
+
"@backstage/plugin-scaffolder-node-test-utils": "0.3.17-next.1"
|
|
66
66
|
},
|
|
67
67
|
"configSchema": "config.schema.json"
|
|
68
68
|
}
|