@kungfu-tech/buildchain 2.5.4-alpha.0 → 2.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/docs/MAP.md
CHANGED
|
@@ -38,6 +38,7 @@ running artifact), *use* (consume / extend) - and a **status**:
|
|
|
38
38
|
| How do I add timestamped logs inside build scripts? | [`toolkit-observability.md`](toolkit-observability.md) | use | stable |
|
|
39
39
|
| What package-owned facts should buildchain.libkungfu.dev render? | [`site-bundle-contract.md`](site-bundle-contract.md) | use | stable |
|
|
40
40
|
| How do I call the reusable build workflow? | [`reusable-build-surface.md`](reusable-build-surface.md) | use | stable |
|
|
41
|
+
| How do self-hosted runners relay large artifacts through S3 before GitHub artifacts? | [`reusable-build-surface.md`](reusable-build-surface.md#artifact-transfer-relay) | use | stable |
|
|
41
42
|
| How do I validate an unreleased Buildchain runtime train while keeping `@v2`? | [`runtime-train-validation.md`](runtime-train-validation.md) | use | stable |
|
|
42
43
|
| How do I deploy a site/app preview, staging, or production surface? | [`web-surface-deployments.md`](web-surface-deployments.md) | use | stable |
|
|
43
44
|
| How do I publish observed infrastructure contracts for downstream consumers? | [`infra-contract.md`](infra-contract.md) | use | preview |
|
|
@@ -69,6 +70,8 @@ running artifact), *use* (consume / extend) - and a **status**:
|
|
|
69
70
|
- **libnode / native artifacts / self-hosted runner matrix** ->
|
|
70
71
|
[`reusable-build-surface.md`](reusable-build-surface.md) and
|
|
71
72
|
[`../fixtures/libnode-shaped/README.md`](../fixtures/libnode-shaped/README.md).
|
|
73
|
+
- **S3 artifact relay / self-hosted runner artifact transfer** ->
|
|
74
|
+
[`reusable-build-surface.md`](reusable-build-surface.md#artifact-transfer-relay).
|
|
72
75
|
- **runtime train validation / temporary `buildchain-ref` override** ->
|
|
73
76
|
[`runtime-train-validation.md`](runtime-train-validation.md) and
|
|
74
77
|
[`reusable-build-surface.md`](reusable-build-surface.md).
|
|
@@ -236,6 +236,77 @@ release jobs can detect drifting diagnostics JSON contracts and missing or
|
|
|
236
236
|
drifting diagnostics sidecar manifests without downloading the per-platform
|
|
237
237
|
diagnostics artifacts first.
|
|
238
238
|
|
|
239
|
+
## Artifact Transfer Relay
|
|
240
|
+
|
|
241
|
+
By default, platform jobs upload payloads, manifests, and diagnostics directly
|
|
242
|
+
to GitHub artifacts:
|
|
243
|
+
|
|
244
|
+
```yaml
|
|
245
|
+
with:
|
|
246
|
+
artifact-transfer-mode: github-artifacts
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
Large self-hosted native builds can opt into the first-class S3 relay path:
|
|
250
|
+
|
|
251
|
+
```yaml
|
|
252
|
+
jobs:
|
|
253
|
+
build:
|
|
254
|
+
uses: kungfu-systems/buildchain/.github/workflows/.build.yml@v2
|
|
255
|
+
with:
|
|
256
|
+
runner-preset: kungfu-v4-self-hosted
|
|
257
|
+
artifact-transfer-mode: s3-to-github-artifacts
|
|
258
|
+
artifact-relay-s3-bucket: ${{ vars.BUILDCHAIN_ARTIFACT_RELAY_S3_BUCKET }}
|
|
259
|
+
artifact-relay-s3-region: ${{ vars.BUILDCHAIN_ARTIFACT_RELAY_S3_REGION }}
|
|
260
|
+
artifact-relay-s3-prefix: ${{ vars.BUILDCHAIN_ARTIFACT_RELAY_S3_PREFIX }}
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
In relay mode, each self-hosted platform job uploads the heavy payload files to
|
|
264
|
+
S3 and uploads only a small `relay-manifest.json` to GitHub. A GitHub-hosted
|
|
265
|
+
`relay-artifacts` job then assumes the configured download role, downloads the
|
|
266
|
+
payloads from S3, verifies every file by SHA256, and re-uploads the normal
|
|
267
|
+
GitHub artifacts under the same artifact names that direct mode uses. Downstream
|
|
268
|
+
summary, release-candidate, and promote-only workflows therefore continue to
|
|
269
|
+
consume GitHub artifacts and do not need custom S3 logic.
|
|
270
|
+
The relay implementation uses Node.js plus the standard AWS environment
|
|
271
|
+
credentials from GitHub OIDC; runner images and build containers do not need the
|
|
272
|
+
AWS CLI installed.
|
|
273
|
+
|
|
274
|
+
After the GitHub artifact uploads succeed, Buildchain deletes the S3 objects
|
|
275
|
+
listed in the relay manifest for that platform. If any download, verification,
|
|
276
|
+
or GitHub artifact upload fails, cleanup is skipped so maintainers can inspect
|
|
277
|
+
the retained S3 payload. Configure a short bucket lifecycle expiration as a
|
|
278
|
+
cost and cleanup backstop.
|
|
279
|
+
|
|
280
|
+
The relay configuration is intentionally generic. Buildchain does not hard-code
|
|
281
|
+
organization buckets, regions, or role ARNs. Callers may pass explicit inputs,
|
|
282
|
+
or set repository/organization variables and secrets using these names:
|
|
283
|
+
|
|
284
|
+
| Variable or secret | Meaning |
|
|
285
|
+
| --- | --- |
|
|
286
|
+
| `BUILDCHAIN_ARTIFACT_RELAY_S3_BUCKET` | Relay bucket name |
|
|
287
|
+
| `BUILDCHAIN_ARTIFACT_RELAY_S3_REGION` | Relay bucket region |
|
|
288
|
+
| `BUILDCHAIN_ARTIFACT_RELAY_S3_PREFIX` | Relay object prefix; defaults to `buildchain-artifacts` |
|
|
289
|
+
| `BUILDCHAIN_ARTIFACT_RELAY_S3_ROLE_ARN` | Shared OIDC role ARN for upload and download |
|
|
290
|
+
| `BUILDCHAIN_ARTIFACT_RELAY_S3_UPLOAD_ROLE_ARN` | Upload OIDC role ARN for self-hosted build jobs |
|
|
291
|
+
| `BUILDCHAIN_ARTIFACT_RELAY_S3_DOWNLOAD_ROLE_ARN` | Download OIDC role ARN for the GitHub-hosted relay job |
|
|
292
|
+
| `BUILDCHAIN_ARTIFACT_RELAY_S3_OIDC_AUDIENCE` | Optional OIDC audience override |
|
|
293
|
+
|
|
294
|
+
For AWS China regions, Buildchain defaults the OIDC audience to
|
|
295
|
+
`sts.amazonaws.com.cn`; other regions default to `sts.amazonaws.com`. The caller
|
|
296
|
+
workflow must allow `id-token: write`, and the target role trust policy should
|
|
297
|
+
restrict GitHub OIDC claims to the expected organization, repository, workflow,
|
|
298
|
+
and branch/ref. The S3 permissions should be scoped to the relay bucket/prefix
|
|
299
|
+
used by the repository.
|
|
300
|
+
Upload roles need write/delete access under the relay prefix; download roles
|
|
301
|
+
need read access plus delete access for successful cleanup.
|
|
302
|
+
|
|
303
|
+
Relay mode is opt-in and does not affect forks or open-source users that do not
|
|
304
|
+
configure S3. Missing bucket, region, upload role, or download role values fail
|
|
305
|
+
before the heavy build matrix is scheduled. Buildchain treats S3 as a transport
|
|
306
|
+
cache, not as the final release evidence store; the final audit entry remains
|
|
307
|
+
the GitHub artifact set plus the Buildchain build summary and release-candidate
|
|
308
|
+
passport.
|
|
309
|
+
|
|
239
310
|
Set `release-candidate: true` when the successful reusable build is meant to be
|
|
240
311
|
the artifact source promoted later. Buildchain then uploads
|
|
241
312
|
`release-candidate-passport.json` under the
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kungfu-tech/buildchain",
|
|
3
|
-
"version": "2.5.4
|
|
3
|
+
"version": "2.5.4",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "Buildchain Release Passport, release governance, CLI toolkit, and site facts.",
|
|
6
6
|
"repository": "https://github.com/kungfu-systems/buildchain",
|
|
@@ -0,0 +1,529 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import crypto from "node:crypto";
|
|
3
|
+
import fs from "node:fs";
|
|
4
|
+
import https from "node:https";
|
|
5
|
+
import os from "node:os";
|
|
6
|
+
import path from "node:path";
|
|
7
|
+
import { pathToFileURL } from "node:url";
|
|
8
|
+
import { writeGitHubOutputs } from "./build-contract-core.mjs";
|
|
9
|
+
|
|
10
|
+
const CONTRACT = "kungfu-buildchain-artifact-relay-s3";
|
|
11
|
+
|
|
12
|
+
function env(name, fallback = "") {
|
|
13
|
+
return process.env[name] || fallback;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function toPosix(value) {
|
|
17
|
+
return String(value || "").replaceAll(path.sep, "/");
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function sha256File(filePath) {
|
|
21
|
+
const hash = crypto.createHash("sha256");
|
|
22
|
+
hash.update(fs.readFileSync(filePath));
|
|
23
|
+
return hash.digest("hex");
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function splitLines(value) {
|
|
27
|
+
return String(value || "")
|
|
28
|
+
.split(/\r?\n/)
|
|
29
|
+
.map((line) => line.trim())
|
|
30
|
+
.filter(Boolean);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function listFiles(root, rel) {
|
|
34
|
+
const normalized = String(rel || "").replace(/\\/g, "/").replace(/\/\*\*\/?\*?$/, "");
|
|
35
|
+
const target = path.isAbsolute(normalized) ? normalized : path.join(root, normalized);
|
|
36
|
+
if (!fs.existsSync(target)) {
|
|
37
|
+
return [];
|
|
38
|
+
}
|
|
39
|
+
const stat = fs.statSync(target);
|
|
40
|
+
if (stat.isFile()) {
|
|
41
|
+
return [target];
|
|
42
|
+
}
|
|
43
|
+
return fs
|
|
44
|
+
.readdirSync(target, { withFileTypes: true })
|
|
45
|
+
.flatMap((entry) => listFiles(root, path.join(target, entry.name)));
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function collectGroupFiles(root, paths) {
|
|
49
|
+
const files = new Set();
|
|
50
|
+
for (const entry of paths) {
|
|
51
|
+
for (const file of listFiles(root, entry)) {
|
|
52
|
+
files.add(path.resolve(file));
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return [...files].sort();
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function safeSegment(value, fallback = "artifact") {
|
|
59
|
+
const segment = String(value || fallback)
|
|
60
|
+
.replace(/[^A-Za-z0-9._-]+/g, "-")
|
|
61
|
+
.replace(/^-+|-+$/g, "");
|
|
62
|
+
return segment || fallback;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function normalizePrefix(value) {
|
|
66
|
+
return String(value || "buildchain-artifacts").replace(/^\/+|\/+$/g, "") || "buildchain-artifacts";
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function assertSafeRelativePath(value) {
|
|
70
|
+
const normalized = path.posix.normalize(String(value || "").replace(/\\/g, "/"));
|
|
71
|
+
if (!normalized || normalized === "." || normalized.startsWith("../") || normalized === ".." || path.isAbsolute(normalized)) {
|
|
72
|
+
throw new Error(`unsafe relay object relative path: ${value}`);
|
|
73
|
+
}
|
|
74
|
+
return normalized;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function hmac(key, value, encoding) {
|
|
78
|
+
return crypto.createHmac("sha256", key).update(value).digest(encoding);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function sha256Hex(value) {
|
|
82
|
+
return crypto.createHash("sha256").update(value).digest("hex");
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function encodePathSegment(value) {
|
|
86
|
+
return encodeURIComponent(value).replace(/[!'()*]/g, (char) => `%${char.charCodeAt(0).toString(16).toUpperCase()}`);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function encodeS3Key(key) {
|
|
90
|
+
return String(key || "")
|
|
91
|
+
.split("/")
|
|
92
|
+
.map((segment) => encodePathSegment(segment))
|
|
93
|
+
.join("/");
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function s3EndpointHost(region) {
|
|
97
|
+
return String(region || "").startsWith("cn-")
|
|
98
|
+
? `s3.${region}.amazonaws.com.cn`
|
|
99
|
+
: `s3.${region}.amazonaws.com`;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function s3RequestTarget({ bucket, key, region }) {
|
|
103
|
+
const safeBucket = String(bucket || "");
|
|
104
|
+
const hostSuffix = s3EndpointHost(region);
|
|
105
|
+
const encodedKey = encodeS3Key(key);
|
|
106
|
+
if (/^[a-z0-9][a-z0-9.-]*[a-z0-9]$/.test(safeBucket) && !safeBucket.includes("..") && !safeBucket.includes(".")) {
|
|
107
|
+
return {
|
|
108
|
+
host: `${safeBucket}.${hostSuffix}`,
|
|
109
|
+
path: `/${encodedKey}`,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
return {
|
|
113
|
+
host: hostSuffix,
|
|
114
|
+
path: `/${encodePathSegment(safeBucket)}/${encodedKey}`,
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function awsCredentials() {
|
|
119
|
+
const accessKeyId = env("AWS_ACCESS_KEY_ID");
|
|
120
|
+
const secretAccessKey = env("AWS_SECRET_ACCESS_KEY");
|
|
121
|
+
const sessionToken = env("AWS_SESSION_TOKEN");
|
|
122
|
+
if (!accessKeyId || !secretAccessKey) {
|
|
123
|
+
throw new Error("AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are required for S3 artifact relay");
|
|
124
|
+
}
|
|
125
|
+
return { accessKeyId, secretAccessKey, sessionToken };
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function signS3Request({ method, bucket, key, region, payloadHash, contentLength = 0 }) {
|
|
129
|
+
if (!region) throw new Error("S3 relay region is required");
|
|
130
|
+
const credentials = awsCredentials();
|
|
131
|
+
const now = new Date();
|
|
132
|
+
const amzDate = now.toISOString().replace(/[:-]|\.\d{3}/g, "");
|
|
133
|
+
const dateStamp = amzDate.slice(0, 8);
|
|
134
|
+
const target = s3RequestTarget({ bucket, key, region });
|
|
135
|
+
const headers = {
|
|
136
|
+
host: target.host,
|
|
137
|
+
"x-amz-content-sha256": payloadHash,
|
|
138
|
+
"x-amz-date": amzDate,
|
|
139
|
+
};
|
|
140
|
+
if (contentLength > 0 || method === "PUT") {
|
|
141
|
+
headers["content-length"] = String(contentLength);
|
|
142
|
+
}
|
|
143
|
+
if (credentials.sessionToken) {
|
|
144
|
+
headers["x-amz-security-token"] = credentials.sessionToken;
|
|
145
|
+
}
|
|
146
|
+
const signedHeaders = Object.keys(headers).sort().join(";");
|
|
147
|
+
const canonicalHeaders = Object.keys(headers)
|
|
148
|
+
.sort()
|
|
149
|
+
.map((name) => `${name}:${String(headers[name]).trim().replace(/\s+/g, " ")}\n`)
|
|
150
|
+
.join("");
|
|
151
|
+
const canonicalRequest = [
|
|
152
|
+
method,
|
|
153
|
+
target.path,
|
|
154
|
+
"",
|
|
155
|
+
canonicalHeaders,
|
|
156
|
+
signedHeaders,
|
|
157
|
+
payloadHash,
|
|
158
|
+
].join("\n");
|
|
159
|
+
const credentialScope = `${dateStamp}/${region}/s3/aws4_request`;
|
|
160
|
+
const stringToSign = [
|
|
161
|
+
"AWS4-HMAC-SHA256",
|
|
162
|
+
amzDate,
|
|
163
|
+
credentialScope,
|
|
164
|
+
sha256Hex(canonicalRequest),
|
|
165
|
+
].join("\n");
|
|
166
|
+
const dateKey = hmac(`AWS4${credentials.secretAccessKey}`, dateStamp);
|
|
167
|
+
const regionKey = hmac(dateKey, region);
|
|
168
|
+
const serviceKey = hmac(regionKey, "s3");
|
|
169
|
+
const signingKey = hmac(serviceKey, "aws4_request");
|
|
170
|
+
const signature = hmac(signingKey, stringToSign, "hex");
|
|
171
|
+
return {
|
|
172
|
+
method,
|
|
173
|
+
host: target.host,
|
|
174
|
+
path: target.path,
|
|
175
|
+
headers: {
|
|
176
|
+
...headers,
|
|
177
|
+
authorization: `AWS4-HMAC-SHA256 Credential=${credentials.accessKeyId}/${credentialScope}, SignedHeaders=${signedHeaders}, Signature=${signature}`,
|
|
178
|
+
},
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function requestS3({ method, bucket, key, region, payloadHash, contentLength = 0, bodyPath = "", outputPath = "" }) {
|
|
183
|
+
return new Promise((resolve, reject) => {
|
|
184
|
+
const request = signS3Request({ method, bucket, key, region, payloadHash, contentLength });
|
|
185
|
+
const req = https.request(
|
|
186
|
+
{
|
|
187
|
+
method: request.method,
|
|
188
|
+
host: request.host,
|
|
189
|
+
path: request.path,
|
|
190
|
+
headers: request.headers,
|
|
191
|
+
},
|
|
192
|
+
(res) => {
|
|
193
|
+
const chunks = [];
|
|
194
|
+
const output = outputPath ? fs.createWriteStream(outputPath) : null;
|
|
195
|
+
if (output) {
|
|
196
|
+
output.on("error", reject);
|
|
197
|
+
}
|
|
198
|
+
res.on("data", (chunk) => {
|
|
199
|
+
if (output && res.statusCode >= 200 && res.statusCode < 300) {
|
|
200
|
+
output.write(chunk);
|
|
201
|
+
} else {
|
|
202
|
+
chunks.push(chunk);
|
|
203
|
+
}
|
|
204
|
+
});
|
|
205
|
+
res.on("end", () => {
|
|
206
|
+
const body = Buffer.concat(chunks).toString("utf8");
|
|
207
|
+
if (res.statusCode < 200 || res.statusCode >= 300) {
|
|
208
|
+
if (output) {
|
|
209
|
+
output.destroy();
|
|
210
|
+
fs.rmSync(outputPath, { force: true });
|
|
211
|
+
}
|
|
212
|
+
reject(new Error(`S3 ${method} s3://${bucket}/${key} failed with HTTP ${res.statusCode}: ${body.slice(0, 500)}`));
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
if (output) {
|
|
216
|
+
output.end(resolve);
|
|
217
|
+
return;
|
|
218
|
+
}
|
|
219
|
+
resolve();
|
|
220
|
+
});
|
|
221
|
+
res.on("error", reject);
|
|
222
|
+
},
|
|
223
|
+
);
|
|
224
|
+
req.on("error", reject);
|
|
225
|
+
if (bodyPath) {
|
|
226
|
+
fs.createReadStream(bodyPath).on("error", reject).pipe(req);
|
|
227
|
+
} else {
|
|
228
|
+
req.end();
|
|
229
|
+
}
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
function fakeS3Path(bucket, key) {
|
|
234
|
+
const fakeRoot = env("BUILDCHAIN_ARTIFACT_RELAY_FAKE_S3_ROOT");
|
|
235
|
+
if (!fakeRoot) return "";
|
|
236
|
+
return path.join(fakeRoot, bucket, ...assertSafeRelativePath(key).split("/"));
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
async function putS3Object({ bucket, key, region, filePath, sha256, size }) {
|
|
240
|
+
const fakePath = fakeS3Path(bucket, key);
|
|
241
|
+
if (fakePath) {
|
|
242
|
+
fs.mkdirSync(path.dirname(fakePath), { recursive: true });
|
|
243
|
+
fs.copyFileSync(filePath, fakePath);
|
|
244
|
+
return;
|
|
245
|
+
}
|
|
246
|
+
await requestS3({
|
|
247
|
+
method: "PUT",
|
|
248
|
+
bucket,
|
|
249
|
+
key,
|
|
250
|
+
region,
|
|
251
|
+
payloadHash: sha256,
|
|
252
|
+
contentLength: size,
|
|
253
|
+
bodyPath: filePath,
|
|
254
|
+
});
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
async function getS3Object({ bucket, key, region, targetPath }) {
|
|
258
|
+
const fakePath = fakeS3Path(bucket, key);
|
|
259
|
+
if (fakePath) {
|
|
260
|
+
fs.copyFileSync(fakePath, targetPath);
|
|
261
|
+
return;
|
|
262
|
+
}
|
|
263
|
+
await requestS3({
|
|
264
|
+
method: "GET",
|
|
265
|
+
bucket,
|
|
266
|
+
key,
|
|
267
|
+
region,
|
|
268
|
+
payloadHash: sha256Hex(""),
|
|
269
|
+
outputPath: targetPath,
|
|
270
|
+
});
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
async function deleteS3Object({ bucket, key, region }) {
|
|
274
|
+
const fakePath = fakeS3Path(bucket, key);
|
|
275
|
+
if (fakePath) {
|
|
276
|
+
fs.rmSync(fakePath, { force: true });
|
|
277
|
+
return;
|
|
278
|
+
}
|
|
279
|
+
await requestS3({
|
|
280
|
+
method: "DELETE",
|
|
281
|
+
bucket,
|
|
282
|
+
key,
|
|
283
|
+
region,
|
|
284
|
+
payloadHash: sha256Hex(""),
|
|
285
|
+
});
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
function writeManifest(manifestPath, manifest) {
|
|
289
|
+
fs.mkdirSync(path.dirname(manifestPath), { recursive: true });
|
|
290
|
+
fs.writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`);
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
function resolveUploadGroups() {
|
|
294
|
+
return [
|
|
295
|
+
{
|
|
296
|
+
role: "payload",
|
|
297
|
+
artifactName: env("BUILDCHAIN_ARTIFACT_RELAY_PAYLOAD_ARTIFACT_NAME"),
|
|
298
|
+
paths: splitLines(env("BUILDCHAIN_ARTIFACT_RELAY_PAYLOAD_PATHS")),
|
|
299
|
+
required: true,
|
|
300
|
+
},
|
|
301
|
+
{
|
|
302
|
+
role: "manifest",
|
|
303
|
+
artifactName: env("BUILDCHAIN_ARTIFACT_RELAY_MANIFEST_ARTIFACT_NAME"),
|
|
304
|
+
paths: splitLines(env("BUILDCHAIN_ARTIFACT_RELAY_MANIFEST_PATHS")),
|
|
305
|
+
required: true,
|
|
306
|
+
},
|
|
307
|
+
{
|
|
308
|
+
role: "diagnostics",
|
|
309
|
+
artifactName: env("BUILDCHAIN_ARTIFACT_RELAY_DIAGNOSTICS_ARTIFACT_NAME"),
|
|
310
|
+
paths: splitLines(env("BUILDCHAIN_ARTIFACT_RELAY_DIAGNOSTICS_PATHS")),
|
|
311
|
+
required: true,
|
|
312
|
+
},
|
|
313
|
+
];
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
export async function uploadRelayArtifacts({
|
|
317
|
+
workspace = env("BUILDCHAIN_ARTIFACT_RELAY_WORKSPACE", process.cwd()),
|
|
318
|
+
manifestPath = env("BUILDCHAIN_ARTIFACT_RELAY_MANIFEST_PATH", ".buildchain/artifacts/relay-manifest.json"),
|
|
319
|
+
bucket = env("BUILDCHAIN_ARTIFACT_RELAY_BUCKET"),
|
|
320
|
+
region = env("BUILDCHAIN_ARTIFACT_RELAY_REGION"),
|
|
321
|
+
prefix = env("BUILDCHAIN_ARTIFACT_RELAY_PREFIX", "buildchain-artifacts"),
|
|
322
|
+
groups = resolveUploadGroups(),
|
|
323
|
+
repository = env("GITHUB_REPOSITORY"),
|
|
324
|
+
runId = env("GITHUB_RUN_ID"),
|
|
325
|
+
runAttempt = env("GITHUB_RUN_ATTEMPT"),
|
|
326
|
+
sourceSha = env("BUILDCHAIN_ARTIFACT_RELAY_SOURCE_SHA", env("GITHUB_SHA")),
|
|
327
|
+
platformId = env("BUILDCHAIN_ARTIFACT_RELAY_PLATFORM_ID", os.platform()),
|
|
328
|
+
platformName = env("BUILDCHAIN_ARTIFACT_RELAY_PLATFORM_NAME", platformId),
|
|
329
|
+
} = {}) {
|
|
330
|
+
if (!bucket) throw new Error("BUILDCHAIN_ARTIFACT_RELAY_BUCKET is required");
|
|
331
|
+
if (!region) throw new Error("BUILDCHAIN_ARTIFACT_RELAY_REGION is required");
|
|
332
|
+
const resolvedWorkspace = path.resolve(workspace);
|
|
333
|
+
const basePrefix = [
|
|
334
|
+
normalizePrefix(prefix),
|
|
335
|
+
safeSegment(repository || "repository"),
|
|
336
|
+
safeSegment(runId || "run"),
|
|
337
|
+
safeSegment(runAttempt || "attempt"),
|
|
338
|
+
safeSegment(sourceSha || "sha"),
|
|
339
|
+
safeSegment(platformId || "platform"),
|
|
340
|
+
].join("/");
|
|
341
|
+
const manifestGroups = [];
|
|
342
|
+
|
|
343
|
+
for (const group of groups) {
|
|
344
|
+
const artifactName = String(group.artifactName || "").trim();
|
|
345
|
+
if (!artifactName) {
|
|
346
|
+
throw new Error(`relay ${group.role} artifact name is required`);
|
|
347
|
+
}
|
|
348
|
+
const files = collectGroupFiles(resolvedWorkspace, group.paths || []);
|
|
349
|
+
if (group.required && files.length === 0) {
|
|
350
|
+
throw new Error(`relay ${group.role} artifact ${artifactName} matched no files`);
|
|
351
|
+
}
|
|
352
|
+
const groupPrefix = `${basePrefix}/${safeSegment(group.role)}/${safeSegment(artifactName)}`;
|
|
353
|
+
const objects = [];
|
|
354
|
+
for (const file of files) {
|
|
355
|
+
const relativePath = assertSafeRelativePath(toPosix(path.relative(resolvedWorkspace, file)));
|
|
356
|
+
const stat = fs.statSync(file);
|
|
357
|
+
const sha256 = sha256File(file);
|
|
358
|
+
const key = `${groupPrefix}/${relativePath}`;
|
|
359
|
+
await putS3Object({ bucket, key, region, filePath: file, sha256, size: stat.size });
|
|
360
|
+
objects.push({
|
|
361
|
+
relativePath,
|
|
362
|
+
size: stat.size,
|
|
363
|
+
sha256,
|
|
364
|
+
bucket,
|
|
365
|
+
key,
|
|
366
|
+
uri: `s3://${bucket}/${key}`,
|
|
367
|
+
});
|
|
368
|
+
}
|
|
369
|
+
manifestGroups.push({
|
|
370
|
+
role: group.role,
|
|
371
|
+
artifactName,
|
|
372
|
+
fileCount: objects.length,
|
|
373
|
+
totalBytes: objects.reduce((sum, object) => sum + object.size, 0),
|
|
374
|
+
objects,
|
|
375
|
+
});
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
const manifest = {
|
|
379
|
+
schemaVersion: 1,
|
|
380
|
+
contract: CONTRACT,
|
|
381
|
+
transferMode: "s3-to-github-artifacts",
|
|
382
|
+
provider: "s3",
|
|
383
|
+
generatedAt: new Date().toISOString(),
|
|
384
|
+
repository,
|
|
385
|
+
runId,
|
|
386
|
+
runAttempt,
|
|
387
|
+
sourceSha,
|
|
388
|
+
platform: {
|
|
389
|
+
id: platformId,
|
|
390
|
+
name: platformName,
|
|
391
|
+
},
|
|
392
|
+
s3: {
|
|
393
|
+
bucket,
|
|
394
|
+
region,
|
|
395
|
+
prefix: basePrefix,
|
|
396
|
+
},
|
|
397
|
+
groups: manifestGroups,
|
|
398
|
+
};
|
|
399
|
+
const resolvedManifestPath = path.resolve(resolvedWorkspace, manifestPath);
|
|
400
|
+
writeManifest(resolvedManifestPath, manifest);
|
|
401
|
+
writeGitHubOutputs({
|
|
402
|
+
"relay-manifest-path": toPosix(path.relative(resolvedWorkspace, resolvedManifestPath)),
|
|
403
|
+
"relay-object-count": String(manifestGroups.reduce((sum, group) => sum + group.fileCount, 0)),
|
|
404
|
+
"relay-total-bytes": String(manifestGroups.reduce((sum, group) => sum + group.totalBytes, 0)),
|
|
405
|
+
});
|
|
406
|
+
return manifest;
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
function findRelayManifest(inputRoot, expectedPlatformId = "") {
|
|
410
|
+
const matches = [];
|
|
411
|
+
function walk(dir) {
|
|
412
|
+
if (!fs.existsSync(dir)) return;
|
|
413
|
+
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
414
|
+
const current = path.join(dir, entry.name);
|
|
415
|
+
if (entry.isDirectory()) {
|
|
416
|
+
walk(current);
|
|
417
|
+
} else if (entry.name === "relay-manifest.json") {
|
|
418
|
+
const manifest = JSON.parse(fs.readFileSync(current, "utf8"));
|
|
419
|
+
if (manifest.contract === CONTRACT && (!expectedPlatformId || manifest.platform?.id === expectedPlatformId)) {
|
|
420
|
+
matches.push({ path: current, manifest });
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
walk(path.resolve(inputRoot));
|
|
426
|
+
if (matches.length !== 1) {
|
|
427
|
+
throw new Error(`expected exactly one relay-manifest.json${expectedPlatformId ? ` for ${expectedPlatformId}` : ""}, found ${matches.length}`);
|
|
428
|
+
}
|
|
429
|
+
return matches[0];
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
export async function downloadRelayArtifacts({
|
|
433
|
+
inputRoot = env("BUILDCHAIN_ARTIFACT_RELAY_INPUT_ROOT", ".buildchain/downloaded-relay-manifests"),
|
|
434
|
+
outputRoot = env("BUILDCHAIN_ARTIFACT_RELAY_OUTPUT_ROOT", ".buildchain/relayed-artifacts"),
|
|
435
|
+
region = env("BUILDCHAIN_ARTIFACT_RELAY_REGION"),
|
|
436
|
+
platformId = env("BUILDCHAIN_ARTIFACT_RELAY_PLATFORM_ID", ""),
|
|
437
|
+
} = {}) {
|
|
438
|
+
const { manifest } = findRelayManifest(inputRoot, platformId);
|
|
439
|
+
const resolvedOutputRoot = path.resolve(outputRoot);
|
|
440
|
+
const effectiveRegion = region || manifest.s3?.region || "";
|
|
441
|
+
if (!effectiveRegion) {
|
|
442
|
+
throw new Error("relay download region is required");
|
|
443
|
+
}
|
|
444
|
+
const outputs = {};
|
|
445
|
+
let objectCount = 0;
|
|
446
|
+
let totalBytes = 0;
|
|
447
|
+
for (const group of manifest.groups || []) {
|
|
448
|
+
const groupDir = path.join(resolvedOutputRoot, safeSegment(group.role));
|
|
449
|
+
fs.mkdirSync(groupDir, { recursive: true });
|
|
450
|
+
for (const object of group.objects || []) {
|
|
451
|
+
const relativePath = assertSafeRelativePath(object.relativePath);
|
|
452
|
+
const targetPath = path.join(groupDir, ...relativePath.split("/"));
|
|
453
|
+
fs.mkdirSync(path.dirname(targetPath), { recursive: true });
|
|
454
|
+
await getS3Object({ bucket: object.bucket, key: object.key, region: effectiveRegion, targetPath });
|
|
455
|
+
const actualSha256 = sha256File(targetPath);
|
|
456
|
+
if (actualSha256 !== object.sha256) {
|
|
457
|
+
throw new Error(`relay sha256 mismatch for ${relativePath}: expected ${object.sha256}, got ${actualSha256}`);
|
|
458
|
+
}
|
|
459
|
+
objectCount += 1;
|
|
460
|
+
totalBytes += Number(object.size || 0);
|
|
461
|
+
}
|
|
462
|
+
outputs[`${group.role}-path`] = toPosix(path.relative(process.cwd(), groupDir));
|
|
463
|
+
outputs[`${group.role}-artifact-name`] = group.artifactName || "";
|
|
464
|
+
}
|
|
465
|
+
const downloadedManifestPath = path.join(resolvedOutputRoot, "relay-manifest.downloaded.json");
|
|
466
|
+
writeManifest(downloadedManifestPath, {
|
|
467
|
+
...manifest,
|
|
468
|
+
downloadedAt: new Date().toISOString(),
|
|
469
|
+
downloadedObjectCount: objectCount,
|
|
470
|
+
downloadedTotalBytes: totalBytes,
|
|
471
|
+
});
|
|
472
|
+
writeGitHubOutputs({
|
|
473
|
+
...outputs,
|
|
474
|
+
"relay-downloaded-manifest-path": toPosix(path.relative(process.cwd(), downloadedManifestPath)),
|
|
475
|
+
"relay-downloaded-object-count": String(objectCount),
|
|
476
|
+
"relay-downloaded-total-bytes": String(totalBytes),
|
|
477
|
+
});
|
|
478
|
+
return { manifest, objectCount, totalBytes };
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
export async function cleanupRelayArtifacts({
|
|
482
|
+
inputRoot = env("BUILDCHAIN_ARTIFACT_RELAY_INPUT_ROOT", ".buildchain/downloaded-relay-manifests"),
|
|
483
|
+
region = env("BUILDCHAIN_ARTIFACT_RELAY_REGION"),
|
|
484
|
+
platformId = env("BUILDCHAIN_ARTIFACT_RELAY_PLATFORM_ID", ""),
|
|
485
|
+
} = {}) {
|
|
486
|
+
const { manifest } = findRelayManifest(inputRoot, platformId);
|
|
487
|
+
const effectiveRegion = region || manifest.s3?.region || "";
|
|
488
|
+
if (!effectiveRegion) {
|
|
489
|
+
throw new Error("relay cleanup region is required");
|
|
490
|
+
}
|
|
491
|
+
let objectCount = 0;
|
|
492
|
+
let totalBytes = 0;
|
|
493
|
+
for (const group of manifest.groups || []) {
|
|
494
|
+
for (const object of group.objects || []) {
|
|
495
|
+
await deleteS3Object({ bucket: object.bucket, key: object.key, region: effectiveRegion });
|
|
496
|
+
objectCount += 1;
|
|
497
|
+
totalBytes += Number(object.size || 0);
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
writeGitHubOutputs({
|
|
501
|
+
"relay-cleaned-object-count": String(objectCount),
|
|
502
|
+
"relay-cleaned-total-bytes": String(totalBytes),
|
|
503
|
+
});
|
|
504
|
+
return { manifest, objectCount, totalBytes };
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
async function main() {
|
|
508
|
+
const command = process.argv[2] || "";
|
|
509
|
+
if (command === "upload") {
|
|
510
|
+
await uploadRelayArtifacts();
|
|
511
|
+
return;
|
|
512
|
+
}
|
|
513
|
+
if (command === "download") {
|
|
514
|
+
await downloadRelayArtifacts();
|
|
515
|
+
return;
|
|
516
|
+
}
|
|
517
|
+
if (command === "cleanup") {
|
|
518
|
+
await cleanupRelayArtifacts();
|
|
519
|
+
return;
|
|
520
|
+
}
|
|
521
|
+
throw new Error("usage: artifact-relay-s3.mjs <upload|download|cleanup>");
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
|
525
|
+
main().catch((error) => {
|
|
526
|
+
console.error(`::error::${String(error.message || error).replace(/\r?\n/g, "%0A")}`);
|
|
527
|
+
process.exitCode = 1;
|
|
528
|
+
});
|
|
529
|
+
}
|
|
@@ -29,6 +29,7 @@ const requiredPaths = [
|
|
|
29
29
|
"scripts/create-release-bundle.mjs",
|
|
30
30
|
"scripts/generate-site-bundle.mjs",
|
|
31
31
|
"scripts/generate-release-candidate-passport.mjs",
|
|
32
|
+
"scripts/artifact-relay-s3.mjs",
|
|
32
33
|
"scripts/npm-publish-dry-run.mjs",
|
|
33
34
|
"scripts/npm-publish-transaction.mjs",
|
|
34
35
|
"scripts/release-candidate-resolver.mjs",
|