@akanjs/devkit 3.1.0-rc.1 → 3.1.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/akanConfig/akanConfig.test.ts +51 -5
- package/akanConfig/akanConfig.ts +31 -12
- package/akanConfig/akanWorkspaceConfig.test.ts +6 -6
- package/akanConfig/akanWorkspaceConfig.ts +1 -1
- package/applicationBuildRunner.test.ts +33 -1
- package/applicationBuildRunner.ts +4 -5
- package/cloud/cloudApi.ts +22 -15
- package/cloud/cloudHttpError.test.ts +32 -0
- package/cloud/cloudHttpError.ts +54 -0
- package/cloud/cloudUpdates.ts +138 -0
- package/cloud/globalConfig.test.ts +17 -0
- package/cloud/globalConfig.ts +8 -2
- package/cloud/index.ts +3 -0
- package/cloud/pipelineSecrets.ts +114 -0
- package/cloud/updatePackUpload.test.ts +100 -45
- package/cloud/updatePackUpload.ts +38 -18
- package/commandDecorators/argMeta.ts +11 -2
- package/commandDecorators/command.ts +40 -6
- package/commandDecorators/commandBuilder.ts +6 -1
- package/commandDecorators/commandDecorators.test.ts +205 -4
- package/commandDecorators/exitCodeError.ts +15 -0
- package/commandDecorators/index.ts +1 -0
- package/executors.test.ts +46 -0
- package/executors.ts +26 -23
- package/frontendBuild/csrArtifactBuilder.ts +18 -1
- package/incrementalBuilder/buildBatch.proc.ts +12 -7
- package/incrementalBuilder/incrementalBuilder.proc.test.ts +63 -0
- package/incrementalBuilder/incrementalBuilder.proc.ts +7 -5
- package/mobile/desktopServerStage.test.ts +13 -0
- package/mobile/desktopServerStage.ts +5 -7
- package/mobile/index.ts +2 -0
- package/mobile/nativeApi.test.ts +26 -0
- package/mobile/nativeApi.ts +5 -2
- package/mobile/nativeApp.test.ts +135 -33
- package/mobile/nativeApp.ts +109 -73
- package/mobile/nativeConfig.ts +2 -1
- package/mobile/nativePublicEnv.test.ts +131 -0
- package/mobile/nativePublicEnv.ts +116 -0
- package/mobile/serverBundleSecrets.test.ts +79 -0
- package/mobile/serverBundleSecrets.ts +60 -16
- package/mobile/updatePackFolder.test.ts +98 -0
- package/mobile/updatePackFolder.ts +131 -0
- package/mobile/updateUploader.test.ts +114 -22
- package/mobile/updateUploader.ts +75 -8
- package/package.json +2 -2
- package/pipeline/PipelineCompiler.test.ts +157 -1
- package/pipeline/PipelineCompiler.ts +80 -46
- package/pipeline/PipelineDefinition.test.ts +9 -0
- package/pipeline/PipelineDefinition.ts +4 -3
- package/pipeline/PipelineDryRun.test.ts +27 -0
- package/pipeline/PipelineDryRun.ts +2 -2
- package/pipeline/PipelineInputReader.ts +157 -253
- package/pipeline/PipelineRules.ts +92 -0
- package/pipeline/index.ts +1 -0
- package/qualityScanner.test.ts +5 -3
- package/slicePlanner.ts +5 -3
- package/subspace.test.ts +90 -19
- package/subspace.ts +52 -500
- package/subspaceClone.ts +361 -0
- package/subspaceConfig.ts +1 -2
- package/subspaceReport.ts +162 -0
- package/transforms/serverEnvPlugin.test.ts +76 -0
- package/transforms/serverEnvPlugin.ts +9 -25
- package/transforms/serverEnvProfile.test.ts +189 -0
- package/transforms/serverEnvProfile.ts +324 -60
- package/typecheck/projectTypecheck.ts +6 -5
package/cloud/globalConfig.ts
CHANGED
|
@@ -54,8 +54,14 @@ export class GlobalConfig {
|
|
|
54
54
|
});
|
|
55
55
|
}
|
|
56
56
|
static async hostFor(workspace: { getConfig(): Promise<{ cloudHost: string | null }> }, host?: string | null) {
|
|
57
|
-
|
|
58
|
-
|
|
57
|
+
return GlobalConfig.#normalizeHost(host || ((await workspace.getConfig()).cloudHost ?? GlobalConfig.akanCloudHost));
|
|
58
|
+
}
|
|
59
|
+
//? A login is stored under the host string, so `--host` and the workspace's `cloud.host` must spell one cloud alike.
|
|
60
|
+
static #normalizeHost(host: string) {
|
|
61
|
+
const url = URL.canParse(host) ? new URL(host) : null;
|
|
62
|
+
return url && (url.protocol === "https:" || url.protocol === "http:")
|
|
63
|
+
? `${url.origin}${url.pathname.replace(/\/+$/, "")}`
|
|
64
|
+
: host.replace(/\/+$/, "");
|
|
59
65
|
}
|
|
60
66
|
static async getHostConfig(host = GlobalConfig.akanCloudHost): Promise<HostConfig> {
|
|
61
67
|
const akanConfig = await GlobalConfig.#getAkanGlobalConfig();
|
package/cloud/index.ts
CHANGED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import { capitalize, Logger, PipelineNames } from "akanjs/common";
|
|
2
|
+
import { CloudApi, type PipelineSecretListing } from "./cloudApi";
|
|
3
|
+
|
|
4
|
+
//* The named secrets an Akan Cloud project hands its pipeline: written and removed by name, listed without values.
|
|
5
|
+
export class PipelineSecrets {
|
|
6
|
+
static readonly #secretBytes = 64 * 1024;
|
|
7
|
+
//? The branches Akan Cloud deploys (its BranchType): a secret scoped to any other is refused by the server.
|
|
8
|
+
static readonly #secretBranches = ["debug", "develop", "main"];
|
|
9
|
+
static readonly #secretsUnavailable = "The server is not configured for pipeline secrets yet";
|
|
10
|
+
|
|
11
|
+
constructor(
|
|
12
|
+
readonly cloudApi: CloudApi,
|
|
13
|
+
readonly devProjectId: string,
|
|
14
|
+
) {}
|
|
15
|
+
|
|
16
|
+
//* Never an argument: a value on the command line lands in shell history and in any `ps` listing.
|
|
17
|
+
static async readValue(name: string, file: string | null) {
|
|
18
|
+
const raw = file
|
|
19
|
+
? await Bun.file(file).text()
|
|
20
|
+
: process.stdin.isTTY
|
|
21
|
+
? await (await import("@inquirer/prompts")).password({
|
|
22
|
+
message: `Value of ${name} (hidden; it can never be read back)`,
|
|
23
|
+
mask: "*",
|
|
24
|
+
})
|
|
25
|
+
: await PipelineSecrets.#readStdin(name);
|
|
26
|
+
const value = raw.replace(/\r?\n$/, "");
|
|
27
|
+
if (!value) throw new Error(`The value of ${name} is empty.`);
|
|
28
|
+
const bytes = new TextEncoder().encode(value).length;
|
|
29
|
+
if (bytes > PipelineSecrets.#secretBytes)
|
|
30
|
+
throw new Error(`The value of ${name} is ${bytes} bytes, over ${PipelineSecrets.#secretBytes}.`);
|
|
31
|
+
return value;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
static async #readStdin(name: string) {
|
|
35
|
+
process.stderr.write(`Reading the value of ${name} from stdin until it closes (or pass --file <path>).\n`);
|
|
36
|
+
return await Bun.stdin.text();
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async upload({ name, branch, value }: { name: string; branch: string | null; value: string }) {
|
|
40
|
+
PipelineSecrets.assertName(name);
|
|
41
|
+
PipelineSecrets.assertBranch(branch);
|
|
42
|
+
const saved = await this.cloudApi
|
|
43
|
+
.uploadSecret(this.devProjectId, { scope: branch ?? "*", name, value })
|
|
44
|
+
.catch((error: unknown) => {
|
|
45
|
+
throw CloudApi.refusal(error, "a secret is written by a project member signed in with `akan login`", {
|
|
46
|
+
unavailable: PipelineSecrets.#secretsUnavailable,
|
|
47
|
+
});
|
|
48
|
+
});
|
|
49
|
+
Logger.rawLog(
|
|
50
|
+
`Saved ${saved.name} for ${saved.scope === "*" ? "every branch" : `branch ${saved.scope}`}. Builds that start from now on use it.`,
|
|
51
|
+
);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
async list(branch: string | null) {
|
|
55
|
+
PipelineSecrets.assertBranch(branch);
|
|
56
|
+
const listed = await this.cloudApi.listSecrets(this.devProjectId).catch((error: unknown) => {
|
|
57
|
+
throw CloudApi.refusal(error, "the secrets are listed for a project member", {
|
|
58
|
+
unavailable: PipelineSecrets.#secretsUnavailable,
|
|
59
|
+
});
|
|
60
|
+
});
|
|
61
|
+
const secrets = listed.filter((secret) => !branch || secret.scope === "*" || secret.scope === branch);
|
|
62
|
+
Logger.rawLog(PipelineSecrets.format(secrets));
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
async remove({ name, branch }: { name: string; branch: string | null }) {
|
|
66
|
+
PipelineSecrets.assertName(name);
|
|
67
|
+
PipelineSecrets.assertBranch(branch);
|
|
68
|
+
const removed = await this.cloudApi
|
|
69
|
+
.removeSecret(this.devProjectId, { scope: branch ?? "*", name })
|
|
70
|
+
.catch((error: unknown) => {
|
|
71
|
+
throw CloudApi.refusal(error, "a secret is removed by a project member signed in with `akan login`", {
|
|
72
|
+
unavailable: PipelineSecrets.#secretsUnavailable,
|
|
73
|
+
});
|
|
74
|
+
});
|
|
75
|
+
Logger.rawLog(
|
|
76
|
+
removed
|
|
77
|
+
? `Removed ${name} for ${branch ? `branch ${branch}` : "every branch"}.`
|
|
78
|
+
: `No ${name} for ${branch ? `branch ${branch}` : "every branch"} to remove.`,
|
|
79
|
+
);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
static format(secrets: PipelineSecretListing[]) {
|
|
83
|
+
if (!secrets.length) return "No pipeline secrets.";
|
|
84
|
+
const rows = [...secrets]
|
|
85
|
+
.sort((a, b) => a.name.localeCompare(b.name) || a.scope.localeCompare(b.scope))
|
|
86
|
+
.map((secret) =>
|
|
87
|
+
[secret.name, secret.scope === "*" ? "every branch" : secret.scope, secret.updatedAt, secret.updatedBy].map(
|
|
88
|
+
(cell) => String(cell ?? "-"),
|
|
89
|
+
),
|
|
90
|
+
);
|
|
91
|
+
const header = ["NAME", "SCOPE", "UPDATED", "BY"];
|
|
92
|
+
const widths = header.map((title, column) =>
|
|
93
|
+
Math.max(title.length, ...rows.map((row) => row[column]?.length ?? 0)),
|
|
94
|
+
);
|
|
95
|
+
return [header, ...rows]
|
|
96
|
+
.map((row) =>
|
|
97
|
+
row
|
|
98
|
+
.map((cell, column) => cell.padEnd(widths[column] ?? 0))
|
|
99
|
+
.join(" ")
|
|
100
|
+
.trimEnd(),
|
|
101
|
+
)
|
|
102
|
+
.join("\n");
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
static assertName(name: string) {
|
|
106
|
+
const problem = PipelineNames.secretNameError(name);
|
|
107
|
+
if (problem) throw new Error(`${capitalize(problem)}.`);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
static assertBranch(branch: string | null) {
|
|
111
|
+
if (branch !== null && !PipelineSecrets.#secretBranches.includes(branch))
|
|
112
|
+
throw new Error(`--branch ${branch} is not a deployed branch: ${PipelineSecrets.#secretBranches.join(", ")}.`);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
@@ -13,14 +13,15 @@ interface Put {
|
|
|
13
13
|
body: Uint8Array;
|
|
14
14
|
}
|
|
15
15
|
|
|
16
|
-
class
|
|
16
|
+
class FakeCloudHost {
|
|
17
17
|
readonly server: ReturnType<typeof Bun.serve>;
|
|
18
18
|
readonly requests: Record<string, unknown>[] = [];
|
|
19
19
|
readonly puts: Put[] = [];
|
|
20
20
|
readonly commits: string[] = [];
|
|
21
21
|
readonly registrations: Record<string, string>[] = [];
|
|
22
|
-
putStatuses: number[] = [];
|
|
23
|
-
|
|
22
|
+
putStatuses: (number | Response)[] = [];
|
|
23
|
+
stallPuts: boolean = false;
|
|
24
|
+
commitAnswers: (number | [number, string] | Response)[] = [];
|
|
24
25
|
release: Record<string, unknown> = { id: "release-1", status: "published", problems: [] };
|
|
25
26
|
requestRefusal: Response | null = null;
|
|
26
27
|
|
|
@@ -38,6 +39,7 @@ class FakeAkasys {
|
|
|
38
39
|
this.commits.length = 0;
|
|
39
40
|
this.registrations.length = 0;
|
|
40
41
|
this.putStatuses = [];
|
|
42
|
+
this.stallPuts = false;
|
|
41
43
|
this.commitAnswers = [];
|
|
42
44
|
this.requestRefusal = null;
|
|
43
45
|
}
|
|
@@ -61,7 +63,9 @@ class FakeAkasys {
|
|
|
61
63
|
headers: Object.fromEntries(request.headers),
|
|
62
64
|
body: new Uint8Array(await request.arrayBuffer()),
|
|
63
65
|
});
|
|
64
|
-
return new Response(
|
|
66
|
+
if (this.stallPuts) return await new Promise<Response>(() => undefined);
|
|
67
|
+
const answer = this.putStatuses.shift() ?? 200;
|
|
68
|
+
return answer instanceof Response ? answer : new Response(null, { status: answer });
|
|
65
69
|
}
|
|
66
70
|
if (url.pathname === "/api/commitUpdateUpload/project-1") {
|
|
67
71
|
const { uploadId } = (await request.json()) as { uploadId: string };
|
|
@@ -85,15 +89,16 @@ class FakeAkasys {
|
|
|
85
89
|
#answer() {
|
|
86
90
|
const answer = this.commitAnswers.shift();
|
|
87
91
|
if (!answer) return null;
|
|
92
|
+
if (answer instanceof Response) return answer;
|
|
88
93
|
const [status, key] = typeof answer === "number" ? [answer, "release.error.uploadCommitInProgress"] : answer;
|
|
89
94
|
return Response.json({ error: key }, { status });
|
|
90
95
|
}
|
|
91
96
|
}
|
|
92
97
|
|
|
93
98
|
const makeDir = tempDirs("akan-update-upload-");
|
|
94
|
-
const
|
|
99
|
+
const cloudHost = new FakeCloudHost();
|
|
95
100
|
const api = new CloudApi({} as Workspace, {
|
|
96
|
-
host:
|
|
101
|
+
host: cloudHost.host,
|
|
97
102
|
auth: { accessToken: { jwt: "jwt-1", refreshToken: null, expiresAt: null } },
|
|
98
103
|
});
|
|
99
104
|
const fields = { appName: "lu", target: "default", platform: "android" as const, channel: "develop", notes: null };
|
|
@@ -105,8 +110,8 @@ const archiveOf = async (bytes: string) => {
|
|
|
105
110
|
return archive;
|
|
106
111
|
};
|
|
107
112
|
|
|
108
|
-
beforeEach(() =>
|
|
109
|
-
afterAll(() =>
|
|
113
|
+
beforeEach(() => cloudHost.reset());
|
|
114
|
+
afterAll(() => cloudHost.server.stop(true));
|
|
110
115
|
|
|
111
116
|
describe("UpdatePackUpload", () => {
|
|
112
117
|
test("announces the archive's size and digest, PUTs it with nothing but its content type, then commits", async () => {
|
|
@@ -115,7 +120,7 @@ describe("UpdatePackUpload", () => {
|
|
|
115
120
|
const release = await new UpdatePackUpload(api, "project-1", quick).send(archive, fields);
|
|
116
121
|
|
|
117
122
|
expect(release).toEqual({ id: "release-1", status: "published", problems: [], uploadId: "upload-1" });
|
|
118
|
-
expect(
|
|
123
|
+
expect(cloudHost.requests).toEqual([
|
|
119
124
|
{
|
|
120
125
|
...fields,
|
|
121
126
|
size: 10,
|
|
@@ -123,98 +128,129 @@ describe("UpdatePackUpload", () => {
|
|
|
123
128
|
authorization: "Bearer jwt-1",
|
|
124
129
|
},
|
|
125
130
|
]);
|
|
126
|
-
const [put] =
|
|
131
|
+
const [put] = cloudHost.puts;
|
|
127
132
|
expect(new TextDecoder().decode(put?.body)).toBe("pack bytes");
|
|
128
133
|
expect(put?.headers["content-type"]).toBe("application/gzip");
|
|
129
134
|
expect(put?.headers["content-length"]).toBe("10");
|
|
130
135
|
expect(put?.headers.authorization).toBeUndefined();
|
|
131
136
|
expect(Object.keys(put?.headers ?? {}).filter((name) => name.startsWith("x-amz-"))).toEqual([]);
|
|
132
|
-
expect(
|
|
137
|
+
expect(cloudHost.commits).toEqual(["upload-1"]);
|
|
133
138
|
});
|
|
134
139
|
|
|
135
140
|
test("takes one fresh ticket when storage answers 403, and gives up on the second without naming the URL", async () => {
|
|
136
141
|
const archive = await archiveOf("pack");
|
|
137
|
-
|
|
142
|
+
cloudHost.putStatuses = [403];
|
|
138
143
|
|
|
139
144
|
await new UpdatePackUpload(api, "project-1", quick).send(archive, fields);
|
|
140
|
-
expect(
|
|
141
|
-
expect(
|
|
145
|
+
expect(cloudHost.puts.map(({ uploadId }) => uploadId)).toEqual(["upload-1", "upload-2"]);
|
|
146
|
+
expect(cloudHost.commits).toEqual(["upload-2"]);
|
|
142
147
|
|
|
143
|
-
|
|
144
|
-
|
|
148
|
+
cloudHost.reset();
|
|
149
|
+
cloudHost.putStatuses = [403, 403];
|
|
145
150
|
const refusal = await new UpdatePackUpload(api, "project-1", quick).send(archive, fields).catch((e: Error) => e);
|
|
146
151
|
expect(String(refusal)).toContain("HTTP 403");
|
|
147
152
|
expect(String(refusal)).not.toContain("secret-signature");
|
|
148
|
-
expect(
|
|
153
|
+
expect(cloudHost.commits).toEqual([]);
|
|
149
154
|
});
|
|
150
155
|
|
|
151
156
|
test("retries a PUT the way curl --retry does, and stops at a refusal that is not transient", async () => {
|
|
152
157
|
const archive = await archiveOf("pack");
|
|
153
|
-
|
|
158
|
+
cloudHost.putStatuses = [503, 502];
|
|
154
159
|
await new UpdatePackUpload(api, "project-1", quick).send(archive, fields);
|
|
155
|
-
expect(
|
|
160
|
+
expect(cloudHost.puts.map(({ uploadId }) => uploadId)).toEqual(["upload-1", "upload-1", "upload-1"]);
|
|
156
161
|
|
|
157
|
-
|
|
158
|
-
|
|
162
|
+
cloudHost.reset();
|
|
163
|
+
cloudHost.putStatuses = [400];
|
|
159
164
|
await expect(new UpdatePackUpload(api, "project-1", quick).send(archive, fields)).rejects.toThrow("HTTP 400");
|
|
160
|
-
expect(
|
|
165
|
+
expect(cloudHost.puts).toHaveLength(1);
|
|
161
166
|
});
|
|
162
167
|
|
|
163
168
|
test("asks again with the same id while the commit is cut off or still running, but not past a real answer", async () => {
|
|
164
169
|
const archive = await archiveOf("pack");
|
|
165
|
-
|
|
170
|
+
cloudHost.commitAnswers = [524, 409, 502];
|
|
166
171
|
await new UpdatePackUpload(api, "project-1", quick).send(archive, fields);
|
|
167
|
-
expect(
|
|
172
|
+
expect(cloudHost.commits).toEqual(["upload-1", "upload-1", "upload-1", "upload-1"]);
|
|
168
173
|
|
|
169
|
-
|
|
170
|
-
|
|
174
|
+
cloudHost.reset();
|
|
175
|
+
cloudHost.commitAnswers = [404];
|
|
171
176
|
await expect(new UpdatePackUpload(api, "project-1", quick).send(archive, fields)).rejects.toThrow("404");
|
|
172
|
-
expect(
|
|
177
|
+
expect(cloudHost.commits).toEqual(["upload-1"]);
|
|
173
178
|
|
|
174
|
-
|
|
175
|
-
|
|
179
|
+
cloudHost.reset();
|
|
180
|
+
cloudHost.commitAnswers = Array.from({ length: 100 }, () => 524);
|
|
176
181
|
const cutOff = { putRetryMs: 1, commitRetryMs: 10, cutOffWindowMs: 35, commitLimitMs: 10_000 };
|
|
177
182
|
await expect(new UpdatePackUpload(api, "project-1", cutOff).send(archive, fields)).rejects.toThrow("524");
|
|
178
|
-
expect(
|
|
183
|
+
expect(cloudHost.commits.length).toBeLessThan(6);
|
|
179
184
|
});
|
|
180
185
|
|
|
181
186
|
test("keeps waiting on a live claim past the cut-off window, up to the overall limit", async () => {
|
|
182
187
|
const archive = await archiveOf("pack");
|
|
183
|
-
|
|
188
|
+
cloudHost.commitAnswers = [...Array.from({ length: 6 }, () => 409), 524, ...Array.from({ length: 6 }, () => 409)];
|
|
184
189
|
const timing = { putRetryMs: 1, commitRetryMs: 10, cutOffWindowMs: 25, commitLimitMs: 10_000 };
|
|
185
190
|
await new UpdatePackUpload(api, "project-1", timing).send(archive, fields);
|
|
186
|
-
expect(
|
|
191
|
+
expect(cloudHost.commits).toHaveLength(14);
|
|
187
192
|
|
|
188
|
-
|
|
189
|
-
|
|
193
|
+
cloudHost.reset();
|
|
194
|
+
cloudHost.commitAnswers = Array.from({ length: 100 }, () => 409);
|
|
190
195
|
const limited = { putRetryMs: 1, commitRetryMs: 10, cutOffWindowMs: 10_000, commitLimitMs: 35 };
|
|
191
196
|
await expect(new UpdatePackUpload(api, "project-1", limited).send(archive, fields)).rejects.toThrow("409");
|
|
192
|
-
expect(
|
|
197
|
+
expect(cloudHost.commits.length).toBeLessThan(6);
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
test("gives up a stalled PUT after a time its size sets, and tries it again as a dropped one", async () => {
|
|
201
|
+
const archive = await archiveOf("pack");
|
|
202
|
+
cloudHost.stallPuts = true;
|
|
203
|
+
|
|
204
|
+
await expect(
|
|
205
|
+
new UpdatePackUpload(api, "project-1", { ...quick, putTimeoutMs: 50 }).send(archive, fields),
|
|
206
|
+
).rejects.toThrow("no response");
|
|
207
|
+
expect(cloudHost.puts).toHaveLength(4);
|
|
208
|
+
expect(cloudHost.commits).toEqual([]);
|
|
209
|
+
});
|
|
210
|
+
|
|
211
|
+
test("asks again past a gateway's 503, but not past one the server answers with its own refusal", async () => {
|
|
212
|
+
const archive = await archiveOf("pack");
|
|
213
|
+
cloudHost.commitAnswers = [
|
|
214
|
+
new Response("No healthy federation child is ready", { status: 503 }),
|
|
215
|
+
[503, "base.error.serverUnavailable"],
|
|
216
|
+
];
|
|
217
|
+
await new UpdatePackUpload(api, "project-1", quick).send(archive, fields);
|
|
218
|
+
expect(cloudHost.commits).toEqual(["upload-1", "upload-1", "upload-1"]);
|
|
219
|
+
|
|
220
|
+
cloudHost.reset();
|
|
221
|
+
cloudHost.commitAnswers = [[503, "release.error.signingUnavailable"]];
|
|
222
|
+
await expect(new UpdatePackUpload(api, "project-1", quick).send(archive, fields)).rejects.toThrow(
|
|
223
|
+
"release.error.signingUnavailable",
|
|
224
|
+
);
|
|
225
|
+
expect(cloudHost.commits).toEqual(["upload-1"]);
|
|
193
226
|
});
|
|
194
227
|
|
|
195
228
|
test("tells a server without staged uploads apart from one whose 404 is a refusal", async () => {
|
|
196
229
|
const archive = await archiveOf("pack");
|
|
197
|
-
|
|
230
|
+
cloudHost.requestRefusal = new Response("Not Found", { status: 404 });
|
|
198
231
|
await expect(new UpdatePackUpload(api, "project-1", quick).send(archive, fields)).rejects.toThrow(
|
|
199
232
|
"does not take direct pack uploads yet",
|
|
200
233
|
);
|
|
201
234
|
|
|
202
|
-
|
|
235
|
+
cloudHost.requestRefusal = Response.json(
|
|
236
|
+
{ error: "devProject.error.appNotFound", statusCode: 404 },
|
|
237
|
+
{ status: 404 },
|
|
238
|
+
);
|
|
203
239
|
const refusal = await new UpdatePackUpload(api, "project-1", quick).send(archive, fields).catch((e: Error) => e);
|
|
204
240
|
expect(String(refusal)).toContain("devProject.error.appNotFound");
|
|
205
|
-
expect(
|
|
241
|
+
expect(cloudHost.puts).toEqual([]);
|
|
206
242
|
});
|
|
207
243
|
|
|
208
244
|
test("stops at once on a 409 that is not a commit in progress: the upload is already used", async () => {
|
|
209
245
|
const archive = await archiveOf("pack");
|
|
210
|
-
|
|
246
|
+
cloudHost.commitAnswers = [[409, "release.error.uploadEmbedded"]];
|
|
211
247
|
await expect(new UpdatePackUpload(api, "project-1", quick).send(archive, fields)).rejects.toThrow("uploadEmbedded");
|
|
212
|
-
expect(
|
|
248
|
+
expect(cloudHost.commits).toHaveLength(1);
|
|
213
249
|
});
|
|
214
250
|
|
|
215
251
|
test("stages a binary's pack and registers it by upload id, asking again while the server is still at it", async () => {
|
|
216
252
|
const archive = await archiveOf("pack");
|
|
217
|
-
|
|
253
|
+
cloudHost.commitAnswers = [524, 409];
|
|
218
254
|
const upload = new UpdatePackUpload(api, "project-1", quick);
|
|
219
255
|
const binary = await upload.stage(archive, fields).then((packUploadId) =>
|
|
220
256
|
upload.settle(() =>
|
|
@@ -230,9 +266,9 @@ describe("UpdatePackUpload", () => {
|
|
|
230
266
|
);
|
|
231
267
|
|
|
232
268
|
expect(binary).toEqual({ id: "binary-1" });
|
|
233
|
-
expect(
|
|
234
|
-
expect(
|
|
235
|
-
expect(
|
|
269
|
+
expect(cloudHost.commits).toEqual([]);
|
|
270
|
+
expect(cloudHost.registrations).toHaveLength(3);
|
|
271
|
+
expect(cloudHost.registrations[2]).toEqual({
|
|
236
272
|
appName: "lu",
|
|
237
273
|
target: "default",
|
|
238
274
|
distribution: "playTesting",
|
|
@@ -242,9 +278,28 @@ describe("UpdatePackUpload", () => {
|
|
|
242
278
|
});
|
|
243
279
|
});
|
|
244
280
|
|
|
281
|
+
test("waits as long as storage's Retry-After asks before trying a PUT again", async () => {
|
|
282
|
+
const archive = await archiveOf("pack");
|
|
283
|
+
cloudHost.putStatuses = [new Response(null, { status: 429, headers: { "Retry-After": "1" } })];
|
|
284
|
+
const startedAt = performance.now();
|
|
285
|
+
|
|
286
|
+
await new UpdatePackUpload(api, "project-1", quick).send(archive, fields);
|
|
287
|
+
|
|
288
|
+
expect(performance.now() - startedAt).toBeGreaterThanOrEqual(950);
|
|
289
|
+
expect(cloudHost.puts).toHaveLength(2);
|
|
290
|
+
});
|
|
291
|
+
|
|
292
|
+
test("never asks again sooner than the server's Retry-After, nor past the limit it would wait to", async () => {
|
|
293
|
+
const archive = await archiveOf("pack");
|
|
294
|
+
cloudHost.commitAnswers = [new Response("busy", { status: 503, headers: { "Retry-After": "120" } })];
|
|
295
|
+
|
|
296
|
+
await expect(new UpdatePackUpload(api, "project-1", quick).send(archive, fields)).rejects.toThrow("503");
|
|
297
|
+
expect(cloudHost.commits).toEqual(["upload-1"]);
|
|
298
|
+
});
|
|
299
|
+
|
|
245
300
|
test("refuses an empty archive before asking for a ticket", async () => {
|
|
246
301
|
const archive = await archiveOf("");
|
|
247
302
|
await expect(new UpdatePackUpload(api, "project-1", quick).send(archive, fields)).rejects.toThrow("0 bytes");
|
|
248
|
-
expect(
|
|
303
|
+
expect(cloudHost.requests).toEqual([]);
|
|
249
304
|
});
|
|
250
305
|
});
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { Logger, sleep } from "akanjs/common";
|
|
2
2
|
import type { BunFile } from "bun";
|
|
3
|
-
import { CloudApi,
|
|
3
|
+
import type { CloudApi, CloudRecord, UpdateUploadRequest } from "./cloudApi";
|
|
4
|
+
import { CloudHttpError } from "./cloudHttpError";
|
|
4
5
|
|
|
5
6
|
interface UpdatePackUploadOptions {
|
|
6
7
|
putRetryMs?: number;
|
|
8
|
+
putTimeoutMs?: number;
|
|
7
9
|
commitRetryMs?: number;
|
|
8
10
|
cutOffWindowMs?: number;
|
|
9
11
|
commitLimitMs?: number;
|
|
@@ -15,9 +17,14 @@ interface UpdatePackUploadOptions {
|
|
|
15
17
|
export class UpdatePackUpload {
|
|
16
18
|
static readonly maxBytes = 200 * 1024 * 1024;
|
|
17
19
|
static readonly #transientPut = new Set([0, 408, 429, 500, 502, 503, 504]);
|
|
20
|
+
//? A stalled PUT never fails on its own, so each try ends once the pack would be through even at this crawl.
|
|
21
|
+
static readonly #putFloorBytesPerSecond = 64 * 1024;
|
|
22
|
+
//? Storage's Retry-After is honoured up to this: past it the presigned URL may lapse, and a fresh one is asked for.
|
|
23
|
+
static readonly #maxPutRetryAfterMs = 60_000;
|
|
18
24
|
readonly #api: CloudApi;
|
|
19
25
|
readonly #devProjectId: string;
|
|
20
26
|
readonly #putRetryMs: number;
|
|
27
|
+
readonly #putTimeoutMs: number;
|
|
21
28
|
readonly #commitRetryMs: number;
|
|
22
29
|
readonly #cutOffWindowMs: number;
|
|
23
30
|
readonly #commitLimitMs: number;
|
|
@@ -27,6 +34,7 @@ export class UpdatePackUpload {
|
|
|
27
34
|
devProjectId: string,
|
|
28
35
|
{
|
|
29
36
|
putRetryMs = 1_000,
|
|
37
|
+
putTimeoutMs = 60_000,
|
|
30
38
|
commitRetryMs = 10_000,
|
|
31
39
|
cutOffWindowMs = 15 * 60_000,
|
|
32
40
|
commitLimitMs = 60 * 60_000,
|
|
@@ -35,12 +43,12 @@ export class UpdatePackUpload {
|
|
|
35
43
|
this.#api = api;
|
|
36
44
|
this.#devProjectId = devProjectId;
|
|
37
45
|
this.#putRetryMs = putRetryMs;
|
|
46
|
+
this.#putTimeoutMs = putTimeoutMs;
|
|
38
47
|
this.#commitRetryMs = commitRetryMs;
|
|
39
48
|
this.#cutOffWindowMs = cutOffWindowMs;
|
|
40
49
|
this.#commitLimitMs = commitLimitMs;
|
|
41
50
|
}
|
|
42
51
|
|
|
43
|
-
/** Stages the pack and commits it as a release of `fields.channel`. */
|
|
44
52
|
async send(archive: string, fields: Omit<UpdateUploadRequest, "size" | "sha256">): Promise<CloudRecord> {
|
|
45
53
|
const uploadId = await this.stage(archive, fields);
|
|
46
54
|
return await this.settle(() => this.#api.commitUpdateUpload(this.#devProjectId, uploadId));
|
|
@@ -67,10 +75,9 @@ export class UpdatePackUpload {
|
|
|
67
75
|
async #request(upload: UpdateUploadRequest) {
|
|
68
76
|
return await this.#api.requestUpdateUpload(this.#devProjectId, upload).catch((error: unknown) => {
|
|
69
77
|
//? A server older than staged uploads has no such route: its 404 is a page, where every refusal is an Err key.
|
|
70
|
-
|
|
71
|
-
if (CloudApi.statusOf(error) === 404 && !/"error"\s*:\s*"[\w$-]+\.error\.[\w$-]+"/.test(message))
|
|
78
|
+
if (error instanceof CloudHttpError && error.status === 404 && !error.errKey)
|
|
72
79
|
throw new Error(
|
|
73
|
-
"The project's server does not take direct pack uploads yet: update
|
|
80
|
+
"The project's server does not take direct pack uploads yet: update Akan Cloud and try again, or upload the pack from the project's release screen.",
|
|
74
81
|
);
|
|
75
82
|
throw error;
|
|
76
83
|
});
|
|
@@ -79,15 +86,26 @@ export class UpdatePackUpload {
|
|
|
79
86
|
// The URL signs only its host: an unsigned x-amz-* header (a checksum included) makes storage refuse the PUT.
|
|
80
87
|
// A Bun fetch error names the URL, a bearer credential, so nothing but the status leaves this method.
|
|
81
88
|
async #put(url: string, contentType: string, file: BunFile) {
|
|
89
|
+
const timeoutMs = this.#putTimeoutMs + Math.ceil((file.size / UpdatePackUpload.#putFloorBytesPerSecond) * 1000);
|
|
82
90
|
for (let retry = 0; ; retry++) {
|
|
83
|
-
const status = await fetch(url, {
|
|
91
|
+
const { status, retryAfterMs } = await fetch(url, {
|
|
92
|
+
method: "PUT",
|
|
93
|
+
body: file,
|
|
94
|
+
headers: { "Content-Type": contentType },
|
|
95
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
96
|
+
})
|
|
84
97
|
.then(async (response) => {
|
|
85
98
|
await response.arrayBuffer().catch(() => undefined);
|
|
86
|
-
return
|
|
99
|
+
return {
|
|
100
|
+
status: response.status,
|
|
101
|
+
retryAfterMs: CloudHttpError.retryAfterMs(response.headers.get("retry-after")),
|
|
102
|
+
};
|
|
87
103
|
})
|
|
88
|
-
.catch(() => 0);
|
|
104
|
+
.catch(() => ({ status: 0, retryAfterMs: null }));
|
|
89
105
|
if (retry === 3 || !UpdatePackUpload.#transientPut.has(status)) return status;
|
|
90
|
-
await sleep(
|
|
106
|
+
await sleep(
|
|
107
|
+
Math.max(this.#putRetryMs * (retry + 1), Math.min(retryAfterMs ?? 0, UpdatePackUpload.#maxPutRetryAfterMs)),
|
|
108
|
+
);
|
|
91
109
|
}
|
|
92
110
|
}
|
|
93
111
|
|
|
@@ -109,24 +127,26 @@ export class UpdatePackUpload {
|
|
|
109
127
|
else if (UpdatePackUpload.#cutOff(error)) cutOffSince ??= now;
|
|
110
128
|
else throw error;
|
|
111
129
|
const deadline = cutOffSince === null ? limit : Math.min(limit, cutOffSince + this.#cutOffWindowMs);
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
);
|
|
116
|
-
await sleep(this.#commitRetryMs);
|
|
130
|
+
const waitMs = Math.max(this.#commitRetryMs, (error instanceof CloudHttpError && error.retryAfterMs) || 0);
|
|
131
|
+
if (now + waitMs > deadline) throw error;
|
|
132
|
+
Logger.rawLog(`The project is still taking the pack; asking again in ${Math.round(waitMs / 1000)}s.`);
|
|
133
|
+
await sleep(waitMs);
|
|
117
134
|
}
|
|
118
135
|
}
|
|
119
136
|
}
|
|
120
137
|
|
|
121
138
|
//? Not every 409: an upload already committed or embedded is answered 409 too, and asking again changes nothing.
|
|
122
139
|
static #inProgress(error: unknown) {
|
|
123
|
-
|
|
124
|
-
|
|
140
|
+
return (
|
|
141
|
+
error instanceof CloudHttpError && error.status === 409 && !!error.errKey?.endsWith(".uploadCommitInProgress")
|
|
142
|
+
);
|
|
125
143
|
}
|
|
126
144
|
|
|
127
145
|
static #cutOff(error: unknown) {
|
|
128
|
-
|
|
129
|
-
|
|
146
|
+
if (!(error instanceof CloudHttpError)) return !(error instanceof SyntaxError);
|
|
147
|
+
const { status, errKey } = error;
|
|
148
|
+
//? A gateway with no replica ready answers 503; the server's own 503 names an Err key and is a final refusal.
|
|
149
|
+
if (status === 503) return errKey === null || errKey === "base.error.serverUnavailable";
|
|
130
150
|
return status === 502 || status === 504 || (status >= 520 && status <= 527);
|
|
131
151
|
}
|
|
132
152
|
}
|
|
@@ -49,11 +49,15 @@ export interface ArgMeta<Context = CommandContext> {
|
|
|
49
49
|
idx: number;
|
|
50
50
|
type: ArgType;
|
|
51
51
|
}
|
|
52
|
+
export interface InternalArgOption {
|
|
53
|
+
nullable?: boolean;
|
|
54
|
+
onlyApp?: boolean;
|
|
55
|
+
}
|
|
52
56
|
export interface InternalArgMeta {
|
|
53
57
|
key: string;
|
|
54
58
|
idx: number;
|
|
55
59
|
type: InternalArgType;
|
|
56
|
-
option?:
|
|
60
|
+
option?: InternalArgOption;
|
|
57
61
|
}
|
|
58
62
|
|
|
59
63
|
export const getArgMetas = (
|
|
@@ -69,9 +73,11 @@ export const getArgMetas = (
|
|
|
69
73
|
export interface InternalArgToken<T = unknown, Type extends InternalArgType = InternalArgType> {
|
|
70
74
|
type: Type;
|
|
71
75
|
_value: T;
|
|
76
|
+
option?: InternalArgOption;
|
|
72
77
|
}
|
|
73
78
|
|
|
74
|
-
const createInternalArgToken = <T, Type extends InternalArgType>(type: Type
|
|
79
|
+
const createInternalArgToken = <T, Type extends InternalArgType>(type: Type, option?: InternalArgOption) =>
|
|
80
|
+
(option ? { type, option } : { type }) as InternalArgToken<T, Type>;
|
|
75
81
|
|
|
76
82
|
export const normalizePrimitiveArgType = (type: PrimitiveArgType): NormalizedPrimitiveArgType => {
|
|
77
83
|
if (type === String) return "string";
|
|
@@ -93,6 +99,9 @@ export type Lib = LibExecutor;
|
|
|
93
99
|
export const Sys = createInternalArgToken<SysExecutor, "Sys">("Sys");
|
|
94
100
|
export type Sys = SysExecutor;
|
|
95
101
|
|
|
102
|
+
/** Sys, except that a one-app workspace given no name means its app, as with App. */
|
|
103
|
+
export const SysOrOnlyApp = createInternalArgToken<SysExecutor, "Sys">("Sys", { onlyApp: true });
|
|
104
|
+
|
|
96
105
|
export const Exec = createInternalArgToken<Executor, "Exec">("Exec");
|
|
97
106
|
export type Exec = Executor;
|
|
98
107
|
|