@samebase/alchemy-kit 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/NOTICE +2 -0
- package/README.md +101 -2
- package/lib/github-state.d.ts +36 -0
- package/lib/github-state.d.ts.map +1 -0
- package/lib/github-state.js +485 -0
- package/lib/github-state.js.map +1 -0
- package/package.json +62 -3
- package/src/github-state/GitHubState.ts +546 -0
- package/src/github-state/git.ts +66 -0
- package/src/github-state/sealing.ts +145 -0
|
@@ -0,0 +1,546 @@
|
|
|
1
|
+
import { execFileSync } from "node:child_process";
|
|
2
|
+
import { State, StateStoreError } from "alchemy/State";
|
|
3
|
+
import {
|
|
4
|
+
type BucketObjects,
|
|
5
|
+
makeBucketState,
|
|
6
|
+
type WriteCondition,
|
|
7
|
+
} from "alchemy/State/BucketState";
|
|
8
|
+
import * as Data from "effect/Data";
|
|
9
|
+
import * as Deferred from "effect/Deferred";
|
|
10
|
+
import * as Effect from "effect/Effect";
|
|
11
|
+
import * as HttpClient from "effect/http/HttpClient";
|
|
12
|
+
import * as HttpClientRequest from "effect/http/HttpClientRequest";
|
|
13
|
+
import * as Layer from "effect/Layer";
|
|
14
|
+
import * as Queue from "effect/Queue";
|
|
15
|
+
import * as Redacted from "effect/Redacted";
|
|
16
|
+
import * as Schedule from "effect/Schedule";
|
|
17
|
+
import * as Schema from "effect/Schema";
|
|
18
|
+
import * as Scope from "effect/Scope";
|
|
19
|
+
import { blobSha, commitMessage, encodePath, LEASE_FILE } from "./git.ts";
|
|
20
|
+
import { KEY_VARIABLE, openRecord, parseKey, sealRecord } from "./sealing.ts";
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* An Alchemy state store on a branch of a GitHub repository.
|
|
24
|
+
*
|
|
25
|
+
* Built on Alchemy's BucketState, like the S3 and Cloud Storage stores. The "bucket" is one Git
|
|
26
|
+
* branch: a key is a path, an object's version is its blob sha. Writes queue up and one worker
|
|
27
|
+
* turns everything queued into one commit through the Git Data API; the ref update with
|
|
28
|
+
* `force: false` is the compare-and-swap, and a lost race starts the batch again on the new head.
|
|
29
|
+
*
|
|
30
|
+
* The lease (`__lease__.json`) lives on a second branch, `<branch>-lock`, so the state branch holds
|
|
31
|
+
* only state.
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
export interface GitHubStateOptions {
|
|
35
|
+
/** `owner/name`. @default GITHUB_REPOSITORY, else the `origin` remote */
|
|
36
|
+
readonly repository?: string;
|
|
37
|
+
/** @default "alchemy-state-store" */
|
|
38
|
+
readonly branch?: string;
|
|
39
|
+
/** Base64 of 32 bytes that seals each secret. @default ALCHEMY_GITHUB_STATE_KEY */
|
|
40
|
+
readonly key?: Redacted.Redacted<string>;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const API = "https://api.github.com";
|
|
44
|
+
const MAX_ATTEMPTS = 20;
|
|
45
|
+
|
|
46
|
+
const STATE_README = `# Alchemy state
|
|
47
|
+
|
|
48
|
+
This branch holds the Alchemy state of this repository. Alchemy deploys write it; do not edit it
|
|
49
|
+
by hand. Each resource is one file, \`{stack}/{stage}/{resource}.json\`.
|
|
50
|
+
|
|
51
|
+
Each secret value is \`{"__redacted__": {"__sealed__": "v1.<iv>.<ciphertext>"}}\`, sealed with
|
|
52
|
+
AES-256-GCM under the key in ${KEY_VARIABLE}. Every other value is plain JSON.
|
|
53
|
+
|
|
54
|
+
The deploy lock of each stage is on the branch of the same name with \`-lock\` at the end.
|
|
55
|
+
`;
|
|
56
|
+
|
|
57
|
+
const LOCK_README = `# Alchemy deploy locks
|
|
58
|
+
|
|
59
|
+
This branch holds the lock (\`{stack}/{stage}/__lease__.json\`) of each stage while a deploy runs.
|
|
60
|
+
The state itself is on the branch without \`-lock\`.
|
|
61
|
+
`;
|
|
62
|
+
|
|
63
|
+
// GitHub responses, validated.
|
|
64
|
+
const RefResponse = Schema.Struct({ object: Schema.Struct({ sha: Schema.String }) });
|
|
65
|
+
const TreeResponse = Schema.Struct({
|
|
66
|
+
sha: Schema.String,
|
|
67
|
+
truncated: Schema.Boolean,
|
|
68
|
+
tree: Schema.Array(
|
|
69
|
+
Schema.Struct({ path: Schema.String, type: Schema.String, sha: Schema.String }),
|
|
70
|
+
),
|
|
71
|
+
});
|
|
72
|
+
const ShaResponse = Schema.Struct({ sha: Schema.String });
|
|
73
|
+
const BlobResponse = Schema.Struct({ encoding: Schema.String, content: Schema.String });
|
|
74
|
+
|
|
75
|
+
class Retryable extends Data.TaggedError("Retryable")<{ readonly message: string }> {}
|
|
76
|
+
|
|
77
|
+
interface Response {
|
|
78
|
+
readonly status: number;
|
|
79
|
+
readonly json: unknown;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** A branch head: its commit, its tree, and every file's blob sha. */
|
|
83
|
+
interface Head {
|
|
84
|
+
readonly commit: string;
|
|
85
|
+
readonly tree: string;
|
|
86
|
+
readonly files: ReadonlyMap<string, string>;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** A queued write (`body`) or delete (`body` undefined). */
|
|
90
|
+
interface Pending {
|
|
91
|
+
readonly key: string;
|
|
92
|
+
readonly body: string | undefined;
|
|
93
|
+
readonly condition: WriteCondition | undefined;
|
|
94
|
+
readonly done: Deferred.Deferred<string | undefined, StateStoreError>;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* The batch applied in order to the files of `base`: the files after it, the new blob sha of each
|
|
99
|
+
* write (`undefined` for a delete or a condition that did not hold), and the paths whose content
|
|
100
|
+
* changes.
|
|
101
|
+
*/
|
|
102
|
+
const applyBatch = (base: Head | undefined, batch: ReadonlyArray<Pending>) => {
|
|
103
|
+
const files = new Map(base?.files ?? []);
|
|
104
|
+
const results: Array<string | undefined> = [];
|
|
105
|
+
const changed = new Map<string, { body: string | undefined; lock: string }>();
|
|
106
|
+
let conditionFailed = false;
|
|
107
|
+
for (const pending of batch) {
|
|
108
|
+
const current = files.get(pending.key);
|
|
109
|
+
const condition = pending.condition;
|
|
110
|
+
if (pending.body === undefined) {
|
|
111
|
+
// A delete of a missing file, or of one at another version, does nothing.
|
|
112
|
+
if (
|
|
113
|
+
current !== undefined &&
|
|
114
|
+
(condition === undefined || ("ifVersion" in condition && condition.ifVersion === current))
|
|
115
|
+
) {
|
|
116
|
+
files.delete(pending.key);
|
|
117
|
+
changed.set(pending.key, { body: undefined, lock: "release lock" });
|
|
118
|
+
} else if (condition !== undefined) conditionFailed = true;
|
|
119
|
+
results.push(undefined);
|
|
120
|
+
continue;
|
|
121
|
+
}
|
|
122
|
+
if (condition !== undefined) {
|
|
123
|
+
const holds =
|
|
124
|
+
"ifAbsent" in condition ? current === undefined : current === condition.ifVersion;
|
|
125
|
+
if (!holds) {
|
|
126
|
+
conditionFailed = true;
|
|
127
|
+
results.push(undefined);
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
const sha = blobSha(pending.body);
|
|
132
|
+
files.set(pending.key, sha);
|
|
133
|
+
changed.set(pending.key, {
|
|
134
|
+
body: pending.body,
|
|
135
|
+
lock: condition === undefined ? "" : "ifAbsent" in condition ? "take lock" : "renew lock",
|
|
136
|
+
});
|
|
137
|
+
results.push(sha);
|
|
138
|
+
}
|
|
139
|
+
const effective = Array.from(changed).filter(([path, change]) =>
|
|
140
|
+
change.body === undefined
|
|
141
|
+
? base?.files.has(path) === true
|
|
142
|
+
: base?.files.get(path) !== blobSha(change.body),
|
|
143
|
+
);
|
|
144
|
+
return { files, results, effective, conditionFailed };
|
|
145
|
+
};
|
|
146
|
+
|
|
147
|
+
const fail = (message: string) => new StateStoreError({ message });
|
|
148
|
+
|
|
149
|
+
const errorMessage = (json: unknown) =>
|
|
150
|
+
typeof json === "object" && json !== null && "message" in json && typeof json.message === "string"
|
|
151
|
+
? json.message
|
|
152
|
+
: "";
|
|
153
|
+
|
|
154
|
+
/** `owner/name` from the option, GITHUB_REPOSITORY, or the `origin` remote. */
|
|
155
|
+
const resolveRepository = (explicit: string | undefined) =>
|
|
156
|
+
Effect.try({
|
|
157
|
+
try: () => {
|
|
158
|
+
const fromEnv = explicit ?? process.env["GITHUB_REPOSITORY"];
|
|
159
|
+
if (fromEnv) return fromEnv;
|
|
160
|
+
const url = execFileSync("git", ["remote", "get-url", "origin"], { encoding: "utf8" }).trim();
|
|
161
|
+
const match = /github\.com[:/]([^/]+\/[^/]+?)(?:\.git)?$/.exec(url);
|
|
162
|
+
if (!match?.[1]) throw new Error(`origin is not a GitHub remote: ${url}`);
|
|
163
|
+
return match[1];
|
|
164
|
+
},
|
|
165
|
+
catch: (cause) =>
|
|
166
|
+
fail(
|
|
167
|
+
`GitHub state store: no repository. Set GITHUB_REPOSITORY or pass { repository }. ${cause instanceof Error ? cause.message : ""}`,
|
|
168
|
+
),
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
/** The BucketObjects of one branch (plus its `-lock` branch), bound to the ambient scope. */
|
|
172
|
+
export const makeGitHubObjects = (options: GitHubStateOptions = {}) =>
|
|
173
|
+
Effect.gen(function* () {
|
|
174
|
+
const client = yield* HttpClient.HttpClient;
|
|
175
|
+
const token = process.env["GITHUB_TOKEN"] || process.env["GH_TOKEN"];
|
|
176
|
+
if (!token) return yield* fail("GitHub state store: set GITHUB_TOKEN or GH_TOKEN");
|
|
177
|
+
const repository = yield* resolveRepository(options.repository);
|
|
178
|
+
const stateBranch = options.branch ?? "alchemy-state-store";
|
|
179
|
+
const repoPath = `/repos/${encodePath(repository)}`;
|
|
180
|
+
|
|
181
|
+
const call = (method: "GET" | "POST" | "PATCH", path: string, body?: unknown) =>
|
|
182
|
+
Effect.gen(function* () {
|
|
183
|
+
const base = HttpClientRequest.make(method)(`${API}${repoPath}${path}`, {
|
|
184
|
+
headers: {
|
|
185
|
+
authorization: `Bearer ${token}`,
|
|
186
|
+
accept: "application/vnd.github+json",
|
|
187
|
+
"x-github-api-version": "2022-11-28",
|
|
188
|
+
"user-agent": "samebase-alchemy-kit",
|
|
189
|
+
},
|
|
190
|
+
});
|
|
191
|
+
const request = body === undefined ? base : HttpClientRequest.bodyJsonUnsafe(base, body);
|
|
192
|
+
// A call with no answer would hold its callers for good: they wait uninterruptibly.
|
|
193
|
+
const [response, text] = yield* client.execute(request).pipe(
|
|
194
|
+
Effect.flatMap((answer) => Effect.map(answer.text, (body) => [answer, body] as const)),
|
|
195
|
+
Effect.mapError((cause) => new Retryable({ message: cause.message })),
|
|
196
|
+
Effect.timeoutOrElse({
|
|
197
|
+
duration: "60 seconds",
|
|
198
|
+
orElse: () =>
|
|
199
|
+
Effect.fail(new Retryable({ message: `${method} ${path}: no answer in 60 seconds` })),
|
|
200
|
+
}),
|
|
201
|
+
);
|
|
202
|
+
const json: unknown = yield* Effect.try({
|
|
203
|
+
try: (): unknown => (text === "" ? undefined : JSON.parse(text)),
|
|
204
|
+
catch: () =>
|
|
205
|
+
new Retryable({ message: `${method} ${path}: not JSON (${response.status})` }),
|
|
206
|
+
});
|
|
207
|
+
const status = response.status;
|
|
208
|
+
const retryAfter = response.headers["retry-after"];
|
|
209
|
+
const limited =
|
|
210
|
+
(status === 403 || status === 429) &&
|
|
211
|
+
(retryAfter !== undefined ||
|
|
212
|
+
response.headers["x-ratelimit-remaining"] === "0" ||
|
|
213
|
+
/rate limit/i.test(errorMessage(json)));
|
|
214
|
+
if (limited) {
|
|
215
|
+
// `retry-after` can also be a date; only a count of at most 120 seconds is waited for.
|
|
216
|
+
const seconds = Number(retryAfter ?? "60");
|
|
217
|
+
if (!(seconds >= 0 && seconds <= 120)) {
|
|
218
|
+
return yield* fail(
|
|
219
|
+
`GitHub rate limit on ${method} ${path}: retry after ${retryAfter} (${errorMessage(json)})`,
|
|
220
|
+
);
|
|
221
|
+
}
|
|
222
|
+
yield* Effect.sleep(`${seconds} seconds`);
|
|
223
|
+
return yield* new Retryable({ message: `${method} ${path}: rate limited (${status})` });
|
|
224
|
+
}
|
|
225
|
+
if (status >= 500) {
|
|
226
|
+
return yield* new Retryable({
|
|
227
|
+
message: `${method} ${path}: ${status} ${errorMessage(json)}`,
|
|
228
|
+
});
|
|
229
|
+
}
|
|
230
|
+
const result: Response = { status, json };
|
|
231
|
+
return result;
|
|
232
|
+
}).pipe(
|
|
233
|
+
Effect.retry({
|
|
234
|
+
while: (error) => error._tag === "Retryable",
|
|
235
|
+
schedule: Schedule.exponential("500 millis"),
|
|
236
|
+
times: 5,
|
|
237
|
+
}),
|
|
238
|
+
Effect.mapError((error) =>
|
|
239
|
+
error._tag === "Retryable" ? fail(`GitHub state store: ${error.message}`) : error,
|
|
240
|
+
),
|
|
241
|
+
);
|
|
242
|
+
|
|
243
|
+
const decode =
|
|
244
|
+
<S extends Schema.Codec<unknown, unknown>>(schema: S, what: string) =>
|
|
245
|
+
(json: unknown) =>
|
|
246
|
+
Schema.decodeUnknownEffect(schema)(json).pipe(
|
|
247
|
+
Effect.mapError((cause) =>
|
|
248
|
+
fail(`GitHub state store: unexpected ${what}: ${cause.message}`),
|
|
249
|
+
),
|
|
250
|
+
);
|
|
251
|
+
|
|
252
|
+
const unexpected = (what: string, response: Response) =>
|
|
253
|
+
fail(
|
|
254
|
+
`GitHub state store: ${what} answered ${response.status} ${errorMessage(response.json)}`,
|
|
255
|
+
);
|
|
256
|
+
|
|
257
|
+
/** The reads of one branch, and the worker that turns every queued write into one commit. */
|
|
258
|
+
const makeBranch = (branch: string, readme: string) =>
|
|
259
|
+
Effect.gen(function* () {
|
|
260
|
+
const queue = yield* Queue.unbounded<Pending>();
|
|
261
|
+
// The newest head this process knows, each head it saw since its last commit, the heads
|
|
262
|
+
// that are behind one of its commits, and the text of each blob it wrote or read.
|
|
263
|
+
let known: Head | undefined;
|
|
264
|
+
const seen = new Set<string>();
|
|
265
|
+
const replaced = new Set<string>();
|
|
266
|
+
const blobs = new Map<string, string>();
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* The branch as GitHub has it now; `undefined` when it does not exist. GitHub can answer
|
|
270
|
+
* with the head from before a write of this process (seen live: 1 read of 40 right after a
|
|
271
|
+
* write). The branch only moves forward, so a head that is behind a commit of this process
|
|
272
|
+
* is an old answer, and the question is asked again.
|
|
273
|
+
*/
|
|
274
|
+
const current = Effect.gen(function* () {
|
|
275
|
+
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
|
|
276
|
+
const ref = yield* call("GET", `/git/ref/heads/${encodePath(branch)}`);
|
|
277
|
+
if (ref.status !== 200 && ref.status !== 404) {
|
|
278
|
+
return yield* unexpected(`GET ref ${branch}`, ref);
|
|
279
|
+
}
|
|
280
|
+
const commit =
|
|
281
|
+
ref.status === 404
|
|
282
|
+
? undefined
|
|
283
|
+
: (yield* decode(RefResponse, "ref")(ref.json)).object.sha;
|
|
284
|
+
if (commit === undefined ? known !== undefined : replaced.has(commit)) {
|
|
285
|
+
yield* Effect.sleep("250 millis");
|
|
286
|
+
continue;
|
|
287
|
+
}
|
|
288
|
+
if (commit === undefined) return undefined;
|
|
289
|
+
seen.add(commit);
|
|
290
|
+
if (known?.commit === commit) return known;
|
|
291
|
+
const treeResponse = yield* call("GET", `/git/trees/${commit}?recursive=1`);
|
|
292
|
+
if (treeResponse.status !== 200) {
|
|
293
|
+
return yield* unexpected(`GET tree ${branch}`, treeResponse);
|
|
294
|
+
}
|
|
295
|
+
const tree = yield* decode(TreeResponse, "tree")(treeResponse.json);
|
|
296
|
+
if (tree.truncated) {
|
|
297
|
+
return yield* fail(
|
|
298
|
+
`GitHub state store: the tree of ${branch} is too large (truncated)`,
|
|
299
|
+
);
|
|
300
|
+
}
|
|
301
|
+
const files = new Map<string, string>();
|
|
302
|
+
for (const entry of tree.tree) {
|
|
303
|
+
if (entry.type === "blob") files.set(entry.path, entry.sha);
|
|
304
|
+
}
|
|
305
|
+
const head: Head = { commit, tree: tree.sha, files };
|
|
306
|
+
// A commit of this process can land while the tree loads.
|
|
307
|
+
if (!replaced.has(commit)) known = head;
|
|
308
|
+
return head;
|
|
309
|
+
}
|
|
310
|
+
return yield* fail(
|
|
311
|
+
`GitHub state store: ${branch} kept answering with a head from before this run's last write`,
|
|
312
|
+
);
|
|
313
|
+
});
|
|
314
|
+
|
|
315
|
+
const read = (path: string) =>
|
|
316
|
+
Effect.gen(function* () {
|
|
317
|
+
const version = (yield* current)?.files.get(path);
|
|
318
|
+
if (version === undefined) return undefined;
|
|
319
|
+
const held = blobs.get(version);
|
|
320
|
+
if (held !== undefined) return { text: held, version };
|
|
321
|
+
const response = yield* call("GET", `/git/blobs/${version}`);
|
|
322
|
+
if (response.status !== 200) return yield* unexpected(`GET blob ${path}`, response);
|
|
323
|
+
const blob = yield* decode(BlobResponse, "blob")(response.json);
|
|
324
|
+
if (blob.encoding !== "base64") {
|
|
325
|
+
return yield* fail(`GitHub state store: ${path} has encoding ${blob.encoding}`);
|
|
326
|
+
}
|
|
327
|
+
const text = Buffer.from(blob.content, "base64").toString("utf8");
|
|
328
|
+
blobs.set(version, text);
|
|
329
|
+
return { text, version };
|
|
330
|
+
});
|
|
331
|
+
|
|
332
|
+
const commitBatch = (batch: ReadonlyArray<Pending>) =>
|
|
333
|
+
Effect.gen(function* () {
|
|
334
|
+
// The first attempt builds on the known head without a question to GitHub: the ref
|
|
335
|
+
// update fails if another writer moved the branch since.
|
|
336
|
+
let trusted = known !== undefined;
|
|
337
|
+
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
|
|
338
|
+
const base = trusted ? known : yield* current;
|
|
339
|
+
const { files, results, effective, conditionFailed } = applyBatch(base, batch);
|
|
340
|
+
// A failed condition or "nothing to write" against the known head may be out of
|
|
341
|
+
// date: check against the branch as it is now.
|
|
342
|
+
if (trusted && (conditionFailed || effective.length === 0)) {
|
|
343
|
+
trusted = false;
|
|
344
|
+
continue;
|
|
345
|
+
}
|
|
346
|
+
if (effective.length === 0) return results;
|
|
347
|
+
const isLock = branch !== stateBranch;
|
|
348
|
+
const message = commitMessage(
|
|
349
|
+
effective.map(([key, change]) => ({
|
|
350
|
+
key,
|
|
351
|
+
deleted: change.body === undefined,
|
|
352
|
+
lock: isLock ? change.lock : "",
|
|
353
|
+
})),
|
|
354
|
+
);
|
|
355
|
+
const entries = effective.map(([path, change]) =>
|
|
356
|
+
change.body === undefined
|
|
357
|
+
? { path, mode: "100644", type: "blob", sha: null }
|
|
358
|
+
: { path, mode: "100644", type: "blob", content: change.body },
|
|
359
|
+
);
|
|
360
|
+
const treeResponse = yield* call(
|
|
361
|
+
"POST",
|
|
362
|
+
"/git/trees",
|
|
363
|
+
base === undefined
|
|
364
|
+
? {
|
|
365
|
+
tree: [
|
|
366
|
+
{ path: "README.md", mode: "100644", type: "blob", content: readme },
|
|
367
|
+
...entries.filter((entry) => !("sha" in entry)),
|
|
368
|
+
],
|
|
369
|
+
}
|
|
370
|
+
: { base_tree: base.tree, tree: entries },
|
|
371
|
+
);
|
|
372
|
+
if (treeResponse.status !== 201) {
|
|
373
|
+
return yield* unexpected(`POST tree ${branch}`, treeResponse);
|
|
374
|
+
}
|
|
375
|
+
const tree = (yield* decode(ShaResponse, "tree")(treeResponse.json)).sha;
|
|
376
|
+
const commitResponse = yield* call("POST", "/git/commits", {
|
|
377
|
+
message,
|
|
378
|
+
tree,
|
|
379
|
+
parents: base === undefined ? [] : [base.commit],
|
|
380
|
+
});
|
|
381
|
+
if (commitResponse.status !== 201) {
|
|
382
|
+
return yield* unexpected(`POST commit ${branch}`, commitResponse);
|
|
383
|
+
}
|
|
384
|
+
const commit = (yield* decode(ShaResponse, "commit")(commitResponse.json)).sha;
|
|
385
|
+
const refResponse =
|
|
386
|
+
base === undefined
|
|
387
|
+
? yield* call("POST", "/git/refs", { ref: `refs/heads/${branch}`, sha: commit })
|
|
388
|
+
: yield* call("PATCH", `/git/refs/heads/${encodePath(branch)}`, {
|
|
389
|
+
sha: commit,
|
|
390
|
+
force: false,
|
|
391
|
+
});
|
|
392
|
+
// 422: another writer moved (or created) the branch. Start again on its head.
|
|
393
|
+
if (refResponse.status === 422) {
|
|
394
|
+
trusted = false;
|
|
395
|
+
continue;
|
|
396
|
+
}
|
|
397
|
+
if (refResponse.status !== 200 && refResponse.status !== 201) {
|
|
398
|
+
return yield* unexpected(`update ref ${branch}`, refResponse);
|
|
399
|
+
}
|
|
400
|
+
// Each head seen so far was the head at some time before this commit landed, so it
|
|
401
|
+
// is behind this commit: the base, and also an earlier head of this process when
|
|
402
|
+
// another writer's commit came between.
|
|
403
|
+
for (const head of seen) replaced.add(head);
|
|
404
|
+
seen.clear();
|
|
405
|
+
seen.add(commit);
|
|
406
|
+
known = { commit, tree, files };
|
|
407
|
+
for (const [, change] of effective) {
|
|
408
|
+
if (change.body !== undefined) blobs.set(blobSha(change.body), change.body);
|
|
409
|
+
}
|
|
410
|
+
return results;
|
|
411
|
+
}
|
|
412
|
+
return yield* fail(
|
|
413
|
+
`GitHub state store: ${branch} kept moving; gave up after ${MAX_ATTEMPTS} attempts`,
|
|
414
|
+
);
|
|
415
|
+
});
|
|
416
|
+
|
|
417
|
+
yield* Queue.takeAll(queue).pipe(
|
|
418
|
+
Effect.flatMap((batch) =>
|
|
419
|
+
commitBatch(batch).pipe(
|
|
420
|
+
Effect.flatMap((results) =>
|
|
421
|
+
Effect.forEach(batch, (pending, i) => Deferred.succeed(pending.done, results[i]), {
|
|
422
|
+
discard: true,
|
|
423
|
+
}),
|
|
424
|
+
),
|
|
425
|
+
// Also a defect goes to the callers: a worker that stops would leave them waiting.
|
|
426
|
+
Effect.catchCause((cause) =>
|
|
427
|
+
Effect.forEach(batch, (pending) => Deferred.failCause(pending.done, cause), {
|
|
428
|
+
discard: true,
|
|
429
|
+
}),
|
|
430
|
+
),
|
|
431
|
+
),
|
|
432
|
+
),
|
|
433
|
+
Effect.forever,
|
|
434
|
+
Effect.forkScoped,
|
|
435
|
+
);
|
|
436
|
+
|
|
437
|
+
return {
|
|
438
|
+
read,
|
|
439
|
+
/** Every file path on the branch, with one call when the head is the known one. */
|
|
440
|
+
paths: current.pipe(Effect.map((head) => Array.from(head?.files.keys() ?? []))),
|
|
441
|
+
/** Queue a write or delete; the deferred resolves once its commit is on the branch. */
|
|
442
|
+
enqueue: (key: string, body: string | undefined, condition: WriteCondition | undefined) =>
|
|
443
|
+
Effect.gen(function* () {
|
|
444
|
+
const done = yield* Deferred.make<string | undefined, StateStoreError>();
|
|
445
|
+
yield* Queue.offer(queue, { key, body, condition, done });
|
|
446
|
+
return done;
|
|
447
|
+
}),
|
|
448
|
+
};
|
|
449
|
+
});
|
|
450
|
+
|
|
451
|
+
const store = yield* makeBranch(stateBranch, STATE_README);
|
|
452
|
+
const lock = yield* makeBranch(`${stateBranch}-lock`, LOCK_README);
|
|
453
|
+
const branchOf = (key: string) =>
|
|
454
|
+
key === LEASE_FILE || key.endsWith(`/${LEASE_FILE}`) ? lock : store;
|
|
455
|
+
const enqueue = (
|
|
456
|
+
key: string,
|
|
457
|
+
body: string | undefined,
|
|
458
|
+
condition: WriteCondition | undefined,
|
|
459
|
+
) => branchOf(key).enqueue(key, body, condition);
|
|
460
|
+
// A queued write lands even when its caller is interrupted, so the caller waits for it.
|
|
461
|
+
// BucketState interrupts a lease renewal before it releases the lease; with an interruptible
|
|
462
|
+
// wait the renewal could land after the release read the lease, and the release then deleted
|
|
463
|
+
// nothing.
|
|
464
|
+
const submit = (key: string, body: string | undefined, condition: WriteCondition | undefined) =>
|
|
465
|
+
enqueue(key, body, condition).pipe(Effect.flatMap(Deferred.await), Effect.uninterruptible);
|
|
466
|
+
|
|
467
|
+
const objects: BucketObjects = {
|
|
468
|
+
listKeys: (prefix) =>
|
|
469
|
+
store.paths.pipe(Effect.map((paths) => paths.filter((path) => path.startsWith(prefix)))),
|
|
470
|
+
listChildren: (prefix) =>
|
|
471
|
+
store.paths.pipe(
|
|
472
|
+
Effect.map((paths) => {
|
|
473
|
+
const children = new Set<string>();
|
|
474
|
+
for (const path of paths) {
|
|
475
|
+
if (!path.startsWith(prefix)) continue;
|
|
476
|
+
const rest = path.slice(prefix.length);
|
|
477
|
+
const slash = rest.indexOf("/");
|
|
478
|
+
if (slash > 0) children.add(rest.slice(0, slash));
|
|
479
|
+
}
|
|
480
|
+
return Array.from(children);
|
|
481
|
+
}),
|
|
482
|
+
),
|
|
483
|
+
read: (key) => branchOf(key).read(key),
|
|
484
|
+
write: (key, body, condition) => submit(key, body, condition),
|
|
485
|
+
delete: (key, ifVersion) =>
|
|
486
|
+
submit(key, undefined, ifVersion === undefined ? undefined : { ifVersion }).pipe(
|
|
487
|
+
Effect.asVoid,
|
|
488
|
+
),
|
|
489
|
+
// Queue every delete before waiting, so they land in one commit.
|
|
490
|
+
deleteMany: (keys) =>
|
|
491
|
+
Effect.forEach(keys, (key) => enqueue(key, undefined, undefined)).pipe(
|
|
492
|
+
Effect.flatMap((pending) => Effect.forEach(pending, Deferred.await, { discard: true })),
|
|
493
|
+
Effect.uninterruptible,
|
|
494
|
+
),
|
|
495
|
+
};
|
|
496
|
+
|
|
497
|
+
return objects;
|
|
498
|
+
});
|
|
499
|
+
|
|
500
|
+
/**
|
|
501
|
+
* The State layer. It reads the key and builds the store on the first state operation, bound to
|
|
502
|
+
* this layer's scope, so a missing key fails the run before any read or write.
|
|
503
|
+
*/
|
|
504
|
+
export const state = (
|
|
505
|
+
options: GitHubStateOptions = {},
|
|
506
|
+
): Layer.Layer<State, never, HttpClient.HttpClient> =>
|
|
507
|
+
Layer.effect(
|
|
508
|
+
State,
|
|
509
|
+
Effect.gen(function* () {
|
|
510
|
+
const scope = yield* Effect.scope;
|
|
511
|
+
const context = yield* Effect.context<HttpClient.HttpClient>();
|
|
512
|
+
const make = Effect.gen(function* () {
|
|
513
|
+
const key = yield* parseKey(
|
|
514
|
+
options.key === undefined ? process.env[KEY_VARIABLE] : Redacted.value(options.key),
|
|
515
|
+
);
|
|
516
|
+
const objects = yield* makeGitHubObjects(options);
|
|
517
|
+
const sealed: BucketObjects = {
|
|
518
|
+
...objects,
|
|
519
|
+
read: (objectKey) =>
|
|
520
|
+
objects
|
|
521
|
+
.read(objectKey)
|
|
522
|
+
.pipe(
|
|
523
|
+
Effect.flatMap((object) =>
|
|
524
|
+
object === undefined
|
|
525
|
+
? Effect.undefined
|
|
526
|
+
: openRecord(key, objectKey, object.text).pipe(
|
|
527
|
+
Effect.map((text) => ({ text, version: object.version })),
|
|
528
|
+
),
|
|
529
|
+
),
|
|
530
|
+
),
|
|
531
|
+
write: (objectKey, body, condition) =>
|
|
532
|
+
sealRecord(key, objectKey, body).pipe(
|
|
533
|
+
Effect.flatMap((text) => objects.write(objectKey, text, condition)),
|
|
534
|
+
),
|
|
535
|
+
};
|
|
536
|
+
return yield* makeBucketState({
|
|
537
|
+
id: "github",
|
|
538
|
+
label: "GitHub",
|
|
539
|
+
objects: Effect.succeed(sealed),
|
|
540
|
+
});
|
|
541
|
+
});
|
|
542
|
+
return yield* Effect.cached(
|
|
543
|
+
Scope.provide(make, scope).pipe(Effect.orDie, Effect.provideContext(context)),
|
|
544
|
+
);
|
|
545
|
+
}),
|
|
546
|
+
);
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
// The paths, blob shas, and commit messages of the state branches.
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
import { decodeFqn } from "alchemy/FQN";
|
|
4
|
+
|
|
5
|
+
/** The lease of a stage, which Alchemy's BucketState rewrites every 20 seconds during a run. */
|
|
6
|
+
export const LEASE_FILE = "__lease__.json";
|
|
7
|
+
|
|
8
|
+
/** A path with each segment URL-encoded and its slashes kept, for a GitHub API URL. */
|
|
9
|
+
export const encodePath = (path: string) => path.split("/").map(encodeURIComponent).join("/");
|
|
10
|
+
|
|
11
|
+
/** The git blob sha of `text`: what GitHub reports as the file's sha. */
|
|
12
|
+
export const blobSha = (text: string) => {
|
|
13
|
+
const bytes = Buffer.from(text, "utf8");
|
|
14
|
+
return createHash("sha1").update(`blob ${bytes.length}\0`).update(bytes).digest("hex");
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
/** What the commit message calls a key: `{stack} {stage}` and the resource. */
|
|
18
|
+
const describeKey = (key: string) => {
|
|
19
|
+
const parts = key.split("/");
|
|
20
|
+
const file = parts[parts.length - 1] ?? key;
|
|
21
|
+
if (parts.length < 3) return { group: "", name: file.replace(/\.json$/, "") };
|
|
22
|
+
const name =
|
|
23
|
+
file === "__stack_output__.json"
|
|
24
|
+
? "stack output"
|
|
25
|
+
: file === LEASE_FILE
|
|
26
|
+
? "lock"
|
|
27
|
+
: decodeFqn(file.replace(/\.json$/, ""));
|
|
28
|
+
return { group: `${parts.slice(0, -2).join("/")} ${parts[parts.length - 2]}`, name };
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
const listNames = (verb: string, names: ReadonlyArray<string>) =>
|
|
32
|
+
names.length === 0
|
|
33
|
+
? []
|
|
34
|
+
: names.length <= 5
|
|
35
|
+
? [`${verb} ${names.join(", ")}`]
|
|
36
|
+
: [`${verb} ${names.length} resources`];
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* One line per stage, such as `app prod: set Worker, Bucket; delete Old`. A lock change carries
|
|
40
|
+
* its `lock` text (`take lock`, `renew lock`, `release lock`) in place of a resource name.
|
|
41
|
+
*/
|
|
42
|
+
export const commitMessage = (
|
|
43
|
+
changes: ReadonlyArray<{
|
|
44
|
+
readonly key: string;
|
|
45
|
+
readonly deleted: boolean;
|
|
46
|
+
readonly lock: string;
|
|
47
|
+
}>,
|
|
48
|
+
) => {
|
|
49
|
+
const groups = new Map<string, { set: string[]; deleted: string[]; lock: string[] }>();
|
|
50
|
+
for (const change of changes) {
|
|
51
|
+
const { group, name } = describeKey(change.key);
|
|
52
|
+
const entry = groups.get(group) ?? { set: [], deleted: [], lock: [] };
|
|
53
|
+
if (change.lock !== "") entry.lock.push(change.lock);
|
|
54
|
+
else if (change.deleted) entry.deleted.push(name);
|
|
55
|
+
else entry.set.push(name);
|
|
56
|
+
groups.set(group, entry);
|
|
57
|
+
}
|
|
58
|
+
return Array.from(groups, ([group, entry]) => {
|
|
59
|
+
const what = [
|
|
60
|
+
...entry.lock,
|
|
61
|
+
...listNames("set", entry.set),
|
|
62
|
+
...listNames("delete", entry.deleted),
|
|
63
|
+
].join("; ");
|
|
64
|
+
return group === "" ? `store: ${what}` : `${group}: ${what}`;
|
|
65
|
+
}).join("\n");
|
|
66
|
+
};
|