@akanjs/devkit 3.0.0-beta.24 → 3.0.0-beta.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DEV_RUNTIME_KNOBS.md +37 -22
- package/akanApp/akanApp.host.ts +34 -1
- package/akanApp/devBootLatch.test.ts +39 -0
- package/akanApp/devBootLatch.ts +39 -0
- package/akanApp/devHostPolicy.test.ts +25 -0
- package/akanApp/devHostPolicy.ts +27 -2
- package/cloud/globalConfig.ts +3 -17
- package/csrE2e/csrE2eHarness.fixture.ts +66 -11
- package/csrE2e/csrPatchFirst.e2e.test.ts +87 -0
- package/csrE2e/devHmrLatency.e2e.test.ts +391 -0
- package/csrE2e/ssrRegistry.e2e.test.ts +644 -0
- package/fileSys.ts +16 -1
- package/frontendBuild/buildRouteClient.test.ts +61 -0
- package/frontendBuild/bundleDefine.test.ts +12 -0
- package/frontendBuild/bundleDefine.ts +5 -1
- package/frontendBuild/clientEntryDiscovery.test.ts +37 -0
- package/frontendBuild/clientEntryDiscovery.ts +10 -5
- package/frontendBuild/csrArtifactBuilder.ts +3 -6
- package/frontendBuild/csrDevArtifactWriter.test.ts +143 -0
- package/frontendBuild/csrDevArtifactWriter.ts +158 -24
- package/frontendBuild/csrDevBundler.ts +44 -205
- package/frontendBuild/csrDevModuleCompiler.ts +51 -9
- package/frontendBuild/csrDevPatcher.test.ts +340 -0
- package/frontendBuild/csrDevPatcher.ts +374 -0
- package/frontendBuild/csrDevPaths.ts +60 -0
- package/frontendBuild/csrDevResolver.test.ts +165 -0
- package/frontendBuild/csrDevResolver.ts +70 -18
- package/frontendBuild/csrDevTypes.ts +14 -1
- package/frontendBuild/csrEntryFiles.ts +24 -0
- package/frontendBuild/cssCompiler.ts +2 -4
- package/frontendBuild/devRegistryBundler.ts +177 -0
- package/frontendBuild/hmrWatcher.test.ts +51 -0
- package/frontendBuild/hmrWatcher.ts +17 -3
- package/frontendBuild/index.ts +3 -0
- package/frontendBuild/pagesBundleBuilder.ts +24 -1
- package/frontendBuild/pagesEntrySourceGenerator.ts +9 -2
- package/frontendBuild/registryMetadataFingerprint.test.ts +63 -0
- package/frontendBuild/registryMetadataFingerprint.ts +28 -0
- package/frontendBuild/routeClientBuilder.ts +79 -18
- package/frontendBuild/serverGraphFile.test.ts +77 -0
- package/frontendBuild/serverGraphFile.ts +93 -0
- package/frontendBuild/ssrDevBundler.ts +91 -0
- package/incrementalBuilder/buildBatch.proc.ts +94 -16
- package/incrementalBuilder/buildBatchProtocol.ts +6 -3
- package/incrementalBuilder/buildBatchRunner.test.ts +31 -0
- package/incrementalBuilder/buildBatchRunner.ts +17 -8
- package/incrementalBuilder/builderWorkQueue.test.ts +94 -0
- package/incrementalBuilder/builderWorkQueue.ts +126 -0
- package/incrementalBuilder/incrementalBuilder.host.test.ts +1 -1
- package/incrementalBuilder/incrementalBuilder.host.ts +32 -3
- package/incrementalBuilder/incrementalBuilder.proc.test.ts +129 -2
- package/incrementalBuilder/incrementalBuilder.proc.ts +518 -52
- package/integration/devStability.integration.test.ts +18 -7
- package/package.json +2 -2
- package/transforms/rscUseClientTransform.ts +7 -0
- package/transforms/transforms.test.ts +13 -0
- package/transforms/useClientBundlePlugin.ts +4 -1
|
@@ -11,10 +11,12 @@ import {
|
|
|
11
11
|
CssCompiler,
|
|
12
12
|
FontOptimizer,
|
|
13
13
|
PagesBundleBuilder,
|
|
14
|
+
ServerGraphFile,
|
|
14
15
|
SsrBaseArtifactBuilder,
|
|
16
|
+
SsrDevBundler,
|
|
15
17
|
} from "@akanjs/devkit/frontendBuild";
|
|
16
18
|
import { Logger } from "akanjs/common";
|
|
17
|
-
import type { BuilderMessage, BuildPhase } from "akanjs/server";
|
|
19
|
+
import type { BuilderMessage, BuildPhase, HmrTrace } from "akanjs/server";
|
|
18
20
|
import { resolveDevCsrMode } from "akanjs/server/hmr/csrDevManifest";
|
|
19
21
|
import type { BuildBatchRequest, BuildBatchResult, OptimizedFonts, PagesBatchCssAssets } from "./buildBatchProtocol";
|
|
20
22
|
|
|
@@ -32,10 +34,11 @@ class BuildBatch {
|
|
|
32
34
|
|
|
33
35
|
async run(): Promise<BuildBatchResult> {
|
|
34
36
|
if (this.#request.needs.includes("base")) await this.#buildBase();
|
|
35
|
-
// Order kept from the in-process build: csr before pages, css last because it depends on the rebuilt client.
|
|
36
37
|
if (this.#request.needs.includes("csr")) await this.#buildCsr();
|
|
37
|
-
if (this.#request.needs.includes("
|
|
38
|
+
if (this.#request.needs.includes("ssr")) await this.#buildSsrRegistry();
|
|
39
|
+
// Css before pages: it scans the sources and reads nothing pages produces, so a class edit need not wait for pages.
|
|
38
40
|
if (this.#request.needs.includes("css")) await this.#buildCss();
|
|
41
|
+
if (this.#request.needs.includes("pages")) await this.#buildPages();
|
|
39
42
|
return this.#result;
|
|
40
43
|
}
|
|
41
44
|
|
|
@@ -45,6 +48,12 @@ class BuildBatch {
|
|
|
45
48
|
process.send?.(message);
|
|
46
49
|
}
|
|
47
50
|
|
|
51
|
+
#sentTrace(): HmrTrace | undefined {
|
|
52
|
+
if (!this.#request.trace) return undefined;
|
|
53
|
+
const now = Date.now();
|
|
54
|
+
return { ...this.#request.trace, patchAt: now, sentAt: now };
|
|
55
|
+
}
|
|
56
|
+
|
|
48
57
|
#emitStatus(phase: BuildPhase, message?: string): void {
|
|
49
58
|
this.#emit({
|
|
50
59
|
type: "build-status",
|
|
@@ -102,7 +111,21 @@ class BuildBatch {
|
|
|
102
111
|
}
|
|
103
112
|
|
|
104
113
|
async #updateCsrRegistry(started: number): Promise<void> {
|
|
105
|
-
const update = await new CsrDevBundler(this.#app).update(this.#request.changedFiles
|
|
114
|
+
const update = await new CsrDevBundler(this.#app).update(this.#request.changedFiles, {
|
|
115
|
+
announce: (announced) =>
|
|
116
|
+
this.#emit({
|
|
117
|
+
type: "csr-updated",
|
|
118
|
+
data: {
|
|
119
|
+
generation: announced.generation,
|
|
120
|
+
mode: "registry",
|
|
121
|
+
reload: announced.reload,
|
|
122
|
+
reason: announced.reason,
|
|
123
|
+
patchUrl: announced.patchUrl,
|
|
124
|
+
changedIds: announced.changedIds,
|
|
125
|
+
trace: this.#sentTrace(),
|
|
126
|
+
},
|
|
127
|
+
}),
|
|
128
|
+
});
|
|
106
129
|
if (!update) {
|
|
107
130
|
this.#logger.verbose(`csr-dev unchanged (${Date.now() - started}ms)`);
|
|
108
131
|
return;
|
|
@@ -110,17 +133,48 @@ class BuildBatch {
|
|
|
110
133
|
this.#logger.verbose(
|
|
111
134
|
`csr-dev generation=${update.generation} ${update.reload ? `reload (${update.reason})` : `patch modules=${update.changedIds.length}`} graph=${update.moduleCount} (${Date.now() - started}ms)`,
|
|
112
135
|
);
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
// Takes every entry the routes reach as a root, so a route build naming any of them finds it in the registry.
|
|
139
|
+
async #buildSsrRegistry(): Promise<void> {
|
|
140
|
+
const started = Date.now();
|
|
141
|
+
try {
|
|
142
|
+
const bundler = new SsrDevBundler(this.#app);
|
|
143
|
+
//? Runs before pages, so the graph is the last build's, as in the resident builder: a save it holds still waits.
|
|
144
|
+
const hold = await ServerGraphFile.touches(
|
|
145
|
+
await ServerGraphFile.read(this.#request.artifactDir),
|
|
146
|
+
this.#request.changedFiles,
|
|
147
|
+
(file) => ServerGraphFile.clientExportsOf(file),
|
|
148
|
+
);
|
|
149
|
+
const update = await bundler.update(this.#request.changedFiles, {
|
|
150
|
+
roots: await bundler.clientEntries(),
|
|
151
|
+
announce: (announced) =>
|
|
152
|
+
this.#emit({
|
|
153
|
+
type: "ssr-updated",
|
|
154
|
+
data: {
|
|
155
|
+
generation: announced.generation,
|
|
156
|
+
reload: announced.reload,
|
|
157
|
+
reason: announced.reason,
|
|
158
|
+
patchUrl: announced.patchUrl,
|
|
159
|
+
changedIds: announced.changedIds,
|
|
160
|
+
trace: this.#sentTrace(),
|
|
161
|
+
...(announced.epoch !== undefined ? { epoch: announced.epoch } : {}),
|
|
162
|
+
...(announced.first ? { first: true } : {}),
|
|
163
|
+
...(hold && this.#request.changedFiles.length > 0
|
|
164
|
+
? { hold, batchGeneration: this.#request.generation }
|
|
165
|
+
: {}),
|
|
166
|
+
},
|
|
167
|
+
}),
|
|
168
|
+
});
|
|
169
|
+
this.#emitStatus("ssr");
|
|
170
|
+
this.#logger.verbose(
|
|
171
|
+
update
|
|
172
|
+
? `ssr-dev generation=${update.generation} ${update.reload ? `reload (${update.reason})` : `patch modules=${update.changedIds.length}`} graph=${update.moduleCount} (${Date.now() - started}ms)`
|
|
173
|
+
: `ssr-dev unchanged (${Date.now() - started}ms)`,
|
|
174
|
+
);
|
|
175
|
+
} catch (err) {
|
|
176
|
+
this.#fail("ssr", "ssr-dev", err);
|
|
177
|
+
}
|
|
124
178
|
}
|
|
125
179
|
|
|
126
180
|
// Rewritten with the bundle: the backend rereads it on `pages-updated` to pick up added, moved or deleted routes.
|
|
@@ -129,8 +183,16 @@ class BuildBatch {
|
|
|
129
183
|
try {
|
|
130
184
|
const pageEntries = await resolveSsrPageEntriesForApp(this.#app, await this.#app.getPageKeys());
|
|
131
185
|
const seedIndex = computeRouteSeedIndex(pageEntries);
|
|
186
|
+
const previousGraph = await ServerGraphFile.read(this.#request.artifactDir);
|
|
132
187
|
const next = await new PagesBundleBuilder(this.#app, "start", pageEntries).build();
|
|
133
188
|
await saveRouteSeedIndex(this.#request.artifactDir, seedIndex);
|
|
189
|
+
const nextGraph = await ServerGraphFile.read(this.#request.artifactDir);
|
|
190
|
+
const serverTouched = await ServerGraphFile.touches(
|
|
191
|
+
previousGraph,
|
|
192
|
+
[...this.#request.changedFiles, ...(previousGraph?.carried ?? [])],
|
|
193
|
+
(file) => nextGraph?.clientExports[file] ?? null,
|
|
194
|
+
nextGraph,
|
|
195
|
+
);
|
|
134
196
|
this.#emit({
|
|
135
197
|
type: "pages-updated",
|
|
136
198
|
data: {
|
|
@@ -138,12 +200,17 @@ class BuildBatch {
|
|
|
138
200
|
buildId: next.buildId,
|
|
139
201
|
generation: this.#request.generation,
|
|
140
202
|
changedFiles: this.#request.changedFiles,
|
|
203
|
+
trace: this.#sentTrace(),
|
|
204
|
+
serverTouched,
|
|
141
205
|
},
|
|
142
206
|
});
|
|
143
207
|
this.#emitStatus("pages");
|
|
144
208
|
this.#logger.verbose(`pages-rebundle ok buildId=${next.buildId} (${Date.now() - started}ms)`);
|
|
145
209
|
} catch (err) {
|
|
146
210
|
this.#fail("pages", "pages-rebundle", err);
|
|
211
|
+
await ServerGraphFile.carry(this.#request.artifactDir, this.#request.changedFiles).catch((carryError: unknown) =>
|
|
212
|
+
this.#logger.warn(`server graph carry failed; the next save may skip its RSC refresh: ${String(carryError)}`),
|
|
213
|
+
);
|
|
147
214
|
}
|
|
148
215
|
}
|
|
149
216
|
|
|
@@ -213,7 +280,13 @@ class BuildBatch {
|
|
|
213
280
|
static async main(): Promise<void> {
|
|
214
281
|
const raw = process.argv[2];
|
|
215
282
|
if (!raw) throw new Error("[build-batch] missing request argument");
|
|
216
|
-
const
|
|
283
|
+
const parsed = JSON.parse(raw) as BuildBatchRequest;
|
|
284
|
+
const request = parsed.trace
|
|
285
|
+
? {
|
|
286
|
+
...parsed,
|
|
287
|
+
trace: { ...parsed.trace, workerStartAt: Math.round(performance.timeOrigin), workerAt: Date.now() },
|
|
288
|
+
}
|
|
289
|
+
: parsed;
|
|
217
290
|
const workspace = WorkspaceExecutor.fromRoot({
|
|
218
291
|
workspaceRoot: request.workspaceRoot,
|
|
219
292
|
repoName: request.repoName,
|
|
@@ -221,8 +294,13 @@ class BuildBatch {
|
|
|
221
294
|
const app = AppExecutor.from(workspace, request.appName);
|
|
222
295
|
// Seeded, not rediscovered: route discovery would be the largest cost of spawning this process.
|
|
223
296
|
if (request.pageKeys) app.setPageKeys(request.pageKeys);
|
|
297
|
+
//? Exits with the builder that spawned it (a kill, a crash, a restart for a metadata save): left running, it would
|
|
298
|
+
//? write a registry its replacement is rebuilding. Removed before returning, since the listener keeps Bun's IPC open.
|
|
299
|
+
const orphaned = () => process.exit(1);
|
|
300
|
+
process.on("disconnect", orphaned);
|
|
224
301
|
const result = await new BuildBatch(request, app).run();
|
|
225
302
|
process.send?.({ type: "build-batch-result", data: result });
|
|
303
|
+
process.off("disconnect", orphaned);
|
|
226
304
|
}
|
|
227
305
|
}
|
|
228
306
|
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import type { FontOptimizer } from "@akanjs/devkit/frontendBuild";
|
|
2
|
-
import type { BaseBuildArtifact, CssPayload, PagesBundlePayload } from "akanjs/server";
|
|
2
|
+
import type { BaseBuildArtifact, CssPayload, HmrTrace, PagesBundlePayload } from "akanjs/server";
|
|
3
3
|
|
|
4
4
|
export type OptimizedFonts = Awaited<ReturnType<FontOptimizer["optimize"]>>;
|
|
5
5
|
|
|
6
6
|
// `base` is the boot build: it always runs alone, before the builder can serve anything.
|
|
7
|
-
export type BuildBatchNeed = "base" | "pages" | "css" | "csr";
|
|
7
|
+
export type BuildBatchNeed = "base" | "pages" | "css" | "csr" | "ssr";
|
|
8
8
|
|
|
9
9
|
// JSON only: the worker is spawned per batch and receives this in its argv.
|
|
10
10
|
export interface BuildBatchRequest {
|
|
@@ -18,6 +18,7 @@ export interface BuildBatchRequest {
|
|
|
18
18
|
optimizedFonts: OptimizedFonts | null; // reused unless this batch touched one of its files
|
|
19
19
|
cssAssets: PagesBatchCssAssets | null; // an unchanged compile skips the broadcast instead of busting hashes
|
|
20
20
|
artifactDir: string;
|
|
21
|
+
trace?: HmrTrace;
|
|
21
22
|
}
|
|
22
23
|
|
|
23
24
|
export type PagesBatchCssAssets = CssPayload["cssAssets"];
|
|
@@ -29,7 +30,9 @@ export interface BuildBatchResult {
|
|
|
29
30
|
optimizedFonts?: OptimizedFonts;
|
|
30
31
|
artifact?: BaseBuildArtifact; // `base` batches only
|
|
31
32
|
errors: Partial<Record<BuildBatchNeed, string>>;
|
|
32
|
-
crashed?: boolean; // died before reporting
|
|
33
|
+
crashed?: boolean; // died before reporting its result
|
|
34
|
+
/** Of a crashed batch, the needs it died before reporting: the others streamed their build-status first. */
|
|
35
|
+
crashedNeeds?: BuildBatchNeed[];
|
|
33
36
|
}
|
|
34
37
|
|
|
35
38
|
export type BuildBatchMessage = { type: "build-batch-result"; data: BuildBatchResult };
|
|
@@ -31,4 +31,35 @@ describe("BuildBatchRunner", () => {
|
|
|
31
31
|
|
|
32
32
|
expect(result.errors.base).toContain('"./not-there" (apps/demo/page/_index.tsx:1:22)');
|
|
33
33
|
}, 30_000);
|
|
34
|
+
|
|
35
|
+
test("a worker that dies mid-batch keeps what each need reported, and fails only the needs it never reached", async () => {
|
|
36
|
+
const { root } = track(await createTempApp("demo"));
|
|
37
|
+
const appDir = path.join(root, "apps/demo");
|
|
38
|
+
await writeText(
|
|
39
|
+
path.join(root, "pkgs/@akanjs/devkit/incrementalBuilder/buildBatch.proc.ts"),
|
|
40
|
+
`const { generation } = JSON.parse(process.argv[2]);
|
|
41
|
+
const status = (phase, message) =>
|
|
42
|
+
process.send({ type: "build-status", data: { generation, phase, ok: !message, files: [], message } });
|
|
43
|
+
status("ssr");
|
|
44
|
+
status("css", "css broke");
|
|
45
|
+
setTimeout(() => process.exit(1), 50);
|
|
46
|
+
`,
|
|
47
|
+
);
|
|
48
|
+
const result = await new BuildBatchRunner({ workspaceRoot: root, cwd: appDir }).run({
|
|
49
|
+
appName: "demo",
|
|
50
|
+
workspaceRoot: root,
|
|
51
|
+
repoName: "repo",
|
|
52
|
+
generation: 7,
|
|
53
|
+
needs: ["ssr", "css", "pages"],
|
|
54
|
+
changedFiles: [],
|
|
55
|
+
pageKeys: null,
|
|
56
|
+
optimizedFonts: null,
|
|
57
|
+
cssAssets: null,
|
|
58
|
+
artifactDir: path.join(appDir, ".akan/artifact"),
|
|
59
|
+
});
|
|
60
|
+
expect(result).toMatchObject({ crashed: true, crashedNeeds: ["pages"] });
|
|
61
|
+
expect(result.errors.ssr).toBeUndefined();
|
|
62
|
+
expect(result.errors.css).toBe("css broke");
|
|
63
|
+
expect(result.errors.pages).toContain("exited with code 1");
|
|
64
|
+
}, 30_000);
|
|
34
65
|
});
|
|
@@ -3,7 +3,8 @@ import { Logger } from "akanjs/common";
|
|
|
3
3
|
import type { BuilderMessage } from "akanjs/server";
|
|
4
4
|
import type { BuildBatchMessage, BuildBatchRequest, BuildBatchResult } from "./buildBatchProtocol";
|
|
5
5
|
|
|
6
|
-
// No pool on purpose:
|
|
6
|
+
// No pool on purpose: a worker exiting after its batch returns the arenas `Bun.build` never frees. The callers order
|
|
7
|
+
// their own work (the slow lane, the css queue, the SSR registry's boot build beside them), so a few can overlap.
|
|
7
8
|
// A worker that dies without reporting must not take the watcher down; its needs come back as errors.
|
|
8
9
|
export class BuildBatchRunner {
|
|
9
10
|
#logger = new Logger("BuildBatchRunner");
|
|
@@ -38,8 +39,10 @@ export class BuildBatchRunner {
|
|
|
38
39
|
const started = Date.now();
|
|
39
40
|
const entry = await this.#resolveEntry();
|
|
40
41
|
let result: BuildBatchResult | null = null;
|
|
42
|
+
const reported = new Map<string, string | undefined>();
|
|
43
|
+
const payload = request.trace ? { ...request, trace: { ...request.trace, spawnAt: Date.now() } } : request;
|
|
41
44
|
// argv rather than IPC, so the worker starts on its first tick instead of waiting for a handshake.
|
|
42
|
-
const proc = Bun.spawn(["bun", entry, JSON.stringify(
|
|
45
|
+
const proc = Bun.spawn(["bun", entry, JSON.stringify(payload)], {
|
|
43
46
|
cwd: this.#cwd,
|
|
44
47
|
env: process.env,
|
|
45
48
|
stdio: ["ignore", "inherit", "inherit"],
|
|
@@ -47,7 +50,10 @@ export class BuildBatchRunner {
|
|
|
47
50
|
ipc: (message: BuildBatchMessage | BuilderMessage) => {
|
|
48
51
|
if (!message || typeof message !== "object") return;
|
|
49
52
|
if (message.type === "build-batch-result") result = message.data;
|
|
50
|
-
else
|
|
53
|
+
else {
|
|
54
|
+
if (message.type === "build-status") reported.set(message.data.phase, message.data.message);
|
|
55
|
+
onMessage(message);
|
|
56
|
+
}
|
|
51
57
|
},
|
|
52
58
|
});
|
|
53
59
|
const exitCode = await proc.exited;
|
|
@@ -64,10 +70,13 @@ export class BuildBatchRunner {
|
|
|
64
70
|
}`
|
|
65
71
|
: `build worker exited with code ${exitCode} before reporting a result`;
|
|
66
72
|
this.#logger.error(`[build-batch] generation=${request.generation} ${message}`);
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
73
|
+
//? A need that reported before the crash keeps what it reported: a worker killed during pages built ssr and css.
|
|
74
|
+
const crashedNeeds = request.needs.filter((need) => !reported.has(need));
|
|
75
|
+
const failed = request.needs.flatMap((need) => {
|
|
76
|
+
if (!reported.has(need)) return [[need, message] as const];
|
|
77
|
+
const error = reported.get(need);
|
|
78
|
+
return error ? [[need, error] as const] : [];
|
|
79
|
+
});
|
|
80
|
+
return { generation: request.generation, errors: Object.fromEntries(failed), crashed: true, crashedNeeds };
|
|
72
81
|
}
|
|
73
82
|
}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { type BatchJob, BuilderWorkQueue } from "./builderWorkQueue";
|
|
3
|
+
|
|
4
|
+
const gate = () => {
|
|
5
|
+
const { promise: opened, resolve: open } = Promise.withResolvers<void>();
|
|
6
|
+
return { open, opened };
|
|
7
|
+
};
|
|
8
|
+
|
|
9
|
+
describe("BuilderWorkQueue", () => {
|
|
10
|
+
test("runs jobs one at a time, in the order they were queued", async () => {
|
|
11
|
+
const order: string[] = [];
|
|
12
|
+
const queue = new BuilderWorkQueue({ runBatch: async () => undefined });
|
|
13
|
+
const first = gate();
|
|
14
|
+
const a = queue.enqueue("a", async () => {
|
|
15
|
+
await first.opened;
|
|
16
|
+
order.push("a");
|
|
17
|
+
});
|
|
18
|
+
const b = queue.enqueue("b", async () => {
|
|
19
|
+
order.push("b");
|
|
20
|
+
return 2;
|
|
21
|
+
});
|
|
22
|
+
expect(queue.size).toBe(2);
|
|
23
|
+
first.open();
|
|
24
|
+
await a;
|
|
25
|
+
expect(await b).toBe(2);
|
|
26
|
+
expect(order).toEqual(["a", "b"]);
|
|
27
|
+
expect(queue.size).toBe(0);
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
test("folds a batch queued behind another batch into it, for the newest generation", async () => {
|
|
31
|
+
const ran: BatchJob[] = [];
|
|
32
|
+
const running = gate();
|
|
33
|
+
const queue = new BuilderWorkQueue({
|
|
34
|
+
runBatch: async (batch) => {
|
|
35
|
+
ran.push(batch);
|
|
36
|
+
if (ran.length === 1) await running.opened;
|
|
37
|
+
},
|
|
38
|
+
});
|
|
39
|
+
const first = queue.enqueueBatch({ generation: 1, needs: ["pages", "css"], changedFiles: ["a.tsx"] });
|
|
40
|
+
const second = queue.enqueueBatch({
|
|
41
|
+
generation: 2,
|
|
42
|
+
needs: ["pages", "css"],
|
|
43
|
+
changedFiles: ["b.tsx"],
|
|
44
|
+
discovery: { files: ["b.tsx"], refresh: false, generation: 2 },
|
|
45
|
+
});
|
|
46
|
+
const third = queue.enqueueBatch({ generation: 3, needs: ["csr", "pages"], changedFiles: ["a.tsx", "c.tsx"] });
|
|
47
|
+
expect(second).toBe(third);
|
|
48
|
+
running.open();
|
|
49
|
+
await Promise.all([first, second, third]);
|
|
50
|
+
expect(ran).toHaveLength(2);
|
|
51
|
+
expect(ran[1]).toEqual({
|
|
52
|
+
generation: 3,
|
|
53
|
+
needs: ["pages", "css", "csr"],
|
|
54
|
+
changedFiles: ["b.tsx", "a.tsx", "c.tsx"],
|
|
55
|
+
discovery: { files: ["b.tsx"], refresh: false, generation: 2 },
|
|
56
|
+
});
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
test("keeps a batch queued after a task behind that task", async () => {
|
|
60
|
+
const order: string[] = [];
|
|
61
|
+
const running = gate();
|
|
62
|
+
const queue = new BuilderWorkQueue({
|
|
63
|
+
runBatch: async (batch) => {
|
|
64
|
+
order.push(`batch:${batch.generation}`);
|
|
65
|
+
if (batch.generation === 1) await running.opened;
|
|
66
|
+
},
|
|
67
|
+
});
|
|
68
|
+
const first = queue.enqueueBatch({ generation: 1, needs: ["pages"], changedFiles: [] });
|
|
69
|
+
const route = queue.enqueue("route", async () => {
|
|
70
|
+
order.push("route");
|
|
71
|
+
});
|
|
72
|
+
const second = queue.enqueueBatch({ generation: 2, needs: ["pages"], changedFiles: [] });
|
|
73
|
+
running.open();
|
|
74
|
+
await Promise.all([first, route, second]);
|
|
75
|
+
expect(order).toEqual(["batch:1", "route", "batch:2"]);
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
test("drains jobs queued while it waits, and survives a job that throws", async () => {
|
|
79
|
+
const queue = new BuilderWorkQueue({ runBatch: async () => undefined });
|
|
80
|
+
const failing = queue.enqueue("fails", async () => {
|
|
81
|
+
throw new Error("boom");
|
|
82
|
+
});
|
|
83
|
+
let late = false;
|
|
84
|
+
void queue.enqueue("queues-more", async () => {
|
|
85
|
+
void queue.enqueue("late", async () => {
|
|
86
|
+
late = true;
|
|
87
|
+
});
|
|
88
|
+
});
|
|
89
|
+
await expect(failing).rejects.toThrow("boom");
|
|
90
|
+
await queue.drain();
|
|
91
|
+
expect(late).toBe(true);
|
|
92
|
+
expect(queue.size).toBe(0);
|
|
93
|
+
});
|
|
94
|
+
});
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import type { HmrTrace } from "akanjs/server";
|
|
2
|
+
import type { BuildBatchNeed } from "./buildBatchProtocol";
|
|
3
|
+
|
|
4
|
+
export interface DiscoveryJob {
|
|
5
|
+
files: string[];
|
|
6
|
+
refresh: boolean;
|
|
7
|
+
/** The newest batch these files came from: a route build that starts after has taken it in. */
|
|
8
|
+
generation: number;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export interface BatchJob {
|
|
12
|
+
generation: number;
|
|
13
|
+
needs: BuildBatchNeed[];
|
|
14
|
+
changedFiles: string[];
|
|
15
|
+
trace?: HmrTrace;
|
|
16
|
+
/** Code files whose client-entry discovery goes stale before this batch builds; `refresh` rebuilds it whole. */
|
|
17
|
+
discovery?: DiscoveryJob;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
interface QueuedJob {
|
|
21
|
+
label: string;
|
|
22
|
+
batch: BatchJob | null;
|
|
23
|
+
run: () => Promise<void>;
|
|
24
|
+
settle: PromiseWithResolvers<void>;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export interface BuilderWorkQueueOptions {
|
|
28
|
+
runBatch: (batch: BatchJob) => Promise<void>;
|
|
29
|
+
onSettled?: (label: string, ms: number) => void;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
//* The builder's slow lane: build-worker batches, route builds and full CSR builds run here one at a time, so a save's
|
|
33
|
+
//* CSR patch never waits behind them. A batch queued behind another batch folds into it: a burst of saves builds pages
|
|
34
|
+
//* and css once for the newest generation instead of once per save.
|
|
35
|
+
export class BuilderWorkQueue {
|
|
36
|
+
readonly #runBatch: BuilderWorkQueueOptions["runBatch"];
|
|
37
|
+
readonly #onSettled: BuilderWorkQueueOptions["onSettled"];
|
|
38
|
+
readonly #pending: QueuedJob[] = [];
|
|
39
|
+
#running: QueuedJob | null = null;
|
|
40
|
+
#idle: Promise<void> = Promise.resolve();
|
|
41
|
+
#wakeIdle: (() => void) | null = null;
|
|
42
|
+
|
|
43
|
+
constructor({ runBatch, onSettled }: BuilderWorkQueueOptions) {
|
|
44
|
+
this.#runBatch = runBatch;
|
|
45
|
+
this.#onSettled = onSettled;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** Jobs waiting or running. */
|
|
49
|
+
get size(): number {
|
|
50
|
+
return this.#pending.length + (this.#running ? 1 : 0);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
enqueueBatch(batch: BatchJob): Promise<void> {
|
|
54
|
+
const tail = this.#pending.at(-1);
|
|
55
|
+
if (tail?.batch) {
|
|
56
|
+
tail.batch = BuilderWorkQueue.merge(tail.batch, batch);
|
|
57
|
+
return tail.settle.promise;
|
|
58
|
+
}
|
|
59
|
+
const job = this.#job("batch", batch, async () => undefined);
|
|
60
|
+
job.run = async () => await this.#runBatch(job.batch ?? batch);
|
|
61
|
+
return this.#push(job);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
enqueue<T>(label: string, run: () => Promise<T>): Promise<T> {
|
|
65
|
+
let result: T | undefined;
|
|
66
|
+
const job = this.#job(label, null, async () => {
|
|
67
|
+
result = await run();
|
|
68
|
+
});
|
|
69
|
+
return this.#push(job).then(() => result as T);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Resolves once nothing is waiting or running, including jobs queued while it waits. */
|
|
73
|
+
async drain(): Promise<void> {
|
|
74
|
+
while (this.size > 0) await this.#idle;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
static merge(into: BatchJob, next: BatchJob): BatchJob {
|
|
78
|
+
const discovery =
|
|
79
|
+
into.discovery || next.discovery
|
|
80
|
+
? {
|
|
81
|
+
files: [...new Set([...(into.discovery?.files ?? []), ...(next.discovery?.files ?? [])])],
|
|
82
|
+
refresh: !!into.discovery?.refresh || !!next.discovery?.refresh,
|
|
83
|
+
generation: Math.max(into.discovery?.generation ?? 0, next.discovery?.generation ?? 0),
|
|
84
|
+
}
|
|
85
|
+
: undefined;
|
|
86
|
+
return {
|
|
87
|
+
generation: Math.max(into.generation, next.generation),
|
|
88
|
+
needs: [...new Set([...into.needs, ...next.needs])],
|
|
89
|
+
changedFiles: [...new Set([...into.changedFiles, ...next.changedFiles])],
|
|
90
|
+
trace: next.trace ?? into.trace,
|
|
91
|
+
...(discovery ? { discovery } : {}),
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
#job(label: string, batch: BatchJob | null, run: () => Promise<void>): QueuedJob {
|
|
96
|
+
return { label, batch, run, settle: Promise.withResolvers<void>() };
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
#push(job: QueuedJob): Promise<void> {
|
|
100
|
+
if (this.size === 0)
|
|
101
|
+
this.#idle = new Promise<void>((resolve) => {
|
|
102
|
+
this.#wakeIdle = resolve;
|
|
103
|
+
});
|
|
104
|
+
this.#pending.push(job);
|
|
105
|
+
if (!this.#running) void this.#pump();
|
|
106
|
+
return job.settle.promise;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
async #pump(): Promise<void> {
|
|
110
|
+
for (let job = this.#pending.shift(); job; job = this.#pending.shift()) {
|
|
111
|
+
this.#running = job;
|
|
112
|
+
const started = Date.now();
|
|
113
|
+
try {
|
|
114
|
+
await job.run();
|
|
115
|
+
job.settle.resolve();
|
|
116
|
+
} catch (error) {
|
|
117
|
+
job.settle.reject(error);
|
|
118
|
+
} finally {
|
|
119
|
+
this.#running = null;
|
|
120
|
+
this.#onSettled?.(job.label, Date.now() - started);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
this.#wakeIdle?.();
|
|
124
|
+
this.#wakeIdle = null;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
@@ -196,7 +196,7 @@ describe("IncrementalBuilderHost.maxRssBytes", () => {
|
|
|
196
196
|
withEnv(
|
|
197
197
|
{ AKAN_BUILDER_MAX_RSS_MB: undefined, AKAN_BUILDER_MAX_RSS: undefined, AKAN_MEMORY_LIMIT: undefined },
|
|
198
198
|
() => {
|
|
199
|
-
expect(IncrementalBuilderHost.maxRssBytes()).toBe(
|
|
199
|
+
expect(IncrementalBuilderHost.maxRssBytes()).toBe(2_048 * 1024 * 1024);
|
|
200
200
|
},
|
|
201
201
|
);
|
|
202
202
|
});
|
|
@@ -1,6 +1,8 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
1
2
|
import path from "node:path";
|
|
2
3
|
import { Logger } from "akanjs/common";
|
|
3
4
|
import type { BuilderMessage } from "akanjs/server";
|
|
5
|
+
import { CSR_DEV_DIRNAME, CSR_DEV_PATCHING_MARKER, SSR_DEV_DIRNAME } from "akanjs/server/hmr/csrDevManifest";
|
|
4
6
|
import { MemoryLimit } from "akanjs/server/memoryLimit";
|
|
5
7
|
import type { App } from "../commandDecorators";
|
|
6
8
|
|
|
@@ -12,8 +14,10 @@ const builderMsgTypeSet = new Set<BuilderMessage["type"]>([
|
|
|
12
14
|
"css-updated",
|
|
13
15
|
"pages-updated",
|
|
14
16
|
"csr-updated",
|
|
17
|
+
"ssr-updated",
|
|
15
18
|
"build-status",
|
|
16
19
|
"builder-metrics",
|
|
20
|
+
"boot-armed",
|
|
17
21
|
]);
|
|
18
22
|
/** Prefer `"pipe"` under a TUI: a Bun child that inherits the terminal restores its spawn-time termios on exit. */
|
|
19
23
|
export type DevStdioMode = "inherit" | "pipe";
|
|
@@ -45,8 +49,9 @@ export class IncrementalBuilderHost {
|
|
|
45
49
|
static readonly #restartBaseDelayMs = 1_000;
|
|
46
50
|
static readonly #restartMaxDelayMs = 30_000;
|
|
47
51
|
static readonly #recycleDrainTimeoutMs = 30_000;
|
|
48
|
-
//
|
|
49
|
-
|
|
52
|
+
// apps/akan's builder peaks near 950MB holding the SSR registry's patcher through a route build, and arming CSR adds
|
|
53
|
+
// a second patcher: this recycles only a builder grown well past both.
|
|
54
|
+
static readonly #devMaxRssBytes = 2_048 * 1024 * 1024;
|
|
50
55
|
logger = new Logger("IncrementalBuilderHost");
|
|
51
56
|
entry: string;
|
|
52
57
|
env: Record<string, string>;
|
|
@@ -62,6 +67,7 @@ export class IncrementalBuilderHost {
|
|
|
62
67
|
#recycleTimer: ReturnType<typeof setTimeout> | null = null;
|
|
63
68
|
#recycleRequested: boolean = false;
|
|
64
69
|
#spawnAfterRecycle: boolean = false;
|
|
70
|
+
#patcherOff: boolean = false;
|
|
65
71
|
#manualStop = false;
|
|
66
72
|
// Nothing else answers a request whose builder exits holding it: a crash or kill sends nothing, and a drain races
|
|
67
73
|
// its own exit, so an unanswered page request would spin forever.
|
|
@@ -95,10 +101,16 @@ export class IncrementalBuilderHost {
|
|
|
95
101
|
this.ready = false;
|
|
96
102
|
const afterRecycle = this.#spawnAfterRecycle;
|
|
97
103
|
this.#spawnAfterRecycle = false;
|
|
104
|
+
this.#checkPatchingMarker(isRestart && !afterRecycle);
|
|
98
105
|
let proc!: Bun.Subprocess<"ignore", "inherit" | "pipe", "inherit" | "pipe">;
|
|
99
106
|
proc = Bun.spawn(["bun", this.entry], {
|
|
100
107
|
cwd: this.app.cwdPath,
|
|
101
|
-
env: {
|
|
108
|
+
env: {
|
|
109
|
+
...this.env,
|
|
110
|
+
AKAN_WATCH: "1",
|
|
111
|
+
...(afterRecycle ? { AKAN_BUILDER_ANNOUNCE_BOOT: "1" } : {}),
|
|
112
|
+
...(this.#patcherOff ? { AKAN_DEV_CSR_PATCHER: "off" } : {}),
|
|
113
|
+
},
|
|
102
114
|
stdio: ["ignore", this.#stdio, this.#stdio],
|
|
103
115
|
ipc: (msg: BuilderMessage) => {
|
|
104
116
|
if (this.#proc !== proc) return;
|
|
@@ -152,6 +164,23 @@ export class IncrementalBuilderHost {
|
|
|
152
164
|
}
|
|
153
165
|
this.logger.verbose(`builder spawned pid=${proc.pid} entry=${this.entry}${isRestart ? " restart=1" : ""}`);
|
|
154
166
|
}
|
|
167
|
+
//? The resident builder patches the dev CSR bundle in its own process, so a bundler crash there takes the file watcher
|
|
168
|
+
//? with it. A builder that died holding the marker hands every later CSR save to build workers for this session;
|
|
169
|
+
//? a marker any other exit left behind (Ctrl-C mid-patch) is cleared so it cannot turn the patcher off next session.
|
|
170
|
+
#checkPatchingMarker(afterCrash: boolean): void {
|
|
171
|
+
const markers = [CSR_DEV_DIRNAME, SSR_DEV_DIRNAME]
|
|
172
|
+
.map((dirName) => path.join(this.app.cwdPath, ".akan/artifact", dirName, CSR_DEV_PATCHING_MARKER))
|
|
173
|
+
.filter((marker) => fs.existsSync(marker));
|
|
174
|
+
if (markers.length === 0) return;
|
|
175
|
+
if (afterCrash && !this.#patcherOff) {
|
|
176
|
+
this.#patcherOff = true;
|
|
177
|
+
this.logger.warn(
|
|
178
|
+
"the builder died while patching a dev module registry; registry saves go to a build worker until the next config or metadata restart (AKAN_DEV_CSR_PATCHER=off)",
|
|
179
|
+
);
|
|
180
|
+
}
|
|
181
|
+
for (const marker of markers) fs.rmSync(marker, { force: true });
|
|
182
|
+
}
|
|
183
|
+
|
|
155
184
|
async #drain(stream: ReadableStream<Uint8Array> | undefined | null, kind: "stdout" | "stderr") {
|
|
156
185
|
if (!stream) return;
|
|
157
186
|
const decoder = new TextDecoder();
|