@akanjs/devkit 2.4.1-rc.3 → 2.4.1-rc.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/akanApp/akanApp.host.test.ts +199 -2
- package/akanApp/akanApp.host.ts +399 -9
- package/executors.test.ts +60 -0
- package/executors.ts +11 -0
- package/frontendBuild/fontOptimizer.test.ts +111 -0
- package/frontendBuild/fontOptimizer.ts +102 -17
- package/frontendBuild/hmrWatcher.test.ts +191 -0
- package/frontendBuild/hmrWatcher.ts +176 -5
- package/frontendBuild/index.ts +1 -0
- package/frontendBuild/sourceMtimeIndex.test.ts +280 -0
- package/frontendBuild/sourceMtimeIndex.ts +326 -0
- package/incrementalBuilder/buildBatch.proc.ts +34 -1
- package/incrementalBuilder/buildBatchProtocol.ts +13 -2
- package/incrementalBuilder/builderChannel.test.ts +144 -0
- package/incrementalBuilder/builderChannel.ts +72 -0
- package/incrementalBuilder/incrementalBuilder.host.test.ts +88 -3
- package/incrementalBuilder/incrementalBuilder.host.ts +50 -3
- package/incrementalBuilder/incrementalBuilder.proc.ts +93 -34
- package/integration/devStability.integration.test.ts +260 -101
- package/integration/devStabilityHarness.test.ts +111 -0
- package/integration/devStabilityHarness.ts +528 -39
- package/package.json +2 -2
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
import type { Dirent, Stats } from "node:fs";
|
|
2
|
+
import { readdir, stat } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { HmrChangeClassifier } from "./hmrChangeClassifier";
|
|
5
|
+
|
|
6
|
+
interface TrackedFile {
|
|
7
|
+
mtimeMs: number;
|
|
8
|
+
size: number;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* A directory the index could not read, held with this mtime so every subsequent scan sees a mismatch
|
|
13
|
+
* and retries it. `NaN !== NaN`, which is exactly the "always stale" semantics wanted here.
|
|
14
|
+
*/
|
|
15
|
+
const UNREADABLE = Number.NaN;
|
|
16
|
+
|
|
17
|
+
export interface SourceMtimeIndexOptions {
|
|
18
|
+
roots: string[];
|
|
19
|
+
classifier?: HmrChangeClassifier;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* A snapshot of every interesting source file's mtime under the watch roots, so the watcher can answer
|
|
24
|
+
* "what actually changed" without trusting `fs.watch` event payloads.
|
|
25
|
+
*
|
|
26
|
+
* Bun 1.3.14's recursive `fs.watch` on macOS reports roughly **one path per ~200ms coalescing window**
|
|
27
|
+
* and discards every other path changed in that window. Measured on this repo: saving 5 files together
|
|
28
|
+
* reports 1; saving 20 reports 2; a single unrelated write landing first in the window hides a
|
|
29
|
+
* concurrent save entirely. Node 22 reports all 20 on the same tree, so this is a Bun defect rather
|
|
30
|
+
* than an FSEvents limitation (`local/optimize-resource/06-watcher-dropped-event.md`).
|
|
31
|
+
*
|
|
32
|
+
* Re-stating the tracked set costs ~15ms against ~1300 source files here, where a fresh walk costs
|
|
33
|
+
* ~70ms — the walk has to visit ~40k entries (`ios/`, `android/`, `public/`) to find those 1300. So
|
|
34
|
+
* changes are found by re-stating known files plus re-reading only the directories whose own mtime
|
|
35
|
+
* moved, which is what adding or removing an entry bumps.
|
|
36
|
+
*/
|
|
37
|
+
export class SourceMtimeIndex {
|
|
38
|
+
readonly #roots: string[];
|
|
39
|
+
readonly #classifier: HmrChangeClassifier;
|
|
40
|
+
readonly #files = new Map<string, TrackedFile>();
|
|
41
|
+
readonly #dirs = new Map<string, number>();
|
|
42
|
+
readonly #unreadable = new Map<string, string>();
|
|
43
|
+
#primed = false;
|
|
44
|
+
#queue: Promise<unknown> = Promise.resolve();
|
|
45
|
+
|
|
46
|
+
constructor({ roots, classifier }: SourceMtimeIndexOptions) {
|
|
47
|
+
this.#roots = SourceMtimeIndex.#pruneNestedRoots(roots);
|
|
48
|
+
this.#classifier = classifier ?? new HmrChangeClassifier();
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
get primed(): boolean {
|
|
52
|
+
return this.#primed;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
get trackedFileCount(): number {
|
|
56
|
+
return this.#files.size;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Paths the index currently cannot read, as `{ path, code }`. Non-empty means change detection has a
|
|
61
|
+
* blind spot right now: anything edited underneath one of these is not reported until it recovers.
|
|
62
|
+
*
|
|
63
|
+
* Worth surfacing rather than swallowing, because the failure is silent by nature — a walk cannot tell
|
|
64
|
+
* how much it did not see. Each entry is retried on every scan and clears itself on success.
|
|
65
|
+
*/
|
|
66
|
+
get coverageGaps(): { path: string; code: string }[] {
|
|
67
|
+
return [...this.#unreadable].map(([file, code]) => ({ path: file, code }));
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Record the current state as the baseline. Reports nothing; call before the first `collectChanges`. */
|
|
71
|
+
async prime(): Promise<void> {
|
|
72
|
+
await this.#serialize(async () => {
|
|
73
|
+
this.#files.clear();
|
|
74
|
+
this.#dirs.clear();
|
|
75
|
+
this.#unreadable.clear();
|
|
76
|
+
await Promise.all(this.#roots.map((root) => this.#walk(root, null)));
|
|
77
|
+
this.#primed = true;
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Absolute paths whose content changed (or which disappeared) since the last call, with the baseline
|
|
83
|
+
* advanced to match. Returns an empty list until primed, so a failed prime degrades to watcher events
|
|
84
|
+
* rather than reporting the entire tree as changed.
|
|
85
|
+
*/
|
|
86
|
+
async collectChanges(): Promise<string[]> {
|
|
87
|
+
return this.#serialize(async () => {
|
|
88
|
+
if (!this.#primed) return [];
|
|
89
|
+
const changed = new Set<string>();
|
|
90
|
+
await this.#collectFileChanges(changed);
|
|
91
|
+
await this.#collectDirChanges(changed);
|
|
92
|
+
return [...changed];
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Every scan runs to completion before the next begins.
|
|
98
|
+
*
|
|
99
|
+
* Overlapping scans corrupt each other rather than merely racing: one advances the baseline and prunes
|
|
100
|
+
* entries while the other is midway through a key snapshot it took earlier, so a path the first has
|
|
101
|
+
* already accounted for reads back as missing and is reported as a change that never happened. Observed
|
|
102
|
+
* directly — a file nothing had written was reported, which downstream is a wasted rebuild.
|
|
103
|
+
*/
|
|
104
|
+
async #serialize<T>(work: () => Promise<T>): Promise<T> {
|
|
105
|
+
const run = this.#queue.then(work, work);
|
|
106
|
+
// Swallowed on the chain only; the caller still sees the rejection through `run`.
|
|
107
|
+
this.#queue = run.catch(() => undefined);
|
|
108
|
+
return run;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Adopt writes the caller made itself — regenerated barrels, inserted imports — into the baseline so
|
|
113
|
+
* the next `collectChanges` does not report them as a user edit.
|
|
114
|
+
*
|
|
115
|
+
* Narrow race worth knowing: a user save of the same path landing between the caller's write and this
|
|
116
|
+
* call is baselined too, and therefore lost. It is sub-millisecond and no wider than the window the
|
|
117
|
+
* event-based watcher already had.
|
|
118
|
+
*/
|
|
119
|
+
async absorb(paths: string[]): Promise<void> {
|
|
120
|
+
await this.#serialize(async () => {
|
|
121
|
+
if (!this.#primed) return;
|
|
122
|
+
await Promise.all(
|
|
123
|
+
paths.map(async (file) => {
|
|
124
|
+
const abs = path.resolve(file);
|
|
125
|
+
const stats = await stat(abs).catch(() => null);
|
|
126
|
+
if (!stats?.isFile()) {
|
|
127
|
+
this.#files.delete(abs);
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
130
|
+
this.#files.set(abs, { mtimeMs: stats.mtimeMs, size: stats.size });
|
|
131
|
+
}),
|
|
132
|
+
);
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
async #collectFileChanges(changed: Set<string>): Promise<void> {
|
|
137
|
+
await Promise.all(
|
|
138
|
+
[...this.#files.keys()].map(async (abs) => {
|
|
139
|
+
const { stats, err } = await SourceMtimeIndex.#statPath(abs);
|
|
140
|
+
if (!stats) {
|
|
141
|
+
// Unreadable is not deleted. Dropping it here would both invent a change nobody made and stop
|
|
142
|
+
// tracking the file, so a later real edit would go unreported.
|
|
143
|
+
if (err && !SourceMtimeIndex.#isMissing(err)) {
|
|
144
|
+
this.#unreadable.set(abs, err.code ?? "EUNKNOWN");
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
this.#files.delete(abs);
|
|
148
|
+
this.#unreadable.delete(abs);
|
|
149
|
+
changed.add(abs);
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
this.#unreadable.delete(abs);
|
|
153
|
+
if (!stats.isFile()) {
|
|
154
|
+
this.#files.delete(abs);
|
|
155
|
+
changed.add(abs);
|
|
156
|
+
return;
|
|
157
|
+
}
|
|
158
|
+
const known = this.#files.get(abs);
|
|
159
|
+
if (known && known.mtimeMs === stats.mtimeMs && known.size === stats.size) return;
|
|
160
|
+
this.#files.set(abs, { mtimeMs: stats.mtimeMs, size: stats.size });
|
|
161
|
+
changed.add(abs);
|
|
162
|
+
}),
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* A directory's own mtime moves when an entry is added or removed but not when a tracked file's
|
|
168
|
+
* content changes, so this finds creations and deletions the file pass cannot see.
|
|
169
|
+
*/
|
|
170
|
+
async #collectDirChanges(changed: Set<string>): Promise<void> {
|
|
171
|
+
const moved: string[] = [];
|
|
172
|
+
const gone: string[] = [];
|
|
173
|
+
await Promise.all(
|
|
174
|
+
[...this.#dirs.entries()].map(async ([dir, mtimeMs]) => {
|
|
175
|
+
const { stats, err } = await SourceMtimeIndex.#statPath(dir);
|
|
176
|
+
if (!stats?.isDirectory()) {
|
|
177
|
+
if (err && !SourceMtimeIndex.#isMissing(err)) {
|
|
178
|
+
this.#markUnreadable(dir, err);
|
|
179
|
+
return;
|
|
180
|
+
}
|
|
181
|
+
gone.push(dir);
|
|
182
|
+
return;
|
|
183
|
+
}
|
|
184
|
+
// `UNREADABLE` is NaN, so a directory retained from a failed read always mismatches and is
|
|
185
|
+
// rewalked here — that retry is what lets a transient failure recover on its own.
|
|
186
|
+
if (stats.mtimeMs !== mtimeMs) moved.push(dir);
|
|
187
|
+
}),
|
|
188
|
+
);
|
|
189
|
+
for (const dir of gone) this.#forget(dir);
|
|
190
|
+
// Sequential: a moved directory can reveal a new subtree, and walking those in order keeps the
|
|
191
|
+
// number of concurrent `readdir` calls proportional to the change rather than to the tree.
|
|
192
|
+
for (const dir of moved) await this.#walk(dir, changed);
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* Re-read one directory, tracking entries that are new or changed. Recurses only into subdirectories
|
|
197
|
+
* that are new to the index; known ones are covered by their own mtime check in `#collectDirChanges`.
|
|
198
|
+
*/
|
|
199
|
+
async #walk(dir: string, changed: Set<string> | null): Promise<void> {
|
|
200
|
+
const [listing, dirStat] = await Promise.all([SourceMtimeIndex.#readdirPath(dir), SourceMtimeIndex.#statPath(dir)]);
|
|
201
|
+
const entries = listing.entries;
|
|
202
|
+
const dirStats = dirStat.stats;
|
|
203
|
+
if (!entries || !dirStats?.isDirectory()) {
|
|
204
|
+
// A directory that cannot be read is not a directory that is gone. Forgetting it here is
|
|
205
|
+
// unrecoverable: nothing re-stats a dir the index has dropped, and the parent's own mtime does not
|
|
206
|
+
// move when a child merely becomes unreadable, so the whole subtree stays invisible for the life of
|
|
207
|
+
// the process. Measured: one `EACCES` at prime silently hid 2 of 3 files, permanently.
|
|
208
|
+
const err = listing.err ?? dirStat.err;
|
|
209
|
+
if (err && !SourceMtimeIndex.#isMissing(err)) {
|
|
210
|
+
this.#markUnreadable(dir, err);
|
|
211
|
+
return;
|
|
212
|
+
}
|
|
213
|
+
this.#forget(dir);
|
|
214
|
+
return;
|
|
215
|
+
}
|
|
216
|
+
const known = this.#dirs.has(dir);
|
|
217
|
+
this.#dirs.set(dir, dirStats.mtimeMs);
|
|
218
|
+
this.#unreadable.delete(dir);
|
|
219
|
+
const descend: string[] = [];
|
|
220
|
+
const present = new Set<string>();
|
|
221
|
+
let blind = false;
|
|
222
|
+
for (const entry of entries) {
|
|
223
|
+
if (SourceMtimeIndex.#skipDirent(entry.name)) continue;
|
|
224
|
+
const abs = path.join(dir, entry.name);
|
|
225
|
+
if (entry.isDirectory()) {
|
|
226
|
+
descend.push(abs);
|
|
227
|
+
continue;
|
|
228
|
+
}
|
|
229
|
+
if (!entry.isFile()) continue;
|
|
230
|
+
if (this.#classifier.classify(abs) === "ignore") continue;
|
|
231
|
+
present.add(abs);
|
|
232
|
+
const { stats, err } = await SourceMtimeIndex.#statPath(abs);
|
|
233
|
+
if (!stats?.isFile()) {
|
|
234
|
+
// A file listed but not stat-able has no baseline, so an edit to it would compare against nothing.
|
|
235
|
+
// Leaving the directory stale is what brings it back; the file pass cannot, since it only revisits
|
|
236
|
+
// paths already tracked.
|
|
237
|
+
if (err && !SourceMtimeIndex.#isMissing(err)) {
|
|
238
|
+
this.#unreadable.set(abs, err.code ?? "EUNKNOWN");
|
|
239
|
+
blind = true;
|
|
240
|
+
}
|
|
241
|
+
continue;
|
|
242
|
+
}
|
|
243
|
+
this.#unreadable.delete(abs);
|
|
244
|
+
const previous = this.#files.get(abs);
|
|
245
|
+
if (previous && previous.mtimeMs === stats.mtimeMs && previous.size === stats.size) continue;
|
|
246
|
+
this.#files.set(abs, { mtimeMs: stats.mtimeMs, size: stats.size });
|
|
247
|
+
changed?.add(abs);
|
|
248
|
+
}
|
|
249
|
+
// Entries this directory used to hold and no longer does. Only worth scanning for a directory the
|
|
250
|
+
// index already knew, which during `prime` is none of them.
|
|
251
|
+
if (known) {
|
|
252
|
+
for (const abs of [...this.#files.keys()]) {
|
|
253
|
+
if (present.has(abs) || path.dirname(abs) !== dir) continue;
|
|
254
|
+
this.#files.delete(abs);
|
|
255
|
+
changed?.add(abs);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
if (blind) this.#dirs.set(dir, UNREADABLE);
|
|
259
|
+
// Known subdirectories carry their own mtime check, so only unseen ones need walking. Concurrent
|
|
260
|
+
// because `prime` reaches every directory through here.
|
|
261
|
+
await Promise.all(descend.filter((sub) => !this.#dirs.has(sub)).map((sub) => this.#walk(sub, changed)));
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
/**
|
|
265
|
+
* Keep a directory the index could not read, flagged for retry on every scan, and keep whatever it
|
|
266
|
+
* already knows underneath it — those files stay tracked so a real edit is still caught the moment
|
|
267
|
+
* access returns.
|
|
268
|
+
*/
|
|
269
|
+
#markUnreadable(dir: string, err: NodeJS.ErrnoException): void {
|
|
270
|
+
this.#dirs.set(dir, UNREADABLE);
|
|
271
|
+
this.#unreadable.set(dir, err.code ?? "EUNKNOWN");
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/** Drop a directory and everything the index holds beneath it. */
|
|
275
|
+
#forget(dir: string): void {
|
|
276
|
+
const prefix = `${dir}${path.sep}`;
|
|
277
|
+
this.#dirs.delete(dir);
|
|
278
|
+
this.#unreadable.delete(dir);
|
|
279
|
+
for (const known of [...this.#dirs.keys()]) if (known.startsWith(prefix)) this.#dirs.delete(known);
|
|
280
|
+
// Otherwise a gap under a deleted directory is reported forever, since nothing revisits it to clear.
|
|
281
|
+
for (const known of [...this.#unreadable.keys()]) if (known.startsWith(prefix)) this.#unreadable.delete(known);
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/** `null` stats with the errno kept, so callers can tell "not there" from "could not look". */
|
|
285
|
+
static async #statPath(abs: string): Promise<{ stats: Stats | null; err: NodeJS.ErrnoException | null }> {
|
|
286
|
+
try {
|
|
287
|
+
return { stats: await stat(abs), err: null };
|
|
288
|
+
} catch (err) {
|
|
289
|
+
return { stats: null, err: err as NodeJS.ErrnoException };
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
static async #readdirPath(dir: string): Promise<{ entries: Dirent[] | null; err: NodeJS.ErrnoException | null }> {
|
|
294
|
+
try {
|
|
295
|
+
return { entries: await readdir(dir, { withFileTypes: true }), err: null };
|
|
296
|
+
} catch (err) {
|
|
297
|
+
return { entries: null, err: err as NodeJS.ErrnoException };
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
/**
|
|
302
|
+
* Whether an errno means the path genuinely is not there, which is the only case where dropping it from
|
|
303
|
+
* the baseline is right. Everything else — `EACCES`, `EIO`, `EMFILE`, a stalled network mount — means the
|
|
304
|
+
* index failed to look, and treating that as a deletion both invents a change and blinds it to real ones.
|
|
305
|
+
*/
|
|
306
|
+
static #isMissing(err: NodeJS.ErrnoException): boolean {
|
|
307
|
+
return err.code === "ENOENT" || err.code === "ENOTDIR";
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/**
|
|
311
|
+
* Mirrors `HmrChangeClassifier`'s path rules for directory names — dotted names cover `.git` and
|
|
312
|
+
* `.akan`, which is where every build artifact lands. Symlinked entries are neither `isFile` nor
|
|
313
|
+
* `isDirectory` and so are skipped, matching `fs.watch`, which does not follow them either.
|
|
314
|
+
*/
|
|
315
|
+
static #skipDirent(name: string): boolean {
|
|
316
|
+
return !name || name.startsWith(".") || name === "node_modules";
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
/** `WatchRootResolver` can return `apps/<app>/page` alongside `apps/`; walking both doubles the work. */
|
|
320
|
+
static #pruneNestedRoots(roots: string[]): string[] {
|
|
321
|
+
const resolved = [...new Set(roots.map((root) => path.resolve(root)))].sort();
|
|
322
|
+
return resolved.filter(
|
|
323
|
+
(root) => !resolved.some((other) => other !== root && root.startsWith(`${other}${path.sep}`)),
|
|
324
|
+
);
|
|
325
|
+
}
|
|
326
|
+
}
|
|
@@ -4,7 +4,13 @@ import type { App } from "@akanjs/devkit/commandDecorators";
|
|
|
4
4
|
// eager import is paid on every save. Measured on a 177-route app: `executors` 24ms, `frontendBuild`
|
|
5
5
|
// ~110ms, app config 5ms.
|
|
6
6
|
import { AppExecutor, WorkspaceExecutor } from "@akanjs/devkit/executors";
|
|
7
|
-
import {
|
|
7
|
+
import {
|
|
8
|
+
CsrArtifactBuilder,
|
|
9
|
+
CssCompiler,
|
|
10
|
+
FontOptimizer,
|
|
11
|
+
PagesBundleBuilder,
|
|
12
|
+
SsrBaseArtifactBuilder,
|
|
13
|
+
} from "@akanjs/devkit/frontendBuild";
|
|
8
14
|
import { Logger } from "akanjs/common";
|
|
9
15
|
import type { BuilderMessage, BuildPhase } from "akanjs/server";
|
|
10
16
|
import type { BuildBatchRequest, BuildBatchResult, OptimizedFonts, PagesBatchCssAssets } from "./buildBatchProtocol";
|
|
@@ -34,6 +40,8 @@ class BuildBatch {
|
|
|
34
40
|
}
|
|
35
41
|
|
|
36
42
|
async run(): Promise<BuildBatchResult> {
|
|
43
|
+
// `base` arrives alone, from a builder that cannot serve anything until it finishes.
|
|
44
|
+
if (this.#request.needs.includes("base")) await this.#buildBase();
|
|
37
45
|
// Ordered the way the watcher used to run them: csr before pages so a csr failure cannot delay the
|
|
38
46
|
// pages bundle the browser is waiting on, and css last because it depends on the rebuilt client.
|
|
39
47
|
if (this.#request.needs.includes("csr")) await this.#buildCsr();
|
|
@@ -48,6 +56,12 @@ class BuildBatch {
|
|
|
48
56
|
* never had, and it would move every artifact write into the window right before the watcher reports
|
|
49
57
|
* the generation complete — which is where a save issued immediately afterwards gets dropped by Bun's
|
|
50
58
|
* recursive `fs.watch` (`local/optimize-resource/06-watcher-dropped-event.md`).
|
|
59
|
+
*
|
|
60
|
+
* A bare `process.send` is safe here, unlike in the watcher, for one reason: this process ends by
|
|
61
|
+
* returning from `main`, and a natural exit flushes a pending ipc write (measured: 1MB delivered
|
|
62
|
+
* 20/20). It is `process.exit` that discards one — so adding an explicit exit to this file, at the end
|
|
63
|
+
* of `main` or anywhere after an emit, would silently start dropping `css-updated` payloads. Route
|
|
64
|
+
* sends through `BuilderChannel` if that ever becomes necessary.
|
|
51
65
|
*/
|
|
52
66
|
#emit(message: BuilderMessage): void {
|
|
53
67
|
process.send?.(message);
|
|
@@ -66,6 +80,25 @@ class BuildBatch {
|
|
|
66
80
|
});
|
|
67
81
|
}
|
|
68
82
|
|
|
83
|
+
/**
|
|
84
|
+
* The boot build. Streams nothing: a builder is not serving yet, so there is no phase board to update
|
|
85
|
+
* and no browser to reload — the watcher learns the outcome from the batch result, and a failure there
|
|
86
|
+
* is what puts it into degraded watch mode.
|
|
87
|
+
*/
|
|
88
|
+
async #buildBase(): Promise<void> {
|
|
89
|
+
const started = Date.now();
|
|
90
|
+
try {
|
|
91
|
+
const { artifact, optimizedFonts } = await new SsrBaseArtifactBuilder(this.#app).build();
|
|
92
|
+
this.#result.artifact = artifact;
|
|
93
|
+
this.#result.optimizedFonts = optimizedFonts;
|
|
94
|
+
this.#logger.verbose(`base-artifact ok buildId=${artifact.pagesBundleBuildId} (${Date.now() - started}ms)`);
|
|
95
|
+
} catch (err) {
|
|
96
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
97
|
+
this.#logger.error(`base-artifact failed: ${message}`);
|
|
98
|
+
this.#result.errors.base = message;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
69
102
|
async #buildCsr(): Promise<void> {
|
|
70
103
|
const started = Date.now();
|
|
71
104
|
try {
|
|
@@ -1,9 +1,15 @@
|
|
|
1
1
|
import type { FontOptimizer } from "@akanjs/devkit/frontendBuild";
|
|
2
|
-
import type { CssPayload, PagesBundlePayload } from "akanjs/server";
|
|
2
|
+
import type { BaseBuildArtifact, CssPayload, PagesBundlePayload } from "akanjs/server";
|
|
3
3
|
|
|
4
4
|
export type OptimizedFonts = Awaited<ReturnType<FontOptimizer["optimize"]>>;
|
|
5
5
|
|
|
6
|
-
|
|
6
|
+
/**
|
|
7
|
+
* `base` is the boot build (`SsrBaseArtifactBuilder`) and is never batched with the others: it is what a
|
|
8
|
+
* builder runs before it can serve anything, where the rest are per-save work. It earns its place here
|
|
9
|
+
* for the same reason as the others — measured on `apps/akan`, it retains **+1143 MB** of bundler arena
|
|
10
|
+
* that only a process exit returns, which was 65 % of the builder's whole idle footprint.
|
|
11
|
+
*/
|
|
12
|
+
export type BuildBatchNeed = "base" | "pages" | "css" | "csr";
|
|
7
13
|
|
|
8
14
|
/**
|
|
9
15
|
* One generation of build work, handed to a process that exits when it is done.
|
|
@@ -44,6 +50,11 @@ export interface BuildBatchResult {
|
|
|
44
50
|
generation: number;
|
|
45
51
|
cssAssets?: PagesBatchCssAssets;
|
|
46
52
|
optimizedFonts?: OptimizedFonts;
|
|
53
|
+
/**
|
|
54
|
+
* Present only for a `base` batch. The same object the worker wrote to `base-artifact.json`, returned
|
|
55
|
+
* here so the watcher does not have to read its own boot build back off disk.
|
|
56
|
+
*/
|
|
57
|
+
artifact?: BaseBuildArtifact;
|
|
47
58
|
errors: Partial<Record<BuildBatchNeed, string>>;
|
|
48
59
|
/** The worker died before reporting, so it streamed no `build-status` of its own for these needs. */
|
|
49
60
|
crashed?: boolean;
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import { afterEach, describe, expect, test } from "bun:test";
|
|
2
|
+
import { mkdtemp, rm } from "node:fs/promises";
|
|
3
|
+
import os from "node:os";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { BuilderChannel } from "./builderChannel";
|
|
6
|
+
|
|
7
|
+
const tempDirs: string[] = [];
|
|
8
|
+
|
|
9
|
+
afterEach(async () => {
|
|
10
|
+
for (const dir of tempDirs.splice(0)) await rm(dir, { recursive: true, force: true });
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
type SendMode = "drain" | "await" | "bare";
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Run a child that sends what the builder sends and then exits the way the recycle drain does, and
|
|
17
|
+
* report what the parent actually received.
|
|
18
|
+
*
|
|
19
|
+
* `payload` picks the message shape, because shape decides whether a bare send survives: `manifest` is a
|
|
20
|
+
* map of many short strings (a `build-route-res` delta), `css` is one long string (a base64 stylesheet).
|
|
21
|
+
*/
|
|
22
|
+
const sendThenExit = async (
|
|
23
|
+
bytes: number,
|
|
24
|
+
{ mode, payload }: { mode: SendMode; payload: "manifest" | "css" },
|
|
25
|
+
): Promise<Array<{ type?: string }>> => {
|
|
26
|
+
const dir = await mkdtemp(path.join(os.tmpdir(), "builder-channel-"));
|
|
27
|
+
tempDirs.push(dir);
|
|
28
|
+
const entry = path.join(dir, "child.ts");
|
|
29
|
+
const modulePath = JSON.stringify(path.join(import.meta.dir, "builderChannel"));
|
|
30
|
+
const build =
|
|
31
|
+
payload === "manifest"
|
|
32
|
+
? [
|
|
33
|
+
'const chunk = "x".repeat(200);',
|
|
34
|
+
"const moduleMap: Record<string, string> = {};",
|
|
35
|
+
`for (let i = 0; i * 210 < ${bytes}; i++) moduleMap["chunk-" + i] = chunk;`,
|
|
36
|
+
'const msg = { type: "build-route-res", id: 7, ok: true, data: { ssrManifestDelta: moduleMap } };',
|
|
37
|
+
]
|
|
38
|
+
: [
|
|
39
|
+
`const css = "y".repeat(${bytes});`,
|
|
40
|
+
'const msg = { type: "css-updated", data: { cssAssets: {}, cssBase64ByUrl: { "/a.css": css } } };',
|
|
41
|
+
];
|
|
42
|
+
const sendLine = {
|
|
43
|
+
drain: "BuilderChannel.emit(msg as never);\nawait BuilderChannel.drain();",
|
|
44
|
+
await: "await BuilderChannel.send(msg as never);",
|
|
45
|
+
bare: "process.send?.(msg);",
|
|
46
|
+
}[mode];
|
|
47
|
+
await Bun.write(
|
|
48
|
+
entry,
|
|
49
|
+
[`import { BuilderChannel } from ${modulePath};`, ...build, sendLine, "process.exit(0);"].join("\n"),
|
|
50
|
+
);
|
|
51
|
+
const received: Array<{ type?: string }> = [];
|
|
52
|
+
const proc = Bun.spawn(["bun", entry], {
|
|
53
|
+
stdio: ["ignore", "inherit", "inherit"],
|
|
54
|
+
// The mode the dev host actually spawns the builder with; it changes where the loss cliff sits.
|
|
55
|
+
serialization: "advanced",
|
|
56
|
+
ipc: (message) => {
|
|
57
|
+
received.push(message as { type?: string });
|
|
58
|
+
},
|
|
59
|
+
});
|
|
60
|
+
await proc.exited;
|
|
61
|
+
// Messages land before the exit callback, never after, but leave room for a straggler to prove it.
|
|
62
|
+
await Bun.sleep(50);
|
|
63
|
+
return received;
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
describe("BuilderChannel", () => {
|
|
67
|
+
test("delivers a reply too large for the pipe buffer before the process exits", async () => {
|
|
68
|
+
expect(await sendThenExit(1_000_000, { mode: "await", payload: "manifest" })).toMatchObject([{ id: 7 }]);
|
|
69
|
+
// A manifest delta is usually well past the buffer, but the small case must keep working too.
|
|
70
|
+
expect(await sendThenExit(0, { mode: "await", payload: "manifest" })).toMatchObject([{ id: 7 }]);
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
test("a drained event survives the exit even though nobody awaited it", async () => {
|
|
74
|
+
// The event path: `css-updated` is relayed with no caller to await it, and the recycle drain exits
|
|
75
|
+
// milliseconds later. Only `drain()` stands between the two.
|
|
76
|
+
expect(await sendThenExit(200_000, { mode: "drain", payload: "css" })).toMatchObject([{ type: "css-updated" }]);
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
test("without the flush wait the same messages are lost, which is why this class exists", async () => {
|
|
80
|
+
// Controls, not requirements: if a future bun flushes ipc writes on exit, these fail and say so.
|
|
81
|
+
expect(await sendThenExit(1_000_000, { mode: "bare", payload: "manifest" })).toEqual([]);
|
|
82
|
+
// 20KB of css, far below the 64KB the manifest shape survives — one long string dies much earlier,
|
|
83
|
+
// so no size threshold would have been safe to special-case.
|
|
84
|
+
expect(await sendThenExit(20_000, { mode: "bare", payload: "css" })).toEqual([]);
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
test("drain resolves only once every tracked send has flushed", async () => {
|
|
88
|
+
const send = process.send;
|
|
89
|
+
const flushes: Array<() => void> = [];
|
|
90
|
+
try {
|
|
91
|
+
(process as { send?: unknown }).send = (_msg: unknown, _h: unknown, _o: unknown, cb: () => void) => {
|
|
92
|
+
flushes.push(cb);
|
|
93
|
+
return true;
|
|
94
|
+
};
|
|
95
|
+
BuilderChannel.emit({ type: "builder-ready" });
|
|
96
|
+
BuilderChannel.emit({ type: "builder-ready" });
|
|
97
|
+
let drained = false;
|
|
98
|
+
const draining = BuilderChannel.drain().then((count) => {
|
|
99
|
+
drained = true;
|
|
100
|
+
return count;
|
|
101
|
+
});
|
|
102
|
+
flushes[0]?.();
|
|
103
|
+
await Bun.sleep(1);
|
|
104
|
+
expect(drained).toBe(false);
|
|
105
|
+
flushes[1]?.();
|
|
106
|
+
expect(await draining).toBe(2);
|
|
107
|
+
} finally {
|
|
108
|
+
(process as { send?: unknown }).send = send;
|
|
109
|
+
}
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
test("a send started while draining is drained too", async () => {
|
|
113
|
+
const send = process.send;
|
|
114
|
+
const flushes: Array<() => void> = [];
|
|
115
|
+
try {
|
|
116
|
+
(process as { send?: unknown }).send = (_msg: unknown, _h: unknown, _o: unknown, cb: () => void) => {
|
|
117
|
+
flushes.push(cb);
|
|
118
|
+
return true;
|
|
119
|
+
};
|
|
120
|
+
BuilderChannel.emit({ type: "builder-ready" });
|
|
121
|
+
const draining = BuilderChannel.drain();
|
|
122
|
+
// What `#reportMetrics` does from a work item's `finally`, after the drain has begun.
|
|
123
|
+
BuilderChannel.emit({ type: "builder-metrics", data: { rssBytes: 1, generation: 1, workCount: 1 } });
|
|
124
|
+
for (let i = 0; i < 4 && flushes.length; i++) {
|
|
125
|
+
flushes.shift()?.();
|
|
126
|
+
await Bun.sleep(1);
|
|
127
|
+
}
|
|
128
|
+
expect(await draining).toBe(2);
|
|
129
|
+
} finally {
|
|
130
|
+
(process as { send?: unknown }).send = send;
|
|
131
|
+
}
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
test("resolves instead of hanging when there is no ipc channel", async () => {
|
|
135
|
+
const send = process.send;
|
|
136
|
+
try {
|
|
137
|
+
(process as { send?: typeof process.send }).send = undefined;
|
|
138
|
+
await BuilderChannel.send({ type: "build-csr-res", id: 1, ok: true });
|
|
139
|
+
expect(await BuilderChannel.drain()).toBe(0);
|
|
140
|
+
} finally {
|
|
141
|
+
(process as { send?: typeof process.send }).send = send;
|
|
142
|
+
}
|
|
143
|
+
});
|
|
144
|
+
});
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import type { BuilderMessage } from "akanjs/server";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Every message the builder sends its host, and the record of which of those writes have actually left
|
|
5
|
+
* the process.
|
|
6
|
+
*
|
|
7
|
+
* `process.exit` discards an ipc write that has not flushed, and this process exits routinely — the
|
|
8
|
+
* recycle that keeps its RSS under the ceiling ends in `process.exit(0)`. Whether a given message
|
|
9
|
+
* survives that depends on its *shape*, not only its size. Measured on bun 1.3.14 (darwin,
|
|
10
|
+
* `serialization: "advanced"`), send-then-exit-immediately, 20 rounds each:
|
|
11
|
+
*
|
|
12
|
+
* - one long string: 8KB delivered 20/20, 16KB delivered **0/20**
|
|
13
|
+
* - many short strings: 60KB delivered 20/20, 100KB delivered **0/20**
|
|
14
|
+
*
|
|
15
|
+
* `css-updated` is the first shape (a base64 stylesheet per url) and `build-route-res` the second (a
|
|
16
|
+
* manifest delta), so neither sits safely under a threshold worth relying on. Nothing here assumes one:
|
|
17
|
+
* every send reports when it flushed, and `drain` is what `shutdown` awaits before exiting.
|
|
18
|
+
*
|
|
19
|
+
* A natural exit does *not* have this problem — a process that simply returns from `main` flushes 1MB
|
|
20
|
+
* 20/20 — which is why the disposable build worker needs none of this. Only an explicit `process.exit`
|
|
21
|
+
* cuts a write short.
|
|
22
|
+
*/
|
|
23
|
+
export class BuilderChannel {
|
|
24
|
+
/**
|
|
25
|
+
* Bound so a runtime that ever stops invoking the flush callback costs one late message instead of
|
|
26
|
+
* wedging the shutdown drain until the host's kill watchdog fires.
|
|
27
|
+
*/
|
|
28
|
+
static readonly #flushTimeoutMs = 5_000;
|
|
29
|
+
/** Sends that have not flushed yet; `drain` only ever needs to await them all. */
|
|
30
|
+
static readonly #flushing = new Set<Promise<void>>();
|
|
31
|
+
|
|
32
|
+
/** Send, and resolve once the write has left this process. */
|
|
33
|
+
static send(message: BuilderMessage): Promise<void> {
|
|
34
|
+
const send = process.send;
|
|
35
|
+
if (!send) return Promise.resolve();
|
|
36
|
+
const flushed = new Promise<void>((resolve) => {
|
|
37
|
+
const timer = setTimeout(resolve, BuilderChannel.#flushTimeoutMs);
|
|
38
|
+
// A failed send has nothing to retry: the host answers an unanswered request id itself when this
|
|
39
|
+
// process exits, and a lost event is superseded by the next generation's.
|
|
40
|
+
send.call(process, message, undefined, undefined, () => {
|
|
41
|
+
clearTimeout(timer);
|
|
42
|
+
resolve();
|
|
43
|
+
});
|
|
44
|
+
});
|
|
45
|
+
BuilderChannel.#flushing.add(flushed);
|
|
46
|
+
void flushed.then(() => BuilderChannel.#flushing.delete(flushed));
|
|
47
|
+
return flushed;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Send an event no caller awaits. Tracked exactly like `send`, which is the whole point: a relayed
|
|
52
|
+
* `css-updated` has nobody to await it, so `drain` is the only thing standing between it and the exit.
|
|
53
|
+
*/
|
|
54
|
+
static emit(message: BuilderMessage): void {
|
|
55
|
+
void BuilderChannel.send(message);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Resolve once every send handed over so far has flushed, and report how many were still in flight.
|
|
60
|
+
* Sends started *while* draining are covered too — `#reportMetrics` can fire from a work item's
|
|
61
|
+
* `finally`, after the drain has already begun.
|
|
62
|
+
*/
|
|
63
|
+
static async drain(): Promise<number> {
|
|
64
|
+
let flushed = 0;
|
|
65
|
+
while (BuilderChannel.#flushing.size) {
|
|
66
|
+
const pending = [...BuilderChannel.#flushing];
|
|
67
|
+
flushed += pending.length;
|
|
68
|
+
await Promise.all(pending);
|
|
69
|
+
}
|
|
70
|
+
return flushed;
|
|
71
|
+
}
|
|
72
|
+
}
|