@wildshard/dev 0.0.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/wildshard.mjs +75 -0
- package/lint/host-callbacks.d.mts +6 -0
- package/lint/host-callbacks.mjs +123 -0
- package/lint/ratchet.json +3 -0
- package/package.json +95 -4
- package/scripts/audit-assets.d.mts +10 -0
- package/scripts/audit-assets.mjs +151 -0
- package/scripts/bake/coatFiles.d.mts +6 -0
- package/scripts/bake/coatFiles.mjs +37 -0
- package/scripts/bake/navmeshGround.ts +22 -0
- package/scripts/bake/near-placeholder.mjs +331 -0
- package/scripts/bake/viewmodelBytes.ts +26 -0
- package/scripts/bake/viewmodelPage.ts +11 -0
- package/scripts/bake-catalogue.mjs +44 -0
- package/scripts/bake-check.mjs +54 -0
- package/scripts/bake-chunk.mjs +124 -0
- package/scripts/bake-cloud-field.mjs +24 -0
- package/scripts/bake-coats.mjs +110 -0
- package/scripts/bake-data-maps.d.mts +24 -0
- package/scripts/bake-data-maps.mjs +78 -0
- package/scripts/bake-entry-programs.mjs +126 -0
- package/scripts/bake-geometry.mjs +106 -0
- package/scripts/bake-ktx2.mjs +477 -0
- package/scripts/bake-map-glb.mjs +127 -0
- package/scripts/bake-map-rest.d.mts +7 -0
- package/scripts/bake-map-rest.mjs +50 -0
- package/scripts/bake-maps.mjs +200 -0
- package/scripts/bake-navmesh.mjs +259 -0
- package/scripts/bake-packs.mjs +129 -0
- package/scripts/bake-pickups.mjs +102 -0
- package/scripts/bake-query-terrain.mjs +64 -0
- package/scripts/bake-textures.mjs +80 -0
- package/scripts/bake-viewmodel-sets.mjs +137 -0
- package/scripts/bake-voxel-ao.mjs +119 -0
- package/scripts/bake-water-ripples.mjs +25 -0
- package/scripts/browser-imports.d.mts +2 -0
- package/scripts/browser-imports.mjs +40 -0
- package/scripts/browser-machine-loader-proof.mjs +39 -0
- package/scripts/browser-machine-loader.d.mts +8 -0
- package/scripts/browser-machine-loader.mjs +87 -0
- package/scripts/browser-playwright.d.mts +10 -0
- package/scripts/browser-playwright.mjs +5 -0
- package/scripts/browser-test.d.mts +4 -0
- package/scripts/browser-test.mjs +18 -0
- package/scripts/build-sdk.mjs +95 -0
- package/scripts/build-shardfiles.d.mts +7 -0
- package/scripts/build-shardfiles.mjs +230 -0
- package/scripts/capture-hero.mjs +211 -0
- package/scripts/chromium-keychain.d.mts +1 -0
- package/scripts/chromium-keychain.mjs +7 -0
- package/scripts/entry-reads/crawl.ts +152 -0
- package/scripts/gen-host-catalogue.d.mts +14 -0
- package/scripts/gen-host-catalogue.mjs +108 -0
- package/scripts/gen-shards.d.mts +17 -0
- package/scripts/gen-shards.mjs +311 -0
- package/scripts/goldens/plot-inputs.d.mts +12 -0
- package/scripts/goldens/plot-inputs.mjs +46 -0
- package/scripts/gpu-texmem.mjs +98 -0
- package/scripts/ktx2-ab.mjs +111 -0
- package/scripts/map-bake-frames.d.mts +4 -0
- package/scripts/map-bake-frames.mjs +23 -0
- package/scripts/map-world.mjs +162 -0
- package/scripts/memory-product.mjs +62 -0
- package/scripts/music/export-midi.mjs +69 -0
- package/scripts/named-blob-output.d.mts +8 -0
- package/scripts/named-blob-output.mjs +48 -0
- package/scripts/practice/verify_unimate_skin.mjs +356 -0
- package/scripts/preparation-browser.mjs +25 -0
- package/scripts/preparation-seed.mjs +15 -0
- package/scripts/prepare-product.d.mts +14 -0
- package/scripts/prepare-product.mjs +133 -0
- package/scripts/prove-walk.mjs +83 -0
- package/scripts/push-tables.d.mts +1 -0
- package/scripts/push-tables.mjs +28 -0
- package/scripts/record-shard-mounts.mjs +133 -0
- package/scripts/section-inventory.ts +209 -0
- package/scripts/shard-build-pool.d.mts +4 -0
- package/scripts/shard-build-pool.mjs +53 -0
- package/scripts/shard-milestones.d.mts +15 -0
- package/scripts/shard-milestones.mjs +105 -0
- package/scripts/shard-progress.mjs +212 -0
- package/scripts/shardfile-cold-boot.mjs +79 -0
- package/scripts/shardfile-rolldown.d.mts +2 -0
- package/scripts/shardfile-rolldown.mjs +32 -0
- package/scripts/shardfile-vite.d.mts +4 -0
- package/scripts/shardfile-vite.mjs +20 -0
- package/scripts/site-facts.mjs +88 -0
- package/scripts/test-template-gate.mjs +44 -0
- package/scripts/tex-tiers.mjs +240 -0
- package/scripts/unused-assets.mjs +173 -0
- package/scripts/wildshard.d.mts +4 -0
- package/scripts/wildshard.mjs +146 -0
- package/scripts/witness-generation.d.mts +10 -0
- package/scripts/witness-generation.mjs +45 -0
- package/src/.oxlintrc.json +12 -0
- package/src/cli/bin.mjs +8 -0
- package/src/cli/content-command.ts +44 -0
- package/src/cli/content.ts +287 -0
- package/src/cli/grid/command.ts +48 -0
- package/src/cli/grid/grid.ts +155 -0
- package/src/cli/help.ts +262 -0
- package/src/cli/hero/adopt.ts +109 -0
- package/src/cli/install-home.ts +40 -0
- package/src/cli/main.ts +142 -0
- package/src/cli/version.ts +63 -0
- package/src/server/app.ts +19 -0
- package/src/server/serve.ts +49 -0
- package/src/sharddev/bake/json.mjs +84 -0
- package/src/sharddev/bake/mapRest.ts +57 -0
- package/src/sharddev/bake/nativeRaster.ts +106 -0
- package/src/sharddev/bake/source-loader.mjs +55 -0
- package/src/sharddev/bake/splatLayers.ts +29 -0
- package/src/sharddev/bootInventory.ts +132 -0
- package/src/sharddev/check.ts +125 -0
- package/src/sharddev/checkProject.ts +134 -0
- package/src/sharddev/cli.ts +207 -0
- package/src/sharddev/dev.ts +150 -0
- package/src/sharddev/devClient.ts +120 -0
- package/src/sharddev/devRoot.ts +6 -0
- package/src/sharddev/engineMusic.ts +23 -0
- package/src/sharddev/projectEntryProof.ts +32 -0
- package/src/sharddev/test/diskScriptScenario.ts +88 -0
- package/src/sharddev/test/scriptScenario.ts +254 -0
- package/src/sharddev/tsconfig.json +3 -0
- package/src/sharddev/watch/page.ts +288 -0
- package/src/sharddev/watch/pageEntry.ts +44 -0
- package/src/sharddev/watch/parity-baseline.json +22 -0
- package/src/sharddev/watch/parity.mjs +194 -0
- package/src/sharddev/watch/proof.mjs +199 -0
- package/src/sharddev/watch/socket.ts +57 -0
- package/src/sharddev/watch/watch.ts +382 -0
- package/src/shared/health.ts +10 -0
- package/src/tools/build-web.ts +59 -0
- package/src/web/app.tsx +53 -0
- package/src/web/index.html +12 -0
- package/src/web/main.tsx +10 -0
- package/tsconfig.author.json +25 -0
- package/tsconfig.json +47 -0
- package/README.md +0 -3
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
// Phone-tier image files (project/archive/2026-09-22-load-perf.md §P1, ask P5 "cold bytes"). Idempotent; needs ImageMagick + cwebp.
|
|
2
|
+
// Commit the outputs: Vercel's builder has neither tool, so nothing here runs at build time.
|
|
3
|
+
//
|
|
4
|
+
// 1. Poly Haven sets: for every public/assets/tex/<id>/{diffuse,nor_gl,arm}.jpg wider than 1024 px or heavier than
|
|
5
|
+
// 350 KB, write <kind>_1k.jpg beside it (≤ 1024², q82). The phone tier requests these (src/client/engine/core/assets.ts texUrl)
|
|
6
|
+
// instead of downloading a 2048² file it would shrink anyway.
|
|
7
|
+
// 2. Model textures: Poly Haven ships the glTF props' 1k JPEGs at q99 — three times the bytes of q85 for no
|
|
8
|
+
// difference anyone can see at 1k. Re-encoded IN PLACE (both tiers) at q85, normal maps with 4:4:4 chroma (their
|
|
9
|
+
// R and G are independent data, 4:2:0 would smear them). Same for the JPEGs embedded in the `<id>_lod.glb`
|
|
10
|
+
// props (scripts/simplify-models.mjs copies them in at q99). A file already ≤ q90 is left alone.
|
|
11
|
+
// 3. `.phone.webp` copies, picked by `tierUrl()` (src/client/engine/boot/bytes.ts) on the phone tier — for fetchImage, and for
|
|
12
|
+
// three's loaders through DefaultLoadingManager's URL modifier (glTF textures). The boot manifest declares
|
|
13
|
+
// through the same function, so declared bytes = downloaded bytes. WebP q75 (-sharp_yuv) is ~20–25 % smaller
|
|
14
|
+
// than the q82 JPEG it replaces AND measures lower error against the source (forest_ground_04: 0.024 vs 0.032); PNG alpha stays lossless (-exact keeps the RGB under transparent
|
|
15
|
+
// texels, which bilinear filtering and the mips still read). Capped at 1024 px (the phone's maxTexture), AO /
|
|
16
|
+
// roughness / metalness planes of the Poly Haven sets and props at 512 px. Encoded from the best source there
|
|
17
|
+
// is (the original 2k map, not its q82 _1k copy). A copy that is not ≥ 15 % smaller is not kept.
|
|
18
|
+
// 4. `<id>_lod.phone.glb`: the LOD props with their embedded JPEGs as WebP (EXT_texture_webp, which three's
|
|
19
|
+
// GLTFLoader reads), ARM planes at 512 px.
|
|
20
|
+
// Content-addressed: encode only missing source/settings keys, verify cached output hashes with --check.
|
|
21
|
+
// --seed-cache trusts today's committed copies and writes only scripts/tex-tiers.cache.json.
|
|
22
|
+
import { readdirSync, existsSync, statSync, readFileSync, rmSync, mkdtempSync } from 'node:fs';
|
|
23
|
+
import { tmpdir } from 'node:os';
|
|
24
|
+
import { byteWriter, outputHash, jsonBytes } from '@wildshard/engine/scripts/bake-output.mjs';
|
|
25
|
+
import { execFileSync } from 'node:child_process';
|
|
26
|
+
import { join } from 'node:path';
|
|
27
|
+
import { textureTierJobs } from '@wildshard/engine/scripts/texture-inputs.mjs';
|
|
28
|
+
|
|
29
|
+
const ROOT = 'public/assets/tex';
|
|
30
|
+
let made = 0;
|
|
31
|
+
const identify = (fmt, file) => execFileSync('magick', ['identify', '-format', fmt, file]).toString().trim();
|
|
32
|
+
const check = process.argv.includes('--check');
|
|
33
|
+
const seed = process.argv.includes('--seed-cache');
|
|
34
|
+
if (check && seed) throw new Error('tex-tiers: choose --check or --seed-cache');
|
|
35
|
+
const output = byteWriter(check, 'tex-tiers');
|
|
36
|
+
const CACHE = 'scripts/tex-tiers.cache.json';
|
|
37
|
+
const cache = existsSync(CACHE) ? JSON.parse(readFileSync(CACHE, 'utf8')) : {};
|
|
38
|
+
const stale = new Set();
|
|
39
|
+
const want = new Set();
|
|
40
|
+
let seeded = 0, encoded = 0, reused = 0;
|
|
41
|
+
const keyOf = (bytes, settings) => outputHash(Buffer.concat([Buffer.from(JSON.stringify(settings)), bytes]));
|
|
42
|
+
|
|
43
|
+
/** Seeding reads only. Every cache hit verifies the committed output bytes; missing keys alone encode. */
|
|
44
|
+
function copy(src, out, settings, make, inPlace = false) {
|
|
45
|
+
const source = readFileSync(src), key = keyOf(source, settings), hit = cache[key];
|
|
46
|
+
if (seed) {
|
|
47
|
+
if (hit === undefined) { cache[key] = existsSync(out) ? outputHash(readFileSync(out)) : 'none'; seeded++; }
|
|
48
|
+
if (existsSync(out)) want.add(out);
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
if (hit !== undefined) {
|
|
52
|
+
if (hit === 'none') { if (existsSync(out)) stale.add(out); }
|
|
53
|
+
else if (!existsSync(out) || outputHash(readFileSync(out)) !== hit) stale.add(out);
|
|
54
|
+
else want.add(out);
|
|
55
|
+
reused++;
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
if (check) { stale.add(out); return; }
|
|
59
|
+
const bytes = make(source);
|
|
60
|
+
encoded++;
|
|
61
|
+
if (bytes === null) { cache[key] = 'none'; output.remove(out); }
|
|
62
|
+
else {
|
|
63
|
+
cache[key] = outputHash(bytes); output.put(out, bytes); want.add(out);
|
|
64
|
+
if (inPlace) cache[keyOf(bytes, settings)] = outputHash(bytes);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
const scratch = mkdtempSync(join(tmpdir(), 'tex-tiers-'));
|
|
68
|
+
let scratchId = 0;
|
|
69
|
+
process.on('exit', () => rmSync(scratch, { recursive: true, force: true }));
|
|
70
|
+
|
|
71
|
+
// ── 1. Poly Haven set _1k siblings ──
|
|
72
|
+
const textureFiles = [];
|
|
73
|
+
const textureWalk = (dir) => { for (const entry of readdirSync(dir, { withFileTypes: true })) { const path = join(dir, entry.name); if (entry.isDirectory()) textureWalk(path); else textureFiles.push(path); } };
|
|
74
|
+
textureWalk(ROOT);
|
|
75
|
+
const tierJobs = textureTierJobs(textureFiles);
|
|
76
|
+
for (const { source: src, output: out } of tierJobs.filter((job) => job.kind === 'resize')) {
|
|
77
|
+
copy(src, out, { format: 'jpg', max: 1024, quality: 82, strip: true, byteLimit: 350 * 1024 }, () => {
|
|
78
|
+
if (Number(identify('%w', src)) <= 1024 && statSync(src).size < 350 * 1024) return null;
|
|
79
|
+
made++;
|
|
80
|
+
return execFileSync('magick', [src, '-resize', '1024x1024', '-quality', '82', '-strip', 'jpg:-'], { maxBuffer: 64 << 20 });
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// ── 2. model textures, in place ──
|
|
85
|
+
const isNormal = (name) => /nor(_gl)?/i.test(name);
|
|
86
|
+
const jpegArgs = (name) => ['-quality', '85', ...(isNormal(name) ? ['-sampling-factor', '1x1'] : []), '-strip'];
|
|
87
|
+
/** q85 re-encode of a JPEG buffer, or null when it is already ≤ q90 (idempotence). */
|
|
88
|
+
function squeeze(buf, name) {
|
|
89
|
+
const q = Number(execFileSync('magick', ['identify', '-format', '%Q', 'jpg:-'], { input: buf }).toString().trim());
|
|
90
|
+
if (!(q > 90)) return null;
|
|
91
|
+
return execFileSync('magick', ['jpg:-', ...jpegArgs(name), 'jpg:-'], { input: buf, maxBuffer: 64 << 20 });
|
|
92
|
+
}
|
|
93
|
+
let squeezed = 0, saved = 0;
|
|
94
|
+
const MODELS = 'public/assets/models';
|
|
95
|
+
for (const id of readdirSync(MODELS)) {
|
|
96
|
+
const texDir = join(MODELS, id, 'textures');
|
|
97
|
+
if (existsSync(texDir)) for (const f of readdirSync(texDir).filter((x) => x.endsWith('.jpg'))) {
|
|
98
|
+
const p = join(texDir, f), buf = readFileSync(p);
|
|
99
|
+
copy(p, p, { format: 'jpg', args: jpegArgs(f), squeezeAbove: 90 }, () => {
|
|
100
|
+
const out = squeeze(buf, f);
|
|
101
|
+
if (out && out.length < buf.length) { squeezed++; saved += buf.length - out.length; return out; }
|
|
102
|
+
return buf;
|
|
103
|
+
}, true);
|
|
104
|
+
}
|
|
105
|
+
const glb = join(MODELS, id, `${id}_lod.glb`);
|
|
106
|
+
if (existsSync(glb)) copy(glb, glb, { format: 'glb', quality: 85, squeezeAbove: 90, normalSampling: '1x1', alignment: 4 }, () => squeezeGlb(glb), true);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Re-encode a GLB's embedded JPEGs: rewrite the BIN chunk with each image's bufferView replaced and every
|
|
111
|
+
* bufferView re-packed (4-byte aligned) in its original order. Returns the output bytes.
|
|
112
|
+
*/
|
|
113
|
+
function squeezeGlb(path) {
|
|
114
|
+
const { json, views, size } = readGlb(path);
|
|
115
|
+
let changed = false;
|
|
116
|
+
for (const img of json.images ?? []) {
|
|
117
|
+
if (img.mimeType !== 'image/jpeg' || img.bufferView === undefined) continue;
|
|
118
|
+
const out = squeeze(views[img.bufferView], img.name ?? '');
|
|
119
|
+
if (out && out.length < views[img.bufferView].length) { views[img.bufferView] = out; changed = true; }
|
|
120
|
+
}
|
|
121
|
+
if (!changed) return readFileSync(path);
|
|
122
|
+
const bytes = glbBytes(json, views);
|
|
123
|
+
squeezed++; saved += size - bytes.length;
|
|
124
|
+
return bytes;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function readGlb(path) {
|
|
128
|
+
const b = readFileSync(path);
|
|
129
|
+
if (b.readUInt32LE(0) !== 0x46546c67) throw new Error(`${path}: not a GLB`);
|
|
130
|
+
const jsonLen = b.readUInt32LE(12);
|
|
131
|
+
const json = JSON.parse(b.subarray(20, 20 + jsonLen).toString('utf8'));
|
|
132
|
+
const binStart = 20 + jsonLen + 8;
|
|
133
|
+
const bin = b.subarray(binStart, binStart + b.readUInt32LE(20 + jsonLen));
|
|
134
|
+
const views = json.bufferViews.map((v) => bin.subarray(v.byteOffset ?? 0, (v.byteOffset ?? 0) + v.byteLength));
|
|
135
|
+
return { json, views, size: b.length };
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** Pack a GLB whose bufferViews are `views`, re-packed 4-byte aligned in order. */
|
|
139
|
+
function glbBytes(json, views) {
|
|
140
|
+
const parts = []; let off = 0;
|
|
141
|
+
json.bufferViews.forEach((v, i) => {
|
|
142
|
+
v.byteOffset = off; v.byteLength = views[i].length;
|
|
143
|
+
parts.push(views[i]); off += views[i].length;
|
|
144
|
+
const pad = (4 - (off % 4)) % 4; if (pad) { parts.push(Buffer.alloc(pad)); off += pad; }
|
|
145
|
+
});
|
|
146
|
+
json.buffers[0].byteLength = off;
|
|
147
|
+
const newBin = Buffer.concat(parts);
|
|
148
|
+
let jsonBuf = Buffer.from(JSON.stringify(json), 'utf8');
|
|
149
|
+
jsonBuf = Buffer.concat([jsonBuf, Buffer.alloc((4 - (jsonBuf.length % 4)) % 4, 0x20)]);
|
|
150
|
+
const header = Buffer.alloc(12); header.writeUInt32LE(0x46546c67, 0); header.writeUInt32LE(2, 4);
|
|
151
|
+
const jh = Buffer.alloc(8); jh.writeUInt32LE(jsonBuf.length, 0); jh.writeUInt32LE(0x4e4f534a, 4);
|
|
152
|
+
const bh = Buffer.alloc(8); bh.writeUInt32LE(newBin.length, 0); bh.writeUInt32LE(0x004e4942, 4);
|
|
153
|
+
const total = 12 + 8 + jsonBuf.length + 8 + newBin.length; header.writeUInt32LE(total, 8);
|
|
154
|
+
return Buffer.concat([header, jh, jsonBuf, bh, newBin]);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// ── 3. .phone.webp siblings ──
|
|
158
|
+
const PHONE_MAX = 1024, ARM_MAX = 512;
|
|
159
|
+
/** AO / roughness / metalness planes: low-frequency, the phone gets them at half resolution */
|
|
160
|
+
const isArm = (p) => /(^|[_/-])(arm|rough|roughness|metal|metallic)([_.-]|$)/i.test(p.split('/').pop() ?? p);
|
|
161
|
+
/**
|
|
162
|
+
* [file the other tiers download, best source to encode from, max px] for every image the phone fetches: baked planes (cards, fur, clouds, planet),
|
|
163
|
+
* the twig atlas, every Poly Haven set (encoded from the ORIGINAL map, not the q82 _1k copy — one lossy pass, not
|
|
164
|
+
* two) and the glTF props' textures (through the loaders' URL modifier, src/client/engine/boot/bytes.ts).
|
|
165
|
+
*/
|
|
166
|
+
function phoneJobs() {
|
|
167
|
+
const jobs = [];
|
|
168
|
+
const walk = (dir) => { for (const n of readdirSync(dir)) { const p = join(dir, n); if (statSync(p).isDirectory()) walk(p); else if (/\.(png|jpg)$/.test(n) && !n.includes('.phone.')) jobs.push([p, p, PHONE_MAX]); } };
|
|
169
|
+
walk('public/assets/baked');
|
|
170
|
+
for (const { source, served, max } of textureTierJobs([...new Set([...textureFiles, ...tierJobs.map((job) => job.output)])].filter(existsSync)).filter((job) => job.kind === 'phone')) jobs.push([served, source, max]);
|
|
171
|
+
for (const id of readdirSync(MODELS)) {
|
|
172
|
+
const t = join(MODELS, id, 'textures');
|
|
173
|
+
if (existsSync(t)) for (const n of readdirSync(t)) if (n.endsWith('.jpg') && !n.includes('.phone.')) jobs.push([join(t, n), join(t, n), isArm(n) ? ARM_MAX : PHONE_MAX]);
|
|
174
|
+
}
|
|
175
|
+
return jobs;
|
|
176
|
+
}
|
|
177
|
+
function phoneName(p) { return p.replace(/\.(png|jpg)$/, '.phone.webp'); }
|
|
178
|
+
|
|
179
|
+
/** WebP of `src` (a file or a JPEG buffer) capped at `max` px. PNG alpha stays lossless; RGB under it is kept (-exact). */
|
|
180
|
+
function webp(src, max, out) {
|
|
181
|
+
const tmp = `${out}.tmp.png`;
|
|
182
|
+
const input = Buffer.isBuffer(src) ? ['jpg:-'] : [src];
|
|
183
|
+
const [w, h] = execFileSync('magick', ['identify', '-format', '%w %h', ...input], Buffer.isBuffer(src) ? { input: src } : {}).toString().trim().split(' ').map(Number);
|
|
184
|
+
const resize = Math.max(w, h) > max ? ['-separate', '-resize', `${max}x${max}`, '-combine'] : []; // per channel: the default alpha-weighted resize blackens the RGB under transparent texels
|
|
185
|
+
execFileSync('magick', [...input, ...resize, '-strip', tmp], Buffer.isBuffer(src) ? { input: src } : {});
|
|
186
|
+
execFileSync('cwebp', ['-quiet', '-q', '75', '-sharp_yuv', '-alpha_q', '100', '-exact', '-m', '6', tmp, '-o', out]);
|
|
187
|
+
rmSync(tmp);
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
let phoneMade = 0, phoneSkipped = 0;
|
|
191
|
+
for (const [served, src, max] of phoneJobs()) {
|
|
192
|
+
const out = phoneName(served);
|
|
193
|
+
copy(src, out, { format: 'webp', max, quality: 75, sharpYuv: true, alphaQuality: 100, exact: true, method: 6, resize: 'separate-combine', minSaving: 0.15, servedHash: outputHash(readFileSync(served)) }, () => {
|
|
194
|
+
const temp = join(scratch, `${++scratchId}.webp`);
|
|
195
|
+
webp(src, max, temp);
|
|
196
|
+
const bytes = readFileSync(temp); rmSync(temp);
|
|
197
|
+
if (bytes.length > statSync(served).size * 0.85) { phoneSkipped++; return null; }
|
|
198
|
+
phoneMade++;
|
|
199
|
+
return bytes;
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
// ── 4. <id>_lod.phone.glb: the props' embedded JPEGs as WebP (EXT_texture_webp), ARM planes at half resolution ──
|
|
204
|
+
for (const id of readdirSync(MODELS)) {
|
|
205
|
+
const glb = join(MODELS, id, `${id}_lod.glb`), out = join(MODELS, id, `${id}_lod.phone.glb`);
|
|
206
|
+
if (!existsSync(glb)) continue;
|
|
207
|
+
copy(glb, out, { format: 'glb-webp', max: PHONE_MAX, armMax: ARM_MAX, quality: 75, sharpYuv: true, alphaQuality: 100, exact: true, method: 6, resize: 'separate-combine', alignment: 4 }, () => {
|
|
208
|
+
const { json, views } = readGlb(glb);
|
|
209
|
+
const tmp = join(scratch, `${++scratchId}.webp`);
|
|
210
|
+
(json.images ?? []).forEach((img, i) => {
|
|
211
|
+
if (img.mimeType !== 'image/jpeg' || img.bufferView === undefined) return;
|
|
212
|
+
webp(views[img.bufferView], isArm(img.name ?? '') ? ARM_MAX : PHONE_MAX, tmp);
|
|
213
|
+
views[img.bufferView] = readFileSync(tmp); rmSync(tmp);
|
|
214
|
+
img.mimeType = 'image/webp';
|
|
215
|
+
for (const t of json.textures ?? []) if (t.source === i) { t.extensions = { ...t.extensions, EXT_texture_webp: { source: i } }; delete t.source; }
|
|
216
|
+
});
|
|
217
|
+
for (const k of ['extensionsUsed', 'extensionsRequired']) json[k] = [...new Set([...(json[k] ?? []), 'EXT_texture_webp'])];
|
|
218
|
+
phoneMade++;
|
|
219
|
+
return glbBytes(json, views);
|
|
220
|
+
});
|
|
221
|
+
}
|
|
222
|
+
// Other pipelines own Nalati rigs, shard-specific phone textures and hand-authored model variants.
|
|
223
|
+
// Sweep only the folders/families this script generates; never remove another pipeline's copies.
|
|
224
|
+
const sweep = (dir) => { for (const n of readdirSync(dir)) { const p = join(dir, n); if (statSync(p).isDirectory()) sweep(p); else if (n.includes('.phone.') && !want.has(p) && !stale.has(p)) output.remove(p); } };
|
|
225
|
+
if (!seed) {
|
|
226
|
+
sweep('public/assets/baked');
|
|
227
|
+
sweep(ROOT);
|
|
228
|
+
for (const id of readdirSync(MODELS)) {
|
|
229
|
+
const out = join(MODELS, id, `${id}_lod.phone.glb`);
|
|
230
|
+
if (existsSync(out) && !want.has(out) && !stale.has(out)) output.remove(out);
|
|
231
|
+
const tex = join(MODELS, id, 'textures');
|
|
232
|
+
if (existsSync(tex)) sweep(tex);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
for (const path of stale) console.log(`tex-tiers: STALE ${path} (source/settings cache entry or output hash missing/mismatched)`);
|
|
236
|
+
if (stale.size > 0) process.exitCode = 1;
|
|
237
|
+
if (!check) output.put(CACHE, jsonBytes(Object.fromEntries(Object.entries(cache).sort(([a], [b]) => a.localeCompare(b)))));
|
|
238
|
+
console.log(`tex-tiers: ${made} _1k encoded · ${squeezed} model files squeezed (−${(saved / 1048576).toFixed(1)} MB) · ${phoneMade} phone copies encoded, ${phoneSkipped} not kept`);
|
|
239
|
+
output.finish();
|
|
240
|
+
console.log(`tex-tiers: ${encoded} encoded, ${reused} reused, ${seeded} seeded${seed ? ' (cache only; assets untouched)' : ''}`);
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// unused-assets.mjs — E161 ("cull and GC stale / unused stuff"): every file under public/ that no build path references.
|
|
3
|
+
//
|
|
4
|
+
// A file is USED when any of these reach it:
|
|
5
|
+
// 1. a shard's boot, per tier (phone, desktop): src/client/engine/boot/manifest.ts `chunkFiles` for every shard — the packs are cut
|
|
6
|
+
// from these same files (scripts/bake-packs.mjs) — plus Rapier's WASM and the navmesh;
|
|
7
|
+
// 2. the audio manifests: every file a public/assets/music/*/music.json or sfx/*/sfx.json names (+ the manifests);
|
|
8
|
+
// 3. a used .gltf's own buffers / images;
|
|
9
|
+
// 4. a sibling of a used file that differs only by its tier / format suffix (`x.phone.webp`, `x.ktx2`, `x.phone.glb`);
|
|
10
|
+
// 5. the game's code and pages naming it: src/ (not the *.generated.ts tables, which list every file),
|
|
11
|
+
// index.html, public/manifest.webmanifest, vite.config.ts, vite/ — by its path, or by its file stem together with its
|
|
12
|
+
// folder's name (`/assets/lut/${slug}.bin`, `${DIR}${kind}.gen.glb` build paths from pieces).
|
|
13
|
+
// A folder the code builds names inside counts whole: its path is named (`'/assets/pine-hollow/journal'` + `${id}.webp`),
|
|
14
|
+
// or it is a texture set / model folder whose id is named (`pbrUrls('stone_wall')` → /assets/tex/stone_wall/*).
|
|
15
|
+
// A file only scripts/ or test/ name is DEV-ONLY: listed, kept.
|
|
16
|
+
// A file (or its folder) a doc under docs/ names is DOCUMENTED — staged for a pass, not wired in yet: listed, kept.
|
|
17
|
+
// Everything else is UNUSED.
|
|
18
|
+
//
|
|
19
|
+
// node scripts/unused-assets.mjs the report (exit 0)
|
|
20
|
+
// node scripts/unused-assets.mjs --delete also delete the UNUSED files (then commit the deletions)
|
|
21
|
+
// node scripts/unused-assets.mjs --warn one warning line when there are unused files (the pre-push gate), exit 0
|
|
22
|
+
// node scripts/unused-assets.mjs --json the lists as JSON
|
|
23
|
+
import { execFileSync } from 'node:child_process';
|
|
24
|
+
import { existsSync, readFileSync, readdirSync, statSync, unlinkSync } from 'node:fs';
|
|
25
|
+
import { dirname, posix, resolve, join } from 'node:path';
|
|
26
|
+
import { pathToFileURL } from 'node:url';
|
|
27
|
+
|
|
28
|
+
const ROOT = resolve(import.meta.dirname, '..');
|
|
29
|
+
const argv = process.argv.slice(2);
|
|
30
|
+
const has = (f) => argv.includes(`--${f}`);
|
|
31
|
+
const collect = argv.find((a) => a.startsWith('--collect='))?.slice(10);
|
|
32
|
+
|
|
33
|
+
// ── child mode: one tier's boot files; initialize its row explicitly before reading assets ──
|
|
34
|
+
if (collect) {
|
|
35
|
+
const imp = (p) => import(pathToFileURL(resolve(ROOT, p)).href);
|
|
36
|
+
const { initializeTier } = await import('@wildshard/engine/core/tier');
|
|
37
|
+
await initializeTier(collect);
|
|
38
|
+
const { SHARDS } = await imp('src/shards.generated.ts');
|
|
39
|
+
const { playable } = await imp('src/client/game/shard/registry.ts');
|
|
40
|
+
const PLAYABLE_SHARDS = SHARDS.filter(playable);
|
|
41
|
+
const { prepareShardAssets } = await imp('src/client/game/shard/load.ts');
|
|
42
|
+
const { registerGpuFiles } = await imp('src/client/engine/boot/gpuFiles.ts');
|
|
43
|
+
await Promise.all(SHARDS.map((m) => prepareShardAssets(m, registerGpuFiles)));
|
|
44
|
+
// Every authored GPU stand-in is shipped runtime data, even when this tier's default boots images.
|
|
45
|
+
const { GPU_FILES: engineGpu } = await imp('src/client/engine/boot/ktx2.generated.ts');
|
|
46
|
+
const { chunkFiles } = await imp('src/client/engine/boot/manifest.ts');
|
|
47
|
+
const out = new Set();
|
|
48
|
+
for (const def of PLAYABLE_SHARDS) for (const list of Object.values(chunkFiles(def))) for (const f of list) out.add(f);
|
|
49
|
+
for (const url of Object.values(engineGpu[collect])) out.add(url);
|
|
50
|
+
for (const def of SHARDS) if (def.ktx2) for (const url of Object.values((await def.ktx2()).GPU_FILES[collect])) out.add(url);
|
|
51
|
+
process.stdout.write(JSON.stringify([...out]));
|
|
52
|
+
process.exit(0);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// ── the files: tracked under public/, minus what every build regenerates (packs, Rapier's WASM: gitignored anyway) ──
|
|
56
|
+
const tracked = execFileSync('git', ['ls-files', '-z', 'public'], { cwd: ROOT, encoding: 'utf8' }).split('\0').filter(Boolean)
|
|
57
|
+
.filter((f) => existsSync(join(ROOT, f)) && !f.endsWith('.DS_Store'));
|
|
58
|
+
const pub = (f) => f.slice('public'.length); // public/assets/x → /assets/x
|
|
59
|
+
const files = tracked.map(pub);
|
|
60
|
+
const size = (p) => statSync(join(ROOT, 'public', p)).size;
|
|
61
|
+
|
|
62
|
+
const used = new Map(); // path → why
|
|
63
|
+
const use = (p, why) => { if (!used.has(p)) used.set(p, why); };
|
|
64
|
+
|
|
65
|
+
// 1. the boot, both tiers
|
|
66
|
+
for (const tier of ['phone', 'desktop']) {
|
|
67
|
+
const json = execFileSync(process.execPath, ['--import', './scripts/bake-loader.mjs', 'scripts/unused-assets.mjs', `--collect=${tier}`], { cwd: ROOT, encoding: 'utf8', stdio: ['ignore', 'pipe', 'inherit'] });
|
|
68
|
+
for (const f of JSON.parse(json)) use(f, `boot (${tier})`);
|
|
69
|
+
}
|
|
70
|
+
use('/assets/physics/rapier.wasm', 'boot (physics)');
|
|
71
|
+
|
|
72
|
+
// 2. the audio manifests
|
|
73
|
+
const AUDIO_RE = /\.(m4a|mp3|ogg|opus|wav|webm|flac)$/;
|
|
74
|
+
const namedIn = (v, out) => {
|
|
75
|
+
if (typeof v === 'string') { if (AUDIO_RE.test(v) && !v.includes('/')) out.push(v); }
|
|
76
|
+
else if (Array.isArray(v)) for (const x of v) namedIn(x, out);
|
|
77
|
+
else if (v && typeof v === 'object') for (const x of Object.values(v)) namedIn(x, out);
|
|
78
|
+
return out;
|
|
79
|
+
};
|
|
80
|
+
for (const [kind, manifest] of [['music', 'music.json'], ['sfx', 'sfx.json']]) {
|
|
81
|
+
const dir = join(ROOT, 'public/assets', kind);
|
|
82
|
+
if (!existsSync(dir)) continue;
|
|
83
|
+
for (const d of readdirSync(dir)) {
|
|
84
|
+
const m = join(dir, d, manifest);
|
|
85
|
+
if (!existsSync(m)) continue;
|
|
86
|
+
use(`/assets/${kind}/${d}/${manifest}`, 'audio manifest');
|
|
87
|
+
for (const f of namedIn(JSON.parse(readFileSync(m, 'utf8')), [])) use(`/assets/${kind}/${d}/${f}`, `${kind}/${d}/${manifest}`);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// 5. the code and pages (dev corpus separately)
|
|
92
|
+
const walk = (dir, out = []) => {
|
|
93
|
+
if (!existsSync(dir)) return out;
|
|
94
|
+
for (const e of readdirSync(dir, { withFileTypes: true })) {
|
|
95
|
+
const p = join(dir, e.name);
|
|
96
|
+
if (e.isDirectory()) { if (e.name !== 'node_modules') walk(p, out); } else if (/\.(ts|tsx|js|mjs|cjs|css|html|json|glsl|py|sh|md|webmanifest)$/.test(e.name)) out.push(p);
|
|
97
|
+
}
|
|
98
|
+
return out;
|
|
99
|
+
};
|
|
100
|
+
const read = (ps) => ps.filter((p) => !p.endsWith('.generated.ts')).map((p) => readFileSync(p, 'utf8')).join('\n');
|
|
101
|
+
const game = read([
|
|
102
|
+
...walk(join(ROOT, 'src')),
|
|
103
|
+
join(ROOT, 'index.html'), join(ROOT, 'public/manifest.webmanifest'), join(ROOT, 'vite.config.ts'), ...walk(join(ROOT, 'vite')),
|
|
104
|
+
].filter((p) => existsSync(p)));
|
|
105
|
+
const dev = read([...walk(join(ROOT, 'scripts')).filter((p) => !p.endsWith('unused-assets.mjs')), ...walk(join(ROOT, 'test'))]);
|
|
106
|
+
const tokenIn = (text, t) => t.length > 2 && new RegExp(`(^|[^\\w-])${t.replaceAll(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`)}([^\\w-]|$)`).test(text);
|
|
107
|
+
/** named by path, or by stem + folder (a path built from pieces) */
|
|
108
|
+
function named(text, p) {
|
|
109
|
+
if (text.includes(p.slice(1)) || text.includes(p.replace(/\.[^./]+$/, '').slice(1))) return true;
|
|
110
|
+
const base = posix.basename(p), stem = base.split('.')[0], folder = posix.basename(posix.dirname(p));
|
|
111
|
+
if (p.split('/').length <= 2) return text.includes(base); // a root file: /icon-192.png, /trailer-15.mp4
|
|
112
|
+
return tokenIn(text, stem) && tokenIn(text, folder);
|
|
113
|
+
}
|
|
114
|
+
/** a folder the code reads by building names inside it: its path (`JOURNAL_ART = '/assets/pine-hollow/journal'`), or a texture
|
|
115
|
+
* set / model folder named by its id (`pbrUrls('stone_wall')`, `gltf('hatchet')`: `/assets/tex|models/${id}/…`) */
|
|
116
|
+
function folderNamed(text, p) {
|
|
117
|
+
const dir = posix.dirname(p);
|
|
118
|
+
if (text.includes(dir.slice(1))) return true;
|
|
119
|
+
const m = /^\/assets\/(tex|models)\/([^/]+)\//.exec(p);
|
|
120
|
+
return m !== null && tokenIn(text, m[2]);
|
|
121
|
+
}
|
|
122
|
+
for (const p of files) if (!used.has(p) && (named(game, p) || folderNamed(game, p))) use(p, 'named in the game code');
|
|
123
|
+
|
|
124
|
+
// 3 + 4, to a fixed point: glTF dependencies and tier / format siblings of what is used
|
|
125
|
+
const stemKey = (p) => { const d = posix.dirname(p), b = posix.basename(p); return `${d}/${b.split('.')[0]}`; };
|
|
126
|
+
const byStem = new Map();
|
|
127
|
+
for (const p of files) { const k = stemKey(p); byStem.set(k, [...(byStem.get(k) ?? []), p]); }
|
|
128
|
+
for (let grew = true; grew;) {
|
|
129
|
+
grew = false;
|
|
130
|
+
for (const [p, why] of used) { // a Map iterates the entries added meanwhile too
|
|
131
|
+
if (p.endsWith('.gltf') && existsSync(join(ROOT, 'public', p))) {
|
|
132
|
+
try {
|
|
133
|
+
const g = JSON.parse(readFileSync(join(ROOT, 'public', p), 'utf8'));
|
|
134
|
+
for (const r of [...(g.buffers ?? []), ...(g.images ?? [])]) {
|
|
135
|
+
if (typeof r.uri !== 'string' || r.uri.startsWith('data:')) continue;
|
|
136
|
+
const dep = posix.normalize(posix.join(posix.dirname(p), decodeURIComponent(r.uri)));
|
|
137
|
+
if (!used.has(dep)) { use(dep, `${p} (${why})`); grew = true; }
|
|
138
|
+
}
|
|
139
|
+
} catch { /* not JSON: left to the other rules */ }
|
|
140
|
+
}
|
|
141
|
+
for (const s of byStem.get(stemKey(p)) ?? []) if (!used.has(s)) { use(s, `a tier / format copy of ${p}`); grew = true; }
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
const devReach = (p) => { if (named(dev, p)) return true; return folderNamed(dev, p); };
|
|
146
|
+
const devOnly = files.filter((p) => !used.has(p) && devReach(p));
|
|
147
|
+
// named by a design doc or a plan (a model set staged for a pass not yet wired in): kept, listed
|
|
148
|
+
const docs = read(walk(join(ROOT, 'docs')));
|
|
149
|
+
/** the file or a folder above it, three levels deep at least (`assets/nalati/sourced`, never plain `assets/nalati`) */
|
|
150
|
+
const ancestors = (p) => { const parts = p.slice(1).split('/'); const out = []; for (let i = parts.length; i >= 3; i--) out.push(parts.slice(0, i).join('/')); return out; };
|
|
151
|
+
const documented = files.filter((p) => !used.has(p) && !devOnly.includes(p) && ancestors(p).some((a) => docs.includes(a)));
|
|
152
|
+
const unused = files.filter((p) => !used.has(p) && !devOnly.includes(p) && !documented.includes(p));
|
|
153
|
+
const sum = (ps) => ps.reduce((s, p) => s + size(p), 0);
|
|
154
|
+
const mb = (b) => `${(b / 1048576).toFixed(2)} MB`;
|
|
155
|
+
|
|
156
|
+
if (has('json')) {
|
|
157
|
+
process.stdout.write(`${JSON.stringify({ files: files.length, used: used.size, devOnly: devOnly.map((p) => [p, size(p)]), documented: documented.map((p) => [p, size(p)]), unused: unused.map((p) => [p, size(p)]) }, null, 1)}\n`);
|
|
158
|
+
process.exit(0);
|
|
159
|
+
}
|
|
160
|
+
if (has('warn')) {
|
|
161
|
+
if (unused.length > 0) console.warn(`unused-assets: WARNING — ${unused.length} file(s) under public/ that nothing references (${mb(sum(unused))}); run node scripts/unused-assets.mjs to list them`);
|
|
162
|
+
process.exit(0);
|
|
163
|
+
}
|
|
164
|
+
console.log(`public/: ${files.length} tracked files, ${mb(sum(files))} · used ${files.length - devOnly.length - documented.length - unused.length} · dev-only ${devOnly.length} (${mb(sum(devOnly))}) · documented ${documented.length} (${mb(sum(documented))}) · UNUSED ${unused.length} (${mb(sum(unused))})`);
|
|
165
|
+
if (devOnly.length > 0) { console.log('\ndev-only (named by scripts/ or test/ — kept):'); for (const p of devOnly) console.log(` ${mb(size(p)).padStart(9)} ${p}`); }
|
|
166
|
+
if (documented.length > 0) { console.log('\ndocumented (named by docs/ — staged for a pass, kept):'); for (const p of documented) console.log(` ${mb(size(p)).padStart(9)} ${p}`); }
|
|
167
|
+
if (unused.length > 0) { console.log('\nUNUSED:'); for (const p of unused) console.log(` ${mb(size(p)).padStart(9)} ${p}`); }
|
|
168
|
+
if (has('delete') && unused.length > 0) {
|
|
169
|
+
const bytes = sum(unused);
|
|
170
|
+
for (const p of unused) unlinkSync(join(ROOT, 'public', p));
|
|
171
|
+
for (const d of new Set(unused.map((p) => dirname(join(ROOT, 'public', p))))) { try { if (readdirSync(d).length === 0) execFileSync('rmdir', [d]); } catch { /* not empty */ } }
|
|
172
|
+
console.log(`\ndeleted ${unused.length} files, ${mb(bytes)}`);
|
|
173
|
+
}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Execute the workspace's author CLI without requiring a previously packed SDK or a Vite dev server.
|
|
3
|
+
import { build } from 'vite';
|
|
4
|
+
import { createBuildCacheCounters, observeBuildCache, formatBuildCacheReport } from '@wildshard/engine/scripts/build-cache-report.mjs';
|
|
5
|
+
import { captureFolderInputs, recordBundle, inputRecorderPlugin } from '@wildshard/engine/scripts/shardfile-inputs.mjs';
|
|
6
|
+
import { basename, dirname, resolve, sep } from 'node:path';
|
|
7
|
+
import { existsSync, readFileSync, realpathSync } from 'node:fs';
|
|
8
|
+
import { createHash } from 'node:crypto';
|
|
9
|
+
import { pathToFileURL } from 'node:url';
|
|
10
|
+
import { createRequire } from 'node:module';
|
|
11
|
+
import { importBundle } from '@wildshard/sdk/scripts/import-bundle.mjs';
|
|
12
|
+
import { shardfileValidationRevision } from '@wildshard/engine/scripts/shardfile-validation-revision.mjs';
|
|
13
|
+
import { rapierAlias } from '@wildshard/engine/vite/rapier.ts';
|
|
14
|
+
import { authorBundleCached } from '@wildshard/engine/scripts/author-bundle-cache.mjs';
|
|
15
|
+
import { decisionLegacyLines } from '@wildshard/build/scripts/decision-legacy-lines.mjs';
|
|
16
|
+
import { outputManifest, shardOutputs } from '@wildshard/build/scripts/shard-outputs.mjs';
|
|
17
|
+
import { findShardSource, shardFolderName } from '@wildshard/upload/sharddev/shardSource.ts';
|
|
18
|
+
|
|
19
|
+
/** The verbs whose first operand is a shard source (or, for validate and prove, a built folder). */
|
|
20
|
+
const SOURCE_VERBS = new Set(['check', 'synth', 'build', 'dev', 'watch', 'validate', 'prove']);
|
|
21
|
+
|
|
22
|
+
let runner;
|
|
23
|
+
/** SA3 (J170): `build` and `check` end by writing the folder's shard file, the step `pack` was. It stays out of the author
|
|
24
|
+
* bundle: the same module serves `wildshard build <folder>` and the deprecated `pack` (src/cli/main.ts). */
|
|
25
|
+
async function writeShardfile(dir, source, options = {}) {
|
|
26
|
+
const { writeShardfile: write } = await import('@wildshard/upload/cli/shardfile/shardfile.ts');
|
|
27
|
+
return (await write({ dir, source, report: line => console.info(line), ...options })).file;
|
|
28
|
+
}
|
|
29
|
+
// The bundle's import.meta.dirname stays src/sdk: headless.ts finds bake/headlessWorker.ts beside it, and src/sdk keeps the
|
|
30
|
+
// author tools' dependency links until the host leaves at P3 (P1-F4).
|
|
31
|
+
const authorRequire = createRequire(resolve(import.meta.dirname, '../src/sharddev/package.json'));
|
|
32
|
+
const DEV_SRC = resolve(import.meta.dirname, '../src/sharddev');
|
|
33
|
+
const SDK_DIR = realpathSync(dirname(authorRequire.resolve('@wildshard/sdk/package.json'))), SDK_SRC = resolve(SDK_DIR, 'src/sharddev');
|
|
34
|
+
// The script compilers and the texture tools are the sdk's dependencies; they load from the sdk's own install.
|
|
35
|
+
const sdkRequire = createRequire(resolve(SDK_SRC, 'package.json'));
|
|
36
|
+
// P4: the project is the repo the author stands in; the tools are this package.
|
|
37
|
+
const projectRoot = () => process.cwd();
|
|
38
|
+
/** Compile the trusted CLI once before product children; later invocations
|
|
39
|
+
* verify the same content-addressed compiler and replay its complete inputs. */
|
|
40
|
+
export function prepareWildshard() {
|
|
41
|
+
const root=projectRoot();
|
|
42
|
+
// Resolver policy is executable compiler input too: a source-only cache hit
|
|
43
|
+
// must not replay a bundle built before an externalization change.
|
|
44
|
+
const policy = createHash('sha256').update(readFileSync(import.meta.filename)).digest('hex');
|
|
45
|
+
// SA18: which installed sdk is compiling is an input too. Receipts skip node_modules and paths outside the repo, and
|
|
46
|
+
// this file is byte-identical across releases, so without its root and version a repo that moved to a newer sdk with
|
|
47
|
+
// the same shard-format replayed the old sdk's bundle. (A sharddev checkout run from another folder is still keyed
|
|
48
|
+
// only by its path: edits to it outside that folder are not seen.)
|
|
49
|
+
const sdkRoot = realpathSync(resolve(import.meta.dirname, '..'));
|
|
50
|
+
const sdk = `${sdkRoot}@${JSON.parse(readFileSync(resolve(sdkRoot, 'package.json'), 'utf8')).version}`;
|
|
51
|
+
return authorBundleCached(root,[shardfileValidationRevision(root),policy,sdk],async()=>{
|
|
52
|
+
const result = await build({ configFile: false, publicDir: false, resolve: { alias: rapierAlias }, logLevel: 'silent', define: { __SHARDFILE_VALIDATOR__: JSON.stringify(shardfileValidationRevision()) }, plugins: [inputRecorderPlugin({execution:true}), { name: 'author-node-vite', enforce: 'pre', transform(code, id) {
|
|
53
|
+
// SDK-SPLIT SP8c: each package keeps its own src/sharddev as import.meta.dirname (the sdk finds its GUIDE.md,
|
|
54
|
+
// template and tools; dev, the engine and upload find dev's): one global value pointed the sdk's modules at dev.
|
|
55
|
+
if (!code.includes('import.meta.dirname')) return null;
|
|
56
|
+
const file = id.split('?')[0] ?? id, owner = file.startsWith(SDK_DIR + sep) ? SDK_SRC : DEV_SRC;
|
|
57
|
+
return { code: code.replaceAll('import.meta.dirname', JSON.stringify(owner)), map: null };
|
|
58
|
+
}, resolveId(id) {
|
|
59
|
+
if (id === 'vite') return { id: pathToFileURL(resolve(import.meta.dirname,'shardfile-vite.mjs')).href, external: true };
|
|
60
|
+
// SP9b: the sdk bundles with rolldown; its bundles record their inputs through dev's bridge, as vite's did.
|
|
61
|
+
if (id === 'rolldown') return { id: pathToFileURL(resolve(import.meta.dirname,'shardfile-rolldown.mjs')).href, external: true };
|
|
62
|
+
if (id === 'sharp' || id === 'binaryen' || id === '@typescript/typescript6' || /^(?:assemblyscript|rolldown)(?:\/|$)/u.test(id)) return { id: pathToFileURL(sdkRequire.resolve(id)).href, external: true };
|
|
63
|
+
return null;
|
|
64
|
+
} }], build: { write: false, minify: false, lib: { entry: resolve(import.meta.dirname, '../src/sharddev/cli.ts'), formats: ['es'], fileName: 'cli' }, rolldownOptions: { platform: 'node', external: [/^node:/u], output: { codeSplitting: false } } } });
|
|
65
|
+
recordBundle(result,{execution:true});
|
|
66
|
+
const output = Array.isArray(result) ? result.at(0) : result;
|
|
67
|
+
if (output === undefined || !('output' in output)) throw new Error('Author CLI build produced no output');
|
|
68
|
+
const chunks = output.output.filter((chunk) => chunk.type === 'chunk');
|
|
69
|
+
if (chunks.length !== 1) throw new Error('Author CLI must bundle to one module');
|
|
70
|
+
return { code: chunks[0].code, modules: chunks[0].moduleIds ?? Object.keys(chunks[0].modules) };
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
/** Bundle trusted workspace tooling once; external Vite resolves against this installed tool, not a data URL. */
|
|
74
|
+
export async function runWildshard(args) {
|
|
75
|
+
const receipt = args.find(arg=>arg.startsWith('--input-receipt='))?.slice('--input-receipt='.length);
|
|
76
|
+
if (receipt !== undefined) { await captureFolderInputs(projectRoot(),receipt,()=>runWildshard(args.filter(arg=>!arg.startsWith('--input-receipt=')))); return; }
|
|
77
|
+
runner ??= (async () => {
|
|
78
|
+
const started = performance.now(), bundle = await prepareWildshard();
|
|
79
|
+
recordBundle({output:[{type:'chunk',moduleIds:bundle.modules}]},{execution:true});
|
|
80
|
+
const cli = await importBundle(bundle.code);
|
|
81
|
+
if (cli === null || typeof cli !== 'object' || !('runCli' in cli) || typeof cli.runCli !== 'function') throw new Error('Author CLI has no runner');
|
|
82
|
+
if (process.env.GITHUB_ACTIONS === 'true') console.info(`author timing: ${JSON.stringify({ pid: process.pid, phase: 'compiler-bundle', reused:bundle.reused, product: args[1] ?? null, seconds: Number(((performance.now() - started) / 1000).toFixed(3)) })}`);
|
|
83
|
+
return cli.runCli;
|
|
84
|
+
})();
|
|
85
|
+
const root = projectRoot();
|
|
86
|
+
// SA16 (J317): a repo path names its shard source at src/ (one shard) or its one src/shards/<slug>/ (several).
|
|
87
|
+
if (SOURCE_VERBS.has(args[0] ?? '') && args[1] !== undefined && !args[1].startsWith('--')) args = [args[0], findShardSource(args[1]), ...args.slice(2)];
|
|
88
|
+
const project = args[1] === undefined ? undefined : resolve(args[1]);
|
|
89
|
+
// The folder build checks the manifest slug against: a src/ source takes its manifest's slug (new <dir> <slug>).
|
|
90
|
+
const folder = project === undefined ? '' : shardFolderName(project);
|
|
91
|
+
// SF88g: `--output=<slug>` builds one declared output of the folder, under its own slug and composed manifest
|
|
92
|
+
const selected = args.find(arg => arg.startsWith('--output='))?.slice(9);
|
|
93
|
+
const output = project === undefined || selected === undefined ? undefined : shardOutputs(root, folder)?.find(row => row.slug === selected);
|
|
94
|
+
if (selected !== undefined && output === undefined) throw new Error(`wildshard: ${basename(project ?? '')} declares no output ${selected}`);
|
|
95
|
+
const manifest = project === undefined || !['build', 'check', 'validate', 'prove'].includes(args[0] ?? '') || !existsSync(resolve(project, 'shard.config.ts')) ? undefined : {
|
|
96
|
+
folder: output?.slug ?? folder, ...(output !== undefined ? { composed: outputManifest(root, folder, output.row) } : existsSync(resolve(project, 'manifest.json')) ? { file: resolve(project, 'manifest.json') } : {}),
|
|
97
|
+
imageRoots: [{ prefix: `/src/shards/${folder}/`, directory: project }, { prefix: '/assets/', directory: resolve(root, 'public/assets') }],
|
|
98
|
+
};
|
|
99
|
+
// TS21: `check` on a folder that declares outputs (and no --output) checks each output with its own composed manifest,
|
|
100
|
+
// into <folder>/<slug> when a folder is named, and gives one verdict, as build-shards builds each output.
|
|
101
|
+
const declared = args[0] === 'check' && selected === undefined && project !== undefined && existsSync(resolve(project, 'shard.config.ts')) ? shardOutputs(root, folder) : null;
|
|
102
|
+
if (declared !== null) {
|
|
103
|
+
if (args.includes('--json')) throw new Error(`wildshard check: ${folder} declares ${declared.length} outputs; --json checks one: add --output=<slug>`);
|
|
104
|
+
const rest = args.slice(2), extra = rest.filter(arg => !arg.startsWith('--'));
|
|
105
|
+
if (extra.length > 1 || rest.some(arg => arg.startsWith('--'))) throw new Error('usage: wildshard check <source> [folder] [--json] [--output=<slug>]');
|
|
106
|
+
const [into] = extra, failed = [], before = process.exitCode;
|
|
107
|
+
for (const { slug } of declared) {
|
|
108
|
+
console.info(`== ${slug}`);
|
|
109
|
+
process.exitCode = undefined;
|
|
110
|
+
// A setup error (a missing manifest, a refused source) fails this output; the rest are still checked.
|
|
111
|
+
try { await runWildshard(['check', project, ...(into === undefined ? [] : [resolve(into, slug)]), `--output=${slug}`]); }
|
|
112
|
+
catch (error) { console.error(`${slug}: ${error instanceof Error ? error.message : String(error)}`); process.exitCode = 1; }
|
|
113
|
+
if (process.exitCode !== undefined && process.exitCode !== 0) failed.push(slug);
|
|
114
|
+
}
|
|
115
|
+
process.exitCode = before;
|
|
116
|
+
console.info(`check ${failed.length === 0 ? 'pass' : 'fail'}: ${declared.length} outputs${failed.length === 0 ? '' : `, failed: ${failed.join(', ')}`}`);
|
|
117
|
+
if (failed.length > 0) process.exitCode = 1;
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
const legacy = project === undefined ? undefined : decisionLegacyLines(root, folder);
|
|
121
|
+
await (await runner)(args, { ...(legacy === undefined ? {} : { decisionLegacyLines: legacy }), ...(manifest === undefined ? {} : { manifest }), writeShardfile });
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
|
|
125
|
+
const counters=createBuildCacheCounters(),stop=observeBuildCache(counters);
|
|
126
|
+
let complete=false;
|
|
127
|
+
try {
|
|
128
|
+
let args=process.argv.slice(2);
|
|
129
|
+
// SA4: `check` takes its flags anywhere; with --json, stdout is the verdict document alone, so every log goes to stderr.
|
|
130
|
+
if(args[0]==='check'){
|
|
131
|
+
const rest=args.slice(1);
|
|
132
|
+
args=['check',...rest.filter(arg=>!arg.startsWith('--')),...rest.filter(arg=>arg.startsWith('--'))];
|
|
133
|
+
if(args.includes('--json'))for(const level of ['info','log','warn','debug'])console[level]=console.error.bind(console);
|
|
134
|
+
}
|
|
135
|
+
if(args[0]==='--prepare-compiler') {
|
|
136
|
+
if(args.length!==1)throw new Error('Compiler preparation accepts no other arguments');
|
|
137
|
+
const started=performance.now(),compiler=await prepareWildshard();
|
|
138
|
+
console.info(`Author compiler ${compiler.reused?'reused':'built'} in ${((performance.now()-started)/1000).toFixed(3)}s (before product workers)`);
|
|
139
|
+
} else await runWildshard(args);
|
|
140
|
+
complete=true;
|
|
141
|
+
}
|
|
142
|
+
catch (error) { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; }
|
|
143
|
+
// The counters line is for the build pool, which sets WILDSHARD_CACHE_REPORT=1 for the children whose report it reads,
|
|
144
|
+
// and for debugging a build's cache; an author's run, refusals included, does not print it.
|
|
145
|
+
finally {stop();if(process.env.WILDSHARD_CACHE_REPORT==='1')console.error(`\n${formatBuildCacheReport(counters.snapshot(complete))}`);}
|
|
146
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/** A native retained TypeScript recorder and its exact loaded input closure. */
|
|
2
|
+
export interface WitnessGenerationSpec {
|
|
3
|
+
root: URL; slug: string; inputs: string; files: Iterable<string>; manifest: URL;
|
|
4
|
+
select: (directory: URL) => void; generate: () => Promise<unknown>;
|
|
5
|
+
record?: boolean; forceCompare?: boolean; compare?: boolean;
|
|
6
|
+
}
|
|
7
|
+
/** Generate into the shared cache, select its output for readers, and optionally record only the small hash manifest. */
|
|
8
|
+
export function generateWitness(spec: WitnessGenerationSpec): Promise<{
|
|
9
|
+
directory: string; key: string; hit: boolean; hashes: Record<string, string>; manifest: string;
|
|
10
|
+
}>;
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
// G292: native witness recorders use the shared generation cache, with their original TypeScript tapes and outcomes.
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
|
+
import { readFileSync, writeFileSync } from 'node:fs';
|
|
4
|
+
import { dirname, resolve } from 'node:path';
|
|
5
|
+
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
6
|
+
import * as v from 'valibot';
|
|
7
|
+
import { runNativeRecorder } from '@wildshard/engine/scripts/native-recorder-cache.mjs';
|
|
8
|
+
import { cachedWitnessManifest, compareWitnessManifests } from '@wildshard/sdk/scripts/witness-checkpoints.mjs';
|
|
9
|
+
|
|
10
|
+
const Payloads = v.record(v.string(), v.pipe(v.string(), v.regex(/^[0-9a-f]{64}$/u)));
|
|
11
|
+
const Header = v.object({ cache: v.object({ payloads: Payloads }) });
|
|
12
|
+
|
|
13
|
+
/** Play a retained native recorder into the shared cache; all consumers read that verified output directory.
|
|
14
|
+
* During migration a legacy manifest is decorated from its still-committed payloads. Those files remain until CI
|
|
15
|
+
* proves the cold cache path. Only explicit recording writes the small manifest back to the repository.
|
|
16
|
+
* @param {{root:URL,slug:string,inputs:string,files:Iterable<string>,manifest:URL,
|
|
17
|
+
* select:(directory:URL)=>void,generate:()=>Promise<unknown>,record?:boolean,forceCompare?:boolean,compare?:boolean}} spec
|
|
18
|
+
* @returns {Promise<{directory:string,key:string,hit:boolean,hashes:Record<string,string>,manifest:string}>} */
|
|
19
|
+
export async function generateWitness(spec) {
|
|
20
|
+
const root = fileURLToPath(spec.root), source = fileURLToPath(spec.manifest);
|
|
21
|
+
const committed = readFileSync(source, 'utf8');
|
|
22
|
+
const parsed = v.safeParse(Header, JSON.parse(committed));
|
|
23
|
+
const expected = parsed.success ? committed : cachedWitnessManifest(dirname(source));
|
|
24
|
+
const outputs = [...Object.keys(v.parse(Header, JSON.parse(expected)).cache.payloads), 'manifest.json'].sort();
|
|
25
|
+
const inputs = Object.fromEntries([...new Set(spec.files)].sort().map(url => {
|
|
26
|
+
if (!url.startsWith(spec.root.href)) throw new Error('Witness input is outside its repository');
|
|
27
|
+
return [decodeURIComponent(url.slice(spec.root.href.length)), createHash('sha256').update(readFileSync(new URL(url))).digest('hex')];
|
|
28
|
+
}));
|
|
29
|
+
const result = await runNativeRecorder(root, {
|
|
30
|
+
id: `witness-${spec.slug}`, inputs, outputs, platform: 'native',
|
|
31
|
+
command: ['node', '--experimental-transform-types', '--import', './scripts/sim-node-loader.mjs', `test/proof/${spec.slug}/run.mjs`, 'checkpoints'],
|
|
32
|
+
}, {
|
|
33
|
+
...(spec.forceCompare === undefined ? {} : { forceCompare: spec.forceCompare }),
|
|
34
|
+
generate: async directory => {
|
|
35
|
+
spec.select(pathToFileURL(`${resolve(directory)}/`));
|
|
36
|
+
await spec.generate();
|
|
37
|
+
writeFileSync(resolve(directory, 'manifest.json'), cachedWitnessManifest(directory));
|
|
38
|
+
},
|
|
39
|
+
});
|
|
40
|
+
const manifest = readFileSync(resolve(result.directory, 'manifest.json'), 'utf8');
|
|
41
|
+
if (spec.compare !== false) compareWitnessManifests(expected, manifest);
|
|
42
|
+
spec.select(pathToFileURL(`${resolve(result.directory)}/`));
|
|
43
|
+
if (spec.record === true) writeFileSync(source, manifest);
|
|
44
|
+
return { directory: result.directory, key: result.key, hit: result.hit, hashes: result.hashes, manifest };
|
|
45
|
+
}
|