@wildshard/shard-format 0.4.5 → 0.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -1
- package/src/format/audio-rules.ts +36 -12
- package/src/format/contentBounds.ts +4 -3
- package/src/format/grid-catalogue.ts +4 -3
- package/src/format/issue-codes.ts +262 -0
- package/src/format/product-stream-admit.ts +31 -17
- package/src/format/product-stream-assets.ts +24 -6
- package/src/format/product-stream-miss.ts +3 -0
- package/src/format/product-stream-packs.ts +6 -1
- package/src/format/product-stream-rows.ts +63 -31
- package/src/format/product-stream-tar.ts +20 -8
- package/src/format/product-stream.ts +25 -12
- package/src/format/product.ts +29 -21
- package/src/format/productValidation.ts +74 -43
- package/src/format/tar-paths.ts +3 -2
- package/src/format/tar.ts +12 -11
- package/src/tools/runtime-hosts.json +497 -0
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
// oxlint-disable import/max-dependencies -- the old validator's steps, each from its defining module
|
|
1
2
|
// `validateShardfileProduct`'s inventory steps, replayed over what one
|
|
2
3
|
// streaming pass kept (API-CLOUDFLARE AC6a): each listed file's hash and
|
|
3
4
|
// size, the blobs' facts, server files, KTX2 device costs, the cook and its
|
|
@@ -7,6 +8,7 @@
|
|
|
7
8
|
|
|
8
9
|
import { cookMountPath, readCookData } from './cook-data.ts'
|
|
9
10
|
import { requireCookPreparation } from './cook-preparation.ts'
|
|
11
|
+
import { issueError, withIssueCode } from './issue-codes.ts'
|
|
10
12
|
import { validatePlatformFile } from './platformFileFamilies.ts'
|
|
11
13
|
import { mountSharesBlob } from './product-reads.ts'
|
|
12
14
|
import { inspectFacts, ktx2Text } from './product-stream-facts.ts'
|
|
@@ -45,13 +47,19 @@ function readListed(
|
|
|
45
47
|
replay.listed.add(entry.path)
|
|
46
48
|
const member = replay.members.get(entry.path)
|
|
47
49
|
if (member === undefined) {
|
|
48
|
-
throw
|
|
50
|
+
throw issueError('file-mismatch', `missing file: ${entry.path}`)
|
|
49
51
|
}
|
|
50
52
|
if (member.size > entry.bytes) {
|
|
51
|
-
throw
|
|
53
|
+
throw issueError(
|
|
54
|
+
'file-mismatch',
|
|
55
|
+
`larger than its declared size: ${entry.path}`,
|
|
56
|
+
)
|
|
52
57
|
}
|
|
53
58
|
if (member.size !== entry.bytes || member.sha !== entry.hash) {
|
|
54
|
-
throw
|
|
59
|
+
throw issueError(
|
|
60
|
+
'file-mismatch',
|
|
61
|
+
`Output hash or size mismatch: ${entry.path}`,
|
|
62
|
+
)
|
|
55
63
|
}
|
|
56
64
|
return member
|
|
57
65
|
}
|
|
@@ -63,7 +71,7 @@ function listedBytes(
|
|
|
63
71
|
): Uint8Array {
|
|
64
72
|
const { bytes } = readListed(replay, entry)
|
|
65
73
|
if (bytes === undefined) {
|
|
66
|
-
throw
|
|
74
|
+
throw issueError('stream-miss', `stream: ${entry.path} was not kept`)
|
|
67
75
|
}
|
|
68
76
|
return bytes
|
|
69
77
|
}
|
|
@@ -76,7 +84,11 @@ function replayFiles(
|
|
|
76
84
|
// oxlint-disable-next-line typescript/prefer-readonly-parameter-types -- format's row type, only read
|
|
77
85
|
scripts: ScriptAdmissionRows,
|
|
78
86
|
): Map<string, KeptMember> {
|
|
79
|
-
const contracts =
|
|
87
|
+
const contracts = withIssueCode(
|
|
88
|
+
'script-admission',
|
|
89
|
+
(): ReturnType<typeof decisionModuleContracts> =>
|
|
90
|
+
decisionModuleContracts(scripts, index.files),
|
|
91
|
+
)
|
|
80
92
|
const assets = new Map<string, KeptMember>()
|
|
81
93
|
for (const entry of index.files) {
|
|
82
94
|
const member = readListed(replay, {
|
|
@@ -84,13 +96,15 @@ function replayFiles(
|
|
|
84
96
|
hash: entry.hash,
|
|
85
97
|
bytes: entry.compressed,
|
|
86
98
|
})
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
99
|
+
withIssueCode('blob-invalid', (): void => {
|
|
100
|
+
inspectFacts(
|
|
101
|
+
entry,
|
|
102
|
+
member,
|
|
103
|
+
entry.kind === 'wasm'
|
|
104
|
+
? (contracts.get(entry.hash) ?? { abi: 0 })
|
|
105
|
+
: undefined,
|
|
106
|
+
)
|
|
107
|
+
})
|
|
94
108
|
assets.set(
|
|
95
109
|
`${entry.commons === true ? 'commons:' : ''}${entry.hash}`,
|
|
96
110
|
member,
|
|
@@ -111,34 +125,41 @@ function replayServerAndKtx2(
|
|
|
111
125
|
): void {
|
|
112
126
|
for (const entry of index.admission.server ?? []) {
|
|
113
127
|
const bytes = listedBytes(replay, entry)
|
|
114
|
-
|
|
115
|
-
if (
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
128
|
+
withIssueCode('server-file-invalid', (): void => {
|
|
129
|
+
if (entry.kind === 'json') {
|
|
130
|
+
if (
|
|
131
|
+
validatePlatformFile(
|
|
132
|
+
entry.name,
|
|
133
|
+
entry.schema,
|
|
134
|
+
readCanonicalJson(bytes),
|
|
135
|
+
) === undefined
|
|
136
|
+
) {
|
|
137
|
+
throw new Error(`Unregistered named JSON: ${entry.name}`)
|
|
138
|
+
}
|
|
139
|
+
} else if (outputBlobKind('binary', bytes) !== entry.kind) {
|
|
140
|
+
throw new Error('Output blob bytes differ from their declared kind')
|
|
123
141
|
}
|
|
124
|
-
}
|
|
125
|
-
throw new Error('Output blob bytes differ from their declared kind')
|
|
126
|
-
}
|
|
142
|
+
})
|
|
127
143
|
}
|
|
128
144
|
for (const [hash, cost] of Object.entries(index.ktx2Costs ?? {})) {
|
|
129
|
-
const actual =
|
|
145
|
+
const actual = withIssueCode('ktx2-costs', (): string | undefined =>
|
|
146
|
+
ktx2Text(assets.get(hash)),
|
|
147
|
+
)
|
|
130
148
|
if (
|
|
131
149
|
actual === undefined ||
|
|
132
150
|
new TextDecoder().decode(validationSourceBytes(cost)) !== actual
|
|
133
151
|
) {
|
|
134
|
-
throw
|
|
152
|
+
throw issueError(
|
|
153
|
+
'ktx2-costs',
|
|
154
|
+
'KTX2 device costs differ from verified bytes',
|
|
155
|
+
)
|
|
135
156
|
}
|
|
136
157
|
}
|
|
137
158
|
}
|
|
138
159
|
|
|
139
|
-
/**
|
|
160
|
+
/** The cook steps `replayCook` tags. */
|
|
140
161
|
// oxlint-disable-next-line typescript/prefer-readonly-parameter-types -- valibot's output type, only read
|
|
141
|
-
function
|
|
162
|
+
function cookSteps(replay: Readonly<Replay>, index: OutputIndex): void {
|
|
142
163
|
const cook =
|
|
143
164
|
index.cook === undefined
|
|
144
165
|
? undefined
|
|
@@ -154,15 +175,26 @@ function replayCook(replay: Readonly<Replay>, index: OutputIndex): void {
|
|
|
154
175
|
}
|
|
155
176
|
}
|
|
156
177
|
|
|
178
|
+
/** C0 (J137): the cook file, and each mount's hash and size. */
|
|
179
|
+
// oxlint-disable-next-line typescript/prefer-readonly-parameter-types -- valibot's output type, only read
|
|
180
|
+
function replayCook(replay: Readonly<Replay>, index: OutputIndex): void {
|
|
181
|
+
withIssueCode('cook-invalid', (): void => {
|
|
182
|
+
cookSteps(replay, index)
|
|
183
|
+
})
|
|
184
|
+
}
|
|
185
|
+
|
|
157
186
|
/** No copied runtime files (the Worker has no import checker) and no file the index does not list. */
|
|
158
187
|
// oxlint-disable-next-line typescript/prefer-readonly-parameter-types -- valibot's output type, only read
|
|
159
188
|
function replayInventory(replay: Readonly<Replay>, index: OutputIndex): void {
|
|
160
189
|
if ((index.runtime?.files ?? []).length > NONE) {
|
|
161
|
-
throw
|
|
190
|
+
throw issueError(
|
|
191
|
+
'runtime-files',
|
|
192
|
+
'Copied runtime files need a runtime import checker',
|
|
193
|
+
)
|
|
162
194
|
}
|
|
163
195
|
for (const path of replay.members.keys()) {
|
|
164
196
|
if (!replay.listed.has(path)) {
|
|
165
|
-
throw
|
|
197
|
+
throw issueError('unlisted-file', `Unlisted output file: ${path}`)
|
|
166
198
|
}
|
|
167
199
|
}
|
|
168
200
|
}
|
|
@@ -191,7 +223,7 @@ function replayPacksToInventory(
|
|
|
191
223
|
}
|
|
192
224
|
const refusal = packs.get(pack.path)
|
|
193
225
|
if (refusal !== undefined) {
|
|
194
|
-
throw
|
|
226
|
+
throw issueError('pack-invalid', refusal)
|
|
195
227
|
}
|
|
196
228
|
}
|
|
197
229
|
replayCook(replay, index)
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// tarball's chunks with the same refusals, in bounded memory. Each regular
|
|
3
3
|
// file is hashed as it streams; the caller says which files it wants whole.
|
|
4
4
|
|
|
5
|
+
import { archiveError, withIssueCodeAsync } from './issue-codes.ts'
|
|
5
6
|
import { BLOCK, padded } from './tar-fields.ts'
|
|
6
7
|
import { MAX_ENTRIES, claimFile, claimFolder, newClaims } from './tar-paths.ts'
|
|
7
8
|
import { readTarHeader } from './tar.ts'
|
|
@@ -95,7 +96,7 @@ async function assertEnd(reader: ChunkReader): Promise<number> {
|
|
|
95
96
|
zero &&= isZero(piece)
|
|
96
97
|
}
|
|
97
98
|
if (rest < END_BLOCKS * BLOCK || !zero) {
|
|
98
|
-
throw
|
|
99
|
+
throw archiveError('tar: bad end-of-archive blocks')
|
|
99
100
|
}
|
|
100
101
|
return rest
|
|
101
102
|
}
|
|
@@ -127,10 +128,10 @@ async function readMember(
|
|
|
127
128
|
? new Uint8Array(entry.size)
|
|
128
129
|
: undefined
|
|
129
130
|
if ((await copyMember(reader, entry.size, { hasher, bytes })) < entry.size) {
|
|
130
|
-
throw
|
|
131
|
+
throw archiveError(`tar: truncated entry ${entry.path}`)
|
|
131
132
|
}
|
|
132
133
|
if (!isZero(await reader.read(padded(entry.size) - entry.size))) {
|
|
133
|
-
throw
|
|
134
|
+
throw archiveError(`tar: non-zero padding after ${entry.path}`)
|
|
134
135
|
}
|
|
135
136
|
return { size: entry.size, sha: hasher.digest(), bytes }
|
|
136
137
|
}
|
|
@@ -144,7 +145,7 @@ async function walkEntry(
|
|
|
144
145
|
const entry = readTarHeader(block)
|
|
145
146
|
if (entry.type === DIRECTORY_TYPE) {
|
|
146
147
|
if (entry.size !== START) {
|
|
147
|
-
throw
|
|
148
|
+
throw archiveError(`tar: directory with a size ${entry.path}`)
|
|
148
149
|
}
|
|
149
150
|
if (entry.path !== '') {
|
|
150
151
|
claimFolder(walk.claims, entry.path)
|
|
@@ -160,13 +161,13 @@ async function walkEntry(
|
|
|
160
161
|
async function nextBlock(reader: ChunkReader): Promise<Uint8Array> {
|
|
161
162
|
const block = await reader.read(BLOCK)
|
|
162
163
|
if (block.length < BLOCK) {
|
|
163
|
-
throw
|
|
164
|
+
throw archiveError('tar: truncated (no end-of-archive blocks)')
|
|
164
165
|
}
|
|
165
166
|
return block
|
|
166
167
|
}
|
|
167
168
|
|
|
168
|
-
/**
|
|
169
|
-
async function
|
|
169
|
+
/** `walkTar`, before its refusals (a bad decode among them) take `archive-invalid`. */
|
|
170
|
+
async function walkMembers(
|
|
170
171
|
tar: AsyncIterable<Uint8Array>,
|
|
171
172
|
ports: WalkPorts,
|
|
172
173
|
): Promise<number> {
|
|
@@ -181,12 +182,23 @@ async function walkTar(
|
|
|
181
182
|
return offset + (await assertEnd(reader))
|
|
182
183
|
}
|
|
183
184
|
if (entries === MAX_ENTRIES) {
|
|
184
|
-
throw
|
|
185
|
+
throw archiveError(`tar: too many entries, over ${String(MAX_ENTRIES)}`)
|
|
185
186
|
}
|
|
186
187
|
// oxlint-disable-next-line eslint/no-await-in-loop -- a tarball's entries come in order
|
|
187
188
|
offset += await walkEntry(reader, block, walk)
|
|
188
189
|
}
|
|
189
190
|
}
|
|
190
191
|
|
|
192
|
+
/** Walk a tarball once, refusing what `indexTar` refuses; resolves with its byte length. */
|
|
193
|
+
async function walkTar(
|
|
194
|
+
tar: AsyncIterable<Uint8Array>,
|
|
195
|
+
ports: WalkPorts,
|
|
196
|
+
): Promise<number> {
|
|
197
|
+
return await withIssueCodeAsync(
|
|
198
|
+
'archive-invalid',
|
|
199
|
+
async (): Promise<number> => await walkMembers(tar, ports),
|
|
200
|
+
)
|
|
201
|
+
}
|
|
202
|
+
|
|
191
203
|
export type { StreamHasher, StreamMember, WalkPorts }
|
|
192
204
|
export { walkTar }
|
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
|
|
18
18
|
import { SHARDFILE_ADMISSION_LIMITS as limits } from './admissionLimits.ts'
|
|
19
19
|
import { assertSectionSides } from './feature-sides.ts'
|
|
20
|
+
import { issueError, withIssueCode } from './issue-codes.ts'
|
|
20
21
|
import { parseOutputIndex, validateOutputSection } from './outputIndex.ts'
|
|
21
22
|
import {
|
|
22
23
|
admitAssets,
|
|
@@ -143,7 +144,10 @@ function collect(ports: StreamValidationPorts): Readonly<{
|
|
|
143
144
|
(keep && now + size > STREAM_KEPT_CAP) ||
|
|
144
145
|
(!keep && size > STREAM_BLOB_CAP)
|
|
145
146
|
) {
|
|
146
|
-
throw
|
|
147
|
+
throw issueError(
|
|
148
|
+
'stream-cap',
|
|
149
|
+
`stream: member over the streaming cap: ${path}`,
|
|
150
|
+
)
|
|
147
151
|
}
|
|
148
152
|
return keep || path.startsWith('blobs/')
|
|
149
153
|
}
|
|
@@ -214,22 +218,30 @@ function readSections(
|
|
|
214
218
|
replay: Readonly<Replay>,
|
|
215
219
|
// oxlint-disable-next-line typescript/prefer-readonly-parameter-types -- valibot's output type, only read
|
|
216
220
|
index: OutputIndex,
|
|
217
|
-
):
|
|
221
|
+
): Pick<StreamedProduct, 'sections' | 'warnings' | 'scripts'> {
|
|
218
222
|
const payloads = new Map<string, unknown>()
|
|
219
223
|
for (const entry of index.sections) {
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
+
withIssueCode('section-invalid', (): void => {
|
|
225
|
+
const value = readCanonicalJson(listedBytes(replay, entry))
|
|
226
|
+
assertSectionSides(entry.name, value)
|
|
227
|
+
validateOutputSection(entry.name, value)
|
|
228
|
+
payloads.set(entry.name, value)
|
|
229
|
+
})
|
|
224
230
|
}
|
|
225
|
-
const
|
|
231
|
+
const sides = {
|
|
226
232
|
paths: (): Iterable<string> => replay.members.keys(),
|
|
227
233
|
read: (path: string): Uint8Array | undefined =>
|
|
228
234
|
replay.members.get(path)?.bytes,
|
|
229
235
|
verify: (bytes: Readonly<Uint8Array>, expected: string): boolean =>
|
|
230
236
|
replay.hash(bytes) === expected,
|
|
231
|
-
}
|
|
232
|
-
|
|
237
|
+
}
|
|
238
|
+
const warnings = withIssueCode('script-side', (): string[] =>
|
|
239
|
+
assertScriptSides(payloads, index, sides),
|
|
240
|
+
)
|
|
241
|
+
const scripts = withIssueCode('script-admission', (): ScriptAdmissionRows =>
|
|
242
|
+
scriptAdmissionRows(payloads),
|
|
243
|
+
)
|
|
244
|
+
return { sections: payloads, warnings, scripts }
|
|
233
245
|
}
|
|
234
246
|
|
|
235
247
|
/** Stream-validate a stored shardfile object (see the file comment): `open` gives a fresh stream of it on each call.
|
|
@@ -246,9 +258,10 @@ async function validateShardfileStream(
|
|
|
246
258
|
listed: new Set(['validation.json']),
|
|
247
259
|
hash: ports.hash,
|
|
248
260
|
}
|
|
249
|
-
const index =
|
|
250
|
-
|
|
251
|
-
|
|
261
|
+
const index = withIssueCode('index-invalid', (): OutputIndex =>
|
|
262
|
+
readIndex(replay),
|
|
263
|
+
)
|
|
264
|
+
const { sections, warnings, scripts } = readSections(replay, index)
|
|
252
265
|
replayBeforePacks(replay, index, scripts)
|
|
253
266
|
const before = admitBeforeAssets(index, sections, scripts)
|
|
254
267
|
const later = await assetPasses({
|
package/src/format/product.ts
CHANGED
|
@@ -18,10 +18,15 @@ import { SCRIPT_LIMITS } from './scriptLimits.ts';
|
|
|
18
18
|
import { runtimeAccountedBytes } from "./runtimeCost.ts";
|
|
19
19
|
import { MemoryAdmission } from './memoryAdmission.ts';
|
|
20
20
|
import { nearPlaceholderRoots } from './nearPlaceholder.ts';
|
|
21
|
+
import type { IssueCodeOf } from './issue-codes.ts';
|
|
22
|
+
|
|
21
23
|
|
|
22
24
|
/** Actual costs derived from a bounded parser, never trusted from the author declaration. */
|
|
23
25
|
import { isNavigationData, navigationCost } from './navigationBytes.ts';
|
|
24
26
|
|
|
27
|
+
/** A performance issue's code and its metric words. */
|
|
28
|
+
type Named = readonly [code: IssueCodeOf<'performance'>, metric: string];
|
|
29
|
+
|
|
25
30
|
/** Admitted decoded CPU bytes, GPU bytes, triangle count and draw count charged to the asset budget. */
|
|
26
31
|
export interface AssetCost { decoded: number; gpu: number; triangles: number; draws: number }
|
|
27
32
|
const MAX_BYTES = 25_000_000;
|
|
@@ -176,7 +181,10 @@ export interface PerformanceObservations {
|
|
|
176
181
|
fuel: { p95: number; max: number; samples: number };
|
|
177
182
|
}
|
|
178
183
|
/** A report issue distinguishes tradeable memory targets from enforceable complete totals and execution budgets. */
|
|
179
|
-
export interface PerformanceIssue {
|
|
184
|
+
export interface PerformanceIssue {
|
|
185
|
+
/** SHARD-AUTHORING SA41: the stable code (issue-codes.ts, docs/VALIDATION-CODES.md); `metric` keeps its words. */
|
|
186
|
+
code: IssueCodeOf<'performance'>; metric: string; actual: number; limit: number; severity: 'warning' | 'refusal';
|
|
187
|
+
}
|
|
180
188
|
/** Build-time estimates retain units, their scope, and the measured execution samples they use. */
|
|
181
189
|
export interface PerformanceReport {
|
|
182
190
|
slug: string; policy: 'refuse' | 'warn'; pass: boolean;
|
|
@@ -219,20 +227,20 @@ const near = (tile: Shardfile['tiles'][number], x: number, z: number): boolean =
|
|
|
219
227
|
/** Cheap target grading before asset reads; format integrity and total-memory admission remain separate. */
|
|
220
228
|
export function performanceTargetIssues(source: Shardfile, policy: 'refuse' | 'warn' = 'refuse'): PerformanceIssue[] {
|
|
221
229
|
const issues: PerformanceIssue[] = [];
|
|
222
|
-
const check = (metric:
|
|
223
|
-
if (actual > limit) issues.push({ metric, actual, limit, severity: provisional || policy === 'warn' ? 'warning' : 'refusal' });
|
|
230
|
+
const check = ([code, metric]: Named, actual: number, limit: number, provisional = false): void => {
|
|
231
|
+
if (actual > limit) issues.push({ code, metric, actual, limit, severity: provisional || policy === 'warn' ? 'warning' : 'refusal' });
|
|
224
232
|
};
|
|
225
233
|
const libraryAllowance = source.budgets.library.downloadAllowance ?? C.library.compressed;
|
|
226
|
-
check('library download budget bytes', source.budgets.library.compressed, libraryAllowance);
|
|
227
|
-
check('critical download budget bytes', source.budgets.sim.compressed, C.sim.compressed);
|
|
228
|
-
check('library download bytes', totals(source, reportCardClosure(source, source.library)).bytes, libraryAllowance);
|
|
229
|
-
check('critical download bytes', totals(source, reportCardClosure(source, source.critical)).bytes, C.sim.compressed);
|
|
234
|
+
check(['perf-library-download-budget', 'library download budget bytes'], source.budgets.library.compressed, libraryAllowance);
|
|
235
|
+
check(['perf-critical-download-budget', 'critical download budget bytes'], source.budgets.sim.compressed, C.sim.compressed);
|
|
236
|
+
check(['perf-library-download', 'library download bytes'], totals(source, reportCardClosure(source, source.library)).bytes, libraryAllowance);
|
|
237
|
+
check(['perf-critical-download', 'critical download bytes'], totals(source, reportCardClosure(source, source.critical)).bytes, C.sim.compressed);
|
|
230
238
|
for (const tile of source.tiles) {
|
|
231
239
|
const cap = tile.lod === 0 ? C.l0 : C.l1, id = `tile ${tile.lod}/${tile.x}/${tile.z}`;
|
|
232
|
-
check(`${id} download bytes
|
|
240
|
+
check(['perf-tile-download', `${id} download bytes`], tile.compressed, cap.compressed, true); check(['perf-tile-draws', `${id} draws`], tile.draws, cap.draws, true); check(['perf-tile-triangles', `${id} triangles`], tile.triangles, cap.triangles, true);
|
|
233
241
|
}
|
|
234
242
|
if (source.far !== null) {
|
|
235
|
-
check('far download bytes', source.far.compressed, C.far.compressed); check('far draws', source.far.draws, C.far.draws); check('far triangles', source.far.triangles, C.far.triangles);
|
|
243
|
+
check(['perf-far-download', 'far download bytes'], source.far.compressed, C.far.compressed); check(['perf-far-draws', 'far draws'], source.far.draws, C.far.draws); check(['perf-far-triangles', 'far triangles'], source.far.triangles, C.far.triangles);
|
|
236
244
|
}
|
|
237
245
|
return issues;
|
|
238
246
|
}
|
|
@@ -293,18 +301,18 @@ export function performanceReport(source: Shardfile, observations: PerformanceOb
|
|
|
293
301
|
const commons = native.resident + Object.values(source.requires.commonsCosts).reduce((sum, row) => sum + row.decoded + row.gpu, 0);
|
|
294
302
|
const worst = worstContentCost(source, commons), spawn = source.spawn;
|
|
295
303
|
const memory = { near: nearbyContentCost(source, spawn.x, spawn.z, commons), worst, playingLimit: C.playing, loadingLimit: C.loading };
|
|
296
|
-
const issues: PerformanceIssue[] = memoryTargetWarnings(source).map(row => ({ metric: `${row.category} resident target (tradeable)`, actual: row.bytes, limit: row.target, severity: 'warning' }));
|
|
304
|
+
const issues: PerformanceIssue[] = memoryTargetWarnings(source).map(row => ({ code: 'perf-memory-target', metric: `${row.category} resident target (tradeable)`, actual: row.bytes, limit: row.target, severity: 'warning' }));
|
|
297
305
|
issues.push(...performanceTargetIssues(source, policy));
|
|
298
|
-
const check = (metric:
|
|
299
|
-
if (!Number.isFinite(actual) || actual < 0 || actual > limit) issues.push({ metric, actual, limit, severity: policy === 'warn' ? 'warning' : 'refusal' });
|
|
306
|
+
const check = ([code, metric]: Named, actual: number, limit: number): void => {
|
|
307
|
+
if (!Number.isFinite(actual) || actual < 0 || actual > limit) issues.push({ code, metric, actual, limit, severity: policy === 'warn' ? 'warning' : 'refusal' });
|
|
300
308
|
};
|
|
301
|
-
if (native.unmeasuredViewAndDownload) check('native view/download measurement required', 1, 0);
|
|
302
|
-
if (source.runtime !== null && source.runtime.cost === undefined) check('native resident measurement required', 1, 0);
|
|
309
|
+
if (native.unmeasuredViewAndDownload) check(['perf-native-view-unmeasured', 'native view/download measurement required'], 1, 0);
|
|
310
|
+
if (source.runtime !== null && source.runtime.cost === undefined) check(['perf-native-resident-unmeasured', 'native resident measurement required'], 1, 0);
|
|
303
311
|
// G433: while src/format/memoryGates.json is report-only the totals print as warnings and never refuse a build.
|
|
304
|
-
const total = (metric:
|
|
305
|
-
if (MEMORY_GATES.reportOnly && actual > limit) issues.push({ metric: `${metric} (report-only, G433)`, actual, limit, severity: 'warning' }); else check(metric, actual, limit);
|
|
312
|
+
const total = ([code, metric]: Named, actual: number, limit: number): void => {
|
|
313
|
+
if (MEMORY_GATES.reportOnly && actual > limit) issues.push({ code, metric: `${metric} (report-only, G433)`, actual, limit, severity: 'warning' }); else check([code, metric], actual, limit);
|
|
306
314
|
};
|
|
307
|
-
total('worst playing total bytes', worst.playing, C.playing); total('worst loading total bytes', worst.loading, C.loading);
|
|
315
|
+
total(['perf-playing-memory', 'worst playing total bytes'], worst.playing, C.playing); total(['perf-loading-memory', 'worst loading total bytes'], worst.loading, C.loading);
|
|
308
316
|
const library = reportCardClosure(source, source.library), critical = reportCardClosure(source, source.critical);
|
|
309
317
|
const tileRoots = source.tiles.flatMap(tile => tile.files), farRoots = source.far?.files ?? [], nearRoots = nearPlaceholderRoots(source.near);
|
|
310
318
|
const playable = reportCardClosure(source, [...source.critical, ...source.library, ...farRoots, ...source.tiles.filter(tile => tile.lod === 1 || near(tile, spawn.x, spawn.z)).flatMap(tile => tile.files)]);
|
|
@@ -324,11 +332,11 @@ export function performanceReport(source: Shardfile, observations: PerformanceOb
|
|
|
324
332
|
const graphs: PerformanceReport['graphs'] = [];
|
|
325
333
|
const scriptLimit = Math.min(source.serverBudget.tickMicros, 1_000_000 / 60 / 4);
|
|
326
334
|
for (const [group, data] of ([['scripts', observations.scripts], ['fuel', observations.fuel]] as const)) for (const [metric, value] of Object.entries(data)) {
|
|
327
|
-
if (!Number.isFinite(value) || value < 0) check(`invalid measured ${group}.${metric}
|
|
335
|
+
if (!Number.isFinite(value) || value < 0) check(['perf-script-sample-invalid', `invalid measured ${group}.${metric}`], value, 0);
|
|
328
336
|
}
|
|
329
|
-
check('script CPU p95 us/tick', observations.scripts.p95Micros, scriptLimit);
|
|
330
|
-
check('script fuel/tick', observations.fuel.max, SCRIPT_LIMITS.fuelPerTick);
|
|
331
|
-
if (observations.scripts.samples < 60 || observations.fuel.samples < 60) check('missing execution samples', 60, Math.min(observations.scripts.samples, observations.fuel.samples));
|
|
337
|
+
check(['perf-script-cpu', 'script CPU p95 us/tick'], observations.scripts.p95Micros, scriptLimit);
|
|
338
|
+
check(['perf-script-fuel', 'script fuel/tick'], observations.fuel.max, SCRIPT_LIMITS.fuelPerTick);
|
|
339
|
+
if (observations.scripts.samples < 60 || observations.fuel.samples < 60) check(['perf-script-samples-missing', 'missing execution samples'], 60, Math.min(observations.scripts.samples, observations.fuel.samples));
|
|
332
340
|
// An explicit cold-network estimate, not a measured phone loading claim. The guide names both assumptions.
|
|
333
341
|
const bytesPerSecond = 1_000_000, setupSeconds = 1;
|
|
334
342
|
return { slug: source.identity.slug, policy, pass: !issues.some(issue => issue.severity === 'refusal'), native, memory, views, graphs, scripts: observations, download,
|
|
@@ -19,6 +19,7 @@ import { assertSectionSides } from './feature-sides.ts';
|
|
|
19
19
|
import { assertScriptSides } from './script-sides.ts';
|
|
20
20
|
import { productRecordings } from './product-audio.ts';
|
|
21
21
|
import type { RecordingMeasure } from './audio-measure.ts';
|
|
22
|
+
import { issueError, withIssueCode as coded } from './issue-codes.ts';
|
|
22
23
|
|
|
23
24
|
/**
|
|
24
25
|
* P1-F3 format validation of a built shardfile product (format 2: `shard.json`, sections, blobs, packs): the schema,
|
|
@@ -108,23 +109,23 @@ export function outputBlobExtension(kind: OutputFile['kind'], bytes: Uint8Array)
|
|
|
108
109
|
export function assertProductTargets(source: Shardfile, policy: 'warn' | 'refuse'): void {
|
|
109
110
|
const issues = performanceTargetIssues(source, policy).filter(issue => issue.severity === 'refusal');
|
|
110
111
|
if (issues.length === 0) return;
|
|
111
|
-
throw
|
|
112
|
+
throw issueError('performance-targets', issues.map(issue => `REFUSAL: ${issue.metric}: ${issue.actual} > ${issue.limit}`).join('\n'));
|
|
112
113
|
}
|
|
113
114
|
|
|
114
115
|
/** Admit a whole shardfile's declared costs and its actual immutable bytes through the defining contracts. */
|
|
115
116
|
export function admitShardfileProduct(input: unknown, assets: ReadonlyMap<string, Uint8Array>, hash: (bytes: Uint8Array) => string,
|
|
116
117
|
policy: 'warn' | 'refuse' = 'refuse', scripts: ScriptAdmissionRows = []): Shardfile {
|
|
117
|
-
const source = parseShardfile(input);
|
|
118
|
-
preflightDeclaredCosts(source, offlineMemoryAdmission(policy));
|
|
119
|
-
assertProductTargets(source, policy);
|
|
120
|
-
return validateShardfileAssets(source, assets, hash, offlineMemoryAdmission(policy), undefined, scripts);
|
|
118
|
+
const source = coded('shard-schema', (): Shardfile => parseShardfile(input));
|
|
119
|
+
coded('declared-costs', (): void => { preflightDeclaredCosts(source, offlineMemoryAdmission(policy)); });
|
|
120
|
+
coded('performance-targets', (): void => { assertProductTargets(source, policy); });
|
|
121
|
+
return coded('asset-admission', (): Shardfile => validateShardfileAssets(source, assets, hash, offlineMemoryAdmission(policy), undefined, scripts));
|
|
121
122
|
}
|
|
122
123
|
|
|
123
124
|
/** Refuse world target rows that name items or quests the shardfile does not declare. */
|
|
124
125
|
export function checkWorldTargets(source: Shardfile, input: unknown): void {
|
|
125
126
|
if (input === undefined) return;
|
|
126
|
-
const errors = worldTargetRules(parseWorldTargets(input), source);
|
|
127
|
-
if (errors.length > 0) throw
|
|
127
|
+
const errors = coded('world-targets', (): string[] => worldTargetRules(parseWorldTargets(input), source));
|
|
128
|
+
if (errors.length > 0) throw issueError('world-targets', errors.join('; '));
|
|
128
129
|
}
|
|
129
130
|
|
|
130
131
|
function inspectBlob(file: OutputFile, bytes: Uint8Array, ports: ProductValidationPorts, script?: ScriptContract): void {
|
|
@@ -148,58 +149,88 @@ function inspectBlob(file: OutputFile, bytes: Uint8Array, ports: ProductValidati
|
|
|
148
149
|
if ((['decoded', 'gpu', 'triangles', 'draws'] as const).some(key => cost[key] > file[key])) throw new Error('Output blob understates parsed costs');
|
|
149
150
|
}
|
|
150
151
|
|
|
152
|
+
type ReadListed = (entry: Readonly<{ path: string; hash: string; bytes: number }>) => Uint8Array;
|
|
153
|
+
|
|
154
|
+
/** Each section: listed, canonical, a declared side, its schema (`section-invalid`). */
|
|
155
|
+
// oxlint-disable-next-line typescript/prefer-readonly-parameter-types -- valibot's output type, only read
|
|
156
|
+
function readSections(index: OutputIndex, read: ReadListed): Map<string, unknown> {
|
|
157
|
+
const payloads = new Map<string, unknown>();
|
|
158
|
+
for (const entry of index.sections) {
|
|
159
|
+
coded('section-invalid', (): void => { const value = readCanonicalJson(read(entry)); assertSectionSides(entry.name, value); validateOutputSection(entry.name, value); payloads.set(entry.name, value); });
|
|
160
|
+
}
|
|
161
|
+
return payloads;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/** J91: server files pass the same hash, size and schema checks; they stay out of the client's admitted assets. Then the
|
|
165
|
+
* declared KTX2 device costs against the verified bytes. */
|
|
166
|
+
// oxlint-disable-next-line typescript/prefer-readonly-parameter-types -- valibot's output type and the admitted bytes, only read
|
|
167
|
+
function checkServerAndKtx2(index: OutputIndex, read: ReadListed, assets: ReadonlyMap<string, Uint8Array>): void {
|
|
168
|
+
for (const entry of index.admission.server ?? []) {
|
|
169
|
+
const bytes = read(entry);
|
|
170
|
+
coded('server-file-invalid', (): void => {
|
|
171
|
+
if (entry.kind === 'json') {
|
|
172
|
+
if (validatePlatformFile(entry.name, entry.schema, readCanonicalJson(bytes)) === undefined) throw new Error(`Unregistered named JSON: ${entry.name}`);
|
|
173
|
+
} else if (outputBlobKind('binary', bytes) !== entry.kind) throw new Error('Output blob bytes differ from their declared kind');
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
coded('ktx2-costs', (): void => {
|
|
177
|
+
for (const [hash, cost] of Object.entries(index.ktx2Costs ?? {})) {
|
|
178
|
+
const bytes = assets.get(hash);
|
|
179
|
+
if (bytes === undefined || decoder().decode(validationSourceBytes(cost)) !== decoder().decode(validationSourceBytes(ktx2DeviceCosts(bytes)))) throw new Error('KTX2 device costs differ from verified bytes');
|
|
180
|
+
}
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/** C0 (J137): the cook file; a mount passes the same hash and size checks as any listed file (`cook-invalid`). */
|
|
185
|
+
// oxlint-disable-next-line typescript/prefer-readonly-parameter-types -- valibot's output type, only read
|
|
186
|
+
function readCook(index: OutputIndex, read: ReadListed, hash: (bytes: Uint8Array) => string): ReturnType<typeof readCookData> | undefined {
|
|
187
|
+
return coded('cook-invalid', (): ReturnType<typeof readCookData> | undefined => {
|
|
188
|
+
const cook = index.cook === undefined ? undefined : readCookData(readCanonicalJson(read(index.cook)));
|
|
189
|
+
requireCookPreparation(index, cook, hash);
|
|
190
|
+
for (const mount of cook?.mounts ?? []) {
|
|
191
|
+
if (!mountSharesBlob(mount, index.files)) {
|
|
192
|
+
if (mount.path !== cookMountPath(mount.url)) { throw new Error(`Mount path differs from its URL: ${mount.url}`); }
|
|
193
|
+
read(mount);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
return cook;
|
|
197
|
+
});
|
|
198
|
+
}
|
|
199
|
+
|
|
151
200
|
/** Validate a built product folder with format alone: the index and section schemas, the exact inventory (no unlisted,
|
|
152
201
|
* missing or changed file), every hash and size, byte kinds against declared kinds and costs, KTX2 device costs, pack
|
|
153
202
|
* members, registered named JSON, budgets and admission, world targets and the declared source hash. A `client/plan`
|
|
154
203
|
* kind with no declared side (SERVER-PARITY SS2, `feature-sides.ts`) is refused by name before its section schema runs, and
|
|
155
|
-
* a script run on the wrong side (SS7, `script-sides.ts`) once the sections are read.
|
|
204
|
+
* a script run on the wrong side (SS7, `script-sides.ts`) once the sections are read. A refusal carries the `code` of the
|
|
205
|
+
* check that refused (SA41, `issue-codes.ts`). */
|
|
156
206
|
export function validateShardfileProduct(files: ProductFiles, ports: ProductValidationPorts): ValidatedProduct {
|
|
157
207
|
const verify = ports.verify ?? ((bytes: Uint8Array, expected: string): boolean => ports.hash(bytes) === expected);
|
|
158
|
-
const index = parseOutputIndex(readCanonicalJson(files.read('shard.json', limits.sourceBytes)));
|
|
159
|
-
const listed = new Set(['shard.json', 'validation.json']),
|
|
160
|
-
const read = (entry:
|
|
208
|
+
const index = coded('index-invalid', (): OutputIndex => parseOutputIndex(readCanonicalJson(files.read('shard.json', limits.sourceBytes))));
|
|
209
|
+
const listed = new Set(['shard.json', 'validation.json']), { assets, keep } = assetStore(files.read, ports.readThrough === true);
|
|
210
|
+
const read: ReadListed = (entry): Uint8Array => coded('file-mismatch', (): Uint8Array => {
|
|
161
211
|
listed.add(entry.path);
|
|
162
212
|
const bytes = files.read(entry.path, entry.bytes);
|
|
163
213
|
if (bytes.length !== entry.bytes || !verify(bytes, entry.hash, true)) throw new Error(`Output hash or size mismatch: ${entry.path}`);
|
|
164
214
|
return bytes;
|
|
165
|
-
};
|
|
166
|
-
|
|
167
|
-
const scripts =
|
|
215
|
+
});
|
|
216
|
+
const payloads = readSections(index, read), warnings = coded('script-side', (): string[] => assertScriptSides(payloads, index, { ...files, verify }));
|
|
217
|
+
const scripts = coded('script-admission', (): ScriptAdmissionRows => scriptAdmissionRows(payloads));
|
|
218
|
+
const contracts = coded('script-admission', (): ReturnType<typeof decisionModuleContracts> => decisionModuleContracts(scripts, index.files));
|
|
168
219
|
for (const entry of index.files) {
|
|
169
220
|
const bytes = read({ path: entry.path, hash: entry.hash, bytes: entry.compressed });
|
|
170
|
-
inspectBlob(entry, bytes, ports, entry.kind === 'wasm' ? contracts.get(entry.hash) ?? { abi: 0 } : undefined);
|
|
221
|
+
coded('blob-invalid', (): void => { inspectBlob(entry, bytes, ports, entry.kind === 'wasm' ? contracts.get(entry.hash) ?? { abi: 0 } : undefined); });
|
|
171
222
|
keep(entry, bytes, index.requires.commons.includes(entry.hash));
|
|
172
223
|
}
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
if (entry.kind === 'json') {
|
|
177
|
-
if (validatePlatformFile(entry.name, entry.schema, readCanonicalJson(bytes)) === undefined) throw new Error(`Unregistered named JSON: ${entry.name}`);
|
|
178
|
-
} else if (outputBlobKind('binary', bytes) !== entry.kind) throw new Error('Output blob bytes differ from their declared kind');
|
|
179
|
-
}
|
|
180
|
-
for (const [hash, cost] of Object.entries(index.ktx2Costs ?? {})) {
|
|
181
|
-
const bytes = assets.get(hash);
|
|
182
|
-
if (bytes === undefined || decoder().decode(validationSourceBytes(cost)) !== decoder().decode(validationSourceBytes(ktx2DeviceCosts(bytes)))) throw new Error('KTX2 device costs differ from verified bytes');
|
|
183
|
-
}
|
|
184
|
-
checkPacks(index, new Set(files.paths()), { readFile: files.read, readListed: read, verify, hash: ports.hash });
|
|
185
|
-
// C0 (J137): a mount passes the same hash and size checks as any listed file; nothing else may sit under `mounts/`.
|
|
186
|
-
const cook = index.cook === undefined ? undefined : readCookData(readCanonicalJson(read(index.cook)));
|
|
187
|
-
requireCookPreparation(index, cook, ports.hash);
|
|
188
|
-
if (cook !== undefined) {
|
|
189
|
-
for (const mount of cook.mounts) {
|
|
190
|
-
if (!mountSharesBlob(mount, index.files)) {
|
|
191
|
-
if (mount.path !== cookMountPath(mount.url)) { throw new Error(`Mount path differs from its URL: ${mount.url}`); }
|
|
192
|
-
read(mount);
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
}
|
|
224
|
+
checkServerAndKtx2(index, read, assets);
|
|
225
|
+
coded('pack-invalid', (): void => { checkPacks(index, new Set(files.paths()), { readFile: files.read, readListed: read, verify, hash: ports.hash }); });
|
|
226
|
+
const cook = readCook(index, read, ports.hash);
|
|
196
227
|
for (const entry of index.runtime?.files ?? []) {
|
|
197
|
-
if (ports.runtimeImports === undefined) throw
|
|
198
|
-
ports.runtimeImports(entry.path, decoder().decode(read(entry)));
|
|
228
|
+
if (ports.runtimeImports === undefined) throw issueError('runtime-files', 'Copied runtime files need a runtime import checker');
|
|
229
|
+
coded('runtime-files', (): void => { ports.runtimeImports?.(entry.path, decoder().decode(read(entry))); });
|
|
199
230
|
}
|
|
200
|
-
for (const path of files.paths()) if (!listed.has(path)) throw
|
|
201
|
-
const shard = admitShardfileProduct(assembleOutputShardfile(index, payloads), assets, ports.hash, 'warn', scripts);
|
|
231
|
+
for (const path of files.paths()) if (!listed.has(path)) throw issueError('unlisted-file', `Unlisted output file: ${path}`);
|
|
232
|
+
const shard = admitShardfileProduct(coded('shard-schema', (): unknown => assembleOutputShardfile(index, payloads)), assets, ports.hash, 'warn', scripts);
|
|
202
233
|
checkWorldTargets(shard, payloads.get('world/targets'));
|
|
203
|
-
if (ports.hash(validationSourceBytes(shard)) !== index.build.sourceHash) throw
|
|
234
|
+
if (ports.hash(validationSourceBytes(shard)) !== index.build.sourceHash) throw issueError('source-hash', 'Output sections differ from the declared source hash');
|
|
204
235
|
return { index, shard, assets, sections: payloads, scripts, warnings, audio: productRecordings({ cook, read: files.read, shard, assets }) };
|
|
205
236
|
}
|
package/src/format/tar-paths.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// a hostile path never reaches a disk, on any host a shard is unpacked on.
|
|
3
3
|
|
|
4
4
|
import { SHARDFILE_ADMISSION_LIMITS } from './admissionLimits.ts'
|
|
5
|
+
import { archiveError } from './issue-codes.ts'
|
|
5
6
|
|
|
6
7
|
// `tar -C product .` adds directories, and a shard adds `shard.json` and
|
|
7
8
|
// `validation.json`, to the files its index may list.
|
|
@@ -57,7 +58,7 @@ function folded(path: string): string {
|
|
|
57
58
|
}
|
|
58
59
|
|
|
59
60
|
function clash(path: string): Error {
|
|
60
|
-
return
|
|
61
|
+
return archiveError(`tar: both a file and a folder: ${path}`)
|
|
61
62
|
}
|
|
62
63
|
|
|
63
64
|
function claimParents(
|
|
@@ -81,7 +82,7 @@ function claimParents(
|
|
|
81
82
|
function claimFile(claims: Readonly<TarClaims>, path: string): void {
|
|
82
83
|
const key = folded(path)
|
|
83
84
|
if (claims.files.has(key)) {
|
|
84
|
-
throw
|
|
85
|
+
throw archiveError(`tar: duplicate entry ${path}`)
|
|
85
86
|
}
|
|
86
87
|
if (claims.folders.has(key)) {
|
|
87
88
|
throw clash(path)
|