@wildshard/upload 0.0.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +26 -8
- package/src/cli/account/account.ts +199 -0
- package/src/cli/account/command.ts +193 -0
- package/src/cli/account/key.ts +154 -0
- package/src/cli/args.ts +100 -0
- package/src/cli/audio/audio.ts +39 -0
- package/src/cli/delete/delete.ts +258 -0
- package/src/cli/hero/hero.ts +163 -0
- package/src/cli/hero/scene-card.ts +46 -0
- package/src/cli/hero/stamp.ts +300 -0
- package/src/cli/report.ts +20 -0
- package/src/cli/shardfile/command.ts +49 -0
- package/src/cli/shardfile/shardfile.ts +192 -0
- package/src/cli/upload/api.ts +286 -0
- package/src/cli/upload/command.ts +239 -0
- package/src/cli/upload/completion.ts +80 -0
- package/src/cli/upload/compress.ts +55 -0
- package/src/cli/upload/hero-gate.ts +208 -0
- package/src/cli/upload/hero.ts +62 -0
- package/src/cli/upload/image-size.ts +89 -0
- package/src/cli/upload/placement.ts +132 -0
- package/src/cli/upload/provenance.ts +204 -0
- package/src/cli/upload/signature.ts +39 -0
- package/src/cli/upload/status-body.ts +41 -0
- package/src/cli/upload/status.ts +176 -0
- package/src/cli/upload/upload.ts +298 -0
- package/src/cli/upload/validate.ts +92 -0
- package/src/cli/upload/wire.ts +189 -0
- package/src/sharddev/draft.ts +11 -0
- package/src/sharddev/refusals.ts +329 -0
- package/src/sharddev/shardSource.ts +75 -0
- package/README.md +0 -3
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
import { createHash, randomUUID } from 'node:crypto'
|
|
2
|
+
import { readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'
|
|
3
|
+
import path from 'node:path'
|
|
4
|
+
|
|
5
|
+
import { shardFolderName } from '../../sharddev/shardSource.ts'
|
|
6
|
+
import { sceneCardUrls } from './scene-card.ts'
|
|
7
|
+
|
|
8
|
+
// A shard's hero (its SHARD SELECT card: thumb, portrait and landscape) is a
|
|
9
|
+
// picture of its world, so it goes stale when the world changes (Jake,
|
|
10
|
+
// 2026-11-12 17:14 Cancún: "you have to update the hero images for the
|
|
11
|
+
// templates"). Rendering it needs a real GPU and a served build of the exact
|
|
12
|
+
// candidate, which no cut step or CI runner has, so the cut refuses instead:
|
|
13
|
+
// `hero.json` beside `manifest.json` records the hash of the shard's content
|
|
14
|
+
// when the hero was captured, and `wildshard build` refuses a product whose
|
|
15
|
+
// source hashes differently now, whose hero files are not the ones stamped,
|
|
16
|
+
// or which does not carry the stamped hero files as blobs (built before the
|
|
17
|
+
// capture). `wildshard hero` captures and re-stamps.
|
|
18
|
+
|
|
19
|
+
const STAMP = 'hero.json'
|
|
20
|
+
const MANIFEST = 'manifest.json'
|
|
21
|
+
const SLOTS = ['thumb', 'portrait', 'landscape'] as const
|
|
22
|
+
// Generated, ignored, test or prose files are not the world a hero shows.
|
|
23
|
+
// The directories are skipped at the shard folder's top level only.
|
|
24
|
+
const SKIPPED_DIRECTORIES = new Set(
|
|
25
|
+
'.cache .git .worktrees dist node_modules public tmp'.split(' '),
|
|
26
|
+
)
|
|
27
|
+
const SKIPPED_SUFFIXES = '.generated.ts .test.ts .test.mjs .md .DS_Store'.split(
|
|
28
|
+
' ',
|
|
29
|
+
)
|
|
30
|
+
const NOT_FOUND = 'ENOENT'
|
|
31
|
+
const VALUE = 1
|
|
32
|
+
const NONE = 0
|
|
33
|
+
const INDENT = 2
|
|
34
|
+
const UNKNOWN_TIME = 'at an unknown time'
|
|
35
|
+
|
|
36
|
+
type HeroCondition = 'fresh' | 'generated' | 'stale' | 'unstamped'
|
|
37
|
+
|
|
38
|
+
interface HeroState {
|
|
39
|
+
readonly state: HeroCondition
|
|
40
|
+
/** The content hash now. */
|
|
41
|
+
readonly content: string
|
|
42
|
+
/** What the stamp recorded, if there is one. */
|
|
43
|
+
readonly stamped?: string | undefined
|
|
44
|
+
/** When the stamp says the hero was captured. */
|
|
45
|
+
readonly captured?: string | undefined
|
|
46
|
+
readonly curated?: boolean | undefined
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
interface HeroStamp {
|
|
50
|
+
readonly content: string
|
|
51
|
+
readonly captured: string
|
|
52
|
+
readonly base: string
|
|
53
|
+
readonly card: Readonly<Record<string, string>>
|
|
54
|
+
readonly camera?: unknown
|
|
55
|
+
/** `"curated": true`: hand-picked art, re-stamped, never recaptured. */
|
|
56
|
+
readonly curated?: true
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function sha256(bytes: Readonly<Uint8Array> | string): string {
|
|
60
|
+
return createHash('sha256').update(bytes).digest('hex')
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function isMissing(error: unknown): boolean {
|
|
64
|
+
return error instanceof Error && 'code' in error && error.code === NOT_FOUND
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
async function readJson(file: string): Promise<unknown> {
|
|
68
|
+
try {
|
|
69
|
+
return JSON.parse(await readFile(file, 'utf8')) as unknown
|
|
70
|
+
} catch (error: unknown) {
|
|
71
|
+
if (isMissing(error)) {
|
|
72
|
+
return undefined
|
|
73
|
+
}
|
|
74
|
+
throw error
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** The stamp, or undefined when it is missing or unreadable (unstamped). */
|
|
79
|
+
async function readStamp(shard: string): Promise<unknown> {
|
|
80
|
+
try {
|
|
81
|
+
return await readJson(path.join(shard, STAMP))
|
|
82
|
+
} catch (error: unknown) {
|
|
83
|
+
if (error instanceof SyntaxError) {
|
|
84
|
+
return undefined
|
|
85
|
+
}
|
|
86
|
+
throw error
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function field(value: unknown, key: string): unknown {
|
|
91
|
+
if (typeof value !== 'object' || value === null) {
|
|
92
|
+
return undefined
|
|
93
|
+
}
|
|
94
|
+
return Object.entries(value).find(
|
|
95
|
+
([name]: readonly [string, unknown]): boolean => name === key,
|
|
96
|
+
)?.[VALUE]
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** The manifest's card urls, by slot; a scene shard's captured card files (F28). */
|
|
100
|
+
async function cardUrls(
|
|
101
|
+
shard: string,
|
|
102
|
+
): Promise<Readonly<Record<string, string>>> {
|
|
103
|
+
const manifest = await readJson(path.join(shard, MANIFEST))
|
|
104
|
+
const card = field(manifest, 'card')
|
|
105
|
+
const urls: Record<string, string> = {}
|
|
106
|
+
for (const slot of SLOTS) {
|
|
107
|
+
const url = field(card, slot)
|
|
108
|
+
if (typeof url === 'string') {
|
|
109
|
+
urls[slot] = url
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return manifest === undefined ? await sceneCardUrls(shard) : urls
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* The files a card url names inside this shard folder: `/src/shards/<folder>/x`
|
|
117
|
+
* or a path relative to the folder. A `data:` card names none.
|
|
118
|
+
*/
|
|
119
|
+
function cardFile(shard: string, url: string): string | undefined {
|
|
120
|
+
if (url.startsWith('data:') || url.includes('://')) {
|
|
121
|
+
return undefined
|
|
122
|
+
}
|
|
123
|
+
// A src/ shard source (J317) is named by its manifest slug, as the build's image root is (scripts/wildshard.mjs).
|
|
124
|
+
const prefix = `/src/shards/${shardFolderName(shard)}/`
|
|
125
|
+
const relative = url.startsWith(prefix) ? url.slice(prefix.length) : url
|
|
126
|
+
const file = path.resolve(shard, relative.replace(/^\/+/u, ''))
|
|
127
|
+
return file.startsWith(`${path.resolve(shard)}${path.sep}`) ? file : undefined
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** The hero file each card slot names; a `data:` slot names none. */
|
|
131
|
+
async function heroSlotFiles(
|
|
132
|
+
shard: string,
|
|
133
|
+
): Promise<Readonly<Record<string, string>>> {
|
|
134
|
+
const files: Record<string, string> = {}
|
|
135
|
+
for (const [slot, url] of Object.entries(await cardUrls(shard))) {
|
|
136
|
+
const file = cardFile(path.resolve(shard), url)
|
|
137
|
+
if (file !== undefined) {
|
|
138
|
+
files[slot] = file
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
return files
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
async function heroFiles(shard: string): Promise<ReadonlySet<string>> {
|
|
145
|
+
return new Set(Object.values(await heroSlotFiles(shard)))
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
async function contentFiles(
|
|
149
|
+
directory: string,
|
|
150
|
+
skip: ReadonlySet<string>,
|
|
151
|
+
top = true,
|
|
152
|
+
): Promise<readonly string[]> {
|
|
153
|
+
const entries = await readdir(directory, { withFileTypes: true })
|
|
154
|
+
const nested = await Promise.all(
|
|
155
|
+
entries.map(
|
|
156
|
+
async (
|
|
157
|
+
entry: Readonly<{
|
|
158
|
+
name: string
|
|
159
|
+
isDirectory: () => boolean
|
|
160
|
+
isFile: () => boolean
|
|
161
|
+
}>,
|
|
162
|
+
): Promise<readonly string[]> => {
|
|
163
|
+
const file = path.join(directory, entry.name)
|
|
164
|
+
if (entry.isDirectory()) {
|
|
165
|
+
return top && SKIPPED_DIRECTORIES.has(entry.name)
|
|
166
|
+
? []
|
|
167
|
+
: await contentFiles(file, skip, false)
|
|
168
|
+
}
|
|
169
|
+
const ignored =
|
|
170
|
+
!entry.isFile() ||
|
|
171
|
+
skip.has(file) ||
|
|
172
|
+
SKIPPED_SUFFIXES.some((suffix: string): boolean =>
|
|
173
|
+
entry.name.endsWith(suffix),
|
|
174
|
+
)
|
|
175
|
+
return ignored ? [] : [file]
|
|
176
|
+
},
|
|
177
|
+
),
|
|
178
|
+
)
|
|
179
|
+
return nested.flat()
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* One hash over every content file's folder-relative path and bytes: the
|
|
184
|
+
* shard as a hero would show it. The hero files and the stamp are left out,
|
|
185
|
+
* so a capture never makes its own hero stale.
|
|
186
|
+
*/
|
|
187
|
+
async function heroContentHash(shard: string): Promise<string> {
|
|
188
|
+
const root = path.resolve(shard)
|
|
189
|
+
const skip = new Set([...(await heroFiles(root)), path.join(root, STAMP)])
|
|
190
|
+
const found = await contentFiles(root, skip)
|
|
191
|
+
const rows = await Promise.all(
|
|
192
|
+
found.map(async (file: string): Promise<string> => {
|
|
193
|
+
const relative = path.relative(root, file).split(path.sep).join('/')
|
|
194
|
+
return `${relative}\0${sha256(await readFile(file))}\n`
|
|
195
|
+
}),
|
|
196
|
+
)
|
|
197
|
+
return sha256(rows.toSorted().join(''))
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
function textOf(value: unknown): string | undefined {
|
|
201
|
+
return typeof value === 'string' ? value : undefined
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/** Each hero file's folder-relative path and hash, for the stamp. */
|
|
205
|
+
async function cardHashes(
|
|
206
|
+
shard: string,
|
|
207
|
+
): Promise<Readonly<Record<string, string>>> {
|
|
208
|
+
const files = [...(await heroFiles(shard))]
|
|
209
|
+
const rows = await Promise.all(
|
|
210
|
+
files.map(async (file: string): Promise<readonly [string, string]> => [
|
|
211
|
+
path.relative(shard, file).split(path.sep).join('/'),
|
|
212
|
+
sha256(await readFile(file)),
|
|
213
|
+
]),
|
|
214
|
+
)
|
|
215
|
+
return Object.fromEntries(rows)
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/** Whether the stamped hero files are the ones on disk now. */
|
|
219
|
+
function sameCard(
|
|
220
|
+
stamped: unknown,
|
|
221
|
+
current: Readonly<Record<string, string>>,
|
|
222
|
+
): boolean {
|
|
223
|
+
const names = Object.keys(current)
|
|
224
|
+
return (
|
|
225
|
+
names.length === Object.keys(stamped ?? {}).length &&
|
|
226
|
+
names.every(
|
|
227
|
+
(name: string): boolean => field(stamped, name) === current[name],
|
|
228
|
+
)
|
|
229
|
+
)
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/** Whether the shard's captured hero is as new as its content. */
|
|
233
|
+
async function heroState(shard: string): Promise<HeroState> {
|
|
234
|
+
const content = await heroContentHash(shard)
|
|
235
|
+
const files = await heroFiles(shard)
|
|
236
|
+
if (files.size === NONE) {
|
|
237
|
+
return { state: 'generated', content }
|
|
238
|
+
}
|
|
239
|
+
const stamp = await readStamp(shard)
|
|
240
|
+
const curated = field(stamp, 'curated') === true
|
|
241
|
+
const stamped = field(stamp, 'content')
|
|
242
|
+
if (typeof stamped !== 'string') {
|
|
243
|
+
return { state: 'unstamped', content, curated }
|
|
244
|
+
}
|
|
245
|
+
return {
|
|
246
|
+
state:
|
|
247
|
+
stamped === content &&
|
|
248
|
+
sameCard(field(stamp, 'card'), await cardHashes(shard))
|
|
249
|
+
? 'fresh'
|
|
250
|
+
: 'stale',
|
|
251
|
+
content,
|
|
252
|
+
stamped,
|
|
253
|
+
captured: textOf(field(stamp, 'captured')),
|
|
254
|
+
curated,
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** Record that the hero files now show this content (`wildshard hero`). */
|
|
259
|
+
async function writeHeroStamp(
|
|
260
|
+
shard: string,
|
|
261
|
+
options: Readonly<{ base: string; captured?: string; camera?: unknown }>,
|
|
262
|
+
): Promise<HeroStamp> {
|
|
263
|
+
const previous = await readStamp(shard)
|
|
264
|
+
const camera = options.camera ?? field(previous, 'camera')
|
|
265
|
+
const curated = field(previous, 'curated') === true
|
|
266
|
+
const stamp: HeroStamp = {
|
|
267
|
+
content: await heroContentHash(shard),
|
|
268
|
+
captured: options.captured ?? new Date().toISOString(),
|
|
269
|
+
base: options.base,
|
|
270
|
+
card: await cardHashes(shard),
|
|
271
|
+
...(camera === undefined ? {} : { camera }),
|
|
272
|
+
...(curated ? { curated: true } : {}),
|
|
273
|
+
}
|
|
274
|
+
const partial = path.join(shard, `${STAMP}.${randomUUID()}.partial`)
|
|
275
|
+
try {
|
|
276
|
+
await writeFile(partial, `${JSON.stringify(stamp, undefined, INDENT)}\n`)
|
|
277
|
+
await rename(partial, path.join(shard, STAMP))
|
|
278
|
+
} finally {
|
|
279
|
+
await rm(partial, { force: true })
|
|
280
|
+
}
|
|
281
|
+
return stamp
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
export {
|
|
285
|
+
type HeroState,
|
|
286
|
+
type HeroStamp,
|
|
287
|
+
MANIFEST,
|
|
288
|
+
STAMP,
|
|
289
|
+
NONE,
|
|
290
|
+
UNKNOWN_TIME,
|
|
291
|
+
cardHashes,
|
|
292
|
+
field,
|
|
293
|
+
heroContentHash,
|
|
294
|
+
heroFiles,
|
|
295
|
+
heroSlotFiles,
|
|
296
|
+
heroState,
|
|
297
|
+
isMissing,
|
|
298
|
+
readJson,
|
|
299
|
+
writeHeroStamp,
|
|
300
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { refusalCode, refusalHint, refusalNote } from '../sharddev/refusals.ts'
|
|
2
|
+
|
|
3
|
+
import type { RefusalStep } from '../sharddev/refusals.ts'
|
|
4
|
+
|
|
5
|
+
/** One line of a command's report on stdout. */
|
|
6
|
+
function reportLine(line: string): void {
|
|
7
|
+
process.stdout.write(`${line}\n`)
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
/** A refusal on stderr, then its rule, fix and GUIDE.md link (SA21). `refusal` is the words, or the error: its message,
|
|
11
|
+
* and its shard-format code names the rule when it has one (SA41). */
|
|
12
|
+
function refuse(step: RefusalStep, prefix: string, refusal: unknown): void {
|
|
13
|
+
const reason = refusal instanceof Error ? refusal.message : String(refusal)
|
|
14
|
+
const hint = refusalHint(
|
|
15
|
+
refusalNote(step, reason, { code: refusalCode(refusal) }),
|
|
16
|
+
)
|
|
17
|
+
process.stderr.write(`sharddev: ${prefix}${reason}\n ${hint}\n`)
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export { refuse, reportLine }
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { refuse, reportLine } from '../report.ts'
|
|
2
|
+
import { writeShardfile } from './shardfile.ts'
|
|
3
|
+
|
|
4
|
+
// `wildshard build <folder> [file]` and `wildshard pack`, its deprecated
|
|
5
|
+
// alias (SHARD-AUTHORING SA3, J170): an author build writes its shard file
|
|
6
|
+
// itself; these write the shard file of a folder already built.
|
|
7
|
+
|
|
8
|
+
const EXIT_OK = 0
|
|
9
|
+
const EXIT_USAGE = 1
|
|
10
|
+
const EXIT_FAILED = 3
|
|
11
|
+
// Refusals name the shard file step's rules, whichever verb ran it.
|
|
12
|
+
const STEP = 'shardfile'
|
|
13
|
+
|
|
14
|
+
const PACK_DEPRECATED =
|
|
15
|
+
'sharddev: pack is deprecated: `wildshard build` writes the shard file (`wildshard build <folder> [file]` for a built folder)\n'
|
|
16
|
+
|
|
17
|
+
/** Write a built folder's shard file, to `out` or beside the folder. */
|
|
18
|
+
async function runShardfile(
|
|
19
|
+
step: 'build' | 'pack',
|
|
20
|
+
dir: string | undefined,
|
|
21
|
+
out: string | undefined,
|
|
22
|
+
): Promise<number> {
|
|
23
|
+
if (dir === undefined) {
|
|
24
|
+
refuse(STEP, '', `${step} needs a shard file folder`)
|
|
25
|
+
return EXIT_USAGE
|
|
26
|
+
}
|
|
27
|
+
try {
|
|
28
|
+
await writeShardfile({
|
|
29
|
+
dir,
|
|
30
|
+
report: reportLine,
|
|
31
|
+
...(out === undefined ? {} : { out }),
|
|
32
|
+
})
|
|
33
|
+
return EXIT_OK
|
|
34
|
+
} catch (error: unknown) {
|
|
35
|
+
refuse(STEP, `${step} failed: `, error)
|
|
36
|
+
return EXIT_FAILED
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** `pack <folder> [--out <file>]`: one line naming `build`, then the same step. */
|
|
41
|
+
async function runPack(
|
|
42
|
+
dir: string | undefined,
|
|
43
|
+
out: string | undefined,
|
|
44
|
+
): Promise<number> {
|
|
45
|
+
process.stderr.write(PACK_DEPRECATED)
|
|
46
|
+
return await runShardfile('pack', dir, out)
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export { runPack, runShardfile }
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
import { createHash, randomUUID } from 'node:crypto'
|
|
2
|
+
import { readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'
|
|
3
|
+
import path from 'node:path'
|
|
4
|
+
|
|
5
|
+
import { installFormatHook } from '@wildshard/format/formatHook'
|
|
6
|
+
|
|
7
|
+
import { DRAFT_SDK, draftMessage } from '../../sharddev/draft.ts'
|
|
8
|
+
import { refusalHint, refusalNote } from '../../sharddev/refusals.ts'
|
|
9
|
+
import { audioLines } from '../audio/audio.ts'
|
|
10
|
+
import { assertHeroCurrent } from '../hero/hero.ts'
|
|
11
|
+
import { validateShardfile } from '../upload/validate.ts'
|
|
12
|
+
|
|
13
|
+
// The shard file (C0, J137): `wildshard build` (SHARD-AUTHORING SA3, J170)
|
|
14
|
+
// turns the shard file folder it built into the one tarball `api-server`
|
|
15
|
+
// takes, so a shard file stands alone. It writes ustar through format's
|
|
16
|
+
// `writeTar` and runs format's validator on the tarball itself, the check the
|
|
17
|
+
// server runs at commit: what this writes, the server admits once it is
|
|
18
|
+
// prepared. A fresh build is a draft (sdk 0.2) that the server refuses, so
|
|
19
|
+
// this warns about one (J198). `wildshard pack` is the deprecated alias.
|
|
20
|
+
//
|
|
21
|
+
// The same folder writes the same bytes, so the same hash, and a re-upload
|
|
22
|
+
// sends nothing.
|
|
23
|
+
|
|
24
|
+
const BYTES_PER_MB = 1_000_000
|
|
25
|
+
const ONE_DECIMAL = 1
|
|
26
|
+
// The cap in api-server's D31, on a shardfile stored or decoded. J137 raises
|
|
27
|
+
// it only if a real shard needs it, so going over is news and not a refusal:
|
|
28
|
+
// the server's cap is the one that counts.
|
|
29
|
+
const SERVER_CAP_BYTES = 450_000_000
|
|
30
|
+
|
|
31
|
+
/** What `readdir` says about one entry. */
|
|
32
|
+
interface ProductEntry {
|
|
33
|
+
readonly name: string
|
|
34
|
+
readonly parentPath: string
|
|
35
|
+
readonly isDirectory: () => boolean
|
|
36
|
+
readonly isFile: () => boolean
|
|
37
|
+
readonly isSymbolicLink: () => boolean
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
interface ShardfileOptions {
|
|
41
|
+
/** The built shard file folder: `shard.json` at its root. */
|
|
42
|
+
readonly dir: string
|
|
43
|
+
/** Where the shard file goes; `<slug>.shardfile` beside the folder if unset. */
|
|
44
|
+
readonly out?: string
|
|
45
|
+
/** The shard source the hero check reads; `<root>/src/shards/<name>` for `<root>/out/<name>` if unset. */
|
|
46
|
+
readonly source?: string
|
|
47
|
+
/** Receives one line per step. */
|
|
48
|
+
readonly report: (line: string) => void
|
|
49
|
+
/** False: the caller reports a draft itself (`check`, in its verdict). */
|
|
50
|
+
readonly draftWarning?: boolean
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
interface Tarball {
|
|
54
|
+
readonly tarball: Uint8Array<ArrayBuffer>
|
|
55
|
+
readonly fileCount: number
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
interface ShardfileOutcome {
|
|
59
|
+
readonly slug: string
|
|
60
|
+
readonly file: string
|
|
61
|
+
readonly hash: string
|
|
62
|
+
readonly bytes: number
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function sha256(bytes: Readonly<Uint8Array>): string {
|
|
66
|
+
return createHash('sha256').update(bytes).digest('hex')
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function mb(bytes: number): string {
|
|
70
|
+
return `${(bytes / BYTES_PER_MB).toFixed(ONE_DECIMAL)} MB`
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// A cook pack repeats files that ship at their own paths: format rebuilds and
|
|
74
|
+
// checks it from them, so a shard file leaves it out (C0). 109 MB of Pine Hollow.
|
|
75
|
+
const PACKS = 'blobs/pack/'
|
|
76
|
+
|
|
77
|
+
function productPath(root: string, entry: ProductEntry): string {
|
|
78
|
+
const file = path.join(entry.parentPath, entry.name)
|
|
79
|
+
if (entry.isSymbolicLink() || !entry.isFile()) {
|
|
80
|
+
throw new Error(`not a regular file: ${file}`)
|
|
81
|
+
}
|
|
82
|
+
return path.relative(root, file).split(path.sep).join('/')
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** Every file in the product by `/`-separated path; a link is refused. */
|
|
86
|
+
async function readProduct(
|
|
87
|
+
root: string,
|
|
88
|
+
): Promise<ReadonlyMap<string, Uint8Array>> {
|
|
89
|
+
const entries = await readdir(root, { recursive: true, withFileTypes: true })
|
|
90
|
+
const paths = entries
|
|
91
|
+
.filter((entry: ProductEntry): boolean => !entry.isDirectory())
|
|
92
|
+
.map((entry: ProductEntry): string => productPath(root, entry))
|
|
93
|
+
.filter((file: string): boolean => !file.startsWith(PACKS))
|
|
94
|
+
const pairs = await Promise.all(
|
|
95
|
+
paths.map(async (file: string): Promise<readonly [string, Uint8Array]> => {
|
|
96
|
+
const bytes = await readFile(path.join(root, file))
|
|
97
|
+
return [
|
|
98
|
+
file,
|
|
99
|
+
new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.length),
|
|
100
|
+
]
|
|
101
|
+
}),
|
|
102
|
+
)
|
|
103
|
+
return new Map(pairs)
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
async function tarOf(dir: string): Promise<Tarball> {
|
|
107
|
+
installFormatHook()
|
|
108
|
+
const { writeTar } = await import('@wildshard/format/tar')
|
|
109
|
+
const files = await readProduct(dir)
|
|
110
|
+
return { tarball: writeTar(files), fileCount: files.size }
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function reportOverCap(options: ShardfileOptions, bytes: number): void {
|
|
114
|
+
if (bytes > SERVER_CAP_BYTES) {
|
|
115
|
+
options.report(
|
|
116
|
+
`warning: ${mb(bytes)} is over api-server's ${mb(SERVER_CAP_BYTES)} cap (its D34)`,
|
|
117
|
+
)
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* J198: a fresh build is a draft (sdk 0.2) that upload and api-server refuse;
|
|
123
|
+
* build still writes it, as `wildshard prepare` works on the folder.
|
|
124
|
+
*/
|
|
125
|
+
function reportDraft(
|
|
126
|
+
options: ShardfileOptions,
|
|
127
|
+
slug: string,
|
|
128
|
+
sdk: string,
|
|
129
|
+
): void {
|
|
130
|
+
if (sdk === DRAFT_SDK && options.draftWarning !== false) {
|
|
131
|
+
const message = draftMessage(slug)
|
|
132
|
+
options.report(`warning: ${message}`)
|
|
133
|
+
options.report(` ${refusalHint(refusalNote('shardfile', message))}`)
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** Never leaves half a tarball under the final name. */
|
|
138
|
+
async function writeAtomically(
|
|
139
|
+
file: string,
|
|
140
|
+
bytes: Readonly<Uint8Array>,
|
|
141
|
+
): Promise<void> {
|
|
142
|
+
const partial = `${file}.${randomUUID()}.partial`
|
|
143
|
+
try {
|
|
144
|
+
await writeFile(partial, bytes)
|
|
145
|
+
await rename(partial, file)
|
|
146
|
+
} finally {
|
|
147
|
+
await rm(partial, { force: true })
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/** Check the hero is as new as the content, tar the folder, check the tarball. */
|
|
152
|
+
async function checkedTarball(
|
|
153
|
+
options: ShardfileOptions,
|
|
154
|
+
): Promise<Readonly<{ tarball: Uint8Array<ArrayBuffer>; slug: string }>> {
|
|
155
|
+
await assertHeroCurrent(options.dir, options.report, options.source)
|
|
156
|
+
const { tarball, fileCount } = await tarOf(options.dir)
|
|
157
|
+
const { slug, name, audio, sdk } = await validateShardfile(tarball)
|
|
158
|
+
options.report(`valid: ${slug} (${name}), ${String(fileCount)} files`)
|
|
159
|
+
reportDraft(options, slug, sdk)
|
|
160
|
+
// AUDIO AU5: build prints the audio rules as AMBER and still writes the
|
|
161
|
+
// shard file; only upload and api-server's admission refuse a red.
|
|
162
|
+
for (const line of await audioLines(audio)) {
|
|
163
|
+
options.report(line)
|
|
164
|
+
}
|
|
165
|
+
reportOverCap(options, tarball.length)
|
|
166
|
+
return { tarball, slug }
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/** The shard file's default name: `<slug>.shardfile` beside its folder (SA3). */
|
|
170
|
+
function shardfilePath(dir: string, slug: string): string {
|
|
171
|
+
return path.join(path.dirname(path.resolve(dir)), `${slug}.shardfile`)
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/** Check the folder, then write its shard file; a refusal writes none. */
|
|
175
|
+
async function writeShardfile(
|
|
176
|
+
options: ShardfileOptions,
|
|
177
|
+
): Promise<ShardfileOutcome> {
|
|
178
|
+
const { tarball, slug } = await checkedTarball(options)
|
|
179
|
+
const bytes = tarball.length
|
|
180
|
+
const file = options.out ?? shardfilePath(options.dir, slug)
|
|
181
|
+
const hash = sha256(tarball)
|
|
182
|
+
await writeAtomically(file, tarball)
|
|
183
|
+
options.report(`shard file: ${file}, ${mb(bytes)} (${hash})`)
|
|
184
|
+
return { slug, file, hash, bytes }
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
export {
|
|
188
|
+
type ShardfileOptions,
|
|
189
|
+
type ShardfileOutcome,
|
|
190
|
+
shardfilePath,
|
|
191
|
+
writeShardfile,
|
|
192
|
+
}
|