@wildshard/upload 0.0.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,300 @@
1
+ import { createHash, randomUUID } from 'node:crypto'
2
+ import { readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'
3
+ import path from 'node:path'
4
+
5
+ import { shardFolderName } from '../../sharddev/shardSource.ts'
6
+ import { sceneCardUrls } from './scene-card.ts'
7
+
8
+ // A shard's hero (its SHARD SELECT card: thumb, portrait and landscape) is a
9
+ // picture of its world, so it goes stale when the world changes (Jake,
10
+ // 2026-11-12 17:14 Cancún: "you have to update the hero images for the
11
+ // templates"). Rendering it needs a real GPU and a served build of the exact
12
+ // candidate, which no cut step or CI runner has, so the cut refuses instead:
13
+ // `hero.json` beside `manifest.json` records the hash of the shard's content
14
+ // when the hero was captured, and `wildshard build` refuses a product whose
15
+ // source hashes differently now, whose hero files are not the ones stamped,
16
+ // or which does not carry the stamped hero files as blobs (built before the
17
+ // capture). `wildshard hero` captures and re-stamps.
18
+
19
+ const STAMP = 'hero.json'
20
+ const MANIFEST = 'manifest.json'
21
+ const SLOTS = ['thumb', 'portrait', 'landscape'] as const
22
+ // Generated, ignored, test or prose files are not the world a hero shows.
23
+ // The directories are skipped at the shard folder's top level only.
24
+ const SKIPPED_DIRECTORIES = new Set(
25
+ '.cache .git .worktrees dist node_modules public tmp'.split(' '),
26
+ )
27
+ const SKIPPED_SUFFIXES = '.generated.ts .test.ts .test.mjs .md .DS_Store'.split(
28
+ ' ',
29
+ )
30
+ const NOT_FOUND = 'ENOENT'
31
+ const VALUE = 1
32
+ const NONE = 0
33
+ const INDENT = 2
34
+ const UNKNOWN_TIME = 'at an unknown time'
35
+
36
+ type HeroCondition = 'fresh' | 'generated' | 'stale' | 'unstamped'
37
+
38
+ interface HeroState {
39
+ readonly state: HeroCondition
40
+ /** The content hash now. */
41
+ readonly content: string
42
+ /** What the stamp recorded, if there is one. */
43
+ readonly stamped?: string | undefined
44
+ /** When the stamp says the hero was captured. */
45
+ readonly captured?: string | undefined
46
+ readonly curated?: boolean | undefined
47
+ }
48
+
49
+ interface HeroStamp {
50
+ readonly content: string
51
+ readonly captured: string
52
+ readonly base: string
53
+ readonly card: Readonly<Record<string, string>>
54
+ readonly camera?: unknown
55
+ /** `"curated": true`: hand-picked art, re-stamped, never recaptured. */
56
+ readonly curated?: true
57
+ }
58
+
59
+ function sha256(bytes: Readonly<Uint8Array> | string): string {
60
+ return createHash('sha256').update(bytes).digest('hex')
61
+ }
62
+
63
+ function isMissing(error: unknown): boolean {
64
+ return error instanceof Error && 'code' in error && error.code === NOT_FOUND
65
+ }
66
+
67
+ async function readJson(file: string): Promise<unknown> {
68
+ try {
69
+ return JSON.parse(await readFile(file, 'utf8')) as unknown
70
+ } catch (error: unknown) {
71
+ if (isMissing(error)) {
72
+ return undefined
73
+ }
74
+ throw error
75
+ }
76
+ }
77
+
78
+ /** The stamp, or undefined when it is missing or unreadable (unstamped). */
79
+ async function readStamp(shard: string): Promise<unknown> {
80
+ try {
81
+ return await readJson(path.join(shard, STAMP))
82
+ } catch (error: unknown) {
83
+ if (error instanceof SyntaxError) {
84
+ return undefined
85
+ }
86
+ throw error
87
+ }
88
+ }
89
+
90
+ function field(value: unknown, key: string): unknown {
91
+ if (typeof value !== 'object' || value === null) {
92
+ return undefined
93
+ }
94
+ return Object.entries(value).find(
95
+ ([name]: readonly [string, unknown]): boolean => name === key,
96
+ )?.[VALUE]
97
+ }
98
+
99
+ /** The manifest's card urls, by slot; a scene shard's captured card files (F28). */
100
+ async function cardUrls(
101
+ shard: string,
102
+ ): Promise<Readonly<Record<string, string>>> {
103
+ const manifest = await readJson(path.join(shard, MANIFEST))
104
+ const card = field(manifest, 'card')
105
+ const urls: Record<string, string> = {}
106
+ for (const slot of SLOTS) {
107
+ const url = field(card, slot)
108
+ if (typeof url === 'string') {
109
+ urls[slot] = url
110
+ }
111
+ }
112
+ return manifest === undefined ? await sceneCardUrls(shard) : urls
113
+ }
114
+
115
+ /**
116
+ * The files a card url names inside this shard folder: `/src/shards/<folder>/x`
117
+ * or a path relative to the folder. A `data:` card names none.
118
+ */
119
+ function cardFile(shard: string, url: string): string | undefined {
120
+ if (url.startsWith('data:') || url.includes('://')) {
121
+ return undefined
122
+ }
123
+ // A src/ shard source (J317) is named by its manifest slug, as the build's image root is (scripts/wildshard.mjs).
124
+ const prefix = `/src/shards/${shardFolderName(shard)}/`
125
+ const relative = url.startsWith(prefix) ? url.slice(prefix.length) : url
126
+ const file = path.resolve(shard, relative.replace(/^\/+/u, ''))
127
+ return file.startsWith(`${path.resolve(shard)}${path.sep}`) ? file : undefined
128
+ }
129
+
130
+ /** The hero file each card slot names; a `data:` slot names none. */
131
+ async function heroSlotFiles(
132
+ shard: string,
133
+ ): Promise<Readonly<Record<string, string>>> {
134
+ const files: Record<string, string> = {}
135
+ for (const [slot, url] of Object.entries(await cardUrls(shard))) {
136
+ const file = cardFile(path.resolve(shard), url)
137
+ if (file !== undefined) {
138
+ files[slot] = file
139
+ }
140
+ }
141
+ return files
142
+ }
143
+
144
+ async function heroFiles(shard: string): Promise<ReadonlySet<string>> {
145
+ return new Set(Object.values(await heroSlotFiles(shard)))
146
+ }
147
+
148
+ async function contentFiles(
149
+ directory: string,
150
+ skip: ReadonlySet<string>,
151
+ top = true,
152
+ ): Promise<readonly string[]> {
153
+ const entries = await readdir(directory, { withFileTypes: true })
154
+ const nested = await Promise.all(
155
+ entries.map(
156
+ async (
157
+ entry: Readonly<{
158
+ name: string
159
+ isDirectory: () => boolean
160
+ isFile: () => boolean
161
+ }>,
162
+ ): Promise<readonly string[]> => {
163
+ const file = path.join(directory, entry.name)
164
+ if (entry.isDirectory()) {
165
+ return top && SKIPPED_DIRECTORIES.has(entry.name)
166
+ ? []
167
+ : await contentFiles(file, skip, false)
168
+ }
169
+ const ignored =
170
+ !entry.isFile() ||
171
+ skip.has(file) ||
172
+ SKIPPED_SUFFIXES.some((suffix: string): boolean =>
173
+ entry.name.endsWith(suffix),
174
+ )
175
+ return ignored ? [] : [file]
176
+ },
177
+ ),
178
+ )
179
+ return nested.flat()
180
+ }
181
+
182
+ /**
183
+ * One hash over every content file's folder-relative path and bytes: the
184
+ * shard as a hero would show it. The hero files and the stamp are left out,
185
+ * so a capture never makes its own hero stale.
186
+ */
187
+ async function heroContentHash(shard: string): Promise<string> {
188
+ const root = path.resolve(shard)
189
+ const skip = new Set([...(await heroFiles(root)), path.join(root, STAMP)])
190
+ const found = await contentFiles(root, skip)
191
+ const rows = await Promise.all(
192
+ found.map(async (file: string): Promise<string> => {
193
+ const relative = path.relative(root, file).split(path.sep).join('/')
194
+ return `${relative}\0${sha256(await readFile(file))}\n`
195
+ }),
196
+ )
197
+ return sha256(rows.toSorted().join(''))
198
+ }
199
+
200
+ function textOf(value: unknown): string | undefined {
201
+ return typeof value === 'string' ? value : undefined
202
+ }
203
+
204
+ /** Each hero file's folder-relative path and hash, for the stamp. */
205
+ async function cardHashes(
206
+ shard: string,
207
+ ): Promise<Readonly<Record<string, string>>> {
208
+ const files = [...(await heroFiles(shard))]
209
+ const rows = await Promise.all(
210
+ files.map(async (file: string): Promise<readonly [string, string]> => [
211
+ path.relative(shard, file).split(path.sep).join('/'),
212
+ sha256(await readFile(file)),
213
+ ]),
214
+ )
215
+ return Object.fromEntries(rows)
216
+ }
217
+
218
+ /** Whether the stamped hero files are the ones on disk now. */
219
+ function sameCard(
220
+ stamped: unknown,
221
+ current: Readonly<Record<string, string>>,
222
+ ): boolean {
223
+ const names = Object.keys(current)
224
+ return (
225
+ names.length === Object.keys(stamped ?? {}).length &&
226
+ names.every(
227
+ (name: string): boolean => field(stamped, name) === current[name],
228
+ )
229
+ )
230
+ }
231
+
232
+ /** Whether the shard's captured hero is as new as its content. */
233
+ async function heroState(shard: string): Promise<HeroState> {
234
+ const content = await heroContentHash(shard)
235
+ const files = await heroFiles(shard)
236
+ if (files.size === NONE) {
237
+ return { state: 'generated', content }
238
+ }
239
+ const stamp = await readStamp(shard)
240
+ const curated = field(stamp, 'curated') === true
241
+ const stamped = field(stamp, 'content')
242
+ if (typeof stamped !== 'string') {
243
+ return { state: 'unstamped', content, curated }
244
+ }
245
+ return {
246
+ state:
247
+ stamped === content &&
248
+ sameCard(field(stamp, 'card'), await cardHashes(shard))
249
+ ? 'fresh'
250
+ : 'stale',
251
+ content,
252
+ stamped,
253
+ captured: textOf(field(stamp, 'captured')),
254
+ curated,
255
+ }
256
+ }
257
+
258
+ /** Record that the hero files now show this content (`wildshard hero`). */
259
+ async function writeHeroStamp(
260
+ shard: string,
261
+ options: Readonly<{ base: string; captured?: string; camera?: unknown }>,
262
+ ): Promise<HeroStamp> {
263
+ const previous = await readStamp(shard)
264
+ const camera = options.camera ?? field(previous, 'camera')
265
+ const curated = field(previous, 'curated') === true
266
+ const stamp: HeroStamp = {
267
+ content: await heroContentHash(shard),
268
+ captured: options.captured ?? new Date().toISOString(),
269
+ base: options.base,
270
+ card: await cardHashes(shard),
271
+ ...(camera === undefined ? {} : { camera }),
272
+ ...(curated ? { curated: true } : {}),
273
+ }
274
+ const partial = path.join(shard, `${STAMP}.${randomUUID()}.partial`)
275
+ try {
276
+ await writeFile(partial, `${JSON.stringify(stamp, undefined, INDENT)}\n`)
277
+ await rename(partial, path.join(shard, STAMP))
278
+ } finally {
279
+ await rm(partial, { force: true })
280
+ }
281
+ return stamp
282
+ }
283
+
284
+ export {
285
+ type HeroState,
286
+ type HeroStamp,
287
+ MANIFEST,
288
+ STAMP,
289
+ NONE,
290
+ UNKNOWN_TIME,
291
+ cardHashes,
292
+ field,
293
+ heroContentHash,
294
+ heroFiles,
295
+ heroSlotFiles,
296
+ heroState,
297
+ isMissing,
298
+ readJson,
299
+ writeHeroStamp,
300
+ }
@@ -0,0 +1,20 @@
1
+ import { refusalCode, refusalHint, refusalNote } from '../sharddev/refusals.ts'
2
+
3
+ import type { RefusalStep } from '../sharddev/refusals.ts'
4
+
5
+ /** One line of a command's report on stdout. */
6
+ function reportLine(line: string): void {
7
+ process.stdout.write(`${line}\n`)
8
+ }
9
+
10
+ /** A refusal on stderr, then its rule, fix and GUIDE.md link (SA21). `refusal` is the words, or the error: its message,
11
+ * and its shard-format code names the rule when it has one (SA41). */
12
+ function refuse(step: RefusalStep, prefix: string, refusal: unknown): void {
13
+ const reason = refusal instanceof Error ? refusal.message : String(refusal)
14
+ const hint = refusalHint(
15
+ refusalNote(step, reason, { code: refusalCode(refusal) }),
16
+ )
17
+ process.stderr.write(`sharddev: ${prefix}${reason}\n ${hint}\n`)
18
+ }
19
+
20
+ export { refuse, reportLine }
@@ -0,0 +1,49 @@
1
+ import { refuse, reportLine } from '../report.ts'
2
+ import { writeShardfile } from './shardfile.ts'
3
+
4
+ // `wildshard build <folder> [file]` and `wildshard pack`, its deprecated
5
+ // alias (SHARD-AUTHORING SA3, J170): an author build writes its shard file
6
+ // itself; these write the shard file of a folder already built.
7
+
8
+ const EXIT_OK = 0
9
+ const EXIT_USAGE = 1
10
+ const EXIT_FAILED = 3
11
+ // Refusals name the shard file step's rules, whichever verb ran it.
12
+ const STEP = 'shardfile'
13
+
14
+ const PACK_DEPRECATED =
15
+ 'sharddev: pack is deprecated: `wildshard build` writes the shard file (`wildshard build <folder> [file]` for a built folder)\n'
16
+
17
+ /** Write a built folder's shard file, to `out` or beside the folder. */
18
+ async function runShardfile(
19
+ step: 'build' | 'pack',
20
+ dir: string | undefined,
21
+ out: string | undefined,
22
+ ): Promise<number> {
23
+ if (dir === undefined) {
24
+ refuse(STEP, '', `${step} needs a shard file folder`)
25
+ return EXIT_USAGE
26
+ }
27
+ try {
28
+ await writeShardfile({
29
+ dir,
30
+ report: reportLine,
31
+ ...(out === undefined ? {} : { out }),
32
+ })
33
+ return EXIT_OK
34
+ } catch (error: unknown) {
35
+ refuse(STEP, `${step} failed: `, error)
36
+ return EXIT_FAILED
37
+ }
38
+ }
39
+
40
+ /** `pack <folder> [--out <file>]`: one line naming `build`, then the same step. */
41
+ async function runPack(
42
+ dir: string | undefined,
43
+ out: string | undefined,
44
+ ): Promise<number> {
45
+ process.stderr.write(PACK_DEPRECATED)
46
+ return await runShardfile('pack', dir, out)
47
+ }
48
+
49
+ export { runPack, runShardfile }
@@ -0,0 +1,192 @@
1
+ import { createHash, randomUUID } from 'node:crypto'
2
+ import { readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'
3
+ import path from 'node:path'
4
+
5
+ import { installFormatHook } from '@wildshard/format/formatHook'
6
+
7
+ import { DRAFT_SDK, draftMessage } from '../../sharddev/draft.ts'
8
+ import { refusalHint, refusalNote } from '../../sharddev/refusals.ts'
9
+ import { audioLines } from '../audio/audio.ts'
10
+ import { assertHeroCurrent } from '../hero/hero.ts'
11
+ import { validateShardfile } from '../upload/validate.ts'
12
+
13
+ // The shard file (C0, J137): `wildshard build` (SHARD-AUTHORING SA3, J170)
14
+ // turns the shard file folder it built into the one tarball `api-server`
15
+ // takes, so a shard file stands alone. It writes ustar through format's
16
+ // `writeTar` and runs format's validator on the tarball itself, the check the
17
+ // server runs at commit: what this writes, the server admits once it is
18
+ // prepared. A fresh build is a draft (sdk 0.2) that the server refuses, so
19
+ // this warns about one (J198). `wildshard pack` is the deprecated alias.
20
+ //
21
+ // The same folder writes the same bytes, so the same hash, and a re-upload
22
+ // sends nothing.
23
+
24
+ const BYTES_PER_MB = 1_000_000
25
+ const ONE_DECIMAL = 1
26
+ // The cap in api-server's D31, on a shardfile stored or decoded. J137 raises
27
+ // it only if a real shard needs it, so going over is news and not a refusal:
28
+ // the server's cap is the one that counts.
29
+ const SERVER_CAP_BYTES = 450_000_000
30
+
31
+ /** What `readdir` says about one entry. */
32
+ interface ProductEntry {
33
+ readonly name: string
34
+ readonly parentPath: string
35
+ readonly isDirectory: () => boolean
36
+ readonly isFile: () => boolean
37
+ readonly isSymbolicLink: () => boolean
38
+ }
39
+
40
+ interface ShardfileOptions {
41
+ /** The built shard file folder: `shard.json` at its root. */
42
+ readonly dir: string
43
+ /** Where the shard file goes; `<slug>.shardfile` beside the folder if unset. */
44
+ readonly out?: string
45
+ /** The shard source the hero check reads; `<root>/src/shards/<name>` for `<root>/out/<name>` if unset. */
46
+ readonly source?: string
47
+ /** Receives one line per step. */
48
+ readonly report: (line: string) => void
49
+ /** False: the caller reports a draft itself (`check`, in its verdict). */
50
+ readonly draftWarning?: boolean
51
+ }
52
+
53
+ interface Tarball {
54
+ readonly tarball: Uint8Array<ArrayBuffer>
55
+ readonly fileCount: number
56
+ }
57
+
58
+ interface ShardfileOutcome {
59
+ readonly slug: string
60
+ readonly file: string
61
+ readonly hash: string
62
+ readonly bytes: number
63
+ }
64
+
65
+ function sha256(bytes: Readonly<Uint8Array>): string {
66
+ return createHash('sha256').update(bytes).digest('hex')
67
+ }
68
+
69
+ function mb(bytes: number): string {
70
+ return `${(bytes / BYTES_PER_MB).toFixed(ONE_DECIMAL)} MB`
71
+ }
72
+
73
+ // A cook pack repeats files that ship at their own paths: format rebuilds and
74
+ // checks it from them, so a shard file leaves it out (C0). 109 MB of Pine Hollow.
75
+ const PACKS = 'blobs/pack/'
76
+
77
+ function productPath(root: string, entry: ProductEntry): string {
78
+ const file = path.join(entry.parentPath, entry.name)
79
+ if (entry.isSymbolicLink() || !entry.isFile()) {
80
+ throw new Error(`not a regular file: ${file}`)
81
+ }
82
+ return path.relative(root, file).split(path.sep).join('/')
83
+ }
84
+
85
+ /** Every file in the product by `/`-separated path; a link is refused. */
86
+ async function readProduct(
87
+ root: string,
88
+ ): Promise<ReadonlyMap<string, Uint8Array>> {
89
+ const entries = await readdir(root, { recursive: true, withFileTypes: true })
90
+ const paths = entries
91
+ .filter((entry: ProductEntry): boolean => !entry.isDirectory())
92
+ .map((entry: ProductEntry): string => productPath(root, entry))
93
+ .filter((file: string): boolean => !file.startsWith(PACKS))
94
+ const pairs = await Promise.all(
95
+ paths.map(async (file: string): Promise<readonly [string, Uint8Array]> => {
96
+ const bytes = await readFile(path.join(root, file))
97
+ return [
98
+ file,
99
+ new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.length),
100
+ ]
101
+ }),
102
+ )
103
+ return new Map(pairs)
104
+ }
105
+
106
+ async function tarOf(dir: string): Promise<Tarball> {
107
+ installFormatHook()
108
+ const { writeTar } = await import('@wildshard/format/tar')
109
+ const files = await readProduct(dir)
110
+ return { tarball: writeTar(files), fileCount: files.size }
111
+ }
112
+
113
+ function reportOverCap(options: ShardfileOptions, bytes: number): void {
114
+ if (bytes > SERVER_CAP_BYTES) {
115
+ options.report(
116
+ `warning: ${mb(bytes)} is over api-server's ${mb(SERVER_CAP_BYTES)} cap (its D34)`,
117
+ )
118
+ }
119
+ }
120
+
121
+ /**
122
+ * J198: a fresh build is a draft (sdk 0.2) that upload and api-server refuse;
123
+ * build still writes it, as `wildshard prepare` works on the folder.
124
+ */
125
+ function reportDraft(
126
+ options: ShardfileOptions,
127
+ slug: string,
128
+ sdk: string,
129
+ ): void {
130
+ if (sdk === DRAFT_SDK && options.draftWarning !== false) {
131
+ const message = draftMessage(slug)
132
+ options.report(`warning: ${message}`)
133
+ options.report(` ${refusalHint(refusalNote('shardfile', message))}`)
134
+ }
135
+ }
136
+
137
+ /** Never leaves half a tarball under the final name. */
138
+ async function writeAtomically(
139
+ file: string,
140
+ bytes: Readonly<Uint8Array>,
141
+ ): Promise<void> {
142
+ const partial = `${file}.${randomUUID()}.partial`
143
+ try {
144
+ await writeFile(partial, bytes)
145
+ await rename(partial, file)
146
+ } finally {
147
+ await rm(partial, { force: true })
148
+ }
149
+ }
150
+
151
+ /** Check the hero is as new as the content, tar the folder, check the tarball. */
152
+ async function checkedTarball(
153
+ options: ShardfileOptions,
154
+ ): Promise<Readonly<{ tarball: Uint8Array<ArrayBuffer>; slug: string }>> {
155
+ await assertHeroCurrent(options.dir, options.report, options.source)
156
+ const { tarball, fileCount } = await tarOf(options.dir)
157
+ const { slug, name, audio, sdk } = await validateShardfile(tarball)
158
+ options.report(`valid: ${slug} (${name}), ${String(fileCount)} files`)
159
+ reportDraft(options, slug, sdk)
160
+ // AUDIO AU5: build prints the audio rules as AMBER and still writes the
161
+ // shard file; only upload and api-server's admission refuse a red.
162
+ for (const line of await audioLines(audio)) {
163
+ options.report(line)
164
+ }
165
+ reportOverCap(options, tarball.length)
166
+ return { tarball, slug }
167
+ }
168
+
169
+ /** The shard file's default name: `<slug>.shardfile` beside its folder (SA3). */
170
+ function shardfilePath(dir: string, slug: string): string {
171
+ return path.join(path.dirname(path.resolve(dir)), `${slug}.shardfile`)
172
+ }
173
+
174
+ /** Check the folder, then write its shard file; a refusal writes none. */
175
+ async function writeShardfile(
176
+ options: ShardfileOptions,
177
+ ): Promise<ShardfileOutcome> {
178
+ const { tarball, slug } = await checkedTarball(options)
179
+ const bytes = tarball.length
180
+ const file = options.out ?? shardfilePath(options.dir, slug)
181
+ const hash = sha256(tarball)
182
+ await writeAtomically(file, tarball)
183
+ options.report(`shard file: ${file}, ${mb(bytes)} (${hash})`)
184
+ return { slug, file, hash, bytes }
185
+ }
186
+
187
+ export {
188
+ type ShardfileOptions,
189
+ type ShardfileOutcome,
190
+ shardfilePath,
191
+ writeShardfile,
192
+ }