@wildshard/upload 0.0.0-stage → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +28 -3
- package/src/cli/account/account.ts +199 -0
- package/src/cli/account/command.ts +193 -0
- package/src/cli/account/key.ts +154 -0
- package/src/cli/args.ts +100 -0
- package/src/cli/audio/audio.ts +39 -0
- package/src/cli/delete/delete.ts +258 -0
- package/src/cli/hero/hero.ts +163 -0
- package/src/cli/hero/scene-card.ts +46 -0
- package/src/cli/hero/stamp.ts +300 -0
- package/src/cli/report.ts +20 -0
- package/src/cli/shardfile/command.ts +49 -0
- package/src/cli/shardfile/shardfile.ts +192 -0
- package/src/cli/upload/api.ts +286 -0
- package/src/cli/upload/command.ts +239 -0
- package/src/cli/upload/completion.ts +80 -0
- package/src/cli/upload/compress.ts +55 -0
- package/src/cli/upload/hero-gate.ts +208 -0
- package/src/cli/upload/hero.ts +62 -0
- package/src/cli/upload/image-size.ts +89 -0
- package/src/cli/upload/placement.ts +132 -0
- package/src/cli/upload/provenance.ts +204 -0
- package/src/cli/upload/signature.ts +39 -0
- package/src/cli/upload/status-body.ts +41 -0
- package/src/cli/upload/status.ts +176 -0
- package/src/cli/upload/upload.ts +298 -0
- package/src/cli/upload/validate.ts +92 -0
- package/src/cli/upload/wire.ts +189 -0
- package/src/sharddev/draft.ts +11 -0
- package/src/sharddev/refusals.ts +329 -0
- package/src/sharddev/shardSource.ts +75 -0
- package/README.md +0 -3
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { createHash, createPrivateKey, sign } from 'node:crypto'
|
|
2
|
+
|
|
3
|
+
// The org creator key (J60, J69): Ed25519 over the raw 32 bytes of the
|
|
4
|
+
// tarball's SHA-256, never the compressed bytes. The private key is PKCS#8
|
|
5
|
+
// DER in base64, as `api-server keygen` prints it.
|
|
6
|
+
|
|
7
|
+
interface UploadIdentity {
|
|
8
|
+
/** Lowercase hex: the shardfile's identity. */
|
|
9
|
+
readonly hash: string
|
|
10
|
+
/** Base64: the `x-wildshard-signature` header. */
|
|
11
|
+
readonly signature?: string
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
interface SignedShardfile extends UploadIdentity {
|
|
15
|
+
readonly signature: string
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** Account-only uploads retain the same byte identity without a creator signature. */
|
|
19
|
+
function hashTarball(tarball: Readonly<Uint8Array>): UploadIdentity {
|
|
20
|
+
return { hash: createHash('sha256').update(tarball).digest('hex') }
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function signTarball(
|
|
24
|
+
privateKey: string,
|
|
25
|
+
tarball: Readonly<Uint8Array>,
|
|
26
|
+
): SignedShardfile {
|
|
27
|
+
const key = createPrivateKey({
|
|
28
|
+
key: Buffer.from(privateKey, 'base64'),
|
|
29
|
+
format: 'der',
|
|
30
|
+
type: 'pkcs8',
|
|
31
|
+
})
|
|
32
|
+
const digest = createHash('sha256').update(tarball).digest()
|
|
33
|
+
return {
|
|
34
|
+
hash: digest.toString('hex'),
|
|
35
|
+
signature: sign(undefined, digest, key).toString('base64'),
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export { type UploadIdentity, type SignedShardfile, hashTarball, signTarball }
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
// The API accepts reports up to 8 MiB. Leave room for the status envelope
|
|
2
|
+
// while bounding an unexpected proxy/server response before parsing JSON.
|
|
3
|
+
const MAX_STATUS_BYTES = 16_777_216
|
|
4
|
+
|
|
5
|
+
async function collect(
|
|
6
|
+
reader: ReadableStreamDefaultReader<Uint8Array>,
|
|
7
|
+
): Promise<string> {
|
|
8
|
+
const state: { decoder: TextDecoder; parts: string[]; bytes: number } = {
|
|
9
|
+
decoder: new TextDecoder(),
|
|
10
|
+
parts: [],
|
|
11
|
+
bytes: 0,
|
|
12
|
+
}
|
|
13
|
+
for (;;) {
|
|
14
|
+
// eslint-disable-next-line no-await-in-loop -- Stream chunks arrive in order; parallel reads would not bound buffering.
|
|
15
|
+
const next = await reader.read()
|
|
16
|
+
if (next.done) {
|
|
17
|
+
return state.parts.join('') + state.decoder.decode()
|
|
18
|
+
}
|
|
19
|
+
state.bytes += next.value.byteLength
|
|
20
|
+
if (state.bytes > MAX_STATUS_BYTES) {
|
|
21
|
+
// eslint-disable-next-line no-await-in-loop -- Cancel the same sequential stream before refusing its oversized body.
|
|
22
|
+
await reader.cancel()
|
|
23
|
+
throw new Error('upload status response exceeds the 16 MiB limit')
|
|
24
|
+
}
|
|
25
|
+
state.parts.push(state.decoder.decode(next.value, { stream: true }))
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
async function readStatusBody(reply: Response): Promise<string> {
|
|
30
|
+
if (reply.body === null) {
|
|
31
|
+
return ''
|
|
32
|
+
}
|
|
33
|
+
const reader = reply.body.getReader()
|
|
34
|
+
try {
|
|
35
|
+
return await collect(reader)
|
|
36
|
+
} finally {
|
|
37
|
+
reader.releaseLock()
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export { readStatusBody }
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
import { setTimeout as sleep } from 'node:timers/promises'
|
|
2
|
+
|
|
3
|
+
import { ApiRefusalError, readUploadStatus } from './api.ts'
|
|
4
|
+
|
|
5
|
+
import type { ApiOptions } from './api.ts'
|
|
6
|
+
import type { UploadState, UploadStatus } from './wire.ts'
|
|
7
|
+
|
|
8
|
+
const INITIAL_DELAY_MS = 2000
|
|
9
|
+
const MAX_DELAY_MS = 15_000
|
|
10
|
+
const MAX_WAIT_MS = 1_200_000
|
|
11
|
+
const REQUEST_TIMEOUT_MS = 15_000
|
|
12
|
+
const BACKOFF = 2
|
|
13
|
+
const SERVER_ERROR = 500
|
|
14
|
+
const TOO_MANY_REQUESTS = 429
|
|
15
|
+
const NONE = 0
|
|
16
|
+
|
|
17
|
+
interface PollPorts {
|
|
18
|
+
readonly now: () => number
|
|
19
|
+
readonly sleep: (milliseconds: number) => Promise<void>
|
|
20
|
+
readonly read: typeof readUploadStatus
|
|
21
|
+
}
|
|
22
|
+
interface PollRequest {
|
|
23
|
+
readonly status?: UploadState
|
|
24
|
+
readonly id: string
|
|
25
|
+
readonly hash: string
|
|
26
|
+
readonly report: (line: string) => void
|
|
27
|
+
}
|
|
28
|
+
interface Polling extends PollRequest {
|
|
29
|
+
readonly options: ApiOptions
|
|
30
|
+
readonly ports: PollPorts
|
|
31
|
+
readonly deadline: number
|
|
32
|
+
}
|
|
33
|
+
interface PollState {
|
|
34
|
+
readonly seen: UploadState | undefined
|
|
35
|
+
readonly delay: number
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function timedOut(id: string): Error {
|
|
39
|
+
return new Error(
|
|
40
|
+
`upload ${id}: timed out waiting for the verdict; resume with wildshard upload status ${id}`,
|
|
41
|
+
)
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function reportStatus(
|
|
45
|
+
status: UploadStatus,
|
|
46
|
+
report: (line: string) => void,
|
|
47
|
+
): void {
|
|
48
|
+
report(`upload ${status.uploadId}: ${status.status}`)
|
|
49
|
+
if (status.status === 'refused' && status.report !== undefined) {
|
|
50
|
+
report(JSON.stringify(status.report, undefined, BACKOFF))
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function requireSuccess(status: UploadStatus): void {
|
|
55
|
+
if (status.status === 'stalled') {
|
|
56
|
+
throw new Error(
|
|
57
|
+
`upload ${status.uploadId}: stalled; it may still settle; read it later with wildshard upload status ${status.uploadId}`,
|
|
58
|
+
)
|
|
59
|
+
}
|
|
60
|
+
if (status.status === 'refused') {
|
|
61
|
+
throw new Error(`upload ${status.uploadId}: refused`)
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function retryable(error: unknown): boolean {
|
|
66
|
+
return (
|
|
67
|
+
error instanceof TypeError ||
|
|
68
|
+
(error instanceof Error &&
|
|
69
|
+
(error.name === 'TimeoutError' || error.name === 'AbortError')) ||
|
|
70
|
+
(error instanceof ApiRefusalError &&
|
|
71
|
+
(error.status >= SERVER_ERROR || error.status === TOO_MANY_REQUESTS))
|
|
72
|
+
)
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
async function readMatching(
|
|
76
|
+
session: Polling,
|
|
77
|
+
remaining: number,
|
|
78
|
+
): Promise<UploadStatus> {
|
|
79
|
+
const timeout = Math.ceil(Math.min(remaining, REQUEST_TIMEOUT_MS))
|
|
80
|
+
const status = await session.ports.read(
|
|
81
|
+
session.options,
|
|
82
|
+
session.id,
|
|
83
|
+
AbortSignal.timeout(timeout),
|
|
84
|
+
)
|
|
85
|
+
if (status.uploadId !== session.id || status.hash !== session.hash) {
|
|
86
|
+
throw new Error('upload status does not match the accepted upload')
|
|
87
|
+
}
|
|
88
|
+
return status
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
async function observe(session: Polling): Promise<UploadStatus | undefined> {
|
|
92
|
+
const remaining = session.deadline - session.ports.now()
|
|
93
|
+
if (remaining <= NONE) {
|
|
94
|
+
throw timedOut(session.id)
|
|
95
|
+
}
|
|
96
|
+
try {
|
|
97
|
+
return await readMatching(session, remaining)
|
|
98
|
+
} catch (error: unknown) {
|
|
99
|
+
if (!retryable(error)) {
|
|
100
|
+
throw error
|
|
101
|
+
}
|
|
102
|
+
if (session.ports.now() >= session.deadline) {
|
|
103
|
+
throw timedOut(session.id)
|
|
104
|
+
}
|
|
105
|
+
return undefined
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function isLive(
|
|
110
|
+
session: Polling,
|
|
111
|
+
status: UploadStatus,
|
|
112
|
+
seen: UploadState | undefined,
|
|
113
|
+
): boolean {
|
|
114
|
+
if (status.status !== seen || status.status === 'refused') {
|
|
115
|
+
reportStatus(status, session.report)
|
|
116
|
+
}
|
|
117
|
+
requireSuccess(status)
|
|
118
|
+
return status.status === 'live'
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
async function poll(session: Polling, state: PollState): Promise<UploadStatus> {
|
|
122
|
+
const status = await observe(session)
|
|
123
|
+
if (status !== undefined && isLive(session, status, state.seen)) {
|
|
124
|
+
return status
|
|
125
|
+
}
|
|
126
|
+
const delay = Math.min(state.delay, session.deadline - session.ports.now())
|
|
127
|
+
if (delay <= NONE) {
|
|
128
|
+
throw timedOut(session.id)
|
|
129
|
+
}
|
|
130
|
+
await session.ports.sleep(delay)
|
|
131
|
+
return await poll(session, {
|
|
132
|
+
seen: status?.status ?? state.seen,
|
|
133
|
+
delay: Math.min(state.delay * BACKOFF, MAX_DELAY_MS),
|
|
134
|
+
})
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** No background worker remains after success, refusal, stall or timeout. */
|
|
138
|
+
async function waitForUpload(
|
|
139
|
+
options: ApiOptions,
|
|
140
|
+
request: PollRequest,
|
|
141
|
+
ports: Partial<PollPorts> = {},
|
|
142
|
+
): Promise<UploadStatus> {
|
|
143
|
+
const clock: PollPorts = {
|
|
144
|
+
now: (): number => performance.now(),
|
|
145
|
+
sleep: async (milliseconds: number): Promise<void> => {
|
|
146
|
+
await sleep(milliseconds)
|
|
147
|
+
},
|
|
148
|
+
read: readUploadStatus,
|
|
149
|
+
...ports,
|
|
150
|
+
}
|
|
151
|
+
return await poll(
|
|
152
|
+
{ ...request, options, ports: clock, deadline: clock.now() + MAX_WAIT_MS },
|
|
153
|
+
{ seen: request.status, delay: INITIAL_DELAY_MS },
|
|
154
|
+
)
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** One authenticated read, also used by `upload status <id>`. */
|
|
158
|
+
async function printUploadStatus(
|
|
159
|
+
options: ApiOptions,
|
|
160
|
+
id: string,
|
|
161
|
+
report: (line: string) => void,
|
|
162
|
+
): Promise<UploadStatus> {
|
|
163
|
+
const status = await readUploadStatus(
|
|
164
|
+
options,
|
|
165
|
+
id,
|
|
166
|
+
AbortSignal.timeout(REQUEST_TIMEOUT_MS),
|
|
167
|
+
)
|
|
168
|
+
if (status.uploadId !== id) {
|
|
169
|
+
throw new Error('upload status does not match the requested upload')
|
|
170
|
+
}
|
|
171
|
+
reportStatus(status, report)
|
|
172
|
+
requireSuccess(status)
|
|
173
|
+
return status
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
export { type PollPorts, printUploadStatus, waitForUpload }
|
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises'
|
|
2
|
+
|
|
3
|
+
import { DRAFT_SDK, draftMessage } from '../../sharddev/draft.ts'
|
|
4
|
+
import { assertAudio } from '../audio/audio.ts'
|
|
5
|
+
import {
|
|
6
|
+
commitUpload,
|
|
7
|
+
findCatalogued,
|
|
8
|
+
findStored,
|
|
9
|
+
putObject,
|
|
10
|
+
requestTicket,
|
|
11
|
+
} from './api.ts'
|
|
12
|
+
import { completeUpload } from './completion.ts'
|
|
13
|
+
import { storedForm } from './compress.ts'
|
|
14
|
+
import { hashTarball, signTarball } from './signature.ts'
|
|
15
|
+
import { assertHero, validateShardfile } from './validate.ts'
|
|
16
|
+
|
|
17
|
+
import type { ApiOptions, Channel } from './api.ts'
|
|
18
|
+
import type { Completion } from './completion.ts'
|
|
19
|
+
import type { StoredForm } from './compress.ts'
|
|
20
|
+
import type { Provenance } from './provenance.ts'
|
|
21
|
+
import type { UploadIdentity } from './signature.ts'
|
|
22
|
+
import type { CommitAnswer } from './wire.ts'
|
|
23
|
+
|
|
24
|
+
// `sharddev upload` (J60, J96): check the shardfile here, compress it here,
|
|
25
|
+
// sign the tarball, then ticket, PUT, commit. api-server only checks and
|
|
26
|
+
// stores what arrives. The commit carries the shard version and commit
|
|
27
|
+
// (API-CLOUDFLARE AC16, V1), and is sent for held bytes too, so the API
|
|
28
|
+
// answers which version holds them: a retry, or a 409 for another version.
|
|
29
|
+
// SHARD-AUTHORING UP3 adds the channel the upload is for (J301) and the
|
|
30
|
+
// versions it was built with (J306), and refuses a hero unchanged since the
|
|
31
|
+
// slug's newest catalogued version before any byte moves, as UP4's API will.
|
|
32
|
+
// An unprepared draft (sdk 0.2) refuses here too, before any request: the
|
|
33
|
+
// API answers it 422 at commit (J259), after the whole file was sent.
|
|
34
|
+
// SA32 sends `place: false` for `--no-place` and prints the API's grid
|
|
35
|
+
// answer and each moved channel's `/slug/<slug>` link (placement.ts).
|
|
36
|
+
|
|
37
|
+
const MS_PER_SECOND = 1000
|
|
38
|
+
const BYTES_PER_MB = 1_000_000
|
|
39
|
+
const PERCENT = 100
|
|
40
|
+
const ONE_DECIMAL = 1
|
|
41
|
+
const WHOLE = 1
|
|
42
|
+
|
|
43
|
+
interface UploadOptions extends ApiOptions, Provenance {
|
|
44
|
+
/** Wait for cooked activation by default; false implements --no-wait. */
|
|
45
|
+
readonly wait?: boolean
|
|
46
|
+
/** The shardfile tarball. */
|
|
47
|
+
readonly file: string
|
|
48
|
+
/** The channel the upload is for, by its wire name (J301). */
|
|
49
|
+
readonly channel: Channel
|
|
50
|
+
/** False for `--no-place` (SA32): the commit sends `place: false`. */
|
|
51
|
+
readonly place?: boolean
|
|
52
|
+
/** Base64 PKCS#8: WILDSHARD_CREATOR_PRIVATE_KEY. */
|
|
53
|
+
readonly privateKey?: string
|
|
54
|
+
/** Brotli quality. */
|
|
55
|
+
readonly level: number
|
|
56
|
+
/** Receives one line per step. */
|
|
57
|
+
readonly report: (line: string) => void
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** The upload that holds the bytes, as the API answered its commit. */
|
|
61
|
+
interface UploadOutcome extends Completion {
|
|
62
|
+
/** The grid line and the links, printed after the commit line (SA32). */
|
|
63
|
+
readonly gridReport: readonly string[]
|
|
64
|
+
readonly hash: string
|
|
65
|
+
readonly shard: string
|
|
66
|
+
readonly revision: number
|
|
67
|
+
/** The bytes were stored already, so nothing new was recorded. */
|
|
68
|
+
readonly alreadyStored: boolean
|
|
69
|
+
/** The shard version the API holds these bytes as. */
|
|
70
|
+
readonly version: string
|
|
71
|
+
/** The API's format version. */
|
|
72
|
+
readonly formatVersion: string
|
|
73
|
+
/** The migrations the API ran. */
|
|
74
|
+
readonly migrations: readonly unknown[]
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
type Encoding = StoredForm['contentEncoding']
|
|
78
|
+
|
|
79
|
+
function mb(bytes: number): string {
|
|
80
|
+
return `${(bytes / BYTES_PER_MB).toFixed(ONE_DECIMAL)} MB`
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function secondsSince(start: number): string {
|
|
84
|
+
const elapsed = (performance.now() - start) / MS_PER_SECOND
|
|
85
|
+
return `${elapsed.toFixed(ONE_DECIMAL)} s`
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Listed bytes still ask for a ticket: the listing says they are held, and
|
|
90
|
+
* only the ticket says today's format takes them. A break refuses them here.
|
|
91
|
+
* True when they are held, so nothing needs compressing or sending.
|
|
92
|
+
*/
|
|
93
|
+
async function held(
|
|
94
|
+
options: UploadOptions,
|
|
95
|
+
signed: UploadIdentity,
|
|
96
|
+
tarballBytes: number,
|
|
97
|
+
): Promise<boolean> {
|
|
98
|
+
const listed = await findStored(options, signed.hash)
|
|
99
|
+
if (listed === undefined) {
|
|
100
|
+
return false
|
|
101
|
+
}
|
|
102
|
+
const ticket = await requestTicket(
|
|
103
|
+
options,
|
|
104
|
+
{ hash: signed.hash, storedBytes: tarballBytes },
|
|
105
|
+
signed.signature,
|
|
106
|
+
)
|
|
107
|
+
return ticket.stored
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
async function compressed(
|
|
111
|
+
options: UploadOptions,
|
|
112
|
+
tarball: Readonly<Uint8Array>,
|
|
113
|
+
): Promise<StoredForm> {
|
|
114
|
+
const start = performance.now()
|
|
115
|
+
const form = await storedForm(tarball, options.level)
|
|
116
|
+
const saved = PERCENT * (WHOLE - form.body.length / tarball.length)
|
|
117
|
+
options.report(
|
|
118
|
+
form.contentEncoding === undefined
|
|
119
|
+
? `sending the shard file uncompressed: Brotli would save under 1%`
|
|
120
|
+
: `brotli-${String(options.level)}: ${mb(tarball.length)} -> ${mb(form.body.length)}, ${saved.toFixed(ONE_DECIMAL)}% saved in ${secondsSince(start)}`,
|
|
121
|
+
)
|
|
122
|
+
return form
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** Compress, ticket, and PUT unless the server has the bytes already. */
|
|
126
|
+
async function deliver(
|
|
127
|
+
options: UploadOptions,
|
|
128
|
+
tarball: Readonly<Uint8Array>,
|
|
129
|
+
signed: UploadIdentity,
|
|
130
|
+
): Promise<Encoding> {
|
|
131
|
+
const { body, contentEncoding } = await compressed(options, tarball)
|
|
132
|
+
const ticket = await requestTicket(
|
|
133
|
+
options,
|
|
134
|
+
{
|
|
135
|
+
hash: signed.hash,
|
|
136
|
+
storedBytes: body.length,
|
|
137
|
+
...(contentEncoding === undefined ? {} : { contentEncoding }),
|
|
138
|
+
},
|
|
139
|
+
signed.signature,
|
|
140
|
+
)
|
|
141
|
+
if (!ticket.stored) {
|
|
142
|
+
const start = performance.now()
|
|
143
|
+
await putObject(ticket, body)
|
|
144
|
+
options.report(`uploaded ${mb(body.length)} in ${secondsSince(start)}`)
|
|
145
|
+
}
|
|
146
|
+
return contentEncoding
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** UP2's echo of the channel and the versions: the fields the API sent. */
|
|
150
|
+
function echoOf(answer: CommitAnswer): string {
|
|
151
|
+
const { channel, sharddev, engine, shardFormat } = answer
|
|
152
|
+
const fields = { channel, sharddev, engine, shardFormat }
|
|
153
|
+
return Object.entries(fields)
|
|
154
|
+
.map(
|
|
155
|
+
([name, value]: readonly [string, string | null | undefined]): string =>
|
|
156
|
+
typeof value === 'string' ? `, ${name} ${value}` : '',
|
|
157
|
+
)
|
|
158
|
+
.join('')
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function reportAnswer(options: UploadOptions, answer: CommitAnswer): void {
|
|
162
|
+
if (answer.uploadId !== undefined) {
|
|
163
|
+
return
|
|
164
|
+
}
|
|
165
|
+
const {
|
|
166
|
+
signedBy,
|
|
167
|
+
created,
|
|
168
|
+
staged,
|
|
169
|
+
notified,
|
|
170
|
+
version,
|
|
171
|
+
formatVersion,
|
|
172
|
+
migrations,
|
|
173
|
+
} = answer
|
|
174
|
+
const { channel, sharddev, engine, shardFormat } = options
|
|
175
|
+
options.report(
|
|
176
|
+
`${options.privateKey === undefined ? 'authorized by' : 'signed by'} ${signedBy}, created: ${String(created)}, staged: ${String(staged)}, notified: ${String(notified)}`,
|
|
177
|
+
)
|
|
178
|
+
options.report(
|
|
179
|
+
`sent: channel ${channel}, sharddev ${sharddev}, engine ${engine}, shardFormat ${shardFormat}`,
|
|
180
|
+
)
|
|
181
|
+
options.report(
|
|
182
|
+
`api: version ${version}, formatVersion ${formatVersion}, migrations ${JSON.stringify(migrations)}${echoOf(answer)}`,
|
|
183
|
+
)
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/** The commit, always: the API names the version that holds the bytes (V1). */
|
|
187
|
+
async function commit(
|
|
188
|
+
options: UploadOptions,
|
|
189
|
+
signed: UploadIdentity,
|
|
190
|
+
contentEncoding: Encoding,
|
|
191
|
+
): Promise<UploadOutcome> {
|
|
192
|
+
const { channel, sharddev, engine, shardFormat } = options
|
|
193
|
+
const place = options.place !== false
|
|
194
|
+
const answer = await commitUpload(options, signed, {
|
|
195
|
+
place,
|
|
196
|
+
contentEncoding,
|
|
197
|
+
version: options.version,
|
|
198
|
+
commit: options.commit,
|
|
199
|
+
channel,
|
|
200
|
+
sharddev,
|
|
201
|
+
engine,
|
|
202
|
+
shardFormat,
|
|
203
|
+
})
|
|
204
|
+
const { hash, shard, revision, created, version, formatVersion, migrations } =
|
|
205
|
+
answer
|
|
206
|
+
if (hash !== signed.hash) {
|
|
207
|
+
throw new Error(
|
|
208
|
+
'the API commit hash does not match the uploaded shard file',
|
|
209
|
+
)
|
|
210
|
+
}
|
|
211
|
+
reportAnswer(options, answer)
|
|
212
|
+
return {
|
|
213
|
+
...(await completeUpload(options, answer)),
|
|
214
|
+
hash,
|
|
215
|
+
shard,
|
|
216
|
+
revision,
|
|
217
|
+
alreadyStored: !created,
|
|
218
|
+
version,
|
|
219
|
+
formatVersion,
|
|
220
|
+
migrations,
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/** What the hero pre-check needs from the validated shardfile. */
|
|
225
|
+
interface Validated {
|
|
226
|
+
readonly slug: string
|
|
227
|
+
/** The hero digest, when the shard has a card. */
|
|
228
|
+
readonly hero: string | undefined
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/** Format's check, the draft check (J198), AU5's audio rules and the hero gate (HERO-GATE HG2): each refuses before a byte is sent. */
|
|
232
|
+
async function validated(
|
|
233
|
+
options: Readonly<UploadOptions>,
|
|
234
|
+
tarball: Readonly<Uint8Array>,
|
|
235
|
+
): Promise<Validated> {
|
|
236
|
+
const { slug, name, audio, heroHash, sdk, hero } =
|
|
237
|
+
await validateShardfile(tarball)
|
|
238
|
+
options.report(`valid: ${slug} (${name}), ${mb(tarball.length)}`)
|
|
239
|
+
if (sdk === DRAFT_SDK) {
|
|
240
|
+
throw new Error(draftMessage(slug))
|
|
241
|
+
}
|
|
242
|
+
await assertAudio(audio)
|
|
243
|
+
assertHero(hero)
|
|
244
|
+
return { slug, hero: heroHash }
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/**
|
|
248
|
+
* UP3's hero pre-check (J306): a hero digest equal to the slug's newest
|
|
249
|
+
* catalogued one refuses here, before any byte moves. A first version, a
|
|
250
|
+
* catalogue with no digest (before UP4) and a retry of the catalogued
|
|
251
|
+
* version pass, as the API's own check does.
|
|
252
|
+
*/
|
|
253
|
+
async function checkHero(
|
|
254
|
+
options: UploadOptions,
|
|
255
|
+
shard: Validated,
|
|
256
|
+
): Promise<void> {
|
|
257
|
+
if (shard.hero === undefined) {
|
|
258
|
+
return
|
|
259
|
+
}
|
|
260
|
+
const newest = await findCatalogued(options, shard.slug)
|
|
261
|
+
if (newest?.heroHash === shard.hero && newest.version !== options.version) {
|
|
262
|
+
throw new Error(
|
|
263
|
+
`the hero is unchanged since ${[shard.slug, newest.version].filter(Boolean).join(' ')}: run wildshard hero, then build and upload again`,
|
|
264
|
+
)
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/** The commit's line, then the grid line and the links (SA32). */
|
|
269
|
+
function reportCommitted(options: UploadOptions, outcome: UploadOutcome): void {
|
|
270
|
+
const verb = outcome.alreadyStored ? 'already stored' : 'committed'
|
|
271
|
+
options.report(
|
|
272
|
+
outcome.uploadId === undefined || outcome.status === 'live'
|
|
273
|
+
? `${verb}: ${outcome.shard} ${outcome.version} catalogue number ${String(outcome.revision)} for ${options.channel} (${outcome.hash})`
|
|
274
|
+
: `queued: ${outcome.shard} ${outcome.version} for ${options.channel} (upload ${outcome.uploadId})`,
|
|
275
|
+
)
|
|
276
|
+
for (const line of outcome.gridReport) {
|
|
277
|
+
options.report(line)
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/** Check, compress, sign, upload and commit; the upload that holds the bytes. */
|
|
282
|
+
async function uploadShardfile(options: UploadOptions): Promise<UploadOutcome> {
|
|
283
|
+
const bytes = await readFile(options.file)
|
|
284
|
+
const tarball = new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.length)
|
|
285
|
+
await checkHero(options, await validated(options, tarball))
|
|
286
|
+
const signed =
|
|
287
|
+
options.privateKey === undefined
|
|
288
|
+
? hashTarball(tarball)
|
|
289
|
+
: signTarball(options.privateKey, tarball)
|
|
290
|
+
const encoding = (await held(options, signed, tarball.length))
|
|
291
|
+
? undefined
|
|
292
|
+
: await deliver(options, tarball, signed)
|
|
293
|
+
const outcome = await commit(options, signed, encoding)
|
|
294
|
+
reportCommitted(options, outcome)
|
|
295
|
+
return outcome
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
export { type UploadOptions, type UploadOutcome, uploadShardfile }
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
2
|
+
|
|
3
|
+
import { installFormatHook } from '@wildshard/format/formatHook'
|
|
4
|
+
|
|
5
|
+
import { heroVerdict } from './hero-gate.ts'
|
|
6
|
+
import { heroHashOf } from './hero.ts'
|
|
7
|
+
|
|
8
|
+
import type { HeroShard, HeroVerdict } from './hero-gate.ts'
|
|
9
|
+
import type { HeroCard } from './hero.ts'
|
|
10
|
+
import type { AudioReport } from '@wildshard/format/audioRules'
|
|
11
|
+
import type { ShardfileFiles as ProductFiles } from '@wildshard/format/shardfileValidation'
|
|
12
|
+
|
|
13
|
+
// The same check api-server runs on commit, run first on the author's
|
|
14
|
+
// machine: a shardfile the server would refuse should not cost a Brotli
|
|
15
|
+
// pass and a 200 MB upload to find out.
|
|
16
|
+
|
|
17
|
+
/** What the upload reports about a valid shardfile. */
|
|
18
|
+
interface ValidShardfile {
|
|
19
|
+
readonly slug: string
|
|
20
|
+
readonly name: string
|
|
21
|
+
/** AUDIO AU5: the shard's own audio against format's rules, as of today. */
|
|
22
|
+
readonly audio: AudioReport
|
|
23
|
+
/** `presentation.card`'s three image blob hashes, when it has one (J306). */
|
|
24
|
+
readonly card: HeroCard | undefined
|
|
25
|
+
/** The hero digest api-server's UP4 takes (`hero.ts`); undefined with no card. */
|
|
26
|
+
readonly heroHash: string | undefined
|
|
27
|
+
/** Meta HERO-GATE HG2: whether the hero is present, captured, carried and the right size. */
|
|
28
|
+
readonly hero: HeroVerdict
|
|
29
|
+
/** `0.2`: a draft as `wildshard build` writes it; `0.3`: prepared (J198). */
|
|
30
|
+
readonly sdk: '0.2' | '0.3'
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function sha256(bytes: Readonly<Uint8Array>): string {
|
|
34
|
+
return createHash('sha256').update(bytes).digest('hex')
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** The tarball's files as format's read-only view of a product folder. */
|
|
38
|
+
function productFiles(files: ReadonlyMap<string, Uint8Array>): ProductFiles {
|
|
39
|
+
return {
|
|
40
|
+
read: (path: string, maxBytes: number): Uint8Array => {
|
|
41
|
+
const bytes = files.get(path)
|
|
42
|
+
if (bytes === undefined) {
|
|
43
|
+
throw new Error(`no ${path} in the shard file`)
|
|
44
|
+
}
|
|
45
|
+
if (bytes.length > maxBytes) {
|
|
46
|
+
throw new Error(`${path} is larger than ${String(maxBytes)} bytes`)
|
|
47
|
+
}
|
|
48
|
+
return bytes
|
|
49
|
+
},
|
|
50
|
+
paths: (): Iterable<string> => files.keys(),
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** HG2's verdict and the digest UP4 compares: a scene shard's card is its discovery mount's. */
|
|
55
|
+
function heroOf(
|
|
56
|
+
shard: Readonly<HeroShard>,
|
|
57
|
+
members: ReadonlyMap<string, Readonly<Uint8Array>>,
|
|
58
|
+
): Pick<ValidShardfile, 'hero' | 'heroHash'> {
|
|
59
|
+
const hero = heroVerdict(shard, members)
|
|
60
|
+
return { hero, heroHash: hero.ok ? hero.digest : heroHashOf(shard) }
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** The shard's identity; throws with format's reason when it is invalid. */
|
|
64
|
+
async function validateShardfile(
|
|
65
|
+
tarball: Readonly<Uint8Array>,
|
|
66
|
+
): Promise<ValidShardfile> {
|
|
67
|
+
installFormatHook()
|
|
68
|
+
const { prepareAssetDecoders } =
|
|
69
|
+
await import('@wildshard/format/shardfileAssets')
|
|
70
|
+
const { validateShardfileFolder: validateShardfileProduct } =
|
|
71
|
+
await import('@wildshard/format/shardfileValidation')
|
|
72
|
+
const { shardfileAudioReport: productAudioReport } =
|
|
73
|
+
await import('@wildshard/format/shardfileAudio')
|
|
74
|
+
const { readTar } = await import('@wildshard/format/tar')
|
|
75
|
+
await prepareAssetDecoders()
|
|
76
|
+
const members = readTar(tarball)
|
|
77
|
+
const valid = validateShardfileProduct(productFiles(members), {
|
|
78
|
+
hash: sha256,
|
|
79
|
+
})
|
|
80
|
+
const { identity, presentation } = valid.shard
|
|
81
|
+
return {
|
|
82
|
+
sdk: valid.index.sdk,
|
|
83
|
+
slug: identity.slug,
|
|
84
|
+
name: identity.name,
|
|
85
|
+
audio: productAudioReport(valid, new Date()),
|
|
86
|
+
card: presentation?.card,
|
|
87
|
+
...heroOf(valid.shard, members),
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export { assertHero } from './hero-gate.ts'
|
|
92
|
+
export { type ValidShardfile, validateShardfile }
|