@wildshard/upload 0.0.0-stage → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +28 -3
- package/src/cli/account/account.ts +199 -0
- package/src/cli/account/command.ts +193 -0
- package/src/cli/account/key.ts +154 -0
- package/src/cli/args.ts +100 -0
- package/src/cli/audio/audio.ts +39 -0
- package/src/cli/delete/delete.ts +258 -0
- package/src/cli/hero/hero.ts +163 -0
- package/src/cli/hero/scene-card.ts +46 -0
- package/src/cli/hero/stamp.ts +300 -0
- package/src/cli/report.ts +20 -0
- package/src/cli/shardfile/command.ts +49 -0
- package/src/cli/shardfile/shardfile.ts +192 -0
- package/src/cli/upload/api.ts +286 -0
- package/src/cli/upload/command.ts +239 -0
- package/src/cli/upload/completion.ts +80 -0
- package/src/cli/upload/compress.ts +55 -0
- package/src/cli/upload/hero-gate.ts +208 -0
- package/src/cli/upload/hero.ts +62 -0
- package/src/cli/upload/image-size.ts +89 -0
- package/src/cli/upload/placement.ts +132 -0
- package/src/cli/upload/provenance.ts +204 -0
- package/src/cli/upload/signature.ts +39 -0
- package/src/cli/upload/status-body.ts +41 -0
- package/src/cli/upload/status.ts +176 -0
- package/src/cli/upload/upload.ts +298 -0
- package/src/cli/upload/validate.ts +92 -0
- package/src/cli/upload/wire.ts +189 -0
- package/src/sharddev/draft.ts +11 -0
- package/src/sharddev/refusals.ts +329 -0
- package/src/sharddev/shardSource.ts +75 -0
- package/README.md +0 -3
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
import { safeParse } from 'valibot'
|
|
2
|
+
|
|
3
|
+
import { readStatusBody } from './status-body.ts'
|
|
4
|
+
import {
|
|
5
|
+
CatalogueSchema,
|
|
6
|
+
CommitAnswerSchema,
|
|
7
|
+
DirectUploadTicketSchema,
|
|
8
|
+
ErrorBodySchema,
|
|
9
|
+
ShardfileListSchema,
|
|
10
|
+
UploadStatusSchema,
|
|
11
|
+
} from './wire.ts'
|
|
12
|
+
|
|
13
|
+
import type { Provenance } from './provenance.ts'
|
|
14
|
+
import type { UploadIdentity } from './signature.ts'
|
|
15
|
+
import type {
|
|
16
|
+
CatalogueShard,
|
|
17
|
+
CommitAnswer,
|
|
18
|
+
DirectUploadTicket,
|
|
19
|
+
ShardfileSummary,
|
|
20
|
+
UploadStatus,
|
|
21
|
+
} from './wire.ts'
|
|
22
|
+
import type { BaseIssue, BaseSchema, InferOutput } from 'valibot'
|
|
23
|
+
|
|
24
|
+
// The large-file protocol of api-server (its docs/design.md, V7): ticket,
|
|
25
|
+
// PUT to R2, commit. Vercel's protection sits in front of the API calls only:
|
|
26
|
+
// the PUT goes straight to R2, so the bypass header never goes with it. The
|
|
27
|
+
// ticket and the commit carry the account key too (meta SHARD-DELETE DEL5,
|
|
28
|
+
// DEL6); the public reads and the PUT never do.
|
|
29
|
+
|
|
30
|
+
const SIGNATURE_HEADER = 'x-wildshard-signature'
|
|
31
|
+
const BYPASS_HEADER = 'x-vercel-protection-bypass'
|
|
32
|
+
const PUT_ATTEMPTS = 3
|
|
33
|
+
const FIRST_SERVER_ERROR = 500
|
|
34
|
+
const FIRST_ATTEMPT = 1
|
|
35
|
+
const BYPASS_HINT =
|
|
36
|
+
'not an api-server reply; a protected deployment needs VERCEL_AUTOMATION_BYPASS_SECRET'
|
|
37
|
+
|
|
38
|
+
interface ApiOptions {
|
|
39
|
+
/** The api-server origin. */
|
|
40
|
+
readonly server: string
|
|
41
|
+
/** The account's `wsk_` key, sent on the ticket and the commit. */
|
|
42
|
+
readonly apiKey: string
|
|
43
|
+
/** Vercel's automation bypass, for a protected deployment. */
|
|
44
|
+
readonly bypassSecret?: string
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
interface UploadRequest {
|
|
48
|
+
readonly hash: string
|
|
49
|
+
readonly storedBytes: number
|
|
50
|
+
readonly contentEncoding?: 'br'
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
interface PutTarget {
|
|
54
|
+
readonly url: string
|
|
55
|
+
readonly headers: Readonly<Record<string, string>>
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
type AnySchema = BaseSchema<unknown, unknown, BaseIssue<unknown>>
|
|
59
|
+
|
|
60
|
+
function apiHeaders(options: ApiOptions): Record<string, string> {
|
|
61
|
+
return options.bypassSecret === undefined
|
|
62
|
+
? {}
|
|
63
|
+
: { [BYPASS_HEADER]: options.bypassSecret }
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function jsonOf(text: string): unknown {
|
|
67
|
+
try {
|
|
68
|
+
const value: unknown = JSON.parse(text)
|
|
69
|
+
return value
|
|
70
|
+
} catch {
|
|
71
|
+
return undefined
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
class ApiRefusalError extends Error {
|
|
76
|
+
public override readonly name = 'ApiRefusalError'
|
|
77
|
+
public readonly status: number
|
|
78
|
+
public constructor(message: string, status: number) {
|
|
79
|
+
super(message)
|
|
80
|
+
this.status = status
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function refusal(path: string, status: number, body?: unknown): Error {
|
|
85
|
+
const parsed = safeParse(ErrorBodySchema, body)
|
|
86
|
+
const reason = parsed.success ? parsed.output.error : BYPASS_HINT
|
|
87
|
+
return new ApiRefusalError(
|
|
88
|
+
`${path} answered ${String(status)}: ${reason}`,
|
|
89
|
+
status,
|
|
90
|
+
)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** The reply's parsed body; a refusal or anything unparseable throws. */
|
|
94
|
+
async function replyOf<Schema extends AnySchema>(
|
|
95
|
+
path: string,
|
|
96
|
+
reply: Response,
|
|
97
|
+
schema: Schema,
|
|
98
|
+
): Promise<InferOutput<Schema>> {
|
|
99
|
+
const body = jsonOf(
|
|
100
|
+
await (path.startsWith('/api/shardfiles/uploads/')
|
|
101
|
+
? readStatusBody(reply)
|
|
102
|
+
: reply.text()),
|
|
103
|
+
)
|
|
104
|
+
if (!reply.ok) {
|
|
105
|
+
throw refusal(path, reply.status, body)
|
|
106
|
+
}
|
|
107
|
+
const parsed = safeParse(schema, body)
|
|
108
|
+
if (!parsed.success) {
|
|
109
|
+
throw refusal(path, reply.status)
|
|
110
|
+
}
|
|
111
|
+
return parsed.output
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** The status is private to the account, unlike catalogue reads. */
|
|
115
|
+
async function readUploadStatus(
|
|
116
|
+
options: ApiOptions,
|
|
117
|
+
id: string,
|
|
118
|
+
signal?: Readonly<AbortSignal>,
|
|
119
|
+
): Promise<UploadStatus> {
|
|
120
|
+
const path = `/api/shardfiles/uploads/${encodeURIComponent(id)}`
|
|
121
|
+
const reply = await fetch(new URL(path, options.server), {
|
|
122
|
+
headers: {
|
|
123
|
+
...apiHeaders(options),
|
|
124
|
+
authorization: `Bearer ${options.apiKey}`,
|
|
125
|
+
},
|
|
126
|
+
...(signal === undefined ? {} : { signal }),
|
|
127
|
+
})
|
|
128
|
+
return await replyOf(path, reply, UploadStatusSchema)
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
async function postUpload<Schema extends AnySchema>(
|
|
132
|
+
options: ApiOptions,
|
|
133
|
+
call: Readonly<{ path: string; signature: string | undefined; body: object }>,
|
|
134
|
+
schema: Schema,
|
|
135
|
+
): Promise<InferOutput<Schema>> {
|
|
136
|
+
const reply = await fetch(new URL(call.path, options.server), {
|
|
137
|
+
method: 'POST',
|
|
138
|
+
headers: {
|
|
139
|
+
...apiHeaders(options),
|
|
140
|
+
'content-type': 'application/json',
|
|
141
|
+
authorization: `Bearer ${options.apiKey}`,
|
|
142
|
+
...(call.signature === undefined
|
|
143
|
+
? {}
|
|
144
|
+
: { [SIGNATURE_HEADER]: call.signature }),
|
|
145
|
+
},
|
|
146
|
+
body: JSON.stringify(call.body),
|
|
147
|
+
})
|
|
148
|
+
return await replyOf(call.path, reply, schema)
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/** The stored shardfile with this hash, if there is one. */
|
|
152
|
+
async function findStored(
|
|
153
|
+
options: ApiOptions,
|
|
154
|
+
hash: string,
|
|
155
|
+
): Promise<ShardfileSummary | undefined> {
|
|
156
|
+
const path = '/api/shardfiles'
|
|
157
|
+
const reply = await fetch(new URL(path, options.server), {
|
|
158
|
+
headers: apiHeaders(options),
|
|
159
|
+
})
|
|
160
|
+
const { shardfiles } = await replyOf(path, reply, ShardfileListSchema)
|
|
161
|
+
return shardfiles.find(
|
|
162
|
+
(stored: Readonly<{ hash: string }>): boolean => stored.hash === hash,
|
|
163
|
+
)
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/** The slug's entry in `GET /api/catalogue`, if it has one. */
|
|
167
|
+
async function findCatalogued(
|
|
168
|
+
options: ApiOptions,
|
|
169
|
+
slug: string,
|
|
170
|
+
): Promise<CatalogueShard | undefined> {
|
|
171
|
+
const path = '/api/catalogue'
|
|
172
|
+
const reply = await fetch(new URL(path, options.server), {
|
|
173
|
+
headers: apiHeaders(options),
|
|
174
|
+
})
|
|
175
|
+
const { shards } = await replyOf(path, reply, CatalogueSchema)
|
|
176
|
+
return shards.find(
|
|
177
|
+
(shard: Readonly<{ slug: string }>): boolean => shard.slug === slug,
|
|
178
|
+
)
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
async function requestTicket(
|
|
182
|
+
options: ApiOptions,
|
|
183
|
+
request: UploadRequest,
|
|
184
|
+
signature: string | undefined,
|
|
185
|
+
): Promise<DirectUploadTicket> {
|
|
186
|
+
return await postUpload(
|
|
187
|
+
options,
|
|
188
|
+
{ path: '/api/shardfiles/uploads', signature, body: request },
|
|
189
|
+
DirectUploadTicketSchema,
|
|
190
|
+
)
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
async function attemptPut(
|
|
194
|
+
target: PutTarget,
|
|
195
|
+
body: Readonly<Uint8Array>,
|
|
196
|
+
): Promise<Response | undefined> {
|
|
197
|
+
try {
|
|
198
|
+
return await fetch(target.url, {
|
|
199
|
+
method: 'PUT',
|
|
200
|
+
headers: target.headers,
|
|
201
|
+
body: new Uint8Array(body),
|
|
202
|
+
})
|
|
203
|
+
} catch {
|
|
204
|
+
return undefined
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
function retries(reply: Response | undefined): boolean {
|
|
209
|
+
return reply === undefined || reply.status >= FIRST_SERVER_ERROR
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/** PUT with exactly the signed headers; a dropped connection or 5xx retries. */
|
|
213
|
+
async function putObject(
|
|
214
|
+
target: PutTarget,
|
|
215
|
+
body: Readonly<Uint8Array>,
|
|
216
|
+
attempt: number = FIRST_ATTEMPT,
|
|
217
|
+
): Promise<void> {
|
|
218
|
+
const reply = await attemptPut(target, body)
|
|
219
|
+
if (reply?.ok === true) {
|
|
220
|
+
return
|
|
221
|
+
}
|
|
222
|
+
if (retries(reply) && attempt < PUT_ATTEMPTS) {
|
|
223
|
+
await putObject(target, body, attempt + FIRST_ATTEMPT)
|
|
224
|
+
return
|
|
225
|
+
}
|
|
226
|
+
const answer =
|
|
227
|
+
reply === undefined
|
|
228
|
+
? 'no reply'
|
|
229
|
+
: `${String(reply.status)}: ${await reply.text()}`
|
|
230
|
+
throw new Error(`the PUT to R2 failed: ${answer}`)
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/** The channels an upload may ask for, by their wire names (J301). */
|
|
234
|
+
type Channel = 'staging' | 'production'
|
|
235
|
+
|
|
236
|
+
interface CommitBody extends Provenance {
|
|
237
|
+
readonly contentEncoding?: 'br' | undefined
|
|
238
|
+
readonly channel: Channel
|
|
239
|
+
/** False sends `place: false` (SA32's `--no-place`): no grid cell. */
|
|
240
|
+
readonly place: boolean
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* The commit: the bytes' encoding, the shard version and commit (V1), the
|
|
245
|
+
* channel it is for (J301), the versions it was built with (J306) and,
|
|
246
|
+
* for `--no-place`, `place: false` (SA32; the API places by default).
|
|
247
|
+
*/
|
|
248
|
+
async function commitUpload(
|
|
249
|
+
options: ApiOptions,
|
|
250
|
+
signed: UploadIdentity,
|
|
251
|
+
body: CommitBody,
|
|
252
|
+
): Promise<CommitAnswer> {
|
|
253
|
+
const { contentEncoding, version, commit, channel, place } = body
|
|
254
|
+
const { sharddev, engine, shardFormat } = body
|
|
255
|
+
return await postUpload(
|
|
256
|
+
options,
|
|
257
|
+
{
|
|
258
|
+
path: `/api/shardfiles/uploads/${signed.hash}/commit`,
|
|
259
|
+
signature: signed.signature,
|
|
260
|
+
body: {
|
|
261
|
+
...(contentEncoding === undefined ? {} : { contentEncoding }),
|
|
262
|
+
version,
|
|
263
|
+
commit,
|
|
264
|
+
channel,
|
|
265
|
+
sharddev,
|
|
266
|
+
engine,
|
|
267
|
+
shardFormat,
|
|
268
|
+
...(place ? {} : { place: false }),
|
|
269
|
+
},
|
|
270
|
+
},
|
|
271
|
+
CommitAnswerSchema,
|
|
272
|
+
)
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
export {
|
|
276
|
+
type ApiOptions,
|
|
277
|
+
type Channel,
|
|
278
|
+
type UploadRequest,
|
|
279
|
+
ApiRefusalError,
|
|
280
|
+
readUploadStatus,
|
|
281
|
+
commitUpload,
|
|
282
|
+
findCatalogued,
|
|
283
|
+
findStored,
|
|
284
|
+
putObject,
|
|
285
|
+
requestTicket,
|
|
286
|
+
}
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
import { apiKeyOf, uploadKeyRefusal } from '../account/command.ts'
|
|
2
|
+
import { refuse, reportLine } from '../report.ts'
|
|
3
|
+
import {
|
|
4
|
+
DEFAULT_BROTLI_LEVEL,
|
|
5
|
+
MAX_BROTLI_LEVEL,
|
|
6
|
+
MIN_BROTLI_LEVEL,
|
|
7
|
+
} from './compress.ts'
|
|
8
|
+
import { resolveProvenance } from './provenance.ts'
|
|
9
|
+
import { printUploadStatus } from './status.ts'
|
|
10
|
+
import { uploadShardfile } from './upload.ts'
|
|
11
|
+
|
|
12
|
+
import type { CliArgs } from '../args.ts'
|
|
13
|
+
import type { Provenance } from './provenance.ts'
|
|
14
|
+
import type { UploadOptions } from './upload.ts'
|
|
15
|
+
|
|
16
|
+
type Env = Readonly<Record<string, string | undefined>>
|
|
17
|
+
const EXIT_OK = 0
|
|
18
|
+
const EXIT_USAGE = 1
|
|
19
|
+
const EXIT_FAILED = 3
|
|
20
|
+
|
|
21
|
+
function parseLevel(raw: string | undefined): number | undefined {
|
|
22
|
+
if (raw === undefined) {
|
|
23
|
+
return DEFAULT_BROTLI_LEVEL
|
|
24
|
+
}
|
|
25
|
+
const level = Number(raw)
|
|
26
|
+
return Number.isInteger(level) &&
|
|
27
|
+
level >= MIN_BROTLI_LEVEL &&
|
|
28
|
+
level <= MAX_BROTLI_LEVEL
|
|
29
|
+
? level
|
|
30
|
+
: undefined
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function bypassOption(env: Env): { readonly bypassSecret?: string } {
|
|
34
|
+
const bypassSecret = env['VERCEL_AUTOMATION_BYPASS_SECRET'] ?? ''
|
|
35
|
+
return bypassSecret === '' ? {} : { bypassSecret }
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** The running `wildshard`, read by its caller: its version (SDK-SPLIT SP2a) and its folder, where the engine and
|
|
39
|
+
* shard-format resolve when the shard has none installed (SP2b). */
|
|
40
|
+
interface Uploader {
|
|
41
|
+
readonly version: string
|
|
42
|
+
readonly installDir?: string | undefined
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** The shard version and commit to send, or what is wrong (AC16). */
|
|
46
|
+
async function provenanceOf(
|
|
47
|
+
args: CliArgs,
|
|
48
|
+
uploader: Uploader,
|
|
49
|
+
): Promise<Provenance | string> {
|
|
50
|
+
try {
|
|
51
|
+
const provenance = await resolveProvenance({
|
|
52
|
+
cwd: process.cwd(),
|
|
53
|
+
sharddev: uploader.version,
|
|
54
|
+
installDir: uploader.installDir,
|
|
55
|
+
version: args.shardVersion,
|
|
56
|
+
commit: args.commit,
|
|
57
|
+
})
|
|
58
|
+
reportLine(
|
|
59
|
+
`shard source version ${provenance.version}, commit ${provenance.commit}`,
|
|
60
|
+
)
|
|
61
|
+
return provenance
|
|
62
|
+
} catch (error: unknown) {
|
|
63
|
+
return error instanceof Error ? error.message : String(error)
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** `--channel`'s wire name (J301): `prod` is the channel `production`. */
|
|
68
|
+
type Channel = UploadOptions['channel']
|
|
69
|
+
|
|
70
|
+
const CHANNELS: Readonly<Record<string, Channel>> = {
|
|
71
|
+
staging: 'staging',
|
|
72
|
+
prod: 'production',
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** The upload's channel by its wire name, if `--channel` names one. */
|
|
76
|
+
function channelOf(raw: string | undefined): Channel | undefined {
|
|
77
|
+
return raw !== undefined && Object.hasOwn(CHANNELS, raw)
|
|
78
|
+
? CHANNELS[raw]
|
|
79
|
+
: undefined
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function channelRefusal(raw: string | undefined): string {
|
|
83
|
+
const given = raw === undefined ? '' : `, not --channel ${raw}`
|
|
84
|
+
return `upload needs --channel staging or --channel prod${given}`
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
type Connection = Omit<
|
|
88
|
+
UploadOptions,
|
|
89
|
+
keyof Provenance | 'file' | 'channel' | 'place'
|
|
90
|
+
>
|
|
91
|
+
|
|
92
|
+
/** `--server`, else WILDSHARD_API_URL, else empty. */
|
|
93
|
+
function serverOf(args: CliArgs, env: Env): string {
|
|
94
|
+
return args.server ?? env['WILDSHARD_API_URL'] ?? ''
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** What stops upload before any request: the server or account credential. */
|
|
98
|
+
function credentialsRefusal(
|
|
99
|
+
server: string,
|
|
100
|
+
apiKey: string | undefined,
|
|
101
|
+
): string | undefined {
|
|
102
|
+
if (server === '') {
|
|
103
|
+
return 'upload needs --server (or WILDSHARD_API_URL)'
|
|
104
|
+
}
|
|
105
|
+
return uploadKeyRefusal(server, apiKey)
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* The level, server and keys, or what is wrong. No account key refuses
|
|
110
|
+
* here, before any request (meta SHARD-DELETE DEL6): api-server answers a
|
|
111
|
+
* keyless ticket 401 since DEL5.
|
|
112
|
+
*/
|
|
113
|
+
function connectionOf(
|
|
114
|
+
args: CliArgs,
|
|
115
|
+
apiKey: string | undefined,
|
|
116
|
+
env: Env,
|
|
117
|
+
): Connection | string {
|
|
118
|
+
const level = parseLevel(args.level)
|
|
119
|
+
const server = serverOf(args, env)
|
|
120
|
+
const privateKey = env['WILDSHARD_CREATOR_PRIVATE_KEY']?.trim()
|
|
121
|
+
if (level === undefined) {
|
|
122
|
+
return `invalid --level ${String(args.level)}`
|
|
123
|
+
}
|
|
124
|
+
const refusal = credentialsRefusal(server, apiKey)
|
|
125
|
+
if (refusal !== undefined || apiKey === undefined) {
|
|
126
|
+
return refusal ?? ''
|
|
127
|
+
}
|
|
128
|
+
return {
|
|
129
|
+
server,
|
|
130
|
+
apiKey,
|
|
131
|
+
...(privateKey === undefined || privateKey === '' ? {} : { privateKey }),
|
|
132
|
+
level,
|
|
133
|
+
report: reportLine,
|
|
134
|
+
...bypassOption(env),
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** Everything but the provenance, or what is wrong. */
|
|
139
|
+
function uploadSettings(
|
|
140
|
+
args: CliArgs,
|
|
141
|
+
apiKey: string | undefined,
|
|
142
|
+
env: Env,
|
|
143
|
+
): Omit<UploadOptions, keyof Provenance> | string {
|
|
144
|
+
const channel = channelOf(args.channel)
|
|
145
|
+
if (args.operand === undefined) {
|
|
146
|
+
return 'upload needs a shard file'
|
|
147
|
+
}
|
|
148
|
+
if (channel === undefined) {
|
|
149
|
+
return channelRefusal(args.channel)
|
|
150
|
+
}
|
|
151
|
+
const connection = connectionOf(args, apiKey, env)
|
|
152
|
+
return typeof connection === 'string'
|
|
153
|
+
? connection
|
|
154
|
+
: {
|
|
155
|
+
file: args.operand,
|
|
156
|
+
channel,
|
|
157
|
+
place: !args.noPlace,
|
|
158
|
+
wait: !args.noWait,
|
|
159
|
+
...connection,
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** The upload's settings from flags and environment, or what is wrong. */
|
|
164
|
+
async function uploadOptions(
|
|
165
|
+
args: CliArgs,
|
|
166
|
+
env: Env,
|
|
167
|
+
uploader: Uploader,
|
|
168
|
+
): Promise<UploadOptions | string> {
|
|
169
|
+
const apiKey = await apiKeyOf(env['WILDSHARD_API_KEY'])
|
|
170
|
+
if (apiKey instanceof Error) {
|
|
171
|
+
return apiKey.message
|
|
172
|
+
}
|
|
173
|
+
const settings = uploadSettings(args, apiKey, env)
|
|
174
|
+
if (typeof settings === 'string') {
|
|
175
|
+
return settings
|
|
176
|
+
}
|
|
177
|
+
const provenance = await provenanceOf(args, uploader)
|
|
178
|
+
return typeof provenance === 'string'
|
|
179
|
+
? provenance
|
|
180
|
+
: { ...settings, ...provenance }
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
async function runConnectedStatus(
|
|
184
|
+
connection: Connection,
|
|
185
|
+
id: string,
|
|
186
|
+
): Promise<number> {
|
|
187
|
+
try {
|
|
188
|
+
await printUploadStatus(connection, id, connection.report)
|
|
189
|
+
return EXIT_OK
|
|
190
|
+
} catch (error: unknown) {
|
|
191
|
+
refuse('upload', 'status failed: ', error)
|
|
192
|
+
return EXIT_FAILED
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
async function runStatus(args: CliArgs, env: Env): Promise<number> {
|
|
197
|
+
const key = await apiKeyOf(env['WILDSHARD_API_KEY'])
|
|
198
|
+
const connection =
|
|
199
|
+
key instanceof Error ? key.message : connectionOf(args, key, env)
|
|
200
|
+
if (typeof connection === 'string' || args.target === undefined) {
|
|
201
|
+
refuse(
|
|
202
|
+
'upload',
|
|
203
|
+
'',
|
|
204
|
+
typeof connection === 'string'
|
|
205
|
+
? connection
|
|
206
|
+
: 'upload status needs an upload id',
|
|
207
|
+
)
|
|
208
|
+
return EXIT_USAGE
|
|
209
|
+
}
|
|
210
|
+
return await runConnectedStatus(connection, args.target)
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
async function sendUpload(options: UploadOptions): Promise<number> {
|
|
214
|
+
try {
|
|
215
|
+
await uploadShardfile(options)
|
|
216
|
+
return EXIT_OK
|
|
217
|
+
} catch (error: unknown) {
|
|
218
|
+
refuse('upload', 'upload failed: ', error)
|
|
219
|
+
return EXIT_FAILED
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
async function runUpload(
|
|
224
|
+
args: CliArgs,
|
|
225
|
+
env: Env,
|
|
226
|
+
uploader: Uploader,
|
|
227
|
+
): Promise<number> {
|
|
228
|
+
if (args.operand === 'status') {
|
|
229
|
+
return await runStatus(args, env)
|
|
230
|
+
}
|
|
231
|
+
const options = await uploadOptions(args, env, uploader)
|
|
232
|
+
if (typeof options === 'string') {
|
|
233
|
+
refuse('upload', '', options)
|
|
234
|
+
return EXIT_USAGE
|
|
235
|
+
}
|
|
236
|
+
return await sendUpload(options)
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
export { runUpload }
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { placementLines } from './placement.ts'
|
|
2
|
+
import { waitForUpload } from './status.ts'
|
|
3
|
+
|
|
4
|
+
import type { ApiOptions, Channel } from './api.ts'
|
|
5
|
+
import type { CommitAnswer, UploadState } from './wire.ts'
|
|
6
|
+
|
|
7
|
+
interface CompletionOptions extends ApiOptions {
|
|
8
|
+
readonly wait?: boolean
|
|
9
|
+
readonly channel: Channel
|
|
10
|
+
readonly place?: boolean
|
|
11
|
+
readonly report: (line: string) => void
|
|
12
|
+
}
|
|
13
|
+
interface Completion {
|
|
14
|
+
readonly uploadId?: string | undefined
|
|
15
|
+
readonly status?: UploadState | undefined
|
|
16
|
+
readonly gridReport: readonly string[]
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function queued(
|
|
20
|
+
accepted: Readonly<{ uploadId: string; status: UploadState }>,
|
|
21
|
+
): Completion {
|
|
22
|
+
if (accepted.status === 'refused' || accepted.status === 'stalled') {
|
|
23
|
+
throw new Error(
|
|
24
|
+
`upload ${accepted.uploadId}: ${accepted.status}; ${accepted.status === 'stalled' ? 'it may still settle; ' : ''}read verdict with wildshard upload status ${accepted.uploadId}`,
|
|
25
|
+
)
|
|
26
|
+
}
|
|
27
|
+
return { ...accepted, gridReport: [] }
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function announce(
|
|
31
|
+
options: CompletionOptions,
|
|
32
|
+
accepted: Readonly<{ uploadId: string; status: UploadState | undefined }>,
|
|
33
|
+
): Readonly<{ uploadId: string; status: UploadState }> {
|
|
34
|
+
if (accepted.status === undefined) {
|
|
35
|
+
throw new Error('the API accepted an upload without its status')
|
|
36
|
+
}
|
|
37
|
+
options.report(`checked: API format accepted (upload ${accepted.uploadId})`)
|
|
38
|
+
options.report(`upload ${accepted.uploadId}: ${accepted.status}`)
|
|
39
|
+
return { uploadId: accepted.uploadId, status: accepted.status }
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Legacy servers finish at commit; the async protocol finishes at live. */
|
|
43
|
+
async function completeUpload(
|
|
44
|
+
options: CompletionOptions,
|
|
45
|
+
answer: CommitAnswer,
|
|
46
|
+
): Promise<Completion> {
|
|
47
|
+
const placing = { channel: options.channel, place: options.place !== false }
|
|
48
|
+
if (answer.uploadId === undefined) {
|
|
49
|
+
return { gridReport: placementLines(answer, placing) }
|
|
50
|
+
}
|
|
51
|
+
const accepted = announce(options, {
|
|
52
|
+
uploadId: answer.uploadId,
|
|
53
|
+
status: answer.status,
|
|
54
|
+
})
|
|
55
|
+
if (options.wait === false && accepted.status === 'live') {
|
|
56
|
+
return {
|
|
57
|
+
...accepted,
|
|
58
|
+
gridReport: placementLines(answer, placing),
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
if (options.wait === false) {
|
|
62
|
+
return queued(accepted)
|
|
63
|
+
}
|
|
64
|
+
const status = await waitForUpload(options, {
|
|
65
|
+
id: answer.uploadId,
|
|
66
|
+
hash: answer.hash,
|
|
67
|
+
report: options.report,
|
|
68
|
+
status: accepted.status,
|
|
69
|
+
})
|
|
70
|
+
return {
|
|
71
|
+
uploadId: answer.uploadId,
|
|
72
|
+
status: status.status,
|
|
73
|
+
gridReport: placementLines(
|
|
74
|
+
{ ...answer, placement: status.placement },
|
|
75
|
+
placing,
|
|
76
|
+
),
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export { type Completion, completeUpload }
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { promisify } from 'node:util'
|
|
2
|
+
import { brotliCompress, constants } from 'node:zlib'
|
|
3
|
+
|
|
4
|
+
// J96: the author's machine compresses. The whole tarball is one Brotli
|
|
5
|
+
// stream, because that is what `Content-Encoding: br` means to R2 and to a
|
|
6
|
+
// browser, so a tar member cannot opt out. It does not need to: the encoder
|
|
7
|
+
// stores incompressible runs raw, so KTX2 costs almost nothing.
|
|
8
|
+
|
|
9
|
+
const compress = promisify(brotliCompress)
|
|
10
|
+
|
|
11
|
+
// Measured on a 271 MB tarball of real game assets mixed to save what Pine
|
|
12
|
+
// Hollow saves at level 5: level 6 stores 216.9 MB in about 2 s, the last
|
|
13
|
+
// cheap step. Level 7 takes three times as long to shed 0.4 MB; level 11 sheds
|
|
14
|
+
// 5.4 MB for 278 s. docs/design.md has the table.
|
|
15
|
+
const DEFAULT_BROTLI_LEVEL = 6
|
|
16
|
+
const MIN_BROTLI_LEVEL = 0
|
|
17
|
+
const MAX_BROTLI_LEVEL = 11
|
|
18
|
+
// A larger window saved nothing at the default level, and every decoder
|
|
19
|
+
// reads this one.
|
|
20
|
+
const WINDOW_BITS = 22
|
|
21
|
+
// Below this, Brotli only adds a decode for every reader of the object.
|
|
22
|
+
const WORTHWHILE_SAVING = 0.01
|
|
23
|
+
const WHOLE = 1
|
|
24
|
+
|
|
25
|
+
interface StoredForm {
|
|
26
|
+
/** What the PUT carries. */
|
|
27
|
+
readonly body: Readonly<Uint8Array>
|
|
28
|
+
/** `br` when `body` is the tarball's Brotli; absent for the tarball. */
|
|
29
|
+
readonly contentEncoding?: 'br'
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** The tarball's Brotli, or the tarball when that saves next to nothing. */
|
|
33
|
+
async function storedForm(
|
|
34
|
+
tarball: Readonly<Uint8Array>,
|
|
35
|
+
level: number,
|
|
36
|
+
): Promise<StoredForm> {
|
|
37
|
+
const brotli = await compress(tarball, {
|
|
38
|
+
params: {
|
|
39
|
+
[constants.BROTLI_PARAM_QUALITY]: level,
|
|
40
|
+
[constants.BROTLI_PARAM_LGWIN]: WINDOW_BITS,
|
|
41
|
+
[constants.BROTLI_PARAM_SIZE_HINT]: tarball.length,
|
|
42
|
+
},
|
|
43
|
+
})
|
|
44
|
+
return brotli.length < tarball.length * (WHOLE - WORTHWHILE_SAVING)
|
|
45
|
+
? { body: brotli, contentEncoding: 'br' }
|
|
46
|
+
: { body: tarball }
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export {
|
|
50
|
+
DEFAULT_BROTLI_LEVEL,
|
|
51
|
+
MAX_BROTLI_LEVEL,
|
|
52
|
+
MIN_BROTLI_LEVEL,
|
|
53
|
+
type StoredForm,
|
|
54
|
+
storedForm,
|
|
55
|
+
}
|