@valbuild/server 0.140.2 → 0.141.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +83 -0
- package/dist/declarations/src/ValOps.d.ts +20 -0
- package/dist/declarations/src/ValOpsFS.d.ts +5 -1
- package/dist/declarations/src/createFixPatch.d.ts +3 -1
- package/dist/declarations/src/extractMetadata.d.ts +47 -1
- package/dist/declarations/src/fixHandlers.d.ts +38 -33
- package/dist/declarations/src/galleryEntryKey.d.ts +35 -0
- package/dist/declarations/src/galleryFiles.d.ts +59 -0
- package/dist/declarations/src/index.d.ts +5 -2
- package/dist/declarations/src/isoBmff.d.ts +41 -0
- package/dist/declarations/src/valServerConfig.d.ts +8 -0
- package/dist/declarations/src/videoFiles.d.ts +22 -0
- package/dist/declarations/src/videosetFixes.d.ts +141 -0
- package/dist/valbuild-server.cjs.dev.js +3376 -432
- package/dist/valbuild-server.cjs.prod.js +3376 -432
- package/dist/valbuild-server.esm.js +3368 -434
- package/package.json +4 -4
|
@@ -8,7 +8,7 @@ import path__default from 'path';
|
|
|
8
8
|
import fs from 'fs';
|
|
9
9
|
import vm from 'node:vm';
|
|
10
10
|
import { Module } from 'node:module';
|
|
11
|
-
import { isSchemaSourceFixError, resolveSchemaSourceFixForError, Patch, getErrorMessageFromUnknownJson, PatchId as PatchId$1, JSONValue, PatchGroup, newestCommitSha, SerializedSchema, VAL_ENABLE_COOKIE_NAME, VAL_STATE_COOKIE, VAL_SESSION_COOKIE, parseJob, VAL_STUDIO_HEADER, Api } from '@valbuild/shared/internal';
|
|
11
|
+
import { isSchemaSourceFixError, resolveSchemaSourceFixForError, Patch, getErrorMessageFromUnknownJson, PatchId as PatchId$1, JSONValue, PatchGroup, newestCommitSha, SerializedSchema, VAL_ENABLE_COOKIE_NAME, VAL_STATE_COOKIE, VAL_SESSION_COOKIE, parseJob, VAL_STUDIO_HEADER, Api, traverseSchemaSource } from '@valbuild/shared/internal';
|
|
12
12
|
import { createUIRequestHandler } from '@valbuild/ui/server';
|
|
13
13
|
import crypto$1 from 'crypto';
|
|
14
14
|
import z$1, { z } from 'zod';
|
|
@@ -18,6 +18,7 @@ import net from 'node:net';
|
|
|
18
18
|
import sizeOf from 'image-size';
|
|
19
19
|
import { fromError } from 'zod-validation-error';
|
|
20
20
|
import os from 'os';
|
|
21
|
+
import { Buffer as Buffer$1 } from 'buffer';
|
|
21
22
|
import { transform } from 'sucrase';
|
|
22
23
|
import http$1 from 'http';
|
|
23
24
|
import https$1 from 'https';
|
|
@@ -2460,6 +2461,24 @@ function defineExternal(definition) {
|
|
|
2460
2461
|
throw new Error("defineExternal is not implemented yet: phase 0 lands the contract, the registry that executes it arrives with the read endpoints.");
|
|
2461
2462
|
}
|
|
2462
2463
|
|
|
2464
|
+
/**
|
|
2465
|
+
* Does saving this commit upload a file to Val's remote host?
|
|
2466
|
+
*
|
|
2467
|
+
* The question `/save` asks before it reads remote credentials. It used to ask
|
|
2468
|
+
* the SCHEMAS instead -- "is any of them remote?" -- and with
|
|
2469
|
+
* `files: { remote: true }` every media schema is, so every save needed a
|
|
2470
|
+
* `val login` token or an api key: a corrected typo included. Only a remote
|
|
2471
|
+
* FILE is uploaded, so only a commit carrying one needs credentials.
|
|
2472
|
+
*/
|
|
2473
|
+
function commitCarriesRemoteFiles(preparedCommit) {
|
|
2474
|
+
return Object.values(preparedCommit.patchedBinaryFilesDescriptors).some(descriptor => descriptor.remote);
|
|
2475
|
+
}
|
|
2476
|
+
|
|
2477
|
+
/** Does this commit carry any file at all, remote or local? */
|
|
2478
|
+
function commitCarriesFiles(preparedCommit) {
|
|
2479
|
+
return Object.keys(preparedCommit.patchedBinaryFilesDescriptors).length > 0;
|
|
2480
|
+
}
|
|
2481
|
+
|
|
2463
2482
|
const JwtPayloadSchema = z.object({
|
|
2464
2483
|
sub: z.string(),
|
|
2465
2484
|
exp: z.number(),
|
|
@@ -3181,6 +3200,291 @@ function readableError(error) {
|
|
|
3181
3200
|
}
|
|
3182
3201
|
}
|
|
3183
3202
|
|
|
3203
|
+
/**
|
|
3204
|
+
* The `Range` request header, for the one route that serves bytes a player
|
|
3205
|
+
* seeks in: `/files`.
|
|
3206
|
+
*
|
|
3207
|
+
* A `<video>` element asks for ranges as a matter of course, and Safari will
|
|
3208
|
+
* not play a video at all from a server that does not answer them. An HLS
|
|
3209
|
+
* stream whose segments are byte ranges of one file (`#EXT-X-BYTERANGE`) asks
|
|
3210
|
+
* for nothing else.
|
|
3211
|
+
*
|
|
3212
|
+
* Only a single range is honoured. A multi-range request (`bytes=0-1,5-6`) is
|
|
3213
|
+
* answered with the whole file, which RFC 9110 permits, and nothing that plays
|
|
3214
|
+
* media sends one. A header that does not parse is ignored the same way.
|
|
3215
|
+
*/
|
|
3216
|
+
|
|
3217
|
+
function parseRangeHeader(header, size) {
|
|
3218
|
+
if (!header) {
|
|
3219
|
+
return {
|
|
3220
|
+
kind: "none"
|
|
3221
|
+
};
|
|
3222
|
+
}
|
|
3223
|
+
const match = /^\s*bytes\s*=\s*(.*)$/i.exec(header);
|
|
3224
|
+
if (!match) {
|
|
3225
|
+
return {
|
|
3226
|
+
kind: "none"
|
|
3227
|
+
};
|
|
3228
|
+
}
|
|
3229
|
+
const spec = match[1].trim();
|
|
3230
|
+
if (spec.includes(",")) {
|
|
3231
|
+
return {
|
|
3232
|
+
kind: "none"
|
|
3233
|
+
};
|
|
3234
|
+
}
|
|
3235
|
+
const suffix = /^-(\d+)$/.exec(spec);
|
|
3236
|
+
if (suffix) {
|
|
3237
|
+
const length = Number(suffix[1]);
|
|
3238
|
+
if (length === 0 || size === 0) {
|
|
3239
|
+
return {
|
|
3240
|
+
kind: "unsatisfiable"
|
|
3241
|
+
};
|
|
3242
|
+
}
|
|
3243
|
+
return {
|
|
3244
|
+
kind: "range",
|
|
3245
|
+
start: Math.max(0, size - length),
|
|
3246
|
+
end: size - 1
|
|
3247
|
+
};
|
|
3248
|
+
}
|
|
3249
|
+
const bounded = /^(\d+)-(\d*)$/.exec(spec);
|
|
3250
|
+
if (!bounded) {
|
|
3251
|
+
return {
|
|
3252
|
+
kind: "none"
|
|
3253
|
+
};
|
|
3254
|
+
}
|
|
3255
|
+
const start = Number(bounded[1]);
|
|
3256
|
+
const end = bounded[2] === "" ? undefined : Number(bounded[2]);
|
|
3257
|
+
if (end !== undefined && end < start) {
|
|
3258
|
+
// Syntactically invalid (RFC 9110 §14.1.1): ignored, not refused.
|
|
3259
|
+
return {
|
|
3260
|
+
kind: "none"
|
|
3261
|
+
};
|
|
3262
|
+
}
|
|
3263
|
+
if (start >= size) {
|
|
3264
|
+
return {
|
|
3265
|
+
kind: "unsatisfiable"
|
|
3266
|
+
};
|
|
3267
|
+
}
|
|
3268
|
+
return {
|
|
3269
|
+
kind: "range",
|
|
3270
|
+
start,
|
|
3271
|
+
end: Math.min(end ?? size - 1, size - 1)
|
|
3272
|
+
};
|
|
3273
|
+
}
|
|
3274
|
+
|
|
3275
|
+
/**
|
|
3276
|
+
* The little of HLS (RFC 8216) the server has to read: the variants of a
|
|
3277
|
+
* master playlist, the segment durations of a media playlist, and every URI a
|
|
3278
|
+
* playlist names.
|
|
3279
|
+
*
|
|
3280
|
+
* Deliberately a line scanner rather than a parser. The playlists Val reads are
|
|
3281
|
+
* the ones the Studio wrote, and the questions asked of them are small — "how
|
|
3282
|
+
* big is the largest rendition", "how long is it", "where does each URI
|
|
3283
|
+
* point" — so a full model of the format would be code nothing exercises.
|
|
3284
|
+
*/
|
|
3285
|
+
|
|
3286
|
+
/** One `#EXT-X-STREAM-INF` entry of a master playlist. */
|
|
3287
|
+
|
|
3288
|
+
/** Lines of a playlist, with Windows line endings tolerated. */
|
|
3289
|
+
function linesOf(text) {
|
|
3290
|
+
return text.split(/\r?\n/);
|
|
3291
|
+
}
|
|
3292
|
+
|
|
3293
|
+
/**
|
|
3294
|
+
* The attributes of a tag, e.g. the part after `#EXT-X-STREAM-INF:`.
|
|
3295
|
+
*
|
|
3296
|
+
* Quoted values may contain commas (`CODECS="avc1.64001f,mp4a.40.2"`), which
|
|
3297
|
+
* is why this is not a `split(",")`. Quotes are removed from the values.
|
|
3298
|
+
*/
|
|
3299
|
+
function parseHlsAttributes(list) {
|
|
3300
|
+
const attributes = {};
|
|
3301
|
+
const re = /([A-Z0-9-]+)=("[^"]*"|[^,]*)/g;
|
|
3302
|
+
let match;
|
|
3303
|
+
while ((match = re.exec(list)) !== null) {
|
|
3304
|
+
const [, name, rawValue] = match;
|
|
3305
|
+
attributes[name] = rawValue.startsWith('"') && rawValue.endsWith('"') ? rawValue.slice(1, -1) : rawValue.trim();
|
|
3306
|
+
}
|
|
3307
|
+
return attributes;
|
|
3308
|
+
}
|
|
3309
|
+
|
|
3310
|
+
/** Whether a playlist is a master (multivariant) playlist. */
|
|
3311
|
+
function isHlsMasterPlaylist(text) {
|
|
3312
|
+
return linesOf(text).some(line => line.trim().startsWith("#EXT-X-STREAM-INF:"));
|
|
3313
|
+
}
|
|
3314
|
+
|
|
3315
|
+
/** The variants of a master playlist, in the order they are listed. */
|
|
3316
|
+
function parseHlsMasterVariants(text) {
|
|
3317
|
+
const variants = [];
|
|
3318
|
+
let pending = null;
|
|
3319
|
+
for (const rawLine of linesOf(text)) {
|
|
3320
|
+
const line = rawLine.trim();
|
|
3321
|
+
if (line === "") {
|
|
3322
|
+
continue;
|
|
3323
|
+
}
|
|
3324
|
+
if (line.startsWith("#EXT-X-STREAM-INF:")) {
|
|
3325
|
+
const attributes = parseHlsAttributes(line.slice("#EXT-X-STREAM-INF:".length));
|
|
3326
|
+
const variant = {};
|
|
3327
|
+
const bandwidth = Number(attributes["BANDWIDTH"]);
|
|
3328
|
+
if (attributes["BANDWIDTH"] !== undefined && Number.isFinite(bandwidth)) {
|
|
3329
|
+
variant.bandwidth = bandwidth;
|
|
3330
|
+
}
|
|
3331
|
+
const resolution = /^(\d+)x(\d+)$/.exec(attributes["RESOLUTION"] ?? "");
|
|
3332
|
+
if (resolution) {
|
|
3333
|
+
variant.width = Number(resolution[1]);
|
|
3334
|
+
variant.height = Number(resolution[2]);
|
|
3335
|
+
}
|
|
3336
|
+
pending = variant;
|
|
3337
|
+
continue;
|
|
3338
|
+
}
|
|
3339
|
+
if (line.startsWith("#")) {
|
|
3340
|
+
continue;
|
|
3341
|
+
}
|
|
3342
|
+
// The URI line that follows a STREAM-INF tag belongs to it.
|
|
3343
|
+
if (pending) {
|
|
3344
|
+
variants.push({
|
|
3345
|
+
...pending,
|
|
3346
|
+
uri: line
|
|
3347
|
+
});
|
|
3348
|
+
pending = null;
|
|
3349
|
+
}
|
|
3350
|
+
}
|
|
3351
|
+
return variants;
|
|
3352
|
+
}
|
|
3353
|
+
|
|
3354
|
+
/**
|
|
3355
|
+
* The variant whose dimensions describe the video: the largest picture, and
|
|
3356
|
+
* among equally large ones the highest bandwidth. `undefined` when no variant
|
|
3357
|
+
* declares a `RESOLUTION`.
|
|
3358
|
+
*/
|
|
3359
|
+
function largestHlsVariant(variants) {
|
|
3360
|
+
let best;
|
|
3361
|
+
for (const variant of variants) {
|
|
3362
|
+
const {
|
|
3363
|
+
width,
|
|
3364
|
+
height
|
|
3365
|
+
} = variant;
|
|
3366
|
+
if (width === undefined || height === undefined) {
|
|
3367
|
+
continue;
|
|
3368
|
+
}
|
|
3369
|
+
if (!best || width * height > best.width * best.height || width * height === best.width * best.height && (variant.bandwidth ?? 0) > (best.bandwidth ?? 0)) {
|
|
3370
|
+
best = {
|
|
3371
|
+
...variant,
|
|
3372
|
+
width,
|
|
3373
|
+
height
|
|
3374
|
+
};
|
|
3375
|
+
}
|
|
3376
|
+
}
|
|
3377
|
+
return best;
|
|
3378
|
+
}
|
|
3379
|
+
|
|
3380
|
+
/**
|
|
3381
|
+
* The sum of a media playlist's `#EXTINF` durations, in seconds, or
|
|
3382
|
+
* `undefined` when it has none (a master playlist, or not a playlist).
|
|
3383
|
+
*/
|
|
3384
|
+
function sumHlsSegmentDurations(text) {
|
|
3385
|
+
let total = 0;
|
|
3386
|
+
let found = false;
|
|
3387
|
+
for (const rawLine of linesOf(text)) {
|
|
3388
|
+
const line = rawLine.trim();
|
|
3389
|
+
if (!line.startsWith("#EXTINF:")) {
|
|
3390
|
+
continue;
|
|
3391
|
+
}
|
|
3392
|
+
const seconds = Number.parseFloat(line.slice("#EXTINF:".length));
|
|
3393
|
+
if (Number.isFinite(seconds)) {
|
|
3394
|
+
total += seconds;
|
|
3395
|
+
found = true;
|
|
3396
|
+
}
|
|
3397
|
+
}
|
|
3398
|
+
return found ? total : undefined;
|
|
3399
|
+
}
|
|
3400
|
+
|
|
3401
|
+
/**
|
|
3402
|
+
* Rewrite every URI a playlist names, leaving everything else byte for byte.
|
|
3403
|
+
*
|
|
3404
|
+
* URIs are the non-comment lines (segments, variant playlists) and the `URI`
|
|
3405
|
+
* attribute of tags such as `#EXT-X-MAP` and `#EXT-X-MEDIA`. Line endings are
|
|
3406
|
+
* kept as they were.
|
|
3407
|
+
*/
|
|
3408
|
+
function mapHlsUris(text, map) {
|
|
3409
|
+
return text.split(/(\r?\n)/).map(part => {
|
|
3410
|
+
if (part === "\n" || part === "\r\n") {
|
|
3411
|
+
return part;
|
|
3412
|
+
}
|
|
3413
|
+
const trimmed = part.trim();
|
|
3414
|
+
if (trimmed === "") {
|
|
3415
|
+
return part;
|
|
3416
|
+
}
|
|
3417
|
+
if (trimmed.startsWith("#")) {
|
|
3418
|
+
// Only an attribute that is exactly `URI` — preceded by the tag's
|
|
3419
|
+
// colon or a comma — and not, say, a `KEYFORMAT-URI` lookalike.
|
|
3420
|
+
return part.replace(/([:,]\s*)URI="([^"]*)"/g, (_match, before, uri) => `${before}URI="${map(uri)}"`);
|
|
3421
|
+
}
|
|
3422
|
+
const leading = part.slice(0, part.indexOf(trimmed));
|
|
3423
|
+
const trailing = part.slice(part.indexOf(trimmed) + trimmed.length);
|
|
3424
|
+
return `${leading}${map(trimmed)}${trailing}`;
|
|
3425
|
+
}).join("");
|
|
3426
|
+
}
|
|
3427
|
+
|
|
3428
|
+
/**
|
|
3429
|
+
* An HLS playlist served as a DRAFT, with every URI in it pointed at the draft
|
|
3430
|
+
* endpoint.
|
|
3431
|
+
*
|
|
3432
|
+
* Why this exists: a draft is served from `/api/val/files<path>?patch_id=…`,
|
|
3433
|
+
* and a player resolves the playlist's URIs against that URL. A RELATIVE URI
|
|
3434
|
+
* (`720p.m3u8`, how a local stream is written so it works wherever the
|
|
3435
|
+
* directory is served from) resolves to `/api/val/files/public/val/x/720p.m3u8`
|
|
3436
|
+
* — the query string is dropped by URL resolution, so the `patch_id` is gone,
|
|
3437
|
+
* and the endpoint looks for a PUBLISHED file that does not exist yet. An
|
|
3438
|
+
* ABSOLUTE remote ref (how a remote stream is written, because the content host
|
|
3439
|
+
* addresses files by hash) points at the content host, which has nothing until
|
|
3440
|
+
* publish. Either way the master plays and nothing it names does.
|
|
3441
|
+
*
|
|
3442
|
+
* So each URI is rewritten to the URL `Internal.mediaUrl` gives the same file
|
|
3443
|
+
* as a draft. Every file of one upload is in the same patch, so they share the
|
|
3444
|
+
* `patch_id` the playlist itself was requested with.
|
|
3445
|
+
*
|
|
3446
|
+
* What is NOT rewritten, and why:
|
|
3447
|
+
* - an absolute URL that is not a Val remote ref (another host's stream): it
|
|
3448
|
+
* was never in a patch;
|
|
3449
|
+
* - a root-relative URI (`/val/x/720p.mp4`): the Studio never writes one, and
|
|
3450
|
+
* it already names a published URL, so it is not this function's to guess at.
|
|
3451
|
+
*
|
|
3452
|
+
* Pure, and the whole rule: the `/files` route calls it and nothing else.
|
|
3453
|
+
*/
|
|
3454
|
+
function rewriteDraftPlaylist(playlist, options) {
|
|
3455
|
+
return mapHlsUris(playlist, uri => rewriteUri(uri, options));
|
|
3456
|
+
}
|
|
3457
|
+
function rewriteUri(uri, {
|
|
3458
|
+
playlistPath,
|
|
3459
|
+
patchId,
|
|
3460
|
+
remote
|
|
3461
|
+
}) {
|
|
3462
|
+
if (Internal.remote.splitRemoteRef(uri).status === "success") {
|
|
3463
|
+
return Internal.mediaUrl({
|
|
3464
|
+
path: uri,
|
|
3465
|
+
patch_id: patchId
|
|
3466
|
+
});
|
|
3467
|
+
}
|
|
3468
|
+
if (/^[a-z][a-z0-9+.-]*:/i.test(uri) || uri.startsWith("/")) {
|
|
3469
|
+
// Another scheme (https:, data:, …), protocol-relative or root-relative.
|
|
3470
|
+
return uri;
|
|
3471
|
+
}
|
|
3472
|
+
// The query and fragment of a relative URI address the published file; the
|
|
3473
|
+
// draft endpoint has its own query, and fragments do not reach a server.
|
|
3474
|
+
const relative = uri.split(/[?#]/)[0];
|
|
3475
|
+
if (relative === "") {
|
|
3476
|
+
return uri;
|
|
3477
|
+
}
|
|
3478
|
+
// `join` from an absolute directory stays absolute: `..` stops at the root.
|
|
3479
|
+
const resolved = path__default.posix.join(path__default.posix.dirname(playlistPath), relative);
|
|
3480
|
+
return `/api/val/files${resolved}?patch_id=${patchId}${remote ? "&remote=true" : ""}`;
|
|
3481
|
+
}
|
|
3482
|
+
|
|
3483
|
+
/** Whether a served file is an HLS playlist, by its extension. */
|
|
3484
|
+
function isHlsPlaylistPath(filePath) {
|
|
3485
|
+
return filePath.split("?")[0].toLowerCase().endsWith(".m3u8");
|
|
3486
|
+
}
|
|
3487
|
+
|
|
3184
3488
|
/* eslint-disable @typescript-eslint/no-unused-vars */
|
|
3185
3489
|
const jsonOps = new JSONOps();
|
|
3186
3490
|
/**
|
|
@@ -4004,9 +4308,20 @@ class ValOps {
|
|
|
4004
4308
|
// when constructing the url we use the patch id (and the file path)
|
|
4005
4309
|
// to fetch the right file
|
|
4006
4310
|
// NOTE: overwrite and use last patch_id if multiple patches modify the same file
|
|
4007
|
-
|
|
4311
|
+
//
|
|
4312
|
+
// Keyed by where the patch_id LANDS — the field's path plus
|
|
4313
|
+
// `nestedFilePath` — not by the field's path alone. A video's
|
|
4314
|
+
// poster and caption files are file ops on the video's own path
|
|
4315
|
+
// with `nestedFilePath` ["poster"] / ["captions", "0"]; keyed by
|
|
4316
|
+
// `op.path` they overwrote the video's entry (and each other),
|
|
4317
|
+
// so only one of the files got a patch_id and the rest were
|
|
4318
|
+
// looked for at a published URL that holds nothing. JSON rather
|
|
4319
|
+
// than a "/" join because a segment can contain "/" (a gallery
|
|
4320
|
+
// key is a file path).
|
|
4321
|
+
const patchIdPath = op.path.concat(...(op.nestedFilePath || [])).concat("patch_id");
|
|
4322
|
+
fileFixOps[JSON.stringify(patchIdPath)] = [{
|
|
4008
4323
|
op: "add",
|
|
4009
|
-
path:
|
|
4324
|
+
path: patchIdPath,
|
|
4010
4325
|
value: patchId
|
|
4011
4326
|
}];
|
|
4012
4327
|
}
|
|
@@ -4204,7 +4519,7 @@ class ValOps {
|
|
|
4204
4519
|
};
|
|
4205
4520
|
if (validationErrors) {
|
|
4206
4521
|
for (const validationError of validationErrors) {
|
|
4207
|
-
var _validationError$fixe, _validationError$fixe2, _validationError$fixe3, _validationError$fixe4, _validationError$fixe5, _validationError$fixe6;
|
|
4522
|
+
var _validationError$fixe, _validationError$fixe2, _validationError$fixe3, _validationError$fixe4, _validationError$fixe5, _validationError$fixe6, _validationError$fixe7, _validationError$fixe8;
|
|
4208
4523
|
if (isOnlyFileCheckValidationError(validationError)) {
|
|
4209
4524
|
if (files[sourcePath]) {
|
|
4210
4525
|
addError({
|
|
@@ -4224,7 +4539,7 @@ class ValOps {
|
|
|
4224
4539
|
addError(resolved.error);
|
|
4225
4540
|
}
|
|
4226
4541
|
// resolved.status === "resolved" → drop silently
|
|
4227
|
-
} else if ((_validationError$fixe3 = validationError.fixes) !== null && _validationError$fixe3 !== void 0 && _validationError$fixe3.includes("images:check-unique-folder") || (_validationError$fixe4 = validationError.fixes) !== null && _validationError$fixe4 !== void 0 && _validationError$fixe4.includes("files:check-unique-folder")) {
|
|
4542
|
+
} else if ((_validationError$fixe3 = validationError.fixes) !== null && _validationError$fixe3 !== void 0 && _validationError$fixe3.includes("images:check-unique-folder") || (_validationError$fixe4 = validationError.fixes) !== null && _validationError$fixe4 !== void 0 && _validationError$fixe4.includes("files:check-unique-folder") || (_validationError$fixe5 = validationError.fixes) !== null && _validationError$fixe5 !== void 0 && _validationError$fixe5.includes("videos:check-unique-folder")) {
|
|
4228
4543
|
const TYPE_ERROR_MESSAGE = `This is most likely a Val version mismatch or Val bug.`;
|
|
4229
4544
|
if (!validationError.value || typeof validationError.value !== "object") {
|
|
4230
4545
|
addError({
|
|
@@ -4249,7 +4564,7 @@ class ValOps {
|
|
|
4249
4564
|
// If conflictingModules is empty, directory is unique — silently drop the error.
|
|
4250
4565
|
}
|
|
4251
4566
|
}
|
|
4252
|
-
} else if ((_validationError$
|
|
4567
|
+
} else if ((_validationError$fixe6 = validationError.fixes) !== null && _validationError$fixe6 !== void 0 && _validationError$fixe6.includes("images:check-all-files") || (_validationError$fixe7 = validationError.fixes) !== null && _validationError$fixe7 !== void 0 && _validationError$fixe7.includes("files:check-all-files") || (_validationError$fixe8 = validationError.fixes) !== null && _validationError$fixe8 !== void 0 && _validationError$fixe8.includes("videos:check-all-files")) ; else {
|
|
4253
4568
|
addError(validationError);
|
|
4254
4569
|
}
|
|
4255
4570
|
}
|
|
@@ -5354,6 +5669,19 @@ class ValOps {
|
|
|
5354
5669
|
async saveBinaryFileFromPatch(filePath, parentRef, patchId, bytes, mimeType, type, metadata) {
|
|
5355
5670
|
return this.saveBase64EncodedBinaryFileFromPatch(filePath, parentRef, patchId, `data:${mimeType};base64,${bytes.toString("base64")}`, type, metadata);
|
|
5356
5671
|
}
|
|
5672
|
+
/**
|
|
5673
|
+
* A file's bytes as something a `Range` request can be answered from
|
|
5674
|
+
* without reading the rest: a video is seeked in by many small ranges, and
|
|
5675
|
+
* each used to load the whole file.
|
|
5676
|
+
*
|
|
5677
|
+
* This default holds the whole file, which is all a mode that only gets
|
|
5678
|
+
* whole files can do (`ValOpsHttp`: the content service answers with the
|
|
5679
|
+
* file in a JSON body). `ValOpsFS` reads only the asked-for range off disk.
|
|
5680
|
+
*/
|
|
5681
|
+
async openBinaryFile(filePath, fromPatch) {
|
|
5682
|
+
const buffer = fromPatch ? await this.getBase64EncodedBinaryFileFromPatch(filePath, fromPatch.patchId, fromPatch.remote) : await this.getBinaryFile(filePath);
|
|
5683
|
+
return buffer === null ? null : bufferReader(buffer);
|
|
5684
|
+
}
|
|
5357
5685
|
|
|
5358
5686
|
// #region history
|
|
5359
5687
|
//
|
|
@@ -5404,8 +5732,8 @@ class ValOps {
|
|
|
5404
5732
|
// #endregion history
|
|
5405
5733
|
}
|
|
5406
5734
|
function isOnlyFileCheckValidationError(validationError) {
|
|
5407
|
-
var _validationError$
|
|
5408
|
-
if ((_validationError$
|
|
5735
|
+
var _validationError$fixe9;
|
|
5736
|
+
if ((_validationError$fixe9 = validationError.fixes) !== null && _validationError$fixe9 !== void 0 && _validationError$fixe9.every(f => f === "file:check-metadata" || f === "image:check-metadata")) {
|
|
5409
5737
|
return true;
|
|
5410
5738
|
}
|
|
5411
5739
|
return false;
|
|
@@ -5595,6 +5923,15 @@ function bufferFromDataUrl(dataUrl) {
|
|
|
5595
5923
|
}
|
|
5596
5924
|
}
|
|
5597
5925
|
|
|
5926
|
+
/** A file of known size whose bytes are read a range at a time. */
|
|
5927
|
+
|
|
5928
|
+
function bufferReader(buffer) {
|
|
5929
|
+
return {
|
|
5930
|
+
size: buffer.length,
|
|
5931
|
+
read: async (start, end) => buffer.subarray(start, end + 1)
|
|
5932
|
+
};
|
|
5933
|
+
}
|
|
5934
|
+
|
|
5598
5935
|
/**
|
|
5599
5936
|
* The `*.val.json` files backing a `.jsonValues()` module's entries, as paths
|
|
5600
5937
|
* relative to the project root.
|
|
@@ -8029,6 +8366,26 @@ class ValOpsFS extends ValOps {
|
|
|
8029
8366
|
errors
|
|
8030
8367
|
};
|
|
8031
8368
|
}
|
|
8369
|
+
async openBinaryFile(filePath, fromPatch) {
|
|
8370
|
+
let absPath;
|
|
8371
|
+
if (fromPatch) {
|
|
8372
|
+
const patchesDir = this.getPatchesDir();
|
|
8373
|
+
absPath = this.wherePatchFileIs(patchBinaryFile(patchesDir, fromPatch.patchId, filePath), stagedPatchBinaryFile(patchesDir, fromPatch.patchId, filePath));
|
|
8374
|
+
} else {
|
|
8375
|
+
absPath = path__default.join(this.rootDir, ...filePath.split("/"));
|
|
8376
|
+
if (!this.host.fileExists(absPath)) {
|
|
8377
|
+
absPath = null;
|
|
8378
|
+
}
|
|
8379
|
+
}
|
|
8380
|
+
if (absPath === null) {
|
|
8381
|
+
return null;
|
|
8382
|
+
}
|
|
8383
|
+
const path = absPath;
|
|
8384
|
+
return {
|
|
8385
|
+
size: this.host.binaryFileSize(path),
|
|
8386
|
+
read: async (start, end) => this.host.readBinaryFileRange(path, start, end)
|
|
8387
|
+
};
|
|
8388
|
+
}
|
|
8032
8389
|
async getBinaryFile(filePath) {
|
|
8033
8390
|
const absPath = path__default.join(this.rootDir, ...filePath.split("/"));
|
|
8034
8391
|
if (!this.host.fileExists(absPath)) {
|
|
@@ -8151,6 +8508,30 @@ class FSOpsHost {
|
|
|
8151
8508
|
readBinaryFile(path) {
|
|
8152
8509
|
return fs.readFileSync(path);
|
|
8153
8510
|
}
|
|
8511
|
+
binaryFileSize(path) {
|
|
8512
|
+
return fs.statSync(path).size;
|
|
8513
|
+
}
|
|
8514
|
+
|
|
8515
|
+
/** Bytes `start` to `end` inclusive, and only those, read off disk. */
|
|
8516
|
+
async readBinaryFileRange(path, start, end) {
|
|
8517
|
+
const handle = await fs.promises.open(path, "r");
|
|
8518
|
+
try {
|
|
8519
|
+
const buffer = Buffer$1.alloc(Math.max(0, end - start + 1));
|
|
8520
|
+
let filled = 0;
|
|
8521
|
+
while (filled < buffer.length) {
|
|
8522
|
+
const {
|
|
8523
|
+
bytesRead
|
|
8524
|
+
} = await handle.read(buffer, filled, buffer.length - filled, start + filled);
|
|
8525
|
+
if (bytesRead === 0) {
|
|
8526
|
+
break;
|
|
8527
|
+
}
|
|
8528
|
+
filled += bytesRead;
|
|
8529
|
+
}
|
|
8530
|
+
return buffer.subarray(0, filled);
|
|
8531
|
+
} finally {
|
|
8532
|
+
await handle.close();
|
|
8533
|
+
}
|
|
8534
|
+
}
|
|
8154
8535
|
readUtf8File(path) {
|
|
8155
8536
|
return fs.readFileSync(path, "utf-8");
|
|
8156
8537
|
}
|
|
@@ -12292,6 +12673,15 @@ const DEFAULT_VAL_BUILD_URL = "https://admin.val.build";
|
|
|
12292
12673
|
*/
|
|
12293
12674
|
const VAL_APP_ENV = "app";
|
|
12294
12675
|
|
|
12676
|
+
/**
|
|
12677
|
+
* What the Val app says when a project does not store its media remotely.
|
|
12678
|
+
*
|
|
12679
|
+
* Exported so that `val publish` -- which publishes to the app, and so is
|
|
12680
|
+
* the last place to catch this before a build goes out -- refuses with the
|
|
12681
|
+
* same sentence the deployed server would.
|
|
12682
|
+
*/
|
|
12683
|
+
const APP_MODE_REQUIRES_REMOTE_FILES = "This project runs in the Val app, which stores every image, file and " + "video on Val's remote content host -- there is no repository for an " + "upload to go into. Add `files: { remote: true }` to `initVal` in " + "val.config.ts, so that `s.image()`, `s.file()`, `s.video()` and the " + "sets are remote everywhere this project is read. Images already in the " + "project then fail validation until they are uploaded: `npx val validate " + "--fix` does that.";
|
|
12684
|
+
|
|
12295
12685
|
/** Which mode the environment SAYS this is, and which variable said so. */
|
|
12296
12686
|
|
|
12297
12687
|
/**
|
|
@@ -12438,7 +12828,7 @@ async function initHandlerOptions(route, opts, config) {
|
|
|
12438
12828
|
valContentUrl
|
|
12439
12829
|
});
|
|
12440
12830
|
if (isProxyMode) {
|
|
12441
|
-
var _opts$versions, _opts$versions2;
|
|
12831
|
+
var _config$files, _opts$versions, _opts$versions2;
|
|
12442
12832
|
/*
|
|
12443
12833
|
* Why this app is in http mode, in the message that says what is missing.
|
|
12444
12834
|
*
|
|
@@ -12491,6 +12881,27 @@ async function initHandlerOptions(route, opts, config) {
|
|
|
12491
12881
|
if (!maybeValProject) {
|
|
12492
12882
|
throw new Error("Proxy mode does not work unless the 'project' option in val.config is defined or the VAL_PROJECT env var is set." + because);
|
|
12493
12883
|
}
|
|
12884
|
+
/*
|
|
12885
|
+
* The Val app has no repository for an uploaded file to go into, so every
|
|
12886
|
+
* image, file and video in it is remote -- and the project has to SAY so,
|
|
12887
|
+
* in `val.config.ts`, rather than this server deciding it.
|
|
12888
|
+
*
|
|
12889
|
+
* Deciding it here would be easy and wrong. The schemas are built when
|
|
12890
|
+
* `val.config.ts` and the modules are evaluated, before this function
|
|
12891
|
+
* sees anything, and the same file is evaluated by `val validate`, the VS
|
|
12892
|
+
* Code extension and `pnpm dev`. A switch only the app could see would
|
|
12893
|
+
* leave every one of those reading the project's media as local: every
|
|
12894
|
+
* remote image reported as a mistake, and `val validate --fix` downloading
|
|
12895
|
+
* it back into the repository.
|
|
12896
|
+
*
|
|
12897
|
+
* So this refuses rather than covers for it. Without the check, the
|
|
12898
|
+
* failure is quieter and later: a local upload that the app has nowhere
|
|
12899
|
+
* to put, found at a publish.
|
|
12900
|
+
*/
|
|
12901
|
+
// `config?.`: `initVal()` with no argument hands back no config at all.
|
|
12902
|
+
if (isAppEnv && (config === null || config === void 0 || (_config$files = config.files) === null || _config$files === void 0 ? void 0 : _config$files.remote) !== true) {
|
|
12903
|
+
throw new Error(APP_MODE_REQUIRES_REMOTE_FILES);
|
|
12904
|
+
}
|
|
12494
12905
|
const coreVersion = (_opts$versions = opts.versions) === null || _opts$versions === void 0 ? void 0 : _opts$versions.core;
|
|
12495
12906
|
if (!coreVersion) {
|
|
12496
12907
|
throw new Error("Could not determine version of @valbuild/core");
|
|
@@ -12823,6 +13234,12 @@ async function resolveRemoteFileAuth(options) {
|
|
|
12823
13234
|
|
|
12824
13235
|
/** What a publish reports back, in either shape. */
|
|
12825
13236
|
|
|
13237
|
+
/**
|
|
13238
|
+
* Where a local file a publish commits is written. Fixed: it was configurable
|
|
13239
|
+
* (`files.directory` in val.config.ts), nobody configured it, and the schemas
|
|
13240
|
+
* have their own `dir` for any collection that wants somewhere else.
|
|
13241
|
+
*/
|
|
13242
|
+
const FILES_DIRECTORY = "/public/val";
|
|
12826
13243
|
const ValServer = (valModules, options, callbacks) => {
|
|
12827
13244
|
var _options$git;
|
|
12828
13245
|
const AIContentBlock = z.union([z.object({
|
|
@@ -14839,7 +15256,6 @@ const ValServer = (valModules, options, callbacks) => {
|
|
|
14839
15256
|
},
|
|
14840
15257
|
"/publish-job-prepare": {
|
|
14841
15258
|
POST: async req => {
|
|
14842
|
-
var _options$config$files2;
|
|
14843
15259
|
const auth = getAuth(req.cookies);
|
|
14844
15260
|
if (auth.error) {
|
|
14845
15261
|
return {
|
|
@@ -14886,7 +15302,6 @@ const ValServer = (valModules, options, callbacks) => {
|
|
|
14886
15302
|
* push either way.
|
|
14887
15303
|
*/
|
|
14888
15304
|
if (serverOps.sourceMode() === "connected") {
|
|
14889
|
-
var _options$config$files;
|
|
14890
15305
|
const jobPatches = await serverOps.fetchPatches({
|
|
14891
15306
|
patchIds,
|
|
14892
15307
|
excludePatchOps: false
|
|
@@ -14939,7 +15354,7 @@ const ValServer = (valModules, options, callbacks) => {
|
|
|
14939
15354
|
const deploymentFiles = commit === undefined ? await serverOps.embeddedValSourceShas() : null;
|
|
14940
15355
|
const sent = await serverOps.prepareJob(jobId, {
|
|
14941
15356
|
tab,
|
|
14942
|
-
filesDirectory:
|
|
15357
|
+
filesDirectory: FILES_DIRECTORY,
|
|
14943
15358
|
...split.archive,
|
|
14944
15359
|
...(commit !== undefined ? {
|
|
14945
15360
|
gitCommit: commit
|
|
@@ -14968,7 +15383,7 @@ const ValServer = (valModules, options, callbacks) => {
|
|
|
14968
15383
|
} = built;
|
|
14969
15384
|
const sent = await serverOps.prepareJob(jobId, {
|
|
14970
15385
|
tab,
|
|
14971
|
-
filesDirectory:
|
|
15386
|
+
filesDirectory: FILES_DIRECTORY,
|
|
14972
15387
|
...split.archive
|
|
14973
15388
|
});
|
|
14974
15389
|
return jobPrepareAnswer(sent, {
|
|
@@ -15298,7 +15713,13 @@ const ValServer = (valModules, options, callbacks) => {
|
|
|
15298
15713
|
* tree; a store with no working tree does only the push.
|
|
15299
15714
|
*/
|
|
15300
15715
|
if (serverOps instanceof ValOpsMemory) {
|
|
15301
|
-
|
|
15716
|
+
/*
|
|
15717
|
+
* A commit with no files uploads nothing and needs no credentials;
|
|
15718
|
+
* see `commitCarriesRemoteFiles`. One with a LOCAL file still goes
|
|
15719
|
+
* through `uploadRemoteFiles` where the project is remote, because
|
|
15720
|
+
* that is where a local file is refused rather than dropped.
|
|
15721
|
+
*/
|
|
15722
|
+
const isRemoteRequired = commitCarriesRemoteFiles(preparedCommit) || commitCarriesFiles(preparedCommit) && getIsRemoteRequired(await serverOps.getSchemas());
|
|
15302
15723
|
if (isRemoteRequired) {
|
|
15303
15724
|
const authRes = await getRemoteFileAuth();
|
|
15304
15725
|
if (authRes.status !== 200) {
|
|
@@ -15321,7 +15742,9 @@ const ValServer = (valModules, options, callbacks) => {
|
|
|
15321
15742
|
}
|
|
15322
15743
|
if (serverOps instanceof ValOpsFS) {
|
|
15323
15744
|
var _remoteFileAuthRes;
|
|
15324
|
-
|
|
15745
|
+
// Credentials when THIS save uploads a remote file, not whenever
|
|
15746
|
+
// the project has a remote schema. See `commitCarriesRemoteFiles`.
|
|
15747
|
+
const isRemoteRequired = commitCarriesRemoteFiles(preparedCommit);
|
|
15325
15748
|
let mode;
|
|
15326
15749
|
let remoteFileAuthRes;
|
|
15327
15750
|
if (isRemoteRequired) {
|
|
@@ -15456,17 +15879,14 @@ const ValServer = (valModules, options, callbacks) => {
|
|
|
15456
15879
|
*/
|
|
15457
15880
|
const managedBranch = serverOps.sourceMode() === "managed" ? serverOps.projectBranch() : null;
|
|
15458
15881
|
const committedBinaries = serverOps.sourceMode() === "managed" ? await readCommittedBinaryFiles(serverOps, preparedCommit.patchedBinaryFilesDescriptors) : null;
|
|
15459
|
-
const commitToGit = () =>
|
|
15460
|
-
|
|
15461
|
-
|
|
15462
|
-
|
|
15463
|
-
|
|
15464
|
-
|
|
15465
|
-
|
|
15466
|
-
|
|
15467
|
-
*/
|
|
15468
|
-
body.patchGroupId);
|
|
15469
|
-
};
|
|
15882
|
+
const commitToGit = () => serverOps.commit(preparedCommit, message, auth.id, FILES_DIRECTORY, undefined,
|
|
15883
|
+
/*
|
|
15884
|
+
* Forwarded verbatim, and only the client can decide it: the
|
|
15885
|
+
* content API closes the group it is named without checking
|
|
15886
|
+
* that the commit shipped all of it, and whether it did needs
|
|
15887
|
+
* the patch sets, which live in the browser.
|
|
15888
|
+
*/
|
|
15889
|
+
body.patchGroupId);
|
|
15470
15890
|
/*
|
|
15471
15891
|
* A host may publish somewhere other than the repository.
|
|
15472
15892
|
*
|
|
@@ -16453,34 +16873,18 @@ const ValServer = (valModules, options, callbacks) => {
|
|
|
16453
16873
|
// nothing fetches it backend-to-backend. Neither half of the argument
|
|
16454
16874
|
// above transfers. See architecture/media.md, "Why /files has no auth,
|
|
16455
16875
|
// and /history/files does".
|
|
16456
|
-
let fileBuffer;
|
|
16457
|
-
let mimeType;
|
|
16458
16876
|
const remote = query.remote === "true";
|
|
16459
|
-
|
|
16460
|
-
|
|
16461
|
-
|
|
16462
|
-
} else {
|
|
16463
|
-
if (serverOps instanceof ValOpsHttp && remote) {
|
|
16464
|
-
console.error(`Remote file: ${filePath} requested without patch id. This is most likely a bug in Val.`);
|
|
16465
|
-
}
|
|
16466
|
-
fileBuffer = await serverOps.getBinaryFile(filePath);
|
|
16877
|
+
const patchId = query.patch_id;
|
|
16878
|
+
if (!patchId && serverOps instanceof ValOpsHttp && remote) {
|
|
16879
|
+
console.error(`Remote file: ${filePath} requested without patch id. This is most likely a bug in Val.`);
|
|
16467
16880
|
}
|
|
16468
|
-
|
|
16469
|
-
|
|
16470
|
-
|
|
16471
|
-
|
|
16472
|
-
|
|
16473
|
-
|
|
16474
|
-
|
|
16475
|
-
// patch, so it could be served "public, max-age=20000, immutable"
|
|
16476
|
-
// instead. There used to be a `cacheControl` variable here for
|
|
16477
|
-
// that, but its only assignment was commented out, so every
|
|
16478
|
-
// response has always taken the revalidate branch.
|
|
16479
|
-
"Cache-Control": "public, max-age=0, must-revalidate"
|
|
16480
|
-
},
|
|
16481
|
-
body: bufferToReadableStream(fileBuffer)
|
|
16482
|
-
};
|
|
16483
|
-
} else {
|
|
16881
|
+
// Opened, not read: a video is seeked in by many small ranges, and
|
|
16882
|
+
// only the asked-for bytes are read where the mode can (fs).
|
|
16883
|
+
const opened = await serverOps.openBinaryFile(filePath, patchId ? {
|
|
16884
|
+
patchId: patchId,
|
|
16885
|
+
remote
|
|
16886
|
+
} : null);
|
|
16887
|
+
if (!opened) {
|
|
16484
16888
|
return {
|
|
16485
16889
|
status: 404,
|
|
16486
16890
|
json: {
|
|
@@ -16488,6 +16892,62 @@ const ValServer = (valModules, options, callbacks) => {
|
|
|
16488
16892
|
}
|
|
16489
16893
|
};
|
|
16490
16894
|
}
|
|
16895
|
+
// A DRAFT playlist names files that are drafts too, at URLs that only
|
|
16896
|
+
// resolve once published. See `rewriteDraftPlaylist`. Playlists are
|
|
16897
|
+
// small, so one is read whole to be rewritten.
|
|
16898
|
+
const file = patchId && isHlsPlaylistPath(filePath) ? bufferReader(Buffer.from(rewriteDraftPlaylist((await opened.read(0, opened.size - 1)).toString("utf-8"), {
|
|
16899
|
+
playlistPath: filePath,
|
|
16900
|
+
patchId,
|
|
16901
|
+
remote
|
|
16902
|
+
}), "utf-8")) : opened;
|
|
16903
|
+
const headers = {
|
|
16904
|
+
// TODO: we could use ETag and return 304 instead
|
|
16905
|
+
// From the extension, published or draft: a `<video>` and an HLS
|
|
16906
|
+
// player both decide what they are holding by it, and it used to be
|
|
16907
|
+
// set for drafts only.
|
|
16908
|
+
"Content-Type": Internal.filenameToMimeType(filePath) || "application/octet-stream",
|
|
16909
|
+
// TODO: a file requested with a patch_id is immutable for that
|
|
16910
|
+
// patch, so it could be served "public, max-age=20000, immutable"
|
|
16911
|
+
// instead. There used to be a `cacheControl` variable here for
|
|
16912
|
+
// that, but its only assignment was commented out, so every
|
|
16913
|
+
// response has always taken the revalidate branch.
|
|
16914
|
+
"Cache-Control": "public, max-age=0, must-revalidate",
|
|
16915
|
+
"Accept-Ranges": "bytes"
|
|
16916
|
+
};
|
|
16917
|
+
const range = parseRangeHeader(req.headers.range, file.size);
|
|
16918
|
+
if (range.kind === "unsatisfiable") {
|
|
16919
|
+
return {
|
|
16920
|
+
status: 416,
|
|
16921
|
+
headers: {
|
|
16922
|
+
...headers,
|
|
16923
|
+
"Content-Range": `bytes */${file.size}`
|
|
16924
|
+
},
|
|
16925
|
+
json: {
|
|
16926
|
+
message: `Range not satisfiable: ${req.headers.range}`
|
|
16927
|
+
}
|
|
16928
|
+
};
|
|
16929
|
+
}
|
|
16930
|
+
if (range.kind === "range") {
|
|
16931
|
+
const slice = await file.read(range.start, range.end);
|
|
16932
|
+
return {
|
|
16933
|
+
status: 206,
|
|
16934
|
+
headers: {
|
|
16935
|
+
...headers,
|
|
16936
|
+
"Content-Range": `bytes ${range.start}-${range.end}/${file.size}`,
|
|
16937
|
+
"Content-Length": String(slice.length)
|
|
16938
|
+
},
|
|
16939
|
+
body: bufferToReadableStream(slice)
|
|
16940
|
+
};
|
|
16941
|
+
}
|
|
16942
|
+
const body = file.size === 0 ? Buffer.alloc(0) : await file.read(0, file.size - 1);
|
|
16943
|
+
return {
|
|
16944
|
+
status: 200,
|
|
16945
|
+
headers: {
|
|
16946
|
+
...headers,
|
|
16947
|
+
"Content-Length": String(body.length)
|
|
16948
|
+
},
|
|
16949
|
+
body: bufferToReadableStream(body)
|
|
16950
|
+
};
|
|
16491
16951
|
}
|
|
16492
16952
|
}
|
|
16493
16953
|
};
|
|
@@ -17835,301 +18295,2692 @@ class ValFSHost {
|
|
|
17835
18295
|
}
|
|
17836
18296
|
}
|
|
17837
18297
|
|
|
17838
|
-
|
|
17839
|
-
|
|
17840
|
-
|
|
17841
|
-
|
|
17842
|
-
|
|
17843
|
-
|
|
17844
|
-
|
|
17845
|
-
|
|
17846
|
-
|
|
17847
|
-
|
|
17848
|
-
|
|
17849
|
-
|
|
17850
|
-
|
|
17851
|
-
|
|
17852
|
-
|
|
17853
|
-
|
|
17854
|
-
|
|
17855
|
-
|
|
17856
|
-
|
|
17857
|
-
|
|
17858
|
-
|
|
17859
|
-
|
|
17860
|
-
height = 0;
|
|
17861
|
-
}
|
|
18298
|
+
/**
|
|
18299
|
+
* The size and length of an ISO base media file (`.mp4`, `.m4v`, `.mov`),
|
|
18300
|
+
* read from its headers. Nothing is decoded.
|
|
18301
|
+
*
|
|
18302
|
+
* Hand-written rather than a media library, deliberately: `@valbuild/server`
|
|
18303
|
+
* is loaded by every app that runs Val, and the four numbers wanted here are
|
|
18304
|
+
* three small boxes away from the start of the `moov` box:
|
|
18305
|
+
*
|
|
18306
|
+
* moov > mvhd timescale and duration (version 0 and 1)
|
|
18307
|
+
* moov > mvex > mehd duration of a FRAGMENTED file, whose mvhd says 0
|
|
18308
|
+
* moov > trak > tkhd width and height (16.16 fixed point) and the
|
|
18309
|
+
* display matrix, for a rotated phone video
|
|
18310
|
+
* moov > trak > mdia > hdlr `vide`: which trak is the picture
|
|
18311
|
+
*
|
|
18312
|
+
* Only the box headers are read on the way to `moov`, and then `moov` itself,
|
|
18313
|
+
* so a multi-GB file costs a few small reads: `mdat` is seeked past, never
|
|
18314
|
+
* loaded. That is what {@link ByteSource} is for.
|
|
18315
|
+
*/
|
|
18316
|
+
|
|
18317
|
+
/** Random access to a file's bytes, so a large `mdat` can be skipped. */
|
|
18318
|
+
|
|
18319
|
+
function bufferByteSource(buffer) {
|
|
17862
18320
|
return {
|
|
17863
|
-
|
|
17864
|
-
|
|
17865
|
-
mimeType
|
|
18321
|
+
size: buffer.length,
|
|
18322
|
+
read: (offset, length) => buffer.subarray(offset, offset + length)
|
|
17866
18323
|
};
|
|
17867
18324
|
}
|
|
17868
|
-
|
|
17869
|
-
|
|
17870
|
-
|
|
17871
|
-
|
|
17872
|
-
|
|
18325
|
+
|
|
18326
|
+
/**
|
|
18327
|
+
* Run `read` against a file on disk, reading only what it asks for. The file
|
|
18328
|
+
* is closed afterwards whatever happens.
|
|
18329
|
+
*/
|
|
18330
|
+
function withFileByteSource(absolutePath, read) {
|
|
18331
|
+
const fd = fs.openSync(absolutePath, "r");
|
|
18332
|
+
try {
|
|
18333
|
+
const size = fs.fstatSync(fd).size;
|
|
18334
|
+
return read({
|
|
18335
|
+
size,
|
|
18336
|
+
read: (offset, length) => {
|
|
18337
|
+
const wanted = Math.max(0, Math.min(length, size - offset));
|
|
18338
|
+
const buffer = Buffer.alloc(wanted);
|
|
18339
|
+
let filled = 0;
|
|
18340
|
+
while (filled < wanted) {
|
|
18341
|
+
const n = fs.readSync(fd, buffer, filled, wanted - filled, offset + filled);
|
|
18342
|
+
if (n === 0) {
|
|
18343
|
+
break;
|
|
18344
|
+
}
|
|
18345
|
+
filled += n;
|
|
18346
|
+
}
|
|
18347
|
+
return buffer.subarray(0, filled);
|
|
18348
|
+
}
|
|
18349
|
+
});
|
|
18350
|
+
} finally {
|
|
18351
|
+
fs.closeSync(fd);
|
|
17873
18352
|
}
|
|
17874
|
-
return {
|
|
17875
|
-
mimeType
|
|
17876
|
-
};
|
|
17877
18353
|
}
|
|
17878
|
-
|
|
17879
18354
|
/**
|
|
17880
|
-
* A
|
|
17881
|
-
*
|
|
17882
|
-
* A gallery is a record keyed by its entries' file paths. A REMOTE gallery keys
|
|
17883
|
-
* an uploaded entry by the remote URL instead — which encodes that same path —
|
|
17884
|
-
* so every entry has a local path, and the interesting question is whether the
|
|
17885
|
-
* bytes are expected to be at it.
|
|
17886
|
-
*
|
|
17887
|
-
* **They are not, for a remote entry added the ordinary way.** `saveOrUploadFiles`
|
|
17888
|
-
* uploads the remote descriptors to the content host and copies only the local
|
|
17889
|
-
* ones into the working tree, so an image added through the Studio (or over MCP)
|
|
17890
|
-
* and published has no file in the repo, by design — putting one there is what
|
|
17891
|
-
* remote storage exists to avoid. A remote entry CAN have one, because
|
|
17892
|
-
* `val validate --fix` promotes a local file to a remote ref and leaves the file
|
|
17893
|
-
* where it was; so "remote" means "do not require it", never "there is not one".
|
|
17894
|
-
*
|
|
17895
|
-
* One copy, in its own file, because this used to be two: a `remoteKeyToLocalPath`
|
|
17896
|
-
* in `fixHandlers` and a `galleryKeyToLocalPath` in `createFixPatch`, each
|
|
17897
|
-
* normalising the key correctly and each then treating the result as a file that
|
|
17898
|
-
* must exist. That agreement is what made a published remote image report as
|
|
17899
|
-
* missing from both sides at once.
|
|
18355
|
+
* A `moov` larger than this is not a header worth reading into memory. A
|
|
18356
|
+
* two-hour film's is a few MB; this is far past any real one.
|
|
17900
18357
|
*/
|
|
18358
|
+
const MAX_MOOV_SIZE = 64 * 1024 * 1024;
|
|
17901
18359
|
|
|
17902
18360
|
/**
|
|
17903
|
-
*
|
|
17904
|
-
*
|
|
17905
|
-
|
|
17906
|
-
|
|
17907
|
-
|
|
18361
|
+
* Whatever of {@link IsoBmffMetadata} the file declares. A field that cannot be
|
|
18362
|
+
* read is absent; nothing here throws on a file that is not ISO BMFF.
|
|
18363
|
+
*/
|
|
18364
|
+
function readIsoBmffMetadata(source) {
|
|
18365
|
+
const moov = findTopLevelBox(source, "moov");
|
|
18366
|
+
if (!moov) {
|
|
18367
|
+
return {};
|
|
18368
|
+
}
|
|
18369
|
+
const metadata = {};
|
|
18370
|
+
const mvhd = childBox(moov, "mvhd");
|
|
18371
|
+
const timescaleAndDuration = mvhd ? readMvhd(mvhd) : undefined;
|
|
18372
|
+
if (timescaleAndDuration) {
|
|
18373
|
+
const {
|
|
18374
|
+
timescale,
|
|
18375
|
+
duration
|
|
18376
|
+
} = timescaleAndDuration;
|
|
18377
|
+
if (duration > 0) {
|
|
18378
|
+
metadata.duration = duration / timescale;
|
|
18379
|
+
} else {
|
|
18380
|
+
// A fragmented file (CMAF) says 0 in mvhd and puts the length in mehd,
|
|
18381
|
+
// in the same timescale.
|
|
18382
|
+
const mehd = childBox(childBox(moov, "mvex"), "mehd");
|
|
18383
|
+
const fragmentDuration = mehd ? readMehd(mehd) : undefined;
|
|
18384
|
+
if (fragmentDuration !== undefined && fragmentDuration > 0) {
|
|
18385
|
+
metadata.duration = fragmentDuration / timescale;
|
|
18386
|
+
}
|
|
18387
|
+
}
|
|
18388
|
+
}
|
|
18389
|
+
for (const trak of childBoxes(moov, "trak")) {
|
|
18390
|
+
const hdlr = childBox(childBox(trak, "mdia"), "hdlr");
|
|
18391
|
+
if (!hdlr || hdlr.length < 12 || hdlr.toString("latin1", 8, 12) !== "vide") {
|
|
18392
|
+
continue;
|
|
18393
|
+
}
|
|
18394
|
+
const tkhd = childBox(trak, "tkhd");
|
|
18395
|
+
const size = tkhd ? readTkhd(tkhd) : undefined;
|
|
18396
|
+
if (size && size.width > 0 && size.height > 0) {
|
|
18397
|
+
metadata.width = size.width;
|
|
18398
|
+
metadata.height = size.height;
|
|
18399
|
+
break;
|
|
18400
|
+
}
|
|
18401
|
+
}
|
|
18402
|
+
return metadata;
|
|
18403
|
+
}
|
|
18404
|
+
|
|
18405
|
+
/** The payload (after the header) of the first top-level box of a type. */
|
|
18406
|
+
function findTopLevelBox(source, type) {
|
|
18407
|
+
let offset = 0;
|
|
18408
|
+
while (offset + 8 <= source.size) {
|
|
18409
|
+
const header = source.read(offset, 16);
|
|
18410
|
+
if (header.length < 8) {
|
|
18411
|
+
return undefined;
|
|
18412
|
+
}
|
|
18413
|
+
let size = header.readUInt32BE(0);
|
|
18414
|
+
const boxType = header.toString("latin1", 4, 8);
|
|
18415
|
+
let headerSize = 8;
|
|
18416
|
+
if (size === 1) {
|
|
18417
|
+
if (header.length < 16) {
|
|
18418
|
+
return undefined;
|
|
18419
|
+
}
|
|
18420
|
+
size = Number(header.readBigUInt64BE(8));
|
|
18421
|
+
headerSize = 16;
|
|
18422
|
+
} else if (size === 0) {
|
|
18423
|
+
size = source.size - offset;
|
|
18424
|
+
}
|
|
18425
|
+
if (size < headerSize) {
|
|
18426
|
+
return undefined;
|
|
18427
|
+
}
|
|
18428
|
+
if (boxType === type) {
|
|
18429
|
+
if (size - headerSize > MAX_MOOV_SIZE) {
|
|
18430
|
+
return undefined;
|
|
18431
|
+
}
|
|
18432
|
+
const payload = source.read(offset + headerSize, size - headerSize);
|
|
18433
|
+
return payload.length === size - headerSize ? payload : undefined;
|
|
18434
|
+
}
|
|
18435
|
+
offset += size;
|
|
18436
|
+
}
|
|
18437
|
+
return undefined;
|
|
18438
|
+
}
|
|
18439
|
+
|
|
18440
|
+
/** Each child box of a container payload, as `[type, payload]`. */
|
|
18441
|
+
function* boxesIn(payload) {
|
|
18442
|
+
let offset = 0;
|
|
18443
|
+
while (offset + 8 <= payload.length) {
|
|
18444
|
+
let size = payload.readUInt32BE(offset);
|
|
18445
|
+
const type = payload.toString("latin1", offset + 4, offset + 8);
|
|
18446
|
+
let headerSize = 8;
|
|
18447
|
+
if (size === 1) {
|
|
18448
|
+
if (offset + 16 > payload.length) {
|
|
18449
|
+
return;
|
|
18450
|
+
}
|
|
18451
|
+
size = Number(payload.readBigUInt64BE(offset + 8));
|
|
18452
|
+
headerSize = 16;
|
|
18453
|
+
} else if (size === 0) {
|
|
18454
|
+
size = payload.length - offset;
|
|
18455
|
+
}
|
|
18456
|
+
if (size < headerSize || offset + size > payload.length) {
|
|
18457
|
+
return;
|
|
18458
|
+
}
|
|
18459
|
+
yield [type, payload.subarray(offset + headerSize, offset + size)];
|
|
18460
|
+
offset += size;
|
|
18461
|
+
}
|
|
18462
|
+
}
|
|
18463
|
+
function childBoxes(payload, type) {
|
|
18464
|
+
if (!payload) {
|
|
18465
|
+
return [];
|
|
18466
|
+
}
|
|
18467
|
+
const found = [];
|
|
18468
|
+
for (const [boxType, child] of boxesIn(payload)) {
|
|
18469
|
+
if (boxType === type) {
|
|
18470
|
+
found.push(child);
|
|
18471
|
+
}
|
|
18472
|
+
}
|
|
18473
|
+
return found;
|
|
18474
|
+
}
|
|
18475
|
+
function childBox(payload, type) {
|
|
18476
|
+
return childBoxes(payload, type)[0];
|
|
18477
|
+
}
|
|
18478
|
+
|
|
18479
|
+
/** "Unknown" in a version 1 box. Not a literal: `0x…n` needs an ES2020 target. */
|
|
18480
|
+
const ALL_ONES_64 = BigInt("0xffffffffffffffff");
|
|
18481
|
+
function readMvhd(mvhd) {
|
|
18482
|
+
const version = mvhd[0];
|
|
18483
|
+
if (version === 1) {
|
|
18484
|
+
if (mvhd.length < 32) return undefined;
|
|
18485
|
+
const timescale = mvhd.readUInt32BE(20);
|
|
18486
|
+
const duration = mvhd.readBigUInt64BE(24);
|
|
18487
|
+
if (timescale === 0 || duration === ALL_ONES_64) {
|
|
18488
|
+
return undefined;
|
|
18489
|
+
}
|
|
18490
|
+
return {
|
|
18491
|
+
timescale,
|
|
18492
|
+
duration: Number(duration)
|
|
18493
|
+
};
|
|
18494
|
+
}
|
|
18495
|
+
if (mvhd.length < 20) return undefined;
|
|
18496
|
+
const timescale = mvhd.readUInt32BE(12);
|
|
18497
|
+
const duration = mvhd.readUInt32BE(16);
|
|
18498
|
+
// All ones means "unknown" in a version 0 box.
|
|
18499
|
+
if (timescale === 0 || duration === 0xffffffff) {
|
|
18500
|
+
return undefined;
|
|
18501
|
+
}
|
|
18502
|
+
return {
|
|
18503
|
+
timescale,
|
|
18504
|
+
duration
|
|
18505
|
+
};
|
|
18506
|
+
}
|
|
18507
|
+
function readMehd(mehd) {
|
|
18508
|
+
if (mehd[0] === 1) {
|
|
18509
|
+
return mehd.length >= 12 ? Number(mehd.readBigUInt64BE(4)) : undefined;
|
|
18510
|
+
}
|
|
18511
|
+
return mehd.length >= 8 ? mehd.readUInt32BE(4) : undefined;
|
|
18512
|
+
}
|
|
18513
|
+
|
|
18514
|
+
/**
|
|
18515
|
+
* The size a track is SHOWN at: tkhd's width and height, swapped when the
|
|
18516
|
+
* display matrix turns the picture a quarter turn — a phone records upright
|
|
18517
|
+
* video sideways and says so here, and a page lays it out as it is shown.
|
|
18518
|
+
*/
|
|
18519
|
+
function readTkhd(tkhd) {
|
|
18520
|
+
// Past the version-specific times and ids, then 16 bytes of reserved,
|
|
18521
|
+
// layer, alternate group, volume and reserved, then the 36-byte matrix.
|
|
18522
|
+
const matrixAt = tkhd[0] === 1 ? 4 + 32 + 16 : 4 + 20 + 16;
|
|
18523
|
+
const widthAt = matrixAt + 36;
|
|
18524
|
+
if (tkhd.length < widthAt + 8) {
|
|
18525
|
+
return undefined;
|
|
18526
|
+
}
|
|
18527
|
+
const a = tkhd.readInt32BE(matrixAt);
|
|
18528
|
+
const b = tkhd.readInt32BE(matrixAt + 4);
|
|
18529
|
+
const c = tkhd.readInt32BE(matrixAt + 12);
|
|
18530
|
+
const d = tkhd.readInt32BE(matrixAt + 16);
|
|
18531
|
+
const width = Math.round(tkhd.readUInt32BE(widthAt) / 0x10000);
|
|
18532
|
+
const height = Math.round(tkhd.readUInt32BE(widthAt + 4) / 0x10000);
|
|
18533
|
+
const quarterTurn = a === 0 && d === 0 && b !== 0 && c !== 0;
|
|
18534
|
+
return quarterTurn ? {
|
|
18535
|
+
width: height,
|
|
18536
|
+
height: width
|
|
18537
|
+
} : {
|
|
18538
|
+
width,
|
|
18539
|
+
height
|
|
18540
|
+
};
|
|
18541
|
+
}
|
|
18542
|
+
|
|
18543
|
+
/**
|
|
18544
|
+
* The size and length of a Matroska file (`.webm`, and `.mkv`, which is the
|
|
18545
|
+
* same container), read from its headers. Nothing is decoded.
|
|
18546
|
+
*
|
|
18547
|
+
* Hand-written for the same reason `isoBmff.ts` is: `@valbuild/server` is
|
|
18548
|
+
* loaded by every app that runs Val, and what is wanted here is a handful of
|
|
18549
|
+
* integers two levels into the file:
|
|
18550
|
+
*
|
|
18551
|
+
* EBML the header; a file without one is not Matroska
|
|
18552
|
+
* Segment > Info TimecodeScale and Duration
|
|
18553
|
+
* Segment > Tracks > TrackEntry TrackType 1 (video) >
|
|
18554
|
+
* Video PixelWidth / PixelHeight, and DisplayWidth /
|
|
18555
|
+
* DisplayHeight when they are in pixels
|
|
18556
|
+
*
|
|
18557
|
+
* Matroska is EBML: every element is a variable-length ID, a variable-length
|
|
18558
|
+
* size and a payload. Only element HEADERS are read on the way through the
|
|
18559
|
+
* Segment, and then the payloads of Info and Tracks, which are a few hundred
|
|
18560
|
+
* bytes. The Clusters — the frames, which are the file — are never read: the
|
|
18561
|
+
* walk stops at the first one.
|
|
18562
|
+
*
|
|
18563
|
+
* That is allowed because the spec (RFC 9559, section 6.3) requires the first
|
|
18564
|
+
* Info and Tracks to come before the first Cluster, or to be named by a
|
|
18565
|
+
* SeekHead that does. A file written by a browser's MediaRecorder, or by
|
|
18566
|
+
* anything else that streams, does not know its own length when it starts:
|
|
18567
|
+
* its Segment and Clusters have the "unknown" size, and its Info has no
|
|
18568
|
+
* Duration. The Segment is then read as running to the end of the file, and
|
|
18569
|
+
* the duration is left out rather than counted from the Clusters — a caller
|
|
18570
|
+
* reports it, as it does for a fragmented mp4.
|
|
18571
|
+
*/
|
|
18572
|
+
|
|
18573
|
+
const EBML_HEADER = 0x1a45dfa3;
|
|
18574
|
+
const SEGMENT = 0x18538067;
|
|
18575
|
+
const SEEK_HEAD = 0x114d9b74;
|
|
18576
|
+
const SEEK = 0x4dbb;
|
|
18577
|
+
const SEEK_ID = 0x53ab;
|
|
18578
|
+
const SEEK_POSITION = 0x53ac;
|
|
18579
|
+
const INFO = 0x1549a966;
|
|
18580
|
+
const TIMECODE_SCALE = 0x2ad7b1;
|
|
18581
|
+
const DURATION = 0x4489;
|
|
18582
|
+
const TRACKS = 0x1654ae6b;
|
|
18583
|
+
const TRACK_ENTRY = 0xae;
|
|
18584
|
+
const TRACK_TYPE = 0x83;
|
|
18585
|
+
const VIDEO = 0xe0;
|
|
18586
|
+
const PIXEL_WIDTH = 0xb0;
|
|
18587
|
+
const PIXEL_HEIGHT = 0xba;
|
|
18588
|
+
const DISPLAY_WIDTH = 0x54b0;
|
|
18589
|
+
const DISPLAY_HEIGHT = 0x54ba;
|
|
18590
|
+
const DISPLAY_UNIT = 0x54b2;
|
|
18591
|
+
const CLUSTER = 0x1f43b675;
|
|
18592
|
+
const TRACK_TYPE_VIDEO = 1;
|
|
18593
|
+
const DISPLAY_UNIT_PIXELS = 0;
|
|
18594
|
+
/** Nanoseconds per tick, when Info does not say. */
|
|
18595
|
+
const DEFAULT_TIMECODE_SCALE = 1_000_000;
|
|
18596
|
+
|
|
18597
|
+
/**
|
|
18598
|
+
* Info, Tracks or a SeekHead larger than this is not a header worth reading
|
|
18599
|
+
* into memory. Tracks is the big one, for its codec private data, and that is
|
|
18600
|
+
* kilobytes.
|
|
18601
|
+
*/
|
|
18602
|
+
const MAX_HEADER_ELEMENT_SIZE = 16 * 1024 * 1024;
|
|
18603
|
+
|
|
18604
|
+
/**
|
|
18605
|
+
* Whatever of {@link EbmlMetadata} the file declares. A field that cannot be
|
|
18606
|
+
* read is absent; nothing here throws on a file that is not Matroska.
|
|
18607
|
+
*/
|
|
18608
|
+
function readEbmlMetadata(source) {
|
|
18609
|
+
const header = readElementHeader(source, 0);
|
|
18610
|
+
if (!header || header.id !== EBML_HEADER || header.size === undefined) {
|
|
18611
|
+
return {};
|
|
18612
|
+
}
|
|
18613
|
+
const segment = findSegment(source, header.dataStart + header.size);
|
|
18614
|
+
if (!segment) {
|
|
18615
|
+
return {};
|
|
18616
|
+
}
|
|
18617
|
+
const segmentStart = segment.dataStart;
|
|
18618
|
+
const segmentEnd = segment.size === undefined ? source.size : Math.min(source.size, segmentStart + segment.size);
|
|
18619
|
+
let info;
|
|
18620
|
+
let tracks;
|
|
18621
|
+
const seekPositions = new Map();
|
|
18622
|
+
let offset = segmentStart;
|
|
18623
|
+
while (offset < segmentEnd && !(info && tracks)) {
|
|
18624
|
+
const element = readElementHeader(source, offset);
|
|
18625
|
+
if (!element || element.id === CLUSTER) {
|
|
18626
|
+
break;
|
|
18627
|
+
}
|
|
18628
|
+
if (element.id === INFO && !info) {
|
|
18629
|
+
info = readPayload(source, element);
|
|
18630
|
+
} else if (element.id === TRACKS && !tracks) {
|
|
18631
|
+
tracks = readPayload(source, element);
|
|
18632
|
+
} else if (element.id === SEEK_HEAD) {
|
|
18633
|
+
const seekHead = readPayload(source, element);
|
|
18634
|
+
if (seekHead) {
|
|
18635
|
+
readSeekHead(seekHead, seekPositions);
|
|
18636
|
+
}
|
|
18637
|
+
}
|
|
18638
|
+
if (element.size === undefined) {
|
|
18639
|
+
// Only a Segment or a Cluster may have an unknown size, and an element
|
|
18640
|
+
// whose end is unknown cannot be stepped over.
|
|
18641
|
+
break;
|
|
18642
|
+
}
|
|
18643
|
+
offset = element.dataStart + element.size;
|
|
18644
|
+
}
|
|
18645
|
+
// Not before the first Cluster: the SeekHead says where.
|
|
18646
|
+
if (!info) {
|
|
18647
|
+
info = readSought(source, segmentStart, seekPositions, INFO);
|
|
18648
|
+
}
|
|
18649
|
+
if (!tracks) {
|
|
18650
|
+
tracks = readSought(source, segmentStart, seekPositions, TRACKS);
|
|
18651
|
+
}
|
|
18652
|
+
const metadata = {};
|
|
18653
|
+
const duration = info ? readDuration(info) : undefined;
|
|
18654
|
+
if (duration !== undefined) {
|
|
18655
|
+
metadata.duration = duration;
|
|
18656
|
+
}
|
|
18657
|
+
const size = tracks ? readVideoSize(tracks) : undefined;
|
|
18658
|
+
if (size) {
|
|
18659
|
+
metadata.width = size.width;
|
|
18660
|
+
metadata.height = size.height;
|
|
18661
|
+
}
|
|
18662
|
+
return metadata;
|
|
18663
|
+
}
|
|
18664
|
+
/** The first Segment among the top-level elements after the EBML header. */
|
|
18665
|
+
function findSegment(source, from) {
|
|
18666
|
+
let offset = from;
|
|
18667
|
+
while (offset < source.size) {
|
|
18668
|
+
const element = readElementHeader(source, offset);
|
|
18669
|
+
if (!element) {
|
|
18670
|
+
return undefined;
|
|
18671
|
+
}
|
|
18672
|
+
if (element.id === SEGMENT) {
|
|
18673
|
+
return element;
|
|
18674
|
+
}
|
|
18675
|
+
if (element.size === undefined) {
|
|
18676
|
+
return undefined;
|
|
18677
|
+
}
|
|
18678
|
+
offset = element.dataStart + element.size;
|
|
18679
|
+
}
|
|
18680
|
+
return undefined;
|
|
18681
|
+
}
|
|
18682
|
+
function readElementHeader(source, offset) {
|
|
18683
|
+
// At most 4 bytes of ID and 8 of size.
|
|
18684
|
+
const bytes = source.read(offset, 12);
|
|
18685
|
+
const id = readVint(bytes, 0, 4);
|
|
18686
|
+
if (!id) {
|
|
18687
|
+
return undefined;
|
|
18688
|
+
}
|
|
18689
|
+
const size = readVint(bytes, id.length, 8);
|
|
18690
|
+
if (!size) {
|
|
18691
|
+
return undefined;
|
|
18692
|
+
}
|
|
18693
|
+
return {
|
|
18694
|
+
id: id.raw,
|
|
18695
|
+
size: size.unknown ? undefined : size.value,
|
|
18696
|
+
dataStart: offset + id.length + size.length
|
|
18697
|
+
};
|
|
18698
|
+
}
|
|
18699
|
+
function readPayload(source, element) {
|
|
18700
|
+
const {
|
|
18701
|
+
size
|
|
18702
|
+
} = element;
|
|
18703
|
+
if (size === undefined || size > MAX_HEADER_ELEMENT_SIZE) {
|
|
18704
|
+
return undefined;
|
|
18705
|
+
}
|
|
18706
|
+
const payload = source.read(element.dataStart, size);
|
|
18707
|
+
return payload.length === size ? payload : undefined;
|
|
18708
|
+
}
|
|
18709
|
+
/**
|
|
18710
|
+
* An EBML variable-length integer: the number of leading zero bits of the
|
|
18711
|
+
* first byte, plus one, is its length in bytes. Built with arithmetic rather
|
|
18712
|
+
* than shifts, which are 32-bit and signed in JavaScript.
|
|
18713
|
+
*/
|
|
18714
|
+
function readVint(bytes, offset, maxLength) {
|
|
18715
|
+
if (offset >= bytes.length) {
|
|
18716
|
+
return undefined;
|
|
18717
|
+
}
|
|
18718
|
+
const first = bytes[offset];
|
|
18719
|
+
if (first === 0) {
|
|
18720
|
+
return undefined;
|
|
18721
|
+
}
|
|
18722
|
+
const length = Math.clz32(first) - 23;
|
|
18723
|
+
if (length > maxLength || offset + length > bytes.length) {
|
|
18724
|
+
return undefined;
|
|
18725
|
+
}
|
|
18726
|
+
const valueMask = 0xff >> length;
|
|
18727
|
+
let raw = first;
|
|
18728
|
+
let value = first & valueMask;
|
|
18729
|
+
let unknown = value === valueMask;
|
|
18730
|
+
for (let i = 1; i < length; i++) {
|
|
18731
|
+
const byte = bytes[offset + i];
|
|
18732
|
+
raw = raw * 256 + byte;
|
|
18733
|
+
value = value * 256 + byte;
|
|
18734
|
+
unknown = unknown && byte === 0xff;
|
|
18735
|
+
}
|
|
18736
|
+
return {
|
|
18737
|
+
length,
|
|
18738
|
+
raw,
|
|
18739
|
+
value,
|
|
18740
|
+
unknown
|
|
18741
|
+
};
|
|
18742
|
+
}
|
|
18743
|
+
|
|
18744
|
+
/** Each child element of a master element's payload, as `[id, payload]`. */
|
|
18745
|
+
function* elementsIn(payload) {
|
|
18746
|
+
let offset = 0;
|
|
18747
|
+
while (offset < payload.length) {
|
|
18748
|
+
const id = readVint(payload, offset, 4);
|
|
18749
|
+
if (!id) {
|
|
18750
|
+
return;
|
|
18751
|
+
}
|
|
18752
|
+
const size = readVint(payload, offset + id.length, 8);
|
|
18753
|
+
if (!size || size.unknown) {
|
|
18754
|
+
return;
|
|
18755
|
+
}
|
|
18756
|
+
const dataStart = offset + id.length + size.length;
|
|
18757
|
+
const end = dataStart + size.value;
|
|
18758
|
+
if (end > payload.length) {
|
|
18759
|
+
return;
|
|
18760
|
+
}
|
|
18761
|
+
yield [id.raw, payload.subarray(dataStart, end)];
|
|
18762
|
+
offset = end;
|
|
18763
|
+
}
|
|
18764
|
+
}
|
|
18765
|
+
function child(payload, id) {
|
|
18766
|
+
for (const [childId, data] of elementsIn(payload)) {
|
|
18767
|
+
if (childId === id) {
|
|
18768
|
+
return data;
|
|
18769
|
+
}
|
|
18770
|
+
}
|
|
18771
|
+
return undefined;
|
|
18772
|
+
}
|
|
18773
|
+
function readUint(data) {
|
|
18774
|
+
if (!data || data.length > 8) {
|
|
18775
|
+
return undefined;
|
|
18776
|
+
}
|
|
18777
|
+
let value = 0;
|
|
18778
|
+
for (const byte of data) {
|
|
18779
|
+
value = value * 256 + byte;
|
|
18780
|
+
}
|
|
18781
|
+
return value;
|
|
18782
|
+
}
|
|
18783
|
+
function readFloat(data) {
|
|
18784
|
+
if ((data === null || data === void 0 ? void 0 : data.length) === 4) {
|
|
18785
|
+
return data.readFloatBE(0);
|
|
18786
|
+
}
|
|
18787
|
+
if ((data === null || data === void 0 ? void 0 : data.length) === 8) {
|
|
18788
|
+
return data.readDoubleBE(0);
|
|
18789
|
+
}
|
|
18790
|
+
return undefined;
|
|
18791
|
+
}
|
|
18792
|
+
|
|
18793
|
+
/** Where each top-level element a SeekHead names is, from the Segment's data. */
|
|
18794
|
+
function readSeekHead(seekHead, into) {
|
|
18795
|
+
for (const [id, seek] of elementsIn(seekHead)) {
|
|
18796
|
+
if (id !== SEEK) {
|
|
18797
|
+
continue;
|
|
18798
|
+
}
|
|
18799
|
+
const seekId = child(seek, SEEK_ID);
|
|
18800
|
+
const position = readUint(child(seek, SEEK_POSITION));
|
|
18801
|
+
if (seekId && seekId.length <= 4 && position !== undefined) {
|
|
18802
|
+
const target = readUint(seekId);
|
|
18803
|
+
if (target !== undefined && !into.has(target)) {
|
|
18804
|
+
into.set(target, position);
|
|
18805
|
+
}
|
|
18806
|
+
}
|
|
18807
|
+
}
|
|
18808
|
+
}
|
|
18809
|
+
function readSought(source, segmentStart, seekPositions, id) {
|
|
18810
|
+
const position = seekPositions.get(id);
|
|
18811
|
+
if (position === undefined) {
|
|
18812
|
+
return undefined;
|
|
18813
|
+
}
|
|
18814
|
+
const element = readElementHeader(source, segmentStart + position);
|
|
18815
|
+
return (element === null || element === void 0 ? void 0 : element.id) === id ? readPayload(source, element) : undefined;
|
|
18816
|
+
}
|
|
18817
|
+
|
|
18818
|
+
/**
|
|
18819
|
+
* Info's Duration is a float in ticks of TimecodeScale nanoseconds. Absent in
|
|
18820
|
+
* a file written as it was recorded, and then absent here too.
|
|
18821
|
+
*/
|
|
18822
|
+
function readDuration(info) {
|
|
18823
|
+
const duration = readFloat(child(info, DURATION));
|
|
18824
|
+
if (duration === undefined || !Number.isFinite(duration) || duration <= 0) {
|
|
18825
|
+
return undefined;
|
|
18826
|
+
}
|
|
18827
|
+
const scale = readUint(child(info, TIMECODE_SCALE)) ?? DEFAULT_TIMECODE_SCALE;
|
|
18828
|
+
if (scale <= 0) {
|
|
18829
|
+
return undefined;
|
|
18830
|
+
}
|
|
18831
|
+
return duration * scale / 1e9;
|
|
18832
|
+
}
|
|
18833
|
+
|
|
18834
|
+
/**
|
|
18835
|
+
* The size the first video track is SHOWN at. DisplayWidth and DisplayHeight
|
|
18836
|
+
* are what a non-square-pixel (anamorphic) video is laid out at, so they win
|
|
18837
|
+
* over the stored PixelWidth and PixelHeight — but only when DisplayUnit says
|
|
18838
|
+
* they are pixels. The other units (centimetres, inches, a bare aspect ratio)
|
|
18839
|
+
* are not a size a page can use, so the pixel size is reported instead.
|
|
18840
|
+
*/
|
|
18841
|
+
function readVideoSize(tracks) {
|
|
18842
|
+
for (const [id, entry] of elementsIn(tracks)) {
|
|
18843
|
+
if (id !== TRACK_ENTRY || readUint(child(entry, TRACK_TYPE)) !== TRACK_TYPE_VIDEO) {
|
|
18844
|
+
continue;
|
|
18845
|
+
}
|
|
18846
|
+
const video = child(entry, VIDEO);
|
|
18847
|
+
if (!video) {
|
|
18848
|
+
continue;
|
|
18849
|
+
}
|
|
18850
|
+
const pixelWidth = readUint(child(video, PIXEL_WIDTH));
|
|
18851
|
+
const pixelHeight = readUint(child(video, PIXEL_HEIGHT));
|
|
18852
|
+
const displayUnit = readUint(child(video, DISPLAY_UNIT)) ?? DISPLAY_UNIT_PIXELS;
|
|
18853
|
+
if (displayUnit === DISPLAY_UNIT_PIXELS) {
|
|
18854
|
+
const displayWidth = readUint(child(video, DISPLAY_WIDTH));
|
|
18855
|
+
const displayHeight = readUint(child(video, DISPLAY_HEIGHT));
|
|
18856
|
+
if (displayWidth && displayHeight) {
|
|
18857
|
+
return {
|
|
18858
|
+
width: displayWidth,
|
|
18859
|
+
height: displayHeight
|
|
18860
|
+
};
|
|
18861
|
+
}
|
|
18862
|
+
}
|
|
18863
|
+
if (pixelWidth && pixelHeight) {
|
|
18864
|
+
return {
|
|
18865
|
+
width: pixelWidth,
|
|
18866
|
+
height: pixelHeight
|
|
18867
|
+
};
|
|
18868
|
+
}
|
|
18869
|
+
}
|
|
18870
|
+
return undefined;
|
|
18871
|
+
}
|
|
18872
|
+
|
|
18873
|
+
async function extractImageMetadata(filename, input) {
|
|
18874
|
+
const imageSize = sizeOf(new Uint8Array(input));
|
|
18875
|
+
let mimeType = null;
|
|
18876
|
+
if (imageSize.type) {
|
|
18877
|
+
const possibleMimeType = `image/${imageSize.type}`;
|
|
18878
|
+
if (Internal.MIME_TYPES_TO_EXT[possibleMimeType]) {
|
|
18879
|
+
mimeType = possibleMimeType;
|
|
18880
|
+
}
|
|
18881
|
+
const filenameBasedLookup = Internal.filenameToMimeType(filename);
|
|
18882
|
+
if (filenameBasedLookup) {
|
|
18883
|
+
mimeType = filenameBasedLookup;
|
|
18884
|
+
}
|
|
18885
|
+
}
|
|
18886
|
+
if (!mimeType) {
|
|
18887
|
+
mimeType = "application/octet-stream";
|
|
18888
|
+
}
|
|
18889
|
+
let {
|
|
18890
|
+
width,
|
|
18891
|
+
height
|
|
18892
|
+
} = imageSize;
|
|
18893
|
+
if (!width || !height) {
|
|
18894
|
+
width = 0;
|
|
18895
|
+
height = 0;
|
|
18896
|
+
}
|
|
18897
|
+
return {
|
|
18898
|
+
width,
|
|
18899
|
+
height,
|
|
18900
|
+
mimeType
|
|
18901
|
+
};
|
|
18902
|
+
}
|
|
18903
|
+
async function extractFileMetadata(filename, _input // TODO: use buffer to determine mimetype
|
|
18904
|
+
) {
|
|
18905
|
+
let mimeType = Internal.filenameToMimeType(filename);
|
|
18906
|
+
if (!mimeType) {
|
|
18907
|
+
mimeType = "application/octet-stream";
|
|
18908
|
+
}
|
|
18909
|
+
return {
|
|
18910
|
+
mimeType
|
|
18911
|
+
};
|
|
18912
|
+
}
|
|
18913
|
+
|
|
18914
|
+
/**
|
|
18915
|
+
* What `val validate --fix` writes into a video: `mimeType`, `width`, `height`
|
|
18916
|
+
* and `duration`, read from the file. A field that cannot be read is left out
|
|
18917
|
+
* rather than guessed, and the caller reports it — see
|
|
18918
|
+
* {@link unreadableVideoMetadataMessage}.
|
|
18919
|
+
*
|
|
18920
|
+
* `mimeType` always comes from the file EXTENSION: validation compares the two,
|
|
18921
|
+
* so any other answer is one validation rejects. The rest depends on the kind
|
|
18922
|
+
* of file:
|
|
18923
|
+
*
|
|
18924
|
+
* - **ISO BMFF** (`.mp4`, `.m4v`, `.mov`): read from the `moov` box by a small
|
|
18925
|
+
* parser of our own (`isoBmff.ts`). No media library: this package is loaded
|
|
18926
|
+
* by every app that runs Val.
|
|
18927
|
+
* - **WebM / Matroska** (`.webm`, `.mkv`): read from the Segment's Info and Tracks by another
|
|
18928
|
+
* (`ebml.ts`). A WebM recorded in a browser does not declare its length, and
|
|
18929
|
+
* then `duration` is left out — the frames are not counted to find it.
|
|
18930
|
+
* - **An HLS master playlist** (`.m3u8`): the dimensions are the largest
|
|
18931
|
+
* rendition's `RESOLUTION`, and the duration is the sum of the `#EXTINF`s of
|
|
18932
|
+
* the first media playlist it names, read from disk beside it — which is why
|
|
18933
|
+
* `filename` must be the playlist's path on disk.
|
|
18934
|
+
* - **Anything else**: the mime type and nothing more. The Studio reads those
|
|
18935
|
+
* in the browser when they are uploaded.
|
|
18936
|
+
*/
|
|
18937
|
+
async function extractVideoMetadata(filename, input, readFile = readFileOrUndefined$2) {
|
|
18938
|
+
if (isHlsPath(filename)) {
|
|
18939
|
+
return extractHlsMetadata(filename, input.toString("utf-8"), readFile);
|
|
18940
|
+
}
|
|
18941
|
+
return extractFromByteSource(filename, bufferByteSource(input));
|
|
18942
|
+
}
|
|
18943
|
+
|
|
18944
|
+
/**
|
|
18945
|
+
* {@link extractVideoMetadata} for a file on disk, reading only the headers
|
|
18946
|
+
* it needs: a video's frames (an mp4's `mdat`, a WebM's Clusters) are most of
|
|
18947
|
+
* the file, and are skipped, not loaded.
|
|
18948
|
+
*/
|
|
18949
|
+
async function extractVideoMetadataFromFile(absolutePath) {
|
|
18950
|
+
if (isHlsPath(absolutePath)) {
|
|
18951
|
+
return extractVideoMetadata(absolutePath, fs.readFileSync(absolutePath));
|
|
18952
|
+
}
|
|
18953
|
+
return withFileByteSource(absolutePath, source => extractFromByteSource(absolutePath, source));
|
|
18954
|
+
}
|
|
18955
|
+
const ISO_BMFF_EXTENSIONS = [".mp4", ".m4v", ".mov"];
|
|
18956
|
+
const EBML_EXTENSIONS = [".webm", ".mkv"];
|
|
18957
|
+
|
|
18958
|
+
/** The extensions whose size and length Val reads itself. */
|
|
18959
|
+
const READABLE_VIDEO_EXTENSIONS = [...ISO_BMFF_EXTENSIONS, ...EBML_EXTENSIONS, ".m3u8"];
|
|
18960
|
+
|
|
18961
|
+
/**
|
|
18962
|
+
* What to tell someone whose video's `fields` could not be read. Names the
|
|
18963
|
+
* way out, because "could not read" alone leaves them nowhere to go.
|
|
18964
|
+
*/
|
|
18965
|
+
function unreadableVideoMetadataMessage(fileRef, fields) {
|
|
18966
|
+
const extension = extensionOf(fileRef);
|
|
18967
|
+
if (!canReadVideoMetadata(fileRef)) {
|
|
18968
|
+
return `Val cannot read the size and length of a ${extension || "file without an extension"} file on the command line. Upload it again in the Val Studio, or add ${listOf(fields)} by hand.`;
|
|
18969
|
+
}
|
|
18970
|
+
return `Val could not read the ${listOf(fields)} of ${fileRef}. Upload it again in the Val Studio, or add ${listOf(fields)} by hand.`;
|
|
18971
|
+
}
|
|
18972
|
+
|
|
18973
|
+
/** Whether Val reads this video's size and length itself, by its extension. */
|
|
18974
|
+
function canReadVideoMetadata(fileRef) {
|
|
18975
|
+
return READABLE_VIDEO_EXTENSIONS.includes(extensionOf(fileRef));
|
|
18976
|
+
}
|
|
18977
|
+
function extensionOf(fileRef) {
|
|
18978
|
+
return path__default.extname(fileRef.split("?")[0]).toLowerCase();
|
|
18979
|
+
}
|
|
18980
|
+
function listOf(fields) {
|
|
18981
|
+
return fields.length <= 1 ? fields.join("") : `${fields.slice(0, -1).join(", ")} and ${fields[fields.length - 1]}`;
|
|
18982
|
+
}
|
|
18983
|
+
function isHlsPath(filename) {
|
|
18984
|
+
return path__default.extname(filename).toLowerCase() === ".m3u8";
|
|
18985
|
+
}
|
|
18986
|
+
function extractFromByteSource(filename, source) {
|
|
18987
|
+
const metadata = {};
|
|
18988
|
+
const mimeType = Internal.filenameToMimeType(filename);
|
|
18989
|
+
if (mimeType) {
|
|
18990
|
+
metadata.mimeType = mimeType;
|
|
18991
|
+
}
|
|
18992
|
+
const extension = extensionOf(filename);
|
|
18993
|
+
const read = ISO_BMFF_EXTENSIONS.includes(extension) ? readIsoBmffMetadata(source) : EBML_EXTENSIONS.includes(extension) ? readEbmlMetadata(source) : undefined;
|
|
18994
|
+
if (!read) {
|
|
18995
|
+
return metadata;
|
|
18996
|
+
}
|
|
18997
|
+
if (read.width !== undefined && read.height !== undefined) {
|
|
18998
|
+
metadata.width = read.width;
|
|
18999
|
+
metadata.height = read.height;
|
|
19000
|
+
}
|
|
19001
|
+
const duration = roundSeconds(read.duration);
|
|
19002
|
+
if (duration !== undefined) {
|
|
19003
|
+
metadata.duration = duration;
|
|
19004
|
+
}
|
|
19005
|
+
return metadata;
|
|
19006
|
+
}
|
|
19007
|
+
|
|
19008
|
+
/**
|
|
19009
|
+
* An HLS stream's metadata from its master playlist's text, and — for the
|
|
19010
|
+
* duration — the first media playlist it names, which `readMediaPlaylist`
|
|
19011
|
+
* fetches by the URI the master gives (relative to it, or a remote ref). The
|
|
19012
|
+
* one implementation for a stream on disk and one on Val Remote.
|
|
19013
|
+
*/
|
|
19014
|
+
async function readHlsMetadata(text, readMediaPlaylist) {
|
|
19015
|
+
const metadata = {
|
|
19016
|
+
mimeType: Internal.media.HLS_MIME_TYPE
|
|
19017
|
+
};
|
|
19018
|
+
if (!isHlsMasterPlaylist(text)) {
|
|
19019
|
+
// A media playlist on its own: it knows how long it is, not how big.
|
|
19020
|
+
const duration = roundSeconds(sumHlsSegmentDurations(text));
|
|
19021
|
+
if (duration !== undefined) {
|
|
19022
|
+
metadata.duration = duration;
|
|
19023
|
+
}
|
|
19024
|
+
return metadata;
|
|
19025
|
+
}
|
|
19026
|
+
const variants = parseHlsMasterVariants(text);
|
|
19027
|
+
const largest = largestHlsVariant(variants);
|
|
19028
|
+
if (largest) {
|
|
19029
|
+
metadata.width = largest.width;
|
|
19030
|
+
metadata.height = largest.height;
|
|
19031
|
+
}
|
|
19032
|
+
// Every rendition is the same length, so the first one listed is as good as
|
|
19033
|
+
// any.
|
|
19034
|
+
const first = variants[0];
|
|
19035
|
+
if (first) {
|
|
19036
|
+
const mediaPlaylist = await readMediaPlaylist(first.uri);
|
|
19037
|
+
if (mediaPlaylist !== undefined) {
|
|
19038
|
+
const duration = roundSeconds(sumHlsSegmentDurations(mediaPlaylist));
|
|
19039
|
+
if (duration !== undefined) {
|
|
19040
|
+
metadata.duration = duration;
|
|
19041
|
+
}
|
|
19042
|
+
}
|
|
19043
|
+
}
|
|
19044
|
+
return metadata;
|
|
19045
|
+
}
|
|
19046
|
+
function extractHlsMetadata(filename, text, readFile) {
|
|
19047
|
+
// Only a relative URI can be read from disk: the Studio writes relative
|
|
19048
|
+
// URIs into a local stream.
|
|
19049
|
+
return readHlsMetadata(text, async uri => {
|
|
19050
|
+
var _readFile;
|
|
19051
|
+
return isRelativeUri(uri) ? (_readFile = readFile(path__default.resolve(path__default.dirname(filename), uri.split(/[?#]/)[0]))) === null || _readFile === void 0 ? void 0 : _readFile.toString("utf-8") : undefined;
|
|
19052
|
+
});
|
|
19053
|
+
}
|
|
19054
|
+
function isRelativeUri(uri) {
|
|
19055
|
+
return !/^[a-z][a-z0-9+.-]*:/i.test(uri) && !uri.startsWith("/");
|
|
19056
|
+
}
|
|
19057
|
+
|
|
19058
|
+
/**
|
|
19059
|
+
* Seconds to the millisecond. A duration summed from `#EXTINF`s or computed
|
|
19060
|
+
* from timestamps carries float noise (`5.999999999`), and this value is
|
|
19061
|
+
* written into a `.val.ts` someone reads.
|
|
19062
|
+
*/
|
|
19063
|
+
function roundSeconds(seconds) {
|
|
19064
|
+
if (seconds === undefined || !Number.isFinite(seconds) || seconds <= 0) {
|
|
19065
|
+
return undefined;
|
|
19066
|
+
}
|
|
19067
|
+
return Math.round(seconds * 1000) / 1000;
|
|
19068
|
+
}
|
|
19069
|
+
function readFileOrUndefined$2(absolutePath) {
|
|
19070
|
+
try {
|
|
19071
|
+
return fs.readFileSync(absolutePath);
|
|
19072
|
+
} catch {
|
|
19073
|
+
return undefined;
|
|
19074
|
+
}
|
|
19075
|
+
}
|
|
19076
|
+
|
|
19077
|
+
/**
|
|
19078
|
+
* A remote video's size and length, read over HTTP without downloading it.
|
|
19079
|
+
*
|
|
19080
|
+
* The parsers (`isoBmff.ts`, `ebml.ts`) are synchronous and ask a
|
|
19081
|
+
* {@link ByteSource} for the few header bytes they need. Here that source is
|
|
19082
|
+
* a cache of byte ranges fetched with `Range` requests: when the parser asks
|
|
19083
|
+
* for bytes the cache does not hold, the read throws {@link NeedBytes}, the
|
|
19084
|
+
* missing range is fetched, and the parser runs again from the start. Parsing
|
|
19085
|
+
* headers is microseconds and fetching is milliseconds, so re-running is the
|
|
19086
|
+
* cheap side of that trade — and it keeps ONE parser for files on disk and
|
|
19087
|
+
* files on Val Remote, rather than an async copy of each.
|
|
19088
|
+
*
|
|
19089
|
+
* An mp4 with `moov` first costs one request; one with `moov` after `mdat`
|
|
19090
|
+
* costs about three (the start, the box header past `mdat`, and `moov`
|
|
19091
|
+
* itself), however large the file. An HLS stream is its master playlist and
|
|
19092
|
+
* one media playlist.
|
|
19093
|
+
*/
|
|
19094
|
+
|
|
19095
|
+
/** How much to fetch past what was asked for: box headers come in runs. */
|
|
19096
|
+
const READ_AHEAD = 64 * 1024;
|
|
19097
|
+
/** A parser that needs more fetches than this is not reading headers. */
|
|
19098
|
+
const MAX_FETCHES = 16;
|
|
19099
|
+
/** Thrown by a cached read that needs bytes not fetched yet. */
|
|
19100
|
+
class NeedBytes {
|
|
19101
|
+
constructor(offset, length) {
|
|
19102
|
+
this.offset = offset;
|
|
19103
|
+
this.length = length;
|
|
19104
|
+
}
|
|
19105
|
+
}
|
|
19106
|
+
|
|
19107
|
+
/**
|
|
19108
|
+
* Read `url`'s metadata. `nameForType` is the file name its type is read off —
|
|
19109
|
+
* for a remote ref, the `/public/...` path inside it, since the ref's own URL
|
|
19110
|
+
* ends in the same name but is not where a reader would look for it.
|
|
19111
|
+
*/
|
|
19112
|
+
async function extractVideoMetadataFromUrl(url, nameForType, fetchImpl = fetch) {
|
|
19113
|
+
if (nameForType.split("?")[0].toLowerCase().endsWith(".m3u8")) {
|
|
19114
|
+
const master = await fetchText(url, fetchImpl);
|
|
19115
|
+
return readHlsMetadata(master, uri => fetchText(new URL(uri, url).toString(), fetchImpl).catch(() => undefined));
|
|
19116
|
+
}
|
|
19117
|
+
const cache = new RangeCache(url, fetchImpl);
|
|
19118
|
+
for (let fetches = 0;; fetches++) {
|
|
19119
|
+
const source = await cache.source();
|
|
19120
|
+
try {
|
|
19121
|
+
return extractFromByteSource(nameForType, source);
|
|
19122
|
+
} catch (err) {
|
|
19123
|
+
if (!(err instanceof NeedBytes)) {
|
|
19124
|
+
throw err;
|
|
19125
|
+
}
|
|
19126
|
+
if (fetches >= MAX_FETCHES) {
|
|
19127
|
+
throw new Error(`Reading the headers of ${url} took more than ${MAX_FETCHES} requests`, {
|
|
19128
|
+
cause: err
|
|
19129
|
+
});
|
|
19130
|
+
}
|
|
19131
|
+
await cache.fetch(err.offset, err.length);
|
|
19132
|
+
}
|
|
19133
|
+
}
|
|
19134
|
+
}
|
|
19135
|
+
|
|
19136
|
+
/** The local `/public/...` path a remote ref names, for its file type. */
|
|
19137
|
+
function nameForTypeOf(ref) {
|
|
19138
|
+
const split = Internal.remote.splitRemoteRef(ref);
|
|
19139
|
+
return split.status === "success" ? `/${split.filePath}` : ref;
|
|
19140
|
+
}
|
|
19141
|
+
async function fetchText(url, fetchImpl) {
|
|
19142
|
+
const res = await fetchImpl(url, {
|
|
19143
|
+
headers: {}
|
|
19144
|
+
});
|
|
19145
|
+
if (!res.ok) {
|
|
19146
|
+
throw new Error(`Could not read ${url}: HTTP ${res.status}`);
|
|
19147
|
+
}
|
|
19148
|
+
return res.text();
|
|
19149
|
+
}
|
|
19150
|
+
class RangeCache {
|
|
19151
|
+
/** Fetched ranges, as [start, bytes]. Few, so a list is enough. */
|
|
19152
|
+
chunks = [];
|
|
19153
|
+
size;
|
|
19154
|
+
constructor(url, fetchImpl) {
|
|
19155
|
+
this.url = url;
|
|
19156
|
+
this.fetchImpl = fetchImpl;
|
|
19157
|
+
}
|
|
19158
|
+
|
|
19159
|
+
/** A source over what is cached, after fetching the start if nothing is. */
|
|
19160
|
+
async source() {
|
|
19161
|
+
if (this.size === undefined) {
|
|
19162
|
+
await this.fetch(0, READ_AHEAD);
|
|
19163
|
+
}
|
|
19164
|
+
const size = this.size ?? 0;
|
|
19165
|
+
return {
|
|
19166
|
+
size,
|
|
19167
|
+
read: (offset, length) => {
|
|
19168
|
+
const end = Math.min(offset + length, size);
|
|
19169
|
+
if (end <= offset) {
|
|
19170
|
+
return Buffer.alloc(0);
|
|
19171
|
+
}
|
|
19172
|
+
for (const [start, bytes] of this.chunks) {
|
|
19173
|
+
if (start <= offset && end <= start + bytes.length) {
|
|
19174
|
+
return bytes.subarray(offset - start, end - start);
|
|
19175
|
+
}
|
|
19176
|
+
}
|
|
19177
|
+
throw new NeedBytes(offset, end - offset);
|
|
19178
|
+
}
|
|
19179
|
+
};
|
|
19180
|
+
}
|
|
19181
|
+
async fetch(offset, length) {
|
|
19182
|
+
const last = offset + Math.max(length, READ_AHEAD) - 1;
|
|
19183
|
+
const res = await this.fetchImpl(this.url, {
|
|
19184
|
+
headers: {
|
|
19185
|
+
Range: `bytes=${offset}-${last}`
|
|
19186
|
+
}
|
|
19187
|
+
});
|
|
19188
|
+
if (res.status === 200) {
|
|
19189
|
+
// The server ignored the range and sent the whole file. That answers
|
|
19190
|
+
// every read there will be.
|
|
19191
|
+
const whole = Buffer.from(await res.arrayBuffer());
|
|
19192
|
+
this.chunks = [[0, whole]];
|
|
19193
|
+
this.size = whole.length;
|
|
19194
|
+
return;
|
|
19195
|
+
}
|
|
19196
|
+
if (res.status !== 206) {
|
|
19197
|
+
throw new Error(`Could not read ${this.url}: HTTP ${res.status}`);
|
|
19198
|
+
}
|
|
19199
|
+
// `bytes 0-65535/1234567`: where the bytes start, and how big the file is.
|
|
19200
|
+
const range = /^bytes (\d+)-(\d+)\/(\d+|\*)$/.exec(res.headers.get("content-range") ?? "");
|
|
19201
|
+
if (!range) {
|
|
19202
|
+
throw new Error(`Could not read ${this.url}: a partial answer without a Content-Range`);
|
|
19203
|
+
}
|
|
19204
|
+
const bytes = Buffer.from(await res.arrayBuffer());
|
|
19205
|
+
if (range[3] !== "*") {
|
|
19206
|
+
this.size = Number(range[3]);
|
|
19207
|
+
} else if (this.size === undefined) {
|
|
19208
|
+
throw new Error(`Could not read ${this.url}: its size is not known`);
|
|
19209
|
+
}
|
|
19210
|
+
this.chunks.push([Number(range[1]), bytes]);
|
|
19211
|
+
}
|
|
19212
|
+
}
|
|
19213
|
+
|
|
19214
|
+
/**
|
|
19215
|
+
* A gallery entry's key, read as what it actually is.
|
|
19216
|
+
*
|
|
19217
|
+
* A gallery is a record keyed by its entries' file paths. A REMOTE gallery keys
|
|
19218
|
+
* an uploaded entry by the remote URL instead — which encodes that same path —
|
|
19219
|
+
* so every entry has a local path, and the interesting question is whether the
|
|
19220
|
+
* bytes are expected to be at it.
|
|
19221
|
+
*
|
|
19222
|
+
* **They are not, for a remote entry added the ordinary way.** `saveOrUploadFiles`
|
|
19223
|
+
* uploads the remote descriptors to the content host and copies only the local
|
|
19224
|
+
* ones into the working tree, so an image added through the Studio (or over MCP)
|
|
19225
|
+
* and published has no file in the repo, by design — putting one there is what
|
|
19226
|
+
* remote storage exists to avoid. A remote entry CAN have one, because
|
|
19227
|
+
* `val validate --fix` promotes a local file to a remote ref and leaves the file
|
|
19228
|
+
* where it was; so "remote" means "do not require it", never "there is not one".
|
|
19229
|
+
*
|
|
19230
|
+
* One copy, in its own file, because this used to be two: a `remoteKeyToLocalPath`
|
|
19231
|
+
* in `fixHandlers` and a `galleryKeyToLocalPath` in `createFixPatch`, each
|
|
19232
|
+
* normalising the key correctly and each then treating the result as a file that
|
|
19233
|
+
* must exist. That agreement is what made a published remote image report as
|
|
19234
|
+
* missing from both sides at once.
|
|
19235
|
+
*/
|
|
19236
|
+
|
|
19237
|
+
/**
|
|
19238
|
+
* A key that names somewhere else, whether or not it is a ref we can read.
|
|
19239
|
+
*
|
|
19240
|
+
* Case-insensitive, because a scheme is (RFC 3986) — and because
|
|
19241
|
+
* `splitRemoteRef`'s pattern is not, so `HTTPS://…` is precisely one of the
|
|
19242
|
+
* spellings that arrives here unparsed and must not be mistaken for a path.
|
|
17908
19243
|
*/
|
|
17909
19244
|
function isRemoteUrl(key) {
|
|
17910
19245
|
return /^https?:\/\//i.test(key);
|
|
17911
19246
|
}
|
|
17912
|
-
function galleryEntryOf(key) {
|
|
17913
|
-
const remoteRefRes = Internal.remote.splitRemoteRef(key);
|
|
17914
|
-
if (remoteRefRes.status === "success") {
|
|
19247
|
+
function galleryEntryOf(key) {
|
|
19248
|
+
const remoteRefRes = Internal.remote.splitRemoteRef(key);
|
|
19249
|
+
if (remoteRefRes.status === "success") {
|
|
19250
|
+
return {
|
|
19251
|
+
localPath: `/${remoteRefRes.filePath}`,
|
|
19252
|
+
remote: true
|
|
19253
|
+
};
|
|
19254
|
+
}
|
|
19255
|
+
if (isRemoteUrl(key)) {
|
|
19256
|
+
// A URL that is not a ref this version can read: truncated by a hand edit,
|
|
19257
|
+
// a path outside `public/`, a `..` segment — or written by a core version
|
|
19258
|
+
// whose format `splitRemoteRef` rejects.
|
|
19259
|
+
//
|
|
19260
|
+
// Still not a path in the working tree, so the on-disk checks must not run
|
|
19261
|
+
// against it. They would find nothing at `<projectRoot>/https://…`, report
|
|
19262
|
+
// the entry as missing, and `--fix` would DELETE it — the exact failure
|
|
19263
|
+
// this module exists to prevent, arriving at the one moment the key is the
|
|
19264
|
+
// only remaining record of where the bytes went.
|
|
19265
|
+
//
|
|
19266
|
+
// Nothing is being hidden by this: a malformed key is already reported, by
|
|
19267
|
+
// the record schema's own "Invalid remote URL format". That is an error for
|
|
19268
|
+
// a person to look at, not a file for `--fix` to go missing.
|
|
19269
|
+
return {
|
|
19270
|
+
localPath: key,
|
|
19271
|
+
remote: true
|
|
19272
|
+
};
|
|
19273
|
+
}
|
|
19274
|
+
return {
|
|
19275
|
+
localPath: key,
|
|
19276
|
+
remote: false
|
|
19277
|
+
};
|
|
19278
|
+
}
|
|
19279
|
+
|
|
19280
|
+
/**
|
|
19281
|
+
* The media path a validation error is about, if it is about media at all.
|
|
19282
|
+
*
|
|
19283
|
+
* There is no schema in hand here — a `ValidationError` carries only the value
|
|
19284
|
+
* it flagged — so the shape is all there is to go on.
|
|
19285
|
+
*/
|
|
19286
|
+
function getValidationErrorFileRef(validationError) {
|
|
19287
|
+
const maybePath = validationError.value && typeof validationError.value === "object" && "path" in validationError.value && typeof validationError.value.path === "string" ? validationError.value.path : undefined;
|
|
19288
|
+
if (!maybePath) {
|
|
19289
|
+
return null;
|
|
19290
|
+
}
|
|
19291
|
+
return maybePath;
|
|
19292
|
+
}
|
|
19293
|
+
|
|
19294
|
+
const textEncoder$1 = new TextEncoder();
|
|
19295
|
+
async function checkRemoteRef(remoteHost, ref, projectRoot, schema, metadata) {
|
|
19296
|
+
const remoteRefDataRes = Internal.remote.splitRemoteRef(ref);
|
|
19297
|
+
if (remoteRefDataRes.status === "error") {
|
|
19298
|
+
return remoteRefDataRes;
|
|
19299
|
+
}
|
|
19300
|
+
const fileHash = remoteRefDataRes.fileHash;
|
|
19301
|
+
if (!fileHash) {
|
|
19302
|
+
return {
|
|
19303
|
+
status: "error",
|
|
19304
|
+
error: `File hash is missing in remote ref: ${ref}`
|
|
19305
|
+
};
|
|
19306
|
+
}
|
|
19307
|
+
const relativeFilePath = remoteRefDataRes.filePath;
|
|
19308
|
+
if (!relativeFilePath) {
|
|
19309
|
+
return {
|
|
19310
|
+
status: "error",
|
|
19311
|
+
error: `File path is missing in remote ref: ${ref}`
|
|
19312
|
+
};
|
|
19313
|
+
}
|
|
19314
|
+
if (!relativeFilePath.startsWith("public/")) {
|
|
19315
|
+
return {
|
|
19316
|
+
status: "error",
|
|
19317
|
+
error: `File path must be within the public/ directory (e.g. public/path/to/file.txt). Got: ${relativeFilePath}`
|
|
19318
|
+
};
|
|
19319
|
+
}
|
|
19320
|
+
const coreVersion = Internal.VERSION.core || "unknown";
|
|
19321
|
+
const fileExt = getFileExt(relativeFilePath);
|
|
19322
|
+
const currentValidationHash = Internal.remote.getValidationHash(coreVersion, schema, fileExt, metadata,
|
|
19323
|
+
// TODO: validate
|
|
19324
|
+
fileHash, textEncoder$1);
|
|
19325
|
+
const validationHashFromRemoteRefValues = Internal.remote.getValidationHash(remoteRefDataRes.version, schema, getFileExt(remoteRefDataRes.filePath), metadata,
|
|
19326
|
+
// TODO: validate
|
|
19327
|
+
fileHash, textEncoder$1);
|
|
19328
|
+
if (
|
|
19329
|
+
// Current validation hash is the same as the remote validation hash
|
|
19330
|
+
// We assume the uploaded file exists and is correct
|
|
19331
|
+
currentValidationHash === remoteRefDataRes.validationHash &&
|
|
19332
|
+
// This can happen if the version or fileExt has been tampered with
|
|
19333
|
+
// (or if the way it is computed has changed)
|
|
19334
|
+
validationHashFromRemoteRefValues === remoteRefDataRes.validationHash) {
|
|
19335
|
+
return {
|
|
19336
|
+
status: "success"
|
|
19337
|
+
};
|
|
19338
|
+
}
|
|
19339
|
+
// Validation hash does not match, so we need to re-validate the remote content
|
|
19340
|
+
const fileBufferRes = await getFileBufferFromRemote(ref, fileExt, remoteRefDataRes.fileHash, projectRoot);
|
|
19341
|
+
|
|
19342
|
+
// We could not download the remote file for some reason (for example, the file hash is wrong)
|
|
19343
|
+
if (fileBufferRes.status === "error") {
|
|
19344
|
+
return fileBufferRes;
|
|
19345
|
+
}
|
|
19346
|
+
|
|
19347
|
+
// At this point we have the file buffer and we know the correct file hash
|
|
19348
|
+
// We must create a new ref since something has changed
|
|
19349
|
+
|
|
19350
|
+
let currentMetadata;
|
|
19351
|
+
try {
|
|
19352
|
+
if (schema.type === "image") {
|
|
19353
|
+
currentMetadata = await extractImageMetadata(remoteRefDataRes.filePath, fileBufferRes.fileBuffer);
|
|
19354
|
+
} else if (schema.type === "file") {
|
|
19355
|
+
currentMetadata = await extractFileMetadata(remoteRefDataRes.filePath, fileBufferRes.fileBuffer);
|
|
19356
|
+
} else {
|
|
19357
|
+
const exhaustiveCheck = schema;
|
|
19358
|
+
return {
|
|
19359
|
+
status: "error",
|
|
19360
|
+
error: `Unknown schema type: ${JSON.stringify(exhaustiveCheck)}`
|
|
19361
|
+
};
|
|
19362
|
+
}
|
|
19363
|
+
} catch (err) {
|
|
19364
|
+
return {
|
|
19365
|
+
status: "error",
|
|
19366
|
+
error: `Failed to extract metadata from remote file: ${err instanceof Error ? err.message : "Unknown error"}. Cached file: ${fileBufferRes.cachedFilePath}`
|
|
19367
|
+
};
|
|
19368
|
+
}
|
|
19369
|
+
const updatedMetadata = {
|
|
19370
|
+
...metadata,
|
|
19371
|
+
...currentMetadata
|
|
19372
|
+
};
|
|
19373
|
+
const nextValidationHash = Internal.remote.getValidationHash(coreVersion, schema, fileExt, updatedMetadata, fileBufferRes.fileHash, textEncoder$1);
|
|
19374
|
+
if (!updatedMetadata.mimeType) {
|
|
19375
|
+
return {
|
|
19376
|
+
status: "error",
|
|
19377
|
+
error: `MIME type is missing in metadata: ${JSON.stringify(updatedMetadata)}`
|
|
19378
|
+
};
|
|
19379
|
+
}
|
|
19380
|
+
const newFileExt = Internal.mimeTypeToFileExt(updatedMetadata.mimeType);
|
|
19381
|
+
const newFilePath = relativeFilePath.slice(0, -fileExt.length) + newFileExt;
|
|
19382
|
+
return {
|
|
19383
|
+
status: "fix-required",
|
|
19384
|
+
metadata: updatedMetadata,
|
|
19385
|
+
ref: Internal.remote.createRemoteRef(remoteHost, {
|
|
19386
|
+
publicProjectId: remoteRefDataRes.projectId,
|
|
19387
|
+
coreVersion,
|
|
19388
|
+
bucket: remoteRefDataRes.bucket,
|
|
19389
|
+
validationHash: nextValidationHash,
|
|
19390
|
+
fileHash: fileBufferRes.fileHash,
|
|
19391
|
+
filePath: newFilePath
|
|
19392
|
+
})
|
|
19393
|
+
};
|
|
19394
|
+
}
|
|
19395
|
+
function getCachedRemoteFileDir(projectRoot) {
|
|
19396
|
+
// store in projectRoot/.val/remote-file-cache
|
|
19397
|
+
const remoteFileCacheDir = path__default.join(projectRoot, ".val", "remote-file-cache");
|
|
19398
|
+
return remoteFileCacheDir;
|
|
19399
|
+
}
|
|
19400
|
+
function getCachedRemoteFilePath(fileExt, currentFileHash, remoteFileCacheDir) {
|
|
19401
|
+
const remoteFilePath = path__default.join(remoteFileCacheDir, currentFileHash + "." + fileExt);
|
|
19402
|
+
return remoteFilePath;
|
|
19403
|
+
}
|
|
19404
|
+
async function getFileBufferFromRemote(ref, fileExt, currentFileHash, projectRoot) {
|
|
19405
|
+
const remoteFileCacheDir = getCachedRemoteFileDir(projectRoot);
|
|
19406
|
+
const remoteFilePath = getCachedRemoteFilePath(fileExt, currentFileHash, remoteFileCacheDir);
|
|
19407
|
+
try {
|
|
19408
|
+
const fileBuffer = await fs.promises.readFile(remoteFilePath);
|
|
19409
|
+
const computedFileHash = Internal.remote.getFileHash(fileBuffer);
|
|
19410
|
+
if (computedFileHash !== currentFileHash) {
|
|
19411
|
+
await fs.promises.unlink(remoteFilePath);
|
|
19412
|
+
throw Error(`Cached file hash does not match the expected hash: ${computedFileHash} !== ${currentFileHash}`);
|
|
19413
|
+
}
|
|
19414
|
+
} catch {
|
|
19415
|
+
try {
|
|
19416
|
+
await fs.promises.mkdir(remoteFileCacheDir, {
|
|
19417
|
+
recursive: true
|
|
19418
|
+
});
|
|
19419
|
+
await downloadFileFromRemote(ref, remoteFilePath);
|
|
19420
|
+
} catch (err) {
|
|
19421
|
+
try {
|
|
19422
|
+
// try to delete in case of partial download
|
|
19423
|
+
await fs.promises.unlink(remoteFilePath);
|
|
19424
|
+
} catch {
|
|
19425
|
+
//
|
|
19426
|
+
}
|
|
19427
|
+
return {
|
|
19428
|
+
status: "error",
|
|
19429
|
+
error: `Failed to download remote file: ${err instanceof Error ? err.message : "Unknown error"}`
|
|
19430
|
+
};
|
|
19431
|
+
}
|
|
19432
|
+
}
|
|
19433
|
+
const fileBuffer = await fs.promises.readFile(remoteFilePath);
|
|
19434
|
+
const computedFileHash = Internal.remote.getFileHash(fileBuffer);
|
|
19435
|
+
return {
|
|
19436
|
+
status: "success",
|
|
19437
|
+
fileBuffer,
|
|
19438
|
+
fileHash: computedFileHash,
|
|
19439
|
+
cachedFilePath: remoteFilePath
|
|
19440
|
+
};
|
|
19441
|
+
}
|
|
19442
|
+
async function downloadFileFromRemote(ref, filePath) {
|
|
19443
|
+
return new Promise((resolve, reject) => {
|
|
19444
|
+
const url = new URL(ref);
|
|
19445
|
+
const client = url.protocol === "https:" ? https$1 : http$1;
|
|
19446
|
+
const request = client.get(url, response => {
|
|
19447
|
+
if (response.statusCode && response.statusCode >= 300 && response.statusCode < 400 && response.headers.location) {
|
|
19448
|
+
// Handle redirects
|
|
19449
|
+
return downloadFileFromRemote(response.headers.location, filePath).then(resolve).catch(reject);
|
|
19450
|
+
}
|
|
19451
|
+
if (response.statusCode == 404) {
|
|
19452
|
+
return reject(new Error(`${ref}. File not found. The URL is most likely incorrect.`));
|
|
19453
|
+
}
|
|
19454
|
+
if (response.statusCode !== 200) {
|
|
19455
|
+
return reject(new Error(`${ref}. Failed to download file. HTTP Status: ${response.statusCode}`));
|
|
19456
|
+
}
|
|
19457
|
+
const writeStream = fs.createWriteStream(filePath);
|
|
19458
|
+
response.pipe(writeStream);
|
|
19459
|
+
writeStream.on("finish", () => resolve({
|
|
19460
|
+
status: "success",
|
|
19461
|
+
filePath
|
|
19462
|
+
}));
|
|
19463
|
+
writeStream.on("error", err => reject(new Error(`${ref}. Error writing file: ${err.message}`)));
|
|
19464
|
+
});
|
|
19465
|
+
request.on("error", err => reject(new Error(`${ref}. Download error: ${err.message}`)));
|
|
19466
|
+
request.end();
|
|
19467
|
+
});
|
|
19468
|
+
}
|
|
19469
|
+
|
|
19470
|
+
const textEncoder = new TextEncoder();
|
|
19471
|
+
|
|
19472
|
+
/**
|
|
19473
|
+
* Everything an upload to Val Remote needs before it has any bytes: the
|
|
19474
|
+
* developer's token, the project's public id and a bucket. Opened once per
|
|
19475
|
+
* fix, so a video's dozen files go up under one session and into one bucket
|
|
19476
|
+
* rather than a bucket each.
|
|
19477
|
+
*/
|
|
19478
|
+
|
|
19479
|
+
async function openRemoteUploadSession(ctx) {
|
|
19480
|
+
const fail = errorMessage => ({
|
|
19481
|
+
success: false,
|
|
19482
|
+
result: {
|
|
19483
|
+
success: false,
|
|
19484
|
+
errorMessage
|
|
19485
|
+
}
|
|
19486
|
+
});
|
|
19487
|
+
const patFile = getPersonalAccessTokenPath(ctx.projectRoot);
|
|
19488
|
+
if (!ctx.fs.fileExists(patFile)) {
|
|
19489
|
+
return fail(`File: ${path__default.join(ctx.projectRoot, ctx.file)} has remote files that are not uploaded and you are not logged in.\n\nFix this error by logging in:\n\t"npx val login"\n`);
|
|
19490
|
+
}
|
|
19491
|
+
const patFileContent = ctx.fs.readFile(patFile);
|
|
19492
|
+
if (patFileContent === undefined) {
|
|
19493
|
+
return fail(`Could not read personal access token file at ${patFile}`);
|
|
19494
|
+
}
|
|
19495
|
+
const parsedPatFile = parsePersonalAccessTokenFile(patFileContent);
|
|
19496
|
+
if (!parsedPatFile.success) {
|
|
19497
|
+
return fail(`Error parsing personal access token file: ${parsedPatFile.error}. You need to login again.`);
|
|
19498
|
+
}
|
|
19499
|
+
const {
|
|
19500
|
+
pat
|
|
19501
|
+
} = parsedPatFile.data;
|
|
19502
|
+
const projectName = ctx.project;
|
|
19503
|
+
let publicProjectId = ctx.publicProjectId;
|
|
19504
|
+
let remoteFileBuckets = ctx.remoteFileBuckets;
|
|
19505
|
+
let remoteFilesCounter = ctx.remoteFilesCounter;
|
|
19506
|
+
if (!publicProjectId || !remoteFileBuckets) {
|
|
19507
|
+
if (!projectName) {
|
|
19508
|
+
return fail("Project name not found. Add project name to val.config or set the VAL_PROJECT environment variable");
|
|
19509
|
+
}
|
|
19510
|
+
const settingsRes = await ctx.remote.getSettings(projectName, {
|
|
19511
|
+
pat
|
|
19512
|
+
});
|
|
19513
|
+
if (!settingsRes.success) {
|
|
19514
|
+
return fail(`Could not get public project id: ${settingsRes.message}.`);
|
|
19515
|
+
}
|
|
19516
|
+
publicProjectId = settingsRes.data.publicProjectId;
|
|
19517
|
+
remoteFileBuckets = settingsRes.data.remoteFileBuckets.map(b => b.bucket);
|
|
19518
|
+
}
|
|
19519
|
+
if (!publicProjectId) {
|
|
19520
|
+
return fail("Could not get public project id");
|
|
19521
|
+
}
|
|
19522
|
+
if (!projectName) {
|
|
19523
|
+
return fail(`Could not get project. Check that your val.config has the 'project' field set, or set it using the VAL_PROJECT environment variable`);
|
|
19524
|
+
}
|
|
19525
|
+
remoteFilesCounter += 1;
|
|
19526
|
+
const bucket = remoteFileBuckets[remoteFilesCounter % remoteFileBuckets.length];
|
|
19527
|
+
if (!bucket) {
|
|
19528
|
+
return fail(`Internal error: could not allocate a bucket for the remote file located at ${ctx.sourcePath}`);
|
|
19529
|
+
}
|
|
19530
|
+
return {
|
|
19531
|
+
success: true,
|
|
19532
|
+
session: {
|
|
19533
|
+
pat,
|
|
19534
|
+
projectName,
|
|
19535
|
+
publicProjectId,
|
|
19536
|
+
bucket,
|
|
19537
|
+
remoteFileBuckets,
|
|
19538
|
+
remoteFilesCounter
|
|
19539
|
+
}
|
|
19540
|
+
};
|
|
19541
|
+
}
|
|
19542
|
+
|
|
19543
|
+
/**
|
|
19544
|
+
* Upload one file's bytes and return the ref they are served at.
|
|
19545
|
+
*
|
|
19546
|
+
* `relativeFilePath` is the file's path under the project (`public/...`),
|
|
19547
|
+
* which the ref encodes; the bytes need not be the bytes on disk there — a
|
|
19548
|
+
* video's playlists are uploaded REWRITTEN, naming the refs of what they list.
|
|
19549
|
+
*/
|
|
19550
|
+
async function uploadBytesToRemote(ctx, session, relativeFilePath, fileBuffer, metadata, schema) {
|
|
19551
|
+
if (!relativeFilePath.startsWith("public/")) {
|
|
19552
|
+
return {
|
|
19553
|
+
success: false,
|
|
19554
|
+
error: `File path must be within the public/ directory (e.g. public/path/to/file.txt). Got: ${relativeFilePath}`
|
|
19555
|
+
};
|
|
19556
|
+
}
|
|
19557
|
+
const fileHash = Internal.remote.getFileHash(fileBuffer);
|
|
19558
|
+
const coreVersion = Internal.VERSION.core || "unknown";
|
|
19559
|
+
const fileExt = getFileExt(relativeFilePath);
|
|
19560
|
+
const ref = Internal.remote.createRemoteRef(ctx.remote.remoteHost, {
|
|
19561
|
+
publicProjectId: session.publicProjectId,
|
|
19562
|
+
coreVersion,
|
|
19563
|
+
bucket: session.bucket,
|
|
19564
|
+
validationHash: Internal.remote.getValidationHash(coreVersion, schema, fileExt, metadata, fileHash, textEncoder),
|
|
19565
|
+
fileHash,
|
|
19566
|
+
filePath: relativeFilePath
|
|
19567
|
+
});
|
|
19568
|
+
const remoteFileUpload = await ctx.remote.uploadFile(session.projectName, session.bucket, fileHash, fileExt, fileBuffer, {
|
|
19569
|
+
pat: session.pat
|
|
19570
|
+
});
|
|
19571
|
+
if (!remoteFileUpload.success) {
|
|
19572
|
+
return {
|
|
19573
|
+
success: false,
|
|
19574
|
+
error: `Could not upload remote file: '${ref}'. Error: ${remoteFileUpload.error}`
|
|
19575
|
+
};
|
|
19576
|
+
}
|
|
19577
|
+
return {
|
|
19578
|
+
success: true,
|
|
19579
|
+
ref
|
|
19580
|
+
};
|
|
19581
|
+
}
|
|
19582
|
+
|
|
19583
|
+
/**
|
|
19584
|
+
* Every file a video value names, as refs (`/public/…`, or a remote URL as
|
|
19585
|
+
* written): the video, its poster, each caption track — and, for a LOCAL HLS
|
|
19586
|
+
* stream, every file its playlists name, followed from the master down through
|
|
19587
|
+
* each media playlist to the segment files, init sections and audio tracks.
|
|
19588
|
+
*
|
|
19589
|
+
* Followed rather than "everything in the master's directory": the Studio
|
|
19590
|
+
* gives each stream a directory of its own, but nothing stops a hand-placed
|
|
19591
|
+
* `/public/val/clip.m3u8`, and the directory of THAT one is every file in
|
|
19592
|
+
* `/public/val`. A file in a stream's directory that no playlist names is not
|
|
19593
|
+
* used by the stream either, so calling it unused is right.
|
|
19594
|
+
*
|
|
19595
|
+
* A remote stream's playlists are on the content host, so its master is all
|
|
19596
|
+
* that is named here.
|
|
19597
|
+
*
|
|
19598
|
+
* The one place that answers this, so `list-unused-files` and anything else
|
|
19599
|
+
* asking "is this file used" agree on what a video holds.
|
|
19600
|
+
*/
|
|
19601
|
+
function filesOfVideo(video, options) {
|
|
19602
|
+
if (!isObject(video) || typeof video.path !== "string") {
|
|
19603
|
+
return [];
|
|
19604
|
+
}
|
|
19605
|
+
const refs = new Set([video.path]);
|
|
19606
|
+
if (isObject(video.poster) && typeof video.poster.path === "string") {
|
|
19607
|
+
refs.add(video.poster.path);
|
|
19608
|
+
}
|
|
19609
|
+
if (Array.isArray(video.captions)) {
|
|
19610
|
+
for (const track of video.captions) {
|
|
19611
|
+
if (isObject(track) && typeof track.path === "string") {
|
|
19612
|
+
refs.add(track.path);
|
|
19613
|
+
}
|
|
19614
|
+
}
|
|
19615
|
+
}
|
|
19616
|
+
const mimeType = typeof video.mimeType === "string" ? video.mimeType : undefined;
|
|
19617
|
+
if (!Internal.isRemoteMediaPath(video.path) && Internal.media.isHlsVideo({
|
|
19618
|
+
path: video.path,
|
|
19619
|
+
mimeType
|
|
19620
|
+
})) {
|
|
19621
|
+
const readFile = options.readFile ?? readFileOrUndefined$1;
|
|
19622
|
+
const pending = [video.path.split("?")[0]];
|
|
19623
|
+
const visited = new Set();
|
|
19624
|
+
while (pending.length > 0) {
|
|
19625
|
+
var _readFile;
|
|
19626
|
+
const playlist = pending.pop();
|
|
19627
|
+
if (visited.has(playlist)) {
|
|
19628
|
+
continue;
|
|
19629
|
+
}
|
|
19630
|
+
visited.add(playlist);
|
|
19631
|
+
const text = (_readFile = readFile(path__default.join(options.projectRoot, ...playlist.split("/")))) === null || _readFile === void 0 ? void 0 : _readFile.toString("utf-8");
|
|
19632
|
+
if (text === undefined) {
|
|
19633
|
+
continue;
|
|
19634
|
+
}
|
|
19635
|
+
mapHlsUris(text, uri => {
|
|
19636
|
+
const ref = localRefOf(uri, playlist);
|
|
19637
|
+
if (ref !== undefined) {
|
|
19638
|
+
refs.add(ref);
|
|
19639
|
+
if (ref.toLowerCase().endsWith(".m3u8")) {
|
|
19640
|
+
pending.push(ref);
|
|
19641
|
+
}
|
|
19642
|
+
}
|
|
19643
|
+
return uri;
|
|
19644
|
+
});
|
|
19645
|
+
}
|
|
19646
|
+
}
|
|
19647
|
+
return [...refs];
|
|
19648
|
+
}
|
|
19649
|
+
|
|
19650
|
+
/** A playlist URI as a local ref, when it is relative; otherwise undefined. */
|
|
19651
|
+
function localRefOf(uri, playlist) {
|
|
19652
|
+
if (/^[a-z][a-z0-9+.-]*:/i.test(uri) || uri.startsWith("/")) {
|
|
19653
|
+
return undefined;
|
|
19654
|
+
}
|
|
19655
|
+
const relative = uri.split(/[?#]/)[0];
|
|
19656
|
+
if (relative === "") {
|
|
19657
|
+
return undefined;
|
|
19658
|
+
}
|
|
19659
|
+
return path__default.posix.join(path__default.posix.dirname(playlist), relative);
|
|
19660
|
+
}
|
|
19661
|
+
function isObject(value) {
|
|
19662
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
19663
|
+
}
|
|
19664
|
+
function readFileOrUndefined$1(absolutePath) {
|
|
19665
|
+
try {
|
|
19666
|
+
return fs.readFileSync(absolutePath);
|
|
19667
|
+
} catch {
|
|
19668
|
+
return undefined;
|
|
19669
|
+
}
|
|
19670
|
+
}
|
|
19671
|
+
|
|
19672
|
+
/**
|
|
19673
|
+
* `video:upload-remote` and `video:download-remote`: move a video between the
|
|
19674
|
+
* repository and Val Remote.
|
|
19675
|
+
*
|
|
19676
|
+
* A video is the one media value that names several files — the video, its
|
|
19677
|
+
* poster, its caption tracks, and for an HLS stream every playlist and segment
|
|
19678
|
+
* its master names — so one fix moves all of them, and the patch then rewrites
|
|
19679
|
+
* every `path` the value holds. Moving only some would leave a video that is
|
|
19680
|
+
* half on each side, which validation rejects and no page can play.
|
|
19681
|
+
*
|
|
19682
|
+
* HLS is the part that is not a copy. The content host serves a file by its
|
|
19683
|
+
* HASH, so a relative name in a playlist resolves to nothing there: on the way
|
|
19684
|
+
* UP every playlist is rewritten to name the refs of what it lists, bottom-up
|
|
19685
|
+
* (a playlist's ref is the hash of its rewritten bytes, so what it names must
|
|
19686
|
+
* be uploaded first). On the way DOWN the same playlists are rewritten back to
|
|
19687
|
+
* relative names, which is what lets a published stream be plain static files.
|
|
19688
|
+
*/
|
|
19689
|
+
function isRecord$2(value) {
|
|
19690
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
19691
|
+
}
|
|
19692
|
+
function isPlaylist$1(ref) {
|
|
19693
|
+
return ref.split("?")[0].toLowerCase().endsWith(".m3u8");
|
|
19694
|
+
}
|
|
19695
|
+
|
|
19696
|
+
/** The paths a video value holds, each with where in the value it sits. */
|
|
19697
|
+
function videoPathsOf(video) {
|
|
19698
|
+
if (!isRecord$2(video) || typeof video.path !== "string") {
|
|
19699
|
+
return [];
|
|
19700
|
+
}
|
|
19701
|
+
const found = [{
|
|
19702
|
+
at: ["path"],
|
|
19703
|
+
path: video.path
|
|
19704
|
+
}];
|
|
19705
|
+
if (isRecord$2(video.poster) && typeof video.poster.path === "string") {
|
|
19706
|
+
found.push({
|
|
19707
|
+
at: ["poster", "path"],
|
|
19708
|
+
path: video.poster.path
|
|
19709
|
+
});
|
|
19710
|
+
}
|
|
19711
|
+
if (Array.isArray(video.captions)) {
|
|
19712
|
+
video.captions.forEach((track, index) => {
|
|
19713
|
+
if (isRecord$2(track) && typeof track.path === "string") {
|
|
19714
|
+
found.push({
|
|
19715
|
+
at: ["captions", String(index), "path"],
|
|
19716
|
+
path: track.path
|
|
19717
|
+
});
|
|
19718
|
+
}
|
|
19719
|
+
});
|
|
19720
|
+
}
|
|
19721
|
+
return found;
|
|
19722
|
+
}
|
|
19723
|
+
|
|
19724
|
+
/**
|
|
19725
|
+
* The patch that points a video at where its files now are: one `add` per
|
|
19726
|
+
* path that moved, so the authored fields beside them — the description, the
|
|
19727
|
+
* times, the caption labels — are never rewritten.
|
|
19728
|
+
*/
|
|
19729
|
+
function rewriteVideoPathsPatch(sourcePath, video, moved) {
|
|
19730
|
+
const base = sourceToPatchPath(sourcePath);
|
|
19731
|
+
const patch = [];
|
|
19732
|
+
for (const {
|
|
19733
|
+
at,
|
|
19734
|
+
path: current
|
|
19735
|
+
} of videoPathsOf(video)) {
|
|
19736
|
+
const to = moved[current];
|
|
19737
|
+
if (to !== undefined && to !== current) {
|
|
19738
|
+
patch.push({
|
|
19739
|
+
op: "add",
|
|
19740
|
+
path: base.concat(at),
|
|
19741
|
+
value: to
|
|
19742
|
+
});
|
|
19743
|
+
}
|
|
19744
|
+
}
|
|
19745
|
+
return patch;
|
|
19746
|
+
}
|
|
19747
|
+
|
|
19748
|
+
/** A relative playlist URI as a `/public/...` ref, or undefined if it is not one. */
|
|
19749
|
+
function resolvePlaylistUri(uri, playlist) {
|
|
19750
|
+
if (/^[a-z][a-z0-9+.-]*:/i.test(uri) || uri.startsWith("/")) {
|
|
19751
|
+
return undefined;
|
|
19752
|
+
}
|
|
19753
|
+
const relative = uri.split(/[?#]/)[0];
|
|
19754
|
+
if (relative === "") {
|
|
19755
|
+
return undefined;
|
|
19756
|
+
}
|
|
19757
|
+
return path__default.posix.join(path__default.posix.dirname(playlist), relative);
|
|
19758
|
+
}
|
|
19759
|
+
function playlistUris(text) {
|
|
19760
|
+
const uris = [];
|
|
19761
|
+
mapHlsUris(text, uri => {
|
|
19762
|
+
if (!uris.includes(uri)) uris.push(uri);
|
|
19763
|
+
return uri;
|
|
19764
|
+
});
|
|
19765
|
+
return uris;
|
|
19766
|
+
}
|
|
19767
|
+
|
|
19768
|
+
/** What a file is to the video that names it, which decides its hash basis. */
|
|
19769
|
+
|
|
19770
|
+
/**
|
|
19771
|
+
* What a file's validation hash is computed from — the same as the Studio
|
|
19772
|
+
* computes for an upload (`createVideoPatch`), so a video moved by the CLI and
|
|
19773
|
+
* one uploaded in the Studio carry refs of the same kind.
|
|
19774
|
+
*/
|
|
19775
|
+
function metadataOf(localRef, role, video) {
|
|
19776
|
+
const mimeType = Internal.filenameToMimeType(localRef);
|
|
19777
|
+
if (role === "video") {
|
|
19778
|
+
return {
|
|
19779
|
+
mimeType,
|
|
19780
|
+
width: video.width,
|
|
19781
|
+
height: video.height
|
|
19782
|
+
};
|
|
19783
|
+
}
|
|
19784
|
+
if (role === "poster" && isRecord$2(video.poster)) {
|
|
19785
|
+
return {
|
|
19786
|
+
mimeType: video.poster.mimeType ?? mimeType,
|
|
19787
|
+
width: video.poster.width,
|
|
19788
|
+
height: video.poster.height
|
|
19789
|
+
};
|
|
19790
|
+
}
|
|
19791
|
+
return {
|
|
19792
|
+
mimeType
|
|
19793
|
+
};
|
|
19794
|
+
}
|
|
19795
|
+
|
|
19796
|
+
/**
|
|
19797
|
+
* Places a video's local files on Val Remote, each once, and remembers where.
|
|
19798
|
+
*
|
|
19799
|
+
* The one implementation of "upload a video" for every fix that does it: a
|
|
19800
|
+
* video field (`video:upload-remote`), and an entry of an `s.videoset()`
|
|
19801
|
+
* (`videos:upload-remote`). What differs between the two is which files go up
|
|
19802
|
+
* and what their refs are hashed against, so those are the arguments.
|
|
19803
|
+
*
|
|
19804
|
+
* A playlist goes up AFTER everything it names, rewritten to name their refs:
|
|
19805
|
+
* the content host serves a file by its hash, so a relative name in a playlist
|
|
19806
|
+
* resolves to nothing there — and a playlist's own ref is the hash of its
|
|
19807
|
+
* rewritten bytes, so what it names must have refs first.
|
|
19808
|
+
*/
|
|
19809
|
+
function createVideoUploader(ctx, session, schema, metadataFor) {
|
|
19810
|
+
const moved = new Map();
|
|
19811
|
+
const events = [];
|
|
19812
|
+
const put = async (localRef, role) => {
|
|
19813
|
+
const already = moved.get(localRef);
|
|
19814
|
+
if (already !== undefined) {
|
|
19815
|
+
return already;
|
|
19816
|
+
}
|
|
19817
|
+
const absolute = path__default.join(ctx.projectRoot, localRef);
|
|
19818
|
+
const buffer = ctx.fs.readBuffer(absolute);
|
|
19819
|
+
if (buffer === undefined) {
|
|
19820
|
+
throw new Error(`Error reading file: ${absolute}`);
|
|
19821
|
+
}
|
|
19822
|
+
let bytes = buffer;
|
|
19823
|
+
if (isPlaylist$1(localRef)) {
|
|
19824
|
+
// Bottom-up: what this playlist names goes up first, so the refs it is
|
|
19825
|
+
// rewritten to exist — and so its own hash is of the rewritten bytes.
|
|
19826
|
+
const text = buffer.toString("utf-8");
|
|
19827
|
+
const refs = new Map();
|
|
19828
|
+
for (const uri of playlistUris(text)) {
|
|
19829
|
+
const target = resolvePlaylistUri(uri, localRef);
|
|
19830
|
+
if (target !== undefined) {
|
|
19831
|
+
refs.set(uri, await put(target, "stream"));
|
|
19832
|
+
}
|
|
19833
|
+
}
|
|
19834
|
+
bytes = Buffer.from(mapHlsUris(text, uri => refs.get(uri) ?? uri), "utf-8");
|
|
19835
|
+
}
|
|
19836
|
+
const uploaded = await uploadBytesToRemote(ctx, session, localRef.replace(/^\//, ""), bytes, metadataFor(localRef, role), schema);
|
|
19837
|
+
if (!uploaded.success) {
|
|
19838
|
+
throw new Error(uploaded.error);
|
|
19839
|
+
}
|
|
19840
|
+
moved.set(localRef, uploaded.ref);
|
|
19841
|
+
events.push({
|
|
19842
|
+
type: "remote-uploaded",
|
|
19843
|
+
ref: uploaded.ref
|
|
19844
|
+
});
|
|
19845
|
+
return uploaded.ref;
|
|
19846
|
+
};
|
|
19847
|
+
return {
|
|
19848
|
+
put,
|
|
19849
|
+
moved,
|
|
19850
|
+
events
|
|
19851
|
+
};
|
|
19852
|
+
}
|
|
19853
|
+
|
|
19854
|
+
/**
|
|
19855
|
+
* The first of `refs` — and of every file an HLS stream among them names —
|
|
19856
|
+
* that is not on disk, as an absolute path.
|
|
19857
|
+
*
|
|
19858
|
+
* Asked before anything goes up: a missing segment found after the master's
|
|
19859
|
+
* siblings were uploaded leaves bytes on the content host that nothing names.
|
|
19860
|
+
*/
|
|
19861
|
+
function firstMissingVideoFile(ctx, refs) {
|
|
19862
|
+
for (const ref of refs) {
|
|
19863
|
+
const files = filesOfVideo({
|
|
19864
|
+
path: ref.path,
|
|
19865
|
+
mimeType: ref.mimeType
|
|
19866
|
+
}, {
|
|
19867
|
+
projectRoot: ctx.projectRoot,
|
|
19868
|
+
readFile: absolute => ctx.fs.readBuffer(absolute)
|
|
19869
|
+
});
|
|
19870
|
+
for (const file of files) {
|
|
19871
|
+
const absolute = path__default.join(ctx.projectRoot, file);
|
|
19872
|
+
if (!ctx.fs.fileExists(absolute)) {
|
|
19873
|
+
return absolute;
|
|
19874
|
+
}
|
|
19875
|
+
}
|
|
19876
|
+
}
|
|
19877
|
+
return undefined;
|
|
19878
|
+
}
|
|
19879
|
+
function resolveVideo(ctx) {
|
|
19880
|
+
var _resolved$schema;
|
|
19881
|
+
const [, modulePath] = Internal.splitModuleFilePathAndModulePath(ctx.sourcePath);
|
|
19882
|
+
if (!ctx.valModule.source || !ctx.valModule.schema) {
|
|
19883
|
+
return {
|
|
19884
|
+
success: false,
|
|
19885
|
+
result: {
|
|
19886
|
+
success: false,
|
|
19887
|
+
errorMessage: `Could not resolve source or schema for ${ctx.sourcePath}`
|
|
19888
|
+
}
|
|
19889
|
+
};
|
|
19890
|
+
}
|
|
19891
|
+
const resolved = Internal.resolvePath(modulePath, ctx.valModule.source, ctx.valModule.schema);
|
|
19892
|
+
const video = resolved.source;
|
|
19893
|
+
if (((_resolved$schema = resolved.schema) === null || _resolved$schema === void 0 ? void 0 : _resolved$schema.type) !== "video" || !isRecord$2(video) || typeof video.path !== "string") {
|
|
19894
|
+
return {
|
|
19895
|
+
success: false,
|
|
19896
|
+
result: {
|
|
19897
|
+
success: false,
|
|
19898
|
+
errorMessage: `Expected a video at ${ctx.sourcePath}`
|
|
19899
|
+
}
|
|
19900
|
+
};
|
|
19901
|
+
}
|
|
19902
|
+
return {
|
|
19903
|
+
success: true,
|
|
19904
|
+
video,
|
|
19905
|
+
schema: resolved.schema
|
|
19906
|
+
};
|
|
19907
|
+
}
|
|
19908
|
+
|
|
19909
|
+
/**
|
|
19910
|
+
* The paths of a video FIELD that its fix moves.
|
|
19911
|
+
*
|
|
19912
|
+
* All of them for a video of its own. For a field picked from an
|
|
19913
|
+
* `s.videoset()` (`referencedModule`), only the poster and the captions: the
|
|
19914
|
+
* video file is the set's — its key there — and moving it from here would
|
|
19915
|
+
* leave the field naming a file the set does not have. The set's own fix
|
|
19916
|
+
* (`videos:upload-remote`) moves it, and rewrites the fields that name it.
|
|
19917
|
+
*/
|
|
19918
|
+
function movablePathsOf(video, schema) {
|
|
19919
|
+
const setBacked = schema.referencedModule !== undefined;
|
|
19920
|
+
return videoPathsOf(video).filter(({
|
|
19921
|
+
at
|
|
19922
|
+
}) => !setBacked || at[0] !== "path");
|
|
19923
|
+
}
|
|
19924
|
+
async function handleVideoUploadRemote(ctx) {
|
|
19925
|
+
if (!ctx.fix) {
|
|
19926
|
+
return {
|
|
19927
|
+
success: false,
|
|
19928
|
+
errorMessage: `Video needs to be uploaded to Val Remote (use --fix to upload)`
|
|
19929
|
+
};
|
|
19930
|
+
}
|
|
19931
|
+
const resolved = resolveVideo(ctx);
|
|
19932
|
+
if (!resolved.success) {
|
|
19933
|
+
return resolved.result;
|
|
19934
|
+
}
|
|
19935
|
+
const {
|
|
19936
|
+
video,
|
|
19937
|
+
schema
|
|
19938
|
+
} = resolved;
|
|
19939
|
+
if (ctx.remoteFiles[ctx.sourcePath]) {
|
|
19940
|
+
return {
|
|
19941
|
+
success: true,
|
|
19942
|
+
shouldApplyPatch: true,
|
|
19943
|
+
events: [{
|
|
19944
|
+
type: "remote-already-uploaded",
|
|
19945
|
+
filePath: String(video.path)
|
|
19946
|
+
}]
|
|
19947
|
+
};
|
|
19948
|
+
}
|
|
19949
|
+
const toUpload = movablePathsOf(video, schema).filter(({
|
|
19950
|
+
path: ref
|
|
19951
|
+
}) => !Internal.isRemoteMediaPath(ref));
|
|
19952
|
+
// Every file is checked before any goes up: a missing caption found after
|
|
19953
|
+
// the video was uploaded leaves bytes on the content host that nothing names.
|
|
19954
|
+
const missing = firstMissingVideoFile(ctx, toUpload.map(({
|
|
19955
|
+
at,
|
|
19956
|
+
path: ref
|
|
19957
|
+
}) => ({
|
|
19958
|
+
path: ref,
|
|
19959
|
+
...(at[0] === "path" && typeof video.mimeType === "string" ? {
|
|
19960
|
+
mimeType: video.mimeType
|
|
19961
|
+
} : {})
|
|
19962
|
+
})));
|
|
19963
|
+
if (missing !== undefined) {
|
|
19964
|
+
return {
|
|
19965
|
+
success: false,
|
|
19966
|
+
errorMessage: `File ${missing} does not exist`
|
|
19967
|
+
};
|
|
19968
|
+
}
|
|
19969
|
+
const opened = await openRemoteUploadSession(ctx);
|
|
19970
|
+
if (!opened.success) {
|
|
19971
|
+
return opened.result;
|
|
19972
|
+
}
|
|
19973
|
+
const {
|
|
19974
|
+
session
|
|
19975
|
+
} = opened;
|
|
19976
|
+
const uploader = createVideoUploader(ctx, session, schema, (localRef, role) => metadataOf(localRef, role, video));
|
|
19977
|
+
try {
|
|
19978
|
+
for (const {
|
|
19979
|
+
at,
|
|
19980
|
+
path: ref
|
|
19981
|
+
} of toUpload) {
|
|
19982
|
+
await uploader.put(ref, at[0] === "poster" ? "poster" : at[0] === "captions" ? "caption" : "video");
|
|
19983
|
+
}
|
|
19984
|
+
} catch (err) {
|
|
19985
|
+
return {
|
|
19986
|
+
success: false,
|
|
19987
|
+
errorMessage: err instanceof Error ? err.message : String(err)
|
|
19988
|
+
};
|
|
19989
|
+
}
|
|
19990
|
+
const refs = Object.fromEntries(uploader.moved);
|
|
19991
|
+
ctx.remoteFiles[ctx.sourcePath] = {
|
|
19992
|
+
ref: refs[String(video.path)] ?? String(video.path),
|
|
19993
|
+
refs
|
|
19994
|
+
};
|
|
19995
|
+
return {
|
|
19996
|
+
success: true,
|
|
19997
|
+
shouldApplyPatch: true,
|
|
19998
|
+
publicProjectId: session.publicProjectId,
|
|
19999
|
+
remoteFileBuckets: session.remoteFileBuckets,
|
|
20000
|
+
remoteFilesCounter: session.remoteFilesCounter,
|
|
20001
|
+
events: uploader.events
|
|
20002
|
+
};
|
|
20003
|
+
}
|
|
20004
|
+
async function handleVideoDownloadRemote(ctx) {
|
|
20005
|
+
if (!ctx.fix) {
|
|
17915
20006
|
return {
|
|
17916
|
-
|
|
17917
|
-
|
|
20007
|
+
success: false,
|
|
20008
|
+
errorMessage: `Video ${ctx.sourcePath} needs to be downloaded (use --fix to download)`
|
|
17918
20009
|
};
|
|
17919
20010
|
}
|
|
17920
|
-
|
|
17921
|
-
|
|
17922
|
-
|
|
17923
|
-
|
|
17924
|
-
|
|
17925
|
-
|
|
17926
|
-
|
|
17927
|
-
|
|
17928
|
-
|
|
17929
|
-
|
|
17930
|
-
|
|
17931
|
-
|
|
17932
|
-
|
|
17933
|
-
|
|
20011
|
+
const resolved = resolveVideo(ctx);
|
|
20012
|
+
if (!resolved.success) {
|
|
20013
|
+
return resolved.result;
|
|
20014
|
+
}
|
|
20015
|
+
const {
|
|
20016
|
+
video,
|
|
20017
|
+
schema
|
|
20018
|
+
} = resolved;
|
|
20019
|
+
const moved = new Map();
|
|
20020
|
+
const fetchTo = async ref => {
|
|
20021
|
+
const already = moved.get(ref);
|
|
20022
|
+
if (already !== undefined) {
|
|
20023
|
+
return already;
|
|
20024
|
+
}
|
|
20025
|
+
const split = Internal.remote.splitRemoteRef(ref);
|
|
20026
|
+
if (split.status === "error") {
|
|
20027
|
+
throw new Error(`Not a Val Remote ref: ${ref} (${split.error})`);
|
|
20028
|
+
}
|
|
20029
|
+
if (!split.filePath.startsWith("public/")) {
|
|
20030
|
+
throw new Error(`Cannot download ${ref}: its file path must start with public/ (got ${split.filePath})`);
|
|
20031
|
+
}
|
|
20032
|
+
const localRef = `/${split.filePath}`;
|
|
20033
|
+
const absolute = path__default.join(ctx.projectRoot, split.filePath);
|
|
20034
|
+
await fs.promises.mkdir(path__default.dirname(absolute), {
|
|
20035
|
+
recursive: true
|
|
20036
|
+
});
|
|
20037
|
+
const res = await downloadFileFromRemote(ref, absolute);
|
|
20038
|
+
if (res.status === "error") {
|
|
20039
|
+
throw new Error(res.error);
|
|
20040
|
+
}
|
|
20041
|
+
moved.set(ref, localRef);
|
|
20042
|
+
if (isPlaylist$1(localRef)) {
|
|
20043
|
+
// Back to relative names: a local stream is plain static files, and the
|
|
20044
|
+
// draft endpoint rewrites relative names, not refs.
|
|
20045
|
+
const text = fs.readFileSync(absolute, "utf-8");
|
|
20046
|
+
const local = new Map();
|
|
20047
|
+
for (const uri of playlistUris(text)) {
|
|
20048
|
+
if (Internal.remote.splitRemoteRef(uri).status === "success") {
|
|
20049
|
+
const target = await fetchTo(uri);
|
|
20050
|
+
local.set(uri, path__default.posix.relative(path__default.posix.dirname(localRef), target));
|
|
20051
|
+
}
|
|
20052
|
+
}
|
|
20053
|
+
fs.writeFileSync(absolute, mapHlsUris(text, uri => local.get(uri) ?? uri), "utf-8");
|
|
20054
|
+
}
|
|
20055
|
+
return localRef;
|
|
20056
|
+
};
|
|
20057
|
+
try {
|
|
20058
|
+
for (const {
|
|
20059
|
+
path: ref
|
|
20060
|
+
} of movablePathsOf(video, schema)) {
|
|
20061
|
+
if (Internal.isRemoteMediaPath(ref)) {
|
|
20062
|
+
await fetchTo(ref);
|
|
20063
|
+
}
|
|
20064
|
+
}
|
|
20065
|
+
} catch (err) {
|
|
20066
|
+
return {
|
|
20067
|
+
success: false,
|
|
20068
|
+
errorMessage: err instanceof Error ? err.message : String(err)
|
|
20069
|
+
};
|
|
20070
|
+
}
|
|
20071
|
+
const refs = Object.fromEntries(moved);
|
|
20072
|
+
ctx.remoteFiles[ctx.sourcePath] = {
|
|
20073
|
+
ref: refs[String(video.path)] ?? String(video.path),
|
|
20074
|
+
refs
|
|
20075
|
+
};
|
|
20076
|
+
return {
|
|
20077
|
+
success: true,
|
|
20078
|
+
shouldApplyPatch: true,
|
|
20079
|
+
events: [{
|
|
20080
|
+
type: "remote-downloading",
|
|
20081
|
+
sourcePath: ctx.sourcePath
|
|
20082
|
+
}]
|
|
20083
|
+
};
|
|
20084
|
+
}
|
|
20085
|
+
|
|
20086
|
+
/**
|
|
20087
|
+
* What is out of step between a gallery's entries and its directory.
|
|
20088
|
+
*
|
|
20089
|
+
* Two questions, and they treat a remote entry differently — which is the whole
|
|
20090
|
+
* reason this is separate from the handler around it:
|
|
20091
|
+
*
|
|
20092
|
+
* - **Missing**: an entry with no bytes at its local path. Asked of LOCAL
|
|
20093
|
+
* entries only. A remote entry's bytes live on the content host, and nothing
|
|
20094
|
+
* puts a copy in the working tree: `saveOrUploadFiles` uploads the remote
|
|
20095
|
+
* descriptors and copies only the local ones into the tree, so a remote entry
|
|
20096
|
+
* added through the Studio (or over MCP) has no local file by design, and
|
|
20097
|
+
* demanding one would mean committing remote bytes to git — which is what
|
|
20098
|
+
* remote storage exists to avoid. Whether those bytes really are on the host
|
|
20099
|
+
* is a different check, `image:check-remote`, which already runs for exactly
|
|
20100
|
+
* these entries.
|
|
20101
|
+
* - **Untracked**: a file in the directory that no entry claims. Asked of every
|
|
20102
|
+
* entry, remote included, and that is why they are normalised to their local
|
|
20103
|
+
* path: `val validate --fix` promotes a local file to a remote ref and leaves
|
|
20104
|
+
* the file where it was, so a remote entry can perfectly well have one.
|
|
20105
|
+
*
|
|
20106
|
+
* An entry can claim more than its key. A video set's HLS stream is keyed by
|
|
20107
|
+
* its master playlist, and the media playlists and segments it names are the
|
|
20108
|
+
* same entry — so `filesOfEntry` says what else an entry holds, and those are
|
|
20109
|
+
* not untracked either.
|
|
20110
|
+
*
|
|
20111
|
+
* Its own module rather than a part of `fixHandlers.ts`, so the video set's
|
|
20112
|
+
* fixes can use it without importing the handler registry they are part of.
|
|
20113
|
+
*/
|
|
20114
|
+
function checkGalleryFiles(input) {
|
|
20115
|
+
const {
|
|
20116
|
+
dir,
|
|
20117
|
+
projectRoot,
|
|
20118
|
+
fs,
|
|
20119
|
+
filesOfEntry
|
|
20120
|
+
} = input;
|
|
20121
|
+
const entries = input.entryKeys.map(key => ({
|
|
20122
|
+
key,
|
|
20123
|
+
entry: galleryEntryOf(key)
|
|
20124
|
+
}));
|
|
20125
|
+
const trackedFiles = new Set();
|
|
20126
|
+
for (const {
|
|
20127
|
+
key,
|
|
20128
|
+
entry
|
|
20129
|
+
} of entries) {
|
|
20130
|
+
trackedFiles.add(entry.localPath);
|
|
20131
|
+
for (const file of (filesOfEntry === null || filesOfEntry === void 0 ? void 0 : filesOfEntry(entry, key)) ?? []) {
|
|
20132
|
+
trackedFiles.add(file);
|
|
20133
|
+
}
|
|
20134
|
+
}
|
|
20135
|
+
const missingTrackedFiles = entries.filter(({
|
|
20136
|
+
entry
|
|
20137
|
+
}) => !entry.remote && !fs.fileExists(path__default.join(projectRoot, entry.localPath))).map(({
|
|
20138
|
+
entry
|
|
20139
|
+
}) => entry.localPath);
|
|
20140
|
+
return {
|
|
20141
|
+
missingTrackedFiles,
|
|
20142
|
+
untrackedFiles: filesInDirectory({
|
|
20143
|
+
dir,
|
|
20144
|
+
projectRoot,
|
|
20145
|
+
fs
|
|
20146
|
+
}).filter(f => !trackedFiles.has(f))
|
|
20147
|
+
};
|
|
20148
|
+
}
|
|
20149
|
+
|
|
20150
|
+
/**
|
|
20151
|
+
* The LOCAL entries whose key is there but some of what they hold is not: a
|
|
20152
|
+
* stream missing a segment plays up to the gap and then stops. Separate from
|
|
20153
|
+
* `checkGalleryFiles`' "missing" because a fix does not remove these — the
|
|
20154
|
+
* entry still names a video someone uploaded and may have the rest of.
|
|
20155
|
+
*/
|
|
20156
|
+
function incompleteGalleryEntries(input) {
|
|
20157
|
+
const exists = file => input.fs.fileExists(path__default.join(input.projectRoot, file));
|
|
20158
|
+
const incomplete = [];
|
|
20159
|
+
for (const key of input.entryKeys) {
|
|
20160
|
+
const entry = galleryEntryOf(key);
|
|
20161
|
+
if (entry.remote || !exists(entry.localPath)) {
|
|
20162
|
+
continue;
|
|
20163
|
+
}
|
|
20164
|
+
const missing = input.filesOfEntry(entry, key).filter(file => file !== entry.localPath && !exists(file));
|
|
20165
|
+
if (missing.length > 0) {
|
|
20166
|
+
incomplete.push({
|
|
20167
|
+
key,
|
|
20168
|
+
missing
|
|
20169
|
+
});
|
|
20170
|
+
}
|
|
20171
|
+
}
|
|
20172
|
+
return incomplete;
|
|
20173
|
+
}
|
|
20174
|
+
|
|
20175
|
+
/** Every file under `dir`, as `/public/…` refs; none for a directory not there. */
|
|
20176
|
+
function filesInDirectory({
|
|
20177
|
+
dir,
|
|
20178
|
+
projectRoot,
|
|
20179
|
+
fs
|
|
20180
|
+
}) {
|
|
20181
|
+
const filesInDir = [];
|
|
20182
|
+
try {
|
|
20183
|
+
const found = fs.readDirectory(path__default.join(projectRoot, dir), undefined, undefined, ["**/*"]);
|
|
20184
|
+
for (const entry of found) {
|
|
20185
|
+
filesInDir.push("/" + path__default.relative(projectRoot, entry).split(path__default.sep).join("/"));
|
|
20186
|
+
}
|
|
20187
|
+
} catch {
|
|
20188
|
+
// directory doesn't exist — no untracked files possible
|
|
20189
|
+
}
|
|
20190
|
+
return filesInDir;
|
|
20191
|
+
}
|
|
20192
|
+
|
|
20193
|
+
/**
|
|
20194
|
+
* The fixes of an `s.videoset()` — the video twin of an image gallery.
|
|
20195
|
+
*
|
|
20196
|
+
* A set is a record keyed by each video's file path (an HLS stream by its
|
|
20197
|
+
* master playlist), so most of this is the gallery fixes again with the key as
|
|
20198
|
+
* the file. Three things are not, and they are why this is a module of its own
|
|
20199
|
+
* rather than another branch in `fixHandlers.ts`:
|
|
20200
|
+
*
|
|
20201
|
+
* - **An entry is several files.** A stream's key is its master playlist, and
|
|
20202
|
+
* the media playlists and segments it names belong to the same entry. They
|
|
20203
|
+
* are tracked by it, uploaded with it, and none of them is an entry itself.
|
|
20204
|
+
* - **The set's directory holds files that are not entries.** A field picked
|
|
20205
|
+
* from the set (`s.video(videosVal)`) keeps its own poster and caption
|
|
20206
|
+
* tracks, and the Studio puts them where the set keeps its videos. A file a
|
|
20207
|
+
* video field names is accounted for, not untracked.
|
|
20208
|
+
* - **Fields elsewhere name an entry by its key.** Uploading an entry renames
|
|
20209
|
+
* its key to the remote ref, and every `s.video(videosVal)` whose `path` was
|
|
20210
|
+
* the old key would then name a video the set does not have. So the upload
|
|
20211
|
+
* rewrites them too (`otherModulePatches`).
|
|
20212
|
+
*
|
|
20213
|
+
* Nothing here imports `fixHandlers.ts` at runtime: that module registers
|
|
20214
|
+
* these handlers, and a cycle would hand it undefined ones.
|
|
20215
|
+
*/
|
|
20216
|
+
|
|
20217
|
+
/** The fields of a set entry that are read from the bytes, in the order written. */
|
|
20218
|
+
const VIDEO_METADATA_FIELDS$1 = ["mimeType", "width", "height", "duration"];
|
|
20219
|
+
|
|
20220
|
+
/** What of a `Service` the project-wide walks need. */
|
|
20221
|
+
|
|
20222
|
+
function isRecord$1(value) {
|
|
20223
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
20224
|
+
}
|
|
20225
|
+
function isPlaylist(ref) {
|
|
20226
|
+
return ref.split("?")[0].toLowerCase().endsWith(".m3u8");
|
|
20227
|
+
}
|
|
20228
|
+
function readFileOrUndefined(absolutePath) {
|
|
20229
|
+
try {
|
|
20230
|
+
return fs.readFileSync(absolutePath);
|
|
20231
|
+
} catch {
|
|
20232
|
+
return undefined;
|
|
20233
|
+
}
|
|
20234
|
+
}
|
|
20235
|
+
|
|
20236
|
+
/** See `Internal.videosetEntryVideoSchema`: the Studio hashes against it too. */
|
|
20237
|
+
function videosetEntryVideoSchema(record) {
|
|
20238
|
+
return Internal.videosetEntryVideoSchema(record);
|
|
20239
|
+
}
|
|
20240
|
+
|
|
20241
|
+
/** The key of the set entry at `sourcePath`: its last module path segment. */
|
|
20242
|
+
function entryKeyOf(sourcePath) {
|
|
20243
|
+
const [, modulePath] = Internal.splitModuleFilePathAndModulePath(sourcePath);
|
|
20244
|
+
const parts = Internal.splitModulePath(modulePath);
|
|
20245
|
+
return parts[parts.length - 1];
|
|
20246
|
+
}
|
|
20247
|
+
|
|
20248
|
+
/**
|
|
20249
|
+
* The patch path of `sourcePath`. Not `sourceToPatchPath`: a set's keys are
|
|
20250
|
+
* file paths, and their dots do not survive its `split(".")`.
|
|
20251
|
+
*/
|
|
20252
|
+
function patchPathOf(sourcePath) {
|
|
20253
|
+
const [, modulePath] = Internal.splitModuleFilePathAndModulePath(sourcePath);
|
|
20254
|
+
return Internal.splitModulePath(modulePath);
|
|
20255
|
+
}
|
|
20256
|
+
|
|
20257
|
+
/**
|
|
20258
|
+
* Every file a set entry holds: its key's file and, for an HLS stream, every
|
|
20259
|
+
* playlist and segment its master names — read from the LOCAL copy, which a
|
|
20260
|
+
* remote entry promoted by `--fix` still has, and which is the only place a
|
|
20261
|
+
* stream's tree can be followed from at all.
|
|
20262
|
+
*/
|
|
20263
|
+
function filesOfVideosetEntry(entry, metadata, options) {
|
|
20264
|
+
const fields = isRecord$1(metadata) ? metadata : {};
|
|
20265
|
+
const mimeType = typeof fields.mimeType === "string" ? fields.mimeType : undefined;
|
|
20266
|
+
// The entry's poster and captions are its defaults' files: the set holds
|
|
20267
|
+
// them as it holds the video, so they are tracked by the entry — a remote
|
|
20268
|
+
// one by the local path it was uploaded from, as the key is.
|
|
20269
|
+
return filesOfVideo({
|
|
20270
|
+
path: entry.localPath,
|
|
20271
|
+
mimeType,
|
|
20272
|
+
poster: fields.poster,
|
|
20273
|
+
captions: fields.captions
|
|
20274
|
+
}, options).map(ref => galleryEntryOf(ref).localPath);
|
|
20275
|
+
}
|
|
20276
|
+
|
|
20277
|
+
/**
|
|
20278
|
+
* An entry with every file it names that `moved` has a new ref for pointed
|
|
20279
|
+
* at that ref: the poster and the captions, which a remote set keeps on the
|
|
20280
|
+
* content host with the video.
|
|
20281
|
+
*/
|
|
20282
|
+
function withMovedFiles(entry, moved) {
|
|
20283
|
+
const movedPath = file => isRecord$1(file) && typeof file.path === "string" && moved.has(file.path) ? {
|
|
20284
|
+
...file,
|
|
20285
|
+
path: moved.get(file.path)
|
|
20286
|
+
} : file;
|
|
20287
|
+
return {
|
|
20288
|
+
...entry,
|
|
20289
|
+
...(entry.poster !== undefined ? {
|
|
20290
|
+
poster: movedPath(entry.poster)
|
|
20291
|
+
} : {}),
|
|
20292
|
+
...(Array.isArray(entry.captions) ? {
|
|
20293
|
+
captions: entry.captions.map(movedPath)
|
|
20294
|
+
} : {})
|
|
20295
|
+
};
|
|
20296
|
+
}
|
|
20297
|
+
|
|
20298
|
+
/** The entry's poster and caption files that are still local. */
|
|
20299
|
+
function localFilesOfEntry(entry) {
|
|
20300
|
+
const files = [];
|
|
20301
|
+
if (isRecord$1(entry.poster) && typeof entry.poster.path === "string") {
|
|
20302
|
+
files.push({
|
|
20303
|
+
path: entry.poster.path,
|
|
20304
|
+
role: "poster"
|
|
20305
|
+
});
|
|
20306
|
+
}
|
|
20307
|
+
if (Array.isArray(entry.captions)) {
|
|
20308
|
+
for (const track of entry.captions) {
|
|
20309
|
+
if (isRecord$1(track) && typeof track.path === "string") {
|
|
20310
|
+
files.push({
|
|
20311
|
+
path: track.path,
|
|
20312
|
+
role: "caption"
|
|
20313
|
+
});
|
|
20314
|
+
}
|
|
20315
|
+
}
|
|
20316
|
+
}
|
|
20317
|
+
return files.filter(file => !Internal.isRemoteMediaPath(file.path));
|
|
20318
|
+
}
|
|
20319
|
+
|
|
20320
|
+
/**
|
|
20321
|
+
* The untracked files of a set's directory, sorted into the ones `--fix` can
|
|
20322
|
+
* make entries of and the ones a person has to decide about.
|
|
20323
|
+
*
|
|
20324
|
+
* - `videos`: an untracked stream (keyed by its master playlist — the media
|
|
20325
|
+
* playlists and segments it names are part of it and listed nowhere), and a
|
|
20326
|
+
* progressive video the set accepts.
|
|
20327
|
+
* - `others`: everything else — a poster or caption no video field names, a
|
|
20328
|
+
* media playlist without a master, a video of a type the set does not
|
|
20329
|
+
* accept. `--fix` does not add these: none of them is a video the set could
|
|
20330
|
+
* hold, and deleting someone's file is not a fix.
|
|
20331
|
+
*/
|
|
20332
|
+
function classifyUntrackedVideosetFiles({
|
|
20333
|
+
untracked,
|
|
20334
|
+
accept,
|
|
20335
|
+
projectRoot,
|
|
20336
|
+
readFile = readFileOrUndefined
|
|
20337
|
+
}) {
|
|
20338
|
+
const masters = untracked.filter(file => {
|
|
20339
|
+
var _readFile;
|
|
20340
|
+
if (!isPlaylist(file)) {
|
|
20341
|
+
return false;
|
|
20342
|
+
}
|
|
20343
|
+
const text = (_readFile = readFile(path__default.join(projectRoot, file))) === null || _readFile === void 0 ? void 0 : _readFile.toString("utf-8");
|
|
20344
|
+
return text !== undefined && isHlsMasterPlaylist(text);
|
|
20345
|
+
});
|
|
20346
|
+
const partOfAStream = new Set();
|
|
20347
|
+
for (const master of masters) {
|
|
20348
|
+
for (const file of filesOfVideo({
|
|
20349
|
+
path: master
|
|
20350
|
+
}, {
|
|
20351
|
+
projectRoot,
|
|
20352
|
+
readFile
|
|
20353
|
+
})) {
|
|
20354
|
+
if (file !== master) {
|
|
20355
|
+
partOfAStream.add(file);
|
|
20356
|
+
}
|
|
20357
|
+
}
|
|
20358
|
+
}
|
|
20359
|
+
const videos = [];
|
|
20360
|
+
const others = [];
|
|
20361
|
+
for (const file of untracked) {
|
|
20362
|
+
if (partOfAStream.has(file)) {
|
|
20363
|
+
continue;
|
|
20364
|
+
}
|
|
20365
|
+
if (masters.includes(file)) {
|
|
20366
|
+
videos.push(file);
|
|
20367
|
+
continue;
|
|
20368
|
+
}
|
|
20369
|
+
const mimeType = Internal.filenameToMimeType(file);
|
|
20370
|
+
if (mimeType !== undefined && mimeType.startsWith("video/") && Internal.mimeTypeMatchesAccept(mimeType, accept)) {
|
|
20371
|
+
videos.push(file);
|
|
20372
|
+
} else {
|
|
20373
|
+
others.push(file);
|
|
20374
|
+
}
|
|
20375
|
+
}
|
|
20376
|
+
return {
|
|
20377
|
+
videos,
|
|
20378
|
+
others
|
|
20379
|
+
};
|
|
20380
|
+
}
|
|
20381
|
+
|
|
20382
|
+
/** Whether a serialized schema has a video field anywhere in it. */
|
|
20383
|
+
function hasVideoField(schema) {
|
|
20384
|
+
if (Array.isArray(schema)) {
|
|
20385
|
+
return schema.some(hasVideoField);
|
|
20386
|
+
}
|
|
20387
|
+
if (!isRecord$1(schema)) {
|
|
20388
|
+
return false;
|
|
20389
|
+
}
|
|
20390
|
+
if (schema.type === "video") {
|
|
20391
|
+
return true;
|
|
20392
|
+
}
|
|
20393
|
+
return Object.values(schema).some(hasVideoField);
|
|
20394
|
+
}
|
|
20395
|
+
/**
|
|
20396
|
+
* Every video field in the project, with its value.
|
|
20397
|
+
*
|
|
20398
|
+
* Validated reads for the modules that have one, because only those come back
|
|
20399
|
+
* with their `.jsonValues()` entries loaded — an unvalidated read hands back
|
|
20400
|
+
* each entry as a marker, and a field inside one would be invisible. Modules
|
|
20401
|
+
* without a video are read once, unvalidated, to find that out.
|
|
20402
|
+
*/
|
|
20403
|
+
async function videoFieldsOf(service) {
|
|
20404
|
+
const found = [];
|
|
20405
|
+
for (const moduleFilePath of service.getModuleFilePaths()) {
|
|
20406
|
+
const shallow = await service.get(moduleFilePath, "", {
|
|
20407
|
+
validate: false
|
|
20408
|
+
});
|
|
20409
|
+
if (!hasVideoField(shallow.schema)) {
|
|
20410
|
+
continue;
|
|
20411
|
+
}
|
|
20412
|
+
const loaded = await service.get(moduleFilePath, "", {
|
|
20413
|
+
validate: true
|
|
20414
|
+
});
|
|
20415
|
+
if (loaded.source === undefined || !loaded.schema) {
|
|
20416
|
+
continue;
|
|
20417
|
+
}
|
|
20418
|
+
traverseSchemaSource(loaded.source, loaded.schema, moduleFilePath, ({
|
|
20419
|
+
source,
|
|
20420
|
+
schema,
|
|
20421
|
+
path: sourcePath
|
|
20422
|
+
}) => {
|
|
20423
|
+
if (schema.type === "video") {
|
|
20424
|
+
found.push({
|
|
20425
|
+
moduleFilePath,
|
|
20426
|
+
sourcePath,
|
|
20427
|
+
source,
|
|
20428
|
+
schema
|
|
20429
|
+
});
|
|
20430
|
+
}
|
|
20431
|
+
});
|
|
20432
|
+
}
|
|
20433
|
+
return found;
|
|
20434
|
+
}
|
|
20435
|
+
|
|
20436
|
+
/**
|
|
20437
|
+
* The files video fields name — a video of its own, and the poster and
|
|
20438
|
+
* captions of every field — as `/public/…` refs, a remote one by the local
|
|
20439
|
+
* path it was uploaded from.
|
|
20440
|
+
*/
|
|
20441
|
+
async function filesNamedByVideoFields(service, options) {
|
|
20442
|
+
const files = new Set();
|
|
20443
|
+
for (const field of await videoFieldsOf(service)) {
|
|
20444
|
+
for (const ref of filesOfVideo(field.source, options)) {
|
|
20445
|
+
// A remote ref by the path it encodes: `--fix` uploads a poster and
|
|
20446
|
+
// leaves the file where it was, exactly as it does a set's entry, and
|
|
20447
|
+
// that file is the field's — not something the set forgot.
|
|
20448
|
+
files.add(galleryEntryOf(ref).localPath);
|
|
20449
|
+
}
|
|
20450
|
+
}
|
|
20451
|
+
return files;
|
|
20452
|
+
}
|
|
20453
|
+
|
|
20454
|
+
/**
|
|
20455
|
+
* The patches that point every `s.video(set)` field at a set entry's new key.
|
|
20456
|
+
*
|
|
20457
|
+
* One `add` on each field's `path`, so the description, poster, times and
|
|
20458
|
+
* captions beside it are never rewritten. A field of another set, or of no
|
|
20459
|
+
* set, is left alone even when its path is the same string: it names its own
|
|
20460
|
+
* file.
|
|
20461
|
+
*/
|
|
20462
|
+
async function videosetReferencePatches(service, setModuleFilePath, renamed) {
|
|
20463
|
+
const byModule = new Map();
|
|
20464
|
+
for (const field of await videoFieldsOf(service)) {
|
|
20465
|
+
if (field.schema.referencedModule !== setModuleFilePath) {
|
|
20466
|
+
continue;
|
|
20467
|
+
}
|
|
20468
|
+
const value = field.source;
|
|
20469
|
+
const current = isRecord$1(value) ? value.path : undefined;
|
|
20470
|
+
if (typeof current !== "string" || !(current in renamed)) {
|
|
20471
|
+
continue;
|
|
20472
|
+
}
|
|
20473
|
+
const patch = byModule.get(field.moduleFilePath) ?? [];
|
|
20474
|
+
patch.push({
|
|
20475
|
+
op: "add",
|
|
20476
|
+
path: patchPathOf(field.sourcePath).concat("path"),
|
|
20477
|
+
value: renamed[current]
|
|
20478
|
+
});
|
|
20479
|
+
byModule.set(field.moduleFilePath, patch);
|
|
20480
|
+
}
|
|
20481
|
+
return [...byModule].map(([moduleFilePath, patch]) => ({
|
|
20482
|
+
moduleFilePath,
|
|
20483
|
+
patch
|
|
20484
|
+
}));
|
|
20485
|
+
}
|
|
20486
|
+
|
|
20487
|
+
/** The record at `sourcePath` and its entries, or why it could not be read. */
|
|
20488
|
+
function resolveVideoset(module, sourcePath) {
|
|
20489
|
+
var _resolved$schema;
|
|
20490
|
+
if (module.source === undefined || !module.schema) {
|
|
20491
|
+
return {
|
|
20492
|
+
success: false,
|
|
20493
|
+
errorMessage: `Could not resolve source or schema for ${sourcePath}`
|
|
20494
|
+
};
|
|
20495
|
+
}
|
|
20496
|
+
const [, modulePath] = Internal.splitModuleFilePathAndModulePath(sourcePath);
|
|
20497
|
+
const resolved = Internal.resolvePath(modulePath, module.source, module.schema);
|
|
20498
|
+
if (((_resolved$schema = resolved.schema) === null || _resolved$schema === void 0 ? void 0 : _resolved$schema.type) !== "record" || resolved.schema.mediaType !== "videos") {
|
|
20499
|
+
var _resolved$schema2;
|
|
20500
|
+
return {
|
|
20501
|
+
success: false,
|
|
20502
|
+
errorMessage: `Expected a video set at ${sourcePath}, got ${((_resolved$schema2 = resolved.schema) === null || _resolved$schema2 === void 0 ? void 0 : _resolved$schema2.type) ?? "nothing"}`
|
|
20503
|
+
};
|
|
20504
|
+
}
|
|
20505
|
+
return {
|
|
20506
|
+
success: true,
|
|
20507
|
+
record: resolved.schema,
|
|
20508
|
+
entries: isRecord$1(resolved.source) ? resolved.source : {}
|
|
20509
|
+
};
|
|
20510
|
+
}
|
|
20511
|
+
|
|
20512
|
+
/**
|
|
20513
|
+
* `videos:add-metadata`: the entry's KEY is its file, and it must be on disk
|
|
20514
|
+
* (or on Val Remote, read over HTTP) and a kind Val can read the size and
|
|
20515
|
+
* length of here — the same preconditions as `video:add-metadata`, asked of
|
|
20516
|
+
* the key instead of `path`.
|
|
20517
|
+
*/
|
|
20518
|
+
async function handleVideosetMetadata(ctx) {
|
|
20519
|
+
const key = entryKeyOf(ctx.sourcePath);
|
|
20520
|
+
if (key === undefined) {
|
|
20521
|
+
return {
|
|
20522
|
+
success: false,
|
|
20523
|
+
errorMessage: `Expected a video set entry at ${ctx.sourcePath}`
|
|
20524
|
+
};
|
|
20525
|
+
}
|
|
20526
|
+
const absolute = path__default.join(ctx.projectRoot, key);
|
|
20527
|
+
if (!galleryEntryOf(key).remote && !ctx.fs.fileExists(absolute)) {
|
|
17934
20528
|
return {
|
|
17935
|
-
|
|
17936
|
-
|
|
20529
|
+
success: false,
|
|
20530
|
+
errorMessage: `File ${absolute} does not exist`
|
|
20531
|
+
};
|
|
20532
|
+
}
|
|
20533
|
+
const entry = ctx.validationError.value;
|
|
20534
|
+
const missing = ["width", "height", "duration"].filter(field => !isRecord$1(entry) || entry[field] === undefined);
|
|
20535
|
+
if (missing.length > 0 && !canReadVideoMetadata(key)) {
|
|
20536
|
+
return {
|
|
20537
|
+
success: false,
|
|
20538
|
+
errorMessage: unreadableVideoMetadataMessage(key, missing)
|
|
17937
20539
|
};
|
|
17938
20540
|
}
|
|
17939
20541
|
return {
|
|
17940
|
-
|
|
17941
|
-
|
|
20542
|
+
success: true,
|
|
20543
|
+
shouldApplyPatch: true
|
|
17942
20544
|
};
|
|
17943
20545
|
}
|
|
17944
20546
|
|
|
17945
20547
|
/**
|
|
17946
|
-
*
|
|
17947
|
-
*
|
|
17948
|
-
* There is
|
|
17949
|
-
* it
|
|
20548
|
+
* `videos:check-remote`: a key that is wrong as a key — a remote URL in a set
|
|
20549
|
+
* that is not `.remote()`, one that does not parse, one outside the set's
|
|
20550
|
+
* directory. There is nothing to look up and nothing to move, so the error is
|
|
20551
|
+
* reported as it is, for a person: only they know which of the set's options
|
|
20552
|
+
* or the key is the mistake.
|
|
17950
20553
|
*/
|
|
17951
|
-
function
|
|
17952
|
-
|
|
17953
|
-
|
|
17954
|
-
|
|
17955
|
-
}
|
|
17956
|
-
return maybePath;
|
|
20554
|
+
async function handleVideosetCheckRemote(ctx) {
|
|
20555
|
+
return {
|
|
20556
|
+
success: false,
|
|
20557
|
+
errorMessage: ctx.validationError.message
|
|
20558
|
+
};
|
|
17957
20559
|
}
|
|
17958
20560
|
|
|
17959
|
-
|
|
17960
|
-
|
|
17961
|
-
|
|
17962
|
-
|
|
17963
|
-
|
|
20561
|
+
/**
|
|
20562
|
+
* `videos:upload-remote`: an entry of a `.remote()` set still keyed by a local
|
|
20563
|
+
* path. Every file it holds goes up — a stream bottom-up, as the Studio places
|
|
20564
|
+
* one — and `createFixPatch` renames the key to the ref, the same way
|
|
20565
|
+
* `images:upload-remote` does. The fields that name the entry by its old key
|
|
20566
|
+
* are rewritten too: without that they name a video the set no longer has.
|
|
20567
|
+
*
|
|
20568
|
+
* The entry's poster and captions — its defaults' files — go up with it, and
|
|
20569
|
+
* are what is left to do for an entry whose key is already remote: the key
|
|
20570
|
+
* stays, and the entry is written back with their refs.
|
|
20571
|
+
*/
|
|
20572
|
+
async function handleVideosetUploadRemote(ctx) {
|
|
20573
|
+
if (!ctx.fix) {
|
|
20574
|
+
return {
|
|
20575
|
+
success: false,
|
|
20576
|
+
// No sourcePath in the message: the reported location already points at it.
|
|
20577
|
+
errorMessage: `Video needs to be uploaded to Val Remote (use --fix to upload)`
|
|
20578
|
+
};
|
|
17964
20579
|
}
|
|
17965
|
-
const
|
|
17966
|
-
if (
|
|
20580
|
+
const key = ctx.validationError.value;
|
|
20581
|
+
if (typeof key !== "string") {
|
|
17967
20582
|
return {
|
|
17968
|
-
|
|
17969
|
-
|
|
20583
|
+
success: false,
|
|
20584
|
+
errorMessage: `Expected a local file path for the video set entry at ${ctx.sourcePath}`
|
|
17970
20585
|
};
|
|
17971
20586
|
}
|
|
17972
|
-
|
|
17973
|
-
if (!relativeFilePath) {
|
|
20587
|
+
if (ctx.remoteFiles[ctx.sourcePath]) {
|
|
17974
20588
|
return {
|
|
17975
|
-
|
|
17976
|
-
|
|
20589
|
+
success: true,
|
|
20590
|
+
shouldApplyPatch: true,
|
|
20591
|
+
events: [{
|
|
20592
|
+
type: "remote-already-uploaded",
|
|
20593
|
+
filePath: key
|
|
20594
|
+
}]
|
|
17977
20595
|
};
|
|
17978
20596
|
}
|
|
17979
|
-
|
|
20597
|
+
const set = resolveVideoset(ctx.valModule, Internal.parentOfSourcePath(ctx.sourcePath));
|
|
20598
|
+
if (!set.success) {
|
|
20599
|
+
return set;
|
|
20600
|
+
}
|
|
20601
|
+
const entry = set.entries[key];
|
|
20602
|
+
if (!isRecord$1(entry)) {
|
|
17980
20603
|
return {
|
|
17981
|
-
|
|
17982
|
-
|
|
20604
|
+
success: false,
|
|
20605
|
+
errorMessage: `The video set has no entry '${key}' at ${ctx.sourcePath}`
|
|
17983
20606
|
};
|
|
17984
20607
|
}
|
|
17985
|
-
|
|
17986
|
-
|
|
17987
|
-
const
|
|
17988
|
-
|
|
17989
|
-
|
|
17990
|
-
|
|
17991
|
-
|
|
17992
|
-
|
|
17993
|
-
|
|
17994
|
-
|
|
17995
|
-
|
|
17996
|
-
|
|
17997
|
-
|
|
17998
|
-
// (or if the way it is computed has changed)
|
|
17999
|
-
validationHashFromRemoteRefValues === remoteRefDataRes.validationHash) {
|
|
20608
|
+
// The key is local (the entry goes up whole), or the key is already on the
|
|
20609
|
+
// content host and only the entry's poster or captions are still here.
|
|
20610
|
+
const keyIsLocal = !Internal.isRemoteMediaPath(key);
|
|
20611
|
+
const entryFiles = localFilesOfEntry(entry);
|
|
20612
|
+
const missing = firstMissingVideoFile(ctx, [...(keyIsLocal ? [{
|
|
20613
|
+
path: key,
|
|
20614
|
+
...(typeof entry.mimeType === "string" ? {
|
|
20615
|
+
mimeType: entry.mimeType
|
|
20616
|
+
} : {})
|
|
20617
|
+
}] : []), ...entryFiles.map(file => ({
|
|
20618
|
+
path: file.path
|
|
20619
|
+
}))]);
|
|
20620
|
+
if (missing !== undefined) {
|
|
18000
20621
|
return {
|
|
18001
|
-
|
|
20622
|
+
success: false,
|
|
20623
|
+
errorMessage: `File ${missing} does not exist`
|
|
18002
20624
|
};
|
|
18003
20625
|
}
|
|
18004
|
-
|
|
18005
|
-
|
|
18006
|
-
|
|
18007
|
-
// We could not download the remote file for some reason (for example, the file hash is wrong)
|
|
18008
|
-
if (fileBufferRes.status === "error") {
|
|
18009
|
-
return fileBufferRes;
|
|
20626
|
+
const opened = await openRemoteUploadSession(ctx);
|
|
20627
|
+
if (!opened.success) {
|
|
20628
|
+
return opened.result;
|
|
18010
20629
|
}
|
|
18011
|
-
|
|
18012
|
-
|
|
18013
|
-
|
|
18014
|
-
|
|
18015
|
-
let
|
|
20630
|
+
const {
|
|
20631
|
+
session
|
|
20632
|
+
} = opened;
|
|
20633
|
+
const uploader = createVideoUploader(ctx, session, videosetEntryVideoSchema(set.record), (localRef, role) => metadataOf(localRef, role, entry));
|
|
20634
|
+
let ref;
|
|
18016
20635
|
try {
|
|
18017
|
-
|
|
18018
|
-
|
|
18019
|
-
|
|
18020
|
-
currentMetadata = await extractFileMetadata(remoteRefDataRes.filePath, fileBufferRes.fileBuffer);
|
|
18021
|
-
} else {
|
|
18022
|
-
const exhaustiveCheck = schema;
|
|
18023
|
-
return {
|
|
18024
|
-
status: "error",
|
|
18025
|
-
error: `Unknown schema type: ${JSON.stringify(exhaustiveCheck)}`
|
|
18026
|
-
};
|
|
20636
|
+
ref = keyIsLocal ? await uploader.put(key, "video") : key;
|
|
20637
|
+
for (const file of entryFiles) {
|
|
20638
|
+
await uploader.put(file.path, file.role);
|
|
18027
20639
|
}
|
|
18028
20640
|
} catch (err) {
|
|
18029
20641
|
return {
|
|
18030
|
-
|
|
18031
|
-
|
|
20642
|
+
success: false,
|
|
20643
|
+
errorMessage: err instanceof Error ? err.message : String(err)
|
|
18032
20644
|
};
|
|
18033
20645
|
}
|
|
18034
|
-
|
|
18035
|
-
|
|
18036
|
-
|
|
20646
|
+
ctx.remoteFiles[ctx.sourcePath] = {
|
|
20647
|
+
ref,
|
|
20648
|
+
metadata: withMovedFiles(entry, uploader.moved),
|
|
20649
|
+
refs: Object.fromEntries(uploader.moved)
|
|
18037
20650
|
};
|
|
18038
|
-
const
|
|
18039
|
-
|
|
20651
|
+
const otherModulePatches = ref === key ? [] : await videosetReferencePatches(ctx.service, ctx.moduleFilePath, {
|
|
20652
|
+
[key]: ref
|
|
20653
|
+
});
|
|
20654
|
+
return {
|
|
20655
|
+
success: true,
|
|
20656
|
+
shouldApplyPatch: true,
|
|
20657
|
+
publicProjectId: session.publicProjectId,
|
|
20658
|
+
remoteFileBuckets: session.remoteFileBuckets,
|
|
20659
|
+
remoteFilesCounter: session.remoteFilesCounter,
|
|
20660
|
+
events: uploader.events,
|
|
20661
|
+
...(otherModulePatches.length > 0 ? {
|
|
20662
|
+
otherModulePatches
|
|
20663
|
+
} : {})
|
|
20664
|
+
};
|
|
20665
|
+
}
|
|
20666
|
+
|
|
20667
|
+
/**
|
|
20668
|
+
* `videos:check-all-files`: the set's entries against its directory.
|
|
20669
|
+
*
|
|
20670
|
+
* What only this handler can answer is which untracked files are accounted
|
|
20671
|
+
* for ELSEWHERE — a poster or caption a video field names — because that needs
|
|
20672
|
+
* every module in the project. Those are fine. An untracked file that is not
|
|
20673
|
+
* a video and that no field names is for a person to look at, and is reported
|
|
20674
|
+
* here. Everything `--fix` can do something about — an entry whose file has
|
|
20675
|
+
* gone, a video in the directory the set does not list — is left to
|
|
20676
|
+
* `createFixPatch` (see {@link videosetCheckAllFilesPatch}), which reports it
|
|
20677
|
+
* per entry, or repairs it.
|
|
20678
|
+
*/
|
|
20679
|
+
async function handleVideosetCheckAllFiles(ctx) {
|
|
20680
|
+
const value = ctx.validationError.value;
|
|
20681
|
+
const dir = isRecord$1(value) ? value.dir : undefined;
|
|
20682
|
+
if (typeof dir !== "string") {
|
|
18040
20683
|
return {
|
|
18041
|
-
|
|
18042
|
-
|
|
20684
|
+
success: false,
|
|
20685
|
+
errorMessage: `Unexpected value in check-all-files for ${ctx.sourcePath}`
|
|
18043
20686
|
};
|
|
18044
20687
|
}
|
|
18045
|
-
const
|
|
18046
|
-
|
|
18047
|
-
|
|
18048
|
-
|
|
18049
|
-
|
|
18050
|
-
|
|
18051
|
-
|
|
18052
|
-
|
|
18053
|
-
|
|
18054
|
-
|
|
18055
|
-
|
|
18056
|
-
|
|
20688
|
+
const set = resolveVideoset(ctx.valModule, ctx.sourcePath);
|
|
20689
|
+
if (!set.success) {
|
|
20690
|
+
return set;
|
|
20691
|
+
}
|
|
20692
|
+
const readFile = absolute => ctx.fs.readBuffer(absolute);
|
|
20693
|
+
const incompleteEntries = incompleteGalleryEntries({
|
|
20694
|
+
entryKeys: Object.keys(set.entries),
|
|
20695
|
+
projectRoot: ctx.projectRoot,
|
|
20696
|
+
fs: ctx.fs,
|
|
20697
|
+
filesOfEntry: (entry, key) => filesOfVideosetEntry(entry, set.entries[key], {
|
|
20698
|
+
projectRoot: ctx.projectRoot,
|
|
20699
|
+
readFile
|
|
20700
|
+
})
|
|
20701
|
+
});
|
|
20702
|
+
const {
|
|
20703
|
+
untrackedFiles
|
|
20704
|
+
} = checkGalleryFiles({
|
|
20705
|
+
entryKeys: Object.keys(set.entries),
|
|
20706
|
+
dir,
|
|
20707
|
+
projectRoot: ctx.projectRoot,
|
|
20708
|
+
fs: ctx.fs,
|
|
20709
|
+
filesOfEntry: (entry, key) => filesOfVideosetEntry(entry, set.entries[key], {
|
|
20710
|
+
projectRoot: ctx.projectRoot,
|
|
20711
|
+
readFile
|
|
18057
20712
|
})
|
|
20713
|
+
});
|
|
20714
|
+
if (incompleteEntries.length > 0) {
|
|
20715
|
+
return {
|
|
20716
|
+
success: false,
|
|
20717
|
+
errorMessage: incompleteMessage(incompleteEntries)
|
|
20718
|
+
};
|
|
20719
|
+
}
|
|
20720
|
+
if (untrackedFiles.length === 0) {
|
|
20721
|
+
return {
|
|
20722
|
+
success: true,
|
|
20723
|
+
shouldApplyPatch: true
|
|
20724
|
+
};
|
|
20725
|
+
}
|
|
20726
|
+
const named = await filesNamedByVideoFields(ctx.service, {
|
|
20727
|
+
projectRoot: ctx.projectRoot,
|
|
20728
|
+
readFile
|
|
20729
|
+
});
|
|
20730
|
+
const {
|
|
20731
|
+
others
|
|
20732
|
+
} = classifyUntrackedVideosetFiles({
|
|
20733
|
+
untracked: untrackedFiles.filter(file => !named.has(file)),
|
|
20734
|
+
accept: set.record.accept ?? "video/*",
|
|
20735
|
+
projectRoot: ctx.projectRoot,
|
|
20736
|
+
readFile
|
|
20737
|
+
});
|
|
20738
|
+
if (others.length > 0) {
|
|
20739
|
+
return {
|
|
20740
|
+
success: false,
|
|
20741
|
+
errorMessage: `Video set in ${ctx.moduleFilePath} has files in '${dir}' that are not in the set, and that no video field uses: ${others.join(", ")}. They are not videos the set accepts, so --fix will not add them: move them out of the directory or remove them.`
|
|
20742
|
+
};
|
|
20743
|
+
}
|
|
20744
|
+
return {
|
|
20745
|
+
success: true,
|
|
20746
|
+
shouldApplyPatch: true
|
|
18058
20747
|
};
|
|
18059
20748
|
}
|
|
18060
|
-
|
|
18061
|
-
|
|
18062
|
-
|
|
18063
|
-
|
|
18064
|
-
|
|
18065
|
-
|
|
18066
|
-
|
|
18067
|
-
|
|
18068
|
-
|
|
18069
|
-
|
|
18070
|
-
|
|
18071
|
-
|
|
18072
|
-
|
|
18073
|
-
|
|
18074
|
-
|
|
18075
|
-
|
|
18076
|
-
|
|
18077
|
-
|
|
20749
|
+
|
|
20750
|
+
/**
|
|
20751
|
+
* The `createFixPatch` half of `videos:check-all-files`: drop the entries
|
|
20752
|
+
* whose file has gone, and add the videos in the directory that the set does
|
|
20753
|
+
* not list — with what is read from their bytes, or not at all.
|
|
20754
|
+
*
|
|
20755
|
+
* What is stored for an entry that IS on disk is not compared against its
|
|
20756
|
+
* bytes, unlike an image gallery's. The Studio reads a video's size and length
|
|
20757
|
+
* in the browser from the file as it was picked, and that is not what a header
|
|
20758
|
+
* parse of what is on disk says: a stream's largest rendition is not the
|
|
20759
|
+
* upload's size whenever the upload was larger than the largest rendition, and
|
|
20760
|
+
* two readers round a duration differently. Correcting one from the other
|
|
20761
|
+
* would rewrite every streamed video in the project, every run. A missing
|
|
20762
|
+
* value is `videos:add-metadata`'s, and is reported on the entry.
|
|
20763
|
+
*/
|
|
20764
|
+
async function videosetCheckAllFilesPatch({
|
|
20765
|
+
projectRoot,
|
|
20766
|
+
apply,
|
|
20767
|
+
sourcePath,
|
|
20768
|
+
validationError,
|
|
20769
|
+
moduleSource,
|
|
20770
|
+
moduleSchema
|
|
20771
|
+
}) {
|
|
20772
|
+
const patch = [];
|
|
20773
|
+
const remainingErrors = [];
|
|
20774
|
+
const fail = message => ({
|
|
20775
|
+
patch,
|
|
20776
|
+
remainingErrors: [{
|
|
20777
|
+
...validationError,
|
|
20778
|
+
message,
|
|
20779
|
+
fixes: undefined
|
|
20780
|
+
}]
|
|
20781
|
+
});
|
|
20782
|
+
const value = validationError.value;
|
|
20783
|
+
const dir = isRecord$1(value) ? value.dir : undefined;
|
|
20784
|
+
if (typeof dir !== "string") {
|
|
20785
|
+
return fail(`Unexpected value in check-all-files for ${sourcePath}`);
|
|
20786
|
+
}
|
|
20787
|
+
if (moduleSource === undefined || moduleSchema === undefined) {
|
|
20788
|
+
return fail("Unexpected error while checking a video set (no module source or schema)");
|
|
20789
|
+
}
|
|
20790
|
+
const set = resolveVideoset({
|
|
20791
|
+
source: moduleSource,
|
|
20792
|
+
schema: moduleSchema
|
|
20793
|
+
}, sourcePath);
|
|
20794
|
+
if (!set.success) {
|
|
20795
|
+
return fail(set.errorMessage);
|
|
20796
|
+
}
|
|
20797
|
+
const recordPath = patchPathOf(sourcePath);
|
|
20798
|
+
const incompleteEntries = incompleteGalleryEntries({
|
|
20799
|
+
entryKeys: Object.keys(set.entries),
|
|
20800
|
+
projectRoot,
|
|
20801
|
+
fs: ts.sys,
|
|
20802
|
+
filesOfEntry: (entry, key) => filesOfVideosetEntry(entry, set.entries[key], {
|
|
20803
|
+
projectRoot
|
|
20804
|
+
})
|
|
20805
|
+
});
|
|
20806
|
+
const {
|
|
20807
|
+
missingTrackedFiles,
|
|
20808
|
+
untrackedFiles
|
|
20809
|
+
} = checkGalleryFiles({
|
|
20810
|
+
entryKeys: Object.keys(set.entries),
|
|
20811
|
+
dir,
|
|
20812
|
+
projectRoot,
|
|
20813
|
+
// The host `createDefaultValFSHost` gives the handlers, and so the same
|
|
20814
|
+
// listing the handler half of this check saw.
|
|
20815
|
+
fs: ts.sys,
|
|
20816
|
+
filesOfEntry: (entry, key) => filesOfVideosetEntry(entry, set.entries[key], {
|
|
20817
|
+
projectRoot
|
|
20818
|
+
})
|
|
20819
|
+
});
|
|
20820
|
+
if (incompleteEntries.length > 0) {
|
|
20821
|
+
remainingErrors.push({
|
|
20822
|
+
...validationError,
|
|
20823
|
+
message: incompleteMessage(incompleteEntries),
|
|
20824
|
+
fixes: undefined
|
|
20825
|
+
});
|
|
20826
|
+
}
|
|
20827
|
+
for (const missing of missingTrackedFiles) {
|
|
20828
|
+
if (apply) {
|
|
20829
|
+
const removePath = recordPath.concat(missing);
|
|
20830
|
+
if (isNotRoot(removePath)) {
|
|
20831
|
+
patch.push({
|
|
20832
|
+
op: "remove",
|
|
20833
|
+
path: removePath
|
|
20834
|
+
});
|
|
20835
|
+
}
|
|
20836
|
+
} else {
|
|
20837
|
+
remainingErrors.push({
|
|
20838
|
+
...validationError,
|
|
20839
|
+
message: `Video '${missing}' is in the set, but its file does not exist. Use --fix to remove it from the set.`,
|
|
20840
|
+
sourcePath: Internal.createValPathOfItem(sourcePath, missing),
|
|
20841
|
+
keyError: true
|
|
20842
|
+
});
|
|
18078
20843
|
}
|
|
18079
|
-
}
|
|
20844
|
+
}
|
|
20845
|
+
const {
|
|
20846
|
+
videos
|
|
20847
|
+
} = classifyUntrackedVideosetFiles({
|
|
20848
|
+
untracked: untrackedFiles,
|
|
20849
|
+
accept: set.record.accept ?? "video/*",
|
|
20850
|
+
projectRoot
|
|
20851
|
+
});
|
|
20852
|
+
for (const file of videos) {
|
|
20853
|
+
let metadata;
|
|
18080
20854
|
try {
|
|
18081
|
-
await
|
|
18082
|
-
|
|
20855
|
+
metadata = await extractVideoMetadataFromFile(path__default.join(projectRoot, file));
|
|
20856
|
+
} catch (err) {
|
|
20857
|
+
remainingErrors.push({
|
|
20858
|
+
...validationError,
|
|
20859
|
+
message: `Video '${file}' is in '${dir}' but not in the set, and its metadata could not be read: ${err instanceof Error ? err.message : String(err)}`,
|
|
20860
|
+
fixes: undefined
|
|
20861
|
+
});
|
|
20862
|
+
continue;
|
|
20863
|
+
}
|
|
20864
|
+
const unreadable = VIDEO_METADATA_FIELDS$1.filter(field => metadata[field] === undefined);
|
|
20865
|
+
if (unreadable.length > 0) {
|
|
20866
|
+
// All or nothing, as for `video:add-metadata`: an entry with a mime type
|
|
20867
|
+
// and no size is no closer to valid, and reads as one someone finished.
|
|
20868
|
+
remainingErrors.push({
|
|
20869
|
+
...validationError,
|
|
20870
|
+
message: `Video '${file}' is in '${dir}' but not in the set. ${unreadableVideoMetadataMessage(file, unreadable)}`,
|
|
20871
|
+
fixes: undefined
|
|
20872
|
+
});
|
|
20873
|
+
continue;
|
|
20874
|
+
}
|
|
20875
|
+
if (!apply) {
|
|
20876
|
+
remainingErrors.push({
|
|
20877
|
+
...validationError,
|
|
20878
|
+
message: `Video '${file}' is in '${dir}' but not in the set. Use --fix to add it.`
|
|
20879
|
+
});
|
|
20880
|
+
continue;
|
|
20881
|
+
}
|
|
20882
|
+
const entry = {};
|
|
20883
|
+
for (const field of VIDEO_METADATA_FIELDS$1) {
|
|
20884
|
+
const read = metadata[field];
|
|
20885
|
+
if (read !== undefined) {
|
|
20886
|
+
entry[field] = read;
|
|
20887
|
+
}
|
|
20888
|
+
}
|
|
20889
|
+
// What an upload writes when nobody has described the video yet.
|
|
20890
|
+
entry.alt = null;
|
|
20891
|
+
const addPath = recordPath.concat(file);
|
|
20892
|
+
if (isNotRoot(addPath)) {
|
|
20893
|
+
patch.push({
|
|
20894
|
+
op: "add",
|
|
20895
|
+
path: addPath,
|
|
20896
|
+
value: entry
|
|
20897
|
+
});
|
|
20898
|
+
}
|
|
20899
|
+
}
|
|
20900
|
+
return {
|
|
20901
|
+
patch,
|
|
20902
|
+
remainingErrors
|
|
20903
|
+
};
|
|
20904
|
+
}
|
|
20905
|
+
|
|
20906
|
+
/**
|
|
20907
|
+
* The `createFixPatch` half of `videos:add-metadata`: read what is missing
|
|
20908
|
+
* from the file at the entry's KEY, and write only that, one op per field —
|
|
20909
|
+
* `alt` and anything already there are someone's, and stay.
|
|
20910
|
+
*/
|
|
20911
|
+
async function videosetAddMetadataPatch({
|
|
20912
|
+
projectRoot,
|
|
20913
|
+
sourcePath,
|
|
20914
|
+
validationError,
|
|
20915
|
+
moduleSource,
|
|
20916
|
+
moduleSchema
|
|
20917
|
+
}) {
|
|
20918
|
+
const fail = message => ({
|
|
20919
|
+
patch: [],
|
|
20920
|
+
remainingErrors: [{
|
|
20921
|
+
...validationError,
|
|
20922
|
+
message,
|
|
20923
|
+
fixes: undefined
|
|
20924
|
+
}]
|
|
20925
|
+
});
|
|
20926
|
+
const key = entryKeyOf(sourcePath);
|
|
20927
|
+
if (key === undefined) {
|
|
20928
|
+
return fail(`Expected a video set entry at ${sourcePath}`);
|
|
20929
|
+
}
|
|
20930
|
+
let current = validationError.value;
|
|
20931
|
+
if (moduleSource !== undefined && moduleSchema !== undefined) {
|
|
20932
|
+
const parent = resolveVideoset({
|
|
20933
|
+
source: moduleSource,
|
|
20934
|
+
schema: moduleSchema
|
|
20935
|
+
}, Internal.parentOfSourcePath(sourcePath));
|
|
20936
|
+
if (!parent.success) {
|
|
20937
|
+
return fail(`Could not add video metadata: ${parent.errorMessage}`);
|
|
20938
|
+
}
|
|
20939
|
+
current = parent.entries[key];
|
|
20940
|
+
}
|
|
20941
|
+
if (!isRecord$1(current)) {
|
|
20942
|
+
return fail("Video set entry is not an object!");
|
|
20943
|
+
}
|
|
20944
|
+
let metadata;
|
|
20945
|
+
try {
|
|
20946
|
+
metadata = galleryEntryOf(key).remote ? await extractVideoMetadataFromUrl(key, nameForTypeOf(key)) : await extractVideoMetadataFromFile(path__default.join(projectRoot, key));
|
|
20947
|
+
} catch (err) {
|
|
20948
|
+
return fail(`Failed to read video metadata from ${key}: ${err instanceof Error ? err.message : String(err)}`);
|
|
20949
|
+
}
|
|
20950
|
+
const entry = current;
|
|
20951
|
+
const missing = VIDEO_METADATA_FIELDS$1.filter(field => entry[field] === undefined);
|
|
20952
|
+
const unreadable = missing.filter(field => metadata[field] === undefined);
|
|
20953
|
+
if (unreadable.length > 0) {
|
|
20954
|
+
return fail(unreadableVideoMetadataMessage(key, unreadable));
|
|
20955
|
+
}
|
|
20956
|
+
const entryPath = patchPathOf(sourcePath);
|
|
20957
|
+
const patch = [];
|
|
20958
|
+
for (const field of missing) {
|
|
20959
|
+
const read = metadata[field];
|
|
20960
|
+
if (read !== undefined) {
|
|
20961
|
+
patch.push({
|
|
20962
|
+
op: "add",
|
|
20963
|
+
path: entryPath.concat(field),
|
|
20964
|
+
value: read
|
|
18083
20965
|
});
|
|
18084
|
-
await downloadFileFromRemote(ref, remoteFilePath);
|
|
18085
|
-
} catch (err) {
|
|
18086
|
-
try {
|
|
18087
|
-
// try to delete in case of partial download
|
|
18088
|
-
await fs.promises.unlink(remoteFilePath);
|
|
18089
|
-
} catch {
|
|
18090
|
-
//
|
|
18091
|
-
}
|
|
18092
|
-
return {
|
|
18093
|
-
status: "error",
|
|
18094
|
-
error: `Failed to download remote file: ${err instanceof Error ? err.message : "Unknown error"}`
|
|
18095
|
-
};
|
|
18096
20966
|
}
|
|
18097
20967
|
}
|
|
18098
|
-
const fileBuffer = await fs.promises.readFile(remoteFilePath);
|
|
18099
|
-
const computedFileHash = Internal.remote.getFileHash(fileBuffer);
|
|
18100
20968
|
return {
|
|
18101
|
-
|
|
18102
|
-
|
|
18103
|
-
fileHash: computedFileHash,
|
|
18104
|
-
cachedFilePath: remoteFilePath
|
|
20969
|
+
patch,
|
|
20970
|
+
remainingErrors: []
|
|
18105
20971
|
};
|
|
18106
20972
|
}
|
|
18107
|
-
|
|
18108
|
-
|
|
18109
|
-
|
|
18110
|
-
|
|
18111
|
-
|
|
18112
|
-
|
|
18113
|
-
|
|
18114
|
-
|
|
18115
|
-
|
|
18116
|
-
|
|
18117
|
-
|
|
18118
|
-
}
|
|
18119
|
-
if (response.statusCode !== 200) {
|
|
18120
|
-
return reject(new Error(`${ref}. Failed to download file. HTTP Status: ${response.statusCode}`));
|
|
18121
|
-
}
|
|
18122
|
-
const writeStream = fs.createWriteStream(filePath);
|
|
18123
|
-
response.pipe(writeStream);
|
|
18124
|
-
writeStream.on("finish", () => resolve({
|
|
18125
|
-
status: "success",
|
|
18126
|
-
filePath
|
|
18127
|
-
}));
|
|
18128
|
-
writeStream.on("error", err => reject(new Error(`${ref}. Error writing file: ${err.message}`)));
|
|
18129
|
-
});
|
|
18130
|
-
request.on("error", err => reject(new Error(`${ref}. Download error: ${err.message}`)));
|
|
18131
|
-
request.end();
|
|
18132
|
-
});
|
|
20973
|
+
|
|
20974
|
+
/**
|
|
20975
|
+
* What to say about streams whose master is there and some of whose files are
|
|
20976
|
+
* not. Not fixable: the entry still names a video someone uploaded, and only
|
|
20977
|
+
* they have the rest of it.
|
|
20978
|
+
*/
|
|
20979
|
+
function incompleteMessage(incomplete) {
|
|
20980
|
+
return incomplete.map(({
|
|
20981
|
+
key,
|
|
20982
|
+
missing
|
|
20983
|
+
}) => `Video '${key}' is missing ${missing.length === 1 ? "a file it names" : `${missing.length} files it names`}: ${missing.join(", ")}. It stops playing where they are. Upload it again, or put the files back.`).join(" ");
|
|
18133
20984
|
}
|
|
18134
20985
|
|
|
18135
20986
|
// A remaining error may optionally carry a more specific `sourcePath` than the
|
|
@@ -18295,6 +21146,86 @@ async function createFixPatch(config, apply, sourcePath, validationError, remote
|
|
|
18295
21146
|
value: fileMetadata.mimeType ?? null
|
|
18296
21147
|
});
|
|
18297
21148
|
}
|
|
21149
|
+
} else if (fix === "video:add-metadata") {
|
|
21150
|
+
// Only the fields that are MISSING, one op each: a video's `alt`,
|
|
21151
|
+
// `poster`, `captions` and times are authored and sit in the same
|
|
21152
|
+
// object, and a field already there is not re-read (see the comment on
|
|
21153
|
+
// the fix code in core — reading a video is not cheap).
|
|
21154
|
+
const current = currentVideoValue(sourcePath, validationError, moduleSource, moduleSchema);
|
|
21155
|
+
if (current.status === "error") {
|
|
21156
|
+
remainingErrors.push({
|
|
21157
|
+
...validationError,
|
|
21158
|
+
message: current.message,
|
|
21159
|
+
fixes: undefined
|
|
21160
|
+
});
|
|
21161
|
+
continue;
|
|
21162
|
+
}
|
|
21163
|
+
const fileRef = current.value.path;
|
|
21164
|
+
if (typeof fileRef !== "string") {
|
|
21165
|
+
remainingErrors.push({
|
|
21166
|
+
...validationError,
|
|
21167
|
+
message: `Cannot read video metadata: the video has no path`,
|
|
21168
|
+
fixes: undefined
|
|
21169
|
+
});
|
|
21170
|
+
continue;
|
|
21171
|
+
}
|
|
21172
|
+
let metadata;
|
|
21173
|
+
try {
|
|
21174
|
+
metadata = await getVideoMetadata(config.projectRoot, fileRef);
|
|
21175
|
+
} catch (err) {
|
|
21176
|
+
remainingErrors.push({
|
|
21177
|
+
...validationError,
|
|
21178
|
+
message: `Failed to read video metadata from ${fileRef}: ${err instanceof Error ? err.message : String(err)}`,
|
|
21179
|
+
fixes: undefined
|
|
21180
|
+
});
|
|
21181
|
+
continue;
|
|
21182
|
+
}
|
|
21183
|
+
const missing = VIDEO_METADATA_FIELDS.filter(field => current.value[field] === undefined);
|
|
21184
|
+
const unreadable = missing.filter(field => metadata[field] === undefined);
|
|
21185
|
+
if (unreadable.length > 0) {
|
|
21186
|
+
// All or nothing: a video with a mime type but no size is no closer
|
|
21187
|
+
// to valid, and a half-written value reads as one someone finished.
|
|
21188
|
+
remainingErrors.push({
|
|
21189
|
+
...validationError,
|
|
21190
|
+
message: unreadableVideoMetadataMessage(fileRef, unreadable),
|
|
21191
|
+
fixes: undefined
|
|
21192
|
+
});
|
|
21193
|
+
continue;
|
|
21194
|
+
}
|
|
21195
|
+
const patchPath = sourceToPatchPath(sourcePath);
|
|
21196
|
+
for (const field of missing) {
|
|
21197
|
+
const value = metadata[field];
|
|
21198
|
+
if (value !== undefined) {
|
|
21199
|
+
patch.push({
|
|
21200
|
+
op: "add",
|
|
21201
|
+
path: patchPath.concat(field),
|
|
21202
|
+
value
|
|
21203
|
+
});
|
|
21204
|
+
}
|
|
21205
|
+
}
|
|
21206
|
+
} else if (fix === "videos:add-metadata") {
|
|
21207
|
+
// A set entry's twin of `video:add-metadata`: the file is the entry's
|
|
21208
|
+
// KEY, and only what is missing is written.
|
|
21209
|
+
const fixed = await videosetAddMetadataPatch({
|
|
21210
|
+
projectRoot: config.projectRoot,
|
|
21211
|
+
sourcePath,
|
|
21212
|
+
validationError,
|
|
21213
|
+
moduleSource,
|
|
21214
|
+
moduleSchema
|
|
21215
|
+
});
|
|
21216
|
+
patch.push(...fixed.patch);
|
|
21217
|
+
remainingErrors.push(...fixed.remainingErrors);
|
|
21218
|
+
} else if (fix === "videos:check-all-files") {
|
|
21219
|
+
const fixed = await videosetCheckAllFilesPatch({
|
|
21220
|
+
projectRoot: config.projectRoot,
|
|
21221
|
+
apply,
|
|
21222
|
+
sourcePath,
|
|
21223
|
+
validationError,
|
|
21224
|
+
moduleSource,
|
|
21225
|
+
moduleSchema
|
|
21226
|
+
});
|
|
21227
|
+
patch.push(...fixed.patch);
|
|
21228
|
+
remainingErrors.push(...fixed.remainingErrors);
|
|
18298
21229
|
} else if (fix === "image:upload-remote" || fix === "file:upload-remote") {
|
|
18299
21230
|
const remoteFile = remoteFiles[sourcePath];
|
|
18300
21231
|
let metadata = remoteFile.metadata;
|
|
@@ -18324,10 +21255,30 @@ async function createFixPatch(config, apply, sourcePath, validationError, remote
|
|
|
18324
21255
|
path: sourceToPatchPath(sourcePath)
|
|
18325
21256
|
});
|
|
18326
21257
|
}
|
|
18327
|
-
} else if (fix === "
|
|
21258
|
+
} else if (fix === "video:upload-remote" || fix === "video:download-remote") {
|
|
21259
|
+
var _remoteFiles$sourcePa;
|
|
21260
|
+
// The handler moved the files (`videoRemote.ts`); what is left is to
|
|
21261
|
+
// point each path the value holds at where its file now is. Read from
|
|
21262
|
+
// the module rather than the error, which carries the value as it was
|
|
21263
|
+
// validated.
|
|
21264
|
+
const moved = (_remoteFiles$sourcePa = remoteFiles[sourcePath]) === null || _remoteFiles$sourcePa === void 0 ? void 0 : _remoteFiles$sourcePa.refs;
|
|
21265
|
+
if (!moved) {
|
|
21266
|
+
remainingErrors.push({
|
|
21267
|
+
...validationError,
|
|
21268
|
+
message: fix === "video:upload-remote" ? "Cannot point the video at Val Remote: its files were not uploaded" : "Cannot point the video at its local files: they were not downloaded",
|
|
21269
|
+
fixes: undefined
|
|
21270
|
+
});
|
|
21271
|
+
continue;
|
|
21272
|
+
}
|
|
21273
|
+
const current = moduleSource !== undefined && moduleSchema !== undefined ? Internal.resolvePath(Internal.splitModuleFilePathAndModulePath(sourcePath)[1], moduleSource, moduleSchema).source : validationError.value;
|
|
21274
|
+
patch.push(...rewriteVideoPathsPatch(sourcePath, current, moved));
|
|
21275
|
+
} else if (fix === "images:upload-remote" || fix === "files:upload-remote" || fix === "videos:upload-remote") {
|
|
18328
21276
|
// Gallery entry: the record is keyed by the file path, so uploading to
|
|
18329
21277
|
// remote means renaming the key from the local path to the remote URL
|
|
18330
|
-
// (remove the old key, add the new one with the same metadata).
|
|
21278
|
+
// (remove the old key, add the new one with the same metadata). A video
|
|
21279
|
+
// set's entry is renamed to its MASTER playlist's ref for a stream: the
|
|
21280
|
+
// playlists and segments it names went up with it, and are named by the
|
|
21281
|
+
// uploaded master, not by the set.
|
|
18331
21282
|
const remoteFile = remoteFiles[sourcePath];
|
|
18332
21283
|
if (!remoteFile) {
|
|
18333
21284
|
remainingErrors.push({
|
|
@@ -18686,6 +21637,52 @@ function getRemoteValueFromValidationError(v) {
|
|
|
18686
21637
|
metadata
|
|
18687
21638
|
};
|
|
18688
21639
|
}
|
|
21640
|
+
|
|
21641
|
+
/** The fields of a video that are read from its bytes, in the order written. */
|
|
21642
|
+
const VIDEO_METADATA_FIELDS = ["mimeType", "width", "height", "duration"];
|
|
21643
|
+
|
|
21644
|
+
/**
|
|
21645
|
+
* The video a `video:add-metadata` error is about, read from the module at its
|
|
21646
|
+
* path when the module is at hand — the schema there decides that it IS a
|
|
21647
|
+
* video — and from the error's own value otherwise.
|
|
21648
|
+
*/
|
|
21649
|
+
function currentVideoValue(sourcePath, validationError, moduleSource, moduleSchema) {
|
|
21650
|
+
if (moduleSource !== undefined && moduleSchema !== undefined) {
|
|
21651
|
+
const [, modulePath] = Internal.splitModuleFilePathAndModulePath(sourcePath);
|
|
21652
|
+
const resolved = Internal.resolvePath(modulePath, moduleSource, moduleSchema);
|
|
21653
|
+
if (resolved.schema.type !== "video") {
|
|
21654
|
+
return {
|
|
21655
|
+
status: "error",
|
|
21656
|
+
message: `Could not add video metadata: schema at ${sourcePath} is '${resolved.schema.type}', not a video`
|
|
21657
|
+
};
|
|
21658
|
+
}
|
|
21659
|
+
const value = mediaValueOf(resolved.source);
|
|
21660
|
+
return value ? {
|
|
21661
|
+
status: "ok",
|
|
21662
|
+
value
|
|
21663
|
+
} : {
|
|
21664
|
+
status: "error",
|
|
21665
|
+
message: "Video is not an object!"
|
|
21666
|
+
};
|
|
21667
|
+
}
|
|
21668
|
+
const value = mediaValueOf(validationError.value);
|
|
21669
|
+
return value ? {
|
|
21670
|
+
status: "ok",
|
|
21671
|
+
value
|
|
21672
|
+
} : {
|
|
21673
|
+
status: "error",
|
|
21674
|
+
message: "Video is not an object!"
|
|
21675
|
+
};
|
|
21676
|
+
}
|
|
21677
|
+
async function getVideoMetadata(projectRoot, fileRef) {
|
|
21678
|
+
// Not read into memory: only the headers are, and a video is mostly not
|
|
21679
|
+
// headers. A remote video's headers are read over HTTP, with Range
|
|
21680
|
+
// requests, so it is not downloaded either.
|
|
21681
|
+
if (Internal.isRemoteMediaPath(fileRef)) {
|
|
21682
|
+
return extractVideoMetadataFromUrl(fileRef, nameForTypeOf(fileRef));
|
|
21683
|
+
}
|
|
21684
|
+
return extractVideoMetadataFromFile(path__default.join(projectRoot, fileRef));
|
|
21685
|
+
}
|
|
18689
21686
|
async function getImageMetadata(projectRoot, validationError) {
|
|
18690
21687
|
const fileRef = getValidationErrorFileRef(validationError);
|
|
18691
21688
|
if (!fileRef) {
|
|
@@ -18859,10 +21856,17 @@ function formatOpsError(error, sourceFile) {
|
|
|
18859
21856
|
* What stays in the CLI is the driver: the `runValidation` async generator that
|
|
18860
21857
|
* walks every module, decides what to report, and renders it to a terminal.
|
|
18861
21858
|
*/
|
|
18862
|
-
const textEncoder = new TextEncoder();
|
|
18863
21859
|
|
|
18864
21860
|
// Types for handler system
|
|
18865
21861
|
|
|
21862
|
+
/**
|
|
21863
|
+
* Where a fix moved a media value's file. `refs` is for a value that names
|
|
21864
|
+
* SEVERAL files — a video's poster, captions and stream — keyed by the path
|
|
21865
|
+
* the value held, so the patch can rewrite each one where it is.
|
|
21866
|
+
*/
|
|
21867
|
+
|
|
21868
|
+
/** A patch, and the module it is for. */
|
|
21869
|
+
|
|
18866
21870
|
// Handler functions
|
|
18867
21871
|
async function handleFileMetadata(ctx) {
|
|
18868
21872
|
var _fileSource$source;
|
|
@@ -18896,6 +21900,53 @@ async function handleFileMetadata(ctx) {
|
|
|
18896
21900
|
};
|
|
18897
21901
|
}
|
|
18898
21902
|
|
|
21903
|
+
/**
|
|
21904
|
+
* `video:add-metadata`: the bytes must be on disk, or on Val Remote (read
|
|
21905
|
+
* over HTTP, headers only — see `remoteVideoMetadata.ts`), and they must be a
|
|
21906
|
+
* kind of file Val can read the size and length of here: mp4/mov boxes,
|
|
21907
|
+
* WebM/Matroska headers and HLS playlists. Anything else is refused with what
|
|
21908
|
+
* to do instead, unless all that is missing is the mime type, which the
|
|
21909
|
+
* extension answers.
|
|
21910
|
+
*/
|
|
21911
|
+
async function handleVideoMetadata(ctx) {
|
|
21912
|
+
const exists = isRemoteVideoAt(ctx) ? {
|
|
21913
|
+
success: true,
|
|
21914
|
+
shouldApplyPatch: true
|
|
21915
|
+
} : await handleFileMetadata(ctx);
|
|
21916
|
+
if (!exists.success) {
|
|
21917
|
+
return exists;
|
|
21918
|
+
}
|
|
21919
|
+
const [, modulePath] = Internal.splitModuleFilePathAndModulePath(ctx.sourcePath);
|
|
21920
|
+
if (!ctx.valModule.source || !ctx.valModule.schema) {
|
|
21921
|
+
return exists;
|
|
21922
|
+
}
|
|
21923
|
+
const video = Internal.resolvePath(modulePath, ctx.valModule.source, ctx.valModule.schema).source;
|
|
21924
|
+
if (!isRecord(video) || typeof video.path !== "string") {
|
|
21925
|
+
return exists;
|
|
21926
|
+
}
|
|
21927
|
+
const missing = ["width", "height", "duration"].filter(field => video[field] === undefined);
|
|
21928
|
+
if (missing.length > 0 && !canReadVideoMetadata(video.path)) {
|
|
21929
|
+
return {
|
|
21930
|
+
success: false,
|
|
21931
|
+
errorMessage: unreadableVideoMetadataMessage(video.path, missing)
|
|
21932
|
+
};
|
|
21933
|
+
}
|
|
21934
|
+
return exists;
|
|
21935
|
+
}
|
|
21936
|
+
|
|
21937
|
+
/** Whether the video at the error's path is on Val Remote. */
|
|
21938
|
+
function isRemoteVideoAt(ctx) {
|
|
21939
|
+
if (!ctx.valModule.source || !ctx.valModule.schema) {
|
|
21940
|
+
return false;
|
|
21941
|
+
}
|
|
21942
|
+
const [, modulePath] = Internal.splitModuleFilePathAndModulePath(ctx.sourcePath);
|
|
21943
|
+
const video = Internal.resolvePath(modulePath, ctx.valModule.source, ctx.valModule.schema).source;
|
|
21944
|
+
return isRecord(video) && typeof video.path === "string" && Internal.isRemoteMediaPath(video.path);
|
|
21945
|
+
}
|
|
21946
|
+
function isRecord(value) {
|
|
21947
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
21948
|
+
}
|
|
21949
|
+
|
|
18899
21950
|
// Shared upload core used by both the single-field (handleRemoteFileUpload)
|
|
18900
21951
|
// and gallery (handleRemoteGalleryFileUpload) handlers. The two differ only in
|
|
18901
21952
|
// how they derive the local file ref, metadata and serialized image/file
|
|
@@ -18908,30 +21959,6 @@ async function uploadRemoteFileCore(ctx, fileRef, metadata, schema) {
|
|
|
18908
21959
|
errorMessage: `File ${filePath} does not exist`
|
|
18909
21960
|
};
|
|
18910
21961
|
}
|
|
18911
|
-
const patFile = getPersonalAccessTokenPath(ctx.projectRoot);
|
|
18912
|
-
if (!ctx.fs.fileExists(patFile)) {
|
|
18913
|
-
return {
|
|
18914
|
-
success: false,
|
|
18915
|
-
errorMessage: `File: ${path__default.join(ctx.projectRoot, ctx.file)} has remote images that are not uploaded and you are not logged in.\n\nFix this error by logging in:\n\t"npx val login"\n`
|
|
18916
|
-
};
|
|
18917
|
-
}
|
|
18918
|
-
const patFileContent = ctx.fs.readFile(patFile);
|
|
18919
|
-
if (patFileContent === undefined) {
|
|
18920
|
-
return {
|
|
18921
|
-
success: false,
|
|
18922
|
-
errorMessage: `Could not read personal access token file at ${patFile}`
|
|
18923
|
-
};
|
|
18924
|
-
}
|
|
18925
|
-
const parsedPatFile = parsePersonalAccessTokenFile(patFileContent);
|
|
18926
|
-
if (!parsedPatFile.success) {
|
|
18927
|
-
return {
|
|
18928
|
-
success: false,
|
|
18929
|
-
errorMessage: `Error parsing personal access token file: ${parsedPatFile.error}. You need to login again.`
|
|
18930
|
-
};
|
|
18931
|
-
}
|
|
18932
|
-
const {
|
|
18933
|
-
pat
|
|
18934
|
-
} = parsedPatFile.data;
|
|
18935
21962
|
if (ctx.remoteFiles[ctx.sourcePath]) {
|
|
18936
21963
|
return {
|
|
18937
21964
|
success: true,
|
|
@@ -18941,49 +21968,13 @@ async function uploadRemoteFileCore(ctx, fileRef, metadata, schema) {
|
|
|
18941
21968
|
}]
|
|
18942
21969
|
};
|
|
18943
21970
|
}
|
|
18944
|
-
const
|
|
18945
|
-
|
|
18946
|
-
|
|
18947
|
-
let remoteFilesCounter = ctx.remoteFilesCounter;
|
|
18948
|
-
if (!publicProjectId || !remoteFileBuckets) {
|
|
18949
|
-
if (!projectName) {
|
|
18950
|
-
return {
|
|
18951
|
-
success: false,
|
|
18952
|
-
errorMessage: "Project name not found. Add project name to val.config or set the VAL_PROJECT environment variable"
|
|
18953
|
-
};
|
|
18954
|
-
}
|
|
18955
|
-
const settingsRes = await ctx.remote.getSettings(projectName, {
|
|
18956
|
-
pat
|
|
18957
|
-
});
|
|
18958
|
-
if (!settingsRes.success) {
|
|
18959
|
-
return {
|
|
18960
|
-
success: false,
|
|
18961
|
-
errorMessage: `Could not get public project id: ${settingsRes.message}.`
|
|
18962
|
-
};
|
|
18963
|
-
}
|
|
18964
|
-
publicProjectId = settingsRes.data.publicProjectId;
|
|
18965
|
-
remoteFileBuckets = settingsRes.data.remoteFileBuckets.map(b => b.bucket);
|
|
18966
|
-
}
|
|
18967
|
-
if (!publicProjectId) {
|
|
18968
|
-
return {
|
|
18969
|
-
success: false,
|
|
18970
|
-
errorMessage: "Could not get public project id"
|
|
18971
|
-
};
|
|
18972
|
-
}
|
|
18973
|
-
if (!projectName) {
|
|
18974
|
-
return {
|
|
18975
|
-
success: false,
|
|
18976
|
-
errorMessage: `Could not get project. Check that your val.config has the 'project' field set, or set it using the VAL_PROJECT environment variable`
|
|
18977
|
-
};
|
|
18978
|
-
}
|
|
18979
|
-
remoteFilesCounter += 1;
|
|
18980
|
-
const bucket = remoteFileBuckets[remoteFilesCounter % remoteFileBuckets.length];
|
|
18981
|
-
if (!bucket) {
|
|
18982
|
-
return {
|
|
18983
|
-
success: false,
|
|
18984
|
-
errorMessage: `Internal error: could not allocate a bucket for the remote file located at ${ctx.sourcePath}`
|
|
18985
|
-
};
|
|
21971
|
+
const opened = await openRemoteUploadSession(ctx);
|
|
21972
|
+
if (!opened.success) {
|
|
21973
|
+
return opened.result;
|
|
18986
21974
|
}
|
|
21975
|
+
const {
|
|
21976
|
+
session
|
|
21977
|
+
} = opened;
|
|
18987
21978
|
const fileBuffer = ctx.fs.readBuffer(filePath);
|
|
18988
21979
|
if (fileBuffer === undefined) {
|
|
18989
21980
|
return {
|
|
@@ -18992,32 +21983,16 @@ async function uploadRemoteFileCore(ctx, fileRef, metadata, schema) {
|
|
|
18992
21983
|
};
|
|
18993
21984
|
}
|
|
18994
21985
|
const relativeFilePath = path__default.relative(ctx.projectRoot, filePath).split(path__default.sep).join("/");
|
|
18995
|
-
|
|
18996
|
-
|
|
18997
|
-
success: false,
|
|
18998
|
-
errorMessage: `File path must be within the public/ directory (e.g. public/path/to/file.txt). Got: ${relativeFilePath}`
|
|
18999
|
-
};
|
|
19000
|
-
}
|
|
19001
|
-
const fileHash = Internal.remote.getFileHash(fileBuffer);
|
|
19002
|
-
const coreVersion = Internal.VERSION.core || "unknown";
|
|
19003
|
-
const fileExt = getFileExt(filePath);
|
|
19004
|
-
const ref = Internal.remote.createRemoteRef(ctx.remote.remoteHost, {
|
|
19005
|
-
publicProjectId,
|
|
19006
|
-
coreVersion,
|
|
19007
|
-
bucket,
|
|
19008
|
-
validationHash: Internal.remote.getValidationHash(coreVersion, schema, fileExt, metadata, fileHash, textEncoder),
|
|
19009
|
-
fileHash,
|
|
19010
|
-
filePath: relativeFilePath
|
|
19011
|
-
});
|
|
19012
|
-
const remoteFileUpload = await ctx.remote.uploadFile(projectName, bucket, fileHash, fileExt, fileBuffer, {
|
|
19013
|
-
pat
|
|
19014
|
-
});
|
|
19015
|
-
if (!remoteFileUpload.success) {
|
|
21986
|
+
const uploaded = await uploadBytesToRemote(ctx, session, relativeFilePath, fileBuffer, metadata, schema);
|
|
21987
|
+
if (!uploaded.success) {
|
|
19016
21988
|
return {
|
|
19017
21989
|
success: false,
|
|
19018
|
-
errorMessage:
|
|
21990
|
+
errorMessage: uploaded.error
|
|
19019
21991
|
};
|
|
19020
21992
|
}
|
|
21993
|
+
const {
|
|
21994
|
+
ref
|
|
21995
|
+
} = uploaded;
|
|
19021
21996
|
ctx.remoteFiles[ctx.sourcePath] = {
|
|
19022
21997
|
ref,
|
|
19023
21998
|
metadata
|
|
@@ -19025,9 +22000,9 @@ async function uploadRemoteFileCore(ctx, fileRef, metadata, schema) {
|
|
|
19025
22000
|
return {
|
|
19026
22001
|
success: true,
|
|
19027
22002
|
shouldApplyPatch: true,
|
|
19028
|
-
publicProjectId,
|
|
19029
|
-
remoteFileBuckets,
|
|
19030
|
-
remoteFilesCounter,
|
|
22003
|
+
publicProjectId: session.publicProjectId,
|
|
22004
|
+
remoteFileBuckets: session.remoteFileBuckets,
|
|
22005
|
+
remoteFilesCounter: session.remoteFilesCounter,
|
|
19031
22006
|
events: [{
|
|
19032
22007
|
type: "remote-uploading",
|
|
19033
22008
|
ref
|
|
@@ -19228,50 +22203,6 @@ async function handleUniqueFolderCheck(ctx) {
|
|
|
19228
22203
|
success: true
|
|
19229
22204
|
};
|
|
19230
22205
|
}
|
|
19231
|
-
|
|
19232
|
-
/**
|
|
19233
|
-
* What is out of step between a gallery's entries and its directory.
|
|
19234
|
-
*
|
|
19235
|
-
* Two questions, and they treat a remote entry differently — which is the whole
|
|
19236
|
-
* reason this is separate from the handler around it:
|
|
19237
|
-
*
|
|
19238
|
-
* - **Missing**: an entry with no bytes at its local path. Asked of LOCAL
|
|
19239
|
-
* entries only. A remote entry's bytes live on the content host, and nothing
|
|
19240
|
-
* puts a copy in the working tree: `saveOrUploadFiles` uploads the remote
|
|
19241
|
-
* descriptors and copies only the local ones into the tree, so a remote entry
|
|
19242
|
-
* added through the Studio (or over MCP) has no local file by design, and
|
|
19243
|
-
* demanding one would mean committing remote bytes to git — which is what
|
|
19244
|
-
* remote storage exists to avoid. Whether those bytes really are on the host
|
|
19245
|
-
* is a different check, `image:check-remote`, which already runs for exactly
|
|
19246
|
-
* these entries.
|
|
19247
|
-
* - **Untracked**: a file in the directory that no entry claims. Asked of every
|
|
19248
|
-
* entry, remote included, and that is why they are normalised to their local
|
|
19249
|
-
* path: `val validate --fix` promotes a local file to a remote ref and leaves
|
|
19250
|
-
* the file where it was, so a remote entry can perfectly well have one.
|
|
19251
|
-
*/
|
|
19252
|
-
function checkGalleryFiles(input) {
|
|
19253
|
-
const {
|
|
19254
|
-
dir,
|
|
19255
|
-
projectRoot,
|
|
19256
|
-
fs
|
|
19257
|
-
} = input;
|
|
19258
|
-
const entries = input.entryKeys.map(galleryEntryOf);
|
|
19259
|
-
const trackedFiles = new Set(entries.map(entry => entry.localPath));
|
|
19260
|
-
const missingTrackedFiles = entries.filter(entry => !entry.remote && !fs.fileExists(path__default.join(projectRoot, entry.localPath))).map(entry => entry.localPath);
|
|
19261
|
-
const filesInDir = [];
|
|
19262
|
-
try {
|
|
19263
|
-
const found = fs.readDirectory(path__default.join(projectRoot, dir), undefined, undefined, ["**/*"]);
|
|
19264
|
-
for (const entry of found) {
|
|
19265
|
-
filesInDir.push("/" + path__default.relative(projectRoot, entry).split(path__default.sep).join("/"));
|
|
19266
|
-
}
|
|
19267
|
-
} catch {
|
|
19268
|
-
// directory doesn't exist — no untracked files possible
|
|
19269
|
-
}
|
|
19270
|
-
return {
|
|
19271
|
-
missingTrackedFiles,
|
|
19272
|
-
untrackedFiles: filesInDir.filter(f => !trackedFiles.has(f))
|
|
19273
|
-
};
|
|
19274
|
-
}
|
|
19275
22206
|
async function handleCheckAllFiles(ctx) {
|
|
19276
22207
|
const value = ctx.validationError.value;
|
|
19277
22208
|
if (!value || typeof value.dir !== "string") {
|
|
@@ -19418,12 +22349,20 @@ const currentFixHandlers = {
|
|
|
19418
22349
|
"image:add-metadata": handleFileMetadata,
|
|
19419
22350
|
"file:check-metadata": handleFileMetadata,
|
|
19420
22351
|
"file:add-metadata": handleFileMetadata,
|
|
22352
|
+
"video:add-metadata": handleVideoMetadata,
|
|
19421
22353
|
"image:upload-remote": handleRemoteFileUpload,
|
|
19422
22354
|
"file:upload-remote": handleRemoteFileUpload,
|
|
19423
22355
|
"images:upload-remote": handleRemoteGalleryFileUpload,
|
|
19424
22356
|
"files:upload-remote": handleRemoteGalleryFileUpload,
|
|
19425
22357
|
"image:download-remote": handleRemoteFileDownload,
|
|
19426
22358
|
"file:download-remote": handleRemoteFileDownload,
|
|
22359
|
+
"video:upload-remote": handleVideoUploadRemote,
|
|
22360
|
+
"video:download-remote": handleVideoDownloadRemote,
|
|
22361
|
+
"videos:add-metadata": handleVideosetMetadata,
|
|
22362
|
+
"videos:upload-remote": handleVideosetUploadRemote,
|
|
22363
|
+
"videos:check-remote": handleVideosetCheckRemote,
|
|
22364
|
+
"videos:check-unique-folder": handleUniqueFolderCheck,
|
|
22365
|
+
"videos:check-all-files": handleVideosetCheckAllFiles,
|
|
19427
22366
|
"image:check-remote": handleRemoteFileCheck,
|
|
19428
22367
|
"images:check-remote": handleRemoteFileCheck,
|
|
19429
22368
|
"file:check-remote": handleRemoteFileCheck,
|
|
@@ -19619,20 +22558,15 @@ function createPrettierFormatter(prettier, options) {
|
|
|
19619
22558
|
}
|
|
19620
22559
|
|
|
19621
22560
|
/**
|
|
19622
|
-
* NOTE: this is intentionally NOT `SharedValConfig` from `@valbuild/shared
|
|
19623
|
-
*
|
|
19624
|
-
*
|
|
19625
|
-
* configs are accepted, so they are kept separate on purpose.
|
|
22561
|
+
* NOTE: this is intentionally NOT `SharedValConfig` from `@valbuild/shared`,
|
|
22562
|
+
* which leaves `files` out: `files.remote` decides how the schemas are built,
|
|
22563
|
+
* so it is read where `val.config.ts` is, not shared with the client.
|
|
19626
22564
|
*/
|
|
19627
22565
|
const ValConfigSchema = z$1.object({
|
|
19628
22566
|
project: z$1.string().optional(),
|
|
19629
22567
|
root: z$1.string().optional(),
|
|
19630
22568
|
files: z$1.object({
|
|
19631
|
-
|
|
19632
|
-
// Reject path traversal so the directory cannot escape /public
|
|
19633
|
-
!val.split("/").some(segment => segment === "." || segment === ".."), {
|
|
19634
|
-
message: "files.directory must start with '/public', must not end with '/' and must not contain '.' or '..' segments"
|
|
19635
|
-
})
|
|
22569
|
+
remote: z$1.boolean().optional()
|
|
19636
22570
|
}).optional(),
|
|
19637
22571
|
gitCommit: z$1.string().optional(),
|
|
19638
22572
|
gitBranch: z$1.string().optional(),
|
|
@@ -20121,7 +23055,7 @@ async function replaySnapshot(snapshotDir) {
|
|
|
20121
23055
|
const serverOps = new ValOpsFS(process.env.VAL_CONTENT_URL || DEFAULT_CONTENT_HOST, root, valModules,
|
|
20122
23056
|
// The snapshot's own val.config, as evaluated by loadValModules - no need to
|
|
20123
23057
|
// re-read it, and this way the replay uses exactly the config the snapshot
|
|
20124
|
-
// carries
|
|
23058
|
+
// carries.
|
|
20125
23059
|
{
|
|
20126
23060
|
config: valModules.config
|
|
20127
23061
|
});
|
|
@@ -20200,4 +23134,4 @@ function readCapturedReport(snapshotDir) {
|
|
|
20200
23134
|
return JSON.parse(fs.readFileSync(reportPath, "utf-8"));
|
|
20201
23135
|
}
|
|
20202
23136
|
|
|
20203
|
-
export { DEFAULT_LOGIN_EXPIRES_IN_SECONDS, DEFAULT_LOGIN_HOST, DEFAULT_LOGIN_POLL_INTERVAL_SECONDS, EXTERNAL_RESULT, InMemoryPatchStore, Service, ValFSHost, ValLoginError, ValModuleLoader, ValOpsFS, ValOpsHttp, ValOpsMemory, ValSourceFileHandler, analyzeValModule, awaitValLoginConfirmation, checkRemoteRef, classifyJsonValuesOp, compareWithCapturedReport, createDefaultValFSHost, createFixPatch, createJsonEntryPathMap, createModulePathMap, createPrettierFormatter, createService, createValApiRouter, createValModuleFileInspector, createValOps, createValServer, currentFixHandlers, decodeJwtWithoutVerifying, defineExternal, describePatchStoreProblems, downloadFileFromRemote, encodeJwt, err, evalValConfigFile, extractFileMetadata, extractImageMetadata, extractJsonValuesEntry, findAndEvalValConfigFile, findDefaultExport, findJsonEntryFilePath, fixHandlers, formatPatchSourceError, formatSyntaxErrorTree, getCachedRemoteFileDir, getCachedRemoteFilePath, getCompilerOptions, getExpire, getFileExt, getModulePathRange, getPersonalAccessTokenPath, getSettings, getValidationErrorFileRef, handleCheckAllFiles, handleExternalUpload, handleFileMetadata, handleJsonValuesExtractEntry, handleRemoteFileCheck, handleRemoteFileDownload, handleRemoteFileUpload, handleRemoteGalleryFileUpload, handleUniqueFolderCheck, initHandlerOptions, isExternalResult, loadValModules, ok, parsePersonalAccessTokenFile, patchSourceFile, persistPersonalAccessToken, planJsonValuesEntryExtraction, readCapturedReport, readPatchStore, rebaseContentOp, replaySnapshot, resolveRemoteFileAuth, safeReadGit, startValLogin, uploadRemoteFile, validateMetadata, verifyJwt };
|
|
23137
|
+
export { APP_MODE_REQUIRES_REMOTE_FILES, DEFAULT_LOGIN_EXPIRES_IN_SECONDS, DEFAULT_LOGIN_HOST, DEFAULT_LOGIN_POLL_INTERVAL_SECONDS, EXTERNAL_RESULT, InMemoryPatchStore, Service, ValFSHost, ValLoginError, ValModuleLoader, ValOpsFS, ValOpsHttp, ValOpsMemory, ValSourceFileHandler, analyzeValModule, awaitValLoginConfirmation, checkRemoteRef, classifyJsonValuesOp, compareWithCapturedReport, createDefaultValFSHost, createFixPatch, createJsonEntryPathMap, createModulePathMap, createPrettierFormatter, createService, createValApiRouter, createValModuleFileInspector, createValOps, createValServer, currentFixHandlers, decodeJwtWithoutVerifying, defineExternal, describePatchStoreProblems, downloadFileFromRemote, encodeJwt, err, evalValConfigFile, extractFileMetadata, extractImageMetadata, extractJsonValuesEntry, extractVideoMetadata, extractVideoMetadataFromFile, filesOfVideo, filesOfVideosetEntry, findAndEvalValConfigFile, findDefaultExport, findJsonEntryFilePath, fixHandlers, formatPatchSourceError, formatSyntaxErrorTree, getCachedRemoteFileDir, getCachedRemoteFilePath, getCompilerOptions, getExpire, getFileExt, getModulePathRange, getPersonalAccessTokenPath, getSettings, getValidationErrorFileRef, handleCheckAllFiles, handleExternalUpload, handleFileMetadata, handleJsonValuesExtractEntry, handleRemoteFileCheck, handleRemoteFileDownload, handleRemoteFileUpload, handleRemoteGalleryFileUpload, handleUniqueFolderCheck, handleVideosetCheckAllFiles, handleVideosetCheckRemote, handleVideosetMetadata, handleVideosetUploadRemote, initHandlerOptions, isExternalResult, loadValModules, ok, parsePersonalAccessTokenFile, patchSourceFile, persistPersonalAccessToken, planJsonValuesEntryExtraction, readCapturedReport, readPatchStore, rebaseContentOp, replaySnapshot, resolveRemoteFileAuth, safeReadGit, startValLogin, uploadRemoteFile, validateMetadata, verifyJwt, videosetEntryVideoSchema };
|