@valbuild/server 0.147.0 → 0.148.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +115 -0
- package/browser/dist/valbuild-server-browser.cjs.dev.js +2 -2
- package/browser/dist/valbuild-server-browser.cjs.prod.js +2 -2
- package/browser/dist/valbuild-server-browser.esm.js +2 -2
- package/dist/{ValRouterCore-7ffcc08c.cjs.prod.js → ValRouterCore-42772c21.cjs.prod.js} +980 -53
- package/dist/{ValRouterCore-f5004f8f.esm.js → ValRouterCore-7ca3dc68.esm.js} +972 -54
- package/dist/{ValRouterCore-3fd3d444.cjs.dev.js → ValRouterCore-9bdaaf89.cjs.dev.js} +980 -53
- package/dist/declarations/src/ValOps.d.ts +32 -11
- package/dist/declarations/src/ValOpsMemory.d.ts +4 -4
- package/dist/declarations/src/ValServer.d.ts +1 -2
- package/dist/declarations/src/checkRemoteRef.d.ts +40 -2
- package/dist/declarations/src/createFixPatch.d.ts +25 -3
- package/dist/declarations/src/fixFiles.d.ts +42 -0
- package/dist/declarations/src/fixHandlers.d.ts +14 -0
- package/dist/declarations/src/getSettings.d.ts +1 -0
- package/dist/declarations/src/index.d.ts +1 -0
- package/dist/declarations/src/platform.d.ts +16 -0
- package/dist/declarations/src/remoteFileCheck.d.ts +116 -0
- package/dist/declarations/src/remoteHostCheck.d.ts +101 -0
- package/dist/declarations/src/studioValidation.d.ts +84 -0
- package/dist/declarations/src/uploadRemoteFile.d.ts +1 -1
- package/dist/declarations/src/videosetFixes.d.ts +3 -2
- package/dist/{getJsonEntryAtCommit-df65d861.cjs.prod.js → getJsonEntryAtCommit-2e36e805.cjs.prod.js} +3 -4
- package/dist/{getJsonEntryAtCommit-1e08d724.esm.js → getJsonEntryAtCommit-40b6d9c1.esm.js} +3 -4
- package/dist/{getJsonEntryAtCommit-e96beb9e.cjs.dev.js → getJsonEntryAtCommit-64f67a50.cjs.dev.js} +3 -4
- package/dist/valbuild-server.cjs.dev.js +1352 -695
- package/dist/valbuild-server.cjs.prod.js +1352 -695
- package/dist/valbuild-server.esm.js +1352 -696
- package/package.json +4 -5
|
@@ -3,29 +3,29 @@ export { a as analyzeValModule, d as formatSyntaxErrorTree } from './jsonValuesM
|
|
|
3
3
|
import { applyPatch, PatchError, JSONOps, deepClone, deepEqual, sourceToPatchPath, isNotRoot } from '@valbuild/core/patch';
|
|
4
4
|
import { T as TSOps, r as removeValJsonEntry, i as insertValJsonEntry } from './ops-0d05feaf.esm.js';
|
|
5
5
|
import { pipe, result } from '@valbuild/core/fp';
|
|
6
|
-
import { derefPatch, Internal, extractValModules, deserializeSchema, DEFAULT_CONTENT_HOST } from '@valbuild/core';
|
|
6
|
+
import { derefPatch, Internal, extractValModules, deserializeSchema, DEFAULT_CONTENT_HOST, DEFAULT_VAL_REMOTE_HOST } from '@valbuild/core';
|
|
7
7
|
export { hasRemoteFileSchema } from '@valbuild/core';
|
|
8
8
|
import ts from 'typescript';
|
|
9
9
|
import path from 'path';
|
|
10
10
|
import fs from 'fs';
|
|
11
11
|
import vm from 'node:vm';
|
|
12
12
|
import { Module } from 'node:module';
|
|
13
|
-
import { r as resolveExistingJsonPath, f as findNestedJsonValuesRecords, v as validateJsonValuesEntries, i as isJsonValuesRootOp, a as rebaseContentOp, c as classifyJsonValuesOp, V as ValOps, b as bufferFromDataUrl, g as getFieldsForType, u as uploadRemoteFile, d as getFileExt, e as guessMimeTypeFromPath, h as createMetadataFromBuffer, j as ValOpsMemory, k as
|
|
14
|
-
export { I as InMemoryPatchStore, j as ValOpsMemory, c as classifyJsonValuesOp,
|
|
13
|
+
import { r as resolveExistingJsonPath, f as findNestedJsonValuesRecords, v as validateJsonValuesEntries, i as isJsonValuesRootOp, a as rebaseContentOp, c as classifyJsonValuesOp, V as ValOps, b as bufferFromDataUrl, g as getFieldsForType, u as uploadRemoteFile, d as getFileExt, e as guessMimeTypeFromPath, h as createMetadataFromBuffer, j as ValOpsMemory, k as isHlsMasterPlaylist, s as sumHlsSegmentDurations, p as parseHlsMasterVariants, l as largestHlsVariant, m as fallbackRefOf, n as mapHlsUris, o as movedRemoteHost, q as needsHostCheck, t as rewriteRefHost, w as fallbackRemoteHost, x as checkRemoteFiles, y as ValServer, z as createValRequestHandler, A as getNewJsonEntryPaths, B as formatPatchSourceError } from './ValRouterCore-7ca3dc68.esm.js';
|
|
14
|
+
export { I as InMemoryPatchStore, j as ValOpsMemory, C as askOncePerHost, c as classifyJsonValuesOp, B as formatPatchSourceError, d as getFileExt, D as getSettings, a as rebaseContentOp, u as uploadRemoteFile } from './ValRouterCore-7ca3dc68.esm.js';
|
|
15
15
|
import { createUIRequestHandler } from '@valbuild/ui/server';
|
|
16
16
|
import z$1, { z } from 'zod';
|
|
17
17
|
import { fromError } from 'zod-validation-error';
|
|
18
|
-
import { Patch, schemaErrorsOf, PatchSets, joinSharedChanges, schemaTypesOfPath, sideFiles, sideSource, JSONValue, PlanRefusalError, PatchId as PatchId$1, SerializedSchema, getErrorMessageFromUnknownJson, newestCommitSha, traverseSchemaSource, richTextImageSchema, forEachRichTextImage } from '@valbuild/shared/internal';
|
|
18
|
+
import { Patch, schemaErrorsOf, PatchSets, joinSharedChanges, schemaTypesOfPath, sideFiles, sideSource, JSONValue, PlanRefusalError, PatchId as PatchId$1, SerializedSchema, getErrorMessageFromUnknownJson, newestCommitSha, traverseSchemaSource, richTextImageSchema, forEachRichTextImage, isStudioFix } from '@valbuild/shared/internal';
|
|
19
19
|
import os from 'os';
|
|
20
20
|
import crypto$1, { createHash } from 'crypto';
|
|
21
21
|
import { Buffer as Buffer$1 } from 'buffer';
|
|
22
22
|
import http from 'node:http';
|
|
23
23
|
import https from 'node:https';
|
|
24
24
|
import net from 'node:net';
|
|
25
|
-
import { transform } from 'sucrase';
|
|
26
25
|
import sizeOf from 'image-size';
|
|
27
26
|
import http$1 from 'http';
|
|
28
27
|
import https$1 from 'https';
|
|
28
|
+
import { transform } from 'sucrase';
|
|
29
29
|
import fs$1 from 'fs/promises';
|
|
30
30
|
|
|
31
31
|
function getSyntheticContainingPath(rootDir) {
|
|
@@ -3153,7 +3153,7 @@ class ValOpsFS extends ValOps {
|
|
|
3153
3153
|
// Cheap, and this is the one path that creates staging directories, so it
|
|
3154
3154
|
// is where the ones nobody claimed get noticed.
|
|
3155
3155
|
sweepStaleUploads(patchesDir);
|
|
3156
|
-
const buffer = bufferFromDataUrl(data);
|
|
3156
|
+
const buffer = bufferFromDataUrl(data, base64 => Buffer$1.from(base64, "base64"));
|
|
3157
3157
|
if (!buffer) {
|
|
3158
3158
|
return {
|
|
3159
3159
|
error: {
|
|
@@ -9222,568 +9222,279 @@ function readableError(error) {
|
|
|
9222
9222
|
}
|
|
9223
9223
|
}
|
|
9224
9224
|
|
|
9225
|
-
/** The platform every server on Node runs with. See {@link ValServerPlatform}. */
|
|
9226
|
-
const nodePlatform = {
|
|
9227
|
-
createOps: createValOps,
|
|
9228
|
-
probeUrl: url => probeUrl(url),
|
|
9229
|
-
jwt: {
|
|
9230
|
-
verifyJwt,
|
|
9231
|
-
decodeJwtWithoutVerifying,
|
|
9232
|
-
encodeJwt,
|
|
9233
|
-
getExpire
|
|
9234
|
-
},
|
|
9235
|
-
readValLoginToken,
|
|
9236
|
-
resolveRemoteFileAuth
|
|
9237
|
-
};
|
|
9238
|
-
|
|
9239
|
-
async function createValServer(valModules, route, opts, config, callbacks, formatter,
|
|
9240
9225
|
/**
|
|
9241
|
-
*
|
|
9242
|
-
*
|
|
9243
|
-
|
|
9244
|
-
|
|
9245
|
-
|
|
9246
|
-
*
|
|
9247
|
-
*
|
|
9226
|
+
* The size and length of an ISO base media file (`.mp4`, `.m4v`, `.mov`),
|
|
9227
|
+
* read from its headers. Nothing is decoded.
|
|
9228
|
+
*
|
|
9229
|
+
* Hand-written rather than a media library, deliberately: `@valbuild/server`
|
|
9230
|
+
* is loaded by every app that runs Val, and the four numbers wanted here are
|
|
9231
|
+
* three small boxes away from the start of the `moov` box:
|
|
9232
|
+
*
|
|
9233
|
+
* moov > mvhd timescale and duration (version 0 and 1)
|
|
9234
|
+
* moov > mvex > mehd duration of a FRAGMENTED file, whose mvhd says 0
|
|
9235
|
+
* moov > trak > tkhd width and height (16.16 fixed point) and the
|
|
9236
|
+
* display matrix, for a rotated phone video
|
|
9237
|
+
* moov > trak > mdia > hdlr `vide`: which trak is the picture
|
|
9238
|
+
*
|
|
9239
|
+
* Only the box headers are read on the way to `moov`, and then `moov` itself,
|
|
9240
|
+
* so a multi-GB file costs a few small reads: `mdat` is seeked past, never
|
|
9241
|
+
* loaded. That is what {@link ByteSource} is for.
|
|
9248
9242
|
*/
|
|
9249
|
-
|
|
9250
|
-
|
|
9251
|
-
|
|
9252
|
-
|
|
9253
|
-
|
|
9254
|
-
|
|
9255
|
-
|
|
9256
|
-
}
|
|
9243
|
+
|
|
9244
|
+
/** Random access to a file's bytes, so a large `mdat` can be skipped. */
|
|
9245
|
+
|
|
9246
|
+
function bufferByteSource(buffer) {
|
|
9247
|
+
return {
|
|
9248
|
+
size: buffer.length,
|
|
9249
|
+
read: (offset, length) => buffer.subarray(offset, offset + length)
|
|
9250
|
+
};
|
|
9257
9251
|
}
|
|
9258
9252
|
|
|
9259
|
-
// TODO: remove
|
|
9260
9253
|
/**
|
|
9261
|
-
*
|
|
9262
|
-
*
|
|
9263
|
-
* This is the only thing in this module that touches either, and it is a local
|
|
9264
|
-
* development convenience: scanning upwards for a `.git` to guess the commit and
|
|
9265
|
-
* branch. A static import put `fs` in the module graph of everything reaching
|
|
9266
|
-
* `createValApiRouter` -- which is every server integration, including ones that
|
|
9267
|
-
* run where there is no filesystem. Workerd provides no `fs`, so such a build
|
|
9268
|
-
* could not be bundled at all without stubbing it.
|
|
9269
|
-
*
|
|
9270
|
-
* The `await import` costs nothing here: the only caller is the CLI, on a
|
|
9271
|
-
* machine that has both.
|
|
9254
|
+
* Run `read` against a file on disk, reading only what it asks for. The file
|
|
9255
|
+
* is closed afterwards whatever happens.
|
|
9272
9256
|
*/
|
|
9273
|
-
|
|
9274
|
-
const
|
|
9275
|
-
|
|
9276
|
-
|
|
9277
|
-
|
|
9278
|
-
|
|
9279
|
-
|
|
9280
|
-
|
|
9281
|
-
|
|
9282
|
-
|
|
9283
|
-
|
|
9284
|
-
|
|
9285
|
-
|
|
9286
|
-
|
|
9287
|
-
|
|
9288
|
-
|
|
9289
|
-
|
|
9290
|
-
|
|
9291
|
-
const branchName = match[1];
|
|
9292
|
-
return {
|
|
9293
|
-
branch: branchName,
|
|
9294
|
-
commit: await readCommit(currentDir, branchName)
|
|
9295
|
-
};
|
|
9296
|
-
} else {
|
|
9297
|
-
return {
|
|
9298
|
-
commit: undefined,
|
|
9299
|
-
branch: undefined
|
|
9300
|
-
};
|
|
9257
|
+
function withFileByteSource(absolutePath, read) {
|
|
9258
|
+
const fd = fs.openSync(absolutePath, "r");
|
|
9259
|
+
try {
|
|
9260
|
+
const size = fs.fstatSync(fd).size;
|
|
9261
|
+
return read({
|
|
9262
|
+
size,
|
|
9263
|
+
read: (offset, length) => {
|
|
9264
|
+
const wanted = Math.max(0, Math.min(length, size - offset));
|
|
9265
|
+
const buffer = Buffer.alloc(wanted);
|
|
9266
|
+
let filled = 0;
|
|
9267
|
+
while (filled < wanted) {
|
|
9268
|
+
const n = fs.readSync(fd, buffer, filled, wanted - filled, offset + filled);
|
|
9269
|
+
if (n === 0) {
|
|
9270
|
+
break;
|
|
9271
|
+
}
|
|
9272
|
+
filled += n;
|
|
9273
|
+
}
|
|
9274
|
+
return buffer.subarray(0, filled);
|
|
9301
9275
|
}
|
|
9302
|
-
}
|
|
9303
|
-
|
|
9276
|
+
});
|
|
9277
|
+
} finally {
|
|
9278
|
+
fs.closeSync(fd);
|
|
9279
|
+
}
|
|
9280
|
+
}
|
|
9281
|
+
/**
|
|
9282
|
+
* A `moov` larger than this is not a header worth reading into memory. A
|
|
9283
|
+
* two-hour film's is a few MB; this is far past any real one.
|
|
9284
|
+
*/
|
|
9285
|
+
const MAX_MOOV_SIZE = 64 * 1024 * 1024;
|
|
9304
9286
|
|
|
9305
|
-
|
|
9306
|
-
|
|
9307
|
-
|
|
9308
|
-
|
|
9309
|
-
|
|
9310
|
-
|
|
9287
|
+
/**
|
|
9288
|
+
* Whatever of {@link IsoBmffMetadata} the file declares. A field that cannot be
|
|
9289
|
+
* read is absent; nothing here throws on a file that is not ISO BMFF.
|
|
9290
|
+
*/
|
|
9291
|
+
function readIsoBmffMetadata(source) {
|
|
9292
|
+
const moov = findTopLevelBox(source, "moov");
|
|
9293
|
+
if (!moov) {
|
|
9294
|
+
return {};
|
|
9295
|
+
}
|
|
9296
|
+
const metadata = {};
|
|
9297
|
+
const mvhd = childBox(moov, "mvhd");
|
|
9298
|
+
const timescaleAndDuration = mvhd ? readMvhd(mvhd) : undefined;
|
|
9299
|
+
if (timescaleAndDuration) {
|
|
9300
|
+
const {
|
|
9301
|
+
timescale,
|
|
9302
|
+
duration
|
|
9303
|
+
} = timescaleAndDuration;
|
|
9304
|
+
if (duration > 0) {
|
|
9305
|
+
metadata.duration = duration / timescale;
|
|
9306
|
+
} else {
|
|
9307
|
+
// A fragmented file (CMAF) says 0 in mvhd and puts the length in mehd,
|
|
9308
|
+
// in the same timescale.
|
|
9309
|
+
const mehd = childBox(childBox(moov, "mvex"), "mehd");
|
|
9310
|
+
const fragmentDuration = mehd ? readMehd(mehd) : undefined;
|
|
9311
|
+
if (fragmentDuration !== undefined && fragmentDuration > 0) {
|
|
9312
|
+
metadata.duration = fragmentDuration / timescale;
|
|
9311
9313
|
}
|
|
9312
|
-
return findGitHead(parentDir, depth + 1);
|
|
9313
9314
|
}
|
|
9314
9315
|
}
|
|
9315
|
-
|
|
9316
|
-
|
|
9317
|
-
|
|
9318
|
-
|
|
9319
|
-
|
|
9320
|
-
|
|
9321
|
-
|
|
9322
|
-
|
|
9323
|
-
|
|
9324
|
-
|
|
9325
|
-
|
|
9326
|
-
|
|
9327
|
-
async function readCommit(gitDir, branchName) {
|
|
9328
|
-
const {
|
|
9329
|
-
promises: fs
|
|
9330
|
-
} = await import('fs');
|
|
9331
|
-
const path = await import('path');
|
|
9332
|
-
try {
|
|
9333
|
-
return (await fs.readFile(path.join(gitDir, ".git", "refs", "heads", branchName), "utf-8")).trim();
|
|
9334
|
-
} catch {
|
|
9335
|
-
return undefined;
|
|
9316
|
+
for (const trak of childBoxes(moov, "trak")) {
|
|
9317
|
+
const hdlr = childBox(childBox(trak, "mdia"), "hdlr");
|
|
9318
|
+
if (!hdlr || hdlr.length < 12 || hdlr.toString("latin1", 8, 12) !== "vide") {
|
|
9319
|
+
continue;
|
|
9320
|
+
}
|
|
9321
|
+
const tkhd = childBox(trak, "tkhd");
|
|
9322
|
+
const size = tkhd ? readTkhd(tkhd) : undefined;
|
|
9323
|
+
if (size && size.width > 0 && size.height > 0) {
|
|
9324
|
+
metadata.width = size.width;
|
|
9325
|
+
metadata.height = size.height;
|
|
9326
|
+
break;
|
|
9327
|
+
}
|
|
9336
9328
|
}
|
|
9329
|
+
return metadata;
|
|
9337
9330
|
}
|
|
9338
|
-
function createValApiRouter(route, valServerPromise, convert) {
|
|
9339
|
-
const uiRequestHandler = createUIRequestHandler();
|
|
9340
|
-
return createValRequestHandler(route, valServerPromise, convert, uiRequestHandler);
|
|
9341
|
-
}
|
|
9342
|
-
|
|
9343
|
-
const JsFileLookupMapping = [
|
|
9344
|
-
// NOTE: first one matching will be used
|
|
9345
|
-
[".cjs.d.ts", [".esm.js", ".mjs.js"]], [".cjs.js", [".esm.js", ".mjs.js"]], [".cjs", [".mjs"]], [".d.ts", [".js", ".esm.js", ".mjs.js"]]];
|
|
9346
|
-
const MAX_CACHE_SIZE = 100 * 1024 * 1024; // 100 mb
|
|
9347
|
-
const MAX_OBJECT_KEY_SIZE = 2 ** 27; // https://stackoverflow.com/questions/13367391/is-there-a-limit-on-length-of-the-key-string-in-js-object
|
|
9348
9331
|
|
|
9349
|
-
|
|
9350
|
-
|
|
9351
|
-
|
|
9352
|
-
|
|
9353
|
-
|
|
9354
|
-
|
|
9355
|
-
|
|
9356
|
-
|
|
9357
|
-
|
|
9358
|
-
|
|
9359
|
-
|
|
9360
|
-
|
|
9361
|
-
|
|
9362
|
-
rmFile: fs.rmSync,
|
|
9363
|
-
readBuffer: fileName => {
|
|
9364
|
-
try {
|
|
9365
|
-
return fs.readFileSync(fileName);
|
|
9366
|
-
} catch {
|
|
9332
|
+
/** The payload (after the header) of the first top-level box of a type. */
|
|
9333
|
+
function findTopLevelBox(source, type) {
|
|
9334
|
+
let offset = 0;
|
|
9335
|
+
while (offset + 8 <= source.size) {
|
|
9336
|
+
const header = source.read(offset, 16);
|
|
9337
|
+
if (header.length < 8) {
|
|
9338
|
+
return undefined;
|
|
9339
|
+
}
|
|
9340
|
+
let size = header.readUInt32BE(0);
|
|
9341
|
+
const boxType = header.toString("latin1", 4, 8);
|
|
9342
|
+
let headerSize = 8;
|
|
9343
|
+
if (size === 1) {
|
|
9344
|
+
if (header.length < 16) {
|
|
9367
9345
|
return undefined;
|
|
9368
9346
|
}
|
|
9347
|
+
size = Number(header.readBigUInt64BE(8));
|
|
9348
|
+
headerSize = 16;
|
|
9349
|
+
} else if (size === 0) {
|
|
9350
|
+
size = source.size - offset;
|
|
9369
9351
|
}
|
|
9370
|
-
|
|
9371
|
-
|
|
9372
|
-
this.compilerOptions = compilerOptions;
|
|
9373
|
-
this.sourceFileHandler = sourceFileHandler;
|
|
9374
|
-
this.host = host;
|
|
9375
|
-
this.disableCache = disableCache;
|
|
9376
|
-
this.cache = {};
|
|
9377
|
-
this.cacheSize = 0;
|
|
9378
|
-
}
|
|
9379
|
-
getModule(modulePath) {
|
|
9380
|
-
if (!modulePath) {
|
|
9381
|
-
throw Error(`Illegal module path: "${modulePath}"`);
|
|
9352
|
+
if (size < headerSize) {
|
|
9353
|
+
return undefined;
|
|
9382
9354
|
}
|
|
9383
|
-
|
|
9384
|
-
|
|
9385
|
-
|
|
9386
|
-
}
|
|
9387
|
-
let compiledCode;
|
|
9388
|
-
if (this.cache[code] && !this.disableCache) {
|
|
9389
|
-
// TODO: use hash instead of code as key
|
|
9390
|
-
compiledCode = this.cache[code];
|
|
9391
|
-
} else {
|
|
9392
|
-
compiledCode = transform(code, {
|
|
9393
|
-
filePath: modulePath,
|
|
9394
|
-
disableESTransforms: true,
|
|
9395
|
-
transforms: ["typescript"]
|
|
9396
|
-
}).code;
|
|
9397
|
-
if (!this.disableCache) {
|
|
9398
|
-
if (this.cacheSize > MAX_CACHE_SIZE) {
|
|
9399
|
-
console.warn("Cache size exceeded, clearing cache");
|
|
9400
|
-
this.cache = {};
|
|
9401
|
-
this.cacheSize = 0;
|
|
9402
|
-
}
|
|
9403
|
-
if (code.length < MAX_OBJECT_KEY_SIZE) {
|
|
9404
|
-
this.cache[code] = compiledCode;
|
|
9405
|
-
this.cacheSize += code.length + compiledCode.length; // code is mostly ASCII so 1 byte per char
|
|
9406
|
-
}
|
|
9355
|
+
if (boxType === type) {
|
|
9356
|
+
if (size - headerSize > MAX_MOOV_SIZE) {
|
|
9357
|
+
return undefined;
|
|
9407
9358
|
}
|
|
9359
|
+
const payload = source.read(offset + headerSize, size - headerSize);
|
|
9360
|
+
return payload.length === size - headerSize ? payload : undefined;
|
|
9408
9361
|
}
|
|
9409
|
-
|
|
9362
|
+
offset += size;
|
|
9410
9363
|
}
|
|
9411
|
-
|
|
9412
|
-
|
|
9413
|
-
|
|
9414
|
-
|
|
9415
|
-
|
|
9416
|
-
|
|
9417
|
-
|
|
9418
|
-
|
|
9419
|
-
|
|
9420
|
-
|
|
9421
|
-
|
|
9364
|
+
return undefined;
|
|
9365
|
+
}
|
|
9366
|
+
|
|
9367
|
+
/** Each child box of a container payload, as `[type, payload]`. */
|
|
9368
|
+
function* boxesIn(payload) {
|
|
9369
|
+
let offset = 0;
|
|
9370
|
+
while (offset + 8 <= payload.length) {
|
|
9371
|
+
let size = payload.readUInt32BE(offset);
|
|
9372
|
+
const type = payload.toString("latin1", offset + 4, offset + 8);
|
|
9373
|
+
let headerSize = 8;
|
|
9374
|
+
if (size === 1) {
|
|
9375
|
+
if (offset + 16 > payload.length) {
|
|
9376
|
+
return;
|
|
9422
9377
|
}
|
|
9423
|
-
|
|
9378
|
+
size = Number(payload.readBigUInt64BE(offset + 8));
|
|
9379
|
+
headerSize = 16;
|
|
9380
|
+
} else if (size === 0) {
|
|
9381
|
+
size = payload.length - offset;
|
|
9424
9382
|
}
|
|
9425
|
-
|
|
9426
|
-
|
|
9427
|
-
const followedPath = ((_this$host$realpath = (_this$host = this.host).realpath) === null || _this$host$realpath === void 0 ? void 0 : _this$host$realpath.call(_this$host, filePath)) ?? filePath;
|
|
9428
|
-
if (!followedPath) {
|
|
9429
|
-
throw Error(`File path was empty: "${filePath}", containing file: "${containingFilePath}", requested module: "${requestedModuleName}"`);
|
|
9383
|
+
if (size < headerSize || offset + size > payload.length) {
|
|
9384
|
+
return;
|
|
9430
9385
|
}
|
|
9431
|
-
|
|
9386
|
+
yield [type, payload.subarray(offset + headerSize, offset + size)];
|
|
9387
|
+
offset += size;
|
|
9432
9388
|
}
|
|
9433
|
-
|
|
9434
|
-
|
|
9435
|
-
|
|
9436
|
-
|
|
9437
|
-
|
|
9438
|
-
|
|
9439
|
-
|
|
9440
|
-
|
|
9441
|
-
|
|
9442
|
-
if (!requiresReplacements) {
|
|
9443
|
-
if (this.host.fileExists(filePath)) {
|
|
9444
|
-
return {
|
|
9445
|
-
match: filePath
|
|
9446
|
-
};
|
|
9447
|
-
}
|
|
9389
|
+
}
|
|
9390
|
+
function childBoxes(payload, type) {
|
|
9391
|
+
if (!payload) {
|
|
9392
|
+
return [];
|
|
9393
|
+
}
|
|
9394
|
+
const found = [];
|
|
9395
|
+
for (const [boxType, child] of boxesIn(payload)) {
|
|
9396
|
+
if (boxType === type) {
|
|
9397
|
+
found.push(child);
|
|
9448
9398
|
}
|
|
9449
|
-
|
|
9450
|
-
|
|
9451
|
-
|
|
9452
|
-
|
|
9453
|
-
|
|
9454
|
-
|
|
9455
|
-
|
|
9456
|
-
|
|
9457
|
-
|
|
9458
|
-
|
|
9459
|
-
|
|
9460
|
-
|
|
9461
|
-
|
|
9462
|
-
|
|
9399
|
+
}
|
|
9400
|
+
return found;
|
|
9401
|
+
}
|
|
9402
|
+
function childBox(payload, type) {
|
|
9403
|
+
return childBoxes(payload, type)[0];
|
|
9404
|
+
}
|
|
9405
|
+
|
|
9406
|
+
/** "Unknown" in a version 1 box. Not a literal: `0x…n` needs an ES2020 target. */
|
|
9407
|
+
const ALL_ONES_64 = BigInt("0xffffffffffffffff");
|
|
9408
|
+
function readMvhd(mvhd) {
|
|
9409
|
+
const version = mvhd[0];
|
|
9410
|
+
if (version === 1) {
|
|
9411
|
+
if (mvhd.length < 32) return undefined;
|
|
9412
|
+
const timescale = mvhd.readUInt32BE(20);
|
|
9413
|
+
const duration = mvhd.readBigUInt64BE(24);
|
|
9414
|
+
if (timescale === 0 || duration === ALL_ONES_64) {
|
|
9415
|
+
return undefined;
|
|
9463
9416
|
}
|
|
9464
9417
|
return {
|
|
9465
|
-
|
|
9466
|
-
|
|
9418
|
+
timescale,
|
|
9419
|
+
duration: Number(duration)
|
|
9467
9420
|
};
|
|
9468
9421
|
}
|
|
9422
|
+
if (mvhd.length < 20) return undefined;
|
|
9423
|
+
const timescale = mvhd.readUInt32BE(12);
|
|
9424
|
+
const duration = mvhd.readUInt32BE(16);
|
|
9425
|
+
// All ones means "unknown" in a version 0 box.
|
|
9426
|
+
if (timescale === 0 || duration === 0xffffffff) {
|
|
9427
|
+
return undefined;
|
|
9428
|
+
}
|
|
9429
|
+
return {
|
|
9430
|
+
timescale,
|
|
9431
|
+
duration
|
|
9432
|
+
};
|
|
9433
|
+
}
|
|
9434
|
+
function readMehd(mehd) {
|
|
9435
|
+
if (mehd[0] === 1) {
|
|
9436
|
+
return mehd.length >= 12 ? Number(mehd.readBigUInt64BE(4)) : undefined;
|
|
9437
|
+
}
|
|
9438
|
+
return mehd.length >= 8 ? mehd.readUInt32BE(4) : undefined;
|
|
9469
9439
|
}
|
|
9470
9440
|
|
|
9471
9441
|
/**
|
|
9472
|
-
*
|
|
9473
|
-
*
|
|
9442
|
+
* The size a track is SHOWN at: tkhd's width and height, swapped when the
|
|
9443
|
+
* display matrix turns the picture a quarter turn — a phone records upright
|
|
9444
|
+
* video sideways and says so here, and a page lays it out as it is shown.
|
|
9474
9445
|
*/
|
|
9475
|
-
|
|
9476
|
-
|
|
9477
|
-
|
|
9478
|
-
|
|
9479
|
-
|
|
9480
|
-
|
|
9481
|
-
|
|
9482
|
-
readDirectory(rootDir, extensions, excludes, includes, depth) {
|
|
9483
|
-
return this.valFS.readDirectory(rootDir, extensions, excludes, includes, depth);
|
|
9484
|
-
}
|
|
9485
|
-
rmFile(fileName) {
|
|
9486
|
-
this.valFS.rmFile(fileName);
|
|
9487
|
-
}
|
|
9488
|
-
writeFile(fileName, text, encoding) {
|
|
9489
|
-
this.valFS.writeFile(fileName, text, encoding);
|
|
9490
|
-
}
|
|
9491
|
-
getCurrentDirectory() {
|
|
9492
|
-
return this.currentDirectory;
|
|
9493
|
-
}
|
|
9494
|
-
getCanonicalFileName(fileName) {
|
|
9495
|
-
if (path.isAbsolute(fileName)) {
|
|
9496
|
-
return path.normalize(fileName);
|
|
9497
|
-
}
|
|
9498
|
-
return path.resolve(this.getCurrentDirectory(), fileName);
|
|
9499
|
-
}
|
|
9500
|
-
fileExists(fileName) {
|
|
9501
|
-
return this.valFS.fileExists(fileName);
|
|
9502
|
-
}
|
|
9503
|
-
readFile(fileName) {
|
|
9504
|
-
return this.valFS.readFile(fileName);
|
|
9505
|
-
}
|
|
9506
|
-
readBuffer(fileName) {
|
|
9507
|
-
return this.valFS.readBuffer(fileName);
|
|
9508
|
-
}
|
|
9509
|
-
realpath(path) {
|
|
9510
|
-
return this.valFS.realpath(path);
|
|
9446
|
+
function readTkhd(tkhd) {
|
|
9447
|
+
// Past the version-specific times and ids, then 16 bytes of reserved,
|
|
9448
|
+
// layer, alternate group, volume and reserved, then the 36-byte matrix.
|
|
9449
|
+
const matrixAt = tkhd[0] === 1 ? 4 + 32 + 16 : 4 + 20 + 16;
|
|
9450
|
+
const widthAt = matrixAt + 36;
|
|
9451
|
+
if (tkhd.length < widthAt + 8) {
|
|
9452
|
+
return undefined;
|
|
9511
9453
|
}
|
|
9454
|
+
const a = tkhd.readInt32BE(matrixAt);
|
|
9455
|
+
const b = tkhd.readInt32BE(matrixAt + 4);
|
|
9456
|
+
const c = tkhd.readInt32BE(matrixAt + 12);
|
|
9457
|
+
const d = tkhd.readInt32BE(matrixAt + 16);
|
|
9458
|
+
const width = Math.round(tkhd.readUInt32BE(widthAt) / 0x10000);
|
|
9459
|
+
const height = Math.round(tkhd.readUInt32BE(widthAt + 4) / 0x10000);
|
|
9460
|
+
const quarterTurn = a === 0 && d === 0 && b !== 0 && c !== 0;
|
|
9461
|
+
return quarterTurn ? {
|
|
9462
|
+
width: height,
|
|
9463
|
+
height: width
|
|
9464
|
+
} : {
|
|
9465
|
+
width,
|
|
9466
|
+
height
|
|
9467
|
+
};
|
|
9512
9468
|
}
|
|
9513
9469
|
|
|
9514
9470
|
/**
|
|
9515
|
-
* The size and length of
|
|
9516
|
-
* read from its headers. Nothing is decoded.
|
|
9471
|
+
* The size and length of a Matroska file (`.webm`, and `.mkv`, which is the
|
|
9472
|
+
* same container), read from its headers. Nothing is decoded.
|
|
9517
9473
|
*
|
|
9518
|
-
* Hand-written
|
|
9519
|
-
*
|
|
9520
|
-
*
|
|
9474
|
+
* Hand-written for the same reason `isoBmff.ts` is: `@valbuild/server` is
|
|
9475
|
+
* loaded by every app that runs Val, and what is wanted here is a handful of
|
|
9476
|
+
* integers two levels into the file:
|
|
9521
9477
|
*
|
|
9522
|
-
*
|
|
9523
|
-
*
|
|
9524
|
-
*
|
|
9525
|
-
*
|
|
9526
|
-
*
|
|
9478
|
+
* EBML the header; a file without one is not Matroska
|
|
9479
|
+
* Segment > Info TimecodeScale and Duration
|
|
9480
|
+
* Segment > Tracks > TrackEntry TrackType 1 (video) >
|
|
9481
|
+
* Video PixelWidth / PixelHeight, and DisplayWidth /
|
|
9482
|
+
* DisplayHeight when they are in pixels
|
|
9527
9483
|
*
|
|
9528
|
-
*
|
|
9529
|
-
*
|
|
9530
|
-
*
|
|
9531
|
-
|
|
9532
|
-
|
|
9533
|
-
|
|
9534
|
-
|
|
9535
|
-
|
|
9536
|
-
|
|
9537
|
-
|
|
9538
|
-
|
|
9539
|
-
|
|
9540
|
-
|
|
9541
|
-
|
|
9542
|
-
/**
|
|
9543
|
-
* Run `read` against a file on disk, reading only what it asks for. The file
|
|
9544
|
-
* is closed afterwards whatever happens.
|
|
9545
|
-
*/
|
|
9546
|
-
function withFileByteSource(absolutePath, read) {
|
|
9547
|
-
const fd = fs.openSync(absolutePath, "r");
|
|
9548
|
-
try {
|
|
9549
|
-
const size = fs.fstatSync(fd).size;
|
|
9550
|
-
return read({
|
|
9551
|
-
size,
|
|
9552
|
-
read: (offset, length) => {
|
|
9553
|
-
const wanted = Math.max(0, Math.min(length, size - offset));
|
|
9554
|
-
const buffer = Buffer.alloc(wanted);
|
|
9555
|
-
let filled = 0;
|
|
9556
|
-
while (filled < wanted) {
|
|
9557
|
-
const n = fs.readSync(fd, buffer, filled, wanted - filled, offset + filled);
|
|
9558
|
-
if (n === 0) {
|
|
9559
|
-
break;
|
|
9560
|
-
}
|
|
9561
|
-
filled += n;
|
|
9562
|
-
}
|
|
9563
|
-
return buffer.subarray(0, filled);
|
|
9564
|
-
}
|
|
9565
|
-
});
|
|
9566
|
-
} finally {
|
|
9567
|
-
fs.closeSync(fd);
|
|
9568
|
-
}
|
|
9569
|
-
}
|
|
9570
|
-
/**
|
|
9571
|
-
* A `moov` larger than this is not a header worth reading into memory. A
|
|
9572
|
-
* two-hour film's is a few MB; this is far past any real one.
|
|
9573
|
-
*/
|
|
9574
|
-
const MAX_MOOV_SIZE = 64 * 1024 * 1024;
|
|
9575
|
-
|
|
9576
|
-
/**
|
|
9577
|
-
* Whatever of {@link IsoBmffMetadata} the file declares. A field that cannot be
|
|
9578
|
-
* read is absent; nothing here throws on a file that is not ISO BMFF.
|
|
9579
|
-
*/
|
|
9580
|
-
function readIsoBmffMetadata(source) {
|
|
9581
|
-
const moov = findTopLevelBox(source, "moov");
|
|
9582
|
-
if (!moov) {
|
|
9583
|
-
return {};
|
|
9584
|
-
}
|
|
9585
|
-
const metadata = {};
|
|
9586
|
-
const mvhd = childBox(moov, "mvhd");
|
|
9587
|
-
const timescaleAndDuration = mvhd ? readMvhd(mvhd) : undefined;
|
|
9588
|
-
if (timescaleAndDuration) {
|
|
9589
|
-
const {
|
|
9590
|
-
timescale,
|
|
9591
|
-
duration
|
|
9592
|
-
} = timescaleAndDuration;
|
|
9593
|
-
if (duration > 0) {
|
|
9594
|
-
metadata.duration = duration / timescale;
|
|
9595
|
-
} else {
|
|
9596
|
-
// A fragmented file (CMAF) says 0 in mvhd and puts the length in mehd,
|
|
9597
|
-
// in the same timescale.
|
|
9598
|
-
const mehd = childBox(childBox(moov, "mvex"), "mehd");
|
|
9599
|
-
const fragmentDuration = mehd ? readMehd(mehd) : undefined;
|
|
9600
|
-
if (fragmentDuration !== undefined && fragmentDuration > 0) {
|
|
9601
|
-
metadata.duration = fragmentDuration / timescale;
|
|
9602
|
-
}
|
|
9603
|
-
}
|
|
9604
|
-
}
|
|
9605
|
-
for (const trak of childBoxes(moov, "trak")) {
|
|
9606
|
-
const hdlr = childBox(childBox(trak, "mdia"), "hdlr");
|
|
9607
|
-
if (!hdlr || hdlr.length < 12 || hdlr.toString("latin1", 8, 12) !== "vide") {
|
|
9608
|
-
continue;
|
|
9609
|
-
}
|
|
9610
|
-
const tkhd = childBox(trak, "tkhd");
|
|
9611
|
-
const size = tkhd ? readTkhd(tkhd) : undefined;
|
|
9612
|
-
if (size && size.width > 0 && size.height > 0) {
|
|
9613
|
-
metadata.width = size.width;
|
|
9614
|
-
metadata.height = size.height;
|
|
9615
|
-
break;
|
|
9616
|
-
}
|
|
9617
|
-
}
|
|
9618
|
-
return metadata;
|
|
9619
|
-
}
|
|
9620
|
-
|
|
9621
|
-
/** The payload (after the header) of the first top-level box of a type. */
|
|
9622
|
-
function findTopLevelBox(source, type) {
|
|
9623
|
-
let offset = 0;
|
|
9624
|
-
while (offset + 8 <= source.size) {
|
|
9625
|
-
const header = source.read(offset, 16);
|
|
9626
|
-
if (header.length < 8) {
|
|
9627
|
-
return undefined;
|
|
9628
|
-
}
|
|
9629
|
-
let size = header.readUInt32BE(0);
|
|
9630
|
-
const boxType = header.toString("latin1", 4, 8);
|
|
9631
|
-
let headerSize = 8;
|
|
9632
|
-
if (size === 1) {
|
|
9633
|
-
if (header.length < 16) {
|
|
9634
|
-
return undefined;
|
|
9635
|
-
}
|
|
9636
|
-
size = Number(header.readBigUInt64BE(8));
|
|
9637
|
-
headerSize = 16;
|
|
9638
|
-
} else if (size === 0) {
|
|
9639
|
-
size = source.size - offset;
|
|
9640
|
-
}
|
|
9641
|
-
if (size < headerSize) {
|
|
9642
|
-
return undefined;
|
|
9643
|
-
}
|
|
9644
|
-
if (boxType === type) {
|
|
9645
|
-
if (size - headerSize > MAX_MOOV_SIZE) {
|
|
9646
|
-
return undefined;
|
|
9647
|
-
}
|
|
9648
|
-
const payload = source.read(offset + headerSize, size - headerSize);
|
|
9649
|
-
return payload.length === size - headerSize ? payload : undefined;
|
|
9650
|
-
}
|
|
9651
|
-
offset += size;
|
|
9652
|
-
}
|
|
9653
|
-
return undefined;
|
|
9654
|
-
}
|
|
9655
|
-
|
|
9656
|
-
/** Each child box of a container payload, as `[type, payload]`. */
|
|
9657
|
-
function* boxesIn(payload) {
|
|
9658
|
-
let offset = 0;
|
|
9659
|
-
while (offset + 8 <= payload.length) {
|
|
9660
|
-
let size = payload.readUInt32BE(offset);
|
|
9661
|
-
const type = payload.toString("latin1", offset + 4, offset + 8);
|
|
9662
|
-
let headerSize = 8;
|
|
9663
|
-
if (size === 1) {
|
|
9664
|
-
if (offset + 16 > payload.length) {
|
|
9665
|
-
return;
|
|
9666
|
-
}
|
|
9667
|
-
size = Number(payload.readBigUInt64BE(offset + 8));
|
|
9668
|
-
headerSize = 16;
|
|
9669
|
-
} else if (size === 0) {
|
|
9670
|
-
size = payload.length - offset;
|
|
9671
|
-
}
|
|
9672
|
-
if (size < headerSize || offset + size > payload.length) {
|
|
9673
|
-
return;
|
|
9674
|
-
}
|
|
9675
|
-
yield [type, payload.subarray(offset + headerSize, offset + size)];
|
|
9676
|
-
offset += size;
|
|
9677
|
-
}
|
|
9678
|
-
}
|
|
9679
|
-
function childBoxes(payload, type) {
|
|
9680
|
-
if (!payload) {
|
|
9681
|
-
return [];
|
|
9682
|
-
}
|
|
9683
|
-
const found = [];
|
|
9684
|
-
for (const [boxType, child] of boxesIn(payload)) {
|
|
9685
|
-
if (boxType === type) {
|
|
9686
|
-
found.push(child);
|
|
9687
|
-
}
|
|
9688
|
-
}
|
|
9689
|
-
return found;
|
|
9690
|
-
}
|
|
9691
|
-
function childBox(payload, type) {
|
|
9692
|
-
return childBoxes(payload, type)[0];
|
|
9693
|
-
}
|
|
9694
|
-
|
|
9695
|
-
/** "Unknown" in a version 1 box. Not a literal: `0x…n` needs an ES2020 target. */
|
|
9696
|
-
const ALL_ONES_64 = BigInt("0xffffffffffffffff");
|
|
9697
|
-
function readMvhd(mvhd) {
|
|
9698
|
-
const version = mvhd[0];
|
|
9699
|
-
if (version === 1) {
|
|
9700
|
-
if (mvhd.length < 32) return undefined;
|
|
9701
|
-
const timescale = mvhd.readUInt32BE(20);
|
|
9702
|
-
const duration = mvhd.readBigUInt64BE(24);
|
|
9703
|
-
if (timescale === 0 || duration === ALL_ONES_64) {
|
|
9704
|
-
return undefined;
|
|
9705
|
-
}
|
|
9706
|
-
return {
|
|
9707
|
-
timescale,
|
|
9708
|
-
duration: Number(duration)
|
|
9709
|
-
};
|
|
9710
|
-
}
|
|
9711
|
-
if (mvhd.length < 20) return undefined;
|
|
9712
|
-
const timescale = mvhd.readUInt32BE(12);
|
|
9713
|
-
const duration = mvhd.readUInt32BE(16);
|
|
9714
|
-
// All ones means "unknown" in a version 0 box.
|
|
9715
|
-
if (timescale === 0 || duration === 0xffffffff) {
|
|
9716
|
-
return undefined;
|
|
9717
|
-
}
|
|
9718
|
-
return {
|
|
9719
|
-
timescale,
|
|
9720
|
-
duration
|
|
9721
|
-
};
|
|
9722
|
-
}
|
|
9723
|
-
function readMehd(mehd) {
|
|
9724
|
-
if (mehd[0] === 1) {
|
|
9725
|
-
return mehd.length >= 12 ? Number(mehd.readBigUInt64BE(4)) : undefined;
|
|
9726
|
-
}
|
|
9727
|
-
return mehd.length >= 8 ? mehd.readUInt32BE(4) : undefined;
|
|
9728
|
-
}
|
|
9729
|
-
|
|
9730
|
-
/**
|
|
9731
|
-
* The size a track is SHOWN at: tkhd's width and height, swapped when the
|
|
9732
|
-
* display matrix turns the picture a quarter turn — a phone records upright
|
|
9733
|
-
* video sideways and says so here, and a page lays it out as it is shown.
|
|
9734
|
-
*/
|
|
9735
|
-
function readTkhd(tkhd) {
|
|
9736
|
-
// Past the version-specific times and ids, then 16 bytes of reserved,
|
|
9737
|
-
// layer, alternate group, volume and reserved, then the 36-byte matrix.
|
|
9738
|
-
const matrixAt = tkhd[0] === 1 ? 4 + 32 + 16 : 4 + 20 + 16;
|
|
9739
|
-
const widthAt = matrixAt + 36;
|
|
9740
|
-
if (tkhd.length < widthAt + 8) {
|
|
9741
|
-
return undefined;
|
|
9742
|
-
}
|
|
9743
|
-
const a = tkhd.readInt32BE(matrixAt);
|
|
9744
|
-
const b = tkhd.readInt32BE(matrixAt + 4);
|
|
9745
|
-
const c = tkhd.readInt32BE(matrixAt + 12);
|
|
9746
|
-
const d = tkhd.readInt32BE(matrixAt + 16);
|
|
9747
|
-
const width = Math.round(tkhd.readUInt32BE(widthAt) / 0x10000);
|
|
9748
|
-
const height = Math.round(tkhd.readUInt32BE(widthAt + 4) / 0x10000);
|
|
9749
|
-
const quarterTurn = a === 0 && d === 0 && b !== 0 && c !== 0;
|
|
9750
|
-
return quarterTurn ? {
|
|
9751
|
-
width: height,
|
|
9752
|
-
height: width
|
|
9753
|
-
} : {
|
|
9754
|
-
width,
|
|
9755
|
-
height
|
|
9756
|
-
};
|
|
9757
|
-
}
|
|
9758
|
-
|
|
9759
|
-
/**
|
|
9760
|
-
* The size and length of a Matroska file (`.webm`, and `.mkv`, which is the
|
|
9761
|
-
* same container), read from its headers. Nothing is decoded.
|
|
9762
|
-
*
|
|
9763
|
-
* Hand-written for the same reason `isoBmff.ts` is: `@valbuild/server` is
|
|
9764
|
-
* loaded by every app that runs Val, and what is wanted here is a handful of
|
|
9765
|
-
* integers two levels into the file:
|
|
9766
|
-
*
|
|
9767
|
-
* EBML the header; a file without one is not Matroska
|
|
9768
|
-
* Segment > Info TimecodeScale and Duration
|
|
9769
|
-
* Segment > Tracks > TrackEntry TrackType 1 (video) >
|
|
9770
|
-
* Video PixelWidth / PixelHeight, and DisplayWidth /
|
|
9771
|
-
* DisplayHeight when they are in pixels
|
|
9772
|
-
*
|
|
9773
|
-
* Matroska is EBML: every element is a variable-length ID, a variable-length
|
|
9774
|
-
* size and a payload. Only element HEADERS are read on the way through the
|
|
9775
|
-
* Segment, and then the payloads of Info and Tracks, which are a few hundred
|
|
9776
|
-
* bytes. The Clusters — the frames, which are the file — are never read: the
|
|
9777
|
-
* walk stops at the first one.
|
|
9778
|
-
*
|
|
9779
|
-
* That is allowed because the spec (RFC 9559, section 6.3) requires the first
|
|
9780
|
-
* Info and Tracks to come before the first Cluster, or to be named by a
|
|
9781
|
-
* SeekHead that does. A file written by a browser's MediaRecorder, or by
|
|
9782
|
-
* anything else that streams, does not know its own length when it starts:
|
|
9783
|
-
* its Segment and Clusters have the "unknown" size, and its Info has no
|
|
9784
|
-
* Duration. The Segment is then read as running to the end of the file, and
|
|
9785
|
-
* the duration is left out rather than counted from the Clusters — a caller
|
|
9786
|
-
* reports it, as it does for a fragmented mp4.
|
|
9484
|
+
* Matroska is EBML: every element is a variable-length ID, a variable-length
|
|
9485
|
+
* size and a payload. Only element HEADERS are read on the way through the
|
|
9486
|
+
* Segment, and then the payloads of Info and Tracks, which are a few hundred
|
|
9487
|
+
* bytes. The Clusters — the frames, which are the file — are never read: the
|
|
9488
|
+
* walk stops at the first one.
|
|
9489
|
+
*
|
|
9490
|
+
* That is allowed because the spec (RFC 9559, section 6.3) requires the first
|
|
9491
|
+
* Info and Tracks to come before the first Cluster, or to be named by a
|
|
9492
|
+
* SeekHead that does. A file written by a browser's MediaRecorder, or by
|
|
9493
|
+
* anything else that streams, does not know its own length when it starts:
|
|
9494
|
+
* its Segment and Clusters have the "unknown" size, and its Info has no
|
|
9495
|
+
* Duration. The Segment is then read as running to the end of the file, and
|
|
9496
|
+
* the duration is left out rather than counted from the Clusters — a caller
|
|
9497
|
+
* reports it, as it does for a fragmented mp4.
|
|
9787
9498
|
*/
|
|
9788
9499
|
|
|
9789
9500
|
const EBML_HEADER = 0x1a45dfa3;
|
|
@@ -10508,7 +10219,18 @@ function getValidationErrorFileRef(validationError) {
|
|
|
10508
10219
|
}
|
|
10509
10220
|
|
|
10510
10221
|
const textEncoder$1 = new TextEncoder();
|
|
10511
|
-
async function checkRemoteRef(
|
|
10222
|
+
async function checkRemoteRef(
|
|
10223
|
+
/**
|
|
10224
|
+
* Unused: a re-derived ref keeps the host it already names (see below).
|
|
10225
|
+
* Kept so the exported signature does not change.
|
|
10226
|
+
*/
|
|
10227
|
+
_remoteHost, ref, projectRoot, schema, metadata,
|
|
10228
|
+
/**
|
|
10229
|
+
* Where the bytes come from when the ref has to be re-derived. Defaults to
|
|
10230
|
+
* a download cached under `projectRoot`; the Studio passes one that keeps
|
|
10231
|
+
* nothing on disk (`FixFiles.readRemoteFile`).
|
|
10232
|
+
*/
|
|
10233
|
+
readRemote = (ref, fileExt, fileHash) => getFileBufferFromRemote(ref, fileExt, fileHash, projectRoot)) {
|
|
10512
10234
|
const remoteRefDataRes = Internal.remote.splitRemoteRef(ref);
|
|
10513
10235
|
if (remoteRefDataRes.status === "error") {
|
|
10514
10236
|
return remoteRefDataRes;
|
|
@@ -10553,7 +10275,7 @@ async function checkRemoteRef(remoteHost, ref, projectRoot, schema, metadata) {
|
|
|
10553
10275
|
};
|
|
10554
10276
|
}
|
|
10555
10277
|
// Validation hash does not match, so we need to re-validate the remote content
|
|
10556
|
-
const fileBufferRes = await
|
|
10278
|
+
const fileBufferRes = await readRemote(ref, fileExt, remoteRefDataRes.fileHash);
|
|
10557
10279
|
|
|
10558
10280
|
// We could not download the remote file for some reason (for example, the file hash is wrong)
|
|
10559
10281
|
if (fileBufferRes.status === "error") {
|
|
@@ -10579,7 +10301,7 @@ async function checkRemoteRef(remoteHost, ref, projectRoot, schema, metadata) {
|
|
|
10579
10301
|
} catch (err) {
|
|
10580
10302
|
return {
|
|
10581
10303
|
status: "error",
|
|
10582
|
-
error: `Failed to extract metadata from remote file: ${err instanceof Error ? err.message : "Unknown error"}. Cached file: ${fileBufferRes.cachedFilePath}`
|
|
10304
|
+
error: `Failed to extract metadata from remote file: ${err instanceof Error ? err.message : "Unknown error"}${fileBufferRes.cachedFilePath ? `. Cached file: ${fileBufferRes.cachedFilePath}` : ""}`
|
|
10583
10305
|
};
|
|
10584
10306
|
}
|
|
10585
10307
|
const updatedMetadata = {
|
|
@@ -10598,7 +10320,11 @@ async function checkRemoteRef(remoteHost, ref, projectRoot, schema, metadata) {
|
|
|
10598
10320
|
return {
|
|
10599
10321
|
status: "fix-required",
|
|
10600
10322
|
metadata: updatedMetadata,
|
|
10601
|
-
ref
|
|
10323
|
+
// On the host the ref was already on. Re-deriving a ref changes what it
|
|
10324
|
+
// says about the file, never where it is served from: a Hosted project's
|
|
10325
|
+
// file is in R2 and served only at its own address, and moving a ref to
|
|
10326
|
+
// another host is a fix of its own (`remote:rewrite-host`).
|
|
10327
|
+
ref: Internal.remote.createRemoteRef(remoteRefDataRes.remoteHost, {
|
|
10602
10328
|
publicProjectId: remoteRefDataRes.projectId,
|
|
10603
10329
|
coreVersion,
|
|
10604
10330
|
bucket: remoteRefDataRes.bucket,
|
|
@@ -10617,6 +10343,12 @@ function getCachedRemoteFilePath(fileExt, currentFileHash, remoteFileCacheDir) {
|
|
|
10617
10343
|
const remoteFilePath = path.join(remoteFileCacheDir, currentFileHash + "." + fileExt);
|
|
10618
10344
|
return remoteFilePath;
|
|
10619
10345
|
}
|
|
10346
|
+
|
|
10347
|
+
/**
|
|
10348
|
+
* A remote file's bytes, and the hash they actually have — which is the one a
|
|
10349
|
+
* re-derived ref is made from, whatever hash the old ref claimed.
|
|
10350
|
+
*/
|
|
10351
|
+
|
|
10620
10352
|
async function getFileBufferFromRemote(ref, fileExt, currentFileHash, projectRoot) {
|
|
10621
10353
|
const remoteFileCacheDir = getCachedRemoteFileDir(projectRoot);
|
|
10622
10354
|
const remoteFilePath = getCachedRemoteFilePath(fileExt, currentFileHash, remoteFileCacheDir);
|
|
@@ -10655,14 +10387,55 @@ async function getFileBufferFromRemote(ref, fileExt, currentFileHash, projectRoo
|
|
|
10655
10387
|
cachedFilePath: remoteFilePath
|
|
10656
10388
|
};
|
|
10657
10389
|
}
|
|
10390
|
+
|
|
10391
|
+
/**
|
|
10392
|
+
* Download a remote file to `filePath`, from where its ref says -- and, for a
|
|
10393
|
+
* platform ref whose host fails, from the same file on the project's own
|
|
10394
|
+
* `valstart.dev` address, which always serves it (`fallbackRefOf`). The bytes
|
|
10395
|
+
* from the fallback are checked against the ref's file hash before they are
|
|
10396
|
+
* kept: a different host answering is only safe because of that check.
|
|
10397
|
+
*
|
|
10398
|
+
* Rejects with the FIRST host's error when both fail: that is the one the ref
|
|
10399
|
+
* names, and the one a person can recognise.
|
|
10400
|
+
*/
|
|
10658
10401
|
async function downloadFileFromRemote(ref, filePath) {
|
|
10659
|
-
|
|
10660
|
-
|
|
10402
|
+
try {
|
|
10403
|
+
return await downloadFromUrl(ref, filePath);
|
|
10404
|
+
} catch (err) {
|
|
10405
|
+
const fallback = fallbackRefOf(ref);
|
|
10406
|
+
const parsed = Internal.remote.splitRemoteRef(ref);
|
|
10407
|
+
if (fallback === null || parsed.status === "error") {
|
|
10408
|
+
throw err;
|
|
10409
|
+
}
|
|
10410
|
+
try {
|
|
10411
|
+
const res = await downloadFromUrl(fallback, filePath);
|
|
10412
|
+
if (res.status === "error") {
|
|
10413
|
+
throw err;
|
|
10414
|
+
}
|
|
10415
|
+
} catch {
|
|
10416
|
+
throw err;
|
|
10417
|
+
}
|
|
10418
|
+
const bytes = await fs.promises.readFile(filePath);
|
|
10419
|
+
if (Internal.remote.getFileHash(bytes) !== parsed.fileHash) {
|
|
10420
|
+
await fs.promises.unlink(filePath).catch(() => undefined);
|
|
10421
|
+
throw new Error(`${ref}. Its host failed, and the file at ${fallback} is not the one the ref names.`, {
|
|
10422
|
+
cause: err
|
|
10423
|
+
});
|
|
10424
|
+
}
|
|
10425
|
+
return {
|
|
10426
|
+
status: "success",
|
|
10427
|
+
filePath
|
|
10428
|
+
};
|
|
10429
|
+
}
|
|
10430
|
+
}
|
|
10431
|
+
async function downloadFromUrl(ref, filePath) {
|
|
10432
|
+
return new Promise((resolve, reject) => {
|
|
10433
|
+
const url = new URL(ref);
|
|
10661
10434
|
const client = url.protocol === "https:" ? https$1 : http$1;
|
|
10662
10435
|
const request = client.get(url, response => {
|
|
10663
10436
|
if (response.statusCode && response.statusCode >= 300 && response.statusCode < 400 && response.headers.location) {
|
|
10664
10437
|
// Handle redirects
|
|
10665
|
-
return
|
|
10438
|
+
return downloadFromUrl(response.headers.location, filePath).then(resolve).catch(reject);
|
|
10666
10439
|
}
|
|
10667
10440
|
if (response.statusCode == 404) {
|
|
10668
10441
|
return reject(new Error(`${ref}. File not found. The URL is most likely incorrect.`));
|
|
@@ -10683,6 +10456,45 @@ async function downloadFileFromRemote(ref, filePath) {
|
|
|
10683
10456
|
});
|
|
10684
10457
|
}
|
|
10685
10458
|
|
|
10459
|
+
/**
|
|
10460
|
+
* Where `createFixPatch` reads a project file's bytes, and where it puts a
|
|
10461
|
+
* remote file it was asked to download.
|
|
10462
|
+
*
|
|
10463
|
+
* Every path is the file's path from the project root, the way a source names
|
|
10464
|
+
* it: `/public/val/logo.png`, never an absolute path on some disk. That is the
|
|
10465
|
+
* point: a project in http mode, or one the platform serves from a Worker, has
|
|
10466
|
+
* no disk to join a path onto, and a fix that reads a file has to work there
|
|
10467
|
+
* too. `diskFixFiles` is the one the CLI and the language server use, and does
|
|
10468
|
+
* exactly what `createFixPatch` did with `fs` before this existed.
|
|
10469
|
+
*
|
|
10470
|
+
* Not yet covered, and still read from `projectRoot` directly: the CLI's
|
|
10471
|
+
* listing of a media directory for files it does not track, and
|
|
10472
|
+
* `videos:check-all-files` (`videosetCheckAllFilesPatch`), which lists one.
|
|
10473
|
+
* See `docs/plans/studio-validate.md`.
|
|
10474
|
+
*/
|
|
10475
|
+
|
|
10476
|
+
/** {@link FixFiles} over a project on disk. */
|
|
10477
|
+
function diskFixFiles(projectRoot) {
|
|
10478
|
+
const absolute = filePath => path.join(projectRoot, filePath);
|
|
10479
|
+
return {
|
|
10480
|
+
readFile: filePath => fs.promises.readFile(absolute(filePath)),
|
|
10481
|
+
readVideoMetadata: filePath => extractVideoMetadataFromFile(absolute(filePath)),
|
|
10482
|
+
// Kept under `.val/remote-file-cache`, so the next run does not download
|
|
10483
|
+
// it again.
|
|
10484
|
+
readRemoteFile: (ref, fileExt, fileHash) => getFileBufferFromRemote(ref, fileExt, fileHash, projectRoot),
|
|
10485
|
+
saveRemoteFile: async (url, filePath) => {
|
|
10486
|
+
const target = absolute(filePath);
|
|
10487
|
+
await fs.promises.mkdir(path.dirname(target), {
|
|
10488
|
+
recursive: true
|
|
10489
|
+
});
|
|
10490
|
+
const res = await downloadFileFromRemote(url, target);
|
|
10491
|
+
return res.status === "error" ? res : {
|
|
10492
|
+
status: "success"
|
|
10493
|
+
};
|
|
10494
|
+
}
|
|
10495
|
+
};
|
|
10496
|
+
}
|
|
10497
|
+
|
|
10686
10498
|
const textEncoder = new TextEncoder();
|
|
10687
10499
|
|
|
10688
10500
|
/**
|
|
@@ -10719,6 +10531,7 @@ async function openRemoteUploadSession(ctx) {
|
|
|
10719
10531
|
let publicProjectId = ctx.publicProjectId;
|
|
10720
10532
|
let remoteFileBuckets = ctx.remoteFileBuckets;
|
|
10721
10533
|
let remoteFilesCounter = ctx.remoteFilesCounter;
|
|
10534
|
+
let remoteHost = ctx.remote.remoteHost;
|
|
10722
10535
|
if (!publicProjectId || !remoteFileBuckets) {
|
|
10723
10536
|
if (!projectName) {
|
|
10724
10537
|
return fail("Project name not found. Add project name to val.config or set the VAL_PROJECT environment variable");
|
|
@@ -10731,6 +10544,7 @@ async function openRemoteUploadSession(ctx) {
|
|
|
10731
10544
|
}
|
|
10732
10545
|
publicProjectId = settingsRes.data.publicProjectId;
|
|
10733
10546
|
remoteFileBuckets = settingsRes.data.remoteFileBuckets.map(b => b.bucket);
|
|
10547
|
+
remoteHost = settingsRes.data.remoteHost ?? remoteHost;
|
|
10734
10548
|
}
|
|
10735
10549
|
if (!publicProjectId) {
|
|
10736
10550
|
return fail("Could not get public project id");
|
|
@@ -10751,7 +10565,8 @@ async function openRemoteUploadSession(ctx) {
|
|
|
10751
10565
|
publicProjectId,
|
|
10752
10566
|
bucket,
|
|
10753
10567
|
remoteFileBuckets,
|
|
10754
|
-
remoteFilesCounter
|
|
10568
|
+
remoteFilesCounter,
|
|
10569
|
+
remoteHost
|
|
10755
10570
|
}
|
|
10756
10571
|
};
|
|
10757
10572
|
}
|
|
@@ -10773,7 +10588,7 @@ async function uploadBytesToRemote(ctx, session, relativeFilePath, fileBuffer, m
|
|
|
10773
10588
|
const fileHash = Internal.remote.getFileHash(fileBuffer);
|
|
10774
10589
|
const coreVersion = Internal.VERSION.core || "unknown";
|
|
10775
10590
|
const fileExt = getFileExt(relativeFilePath);
|
|
10776
|
-
const ref = Internal.remote.createRemoteRef(
|
|
10591
|
+
const ref = Internal.remote.createRemoteRef(session.remoteHost, {
|
|
10777
10592
|
publicProjectId: session.publicProjectId,
|
|
10778
10593
|
coreVersion,
|
|
10779
10594
|
bucket: session.bucket,
|
|
@@ -12291,7 +12106,7 @@ async function videosetCheckAllFilesPatch({
|
|
|
12291
12106
|
* `alt` and anything already there are someone's, and stay.
|
|
12292
12107
|
*/
|
|
12293
12108
|
async function videosetAddMetadataPatch({
|
|
12294
|
-
|
|
12109
|
+
files,
|
|
12295
12110
|
sourcePath,
|
|
12296
12111
|
validationError,
|
|
12297
12112
|
moduleSource,
|
|
@@ -12325,7 +12140,7 @@ async function videosetAddMetadataPatch({
|
|
|
12325
12140
|
}
|
|
12326
12141
|
let metadata;
|
|
12327
12142
|
try {
|
|
12328
|
-
metadata = galleryEntryOf(key).remote ? await extractVideoMetadataFromUrl(key, nameForTypeOf(key)) : await
|
|
12143
|
+
metadata = galleryEntryOf(key).remote ? await extractVideoMetadataFromUrl(key, nameForTypeOf(key)) : await files.readVideoMetadata(key);
|
|
12329
12144
|
} catch (err) {
|
|
12330
12145
|
return fail(`Failed to read video metadata from ${key}: ${err instanceof Error ? err.message : String(err)}`);
|
|
12331
12146
|
}
|
|
@@ -12423,9 +12238,10 @@ function mediaValueOf(value) {
|
|
|
12423
12238
|
async function createFixPatch(config, apply, sourcePath, validationError, remoteFiles, moduleSource, moduleSchema) {
|
|
12424
12239
|
const remainingErrors = [];
|
|
12425
12240
|
const patch = [];
|
|
12241
|
+
const files = config.files ?? diskFixFiles(config.projectRoot);
|
|
12426
12242
|
for (const fix of validationError.fixes || []) {
|
|
12427
12243
|
if (fix === "image:check-metadata" || fix === "image:add-metadata") {
|
|
12428
|
-
const imageMetadata = await getImageMetadata(
|
|
12244
|
+
const imageMetadata = await getImageMetadata(files, validationError);
|
|
12429
12245
|
if (imageMetadata.width === undefined || imageMetadata.height === undefined) {
|
|
12430
12246
|
remainingErrors.push({
|
|
12431
12247
|
...validationError,
|
|
@@ -12492,7 +12308,7 @@ async function createFixPatch(config, apply, sourcePath, validationError, remote
|
|
|
12492
12308
|
}
|
|
12493
12309
|
}
|
|
12494
12310
|
} else if (fix === "file:add-metadata" || fix === "file:check-metadata") {
|
|
12495
|
-
const fileMetadata = await getFileMetadata(
|
|
12311
|
+
const fileMetadata = await getFileMetadata(files, validationError);
|
|
12496
12312
|
if (fileMetadata === undefined) {
|
|
12497
12313
|
remainingErrors.push({
|
|
12498
12314
|
...validationError,
|
|
@@ -12553,7 +12369,7 @@ async function createFixPatch(config, apply, sourcePath, validationError, remote
|
|
|
12553
12369
|
}
|
|
12554
12370
|
let metadata;
|
|
12555
12371
|
try {
|
|
12556
|
-
metadata = await getVideoMetadata(
|
|
12372
|
+
metadata = await getVideoMetadata(files, fileRef);
|
|
12557
12373
|
} catch (err) {
|
|
12558
12374
|
remainingErrors.push({
|
|
12559
12375
|
...validationError,
|
|
@@ -12589,7 +12405,7 @@ async function createFixPatch(config, apply, sourcePath, validationError, remote
|
|
|
12589
12405
|
// A set entry's twin of `video:add-metadata`: the file is the entry's
|
|
12590
12406
|
// KEY, and only what is missing is written.
|
|
12591
12407
|
const fixed = await videosetAddMetadataPatch({
|
|
12592
|
-
|
|
12408
|
+
files,
|
|
12593
12409
|
sourcePath,
|
|
12594
12410
|
validationError,
|
|
12595
12411
|
moduleSource,
|
|
@@ -12613,9 +12429,9 @@ async function createFixPatch(config, apply, sourcePath, validationError, remote
|
|
|
12613
12429
|
let metadata = remoteFile.metadata;
|
|
12614
12430
|
if (!metadata) {
|
|
12615
12431
|
if (fix === "image:upload-remote") {
|
|
12616
|
-
metadata = await getImageMetadata(
|
|
12432
|
+
metadata = await getImageMetadata(files, validationError);
|
|
12617
12433
|
} else if (fix === "file:upload-remote") {
|
|
12618
|
-
metadata = await getFileMetadata(
|
|
12434
|
+
metadata = await getFileMetadata(files, validationError);
|
|
12619
12435
|
}
|
|
12620
12436
|
}
|
|
12621
12437
|
if (!metadata) {
|
|
@@ -12749,11 +12565,7 @@ async function createFixPatch(config, apply, sourcePath, validationError, remote
|
|
|
12749
12565
|
});
|
|
12750
12566
|
continue;
|
|
12751
12567
|
}
|
|
12752
|
-
const
|
|
12753
|
-
await fs.promises.mkdir(path.dirname(absoluteFilePath), {
|
|
12754
|
-
recursive: true
|
|
12755
|
-
});
|
|
12756
|
-
const res = await downloadFileFromRemote(url, absoluteFilePath);
|
|
12568
|
+
const res = await files.saveRemoteFile(url, filePath);
|
|
12757
12569
|
if (res.status === "error") {
|
|
12758
12570
|
remainingErrors.push({
|
|
12759
12571
|
...validationError,
|
|
@@ -12789,10 +12601,9 @@ async function createFixPatch(config, apply, sourcePath, validationError, remote
|
|
|
12789
12601
|
// image as unreadable, and with `--fix` REMOVED it from the gallery.
|
|
12790
12602
|
continue;
|
|
12791
12603
|
}
|
|
12792
|
-
const filename = path.join(config.projectRoot, entry.localPath);
|
|
12793
12604
|
let buffer;
|
|
12794
12605
|
try {
|
|
12795
|
-
buffer =
|
|
12606
|
+
buffer = await files.readFile(entry.localPath);
|
|
12796
12607
|
} catch {
|
|
12797
12608
|
if (apply) {
|
|
12798
12609
|
const removePath = sourceToPatchPath(sourcePath).concat([entryKey]);
|
|
@@ -12805,16 +12616,16 @@ async function createFixPatch(config, apply, sourcePath, validationError, remote
|
|
|
12805
12616
|
} else {
|
|
12806
12617
|
remainingErrors.push({
|
|
12807
12618
|
...validationError,
|
|
12808
|
-
message: `Could not read file: ${
|
|
12619
|
+
message: `Could not read file: ${path.join(config.projectRoot, entry.localPath)} - file might not exist or can not be accessed`,
|
|
12809
12620
|
fixes: undefined
|
|
12810
12621
|
});
|
|
12811
12622
|
}
|
|
12812
12623
|
continue;
|
|
12813
12624
|
}
|
|
12814
12625
|
if (fix === "images:check-all-files") {
|
|
12815
|
-
const actualMetadata = await extractImageMetadata(
|
|
12626
|
+
const actualMetadata = await extractImageMetadata(entry.localPath, buffer);
|
|
12816
12627
|
const stored = storedEntry;
|
|
12817
|
-
const metadataIsCorrect =
|
|
12628
|
+
const metadataIsCorrect = mediaDirEntryMismatches("images", stored, actualMetadata).length === 0;
|
|
12818
12629
|
if (!metadataIsCorrect) {
|
|
12819
12630
|
if (apply) {
|
|
12820
12631
|
patch.push({
|
|
@@ -12839,9 +12650,9 @@ async function createFixPatch(config, apply, sourcePath, validationError, remote
|
|
|
12839
12650
|
}
|
|
12840
12651
|
}
|
|
12841
12652
|
} else if (fix === "files:check-all-files") {
|
|
12842
|
-
const actualMetadata = await extractFileMetadata(
|
|
12653
|
+
const actualMetadata = await extractFileMetadata(entry.localPath);
|
|
12843
12654
|
const stored = storedEntry;
|
|
12844
|
-
const metadataIsCorrect = stored.
|
|
12655
|
+
const metadataIsCorrect = mediaDirEntryMismatches("files", stored, actualMetadata).length === 0;
|
|
12845
12656
|
if (!metadataIsCorrect) {
|
|
12846
12657
|
if (apply) {
|
|
12847
12658
|
patch.push({
|
|
@@ -12899,7 +12710,27 @@ async function createFixPatch(config, apply, sourcePath, validationError, remote
|
|
|
12899
12710
|
schema: schemaAtPath
|
|
12900
12711
|
} = Internal.resolvePath(modulePath, moduleSource, moduleSchema);
|
|
12901
12712
|
if (schemaAtPath.type === "image" || schemaAtPath.type === "file") {
|
|
12902
|
-
|
|
12713
|
+
// First, whether the ref's host still serves its project at all: a
|
|
12714
|
+
// ref on a domain the project dropped is pointed at the project's own
|
|
12715
|
+
// address, not re-derived from bytes nobody can download from there.
|
|
12716
|
+
const moved = await movedRemoteHost(v.path, config.askHostProject);
|
|
12717
|
+
if (moved) {
|
|
12718
|
+
if (apply) {
|
|
12719
|
+
patch.push({
|
|
12720
|
+
op: "replace",
|
|
12721
|
+
path: sourceToPatchPath(sourcePath).concat(["path"]),
|
|
12722
|
+
value: moved.ref
|
|
12723
|
+
});
|
|
12724
|
+
} else {
|
|
12725
|
+
remainingErrors.push({
|
|
12726
|
+
...validationError,
|
|
12727
|
+
message: `${moved.why}. Run 'val validate --fix' to point it at ${moved.to}, which always serves it.`,
|
|
12728
|
+
fixes: ["remote:rewrite-host"]
|
|
12729
|
+
});
|
|
12730
|
+
}
|
|
12731
|
+
continue;
|
|
12732
|
+
}
|
|
12733
|
+
const res = await checkRemoteRef(config.remoteHost, v.path, config.projectRoot, schemaAtPath, v.metadata, files.readRemoteFile);
|
|
12903
12734
|
if (res.status === "success") ; else if (res.status === "error") {
|
|
12904
12735
|
remainingErrors.push({
|
|
12905
12736
|
...validationError,
|
|
@@ -12928,162 +12759,966 @@ async function createFixPatch(config, apply, sourcePath, validationError, remote
|
|
|
12928
12759
|
fixes: undefined
|
|
12929
12760
|
});
|
|
12930
12761
|
}
|
|
12931
|
-
} else {
|
|
12932
|
-
remainingErrors.push({
|
|
12933
|
-
...validationError,
|
|
12934
|
-
message: "Could not check remote ref: schema type is not image or file: " + (schemaAtPath === null || schemaAtPath === void 0 ? void 0 : schemaAtPath.type),
|
|
12935
|
-
fixes: undefined
|
|
12936
|
-
});
|
|
12762
|
+
} else {
|
|
12763
|
+
remainingErrors.push({
|
|
12764
|
+
...validationError,
|
|
12765
|
+
message: "Could not check remote ref: schema type is not image or file: " + (schemaAtPath === null || schemaAtPath === void 0 ? void 0 : schemaAtPath.type),
|
|
12766
|
+
fixes: undefined
|
|
12767
|
+
});
|
|
12768
|
+
}
|
|
12769
|
+
} else if (fix === "remote:rewrite-host") {
|
|
12770
|
+
// The new ref is derived from the old one alone, so nothing the error
|
|
12771
|
+
// carries is trusted: the value is read from the module, and only a
|
|
12772
|
+
// platform ref that is not already on the project's own address moves.
|
|
12773
|
+
const [, modulePath] = Internal.splitModuleFilePathAndModulePath(sourcePath);
|
|
12774
|
+
const current = moduleSource !== undefined && moduleSchema !== undefined ? Internal.resolvePath(modulePath, moduleSource, moduleSchema).source : undefined;
|
|
12775
|
+
const ref = typeof current === "object" && current !== null && "path" in current && typeof current.path === "string" ? current.path : undefined;
|
|
12776
|
+
const parsed = ref === undefined ? undefined : Internal.remote.splitRemoteRef(ref);
|
|
12777
|
+
const rewritten = ref !== undefined && (parsed === null || parsed === void 0 ? void 0 : parsed.status) === "success" && needsHostCheck(parsed.remoteHost, parsed.projectId) ? rewriteRefHost(ref, fallbackRemoteHost(parsed.projectId)) : null;
|
|
12778
|
+
if (rewritten === null) {
|
|
12779
|
+
remainingErrors.push({
|
|
12780
|
+
...validationError,
|
|
12781
|
+
message: "Cannot point this at the project's own address: it is not a platform ref on another host",
|
|
12782
|
+
fixes: undefined
|
|
12783
|
+
});
|
|
12784
|
+
continue;
|
|
12785
|
+
}
|
|
12786
|
+
patch.push({
|
|
12787
|
+
op: "replace",
|
|
12788
|
+
path: sourceToPatchPath(sourcePath).concat(["path"]),
|
|
12789
|
+
value: rewritten
|
|
12790
|
+
});
|
|
12791
|
+
} else if (fix === "view:check-module") {
|
|
12792
|
+
// The schema is the authority: it names the module, and there is exactly
|
|
12793
|
+
// one valid pointer. So this is written rather than reported — nobody has
|
|
12794
|
+
// a decision to make here.
|
|
12795
|
+
const [, modulePath] = Internal.splitModuleFilePathAndModulePath(sourcePath);
|
|
12796
|
+
if (moduleSource === undefined || moduleSchema === undefined) {
|
|
12797
|
+
remainingErrors.push({
|
|
12798
|
+
...validationError,
|
|
12799
|
+
message: "Unexpected error while checking a view (no module source or schema)",
|
|
12800
|
+
fixes: undefined
|
|
12801
|
+
});
|
|
12802
|
+
continue;
|
|
12803
|
+
}
|
|
12804
|
+
const {
|
|
12805
|
+
schema: schemaAtPath
|
|
12806
|
+
} = Internal.resolvePath(modulePath, moduleSource, moduleSchema);
|
|
12807
|
+
if (schemaAtPath.type !== "view") {
|
|
12808
|
+
remainingErrors.push({
|
|
12809
|
+
...validationError,
|
|
12810
|
+
message: `Could not fix view: schema at ${sourcePath} is '${schemaAtPath.type}', not a view`,
|
|
12811
|
+
fixes: undefined
|
|
12812
|
+
});
|
|
12813
|
+
continue;
|
|
12814
|
+
}
|
|
12815
|
+
if (apply) {
|
|
12816
|
+
patch.push({
|
|
12817
|
+
op: "replace",
|
|
12818
|
+
path: Internal.createPatchPath(modulePath),
|
|
12819
|
+
value: {
|
|
12820
|
+
view: schemaAtPath.moduleFilePath
|
|
12821
|
+
}
|
|
12822
|
+
});
|
|
12823
|
+
} else {
|
|
12824
|
+
remainingErrors.push({
|
|
12825
|
+
...validationError,
|
|
12826
|
+
message: `This view points at the wrong module. Expected '${schemaAtPath.moduleFilePath}'.`
|
|
12827
|
+
});
|
|
12828
|
+
}
|
|
12829
|
+
}
|
|
12830
|
+
}
|
|
12831
|
+
if (!validationError.fixes || validationError.fixes.length === 0) {
|
|
12832
|
+
remainingErrors.push(validationError);
|
|
12833
|
+
}
|
|
12834
|
+
return {
|
|
12835
|
+
patch,
|
|
12836
|
+
remainingErrors
|
|
12837
|
+
};
|
|
12838
|
+
}
|
|
12839
|
+
function getRemoteValueFromValidationError(v) {
|
|
12840
|
+
if (v.value && typeof v.value !== "object") {
|
|
12841
|
+
return {
|
|
12842
|
+
success: false,
|
|
12843
|
+
message: "Unexpected error while checking remote (not an object)"
|
|
12844
|
+
};
|
|
12845
|
+
}
|
|
12846
|
+
if (!v.value) {
|
|
12847
|
+
return {
|
|
12848
|
+
success: false,
|
|
12849
|
+
message: "Unexpected error while checking remote (no value)"
|
|
12850
|
+
};
|
|
12851
|
+
}
|
|
12852
|
+
if (typeof v.value !== "object" || v.value === null || !("path" in v.value)) {
|
|
12853
|
+
return {
|
|
12854
|
+
success: false,
|
|
12855
|
+
message: "Unexpected error while checking remote (no path in value)"
|
|
12856
|
+
};
|
|
12857
|
+
}
|
|
12858
|
+
if (typeof v.value.path !== "string") {
|
|
12859
|
+
return {
|
|
12860
|
+
success: false,
|
|
12861
|
+
message: "Unexpected error while checking remote (path is not a string)"
|
|
12862
|
+
};
|
|
12863
|
+
}
|
|
12864
|
+
// Everything but the path is what Val read from the bytes.
|
|
12865
|
+
const {
|
|
12866
|
+
path,
|
|
12867
|
+
...metadata
|
|
12868
|
+
} = v.value;
|
|
12869
|
+
return {
|
|
12870
|
+
success: true,
|
|
12871
|
+
path,
|
|
12872
|
+
metadata
|
|
12873
|
+
};
|
|
12874
|
+
}
|
|
12875
|
+
|
|
12876
|
+
/** The fields of a video that are read from its bytes, in the order written. */
|
|
12877
|
+
const VIDEO_METADATA_FIELDS = ["mimeType", "width", "height", "duration"];
|
|
12878
|
+
|
|
12879
|
+
/**
|
|
12880
|
+
* The video a `video:add-metadata` error is about, read from the module at its
|
|
12881
|
+
* path when the module is at hand — the schema there decides that it IS a
|
|
12882
|
+
* video — and from the error's own value otherwise.
|
|
12883
|
+
*/
|
|
12884
|
+
function currentVideoValue(sourcePath, validationError, moduleSource, moduleSchema) {
|
|
12885
|
+
if (moduleSource !== undefined && moduleSchema !== undefined) {
|
|
12886
|
+
const [, modulePath] = Internal.splitModuleFilePathAndModulePath(sourcePath);
|
|
12887
|
+
const resolved = Internal.resolvePath(modulePath, moduleSource, moduleSchema);
|
|
12888
|
+
if (resolved.schema.type !== "video") {
|
|
12889
|
+
return {
|
|
12890
|
+
status: "error",
|
|
12891
|
+
message: `Could not add video metadata: schema at ${sourcePath} is '${resolved.schema.type}', not a video`
|
|
12892
|
+
};
|
|
12893
|
+
}
|
|
12894
|
+
const value = mediaValueOf(resolved.source);
|
|
12895
|
+
return value ? {
|
|
12896
|
+
status: "ok",
|
|
12897
|
+
value
|
|
12898
|
+
} : {
|
|
12899
|
+
status: "error",
|
|
12900
|
+
message: "Video is not an object!"
|
|
12901
|
+
};
|
|
12902
|
+
}
|
|
12903
|
+
const value = mediaValueOf(validationError.value);
|
|
12904
|
+
return value ? {
|
|
12905
|
+
status: "ok",
|
|
12906
|
+
value
|
|
12907
|
+
} : {
|
|
12908
|
+
status: "error",
|
|
12909
|
+
message: "Video is not an object!"
|
|
12910
|
+
};
|
|
12911
|
+
}
|
|
12912
|
+
async function getVideoMetadata(files, fileRef) {
|
|
12913
|
+
// Not read into memory: only the headers are, and a video is mostly not
|
|
12914
|
+
// headers. A remote video's headers are read over HTTP, with Range
|
|
12915
|
+
// requests, so it is not downloaded either.
|
|
12916
|
+
if (Internal.isRemoteMediaPath(fileRef)) {
|
|
12917
|
+
return extractVideoMetadataFromUrl(fileRef, nameForTypeOf(fileRef));
|
|
12918
|
+
}
|
|
12919
|
+
return files.readVideoMetadata(fileRef);
|
|
12920
|
+
}
|
|
12921
|
+
async function getImageMetadata(files, validationError) {
|
|
12922
|
+
const fileRef = getValidationErrorFileRef(validationError);
|
|
12923
|
+
if (!fileRef) {
|
|
12924
|
+
// TODO:
|
|
12925
|
+
throw Error("Cannot fix image without a file reference");
|
|
12926
|
+
}
|
|
12927
|
+
return extractImageMetadata(fileRef, await files.readFile(fileRef));
|
|
12928
|
+
}
|
|
12929
|
+
async function getFileMetadata(files, validationError) {
|
|
12930
|
+
const fileRef = getValidationErrorFileRef(validationError);
|
|
12931
|
+
if (!fileRef) {
|
|
12932
|
+
// TODO:
|
|
12933
|
+
throw Error("Cannot fix file without a file reference");
|
|
12934
|
+
}
|
|
12935
|
+
return extractFileMetadata(fileRef, await files.readFile(fileRef));
|
|
12936
|
+
}
|
|
12937
|
+
|
|
12938
|
+
/**
|
|
12939
|
+
* What an image or file directory's entry records that its bytes disagree
|
|
12940
|
+
* with, as `[field, recorded, actual]`. The one comparison: the CLI's
|
|
12941
|
+
* `*:check-all-files` fix and the Studio's full check both ask it.
|
|
12942
|
+
*/
|
|
12943
|
+
function mediaDirEntryMismatches(type, stored, actual) {
|
|
12944
|
+
const fields = type === "images" ? ["width", "height", "mimeType"] : ["mimeType"];
|
|
12945
|
+
return fields.filter(field => stored[field] !== actual[field]).map(field => [field, stored[field], actual[field]]);
|
|
12946
|
+
}
|
|
12947
|
+
|
|
12948
|
+
/**
|
|
12949
|
+
* The Studio's full check, one module at a time: what `/validate` and
|
|
12950
|
+
* `/validate/fix` answer with. See `docs/plans/studio-validate.md`.
|
|
12951
|
+
*
|
|
12952
|
+
* Both take the sources the caller is looking at — base plus the patches it
|
|
12953
|
+
* names — so an error is about the content on the editor's screen, and a fix
|
|
12954
|
+
* is a patch on top of it.
|
|
12955
|
+
*/
|
|
12956
|
+
|
|
12957
|
+
/**
|
|
12958
|
+
* Every error in one module: its sources against its schema, and its local
|
|
12959
|
+
* files against their bytes.
|
|
12960
|
+
*
|
|
12961
|
+
* Not filtered the way the publish gate filters (`partitionValidationErrors`):
|
|
12962
|
+
* the errors the gate hides because "the server repairs them on save" are the
|
|
12963
|
+
* ones this check exists to show, since nothing repairs them in http mode.
|
|
12964
|
+
*
|
|
12965
|
+
* Remote files are checked too: a stale ref is an error with a fix, and a
|
|
12966
|
+
* host that does not serve them is reported once, as a host, rather than as
|
|
12967
|
+
* an error on every image it holds (`checkRemoteFiles`).
|
|
12968
|
+
*
|
|
12969
|
+
* Not yet here: the media directories' own entries, step 5 of the plan.
|
|
12970
|
+
*/
|
|
12971
|
+
async function checkModuleForStudio(ops, input) {
|
|
12972
|
+
const {
|
|
12973
|
+
moduleFilePath,
|
|
12974
|
+
schemas,
|
|
12975
|
+
sources
|
|
12976
|
+
} = input;
|
|
12977
|
+
if (!schemas[moduleFilePath]) {
|
|
12978
|
+
return {
|
|
12979
|
+
status: "unknown-module"
|
|
12980
|
+
};
|
|
12981
|
+
}
|
|
12982
|
+
const validation = await ops.validateSources(schemas, sources, undefined, [moduleFilePath]);
|
|
12983
|
+
const moduleErrors = validation.errors[moduleFilePath];
|
|
12984
|
+
if (moduleErrors !== null && moduleErrors !== void 0 && moduleErrors.invalidSource) {
|
|
12985
|
+
return {
|
|
12986
|
+
status: "invalid-source",
|
|
12987
|
+
message: moduleErrors.invalidSource.message
|
|
12988
|
+
};
|
|
12989
|
+
}
|
|
12990
|
+
const errors = {};
|
|
12991
|
+
const add = (sourcePath, found) => {
|
|
12992
|
+
errors[sourcePath] = (errors[sourcePath] ?? []).concat(found);
|
|
12993
|
+
};
|
|
12994
|
+
for (const [sourcePath, found] of Object.entries((moduleErrors === null || moduleErrors === void 0 ? void 0 : moduleErrors.validations) ?? {})) {
|
|
12995
|
+
add(sourcePath, found);
|
|
12996
|
+
}
|
|
12997
|
+
// `validateSources` hands back every local file whose metadata it could
|
|
12998
|
+
// not judge without the bytes — which is every image that has metadata at
|
|
12999
|
+
// all. Reading them is what turns "has metadata" into "has the RIGHT
|
|
13000
|
+
// metadata".
|
|
13001
|
+
const fileErrors = await ops.validateFiles(schemas, sources, validation.files, input.fileLastUpdatedByPatchId);
|
|
13002
|
+
for (const [sourcePath, found] of Object.entries(fileErrors)) {
|
|
13003
|
+
add(sourcePath, found);
|
|
13004
|
+
}
|
|
13005
|
+
const remote = await ops.validateRemoteFiles(schemas, sources, validation.remoteFiles, input.fileLastUpdatedByPatchId, input.network);
|
|
13006
|
+
for (const [sourcePath, found] of Object.entries(remote.errors)) {
|
|
13007
|
+
add(sourcePath, found);
|
|
13008
|
+
}
|
|
13009
|
+
const mediaDir = await checkMediaDirEntries(ops, input);
|
|
13010
|
+
for (const [sourcePath, found] of Object.entries(mediaDir.errors)) {
|
|
13011
|
+
add(sourcePath, found);
|
|
13012
|
+
}
|
|
13013
|
+
return {
|
|
13014
|
+
status: "ok",
|
|
13015
|
+
errors,
|
|
13016
|
+
remoteHosts: [...remote.hosts, ...mediaDir.hosts]
|
|
13017
|
+
};
|
|
13018
|
+
}
|
|
13019
|
+
|
|
13020
|
+
/** A module that is an `s.imagedir()`, `s.filedir()` or `s.videodir()`. */
|
|
13021
|
+
|
|
13022
|
+
function mediaDirOf(input) {
|
|
13023
|
+
var _input$schemas$input$;
|
|
13024
|
+
const schema = (_input$schemas$input$ = input.schemas[input.moduleFilePath]) === null || _input$schemas$input$ === void 0 ? void 0 : _input$schemas$input$["executeSerialize"]();
|
|
13025
|
+
const source = input.sources[input.moduleFilePath];
|
|
13026
|
+
if ((schema === null || schema === void 0 ? void 0 : schema.type) !== "record" || !schema.mediaType || typeof schema.dir !== "string" || !isEntries(source)) {
|
|
13027
|
+
return undefined;
|
|
13028
|
+
}
|
|
13029
|
+
return {
|
|
13030
|
+
type: schema.mediaType,
|
|
13031
|
+
dir: schema.dir,
|
|
13032
|
+
entries: source
|
|
13033
|
+
};
|
|
13034
|
+
}
|
|
13035
|
+
function isEntries(source) {
|
|
13036
|
+
return typeof source === "object" && source !== null && !Array.isArray(source);
|
|
13037
|
+
}
|
|
13038
|
+
|
|
13039
|
+
/**
|
|
13040
|
+
* A media directory's own entries against their files: what
|
|
13041
|
+
* `*:check-all-files` asks, minus the listing of the directory on disk.
|
|
13042
|
+
*
|
|
13043
|
+
* Only what the directory TRACKS is checked. Listing the directory for files
|
|
13044
|
+
* it does not track stays the CLI's: in a managed project nothing puts a file
|
|
13045
|
+
* there but the directory itself, and in a connected one the files are in the
|
|
13046
|
+
* repository, where `val validate` sees them. See
|
|
13047
|
+
* `docs/plans/studio-validate.md`, "Media directories are checked by what
|
|
13048
|
+
* they track".
|
|
13049
|
+
*
|
|
13050
|
+
* - **A local entry** is read through the ops, from the pending patch that
|
|
13051
|
+
* wrote it if there is one. Gone: the row's **Fix** removes the entry. Wrong
|
|
13052
|
+
* size or type: the row's **Fix** writes what the bytes say. Both are
|
|
13053
|
+
* `*:check-all-files`, built for that one entry (see `fixForStudio`).
|
|
13054
|
+
* - **A video entry** is only asked whether its file is there, and gets no
|
|
13055
|
+
* fix: what a videodir records is read in the browser from the upload, and
|
|
13056
|
+
* is never compared with the file (`videosetCheckAllFilesPatch` says why).
|
|
13057
|
+
* - **A remote entry** is asked of its host, as any remote file is, but only
|
|
13058
|
+
* whether the host has it (`existenceOnly`).
|
|
13059
|
+
*/
|
|
13060
|
+
async function checkMediaDirEntries(ops, input, onlyKey) {
|
|
13061
|
+
const errors = {};
|
|
13062
|
+
const mediaDir = mediaDirOf(input);
|
|
13063
|
+
if (!mediaDir) {
|
|
13064
|
+
return {
|
|
13065
|
+
errors,
|
|
13066
|
+
hosts: []
|
|
13067
|
+
};
|
|
13068
|
+
}
|
|
13069
|
+
const files = opsFixFiles(ops, input.fileLastUpdatedByPatchId);
|
|
13070
|
+
const remoteEntries = {};
|
|
13071
|
+
const type = mediaDir.type;
|
|
13072
|
+
const keys = Object.keys(mediaDir.entries).filter(key => onlyKey === undefined || key === onlyKey).sort();
|
|
13073
|
+
for (const key of keys) {
|
|
13074
|
+
const entryPath = Internal.createValPathOfItem(input.moduleFilePath, key);
|
|
13075
|
+
if (!entryPath) {
|
|
13076
|
+
continue;
|
|
13077
|
+
}
|
|
13078
|
+
const entry = galleryEntryOf(key);
|
|
13079
|
+
if (entry.remote) {
|
|
13080
|
+
remoteEntries[entryPath] = {
|
|
13081
|
+
path: key
|
|
13082
|
+
};
|
|
13083
|
+
continue;
|
|
13084
|
+
}
|
|
13085
|
+
if (type === "videos") {
|
|
13086
|
+
// A video: there or not, and nothing else. See above.
|
|
13087
|
+
const fromPatch = input.fileLastUpdatedByPatchId[entry.localPath] ?? null;
|
|
13088
|
+
const reader = await ops.openBinaryFile(entry.localPath, fromPatch);
|
|
13089
|
+
if (reader === null) {
|
|
13090
|
+
errors[entryPath] = [{
|
|
13091
|
+
message: "The file is gone, but the directory still lists it. Remove it from the directory, or put the file back.",
|
|
13092
|
+
keyError: true
|
|
13093
|
+
}];
|
|
13094
|
+
}
|
|
13095
|
+
continue;
|
|
13096
|
+
}
|
|
13097
|
+
const checkFix = type === "images" ? "images:check-all-files" : "files:check-all-files";
|
|
13098
|
+
let buffer;
|
|
13099
|
+
try {
|
|
13100
|
+
buffer = await files.readFile(entry.localPath);
|
|
13101
|
+
} catch {
|
|
13102
|
+
errors[entryPath] = [{
|
|
13103
|
+
message: "The file is gone, but the directory still lists it. Fix removes it from the directory.",
|
|
13104
|
+
keyError: true,
|
|
13105
|
+
fixes: [checkFix]
|
|
13106
|
+
}];
|
|
13107
|
+
continue;
|
|
13108
|
+
}
|
|
13109
|
+
const stored = mediaDir.entries[key];
|
|
13110
|
+
if (typeof stored !== "object" || stored === null) {
|
|
13111
|
+
// The record schema already reports an entry that is not an object.
|
|
13112
|
+
continue;
|
|
13113
|
+
}
|
|
13114
|
+
const actual = type === "images" ? await extractImageMetadata(entry.localPath, buffer) : await extractFileMetadata(entry.localPath);
|
|
13115
|
+
const mismatches = mediaDirEntryMismatches(type, {
|
|
13116
|
+
...stored
|
|
13117
|
+
}, actual);
|
|
13118
|
+
if (mismatches.length > 0) {
|
|
13119
|
+
errors[entryPath] = [{
|
|
13120
|
+
message: `Does not match the file: ${mismatches.map(([field, recorded, read]) => `${field} ${String(recorded ?? "missing")} → ${String(read)}`).join(", ")}.`,
|
|
13121
|
+
keyError: true,
|
|
13122
|
+
fixes: [checkFix]
|
|
13123
|
+
}];
|
|
13124
|
+
}
|
|
13125
|
+
}
|
|
13126
|
+
if (Object.keys(remoteEntries).length === 0) {
|
|
13127
|
+
return {
|
|
13128
|
+
errors,
|
|
13129
|
+
hosts: []
|
|
13130
|
+
};
|
|
13131
|
+
}
|
|
13132
|
+
const sampleByHost = ops.sourceMode() === "managed";
|
|
13133
|
+
const published = sampleByHost ? (await ops.getSources()).sources[input.moduleFilePath] : undefined;
|
|
13134
|
+
const remote = await checkRemoteFiles({
|
|
13135
|
+
remoteFiles: remoteEntries,
|
|
13136
|
+
existenceOnly: true,
|
|
13137
|
+
schemaAt: () => undefined,
|
|
13138
|
+
// A directory's entry IS its key: published when the published directory
|
|
13139
|
+
// has the same key.
|
|
13140
|
+
isPublished: (_, ref) => typeof published === "object" && published !== null && !Array.isArray(published) && ref in published,
|
|
13141
|
+
pendingBytes: ops.pendingBytesOf(input.fileLastUpdatedByPatchId),
|
|
13142
|
+
sampleByHost,
|
|
13143
|
+
...input.network
|
|
13144
|
+
});
|
|
13145
|
+
for (const [path, found] of Object.entries(remote.errors)) {
|
|
13146
|
+
errors[path] = (errors[path] ?? []).concat(found.map(error => ({
|
|
13147
|
+
...error,
|
|
13148
|
+
keyError: true
|
|
13149
|
+
})));
|
|
13150
|
+
}
|
|
13151
|
+
return {
|
|
13152
|
+
errors,
|
|
13153
|
+
hosts: remote.hosts
|
|
13154
|
+
};
|
|
13155
|
+
}
|
|
13156
|
+
/**
|
|
13157
|
+
* The patch that fixes one error, built from the server's own reading of the
|
|
13158
|
+
* module rather than from the error the caller saw.
|
|
13159
|
+
*
|
|
13160
|
+
* The caller names the error by where it is and which fix it wants; the error
|
|
13161
|
+
* itself is found again here. That is what keeps a stale or hand-written
|
|
13162
|
+
* request from steering `createFixPatch` (CLAUDE.md, "Adding a ValidationFix
|
|
13163
|
+
* code"), and it is what lets an error reported at a FIELD be fixed at the
|
|
13164
|
+
* media value it belongs to: `validateFiles` reports a wrong width at
|
|
13165
|
+
* `…"hero"."width"`, while the fix rewrites `…"hero"`.
|
|
13166
|
+
*
|
|
13167
|
+
* Only {@link isStudioFix} codes are built. The others need a disk or a
|
|
13168
|
+
* remote host this server may not have, and are the CLI's for now.
|
|
13169
|
+
*/
|
|
13170
|
+
async function fixForStudio(ops, input) {
|
|
13171
|
+
var _validation$errors$mo, _input$network;
|
|
13172
|
+
const {
|
|
13173
|
+
moduleFilePath,
|
|
13174
|
+
schemas,
|
|
13175
|
+
sources,
|
|
13176
|
+
fix
|
|
13177
|
+
} = input;
|
|
13178
|
+
if (!isStudioFix(fix)) {
|
|
13179
|
+
return {
|
|
13180
|
+
status: "not-in-studio"
|
|
13181
|
+
};
|
|
13182
|
+
}
|
|
13183
|
+
const schema = schemas[moduleFilePath];
|
|
13184
|
+
if (!schema) {
|
|
13185
|
+
return {
|
|
13186
|
+
status: "unknown-module"
|
|
13187
|
+
};
|
|
13188
|
+
}
|
|
13189
|
+
const validation = await ops.validateSources(schemas, sources, undefined, [moduleFilePath]);
|
|
13190
|
+
const found = await findError(ops, ((_validation$errors$mo = validation.errors[moduleFilePath]) === null || _validation$errors$mo === void 0 ? void 0 : _validation$errors$mo.validations) ?? {}, validation.files, validation.remoteFiles, input);
|
|
13191
|
+
if (!found) {
|
|
13192
|
+
return {
|
|
13193
|
+
status: "not-found"
|
|
13194
|
+
};
|
|
13195
|
+
}
|
|
13196
|
+
const res = await createFixPatch({
|
|
13197
|
+
// Nothing a Studio fix runs reads `projectRoot`: every byte goes through
|
|
13198
|
+
// `files`. An empty root means a fix that slipped past that fails to
|
|
13199
|
+
// find its file rather than reading one off this server's disk.
|
|
13200
|
+
projectRoot: "",
|
|
13201
|
+
// A re-derived ref stays on the host it was on. Moving refs to another
|
|
13202
|
+
// host is its own fix (step 6 of the plan), not a side effect of this.
|
|
13203
|
+
remoteHost: found.remoteHost ?? DEFAULT_VAL_REMOTE_HOST,
|
|
13204
|
+
files: opsFixFiles(ops, input.fileLastUpdatedByPatchId),
|
|
13205
|
+
askHostProject: (_input$network = input.network) === null || _input$network === void 0 ? void 0 : _input$network.askHostProject
|
|
13206
|
+
}, true, found.sourcePath, found.error, {}, found.moduleSource ?? sources[moduleFilePath], schema["executeSerialize"]());
|
|
13207
|
+
if (!res) {
|
|
13208
|
+
return {
|
|
13209
|
+
status: "not-found"
|
|
13210
|
+
};
|
|
13211
|
+
}
|
|
13212
|
+
return {
|
|
13213
|
+
status: "ok",
|
|
13214
|
+
patch: res.patch,
|
|
13215
|
+
remainingErrors: res.remainingErrors
|
|
13216
|
+
};
|
|
13217
|
+
}
|
|
13218
|
+
async function findError(ops, validations, files, remoteFiles, input) {
|
|
13219
|
+
var _validations$sourcePa;
|
|
13220
|
+
const {
|
|
13221
|
+
sourcePath,
|
|
13222
|
+
fix
|
|
13223
|
+
} = input;
|
|
13224
|
+
const reported = (_validations$sourcePa = validations[sourcePath]) === null || _validations$sourcePa === void 0 ? void 0 : _validations$sourcePa.find(error => {
|
|
13225
|
+
var _error$fixes;
|
|
13226
|
+
return (_error$fixes = error.fixes) === null || _error$fixes === void 0 ? void 0 : _error$fixes.includes(fix);
|
|
13227
|
+
});
|
|
13228
|
+
if (reported) {
|
|
13229
|
+
return {
|
|
13230
|
+
sourcePath,
|
|
13231
|
+
error: reported
|
|
13232
|
+
};
|
|
13233
|
+
}
|
|
13234
|
+
if (fix === "images:check-all-files" || fix === "files:check-all-files") {
|
|
13235
|
+
var _checked$errors$sourc;
|
|
13236
|
+
// One entry, checked again on its own: a row whose file someone has since
|
|
13237
|
+
// put back, or whose entry someone already fixed, is "not found".
|
|
13238
|
+
const mediaDir = mediaDirOf(input);
|
|
13239
|
+
const [, modulePath] = Internal.splitModuleFilePathAndModulePath(sourcePath);
|
|
13240
|
+
const [key, ...rest] = modulePath ? Internal.splitModulePath(modulePath) : [];
|
|
13241
|
+
if (!mediaDir || key === undefined || rest.length > 0) {
|
|
13242
|
+
return undefined;
|
|
13243
|
+
}
|
|
13244
|
+
const checked = await checkMediaDirEntries(ops, input, key);
|
|
13245
|
+
const error = (_checked$errors$sourc = checked.errors[sourcePath]) === null || _checked$errors$sourc === void 0 ? void 0 : _checked$errors$sourc.find(candidate => {
|
|
13246
|
+
var _candidate$fixes;
|
|
13247
|
+
return (_candidate$fixes = candidate.fixes) === null || _candidate$fixes === void 0 ? void 0 : _candidate$fixes.includes(fix);
|
|
13248
|
+
});
|
|
13249
|
+
if (!error) {
|
|
13250
|
+
return undefined;
|
|
13251
|
+
}
|
|
13252
|
+
return {
|
|
13253
|
+
// `createFixPatch` fixes a directory at its root, every entry it is
|
|
13254
|
+
// handed: so it is handed this one.
|
|
13255
|
+
sourcePath: Internal.parentOfSourcePath(sourcePath),
|
|
13256
|
+
error: {
|
|
13257
|
+
message: error.message,
|
|
13258
|
+
value: {
|
|
13259
|
+
dir: mediaDir.dir,
|
|
13260
|
+
type: mediaDir.type
|
|
13261
|
+
},
|
|
13262
|
+
fixes: [fix]
|
|
13263
|
+
},
|
|
13264
|
+
moduleSource: {
|
|
13265
|
+
[key]: mediaDir.entries[key]
|
|
13266
|
+
}
|
|
13267
|
+
};
|
|
13268
|
+
}
|
|
13269
|
+
if (fix === "image:check-remote" || fix === "file:check-remote" || fix === "remote:rewrite-host") {
|
|
13270
|
+
var _checked$errors$sourc2;
|
|
13271
|
+
// Every remote value is in `remoteFiles`, sound or not, so check it again
|
|
13272
|
+
// before rewriting it: a ref someone already fixed, or whose host serves
|
|
13273
|
+
// the project again, is "not found", not a download and an identical ref.
|
|
13274
|
+
const value = remoteFiles[sourcePath];
|
|
13275
|
+
if (!value) {
|
|
13276
|
+
return undefined;
|
|
13277
|
+
}
|
|
13278
|
+
const checked = await ops.validateRemoteFiles(input.schemas, input.sources, {
|
|
13279
|
+
[sourcePath]: value
|
|
13280
|
+
}, input.fileLastUpdatedByPatchId, input.network);
|
|
13281
|
+
const found = (_checked$errors$sourc2 = checked.errors[sourcePath]) === null || _checked$errors$sourc2 === void 0 ? void 0 : _checked$errors$sourc2.find(error => {
|
|
13282
|
+
var _error$fixes2;
|
|
13283
|
+
return (_error$fixes2 = error.fixes) === null || _error$fixes2 === void 0 ? void 0 : _error$fixes2.includes(fix);
|
|
13284
|
+
});
|
|
13285
|
+
const parsed = Internal.remote.splitRemoteRef(value.path);
|
|
13286
|
+
if (!found || parsed.status === "error") {
|
|
13287
|
+
return undefined;
|
|
13288
|
+
}
|
|
13289
|
+
return {
|
|
13290
|
+
sourcePath,
|
|
13291
|
+
error: found,
|
|
13292
|
+
remoteHost: parsed.remoteHost
|
|
13293
|
+
};
|
|
13294
|
+
}
|
|
13295
|
+
if (fix !== "image:check-metadata" && fix !== "file:check-metadata") {
|
|
13296
|
+
return undefined;
|
|
13297
|
+
}
|
|
13298
|
+
// A metadata mismatch is reported by `validateFiles`, at the field that is
|
|
13299
|
+
// wrong, and always as `image:check-metadata` — even for a file, which has
|
|
13300
|
+
// no width to read. So the media value is found from the field, and the fix
|
|
13301
|
+
// is chosen by what its schema says it is.
|
|
13302
|
+
for (const [mediaPathS, value] of Object.entries(files)) {
|
|
13303
|
+
const mediaPath = mediaPathS;
|
|
13304
|
+
const type = mediaTypeAt(mediaPath, input);
|
|
13305
|
+
if (!type) {
|
|
13306
|
+
continue;
|
|
13307
|
+
}
|
|
13308
|
+
const fieldPaths = getFieldsForType(type).map(field => Internal.createValPathOfItem(mediaPath, field));
|
|
13309
|
+
if (mediaPath === sourcePath || fieldPaths.includes(sourcePath)) {
|
|
13310
|
+
// Every image with metadata is in `files`, right or wrong — being there
|
|
13311
|
+
// only means "not yet compared with its bytes". So compare it, and fix
|
|
13312
|
+
// only one that is actually wrong: a stale request for an image someone
|
|
13313
|
+
// has since fixed is "not found", not an empty patch.
|
|
13314
|
+
const mismatches = await ops.validateFiles(input.schemas, input.sources, {
|
|
13315
|
+
[mediaPath]: value
|
|
13316
|
+
}, input.fileLastUpdatedByPatchId);
|
|
13317
|
+
if (Object.keys(mismatches).length === 0) {
|
|
13318
|
+
return undefined;
|
|
13319
|
+
}
|
|
13320
|
+
return {
|
|
13321
|
+
sourcePath: mediaPath,
|
|
13322
|
+
error: {
|
|
13323
|
+
message: "Metadata does not match the file",
|
|
13324
|
+
value,
|
|
13325
|
+
fixes: [type === "image" ? "image:check-metadata" : "file:check-metadata"]
|
|
13326
|
+
}
|
|
13327
|
+
};
|
|
13328
|
+
}
|
|
13329
|
+
}
|
|
13330
|
+
return undefined;
|
|
13331
|
+
}
|
|
13332
|
+
function mediaTypeAt(sourcePath, {
|
|
13333
|
+
moduleFilePath,
|
|
13334
|
+
schemas,
|
|
13335
|
+
sources
|
|
13336
|
+
}) {
|
|
13337
|
+
const schema = schemas[moduleFilePath];
|
|
13338
|
+
if (!schema) {
|
|
13339
|
+
return undefined;
|
|
13340
|
+
}
|
|
13341
|
+
const [, modulePath] = Internal.splitModuleFilePathAndModulePath(sourcePath);
|
|
13342
|
+
try {
|
|
13343
|
+
const {
|
|
13344
|
+
schema: atPath
|
|
13345
|
+
} = Internal.resolvePath(modulePath, sources[moduleFilePath], schema["executeSerialize"]());
|
|
13346
|
+
return atPath.type === "image" || atPath.type === "file" ? atPath.type : undefined;
|
|
13347
|
+
} catch {
|
|
13348
|
+
return undefined;
|
|
13349
|
+
}
|
|
13350
|
+
}
|
|
13351
|
+
|
|
13352
|
+
/**
|
|
13353
|
+
* {@link FixFiles} over whatever store this server has: the disk in fs mode,
|
|
13354
|
+
* the content service in http mode. A file a pending patch wrote is read from
|
|
13355
|
+
* that patch, as `validateFiles` reads it, so a fix is made from the bytes
|
|
13356
|
+
* the check was made from.
|
|
13357
|
+
*/
|
|
13358
|
+
function opsFixFiles(ops, fileLastUpdatedByPatchId) {
|
|
13359
|
+
const readFile = async filePath => {
|
|
13360
|
+
const fromPatch = fileLastUpdatedByPatchId[filePath];
|
|
13361
|
+
const buffer = fromPatch ? await ops.getBase64EncodedBinaryFileFromPatch(filePath, fromPatch.patchId, fromPatch.remote) : await ops.getBinaryFile(filePath);
|
|
13362
|
+
if (buffer === null) {
|
|
13363
|
+
throw new Error(`Could not read ${filePath}`);
|
|
13364
|
+
}
|
|
13365
|
+
// A view of the same bytes, not a copy: the readers below take a Buffer,
|
|
13366
|
+
// and ops answer with a Uint8Array because they run in a browser too.
|
|
13367
|
+
return Buffer.from(buffer.buffer, buffer.byteOffset, buffer.byteLength);
|
|
13368
|
+
};
|
|
13369
|
+
return {
|
|
13370
|
+
readFile,
|
|
13371
|
+
readVideoMetadata: async filePath => extractVideoMetadata(filePath, await readFile(filePath)),
|
|
13372
|
+
readRemoteFile: readRemoteFileOverHttp,
|
|
13373
|
+
saveRemoteFile: async () => ({
|
|
13374
|
+
status: "error",
|
|
13375
|
+
error: "Downloading a remote file is not available in the Studio yet. Run `val validate --fix` in the project."
|
|
13376
|
+
})
|
|
13377
|
+
};
|
|
13378
|
+
}
|
|
13379
|
+
|
|
13380
|
+
/**
|
|
13381
|
+
* A remote file's bytes, downloaded and kept nowhere: this server may have no
|
|
13382
|
+
* disk to keep them on (http mode, a Worker on the platform).
|
|
13383
|
+
*
|
|
13384
|
+
* A platform ref whose host fails is read from the project's own
|
|
13385
|
+
* `valstart.dev` address instead (`fallbackRefOf`), and those bytes are kept
|
|
13386
|
+
* only if they hash to what the ref names: a different host answering is safe
|
|
13387
|
+
* because of that check, and only because of it.
|
|
13388
|
+
*/
|
|
13389
|
+
const readRemoteFileOverHttp = async ref => {
|
|
13390
|
+
const first = await fetchBytes(ref);
|
|
13391
|
+
if (first.status === "success") {
|
|
13392
|
+
return first;
|
|
13393
|
+
}
|
|
13394
|
+
const fallback = fallbackRefOf(ref);
|
|
13395
|
+
const parsed = Internal.remote.splitRemoteRef(ref);
|
|
13396
|
+
if (fallback === null || parsed.status === "error") {
|
|
13397
|
+
return first;
|
|
13398
|
+
}
|
|
13399
|
+
const second = await fetchBytes(fallback);
|
|
13400
|
+
if (second.status === "success" && second.fileHash === parsed.fileHash) {
|
|
13401
|
+
return second;
|
|
13402
|
+
}
|
|
13403
|
+
return first;
|
|
13404
|
+
};
|
|
13405
|
+
async function fetchBytes(url) {
|
|
13406
|
+
try {
|
|
13407
|
+
const res = await fetch(url, {
|
|
13408
|
+
signal: AbortSignal.timeout(30_000)
|
|
13409
|
+
});
|
|
13410
|
+
if (!res.ok) {
|
|
13411
|
+
return {
|
|
13412
|
+
status: "error",
|
|
13413
|
+
error: `The file could not be downloaded from its address: HTTP ${res.status}.`
|
|
13414
|
+
};
|
|
13415
|
+
}
|
|
13416
|
+
const fileBuffer = Buffer.from(await res.arrayBuffer());
|
|
13417
|
+
return {
|
|
13418
|
+
status: "success",
|
|
13419
|
+
fileBuffer,
|
|
13420
|
+
fileHash: Internal.remote.getFileHash(fileBuffer)
|
|
13421
|
+
};
|
|
13422
|
+
} catch (err) {
|
|
13423
|
+
return {
|
|
13424
|
+
status: "error",
|
|
13425
|
+
error: `The file could not be downloaded from its address: ${err instanceof Error ? err.message : "unknown error"}.`
|
|
13426
|
+
};
|
|
13427
|
+
}
|
|
13428
|
+
}
|
|
13429
|
+
|
|
13430
|
+
/** The platform every server on Node runs with. See {@link ValServerPlatform}. */
|
|
13431
|
+
const nodePlatform = {
|
|
13432
|
+
createOps: createValOps,
|
|
13433
|
+
probeUrl: url => probeUrl(url),
|
|
13434
|
+
jwt: {
|
|
13435
|
+
verifyJwt,
|
|
13436
|
+
decodeJwtWithoutVerifying,
|
|
13437
|
+
encodeJwt,
|
|
13438
|
+
getExpire
|
|
13439
|
+
},
|
|
13440
|
+
bytesToBase64: bytes => Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString("base64"),
|
|
13441
|
+
readValLoginToken,
|
|
13442
|
+
resolveRemoteFileAuth,
|
|
13443
|
+
studioValidation: {
|
|
13444
|
+
checkModuleForStudio,
|
|
13445
|
+
fixForStudio
|
|
13446
|
+
}
|
|
13447
|
+
};
|
|
13448
|
+
|
|
13449
|
+
async function createValServer(valModules, route, opts, config, callbacks, formatter,
|
|
13450
|
+
/**
|
|
13451
|
+
* Called after a save has applied its patches. EXPERIMENTAL — see
|
|
13452
|
+
* `ValServerOptions.commitPrepared`.
|
|
13453
|
+
*/
|
|
13454
|
+
commitPrepared,
|
|
13455
|
+
/**
|
|
13456
|
+
* What a publish does in http mode. EXPERIMENTAL — see
|
|
13457
|
+
* `ValServerOptions.publishOverride`.
|
|
13458
|
+
*/
|
|
13459
|
+
publishOverride) {
|
|
13460
|
+
const valServerConfig = await initHandlerOptions(route, opts, config);
|
|
13461
|
+
return ValServer(valModules, {
|
|
13462
|
+
formatter,
|
|
13463
|
+
commitPrepared,
|
|
13464
|
+
publishOverride,
|
|
13465
|
+
...valServerConfig
|
|
13466
|
+
}, callbacks, nodePlatform);
|
|
13467
|
+
}
|
|
13468
|
+
|
|
13469
|
+
// TODO: remove
|
|
13470
|
+
/**
|
|
13471
|
+
* `fs` and `path` are imported INSIDE this function, not at the top of the file.
|
|
13472
|
+
*
|
|
13473
|
+
* This is the only thing in this module that touches either, and it is a local
|
|
13474
|
+
* development convenience: scanning upwards for a `.git` to guess the commit and
|
|
13475
|
+
* branch. A static import put `fs` in the module graph of everything reaching
|
|
13476
|
+
* `createValApiRouter` -- which is every server integration, including ones that
|
|
13477
|
+
* run where there is no filesystem. Workerd provides no `fs`, so such a build
|
|
13478
|
+
* could not be bundled at all without stubbing it.
|
|
13479
|
+
*
|
|
13480
|
+
* The `await import` costs nothing here: the only caller is the CLI, on a
|
|
13481
|
+
* machine that has both.
|
|
13482
|
+
*/
|
|
13483
|
+
async function safeReadGit(cwd) {
|
|
13484
|
+
const {
|
|
13485
|
+
promises: fs
|
|
13486
|
+
} = await import('fs');
|
|
13487
|
+
const path = await import('path');
|
|
13488
|
+
async function findGitHead(currentDir, depth) {
|
|
13489
|
+
const gitHeadPath = path.join(currentDir, ".git", "HEAD");
|
|
13490
|
+
if (depth > 1000) {
|
|
13491
|
+
console.error(`Reached max depth while scanning for .git folder. Current working dir: ${cwd}.`);
|
|
13492
|
+
return {
|
|
13493
|
+
commit: undefined,
|
|
13494
|
+
branch: undefined
|
|
13495
|
+
};
|
|
13496
|
+
}
|
|
13497
|
+
try {
|
|
13498
|
+
const headContents = await fs.readFile(gitHeadPath, "utf-8");
|
|
13499
|
+
const match = headContents.match(/^ref: refs\/heads\/(.+)/);
|
|
13500
|
+
if (match) {
|
|
13501
|
+
const branchName = match[1];
|
|
13502
|
+
return {
|
|
13503
|
+
branch: branchName,
|
|
13504
|
+
commit: await readCommit(currentDir, branchName)
|
|
13505
|
+
};
|
|
13506
|
+
} else {
|
|
13507
|
+
return {
|
|
13508
|
+
commit: undefined,
|
|
13509
|
+
branch: undefined
|
|
13510
|
+
};
|
|
13511
|
+
}
|
|
13512
|
+
} catch {
|
|
13513
|
+
const parentDir = path.dirname(currentDir);
|
|
13514
|
+
|
|
13515
|
+
// We've reached the root directory
|
|
13516
|
+
if (parentDir === currentDir) {
|
|
13517
|
+
return {
|
|
13518
|
+
commit: undefined,
|
|
13519
|
+
branch: undefined
|
|
13520
|
+
};
|
|
13521
|
+
}
|
|
13522
|
+
return findGitHead(parentDir, depth + 1);
|
|
13523
|
+
}
|
|
13524
|
+
}
|
|
13525
|
+
try {
|
|
13526
|
+
return findGitHead(cwd, 0);
|
|
13527
|
+
} catch (err) {
|
|
13528
|
+
console.error("Error while reading .git", err);
|
|
13529
|
+
return {
|
|
13530
|
+
commit: undefined,
|
|
13531
|
+
branch: undefined
|
|
13532
|
+
};
|
|
13533
|
+
}
|
|
13534
|
+
}
|
|
13535
|
+
|
|
13536
|
+
/** Only reached from {@link safeReadGit}; same reason for the local imports. */
|
|
13537
|
+
async function readCommit(gitDir, branchName) {
|
|
13538
|
+
const {
|
|
13539
|
+
promises: fs
|
|
13540
|
+
} = await import('fs');
|
|
13541
|
+
const path = await import('path');
|
|
13542
|
+
try {
|
|
13543
|
+
return (await fs.readFile(path.join(gitDir, ".git", "refs", "heads", branchName), "utf-8")).trim();
|
|
13544
|
+
} catch {
|
|
13545
|
+
return undefined;
|
|
13546
|
+
}
|
|
13547
|
+
}
|
|
13548
|
+
function createValApiRouter(route, valServerPromise, convert) {
|
|
13549
|
+
const uiRequestHandler = createUIRequestHandler();
|
|
13550
|
+
return createValRequestHandler(route, valServerPromise, convert, uiRequestHandler);
|
|
13551
|
+
}
|
|
13552
|
+
|
|
13553
|
+
const JsFileLookupMapping = [
|
|
13554
|
+
// NOTE: first one matching will be used
|
|
13555
|
+
[".cjs.d.ts", [".esm.js", ".mjs.js"]], [".cjs.js", [".esm.js", ".mjs.js"]], [".cjs", [".mjs"]], [".d.ts", [".js", ".esm.js", ".mjs.js"]]];
|
|
13556
|
+
const MAX_CACHE_SIZE = 100 * 1024 * 1024; // 100 mb
|
|
13557
|
+
const MAX_OBJECT_KEY_SIZE = 2 ** 27; // https://stackoverflow.com/questions/13367391/is-there-a-limit-on-length-of-the-key-string-in-js-object
|
|
13558
|
+
|
|
13559
|
+
class ValModuleLoader {
|
|
13560
|
+
cache;
|
|
13561
|
+
cacheSize;
|
|
13562
|
+
constructor(projectRoot, compilerOptions,
|
|
13563
|
+
// TODO: remove this?
|
|
13564
|
+
sourceFileHandler, host = {
|
|
13565
|
+
...ts.sys,
|
|
13566
|
+
writeFile: (fileName, data, encoding) => {
|
|
13567
|
+
fs.mkdirSync(path.dirname(fileName), {
|
|
13568
|
+
recursive: true
|
|
13569
|
+
});
|
|
13570
|
+
fs.writeFileSync(fileName, typeof data === "string" ? data : new Uint8Array(data), encoding);
|
|
13571
|
+
},
|
|
13572
|
+
rmFile: fs.rmSync,
|
|
13573
|
+
readBuffer: fileName => {
|
|
13574
|
+
try {
|
|
13575
|
+
return fs.readFileSync(fileName);
|
|
13576
|
+
} catch {
|
|
13577
|
+
return undefined;
|
|
13578
|
+
}
|
|
13579
|
+
}
|
|
13580
|
+
}, disableCache = false) {
|
|
13581
|
+
this.projectRoot = projectRoot;
|
|
13582
|
+
this.compilerOptions = compilerOptions;
|
|
13583
|
+
this.sourceFileHandler = sourceFileHandler;
|
|
13584
|
+
this.host = host;
|
|
13585
|
+
this.disableCache = disableCache;
|
|
13586
|
+
this.cache = {};
|
|
13587
|
+
this.cacheSize = 0;
|
|
13588
|
+
}
|
|
13589
|
+
getModule(modulePath) {
|
|
13590
|
+
if (!modulePath) {
|
|
13591
|
+
throw Error(`Illegal module path: "${modulePath}"`);
|
|
13592
|
+
}
|
|
13593
|
+
const code = this.host.readFile(modulePath);
|
|
13594
|
+
if (!code) {
|
|
13595
|
+
throw Error(`Could not read file "${modulePath}"`);
|
|
13596
|
+
}
|
|
13597
|
+
let compiledCode;
|
|
13598
|
+
if (this.cache[code] && !this.disableCache) {
|
|
13599
|
+
// TODO: use hash instead of code as key
|
|
13600
|
+
compiledCode = this.cache[code];
|
|
13601
|
+
} else {
|
|
13602
|
+
compiledCode = transform(code, {
|
|
13603
|
+
filePath: modulePath,
|
|
13604
|
+
disableESTransforms: true,
|
|
13605
|
+
transforms: ["typescript"]
|
|
13606
|
+
}).code;
|
|
13607
|
+
if (!this.disableCache) {
|
|
13608
|
+
if (this.cacheSize > MAX_CACHE_SIZE) {
|
|
13609
|
+
console.warn("Cache size exceeded, clearing cache");
|
|
13610
|
+
this.cache = {};
|
|
13611
|
+
this.cacheSize = 0;
|
|
13612
|
+
}
|
|
13613
|
+
if (code.length < MAX_OBJECT_KEY_SIZE) {
|
|
13614
|
+
this.cache[code] = compiledCode;
|
|
13615
|
+
this.cacheSize += code.length + compiledCode.length; // code is mostly ASCII so 1 byte per char
|
|
13616
|
+
}
|
|
12937
13617
|
}
|
|
12938
|
-
}
|
|
12939
|
-
|
|
12940
|
-
|
|
12941
|
-
|
|
12942
|
-
|
|
12943
|
-
|
|
12944
|
-
|
|
12945
|
-
|
|
12946
|
-
|
|
12947
|
-
|
|
12948
|
-
|
|
12949
|
-
|
|
13618
|
+
}
|
|
13619
|
+
return compiledCode;
|
|
13620
|
+
}
|
|
13621
|
+
resolveModulePath(containingFilePath, requestedModuleName) {
|
|
13622
|
+
var _this$host$realpath, _this$host;
|
|
13623
|
+
let sourceFileName = this.sourceFileHandler.resolveSourceModulePath(containingFilePath, requestedModuleName);
|
|
13624
|
+
if (requestedModuleName === "@vercel/stega") {
|
|
13625
|
+
sourceFileName = this.sourceFileHandler.resolveSourceModulePath(containingFilePath, "@vercel/stega").replace("stega/dist", "stega/dist/esm");
|
|
13626
|
+
}
|
|
13627
|
+
const matches = this.findMatchingJsFile(sourceFileName);
|
|
13628
|
+
if (matches.match === false) {
|
|
13629
|
+
let debugInfo = "";
|
|
13630
|
+
if (sourceFileName.includes("val.config")) {
|
|
13631
|
+
debugInfo = `\n@valbuild directory scan:\n${this.host.readDirectory("/", ["js", "ts", "json"], [], ["**/@valbuild/*"]).join("\n")}`;
|
|
12950
13632
|
}
|
|
12951
|
-
|
|
12952
|
-
|
|
12953
|
-
|
|
12954
|
-
|
|
12955
|
-
|
|
12956
|
-
|
|
12957
|
-
|
|
12958
|
-
|
|
12959
|
-
|
|
12960
|
-
|
|
13633
|
+
throw Error(`Could not find matching js file for module "${requestedModuleName}" requested by: "${containingFilePath}". Tried:\n${matches.tried.join("\n")}${debugInfo}`);
|
|
13634
|
+
}
|
|
13635
|
+
const filePath = matches.match;
|
|
13636
|
+
// resolve all symlinks (preconstruct for example symlinks the dist folder)
|
|
13637
|
+
const followedPath = ((_this$host$realpath = (_this$host = this.host).realpath) === null || _this$host$realpath === void 0 ? void 0 : _this$host$realpath.call(_this$host, filePath)) ?? filePath;
|
|
13638
|
+
if (!followedPath) {
|
|
13639
|
+
throw Error(`File path was empty: "${filePath}", containing file: "${containingFilePath}", requested module: "${requestedModuleName}"`);
|
|
13640
|
+
}
|
|
13641
|
+
return followedPath;
|
|
13642
|
+
}
|
|
13643
|
+
findMatchingJsFile(filePath) {
|
|
13644
|
+
let requiresReplacements = false;
|
|
13645
|
+
for (const [currentEnding] of JsFileLookupMapping) {
|
|
13646
|
+
if (filePath.endsWith(currentEnding)) {
|
|
13647
|
+
requiresReplacements = true;
|
|
13648
|
+
break;
|
|
12961
13649
|
}
|
|
12962
|
-
|
|
12963
|
-
|
|
12964
|
-
|
|
12965
|
-
|
|
12966
|
-
|
|
12967
|
-
|
|
13650
|
+
}
|
|
13651
|
+
// avoid unnecessary calls to fileExists if we don't need to replace anything
|
|
13652
|
+
if (!requiresReplacements) {
|
|
13653
|
+
if (this.host.fileExists(filePath)) {
|
|
13654
|
+
return {
|
|
13655
|
+
match: filePath
|
|
13656
|
+
};
|
|
13657
|
+
}
|
|
13658
|
+
}
|
|
13659
|
+
const tried = [];
|
|
13660
|
+
for (const [currentEnding, replacements] of JsFileLookupMapping) {
|
|
13661
|
+
if (filePath.endsWith(currentEnding)) {
|
|
13662
|
+
for (const replacement of replacements) {
|
|
13663
|
+
const newFilePath = filePath.slice(0, -currentEnding.length) + replacement;
|
|
13664
|
+
if (this.host.fileExists(newFilePath)) {
|
|
13665
|
+
return {
|
|
13666
|
+
match: newFilePath
|
|
13667
|
+
};
|
|
13668
|
+
} else {
|
|
13669
|
+
tried.push(newFilePath);
|
|
12968
13670
|
}
|
|
12969
|
-
}
|
|
12970
|
-
} else {
|
|
12971
|
-
remainingErrors.push({
|
|
12972
|
-
...validationError,
|
|
12973
|
-
message: `This view points at the wrong module. Expected '${schemaAtPath.moduleFilePath}'.`
|
|
12974
|
-
});
|
|
13671
|
+
}
|
|
12975
13672
|
}
|
|
12976
13673
|
}
|
|
12977
|
-
}
|
|
12978
|
-
if (!validationError.fixes || validationError.fixes.length === 0) {
|
|
12979
|
-
remainingErrors.push(validationError);
|
|
12980
|
-
}
|
|
12981
|
-
return {
|
|
12982
|
-
patch,
|
|
12983
|
-
remainingErrors
|
|
12984
|
-
};
|
|
12985
|
-
}
|
|
12986
|
-
function getRemoteValueFromValidationError(v) {
|
|
12987
|
-
if (v.value && typeof v.value !== "object") {
|
|
12988
|
-
return {
|
|
12989
|
-
success: false,
|
|
12990
|
-
message: "Unexpected error while checking remote (not an object)"
|
|
12991
|
-
};
|
|
12992
|
-
}
|
|
12993
|
-
if (!v.value) {
|
|
12994
|
-
return {
|
|
12995
|
-
success: false,
|
|
12996
|
-
message: "Unexpected error while checking remote (no value)"
|
|
12997
|
-
};
|
|
12998
|
-
}
|
|
12999
|
-
if (typeof v.value !== "object" || v.value === null || !("path" in v.value)) {
|
|
13000
|
-
return {
|
|
13001
|
-
success: false,
|
|
13002
|
-
message: "Unexpected error while checking remote (no path in value)"
|
|
13003
|
-
};
|
|
13004
|
-
}
|
|
13005
|
-
if (typeof v.value.path !== "string") {
|
|
13006
13674
|
return {
|
|
13007
|
-
|
|
13008
|
-
|
|
13675
|
+
match: false,
|
|
13676
|
+
tried: tried.concat(filePath)
|
|
13009
13677
|
};
|
|
13010
13678
|
}
|
|
13011
|
-
// Everything but the path is what Val read from the bytes.
|
|
13012
|
-
const {
|
|
13013
|
-
path,
|
|
13014
|
-
...metadata
|
|
13015
|
-
} = v.value;
|
|
13016
|
-
return {
|
|
13017
|
-
success: true,
|
|
13018
|
-
path,
|
|
13019
|
-
metadata
|
|
13020
|
-
};
|
|
13021
13679
|
}
|
|
13022
13680
|
|
|
13023
|
-
/** The fields of a video that are read from its bytes, in the order written. */
|
|
13024
|
-
const VIDEO_METADATA_FIELDS = ["mimeType", "width", "height", "duration"];
|
|
13025
|
-
|
|
13026
13681
|
/**
|
|
13027
|
-
*
|
|
13028
|
-
*
|
|
13029
|
-
* video — and from the error's own value otherwise.
|
|
13682
|
+
* An implementation of methods in the various ts.*Host interfaces
|
|
13683
|
+
* that uses ValFS to resolve modules and read/write files.
|
|
13030
13684
|
*/
|
|
13031
|
-
|
|
13032
|
-
|
|
13033
|
-
|
|
13034
|
-
|
|
13035
|
-
|
|
13036
|
-
|
|
13037
|
-
|
|
13038
|
-
|
|
13039
|
-
|
|
13685
|
+
|
|
13686
|
+
class ValFSHost {
|
|
13687
|
+
constructor(valFS, currentDirectory) {
|
|
13688
|
+
this.valFS = valFS;
|
|
13689
|
+
this.currentDirectory = currentDirectory;
|
|
13690
|
+
}
|
|
13691
|
+
useCaseSensitiveFileNames = true;
|
|
13692
|
+
readDirectory(rootDir, extensions, excludes, includes, depth) {
|
|
13693
|
+
return this.valFS.readDirectory(rootDir, extensions, excludes, includes, depth);
|
|
13694
|
+
}
|
|
13695
|
+
rmFile(fileName) {
|
|
13696
|
+
this.valFS.rmFile(fileName);
|
|
13697
|
+
}
|
|
13698
|
+
writeFile(fileName, text, encoding) {
|
|
13699
|
+
this.valFS.writeFile(fileName, text, encoding);
|
|
13700
|
+
}
|
|
13701
|
+
getCurrentDirectory() {
|
|
13702
|
+
return this.currentDirectory;
|
|
13703
|
+
}
|
|
13704
|
+
getCanonicalFileName(fileName) {
|
|
13705
|
+
if (path.isAbsolute(fileName)) {
|
|
13706
|
+
return path.normalize(fileName);
|
|
13040
13707
|
}
|
|
13041
|
-
|
|
13042
|
-
return value ? {
|
|
13043
|
-
status: "ok",
|
|
13044
|
-
value
|
|
13045
|
-
} : {
|
|
13046
|
-
status: "error",
|
|
13047
|
-
message: "Video is not an object!"
|
|
13048
|
-
};
|
|
13708
|
+
return path.resolve(this.getCurrentDirectory(), fileName);
|
|
13049
13709
|
}
|
|
13050
|
-
|
|
13051
|
-
|
|
13052
|
-
status: "ok",
|
|
13053
|
-
value
|
|
13054
|
-
} : {
|
|
13055
|
-
status: "error",
|
|
13056
|
-
message: "Video is not an object!"
|
|
13057
|
-
};
|
|
13058
|
-
}
|
|
13059
|
-
async function getVideoMetadata(projectRoot, fileRef) {
|
|
13060
|
-
// Not read into memory: only the headers are, and a video is mostly not
|
|
13061
|
-
// headers. A remote video's headers are read over HTTP, with Range
|
|
13062
|
-
// requests, so it is not downloaded either.
|
|
13063
|
-
if (Internal.isRemoteMediaPath(fileRef)) {
|
|
13064
|
-
return extractVideoMetadataFromUrl(fileRef, nameForTypeOf(fileRef));
|
|
13710
|
+
fileExists(fileName) {
|
|
13711
|
+
return this.valFS.fileExists(fileName);
|
|
13065
13712
|
}
|
|
13066
|
-
|
|
13067
|
-
|
|
13068
|
-
async function getImageMetadata(projectRoot, validationError) {
|
|
13069
|
-
const fileRef = getValidationErrorFileRef(validationError);
|
|
13070
|
-
if (!fileRef) {
|
|
13071
|
-
// TODO:
|
|
13072
|
-
throw Error("Cannot fix image without a file reference");
|
|
13713
|
+
readFile(fileName) {
|
|
13714
|
+
return this.valFS.readFile(fileName);
|
|
13073
13715
|
}
|
|
13074
|
-
|
|
13075
|
-
|
|
13076
|
-
|
|
13077
|
-
|
|
13078
|
-
|
|
13079
|
-
const fileRef = getValidationErrorFileRef(validationError);
|
|
13080
|
-
if (!fileRef) {
|
|
13081
|
-
// TODO:
|
|
13082
|
-
throw Error("Cannot fix file without a file reference");
|
|
13716
|
+
readBuffer(fileName) {
|
|
13717
|
+
return this.valFS.readBuffer(fileName);
|
|
13718
|
+
}
|
|
13719
|
+
realpath(path) {
|
|
13720
|
+
return this.valFS.realpath(path);
|
|
13083
13721
|
}
|
|
13084
|
-
const filename = path.join(projectRoot, fileRef);
|
|
13085
|
-
fs.readFileSync(filename);
|
|
13086
|
-
return extractFileMetadata(fileRef);
|
|
13087
13722
|
}
|
|
13088
13723
|
|
|
13089
13724
|
/**
|
|
@@ -13766,6 +14401,26 @@ async function handleViewCheckModule(ctx) {
|
|
|
13766
14401
|
shouldApplyPatch: true
|
|
13767
14402
|
};
|
|
13768
14403
|
}
|
|
14404
|
+
|
|
14405
|
+
/**
|
|
14406
|
+
* `remote:rewrite-host`: a platform ref whose host no longer serves its
|
|
14407
|
+
* project. The server's full check reports it, and so does `createFixPatch`'s
|
|
14408
|
+
* `image:check-remote` branch, which is how `val validate` finds it: that
|
|
14409
|
+
* branch asks the host first. Either way the patch is `createFixPatch`'s, and
|
|
14410
|
+
* needs nothing but the ref -- the new host is derived from it.
|
|
14411
|
+
*/
|
|
14412
|
+
async function handleRemoteRewriteHost(ctx) {
|
|
14413
|
+
if (!ctx.fix) {
|
|
14414
|
+
return {
|
|
14415
|
+
success: true,
|
|
14416
|
+
fixableErrorMessage: `${ctx.validationError.message} ` + "Run 'val validate --fix' to point it at the project's own address."
|
|
14417
|
+
};
|
|
14418
|
+
}
|
|
14419
|
+
return {
|
|
14420
|
+
success: true,
|
|
14421
|
+
shouldApplyPatch: true
|
|
14422
|
+
};
|
|
14423
|
+
}
|
|
13769
14424
|
async function handleExternalUpload() {
|
|
13770
14425
|
return {
|
|
13771
14426
|
success: true,
|
|
@@ -13807,7 +14462,8 @@ const currentFixHandlers = {
|
|
|
13807
14462
|
"files:check-all-files": handleCheckAllFiles,
|
|
13808
14463
|
"jsonValues:extract-entry": handleJsonValuesExtractEntry,
|
|
13809
14464
|
"external:upload": handleExternalUpload,
|
|
13810
|
-
"view:check-module": handleViewCheckModule
|
|
14465
|
+
"view:check-module": handleViewCheckModule,
|
|
14466
|
+
"remote:rewrite-host": handleRemoteRewriteHost
|
|
13811
14467
|
};
|
|
13812
14468
|
const deprecatedFixHandlers = {
|
|
13813
14469
|
"image:replace-metadata": handleFileMetadata
|