@valbuild/server 0.147.0 → 0.148.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +115 -0
- package/browser/dist/valbuild-server-browser.cjs.dev.js +2 -2
- package/browser/dist/valbuild-server-browser.cjs.prod.js +2 -2
- package/browser/dist/valbuild-server-browser.esm.js +2 -2
- package/dist/{ValRouterCore-7ffcc08c.cjs.prod.js → ValRouterCore-42772c21.cjs.prod.js} +980 -53
- package/dist/{ValRouterCore-f5004f8f.esm.js → ValRouterCore-7ca3dc68.esm.js} +972 -54
- package/dist/{ValRouterCore-3fd3d444.cjs.dev.js → ValRouterCore-9bdaaf89.cjs.dev.js} +980 -53
- package/dist/declarations/src/ValOps.d.ts +32 -11
- package/dist/declarations/src/ValOpsMemory.d.ts +4 -4
- package/dist/declarations/src/ValServer.d.ts +1 -2
- package/dist/declarations/src/checkRemoteRef.d.ts +40 -2
- package/dist/declarations/src/createFixPatch.d.ts +25 -3
- package/dist/declarations/src/fixFiles.d.ts +42 -0
- package/dist/declarations/src/fixHandlers.d.ts +14 -0
- package/dist/declarations/src/getSettings.d.ts +1 -0
- package/dist/declarations/src/index.d.ts +1 -0
- package/dist/declarations/src/platform.d.ts +16 -0
- package/dist/declarations/src/remoteFileCheck.d.ts +116 -0
- package/dist/declarations/src/remoteHostCheck.d.ts +101 -0
- package/dist/declarations/src/studioValidation.d.ts +84 -0
- package/dist/declarations/src/uploadRemoteFile.d.ts +1 -1
- package/dist/declarations/src/videosetFixes.d.ts +3 -2
- package/dist/{getJsonEntryAtCommit-df65d861.cjs.prod.js → getJsonEntryAtCommit-2e36e805.cjs.prod.js} +3 -4
- package/dist/{getJsonEntryAtCommit-1e08d724.esm.js → getJsonEntryAtCommit-40b6d9c1.esm.js} +3 -4
- package/dist/{getJsonEntryAtCommit-e96beb9e.cjs.dev.js → getJsonEntryAtCommit-64f67a50.cjs.dev.js} +3 -4
- package/dist/valbuild-server.cjs.dev.js +1352 -695
- package/dist/valbuild-server.cjs.prod.js +1352 -695
- package/dist/valbuild-server.esm.js +1352 -696
- package/package.json +4 -5
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { Buffer } from 'buffer';
|
|
2
1
|
import { Internal, VAL_EXTENSION, extractValModules, computeValModuleShas, ImageSchema, DEFAULT_CONTENT_HOST, hasRemoteFileSchema } from '@valbuild/core';
|
|
3
2
|
import { isSchemaSourceFixError, resolveSchemaSourceFixForError, SerializedSchema, VAL_ENABLE_COOKIE_NAME, VAL_STATE_COOKIE, VAL_SESSION_COOKIE, newestCommitSha, parseJob, VAL_STUDIO_HEADER, getErrorMessageFromUnknownJson, Api } from '@valbuild/shared/internal';
|
|
4
3
|
import { z } from 'zod';
|
|
@@ -922,6 +921,118 @@ async function validateJsonValuesEntries(schema, source, modulePath) {
|
|
|
922
921
|
return res;
|
|
923
922
|
}
|
|
924
923
|
|
|
924
|
+
/**
|
|
925
|
+
* Bytes as text and back, without Node's `Buffer`.
|
|
926
|
+
*
|
|
927
|
+
* The request handlers run in a browser tab too (`@valbuild/server/browser`),
|
|
928
|
+
* and a browser has no `Buffer`. Importing npm's `buffer` instead is what broke
|
|
929
|
+
* the demo under `vite dev`: it is CommonJS, and a dependency the dev server
|
|
930
|
+
* did not see in its scan is served unconverted. So bytes are `Uint8Array`
|
|
931
|
+
* here, and these are the conversions `Buffer` used to do -- with the same
|
|
932
|
+
* answers, which `bytes.test.ts` checks against `Buffer` itself.
|
|
933
|
+
*/
|
|
934
|
+
|
|
935
|
+
/**
|
|
936
|
+
* The value of each character code, or -1 for one that is not base64. Both
|
|
937
|
+
* alphabets, because `Buffer.from(s, "base64")` takes either: `-` and `_` are
|
|
938
|
+
* the URL-safe spellings of `+` and `/`.
|
|
939
|
+
*/
|
|
940
|
+
const BASE64_VALUES = (() => {
|
|
941
|
+
const BASE64_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
|
942
|
+
const values = new Int16Array(128).fill(-1);
|
|
943
|
+
for (let i = 0; i < BASE64_ALPHABET.length; i++) {
|
|
944
|
+
values[BASE64_ALPHABET.charCodeAt(i)] = i;
|
|
945
|
+
}
|
|
946
|
+
values["-".charCodeAt(0)] = 62;
|
|
947
|
+
values["_".charCodeAt(0)] = 63;
|
|
948
|
+
return values;
|
|
949
|
+
})();
|
|
950
|
+
const PADDING = "=".charCodeAt(0);
|
|
951
|
+
|
|
952
|
+
/**
|
|
953
|
+
* Decodes base64 the way `Buffer.from(base64, "base64")` does: either
|
|
954
|
+
* alphabet, padding optional, anything that is not base64 skipped, and
|
|
955
|
+
* decoding stops at the first `=`. A trailing partial group gives the whole
|
|
956
|
+
* bytes it holds.
|
|
957
|
+
*
|
|
958
|
+
* `atob` first, because it is native and this is called with whole uploads;
|
|
959
|
+
* it agrees with `Buffer` on everything it accepts, and refuses the rest (the
|
|
960
|
+
* URL-safe alphabet, stray characters, `=` in the middle), which then goes the
|
|
961
|
+
* slow way.
|
|
962
|
+
*/
|
|
963
|
+
function base64ToBytes(base64) {
|
|
964
|
+
let binary;
|
|
965
|
+
try {
|
|
966
|
+
binary = atob(base64);
|
|
967
|
+
} catch {
|
|
968
|
+
return lenientBase64ToBytes(base64);
|
|
969
|
+
}
|
|
970
|
+
const bytes = new Uint8Array(binary.length);
|
|
971
|
+
for (let i = 0; i < binary.length; i++) {
|
|
972
|
+
bytes[i] = binary.charCodeAt(i);
|
|
973
|
+
}
|
|
974
|
+
return bytes;
|
|
975
|
+
}
|
|
976
|
+
function lenientBase64ToBytes(base64) {
|
|
977
|
+
const bytes = new Uint8Array(Math.floor(base64.length * 3 / 4));
|
|
978
|
+
let length = 0;
|
|
979
|
+
let bits = 0;
|
|
980
|
+
let bitCount = 0;
|
|
981
|
+
for (let i = 0; i < base64.length; i++) {
|
|
982
|
+
const code = base64.charCodeAt(i);
|
|
983
|
+
if (code === PADDING) break;
|
|
984
|
+
const value = code < 128 ? BASE64_VALUES[code] : -1;
|
|
985
|
+
if (value < 0) continue;
|
|
986
|
+
bits = (bits << 6 | value) & 0xffffff;
|
|
987
|
+
bitCount += 6;
|
|
988
|
+
if (bitCount >= 8) {
|
|
989
|
+
bitCount -= 8;
|
|
990
|
+
bytes[length++] = bits >> bitCount & 0xff;
|
|
991
|
+
}
|
|
992
|
+
}
|
|
993
|
+
return bytes.subarray(0, length);
|
|
994
|
+
}
|
|
995
|
+
|
|
996
|
+
/**
|
|
997
|
+
* How many bytes go to `String.fromCharCode` at once: it takes them as
|
|
998
|
+
* arguments, and an engine has a limit on how many a call may have.
|
|
999
|
+
*/
|
|
1000
|
+
const CHUNK = 0x8000;
|
|
1001
|
+
|
|
1002
|
+
/**
|
|
1003
|
+
* `Buffer.from(bytes).toString("base64")`: padded, standard alphabet.
|
|
1004
|
+
*
|
|
1005
|
+
* Through `btoa`, which is native, rather than building the string here: that
|
|
1006
|
+
* was five times slower, and this is called with whole files. Node is still
|
|
1007
|
+
* faster with `Buffer`, which is why `ValServerPlatform.bytesToBase64` exists
|
|
1008
|
+
* for the paths that encode every file of a publish.
|
|
1009
|
+
*/
|
|
1010
|
+
function bytesToBase64(bytes) {
|
|
1011
|
+
const binary = [];
|
|
1012
|
+
for (let i = 0; i < bytes.length; i += CHUNK) {
|
|
1013
|
+
// `Reflect.apply` rather than a spread, which copies the chunk into an
|
|
1014
|
+
// argument list first and is four times slower.
|
|
1015
|
+
const chunk = Reflect.apply(String.fromCharCode, null, bytes.subarray(i, i + CHUNK));
|
|
1016
|
+
binary.push(chunk);
|
|
1017
|
+
}
|
|
1018
|
+
return btoa(binary.join(""));
|
|
1019
|
+
}
|
|
1020
|
+
|
|
1021
|
+
/** `Buffer.from(text, "utf8")`. */
|
|
1022
|
+
function utf8ToBytes(text) {
|
|
1023
|
+
return new TextEncoder().encode(text);
|
|
1024
|
+
}
|
|
1025
|
+
|
|
1026
|
+
/**
|
|
1027
|
+
* `Buffer.from(bytes).toString("utf8")`. `ignoreBOM` because `Buffer` keeps a
|
|
1028
|
+
* leading byte order mark, where a `TextDecoder` drops it by default.
|
|
1029
|
+
*/
|
|
1030
|
+
function bytesToUtf8(bytes) {
|
|
1031
|
+
return new TextDecoder("utf-8", {
|
|
1032
|
+
ignoreBOM: true
|
|
1033
|
+
}).decode(bytes);
|
|
1034
|
+
}
|
|
1035
|
+
|
|
925
1036
|
/**
|
|
926
1037
|
* Does saving this commit upload a file to Val's remote host?
|
|
927
1038
|
*
|
|
@@ -1336,7 +1447,8 @@ function isHlsPlaylistPath(filePath) {
|
|
|
1336
1447
|
* patch's to read. Generic over the patch id so it asks for exactly the one
|
|
1337
1448
|
* method it uses, and a test can hand it a plain object.
|
|
1338
1449
|
*/
|
|
1339
|
-
async function readCommittedBinaryFiles(ops, descriptors
|
|
1450
|
+
async function readCommittedBinaryFiles(ops, descriptors, /** `ValServerPlatform.bytesToBase64`, which is faster in Node. */
|
|
1451
|
+
encode = bytesToBase64) {
|
|
1340
1452
|
const files = {};
|
|
1341
1453
|
const unread = [];
|
|
1342
1454
|
await Promise.all(Object.entries(descriptors).map(async ([filePath, {
|
|
@@ -1348,7 +1460,7 @@ async function readCommittedBinaryFiles(ops, descriptors) {
|
|
|
1348
1460
|
console.error("Could not read a committed file", filePath, error);
|
|
1349
1461
|
return null;
|
|
1350
1462
|
});
|
|
1351
|
-
if (bytes === null) unread.push(filePath);else files[filePath] = bytes
|
|
1463
|
+
if (bytes === null) unread.push(filePath);else files[filePath] = encode(bytes);
|
|
1352
1464
|
}));
|
|
1353
1465
|
return {
|
|
1354
1466
|
files,
|
|
@@ -1515,10 +1627,537 @@ function computePatchesToDrop(prepared) {
|
|
|
1515
1627
|
return dropped;
|
|
1516
1628
|
}
|
|
1517
1629
|
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1630
|
+
function getFileExt(filePath) {
|
|
1631
|
+
// NOTE: We do not import the path module. This code is copied in different projects. We want the same implementation and which means that this might running in browser where path is not available).
|
|
1632
|
+
return filePath.split(".").pop() || "";
|
|
1633
|
+
}
|
|
1634
|
+
|
|
1635
|
+
/**
|
|
1636
|
+
* Does the host a platform ref names still serve the project the ref names?
|
|
1637
|
+
*
|
|
1638
|
+
* A ref served by the platform loader names the project's host at the time
|
|
1639
|
+
* the file was added: `https://www.acme.no/_val/file/p/3f9a2c1/…`. When the
|
|
1640
|
+
* project drops `www.acme.no`, the bytes are still there, at
|
|
1641
|
+
* `3f9a2c1.valstart.dev`, but every ref that names the old domain points at a
|
|
1642
|
+
* host that no longer serves them. This is the question that finds those refs,
|
|
1643
|
+
* and `rewriteRefHost` is what the fix (`remote:rewrite-host`) does about it.
|
|
1644
|
+
* See valbuild/home `docs/remote-files-on-platform.md`, "Rewriting refs after a
|
|
1645
|
+
* domain is removed", and `docs/plans/studio-validate.md`, step 6.
|
|
1646
|
+
*
|
|
1647
|
+
* Asked of the host itself (`GET <host>/_val/project`), not of content: the
|
|
1648
|
+
* host's answer covers DNS, the certificate and the loader's mapping in one,
|
|
1649
|
+
* it needs no credential, and it works for a ref that names a project other
|
|
1650
|
+
* than this one -- a project seeded from a template has refs naming the
|
|
1651
|
+
* template's.
|
|
1652
|
+
*/
|
|
1653
|
+
|
|
1654
|
+
/** The domain every platform project is always served at, by its public id. */
|
|
1655
|
+
const PLATFORM_SITES_DOMAIN = "valstart.dev";
|
|
1656
|
+
|
|
1657
|
+
/** What a platform ref's host part ends with. */
|
|
1658
|
+
const PLATFORM_PREFIX = "/_val";
|
|
1659
|
+
|
|
1660
|
+
/**
|
|
1661
|
+
* The host every platform ref of `publicProjectId` can always be pointed at:
|
|
1662
|
+
* `<public id>.valstart.dev` is permanent (decision 4 of the remote-files
|
|
1663
|
+
* plan), and the loader serves a project's files there whatever else it has.
|
|
1664
|
+
*/
|
|
1665
|
+
function fallbackRemoteHost(publicProjectId) {
|
|
1666
|
+
return `https://${publicProjectId}.${PLATFORM_SITES_DOMAIN}${PLATFORM_PREFIX}`;
|
|
1667
|
+
}
|
|
1668
|
+
|
|
1669
|
+
/**
|
|
1670
|
+
* Only a platform ref is asked about. `remote.val.build` is not a platform
|
|
1671
|
+
* host and is not going away, and a ref already on the project's own
|
|
1672
|
+
* `valstart.dev` address is on the fallback, which always serves it.
|
|
1673
|
+
*/
|
|
1674
|
+
function needsHostCheck(remoteHost, projectId) {
|
|
1675
|
+
return remoteHost.endsWith(PLATFORM_PREFIX) && remoteHost !== fallbackRemoteHost(projectId);
|
|
1676
|
+
}
|
|
1677
|
+
|
|
1678
|
+
/** `ref` with its host part replaced, and everything else as it was. */
|
|
1679
|
+
function rewriteRefHost(ref, remoteHost) {
|
|
1680
|
+
const parsed = Internal.remote.splitRemoteRef(ref);
|
|
1681
|
+
if (parsed.status === "error") {
|
|
1682
|
+
return null;
|
|
1683
|
+
}
|
|
1684
|
+
return Internal.remote.createRemoteRef(remoteHost, {
|
|
1685
|
+
publicProjectId: parsed.projectId,
|
|
1686
|
+
coreVersion: parsed.version,
|
|
1687
|
+
bucket: parsed.bucket,
|
|
1688
|
+
validationHash: parsed.validationHash,
|
|
1689
|
+
fileHash: parsed.fileHash,
|
|
1690
|
+
filePath: parsed.filePath
|
|
1691
|
+
});
|
|
1692
|
+
}
|
|
1693
|
+
|
|
1694
|
+
/**
|
|
1695
|
+
* - `serves`: the host answered for the project the ref names.
|
|
1696
|
+
* - `other-project`: it answered, for another one -- the domain was moved to
|
|
1697
|
+
* a different project, and decision 5 means it will not serve these files.
|
|
1698
|
+
* - `no-site`: it answered, but serves no project at all (the loader's 404 for
|
|
1699
|
+
* a domain nobody has mapped any more).
|
|
1700
|
+
* - `unreachable`: no answer. A domain whose DNS was removed looks like this,
|
|
1701
|
+
* and so does a host that is down for a minute, which is why the row says
|
|
1702
|
+
* which it saw.
|
|
1703
|
+
*/
|
|
1704
|
+
|
|
1705
|
+
const ASK_TIMEOUT_MS = 8_000;
|
|
1706
|
+
|
|
1707
|
+
/** `GET <host>/_val/project`, answered by the platform loader. */
|
|
1708
|
+
const askHostProject = async remoteHost => {
|
|
1709
|
+
try {
|
|
1710
|
+
const res = await fetch(`${remoteHost}/project`, {
|
|
1711
|
+
signal: AbortSignal.timeout(ASK_TIMEOUT_MS)
|
|
1712
|
+
});
|
|
1713
|
+
if (!res.ok) {
|
|
1714
|
+
return {
|
|
1715
|
+
status: "no-site",
|
|
1716
|
+
httpStatus: res.status
|
|
1717
|
+
};
|
|
1718
|
+
}
|
|
1719
|
+
const body = await res.json();
|
|
1720
|
+
if (typeof body === "object" && body !== null && "publicProjectId" in body && typeof body.publicProjectId === "string") {
|
|
1721
|
+
return {
|
|
1722
|
+
status: "answered",
|
|
1723
|
+
publicProjectId: body.publicProjectId
|
|
1724
|
+
};
|
|
1725
|
+
}
|
|
1726
|
+
return {
|
|
1727
|
+
status: "no-site",
|
|
1728
|
+
httpStatus: res.status
|
|
1729
|
+
};
|
|
1730
|
+
} catch (err) {
|
|
1731
|
+
return {
|
|
1732
|
+
status: "unreachable",
|
|
1733
|
+
reason: err instanceof Error && err.name === "TimeoutError" ? `No answer within ${ASK_TIMEOUT_MS / 1000} s` : err instanceof Error ? err.message : "Unknown error"
|
|
1734
|
+
};
|
|
1735
|
+
}
|
|
1736
|
+
};
|
|
1737
|
+
|
|
1738
|
+
/**
|
|
1739
|
+
* Hosts that answered for a project, and when: one question per host, not per
|
|
1740
|
+
* ref, and not again for a few minutes. Only a "serves" is kept, for the same
|
|
1741
|
+
* reason `remoteFileCheck` keeps only "found": a host that stopped serving has
|
|
1742
|
+
* to be noticed, and one that did not answer has to be asked again.
|
|
1743
|
+
*/
|
|
1744
|
+
const SERVES_FOR_MS = 5 * 60_000;
|
|
1745
|
+
const servesAt = new Map();
|
|
1746
|
+
async function hostServes(remoteHost, projectId, ask = askHostProject) {
|
|
1747
|
+
const key = `${remoteHost} ${projectId}`;
|
|
1748
|
+
const at = servesAt.get(key);
|
|
1749
|
+
if (at !== undefined && Date.now() - at < SERVES_FOR_MS) {
|
|
1750
|
+
return {
|
|
1751
|
+
status: "serves"
|
|
1752
|
+
};
|
|
1753
|
+
}
|
|
1754
|
+
const answer = await ask(remoteHost);
|
|
1755
|
+
if (answer.status !== "answered") {
|
|
1756
|
+
return answer;
|
|
1757
|
+
}
|
|
1758
|
+
if (answer.publicProjectId !== projectId) {
|
|
1759
|
+
return {
|
|
1760
|
+
status: "other-project",
|
|
1761
|
+
publicProjectId: answer.publicProjectId
|
|
1762
|
+
};
|
|
1763
|
+
}
|
|
1764
|
+
servesAt.set(key, Date.now());
|
|
1765
|
+
return {
|
|
1766
|
+
status: "serves"
|
|
1767
|
+
};
|
|
1768
|
+
}
|
|
1769
|
+
|
|
1770
|
+
/**
|
|
1771
|
+
* `ask`, asking each host once for as long as the returned function is kept:
|
|
1772
|
+
* one check, not the life of the server. A check walks its refs in order, so
|
|
1773
|
+
* without it every ref on a host that does not answer waits out its own
|
|
1774
|
+
* timeout -- 150 images on a dead domain is 20 minutes. Kept per check rather
|
|
1775
|
+
* than in {@link hostServes}'s cache because a host that did not answer has
|
|
1776
|
+
* to be asked again by the next one.
|
|
1777
|
+
*/
|
|
1778
|
+
function askOncePerHost(ask = askHostProject) {
|
|
1779
|
+
const answers = new Map();
|
|
1780
|
+
return remoteHost => {
|
|
1781
|
+
let answer = answers.get(remoteHost);
|
|
1782
|
+
if (answer === undefined) {
|
|
1783
|
+
answer = ask(remoteHost);
|
|
1784
|
+
answers.set(remoteHost, answer);
|
|
1785
|
+
}
|
|
1786
|
+
return answer;
|
|
1787
|
+
};
|
|
1788
|
+
}
|
|
1789
|
+
|
|
1790
|
+
/** `www.acme.no`, from `https://www.acme.no/_val`: how a person names a host. */
|
|
1791
|
+
function hostNameOfRemoteHost(remoteHost) {
|
|
1792
|
+
try {
|
|
1793
|
+
return new URL(remoteHost).host;
|
|
1794
|
+
} catch {
|
|
1795
|
+
return remoteHost;
|
|
1796
|
+
}
|
|
1797
|
+
}
|
|
1798
|
+
|
|
1799
|
+
/**
|
|
1800
|
+
* Where a platform ref has to move to, when its host no longer serves the
|
|
1801
|
+
* project it names; `undefined` when it does not have to (or is not a
|
|
1802
|
+
* platform ref). `why` and `to` are for the caller's sentence: what can be
|
|
1803
|
+
* done about it differs by caller (a Studio **Fix**, `val validate --fix`, or
|
|
1804
|
+
* nothing yet).
|
|
1805
|
+
*/
|
|
1806
|
+
async function movedRemoteHost(ref, ask) {
|
|
1807
|
+
const parsed = Internal.remote.splitRemoteRef(ref);
|
|
1808
|
+
if (parsed.status === "error" || !needsHostCheck(parsed.remoteHost, parsed.projectId)) {
|
|
1809
|
+
return undefined;
|
|
1810
|
+
}
|
|
1811
|
+
const serves = await hostServes(parsed.remoteHost, parsed.projectId, ask);
|
|
1812
|
+
// Only an answer that says so moves a ref: the host serves another
|
|
1813
|
+
// project, or the loader says it serves none (its 404 for a domain nobody
|
|
1814
|
+
// maps). A host that did not answer, or answered 5xx, may be down for a
|
|
1815
|
+
// minute -- and a moved ref does not move back, so rewriting on that would
|
|
1816
|
+
// take a site's images off its own domain for good over one blip. Those are
|
|
1817
|
+
// left to the existence check, which reports the host once with Try again.
|
|
1818
|
+
if (serves.status === "serves" || serves.status === "unreachable" || serves.status === "no-site" && serves.httpStatus !== 404) {
|
|
1819
|
+
return undefined;
|
|
1820
|
+
}
|
|
1821
|
+
const target = fallbackRemoteHost(parsed.projectId);
|
|
1822
|
+
const moved = rewriteRefHost(ref, target);
|
|
1823
|
+
if (moved === null) {
|
|
1824
|
+
return undefined;
|
|
1825
|
+
}
|
|
1826
|
+
const from = hostNameOfRemoteHost(parsed.remoteHost);
|
|
1827
|
+
return {
|
|
1828
|
+
ref: moved,
|
|
1829
|
+
why: serves.status === "other-project" ? `${from} now serves another project, so this file is not shown from there` : `${from} no longer serves this project, so this file is not shown from there`,
|
|
1830
|
+
to: hostNameOfRemoteHost(target)
|
|
1831
|
+
};
|
|
1832
|
+
}
|
|
1833
|
+
|
|
1834
|
+
/**
|
|
1835
|
+
* Where else a ref's file can be downloaded from: the same file on the
|
|
1836
|
+
* project's own `valstart.dev` address, for a platform ref on any other host.
|
|
1837
|
+
* `null` when there is nowhere else -- a `remote.val.build` ref, or one already
|
|
1838
|
+
* on the fallback. See valbuild/home `docs/remote-files-on-platform.md`,
|
|
1839
|
+
* "Tooling: the fallback".
|
|
1521
1840
|
*/
|
|
1841
|
+
function fallbackRefOf(ref) {
|
|
1842
|
+
const parsed = Internal.remote.splitRemoteRef(ref);
|
|
1843
|
+
if (parsed.status === "error" || !needsHostCheck(parsed.remoteHost, parsed.projectId)) {
|
|
1844
|
+
return null;
|
|
1845
|
+
}
|
|
1846
|
+
return rewriteRefHost(ref, fallbackRemoteHost(parsed.projectId));
|
|
1847
|
+
}
|
|
1848
|
+
|
|
1849
|
+
/**
|
|
1850
|
+
* Does the host a remote ref names still serve it?
|
|
1851
|
+
*
|
|
1852
|
+
* `found` and `missing` are the host's answer. `unreachable` is no answer at
|
|
1853
|
+
* all — a timeout, a refused connection, a 5xx — and says nothing about the
|
|
1854
|
+
* file, only about the host.
|
|
1855
|
+
*/
|
|
1856
|
+
|
|
1857
|
+
/**
|
|
1858
|
+
* How the remote check reaches the network: what it asks a host about a file,
|
|
1859
|
+
* and which project a platform host serves. Both default to the real thing;
|
|
1860
|
+
* tests replace them.
|
|
1861
|
+
*/
|
|
1862
|
+
|
|
1863
|
+
const HEAD_TIMEOUT_MS = 8_000;
|
|
1864
|
+
|
|
1865
|
+
/**
|
|
1866
|
+
* A HEAD, falling back to the first byte of a GET for a host that does not
|
|
1867
|
+
* answer HEAD (405, 501): either way no file is downloaded to learn that it is
|
|
1868
|
+
* there.
|
|
1869
|
+
*/
|
|
1870
|
+
const headRemoteFile = async url => {
|
|
1871
|
+
const ask = async init => fetch(url, {
|
|
1872
|
+
...init,
|
|
1873
|
+
redirect: "follow",
|
|
1874
|
+
signal: AbortSignal.timeout(HEAD_TIMEOUT_MS)
|
|
1875
|
+
});
|
|
1876
|
+
try {
|
|
1877
|
+
let res = await ask({
|
|
1878
|
+
method: "HEAD"
|
|
1879
|
+
});
|
|
1880
|
+
if (res.status === 405 || res.status === 501) {
|
|
1881
|
+
var _res$body;
|
|
1882
|
+
res = await ask({
|
|
1883
|
+
method: "GET",
|
|
1884
|
+
headers: {
|
|
1885
|
+
Range: "bytes=0-0"
|
|
1886
|
+
}
|
|
1887
|
+
});
|
|
1888
|
+
await ((_res$body = res.body) === null || _res$body === void 0 ? void 0 : _res$body.cancel());
|
|
1889
|
+
}
|
|
1890
|
+
if (res.ok) {
|
|
1891
|
+
return {
|
|
1892
|
+
status: "found"
|
|
1893
|
+
};
|
|
1894
|
+
}
|
|
1895
|
+
if (res.status === 404 || res.status === 410) {
|
|
1896
|
+
return {
|
|
1897
|
+
status: "missing",
|
|
1898
|
+
httpStatus: res.status
|
|
1899
|
+
};
|
|
1900
|
+
}
|
|
1901
|
+
return {
|
|
1902
|
+
status: "unreachable",
|
|
1903
|
+
reason: `HTTP ${res.status}`
|
|
1904
|
+
};
|
|
1905
|
+
} catch (err) {
|
|
1906
|
+
return {
|
|
1907
|
+
status: "unreachable",
|
|
1908
|
+
reason: describeFetchError(err)
|
|
1909
|
+
};
|
|
1910
|
+
}
|
|
1911
|
+
};
|
|
1912
|
+
function describeFetchError(err) {
|
|
1913
|
+
if (!(err instanceof Error)) {
|
|
1914
|
+
return "Unknown error";
|
|
1915
|
+
}
|
|
1916
|
+
if (err.name === "TimeoutError") {
|
|
1917
|
+
return `No answer within ${HEAD_TIMEOUT_MS / 1000} s`;
|
|
1918
|
+
}
|
|
1919
|
+
// undici puts what actually went wrong (ENOTFOUND, ECONNREFUSED) on `cause`.
|
|
1920
|
+
const cause = err.cause;
|
|
1921
|
+
if (cause instanceof Error) {
|
|
1922
|
+
return `${err.message}: ${cause.message}`;
|
|
1923
|
+
}
|
|
1924
|
+
return err.message;
|
|
1925
|
+
}
|
|
1926
|
+
|
|
1927
|
+
/**
|
|
1928
|
+
* Refs a host has served, and when.
|
|
1929
|
+
*
|
|
1930
|
+
* So that pressing **Check again**, or opening a second module that uses the
|
|
1931
|
+
* same images, does not ask about every file again. Kept for minutes, not for
|
|
1932
|
+
* the life of the process: a ref is content-addressed, so its answer only
|
|
1933
|
+
* changes when its host stops serving it -- which is exactly what this check
|
|
1934
|
+
* is for, and a server that remembered "found" forever would never report a
|
|
1935
|
+
* host that went down after it was first asked. Only `found` is kept. A host
|
|
1936
|
+
* that did not answer is asked again on the next check, which is what makes
|
|
1937
|
+
* **Try again** mean something.
|
|
1938
|
+
*/
|
|
1939
|
+
const FOUND_FOR_MS = 5 * 60_000;
|
|
1940
|
+
const foundAt = new Map();
|
|
1941
|
+
function wasFound(ref) {
|
|
1942
|
+
const at = foundAt.get(ref);
|
|
1943
|
+
return at !== undefined && Date.now() - at < FOUND_FOR_MS;
|
|
1944
|
+
}
|
|
1945
|
+
function markFound(ref) {
|
|
1946
|
+
foundAt.set(ref, Date.now());
|
|
1947
|
+
}
|
|
1948
|
+
|
|
1949
|
+
/** A host that did not serve the refs of a module it was asked about. */
|
|
1950
|
+
|
|
1951
|
+
const textEncoder = new TextEncoder();
|
|
1952
|
+
|
|
1953
|
+
/**
|
|
1954
|
+
* The remote half of the Studio's full check, for one module's remote values.
|
|
1955
|
+
*
|
|
1956
|
+
* Four questions, cheapest first, and a ref stops at the first one it fails:
|
|
1957
|
+
*
|
|
1958
|
+
* 1. **Is it a ref at all?** `splitRemoteRef` parses it.
|
|
1959
|
+
* 2. **Does the ref agree with what is recorded about it?** A ref carries a
|
|
1960
|
+
* hash of the schema's options, the file's extension and its width, height
|
|
1961
|
+
* and mime type. When the value or the schema changed and the ref did not,
|
|
1962
|
+
* the ref is stale, and **Fix** (`image:check-remote`) re-reads the file and
|
|
1963
|
+
* writes the ref it should have. Computed with the ref's OWN core version:
|
|
1964
|
+
* the CLI also re-derives every ref after a Val upgrade, and listing every
|
|
1965
|
+
* remote image in the project after each upgrade would bury the rows that
|
|
1966
|
+
* matter. Nothing here downloads.
|
|
1967
|
+
* 3. **Does its host still serve its project?** Only for a platform ref
|
|
1968
|
+
* (`…/_val/file/…`) that is not on the project's own `valstart.dev`
|
|
1969
|
+
* address: one `GET <host>/_val/project` per host. A host that dropped the
|
|
1970
|
+
* project gets a **Fix** (`remote:rewrite-host`) that points the ref at
|
|
1971
|
+
* `<public id>.valstart.dev`. See `remoteHostCheck.ts`.
|
|
1972
|
+
* 4. **Does the host have it?** A HEAD.
|
|
1973
|
+
*
|
|
1974
|
+
* Neither host question is asked of a ref whose bytes are in a pending patch,
|
|
1975
|
+
* which the host will not have until publish: there the question is whether
|
|
1976
|
+
* the patch still has them.
|
|
1977
|
+
*/
|
|
1978
|
+
async function checkRemoteFiles(input) {
|
|
1979
|
+
const head = input.head ?? headRemoteFile;
|
|
1980
|
+
const askHostProject = askOncePerHost(input.askHostProject);
|
|
1981
|
+
const errors = {};
|
|
1982
|
+
const addError = (sourcePath, error) => {
|
|
1983
|
+
errors[sourcePath] = (errors[sourcePath] ?? []).concat(error);
|
|
1984
|
+
};
|
|
1985
|
+
|
|
1986
|
+
/** Refs still to ask a host about, by host. */
|
|
1987
|
+
const toAsk = new Map();
|
|
1988
|
+
const sorted = Object.entries(input.remoteFiles).sort(([a], [b]) => a.localeCompare(b));
|
|
1989
|
+
for (const [sourcePathS, value] of sorted) {
|
|
1990
|
+
const sourcePath = sourcePathS;
|
|
1991
|
+
const ref = value.path;
|
|
1992
|
+
const parsed = Internal.remote.splitRemoteRef(ref);
|
|
1993
|
+
if (parsed.status === "error") {
|
|
1994
|
+
addError(sourcePath, {
|
|
1995
|
+
message: `This is not an address Val can check: ${parsed.error}`,
|
|
1996
|
+
value
|
|
1997
|
+
});
|
|
1998
|
+
continue;
|
|
1999
|
+
}
|
|
2000
|
+
let noun = "file";
|
|
2001
|
+
if (!input.existenceOnly) {
|
|
2002
|
+
const schema = input.schemaAt(sourcePath);
|
|
2003
|
+
if ((schema === null || schema === void 0 ? void 0 : schema.type) !== "image" && (schema === null || schema === void 0 ? void 0 : schema.type) !== "file") {
|
|
2004
|
+
continue;
|
|
2005
|
+
}
|
|
2006
|
+
noun = schema.type;
|
|
2007
|
+
if (isStale(parsed, schema, value)) {
|
|
2008
|
+
addError(sourcePath, {
|
|
2009
|
+
message: `The address of this ${schema.type} does not match what is recorded about it, so it may be served with the wrong size or type.`,
|
|
2010
|
+
value,
|
|
2011
|
+
fixes: [schema.type === "image" ? "image:check-remote" : "file:check-remote"]
|
|
2012
|
+
});
|
|
2013
|
+
continue;
|
|
2014
|
+
}
|
|
2015
|
+
}
|
|
2016
|
+
const pending = await input.pendingBytes(ref);
|
|
2017
|
+
if (pending === "present") {
|
|
2018
|
+
continue;
|
|
2019
|
+
}
|
|
2020
|
+
if (pending === "missing") {
|
|
2021
|
+
addError(sourcePath, {
|
|
2022
|
+
message: `This ${noun} is in your unpublished changes, but its file is not. Choose the ${noun} again.`,
|
|
2023
|
+
value
|
|
2024
|
+
});
|
|
2025
|
+
continue;
|
|
2026
|
+
}
|
|
2027
|
+
// Asked after the pending check: a ref whose bytes are still in a patch
|
|
2028
|
+
// is the key those bytes are stored under, and repointing it would cut
|
|
2029
|
+
// it off from them.
|
|
2030
|
+
const moved = await movedRemoteHost(ref, askHostProject);
|
|
2031
|
+
if (moved) {
|
|
2032
|
+
addError(sourcePath, input.existenceOnly ? {
|
|
2033
|
+
// A directory's key: the fields picking from it hold the same
|
|
2034
|
+
// ref, and would have to follow. Not built yet.
|
|
2035
|
+
message: `${moved.why}. Pointing a file in a media directory at ${moved.to} is not something the Studio can do yet.`,
|
|
2036
|
+
value
|
|
2037
|
+
} : {
|
|
2038
|
+
message: `${moved.why}. Fix points it at ${moved.to}, which always serves it.`,
|
|
2039
|
+
value,
|
|
2040
|
+
fixes: ["remote:rewrite-host"]
|
|
2041
|
+
});
|
|
2042
|
+
continue;
|
|
2043
|
+
}
|
|
2044
|
+
if (wasFound(ref)) {
|
|
2045
|
+
continue;
|
|
2046
|
+
}
|
|
2047
|
+
const refs = toAsk.get(parsed.remoteHost) ?? [];
|
|
2048
|
+
refs.push({
|
|
2049
|
+
sourcePath,
|
|
2050
|
+
ref,
|
|
2051
|
+
published: input.isPublished(sourcePath, ref)
|
|
2052
|
+
});
|
|
2053
|
+
toAsk.set(parsed.remoteHost, refs);
|
|
2054
|
+
}
|
|
2055
|
+
const hosts = [];
|
|
2056
|
+
await Promise.all([...toAsk.entries()].map(async ([host, refs]) => {
|
|
2057
|
+
const problem = await askHost(host, refs, input.sampleByHost, head);
|
|
2058
|
+
if (problem.host) {
|
|
2059
|
+
hosts.push(problem.host);
|
|
2060
|
+
}
|
|
2061
|
+
for (const {
|
|
2062
|
+
sourcePath,
|
|
2063
|
+
error
|
|
2064
|
+
} of problem.refErrors) {
|
|
2065
|
+
addError(sourcePath, error);
|
|
2066
|
+
}
|
|
2067
|
+
}));
|
|
2068
|
+
hosts.sort((a, b) => a.host.localeCompare(b.host));
|
|
2069
|
+
return {
|
|
2070
|
+
errors,
|
|
2071
|
+
hosts
|
|
2072
|
+
};
|
|
2073
|
+
}
|
|
2074
|
+
|
|
2075
|
+
/**
|
|
2076
|
+
* One host's refs, asked one at a time.
|
|
2077
|
+
*
|
|
2078
|
+
* In order rather than all at once: a host that does not answer is reported
|
|
2079
|
+
* after the FIRST timeout, not after one timeout per image. A host that
|
|
2080
|
+
* answers "missing" for one ref is still asked about the next, since a file
|
|
2081
|
+
* that is gone is a fact about that file.
|
|
2082
|
+
*/
|
|
2083
|
+
async function askHost(host, refs, sampleByHost, head) {
|
|
2084
|
+
const vouchable = sampleByHost ? refs.filter(r => r.published) : [];
|
|
2085
|
+
const individually = sampleByHost ? refs.filter(r => !r.published) : refs;
|
|
2086
|
+
const allPaths = refs.map(r => r.sourcePath);
|
|
2087
|
+
const refErrors = [];
|
|
2088
|
+
if (vouchable.length > 0) {
|
|
2089
|
+
const sample = vouchable[0];
|
|
2090
|
+
const answer = await head(sample.ref);
|
|
2091
|
+
if (answer.status === "found") {
|
|
2092
|
+
for (const r of vouchable) {
|
|
2093
|
+
markFound(r.ref);
|
|
2094
|
+
}
|
|
2095
|
+
} else {
|
|
2096
|
+
// In a managed project a published ref cannot break on its own, so the
|
|
2097
|
+
// sample's answer is the host's: one row, every ref on it.
|
|
2098
|
+
return {
|
|
2099
|
+
host: {
|
|
2100
|
+
host,
|
|
2101
|
+
status: answer.status,
|
|
2102
|
+
details: detailsOf(sample.ref, answer),
|
|
2103
|
+
sourcePaths: allPaths
|
|
2104
|
+
},
|
|
2105
|
+
refErrors
|
|
2106
|
+
};
|
|
2107
|
+
}
|
|
2108
|
+
}
|
|
2109
|
+
for (const r of individually) {
|
|
2110
|
+
const answer = await head(r.ref);
|
|
2111
|
+
if (answer.status === "found") {
|
|
2112
|
+
markFound(r.ref);
|
|
2113
|
+
} else if (answer.status === "missing") {
|
|
2114
|
+
refErrors.push({
|
|
2115
|
+
sourcePath: r.sourcePath,
|
|
2116
|
+
error: {
|
|
2117
|
+
message: `The file is not at its address: ${hostName(host)} answered ${answer.httpStatus}.`
|
|
2118
|
+
}
|
|
2119
|
+
});
|
|
2120
|
+
} else {
|
|
2121
|
+
return {
|
|
2122
|
+
host: {
|
|
2123
|
+
host,
|
|
2124
|
+
status: "unreachable",
|
|
2125
|
+
details: detailsOf(r.ref, answer),
|
|
2126
|
+
sourcePaths: allPaths
|
|
2127
|
+
},
|
|
2128
|
+
refErrors
|
|
2129
|
+
};
|
|
2130
|
+
}
|
|
2131
|
+
}
|
|
2132
|
+
return {
|
|
2133
|
+
refErrors
|
|
2134
|
+
};
|
|
2135
|
+
}
|
|
2136
|
+
function detailsOf(ref, answer) {
|
|
2137
|
+
return `HEAD ${ref}\n→ ${answer.status === "missing" ? `${answer.httpStatus}` : answer.status === "unreachable" ? answer.reason : "200"}`;
|
|
2138
|
+
}
|
|
2139
|
+
|
|
2140
|
+
/** `www.acme.no`, from `https://www.acme.no`: how a person names a host. */
|
|
2141
|
+
function hostName(host) {
|
|
2142
|
+
try {
|
|
2143
|
+
return new URL(host).host;
|
|
2144
|
+
} catch {
|
|
2145
|
+
return host;
|
|
2146
|
+
}
|
|
2147
|
+
}
|
|
2148
|
+
function isStale(parsed, schema, value) {
|
|
2149
|
+
// The hash covers only these of what was recorded (`getValidationBasis`):
|
|
2150
|
+
// `alt` and `hotspot` are the editor's, and change no byte.
|
|
2151
|
+
const metadata = {
|
|
2152
|
+
width: value.width,
|
|
2153
|
+
height: value.height,
|
|
2154
|
+
mimeType: value.mimeType
|
|
2155
|
+
};
|
|
2156
|
+
const expected = Internal.remote.getValidationHash(parsed.version, schema, getFileExt(parsed.filePath), metadata, parsed.fileHash, textEncoder);
|
|
2157
|
+
return expected !== parsed.validationHash;
|
|
2158
|
+
}
|
|
2159
|
+
|
|
2160
|
+
/* eslint-disable @typescript-eslint/no-unused-vars */
|
|
1522
2161
|
const jsonOps = new JSONOps();
|
|
1523
2162
|
/**
|
|
1524
2163
|
* How many `.jsonValues()` entry files a whole-record write reads at a time.
|
|
@@ -2559,7 +3198,14 @@ class ValOps {
|
|
|
2559
3198
|
}
|
|
2560
3199
|
|
|
2561
3200
|
// #region validateSources
|
|
2562
|
-
async validateSources(schemas, sources, patchesByModule
|
|
3201
|
+
async validateSources(schemas, sources, patchesByModule,
|
|
3202
|
+
/**
|
|
3203
|
+
* Validate only these modules. The snapshot the cross-module checks read
|
|
3204
|
+
* (a keyOf's keys, a route, a set's directory) is still built from every
|
|
3205
|
+
* module, which is what makes checking one module at a time give the same
|
|
3206
|
+
* answer for it as checking them all. Wins over `patchesByModule`.
|
|
3207
|
+
*/
|
|
3208
|
+
onlyModules) {
|
|
2563
3209
|
const errors = {};
|
|
2564
3210
|
const files = {};
|
|
2565
3211
|
const remoteFiles = {};
|
|
@@ -2592,7 +3238,18 @@ class ValOps {
|
|
|
2592
3238
|
}
|
|
2593
3239
|
}
|
|
2594
3240
|
}
|
|
2595
|
-
const
|
|
3241
|
+
const addModuleError = (path, sourcePath, validationError) => {
|
|
3242
|
+
if (!errors[path]) {
|
|
3243
|
+
errors[path] = {
|
|
3244
|
+
validations: {}
|
|
3245
|
+
};
|
|
3246
|
+
}
|
|
3247
|
+
if (!errors[path].validations[sourcePath]) {
|
|
3248
|
+
errors[path].validations[sourcePath] = [];
|
|
3249
|
+
}
|
|
3250
|
+
errors[path].validations[sourcePath].push(validationError);
|
|
3251
|
+
};
|
|
3252
|
+
const modulePathsToValidate = onlyModules ?? (patchesByModule && Object.keys(patchesByModule));
|
|
2596
3253
|
for (const [pathS, schema] of entries) {
|
|
2597
3254
|
if (modulePathsToValidate && !modulePathsToValidate.includes(pathS)) {
|
|
2598
3255
|
continue;
|
|
@@ -2621,15 +3278,35 @@ class ValOps {
|
|
|
2621
3278
|
} = await validateJsonValuesEntries(schema, source, path);
|
|
2622
3279
|
for (const [sourcePathS, entryErrors] of Object.entries(jsonValuesErrors)) {
|
|
2623
3280
|
const sourcePath = sourcePathS;
|
|
2624
|
-
|
|
2625
|
-
errors
|
|
2626
|
-
|
|
2627
|
-
|
|
2628
|
-
|
|
2629
|
-
|
|
2630
|
-
|
|
3281
|
+
for (const entryError of entryErrors) {
|
|
3282
|
+
// An entry's errors get the same project-wide resolution as every
|
|
3283
|
+
// other module's. Added raw, every keyOf and route in a
|
|
3284
|
+
// `.jsonValues()` record was reported as unresolved -- 240 of them in
|
|
3285
|
+
// `examples/next`'s kb.val.ts, none real.
|
|
3286
|
+
if (isSchemaSourceFixError(entryError)) {
|
|
3287
|
+
const resolved = resolveSchemaSourceFixForError(entryError, snapshot);
|
|
3288
|
+
if (!resolved || resolved.status !== "remaining") {
|
|
3289
|
+
continue;
|
|
3290
|
+
}
|
|
3291
|
+
addModuleError(path, sourcePath, resolved.error);
|
|
3292
|
+
continue;
|
|
3293
|
+
}
|
|
3294
|
+
// "Not yet compared with its bytes" is a request, not an error, and
|
|
3295
|
+
// for an entry it cannot be answered here: `validateFiles` resolves a
|
|
3296
|
+
// file's schema through the module's source, which for a
|
|
3297
|
+
// `.jsonValues()` record holds markers rather than the entry. The
|
|
3298
|
+
// publish gate ignores it anyway (`partitionValidationErrors`).
|
|
3299
|
+
//
|
|
3300
|
+
// A remote file's "not yet checked against its host" is the same
|
|
3301
|
+
// request, for the same reason: `validateRemoteFiles` and its fix
|
|
3302
|
+
// resolve the path through the module's source too. Added as an
|
|
3303
|
+
// error, every remote image in an entry was a row on the Validate
|
|
3304
|
+
// page whose Fix could not find it.
|
|
3305
|
+
if (isOnlyFileCheckValidationError(entryError) || isOnlyRemoteCheckValidationError(entryError)) {
|
|
3306
|
+
continue;
|
|
3307
|
+
}
|
|
3308
|
+
addModuleError(path, sourcePath, entryError);
|
|
2631
3309
|
}
|
|
2632
|
-
errors[path].validations[sourcePath].push(...entryErrors);
|
|
2633
3310
|
}
|
|
2634
3311
|
if (res === false) {
|
|
2635
3312
|
continue;
|
|
@@ -2707,9 +3384,64 @@ class ValOps {
|
|
|
2707
3384
|
remoteFiles
|
|
2708
3385
|
};
|
|
2709
3386
|
}
|
|
2710
|
-
|
|
2711
|
-
|
|
2712
|
-
|
|
3387
|
+
|
|
3388
|
+
/**
|
|
3389
|
+
* The remote values `validateSources` set aside, checked: is each ref
|
|
3390
|
+
* sound, and does its host serve it. See `checkRemoteFiles`.
|
|
3391
|
+
*
|
|
3392
|
+
* `fileLastUpdatedByPatchId` names the refs whose bytes are still in a
|
|
3393
|
+
* pending patch, which the host will not have until publish.
|
|
3394
|
+
*/
|
|
3395
|
+
async validateRemoteFiles(schemas, sources, remoteFiles, fileLastUpdatedByPatchId = {}, network = {}) {
|
|
3396
|
+
const sampleByHost = this.sourceMode() === "managed";
|
|
3397
|
+
// Only a managed project vouches for one published ref with another, so
|
|
3398
|
+
// only it needs to know which refs are published.
|
|
3399
|
+
const baseSources = sampleByHost ? (await this.getSources()).sources : undefined;
|
|
3400
|
+
const valueAt = (sourcePath, from) => {
|
|
3401
|
+
const [moduleFilePath, modulePath] = Internal.splitModuleFilePathAndModulePath(sourcePath);
|
|
3402
|
+
const schema = schemas[moduleFilePath];
|
|
3403
|
+
if (!schema || from[moduleFilePath] === undefined) {
|
|
3404
|
+
return undefined;
|
|
3405
|
+
}
|
|
3406
|
+
try {
|
|
3407
|
+
return Internal.resolvePath(modulePath, from[moduleFilePath], schema["executeSerialize"]());
|
|
3408
|
+
} catch {
|
|
3409
|
+
return undefined;
|
|
3410
|
+
}
|
|
3411
|
+
};
|
|
3412
|
+
return checkRemoteFiles({
|
|
3413
|
+
remoteFiles,
|
|
3414
|
+
sampleByHost,
|
|
3415
|
+
...network,
|
|
3416
|
+
schemaAt: sourcePath => {
|
|
3417
|
+
var _valueAt;
|
|
3418
|
+
return (_valueAt = valueAt(sourcePath, sources)) === null || _valueAt === void 0 ? void 0 : _valueAt.schema;
|
|
3419
|
+
},
|
|
3420
|
+
isPublished: (sourcePath, ref) => {
|
|
3421
|
+
var _valueAt2;
|
|
3422
|
+
if (!baseSources) {
|
|
3423
|
+
return false;
|
|
3424
|
+
}
|
|
3425
|
+
const published = (_valueAt2 = valueAt(sourcePath, baseSources)) === null || _valueAt2 === void 0 ? void 0 : _valueAt2.source;
|
|
3426
|
+
return typeof published === "object" && published !== null && "path" in published && published.path === ref;
|
|
3427
|
+
},
|
|
3428
|
+
pendingBytes: this.pendingBytesOf(fileLastUpdatedByPatchId)
|
|
3429
|
+
});
|
|
3430
|
+
}
|
|
3431
|
+
|
|
3432
|
+
/**
|
|
3433
|
+
* Is a remote file's bytes in a pending patch? `not-pending` when no patch
|
|
3434
|
+
* wrote it; `missing` when one did and its bytes are not there.
|
|
3435
|
+
*/
|
|
3436
|
+
pendingBytesOf(fileLastUpdatedByPatchId) {
|
|
3437
|
+
return async ref => {
|
|
3438
|
+
const fromPatch = fileLastUpdatedByPatchId[ref];
|
|
3439
|
+
if (!fromPatch) {
|
|
3440
|
+
return "not-pending";
|
|
3441
|
+
}
|
|
3442
|
+
const bytes = await this.getBase64EncodedBinaryFileFromPatch(ref, fromPatch.patchId, fromPatch.remote);
|
|
3443
|
+
return bytes === null ? "missing" : "present";
|
|
3444
|
+
};
|
|
2713
3445
|
}
|
|
2714
3446
|
|
|
2715
3447
|
// #region validateFiles
|
|
@@ -2781,7 +3513,9 @@ class ValOps {
|
|
|
2781
3513
|
if (metadataErrors && metadataErrors.length > 0) {
|
|
2782
3514
|
return {
|
|
2783
3515
|
[sourcePath]: metadataErrors.map(e => ({
|
|
2784
|
-
|
|
3516
|
+
// Which file: "File not found" on its own names nothing, and the
|
|
3517
|
+
// row it is shown on is the field, not the file.
|
|
3518
|
+
message: `${e.message}: ${filePath}`,
|
|
2785
3519
|
value: {
|
|
2786
3520
|
filePath,
|
|
2787
3521
|
patchId: (fileData === null || fileData === void 0 ? void 0 : fileData.patchId) ?? null
|
|
@@ -3830,7 +4564,7 @@ class ValOps {
|
|
|
3830
4564
|
* override it.
|
|
3831
4565
|
*/
|
|
3832
4566
|
async saveBinaryFileFromPatch(filePath, parentRef, patchId, bytes, mimeType, type, metadata) {
|
|
3833
|
-
return this.saveBase64EncodedBinaryFileFromPatch(filePath, parentRef, patchId, `data:${mimeType};base64,${bytes
|
|
4567
|
+
return this.saveBase64EncodedBinaryFileFromPatch(filePath, parentRef, patchId, `data:${mimeType};base64,${bytesToBase64(bytes)}`, type, metadata);
|
|
3834
4568
|
}
|
|
3835
4569
|
/**
|
|
3836
4570
|
* A file's bytes as something a `Range` request can be answered from
|
|
@@ -3894,6 +4628,10 @@ class ValOps {
|
|
|
3894
4628
|
|
|
3895
4629
|
// #endregion history
|
|
3896
4630
|
}
|
|
4631
|
+
function isOnlyRemoteCheckValidationError(validationError) {
|
|
4632
|
+
const fixes = validationError.fixes ?? [];
|
|
4633
|
+
return fixes.length > 0 && fixes.every(f => f === "image:check-remote" || f === "file:check-remote");
|
|
4634
|
+
}
|
|
3897
4635
|
function isOnlyFileCheckValidationError(validationError) {
|
|
3898
4636
|
var _validationError$fixe9;
|
|
3899
4637
|
if ((_validationError$fixe9 = validationError.fixes) !== null && _validationError$fixe9 !== void 0 && _validationError$fixe9.every(f => f === "file:check-metadata" || f === "image:check-metadata")) {
|
|
@@ -4052,15 +4790,16 @@ function createMetadataFromBuffer(type, mimeType, buffer) {
|
|
|
4052
4790
|
metadata
|
|
4053
4791
|
};
|
|
4054
4792
|
}
|
|
4055
|
-
function bufferFromDataUrl(dataUrl
|
|
4793
|
+
function bufferFromDataUrl(dataUrl, /** Node passes `Buffer`'s, which is faster: this is a whole upload. */
|
|
4794
|
+
decodeBase64 = base64ToBytes) {
|
|
4056
4795
|
let base64Data;
|
|
4057
4796
|
const base64Index = dataUrl.indexOf(";base64,");
|
|
4058
4797
|
if (base64Index > -1) {
|
|
4059
4798
|
base64Data = dataUrl.slice(base64Index + ";base64,".length);
|
|
4060
4799
|
}
|
|
4061
4800
|
if (base64Data) {
|
|
4062
|
-
|
|
4063
|
-
);
|
|
4801
|
+
// TODO: why does it not work with base64url?
|
|
4802
|
+
return decodeBase64(base64Data);
|
|
4064
4803
|
}
|
|
4065
4804
|
}
|
|
4066
4805
|
|
|
@@ -4211,7 +4950,7 @@ async function resolveJsonEntriesAtCommit(ops, commitSha, affectedFiles) {
|
|
|
4211
4950
|
continue;
|
|
4212
4951
|
}
|
|
4213
4952
|
try {
|
|
4214
|
-
entries[gitPath] = JSON.parse(res.value
|
|
4953
|
+
entries[gitPath] = JSON.parse(bytesToUtf8(res.value));
|
|
4215
4954
|
} catch (err) {
|
|
4216
4955
|
failures.push({
|
|
4217
4956
|
kind: "file-unavailable",
|
|
@@ -4533,7 +5272,14 @@ const SettingsSchema = z.object({
|
|
|
4533
5272
|
publicProjectId: z.string(),
|
|
4534
5273
|
remoteFileBuckets: z.array(z.object({
|
|
4535
5274
|
bucket: z.string()
|
|
4536
|
-
}))
|
|
5275
|
+
})),
|
|
5276
|
+
/**
|
|
5277
|
+
* The host a new remote file's ref names: the project's own address with
|
|
5278
|
+
* `/_val` for a Hosted project whose files are in R2, `remote.val.build`
|
|
5279
|
+
* otherwise. Absent from a content service that predates it, where every
|
|
5280
|
+
* project's files are on `remote.val.build`.
|
|
5281
|
+
*/
|
|
5282
|
+
remoteHost: z.string().optional()
|
|
4537
5283
|
});
|
|
4538
5284
|
async function getSettings(projectName, auth, /** Overrides {@link defaultHost}. See the note there for why this exists. */
|
|
4539
5285
|
contentUrl) {
|
|
@@ -4709,11 +5455,6 @@ function mapStatus(status) {
|
|
|
4709
5455
|
return 500;
|
|
4710
5456
|
}
|
|
4711
5457
|
|
|
4712
|
-
/*
|
|
4713
|
-
* Imported rather than global: under Node this is the built-in, and in a
|
|
4714
|
-
* browser (the demo, `@valbuild/server/browser`) the `buffer` package.
|
|
4715
|
-
*/
|
|
4716
|
-
|
|
4717
5458
|
/** What {@link ValServerOptions.publishOverride} is given. */
|
|
4718
5459
|
|
|
4719
5460
|
/** What a publish reports back, in either shape. */
|
|
@@ -4724,6 +5465,18 @@ function mapStatus(status) {
|
|
|
4724
5465
|
* have their own `dir` for any collection that wants somewhere else.
|
|
4725
5466
|
*/
|
|
4726
5467
|
const FILES_DIRECTORY = "/public/val";
|
|
5468
|
+
|
|
5469
|
+
/**
|
|
5470
|
+
* `/validate` and `/validate/fix` on a platform without the full check (a
|
|
5471
|
+
* server in a browser tab). 501: nothing the caller does makes it available,
|
|
5472
|
+
* so the Studio offers no Try again.
|
|
5473
|
+
*/
|
|
5474
|
+
const studioValidationUnavailable = {
|
|
5475
|
+
status: 501,
|
|
5476
|
+
json: {
|
|
5477
|
+
message: "Checking everything needs the project's files and Val's file hosts, and this Studio runs without them."
|
|
5478
|
+
}
|
|
5479
|
+
};
|
|
4727
5480
|
const ValServer = (valModules, options, callbacks, platform) => {
|
|
4728
5481
|
var _options$git;
|
|
4729
5482
|
const {
|
|
@@ -5127,6 +5880,82 @@ const ValServer = (valModules, options, callbacks, platform) => {
|
|
|
5127
5880
|
};
|
|
5128
5881
|
}
|
|
5129
5882
|
};
|
|
5883
|
+
|
|
5884
|
+
/**
|
|
5885
|
+
* What the Studio's full check reads: base plus the patches the caller is
|
|
5886
|
+
* showing, as `/sources/~` resolves an explicit `patch_id` list -- including
|
|
5887
|
+
* `draftOverlay`, so a module changed by a publish after this build is
|
|
5888
|
+
* checked as the caller sees it, not as this build has it.
|
|
5889
|
+
*/
|
|
5890
|
+
const studioCheckInput = async (cookies, moduleFilePath, patchIds) => {
|
|
5891
|
+
const auth = getAuth(cookies);
|
|
5892
|
+
if (auth.error) {
|
|
5893
|
+
return {
|
|
5894
|
+
status: 401,
|
|
5895
|
+
json: {
|
|
5896
|
+
message: auth.error
|
|
5897
|
+
}
|
|
5898
|
+
};
|
|
5899
|
+
}
|
|
5900
|
+
if (isHttpOps(serverOps) && !("id" in auth)) {
|
|
5901
|
+
return {
|
|
5902
|
+
status: 401,
|
|
5903
|
+
json: {
|
|
5904
|
+
message: "Unauthorized"
|
|
5905
|
+
}
|
|
5906
|
+
};
|
|
5907
|
+
}
|
|
5908
|
+
const moduleErrors = await serverOps.getModuleErrors();
|
|
5909
|
+
if ((moduleErrors === null || moduleErrors === void 0 ? void 0 : moduleErrors.length) > 0) {
|
|
5910
|
+
return {
|
|
5911
|
+
status: 500,
|
|
5912
|
+
json: {
|
|
5913
|
+
message: "Val could not load this project's modules, so nothing can be checked. Check val.modules."
|
|
5914
|
+
}
|
|
5915
|
+
};
|
|
5916
|
+
}
|
|
5917
|
+
// An EXPLICITLY empty list is base, never "everything": `fetchPatches`
|
|
5918
|
+
// reads an empty filter as no filter (see `/sources/~`).
|
|
5919
|
+
let patchOps = patchIds !== undefined && patchIds.length === 0 ? {
|
|
5920
|
+
patches: []
|
|
5921
|
+
} : await serverOps.fetchPatches({
|
|
5922
|
+
patchIds,
|
|
5923
|
+
excludePatchOps: false
|
|
5924
|
+
});
|
|
5925
|
+
if ("error" in patchOps && patchOps.error) {
|
|
5926
|
+
return "unauthorized" in patchOps && patchOps.unauthorized ? {
|
|
5927
|
+
status: 401,
|
|
5928
|
+
json: {
|
|
5929
|
+
message: "Unauthorized"
|
|
5930
|
+
}
|
|
5931
|
+
} : {
|
|
5932
|
+
status: 500,
|
|
5933
|
+
json: {
|
|
5934
|
+
message: "The unpublished changes could not be read, so nothing was checked."
|
|
5935
|
+
}
|
|
5936
|
+
};
|
|
5937
|
+
}
|
|
5938
|
+
patchOps = {
|
|
5939
|
+
...patchOps,
|
|
5940
|
+
patches: draftOverlay(patchOps.patches, patchOps.commits)
|
|
5941
|
+
};
|
|
5942
|
+
const analysis = serverOps.analyzePatches(patchOps.patches);
|
|
5943
|
+
const {
|
|
5944
|
+
sources
|
|
5945
|
+
} = await serverOps.getSourcesWithPatchesApplied({
|
|
5946
|
+
...analysis,
|
|
5947
|
+
...patchOps
|
|
5948
|
+
});
|
|
5949
|
+
return {
|
|
5950
|
+
status: "ok",
|
|
5951
|
+
input: {
|
|
5952
|
+
moduleFilePath,
|
|
5953
|
+
schemas: await serverOps.getSchemas(),
|
|
5954
|
+
sources,
|
|
5955
|
+
fileLastUpdatedByPatchId: analysis.fileLastUpdatedByPatchId
|
|
5956
|
+
}
|
|
5957
|
+
};
|
|
5958
|
+
};
|
|
5130
5959
|
return {
|
|
5131
5960
|
"/draft/enable": {
|
|
5132
5961
|
GET: async req => {
|
|
@@ -5477,7 +6306,10 @@ const ValServer = (valModules, options, callbacks, platform) => {
|
|
|
5477
6306
|
json: {
|
|
5478
6307
|
publicProjectId: settingsRes.data.publicProjectId,
|
|
5479
6308
|
remoteFileBuckets: settingsRes.data.remoteFileBuckets,
|
|
5480
|
-
coreVersion: Internal.VERSION.core || "unknown"
|
|
6309
|
+
coreVersion: Internal.VERSION.core || "unknown",
|
|
6310
|
+
...(settingsRes.data.remoteHost !== undefined ? {
|
|
6311
|
+
remoteHost: settingsRes.data.remoteHost
|
|
6312
|
+
} : {})
|
|
5481
6313
|
}
|
|
5482
6314
|
};
|
|
5483
6315
|
}
|
|
@@ -5758,6 +6590,92 @@ const ValServer = (valModules, options, callbacks, platform) => {
|
|
|
5758
6590
|
};
|
|
5759
6591
|
}
|
|
5760
6592
|
},
|
|
6593
|
+
"/validate": {
|
|
6594
|
+
POST: async req => {
|
|
6595
|
+
const checked = await studioCheckInput(req.cookies, req.body.moduleFilePath, req.body.patchIds);
|
|
6596
|
+
if (checked.status !== "ok") {
|
|
6597
|
+
return checked;
|
|
6598
|
+
}
|
|
6599
|
+
if (!platform.studioValidation) {
|
|
6600
|
+
return studioValidationUnavailable;
|
|
6601
|
+
}
|
|
6602
|
+
const res = await platform.studioValidation.checkModuleForStudio(serverOps, checked.input);
|
|
6603
|
+
switch (res.status) {
|
|
6604
|
+
case "unknown-module":
|
|
6605
|
+
return {
|
|
6606
|
+
status: 404,
|
|
6607
|
+
json: {
|
|
6608
|
+
message: `There is no module '${req.body.moduleFilePath}' in this project.`
|
|
6609
|
+
}
|
|
6610
|
+
};
|
|
6611
|
+
case "invalid-source":
|
|
6612
|
+
return {
|
|
6613
|
+
status: 200,
|
|
6614
|
+
json: {
|
|
6615
|
+
moduleFilePath: req.body.moduleFilePath,
|
|
6616
|
+
errors: {},
|
|
6617
|
+
invalidSource: res.message
|
|
6618
|
+
}
|
|
6619
|
+
};
|
|
6620
|
+
case "ok":
|
|
6621
|
+
return {
|
|
6622
|
+
status: 200,
|
|
6623
|
+
json: {
|
|
6624
|
+
moduleFilePath: req.body.moduleFilePath,
|
|
6625
|
+
errors: res.errors,
|
|
6626
|
+
remoteHosts: res.remoteHosts
|
|
6627
|
+
}
|
|
6628
|
+
};
|
|
6629
|
+
}
|
|
6630
|
+
}
|
|
6631
|
+
},
|
|
6632
|
+
"/validate/fix": {
|
|
6633
|
+
POST: async req => {
|
|
6634
|
+
const checked = await studioCheckInput(req.cookies, req.body.moduleFilePath, req.body.patchIds);
|
|
6635
|
+
if (checked.status !== "ok") {
|
|
6636
|
+
return checked;
|
|
6637
|
+
}
|
|
6638
|
+
if (!platform.studioValidation) {
|
|
6639
|
+
return studioValidationUnavailable;
|
|
6640
|
+
}
|
|
6641
|
+
const res = await platform.studioValidation.fixForStudio(serverOps, {
|
|
6642
|
+
...checked.input,
|
|
6643
|
+
sourcePath: req.body.sourcePath,
|
|
6644
|
+
fix: req.body.fix
|
|
6645
|
+
});
|
|
6646
|
+
switch (res.status) {
|
|
6647
|
+
case "unknown-module":
|
|
6648
|
+
return {
|
|
6649
|
+
status: 404,
|
|
6650
|
+
json: {
|
|
6651
|
+
message: `There is no module '${req.body.moduleFilePath}' in this project.`
|
|
6652
|
+
}
|
|
6653
|
+
};
|
|
6654
|
+
case "not-found":
|
|
6655
|
+
return {
|
|
6656
|
+
status: 404,
|
|
6657
|
+
json: {
|
|
6658
|
+
message: "This error is no longer there. Check the module again to see what is left."
|
|
6659
|
+
}
|
|
6660
|
+
};
|
|
6661
|
+
case "not-in-studio":
|
|
6662
|
+
return {
|
|
6663
|
+
status: 400,
|
|
6664
|
+
json: {
|
|
6665
|
+
message: "This fix cannot be made from the Studio yet. Run `val validate --fix` in the project."
|
|
6666
|
+
}
|
|
6667
|
+
};
|
|
6668
|
+
case "ok":
|
|
6669
|
+
return {
|
|
6670
|
+
status: 200,
|
|
6671
|
+
json: {
|
|
6672
|
+
patch: res.patch,
|
|
6673
|
+
remainingErrors: res.remainingErrors
|
|
6674
|
+
}
|
|
6675
|
+
};
|
|
6676
|
+
}
|
|
6677
|
+
}
|
|
6678
|
+
},
|
|
5761
6679
|
"/external-urls/check": {
|
|
5762
6680
|
POST: async req => {
|
|
5763
6681
|
/*
|
|
@@ -6431,8 +7349,9 @@ const ValServer = (valModules, options, callbacks, platform) => {
|
|
|
6431
7349
|
// TODO: send binary files validation errors
|
|
6432
7350
|
if (query.validate_binary_files) {
|
|
6433
7351
|
await serverOps.validateFiles(schemas, sourcesRes.sources, sourcesValidation.files);
|
|
6434
|
-
//
|
|
6435
|
-
|
|
7352
|
+
// Remote files are not checked here: their check asks each host
|
|
7353
|
+
// over the network, so it is the Studio's full check that runs it,
|
|
7354
|
+
// one module at a time (`/validate`).
|
|
6436
7355
|
}
|
|
6437
7356
|
}
|
|
6438
7357
|
const schemaSha = await serverOps.getSchemaSha();
|
|
@@ -6580,7 +7499,7 @@ const ValServer = (valModules, options, callbacks, platform) => {
|
|
|
6580
7499
|
withJob: prepared,
|
|
6581
7500
|
jobPatchIds: patchIds
|
|
6582
7501
|
});
|
|
6583
|
-
const build = serverOps.embedsSource() ? await buildSourcesOf(serverOps, patchIds, commit) : null;
|
|
7502
|
+
const build = serverOps.embedsSource() ? await buildSourcesOf(platform, serverOps, patchIds, commit) : null;
|
|
6584
7503
|
if (build !== null && "status" in build) return build;
|
|
6585
7504
|
/*
|
|
6586
7505
|
* A build of no commit says what Val source it holds instead, so
|
|
@@ -6612,7 +7531,7 @@ const ValServer = (valModules, options, callbacks, platform) => {
|
|
|
6612
7531
|
buildable: build !== null
|
|
6613
7532
|
});
|
|
6614
7533
|
}
|
|
6615
|
-
const built = await buildSourcesOf(serverOps, patchIds, commit);
|
|
7534
|
+
const built = await buildSourcesOf(platform, serverOps, patchIds, commit);
|
|
6616
7535
|
if ("status" in built) return built;
|
|
6617
7536
|
const {
|
|
6618
7537
|
split
|
|
@@ -6682,7 +7601,7 @@ const ValServer = (valModules, options, callbacks, platform) => {
|
|
|
6682
7601
|
* and the selection on top. Only what the selection touched is saved,
|
|
6683
7602
|
* so the proposal's save is the selection and nothing else.
|
|
6684
7603
|
*/
|
|
6685
|
-
const built = await buildSourcesOf(serverOps, patchIds, commit, {
|
|
7604
|
+
const built = await buildSourcesOf(platform, serverOps, patchIds, commit, {
|
|
6686
7605
|
binaries: false
|
|
6687
7606
|
});
|
|
6688
7607
|
if ("status" in built) return built;
|
|
@@ -7227,7 +8146,7 @@ const ValServer = (valModules, options, callbacks, platform) => {
|
|
|
7227
8146
|
* patch's to read. See `binaryFiles` on the route.
|
|
7228
8147
|
*/
|
|
7229
8148
|
const managedBranch = serverOps.sourceMode() === "managed" ? serverOps.projectBranch() : null;
|
|
7230
|
-
const committedBinaries = serverOps.sourceMode() === "managed" ? await readCommittedBinaryFiles(serverOps, preparedCommit.patchedBinaryFilesDescriptors) : null;
|
|
8149
|
+
const committedBinaries = serverOps.sourceMode() === "managed" ? await readCommittedBinaryFiles(serverOps, preparedCommit.patchedBinaryFilesDescriptors, platform.bytesToBase64) : null;
|
|
7231
8150
|
const commitToGit = () => serverOps.commit(preparedCommit, message, auth.id, FILES_DIRECTORY, undefined);
|
|
7232
8151
|
/*
|
|
7233
8152
|
* A host may publish somewhere other than the repository.
|
|
@@ -7887,7 +8806,7 @@ const ValServer = (valModules, options, callbacks, platform) => {
|
|
|
7887
8806
|
// wrap it in a data: URL, and hand that to a method whose first
|
|
7888
8807
|
// act was to unwrap it again - ceremony to satisfy a convention
|
|
7889
8808
|
// that no longer exists.
|
|
7890
|
-
const bytes =
|
|
8809
|
+
const bytes = new Uint8Array(await binaryRes.arrayBuffer());
|
|
7891
8810
|
const type = file.metadata.mimeType.startsWith("image/") ? "image" : "file";
|
|
7892
8811
|
const saveRes = await serverOps.saveBinaryFileFromPatch(file.filePath, req.body.parentRef, req.body.patchId, bytes, file.metadata.mimeType, type, file.metadata);
|
|
7893
8812
|
if (saveRes.error) {
|
|
@@ -8133,7 +9052,7 @@ const ValServer = (valModules, options, callbacks, platform) => {
|
|
|
8133
9052
|
// TypeScript compiler. See `patch/ts/publishKit.ts`.
|
|
8134
9053
|
const {
|
|
8135
9054
|
getJsonEntryAtCommit
|
|
8136
|
-
} = await import('./getJsonEntryAtCommit-
|
|
9055
|
+
} = await import('./getJsonEntryAtCommit-40b6d9c1.esm.js');
|
|
8137
9056
|
const res = await getJsonEntryAtCommit(serverOps, req.query.commit_sha, req.query.path, req.query.key);
|
|
8138
9057
|
if (result.isErr(res)) {
|
|
8139
9058
|
return historyErrorResponse(res.error);
|
|
@@ -8241,11 +9160,11 @@ const ValServer = (valModules, options, callbacks, platform) => {
|
|
|
8241
9160
|
// A DRAFT playlist names files that are drafts too, at URLs that only
|
|
8242
9161
|
// resolve once published. See `rewriteDraftPlaylist`. Playlists are
|
|
8243
9162
|
// small, so one is read whole to be rewritten.
|
|
8244
|
-
const file = patchId && isHlsPlaylistPath(filePath) ? bufferReader(
|
|
9163
|
+
const file = patchId && isHlsPlaylistPath(filePath) ? bufferReader(utf8ToBytes(rewriteDraftPlaylist(bytesToUtf8(await opened.read(0, opened.size - 1)), {
|
|
8245
9164
|
playlistPath: filePath,
|
|
8246
9165
|
patchId,
|
|
8247
9166
|
remote
|
|
8248
|
-
})
|
|
9167
|
+
}))) : opened;
|
|
8249
9168
|
const headers = {
|
|
8250
9169
|
// TODO: we could use ETag and return 304 instead
|
|
8251
9170
|
// From the extension, published or draft: a `<video>` and an HLS
|
|
@@ -8285,7 +9204,7 @@ const ValServer = (valModules, options, callbacks, platform) => {
|
|
|
8285
9204
|
body: bufferToReadableStream(slice)
|
|
8286
9205
|
};
|
|
8287
9206
|
}
|
|
8288
|
-
const body = file.size === 0 ?
|
|
9207
|
+
const body = file.size === 0 ? new Uint8Array(0) : await file.read(0, file.size - 1);
|
|
8289
9208
|
return {
|
|
8290
9209
|
status: 200,
|
|
8291
9210
|
headers: {
|
|
@@ -8347,7 +9266,7 @@ function verifyCallbackReq(stateCookie, queryParams) {
|
|
|
8347
9266
|
}
|
|
8348
9267
|
function getStateFromCookie(stateCookie) {
|
|
8349
9268
|
try {
|
|
8350
|
-
const decoded =
|
|
9269
|
+
const decoded = bytesToUtf8(base64ToBytes(stateCookie));
|
|
8351
9270
|
const parsed = JSON.parse(decoded);
|
|
8352
9271
|
if (!parsed) {
|
|
8353
9272
|
return {
|
|
@@ -8399,7 +9318,7 @@ function getStateFromCookie(stateCookie) {
|
|
|
8399
9318
|
}
|
|
8400
9319
|
}
|
|
8401
9320
|
function createStateCookie(state) {
|
|
8402
|
-
return
|
|
9321
|
+
return bytesToBase64(utf8ToBytes(JSON.stringify(state)));
|
|
8403
9322
|
}
|
|
8404
9323
|
const ValAppJwtPayload = z.object({
|
|
8405
9324
|
sub: z.string(),
|
|
@@ -8682,7 +9601,7 @@ function getIsRemoteRequired(schemas) {
|
|
|
8682
9601
|
* `split.archive` is the managed job's archive; a connected job's archive is
|
|
8683
9602
|
* the push's, made from its own changes alone, by the caller.
|
|
8684
9603
|
*/
|
|
8685
|
-
async function buildSourcesOf(serverOps, patchIds, commit,
|
|
9604
|
+
async function buildSourcesOf(platform, serverOps, patchIds, commit,
|
|
8686
9605
|
/**
|
|
8687
9606
|
* Read the job's binary files back for a build. Off for a caller that only
|
|
8688
9607
|
* commits what was prepared, which names the files rather than carrying them.
|
|
@@ -8748,7 +9667,7 @@ options = {
|
|
|
8748
9667
|
jobPatchIds: patchIds
|
|
8749
9668
|
});
|
|
8750
9669
|
// Read now, while the job's changes still hold their files.
|
|
8751
|
-
const binaries = options.binaries ? await readCommittedBinaryFiles(serverOps, split.archive.patchedBinaryFilesDescriptors) : {
|
|
9670
|
+
const binaries = options.binaries ? await readCommittedBinaryFiles(serverOps, split.archive.patchedBinaryFilesDescriptors, platform.bytesToBase64) : {
|
|
8752
9671
|
files: {},
|
|
8753
9672
|
unread: []
|
|
8754
9673
|
};
|
|
@@ -8863,11 +9782,6 @@ async function uploadRemoteFile(contentHost, project, bucket, fileHash, fileExt,
|
|
|
8863
9782
|
};
|
|
8864
9783
|
}
|
|
8865
9784
|
|
|
8866
|
-
function getFileExt(filePath) {
|
|
8867
|
-
// NOTE: We do not import the path module. This code is copied in different projects. We want the same implementation and which means that this might running in browser where path is not available).
|
|
8868
|
-
return filePath.split(".").pop() || "";
|
|
8869
|
-
}
|
|
8870
|
-
|
|
8871
9785
|
/**
|
|
8872
9786
|
* One stored patch. Everything the ordered chain needs and nothing else.
|
|
8873
9787
|
*/
|
|
@@ -9568,7 +10482,11 @@ function createValRequestHandler(route, valServerPromise, convert, serveStatic)
|
|
|
9568
10482
|
const method = (_req$method = req.method) === null || _req$method === void 0 ? void 0 : _req$method.toUpperCase();
|
|
9569
10483
|
let route = null;
|
|
9570
10484
|
let path = undefined;
|
|
9571
|
-
|
|
10485
|
+
// An exact match wins over every prefix match, wherever it sits in
|
|
10486
|
+
// `Api`: checked per key, in order, `/validate` took `/validate/fix`
|
|
10487
|
+
// because it was declared first.
|
|
10488
|
+
const exactRouteDef = Object.keys(Api).find(routeDef => routeDef === reqApiRoutePath);
|
|
10489
|
+
for (const routeDef of exactRouteDef ? [exactRouteDef] : Object.keys(Api)) {
|
|
9572
10490
|
if (routeDef === reqApiRoutePath) {
|
|
9573
10491
|
route = routeDef;
|
|
9574
10492
|
break;
|
|
@@ -9863,4 +10781,4 @@ function getCookies(req, cookiesDef) {
|
|
|
9863
10781
|
return z.object(cookiesDef).safeParse(input);
|
|
9864
10782
|
}
|
|
9865
10783
|
|
|
9866
|
-
export { InMemoryPatchStore as I, ValOps as V, rebaseContentOp as a, bufferFromDataUrl as b, classifyJsonValuesOp as c, getFileExt as d, guessMimeTypeFromPath as e, findNestedJsonValuesRecords as f, getFieldsForType as g, createMetadataFromBuffer as h, isJsonValuesRootOp as i, ValOpsMemory as j,
|
|
10784
|
+
export { getNewJsonEntryPaths as A, formatPatchSourceError as B, askOncePerHost as C, getSettings as D, bytesToBase64 as E, bytesToUtf8 as F, posixPath as G, InMemoryPatchStore as I, ValOps as V, rebaseContentOp as a, bufferFromDataUrl as b, classifyJsonValuesOp as c, getFileExt as d, guessMimeTypeFromPath as e, findNestedJsonValuesRecords as f, getFieldsForType as g, createMetadataFromBuffer as h, isJsonValuesRootOp as i, ValOpsMemory as j, isHlsMasterPlaylist as k, largestHlsVariant as l, fallbackRefOf as m, mapHlsUris as n, movedRemoteHost as o, parseHlsMasterVariants as p, needsHostCheck as q, resolveExistingJsonPath as r, sumHlsSegmentDurations as s, rewriteRefHost as t, uploadRemoteFile as u, validateJsonValuesEntries as v, fallbackRemoteHost as w, checkRemoteFiles as x, ValServer as y, createValRequestHandler as z };
|