@valbuild/server 0.147.0 → 0.148.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +108 -0
- package/browser/dist/valbuild-server-browser.cjs.dev.js +2 -2
- package/browser/dist/valbuild-server-browser.cjs.prod.js +2 -2
- package/browser/dist/valbuild-server-browser.esm.js +2 -2
- package/dist/{ValRouterCore-7ffcc08c.cjs.prod.js → ValRouterCore-42772c21.cjs.prod.js} +980 -53
- package/dist/{ValRouterCore-f5004f8f.esm.js → ValRouterCore-7ca3dc68.esm.js} +972 -54
- package/dist/{ValRouterCore-3fd3d444.cjs.dev.js → ValRouterCore-9bdaaf89.cjs.dev.js} +980 -53
- package/dist/declarations/src/ValOps.d.ts +32 -11
- package/dist/declarations/src/ValOpsMemory.d.ts +4 -4
- package/dist/declarations/src/ValServer.d.ts +1 -2
- package/dist/declarations/src/checkRemoteRef.d.ts +40 -2
- package/dist/declarations/src/createFixPatch.d.ts +25 -3
- package/dist/declarations/src/fixFiles.d.ts +42 -0
- package/dist/declarations/src/fixHandlers.d.ts +14 -0
- package/dist/declarations/src/getSettings.d.ts +1 -0
- package/dist/declarations/src/index.d.ts +1 -0
- package/dist/declarations/src/platform.d.ts +16 -0
- package/dist/declarations/src/remoteFileCheck.d.ts +116 -0
- package/dist/declarations/src/remoteHostCheck.d.ts +101 -0
- package/dist/declarations/src/studioValidation.d.ts +84 -0
- package/dist/declarations/src/uploadRemoteFile.d.ts +1 -1
- package/dist/declarations/src/videosetFixes.d.ts +3 -2
- package/dist/{getJsonEntryAtCommit-df65d861.cjs.prod.js → getJsonEntryAtCommit-2e36e805.cjs.prod.js} +3 -4
- package/dist/{getJsonEntryAtCommit-1e08d724.esm.js → getJsonEntryAtCommit-40b6d9c1.esm.js} +3 -4
- package/dist/{getJsonEntryAtCommit-e96beb9e.cjs.dev.js → getJsonEntryAtCommit-64f67a50.cjs.dev.js} +3 -4
- package/dist/valbuild-server.cjs.dev.js +1352 -695
- package/dist/valbuild-server.cjs.prod.js +1352 -695
- package/dist/valbuild-server.esm.js +1352 -696
- package/package.json +4 -5
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
var buffer = require('buffer');
|
|
4
3
|
var core = require('@valbuild/core');
|
|
5
4
|
var internal = require('@valbuild/shared/internal');
|
|
6
5
|
var z = require('zod');
|
|
@@ -928,6 +927,118 @@ async function validateJsonValuesEntries(schema, source, modulePath) {
|
|
|
928
927
|
return res;
|
|
929
928
|
}
|
|
930
929
|
|
|
930
|
+
/**
|
|
931
|
+
* Bytes as text and back, without Node's `Buffer`.
|
|
932
|
+
*
|
|
933
|
+
* The request handlers run in a browser tab too (`@valbuild/server/browser`),
|
|
934
|
+
* and a browser has no `Buffer`. Importing npm's `buffer` instead is what broke
|
|
935
|
+
* the demo under `vite dev`: it is CommonJS, and a dependency the dev server
|
|
936
|
+
* did not see in its scan is served unconverted. So bytes are `Uint8Array`
|
|
937
|
+
* here, and these are the conversions `Buffer` used to do -- with the same
|
|
938
|
+
* answers, which `bytes.test.ts` checks against `Buffer` itself.
|
|
939
|
+
*/
|
|
940
|
+
|
|
941
|
+
/**
|
|
942
|
+
* The value of each character code, or -1 for one that is not base64. Both
|
|
943
|
+
* alphabets, because `Buffer.from(s, "base64")` takes either: `-` and `_` are
|
|
944
|
+
* the URL-safe spellings of `+` and `/`.
|
|
945
|
+
*/
|
|
946
|
+
const BASE64_VALUES = (() => {
|
|
947
|
+
const BASE64_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
|
948
|
+
const values = new Int16Array(128).fill(-1);
|
|
949
|
+
for (let i = 0; i < BASE64_ALPHABET.length; i++) {
|
|
950
|
+
values[BASE64_ALPHABET.charCodeAt(i)] = i;
|
|
951
|
+
}
|
|
952
|
+
values["-".charCodeAt(0)] = 62;
|
|
953
|
+
values["_".charCodeAt(0)] = 63;
|
|
954
|
+
return values;
|
|
955
|
+
})();
|
|
956
|
+
const PADDING = "=".charCodeAt(0);
|
|
957
|
+
|
|
958
|
+
/**
|
|
959
|
+
* Decodes base64 the way `Buffer.from(base64, "base64")` does: either
|
|
960
|
+
* alphabet, padding optional, anything that is not base64 skipped, and
|
|
961
|
+
* decoding stops at the first `=`. A trailing partial group gives the whole
|
|
962
|
+
* bytes it holds.
|
|
963
|
+
*
|
|
964
|
+
* `atob` first, because it is native and this is called with whole uploads;
|
|
965
|
+
* it agrees with `Buffer` on everything it accepts, and refuses the rest (the
|
|
966
|
+
* URL-safe alphabet, stray characters, `=` in the middle), which then goes the
|
|
967
|
+
* slow way.
|
|
968
|
+
*/
|
|
969
|
+
function base64ToBytes(base64) {
|
|
970
|
+
let binary;
|
|
971
|
+
try {
|
|
972
|
+
binary = atob(base64);
|
|
973
|
+
} catch {
|
|
974
|
+
return lenientBase64ToBytes(base64);
|
|
975
|
+
}
|
|
976
|
+
const bytes = new Uint8Array(binary.length);
|
|
977
|
+
for (let i = 0; i < binary.length; i++) {
|
|
978
|
+
bytes[i] = binary.charCodeAt(i);
|
|
979
|
+
}
|
|
980
|
+
return bytes;
|
|
981
|
+
}
|
|
982
|
+
function lenientBase64ToBytes(base64) {
|
|
983
|
+
const bytes = new Uint8Array(Math.floor(base64.length * 3 / 4));
|
|
984
|
+
let length = 0;
|
|
985
|
+
let bits = 0;
|
|
986
|
+
let bitCount = 0;
|
|
987
|
+
for (let i = 0; i < base64.length; i++) {
|
|
988
|
+
const code = base64.charCodeAt(i);
|
|
989
|
+
if (code === PADDING) break;
|
|
990
|
+
const value = code < 128 ? BASE64_VALUES[code] : -1;
|
|
991
|
+
if (value < 0) continue;
|
|
992
|
+
bits = (bits << 6 | value) & 0xffffff;
|
|
993
|
+
bitCount += 6;
|
|
994
|
+
if (bitCount >= 8) {
|
|
995
|
+
bitCount -= 8;
|
|
996
|
+
bytes[length++] = bits >> bitCount & 0xff;
|
|
997
|
+
}
|
|
998
|
+
}
|
|
999
|
+
return bytes.subarray(0, length);
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
/**
|
|
1003
|
+
* How many bytes go to `String.fromCharCode` at once: it takes them as
|
|
1004
|
+
* arguments, and an engine has a limit on how many a call may have.
|
|
1005
|
+
*/
|
|
1006
|
+
const CHUNK = 0x8000;
|
|
1007
|
+
|
|
1008
|
+
/**
|
|
1009
|
+
* `Buffer.from(bytes).toString("base64")`: padded, standard alphabet.
|
|
1010
|
+
*
|
|
1011
|
+
* Through `btoa`, which is native, rather than building the string here: that
|
|
1012
|
+
* was five times slower, and this is called with whole files. Node is still
|
|
1013
|
+
* faster with `Buffer`, which is why `ValServerPlatform.bytesToBase64` exists
|
|
1014
|
+
* for the paths that encode every file of a publish.
|
|
1015
|
+
*/
|
|
1016
|
+
function bytesToBase64(bytes) {
|
|
1017
|
+
const binary = [];
|
|
1018
|
+
for (let i = 0; i < bytes.length; i += CHUNK) {
|
|
1019
|
+
// `Reflect.apply` rather than a spread, which copies the chunk into an
|
|
1020
|
+
// argument list first and is four times slower.
|
|
1021
|
+
const chunk = Reflect.apply(String.fromCharCode, null, bytes.subarray(i, i + CHUNK));
|
|
1022
|
+
binary.push(chunk);
|
|
1023
|
+
}
|
|
1024
|
+
return btoa(binary.join(""));
|
|
1025
|
+
}
|
|
1026
|
+
|
|
1027
|
+
/** `Buffer.from(text, "utf8")`. */
|
|
1028
|
+
function utf8ToBytes(text) {
|
|
1029
|
+
return new TextEncoder().encode(text);
|
|
1030
|
+
}
|
|
1031
|
+
|
|
1032
|
+
/**
|
|
1033
|
+
* `Buffer.from(bytes).toString("utf8")`. `ignoreBOM` because `Buffer` keeps a
|
|
1034
|
+
* leading byte order mark, where a `TextDecoder` drops it by default.
|
|
1035
|
+
*/
|
|
1036
|
+
function bytesToUtf8(bytes) {
|
|
1037
|
+
return new TextDecoder("utf-8", {
|
|
1038
|
+
ignoreBOM: true
|
|
1039
|
+
}).decode(bytes);
|
|
1040
|
+
}
|
|
1041
|
+
|
|
931
1042
|
/**
|
|
932
1043
|
* Does saving this commit upload a file to Val's remote host?
|
|
933
1044
|
*
|
|
@@ -1342,7 +1453,8 @@ function isHlsPlaylistPath(filePath) {
|
|
|
1342
1453
|
* patch's to read. Generic over the patch id so it asks for exactly the one
|
|
1343
1454
|
* method it uses, and a test can hand it a plain object.
|
|
1344
1455
|
*/
|
|
1345
|
-
async function readCommittedBinaryFiles(ops, descriptors
|
|
1456
|
+
async function readCommittedBinaryFiles(ops, descriptors, /** `ValServerPlatform.bytesToBase64`, which is faster in Node. */
|
|
1457
|
+
encode = bytesToBase64) {
|
|
1346
1458
|
const files = {};
|
|
1347
1459
|
const unread = [];
|
|
1348
1460
|
await Promise.all(Object.entries(descriptors).map(async ([filePath, {
|
|
@@ -1354,7 +1466,7 @@ async function readCommittedBinaryFiles(ops, descriptors) {
|
|
|
1354
1466
|
console.error("Could not read a committed file", filePath, error);
|
|
1355
1467
|
return null;
|
|
1356
1468
|
});
|
|
1357
|
-
if (bytes === null) unread.push(filePath);else files[filePath] = bytes
|
|
1469
|
+
if (bytes === null) unread.push(filePath);else files[filePath] = encode(bytes);
|
|
1358
1470
|
}));
|
|
1359
1471
|
return {
|
|
1360
1472
|
files,
|
|
@@ -1521,10 +1633,537 @@ function computePatchesToDrop(prepared) {
|
|
|
1521
1633
|
return dropped;
|
|
1522
1634
|
}
|
|
1523
1635
|
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1636
|
+
function getFileExt(filePath) {
|
|
1637
|
+
// NOTE: We do not import the path module. This code is copied in different projects. We want the same implementation and which means that this might running in browser where path is not available).
|
|
1638
|
+
return filePath.split(".").pop() || "";
|
|
1639
|
+
}
|
|
1640
|
+
|
|
1641
|
+
/**
|
|
1642
|
+
* Does the host a platform ref names still serve the project the ref names?
|
|
1643
|
+
*
|
|
1644
|
+
* A ref served by the platform loader names the project's host at the time
|
|
1645
|
+
* the file was added: `https://www.acme.no/_val/file/p/3f9a2c1/…`. When the
|
|
1646
|
+
* project drops `www.acme.no`, the bytes are still there, at
|
|
1647
|
+
* `3f9a2c1.valstart.dev`, but every ref that names the old domain points at a
|
|
1648
|
+
* host that no longer serves them. This is the question that finds those refs,
|
|
1649
|
+
* and `rewriteRefHost` is what the fix (`remote:rewrite-host`) does about it.
|
|
1650
|
+
* See valbuild/home `docs/remote-files-on-platform.md`, "Rewriting refs after a
|
|
1651
|
+
* domain is removed", and `docs/plans/studio-validate.md`, step 6.
|
|
1652
|
+
*
|
|
1653
|
+
* Asked of the host itself (`GET <host>/_val/project`), not of content: the
|
|
1654
|
+
* host's answer covers DNS, the certificate and the loader's mapping in one,
|
|
1655
|
+
* it needs no credential, and it works for a ref that names a project other
|
|
1656
|
+
* than this one -- a project seeded from a template has refs naming the
|
|
1657
|
+
* template's.
|
|
1658
|
+
*/
|
|
1659
|
+
|
|
1660
|
+
/** The domain every platform project is always served at, by its public id. */
|
|
1661
|
+
const PLATFORM_SITES_DOMAIN = "valstart.dev";
|
|
1662
|
+
|
|
1663
|
+
/** What a platform ref's host part ends with. */
|
|
1664
|
+
const PLATFORM_PREFIX = "/_val";
|
|
1665
|
+
|
|
1666
|
+
/**
|
|
1667
|
+
* The host every platform ref of `publicProjectId` can always be pointed at:
|
|
1668
|
+
* `<public id>.valstart.dev` is permanent (decision 4 of the remote-files
|
|
1669
|
+
* plan), and the loader serves a project's files there whatever else it has.
|
|
1670
|
+
*/
|
|
1671
|
+
function fallbackRemoteHost(publicProjectId) {
|
|
1672
|
+
return `https://${publicProjectId}.${PLATFORM_SITES_DOMAIN}${PLATFORM_PREFIX}`;
|
|
1673
|
+
}
|
|
1674
|
+
|
|
1675
|
+
/**
|
|
1676
|
+
* Only a platform ref is asked about. `remote.val.build` is not a platform
|
|
1677
|
+
* host and is not going away, and a ref already on the project's own
|
|
1678
|
+
* `valstart.dev` address is on the fallback, which always serves it.
|
|
1679
|
+
*/
|
|
1680
|
+
function needsHostCheck(remoteHost, projectId) {
|
|
1681
|
+
return remoteHost.endsWith(PLATFORM_PREFIX) && remoteHost !== fallbackRemoteHost(projectId);
|
|
1682
|
+
}
|
|
1683
|
+
|
|
1684
|
+
/** `ref` with its host part replaced, and everything else as it was. */
|
|
1685
|
+
function rewriteRefHost(ref, remoteHost) {
|
|
1686
|
+
const parsed = core.Internal.remote.splitRemoteRef(ref);
|
|
1687
|
+
if (parsed.status === "error") {
|
|
1688
|
+
return null;
|
|
1689
|
+
}
|
|
1690
|
+
return core.Internal.remote.createRemoteRef(remoteHost, {
|
|
1691
|
+
publicProjectId: parsed.projectId,
|
|
1692
|
+
coreVersion: parsed.version,
|
|
1693
|
+
bucket: parsed.bucket,
|
|
1694
|
+
validationHash: parsed.validationHash,
|
|
1695
|
+
fileHash: parsed.fileHash,
|
|
1696
|
+
filePath: parsed.filePath
|
|
1697
|
+
});
|
|
1698
|
+
}
|
|
1699
|
+
|
|
1700
|
+
/**
|
|
1701
|
+
* - `serves`: the host answered for the project the ref names.
|
|
1702
|
+
* - `other-project`: it answered, for another one -- the domain was moved to
|
|
1703
|
+
* a different project, and decision 5 means it will not serve these files.
|
|
1704
|
+
* - `no-site`: it answered, but serves no project at all (the loader's 404 for
|
|
1705
|
+
* a domain nobody has mapped any more).
|
|
1706
|
+
* - `unreachable`: no answer. A domain whose DNS was removed looks like this,
|
|
1707
|
+
* and so does a host that is down for a minute, which is why the row says
|
|
1708
|
+
* which it saw.
|
|
1709
|
+
*/
|
|
1710
|
+
|
|
1711
|
+
const ASK_TIMEOUT_MS = 8_000;
|
|
1712
|
+
|
|
1713
|
+
/** `GET <host>/_val/project`, answered by the platform loader. */
|
|
1714
|
+
const askHostProject = async remoteHost => {
|
|
1715
|
+
try {
|
|
1716
|
+
const res = await fetch(`${remoteHost}/project`, {
|
|
1717
|
+
signal: AbortSignal.timeout(ASK_TIMEOUT_MS)
|
|
1718
|
+
});
|
|
1719
|
+
if (!res.ok) {
|
|
1720
|
+
return {
|
|
1721
|
+
status: "no-site",
|
|
1722
|
+
httpStatus: res.status
|
|
1723
|
+
};
|
|
1724
|
+
}
|
|
1725
|
+
const body = await res.json();
|
|
1726
|
+
if (typeof body === "object" && body !== null && "publicProjectId" in body && typeof body.publicProjectId === "string") {
|
|
1727
|
+
return {
|
|
1728
|
+
status: "answered",
|
|
1729
|
+
publicProjectId: body.publicProjectId
|
|
1730
|
+
};
|
|
1731
|
+
}
|
|
1732
|
+
return {
|
|
1733
|
+
status: "no-site",
|
|
1734
|
+
httpStatus: res.status
|
|
1735
|
+
};
|
|
1736
|
+
} catch (err) {
|
|
1737
|
+
return {
|
|
1738
|
+
status: "unreachable",
|
|
1739
|
+
reason: err instanceof Error && err.name === "TimeoutError" ? `No answer within ${ASK_TIMEOUT_MS / 1000} s` : err instanceof Error ? err.message : "Unknown error"
|
|
1740
|
+
};
|
|
1741
|
+
}
|
|
1742
|
+
};
|
|
1743
|
+
|
|
1744
|
+
/**
|
|
1745
|
+
* Hosts that answered for a project, and when: one question per host, not per
|
|
1746
|
+
* ref, and not again for a few minutes. Only a "serves" is kept, for the same
|
|
1747
|
+
* reason `remoteFileCheck` keeps only "found": a host that stopped serving has
|
|
1748
|
+
* to be noticed, and one that did not answer has to be asked again.
|
|
1749
|
+
*/
|
|
1750
|
+
const SERVES_FOR_MS = 5 * 60_000;
|
|
1751
|
+
const servesAt = new Map();
|
|
1752
|
+
async function hostServes(remoteHost, projectId, ask = askHostProject) {
|
|
1753
|
+
const key = `${remoteHost} ${projectId}`;
|
|
1754
|
+
const at = servesAt.get(key);
|
|
1755
|
+
if (at !== undefined && Date.now() - at < SERVES_FOR_MS) {
|
|
1756
|
+
return {
|
|
1757
|
+
status: "serves"
|
|
1758
|
+
};
|
|
1759
|
+
}
|
|
1760
|
+
const answer = await ask(remoteHost);
|
|
1761
|
+
if (answer.status !== "answered") {
|
|
1762
|
+
return answer;
|
|
1763
|
+
}
|
|
1764
|
+
if (answer.publicProjectId !== projectId) {
|
|
1765
|
+
return {
|
|
1766
|
+
status: "other-project",
|
|
1767
|
+
publicProjectId: answer.publicProjectId
|
|
1768
|
+
};
|
|
1769
|
+
}
|
|
1770
|
+
servesAt.set(key, Date.now());
|
|
1771
|
+
return {
|
|
1772
|
+
status: "serves"
|
|
1773
|
+
};
|
|
1774
|
+
}
|
|
1775
|
+
|
|
1776
|
+
/**
|
|
1777
|
+
* `ask`, asking each host once for as long as the returned function is kept:
|
|
1778
|
+
* one check, not the life of the server. A check walks its refs in order, so
|
|
1779
|
+
* without it every ref on a host that does not answer waits out its own
|
|
1780
|
+
* timeout -- 150 images on a dead domain is 20 minutes. Kept per check rather
|
|
1781
|
+
* than in {@link hostServes}'s cache because a host that did not answer has
|
|
1782
|
+
* to be asked again by the next one.
|
|
1783
|
+
*/
|
|
1784
|
+
function askOncePerHost(ask = askHostProject) {
|
|
1785
|
+
const answers = new Map();
|
|
1786
|
+
return remoteHost => {
|
|
1787
|
+
let answer = answers.get(remoteHost);
|
|
1788
|
+
if (answer === undefined) {
|
|
1789
|
+
answer = ask(remoteHost);
|
|
1790
|
+
answers.set(remoteHost, answer);
|
|
1791
|
+
}
|
|
1792
|
+
return answer;
|
|
1793
|
+
};
|
|
1794
|
+
}
|
|
1795
|
+
|
|
1796
|
+
/** `www.acme.no`, from `https://www.acme.no/_val`: how a person names a host. */
|
|
1797
|
+
function hostNameOfRemoteHost(remoteHost) {
|
|
1798
|
+
try {
|
|
1799
|
+
return new URL(remoteHost).host;
|
|
1800
|
+
} catch {
|
|
1801
|
+
return remoteHost;
|
|
1802
|
+
}
|
|
1803
|
+
}
|
|
1804
|
+
|
|
1805
|
+
/**
|
|
1806
|
+
* Where a platform ref has to move to, when its host no longer serves the
|
|
1807
|
+
* project it names; `undefined` when it does not have to (or is not a
|
|
1808
|
+
* platform ref). `why` and `to` are for the caller's sentence: what can be
|
|
1809
|
+
* done about it differs by caller (a Studio **Fix**, `val validate --fix`, or
|
|
1810
|
+
* nothing yet).
|
|
1811
|
+
*/
|
|
1812
|
+
async function movedRemoteHost(ref, ask) {
|
|
1813
|
+
const parsed = core.Internal.remote.splitRemoteRef(ref);
|
|
1814
|
+
if (parsed.status === "error" || !needsHostCheck(parsed.remoteHost, parsed.projectId)) {
|
|
1815
|
+
return undefined;
|
|
1816
|
+
}
|
|
1817
|
+
const serves = await hostServes(parsed.remoteHost, parsed.projectId, ask);
|
|
1818
|
+
// Only an answer that says so moves a ref: the host serves another
|
|
1819
|
+
// project, or the loader says it serves none (its 404 for a domain nobody
|
|
1820
|
+
// maps). A host that did not answer, or answered 5xx, may be down for a
|
|
1821
|
+
// minute -- and a moved ref does not move back, so rewriting on that would
|
|
1822
|
+
// take a site's images off its own domain for good over one blip. Those are
|
|
1823
|
+
// left to the existence check, which reports the host once with Try again.
|
|
1824
|
+
if (serves.status === "serves" || serves.status === "unreachable" || serves.status === "no-site" && serves.httpStatus !== 404) {
|
|
1825
|
+
return undefined;
|
|
1826
|
+
}
|
|
1827
|
+
const target = fallbackRemoteHost(parsed.projectId);
|
|
1828
|
+
const moved = rewriteRefHost(ref, target);
|
|
1829
|
+
if (moved === null) {
|
|
1830
|
+
return undefined;
|
|
1831
|
+
}
|
|
1832
|
+
const from = hostNameOfRemoteHost(parsed.remoteHost);
|
|
1833
|
+
return {
|
|
1834
|
+
ref: moved,
|
|
1835
|
+
why: serves.status === "other-project" ? `${from} now serves another project, so this file is not shown from there` : `${from} no longer serves this project, so this file is not shown from there`,
|
|
1836
|
+
to: hostNameOfRemoteHost(target)
|
|
1837
|
+
};
|
|
1838
|
+
}
|
|
1839
|
+
|
|
1840
|
+
/**
|
|
1841
|
+
* Where else a ref's file can be downloaded from: the same file on the
|
|
1842
|
+
* project's own `valstart.dev` address, for a platform ref on any other host.
|
|
1843
|
+
* `null` when there is nowhere else -- a `remote.val.build` ref, or one already
|
|
1844
|
+
* on the fallback. See valbuild/home `docs/remote-files-on-platform.md`,
|
|
1845
|
+
* "Tooling: the fallback".
|
|
1527
1846
|
*/
|
|
1847
|
+
function fallbackRefOf(ref) {
|
|
1848
|
+
const parsed = core.Internal.remote.splitRemoteRef(ref);
|
|
1849
|
+
if (parsed.status === "error" || !needsHostCheck(parsed.remoteHost, parsed.projectId)) {
|
|
1850
|
+
return null;
|
|
1851
|
+
}
|
|
1852
|
+
return rewriteRefHost(ref, fallbackRemoteHost(parsed.projectId));
|
|
1853
|
+
}
|
|
1854
|
+
|
|
1855
|
+
/**
|
|
1856
|
+
* Does the host a remote ref names still serve it?
|
|
1857
|
+
*
|
|
1858
|
+
* `found` and `missing` are the host's answer. `unreachable` is no answer at
|
|
1859
|
+
* all — a timeout, a refused connection, a 5xx — and says nothing about the
|
|
1860
|
+
* file, only about the host.
|
|
1861
|
+
*/
|
|
1862
|
+
|
|
1863
|
+
/**
|
|
1864
|
+
* How the remote check reaches the network: what it asks a host about a file,
|
|
1865
|
+
* and which project a platform host serves. Both default to the real thing;
|
|
1866
|
+
* tests replace them.
|
|
1867
|
+
*/
|
|
1868
|
+
|
|
1869
|
+
const HEAD_TIMEOUT_MS = 8_000;
|
|
1870
|
+
|
|
1871
|
+
/**
|
|
1872
|
+
* A HEAD, falling back to the first byte of a GET for a host that does not
|
|
1873
|
+
* answer HEAD (405, 501): either way no file is downloaded to learn that it is
|
|
1874
|
+
* there.
|
|
1875
|
+
*/
|
|
1876
|
+
const headRemoteFile = async url => {
|
|
1877
|
+
const ask = async init => fetch(url, {
|
|
1878
|
+
...init,
|
|
1879
|
+
redirect: "follow",
|
|
1880
|
+
signal: AbortSignal.timeout(HEAD_TIMEOUT_MS)
|
|
1881
|
+
});
|
|
1882
|
+
try {
|
|
1883
|
+
let res = await ask({
|
|
1884
|
+
method: "HEAD"
|
|
1885
|
+
});
|
|
1886
|
+
if (res.status === 405 || res.status === 501) {
|
|
1887
|
+
var _res$body;
|
|
1888
|
+
res = await ask({
|
|
1889
|
+
method: "GET",
|
|
1890
|
+
headers: {
|
|
1891
|
+
Range: "bytes=0-0"
|
|
1892
|
+
}
|
|
1893
|
+
});
|
|
1894
|
+
await ((_res$body = res.body) === null || _res$body === void 0 ? void 0 : _res$body.cancel());
|
|
1895
|
+
}
|
|
1896
|
+
if (res.ok) {
|
|
1897
|
+
return {
|
|
1898
|
+
status: "found"
|
|
1899
|
+
};
|
|
1900
|
+
}
|
|
1901
|
+
if (res.status === 404 || res.status === 410) {
|
|
1902
|
+
return {
|
|
1903
|
+
status: "missing",
|
|
1904
|
+
httpStatus: res.status
|
|
1905
|
+
};
|
|
1906
|
+
}
|
|
1907
|
+
return {
|
|
1908
|
+
status: "unreachable",
|
|
1909
|
+
reason: `HTTP ${res.status}`
|
|
1910
|
+
};
|
|
1911
|
+
} catch (err) {
|
|
1912
|
+
return {
|
|
1913
|
+
status: "unreachable",
|
|
1914
|
+
reason: describeFetchError(err)
|
|
1915
|
+
};
|
|
1916
|
+
}
|
|
1917
|
+
};
|
|
1918
|
+
function describeFetchError(err) {
|
|
1919
|
+
if (!(err instanceof Error)) {
|
|
1920
|
+
return "Unknown error";
|
|
1921
|
+
}
|
|
1922
|
+
if (err.name === "TimeoutError") {
|
|
1923
|
+
return `No answer within ${HEAD_TIMEOUT_MS / 1000} s`;
|
|
1924
|
+
}
|
|
1925
|
+
// undici puts what actually went wrong (ENOTFOUND, ECONNREFUSED) on `cause`.
|
|
1926
|
+
const cause = err.cause;
|
|
1927
|
+
if (cause instanceof Error) {
|
|
1928
|
+
return `${err.message}: ${cause.message}`;
|
|
1929
|
+
}
|
|
1930
|
+
return err.message;
|
|
1931
|
+
}
|
|
1932
|
+
|
|
1933
|
+
/**
|
|
1934
|
+
* Refs a host has served, and when.
|
|
1935
|
+
*
|
|
1936
|
+
* So that pressing **Check again**, or opening a second module that uses the
|
|
1937
|
+
* same images, does not ask about every file again. Kept for minutes, not for
|
|
1938
|
+
* the life of the process: a ref is content-addressed, so its answer only
|
|
1939
|
+
* changes when its host stops serving it -- which is exactly what this check
|
|
1940
|
+
* is for, and a server that remembered "found" forever would never report a
|
|
1941
|
+
* host that went down after it was first asked. Only `found` is kept. A host
|
|
1942
|
+
* that did not answer is asked again on the next check, which is what makes
|
|
1943
|
+
* **Try again** mean something.
|
|
1944
|
+
*/
|
|
1945
|
+
const FOUND_FOR_MS = 5 * 60_000;
|
|
1946
|
+
const foundAt = new Map();
|
|
1947
|
+
function wasFound(ref) {
|
|
1948
|
+
const at = foundAt.get(ref);
|
|
1949
|
+
return at !== undefined && Date.now() - at < FOUND_FOR_MS;
|
|
1950
|
+
}
|
|
1951
|
+
function markFound(ref) {
|
|
1952
|
+
foundAt.set(ref, Date.now());
|
|
1953
|
+
}
|
|
1954
|
+
|
|
1955
|
+
/** A host that did not serve the refs of a module it was asked about. */
|
|
1956
|
+
|
|
1957
|
+
const textEncoder = new TextEncoder();
|
|
1958
|
+
|
|
1959
|
+
/**
|
|
1960
|
+
* The remote half of the Studio's full check, for one module's remote values.
|
|
1961
|
+
*
|
|
1962
|
+
* Four questions, cheapest first, and a ref stops at the first one it fails:
|
|
1963
|
+
*
|
|
1964
|
+
* 1. **Is it a ref at all?** `splitRemoteRef` parses it.
|
|
1965
|
+
* 2. **Does the ref agree with what is recorded about it?** A ref carries a
|
|
1966
|
+
* hash of the schema's options, the file's extension and its width, height
|
|
1967
|
+
* and mime type. When the value or the schema changed and the ref did not,
|
|
1968
|
+
* the ref is stale, and **Fix** (`image:check-remote`) re-reads the file and
|
|
1969
|
+
* writes the ref it should have. Computed with the ref's OWN core version:
|
|
1970
|
+
* the CLI also re-derives every ref after a Val upgrade, and listing every
|
|
1971
|
+
* remote image in the project after each upgrade would bury the rows that
|
|
1972
|
+
* matter. Nothing here downloads.
|
|
1973
|
+
* 3. **Does its host still serve its project?** Only for a platform ref
|
|
1974
|
+
* (`…/_val/file/…`) that is not on the project's own `valstart.dev`
|
|
1975
|
+
* address: one `GET <host>/_val/project` per host. A host that dropped the
|
|
1976
|
+
* project gets a **Fix** (`remote:rewrite-host`) that points the ref at
|
|
1977
|
+
* `<public id>.valstart.dev`. See `remoteHostCheck.ts`.
|
|
1978
|
+
* 4. **Does the host have it?** A HEAD.
|
|
1979
|
+
*
|
|
1980
|
+
* Neither host question is asked of a ref whose bytes are in a pending patch,
|
|
1981
|
+
* which the host will not have until publish: there the question is whether
|
|
1982
|
+
* the patch still has them.
|
|
1983
|
+
*/
|
|
1984
|
+
async function checkRemoteFiles(input) {
|
|
1985
|
+
const head = input.head ?? headRemoteFile;
|
|
1986
|
+
const askHostProject = askOncePerHost(input.askHostProject);
|
|
1987
|
+
const errors = {};
|
|
1988
|
+
const addError = (sourcePath, error) => {
|
|
1989
|
+
errors[sourcePath] = (errors[sourcePath] ?? []).concat(error);
|
|
1990
|
+
};
|
|
1991
|
+
|
|
1992
|
+
/** Refs still to ask a host about, by host. */
|
|
1993
|
+
const toAsk = new Map();
|
|
1994
|
+
const sorted = Object.entries(input.remoteFiles).sort(([a], [b]) => a.localeCompare(b));
|
|
1995
|
+
for (const [sourcePathS, value] of sorted) {
|
|
1996
|
+
const sourcePath = sourcePathS;
|
|
1997
|
+
const ref = value.path;
|
|
1998
|
+
const parsed = core.Internal.remote.splitRemoteRef(ref);
|
|
1999
|
+
if (parsed.status === "error") {
|
|
2000
|
+
addError(sourcePath, {
|
|
2001
|
+
message: `This is not an address Val can check: ${parsed.error}`,
|
|
2002
|
+
value
|
|
2003
|
+
});
|
|
2004
|
+
continue;
|
|
2005
|
+
}
|
|
2006
|
+
let noun = "file";
|
|
2007
|
+
if (!input.existenceOnly) {
|
|
2008
|
+
const schema = input.schemaAt(sourcePath);
|
|
2009
|
+
if ((schema === null || schema === void 0 ? void 0 : schema.type) !== "image" && (schema === null || schema === void 0 ? void 0 : schema.type) !== "file") {
|
|
2010
|
+
continue;
|
|
2011
|
+
}
|
|
2012
|
+
noun = schema.type;
|
|
2013
|
+
if (isStale(parsed, schema, value)) {
|
|
2014
|
+
addError(sourcePath, {
|
|
2015
|
+
message: `The address of this ${schema.type} does not match what is recorded about it, so it may be served with the wrong size or type.`,
|
|
2016
|
+
value,
|
|
2017
|
+
fixes: [schema.type === "image" ? "image:check-remote" : "file:check-remote"]
|
|
2018
|
+
});
|
|
2019
|
+
continue;
|
|
2020
|
+
}
|
|
2021
|
+
}
|
|
2022
|
+
const pending = await input.pendingBytes(ref);
|
|
2023
|
+
if (pending === "present") {
|
|
2024
|
+
continue;
|
|
2025
|
+
}
|
|
2026
|
+
if (pending === "missing") {
|
|
2027
|
+
addError(sourcePath, {
|
|
2028
|
+
message: `This ${noun} is in your unpublished changes, but its file is not. Choose the ${noun} again.`,
|
|
2029
|
+
value
|
|
2030
|
+
});
|
|
2031
|
+
continue;
|
|
2032
|
+
}
|
|
2033
|
+
// Asked after the pending check: a ref whose bytes are still in a patch
|
|
2034
|
+
// is the key those bytes are stored under, and repointing it would cut
|
|
2035
|
+
// it off from them.
|
|
2036
|
+
const moved = await movedRemoteHost(ref, askHostProject);
|
|
2037
|
+
if (moved) {
|
|
2038
|
+
addError(sourcePath, input.existenceOnly ? {
|
|
2039
|
+
// A directory's key: the fields picking from it hold the same
|
|
2040
|
+
// ref, and would have to follow. Not built yet.
|
|
2041
|
+
message: `${moved.why}. Pointing a file in a media directory at ${moved.to} is not something the Studio can do yet.`,
|
|
2042
|
+
value
|
|
2043
|
+
} : {
|
|
2044
|
+
message: `${moved.why}. Fix points it at ${moved.to}, which always serves it.`,
|
|
2045
|
+
value,
|
|
2046
|
+
fixes: ["remote:rewrite-host"]
|
|
2047
|
+
});
|
|
2048
|
+
continue;
|
|
2049
|
+
}
|
|
2050
|
+
if (wasFound(ref)) {
|
|
2051
|
+
continue;
|
|
2052
|
+
}
|
|
2053
|
+
const refs = toAsk.get(parsed.remoteHost) ?? [];
|
|
2054
|
+
refs.push({
|
|
2055
|
+
sourcePath,
|
|
2056
|
+
ref,
|
|
2057
|
+
published: input.isPublished(sourcePath, ref)
|
|
2058
|
+
});
|
|
2059
|
+
toAsk.set(parsed.remoteHost, refs);
|
|
2060
|
+
}
|
|
2061
|
+
const hosts = [];
|
|
2062
|
+
await Promise.all([...toAsk.entries()].map(async ([host, refs]) => {
|
|
2063
|
+
const problem = await askHost(host, refs, input.sampleByHost, head);
|
|
2064
|
+
if (problem.host) {
|
|
2065
|
+
hosts.push(problem.host);
|
|
2066
|
+
}
|
|
2067
|
+
for (const {
|
|
2068
|
+
sourcePath,
|
|
2069
|
+
error
|
|
2070
|
+
} of problem.refErrors) {
|
|
2071
|
+
addError(sourcePath, error);
|
|
2072
|
+
}
|
|
2073
|
+
}));
|
|
2074
|
+
hosts.sort((a, b) => a.host.localeCompare(b.host));
|
|
2075
|
+
return {
|
|
2076
|
+
errors,
|
|
2077
|
+
hosts
|
|
2078
|
+
};
|
|
2079
|
+
}
|
|
2080
|
+
|
|
2081
|
+
/**
|
|
2082
|
+
* One host's refs, asked one at a time.
|
|
2083
|
+
*
|
|
2084
|
+
* In order rather than all at once: a host that does not answer is reported
|
|
2085
|
+
* after the FIRST timeout, not after one timeout per image. A host that
|
|
2086
|
+
* answers "missing" for one ref is still asked about the next, since a file
|
|
2087
|
+
* that is gone is a fact about that file.
|
|
2088
|
+
*/
|
|
2089
|
+
async function askHost(host, refs, sampleByHost, head) {
|
|
2090
|
+
const vouchable = sampleByHost ? refs.filter(r => r.published) : [];
|
|
2091
|
+
const individually = sampleByHost ? refs.filter(r => !r.published) : refs;
|
|
2092
|
+
const allPaths = refs.map(r => r.sourcePath);
|
|
2093
|
+
const refErrors = [];
|
|
2094
|
+
if (vouchable.length > 0) {
|
|
2095
|
+
const sample = vouchable[0];
|
|
2096
|
+
const answer = await head(sample.ref);
|
|
2097
|
+
if (answer.status === "found") {
|
|
2098
|
+
for (const r of vouchable) {
|
|
2099
|
+
markFound(r.ref);
|
|
2100
|
+
}
|
|
2101
|
+
} else {
|
|
2102
|
+
// In a managed project a published ref cannot break on its own, so the
|
|
2103
|
+
// sample's answer is the host's: one row, every ref on it.
|
|
2104
|
+
return {
|
|
2105
|
+
host: {
|
|
2106
|
+
host,
|
|
2107
|
+
status: answer.status,
|
|
2108
|
+
details: detailsOf(sample.ref, answer),
|
|
2109
|
+
sourcePaths: allPaths
|
|
2110
|
+
},
|
|
2111
|
+
refErrors
|
|
2112
|
+
};
|
|
2113
|
+
}
|
|
2114
|
+
}
|
|
2115
|
+
for (const r of individually) {
|
|
2116
|
+
const answer = await head(r.ref);
|
|
2117
|
+
if (answer.status === "found") {
|
|
2118
|
+
markFound(r.ref);
|
|
2119
|
+
} else if (answer.status === "missing") {
|
|
2120
|
+
refErrors.push({
|
|
2121
|
+
sourcePath: r.sourcePath,
|
|
2122
|
+
error: {
|
|
2123
|
+
message: `The file is not at its address: ${hostName(host)} answered ${answer.httpStatus}.`
|
|
2124
|
+
}
|
|
2125
|
+
});
|
|
2126
|
+
} else {
|
|
2127
|
+
return {
|
|
2128
|
+
host: {
|
|
2129
|
+
host,
|
|
2130
|
+
status: "unreachable",
|
|
2131
|
+
details: detailsOf(r.ref, answer),
|
|
2132
|
+
sourcePaths: allPaths
|
|
2133
|
+
},
|
|
2134
|
+
refErrors
|
|
2135
|
+
};
|
|
2136
|
+
}
|
|
2137
|
+
}
|
|
2138
|
+
return {
|
|
2139
|
+
refErrors
|
|
2140
|
+
};
|
|
2141
|
+
}
|
|
2142
|
+
function detailsOf(ref, answer) {
|
|
2143
|
+
return `HEAD ${ref}\n→ ${answer.status === "missing" ? `${answer.httpStatus}` : answer.status === "unreachable" ? answer.reason : "200"}`;
|
|
2144
|
+
}
|
|
2145
|
+
|
|
2146
|
+
/** `www.acme.no`, from `https://www.acme.no`: how a person names a host. */
|
|
2147
|
+
function hostName(host) {
|
|
2148
|
+
try {
|
|
2149
|
+
return new URL(host).host;
|
|
2150
|
+
} catch {
|
|
2151
|
+
return host;
|
|
2152
|
+
}
|
|
2153
|
+
}
|
|
2154
|
+
function isStale(parsed, schema, value) {
|
|
2155
|
+
// The hash covers only these of what was recorded (`getValidationBasis`):
|
|
2156
|
+
// `alt` and `hotspot` are the editor's, and change no byte.
|
|
2157
|
+
const metadata = {
|
|
2158
|
+
width: value.width,
|
|
2159
|
+
height: value.height,
|
|
2160
|
+
mimeType: value.mimeType
|
|
2161
|
+
};
|
|
2162
|
+
const expected = core.Internal.remote.getValidationHash(parsed.version, schema, getFileExt(parsed.filePath), metadata, parsed.fileHash, textEncoder);
|
|
2163
|
+
return expected !== parsed.validationHash;
|
|
2164
|
+
}
|
|
2165
|
+
|
|
2166
|
+
/* eslint-disable @typescript-eslint/no-unused-vars */
|
|
1528
2167
|
const jsonOps = new patch.JSONOps();
|
|
1529
2168
|
/**
|
|
1530
2169
|
* How many `.jsonValues()` entry files a whole-record write reads at a time.
|
|
@@ -2565,7 +3204,14 @@ class ValOps {
|
|
|
2565
3204
|
}
|
|
2566
3205
|
|
|
2567
3206
|
// #region validateSources
|
|
2568
|
-
async validateSources(schemas, sources, patchesByModule
|
|
3207
|
+
async validateSources(schemas, sources, patchesByModule,
|
|
3208
|
+
/**
|
|
3209
|
+
* Validate only these modules. The snapshot the cross-module checks read
|
|
3210
|
+
* (a keyOf's keys, a route, a set's directory) is still built from every
|
|
3211
|
+
* module, which is what makes checking one module at a time give the same
|
|
3212
|
+
* answer for it as checking them all. Wins over `patchesByModule`.
|
|
3213
|
+
*/
|
|
3214
|
+
onlyModules) {
|
|
2569
3215
|
const errors = {};
|
|
2570
3216
|
const files = {};
|
|
2571
3217
|
const remoteFiles = {};
|
|
@@ -2598,7 +3244,18 @@ class ValOps {
|
|
|
2598
3244
|
}
|
|
2599
3245
|
}
|
|
2600
3246
|
}
|
|
2601
|
-
const
|
|
3247
|
+
const addModuleError = (path, sourcePath, validationError) => {
|
|
3248
|
+
if (!errors[path]) {
|
|
3249
|
+
errors[path] = {
|
|
3250
|
+
validations: {}
|
|
3251
|
+
};
|
|
3252
|
+
}
|
|
3253
|
+
if (!errors[path].validations[sourcePath]) {
|
|
3254
|
+
errors[path].validations[sourcePath] = [];
|
|
3255
|
+
}
|
|
3256
|
+
errors[path].validations[sourcePath].push(validationError);
|
|
3257
|
+
};
|
|
3258
|
+
const modulePathsToValidate = onlyModules ?? (patchesByModule && Object.keys(patchesByModule));
|
|
2602
3259
|
for (const [pathS, schema] of entries) {
|
|
2603
3260
|
if (modulePathsToValidate && !modulePathsToValidate.includes(pathS)) {
|
|
2604
3261
|
continue;
|
|
@@ -2627,15 +3284,35 @@ class ValOps {
|
|
|
2627
3284
|
} = await validateJsonValuesEntries(schema, source, path);
|
|
2628
3285
|
for (const [sourcePathS, entryErrors] of Object.entries(jsonValuesErrors)) {
|
|
2629
3286
|
const sourcePath = sourcePathS;
|
|
2630
|
-
|
|
2631
|
-
errors
|
|
2632
|
-
|
|
2633
|
-
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
3287
|
+
for (const entryError of entryErrors) {
|
|
3288
|
+
// An entry's errors get the same project-wide resolution as every
|
|
3289
|
+
// other module's. Added raw, every keyOf and route in a
|
|
3290
|
+
// `.jsonValues()` record was reported as unresolved -- 240 of them in
|
|
3291
|
+
// `examples/next`'s kb.val.ts, none real.
|
|
3292
|
+
if (internal.isSchemaSourceFixError(entryError)) {
|
|
3293
|
+
const resolved = internal.resolveSchemaSourceFixForError(entryError, snapshot);
|
|
3294
|
+
if (!resolved || resolved.status !== "remaining") {
|
|
3295
|
+
continue;
|
|
3296
|
+
}
|
|
3297
|
+
addModuleError(path, sourcePath, resolved.error);
|
|
3298
|
+
continue;
|
|
3299
|
+
}
|
|
3300
|
+
// "Not yet compared with its bytes" is a request, not an error, and
|
|
3301
|
+
// for an entry it cannot be answered here: `validateFiles` resolves a
|
|
3302
|
+
// file's schema through the module's source, which for a
|
|
3303
|
+
// `.jsonValues()` record holds markers rather than the entry. The
|
|
3304
|
+
// publish gate ignores it anyway (`partitionValidationErrors`).
|
|
3305
|
+
//
|
|
3306
|
+
// A remote file's "not yet checked against its host" is the same
|
|
3307
|
+
// request, for the same reason: `validateRemoteFiles` and its fix
|
|
3308
|
+
// resolve the path through the module's source too. Added as an
|
|
3309
|
+
// error, every remote image in an entry was a row on the Validate
|
|
3310
|
+
// page whose Fix could not find it.
|
|
3311
|
+
if (isOnlyFileCheckValidationError(entryError) || isOnlyRemoteCheckValidationError(entryError)) {
|
|
3312
|
+
continue;
|
|
3313
|
+
}
|
|
3314
|
+
addModuleError(path, sourcePath, entryError);
|
|
2637
3315
|
}
|
|
2638
|
-
errors[path].validations[sourcePath].push(...entryErrors);
|
|
2639
3316
|
}
|
|
2640
3317
|
if (res === false) {
|
|
2641
3318
|
continue;
|
|
@@ -2713,9 +3390,64 @@ class ValOps {
|
|
|
2713
3390
|
remoteFiles
|
|
2714
3391
|
};
|
|
2715
3392
|
}
|
|
2716
|
-
|
|
2717
|
-
|
|
2718
|
-
|
|
3393
|
+
|
|
3394
|
+
/**
|
|
3395
|
+
* The remote values `validateSources` set aside, checked: is each ref
|
|
3396
|
+
* sound, and does its host serve it. See `checkRemoteFiles`.
|
|
3397
|
+
*
|
|
3398
|
+
* `fileLastUpdatedByPatchId` names the refs whose bytes are still in a
|
|
3399
|
+
* pending patch, which the host will not have until publish.
|
|
3400
|
+
*/
|
|
3401
|
+
async validateRemoteFiles(schemas, sources, remoteFiles, fileLastUpdatedByPatchId = {}, network = {}) {
|
|
3402
|
+
const sampleByHost = this.sourceMode() === "managed";
|
|
3403
|
+
// Only a managed project vouches for one published ref with another, so
|
|
3404
|
+
// only it needs to know which refs are published.
|
|
3405
|
+
const baseSources = sampleByHost ? (await this.getSources()).sources : undefined;
|
|
3406
|
+
const valueAt = (sourcePath, from) => {
|
|
3407
|
+
const [moduleFilePath, modulePath] = core.Internal.splitModuleFilePathAndModulePath(sourcePath);
|
|
3408
|
+
const schema = schemas[moduleFilePath];
|
|
3409
|
+
if (!schema || from[moduleFilePath] === undefined) {
|
|
3410
|
+
return undefined;
|
|
3411
|
+
}
|
|
3412
|
+
try {
|
|
3413
|
+
return core.Internal.resolvePath(modulePath, from[moduleFilePath], schema["executeSerialize"]());
|
|
3414
|
+
} catch {
|
|
3415
|
+
return undefined;
|
|
3416
|
+
}
|
|
3417
|
+
};
|
|
3418
|
+
return checkRemoteFiles({
|
|
3419
|
+
remoteFiles,
|
|
3420
|
+
sampleByHost,
|
|
3421
|
+
...network,
|
|
3422
|
+
schemaAt: sourcePath => {
|
|
3423
|
+
var _valueAt;
|
|
3424
|
+
return (_valueAt = valueAt(sourcePath, sources)) === null || _valueAt === void 0 ? void 0 : _valueAt.schema;
|
|
3425
|
+
},
|
|
3426
|
+
isPublished: (sourcePath, ref) => {
|
|
3427
|
+
var _valueAt2;
|
|
3428
|
+
if (!baseSources) {
|
|
3429
|
+
return false;
|
|
3430
|
+
}
|
|
3431
|
+
const published = (_valueAt2 = valueAt(sourcePath, baseSources)) === null || _valueAt2 === void 0 ? void 0 : _valueAt2.source;
|
|
3432
|
+
return typeof published === "object" && published !== null && "path" in published && published.path === ref;
|
|
3433
|
+
},
|
|
3434
|
+
pendingBytes: this.pendingBytesOf(fileLastUpdatedByPatchId)
|
|
3435
|
+
});
|
|
3436
|
+
}
|
|
3437
|
+
|
|
3438
|
+
/**
|
|
3439
|
+
* Is a remote file's bytes in a pending patch? `not-pending` when no patch
|
|
3440
|
+
* wrote it; `missing` when one did and its bytes are not there.
|
|
3441
|
+
*/
|
|
3442
|
+
pendingBytesOf(fileLastUpdatedByPatchId) {
|
|
3443
|
+
return async ref => {
|
|
3444
|
+
const fromPatch = fileLastUpdatedByPatchId[ref];
|
|
3445
|
+
if (!fromPatch) {
|
|
3446
|
+
return "not-pending";
|
|
3447
|
+
}
|
|
3448
|
+
const bytes = await this.getBase64EncodedBinaryFileFromPatch(ref, fromPatch.patchId, fromPatch.remote);
|
|
3449
|
+
return bytes === null ? "missing" : "present";
|
|
3450
|
+
};
|
|
2719
3451
|
}
|
|
2720
3452
|
|
|
2721
3453
|
// #region validateFiles
|
|
@@ -2787,7 +3519,9 @@ class ValOps {
|
|
|
2787
3519
|
if (metadataErrors && metadataErrors.length > 0) {
|
|
2788
3520
|
return {
|
|
2789
3521
|
[sourcePath]: metadataErrors.map(e => ({
|
|
2790
|
-
|
|
3522
|
+
// Which file: "File not found" on its own names nothing, and the
|
|
3523
|
+
// row it is shown on is the field, not the file.
|
|
3524
|
+
message: `${e.message}: ${filePath}`,
|
|
2791
3525
|
value: {
|
|
2792
3526
|
filePath,
|
|
2793
3527
|
patchId: (fileData === null || fileData === void 0 ? void 0 : fileData.patchId) ?? null
|
|
@@ -3836,7 +4570,7 @@ class ValOps {
|
|
|
3836
4570
|
* override it.
|
|
3837
4571
|
*/
|
|
3838
4572
|
async saveBinaryFileFromPatch(filePath, parentRef, patchId, bytes, mimeType, type, metadata) {
|
|
3839
|
-
return this.saveBase64EncodedBinaryFileFromPatch(filePath, parentRef, patchId, `data:${mimeType};base64,${bytes
|
|
4573
|
+
return this.saveBase64EncodedBinaryFileFromPatch(filePath, parentRef, patchId, `data:${mimeType};base64,${bytesToBase64(bytes)}`, type, metadata);
|
|
3840
4574
|
}
|
|
3841
4575
|
/**
|
|
3842
4576
|
* A file's bytes as something a `Range` request can be answered from
|
|
@@ -3900,6 +4634,10 @@ class ValOps {
|
|
|
3900
4634
|
|
|
3901
4635
|
// #endregion history
|
|
3902
4636
|
}
|
|
4637
|
+
function isOnlyRemoteCheckValidationError(validationError) {
|
|
4638
|
+
const fixes = validationError.fixes ?? [];
|
|
4639
|
+
return fixes.length > 0 && fixes.every(f => f === "image:check-remote" || f === "file:check-remote");
|
|
4640
|
+
}
|
|
3903
4641
|
function isOnlyFileCheckValidationError(validationError) {
|
|
3904
4642
|
var _validationError$fixe9;
|
|
3905
4643
|
if ((_validationError$fixe9 = validationError.fixes) !== null && _validationError$fixe9 !== void 0 && _validationError$fixe9.every(f => f === "file:check-metadata" || f === "image:check-metadata")) {
|
|
@@ -4058,15 +4796,16 @@ function createMetadataFromBuffer(type, mimeType, buffer) {
|
|
|
4058
4796
|
metadata
|
|
4059
4797
|
};
|
|
4060
4798
|
}
|
|
4061
|
-
function bufferFromDataUrl(dataUrl
|
|
4799
|
+
function bufferFromDataUrl(dataUrl, /** Node passes `Buffer`'s, which is faster: this is a whole upload. */
|
|
4800
|
+
decodeBase64 = base64ToBytes) {
|
|
4062
4801
|
let base64Data;
|
|
4063
4802
|
const base64Index = dataUrl.indexOf(";base64,");
|
|
4064
4803
|
if (base64Index > -1) {
|
|
4065
4804
|
base64Data = dataUrl.slice(base64Index + ";base64,".length);
|
|
4066
4805
|
}
|
|
4067
4806
|
if (base64Data) {
|
|
4068
|
-
|
|
4069
|
-
);
|
|
4807
|
+
// TODO: why does it not work with base64url?
|
|
4808
|
+
return decodeBase64(base64Data);
|
|
4070
4809
|
}
|
|
4071
4810
|
}
|
|
4072
4811
|
|
|
@@ -4217,7 +4956,7 @@ async function resolveJsonEntriesAtCommit(ops, commitSha, affectedFiles) {
|
|
|
4217
4956
|
continue;
|
|
4218
4957
|
}
|
|
4219
4958
|
try {
|
|
4220
|
-
entries[gitPath] = JSON.parse(res.value
|
|
4959
|
+
entries[gitPath] = JSON.parse(bytesToUtf8(res.value));
|
|
4221
4960
|
} catch (err) {
|
|
4222
4961
|
failures.push({
|
|
4223
4962
|
kind: "file-unavailable",
|
|
@@ -4539,7 +5278,14 @@ const SettingsSchema = z.z.object({
|
|
|
4539
5278
|
publicProjectId: z.z.string(),
|
|
4540
5279
|
remoteFileBuckets: z.z.array(z.z.object({
|
|
4541
5280
|
bucket: z.z.string()
|
|
4542
|
-
}))
|
|
5281
|
+
})),
|
|
5282
|
+
/**
|
|
5283
|
+
* The host a new remote file's ref names: the project's own address with
|
|
5284
|
+
* `/_val` for a Hosted project whose files are in R2, `remote.val.build`
|
|
5285
|
+
* otherwise. Absent from a content service that predates it, where every
|
|
5286
|
+
* project's files are on `remote.val.build`.
|
|
5287
|
+
*/
|
|
5288
|
+
remoteHost: z.z.string().optional()
|
|
4543
5289
|
});
|
|
4544
5290
|
async function getSettings(projectName, auth, /** Overrides {@link defaultHost}. See the note there for why this exists. */
|
|
4545
5291
|
contentUrl) {
|
|
@@ -4715,11 +5461,6 @@ function mapStatus(status) {
|
|
|
4715
5461
|
return 500;
|
|
4716
5462
|
}
|
|
4717
5463
|
|
|
4718
|
-
/*
|
|
4719
|
-
* Imported rather than global: under Node this is the built-in, and in a
|
|
4720
|
-
* browser (the demo, `@valbuild/server/browser`) the `buffer` package.
|
|
4721
|
-
*/
|
|
4722
|
-
|
|
4723
5464
|
/** What {@link ValServerOptions.publishOverride} is given. */
|
|
4724
5465
|
|
|
4725
5466
|
/** What a publish reports back, in either shape. */
|
|
@@ -4730,6 +5471,18 @@ function mapStatus(status) {
|
|
|
4730
5471
|
* have their own `dir` for any collection that wants somewhere else.
|
|
4731
5472
|
*/
|
|
4732
5473
|
const FILES_DIRECTORY = "/public/val";
|
|
5474
|
+
|
|
5475
|
+
/**
|
|
5476
|
+
* `/validate` and `/validate/fix` on a platform without the full check (a
|
|
5477
|
+
* server in a browser tab). 501: nothing the caller does makes it available,
|
|
5478
|
+
* so the Studio offers no Try again.
|
|
5479
|
+
*/
|
|
5480
|
+
const studioValidationUnavailable = {
|
|
5481
|
+
status: 501,
|
|
5482
|
+
json: {
|
|
5483
|
+
message: "Checking everything needs the project's files and Val's file hosts, and this Studio runs without them."
|
|
5484
|
+
}
|
|
5485
|
+
};
|
|
4733
5486
|
const ValServer = (valModules, options, callbacks, platform) => {
|
|
4734
5487
|
var _options$git;
|
|
4735
5488
|
const {
|
|
@@ -5133,6 +5886,82 @@ const ValServer = (valModules, options, callbacks, platform) => {
|
|
|
5133
5886
|
};
|
|
5134
5887
|
}
|
|
5135
5888
|
};
|
|
5889
|
+
|
|
5890
|
+
/**
|
|
5891
|
+
* What the Studio's full check reads: base plus the patches the caller is
|
|
5892
|
+
* showing, as `/sources/~` resolves an explicit `patch_id` list -- including
|
|
5893
|
+
* `draftOverlay`, so a module changed by a publish after this build is
|
|
5894
|
+
* checked as the caller sees it, not as this build has it.
|
|
5895
|
+
*/
|
|
5896
|
+
const studioCheckInput = async (cookies, moduleFilePath, patchIds) => {
|
|
5897
|
+
const auth = getAuth(cookies);
|
|
5898
|
+
if (auth.error) {
|
|
5899
|
+
return {
|
|
5900
|
+
status: 401,
|
|
5901
|
+
json: {
|
|
5902
|
+
message: auth.error
|
|
5903
|
+
}
|
|
5904
|
+
};
|
|
5905
|
+
}
|
|
5906
|
+
if (isHttpOps(serverOps) && !("id" in auth)) {
|
|
5907
|
+
return {
|
|
5908
|
+
status: 401,
|
|
5909
|
+
json: {
|
|
5910
|
+
message: "Unauthorized"
|
|
5911
|
+
}
|
|
5912
|
+
};
|
|
5913
|
+
}
|
|
5914
|
+
const moduleErrors = await serverOps.getModuleErrors();
|
|
5915
|
+
if ((moduleErrors === null || moduleErrors === void 0 ? void 0 : moduleErrors.length) > 0) {
|
|
5916
|
+
return {
|
|
5917
|
+
status: 500,
|
|
5918
|
+
json: {
|
|
5919
|
+
message: "Val could not load this project's modules, so nothing can be checked. Check val.modules."
|
|
5920
|
+
}
|
|
5921
|
+
};
|
|
5922
|
+
}
|
|
5923
|
+
// An EXPLICITLY empty list is base, never "everything": `fetchPatches`
|
|
5924
|
+
// reads an empty filter as no filter (see `/sources/~`).
|
|
5925
|
+
let patchOps = patchIds !== undefined && patchIds.length === 0 ? {
|
|
5926
|
+
patches: []
|
|
5927
|
+
} : await serverOps.fetchPatches({
|
|
5928
|
+
patchIds,
|
|
5929
|
+
excludePatchOps: false
|
|
5930
|
+
});
|
|
5931
|
+
if ("error" in patchOps && patchOps.error) {
|
|
5932
|
+
return "unauthorized" in patchOps && patchOps.unauthorized ? {
|
|
5933
|
+
status: 401,
|
|
5934
|
+
json: {
|
|
5935
|
+
message: "Unauthorized"
|
|
5936
|
+
}
|
|
5937
|
+
} : {
|
|
5938
|
+
status: 500,
|
|
5939
|
+
json: {
|
|
5940
|
+
message: "The unpublished changes could not be read, so nothing was checked."
|
|
5941
|
+
}
|
|
5942
|
+
};
|
|
5943
|
+
}
|
|
5944
|
+
patchOps = {
|
|
5945
|
+
...patchOps,
|
|
5946
|
+
patches: draftOverlay(patchOps.patches, patchOps.commits)
|
|
5947
|
+
};
|
|
5948
|
+
const analysis = serverOps.analyzePatches(patchOps.patches);
|
|
5949
|
+
const {
|
|
5950
|
+
sources
|
|
5951
|
+
} = await serverOps.getSourcesWithPatchesApplied({
|
|
5952
|
+
...analysis,
|
|
5953
|
+
...patchOps
|
|
5954
|
+
});
|
|
5955
|
+
return {
|
|
5956
|
+
status: "ok",
|
|
5957
|
+
input: {
|
|
5958
|
+
moduleFilePath,
|
|
5959
|
+
schemas: await serverOps.getSchemas(),
|
|
5960
|
+
sources,
|
|
5961
|
+
fileLastUpdatedByPatchId: analysis.fileLastUpdatedByPatchId
|
|
5962
|
+
}
|
|
5963
|
+
};
|
|
5964
|
+
};
|
|
5136
5965
|
return {
|
|
5137
5966
|
"/draft/enable": {
|
|
5138
5967
|
GET: async req => {
|
|
@@ -5483,7 +6312,10 @@ const ValServer = (valModules, options, callbacks, platform) => {
|
|
|
5483
6312
|
json: {
|
|
5484
6313
|
publicProjectId: settingsRes.data.publicProjectId,
|
|
5485
6314
|
remoteFileBuckets: settingsRes.data.remoteFileBuckets,
|
|
5486
|
-
coreVersion: core.Internal.VERSION.core || "unknown"
|
|
6315
|
+
coreVersion: core.Internal.VERSION.core || "unknown",
|
|
6316
|
+
...(settingsRes.data.remoteHost !== undefined ? {
|
|
6317
|
+
remoteHost: settingsRes.data.remoteHost
|
|
6318
|
+
} : {})
|
|
5487
6319
|
}
|
|
5488
6320
|
};
|
|
5489
6321
|
}
|
|
@@ -5764,6 +6596,92 @@ const ValServer = (valModules, options, callbacks, platform) => {
|
|
|
5764
6596
|
};
|
|
5765
6597
|
}
|
|
5766
6598
|
},
|
|
6599
|
+
"/validate": {
|
|
6600
|
+
POST: async req => {
|
|
6601
|
+
const checked = await studioCheckInput(req.cookies, req.body.moduleFilePath, req.body.patchIds);
|
|
6602
|
+
if (checked.status !== "ok") {
|
|
6603
|
+
return checked;
|
|
6604
|
+
}
|
|
6605
|
+
if (!platform.studioValidation) {
|
|
6606
|
+
return studioValidationUnavailable;
|
|
6607
|
+
}
|
|
6608
|
+
const res = await platform.studioValidation.checkModuleForStudio(serverOps, checked.input);
|
|
6609
|
+
switch (res.status) {
|
|
6610
|
+
case "unknown-module":
|
|
6611
|
+
return {
|
|
6612
|
+
status: 404,
|
|
6613
|
+
json: {
|
|
6614
|
+
message: `There is no module '${req.body.moduleFilePath}' in this project.`
|
|
6615
|
+
}
|
|
6616
|
+
};
|
|
6617
|
+
case "invalid-source":
|
|
6618
|
+
return {
|
|
6619
|
+
status: 200,
|
|
6620
|
+
json: {
|
|
6621
|
+
moduleFilePath: req.body.moduleFilePath,
|
|
6622
|
+
errors: {},
|
|
6623
|
+
invalidSource: res.message
|
|
6624
|
+
}
|
|
6625
|
+
};
|
|
6626
|
+
case "ok":
|
|
6627
|
+
return {
|
|
6628
|
+
status: 200,
|
|
6629
|
+
json: {
|
|
6630
|
+
moduleFilePath: req.body.moduleFilePath,
|
|
6631
|
+
errors: res.errors,
|
|
6632
|
+
remoteHosts: res.remoteHosts
|
|
6633
|
+
}
|
|
6634
|
+
};
|
|
6635
|
+
}
|
|
6636
|
+
}
|
|
6637
|
+
},
|
|
6638
|
+
"/validate/fix": {
|
|
6639
|
+
POST: async req => {
|
|
6640
|
+
const checked = await studioCheckInput(req.cookies, req.body.moduleFilePath, req.body.patchIds);
|
|
6641
|
+
if (checked.status !== "ok") {
|
|
6642
|
+
return checked;
|
|
6643
|
+
}
|
|
6644
|
+
if (!platform.studioValidation) {
|
|
6645
|
+
return studioValidationUnavailable;
|
|
6646
|
+
}
|
|
6647
|
+
const res = await platform.studioValidation.fixForStudio(serverOps, {
|
|
6648
|
+
...checked.input,
|
|
6649
|
+
sourcePath: req.body.sourcePath,
|
|
6650
|
+
fix: req.body.fix
|
|
6651
|
+
});
|
|
6652
|
+
switch (res.status) {
|
|
6653
|
+
case "unknown-module":
|
|
6654
|
+
return {
|
|
6655
|
+
status: 404,
|
|
6656
|
+
json: {
|
|
6657
|
+
message: `There is no module '${req.body.moduleFilePath}' in this project.`
|
|
6658
|
+
}
|
|
6659
|
+
};
|
|
6660
|
+
case "not-found":
|
|
6661
|
+
return {
|
|
6662
|
+
status: 404,
|
|
6663
|
+
json: {
|
|
6664
|
+
message: "This error is no longer there. Check the module again to see what is left."
|
|
6665
|
+
}
|
|
6666
|
+
};
|
|
6667
|
+
case "not-in-studio":
|
|
6668
|
+
return {
|
|
6669
|
+
status: 400,
|
|
6670
|
+
json: {
|
|
6671
|
+
message: "This fix cannot be made from the Studio yet. Run `val validate --fix` in the project."
|
|
6672
|
+
}
|
|
6673
|
+
};
|
|
6674
|
+
case "ok":
|
|
6675
|
+
return {
|
|
6676
|
+
status: 200,
|
|
6677
|
+
json: {
|
|
6678
|
+
patch: res.patch,
|
|
6679
|
+
remainingErrors: res.remainingErrors
|
|
6680
|
+
}
|
|
6681
|
+
};
|
|
6682
|
+
}
|
|
6683
|
+
}
|
|
6684
|
+
},
|
|
5767
6685
|
"/external-urls/check": {
|
|
5768
6686
|
POST: async req => {
|
|
5769
6687
|
/*
|
|
@@ -6437,8 +7355,9 @@ const ValServer = (valModules, options, callbacks, platform) => {
|
|
|
6437
7355
|
// TODO: send binary files validation errors
|
|
6438
7356
|
if (query.validate_binary_files) {
|
|
6439
7357
|
await serverOps.validateFiles(schemas, sourcesRes.sources, sourcesValidation.files);
|
|
6440
|
-
//
|
|
6441
|
-
|
|
7358
|
+
// Remote files are not checked here: their check asks each host
|
|
7359
|
+
// over the network, so it is the Studio's full check that runs it,
|
|
7360
|
+
// one module at a time (`/validate`).
|
|
6442
7361
|
}
|
|
6443
7362
|
}
|
|
6444
7363
|
const schemaSha = await serverOps.getSchemaSha();
|
|
@@ -6586,7 +7505,7 @@ const ValServer = (valModules, options, callbacks, platform) => {
|
|
|
6586
7505
|
withJob: prepared,
|
|
6587
7506
|
jobPatchIds: patchIds
|
|
6588
7507
|
});
|
|
6589
|
-
const build = serverOps.embedsSource() ? await buildSourcesOf(serverOps, patchIds, commit) : null;
|
|
7508
|
+
const build = serverOps.embedsSource() ? await buildSourcesOf(platform, serverOps, patchIds, commit) : null;
|
|
6590
7509
|
if (build !== null && "status" in build) return build;
|
|
6591
7510
|
/*
|
|
6592
7511
|
* A build of no commit says what Val source it holds instead, so
|
|
@@ -6618,7 +7537,7 @@ const ValServer = (valModules, options, callbacks, platform) => {
|
|
|
6618
7537
|
buildable: build !== null
|
|
6619
7538
|
});
|
|
6620
7539
|
}
|
|
6621
|
-
const built = await buildSourcesOf(serverOps, patchIds, commit);
|
|
7540
|
+
const built = await buildSourcesOf(platform, serverOps, patchIds, commit);
|
|
6622
7541
|
if ("status" in built) return built;
|
|
6623
7542
|
const {
|
|
6624
7543
|
split
|
|
@@ -6688,7 +7607,7 @@ const ValServer = (valModules, options, callbacks, platform) => {
|
|
|
6688
7607
|
* and the selection on top. Only what the selection touched is saved,
|
|
6689
7608
|
* so the proposal's save is the selection and nothing else.
|
|
6690
7609
|
*/
|
|
6691
|
-
const built = await buildSourcesOf(serverOps, patchIds, commit, {
|
|
7610
|
+
const built = await buildSourcesOf(platform, serverOps, patchIds, commit, {
|
|
6692
7611
|
binaries: false
|
|
6693
7612
|
});
|
|
6694
7613
|
if ("status" in built) return built;
|
|
@@ -7233,7 +8152,7 @@ const ValServer = (valModules, options, callbacks, platform) => {
|
|
|
7233
8152
|
* patch's to read. See `binaryFiles` on the route.
|
|
7234
8153
|
*/
|
|
7235
8154
|
const managedBranch = serverOps.sourceMode() === "managed" ? serverOps.projectBranch() : null;
|
|
7236
|
-
const committedBinaries = serverOps.sourceMode() === "managed" ? await readCommittedBinaryFiles(serverOps, preparedCommit.patchedBinaryFilesDescriptors) : null;
|
|
8155
|
+
const committedBinaries = serverOps.sourceMode() === "managed" ? await readCommittedBinaryFiles(serverOps, preparedCommit.patchedBinaryFilesDescriptors, platform.bytesToBase64) : null;
|
|
7237
8156
|
const commitToGit = () => serverOps.commit(preparedCommit, message, auth.id, FILES_DIRECTORY, undefined);
|
|
7238
8157
|
/*
|
|
7239
8158
|
* A host may publish somewhere other than the repository.
|
|
@@ -7893,7 +8812,7 @@ const ValServer = (valModules, options, callbacks, platform) => {
|
|
|
7893
8812
|
// wrap it in a data: URL, and hand that to a method whose first
|
|
7894
8813
|
// act was to unwrap it again - ceremony to satisfy a convention
|
|
7895
8814
|
// that no longer exists.
|
|
7896
|
-
const bytes =
|
|
8815
|
+
const bytes = new Uint8Array(await binaryRes.arrayBuffer());
|
|
7897
8816
|
const type = file.metadata.mimeType.startsWith("image/") ? "image" : "file";
|
|
7898
8817
|
const saveRes = await serverOps.saveBinaryFileFromPatch(file.filePath, req.body.parentRef, req.body.patchId, bytes, file.metadata.mimeType, type, file.metadata);
|
|
7899
8818
|
if (saveRes.error) {
|
|
@@ -8139,7 +9058,7 @@ const ValServer = (valModules, options, callbacks, platform) => {
|
|
|
8139
9058
|
// TypeScript compiler. See `patch/ts/publishKit.ts`.
|
|
8140
9059
|
const {
|
|
8141
9060
|
getJsonEntryAtCommit
|
|
8142
|
-
} = await Promise.resolve().then(function () { return require('./getJsonEntryAtCommit-
|
|
9061
|
+
} = await Promise.resolve().then(function () { return require('./getJsonEntryAtCommit-64f67a50.cjs.dev.js'); });
|
|
8143
9062
|
const res = await getJsonEntryAtCommit(serverOps, req.query.commit_sha, req.query.path, req.query.key);
|
|
8144
9063
|
if (fp.result.isErr(res)) {
|
|
8145
9064
|
return historyErrorResponse(res.error);
|
|
@@ -8247,11 +9166,11 @@ const ValServer = (valModules, options, callbacks, platform) => {
|
|
|
8247
9166
|
// A DRAFT playlist names files that are drafts too, at URLs that only
|
|
8248
9167
|
// resolve once published. See `rewriteDraftPlaylist`. Playlists are
|
|
8249
9168
|
// small, so one is read whole to be rewritten.
|
|
8250
|
-
const file = patchId && isHlsPlaylistPath(filePath) ? bufferReader(
|
|
9169
|
+
const file = patchId && isHlsPlaylistPath(filePath) ? bufferReader(utf8ToBytes(rewriteDraftPlaylist(bytesToUtf8(await opened.read(0, opened.size - 1)), {
|
|
8251
9170
|
playlistPath: filePath,
|
|
8252
9171
|
patchId,
|
|
8253
9172
|
remote
|
|
8254
|
-
})
|
|
9173
|
+
}))) : opened;
|
|
8255
9174
|
const headers = {
|
|
8256
9175
|
// TODO: we could use ETag and return 304 instead
|
|
8257
9176
|
// From the extension, published or draft: a `<video>` and an HLS
|
|
@@ -8291,7 +9210,7 @@ const ValServer = (valModules, options, callbacks, platform) => {
|
|
|
8291
9210
|
body: bufferToReadableStream(slice)
|
|
8292
9211
|
};
|
|
8293
9212
|
}
|
|
8294
|
-
const body = file.size === 0 ?
|
|
9213
|
+
const body = file.size === 0 ? new Uint8Array(0) : await file.read(0, file.size - 1);
|
|
8295
9214
|
return {
|
|
8296
9215
|
status: 200,
|
|
8297
9216
|
headers: {
|
|
@@ -8353,7 +9272,7 @@ function verifyCallbackReq(stateCookie, queryParams) {
|
|
|
8353
9272
|
}
|
|
8354
9273
|
function getStateFromCookie(stateCookie) {
|
|
8355
9274
|
try {
|
|
8356
|
-
const decoded =
|
|
9275
|
+
const decoded = bytesToUtf8(base64ToBytes(stateCookie));
|
|
8357
9276
|
const parsed = JSON.parse(decoded);
|
|
8358
9277
|
if (!parsed) {
|
|
8359
9278
|
return {
|
|
@@ -8405,7 +9324,7 @@ function getStateFromCookie(stateCookie) {
|
|
|
8405
9324
|
}
|
|
8406
9325
|
}
|
|
8407
9326
|
function createStateCookie(state) {
|
|
8408
|
-
return
|
|
9327
|
+
return bytesToBase64(utf8ToBytes(JSON.stringify(state)));
|
|
8409
9328
|
}
|
|
8410
9329
|
const ValAppJwtPayload = z.z.object({
|
|
8411
9330
|
sub: z.z.string(),
|
|
@@ -8688,7 +9607,7 @@ function getIsRemoteRequired(schemas) {
|
|
|
8688
9607
|
* `split.archive` is the managed job's archive; a connected job's archive is
|
|
8689
9608
|
* the push's, made from its own changes alone, by the caller.
|
|
8690
9609
|
*/
|
|
8691
|
-
async function buildSourcesOf(serverOps, patchIds, commit,
|
|
9610
|
+
async function buildSourcesOf(platform, serverOps, patchIds, commit,
|
|
8692
9611
|
/**
|
|
8693
9612
|
* Read the job's binary files back for a build. Off for a caller that only
|
|
8694
9613
|
* commits what was prepared, which names the files rather than carrying them.
|
|
@@ -8754,7 +9673,7 @@ options = {
|
|
|
8754
9673
|
jobPatchIds: patchIds
|
|
8755
9674
|
});
|
|
8756
9675
|
// Read now, while the job's changes still hold their files.
|
|
8757
|
-
const binaries = options.binaries ? await readCommittedBinaryFiles(serverOps, split.archive.patchedBinaryFilesDescriptors) : {
|
|
9676
|
+
const binaries = options.binaries ? await readCommittedBinaryFiles(serverOps, split.archive.patchedBinaryFilesDescriptors, platform.bytesToBase64) : {
|
|
8758
9677
|
files: {},
|
|
8759
9678
|
unread: []
|
|
8760
9679
|
};
|
|
@@ -8869,11 +9788,6 @@ async function uploadRemoteFile(contentHost, project, bucket, fileHash, fileExt,
|
|
|
8869
9788
|
};
|
|
8870
9789
|
}
|
|
8871
9790
|
|
|
8872
|
-
function getFileExt(filePath) {
|
|
8873
|
-
// NOTE: We do not import the path module. This code is copied in different projects. We want the same implementation and which means that this might running in browser where path is not available).
|
|
8874
|
-
return filePath.split(".").pop() || "";
|
|
8875
|
-
}
|
|
8876
|
-
|
|
8877
9791
|
/**
|
|
8878
9792
|
* One stored patch. Everything the ordered chain needs and nothing else.
|
|
8879
9793
|
*/
|
|
@@ -9574,7 +10488,11 @@ function createValRequestHandler(route, valServerPromise, convert, serveStatic)
|
|
|
9574
10488
|
const method = (_req$method = req.method) === null || _req$method === void 0 ? void 0 : _req$method.toUpperCase();
|
|
9575
10489
|
let route = null;
|
|
9576
10490
|
let path = undefined;
|
|
9577
|
-
|
|
10491
|
+
// An exact match wins over every prefix match, wherever it sits in
|
|
10492
|
+
// `Api`: checked per key, in order, `/validate` took `/validate/fix`
|
|
10493
|
+
// because it was declared first.
|
|
10494
|
+
const exactRouteDef = Object.keys(internal.Api).find(routeDef => routeDef === reqApiRoutePath);
|
|
10495
|
+
for (const routeDef of exactRouteDef ? [exactRouteDef] : Object.keys(internal.Api)) {
|
|
9578
10496
|
if (routeDef === reqApiRoutePath) {
|
|
9579
10497
|
route = routeDef;
|
|
9580
10498
|
break;
|
|
@@ -9873,10 +10791,16 @@ exports.InMemoryPatchStore = InMemoryPatchStore;
|
|
|
9873
10791
|
exports.ValOps = ValOps;
|
|
9874
10792
|
exports.ValOpsMemory = ValOpsMemory;
|
|
9875
10793
|
exports.ValServer = ValServer;
|
|
10794
|
+
exports.askOncePerHost = askOncePerHost;
|
|
9876
10795
|
exports.bufferFromDataUrl = bufferFromDataUrl;
|
|
10796
|
+
exports.bytesToBase64 = bytesToBase64;
|
|
10797
|
+
exports.bytesToUtf8 = bytesToUtf8;
|
|
10798
|
+
exports.checkRemoteFiles = checkRemoteFiles;
|
|
9877
10799
|
exports.classifyJsonValuesOp = classifyJsonValuesOp;
|
|
9878
10800
|
exports.createMetadataFromBuffer = createMetadataFromBuffer;
|
|
9879
10801
|
exports.createValRequestHandler = createValRequestHandler;
|
|
10802
|
+
exports.fallbackRefOf = fallbackRefOf;
|
|
10803
|
+
exports.fallbackRemoteHost = fallbackRemoteHost;
|
|
9880
10804
|
exports.findNestedJsonValuesRecords = findNestedJsonValuesRecords;
|
|
9881
10805
|
exports.formatPatchSourceError = formatPatchSourceError;
|
|
9882
10806
|
exports.getFieldsForType = getFieldsForType;
|
|
@@ -9888,10 +10812,13 @@ exports.isHlsMasterPlaylist = isHlsMasterPlaylist;
|
|
|
9888
10812
|
exports.isJsonValuesRootOp = isJsonValuesRootOp;
|
|
9889
10813
|
exports.largestHlsVariant = largestHlsVariant;
|
|
9890
10814
|
exports.mapHlsUris = mapHlsUris;
|
|
10815
|
+
exports.movedRemoteHost = movedRemoteHost;
|
|
10816
|
+
exports.needsHostCheck = needsHostCheck;
|
|
9891
10817
|
exports.parseHlsMasterVariants = parseHlsMasterVariants;
|
|
9892
10818
|
exports.posixPath = posixPath;
|
|
9893
10819
|
exports.rebaseContentOp = rebaseContentOp;
|
|
9894
10820
|
exports.resolveExistingJsonPath = resolveExistingJsonPath;
|
|
10821
|
+
exports.rewriteRefHost = rewriteRefHost;
|
|
9895
10822
|
exports.sumHlsSegmentDurations = sumHlsSegmentDurations;
|
|
9896
10823
|
exports.uploadRemoteFile = uploadRemoteFile;
|
|
9897
10824
|
exports.validateJsonValuesEntries = validateJsonValuesEntries;
|