@valbuild/server 0.147.0 → 0.148.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. package/CHANGELOG.md +115 -0
  2. package/browser/dist/valbuild-server-browser.cjs.dev.js +2 -2
  3. package/browser/dist/valbuild-server-browser.cjs.prod.js +2 -2
  4. package/browser/dist/valbuild-server-browser.esm.js +2 -2
  5. package/dist/{ValRouterCore-7ffcc08c.cjs.prod.js → ValRouterCore-42772c21.cjs.prod.js} +980 -53
  6. package/dist/{ValRouterCore-f5004f8f.esm.js → ValRouterCore-7ca3dc68.esm.js} +972 -54
  7. package/dist/{ValRouterCore-3fd3d444.cjs.dev.js → ValRouterCore-9bdaaf89.cjs.dev.js} +980 -53
  8. package/dist/declarations/src/ValOps.d.ts +32 -11
  9. package/dist/declarations/src/ValOpsMemory.d.ts +4 -4
  10. package/dist/declarations/src/ValServer.d.ts +1 -2
  11. package/dist/declarations/src/checkRemoteRef.d.ts +40 -2
  12. package/dist/declarations/src/createFixPatch.d.ts +25 -3
  13. package/dist/declarations/src/fixFiles.d.ts +42 -0
  14. package/dist/declarations/src/fixHandlers.d.ts +14 -0
  15. package/dist/declarations/src/getSettings.d.ts +1 -0
  16. package/dist/declarations/src/index.d.ts +1 -0
  17. package/dist/declarations/src/platform.d.ts +16 -0
  18. package/dist/declarations/src/remoteFileCheck.d.ts +116 -0
  19. package/dist/declarations/src/remoteHostCheck.d.ts +101 -0
  20. package/dist/declarations/src/studioValidation.d.ts +84 -0
  21. package/dist/declarations/src/uploadRemoteFile.d.ts +1 -1
  22. package/dist/declarations/src/videosetFixes.d.ts +3 -2
  23. package/dist/{getJsonEntryAtCommit-df65d861.cjs.prod.js → getJsonEntryAtCommit-2e36e805.cjs.prod.js} +3 -4
  24. package/dist/{getJsonEntryAtCommit-1e08d724.esm.js → getJsonEntryAtCommit-40b6d9c1.esm.js} +3 -4
  25. package/dist/{getJsonEntryAtCommit-e96beb9e.cjs.dev.js → getJsonEntryAtCommit-64f67a50.cjs.dev.js} +3 -4
  26. package/dist/valbuild-server.cjs.dev.js +1352 -695
  27. package/dist/valbuild-server.cjs.prod.js +1352 -695
  28. package/dist/valbuild-server.esm.js +1352 -696
  29. package/package.json +4 -5
@@ -1,6 +1,5 @@
1
1
  'use strict';
2
2
 
3
- var buffer = require('buffer');
4
3
  var core = require('@valbuild/core');
5
4
  var internal = require('@valbuild/shared/internal');
6
5
  var z = require('zod');
@@ -928,6 +927,118 @@ async function validateJsonValuesEntries(schema, source, modulePath) {
928
927
  return res;
929
928
  }
930
929
 
930
+ /**
931
+ * Bytes as text and back, without Node's `Buffer`.
932
+ *
933
+ * The request handlers run in a browser tab too (`@valbuild/server/browser`),
934
+ * and a browser has no `Buffer`. Importing npm's `buffer` instead is what broke
935
+ * the demo under `vite dev`: it is CommonJS, and a dependency the dev server
936
+ * did not see in its scan is served unconverted. So bytes are `Uint8Array`
937
+ * here, and these are the conversions `Buffer` used to do -- with the same
938
+ * answers, which `bytes.test.ts` checks against `Buffer` itself.
939
+ */
940
+
941
+ /**
942
+ * The value of each character code, or -1 for one that is not base64. Both
943
+ * alphabets, because `Buffer.from(s, "base64")` takes either: `-` and `_` are
944
+ * the URL-safe spellings of `+` and `/`.
945
+ */
946
+ const BASE64_VALUES = (() => {
947
+ const BASE64_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
948
+ const values = new Int16Array(128).fill(-1);
949
+ for (let i = 0; i < BASE64_ALPHABET.length; i++) {
950
+ values[BASE64_ALPHABET.charCodeAt(i)] = i;
951
+ }
952
+ values["-".charCodeAt(0)] = 62;
953
+ values["_".charCodeAt(0)] = 63;
954
+ return values;
955
+ })();
956
+ const PADDING = "=".charCodeAt(0);
957
+
958
+ /**
959
+ * Decodes base64 the way `Buffer.from(base64, "base64")` does: either
960
+ * alphabet, padding optional, anything that is not base64 skipped, and
961
+ * decoding stops at the first `=`. A trailing partial group gives the whole
962
+ * bytes it holds.
963
+ *
964
+ * `atob` first, because it is native and this is called with whole uploads;
965
+ * it agrees with `Buffer` on everything it accepts, and refuses the rest (the
966
+ * URL-safe alphabet, stray characters, `=` in the middle), which then goes the
967
+ * slow way.
968
+ */
969
+ function base64ToBytes(base64) {
970
+ let binary;
971
+ try {
972
+ binary = atob(base64);
973
+ } catch {
974
+ return lenientBase64ToBytes(base64);
975
+ }
976
+ const bytes = new Uint8Array(binary.length);
977
+ for (let i = 0; i < binary.length; i++) {
978
+ bytes[i] = binary.charCodeAt(i);
979
+ }
980
+ return bytes;
981
+ }
982
+ function lenientBase64ToBytes(base64) {
983
+ const bytes = new Uint8Array(Math.floor(base64.length * 3 / 4));
984
+ let length = 0;
985
+ let bits = 0;
986
+ let bitCount = 0;
987
+ for (let i = 0; i < base64.length; i++) {
988
+ const code = base64.charCodeAt(i);
989
+ if (code === PADDING) break;
990
+ const value = code < 128 ? BASE64_VALUES[code] : -1;
991
+ if (value < 0) continue;
992
+ bits = (bits << 6 | value) & 0xffffff;
993
+ bitCount += 6;
994
+ if (bitCount >= 8) {
995
+ bitCount -= 8;
996
+ bytes[length++] = bits >> bitCount & 0xff;
997
+ }
998
+ }
999
+ return bytes.subarray(0, length);
1000
+ }
1001
+
1002
+ /**
1003
+ * How many bytes go to `String.fromCharCode` at once: it takes them as
1004
+ * arguments, and an engine has a limit on how many a call may have.
1005
+ */
1006
+ const CHUNK = 0x8000;
1007
+
1008
+ /**
1009
+ * `Buffer.from(bytes).toString("base64")`: padded, standard alphabet.
1010
+ *
1011
+ * Through `btoa`, which is native, rather than building the string here: that
1012
+ * was five times slower, and this is called with whole files. Node is still
1013
+ * faster with `Buffer`, which is why `ValServerPlatform.bytesToBase64` exists
1014
+ * for the paths that encode every file of a publish.
1015
+ */
1016
+ function bytesToBase64(bytes) {
1017
+ const binary = [];
1018
+ for (let i = 0; i < bytes.length; i += CHUNK) {
1019
+ // `Reflect.apply` rather than a spread, which copies the chunk into an
1020
+ // argument list first and is four times slower.
1021
+ const chunk = Reflect.apply(String.fromCharCode, null, bytes.subarray(i, i + CHUNK));
1022
+ binary.push(chunk);
1023
+ }
1024
+ return btoa(binary.join(""));
1025
+ }
1026
+
1027
+ /** `Buffer.from(text, "utf8")`. */
1028
+ function utf8ToBytes(text) {
1029
+ return new TextEncoder().encode(text);
1030
+ }
1031
+
1032
+ /**
1033
+ * `Buffer.from(bytes).toString("utf8")`. `ignoreBOM` because `Buffer` keeps a
1034
+ * leading byte order mark, where a `TextDecoder` drops it by default.
1035
+ */
1036
+ function bytesToUtf8(bytes) {
1037
+ return new TextDecoder("utf-8", {
1038
+ ignoreBOM: true
1039
+ }).decode(bytes);
1040
+ }
1041
+
931
1042
  /**
932
1043
  * Does saving this commit upload a file to Val's remote host?
933
1044
  *
@@ -1342,7 +1453,8 @@ function isHlsPlaylistPath(filePath) {
1342
1453
  * patch's to read. Generic over the patch id so it asks for exactly the one
1343
1454
  * method it uses, and a test can hand it a plain object.
1344
1455
  */
1345
- async function readCommittedBinaryFiles(ops, descriptors) {
1456
+ async function readCommittedBinaryFiles(ops, descriptors, /** `ValServerPlatform.bytesToBase64`, which is faster in Node. */
1457
+ encode = bytesToBase64) {
1346
1458
  const files = {};
1347
1459
  const unread = [];
1348
1460
  await Promise.all(Object.entries(descriptors).map(async ([filePath, {
@@ -1354,7 +1466,7 @@ async function readCommittedBinaryFiles(ops, descriptors) {
1354
1466
  console.error("Could not read a committed file", filePath, error);
1355
1467
  return null;
1356
1468
  });
1357
- if (bytes === null) unread.push(filePath);else files[filePath] = bytes.toString("base64");
1469
+ if (bytes === null) unread.push(filePath);else files[filePath] = encode(bytes);
1358
1470
  }));
1359
1471
  return {
1360
1472
  files,
@@ -1521,10 +1633,537 @@ function computePatchesToDrop(prepared) {
1521
1633
  return dropped;
1522
1634
  }
1523
1635
 
1524
- /*
1525
- * Imported rather than global: under Node this is the built-in, and in a
1526
- * browser (the demo, `@valbuild/server/browser`) the `buffer` package.
1636
+ function getFileExt(filePath) {
1637
+ // NOTE: We do not import the path module. This code is copied in different projects. We want the same implementation and which means that this might running in browser where path is not available).
1638
+ return filePath.split(".").pop() || "";
1639
+ }
1640
+
1641
+ /**
1642
+ * Does the host a platform ref names still serve the project the ref names?
1643
+ *
1644
+ * A ref served by the platform loader names the project's host at the time
1645
+ * the file was added: `https://www.acme.no/_val/file/p/3f9a2c1/…`. When the
1646
+ * project drops `www.acme.no`, the bytes are still there, at
1647
+ * `3f9a2c1.valstart.dev`, but every ref that names the old domain points at a
1648
+ * host that no longer serves them. This is the question that finds those refs,
1649
+ * and `rewriteRefHost` is what the fix (`remote:rewrite-host`) does about it.
1650
+ * See valbuild/home `docs/remote-files-on-platform.md`, "Rewriting refs after a
1651
+ * domain is removed", and `docs/plans/studio-validate.md`, step 6.
1652
+ *
1653
+ * Asked of the host itself (`GET <host>/_val/project`), not of content: the
1654
+ * host's answer covers DNS, the certificate and the loader's mapping in one,
1655
+ * it needs no credential, and it works for a ref that names a project other
1656
+ * than this one -- a project seeded from a template has refs naming the
1657
+ * template's.
1658
+ */
1659
+
1660
+ /** The domain every platform project is always served at, by its public id. */
1661
+ const PLATFORM_SITES_DOMAIN = "valstart.dev";
1662
+
1663
+ /** What a platform ref's host part ends with. */
1664
+ const PLATFORM_PREFIX = "/_val";
1665
+
1666
+ /**
1667
+ * The host every platform ref of `publicProjectId` can always be pointed at:
1668
+ * `<public id>.valstart.dev` is permanent (decision 4 of the remote-files
1669
+ * plan), and the loader serves a project's files there whatever else it has.
1670
+ */
1671
+ function fallbackRemoteHost(publicProjectId) {
1672
+ return `https://${publicProjectId}.${PLATFORM_SITES_DOMAIN}${PLATFORM_PREFIX}`;
1673
+ }
1674
+
1675
+ /**
1676
+ * Only a platform ref is asked about. `remote.val.build` is not a platform
1677
+ * host and is not going away, and a ref already on the project's own
1678
+ * `valstart.dev` address is on the fallback, which always serves it.
1679
+ */
1680
+ function needsHostCheck(remoteHost, projectId) {
1681
+ return remoteHost.endsWith(PLATFORM_PREFIX) && remoteHost !== fallbackRemoteHost(projectId);
1682
+ }
1683
+
1684
+ /** `ref` with its host part replaced, and everything else as it was. */
1685
+ function rewriteRefHost(ref, remoteHost) {
1686
+ const parsed = core.Internal.remote.splitRemoteRef(ref);
1687
+ if (parsed.status === "error") {
1688
+ return null;
1689
+ }
1690
+ return core.Internal.remote.createRemoteRef(remoteHost, {
1691
+ publicProjectId: parsed.projectId,
1692
+ coreVersion: parsed.version,
1693
+ bucket: parsed.bucket,
1694
+ validationHash: parsed.validationHash,
1695
+ fileHash: parsed.fileHash,
1696
+ filePath: parsed.filePath
1697
+ });
1698
+ }
1699
+
1700
+ /**
1701
+ * - `serves`: the host answered for the project the ref names.
1702
+ * - `other-project`: it answered, for another one -- the domain was moved to
1703
+ * a different project, and decision 5 means it will not serve these files.
1704
+ * - `no-site`: it answered, but serves no project at all (the loader's 404 for
1705
+ * a domain nobody has mapped any more).
1706
+ * - `unreachable`: no answer. A domain whose DNS was removed looks like this,
1707
+ * and so does a host that is down for a minute, which is why the row says
1708
+ * which it saw.
1709
+ */
1710
+
1711
+ const ASK_TIMEOUT_MS = 8_000;
1712
+
1713
+ /** `GET <host>/_val/project`, answered by the platform loader. */
1714
+ const askHostProject = async remoteHost => {
1715
+ try {
1716
+ const res = await fetch(`${remoteHost}/project`, {
1717
+ signal: AbortSignal.timeout(ASK_TIMEOUT_MS)
1718
+ });
1719
+ if (!res.ok) {
1720
+ return {
1721
+ status: "no-site",
1722
+ httpStatus: res.status
1723
+ };
1724
+ }
1725
+ const body = await res.json();
1726
+ if (typeof body === "object" && body !== null && "publicProjectId" in body && typeof body.publicProjectId === "string") {
1727
+ return {
1728
+ status: "answered",
1729
+ publicProjectId: body.publicProjectId
1730
+ };
1731
+ }
1732
+ return {
1733
+ status: "no-site",
1734
+ httpStatus: res.status
1735
+ };
1736
+ } catch (err) {
1737
+ return {
1738
+ status: "unreachable",
1739
+ reason: err instanceof Error && err.name === "TimeoutError" ? `No answer within ${ASK_TIMEOUT_MS / 1000} s` : err instanceof Error ? err.message : "Unknown error"
1740
+ };
1741
+ }
1742
+ };
1743
+
1744
+ /**
1745
+ * Hosts that answered for a project, and when: one question per host, not per
1746
+ * ref, and not again for a few minutes. Only a "serves" is kept, for the same
1747
+ * reason `remoteFileCheck` keeps only "found": a host that stopped serving has
1748
+ * to be noticed, and one that did not answer has to be asked again.
1749
+ */
1750
+ const SERVES_FOR_MS = 5 * 60_000;
1751
+ const servesAt = new Map();
1752
+ async function hostServes(remoteHost, projectId, ask = askHostProject) {
1753
+ const key = `${remoteHost} ${projectId}`;
1754
+ const at = servesAt.get(key);
1755
+ if (at !== undefined && Date.now() - at < SERVES_FOR_MS) {
1756
+ return {
1757
+ status: "serves"
1758
+ };
1759
+ }
1760
+ const answer = await ask(remoteHost);
1761
+ if (answer.status !== "answered") {
1762
+ return answer;
1763
+ }
1764
+ if (answer.publicProjectId !== projectId) {
1765
+ return {
1766
+ status: "other-project",
1767
+ publicProjectId: answer.publicProjectId
1768
+ };
1769
+ }
1770
+ servesAt.set(key, Date.now());
1771
+ return {
1772
+ status: "serves"
1773
+ };
1774
+ }
1775
+
1776
+ /**
1777
+ * `ask`, asking each host once for as long as the returned function is kept:
1778
+ * one check, not the life of the server. A check walks its refs in order, so
1779
+ * without it every ref on a host that does not answer waits out its own
1780
+ * timeout -- 150 images on a dead domain is 20 minutes. Kept per check rather
1781
+ * than in {@link hostServes}'s cache because a host that did not answer has
1782
+ * to be asked again by the next one.
1783
+ */
1784
+ function askOncePerHost(ask = askHostProject) {
1785
+ const answers = new Map();
1786
+ return remoteHost => {
1787
+ let answer = answers.get(remoteHost);
1788
+ if (answer === undefined) {
1789
+ answer = ask(remoteHost);
1790
+ answers.set(remoteHost, answer);
1791
+ }
1792
+ return answer;
1793
+ };
1794
+ }
1795
+
1796
+ /** `www.acme.no`, from `https://www.acme.no/_val`: how a person names a host. */
1797
+ function hostNameOfRemoteHost(remoteHost) {
1798
+ try {
1799
+ return new URL(remoteHost).host;
1800
+ } catch {
1801
+ return remoteHost;
1802
+ }
1803
+ }
1804
+
1805
+ /**
1806
+ * Where a platform ref has to move to, when its host no longer serves the
1807
+ * project it names; `undefined` when it does not have to (or is not a
1808
+ * platform ref). `why` and `to` are for the caller's sentence: what can be
1809
+ * done about it differs by caller (a Studio **Fix**, `val validate --fix`, or
1810
+ * nothing yet).
1811
+ */
1812
+ async function movedRemoteHost(ref, ask) {
1813
+ const parsed = core.Internal.remote.splitRemoteRef(ref);
1814
+ if (parsed.status === "error" || !needsHostCheck(parsed.remoteHost, parsed.projectId)) {
1815
+ return undefined;
1816
+ }
1817
+ const serves = await hostServes(parsed.remoteHost, parsed.projectId, ask);
1818
+ // Only an answer that says so moves a ref: the host serves another
1819
+ // project, or the loader says it serves none (its 404 for a domain nobody
1820
+ // maps). A host that did not answer, or answered 5xx, may be down for a
1821
+ // minute -- and a moved ref does not move back, so rewriting on that would
1822
+ // take a site's images off its own domain for good over one blip. Those are
1823
+ // left to the existence check, which reports the host once with Try again.
1824
+ if (serves.status === "serves" || serves.status === "unreachable" || serves.status === "no-site" && serves.httpStatus !== 404) {
1825
+ return undefined;
1826
+ }
1827
+ const target = fallbackRemoteHost(parsed.projectId);
1828
+ const moved = rewriteRefHost(ref, target);
1829
+ if (moved === null) {
1830
+ return undefined;
1831
+ }
1832
+ const from = hostNameOfRemoteHost(parsed.remoteHost);
1833
+ return {
1834
+ ref: moved,
1835
+ why: serves.status === "other-project" ? `${from} now serves another project, so this file is not shown from there` : `${from} no longer serves this project, so this file is not shown from there`,
1836
+ to: hostNameOfRemoteHost(target)
1837
+ };
1838
+ }
1839
+
1840
+ /**
1841
+ * Where else a ref's file can be downloaded from: the same file on the
1842
+ * project's own `valstart.dev` address, for a platform ref on any other host.
1843
+ * `null` when there is nowhere else -- a `remote.val.build` ref, or one already
1844
+ * on the fallback. See valbuild/home `docs/remote-files-on-platform.md`,
1845
+ * "Tooling: the fallback".
1527
1846
  */
1847
+ function fallbackRefOf(ref) {
1848
+ const parsed = core.Internal.remote.splitRemoteRef(ref);
1849
+ if (parsed.status === "error" || !needsHostCheck(parsed.remoteHost, parsed.projectId)) {
1850
+ return null;
1851
+ }
1852
+ return rewriteRefHost(ref, fallbackRemoteHost(parsed.projectId));
1853
+ }
1854
+
1855
+ /**
1856
+ * Does the host a remote ref names still serve it?
1857
+ *
1858
+ * `found` and `missing` are the host's answer. `unreachable` is no answer at
1859
+ * all — a timeout, a refused connection, a 5xx — and says nothing about the
1860
+ * file, only about the host.
1861
+ */
1862
+
1863
+ /**
1864
+ * How the remote check reaches the network: what it asks a host about a file,
1865
+ * and which project a platform host serves. Both default to the real thing;
1866
+ * tests replace them.
1867
+ */
1868
+
1869
+ const HEAD_TIMEOUT_MS = 8_000;
1870
+
1871
+ /**
1872
+ * A HEAD, falling back to the first byte of a GET for a host that does not
1873
+ * answer HEAD (405, 501): either way no file is downloaded to learn that it is
1874
+ * there.
1875
+ */
1876
+ const headRemoteFile = async url => {
1877
+ const ask = async init => fetch(url, {
1878
+ ...init,
1879
+ redirect: "follow",
1880
+ signal: AbortSignal.timeout(HEAD_TIMEOUT_MS)
1881
+ });
1882
+ try {
1883
+ let res = await ask({
1884
+ method: "HEAD"
1885
+ });
1886
+ if (res.status === 405 || res.status === 501) {
1887
+ var _res$body;
1888
+ res = await ask({
1889
+ method: "GET",
1890
+ headers: {
1891
+ Range: "bytes=0-0"
1892
+ }
1893
+ });
1894
+ await ((_res$body = res.body) === null || _res$body === void 0 ? void 0 : _res$body.cancel());
1895
+ }
1896
+ if (res.ok) {
1897
+ return {
1898
+ status: "found"
1899
+ };
1900
+ }
1901
+ if (res.status === 404 || res.status === 410) {
1902
+ return {
1903
+ status: "missing",
1904
+ httpStatus: res.status
1905
+ };
1906
+ }
1907
+ return {
1908
+ status: "unreachable",
1909
+ reason: `HTTP ${res.status}`
1910
+ };
1911
+ } catch (err) {
1912
+ return {
1913
+ status: "unreachable",
1914
+ reason: describeFetchError(err)
1915
+ };
1916
+ }
1917
+ };
1918
+ function describeFetchError(err) {
1919
+ if (!(err instanceof Error)) {
1920
+ return "Unknown error";
1921
+ }
1922
+ if (err.name === "TimeoutError") {
1923
+ return `No answer within ${HEAD_TIMEOUT_MS / 1000} s`;
1924
+ }
1925
+ // undici puts what actually went wrong (ENOTFOUND, ECONNREFUSED) on `cause`.
1926
+ const cause = err.cause;
1927
+ if (cause instanceof Error) {
1928
+ return `${err.message}: ${cause.message}`;
1929
+ }
1930
+ return err.message;
1931
+ }
1932
+
1933
+ /**
1934
+ * Refs a host has served, and when.
1935
+ *
1936
+ * So that pressing **Check again**, or opening a second module that uses the
1937
+ * same images, does not ask about every file again. Kept for minutes, not for
1938
+ * the life of the process: a ref is content-addressed, so its answer only
1939
+ * changes when its host stops serving it -- which is exactly what this check
1940
+ * is for, and a server that remembered "found" forever would never report a
1941
+ * host that went down after it was first asked. Only `found` is kept. A host
1942
+ * that did not answer is asked again on the next check, which is what makes
1943
+ * **Try again** mean something.
1944
+ */
1945
+ const FOUND_FOR_MS = 5 * 60_000;
1946
+ const foundAt = new Map();
1947
+ function wasFound(ref) {
1948
+ const at = foundAt.get(ref);
1949
+ return at !== undefined && Date.now() - at < FOUND_FOR_MS;
1950
+ }
1951
+ function markFound(ref) {
1952
+ foundAt.set(ref, Date.now());
1953
+ }
1954
+
1955
+ /** A host that did not serve the refs of a module it was asked about. */
1956
+
1957
+ const textEncoder = new TextEncoder();
1958
+
1959
+ /**
1960
+ * The remote half of the Studio's full check, for one module's remote values.
1961
+ *
1962
+ * Four questions, cheapest first, and a ref stops at the first one it fails:
1963
+ *
1964
+ * 1. **Is it a ref at all?** `splitRemoteRef` parses it.
1965
+ * 2. **Does the ref agree with what is recorded about it?** A ref carries a
1966
+ * hash of the schema's options, the file's extension and its width, height
1967
+ * and mime type. When the value or the schema changed and the ref did not,
1968
+ * the ref is stale, and **Fix** (`image:check-remote`) re-reads the file and
1969
+ * writes the ref it should have. Computed with the ref's OWN core version:
1970
+ * the CLI also re-derives every ref after a Val upgrade, and listing every
1971
+ * remote image in the project after each upgrade would bury the rows that
1972
+ * matter. Nothing here downloads.
1973
+ * 3. **Does its host still serve its project?** Only for a platform ref
1974
+ * (`…/_val/file/…`) that is not on the project's own `valstart.dev`
1975
+ * address: one `GET <host>/_val/project` per host. A host that dropped the
1976
+ * project gets a **Fix** (`remote:rewrite-host`) that points the ref at
1977
+ * `<public id>.valstart.dev`. See `remoteHostCheck.ts`.
1978
+ * 4. **Does the host have it?** A HEAD.
1979
+ *
1980
+ * Neither host question is asked of a ref whose bytes are in a pending patch,
1981
+ * which the host will not have until publish: there the question is whether
1982
+ * the patch still has them.
1983
+ */
1984
+ async function checkRemoteFiles(input) {
1985
+ const head = input.head ?? headRemoteFile;
1986
+ const askHostProject = askOncePerHost(input.askHostProject);
1987
+ const errors = {};
1988
+ const addError = (sourcePath, error) => {
1989
+ errors[sourcePath] = (errors[sourcePath] ?? []).concat(error);
1990
+ };
1991
+
1992
+ /** Refs still to ask a host about, by host. */
1993
+ const toAsk = new Map();
1994
+ const sorted = Object.entries(input.remoteFiles).sort(([a], [b]) => a.localeCompare(b));
1995
+ for (const [sourcePathS, value] of sorted) {
1996
+ const sourcePath = sourcePathS;
1997
+ const ref = value.path;
1998
+ const parsed = core.Internal.remote.splitRemoteRef(ref);
1999
+ if (parsed.status === "error") {
2000
+ addError(sourcePath, {
2001
+ message: `This is not an address Val can check: ${parsed.error}`,
2002
+ value
2003
+ });
2004
+ continue;
2005
+ }
2006
+ let noun = "file";
2007
+ if (!input.existenceOnly) {
2008
+ const schema = input.schemaAt(sourcePath);
2009
+ if ((schema === null || schema === void 0 ? void 0 : schema.type) !== "image" && (schema === null || schema === void 0 ? void 0 : schema.type) !== "file") {
2010
+ continue;
2011
+ }
2012
+ noun = schema.type;
2013
+ if (isStale(parsed, schema, value)) {
2014
+ addError(sourcePath, {
2015
+ message: `The address of this ${schema.type} does not match what is recorded about it, so it may be served with the wrong size or type.`,
2016
+ value,
2017
+ fixes: [schema.type === "image" ? "image:check-remote" : "file:check-remote"]
2018
+ });
2019
+ continue;
2020
+ }
2021
+ }
2022
+ const pending = await input.pendingBytes(ref);
2023
+ if (pending === "present") {
2024
+ continue;
2025
+ }
2026
+ if (pending === "missing") {
2027
+ addError(sourcePath, {
2028
+ message: `This ${noun} is in your unpublished changes, but its file is not. Choose the ${noun} again.`,
2029
+ value
2030
+ });
2031
+ continue;
2032
+ }
2033
+ // Asked after the pending check: a ref whose bytes are still in a patch
2034
+ // is the key those bytes are stored under, and repointing it would cut
2035
+ // it off from them.
2036
+ const moved = await movedRemoteHost(ref, askHostProject);
2037
+ if (moved) {
2038
+ addError(sourcePath, input.existenceOnly ? {
2039
+ // A directory's key: the fields picking from it hold the same
2040
+ // ref, and would have to follow. Not built yet.
2041
+ message: `${moved.why}. Pointing a file in a media directory at ${moved.to} is not something the Studio can do yet.`,
2042
+ value
2043
+ } : {
2044
+ message: `${moved.why}. Fix points it at ${moved.to}, which always serves it.`,
2045
+ value,
2046
+ fixes: ["remote:rewrite-host"]
2047
+ });
2048
+ continue;
2049
+ }
2050
+ if (wasFound(ref)) {
2051
+ continue;
2052
+ }
2053
+ const refs = toAsk.get(parsed.remoteHost) ?? [];
2054
+ refs.push({
2055
+ sourcePath,
2056
+ ref,
2057
+ published: input.isPublished(sourcePath, ref)
2058
+ });
2059
+ toAsk.set(parsed.remoteHost, refs);
2060
+ }
2061
+ const hosts = [];
2062
+ await Promise.all([...toAsk.entries()].map(async ([host, refs]) => {
2063
+ const problem = await askHost(host, refs, input.sampleByHost, head);
2064
+ if (problem.host) {
2065
+ hosts.push(problem.host);
2066
+ }
2067
+ for (const {
2068
+ sourcePath,
2069
+ error
2070
+ } of problem.refErrors) {
2071
+ addError(sourcePath, error);
2072
+ }
2073
+ }));
2074
+ hosts.sort((a, b) => a.host.localeCompare(b.host));
2075
+ return {
2076
+ errors,
2077
+ hosts
2078
+ };
2079
+ }
2080
+
2081
+ /**
2082
+ * One host's refs, asked one at a time.
2083
+ *
2084
+ * In order rather than all at once: a host that does not answer is reported
2085
+ * after the FIRST timeout, not after one timeout per image. A host that
2086
+ * answers "missing" for one ref is still asked about the next, since a file
2087
+ * that is gone is a fact about that file.
2088
+ */
2089
+ async function askHost(host, refs, sampleByHost, head) {
2090
+ const vouchable = sampleByHost ? refs.filter(r => r.published) : [];
2091
+ const individually = sampleByHost ? refs.filter(r => !r.published) : refs;
2092
+ const allPaths = refs.map(r => r.sourcePath);
2093
+ const refErrors = [];
2094
+ if (vouchable.length > 0) {
2095
+ const sample = vouchable[0];
2096
+ const answer = await head(sample.ref);
2097
+ if (answer.status === "found") {
2098
+ for (const r of vouchable) {
2099
+ markFound(r.ref);
2100
+ }
2101
+ } else {
2102
+ // In a managed project a published ref cannot break on its own, so the
2103
+ // sample's answer is the host's: one row, every ref on it.
2104
+ return {
2105
+ host: {
2106
+ host,
2107
+ status: answer.status,
2108
+ details: detailsOf(sample.ref, answer),
2109
+ sourcePaths: allPaths
2110
+ },
2111
+ refErrors
2112
+ };
2113
+ }
2114
+ }
2115
+ for (const r of individually) {
2116
+ const answer = await head(r.ref);
2117
+ if (answer.status === "found") {
2118
+ markFound(r.ref);
2119
+ } else if (answer.status === "missing") {
2120
+ refErrors.push({
2121
+ sourcePath: r.sourcePath,
2122
+ error: {
2123
+ message: `The file is not at its address: ${hostName(host)} answered ${answer.httpStatus}.`
2124
+ }
2125
+ });
2126
+ } else {
2127
+ return {
2128
+ host: {
2129
+ host,
2130
+ status: "unreachable",
2131
+ details: detailsOf(r.ref, answer),
2132
+ sourcePaths: allPaths
2133
+ },
2134
+ refErrors
2135
+ };
2136
+ }
2137
+ }
2138
+ return {
2139
+ refErrors
2140
+ };
2141
+ }
2142
+ function detailsOf(ref, answer) {
2143
+ return `HEAD ${ref}\n→ ${answer.status === "missing" ? `${answer.httpStatus}` : answer.status === "unreachable" ? answer.reason : "200"}`;
2144
+ }
2145
+
2146
+ /** `www.acme.no`, from `https://www.acme.no`: how a person names a host. */
2147
+ function hostName(host) {
2148
+ try {
2149
+ return new URL(host).host;
2150
+ } catch {
2151
+ return host;
2152
+ }
2153
+ }
2154
+ function isStale(parsed, schema, value) {
2155
+ // The hash covers only these of what was recorded (`getValidationBasis`):
2156
+ // `alt` and `hotspot` are the editor's, and change no byte.
2157
+ const metadata = {
2158
+ width: value.width,
2159
+ height: value.height,
2160
+ mimeType: value.mimeType
2161
+ };
2162
+ const expected = core.Internal.remote.getValidationHash(parsed.version, schema, getFileExt(parsed.filePath), metadata, parsed.fileHash, textEncoder);
2163
+ return expected !== parsed.validationHash;
2164
+ }
2165
+
2166
+ /* eslint-disable @typescript-eslint/no-unused-vars */
1528
2167
  const jsonOps = new patch.JSONOps();
1529
2168
  /**
1530
2169
  * How many `.jsonValues()` entry files a whole-record write reads at a time.
@@ -2565,7 +3204,14 @@ class ValOps {
2565
3204
  }
2566
3205
 
2567
3206
  // #region validateSources
2568
- async validateSources(schemas, sources, patchesByModule) {
3207
+ async validateSources(schemas, sources, patchesByModule,
3208
+ /**
3209
+ * Validate only these modules. The snapshot the cross-module checks read
3210
+ * (a keyOf's keys, a route, a set's directory) is still built from every
3211
+ * module, which is what makes checking one module at a time give the same
3212
+ * answer for it as checking them all. Wins over `patchesByModule`.
3213
+ */
3214
+ onlyModules) {
2569
3215
  const errors = {};
2570
3216
  const files = {};
2571
3217
  const remoteFiles = {};
@@ -2598,7 +3244,18 @@ class ValOps {
2598
3244
  }
2599
3245
  }
2600
3246
  }
2601
- const modulePathsToValidate = patchesByModule && Object.keys(patchesByModule);
3247
+ const addModuleError = (path, sourcePath, validationError) => {
3248
+ if (!errors[path]) {
3249
+ errors[path] = {
3250
+ validations: {}
3251
+ };
3252
+ }
3253
+ if (!errors[path].validations[sourcePath]) {
3254
+ errors[path].validations[sourcePath] = [];
3255
+ }
3256
+ errors[path].validations[sourcePath].push(validationError);
3257
+ };
3258
+ const modulePathsToValidate = onlyModules ?? (patchesByModule && Object.keys(patchesByModule));
2602
3259
  for (const [pathS, schema] of entries) {
2603
3260
  if (modulePathsToValidate && !modulePathsToValidate.includes(pathS)) {
2604
3261
  continue;
@@ -2627,15 +3284,35 @@ class ValOps {
2627
3284
  } = await validateJsonValuesEntries(schema, source, path);
2628
3285
  for (const [sourcePathS, entryErrors] of Object.entries(jsonValuesErrors)) {
2629
3286
  const sourcePath = sourcePathS;
2630
- if (!errors[path]) {
2631
- errors[path] = {
2632
- validations: {}
2633
- };
2634
- }
2635
- if (!errors[path].validations[sourcePath]) {
2636
- errors[path].validations[sourcePath] = [];
3287
+ for (const entryError of entryErrors) {
3288
+ // An entry's errors get the same project-wide resolution as every
3289
+ // other module's. Added raw, every keyOf and route in a
3290
+ // `.jsonValues()` record was reported as unresolved -- 240 of them in
3291
+ // `examples/next`'s kb.val.ts, none real.
3292
+ if (internal.isSchemaSourceFixError(entryError)) {
3293
+ const resolved = internal.resolveSchemaSourceFixForError(entryError, snapshot);
3294
+ if (!resolved || resolved.status !== "remaining") {
3295
+ continue;
3296
+ }
3297
+ addModuleError(path, sourcePath, resolved.error);
3298
+ continue;
3299
+ }
3300
+ // "Not yet compared with its bytes" is a request, not an error, and
3301
+ // for an entry it cannot be answered here: `validateFiles` resolves a
3302
+ // file's schema through the module's source, which for a
3303
+ // `.jsonValues()` record holds markers rather than the entry. The
3304
+ // publish gate ignores it anyway (`partitionValidationErrors`).
3305
+ //
3306
+ // A remote file's "not yet checked against its host" is the same
3307
+ // request, for the same reason: `validateRemoteFiles` and its fix
3308
+ // resolve the path through the module's source too. Added as an
3309
+ // error, every remote image in an entry was a row on the Validate
3310
+ // page whose Fix could not find it.
3311
+ if (isOnlyFileCheckValidationError(entryError) || isOnlyRemoteCheckValidationError(entryError)) {
3312
+ continue;
3313
+ }
3314
+ addModuleError(path, sourcePath, entryError);
2637
3315
  }
2638
- errors[path].validations[sourcePath].push(...entryErrors);
2639
3316
  }
2640
3317
  if (res === false) {
2641
3318
  continue;
@@ -2713,9 +3390,64 @@ class ValOps {
2713
3390
  remoteFiles
2714
3391
  };
2715
3392
  }
2716
- async validateRemoteFiles(schemas, sources, remoteFiles) {
2717
- // TODO: Implement
2718
- return {};
3393
+
3394
+ /**
3395
+ * The remote values `validateSources` set aside, checked: is each ref
3396
+ * sound, and does its host serve it. See `checkRemoteFiles`.
3397
+ *
3398
+ * `fileLastUpdatedByPatchId` names the refs whose bytes are still in a
3399
+ * pending patch, which the host will not have until publish.
3400
+ */
3401
+ async validateRemoteFiles(schemas, sources, remoteFiles, fileLastUpdatedByPatchId = {}, network = {}) {
3402
+ const sampleByHost = this.sourceMode() === "managed";
3403
+ // Only a managed project vouches for one published ref with another, so
3404
+ // only it needs to know which refs are published.
3405
+ const baseSources = sampleByHost ? (await this.getSources()).sources : undefined;
3406
+ const valueAt = (sourcePath, from) => {
3407
+ const [moduleFilePath, modulePath] = core.Internal.splitModuleFilePathAndModulePath(sourcePath);
3408
+ const schema = schemas[moduleFilePath];
3409
+ if (!schema || from[moduleFilePath] === undefined) {
3410
+ return undefined;
3411
+ }
3412
+ try {
3413
+ return core.Internal.resolvePath(modulePath, from[moduleFilePath], schema["executeSerialize"]());
3414
+ } catch {
3415
+ return undefined;
3416
+ }
3417
+ };
3418
+ return checkRemoteFiles({
3419
+ remoteFiles,
3420
+ sampleByHost,
3421
+ ...network,
3422
+ schemaAt: sourcePath => {
3423
+ var _valueAt;
3424
+ return (_valueAt = valueAt(sourcePath, sources)) === null || _valueAt === void 0 ? void 0 : _valueAt.schema;
3425
+ },
3426
+ isPublished: (sourcePath, ref) => {
3427
+ var _valueAt2;
3428
+ if (!baseSources) {
3429
+ return false;
3430
+ }
3431
+ const published = (_valueAt2 = valueAt(sourcePath, baseSources)) === null || _valueAt2 === void 0 ? void 0 : _valueAt2.source;
3432
+ return typeof published === "object" && published !== null && "path" in published && published.path === ref;
3433
+ },
3434
+ pendingBytes: this.pendingBytesOf(fileLastUpdatedByPatchId)
3435
+ });
3436
+ }
3437
+
3438
+ /**
3439
+ * Is a remote file's bytes in a pending patch? `not-pending` when no patch
3440
+ * wrote it; `missing` when one did and its bytes are not there.
3441
+ */
3442
+ pendingBytesOf(fileLastUpdatedByPatchId) {
3443
+ return async ref => {
3444
+ const fromPatch = fileLastUpdatedByPatchId[ref];
3445
+ if (!fromPatch) {
3446
+ return "not-pending";
3447
+ }
3448
+ const bytes = await this.getBase64EncodedBinaryFileFromPatch(ref, fromPatch.patchId, fromPatch.remote);
3449
+ return bytes === null ? "missing" : "present";
3450
+ };
2719
3451
  }
2720
3452
 
2721
3453
  // #region validateFiles
@@ -2787,7 +3519,9 @@ class ValOps {
2787
3519
  if (metadataErrors && metadataErrors.length > 0) {
2788
3520
  return {
2789
3521
  [sourcePath]: metadataErrors.map(e => ({
2790
- message: e.message,
3522
+ // Which file: "File not found" on its own names nothing, and the
3523
+ // row it is shown on is the field, not the file.
3524
+ message: `${e.message}: ${filePath}`,
2791
3525
  value: {
2792
3526
  filePath,
2793
3527
  patchId: (fileData === null || fileData === void 0 ? void 0 : fileData.patchId) ?? null
@@ -3836,7 +4570,7 @@ class ValOps {
3836
4570
  * override it.
3837
4571
  */
3838
4572
  async saveBinaryFileFromPatch(filePath, parentRef, patchId, bytes, mimeType, type, metadata) {
3839
- return this.saveBase64EncodedBinaryFileFromPatch(filePath, parentRef, patchId, `data:${mimeType};base64,${bytes.toString("base64")}`, type, metadata);
4573
+ return this.saveBase64EncodedBinaryFileFromPatch(filePath, parentRef, patchId, `data:${mimeType};base64,${bytesToBase64(bytes)}`, type, metadata);
3840
4574
  }
3841
4575
  /**
3842
4576
  * A file's bytes as something a `Range` request can be answered from
@@ -3900,6 +4634,10 @@ class ValOps {
3900
4634
 
3901
4635
  // #endregion history
3902
4636
  }
4637
+ function isOnlyRemoteCheckValidationError(validationError) {
4638
+ const fixes = validationError.fixes ?? [];
4639
+ return fixes.length > 0 && fixes.every(f => f === "image:check-remote" || f === "file:check-remote");
4640
+ }
3903
4641
  function isOnlyFileCheckValidationError(validationError) {
3904
4642
  var _validationError$fixe9;
3905
4643
  if ((_validationError$fixe9 = validationError.fixes) !== null && _validationError$fixe9 !== void 0 && _validationError$fixe9.every(f => f === "file:check-metadata" || f === "image:check-metadata")) {
@@ -4058,15 +4796,16 @@ function createMetadataFromBuffer(type, mimeType, buffer) {
4058
4796
  metadata
4059
4797
  };
4060
4798
  }
4061
- function bufferFromDataUrl(dataUrl) {
4799
+ function bufferFromDataUrl(dataUrl, /** Node passes `Buffer`'s, which is faster: this is a whole upload. */
4800
+ decodeBase64 = base64ToBytes) {
4062
4801
  let base64Data;
4063
4802
  const base64Index = dataUrl.indexOf(";base64,");
4064
4803
  if (base64Index > -1) {
4065
4804
  base64Data = dataUrl.slice(base64Index + ";base64,".length);
4066
4805
  }
4067
4806
  if (base64Data) {
4068
- return buffer.Buffer.from(base64Data, "base64" // TODO: why does it not work with base64url?
4069
- );
4807
+ // TODO: why does it not work with base64url?
4808
+ return decodeBase64(base64Data);
4070
4809
  }
4071
4810
  }
4072
4811
 
@@ -4217,7 +4956,7 @@ async function resolveJsonEntriesAtCommit(ops, commitSha, affectedFiles) {
4217
4956
  continue;
4218
4957
  }
4219
4958
  try {
4220
- entries[gitPath] = JSON.parse(res.value.toString("utf-8"));
4959
+ entries[gitPath] = JSON.parse(bytesToUtf8(res.value));
4221
4960
  } catch (err) {
4222
4961
  failures.push({
4223
4962
  kind: "file-unavailable",
@@ -4539,7 +5278,14 @@ const SettingsSchema = z.z.object({
4539
5278
  publicProjectId: z.z.string(),
4540
5279
  remoteFileBuckets: z.z.array(z.z.object({
4541
5280
  bucket: z.z.string()
4542
- }))
5281
+ })),
5282
+ /**
5283
+ * The host a new remote file's ref names: the project's own address with
5284
+ * `/_val` for a Hosted project whose files are in R2, `remote.val.build`
5285
+ * otherwise. Absent from a content service that predates it, where every
5286
+ * project's files are on `remote.val.build`.
5287
+ */
5288
+ remoteHost: z.z.string().optional()
4543
5289
  });
4544
5290
  async function getSettings(projectName, auth, /** Overrides {@link defaultHost}. See the note there for why this exists. */
4545
5291
  contentUrl) {
@@ -4715,11 +5461,6 @@ function mapStatus(status) {
4715
5461
  return 500;
4716
5462
  }
4717
5463
 
4718
- /*
4719
- * Imported rather than global: under Node this is the built-in, and in a
4720
- * browser (the demo, `@valbuild/server/browser`) the `buffer` package.
4721
- */
4722
-
4723
5464
  /** What {@link ValServerOptions.publishOverride} is given. */
4724
5465
 
4725
5466
  /** What a publish reports back, in either shape. */
@@ -4730,6 +5471,18 @@ function mapStatus(status) {
4730
5471
  * have their own `dir` for any collection that wants somewhere else.
4731
5472
  */
4732
5473
  const FILES_DIRECTORY = "/public/val";
5474
+
5475
+ /**
5476
+ * `/validate` and `/validate/fix` on a platform without the full check (a
5477
+ * server in a browser tab). 501: nothing the caller does makes it available,
5478
+ * so the Studio offers no Try again.
5479
+ */
5480
+ const studioValidationUnavailable = {
5481
+ status: 501,
5482
+ json: {
5483
+ message: "Checking everything needs the project's files and Val's file hosts, and this Studio runs without them."
5484
+ }
5485
+ };
4733
5486
  const ValServer = (valModules, options, callbacks, platform) => {
4734
5487
  var _options$git;
4735
5488
  const {
@@ -5133,6 +5886,82 @@ const ValServer = (valModules, options, callbacks, platform) => {
5133
5886
  };
5134
5887
  }
5135
5888
  };
5889
+
5890
+ /**
5891
+ * What the Studio's full check reads: base plus the patches the caller is
5892
+ * showing, as `/sources/~` resolves an explicit `patch_id` list -- including
5893
+ * `draftOverlay`, so a module changed by a publish after this build is
5894
+ * checked as the caller sees it, not as this build has it.
5895
+ */
5896
+ const studioCheckInput = async (cookies, moduleFilePath, patchIds) => {
5897
+ const auth = getAuth(cookies);
5898
+ if (auth.error) {
5899
+ return {
5900
+ status: 401,
5901
+ json: {
5902
+ message: auth.error
5903
+ }
5904
+ };
5905
+ }
5906
+ if (isHttpOps(serverOps) && !("id" in auth)) {
5907
+ return {
5908
+ status: 401,
5909
+ json: {
5910
+ message: "Unauthorized"
5911
+ }
5912
+ };
5913
+ }
5914
+ const moduleErrors = await serverOps.getModuleErrors();
5915
+ if ((moduleErrors === null || moduleErrors === void 0 ? void 0 : moduleErrors.length) > 0) {
5916
+ return {
5917
+ status: 500,
5918
+ json: {
5919
+ message: "Val could not load this project's modules, so nothing can be checked. Check val.modules."
5920
+ }
5921
+ };
5922
+ }
5923
+ // An EXPLICITLY empty list is base, never "everything": `fetchPatches`
5924
+ // reads an empty filter as no filter (see `/sources/~`).
5925
+ let patchOps = patchIds !== undefined && patchIds.length === 0 ? {
5926
+ patches: []
5927
+ } : await serverOps.fetchPatches({
5928
+ patchIds,
5929
+ excludePatchOps: false
5930
+ });
5931
+ if ("error" in patchOps && patchOps.error) {
5932
+ return "unauthorized" in patchOps && patchOps.unauthorized ? {
5933
+ status: 401,
5934
+ json: {
5935
+ message: "Unauthorized"
5936
+ }
5937
+ } : {
5938
+ status: 500,
5939
+ json: {
5940
+ message: "The unpublished changes could not be read, so nothing was checked."
5941
+ }
5942
+ };
5943
+ }
5944
+ patchOps = {
5945
+ ...patchOps,
5946
+ patches: draftOverlay(patchOps.patches, patchOps.commits)
5947
+ };
5948
+ const analysis = serverOps.analyzePatches(patchOps.patches);
5949
+ const {
5950
+ sources
5951
+ } = await serverOps.getSourcesWithPatchesApplied({
5952
+ ...analysis,
5953
+ ...patchOps
5954
+ });
5955
+ return {
5956
+ status: "ok",
5957
+ input: {
5958
+ moduleFilePath,
5959
+ schemas: await serverOps.getSchemas(),
5960
+ sources,
5961
+ fileLastUpdatedByPatchId: analysis.fileLastUpdatedByPatchId
5962
+ }
5963
+ };
5964
+ };
5136
5965
  return {
5137
5966
  "/draft/enable": {
5138
5967
  GET: async req => {
@@ -5483,7 +6312,10 @@ const ValServer = (valModules, options, callbacks, platform) => {
5483
6312
  json: {
5484
6313
  publicProjectId: settingsRes.data.publicProjectId,
5485
6314
  remoteFileBuckets: settingsRes.data.remoteFileBuckets,
5486
- coreVersion: core.Internal.VERSION.core || "unknown"
6315
+ coreVersion: core.Internal.VERSION.core || "unknown",
6316
+ ...(settingsRes.data.remoteHost !== undefined ? {
6317
+ remoteHost: settingsRes.data.remoteHost
6318
+ } : {})
5487
6319
  }
5488
6320
  };
5489
6321
  }
@@ -5764,6 +6596,92 @@ const ValServer = (valModules, options, callbacks, platform) => {
5764
6596
  };
5765
6597
  }
5766
6598
  },
6599
+ "/validate": {
6600
+ POST: async req => {
6601
+ const checked = await studioCheckInput(req.cookies, req.body.moduleFilePath, req.body.patchIds);
6602
+ if (checked.status !== "ok") {
6603
+ return checked;
6604
+ }
6605
+ if (!platform.studioValidation) {
6606
+ return studioValidationUnavailable;
6607
+ }
6608
+ const res = await platform.studioValidation.checkModuleForStudio(serverOps, checked.input);
6609
+ switch (res.status) {
6610
+ case "unknown-module":
6611
+ return {
6612
+ status: 404,
6613
+ json: {
6614
+ message: `There is no module '${req.body.moduleFilePath}' in this project.`
6615
+ }
6616
+ };
6617
+ case "invalid-source":
6618
+ return {
6619
+ status: 200,
6620
+ json: {
6621
+ moduleFilePath: req.body.moduleFilePath,
6622
+ errors: {},
6623
+ invalidSource: res.message
6624
+ }
6625
+ };
6626
+ case "ok":
6627
+ return {
6628
+ status: 200,
6629
+ json: {
6630
+ moduleFilePath: req.body.moduleFilePath,
6631
+ errors: res.errors,
6632
+ remoteHosts: res.remoteHosts
6633
+ }
6634
+ };
6635
+ }
6636
+ }
6637
+ },
6638
+ "/validate/fix": {
6639
+ POST: async req => {
6640
+ const checked = await studioCheckInput(req.cookies, req.body.moduleFilePath, req.body.patchIds);
6641
+ if (checked.status !== "ok") {
6642
+ return checked;
6643
+ }
6644
+ if (!platform.studioValidation) {
6645
+ return studioValidationUnavailable;
6646
+ }
6647
+ const res = await platform.studioValidation.fixForStudio(serverOps, {
6648
+ ...checked.input,
6649
+ sourcePath: req.body.sourcePath,
6650
+ fix: req.body.fix
6651
+ });
6652
+ switch (res.status) {
6653
+ case "unknown-module":
6654
+ return {
6655
+ status: 404,
6656
+ json: {
6657
+ message: `There is no module '${req.body.moduleFilePath}' in this project.`
6658
+ }
6659
+ };
6660
+ case "not-found":
6661
+ return {
6662
+ status: 404,
6663
+ json: {
6664
+ message: "This error is no longer there. Check the module again to see what is left."
6665
+ }
6666
+ };
6667
+ case "not-in-studio":
6668
+ return {
6669
+ status: 400,
6670
+ json: {
6671
+ message: "This fix cannot be made from the Studio yet. Run `val validate --fix` in the project."
6672
+ }
6673
+ };
6674
+ case "ok":
6675
+ return {
6676
+ status: 200,
6677
+ json: {
6678
+ patch: res.patch,
6679
+ remainingErrors: res.remainingErrors
6680
+ }
6681
+ };
6682
+ }
6683
+ }
6684
+ },
5767
6685
  "/external-urls/check": {
5768
6686
  POST: async req => {
5769
6687
  /*
@@ -6437,8 +7355,9 @@ const ValServer = (valModules, options, callbacks, platform) => {
6437
7355
  // TODO: send binary files validation errors
6438
7356
  if (query.validate_binary_files) {
6439
7357
  await serverOps.validateFiles(schemas, sourcesRes.sources, sourcesValidation.files);
6440
- // TODO : actually do something with this
6441
- await serverOps.validateRemoteFiles(schemas, sourcesRes.sources, sourcesValidation.remoteFiles);
7358
+ // Remote files are not checked here: their check asks each host
7359
+ // over the network, so it is the Studio's full check that runs it,
7360
+ // one module at a time (`/validate`).
6442
7361
  }
6443
7362
  }
6444
7363
  const schemaSha = await serverOps.getSchemaSha();
@@ -6586,7 +7505,7 @@ const ValServer = (valModules, options, callbacks, platform) => {
6586
7505
  withJob: prepared,
6587
7506
  jobPatchIds: patchIds
6588
7507
  });
6589
- const build = serverOps.embedsSource() ? await buildSourcesOf(serverOps, patchIds, commit) : null;
7508
+ const build = serverOps.embedsSource() ? await buildSourcesOf(platform, serverOps, patchIds, commit) : null;
6590
7509
  if (build !== null && "status" in build) return build;
6591
7510
  /*
6592
7511
  * A build of no commit says what Val source it holds instead, so
@@ -6618,7 +7537,7 @@ const ValServer = (valModules, options, callbacks, platform) => {
6618
7537
  buildable: build !== null
6619
7538
  });
6620
7539
  }
6621
- const built = await buildSourcesOf(serverOps, patchIds, commit);
7540
+ const built = await buildSourcesOf(platform, serverOps, patchIds, commit);
6622
7541
  if ("status" in built) return built;
6623
7542
  const {
6624
7543
  split
@@ -6688,7 +7607,7 @@ const ValServer = (valModules, options, callbacks, platform) => {
6688
7607
  * and the selection on top. Only what the selection touched is saved,
6689
7608
  * so the proposal's save is the selection and nothing else.
6690
7609
  */
6691
- const built = await buildSourcesOf(serverOps, patchIds, commit, {
7610
+ const built = await buildSourcesOf(platform, serverOps, patchIds, commit, {
6692
7611
  binaries: false
6693
7612
  });
6694
7613
  if ("status" in built) return built;
@@ -7233,7 +8152,7 @@ const ValServer = (valModules, options, callbacks, platform) => {
7233
8152
  * patch's to read. See `binaryFiles` on the route.
7234
8153
  */
7235
8154
  const managedBranch = serverOps.sourceMode() === "managed" ? serverOps.projectBranch() : null;
7236
- const committedBinaries = serverOps.sourceMode() === "managed" ? await readCommittedBinaryFiles(serverOps, preparedCommit.patchedBinaryFilesDescriptors) : null;
8155
+ const committedBinaries = serverOps.sourceMode() === "managed" ? await readCommittedBinaryFiles(serverOps, preparedCommit.patchedBinaryFilesDescriptors, platform.bytesToBase64) : null;
7237
8156
  const commitToGit = () => serverOps.commit(preparedCommit, message, auth.id, FILES_DIRECTORY, undefined);
7238
8157
  /*
7239
8158
  * A host may publish somewhere other than the repository.
@@ -7893,7 +8812,7 @@ const ValServer = (valModules, options, callbacks, platform) => {
7893
8812
  // wrap it in a data: URL, and hand that to a method whose first
7894
8813
  // act was to unwrap it again - ceremony to satisfy a convention
7895
8814
  // that no longer exists.
7896
- const bytes = buffer.Buffer.from(await binaryRes.arrayBuffer());
8815
+ const bytes = new Uint8Array(await binaryRes.arrayBuffer());
7897
8816
  const type = file.metadata.mimeType.startsWith("image/") ? "image" : "file";
7898
8817
  const saveRes = await serverOps.saveBinaryFileFromPatch(file.filePath, req.body.parentRef, req.body.patchId, bytes, file.metadata.mimeType, type, file.metadata);
7899
8818
  if (saveRes.error) {
@@ -8139,7 +9058,7 @@ const ValServer = (valModules, options, callbacks, platform) => {
8139
9058
  // TypeScript compiler. See `patch/ts/publishKit.ts`.
8140
9059
  const {
8141
9060
  getJsonEntryAtCommit
8142
- } = await Promise.resolve().then(function () { return require('./getJsonEntryAtCommit-e96beb9e.cjs.dev.js'); });
9061
+ } = await Promise.resolve().then(function () { return require('./getJsonEntryAtCommit-64f67a50.cjs.dev.js'); });
8143
9062
  const res = await getJsonEntryAtCommit(serverOps, req.query.commit_sha, req.query.path, req.query.key);
8144
9063
  if (fp.result.isErr(res)) {
8145
9064
  return historyErrorResponse(res.error);
@@ -8247,11 +9166,11 @@ const ValServer = (valModules, options, callbacks, platform) => {
8247
9166
  // A DRAFT playlist names files that are drafts too, at URLs that only
8248
9167
  // resolve once published. See `rewriteDraftPlaylist`. Playlists are
8249
9168
  // small, so one is read whole to be rewritten.
8250
- const file = patchId && isHlsPlaylistPath(filePath) ? bufferReader(buffer.Buffer.from(rewriteDraftPlaylist((await opened.read(0, opened.size - 1)).toString("utf-8"), {
9169
+ const file = patchId && isHlsPlaylistPath(filePath) ? bufferReader(utf8ToBytes(rewriteDraftPlaylist(bytesToUtf8(await opened.read(0, opened.size - 1)), {
8251
9170
  playlistPath: filePath,
8252
9171
  patchId,
8253
9172
  remote
8254
- }), "utf-8")) : opened;
9173
+ }))) : opened;
8255
9174
  const headers = {
8256
9175
  // TODO: we could use ETag and return 304 instead
8257
9176
  // From the extension, published or draft: a `<video>` and an HLS
@@ -8291,7 +9210,7 @@ const ValServer = (valModules, options, callbacks, platform) => {
8291
9210
  body: bufferToReadableStream(slice)
8292
9211
  };
8293
9212
  }
8294
- const body = file.size === 0 ? buffer.Buffer.alloc(0) : await file.read(0, file.size - 1);
9213
+ const body = file.size === 0 ? new Uint8Array(0) : await file.read(0, file.size - 1);
8295
9214
  return {
8296
9215
  status: 200,
8297
9216
  headers: {
@@ -8353,7 +9272,7 @@ function verifyCallbackReq(stateCookie, queryParams) {
8353
9272
  }
8354
9273
  function getStateFromCookie(stateCookie) {
8355
9274
  try {
8356
- const decoded = buffer.Buffer.from(stateCookie, "base64").toString("utf8");
9275
+ const decoded = bytesToUtf8(base64ToBytes(stateCookie));
8357
9276
  const parsed = JSON.parse(decoded);
8358
9277
  if (!parsed) {
8359
9278
  return {
@@ -8405,7 +9324,7 @@ function getStateFromCookie(stateCookie) {
8405
9324
  }
8406
9325
  }
8407
9326
  function createStateCookie(state) {
8408
- return buffer.Buffer.from(JSON.stringify(state), "utf8").toString("base64");
9327
+ return bytesToBase64(utf8ToBytes(JSON.stringify(state)));
8409
9328
  }
8410
9329
  const ValAppJwtPayload = z.z.object({
8411
9330
  sub: z.z.string(),
@@ -8688,7 +9607,7 @@ function getIsRemoteRequired(schemas) {
8688
9607
  * `split.archive` is the managed job's archive; a connected job's archive is
8689
9608
  * the push's, made from its own changes alone, by the caller.
8690
9609
  */
8691
- async function buildSourcesOf(serverOps, patchIds, commit,
9610
+ async function buildSourcesOf(platform, serverOps, patchIds, commit,
8692
9611
  /**
8693
9612
  * Read the job's binary files back for a build. Off for a caller that only
8694
9613
  * commits what was prepared, which names the files rather than carrying them.
@@ -8754,7 +9673,7 @@ options = {
8754
9673
  jobPatchIds: patchIds
8755
9674
  });
8756
9675
  // Read now, while the job's changes still hold their files.
8757
- const binaries = options.binaries ? await readCommittedBinaryFiles(serverOps, split.archive.patchedBinaryFilesDescriptors) : {
9676
+ const binaries = options.binaries ? await readCommittedBinaryFiles(serverOps, split.archive.patchedBinaryFilesDescriptors, platform.bytesToBase64) : {
8758
9677
  files: {},
8759
9678
  unread: []
8760
9679
  };
@@ -8869,11 +9788,6 @@ async function uploadRemoteFile(contentHost, project, bucket, fileHash, fileExt,
8869
9788
  };
8870
9789
  }
8871
9790
 
8872
- function getFileExt(filePath) {
8873
- // NOTE: We do not import the path module. This code is copied in different projects. We want the same implementation and which means that this might running in browser where path is not available).
8874
- return filePath.split(".").pop() || "";
8875
- }
8876
-
8877
9791
  /**
8878
9792
  * One stored patch. Everything the ordered chain needs and nothing else.
8879
9793
  */
@@ -9574,7 +10488,11 @@ function createValRequestHandler(route, valServerPromise, convert, serveStatic)
9574
10488
  const method = (_req$method = req.method) === null || _req$method === void 0 ? void 0 : _req$method.toUpperCase();
9575
10489
  let route = null;
9576
10490
  let path = undefined;
9577
- for (const routeDef of Object.keys(internal.Api)) {
10491
+ // An exact match wins over every prefix match, wherever it sits in
10492
+ // `Api`: checked per key, in order, `/validate` took `/validate/fix`
10493
+ // because it was declared first.
10494
+ const exactRouteDef = Object.keys(internal.Api).find(routeDef => routeDef === reqApiRoutePath);
10495
+ for (const routeDef of exactRouteDef ? [exactRouteDef] : Object.keys(internal.Api)) {
9578
10496
  if (routeDef === reqApiRoutePath) {
9579
10497
  route = routeDef;
9580
10498
  break;
@@ -9873,10 +10791,16 @@ exports.InMemoryPatchStore = InMemoryPatchStore;
9873
10791
  exports.ValOps = ValOps;
9874
10792
  exports.ValOpsMemory = ValOpsMemory;
9875
10793
  exports.ValServer = ValServer;
10794
+ exports.askOncePerHost = askOncePerHost;
9876
10795
  exports.bufferFromDataUrl = bufferFromDataUrl;
10796
+ exports.bytesToBase64 = bytesToBase64;
10797
+ exports.bytesToUtf8 = bytesToUtf8;
10798
+ exports.checkRemoteFiles = checkRemoteFiles;
9877
10799
  exports.classifyJsonValuesOp = classifyJsonValuesOp;
9878
10800
  exports.createMetadataFromBuffer = createMetadataFromBuffer;
9879
10801
  exports.createValRequestHandler = createValRequestHandler;
10802
+ exports.fallbackRefOf = fallbackRefOf;
10803
+ exports.fallbackRemoteHost = fallbackRemoteHost;
9880
10804
  exports.findNestedJsonValuesRecords = findNestedJsonValuesRecords;
9881
10805
  exports.formatPatchSourceError = formatPatchSourceError;
9882
10806
  exports.getFieldsForType = getFieldsForType;
@@ -9888,10 +10812,13 @@ exports.isHlsMasterPlaylist = isHlsMasterPlaylist;
9888
10812
  exports.isJsonValuesRootOp = isJsonValuesRootOp;
9889
10813
  exports.largestHlsVariant = largestHlsVariant;
9890
10814
  exports.mapHlsUris = mapHlsUris;
10815
+ exports.movedRemoteHost = movedRemoteHost;
10816
+ exports.needsHostCheck = needsHostCheck;
9891
10817
  exports.parseHlsMasterVariants = parseHlsMasterVariants;
9892
10818
  exports.posixPath = posixPath;
9893
10819
  exports.rebaseContentOp = rebaseContentOp;
9894
10820
  exports.resolveExistingJsonPath = resolveExistingJsonPath;
10821
+ exports.rewriteRefHost = rewriteRefHost;
9895
10822
  exports.sumHlsSegmentDurations = sumHlsSegmentDurations;
9896
10823
  exports.uploadRemoteFile = uploadRemoteFile;
9897
10824
  exports.validateJsonValuesEntries = validateJsonValuesEntries;