@valbuild/server 0.147.0 → 0.148.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. package/CHANGELOG.md +108 -0
  2. package/browser/dist/valbuild-server-browser.cjs.dev.js +2 -2
  3. package/browser/dist/valbuild-server-browser.cjs.prod.js +2 -2
  4. package/browser/dist/valbuild-server-browser.esm.js +2 -2
  5. package/dist/{ValRouterCore-7ffcc08c.cjs.prod.js → ValRouterCore-42772c21.cjs.prod.js} +980 -53
  6. package/dist/{ValRouterCore-f5004f8f.esm.js → ValRouterCore-7ca3dc68.esm.js} +972 -54
  7. package/dist/{ValRouterCore-3fd3d444.cjs.dev.js → ValRouterCore-9bdaaf89.cjs.dev.js} +980 -53
  8. package/dist/declarations/src/ValOps.d.ts +32 -11
  9. package/dist/declarations/src/ValOpsMemory.d.ts +4 -4
  10. package/dist/declarations/src/ValServer.d.ts +1 -2
  11. package/dist/declarations/src/checkRemoteRef.d.ts +40 -2
  12. package/dist/declarations/src/createFixPatch.d.ts +25 -3
  13. package/dist/declarations/src/fixFiles.d.ts +42 -0
  14. package/dist/declarations/src/fixHandlers.d.ts +14 -0
  15. package/dist/declarations/src/getSettings.d.ts +1 -0
  16. package/dist/declarations/src/index.d.ts +1 -0
  17. package/dist/declarations/src/platform.d.ts +16 -0
  18. package/dist/declarations/src/remoteFileCheck.d.ts +116 -0
  19. package/dist/declarations/src/remoteHostCheck.d.ts +101 -0
  20. package/dist/declarations/src/studioValidation.d.ts +84 -0
  21. package/dist/declarations/src/uploadRemoteFile.d.ts +1 -1
  22. package/dist/declarations/src/videosetFixes.d.ts +3 -2
  23. package/dist/{getJsonEntryAtCommit-df65d861.cjs.prod.js → getJsonEntryAtCommit-2e36e805.cjs.prod.js} +3 -4
  24. package/dist/{getJsonEntryAtCommit-1e08d724.esm.js → getJsonEntryAtCommit-40b6d9c1.esm.js} +3 -4
  25. package/dist/{getJsonEntryAtCommit-e96beb9e.cjs.dev.js → getJsonEntryAtCommit-64f67a50.cjs.dev.js} +3 -4
  26. package/dist/valbuild-server.cjs.dev.js +1352 -695
  27. package/dist/valbuild-server.cjs.prod.js +1352 -695
  28. package/dist/valbuild-server.esm.js +1352 -696
  29. package/package.json +4 -5
@@ -1,4 +1,3 @@
1
- import { Buffer } from 'buffer';
2
1
  import { Internal, VAL_EXTENSION, extractValModules, computeValModuleShas, ImageSchema, DEFAULT_CONTENT_HOST, hasRemoteFileSchema } from '@valbuild/core';
3
2
  import { isSchemaSourceFixError, resolveSchemaSourceFixForError, SerializedSchema, VAL_ENABLE_COOKIE_NAME, VAL_STATE_COOKIE, VAL_SESSION_COOKIE, newestCommitSha, parseJob, VAL_STUDIO_HEADER, getErrorMessageFromUnknownJson, Api } from '@valbuild/shared/internal';
4
3
  import { z } from 'zod';
@@ -922,6 +921,118 @@ async function validateJsonValuesEntries(schema, source, modulePath) {
922
921
  return res;
923
922
  }
924
923
 
924
+ /**
925
+ * Bytes as text and back, without Node's `Buffer`.
926
+ *
927
+ * The request handlers run in a browser tab too (`@valbuild/server/browser`),
928
+ * and a browser has no `Buffer`. Importing npm's `buffer` instead is what broke
929
+ * the demo under `vite dev`: it is CommonJS, and a dependency the dev server
930
+ * did not see in its scan is served unconverted. So bytes are `Uint8Array`
931
+ * here, and these are the conversions `Buffer` used to do -- with the same
932
+ * answers, which `bytes.test.ts` checks against `Buffer` itself.
933
+ */
934
+
935
+ /**
936
+ * The value of each character code, or -1 for one that is not base64. Both
937
+ * alphabets, because `Buffer.from(s, "base64")` takes either: `-` and `_` are
938
+ * the URL-safe spellings of `+` and `/`.
939
+ */
940
+ const BASE64_VALUES = (() => {
941
+ const BASE64_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
942
+ const values = new Int16Array(128).fill(-1);
943
+ for (let i = 0; i < BASE64_ALPHABET.length; i++) {
944
+ values[BASE64_ALPHABET.charCodeAt(i)] = i;
945
+ }
946
+ values["-".charCodeAt(0)] = 62;
947
+ values["_".charCodeAt(0)] = 63;
948
+ return values;
949
+ })();
950
+ const PADDING = "=".charCodeAt(0);
951
+
952
+ /**
953
+ * Decodes base64 the way `Buffer.from(base64, "base64")` does: either
954
+ * alphabet, padding optional, anything that is not base64 skipped, and
955
+ * decoding stops at the first `=`. A trailing partial group gives the whole
956
+ * bytes it holds.
957
+ *
958
+ * `atob` first, because it is native and this is called with whole uploads;
959
+ * it agrees with `Buffer` on everything it accepts, and refuses the rest (the
960
+ * URL-safe alphabet, stray characters, `=` in the middle), which then goes the
961
+ * slow way.
962
+ */
963
+ function base64ToBytes(base64) {
964
+ let binary;
965
+ try {
966
+ binary = atob(base64);
967
+ } catch {
968
+ return lenientBase64ToBytes(base64);
969
+ }
970
+ const bytes = new Uint8Array(binary.length);
971
+ for (let i = 0; i < binary.length; i++) {
972
+ bytes[i] = binary.charCodeAt(i);
973
+ }
974
+ return bytes;
975
+ }
976
+ function lenientBase64ToBytes(base64) {
977
+ const bytes = new Uint8Array(Math.floor(base64.length * 3 / 4));
978
+ let length = 0;
979
+ let bits = 0;
980
+ let bitCount = 0;
981
+ for (let i = 0; i < base64.length; i++) {
982
+ const code = base64.charCodeAt(i);
983
+ if (code === PADDING) break;
984
+ const value = code < 128 ? BASE64_VALUES[code] : -1;
985
+ if (value < 0) continue;
986
+ bits = (bits << 6 | value) & 0xffffff;
987
+ bitCount += 6;
988
+ if (bitCount >= 8) {
989
+ bitCount -= 8;
990
+ bytes[length++] = bits >> bitCount & 0xff;
991
+ }
992
+ }
993
+ return bytes.subarray(0, length);
994
+ }
995
+
996
+ /**
997
+ * How many bytes go to `String.fromCharCode` at once: it takes them as
998
+ * arguments, and an engine has a limit on how many a call may have.
999
+ */
1000
+ const CHUNK = 0x8000;
1001
+
1002
+ /**
1003
+ * `Buffer.from(bytes).toString("base64")`: padded, standard alphabet.
1004
+ *
1005
+ * Through `btoa`, which is native, rather than building the string here: that
1006
+ * was five times slower, and this is called with whole files. Node is still
1007
+ * faster with `Buffer`, which is why `ValServerPlatform.bytesToBase64` exists
1008
+ * for the paths that encode every file of a publish.
1009
+ */
1010
+ function bytesToBase64(bytes) {
1011
+ const binary = [];
1012
+ for (let i = 0; i < bytes.length; i += CHUNK) {
1013
+ // `Reflect.apply` rather than a spread, which copies the chunk into an
1014
+ // argument list first and is four times slower.
1015
+ const chunk = Reflect.apply(String.fromCharCode, null, bytes.subarray(i, i + CHUNK));
1016
+ binary.push(chunk);
1017
+ }
1018
+ return btoa(binary.join(""));
1019
+ }
1020
+
1021
+ /** `Buffer.from(text, "utf8")`. */
1022
+ function utf8ToBytes(text) {
1023
+ return new TextEncoder().encode(text);
1024
+ }
1025
+
1026
+ /**
1027
+ * `Buffer.from(bytes).toString("utf8")`. `ignoreBOM` because `Buffer` keeps a
1028
+ * leading byte order mark, where a `TextDecoder` drops it by default.
1029
+ */
1030
+ function bytesToUtf8(bytes) {
1031
+ return new TextDecoder("utf-8", {
1032
+ ignoreBOM: true
1033
+ }).decode(bytes);
1034
+ }
1035
+
925
1036
  /**
926
1037
  * Does saving this commit upload a file to Val's remote host?
927
1038
  *
@@ -1336,7 +1447,8 @@ function isHlsPlaylistPath(filePath) {
1336
1447
  * patch's to read. Generic over the patch id so it asks for exactly the one
1337
1448
  * method it uses, and a test can hand it a plain object.
1338
1449
  */
1339
- async function readCommittedBinaryFiles(ops, descriptors) {
1450
+ async function readCommittedBinaryFiles(ops, descriptors, /** `ValServerPlatform.bytesToBase64`, which is faster in Node. */
1451
+ encode = bytesToBase64) {
1340
1452
  const files = {};
1341
1453
  const unread = [];
1342
1454
  await Promise.all(Object.entries(descriptors).map(async ([filePath, {
@@ -1348,7 +1460,7 @@ async function readCommittedBinaryFiles(ops, descriptors) {
1348
1460
  console.error("Could not read a committed file", filePath, error);
1349
1461
  return null;
1350
1462
  });
1351
- if (bytes === null) unread.push(filePath);else files[filePath] = bytes.toString("base64");
1463
+ if (bytes === null) unread.push(filePath);else files[filePath] = encode(bytes);
1352
1464
  }));
1353
1465
  return {
1354
1466
  files,
@@ -1515,10 +1627,537 @@ function computePatchesToDrop(prepared) {
1515
1627
  return dropped;
1516
1628
  }
1517
1629
 
1518
- /*
1519
- * Imported rather than global: under Node this is the built-in, and in a
1520
- * browser (the demo, `@valbuild/server/browser`) the `buffer` package.
1630
+ function getFileExt(filePath) {
1631
+ // NOTE: We do not import the path module. This code is copied in different projects. We want the same implementation and which means that this might running in browser where path is not available).
1632
+ return filePath.split(".").pop() || "";
1633
+ }
1634
+
1635
+ /**
1636
+ * Does the host a platform ref names still serve the project the ref names?
1637
+ *
1638
+ * A ref served by the platform loader names the project's host at the time
1639
+ * the file was added: `https://www.acme.no/_val/file/p/3f9a2c1/…`. When the
1640
+ * project drops `www.acme.no`, the bytes are still there, at
1641
+ * `3f9a2c1.valstart.dev`, but every ref that names the old domain points at a
1642
+ * host that no longer serves them. This is the question that finds those refs,
1643
+ * and `rewriteRefHost` is what the fix (`remote:rewrite-host`) does about it.
1644
+ * See valbuild/home `docs/remote-files-on-platform.md`, "Rewriting refs after a
1645
+ * domain is removed", and `docs/plans/studio-validate.md`, step 6.
1646
+ *
1647
+ * Asked of the host itself (`GET <host>/_val/project`), not of content: the
1648
+ * host's answer covers DNS, the certificate and the loader's mapping in one,
1649
+ * it needs no credential, and it works for a ref that names a project other
1650
+ * than this one -- a project seeded from a template has refs naming the
1651
+ * template's.
1652
+ */
1653
+
1654
+ /** The domain every platform project is always served at, by its public id. */
1655
+ const PLATFORM_SITES_DOMAIN = "valstart.dev";
1656
+
1657
+ /** What a platform ref's host part ends with. */
1658
+ const PLATFORM_PREFIX = "/_val";
1659
+
1660
+ /**
1661
+ * The host every platform ref of `publicProjectId` can always be pointed at:
1662
+ * `<public id>.valstart.dev` is permanent (decision 4 of the remote-files
1663
+ * plan), and the loader serves a project's files there whatever else it has.
1664
+ */
1665
+ function fallbackRemoteHost(publicProjectId) {
1666
+ return `https://${publicProjectId}.${PLATFORM_SITES_DOMAIN}${PLATFORM_PREFIX}`;
1667
+ }
1668
+
1669
+ /**
1670
+ * Only a platform ref is asked about. `remote.val.build` is not a platform
1671
+ * host and is not going away, and a ref already on the project's own
1672
+ * `valstart.dev` address is on the fallback, which always serves it.
1673
+ */
1674
+ function needsHostCheck(remoteHost, projectId) {
1675
+ return remoteHost.endsWith(PLATFORM_PREFIX) && remoteHost !== fallbackRemoteHost(projectId);
1676
+ }
1677
+
1678
+ /** `ref` with its host part replaced, and everything else as it was. */
1679
+ function rewriteRefHost(ref, remoteHost) {
1680
+ const parsed = Internal.remote.splitRemoteRef(ref);
1681
+ if (parsed.status === "error") {
1682
+ return null;
1683
+ }
1684
+ return Internal.remote.createRemoteRef(remoteHost, {
1685
+ publicProjectId: parsed.projectId,
1686
+ coreVersion: parsed.version,
1687
+ bucket: parsed.bucket,
1688
+ validationHash: parsed.validationHash,
1689
+ fileHash: parsed.fileHash,
1690
+ filePath: parsed.filePath
1691
+ });
1692
+ }
1693
+
1694
+ /**
1695
+ * - `serves`: the host answered for the project the ref names.
1696
+ * - `other-project`: it answered, for another one -- the domain was moved to
1697
+ * a different project, and decision 5 means it will not serve these files.
1698
+ * - `no-site`: it answered, but serves no project at all (the loader's 404 for
1699
+ * a domain nobody has mapped any more).
1700
+ * - `unreachable`: no answer. A domain whose DNS was removed looks like this,
1701
+ * and so does a host that is down for a minute, which is why the row says
1702
+ * which it saw.
1703
+ */
1704
+
1705
+ const ASK_TIMEOUT_MS = 8_000;
1706
+
1707
+ /** `GET <host>/_val/project`, answered by the platform loader. */
1708
+ const askHostProject = async remoteHost => {
1709
+ try {
1710
+ const res = await fetch(`${remoteHost}/project`, {
1711
+ signal: AbortSignal.timeout(ASK_TIMEOUT_MS)
1712
+ });
1713
+ if (!res.ok) {
1714
+ return {
1715
+ status: "no-site",
1716
+ httpStatus: res.status
1717
+ };
1718
+ }
1719
+ const body = await res.json();
1720
+ if (typeof body === "object" && body !== null && "publicProjectId" in body && typeof body.publicProjectId === "string") {
1721
+ return {
1722
+ status: "answered",
1723
+ publicProjectId: body.publicProjectId
1724
+ };
1725
+ }
1726
+ return {
1727
+ status: "no-site",
1728
+ httpStatus: res.status
1729
+ };
1730
+ } catch (err) {
1731
+ return {
1732
+ status: "unreachable",
1733
+ reason: err instanceof Error && err.name === "TimeoutError" ? `No answer within ${ASK_TIMEOUT_MS / 1000} s` : err instanceof Error ? err.message : "Unknown error"
1734
+ };
1735
+ }
1736
+ };
1737
+
1738
+ /**
1739
+ * Hosts that answered for a project, and when: one question per host, not per
1740
+ * ref, and not again for a few minutes. Only a "serves" is kept, for the same
1741
+ * reason `remoteFileCheck` keeps only "found": a host that stopped serving has
1742
+ * to be noticed, and one that did not answer has to be asked again.
1743
+ */
1744
+ const SERVES_FOR_MS = 5 * 60_000;
1745
+ const servesAt = new Map();
1746
+ async function hostServes(remoteHost, projectId, ask = askHostProject) {
1747
+ const key = `${remoteHost} ${projectId}`;
1748
+ const at = servesAt.get(key);
1749
+ if (at !== undefined && Date.now() - at < SERVES_FOR_MS) {
1750
+ return {
1751
+ status: "serves"
1752
+ };
1753
+ }
1754
+ const answer = await ask(remoteHost);
1755
+ if (answer.status !== "answered") {
1756
+ return answer;
1757
+ }
1758
+ if (answer.publicProjectId !== projectId) {
1759
+ return {
1760
+ status: "other-project",
1761
+ publicProjectId: answer.publicProjectId
1762
+ };
1763
+ }
1764
+ servesAt.set(key, Date.now());
1765
+ return {
1766
+ status: "serves"
1767
+ };
1768
+ }
1769
+
1770
+ /**
1771
+ * `ask`, asking each host once for as long as the returned function is kept:
1772
+ * one check, not the life of the server. A check walks its refs in order, so
1773
+ * without it every ref on a host that does not answer waits out its own
1774
+ * timeout -- 150 images on a dead domain is 20 minutes. Kept per check rather
1775
+ * than in {@link hostServes}'s cache because a host that did not answer has
1776
+ * to be asked again by the next one.
1777
+ */
1778
+ function askOncePerHost(ask = askHostProject) {
1779
+ const answers = new Map();
1780
+ return remoteHost => {
1781
+ let answer = answers.get(remoteHost);
1782
+ if (answer === undefined) {
1783
+ answer = ask(remoteHost);
1784
+ answers.set(remoteHost, answer);
1785
+ }
1786
+ return answer;
1787
+ };
1788
+ }
1789
+
1790
+ /** `www.acme.no`, from `https://www.acme.no/_val`: how a person names a host. */
1791
+ function hostNameOfRemoteHost(remoteHost) {
1792
+ try {
1793
+ return new URL(remoteHost).host;
1794
+ } catch {
1795
+ return remoteHost;
1796
+ }
1797
+ }
1798
+
1799
+ /**
1800
+ * Where a platform ref has to move to, when its host no longer serves the
1801
+ * project it names; `undefined` when it does not have to (or is not a
1802
+ * platform ref). `why` and `to` are for the caller's sentence: what can be
1803
+ * done about it differs by caller (a Studio **Fix**, `val validate --fix`, or
1804
+ * nothing yet).
1805
+ */
1806
+ async function movedRemoteHost(ref, ask) {
1807
+ const parsed = Internal.remote.splitRemoteRef(ref);
1808
+ if (parsed.status === "error" || !needsHostCheck(parsed.remoteHost, parsed.projectId)) {
1809
+ return undefined;
1810
+ }
1811
+ const serves = await hostServes(parsed.remoteHost, parsed.projectId, ask);
1812
+ // Only an answer that says so moves a ref: the host serves another
1813
+ // project, or the loader says it serves none (its 404 for a domain nobody
1814
+ // maps). A host that did not answer, or answered 5xx, may be down for a
1815
+ // minute -- and a moved ref does not move back, so rewriting on that would
1816
+ // take a site's images off its own domain for good over one blip. Those are
1817
+ // left to the existence check, which reports the host once with Try again.
1818
+ if (serves.status === "serves" || serves.status === "unreachable" || serves.status === "no-site" && serves.httpStatus !== 404) {
1819
+ return undefined;
1820
+ }
1821
+ const target = fallbackRemoteHost(parsed.projectId);
1822
+ const moved = rewriteRefHost(ref, target);
1823
+ if (moved === null) {
1824
+ return undefined;
1825
+ }
1826
+ const from = hostNameOfRemoteHost(parsed.remoteHost);
1827
+ return {
1828
+ ref: moved,
1829
+ why: serves.status === "other-project" ? `${from} now serves another project, so this file is not shown from there` : `${from} no longer serves this project, so this file is not shown from there`,
1830
+ to: hostNameOfRemoteHost(target)
1831
+ };
1832
+ }
1833
+
1834
+ /**
1835
+ * Where else a ref's file can be downloaded from: the same file on the
1836
+ * project's own `valstart.dev` address, for a platform ref on any other host.
1837
+ * `null` when there is nowhere else -- a `remote.val.build` ref, or one already
1838
+ * on the fallback. See valbuild/home `docs/remote-files-on-platform.md`,
1839
+ * "Tooling: the fallback".
1521
1840
  */
1841
+ function fallbackRefOf(ref) {
1842
+ const parsed = Internal.remote.splitRemoteRef(ref);
1843
+ if (parsed.status === "error" || !needsHostCheck(parsed.remoteHost, parsed.projectId)) {
1844
+ return null;
1845
+ }
1846
+ return rewriteRefHost(ref, fallbackRemoteHost(parsed.projectId));
1847
+ }
1848
+
1849
+ /**
1850
+ * Does the host a remote ref names still serve it?
1851
+ *
1852
+ * `found` and `missing` are the host's answer. `unreachable` is no answer at
1853
+ * all — a timeout, a refused connection, a 5xx — and says nothing about the
1854
+ * file, only about the host.
1855
+ */
1856
+
1857
+ /**
1858
+ * How the remote check reaches the network: what it asks a host about a file,
1859
+ * and which project a platform host serves. Both default to the real thing;
1860
+ * tests replace them.
1861
+ */
1862
+
1863
+ const HEAD_TIMEOUT_MS = 8_000;
1864
+
1865
+ /**
1866
+ * A HEAD, falling back to the first byte of a GET for a host that does not
1867
+ * answer HEAD (405, 501): either way no file is downloaded to learn that it is
1868
+ * there.
1869
+ */
1870
+ const headRemoteFile = async url => {
1871
+ const ask = async init => fetch(url, {
1872
+ ...init,
1873
+ redirect: "follow",
1874
+ signal: AbortSignal.timeout(HEAD_TIMEOUT_MS)
1875
+ });
1876
+ try {
1877
+ let res = await ask({
1878
+ method: "HEAD"
1879
+ });
1880
+ if (res.status === 405 || res.status === 501) {
1881
+ var _res$body;
1882
+ res = await ask({
1883
+ method: "GET",
1884
+ headers: {
1885
+ Range: "bytes=0-0"
1886
+ }
1887
+ });
1888
+ await ((_res$body = res.body) === null || _res$body === void 0 ? void 0 : _res$body.cancel());
1889
+ }
1890
+ if (res.ok) {
1891
+ return {
1892
+ status: "found"
1893
+ };
1894
+ }
1895
+ if (res.status === 404 || res.status === 410) {
1896
+ return {
1897
+ status: "missing",
1898
+ httpStatus: res.status
1899
+ };
1900
+ }
1901
+ return {
1902
+ status: "unreachable",
1903
+ reason: `HTTP ${res.status}`
1904
+ };
1905
+ } catch (err) {
1906
+ return {
1907
+ status: "unreachable",
1908
+ reason: describeFetchError(err)
1909
+ };
1910
+ }
1911
+ };
1912
+ function describeFetchError(err) {
1913
+ if (!(err instanceof Error)) {
1914
+ return "Unknown error";
1915
+ }
1916
+ if (err.name === "TimeoutError") {
1917
+ return `No answer within ${HEAD_TIMEOUT_MS / 1000} s`;
1918
+ }
1919
+ // undici puts what actually went wrong (ENOTFOUND, ECONNREFUSED) on `cause`.
1920
+ const cause = err.cause;
1921
+ if (cause instanceof Error) {
1922
+ return `${err.message}: ${cause.message}`;
1923
+ }
1924
+ return err.message;
1925
+ }
1926
+
1927
+ /**
1928
+ * Refs a host has served, and when.
1929
+ *
1930
+ * So that pressing **Check again**, or opening a second module that uses the
1931
+ * same images, does not ask about every file again. Kept for minutes, not for
1932
+ * the life of the process: a ref is content-addressed, so its answer only
1933
+ * changes when its host stops serving it -- which is exactly what this check
1934
+ * is for, and a server that remembered "found" forever would never report a
1935
+ * host that went down after it was first asked. Only `found` is kept. A host
1936
+ * that did not answer is asked again on the next check, which is what makes
1937
+ * **Try again** mean something.
1938
+ */
1939
+ const FOUND_FOR_MS = 5 * 60_000;
1940
+ const foundAt = new Map();
1941
+ function wasFound(ref) {
1942
+ const at = foundAt.get(ref);
1943
+ return at !== undefined && Date.now() - at < FOUND_FOR_MS;
1944
+ }
1945
+ function markFound(ref) {
1946
+ foundAt.set(ref, Date.now());
1947
+ }
1948
+
1949
+ /** A host that did not serve the refs of a module it was asked about. */
1950
+
1951
+ const textEncoder = new TextEncoder();
1952
+
1953
+ /**
1954
+ * The remote half of the Studio's full check, for one module's remote values.
1955
+ *
1956
+ * Four questions, cheapest first, and a ref stops at the first one it fails:
1957
+ *
1958
+ * 1. **Is it a ref at all?** `splitRemoteRef` parses it.
1959
+ * 2. **Does the ref agree with what is recorded about it?** A ref carries a
1960
+ * hash of the schema's options, the file's extension and its width, height
1961
+ * and mime type. When the value or the schema changed and the ref did not,
1962
+ * the ref is stale, and **Fix** (`image:check-remote`) re-reads the file and
1963
+ * writes the ref it should have. Computed with the ref's OWN core version:
1964
+ * the CLI also re-derives every ref after a Val upgrade, and listing every
1965
+ * remote image in the project after each upgrade would bury the rows that
1966
+ * matter. Nothing here downloads.
1967
+ * 3. **Does its host still serve its project?** Only for a platform ref
1968
+ * (`…/_val/file/…`) that is not on the project's own `valstart.dev`
1969
+ * address: one `GET <host>/_val/project` per host. A host that dropped the
1970
+ * project gets a **Fix** (`remote:rewrite-host`) that points the ref at
1971
+ * `<public id>.valstart.dev`. See `remoteHostCheck.ts`.
1972
+ * 4. **Does the host have it?** A HEAD.
1973
+ *
1974
+ * Neither host question is asked of a ref whose bytes are in a pending patch,
1975
+ * which the host will not have until publish: there the question is whether
1976
+ * the patch still has them.
1977
+ */
1978
+ async function checkRemoteFiles(input) {
1979
+ const head = input.head ?? headRemoteFile;
1980
+ const askHostProject = askOncePerHost(input.askHostProject);
1981
+ const errors = {};
1982
+ const addError = (sourcePath, error) => {
1983
+ errors[sourcePath] = (errors[sourcePath] ?? []).concat(error);
1984
+ };
1985
+
1986
+ /** Refs still to ask a host about, by host. */
1987
+ const toAsk = new Map();
1988
+ const sorted = Object.entries(input.remoteFiles).sort(([a], [b]) => a.localeCompare(b));
1989
+ for (const [sourcePathS, value] of sorted) {
1990
+ const sourcePath = sourcePathS;
1991
+ const ref = value.path;
1992
+ const parsed = Internal.remote.splitRemoteRef(ref);
1993
+ if (parsed.status === "error") {
1994
+ addError(sourcePath, {
1995
+ message: `This is not an address Val can check: ${parsed.error}`,
1996
+ value
1997
+ });
1998
+ continue;
1999
+ }
2000
+ let noun = "file";
2001
+ if (!input.existenceOnly) {
2002
+ const schema = input.schemaAt(sourcePath);
2003
+ if ((schema === null || schema === void 0 ? void 0 : schema.type) !== "image" && (schema === null || schema === void 0 ? void 0 : schema.type) !== "file") {
2004
+ continue;
2005
+ }
2006
+ noun = schema.type;
2007
+ if (isStale(parsed, schema, value)) {
2008
+ addError(sourcePath, {
2009
+ message: `The address of this ${schema.type} does not match what is recorded about it, so it may be served with the wrong size or type.`,
2010
+ value,
2011
+ fixes: [schema.type === "image" ? "image:check-remote" : "file:check-remote"]
2012
+ });
2013
+ continue;
2014
+ }
2015
+ }
2016
+ const pending = await input.pendingBytes(ref);
2017
+ if (pending === "present") {
2018
+ continue;
2019
+ }
2020
+ if (pending === "missing") {
2021
+ addError(sourcePath, {
2022
+ message: `This ${noun} is in your unpublished changes, but its file is not. Choose the ${noun} again.`,
2023
+ value
2024
+ });
2025
+ continue;
2026
+ }
2027
+ // Asked after the pending check: a ref whose bytes are still in a patch
2028
+ // is the key those bytes are stored under, and repointing it would cut
2029
+ // it off from them.
2030
+ const moved = await movedRemoteHost(ref, askHostProject);
2031
+ if (moved) {
2032
+ addError(sourcePath, input.existenceOnly ? {
2033
+ // A directory's key: the fields picking from it hold the same
2034
+ // ref, and would have to follow. Not built yet.
2035
+ message: `${moved.why}. Pointing a file in a media directory at ${moved.to} is not something the Studio can do yet.`,
2036
+ value
2037
+ } : {
2038
+ message: `${moved.why}. Fix points it at ${moved.to}, which always serves it.`,
2039
+ value,
2040
+ fixes: ["remote:rewrite-host"]
2041
+ });
2042
+ continue;
2043
+ }
2044
+ if (wasFound(ref)) {
2045
+ continue;
2046
+ }
2047
+ const refs = toAsk.get(parsed.remoteHost) ?? [];
2048
+ refs.push({
2049
+ sourcePath,
2050
+ ref,
2051
+ published: input.isPublished(sourcePath, ref)
2052
+ });
2053
+ toAsk.set(parsed.remoteHost, refs);
2054
+ }
2055
+ const hosts = [];
2056
+ await Promise.all([...toAsk.entries()].map(async ([host, refs]) => {
2057
+ const problem = await askHost(host, refs, input.sampleByHost, head);
2058
+ if (problem.host) {
2059
+ hosts.push(problem.host);
2060
+ }
2061
+ for (const {
2062
+ sourcePath,
2063
+ error
2064
+ } of problem.refErrors) {
2065
+ addError(sourcePath, error);
2066
+ }
2067
+ }));
2068
+ hosts.sort((a, b) => a.host.localeCompare(b.host));
2069
+ return {
2070
+ errors,
2071
+ hosts
2072
+ };
2073
+ }
2074
+
2075
+ /**
2076
+ * One host's refs, asked one at a time.
2077
+ *
2078
+ * In order rather than all at once: a host that does not answer is reported
2079
+ * after the FIRST timeout, not after one timeout per image. A host that
2080
+ * answers "missing" for one ref is still asked about the next, since a file
2081
+ * that is gone is a fact about that file.
2082
+ */
2083
+ async function askHost(host, refs, sampleByHost, head) {
2084
+ const vouchable = sampleByHost ? refs.filter(r => r.published) : [];
2085
+ const individually = sampleByHost ? refs.filter(r => !r.published) : refs;
2086
+ const allPaths = refs.map(r => r.sourcePath);
2087
+ const refErrors = [];
2088
+ if (vouchable.length > 0) {
2089
+ const sample = vouchable[0];
2090
+ const answer = await head(sample.ref);
2091
+ if (answer.status === "found") {
2092
+ for (const r of vouchable) {
2093
+ markFound(r.ref);
2094
+ }
2095
+ } else {
2096
+ // In a managed project a published ref cannot break on its own, so the
2097
+ // sample's answer is the host's: one row, every ref on it.
2098
+ return {
2099
+ host: {
2100
+ host,
2101
+ status: answer.status,
2102
+ details: detailsOf(sample.ref, answer),
2103
+ sourcePaths: allPaths
2104
+ },
2105
+ refErrors
2106
+ };
2107
+ }
2108
+ }
2109
+ for (const r of individually) {
2110
+ const answer = await head(r.ref);
2111
+ if (answer.status === "found") {
2112
+ markFound(r.ref);
2113
+ } else if (answer.status === "missing") {
2114
+ refErrors.push({
2115
+ sourcePath: r.sourcePath,
2116
+ error: {
2117
+ message: `The file is not at its address: ${hostName(host)} answered ${answer.httpStatus}.`
2118
+ }
2119
+ });
2120
+ } else {
2121
+ return {
2122
+ host: {
2123
+ host,
2124
+ status: "unreachable",
2125
+ details: detailsOf(r.ref, answer),
2126
+ sourcePaths: allPaths
2127
+ },
2128
+ refErrors
2129
+ };
2130
+ }
2131
+ }
2132
+ return {
2133
+ refErrors
2134
+ };
2135
+ }
2136
+ function detailsOf(ref, answer) {
2137
+ return `HEAD ${ref}\n→ ${answer.status === "missing" ? `${answer.httpStatus}` : answer.status === "unreachable" ? answer.reason : "200"}`;
2138
+ }
2139
+
2140
+ /** `www.acme.no`, from `https://www.acme.no`: how a person names a host. */
2141
+ function hostName(host) {
2142
+ try {
2143
+ return new URL(host).host;
2144
+ } catch {
2145
+ return host;
2146
+ }
2147
+ }
2148
+ function isStale(parsed, schema, value) {
2149
+ // The hash covers only these of what was recorded (`getValidationBasis`):
2150
+ // `alt` and `hotspot` are the editor's, and change no byte.
2151
+ const metadata = {
2152
+ width: value.width,
2153
+ height: value.height,
2154
+ mimeType: value.mimeType
2155
+ };
2156
+ const expected = Internal.remote.getValidationHash(parsed.version, schema, getFileExt(parsed.filePath), metadata, parsed.fileHash, textEncoder);
2157
+ return expected !== parsed.validationHash;
2158
+ }
2159
+
2160
+ /* eslint-disable @typescript-eslint/no-unused-vars */
1522
2161
  const jsonOps = new JSONOps();
1523
2162
  /**
1524
2163
  * How many `.jsonValues()` entry files a whole-record write reads at a time.
@@ -2559,7 +3198,14 @@ class ValOps {
2559
3198
  }
2560
3199
 
2561
3200
  // #region validateSources
2562
- async validateSources(schemas, sources, patchesByModule) {
3201
+ async validateSources(schemas, sources, patchesByModule,
3202
+ /**
3203
+ * Validate only these modules. The snapshot the cross-module checks read
3204
+ * (a keyOf's keys, a route, a set's directory) is still built from every
3205
+ * module, which is what makes checking one module at a time give the same
3206
+ * answer for it as checking them all. Wins over `patchesByModule`.
3207
+ */
3208
+ onlyModules) {
2563
3209
  const errors = {};
2564
3210
  const files = {};
2565
3211
  const remoteFiles = {};
@@ -2592,7 +3238,18 @@ class ValOps {
2592
3238
  }
2593
3239
  }
2594
3240
  }
2595
- const modulePathsToValidate = patchesByModule && Object.keys(patchesByModule);
3241
+ const addModuleError = (path, sourcePath, validationError) => {
3242
+ if (!errors[path]) {
3243
+ errors[path] = {
3244
+ validations: {}
3245
+ };
3246
+ }
3247
+ if (!errors[path].validations[sourcePath]) {
3248
+ errors[path].validations[sourcePath] = [];
3249
+ }
3250
+ errors[path].validations[sourcePath].push(validationError);
3251
+ };
3252
+ const modulePathsToValidate = onlyModules ?? (patchesByModule && Object.keys(patchesByModule));
2596
3253
  for (const [pathS, schema] of entries) {
2597
3254
  if (modulePathsToValidate && !modulePathsToValidate.includes(pathS)) {
2598
3255
  continue;
@@ -2621,15 +3278,35 @@ class ValOps {
2621
3278
  } = await validateJsonValuesEntries(schema, source, path);
2622
3279
  for (const [sourcePathS, entryErrors] of Object.entries(jsonValuesErrors)) {
2623
3280
  const sourcePath = sourcePathS;
2624
- if (!errors[path]) {
2625
- errors[path] = {
2626
- validations: {}
2627
- };
2628
- }
2629
- if (!errors[path].validations[sourcePath]) {
2630
- errors[path].validations[sourcePath] = [];
3281
+ for (const entryError of entryErrors) {
3282
+ // An entry's errors get the same project-wide resolution as every
3283
+ // other module's. Added raw, every keyOf and route in a
3284
+ // `.jsonValues()` record was reported as unresolved -- 240 of them in
3285
+ // `examples/next`'s kb.val.ts, none real.
3286
+ if (isSchemaSourceFixError(entryError)) {
3287
+ const resolved = resolveSchemaSourceFixForError(entryError, snapshot);
3288
+ if (!resolved || resolved.status !== "remaining") {
3289
+ continue;
3290
+ }
3291
+ addModuleError(path, sourcePath, resolved.error);
3292
+ continue;
3293
+ }
3294
+ // "Not yet compared with its bytes" is a request, not an error, and
3295
+ // for an entry it cannot be answered here: `validateFiles` resolves a
3296
+ // file's schema through the module's source, which for a
3297
+ // `.jsonValues()` record holds markers rather than the entry. The
3298
+ // publish gate ignores it anyway (`partitionValidationErrors`).
3299
+ //
3300
+ // A remote file's "not yet checked against its host" is the same
3301
+ // request, for the same reason: `validateRemoteFiles` and its fix
3302
+ // resolve the path through the module's source too. Added as an
3303
+ // error, every remote image in an entry was a row on the Validate
3304
+ // page whose Fix could not find it.
3305
+ if (isOnlyFileCheckValidationError(entryError) || isOnlyRemoteCheckValidationError(entryError)) {
3306
+ continue;
3307
+ }
3308
+ addModuleError(path, sourcePath, entryError);
2631
3309
  }
2632
- errors[path].validations[sourcePath].push(...entryErrors);
2633
3310
  }
2634
3311
  if (res === false) {
2635
3312
  continue;
@@ -2707,9 +3384,64 @@ class ValOps {
2707
3384
  remoteFiles
2708
3385
  };
2709
3386
  }
2710
- async validateRemoteFiles(schemas, sources, remoteFiles) {
2711
- // TODO: Implement
2712
- return {};
3387
+
3388
+ /**
3389
+ * The remote values `validateSources` set aside, checked: is each ref
3390
+ * sound, and does its host serve it. See `checkRemoteFiles`.
3391
+ *
3392
+ * `fileLastUpdatedByPatchId` names the refs whose bytes are still in a
3393
+ * pending patch, which the host will not have until publish.
3394
+ */
3395
+ async validateRemoteFiles(schemas, sources, remoteFiles, fileLastUpdatedByPatchId = {}, network = {}) {
3396
+ const sampleByHost = this.sourceMode() === "managed";
3397
+ // Only a managed project vouches for one published ref with another, so
3398
+ // only it needs to know which refs are published.
3399
+ const baseSources = sampleByHost ? (await this.getSources()).sources : undefined;
3400
+ const valueAt = (sourcePath, from) => {
3401
+ const [moduleFilePath, modulePath] = Internal.splitModuleFilePathAndModulePath(sourcePath);
3402
+ const schema = schemas[moduleFilePath];
3403
+ if (!schema || from[moduleFilePath] === undefined) {
3404
+ return undefined;
3405
+ }
3406
+ try {
3407
+ return Internal.resolvePath(modulePath, from[moduleFilePath], schema["executeSerialize"]());
3408
+ } catch {
3409
+ return undefined;
3410
+ }
3411
+ };
3412
+ return checkRemoteFiles({
3413
+ remoteFiles,
3414
+ sampleByHost,
3415
+ ...network,
3416
+ schemaAt: sourcePath => {
3417
+ var _valueAt;
3418
+ return (_valueAt = valueAt(sourcePath, sources)) === null || _valueAt === void 0 ? void 0 : _valueAt.schema;
3419
+ },
3420
+ isPublished: (sourcePath, ref) => {
3421
+ var _valueAt2;
3422
+ if (!baseSources) {
3423
+ return false;
3424
+ }
3425
+ const published = (_valueAt2 = valueAt(sourcePath, baseSources)) === null || _valueAt2 === void 0 ? void 0 : _valueAt2.source;
3426
+ return typeof published === "object" && published !== null && "path" in published && published.path === ref;
3427
+ },
3428
+ pendingBytes: this.pendingBytesOf(fileLastUpdatedByPatchId)
3429
+ });
3430
+ }
3431
+
3432
+ /**
3433
+ * Is a remote file's bytes in a pending patch? `not-pending` when no patch
3434
+ * wrote it; `missing` when one did and its bytes are not there.
3435
+ */
3436
+ pendingBytesOf(fileLastUpdatedByPatchId) {
3437
+ return async ref => {
3438
+ const fromPatch = fileLastUpdatedByPatchId[ref];
3439
+ if (!fromPatch) {
3440
+ return "not-pending";
3441
+ }
3442
+ const bytes = await this.getBase64EncodedBinaryFileFromPatch(ref, fromPatch.patchId, fromPatch.remote);
3443
+ return bytes === null ? "missing" : "present";
3444
+ };
2713
3445
  }
2714
3446
 
2715
3447
  // #region validateFiles
@@ -2781,7 +3513,9 @@ class ValOps {
2781
3513
  if (metadataErrors && metadataErrors.length > 0) {
2782
3514
  return {
2783
3515
  [sourcePath]: metadataErrors.map(e => ({
2784
- message: e.message,
3516
+ // Which file: "File not found" on its own names nothing, and the
3517
+ // row it is shown on is the field, not the file.
3518
+ message: `${e.message}: ${filePath}`,
2785
3519
  value: {
2786
3520
  filePath,
2787
3521
  patchId: (fileData === null || fileData === void 0 ? void 0 : fileData.patchId) ?? null
@@ -3830,7 +4564,7 @@ class ValOps {
3830
4564
  * override it.
3831
4565
  */
3832
4566
  async saveBinaryFileFromPatch(filePath, parentRef, patchId, bytes, mimeType, type, metadata) {
3833
- return this.saveBase64EncodedBinaryFileFromPatch(filePath, parentRef, patchId, `data:${mimeType};base64,${bytes.toString("base64")}`, type, metadata);
4567
+ return this.saveBase64EncodedBinaryFileFromPatch(filePath, parentRef, patchId, `data:${mimeType};base64,${bytesToBase64(bytes)}`, type, metadata);
3834
4568
  }
3835
4569
  /**
3836
4570
  * A file's bytes as something a `Range` request can be answered from
@@ -3894,6 +4628,10 @@ class ValOps {
3894
4628
 
3895
4629
  // #endregion history
3896
4630
  }
4631
+ function isOnlyRemoteCheckValidationError(validationError) {
4632
+ const fixes = validationError.fixes ?? [];
4633
+ return fixes.length > 0 && fixes.every(f => f === "image:check-remote" || f === "file:check-remote");
4634
+ }
3897
4635
  function isOnlyFileCheckValidationError(validationError) {
3898
4636
  var _validationError$fixe9;
3899
4637
  if ((_validationError$fixe9 = validationError.fixes) !== null && _validationError$fixe9 !== void 0 && _validationError$fixe9.every(f => f === "file:check-metadata" || f === "image:check-metadata")) {
@@ -4052,15 +4790,16 @@ function createMetadataFromBuffer(type, mimeType, buffer) {
4052
4790
  metadata
4053
4791
  };
4054
4792
  }
4055
- function bufferFromDataUrl(dataUrl) {
4793
+ function bufferFromDataUrl(dataUrl, /** Node passes `Buffer`'s, which is faster: this is a whole upload. */
4794
+ decodeBase64 = base64ToBytes) {
4056
4795
  let base64Data;
4057
4796
  const base64Index = dataUrl.indexOf(";base64,");
4058
4797
  if (base64Index > -1) {
4059
4798
  base64Data = dataUrl.slice(base64Index + ";base64,".length);
4060
4799
  }
4061
4800
  if (base64Data) {
4062
- return Buffer.from(base64Data, "base64" // TODO: why does it not work with base64url?
4063
- );
4801
+ // TODO: why does it not work with base64url?
4802
+ return decodeBase64(base64Data);
4064
4803
  }
4065
4804
  }
4066
4805
 
@@ -4211,7 +4950,7 @@ async function resolveJsonEntriesAtCommit(ops, commitSha, affectedFiles) {
4211
4950
  continue;
4212
4951
  }
4213
4952
  try {
4214
- entries[gitPath] = JSON.parse(res.value.toString("utf-8"));
4953
+ entries[gitPath] = JSON.parse(bytesToUtf8(res.value));
4215
4954
  } catch (err) {
4216
4955
  failures.push({
4217
4956
  kind: "file-unavailable",
@@ -4533,7 +5272,14 @@ const SettingsSchema = z.object({
4533
5272
  publicProjectId: z.string(),
4534
5273
  remoteFileBuckets: z.array(z.object({
4535
5274
  bucket: z.string()
4536
- }))
5275
+ })),
5276
+ /**
5277
+ * The host a new remote file's ref names: the project's own address with
5278
+ * `/_val` for a Hosted project whose files are in R2, `remote.val.build`
5279
+ * otherwise. Absent from a content service that predates it, where every
5280
+ * project's files are on `remote.val.build`.
5281
+ */
5282
+ remoteHost: z.string().optional()
4537
5283
  });
4538
5284
  async function getSettings(projectName, auth, /** Overrides {@link defaultHost}. See the note there for why this exists. */
4539
5285
  contentUrl) {
@@ -4709,11 +5455,6 @@ function mapStatus(status) {
4709
5455
  return 500;
4710
5456
  }
4711
5457
 
4712
- /*
4713
- * Imported rather than global: under Node this is the built-in, and in a
4714
- * browser (the demo, `@valbuild/server/browser`) the `buffer` package.
4715
- */
4716
-
4717
5458
  /** What {@link ValServerOptions.publishOverride} is given. */
4718
5459
 
4719
5460
  /** What a publish reports back, in either shape. */
@@ -4724,6 +5465,18 @@ function mapStatus(status) {
4724
5465
  * have their own `dir` for any collection that wants somewhere else.
4725
5466
  */
4726
5467
  const FILES_DIRECTORY = "/public/val";
5468
+
5469
+ /**
5470
+ * `/validate` and `/validate/fix` on a platform without the full check (a
5471
+ * server in a browser tab). 501: nothing the caller does makes it available,
5472
+ * so the Studio offers no Try again.
5473
+ */
5474
+ const studioValidationUnavailable = {
5475
+ status: 501,
5476
+ json: {
5477
+ message: "Checking everything needs the project's files and Val's file hosts, and this Studio runs without them."
5478
+ }
5479
+ };
4727
5480
  const ValServer = (valModules, options, callbacks, platform) => {
4728
5481
  var _options$git;
4729
5482
  const {
@@ -5127,6 +5880,82 @@ const ValServer = (valModules, options, callbacks, platform) => {
5127
5880
  };
5128
5881
  }
5129
5882
  };
5883
+
5884
+ /**
5885
+ * What the Studio's full check reads: base plus the patches the caller is
5886
+ * showing, as `/sources/~` resolves an explicit `patch_id` list -- including
5887
+ * `draftOverlay`, so a module changed by a publish after this build is
5888
+ * checked as the caller sees it, not as this build has it.
5889
+ */
5890
+ const studioCheckInput = async (cookies, moduleFilePath, patchIds) => {
5891
+ const auth = getAuth(cookies);
5892
+ if (auth.error) {
5893
+ return {
5894
+ status: 401,
5895
+ json: {
5896
+ message: auth.error
5897
+ }
5898
+ };
5899
+ }
5900
+ if (isHttpOps(serverOps) && !("id" in auth)) {
5901
+ return {
5902
+ status: 401,
5903
+ json: {
5904
+ message: "Unauthorized"
5905
+ }
5906
+ };
5907
+ }
5908
+ const moduleErrors = await serverOps.getModuleErrors();
5909
+ if ((moduleErrors === null || moduleErrors === void 0 ? void 0 : moduleErrors.length) > 0) {
5910
+ return {
5911
+ status: 500,
5912
+ json: {
5913
+ message: "Val could not load this project's modules, so nothing can be checked. Check val.modules."
5914
+ }
5915
+ };
5916
+ }
5917
+ // An EXPLICITLY empty list is base, never "everything": `fetchPatches`
5918
+ // reads an empty filter as no filter (see `/sources/~`).
5919
+ let patchOps = patchIds !== undefined && patchIds.length === 0 ? {
5920
+ patches: []
5921
+ } : await serverOps.fetchPatches({
5922
+ patchIds,
5923
+ excludePatchOps: false
5924
+ });
5925
+ if ("error" in patchOps && patchOps.error) {
5926
+ return "unauthorized" in patchOps && patchOps.unauthorized ? {
5927
+ status: 401,
5928
+ json: {
5929
+ message: "Unauthorized"
5930
+ }
5931
+ } : {
5932
+ status: 500,
5933
+ json: {
5934
+ message: "The unpublished changes could not be read, so nothing was checked."
5935
+ }
5936
+ };
5937
+ }
5938
+ patchOps = {
5939
+ ...patchOps,
5940
+ patches: draftOverlay(patchOps.patches, patchOps.commits)
5941
+ };
5942
+ const analysis = serverOps.analyzePatches(patchOps.patches);
5943
+ const {
5944
+ sources
5945
+ } = await serverOps.getSourcesWithPatchesApplied({
5946
+ ...analysis,
5947
+ ...patchOps
5948
+ });
5949
+ return {
5950
+ status: "ok",
5951
+ input: {
5952
+ moduleFilePath,
5953
+ schemas: await serverOps.getSchemas(),
5954
+ sources,
5955
+ fileLastUpdatedByPatchId: analysis.fileLastUpdatedByPatchId
5956
+ }
5957
+ };
5958
+ };
5130
5959
  return {
5131
5960
  "/draft/enable": {
5132
5961
  GET: async req => {
@@ -5477,7 +6306,10 @@ const ValServer = (valModules, options, callbacks, platform) => {
5477
6306
  json: {
5478
6307
  publicProjectId: settingsRes.data.publicProjectId,
5479
6308
  remoteFileBuckets: settingsRes.data.remoteFileBuckets,
5480
- coreVersion: Internal.VERSION.core || "unknown"
6309
+ coreVersion: Internal.VERSION.core || "unknown",
6310
+ ...(settingsRes.data.remoteHost !== undefined ? {
6311
+ remoteHost: settingsRes.data.remoteHost
6312
+ } : {})
5481
6313
  }
5482
6314
  };
5483
6315
  }
@@ -5758,6 +6590,92 @@ const ValServer = (valModules, options, callbacks, platform) => {
5758
6590
  };
5759
6591
  }
5760
6592
  },
6593
+ "/validate": {
6594
+ POST: async req => {
6595
+ const checked = await studioCheckInput(req.cookies, req.body.moduleFilePath, req.body.patchIds);
6596
+ if (checked.status !== "ok") {
6597
+ return checked;
6598
+ }
6599
+ if (!platform.studioValidation) {
6600
+ return studioValidationUnavailable;
6601
+ }
6602
+ const res = await platform.studioValidation.checkModuleForStudio(serverOps, checked.input);
6603
+ switch (res.status) {
6604
+ case "unknown-module":
6605
+ return {
6606
+ status: 404,
6607
+ json: {
6608
+ message: `There is no module '${req.body.moduleFilePath}' in this project.`
6609
+ }
6610
+ };
6611
+ case "invalid-source":
6612
+ return {
6613
+ status: 200,
6614
+ json: {
6615
+ moduleFilePath: req.body.moduleFilePath,
6616
+ errors: {},
6617
+ invalidSource: res.message
6618
+ }
6619
+ };
6620
+ case "ok":
6621
+ return {
6622
+ status: 200,
6623
+ json: {
6624
+ moduleFilePath: req.body.moduleFilePath,
6625
+ errors: res.errors,
6626
+ remoteHosts: res.remoteHosts
6627
+ }
6628
+ };
6629
+ }
6630
+ }
6631
+ },
6632
+ "/validate/fix": {
6633
+ POST: async req => {
6634
+ const checked = await studioCheckInput(req.cookies, req.body.moduleFilePath, req.body.patchIds);
6635
+ if (checked.status !== "ok") {
6636
+ return checked;
6637
+ }
6638
+ if (!platform.studioValidation) {
6639
+ return studioValidationUnavailable;
6640
+ }
6641
+ const res = await platform.studioValidation.fixForStudio(serverOps, {
6642
+ ...checked.input,
6643
+ sourcePath: req.body.sourcePath,
6644
+ fix: req.body.fix
6645
+ });
6646
+ switch (res.status) {
6647
+ case "unknown-module":
6648
+ return {
6649
+ status: 404,
6650
+ json: {
6651
+ message: `There is no module '${req.body.moduleFilePath}' in this project.`
6652
+ }
6653
+ };
6654
+ case "not-found":
6655
+ return {
6656
+ status: 404,
6657
+ json: {
6658
+ message: "This error is no longer there. Check the module again to see what is left."
6659
+ }
6660
+ };
6661
+ case "not-in-studio":
6662
+ return {
6663
+ status: 400,
6664
+ json: {
6665
+ message: "This fix cannot be made from the Studio yet. Run `val validate --fix` in the project."
6666
+ }
6667
+ };
6668
+ case "ok":
6669
+ return {
6670
+ status: 200,
6671
+ json: {
6672
+ patch: res.patch,
6673
+ remainingErrors: res.remainingErrors
6674
+ }
6675
+ };
6676
+ }
6677
+ }
6678
+ },
5761
6679
  "/external-urls/check": {
5762
6680
  POST: async req => {
5763
6681
  /*
@@ -6431,8 +7349,9 @@ const ValServer = (valModules, options, callbacks, platform) => {
6431
7349
  // TODO: send binary files validation errors
6432
7350
  if (query.validate_binary_files) {
6433
7351
  await serverOps.validateFiles(schemas, sourcesRes.sources, sourcesValidation.files);
6434
- // TODO : actually do something with this
6435
- await serverOps.validateRemoteFiles(schemas, sourcesRes.sources, sourcesValidation.remoteFiles);
7352
+ // Remote files are not checked here: their check asks each host
7353
+ // over the network, so it is the Studio's full check that runs it,
7354
+ // one module at a time (`/validate`).
6436
7355
  }
6437
7356
  }
6438
7357
  const schemaSha = await serverOps.getSchemaSha();
@@ -6580,7 +7499,7 @@ const ValServer = (valModules, options, callbacks, platform) => {
6580
7499
  withJob: prepared,
6581
7500
  jobPatchIds: patchIds
6582
7501
  });
6583
- const build = serverOps.embedsSource() ? await buildSourcesOf(serverOps, patchIds, commit) : null;
7502
+ const build = serverOps.embedsSource() ? await buildSourcesOf(platform, serverOps, patchIds, commit) : null;
6584
7503
  if (build !== null && "status" in build) return build;
6585
7504
  /*
6586
7505
  * A build of no commit says what Val source it holds instead, so
@@ -6612,7 +7531,7 @@ const ValServer = (valModules, options, callbacks, platform) => {
6612
7531
  buildable: build !== null
6613
7532
  });
6614
7533
  }
6615
- const built = await buildSourcesOf(serverOps, patchIds, commit);
7534
+ const built = await buildSourcesOf(platform, serverOps, patchIds, commit);
6616
7535
  if ("status" in built) return built;
6617
7536
  const {
6618
7537
  split
@@ -6682,7 +7601,7 @@ const ValServer = (valModules, options, callbacks, platform) => {
6682
7601
  * and the selection on top. Only what the selection touched is saved,
6683
7602
  * so the proposal's save is the selection and nothing else.
6684
7603
  */
6685
- const built = await buildSourcesOf(serverOps, patchIds, commit, {
7604
+ const built = await buildSourcesOf(platform, serverOps, patchIds, commit, {
6686
7605
  binaries: false
6687
7606
  });
6688
7607
  if ("status" in built) return built;
@@ -7227,7 +8146,7 @@ const ValServer = (valModules, options, callbacks, platform) => {
7227
8146
  * patch's to read. See `binaryFiles` on the route.
7228
8147
  */
7229
8148
  const managedBranch = serverOps.sourceMode() === "managed" ? serverOps.projectBranch() : null;
7230
- const committedBinaries = serverOps.sourceMode() === "managed" ? await readCommittedBinaryFiles(serverOps, preparedCommit.patchedBinaryFilesDescriptors) : null;
8149
+ const committedBinaries = serverOps.sourceMode() === "managed" ? await readCommittedBinaryFiles(serverOps, preparedCommit.patchedBinaryFilesDescriptors, platform.bytesToBase64) : null;
7231
8150
  const commitToGit = () => serverOps.commit(preparedCommit, message, auth.id, FILES_DIRECTORY, undefined);
7232
8151
  /*
7233
8152
  * A host may publish somewhere other than the repository.
@@ -7887,7 +8806,7 @@ const ValServer = (valModules, options, callbacks, platform) => {
7887
8806
  // wrap it in a data: URL, and hand that to a method whose first
7888
8807
  // act was to unwrap it again - ceremony to satisfy a convention
7889
8808
  // that no longer exists.
7890
- const bytes = Buffer.from(await binaryRes.arrayBuffer());
8809
+ const bytes = new Uint8Array(await binaryRes.arrayBuffer());
7891
8810
  const type = file.metadata.mimeType.startsWith("image/") ? "image" : "file";
7892
8811
  const saveRes = await serverOps.saveBinaryFileFromPatch(file.filePath, req.body.parentRef, req.body.patchId, bytes, file.metadata.mimeType, type, file.metadata);
7893
8812
  if (saveRes.error) {
@@ -8133,7 +9052,7 @@ const ValServer = (valModules, options, callbacks, platform) => {
8133
9052
  // TypeScript compiler. See `patch/ts/publishKit.ts`.
8134
9053
  const {
8135
9054
  getJsonEntryAtCommit
8136
- } = await import('./getJsonEntryAtCommit-1e08d724.esm.js');
9055
+ } = await import('./getJsonEntryAtCommit-40b6d9c1.esm.js');
8137
9056
  const res = await getJsonEntryAtCommit(serverOps, req.query.commit_sha, req.query.path, req.query.key);
8138
9057
  if (result.isErr(res)) {
8139
9058
  return historyErrorResponse(res.error);
@@ -8241,11 +9160,11 @@ const ValServer = (valModules, options, callbacks, platform) => {
8241
9160
  // A DRAFT playlist names files that are drafts too, at URLs that only
8242
9161
  // resolve once published. See `rewriteDraftPlaylist`. Playlists are
8243
9162
  // small, so one is read whole to be rewritten.
8244
- const file = patchId && isHlsPlaylistPath(filePath) ? bufferReader(Buffer.from(rewriteDraftPlaylist((await opened.read(0, opened.size - 1)).toString("utf-8"), {
9163
+ const file = patchId && isHlsPlaylistPath(filePath) ? bufferReader(utf8ToBytes(rewriteDraftPlaylist(bytesToUtf8(await opened.read(0, opened.size - 1)), {
8245
9164
  playlistPath: filePath,
8246
9165
  patchId,
8247
9166
  remote
8248
- }), "utf-8")) : opened;
9167
+ }))) : opened;
8249
9168
  const headers = {
8250
9169
  // TODO: we could use ETag and return 304 instead
8251
9170
  // From the extension, published or draft: a `<video>` and an HLS
@@ -8285,7 +9204,7 @@ const ValServer = (valModules, options, callbacks, platform) => {
8285
9204
  body: bufferToReadableStream(slice)
8286
9205
  };
8287
9206
  }
8288
- const body = file.size === 0 ? Buffer.alloc(0) : await file.read(0, file.size - 1);
9207
+ const body = file.size === 0 ? new Uint8Array(0) : await file.read(0, file.size - 1);
8289
9208
  return {
8290
9209
  status: 200,
8291
9210
  headers: {
@@ -8347,7 +9266,7 @@ function verifyCallbackReq(stateCookie, queryParams) {
8347
9266
  }
8348
9267
  function getStateFromCookie(stateCookie) {
8349
9268
  try {
8350
- const decoded = Buffer.from(stateCookie, "base64").toString("utf8");
9269
+ const decoded = bytesToUtf8(base64ToBytes(stateCookie));
8351
9270
  const parsed = JSON.parse(decoded);
8352
9271
  if (!parsed) {
8353
9272
  return {
@@ -8399,7 +9318,7 @@ function getStateFromCookie(stateCookie) {
8399
9318
  }
8400
9319
  }
8401
9320
  function createStateCookie(state) {
8402
- return Buffer.from(JSON.stringify(state), "utf8").toString("base64");
9321
+ return bytesToBase64(utf8ToBytes(JSON.stringify(state)));
8403
9322
  }
8404
9323
  const ValAppJwtPayload = z.object({
8405
9324
  sub: z.string(),
@@ -8682,7 +9601,7 @@ function getIsRemoteRequired(schemas) {
8682
9601
  * `split.archive` is the managed job's archive; a connected job's archive is
8683
9602
  * the push's, made from its own changes alone, by the caller.
8684
9603
  */
8685
- async function buildSourcesOf(serverOps, patchIds, commit,
9604
+ async function buildSourcesOf(platform, serverOps, patchIds, commit,
8686
9605
  /**
8687
9606
  * Read the job's binary files back for a build. Off for a caller that only
8688
9607
  * commits what was prepared, which names the files rather than carrying them.
@@ -8748,7 +9667,7 @@ options = {
8748
9667
  jobPatchIds: patchIds
8749
9668
  });
8750
9669
  // Read now, while the job's changes still hold their files.
8751
- const binaries = options.binaries ? await readCommittedBinaryFiles(serverOps, split.archive.patchedBinaryFilesDescriptors) : {
9670
+ const binaries = options.binaries ? await readCommittedBinaryFiles(serverOps, split.archive.patchedBinaryFilesDescriptors, platform.bytesToBase64) : {
8752
9671
  files: {},
8753
9672
  unread: []
8754
9673
  };
@@ -8863,11 +9782,6 @@ async function uploadRemoteFile(contentHost, project, bucket, fileHash, fileExt,
8863
9782
  };
8864
9783
  }
8865
9784
 
8866
- function getFileExt(filePath) {
8867
- // NOTE: We do not import the path module. This code is copied in different projects. We want the same implementation and which means that this might running in browser where path is not available).
8868
- return filePath.split(".").pop() || "";
8869
- }
8870
-
8871
9785
  /**
8872
9786
  * One stored patch. Everything the ordered chain needs and nothing else.
8873
9787
  */
@@ -9568,7 +10482,11 @@ function createValRequestHandler(route, valServerPromise, convert, serveStatic)
9568
10482
  const method = (_req$method = req.method) === null || _req$method === void 0 ? void 0 : _req$method.toUpperCase();
9569
10483
  let route = null;
9570
10484
  let path = undefined;
9571
- for (const routeDef of Object.keys(Api)) {
10485
+ // An exact match wins over every prefix match, wherever it sits in
10486
+ // `Api`: checked per key, in order, `/validate` took `/validate/fix`
10487
+ // because it was declared first.
10488
+ const exactRouteDef = Object.keys(Api).find(routeDef => routeDef === reqApiRoutePath);
10489
+ for (const routeDef of exactRouteDef ? [exactRouteDef] : Object.keys(Api)) {
9572
10490
  if (routeDef === reqApiRoutePath) {
9573
10491
  route = routeDef;
9574
10492
  break;
@@ -9863,4 +10781,4 @@ function getCookies(req, cookiesDef) {
9863
10781
  return z.object(cookiesDef).safeParse(input);
9864
10782
  }
9865
10783
 
9866
- export { InMemoryPatchStore as I, ValOps as V, rebaseContentOp as a, bufferFromDataUrl as b, classifyJsonValuesOp as c, getFileExt as d, guessMimeTypeFromPath as e, findNestedJsonValuesRecords as f, getFieldsForType as g, createMetadataFromBuffer as h, isJsonValuesRootOp as i, ValOpsMemory as j, ValServer as k, createValRequestHandler as l, isHlsMasterPlaylist as m, largestHlsVariant as n, mapHlsUris as o, parseHlsMasterVariants as p, getNewJsonEntryPaths as q, resolveExistingJsonPath as r, sumHlsSegmentDurations as s, formatPatchSourceError as t, uploadRemoteFile as u, validateJsonValuesEntries as v, getSettings as w, posixPath as x };
10784
+ export { getNewJsonEntryPaths as A, formatPatchSourceError as B, askOncePerHost as C, getSettings as D, bytesToBase64 as E, bytesToUtf8 as F, posixPath as G, InMemoryPatchStore as I, ValOps as V, rebaseContentOp as a, bufferFromDataUrl as b, classifyJsonValuesOp as c, getFileExt as d, guessMimeTypeFromPath as e, findNestedJsonValuesRecords as f, getFieldsForType as g, createMetadataFromBuffer as h, isJsonValuesRootOp as i, ValOpsMemory as j, isHlsMasterPlaylist as k, largestHlsVariant as l, fallbackRefOf as m, mapHlsUris as n, movedRemoteHost as o, parseHlsMasterVariants as p, needsHostCheck as q, resolveExistingJsonPath as r, sumHlsSegmentDurations as s, rewriteRefHost as t, uploadRemoteFile as u, validateJsonValuesEntries as v, fallbackRemoteHost as w, checkRemoteFiles as x, ValServer as y, createValRequestHandler as z };