@valbuild/server 0.72.3 → 0.73.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -2
- package/dist/declarations/src/ValFSHost.d.ts +1 -0
- package/dist/declarations/src/createFixPatch.d.ts +10 -2
- package/dist/declarations/src/getSettings.d.ts +34 -0
- package/dist/declarations/src/index.d.ts +3 -0
- package/dist/declarations/src/patch/ts/syntax.d.ts +3 -0
- package/dist/declarations/src/personalAccessTokens.d.ts +10 -0
- package/dist/declarations/src/uploadRemoteFile.d.ts +25 -0
- package/dist/valbuild-server.cjs.dev.js +1016 -101
- package/dist/valbuild-server.cjs.prod.js +1016 -101
- package/dist/valbuild-server.esm.js +1012 -103
- package/package.json +4 -4
@@ -17,6 +17,8 @@ var crypto$1 = require('crypto');
|
|
17
17
|
var zod = require('zod');
|
18
18
|
var sizeOf = require('image-size');
|
19
19
|
var zodValidationError = require('zod-validation-error');
|
20
|
+
var http = require('http');
|
21
|
+
var https = require('https');
|
20
22
|
|
21
23
|
function _interopDefault (e) { return e && e.__esModule ? e : { 'default': e }; }
|
22
24
|
|
@@ -43,6 +45,8 @@ var fsPath__namespace = /*#__PURE__*/_interopNamespace(fsPath);
|
|
43
45
|
var fs__default = /*#__PURE__*/_interopDefault(fs);
|
44
46
|
var crypto__default = /*#__PURE__*/_interopDefault(crypto$1);
|
45
47
|
var sizeOf__default = /*#__PURE__*/_interopDefault(sizeOf);
|
48
|
+
var http__default = /*#__PURE__*/_interopDefault(http);
|
49
|
+
var https__default = /*#__PURE__*/_interopDefault(https);
|
46
50
|
|
47
51
|
class ValSyntaxError {
|
48
52
|
constructor(message, node) {
|
@@ -94,7 +98,7 @@ function validateObjectProperties(nodes) {
|
|
94
98
|
* validating its children.
|
95
99
|
*/
|
96
100
|
function shallowValidateExpression(value) {
|
97
|
-
return ts__default["default"].isStringLiteralLike(value) || ts__default["default"].isNumericLiteral(value) || value.kind === ts__default["default"].SyntaxKind.TrueKeyword || value.kind === ts__default["default"].SyntaxKind.FalseKeyword || value.kind === ts__default["default"].SyntaxKind.NullKeyword || ts__default["default"].isArrayLiteralExpression(value) || ts__default["default"].isObjectLiteralExpression(value) || isValFileMethodCall(value) || isValImageMethodCall(value) ? undefined : new ValSyntaxError("Expression must be a literal or call c.file", value);
|
101
|
+
return ts__default["default"].isStringLiteralLike(value) || ts__default["default"].isNumericLiteral(value) || value.kind === ts__default["default"].SyntaxKind.TrueKeyword || value.kind === ts__default["default"].SyntaxKind.FalseKeyword || value.kind === ts__default["default"].SyntaxKind.NullKeyword || ts__default["default"].isArrayLiteralExpression(value) || ts__default["default"].isObjectLiteralExpression(value) || isValFileMethodCall(value) || isValImageMethodCall(value) || isValRemoteMethodCall(value) ? undefined : new ValSyntaxError("Expression must be a literal or call c.file / c.image / c.remote", value);
|
98
102
|
}
|
99
103
|
|
100
104
|
/**
|
@@ -152,8 +156,23 @@ function evaluateExpression(value) {
|
|
152
156
|
});
|
153
157
|
}
|
154
158
|
}));
|
159
|
+
} else if (isValRemoteMethodCall(value)) {
|
160
|
+
return fp.pipe(findValRemoteNodeArg(value), fp.result.flatMap(ref => {
|
161
|
+
if (value.arguments.length === 2) {
|
162
|
+
return fp.pipe(evaluateExpression(value.arguments[1]), fp.result.map(metadata => ({
|
163
|
+
[core.FILE_REF_PROP]: ref.text,
|
164
|
+
_type: "remote",
|
165
|
+
metadata
|
166
|
+
})));
|
167
|
+
} else {
|
168
|
+
return fp.result.ok({
|
169
|
+
[core.FILE_REF_PROP]: ref.text,
|
170
|
+
_type: "remote"
|
171
|
+
});
|
172
|
+
}
|
173
|
+
}));
|
155
174
|
} else {
|
156
|
-
return fp.result.err(new ValSyntaxError("Expression must be a literal or call c.file", value));
|
175
|
+
return fp.result.err(new ValSyntaxError("Expression must be a literal or call c.file / c.image / c.remote", value));
|
157
176
|
}
|
158
177
|
}
|
159
178
|
function findObjectPropertyAssignment(value, key) {
|
@@ -173,6 +192,9 @@ function isValFileMethodCall(node) {
|
|
173
192
|
function isValImageMethodCall(node) {
|
174
193
|
return ts__default["default"].isCallExpression(node) && ts__default["default"].isPropertyAccessExpression(node.expression) && ts__default["default"].isIdentifier(node.expression.expression) && node.expression.expression.text === "c" && node.expression.name.text === "image";
|
175
194
|
}
|
195
|
+
function isValRemoteMethodCall(node) {
|
196
|
+
return ts__default["default"].isCallExpression(node) && ts__default["default"].isPropertyAccessExpression(node.expression) && ts__default["default"].isIdentifier(node.expression.expression) && node.expression.expression.text === "c" && node.expression.name.text === "remote";
|
197
|
+
}
|
176
198
|
function findValFileNodeArg(node) {
|
177
199
|
if (node.arguments.length === 0) {
|
178
200
|
return fp.result.err(new ValSyntaxError(`Invalid c.file() call: missing ref argument`, node));
|
@@ -184,6 +206,17 @@ function findValFileNodeArg(node) {
|
|
184
206
|
const refNode = node.arguments[0];
|
185
207
|
return fp.result.ok(refNode);
|
186
208
|
}
|
209
|
+
function findValRemoteNodeArg(node) {
|
210
|
+
if (node.arguments.length === 0) {
|
211
|
+
return fp.result.err(new ValSyntaxError(`Invalid c.remote() call: missing ref argument`, node));
|
212
|
+
} else if (node.arguments.length > 2) {
|
213
|
+
return fp.result.err(new ValSyntaxError(`Invalid c.remote() call: too many arguments ${node.arguments.length}}`, node));
|
214
|
+
} else if (!ts__default["default"].isStringLiteral(node.arguments[0])) {
|
215
|
+
return fp.result.err(new ValSyntaxError(`Invalid c.remote() call: ref must be a string literal`, node));
|
216
|
+
}
|
217
|
+
const refNode = node.arguments[0];
|
218
|
+
return fp.result.ok(refNode);
|
219
|
+
}
|
187
220
|
function findValImageNodeArg(node) {
|
188
221
|
if (node.arguments.length === 0) {
|
189
222
|
return fp.result.err(new ValSyntaxError(`Invalid c.image() call: missing ref argument`, node));
|
@@ -221,6 +254,19 @@ function findValImageMetadataArg(node) {
|
|
221
254
|
}
|
222
255
|
return fp.result.ok(undefined);
|
223
256
|
}
|
257
|
+
function findValRemoteMetadataArg(node) {
|
258
|
+
if (node.arguments.length === 0) {
|
259
|
+
return fp.result.err(new ValSyntaxError(`Invalid c.remote() call: missing ref argument`, node));
|
260
|
+
} else if (node.arguments.length > 2) {
|
261
|
+
return fp.result.err(new ValSyntaxError(`Invalid c.remote() call: too many arguments ${node.arguments.length}}`, node));
|
262
|
+
} else if (node.arguments.length === 2) {
|
263
|
+
if (!ts__default["default"].isObjectLiteralExpression(node.arguments[1])) {
|
264
|
+
return fp.result.err(new ValSyntaxError(`Invalid c.remote() call: metadata must be a object literal`, node));
|
265
|
+
}
|
266
|
+
return fp.result.ok(node.arguments[1]);
|
267
|
+
}
|
268
|
+
return fp.result.ok(undefined);
|
269
|
+
}
|
224
270
|
|
225
271
|
/**
|
226
272
|
* Given a list of expressions, validates that all the expressions are not
|
@@ -322,6 +368,13 @@ function createValImageReference(value) {
|
|
322
368
|
}
|
323
369
|
return ts__default["default"].factory.createCallExpression(ts__default["default"].factory.createPropertyAccessExpression(ts__default["default"].factory.createIdentifier("c"), ts__default["default"].factory.createIdentifier("image")), undefined, args);
|
324
370
|
}
|
371
|
+
function createValRemoteReference(value) {
|
372
|
+
const args = [ts__default["default"].factory.createStringLiteral(value[core.FILE_REF_PROP])];
|
373
|
+
if (value.metadata) {
|
374
|
+
args.push(toExpression(value.metadata));
|
375
|
+
}
|
376
|
+
return ts__default["default"].factory.createCallExpression(ts__default["default"].factory.createPropertyAccessExpression(ts__default["default"].factory.createIdentifier("c"), ts__default["default"].factory.createIdentifier("remote")), undefined, args);
|
377
|
+
}
|
325
378
|
function toExpression(value) {
|
326
379
|
if (typeof value === "string") {
|
327
380
|
// TODO: Use configuration/heuristics to determine use of single quote or double quote
|
@@ -354,6 +407,9 @@ function toExpression(value) {
|
|
354
407
|
if (isValImageValue(value)) {
|
355
408
|
return createValImageReference(value);
|
356
409
|
}
|
410
|
+
if (isValRemoteValue(value)) {
|
411
|
+
return createValRemoteReference(value);
|
412
|
+
}
|
357
413
|
return ts__default["default"].factory.createObjectLiteralExpression(Object.entries(value).map(([key, value]) => createPropertyAssignment(key, value)));
|
358
414
|
} else {
|
359
415
|
return ts__default["default"].factory.createStringLiteral(value);
|
@@ -525,6 +581,25 @@ function replaceInNode(document, node, key, value) {
|
|
525
581
|
ts__default["default"].factory.createObjectLiteralExpression([ts__default["default"].factory.createPropertyAssignment(key, metadataArgNode)]), key, value);
|
526
582
|
}));
|
527
583
|
}
|
584
|
+
} else if (isValRemoteMethodCall(node)) {
|
585
|
+
if (key === core.FILE_REF_PROP) {
|
586
|
+
if (typeof value !== "string") {
|
587
|
+
return fp.result.err(new patch.PatchError("Cannot replace c.image reference with non-string value"));
|
588
|
+
}
|
589
|
+
return fp.pipe(findValRemoteNodeArg(node), fp.result.map(refNode => replaceNodeValue(document, refNode, value)));
|
590
|
+
} else {
|
591
|
+
return fp.pipe(findValRemoteMetadataArg(node), fp.result.flatMap(metadataArgNode => {
|
592
|
+
if (!metadataArgNode) {
|
593
|
+
return fp.result.err(new patch.PatchError("Cannot replace in c.remote metadata when it does not exist"));
|
594
|
+
}
|
595
|
+
if (key !== "metadata") {
|
596
|
+
return fp.result.err(new patch.PatchError(`Cannot replace c.remote metadata key ${key} when it does not exist`));
|
597
|
+
}
|
598
|
+
return replaceInNode(document,
|
599
|
+
// TODO: creating a fake object here might not be right - seems to work though
|
600
|
+
ts__default["default"].factory.createObjectLiteralExpression([ts__default["default"].factory.createPropertyAssignment(key, metadataArgNode)]), key, value);
|
601
|
+
}));
|
602
|
+
}
|
528
603
|
} else {
|
529
604
|
return fp.result.err(shallowValidateExpression(node) ?? new patch.PatchError("Cannot replace in non-object/array"));
|
530
605
|
}
|
@@ -551,6 +626,11 @@ function getFromNode(node, key) {
|
|
551
626
|
return findValImageNodeArg(node);
|
552
627
|
}
|
553
628
|
return findValImageMetadataArg(node);
|
629
|
+
} else if (isValRemoteMethodCall(node)) {
|
630
|
+
if (key === core.FILE_REF_PROP) {
|
631
|
+
return findValRemoteNodeArg(node);
|
632
|
+
}
|
633
|
+
return findValRemoteMetadataArg(node);
|
554
634
|
} else {
|
555
635
|
return fp.result.err(shallowValidateExpression(node) ?? new patch.PatchError("Cannot access non-object/array"));
|
556
636
|
}
|
@@ -596,7 +676,7 @@ function removeFromNode(document, node, key) {
|
|
596
676
|
}
|
597
677
|
} else if (isValImageMethodCall(node)) {
|
598
678
|
if (key === core.FILE_REF_PROP) {
|
599
|
-
return fp.result.err(new patch.PatchError("Cannot remove a ref from c.
|
679
|
+
return fp.result.err(new patch.PatchError("Cannot remove a ref from c.image"));
|
600
680
|
} else {
|
601
681
|
return fp.pipe(findValImageMetadataArg(node), fp.result.flatMap(metadataArgNode => {
|
602
682
|
if (!metadataArgNode) {
|
@@ -605,6 +685,17 @@ function removeFromNode(document, node, key) {
|
|
605
685
|
return removeFromNode(document, metadataArgNode, key);
|
606
686
|
}));
|
607
687
|
}
|
688
|
+
} else if (isValRemoteMethodCall(node)) {
|
689
|
+
if (key === core.FILE_REF_PROP) {
|
690
|
+
return fp.result.err(new patch.PatchError("Cannot remove a ref from c.remote"));
|
691
|
+
} else {
|
692
|
+
return fp.pipe(findValRemoteMetadataArg(node), fp.result.flatMap(metadataArgNode => {
|
693
|
+
if (!metadataArgNode) {
|
694
|
+
return fp.result.err(new patch.PatchError("Cannot remove from c.remote metadata when it does not exist"));
|
695
|
+
}
|
696
|
+
return removeFromNode(document, metadataArgNode, key);
|
697
|
+
}));
|
698
|
+
}
|
608
699
|
} else {
|
609
700
|
return fp.result.err(shallowValidateExpression(node) ?? new patch.PatchError("Cannot remove from non-object/array"));
|
610
701
|
}
|
@@ -613,18 +704,13 @@ function removeAtPath(document, rootNode, path) {
|
|
613
704
|
return fp.pipe(getPointerFromPath(rootNode, path), fp.result.flatMap(([node, key]) => removeFromNode(document, node, key)));
|
614
705
|
}
|
615
706
|
function isValFileValue(value) {
|
616
|
-
return !!(typeof value === "object" && value &&
|
617
|
-
// TODO: replace the below with this:
|
618
|
-
// VAL_EXTENSION in value &&
|
619
|
-
// value[VAL_EXTENSION] === "file" &&
|
620
|
-
core.FILE_REF_PROP in value && typeof value[core.FILE_REF_PROP] === "string" && value[core.FILE_REF_SUBTYPE_TAG] !== "image");
|
707
|
+
return !!(typeof value === "object" && value && core.VAL_EXTENSION in value && value[core.VAL_EXTENSION] === "file" && core.FILE_REF_PROP in value && typeof value[core.FILE_REF_PROP] === "string" && value[core.FILE_REF_SUBTYPE_TAG] !== "image");
|
621
708
|
}
|
622
709
|
function isValImageValue(value) {
|
623
|
-
return !!(typeof value === "object" && value &&
|
624
|
-
|
625
|
-
|
626
|
-
|
627
|
-
core.FILE_REF_PROP in value && typeof value[core.FILE_REF_PROP] === "string" && value[core.FILE_REF_SUBTYPE_TAG] === "image");
|
710
|
+
return !!(typeof value === "object" && value && core.VAL_EXTENSION in value && value[core.VAL_EXTENSION] === "file" && core.FILE_REF_PROP in value && typeof value[core.FILE_REF_PROP] === "string" && value[core.FILE_REF_SUBTYPE_TAG] === "image");
|
711
|
+
}
|
712
|
+
function isValRemoteValue(value) {
|
713
|
+
return !!(typeof value === "object" && value && core.VAL_EXTENSION in value && value[core.VAL_EXTENSION] === "remote" && core.FILE_REF_PROP in value && typeof value[core.FILE_REF_PROP] === "string");
|
628
714
|
}
|
629
715
|
function addToNode(document, node, key, value) {
|
630
716
|
if (ts__default["default"].isArrayLiteralExpression(node)) {
|
@@ -674,6 +760,23 @@ function addToNode(document, node, key, value) {
|
|
674
760
|
return fp.result.ok([insertAt(document, node.arguments, node.arguments.length, toExpression(value))]);
|
675
761
|
}));
|
676
762
|
}
|
763
|
+
} else if (isValRemoteMethodCall(node)) {
|
764
|
+
if (key === core.FILE_REF_PROP) {
|
765
|
+
if (typeof value !== "string") {
|
766
|
+
return fp.result.err(new patch.PatchError(`Cannot add ${core.FILE_REF_PROP} key to c.remote with non-string value`));
|
767
|
+
}
|
768
|
+
return fp.pipe(findValRemoteNodeArg(node), fp.result.map(arg => replaceNodeValue(document, arg, value)));
|
769
|
+
} else {
|
770
|
+
return fp.pipe(findValRemoteMetadataArg(node), fp.result.flatMap(metadataArgNode => {
|
771
|
+
if (metadataArgNode) {
|
772
|
+
return fp.result.err(new patch.PatchError("Cannot add metadata to c.remote when it already exists"));
|
773
|
+
}
|
774
|
+
if (key !== "metadata") {
|
775
|
+
return fp.result.err(new patch.PatchError(`Cannot add ${key} key to c.remote: only metadata is allowed`));
|
776
|
+
}
|
777
|
+
return fp.result.ok([insertAt(document, node.arguments, node.arguments.length, toExpression(value))]);
|
778
|
+
}));
|
779
|
+
}
|
677
780
|
} else {
|
678
781
|
return fp.result.err(shallowValidateExpression(node) ?? new patch.PatchError("Cannot add to non-object/array"));
|
679
782
|
}
|
@@ -922,7 +1025,7 @@ class ValSourceFileHandler {
|
|
922
1025
|
fs__default["default"].mkdirSync(fsPath__namespace["default"].dirname(fileName), {
|
923
1026
|
recursive: true
|
924
1027
|
});
|
925
|
-
fs__default["default"].writeFileSync(fileName, data, encoding);
|
1028
|
+
fs__default["default"].writeFileSync(fileName, typeof data === "string" ? data : new Uint8Array(data), encoding);
|
926
1029
|
},
|
927
1030
|
rmFile: fs__default["default"].rmSync
|
928
1031
|
}) {
|
@@ -972,7 +1075,7 @@ class ValModuleLoader {
|
|
972
1075
|
fs__default["default"].mkdirSync(fsPath__namespace["default"].dirname(fileName), {
|
973
1076
|
recursive: true
|
974
1077
|
});
|
975
|
-
fs__default["default"].writeFileSync(fileName, data, encoding);
|
1078
|
+
fs__default["default"].writeFileSync(fileName, typeof data === "string" ? data : new Uint8Array(data), encoding);
|
976
1079
|
},
|
977
1080
|
rmFile: fs__default["default"].rmSync
|
978
1081
|
}, disableCache = false) {
|
@@ -1264,7 +1367,7 @@ async function createService(projectRoot, opts, host = {
|
|
1264
1367
|
fs__default["default"].mkdirSync(fsPath__namespace["default"].dirname(fileName), {
|
1265
1368
|
recursive: true
|
1266
1369
|
});
|
1267
|
-
fs__default["default"].writeFileSync(fileName, data, encoding);
|
1370
|
+
fs__default["default"].writeFileSync(fileName, typeof data === "string" ? data : new Uint8Array(data), encoding);
|
1268
1371
|
},
|
1269
1372
|
rmFile: fs__default["default"].rmSync
|
1270
1373
|
}, loader) {
|
@@ -1371,7 +1474,7 @@ function encodeJwt(payload, sessionKey) {
|
|
1371
1474
|
}
|
1372
1475
|
|
1373
1476
|
/* eslint-disable @typescript-eslint/no-unused-vars */
|
1374
|
-
const textEncoder$
|
1477
|
+
const textEncoder$3 = new TextEncoder();
|
1375
1478
|
const jsonOps = new patch.JSONOps();
|
1376
1479
|
const tsOps = new TSOps(document => {
|
1377
1480
|
return fp.pipe(analyzeValModule(document), fp.result.map(({
|
@@ -1401,12 +1504,12 @@ class ValOps {
|
|
1401
1504
|
if (typeof input === "object") {
|
1402
1505
|
return this.hashObject(input);
|
1403
1506
|
}
|
1404
|
-
return core.Internal.getSHA256Hash(textEncoder$
|
1507
|
+
return core.Internal.getSHA256Hash(textEncoder$3.encode(input));
|
1405
1508
|
}
|
1406
1509
|
hashObject(obj) {
|
1407
1510
|
const collector = [];
|
1408
1511
|
this.collectObjectRecursive(obj, collector);
|
1409
|
-
return core.Internal.getSHA256Hash(textEncoder$
|
1512
|
+
return core.Internal.getSHA256Hash(textEncoder$3.encode(collector.join("")));
|
1410
1513
|
}
|
1411
1514
|
collectObjectRecursive(item, collector) {
|
1412
1515
|
if (typeof item === "string") {
|
@@ -1575,7 +1678,10 @@ class ValOps {
|
|
1575
1678
|
for (const op of patch.patch) {
|
1576
1679
|
if (op.op === "file") {
|
1577
1680
|
const filePath = op.filePath;
|
1578
|
-
fileLastUpdatedByPatchId[filePath] =
|
1681
|
+
fileLastUpdatedByPatchId[filePath] = {
|
1682
|
+
patchId: patch.patchId,
|
1683
|
+
remote: op.remote
|
1684
|
+
};
|
1579
1685
|
}
|
1580
1686
|
const path = patch.path;
|
1581
1687
|
if (!patchesByModule[path]) {
|
@@ -1711,6 +1817,7 @@ class ValOps {
|
|
1711
1817
|
};
|
1712
1818
|
const errors = {};
|
1713
1819
|
const files = {};
|
1820
|
+
const remoteFiles = {};
|
1714
1821
|
const entries = Object.entries(schemas);
|
1715
1822
|
const modulePathsToValidate = patchesByModule && Object.keys(patchesByModule);
|
1716
1823
|
for (const [pathS, schema] of entries) {
|
@@ -1752,7 +1859,7 @@ class ValOps {
|
|
1752
1859
|
};
|
1753
1860
|
if (validationErrors) {
|
1754
1861
|
for (const validationError of validationErrors) {
|
1755
|
-
var _validationError$fixe;
|
1862
|
+
var _validationError$fixe, _validationError$fixe2, _validationError$fixe3;
|
1756
1863
|
if (isOnlyFileCheckValidationError(validationError)) {
|
1757
1864
|
if (files[sourcePath]) {
|
1758
1865
|
throw new Error("Cannot have multiple files with same path. Path: " + sourcePath + "; Module: " + path);
|
@@ -1761,7 +1868,9 @@ class ValOps {
|
|
1761
1868
|
if (isFileSource(value)) {
|
1762
1869
|
files[sourcePath] = value;
|
1763
1870
|
}
|
1764
|
-
} else if ((_validationError$fixe = validationError.fixes) !== null && _validationError$fixe !== void 0 && _validationError$fixe.includes("
|
1871
|
+
} else if ((_validationError$fixe = validationError.fixes) !== null && _validationError$fixe !== void 0 && _validationError$fixe.includes("image:check-remote") || (_validationError$fixe2 = validationError.fixes) !== null && _validationError$fixe2 !== void 0 && _validationError$fixe2.includes("file:check-remote")) {
|
1872
|
+
remoteFiles[sourcePath] = validationError.value;
|
1873
|
+
} else if ((_validationError$fixe3 = validationError.fixes) !== null && _validationError$fixe3 !== void 0 && _validationError$fixe3.includes("keyof:check-keys")) {
|
1765
1874
|
const TYPE_ERROR_MESSAGE = `This is most likely a Val version mismatch or Val bug.`;
|
1766
1875
|
if (!validationError.value) {
|
1767
1876
|
addError({
|
@@ -1810,9 +1919,14 @@ class ValOps {
|
|
1810
1919
|
}
|
1811
1920
|
return {
|
1812
1921
|
errors,
|
1813
|
-
files
|
1922
|
+
files,
|
1923
|
+
remoteFiles
|
1814
1924
|
};
|
1815
1925
|
}
|
1926
|
+
async validateRemoteFiles(schemas, sources, remoteFiles) {
|
1927
|
+
// TODO: Implement
|
1928
|
+
return {};
|
1929
|
+
}
|
1816
1930
|
|
1817
1931
|
// #region validateFiles
|
1818
1932
|
async validateFiles(schemas, sources, files, fileLastUpdatedByPatchId) {
|
@@ -1860,13 +1974,13 @@ class ValOps {
|
|
1860
1974
|
}
|
1861
1975
|
const type = schemaAtPath instanceof core.ImageSchema ? "image" : "file";
|
1862
1976
|
const filePath = value[core.FILE_REF_PROP];
|
1863
|
-
const
|
1977
|
+
const fileData = (fileLastUpdatedByPatchId === null || fileLastUpdatedByPatchId === void 0 ? void 0 : fileLastUpdatedByPatchId[filePath]) || null;
|
1864
1978
|
let metadata;
|
1865
1979
|
let metadataErrors;
|
1866
1980
|
|
1867
1981
|
// TODO: refactor so we call get metadata once instead of iterating like this. Reason: should be a lot faster
|
1868
|
-
if (
|
1869
|
-
const patchFileMetadata = await this.getBase64EncodedBinaryFileMetadataFromPatch(filePath, type, patchId);
|
1982
|
+
if (fileData) {
|
1983
|
+
const patchFileMetadata = await this.getBase64EncodedBinaryFileMetadataFromPatch(filePath, type, fileData.patchId, fileData.remote);
|
1870
1984
|
if (patchFileMetadata.errors) {
|
1871
1985
|
metadataErrors = patchFileMetadata.errors;
|
1872
1986
|
} else {
|
@@ -1886,7 +2000,7 @@ class ValOps {
|
|
1886
2000
|
message: e.message,
|
1887
2001
|
value: {
|
1888
2002
|
filePath,
|
1889
|
-
patchId
|
2003
|
+
patchId: (fileData === null || fileData === void 0 ? void 0 : fileData.patchId) ?? null
|
1890
2004
|
}
|
1891
2005
|
}))
|
1892
2006
|
};
|
@@ -2073,13 +2187,18 @@ class ValOps {
|
|
2073
2187
|
}
|
2074
2188
|
const patchedBinaryFilesDescriptors = {};
|
2075
2189
|
const binaryFilePatchErrors = {};
|
2076
|
-
await Promise.all(Object.entries(fileLastUpdatedByPatchId).map(async ([filePath,
|
2190
|
+
await Promise.all(Object.entries(fileLastUpdatedByPatchId).map(async ([filePath, patchData]) => {
|
2191
|
+
const {
|
2192
|
+
patchId,
|
2193
|
+
remote
|
2194
|
+
} = patchData;
|
2077
2195
|
if (globalAppliedPatches.includes(patchId)) {
|
2078
2196
|
// TODO: do we want to make sure the file is there? Then again, it should be rare that it happens (unless there's a Val bug) so it might be enough to fail later (at commit)
|
2079
2197
|
// TODO: include sha256? This way we can make sure we pick the right file since theoretically there could be multiple files with the same path in the same patch
|
2080
2198
|
// or is that the case? We are picking the latest file by path so, that should be enough?
|
2081
2199
|
patchedBinaryFilesDescriptors[filePath] = {
|
2082
|
-
patchId
|
2200
|
+
patchId,
|
2201
|
+
remote
|
2083
2202
|
};
|
2084
2203
|
} else {
|
2085
2204
|
hasErrors = true;
|
@@ -2174,18 +2293,20 @@ class ValOps {
|
|
2174
2293
|
error: new patch.PatchError("Value is not a string")
|
2175
2294
|
};
|
2176
2295
|
} else {
|
2177
|
-
const sha256 = core.Internal.getSHA256Hash(textEncoder$
|
2296
|
+
const sha256 = core.Internal.getSHA256Hash(textEncoder$3.encode(value));
|
2178
2297
|
files[filePath] = {
|
2179
2298
|
value,
|
2180
2299
|
sha256,
|
2181
|
-
path: op.path
|
2300
|
+
path: op.path,
|
2301
|
+
remote: op.remote
|
2182
2302
|
};
|
2183
2303
|
sourceFileOps.push({
|
2184
2304
|
op: "file",
|
2185
2305
|
path: op.path,
|
2186
2306
|
filePath,
|
2187
2307
|
nestedFilePath: op.nestedFilePath,
|
2188
|
-
value: sha256
|
2308
|
+
value: sha256,
|
2309
|
+
remote: op.remote
|
2189
2310
|
});
|
2190
2311
|
}
|
2191
2312
|
}
|
@@ -2268,7 +2389,7 @@ class ValOps {
|
|
2268
2389
|
const MaxRetries = 3;
|
2269
2390
|
let lastRes;
|
2270
2391
|
for (let i = 0; i < MaxRetries; i++) {
|
2271
|
-
lastRes = await this.saveBase64EncodedBinaryFileFromPatch(filePath, parentRef, patchId, data.value, type, metadataOps.metadata);
|
2392
|
+
lastRes = await this.saveBase64EncodedBinaryFileFromPatch(filePath, parentRef, patchId, data.value, type, metadataOps.metadata, data.remote);
|
2272
2393
|
if (!lastRes.error) {
|
2273
2394
|
return {
|
2274
2395
|
filePath
|
@@ -2300,8 +2421,8 @@ class ValOps {
|
|
2300
2421
|
// #region abstract ops
|
2301
2422
|
}
|
2302
2423
|
function isOnlyFileCheckValidationError(validationError) {
|
2303
|
-
var _validationError$
|
2304
|
-
if ((_validationError$
|
2424
|
+
var _validationError$fixe4;
|
2425
|
+
if ((_validationError$fixe4 = validationError.fixes) !== null && _validationError$fixe4 !== void 0 && _validationError$fixe4.every(f => f === "file:check-metadata" || f === "image:check-metadata")) {
|
2305
2426
|
return true;
|
2306
2427
|
}
|
2307
2428
|
return false;
|
@@ -2328,7 +2449,7 @@ function createMetadataFromBuffer(type, mimeType, buffer) {
|
|
2328
2449
|
width,
|
2329
2450
|
height,
|
2330
2451
|
type
|
2331
|
-
} = sizeOf__default["default"](buffer);
|
2452
|
+
} = sizeOf__default["default"](new Uint8Array(buffer));
|
2332
2453
|
const normalizedType = type === "jpg" ? "jpeg" : type === "svg" ? "svg+xml" : type;
|
2333
2454
|
if (type !== undefined && `image/${normalizedType}` !== mimeType) {
|
2334
2455
|
return {
|
@@ -2437,6 +2558,74 @@ function computeChangedPatchParentRefs(currentPatches, deletePatchIds) {
|
|
2437
2558
|
};
|
2438
2559
|
}
|
2439
2560
|
|
2561
|
+
function getFileExt(filePath) {
|
2562
|
+
// NOTE: We do not import the path module. This code is copied in different projects. We want the same implementation and which means that this might running in browser where path is not available).
|
2563
|
+
return filePath.split(".").pop() || "";
|
2564
|
+
}
|
2565
|
+
|
2566
|
+
const textEncoder$2 = new TextEncoder();
|
2567
|
+
async function uploadRemoteFile(remoteHost, fileBuffer, publicProjectId, bucket, filePath, schema, metadata, auth) {
|
2568
|
+
const fileHash = core.Internal.remote.getFileHash(fileBuffer);
|
2569
|
+
const coreVersion = core.Internal.VERSION.core || "unknown";
|
2570
|
+
const fileExt = getFileExt(filePath);
|
2571
|
+
const ref = core.Internal.remote.createRemoteRef(remoteHost, {
|
2572
|
+
publicProjectId,
|
2573
|
+
coreVersion,
|
2574
|
+
bucket,
|
2575
|
+
validationHash: core.Internal.remote.getValidationHash(coreVersion, schema, fileExt, metadata, fileHash, textEncoder$2),
|
2576
|
+
fileHash,
|
2577
|
+
filePath
|
2578
|
+
});
|
2579
|
+
return uploadRemoteRef(fileBuffer, ref, auth);
|
2580
|
+
}
|
2581
|
+
async function uploadRemoteRef(fileBuffer, ref, auth) {
|
2582
|
+
const authHeader = "apiKey" in auth ? {
|
2583
|
+
Authorization: `Bearer ${auth.apiKey}`
|
2584
|
+
} : {
|
2585
|
+
"x-val-pat": auth.pat
|
2586
|
+
};
|
2587
|
+
const res = await fetch(ref, {
|
2588
|
+
method: "PUT",
|
2589
|
+
headers: {
|
2590
|
+
...authHeader,
|
2591
|
+
"Content-Type": "application/octet-stream"
|
2592
|
+
},
|
2593
|
+
body: fileBuffer
|
2594
|
+
});
|
2595
|
+
if (!res.ok) {
|
2596
|
+
var _res$headers$get;
|
2597
|
+
if (res.status === 409) {
|
2598
|
+
// File already exists
|
2599
|
+
return {
|
2600
|
+
success: true,
|
2601
|
+
ref
|
2602
|
+
};
|
2603
|
+
}
|
2604
|
+
if ((_res$headers$get = res.headers.get("content-type")) !== null && _res$headers$get !== void 0 && _res$headers$get.includes("application/json")) {
|
2605
|
+
const json = await res.json();
|
2606
|
+
if (json.message) {
|
2607
|
+
return {
|
2608
|
+
success: false,
|
2609
|
+
error: `${ref}. Failed to upload file: ${json.message}.`
|
2610
|
+
};
|
2611
|
+
} else {
|
2612
|
+
return {
|
2613
|
+
success: false,
|
2614
|
+
error: `${ref}. Failed to upload file: ${JSON.stringify(json)}.`
|
2615
|
+
};
|
2616
|
+
}
|
2617
|
+
}
|
2618
|
+
return {
|
2619
|
+
success: false,
|
2620
|
+
error: `${ref}. Failed to upload file: ${await res.text()}.`
|
2621
|
+
};
|
2622
|
+
}
|
2623
|
+
return {
|
2624
|
+
success: true,
|
2625
|
+
ref
|
2626
|
+
};
|
2627
|
+
}
|
2628
|
+
|
2440
2629
|
class ValOpsFS extends ValOps {
|
2441
2630
|
static VAL_DIR = ".val";
|
2442
2631
|
constructor(rootDir, valModules, options) {
|
@@ -2878,10 +3067,10 @@ class ValOpsFS extends ValOps {
|
|
2878
3067
|
};
|
2879
3068
|
}
|
2880
3069
|
}
|
2881
|
-
async saveBase64EncodedBinaryFileFromPatch(filePath, parentRef, patchId, data, _type, metadata) {
|
3070
|
+
async saveBase64EncodedBinaryFileFromPatch(filePath, parentRef, patchId, data, _type, metadata, remote) {
|
2882
3071
|
const patchDir = this.getParentPatchIdFromParentRef(parentRef);
|
2883
|
-
const patchFilePath = this.getBinaryFilePath(filePath, patchDir);
|
2884
|
-
const metadataFilePath = this.getBinaryFileMetadataPath(filePath, patchDir);
|
3072
|
+
const patchFilePath = this.getBinaryFilePath(filePath, patchDir, remote);
|
3073
|
+
const metadataFilePath = this.getBinaryFileMetadataPath(filePath, patchDir, remote);
|
2885
3074
|
try {
|
2886
3075
|
const buffer = bufferFromDataUrl(data);
|
2887
3076
|
if (!buffer) {
|
@@ -2912,7 +3101,7 @@ class ValOpsFS extends ValOps {
|
|
2912
3101
|
};
|
2913
3102
|
}
|
2914
3103
|
}
|
2915
|
-
async getBase64EncodedBinaryFileMetadataFromPatch(filePath, type, patchId) {
|
3104
|
+
async getBase64EncodedBinaryFileMetadataFromPatch(filePath, type, patchId, remote) {
|
2916
3105
|
const patchDirRes = await this.getParentPatchIdFromPatchId(patchId);
|
2917
3106
|
if (fp.result.isErr(patchDirRes)) {
|
2918
3107
|
return {
|
@@ -2921,7 +3110,7 @@ class ValOpsFS extends ValOps {
|
|
2921
3110
|
}]
|
2922
3111
|
};
|
2923
3112
|
}
|
2924
|
-
const metadataFilePath = this.getBinaryFileMetadataPath(filePath, patchDirRes.value);
|
3113
|
+
const metadataFilePath = this.getBinaryFileMetadataPath(filePath, patchDirRes.value, remote);
|
2925
3114
|
if (!this.host.fileExists(metadataFilePath)) {
|
2926
3115
|
return {
|
2927
3116
|
errors: [{
|
@@ -2961,7 +3150,10 @@ class ValOpsFS extends ValOps {
|
|
2961
3150
|
if (!fp.result.isOk(patchDirRes)) {
|
2962
3151
|
return null;
|
2963
3152
|
}
|
2964
|
-
const absPath = this.getBinaryFilePath(filePath, patchDirRes.value
|
3153
|
+
const absPath = this.getBinaryFilePath(filePath, patchDirRes.value,
|
3154
|
+
// We save remote remote files using the filepath (so not the remote reference) and we also retrieve them using the filepath. Therefore remote is always false
|
3155
|
+
false // remote = false
|
3156
|
+
);
|
2965
3157
|
if (!this.host.fileExists(absPath)) {
|
2966
3158
|
return null;
|
2967
3159
|
}
|
@@ -3025,43 +3217,90 @@ class ValOpsFS extends ValOps {
|
|
3025
3217
|
}
|
3026
3218
|
}
|
3027
3219
|
}
|
3028
|
-
async
|
3220
|
+
async saveOrUploadFiles(preparedCommit, auth, testMode) {
|
3029
3221
|
const updatedFiles = [];
|
3222
|
+
const uploadedRemoteRefs = [];
|
3030
3223
|
const errors = {};
|
3031
|
-
|
3032
|
-
|
3033
|
-
|
3034
|
-
|
3035
|
-
|
3036
|
-
|
3037
|
-
|
3038
|
-
|
3039
|
-
|
3224
|
+
const remoteFileDescriptors = Object.entries(preparedCommit.patchedBinaryFilesDescriptors).filter(([, {
|
3225
|
+
remote
|
3226
|
+
}]) => remote).map(([ref, {
|
3227
|
+
patchId
|
3228
|
+
}]) => [ref, {
|
3229
|
+
patchId
|
3230
|
+
}]);
|
3231
|
+
const localFileDescriptors = Object.entries(preparedCommit.patchedBinaryFilesDescriptors).filter(([, {
|
3232
|
+
remote
|
3233
|
+
}]) => !remote).map(([ref, {
|
3234
|
+
patchId
|
3235
|
+
}]) => [ref, {
|
3236
|
+
patchId
|
3237
|
+
}]);
|
3238
|
+
for (const [ref, {
|
3239
|
+
patchId
|
3240
|
+
}] of remoteFileDescriptors) {
|
3241
|
+
const splitRemoteRefRes = core.Internal.remote.splitRemoteRef(ref);
|
3242
|
+
if (splitRemoteRefRes.status === "error") {
|
3243
|
+
errors[ref] = {
|
3244
|
+
message: "Failed to split remote ref: " + ref
|
3245
|
+
};
|
3246
|
+
continue;
|
3247
|
+
}
|
3248
|
+
const fileBuffer = await this.getBase64EncodedBinaryFileFromPatch(splitRemoteRefRes.filePath, patchId);
|
3249
|
+
if (!fileBuffer) {
|
3250
|
+
errors[ref] = {
|
3251
|
+
message: "Failed to get binary file from patch. Ref: " + ref + ". PatchId: " + patchId
|
3040
3252
|
};
|
3253
|
+
continue;
|
3254
|
+
}
|
3255
|
+
if (testMode === "test-skip-remote") {
|
3256
|
+
console.log("Skip remote flag enabled. Skipping file upload", ref);
|
3257
|
+
continue;
|
3041
3258
|
}
|
3259
|
+
console.log("Uploading remote file", ref);
|
3260
|
+
const res = await uploadRemoteRef(fileBuffer, ref, auth);
|
3261
|
+
if (!res.success) {
|
3262
|
+
console.error("Failed to upload remote file", ref, res.error);
|
3263
|
+
throw new Error(`Failed to upload remote file: ${ref}. ${res.error}`);
|
3264
|
+
}
|
3265
|
+
console.log("Completed remote file", ref);
|
3266
|
+
uploadedRemoteRefs.push(ref);
|
3042
3267
|
}
|
3043
3268
|
const patchIdToPatchDirMapRes = await this.getParentPatchIdFromPatchIdMap();
|
3044
3269
|
if (fp.result.isErr(patchIdToPatchDirMapRes)) {
|
3045
3270
|
return {
|
3046
3271
|
updatedFiles,
|
3272
|
+
uploadedRemoteRefs,
|
3047
3273
|
errors
|
3048
3274
|
};
|
3049
3275
|
}
|
3050
3276
|
const patchIdToPatchDirMap = patchIdToPatchDirMapRes.value;
|
3051
|
-
for (const [
|
3277
|
+
for (const [ref, {
|
3052
3278
|
patchId
|
3053
|
-
}] of
|
3279
|
+
}] of localFileDescriptors) {
|
3280
|
+
const filePath = ref;
|
3054
3281
|
const absPath = fsPath__namespace["default"].join(this.rootDir, ...filePath.split("/"));
|
3055
3282
|
try {
|
3056
3283
|
const patchDir = patchIdToPatchDirMap[patchId];
|
3057
3284
|
if (!patchDir) {
|
3058
3285
|
errors[absPath] = {
|
3059
3286
|
message: "Failed to find PatchDir for PatchId " + patchId,
|
3060
|
-
filePath
|
3287
|
+
filePath: filePath
|
3061
3288
|
};
|
3062
3289
|
continue;
|
3063
3290
|
}
|
3064
|
-
this.host.copyFile(this.getBinaryFilePath(filePath, patchDir), absPath);
|
3291
|
+
this.host.copyFile(this.getBinaryFilePath(filePath, patchDir, false), absPath);
|
3292
|
+
updatedFiles.push(absPath);
|
3293
|
+
} catch (err) {
|
3294
|
+
errors[absPath] = {
|
3295
|
+
message: err instanceof Error ? err.message : "Unknown error",
|
3296
|
+
filePath: filePath
|
3297
|
+
};
|
3298
|
+
}
|
3299
|
+
}
|
3300
|
+
for (const [filePath, data] of Object.entries(preparedCommit.patchedSourceFiles)) {
|
3301
|
+
const absPath = fsPath__namespace["default"].join(this.rootDir, ...filePath.split("/"));
|
3302
|
+
try {
|
3303
|
+
this.host.writeUf8File(absPath, data);
|
3065
3304
|
updatedFiles.push(absPath);
|
3066
3305
|
} catch (err) {
|
3067
3306
|
errors[absPath] = {
|
@@ -3094,6 +3333,7 @@ class ValOpsFS extends ValOps {
|
|
3094
3333
|
}
|
3095
3334
|
return {
|
3096
3335
|
updatedFiles,
|
3336
|
+
uploadedRemoteRefs,
|
3097
3337
|
errors
|
3098
3338
|
};
|
3099
3339
|
}
|
@@ -3162,10 +3402,26 @@ class ValOpsFS extends ValOps {
|
|
3162
3402
|
getFullPatchDir(patchDir) {
|
3163
3403
|
return fsPath__namespace["default"].join(this.getPatchesDir(), patchDir);
|
3164
3404
|
}
|
3165
|
-
getBinaryFilePath(filePath, patchDir) {
|
3405
|
+
getBinaryFilePath(filePath, patchDir, remote) {
|
3406
|
+
if (remote) {
|
3407
|
+
const res = core.Internal.remote.splitRemoteRef(filePath);
|
3408
|
+
if (res.status === "error") {
|
3409
|
+
throw new Error("Failed to split remote ref: " + filePath);
|
3410
|
+
}
|
3411
|
+
const actualFilePath = res.filePath;
|
3412
|
+
return fsPath__namespace["default"].join(this.getFullPatchDir(patchDir), "files", actualFilePath, fsPath__namespace["default"].basename(actualFilePath));
|
3413
|
+
}
|
3166
3414
|
return fsPath__namespace["default"].join(this.getFullPatchDir(patchDir), "files", filePath, fsPath__namespace["default"].basename(filePath));
|
3167
3415
|
}
|
3168
|
-
getBinaryFileMetadataPath(filePath, patchDir) {
|
3416
|
+
getBinaryFileMetadataPath(filePath, patchDir, remote) {
|
3417
|
+
if (remote) {
|
3418
|
+
const res = core.Internal.remote.splitRemoteRef(filePath);
|
3419
|
+
if (res.status === "error") {
|
3420
|
+
throw new Error("Failed to split remote ref (in metadata path): " + filePath);
|
3421
|
+
}
|
3422
|
+
const actualFilePath = res.filePath;
|
3423
|
+
return fsPath__namespace["default"].join(this.getFullPatchDir(patchDir), "files", actualFilePath, fsPath__namespace["default"].basename(actualFilePath));
|
3424
|
+
}
|
3169
3425
|
return fsPath__namespace["default"].join(this.getFullPatchDir(patchDir), "files", filePath, "metadata.json");
|
3170
3426
|
}
|
3171
3427
|
getPatchFilePath(patchDir) {
|
@@ -3245,7 +3501,7 @@ class FSOpsHost {
|
|
3245
3501
|
fs__default["default"].mkdirSync(fsPath__namespace["default"].dirname(path), {
|
3246
3502
|
recursive: true
|
3247
3503
|
});
|
3248
|
-
fs__default["default"].writeFileSync(path, data, "base64url");
|
3504
|
+
fs__default["default"].writeFileSync(path, new Uint8Array(data), "base64url");
|
3249
3505
|
}
|
3250
3506
|
copyFile(from, to) {
|
3251
3507
|
fs__default["default"].mkdirSync(fsPath__namespace["default"].dirname(to), {
|
@@ -3269,7 +3525,7 @@ const FSPatchBase = zod.z.object({
|
|
3269
3525
|
timestamp: zod.z.string().datetime()
|
3270
3526
|
});
|
3271
3527
|
|
3272
|
-
const textEncoder = new TextEncoder();
|
3528
|
+
const textEncoder$1 = new TextEncoder();
|
3273
3529
|
const PatchId = zod.z.string().refine(s => !!s); // TODO: validate
|
3274
3530
|
const CommitSha = zod.z.string().refine(s => !!s); // TODO: validate
|
3275
3531
|
zod.z.string().refine(s => !!s); // TODO: validate
|
@@ -3316,7 +3572,8 @@ const FilesResponse = zod.z.object({
|
|
3316
3572
|
filePath: zod.z.string(),
|
3317
3573
|
location: zod.z.literal("patch"),
|
3318
3574
|
patchId: PatchId,
|
3319
|
-
value: zod.z.string()
|
3575
|
+
value: zod.z.string(),
|
3576
|
+
remote: zod.z.boolean()
|
3320
3577
|
}), zod.z.object({
|
3321
3578
|
filePath: zod.z.string(),
|
3322
3579
|
location: zod.z.literal("repo"),
|
@@ -3327,7 +3584,8 @@ const FilesResponse = zod.z.object({
|
|
3327
3584
|
filePath: zod.z.string(),
|
3328
3585
|
location: zod.z.literal("patch"),
|
3329
3586
|
patchId: PatchId,
|
3330
|
-
message: zod.z.string()
|
3587
|
+
message: zod.z.string(),
|
3588
|
+
remote: zod.z.boolean()
|
3331
3589
|
}), zod.z.object({
|
3332
3590
|
filePath: zod.z.string(),
|
3333
3591
|
location: zod.z.literal("repo"),
|
@@ -3763,7 +4021,23 @@ class ValOpsHttp extends ValOps {
|
|
3763
4021
|
});
|
3764
4022
|
});
|
3765
4023
|
}
|
3766
|
-
async saveBase64EncodedBinaryFileFromPatch(
|
4024
|
+
async saveBase64EncodedBinaryFileFromPatch(filePathOrRef, _parentRef,
|
4025
|
+
// this is required for the FS implementation, but not for the HTTP implementation (patchId is enough)
|
4026
|
+
patchId, data, type, metadata, remote) {
|
4027
|
+
let filePath;
|
4028
|
+
if (remote) {
|
4029
|
+
const splitRemoteRefDataRes = core.Internal.remote.splitRemoteRef(filePathOrRef);
|
4030
|
+
if (splitRemoteRefDataRes.status === "error") {
|
4031
|
+
return {
|
4032
|
+
error: {
|
4033
|
+
message: `Could not split remote ref: ${splitRemoteRefDataRes.error}`
|
4034
|
+
}
|
4035
|
+
};
|
4036
|
+
}
|
4037
|
+
filePath = "/" + splitRemoteRefDataRes.filePath;
|
4038
|
+
} else {
|
4039
|
+
filePath = filePathOrRef;
|
4040
|
+
}
|
3767
4041
|
return fetch(`${this.contentUrl}/v1/${this.project}/patches/${patchId}/files`, {
|
3768
4042
|
method: "POST",
|
3769
4043
|
headers: {
|
@@ -3771,10 +4045,11 @@ class ValOpsHttp extends ValOps {
|
|
3771
4045
|
"Content-Type": "application/json"
|
3772
4046
|
},
|
3773
4047
|
body: JSON.stringify({
|
3774
|
-
filePath
|
4048
|
+
filePath,
|
3775
4049
|
data,
|
3776
4050
|
type,
|
3777
|
-
metadata
|
4051
|
+
metadata,
|
4052
|
+
remote
|
3778
4053
|
})
|
3779
4054
|
}).then(async res => {
|
3780
4055
|
if (res.ok) {
|
@@ -3812,7 +4087,7 @@ class ValOpsHttp extends ValOps {
|
|
3812
4087
|
});
|
3813
4088
|
params.set("body_sha",
|
3814
4089
|
// We use this for cache invalidation
|
3815
|
-
core.Internal.getSHA256Hash(textEncoder.encode(stringifiedFiles)));
|
4090
|
+
core.Internal.getSHA256Hash(textEncoder$1.encode(stringifiedFiles)));
|
3816
4091
|
return fetch(`${this.contentUrl}/v1/${this.project}/files?${params}`, {
|
3817
4092
|
method: "PUT",
|
3818
4093
|
// Yes, PUT is weird. Weirder to have a body in a GET request.
|
@@ -3872,39 +4147,53 @@ class ValOpsHttp extends ValOps {
|
|
3872
4147
|
}
|
3873
4148
|
async getBinaryFile(filePath) {
|
3874
4149
|
// We could also just get this from public/ on the running server. Current approach feels more clean, but will be slower / puts more server load... We might want to change this
|
3875
|
-
const
|
4150
|
+
const requestFiles = [];
|
4151
|
+
requestFiles.push({
|
3876
4152
|
filePath: filePath,
|
3877
4153
|
location: "repo",
|
3878
4154
|
root: this.root,
|
3879
4155
|
commitSha: this.commitSha
|
3880
|
-
}
|
4156
|
+
});
|
4157
|
+
const filesRes = await this.getHttpFiles(requestFiles);
|
3881
4158
|
if (filesRes.error) {
|
3882
4159
|
return null;
|
3883
4160
|
}
|
3884
|
-
const file = filesRes.files
|
4161
|
+
const file = filesRes.files[0];
|
4162
|
+
if (filesRes.files.length > 1) {
|
4163
|
+
console.error("Expected 1 file, got more:", filesRes.files);
|
4164
|
+
}
|
3885
4165
|
if (!file) {
|
3886
4166
|
return null;
|
3887
4167
|
}
|
3888
4168
|
return Buffer.from(file.value, "base64");
|
3889
4169
|
}
|
3890
|
-
async getBase64EncodedBinaryFileFromPatch(filePath, patchId) {
|
4170
|
+
async getBase64EncodedBinaryFileFromPatch(filePath, patchId, remote) {
|
3891
4171
|
const filesRes = await this.getHttpFiles([{
|
3892
4172
|
filePath: filePath,
|
3893
4173
|
location: "patch",
|
3894
|
-
patchId
|
4174
|
+
patchId,
|
4175
|
+
remote
|
3895
4176
|
}]);
|
3896
4177
|
if (filesRes.error) {
|
4178
|
+
console.error("Error getting file:", filePath, filesRes.error);
|
3897
4179
|
return null;
|
3898
4180
|
}
|
3899
|
-
|
4181
|
+
if (filesRes.errors) {
|
4182
|
+
console.error("Failed while retrieving files", filePath, filesRes.errors);
|
4183
|
+
}
|
4184
|
+
const file = filesRes.files[0];
|
4185
|
+
if (filesRes.files.length > 1) {
|
4186
|
+
console.error("Expected 1 file, got more:", filesRes.files);
|
4187
|
+
}
|
3900
4188
|
if (!file) {
|
3901
4189
|
return null;
|
3902
4190
|
}
|
3903
4191
|
return bufferFromDataUrl(file.value) ?? null;
|
3904
4192
|
}
|
3905
|
-
async getBase64EncodedBinaryFileMetadataFromPatch(filePath, type, patchId) {
|
4193
|
+
async getBase64EncodedBinaryFileMetadataFromPatch(filePath, type, patchId, remote) {
|
3906
4194
|
const params = new URLSearchParams();
|
3907
4195
|
params.set("file_path", filePath);
|
4196
|
+
params.set("remote", remote.toString());
|
3908
4197
|
try {
|
3909
4198
|
const metadataRes = await fetch(`${this.contentUrl}/v1/${this.project}/patches/${patchId}/files?${params}`, {
|
3910
4199
|
headers: {
|
@@ -4133,8 +4422,112 @@ class ValOpsHttp extends ValOps {
|
|
4133
4422
|
}
|
4134
4423
|
}
|
4135
4424
|
|
4425
|
+
const host = process.env.VAL_CONTENT_URL || "https://content.val.build";
|
4426
|
+
const SettingsSchema = zod.z.object({
|
4427
|
+
publicProjectId: zod.z.string(),
|
4428
|
+
remoteFileBuckets: zod.z.array(zod.z.object({
|
4429
|
+
bucket: zod.z.string()
|
4430
|
+
}))
|
4431
|
+
});
|
4432
|
+
async function getSettings(projectName, auth) {
|
4433
|
+
try {
|
4434
|
+
const response = await fetch(`${host}/v1/${projectName}/settings`, {
|
4435
|
+
headers: "pat" in auth ? {
|
4436
|
+
"x-val-pat": auth.pat,
|
4437
|
+
"Content-Type": "application/json"
|
4438
|
+
} : {
|
4439
|
+
Authorization: `Bearer ${auth.apiKey}`,
|
4440
|
+
"Content-Type": "application/json"
|
4441
|
+
}
|
4442
|
+
});
|
4443
|
+
if (response.status === 404) {
|
4444
|
+
return {
|
4445
|
+
success: false,
|
4446
|
+
message: `Project ${projectName} not found. Verify that user has access to project and that it exists.`
|
4447
|
+
};
|
4448
|
+
}
|
4449
|
+
if (response.status !== 200) {
|
4450
|
+
return {
|
4451
|
+
success: false,
|
4452
|
+
message: `Failed to get project id: ${response.statusText}`
|
4453
|
+
};
|
4454
|
+
}
|
4455
|
+
const json = await response.json();
|
4456
|
+
const parseRes = SettingsSchema.safeParse(json);
|
4457
|
+
if (!parseRes.success) {
|
4458
|
+
return {
|
4459
|
+
success: false,
|
4460
|
+
message: `Failed to parse settings data: ${parseRes.error.message}`
|
4461
|
+
};
|
4462
|
+
}
|
4463
|
+
return {
|
4464
|
+
success: true,
|
4465
|
+
data: parseRes.data
|
4466
|
+
};
|
4467
|
+
} catch (err) {
|
4468
|
+
return {
|
4469
|
+
success: false,
|
4470
|
+
message: `Failed to get project id. Check network connection and try again.`
|
4471
|
+
};
|
4472
|
+
}
|
4473
|
+
}
|
4474
|
+
|
4475
|
+
function getPersonalAccessTokenPath(root) {
|
4476
|
+
return fsPath__namespace["default"].join(fsPath__namespace["default"].resolve(root), ".val", "pat.json");
|
4477
|
+
}
|
4478
|
+
function parsePersonalAccessTokenFile(content) {
|
4479
|
+
if (!content) {
|
4480
|
+
return {
|
4481
|
+
success: false,
|
4482
|
+
error: "Invalid content: undefined"
|
4483
|
+
};
|
4484
|
+
}
|
4485
|
+
let patFileContent;
|
4486
|
+
try {
|
4487
|
+
patFileContent = JSON.parse(content);
|
4488
|
+
} catch {
|
4489
|
+
return {
|
4490
|
+
success: false,
|
4491
|
+
error: `Invalid content: file is not a valid JSON file`
|
4492
|
+
};
|
4493
|
+
}
|
4494
|
+
if (typeof patFileContent !== "object") {
|
4495
|
+
return {
|
4496
|
+
success: false,
|
4497
|
+
error: "Invalid content: not an object"
|
4498
|
+
};
|
4499
|
+
}
|
4500
|
+
if (!patFileContent) {
|
4501
|
+
return {
|
4502
|
+
success: false,
|
4503
|
+
error: "Invalid content: null"
|
4504
|
+
};
|
4505
|
+
}
|
4506
|
+
if (!("pat" in patFileContent)) {
|
4507
|
+
return {
|
4508
|
+
success: false,
|
4509
|
+
error: "Invalid content: key 'pat' is missing"
|
4510
|
+
};
|
4511
|
+
}
|
4512
|
+
const patField = patFileContent.pat;
|
4513
|
+
if (typeof patField === "string") {
|
4514
|
+
return {
|
4515
|
+
success: true,
|
4516
|
+
data: {
|
4517
|
+
pat: patField
|
4518
|
+
}
|
4519
|
+
};
|
4520
|
+
} else {
|
4521
|
+
return {
|
4522
|
+
success: false,
|
4523
|
+
error: "Invalid content: pat is not a string"
|
4524
|
+
};
|
4525
|
+
}
|
4526
|
+
}
|
4527
|
+
|
4136
4528
|
/* eslint-disable @typescript-eslint/no-unused-vars */
|
4137
4529
|
const ValServer = (valModules, options, callbacks) => {
|
4530
|
+
process.env.VAL_REMOTE_HOST || core.DEFAULT_VAL_REMOTE_HOST;
|
4138
4531
|
let serverOps;
|
4139
4532
|
if (options.mode === "fs") {
|
4140
4533
|
serverOps = new ValOpsFS(options.cwd, valModules, {
|
@@ -4288,6 +4681,76 @@ const ValServer = (valModules, options, callbacks) => {
|
|
4288
4681
|
redirectTo: appAuthorizeUrl
|
4289
4682
|
};
|
4290
4683
|
};
|
4684
|
+
let remoteFileAuth = null;
|
4685
|
+
const getRemoteFileAuth = async () => {
|
4686
|
+
if (remoteFileAuth) {
|
4687
|
+
return {
|
4688
|
+
status: 200,
|
4689
|
+
json: {
|
4690
|
+
remoteFileAuth
|
4691
|
+
}
|
4692
|
+
};
|
4693
|
+
}
|
4694
|
+
if (options.apiKey) {
|
4695
|
+
remoteFileAuth = {
|
4696
|
+
apiKey: options.apiKey
|
4697
|
+
};
|
4698
|
+
} else if (serverOps instanceof ValOpsFS) {
|
4699
|
+
const projectRootDir = options.config.root;
|
4700
|
+
if (!projectRootDir) {
|
4701
|
+
return {
|
4702
|
+
status: 400,
|
4703
|
+
json: {
|
4704
|
+
errorCode: "project-not-configured",
|
4705
|
+
message: "Root directory was empty"
|
4706
|
+
}
|
4707
|
+
};
|
4708
|
+
}
|
4709
|
+
const fs = await Promise.resolve().then(function () { return /*#__PURE__*/_interopNamespace(require('fs')); });
|
4710
|
+
const patPath = getPersonalAccessTokenPath(fsPath__namespace["default"].join(process.cwd()));
|
4711
|
+
let patFile;
|
4712
|
+
try {
|
4713
|
+
patFile = await fs.promises.readFile(patPath, "utf-8");
|
4714
|
+
} catch (err) {
|
4715
|
+
return {
|
4716
|
+
status: 400,
|
4717
|
+
json: {
|
4718
|
+
errorCode: "pat-error",
|
4719
|
+
message: "Could not read personal access token file"
|
4720
|
+
}
|
4721
|
+
};
|
4722
|
+
}
|
4723
|
+
const patRes = parsePersonalAccessTokenFile(patFile);
|
4724
|
+
if (patRes.success) {
|
4725
|
+
remoteFileAuth = {
|
4726
|
+
pat: patRes.data.pat
|
4727
|
+
};
|
4728
|
+
} else {
|
4729
|
+
return {
|
4730
|
+
status: 400,
|
4731
|
+
json: {
|
4732
|
+
errorCode: "pat-error",
|
4733
|
+
message: "Could not parse personal access token file"
|
4734
|
+
}
|
4735
|
+
};
|
4736
|
+
}
|
4737
|
+
}
|
4738
|
+
if (!remoteFileAuth) {
|
4739
|
+
return {
|
4740
|
+
status: 400,
|
4741
|
+
json: {
|
4742
|
+
errorCode: "project-not-configured",
|
4743
|
+
message: "Remote file auth is not configured"
|
4744
|
+
}
|
4745
|
+
};
|
4746
|
+
}
|
4747
|
+
return {
|
4748
|
+
status: 200,
|
4749
|
+
json: {
|
4750
|
+
remoteFileAuth
|
4751
|
+
}
|
4752
|
+
};
|
4753
|
+
};
|
4291
4754
|
return {
|
4292
4755
|
"/draft/enable": {
|
4293
4756
|
GET: async req => {
|
@@ -4601,6 +5064,63 @@ const ValServer = (valModules, options, callbacks) => {
|
|
4601
5064
|
};
|
4602
5065
|
}
|
4603
5066
|
},
|
5067
|
+
"/remote/settings": {
|
5068
|
+
GET: async req => {
|
5069
|
+
const cookies = req.cookies;
|
5070
|
+
const auth = getAuth(cookies);
|
5071
|
+
if (auth.error) {
|
5072
|
+
return {
|
5073
|
+
status: 401,
|
5074
|
+
json: {
|
5075
|
+
errorCode: "unauthorized",
|
5076
|
+
message: auth.error
|
5077
|
+
}
|
5078
|
+
};
|
5079
|
+
}
|
5080
|
+
if (serverOps instanceof ValOpsHttp && !("id" in auth)) {
|
5081
|
+
return {
|
5082
|
+
status: 401,
|
5083
|
+
json: {
|
5084
|
+
errorCode: "unauthorized",
|
5085
|
+
message: "Unauthorized"
|
5086
|
+
}
|
5087
|
+
};
|
5088
|
+
}
|
5089
|
+
if (!options.project) {
|
5090
|
+
return {
|
5091
|
+
status: 400,
|
5092
|
+
json: {
|
5093
|
+
errorCode: "project-not-configured",
|
5094
|
+
message: "Project is not configured. Update your val.config file with a valid remote project"
|
5095
|
+
}
|
5096
|
+
};
|
5097
|
+
}
|
5098
|
+
const remoteFileAuthRes = await getRemoteFileAuth();
|
5099
|
+
if (remoteFileAuthRes.status !== 200) {
|
5100
|
+
return remoteFileAuthRes;
|
5101
|
+
}
|
5102
|
+
const remoteFileAuth = remoteFileAuthRes.json.remoteFileAuth;
|
5103
|
+
const settingsRes = await getSettings(options.project, remoteFileAuth);
|
5104
|
+
if (!settingsRes.success) {
|
5105
|
+
console.warn("Could not get public project id: " + settingsRes.message);
|
5106
|
+
return {
|
5107
|
+
status: 400,
|
5108
|
+
json: {
|
5109
|
+
errorCode: "error-could-not-get-settings",
|
5110
|
+
message: `Could not get settings: ${settingsRes.message}`
|
5111
|
+
}
|
5112
|
+
};
|
5113
|
+
}
|
5114
|
+
return {
|
5115
|
+
status: 200,
|
5116
|
+
json: {
|
5117
|
+
publicProjectId: settingsRes.data.publicProjectId,
|
5118
|
+
remoteFileBuckets: settingsRes.data.remoteFileBuckets,
|
5119
|
+
coreVersion: core.Internal.VERSION.core || "unknown"
|
5120
|
+
}
|
5121
|
+
};
|
5122
|
+
}
|
5123
|
+
},
|
4604
5124
|
//#region stat
|
4605
5125
|
"/stat": {
|
4606
5126
|
POST: async req => {
|
@@ -4960,7 +5480,8 @@ const ValServer = (valModules, options, callbacks) => {
|
|
4960
5480
|
};
|
4961
5481
|
let sourcesValidation = {
|
4962
5482
|
errors: {},
|
4963
|
-
files: {}
|
5483
|
+
files: {},
|
5484
|
+
remoteFiles: {}
|
4964
5485
|
};
|
4965
5486
|
if (query.validate_sources || query.validate_binary_files) {
|
4966
5487
|
const schemas = await serverOps.getSchemas();
|
@@ -4969,6 +5490,8 @@ const ValServer = (valModules, options, callbacks) => {
|
|
4969
5490
|
// TODO: send binary files validation errors
|
4970
5491
|
if (query.validate_binary_files) {
|
4971
5492
|
await serverOps.validateFiles(schemas, sourcesRes.sources, sourcesValidation.files);
|
5493
|
+
// TODO : actually do something with this
|
5494
|
+
await serverOps.validateRemoteFiles(schemas, sourcesRes.sources, sourcesValidation.remoteFiles);
|
4972
5495
|
}
|
4973
5496
|
}
|
4974
5497
|
const schemaSha = await serverOps.getSchemaSha();
|
@@ -5140,7 +5663,12 @@ const ValServer = (valModules, options, callbacks) => {
|
|
5140
5663
|
};
|
5141
5664
|
}
|
5142
5665
|
if (serverOps instanceof ValOpsFS) {
|
5143
|
-
await
|
5666
|
+
const remoteFileAuthRes = await getRemoteFileAuth();
|
5667
|
+
if (remoteFileAuthRes.status !== 200) {
|
5668
|
+
return remoteFileAuthRes;
|
5669
|
+
}
|
5670
|
+
const remoteFileAuth = remoteFileAuthRes.json.remoteFileAuth;
|
5671
|
+
await serverOps.saveOrUploadFiles(preparedCommit, remoteFileAuth);
|
5144
5672
|
await serverOps.deletePatches(patchIds);
|
5145
5673
|
return {
|
5146
5674
|
status: 200,
|
@@ -5216,11 +5744,16 @@ const ValServer = (valModules, options, callbacks) => {
|
|
5216
5744
|
let cacheControl;
|
5217
5745
|
let fileBuffer;
|
5218
5746
|
let mimeType;
|
5747
|
+
const remote = query.remote === "true";
|
5219
5748
|
if (query.patch_id) {
|
5220
|
-
fileBuffer = await serverOps.getBase64EncodedBinaryFileFromPatch(filePath, query.patch_id);
|
5749
|
+
fileBuffer = await serverOps.getBase64EncodedBinaryFileFromPatch(filePath, query.patch_id, remote);
|
5221
5750
|
mimeType = core.Internal.filenameToMimeType(filePath);
|
5222
|
-
|
5751
|
+
// TODO: reenable this:
|
5752
|
+
// cacheControl = "public, max-age=20000, immutable";
|
5223
5753
|
} else {
|
5754
|
+
if (serverOps instanceof ValOpsHttp && remote) {
|
5755
|
+
console.error(`Remote file: ${filePath} requested without patch id. This is most likely a bug in Val.`);
|
5756
|
+
}
|
5224
5757
|
fileBuffer = await serverOps.getBinaryFile(filePath);
|
5225
5758
|
}
|
5226
5759
|
if (fileBuffer) {
|
@@ -5967,7 +6500,7 @@ class ValFSHost {
|
|
5967
6500
|
}
|
5968
6501
|
|
5969
6502
|
async function extractImageMetadata(filename, input) {
|
5970
|
-
const imageSize = sizeOf__default["default"](input);
|
6503
|
+
const imageSize = sizeOf__default["default"](new Uint8Array(input));
|
5971
6504
|
let mimeType = null;
|
5972
6505
|
if (imageSize.type) {
|
5973
6506
|
const possibleMimeType = `image/${imageSize.type}`;
|
@@ -6017,33 +6550,189 @@ function getValidationErrorFileRef(validationError) {
|
|
6017
6550
|
return maybeRef;
|
6018
6551
|
}
|
6019
6552
|
|
6020
|
-
|
6021
|
-
async function
|
6022
|
-
|
6023
|
-
|
6024
|
-
|
6025
|
-
|
6026
|
-
|
6027
|
-
|
6028
|
-
|
6029
|
-
|
6030
|
-
|
6031
|
-
|
6032
|
-
|
6033
|
-
|
6034
|
-
|
6035
|
-
|
6036
|
-
|
6037
|
-
|
6038
|
-
|
6039
|
-
|
6040
|
-
|
6553
|
+
const textEncoder = new TextEncoder();
|
6554
|
+
async function checkRemoteRef(remoteHost, ref, projectRoot, schema, metadata) {
|
6555
|
+
const remoteRefDataRes = core.Internal.remote.splitRemoteRef(ref);
|
6556
|
+
if (remoteRefDataRes.status === "error") {
|
6557
|
+
return remoteRefDataRes;
|
6558
|
+
}
|
6559
|
+
const fileHash = remoteRefDataRes.fileHash;
|
6560
|
+
if (!fileHash) {
|
6561
|
+
return {
|
6562
|
+
status: "error",
|
6563
|
+
error: `File hash is missing in remote ref: ${ref}`
|
6564
|
+
};
|
6565
|
+
}
|
6566
|
+
const relativeFilePath = remoteRefDataRes.filePath;
|
6567
|
+
if (!relativeFilePath) {
|
6568
|
+
return {
|
6569
|
+
status: "error",
|
6570
|
+
error: `File path is missing in remote ref: ${ref}`
|
6571
|
+
};
|
6572
|
+
}
|
6573
|
+
if (!relativeFilePath.startsWith("public/val/")) {
|
6574
|
+
return {
|
6575
|
+
status: "error",
|
6576
|
+
error: `File path must be within the public/val/ directory (e.g. public/val/path/to/file.txt). Got: ${relativeFilePath}`
|
6577
|
+
};
|
6578
|
+
}
|
6579
|
+
const coreVersion = core.Internal.VERSION.core || "unknown";
|
6580
|
+
const fileExt = getFileExt(relativeFilePath);
|
6581
|
+
const currentValidationHash = core.Internal.remote.getValidationHash(coreVersion, schema, fileExt, metadata,
|
6582
|
+
// TODO: validate
|
6583
|
+
fileHash, textEncoder);
|
6584
|
+
const validationHashFromRemoteRefValues = core.Internal.remote.getValidationHash(remoteRefDataRes.version, schema, getFileExt(remoteRefDataRes.filePath), metadata,
|
6585
|
+
// TODO: validate
|
6586
|
+
fileHash, textEncoder);
|
6587
|
+
if (
|
6588
|
+
// Current validation hash is the same as the remote validation hash
|
6589
|
+
// We assume the uploaded file exists and is correct
|
6590
|
+
currentValidationHash === remoteRefDataRes.validationHash &&
|
6591
|
+
// This can happen if the version or fileExt has been tampered with
|
6592
|
+
// (or if the way it is computed has changed)
|
6593
|
+
validationHashFromRemoteRefValues === remoteRefDataRes.validationHash) {
|
6594
|
+
return {
|
6595
|
+
status: "success"
|
6596
|
+
};
|
6597
|
+
}
|
6598
|
+
// Validation hash does not match, so we need to re-validate the remote content
|
6599
|
+
const fileBufferRes = await getFileBufferFromRemote(ref, fileExt, remoteRefDataRes.fileHash, projectRoot);
|
6600
|
+
|
6601
|
+
// We could not download the remote file for some reason (for example, the file hash is wrong)
|
6602
|
+
if (fileBufferRes.status === "error") {
|
6603
|
+
return fileBufferRes;
|
6604
|
+
}
|
6605
|
+
|
6606
|
+
// At this point we have the file buffer and we know the correct file hash
|
6607
|
+
// We must create a new ref since something has changed
|
6608
|
+
|
6609
|
+
let currentMetadata;
|
6610
|
+
try {
|
6611
|
+
if (schema.type === "image") {
|
6612
|
+
currentMetadata = await extractImageMetadata(remoteRefDataRes.filePath, fileBufferRes.fileBuffer);
|
6613
|
+
} else if (schema.type === "file") {
|
6614
|
+
currentMetadata = await extractFileMetadata(remoteRefDataRes.filePath, fileBufferRes.fileBuffer);
|
6615
|
+
} else {
|
6616
|
+
const exhaustiveCheck = schema;
|
6617
|
+
return {
|
6618
|
+
status: "error",
|
6619
|
+
error: `Unknown schema type: ${JSON.stringify(exhaustiveCheck)}`
|
6620
|
+
};
|
6621
|
+
}
|
6622
|
+
} catch (err) {
|
6623
|
+
return {
|
6624
|
+
status: "error",
|
6625
|
+
error: `Failed to extract metadata from remote file: ${err instanceof Error ? err.message : "Unknown error"}. Cached file: ${fileBufferRes.cachedFilePath}`
|
6626
|
+
};
|
6627
|
+
}
|
6628
|
+
const updatedMetadata = {
|
6629
|
+
...metadata,
|
6630
|
+
...currentMetadata
|
6631
|
+
};
|
6632
|
+
const nextValidationHash = core.Internal.remote.getValidationHash(coreVersion, schema, fileExt, updatedMetadata, fileBufferRes.fileHash, textEncoder);
|
6633
|
+
if (!updatedMetadata.mimeType) {
|
6634
|
+
return {
|
6635
|
+
status: "error",
|
6636
|
+
error: `MIME type is missing in metadata: ${JSON.stringify(updatedMetadata)}`
|
6637
|
+
};
|
6638
|
+
}
|
6639
|
+
const newFileExt = core.Internal.mimeTypeToFileExt(updatedMetadata.mimeType);
|
6640
|
+
const newFilePath = relativeFilePath.slice(0, -fileExt.length) + newFileExt;
|
6641
|
+
return {
|
6642
|
+
status: "fix-required",
|
6643
|
+
metadata: updatedMetadata,
|
6644
|
+
ref: core.Internal.remote.createRemoteRef(remoteHost, {
|
6645
|
+
publicProjectId: remoteRefDataRes.projectId,
|
6646
|
+
coreVersion,
|
6647
|
+
bucket: remoteRefDataRes.bucket,
|
6648
|
+
validationHash: nextValidationHash,
|
6649
|
+
fileHash: fileBufferRes.fileHash,
|
6650
|
+
filePath: newFilePath
|
6651
|
+
})
|
6652
|
+
};
|
6653
|
+
}
|
6654
|
+
function getCachedRemoteFileDir(projectRoot) {
|
6655
|
+
// store in projectRoot/.val/remote-file-cache
|
6656
|
+
const remoteFileCacheDir = fsPath__namespace["default"].join(projectRoot, ".val", "remote-file-cache");
|
6657
|
+
return remoteFileCacheDir;
|
6658
|
+
}
|
6659
|
+
function getCachedRemoteFilePath(fileExt, currentFileHash, remoteFileCacheDir) {
|
6660
|
+
const remoteFilePath = fsPath__namespace["default"].join(remoteFileCacheDir, currentFileHash + "." + fileExt);
|
6661
|
+
return remoteFilePath;
|
6662
|
+
}
|
6663
|
+
async function getFileBufferFromRemote(ref, fileExt, currentFileHash, projectRoot) {
|
6664
|
+
const remoteFileCacheDir = getCachedRemoteFileDir(projectRoot);
|
6665
|
+
const remoteFilePath = getCachedRemoteFilePath(fileExt, currentFileHash, remoteFileCacheDir);
|
6666
|
+
try {
|
6667
|
+
const fileBuffer = await fs__default["default"].promises.readFile(remoteFilePath);
|
6668
|
+
const computedFileHash = core.Internal.remote.getFileHash(fileBuffer);
|
6669
|
+
if (computedFileHash !== currentFileHash) {
|
6670
|
+
await fs__default["default"].promises.unlink(remoteFilePath);
|
6671
|
+
throw Error(`Cached file hash does not match the expected hash: ${computedFileHash} !== ${currentFileHash}`);
|
6672
|
+
}
|
6673
|
+
} catch (err) {
|
6674
|
+
try {
|
6675
|
+
await fs__default["default"].promises.mkdir(remoteFileCacheDir, {
|
6676
|
+
recursive: true
|
6677
|
+
});
|
6678
|
+
await downloadFileFromRemote(ref, remoteFilePath);
|
6679
|
+
} catch (err) {
|
6680
|
+
try {
|
6681
|
+
// try to delete in case of partial download
|
6682
|
+
await fs__default["default"].promises.unlink(remoteFilePath);
|
6683
|
+
} catch {
|
6684
|
+
//
|
6685
|
+
}
|
6686
|
+
return {
|
6687
|
+
status: "error",
|
6688
|
+
error: `Failed to download remote file: ${err instanceof Error ? err.message : "Unknown error"}`
|
6689
|
+
};
|
6690
|
+
}
|
6041
6691
|
}
|
6692
|
+
const fileBuffer = await fs__default["default"].promises.readFile(remoteFilePath);
|
6693
|
+
const computedFileHash = core.Internal.remote.getFileHash(fileBuffer);
|
6694
|
+
return {
|
6695
|
+
status: "success",
|
6696
|
+
fileBuffer,
|
6697
|
+
fileHash: computedFileHash,
|
6698
|
+
cachedFilePath: remoteFilePath
|
6699
|
+
};
|
6700
|
+
}
|
6701
|
+
async function downloadFileFromRemote(ref, filePath) {
|
6702
|
+
return new Promise((resolve, reject) => {
|
6703
|
+
const url = new URL(ref);
|
6704
|
+
const client = url.protocol === "https:" ? https__default["default"] : http__default["default"];
|
6705
|
+
const request = client.get(url, response => {
|
6706
|
+
if (response.statusCode && response.statusCode >= 300 && response.statusCode < 400 && response.headers.location) {
|
6707
|
+
// Handle redirects
|
6708
|
+
return downloadFileFromRemote(response.headers.location, filePath).then(resolve).catch(reject);
|
6709
|
+
}
|
6710
|
+
if (response.statusCode == 404) {
|
6711
|
+
return reject(new Error(`${ref}. File not found. The URL is most likely incorrect.`));
|
6712
|
+
}
|
6713
|
+
if (response.statusCode !== 200) {
|
6714
|
+
return reject(new Error(`${ref}. Failed to download file. HTTP Status: ${response.statusCode}`));
|
6715
|
+
}
|
6716
|
+
const writeStream = fs__default["default"].createWriteStream(filePath);
|
6717
|
+
response.pipe(writeStream);
|
6718
|
+
writeStream.on("finish", () => resolve({
|
6719
|
+
status: "success",
|
6720
|
+
filePath
|
6721
|
+
}));
|
6722
|
+
writeStream.on("error", err => reject(new Error(`${ref}. Error writing file: ${err.message}`)));
|
6723
|
+
});
|
6724
|
+
request.on("error", err => reject(new Error(`${ref}. Download error: ${err.message}`)));
|
6725
|
+
request.end();
|
6726
|
+
});
|
6727
|
+
}
|
6728
|
+
|
6729
|
+
// TODO: find a better name? transformFixesToPatch?
|
6730
|
+
async function createFixPatch(config, apply, sourcePath, validationError, remoteFiles, moduleSource, moduleSchema) {
|
6042
6731
|
const remainingErrors = [];
|
6043
6732
|
const patch$1 = [];
|
6044
6733
|
for (const fix of validationError.fixes || []) {
|
6045
6734
|
if (fix === "image:check-metadata" || fix === "image:add-metadata") {
|
6046
|
-
const imageMetadata = await getImageMetadata();
|
6735
|
+
const imageMetadata = await getImageMetadata(config.projectRoot, validationError);
|
6047
6736
|
if (imageMetadata.width === undefined || imageMetadata.height === undefined) {
|
6048
6737
|
remainingErrors.push({
|
6049
6738
|
...validationError,
|
@@ -6116,7 +6805,7 @@ async function createFixPatch(config, apply, sourcePath, validationError) {
|
|
6116
6805
|
});
|
6117
6806
|
}
|
6118
6807
|
} else if (fix === "file:add-metadata" || fix === "file:check-metadata") {
|
6119
|
-
const fileMetadata = await getFileMetadata();
|
6808
|
+
const fileMetadata = await getFileMetadata(config.projectRoot, validationError);
|
6120
6809
|
if (fileMetadata === undefined) {
|
6121
6810
|
remainingErrors.push({
|
6122
6811
|
...validationError,
|
@@ -6173,6 +6862,161 @@ async function createFixPatch(config, apply, sourcePath, validationError) {
|
|
6173
6862
|
}
|
6174
6863
|
});
|
6175
6864
|
}
|
6865
|
+
} else if (fix === "image:upload-remote" || fix === "file:upload-remote") {
|
6866
|
+
const remoteFile = remoteFiles[sourcePath];
|
6867
|
+
if (!remoteFile) {
|
6868
|
+
remainingErrors.push({
|
6869
|
+
...validationError,
|
6870
|
+
message: "Cannot fix local to remote image: remote image was not uploaded",
|
6871
|
+
fixes: undefined
|
6872
|
+
});
|
6873
|
+
} else {
|
6874
|
+
patch$1.push({
|
6875
|
+
op: "replace",
|
6876
|
+
value: {
|
6877
|
+
_type: "remote",
|
6878
|
+
_ref: remoteFile.ref,
|
6879
|
+
metadata: remoteFile.metadata
|
6880
|
+
},
|
6881
|
+
path: patch.sourceToPatchPath(sourcePath)
|
6882
|
+
});
|
6883
|
+
}
|
6884
|
+
} else if (fix === "image:download-remote" || fix === "file:download-remote") {
|
6885
|
+
const v = getRemoteValueFromValidationError(validationError);
|
6886
|
+
if (!v.success) {
|
6887
|
+
remainingErrors.push({
|
6888
|
+
...validationError,
|
6889
|
+
message: v.message,
|
6890
|
+
fixes: undefined
|
6891
|
+
});
|
6892
|
+
continue;
|
6893
|
+
}
|
6894
|
+
const splitRemoteRefDataRes = core.Internal.remote.splitRemoteRef(v._ref);
|
6895
|
+
if (splitRemoteRefDataRes.status === "error") {
|
6896
|
+
remainingErrors.push({
|
6897
|
+
...validationError,
|
6898
|
+
message: splitRemoteRefDataRes.error,
|
6899
|
+
fixes: undefined
|
6900
|
+
});
|
6901
|
+
continue;
|
6902
|
+
}
|
6903
|
+
const url = v._ref;
|
6904
|
+
const filePath = splitRemoteRefDataRes.filePath;
|
6905
|
+
if (!filePath) {
|
6906
|
+
remainingErrors.push({
|
6907
|
+
...validationError,
|
6908
|
+
message: "Unexpected error while downloading remote (no filePath)",
|
6909
|
+
fixes: undefined
|
6910
|
+
});
|
6911
|
+
continue;
|
6912
|
+
}
|
6913
|
+
if (!filePath.startsWith("public/val/")) {
|
6914
|
+
remainingErrors.push({
|
6915
|
+
...validationError,
|
6916
|
+
message: "Unexpected error while downloading remote (invalid file path - must start with public/val/)",
|
6917
|
+
fixes: undefined
|
6918
|
+
});
|
6919
|
+
continue;
|
6920
|
+
}
|
6921
|
+
const absoluteFilePath = fsPath__namespace["default"].join(config.projectRoot, splitRemoteRefDataRes.filePath);
|
6922
|
+
await fs__default["default"].promises.mkdir(fsPath__namespace["default"].dirname(absoluteFilePath), {
|
6923
|
+
recursive: true
|
6924
|
+
});
|
6925
|
+
const res = await downloadFileFromRemote(url, absoluteFilePath);
|
6926
|
+
if (res.status === "error") {
|
6927
|
+
remainingErrors.push({
|
6928
|
+
...validationError,
|
6929
|
+
message: res.error,
|
6930
|
+
fixes: undefined
|
6931
|
+
});
|
6932
|
+
continue;
|
6933
|
+
}
|
6934
|
+
const value = {
|
6935
|
+
[core.VAL_EXTENSION]: "file",
|
6936
|
+
[core.FILE_REF_PROP]: `/${filePath}`,
|
6937
|
+
...(fix === "image:download-remote" ? {
|
6938
|
+
[core.FILE_REF_SUBTYPE_TAG]: "image"
|
6939
|
+
} : {})
|
6940
|
+
};
|
6941
|
+
patch$1.push({
|
6942
|
+
op: "replace",
|
6943
|
+
path: patch.sourceToPatchPath(sourcePath),
|
6944
|
+
value: v.metadata ? {
|
6945
|
+
...value,
|
6946
|
+
metadata: v.metadata
|
6947
|
+
} : value
|
6948
|
+
});
|
6949
|
+
} else if (fix === "file:check-remote" || fix === "image:check-remote") {
|
6950
|
+
const v = getRemoteValueFromValidationError(validationError);
|
6951
|
+
if (!v.success) {
|
6952
|
+
remainingErrors.push({
|
6953
|
+
...validationError,
|
6954
|
+
message: v.message,
|
6955
|
+
fixes: undefined
|
6956
|
+
});
|
6957
|
+
continue;
|
6958
|
+
}
|
6959
|
+
const [, modulePath] = core.Internal.splitModuleFilePathAndModulePath(sourcePath);
|
6960
|
+
if (moduleSource === undefined) {
|
6961
|
+
remainingErrors.push({
|
6962
|
+
...validationError,
|
6963
|
+
message: "Unexpected error while checking remote (no moduleSource)",
|
6964
|
+
fixes: undefined
|
6965
|
+
});
|
6966
|
+
continue;
|
6967
|
+
}
|
6968
|
+
if (moduleSchema === undefined) {
|
6969
|
+
remainingErrors.push({
|
6970
|
+
...validationError,
|
6971
|
+
message: "Unexpected error while checking remote (no moduleSchema)",
|
6972
|
+
fixes: undefined
|
6973
|
+
});
|
6974
|
+
continue;
|
6975
|
+
}
|
6976
|
+
const {
|
6977
|
+
schema: schemaAtPath
|
6978
|
+
} = core.Internal.resolvePath(modulePath, moduleSource, moduleSchema);
|
6979
|
+
if (schemaAtPath.type === "image" || schemaAtPath.type === "file") {
|
6980
|
+
const res = await checkRemoteRef(config.remoteHost, v._ref, config.projectRoot, schemaAtPath, v.metadata);
|
6981
|
+
if (res.status === "success") ; else if (res.status === "error") {
|
6982
|
+
remainingErrors.push({
|
6983
|
+
...validationError,
|
6984
|
+
message: res.error,
|
6985
|
+
fixes: undefined
|
6986
|
+
});
|
6987
|
+
} else if (res.status === "fix-required") {
|
6988
|
+
if (apply) {
|
6989
|
+
patch$1.push({
|
6990
|
+
op: "replace",
|
6991
|
+
path: patch.sourceToPatchPath(sourcePath),
|
6992
|
+
value: {
|
6993
|
+
_type: "remote",
|
6994
|
+
_ref: res.ref,
|
6995
|
+
metadata: res.metadata
|
6996
|
+
}
|
6997
|
+
});
|
6998
|
+
} else {
|
6999
|
+
remainingErrors.push({
|
7000
|
+
...validationError,
|
7001
|
+
message: `Remote ref: ${res.ref} is not valid. Use the --fix flag to fix this issue.`,
|
7002
|
+
fixes: undefined
|
7003
|
+
});
|
7004
|
+
}
|
7005
|
+
} else {
|
7006
|
+
const exhaustiveCheck = res;
|
7007
|
+
remainingErrors.push({
|
7008
|
+
...validationError,
|
7009
|
+
message: `Internal error found found unexpected status: ${JSON.stringify(exhaustiveCheck)}`,
|
7010
|
+
fixes: undefined
|
7011
|
+
});
|
7012
|
+
}
|
7013
|
+
} else {
|
7014
|
+
remainingErrors.push({
|
7015
|
+
...validationError,
|
7016
|
+
message: "Could not check remote ref: schema type is not image or file: " + (schemaAtPath === null || schemaAtPath === void 0 ? void 0 : schemaAtPath.type),
|
7017
|
+
fixes: undefined
|
7018
|
+
});
|
7019
|
+
}
|
6176
7020
|
}
|
6177
7021
|
}
|
6178
7022
|
if (!validationError.fixes || validationError.fixes.length === 0) {
|
@@ -6183,6 +7027,73 @@ async function createFixPatch(config, apply, sourcePath, validationError) {
|
|
6183
7027
|
remainingErrors
|
6184
7028
|
};
|
6185
7029
|
}
|
7030
|
+
function getRemoteValueFromValidationError(v) {
|
7031
|
+
if (v.value && typeof v.value !== "object") {
|
7032
|
+
return {
|
7033
|
+
success: false,
|
7034
|
+
message: "Unexpected error while checking remote (not an object)"
|
7035
|
+
};
|
7036
|
+
}
|
7037
|
+
if (!v.value) {
|
7038
|
+
return {
|
7039
|
+
success: false,
|
7040
|
+
message: "Unexpected error while checking remote (no value)"
|
7041
|
+
};
|
7042
|
+
}
|
7043
|
+
if (typeof v.value !== "object" || v.value === null || !(core.FILE_REF_PROP in v.value)) {
|
7044
|
+
return {
|
7045
|
+
success: false,
|
7046
|
+
message: "Unexpected error while checking remote (no _ref in value)"
|
7047
|
+
};
|
7048
|
+
}
|
7049
|
+
if (typeof v.value._ref !== "string") {
|
7050
|
+
return {
|
7051
|
+
success: false,
|
7052
|
+
message: "Unexpected error while checking remote (_ref is not a string)"
|
7053
|
+
};
|
7054
|
+
}
|
7055
|
+
let metadata;
|
7056
|
+
if ("metadata" in v.value && typeof v.value.metadata === "object") {
|
7057
|
+
if (v.value.metadata === null) {
|
7058
|
+
return {
|
7059
|
+
success: false,
|
7060
|
+
message: "Unexpected error while checking remote (metadata is null)"
|
7061
|
+
};
|
7062
|
+
}
|
7063
|
+
if (Array.isArray(v.value.metadata)) {
|
7064
|
+
return {
|
7065
|
+
success: false,
|
7066
|
+
message: "Unexpected error while checking remote (metadata is an array)"
|
7067
|
+
};
|
7068
|
+
}
|
7069
|
+
metadata = v.value.metadata;
|
7070
|
+
}
|
7071
|
+
return {
|
7072
|
+
success: true,
|
7073
|
+
_ref: v.value._ref,
|
7074
|
+
metadata
|
7075
|
+
};
|
7076
|
+
}
|
7077
|
+
async function getImageMetadata(projectRoot, validationError) {
|
7078
|
+
const fileRef = getValidationErrorFileRef(validationError);
|
7079
|
+
if (!fileRef) {
|
7080
|
+
// TODO:
|
7081
|
+
throw Error("Cannot fix image without a file reference");
|
7082
|
+
}
|
7083
|
+
const filename = fsPath__namespace["default"].join(projectRoot, fileRef);
|
7084
|
+
const buffer = fs__default["default"].readFileSync(filename);
|
7085
|
+
return extractImageMetadata(filename, buffer);
|
7086
|
+
}
|
7087
|
+
async function getFileMetadata(projectRoot, validationError) {
|
7088
|
+
const fileRef = getValidationErrorFileRef(validationError);
|
7089
|
+
if (!fileRef) {
|
7090
|
+
// TODO:
|
7091
|
+
throw Error("Cannot fix file without a file reference");
|
7092
|
+
}
|
7093
|
+
const filename = fsPath__namespace["default"].join(projectRoot, fileRef);
|
7094
|
+
fs__default["default"].readFileSync(filename);
|
7095
|
+
return extractFileMetadata(fileRef);
|
7096
|
+
}
|
6186
7097
|
|
6187
7098
|
exports.Service = Service;
|
6188
7099
|
exports.ValFSHost = ValFSHost;
|
@@ -6197,5 +7108,9 @@ exports.encodeJwt = encodeJwt;
|
|
6197
7108
|
exports.formatSyntaxErrorTree = formatSyntaxErrorTree;
|
6198
7109
|
exports.getCompilerOptions = getCompilerOptions;
|
6199
7110
|
exports.getExpire = getExpire;
|
7111
|
+
exports.getPersonalAccessTokenPath = getPersonalAccessTokenPath;
|
7112
|
+
exports.getSettings = getSettings;
|
7113
|
+
exports.parsePersonalAccessTokenFile = parsePersonalAccessTokenFile;
|
6200
7114
|
exports.patchSourceFile = patchSourceFile;
|
6201
7115
|
exports.safeReadGit = safeReadGit;
|
7116
|
+
exports.uploadRemoteFile = uploadRemoteFile;
|