@valbuild/server 0.103.0 → 0.103.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/declarations/src/ValOps.d.ts +9 -0
- package/dist/declarations/src/ValOpsFS.d.ts +46 -37
- package/dist/declarations/src/index.d.ts +7 -0
- package/dist/declarations/src/patchLog.d.ts +124 -0
- package/dist/declarations/src/patchStore.d.ts +216 -0
- package/dist/valbuild-server.cjs.dev.js +1528 -346
- package/dist/valbuild-server.cjs.prod.js +1528 -346
- package/dist/valbuild-server.esm.js +1527 -348
- package/package.json +2 -2
|
@@ -16,6 +16,7 @@ var crypto$1 = require('crypto');
|
|
|
16
16
|
var z = require('zod');
|
|
17
17
|
var sizeOf = require('image-size');
|
|
18
18
|
var zodValidationError = require('zod-validation-error');
|
|
19
|
+
var os = require('os');
|
|
19
20
|
var sucrase = require('sucrase');
|
|
20
21
|
var http = require('http');
|
|
21
22
|
var https = require('https');
|
|
@@ -48,6 +49,7 @@ var vm__default = /*#__PURE__*/_interopDefault(vm);
|
|
|
48
49
|
var crypto__default = /*#__PURE__*/_interopDefault(crypto$1);
|
|
49
50
|
var z__default = /*#__PURE__*/_interopDefault(z);
|
|
50
51
|
var sizeOf__default = /*#__PURE__*/_interopDefault(sizeOf);
|
|
52
|
+
var os__default = /*#__PURE__*/_interopDefault(os);
|
|
51
53
|
var http__default = /*#__PURE__*/_interopDefault(http);
|
|
52
54
|
var https__default = /*#__PURE__*/_interopDefault(https);
|
|
53
55
|
var fs__default$1 = /*#__PURE__*/_interopDefault(fs$1);
|
|
@@ -3301,49 +3303,1030 @@ class JsonEntryFilesFingerprint {
|
|
|
3301
3303
|
}
|
|
3302
3304
|
|
|
3303
3305
|
/**
|
|
3304
|
-
*
|
|
3306
|
+
* The order of the pending patches, held in one flat, append-only text file.
|
|
3305
3307
|
*
|
|
3306
|
-
*
|
|
3308
|
+
* ## Why a log and not links between the patches
|
|
3307
3309
|
*
|
|
3308
|
-
*
|
|
3309
|
-
*
|
|
3310
|
-
*
|
|
3310
|
+
* The store this replaces kept the order in the patches themselves: every record
|
|
3311
|
+
* carried a `parentRef`, and the directory a record lived in was named after that
|
|
3312
|
+
* parent. Reading the chain meant walking those links from `head`, and the walk
|
|
3313
|
+
* stopped dead — silently — at the first id nothing on disk answered for.
|
|
3314
|
+
*
|
|
3315
|
+
* That is not a hypothetical. A store with 410 patches lost exactly one record,
|
|
3316
|
+
* and the 51 patches written after it became unreachable: `/stat` counted the
|
|
3317
|
+
* directories and announced 410, `GET /patches` walked the links and delivered
|
|
3318
|
+
* 359, and the studio waited forever for 51 ids that no longer had a path back to
|
|
3319
|
+
* `head`. One missing file cost every edit made after it.
|
|
3320
|
+
*
|
|
3321
|
+
* So the order lives in one place and the patches reference nothing. A patch
|
|
3322
|
+
* directory is self-contained data; this file says what order the directories go
|
|
3323
|
+
* in. Removing a line cannot orphan the lines below it, because there is nothing
|
|
3324
|
+
* below it to orphan — the entry after a dropped one is simply next.
|
|
3325
|
+
*
|
|
3326
|
+
* ## Why text
|
|
3327
|
+
*
|
|
3328
|
+
* It is read by people during exactly the incidents that make it interesting, and
|
|
3329
|
+
* `cat` should be enough. One entry per line, whitespace-separated, id first:
|
|
3330
|
+
*
|
|
3331
|
+
* ```
|
|
3332
|
+
* val-patch-log v1
|
|
3333
|
+
* 659b8cfa-065c-47d1-8d82-fc69a2ac72a9 2026-08-27T11:46:13.730Z /content/authors.val.ts
|
|
3334
|
+
* ```
|
|
3335
|
+
*
|
|
3336
|
+
* Position in the file IS the order. There is deliberately no sequence number:
|
|
3337
|
+
* a number stored beside the thing it describes is a number that can disagree
|
|
3338
|
+
* with it, and then something has to decide which one lies.
|
|
3339
|
+
*
|
|
3340
|
+
* The timestamp and path are there to make the file readable, not to be believed
|
|
3341
|
+
* — `patch.json` owns those fields. Nothing here is a second copy of state that
|
|
3342
|
+
* anything reads back.
|
|
3311
3343
|
*/
|
|
3312
|
-
|
|
3313
|
-
|
|
3314
|
-
|
|
3315
|
-
|
|
3316
|
-
|
|
3317
|
-
|
|
3318
|
-
|
|
3319
|
-
|
|
3320
|
-
|
|
3321
|
-
|
|
3322
|
-
|
|
3323
|
-
|
|
3324
|
-
|
|
3325
|
-
|
|
3326
|
-
|
|
3327
|
-
|
|
3328
|
-
|
|
3329
|
-
|
|
3330
|
-
|
|
3331
|
-
|
|
3332
|
-
|
|
3333
|
-
|
|
3334
|
-
|
|
3335
|
-
|
|
3336
|
-
|
|
3337
|
-
|
|
3344
|
+
|
|
3345
|
+
const PATCH_LOG_FILE_NAME = "patches.log";
|
|
3346
|
+
const HEADER = "val-patch-log v1";
|
|
3347
|
+
|
|
3348
|
+
/**
|
|
3349
|
+
* Something that is wrong with the file but does not stop it being read.
|
|
3350
|
+
*
|
|
3351
|
+
* Reported rather than thrown, because a log that is 99% intact is worth reading
|
|
3352
|
+
* and then repairing. Only a file that cannot be understood at all is fatal, and
|
|
3353
|
+
* that is a `status` on the read result, not a problem in this list.
|
|
3354
|
+
*/
|
|
3355
|
+
|
|
3356
|
+
function isModuleFilePath(path) {
|
|
3357
|
+
return path.startsWith("/") && path.includes(".val.");
|
|
3358
|
+
}
|
|
3359
|
+
function isPatchId(patchId) {
|
|
3360
|
+
return patchId.length > 0 && !/\s/.test(patchId);
|
|
3361
|
+
}
|
|
3362
|
+
function formatPatchLogLine(entry) {
|
|
3363
|
+
return `${entry.patchId} ${entry.createdAt} ${entry.path}`;
|
|
3364
|
+
}
|
|
3365
|
+
|
|
3366
|
+
/**
|
|
3367
|
+
* Split on the first two runs of whitespace only: a module file path may contain
|
|
3368
|
+
* spaces, and it is last precisely so that it can.
|
|
3369
|
+
*/
|
|
3370
|
+
function parsePatchLogLine(line) {
|
|
3371
|
+
const trimmed = line.trim();
|
|
3372
|
+
if (trimmed === "") {
|
|
3373
|
+
return null;
|
|
3374
|
+
}
|
|
3375
|
+
const match = /^(\S+)\s+(\S+)\s+(.+)$/.exec(trimmed);
|
|
3376
|
+
if (!match) {
|
|
3377
|
+
return null;
|
|
3378
|
+
}
|
|
3379
|
+
const [, patchId, createdAt, path] = match;
|
|
3380
|
+
if (!isPatchId(patchId) || !isModuleFilePath(path)) {
|
|
3381
|
+
return null;
|
|
3382
|
+
}
|
|
3383
|
+
return {
|
|
3384
|
+
patchId,
|
|
3385
|
+
createdAt,
|
|
3386
|
+
path
|
|
3387
|
+
};
|
|
3388
|
+
}
|
|
3389
|
+
function serializePatchLog(entries) {
|
|
3390
|
+
return [HEADER, ...entries.map(formatPatchLogLine), ""].join("\n");
|
|
3391
|
+
}
|
|
3392
|
+
function readPatchLog(logFilePath) {
|
|
3393
|
+
let raw;
|
|
3394
|
+
try {
|
|
3395
|
+
raw = fs__default["default"].readFileSync(logFilePath, "utf-8");
|
|
3396
|
+
} catch (err) {
|
|
3397
|
+
if (typeof err === "object" && err !== null && "code" in err && err.code === "ENOENT") {
|
|
3398
|
+
return {
|
|
3399
|
+
status: "absent"
|
|
3400
|
+
};
|
|
3401
|
+
}
|
|
3402
|
+
return {
|
|
3403
|
+
status: "unreadable",
|
|
3404
|
+
message: `Could not read ${logFilePath}: ${err instanceof Error ? err.message : "unknown error"}`
|
|
3405
|
+
};
|
|
3406
|
+
}
|
|
3407
|
+
return parsePatchLog(raw);
|
|
3408
|
+
}
|
|
3409
|
+
function parsePatchLog(raw) {
|
|
3410
|
+
const problems = [];
|
|
3411
|
+
// A zero-byte file is a store that was created and then interrupted before the
|
|
3412
|
+
// header landed. Empty, not broken: there are no patches either way.
|
|
3413
|
+
if (raw === "") {
|
|
3414
|
+
return {
|
|
3415
|
+
status: "ok",
|
|
3416
|
+
entries: [],
|
|
3417
|
+
problems
|
|
3418
|
+
};
|
|
3419
|
+
}
|
|
3420
|
+
const endsWithNewline = raw.endsWith("\n");
|
|
3421
|
+
const lines = raw.split("\n");
|
|
3422
|
+
// `split` always leaves a trailing "" when the text ends in a newline; when it
|
|
3423
|
+
// does not, the last element is a line whose write never finished.
|
|
3424
|
+
const tornFinalLine = endsWithNewline ? null : lines[lines.length - 1];
|
|
3425
|
+
lines.pop();
|
|
3426
|
+
if (tornFinalLine !== null && tornFinalLine.trim() !== "") {
|
|
3427
|
+
problems.push({
|
|
3428
|
+
type: "torn-final-line",
|
|
3429
|
+
line: tornFinalLine
|
|
3430
|
+
});
|
|
3431
|
+
}
|
|
3432
|
+
|
|
3433
|
+
// Only consume the first line when it really is the header. A headerless file
|
|
3434
|
+
// is reported, but its first line is still offered to the entry parser rather
|
|
3435
|
+
// than eaten: dropping a real patch to pay for a missing header would be the
|
|
3436
|
+
// same silent data loss this store exists to stop.
|
|
3437
|
+
const firstLine = (lines[0] ?? "").replace(/\r$/, "");
|
|
3438
|
+
let firstEntryIndex = 0;
|
|
3439
|
+
if (firstLine === HEADER) {
|
|
3440
|
+
firstEntryIndex = 1;
|
|
3441
|
+
} else if (firstLine.startsWith("val-patch-log ")) {
|
|
3442
|
+
problems.push({
|
|
3443
|
+
type: "unsupported-version",
|
|
3444
|
+
header: firstLine
|
|
3445
|
+
});
|
|
3446
|
+
firstEntryIndex = 1;
|
|
3447
|
+
} else {
|
|
3448
|
+
problems.push({
|
|
3449
|
+
type: "missing-header",
|
|
3450
|
+
firstLine
|
|
3451
|
+
});
|
|
3452
|
+
}
|
|
3453
|
+
const entries = [];
|
|
3454
|
+
const seen = new Set();
|
|
3455
|
+
lines.slice(firstEntryIndex).forEach((rawLine, index) => {
|
|
3456
|
+
const line = rawLine.replace(/\r$/, "");
|
|
3457
|
+
// 1-based and counted from the top of the file, so they match what an editor
|
|
3458
|
+
// shows the person reading this report.
|
|
3459
|
+
const lineNumber = firstEntryIndex + index + 1;
|
|
3460
|
+
if (line.trim() === "") {
|
|
3461
|
+
return;
|
|
3462
|
+
}
|
|
3463
|
+
const entry = parsePatchLogLine(line);
|
|
3464
|
+
if (entry === null) {
|
|
3465
|
+
problems.push({
|
|
3466
|
+
type: "unparseable-line",
|
|
3467
|
+
lineNumber,
|
|
3468
|
+
line
|
|
3469
|
+
});
|
|
3470
|
+
return;
|
|
3471
|
+
}
|
|
3472
|
+
if (seen.has(entry.patchId)) {
|
|
3473
|
+
problems.push({
|
|
3474
|
+
type: "duplicate-entry",
|
|
3475
|
+
patchId: entry.patchId,
|
|
3476
|
+
lineNumber
|
|
3477
|
+
});
|
|
3478
|
+
return;
|
|
3479
|
+
}
|
|
3480
|
+
seen.add(entry.patchId);
|
|
3481
|
+
entries.push(entry);
|
|
3482
|
+
});
|
|
3483
|
+
return {
|
|
3484
|
+
status: "ok",
|
|
3485
|
+
entries,
|
|
3486
|
+
problems
|
|
3487
|
+
};
|
|
3488
|
+
}
|
|
3489
|
+
|
|
3490
|
+
/**
|
|
3491
|
+
* fsync the directory holding a file, so a rename or create survives a power cut.
|
|
3492
|
+
*
|
|
3493
|
+
* Best-effort: opening a directory is not permitted on Windows, and a store that
|
|
3494
|
+
* refuses to work there to buy durability nobody asked for is the worse trade.
|
|
3495
|
+
*/
|
|
3496
|
+
function fsyncDir(dirPath) {
|
|
3497
|
+
let fd;
|
|
3498
|
+
try {
|
|
3499
|
+
fd = fs__default["default"].openSync(dirPath, "r");
|
|
3500
|
+
fs__default["default"].fsyncSync(fd);
|
|
3501
|
+
} catch {
|
|
3502
|
+
// ignore: see above
|
|
3503
|
+
} finally {
|
|
3504
|
+
if (fd !== undefined) {
|
|
3505
|
+
try {
|
|
3506
|
+
fs__default["default"].closeSync(fd);
|
|
3507
|
+
} catch {
|
|
3508
|
+
// ignore
|
|
3509
|
+
}
|
|
3510
|
+
}
|
|
3511
|
+
}
|
|
3512
|
+
}
|
|
3513
|
+
|
|
3514
|
+
/**
|
|
3515
|
+
* Add one entry to the end of the log.
|
|
3516
|
+
*
|
|
3517
|
+
* A single `writeSync` on an append-only descriptor, then fsync: the whole line
|
|
3518
|
+
* reaches the file or none of it does, and a reader that catches it mid-flight
|
|
3519
|
+
* discards the partial last line rather than misreading it.
|
|
3520
|
+
*
|
|
3521
|
+
* Callers must hold the patch lock. That is what makes "the torn line can only be
|
|
3522
|
+
* the last one" true, and it is why this does not try to be safe against
|
|
3523
|
+
* concurrent appends on its own.
|
|
3524
|
+
*/
|
|
3525
|
+
function appendPatchLogEntry(logFilePath, entry) {
|
|
3526
|
+
const dirPath = path__namespace["default"].dirname(logFilePath);
|
|
3527
|
+
fs__default["default"].mkdirSync(dirPath, {
|
|
3528
|
+
recursive: true
|
|
3529
|
+
});
|
|
3530
|
+
if (!fs__default["default"].existsSync(logFilePath)) {
|
|
3531
|
+
writePatchLogFile(logFilePath, []);
|
|
3532
|
+
}
|
|
3533
|
+
const fd = fs__default["default"].openSync(logFilePath, "a");
|
|
3534
|
+
try {
|
|
3535
|
+
fs__default["default"].writeSync(fd, `${formatPatchLogLine(entry)}\n`);
|
|
3536
|
+
fs__default["default"].fsyncSync(fd);
|
|
3537
|
+
} finally {
|
|
3538
|
+
fs__default["default"].closeSync(fd);
|
|
3539
|
+
}
|
|
3540
|
+
}
|
|
3541
|
+
|
|
3542
|
+
/**
|
|
3543
|
+
* Replace the log wholesale — used by delete and by repair.
|
|
3544
|
+
*
|
|
3545
|
+
* Write a sibling temp file, fsync it, then rename over the original: a rename is
|
|
3546
|
+
* atomic, so a reader either sees the old log or the new one and never a
|
|
3547
|
+
* half-rewritten file. In-place truncation would have a window where the log is
|
|
3548
|
+
* short and the store looks like it lost patches.
|
|
3549
|
+
*
|
|
3550
|
+
* Callers must hold the patch lock.
|
|
3551
|
+
*/
|
|
3552
|
+
function writePatchLogFile(logFilePath, entries) {
|
|
3553
|
+
const dirPath = path__namespace["default"].dirname(logFilePath);
|
|
3554
|
+
fs__default["default"].mkdirSync(dirPath, {
|
|
3555
|
+
recursive: true
|
|
3556
|
+
});
|
|
3557
|
+
const tmpPath = `${logFilePath}.tmp-${process.pid}`;
|
|
3558
|
+
const fd = fs__default["default"].openSync(tmpPath, "w");
|
|
3559
|
+
try {
|
|
3560
|
+
fs__default["default"].writeSync(fd, serializePatchLog(entries));
|
|
3561
|
+
fs__default["default"].fsyncSync(fd);
|
|
3562
|
+
} finally {
|
|
3563
|
+
fs__default["default"].closeSync(fd);
|
|
3564
|
+
}
|
|
3565
|
+
fs__default["default"].renameSync(tmpPath, logFilePath);
|
|
3566
|
+
fsyncDir(dirPath);
|
|
3567
|
+
}
|
|
3568
|
+
|
|
3569
|
+
/**
|
|
3570
|
+
* The on-disk shape of the local-dev patch store.
|
|
3571
|
+
*
|
|
3572
|
+
* ```
|
|
3573
|
+
* .val/patches/
|
|
3574
|
+
* patches.log the order, and the only place it lives
|
|
3575
|
+
* patches.repair.log what repair has done, for the person who has to know
|
|
3576
|
+
* <patchId>/patch.json one plain, self-contained directory per patch
|
|
3577
|
+
* <patchId>/base.json written when the patch is published
|
|
3578
|
+
* <patchId>/files/… binary payloads for this patch's file ops
|
|
3579
|
+
* ```
|
|
3580
|
+
*
|
|
3581
|
+
* A directory is named after the patch it holds, and a record references nothing
|
|
3582
|
+
* outside itself. That is the whole design, and it is a direct answer to how the
|
|
3583
|
+
* old layout failed: there, a directory was named after a record's PARENT, so
|
|
3584
|
+
* reading the store meant following links, and one absent record silently cut off
|
|
3585
|
+
* every patch written after it.
|
|
3586
|
+
*
|
|
3587
|
+
* The invariant this module exists to hold up is narrow and worth stating: **the
|
|
3588
|
+
* announced set and the delivered set are the same array.** `getStat` and
|
|
3589
|
+
* `fetchPatches` both come out of one `readPatchStore` call, so they cannot report
|
|
3590
|
+
* different numbers - which is exactly what they did when this broke, announcing
|
|
3591
|
+
* 410 patches and delivering 359 with no error in between.
|
|
3592
|
+
*/
|
|
3593
|
+
|
|
3594
|
+
const PATCH_REPAIR_LOG_FILE_NAME = "patches.repair.log";
|
|
3595
|
+
const FSPatch = z.z.object({
|
|
3596
|
+
path: z.z.string().refine(p => p.startsWith("/") && p.includes(".val."), "Path is not valid. Must start with '/' and include '.val.'"),
|
|
3597
|
+
patch: internal.Patch,
|
|
3598
|
+
patchId: z.z.string(),
|
|
3599
|
+
baseSha: z.z.string().refine(p => true),
|
|
3600
|
+
authorId: z.z.string().refine(p => true).nullable(),
|
|
3601
|
+
createdAt: z.z.string().datetime(),
|
|
3602
|
+
coreVersion: z.z.string().nullable(),
|
|
3603
|
+
sessionId: z.z.string().nullable()
|
|
3604
|
+
});
|
|
3605
|
+
const FSPatchBase = z.z.object({
|
|
3606
|
+
baseSha: z.z.string().refine(p => true),
|
|
3607
|
+
timestamp: z.z.string().datetime()
|
|
3608
|
+
});
|
|
3609
|
+
|
|
3610
|
+
/**
|
|
3611
|
+
* Something wrong with the store that a reader can see.
|
|
3612
|
+
*
|
|
3613
|
+
* Every one of these used to be invisible. Reporting them is the point: the
|
|
3614
|
+
* failure that motivated this rewrite was not that the store broke, it was that
|
|
3615
|
+
* breaking looked exactly like working.
|
|
3616
|
+
*/
|
|
3617
|
+
|
|
3618
|
+
function patchesLogFile(patchesDir) {
|
|
3619
|
+
return path__namespace["default"].join(patchesDir, PATCH_LOG_FILE_NAME);
|
|
3620
|
+
}
|
|
3621
|
+
function patchRepairLogFile(patchesDir) {
|
|
3622
|
+
return path__namespace["default"].join(patchesDir, PATCH_REPAIR_LOG_FILE_NAME);
|
|
3623
|
+
}
|
|
3624
|
+
function patchDir(patchesDir, patchId) {
|
|
3625
|
+
return path__namespace["default"].join(patchesDir, patchId);
|
|
3626
|
+
}
|
|
3627
|
+
function patchRecordFile(patchesDir, patchId) {
|
|
3628
|
+
return path__namespace["default"].join(patchDir(patchesDir, patchId), "patch.json");
|
|
3629
|
+
}
|
|
3630
|
+
function patchBaseFile(patchesDir, patchId) {
|
|
3631
|
+
return path__namespace["default"].join(patchDir(patchesDir, patchId), "base.json");
|
|
3632
|
+
}
|
|
3633
|
+
function patchBinaryFile(patchesDir, patchId, filePath) {
|
|
3634
|
+
return path__namespace["default"].join(patchDir(patchesDir, patchId), "files", filePath, path__namespace["default"].basename(filePath));
|
|
3635
|
+
}
|
|
3636
|
+
function patchBinaryFileMetadata(patchesDir, patchId, filePath) {
|
|
3637
|
+
return path__namespace["default"].join(patchDir(patchesDir, patchId), "files", filePath, "metadata.json");
|
|
3638
|
+
}
|
|
3639
|
+
|
|
3640
|
+
/** Names that live in the patches directory but are not patches. */
|
|
3641
|
+
const RESERVED_NAMES = new Set([PATCH_LOG_FILE_NAME, PATCH_REPAIR_LOG_FILE_NAME]);
|
|
3642
|
+
function readJsonFile(filePath, parser) {
|
|
3643
|
+
let raw;
|
|
3644
|
+
try {
|
|
3645
|
+
raw = fs__default["default"].readFileSync(filePath, "utf-8");
|
|
3646
|
+
} catch (err) {
|
|
3647
|
+
return {
|
|
3648
|
+
error: err instanceof Error ? err.message : "unknown error"
|
|
3649
|
+
};
|
|
3650
|
+
}
|
|
3651
|
+
if (raw === "") {
|
|
3652
|
+
return {
|
|
3653
|
+
error: "the file is empty"
|
|
3654
|
+
};
|
|
3655
|
+
}
|
|
3656
|
+
let json;
|
|
3657
|
+
try {
|
|
3658
|
+
json = JSON.parse(raw);
|
|
3659
|
+
} catch (err) {
|
|
3660
|
+
return {
|
|
3661
|
+
error: `it is not valid JSON: ${err instanceof Error ? err.message : "unknown error"}`
|
|
3662
|
+
};
|
|
3663
|
+
}
|
|
3664
|
+
const parsed = parser.safeParse(json);
|
|
3665
|
+
if (!parsed.success) {
|
|
3666
|
+
return {
|
|
3667
|
+
error: zodValidationError.fromError(parsed.error).toString()
|
|
3668
|
+
};
|
|
3669
|
+
}
|
|
3670
|
+
return {
|
|
3671
|
+
data: parsed.data
|
|
3672
|
+
};
|
|
3673
|
+
}
|
|
3674
|
+
function listPatchDirNames(patchesDir) {
|
|
3675
|
+
return fs__default["default"].readdirSync(patchesDir, {
|
|
3676
|
+
withFileTypes: true
|
|
3677
|
+
}).filter(dirent => dirent.isDirectory() && !RESERVED_NAMES.has(dirent.name)).map(dirent => dirent.name);
|
|
3678
|
+
}
|
|
3679
|
+
|
|
3680
|
+
/**
|
|
3681
|
+
* Read the whole store: the order, the records, and everything wrong with it.
|
|
3682
|
+
*
|
|
3683
|
+
* One call, one answer. Callers that need only the ids and callers that need the
|
|
3684
|
+
* ops both use this, which is what stops them disagreeing.
|
|
3685
|
+
*/
|
|
3686
|
+
function readPatchStore(patchesDir) {
|
|
3687
|
+
if (!fs__default["default"].existsSync(patchesDir)) {
|
|
3688
|
+
return {
|
|
3689
|
+
status: "ok",
|
|
3690
|
+
entries: [],
|
|
3691
|
+
problems: []
|
|
3692
|
+
};
|
|
3693
|
+
}
|
|
3694
|
+
let dirNames;
|
|
3695
|
+
try {
|
|
3696
|
+
dirNames = listPatchDirNames(patchesDir);
|
|
3697
|
+
} catch (err) {
|
|
3698
|
+
return {
|
|
3699
|
+
status: "unreadable",
|
|
3700
|
+
message: `Could not list ${patchesDir}: ${err instanceof Error ? err.message : "unknown error"}`
|
|
3701
|
+
};
|
|
3702
|
+
}
|
|
3703
|
+
const problems = [];
|
|
3704
|
+
const logRes = readPatchLog(patchesLogFile(patchesDir));
|
|
3705
|
+
if (logRes.status === "unreadable") {
|
|
3706
|
+
return {
|
|
3707
|
+
status: "unreadable",
|
|
3708
|
+
message: logRes.message
|
|
3709
|
+
};
|
|
3710
|
+
}
|
|
3711
|
+
let logEntries;
|
|
3712
|
+
if (logRes.status === "absent") {
|
|
3713
|
+
if (dirNames.length === 0) {
|
|
3714
|
+
return {
|
|
3715
|
+
status: "ok",
|
|
3716
|
+
entries: [],
|
|
3717
|
+
problems: []
|
|
3718
|
+
};
|
|
3719
|
+
}
|
|
3720
|
+
logEntries = reconstructLogEntries(patchesDir, dirNames);
|
|
3721
|
+
if (logEntries.length > 0) {
|
|
3722
|
+
problems.push({
|
|
3723
|
+
type: "reconstructed-log",
|
|
3724
|
+
entryCount: logEntries.length
|
|
3725
|
+
});
|
|
3726
|
+
}
|
|
3727
|
+
} else {
|
|
3728
|
+
logEntries = logRes.entries;
|
|
3729
|
+
for (const problem of logRes.problems) {
|
|
3730
|
+
problems.push({
|
|
3731
|
+
type: "log",
|
|
3732
|
+
problem
|
|
3733
|
+
});
|
|
3734
|
+
}
|
|
3735
|
+
}
|
|
3736
|
+
const entries = [];
|
|
3737
|
+
const named = new Set();
|
|
3738
|
+
for (const logEntry of logEntries) {
|
|
3739
|
+
named.add(logEntry.patchId);
|
|
3740
|
+
const read = readPatchRecord(patchesDir, logEntry.patchId);
|
|
3741
|
+
if (read.error !== undefined) {
|
|
3742
|
+
problems.push({
|
|
3743
|
+
type: "unreadable-patch",
|
|
3744
|
+
name: logEntry.patchId,
|
|
3745
|
+
dir: patchDir(patchesDir, logEntry.patchId),
|
|
3746
|
+
message: read.error
|
|
3747
|
+
});
|
|
3748
|
+
continue;
|
|
3749
|
+
}
|
|
3750
|
+
const baseFile = patchBaseFile(patchesDir, logEntry.patchId);
|
|
3751
|
+
const baseRes = fs__default["default"].existsSync(baseFile) ? readJsonFile(baseFile, FSPatchBase) : undefined;
|
|
3752
|
+
entries.push({
|
|
3753
|
+
patchId: logEntry.patchId,
|
|
3754
|
+
record: read.data,
|
|
3755
|
+
base: (baseRes === null || baseRes === void 0 ? void 0 : baseRes.data) ?? null
|
|
3756
|
+
});
|
|
3757
|
+
}
|
|
3758
|
+
for (const name of dirNames) {
|
|
3759
|
+
if (named.has(name)) {
|
|
3760
|
+
continue;
|
|
3761
|
+
}
|
|
3762
|
+
// Told apart by whether the directory holds a usable record. One is a patch
|
|
3763
|
+
// whose contents are lost and has to be reported; the other is a crash
|
|
3764
|
+
// leftover nothing ever read.
|
|
3765
|
+
const read = readPatchRecord(patchesDir, name);
|
|
3766
|
+
if (read.error !== undefined) {
|
|
3767
|
+
problems.push({
|
|
3768
|
+
type: "unreadable-patch",
|
|
3769
|
+
name,
|
|
3770
|
+
dir: path__namespace["default"].join(patchesDir, name),
|
|
3771
|
+
message: read.error
|
|
3772
|
+
});
|
|
3773
|
+
} else {
|
|
3774
|
+
problems.push({
|
|
3775
|
+
type: "orphan-directory",
|
|
3776
|
+
name,
|
|
3777
|
+
dir: path__namespace["default"].join(patchesDir, name)
|
|
3778
|
+
});
|
|
3779
|
+
}
|
|
3780
|
+
}
|
|
3781
|
+
return {
|
|
3782
|
+
status: "ok",
|
|
3783
|
+
entries,
|
|
3784
|
+
problems
|
|
3785
|
+
};
|
|
3786
|
+
}
|
|
3787
|
+
|
|
3788
|
+
/**
|
|
3789
|
+
* Read one patch directory, insisting it holds the patch it is named after.
|
|
3790
|
+
*
|
|
3791
|
+
* The name check is not a formality. A directory named after something other
|
|
3792
|
+
* than the record inside is how the previous layout worked - the name was the
|
|
3793
|
+
* record's PARENT - so this is what tells a store this version can use from one
|
|
3794
|
+
* it cannot.
|
|
3795
|
+
*/
|
|
3796
|
+
function readPatchRecord(patchesDir, name) {
|
|
3797
|
+
const file = patchRecordFile(patchesDir, name);
|
|
3798
|
+
if (!fs__default["default"].existsSync(file)) {
|
|
3799
|
+
return {
|
|
3800
|
+
error: `there is no ${file}`
|
|
3801
|
+
};
|
|
3802
|
+
}
|
|
3803
|
+
const res = readJsonFile(file, FSPatch);
|
|
3804
|
+
if (res.error !== undefined) {
|
|
3805
|
+
return {
|
|
3806
|
+
error: res.error
|
|
3807
|
+
};
|
|
3808
|
+
}
|
|
3809
|
+
if (res.data.patchId !== name) {
|
|
3810
|
+
return {
|
|
3811
|
+
error: `it holds ${res.data.patchId}, not the patch its directory is named after`
|
|
3812
|
+
};
|
|
3813
|
+
}
|
|
3814
|
+
return {
|
|
3815
|
+
data: res.data
|
|
3816
|
+
};
|
|
3817
|
+
}
|
|
3818
|
+
|
|
3819
|
+
/**
|
|
3820
|
+
* Rebuild the order from the records when the log itself is gone.
|
|
3821
|
+
*
|
|
3822
|
+
* `createdAt` is the only ordering left, and it is not a perfect one, so the
|
|
3823
|
+
* caller reports that the order was guessed rather than presenting it as fact.
|
|
3824
|
+
*/
|
|
3825
|
+
function reconstructLogEntries(patchesDir, dirNames) {
|
|
3826
|
+
const entries = [];
|
|
3827
|
+
for (const name of dirNames) {
|
|
3828
|
+
const res = readJsonFile(path__namespace["default"].join(patchesDir, name, "patch.json"), FSPatch);
|
|
3829
|
+
if (res.error !== undefined || res.data.patchId !== name) {
|
|
3830
|
+
continue;
|
|
3831
|
+
}
|
|
3832
|
+
entries.push({
|
|
3833
|
+
patchId: res.data.patchId,
|
|
3834
|
+
createdAt: res.data.createdAt,
|
|
3835
|
+
path: res.data.path
|
|
3836
|
+
});
|
|
3837
|
+
}
|
|
3838
|
+
entries.sort((a, b) => a.createdAt.localeCompare(b.createdAt));
|
|
3839
|
+
return entries;
|
|
3840
|
+
}
|
|
3841
|
+
|
|
3842
|
+
/** Write a patch record so that a reader sees all of it or none of it. */
|
|
3843
|
+
function writePatchRecord(patchesDir, patchId, record) {
|
|
3844
|
+
const dir = patchDir(patchesDir, patchId);
|
|
3845
|
+
fs__default["default"].mkdirSync(dir, {
|
|
3846
|
+
recursive: true
|
|
3847
|
+
});
|
|
3848
|
+
const file = patchRecordFile(patchesDir, patchId);
|
|
3849
|
+
const tmpFile = `${file}.tmp-${process.pid}`;
|
|
3850
|
+
const fd = fs__default["default"].openSync(tmpFile, "w");
|
|
3851
|
+
try {
|
|
3852
|
+
fs__default["default"].writeSync(fd, JSON.stringify(record));
|
|
3853
|
+
fs__default["default"].fsyncSync(fd);
|
|
3854
|
+
} finally {
|
|
3855
|
+
fs__default["default"].closeSync(fd);
|
|
3856
|
+
}
|
|
3857
|
+
fs__default["default"].renameSync(tmpFile, file);
|
|
3858
|
+
}
|
|
3859
|
+
|
|
3860
|
+
/**
|
|
3861
|
+
* Add a patch to the store.
|
|
3862
|
+
*
|
|
3863
|
+
* Record first, then the log line, and the order is the safety property: an
|
|
3864
|
+
* interrupted append leaves a directory nothing points at, which repair sweeps
|
|
3865
|
+
* up. The reverse order would leave the log naming a patch that is not there -
|
|
3866
|
+
* the exact state this whole rewrite exists to make unreachable.
|
|
3867
|
+
*
|
|
3868
|
+
* Callers must hold the patch lock.
|
|
3869
|
+
*/
|
|
3870
|
+
function appendPatch(patchesDir, record) {
|
|
3871
|
+
const patchId = record.patchId;
|
|
3872
|
+
writePatchRecord(patchesDir, patchId, record);
|
|
3873
|
+
const entry = {
|
|
3874
|
+
patchId,
|
|
3875
|
+
createdAt: record.createdAt,
|
|
3876
|
+
path: record.path
|
|
3877
|
+
};
|
|
3878
|
+
appendPatchLogEntry(patchesLogFile(patchesDir), entry);
|
|
3879
|
+
return entry;
|
|
3880
|
+
}
|
|
3881
|
+
/**
|
|
3882
|
+
* Bring the store back to a state where the log and the directories agree.
|
|
3883
|
+
*
|
|
3884
|
+
* Safe in a way the old layout's repair could never be: the log is a flat list,
|
|
3885
|
+
* so dropping an entry does not orphan the entries after it. There is nothing to
|
|
3886
|
+
* re-link, which is why this can run unattended where re-parenting a chain could
|
|
3887
|
+
* not.
|
|
3888
|
+
*
|
|
3889
|
+
* A patch that cannot be read is removed rather than kept around to fail again
|
|
3890
|
+
* on every load. That does discard unpublished work, which is why
|
|
3891
|
+
* {@link RepairAction} carries the reason, why it is written to
|
|
3892
|
+
* `patches.repair.log`, and why the caller is expected to tell the person
|
|
3893
|
+
* editing.
|
|
3894
|
+
*
|
|
3895
|
+
* Callers must hold the patch lock.
|
|
3896
|
+
*/
|
|
3897
|
+
function repairPatchStore(patchesDir, read) {
|
|
3898
|
+
const actions = [];
|
|
3899
|
+
const needsRewrite = read.problems.some(problem => problem.type === "unreadable-patch" || problem.type === "reconstructed-log" || problem.type === "log");
|
|
3900
|
+
for (const problem of read.problems) {
|
|
3901
|
+
if (problem.type === "unreadable-patch") {
|
|
3902
|
+
remove(problem.dir);
|
|
3903
|
+
actions.push({
|
|
3904
|
+
type: "removed-unreadable-patch",
|
|
3905
|
+
name: problem.name,
|
|
3906
|
+
because: problem.message
|
|
3907
|
+
});
|
|
3908
|
+
} else if (problem.type === "orphan-directory") {
|
|
3909
|
+
remove(problem.dir);
|
|
3910
|
+
actions.push({
|
|
3911
|
+
type: "removed-orphan-directory",
|
|
3912
|
+
name: problem.name
|
|
3913
|
+
});
|
|
3914
|
+
}
|
|
3915
|
+
}
|
|
3916
|
+
if (needsRewrite) {
|
|
3917
|
+
writePatchLogFile(patchesLogFile(patchesDir), read.entries.map(entry => ({
|
|
3918
|
+
patchId: entry.patchId,
|
|
3919
|
+
createdAt: entry.record.createdAt,
|
|
3920
|
+
path: entry.record.path
|
|
3921
|
+
})));
|
|
3922
|
+
actions.push({
|
|
3923
|
+
type: "rewrote-log",
|
|
3924
|
+
entryCount: read.entries.length
|
|
3925
|
+
});
|
|
3926
|
+
}
|
|
3927
|
+
if (actions.length > 0) {
|
|
3928
|
+
recordRepair(patchesDir, actions);
|
|
3929
|
+
}
|
|
3930
|
+
return actions;
|
|
3931
|
+
}
|
|
3932
|
+
function remove(dir) {
|
|
3933
|
+
try {
|
|
3934
|
+
fs__default["default"].rmSync(dir, {
|
|
3935
|
+
recursive: true,
|
|
3936
|
+
force: true
|
|
3937
|
+
});
|
|
3938
|
+
} catch {
|
|
3939
|
+
// Left in place. It is inert either way - nothing reads a directory the log
|
|
3940
|
+
// does not name, and the log has already been rewritten without it.
|
|
3941
|
+
}
|
|
3942
|
+
}
|
|
3943
|
+
|
|
3944
|
+
/**
|
|
3945
|
+
* Leave a trail. Repair drops a person's unpublished edits, and doing that
|
|
3946
|
+
* without saying so anywhere durable is how you get someone certain they saved
|
|
3947
|
+
* something and no way to tell them what happened to it.
|
|
3948
|
+
*/
|
|
3949
|
+
function recordRepair(patchesDir, actions) {
|
|
3950
|
+
const at = new Date().toISOString();
|
|
3951
|
+
const lines = actions.map(action => {
|
|
3952
|
+
if (action.type === "removed-unreadable-patch") {
|
|
3953
|
+
return `${at} removed ${action.name}: ${action.because}`;
|
|
3954
|
+
}
|
|
3955
|
+
if (action.type === "removed-orphan-directory") {
|
|
3956
|
+
return `${at} removed orphan directory ${action.name}`;
|
|
3957
|
+
}
|
|
3958
|
+
return `${at} rewrote the log with ${action.entryCount} entries`;
|
|
3959
|
+
});
|
|
3960
|
+
try {
|
|
3961
|
+
fs__default["default"].appendFileSync(patchRepairLogFile(patchesDir), `${lines.join("\n")}\n`, "utf-8");
|
|
3962
|
+
} catch {
|
|
3963
|
+
// An audit trail that cannot be written is not a reason to fail the repair.
|
|
3964
|
+
}
|
|
3965
|
+
}
|
|
3966
|
+
|
|
3967
|
+
/**
|
|
3968
|
+
* Last resort: move the whole store aside and start empty.
|
|
3969
|
+
*
|
|
3970
|
+
* A rename, never a delete. What is being given up on here is someone's
|
|
3971
|
+
* unpublished work, and the least this can do is say where it went.
|
|
3972
|
+
*
|
|
3973
|
+
* Callers must hold the patch lock.
|
|
3974
|
+
*/
|
|
3975
|
+
function resetPatchStore(patchesDir, reason) {
|
|
3976
|
+
if (!fs__default["default"].existsSync(patchesDir)) {
|
|
3977
|
+
return {
|
|
3978
|
+
movedTo: patchesDir
|
|
3979
|
+
};
|
|
3980
|
+
}
|
|
3981
|
+
const movedTo = `${patchesDir}-corrupt-${new Date().toISOString().replace(/[:.]/g, "-")}`;
|
|
3982
|
+
try {
|
|
3983
|
+
fs__default["default"].renameSync(patchesDir, movedTo);
|
|
3984
|
+
} catch (err) {
|
|
3985
|
+
return {
|
|
3986
|
+
error: `Could not move ${patchesDir} aside: ${err instanceof Error ? err.message : "unknown error"}`
|
|
3987
|
+
};
|
|
3988
|
+
}
|
|
3989
|
+
try {
|
|
3990
|
+
fs__default["default"].mkdirSync(patchesDir, {
|
|
3991
|
+
recursive: true
|
|
3992
|
+
});
|
|
3993
|
+
fs__default["default"].appendFileSync(patchRepairLogFile(patchesDir), `${new Date().toISOString()} reset the store (${reason}); the previous contents are in ${movedTo}\n`, "utf-8");
|
|
3994
|
+
} catch {
|
|
3995
|
+
// The rename is the part that mattered.
|
|
3996
|
+
}
|
|
3997
|
+
return {
|
|
3998
|
+
movedTo
|
|
3999
|
+
};
|
|
4000
|
+
}
|
|
4001
|
+
|
|
4002
|
+
/** One line per problem, for a log line or an API error a person has to act on. */
|
|
4003
|
+
function describePatchStoreProblems(problems) {
|
|
4004
|
+
return problems.map(problem => {
|
|
4005
|
+
switch (problem.type) {
|
|
4006
|
+
case "unreadable-patch":
|
|
4007
|
+
return `The change ${problem.name} could not be read: ${problem.message} (${problem.dir})`;
|
|
4008
|
+
case "orphan-directory":
|
|
4009
|
+
return `${problem.dir} is not listed in the log and is not being used.`;
|
|
4010
|
+
case "reconstructed-log":
|
|
4011
|
+
return `The log was missing, so the order of ${problem.entryCount} change(s) was reconstructed from their timestamps and may not be exact.`;
|
|
4012
|
+
case "log":
|
|
4013
|
+
return describeLogProblem(problem.problem);
|
|
4014
|
+
}
|
|
4015
|
+
});
|
|
4016
|
+
}
|
|
4017
|
+
function describeLogProblem(problem) {
|
|
4018
|
+
switch (problem.type) {
|
|
4019
|
+
case "missing-header":
|
|
4020
|
+
return `The log does not start with its header (it starts with ${JSON.stringify(problem.firstLine)}).`;
|
|
4021
|
+
case "unsupported-version":
|
|
4022
|
+
return `The log says ${JSON.stringify(problem.header)}, which this version of Val does not understand.`;
|
|
4023
|
+
case "unparseable-line":
|
|
4024
|
+
return `Line ${problem.lineNumber} of the log could not be read: ${JSON.stringify(problem.line)}`;
|
|
4025
|
+
case "duplicate-entry":
|
|
4026
|
+
return `The change ${problem.patchId} is listed twice in the log (line ${problem.lineNumber}).`;
|
|
4027
|
+
case "torn-final-line":
|
|
4028
|
+
return `The last line of the log was not finished being written and was ignored.`;
|
|
4029
|
+
}
|
|
4030
|
+
}
|
|
4031
|
+
|
|
4032
|
+
/**
|
|
4033
|
+
* A timed, cross-process lock over the patches directory.
|
|
4034
|
+
*
|
|
4035
|
+
* ## Why there is one at all
|
|
4036
|
+
*
|
|
4037
|
+
* Nothing serialized writes to the patch store before this. Two dev-server route
|
|
4038
|
+
* handlers, two browser tabs and the CLI all wrote to the same directory, and the
|
|
4039
|
+
* one guard that existed — a `mkdir` whose stated purpose was to throw when the
|
|
4040
|
+
* directory already existed — had its `EEXIST` swallowed by a bare `catch {}`, so
|
|
4041
|
+
* a colliding write silently overwrote the patch that was already there. A store
|
|
4042
|
+
* that loses a patch that way loses every patch chained behind it too.
|
|
4043
|
+
*
|
|
4044
|
+
* ## Why a file, and why timed
|
|
4045
|
+
*
|
|
4046
|
+
* A real lock manager is more than this problem needs: the contenders are a
|
|
4047
|
+
* handful of processes on one machine, holding it for milliseconds. What that
|
|
4048
|
+
* does need is to survive a holder that dies — `kill -9` on a dev server, a
|
|
4049
|
+
* laptop lid, a crashed test — without leaving the store permanently unwritable.
|
|
4050
|
+
* Hence an expiry: a lock nobody renews stops being a lock.
|
|
4051
|
+
*
|
|
4052
|
+
* The file is plain `key: value` text on purpose. When something is stuck, the
|
|
4053
|
+
* question is always "who is holding it and since when", and `cat .val/patches.lock`
|
|
4054
|
+
* should answer it without a tool.
|
|
4055
|
+
*
|
|
4056
|
+
* ## The takeover race
|
|
4057
|
+
*
|
|
4058
|
+
* Two processes can decide the same lock is stale at the same moment, and a blind
|
|
4059
|
+
* `unlink` would then let the second one delete the FIRST one's fresh lock. So a
|
|
4060
|
+
* stale lock is only removed after being read twice, unchanged, across a poll
|
|
4061
|
+
* interval, and the `owner` nonce is what "unchanged" means. Whoever wins the
|
|
4062
|
+
* `wx` create afterwards is unique regardless, because that create is atomic —
|
|
4063
|
+
* the double read is what stops a winner being deleted a moment after winning.
|
|
4064
|
+
*/
|
|
4065
|
+
|
|
4066
|
+
const PATCH_LOCK_FILE_NAME = "patches.lock";
|
|
4067
|
+
const DEFAULT_TIMEOUT_MS = 10_000;
|
|
4068
|
+
const DEFAULT_POLL_MS = 25;
|
|
4069
|
+
|
|
4070
|
+
/**
|
|
4071
|
+
* Locks this process is currently holding, released on exit.
|
|
4072
|
+
*
|
|
4073
|
+
* A best-effort courtesy: it turns the common crash — an unhandled throw, a
|
|
4074
|
+
* ctrl-C — into an immediately reusable store instead of one that waits out a
|
|
4075
|
+
* TTL. The TTL is still what makes correctness not depend on this running.
|
|
4076
|
+
*/
|
|
4077
|
+
const heldLocks = new Set();
|
|
4078
|
+
let exitHookInstalled = false;
|
|
4079
|
+
function installExitHook() {
|
|
4080
|
+
if (exitHookInstalled) {
|
|
4081
|
+
return;
|
|
4082
|
+
}
|
|
4083
|
+
exitHookInstalled = true;
|
|
4084
|
+
process.once("exit", () => {
|
|
4085
|
+
heldLocks.forEach(lock => {
|
|
4086
|
+
try {
|
|
4087
|
+
lock.release();
|
|
4088
|
+
} catch {
|
|
4089
|
+
// Nothing useful can be done during exit.
|
|
4090
|
+
}
|
|
4091
|
+
});
|
|
4092
|
+
});
|
|
4093
|
+
}
|
|
4094
|
+
function formatPatchLockInfo(info) {
|
|
4095
|
+
return [`owner: ${info.owner}`, `pid: ${info.pid}`, `host: ${info.host}`, `op: ${info.op}`, `acquired: ${info.acquiredAt}`, `expires: ${info.expiresAt}`, ""].join("\n");
|
|
4096
|
+
}
|
|
4097
|
+
function parsePatchLockInfo(raw) {
|
|
4098
|
+
const fields = {};
|
|
4099
|
+
for (const line of raw.split("\n")) {
|
|
4100
|
+
const separator = line.indexOf(":");
|
|
4101
|
+
if (separator === -1) {
|
|
4102
|
+
continue;
|
|
4103
|
+
}
|
|
4104
|
+
fields[line.slice(0, separator).trim()] = line.slice(separator + 1).trim();
|
|
4105
|
+
}
|
|
4106
|
+
const {
|
|
4107
|
+
owner,
|
|
4108
|
+
pid,
|
|
4109
|
+
host,
|
|
4110
|
+
op,
|
|
4111
|
+
acquired,
|
|
4112
|
+
expires
|
|
4113
|
+
} = fields;
|
|
4114
|
+
if (!owner || !acquired || !expires) {
|
|
4115
|
+
return null;
|
|
4116
|
+
}
|
|
4117
|
+
const parsedPid = Number.parseInt(pid ?? "", 10);
|
|
4118
|
+
return {
|
|
4119
|
+
owner,
|
|
4120
|
+
pid: Number.isNaN(parsedPid) ? -1 : parsedPid,
|
|
4121
|
+
host: host ?? "",
|
|
4122
|
+
op: op ?? "",
|
|
4123
|
+
acquiredAt: acquired,
|
|
4124
|
+
expiresAt: expires
|
|
4125
|
+
};
|
|
4126
|
+
}
|
|
4127
|
+
function readRaw(lockFilePath) {
|
|
4128
|
+
try {
|
|
4129
|
+
return fs__default["default"].readFileSync(lockFilePath, "utf-8");
|
|
4130
|
+
} catch {
|
|
4131
|
+
return null;
|
|
4132
|
+
}
|
|
4133
|
+
}
|
|
4134
|
+
function isExpired(info, nowMs) {
|
|
4135
|
+
// An unparseable lock file is treated as expired. It is either a half-written
|
|
4136
|
+
// acquisition or something that is not a lock at all, and neither should be
|
|
4137
|
+
// able to wedge the store forever.
|
|
4138
|
+
if (info === null) {
|
|
4139
|
+
return true;
|
|
4140
|
+
}
|
|
4141
|
+
const expiresAt = Date.parse(info.expiresAt);
|
|
4142
|
+
if (Number.isNaN(expiresAt)) {
|
|
4143
|
+
return true;
|
|
4144
|
+
}
|
|
4145
|
+
return expiresAt <= nowMs;
|
|
4146
|
+
}
|
|
4147
|
+
function describeHolder(info) {
|
|
4148
|
+
if (info === null) {
|
|
4149
|
+
return "the lock file could not be read";
|
|
4150
|
+
}
|
|
4151
|
+
return `held by pid ${info.pid} on ${info.host || "unknown host"} for "${info.op}" since ${info.acquiredAt} (expires ${info.expiresAt})`;
|
|
4152
|
+
}
|
|
4153
|
+
function sleep(ms) {
|
|
4154
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
4155
|
+
}
|
|
4156
|
+
|
|
4157
|
+
/**
|
|
4158
|
+
* Try to create the lock file exclusively.
|
|
4159
|
+
*
|
|
4160
|
+
* `wx` is `O_CREAT | O_EXCL`: the create either happens or reports EEXIST, with
|
|
4161
|
+
* no window in between. Everything else in this module exists to handle the
|
|
4162
|
+
* EEXIST case; this is the part that actually provides mutual exclusion.
|
|
4163
|
+
*/
|
|
4164
|
+
function tryCreate(lockFilePath, info) {
|
|
4165
|
+
let fd;
|
|
4166
|
+
try {
|
|
4167
|
+
fd = fs__default["default"].openSync(lockFilePath, "wx");
|
|
4168
|
+
} catch (err) {
|
|
4169
|
+
if (typeof err === "object" && err !== null && "code" in err && err.code === "EEXIST") {
|
|
4170
|
+
return "exists";
|
|
4171
|
+
}
|
|
4172
|
+
return {
|
|
4173
|
+
error: err instanceof Error ? err.message : "unknown error"
|
|
4174
|
+
};
|
|
4175
|
+
}
|
|
4176
|
+
try {
|
|
4177
|
+
fs__default["default"].writeSync(fd, formatPatchLockInfo(info));
|
|
4178
|
+
fs__default["default"].fsyncSync(fd);
|
|
4179
|
+
} finally {
|
|
4180
|
+
fs__default["default"].closeSync(fd);
|
|
4181
|
+
}
|
|
4182
|
+
return "created";
|
|
4183
|
+
}
|
|
4184
|
+
async function acquirePatchLock(lockFilePath, options) {
|
|
4185
|
+
const now = options.now ?? Date.now;
|
|
4186
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
4187
|
+
const pollMs = options.pollMs ?? DEFAULT_POLL_MS;
|
|
4188
|
+
const deadline = now() + timeoutMs;
|
|
4189
|
+
try {
|
|
4190
|
+
fs__default["default"].mkdirSync(path__namespace["default"].dirname(lockFilePath), {
|
|
4191
|
+
recursive: true
|
|
4192
|
+
});
|
|
4193
|
+
} catch (err) {
|
|
4194
|
+
return {
|
|
4195
|
+
status: "error",
|
|
4196
|
+
message: `Could not create the directory for ${lockFilePath}: ${err instanceof Error ? err.message : "unknown error"}`
|
|
4197
|
+
};
|
|
4198
|
+
}
|
|
4199
|
+
let lastSeenHolder = null;
|
|
4200
|
+
for (;;) {
|
|
4201
|
+
const acquiredAt = now();
|
|
4202
|
+
const info = {
|
|
4203
|
+
owner: crypto__default["default"].randomUUID(),
|
|
4204
|
+
pid: process.pid,
|
|
4205
|
+
host: os__default["default"].hostname(),
|
|
4206
|
+
op: options.op,
|
|
4207
|
+
acquiredAt: new Date(acquiredAt).toISOString(),
|
|
4208
|
+
expiresAt: new Date(acquiredAt + options.ttlMs).toISOString()
|
|
4209
|
+
};
|
|
4210
|
+
const created = tryCreate(lockFilePath, info);
|
|
4211
|
+
if (created === "created") {
|
|
4212
|
+
const lock = createLock(lockFilePath, info, options.ttlMs, now);
|
|
4213
|
+
installExitHook();
|
|
4214
|
+
heldLocks.add(lock);
|
|
4215
|
+
return {
|
|
4216
|
+
status: "acquired",
|
|
4217
|
+
lock
|
|
4218
|
+
};
|
|
4219
|
+
}
|
|
4220
|
+
if (created !== "exists") {
|
|
4221
|
+
return {
|
|
4222
|
+
status: "error",
|
|
4223
|
+
message: `Could not create ${lockFilePath}: ${created.error}`
|
|
4224
|
+
};
|
|
4225
|
+
}
|
|
4226
|
+
const rawBefore = readRaw(lockFilePath);
|
|
4227
|
+
if (rawBefore === null) {
|
|
4228
|
+
// Released between the failed create and the read. Go straight round again.
|
|
4229
|
+
continue;
|
|
4230
|
+
}
|
|
4231
|
+
lastSeenHolder = parsePatchLockInfo(rawBefore);
|
|
4232
|
+
if (isExpired(lastSeenHolder, now())) {
|
|
4233
|
+
// Read twice across a poll interval before removing it: an unchanged stale
|
|
4234
|
+
// file is one nobody is renewing, and requiring the second read is what
|
|
4235
|
+
// stops two processes both deciding to clear it and the loser wiping the
|
|
4236
|
+
// winner's brand-new lock.
|
|
4237
|
+
await sleep(pollMs);
|
|
4238
|
+
const rawAfter = readRaw(lockFilePath);
|
|
4239
|
+
if (rawAfter === rawBefore && isExpired(lastSeenHolder, now())) {
|
|
4240
|
+
try {
|
|
4241
|
+
fs__default["default"].unlinkSync(lockFilePath);
|
|
4242
|
+
} catch {
|
|
4243
|
+
// Someone else got there first, which is the outcome we wanted anyway.
|
|
4244
|
+
}
|
|
4245
|
+
}
|
|
4246
|
+
continue;
|
|
4247
|
+
}
|
|
4248
|
+
if (now() >= deadline) {
|
|
4249
|
+
return {
|
|
4250
|
+
status: "busy",
|
|
4251
|
+
holder: lastSeenHolder,
|
|
4252
|
+
message: `Timed out after ${timeoutMs}ms waiting for the Val patch lock at ${lockFilePath}: ${describeHolder(lastSeenHolder)}`
|
|
4253
|
+
};
|
|
4254
|
+
}
|
|
4255
|
+
await sleep(pollMs);
|
|
4256
|
+
}
|
|
4257
|
+
}
|
|
4258
|
+
function createLock(lockFilePath, initialInfo, defaultTtlMs, now) {
|
|
4259
|
+
let info = initialInfo;
|
|
4260
|
+
let released = false;
|
|
4261
|
+
const stillOurs = () => {
|
|
4262
|
+
var _parsePatchLockInfo;
|
|
4263
|
+
const raw = readRaw(lockFilePath);
|
|
4264
|
+
if (raw === null) {
|
|
4265
|
+
return false;
|
|
4266
|
+
}
|
|
4267
|
+
return ((_parsePatchLockInfo = parsePatchLockInfo(raw)) === null || _parsePatchLockInfo === void 0 ? void 0 : _parsePatchLockInfo.owner) === info.owner;
|
|
4268
|
+
};
|
|
4269
|
+
const lock = {
|
|
4270
|
+
get info() {
|
|
4271
|
+
return info;
|
|
4272
|
+
},
|
|
4273
|
+
renew(ttlMs) {
|
|
4274
|
+
if (released || !stillOurs()) {
|
|
4275
|
+
return false;
|
|
4276
|
+
}
|
|
4277
|
+
const next = {
|
|
4278
|
+
...info,
|
|
4279
|
+
expiresAt: new Date(now() + (ttlMs ?? defaultTtlMs)).toISOString()
|
|
4280
|
+
};
|
|
4281
|
+
const fd = fs__default["default"].openSync(lockFilePath, "w");
|
|
4282
|
+
try {
|
|
4283
|
+
fs__default["default"].writeSync(fd, formatPatchLockInfo(next));
|
|
4284
|
+
fs__default["default"].fsyncSync(fd);
|
|
4285
|
+
} finally {
|
|
4286
|
+
fs__default["default"].closeSync(fd);
|
|
4287
|
+
}
|
|
4288
|
+
info = next;
|
|
4289
|
+
return true;
|
|
4290
|
+
},
|
|
4291
|
+
release() {
|
|
4292
|
+
if (released) {
|
|
4293
|
+
return;
|
|
4294
|
+
}
|
|
4295
|
+
released = true;
|
|
4296
|
+
heldLocks.delete(lock);
|
|
4297
|
+
// Only ours. If the lock expired and someone else took it, the file now
|
|
4298
|
+
// belongs to them and deleting it would hand the store to a third party.
|
|
4299
|
+
if (stillOurs()) {
|
|
4300
|
+
try {
|
|
4301
|
+
fs__default["default"].unlinkSync(lockFilePath);
|
|
4302
|
+
} catch {
|
|
4303
|
+
// Already gone.
|
|
3338
4304
|
}
|
|
3339
4305
|
}
|
|
3340
|
-
} else {
|
|
3341
|
-
lastNonDeletedPatchIndex = i;
|
|
3342
4306
|
}
|
|
3343
|
-
}
|
|
3344
|
-
return {
|
|
3345
|
-
changedPatches
|
|
3346
4307
|
};
|
|
4308
|
+
return lock;
|
|
4309
|
+
}
|
|
4310
|
+
|
|
4311
|
+
/**
|
|
4312
|
+
* Run `fn` while holding the lock, and always give it back.
|
|
4313
|
+
*
|
|
4314
|
+
* The only shape callers should use: a `release()` that a throw can skip is how
|
|
4315
|
+
* a store ends up locked by a process that has long since moved on.
|
|
4316
|
+
*/
|
|
4317
|
+
async function withPatchLock(lockFilePath, options, fn) {
|
|
4318
|
+
const acquired = await acquirePatchLock(lockFilePath, options);
|
|
4319
|
+
if (acquired.status !== "acquired") {
|
|
4320
|
+
return acquired;
|
|
4321
|
+
}
|
|
4322
|
+
try {
|
|
4323
|
+
return {
|
|
4324
|
+
status: "ok",
|
|
4325
|
+
value: await fn(acquired.lock)
|
|
4326
|
+
};
|
|
4327
|
+
} finally {
|
|
4328
|
+
acquired.lock.release();
|
|
4329
|
+
}
|
|
3347
4330
|
}
|
|
3348
4331
|
|
|
3349
4332
|
async function uploadRemoteFile(contentHost, project, bucket, fileHash, fileExt, fileBuffer, auth) {
|
|
@@ -3414,6 +4397,20 @@ class ValOpsFS extends ValOps {
|
|
|
3414
4397
|
* content lives behind a thunk that `JSON.stringify` drops).
|
|
3415
4398
|
*/
|
|
3416
4399
|
|
|
4400
|
+
/**
|
|
4401
|
+
* Unpublished changes repair threw away, waiting to be told to somebody.
|
|
4402
|
+
*
|
|
4403
|
+
* Held here and drained by the next `getStat`, because stat is the channel
|
|
4404
|
+
* that always flows. The case worth reporting is a repair that removed
|
|
4405
|
+
* EVERYTHING - which is what an old store looks like - and then there is
|
|
4406
|
+
* nothing left for the studio to fetch, so a notice riding on `fetchPatches`
|
|
4407
|
+
* would never be collected. Drained rather than kept, because this has to be
|
|
4408
|
+
* said once: a permanent flag would put a toast on the screen forever.
|
|
4409
|
+
*
|
|
4410
|
+
* Held in memory only: a restart loses it, and that is the right trade. The
|
|
4411
|
+
* durable record is `patches.repair.log`.
|
|
4412
|
+
*/
|
|
4413
|
+
removedPatchNotices = [];
|
|
3417
4414
|
async onInit() {
|
|
3418
4415
|
// do nothing
|
|
3419
4416
|
}
|
|
@@ -3507,14 +4504,28 @@ class ValOpsFS extends ValOps {
|
|
|
3507
4504
|
const serializedSchemas = Object.fromEntries(Object.entries(schemas).map(([path, schema]) => [path, serializeSchemaSafely(schema)]));
|
|
3508
4505
|
const currentJsonEntriesSha = this.jsonEntryFilesFingerprint.compute(serializedSchemas);
|
|
3509
4506
|
const jsonEntryFilePaths = this.jsonEntryFilesFingerprint.entryFilePaths(serializedSchemas);
|
|
3510
|
-
|
|
3511
|
-
|
|
3512
|
-
//
|
|
3513
|
-
|
|
3514
|
-
|
|
3515
|
-
|
|
3516
|
-
|
|
4507
|
+
|
|
4508
|
+
// The SAME read `fetchPatches` delivers from. Announcing out of one source
|
|
4509
|
+
// and delivering out of another is what let this store tell a studio about
|
|
4510
|
+
// 410 unpublished changes and then hand over 359 of them, with nothing in
|
|
4511
|
+
// between reporting that anything was wrong.
|
|
4512
|
+
const announceRes = await this.readStore();
|
|
4513
|
+
if (announceRes.status === "error") {
|
|
4514
|
+
return {
|
|
4515
|
+
type: "error",
|
|
4516
|
+
error: {
|
|
4517
|
+
message: announceRes.message
|
|
4518
|
+
}
|
|
4519
|
+
};
|
|
3517
4520
|
}
|
|
4521
|
+
const patches = announceRes.entries.map(entry => entry.patchId);
|
|
4522
|
+
// Drained here, on the channel that always flows. A repair that removed
|
|
4523
|
+
// everything leaves the studio nothing to fetch, so a notice riding on
|
|
4524
|
+
// `GET /patches` would sit here unread.
|
|
4525
|
+
const removed = this.removedPatchNotices.splice(0, this.removedPatchNotices.length);
|
|
4526
|
+
const removedNotice = removed.length > 0 ? {
|
|
4527
|
+
removed
|
|
4528
|
+
} : {};
|
|
3518
4529
|
// something changed: return immediately
|
|
3519
4530
|
const didChange = !params ||
|
|
3520
4531
|
// An entry file changed on disk: nothing else here can see that, since a
|
|
@@ -3529,6 +4540,7 @@ class ValOpsFS extends ValOps {
|
|
|
3529
4540
|
schemaSha: currentSchemaSha,
|
|
3530
4541
|
sourcesSha: currentSourcesSha,
|
|
3531
4542
|
patches,
|
|
4543
|
+
...removedNotice,
|
|
3532
4544
|
jsonEntriesSha: currentJsonEntriesSha
|
|
3533
4545
|
};
|
|
3534
4546
|
}
|
|
@@ -3653,6 +4665,7 @@ class ValOpsFS extends ValOps {
|
|
|
3653
4665
|
schemaSha: currentSchemaSha,
|
|
3654
4666
|
sourcesSha: currentSourcesSha,
|
|
3655
4667
|
patches,
|
|
4668
|
+
...removedNotice,
|
|
3656
4669
|
jsonEntriesSha: currentJsonEntriesSha
|
|
3657
4670
|
};
|
|
3658
4671
|
} catch (err) {
|
|
@@ -3672,134 +4685,152 @@ class ValOpsFS extends ValOps {
|
|
|
3672
4685
|
};
|
|
3673
4686
|
}
|
|
3674
4687
|
}
|
|
3675
|
-
async readPatches(includes) {
|
|
3676
|
-
const patchesCacheDir = this.getPatchesDir();
|
|
3677
|
-
let patchJsonFiles = [];
|
|
3678
|
-
if (!this.host.directoryExists || this.host.directoryExists && this.host.directoryExists(patchesCacheDir)) {
|
|
3679
|
-
patchJsonFiles = this.host.readDirectory(patchesCacheDir, ["patch.json"], [], []);
|
|
3680
|
-
}
|
|
3681
|
-
const patches = {};
|
|
3682
|
-
const errors = [];
|
|
3683
|
-
const parsedUnsortedFsPatches = patchJsonFiles.map(file => path__namespace["default"].basename(path__namespace["default"].dirname(file))).map(patchDir => [patchDir, this.parseJsonFile(this.getPatchFilePath(patchDir), FSPatch), this.host.fileExists(this.getPatchBaseFile(patchDir)) ? this.parseJsonFile(this.getPatchBaseFile(patchDir), FSPatchBase) : undefined]);
|
|
3684
|
-
parsedUnsortedFsPatches.forEach(([dir, parsedPatch, parsedBase]) => {
|
|
3685
|
-
if (parsedPatch.error) {
|
|
3686
|
-
errors.push({
|
|
3687
|
-
...parsedPatch.error,
|
|
3688
|
-
parentPatchId: dir
|
|
3689
|
-
});
|
|
3690
|
-
} else if (parsedBase && parsedBase.error) {
|
|
3691
|
-
errors.push({
|
|
3692
|
-
...parsedBase.error,
|
|
3693
|
-
parentPatchId: dir
|
|
3694
|
-
});
|
|
3695
|
-
} else {
|
|
3696
|
-
const patchId = parsedPatch.data.patchId;
|
|
3697
|
-
if (includes && includes.length > 0 && !includes.includes(patchId)) {
|
|
3698
|
-
return;
|
|
3699
|
-
}
|
|
3700
|
-
patches[patchId] = {
|
|
3701
|
-
path: parsedPatch.data.path,
|
|
3702
|
-
patch: parsedPatch.data.patch,
|
|
3703
|
-
parentRef: parsedPatch.data.parentRef,
|
|
3704
|
-
baseSha: parsedPatch.data.baseSha,
|
|
3705
|
-
createdAt: parsedPatch.data.createdAt,
|
|
3706
|
-
authorId: parsedPatch.data.authorId,
|
|
3707
|
-
appliedAt: null
|
|
3708
|
-
};
|
|
3709
|
-
}
|
|
3710
|
-
});
|
|
3711
4688
|
|
|
3712
|
-
|
|
3713
|
-
|
|
4689
|
+
/**
|
|
4690
|
+
* The one read every other read comes out of.
|
|
4691
|
+
*
|
|
4692
|
+
* `getStat` announces from this and `fetchPatches` delivers from this, so the
|
|
4693
|
+
* two cannot disagree. They used to: `getStat` counted the directories on disk
|
|
4694
|
+
* while `fetchPatches` walked the parent links between them, and when a single
|
|
4695
|
+
* record went missing the first said 410 and the second said 359 — with no
|
|
4696
|
+
* error anywhere, because a walk that runs out of links just stops.
|
|
4697
|
+
*
|
|
4698
|
+
* Anything wrong is reported, then repaired, and reset only if repair does not
|
|
4699
|
+
* settle it. In that order, and never silently.
|
|
4700
|
+
*/
|
|
4701
|
+
async readStore() {
|
|
4702
|
+
const read = readPatchStore(this.getPatchesDir());
|
|
4703
|
+
if (read.status === "unreadable") {
|
|
3714
4704
|
return {
|
|
3715
|
-
|
|
3716
|
-
|
|
4705
|
+
status: "error",
|
|
4706
|
+
message: read.message
|
|
3717
4707
|
};
|
|
3718
4708
|
}
|
|
3719
|
-
|
|
3720
|
-
|
|
3721
|
-
|
|
3722
|
-
|
|
3723
|
-
|
|
3724
|
-
|
|
4709
|
+
if (read.problems.length === 0) {
|
|
4710
|
+
return {
|
|
4711
|
+
status: "ok",
|
|
4712
|
+
entries: read.entries
|
|
4713
|
+
};
|
|
4714
|
+
}
|
|
4715
|
+
return this.repairStore(read);
|
|
3725
4716
|
}
|
|
3726
|
-
|
|
3727
|
-
|
|
3728
|
-
|
|
3729
|
-
|
|
4717
|
+
|
|
4718
|
+
/**
|
|
4719
|
+
* Report, repair, and reset only if repair did not take.
|
|
4720
|
+
*
|
|
4721
|
+
* Reached only when something is actually wrong, so the healthy path — which
|
|
4722
|
+
* is every stat poll — never touches the lock.
|
|
4723
|
+
*/
|
|
4724
|
+
async repairStore(read) {
|
|
4725
|
+
const patchesDir = this.getPatchesDir();
|
|
4726
|
+
for (const problem of describePatchStoreProblems(read.problems)) {
|
|
4727
|
+
console.warn("Val: something is wrong with the patch store.", problem);
|
|
4728
|
+
}
|
|
4729
|
+
const locked = await withPatchLock(this.getPatchLockFile(), {
|
|
4730
|
+
ttlMs: 30_000,
|
|
4731
|
+
op: "repair the patch store"
|
|
4732
|
+
}, () => {
|
|
4733
|
+
// Read again under the lock. What was seen above was seen without it, so
|
|
4734
|
+
// a write may have landed since — and repairing against a stale picture
|
|
4735
|
+
// is how the old delete path destroyed live patches.
|
|
4736
|
+
const current = readPatchStore(patchesDir);
|
|
4737
|
+
if (current.status !== "ok") {
|
|
4738
|
+
return {
|
|
4739
|
+
status: "error",
|
|
4740
|
+
message: current.message
|
|
4741
|
+
};
|
|
4742
|
+
}
|
|
4743
|
+
if (current.problems.length === 0) {
|
|
3730
4744
|
return {
|
|
3731
|
-
|
|
3732
|
-
|
|
4745
|
+
status: "ok",
|
|
4746
|
+
entries: current.entries
|
|
3733
4747
|
};
|
|
3734
4748
|
}
|
|
3735
|
-
|
|
3736
|
-
|
|
3737
|
-
|
|
3738
|
-
|
|
4749
|
+
for (const action of repairPatchStore(patchesDir, current)) {
|
|
4750
|
+
if (action.type === "removed-unreadable-patch") {
|
|
4751
|
+
console.warn(`Val: removed the unpublished change ${action.name} because ${action.because}. See ${patchesDir}/${PATCH_REPAIR_LOG_FILE_NAME}.`);
|
|
4752
|
+
// Queued rather than only logged: this is someone's unpublished
|
|
4753
|
+
// work being thrown away, and a server console is not somewhere the
|
|
4754
|
+
// person editing is looking.
|
|
4755
|
+
this.removedPatchNotices.push({
|
|
4756
|
+
patchId: action.name,
|
|
4757
|
+
reason: action.because
|
|
4758
|
+
});
|
|
4759
|
+
} else if (action.type === "removed-orphan-directory") {
|
|
4760
|
+
// Not queued. Nothing ever read this - it is a record whose log line
|
|
4761
|
+
// never landed - so there is no edit to tell anyone about.
|
|
4762
|
+
console.warn(`Val: removed the unused patch directory ${action.name}.`);
|
|
4763
|
+
}
|
|
3739
4764
|
}
|
|
3740
|
-
|
|
4765
|
+
const after = readPatchStore(patchesDir);
|
|
4766
|
+
if (after.status === "ok" && after.problems.length === 0) {
|
|
4767
|
+
return {
|
|
4768
|
+
status: "ok",
|
|
4769
|
+
entries: after.entries
|
|
4770
|
+
};
|
|
4771
|
+
}
|
|
4772
|
+
// Repair did not settle it, so the store is not something anyone can
|
|
4773
|
+
// describe any more. Move it aside — a rename, never a delete — and say
|
|
4774
|
+
// where it went.
|
|
4775
|
+
const reset = resetPatchStore(patchesDir, "it could not be repaired automatically");
|
|
4776
|
+
if ("error" in reset) {
|
|
4777
|
+
return {
|
|
4778
|
+
status: "error",
|
|
4779
|
+
message: reset.error
|
|
4780
|
+
};
|
|
4781
|
+
}
|
|
4782
|
+
console.warn(`Val: the patch store could not be repaired, so it was reset. The previous contents are in ${reset.movedTo}.`);
|
|
4783
|
+
return {
|
|
4784
|
+
status: "ok",
|
|
4785
|
+
entries: []
|
|
4786
|
+
};
|
|
3741
4787
|
});
|
|
3742
|
-
|
|
3743
|
-
|
|
3744
|
-
|
|
3745
|
-
|
|
3746
|
-
|
|
3747
|
-
|
|
3748
|
-
|
|
3749
|
-
|
|
3750
|
-
errors,
|
|
3751
|
-
patches: allPatches
|
|
3752
|
-
} = await this.readPatches();
|
|
3753
|
-
for (const [patchIdS, patch] of Object.entries(allPatches)) {
|
|
3754
|
-
const patchId = patchIdS;
|
|
3755
|
-
patches[patchId] = {
|
|
3756
|
-
patch: excludePatchOps ? undefined : patch.patch,
|
|
3757
|
-
parentRef: patch.parentRef,
|
|
3758
|
-
path: patch.path,
|
|
3759
|
-
baseSha: patch.baseSha,
|
|
3760
|
-
createdAt: patch.createdAt,
|
|
3761
|
-
authorId: patch.authorId,
|
|
3762
|
-
appliedAt: patch.appliedAt
|
|
4788
|
+
if (locked.status !== "ok") {
|
|
4789
|
+
// Someone else holds the lock, quite possibly to run this same repair.
|
|
4790
|
+
// Serving what was read is safe — those entries are the ones that ARE
|
|
4791
|
+
// readable — and the next read tries again.
|
|
4792
|
+
console.warn(`Val: could not repair the patch store now. ${locked.message}`);
|
|
4793
|
+
return {
|
|
4794
|
+
status: "ok",
|
|
4795
|
+
entries: read.entries
|
|
3763
4796
|
};
|
|
3764
4797
|
}
|
|
3765
|
-
|
|
4798
|
+
return locked.value;
|
|
4799
|
+
}
|
|
4800
|
+
async fetchPatches(filters) {
|
|
4801
|
+
const storeRes = await this.readStore();
|
|
4802
|
+
if (storeRes.status === "error") {
|
|
4803
|
+
// Surfaced, not returned as an empty list. An empty answer to "what is
|
|
4804
|
+
// pending" is indistinguishable from "nothing is pending", and the studio
|
|
4805
|
+
// would render published content over unpublished edits without a word.
|
|
4806
|
+
const none = [];
|
|
3766
4807
|
return {
|
|
3767
|
-
patches,
|
|
3768
|
-
|
|
4808
|
+
patches: none,
|
|
4809
|
+
error: {
|
|
4810
|
+
message: storeRes.message
|
|
4811
|
+
}
|
|
4812
|
+
// The cast is unavoidable: the return type is conditional on a generic
|
|
4813
|
+
// that TypeScript cannot narrow from a value.
|
|
3769
4814
|
};
|
|
3770
4815
|
}
|
|
4816
|
+
const requested = filters.patchIds && filters.patchIds.length > 0 ? new Set(filters.patchIds) : null;
|
|
4817
|
+
const patches = storeRes.entries.filter(entry => requested === null || requested.has(entry.patchId)).map(entry => ({
|
|
4818
|
+
patchId: entry.patchId,
|
|
4819
|
+
path: entry.record.path,
|
|
4820
|
+
patch: filters.excludePatchOps ? undefined : entry.record.patch,
|
|
4821
|
+
createdAt: entry.record.createdAt,
|
|
4822
|
+
authorId: entry.record.authorId,
|
|
4823
|
+
baseSha: entry.record.baseSha,
|
|
4824
|
+
// Publishing deletes the whole store, so in fs mode a patch that is
|
|
4825
|
+
// still here has not been applied. `base.json` is read for its parse
|
|
4826
|
+
// errors, not for this.
|
|
4827
|
+
appliedAt: null
|
|
4828
|
+
}));
|
|
3771
4829
|
return {
|
|
3772
4830
|
patches
|
|
3773
4831
|
};
|
|
3774
4832
|
}
|
|
3775
4833
|
|
|
3776
|
-
// #region createPatchChain
|
|
3777
|
-
createPatchChain(unsortedPatchRecord) {
|
|
3778
|
-
var _Object$entries$find;
|
|
3779
|
-
// TODO: Error handling
|
|
3780
|
-
const nextPatch = {};
|
|
3781
|
-
Object.keys(unsortedPatchRecord).forEach(patchId => {
|
|
3782
|
-
const patch = unsortedPatchRecord[patchId];
|
|
3783
|
-
if (patch.parentRef.type === "head") {
|
|
3784
|
-
nextPatch["head"] = patchId;
|
|
3785
|
-
} else {
|
|
3786
|
-
nextPatch[patch.parentRef.patchId] = patchId;
|
|
3787
|
-
}
|
|
3788
|
-
});
|
|
3789
|
-
const sortedPatches = [];
|
|
3790
|
-
let nextPatchId = (_Object$entries$find = Object.entries(unsortedPatchRecord).find(([, patch]) => patch.parentRef.type === "head")) === null || _Object$entries$find === void 0 ? void 0 : _Object$entries$find[0];
|
|
3791
|
-
while (!!nextPatchId && nextPatchId in unsortedPatchRecord) {
|
|
3792
|
-
const patch = unsortedPatchRecord[nextPatchId];
|
|
3793
|
-
delete patch["parentRef"];
|
|
3794
|
-
sortedPatches.push({
|
|
3795
|
-
...patch,
|
|
3796
|
-
patchId: nextPatchId
|
|
3797
|
-
});
|
|
3798
|
-
nextPatchId = nextPatch[nextPatchId];
|
|
3799
|
-
}
|
|
3800
|
-
return sortedPatches;
|
|
3801
|
-
}
|
|
3802
|
-
|
|
3803
4834
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
3804
4835
|
parseJsonFile(filePath, parser) {
|
|
3805
4836
|
if (!this.host.fileExists(filePath)) {
|
|
@@ -3874,53 +4905,60 @@ class ValOpsFS extends ValOps {
|
|
|
3874
4905
|
};
|
|
3875
4906
|
}
|
|
3876
4907
|
}
|
|
3877
|
-
async saveSourceFilePatch(path, patch, patchId,
|
|
3878
|
-
const
|
|
3879
|
-
|
|
3880
|
-
|
|
3881
|
-
|
|
3882
|
-
|
|
3883
|
-
|
|
3884
|
-
|
|
3885
|
-
|
|
3886
|
-
|
|
3887
|
-
|
|
3888
|
-
|
|
3889
|
-
|
|
3890
|
-
|
|
3891
|
-
|
|
3892
|
-
|
|
3893
|
-
|
|
3894
|
-
|
|
3895
|
-
|
|
3896
|
-
|
|
3897
|
-
|
|
3898
|
-
|
|
3899
|
-
|
|
3900
|
-
|
|
4908
|
+
async saveSourceFilePatch(path, patch, patchId, _parentRef, authorId, sessionId) {
|
|
4909
|
+
const patchesDir = this.getPatchesDir();
|
|
4910
|
+
const record = {
|
|
4911
|
+
patch,
|
|
4912
|
+
patchId,
|
|
4913
|
+
path,
|
|
4914
|
+
authorId,
|
|
4915
|
+
sessionId,
|
|
4916
|
+
baseSha: await this.getBaseSha(),
|
|
4917
|
+
coreVersion: core.Internal.VERSION.core,
|
|
4918
|
+
createdAt: new Date().toISOString()
|
|
4919
|
+
};
|
|
4920
|
+
/*
|
|
4921
|
+
* `parentRef` is ignored here, and that is the point of the log.
|
|
4922
|
+
*
|
|
4923
|
+
* It used to be the directory name, which is how a client working from a
|
|
4924
|
+
* stale view could write a patch whose parent had never landed and strand
|
|
4925
|
+
* every patch behind it. With the order held in one append-only list the
|
|
4926
|
+
* server decides where a patch goes: it goes last. There is no parent to
|
|
4927
|
+
* name, nothing to point at, and so nothing that can point at nothing.
|
|
4928
|
+
*
|
|
4929
|
+
* The parameter stays because `ValOpsHttp` genuinely needs it — the content
|
|
4930
|
+
* api runs its own chain and is told the parent — but in fs mode a stale
|
|
4931
|
+
* parent is not an error to refuse, it is a fact with no consequences.
|
|
4932
|
+
*
|
|
4933
|
+
* What that gives up is early detection of a patch computed against a
|
|
4934
|
+
* different state: two tabs editing the same array can produce an op that no
|
|
4935
|
+
* longer fits. That is caught where it actually shows — applying the patch —
|
|
4936
|
+
* and handled by dropping it and saying so, rather than by refusing writes
|
|
4937
|
+
* up front and making every interleaved edit a round trip.
|
|
4938
|
+
*/
|
|
4939
|
+
const locked = await withPatchLock(this.getPatchLockFile(), {
|
|
4940
|
+
ttlMs: 10_000,
|
|
4941
|
+
op: "PUT /patches"
|
|
4942
|
+
}, () => {
|
|
4943
|
+
try {
|
|
4944
|
+
appendPatch(patchesDir, record);
|
|
4945
|
+
} catch (err) {
|
|
3901
4946
|
return fp.result.err({
|
|
3902
4947
|
errorType: "other",
|
|
3903
|
-
|
|
3904
|
-
message: "Failed to write patch file"
|
|
4948
|
+
message: `Failed to write the patch: ${err instanceof Error ? err.message : "unknown error"}`
|
|
3905
4949
|
});
|
|
3906
4950
|
}
|
|
3907
4951
|
return fp.result.ok({
|
|
3908
4952
|
patchId
|
|
3909
4953
|
});
|
|
3910
|
-
}
|
|
3911
|
-
|
|
3912
|
-
return fp.result.err({
|
|
3913
|
-
errorType: "other",
|
|
3914
|
-
error: err,
|
|
3915
|
-
message: err.message
|
|
3916
|
-
});
|
|
3917
|
-
}
|
|
4954
|
+
});
|
|
4955
|
+
if (locked.status !== "ok") {
|
|
3918
4956
|
return fp.result.err({
|
|
3919
4957
|
errorType: "other",
|
|
3920
|
-
|
|
3921
|
-
message: "Unknown error"
|
|
4958
|
+
message: locked.message
|
|
3922
4959
|
});
|
|
3923
4960
|
}
|
|
4961
|
+
return locked.value;
|
|
3924
4962
|
}
|
|
3925
4963
|
async getSourceFile(path) {
|
|
3926
4964
|
const filePath = path__namespace["default"].join(this.rootDir, path);
|
|
@@ -3957,10 +4995,14 @@ class ValOpsFS extends ValOps {
|
|
|
3957
4995
|
};
|
|
3958
4996
|
}
|
|
3959
4997
|
}
|
|
3960
|
-
async saveBase64EncodedBinaryFileFromPatch(filePath,
|
|
3961
|
-
|
|
3962
|
-
|
|
3963
|
-
|
|
4998
|
+
async saveBase64EncodedBinaryFileFromPatch(filePath, _parentRef, patchId, data, _type, metadata) {
|
|
4999
|
+
// Keyed by the patch's own id, so the parent is not needed and is not asked
|
|
5000
|
+
// for. Uploads arrive before the patch record does, which is fine: the
|
|
5001
|
+
// directory sits there unreferenced until the log line that names it lands,
|
|
5002
|
+
// and repair sweeps it up if that never happens.
|
|
5003
|
+
const patchesDir = this.getPatchesDir();
|
|
5004
|
+
const patchFilePath = patchBinaryFile(patchesDir, patchId, filePath);
|
|
5005
|
+
const metadataFilePath = patchBinaryFileMetadata(patchesDir, patchId, filePath);
|
|
3964
5006
|
try {
|
|
3965
5007
|
if (data === null) {
|
|
3966
5008
|
this.host.deleteFile(patchFilePath);
|
|
@@ -4000,15 +5042,7 @@ class ValOpsFS extends ValOps {
|
|
|
4000
5042
|
}
|
|
4001
5043
|
}
|
|
4002
5044
|
async getBase64EncodedBinaryFileMetadataFromPatch(filePath, type, patchId) {
|
|
4003
|
-
const
|
|
4004
|
-
if (fp.result.isErr(patchDirRes)) {
|
|
4005
|
-
return {
|
|
4006
|
-
errors: [{
|
|
4007
|
-
message: "Failed to get patch dir from patch id"
|
|
4008
|
-
}]
|
|
4009
|
-
};
|
|
4010
|
-
}
|
|
4011
|
-
const metadataFilePath = this.getBinaryFileMetadataPath(filePath, patchDirRes.value);
|
|
5045
|
+
const metadataFilePath = patchBinaryFileMetadata(this.getPatchesDir(), patchId, filePath);
|
|
4012
5046
|
if (!this.host.fileExists(metadataFilePath)) {
|
|
4013
5047
|
return {
|
|
4014
5048
|
errors: [{
|
|
@@ -4044,95 +5078,132 @@ class ValOpsFS extends ValOps {
|
|
|
4044
5078
|
};
|
|
4045
5079
|
}
|
|
4046
5080
|
async getBase64EncodedBinaryFileFromPatch(filePath, patchId) {
|
|
4047
|
-
|
|
4048
|
-
|
|
4049
|
-
|
|
4050
|
-
}
|
|
4051
|
-
const absPath = this.getBinaryFilePath(filePath, patchDirRes.value);
|
|
5081
|
+
// Straight from the id. This used to read and parse every patch on disk to
|
|
5082
|
+
// work out which directory the file was under, on every single image request.
|
|
5083
|
+
const absPath = patchBinaryFile(this.getPatchesDir(), patchId, filePath);
|
|
4052
5084
|
if (!this.host.fileExists(absPath)) {
|
|
4053
5085
|
return null;
|
|
4054
5086
|
}
|
|
4055
5087
|
return this.host.readBinaryFile(absPath);
|
|
4056
5088
|
}
|
|
4057
5089
|
async deletePatches(patchIds) {
|
|
4058
|
-
const
|
|
4059
|
-
const
|
|
4060
|
-
|
|
5090
|
+
const patchesDir = this.getPatchesDir();
|
|
5091
|
+
const requested = new Set(patchIds);
|
|
5092
|
+
const locked = await withPatchLock(this.getPatchLockFile(), {
|
|
5093
|
+
ttlMs: 30_000,
|
|
5094
|
+
op: "DELETE /patches"
|
|
5095
|
+
}, () => {
|
|
5096
|
+
const read = readPatchStore(patchesDir);
|
|
5097
|
+
if (read.status !== "ok") {
|
|
5098
|
+
return {
|
|
5099
|
+
status: "error",
|
|
5100
|
+
message: read.message
|
|
5101
|
+
};
|
|
5102
|
+
}
|
|
5103
|
+
/*
|
|
5104
|
+
* Deleting is now dropping lines from a list.
|
|
5105
|
+
*
|
|
5106
|
+
* It used to be a three-step re-link per surviving patch — rewrite a
|
|
5107
|
+
* record, remove the destination directory, rename another one over it —
|
|
5108
|
+
* with the plan computed from a snapshot taken before hundreds of file
|
|
5109
|
+
* reads, every failure swallowed by a `console.error`, and one step that
|
|
5110
|
+
* unconditionally removed whatever occupied the destination. An
|
|
5111
|
+
* interruption anywhere in it stranded every patch after the hole.
|
|
5112
|
+
*/
|
|
5113
|
+
writePatchLogFile(patchesLogFile(patchesDir), read.entries.filter(entry => !requested.has(entry.patchId)).map(entry => ({
|
|
5114
|
+
patchId: entry.patchId,
|
|
5115
|
+
createdAt: entry.record.createdAt,
|
|
5116
|
+
path: entry.record.path
|
|
5117
|
+
})));
|
|
5118
|
+
// The log first, then the directories. A crash in between leaves
|
|
5119
|
+
// directories nothing names, which is inert and swept up on the next
|
|
5120
|
+
// read; the other order would leave the log naming patches that are gone.
|
|
5121
|
+
const deleted = [];
|
|
5122
|
+
const errors = {};
|
|
5123
|
+
for (const patchId of patchIds) {
|
|
5124
|
+
try {
|
|
5125
|
+
fs__default["default"].rmSync(patchDir(patchesDir, patchId), {
|
|
5126
|
+
recursive: true,
|
|
5127
|
+
force: true
|
|
5128
|
+
});
|
|
5129
|
+
deleted.push(patchId);
|
|
5130
|
+
} catch (err) {
|
|
5131
|
+
// Reported. This endpoint used to answer "deleted" unconditionally —
|
|
5132
|
+
// `deleted` and `errors` were never written to at all — so a delete
|
|
5133
|
+
// that failed looked exactly like one that worked.
|
|
5134
|
+
errors[patchId] = {
|
|
5135
|
+
message: `Could not remove the files for ${patchId}: ${err instanceof Error ? err.message : "unknown error"}`
|
|
5136
|
+
};
|
|
5137
|
+
}
|
|
5138
|
+
}
|
|
5139
|
+
return {
|
|
5140
|
+
status: "ok",
|
|
5141
|
+
deleted,
|
|
5142
|
+
errors
|
|
5143
|
+
};
|
|
5144
|
+
});
|
|
5145
|
+
if (locked.status !== "ok") {
|
|
5146
|
+
return {
|
|
5147
|
+
error: {
|
|
5148
|
+
message: locked.message
|
|
5149
|
+
}
|
|
5150
|
+
};
|
|
5151
|
+
}
|
|
5152
|
+
if (locked.value.status === "error") {
|
|
4061
5153
|
return {
|
|
4062
5154
|
error: {
|
|
4063
|
-
message:
|
|
5155
|
+
message: locked.value.message
|
|
4064
5156
|
}
|
|
4065
5157
|
};
|
|
4066
5158
|
}
|
|
4067
|
-
const
|
|
4068
|
-
|
|
5159
|
+
const {
|
|
5160
|
+
deleted,
|
|
5161
|
+
errors
|
|
5162
|
+
} = locked.value;
|
|
5163
|
+
if (Object.keys(errors).length > 0) {
|
|
5164
|
+
return {
|
|
5165
|
+
deleted,
|
|
5166
|
+
errors
|
|
5167
|
+
};
|
|
5168
|
+
}
|
|
4069
5169
|
return {
|
|
4070
5170
|
deleted
|
|
4071
5171
|
};
|
|
4072
5172
|
}
|
|
4073
5173
|
async deleteAllPatches() {
|
|
4074
|
-
const
|
|
4075
|
-
const
|
|
4076
|
-
|
|
4077
|
-
|
|
4078
|
-
|
|
4079
|
-
|
|
4080
|
-
|
|
4081
|
-
|
|
5174
|
+
const patchesDir = this.getPatchesDir();
|
|
5175
|
+
const locked = await withPatchLock(this.getPatchLockFile(), {
|
|
5176
|
+
ttlMs: 30_000,
|
|
5177
|
+
op: "delete all patches"
|
|
5178
|
+
}, () => {
|
|
5179
|
+
if (!fs__default["default"].existsSync(patchesDir)) {
|
|
5180
|
+
return {};
|
|
5181
|
+
}
|
|
5182
|
+
const tmpDir = path__namespace["default"].join(this.rootDir, ValOpsFS.VAL_DIR, "patches-deleted-" + crypto.randomUUID());
|
|
5183
|
+
try {
|
|
5184
|
+
// One rename takes the log and the directories together, so the store
|
|
5185
|
+
// is empty and self-consistent from the first instant rather than
|
|
5186
|
+
// part-way through. It is also the cleanup route for a store this
|
|
5187
|
+
// version refuses to read: discarding everything always works.
|
|
5188
|
+
this.host.moveDir(patchesDir, tmpDir);
|
|
5189
|
+
this.host.deleteDir(tmpDir);
|
|
5190
|
+
return {};
|
|
5191
|
+
} catch (err) {
|
|
4082
5192
|
return {
|
|
4083
5193
|
error: {
|
|
4084
|
-
message: `Got an error while deleting patches: ${err.message}`
|
|
5194
|
+
message: `Got an error while deleting patches: ${err instanceof Error ? err.message : "unknown error"}`
|
|
4085
5195
|
}
|
|
4086
5196
|
};
|
|
4087
5197
|
}
|
|
5198
|
+
});
|
|
5199
|
+
if (locked.status !== "ok") {
|
|
4088
5200
|
return {
|
|
4089
5201
|
error: {
|
|
4090
|
-
message:
|
|
5202
|
+
message: locked.message
|
|
4091
5203
|
}
|
|
4092
5204
|
};
|
|
4093
5205
|
}
|
|
4094
|
-
|
|
4095
|
-
updateOrderedPatches(updates, patchDirMap, deletePatchIds) {
|
|
4096
|
-
for (const patchId of deletePatchIds) {
|
|
4097
|
-
const patchDir = patchDirMap[patchId];
|
|
4098
|
-
if (!patchDir) {
|
|
4099
|
-
console.error("Could not find patch dir for patch id scheduled for deletion: ", patchId);
|
|
4100
|
-
continue;
|
|
4101
|
-
}
|
|
4102
|
-
try {
|
|
4103
|
-
this.host.deleteDir(this.getFullPatchDir(patchDir));
|
|
4104
|
-
} catch (err) {
|
|
4105
|
-
console.error("Failed to delete patch dir", err);
|
|
4106
|
-
}
|
|
4107
|
-
}
|
|
4108
|
-
for (const [patchIdS, parentRef] of Object.entries(updates.changedPatches)) {
|
|
4109
|
-
const prevParentPatchId = patchDirMap[patchIdS];
|
|
4110
|
-
if (!prevParentPatchId) {
|
|
4111
|
-
console.error("Could not find previous parent patch id for deleted patch id: ", patchIdS);
|
|
4112
|
-
continue;
|
|
4113
|
-
}
|
|
4114
|
-
const newParentPatchId = parentRef.type === "head" ? "head" : parentRef.patchId;
|
|
4115
|
-
const currentPatchDataRes = this.parseJsonFile(this.getPatchFilePath(prevParentPatchId), FSPatch);
|
|
4116
|
-
if (currentPatchDataRes.error) {
|
|
4117
|
-
console.error("Failed to parse patch file while fixing patch chain after deleted patch", {
|
|
4118
|
-
updates
|
|
4119
|
-
}, currentPatchDataRes.error);
|
|
4120
|
-
continue;
|
|
4121
|
-
}
|
|
4122
|
-
const newPatchData = currentPatchDataRes.data;
|
|
4123
|
-
newPatchData.parentRef = parentRef;
|
|
4124
|
-
try {
|
|
4125
|
-
this.host.writeUf8File(this.getPatchFilePath(prevParentPatchId), JSON.stringify(newPatchData));
|
|
4126
|
-
if (this.host.directoryExists(this.getFullPatchDir(newParentPatchId))) {
|
|
4127
|
-
this.host.deleteDir(this.getFullPatchDir(newParentPatchId));
|
|
4128
|
-
}
|
|
4129
|
-
this.host.moveDir(this.getFullPatchDir(prevParentPatchId), this.getFullPatchDir(newParentPatchId));
|
|
4130
|
-
} catch (err) {
|
|
4131
|
-
console.error("Failed fix patch chain after deleted patch", {
|
|
4132
|
-
updates
|
|
4133
|
-
}, err);
|
|
4134
|
-
}
|
|
4135
|
-
}
|
|
5206
|
+
return locked.value;
|
|
4136
5207
|
}
|
|
4137
5208
|
async saveOrUploadFiles(preparedCommit, mode, auth) {
|
|
4138
5209
|
const updatedFiles = [];
|
|
@@ -4195,30 +5266,13 @@ class ValOpsFS extends ValOps {
|
|
|
4195
5266
|
uploadedRemoteRefs.push(ref);
|
|
4196
5267
|
}
|
|
4197
5268
|
}
|
|
4198
|
-
const patchIdToPatchDirMapRes = await this.getParentPatchIdFromPatchIdMap();
|
|
4199
|
-
if (fp.result.isErr(patchIdToPatchDirMapRes)) {
|
|
4200
|
-
return {
|
|
4201
|
-
updatedFiles,
|
|
4202
|
-
uploadedRemoteRefs,
|
|
4203
|
-
errors
|
|
4204
|
-
};
|
|
4205
|
-
}
|
|
4206
|
-
const patchIdToPatchDirMap = patchIdToPatchDirMapRes.value;
|
|
4207
5269
|
for (const [ref, {
|
|
4208
5270
|
patchId
|
|
4209
5271
|
}] of localFileDescriptors) {
|
|
4210
5272
|
const filePath = ref;
|
|
4211
5273
|
const absPath = path__namespace["default"].join(this.rootDir, ...filePath.split("/"));
|
|
4212
5274
|
try {
|
|
4213
|
-
|
|
4214
|
-
if (!patchDir) {
|
|
4215
|
-
errors[absPath] = {
|
|
4216
|
-
message: "Failed to find PatchDir for PatchId " + patchId,
|
|
4217
|
-
filePath: filePath
|
|
4218
|
-
};
|
|
4219
|
-
continue;
|
|
4220
|
-
}
|
|
4221
|
-
this.host.copyFile(this.getBinaryFilePath(filePath, patchDir), absPath);
|
|
5275
|
+
this.host.copyFile(patchBinaryFile(this.getPatchesDir(), patchId, filePath), absPath);
|
|
4222
5276
|
updatedFiles.push(absPath);
|
|
4223
5277
|
} catch (err) {
|
|
4224
5278
|
errors[absPath] = {
|
|
@@ -4248,14 +5302,7 @@ class ValOpsFS extends ValOps {
|
|
|
4248
5302
|
baseSha: await this.getBaseSha(),
|
|
4249
5303
|
timestamp: new Date().toISOString()
|
|
4250
5304
|
};
|
|
4251
|
-
const
|
|
4252
|
-
if (!patchDir) {
|
|
4253
|
-
errors[`patchId:${patchId}`] = {
|
|
4254
|
-
message: "Failed to find PatchDir for PatchId " + patchId
|
|
4255
|
-
};
|
|
4256
|
-
continue;
|
|
4257
|
-
}
|
|
4258
|
-
const absPath = this.getPatchBaseFile(patchDir);
|
|
5305
|
+
const absPath = patchBaseFile(this.getPatchesDir(), patchId);
|
|
4259
5306
|
try {
|
|
4260
5307
|
this.host.writeUf8File(absPath, JSON.stringify(appliedAt));
|
|
4261
5308
|
} catch (err) {
|
|
@@ -4300,47 +5347,18 @@ class ValOpsFS extends ValOps {
|
|
|
4300
5347
|
}
|
|
4301
5348
|
return createMetadataFromBuffer(type, mimeType, buffer);
|
|
4302
5349
|
}
|
|
4303
|
-
async getParentPatchIdFromPatchId(patchId) {
|
|
4304
|
-
// This is not great. If needed we should find a better way
|
|
4305
|
-
const patches = await this.readPatches();
|
|
4306
|
-
if (patches.errors || patches.error) {
|
|
4307
|
-
console.error("Failed to read patches", JSON.stringify(patches));
|
|
4308
|
-
return fp.result.err("failed-to-read-patches");
|
|
4309
|
-
}
|
|
4310
|
-
const patch = patches.patches[patchId];
|
|
4311
|
-
if (!patch) {
|
|
4312
|
-
console.error("Could not find patch with patchId: ", patchId);
|
|
4313
|
-
return fp.result.err("patch-not-found");
|
|
4314
|
-
}
|
|
4315
|
-
return fp.result.ok(this.getParentPatchIdFromParentRef(patch.parentRef));
|
|
4316
|
-
}
|
|
4317
|
-
async getParentPatchIdFromPatchIdMap() {
|
|
4318
|
-
const patches = await this.readPatches();
|
|
4319
|
-
if (patches.errors || patches.error) {
|
|
4320
|
-
console.error("Failed to read patches", JSON.stringify(patches));
|
|
4321
|
-
return fp.result.err("failed-to-read-patches");
|
|
4322
|
-
}
|
|
4323
|
-
return fp.result.ok(Object.fromEntries(Object.entries(patches.patches).map(([patchId, value]) => [patchId, this.getParentPatchIdFromParentRef(value.parentRef)])));
|
|
4324
|
-
}
|
|
4325
5350
|
|
|
4326
5351
|
// #region fs file path helpers
|
|
4327
5352
|
getPatchesDir() {
|
|
4328
5353
|
return path__namespace["default"].join(this.rootDir, ValOpsFS.VAL_DIR, "patches");
|
|
4329
5354
|
}
|
|
4330
|
-
|
|
4331
|
-
|
|
4332
|
-
|
|
4333
|
-
|
|
4334
|
-
|
|
4335
|
-
|
|
4336
|
-
|
|
4337
|
-
return path__namespace["default"].join(this.getFullPatchDir(patchDir), "files", filePath, "metadata.json");
|
|
4338
|
-
}
|
|
4339
|
-
getPatchFilePath(patchDir) {
|
|
4340
|
-
return path__namespace["default"].join(this.getFullPatchDir(patchDir), "patch.json");
|
|
4341
|
-
}
|
|
4342
|
-
getPatchBaseFile(patchDir) {
|
|
4343
|
-
return path__namespace["default"].join(this.getFullPatchDir(patchDir), "base.json");
|
|
5355
|
+
|
|
5356
|
+
/**
|
|
5357
|
+
* Deliberately outside the patches directory: delete-all and reset rename that
|
|
5358
|
+
* whole directory, and a lock that moves away with it is not holding anything.
|
|
5359
|
+
*/
|
|
5360
|
+
getPatchLockFile() {
|
|
5361
|
+
return path__namespace["default"].join(this.rootDir, ValOpsFS.VAL_DIR, PATCH_LOCK_FILE_NAME);
|
|
4344
5362
|
}
|
|
4345
5363
|
}
|
|
4346
5364
|
class FSOpsHost {
|
|
@@ -4423,22 +5441,112 @@ class FSOpsHost {
|
|
|
4423
5441
|
fs__default["default"].copyFileSync(from, to);
|
|
4424
5442
|
}
|
|
4425
5443
|
}
|
|
4426
|
-
|
|
4427
|
-
|
|
4428
|
-
|
|
4429
|
-
|
|
4430
|
-
|
|
4431
|
-
|
|
4432
|
-
|
|
4433
|
-
|
|
4434
|
-
|
|
4435
|
-
|
|
4436
|
-
|
|
4437
|
-
|
|
4438
|
-
|
|
4439
|
-
|
|
4440
|
-
|
|
4441
|
-
|
|
5444
|
+
|
|
5445
|
+
/**
|
|
5446
|
+
* Which unpublished changes a failed save has to throw away to make progress.
|
|
5447
|
+
*
|
|
5448
|
+
* `/save` used to refuse the whole commit when any one patch could not be
|
|
5449
|
+
* applied: nothing was written, for any module, and the change stayed on disk to
|
|
5450
|
+
* fail again on the next attempt. With auto-save that is not a refusal, it is a
|
|
5451
|
+
* dead stop — the editor keeps typing, every save fails on the same patch, and
|
|
5452
|
+
* nothing is ever written again.
|
|
5453
|
+
*
|
|
5454
|
+
* So in `fs` mode the failing change is removed instead, and the person editing
|
|
5455
|
+
* is told. The rule for what goes with it:
|
|
5456
|
+
*
|
|
5457
|
+
* **From the first failure onward, within that module.** A patch is a set of ops
|
|
5458
|
+
* against the source as the patches before it left it. Once one has not been
|
|
5459
|
+
* applied, every later patch in that module was computed against a state that no
|
|
5460
|
+
* longer exists, so keeping them would be keeping changes whose meaning has
|
|
5461
|
+
* already been lost. That is `triedPatches` (the one that failed) plus
|
|
5462
|
+
* `skippedPatches` (the tail that was never attempted).
|
|
5463
|
+
*
|
|
5464
|
+
* **Other modules are untouched.** `prepare` walks each module's chain
|
|
5465
|
+
* independently, so a broken chain in one file says nothing about another.
|
|
5466
|
+
*
|
|
5467
|
+
* **And a module that failed as a WHOLE keeps everything.** When the `.val.ts`
|
|
5468
|
+
* cannot be read at all, `prepare` reports every one of its patches as skipped
|
|
5469
|
+
* and names none of them as unappliable — the file is what failed, not the
|
|
5470
|
+
* changes. Deleting them there would be destroying good edits because a file was
|
|
5471
|
+
* missing for a moment: rename a module, or switch branch, with unsaved changes
|
|
5472
|
+
* to it and the next auto-save would silently take them all. Skipped is only
|
|
5473
|
+
* collateral when there is something in that module for it to be collateral OF.
|
|
5474
|
+
*/
|
|
5475
|
+
|
|
5476
|
+
/**
|
|
5477
|
+
* The message for a patch that did not itself fail but sits behind one that did.
|
|
5478
|
+
*
|
|
5479
|
+
* Worth distinguishing: "yours was broken" and "something before yours was
|
|
5480
|
+
* broken" are different things to be told, and only the first is a bug in the
|
|
5481
|
+
* change itself.
|
|
5482
|
+
*/
|
|
5483
|
+
function collateralMessage(moduleFilePath) {
|
|
5484
|
+
return `An earlier change to ${moduleFilePath} could not be applied, so this one no longer fits the content it was written against.`;
|
|
5485
|
+
}
|
|
5486
|
+
function computePatchesToDrop(prepared) {
|
|
5487
|
+
const dropped = [];
|
|
5488
|
+
const seen = new Set();
|
|
5489
|
+
const add = (moduleFilePath, patchId) => {
|
|
5490
|
+
var _prepared$unappliable;
|
|
5491
|
+
if (seen.has(patchId)) {
|
|
5492
|
+
return;
|
|
5493
|
+
}
|
|
5494
|
+
seen.add(patchId);
|
|
5495
|
+
dropped.push({
|
|
5496
|
+
patchId,
|
|
5497
|
+
moduleFilePath,
|
|
5498
|
+
message: ((_prepared$unappliable = prepared.unappliablePatches[patchId]) === null || _prepared$unappliable === void 0 ? void 0 : _prepared$unappliable.message) ?? collateralMessage(moduleFilePath)
|
|
5499
|
+
});
|
|
5500
|
+
};
|
|
5501
|
+
|
|
5502
|
+
// Tried before skipped, so the change that actually failed is named first —
|
|
5503
|
+
// it is the one worth reading.
|
|
5504
|
+
for (const [moduleFilePath, patchIds] of Object.entries(prepared.triedPatches)) {
|
|
5505
|
+
for (const patchId of patchIds) {
|
|
5506
|
+
add(moduleFilePath, patchId);
|
|
5507
|
+
}
|
|
5508
|
+
}
|
|
5509
|
+
/*
|
|
5510
|
+
* Which modules actually had a change fail, as opposed to being unreadable.
|
|
5511
|
+
*
|
|
5512
|
+
* `triedPatches` holds only failures, and `unappliablePatches` names the
|
|
5513
|
+
* failure itself, so between them they are the whole answer. A module in
|
|
5514
|
+
* neither has nothing to blame its skipped tail on.
|
|
5515
|
+
*/
|
|
5516
|
+
const failedModules = new Set();
|
|
5517
|
+
for (const [moduleFilePath, patchIds] of Object.entries(prepared.triedPatches)) {
|
|
5518
|
+
if (patchIds.length > 0) {
|
|
5519
|
+
failedModules.add(moduleFilePath);
|
|
5520
|
+
}
|
|
5521
|
+
}
|
|
5522
|
+
for (const failure of Object.values(prepared.unappliablePatches)) {
|
|
5523
|
+
failedModules.add(failure.moduleFilePath);
|
|
5524
|
+
}
|
|
5525
|
+
for (const [moduleFilePath, patchIds] of Object.entries(prepared.skippedPatches)) {
|
|
5526
|
+
if (!failedModules.has(moduleFilePath)) {
|
|
5527
|
+
// The module failed as a whole. Its changes are fine and are kept, so the
|
|
5528
|
+
// save refuses instead — which is what it did before auto-save, and is
|
|
5529
|
+
// right here: this one resolves itself when the file comes back.
|
|
5530
|
+
continue;
|
|
5531
|
+
}
|
|
5532
|
+
for (const patchId of patchIds) {
|
|
5533
|
+
add(moduleFilePath, patchId);
|
|
5534
|
+
}
|
|
5535
|
+
}
|
|
5536
|
+
|
|
5537
|
+
/*
|
|
5538
|
+
* A patch reported unappliable that neither list holds.
|
|
5539
|
+
*
|
|
5540
|
+
* Belt and braces: the two lists are how `prepare` reports the shape of the
|
|
5541
|
+
* failure, and this is how it reports the failure itself. If they ever
|
|
5542
|
+
* disagree, leaving a patch behind means the next save fails on it again —
|
|
5543
|
+
* the exact dead stop this function exists to prevent.
|
|
5544
|
+
*/
|
|
5545
|
+
for (const [patchId, failure] of Object.entries(prepared.unappliablePatches)) {
|
|
5546
|
+
add(failure.moduleFilePath, patchId);
|
|
5547
|
+
}
|
|
5548
|
+
return dropped;
|
|
5549
|
+
}
|
|
4442
5550
|
|
|
4443
5551
|
const textEncoder$2 = new TextEncoder();
|
|
4444
5552
|
const PatchId = z.z.string().refine(s => !!s); // TODO: validate
|
|
@@ -6464,7 +7572,7 @@ const ValServer = (valModules, options, callbacks) => {
|
|
|
6464
7572
|
status: 409,
|
|
6465
7573
|
json: {
|
|
6466
7574
|
type: "patch-head-conflict",
|
|
6467
|
-
message: "
|
|
7575
|
+
message: "The change was written against a different starting point than the server has."
|
|
6468
7576
|
}
|
|
6469
7577
|
};
|
|
6470
7578
|
} else {
|
|
@@ -7137,11 +8245,65 @@ const ValServer = (valModules, options, callbacks) => {
|
|
|
7137
8245
|
patchIds,
|
|
7138
8246
|
excludePatchOps: false
|
|
7139
8247
|
});
|
|
8248
|
+
/*
|
|
8249
|
+
* Exactly the patches this request consumes, and the ONLY ones it may
|
|
8250
|
+
* delete afterwards.
|
|
8251
|
+
*
|
|
8252
|
+
* Taken from what was fetched rather than from what was asked for, and
|
|
8253
|
+
* held across everything below, because the store keeps accepting writes
|
|
8254
|
+
* while this runs. A change typed during a save is not in this list, is
|
|
8255
|
+
* not applied, and must still be here when the save finishes — that is
|
|
8256
|
+
* the whole contract, and `deleteAllPatches` broke it by removing the
|
|
8257
|
+
* store wholesale.
|
|
8258
|
+
*/
|
|
8259
|
+
const consumed = patches.patches.map(patch => patch.patchId);
|
|
7140
8260
|
const analysis = serverOps.analyzePatches(patches.patches, patches.commits, commit);
|
|
7141
|
-
|
|
8261
|
+
let preparedCommit = await serverOps.prepare({
|
|
7142
8262
|
...analysis,
|
|
7143
8263
|
...patches
|
|
7144
8264
|
});
|
|
8265
|
+
/*
|
|
8266
|
+
* In `fs` mode a change that cannot be applied is removed rather than
|
|
8267
|
+
* blocking the save.
|
|
8268
|
+
*
|
|
8269
|
+
* Refusing the whole commit is right for a git commit — it is one
|
|
8270
|
+
* atomic thing, and the content api owns its own patches. It is wrong
|
|
8271
|
+
* for auto-save: the editor keeps typing, every save fails on the same
|
|
8272
|
+
* patch, and nothing is ever written again. So the failing change and
|
|
8273
|
+
* the rest of its module's chain go, the rest is written, and the person
|
|
8274
|
+
* editing is told what was thrown away.
|
|
8275
|
+
*/
|
|
8276
|
+
let removed = [];
|
|
8277
|
+
if (preparedCommit.hasErrors && serverOps instanceof ValOpsFS) {
|
|
8278
|
+
removed = computePatchesToDrop(preparedCommit);
|
|
8279
|
+
/*
|
|
8280
|
+
* Nothing to drop means nothing a second `prepare` could do better.
|
|
8281
|
+
*
|
|
8282
|
+
* `hasErrors` covers failures no patch is to blame for — a binary file
|
|
8283
|
+
* that could not be written, a module that could not be formatted, a
|
|
8284
|
+
* `.val.ts` that could not be read at all. Re-preparing every module
|
|
8285
|
+
* with an unchanged patch set would fail identically, and logging
|
|
8286
|
+
* "removed unpublished changes []" on the way would say something
|
|
8287
|
+
* untrue. It falls through to the 400 instead, which is the honest
|
|
8288
|
+
* answer: this save did not happen and the changes are still here.
|
|
8289
|
+
*/
|
|
8290
|
+
if (removed.length > 0) {
|
|
8291
|
+
const doomed = new Set(removed.map(entry => entry.patchId));
|
|
8292
|
+
const survivors = patches.patches.filter(patch => !doomed.has(patch.patchId));
|
|
8293
|
+
const survivingPatches = {
|
|
8294
|
+
...patches,
|
|
8295
|
+
patches: survivors
|
|
8296
|
+
};
|
|
8297
|
+
// One retry is enough: what is left of each module is the prefix
|
|
8298
|
+
// that already applied, and `prepare` walks modules independently,
|
|
8299
|
+
// so nothing new can fail.
|
|
8300
|
+
preparedCommit = await serverOps.prepare({
|
|
8301
|
+
...serverOps.analyzePatches(survivors, patches.commits, commit),
|
|
8302
|
+
...survivingPatches
|
|
8303
|
+
});
|
|
8304
|
+
console.error("Val: removed unpublished changes that could not be applied", removed);
|
|
8305
|
+
}
|
|
8306
|
+
}
|
|
7145
8307
|
if (preparedCommit.hasErrors) {
|
|
7146
8308
|
console.error("Failed to create commit", JSON.stringify({
|
|
7147
8309
|
sourceFilePatchErrors: preparedCommit.sourceFilePatchErrors,
|
|
@@ -7194,13 +8356,31 @@ const ValServer = (valModules, options, callbacks) => {
|
|
|
7194
8356
|
}
|
|
7195
8357
|
};
|
|
7196
8358
|
}
|
|
7197
|
-
|
|
7198
|
-
|
|
8359
|
+
/*
|
|
8360
|
+
* Only what this request consumed.
|
|
8361
|
+
*
|
|
8362
|
+
* This used to be `deleteAllPatches()`, which renamed the whole store
|
|
8363
|
+
* aside and deleted it. Anything typed while the save was in flight
|
|
8364
|
+
* went with it — never applied, and indistinguishable on the client
|
|
8365
|
+
* from having been published, because both look like "the patch is
|
|
8366
|
+
* gone and the base moved". A visible edit that exists nowhere on
|
|
8367
|
+
* disk, with nothing reporting a problem.
|
|
8368
|
+
*/
|
|
8369
|
+
const deleteRes = await serverOps.deletePatches(consumed);
|
|
8370
|
+
if ("error" in deleteRes && deleteRes.error !== undefined) {
|
|
7199
8371
|
console.error(`Val got an error while cleaning up patches after publish: ${deleteRes.error.message}`);
|
|
8372
|
+
} else if ("errors" in deleteRes && deleteRes.errors !== undefined) {
|
|
8373
|
+
// Per patch, and not fatal: the files are written, so the save
|
|
8374
|
+
// succeeded. A patch left behind is announced again on the next
|
|
8375
|
+
// stat, applies to a base that already contains it, and is removed
|
|
8376
|
+
// then.
|
|
8377
|
+
console.error("Val: could not clean up some patches after publish", deleteRes.errors);
|
|
7200
8378
|
}
|
|
7201
8379
|
return {
|
|
7202
8380
|
status: 200,
|
|
7203
|
-
json:
|
|
8381
|
+
json: removed.length > 0 ? {
|
|
8382
|
+
removed
|
|
8383
|
+
} : {}
|
|
7204
8384
|
};
|
|
7205
8385
|
} else if (serverOps instanceof ValOpsHttp) {
|
|
7206
8386
|
if (auth.error === undefined && auth.id) {
|
|
@@ -10819,6 +11999,7 @@ exports.createValApiRouter = createValApiRouter;
|
|
|
10819
11999
|
exports.createValServer = createValServer;
|
|
10820
12000
|
exports.currentFixHandlers = currentFixHandlers;
|
|
10821
12001
|
exports.decodeJwt = decodeJwt;
|
|
12002
|
+
exports.describePatchStoreProblems = describePatchStoreProblems;
|
|
10822
12003
|
exports.downloadFileFromRemote = downloadFileFromRemote;
|
|
10823
12004
|
exports.encodeJwt = encodeJwt;
|
|
10824
12005
|
exports.evalValConfigFile = evalValConfigFile;
|
|
@@ -10852,6 +12033,7 @@ exports.parsePersonalAccessTokenFile = parsePersonalAccessTokenFile;
|
|
|
10852
12033
|
exports.patchSourceFile = patchSourceFile;
|
|
10853
12034
|
exports.persistPersonalAccessToken = persistPersonalAccessToken;
|
|
10854
12035
|
exports.readCapturedReport = readCapturedReport;
|
|
12036
|
+
exports.readPatchStore = readPatchStore;
|
|
10855
12037
|
exports.replaySnapshot = replaySnapshot;
|
|
10856
12038
|
exports.safeReadGit = safeReadGit;
|
|
10857
12039
|
exports.startValLogin = startValLogin;
|