@unotest/core 0.25.0 → 0.26.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +124 -0
- package/README.md +7 -0
- package/dist/index.d.ts +224 -2
- package/dist/index.js +985 -4
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -298,9 +298,9 @@ async function startDebugCommandsWatcher(deps) {
|
|
|
298
298
|
if (stopped || !handle) return;
|
|
299
299
|
inflight = inflight.then(async () => {
|
|
300
300
|
if (stopped || !handle) return;
|
|
301
|
-
const
|
|
302
|
-
if (
|
|
303
|
-
const len =
|
|
301
|
+
const stat2 = await handle.stat();
|
|
302
|
+
if (stat2.size <= offset) return;
|
|
303
|
+
const len = stat2.size - offset;
|
|
304
304
|
const buf = Buffer.alloc(len);
|
|
305
305
|
const { bytesRead } = await handle.read(buf, 0, len, offset);
|
|
306
306
|
offset += bytesRead;
|
|
@@ -586,26 +586,1007 @@ var UnotestError = class extends Error {
|
|
|
586
586
|
}
|
|
587
587
|
}
|
|
588
588
|
};
|
|
589
|
+
|
|
590
|
+
// src/index.ts
|
|
591
|
+
import { isRecord } from "@unotest/protocol";
|
|
592
|
+
|
|
593
|
+
// src/run-queue/fs-run-queue.ts
|
|
594
|
+
import { mkdir as mkdir6 } from "fs/promises";
|
|
595
|
+
import { join as join10 } from "path";
|
|
596
|
+
import {
|
|
597
|
+
QUEUE_GLOBAL_DIR_ENV,
|
|
598
|
+
QUEUE_GLOBAL_SLOTS_ENV,
|
|
599
|
+
QUEUE_LEASE_ENV,
|
|
600
|
+
QUEUE_MUTEX_FILE,
|
|
601
|
+
QUEUE_RUNNING_DIR,
|
|
602
|
+
QUEUE_SCHEMA_VERSION,
|
|
603
|
+
QUEUE_SLOTS_DIR,
|
|
604
|
+
QUEUE_TICKETS_DIR,
|
|
605
|
+
compareQueueTicketNames,
|
|
606
|
+
makeQueueTicketName,
|
|
607
|
+
parseQueueTicket,
|
|
608
|
+
parseQueueTicketName,
|
|
609
|
+
queuePriorityRank
|
|
610
|
+
} from "@unotest/protocol";
|
|
611
|
+
|
|
612
|
+
// src/run-queue/errors.ts
|
|
613
|
+
var QueueTicketLostError = class extends UnotestError {
|
|
614
|
+
static {
|
|
615
|
+
__name(this, "QueueTicketLostError");
|
|
616
|
+
}
|
|
617
|
+
constructor(ticketName) {
|
|
618
|
+
super(
|
|
619
|
+
`queue ticket ${ticketName} is gone \u2014 it was cancelled or reaped while waiting for a slot`,
|
|
620
|
+
{ ticketName }
|
|
621
|
+
);
|
|
622
|
+
}
|
|
623
|
+
};
|
|
624
|
+
var QueueWaitAbortedError = class extends UnotestError {
|
|
625
|
+
static {
|
|
626
|
+
__name(this, "QueueWaitAbortedError");
|
|
627
|
+
}
|
|
628
|
+
constructor(ticketName) {
|
|
629
|
+
super(`stopped waiting for a queue slot (ticket ${ticketName} withdrawn)`, {
|
|
630
|
+
ticketName
|
|
631
|
+
});
|
|
632
|
+
}
|
|
633
|
+
};
|
|
634
|
+
var QueueUnavailableError = class extends UnotestError {
|
|
635
|
+
static {
|
|
636
|
+
__name(this, "QueueUnavailableError");
|
|
637
|
+
}
|
|
638
|
+
constructor(root, cause) {
|
|
639
|
+
super(
|
|
640
|
+
`run queue at ${root} is unusable: ${cause}. Fix the directory permissions, or set UNOTEST_NO_QUEUE=1 to run without queueing.`,
|
|
641
|
+
{ root, cause }
|
|
642
|
+
);
|
|
643
|
+
}
|
|
644
|
+
};
|
|
645
|
+
|
|
646
|
+
// src/run-queue/fs-primitives.ts
|
|
647
|
+
import { unlinkSync } from "fs";
|
|
648
|
+
import {
|
|
649
|
+
open as open2,
|
|
650
|
+
readFile,
|
|
651
|
+
readdir,
|
|
652
|
+
rename as rename4,
|
|
653
|
+
stat,
|
|
654
|
+
unlink,
|
|
655
|
+
utimes as utimes2,
|
|
656
|
+
writeFile as writeFile6
|
|
657
|
+
} from "fs/promises";
|
|
658
|
+
import { dirname as dirname3, join as join8 } from "path";
|
|
659
|
+
async function createExclusive(path, contents) {
|
|
660
|
+
let handle;
|
|
661
|
+
try {
|
|
662
|
+
handle = await open2(path, "wx");
|
|
663
|
+
} catch (e) {
|
|
664
|
+
if (isErrno(e, "EEXIST")) return false;
|
|
665
|
+
throw e;
|
|
666
|
+
}
|
|
667
|
+
try {
|
|
668
|
+
await handle.writeFile(contents, "utf8");
|
|
669
|
+
} finally {
|
|
670
|
+
await handle.close();
|
|
671
|
+
}
|
|
672
|
+
return true;
|
|
673
|
+
}
|
|
674
|
+
__name(createExclusive, "createExclusive");
|
|
675
|
+
async function writeFileAtomic(path, contents, tmpToken) {
|
|
676
|
+
const tmp = join8(dirname3(path), `.tmp-${tmpToken}`);
|
|
677
|
+
await writeFile6(tmp, contents, "utf8");
|
|
678
|
+
await rename4(tmp, path);
|
|
679
|
+
}
|
|
680
|
+
__name(writeFileAtomic, "writeFileAtomic");
|
|
681
|
+
async function touchFile(path, now) {
|
|
682
|
+
const seconds = now / 1e3;
|
|
683
|
+
try {
|
|
684
|
+
await utimes2(path, seconds, seconds);
|
|
685
|
+
} catch (e) {
|
|
686
|
+
if (!isErrno(e, "ENOENT")) throw e;
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
__name(touchFile, "touchFile");
|
|
690
|
+
async function fileAgeMs(path, now) {
|
|
691
|
+
try {
|
|
692
|
+
const s = await stat(path);
|
|
693
|
+
return now - s.mtimeMs;
|
|
694
|
+
} catch (e) {
|
|
695
|
+
if (isErrno(e, "ENOENT")) return null;
|
|
696
|
+
throw e;
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
__name(fileAgeMs, "fileAgeMs");
|
|
700
|
+
async function removeFile(path) {
|
|
701
|
+
try {
|
|
702
|
+
await unlink(path);
|
|
703
|
+
return true;
|
|
704
|
+
} catch (e) {
|
|
705
|
+
if (isErrno(e, "ENOENT")) return false;
|
|
706
|
+
throw e;
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
__name(removeFile, "removeFile");
|
|
710
|
+
async function readTextFile(path) {
|
|
711
|
+
try {
|
|
712
|
+
return await readFile(path, "utf8");
|
|
713
|
+
} catch (e) {
|
|
714
|
+
if (isErrno(e, "ENOENT")) return null;
|
|
715
|
+
throw e;
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
__name(readTextFile, "readTextFile");
|
|
719
|
+
function removeFileSync(path) {
|
|
720
|
+
try {
|
|
721
|
+
unlinkSync(path);
|
|
722
|
+
} catch (e) {
|
|
723
|
+
if (!isErrno(e, "ENOENT")) throw e;
|
|
724
|
+
}
|
|
725
|
+
}
|
|
726
|
+
__name(removeFileSync, "removeFileSync");
|
|
727
|
+
async function listDir(dir) {
|
|
728
|
+
try {
|
|
729
|
+
return await readdir(dir);
|
|
730
|
+
} catch (e) {
|
|
731
|
+
if (isErrno(e, "ENOENT")) return [];
|
|
732
|
+
throw e;
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
__name(listDir, "listDir");
|
|
736
|
+
async function quarantineAndRemove(path, token) {
|
|
737
|
+
const dir = dirname3(path);
|
|
738
|
+
const quarantined = join8(dir, `stale-${token}`);
|
|
739
|
+
try {
|
|
740
|
+
await rename4(path, quarantined);
|
|
741
|
+
} catch (e) {
|
|
742
|
+
if (isErrno(e, "ENOENT")) return false;
|
|
743
|
+
throw e;
|
|
744
|
+
}
|
|
745
|
+
await removeFile(quarantined);
|
|
746
|
+
return true;
|
|
747
|
+
}
|
|
748
|
+
__name(quarantineAndRemove, "quarantineAndRemove");
|
|
749
|
+
async function moveFile(from, to) {
|
|
750
|
+
try {
|
|
751
|
+
await rename4(from, to);
|
|
752
|
+
return true;
|
|
753
|
+
} catch (e) {
|
|
754
|
+
if (isErrno(e, "ENOENT")) return false;
|
|
755
|
+
throw e;
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
__name(moveFile, "moveFile");
|
|
759
|
+
function isErrno(e, code) {
|
|
760
|
+
return typeof e === "object" && e !== null && e.code === code;
|
|
761
|
+
}
|
|
762
|
+
__name(isErrno, "isErrno");
|
|
763
|
+
|
|
764
|
+
// src/run-queue/slot-pool.ts
|
|
765
|
+
import { mkdir as mkdir5 } from "fs/promises";
|
|
766
|
+
import { join as join9 } from "path";
|
|
767
|
+
var SLOT_PREFIX = "slot-";
|
|
768
|
+
var SlotPool = class {
|
|
769
|
+
constructor(opts) {
|
|
770
|
+
this.opts = opts;
|
|
771
|
+
}
|
|
772
|
+
opts;
|
|
773
|
+
static {
|
|
774
|
+
__name(this, "SlotPool");
|
|
775
|
+
}
|
|
776
|
+
get total() {
|
|
777
|
+
return this.opts.total;
|
|
778
|
+
}
|
|
779
|
+
/** Take `count` leases, all or nothing. Returns the lease names, or
|
|
780
|
+
* null when the pool cannot satisfy the request right now — a normal
|
|
781
|
+
* outcome the caller answers by waiting. */
|
|
782
|
+
async claim(count, payload) {
|
|
783
|
+
if (count > this.opts.total) return null;
|
|
784
|
+
await mkdir5(this.opts.dir, { recursive: true });
|
|
785
|
+
if (!this.opts.mutex) return this.claimUnguarded(count, payload);
|
|
786
|
+
const claimed = await this.withMutex(
|
|
787
|
+
(keepAlive) => this.claimUnguarded(count, payload, keepAlive)
|
|
788
|
+
);
|
|
789
|
+
return claimed ?? null;
|
|
790
|
+
}
|
|
791
|
+
async release(names) {
|
|
792
|
+
for (const name of names) await removeFile(join9(this.opts.dir, name));
|
|
793
|
+
}
|
|
794
|
+
/** For a holder that is exiting inside a signal handler. */
|
|
795
|
+
releaseSync(names) {
|
|
796
|
+
for (const name of names) removeFileSync(join9(this.opts.dir, name));
|
|
797
|
+
}
|
|
798
|
+
/** Keep the leases alive. A holder that stops touching is treated as
|
|
799
|
+
* dead after `staleAfterMs` and its leases are reaped. */
|
|
800
|
+
async touch(names) {
|
|
801
|
+
const now = this.opts.now();
|
|
802
|
+
for (const name of names) await touchFile(join9(this.opts.dir, name), now);
|
|
803
|
+
}
|
|
804
|
+
/** Reap dead leases, then report what is left. */
|
|
805
|
+
sweep() {
|
|
806
|
+
return this.scan(true);
|
|
807
|
+
}
|
|
808
|
+
/** Report without reaping — for a snapshot the UI renders. */
|
|
809
|
+
inspect() {
|
|
810
|
+
return this.scan(false);
|
|
811
|
+
}
|
|
812
|
+
async claimUnguarded(count, payload, keepAlive) {
|
|
813
|
+
const taken = [];
|
|
814
|
+
const body = JSON.stringify(payload);
|
|
815
|
+
for (let n = 0; n < this.opts.total && taken.length < count; n++) {
|
|
816
|
+
await keepAlive?.();
|
|
817
|
+
const name = `${SLOT_PREFIX}${n}`;
|
|
818
|
+
if (await createExclusive(join9(this.opts.dir, name), body)) {
|
|
819
|
+
taken.push(name);
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
if (taken.length < count) {
|
|
823
|
+
await this.release(taken);
|
|
824
|
+
return null;
|
|
825
|
+
}
|
|
826
|
+
return taken;
|
|
827
|
+
}
|
|
828
|
+
async scan(reap) {
|
|
829
|
+
const runIds = /* @__PURE__ */ new Set();
|
|
830
|
+
let used = 0;
|
|
831
|
+
for (const name of await listDir(this.opts.dir)) {
|
|
832
|
+
if (!name.startsWith(SLOT_PREFIX)) continue;
|
|
833
|
+
const path = join9(this.opts.dir, name);
|
|
834
|
+
const age = await fileAgeMs(path, this.opts.now());
|
|
835
|
+
if (age === null) continue;
|
|
836
|
+
if (age > this.opts.staleAfterMs) {
|
|
837
|
+
if (reap) {
|
|
838
|
+
await quarantineAndRemove(path, `${name}-${this.opts.token()}`);
|
|
839
|
+
continue;
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
used++;
|
|
843
|
+
const payload = parseSlotPayload(await readTextFile(path));
|
|
844
|
+
if (payload) runIds.add(payload.runId);
|
|
845
|
+
}
|
|
846
|
+
return { total: this.opts.total, used, runIds };
|
|
847
|
+
}
|
|
848
|
+
/** Run `body` while holding the pool's mutex. Returns undefined when
|
|
849
|
+
* the mutex could not be taken in time — the caller retries on its
|
|
850
|
+
* own cadence. `body` receives a keep-alive callback it should invoke
|
|
851
|
+
* between filesystem operations so a slow critical section is not
|
|
852
|
+
* mistaken for a dead holder. */
|
|
853
|
+
async withMutex(body) {
|
|
854
|
+
const mutex = this.opts.mutex;
|
|
855
|
+
if (!mutex) return body(async () => {
|
|
856
|
+
});
|
|
857
|
+
const token = this.opts.token();
|
|
858
|
+
for (let attempt = 0; attempt < mutex.attempts; attempt++) {
|
|
859
|
+
if (await createExclusive(mutex.path, token)) {
|
|
860
|
+
try {
|
|
861
|
+
return await body(() => touchFile(mutex.path, this.opts.now()));
|
|
862
|
+
} finally {
|
|
863
|
+
if (await readTextFile(mutex.path) === token) {
|
|
864
|
+
await removeFile(mutex.path);
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
}
|
|
868
|
+
const age = await fileAgeMs(mutex.path, this.opts.now());
|
|
869
|
+
if (age !== null && age > mutex.staleAfterMs) {
|
|
870
|
+
await quarantineAndRemove(mutex.path, `mutex-${this.opts.token()}`);
|
|
871
|
+
continue;
|
|
872
|
+
}
|
|
873
|
+
await this.opts.sleep(mutex.waitMs);
|
|
874
|
+
}
|
|
875
|
+
return void 0;
|
|
876
|
+
}
|
|
877
|
+
};
|
|
878
|
+
function parseSlotPayload(text) {
|
|
879
|
+
if (text === null) return null;
|
|
880
|
+
let value;
|
|
881
|
+
try {
|
|
882
|
+
value = JSON.parse(text);
|
|
883
|
+
} catch {
|
|
884
|
+
return null;
|
|
885
|
+
}
|
|
886
|
+
if (typeof value !== "object" || value === null) return null;
|
|
887
|
+
const { runId, ticket, pid, host, at } = value;
|
|
888
|
+
if (typeof runId !== "string" || typeof ticket !== "string") return null;
|
|
889
|
+
if (typeof pid !== "number" || typeof host !== "string") return null;
|
|
890
|
+
return { runId, ticket, pid, host, at: typeof at === "number" ? at : 0 };
|
|
891
|
+
}
|
|
892
|
+
__name(parseSlotPayload, "parseSlotPayload");
|
|
893
|
+
|
|
894
|
+
// src/run-queue/fs-run-queue.ts
|
|
895
|
+
var DEFAULT_STALE_AFTER_MS = 6e4;
|
|
896
|
+
var DEFAULT_TOUCH_INTERVAL_MS = 1e4;
|
|
897
|
+
var DEFAULT_POLL_INTERVAL_MS = 500;
|
|
898
|
+
var MUTEX_STALE_MS = 1e4;
|
|
899
|
+
var MUTEX_ATTEMPTS = 20;
|
|
900
|
+
var MUTEX_WAIT_MS = 25;
|
|
901
|
+
var MAINTENANCE_PREFIX = `${queuePriorityRank("maintenance")}-`;
|
|
902
|
+
function globalPoolFromEnv(env = process.env) {
|
|
903
|
+
const dir = env[QUEUE_GLOBAL_DIR_ENV]?.trim();
|
|
904
|
+
if (!dir) return null;
|
|
905
|
+
const raw = env[QUEUE_GLOBAL_SLOTS_ENV]?.trim();
|
|
906
|
+
const slots = raw ? Number(raw) : NaN;
|
|
907
|
+
return {
|
|
908
|
+
dir,
|
|
909
|
+
// A pool directory without a usable size is a configuration slip, not
|
|
910
|
+
// a reason to refuse to run: fall back to one, which is the safest
|
|
911
|
+
// possible host budget.
|
|
912
|
+
slots: Number.isInteger(slots) && slots > 0 ? slots : 1
|
|
913
|
+
};
|
|
914
|
+
}
|
|
915
|
+
__name(globalPoolFromEnv, "globalPoolFromEnv");
|
|
916
|
+
var FsRunQueue = class {
|
|
917
|
+
static {
|
|
918
|
+
__name(this, "FsRunQueue");
|
|
919
|
+
}
|
|
920
|
+
root;
|
|
921
|
+
ticketsDir;
|
|
922
|
+
runningDir;
|
|
923
|
+
staleAfterMs;
|
|
924
|
+
touchIntervalMs;
|
|
925
|
+
pollIntervalMs;
|
|
926
|
+
now;
|
|
927
|
+
random;
|
|
928
|
+
sleep;
|
|
929
|
+
envPool;
|
|
930
|
+
globalPool;
|
|
931
|
+
constructor(opts) {
|
|
932
|
+
this.root = opts.root;
|
|
933
|
+
this.ticketsDir = join10(opts.root, QUEUE_TICKETS_DIR);
|
|
934
|
+
this.runningDir = join10(opts.root, QUEUE_RUNNING_DIR);
|
|
935
|
+
this.staleAfterMs = opts.staleAfterMs ?? DEFAULT_STALE_AFTER_MS;
|
|
936
|
+
this.touchIntervalMs = opts.touchIntervalMs ?? DEFAULT_TOUCH_INTERVAL_MS;
|
|
937
|
+
this.pollIntervalMs = opts.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
|
|
938
|
+
this.now = opts.now ?? (() => Date.now());
|
|
939
|
+
this.random = opts.random ?? randomToken;
|
|
940
|
+
this.sleep = opts.sleep ?? ((ms) => new Promise((r) => {
|
|
941
|
+
setTimeout(r, ms);
|
|
942
|
+
}));
|
|
943
|
+
const shared = {
|
|
944
|
+
staleAfterMs: this.staleAfterMs,
|
|
945
|
+
now: this.now,
|
|
946
|
+
token: this.random,
|
|
947
|
+
sleep: this.sleep
|
|
948
|
+
};
|
|
949
|
+
this.envPool = new SlotPool({
|
|
950
|
+
dir: join10(opts.root, QUEUE_SLOTS_DIR),
|
|
951
|
+
total: Math.max(1, Math.floor(opts.slots)),
|
|
952
|
+
...shared
|
|
953
|
+
});
|
|
954
|
+
this.globalPool = opts.global ? new SlotPool({
|
|
955
|
+
dir: join10(opts.global.dir, QUEUE_SLOTS_DIR),
|
|
956
|
+
total: Math.max(1, Math.floor(opts.global.slots)),
|
|
957
|
+
mutex: {
|
|
958
|
+
path: join10(opts.global.dir, QUEUE_MUTEX_FILE),
|
|
959
|
+
staleAfterMs: MUTEX_STALE_MS,
|
|
960
|
+
attempts: MUTEX_ATTEMPTS,
|
|
961
|
+
waitMs: MUTEX_WAIT_MS
|
|
962
|
+
},
|
|
963
|
+
...shared
|
|
964
|
+
}) : null;
|
|
965
|
+
}
|
|
966
|
+
async enqueue(input) {
|
|
967
|
+
const createdAt = this.now();
|
|
968
|
+
const priority = input.kind === "maintenance" ? "maintenance" : input.priority;
|
|
969
|
+
const ticket = {
|
|
970
|
+
schemaVersion: QUEUE_SCHEMA_VERSION,
|
|
971
|
+
runId: input.runId,
|
|
972
|
+
kind: input.kind,
|
|
973
|
+
ref: input.ref,
|
|
974
|
+
priority,
|
|
975
|
+
// A weight larger than the pool would never be satisfiable; the
|
|
976
|
+
// run still has to happen, so it takes the whole pool instead.
|
|
977
|
+
weight: this.clampWeight(input.weight ?? 1),
|
|
978
|
+
producer: input.producer,
|
|
979
|
+
createdAt,
|
|
980
|
+
...input.env ? { env: input.env } : {},
|
|
981
|
+
// Recorded at ENQUEUE time on purpose: an environment switched while
|
|
982
|
+
// this ticket waits must not change what it runs.
|
|
983
|
+
...input.source ? { source: input.source } : {},
|
|
984
|
+
...input.ci ? { ci: input.ci } : {}
|
|
985
|
+
};
|
|
986
|
+
const name = makeQueueTicketName(priority, createdAt, this.random());
|
|
987
|
+
try {
|
|
988
|
+
await mkdir6(this.ticketsDir, { recursive: true });
|
|
989
|
+
await mkdir6(this.runningDir, { recursive: true });
|
|
990
|
+
await writeFileAtomic(
|
|
991
|
+
join10(this.ticketsDir, name),
|
|
992
|
+
JSON.stringify(ticket),
|
|
993
|
+
`${name}-${this.random()}`
|
|
994
|
+
);
|
|
995
|
+
} catch (e) {
|
|
996
|
+
throw new QueueUnavailableError(
|
|
997
|
+
this.root,
|
|
998
|
+
e instanceof Error ? e.message : String(e)
|
|
999
|
+
);
|
|
1000
|
+
}
|
|
1001
|
+
return new FsTicketHandle(this, name, ticket);
|
|
1002
|
+
}
|
|
1003
|
+
async snapshot() {
|
|
1004
|
+
const waiting = await this.readEntries(this.ticketsDir);
|
|
1005
|
+
const running = await this.readEntries(this.runningDir);
|
|
1006
|
+
const slots = await this.envPool.inspect();
|
|
1007
|
+
const global = this.globalPool ? await this.globalPool.inspect() : null;
|
|
1008
|
+
return {
|
|
1009
|
+
waiting,
|
|
1010
|
+
running,
|
|
1011
|
+
slots: { total: slots.total, used: slots.used },
|
|
1012
|
+
...global ? { global: { total: global.total, used: global.used } } : {}
|
|
1013
|
+
};
|
|
1014
|
+
}
|
|
1015
|
+
async cancel(ticketName) {
|
|
1016
|
+
if (parseQueueTicketName(ticketName) === null) return false;
|
|
1017
|
+
return removeFile(join10(this.ticketsDir, ticketName));
|
|
1018
|
+
}
|
|
1019
|
+
/** Reap what died, then drop `running/` entries with no lease behind
|
|
1020
|
+
* them. Every producer calls this on its way past — a queue with any
|
|
1021
|
+
* traffic needs no janitor process. */
|
|
1022
|
+
async sweep() {
|
|
1023
|
+
const now = this.now();
|
|
1024
|
+
for (const name of await listDir(this.ticketsDir)) {
|
|
1025
|
+
const path = join10(this.ticketsDir, name);
|
|
1026
|
+
const age = await fileAgeMs(path, now);
|
|
1027
|
+
if (age === null || age <= this.staleAfterMs) continue;
|
|
1028
|
+
await quarantineAndRemove(path, `${name}-${this.random()}`);
|
|
1029
|
+
}
|
|
1030
|
+
const live = await this.envPool.sweep();
|
|
1031
|
+
if (this.globalPool) await this.globalPool.sweep();
|
|
1032
|
+
for (const name of await listDir(this.runningDir)) {
|
|
1033
|
+
const path = join10(this.runningDir, name);
|
|
1034
|
+
const age = await fileAgeMs(path, now);
|
|
1035
|
+
if (age === null || age <= this.staleAfterMs) continue;
|
|
1036
|
+
const ticket = parseQueueTicket(await readJson(path));
|
|
1037
|
+
if (ticket && live.runIds.has(ticket.runId)) continue;
|
|
1038
|
+
await quarantineAndRemove(path, `${name}-${this.random()}`);
|
|
1039
|
+
}
|
|
1040
|
+
}
|
|
1041
|
+
/** Wait until `handle` owns a slot. Public only for the handle it
|
|
1042
|
+
* belongs to. */
|
|
1043
|
+
async acquire(handle, signal) {
|
|
1044
|
+
let lastTouch = this.now();
|
|
1045
|
+
for (; ; ) {
|
|
1046
|
+
if (signal?.aborted) {
|
|
1047
|
+
await this.cancel(handle.name);
|
|
1048
|
+
throw new QueueWaitAbortedError(handle.name);
|
|
1049
|
+
}
|
|
1050
|
+
await this.sweep();
|
|
1051
|
+
const order = await this.waitingOrder();
|
|
1052
|
+
const index = order.indexOf(handle.name);
|
|
1053
|
+
if (index < 0) throw new QueueTicketLostError(handle.name);
|
|
1054
|
+
const lease = await this.tryClaim(handle, index, order);
|
|
1055
|
+
if (lease) return lease;
|
|
1056
|
+
const now = this.now();
|
|
1057
|
+
if (now - lastTouch >= this.touchIntervalMs) {
|
|
1058
|
+
await touchFile(join10(this.ticketsDir, handle.name), now);
|
|
1059
|
+
lastTouch = now;
|
|
1060
|
+
}
|
|
1061
|
+
await this.sleep(this.pollIntervalMs);
|
|
1062
|
+
}
|
|
1063
|
+
}
|
|
1064
|
+
/** One claim attempt for the ticket sitting at `index` of the live
|
|
1065
|
+
* queue. Null means "not now" — the caller waits and retries.
|
|
1066
|
+
*
|
|
1067
|
+
* Position rule: a ticket may attempt a claim when its index is below
|
|
1068
|
+
* the environment's slot count. With the default concurrency of 1
|
|
1069
|
+
* that is strict FIFO; above it, two eligible tickets may take a
|
|
1070
|
+
* freed slot out of order, which is the price of not serialising
|
|
1071
|
+
* every waiter behind the head of the queue.
|
|
1072
|
+
*
|
|
1073
|
+
* A maintenance ticket is exclusive: it waits for the head position
|
|
1074
|
+
* and then takes EVERY slot, so nothing else runs while it mutates
|
|
1075
|
+
* the environment. While one is at the head, NOBODY else may claim —
|
|
1076
|
+
* without that drain rule a steady stream of later tickets would keep
|
|
1077
|
+
* grabbing each freed slot and the head's all-or-nothing claim would
|
|
1078
|
+
* starve forever. Detected from ticket names (the maintenance rank is
|
|
1079
|
+
* the first character), so waiters never read each other's files. */
|
|
1080
|
+
async tryClaim(handle, index, order) {
|
|
1081
|
+
const exclusive = handle.ticket.kind === "maintenance";
|
|
1082
|
+
if (exclusive ? index !== 0 : index >= this.envPool.total) return null;
|
|
1083
|
+
if (!exclusive && index > 0 && order[0].startsWith(MAINTENANCE_PREFIX)) {
|
|
1084
|
+
return null;
|
|
1085
|
+
}
|
|
1086
|
+
const payload = {
|
|
1087
|
+
runId: handle.ticket.runId,
|
|
1088
|
+
ticket: handle.name,
|
|
1089
|
+
pid: handle.ticket.producer.pid,
|
|
1090
|
+
host: handle.ticket.producer.host,
|
|
1091
|
+
at: this.now()
|
|
1092
|
+
};
|
|
1093
|
+
const envSlots = await this.envPool.claim(
|
|
1094
|
+
exclusive ? this.envPool.total : 1,
|
|
1095
|
+
payload
|
|
1096
|
+
);
|
|
1097
|
+
if (!envSlots) return null;
|
|
1098
|
+
let globalSlots = null;
|
|
1099
|
+
if (this.globalPool) {
|
|
1100
|
+
globalSlots = await this.globalPool.claim(handle.ticket.weight, payload);
|
|
1101
|
+
if (!globalSlots) {
|
|
1102
|
+
await this.envPool.release(envSlots);
|
|
1103
|
+
return null;
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
1106
|
+
const promoted = await moveFile(
|
|
1107
|
+
join10(this.ticketsDir, handle.name),
|
|
1108
|
+
join10(this.runningDir, handle.name)
|
|
1109
|
+
);
|
|
1110
|
+
if (!promoted) {
|
|
1111
|
+
await this.envPool.release(envSlots);
|
|
1112
|
+
if (globalSlots) await this.globalPool?.release(globalSlots);
|
|
1113
|
+
throw new QueueTicketLostError(handle.name);
|
|
1114
|
+
}
|
|
1115
|
+
return new FsLease(this, handle, envSlots, globalSlots);
|
|
1116
|
+
}
|
|
1117
|
+
/** Live waiting tickets, in service order. */
|
|
1118
|
+
async waitingOrder() {
|
|
1119
|
+
return (await listDir(this.ticketsDir)).filter((name) => parseQueueTicketName(name) !== null).sort(compareQueueTicketNames);
|
|
1120
|
+
}
|
|
1121
|
+
async readEntries(dir) {
|
|
1122
|
+
const names = (await listDir(dir)).filter((name) => parseQueueTicketName(name) !== null).sort(compareQueueTicketNames);
|
|
1123
|
+
const out = [];
|
|
1124
|
+
for (const name of names) {
|
|
1125
|
+
const ticket = parseQueueTicket(await readJson(join10(dir, name)));
|
|
1126
|
+
if (ticket) out.push({ name, ticket });
|
|
1127
|
+
}
|
|
1128
|
+
return out;
|
|
1129
|
+
}
|
|
1130
|
+
clampWeight(weight) {
|
|
1131
|
+
const w = Number.isFinite(weight) ? Math.floor(weight) : 1;
|
|
1132
|
+
const capped = this.globalPool ? Math.min(w, this.globalPool.total) : w;
|
|
1133
|
+
return Math.max(1, capped);
|
|
1134
|
+
}
|
|
1135
|
+
/** Heartbeat + teardown for a lease — kept here so `FsLease` stays a
|
|
1136
|
+
* handle and the filesystem knowledge stays in one class. */
|
|
1137
|
+
async releaseLease(handle, envSlots, globalSlots) {
|
|
1138
|
+
await removeFile(join10(this.runningDir, handle.name));
|
|
1139
|
+
if (globalSlots) await this.globalPool?.release(globalSlots);
|
|
1140
|
+
await this.envPool.release(envSlots);
|
|
1141
|
+
}
|
|
1142
|
+
releaseLeaseSync(handle, envSlots, globalSlots) {
|
|
1143
|
+
removeFileSync(join10(this.runningDir, handle.name));
|
|
1144
|
+
if (globalSlots) this.globalPool?.releaseSync(globalSlots);
|
|
1145
|
+
this.envPool.releaseSync(envSlots);
|
|
1146
|
+
}
|
|
1147
|
+
async touchLease(handle, envSlots, globalSlots) {
|
|
1148
|
+
await touchFile(join10(this.runningDir, handle.name), this.now());
|
|
1149
|
+
await this.envPool.touch(envSlots);
|
|
1150
|
+
if (globalSlots) await this.globalPool?.touch(globalSlots);
|
|
1151
|
+
}
|
|
1152
|
+
get heartbeatIntervalMs() {
|
|
1153
|
+
return this.touchIntervalMs;
|
|
1154
|
+
}
|
|
1155
|
+
};
|
|
1156
|
+
var FsTicketHandle = class {
|
|
1157
|
+
constructor(queue, name, ticket) {
|
|
1158
|
+
this.queue = queue;
|
|
1159
|
+
this.name = name;
|
|
1160
|
+
this.ticket = ticket;
|
|
1161
|
+
}
|
|
1162
|
+
queue;
|
|
1163
|
+
name;
|
|
1164
|
+
ticket;
|
|
1165
|
+
static {
|
|
1166
|
+
__name(this, "FsTicketHandle");
|
|
1167
|
+
}
|
|
1168
|
+
acquire(opts) {
|
|
1169
|
+
return this.queue.acquire(this, opts?.signal);
|
|
1170
|
+
}
|
|
1171
|
+
async cancel() {
|
|
1172
|
+
await this.queue.cancel(this.name);
|
|
1173
|
+
}
|
|
1174
|
+
};
|
|
1175
|
+
var FsLease = class {
|
|
1176
|
+
constructor(queue, handle, envSlots, globalSlots) {
|
|
1177
|
+
this.queue = queue;
|
|
1178
|
+
this.handle = handle;
|
|
1179
|
+
this.envSlots = envSlots;
|
|
1180
|
+
this.globalSlots = globalSlots;
|
|
1181
|
+
this.heartbeat = setInterval(() => {
|
|
1182
|
+
void this.queue.touchLease(this.handle, this.envSlots, this.globalSlots).catch(() => {
|
|
1183
|
+
});
|
|
1184
|
+
}, this.queue.heartbeatIntervalMs);
|
|
1185
|
+
this.heartbeat.unref?.();
|
|
1186
|
+
}
|
|
1187
|
+
queue;
|
|
1188
|
+
handle;
|
|
1189
|
+
envSlots;
|
|
1190
|
+
globalSlots;
|
|
1191
|
+
static {
|
|
1192
|
+
__name(this, "FsLease");
|
|
1193
|
+
}
|
|
1194
|
+
released = false;
|
|
1195
|
+
heartbeat;
|
|
1196
|
+
get runId() {
|
|
1197
|
+
return this.handle.ticket.runId;
|
|
1198
|
+
}
|
|
1199
|
+
get ticketName() {
|
|
1200
|
+
return this.handle.name;
|
|
1201
|
+
}
|
|
1202
|
+
get env() {
|
|
1203
|
+
return leaseEnv(this.handle.ticket.runId);
|
|
1204
|
+
}
|
|
1205
|
+
async release() {
|
|
1206
|
+
if (this.released) return;
|
|
1207
|
+
this.released = true;
|
|
1208
|
+
clearInterval(this.heartbeat);
|
|
1209
|
+
await this.queue.releaseLease(this.handle, this.envSlots, this.globalSlots);
|
|
1210
|
+
}
|
|
1211
|
+
releaseSync() {
|
|
1212
|
+
if (this.released) return;
|
|
1213
|
+
this.released = true;
|
|
1214
|
+
clearInterval(this.heartbeat);
|
|
1215
|
+
this.queue.releaseLeaseSync(this.handle, this.envSlots, this.globalSlots);
|
|
1216
|
+
}
|
|
1217
|
+
};
|
|
1218
|
+
function leaseEnv(runId) {
|
|
1219
|
+
return Object.freeze({ [QUEUE_LEASE_ENV]: runId });
|
|
1220
|
+
}
|
|
1221
|
+
__name(leaseEnv, "leaseEnv");
|
|
1222
|
+
async function readJson(path) {
|
|
1223
|
+
const text = await readTextFile(path);
|
|
1224
|
+
if (text === null) return null;
|
|
1225
|
+
try {
|
|
1226
|
+
return JSON.parse(text);
|
|
1227
|
+
} catch {
|
|
1228
|
+
return null;
|
|
1229
|
+
}
|
|
1230
|
+
}
|
|
1231
|
+
__name(readJson, "readJson");
|
|
1232
|
+
function randomToken() {
|
|
1233
|
+
const bytes = new Uint8Array(6);
|
|
1234
|
+
globalThis.crypto.getRandomValues(bytes);
|
|
1235
|
+
let out = "";
|
|
1236
|
+
for (const b of bytes) out += b.toString(16).padStart(2, "0");
|
|
1237
|
+
return out;
|
|
1238
|
+
}
|
|
1239
|
+
__name(randomToken, "randomToken");
|
|
1240
|
+
|
|
1241
|
+
// src/bundle/bundle-archive.ts
|
|
1242
|
+
import {
|
|
1243
|
+
BUNDLE_MANIFEST_FILE,
|
|
1244
|
+
BUNDLE_SCHEMA_VERSION,
|
|
1245
|
+
parseBundleManifest
|
|
1246
|
+
} from "@unotest/protocol";
|
|
1247
|
+
import { createHash } from "crypto";
|
|
1248
|
+
import { gunzipSync, gzipSync } from "zlib";
|
|
1249
|
+
|
|
1250
|
+
// src/bundle/errors.ts
|
|
1251
|
+
var BundleFormatError = class extends UnotestError {
|
|
1252
|
+
static {
|
|
1253
|
+
__name(this, "BundleFormatError");
|
|
1254
|
+
}
|
|
1255
|
+
constructor(reason, context = {}) {
|
|
1256
|
+
super(`not a readable test bundle: ${reason}`, { reason, ...context });
|
|
1257
|
+
}
|
|
1258
|
+
};
|
|
1259
|
+
var BundlePathError = class extends UnotestError {
|
|
1260
|
+
static {
|
|
1261
|
+
__name(this, "BundlePathError");
|
|
1262
|
+
}
|
|
1263
|
+
constructor(path, reason) {
|
|
1264
|
+
super(`bundle path "${path}" is not allowed: ${reason}`, { path, reason });
|
|
1265
|
+
}
|
|
1266
|
+
};
|
|
1267
|
+
var BundleTooLargeError = class extends UnotestError {
|
|
1268
|
+
static {
|
|
1269
|
+
__name(this, "BundleTooLargeError");
|
|
1270
|
+
}
|
|
1271
|
+
constructor(limitBytes) {
|
|
1272
|
+
super(
|
|
1273
|
+
`test bundle unpacks to more than ${limitBytes} bytes \u2014 the limit is there because a bundle is unpacked before anything about it is known`,
|
|
1274
|
+
{ limitBytes }
|
|
1275
|
+
);
|
|
1276
|
+
}
|
|
1277
|
+
};
|
|
1278
|
+
var BundleIdMismatchError = class extends UnotestError {
|
|
1279
|
+
static {
|
|
1280
|
+
__name(this, "BundleIdMismatchError");
|
|
1281
|
+
}
|
|
1282
|
+
constructor(claimed, computed) {
|
|
1283
|
+
super(
|
|
1284
|
+
`bundle content hashes to ${computed} but the manifest says ${claimed} \u2014 the archive was altered or truncated in transit`,
|
|
1285
|
+
{ claimed, computed }
|
|
1286
|
+
);
|
|
1287
|
+
}
|
|
1288
|
+
};
|
|
1289
|
+
|
|
1290
|
+
// src/bundle/ustar.ts
|
|
1291
|
+
var BLOCK = 512;
|
|
1292
|
+
var NAME_FIELD = 100;
|
|
1293
|
+
var PREFIX_FIELD = 155;
|
|
1294
|
+
var FILE_MODE = 420;
|
|
1295
|
+
var EXEC_MODE = 493;
|
|
1296
|
+
function assertSafeBundlePath(path) {
|
|
1297
|
+
if (path.length === 0) throw new BundlePathError(path, "it is empty");
|
|
1298
|
+
if (path.includes("\0")) throw new BundlePathError(path, "it contains a NUL byte");
|
|
1299
|
+
if (path.includes("\\")) {
|
|
1300
|
+
throw new BundlePathError(path, "paths are posix \u2014 use `/`, never `\\`");
|
|
1301
|
+
}
|
|
1302
|
+
if (path.startsWith("/")) throw new BundlePathError(path, "it is absolute");
|
|
1303
|
+
if (/^[a-zA-Z]:/.test(path)) {
|
|
1304
|
+
throw new BundlePathError(path, "it carries a Windows drive letter");
|
|
1305
|
+
}
|
|
1306
|
+
for (const segment of path.split("/")) {
|
|
1307
|
+
if (segment === "") throw new BundlePathError(path, "it has an empty segment");
|
|
1308
|
+
if (segment === "." || segment === "..") {
|
|
1309
|
+
throw new BundlePathError(path, "it walks out of the bundle with `.` / `..`");
|
|
1310
|
+
}
|
|
1311
|
+
}
|
|
1312
|
+
}
|
|
1313
|
+
__name(assertSafeBundlePath, "assertSafeBundlePath");
|
|
1314
|
+
function splitUstarPath(path) {
|
|
1315
|
+
if (Buffer.byteLength(path) <= NAME_FIELD) return { name: path, prefix: "" };
|
|
1316
|
+
const cut = path.lastIndexOf("/", PREFIX_FIELD);
|
|
1317
|
+
const prefix = cut > 0 ? path.slice(0, cut) : "";
|
|
1318
|
+
const name = cut > 0 ? path.slice(cut + 1) : path;
|
|
1319
|
+
if (prefix === "" || Buffer.byteLength(name) > NAME_FIELD || Buffer.byteLength(prefix) > PREFIX_FIELD) {
|
|
1320
|
+
throw new BundlePathError(
|
|
1321
|
+
path,
|
|
1322
|
+
`it does not fit the tar header (${NAME_FIELD} bytes of file name, ${PREFIX_FIELD} of directory) \u2014 shorten it`
|
|
1323
|
+
);
|
|
1324
|
+
}
|
|
1325
|
+
return { name, prefix };
|
|
1326
|
+
}
|
|
1327
|
+
__name(splitUstarPath, "splitUstarPath");
|
|
1328
|
+
function writeString(block, value, offset, size) {
|
|
1329
|
+
const bytes = Buffer.from(value, "utf8");
|
|
1330
|
+
if (bytes.length > size) {
|
|
1331
|
+
throw new BundleFormatError(`"${value}" does not fit a ${size}-byte header field`);
|
|
1332
|
+
}
|
|
1333
|
+
bytes.copy(block, offset);
|
|
1334
|
+
}
|
|
1335
|
+
__name(writeString, "writeString");
|
|
1336
|
+
function writeOctal(block, value, offset, size) {
|
|
1337
|
+
const text = value.toString(8).padStart(size - 1, "0");
|
|
1338
|
+
if (text.length > size - 1) {
|
|
1339
|
+
throw new BundleFormatError(`value ${value} does not fit a ${size}-byte octal field`);
|
|
1340
|
+
}
|
|
1341
|
+
block.write(`${text}\0`, offset, "ascii");
|
|
1342
|
+
}
|
|
1343
|
+
__name(writeOctal, "writeOctal");
|
|
1344
|
+
function readOctal(block, offset, size) {
|
|
1345
|
+
const raw = Buffer.from(block.subarray(offset, offset + size)).toString("ascii").replace(/\0.*$/s, "").trim();
|
|
1346
|
+
if (raw === "") return 0;
|
|
1347
|
+
if (!/^[0-7]+$/.test(raw)) {
|
|
1348
|
+
throw new BundleFormatError(`"${raw}" is not an octal header field`);
|
|
1349
|
+
}
|
|
1350
|
+
return parseInt(raw, 8);
|
|
1351
|
+
}
|
|
1352
|
+
__name(readOctal, "readOctal");
|
|
1353
|
+
function checksum(header) {
|
|
1354
|
+
let sum = 0;
|
|
1355
|
+
for (let i = 0; i < BLOCK; i++) {
|
|
1356
|
+
sum += i >= 148 && i < 156 ? 32 : header[i];
|
|
1357
|
+
}
|
|
1358
|
+
return sum;
|
|
1359
|
+
}
|
|
1360
|
+
__name(checksum, "checksum");
|
|
1361
|
+
function buildHeader(entry) {
|
|
1362
|
+
const { name, prefix } = splitUstarPath(entry.path);
|
|
1363
|
+
const header = Buffer.alloc(BLOCK);
|
|
1364
|
+
writeString(header, name, 0, NAME_FIELD);
|
|
1365
|
+
writeOctal(header, entry.executable ? EXEC_MODE : FILE_MODE, 100, 8);
|
|
1366
|
+
writeOctal(header, 0, 108, 8);
|
|
1367
|
+
writeOctal(header, 0, 116, 8);
|
|
1368
|
+
writeOctal(header, entry.content.byteLength, 124, 12);
|
|
1369
|
+
writeOctal(header, 0, 136, 12);
|
|
1370
|
+
header.fill(32, 148, 156);
|
|
1371
|
+
header.write("0", 156, "ascii");
|
|
1372
|
+
header.write("ustar\0", 257, "ascii");
|
|
1373
|
+
header.write("00", 263, "ascii");
|
|
1374
|
+
writeString(header, prefix, 345, PREFIX_FIELD);
|
|
1375
|
+
header.write(`${checksum(header).toString(8).padStart(6, "0")}\0 `, 148, "ascii");
|
|
1376
|
+
return header;
|
|
1377
|
+
}
|
|
1378
|
+
__name(buildHeader, "buildHeader");
|
|
1379
|
+
function writeTar(entries) {
|
|
1380
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1381
|
+
const sorted = [...entries].sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
|
|
1382
|
+
const blocks = [];
|
|
1383
|
+
for (const entry of sorted) {
|
|
1384
|
+
assertSafeBundlePath(entry.path);
|
|
1385
|
+
if (seen.has(entry.path)) {
|
|
1386
|
+
throw new BundleFormatError(`"${entry.path}" appears twice`, { path: entry.path });
|
|
1387
|
+
}
|
|
1388
|
+
seen.add(entry.path);
|
|
1389
|
+
blocks.push(buildHeader(entry));
|
|
1390
|
+
const padded = Math.ceil(entry.content.byteLength / BLOCK) * BLOCK;
|
|
1391
|
+
const body = Buffer.alloc(padded);
|
|
1392
|
+
Buffer.from(entry.content).copy(body);
|
|
1393
|
+
blocks.push(body);
|
|
1394
|
+
}
|
|
1395
|
+
blocks.push(Buffer.alloc(BLOCK * 2));
|
|
1396
|
+
return Buffer.concat(blocks);
|
|
1397
|
+
}
|
|
1398
|
+
__name(writeTar, "writeTar");
|
|
1399
|
+
function isZeroBlock(block) {
|
|
1400
|
+
return block.every((byte) => byte === 0);
|
|
1401
|
+
}
|
|
1402
|
+
__name(isZeroBlock, "isZeroBlock");
|
|
1403
|
+
function readTar(data) {
|
|
1404
|
+
const entries = [];
|
|
1405
|
+
let offset = 0;
|
|
1406
|
+
while (offset + BLOCK <= data.byteLength) {
|
|
1407
|
+
const header = data.subarray(offset, offset + BLOCK);
|
|
1408
|
+
if (isZeroBlock(header)) break;
|
|
1409
|
+
const magic = Buffer.from(header.subarray(257, 262)).toString("ascii");
|
|
1410
|
+
if (magic !== "ustar") {
|
|
1411
|
+
throw new BundleFormatError("a header without the ustar magic", { offset });
|
|
1412
|
+
}
|
|
1413
|
+
const stored = readOctal(header, 148, 8);
|
|
1414
|
+
if (stored !== checksum(header)) {
|
|
1415
|
+
throw new BundleFormatError("a header whose checksum does not match", { offset });
|
|
1416
|
+
}
|
|
1417
|
+
const typeflag = String.fromCharCode(header[156]);
|
|
1418
|
+
if (typeflag !== "0" && typeflag !== "\0") {
|
|
1419
|
+
throw new BundleFormatError(
|
|
1420
|
+
`a member of type "${typeflag}" \u2014 a bundle carries plain files only`,
|
|
1421
|
+
{ offset, typeflag }
|
|
1422
|
+
);
|
|
1423
|
+
}
|
|
1424
|
+
const name = Buffer.from(header.subarray(0, NAME_FIELD)).toString("utf8").replace(/\0.*$/s, "");
|
|
1425
|
+
const prefix = Buffer.from(header.subarray(345, 345 + PREFIX_FIELD)).toString("utf8").replace(/\0.*$/s, "");
|
|
1426
|
+
const path = prefix === "" ? name : `${prefix}/${name}`;
|
|
1427
|
+
assertSafeBundlePath(path);
|
|
1428
|
+
const size = readOctal(header, 124, 12);
|
|
1429
|
+
const start = offset + BLOCK;
|
|
1430
|
+
if (start + size > data.byteLength) {
|
|
1431
|
+
throw new BundleFormatError(`"${path}" is truncated`, { path });
|
|
1432
|
+
}
|
|
1433
|
+
const mode = readOctal(header, 100, 8);
|
|
1434
|
+
entries.push({
|
|
1435
|
+
path,
|
|
1436
|
+
// Copied, not a view: a caller that keeps one small file must not
|
|
1437
|
+
// pin the whole uploaded archive in memory.
|
|
1438
|
+
content: Buffer.from(data.subarray(start, start + size)),
|
|
1439
|
+
...(mode & 73) !== 0 ? { executable: true } : {}
|
|
1440
|
+
});
|
|
1441
|
+
offset = start + Math.ceil(size / BLOCK) * BLOCK;
|
|
1442
|
+
}
|
|
1443
|
+
return entries;
|
|
1444
|
+
}
|
|
1445
|
+
__name(readTar, "readTar");
|
|
1446
|
+
|
|
1447
|
+
// src/bundle/bundle-archive.ts
|
|
1448
|
+
var BUNDLE_ID_CHARS = 16;
|
|
1449
|
+
function byPath(a, b) {
|
|
1450
|
+
return a.path < b.path ? -1 : a.path > b.path ? 1 : 0;
|
|
1451
|
+
}
|
|
1452
|
+
__name(byPath, "byPath");
|
|
1453
|
+
function computeTreeSha256(files) {
|
|
1454
|
+
const tree = createHash("sha256");
|
|
1455
|
+
for (const file of [...files].sort(byPath)) {
|
|
1456
|
+
const content = createHash("sha256").update(file.content).digest("hex");
|
|
1457
|
+
tree.update(`${file.path}\0${file.executable ? "1" : "0"}\0${content}
|
|
1458
|
+
`);
|
|
1459
|
+
}
|
|
1460
|
+
return tree.digest("hex");
|
|
1461
|
+
}
|
|
1462
|
+
__name(computeTreeSha256, "computeTreeSha256");
|
|
1463
|
+
function computeBundleId(files) {
|
|
1464
|
+
return computeTreeSha256(files).slice(0, BUNDLE_ID_CHARS);
|
|
1465
|
+
}
|
|
1466
|
+
__name(computeBundleId, "computeBundleId");
|
|
1467
|
+
function packBundle(files, provenance) {
|
|
1468
|
+
if (files.length === 0) {
|
|
1469
|
+
throw new BundleFormatError("there is nothing to pack");
|
|
1470
|
+
}
|
|
1471
|
+
if (files.some((file) => file.path === BUNDLE_MANIFEST_FILE)) {
|
|
1472
|
+
throw new BundleFormatError(
|
|
1473
|
+
`${BUNDLE_MANIFEST_FILE} is the manifest's own name and cannot be a suite file`
|
|
1474
|
+
);
|
|
1475
|
+
}
|
|
1476
|
+
const sha256 = computeTreeSha256(files);
|
|
1477
|
+
const manifest = {
|
|
1478
|
+
schemaVersion: BUNDLE_SCHEMA_VERSION,
|
|
1479
|
+
bundleId: sha256.slice(0, BUNDLE_ID_CHARS),
|
|
1480
|
+
sha256,
|
|
1481
|
+
createdAt: provenance.createdAt,
|
|
1482
|
+
webVersion: provenance.webVersion,
|
|
1483
|
+
fileCount: files.length,
|
|
1484
|
+
byteSize: files.reduce((total, file) => total + file.content.byteLength, 0),
|
|
1485
|
+
...provenance.author !== void 0 ? { author: provenance.author } : {},
|
|
1486
|
+
...provenance.git !== void 0 ? { git: provenance.git } : {},
|
|
1487
|
+
...provenance.declaredVariables && provenance.declaredVariables.length > 0 ? { declaredVariables: [...provenance.declaredVariables] } : {}
|
|
1488
|
+
};
|
|
1489
|
+
const archive = writeTar([
|
|
1490
|
+
...files,
|
|
1491
|
+
{
|
|
1492
|
+
path: BUNDLE_MANIFEST_FILE,
|
|
1493
|
+
content: Buffer.from(`${JSON.stringify(manifest, null, 2)}
|
|
1494
|
+
`, "utf8")
|
|
1495
|
+
}
|
|
1496
|
+
]);
|
|
1497
|
+
return {
|
|
1498
|
+
bundleId: manifest.bundleId,
|
|
1499
|
+
manifest,
|
|
1500
|
+
// Level 9: a bundle is written once and uploaded once, and the box's
|
|
1501
|
+
// size limit is the thing this is competing against.
|
|
1502
|
+
archive: gzipSync(archive, { level: 9 })
|
|
1503
|
+
};
|
|
1504
|
+
}
|
|
1505
|
+
__name(packBundle, "packBundle");
|
|
1506
|
+
function unpackBundle(archive, options = {}) {
|
|
1507
|
+
const tar = gunzip(archive, options.maxBytes);
|
|
1508
|
+
const entries = readTar(tar);
|
|
1509
|
+
const manifestEntry = entries.find((entry) => entry.path === BUNDLE_MANIFEST_FILE);
|
|
1510
|
+
if (!manifestEntry) {
|
|
1511
|
+
throw new BundleFormatError(`it has no ${BUNDLE_MANIFEST_FILE}`);
|
|
1512
|
+
}
|
|
1513
|
+
let parsedJson;
|
|
1514
|
+
try {
|
|
1515
|
+
parsedJson = JSON.parse(Buffer.from(manifestEntry.content).toString("utf8"));
|
|
1516
|
+
} catch {
|
|
1517
|
+
throw new BundleFormatError(`${BUNDLE_MANIFEST_FILE} is not JSON`);
|
|
1518
|
+
}
|
|
1519
|
+
const manifest = parseBundleManifest(parsedJson);
|
|
1520
|
+
if (!manifest) {
|
|
1521
|
+
throw new BundleFormatError(
|
|
1522
|
+
`${BUNDLE_MANIFEST_FILE} is not a manifest this version understands (schema ${BUNDLE_SCHEMA_VERSION})`
|
|
1523
|
+
);
|
|
1524
|
+
}
|
|
1525
|
+
const files = entries.filter((entry) => entry.path !== BUNDLE_MANIFEST_FILE);
|
|
1526
|
+
const computed = computeTreeSha256(files);
|
|
1527
|
+
if (manifest.sha256 !== void 0 && computed !== manifest.sha256) {
|
|
1528
|
+
throw new BundleIdMismatchError(manifest.sha256, computed);
|
|
1529
|
+
}
|
|
1530
|
+
if (computed.slice(0, manifest.bundleId.length) !== manifest.bundleId) {
|
|
1531
|
+
throw new BundleIdMismatchError(manifest.bundleId, computed);
|
|
1532
|
+
}
|
|
1533
|
+
return { manifest, files };
|
|
1534
|
+
}
|
|
1535
|
+
__name(unpackBundle, "unpackBundle");
|
|
1536
|
+
function gunzip(archive, maxBytes) {
|
|
1537
|
+
if (archive.byteLength < 2 || archive[0] !== 31 || archive[1] !== 139) {
|
|
1538
|
+
throw new BundleFormatError("it is not gzip data");
|
|
1539
|
+
}
|
|
1540
|
+
try {
|
|
1541
|
+
return gunzipSync(archive, maxBytes !== void 0 ? { maxOutputLength: maxBytes } : {});
|
|
1542
|
+
} catch (e) {
|
|
1543
|
+
if (maxBytes !== void 0 && typeof e.code === "string" && e.code === "ERR_BUFFER_TOO_LARGE") {
|
|
1544
|
+
throw new BundleTooLargeError(maxBytes);
|
|
1545
|
+
}
|
|
1546
|
+
throw new BundleFormatError(
|
|
1547
|
+
`gzip could not be read (${e instanceof Error ? e.message : String(e)})`
|
|
1548
|
+
);
|
|
1549
|
+
}
|
|
1550
|
+
}
|
|
1551
|
+
__name(gunzip, "gunzip");
|
|
589
1552
|
export {
|
|
1553
|
+
BundleFormatError,
|
|
1554
|
+
BundleIdMismatchError,
|
|
1555
|
+
BundlePathError,
|
|
1556
|
+
BundleTooLargeError,
|
|
1557
|
+
FsRunQueue,
|
|
590
1558
|
JsonlRunArtifactWriter,
|
|
1559
|
+
QueueTicketLostError,
|
|
1560
|
+
QueueUnavailableError,
|
|
1561
|
+
QueueWaitAbortedError,
|
|
591
1562
|
UnotestError,
|
|
592
1563
|
appendUniqueLines,
|
|
593
1564
|
applyEnvLayers,
|
|
1565
|
+
assertSafeBundlePath,
|
|
594
1566
|
buildRuntimeInspection,
|
|
1567
|
+
computeBundleId,
|
|
1568
|
+
computeTreeSha256,
|
|
595
1569
|
createRunArtifactWriter,
|
|
596
1570
|
createRuntimeStateWriter,
|
|
597
1571
|
currentLocation,
|
|
598
1572
|
extractCallStack,
|
|
1573
|
+
globalPoolFromEnv,
|
|
1574
|
+
isRecord,
|
|
1575
|
+
leaseEnv,
|
|
599
1576
|
levenshteinDistance,
|
|
1577
|
+
packBundle,
|
|
600
1578
|
parseE2EFlags,
|
|
601
1579
|
readDebuggerBreakpoints,
|
|
602
1580
|
readEnvFile,
|
|
603
1581
|
readEnvLayers,
|
|
1582
|
+
readTar,
|
|
604
1583
|
startDebugCommandsWatcher,
|
|
605
1584
|
startRunHeartbeat,
|
|
606
1585
|
suggestClosest,
|
|
607
1586
|
toProtocolRuntimeState,
|
|
1587
|
+
unpackBundle,
|
|
608
1588
|
writeRunManifest,
|
|
609
|
-
writeRunSources
|
|
1589
|
+
writeRunSources,
|
|
1590
|
+
writeTar
|
|
610
1591
|
};
|
|
611
1592
|
//# sourceMappingURL=index.js.map
|