icechunk-js 0.3.2 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -10
- package/dist/index.cjs +1231 -215
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +105 -9
- package/dist/index.d.ts +105 -9
- package/dist/index.js +1230 -214
- package/dist/index.js.map +1 -1
- package/package.json +2 -3
package/dist/index.js
CHANGED
|
@@ -18,12 +18,12 @@ var StorageError = class extends Error {
|
|
|
18
18
|
this.name = "StorageError";
|
|
19
19
|
}
|
|
20
20
|
};
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
}
|
|
21
|
+
function hasErrorName(error) {
|
|
22
|
+
return typeof error === "object" && error !== null && "name" in error;
|
|
23
|
+
}
|
|
24
|
+
function isAbortError(error) {
|
|
25
|
+
return hasErrorName(error) && error.name === "AbortError";
|
|
26
|
+
}
|
|
27
27
|
|
|
28
28
|
// src/format/constants.ts
|
|
29
29
|
var PATHS = {
|
|
@@ -145,6 +145,7 @@ function decodeObjectId8(str) {
|
|
|
145
145
|
|
|
146
146
|
// src/format/flatbuffers/repo-parser.ts
|
|
147
147
|
import { ByteBuffer } from "flatbuffers";
|
|
148
|
+
import * as flexbuffers from "flatbuffers/js/flexbuffers.js";
|
|
148
149
|
|
|
149
150
|
// src/format/flatbuffers/generated/metadata-item.ts
|
|
150
151
|
var MetadataItem = class _MetadataItem {
|
|
@@ -298,6 +299,17 @@ var SnapshotInfo = class _SnapshotInfo {
|
|
|
298
299
|
const offset = this.bb.__offset(this.bb_pos, 12);
|
|
299
300
|
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
|
|
300
301
|
}
|
|
302
|
+
prunedAncestorTxLogs(index, obj) {
|
|
303
|
+
const offset = this.bb.__offset(this.bb_pos, 14);
|
|
304
|
+
return offset ? (obj || new ObjectId12()).__init(
|
|
305
|
+
this.bb.__vector(this.bb_pos + offset) + index * 12,
|
|
306
|
+
this.bb
|
|
307
|
+
) : null;
|
|
308
|
+
}
|
|
309
|
+
prunedAncestorTxLogsLength() {
|
|
310
|
+
const offset = this.bb.__offset(this.bb_pos, 14);
|
|
311
|
+
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
|
|
312
|
+
}
|
|
301
313
|
};
|
|
302
314
|
|
|
303
315
|
// src/format/flatbuffers/generated/update.ts
|
|
@@ -505,8 +517,587 @@ var Repo = class _Repo {
|
|
|
505
517
|
}
|
|
506
518
|
};
|
|
507
519
|
|
|
508
|
-
// src/
|
|
509
|
-
|
|
520
|
+
// src/vendor/fzstd/index.ts
|
|
521
|
+
var ab = ArrayBuffer;
|
|
522
|
+
var u8 = Uint8Array;
|
|
523
|
+
var u16 = Uint16Array;
|
|
524
|
+
var i16 = Int16Array;
|
|
525
|
+
var i32 = Int32Array;
|
|
526
|
+
var slc = (v, s, e) => {
|
|
527
|
+
if (u8.prototype.slice) return u8.prototype.slice.call(v, s, e);
|
|
528
|
+
if (s == null || s < 0) s = 0;
|
|
529
|
+
if (e == null || e > v.length) e = v.length;
|
|
530
|
+
const n = new u8(e - s);
|
|
531
|
+
n.set(v.subarray(s, e));
|
|
532
|
+
return n;
|
|
533
|
+
};
|
|
534
|
+
var fill = (v, n, s, e) => {
|
|
535
|
+
if (u8.prototype.fill) return u8.prototype.fill.call(v, n, s, e);
|
|
536
|
+
if (s == null || s < 0) s = 0;
|
|
537
|
+
if (e == null || e > v.length) e = v.length;
|
|
538
|
+
for (; s < e; ++s) v[s] = n;
|
|
539
|
+
return v;
|
|
540
|
+
};
|
|
541
|
+
var cpw = (v, t, s, e) => {
|
|
542
|
+
if (u8.prototype.copyWithin) return u8.prototype.copyWithin.call(v, t, s, e);
|
|
543
|
+
if (s == null || s < 0) s = 0;
|
|
544
|
+
if (e == null || e > v.length) e = v.length;
|
|
545
|
+
while (s < e) {
|
|
546
|
+
v[t++] = v[s++];
|
|
547
|
+
}
|
|
548
|
+
};
|
|
549
|
+
var ec = [
|
|
550
|
+
"invalid zstd data",
|
|
551
|
+
"window size too large (>2046MB)",
|
|
552
|
+
"invalid block type",
|
|
553
|
+
"FSE accuracy too high",
|
|
554
|
+
"match distance too far back",
|
|
555
|
+
"unexpected EOF",
|
|
556
|
+
"wrong dictionary"
|
|
557
|
+
];
|
|
558
|
+
var err = (ind, msg, nt) => {
|
|
559
|
+
const e = new Error(msg || ec[ind]);
|
|
560
|
+
e.code = ind;
|
|
561
|
+
if (Error.captureStackTrace) Error.captureStackTrace(e, err);
|
|
562
|
+
if (!nt) throw e;
|
|
563
|
+
return e;
|
|
564
|
+
};
|
|
565
|
+
var rb = (d, b, n) => {
|
|
566
|
+
let i = 0, o = 0;
|
|
567
|
+
for (; i < n; ++i) o |= d[b++] << (i << 3);
|
|
568
|
+
return o;
|
|
569
|
+
};
|
|
570
|
+
var b4 = (d, b) => (d[b] | d[b + 1] << 8 | d[b + 2] << 16 | d[b + 3] << 24) >>> 0;
|
|
571
|
+
var rdic = (dic, st) => {
|
|
572
|
+
if (b4(dic, 0) == 3962610743) {
|
|
573
|
+
if (st.d && st.d !== b4(dic, 4)) err(6);
|
|
574
|
+
let bt = 8;
|
|
575
|
+
[bt, st.h] = rhu(dic, bt);
|
|
576
|
+
let mlt, olt, llt;
|
|
577
|
+
[bt, olt] = rfse(dic, bt, 8);
|
|
578
|
+
[bt, mlt] = rfse(dic, bt, 9);
|
|
579
|
+
[bt, llt] = rfse(dic, bt, 9);
|
|
580
|
+
st.t = [mlt, olt, llt];
|
|
581
|
+
st.o = new i32([b4(dic, bt), b4(dic, bt + 4), b4(dic, bt + 8)]);
|
|
582
|
+
dic = dic.subarray(bt + 12);
|
|
583
|
+
}
|
|
584
|
+
const wl = Math.max(dic.length, st.u ? Math.min(st.e, dic.length + st.u) : st.e);
|
|
585
|
+
st.w = new u8(wl);
|
|
586
|
+
st.w.set(dic, wl - dic.length);
|
|
587
|
+
st.e = wl;
|
|
588
|
+
};
|
|
589
|
+
var rzfh = (dat, w) => {
|
|
590
|
+
const n3 = dat[0] | dat[1] << 8 | dat[2] << 16;
|
|
591
|
+
if (n3 == 3126568 && dat[3] == 253) {
|
|
592
|
+
const flg = dat[4];
|
|
593
|
+
const ss = flg >> 5 & 1, cc = flg >> 2 & 1, df = flg & 3, fcf = flg >> 6;
|
|
594
|
+
if (flg & 8) err(0);
|
|
595
|
+
let bt = 6 - ss;
|
|
596
|
+
const db = df == 3 ? 4 : df;
|
|
597
|
+
const di = rb(dat, bt, db);
|
|
598
|
+
bt += db;
|
|
599
|
+
const fsb = fcf ? 1 << fcf : ss;
|
|
600
|
+
const fss = rb(dat, bt, fsb) + (fcf == 1 && 256);
|
|
601
|
+
let ws = fss;
|
|
602
|
+
if (!ss) {
|
|
603
|
+
const wb = 1 << 10 + (dat[5] >> 3);
|
|
604
|
+
ws = wb + (wb >> 3) * (dat[5] & 7);
|
|
605
|
+
}
|
|
606
|
+
if (ws > 2145386496) err(1);
|
|
607
|
+
const buf = new u8((w == 1 ? fss || ws : w ? 0 : ws) + 12);
|
|
608
|
+
buf[0] = 1, buf[4] = 4, buf[8] = 8;
|
|
609
|
+
return {
|
|
610
|
+
b: bt + fsb,
|
|
611
|
+
y: 0,
|
|
612
|
+
l: 0,
|
|
613
|
+
d: di,
|
|
614
|
+
w: w && w != 1 ? w : buf.subarray(12),
|
|
615
|
+
e: ws,
|
|
616
|
+
o: new i32(buf.buffer, 0, 3),
|
|
617
|
+
u: fss,
|
|
618
|
+
c: cc,
|
|
619
|
+
m: Math.min(131072, ws)
|
|
620
|
+
};
|
|
621
|
+
} else if ((n3 >> 4 | dat[3] << 20) == 25481893) {
|
|
622
|
+
return b4(dat, 4) + 8;
|
|
623
|
+
}
|
|
624
|
+
err(0);
|
|
625
|
+
};
|
|
626
|
+
var msb = (val) => {
|
|
627
|
+
let bits = 0;
|
|
628
|
+
for (; 1 << bits <= val; ++bits) ;
|
|
629
|
+
return bits - 1;
|
|
630
|
+
};
|
|
631
|
+
var rfse = (dat, bt, mal) => {
|
|
632
|
+
let tpos = (bt << 3) + 4;
|
|
633
|
+
const al = (dat[bt] & 15) + 5;
|
|
634
|
+
if (al > mal) err(3);
|
|
635
|
+
const sz = 1 << al;
|
|
636
|
+
let probs = sz, sym = -1, re = -1, i = -1, ht = sz;
|
|
637
|
+
const buf = new ab(512 + (sz << 2));
|
|
638
|
+
const freq = new i16(buf, 0, 256);
|
|
639
|
+
const dstate = new u16(buf, 0, 256);
|
|
640
|
+
const nstate = new u16(buf, 512, sz);
|
|
641
|
+
const bb1 = 512 + (sz << 1);
|
|
642
|
+
const syms = new u8(buf, bb1, sz);
|
|
643
|
+
const nbits = new u8(buf, bb1 + sz);
|
|
644
|
+
while (sym < 255 && probs > 0) {
|
|
645
|
+
const bits = msb(probs + 1);
|
|
646
|
+
const cbt = tpos >> 3;
|
|
647
|
+
const msk = (1 << bits + 1) - 1;
|
|
648
|
+
let val = (dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (tpos & 7) & msk;
|
|
649
|
+
const msk1fb = (1 << bits) - 1;
|
|
650
|
+
const msv = msk - probs - 1;
|
|
651
|
+
const sval = val & msk1fb;
|
|
652
|
+
if (sval < msv) tpos += bits, val = sval;
|
|
653
|
+
else {
|
|
654
|
+
tpos += bits + 1;
|
|
655
|
+
if (val > msk1fb) val -= msv;
|
|
656
|
+
}
|
|
657
|
+
freq[++sym] = --val;
|
|
658
|
+
if (val == -1) {
|
|
659
|
+
probs += val;
|
|
660
|
+
syms[--ht] = sym;
|
|
661
|
+
} else probs -= val;
|
|
662
|
+
if (!val) {
|
|
663
|
+
do {
|
|
664
|
+
const rbt = tpos >> 3;
|
|
665
|
+
re = (dat[rbt] | dat[rbt + 1] << 8) >> (tpos & 7) & 3;
|
|
666
|
+
tpos += 2;
|
|
667
|
+
sym += re;
|
|
668
|
+
} while (re == 3);
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
if (sym > 255 || probs) err(0);
|
|
672
|
+
let sympos = 0;
|
|
673
|
+
const sstep = (sz >> 1) + (sz >> 3) + 3;
|
|
674
|
+
const smask = sz - 1;
|
|
675
|
+
for (let s = 0; s <= sym; ++s) {
|
|
676
|
+
const sf = freq[s];
|
|
677
|
+
if (sf < 1) {
|
|
678
|
+
dstate[s] = -sf;
|
|
679
|
+
continue;
|
|
680
|
+
}
|
|
681
|
+
for (i = 0; i < sf; ++i) {
|
|
682
|
+
syms[sympos] = s;
|
|
683
|
+
do {
|
|
684
|
+
sympos = sympos + sstep & smask;
|
|
685
|
+
} while (sympos >= ht);
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
if (sympos) err(0);
|
|
689
|
+
for (i = 0; i < sz; ++i) {
|
|
690
|
+
const ns = dstate[syms[i]]++;
|
|
691
|
+
const nb = nbits[i] = al - msb(ns);
|
|
692
|
+
nstate[i] = (ns << nb) - sz;
|
|
693
|
+
}
|
|
694
|
+
return [tpos + 7 >> 3, {
|
|
695
|
+
b: al,
|
|
696
|
+
s: syms,
|
|
697
|
+
n: nbits,
|
|
698
|
+
t: nstate
|
|
699
|
+
}];
|
|
700
|
+
};
|
|
701
|
+
var rhu = (dat, bt) => {
|
|
702
|
+
let i = 0, wc = -1;
|
|
703
|
+
const buf = new u8(292), hb = dat[bt];
|
|
704
|
+
const hw = buf.subarray(0, 256);
|
|
705
|
+
const rc = buf.subarray(256, 268);
|
|
706
|
+
const ri = new u16(buf.buffer, 268);
|
|
707
|
+
if (hb < 128) {
|
|
708
|
+
const [ebt, fdt] = rfse(dat, bt + 1, 6);
|
|
709
|
+
bt += hb;
|
|
710
|
+
const epos = ebt << 3;
|
|
711
|
+
const lb = dat[bt];
|
|
712
|
+
if (!lb) err(0);
|
|
713
|
+
let st1 = 0, st2 = 0, btr1 = fdt.b, btr2 = btr1;
|
|
714
|
+
let fpos = (++bt << 3) - 8 + msb(lb);
|
|
715
|
+
for (; ; ) {
|
|
716
|
+
fpos -= btr1;
|
|
717
|
+
if (fpos < epos) break;
|
|
718
|
+
let cbt = fpos >> 3;
|
|
719
|
+
st1 += (dat[cbt] | dat[cbt + 1] << 8) >> (fpos & 7) & (1 << btr1) - 1;
|
|
720
|
+
hw[++wc] = fdt.s[st1];
|
|
721
|
+
fpos -= btr2;
|
|
722
|
+
if (fpos < epos) break;
|
|
723
|
+
cbt = fpos >> 3;
|
|
724
|
+
st2 += (dat[cbt] | dat[cbt + 1] << 8) >> (fpos & 7) & (1 << btr2) - 1;
|
|
725
|
+
hw[++wc] = fdt.s[st2];
|
|
726
|
+
btr1 = fdt.n[st1];
|
|
727
|
+
st1 = fdt.t[st1];
|
|
728
|
+
btr2 = fdt.n[st2];
|
|
729
|
+
st2 = fdt.t[st2];
|
|
730
|
+
}
|
|
731
|
+
if (++wc > 255) err(0);
|
|
732
|
+
} else {
|
|
733
|
+
wc = hb - 127;
|
|
734
|
+
for (; i < wc; i += 2) {
|
|
735
|
+
const byte = dat[++bt];
|
|
736
|
+
hw[i] = byte >> 4;
|
|
737
|
+
hw[i + 1] = byte & 15;
|
|
738
|
+
}
|
|
739
|
+
++bt;
|
|
740
|
+
}
|
|
741
|
+
let wes = 0;
|
|
742
|
+
for (i = 0; i < wc; ++i) {
|
|
743
|
+
const wt = hw[i];
|
|
744
|
+
if (wt > 11) err(0);
|
|
745
|
+
wes += wt && 1 << wt - 1;
|
|
746
|
+
}
|
|
747
|
+
const mb = msb(wes) + 1;
|
|
748
|
+
const ts = 1 << mb;
|
|
749
|
+
const rem = ts - wes;
|
|
750
|
+
if (rem & rem - 1) err(0);
|
|
751
|
+
hw[wc++] = msb(rem) + 1;
|
|
752
|
+
for (i = 0; i < wc; ++i) {
|
|
753
|
+
const wt = hw[i];
|
|
754
|
+
++rc[hw[i] = wt && mb + 1 - wt];
|
|
755
|
+
}
|
|
756
|
+
const hbuf = new u8(ts << 1);
|
|
757
|
+
const syms = hbuf.subarray(0, ts), nb = hbuf.subarray(ts);
|
|
758
|
+
ri[mb] = 0;
|
|
759
|
+
for (i = mb; i > 0; --i) {
|
|
760
|
+
const pv = ri[i];
|
|
761
|
+
fill(nb, i, pv, ri[i - 1] = pv + rc[i] * (1 << mb - i));
|
|
762
|
+
}
|
|
763
|
+
if (ri[0] != ts) err(0);
|
|
764
|
+
for (i = 0; i < wc; ++i) {
|
|
765
|
+
const bits = hw[i];
|
|
766
|
+
if (bits) {
|
|
767
|
+
const code = ri[bits];
|
|
768
|
+
fill(syms, i, code, ri[bits] = code + (1 << mb - bits));
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
return [bt, {
|
|
772
|
+
n: nb,
|
|
773
|
+
b: mb,
|
|
774
|
+
s: syms
|
|
775
|
+
}];
|
|
776
|
+
};
|
|
777
|
+
var dllt = rfse(/* @__PURE__ */ new u8([
|
|
778
|
+
81,
|
|
779
|
+
16,
|
|
780
|
+
99,
|
|
781
|
+
140,
|
|
782
|
+
49,
|
|
783
|
+
198,
|
|
784
|
+
24,
|
|
785
|
+
99,
|
|
786
|
+
12,
|
|
787
|
+
33,
|
|
788
|
+
196,
|
|
789
|
+
24,
|
|
790
|
+
99,
|
|
791
|
+
102,
|
|
792
|
+
102,
|
|
793
|
+
134,
|
|
794
|
+
70,
|
|
795
|
+
146,
|
|
796
|
+
4
|
|
797
|
+
]), 0, 6)[1];
|
|
798
|
+
var dmlt = rfse(/* @__PURE__ */ new u8([
|
|
799
|
+
33,
|
|
800
|
+
20,
|
|
801
|
+
196,
|
|
802
|
+
24,
|
|
803
|
+
99,
|
|
804
|
+
140,
|
|
805
|
+
33,
|
|
806
|
+
132,
|
|
807
|
+
16,
|
|
808
|
+
66,
|
|
809
|
+
8,
|
|
810
|
+
33,
|
|
811
|
+
132,
|
|
812
|
+
16,
|
|
813
|
+
66,
|
|
814
|
+
8,
|
|
815
|
+
33,
|
|
816
|
+
68,
|
|
817
|
+
68,
|
|
818
|
+
68,
|
|
819
|
+
68,
|
|
820
|
+
68,
|
|
821
|
+
68,
|
|
822
|
+
68,
|
|
823
|
+
68,
|
|
824
|
+
36,
|
|
825
|
+
9
|
|
826
|
+
]), 0, 6)[1];
|
|
827
|
+
var doct = rfse(/* @__PURE__ */ new u8([
|
|
828
|
+
32,
|
|
829
|
+
132,
|
|
830
|
+
16,
|
|
831
|
+
66,
|
|
832
|
+
102,
|
|
833
|
+
70,
|
|
834
|
+
68,
|
|
835
|
+
68,
|
|
836
|
+
68,
|
|
837
|
+
68,
|
|
838
|
+
36,
|
|
839
|
+
73,
|
|
840
|
+
2
|
|
841
|
+
]), 0, 5)[1];
|
|
842
|
+
var b2bl = (b, s) => {
|
|
843
|
+
const len = b.length, bl = new i32(len);
|
|
844
|
+
for (let i = 0; i < len; ++i) {
|
|
845
|
+
bl[i] = s;
|
|
846
|
+
s += 1 << b[i];
|
|
847
|
+
}
|
|
848
|
+
return bl;
|
|
849
|
+
};
|
|
850
|
+
var llb = /* @__PURE__ */ new u8((/* @__PURE__ */ new i32([
|
|
851
|
+
0,
|
|
852
|
+
0,
|
|
853
|
+
0,
|
|
854
|
+
0,
|
|
855
|
+
16843009,
|
|
856
|
+
50528770,
|
|
857
|
+
134678020,
|
|
858
|
+
202050057,
|
|
859
|
+
269422093
|
|
860
|
+
])).buffer, 0, 36);
|
|
861
|
+
var llbl = /* @__PURE__ */ b2bl(llb, 0);
|
|
862
|
+
var mlb = /* @__PURE__ */ new u8((/* @__PURE__ */ new i32([
|
|
863
|
+
0,
|
|
864
|
+
0,
|
|
865
|
+
0,
|
|
866
|
+
0,
|
|
867
|
+
0,
|
|
868
|
+
0,
|
|
869
|
+
0,
|
|
870
|
+
0,
|
|
871
|
+
16843009,
|
|
872
|
+
50528770,
|
|
873
|
+
117769220,
|
|
874
|
+
185207048,
|
|
875
|
+
252579084,
|
|
876
|
+
16
|
|
877
|
+
])).buffer, 0, 53);
|
|
878
|
+
var mlbl = /* @__PURE__ */ b2bl(mlb, 3);
|
|
879
|
+
var dhu = (dat, out, hu) => {
|
|
880
|
+
const len = dat.length, ss = out.length, lb = dat[len - 1], msk = (1 << hu.b) - 1, eb = -hu.b;
|
|
881
|
+
if (!lb) err(0);
|
|
882
|
+
let st = 0, btr = hu.b, pos = (len << 3) - 8 + msb(lb) - btr, i = -1;
|
|
883
|
+
for (; pos > eb && i < ss; ) {
|
|
884
|
+
const cbt = pos >> 3;
|
|
885
|
+
const val = (dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (pos & 7);
|
|
886
|
+
st = (st << btr | val) & msk;
|
|
887
|
+
out[++i] = hu.s[st];
|
|
888
|
+
pos -= btr = hu.n[st];
|
|
889
|
+
}
|
|
890
|
+
if (pos != eb || i + 1 != ss) err(0);
|
|
891
|
+
};
|
|
892
|
+
var dhu4 = (dat, out, hu) => {
|
|
893
|
+
let bt = 6;
|
|
894
|
+
const ss = out.length, sz1 = ss + 3 >> 2, sz2 = sz1 << 1, sz3 = sz1 + sz2;
|
|
895
|
+
dhu(dat.subarray(bt, bt += dat[0] | dat[1] << 8), out.subarray(0, sz1), hu);
|
|
896
|
+
dhu(dat.subarray(bt, bt += dat[2] | dat[3] << 8), out.subarray(sz1, sz2), hu);
|
|
897
|
+
dhu(dat.subarray(bt, bt += dat[4] | dat[5] << 8), out.subarray(sz2, sz3), hu);
|
|
898
|
+
dhu(dat.subarray(bt), out.subarray(sz3), hu);
|
|
899
|
+
};
|
|
900
|
+
var rzb = (dat, st, out) => {
|
|
901
|
+
let bt = st.b;
|
|
902
|
+
const b0 = dat[bt], btype = b0 >> 1 & 3;
|
|
903
|
+
st.l = b0 & 1;
|
|
904
|
+
const sz = b0 >> 3 | dat[bt + 1] << 5 | dat[bt + 2] << 13;
|
|
905
|
+
const ebt = (bt += 3) + sz;
|
|
906
|
+
if (btype == 1) {
|
|
907
|
+
if (bt >= dat.length) return;
|
|
908
|
+
st.b = bt + 1;
|
|
909
|
+
if (out) {
|
|
910
|
+
fill(out, dat[bt], st.y, st.y += sz);
|
|
911
|
+
return out;
|
|
912
|
+
}
|
|
913
|
+
return fill(new u8(sz), dat[bt]);
|
|
914
|
+
}
|
|
915
|
+
if (ebt > dat.length) return;
|
|
916
|
+
if (btype == 0) {
|
|
917
|
+
st.b = ebt;
|
|
918
|
+
if (out) {
|
|
919
|
+
out.set(dat.subarray(bt, ebt), st.y);
|
|
920
|
+
st.y += sz;
|
|
921
|
+
return out;
|
|
922
|
+
}
|
|
923
|
+
return slc(dat, bt, ebt);
|
|
924
|
+
}
|
|
925
|
+
if (btype == 2) {
|
|
926
|
+
const b3 = dat[bt], lbt = b3 & 3, sf = b3 >> 2 & 3;
|
|
927
|
+
let lss = b3 >> 4, lcs = 0, s4 = 0;
|
|
928
|
+
if (lbt < 2) {
|
|
929
|
+
if (sf & 1) lss |= dat[++bt] << 4 | (sf & 2 && dat[++bt] << 12);
|
|
930
|
+
else lss = b3 >> 3;
|
|
931
|
+
} else {
|
|
932
|
+
s4 = sf;
|
|
933
|
+
if (sf < 2) lss |= (dat[++bt] & 63) << 4, lcs = dat[bt] >> 6 | dat[++bt] << 2;
|
|
934
|
+
else if (sf == 2) lss |= dat[++bt] << 4 | (dat[++bt] & 3) << 12, lcs = dat[bt] >> 2 | dat[++bt] << 6;
|
|
935
|
+
else lss |= dat[++bt] << 4 | (dat[++bt] & 63) << 12, lcs = dat[bt] >> 6 | dat[++bt] << 2 | dat[++bt] << 10;
|
|
936
|
+
}
|
|
937
|
+
++bt;
|
|
938
|
+
let buf = out ? out.subarray(st.y, st.y + st.m) : new u8(st.m);
|
|
939
|
+
let spl = buf.length - lss;
|
|
940
|
+
if (lbt == 0) buf.set(dat.subarray(bt, bt += lss), spl);
|
|
941
|
+
else if (lbt == 1) fill(buf, dat[bt++], spl);
|
|
942
|
+
else {
|
|
943
|
+
let hu = st.h;
|
|
944
|
+
if (lbt == 2) {
|
|
945
|
+
const hud = rhu(dat, bt);
|
|
946
|
+
lcs += bt - (bt = hud[0]);
|
|
947
|
+
st.h = hu = hud[1];
|
|
948
|
+
} else if (!hu) err(0);
|
|
949
|
+
(s4 ? dhu4 : dhu)(dat.subarray(bt, bt += lcs), buf.subarray(spl), hu);
|
|
950
|
+
}
|
|
951
|
+
let ns = dat[bt++];
|
|
952
|
+
if (ns) {
|
|
953
|
+
if (ns == 255) ns = (dat[bt++] | dat[bt++] << 8) + 32512;
|
|
954
|
+
else if (ns > 127) ns = ns - 128 << 8 | dat[bt++];
|
|
955
|
+
const scm = dat[bt++];
|
|
956
|
+
if (scm & 3) err(0);
|
|
957
|
+
const dts = [dmlt, doct, dllt];
|
|
958
|
+
for (let i = 2; i > -1; --i) {
|
|
959
|
+
const md = scm >> (i << 1) + 2 & 3;
|
|
960
|
+
if (md == 1) {
|
|
961
|
+
const rbuf = new u8([0, 0, dat[bt++]]);
|
|
962
|
+
dts[i] = {
|
|
963
|
+
s: rbuf.subarray(2, 3),
|
|
964
|
+
n: rbuf.subarray(0, 1),
|
|
965
|
+
t: new u16(rbuf.buffer, 0, 1),
|
|
966
|
+
b: 0
|
|
967
|
+
};
|
|
968
|
+
} else if (md == 2) {
|
|
969
|
+
[bt, dts[i]] = rfse(dat, bt, 9 - (i & 1));
|
|
970
|
+
} else if (md == 3) {
|
|
971
|
+
if (!st.t) err(0);
|
|
972
|
+
dts[i] = st.t[i];
|
|
973
|
+
}
|
|
974
|
+
}
|
|
975
|
+
const [mlt, oct, llt] = st.t = dts;
|
|
976
|
+
const lb = dat[ebt - 1];
|
|
977
|
+
if (!lb) err(0);
|
|
978
|
+
let spos = (ebt << 3) - 8 + msb(lb) - llt.b, cbt = spos >> 3, oubt = 0;
|
|
979
|
+
let lst = (dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << llt.b) - 1;
|
|
980
|
+
cbt = (spos -= oct.b) >> 3;
|
|
981
|
+
let ost = (dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << oct.b) - 1;
|
|
982
|
+
cbt = (spos -= mlt.b) >> 3;
|
|
983
|
+
let mst = (dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << mlt.b) - 1;
|
|
984
|
+
for (++ns; --ns; ) {
|
|
985
|
+
const llc = llt.s[lst];
|
|
986
|
+
const lbtr = llt.n[lst];
|
|
987
|
+
const mlc = mlt.s[mst];
|
|
988
|
+
const mbtr = mlt.n[mst];
|
|
989
|
+
const ofc = oct.s[ost];
|
|
990
|
+
const obtr = oct.n[ost];
|
|
991
|
+
cbt = (spos -= ofc) >> 3;
|
|
992
|
+
const ofp = 1 << ofc;
|
|
993
|
+
let off = ofp + ((dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16 | dat[cbt + 3] << 24) >>> (spos & 7) & ofp - 1);
|
|
994
|
+
cbt = (spos -= mlb[mlc]) >> 3;
|
|
995
|
+
let ml = mlbl[mlc] + ((dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (spos & 7) & (1 << mlb[mlc]) - 1);
|
|
996
|
+
cbt = (spos -= llb[llc]) >> 3;
|
|
997
|
+
const ll = llbl[llc] + ((dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (spos & 7) & (1 << llb[llc]) - 1);
|
|
998
|
+
cbt = (spos -= lbtr) >> 3;
|
|
999
|
+
lst = llt.t[lst] + ((dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << lbtr) - 1);
|
|
1000
|
+
cbt = (spos -= mbtr) >> 3;
|
|
1001
|
+
mst = mlt.t[mst] + ((dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << mbtr) - 1);
|
|
1002
|
+
cbt = (spos -= obtr) >> 3;
|
|
1003
|
+
ost = oct.t[ost] + ((dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << obtr) - 1);
|
|
1004
|
+
if (off > 3) {
|
|
1005
|
+
st.o[2] = st.o[1];
|
|
1006
|
+
st.o[1] = st.o[0];
|
|
1007
|
+
st.o[0] = off -= 3;
|
|
1008
|
+
} else {
|
|
1009
|
+
const idx = off - (ll != 0);
|
|
1010
|
+
if (idx) {
|
|
1011
|
+
off = idx == 3 ? st.o[0] - 1 : st.o[idx];
|
|
1012
|
+
if (idx > 1) st.o[2] = st.o[1];
|
|
1013
|
+
st.o[1] = st.o[0];
|
|
1014
|
+
st.o[0] = off;
|
|
1015
|
+
} else off = st.o[0];
|
|
1016
|
+
}
|
|
1017
|
+
for (let i = 0; i < ll; ++i) {
|
|
1018
|
+
buf[oubt + i] = buf[spl + i];
|
|
1019
|
+
}
|
|
1020
|
+
oubt += ll, spl += ll;
|
|
1021
|
+
let stin = oubt - off;
|
|
1022
|
+
if (stin < 0) {
|
|
1023
|
+
let len = -stin;
|
|
1024
|
+
const bs = st.e + stin;
|
|
1025
|
+
if (len > ml) len = ml;
|
|
1026
|
+
for (let i = 0; i < len; ++i) {
|
|
1027
|
+
buf[oubt + i] = st.w[bs + i];
|
|
1028
|
+
}
|
|
1029
|
+
oubt += len, ml -= len, stin = 0;
|
|
1030
|
+
}
|
|
1031
|
+
for (let i = 0; i < ml; ++i) {
|
|
1032
|
+
buf[oubt + i] = buf[stin + i];
|
|
1033
|
+
}
|
|
1034
|
+
oubt += ml;
|
|
1035
|
+
}
|
|
1036
|
+
if (oubt != spl) {
|
|
1037
|
+
while (spl < buf.length) {
|
|
1038
|
+
buf[oubt++] = buf[spl++];
|
|
1039
|
+
}
|
|
1040
|
+
} else oubt = buf.length;
|
|
1041
|
+
if (out) st.y += oubt;
|
|
1042
|
+
else buf = slc(buf, 0, oubt);
|
|
1043
|
+
} else if (out) {
|
|
1044
|
+
st.y += lss;
|
|
1045
|
+
if (spl) {
|
|
1046
|
+
for (let i = 0; i < lss; ++i) {
|
|
1047
|
+
buf[i] = buf[spl + i];
|
|
1048
|
+
}
|
|
1049
|
+
}
|
|
1050
|
+
} else if (spl) buf = slc(buf, spl);
|
|
1051
|
+
st.b = ebt;
|
|
1052
|
+
return buf;
|
|
1053
|
+
}
|
|
1054
|
+
err(2);
|
|
1055
|
+
};
|
|
1056
|
+
var cct = (bufs, ol) => {
|
|
1057
|
+
if (bufs.length == 1) return bufs[0];
|
|
1058
|
+
const buf = new u8(ol);
|
|
1059
|
+
for (let i = 0, b = 0; i < bufs.length; ++i) {
|
|
1060
|
+
const chk = bufs[i];
|
|
1061
|
+
buf.set(chk, b);
|
|
1062
|
+
b += chk.length;
|
|
1063
|
+
}
|
|
1064
|
+
return buf;
|
|
1065
|
+
};
|
|
1066
|
+
function decompress(dat, buf, dic) {
|
|
1067
|
+
const bufs = [], nb = +!buf;
|
|
1068
|
+
let bt = 0, ol = 0;
|
|
1069
|
+
for (; dat.length; ) {
|
|
1070
|
+
const st = rzfh(dat, nb || buf);
|
|
1071
|
+
if (typeof st == "object") {
|
|
1072
|
+
if (st.d && !dic) err(6);
|
|
1073
|
+
if (dic) rdic(dic, st);
|
|
1074
|
+
if (nb) {
|
|
1075
|
+
buf = null;
|
|
1076
|
+
if (!dic && st.w.length == st.u) {
|
|
1077
|
+
bufs.push(buf = st.w);
|
|
1078
|
+
ol += st.u;
|
|
1079
|
+
}
|
|
1080
|
+
} else {
|
|
1081
|
+
bufs.push(buf);
|
|
1082
|
+
st.e = 0;
|
|
1083
|
+
}
|
|
1084
|
+
for (; !st.l; ) {
|
|
1085
|
+
const blk = rzb(dat, st, buf);
|
|
1086
|
+
if (!blk) err(5);
|
|
1087
|
+
if (buf) st.e = st.y;
|
|
1088
|
+
else {
|
|
1089
|
+
bufs.push(blk);
|
|
1090
|
+
ol += blk.length;
|
|
1091
|
+
cpw(st.w, 0, blk.length);
|
|
1092
|
+
st.w.set(blk, st.w.length - blk.length);
|
|
1093
|
+
}
|
|
1094
|
+
}
|
|
1095
|
+
bt = st.b + st.c * 4;
|
|
1096
|
+
} else bt = st;
|
|
1097
|
+
dat = dat.subarray(bt);
|
|
1098
|
+
}
|
|
1099
|
+
return cct(bufs, ol);
|
|
1100
|
+
}
|
|
510
1101
|
|
|
511
1102
|
// src/format/header.ts
|
|
512
1103
|
var HEADER_SIZE = 39;
|
|
@@ -631,7 +1222,36 @@ function parseRepo(data) {
|
|
|
631
1222
|
);
|
|
632
1223
|
}
|
|
633
1224
|
const snapshotsLength = repo.snapshotsLength();
|
|
634
|
-
|
|
1225
|
+
const virtualChunkContainers = parseVirtualChunkContainers(repo);
|
|
1226
|
+
return { repo, specVersion, snapshotsLength, virtualChunkContainers };
|
|
1227
|
+
}
|
|
1228
|
+
function parseVirtualChunkContainers(repo) {
|
|
1229
|
+
const result = /* @__PURE__ */ new Map();
|
|
1230
|
+
const configBytes = repo.configArray();
|
|
1231
|
+
if (!configBytes || configBytes.length === 0) return result;
|
|
1232
|
+
let config;
|
|
1233
|
+
try {
|
|
1234
|
+
const ab2 = configBytes.buffer.slice(
|
|
1235
|
+
configBytes.byteOffset,
|
|
1236
|
+
configBytes.byteOffset + configBytes.byteLength
|
|
1237
|
+
);
|
|
1238
|
+
config = flexbuffers.toObject(ab2);
|
|
1239
|
+
} catch {
|
|
1240
|
+
return result;
|
|
1241
|
+
}
|
|
1242
|
+
if (!isRecord(config)) return result;
|
|
1243
|
+
const containers = config.virtual_chunk_containers;
|
|
1244
|
+
if (!isRecord(containers)) return result;
|
|
1245
|
+
for (const container of Object.values(containers)) {
|
|
1246
|
+
if (!isRecord(container)) continue;
|
|
1247
|
+
const { name, url_prefix: urlPrefix } = container;
|
|
1248
|
+
if (typeof name !== "string" || typeof urlPrefix !== "string") continue;
|
|
1249
|
+
result.set(name, urlPrefix);
|
|
1250
|
+
}
|
|
1251
|
+
return result;
|
|
1252
|
+
}
|
|
1253
|
+
function isRecord(value) {
|
|
1254
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
635
1255
|
}
|
|
636
1256
|
function compareUtf8ByteOrder(aBytes, bBytes) {
|
|
637
1257
|
const minLen = Math.min(aBytes.length, bBytes.length);
|
|
@@ -728,9 +1348,6 @@ function listTagsFromRepo(parsedRepo) {
|
|
|
728
1348
|
return listRefs((i) => repo.tags(i), repo.tagsLength());
|
|
729
1349
|
}
|
|
730
1350
|
|
|
731
|
-
// src/reader/session.ts
|
|
732
|
-
import { decompress as decompress2 } from "fzstd";
|
|
733
|
-
|
|
734
1351
|
// src/cache/lru.ts
|
|
735
1352
|
var LRUCache = class {
|
|
736
1353
|
cache;
|
|
@@ -774,6 +1391,85 @@ var LRUCache = class {
|
|
|
774
1391
|
}
|
|
775
1392
|
};
|
|
776
1393
|
|
|
1394
|
+
// src/cache/single-flight.ts
|
|
1395
|
+
function singleFlight(cache) {
|
|
1396
|
+
const pending = /* @__PURE__ */ new Map();
|
|
1397
|
+
return {
|
|
1398
|
+
load(key, fetcher, signal) {
|
|
1399
|
+
if (signal?.aborted) return Promise.reject(makeAbortError());
|
|
1400
|
+
const hit = cache.get(key);
|
|
1401
|
+
if (hit !== void 0) return Promise.resolve(hit);
|
|
1402
|
+
let entry = pending.get(key);
|
|
1403
|
+
if (!entry) {
|
|
1404
|
+
const controller = new AbortController();
|
|
1405
|
+
const promise = invokeFetcher(fetcher, controller.signal).then((value) => {
|
|
1406
|
+
cache.set(key, value);
|
|
1407
|
+
return value;
|
|
1408
|
+
}).finally(() => {
|
|
1409
|
+
if (pending.get(key)?.promise === promise) pending.delete(key);
|
|
1410
|
+
});
|
|
1411
|
+
const fresh = { promise, controller, refCount: 0 };
|
|
1412
|
+
pending.set(key, fresh);
|
|
1413
|
+
entry = fresh;
|
|
1414
|
+
}
|
|
1415
|
+
const owned = entry;
|
|
1416
|
+
owned.refCount++;
|
|
1417
|
+
const release = () => {
|
|
1418
|
+
owned.refCount--;
|
|
1419
|
+
if (owned.refCount > 0) return;
|
|
1420
|
+
if (pending.get(key) === owned) {
|
|
1421
|
+
pending.delete(key);
|
|
1422
|
+
owned.controller.abort();
|
|
1423
|
+
}
|
|
1424
|
+
};
|
|
1425
|
+
if (!signal) {
|
|
1426
|
+
return owned.promise.then(
|
|
1427
|
+
(value) => {
|
|
1428
|
+
release();
|
|
1429
|
+
return value;
|
|
1430
|
+
},
|
|
1431
|
+
(error) => {
|
|
1432
|
+
release();
|
|
1433
|
+
throw error;
|
|
1434
|
+
}
|
|
1435
|
+
);
|
|
1436
|
+
}
|
|
1437
|
+
return new Promise((resolve, reject) => {
|
|
1438
|
+
let done = false;
|
|
1439
|
+
const finish = () => {
|
|
1440
|
+
if (done) return false;
|
|
1441
|
+
done = true;
|
|
1442
|
+
signal.removeEventListener("abort", handleAbort);
|
|
1443
|
+
release();
|
|
1444
|
+
return true;
|
|
1445
|
+
};
|
|
1446
|
+
const handleAbort = () => {
|
|
1447
|
+
if (finish()) reject(makeAbortError());
|
|
1448
|
+
};
|
|
1449
|
+
signal.addEventListener("abort", handleAbort, { once: true });
|
|
1450
|
+
owned.promise.then(
|
|
1451
|
+
(value) => {
|
|
1452
|
+
if (finish()) resolve(value);
|
|
1453
|
+
},
|
|
1454
|
+
(error) => {
|
|
1455
|
+
if (finish()) reject(error);
|
|
1456
|
+
}
|
|
1457
|
+
);
|
|
1458
|
+
});
|
|
1459
|
+
}
|
|
1460
|
+
};
|
|
1461
|
+
}
|
|
1462
|
+
function invokeFetcher(fetcher, signal) {
|
|
1463
|
+
try {
|
|
1464
|
+
return fetcher(signal);
|
|
1465
|
+
} catch (error) {
|
|
1466
|
+
return Promise.reject(error);
|
|
1467
|
+
}
|
|
1468
|
+
}
|
|
1469
|
+
function makeAbortError() {
|
|
1470
|
+
return new DOMException("The operation was aborted.", "AbortError");
|
|
1471
|
+
}
|
|
1472
|
+
|
|
777
1473
|
// src/format/flatbuffers/types.ts
|
|
778
1474
|
function asObjectId12(bytes) {
|
|
779
1475
|
if (bytes.length !== 12) {
|
|
@@ -1624,6 +2320,60 @@ var Manifest = class _Manifest {
|
|
|
1624
2320
|
};
|
|
1625
2321
|
|
|
1626
2322
|
// src/format/flatbuffers/manifest-parser.ts
|
|
2323
|
+
var COMPRESSION_ALG_ZSTD_DICT = 1;
|
|
2324
|
+
var MAX_DECOMPRESSED_LOCATION_SIZE = 1024;
|
|
2325
|
+
function zstdFrameAllocSize(frame) {
|
|
2326
|
+
if (frame.length < 5 || frame[0] !== 40 || frame[1] !== 181 || frame[2] !== 47 || frame[3] !== 253) {
|
|
2327
|
+
return Infinity;
|
|
2328
|
+
}
|
|
2329
|
+
const flg = frame[4];
|
|
2330
|
+
const singleSegment = flg >> 5 & 1;
|
|
2331
|
+
const contentChecksum = flg >> 2 & 1;
|
|
2332
|
+
const dictIdFlag = flg & 3;
|
|
2333
|
+
const contentSizeFlag = flg >> 6;
|
|
2334
|
+
let pos = 5;
|
|
2335
|
+
let windowSize = 0;
|
|
2336
|
+
if (!singleSegment) {
|
|
2337
|
+
if (pos >= frame.length) return Infinity;
|
|
2338
|
+
const wd = frame[pos++];
|
|
2339
|
+
const base = 2 ** (10 + (wd >> 3));
|
|
2340
|
+
windowSize = base + base / 8 * (wd & 7);
|
|
2341
|
+
}
|
|
2342
|
+
pos += dictIdFlag === 3 ? 4 : dictIdFlag;
|
|
2343
|
+
const fcsBytes = contentSizeFlag ? 1 << contentSizeFlag : singleSegment;
|
|
2344
|
+
let contentSize = 0;
|
|
2345
|
+
if (fcsBytes) {
|
|
2346
|
+
if (pos + fcsBytes > frame.length) return Infinity;
|
|
2347
|
+
for (let i = 0; i < fcsBytes; i++) {
|
|
2348
|
+
contentSize += frame[pos + i] * 2 ** (8 * i);
|
|
2349
|
+
}
|
|
2350
|
+
if (contentSizeFlag === 1) contentSize += 256;
|
|
2351
|
+
if (singleSegment) windowSize = contentSize;
|
|
2352
|
+
}
|
|
2353
|
+
pos += fcsBytes;
|
|
2354
|
+
let blockOutput = 0;
|
|
2355
|
+
for (; ; ) {
|
|
2356
|
+
if (pos + 3 > frame.length) return Infinity;
|
|
2357
|
+
const blockHeader = frame[pos] | frame[pos + 1] << 8 | frame[pos + 2] << 16;
|
|
2358
|
+
pos += 3;
|
|
2359
|
+
const lastBlock = blockHeader & 1;
|
|
2360
|
+
const blockType = blockHeader >> 1 & 3;
|
|
2361
|
+
const blockSize = blockHeader >> 3;
|
|
2362
|
+
if (blockType === 3) return Infinity;
|
|
2363
|
+
if (blockType === 1) {
|
|
2364
|
+
blockOutput += blockSize;
|
|
2365
|
+
pos += 1;
|
|
2366
|
+
} else {
|
|
2367
|
+
blockOutput += blockType === 0 ? blockSize : Math.min(windowSize, 131072);
|
|
2368
|
+
pos += blockSize;
|
|
2369
|
+
}
|
|
2370
|
+
if (pos > frame.length) return Infinity;
|
|
2371
|
+
if (lastBlock) break;
|
|
2372
|
+
}
|
|
2373
|
+
if (contentChecksum) pos += 4;
|
|
2374
|
+
if (pos !== frame.length) return Infinity;
|
|
2375
|
+
return Math.max(contentSize || windowSize, blockOutput);
|
|
2376
|
+
}
|
|
1627
2377
|
function parseManifest(data) {
|
|
1628
2378
|
const bb = new ByteBuffer3(data);
|
|
1629
2379
|
const fbsManifest = Manifest.getRootAsManifest(bb);
|
|
@@ -1632,17 +2382,41 @@ function parseManifest(data) {
|
|
|
1632
2382
|
const id = asObjectId12(
|
|
1633
2383
|
idObj.bb.bytes().slice(idObj.bb_pos, idObj.bb_pos + 12)
|
|
1634
2384
|
);
|
|
2385
|
+
const decodeLocation = makeLocationDecoder(fbsManifest);
|
|
1635
2386
|
const arraysLength = fbsManifest.arraysLength();
|
|
1636
2387
|
const arrays = [];
|
|
1637
2388
|
for (let i = 0; i < arraysLength; i++) {
|
|
1638
2389
|
const fbsArray = fbsManifest.arrays(i);
|
|
1639
2390
|
if (fbsArray) {
|
|
1640
|
-
arrays.push(parseArrayManifest(fbsArray));
|
|
2391
|
+
arrays.push(parseArrayManifest(fbsArray, decodeLocation));
|
|
1641
2392
|
}
|
|
1642
2393
|
}
|
|
1643
2394
|
return { id, arrays };
|
|
1644
2395
|
}
|
|
1645
|
-
function
|
|
2396
|
+
function makeLocationDecoder(fbsManifest) {
|
|
2397
|
+
const dictionary = fbsManifest.compressionAlgorithm() === COMPRESSION_ALG_ZSTD_DICT ? fbsManifest.locationDictionaryArray() : null;
|
|
2398
|
+
const textDecoder = new TextDecoder("utf-8", { fatal: true });
|
|
2399
|
+
return (compressed) => {
|
|
2400
|
+
if (!dictionary) {
|
|
2401
|
+
throw new Error(
|
|
2402
|
+
"ChunkRef has a compressed_location but the manifest has no location dictionary"
|
|
2403
|
+
);
|
|
2404
|
+
}
|
|
2405
|
+
if (zstdFrameAllocSize(compressed) > MAX_DECOMPRESSED_LOCATION_SIZE) {
|
|
2406
|
+
throw new Error(
|
|
2407
|
+
`Compressed location declares a size over ${MAX_DECOMPRESSED_LOCATION_SIZE} bytes`
|
|
2408
|
+
);
|
|
2409
|
+
}
|
|
2410
|
+
const decompressed = decompress(compressed, void 0, dictionary);
|
|
2411
|
+
if (decompressed.length > MAX_DECOMPRESSED_LOCATION_SIZE) {
|
|
2412
|
+
throw new Error(
|
|
2413
|
+
`Decompressed location exceeds ${MAX_DECOMPRESSED_LOCATION_SIZE} bytes`
|
|
2414
|
+
);
|
|
2415
|
+
}
|
|
2416
|
+
return textDecoder.decode(decompressed);
|
|
2417
|
+
};
|
|
2418
|
+
}
|
|
2419
|
+
function parseArrayManifest(fbsArray, decodeLocation) {
|
|
1646
2420
|
const nodeIdObj = fbsArray.nodeId();
|
|
1647
2421
|
if (!nodeIdObj)
|
|
1648
2422
|
throw new Error("ArrayManifest missing required node_id field");
|
|
@@ -1654,12 +2428,12 @@ function parseArrayManifest(fbsArray) {
|
|
|
1654
2428
|
for (let i = 0; i < refsLength; i++) {
|
|
1655
2429
|
const fbsRef = fbsArray.refs(i);
|
|
1656
2430
|
if (fbsRef) {
|
|
1657
|
-
refs.push(parseChunkRef(fbsRef));
|
|
2431
|
+
refs.push(parseChunkRef(fbsRef, decodeLocation));
|
|
1658
2432
|
}
|
|
1659
2433
|
}
|
|
1660
2434
|
return { nodeId, refs };
|
|
1661
2435
|
}
|
|
1662
|
-
function parseChunkRef(fbsRef) {
|
|
2436
|
+
function parseChunkRef(fbsRef, decodeLocation) {
|
|
1663
2437
|
const indexLength = fbsRef.indexLength();
|
|
1664
2438
|
const index = [];
|
|
1665
2439
|
for (let i = 0; i < indexLength; i++) {
|
|
@@ -1673,19 +2447,37 @@ function parseChunkRef(fbsRef) {
|
|
|
1673
2447
|
const chunkId = chunkIdObj ? asObjectId12(
|
|
1674
2448
|
chunkIdObj.bb.bytes().slice(chunkIdObj.bb_pos, chunkIdObj.bb_pos + 12)
|
|
1675
2449
|
) : null;
|
|
1676
|
-
const location = fbsRef.location();
|
|
1677
2450
|
const checksumEtag = fbsRef.checksumEtag();
|
|
1678
2451
|
const checksumLastModified = fbsRef.checksumLastModified();
|
|
1679
|
-
|
|
2452
|
+
const ref = {
|
|
1680
2453
|
index,
|
|
1681
2454
|
inline,
|
|
1682
2455
|
offset,
|
|
1683
2456
|
length,
|
|
1684
2457
|
chunkId,
|
|
1685
|
-
location,
|
|
2458
|
+
location: fbsRef.location(),
|
|
1686
2459
|
checksumEtag,
|
|
1687
2460
|
checksumLastModified
|
|
1688
2461
|
};
|
|
2462
|
+
if (chunkId === null) {
|
|
2463
|
+
const compressed = fbsRef.compressedLocationArray();
|
|
2464
|
+
if (compressed) {
|
|
2465
|
+
let decoded = null;
|
|
2466
|
+
let resolved = false;
|
|
2467
|
+
Object.defineProperty(ref, "location", {
|
|
2468
|
+
configurable: true,
|
|
2469
|
+
enumerable: true,
|
|
2470
|
+
get() {
|
|
2471
|
+
if (!resolved) {
|
|
2472
|
+
decoded = decodeLocation(compressed);
|
|
2473
|
+
resolved = true;
|
|
2474
|
+
}
|
|
2475
|
+
return decoded;
|
|
2476
|
+
}
|
|
2477
|
+
});
|
|
2478
|
+
}
|
|
2479
|
+
}
|
|
2480
|
+
return ref;
|
|
1689
2481
|
}
|
|
1690
2482
|
function findChunkRef(manifest, nodeId, coords) {
|
|
1691
2483
|
const arrayManifest = binarySearchArrayManifest(manifest.arrays, nodeId);
|
|
@@ -1739,9 +2531,6 @@ function binarySearchChunkRef(refs, coords) {
|
|
|
1739
2531
|
return null;
|
|
1740
2532
|
}
|
|
1741
2533
|
function getChunkPayload(ref) {
|
|
1742
|
-
if (ref.inline !== null) {
|
|
1743
|
-
return { type: "inline", data: ref.inline };
|
|
1744
|
-
}
|
|
1745
2534
|
if (ref.chunkId !== null) {
|
|
1746
2535
|
return {
|
|
1747
2536
|
type: "native",
|
|
@@ -1760,16 +2549,19 @@ function getChunkPayload(ref) {
|
|
|
1760
2549
|
checksumLastModified: ref.checksumLastModified
|
|
1761
2550
|
};
|
|
1762
2551
|
}
|
|
1763
|
-
|
|
2552
|
+
if (ref.inline !== null) {
|
|
2553
|
+
return { type: "inline", data: ref.inline };
|
|
2554
|
+
}
|
|
2555
|
+
throw new Error("Invalid ChunkRef: no chunkId, location, or inline");
|
|
1764
2556
|
}
|
|
1765
2557
|
|
|
1766
2558
|
// src/format/flatbuffers/metadata.ts
|
|
1767
2559
|
import { decode } from "@msgpack/msgpack";
|
|
1768
|
-
import * as
|
|
2560
|
+
import * as flexbuffers2 from "flatbuffers/js/flexbuffers.js";
|
|
1769
2561
|
function deserializeMetadata(items, specVersion) {
|
|
1770
2562
|
const result = /* @__PURE__ */ Object.create(null);
|
|
1771
2563
|
for (const item of items) {
|
|
1772
|
-
result[item.name] = specVersion === 1 /* V1_0 */ ? decode(item.value) :
|
|
2564
|
+
result[item.name] = specVersion === 1 /* V1_0 */ ? decode(item.value) : flexbuffers2.toObject(
|
|
1773
2565
|
item.value.buffer.slice(
|
|
1774
2566
|
item.value.byteOffset,
|
|
1775
2567
|
item.value.byteOffset + item.value.byteLength
|
|
@@ -2101,17 +2893,219 @@ function parseTransactionLog(data) {
|
|
|
2101
2893
|
};
|
|
2102
2894
|
}
|
|
2103
2895
|
|
|
2896
|
+
// src/reader/range-coalescer.ts
|
|
2897
|
+
function expectedRangeLength(range) {
|
|
2898
|
+
return "suffixLength" in range ? range.suffixLength : range.length;
|
|
2899
|
+
}
|
|
2900
|
+
function makeUrlStore(opts) {
|
|
2901
|
+
const { url, fetchClient, conditionalHeaders } = opts;
|
|
2902
|
+
async function doFetch(init) {
|
|
2903
|
+
return fetchClient ? await fetchClient.fetch(url, init) : await fetch(url, init);
|
|
2904
|
+
}
|
|
2905
|
+
return {
|
|
2906
|
+
async get() {
|
|
2907
|
+
throw new Error(
|
|
2908
|
+
`Virtual chunk URL store for ${url} only supports ranged reads`
|
|
2909
|
+
);
|
|
2910
|
+
},
|
|
2911
|
+
async getRange(_key, range, options) {
|
|
2912
|
+
const headers = conditionalHeaders ? { ...conditionalHeaders } : {};
|
|
2913
|
+
headers.Range = "suffixLength" in range ? `bytes=-${range.suffixLength}` : `bytes=${range.offset}-${range.offset + range.length - 1}`;
|
|
2914
|
+
const response = await doFetch({ headers, signal: options?.signal });
|
|
2915
|
+
if (response.status === 412) {
|
|
2916
|
+
throw new Error(
|
|
2917
|
+
`Virtual chunk at ${url} failed integrity check \u2014 data has been modified since snapshot was created`
|
|
2918
|
+
);
|
|
2919
|
+
}
|
|
2920
|
+
if (response.status !== 200 && response.status !== 206) {
|
|
2921
|
+
throw new Error(
|
|
2922
|
+
`Failed to fetch virtual chunk from ${url}: ${response.status} ${response.statusText}`
|
|
2923
|
+
);
|
|
2924
|
+
}
|
|
2925
|
+
const data = new Uint8Array(await response.arrayBuffer());
|
|
2926
|
+
if (response.status === 206) {
|
|
2927
|
+
const expected = expectedRangeLength(range);
|
|
2928
|
+
if (data.length === expected) return data;
|
|
2929
|
+
throw new Error(
|
|
2930
|
+
`Virtual range response size mismatch for ${url}: expected ${expected} bytes, got ${data.length}`
|
|
2931
|
+
);
|
|
2932
|
+
}
|
|
2933
|
+
if ("offset" in range) {
|
|
2934
|
+
const end = range.offset + range.length;
|
|
2935
|
+
if (data.length >= end) return data.slice(range.offset, end);
|
|
2936
|
+
throw new Error(
|
|
2937
|
+
`Virtual range request not honored for ${url}: need at least ${end} bytes for fallback slicing, got ${data.length}`
|
|
2938
|
+
);
|
|
2939
|
+
}
|
|
2940
|
+
if (data.length >= range.suffixLength) {
|
|
2941
|
+
return data.slice(data.length - range.suffixLength);
|
|
2942
|
+
}
|
|
2943
|
+
throw new Error(
|
|
2944
|
+
`Virtual suffix range request not honored for ${url}: need at least ${range.suffixLength} bytes for fallback slicing, got ${data.length}`
|
|
2945
|
+
);
|
|
2946
|
+
}
|
|
2947
|
+
};
|
|
2948
|
+
}
|
|
2949
|
+
function makeStorageStore(storage) {
|
|
2950
|
+
return {
|
|
2951
|
+
async get(key, options) {
|
|
2952
|
+
const storageOptions = options?.signal ? { signal: options.signal } : void 0;
|
|
2953
|
+
return storage.getObject(key, void 0, storageOptions);
|
|
2954
|
+
},
|
|
2955
|
+
async getRange(key, range, options) {
|
|
2956
|
+
const storageOptions = options?.signal ? { signal: options.signal } : void 0;
|
|
2957
|
+
if ("suffixLength" in range) {
|
|
2958
|
+
throw new Error(
|
|
2959
|
+
`Storage suffix ranges are not supported for ${key}; convert suffixLength to offset/length before reading`
|
|
2960
|
+
);
|
|
2961
|
+
}
|
|
2962
|
+
const storageRange = {
|
|
2963
|
+
start: range.offset,
|
|
2964
|
+
end: range.offset + range.length
|
|
2965
|
+
};
|
|
2966
|
+
const data = await storage.getObject(key, storageRange, storageOptions);
|
|
2967
|
+
if (data.length === range.length) return data;
|
|
2968
|
+
if (data.length >= storageRange.end) {
|
|
2969
|
+
return data.slice(storageRange.start, storageRange.end);
|
|
2970
|
+
}
|
|
2971
|
+
throw new Error(
|
|
2972
|
+
`Storage returned ${data.length} bytes for ${key} range ${storageRange.start}-${storageRange.end - 1}; expected ${range.length} bytes`
|
|
2973
|
+
);
|
|
2974
|
+
}
|
|
2975
|
+
};
|
|
2976
|
+
}
|
|
2977
|
+
|
|
2104
2978
|
// src/reader/session.ts
|
|
2979
|
+
var RANGE_COALESCE_SIZE = 32 * 1024;
|
|
2980
|
+
var RANGE_STORE_CACHE_SIZE = 256;
|
|
2981
|
+
function makeRangeStoreCacheKey(parts) {
|
|
2982
|
+
return JSON.stringify(parts);
|
|
2983
|
+
}
|
|
2105
2984
|
var ReadSession = class _ReadSession {
|
|
2106
2985
|
storage;
|
|
2107
2986
|
snapshot;
|
|
2108
2987
|
specVersion;
|
|
2109
2988
|
manifestCache;
|
|
2110
|
-
|
|
2989
|
+
/**
|
|
2990
|
+
* Single-flight loader over `manifestCache`: concurrent misses for the
|
|
2991
|
+
* same manifest share one fetch instead of racing N identical GETs.
|
|
2992
|
+
* Critical for fan-out reads that all need the same manifest (e.g. the
|
|
2993
|
+
* many parallel chunk reads issued when a renderer crosses into a new
|
|
2994
|
+
* pyramid level).
|
|
2995
|
+
*/
|
|
2996
|
+
manifestLoader;
|
|
2997
|
+
/**
|
|
2998
|
+
* Bounded cache of `AsyncReadable`s wrapped with zarrita's range coalescer.
|
|
2999
|
+
* The cache key includes request-option identities that must not share one
|
|
3000
|
+
* coalescing queue, notably virtual fetch clients and checksum headers.
|
|
3001
|
+
*
|
|
3002
|
+
* Promise slots are inserted synchronously on first use, so
|
|
3003
|
+
* concurrent requests for the same partition share one coalescing window
|
|
3004
|
+
* instead of racing to create parallel stores.
|
|
3005
|
+
*/
|
|
3006
|
+
rangeStores;
|
|
3007
|
+
nativeStore;
|
|
3008
|
+
fetchClientIds;
|
|
3009
|
+
nextFetchClientId = 1;
|
|
3010
|
+
rangeCoalescerIds;
|
|
3011
|
+
nextRangeCoalescerId = 1;
|
|
3012
|
+
/** VCC name → url_prefix map for resolving `vcc://` chunk locations. */
|
|
3013
|
+
virtualChunkContainers;
|
|
3014
|
+
constructor(storage, snapshot, specVersion, maxManifestCacheSize = 100, virtualChunkContainers) {
|
|
2111
3015
|
this.storage = storage;
|
|
2112
3016
|
this.snapshot = snapshot;
|
|
2113
3017
|
this.specVersion = specVersion;
|
|
2114
3018
|
this.manifestCache = new LRUCache(maxManifestCacheSize);
|
|
3019
|
+
this.manifestLoader = singleFlight(this.manifestCache);
|
|
3020
|
+
this.virtualChunkContainers = virtualChunkContainers ?? /* @__PURE__ */ new Map();
|
|
3021
|
+
}
|
|
3022
|
+
getFetchClientKey(fetchClient) {
|
|
3023
|
+
if (!fetchClient) return "default";
|
|
3024
|
+
if (!this.fetchClientIds) this.fetchClientIds = /* @__PURE__ */ new WeakMap();
|
|
3025
|
+
let id = this.fetchClientIds.get(fetchClient);
|
|
3026
|
+
if (id === void 0) {
|
|
3027
|
+
id = this.nextFetchClientId;
|
|
3028
|
+
this.nextFetchClientId = id + 1;
|
|
3029
|
+
this.fetchClientIds.set(fetchClient, id);
|
|
3030
|
+
}
|
|
3031
|
+
return String(id);
|
|
3032
|
+
}
|
|
3033
|
+
getRangeCoalescerKey(withRangeCoalescing) {
|
|
3034
|
+
if (!this.rangeCoalescerIds) this.rangeCoalescerIds = /* @__PURE__ */ new WeakMap();
|
|
3035
|
+
let id = this.rangeCoalescerIds.get(withRangeCoalescing);
|
|
3036
|
+
if (id === void 0) {
|
|
3037
|
+
id = this.nextRangeCoalescerId;
|
|
3038
|
+
this.nextRangeCoalescerId = id + 1;
|
|
3039
|
+
this.rangeCoalescerIds.set(withRangeCoalescing, id);
|
|
3040
|
+
}
|
|
3041
|
+
return String(id);
|
|
3042
|
+
}
|
|
3043
|
+
getRangeStore(partitionKey, createStore, withRangeCoalescing) {
|
|
3044
|
+
if (!this.rangeStores) {
|
|
3045
|
+
this.rangeStores = new LRUCache(RANGE_STORE_CACHE_SIZE);
|
|
3046
|
+
}
|
|
3047
|
+
const stores = this.rangeStores;
|
|
3048
|
+
const cacheKey = makeRangeStoreCacheKey([
|
|
3049
|
+
...partitionKey,
|
|
3050
|
+
["coalescer", this.getRangeCoalescerKey(withRangeCoalescing)]
|
|
3051
|
+
]);
|
|
3052
|
+
const cached = stores.get(cacheKey);
|
|
3053
|
+
if (cached) return cached;
|
|
3054
|
+
const raw = createStore();
|
|
3055
|
+
const promise = Promise.resolve().then(
|
|
3056
|
+
() => withRangeCoalescing(raw, { coalesceSize: RANGE_COALESCE_SIZE })
|
|
3057
|
+
).catch((error) => {
|
|
3058
|
+
if (stores.get(cacheKey) === promise) stores.delete(cacheKey);
|
|
3059
|
+
throw error;
|
|
3060
|
+
});
|
|
3061
|
+
stores.set(cacheKey, promise);
|
|
3062
|
+
return promise;
|
|
3063
|
+
}
|
|
3064
|
+
getNativeStore(options) {
|
|
3065
|
+
if (!this.nativeStore) {
|
|
3066
|
+
this.nativeStore = makeStorageStore(this.storage);
|
|
3067
|
+
}
|
|
3068
|
+
const raw = this.nativeStore;
|
|
3069
|
+
const withRangeCoalescing = options?.withRangeCoalescing;
|
|
3070
|
+
if (!withRangeCoalescing) {
|
|
3071
|
+
return Promise.resolve(raw);
|
|
3072
|
+
}
|
|
3073
|
+
return this.getRangeStore(["native"], () => raw, withRangeCoalescing);
|
|
3074
|
+
}
|
|
3075
|
+
getVirtualStoreForPayload(httpUrl, payload, options) {
|
|
3076
|
+
const validate = !!options?.validateChecksums;
|
|
3077
|
+
let conditionalHeaders;
|
|
3078
|
+
if (validate) {
|
|
3079
|
+
conditionalHeaders = {};
|
|
3080
|
+
if (payload.checksumEtag) {
|
|
3081
|
+
conditionalHeaders["If-Match"] = payload.checksumEtag;
|
|
3082
|
+
}
|
|
3083
|
+
if (payload.checksumLastModified > 0) {
|
|
3084
|
+
conditionalHeaders["If-Unmodified-Since"] = new Date(
|
|
3085
|
+
payload.checksumLastModified * 1e3
|
|
3086
|
+
).toUTCString();
|
|
3087
|
+
}
|
|
3088
|
+
}
|
|
3089
|
+
const checksumKey = validate ? ["checked", payload.checksumEtag ?? "", payload.checksumLastModified] : ["unchecked"];
|
|
3090
|
+
const createStore = () => makeUrlStore({
|
|
3091
|
+
url: httpUrl,
|
|
3092
|
+
fetchClient: options?.fetchClient,
|
|
3093
|
+
conditionalHeaders
|
|
3094
|
+
});
|
|
3095
|
+
const withRangeCoalescing = options?.withRangeCoalescing;
|
|
3096
|
+
if (!withRangeCoalescing) {
|
|
3097
|
+
return Promise.resolve(createStore());
|
|
3098
|
+
}
|
|
3099
|
+
return this.getRangeStore(
|
|
3100
|
+
[
|
|
3101
|
+
"virtual",
|
|
3102
|
+
httpUrl,
|
|
3103
|
+
["fetch", this.getFetchClientKey(options?.fetchClient)],
|
|
3104
|
+
checksumKey
|
|
3105
|
+
],
|
|
3106
|
+
createStore,
|
|
3107
|
+
withRangeCoalescing
|
|
3108
|
+
);
|
|
2115
3109
|
}
|
|
2116
3110
|
/**
|
|
2117
3111
|
* Open a read session for a specific snapshot.
|
|
@@ -2131,7 +3125,8 @@ var ReadSession = class _ReadSession {
|
|
|
2131
3125
|
storage,
|
|
2132
3126
|
snapshot,
|
|
2133
3127
|
specVersion,
|
|
2134
|
-
options?.maxManifestCacheSize
|
|
3128
|
+
options?.maxManifestCacheSize,
|
|
3129
|
+
options?.virtualChunkContainers
|
|
2135
3130
|
);
|
|
2136
3131
|
}
|
|
2137
3132
|
/** Load and parse a snapshot from storage */
|
|
@@ -2142,29 +3137,47 @@ var ReadSession = class _ReadSession {
|
|
|
2142
3137
|
validateFileType(header, 1 /* Snapshot */);
|
|
2143
3138
|
let flatbufferData = getDataAfterHeader(data);
|
|
2144
3139
|
if (header.compression === 1 /* Zstd */) {
|
|
2145
|
-
flatbufferData =
|
|
3140
|
+
flatbufferData = decompress(flatbufferData);
|
|
2146
3141
|
}
|
|
2147
3142
|
return {
|
|
2148
3143
|
snapshot: parseSnapshot(flatbufferData),
|
|
2149
3144
|
specVersion: header.specVersion
|
|
2150
3145
|
};
|
|
2151
3146
|
}
|
|
2152
|
-
/**
|
|
2153
|
-
|
|
3147
|
+
/**
|
|
3148
|
+
* Load and parse a manifest from storage.
|
|
3149
|
+
*
|
|
3150
|
+
* Concurrent calls for the same manifest are coalesced through
|
|
3151
|
+
* `manifestLoader` so a fan-out of chunk reads — the typical case
|
|
3152
|
+
* when many tiles cross into a new pyramid level at once — issues one
|
|
3153
|
+
* HTTP GET instead of one per caller. A caller's `signal` rejects only
|
|
3154
|
+
* that caller's await; the shared fetch is aborted only once every
|
|
3155
|
+
* active waiter has aborted.
|
|
3156
|
+
*/
|
|
3157
|
+
loadManifest(manifestId, options) {
|
|
2154
3158
|
const idStr = encodeObjectId12(manifestId);
|
|
2155
|
-
|
|
2156
|
-
|
|
3159
|
+
return this.manifestLoader.load(
|
|
3160
|
+
idStr,
|
|
3161
|
+
(signal) => this.fetchManifest(idStr, signal),
|
|
3162
|
+
options?.signal
|
|
3163
|
+
);
|
|
3164
|
+
}
|
|
3165
|
+
/**
|
|
3166
|
+
* Fetch and parse a single manifest. The `signal` here is the
|
|
3167
|
+
* single-flight loader's own signal — it fires only when every waiter
|
|
3168
|
+
* has aborted, so a hung manifest fetch doesn't trap later callers
|
|
3169
|
+
* behind it.
|
|
3170
|
+
*/
|
|
3171
|
+
async fetchManifest(idStr, signal) {
|
|
2157
3172
|
const path = getManifestPath(idStr);
|
|
2158
|
-
const data = await this.storage.getObject(path, void 0,
|
|
3173
|
+
const data = await this.storage.getObject(path, void 0, { signal });
|
|
2159
3174
|
const header = parseHeader(data);
|
|
2160
3175
|
validateFileType(header, 2 /* Manifest */);
|
|
2161
3176
|
let flatbufferData = getDataAfterHeader(data);
|
|
2162
3177
|
if (header.compression === 1 /* Zstd */) {
|
|
2163
|
-
flatbufferData =
|
|
3178
|
+
flatbufferData = decompress(flatbufferData);
|
|
2164
3179
|
}
|
|
2165
|
-
|
|
2166
|
-
this.manifestCache.set(idStr, manifest);
|
|
2167
|
-
return manifest;
|
|
3180
|
+
return parseManifest(flatbufferData);
|
|
2168
3181
|
}
|
|
2169
3182
|
/**
|
|
2170
3183
|
* Get the snapshot ID.
|
|
@@ -2226,7 +3239,7 @@ var ReadSession = class _ReadSession {
|
|
|
2226
3239
|
validateFileType(header, 4 /* TransactionLog */);
|
|
2227
3240
|
let flatbufferData = getDataAfterHeader(data);
|
|
2228
3241
|
if (header.compression === 1 /* Zstd */) {
|
|
2229
|
-
flatbufferData =
|
|
3242
|
+
flatbufferData = decompress(flatbufferData);
|
|
2230
3243
|
}
|
|
2231
3244
|
return parseTransactionLog(flatbufferData);
|
|
2232
3245
|
}
|
|
@@ -2295,26 +3308,25 @@ var ReadSession = class _ReadSession {
|
|
|
2295
3308
|
* @returns Chunk data bytes or null if not found
|
|
2296
3309
|
*/
|
|
2297
3310
|
async getChunk(path, coords, options) {
|
|
2298
|
-
|
|
3311
|
+
options?.signal?.throwIfAborted();
|
|
3312
|
+
const requestOptions = toRequestOptions(options);
|
|
2299
3313
|
const node = this.getNode(path);
|
|
2300
3314
|
if (!node || node.nodeData.type !== "array") {
|
|
2301
3315
|
return null;
|
|
2302
3316
|
}
|
|
2303
3317
|
const arrayData = node.nodeData;
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
continue;
|
|
2308
|
-
}
|
|
2309
|
-
const manifest = await this.loadManifest(manifestRef.objectId, options);
|
|
2310
|
-
const chunkRef = findChunkRef(manifest, node.id, coords);
|
|
2311
|
-
if (!chunkRef) continue;
|
|
2312
|
-
const payload = getChunkPayload(chunkRef);
|
|
2313
|
-
return this.fetchChunkPayload(payload, options);
|
|
3318
|
+
for (const manifestRef of arrayData.manifests) {
|
|
3319
|
+
if (!this.coordsInExtents(coords, manifestRef.extents)) {
|
|
3320
|
+
continue;
|
|
2314
3321
|
}
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
3322
|
+
const manifest = await this.loadManifest(
|
|
3323
|
+
manifestRef.objectId,
|
|
3324
|
+
requestOptions
|
|
3325
|
+
);
|
|
3326
|
+
const chunkRef = findChunkRef(manifest, node.id, coords);
|
|
3327
|
+
if (!chunkRef) continue;
|
|
3328
|
+
const payload = getChunkPayload(chunkRef);
|
|
3329
|
+
return this.fetchChunkPayload(payload, options);
|
|
2318
3330
|
}
|
|
2319
3331
|
return null;
|
|
2320
3332
|
}
|
|
@@ -2332,26 +3344,25 @@ var ReadSession = class _ReadSession {
|
|
|
2332
3344
|
* @returns Chunk data bytes or null if not found
|
|
2333
3345
|
*/
|
|
2334
3346
|
async getChunkRange(path, coords, range, options) {
|
|
2335
|
-
|
|
3347
|
+
options?.signal?.throwIfAborted();
|
|
3348
|
+
const requestOptions = toRequestOptions(options);
|
|
2336
3349
|
const node = this.getNode(path);
|
|
2337
3350
|
if (!node || node.nodeData.type !== "array") {
|
|
2338
3351
|
return null;
|
|
2339
3352
|
}
|
|
2340
3353
|
const arrayData = node.nodeData;
|
|
2341
|
-
|
|
2342
|
-
|
|
2343
|
-
|
|
2344
|
-
continue;
|
|
2345
|
-
}
|
|
2346
|
-
const manifest = await this.loadManifest(manifestRef.objectId, options);
|
|
2347
|
-
const chunkRef = findChunkRef(manifest, node.id, coords);
|
|
2348
|
-
if (!chunkRef) continue;
|
|
2349
|
-
const payload = getChunkPayload(chunkRef);
|
|
2350
|
-
return this.fetchChunkPayloadRange(payload, range, options);
|
|
3354
|
+
for (const manifestRef of arrayData.manifests) {
|
|
3355
|
+
if (!this.coordsInExtents(coords, manifestRef.extents)) {
|
|
3356
|
+
continue;
|
|
2351
3357
|
}
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
3358
|
+
const manifest = await this.loadManifest(
|
|
3359
|
+
manifestRef.objectId,
|
|
3360
|
+
requestOptions
|
|
3361
|
+
);
|
|
3362
|
+
const chunkRef = findChunkRef(manifest, node.id, coords);
|
|
3363
|
+
if (!chunkRef) continue;
|
|
3364
|
+
const payload = getChunkPayload(chunkRef);
|
|
3365
|
+
return this.fetchChunkPayloadRange(payload, range, options);
|
|
2355
3366
|
}
|
|
2356
3367
|
return null;
|
|
2357
3368
|
}
|
|
@@ -2373,77 +3384,54 @@ var ReadSession = class _ReadSession {
|
|
|
2373
3384
|
return payload.data;
|
|
2374
3385
|
case "native": {
|
|
2375
3386
|
const path = getChunkPath(encodeObjectId12(payload.chunkId));
|
|
2376
|
-
const
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
|
|
2382
|
-
|
|
2383
|
-
|
|
3387
|
+
const requestedStart = payload.offset;
|
|
3388
|
+
const requestedEnd = payload.offset + payload.length;
|
|
3389
|
+
const store = await this.getNativeStore(options);
|
|
3390
|
+
const data = await store.getRange(
|
|
3391
|
+
path,
|
|
3392
|
+
{ offset: requestedStart, length: payload.length },
|
|
3393
|
+
{ signal: options?.signal }
|
|
3394
|
+
);
|
|
3395
|
+
if (!data) {
|
|
3396
|
+
throw new Error(
|
|
3397
|
+
`Failed to fetch native chunk from ${path} range ${requestedStart}-${requestedEnd - 1}: empty response`
|
|
3398
|
+
);
|
|
2384
3399
|
}
|
|
3400
|
+
if (data.length === payload.length) return data;
|
|
2385
3401
|
throw new Error(
|
|
2386
|
-
`Storage returned ${data.length} bytes for ${path} range ${
|
|
3402
|
+
`Storage returned ${data.length} bytes for ${path} range ${requestedStart}-${requestedEnd - 1}; expected ${payload.length} bytes`
|
|
2387
3403
|
);
|
|
2388
3404
|
}
|
|
2389
3405
|
case "virtual": {
|
|
2390
|
-
const
|
|
3406
|
+
const absoluteLocation = expandVccUrl(
|
|
2391
3407
|
payload.location,
|
|
3408
|
+
this.virtualChunkContainers
|
|
3409
|
+
);
|
|
3410
|
+
const httpUrl = translateToHttpUrl(
|
|
3411
|
+
absoluteLocation,
|
|
2392
3412
|
options?.azureAccount
|
|
2393
3413
|
);
|
|
2394
|
-
const
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
}
|
|
2406
|
-
}
|
|
2407
|
-
const fetchInit = {
|
|
2408
|
-
headers,
|
|
2409
|
-
signal: options?.signal
|
|
2410
|
-
};
|
|
2411
|
-
let response;
|
|
2412
|
-
try {
|
|
2413
|
-
const client = options?.fetchClient;
|
|
2414
|
-
response = client ? await client.fetch(httpUrl, fetchInit) : await fetch(httpUrl, fetchInit);
|
|
2415
|
-
} catch (error) {
|
|
2416
|
-
if (error instanceof Error && error.name === "AbortError") {
|
|
2417
|
-
throw new AbortError();
|
|
2418
|
-
}
|
|
2419
|
-
throw error;
|
|
2420
|
-
}
|
|
2421
|
-
if (response.status === 412) {
|
|
3414
|
+
const store = await this.getVirtualStoreForPayload(
|
|
3415
|
+
httpUrl,
|
|
3416
|
+
payload,
|
|
3417
|
+
options
|
|
3418
|
+
);
|
|
3419
|
+
const data = await store.getRange(
|
|
3420
|
+
"/",
|
|
3421
|
+
{ offset: payload.offset, length: payload.length },
|
|
3422
|
+
{ signal: options?.signal }
|
|
3423
|
+
);
|
|
3424
|
+
if (!data) {
|
|
2422
3425
|
throw new Error(
|
|
2423
|
-
`
|
|
3426
|
+
`Failed to fetch virtual chunk from ${httpUrl}: empty response`
|
|
2424
3427
|
);
|
|
2425
3428
|
}
|
|
2426
|
-
if (
|
|
3429
|
+
if (data.length !== payload.length) {
|
|
2427
3430
|
throw new Error(
|
|
2428
|
-
`
|
|
3431
|
+
`Virtual range response size mismatch for ${httpUrl}: expected ${payload.length} bytes, got ${data.length}`
|
|
2429
3432
|
);
|
|
2430
3433
|
}
|
|
2431
|
-
|
|
2432
|
-
if (response.status === 206) {
|
|
2433
|
-
if (data.length !== payload.length) {
|
|
2434
|
-
throw new Error(
|
|
2435
|
-
`Virtual range response size mismatch for ${httpUrl}: expected ${payload.length} bytes, got ${data.length}`
|
|
2436
|
-
);
|
|
2437
|
-
}
|
|
2438
|
-
return data;
|
|
2439
|
-
}
|
|
2440
|
-
const absoluteEnd = payload.offset + payload.length;
|
|
2441
|
-
if (data.length >= absoluteEnd) {
|
|
2442
|
-
return data.slice(payload.offset, absoluteEnd);
|
|
2443
|
-
}
|
|
2444
|
-
throw new Error(
|
|
2445
|
-
`Virtual range request not honored for ${httpUrl}: need at least ${absoluteEnd} bytes for fallback slicing, got ${data.length}`
|
|
2446
|
-
);
|
|
3434
|
+
return data;
|
|
2447
3435
|
}
|
|
2448
3436
|
}
|
|
2449
3437
|
}
|
|
@@ -2463,80 +3451,57 @@ var ReadSession = class _ReadSession {
|
|
|
2463
3451
|
return payload.data.slice(rangeStart, rangeEnd);
|
|
2464
3452
|
case "native": {
|
|
2465
3453
|
const path = getChunkPath(encodeObjectId12(payload.chunkId));
|
|
2466
|
-
const
|
|
2467
|
-
start: payload.offset + rangeStart,
|
|
2468
|
-
end: payload.offset + rangeEnd
|
|
2469
|
-
};
|
|
3454
|
+
const requestedStart = payload.offset + rangeStart;
|
|
2470
3455
|
const expectedSize = rangeEnd - rangeStart;
|
|
2471
|
-
const
|
|
2472
|
-
|
|
2473
|
-
|
|
2474
|
-
|
|
3456
|
+
const requestedEnd = requestedStart + expectedSize;
|
|
3457
|
+
const store = await this.getNativeStore(options);
|
|
3458
|
+
const data = await store.getRange(
|
|
3459
|
+
path,
|
|
3460
|
+
{ offset: requestedStart, length: expectedSize },
|
|
3461
|
+
{ signal: options?.signal }
|
|
3462
|
+
);
|
|
3463
|
+
if (!data) {
|
|
3464
|
+
throw new Error(
|
|
3465
|
+
`Failed to fetch native chunk from ${path} range ${requestedStart}-${requestedEnd - 1}: empty response`
|
|
3466
|
+
);
|
|
2475
3467
|
}
|
|
3468
|
+
if (data.length === expectedSize) return data;
|
|
2476
3469
|
throw new Error(
|
|
2477
|
-
`Storage returned ${data.length} bytes for ${path} range ${
|
|
3470
|
+
`Storage returned ${data.length} bytes for ${path} range ${requestedStart}-${requestedEnd - 1}; expected ${expectedSize} bytes`
|
|
2478
3471
|
);
|
|
2479
3472
|
}
|
|
2480
3473
|
case "virtual": {
|
|
2481
3474
|
const absoluteStart = payload.offset + rangeStart;
|
|
2482
|
-
const
|
|
2483
|
-
const
|
|
3475
|
+
const expectedSize = rangeEnd - rangeStart;
|
|
3476
|
+
const absoluteLocation = expandVccUrl(
|
|
2484
3477
|
payload.location,
|
|
3478
|
+
this.virtualChunkContainers
|
|
3479
|
+
);
|
|
3480
|
+
const httpUrl = translateToHttpUrl(
|
|
3481
|
+
absoluteLocation,
|
|
2485
3482
|
options?.azureAccount
|
|
2486
3483
|
);
|
|
2487
|
-
const
|
|
2488
|
-
|
|
2489
|
-
|
|
2490
|
-
|
|
2491
|
-
|
|
2492
|
-
|
|
2493
|
-
|
|
2494
|
-
|
|
2495
|
-
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
}
|
|
2499
|
-
}
|
|
2500
|
-
const fetchInit = {
|
|
2501
|
-
headers,
|
|
2502
|
-
signal: options?.signal
|
|
2503
|
-
};
|
|
2504
|
-
let response;
|
|
2505
|
-
try {
|
|
2506
|
-
const client = options?.fetchClient;
|
|
2507
|
-
response = client ? await client.fetch(httpUrl, fetchInit) : await fetch(httpUrl, fetchInit);
|
|
2508
|
-
} catch (error) {
|
|
2509
|
-
if (error instanceof Error && error.name === "AbortError") {
|
|
2510
|
-
throw new AbortError();
|
|
2511
|
-
}
|
|
2512
|
-
throw error;
|
|
2513
|
-
}
|
|
2514
|
-
if (response.status === 412) {
|
|
3484
|
+
const store = await this.getVirtualStoreForPayload(
|
|
3485
|
+
httpUrl,
|
|
3486
|
+
payload,
|
|
3487
|
+
options
|
|
3488
|
+
);
|
|
3489
|
+
const data = await store.getRange(
|
|
3490
|
+
"/",
|
|
3491
|
+
{ offset: absoluteStart, length: expectedSize },
|
|
3492
|
+
{ signal: options?.signal }
|
|
3493
|
+
);
|
|
3494
|
+
if (!data) {
|
|
2515
3495
|
throw new Error(
|
|
2516
|
-
`
|
|
3496
|
+
`Failed to fetch virtual chunk from ${httpUrl}: empty response`
|
|
2517
3497
|
);
|
|
2518
3498
|
}
|
|
2519
|
-
if (
|
|
3499
|
+
if (data.length !== expectedSize) {
|
|
2520
3500
|
throw new Error(
|
|
2521
|
-
`
|
|
3501
|
+
`Virtual range response size mismatch for ${httpUrl}: expected ${expectedSize} bytes, got ${data.length}`
|
|
2522
3502
|
);
|
|
2523
3503
|
}
|
|
2524
|
-
|
|
2525
|
-
const data = new Uint8Array(await response.arrayBuffer());
|
|
2526
|
-
if (response.status === 206) {
|
|
2527
|
-
if (data.length !== expectedSize) {
|
|
2528
|
-
throw new Error(
|
|
2529
|
-
`Virtual range response size mismatch for ${httpUrl}: expected ${expectedSize} bytes, got ${data.length}`
|
|
2530
|
-
);
|
|
2531
|
-
}
|
|
2532
|
-
return data;
|
|
2533
|
-
}
|
|
2534
|
-
if (data.length >= absoluteEnd) {
|
|
2535
|
-
return data.slice(absoluteStart, absoluteEnd);
|
|
2536
|
-
}
|
|
2537
|
-
throw new Error(
|
|
2538
|
-
`Virtual range request not honored for ${httpUrl}: need at least ${absoluteEnd} bytes for fallback slicing, got ${data.length}`
|
|
2539
|
-
);
|
|
3504
|
+
return data;
|
|
2540
3505
|
}
|
|
2541
3506
|
}
|
|
2542
3507
|
}
|
|
@@ -2560,6 +3525,19 @@ var ReadSession = class _ReadSession {
|
|
|
2560
3525
|
}
|
|
2561
3526
|
};
|
|
2562
3527
|
var utf8Encoder = new TextEncoder();
|
|
3528
|
+
function toRequestOptions(options) {
|
|
3529
|
+
if (!options) return void 0;
|
|
3530
|
+
return {
|
|
3531
|
+
...options.signal && { signal: options.signal },
|
|
3532
|
+
...options.fetchClient && { fetchClient: options.fetchClient },
|
|
3533
|
+
...options.validateChecksums !== void 0 && {
|
|
3534
|
+
validateChecksums: options.validateChecksums
|
|
3535
|
+
},
|
|
3536
|
+
...options.azureAccount !== void 0 && {
|
|
3537
|
+
azureAccount: options.azureAccount
|
|
3538
|
+
}
|
|
3539
|
+
};
|
|
3540
|
+
}
|
|
2563
3541
|
function compareUtf8Bytes(a, b) {
|
|
2564
3542
|
const bytesA = utf8Encoder.encode(a);
|
|
2565
3543
|
const bytesB = utf8Encoder.encode(b);
|
|
@@ -2571,6 +3549,28 @@ function compareUtf8Bytes(a, b) {
|
|
|
2571
3549
|
}
|
|
2572
3550
|
return bytesA.length - bytesB.length;
|
|
2573
3551
|
}
|
|
3552
|
+
var VCC_SCHEME = "vcc://";
|
|
3553
|
+
function expandVccUrl(location, containers) {
|
|
3554
|
+
if (!location.startsWith(VCC_SCHEME)) return location;
|
|
3555
|
+
const rest = location.slice(VCC_SCHEME.length);
|
|
3556
|
+
const slash = rest.indexOf("/");
|
|
3557
|
+
if (slash === -1) {
|
|
3558
|
+
throw new Error(
|
|
3559
|
+
`Invalid vcc:// URL "${location}": missing "/" after container name`
|
|
3560
|
+
);
|
|
3561
|
+
}
|
|
3562
|
+
const name = rest.slice(0, slash);
|
|
3563
|
+
const relativePath = rest.slice(slash + 1);
|
|
3564
|
+
const urlPrefix = containers.get(name);
|
|
3565
|
+
if (urlPrefix === void 0) {
|
|
3566
|
+
if (containers.size === 0) return location;
|
|
3567
|
+
throw new Error(
|
|
3568
|
+
`Unknown virtual chunk container "${name}" referenced by ${location}`
|
|
3569
|
+
);
|
|
3570
|
+
}
|
|
3571
|
+
const normalizedPrefix = urlPrefix.endsWith("/") ? urlPrefix : `${urlPrefix}/`;
|
|
3572
|
+
return normalizedPrefix + relativePath;
|
|
3573
|
+
}
|
|
2574
3574
|
function translateToHttpUrl(url, azureAccount) {
|
|
2575
3575
|
if (url.startsWith("s3://")) {
|
|
2576
3576
|
const rest = url.slice(5);
|
|
@@ -2646,7 +3646,7 @@ var Repository = class _Repository {
|
|
|
2646
3646
|
* @param options - Optional request options (signal for cancellation)
|
|
2647
3647
|
* @returns ParsedRepo if v2 format, null if v1 format
|
|
2648
3648
|
* @throws Error if repo file exists but fails to parse
|
|
2649
|
-
* @throws AbortError if the operation was aborted
|
|
3649
|
+
* @throws An error named "AbortError" if the operation was aborted
|
|
2650
3650
|
*/
|
|
2651
3651
|
async loadRepoInfo(options) {
|
|
2652
3652
|
if (this.repoInfoAttempted) {
|
|
@@ -2662,7 +3662,7 @@ var Repository = class _Repository {
|
|
|
2662
3662
|
this.repoInfo = parseRepo(data);
|
|
2663
3663
|
return this.repoInfo;
|
|
2664
3664
|
} catch (error) {
|
|
2665
|
-
if (error
|
|
3665
|
+
if (isAbortError(error)) {
|
|
2666
3666
|
this.repoInfoAttempted = false;
|
|
2667
3667
|
throw error;
|
|
2668
3668
|
}
|
|
@@ -2727,7 +3727,7 @@ var Repository = class _Repository {
|
|
|
2727
3727
|
}
|
|
2728
3728
|
}
|
|
2729
3729
|
} catch (error) {
|
|
2730
|
-
if (error
|
|
3730
|
+
if (isAbortError(error)) {
|
|
2731
3731
|
throw error;
|
|
2732
3732
|
}
|
|
2733
3733
|
if (legacyPath) {
|
|
@@ -2760,7 +3760,7 @@ var Repository = class _Repository {
|
|
|
2760
3760
|
}
|
|
2761
3761
|
}
|
|
2762
3762
|
} catch (error) {
|
|
2763
|
-
if (error
|
|
3763
|
+
if (isAbortError(error)) {
|
|
2764
3764
|
throw error;
|
|
2765
3765
|
}
|
|
2766
3766
|
return legacyPath || null;
|
|
@@ -2801,7 +3801,7 @@ var Repository = class _Repository {
|
|
|
2801
3801
|
}
|
|
2802
3802
|
}
|
|
2803
3803
|
} catch (error) {
|
|
2804
|
-
if (error
|
|
3804
|
+
if (isAbortError(error)) {
|
|
2805
3805
|
throw error;
|
|
2806
3806
|
}
|
|
2807
3807
|
throw new Error("Cannot list branches: storage does not support listing");
|
|
@@ -2843,7 +3843,7 @@ var Repository = class _Repository {
|
|
|
2843
3843
|
}
|
|
2844
3844
|
}
|
|
2845
3845
|
} catch (error) {
|
|
2846
|
-
if (error
|
|
3846
|
+
if (isAbortError(error)) {
|
|
2847
3847
|
throw error;
|
|
2848
3848
|
}
|
|
2849
3849
|
throw new Error("Cannot list tags: storage does not support listing");
|
|
@@ -2873,7 +3873,10 @@ var Repository = class _Repository {
|
|
|
2873
3873
|
if (!snapshotId2) {
|
|
2874
3874
|
throw new Error(`Branch not found: ${name}`);
|
|
2875
3875
|
}
|
|
2876
|
-
return ReadSession.open(this.storage, snapshotId2,
|
|
3876
|
+
return ReadSession.open(this.storage, snapshotId2, {
|
|
3877
|
+
...options,
|
|
3878
|
+
virtualChunkContainers: repoInfo.virtualChunkContainers
|
|
3879
|
+
});
|
|
2877
3880
|
}
|
|
2878
3881
|
const refDirPath = getBranchRefDirPath(name);
|
|
2879
3882
|
const refPath = await this.findLatestRefFile(
|
|
@@ -2900,7 +3903,10 @@ var Repository = class _Repository {
|
|
|
2900
3903
|
if (!snapshotId2) {
|
|
2901
3904
|
throw new Error(`Tag not found: ${name}`);
|
|
2902
3905
|
}
|
|
2903
|
-
return ReadSession.open(this.storage, snapshotId2,
|
|
3906
|
+
return ReadSession.open(this.storage, snapshotId2, {
|
|
3907
|
+
...options,
|
|
3908
|
+
virtualChunkContainers: repoInfo.virtualChunkContainers
|
|
3909
|
+
});
|
|
2904
3910
|
}
|
|
2905
3911
|
const refDirPath = getTagRefDirPath(name);
|
|
2906
3912
|
const legacyPath = getTagRefPath(name);
|
|
@@ -2923,7 +3929,12 @@ var Repository = class _Repository {
|
|
|
2923
3929
|
*/
|
|
2924
3930
|
async checkoutSnapshot(snapshotId, options) {
|
|
2925
3931
|
const id = typeof snapshotId === "string" ? decodeObjectId12(snapshotId) : snapshotId;
|
|
2926
|
-
|
|
3932
|
+
const repoInfo = await this.loadRepoInfo(options);
|
|
3933
|
+
const virtualChunkContainers = repoInfo?.virtualChunkContainers;
|
|
3934
|
+
return ReadSession.open(this.storage, id, {
|
|
3935
|
+
...options,
|
|
3936
|
+
...virtualChunkContainers && { virtualChunkContainers }
|
|
3937
|
+
});
|
|
2927
3938
|
}
|
|
2928
3939
|
/**
|
|
2929
3940
|
* Walk the snapshot history chain starting from the given session.
|
|
@@ -3002,9 +4013,7 @@ var HttpStorage = class {
|
|
|
3002
4013
|
return headers;
|
|
3003
4014
|
}
|
|
3004
4015
|
async getObject(path, range, options) {
|
|
3005
|
-
|
|
3006
|
-
throw new AbortError();
|
|
3007
|
-
}
|
|
4016
|
+
options?.signal?.throwIfAborted();
|
|
3008
4017
|
const url = this.getUrl(path);
|
|
3009
4018
|
const headers = this.getHeaders(range);
|
|
3010
4019
|
let response;
|
|
@@ -3017,9 +4026,7 @@ var HttpStorage = class {
|
|
|
3017
4026
|
signal: options?.signal
|
|
3018
4027
|
});
|
|
3019
4028
|
} catch (error) {
|
|
3020
|
-
if (error
|
|
3021
|
-
throw new AbortError();
|
|
3022
|
-
}
|
|
4029
|
+
if (isAbortError(error)) throw error;
|
|
3023
4030
|
throw new StorageError(
|
|
3024
4031
|
`Failed to fetch ${url}: ${error instanceof Error ? error.message : String(error)}`,
|
|
3025
4032
|
error instanceof Error ? error : void 0
|
|
@@ -3037,9 +4044,7 @@ var HttpStorage = class {
|
|
|
3037
4044
|
return new Uint8Array(arrayBuffer);
|
|
3038
4045
|
}
|
|
3039
4046
|
async exists(path, options) {
|
|
3040
|
-
|
|
3041
|
-
throw new AbortError();
|
|
3042
|
-
}
|
|
4047
|
+
options?.signal?.throwIfAborted();
|
|
3043
4048
|
const url = this.getUrl(path);
|
|
3044
4049
|
try {
|
|
3045
4050
|
const response = await fetch(url, {
|
|
@@ -3050,9 +4055,7 @@ var HttpStorage = class {
|
|
|
3050
4055
|
});
|
|
3051
4056
|
return response.ok;
|
|
3052
4057
|
} catch (error) {
|
|
3053
|
-
if (error
|
|
3054
|
-
throw new AbortError();
|
|
3055
|
-
}
|
|
4058
|
+
if (isAbortError(error)) throw error;
|
|
3056
4059
|
return false;
|
|
3057
4060
|
}
|
|
3058
4061
|
}
|
|
@@ -3070,8 +4073,9 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3070
4073
|
fetchClient;
|
|
3071
4074
|
validateChecksums;
|
|
3072
4075
|
azureAccount;
|
|
4076
|
+
withRangeCoalescing;
|
|
3073
4077
|
basePath = "";
|
|
3074
|
-
constructor(session, fetchClient, validateChecksums, azureAccount) {
|
|
4078
|
+
constructor(session, fetchClient, validateChecksums, azureAccount, withRangeCoalescing) {
|
|
3075
4079
|
if (!(session instanceof ReadSession)) {
|
|
3076
4080
|
throw new Error(
|
|
3077
4081
|
"IcechunkStore constructor is private. Use IcechunkStore.open() instead."
|
|
@@ -3081,6 +4085,7 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3081
4085
|
this.fetchClient = fetchClient;
|
|
3082
4086
|
this.validateChecksums = validateChecksums ?? false;
|
|
3083
4087
|
this.azureAccount = azureAccount;
|
|
4088
|
+
this.withRangeCoalescing = withRangeCoalescing;
|
|
3084
4089
|
}
|
|
3085
4090
|
static async open(arg, options = {}) {
|
|
3086
4091
|
if (arg instanceof ReadSession) {
|
|
@@ -3088,7 +4093,8 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3088
4093
|
arg,
|
|
3089
4094
|
options.fetchClient,
|
|
3090
4095
|
options.validateChecksums,
|
|
3091
|
-
options.azureAccount
|
|
4096
|
+
options.azureAccount,
|
|
4097
|
+
options.withRangeCoalescing
|
|
3092
4098
|
);
|
|
3093
4099
|
}
|
|
3094
4100
|
const storage = typeof arg === "string" ? new HttpStorage(arg) : arg;
|
|
@@ -3116,7 +4122,8 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3116
4122
|
session,
|
|
3117
4123
|
options.fetchClient,
|
|
3118
4124
|
options.validateChecksums,
|
|
3119
|
-
options.azureAccount
|
|
4125
|
+
options.azureAccount,
|
|
4126
|
+
options.withRangeCoalescing
|
|
3120
4127
|
);
|
|
3121
4128
|
}
|
|
3122
4129
|
/**
|
|
@@ -3132,7 +4139,7 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3132
4139
|
* @returns Data bytes or undefined if not found
|
|
3133
4140
|
*/
|
|
3134
4141
|
async get(key, opts) {
|
|
3135
|
-
|
|
4142
|
+
opts?.signal?.throwIfAborted();
|
|
3136
4143
|
const parsed = parseZarrKey(key);
|
|
3137
4144
|
const resolvedPath = this.resolvePath(parsed.path);
|
|
3138
4145
|
try {
|
|
@@ -3144,11 +4151,15 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3144
4151
|
signal: opts?.signal,
|
|
3145
4152
|
...this.fetchClient && { fetchClient: this.fetchClient },
|
|
3146
4153
|
validateChecksums: this.validateChecksums,
|
|
3147
|
-
azureAccount: this.azureAccount
|
|
4154
|
+
azureAccount: this.azureAccount,
|
|
4155
|
+
...this.withRangeCoalescing && {
|
|
4156
|
+
withRangeCoalescing: this.withRangeCoalescing
|
|
4157
|
+
}
|
|
3148
4158
|
});
|
|
3149
4159
|
return chunk ?? void 0;
|
|
3150
|
-
} catch {
|
|
3151
|
-
return void 0;
|
|
4160
|
+
} catch (err2) {
|
|
4161
|
+
if (err2 instanceof NotFoundError) return void 0;
|
|
4162
|
+
throw err2;
|
|
3152
4163
|
}
|
|
3153
4164
|
}
|
|
3154
4165
|
/**
|
|
@@ -3164,7 +4175,7 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3164
4175
|
* @returns Data bytes or undefined if not found
|
|
3165
4176
|
*/
|
|
3166
4177
|
async getRange(key, range, opts) {
|
|
3167
|
-
|
|
4178
|
+
opts?.signal?.throwIfAborted();
|
|
3168
4179
|
const parsed = parseZarrKey(key);
|
|
3169
4180
|
const resolvedPath = this.resolvePath(parsed.path);
|
|
3170
4181
|
try {
|
|
@@ -3177,7 +4188,10 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3177
4188
|
signal: opts?.signal,
|
|
3178
4189
|
...this.fetchClient && { fetchClient: this.fetchClient },
|
|
3179
4190
|
validateChecksums: this.validateChecksums,
|
|
3180
|
-
azureAccount: this.azureAccount
|
|
4191
|
+
azureAccount: this.azureAccount,
|
|
4192
|
+
...this.withRangeCoalescing && {
|
|
4193
|
+
withRangeCoalescing: this.withRangeCoalescing
|
|
4194
|
+
}
|
|
3181
4195
|
}
|
|
3182
4196
|
);
|
|
3183
4197
|
return data2 ?? void 0;
|
|
@@ -3188,8 +4202,9 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3188
4202
|
return data.slice(-range.suffixLength);
|
|
3189
4203
|
}
|
|
3190
4204
|
return data.slice(range.offset, range.offset + range.length);
|
|
3191
|
-
} catch {
|
|
3192
|
-
return void 0;
|
|
4205
|
+
} catch (err2) {
|
|
4206
|
+
if (err2 instanceof NotFoundError) return void 0;
|
|
4207
|
+
throw err2;
|
|
3193
4208
|
}
|
|
3194
4209
|
}
|
|
3195
4210
|
/** Prepend basePath to a parsed path. */
|
|
@@ -3213,7 +4228,8 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3213
4228
|
this.session,
|
|
3214
4229
|
this.fetchClient,
|
|
3215
4230
|
this.validateChecksums,
|
|
3216
|
-
this.azureAccount
|
|
4231
|
+
this.azureAccount,
|
|
4232
|
+
this.withRangeCoalescing
|
|
3217
4233
|
);
|
|
3218
4234
|
const cleanPath = path.replace(/^\/+|\/+$/g, "");
|
|
3219
4235
|
scoped.basePath = this.basePath ? `${this.basePath}/${cleanPath}` : cleanPath;
|