icechunk-js 0.3.2 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -10
- package/dist/index.cjs +1231 -215
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +105 -9
- package/dist/index.d.ts +105 -9
- package/dist/index.js +1230 -214
- package/dist/index.js.map +1 -1
- package/package.json +2 -3
package/dist/index.cjs
CHANGED
|
@@ -71,12 +71,12 @@ var StorageError = class extends Error {
|
|
|
71
71
|
this.name = "StorageError";
|
|
72
72
|
}
|
|
73
73
|
};
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
}
|
|
74
|
+
function hasErrorName(error) {
|
|
75
|
+
return typeof error === "object" && error !== null && "name" in error;
|
|
76
|
+
}
|
|
77
|
+
function isAbortError(error) {
|
|
78
|
+
return hasErrorName(error) && error.name === "AbortError";
|
|
79
|
+
}
|
|
80
80
|
|
|
81
81
|
// src/format/constants.ts
|
|
82
82
|
var PATHS = {
|
|
@@ -198,6 +198,7 @@ function decodeObjectId8(str) {
|
|
|
198
198
|
|
|
199
199
|
// src/format/flatbuffers/repo-parser.ts
|
|
200
200
|
var import_flatbuffers = require("flatbuffers");
|
|
201
|
+
var flexbuffers = __toESM(require("flatbuffers/js/flexbuffers.js"), 1);
|
|
201
202
|
|
|
202
203
|
// src/format/flatbuffers/generated/metadata-item.ts
|
|
203
204
|
var MetadataItem = class _MetadataItem {
|
|
@@ -351,6 +352,17 @@ var SnapshotInfo = class _SnapshotInfo {
|
|
|
351
352
|
const offset = this.bb.__offset(this.bb_pos, 12);
|
|
352
353
|
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
|
|
353
354
|
}
|
|
355
|
+
prunedAncestorTxLogs(index, obj) {
|
|
356
|
+
const offset = this.bb.__offset(this.bb_pos, 14);
|
|
357
|
+
return offset ? (obj || new ObjectId12()).__init(
|
|
358
|
+
this.bb.__vector(this.bb_pos + offset) + index * 12,
|
|
359
|
+
this.bb
|
|
360
|
+
) : null;
|
|
361
|
+
}
|
|
362
|
+
prunedAncestorTxLogsLength() {
|
|
363
|
+
const offset = this.bb.__offset(this.bb_pos, 14);
|
|
364
|
+
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
|
|
365
|
+
}
|
|
354
366
|
};
|
|
355
367
|
|
|
356
368
|
// src/format/flatbuffers/generated/update.ts
|
|
@@ -558,8 +570,587 @@ var Repo = class _Repo {
|
|
|
558
570
|
}
|
|
559
571
|
};
|
|
560
572
|
|
|
561
|
-
// src/
|
|
562
|
-
var
|
|
573
|
+
// src/vendor/fzstd/index.ts
|
|
574
|
+
var ab = ArrayBuffer;
|
|
575
|
+
var u8 = Uint8Array;
|
|
576
|
+
var u16 = Uint16Array;
|
|
577
|
+
var i16 = Int16Array;
|
|
578
|
+
var i32 = Int32Array;
|
|
579
|
+
var slc = (v, s, e) => {
|
|
580
|
+
if (u8.prototype.slice) return u8.prototype.slice.call(v, s, e);
|
|
581
|
+
if (s == null || s < 0) s = 0;
|
|
582
|
+
if (e == null || e > v.length) e = v.length;
|
|
583
|
+
const n = new u8(e - s);
|
|
584
|
+
n.set(v.subarray(s, e));
|
|
585
|
+
return n;
|
|
586
|
+
};
|
|
587
|
+
var fill = (v, n, s, e) => {
|
|
588
|
+
if (u8.prototype.fill) return u8.prototype.fill.call(v, n, s, e);
|
|
589
|
+
if (s == null || s < 0) s = 0;
|
|
590
|
+
if (e == null || e > v.length) e = v.length;
|
|
591
|
+
for (; s < e; ++s) v[s] = n;
|
|
592
|
+
return v;
|
|
593
|
+
};
|
|
594
|
+
var cpw = (v, t, s, e) => {
|
|
595
|
+
if (u8.prototype.copyWithin) return u8.prototype.copyWithin.call(v, t, s, e);
|
|
596
|
+
if (s == null || s < 0) s = 0;
|
|
597
|
+
if (e == null || e > v.length) e = v.length;
|
|
598
|
+
while (s < e) {
|
|
599
|
+
v[t++] = v[s++];
|
|
600
|
+
}
|
|
601
|
+
};
|
|
602
|
+
var ec = [
|
|
603
|
+
"invalid zstd data",
|
|
604
|
+
"window size too large (>2046MB)",
|
|
605
|
+
"invalid block type",
|
|
606
|
+
"FSE accuracy too high",
|
|
607
|
+
"match distance too far back",
|
|
608
|
+
"unexpected EOF",
|
|
609
|
+
"wrong dictionary"
|
|
610
|
+
];
|
|
611
|
+
var err = (ind, msg, nt) => {
|
|
612
|
+
const e = new Error(msg || ec[ind]);
|
|
613
|
+
e.code = ind;
|
|
614
|
+
if (Error.captureStackTrace) Error.captureStackTrace(e, err);
|
|
615
|
+
if (!nt) throw e;
|
|
616
|
+
return e;
|
|
617
|
+
};
|
|
618
|
+
var rb = (d, b, n) => {
|
|
619
|
+
let i = 0, o = 0;
|
|
620
|
+
for (; i < n; ++i) o |= d[b++] << (i << 3);
|
|
621
|
+
return o;
|
|
622
|
+
};
|
|
623
|
+
var b4 = (d, b) => (d[b] | d[b + 1] << 8 | d[b + 2] << 16 | d[b + 3] << 24) >>> 0;
|
|
624
|
+
var rdic = (dic, st) => {
|
|
625
|
+
if (b4(dic, 0) == 3962610743) {
|
|
626
|
+
if (st.d && st.d !== b4(dic, 4)) err(6);
|
|
627
|
+
let bt = 8;
|
|
628
|
+
[bt, st.h] = rhu(dic, bt);
|
|
629
|
+
let mlt, olt, llt;
|
|
630
|
+
[bt, olt] = rfse(dic, bt, 8);
|
|
631
|
+
[bt, mlt] = rfse(dic, bt, 9);
|
|
632
|
+
[bt, llt] = rfse(dic, bt, 9);
|
|
633
|
+
st.t = [mlt, olt, llt];
|
|
634
|
+
st.o = new i32([b4(dic, bt), b4(dic, bt + 4), b4(dic, bt + 8)]);
|
|
635
|
+
dic = dic.subarray(bt + 12);
|
|
636
|
+
}
|
|
637
|
+
const wl = Math.max(dic.length, st.u ? Math.min(st.e, dic.length + st.u) : st.e);
|
|
638
|
+
st.w = new u8(wl);
|
|
639
|
+
st.w.set(dic, wl - dic.length);
|
|
640
|
+
st.e = wl;
|
|
641
|
+
};
|
|
642
|
+
var rzfh = (dat, w) => {
|
|
643
|
+
const n3 = dat[0] | dat[1] << 8 | dat[2] << 16;
|
|
644
|
+
if (n3 == 3126568 && dat[3] == 253) {
|
|
645
|
+
const flg = dat[4];
|
|
646
|
+
const ss = flg >> 5 & 1, cc = flg >> 2 & 1, df = flg & 3, fcf = flg >> 6;
|
|
647
|
+
if (flg & 8) err(0);
|
|
648
|
+
let bt = 6 - ss;
|
|
649
|
+
const db = df == 3 ? 4 : df;
|
|
650
|
+
const di = rb(dat, bt, db);
|
|
651
|
+
bt += db;
|
|
652
|
+
const fsb = fcf ? 1 << fcf : ss;
|
|
653
|
+
const fss = rb(dat, bt, fsb) + (fcf == 1 && 256);
|
|
654
|
+
let ws = fss;
|
|
655
|
+
if (!ss) {
|
|
656
|
+
const wb = 1 << 10 + (dat[5] >> 3);
|
|
657
|
+
ws = wb + (wb >> 3) * (dat[5] & 7);
|
|
658
|
+
}
|
|
659
|
+
if (ws > 2145386496) err(1);
|
|
660
|
+
const buf = new u8((w == 1 ? fss || ws : w ? 0 : ws) + 12);
|
|
661
|
+
buf[0] = 1, buf[4] = 4, buf[8] = 8;
|
|
662
|
+
return {
|
|
663
|
+
b: bt + fsb,
|
|
664
|
+
y: 0,
|
|
665
|
+
l: 0,
|
|
666
|
+
d: di,
|
|
667
|
+
w: w && w != 1 ? w : buf.subarray(12),
|
|
668
|
+
e: ws,
|
|
669
|
+
o: new i32(buf.buffer, 0, 3),
|
|
670
|
+
u: fss,
|
|
671
|
+
c: cc,
|
|
672
|
+
m: Math.min(131072, ws)
|
|
673
|
+
};
|
|
674
|
+
} else if ((n3 >> 4 | dat[3] << 20) == 25481893) {
|
|
675
|
+
return b4(dat, 4) + 8;
|
|
676
|
+
}
|
|
677
|
+
err(0);
|
|
678
|
+
};
|
|
679
|
+
var msb = (val) => {
|
|
680
|
+
let bits = 0;
|
|
681
|
+
for (; 1 << bits <= val; ++bits) ;
|
|
682
|
+
return bits - 1;
|
|
683
|
+
};
|
|
684
|
+
var rfse = (dat, bt, mal) => {
|
|
685
|
+
let tpos = (bt << 3) + 4;
|
|
686
|
+
const al = (dat[bt] & 15) + 5;
|
|
687
|
+
if (al > mal) err(3);
|
|
688
|
+
const sz = 1 << al;
|
|
689
|
+
let probs = sz, sym = -1, re = -1, i = -1, ht = sz;
|
|
690
|
+
const buf = new ab(512 + (sz << 2));
|
|
691
|
+
const freq = new i16(buf, 0, 256);
|
|
692
|
+
const dstate = new u16(buf, 0, 256);
|
|
693
|
+
const nstate = new u16(buf, 512, sz);
|
|
694
|
+
const bb1 = 512 + (sz << 1);
|
|
695
|
+
const syms = new u8(buf, bb1, sz);
|
|
696
|
+
const nbits = new u8(buf, bb1 + sz);
|
|
697
|
+
while (sym < 255 && probs > 0) {
|
|
698
|
+
const bits = msb(probs + 1);
|
|
699
|
+
const cbt = tpos >> 3;
|
|
700
|
+
const msk = (1 << bits + 1) - 1;
|
|
701
|
+
let val = (dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (tpos & 7) & msk;
|
|
702
|
+
const msk1fb = (1 << bits) - 1;
|
|
703
|
+
const msv = msk - probs - 1;
|
|
704
|
+
const sval = val & msk1fb;
|
|
705
|
+
if (sval < msv) tpos += bits, val = sval;
|
|
706
|
+
else {
|
|
707
|
+
tpos += bits + 1;
|
|
708
|
+
if (val > msk1fb) val -= msv;
|
|
709
|
+
}
|
|
710
|
+
freq[++sym] = --val;
|
|
711
|
+
if (val == -1) {
|
|
712
|
+
probs += val;
|
|
713
|
+
syms[--ht] = sym;
|
|
714
|
+
} else probs -= val;
|
|
715
|
+
if (!val) {
|
|
716
|
+
do {
|
|
717
|
+
const rbt = tpos >> 3;
|
|
718
|
+
re = (dat[rbt] | dat[rbt + 1] << 8) >> (tpos & 7) & 3;
|
|
719
|
+
tpos += 2;
|
|
720
|
+
sym += re;
|
|
721
|
+
} while (re == 3);
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
if (sym > 255 || probs) err(0);
|
|
725
|
+
let sympos = 0;
|
|
726
|
+
const sstep = (sz >> 1) + (sz >> 3) + 3;
|
|
727
|
+
const smask = sz - 1;
|
|
728
|
+
for (let s = 0; s <= sym; ++s) {
|
|
729
|
+
const sf = freq[s];
|
|
730
|
+
if (sf < 1) {
|
|
731
|
+
dstate[s] = -sf;
|
|
732
|
+
continue;
|
|
733
|
+
}
|
|
734
|
+
for (i = 0; i < sf; ++i) {
|
|
735
|
+
syms[sympos] = s;
|
|
736
|
+
do {
|
|
737
|
+
sympos = sympos + sstep & smask;
|
|
738
|
+
} while (sympos >= ht);
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
if (sympos) err(0);
|
|
742
|
+
for (i = 0; i < sz; ++i) {
|
|
743
|
+
const ns = dstate[syms[i]]++;
|
|
744
|
+
const nb = nbits[i] = al - msb(ns);
|
|
745
|
+
nstate[i] = (ns << nb) - sz;
|
|
746
|
+
}
|
|
747
|
+
return [tpos + 7 >> 3, {
|
|
748
|
+
b: al,
|
|
749
|
+
s: syms,
|
|
750
|
+
n: nbits,
|
|
751
|
+
t: nstate
|
|
752
|
+
}];
|
|
753
|
+
};
|
|
754
|
+
var rhu = (dat, bt) => {
|
|
755
|
+
let i = 0, wc = -1;
|
|
756
|
+
const buf = new u8(292), hb = dat[bt];
|
|
757
|
+
const hw = buf.subarray(0, 256);
|
|
758
|
+
const rc = buf.subarray(256, 268);
|
|
759
|
+
const ri = new u16(buf.buffer, 268);
|
|
760
|
+
if (hb < 128) {
|
|
761
|
+
const [ebt, fdt] = rfse(dat, bt + 1, 6);
|
|
762
|
+
bt += hb;
|
|
763
|
+
const epos = ebt << 3;
|
|
764
|
+
const lb = dat[bt];
|
|
765
|
+
if (!lb) err(0);
|
|
766
|
+
let st1 = 0, st2 = 0, btr1 = fdt.b, btr2 = btr1;
|
|
767
|
+
let fpos = (++bt << 3) - 8 + msb(lb);
|
|
768
|
+
for (; ; ) {
|
|
769
|
+
fpos -= btr1;
|
|
770
|
+
if (fpos < epos) break;
|
|
771
|
+
let cbt = fpos >> 3;
|
|
772
|
+
st1 += (dat[cbt] | dat[cbt + 1] << 8) >> (fpos & 7) & (1 << btr1) - 1;
|
|
773
|
+
hw[++wc] = fdt.s[st1];
|
|
774
|
+
fpos -= btr2;
|
|
775
|
+
if (fpos < epos) break;
|
|
776
|
+
cbt = fpos >> 3;
|
|
777
|
+
st2 += (dat[cbt] | dat[cbt + 1] << 8) >> (fpos & 7) & (1 << btr2) - 1;
|
|
778
|
+
hw[++wc] = fdt.s[st2];
|
|
779
|
+
btr1 = fdt.n[st1];
|
|
780
|
+
st1 = fdt.t[st1];
|
|
781
|
+
btr2 = fdt.n[st2];
|
|
782
|
+
st2 = fdt.t[st2];
|
|
783
|
+
}
|
|
784
|
+
if (++wc > 255) err(0);
|
|
785
|
+
} else {
|
|
786
|
+
wc = hb - 127;
|
|
787
|
+
for (; i < wc; i += 2) {
|
|
788
|
+
const byte = dat[++bt];
|
|
789
|
+
hw[i] = byte >> 4;
|
|
790
|
+
hw[i + 1] = byte & 15;
|
|
791
|
+
}
|
|
792
|
+
++bt;
|
|
793
|
+
}
|
|
794
|
+
let wes = 0;
|
|
795
|
+
for (i = 0; i < wc; ++i) {
|
|
796
|
+
const wt = hw[i];
|
|
797
|
+
if (wt > 11) err(0);
|
|
798
|
+
wes += wt && 1 << wt - 1;
|
|
799
|
+
}
|
|
800
|
+
const mb = msb(wes) + 1;
|
|
801
|
+
const ts = 1 << mb;
|
|
802
|
+
const rem = ts - wes;
|
|
803
|
+
if (rem & rem - 1) err(0);
|
|
804
|
+
hw[wc++] = msb(rem) + 1;
|
|
805
|
+
for (i = 0; i < wc; ++i) {
|
|
806
|
+
const wt = hw[i];
|
|
807
|
+
++rc[hw[i] = wt && mb + 1 - wt];
|
|
808
|
+
}
|
|
809
|
+
const hbuf = new u8(ts << 1);
|
|
810
|
+
const syms = hbuf.subarray(0, ts), nb = hbuf.subarray(ts);
|
|
811
|
+
ri[mb] = 0;
|
|
812
|
+
for (i = mb; i > 0; --i) {
|
|
813
|
+
const pv = ri[i];
|
|
814
|
+
fill(nb, i, pv, ri[i - 1] = pv + rc[i] * (1 << mb - i));
|
|
815
|
+
}
|
|
816
|
+
if (ri[0] != ts) err(0);
|
|
817
|
+
for (i = 0; i < wc; ++i) {
|
|
818
|
+
const bits = hw[i];
|
|
819
|
+
if (bits) {
|
|
820
|
+
const code = ri[bits];
|
|
821
|
+
fill(syms, i, code, ri[bits] = code + (1 << mb - bits));
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
return [bt, {
|
|
825
|
+
n: nb,
|
|
826
|
+
b: mb,
|
|
827
|
+
s: syms
|
|
828
|
+
}];
|
|
829
|
+
};
|
|
830
|
+
var dllt = rfse(/* @__PURE__ */ new u8([
|
|
831
|
+
81,
|
|
832
|
+
16,
|
|
833
|
+
99,
|
|
834
|
+
140,
|
|
835
|
+
49,
|
|
836
|
+
198,
|
|
837
|
+
24,
|
|
838
|
+
99,
|
|
839
|
+
12,
|
|
840
|
+
33,
|
|
841
|
+
196,
|
|
842
|
+
24,
|
|
843
|
+
99,
|
|
844
|
+
102,
|
|
845
|
+
102,
|
|
846
|
+
134,
|
|
847
|
+
70,
|
|
848
|
+
146,
|
|
849
|
+
4
|
|
850
|
+
]), 0, 6)[1];
|
|
851
|
+
var dmlt = rfse(/* @__PURE__ */ new u8([
|
|
852
|
+
33,
|
|
853
|
+
20,
|
|
854
|
+
196,
|
|
855
|
+
24,
|
|
856
|
+
99,
|
|
857
|
+
140,
|
|
858
|
+
33,
|
|
859
|
+
132,
|
|
860
|
+
16,
|
|
861
|
+
66,
|
|
862
|
+
8,
|
|
863
|
+
33,
|
|
864
|
+
132,
|
|
865
|
+
16,
|
|
866
|
+
66,
|
|
867
|
+
8,
|
|
868
|
+
33,
|
|
869
|
+
68,
|
|
870
|
+
68,
|
|
871
|
+
68,
|
|
872
|
+
68,
|
|
873
|
+
68,
|
|
874
|
+
68,
|
|
875
|
+
68,
|
|
876
|
+
68,
|
|
877
|
+
36,
|
|
878
|
+
9
|
|
879
|
+
]), 0, 6)[1];
|
|
880
|
+
var doct = rfse(/* @__PURE__ */ new u8([
|
|
881
|
+
32,
|
|
882
|
+
132,
|
|
883
|
+
16,
|
|
884
|
+
66,
|
|
885
|
+
102,
|
|
886
|
+
70,
|
|
887
|
+
68,
|
|
888
|
+
68,
|
|
889
|
+
68,
|
|
890
|
+
68,
|
|
891
|
+
36,
|
|
892
|
+
73,
|
|
893
|
+
2
|
|
894
|
+
]), 0, 5)[1];
|
|
895
|
+
var b2bl = (b, s) => {
|
|
896
|
+
const len = b.length, bl = new i32(len);
|
|
897
|
+
for (let i = 0; i < len; ++i) {
|
|
898
|
+
bl[i] = s;
|
|
899
|
+
s += 1 << b[i];
|
|
900
|
+
}
|
|
901
|
+
return bl;
|
|
902
|
+
};
|
|
903
|
+
var llb = /* @__PURE__ */ new u8((/* @__PURE__ */ new i32([
|
|
904
|
+
0,
|
|
905
|
+
0,
|
|
906
|
+
0,
|
|
907
|
+
0,
|
|
908
|
+
16843009,
|
|
909
|
+
50528770,
|
|
910
|
+
134678020,
|
|
911
|
+
202050057,
|
|
912
|
+
269422093
|
|
913
|
+
])).buffer, 0, 36);
|
|
914
|
+
var llbl = /* @__PURE__ */ b2bl(llb, 0);
|
|
915
|
+
var mlb = /* @__PURE__ */ new u8((/* @__PURE__ */ new i32([
|
|
916
|
+
0,
|
|
917
|
+
0,
|
|
918
|
+
0,
|
|
919
|
+
0,
|
|
920
|
+
0,
|
|
921
|
+
0,
|
|
922
|
+
0,
|
|
923
|
+
0,
|
|
924
|
+
16843009,
|
|
925
|
+
50528770,
|
|
926
|
+
117769220,
|
|
927
|
+
185207048,
|
|
928
|
+
252579084,
|
|
929
|
+
16
|
|
930
|
+
])).buffer, 0, 53);
|
|
931
|
+
var mlbl = /* @__PURE__ */ b2bl(mlb, 3);
|
|
932
|
+
var dhu = (dat, out, hu) => {
|
|
933
|
+
const len = dat.length, ss = out.length, lb = dat[len - 1], msk = (1 << hu.b) - 1, eb = -hu.b;
|
|
934
|
+
if (!lb) err(0);
|
|
935
|
+
let st = 0, btr = hu.b, pos = (len << 3) - 8 + msb(lb) - btr, i = -1;
|
|
936
|
+
for (; pos > eb && i < ss; ) {
|
|
937
|
+
const cbt = pos >> 3;
|
|
938
|
+
const val = (dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (pos & 7);
|
|
939
|
+
st = (st << btr | val) & msk;
|
|
940
|
+
out[++i] = hu.s[st];
|
|
941
|
+
pos -= btr = hu.n[st];
|
|
942
|
+
}
|
|
943
|
+
if (pos != eb || i + 1 != ss) err(0);
|
|
944
|
+
};
|
|
945
|
+
var dhu4 = (dat, out, hu) => {
|
|
946
|
+
let bt = 6;
|
|
947
|
+
const ss = out.length, sz1 = ss + 3 >> 2, sz2 = sz1 << 1, sz3 = sz1 + sz2;
|
|
948
|
+
dhu(dat.subarray(bt, bt += dat[0] | dat[1] << 8), out.subarray(0, sz1), hu);
|
|
949
|
+
dhu(dat.subarray(bt, bt += dat[2] | dat[3] << 8), out.subarray(sz1, sz2), hu);
|
|
950
|
+
dhu(dat.subarray(bt, bt += dat[4] | dat[5] << 8), out.subarray(sz2, sz3), hu);
|
|
951
|
+
dhu(dat.subarray(bt), out.subarray(sz3), hu);
|
|
952
|
+
};
|
|
953
|
+
var rzb = (dat, st, out) => {
|
|
954
|
+
let bt = st.b;
|
|
955
|
+
const b0 = dat[bt], btype = b0 >> 1 & 3;
|
|
956
|
+
st.l = b0 & 1;
|
|
957
|
+
const sz = b0 >> 3 | dat[bt + 1] << 5 | dat[bt + 2] << 13;
|
|
958
|
+
const ebt = (bt += 3) + sz;
|
|
959
|
+
if (btype == 1) {
|
|
960
|
+
if (bt >= dat.length) return;
|
|
961
|
+
st.b = bt + 1;
|
|
962
|
+
if (out) {
|
|
963
|
+
fill(out, dat[bt], st.y, st.y += sz);
|
|
964
|
+
return out;
|
|
965
|
+
}
|
|
966
|
+
return fill(new u8(sz), dat[bt]);
|
|
967
|
+
}
|
|
968
|
+
if (ebt > dat.length) return;
|
|
969
|
+
if (btype == 0) {
|
|
970
|
+
st.b = ebt;
|
|
971
|
+
if (out) {
|
|
972
|
+
out.set(dat.subarray(bt, ebt), st.y);
|
|
973
|
+
st.y += sz;
|
|
974
|
+
return out;
|
|
975
|
+
}
|
|
976
|
+
return slc(dat, bt, ebt);
|
|
977
|
+
}
|
|
978
|
+
if (btype == 2) {
|
|
979
|
+
const b3 = dat[bt], lbt = b3 & 3, sf = b3 >> 2 & 3;
|
|
980
|
+
let lss = b3 >> 4, lcs = 0, s4 = 0;
|
|
981
|
+
if (lbt < 2) {
|
|
982
|
+
if (sf & 1) lss |= dat[++bt] << 4 | (sf & 2 && dat[++bt] << 12);
|
|
983
|
+
else lss = b3 >> 3;
|
|
984
|
+
} else {
|
|
985
|
+
s4 = sf;
|
|
986
|
+
if (sf < 2) lss |= (dat[++bt] & 63) << 4, lcs = dat[bt] >> 6 | dat[++bt] << 2;
|
|
987
|
+
else if (sf == 2) lss |= dat[++bt] << 4 | (dat[++bt] & 3) << 12, lcs = dat[bt] >> 2 | dat[++bt] << 6;
|
|
988
|
+
else lss |= dat[++bt] << 4 | (dat[++bt] & 63) << 12, lcs = dat[bt] >> 6 | dat[++bt] << 2 | dat[++bt] << 10;
|
|
989
|
+
}
|
|
990
|
+
++bt;
|
|
991
|
+
let buf = out ? out.subarray(st.y, st.y + st.m) : new u8(st.m);
|
|
992
|
+
let spl = buf.length - lss;
|
|
993
|
+
if (lbt == 0) buf.set(dat.subarray(bt, bt += lss), spl);
|
|
994
|
+
else if (lbt == 1) fill(buf, dat[bt++], spl);
|
|
995
|
+
else {
|
|
996
|
+
let hu = st.h;
|
|
997
|
+
if (lbt == 2) {
|
|
998
|
+
const hud = rhu(dat, bt);
|
|
999
|
+
lcs += bt - (bt = hud[0]);
|
|
1000
|
+
st.h = hu = hud[1];
|
|
1001
|
+
} else if (!hu) err(0);
|
|
1002
|
+
(s4 ? dhu4 : dhu)(dat.subarray(bt, bt += lcs), buf.subarray(spl), hu);
|
|
1003
|
+
}
|
|
1004
|
+
let ns = dat[bt++];
|
|
1005
|
+
if (ns) {
|
|
1006
|
+
if (ns == 255) ns = (dat[bt++] | dat[bt++] << 8) + 32512;
|
|
1007
|
+
else if (ns > 127) ns = ns - 128 << 8 | dat[bt++];
|
|
1008
|
+
const scm = dat[bt++];
|
|
1009
|
+
if (scm & 3) err(0);
|
|
1010
|
+
const dts = [dmlt, doct, dllt];
|
|
1011
|
+
for (let i = 2; i > -1; --i) {
|
|
1012
|
+
const md = scm >> (i << 1) + 2 & 3;
|
|
1013
|
+
if (md == 1) {
|
|
1014
|
+
const rbuf = new u8([0, 0, dat[bt++]]);
|
|
1015
|
+
dts[i] = {
|
|
1016
|
+
s: rbuf.subarray(2, 3),
|
|
1017
|
+
n: rbuf.subarray(0, 1),
|
|
1018
|
+
t: new u16(rbuf.buffer, 0, 1),
|
|
1019
|
+
b: 0
|
|
1020
|
+
};
|
|
1021
|
+
} else if (md == 2) {
|
|
1022
|
+
[bt, dts[i]] = rfse(dat, bt, 9 - (i & 1));
|
|
1023
|
+
} else if (md == 3) {
|
|
1024
|
+
if (!st.t) err(0);
|
|
1025
|
+
dts[i] = st.t[i];
|
|
1026
|
+
}
|
|
1027
|
+
}
|
|
1028
|
+
const [mlt, oct, llt] = st.t = dts;
|
|
1029
|
+
const lb = dat[ebt - 1];
|
|
1030
|
+
if (!lb) err(0);
|
|
1031
|
+
let spos = (ebt << 3) - 8 + msb(lb) - llt.b, cbt = spos >> 3, oubt = 0;
|
|
1032
|
+
let lst = (dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << llt.b) - 1;
|
|
1033
|
+
cbt = (spos -= oct.b) >> 3;
|
|
1034
|
+
let ost = (dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << oct.b) - 1;
|
|
1035
|
+
cbt = (spos -= mlt.b) >> 3;
|
|
1036
|
+
let mst = (dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << mlt.b) - 1;
|
|
1037
|
+
for (++ns; --ns; ) {
|
|
1038
|
+
const llc = llt.s[lst];
|
|
1039
|
+
const lbtr = llt.n[lst];
|
|
1040
|
+
const mlc = mlt.s[mst];
|
|
1041
|
+
const mbtr = mlt.n[mst];
|
|
1042
|
+
const ofc = oct.s[ost];
|
|
1043
|
+
const obtr = oct.n[ost];
|
|
1044
|
+
cbt = (spos -= ofc) >> 3;
|
|
1045
|
+
const ofp = 1 << ofc;
|
|
1046
|
+
let off = ofp + ((dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16 | dat[cbt + 3] << 24) >>> (spos & 7) & ofp - 1);
|
|
1047
|
+
cbt = (spos -= mlb[mlc]) >> 3;
|
|
1048
|
+
let ml = mlbl[mlc] + ((dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (spos & 7) & (1 << mlb[mlc]) - 1);
|
|
1049
|
+
cbt = (spos -= llb[llc]) >> 3;
|
|
1050
|
+
const ll = llbl[llc] + ((dat[cbt] | dat[cbt + 1] << 8 | dat[cbt + 2] << 16) >> (spos & 7) & (1 << llb[llc]) - 1);
|
|
1051
|
+
cbt = (spos -= lbtr) >> 3;
|
|
1052
|
+
lst = llt.t[lst] + ((dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << lbtr) - 1);
|
|
1053
|
+
cbt = (spos -= mbtr) >> 3;
|
|
1054
|
+
mst = mlt.t[mst] + ((dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << mbtr) - 1);
|
|
1055
|
+
cbt = (spos -= obtr) >> 3;
|
|
1056
|
+
ost = oct.t[ost] + ((dat[cbt] | dat[cbt + 1] << 8) >> (spos & 7) & (1 << obtr) - 1);
|
|
1057
|
+
if (off > 3) {
|
|
1058
|
+
st.o[2] = st.o[1];
|
|
1059
|
+
st.o[1] = st.o[0];
|
|
1060
|
+
st.o[0] = off -= 3;
|
|
1061
|
+
} else {
|
|
1062
|
+
const idx = off - (ll != 0);
|
|
1063
|
+
if (idx) {
|
|
1064
|
+
off = idx == 3 ? st.o[0] - 1 : st.o[idx];
|
|
1065
|
+
if (idx > 1) st.o[2] = st.o[1];
|
|
1066
|
+
st.o[1] = st.o[0];
|
|
1067
|
+
st.o[0] = off;
|
|
1068
|
+
} else off = st.o[0];
|
|
1069
|
+
}
|
|
1070
|
+
for (let i = 0; i < ll; ++i) {
|
|
1071
|
+
buf[oubt + i] = buf[spl + i];
|
|
1072
|
+
}
|
|
1073
|
+
oubt += ll, spl += ll;
|
|
1074
|
+
let stin = oubt - off;
|
|
1075
|
+
if (stin < 0) {
|
|
1076
|
+
let len = -stin;
|
|
1077
|
+
const bs = st.e + stin;
|
|
1078
|
+
if (len > ml) len = ml;
|
|
1079
|
+
for (let i = 0; i < len; ++i) {
|
|
1080
|
+
buf[oubt + i] = st.w[bs + i];
|
|
1081
|
+
}
|
|
1082
|
+
oubt += len, ml -= len, stin = 0;
|
|
1083
|
+
}
|
|
1084
|
+
for (let i = 0; i < ml; ++i) {
|
|
1085
|
+
buf[oubt + i] = buf[stin + i];
|
|
1086
|
+
}
|
|
1087
|
+
oubt += ml;
|
|
1088
|
+
}
|
|
1089
|
+
if (oubt != spl) {
|
|
1090
|
+
while (spl < buf.length) {
|
|
1091
|
+
buf[oubt++] = buf[spl++];
|
|
1092
|
+
}
|
|
1093
|
+
} else oubt = buf.length;
|
|
1094
|
+
if (out) st.y += oubt;
|
|
1095
|
+
else buf = slc(buf, 0, oubt);
|
|
1096
|
+
} else if (out) {
|
|
1097
|
+
st.y += lss;
|
|
1098
|
+
if (spl) {
|
|
1099
|
+
for (let i = 0; i < lss; ++i) {
|
|
1100
|
+
buf[i] = buf[spl + i];
|
|
1101
|
+
}
|
|
1102
|
+
}
|
|
1103
|
+
} else if (spl) buf = slc(buf, spl);
|
|
1104
|
+
st.b = ebt;
|
|
1105
|
+
return buf;
|
|
1106
|
+
}
|
|
1107
|
+
err(2);
|
|
1108
|
+
};
|
|
1109
|
+
var cct = (bufs, ol) => {
|
|
1110
|
+
if (bufs.length == 1) return bufs[0];
|
|
1111
|
+
const buf = new u8(ol);
|
|
1112
|
+
for (let i = 0, b = 0; i < bufs.length; ++i) {
|
|
1113
|
+
const chk = bufs[i];
|
|
1114
|
+
buf.set(chk, b);
|
|
1115
|
+
b += chk.length;
|
|
1116
|
+
}
|
|
1117
|
+
return buf;
|
|
1118
|
+
};
|
|
1119
|
+
function decompress(dat, buf, dic) {
|
|
1120
|
+
const bufs = [], nb = +!buf;
|
|
1121
|
+
let bt = 0, ol = 0;
|
|
1122
|
+
for (; dat.length; ) {
|
|
1123
|
+
const st = rzfh(dat, nb || buf);
|
|
1124
|
+
if (typeof st == "object") {
|
|
1125
|
+
if (st.d && !dic) err(6);
|
|
1126
|
+
if (dic) rdic(dic, st);
|
|
1127
|
+
if (nb) {
|
|
1128
|
+
buf = null;
|
|
1129
|
+
if (!dic && st.w.length == st.u) {
|
|
1130
|
+
bufs.push(buf = st.w);
|
|
1131
|
+
ol += st.u;
|
|
1132
|
+
}
|
|
1133
|
+
} else {
|
|
1134
|
+
bufs.push(buf);
|
|
1135
|
+
st.e = 0;
|
|
1136
|
+
}
|
|
1137
|
+
for (; !st.l; ) {
|
|
1138
|
+
const blk = rzb(dat, st, buf);
|
|
1139
|
+
if (!blk) err(5);
|
|
1140
|
+
if (buf) st.e = st.y;
|
|
1141
|
+
else {
|
|
1142
|
+
bufs.push(blk);
|
|
1143
|
+
ol += blk.length;
|
|
1144
|
+
cpw(st.w, 0, blk.length);
|
|
1145
|
+
st.w.set(blk, st.w.length - blk.length);
|
|
1146
|
+
}
|
|
1147
|
+
}
|
|
1148
|
+
bt = st.b + st.c * 4;
|
|
1149
|
+
} else bt = st;
|
|
1150
|
+
dat = dat.subarray(bt);
|
|
1151
|
+
}
|
|
1152
|
+
return cct(bufs, ol);
|
|
1153
|
+
}
|
|
563
1154
|
|
|
564
1155
|
// src/format/header.ts
|
|
565
1156
|
var HEADER_SIZE = 39;
|
|
@@ -673,7 +1264,7 @@ function parseRepo(data) {
|
|
|
673
1264
|
validateFileType(header, 6 /* RepoInfo */);
|
|
674
1265
|
let fbData = getDataAfterHeader(data);
|
|
675
1266
|
if (header.compression === 1 /* Zstd */) {
|
|
676
|
-
fbData =
|
|
1267
|
+
fbData = decompress(fbData);
|
|
677
1268
|
}
|
|
678
1269
|
const bb = new import_flatbuffers.ByteBuffer(fbData);
|
|
679
1270
|
const repo = Repo.getRootAsRepo(bb);
|
|
@@ -684,7 +1275,36 @@ function parseRepo(data) {
|
|
|
684
1275
|
);
|
|
685
1276
|
}
|
|
686
1277
|
const snapshotsLength = repo.snapshotsLength();
|
|
687
|
-
|
|
1278
|
+
const virtualChunkContainers = parseVirtualChunkContainers(repo);
|
|
1279
|
+
return { repo, specVersion, snapshotsLength, virtualChunkContainers };
|
|
1280
|
+
}
|
|
1281
|
+
function parseVirtualChunkContainers(repo) {
|
|
1282
|
+
const result = /* @__PURE__ */ new Map();
|
|
1283
|
+
const configBytes = repo.configArray();
|
|
1284
|
+
if (!configBytes || configBytes.length === 0) return result;
|
|
1285
|
+
let config;
|
|
1286
|
+
try {
|
|
1287
|
+
const ab2 = configBytes.buffer.slice(
|
|
1288
|
+
configBytes.byteOffset,
|
|
1289
|
+
configBytes.byteOffset + configBytes.byteLength
|
|
1290
|
+
);
|
|
1291
|
+
config = flexbuffers.toObject(ab2);
|
|
1292
|
+
} catch {
|
|
1293
|
+
return result;
|
|
1294
|
+
}
|
|
1295
|
+
if (!isRecord(config)) return result;
|
|
1296
|
+
const containers = config.virtual_chunk_containers;
|
|
1297
|
+
if (!isRecord(containers)) return result;
|
|
1298
|
+
for (const container of Object.values(containers)) {
|
|
1299
|
+
if (!isRecord(container)) continue;
|
|
1300
|
+
const { name, url_prefix: urlPrefix } = container;
|
|
1301
|
+
if (typeof name !== "string" || typeof urlPrefix !== "string") continue;
|
|
1302
|
+
result.set(name, urlPrefix);
|
|
1303
|
+
}
|
|
1304
|
+
return result;
|
|
1305
|
+
}
|
|
1306
|
+
function isRecord(value) {
|
|
1307
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
688
1308
|
}
|
|
689
1309
|
function compareUtf8ByteOrder(aBytes, bBytes) {
|
|
690
1310
|
const minLen = Math.min(aBytes.length, bBytes.length);
|
|
@@ -781,9 +1401,6 @@ function listTagsFromRepo(parsedRepo) {
|
|
|
781
1401
|
return listRefs((i) => repo.tags(i), repo.tagsLength());
|
|
782
1402
|
}
|
|
783
1403
|
|
|
784
|
-
// src/reader/session.ts
|
|
785
|
-
var import_fzstd2 = require("fzstd");
|
|
786
|
-
|
|
787
1404
|
// src/cache/lru.ts
|
|
788
1405
|
var LRUCache = class {
|
|
789
1406
|
cache;
|
|
@@ -827,6 +1444,85 @@ var LRUCache = class {
|
|
|
827
1444
|
}
|
|
828
1445
|
};
|
|
829
1446
|
|
|
1447
|
+
// src/cache/single-flight.ts
|
|
1448
|
+
function singleFlight(cache) {
|
|
1449
|
+
const pending = /* @__PURE__ */ new Map();
|
|
1450
|
+
return {
|
|
1451
|
+
load(key, fetcher, signal) {
|
|
1452
|
+
if (signal?.aborted) return Promise.reject(makeAbortError());
|
|
1453
|
+
const hit = cache.get(key);
|
|
1454
|
+
if (hit !== void 0) return Promise.resolve(hit);
|
|
1455
|
+
let entry = pending.get(key);
|
|
1456
|
+
if (!entry) {
|
|
1457
|
+
const controller = new AbortController();
|
|
1458
|
+
const promise = invokeFetcher(fetcher, controller.signal).then((value) => {
|
|
1459
|
+
cache.set(key, value);
|
|
1460
|
+
return value;
|
|
1461
|
+
}).finally(() => {
|
|
1462
|
+
if (pending.get(key)?.promise === promise) pending.delete(key);
|
|
1463
|
+
});
|
|
1464
|
+
const fresh = { promise, controller, refCount: 0 };
|
|
1465
|
+
pending.set(key, fresh);
|
|
1466
|
+
entry = fresh;
|
|
1467
|
+
}
|
|
1468
|
+
const owned = entry;
|
|
1469
|
+
owned.refCount++;
|
|
1470
|
+
const release = () => {
|
|
1471
|
+
owned.refCount--;
|
|
1472
|
+
if (owned.refCount > 0) return;
|
|
1473
|
+
if (pending.get(key) === owned) {
|
|
1474
|
+
pending.delete(key);
|
|
1475
|
+
owned.controller.abort();
|
|
1476
|
+
}
|
|
1477
|
+
};
|
|
1478
|
+
if (!signal) {
|
|
1479
|
+
return owned.promise.then(
|
|
1480
|
+
(value) => {
|
|
1481
|
+
release();
|
|
1482
|
+
return value;
|
|
1483
|
+
},
|
|
1484
|
+
(error) => {
|
|
1485
|
+
release();
|
|
1486
|
+
throw error;
|
|
1487
|
+
}
|
|
1488
|
+
);
|
|
1489
|
+
}
|
|
1490
|
+
return new Promise((resolve, reject) => {
|
|
1491
|
+
let done = false;
|
|
1492
|
+
const finish = () => {
|
|
1493
|
+
if (done) return false;
|
|
1494
|
+
done = true;
|
|
1495
|
+
signal.removeEventListener("abort", handleAbort);
|
|
1496
|
+
release();
|
|
1497
|
+
return true;
|
|
1498
|
+
};
|
|
1499
|
+
const handleAbort = () => {
|
|
1500
|
+
if (finish()) reject(makeAbortError());
|
|
1501
|
+
};
|
|
1502
|
+
signal.addEventListener("abort", handleAbort, { once: true });
|
|
1503
|
+
owned.promise.then(
|
|
1504
|
+
(value) => {
|
|
1505
|
+
if (finish()) resolve(value);
|
|
1506
|
+
},
|
|
1507
|
+
(error) => {
|
|
1508
|
+
if (finish()) reject(error);
|
|
1509
|
+
}
|
|
1510
|
+
);
|
|
1511
|
+
});
|
|
1512
|
+
}
|
|
1513
|
+
};
|
|
1514
|
+
}
|
|
1515
|
+
function invokeFetcher(fetcher, signal) {
|
|
1516
|
+
try {
|
|
1517
|
+
return fetcher(signal);
|
|
1518
|
+
} catch (error) {
|
|
1519
|
+
return Promise.reject(error);
|
|
1520
|
+
}
|
|
1521
|
+
}
|
|
1522
|
+
function makeAbortError() {
|
|
1523
|
+
return new DOMException("The operation was aborted.", "AbortError");
|
|
1524
|
+
}
|
|
1525
|
+
|
|
830
1526
|
// src/format/flatbuffers/types.ts
|
|
831
1527
|
function asObjectId12(bytes) {
|
|
832
1528
|
if (bytes.length !== 12) {
|
|
@@ -1677,6 +2373,60 @@ var Manifest = class _Manifest {
|
|
|
1677
2373
|
};
|
|
1678
2374
|
|
|
1679
2375
|
// src/format/flatbuffers/manifest-parser.ts
|
|
2376
|
+
var COMPRESSION_ALG_ZSTD_DICT = 1;
|
|
2377
|
+
var MAX_DECOMPRESSED_LOCATION_SIZE = 1024;
|
|
2378
|
+
function zstdFrameAllocSize(frame) {
|
|
2379
|
+
if (frame.length < 5 || frame[0] !== 40 || frame[1] !== 181 || frame[2] !== 47 || frame[3] !== 253) {
|
|
2380
|
+
return Infinity;
|
|
2381
|
+
}
|
|
2382
|
+
const flg = frame[4];
|
|
2383
|
+
const singleSegment = flg >> 5 & 1;
|
|
2384
|
+
const contentChecksum = flg >> 2 & 1;
|
|
2385
|
+
const dictIdFlag = flg & 3;
|
|
2386
|
+
const contentSizeFlag = flg >> 6;
|
|
2387
|
+
let pos = 5;
|
|
2388
|
+
let windowSize = 0;
|
|
2389
|
+
if (!singleSegment) {
|
|
2390
|
+
if (pos >= frame.length) return Infinity;
|
|
2391
|
+
const wd = frame[pos++];
|
|
2392
|
+
const base = 2 ** (10 + (wd >> 3));
|
|
2393
|
+
windowSize = base + base / 8 * (wd & 7);
|
|
2394
|
+
}
|
|
2395
|
+
pos += dictIdFlag === 3 ? 4 : dictIdFlag;
|
|
2396
|
+
const fcsBytes = contentSizeFlag ? 1 << contentSizeFlag : singleSegment;
|
|
2397
|
+
let contentSize = 0;
|
|
2398
|
+
if (fcsBytes) {
|
|
2399
|
+
if (pos + fcsBytes > frame.length) return Infinity;
|
|
2400
|
+
for (let i = 0; i < fcsBytes; i++) {
|
|
2401
|
+
contentSize += frame[pos + i] * 2 ** (8 * i);
|
|
2402
|
+
}
|
|
2403
|
+
if (contentSizeFlag === 1) contentSize += 256;
|
|
2404
|
+
if (singleSegment) windowSize = contentSize;
|
|
2405
|
+
}
|
|
2406
|
+
pos += fcsBytes;
|
|
2407
|
+
let blockOutput = 0;
|
|
2408
|
+
for (; ; ) {
|
|
2409
|
+
if (pos + 3 > frame.length) return Infinity;
|
|
2410
|
+
const blockHeader = frame[pos] | frame[pos + 1] << 8 | frame[pos + 2] << 16;
|
|
2411
|
+
pos += 3;
|
|
2412
|
+
const lastBlock = blockHeader & 1;
|
|
2413
|
+
const blockType = blockHeader >> 1 & 3;
|
|
2414
|
+
const blockSize = blockHeader >> 3;
|
|
2415
|
+
if (blockType === 3) return Infinity;
|
|
2416
|
+
if (blockType === 1) {
|
|
2417
|
+
blockOutput += blockSize;
|
|
2418
|
+
pos += 1;
|
|
2419
|
+
} else {
|
|
2420
|
+
blockOutput += blockType === 0 ? blockSize : Math.min(windowSize, 131072);
|
|
2421
|
+
pos += blockSize;
|
|
2422
|
+
}
|
|
2423
|
+
if (pos > frame.length) return Infinity;
|
|
2424
|
+
if (lastBlock) break;
|
|
2425
|
+
}
|
|
2426
|
+
if (contentChecksum) pos += 4;
|
|
2427
|
+
if (pos !== frame.length) return Infinity;
|
|
2428
|
+
return Math.max(contentSize || windowSize, blockOutput);
|
|
2429
|
+
}
|
|
1680
2430
|
function parseManifest(data) {
|
|
1681
2431
|
const bb = new import_flatbuffers3.ByteBuffer(data);
|
|
1682
2432
|
const fbsManifest = Manifest.getRootAsManifest(bb);
|
|
@@ -1685,17 +2435,41 @@ function parseManifest(data) {
|
|
|
1685
2435
|
const id = asObjectId12(
|
|
1686
2436
|
idObj.bb.bytes().slice(idObj.bb_pos, idObj.bb_pos + 12)
|
|
1687
2437
|
);
|
|
2438
|
+
const decodeLocation = makeLocationDecoder(fbsManifest);
|
|
1688
2439
|
const arraysLength = fbsManifest.arraysLength();
|
|
1689
2440
|
const arrays = [];
|
|
1690
2441
|
for (let i = 0; i < arraysLength; i++) {
|
|
1691
2442
|
const fbsArray = fbsManifest.arrays(i);
|
|
1692
2443
|
if (fbsArray) {
|
|
1693
|
-
arrays.push(parseArrayManifest(fbsArray));
|
|
2444
|
+
arrays.push(parseArrayManifest(fbsArray, decodeLocation));
|
|
1694
2445
|
}
|
|
1695
2446
|
}
|
|
1696
2447
|
return { id, arrays };
|
|
1697
2448
|
}
|
|
1698
|
-
function
|
|
2449
|
+
function makeLocationDecoder(fbsManifest) {
|
|
2450
|
+
const dictionary = fbsManifest.compressionAlgorithm() === COMPRESSION_ALG_ZSTD_DICT ? fbsManifest.locationDictionaryArray() : null;
|
|
2451
|
+
const textDecoder = new TextDecoder("utf-8", { fatal: true });
|
|
2452
|
+
return (compressed) => {
|
|
2453
|
+
if (!dictionary) {
|
|
2454
|
+
throw new Error(
|
|
2455
|
+
"ChunkRef has a compressed_location but the manifest has no location dictionary"
|
|
2456
|
+
);
|
|
2457
|
+
}
|
|
2458
|
+
if (zstdFrameAllocSize(compressed) > MAX_DECOMPRESSED_LOCATION_SIZE) {
|
|
2459
|
+
throw new Error(
|
|
2460
|
+
`Compressed location declares a size over ${MAX_DECOMPRESSED_LOCATION_SIZE} bytes`
|
|
2461
|
+
);
|
|
2462
|
+
}
|
|
2463
|
+
const decompressed = decompress(compressed, void 0, dictionary);
|
|
2464
|
+
if (decompressed.length > MAX_DECOMPRESSED_LOCATION_SIZE) {
|
|
2465
|
+
throw new Error(
|
|
2466
|
+
`Decompressed location exceeds ${MAX_DECOMPRESSED_LOCATION_SIZE} bytes`
|
|
2467
|
+
);
|
|
2468
|
+
}
|
|
2469
|
+
return textDecoder.decode(decompressed);
|
|
2470
|
+
};
|
|
2471
|
+
}
|
|
2472
|
+
function parseArrayManifest(fbsArray, decodeLocation) {
|
|
1699
2473
|
const nodeIdObj = fbsArray.nodeId();
|
|
1700
2474
|
if (!nodeIdObj)
|
|
1701
2475
|
throw new Error("ArrayManifest missing required node_id field");
|
|
@@ -1707,12 +2481,12 @@ function parseArrayManifest(fbsArray) {
|
|
|
1707
2481
|
for (let i = 0; i < refsLength; i++) {
|
|
1708
2482
|
const fbsRef = fbsArray.refs(i);
|
|
1709
2483
|
if (fbsRef) {
|
|
1710
|
-
refs.push(parseChunkRef(fbsRef));
|
|
2484
|
+
refs.push(parseChunkRef(fbsRef, decodeLocation));
|
|
1711
2485
|
}
|
|
1712
2486
|
}
|
|
1713
2487
|
return { nodeId, refs };
|
|
1714
2488
|
}
|
|
1715
|
-
function parseChunkRef(fbsRef) {
|
|
2489
|
+
function parseChunkRef(fbsRef, decodeLocation) {
|
|
1716
2490
|
const indexLength = fbsRef.indexLength();
|
|
1717
2491
|
const index = [];
|
|
1718
2492
|
for (let i = 0; i < indexLength; i++) {
|
|
@@ -1726,19 +2500,37 @@ function parseChunkRef(fbsRef) {
|
|
|
1726
2500
|
const chunkId = chunkIdObj ? asObjectId12(
|
|
1727
2501
|
chunkIdObj.bb.bytes().slice(chunkIdObj.bb_pos, chunkIdObj.bb_pos + 12)
|
|
1728
2502
|
) : null;
|
|
1729
|
-
const location = fbsRef.location();
|
|
1730
2503
|
const checksumEtag = fbsRef.checksumEtag();
|
|
1731
2504
|
const checksumLastModified = fbsRef.checksumLastModified();
|
|
1732
|
-
|
|
2505
|
+
const ref = {
|
|
1733
2506
|
index,
|
|
1734
2507
|
inline,
|
|
1735
2508
|
offset,
|
|
1736
2509
|
length,
|
|
1737
2510
|
chunkId,
|
|
1738
|
-
location,
|
|
2511
|
+
location: fbsRef.location(),
|
|
1739
2512
|
checksumEtag,
|
|
1740
2513
|
checksumLastModified
|
|
1741
2514
|
};
|
|
2515
|
+
if (chunkId === null) {
|
|
2516
|
+
const compressed = fbsRef.compressedLocationArray();
|
|
2517
|
+
if (compressed) {
|
|
2518
|
+
let decoded = null;
|
|
2519
|
+
let resolved = false;
|
|
2520
|
+
Object.defineProperty(ref, "location", {
|
|
2521
|
+
configurable: true,
|
|
2522
|
+
enumerable: true,
|
|
2523
|
+
get() {
|
|
2524
|
+
if (!resolved) {
|
|
2525
|
+
decoded = decodeLocation(compressed);
|
|
2526
|
+
resolved = true;
|
|
2527
|
+
}
|
|
2528
|
+
return decoded;
|
|
2529
|
+
}
|
|
2530
|
+
});
|
|
2531
|
+
}
|
|
2532
|
+
}
|
|
2533
|
+
return ref;
|
|
1742
2534
|
}
|
|
1743
2535
|
function findChunkRef(manifest, nodeId, coords) {
|
|
1744
2536
|
const arrayManifest = binarySearchArrayManifest(manifest.arrays, nodeId);
|
|
@@ -1792,9 +2584,6 @@ function binarySearchChunkRef(refs, coords) {
|
|
|
1792
2584
|
return null;
|
|
1793
2585
|
}
|
|
1794
2586
|
function getChunkPayload(ref) {
|
|
1795
|
-
if (ref.inline !== null) {
|
|
1796
|
-
return { type: "inline", data: ref.inline };
|
|
1797
|
-
}
|
|
1798
2587
|
if (ref.chunkId !== null) {
|
|
1799
2588
|
return {
|
|
1800
2589
|
type: "native",
|
|
@@ -1813,16 +2602,19 @@ function getChunkPayload(ref) {
|
|
|
1813
2602
|
checksumLastModified: ref.checksumLastModified
|
|
1814
2603
|
};
|
|
1815
2604
|
}
|
|
1816
|
-
|
|
2605
|
+
if (ref.inline !== null) {
|
|
2606
|
+
return { type: "inline", data: ref.inline };
|
|
2607
|
+
}
|
|
2608
|
+
throw new Error("Invalid ChunkRef: no chunkId, location, or inline");
|
|
1817
2609
|
}
|
|
1818
2610
|
|
|
1819
2611
|
// src/format/flatbuffers/metadata.ts
|
|
1820
2612
|
var import_msgpack = require("@msgpack/msgpack");
|
|
1821
|
-
var
|
|
2613
|
+
var flexbuffers2 = __toESM(require("flatbuffers/js/flexbuffers.js"), 1);
|
|
1822
2614
|
function deserializeMetadata(items, specVersion) {
|
|
1823
2615
|
const result = /* @__PURE__ */ Object.create(null);
|
|
1824
2616
|
for (const item of items) {
|
|
1825
|
-
result[item.name] = specVersion === 1 /* V1_0 */ ? (0, import_msgpack.decode)(item.value) :
|
|
2617
|
+
result[item.name] = specVersion === 1 /* V1_0 */ ? (0, import_msgpack.decode)(item.value) : flexbuffers2.toObject(
|
|
1826
2618
|
item.value.buffer.slice(
|
|
1827
2619
|
item.value.byteOffset,
|
|
1828
2620
|
item.value.byteOffset + item.value.byteLength
|
|
@@ -2154,17 +2946,219 @@ function parseTransactionLog(data) {
|
|
|
2154
2946
|
};
|
|
2155
2947
|
}
|
|
2156
2948
|
|
|
2949
|
+
// src/reader/range-coalescer.ts
|
|
2950
|
+
function expectedRangeLength(range) {
|
|
2951
|
+
return "suffixLength" in range ? range.suffixLength : range.length;
|
|
2952
|
+
}
|
|
2953
|
+
function makeUrlStore(opts) {
|
|
2954
|
+
const { url, fetchClient, conditionalHeaders } = opts;
|
|
2955
|
+
async function doFetch(init) {
|
|
2956
|
+
return fetchClient ? await fetchClient.fetch(url, init) : await fetch(url, init);
|
|
2957
|
+
}
|
|
2958
|
+
return {
|
|
2959
|
+
async get() {
|
|
2960
|
+
throw new Error(
|
|
2961
|
+
`Virtual chunk URL store for ${url} only supports ranged reads`
|
|
2962
|
+
);
|
|
2963
|
+
},
|
|
2964
|
+
async getRange(_key, range, options) {
|
|
2965
|
+
const headers = conditionalHeaders ? { ...conditionalHeaders } : {};
|
|
2966
|
+
headers.Range = "suffixLength" in range ? `bytes=-${range.suffixLength}` : `bytes=${range.offset}-${range.offset + range.length - 1}`;
|
|
2967
|
+
const response = await doFetch({ headers, signal: options?.signal });
|
|
2968
|
+
if (response.status === 412) {
|
|
2969
|
+
throw new Error(
|
|
2970
|
+
`Virtual chunk at ${url} failed integrity check \u2014 data has been modified since snapshot was created`
|
|
2971
|
+
);
|
|
2972
|
+
}
|
|
2973
|
+
if (response.status !== 200 && response.status !== 206) {
|
|
2974
|
+
throw new Error(
|
|
2975
|
+
`Failed to fetch virtual chunk from ${url}: ${response.status} ${response.statusText}`
|
|
2976
|
+
);
|
|
2977
|
+
}
|
|
2978
|
+
const data = new Uint8Array(await response.arrayBuffer());
|
|
2979
|
+
if (response.status === 206) {
|
|
2980
|
+
const expected = expectedRangeLength(range);
|
|
2981
|
+
if (data.length === expected) return data;
|
|
2982
|
+
throw new Error(
|
|
2983
|
+
`Virtual range response size mismatch for ${url}: expected ${expected} bytes, got ${data.length}`
|
|
2984
|
+
);
|
|
2985
|
+
}
|
|
2986
|
+
if ("offset" in range) {
|
|
2987
|
+
const end = range.offset + range.length;
|
|
2988
|
+
if (data.length >= end) return data.slice(range.offset, end);
|
|
2989
|
+
throw new Error(
|
|
2990
|
+
`Virtual range request not honored for ${url}: need at least ${end} bytes for fallback slicing, got ${data.length}`
|
|
2991
|
+
);
|
|
2992
|
+
}
|
|
2993
|
+
if (data.length >= range.suffixLength) {
|
|
2994
|
+
return data.slice(data.length - range.suffixLength);
|
|
2995
|
+
}
|
|
2996
|
+
throw new Error(
|
|
2997
|
+
`Virtual suffix range request not honored for ${url}: need at least ${range.suffixLength} bytes for fallback slicing, got ${data.length}`
|
|
2998
|
+
);
|
|
2999
|
+
}
|
|
3000
|
+
};
|
|
3001
|
+
}
|
|
3002
|
+
function makeStorageStore(storage) {
|
|
3003
|
+
return {
|
|
3004
|
+
async get(key, options) {
|
|
3005
|
+
const storageOptions = options?.signal ? { signal: options.signal } : void 0;
|
|
3006
|
+
return storage.getObject(key, void 0, storageOptions);
|
|
3007
|
+
},
|
|
3008
|
+
async getRange(key, range, options) {
|
|
3009
|
+
const storageOptions = options?.signal ? { signal: options.signal } : void 0;
|
|
3010
|
+
if ("suffixLength" in range) {
|
|
3011
|
+
throw new Error(
|
|
3012
|
+
`Storage suffix ranges are not supported for ${key}; convert suffixLength to offset/length before reading`
|
|
3013
|
+
);
|
|
3014
|
+
}
|
|
3015
|
+
const storageRange = {
|
|
3016
|
+
start: range.offset,
|
|
3017
|
+
end: range.offset + range.length
|
|
3018
|
+
};
|
|
3019
|
+
const data = await storage.getObject(key, storageRange, storageOptions);
|
|
3020
|
+
if (data.length === range.length) return data;
|
|
3021
|
+
if (data.length >= storageRange.end) {
|
|
3022
|
+
return data.slice(storageRange.start, storageRange.end);
|
|
3023
|
+
}
|
|
3024
|
+
throw new Error(
|
|
3025
|
+
`Storage returned ${data.length} bytes for ${key} range ${storageRange.start}-${storageRange.end - 1}; expected ${range.length} bytes`
|
|
3026
|
+
);
|
|
3027
|
+
}
|
|
3028
|
+
};
|
|
3029
|
+
}
|
|
3030
|
+
|
|
2157
3031
|
// src/reader/session.ts
|
|
3032
|
+
var RANGE_COALESCE_SIZE = 32 * 1024;
|
|
3033
|
+
var RANGE_STORE_CACHE_SIZE = 256;
|
|
3034
|
+
function makeRangeStoreCacheKey(parts) {
|
|
3035
|
+
return JSON.stringify(parts);
|
|
3036
|
+
}
|
|
2158
3037
|
var ReadSession = class _ReadSession {
|
|
2159
3038
|
storage;
|
|
2160
3039
|
snapshot;
|
|
2161
3040
|
specVersion;
|
|
2162
3041
|
manifestCache;
|
|
2163
|
-
|
|
3042
|
+
/**
|
|
3043
|
+
* Single-flight loader over `manifestCache`: concurrent misses for the
|
|
3044
|
+
* same manifest share one fetch instead of racing N identical GETs.
|
|
3045
|
+
* Critical for fan-out reads that all need the same manifest (e.g. the
|
|
3046
|
+
* many parallel chunk reads issued when a renderer crosses into a new
|
|
3047
|
+
* pyramid level).
|
|
3048
|
+
*/
|
|
3049
|
+
manifestLoader;
|
|
3050
|
+
/**
|
|
3051
|
+
* Bounded cache of `AsyncReadable`s wrapped with zarrita's range coalescer.
|
|
3052
|
+
* The cache key includes request-option identities that must not share one
|
|
3053
|
+
* coalescing queue, notably virtual fetch clients and checksum headers.
|
|
3054
|
+
*
|
|
3055
|
+
* Promise slots are inserted synchronously on first use, so
|
|
3056
|
+
* concurrent requests for the same partition share one coalescing window
|
|
3057
|
+
* instead of racing to create parallel stores.
|
|
3058
|
+
*/
|
|
3059
|
+
rangeStores;
|
|
3060
|
+
nativeStore;
|
|
3061
|
+
fetchClientIds;
|
|
3062
|
+
nextFetchClientId = 1;
|
|
3063
|
+
rangeCoalescerIds;
|
|
3064
|
+
nextRangeCoalescerId = 1;
|
|
3065
|
+
/** VCC name → url_prefix map for resolving `vcc://` chunk locations. */
|
|
3066
|
+
virtualChunkContainers;
|
|
3067
|
+
constructor(storage, snapshot, specVersion, maxManifestCacheSize = 100, virtualChunkContainers) {
|
|
2164
3068
|
this.storage = storage;
|
|
2165
3069
|
this.snapshot = snapshot;
|
|
2166
3070
|
this.specVersion = specVersion;
|
|
2167
3071
|
this.manifestCache = new LRUCache(maxManifestCacheSize);
|
|
3072
|
+
this.manifestLoader = singleFlight(this.manifestCache);
|
|
3073
|
+
this.virtualChunkContainers = virtualChunkContainers ?? /* @__PURE__ */ new Map();
|
|
3074
|
+
}
|
|
3075
|
+
getFetchClientKey(fetchClient) {
|
|
3076
|
+
if (!fetchClient) return "default";
|
|
3077
|
+
if (!this.fetchClientIds) this.fetchClientIds = /* @__PURE__ */ new WeakMap();
|
|
3078
|
+
let id = this.fetchClientIds.get(fetchClient);
|
|
3079
|
+
if (id === void 0) {
|
|
3080
|
+
id = this.nextFetchClientId;
|
|
3081
|
+
this.nextFetchClientId = id + 1;
|
|
3082
|
+
this.fetchClientIds.set(fetchClient, id);
|
|
3083
|
+
}
|
|
3084
|
+
return String(id);
|
|
3085
|
+
}
|
|
3086
|
+
getRangeCoalescerKey(withRangeCoalescing) {
|
|
3087
|
+
if (!this.rangeCoalescerIds) this.rangeCoalescerIds = /* @__PURE__ */ new WeakMap();
|
|
3088
|
+
let id = this.rangeCoalescerIds.get(withRangeCoalescing);
|
|
3089
|
+
if (id === void 0) {
|
|
3090
|
+
id = this.nextRangeCoalescerId;
|
|
3091
|
+
this.nextRangeCoalescerId = id + 1;
|
|
3092
|
+
this.rangeCoalescerIds.set(withRangeCoalescing, id);
|
|
3093
|
+
}
|
|
3094
|
+
return String(id);
|
|
3095
|
+
}
|
|
3096
|
+
getRangeStore(partitionKey, createStore, withRangeCoalescing) {
|
|
3097
|
+
if (!this.rangeStores) {
|
|
3098
|
+
this.rangeStores = new LRUCache(RANGE_STORE_CACHE_SIZE);
|
|
3099
|
+
}
|
|
3100
|
+
const stores = this.rangeStores;
|
|
3101
|
+
const cacheKey = makeRangeStoreCacheKey([
|
|
3102
|
+
...partitionKey,
|
|
3103
|
+
["coalescer", this.getRangeCoalescerKey(withRangeCoalescing)]
|
|
3104
|
+
]);
|
|
3105
|
+
const cached = stores.get(cacheKey);
|
|
3106
|
+
if (cached) return cached;
|
|
3107
|
+
const raw = createStore();
|
|
3108
|
+
const promise = Promise.resolve().then(
|
|
3109
|
+
() => withRangeCoalescing(raw, { coalesceSize: RANGE_COALESCE_SIZE })
|
|
3110
|
+
).catch((error) => {
|
|
3111
|
+
if (stores.get(cacheKey) === promise) stores.delete(cacheKey);
|
|
3112
|
+
throw error;
|
|
3113
|
+
});
|
|
3114
|
+
stores.set(cacheKey, promise);
|
|
3115
|
+
return promise;
|
|
3116
|
+
}
|
|
3117
|
+
getNativeStore(options) {
|
|
3118
|
+
if (!this.nativeStore) {
|
|
3119
|
+
this.nativeStore = makeStorageStore(this.storage);
|
|
3120
|
+
}
|
|
3121
|
+
const raw = this.nativeStore;
|
|
3122
|
+
const withRangeCoalescing = options?.withRangeCoalescing;
|
|
3123
|
+
if (!withRangeCoalescing) {
|
|
3124
|
+
return Promise.resolve(raw);
|
|
3125
|
+
}
|
|
3126
|
+
return this.getRangeStore(["native"], () => raw, withRangeCoalescing);
|
|
3127
|
+
}
|
|
3128
|
+
getVirtualStoreForPayload(httpUrl, payload, options) {
|
|
3129
|
+
const validate = !!options?.validateChecksums;
|
|
3130
|
+
let conditionalHeaders;
|
|
3131
|
+
if (validate) {
|
|
3132
|
+
conditionalHeaders = {};
|
|
3133
|
+
if (payload.checksumEtag) {
|
|
3134
|
+
conditionalHeaders["If-Match"] = payload.checksumEtag;
|
|
3135
|
+
}
|
|
3136
|
+
if (payload.checksumLastModified > 0) {
|
|
3137
|
+
conditionalHeaders["If-Unmodified-Since"] = new Date(
|
|
3138
|
+
payload.checksumLastModified * 1e3
|
|
3139
|
+
).toUTCString();
|
|
3140
|
+
}
|
|
3141
|
+
}
|
|
3142
|
+
const checksumKey = validate ? ["checked", payload.checksumEtag ?? "", payload.checksumLastModified] : ["unchecked"];
|
|
3143
|
+
const createStore = () => makeUrlStore({
|
|
3144
|
+
url: httpUrl,
|
|
3145
|
+
fetchClient: options?.fetchClient,
|
|
3146
|
+
conditionalHeaders
|
|
3147
|
+
});
|
|
3148
|
+
const withRangeCoalescing = options?.withRangeCoalescing;
|
|
3149
|
+
if (!withRangeCoalescing) {
|
|
3150
|
+
return Promise.resolve(createStore());
|
|
3151
|
+
}
|
|
3152
|
+
return this.getRangeStore(
|
|
3153
|
+
[
|
|
3154
|
+
"virtual",
|
|
3155
|
+
httpUrl,
|
|
3156
|
+
["fetch", this.getFetchClientKey(options?.fetchClient)],
|
|
3157
|
+
checksumKey
|
|
3158
|
+
],
|
|
3159
|
+
createStore,
|
|
3160
|
+
withRangeCoalescing
|
|
3161
|
+
);
|
|
2168
3162
|
}
|
|
2169
3163
|
/**
|
|
2170
3164
|
* Open a read session for a specific snapshot.
|
|
@@ -2184,7 +3178,8 @@ var ReadSession = class _ReadSession {
|
|
|
2184
3178
|
storage,
|
|
2185
3179
|
snapshot,
|
|
2186
3180
|
specVersion,
|
|
2187
|
-
options?.maxManifestCacheSize
|
|
3181
|
+
options?.maxManifestCacheSize,
|
|
3182
|
+
options?.virtualChunkContainers
|
|
2188
3183
|
);
|
|
2189
3184
|
}
|
|
2190
3185
|
/** Load and parse a snapshot from storage */
|
|
@@ -2195,29 +3190,47 @@ var ReadSession = class _ReadSession {
|
|
|
2195
3190
|
validateFileType(header, 1 /* Snapshot */);
|
|
2196
3191
|
let flatbufferData = getDataAfterHeader(data);
|
|
2197
3192
|
if (header.compression === 1 /* Zstd */) {
|
|
2198
|
-
flatbufferData =
|
|
3193
|
+
flatbufferData = decompress(flatbufferData);
|
|
2199
3194
|
}
|
|
2200
3195
|
return {
|
|
2201
3196
|
snapshot: parseSnapshot(flatbufferData),
|
|
2202
3197
|
specVersion: header.specVersion
|
|
2203
3198
|
};
|
|
2204
3199
|
}
|
|
2205
|
-
/**
|
|
2206
|
-
|
|
3200
|
+
/**
|
|
3201
|
+
* Load and parse a manifest from storage.
|
|
3202
|
+
*
|
|
3203
|
+
* Concurrent calls for the same manifest are coalesced through
|
|
3204
|
+
* `manifestLoader` so a fan-out of chunk reads — the typical case
|
|
3205
|
+
* when many tiles cross into a new pyramid level at once — issues one
|
|
3206
|
+
* HTTP GET instead of one per caller. A caller's `signal` rejects only
|
|
3207
|
+
* that caller's await; the shared fetch is aborted only once every
|
|
3208
|
+
* active waiter has aborted.
|
|
3209
|
+
*/
|
|
3210
|
+
loadManifest(manifestId, options) {
|
|
2207
3211
|
const idStr = encodeObjectId12(manifestId);
|
|
2208
|
-
|
|
2209
|
-
|
|
3212
|
+
return this.manifestLoader.load(
|
|
3213
|
+
idStr,
|
|
3214
|
+
(signal) => this.fetchManifest(idStr, signal),
|
|
3215
|
+
options?.signal
|
|
3216
|
+
);
|
|
3217
|
+
}
|
|
3218
|
+
/**
|
|
3219
|
+
* Fetch and parse a single manifest. The `signal` here is the
|
|
3220
|
+
* single-flight loader's own signal — it fires only when every waiter
|
|
3221
|
+
* has aborted, so a hung manifest fetch doesn't trap later callers
|
|
3222
|
+
* behind it.
|
|
3223
|
+
*/
|
|
3224
|
+
async fetchManifest(idStr, signal) {
|
|
2210
3225
|
const path = getManifestPath(idStr);
|
|
2211
|
-
const data = await this.storage.getObject(path, void 0,
|
|
3226
|
+
const data = await this.storage.getObject(path, void 0, { signal });
|
|
2212
3227
|
const header = parseHeader(data);
|
|
2213
3228
|
validateFileType(header, 2 /* Manifest */);
|
|
2214
3229
|
let flatbufferData = getDataAfterHeader(data);
|
|
2215
3230
|
if (header.compression === 1 /* Zstd */) {
|
|
2216
|
-
flatbufferData =
|
|
3231
|
+
flatbufferData = decompress(flatbufferData);
|
|
2217
3232
|
}
|
|
2218
|
-
|
|
2219
|
-
this.manifestCache.set(idStr, manifest);
|
|
2220
|
-
return manifest;
|
|
3233
|
+
return parseManifest(flatbufferData);
|
|
2221
3234
|
}
|
|
2222
3235
|
/**
|
|
2223
3236
|
* Get the snapshot ID.
|
|
@@ -2279,7 +3292,7 @@ var ReadSession = class _ReadSession {
|
|
|
2279
3292
|
validateFileType(header, 4 /* TransactionLog */);
|
|
2280
3293
|
let flatbufferData = getDataAfterHeader(data);
|
|
2281
3294
|
if (header.compression === 1 /* Zstd */) {
|
|
2282
|
-
flatbufferData =
|
|
3295
|
+
flatbufferData = decompress(flatbufferData);
|
|
2283
3296
|
}
|
|
2284
3297
|
return parseTransactionLog(flatbufferData);
|
|
2285
3298
|
}
|
|
@@ -2348,26 +3361,25 @@ var ReadSession = class _ReadSession {
|
|
|
2348
3361
|
* @returns Chunk data bytes or null if not found
|
|
2349
3362
|
*/
|
|
2350
3363
|
async getChunk(path, coords, options) {
|
|
2351
|
-
|
|
3364
|
+
options?.signal?.throwIfAborted();
|
|
3365
|
+
const requestOptions = toRequestOptions(options);
|
|
2352
3366
|
const node = this.getNode(path);
|
|
2353
3367
|
if (!node || node.nodeData.type !== "array") {
|
|
2354
3368
|
return null;
|
|
2355
3369
|
}
|
|
2356
3370
|
const arrayData = node.nodeData;
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
continue;
|
|
2361
|
-
}
|
|
2362
|
-
const manifest = await this.loadManifest(manifestRef.objectId, options);
|
|
2363
|
-
const chunkRef = findChunkRef(manifest, node.id, coords);
|
|
2364
|
-
if (!chunkRef) continue;
|
|
2365
|
-
const payload = getChunkPayload(chunkRef);
|
|
2366
|
-
return this.fetchChunkPayload(payload, options);
|
|
3371
|
+
for (const manifestRef of arrayData.manifests) {
|
|
3372
|
+
if (!this.coordsInExtents(coords, manifestRef.extents)) {
|
|
3373
|
+
continue;
|
|
2367
3374
|
}
|
|
2368
|
-
|
|
2369
|
-
|
|
2370
|
-
|
|
3375
|
+
const manifest = await this.loadManifest(
|
|
3376
|
+
manifestRef.objectId,
|
|
3377
|
+
requestOptions
|
|
3378
|
+
);
|
|
3379
|
+
const chunkRef = findChunkRef(manifest, node.id, coords);
|
|
3380
|
+
if (!chunkRef) continue;
|
|
3381
|
+
const payload = getChunkPayload(chunkRef);
|
|
3382
|
+
return this.fetchChunkPayload(payload, options);
|
|
2371
3383
|
}
|
|
2372
3384
|
return null;
|
|
2373
3385
|
}
|
|
@@ -2385,26 +3397,25 @@ var ReadSession = class _ReadSession {
|
|
|
2385
3397
|
* @returns Chunk data bytes or null if not found
|
|
2386
3398
|
*/
|
|
2387
3399
|
async getChunkRange(path, coords, range, options) {
|
|
2388
|
-
|
|
3400
|
+
options?.signal?.throwIfAborted();
|
|
3401
|
+
const requestOptions = toRequestOptions(options);
|
|
2389
3402
|
const node = this.getNode(path);
|
|
2390
3403
|
if (!node || node.nodeData.type !== "array") {
|
|
2391
3404
|
return null;
|
|
2392
3405
|
}
|
|
2393
3406
|
const arrayData = node.nodeData;
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
continue;
|
|
2398
|
-
}
|
|
2399
|
-
const manifest = await this.loadManifest(manifestRef.objectId, options);
|
|
2400
|
-
const chunkRef = findChunkRef(manifest, node.id, coords);
|
|
2401
|
-
if (!chunkRef) continue;
|
|
2402
|
-
const payload = getChunkPayload(chunkRef);
|
|
2403
|
-
return this.fetchChunkPayloadRange(payload, range, options);
|
|
3407
|
+
for (const manifestRef of arrayData.manifests) {
|
|
3408
|
+
if (!this.coordsInExtents(coords, manifestRef.extents)) {
|
|
3409
|
+
continue;
|
|
2404
3410
|
}
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
3411
|
+
const manifest = await this.loadManifest(
|
|
3412
|
+
manifestRef.objectId,
|
|
3413
|
+
requestOptions
|
|
3414
|
+
);
|
|
3415
|
+
const chunkRef = findChunkRef(manifest, node.id, coords);
|
|
3416
|
+
if (!chunkRef) continue;
|
|
3417
|
+
const payload = getChunkPayload(chunkRef);
|
|
3418
|
+
return this.fetchChunkPayloadRange(payload, range, options);
|
|
2408
3419
|
}
|
|
2409
3420
|
return null;
|
|
2410
3421
|
}
|
|
@@ -2426,77 +3437,54 @@ var ReadSession = class _ReadSession {
|
|
|
2426
3437
|
return payload.data;
|
|
2427
3438
|
case "native": {
|
|
2428
3439
|
const path = getChunkPath(encodeObjectId12(payload.chunkId));
|
|
2429
|
-
const
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
|
|
3440
|
+
const requestedStart = payload.offset;
|
|
3441
|
+
const requestedEnd = payload.offset + payload.length;
|
|
3442
|
+
const store = await this.getNativeStore(options);
|
|
3443
|
+
const data = await store.getRange(
|
|
3444
|
+
path,
|
|
3445
|
+
{ offset: requestedStart, length: payload.length },
|
|
3446
|
+
{ signal: options?.signal }
|
|
3447
|
+
);
|
|
3448
|
+
if (!data) {
|
|
3449
|
+
throw new Error(
|
|
3450
|
+
`Failed to fetch native chunk from ${path} range ${requestedStart}-${requestedEnd - 1}: empty response`
|
|
3451
|
+
);
|
|
2437
3452
|
}
|
|
3453
|
+
if (data.length === payload.length) return data;
|
|
2438
3454
|
throw new Error(
|
|
2439
|
-
`Storage returned ${data.length} bytes for ${path} range ${
|
|
3455
|
+
`Storage returned ${data.length} bytes for ${path} range ${requestedStart}-${requestedEnd - 1}; expected ${payload.length} bytes`
|
|
2440
3456
|
);
|
|
2441
3457
|
}
|
|
2442
3458
|
case "virtual": {
|
|
2443
|
-
const
|
|
3459
|
+
const absoluteLocation = expandVccUrl(
|
|
2444
3460
|
payload.location,
|
|
3461
|
+
this.virtualChunkContainers
|
|
3462
|
+
);
|
|
3463
|
+
const httpUrl = translateToHttpUrl(
|
|
3464
|
+
absoluteLocation,
|
|
2445
3465
|
options?.azureAccount
|
|
2446
3466
|
);
|
|
2447
|
-
const
|
|
2448
|
-
|
|
2449
|
-
|
|
2450
|
-
|
|
2451
|
-
|
|
2452
|
-
|
|
2453
|
-
|
|
2454
|
-
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
|
|
2458
|
-
}
|
|
2459
|
-
}
|
|
2460
|
-
const fetchInit = {
|
|
2461
|
-
headers,
|
|
2462
|
-
signal: options?.signal
|
|
2463
|
-
};
|
|
2464
|
-
let response;
|
|
2465
|
-
try {
|
|
2466
|
-
const client = options?.fetchClient;
|
|
2467
|
-
response = client ? await client.fetch(httpUrl, fetchInit) : await fetch(httpUrl, fetchInit);
|
|
2468
|
-
} catch (error) {
|
|
2469
|
-
if (error instanceof Error && error.name === "AbortError") {
|
|
2470
|
-
throw new AbortError();
|
|
2471
|
-
}
|
|
2472
|
-
throw error;
|
|
2473
|
-
}
|
|
2474
|
-
if (response.status === 412) {
|
|
3467
|
+
const store = await this.getVirtualStoreForPayload(
|
|
3468
|
+
httpUrl,
|
|
3469
|
+
payload,
|
|
3470
|
+
options
|
|
3471
|
+
);
|
|
3472
|
+
const data = await store.getRange(
|
|
3473
|
+
"/",
|
|
3474
|
+
{ offset: payload.offset, length: payload.length },
|
|
3475
|
+
{ signal: options?.signal }
|
|
3476
|
+
);
|
|
3477
|
+
if (!data) {
|
|
2475
3478
|
throw new Error(
|
|
2476
|
-
`
|
|
3479
|
+
`Failed to fetch virtual chunk from ${httpUrl}: empty response`
|
|
2477
3480
|
);
|
|
2478
3481
|
}
|
|
2479
|
-
if (
|
|
3482
|
+
if (data.length !== payload.length) {
|
|
2480
3483
|
throw new Error(
|
|
2481
|
-
`
|
|
3484
|
+
`Virtual range response size mismatch for ${httpUrl}: expected ${payload.length} bytes, got ${data.length}`
|
|
2482
3485
|
);
|
|
2483
3486
|
}
|
|
2484
|
-
|
|
2485
|
-
if (response.status === 206) {
|
|
2486
|
-
if (data.length !== payload.length) {
|
|
2487
|
-
throw new Error(
|
|
2488
|
-
`Virtual range response size mismatch for ${httpUrl}: expected ${payload.length} bytes, got ${data.length}`
|
|
2489
|
-
);
|
|
2490
|
-
}
|
|
2491
|
-
return data;
|
|
2492
|
-
}
|
|
2493
|
-
const absoluteEnd = payload.offset + payload.length;
|
|
2494
|
-
if (data.length >= absoluteEnd) {
|
|
2495
|
-
return data.slice(payload.offset, absoluteEnd);
|
|
2496
|
-
}
|
|
2497
|
-
throw new Error(
|
|
2498
|
-
`Virtual range request not honored for ${httpUrl}: need at least ${absoluteEnd} bytes for fallback slicing, got ${data.length}`
|
|
2499
|
-
);
|
|
3487
|
+
return data;
|
|
2500
3488
|
}
|
|
2501
3489
|
}
|
|
2502
3490
|
}
|
|
@@ -2516,80 +3504,57 @@ var ReadSession = class _ReadSession {
|
|
|
2516
3504
|
return payload.data.slice(rangeStart, rangeEnd);
|
|
2517
3505
|
case "native": {
|
|
2518
3506
|
const path = getChunkPath(encodeObjectId12(payload.chunkId));
|
|
2519
|
-
const
|
|
2520
|
-
start: payload.offset + rangeStart,
|
|
2521
|
-
end: payload.offset + rangeEnd
|
|
2522
|
-
};
|
|
3507
|
+
const requestedStart = payload.offset + rangeStart;
|
|
2523
3508
|
const expectedSize = rangeEnd - rangeStart;
|
|
2524
|
-
const
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
|
|
3509
|
+
const requestedEnd = requestedStart + expectedSize;
|
|
3510
|
+
const store = await this.getNativeStore(options);
|
|
3511
|
+
const data = await store.getRange(
|
|
3512
|
+
path,
|
|
3513
|
+
{ offset: requestedStart, length: expectedSize },
|
|
3514
|
+
{ signal: options?.signal }
|
|
3515
|
+
);
|
|
3516
|
+
if (!data) {
|
|
3517
|
+
throw new Error(
|
|
3518
|
+
`Failed to fetch native chunk from ${path} range ${requestedStart}-${requestedEnd - 1}: empty response`
|
|
3519
|
+
);
|
|
2528
3520
|
}
|
|
3521
|
+
if (data.length === expectedSize) return data;
|
|
2529
3522
|
throw new Error(
|
|
2530
|
-
`Storage returned ${data.length} bytes for ${path} range ${
|
|
3523
|
+
`Storage returned ${data.length} bytes for ${path} range ${requestedStart}-${requestedEnd - 1}; expected ${expectedSize} bytes`
|
|
2531
3524
|
);
|
|
2532
3525
|
}
|
|
2533
3526
|
case "virtual": {
|
|
2534
3527
|
const absoluteStart = payload.offset + rangeStart;
|
|
2535
|
-
const
|
|
2536
|
-
const
|
|
3528
|
+
const expectedSize = rangeEnd - rangeStart;
|
|
3529
|
+
const absoluteLocation = expandVccUrl(
|
|
2537
3530
|
payload.location,
|
|
3531
|
+
this.virtualChunkContainers
|
|
3532
|
+
);
|
|
3533
|
+
const httpUrl = translateToHttpUrl(
|
|
3534
|
+
absoluteLocation,
|
|
2538
3535
|
options?.azureAccount
|
|
2539
3536
|
);
|
|
2540
|
-
const
|
|
2541
|
-
|
|
2542
|
-
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
|
|
2548
|
-
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
}
|
|
2552
|
-
}
|
|
2553
|
-
const fetchInit = {
|
|
2554
|
-
headers,
|
|
2555
|
-
signal: options?.signal
|
|
2556
|
-
};
|
|
2557
|
-
let response;
|
|
2558
|
-
try {
|
|
2559
|
-
const client = options?.fetchClient;
|
|
2560
|
-
response = client ? await client.fetch(httpUrl, fetchInit) : await fetch(httpUrl, fetchInit);
|
|
2561
|
-
} catch (error) {
|
|
2562
|
-
if (error instanceof Error && error.name === "AbortError") {
|
|
2563
|
-
throw new AbortError();
|
|
2564
|
-
}
|
|
2565
|
-
throw error;
|
|
2566
|
-
}
|
|
2567
|
-
if (response.status === 412) {
|
|
3537
|
+
const store = await this.getVirtualStoreForPayload(
|
|
3538
|
+
httpUrl,
|
|
3539
|
+
payload,
|
|
3540
|
+
options
|
|
3541
|
+
);
|
|
3542
|
+
const data = await store.getRange(
|
|
3543
|
+
"/",
|
|
3544
|
+
{ offset: absoluteStart, length: expectedSize },
|
|
3545
|
+
{ signal: options?.signal }
|
|
3546
|
+
);
|
|
3547
|
+
if (!data) {
|
|
2568
3548
|
throw new Error(
|
|
2569
|
-
`
|
|
3549
|
+
`Failed to fetch virtual chunk from ${httpUrl}: empty response`
|
|
2570
3550
|
);
|
|
2571
3551
|
}
|
|
2572
|
-
if (
|
|
3552
|
+
if (data.length !== expectedSize) {
|
|
2573
3553
|
throw new Error(
|
|
2574
|
-
`
|
|
3554
|
+
`Virtual range response size mismatch for ${httpUrl}: expected ${expectedSize} bytes, got ${data.length}`
|
|
2575
3555
|
);
|
|
2576
3556
|
}
|
|
2577
|
-
|
|
2578
|
-
const data = new Uint8Array(await response.arrayBuffer());
|
|
2579
|
-
if (response.status === 206) {
|
|
2580
|
-
if (data.length !== expectedSize) {
|
|
2581
|
-
throw new Error(
|
|
2582
|
-
`Virtual range response size mismatch for ${httpUrl}: expected ${expectedSize} bytes, got ${data.length}`
|
|
2583
|
-
);
|
|
2584
|
-
}
|
|
2585
|
-
return data;
|
|
2586
|
-
}
|
|
2587
|
-
if (data.length >= absoluteEnd) {
|
|
2588
|
-
return data.slice(absoluteStart, absoluteEnd);
|
|
2589
|
-
}
|
|
2590
|
-
throw new Error(
|
|
2591
|
-
`Virtual range request not honored for ${httpUrl}: need at least ${absoluteEnd} bytes for fallback slicing, got ${data.length}`
|
|
2592
|
-
);
|
|
3557
|
+
return data;
|
|
2593
3558
|
}
|
|
2594
3559
|
}
|
|
2595
3560
|
}
|
|
@@ -2613,6 +3578,19 @@ var ReadSession = class _ReadSession {
|
|
|
2613
3578
|
}
|
|
2614
3579
|
};
|
|
2615
3580
|
var utf8Encoder = new TextEncoder();
|
|
3581
|
+
function toRequestOptions(options) {
|
|
3582
|
+
if (!options) return void 0;
|
|
3583
|
+
return {
|
|
3584
|
+
...options.signal && { signal: options.signal },
|
|
3585
|
+
...options.fetchClient && { fetchClient: options.fetchClient },
|
|
3586
|
+
...options.validateChecksums !== void 0 && {
|
|
3587
|
+
validateChecksums: options.validateChecksums
|
|
3588
|
+
},
|
|
3589
|
+
...options.azureAccount !== void 0 && {
|
|
3590
|
+
azureAccount: options.azureAccount
|
|
3591
|
+
}
|
|
3592
|
+
};
|
|
3593
|
+
}
|
|
2616
3594
|
function compareUtf8Bytes(a, b) {
|
|
2617
3595
|
const bytesA = utf8Encoder.encode(a);
|
|
2618
3596
|
const bytesB = utf8Encoder.encode(b);
|
|
@@ -2624,6 +3602,28 @@ function compareUtf8Bytes(a, b) {
|
|
|
2624
3602
|
}
|
|
2625
3603
|
return bytesA.length - bytesB.length;
|
|
2626
3604
|
}
|
|
3605
|
+
var VCC_SCHEME = "vcc://";
|
|
3606
|
+
function expandVccUrl(location, containers) {
|
|
3607
|
+
if (!location.startsWith(VCC_SCHEME)) return location;
|
|
3608
|
+
const rest = location.slice(VCC_SCHEME.length);
|
|
3609
|
+
const slash = rest.indexOf("/");
|
|
3610
|
+
if (slash === -1) {
|
|
3611
|
+
throw new Error(
|
|
3612
|
+
`Invalid vcc:// URL "${location}": missing "/" after container name`
|
|
3613
|
+
);
|
|
3614
|
+
}
|
|
3615
|
+
const name = rest.slice(0, slash);
|
|
3616
|
+
const relativePath = rest.slice(slash + 1);
|
|
3617
|
+
const urlPrefix = containers.get(name);
|
|
3618
|
+
if (urlPrefix === void 0) {
|
|
3619
|
+
if (containers.size === 0) return location;
|
|
3620
|
+
throw new Error(
|
|
3621
|
+
`Unknown virtual chunk container "${name}" referenced by ${location}`
|
|
3622
|
+
);
|
|
3623
|
+
}
|
|
3624
|
+
const normalizedPrefix = urlPrefix.endsWith("/") ? urlPrefix : `${urlPrefix}/`;
|
|
3625
|
+
return normalizedPrefix + relativePath;
|
|
3626
|
+
}
|
|
2627
3627
|
function translateToHttpUrl(url, azureAccount) {
|
|
2628
3628
|
if (url.startsWith("s3://")) {
|
|
2629
3629
|
const rest = url.slice(5);
|
|
@@ -2699,7 +3699,7 @@ var Repository = class _Repository {
|
|
|
2699
3699
|
* @param options - Optional request options (signal for cancellation)
|
|
2700
3700
|
* @returns ParsedRepo if v2 format, null if v1 format
|
|
2701
3701
|
* @throws Error if repo file exists but fails to parse
|
|
2702
|
-
* @throws AbortError if the operation was aborted
|
|
3702
|
+
* @throws An error named "AbortError" if the operation was aborted
|
|
2703
3703
|
*/
|
|
2704
3704
|
async loadRepoInfo(options) {
|
|
2705
3705
|
if (this.repoInfoAttempted) {
|
|
@@ -2715,7 +3715,7 @@ var Repository = class _Repository {
|
|
|
2715
3715
|
this.repoInfo = parseRepo(data);
|
|
2716
3716
|
return this.repoInfo;
|
|
2717
3717
|
} catch (error) {
|
|
2718
|
-
if (error
|
|
3718
|
+
if (isAbortError(error)) {
|
|
2719
3719
|
this.repoInfoAttempted = false;
|
|
2720
3720
|
throw error;
|
|
2721
3721
|
}
|
|
@@ -2780,7 +3780,7 @@ var Repository = class _Repository {
|
|
|
2780
3780
|
}
|
|
2781
3781
|
}
|
|
2782
3782
|
} catch (error) {
|
|
2783
|
-
if (error
|
|
3783
|
+
if (isAbortError(error)) {
|
|
2784
3784
|
throw error;
|
|
2785
3785
|
}
|
|
2786
3786
|
if (legacyPath) {
|
|
@@ -2813,7 +3813,7 @@ var Repository = class _Repository {
|
|
|
2813
3813
|
}
|
|
2814
3814
|
}
|
|
2815
3815
|
} catch (error) {
|
|
2816
|
-
if (error
|
|
3816
|
+
if (isAbortError(error)) {
|
|
2817
3817
|
throw error;
|
|
2818
3818
|
}
|
|
2819
3819
|
return legacyPath || null;
|
|
@@ -2854,7 +3854,7 @@ var Repository = class _Repository {
|
|
|
2854
3854
|
}
|
|
2855
3855
|
}
|
|
2856
3856
|
} catch (error) {
|
|
2857
|
-
if (error
|
|
3857
|
+
if (isAbortError(error)) {
|
|
2858
3858
|
throw error;
|
|
2859
3859
|
}
|
|
2860
3860
|
throw new Error("Cannot list branches: storage does not support listing");
|
|
@@ -2896,7 +3896,7 @@ var Repository = class _Repository {
|
|
|
2896
3896
|
}
|
|
2897
3897
|
}
|
|
2898
3898
|
} catch (error) {
|
|
2899
|
-
if (error
|
|
3899
|
+
if (isAbortError(error)) {
|
|
2900
3900
|
throw error;
|
|
2901
3901
|
}
|
|
2902
3902
|
throw new Error("Cannot list tags: storage does not support listing");
|
|
@@ -2926,7 +3926,10 @@ var Repository = class _Repository {
|
|
|
2926
3926
|
if (!snapshotId2) {
|
|
2927
3927
|
throw new Error(`Branch not found: ${name}`);
|
|
2928
3928
|
}
|
|
2929
|
-
return ReadSession.open(this.storage, snapshotId2,
|
|
3929
|
+
return ReadSession.open(this.storage, snapshotId2, {
|
|
3930
|
+
...options,
|
|
3931
|
+
virtualChunkContainers: repoInfo.virtualChunkContainers
|
|
3932
|
+
});
|
|
2930
3933
|
}
|
|
2931
3934
|
const refDirPath = getBranchRefDirPath(name);
|
|
2932
3935
|
const refPath = await this.findLatestRefFile(
|
|
@@ -2953,7 +3956,10 @@ var Repository = class _Repository {
|
|
|
2953
3956
|
if (!snapshotId2) {
|
|
2954
3957
|
throw new Error(`Tag not found: ${name}`);
|
|
2955
3958
|
}
|
|
2956
|
-
return ReadSession.open(this.storage, snapshotId2,
|
|
3959
|
+
return ReadSession.open(this.storage, snapshotId2, {
|
|
3960
|
+
...options,
|
|
3961
|
+
virtualChunkContainers: repoInfo.virtualChunkContainers
|
|
3962
|
+
});
|
|
2957
3963
|
}
|
|
2958
3964
|
const refDirPath = getTagRefDirPath(name);
|
|
2959
3965
|
const legacyPath = getTagRefPath(name);
|
|
@@ -2976,7 +3982,12 @@ var Repository = class _Repository {
|
|
|
2976
3982
|
*/
|
|
2977
3983
|
async checkoutSnapshot(snapshotId, options) {
|
|
2978
3984
|
const id = typeof snapshotId === "string" ? decodeObjectId12(snapshotId) : snapshotId;
|
|
2979
|
-
|
|
3985
|
+
const repoInfo = await this.loadRepoInfo(options);
|
|
3986
|
+
const virtualChunkContainers = repoInfo?.virtualChunkContainers;
|
|
3987
|
+
return ReadSession.open(this.storage, id, {
|
|
3988
|
+
...options,
|
|
3989
|
+
...virtualChunkContainers && { virtualChunkContainers }
|
|
3990
|
+
});
|
|
2980
3991
|
}
|
|
2981
3992
|
/**
|
|
2982
3993
|
* Walk the snapshot history chain starting from the given session.
|
|
@@ -3055,9 +4066,7 @@ var HttpStorage = class {
|
|
|
3055
4066
|
return headers;
|
|
3056
4067
|
}
|
|
3057
4068
|
async getObject(path, range, options) {
|
|
3058
|
-
|
|
3059
|
-
throw new AbortError();
|
|
3060
|
-
}
|
|
4069
|
+
options?.signal?.throwIfAborted();
|
|
3061
4070
|
const url = this.getUrl(path);
|
|
3062
4071
|
const headers = this.getHeaders(range);
|
|
3063
4072
|
let response;
|
|
@@ -3070,9 +4079,7 @@ var HttpStorage = class {
|
|
|
3070
4079
|
signal: options?.signal
|
|
3071
4080
|
});
|
|
3072
4081
|
} catch (error) {
|
|
3073
|
-
if (error
|
|
3074
|
-
throw new AbortError();
|
|
3075
|
-
}
|
|
4082
|
+
if (isAbortError(error)) throw error;
|
|
3076
4083
|
throw new StorageError(
|
|
3077
4084
|
`Failed to fetch ${url}: ${error instanceof Error ? error.message : String(error)}`,
|
|
3078
4085
|
error instanceof Error ? error : void 0
|
|
@@ -3090,9 +4097,7 @@ var HttpStorage = class {
|
|
|
3090
4097
|
return new Uint8Array(arrayBuffer);
|
|
3091
4098
|
}
|
|
3092
4099
|
async exists(path, options) {
|
|
3093
|
-
|
|
3094
|
-
throw new AbortError();
|
|
3095
|
-
}
|
|
4100
|
+
options?.signal?.throwIfAborted();
|
|
3096
4101
|
const url = this.getUrl(path);
|
|
3097
4102
|
try {
|
|
3098
4103
|
const response = await fetch(url, {
|
|
@@ -3103,9 +4108,7 @@ var HttpStorage = class {
|
|
|
3103
4108
|
});
|
|
3104
4109
|
return response.ok;
|
|
3105
4110
|
} catch (error) {
|
|
3106
|
-
if (error
|
|
3107
|
-
throw new AbortError();
|
|
3108
|
-
}
|
|
4111
|
+
if (isAbortError(error)) throw error;
|
|
3109
4112
|
return false;
|
|
3110
4113
|
}
|
|
3111
4114
|
}
|
|
@@ -3123,8 +4126,9 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3123
4126
|
fetchClient;
|
|
3124
4127
|
validateChecksums;
|
|
3125
4128
|
azureAccount;
|
|
4129
|
+
withRangeCoalescing;
|
|
3126
4130
|
basePath = "";
|
|
3127
|
-
constructor(session, fetchClient, validateChecksums, azureAccount) {
|
|
4131
|
+
constructor(session, fetchClient, validateChecksums, azureAccount, withRangeCoalescing) {
|
|
3128
4132
|
if (!(session instanceof ReadSession)) {
|
|
3129
4133
|
throw new Error(
|
|
3130
4134
|
"IcechunkStore constructor is private. Use IcechunkStore.open() instead."
|
|
@@ -3134,6 +4138,7 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3134
4138
|
this.fetchClient = fetchClient;
|
|
3135
4139
|
this.validateChecksums = validateChecksums ?? false;
|
|
3136
4140
|
this.azureAccount = azureAccount;
|
|
4141
|
+
this.withRangeCoalescing = withRangeCoalescing;
|
|
3137
4142
|
}
|
|
3138
4143
|
static async open(arg, options = {}) {
|
|
3139
4144
|
if (arg instanceof ReadSession) {
|
|
@@ -3141,7 +4146,8 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3141
4146
|
arg,
|
|
3142
4147
|
options.fetchClient,
|
|
3143
4148
|
options.validateChecksums,
|
|
3144
|
-
options.azureAccount
|
|
4149
|
+
options.azureAccount,
|
|
4150
|
+
options.withRangeCoalescing
|
|
3145
4151
|
);
|
|
3146
4152
|
}
|
|
3147
4153
|
const storage = typeof arg === "string" ? new HttpStorage(arg) : arg;
|
|
@@ -3169,7 +4175,8 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3169
4175
|
session,
|
|
3170
4176
|
options.fetchClient,
|
|
3171
4177
|
options.validateChecksums,
|
|
3172
|
-
options.azureAccount
|
|
4178
|
+
options.azureAccount,
|
|
4179
|
+
options.withRangeCoalescing
|
|
3173
4180
|
);
|
|
3174
4181
|
}
|
|
3175
4182
|
/**
|
|
@@ -3185,7 +4192,7 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3185
4192
|
* @returns Data bytes or undefined if not found
|
|
3186
4193
|
*/
|
|
3187
4194
|
async get(key, opts) {
|
|
3188
|
-
|
|
4195
|
+
opts?.signal?.throwIfAborted();
|
|
3189
4196
|
const parsed = parseZarrKey(key);
|
|
3190
4197
|
const resolvedPath = this.resolvePath(parsed.path);
|
|
3191
4198
|
try {
|
|
@@ -3197,11 +4204,15 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3197
4204
|
signal: opts?.signal,
|
|
3198
4205
|
...this.fetchClient && { fetchClient: this.fetchClient },
|
|
3199
4206
|
validateChecksums: this.validateChecksums,
|
|
3200
|
-
azureAccount: this.azureAccount
|
|
4207
|
+
azureAccount: this.azureAccount,
|
|
4208
|
+
...this.withRangeCoalescing && {
|
|
4209
|
+
withRangeCoalescing: this.withRangeCoalescing
|
|
4210
|
+
}
|
|
3201
4211
|
});
|
|
3202
4212
|
return chunk ?? void 0;
|
|
3203
|
-
} catch {
|
|
3204
|
-
return void 0;
|
|
4213
|
+
} catch (err2) {
|
|
4214
|
+
if (err2 instanceof NotFoundError) return void 0;
|
|
4215
|
+
throw err2;
|
|
3205
4216
|
}
|
|
3206
4217
|
}
|
|
3207
4218
|
/**
|
|
@@ -3217,7 +4228,7 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3217
4228
|
* @returns Data bytes or undefined if not found
|
|
3218
4229
|
*/
|
|
3219
4230
|
async getRange(key, range, opts) {
|
|
3220
|
-
|
|
4231
|
+
opts?.signal?.throwIfAborted();
|
|
3221
4232
|
const parsed = parseZarrKey(key);
|
|
3222
4233
|
const resolvedPath = this.resolvePath(parsed.path);
|
|
3223
4234
|
try {
|
|
@@ -3230,7 +4241,10 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3230
4241
|
signal: opts?.signal,
|
|
3231
4242
|
...this.fetchClient && { fetchClient: this.fetchClient },
|
|
3232
4243
|
validateChecksums: this.validateChecksums,
|
|
3233
|
-
azureAccount: this.azureAccount
|
|
4244
|
+
azureAccount: this.azureAccount,
|
|
4245
|
+
...this.withRangeCoalescing && {
|
|
4246
|
+
withRangeCoalescing: this.withRangeCoalescing
|
|
4247
|
+
}
|
|
3234
4248
|
}
|
|
3235
4249
|
);
|
|
3236
4250
|
return data2 ?? void 0;
|
|
@@ -3241,8 +4255,9 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3241
4255
|
return data.slice(-range.suffixLength);
|
|
3242
4256
|
}
|
|
3243
4257
|
return data.slice(range.offset, range.offset + range.length);
|
|
3244
|
-
} catch {
|
|
3245
|
-
return void 0;
|
|
4258
|
+
} catch (err2) {
|
|
4259
|
+
if (err2 instanceof NotFoundError) return void 0;
|
|
4260
|
+
throw err2;
|
|
3246
4261
|
}
|
|
3247
4262
|
}
|
|
3248
4263
|
/** Prepend basePath to a parsed path. */
|
|
@@ -3266,7 +4281,8 @@ var IcechunkStore = class _IcechunkStore {
|
|
|
3266
4281
|
this.session,
|
|
3267
4282
|
this.fetchClient,
|
|
3268
4283
|
this.validateChecksums,
|
|
3269
|
-
this.azureAccount
|
|
4284
|
+
this.azureAccount,
|
|
4285
|
+
this.withRangeCoalescing
|
|
3270
4286
|
);
|
|
3271
4287
|
const cleanPath = path.replace(/^\/+|\/+$/g, "");
|
|
3272
4288
|
scoped.basePath = this.basePath ? `${this.basePath}/${cleanPath}` : cleanPath;
|