icechunk-js 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +32 -10
- package/dist/index.cjs +368 -57
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +8 -4
- package/dist/index.d.ts +8 -4
- package/dist/index.js +368 -57
- package/dist/index.js.map +1 -1
- package/package.json +18 -10
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Shane Loeffler and Shane Grigsby
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,10 +1,14 @@
|
|
|
1
|
-
#
|
|
1
|
+
# icechunk-js
|
|
2
|
+
|
|
3
|
+
[](https://www.npmjs.com/package/icechunk-js)
|
|
4
|
+
[](https://github.com/earth-mover/icechunk)
|
|
5
|
+
[](https://github.com/manzt/zarrita.js)
|
|
6
|
+
[](https://github.com/EarthyScience/icechunk-js/blob/main/LICENSE)
|
|
2
7
|
|
|
3
8
|
Read-only JavaScript/TypeScript reader for Icechunk repositories, designed for
|
|
4
9
|
use with [zarrita](https://github.com/manzt/zarrita.js).
|
|
5
10
|
|
|
6
|
-
-
|
|
7
|
-
- Works in browsers, Node.js 18+, Deno, and Bun
|
|
11
|
+
- Pure TypeScript, works in browsers and Node.js 18+
|
|
8
12
|
- Icechunk v1 and v2 format auto-detection
|
|
9
13
|
- All chunk payload types: inline, native, and virtual
|
|
10
14
|
|
|
@@ -24,7 +28,7 @@ import { open, get } from "zarrita";
|
|
|
24
28
|
const store = await IcechunkStore.open("https://bucket.s3.amazonaws.com/repo");
|
|
25
29
|
|
|
26
30
|
// Open an array and read data
|
|
27
|
-
const array = await open(store, { kind: "array"
|
|
31
|
+
const array = await open(store.resolve("/temperature"), { kind: "array" });
|
|
28
32
|
const data = await get(array, [0, 0, null]);
|
|
29
33
|
```
|
|
30
34
|
|
|
@@ -40,6 +44,10 @@ npm run test
|
|
|
40
44
|
npm run typecheck
|
|
41
45
|
```
|
|
42
46
|
|
|
47
|
+
To regenerate FlatBuffers TypeScript after syncing schemas, run `npm run generate:fbs`.
|
|
48
|
+
The pinned `flatc` compiler is downloaded automatically if not already available.
|
|
49
|
+
The version is set in `scripts/ensure-flatc.sh`.
|
|
50
|
+
|
|
43
51
|
## API
|
|
44
52
|
|
|
45
53
|
### IcechunkStore
|
|
@@ -57,6 +65,9 @@ const store = await IcechunkStore.open("https://example.com/repo", {
|
|
|
57
65
|
// snapshot: 'ABC123...',
|
|
58
66
|
// formatVersion: 'v1', // skip format auto-detection for v1 repos
|
|
59
67
|
// maxManifestCacheSize: 50, // LRU cache size (default: 100)
|
|
68
|
+
// signal: abortController.signal, // cancel initialization
|
|
69
|
+
// validateChecksums: true, // integrity headers for virtual chunks
|
|
70
|
+
// azureAccount: 'myaccount', // required for az:// virtual chunks
|
|
60
71
|
});
|
|
61
72
|
|
|
62
73
|
// Open from an existing ReadSession
|
|
@@ -94,7 +105,7 @@ const fetchClient: FetchClient = {
|
|
|
94
105
|
async fetch(url, init) {
|
|
95
106
|
// URL rewriting, pre-signing, or header injection happens here.
|
|
96
107
|
// icechunk-js has already translated s3:// → https:// and built
|
|
97
|
-
// Range
|
|
108
|
+
// Range headers in `init` (plus If-Match when validateChecksums is on).
|
|
98
109
|
const signedUrl = await presign(url);
|
|
99
110
|
return globalThis.fetch(signedUrl, {
|
|
100
111
|
...init,
|
|
@@ -111,17 +122,28 @@ const store = await IcechunkStore.open("https://example.com/repo", {
|
|
|
111
122
|
Cloud storage URLs in virtual chunk references are automatically translated:
|
|
112
123
|
|
|
113
124
|
- `s3://bucket/key` → `https://bucket.s3.amazonaws.com/key`
|
|
114
|
-
- `gs://bucket/key` → `https://storage.googleapis.com/bucket/key`
|
|
115
|
-
- `az://
|
|
116
|
-
|
|
125
|
+
- `gs://bucket/key` (or `gcs://`) → `https://storage.googleapis.com/bucket/key`
|
|
126
|
+
- `az://container/path` (or `azure://`) →
|
|
127
|
+
`https://{azureAccount}.blob.core.windows.net/container/path`
|
|
128
|
+
- `abfs://container@account.dfs.core.windows.net/path` →
|
|
129
|
+
`https://account.blob.core.windows.net/container/path`
|
|
117
130
|
|
|
118
131
|
### Repository
|
|
119
132
|
|
|
120
133
|
For direct access to branches, tags, and checkouts.
|
|
121
134
|
|
|
135
|
+
> **Note:** Over plain HTTP, `listBranches()` and `listTags()` only work
|
|
136
|
+
> reliably with v2 repos, which embed refs in the top-level `repo` file. For
|
|
137
|
+
> v1 repos, direct `checkoutBranch()` / `checkoutTag()` can work when the
|
|
138
|
+
> target ref still lives at the legacy `ref.json` path, but versioned ref
|
|
139
|
+
> filenames still require `listPrefix()` discovery, which `HttpStorage` does
|
|
140
|
+
> not provide. Use a listing-capable storage backend for full v1 branch/tag
|
|
141
|
+
> support.
|
|
142
|
+
|
|
122
143
|
```typescript
|
|
123
144
|
import { Repository, HttpStorage } from "icechunk-js";
|
|
124
145
|
|
|
146
|
+
// Replace with the root URL of a real Icechunk repository.
|
|
125
147
|
const storage = new HttpStorage("https://example.com/repo");
|
|
126
148
|
|
|
127
149
|
// Auto-detect format (default)
|
|
@@ -130,7 +152,7 @@ const repo = await Repository.open({ storage });
|
|
|
130
152
|
// Or with format version hint (skips /repo request for v1 stores)
|
|
131
153
|
// const repo = await Repository.open({ storage, formatVersion: 'v1' });
|
|
132
154
|
|
|
133
|
-
// List branches and tags
|
|
155
|
+
// List branches and tags (v2 repos, or storage backends that support listing)
|
|
134
156
|
const branches = await repo.listBranches();
|
|
135
157
|
const tags = await repo.listTags();
|
|
136
158
|
|
|
@@ -144,7 +166,7 @@ const session = await repo.checkoutBranch("main");
|
|
|
144
166
|
|
|
145
167
|
```typescript
|
|
146
168
|
for await (const entry of repo.walkHistory(session)) {
|
|
147
|
-
console.log(entry.id, entry.message, entry.flushedAt);
|
|
169
|
+
console.log(entry.id, entry.message, entry.flushedAt, entry.metadata);
|
|
148
170
|
}
|
|
149
171
|
```
|
|
150
172
|
|
package/dist/index.cjs
CHANGED
|
@@ -492,6 +492,70 @@ var Repo = class _Repo {
|
|
|
492
492
|
const offset = this.bb.__offset(this.bb_pos, 20);
|
|
493
493
|
return offset ? this.bb.__string(this.bb_pos + offset, optionalEncoding) : null;
|
|
494
494
|
}
|
|
495
|
+
config(index) {
|
|
496
|
+
const offset = this.bb.__offset(this.bb_pos, 22);
|
|
497
|
+
return offset ? this.bb.readUint8(this.bb.__vector(this.bb_pos + offset) + index) : 0;
|
|
498
|
+
}
|
|
499
|
+
configLength() {
|
|
500
|
+
const offset = this.bb.__offset(this.bb_pos, 22);
|
|
501
|
+
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
|
|
502
|
+
}
|
|
503
|
+
configArray() {
|
|
504
|
+
const offset = this.bb.__offset(this.bb_pos, 22);
|
|
505
|
+
return offset ? new Uint8Array(
|
|
506
|
+
this.bb.bytes().buffer,
|
|
507
|
+
this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset),
|
|
508
|
+
this.bb.__vector_len(this.bb_pos + offset)
|
|
509
|
+
) : null;
|
|
510
|
+
}
|
|
511
|
+
enabledFeatureFlags(index) {
|
|
512
|
+
const offset = this.bb.__offset(this.bb_pos, 24);
|
|
513
|
+
return offset ? this.bb.readUint16(this.bb.__vector(this.bb_pos + offset) + index * 2) : 0;
|
|
514
|
+
}
|
|
515
|
+
enabledFeatureFlagsLength() {
|
|
516
|
+
const offset = this.bb.__offset(this.bb_pos, 24);
|
|
517
|
+
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
|
|
518
|
+
}
|
|
519
|
+
enabledFeatureFlagsArray() {
|
|
520
|
+
const offset = this.bb.__offset(this.bb_pos, 24);
|
|
521
|
+
return offset ? new Uint16Array(
|
|
522
|
+
this.bb.bytes().buffer,
|
|
523
|
+
this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset),
|
|
524
|
+
this.bb.__vector_len(this.bb_pos + offset)
|
|
525
|
+
) : null;
|
|
526
|
+
}
|
|
527
|
+
disabledFeatureFlags(index) {
|
|
528
|
+
const offset = this.bb.__offset(this.bb_pos, 26);
|
|
529
|
+
return offset ? this.bb.readUint16(this.bb.__vector(this.bb_pos + offset) + index * 2) : 0;
|
|
530
|
+
}
|
|
531
|
+
disabledFeatureFlagsLength() {
|
|
532
|
+
const offset = this.bb.__offset(this.bb_pos, 26);
|
|
533
|
+
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
|
|
534
|
+
}
|
|
535
|
+
disabledFeatureFlagsArray() {
|
|
536
|
+
const offset = this.bb.__offset(this.bb_pos, 26);
|
|
537
|
+
return offset ? new Uint16Array(
|
|
538
|
+
this.bb.bytes().buffer,
|
|
539
|
+
this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset),
|
|
540
|
+
this.bb.__vector_len(this.bb_pos + offset)
|
|
541
|
+
) : null;
|
|
542
|
+
}
|
|
543
|
+
extra(index) {
|
|
544
|
+
const offset = this.bb.__offset(this.bb_pos, 28);
|
|
545
|
+
return offset ? this.bb.readUint8(this.bb.__vector(this.bb_pos + offset) + index) : 0;
|
|
546
|
+
}
|
|
547
|
+
extraLength() {
|
|
548
|
+
const offset = this.bb.__offset(this.bb_pos, 28);
|
|
549
|
+
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
|
|
550
|
+
}
|
|
551
|
+
extraArray() {
|
|
552
|
+
const offset = this.bb.__offset(this.bb_pos, 28);
|
|
553
|
+
return offset ? new Uint8Array(
|
|
554
|
+
this.bb.bytes().buffer,
|
|
555
|
+
this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset),
|
|
556
|
+
this.bb.__vector_len(this.bb_pos + offset)
|
|
557
|
+
) : null;
|
|
558
|
+
}
|
|
495
559
|
};
|
|
496
560
|
|
|
497
561
|
// src/format/flatbuffers/repo-parser.ts
|
|
@@ -805,6 +869,51 @@ var ManifestFileInfo = class {
|
|
|
805
869
|
}
|
|
806
870
|
};
|
|
807
871
|
|
|
872
|
+
// src/format/flatbuffers/generated/manifest-file-info-v2.ts
|
|
873
|
+
var ManifestFileInfoV2 = class _ManifestFileInfoV2 {
|
|
874
|
+
bb = null;
|
|
875
|
+
bb_pos = 0;
|
|
876
|
+
__init(i, bb) {
|
|
877
|
+
this.bb_pos = i;
|
|
878
|
+
this.bb = bb;
|
|
879
|
+
return this;
|
|
880
|
+
}
|
|
881
|
+
static getRootAsManifestFileInfoV2(bb, obj) {
|
|
882
|
+
return (obj || new _ManifestFileInfoV2()).__init(
|
|
883
|
+
bb.readInt32(bb.position()) + bb.position(),
|
|
884
|
+
bb
|
|
885
|
+
);
|
|
886
|
+
}
|
|
887
|
+
id(obj) {
|
|
888
|
+
const offset = this.bb.__offset(this.bb_pos, 4);
|
|
889
|
+
return offset ? (obj || new ObjectId12()).__init(this.bb_pos + offset, this.bb) : null;
|
|
890
|
+
}
|
|
891
|
+
sizeBytes() {
|
|
892
|
+
const offset = this.bb.__offset(this.bb_pos, 6);
|
|
893
|
+
return offset ? this.bb.readUint64(this.bb_pos + offset) : BigInt("0");
|
|
894
|
+
}
|
|
895
|
+
numChunkRefs() {
|
|
896
|
+
const offset = this.bb.__offset(this.bb_pos, 8);
|
|
897
|
+
return offset ? this.bb.readUint32(this.bb_pos + offset) : 0;
|
|
898
|
+
}
|
|
899
|
+
extra(index) {
|
|
900
|
+
const offset = this.bb.__offset(this.bb_pos, 10);
|
|
901
|
+
return offset ? this.bb.readUint8(this.bb.__vector(this.bb_pos + offset) + index) : 0;
|
|
902
|
+
}
|
|
903
|
+
extraLength() {
|
|
904
|
+
const offset = this.bb.__offset(this.bb_pos, 10);
|
|
905
|
+
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
|
|
906
|
+
}
|
|
907
|
+
extraArray() {
|
|
908
|
+
const offset = this.bb.__offset(this.bb_pos, 10);
|
|
909
|
+
return offset ? new Uint8Array(
|
|
910
|
+
this.bb.bytes().buffer,
|
|
911
|
+
this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset),
|
|
912
|
+
this.bb.__vector_len(this.bb_pos + offset)
|
|
913
|
+
) : null;
|
|
914
|
+
}
|
|
915
|
+
};
|
|
916
|
+
|
|
808
917
|
// src/format/flatbuffers/generated/dimension-name.ts
|
|
809
918
|
var DimensionName = class _DimensionName {
|
|
810
919
|
bb = null;
|
|
@@ -846,6 +955,31 @@ var DimensionShape = class {
|
|
|
846
955
|
}
|
|
847
956
|
};
|
|
848
957
|
|
|
958
|
+
// src/format/flatbuffers/generated/dimension-shape-v2.ts
|
|
959
|
+
var DimensionShapeV2 = class _DimensionShapeV2 {
|
|
960
|
+
bb = null;
|
|
961
|
+
bb_pos = 0;
|
|
962
|
+
__init(i, bb) {
|
|
963
|
+
this.bb_pos = i;
|
|
964
|
+
this.bb = bb;
|
|
965
|
+
return this;
|
|
966
|
+
}
|
|
967
|
+
static getRootAsDimensionShapeV2(bb, obj) {
|
|
968
|
+
return (obj || new _DimensionShapeV2()).__init(
|
|
969
|
+
bb.readInt32(bb.position()) + bb.position(),
|
|
970
|
+
bb
|
|
971
|
+
);
|
|
972
|
+
}
|
|
973
|
+
arrayLength() {
|
|
974
|
+
const offset = this.bb.__offset(this.bb_pos, 4);
|
|
975
|
+
return offset ? this.bb.readUint64(this.bb_pos + offset) : BigInt("0");
|
|
976
|
+
}
|
|
977
|
+
numChunks() {
|
|
978
|
+
const offset = this.bb.__offset(this.bb_pos, 6);
|
|
979
|
+
return offset ? this.bb.readUint32(this.bb_pos + offset) : 0;
|
|
980
|
+
}
|
|
981
|
+
};
|
|
982
|
+
|
|
849
983
|
// src/format/flatbuffers/generated/chunk-index-range.ts
|
|
850
984
|
var ChunkIndexRange = class {
|
|
851
985
|
bb = null;
|
|
@@ -950,6 +1084,19 @@ var ArrayNodeData = class _ArrayNodeData {
|
|
|
950
1084
|
const offset = this.bb.__offset(this.bb_pos, 8);
|
|
951
1085
|
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
|
|
952
1086
|
}
|
|
1087
|
+
shapeV2(index, obj) {
|
|
1088
|
+
const offset = this.bb.__offset(this.bb_pos, 10);
|
|
1089
|
+
return offset ? (obj || new DimensionShapeV2()).__init(
|
|
1090
|
+
this.bb.__indirect(
|
|
1091
|
+
this.bb.__vector(this.bb_pos + offset) + index * 4
|
|
1092
|
+
),
|
|
1093
|
+
this.bb
|
|
1094
|
+
) : null;
|
|
1095
|
+
}
|
|
1096
|
+
shapeV2Length() {
|
|
1097
|
+
const offset = this.bb.__offset(this.bb_pos, 10);
|
|
1098
|
+
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
|
|
1099
|
+
}
|
|
953
1100
|
};
|
|
954
1101
|
|
|
955
1102
|
// src/format/flatbuffers/generated/object-id8.ts
|
|
@@ -1016,6 +1163,22 @@ var NodeSnapshot = class _NodeSnapshot {
|
|
|
1016
1163
|
const offset = this.bb.__offset(this.bb_pos, 12);
|
|
1017
1164
|
return offset ? this.bb.__union(obj, this.bb_pos + offset) : null;
|
|
1018
1165
|
}
|
|
1166
|
+
extra(index) {
|
|
1167
|
+
const offset = this.bb.__offset(this.bb_pos, 14);
|
|
1168
|
+
return offset ? this.bb.readUint8(this.bb.__vector(this.bb_pos + offset) + index) : 0;
|
|
1169
|
+
}
|
|
1170
|
+
extraLength() {
|
|
1171
|
+
const offset = this.bb.__offset(this.bb_pos, 14);
|
|
1172
|
+
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
|
|
1173
|
+
}
|
|
1174
|
+
extraArray() {
|
|
1175
|
+
const offset = this.bb.__offset(this.bb_pos, 14);
|
|
1176
|
+
return offset ? new Uint8Array(
|
|
1177
|
+
this.bb.bytes().buffer,
|
|
1178
|
+
this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset),
|
|
1179
|
+
this.bb.__vector_len(this.bb_pos + offset)
|
|
1180
|
+
) : null;
|
|
1181
|
+
}
|
|
1019
1182
|
};
|
|
1020
1183
|
|
|
1021
1184
|
// src/format/flatbuffers/generated/snapshot.ts
|
|
@@ -1086,6 +1249,35 @@ var Snapshot = class _Snapshot {
|
|
|
1086
1249
|
const offset = this.bb.__offset(this.bb_pos, 16);
|
|
1087
1250
|
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
|
|
1088
1251
|
}
|
|
1252
|
+
manifestFilesV2(index, obj) {
|
|
1253
|
+
const offset = this.bb.__offset(this.bb_pos, 18);
|
|
1254
|
+
return offset ? (obj || new ManifestFileInfoV2()).__init(
|
|
1255
|
+
this.bb.__indirect(
|
|
1256
|
+
this.bb.__vector(this.bb_pos + offset) + index * 4
|
|
1257
|
+
),
|
|
1258
|
+
this.bb
|
|
1259
|
+
) : null;
|
|
1260
|
+
}
|
|
1261
|
+
manifestFilesV2Length() {
|
|
1262
|
+
const offset = this.bb.__offset(this.bb_pos, 18);
|
|
1263
|
+
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
|
|
1264
|
+
}
|
|
1265
|
+
extra(index) {
|
|
1266
|
+
const offset = this.bb.__offset(this.bb_pos, 20);
|
|
1267
|
+
return offset ? this.bb.readUint8(this.bb.__vector(this.bb_pos + offset) + index) : 0;
|
|
1268
|
+
}
|
|
1269
|
+
extraLength() {
|
|
1270
|
+
const offset = this.bb.__offset(this.bb_pos, 20);
|
|
1271
|
+
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
|
|
1272
|
+
}
|
|
1273
|
+
extraArray() {
|
|
1274
|
+
const offset = this.bb.__offset(this.bb_pos, 20);
|
|
1275
|
+
return offset ? new Uint8Array(
|
|
1276
|
+
this.bb.bytes().buffer,
|
|
1277
|
+
this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset),
|
|
1278
|
+
this.bb.__vector_len(this.bb_pos + offset)
|
|
1279
|
+
) : null;
|
|
1280
|
+
}
|
|
1089
1281
|
};
|
|
1090
1282
|
|
|
1091
1283
|
// src/format/flatbuffers/snapshot-parser.ts
|
|
@@ -1125,18 +1317,34 @@ function parseSnapshot(data) {
|
|
|
1125
1317
|
}
|
|
1126
1318
|
}
|
|
1127
1319
|
}
|
|
1128
|
-
const manifestFilesLength = fbsSnapshot.manifestFilesLength();
|
|
1129
1320
|
const manifestFiles = [];
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
const
|
|
1134
|
-
if (
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1321
|
+
const v2Length = fbsSnapshot.manifestFilesV2Length();
|
|
1322
|
+
if (v2Length > 0) {
|
|
1323
|
+
for (let i = 0; i < v2Length; i++) {
|
|
1324
|
+
const fbsMfi = fbsSnapshot.manifestFilesV2(i);
|
|
1325
|
+
if (fbsMfi) {
|
|
1326
|
+
const mfiIdObj = fbsMfi.id();
|
|
1327
|
+
if (!mfiIdObj) throw new Error("ManifestFileInfoV2 missing id");
|
|
1328
|
+
manifestFiles.push({
|
|
1329
|
+
id: asObjectId12(readId12(mfiIdObj.bb, mfiIdObj.bb_pos)),
|
|
1330
|
+
sizeBytes: Number(fbsMfi.sizeBytes()),
|
|
1331
|
+
numChunkRefs: fbsMfi.numChunkRefs()
|
|
1332
|
+
});
|
|
1333
|
+
}
|
|
1334
|
+
}
|
|
1335
|
+
} else {
|
|
1336
|
+
const v1Length = fbsSnapshot.manifestFilesLength();
|
|
1337
|
+
for (let i = 0; i < v1Length; i++) {
|
|
1338
|
+
const fbsMfi = fbsSnapshot.manifestFiles(i);
|
|
1339
|
+
if (fbsMfi) {
|
|
1340
|
+
const mfiIdObj = fbsMfi.id();
|
|
1341
|
+
if (!mfiIdObj) throw new Error("ManifestFileInfo missing id");
|
|
1342
|
+
manifestFiles.push({
|
|
1343
|
+
id: asObjectId12(readId12(mfiIdObj.bb, mfiIdObj.bb_pos)),
|
|
1344
|
+
sizeBytes: Number(fbsMfi.sizeBytes()),
|
|
1345
|
+
numChunkRefs: fbsMfi.numChunkRefs()
|
|
1346
|
+
});
|
|
1347
|
+
}
|
|
1140
1348
|
}
|
|
1141
1349
|
}
|
|
1142
1350
|
return {
|
|
@@ -1173,15 +1381,31 @@ function parseNodeData(fbsNode, unionType) {
|
|
|
1173
1381
|
throw new Error(`Unknown node data union type: ${unionType}`);
|
|
1174
1382
|
}
|
|
1175
1383
|
function parseArrayNodeData(fbsArray) {
|
|
1176
|
-
const shapeLength = fbsArray.shapeLength();
|
|
1177
1384
|
const shape = [];
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1385
|
+
const shapeV2Length = fbsArray.shapeV2Length();
|
|
1386
|
+
if (shapeV2Length > 0) {
|
|
1387
|
+
for (let i = 0; i < shapeV2Length; i++) {
|
|
1388
|
+
const fbsShape = fbsArray.shapeV2(i);
|
|
1389
|
+
if (fbsShape) {
|
|
1390
|
+
const arrayLength = Number(fbsShape.arrayLength());
|
|
1391
|
+
const numChunks = fbsShape.numChunks();
|
|
1392
|
+
shape.push({
|
|
1393
|
+
arrayLength,
|
|
1394
|
+
chunkLength: numChunks > 0 ? Math.ceil(arrayLength / numChunks) : 0,
|
|
1395
|
+
numChunks
|
|
1396
|
+
});
|
|
1397
|
+
}
|
|
1398
|
+
}
|
|
1399
|
+
} else {
|
|
1400
|
+
const shapeLength = fbsArray.shapeLength();
|
|
1401
|
+
for (let i = 0; i < shapeLength; i++) {
|
|
1402
|
+
const fbsShape = fbsArray.shape(i);
|
|
1403
|
+
if (fbsShape) {
|
|
1404
|
+
shape.push({
|
|
1405
|
+
arrayLength: Number(fbsShape.arrayLength()),
|
|
1406
|
+
chunkLength: Number(fbsShape.chunkLength())
|
|
1407
|
+
});
|
|
1408
|
+
}
|
|
1185
1409
|
}
|
|
1186
1410
|
}
|
|
1187
1411
|
const dimNamesLength = fbsArray.dimensionNamesLength();
|
|
@@ -1298,6 +1522,38 @@ var ChunkRef = class _ChunkRef {
|
|
|
1298
1522
|
const offset = this.bb.__offset(this.bb_pos, 18);
|
|
1299
1523
|
return offset ? this.bb.readUint32(this.bb_pos + offset) : 0;
|
|
1300
1524
|
}
|
|
1525
|
+
compressedLocation(index) {
|
|
1526
|
+
const offset = this.bb.__offset(this.bb_pos, 20);
|
|
1527
|
+
return offset ? this.bb.readUint8(this.bb.__vector(this.bb_pos + offset) + index) : 0;
|
|
1528
|
+
}
|
|
1529
|
+
compressedLocationLength() {
|
|
1530
|
+
const offset = this.bb.__offset(this.bb_pos, 20);
|
|
1531
|
+
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
|
|
1532
|
+
}
|
|
1533
|
+
compressedLocationArray() {
|
|
1534
|
+
const offset = this.bb.__offset(this.bb_pos, 20);
|
|
1535
|
+
return offset ? new Uint8Array(
|
|
1536
|
+
this.bb.bytes().buffer,
|
|
1537
|
+
this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset),
|
|
1538
|
+
this.bb.__vector_len(this.bb_pos + offset)
|
|
1539
|
+
) : null;
|
|
1540
|
+
}
|
|
1541
|
+
extra(index) {
|
|
1542
|
+
const offset = this.bb.__offset(this.bb_pos, 22);
|
|
1543
|
+
return offset ? this.bb.readUint8(this.bb.__vector(this.bb_pos + offset) + index) : 0;
|
|
1544
|
+
}
|
|
1545
|
+
extraLength() {
|
|
1546
|
+
const offset = this.bb.__offset(this.bb_pos, 22);
|
|
1547
|
+
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
|
|
1548
|
+
}
|
|
1549
|
+
extraArray() {
|
|
1550
|
+
const offset = this.bb.__offset(this.bb_pos, 22);
|
|
1551
|
+
return offset ? new Uint8Array(
|
|
1552
|
+
this.bb.bytes().buffer,
|
|
1553
|
+
this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset),
|
|
1554
|
+
this.bb.__vector_len(this.bb_pos + offset)
|
|
1555
|
+
) : null;
|
|
1556
|
+
}
|
|
1301
1557
|
};
|
|
1302
1558
|
|
|
1303
1559
|
// src/format/flatbuffers/generated/array-manifest.ts
|
|
@@ -1332,6 +1588,22 @@ var ArrayManifest = class _ArrayManifest {
|
|
|
1332
1588
|
const offset = this.bb.__offset(this.bb_pos, 6);
|
|
1333
1589
|
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
|
|
1334
1590
|
}
|
|
1591
|
+
extra(index) {
|
|
1592
|
+
const offset = this.bb.__offset(this.bb_pos, 8);
|
|
1593
|
+
return offset ? this.bb.readUint8(this.bb.__vector(this.bb_pos + offset) + index) : 0;
|
|
1594
|
+
}
|
|
1595
|
+
extraLength() {
|
|
1596
|
+
const offset = this.bb.__offset(this.bb_pos, 8);
|
|
1597
|
+
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
|
|
1598
|
+
}
|
|
1599
|
+
extraArray() {
|
|
1600
|
+
const offset = this.bb.__offset(this.bb_pos, 8);
|
|
1601
|
+
return offset ? new Uint8Array(
|
|
1602
|
+
this.bb.bytes().buffer,
|
|
1603
|
+
this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset),
|
|
1604
|
+
this.bb.__vector_len(this.bb_pos + offset)
|
|
1605
|
+
) : null;
|
|
1606
|
+
}
|
|
1335
1607
|
};
|
|
1336
1608
|
|
|
1337
1609
|
// src/format/flatbuffers/generated/manifest.ts
|
|
@@ -1366,6 +1638,42 @@ var Manifest = class _Manifest {
|
|
|
1366
1638
|
const offset = this.bb.__offset(this.bb_pos, 6);
|
|
1367
1639
|
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
|
|
1368
1640
|
}
|
|
1641
|
+
locationDictionary(index) {
|
|
1642
|
+
const offset = this.bb.__offset(this.bb_pos, 8);
|
|
1643
|
+
return offset ? this.bb.readUint8(this.bb.__vector(this.bb_pos + offset) + index) : 0;
|
|
1644
|
+
}
|
|
1645
|
+
locationDictionaryLength() {
|
|
1646
|
+
const offset = this.bb.__offset(this.bb_pos, 8);
|
|
1647
|
+
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
|
|
1648
|
+
}
|
|
1649
|
+
locationDictionaryArray() {
|
|
1650
|
+
const offset = this.bb.__offset(this.bb_pos, 8);
|
|
1651
|
+
return offset ? new Uint8Array(
|
|
1652
|
+
this.bb.bytes().buffer,
|
|
1653
|
+
this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset),
|
|
1654
|
+
this.bb.__vector_len(this.bb_pos + offset)
|
|
1655
|
+
) : null;
|
|
1656
|
+
}
|
|
1657
|
+
compressionAlgorithm() {
|
|
1658
|
+
const offset = this.bb.__offset(this.bb_pos, 10);
|
|
1659
|
+
return offset ? this.bb.readUint8(this.bb_pos + offset) : 1;
|
|
1660
|
+
}
|
|
1661
|
+
extra(index) {
|
|
1662
|
+
const offset = this.bb.__offset(this.bb_pos, 12);
|
|
1663
|
+
return offset ? this.bb.readUint8(this.bb.__vector(this.bb_pos + offset) + index) : 0;
|
|
1664
|
+
}
|
|
1665
|
+
extraLength() {
|
|
1666
|
+
const offset = this.bb.__offset(this.bb_pos, 12);
|
|
1667
|
+
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
|
|
1668
|
+
}
|
|
1669
|
+
extraArray() {
|
|
1670
|
+
const offset = this.bb.__offset(this.bb_pos, 12);
|
|
1671
|
+
return offset ? new Uint8Array(
|
|
1672
|
+
this.bb.bytes().buffer,
|
|
1673
|
+
this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset),
|
|
1674
|
+
this.bb.__vector_len(this.bb_pos + offset)
|
|
1675
|
+
) : null;
|
|
1676
|
+
}
|
|
1369
1677
|
};
|
|
1370
1678
|
|
|
1371
1679
|
// src/format/flatbuffers/manifest-parser.ts
|
|
@@ -1617,6 +1925,14 @@ var MoveOperation = class _MoveOperation {
|
|
|
1617
1925
|
const offset = this.bb.__offset(this.bb_pos, 6);
|
|
1618
1926
|
return offset ? this.bb.__string(this.bb_pos + offset, optionalEncoding) : null;
|
|
1619
1927
|
}
|
|
1928
|
+
nodeId(obj) {
|
|
1929
|
+
const offset = this.bb.__offset(this.bb_pos, 8);
|
|
1930
|
+
return offset ? (obj || new ObjectId8()).__init(this.bb_pos + offset, this.bb) : null;
|
|
1931
|
+
}
|
|
1932
|
+
nodeType() {
|
|
1933
|
+
const offset = this.bb.__offset(this.bb_pos, 10);
|
|
1934
|
+
return offset ? this.bb.readUint8(this.bb_pos + offset) : 0 /* Group */;
|
|
1935
|
+
}
|
|
1620
1936
|
};
|
|
1621
1937
|
|
|
1622
1938
|
// src/format/flatbuffers/generated/transaction-log.ts
|
|
@@ -1730,6 +2046,22 @@ var TransactionLog = class _TransactionLog {
|
|
|
1730
2046
|
const offset = this.bb.__offset(this.bb_pos, 20);
|
|
1731
2047
|
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
|
|
1732
2048
|
}
|
|
2049
|
+
extra(index) {
|
|
2050
|
+
const offset = this.bb.__offset(this.bb_pos, 22);
|
|
2051
|
+
return offset ? this.bb.readUint8(this.bb.__vector(this.bb_pos + offset) + index) : 0;
|
|
2052
|
+
}
|
|
2053
|
+
extraLength() {
|
|
2054
|
+
const offset = this.bb.__offset(this.bb_pos, 22);
|
|
2055
|
+
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
|
|
2056
|
+
}
|
|
2057
|
+
extraArray() {
|
|
2058
|
+
const offset = this.bb.__offset(this.bb_pos, 22);
|
|
2059
|
+
return offset ? new Uint8Array(
|
|
2060
|
+
this.bb.bytes().buffer,
|
|
2061
|
+
this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset),
|
|
2062
|
+
this.bb.__vector_len(this.bb_pos + offset)
|
|
2063
|
+
) : null;
|
|
2064
|
+
}
|
|
1733
2065
|
};
|
|
1734
2066
|
|
|
1735
2067
|
// src/format/flatbuffers/transaction-log-parser.ts
|
|
@@ -2464,7 +2796,7 @@ var Repository = class _Repository {
|
|
|
2464
2796
|
* version has the highest filename. Deletion tombstones are checked.
|
|
2465
2797
|
*
|
|
2466
2798
|
* When listing is not supported (e.g., HTTP storage), only the legacy
|
|
2467
|
-
* ref.json path is
|
|
2799
|
+
* ref.json path is returned (caller must handle NotFoundError on read).
|
|
2468
2800
|
*
|
|
2469
2801
|
* @param dirPrefix - Directory prefix to search
|
|
2470
2802
|
* @param legacyPath - Optional legacy ref.json path to try if listing fails
|
|
@@ -2500,7 +2832,9 @@ var Repository = class _Repository {
|
|
|
2500
2832
|
/**
|
|
2501
2833
|
* List all branches in the repository.
|
|
2502
2834
|
*
|
|
2503
|
-
*
|
|
2835
|
+
* Unlike tags, branches in icechunk v1 have no tombstone mechanism —
|
|
2836
|
+
* deletion removes the ref file outright. A branch is present iff it has
|
|
2837
|
+
* at least one ref file in its directory.
|
|
2504
2838
|
*
|
|
2505
2839
|
* @returns Array of branch names
|
|
2506
2840
|
*/
|
|
@@ -2509,49 +2843,23 @@ var Repository = class _Repository {
|
|
|
2509
2843
|
if (repoInfo) {
|
|
2510
2844
|
return listBranchesFromRepo(repoInfo);
|
|
2511
2845
|
}
|
|
2512
|
-
const
|
|
2846
|
+
const branches = /* @__PURE__ */ new Set();
|
|
2513
2847
|
try {
|
|
2514
2848
|
for await (const path of this.storage.listPrefix(
|
|
2515
2849
|
`${PATHS.REFS}/branch.`
|
|
2516
2850
|
)) {
|
|
2517
|
-
const
|
|
2518
|
-
|
|
2519
|
-
|
|
2520
|
-
);
|
|
2521
|
-
if (deletedMatch) {
|
|
2522
|
-
const branchName = deletedMatch[1];
|
|
2523
|
-
const refFile = `refs/branch.${branchName}/${deletedMatch[2]}`;
|
|
2524
|
-
if (!branchFiles.has(branchName)) {
|
|
2525
|
-
branchFiles.set(branchName, {
|
|
2526
|
-
jsonFiles: [],
|
|
2527
|
-
deletedFiles: /* @__PURE__ */ new Set()
|
|
2528
|
-
});
|
|
2529
|
-
}
|
|
2530
|
-
branchFiles.get(branchName).deletedFiles.add(refFile);
|
|
2531
|
-
} else if (jsonMatch) {
|
|
2532
|
-
const branchName = jsonMatch[1];
|
|
2533
|
-
if (!branchFiles.has(branchName)) {
|
|
2534
|
-
branchFiles.set(branchName, {
|
|
2535
|
-
jsonFiles: [],
|
|
2536
|
-
deletedFiles: /* @__PURE__ */ new Set()
|
|
2537
|
-
});
|
|
2538
|
-
}
|
|
2539
|
-
branchFiles.get(branchName).jsonFiles.push(path);
|
|
2851
|
+
const match = path.match(/^refs\/branch\.(.+)\/([^/]+\.json)$/);
|
|
2852
|
+
if (match) {
|
|
2853
|
+
branches.add(match[1]);
|
|
2540
2854
|
}
|
|
2541
2855
|
}
|
|
2542
|
-
} catch {
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
const result = [];
|
|
2546
|
-
for (const [branchName, { jsonFiles, deletedFiles }] of branchFiles) {
|
|
2547
|
-
if (jsonFiles.length === 0) continue;
|
|
2548
|
-
jsonFiles.sort();
|
|
2549
|
-
const latestRef = jsonFiles[jsonFiles.length - 1];
|
|
2550
|
-
if (!deletedFiles.has(latestRef)) {
|
|
2551
|
-
result.push(branchName);
|
|
2856
|
+
} catch (error) {
|
|
2857
|
+
if (error instanceof AbortError) {
|
|
2858
|
+
throw error;
|
|
2552
2859
|
}
|
|
2860
|
+
throw new Error("Cannot list branches: storage does not support listing");
|
|
2553
2861
|
}
|
|
2554
|
-
return
|
|
2862
|
+
return [...branches];
|
|
2555
2863
|
}
|
|
2556
2864
|
/**
|
|
2557
2865
|
* List all tags in the repository.
|
|
@@ -2587,7 +2895,10 @@ var Repository = class _Repository {
|
|
|
2587
2895
|
tagFiles.get(tagName).jsonFiles.push(path);
|
|
2588
2896
|
}
|
|
2589
2897
|
}
|
|
2590
|
-
} catch {
|
|
2898
|
+
} catch (error) {
|
|
2899
|
+
if (error instanceof AbortError) {
|
|
2900
|
+
throw error;
|
|
2901
|
+
}
|
|
2591
2902
|
throw new Error("Cannot list tags: storage does not support listing");
|
|
2592
2903
|
}
|
|
2593
2904
|
const result = [];
|