@fideus-labs/fizarrita 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.txt +9 -0
- package/dist/codec-worker.d.ts +23 -0
- package/dist/codec-worker.d.ts.map +1 -0
- package/dist/codec-worker.js +149 -0
- package/dist/codec-worker.js.map +1 -0
- package/dist/get-worker.d.ts +40 -0
- package/dist/get-worker.d.ts.map +1 -0
- package/dist/get-worker.js +203 -0
- package/dist/get-worker.js.map +1 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +10 -0
- package/dist/index.js.map +1 -0
- package/dist/internals/codec-pipeline.d.ts +29 -0
- package/dist/internals/codec-pipeline.d.ts.map +1 -0
- package/dist/internals/codec-pipeline.js +135 -0
- package/dist/internals/codec-pipeline.js.map +1 -0
- package/dist/internals/indexer.d.ts +81 -0
- package/dist/internals/indexer.d.ts.map +1 -0
- package/dist/internals/indexer.js +249 -0
- package/dist/internals/indexer.js.map +1 -0
- package/dist/internals/setter.d.ts +27 -0
- package/dist/internals/setter.d.ts.map +1 -0
- package/dist/internals/setter.js +132 -0
- package/dist/internals/setter.js.map +1 -0
- package/dist/internals/util.d.ts +59 -0
- package/dist/internals/util.d.ts.map +1 -0
- package/dist/internals/util.js +137 -0
- package/dist/internals/util.js.map +1 -0
- package/dist/set-worker.d.ts +44 -0
- package/dist/set-worker.d.ts.map +1 -0
- package/dist/set-worker.js +234 -0
- package/dist/set-worker.js.map +1 -0
- package/dist/types.d.ts +129 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +8 -0
- package/dist/types.js.map +1 -0
- package/dist/worker-rpc.d.ts +43 -0
- package/dist/worker-rpc.d.ts.map +1 -0
- package/dist/worker-rpc.js +219 -0
- package/dist/worker-rpc.js.map +1 -0
- package/package.json +53 -0
- package/src/codec-worker.ts +207 -0
- package/src/get-worker.ts +275 -0
- package/src/index.ts +15 -0
- package/src/internals/codec-pipeline.ts +200 -0
- package/src/internals/indexer.ts +346 -0
- package/src/internals/setter.ts +227 -0
- package/src/internals/util.ts +183 -0
- package/src/set-worker.ts +305 -0
- package/src/types.ts +158 -0
- package/src/worker-rpc.ts +347 -0
package/LICENSE.txt
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026-present Fideus Labs LLC <info@fideus.io>
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
6
|
+
|
|
7
|
+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
8
|
+
|
|
9
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Web Worker that handles encode/decode operations using zarrita's codec pipeline.
|
|
3
|
+
*
|
|
4
|
+
* Uses our self-contained codec pipeline builder (which references zarrita's
|
|
5
|
+
* publicly exported `registry`) to decode/encode chunks with real codecs
|
|
6
|
+
* (gzip, blosc, zstd, bytes, transpose, etc.).
|
|
7
|
+
*
|
|
8
|
+
* Message protocol:
|
|
9
|
+
* init: { type: 'init', id, metaId, meta: CodecChunkMeta }
|
|
10
|
+
* -> { type: 'init_ok', id }
|
|
11
|
+
*
|
|
12
|
+
* decode: { type: 'decode', id, bytes: ArrayBuffer, metaId }
|
|
13
|
+
* -> { type: 'decoded', id, data: ArrayBuffer, shape, stride }
|
|
14
|
+
*
|
|
15
|
+
* decode_into: { type: 'decode_into', id, bytes, metaId, output: SAB, ... }
|
|
16
|
+
* -> { type: 'decode_into_ok', id }
|
|
17
|
+
* Worker decodes and writes directly into SharedArrayBuffer.
|
|
18
|
+
*
|
|
19
|
+
* encode: { type: 'encode', id, data: ArrayBuffer, metaId }
|
|
20
|
+
* -> { type: 'encoded', id, bytes: ArrayBuffer }
|
|
21
|
+
*/
|
|
22
|
+
export {};
|
|
23
|
+
//# sourceMappingURL=codec-worker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"codec-worker.d.ts","sourceRoot":"","sources":["../src/codec-worker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG"}
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Web Worker that handles encode/decode operations using zarrita's codec pipeline.
|
|
3
|
+
*
|
|
4
|
+
* Uses our self-contained codec pipeline builder (which references zarrita's
|
|
5
|
+
* publicly exported `registry`) to decode/encode chunks with real codecs
|
|
6
|
+
* (gzip, blosc, zstd, bytes, transpose, etc.).
|
|
7
|
+
*
|
|
8
|
+
* Message protocol:
|
|
9
|
+
* init: { type: 'init', id, metaId, meta: CodecChunkMeta }
|
|
10
|
+
* -> { type: 'init_ok', id }
|
|
11
|
+
*
|
|
12
|
+
* decode: { type: 'decode', id, bytes: ArrayBuffer, metaId }
|
|
13
|
+
* -> { type: 'decoded', id, data: ArrayBuffer, shape, stride }
|
|
14
|
+
*
|
|
15
|
+
* decode_into: { type: 'decode_into', id, bytes, metaId, output: SAB, ... }
|
|
16
|
+
* -> { type: 'decode_into_ok', id }
|
|
17
|
+
* Worker decodes and writes directly into SharedArrayBuffer.
|
|
18
|
+
*
|
|
19
|
+
* encode: { type: 'encode', id, data: ArrayBuffer, metaId }
|
|
20
|
+
* -> { type: 'encoded', id, bytes: ArrayBuffer }
|
|
21
|
+
*/
|
|
22
|
+
import { create_codec_pipeline } from './internals/codec-pipeline.js';
|
|
23
|
+
import { compat_chunk, set_from_chunk_binary } from './internals/setter.js';
|
|
24
|
+
import { get_ctr, get_strides } from './internals/util.js';
|
|
25
|
+
const ctx = self;
|
|
26
|
+
// ---------------------------------------------------------------------------
|
|
27
|
+
// Codec pipeline cache — keyed by metaId (integer lookup, no JSON.stringify)
|
|
28
|
+
// ---------------------------------------------------------------------------
|
|
29
|
+
const pipelineByMetaId = new Map();
|
|
30
|
+
const metaByMetaId = new Map();
|
|
31
|
+
// Legacy fallback: pipeline cache keyed by JSON string (for any decode/encode
|
|
32
|
+
// messages that arrive with a full `meta` object instead of `metaId`)
|
|
33
|
+
const pipelineByKey = new Map();
|
|
34
|
+
function getPipeline(metaId) {
|
|
35
|
+
const pipeline = pipelineByMetaId.get(metaId);
|
|
36
|
+
if (!pipeline) {
|
|
37
|
+
throw new Error(`No pipeline for metaId ${metaId}. Send an 'init' message first.`);
|
|
38
|
+
}
|
|
39
|
+
return pipeline;
|
|
40
|
+
}
|
|
41
|
+
function getOrCreatePipelineLegacy(meta) {
|
|
42
|
+
const key = JSON.stringify(meta);
|
|
43
|
+
let pipeline = pipelineByKey.get(key);
|
|
44
|
+
if (!pipeline) {
|
|
45
|
+
pipeline = create_codec_pipeline({
|
|
46
|
+
data_type: meta.data_type,
|
|
47
|
+
shape: meta.chunk_shape,
|
|
48
|
+
codecs: meta.codecs,
|
|
49
|
+
});
|
|
50
|
+
pipelineByKey.set(key, pipeline);
|
|
51
|
+
}
|
|
52
|
+
return pipeline;
|
|
53
|
+
}
|
|
54
|
+
ctx.addEventListener('message', async (event) => {
|
|
55
|
+
const msg = event.data;
|
|
56
|
+
try {
|
|
57
|
+
if (msg.type === 'init') {
|
|
58
|
+
// Register codec metadata and pre-create pipeline
|
|
59
|
+
metaByMetaId.set(msg.metaId, msg.meta);
|
|
60
|
+
const pipeline = create_codec_pipeline({
|
|
61
|
+
data_type: msg.meta.data_type,
|
|
62
|
+
shape: msg.meta.chunk_shape,
|
|
63
|
+
codecs: msg.meta.codecs,
|
|
64
|
+
});
|
|
65
|
+
pipelineByMetaId.set(msg.metaId, pipeline);
|
|
66
|
+
ctx.postMessage({ type: 'init_ok', id: msg.id });
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
if (msg.type === 'decode') {
|
|
70
|
+
// Resolve pipeline: prefer metaId, fall back to legacy meta
|
|
71
|
+
const pipeline = msg.metaId !== undefined && pipelineByMetaId.has(msg.metaId)
|
|
72
|
+
? getPipeline(msg.metaId)
|
|
73
|
+
: getOrCreatePipelineLegacy(msg.meta);
|
|
74
|
+
const bytes = new Uint8Array(msg.bytes);
|
|
75
|
+
const chunk = await pipeline.decode(bytes);
|
|
76
|
+
// Extract the underlying ArrayBuffer from the decoded TypedArray
|
|
77
|
+
const dataView = chunk.data;
|
|
78
|
+
const buffer = dataView.buffer;
|
|
79
|
+
const byteOffset = dataView.byteOffset;
|
|
80
|
+
const byteLength = dataView.byteLength;
|
|
81
|
+
// If the decoded data doesn't own the full buffer, copy for clean transfer
|
|
82
|
+
let transferBuffer;
|
|
83
|
+
if (byteOffset === 0 && byteLength === buffer.byteLength) {
|
|
84
|
+
transferBuffer = buffer;
|
|
85
|
+
}
|
|
86
|
+
else {
|
|
87
|
+
transferBuffer = buffer.slice(byteOffset, byteOffset + byteLength);
|
|
88
|
+
}
|
|
89
|
+
ctx.postMessage({
|
|
90
|
+
type: 'decoded',
|
|
91
|
+
id: msg.id,
|
|
92
|
+
data: transferBuffer,
|
|
93
|
+
shape: chunk.shape,
|
|
94
|
+
stride: chunk.stride,
|
|
95
|
+
}, [transferBuffer]);
|
|
96
|
+
}
|
|
97
|
+
else if (msg.type === 'decode_into') {
|
|
98
|
+
// Decode and write directly into SharedArrayBuffer — no transfer back
|
|
99
|
+
const pipeline = getPipeline(msg.metaId);
|
|
100
|
+
const bytes = new Uint8Array(msg.bytes);
|
|
101
|
+
const chunk = await pipeline.decode(bytes);
|
|
102
|
+
// Create a Uint8Array view over the shared output buffer
|
|
103
|
+
const destView = new Uint8Array(msg.output, 0, msg.outputByteLength);
|
|
104
|
+
// Convert decoded chunk to byte-level representation
|
|
105
|
+
const src = compat_chunk(chunk);
|
|
106
|
+
// Write decoded data directly into the shared output memory
|
|
107
|
+
set_from_chunk_binary({ data: destView, stride: msg.outputStride }, src, msg.bytesPerElement, msg.projections);
|
|
108
|
+
ctx.postMessage({ type: 'decode_into_ok', id: msg.id });
|
|
109
|
+
}
|
|
110
|
+
else if (msg.type === 'encode') {
|
|
111
|
+
// Resolve pipeline and meta
|
|
112
|
+
let pipeline;
|
|
113
|
+
let meta;
|
|
114
|
+
if (msg.metaId !== undefined && pipelineByMetaId.has(msg.metaId)) {
|
|
115
|
+
pipeline = getPipeline(msg.metaId);
|
|
116
|
+
meta = metaByMetaId.get(msg.metaId);
|
|
117
|
+
}
|
|
118
|
+
else {
|
|
119
|
+
meta = msg.meta;
|
|
120
|
+
pipeline = getOrCreatePipelineLegacy(meta);
|
|
121
|
+
}
|
|
122
|
+
// Reconstruct a Chunk from the transferred ArrayBuffer
|
|
123
|
+
const Ctr = get_ctr(meta.data_type);
|
|
124
|
+
const data = new Ctr(msg.data, 0, msg.data.byteLength / Ctr.BYTES_PER_ELEMENT);
|
|
125
|
+
const shape = meta.chunk_shape;
|
|
126
|
+
const stride = get_strides(shape, 'C');
|
|
127
|
+
const chunk = { data, shape, stride };
|
|
128
|
+
const encoded = await pipeline.encode(chunk);
|
|
129
|
+
// Transfer the encoded bytes back
|
|
130
|
+
const transferBuffer = encoded.byteOffset === 0 && encoded.byteLength === encoded.buffer.byteLength
|
|
131
|
+
? encoded.buffer
|
|
132
|
+
: encoded.buffer.slice(encoded.byteOffset, encoded.byteOffset + encoded.byteLength);
|
|
133
|
+
ctx.postMessage({
|
|
134
|
+
type: 'encoded',
|
|
135
|
+
id: msg.id,
|
|
136
|
+
bytes: transferBuffer,
|
|
137
|
+
}, [transferBuffer]);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
catch (error) {
|
|
141
|
+
// Send error back to main thread
|
|
142
|
+
ctx.postMessage({
|
|
143
|
+
type: msg.type === 'decode' ? 'decoded' : msg.type === 'encode' ? 'encoded' : msg.type === 'decode_into' ? 'decode_into_ok' : 'init_ok',
|
|
144
|
+
id: msg.id,
|
|
145
|
+
error: error instanceof Error ? error.message : String(error),
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
});
|
|
149
|
+
//# sourceMappingURL=codec-worker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"codec-worker.js","sourceRoot":"","sources":["../src/codec-worker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAA;AACrE,OAAO,EAAE,YAAY,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAA;AAC3E,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAA;AAI1D,MAAM,GAAG,GAAG,IAA6C,CAAA;AAEzD,8EAA8E;AAC9E,6EAA6E;AAC7E,8EAA8E;AAE9E,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAG7B,CAAA;AAEH,MAAM,YAAY,GAAG,IAAI,GAAG,EAA0B,CAAA;AAEtD,8EAA8E;AAC9E,sEAAsE;AACtE,MAAM,aAAa,GAAG,IAAI,GAAG,EAG1B,CAAA;AAEH,SAAS,WAAW,CAAC,MAAc;IACjC,MAAM,QAAQ,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IAC7C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,0BAA0B,MAAM,iCAAiC,CAAC,CAAA;IACpF,CAAC;IACD,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED,SAAS,yBAAyB,CAAC,IAAoB;IACrD,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;IAChC,IAAI,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACrC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,QAAQ,GAAG,qBAAqB,CAAC;YAC/B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,KAAK,EAAE,IAAI,CAAC,WAAW;YACvB,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAA;QACF,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;IAClC,CAAC;IACD,OAAO,QAAQ,CAAA;AACjB,CAAC;AAYD,GAAG,CAAC,gBAAgB,CAAC,SAAS,EAAE,KAAK,EAAE,KAAkC,EAAE,EAAE;IAC3E,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAA;IAEtB,IAAI,CAAC;QACH,IAAI,GAAG,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACxB,kDAAkD;YAClD,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,CAAA;YACtC,MAAM,QAAQ,GAAG,qBAAqB,CAAC;gBACrC,SAAS,EAAE,GAAG,CAAC,IAAI,CAAC,SAAS;gBAC7B,KAAK,EAAE,GAAG,CAAC,IAAI,CAAC,WAAW;gBAC3B,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM;aACxB,CAAC,CAAA;YACF,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAA;YAC1C,GAAG,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAA;YAChD,OAAM;QACR,CAAC;QAED,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC1B,4DAA4D;YAC5D,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC;gBAC3E,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC;gBACzB,CAAC,CAAC,yBAAyB,CAAC,GAAG,CAAC,IAAK,CAAC,CAAA;YAExC,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;YACvC,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAoB,CAAA;YAE7D,iEAAiE;YACjE,MAAM,QAAQ,GAAG,KAAK,CAAC,IAItB,CAAA;YACD,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAA;YAC9B,MAAM,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAA;YACtC,MAAM,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAA;YAEtC,2EAA2E;YAC3E,IAAI,cAA2B,CAAA;YAC/B,IAAI,UAAU,KAAK,CAAC,IAAI,UAAU,KAAK,MAAM,CAAC,UAAU,EAAE,CAAC;gBACzD,cAAc,GAAG,MAAM,CAAA;YACzB,CAAC;iBAAM,CAAC;gBACN,cAAc,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,UAAU,GAAG,UAAU,CAAC,CAAA;YACpE,CAAC;YAED,GAAG,CAAC,WAAW,CACb;gBACE,IAAI,EAAE,SAAkB;gBACxB,EAAE,EAAE,GAAG,CAAC,EAAE;gBACV,IAAI,EAAE,cAAc;gBACpB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,MAAM,EAAE,KAAK,CAAC,MAAM;aACrB,EACD,CAAC,cAAc,CAAC,CACjB,CAAA;QACH,CAAC;aAAM,IAAI,GAAG,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;YACtC,sEAAsE;YACtE,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;YACxC,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;YACvC,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAoB,CAAA;YAE7D,yDAAyD;YACzD,MAAM,QAAQ,GAAG,IAAI,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,GAAG,CAAC,gBAAgB,CAAC,CAAA;YAEpE,qDAAqD;YACrD,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,CAAA;YAE/B,4DAA4D;YAC5D,qBAAqB,CACnB,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,CAAC,YAAY,EAAE,EAC5C,GAAG,EACH,GAAG,CAAC,eAAe,EACnB,GAAG,CAAC,WAAW,CAChB,CAAA;YAED,GAAG,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAA;QACzD,CAAC;aAAM,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACjC,4BAA4B;YAC5B,IAAI,QAAkD,CAAA;YACtD,IAAI,IAAoB,CAAA;YACxB,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBACjE,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;gBAClC,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAE,CAAA;YACtC,CAAC;iBAAM,CAAC;gBACN,IAAI,GAAG,GAAG,CAAC,IAAK,CAAA;gBAChB,QAAQ,GAAG,yBAAyB,CAAC,IAAI,CAAC,CAAA;YAC5C,CAAC;YAED,uDAAuD;YACvD,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,SAAS,CAGjC,CAAA;YACD,MAAM,IAAI,GAAG,IAAI,GAAG,CAClB,GAAG,CAAC,IAAI,EACR,CAAC,EACD,GAAG,CAAC,IAAI,CAAC,UAAU,GAAG,GAAG,CAAC,iBAAiB,CAC5C,CAAA;YACD,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAA;YAC9B,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;YAEtC,MAAM,KAAK,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAqB,CAAA;YACxD,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YAE5C,kCAAkC;YAClC,MAAM,cAAc,GAClB,OAAO,CAAC,UAAU,KAAK,CAAC,IAAI,OAAO,CAAC,UAAU,KAAK,OAAO,CAAC,MAAM,CAAC,UAAU;gBAC1E,CAAC,CAAE,OAAO,CAAC,MAAsB;gBACjC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;YAEvF,GAAG,CAAC,WAAW,CACb;gBACE,IAAI,EAAE,SAAkB;gBACxB,EAAE,EAAE,GAAG,CAAC,EAAE;gBACV,KAAK,EAAE,cAAc;aACtB,EACD,CAAC,cAAc,CAAC,CACjB,CAAA;QACH,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,iCAAiC;QACjC,GAAG,CAAC,WAAW,CAAC;YACd,IAAI,EAAE,GAAG,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,aAAa,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,SAAS;YACvI,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;SAC9D,CAAC,CAAA;IACJ,CAAC;AACH,CAAC,CAAC,CAAA"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* getWorker() — Worker-accelerated get for zarrita arrays.
|
|
3
|
+
*
|
|
4
|
+
* Reads data from a zarrita Array, offloading codec decode operations to a
|
|
5
|
+
* WorkerPool. The main thread fetches raw bytes from the store, transfers
|
|
6
|
+
* them to a worker for decoding, then copies the decoded chunk into the
|
|
7
|
+
* output array on the main thread.
|
|
8
|
+
*
|
|
9
|
+
* Uses WorkerPool.runTasks() for bounded-concurrency scheduling.
|
|
10
|
+
*/
|
|
11
|
+
import type { Array as ZarrArray, Chunk, DataType, Readable, Scalar, Slice } from 'zarrita';
|
|
12
|
+
import type { GetWorkerOptions } from './types.js';
|
|
13
|
+
/**
|
|
14
|
+
* Read data from a zarrita Array with codec decoding offloaded to Web Workers.
|
|
15
|
+
*
|
|
16
|
+
* Drop-in replacement for zarrita's `get()` with worker acceleration.
|
|
17
|
+
* The main thread fetches raw bytes from the store, then workers handle
|
|
18
|
+
* the (potentially expensive) codec decode operations in parallel.
|
|
19
|
+
*
|
|
20
|
+
* @param arr - The zarrita Array to read from.
|
|
21
|
+
* @param selection - Index selection (null for full array, or per-dimension slices/indices).
|
|
22
|
+
* @param opts - Options including the WorkerPool and store options.
|
|
23
|
+
* @returns The result chunk, or a scalar if all dimensions are integer-indexed.
|
|
24
|
+
*
|
|
25
|
+
* @example
|
|
26
|
+
* ```ts
|
|
27
|
+
* import { WorkerPool } from '@fideus-labs/worker-pool'
|
|
28
|
+
* import { getWorker } from '@fideus-labs/fizarrita'
|
|
29
|
+
* import * as zarr from 'zarrita'
|
|
30
|
+
*
|
|
31
|
+
* const pool = new WorkerPool(4)
|
|
32
|
+
* const store = new zarr.FetchStore('https://example.com/data.zarr')
|
|
33
|
+
* const arr = await zarr.open(store, { kind: 'array' })
|
|
34
|
+
* const result = await getWorker(arr, null, { pool })
|
|
35
|
+
*
|
|
36
|
+
* pool.terminateWorkers()
|
|
37
|
+
* ```
|
|
38
|
+
*/
|
|
39
|
+
export declare function getWorker<D extends DataType, Store extends Readable, Sel extends (null | Slice | number)[]>(arr: ZarrArray<D, Store>, selection: (Sel | null) | undefined, opts: GetWorkerOptions<Parameters<Store['get']>[1]>): Promise<null extends Sel[number] ? Chunk<D> : Slice extends Sel[number] ? Chunk<D> : Scalar<D>>;
|
|
40
|
+
//# sourceMappingURL=get-worker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"get-worker.d.ts","sourceRoot":"","sources":["../src/get-worker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,KAAK,EACV,KAAK,IAAI,SAAS,EAClB,KAAK,EACL,QAAQ,EACR,QAAQ,EACR,MAAM,EACN,KAAK,EACN,MAAM,SAAS,CAAA;AAUhB,OAAO,KAAK,EAAE,gBAAgB,EAAkB,MAAM,YAAY,CAAA;AAyFlE;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAsB,SAAS,CAC7B,CAAC,SAAS,QAAQ,EAClB,KAAK,SAAS,QAAQ,EACtB,GAAG,SAAS,CAAC,IAAI,GAAG,KAAK,GAAG,MAAM,CAAC,EAAE,EAErC,GAAG,EAAE,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,EACxB,SAAS,GAAE,GAAG,GAAG,IAAI,aAAO,EAC5B,IAAI,EAAE,gBAAgB,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAClD,OAAO,CACR,IAAI,SAAS,GAAG,CAAC,MAAM,CAAC,GACpB,KAAK,CAAC,CAAC,CAAC,GACR,KAAK,SAAS,GAAG,CAAC,MAAM,CAAC,GACvB,KAAK,CAAC,CAAC,CAAC,GACR,MAAM,CAAC,CAAC,CAAC,CAChB,CAoHA"}
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* getWorker() — Worker-accelerated get for zarrita arrays.
|
|
3
|
+
*
|
|
4
|
+
* Reads data from a zarrita Array, offloading codec decode operations to a
|
|
5
|
+
* WorkerPool. The main thread fetches raw bytes from the store, transfers
|
|
6
|
+
* them to a worker for decoding, then copies the decoded chunk into the
|
|
7
|
+
* output array on the main thread.
|
|
8
|
+
*
|
|
9
|
+
* Uses WorkerPool.runTasks() for bounded-concurrency scheduling.
|
|
10
|
+
*/
|
|
11
|
+
import { BasicIndexer } from './internals/indexer.js';
|
|
12
|
+
import { setter } from './internals/setter.js';
|
|
13
|
+
import { get_ctr, get_strides, create_chunk_key_encoder, assertSharedArrayBufferAvailable, createBuffer, } from './internals/util.js';
|
|
14
|
+
import { workerDecode, workerDecodeInto, getMetaId } from './worker-rpc.js';
|
|
15
|
+
/**
|
|
16
|
+
* Default URL for the codec worker. Uses `import.meta.url` to resolve
|
|
17
|
+
* relative to this module.
|
|
18
|
+
*/
|
|
19
|
+
const DEFAULT_WORKER_URL = new URL('./codec-worker.js', import.meta.url);
|
|
20
|
+
/** Shared TextDecoder instance. */
|
|
21
|
+
const decoder = new TextDecoder();
|
|
22
|
+
async function readArrayMetadata(arr) {
|
|
23
|
+
const store = arr.store;
|
|
24
|
+
// Try v3 first: read zarr.json
|
|
25
|
+
const v3Path = (arr.path === '/' ? '/zarr.json' : `${arr.path}/zarr.json`);
|
|
26
|
+
const v3Bytes = await store.get(v3Path);
|
|
27
|
+
if (v3Bytes) {
|
|
28
|
+
const metadata = JSON.parse(decoder.decode(v3Bytes));
|
|
29
|
+
return {
|
|
30
|
+
codecMeta: {
|
|
31
|
+
data_type: metadata.data_type,
|
|
32
|
+
chunk_shape: metadata.chunk_grid.configuration.chunk_shape,
|
|
33
|
+
codecs: metadata.codecs,
|
|
34
|
+
},
|
|
35
|
+
encodeChunkKey: create_chunk_key_encoder(metadata.chunk_key_encoding),
|
|
36
|
+
fillValue: metadata.fill_value ?? null,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
// Try v2: read .zarray
|
|
40
|
+
const v2Path = (arr.path === '/' ? '/.zarray' : `${arr.path}/.zarray`);
|
|
41
|
+
const v2Bytes = await store.get(v2Path);
|
|
42
|
+
if (v2Bytes) {
|
|
43
|
+
const metadata = JSON.parse(decoder.decode(v2Bytes));
|
|
44
|
+
const codecs = [];
|
|
45
|
+
if (metadata.order === 'F') {
|
|
46
|
+
codecs.push({ name: 'transpose', configuration: { order: 'F' } });
|
|
47
|
+
}
|
|
48
|
+
if (metadata.compressor) {
|
|
49
|
+
const { id, ...configuration } = metadata.compressor;
|
|
50
|
+
codecs.push({ name: id, configuration });
|
|
51
|
+
}
|
|
52
|
+
for (const { id, ...configuration } of metadata.filters ?? []) {
|
|
53
|
+
codecs.push({ name: id, configuration });
|
|
54
|
+
}
|
|
55
|
+
return {
|
|
56
|
+
codecMeta: {
|
|
57
|
+
data_type: arr.dtype,
|
|
58
|
+
chunk_shape: arr.chunks,
|
|
59
|
+
codecs: codecs.length > 0 ? codecs : [{ name: 'bytes', configuration: { endian: 'little' } }],
|
|
60
|
+
},
|
|
61
|
+
encodeChunkKey: create_chunk_key_encoder({
|
|
62
|
+
name: 'v2',
|
|
63
|
+
configuration: { separator: metadata.dimension_separator ?? '.' },
|
|
64
|
+
}),
|
|
65
|
+
fillValue: metadata.fill_value ?? null,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
// Fallback: BytesCodec only, default v3 key encoding
|
|
69
|
+
return {
|
|
70
|
+
codecMeta: {
|
|
71
|
+
data_type: arr.dtype,
|
|
72
|
+
chunk_shape: arr.chunks,
|
|
73
|
+
codecs: [{ name: 'bytes', configuration: { endian: 'little' } }],
|
|
74
|
+
},
|
|
75
|
+
encodeChunkKey: create_chunk_key_encoder({ name: 'default' }),
|
|
76
|
+
fillValue: null,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
// ---------------------------------------------------------------------------
|
|
80
|
+
// getWorker
|
|
81
|
+
// ---------------------------------------------------------------------------
|
|
82
|
+
/**
|
|
83
|
+
* Read data from a zarrita Array with codec decoding offloaded to Web Workers.
|
|
84
|
+
*
|
|
85
|
+
* Drop-in replacement for zarrita's `get()` with worker acceleration.
|
|
86
|
+
* The main thread fetches raw bytes from the store, then workers handle
|
|
87
|
+
* the (potentially expensive) codec decode operations in parallel.
|
|
88
|
+
*
|
|
89
|
+
* @param arr - The zarrita Array to read from.
|
|
90
|
+
* @param selection - Index selection (null for full array, or per-dimension slices/indices).
|
|
91
|
+
* @param opts - Options including the WorkerPool and store options.
|
|
92
|
+
* @returns The result chunk, or a scalar if all dimensions are integer-indexed.
|
|
93
|
+
*
|
|
94
|
+
* @example
|
|
95
|
+
* ```ts
|
|
96
|
+
* import { WorkerPool } from '@fideus-labs/worker-pool'
|
|
97
|
+
* import { getWorker } from '@fideus-labs/fizarrita'
|
|
98
|
+
* import * as zarr from 'zarrita'
|
|
99
|
+
*
|
|
100
|
+
* const pool = new WorkerPool(4)
|
|
101
|
+
* const store = new zarr.FetchStore('https://example.com/data.zarr')
|
|
102
|
+
* const arr = await zarr.open(store, { kind: 'array' })
|
|
103
|
+
* const result = await getWorker(arr, null, { pool })
|
|
104
|
+
*
|
|
105
|
+
* pool.terminateWorkers()
|
|
106
|
+
* ```
|
|
107
|
+
*/
|
|
108
|
+
export async function getWorker(arr, selection = null, opts) {
|
|
109
|
+
const { pool, workerUrl } = opts;
|
|
110
|
+
const resolvedWorkerUrl = workerUrl ?? DEFAULT_WORKER_URL;
|
|
111
|
+
const useShared = !!opts.useSharedArrayBuffer;
|
|
112
|
+
if (useShared) {
|
|
113
|
+
assertSharedArrayBufferAvailable();
|
|
114
|
+
}
|
|
115
|
+
// Read metadata from store — single read, single parse
|
|
116
|
+
const { codecMeta, encodeChunkKey, fillValue } = await readArrayMetadata(arr);
|
|
117
|
+
// Get stable metaId for the codec metadata (used by worker-rpc meta-init)
|
|
118
|
+
const metaId = getMetaId(codecMeta);
|
|
119
|
+
const Ctr = get_ctr(arr.dtype);
|
|
120
|
+
const bytesPerElement = Ctr.BYTES_PER_ELEMENT;
|
|
121
|
+
// Set up the indexer
|
|
122
|
+
const indexer = new BasicIndexer({
|
|
123
|
+
selection,
|
|
124
|
+
shape: arr.shape,
|
|
125
|
+
chunk_shape: arr.chunks,
|
|
126
|
+
});
|
|
127
|
+
// Allocate output — backed by SharedArrayBuffer when requested
|
|
128
|
+
const size = indexer.shape.reduce((a, b) => a * b, 1);
|
|
129
|
+
const buffer = createBuffer(size * bytesPerElement, useShared);
|
|
130
|
+
const data = new Ctr(buffer, 0, size);
|
|
131
|
+
const outStride = get_strides(indexer.shape);
|
|
132
|
+
const out = setter.prepare(data, indexer.shape, outStride);
|
|
133
|
+
// Pre-compute chunk invariants (hoisted out of loop)
|
|
134
|
+
const chunkShape = arr.chunks;
|
|
135
|
+
const chunkStrides = get_strides(chunkShape);
|
|
136
|
+
const chunkSize = chunkShape.reduce((a, b) => a * b, 1);
|
|
137
|
+
// Build tasks — one per chunk
|
|
138
|
+
const tasks = [];
|
|
139
|
+
for (const { chunk_coords, mapping } of indexer) {
|
|
140
|
+
const chunkKey = encodeChunkKey(chunk_coords);
|
|
141
|
+
const chunkPath = arr.resolve(chunkKey).path;
|
|
142
|
+
tasks.push(async (workerSlot) => {
|
|
143
|
+
const worker = workerSlot ?? new Worker(resolvedWorkerUrl, { type: 'module' });
|
|
144
|
+
// Fetch raw bytes from store on main thread
|
|
145
|
+
const rawBytes = await arr.store.get(chunkPath, opts.opts);
|
|
146
|
+
if (!rawBytes) {
|
|
147
|
+
// Missing chunk — fill value, no worker needed
|
|
148
|
+
const chunkData = new Ctr(chunkSize);
|
|
149
|
+
if (fillValue != null) {
|
|
150
|
+
// @ts-expect-error: fill_value type is union
|
|
151
|
+
chunkData.fill(fillValue);
|
|
152
|
+
}
|
|
153
|
+
const chunk = {
|
|
154
|
+
data: chunkData,
|
|
155
|
+
shape: chunkShape.slice(),
|
|
156
|
+
stride: chunkStrides.slice(),
|
|
157
|
+
};
|
|
158
|
+
// Copy fill-value chunk into output on main thread
|
|
159
|
+
setter.set_from_chunk(out, chunk, mapping);
|
|
160
|
+
}
|
|
161
|
+
else if (useShared) {
|
|
162
|
+
// Decode-into-shared: worker decodes AND writes directly into
|
|
163
|
+
// the SharedArrayBuffer output — no transfer back, no main-thread copy
|
|
164
|
+
try {
|
|
165
|
+
await workerDecodeInto(worker, rawBytes, metaId, codecMeta, buffer, size * bytesPerElement, outStride, mapping, bytesPerElement);
|
|
166
|
+
}
|
|
167
|
+
catch (error) {
|
|
168
|
+
worker.terminate();
|
|
169
|
+
throw error;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
else {
|
|
173
|
+
// Standard path: worker decodes, transfers back, main thread copies
|
|
174
|
+
let chunk;
|
|
175
|
+
try {
|
|
176
|
+
chunk = await workerDecode(worker, rawBytes, metaId, codecMeta);
|
|
177
|
+
}
|
|
178
|
+
catch (error) {
|
|
179
|
+
worker.terminate();
|
|
180
|
+
throw error;
|
|
181
|
+
}
|
|
182
|
+
setter.set_from_chunk(out, chunk, mapping);
|
|
183
|
+
}
|
|
184
|
+
return { worker, result: undefined };
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
// Execute all tasks with bounded concurrency via WorkerPool
|
|
188
|
+
if (tasks.length > 0) {
|
|
189
|
+
const { promise } = pool.runTasks(tasks);
|
|
190
|
+
await promise;
|
|
191
|
+
}
|
|
192
|
+
// If the final shape is empty (all integer selections), return a scalar
|
|
193
|
+
if (indexer.shape.length === 0) {
|
|
194
|
+
const unwrap = 'get' in out.data
|
|
195
|
+
? out.data.get(0)
|
|
196
|
+
: out.data[0];
|
|
197
|
+
// @ts-expect-error: TS can't narrow conditional type
|
|
198
|
+
return unwrap;
|
|
199
|
+
}
|
|
200
|
+
// @ts-expect-error: TS can't narrow conditional type
|
|
201
|
+
return out;
|
|
202
|
+
}
|
|
203
|
+
//# sourceMappingURL=get-worker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"get-worker.js","sourceRoot":"","sources":["../src/get-worker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAWH,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AACrD,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAA;AAC9C,OAAO,EACL,OAAO,EACP,WAAW,EACX,wBAAwB,EACxB,gCAAgC,EAChC,YAAY,GACb,MAAM,qBAAqB,CAAA;AAE5B,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAE3E;;;GAGG;AACH,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,mBAAmB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AAExE,mCAAmC;AACnC,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;AAYjC,KAAK,UAAU,iBAAiB,CAC9B,GAAwB;IAExB,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAA;IAEvB,+BAA+B;IAC/B,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,IAAI,YAAY,CAAiB,CAAA;IAC1F,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IACvC,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAA;QACpD,OAAO;YACL,SAAS,EAAE;gBACT,SAAS,EAAE,QAAQ,CAAC,SAAS;gBAC7B,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,aAAa,CAAC,WAAW;gBAC1D,MAAM,EAAE,QAAQ,CAAC,MAAM;aACxB;YACD,cAAc,EAAE,wBAAwB,CAAC,QAAQ,CAAC,kBAAkB,CAAC;YACrE,SAAS,EAAE,QAAQ,CAAC,UAAU,IAAI,IAAI;SACvC,CAAA;IACH,CAAC;IAED,uBAAuB;IACvB,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,IAAI,UAAU,CAAiB,CAAA;IACtF,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IACvC,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAA;QACpD,MAAM,MAAM,GAAoE,EAAE,CAAA;QAClF,IAAI,QAAQ,CAAC,KAAK,KAAK,GAAG,EAAE,CAAC;YAC3B,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,aAAa,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAA;QACnE,CAAC;QACD,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;YACxB,MAAM,EAAE,EAAE,EAAE,GAAG,aAAa,EAAE,GAAG,QAAQ,CAAC,UAAU,CAAA;YACpD,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,aAAa,EAAE,CAAC,CAAA;QAC1C,CAAC;QACD,KAAK,MAAM,EAAE,EAAE,EAAE,GAAG,aAAa,EAAE,IAAI,QAAQ,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;YAC9D,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,aAAa,EAAE,CAAC,CAAA;QAC1C,CAAC;QACD,OAAO;YACL,SAAS,EAAE;gBACT,SAAS,EAAE,GAAG,CAAC,KAAK;gBACpB,WAAW,EAAE,GAAG,CAAC,MAAM;gBACvB,MAAM,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,aAAa,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,CAAC;aAC9F;YACD,cAAc,EAAE,wBAAwB,CAAC;gBACvC,IAAI,EAAE,IAAI;gBACV,aAAa,EAAE,EAAE,SAAS,EAAE,QAAQ,CAAC,mBAAmB,IAAI,GAAG,EAAE;aAClE,CAAC;YACF,SAAS,EAAE,QAAQ,CAAC,UAAU,IAAI,IAAI;SACvC,CAAA;IACH,CAAC;IAED,qDAAqD;IACrD,OAAO;QACL,SAAS,EAAE;YACT,SAAS,EAAE,GAAG,CAAC,KAAK;YACpB,WAAW,EAAE,GAAG,CAAC,MAAM;YACvB,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,aAAa,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,CAAC;SACjE;QACD,cAAc,EAAE,wBAAwB,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QAC7D,SAAS,EAAE,IAAI;KAChB,CAAA;AACH,CAAC;AAED,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E;AAE9E;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAK7B,GAAwB,EACxB,YAAwB,IAAI,EAC5B,IAAmD;IAQnD,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,IAAI,CAAA;IAChC,MAAM,iBAAiB,GAAG,SAAS,IAAI,kBAAkB,CAAA;IACzD,MAAM,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAA;IAE7C,IAAI,SAAS,EAAE,CAAC;QACd,gCAAgC,EAAE,CAAA;IACpC,CAAC;IAED,uDAAuD;IACvD,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,SAAS,EAAE,GAAG,MAAM,iBAAiB,CAAC,GAAG,CAAC,CAAA;IAE7E,0EAA0E;IAC1E,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC,CAAA;IAEnC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;IAC9B,MAAM,eAAe,GAAI,GAAgD,CAAC,iBAAiB,CAAA;IAE3F,qBAAqB;IACrB,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC;QAC/B,SAAS;QACT,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,WAAW,EAAE,GAAG,CAAC,MAAM;KACxB,CAAC,CAAA;IAEF,+DAA+D;IAC/D,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;IACrE,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,GAAG,eAAe,EAAE,SAAS,CAAC,CAAA;IAC9D,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAqB,EAAE,CAAC,EAAE,IAAI,CAAC,CAAA;IACpD,MAAM,SAAS,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IAC5C,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK,EAAE,SAAS,CAAa,CAAA;IAEtE,qDAAqD;IACrD,MAAM,UAAU,GAAG,GAAG,CAAC,MAAM,CAAA;IAC7B,MAAM,YAAY,GAAG,WAAW,CAAC,UAAU,CAAC,CAAA;IAC5C,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;IAEvE,8BAA8B;IAC9B,MAAM,KAAK,GAA2B,EAAE,CAAA;IAExC,KAAK,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,IAAI,OAAO,EAAE,CAAC;QAChD,MAAM,QAAQ,GAAG,cAAc,CAAC,YAAY,CAAC,CAAA;QAC7C,MAAM,SAAS,GAAG,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAA;QAE5C,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,UAAyB,EAAE,EAAE;YAC7C,MAAM,MAAM,GAAG,UAAU,IAAI,IAAI,MAAM,CAAC,iBAAiB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAA;YAE9E,4CAA4C;YAC5C,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;YAE1D,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,+CAA+C;gBAC/C,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAA;gBACpC,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;oBACtB,6CAA6C;oBAC7C,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;gBAC3B,CAAC;gBACD,MAAM,KAAK,GAAa;oBACtB,IAAI,EAAE,SAA6B;oBACnC,KAAK,EAAE,UAAU,CAAC,KAAK,EAAE;oBACzB,MAAM,EAAE,YAAY,CAAC,KAAK,EAAE;iBAC7B,CAAA;gBACD,mDAAmD;gBACnD,MAAM,CAAC,cAAc,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAA;YAC5C,CAAC;iBAAM,IAAI,SAAS,EAAE,CAAC;gBACrB,8DAA8D;gBAC9D,uEAAuE;gBACvE,IAAI,CAAC;oBACH,MAAM,gBAAgB,CACpB,MAAM,EACN,QAAQ,EACR,MAAM,EACN,SAAS,EACT,MAA2B,EAC3B,IAAI,GAAG,eAAe,EACtB,SAAS,EACT,OAAO,EACP,eAAe,CAChB,CAAA;gBACH,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,CAAC,SAAS,EAAE,CAAA;oBAClB,MAAM,KAAK,CAAA;gBACb,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,oEAAoE;gBACpE,IAAI,KAAe,CAAA;gBACnB,IAAI,CAAC;oBACH,KAAK,GAAG,MAAM,YAAY,CAAI,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,CAAA;gBACpE,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,CAAC,SAAS,EAAE,CAAA;oBAClB,MAAM,KAAK,CAAA;gBACb,CAAC;gBACD,MAAM,CAAC,cAAc,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAA;YAC5C,CAAC;YAED,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,SAAiB,EAAE,CAAA;QAC9C,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,4DAA4D;IAC5D,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;QACxC,MAAM,OAAO,CAAA;IACf,CAAC;IAED,wEAAwE;IACxE,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,KAAK,IAAI,GAAG,CAAC,IAAI;YAC9B,CAAC,CAAE,GAAG,CAAC,IAAmD,CAAC,GAAG,CAAC,CAAC,CAAC;YACjE,CAAC,CAAE,GAAG,CAAC,IAAwC,CAAC,CAAC,CAAC,CAAA;QACpD,qDAAqD;QACrD,OAAO,MAAM,CAAA;IACf,CAAC;IAED,qDAAqD;IACrD,OAAO,GAAG,CAAA;AACZ,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fideus-labs/fizarrita — Worker-pool-accelerated get/set for zarrita.
|
|
3
|
+
*
|
|
4
|
+
* Provides `getWorker` and `setWorker` as drop-in replacements for zarrita's
|
|
5
|
+
* `get` and `set` that offload codec encode/decode to Web Workers via a
|
|
6
|
+
* WorkerPool with bounded concurrency.
|
|
7
|
+
*/
|
|
8
|
+
export { getWorker } from './get-worker.js';
|
|
9
|
+
export { setWorker } from './set-worker.js';
|
|
10
|
+
export type { GetWorkerOptions, SetWorkerOptions, CodecChunkMeta, } from './types.js';
|
|
11
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAC3C,YAAY,EACV,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,GACf,MAAM,YAAY,CAAA"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fideus-labs/fizarrita — Worker-pool-accelerated get/set for zarrita.
|
|
3
|
+
*
|
|
4
|
+
* Provides `getWorker` and `setWorker` as drop-in replacements for zarrita's
|
|
5
|
+
* `get` and `set` that offload codec encode/decode to Web Workers via a
|
|
6
|
+
* WorkerPool with bounded concurrency.
|
|
7
|
+
*/
|
|
8
|
+
export { getWorker } from './get-worker.js';
|
|
9
|
+
export { setWorker } from './set-worker.js';
|
|
10
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Codec pipeline builder — adapted from zarrita.js/src/codecs.ts
|
|
3
|
+
*
|
|
4
|
+
* Uses zarrita's publicly exported `registry` to load codecs and
|
|
5
|
+
* builds encode/decode pipelines from codec metadata.
|
|
6
|
+
*
|
|
7
|
+
* Self-contained — only imports `registry` from zarrita's public API.
|
|
8
|
+
*/
|
|
9
|
+
import type { Chunk, CodecMetadata, DataType } from 'zarrita';
|
|
10
|
+
interface ChunkMetadata<D extends DataType> {
|
|
11
|
+
data_type: D;
|
|
12
|
+
shape: number[];
|
|
13
|
+
codecs: CodecMetadata[];
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Create a codec pipeline from chunk metadata.
|
|
17
|
+
*
|
|
18
|
+
* Uses zarrita's publicly exported `registry` to resolve codec implementations.
|
|
19
|
+
* Lazily loads codecs on first encode/decode call, then caches them.
|
|
20
|
+
*
|
|
21
|
+
* @param chunk_metadata - The data_type, chunk_shape, and codecs array
|
|
22
|
+
* @returns An object with encode and decode methods
|
|
23
|
+
*/
|
|
24
|
+
export declare function create_codec_pipeline<D extends DataType>(chunk_metadata: ChunkMetadata<D>): {
|
|
25
|
+
encode(chunk: Chunk<D>): Promise<Uint8Array>;
|
|
26
|
+
decode(bytes: Uint8Array): Promise<Chunk<D>>;
|
|
27
|
+
};
|
|
28
|
+
export {};
|
|
29
|
+
//# sourceMappingURL=codec-pipeline.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"codec-pipeline.d.ts","sourceRoot":"","sources":["../../src/internals/codec-pipeline.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AAO7D,UAAU,aAAa,CAAC,CAAC,SAAS,QAAQ;IACxC,SAAS,EAAE,CAAC,CAAA;IACZ,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,MAAM,EAAE,aAAa,EAAE,CAAA;CACxB;AAyID;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CAAC,CAAC,SAAS,QAAQ,EACtD,cAAc,EAAE,aAAa,CAAC,CAAC,CAAC,GAC/B;IACD,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;IAC5C,MAAM,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;CAC7C,CA2BA"}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Codec pipeline builder — adapted from zarrita.js/src/codecs.ts
|
|
3
|
+
*
|
|
4
|
+
* Uses zarrita's publicly exported `registry` to load codecs and
|
|
5
|
+
* builds encode/decode pipelines from codec metadata.
|
|
6
|
+
*
|
|
7
|
+
* Self-contained — only imports `registry` from zarrita's public API.
|
|
8
|
+
*/
|
|
9
|
+
import { registry } from 'zarrita';
|
|
10
|
+
import { get_ctr, get_strides } from './util.js';
|
|
11
|
+
// ---------------------------------------------------------------------------
|
|
12
|
+
// Fallback BytesCodec
|
|
13
|
+
// ---------------------------------------------------------------------------
|
|
14
|
+
const LITTLE_ENDIAN_OS = (() => {
|
|
15
|
+
const a = new Uint32Array([0x12345678]);
|
|
16
|
+
const b = new Uint8Array(a.buffer, a.byteOffset, a.byteLength);
|
|
17
|
+
return !(b[0] === 0x12);
|
|
18
|
+
})();
|
|
19
|
+
function byteswap_inplace(view, bytes_per_element) {
|
|
20
|
+
const numFlips = bytes_per_element / 2;
|
|
21
|
+
const endByteIndex = bytes_per_element - 1;
|
|
22
|
+
let t = 0;
|
|
23
|
+
for (let i = 0; i < view.length; i += bytes_per_element) {
|
|
24
|
+
for (let j = 0; j < numFlips; j += 1) {
|
|
25
|
+
t = view[i + j];
|
|
26
|
+
view[i + j] = view[i + endByteIndex - j];
|
|
27
|
+
view[i + endByteIndex - j] = t;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Fallback BytesCodec for when no explicit array_to_bytes codec is specified.
|
|
33
|
+
* Handles TypedArray <-> Uint8Array conversion with optional endian swap.
|
|
34
|
+
*/
|
|
35
|
+
function createBytesCodec(endian, meta) {
|
|
36
|
+
const Ctr = get_ctr(meta.data_type);
|
|
37
|
+
const shape = meta.shape;
|
|
38
|
+
const stride = get_strides(shape, 'C');
|
|
39
|
+
const sample = new Ctr(0);
|
|
40
|
+
const BYTES_PER_ELEMENT = sample.BYTES_PER_ELEMENT;
|
|
41
|
+
return {
|
|
42
|
+
encode(chunk) {
|
|
43
|
+
const bytes = new Uint8Array(chunk.data.buffer, chunk.data.byteOffset, chunk.data.byteLength);
|
|
44
|
+
if (LITTLE_ENDIAN_OS && endian === 'big') {
|
|
45
|
+
byteswap_inplace(bytes, BYTES_PER_ELEMENT);
|
|
46
|
+
}
|
|
47
|
+
return bytes;
|
|
48
|
+
},
|
|
49
|
+
decode(bytes) {
|
|
50
|
+
if (LITTLE_ENDIAN_OS && endian === 'big') {
|
|
51
|
+
byteswap_inplace(bytes, BYTES_PER_ELEMENT);
|
|
52
|
+
}
|
|
53
|
+
return {
|
|
54
|
+
data: new Ctr(bytes.buffer, bytes.byteOffset, bytes.byteLength / BYTES_PER_ELEMENT),
|
|
55
|
+
shape,
|
|
56
|
+
stride,
|
|
57
|
+
};
|
|
58
|
+
},
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
// ---------------------------------------------------------------------------
|
|
62
|
+
// Load codecs from registry
|
|
63
|
+
// ---------------------------------------------------------------------------
|
|
64
|
+
async function load_codecs(chunk_meta) {
|
|
65
|
+
const promises = chunk_meta.codecs.map(async (meta) => {
|
|
66
|
+
const factory = registry.get(meta.name);
|
|
67
|
+
if (!factory)
|
|
68
|
+
throw new Error(`Unknown codec: ${meta.name}`);
|
|
69
|
+
const CodecClass = await factory();
|
|
70
|
+
return { CodecClass: CodecClass, meta };
|
|
71
|
+
});
|
|
72
|
+
const array_to_array = [];
|
|
73
|
+
let array_to_bytes;
|
|
74
|
+
const bytes_to_bytes = [];
|
|
75
|
+
for await (const { CodecClass, meta } of promises) {
|
|
76
|
+
const codec = CodecClass.fromConfig(meta.configuration, chunk_meta);
|
|
77
|
+
switch (codec.kind) {
|
|
78
|
+
case 'array_to_array':
|
|
79
|
+
array_to_array.push(codec);
|
|
80
|
+
break;
|
|
81
|
+
case 'array_to_bytes':
|
|
82
|
+
array_to_bytes = codec;
|
|
83
|
+
break;
|
|
84
|
+
default:
|
|
85
|
+
bytes_to_bytes.push(codec);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
if (!array_to_bytes) {
|
|
89
|
+
// Default BytesCodec for numeric types
|
|
90
|
+
array_to_bytes = createBytesCodec('little', chunk_meta);
|
|
91
|
+
}
|
|
92
|
+
return { array_to_array, array_to_bytes, bytes_to_bytes };
|
|
93
|
+
}
|
|
94
|
+
// ---------------------------------------------------------------------------
|
|
95
|
+
// create_codec_pipeline
|
|
96
|
+
// ---------------------------------------------------------------------------
|
|
97
|
+
/**
|
|
98
|
+
* Create a codec pipeline from chunk metadata.
|
|
99
|
+
*
|
|
100
|
+
* Uses zarrita's publicly exported `registry` to resolve codec implementations.
|
|
101
|
+
* Lazily loads codecs on first encode/decode call, then caches them.
|
|
102
|
+
*
|
|
103
|
+
* @param chunk_metadata - The data_type, chunk_shape, and codecs array
|
|
104
|
+
* @returns An object with encode and decode methods
|
|
105
|
+
*/
|
|
106
|
+
export function create_codec_pipeline(chunk_metadata) {
|
|
107
|
+
let codecs;
|
|
108
|
+
return {
|
|
109
|
+
async encode(chunk) {
|
|
110
|
+
if (!codecs)
|
|
111
|
+
codecs = await load_codecs(chunk_metadata);
|
|
112
|
+
for (const codec of codecs.array_to_array) {
|
|
113
|
+
chunk = await codec.encode(chunk);
|
|
114
|
+
}
|
|
115
|
+
let bytes = await codecs.array_to_bytes.encode(chunk);
|
|
116
|
+
for (const codec of codecs.bytes_to_bytes) {
|
|
117
|
+
bytes = await codec.encode(bytes);
|
|
118
|
+
}
|
|
119
|
+
return bytes;
|
|
120
|
+
},
|
|
121
|
+
async decode(bytes) {
|
|
122
|
+
if (!codecs)
|
|
123
|
+
codecs = await load_codecs(chunk_metadata);
|
|
124
|
+
for (let i = codecs.bytes_to_bytes.length - 1; i >= 0; i--) {
|
|
125
|
+
bytes = await codecs.bytes_to_bytes[i].decode(bytes);
|
|
126
|
+
}
|
|
127
|
+
let chunk = await codecs.array_to_bytes.decode(bytes);
|
|
128
|
+
for (let i = codecs.array_to_array.length - 1; i >= 0; i--) {
|
|
129
|
+
chunk = await codecs.array_to_array[i].decode(chunk);
|
|
130
|
+
}
|
|
131
|
+
return chunk;
|
|
132
|
+
},
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
//# sourceMappingURL=codec-pipeline.js.map
|