@walkthru-earth/objex 0.1.0 → 1.1.0

package/dist/utils/zarr.js

@@ -1,11 +1,123 @@
 /**
  * Zarr metadata parsing utilities.
  *
- * Handles consolidated metadata for both Zarr v2 (.zmetadata) and v3 (zarr.json),
- * plus a zarrita fallback for non-consolidated stores.
+ * Builds a hierarchical tree of groups and arrays from consolidated metadata
+ * (Zarr v2 .zmetadata, v3 zarr.json), with zarrita fallback for non-consolidated stores.
  */
+import { formatFileSize } from './format.js';
+// ---------------------------------------------------------------------------
+// Register numcodecs-wrapped codecs with zarrita's codec registry.
+// Zarr v3 stores produced by Python's zarr-python may wrap codecs with the
+// "numcodecs." prefix (e.g. "numcodecs.zlib", "numcodecs.shuffle").
+// zarrita only registers the bare names, so we add aliases + a byte shuffle.
+// ---------------------------------------------------------------------------
+/**
+ * Byte shuffle codec (HDF5 / numcodecs byte shuffle).
+ * Rearranges bytes within elements to improve downstream compression ratios.
+ * Operates as a bytes_to_bytes filter in the codec pipeline.
+ */
+const ShuffleCodec = {
+    kind: 'bytes_to_bytes',
+    fromConfig(config) {
+        const elementsize = config?.elementsize ?? 4;
+        return {
+            kind: 'bytes_to_bytes',
+            encode(data) {
+                const count = data.length / elementsize;
+                const out = new Uint8Array(data.length);
+                for (let i = 0; i < count; i++) {
+                    for (let j = 0; j < elementsize; j++) {
+                        out[j * count + i] = data[i * elementsize + j];
+                    }
+                }
+                return out;
+            },
+            decode(data) {
+                const count = data.length / elementsize;
+                const out = new Uint8Array(data.length);
+                for (let i = 0; i < count; i++) {
+                    for (let j = 0; j < elementsize; j++) {
+                        out[i * elementsize + j] = data[j * count + i];
+                    }
+                }
+                return out;
+            }
+        };
+    }
+};
+/** Register numcodecs-prefixed aliases with zarrita's codec registry. */
+async function registerNumcodecs() {
+    try {
+        const { registry } = await import('zarrita');
+        // Alias existing codecs with numcodecs. prefix
+        for (const name of ['zlib', 'gzip', 'blosc', 'lz4', 'zstd']) {
+            const existing = registry.get(name);
+            if (existing)
+                registry.set(`numcodecs.${name}`, existing);
+        }
+        // Register shuffle codec
+        registry.set('numcodecs.shuffle', () => Promise.resolve(ShuffleCodec));
+        registry.set('shuffle', () => Promise.resolve(ShuffleCodec));
+    }
+    catch {
+        // zarrita not available — skip registration
+    }
+}
+let _codecsPromise = null;
+/** Ensure numcodecs codecs are registered. Await before creating ZarrLayer. */
+export function ensureCodecsRegistered() {
+    if (!_codecsPromise)
+        _codecsPromise = registerNumcodecs();
+    return _codecsPromise;
+}
+// Start registration immediately at module load
+ensureCodecsRegistered();
+/** Zarr store marker files — presence of any indicates a Zarr store. */
+export const ZARR_MARKER_FILES = new Set([
+    'zarr.json',
+    '.zmetadata',
+    '.zgroup',
+    '.zarray',
+    '.zattrs'
+]);
+/** Zarr marker file suffixes used in URLs (with leading slash). */
+const ZARR_MARKER_SUFFIXES = ['/zarr.json', '/.zmetadata', '/.zgroup', '/.zarray', '/.zattrs'];
+/**
+ * Detect whether a set of file names contains Zarr marker files.
+ * Returns the detected version (2 or 3) or null if not detected.
+ */
+export function detectZarrMarkers(fileNames) {
+    let hasV3 = false;
+    let hasV2 = false;
+    for (const name of fileNames) {
+        if (name === 'zarr.json')
+            hasV3 = true;
+        else if (ZARR_MARKER_FILES.has(name))
+            hasV2 = true;
+    }
+    if (hasV3)
+        return { detected: true, version: 3 };
+    if (hasV2)
+        return { detected: true, version: 2 };
+    return { detected: false, version: null };
+}
+/**
+ * If a URL points to a Zarr marker file, strip the marker suffix and return the store URL.
+ * Returns null if the URL doesn't end with a known marker suffix.
+ */
+export function extractZarrStoreUrl(url) {
+    for (const suffix of ZARR_MARKER_SUFFIXES) {
+        if (url.endsWith(suffix)) {
+            return url.slice(0, -suffix.length);
+        }
+    }
+    return null;
+}
+// ---------------------------------------------------------------------------
+// Kept helpers
+// ---------------------------------------------------------------------------
 /** Dimension-like variable names treated as coordinates. */
-const DIM_LIKE_NAMES = new Set([
+export const DIM_LIKE_NAMES = new Set([
     'x',
     'y',
     'lat',
@@ -36,110 +148,279 @@ export function formatShape(shape) {
         return 'scalar';
     return `[${shape.join(' × ')}]`;
 }
-/** Collect coordinate variable names from `coordinates` attributes across all entries. */
-function collectCoordNames(entries) {
-    const names = new Set();
-    for (const [, info] of entries) {
-        const coordStr = info.attributes?.coordinates;
-        if (typeof coordStr === 'string') {
-            for (const c of coordStr.split(/\s+/))
-                names.add(c);
+// ---------------------------------------------------------------------------
+// Metadata display helpers
+// ---------------------------------------------------------------------------
+/** Byte size of a dtype string. Handles v2 (`<f4`, `|u1`) and v3 (`float32`, `uint8`). */
+export function dtypeByteSize(dtype) {
+    // v2 style: last char(s) are the byte width e.g. "<f4" → 4, "|u1" → 1
+    const v2 = /^[<>|]?[a-zA-Z](\d+)$/.exec(dtype);
+    if (v2)
+        return Number(v2[1]);
+    // v3 style: "float32" → 4, "int16" → 2, "uint8" → 1, "complex128" → 16
+    const v3 = /(\d+)$/.exec(dtype);
+    if (v3)
+        return Number(v3[1]) / 8;
+    // bool
+    if (dtype === 'bool')
+        return 1;
+    return 1; // fallback
+}
+/** Format chunk count: `"36 [6 × 6]"` */
+export function computeChunkCount(shape, chunks) {
+    if (!shape?.length || !chunks?.length || shape.length !== chunks.length)
+        return null;
+    const dims = shape.map((s, i) => Math.ceil(s / chunks[i]));
+    const total = dims.reduce((a, b) => a * b, 1);
+    return `${total.toLocaleString()} [${dims.join(' × ')}]`;
+}
+/** Format chunk size in bytes: `"817.6 KB"` */
+export function computeChunkSize(chunks, dtype) {
+    if (!chunks?.length || !dtype)
+        return null;
+    const elements = chunks.reduce((a, b) => a * b, 1);
+    return formatFileSize(elements * dtypeByteSize(dtype));
+}
+/** Format uncompressed size: `"28.7 MB"` */
+export function computeUncompressed(shape, dtype) {
+    if (!shape?.length || !dtype)
+        return null;
+    const elements = shape.reduce((a, b) => a * b, 1);
+    return formatFileSize(elements * dtypeByteSize(dtype));
+}
+/** Format codec pipeline for display. */
+export function formatCodecs(node) {
+    // v3: codecs array
+    if (node.codecs?.length) {
+        return node.codecs
+            .map((c) => {
+            const name = c.name ?? c.codec_id ?? (typeof c === 'string' ? c : '');
+            if (!name)
+                return JSON.stringify(c);
+            const cfg = c.configuration ?? c.codec_config ?? {};
+            const parts = [];
+            if (cfg.cname)
+                parts.push(cfg.cname);
+            const level = cfg.clevel ?? cfg.level;
+            if (level != null)
+                parts.push(`level ${level}`);
+            if (cfg.shuffle != null)
+                parts.push(cfg.shuffle ? 'shuffle' : 'no shuffle');
+            if (cfg.typesize)
+                parts.push(`typesize ${cfg.typesize}`);
+            if (cfg.elementsize)
+                parts.push(`elementsize ${cfg.elementsize}`);
+            return parts.length ? `${name} (${parts.join(', ')})` : name;
+        })
+            .join(' → ');
+    }
+    // v2: compressor + filters
+    if (node.compressor) {
+        const c = node.compressor;
+        const name = c.id ?? c.codec ?? 'unknown';
+        const parts = [];
+        if (c.cname)
+            parts.push(c.cname);
+        if (c.clevel != null)
+            parts.push(`level ${c.clevel}`);
+        if (c.shuffle != null)
+            parts.push(c.shuffle ? 'shuffle' : 'no shuffle');
+        const base = parts.length ? `${name} (${parts.join(', ')})` : name;
+        if (node.filters?.length) {
+            const filterStr = node.filters.map((f) => f.id ?? JSON.stringify(f)).join(', ');
+            return `${filterStr} → ${base}`;
         }
+        return base;
     }
-    return names;
+    return null;
 }
-/** Parse Zarr v3 consolidated metadata (zarr.json). */
-export function parseV3Consolidated(data) {
-    const meta = data.consolidated_metadata?.metadata ?? {};
-    const attrs = data.attributes ?? {};
-    const vars = [];
-    const coords = [];
-    let srAttrs = null;
-    const coordNames = collectCoordNames(Object.entries(meta));
-    for (const [name, info] of Object.entries(meta)) {
-        if (!info.shape)
-            continue;
-        const v = {
-            name,
-            shape: info.shape,
-            dtype: info.data_type ?? 'unknown',
-            dims: info.attributes?._ARRAY_DIMENSIONS ?? inferDims(name, info.shape),
-            chunks: info.chunk_grid?.configuration?.chunk_shape ?? [],
-            attributes: info.attributes ?? {}
-        };
-        if (name === 'spatial_ref') {
-            srAttrs = v.attributes;
-            coords.push(v);
+/** Format chunk_key_encoding for display: `"default (sep: "/")"` */
+export function formatChunkKeys(node) {
+    return node.chunkKeyEncoding ?? null;
+}
+/** Find a node by slash-delimited path. */
+export function findNodeByPath(root, path) {
+    if (path === '/' || path === '')
+        return root;
+    const parts = path.replace(/^\//, '').split('/');
+    let current = root;
+    for (const part of parts) {
+        const child = current.children.find((c) => c.name === part);
+        if (!child)
+            return null;
+        current = child;
+    }
+    return current;
+}
+// ---------------------------------------------------------------------------
+// Tree builders
+// ---------------------------------------------------------------------------
+function makeNode(path, kind, attrs) {
+    const name = path === '/' ? '/' : path.split('/').pop();
+    return { path, name, kind, children: [], attributes: attrs };
+}
+/** Ensure all intermediate groups exist and return the parent for `path`. */
+function ensureParent(root, path) {
+    const parts = path.replace(/^\//, '').split('/');
+    parts.pop(); // remove leaf
+    let current = root;
+    let currentPath = '';
+    for (const part of parts) {
+        currentPath += `/${part}`;
+        let child = current.children.find((c) => c.name === part);
+        if (!child) {
+            child = makeNode(currentPath, 'group', {});
+            current.children.push(child);
         }
-        else if (coordNames.has(name) || DIM_LIKE_NAMES.has(name) || v.shape.length <= 1) {
-            coords.push(v);
+        current = child;
+    }
+    return current;
+}
+/** Build tree from Zarr v3 consolidated metadata (zarr.json). */
+export function buildV3Tree(data) {
+    const meta = data.consolidated_metadata?.metadata ?? {};
+    const rootAttrs = data.attributes ?? {};
+    const root = makeNode('/', 'group', rootAttrs);
+    let totalNodes = 1;
+    let spatialRefAttrs = null;
+    for (const [key, info] of Object.entries(meta)) {
+        const path = `/${key}`;
+        const attrs = info.attributes ?? {};
+        if (info.node_type === 'group' || (!info.shape && !info.data_type)) {
+            // Group node
+            const node = makeNode(path, 'group', attrs);
+            const parent = ensureParent(root, path);
+            // Avoid duplicating if ensureParent already created this node
+            const existing = parent.children.find((c) => c.name === node.name);
+            if (existing) {
+                existing.attributes = { ...existing.attributes, ...attrs };
+            }
+            else {
+                parent.children.push(node);
+            }
+            totalNodes++;
         }
         else {
-            vars.push(v);
+            // Array node
+            const node = makeNode(path, 'array', attrs);
+            node.shape = info.shape;
+            node.dtype = info.data_type ?? 'unknown';
+            node.dims =
+                info.dimension_names ?? attrs._ARRAY_DIMENSIONS ?? inferDims(node.name, node.shape ?? []);
+            node.chunks = info.chunk_grid?.configuration?.chunk_shape ?? [];
+            node.fillValue = info.fill_value;
+            node.codecs = info.codecs ?? [];
+            // Parse chunk_key_encoding for v3
+            const cke = info.chunk_key_encoding;
+            if (cke) {
+                const sep = cke.configuration?.separator ?? '/';
+                node.chunkKeyEncoding = `${cke.name ?? 'default'} (sep: "${sep}")`;
+            }
+            const parent = ensureParent(root, path);
+            parent.children.push(node);
+            totalNodes++;
+            if (node.name === 'spatial_ref') {
+                spatialRefAttrs = attrs;
+            }
         }
     }
-    return { storeAttrs: attrs, variables: vars, coords, spatialRefAttrs: srAttrs };
+    // Sort children alphabetically, groups first
+    sortTree(root);
+    return {
+        root,
+        zarrVersion: 3,
+        totalNodes,
+        storeAttrs: rootAttrs,
+        spatialRefAttrs
+    };
 }
-/** Parse Zarr v2 consolidated metadata (.zmetadata). */
-export function parseV2Consolidated(data) {
+/** Build tree from Zarr v2 consolidated metadata (.zmetadata). */
+export function buildV2Tree(data) {
     const meta = data.metadata ?? {};
-    const attrs = meta['.zattrs'] ?? {};
-    const vars = [];
-    const coords = [];
-    let srAttrs = null;
-    const arrayKeys = Object.keys(meta).filter((k) => k.endsWith('/.zarray'));
-    // Collect coordinate names from variable attributes
-    const coordNames = new Set();
-    for (const key of arrayKeys) {
-        const name = key.replace('/.zarray', '');
-        const varAttrs = meta[`${name}/.zattrs`] ?? {};
-        const coordStr = varAttrs.coordinates;
-        if (typeof coordStr === 'string') {
-            for (const c of coordStr.split(/\s+/))
-                coordNames.add(c);
+    const rootAttrs = meta['.zattrs'] ?? {};
+    const root = makeNode('/', 'group', rootAttrs);
+    let totalNodes = 1;
+    let spatialRefAttrs = null;
+    // Collect group paths
+    const groupKeys = Object.keys(meta).filter((k) => k.endsWith('/.zgroup'));
+    for (const key of groupKeys) {
+        const name = key.replace('/.zgroup', '');
+        if (!name)
+            continue; // root
+        const path = `/${name}`;
+        const attrs = meta[`${name}/.zattrs`] ?? {};
+        const parent = ensureParent(root, path);
+        const existing = parent.children.find((c) => c.name === name.split('/').pop());
+        if (existing) {
+            existing.attributes = { ...existing.attributes, ...attrs };
+        }
+        else {
+            parent.children.push(makeNode(path, 'group', attrs));
         }
+        totalNodes++;
     }
+    // Collect array paths
+    const arrayKeys = Object.keys(meta).filter((k) => k.endsWith('/.zarray'));
     for (const key of arrayKeys) {
         const name = key.replace('/.zarray', '');
+        const path = `/${name}`;
         const zarray = meta[key];
-        const varAttrs = meta[`${name}/.zattrs`] ?? {};
-        const v = {
-            name,
-            shape: zarray.shape ?? [],
-            dtype: zarray.dtype ?? 'unknown',
-            dims: varAttrs._ARRAY_DIMENSIONS ?? [],
-            chunks: zarray.chunks ?? [],
-            attributes: varAttrs
-        };
-        if (name === 'spatial_ref') {
-            srAttrs = v.attributes;
-            coords.push(v);
-        }
-        else if (coordNames.has(name) || DIM_LIKE_NAMES.has(name) || v.shape.length <= 1) {
-            coords.push(v);
-        }
-        else {
-            vars.push(v);
+        const attrs = meta[`${name}/.zattrs`] ?? {};
+        const shape = zarray.shape ?? [];
+        const node = makeNode(path, 'array', attrs);
+        node.shape = shape;
+        node.dtype = zarray.dtype ?? 'unknown';
+        node.dims = attrs._ARRAY_DIMENSIONS ?? inferDims(node.name, shape);
+        node.chunks = zarray.chunks ?? [];
+        node.fillValue = zarray.fill_value;
+        node.compressor = zarray.compressor ?? null;
+        node.filters = zarray.filters ?? [];
+        const parent = ensureParent(root, path);
+        parent.children.push(node);
+        totalNodes++;
+        if (node.name === 'spatial_ref') {
+            spatialRefAttrs = attrs;
         }
     }
-    return { storeAttrs: attrs, variables: vars, coords, spatialRefAttrs: srAttrs };
+    sortTree(root);
+    return {
+        root,
+        zarrVersion: 2,
+        totalNodes,
+        storeAttrs: rootAttrs,
+        spatialRefAttrs
+    };
+}
+/** Recursively sort children: groups first, then alphabetically. */
+function sortTree(node) {
+    node.children.sort((a, b) => {
+        if (a.kind !== b.kind)
+            return a.kind === 'group' ? -1 : 1;
+        return a.name.localeCompare(b.name);
+    });
+    for (const child of node.children) {
+        sortTree(child);
+    }
 }
+// ---------------------------------------------------------------------------
+// Fetchers
+// ---------------------------------------------------------------------------
 /**
- * Fetch consolidated metadata from a Zarr store URL.
- * Tries v3 (zarr.json) first, then v2 (.zmetadata).
+ * Fetch hierarchy from a Zarr store URL.
+ * Tries v3 (zarr.json) first, then v2 (.zmetadata), then zarrita fallback.
  */
-export async function fetchConsolidated(storeUrl) {
+export async function fetchHierarchy(storeUrl, storeName, signal) {
     // Try Zarr v3 zarr.json
     try {
-        const res = await fetch(`${storeUrl}/zarr.json`);
+        const res = await fetch(`${storeUrl}/zarr.json`, { signal });
         if (res.ok) {
             const data = await res.json();
             if (data.zarr_format === 3) {
                 if (data.consolidated_metadata) {
-                    return { ...parseV3Consolidated(data), zarrVersion: 3 };
+                    return buildV3Tree(data);
                 }
-                // v3 without consolidated metadata — skip v2 probe
-                return null;
+                // V3 without consolidated metadata — discover children
+                return discoverV3Children(storeUrl, data, signal);
             }
         }
     }
@@ -148,53 +429,119 @@ export async function fetchConsolidated(storeUrl) {
     }
     // Try Zarr v2 .zmetadata
     try {
-        const res = await fetch(`${storeUrl}/.zmetadata`);
+        const res = await fetch(`${storeUrl}/.zmetadata`, { signal });
         if (res.ok) {
             const data = await res.json();
             if (data.metadata) {
-                return { ...parseV2Consolidated(data), zarrVersion: 2 };
+                return buildV2Tree(data);
             }
         }
     }
     catch {
         /* ignore */
     }
-    return null;
+    // Fallback: zarrita probe
+    return probeHierarchy(storeUrl, storeName);
+}
+/**
+ * Discover children of a v3 store without consolidated metadata.
+ * Probes child paths from zarr.json attributes (e.g. multiscales convention)
+ * and by fetching individual zarr.json files for discovered paths.
+ */
+async function discoverV3Children(storeUrl, rootData, signal) {
+    const rootAttrs = rootData.attributes ?? {};
+    const root = makeNode('/', 'group', rootAttrs);
+    let totalNodes = 1;
+    // Collect candidate child names from conventions and common patterns
+    const candidates = new Set();
+    // Multiscales convention: layout[].asset lists child array names
+    const multiscales = rootAttrs.multiscales;
+    if (multiscales?.layout && Array.isArray(multiscales.layout)) {
+        for (const entry of multiscales.layout) {
+            if (entry.asset)
+                candidates.add(String(entry.asset));
+        }
+    }
+    // Probe each candidate path for zarr.json
+    const probes = [...candidates].map(async (name) => {
+        try {
+            const res = await fetch(`${storeUrl}/${name}/zarr.json`, { signal });
+            if (!res.ok)
+                return null;
+            const data = await res.json();
+            if (data.node_type === 'array' && data.shape) {
+                const node = makeNode(`/${name}`, 'array', data.attributes ?? {});
+                node.shape = data.shape;
+                node.dtype = data.data_type ?? 'unknown';
+                node.dims = data.dimension_names ?? inferDims(name, data.shape);
+                node.chunks = data.chunk_grid?.configuration?.chunk_shape ?? [];
+                node.fillValue = data.fill_value;
+                node.codecs = data.codecs ?? [];
+                const cke = data.chunk_key_encoding;
+                if (cke) {
+                    const sep = cke.configuration?.separator ?? '/';
+                    node.chunkKeyEncoding = `${cke.name ?? 'default'} (sep: "${sep}")`;
+                }
+                return node;
+            }
+            if (data.node_type === 'group') {
+                return makeNode(`/${name}`, 'group', data.attributes ?? {});
+            }
+            return null;
+        }
+        catch {
+            return null;
+        }
+    });
+    const results = await Promise.all(probes);
+    for (const node of results) {
+        if (node) {
+            root.children.push(node);
+            totalNodes++;
+        }
+    }
+    sortTree(root);
+    return {
+        root,
+        zarrVersion: 3,
+        totalNodes,
+        storeAttrs: rootAttrs,
+        spatialRefAttrs: null
+    };
 }
 /**
  * Fallback: probe a Zarr store using zarrita when consolidated metadata is unavailable.
- * @param storeName - Display name for the root array (e.g. file name without .zarr)
  */
-export async function probeWithZarrita(storeUrl, storeName) {
+export async function probeHierarchy(storeUrl, storeName) {
     const zarrita = await import('zarrita');
     const store = new zarrita.FetchStore(storeUrl);
     try {
         const arr = await zarrita.open(store, { kind: 'array' });
-        const v = {
-            name: storeName,
-            shape: arr.shape ?? [],
-            dtype: String(arr.dtype ?? 'unknown'),
-            dims: arr.attrs?._ARRAY_DIMENSIONS ?? [],
-            chunks: arr.chunks ?? [],
-            attributes: arr.attrs ?? {}
-        };
+        const root = makeNode('/', 'group', {});
+        const node = makeNode(`/${storeName}`, 'array', arr.attrs ?? {});
+        node.shape = arr.shape ?? [];
+        node.dtype = String(arr.dtype ?? 'unknown');
+        node.dims = arr.attrs?._ARRAY_DIMENSIONS ?? [];
+        node.chunks = arr.chunks ?? [];
+        root.children.push(node);
         return {
+            root,
+            zarrVersion: null,
+            totalNodes: 2,
             storeAttrs: {},
-            variables: [v],
-            coords: [],
-            spatialRefAttrs: null,
-            zarrVersion: null
+            spatialRefAttrs: null
         };
     }
     catch {
         try {
-            await zarrita.open(store, { kind: 'group' });
+            const grp = await zarrita.open(store, { kind: 'group' });
+            const attrs = grp.attrs ?? {};
             return {
-                storeAttrs: {},
-                variables: [],
-                coords: [],
-                spatialRefAttrs: null,
-                zarrVersion: null
+                root: makeNode('/', 'group', attrs),
+                zarrVersion: null,
+                totalNodes: 1,
+                storeAttrs: attrs,
+                spatialRefAttrs: null
             };
         }
         catch {