@loaders.gl/flatgeobuf 4.2.0-alpha.4 → 4.2.0-alpha.6

package/dist/flatgeobuf/3.27.2/packedrtree.js

@@ -1,138 +1,191 @@
 import Config from "./config.js";
 import Logger from "./logger.js";
 export const NODE_ITEM_BYTE_LEN = 8 * 4 + 8;
+/**
+ * @deprecated Use `NODE_ITEM_BYTE_LEN` instead.
+ */
 export const NODE_ITEM_LEN = NODE_ITEM_BYTE_LEN;
+// default branching factor of a node in the rtree
+//
+// actual value will be specified in the header but
+// this can be useful for having reasonably sized guesses for fetch-sizes when
+// streaming results
 export const DEFAULT_NODE_SIZE = 16;
 export function calcTreeSize(numItems, nodeSize) {
-  nodeSize = Math.min(Math.max(Number(nodeSize), 2), 65535);
-  let n = numItems;
-  let numNodes = n;
-  do {
-    n = Math.ceil(n / nodeSize);
-    numNodes += n;
-  } while (n !== 1);
-  return numNodes * NODE_ITEM_BYTE_LEN;
+    nodeSize = Math.min(Math.max(Number(nodeSize), 2), 65535);
+    let n = numItems;
+    let numNodes = n;
+    do {
+        n = Math.ceil(n / nodeSize);
+        numNodes += n;
+    } while (n !== 1);
+    return numNodes * NODE_ITEM_BYTE_LEN;
 }
+/**
+ * returns [levelOffset, numNodes] for each level
+ */
 export function generateLevelBounds(numItems, nodeSize) {
-  if (nodeSize < 2) throw new Error('Node size must be at least 2');
-  if (numItems === 0) throw new Error('Number of items must be greater than 0');
-  let n = numItems;
-  let numNodes = n;
-  const levelNumNodes = [n];
-  do {
-    n = Math.ceil(n / nodeSize);
-    numNodes += n;
-    levelNumNodes.push(n);
-  } while (n !== 1);
-  const levelOffsets = [];
-  n = numNodes;
-  for (const size of levelNumNodes) {
-    levelOffsets.push(n - size);
-    n -= size;
-  }
-  const levelBounds = [];
-  for (let i = 0; i < levelNumNodes.length; i++) levelBounds.push([levelOffsets[i], levelOffsets[i] + levelNumNodes[i]]);
-  return levelBounds;
+    if (nodeSize < 2)
+        throw new Error('Node size must be at least 2');
+    if (numItems === 0)
+        throw new Error('Number of items must be greater than 0');
+    // number of nodes per level in bottom-up order
+    let n = numItems;
+    let numNodes = n;
+    const levelNumNodes = [n];
+    do {
+        n = Math.ceil(n / nodeSize);
+        numNodes += n;
+        levelNumNodes.push(n);
+    } while (n !== 1);
+    // bounds per level in reversed storage order (top-down)
+    const levelOffsets = [];
+    n = numNodes;
+    for (const size of levelNumNodes) {
+        levelOffsets.push(n - size);
+        n -= size;
+    }
+    const levelBounds = [];
+    for (let i = 0; i < levelNumNodes.length; i++)
+        levelBounds.push([levelOffsets[i], levelOffsets[i] + levelNumNodes[i]]);
+    return levelBounds;
 }
+/**
+ * Yield's a `SearchResult` for each feature within the bounds of `rect`.
+ *
+ * Every node in the FGB index tree has a bounding rect, all of the nodes children
+ * are contained within that bounding rect. The leaf nodes of the tree represent
+ * the features of the collection.
+ *
+ * As we traverse the tree, starting from the root, we'll need to read more data
+ * from the index. When we don't already have this range data buffered locally,
+ * an HTTP fetch is triggered. For performance, we merge adjacent and nearby
+ * request ranges into a single request, reasoning that fetching a few extra
+ * bytes is a good tradeoff if it means we can reduce the number of requests.
+ */
 export async function* streamSearch(numItems, nodeSize, rect, readNode) {
-  class NodeRange {
-    constructor(nodes, level) {
-      this._level = void 0;
-      this.nodes = void 0;
-      this._level = level;
-      this.nodes = nodes;
-    }
-    level() {
-      return this._level;
-    }
-    startNodeIdx() {
-      return this.nodes[0];
-    }
-    endNodeIdx() {
-      return this.nodes[1];
+    class NodeRange {
+        _level;
+        nodes;
+        constructor(nodes, level) {
+            this._level = level;
+            this.nodes = nodes;
+        }
+        level() {
+            return this._level;
+        }
+        startNodeIdx() {
+            return this.nodes[0];
+        }
+        endNodeIdx() {
+            return this.nodes[1];
+        }
+        extendEndNodeIdx(newIdx) {
+            console.assert(newIdx > this.nodes[1]);
+            this.nodes[1] = newIdx;
+        }
+        toString() {
+            return `[NodeRange level: ${this._level}, nodes: ${this.nodes[0]}-${this.nodes[1]}]`;
+        }
     }
-    extendEndNodeIdx(newIdx) {
-      console.assert(newIdx > this.nodes[1]);
-      this.nodes[1] = newIdx;
-    }
-    toString() {
-      return `[NodeRange level: ${this._level}, nodes: ${this.nodes[0]}-${this.nodes[1]}]`;
-    }
-  }
-  const {
-    minX,
-    minY,
-    maxX,
-    maxY
-  } = rect;
-  Logger.info(`tree items: ${numItems}, nodeSize: ${nodeSize}`);
-  const levelBounds = generateLevelBounds(numItems, nodeSize);
-  const firstLeafNodeIdx = levelBounds[0][0];
-  const rootNodeRange = (() => {
-    const range = [0, 1];
-    const level = levelBounds.length - 1;
-    return new NodeRange(range, level);
-  })();
-  const queue = [rootNodeRange];
-  Logger.debug(`starting stream search with queue: ${queue}, numItems: ${numItems}, nodeSize: ${nodeSize}, levelBounds: ${levelBounds}`);
-  while (queue.length != 0) {
-    const nodeRange = queue.shift();
-    Logger.debug(`popped node: ${nodeRange}, queueLength: ${queue.length}`);
-    const nodeRangeStartIdx = nodeRange.startNodeIdx();
-    const isLeafNode = nodeRangeStartIdx >= firstLeafNodeIdx;
-    const nodeRangeEndIdx = (() => {
-      const [, levelBound] = levelBounds[nodeRange.level()];
-      const nodeIdx = Math.min(nodeRange.endNodeIdx() + nodeSize, levelBound);
-      if (isLeafNode && nodeIdx < levelBound) {
-        return nodeIdx + 1;
-      }
-      return nodeIdx;
+    const { minX, minY, maxX, maxY } = rect;
+    Logger.info(`tree items: ${numItems}, nodeSize: ${nodeSize}`);
+    const levelBounds = generateLevelBounds(numItems, nodeSize);
+    const firstLeafNodeIdx = levelBounds[0][0];
+    const rootNodeRange = (() => {
+        const range = [0, 1];
+        const level = levelBounds.length - 1;
+        return new NodeRange(range, level);
     })();
-    const numNodesInRange = nodeRangeEndIdx - nodeRangeStartIdx;
-    const buffer = await readNode(nodeRangeStartIdx * NODE_ITEM_BYTE_LEN, numNodesInRange * NODE_ITEM_BYTE_LEN);
-    const dataView = new DataView(buffer);
-    for (let nodeIdx = nodeRangeStartIdx; nodeIdx < nodeRangeEndIdx; nodeIdx++) {
-      const nodeIdxInDataView = nodeIdx - nodeRangeStartIdx;
-      const dataViewByteStart = nodeIdxInDataView * NODE_ITEM_BYTE_LEN;
-      if (maxX < dataView.getFloat64(dataViewByteStart + 0, true)) continue;
-      if (maxY < dataView.getFloat64(dataViewByteStart + 8, true)) continue;
-      if (minX > dataView.getFloat64(dataViewByteStart + 16, true)) continue;
-      if (minY > dataView.getFloat64(dataViewByteStart + 24, true)) continue;
-      const offset = dataView.getBigUint64(dataViewByteStart + 32, true);
-      if (isLeafNode) {
-        const featureByteOffset = offset;
-        const featureLength = (() => {
-          if (nodeIdx < numItems - 1) {
-            const nextPos = (nodeIdxInDataView + 1) * NODE_ITEM_BYTE_LEN;
-            const nextOffset = dataView.getBigUint64(nextPos + 32, true);
-            return nextOffset - featureByteOffset;
-          }
-          return null;
+    const queue = [rootNodeRange];
+    Logger.debug(`starting stream search with queue: ${queue}, numItems: ${numItems}, nodeSize: ${nodeSize}, levelBounds: ${levelBounds}`);
+    while (queue.length != 0) {
+        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+        const nodeRange = queue.shift();
+        Logger.debug(`popped node: ${nodeRange}, queueLength: ${queue.length}`);
+        const nodeRangeStartIdx = nodeRange.startNodeIdx();
+        const isLeafNode = nodeRangeStartIdx >= firstLeafNodeIdx;
+        // find the end index of the node
+        const nodeRangeEndIdx = (() => {
+            const [, levelBound] = levelBounds[nodeRange.level()];
+            const nodeIdx = Math.min(nodeRange.endNodeIdx() + nodeSize, levelBound);
+            if (isLeafNode && nodeIdx < levelBound) {
+                // We can infer the length of *this* feature by getting the start of the *next*
+                // feature, so we get an extra node.
+                // This approach doesn't work for the final node in the index,
+                // but in that case we know that the feature runs to the end of the FGB file and
+                // could make an open ended range request to get "the rest of the data".
+                return nodeIdx + 1;
+            }
+            return nodeIdx;
         })();
-        const featureIdx = nodeIdx - firstLeafNodeIdx;
-        yield [Number(featureByteOffset), featureIdx, Number(featureLength)];
-        continue;
-      }
-      const firstChildNodeIdx = offset;
-      const extraRequestThresholdNodes = Config.global.extraRequestThreshold() / NODE_ITEM_BYTE_LEN;
-      const nearestNodeRange = queue[queue.length - 1];
-      if (nearestNodeRange !== undefined && nearestNodeRange.level() == nodeRange.level() - 1 && firstChildNodeIdx < nearestNodeRange.endNodeIdx() + extraRequestThresholdNodes) {
-        Logger.debug(`Merging "nodeRange" request into existing range: ${nearestNodeRange}, newEndNodeIdx: ${nearestNodeRange.endNodeIdx()} -> ${firstChildNodeIdx}`);
-        nearestNodeRange.extendEndNodeIdx(Number(firstChildNodeIdx));
-        continue;
-      }
-      const newNodeRange = (() => {
-        const level = nodeRange.level() - 1;
-        const range = [Number(firstChildNodeIdx), Number(firstChildNodeIdx) + 1];
-        return new NodeRange(range, level);
-      })();
-      if (nearestNodeRange !== undefined && nearestNodeRange.level() == newNodeRange.level()) {
-        Logger.info(`Same level, but too far away. Pushing new request for nodeIdx: ${firstChildNodeIdx} rather than merging with distant ${nearestNodeRange}`);
-      } else {
-        Logger.info(`Pushing new level for ${newNodeRange} onto queue with nearestNodeRange: ${nearestNodeRange} since there's not already a range for this level.`);
-      }
-      queue.push(newNodeRange);
+        const numNodesInRange = nodeRangeEndIdx - nodeRangeStartIdx;
+        const buffer = await readNode(nodeRangeStartIdx * NODE_ITEM_BYTE_LEN, numNodesInRange * NODE_ITEM_BYTE_LEN);
+        const dataView = new DataView(buffer);
+        for (let nodeIdx = nodeRangeStartIdx; nodeIdx < nodeRangeEndIdx; nodeIdx++) {
+            const nodeIdxInDataView = nodeIdx - nodeRangeStartIdx;
+            const dataViewByteStart = nodeIdxInDataView * NODE_ITEM_BYTE_LEN;
+            if (maxX < dataView.getFloat64(dataViewByteStart + 0, true))
+                continue; // maxX < nodeMinX
+            if (maxY < dataView.getFloat64(dataViewByteStart + 8, true))
+                continue; // maxY < nodeMinY
+            if (minX > dataView.getFloat64(dataViewByteStart + 16, true))
+                continue; // minX > nodeMaxX
+            if (minY > dataView.getFloat64(dataViewByteStart + 24, true))
+                continue; // minY > nodeMaxY
+            // `offset` is:
+            // For leaf nodes: the byte-offset into the feature buffer.
+            // For inner nodes: the node-idx of its first child.
+            const offset = dataView.getBigUint64(dataViewByteStart + 32, true);
+            if (isLeafNode) {
+                const featureByteOffset = offset;
+                const featureLength = (() => {
+                    if (nodeIdx < numItems - 1) {
+                        // Since features are tightly packed, we infer the
+                        // length of _this_ feature by measuring to the _next_
+                        // feature's start.
+                        const nextPos = (nodeIdxInDataView + 1) * NODE_ITEM_BYTE_LEN;
+                        // console.debug(`nodeIdx: ${nodeIdx} of ${numItems}, nodeRangeStartIdx: ${nodeRangeStartIdx}, nextPos: ${nextPos}, dataView.byteLength: ${dataView.byteLength}`,);
+                        const nextOffset = dataView.getBigUint64(nextPos + 32, true);
+                        return nextOffset - featureByteOffset;
+                    }
+                    // This is the last feature - there's no "next" feature
+                    // to measure to, so we can't know it's length.
+                    return null;
+                })();
+                // Logger.debug(`featureByteOffset: ${featureByteOffset}, nodeIdx: ${nodeIdx}, featureLength: ${featureLength}`);
+                const featureIdx = nodeIdx - firstLeafNodeIdx;
+                yield [Number(featureByteOffset), featureIdx, Number(featureLength)];
+                continue;
+            }
+            const firstChildNodeIdx = offset;
+            // request up to this many nodes if it means we can eliminate an
+            // extra request
+            const extraRequestThresholdNodes = Config.global.extraRequestThreshold() / NODE_ITEM_BYTE_LEN;
+            // Since we're traversing the tree by monotonically increasing byte
+            // offset, the most recently enqueued node range will be the
+            // nearest, and thus presents the best candidate for merging.
+            const nearestNodeRange = queue[queue.length - 1];
+            if (nearestNodeRange !== undefined &&
+                nearestNodeRange.level() == nodeRange.level() - 1 &&
+                firstChildNodeIdx < nearestNodeRange.endNodeIdx() + extraRequestThresholdNodes) {
+                Logger.debug(`Merging "nodeRange" request into existing range: ${nearestNodeRange}, newEndNodeIdx: ${nearestNodeRange.endNodeIdx()} -> ${firstChildNodeIdx}`);
+                nearestNodeRange.extendEndNodeIdx(Number(firstChildNodeIdx));
+                continue;
+            }
+            const newNodeRange = (() => {
+                const level = nodeRange.level() - 1;
+                const range = [Number(firstChildNodeIdx), Number(firstChildNodeIdx) + 1];
+                return new NodeRange(range, level);
+            })();
+            // We're going to add a new node range - log the reason
+            if (nearestNodeRange !== undefined && nearestNodeRange.level() == newNodeRange.level()) {
+                Logger.info(`Same level, but too far away. Pushing new request for nodeIdx: ${firstChildNodeIdx} rather than merging with distant ${nearestNodeRange}`);
+            }
+            else {
+                Logger.info(`Pushing new level for ${newNodeRange} onto queue with nearestNodeRange: ${nearestNodeRange} since there's not already a range for this level.`);
+            }
+            queue.push(newNodeRange);
+        }
     }
-  }
 }
-//# sourceMappingURL=packedrtree.js.map

package/dist/flatgeobuf/3.27.2/streams/utils.js

@@ -1,67 +1,68 @@
 import { ReadableStream } from 'web-streams-polyfill';
 import { ReadableStreamBuffer } from 'stream-buffers';
 export function arrayToStream(array) {
-  const myReadableStreamBuffer = new ReadableStreamBuffer({
-    frequency: 10,
-    chunkSize: 2048
-  });
-  myReadableStreamBuffer.put(Buffer.from(array));
-  myReadableStreamBuffer.stop();
-  const webReader = nodeToWeb(myReadableStreamBuffer);
-  return webReader;
+    const myReadableStreamBuffer = new ReadableStreamBuffer({
+        frequency: 10, // in milliseconds.
+        chunkSize: 2048 // in bytes.
+    });
+    myReadableStreamBuffer.put(Buffer.from(array));
+    myReadableStreamBuffer.stop();
+    const webReader = nodeToWeb(myReadableStreamBuffer);
+    return webReader;
 }
-export async function takeAsync(asyncIterable) {
-  let count = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : Infinity;
-  const result = [];
-  const iterator = asyncIterable[Symbol.asyncIterator]();
-  while (result.length < count) {
-    const {
-      value,
-      done
-    } = await iterator.next();
-    if (done) break;
-    result.push(value);
-  }
-  return result;
+export async function takeAsync(asyncIterable, count = Infinity) {
+    const result = [];
+    const iterator = asyncIterable[Symbol.asyncIterator]();
+    while (result.length < count) {
+        const { value, done } = await iterator.next();
+        if (done)
+            break;
+        result.push(value);
+    }
+    return result;
 }
 export function nodeToWeb(nodeStream) {
-  let destroyed = false;
-  const listeners = {};
-  function start(controller) {
-    listeners.data = onData;
-    listeners.end = onData;
-    listeners.end = onDestroy;
-    listeners.close = onDestroy;
-    listeners.error = onDestroy;
-    for (const name in listeners) nodeStream.on(name, listeners[name]);
-    nodeStream.pause();
-    function onData(chunk) {
-      if (destroyed) return;
-      controller.enqueue(chunk);
-      nodeStream.pause();
+    let destroyed = false;
+    const listeners = {};
+    function start(controller) {
+        listeners.data = onData;
+        listeners.end = onData;
+        listeners.end = onDestroy;
+        listeners.close = onDestroy;
+        listeners.error = onDestroy;
+        for (const name in listeners)
+            nodeStream.on(name, listeners[name]);
+        nodeStream.pause();
+        function onData(chunk) {
+            if (destroyed)
+                return;
+            controller.enqueue(chunk);
+            nodeStream.pause();
+        }
+        function onDestroy(err) {
+            if (destroyed)
+                return;
+            destroyed = true;
+            for (const name in listeners)
+                nodeStream.removeListener(name, listeners[name]);
+            if (err)
+                controller.error(err);
+            else
+                controller.close();
+        }
+    }
+    function pull() {
+        if (destroyed)
+            return;
+        nodeStream.resume();
     }
-    function onDestroy(err) {
-      if (destroyed) return;
-      destroyed = true;
-      for (const name in listeners) nodeStream.removeListener(name, listeners[name]);
-      if (err) controller.error(err);else controller.close();
+    function cancel() {
+        destroyed = true;
+        for (const name in listeners)
+            nodeStream.removeListener(name, listeners[name]);
+        nodeStream.push(null);
+        nodeStream.pause();
+        nodeStream.destroy();
     }
-  }
-  function pull() {
-    if (destroyed) return;
-    nodeStream.resume();
-  }
-  function cancel() {
-    destroyed = true;
-    for (const name in listeners) nodeStream.removeListener(name, listeners[name]);
-    nodeStream.push(null);
-    nodeStream.pause();
-    nodeStream.destroy();
-  }
-  return new ReadableStream({
-    start,
-    pull,
-    cancel
-  });
+    return new ReadableStream({ start, pull, cancel });
 }
-//# sourceMappingURL=utils.js.map

package/dist/flatgeobuf-loader.js

@@ -1,30 +1,36 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
 import { parseFlatGeobuf, parseFlatGeobufInBatches } from "./lib/parse-flatgeobuf.js";
-const VERSION = typeof "4.2.0-alpha.4" !== 'undefined' ? "4.2.0-alpha.4" : 'latest';
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof "4.2.0-alpha.5" !== 'undefined' ? "4.2.0-alpha.5" : 'latest';
+// FGB\3FGB\1
 const FGB_MAGIC_NUMBER = [0x66, 0x67, 0x62, 0x03, 0x66, 0x67, 0x62, 0x01];
 export const FlatGeobufWorkerLoader = {
-  id: 'flatgeobuf',
-  name: 'FlatGeobuf',
-  module: 'flatgeobuf',
-  version: VERSION,
-  worker: true,
-  extensions: ['fgb'],
-  mimeTypes: ['application/octet-stream'],
-  category: 'geometry',
-  tests: [new Uint8Array(FGB_MAGIC_NUMBER).buffer],
-  options: {
-    flatgeobuf: {
-      shape: 'geojson-table'
-    },
-    gis: {
-      reproject: false
+    id: 'flatgeobuf',
+    name: 'FlatGeobuf',
+    module: 'flatgeobuf',
+    version: VERSION,
+    worker: true,
+    extensions: ['fgb'],
+    mimeTypes: ['application/octet-stream'],
+    category: 'geometry',
+    tests: [new Uint8Array(FGB_MAGIC_NUMBER).buffer],
+    options: {
+        flatgeobuf: {
+            shape: 'geojson-table'
+        },
+        gis: {
+            reproject: false
+        }
     }
-  }
 };
 export const FlatGeobufLoader = {
-  ...FlatGeobufWorkerLoader,
-  parse: async (arrayBuffer, options) => parseFlatGeobuf(arrayBuffer, options),
-  parseSync: parseFlatGeobuf,
-  parseInBatchesFromStream: parseFlatGeobufInBatches,
-  binary: true
+    ...FlatGeobufWorkerLoader,
+    parse: async (arrayBuffer, options) => parseFlatGeobuf(arrayBuffer, options),
+    parseSync: parseFlatGeobuf,
+    // @ts-expect-error this is a stream parser not an async iterator parser
+    parseInBatchesFromStream: parseFlatGeobufInBatches,
+    binary: true
 };
-//# sourceMappingURL=flatgeobuf-loader.js.map