@loaders.gl/shapefile 4.2.0-alpha.4 → 4.2.0-alpha.6

package/dist/lib/parsers/parse-shp.js

@@ -4,130 +4,167 @@ import { parseRecord } from "./parse-shp-geometry.js";
 const LITTLE_ENDIAN = true;
 const BIG_ENDIAN = false;
 const SHP_HEADER_SIZE = 100;
+// According to the spec, the record header is just 8 bytes, but here we set it
+// to 12 so that we can also access the record's type
 const SHP_RECORD_HEADER_SIZE = 12;
 const STATE = {
-  EXPECTING_HEADER: 0,
-  EXPECTING_RECORD: 1,
-  END: 2,
-  ERROR: 3
+    EXPECTING_HEADER: 0,
+    EXPECTING_RECORD: 1,
+    END: 2,
+    ERROR: 3
 };
 class SHPParser {
-  constructor(options) {
-    this.options = {};
-    this.binaryReader = new BinaryChunkReader({
-      maxRewindBytes: SHP_RECORD_HEADER_SIZE
-    });
-    this.state = STATE.EXPECTING_HEADER;
-    this.result = {
-      geometries: [],
-      progress: {
-        bytesTotal: NaN,
-        bytesUsed: NaN,
-        rows: NaN
-      },
-      currentIndex: NaN
+    options = {};
+    binaryReader = new BinaryChunkReader({ maxRewindBytes: SHP_RECORD_HEADER_SIZE });
+    state = STATE.EXPECTING_HEADER;
+    result = {
+        geometries: [],
+        // Initialize with number values to make TS happy
+        // These are initialized for real in STATE.EXPECTING_HEADER
+        progress: {
+            bytesTotal: NaN,
+            bytesUsed: NaN,
+            rows: NaN
+        },
+        currentIndex: NaN
     };
-    this.options = options;
-  }
-  write(arrayBuffer) {
-    this.binaryReader.write(arrayBuffer);
-    this.state = parseState(this.state, this.result, this.binaryReader, this.options);
-  }
-  end() {
-    this.binaryReader.end();
-    this.state = parseState(this.state, this.result, this.binaryReader, this.options);
-    if (this.state !== STATE.END) {
-      this.state = STATE.ERROR;
-      this.result.error = 'SHP incomplete file';
+    constructor(options) {
+        this.options = options;
+    }
+    write(arrayBuffer) {
+        this.binaryReader.write(arrayBuffer);
+        this.state = parseState(this.state, this.result, this.binaryReader, this.options);
+    }
+    end() {
+        this.binaryReader.end();
+        this.state = parseState(this.state, this.result, this.binaryReader, this.options);
+        // this.result.progress.bytesUsed = this.binaryReader.bytesUsed();
+        if (this.state !== STATE.END) {
+            this.state = STATE.ERROR;
+            this.result.error = 'SHP incomplete file';
+        }
     }
-  }
 }
 export function parseSHP(arrayBuffer, options) {
-  const shpParser = new SHPParser(options);
-  shpParser.write(arrayBuffer);
-  shpParser.end();
-  return shpParser.result;
+    const shpParser = new SHPParser(options);
+    shpParser.write(arrayBuffer);
+    shpParser.end();
+    // @ts-ignore
+    return shpParser.result;
 }
+/**
+ * @param asyncIterator
+ * @param options
+ * @returns
+ */
 export async function* parseSHPInBatches(asyncIterator, options) {
-  const parser = new SHPParser(options);
-  let headerReturned = false;
-  for await (const arrayBuffer of asyncIterator) {
-    parser.write(arrayBuffer);
-    if (!headerReturned && parser.result.header) {
-      headerReturned = true;
-      yield parser.result.header;
+    const parser = new SHPParser(options);
+    let headerReturned = false;
+    for await (const arrayBuffer of asyncIterator) {
+        parser.write(arrayBuffer);
+        if (!headerReturned && parser.result.header) {
+            headerReturned = true;
+            yield parser.result.header;
+        }
+        if (parser.result.geometries.length > 0) {
+            yield parser.result.geometries;
+            parser.result.geometries = [];
+        }
     }
+    parser.end();
     if (parser.result.geometries.length > 0) {
-      yield parser.result.geometries;
-      parser.result.geometries = [];
+        yield parser.result.geometries;
     }
-  }
-  parser.end();
-  if (parser.result.geometries.length > 0) {
-    yield parser.result.geometries;
-  }
-  return;
+    return;
 }
+/**
+ * State-machine parser for SHP data
+ *
+ * Note that whenever more data is needed, a `return`, not a `break`, is
+ * necessary, as the `break` keeps the context within `parseState`, while
+ * `return` releases context so that more data can be written into the
+ * BinaryChunkReader.
+ *
+ * @param  state Current state
+ * @param  result An object to hold result data
+ * @param  binaryReader
+ * @return State at end of current parsing
+ */
+/* eslint-disable complexity, max-depth */
 function parseState(state, result, binaryReader, options) {
-  while (true) {
-    try {
-      switch (state) {
-        case STATE.ERROR:
-        case STATE.END:
-          return state;
-        case STATE.EXPECTING_HEADER:
-          const dataView = binaryReader.getDataView(SHP_HEADER_SIZE);
-          if (!dataView) {
-            return state;
-          }
-          result.header = parseSHPHeader(dataView);
-          result.progress = {
-            bytesUsed: 0,
-            bytesTotal: result.header.length,
-            rows: 0
-          };
-          result.currentIndex = 1;
-          state = STATE.EXPECTING_RECORD;
-          break;
-        case STATE.EXPECTING_RECORD:
-          while (binaryReader.hasAvailableBytes(SHP_RECORD_HEADER_SIZE)) {
-            var _result$header;
-            const recordHeaderView = binaryReader.getDataView(SHP_RECORD_HEADER_SIZE);
-            const recordHeader = {
-              recordNumber: recordHeaderView.getInt32(0, BIG_ENDIAN),
-              byteLength: recordHeaderView.getInt32(4, BIG_ENDIAN) * 2,
-              type: recordHeaderView.getInt32(8, LITTLE_ENDIAN)
-            };
-            if (!binaryReader.hasAvailableBytes(recordHeader.byteLength - 4)) {
-              binaryReader.rewind(SHP_RECORD_HEADER_SIZE);
-              return state;
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        try {
+            switch (state) {
+                case STATE.ERROR:
+                case STATE.END:
+                    return state;
+                case STATE.EXPECTING_HEADER:
+                    // Parse initial file header
+                    const dataView = binaryReader.getDataView(SHP_HEADER_SIZE);
+                    if (!dataView) {
+                        return state;
+                    }
+                    result.header = parseSHPHeader(dataView);
+                    result.progress = {
+                        bytesUsed: 0,
+                        bytesTotal: result.header.length,
+                        rows: 0
+                    };
+                    // index numbering starts at 1
+                    result.currentIndex = 1;
+                    state = STATE.EXPECTING_RECORD;
+                    break;
+                case STATE.EXPECTING_RECORD:
+                    while (binaryReader.hasAvailableBytes(SHP_RECORD_HEADER_SIZE)) {
+                        const recordHeaderView = binaryReader.getDataView(SHP_RECORD_HEADER_SIZE);
+                        const recordHeader = {
+                            recordNumber: recordHeaderView.getInt32(0, BIG_ENDIAN),
+                            // 2 byte words; includes the four words of record header
+                            byteLength: recordHeaderView.getInt32(4, BIG_ENDIAN) * 2,
+                            // This is actually part of the record, not the header...
+                            type: recordHeaderView.getInt32(8, LITTLE_ENDIAN)
+                        };
+                        if (!binaryReader.hasAvailableBytes(recordHeader.byteLength - 4)) {
+                            binaryReader.rewind(SHP_RECORD_HEADER_SIZE);
+                            return state;
+                        }
+                        const invalidRecord = recordHeader.byteLength < 4 ||
+                            recordHeader.type !== result.header?.type ||
+                            recordHeader.recordNumber !== result.currentIndex;
+                        // All records must have at least four bytes (for the record shape type)
+                        if (invalidRecord) {
+                            // Malformed record, try again, advancing just 4 bytes
+                            // Note: this is a rewind because binaryReader.getDataView above
+                            // moved the pointer forward 12 bytes, so rewinding 8 bytes still
+                            // leaves us 4 bytes ahead
+                            binaryReader.rewind(SHP_RECORD_HEADER_SIZE - 4);
+                        }
+                        else {
+                            // Note: type is actually part of the record, not the header, so
+                            // rewind 4 bytes before reading record
+                            binaryReader.rewind(4);
+                            const recordView = binaryReader.getDataView(recordHeader.byteLength);
+                            const geometry = parseRecord(recordView, options);
+                            result.geometries.push(geometry);
+                            result.currentIndex++;
+                            result.progress.rows = result.currentIndex - 1;
+                        }
+                    }
+                    if (binaryReader.ended) {
+                        state = STATE.END;
+                    }
+                    return state;
+                default:
+                    state = STATE.ERROR;
+                    result.error = `illegal parser state ${state}`;
+                    return state;
             }
-            const invalidRecord = recordHeader.byteLength < 4 || recordHeader.type !== ((_result$header = result.header) === null || _result$header === void 0 ? void 0 : _result$header.type) || recordHeader.recordNumber !== result.currentIndex;
-            if (invalidRecord) {
-              binaryReader.rewind(SHP_RECORD_HEADER_SIZE - 4);
-            } else {
-              binaryReader.rewind(4);
-              const recordView = binaryReader.getDataView(recordHeader.byteLength);
-              const geometry = parseRecord(recordView, options);
-              result.geometries.push(geometry);
-              result.currentIndex++;
-              result.progress.rows = result.currentIndex - 1;
-            }
-          }
-          if (binaryReader.ended) {
-            state = STATE.END;
-          }
-          return state;
-        default:
-          state = STATE.ERROR;
-          result.error = `illegal parser state ${state}`;
-          return state;
-      }
-    } catch (error) {
-      state = STATE.ERROR;
-      result.error = `SHP parsing failed: ${error === null || error === void 0 ? void 0 : error.message}`;
-      return state;
+        }
+        catch (error) {
+            state = STATE.ERROR;
+            result.error = `SHP parsing failed: ${error?.message}`;
+            return state;
+        }
     }
-  }
 }
-//# sourceMappingURL=parse-shp.js.map

package/dist/lib/parsers/parse-shx.js

@@ -1,20 +1,24 @@
 import { parseSHPHeader } from "./parse-shp-header.js";
 const SHX_HEADER_SIZE = 100;
 const BIG_ENDIAN = false;
+/**
+ * @param arrayBuffer
+ * @returns SHXOutput
+ */
 export function parseShx(arrayBuffer) {
-  const headerView = new DataView(arrayBuffer, 0, SHX_HEADER_SIZE);
-  const header = parseSHPHeader(headerView);
-  const contentLength = header.length - SHX_HEADER_SIZE;
-  const contentView = new DataView(arrayBuffer, SHX_HEADER_SIZE, contentLength);
-  const offsets = new Int32Array(contentLength);
-  const lengths = new Int32Array(contentLength);
-  for (let i = 0; i < contentLength / 8; i++) {
-    offsets[i] = contentView.getInt32(i * 8, BIG_ENDIAN);
-    lengths[i] = contentView.getInt32(i * 8 + 4, BIG_ENDIAN);
-  }
-  return {
-    offsets,
-    lengths
-  };
+    // SHX header is identical to SHP Header
+    const headerView = new DataView(arrayBuffer, 0, SHX_HEADER_SIZE);
+    const header = parseSHPHeader(headerView);
+    const contentLength = header.length - SHX_HEADER_SIZE;
+    const contentView = new DataView(arrayBuffer, SHX_HEADER_SIZE, contentLength);
+    const offsets = new Int32Array(contentLength);
+    const lengths = new Int32Array(contentLength);
+    for (let i = 0; i < contentLength / 8; i++) {
+        offsets[i] = contentView.getInt32(i * 8, BIG_ENDIAN);
+        lengths[i] = contentView.getInt32(i * 8 + 4, BIG_ENDIAN);
+    }
+    return {
+        offsets,
+        lengths
+    };
 }
-//# sourceMappingURL=parse-shx.js.map

package/dist/lib/parsers/types.js

@@ -1,2 +1 @@
 export {};
-//# sourceMappingURL=types.js.map

package/dist/lib/streaming/binary-chunk-reader.js

@@ -1,105 +1,164 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
 export class BinaryChunkReader {
-  constructor(options) {
-    this.offset = void 0;
-    this.arrayBuffers = void 0;
-    this.ended = void 0;
-    this.maxRewindBytes = void 0;
-    const {
-      maxRewindBytes = 0
-    } = options || {};
-    this.offset = 0;
-    this.arrayBuffers = [];
-    this.ended = false;
-    this.maxRewindBytes = maxRewindBytes;
-  }
-  write(arrayBuffer) {
-    this.arrayBuffers.push(arrayBuffer);
-  }
-  end() {
-    this.arrayBuffers = [];
-    this.ended = true;
-  }
-  hasAvailableBytes(bytes) {
-    let bytesAvailable = -this.offset;
-    for (const arrayBuffer of this.arrayBuffers) {
-      bytesAvailable += arrayBuffer.byteLength;
-      if (bytesAvailable >= bytes) {
-        return true;
-      }
+    offset;
+    arrayBuffers;
+    ended;
+    maxRewindBytes;
+    constructor(options) {
+        const { maxRewindBytes = 0 } = options || {};
+        /** current global offset into current array buffer*/
+        this.offset = 0;
+        /** current buffer from iterator */
+        this.arrayBuffers = [];
+        this.ended = false;
+        /** bytes behind offset to hold on to */
+        this.maxRewindBytes = maxRewindBytes;
     }
-    return false;
-  }
-  findBufferOffsets(bytes) {
-    let offset = -this.offset;
-    const selectedBuffers = [];
-    for (let i = 0; i < this.arrayBuffers.length; i++) {
-      const buf = this.arrayBuffers[i];
-      if (offset + buf.byteLength <= 0) {
-        offset += buf.byteLength;
-        continue;
-      }
-      const start = offset <= 0 ? Math.abs(offset) : 0;
-      let end;
-      if (start + bytes <= buf.byteLength) {
-        end = start + bytes;
-        selectedBuffers.push([i, [start, end]]);
-        return selectedBuffers;
-      }
-      end = buf.byteLength;
-      selectedBuffers.push([i, [start, end]]);
-      bytes -= buf.byteLength - start;
-      offset += buf.byteLength;
+    /**
+     * @param arrayBuffer
+     */
+    write(arrayBuffer) {
+        this.arrayBuffers.push(arrayBuffer);
     }
-    return null;
-  }
-  getDataView(bytes) {
-    const bufferOffsets = this.findBufferOffsets(bytes);
-    if (!bufferOffsets && this.ended) {
-      throw new Error('binary data exhausted');
+    end() {
+        this.arrayBuffers = [];
+        this.ended = true;
     }
-    if (!bufferOffsets) {
-      return null;
+    /**
+     * Has enough bytes available in array buffers
+     *
+     * @param bytes Number of bytes
+     * @return boolean
+     */
+    hasAvailableBytes(bytes) {
+        let bytesAvailable = -this.offset;
+        for (const arrayBuffer of this.arrayBuffers) {
+            bytesAvailable += arrayBuffer.byteLength;
+            if (bytesAvailable >= bytes) {
+                return true;
+            }
+        }
+        return false;
     }
-    if (bufferOffsets.length === 1) {
-      const [bufferIndex, [start, end]] = bufferOffsets[0];
-      const arrayBuffer = this.arrayBuffers[bufferIndex];
-      const view = new DataView(arrayBuffer, start, end - start);
-      this.offset += bytes;
-      this.disposeBuffers();
-      return view;
+    /**
+     * Find offsets of byte ranges within this.arrayBuffers
+     *
+     * @param  bytes Byte length to read
+     * @return Arrays with byte ranges pointing to this.arrayBuffers, Output type is nested array, e.g. [ [0, [1, 2]], ...]
+     */
+    findBufferOffsets(bytes) {
+        let offset = -this.offset;
+        const selectedBuffers = [];
+        for (let i = 0; i < this.arrayBuffers.length; i++) {
+            const buf = this.arrayBuffers[i];
+            // Current buffer isn't long enough to reach global offset
+            if (offset + buf.byteLength <= 0) {
+                offset += buf.byteLength;
+                // eslint-disable-next-line no-continue
+                continue;
+            }
+            // Find start/end offsets for this buffer
+            // When offset < 0, need to skip over Math.abs(offset) bytes
+            // When offset > 0, implies bytes in previous buffer, start at 0
+            const start = offset <= 0 ? Math.abs(offset) : 0;
+            let end;
+            // Length of requested bytes is contained in current buffer
+            if (start + bytes <= buf.byteLength) {
+                end = start + bytes;
+                selectedBuffers.push([i, [start, end]]);
+                return selectedBuffers;
+            }
+            // Will need to look into next buffer
+            end = buf.byteLength;
+            selectedBuffers.push([i, [start, end]]);
+            // Need to read fewer bytes in next iter
+            bytes -= buf.byteLength - start;
+            offset += buf.byteLength;
+        }
+        // Should only finish loop if exhausted all arrays
+        return null;
     }
-    const view = new DataView(this._combineArrayBuffers(bufferOffsets));
-    this.offset += bytes;
-    this.disposeBuffers();
-    return view;
-  }
-  disposeBuffers() {
-    while (this.arrayBuffers.length > 0 && this.offset - this.maxRewindBytes >= this.arrayBuffers[0].byteLength) {
-      this.offset -= this.arrayBuffers[0].byteLength;
-      this.arrayBuffers.shift();
+    /**
+     * Get the required number of bytes from the iterator
+     *
+     * @param bytes Number of bytes
+     * @return DataView with data
+     */
+    getDataView(bytes) {
+        const bufferOffsets = this.findBufferOffsets(bytes);
+        // return `null` if not enough data, except if end() already called, in
+        // which case throw an error.
+        if (!bufferOffsets && this.ended) {
+            throw new Error('binary data exhausted');
+        }
+        if (!bufferOffsets) {
+            return null;
+        }
+        // If only one arrayBuffer needed, return DataView directly
+        if (bufferOffsets.length === 1) {
+            const [bufferIndex, [start, end]] = bufferOffsets[0];
+            const arrayBuffer = this.arrayBuffers[bufferIndex];
+            const view = new DataView(arrayBuffer, start, end - start);
+            this.offset += bytes;
+            this.disposeBuffers();
+            return view;
+        }
+        // Concatenate portions of multiple ArrayBuffers
+        const view = new DataView(this._combineArrayBuffers(bufferOffsets));
+        this.offset += bytes;
+        this.disposeBuffers();
+        return view;
     }
-  }
-  _combineArrayBuffers(bufferOffsets) {
-    let byteLength = 0;
-    for (const bufferOffset of bufferOffsets) {
-      const [start, end] = bufferOffset[1];
-      byteLength += end - start;
+    /**
+     * Dispose of old array buffers
+     */
+    disposeBuffers() {
+        while (this.arrayBuffers.length > 0 &&
+            this.offset - this.maxRewindBytes >= this.arrayBuffers[0].byteLength) {
+            this.offset -= this.arrayBuffers[0].byteLength;
+            this.arrayBuffers.shift();
+        }
     }
-    const result = new Uint8Array(byteLength);
-    let resultOffset = 0;
-    for (const bufferOffset of bufferOffsets) {
-      const [bufferIndex, [start, end]] = bufferOffset;
-      const sourceArray = new Uint8Array(this.arrayBuffers[bufferIndex]);
-      result.set(sourceArray.subarray(start, end), resultOffset);
-      resultOffset += end - start;
+    /**
+     * Copy multiple ArrayBuffers into one contiguous ArrayBuffer
+     *
+     * In contrast to concatenateArrayBuffers, this only copies the necessary
+     * portions of the source arrays, rather than first copying the entire arrays
+     * then taking a part of them.
+     *
+     * @param bufferOffsets List of internal array offsets
+     * @return New contiguous ArrayBuffer
+     */
+    _combineArrayBuffers(bufferOffsets) {
+        let byteLength = 0;
+        for (const bufferOffset of bufferOffsets) {
+            const [start, end] = bufferOffset[1];
+            byteLength += end - start;
+        }
+        const result = new Uint8Array(byteLength);
+        // Copy the subarrays
+        let resultOffset = 0;
+        for (const bufferOffset of bufferOffsets) {
+            const [bufferIndex, [start, end]] = bufferOffset;
+            const sourceArray = new Uint8Array(this.arrayBuffers[bufferIndex]);
+            result.set(sourceArray.subarray(start, end), resultOffset);
+            resultOffset += end - start;
+        }
+        return result.buffer;
+    }
+    /**
+     * @param bytes
+     */
+    skip(bytes) {
+        this.offset += bytes;
+    }
+    /**
+     * @param bytes
+     */
+    rewind(bytes) {
+        // TODO - only works if offset is already set
+        this.offset -= bytes;
     }
-    return result.buffer;
-  }
-  skip(bytes) {
-    this.offset += bytes;
-  }
-  rewind(bytes) {
-    this.offset -= bytes;
-  }
 }
-//# sourceMappingURL=binary-chunk-reader.js.map

package/dist/lib/streaming/binary-reader.js

@@ -1,26 +1,54 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
 export class BinaryReader {
-  constructor(arrayBuffer) {
-    this.offset = void 0;
-    this.arrayBuffer = void 0;
-    this.offset = 0;
-    this.arrayBuffer = arrayBuffer;
-  }
-  hasAvailableBytes(bytes) {
-    return this.arrayBuffer.byteLength - this.offset >= bytes;
-  }
-  getDataView(bytes) {
-    if (bytes && !this.hasAvailableBytes(bytes)) {
-      throw new Error('binary data exhausted');
+    offset;
+    arrayBuffer;
+    constructor(arrayBuffer) {
+        /** current global (stream) offset */
+        this.offset = 0;
+        /** current buffer from iterator */
+        this.arrayBuffer = arrayBuffer;
+    }
+    /**
+     * Checks if there are available bytes in data
+     *
+     * @param bytes
+     * @returns boolean
+     */
+    hasAvailableBytes(bytes) {
+        return this.arrayBuffer.byteLength - this.offset >= bytes;
+    }
+    /**
+     * Get the required number of bytes from the iterator
+     *
+     * @param bytes
+     * @returns Dataview
+     */
+    getDataView(bytes) {
+        if (bytes && !this.hasAvailableBytes(bytes)) {
+            throw new Error('binary data exhausted');
+        }
+        const dataView = bytes
+            ? new DataView(this.arrayBuffer, this.offset, bytes)
+            : new DataView(this.arrayBuffer, this.offset);
+        this.offset += bytes;
+        return dataView;
+    }
+    /**
+     * Skipping
+     *
+     * @param bytes
+     */
+    skip(bytes) {
+        this.offset += bytes;
+    }
+    /**
+     * Rewinding
+     *
+     * @param bytes
+     */
+    rewind(bytes) {
+        this.offset -= bytes;
     }
-    const dataView = bytes ? new DataView(this.arrayBuffer, this.offset, bytes) : new DataView(this.arrayBuffer, this.offset);
-    this.offset += bytes;
-    return dataView;
-  }
-  skip(bytes) {
-    this.offset += bytes;
-  }
-  rewind(bytes) {
-    this.offset -= bytes;
-  }
 }
-//# sourceMappingURL=binary-reader.js.map