@loaders.gl/shapefile 4.0.0-alpha.5 → 4.0.0-alpha.6

package/dist/lib/parsers/parse-shp.js

@@ -1,154 +1,178 @@
-import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
-import BinaryChunkReader from '../streaming/binary-chunk-reader';
-import { parseSHPHeader } from './parse-shp-header';
-import { parseRecord } from './parse-shp-geometry';
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseSHPInBatches = exports.parseSHP = void 0;
+const binary_chunk_reader_1 = __importDefault(require("../streaming/binary-chunk-reader"));
+const parse_shp_header_1 = require("./parse-shp-header");
+const parse_shp_geometry_1 = require("./parse-shp-geometry");
 const LITTLE_ENDIAN = true;
 const BIG_ENDIAN = false;
 const SHP_HEADER_SIZE = 100;
+// According to the spec, the record header is just 8 bytes, but here we set it
+// to 12 so that we can also access the record's type
 const SHP_RECORD_HEADER_SIZE = 12;
 const STATE = {
-  EXPECTING_HEADER: 0,
-  EXPECTING_RECORD: 1,
-  END: 2,
-  ERROR: 3
+    EXPECTING_HEADER: 0,
+    EXPECTING_RECORD: 1,
+    END: 2,
+    ERROR: 3
 };
-
 class SHPParser {
-  constructor(options) {
-    _defineProperty(this, "options", {});
-
-    _defineProperty(this, "binaryReader", new BinaryChunkReader({
-      maxRewindBytes: SHP_RECORD_HEADER_SIZE
-    }));
-
-    _defineProperty(this, "state", STATE.EXPECTING_HEADER);
-
-    _defineProperty(this, "result", {
-      geometries: []
-    });
-
-    this.options = options;
-  }
-
-  write(arrayBuffer) {
-    this.binaryReader.write(arrayBuffer);
-    this.state = parseState(this.state, this.result, this.binaryReader, this.options);
-  }
-
-  end() {
-    this.binaryReader.end();
-    this.state = parseState(this.state, this.result, this.binaryReader, this.options);
-
-    if (this.state !== STATE.END) {
-      this.state = STATE.ERROR;
-      this.result.error = 'SHP incomplete file';
+    constructor(options) {
+        this.options = {};
+        this.binaryReader = new binary_chunk_reader_1.default({ maxRewindBytes: SHP_RECORD_HEADER_SIZE });
+        this.state = STATE.EXPECTING_HEADER;
+        this.result = {
+            geometries: [],
+            // Initialize with number values to make TS happy
+            // These are initialized for real in STATE.EXPECTING_HEADER
+            progress: {
+                bytesTotal: NaN,
+                bytesUsed: NaN,
+                rows: NaN
+            },
+            currentIndex: NaN
+        };
+        this.options = options;
+    }
+    write(arrayBuffer) {
+        this.binaryReader.write(arrayBuffer);
+        this.state = parseState(this.state, this.result, this.binaryReader, this.options);
+    }
+    end() {
+        this.binaryReader.end();
+        this.state = parseState(this.state, this.result, this.binaryReader, this.options);
+        // this.result.progress.bytesUsed = this.binaryReader.bytesUsed();
+        if (this.state !== STATE.END) {
+            this.state = STATE.ERROR;
+            this.result.error = 'SHP incomplete file';
+        }
     }
-  }
-
 }
-
-export function parseSHP(arrayBuffer, options) {
-  const shpParser = new SHPParser(options);
-  shpParser.write(arrayBuffer);
-  shpParser.end();
-  return shpParser.result;
+function parseSHP(arrayBuffer, options) {
+    const shpParser = new SHPParser(options);
+    shpParser.write(arrayBuffer);
+    shpParser.end();
+    // @ts-ignore
+    return shpParser.result;
 }
-export async function* parseSHPInBatches(asyncIterator, options) {
-  const parser = new SHPParser(options);
-  let headerReturned = false;
-
-  for await (const arrayBuffer of asyncIterator) {
-    parser.write(arrayBuffer);
-
-    if (!headerReturned && parser.result.header) {
-      headerReturned = true;
-      yield parser.result.header;
+exports.parseSHP = parseSHP;
+/**
+ * @param asyncIterator
+ * @param options
+ * @returns
+ */
+async function* parseSHPInBatches(asyncIterator, options) {
+    const parser = new SHPParser(options);
+    let headerReturned = false;
+    for await (const arrayBuffer of asyncIterator) {
+        parser.write(arrayBuffer);
+        if (!headerReturned && parser.result.header) {
+            headerReturned = true;
+            yield parser.result.header;
+        }
+        if (parser.result.geometries.length > 0) {
+            yield parser.result.geometries;
+            parser.result.geometries = [];
+        }
     }
-
+    parser.end();
     if (parser.result.geometries.length > 0) {
-      yield parser.result.geometries;
-      parser.result.geometries = [];
+        yield parser.result.geometries;
     }
-  }
-
-  parser.end();
-
-  if (parser.result.geometries.length > 0) {
-    yield parser.result.geometries;
-  }
-
-  return;
+    return;
 }
-
+exports.parseSHPInBatches = parseSHPInBatches;
+/**
+ * State-machine parser for SHP data
+ *
+ * Note that whenever more data is needed, a `return`, not a `break`, is
+ * necessary, as the `break` keeps the context within `parseState`, while
+ * `return` releases context so that more data can be written into the
+ * BinaryChunkReader.
+ *
+ * @param  state Current state
+ * @param  result An object to hold result data
+ * @param  binaryReader
+ * @return State at end of current parsing
+ */
+/* eslint-disable complexity, max-depth */
 function parseState(state, result, binaryReader, options) {
-  while (true) {
-    try {
-      switch (state) {
-        case STATE.ERROR:
-        case STATE.END:
-          return state;
-
-        case STATE.EXPECTING_HEADER:
-          const dataView = binaryReader.getDataView(SHP_HEADER_SIZE);
-
-          if (!dataView) {
-            return state;
-          }
-
-          result.header = parseSHPHeader(dataView);
-          result.progress = {
-            bytesUsed: 0,
-            bytesTotal: result.header.length,
-            rows: 0
-          };
-          result.currentIndex = 1;
-          state = STATE.EXPECTING_RECORD;
-          break;
-
-        case STATE.EXPECTING_RECORD:
-          while (binaryReader.hasAvailableBytes(SHP_RECORD_HEADER_SIZE)) {
-            const recordHeaderView = binaryReader.getDataView(SHP_RECORD_HEADER_SIZE);
-            const recordHeader = {
-              recordNumber: recordHeaderView.getInt32(0, BIG_ENDIAN),
-              byteLength: recordHeaderView.getInt32(4, BIG_ENDIAN) * 2,
-              type: recordHeaderView.getInt32(8, LITTLE_ENDIAN)
-            };
-
-            if (!binaryReader.hasAvailableBytes(recordHeader.byteLength - 4)) {
-              binaryReader.rewind(SHP_RECORD_HEADER_SIZE);
-              return state;
-            }
-
-            const invalidRecord = recordHeader.byteLength < 4 || recordHeader.type !== result.header.type || recordHeader.recordNumber !== result.currentIndex;
-
-            if (invalidRecord) {
-              binaryReader.rewind(SHP_RECORD_HEADER_SIZE - 4);
-            } else {
-              binaryReader.rewind(4);
-              const recordView = binaryReader.getDataView(recordHeader.byteLength);
-              const geometry = parseRecord(recordView, options);
-              result.geometries.push(geometry);
-              result.currentIndex++;
-              result.progress.rows = result.currentIndex - 1;
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        try {
+            switch (state) {
+                case STATE.ERROR:
+                case STATE.END:
+                    return state;
+                case STATE.EXPECTING_HEADER:
+                    // Parse initial file header
+                    const dataView = binaryReader.getDataView(SHP_HEADER_SIZE);
+                    if (!dataView) {
+                        return state;
+                    }
+                    result.header = (0, parse_shp_header_1.parseSHPHeader)(dataView);
+                    result.progress = {
+                        bytesUsed: 0,
+                        bytesTotal: result.header.length,
+                        rows: 0
+                    };
+                    // index numbering starts at 1
+                    result.currentIndex = 1;
+                    state = STATE.EXPECTING_RECORD;
+                    break;
+                case STATE.EXPECTING_RECORD:
+                    while (binaryReader.hasAvailableBytes(SHP_RECORD_HEADER_SIZE)) {
+                        const recordHeaderView = binaryReader.getDataView(SHP_RECORD_HEADER_SIZE);
+                        const recordHeader = {
+                            recordNumber: recordHeaderView.getInt32(0, BIG_ENDIAN),
+                            // 2 byte words; includes the four words of record header
+                            byteLength: recordHeaderView.getInt32(4, BIG_ENDIAN) * 2,
+                            // This is actually part of the record, not the header...
+                            type: recordHeaderView.getInt32(8, LITTLE_ENDIAN)
+                        };
+                        if (!binaryReader.hasAvailableBytes(recordHeader.byteLength - 4)) {
+                            binaryReader.rewind(SHP_RECORD_HEADER_SIZE);
+                            return state;
+                        }
+                        const invalidRecord = recordHeader.byteLength < 4 ||
+                            recordHeader.type !== result.header?.type ||
+                            recordHeader.recordNumber !== result.currentIndex;
+                        // All records must have at least four bytes (for the record shape type)
+                        if (invalidRecord) {
+                            // Malformed record, try again, advancing just 4 bytes
+                            // Note: this is a rewind because binaryReader.getDataView above
+                            // moved the pointer forward 12 bytes, so rewinding 8 bytes still
+                            // leaves us 4 bytes ahead
+                            binaryReader.rewind(SHP_RECORD_HEADER_SIZE - 4);
+                        }
+                        else {
+                            // Note: type is actually part of the record, not the header, so
+                            // rewind 4 bytes before reading record
+                            binaryReader.rewind(4);
+                            const recordView = binaryReader.getDataView(recordHeader.byteLength);
+                            const geometry = (0, parse_shp_geometry_1.parseRecord)(recordView, options);
+                            result.geometries.push(geometry);
+                            result.currentIndex++;
+                            result.progress.rows = result.currentIndex - 1;
+                        }
+                    }
+                    if (binaryReader.ended) {
+                        state = STATE.END;
+                    }
+                    return state;
+                default:
+                    state = STATE.ERROR;
+                    result.error = `illegal parser state ${state}`;
+                    return state;
             }
-          }
-
-          if (binaryReader.ended) {
-            state = STATE.END;
-          }
-
-          return state;
-
-        default:
-          state = STATE.ERROR;
-          result.error = "illegal parser state ".concat(state);
-          return state;
-      }
-    } catch (error) {
-      state = STATE.ERROR;
-      result.error = "SHP parsing failed: ".concat(error === null || error === void 0 ? void 0 : error.message);
-      return state;
+        }
+        catch (error) {
+            state = STATE.ERROR;
+            result.error = `SHP parsing failed: ${error?.message}`;
+            return state;
+        }
     }
-  }
 }
-//# sourceMappingURL=parse-shp.js.map

package/dist/lib/parsers/parse-shx.js

@@ -1,22 +1,28 @@
-import { parseSHPHeader } from './parse-shp-header';
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseShx = void 0;
+const parse_shp_header_1 = require("./parse-shp-header");
 const SHX_HEADER_SIZE = 100;
 const BIG_ENDIAN = false;
-export function parseShx(arrayBuffer) {
-  const headerView = new DataView(arrayBuffer, 0, SHX_HEADER_SIZE);
-  const header = parseSHPHeader(headerView);
-  const contentLength = header.length - SHX_HEADER_SIZE;
-  const contentView = new DataView(arrayBuffer, SHX_HEADER_SIZE, contentLength);
-  const offsets = new Int32Array(contentLength);
-  const lengths = new Int32Array(contentLength);
-
-  for (let i = 0; i < contentLength / 8; i++) {
-    offsets[i] = contentView.getInt32(i * 8, BIG_ENDIAN);
-    lengths[i] = contentView.getInt32(i * 8 + 4, BIG_ENDIAN);
-  }
-
-  return {
-    offsets,
-    lengths
-  };
+/**
+ * @param arrayBuffer
+ * @returns SHXOutput
+ */
+function parseShx(arrayBuffer) {
+    // SHX header is identical to SHP Header
+    const headerView = new DataView(arrayBuffer, 0, SHX_HEADER_SIZE);
+    const header = (0, parse_shp_header_1.parseSHPHeader)(headerView);
+    const contentLength = header.length - SHX_HEADER_SIZE;
+    const contentView = new DataView(arrayBuffer, SHX_HEADER_SIZE, contentLength);
+    const offsets = new Int32Array(contentLength);
+    const lengths = new Int32Array(contentLength);
+    for (let i = 0; i < contentLength / 8; i++) {
+        offsets[i] = contentView.getInt32(i * 8, BIG_ENDIAN);
+        lengths[i] = contentView.getInt32(i * 8 + 4, BIG_ENDIAN);
+    }
+    return {
+        offsets,
+        lengths
+    };
 }
-//# sourceMappingURL=parse-shx.js.map
+exports.parseShx = parseShx;

package/dist/lib/parsers/types.d.ts

@@ -0,0 +1,68 @@
+import { Schema, ObjectRowTable } from '@loaders.gl/schema';
+import type { LoaderOptions } from '@loaders.gl/loader-utils';
+export type SHPLoaderOptions = LoaderOptions & {
+    shp?: {
+        _maxDimensions?: number;
+    };
+};
+export type DBFLoaderOptions = LoaderOptions & {
+    dbf?: {
+        encoding?: string;
+        shape?: 'rows' | 'table' | 'object-row-table';
+    };
+    /** @deprecated */
+    tables?: {
+        /** @deprecated */
+        format?: 'rows' | 'table' | 'object-row-table';
+    };
+};
+export type ShapefileLoaderOptions = LoaderOptions & SHPLoaderOptions & {
+    shapefile?: {
+        shape?: 'geojson';
+    };
+    gis?: {
+        reproject?: boolean;
+        _targetCrs?: string;
+        /** @deprecated. Use options.shapefile.shape */
+        format?: 'geojson';
+    };
+};
+export type DBFRowsOutput = ObjectRowTable['data'];
+/**
+ * DBF Table output. Deprecated in favor of ObjectRowTable
+ * @deprecated
+ */
+export interface DBFTableOutput {
+    schema?: Schema;
+    rows: DBFRowsOutput;
+}
+export type DBFHeader = {
+    year: number;
+    month: number;
+    day: number;
+    nRecords: number;
+    headerLength: number;
+    recordLength: number;
+    languageDriver: number;
+};
+export type DBFField = {
+    name: string;
+    dataType: string;
+    fieldLength: number;
+    decimal: number;
+};
+export type DBFResult = {
+    data: {
+        [key: string]: any;
+    }[];
+    schema?: Schema;
+    error?: string;
+    dbfHeader?: DBFHeader;
+    dbfFields?: DBFField[];
+    progress?: {
+        bytesUsed: number;
+        rowsTotal: number;
+        rows: number;
+    };
+};
+//# sourceMappingURL=types.d.ts.map

package/dist/lib/parsers/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,MAAM,EAAE,cAAc,EAAC,MAAM,oBAAoB,CAAC;AAC1D,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAE5D,MAAM,MAAM,gBAAgB,GAAG,aAAa,GAAG;IAC7C,GAAG,CAAC,EAAE;QACJ,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG,aAAa,GAAG;IAC7C,GAAG,CAAC,EAAE;QACJ,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,kBAAkB,CAAC;KAC/C,CAAC;IACF,kBAAkB;IAClB,MAAM,CAAC,EAAE;QACP,kBAAkB;QAClB,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,kBAAkB,CAAC;KAChD,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG,aAAa,GAChD,gBAAgB,GAAG;IACjB,SAAS,CAAC,EAAE;QACV,KAAK,CAAC,EAAE,SAAS,CAAC;KACnB,CAAC;IACF,GAAG,CAAC,EAAE;QACJ,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,+CAA+C;QAC/C,MAAM,CAAC,EAAE,SAAS,CAAC;KACpB,CAAC;CACH,CAAC;AAEJ,MAAM,MAAM,aAAa,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;AAEnD;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,aAAa,CAAC;CACrB;AAED,MAAM,MAAM,SAAS,GAAG;IAEtB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IAEZ,QAAQ,EAAE,MAAM,CAAC;IAEjB,YAAY,EAAE,MAAM,CAAC;IAErB,YAAY,EAAE,MAAM,CAAC;IAErB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,MAAM,MAAM,QAAQ,GAAG;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,SAAS,GAAG;IACtB,IAAI,EAAE;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,EAAE,CAAC;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IACvB,QAAQ,CAAC,EAAE;QACT,SAAS,EAAE,MAAM,CAAC;QAClB,SAAS,EAAE,MAAM,CAAC;QAClB,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;CACH,CAAC"}

package/dist/lib/parsers/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/lib/streaming/binary-chunk-reader.d.ts

@@ -1,11 +1,12 @@
+type BinaryChunkReaderOptions = {
+    maxRewindBytes: number;
+};
 export default class BinaryChunkReader {
     offset: number;
     arrayBuffers: ArrayBuffer[];
     ended: boolean;
     maxRewindBytes: number;
-    constructor(options?: {
-        [key: string]: any;
-    });
+    constructor(options?: BinaryChunkReaderOptions);
     /**
      * @param arrayBuffer
      */
@@ -56,4 +57,5 @@ export default class BinaryChunkReader {
      */
     rewind(bytes: number): void;
 }
+export {};
 //# sourceMappingURL=binary-chunk-reader.d.ts.map

package/dist/lib/streaming/binary-chunk-reader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"binary-chunk-reader.d.ts","sourceRoot":"","sources":["../../../src/lib/streaming/binary-chunk-reader.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,OAAO,iBAAiB;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,WAAW,EAAE,CAAC;IAC5B,KAAK,EAAE,OAAO,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;gBAEX,OAAO,CAAC,EAAE;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC;IAY1C;;OAEG;IACH,KAAK,CAAC,WAAW,EAAE,WAAW,GAAG,IAAI;IAIrC,GAAG,IAAI,IAAI;IAKX;;;;;OAKG;IACH,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO;IAWzC;;;;;OAKG;IACH,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,GAAG,EAAE,GAAG,IAAI;IAwC9C;;;;;OAKG;IACH,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,QAAQ,GAAG,IAAI;IA+B3C;;OAEG;IACH,cAAc,IAAI,IAAI;IAUtB;;;;;;;;;OASG;IACH,oBAAoB,CAAC,aAAa,EAAE,GAAG,EAAE,GAAG,eAAe;IAoB3D;;OAEG;IACH,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAGzB;;OAEG;IACH,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;CAI5B"}
+{"version":3,"file":"binary-chunk-reader.d.ts","sourceRoot":"","sources":["../../../src/lib/streaming/binary-chunk-reader.ts"],"names":[],"mappings":"AAAA,KAAK,wBAAwB,GAAG;IAC9B,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,MAAM,CAAC,OAAO,OAAO,iBAAiB;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,WAAW,EAAE,CAAC;IAC5B,KAAK,EAAE,OAAO,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;gBAEX,OAAO,CAAC,EAAE,wBAAwB;IAY9C;;OAEG;IACH,KAAK,CAAC,WAAW,EAAE,WAAW,GAAG,IAAI;IAIrC,GAAG,IAAI,IAAI;IAKX;;;;;OAKG;IACH,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO;IAWzC;;;;;OAKG;IACH,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,GAAG,EAAE,GAAG,IAAI;IAwC9C;;;;;OAKG;IACH,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,QAAQ,GAAG,IAAI;IA+B3C;;OAEG;IACH,cAAc,IAAI,IAAI;IAUtB;;;;;;;;;OASG;IACH,oBAAoB,CAAC,aAAa,EAAE,GAAG,EAAE,GAAG,eAAe;IAoB3D;;OAEG;IACH,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAGzB;;OAEG;IACH,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;CAI5B"}