@loaders.gl/shapefile 4.2.0-alpha.4 → 4.2.0-alpha.5

package/dist/lib/parsers/parse-dbf.js

@@ -1,295 +1,336 @@
 import { BinaryChunkReader } from "../streaming/binary-chunk-reader.js";
 const LITTLE_ENDIAN = true;
 const DBF_HEADER_SIZE = 32;
-var STATE = function (STATE) {
-  STATE[STATE["START"] = 0] = "START";
-  STATE[STATE["FIELD_DESCRIPTORS"] = 1] = "FIELD_DESCRIPTORS";
-  STATE[STATE["FIELD_PROPERTIES"] = 2] = "FIELD_PROPERTIES";
-  STATE[STATE["END"] = 3] = "END";
-  STATE[STATE["ERROR"] = 4] = "ERROR";
-  return STATE;
-}(STATE || {});
+var STATE;
+(function (STATE) {
+    STATE[STATE["START"] = 0] = "START";
+    STATE[STATE["FIELD_DESCRIPTORS"] = 1] = "FIELD_DESCRIPTORS";
+    STATE[STATE["FIELD_PROPERTIES"] = 2] = "FIELD_PROPERTIES";
+    STATE[STATE["END"] = 3] = "END";
+    STATE[STATE["ERROR"] = 4] = "ERROR";
+})(STATE || (STATE = {}));
 class DBFParser {
-  constructor(options) {
-    this.binaryReader = new BinaryChunkReader();
-    this.textDecoder = void 0;
-    this.state = STATE.START;
-    this.result = {
-      data: []
-    };
-    this.textDecoder = new TextDecoder(options.encoding);
-  }
-  write(arrayBuffer) {
-    this.binaryReader.write(arrayBuffer);
-    this.state = parseState(this.state, this.result, this.binaryReader, this.textDecoder);
-  }
-  end() {
-    this.binaryReader.end();
-    this.state = parseState(this.state, this.result, this.binaryReader, this.textDecoder);
-    if (this.state !== STATE.END) {
-      this.state = STATE.ERROR;
-      this.result.error = 'DBF incomplete file';
+    constructor(options) {
+        this.binaryReader = new BinaryChunkReader();
+        this.state = STATE.START;
+        this.result = {
+            data: []
+        };
+        this.textDecoder = new TextDecoder(options.encoding);
+    }
+    /**
+     * @param arrayBuffer
+     */
+    write(arrayBuffer) {
+        this.binaryReader.write(arrayBuffer);
+        this.state = parseState(this.state, this.result, this.binaryReader, this.textDecoder);
+        // this.result.progress.bytesUsed = this.binaryReader.bytesUsed();
+        // important events:
+        // - schema available
+        // - first rows available
+        // - all rows available
+    }
+    end() {
+        this.binaryReader.end();
+        this.state = parseState(this.state, this.result, this.binaryReader, this.textDecoder);
+        // this.result.progress.bytesUsed = this.binaryReader.bytesUsed();
+        if (this.state !== STATE.END) {
+            this.state = STATE.ERROR;
+            this.result.error = 'DBF incomplete file';
+        }
     }
-  }
 }
-export function parseDBF(arrayBuffer) {
-  var _options$dbf;
-  let options = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
-  const {
-    encoding = 'latin1'
-  } = options.dbf || {};
-  const dbfParser = new DBFParser({
-    encoding
-  });
-  dbfParser.write(arrayBuffer);
-  dbfParser.end();
-  const {
-    data,
-    schema
-  } = dbfParser.result;
-  const shape = options === null || options === void 0 ? void 0 : (_options$dbf = options.dbf) === null || _options$dbf === void 0 ? void 0 : _options$dbf.shape;
-  switch (shape) {
-    case 'object-row-table':
-      {
-        const table = {
-          shape: 'object-row-table',
-          schema,
-          data
-        };
-        return table;
-      }
-    case 'table':
-      return {
-        schema,
-        rows: data
-      };
-    case 'rows':
-    default:
-      return data;
-  }
+/**
+ * @param arrayBuffer
+ * @param options
+ * @returns DBFTable or rows
+ */
+export function parseDBF(arrayBuffer, options = {}) {
+    const { encoding = 'latin1' } = options.dbf || {};
+    const dbfParser = new DBFParser({ encoding });
+    dbfParser.write(arrayBuffer);
+    dbfParser.end();
+    const { data, schema } = dbfParser.result;
+    const shape = options?.dbf?.shape;
+    switch (shape) {
+        case 'object-row-table': {
+            const table = {
+                shape: 'object-row-table',
+                schema,
+                data
+            };
+            return table;
+        }
+        case 'table':
+            return { schema, rows: data };
+        case 'rows':
+        default:
+            return data;
+    }
 }
-export function parseDBFInBatches(asyncIterator) {
-  try {
-    let options = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
-    return async function* () {
-      const {
-        encoding = 'latin1'
-      } = options.dbf || {};
-      const parser = new DBFParser({
-        encoding
-      });
-      let headerReturned = false;
-      for await (const arrayBuffer of asyncIterator) {
+/**
+ * @param asyncIterator
+ * @param options
+ */
+export async function* parseDBFInBatches(asyncIterator, options = {}) {
+    const { encoding = 'latin1' } = options.dbf || {};
+    const parser = new DBFParser({ encoding });
+    let headerReturned = false;
+    for await (const arrayBuffer of asyncIterator) {
         parser.write(arrayBuffer);
         if (!headerReturned && parser.result.dbfHeader) {
-          headerReturned = true;
-          yield parser.result.dbfHeader;
+            headerReturned = true;
+            yield parser.result.dbfHeader;
         }
         if (parser.result.data.length > 0) {
-          yield parser.result.data;
-          parser.result.data = [];
+            yield parser.result.data;
+            parser.result.data = [];
         }
-      }
-      parser.end();
-      if (parser.result.data.length > 0) {
+    }
+    parser.end();
+    if (parser.result.data.length > 0) {
         yield parser.result.data;
-      }
-    }();
-  } catch (e) {
-    return Promise.reject(e);
-  }
+    }
 }
+/**
+ * https://www.dbase.com/Knowledgebase/INT/db7_file_fmt.htm
+ * @param state
+ * @param result
+ * @param binaryReader
+ * @param textDecoder
+ * @returns
+ */
+/* eslint-disable complexity, max-depth */
 function parseState(state, result, binaryReader, textDecoder) {
-  while (true) {
-    try {
-      switch (state) {
-        case STATE.ERROR:
-        case STATE.END:
-          return state;
-        case STATE.START:
-          const dataView = binaryReader.getDataView(DBF_HEADER_SIZE);
-          if (!dataView) {
-            return state;
-          }
-          result.dbfHeader = parseDBFHeader(dataView);
-          result.progress = {
-            bytesUsed: 0,
-            rowsTotal: result.dbfHeader.nRecords,
-            rows: 0
-          };
-          state = STATE.FIELD_DESCRIPTORS;
-          break;
-        case STATE.FIELD_DESCRIPTORS:
-          const fieldDescriptorView = binaryReader.getDataView(result.dbfHeader.headerLength - DBF_HEADER_SIZE);
-          if (!fieldDescriptorView) {
-            return state;
-          }
-          result.dbfFields = parseFieldDescriptors(fieldDescriptorView, textDecoder);
-          result.schema = {
-            fields: result.dbfFields.map(dbfField => makeField(dbfField)),
-            metadata: {}
-          };
-          state = STATE.FIELD_PROPERTIES;
-          binaryReader.skip(1);
-          break;
-        case STATE.FIELD_PROPERTIES:
-          const {
-            recordLength = 0,
-            nRecords = 0
-          } = (result === null || result === void 0 ? void 0 : result.dbfHeader) || {};
-          while (result.data.length < nRecords) {
-            const recordView = binaryReader.getDataView(recordLength - 1);
-            if (!recordView) {
-              return state;
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        try {
+            switch (state) {
+                case STATE.ERROR:
+                case STATE.END:
+                    return state;
+                case STATE.START:
+                    // Parse initial file header
+                    // DBF Header
+                    const dataView = binaryReader.getDataView(DBF_HEADER_SIZE);
+                    if (!dataView) {
+                        return state;
+                    }
+                    result.dbfHeader = parseDBFHeader(dataView);
+                    result.progress = {
+                        bytesUsed: 0,
+                        rowsTotal: result.dbfHeader.nRecords,
+                        rows: 0
+                    };
+                    state = STATE.FIELD_DESCRIPTORS;
+                    break;
+                case STATE.FIELD_DESCRIPTORS:
+                    // Parse DBF field descriptors (schema)
+                    const fieldDescriptorView = binaryReader.getDataView(
+                    // @ts-ignore
+                    result.dbfHeader.headerLength - DBF_HEADER_SIZE);
+                    if (!fieldDescriptorView) {
+                        return state;
+                    }
+                    result.dbfFields = parseFieldDescriptors(fieldDescriptorView, textDecoder);
+                    result.schema = {
+                        fields: result.dbfFields.map((dbfField) => makeField(dbfField)),
+                        metadata: {}
+                    };
+                    state = STATE.FIELD_PROPERTIES;
+                    // TODO(kyle) Not exactly sure why start offset needs to be headerLength + 1?
+                    // parsedbf uses ((fields.length + 1) << 5) + 2;
+                    binaryReader.skip(1);
+                    break;
+                case STATE.FIELD_PROPERTIES:
+                    const { recordLength = 0, nRecords = 0 } = result?.dbfHeader || {};
+                    while (result.data.length < nRecords) {
+                        const recordView = binaryReader.getDataView(recordLength - 1);
+                        if (!recordView) {
+                            return state;
+                        }
+                        // Note: Avoid actually reading the last byte, which may not be present
+                        binaryReader.skip(1);
+                        // @ts-ignore
+                        const row = parseRow(recordView, result.dbfFields, textDecoder);
+                        result.data.push(row);
+                        // @ts-ignore
+                        result.progress.rows = result.data.length;
+                    }
+                    state = STATE.END;
+                    break;
+                default:
+                    state = STATE.ERROR;
+                    result.error = `illegal parser state ${state}`;
+                    return state;
             }
-            binaryReader.skip(1);
-            const row = parseRow(recordView, result.dbfFields, textDecoder);
-            result.data.push(row);
-            result.progress.rows = result.data.length;
-          }
-          state = STATE.END;
-          break;
-        default:
-          state = STATE.ERROR;
-          result.error = `illegal parser state ${state}`;
-          return state;
-      }
-    } catch (error) {
-      state = STATE.ERROR;
-      result.error = `DBF parsing failed: ${error.message}`;
-      return state;
+        }
+        catch (error) {
+            state = STATE.ERROR;
+            result.error = `DBF parsing failed: ${error.message}`;
+            return state;
+        }
     }
-  }
 }
+/**
+ * @param headerView
+ */
 function parseDBFHeader(headerView) {
-  return {
-    year: headerView.getUint8(1) + 1900,
-    month: headerView.getUint8(2),
-    day: headerView.getUint8(3),
-    nRecords: headerView.getUint32(4, LITTLE_ENDIAN),
-    headerLength: headerView.getUint16(8, LITTLE_ENDIAN),
-    recordLength: headerView.getUint16(10, LITTLE_ENDIAN),
-    languageDriver: headerView.getUint8(29)
-  };
+    return {
+        // Last updated date
+        year: headerView.getUint8(1) + 1900,
+        month: headerView.getUint8(2),
+        day: headerView.getUint8(3),
+        // Number of records in data file
+        nRecords: headerView.getUint32(4, LITTLE_ENDIAN),
+        // Length of header in bytes
+        headerLength: headerView.getUint16(8, LITTLE_ENDIAN),
+        // Length of each record
+        recordLength: headerView.getUint16(10, LITTLE_ENDIAN),
+        // Not sure if this is usually set
+        languageDriver: headerView.getUint8(29)
+    };
 }
+/**
+ * @param view
+ */
 function parseFieldDescriptors(view, textDecoder) {
-  const nFields = (view.byteLength - 1) / 32;
-  const fields = [];
-  let offset = 0;
-  for (let i = 0; i < nFields; i++) {
-    const name = textDecoder.decode(new Uint8Array(view.buffer, view.byteOffset + offset, 11)).replace(/\u0000/g, '');
-    fields.push({
-      name,
-      dataType: String.fromCharCode(view.getUint8(offset + 11)),
-      fieldLength: view.getUint8(offset + 16),
-      decimal: view.getUint8(offset + 17)
-    });
-    offset += 32;
+    // NOTE: this might overestimate the number of fields if the "Database
+    // Container" container exists and is included in the headerLength
+    const nFields = (view.byteLength - 1) / 32;
+    const fields = [];
+    let offset = 0;
+    for (let i = 0; i < nFields; i++) {
+        const name = textDecoder
+            .decode(new Uint8Array(view.buffer, view.byteOffset + offset, 11))
+            // eslint-disable-next-line no-control-regex
+            .replace(/\u0000/g, '');
+        fields.push({
+            name,
+            dataType: String.fromCharCode(view.getUint8(offset + 11)),
+            fieldLength: view.getUint8(offset + 16),
+            decimal: view.getUint8(offset + 17)
+        });
+        offset += 32;
+    }
+    return fields;
+}
+/*
+ * @param {BinaryChunkReader} binaryReader
+function parseRows(binaryReader, fields, nRecords, recordLength, textDecoder) {
+  const rows = [];
+  for (let i = 0; i < nRecords; i++) {
+    const recordView = binaryReader.getDataView(recordLength - 1);
+    binaryReader.skip(1);
+    // @ts-ignore
+    rows.push(parseRow(recordView, fields, textDecoder));
   }
-  return fields;
+  return rows;
 }
+ */
+/**
+ *
+ * @param view
+ * @param fields
+ * @param textDecoder
+ * @returns
+ */
 function parseRow(view, fields, textDecoder) {
-  const out = {};
-  let offset = 0;
-  for (const field of fields) {
-    const text = textDecoder.decode(new Uint8Array(view.buffer, view.byteOffset + offset, field.fieldLength));
-    out[field.name] = parseField(text, field.dataType);
-    offset += field.fieldLength;
-  }
-  return out;
+    const out = {};
+    let offset = 0;
+    for (const field of fields) {
+        const text = textDecoder.decode(new Uint8Array(view.buffer, view.byteOffset + offset, field.fieldLength));
+        out[field.name] = parseField(text, field.dataType);
+        offset += field.fieldLength;
+    }
+    return out;
 }
+/**
+ * Should NaN be coerced to null?
+ * @param text
+ * @param dataType
+ * @returns Field depends on a type of the data
+ */
 function parseField(text, dataType) {
-  switch (dataType) {
-    case 'B':
-      return parseNumber(text);
-    case 'C':
-      return parseCharacter(text);
-    case 'F':
-      return parseNumber(text);
-    case 'N':
-      return parseNumber(text);
-    case 'O':
-      return parseNumber(text);
-    case 'D':
-      return parseDate(text);
-    case 'L':
-      return parseBoolean(text);
-    default:
-      throw new Error('Unsupported data type');
-  }
+    switch (dataType) {
+        case 'B':
+            return parseNumber(text);
+        case 'C':
+            return parseCharacter(text);
+        case 'F':
+            return parseNumber(text);
+        case 'N':
+            return parseNumber(text);
+        case 'O':
+            return parseNumber(text);
+        case 'D':
+            return parseDate(text);
+        case 'L':
+            return parseBoolean(text);
+        default:
+            throw new Error('Unsupported data type');
+    }
 }
+/**
+ * Parse YYYYMMDD to date in milliseconds
+ * @param str YYYYMMDD
+ * @returns new Date as a number
+ */
 function parseDate(str) {
-  return Date.UTC(str.slice(0, 4), parseInt(str.slice(4, 6), 10) - 1, str.slice(6, 8));
+    return Date.UTC(str.slice(0, 4), parseInt(str.slice(4, 6), 10) - 1, str.slice(6, 8));
 }
+/**
+ * Read boolean value
+ * any of Y, y, T, t coerce to true
+ * any of N, n, F, f coerce to false
+ * otherwise null
+ * @param value
+ * @returns boolean | null
+ */
 function parseBoolean(value) {
-  return /^[nf]$/i.test(value) ? false : /^[yt]$/i.test(value) ? true : null;
+    return /^[nf]$/i.test(value) ? false : /^[yt]$/i.test(value) ? true : null;
 }
+/**
+ * Return null instead of NaN
+ * @param text
+ * @returns number | null
+ */
 function parseNumber(text) {
-  const number = parseFloat(text);
-  return isNaN(number) ? null : number;
+    const number = parseFloat(text);
+    return isNaN(number) ? null : number;
 }
+/**
+ *
+ * @param text
+ * @returns string | null
+ */
 function parseCharacter(text) {
-  return text.trim() || null;
+    return text.trim() || null;
 }
-function makeField(_ref) {
-  let {
-    name,
-    dataType,
-    fieldLength,
-    decimal
-  } = _ref;
-  switch (dataType) {
-    case 'B':
-      return {
-        name,
-        type: 'float64',
-        nullable: true,
-        metadata: {}
-      };
-    case 'C':
-      return {
-        name,
-        type: 'utf8',
-        nullable: true,
-        metadata: {}
-      };
-    case 'F':
-      return {
-        name,
-        type: 'float64',
-        nullable: true,
-        metadata: {}
-      };
-    case 'N':
-      return {
-        name,
-        type: 'float64',
-        nullable: true,
-        metadata: {}
-      };
-    case 'O':
-      return {
-        name,
-        type: 'float64',
-        nullable: true,
-        metadata: {}
-      };
-    case 'D':
-      return {
-        name,
-        type: 'timestamp-millisecond',
-        nullable: true,
-        metadata: {}
-      };
-    case 'L':
-      return {
-        name,
-        type: 'bool',
-        nullable: true,
-        metadata: {}
-      };
-    default:
-      throw new Error('Unsupported data type');
-  }
+/**
+ * Create a standard Arrow-style `Field` from field descriptor.
+ * TODO - use `fieldLength` and `decimal` to generate smaller types?
+ * @param param0
+ * @returns Field
+ */
+// eslint-disable
+function makeField({ name, dataType, fieldLength, decimal }) {
+    switch (dataType) {
+        case 'B':
+            return { name, type: 'float64', nullable: true, metadata: {} };
+        case 'C':
+            return { name, type: 'utf8', nullable: true, metadata: {} };
+        case 'F':
+            return { name, type: 'float64', nullable: true, metadata: {} };
+        case 'N':
+            return { name, type: 'float64', nullable: true, metadata: {} };
+        case 'O':
+            return { name, type: 'float64', nullable: true, metadata: {} };
+        case 'D':
+            return { name, type: 'timestamp-millisecond', nullable: true, metadata: {} };
+        case 'L':
+            return { name, type: 'bool', nullable: true, metadata: {} };
+        default:
+            throw new Error('Unsupported data type');
+    }
 }
-//# sourceMappingURL=parse-dbf.js.map

package/dist/lib/parsers/parse-shapefile.d.ts

@@ -1,7 +1,7 @@
 import { LoaderContext } from '@loaders.gl/loader-utils';
-import type { SHXOutput } from './parse-shx';
-import type { SHPHeader } from './parse-shp-header';
-import type { ShapefileLoaderOptions } from './types';
+import type { SHXOutput } from "./parse-shx.js";
+import type { SHPHeader } from "./parse-shp-header.js";
+import type { ShapefileLoaderOptions } from "./types.js";
 interface ShapefileOutput {
     encoding?: string;
     prj?: string;

package/dist/lib/parsers/parse-shapefile.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parse-shapefile.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-shapefile.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,aAAa,EAA8C,MAAM,0BAA0B,CAAC;AAUpG,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,aAAa,CAAC;AAC3C,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,oBAAoB,CAAC;AAClD,OAAO,KAAK,EAAC,sBAAsB,EAAC,MAAM,SAAS,CAAC;AAOpD,UAAU,eAAe;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,SAAS,CAAC;IAClB,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB;AACD;;GAEG;AAEH,wBAAuB,uBAAuB,CAC5C,aAAa,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,EACjE,OAAO,CAAC,EAAE,sBAAsB,EAChC,OAAO,CAAC,EAAE,aAAa,GACtB,aAAa,CAAC,eAAe,CAAC,CAkFhC;AAED;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAClC,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,sBAAsB,EAChC,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,eAAe,CAAC,CAiD1B;AAwDD;;;;;GAKG;AAEH,wBAAsB,yBAAyB,CAC7C,OAAO,CAAC,EAAE,sBAAsB,EAChC,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC;IACT,GAAG,CAAC,EAAE,SAAS,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;CACd,CAAC,CAkCD;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,MAAM,CAQ1E"}
+{"version":3,"file":"parse-shapefile.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-shapefile.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,aAAa,EAA8C,MAAM,0BAA0B,CAAC;AAUpG,OAAO,KAAK,EAAC,SAAS,EAAC,uBAAoB;AAC3C,OAAO,KAAK,EAAC,SAAS,EAAC,8BAA2B;AAClD,OAAO,KAAK,EAAC,sBAAsB,EAAC,mBAAgB;AAOpD,UAAU,eAAe;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,SAAS,CAAC;IAChB,MAAM,EAAE,SAAS,CAAC;IAClB,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB;AACD;;GAEG;AAEH,wBAAuB,uBAAuB,CAC5C,aAAa,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,EACjE,OAAO,CAAC,EAAE,sBAAsB,EAChC,OAAO,CAAC,EAAE,aAAa,GACtB,aAAa,CAAC,eAAe,CAAC,CAkFhC;AAED;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAClC,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,sBAAsB,EAChC,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,eAAe,CAAC,CAiD1B;AAwDD;;;;;GAKG;AAEH,wBAAsB,yBAAyB,CAC7C,OAAO,CAAC,EAAE,sBAAsB,EAChC,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC;IACT,GAAG,CAAC,EAAE,SAAS,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;CACd,CAAC,CAkCD;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,MAAM,CAQ1E"}