@loaders.gl/shapefile 3.2.0-alpha.1 → 3.2.0-alpha.4

package/dist/shp-worker.js

@@ -132,7 +132,7 @@
   // src/lib/parsers/parse-shp-geometry.ts
   var LITTLE_ENDIAN2 = true;
   function parseRecord(view, options) {
-    const { _maxDimensions } = options?.shp || {};
+    const { _maxDimensions = 4 } = options?.shp || {};
     let offset = 0;
     const type = view.getInt32(offset, LITTLE_ENDIAN2);
     offset += Int32Array.BYTES_PER_ELEMENT;
@@ -318,7 +318,13 @@
       this.binaryReader = new BinaryChunkReader({ maxRewindBytes: SHP_RECORD_HEADER_SIZE });
       this.state = STATE.EXPECTING_HEADER;
       this.result = {
-        geometries: []
+        geometries: [],
+        progress: {
+          bytesTotal: NaN,
+          bytesUsed: NaN,
+          rows: NaN
+        },
+        currentIndex: NaN
       };
       this.options = options;
     }
@@ -394,7 +400,7 @@
                 binaryReader.rewind(SHP_RECORD_HEADER_SIZE);
                 return state;
               }
-              const invalidRecord = recordHeader.byteLength < 4 || recordHeader.type !== result.header.type || recordHeader.recordNumber !== result.currentIndex;
+              const invalidRecord = recordHeader.byteLength < 4 || recordHeader.type !== result.header?.type || recordHeader.recordNumber !== result.currentIndex;
               if (invalidRecord) {
                 binaryReader.rewind(SHP_RECORD_HEADER_SIZE - 4);
               } else {
@@ -424,7 +430,7 @@
   }
 
   // src/shp-loader.ts
-  var VERSION = true ? "3.2.0-alpha.1" : "latest";
+  var VERSION = true ? "3.2.0-alpha.4" : "latest";
   var SHP_MAGIC_NUMBER2 = [0, 0, 39, 10];
   var SHPWorkerLoader = {
     name: "SHP",
@@ -568,12 +574,13 @@
       switch (type) {
         case "process":
           try {
-            const { input, options = {} } = payload;
+            const { input, options = {}, context = {} } = payload;
             const result = await parseData({
               loader,
               arrayBuffer: input,
               options,
               context: {
+                ...context,
                 parse: parseOnMainThread
               }
             });

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@loaders.gl/shapefile",
   "description": "Loader for the Shapefile Format",
-  "version": "3.2.0-alpha.1",
+  "version": "3.2.0-alpha.4",
   "license": "MIT",
   "publishConfig": {
     "access": "public"
@@ -37,10 +37,10 @@
     "build-worker-dbf": "esbuild src/workers/dbf-worker.ts --bundle --outfile=dist/dbf-worker.js --define:__VERSION__=\\\"$npm_package_version\\\""
   },
   "dependencies": {
-    "@loaders.gl/gis": "3.2.0-alpha.1",
-    "@loaders.gl/loader-utils": "3.2.0-alpha.1",
-    "@loaders.gl/schema": "3.2.0-alpha.1",
+    "@loaders.gl/gis": "3.2.0-alpha.4",
+    "@loaders.gl/loader-utils": "3.2.0-alpha.4",
+    "@loaders.gl/schema": "3.2.0-alpha.4",
     "@math.gl/proj4": "^3.5.1"
   },
-  "gitHead": "423a2815092b08dcf93ad5b7dc2940b167305afe"
+  "gitHead": "f030d39d5cdedb137e43e757a9da10dd637857fd"
 }

package/src/lib/parsers/parse-dbf.ts

@@ -1,47 +1,21 @@
-import {Schema, Field, Bool, Utf8, Float64, TimestampMillisecond} from '@loaders.gl/schema';
+import {
+  Schema,
+  Field,
+  Bool,
+  Utf8,
+  Float64,
+  TimestampMillisecond,
+  ObjectRowTable
+} from '@loaders.gl/schema';
 import BinaryChunkReader from '../streaming/binary-chunk-reader';
-
-type DBFRowsOutput = object[];
-
-interface DBFTableOutput {
-  schema?: Schema;
-  rows: DBFRowsOutput;
-}
-
-type DBFHeader = {
-  // Last updated date
-  year: number;
-  month: number;
-  day: number;
-  // Number of records in data file
-  nRecords: number;
-  // Length of header in bytes
-  headerLength: number;
-  // Length of each record
-  recordLength: number;
-  // Not sure if this is usually set
-  languageDriver: number;
-};
-
-type DBFField = {
-  name: string;
-  dataType: string;
-  fieldLength: number;
-  decimal: number;
-};
-
-type DBFResult = {
-  data: {[key: string]: any}[];
-  schema?: Schema;
-  error?: string;
-  dbfHeader?: DBFHeader;
-  dbfFields?: DBFField[];
-  progress?: {
-    bytesUsed: number;
-    rowsTotal: number;
-    rows: number;
-  };
-};
+import {
+  DBFLoaderOptions,
+  DBFResult,
+  DBFTableOutput,
+  DBFHeader,
+  DBFRowsOutput,
+  DBFField
+} from './types';
 
 const LITTLE_ENDIAN = true;
 const DBF_HEADER_SIZE = 32;
@@ -98,21 +72,27 @@ class DBFParser {
  */
 export function parseDBF(
   arrayBuffer: ArrayBuffer,
-  options: any = {}
-): DBFRowsOutput | DBFTableOutput {
-  const loaderOptions = options.dbf || {};
-  const {encoding} = loaderOptions;
+  options: DBFLoaderOptions = {}
+): DBFRowsOutput | DBFTableOutput | ObjectRowTable {
+  const {encoding = 'latin1'} = options.dbf || {};
 
   const dbfParser = new DBFParser({encoding});
   dbfParser.write(arrayBuffer);
   dbfParser.end();
 
   const {data, schema} = dbfParser.result;
-  switch (options.tables && options.tables.format) {
+  const shape = options?.tables?.format || options?.dbf?.shape;
+  switch (shape) {
+    case 'object-row-table': {
+      const table: ObjectRowTable = {
+        shape: 'object-row-table',
+        schema,
+        data
+      };
+      return table;
+    }
     case 'table':
-      // TODO - parse columns
       return {schema, rows: data};
-
     case 'rows':
     default:
       return data;
@@ -124,10 +104,9 @@ export function parseDBF(
  */
 export async function* parseDBFInBatches(
   asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,
-  options: any = {}
+  options: DBFLoaderOptions = {}
 ): AsyncIterable<DBFHeader | DBFRowsOutput | DBFTableOutput> {
-  const loaderOptions = options.dbf || {};
-  const {encoding} = loaderOptions;
+  const {encoding = 'latin1'} = options.dbf || {};
 
   const parser = new DBFParser({encoding});
   let headerReturned = false;
@@ -160,7 +139,7 @@ export async function* parseDBFInBatches(
 function parseState(
   state: STATE,
   result: DBFResult,
-  binaryReader: {[key: string]: any},
+  binaryReader: BinaryChunkReader,
   textDecoder: TextDecoder
 ): STATE {
   // eslint-disable-next-line no-constant-condition
@@ -173,7 +152,8 @@ function parseState(
 
         case STATE.START:
           // Parse initial file header
-          const dataView = binaryReader.getDataView(DBF_HEADER_SIZE, 'DBF header');
+          // DBF Header
+          const dataView = binaryReader.getDataView(DBF_HEADER_SIZE);
           if (!dataView) {
             return state;
           }
@@ -190,8 +170,7 @@ function parseState(
           // Parse DBF field descriptors (schema)
           const fieldDescriptorView = binaryReader.getDataView(
             // @ts-ignore
-            result.dbfHeader.headerLength - DBF_HEADER_SIZE,
-            'DBF field descriptors'
+            result.dbfHeader.headerLength - DBF_HEADER_SIZE
           );
           if (!fieldDescriptorView) {
             return state;
@@ -398,7 +377,7 @@ function parseCharacter(text: string): string | null {
  * @returns Field
  */
 // eslint-disable
-function makeField({name, dataType, fieldLength, decimal}): Field {
+function makeField({name, dataType, fieldLength, decimal}: DBFField): Field {
   switch (dataType) {
     case 'B':
       return new Field(name, new Float64(), true);

package/src/lib/parsers/parse-shapefile.ts

@@ -2,6 +2,7 @@
 import type {SHXOutput} from './parse-shx';
 import type {SHPHeader} from './parse-shp-header';
 import type {LoaderContext} from '@loaders.gl/loader-utils';
+import type {ShapefileLoaderOptions} from './types';
 
 import {binaryToGeometry, transformGeoJsonCoords} from '@loaders.gl/gis';
 import {Proj4Projection} from '@math.gl/proj4';
@@ -20,15 +21,11 @@ interface ShapefileOutput {
 }
 /**
  * Parsing of file in batches
- *
- * @param asyncIterator
- * @param options
- * @param context
  */
 // eslint-disable-next-line max-statements, complexity
 export async function* parseShapefileInBatches(
   asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,
-  options?: any,
+  options?: ShapefileLoaderOptions,
   context?: LoaderContext
 ): AsyncIterable<ShapefileOutput> {
   const {reproject = false, _targetCrs = 'WGS84'} = options?.gis || {};
@@ -109,7 +106,7 @@ export async function* parseShapefileInBatches(
  */
 export async function parseShapefile(
   arrayBuffer: ArrayBuffer,
-  options?: {[key: string]: any},
+  options?: ShapefileLoaderOptions,
   context?: LoaderContext
 ): Promise<ShapefileOutput> {
   const {reproject = false, _targetCrs = 'WGS84'} = options?.gis || {};

package/src/lib/parsers/parse-shp-geometry.ts

@@ -1,4 +1,5 @@
 import {BinaryGeometry, BinaryGeometryType} from '@loaders.gl/schema';
+import {SHPLoaderOptions} from './types';
 
 const LITTLE_ENDIAN = true;
 
@@ -9,8 +10,8 @@ const LITTLE_ENDIAN = true;
  * @return Binary Geometry Object
  */
 // eslint-disable-next-line complexity
-export function parseRecord(view: DataView, options?: {shp?: any}): BinaryGeometry | null {
-  const {_maxDimensions} = options?.shp || {};
+export function parseRecord(view: DataView, options?: SHPLoaderOptions): BinaryGeometry | null {
+  const {_maxDimensions = 4} = options?.shp || {};
 
   let offset = 0;
   const type: number = view.getInt32(offset, LITTLE_ENDIAN);

package/src/lib/parsers/parse-shp.ts

@@ -1,8 +1,8 @@
-import type {LoaderOptions} from '@loaders.gl/loader-utils';
 import type {BinaryGeometry} from '@loaders.gl/schema';
 import BinaryChunkReader from '../streaming/binary-chunk-reader';
-import {parseSHPHeader} from './parse-shp-header';
+import {parseSHPHeader, SHPHeader} from './parse-shp-header';
 import {parseRecord} from './parse-shp-geometry';
+import {SHPLoaderOptions} from './types';
 
 const LITTLE_ENDIAN = true;
 const BIG_ENDIAN = false;
@@ -20,20 +20,34 @@ const STATE = {
 };
 
 type SHPResult = {
-  geometries: [];
-  header?: {};
+  geometries: (BinaryGeometry | null)[];
+  header?: SHPHeader;
   error?: string;
+  progress: {
+    bytesUsed: number;
+    bytesTotal: number;
+    rows: number;
+  };
+  currentIndex: number;
 };
 
 class SHPParser {
-  options?: any = {};
+  options?: SHPLoaderOptions = {};
   binaryReader = new BinaryChunkReader({maxRewindBytes: SHP_RECORD_HEADER_SIZE});
   state = STATE.EXPECTING_HEADER;
   result: SHPResult = {
-    geometries: []
+    geometries: [],
+    // Initialize with number values to make TS happy
+    // These are initialized for real in STATE.EXPECTING_HEADER
+    progress: {
+      bytesTotal: NaN,
+      bytesUsed: NaN,
+      rows: NaN
+    },
+    currentIndex: NaN
   };
 
-  constructor(options?: LoaderOptions) {
+  constructor(options?: SHPLoaderOptions) {
     this.options = options;
   }
 
@@ -109,9 +123,9 @@ export async function* parseSHPInBatches(
 /* eslint-disable complexity, max-depth */
 function parseState(
   state: number,
-  result: {[key: string]: any},
+  result: SHPResult,
   binaryReader: BinaryChunkReader,
-  options: {shp?: any}
+  options?: SHPLoaderOptions
 ): number {
   // eslint-disable-next-line no-constant-condition
   while (true) {
@@ -157,7 +171,7 @@ function parseState(
 
             const invalidRecord =
               recordHeader.byteLength < 4 ||
-              recordHeader.type !== result.header.type ||
+              recordHeader.type !== result.header?.type ||
               recordHeader.recordNumber !== result.currentIndex;
 
             // All records must have at least four bytes (for the record shape type)

package/src/lib/parsers/types.ts

@@ -0,0 +1,74 @@
+import {Schema, ObjectRowTable} from '@loaders.gl/schema';
+import type {LoaderOptions} from '@loaders.gl/loader-utils';
+
+export type SHPLoaderOptions = LoaderOptions & {
+  shp?: {
+    _maxDimensions?: number;
+  };
+};
+
+export type DBFLoaderOptions = LoaderOptions & {
+  dbf?: {
+    encoding?: string;
+    shape?: 'rows' | 'table' | 'object-row-table';
+  };
+};
+
+export type ShapefileLoaderOptions = LoaderOptions &
+  SHPLoaderOptions & {
+    shapefile?: {
+      shape?: 'geojson';
+    };
+    gis?: {
+      reproject?: boolean;
+      _targetCrs?: string;
+      /** @deprecated. Use options.shapefile.shape */
+      format?: 'geojson';
+    };
+  };
+
+export type DBFRowsOutput = ObjectRowTable['data'];
+
+/**
+ * DBF Table output. Deprecated in favor of ObjectRowTable
+ * @deprecated
+ */
+export interface DBFTableOutput {
+  schema?: Schema;
+  rows: DBFRowsOutput;
+}
+
+export type DBFHeader = {
+  // Last updated date
+  year: number;
+  month: number;
+  day: number;
+  // Number of records in data file
+  nRecords: number;
+  // Length of header in bytes
+  headerLength: number;
+  // Length of each record
+  recordLength: number;
+  // Not sure if this is usually set
+  languageDriver: number;
+};
+
+export type DBFField = {
+  name: string;
+  dataType: string;
+  fieldLength: number;
+  decimal: number;
+};
+
+export type DBFResult = {
+  data: {[key: string]: any}[];
+  schema?: Schema;
+  error?: string;
+  dbfHeader?: DBFHeader;
+  dbfFields?: DBFField[];
+  progress?: {
+    bytesUsed: number;
+    rowsTotal: number;
+    rows: number;
+  };
+};

package/src/lib/streaming/binary-chunk-reader.ts

@@ -1,10 +1,14 @@
+type BinaryChunkReaderOptions = {
+  maxRewindBytes: number;
+};
+
 export default class BinaryChunkReader {
   offset: number;
   arrayBuffers: ArrayBuffer[];
   ended: boolean;
   maxRewindBytes: number;
 
-  constructor(options?: {[key: string]: any}) {
+  constructor(options?: BinaryChunkReaderOptions) {
     const {maxRewindBytes = 0} = options || {};
 
     /** current global offset into current array buffer*/

package/src/lib/streaming/zip-batch-iterators.ts

@@ -8,8 +8,8 @@ export async function* zipBatchIterators(
   iterator1: AsyncIterator<any[]>,
   iterator2: AsyncIterator<any[]>
 ): AsyncGenerator<number[][], void, unknown> {
-  let batch1 = [];
-  let batch2 = [];
+  let batch1: number[] = [];
+  let batch2: number[] = [];
   let iterator1Done: boolean = false;
   let iterator2Done: boolean = false;