@loaders.gl/shapefile 4.2.0-alpha.3 → 4.2.0-alpha.5

package/dist/lib/parsers/parse-shapefile.js

@@ -1,3 +1,4 @@
+// import type {Feature} from '@loaders.gl/gis';
 import { parseInBatchesFromContext, parseFromContext } from '@loaders.gl/loader-utils';
 import { binaryToGeometry, transformGeoJsonCoords } from '@loaders.gl/gis';
 import { Proj4Projection } from '@math.gl/proj4';
@@ -5,207 +6,247 @@ import { parseShx } from "./parse-shx.js";
 import { zipBatchIterators } from "../streaming/zip-batch-iterators.js";
 import { SHPLoader } from "../../shp-loader.js";
 import { DBFLoader } from "../../dbf-loader.js";
+/**
+ * Parsing of file in batches
+ */
+// eslint-disable-next-line max-statements, complexity
 export async function* parseShapefileInBatches(asyncIterator, options, context) {
-  var _shapeIterable$Symbol, _shapeIterable$Symbol2;
-  const {
-    reproject = false,
-    _targetCrs = 'WGS84'
-  } = (options === null || options === void 0 ? void 0 : options.gis) || {};
-  const {
-    shx,
-    cpg,
-    prj
-  } = await loadShapefileSidecarFiles(options, context);
-  const shapeIterable = await parseInBatchesFromContext(asyncIterator, SHPLoader, options, context);
-  const shapeIterator = ((_shapeIterable$Symbol = shapeIterable[Symbol.asyncIterator]) === null || _shapeIterable$Symbol === void 0 ? void 0 : _shapeIterable$Symbol.call(shapeIterable)) || ((_shapeIterable$Symbol2 = shapeIterable[Symbol.iterator]) === null || _shapeIterable$Symbol2 === void 0 ? void 0 : _shapeIterable$Symbol2.call(shapeIterable));
-  let propertyIterator = null;
-  const dbfResponse = await (context === null || context === void 0 ? void 0 : context.fetch(replaceExtension((context === null || context === void 0 ? void 0 : context.url) || '', 'dbf')));
-  if (dbfResponse !== null && dbfResponse !== void 0 && dbfResponse.ok) {
-    var _propertyIterable$Sym;
-    const propertyIterable = await parseInBatchesFromContext(dbfResponse, DBFLoader, {
-      ...options,
-      dbf: {
-        encoding: cpg || 'latin1'
-      }
-    }, context);
-    propertyIterator = ((_propertyIterable$Sym = propertyIterable[Symbol.asyncIterator]) === null || _propertyIterable$Sym === void 0 ? void 0 : _propertyIterable$Sym.call(propertyIterable)) || propertyIterable[Symbol.iterator]();
-  }
-  let shapeHeader = (await shapeIterator.next()).value;
-  if (shapeHeader && shapeHeader.batchType === 'metadata') {
-    shapeHeader = (await shapeIterator.next()).value;
-  }
-  let dbfHeader = {};
-  if (propertyIterator) {
-    dbfHeader = (await propertyIterator.next()).value;
-    if (dbfHeader && dbfHeader.batchType === 'metadata') {
-      dbfHeader = (await propertyIterator.next()).value;
-    }
-  }
-  const zippedIterator = propertyIterator ? zipBatchIterators(shapeIterator, propertyIterator, 'object-row-table') : shapeIterator;
-  const zippedBatchIterable = {
-    [Symbol.asyncIterator]() {
-      return zippedIterator;
-    }
-  };
-  for await (const batch of zippedBatchIterable) {
-    let geometries;
-    let properties;
-    if (!propertyIterator) {
-      geometries = batch;
-    } else {
-      [geometries, properties] = batch.data;
+    const { reproject = false, _targetCrs = 'WGS84' } = options?.gis || {};
+    const { shx, cpg, prj } = await loadShapefileSidecarFiles(options, context);
+    // parse geometries
+    const shapeIterable = await parseInBatchesFromContext(asyncIterator, SHPLoader, options, context);
+    const shapeIterator = shapeIterable[Symbol.asyncIterator]?.() || shapeIterable[Symbol.iterator]?.();
+    // parse properties
+    let propertyIterator = null;
+    const dbfResponse = await context?.fetch(replaceExtension(context?.url || '', 'dbf'));
+    if (dbfResponse?.ok) {
+        const propertyIterable = await parseInBatchesFromContext(dbfResponse, DBFLoader, {
+            ...options,
+            dbf: { encoding: cpg || 'latin1' }
+        }, context);
+        propertyIterator =
+            propertyIterable[Symbol.asyncIterator]?.() || propertyIterable[Symbol.iterator]();
     }
-    const geojsonGeometries = parseGeometries(geometries);
-    let features = joinProperties(geojsonGeometries, properties);
-    if (reproject) {
-      features = reprojectFeatures(features, prj, _targetCrs);
-    }
-    yield {
-      encoding: cpg,
-      prj,
-      shx,
-      header: shapeHeader,
-      data: features
+    // When `options.metadata` is `true`, there's an extra initial `metadata`
+    // object before the iterator starts. zipBatchIterators expects to receive
+    // batches of Array objects, and will fail with non-iterable batches, so it's
+    // important to skip over the first batch.
+    let shapeHeader = (await shapeIterator.next()).value;
+    if (shapeHeader && shapeHeader.batchType === 'metadata') {
+        shapeHeader = (await shapeIterator.next()).value;
+    }
+    let dbfHeader = {};
+    if (propertyIterator) {
+        dbfHeader = (await propertyIterator.next()).value;
+        if (dbfHeader && dbfHeader.batchType === 'metadata') {
+            dbfHeader = (await propertyIterator.next()).value;
+        }
+    }
+    const zippedIterator = propertyIterator
+        ? zipBatchIterators(shapeIterator, propertyIterator, 'object-row-table')
+        : shapeIterator;
+    const zippedBatchIterable = {
+        [Symbol.asyncIterator]() {
+            return zippedIterator;
+        }
     };
-  }
+    for await (const batch of zippedBatchIterable) {
+        let geometries;
+        let properties;
+        if (!propertyIterator) {
+            geometries = batch;
+        }
+        else {
+            [geometries, properties] = batch.data;
+        }
+        const geojsonGeometries = parseGeometries(geometries);
+        let features = joinProperties(geojsonGeometries, properties);
+        if (reproject) {
+            // @ts-ignore
+            features = reprojectFeatures(features, prj, _targetCrs);
+        }
+        yield {
+            encoding: cpg,
+            prj,
+            shx,
+            header: shapeHeader,
+            data: features
+        };
+    }
 }
+/**
+ * Parse shapefile
+ *
+ * @param arrayBuffer
+ * @param options
+ * @param context
+ * @returns output of shapefile
+ */
 export async function parseShapefile(arrayBuffer, options, context) {
-  var _propertyTable, _options$shapefile, _propertyTable2;
-  const {
-    reproject = false,
-    _targetCrs = 'WGS84'
-  } = (options === null || options === void 0 ? void 0 : options.gis) || {};
-  const {
-    shx,
-    cpg,
-    prj
-  } = await loadShapefileSidecarFiles(options, context);
-  const {
-    header,
-    geometries
-  } = await parseFromContext(arrayBuffer, SHPLoader, options, context);
-  const geojsonGeometries = parseGeometries(geometries);
-  let propertyTable;
-  const dbfResponse = await (context === null || context === void 0 ? void 0 : context.fetch(replaceExtension(context === null || context === void 0 ? void 0 : context.url, 'dbf')));
-  if (dbfResponse !== null && dbfResponse !== void 0 && dbfResponse.ok) {
-    propertyTable = await parseFromContext(dbfResponse, DBFLoader, {
-      dbf: {
-        shape: 'object-row-table',
-        encoding: cpg || 'latin1'
-      }
-    }, context);
-  }
-  let features = joinProperties(geojsonGeometries, ((_propertyTable = propertyTable) === null || _propertyTable === void 0 ? void 0 : _propertyTable.data) || []);
-  if (reproject) {
-    features = reprojectFeatures(features, prj, _targetCrs);
-  }
-  switch (options === null || options === void 0 ? void 0 : (_options$shapefile = options.shapefile) === null || _options$shapefile === void 0 ? void 0 : _options$shapefile.shape) {
-    case 'geojson-table':
-      return {
-        shape: 'geojson-table',
-        type: 'FeatureCollection',
-        encoding: cpg,
-        schema: ((_propertyTable2 = propertyTable) === null || _propertyTable2 === void 0 ? void 0 : _propertyTable2.schema) || {
-          metadata: {},
-          fields: []
-        },
-        prj,
-        shx,
-        header,
-        features
-      };
-    default:
-      return {
-        encoding: cpg,
-        prj,
-        shx,
-        header,
-        data: features
-      };
-  }
+    const { reproject = false, _targetCrs = 'WGS84' } = options?.gis || {};
+    const { shx, cpg, prj } = await loadShapefileSidecarFiles(options, context);
+    // parse geometries
+    const { header, geometries } = await parseFromContext(arrayBuffer, SHPLoader, options, context); // {shp: shx}
+    const geojsonGeometries = parseGeometries(geometries);
+    // parse properties
+    let propertyTable;
+    const dbfResponse = await context?.fetch(replaceExtension(context?.url, 'dbf'));
+    if (dbfResponse?.ok) {
+        propertyTable = await parseFromContext(dbfResponse, DBFLoader, { dbf: { shape: 'object-row-table', encoding: cpg || 'latin1' } }, context);
+    }
+    let features = joinProperties(geojsonGeometries, propertyTable?.data || []);
+    if (reproject) {
+        features = reprojectFeatures(features, prj, _targetCrs);
+    }
+    switch (options?.shapefile?.shape) {
+        case 'geojson-table':
+            return {
+                // @ts-expect-error
+                shape: 'geojson-table',
+                type: 'FeatureCollection',
+                encoding: cpg,
+                schema: propertyTable?.schema || { metadata: {}, fields: [] },
+                prj,
+                shx,
+                header,
+                features
+            };
+        default:
+            return {
+                encoding: cpg,
+                prj,
+                shx,
+                header,
+                data: features
+            };
+    }
 }
+/**
+ * Parse geometries
+ *
+ * @param geometries
+ * @returns geometries as an array
+ */
 function parseGeometries(geometries) {
-  const geojsonGeometries = [];
-  for (const geom of geometries) {
-    geojsonGeometries.push(binaryToGeometry(geom));
-  }
-  return geojsonGeometries;
+    const geojsonGeometries = [];
+    for (const geom of geometries) {
+        geojsonGeometries.push(binaryToGeometry(geom));
+    }
+    return geojsonGeometries;
 }
+/**
+ * Join properties and geometries into features
+ *
+ * @param geometries [description]
+ * @param  properties [description]
+ * @return [description]
+ */
 function joinProperties(geometries, properties) {
-  const features = [];
-  for (let i = 0; i < geometries.length; i++) {
-    const geometry = geometries[i];
-    const feature = {
-      type: 'Feature',
-      geometry,
-      properties: properties && properties[i] || {}
-    };
-    features.push(feature);
-  }
-  return features;
+    const features = [];
+    for (let i = 0; i < geometries.length; i++) {
+        const geometry = geometries[i];
+        const feature = {
+            type: 'Feature',
+            geometry,
+            // properties can be undefined if dbfResponse above was empty
+            properties: (properties && properties[i]) || {}
+        };
+        features.push(feature);
+    }
+    return features;
 }
+/**
+ * Reproject GeoJSON features to output CRS
+ *
+ * @param features parsed GeoJSON features
+ * @param sourceCrs source coordinate reference system
+ * @param targetCrs †arget coordinate reference system
+ * @return Reprojected Features
+ */
 function reprojectFeatures(features, sourceCrs, targetCrs) {
-  if (!sourceCrs && !targetCrs) {
-    return features;
-  }
-  const projection = new Proj4Projection({
-    from: sourceCrs || 'WGS84',
-    to: targetCrs || 'WGS84'
-  });
-  return transformGeoJsonCoords(features, coord => projection.project(coord));
+    if (!sourceCrs && !targetCrs) {
+        return features;
+    }
+    const projection = new Proj4Projection({ from: sourceCrs || 'WGS84', to: targetCrs || 'WGS84' });
+    return transformGeoJsonCoords(features, (coord) => projection.project(coord));
 }
+/**
+ *
+ * @param options
+ * @param context
+ * @returns Promise
+ */
+// eslint-disable-next-line max-statements
 export async function loadShapefileSidecarFiles(options, context) {
-  const {
-    url,
-    fetch
-  } = context;
-  const shxPromise = fetch(replaceExtension(url, 'shx'));
-  const cpgPromise = fetch(replaceExtension(url, 'cpg'));
-  const prjPromise = fetch(replaceExtension(url, 'prj'));
-  await Promise.all([shxPromise, cpgPromise, prjPromise]);
-  let shx;
-  let cpg;
-  let prj;
-  const shxResponse = await shxPromise;
-  if (shxResponse.ok) {
-    const arrayBuffer = await shxResponse.arrayBuffer();
-    shx = parseShx(arrayBuffer);
-  }
-  const cpgResponse = await cpgPromise;
-  if (cpgResponse.ok) {
-    cpg = await cpgResponse.text();
-  }
-  const prjResponse = await prjPromise;
-  if (prjResponse.ok) {
-    prj = await prjResponse.text();
-  }
-  return {
-    shx,
-    cpg,
-    prj
-  };
+    // Attempt a parallel load of the small sidecar files
+    // @ts-ignore context must be defined
+    const { url, fetch } = context;
+    const shxPromise = fetch(replaceExtension(url, 'shx'));
+    const cpgPromise = fetch(replaceExtension(url, 'cpg'));
+    const prjPromise = fetch(replaceExtension(url, 'prj'));
+    await Promise.all([shxPromise, cpgPromise, prjPromise]);
+    let shx;
+    let cpg;
+    let prj;
+    const shxResponse = await shxPromise;
+    if (shxResponse.ok) {
+        const arrayBuffer = await shxResponse.arrayBuffer();
+        shx = parseShx(arrayBuffer);
+    }
+    const cpgResponse = await cpgPromise;
+    if (cpgResponse.ok) {
+        cpg = await cpgResponse.text();
+    }
+    const prjResponse = await prjPromise;
+    if (prjResponse.ok) {
+        prj = await prjResponse.text();
+    }
+    return {
+        shx,
+        cpg,
+        prj
+    };
 }
+/**
+ * Replace the extension at the end of a path.
+ *
+ * Matches the case of new extension with the case of the original file extension,
+ * to increase the chance of finding files without firing off a request storm looking for various case combinations
+ *
+ * NOTE: Extensions can be both lower and uppercase
+ * per spec, extensions should be lower case, but that doesn't mean they always are. See:
+ * calvinmetcalf/shapefile-js#64, mapserver/mapserver#4712
+ * https://trac.osgeo.org/mapserver/ticket/166
+ */
 export function replaceExtension(url, newExtension) {
-  const baseName = basename(url);
-  const extension = extname(url);
-  const isUpperCase = extension === extension.toUpperCase();
-  if (isUpperCase) {
-    newExtension = newExtension.toUpperCase();
-  }
-  return `${baseName}.${newExtension}`;
+    const baseName = basename(url);
+    const extension = extname(url);
+    const isUpperCase = extension === extension.toUpperCase();
+    if (isUpperCase) {
+        newExtension = newExtension.toUpperCase();
+    }
+    return `${baseName}.${newExtension}`;
 }
+// NOTE - this gives the entire path minus extension (i.e. NOT same as path.basename)
+/**
+ * @param url
+ * @returns string
+ */
 function basename(url) {
-  const extIndex = url && url.lastIndexOf('.');
-  if (typeof extIndex === 'number') {
-    return extIndex >= 0 ? url.substr(0, extIndex) : '';
-  }
-  return extIndex;
+    const extIndex = url && url.lastIndexOf('.');
+    if (typeof extIndex === 'number') {
+        return extIndex >= 0 ? url.substr(0, extIndex) : '';
+    }
+    return extIndex;
 }
+/**
+ * @param url
+ * @returns string
+ */
 function extname(url) {
-  const extIndex = url && url.lastIndexOf('.');
-  if (typeof extIndex === 'number') {
-    return extIndex >= 0 ? url.substr(extIndex + 1) : '';
-  }
-  return extIndex;
+    const extIndex = url && url.lastIndexOf('.');
+    if (typeof extIndex === 'number') {
+        return extIndex >= 0 ? url.substr(extIndex + 1) : '';
+    }
+    return extIndex;
 }
-//# sourceMappingURL=parse-shapefile.js.map

package/dist/lib/parsers/parse-shp-geometry.d.ts

@@ -1,5 +1,5 @@
 import { BinaryGeometry } from '@loaders.gl/schema';
-import { SHPLoaderOptions } from './types';
+import { SHPLoaderOptions } from "./types.js";
 /**
  * Parse individual record
  *

package/dist/lib/parsers/parse-shp-geometry.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parse-shp-geometry.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-shp-geometry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,cAAc,EAAqB,MAAM,oBAAoB,CAAC;AACtE,OAAO,EAAC,gBAAgB,EAAC,MAAM,SAAS,CAAC;AAIzC;;;;;GAKG;AAEH,wBAAgB,WAAW,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,cAAc,GAAG,IAAI,CAoD7F"}
+{"version":3,"file":"parse-shp-geometry.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-shp-geometry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,cAAc,EAAqB,MAAM,oBAAoB,CAAC;AACtE,OAAO,EAAC,gBAAgB,EAAC,mBAAgB;AAIzC;;;;;GAKG;AAEH,wBAAgB,WAAW,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,cAAc,GAAG,IAAI,CAoD7F"}