@loaders.gl/shapefile 4.4.0-alpha.1 → 4.4.0-alpha.9

package/dist/shp-worker.js

@@ -1,5 +1,230 @@
 "use strict";
 (() => {
+  // ../worker-utils/src/lib/node/worker_threads-browser.ts
+  var parentPort = null;
+
+  // ../worker-utils/src/lib/worker-utils/get-transfer-list.ts
+  function getTransferList(object, recursive = true, transfers) {
+    const transfersSet = transfers || /* @__PURE__ */ new Set();
+    if (!object) {
+    } else if (isTransferable(object)) {
+      transfersSet.add(object);
+    } else if (isTransferable(object.buffer)) {
+      transfersSet.add(object.buffer);
+    } else if (ArrayBuffer.isView(object)) {
+    } else if (recursive && typeof object === "object") {
+      for (const key in object) {
+        getTransferList(object[key], recursive, transfersSet);
+      }
+    }
+    return transfers === void 0 ? Array.from(transfersSet) : [];
+  }
+  function isTransferable(object) {
+    if (!object) {
+      return false;
+    }
+    if (object instanceof ArrayBuffer) {
+      return true;
+    }
+    if (typeof MessagePort !== "undefined" && object instanceof MessagePort) {
+      return true;
+    }
+    if (typeof ImageBitmap !== "undefined" && object instanceof ImageBitmap) {
+      return true;
+    }
+    if (typeof OffscreenCanvas !== "undefined" && object instanceof OffscreenCanvas) {
+      return true;
+    }
+    return false;
+  }
+
+  // ../worker-utils/src/lib/worker-farm/worker-body.ts
+  async function getParentPort() {
+    return parentPort;
+  }
+  var onMessageWrapperMap = /* @__PURE__ */ new Map();
+  var WorkerBody = class {
+    /** Check that we are actually in a worker thread */
+    static async inWorkerThread() {
+      return typeof self !== "undefined" || Boolean(await getParentPort());
+    }
+    /*
+     * (type: WorkerMessageType, payload: WorkerMessagePayload) => any
+     */
+    static set onmessage(onMessage) {
+      async function handleMessage(message) {
+        const parentPort2 = await getParentPort();
+        const { type, payload } = parentPort2 ? message : message.data;
+        onMessage(type, payload);
+      }
+      getParentPort().then((parentPort2) => {
+        if (parentPort2) {
+          parentPort2.on("message", (message) => {
+            handleMessage(message);
+          });
+          parentPort2.on("exit", () => console.debug("Node worker closing"));
+        } else {
+          globalThis.onmessage = handleMessage;
+        }
+      });
+    }
+    static async addEventListener(onMessage) {
+      let onMessageWrapper = onMessageWrapperMap.get(onMessage);
+      if (!onMessageWrapper) {
+        onMessageWrapper = async (message) => {
+          if (!isKnownMessage(message)) {
+            return;
+          }
+          const parentPort3 = await getParentPort();
+          const { type, payload } = parentPort3 ? message : message.data;
+          onMessage(type, payload);
+        };
+      }
+      const parentPort2 = await getParentPort();
+      if (parentPort2) {
+        console.error("not implemented");
+      } else {
+        globalThis.addEventListener("message", onMessageWrapper);
+      }
+    }
+    static async removeEventListener(onMessage) {
+      const onMessageWrapper = onMessageWrapperMap.get(onMessage);
+      onMessageWrapperMap.delete(onMessage);
+      const parentPort2 = await getParentPort();
+      if (parentPort2) {
+        console.error("not implemented");
+      } else {
+        globalThis.removeEventListener("message", onMessageWrapper);
+      }
+    }
+    /**
+     * Send a message from a worker to creating thread (main thread)
+     * @param type
+     * @param payload
+     */
+    static async postMessage(type, payload) {
+      const data = { source: "loaders.gl", type, payload };
+      const transferList = getTransferList(payload);
+      const parentPort2 = await getParentPort();
+      if (parentPort2) {
+        parentPort2.postMessage(data, transferList);
+      } else {
+        globalThis.postMessage(data, transferList);
+      }
+    }
+  };
+  function isKnownMessage(message) {
+    const { type, data } = message;
+    return type === "message" && data && typeof data.source === "string" && data.source.startsWith("loaders.gl");
+  }
+
+  // ../loader-utils/src/lib/worker-loader-utils/create-loader-worker.ts
+  var requestId = 0;
+  async function createLoaderWorker(loader) {
+    if (!await WorkerBody.inWorkerThread()) {
+      return;
+    }
+    WorkerBody.onmessage = async (type, payload) => {
+      switch (type) {
+        case "process":
+          try {
+            const { input, options = {}, context = {} } = payload;
+            const result = await parseData({
+              loader,
+              arrayBuffer: input,
+              options,
+              // @ts-expect-error fetch missing
+              context: {
+                ...context,
+                _parse: parseOnMainThread
+              }
+            });
+            WorkerBody.postMessage("done", { result });
+          } catch (error) {
+            const message = error instanceof Error ? error.message : "";
+            WorkerBody.postMessage("error", { error: message });
+          }
+          break;
+        default:
+      }
+    };
+  }
+  function parseOnMainThread(arrayBuffer, loader, options, context) {
+    return new Promise((resolve, reject) => {
+      const id = requestId++;
+      const onMessage = (type, payload2) => {
+        if (payload2.id !== id) {
+          return;
+        }
+        switch (type) {
+          case "done":
+            WorkerBody.removeEventListener(onMessage);
+            resolve(payload2.result);
+            break;
+          case "error":
+            WorkerBody.removeEventListener(onMessage);
+            reject(payload2.error);
+            break;
+          default:
+        }
+      };
+      WorkerBody.addEventListener(onMessage);
+      const payload = { id, input: arrayBuffer, options };
+      WorkerBody.postMessage("process", payload);
+    });
+  }
+  async function parseData({
+    loader,
+    arrayBuffer,
+    options,
+    context
+  }) {
+    let data;
+    let parser;
+    if (loader.parseSync || loader.parse) {
+      data = arrayBuffer;
+      parser = loader.parseSync || loader.parse;
+    } else if (loader.parseTextSync) {
+      const textDecoder = new TextDecoder();
+      data = textDecoder.decode(arrayBuffer);
+      parser = loader.parseTextSync;
+    } else {
+      throw new Error(`Could not load data with ${loader.name} loader`);
+    }
+    options = {
+      ...options,
+      modules: loader && loader.options && loader.options.modules || {},
+      core: {
+        ...options.core,
+        worker: false
+      }
+    };
+    return await parser(data, { ...options }, context, loader);
+  }
+
+  // ../loader-utils/src/lib/iterators/async-iteration.ts
+  async function* toArrayBufferIterator(asyncIterator) {
+    for await (const chunk of asyncIterator) {
+      yield copyToArrayBuffer(chunk);
+    }
+  }
+  function copyToArrayBuffer(chunk) {
+    if (chunk instanceof ArrayBuffer) {
+      return chunk;
+    }
+    if (ArrayBuffer.isView(chunk)) {
+      const { buffer, byteOffset, byteLength } = chunk;
+      return copyFromBuffer(buffer, byteOffset, byteLength);
+    }
+    return copyFromBuffer(chunk);
+  }
+  function copyFromBuffer(buffer, byteOffset = 0, byteLength = buffer.byteLength - byteOffset) {
+    const view = new Uint8Array(buffer, byteOffset, byteLength);
+    const copy = new Uint8Array(view.length);
+    copy.set(view);
+    return copy.buffer;
+  }
+
   // src/lib/streaming/binary-chunk-reader.ts
   var BinaryChunkReader = class {
     offset;
@@ -402,7 +627,7 @@
   async function* parseSHPInBatches(asyncIterator, options) {
     const parser = new SHPParser(options);
     let headerReturned = false;
-    for await (const arrayBuffer of asyncIterator) {
+    for await (const arrayBuffer of toArrayBufferIterator(asyncIterator)) {
       parser.write(arrayBuffer);
       if (!headerReturned && parser.result.header) {
         headerReturned = true;
@@ -484,7 +709,7 @@
   }
 
   // src/shp-loader.ts
-  var VERSION = true ? "4.4.0-alpha.0" : "latest";
+  var VERSION = true ? "4.4.0-alpha.9" : "latest";
   var SHP_MAGIC_NUMBER2 = [0, 0, 39, 10];
   var SHPWorkerLoader = {
     dataType: null,
@@ -512,205 +737,6 @@
     parseInBatches: (arrayBufferIterator, options) => parseSHPInBatches(arrayBufferIterator, options)
   };
 
-  // ../worker-utils/src/lib/node/worker_threads-browser.ts
-  var parentPort = null;
-
-  // ../worker-utils/src/lib/worker-utils/get-transfer-list.ts
-  function getTransferList(object, recursive = true, transfers) {
-    const transfersSet = transfers || /* @__PURE__ */ new Set();
-    if (!object) {
-    } else if (isTransferable(object)) {
-      transfersSet.add(object);
-    } else if (isTransferable(object.buffer)) {
-      transfersSet.add(object.buffer);
-    } else if (ArrayBuffer.isView(object)) {
-    } else if (recursive && typeof object === "object") {
-      for (const key in object) {
-        getTransferList(object[key], recursive, transfersSet);
-      }
-    }
-    return transfers === void 0 ? Array.from(transfersSet) : [];
-  }
-  function isTransferable(object) {
-    if (!object) {
-      return false;
-    }
-    if (object instanceof ArrayBuffer) {
-      return true;
-    }
-    if (typeof MessagePort !== "undefined" && object instanceof MessagePort) {
-      return true;
-    }
-    if (typeof ImageBitmap !== "undefined" && object instanceof ImageBitmap) {
-      return true;
-    }
-    if (typeof OffscreenCanvas !== "undefined" && object instanceof OffscreenCanvas) {
-      return true;
-    }
-    return false;
-  }
-
-  // ../worker-utils/src/lib/worker-farm/worker-body.ts
-  async function getParentPort() {
-    return parentPort;
-  }
-  var onMessageWrapperMap = /* @__PURE__ */ new Map();
-  var WorkerBody = class {
-    /** Check that we are actually in a worker thread */
-    static async inWorkerThread() {
-      return typeof self !== "undefined" || Boolean(await getParentPort());
-    }
-    /*
-     * (type: WorkerMessageType, payload: WorkerMessagePayload) => any
-     */
-    static set onmessage(onMessage) {
-      async function handleMessage(message) {
-        const parentPort2 = await getParentPort();
-        const { type, payload } = parentPort2 ? message : message.data;
-        onMessage(type, payload);
-      }
-      getParentPort().then((parentPort2) => {
-        if (parentPort2) {
-          parentPort2.on("message", (message) => {
-            handleMessage(message);
-          });
-          parentPort2.on("exit", () => console.debug("Node worker closing"));
-        } else {
-          globalThis.onmessage = handleMessage;
-        }
-      });
-    }
-    static async addEventListener(onMessage) {
-      let onMessageWrapper = onMessageWrapperMap.get(onMessage);
-      if (!onMessageWrapper) {
-        onMessageWrapper = async (message) => {
-          if (!isKnownMessage(message)) {
-            return;
-          }
-          const parentPort3 = await getParentPort();
-          const { type, payload } = parentPort3 ? message : message.data;
-          onMessage(type, payload);
-        };
-      }
-      const parentPort2 = await getParentPort();
-      if (parentPort2) {
-        console.error("not implemented");
-      } else {
-        globalThis.addEventListener("message", onMessageWrapper);
-      }
-    }
-    static async removeEventListener(onMessage) {
-      const onMessageWrapper = onMessageWrapperMap.get(onMessage);
-      onMessageWrapperMap.delete(onMessage);
-      const parentPort2 = await getParentPort();
-      if (parentPort2) {
-        console.error("not implemented");
-      } else {
-        globalThis.removeEventListener("message", onMessageWrapper);
-      }
-    }
-    /**
-     * Send a message from a worker to creating thread (main thread)
-     * @param type
-     * @param payload
-     */
-    static async postMessage(type, payload) {
-      const data = { source: "loaders.gl", type, payload };
-      const transferList = getTransferList(payload);
-      const parentPort2 = await getParentPort();
-      if (parentPort2) {
-        parentPort2.postMessage(data, transferList);
-      } else {
-        globalThis.postMessage(data, transferList);
-      }
-    }
-  };
-  function isKnownMessage(message) {
-    const { type, data } = message;
-    return type === "message" && data && typeof data.source === "string" && data.source.startsWith("loaders.gl");
-  }
-
-  // ../loader-utils/src/lib/worker-loader-utils/create-loader-worker.ts
-  var requestId = 0;
-  async function createLoaderWorker(loader) {
-    if (!await WorkerBody.inWorkerThread()) {
-      return;
-    }
-    WorkerBody.onmessage = async (type, payload) => {
-      switch (type) {
-        case "process":
-          try {
-            const { input, options = {}, context = {} } = payload;
-            const result = await parseData({
-              loader,
-              arrayBuffer: input,
-              options,
-              // @ts-expect-error fetch missing
-              context: {
-                ...context,
-                _parse: parseOnMainThread
-              }
-            });
-            WorkerBody.postMessage("done", { result });
-          } catch (error) {
-            const message = error instanceof Error ? error.message : "";
-            WorkerBody.postMessage("error", { error: message });
-          }
-          break;
-        default:
-      }
-    };
-  }
-  function parseOnMainThread(arrayBuffer, loader, options, context) {
-    return new Promise((resolve, reject) => {
-      const id = requestId++;
-      const onMessage = (type, payload2) => {
-        if (payload2.id !== id) {
-          return;
-        }
-        switch (type) {
-          case "done":
-            WorkerBody.removeEventListener(onMessage);
-            resolve(payload2.result);
-            break;
-          case "error":
-            WorkerBody.removeEventListener(onMessage);
-            reject(payload2.error);
-            break;
-          default:
-        }
-      };
-      WorkerBody.addEventListener(onMessage);
-      const payload = { id, input: arrayBuffer, options };
-      WorkerBody.postMessage("process", payload);
-    });
-  }
-  async function parseData({
-    loader,
-    arrayBuffer,
-    options,
-    context
-  }) {
-    let data;
-    let parser;
-    if (loader.parseSync || loader.parse) {
-      data = arrayBuffer;
-      parser = loader.parseSync || loader.parse;
-    } else if (loader.parseTextSync) {
-      const textDecoder = new TextDecoder();
-      data = textDecoder.decode(arrayBuffer);
-      parser = loader.parseTextSync;
-    } else {
-      throw new Error(`Could not load data with ${loader.name} loader`);
-    }
-    options = {
-      ...options,
-      modules: loader && loader.options && loader.options.modules || {},
-      worker: false
-    };
-    return await parser(data, { ...options }, context, loader);
-  }
-
   // src/workers/shp-worker.ts
   createLoaderWorker(SHPLoader);
 })();

package/dist/workers/dbf-worker.js

@@ -4,3 +4,4 @@
 import { DBFLoader } from "../dbf-loader.js";
 import { createLoaderWorker } from '@loaders.gl/loader-utils';
 createLoaderWorker(DBFLoader);
+//# sourceMappingURL=dbf-worker.js.map

package/dist/workers/dbf-worker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dbf-worker.js","sourceRoot":"","sources":["../../src/workers/dbf-worker.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAEpC,OAAO,EAAC,SAAS,EAAC,yBAAsB;AACxC,OAAO,EAAC,kBAAkB,EAAC,MAAM,0BAA0B,CAAC;AAE5D,kBAAkB,CAAC,SAAS,CAAC,CAAC"}

package/dist/workers/shp-worker.js

@@ -4,3 +4,4 @@
 import { SHPLoader } from "../shp-loader.js";
 import { createLoaderWorker } from '@loaders.gl/loader-utils';
 createLoaderWorker(SHPLoader);
+//# sourceMappingURL=shp-worker.js.map

package/dist/workers/shp-worker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"shp-worker.js","sourceRoot":"","sources":["../../src/workers/shp-worker.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAEpC,OAAO,EAAC,SAAS,EAAC,yBAAsB;AACxC,OAAO,EAAC,kBAAkB,EAAC,MAAM,0BAA0B,CAAC;AAE5D,kBAAkB,CAAC,SAAS,CAAC,CAAC"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@loaders.gl/shapefile",
   "description": "Loader for the Shapefile Format",
-  "version": "4.4.0-alpha.1",
+  "version": "4.4.0-alpha.9",
   "license": "MIT",
   "type": "module",
   "publishConfig": {
@@ -26,6 +26,12 @@
       "types": "./dist/index.d.ts",
       "import": "./dist/index.js",
       "require": "./dist/index.cjs"
+    },
+    "./shp-worker.js": {
+      "import": "./dist/shp-worker.js"
+    },
+    "./dbf-worker.js": {
+      "import": "./dist/dbf-worker.js"
     }
   },
   "sideEffects": false,
@@ -45,13 +51,13 @@
     "build-worker-dbf": "esbuild src/workers/dbf-worker.ts --bundle --outfile=dist/dbf-worker.js --define:__VERSION__=\\\"$npm_package_version\\\""
   },
   "dependencies": {
-    "@loaders.gl/gis": "4.4.0-alpha.1",
-    "@loaders.gl/loader-utils": "4.4.0-alpha.1",
-    "@loaders.gl/schema": "4.4.0-alpha.1",
+    "@loaders.gl/gis": "4.4.0-alpha.9",
+    "@loaders.gl/loader-utils": "4.4.0-alpha.9",
+    "@loaders.gl/schema": "4.4.0-alpha.9",
     "@math.gl/proj4": "^4.1.0"
   },
   "peerDependencies": {
-    "@loaders.gl/core": "4.4.0-alpha.0"
+    "@loaders.gl/core": "4.4.0-alpha.1"
   },
-  "gitHead": "f1732de45907bd500bf4eedb4803beca8bf4bfb0"
+  "gitHead": "e9e6710379718c7663e97eba868c76e15de4cb84"
 }

package/src/dbf-arrow-loader.ts

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
+import type {Loader, LoaderWithParser, StrictLoaderOptions} from '@loaders.gl/loader-utils';
 import type {ArrowTable, ArrowTableBatch} from '@loaders.gl/schema';
 import {parseDBF, parseDBFInBatches} from './lib/parsers/parse-dbf-to-arrow';
 import {DBFFormat} from './dbf-format';
@@ -11,7 +11,7 @@ import {DBFFormat} from './dbf-format';
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
-export type DBFLoaderOptions = LoaderOptions & {
+export type DBFLoaderOptions = StrictLoaderOptions & {
   dbf?: {
     encoding?: string;
     /** Override the URL to the worker bundle (by default loads from unpkg.com) */
@@ -40,7 +40,12 @@ export const DBFArrowLoader = {
   ...DBFArrowWorkerLoader,
   parse: async (arrayBuffer, options) => parseDBF(arrayBuffer, options),
   parseSync: parseDBF,
-  parseInBatches(arrayBufferIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>, options) {
+  parseInBatches(
+    arrayBufferIterator:
+      | AsyncIterable<ArrayBufferLike | ArrayBufferView>
+      | Iterable<ArrayBufferLike | ArrayBufferView>,
+    options
+  ) {
     return parseDBFInBatches(arrayBufferIterator, options);
   }
 } as const satisfies LoaderWithParser<ArrowTable, ArrowTableBatch, DBFLoaderOptions>;

package/src/dbf-loader.ts

@@ -2,16 +2,17 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
+import type {Loader, LoaderWithParser, StrictLoaderOptions} from '@loaders.gl/loader-utils';
 import {parseDBF, parseDBFInBatches} from './lib/parsers/parse-dbf';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
-export type DBFLoaderOptions = LoaderOptions & {
+export type DBFLoaderOptions = StrictLoaderOptions & {
   dbf?: {
     encoding?: string;
+    shape?: 'rows' | 'table' | 'object-row-table';
     /** Override the URL to the worker bundle (by default loads from unpkg.com) */
     workerUrl?: string;
   };
@@ -44,7 +45,12 @@ export const DBFLoader: LoaderWithParser = {
   ...DBFWorkerLoader,
   parse: async (arrayBuffer, options) => parseDBF(arrayBuffer, options),
   parseSync: parseDBF,
-  parseInBatches(arrayBufferIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>, options) {
+  parseInBatches(
+    arrayBufferIterator:
+      | AsyncIterable<ArrayBufferLike | ArrayBufferView>
+      | Iterable<ArrayBufferLike | ArrayBufferView>,
+    options
+  ) {
     return parseDBFInBatches(arrayBufferIterator, options);
   }
 };

package/src/lib/parsers/parse-dbf-to-arrow.ts

@@ -3,6 +3,7 @@
 // Copyright (c) vis.gl contributors
 
 import type {Schema, Field, ArrowTable, ArrowTableBatch} from '@loaders.gl/schema';
+import {toArrayBufferIterator} from '@loaders.gl/loader-utils';
 import {ArrowTableBuilder} from '@loaders.gl/schema-utils';
 import {BinaryChunkReader} from '../streaming/binary-chunk-reader';
 import {DBFLoaderOptions, DBFHeader, DBFField} from './types';
@@ -87,14 +88,16 @@ export function parseDBF(arrayBuffer: ArrayBuffer, options: DBFLoaderOptions = {
  * @param options
  */
 export async function* parseDBFInBatches(
-  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,
+  asyncIterator:
+    | AsyncIterable<ArrayBufferLike | ArrayBufferView>
+    | Iterable<ArrayBufferLike | ArrayBufferView>,
   options: DBFLoaderOptions = {}
 ): AsyncIterable<ArrowTableBatch> {
   const {encoding = 'latin1'} = options.dbf || {};
 
   const parser = new DBFParser({encoding});
   let headerReturned = false;
-  for await (const arrayBuffer of asyncIterator) {
+  for await (const arrayBuffer of toArrayBufferIterator(asyncIterator)) {
     parser.write(arrayBuffer);
     if (!headerReturned && parser.result.dbfHeader) {
       headerReturned = true;

package/src/lib/parsers/parse-dbf.ts

@@ -3,6 +3,7 @@
 // Copyright (c) vis.gl contributors
 
 import type {Field, ObjectRowTable} from '@loaders.gl/schema';
+import {toArrayBufferIterator} from '@loaders.gl/loader-utils';
 import {BinaryChunkReader} from '../streaming/binary-chunk-reader';
 import {
   DBFLoaderOptions,
@@ -99,14 +100,16 @@ export function parseDBF(
  * @param options
  */
 export async function* parseDBFInBatches(
-  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,
+  asyncIterator:
+    | AsyncIterable<ArrayBufferLike | ArrayBufferView>
+    | Iterable<ArrayBufferLike | ArrayBufferView>,
   options: DBFLoaderOptions = {}
 ): AsyncIterable<DBFHeader | DBFRowsOutput | DBFTableOutput> {
   const {encoding = 'latin1'} = options.dbf || {};
 
   const parser = new DBFParser({encoding});
   let headerReturned = false;
-  for await (const arrayBuffer of asyncIterator) {
+  for await (const arrayBuffer of toArrayBufferIterator(asyncIterator)) {
     parser.write(arrayBuffer);
     if (!headerReturned && parser.result.dbfHeader) {
       headerReturned = true;