@loaders.gl/core 5.0.0-alpha.12 → 5.0.0-alpha.13

package/src/loader-utils/lib/iterators/async-iteration.ts

@@ -0,0 +1,136 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {concatenateArrayBuffers} from '../binary-utils/array-buffer-utils';
+
+// GENERAL UTILITIES
+
+/**
+ * Iterates over an {@link AsyncIterable} or {@link Iterable}, invoking `visitor` for each yielded
+ * value without rewinding the iterator when exiting early. This enables the caller to continue
+ * iterating in another loop after `visitor` signals cancellation.
+ */
+export async function forEach<TValue>(
+  iterable: AsyncIterable<TValue> | Iterable<TValue> | AsyncIterator<TValue>,
+  visitor: (value: TValue) => any
+) {
+  const iterator = toAsyncIterator(iterable);
+  // eslint-disable-next-line
+  while (true) {
+    const {done, value} = await iterator.next();
+    if (done) {
+      if (iterator.return) {
+        iterator.return();
+      }
+      return;
+    }
+    const cancel = visitor(value);
+    if (cancel) {
+      return;
+    }
+  }
+}
+
+/**
+ * Concatenates all binary chunks yielded by an async or sync iterator.
+ * Supports `ArrayBuffer`, typed array views, and `ArrayBufferLike` sources (e.g. `SharedArrayBuffer`).
+ * This allows atomic parsers to operate on iterator inputs by materializing them into a single buffer.
+ */
+export async function concatenateArrayBuffersAsync(
+  asyncIterator:
+    | AsyncIterable<ArrayBufferLike | ArrayBufferView>
+    | Iterable<ArrayBufferLike | ArrayBufferView>
+): Promise<ArrayBuffer> {
+  const arrayBuffers: ArrayBuffer[] = [];
+  for await (const chunk of asyncIterator) {
+    arrayBuffers.push(copyToArrayBuffer(chunk));
+  }
+  return concatenateArrayBuffers(...arrayBuffers);
+}
+
+export async function concatenateStringsAsync(
+  asyncIterator: AsyncIterable<string> | Iterable<string>
+): Promise<string> {
+  const strings: string[] = [];
+  for await (const chunk of asyncIterator) {
+    strings.push(chunk);
+  }
+  return strings.join('');
+}
+
+/**
+ * Normalizes binary chunk iterators to yield `ArrayBuffer` instances.
+ * Accepts `ArrayBuffer`, `ArrayBufferView`, and `ArrayBufferLike` sources
+ * (e.g. `SharedArrayBuffer`) and returns a copied `ArrayBuffer` for each chunk.
+ */
+export async function* toArrayBufferIterator(
+  asyncIterator:
+    | AsyncIterable<ArrayBufferLike | ArrayBufferView>
+    | Iterable<ArrayBufferLike | ArrayBufferView>
+): AsyncIterable<ArrayBuffer> {
+  for await (const chunk of asyncIterator) {
+    yield copyToArrayBuffer(chunk);
+  }
+}
+
+function copyToArrayBuffer(chunk: ArrayBufferLike | ArrayBufferView | ArrayBuffer): ArrayBuffer {
+  if (chunk instanceof ArrayBuffer) {
+    return chunk;
+  }
+
+  if (ArrayBuffer.isView(chunk)) {
+    const {buffer, byteOffset, byteLength} = chunk;
+    return copyFromBuffer(buffer, byteOffset, byteLength);
+  }
+
+  return copyFromBuffer(chunk as ArrayBufferLike);
+}
+
+function copyFromBuffer(
+  buffer: ArrayBufferLike,
+  byteOffset = 0,
+  byteLength = buffer.byteLength - byteOffset
+): ArrayBuffer {
+  const view = new Uint8Array(buffer, byteOffset, byteLength);
+  const copy = new Uint8Array(view.length);
+  copy.set(view);
+  return copy.buffer;
+}
+
+function toAsyncIterator<TValue>(
+  iterable: AsyncIterable<TValue> | Iterable<TValue> | AsyncIterator<TValue>
+): AsyncIterator<TValue> {
+  if (typeof iterable[Symbol.asyncIterator] === 'function') {
+    return iterable[Symbol.asyncIterator]();
+  }
+
+  if (typeof iterable[Symbol.iterator] === 'function') {
+    const iterator = iterable[Symbol.iterator]();
+    return iteratorToAsyncIterator(iterator);
+  }
+
+  return iterable as AsyncIterator<TValue>;
+}
+
+function iteratorToAsyncIterator<T>(iterator: Iterator<T>): AsyncIterator<T> {
+  return {
+    next(value?: any) {
+      return Promise.resolve(iterator.next(value));
+    },
+
+    return(value?: any) {
+      if (typeof iterator.return === 'function') {
+        return Promise.resolve(iterator.return(value));
+      }
+      return Promise.resolve({done: true, value});
+    },
+
+    throw(error?: any) {
+      if (typeof iterator.throw === 'function') {
+        return Promise.resolve(iterator.throw(error));
+      }
+      return Promise.reject(error);
+    }
+  };
+}

package/src/loader-utils/lib/iterators/text-iterators.ts

@@ -0,0 +1,76 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// TextDecoder iterators
+// TextDecoder will keep any partial undecoded bytes between calls to `decode`
+
+export async function* makeTextDecoderIterator(
+  arrayBufferIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,
+  options: TextDecoderOptions = {}
+): AsyncIterable<string> {
+  const textDecoder = new TextDecoder(undefined, options);
+  for await (const arrayBuffer of arrayBufferIterator) {
+    yield typeof arrayBuffer === 'string'
+      ? arrayBuffer
+      : textDecoder.decode(arrayBuffer, {stream: true});
+  }
+}
+
+// TextEncoder iterator
+// TODO - this is not useful unless min chunk size is given
+// TextEncoder will keep any partial undecoded bytes between calls to `encode`
+// If iterator does not yield strings, assume arrayBuffer and return unencoded
+
+export async function* makeTextEncoderIterator(
+  textIterator: AsyncIterable<string> | Iterable<ArrayBuffer>
+): AsyncIterable<ArrayBuffer> {
+  const textEncoder = new TextEncoder();
+  for await (const text of textIterator) {
+    yield typeof text === 'string' ? textEncoder.encode(text).buffer : text;
+  }
+}
+
+/**
+ * @param textIterator async iterable yielding strings
+ * @returns an async iterable over lines
+ * See http://2ality.com/2018/04/async-iter-nodejs.html
+ */
+
+export async function* makeLineIterator(
+  textIterator: AsyncIterable<string>
+): AsyncIterable<string> {
+  let previous = '';
+  for await (const textChunk of textIterator) {
+    previous += textChunk;
+    let eolIndex;
+    eolIndex = previous.indexOf('\n');
+    while (eolIndex >= 0) {
+      // line includes the EOL
+      const line = previous.slice(0, eolIndex + 1);
+      previous = previous.slice(eolIndex + 1);
+      yield line;
+      eolIndex = previous.indexOf('\n');
+    }
+  }
+
+  if (previous.length > 0) {
+    yield previous;
+  }
+}
+
+/**
+ * @param lineIterator async iterable yielding lines
+ * @returns async iterable yielding numbered lines
+ *
+ * See http://2ality.com/2018/04/async-iter-nodejs.html
+ */
+export async function* makeNumberedLineIterator(
+  lineIterator: AsyncIterable<string>
+): AsyncIterable<{counter: number; line: string}> {
+  let counter = 1;
+  for await (const line of lineIterator) {
+    yield {counter, line};
+    counter++;
+  }
+}

package/src/loader-utils/lib/javascript-utils/is-type.ts

@@ -0,0 +1,130 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/* eslint-disable @typescript-eslint/unbound-method */
+
+import type {Readable} from 'stream';
+
+/** Minimal shape for Node.js Buffer-like values */
+type NodeBufferLike = {buffer: ArrayBufferLike; isBuffer: true};
+
+/** Minimal shape for Node.js writable streams */
+type NodeWritableStream = {
+  end: (...args: unknown[]) => unknown;
+  write: (...args: unknown[]) => unknown;
+  writable: boolean;
+};
+
+/** Minimal shape for WritableStream-like DOM implementations */
+type WritableDOMStreamLike = WritableStream | {abort: () => unknown; getWriter: () => unknown};
+
+/** A DOM or Node readable stream */
+export type ReadableStreamType = ReadableStream | Readable;
+
+/** Checks whether a value is a boolean */
+const isBoolean = (value: unknown): value is boolean => typeof value === 'boolean';
+
+/** Checks whether a value is a function */
+const isFunction = (value: unknown): value is (...args: unknown[]) => unknown =>
+  typeof value === 'function';
+
+/** Checks whether a value is a non-null object */
+export const isObject = (value: unknown): value is object =>
+  value !== null && typeof value === 'object';
+
+/** Checks whether a value is a plain object (created by the Object constructor) */
+export const isPureObject = (value: unknown): value is Record<string, unknown> =>
+  isObject(value) && value.constructor === {}.constructor;
+
+/** Checks whether a value is an ArrayBuffer */
+export const isArrayBuffer = (value: unknown): value is ArrayBuffer =>
+  typeof ArrayBuffer !== 'undefined' && value instanceof ArrayBuffer;
+
+/** Checks whether a value is an ArrayBuffer */
+export const isSharedArrayBuffer = (value: unknown): value is SharedArrayBuffer =>
+  typeof SharedArrayBuffer !== 'undefined' && value instanceof SharedArrayBuffer;
+
+/** Checks whether a value is ArrayBuffer-like */
+export const isArrayBufferLike = (value: unknown): value is ArrayBufferLike =>
+  isObject(value) &&
+  typeof (value as ArrayBufferLike).byteLength === 'number' &&
+  typeof (value as ArrayBufferLike).slice === 'function';
+
+/** Checks whether a value behaves like a promise */
+export const isPromise = (value: unknown): value is Promise<unknown> =>
+  isObject(value) && 'then' in value && isFunction((value as {then: unknown}).then);
+
+/** Checks whether a value implements the iterable protocol */
+export const isIterable = (value: unknown): value is Iterable<unknown> =>
+  Boolean(value) && isFunction((value as Iterable<unknown>)[Symbol.iterator]);
+
+/** Checks whether a value implements the async iterable protocol */
+export const isAsyncIterable = (value: unknown): value is AsyncIterable<unknown> =>
+  Boolean(value) && isFunction((value as AsyncIterable<unknown>)[Symbol.asyncIterator]);
+
+/** Checks whether a value is an iterator (has a next function) */
+export const isIterator = (value: unknown): value is Iterator<unknown> =>
+  Boolean(value) && isFunction((value as Iterator<unknown>).next);
+
+/** Checks whether a value is a fetch Response or a duck-typed equivalent */
+export const isResponse = (value: unknown): value is Response =>
+  (typeof Response !== 'undefined' && value instanceof Response) ||
+  (isObject(value) &&
+    isFunction((value as {arrayBuffer?: unknown}).arrayBuffer) &&
+    isFunction((value as {text?: unknown}).text) &&
+    isFunction((value as {json?: unknown}).json));
+
+/** Checks whether a value is a File */
+export const isFile = (value: unknown): value is File =>
+  typeof File !== 'undefined' && value instanceof File;
+
+/** Checks whether a value is a Blob */
+export const isBlob = (value: unknown): value is Blob =>
+  typeof Blob !== 'undefined' && value instanceof Blob;
+
+/** Checks for Node.js Buffers without triggering bundlers to include the Buffer polyfill */
+export const isBuffer = (value: unknown): value is NodeBufferLike =>
+  Boolean(
+    value &&
+      typeof value === 'object' &&
+      (value as Partial<NodeBufferLike>).isBuffer &&
+      'buffer' in (value as NodeBufferLike)
+  );
+
+/** Checks whether a value looks like a DOM WritableStream */
+export const isWritableDOMStream = (value: unknown): value is WritableDOMStreamLike =>
+  isObject(value) &&
+  isFunction((value as WritableDOMStreamLike).abort) &&
+  isFunction((value as WritableDOMStreamLike).getWriter);
+
+/** Checks whether a value looks like a DOM ReadableStream */
+export const isReadableDOMStream = (value: unknown): value is ReadableStream =>
+  (typeof ReadableStream !== 'undefined' && value instanceof ReadableStream) ||
+  (isObject(value) &&
+    isFunction((value as ReadableStream).tee) &&
+    isFunction((value as ReadableStream).cancel) &&
+    isFunction((value as ReadableStream).getReader));
+// Not implemented in Firefox: && isFunction(x.pipeTo)
+
+/** Checks whether a value looks like a Node.js writable stream */
+export const isWritableNodeStream = (value: unknown): value is NodeWritableStream =>
+  isObject(value) &&
+  isFunction((value as NodeWritableStream).end) &&
+  isFunction((value as NodeWritableStream).write) &&
+  isBoolean((value as NodeWritableStream).writable);
+
+/** Checks whether a value looks like a Node.js readable stream */
+export const isReadableNodeStream = (value: unknown): value is Readable =>
+  isObject(value) &&
+  isFunction((value as Readable).read) &&
+  isFunction((value as Readable).pipe) &&
+  isBoolean((value as Readable).readable);
+
+/** Checks whether a value is any readable stream (DOM or Node.js) */
+export const isReadableStream = (value: unknown): value is ReadableStreamType =>
+  isReadableDOMStream(value) || isReadableNodeStream(value);
+
+/** Checks whether a value is any writable stream (DOM or Node.js) */
+export const isWritableStream = (value: unknown): value is WritableStream | NodeWritableStream =>
+  isWritableDOMStream(value) || isWritableNodeStream(value);

package/src/loader-utils/lib/laz/LASZIP-LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2022 Rapidlasso, GmbH
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

package/src/loader-utils/lib/laz/LASZIP-NOTICE

@@ -0,0 +1,6 @@
+The LASzip item version 1 decoding algorithms in laz-chunk-decoder.ts are adapted
+from LASzip, https://github.com/LASzip/LASzip (src/lasreaditemcompressed_v1.cpp).
+Copyright (c) 2007-2022 rapidlasso GmbH.
+Licensed under the Apache License, Version 2.0; see LASZIP-LICENSE.
+The TypeScript adaptation, direct column output, and incremental input support
+were added by the vis.gl contributors.