@loaders.gl/gis 4.3.1 → 4.4.0-alpha.1

package/src/lib/table-converters/convert-geoarrow-table.ts

@@ -0,0 +1,218 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import * as arrow from 'apache-arrow';
+import type {
+  Table,
+  ArrayRowTable,
+  ColumnarTable,
+  ObjectRowTable,
+  GeoJSONTable,
+  ArrowTable,
+  Feature
+} from '@loaders.gl/schema';
+
+import {
+  convertTable,
+  convertArrowToSchema,
+  convertSchemaToArrow,
+  getTableLength,
+  getTableNumCols,
+  getTableCellAt
+} from '@loaders.gl/schema-utils';
+import {getGeometryColumnsFromSchema} from '@loaders.gl/geoarrow';
+import {convertGeoArrowGeometryToGeoJSON} from '../geometry-converters/convert-geoarrow-to-geojson';
+
+/**
+ * * Convert a loaders.gl Table to an Apache Arrow Table
+ * @param mesh
+ * @param metadata
+ * @param batchSize
+ * @returns
+ */
+export function convertTableToArrow(table: Table, options?: {batchSize?: number}): arrow.Table {
+  switch (table.shape) {
+    case 'arrow-table':
+      return table.data;
+
+    case 'columnar-table':
+    // TODO - optimized implementation is possible
+    // return convertColumnarTableToArrow(table, options);
+
+    // fall through
+
+    default:
+      const arrowBatchIterator = makeTableToArrowBatchesIterator(table, options);
+      return new arrow.Table(arrowBatchIterator);
+  }
+}
+
+export function* makeTableToArrowBatchesIterator(
+  table: Table,
+  options?: {batchSize?: number}
+): IterableIterator<arrow.RecordBatch> {
+  const arrowSchema = convertSchemaToArrow(table.schema!);
+
+  const length = getTableLength(table);
+  const numColumns = getTableNumCols(table);
+  const batchSize = options?.batchSize || length;
+
+  const builders = arrowSchema?.fields.map((arrowField) => arrow.makeBuilder(arrowField));
+  const structField = new arrow.Struct(arrowSchema.fields);
+
+  let batchLength = 0;
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    for (let columnIndex = 0; columnIndex < numColumns; ++columnIndex) {
+      const value = getTableCellAt(table, rowIndex, columnIndex);
+
+      const builder = builders[columnIndex];
+      builder.append(value);
+      batchLength++;
+
+      if (batchLength >= batchSize) {
+        const datas = builders.map((builder) => builder.flush());
+        const structData = new arrow.Data(structField, 0, batchLength, 0, undefined, datas);
+        yield new arrow.RecordBatch(arrowSchema, structData);
+        batchLength = 0;
+      }
+    }
+  }
+
+  if (batchLength > 0) {
+    const datas = builders.map((builder) => builder.flush());
+    const structData = new arrow.Data(structField, 0, batchLength, 0, undefined, datas);
+    yield new arrow.RecordBatch(arrowSchema, structData);
+    batchLength = 0;
+  }
+
+  builders.map((builder) => builder.finish());
+}
+
+/**
+ * Convert an Apache Arrow table to a loaders.gl Table
+ * @note Currently does not convert schema
+ */
+export function convertGeoArrowToTable(arrow: arrow.Table, shape: 'arrow-table'): ArrowTable;
+export function convertGeoArrowToTable(arrow: arrow.Table, shape: 'columnar-table'): ColumnarTable;
+export function convertGeoArrowToTable(
+  arrow: arrow.Table,
+  shape: 'object-row-table'
+): ObjectRowTable;
+export function convertGeoArrowToTable(arrow: arrow.Table, shape: 'array-row-table'): ArrayRowTable;
+export function convertGeoArrowToTable(arrow: arrow.Table, shape: 'geojson-table'): GeoJSONTable;
+export function convertGeoArrowToTable(arrow: arrow.Table, shape: 'columnar-table'): ColumnarTable;
+export function convertGeoArrowToTable(arrow: arrow.Table, shape: Table['shape']): Table;
+
+/**
+ * Convert an Apache Arrow table to a loaders.gl Table
+ * @note Currently does not convert schema
+ */
+export function convertGeoArrowToTable(arrowTable: arrow.Table, shape: Table['shape']): Table {
+  switch (shape) {
+    case 'arrow-table':
+      return convertArrowToArrowTable(arrowTable);
+    case 'array-row-table':
+      return convertArrowToArrayRowTable(arrowTable);
+    case 'object-row-table':
+      return convertArrowToObjectRowTable(arrowTable);
+    case 'columnar-table':
+      return convertArrowToColumnarTable(arrowTable);
+    case 'geojson-table':
+      return convertArrowToGeoJSONTable(arrowTable);
+    default:
+      throw new Error(shape);
+  }
+}
+
+/**
+ * Wrap an apache arrow table in a loaders.gl table wrapper.
+ * From this additional conversions are available.
+ * @param arrowTable
+ * @returns
+ */
+function convertArrowToArrowTable(arrowTable: arrow.Table): ArrowTable {
+  return {
+    shape: 'arrow-table',
+    schema: convertArrowToSchema(arrowTable.schema),
+    data: arrowTable
+  };
+}
+
+function convertArrowToArrayRowTable(arrowTable: arrow.Table): Table {
+  const columnarTable = convertArrowToColumnarTable(arrowTable);
+  return convertTable(columnarTable, 'array-row-table');
+}
+
+function convertArrowToObjectRowTable(arrowTable: arrow.Table): Table {
+  const columnarTable = convertArrowToColumnarTable(arrowTable);
+  return convertTable(columnarTable, 'object-row-table');
+}
+
+/**
+ * Convert an Apache Arrow table to a ColumnarTable
+ * @note Currently does not convert schema
+ */
+function convertArrowToColumnarTable(arrowTable: arrow.Table): ColumnarTable {
+  // TODO - avoid calling `getColumn` on columns we are not interested in?
+  // Add options object?
+
+  const columns: ColumnarTable['data'] = {};
+
+  for (const field of arrowTable.schema.fields) {
+    // This (is intended to) coalesce all record batches into a single typed array
+    const arrowColumn = arrowTable.getChild(field.name);
+    const values = arrowColumn?.toArray();
+    columns[field.name] = values;
+  }
+
+  const schema = convertArrowToSchema(arrowTable.schema);
+
+  return {
+    shape: 'columnar-table',
+    schema,
+    data: columns
+  };
+}
+
+/**
+ * Convert an Apache Arrow table to a GeoJSONTable
+ * @note Currently does not convert schema
+ */
+function convertArrowToGeoJSONTable(arrowTable: arrow.Table): GeoJSONTable {
+  const schema = convertArrowToSchema(arrowTable.schema);
+  const geometryColumns = getGeometryColumnsFromSchema(schema);
+
+  // get encoding from geometryColumns['geometry']
+  const encoding = geometryColumns.geometry.encoding;
+
+  const features: Feature[] = [];
+
+  // Remove geometry columns
+  const propertyColumnNames = arrowTable.schema.fields
+    .map((field) => field.name)
+    // TODO - this deletes all geometry columns
+    .filter((name) => !(name in geometryColumns));
+  const propertiesTable = arrowTable.select(propertyColumnNames);
+
+  const arrowGeometryColumn = arrowTable.getChild('geometry');
+
+  for (let row = 0; row < arrowTable.numRows; row++) {
+    // get the geometry value from arrow geometry column
+    // Note that type can vary
+    const arrowGeometry = arrowGeometryColumn?.get(row);
+    // parse arrow geometry to geojson feature
+    const feature = convertGeoArrowGeometryToGeoJSON(arrowGeometry, encoding);
+    if (feature) {
+      const properties = propertiesTable.get(row)?.toJSON() || {};
+      features.push({type: 'Feature', geometry: feature, properties});
+    }
+  }
+
+  return {
+    shape: 'geojson-table',
+    type: 'FeatureCollection',
+    schema,
+    features
+  };
+}

package/src/lib/{tables/convert-table-to-geojson.ts → table-converters/convert-wkb-table-to-geojson.ts}

@@ -2,7 +2,6 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import type {LoaderWithParser} from '@loaders.gl/loader-utils';
 import type {
   ArrayRowTable,
   GeoJSONTable,
@@ -11,17 +10,17 @@ import type {
   Feature,
   Geometry
 } from '@loaders.gl/schema';
-import {getTableLength, getTableRowAsObject} from '@loaders.gl/schema';
-
-import {GeoColumnMetadata, getGeoMetadata} from '../geo/geoparquet-metadata';
+import {getTableLength, getTableRowAsObject} from '@loaders.gl/schema-utils';
+import {GeoColumnMetadata, getGeoMetadata} from '@loaders.gl/geoarrow';
+import {convertWKBToGeometry} from '../geometry-converters/wkb/convert-wkb-to-geometry';
+import {convertWKTToGeometry} from '../geometry-converters/wkb/convert-wkt-to-geometry';
 
 /** TODO - move to loaders.gl/gis? */
 export function convertWKBTableToGeoJSON(
   table: ArrayRowTable | ObjectRowTable,
-  schema: Schema,
-  loaders: LoaderWithParser[]
+  schema: Schema
 ): GeoJSONTable {
-  const geoMetadata = getGeoMetadata(schema);
+  const geoMetadata = getGeoMetadata(schema.metadata);
   const primaryColumn = geoMetadata?.primary_column;
   if (!primaryColumn) {
     throw new Error('no geometry column');
@@ -33,7 +32,7 @@ export function convertWKBTableToGeoJSON(
   const length = getTableLength(table);
   for (let rowIndex = 0; rowIndex < length; rowIndex++) {
     const row = getTableRowAsObject(table, rowIndex);
-    const geometry = parseGeometry(row[primaryColumn], columnMetadata, loaders);
+    const geometry = parseGeometry(row[primaryColumn], columnMetadata);
     delete row[primaryColumn];
     const feature: Feature = {type: 'Feature', geometry: geometry!, properties: row};
     features.push(feature);
@@ -42,27 +41,15 @@ export function convertWKBTableToGeoJSON(
   return {shape: 'geojson-table', schema, type: 'FeatureCollection', features};
 }
 
-function parseGeometry(
-  geometry: unknown,
-  columnMetadata: GeoColumnMetadata,
-  loaders: LoaderWithParser[]
-): Geometry | null {
+function parseGeometry(geometry: unknown, columnMetadata: GeoColumnMetadata): Geometry | null {
   switch (columnMetadata.encoding) {
     case 'wkt':
-      const wktLoader = loaders.find((loader) => loader.id === 'wkt');
-      return wktLoader?.parseTextSync?.(geometry as string) || null;
+      return convertWKTToGeometry(geometry as string) || null;
     case 'wkb':
     default:
-      const wkbLoader = loaders.find((loader) => loader.id === 'wkb');
       const arrayBuffer = ArrayBuffer.isView(geometry)
         ? geometry.buffer.slice(geometry.byteOffset, geometry.byteOffset + geometry.byteLength)
         : (geometry as ArrayBuffer);
-      const geojson = wkbLoader?.parseSync?.(arrayBuffer, {
-        wkb: {shape: 'geojson-geometry'}
-      }) as unknown as Geometry;
-      return geojson; // binaryGeometry ? binaryToGeometry(binaryGeometry) : null;
-    // const binaryGeometry = WKBLoader.parseSync?.(geometry);
-    // ts-ignore
-    // return binaryGeometry ? binaryToGeometry(binaryGeometry) : null;
+      return convertWKBToGeometry(arrayBuffer);
   }
 }

package/src/lib/table-converters/make-arrow-batch-iterator.ts

@@ -0,0 +1,53 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import * as arrow from 'apache-arrow';
+import type {Table} from '@loaders.gl/schema';
+import {
+  convertSchemaToArrow,
+  getTableLength,
+  getTableNumCols,
+  getTableCellAt
+} from '@loaders.gl/schema-utils';
+
+export function* makeTableToArrowBatchesIterator(
+  table: Table,
+  options?: {batchSize?: number}
+): IterableIterator<arrow.RecordBatch> {
+  const arrowSchema = convertSchemaToArrow(table.schema!);
+
+  const length = getTableLength(table);
+  const numColumns = getTableNumCols(table);
+  const batchSize = options?.batchSize || length;
+
+  const builders = arrowSchema?.fields.map((arrowField) => arrow.makeBuilder(arrowField));
+  const structField = new arrow.Struct(arrowSchema.fields);
+
+  let batchLength = 0;
+  for (let rowIndex = 0; rowIndex < length; rowIndex++) {
+    for (let columnIndex = 0; columnIndex < numColumns; ++columnIndex) {
+      const value = getTableCellAt(table, rowIndex, columnIndex);
+
+      const builder = builders[columnIndex];
+      builder.append(value);
+      batchLength++;
+
+      if (batchLength >= batchSize) {
+        const datas = builders.map((builder) => builder.flush());
+        const structData = new arrow.Data(structField, 0, batchLength, 0, undefined, datas);
+        yield new arrow.RecordBatch(arrowSchema, structData);
+        batchLength = 0;
+      }
+    }
+  }
+
+  if (batchLength > 0) {
+    const datas = builders.map((builder) => builder.flush());
+    const structData = new arrow.Data(structField, 0, batchLength, 0, undefined, datas);
+    yield new arrow.RecordBatch(arrowSchema, structData);
+    batchLength = 0;
+  }
+
+  builders.map((builder) => builder.finish());
+}

package/src/lib/utils/base64-encoder.ts

@@ -0,0 +1,157 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/* 
+const binary_to_b64_map = [
+  'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I',
+  'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R',
+  'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a',
+  'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j',
+  'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's',
+  't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1',
+  '2', '3', '4', '5', '6', '7', '8', '9', '+',
+  '/', '='
+];
+const b64_to_binary_map = {
+  '0': 52, '1': 53, '2': 54, '3': 55, '4': 56, '5': 57, '6': 58, '7': 59, '8': 60, '9': 61,
+  A: 0, B: 1, C: 2, D: 3, E: 4, F: 5, G: 6, H: 7, I: 8, J: 9, K: 10, L: 11, M: 12, N: 13, O: 14,
+  P: 15, Q: 16, R: 17, S: 18, T: 19, U: 20, V: 21, W: 22, X: 23, Y: 24, Z: 25, a: 26, b: 27,
+  c: 28, d: 29, e: 30, f: 31, g: 32, h: 33, i: 34, j: 35, k: 36, l: 37, m: 38, n: 39, o: 40,
+  p: 41, q: 42, r: 43, s: 44, t: 45, u: 46, v: 47, w: 48, x: 49, y: 50, z: 51, '+': 62, '/': 63,
+  '=': 64
+};
+
+*/
+
+export class Base64Encoder {
+  getDecodedLength(array: Uint8Array): number {
+    return Math.ceil(array.byteLength / 4) * 3;
+  }
+
+  decode(array: Uint8Array, target?: ArrayBuffer) {} // eslint-disable-line
+}
+
+/*
+
+//generates an array iterator that returns 3 elements at a time. Use to loop through the Uint8Array Array Buffer
+// to be converted to Base64.  (binary array buffer) 8bits * 3 = 6bits * 4 (base64 representation)
+const generateTripleIterator = (arr) => {
+  return {
+      *[Symbol.iterator]() {
+          for(let n = 0; n < arr.length; n+=3) {
+              let result = [];
+              result.push(arr[n]);
+
+              if(n+1 < arr.length)
+                  result.push(arr[n+1]);
+              if(n+2 < arr.length)
+                  result.push(arr[n+2]);
+
+              yield result;
+          }
+      }
+  };
+};
+
+//generates an array iterator that returns 4 elements at a time.  Use to loop through
+// Base64 string because Base64 string is multiples of 4 characters.
+const generateQuadrupleIterator = (arr) => {
+  return {
+      *[Symbol.iterator]() {
+          for(let n = 0; n < arr.length; n+=4) {
+              yield [...arr.slice(n, n+4)];
+          }
+      }
+  };
+};
+
+// Converts a triple of 8 bits into a quadruple of 6 bits.  use to convert binary to base64 representation
+const tripleConvert = (first, second, third) => {
+  let [] = triple;
+  let binary = null, a = null, b = null, c = null, d = null;
+  if (triple.length === 1) {
+      binary = (first << 4);
+      a = ((binary & 4032) >>> 6);
+      b = (binary & 63);
+      c = 64;
+      d = 64;
+  } else if (triple.length === 2) {
+      binary = ((first << 10) | (second << 2));
+      a = ((binary & 258048) >>> 12);
+      b = ((binary & 4032) >>> 6);
+      c = (binary & 63);
+      d = 64;
+  } else {
+      binary = ((first << 16) | (second << 8) | third);
+      a = ((binary & 16515072) >>> 18);
+      b = ((binary & 258048) >>> 12);
+      c = ((binary & 4032) >>> 6);
+      d = (binary & 63);
+  }
+
+  return [a, b, c, d];
+};
+
+// Converts a quadruple of 6 bits into a triple of 8 bits.  use to convert base64 representation into binary
+const quadrupleConvert = (quadruple) => {
+  let [a, b, c, d] = quadruple;
+  let binary = null, first = null, second = null, third = null;
+
+  if(c === 64 && d === 64) {
+      //two padding
+      binary = ((a << 6) | b);
+      first = (binary >> 4);   //shift off 4 bits, 2 bits per padding
+  } else if(d === 64) {
+      //one padding
+      binary = ((a << 12) | (b << 6) | c );
+      binary = (binary >> 2); //shift off 2 bits
+      first = binary >> 8;
+      second = ((binary << 24) >>> 24);
+  } else {
+      //no padding
+      binary = ((a << 18) | (b << 12) | (c << 6) | d );
+      first = (binary >>> 16);
+      second = ((binary << 16) >>> 24);
+      third = ((binary << 24) >>> 24);
+  }
+
+  return [first, second, third];
+};
+
+// Convert 8Bits Array Buffer to Base64 string
+export const ab2b64 = (buffer) => {
+  const b64strArray = [];
+  const view = new Uint8Array(buffer);
+  let iterator = generateTripleIterator(view);
+  for(let triple of iterator) {
+      b64strArray.push(...tripleConvert(triple));
+  }
+  return b64strArray.map(b64CharCodePoint => binary_to_b64_map[b64CharCodePoint]).join("");
+};
+
+// Convert Base64 String to 8Bits Array Buffer
+export const b642ab = (b64str) => {
+  let buffer_length = (b64str.length / 4) * 3;
+  if(b64str.slice(-2) === '==') {
+      buffer_length -= 2;
+  } else if(b64str.slice(-1) === '=') {
+      buffer_length -= 1;
+  }
+
+  let buffer = new ArrayBuffer(buffer_length);
+  const view = new Uint8Array(buffer);
+  let iterator = generateQuadrupleIterator(b64str.split("").map(b64char => b64_to_binary_map[b64char]));
+  let byteIndex = 0;
+  for(let quadruple of iterator) {
+      quadrupleConvert(quadruple).forEach(byte => {
+          if(byte != null) {
+              view[byteIndex] = byte;
+              byteIndex++;
+          }
+      });
+  }
+  return buffer;
+};
+
+*/

package/src/lib/utils/binary-reader.ts

@@ -0,0 +1,76 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/** A DataView that tracks byte offset when reading. */
+export class BinaryReader {
+  arrayBuffer: ArrayBuffer;
+  dataView: DataView;
+  byteOffset: number;
+  littleEndian: boolean;
+
+  constructor(arrayBuffer: ArrayBuffer, isBigEndian: boolean = false) {
+    this.arrayBuffer = arrayBuffer;
+    this.dataView = new DataView(arrayBuffer);
+    this.byteOffset = 0;
+    this.littleEndian = !isBigEndian;
+  }
+
+  readUInt8() {
+    const value = this.dataView.getUint8(this.byteOffset);
+    this.byteOffset += 1;
+    return value;
+  }
+  readUInt16() {
+    const value = this.dataView.getUint16(this.byteOffset, this.littleEndian);
+    this.byteOffset += 2;
+    return value;
+  }
+  readUInt32() {
+    const value = this.dataView.getUint32(this.byteOffset, this.littleEndian);
+    this.byteOffset += 4;
+    return value;
+  }
+  readInt8() {
+    const value = this.dataView.getInt8(this.byteOffset);
+    this.byteOffset += 1;
+    return value;
+  }
+  readInt16() {
+    const value = this.dataView.getInt16(this.byteOffset, this.littleEndian);
+    this.byteOffset += 2;
+    return value;
+  }
+  readInt32() {
+    const value = this.dataView.getInt32(this.byteOffset, this.littleEndian);
+    this.byteOffset += 4;
+    return value;
+  }
+  readFloat() {
+    const value = this.dataView.getFloat32(this.byteOffset, this.littleEndian);
+    this.byteOffset += 4;
+    return value;
+  }
+  readDouble() {
+    const value = this.dataView.getFloat64(this.byteOffset, this.littleEndian);
+    this.byteOffset += 8;
+    return value;
+  }
+
+  readVarInt() {
+    let result = 0;
+    let bytesRead = 0;
+
+    let nextByte;
+    do {
+      // TODO - this needs to be accessed via data view?
+      nextByte = this.dataView.getUint8(this.byteOffset + bytesRead);
+      result += (nextByte & 0x7f) << (7 * bytesRead);
+      bytesRead++;
+    } while (nextByte >= 0x80);
+
+    this.byteOffset += bytesRead;
+
+    return result;
+  }
+}