@loaders.gl/arrow 4.3.0-alpha.8 → 4.3.0-beta.1

package/dist/tables/convert-arrow-to-geojson-table.d.ts

@@ -1,16 +0,0 @@
-import type { GeoJSONTable } from '@loaders.gl/schema';
-import * as arrow from 'apache-arrow';
-import type { ArrowTable } from "../lib/arrow-table.js";
-/**
- * Wrap an apache arrow table in a loaders.gl table wrapper.
- * From this additional conversions are available.
- * @param arrowTable
- * @returns
- */
-export declare function convertApacheArrowToArrowTable(arrowTable: arrow.Table): ArrowTable;
-/**
- * Convert an Apache Arrow table to a GeoJSONTable
- * @note Currently does not convert schema
- */
-export declare function convertArrowToGeoJSONTable(table: ArrowTable): GeoJSONTable;
-//# sourceMappingURL=convert-arrow-to-geojson-table.d.ts.map

package/dist/tables/convert-arrow-to-geojson-table.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"convert-arrow-to-geojson-table.d.ts","sourceRoot":"","sources":["../../src/tables/convert-arrow-to-geojson-table.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAU,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAC9D,OAAO,KAAK,KAAK,MAAM,cAAc,CAAC;AACtC,OAAO,KAAK,EAAC,UAAU,EAAC,8BAA2B;AAInD;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAAC,UAAU,EAAE,KAAK,CAAC,KAAK,GAAG,UAAU,CAMlF;AAED;;;GAGG;AACH,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,UAAU,GAAG,YAAY,CAqC1E"}

package/dist/tables/convert-arrow-to-geojson-table.js

@@ -1,54 +0,0 @@
-// loaders.gl
-// SPDX-License-Identifier: MIT
-// Copyright (c) vis.gl contributors
-import { serializeArrowSchema, parseGeometryFromArrow } from '@loaders.gl/arrow';
-import { getGeometryColumnsFromSchema } from '@loaders.gl/gis';
-/**
- * Wrap an apache arrow table in a loaders.gl table wrapper.
- * From this additional conversions are available.
- * @param arrowTable
- * @returns
- */
-export function convertApacheArrowToArrowTable(arrowTable) {
-    return {
-        shape: 'arrow-table',
-        schema: serializeArrowSchema(arrowTable.schema),
-        data: arrowTable
-    };
-}
-/**
- * Convert an Apache Arrow table to a GeoJSONTable
- * @note Currently does not convert schema
- */
-export function convertArrowToGeoJSONTable(table) {
-    const arrowTable = table.data;
-    const schema = serializeArrowSchema(arrowTable.schema);
-    const geometryColumns = getGeometryColumnsFromSchema(schema);
-    // get encoding from geometryColumns['geometry']
-    const encoding = geometryColumns.geometry.encoding;
-    const features = [];
-    // Remove geometry columns
-    const propertyColumnNames = arrowTable.schema.fields
-        .map((field) => field.name)
-        // TODO - this deletes all geometry columns
-        .filter((name) => !(name in geometryColumns));
-    const propertiesTable = arrowTable.select(propertyColumnNames);
-    const arrowGeometryColumn = arrowTable.getChild('geometry');
-    for (let row = 0; row < arrowTable.numRows; row++) {
-        // get the geometry value from arrow geometry column
-        // Note that type can vary
-        const arrowGeometry = arrowGeometryColumn?.get(row);
-        // parse arrow geometry to geojson feature
-        const feature = parseGeometryFromArrow(arrowGeometry, encoding);
-        if (feature) {
-            const properties = propertiesTable.get(row)?.toJSON() || {};
-            features.push({ type: 'Feature', geometry: feature, properties });
-        }
-    }
-    return {
-        shape: 'geojson-table',
-        type: 'FeatureCollection',
-        schema: table.schema,
-        features
-    };
-}

package/dist/tables/convert-columnar-to-row-table.d.ts

@@ -1,7 +0,0 @@
-import type { ColumnarTable, ObjectRowTable } from '@loaders.gl/schema';
-/**
- *
- * @note - should be part of schema module
- */
-export declare function convertColumnarToRowFormatTable(columnarTable: ColumnarTable): ObjectRowTable;
-//# sourceMappingURL=convert-columnar-to-row-table.d.ts.map

package/dist/tables/convert-columnar-to-row-table.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"convert-columnar-to-row-table.d.ts","sourceRoot":"","sources":["../../src/tables/convert-columnar-to-row-table.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,aAAa,EAAE,cAAc,EAAC,MAAM,oBAAoB,CAAC;AAEtE;;;GAGG;AACH,wBAAgB,+BAA+B,CAAC,aAAa,EAAE,aAAa,GAAG,cAAc,CAoB5F"}

package/dist/tables/convert-columnar-to-row-table.js

@@ -1,25 +0,0 @@
-// loaders.gl
-// SPDX-License-Identifier: MIT
-// Copyright (c) vis.gl contributors
-/**
- *
- * @note - should be part of schema module
- */
-export function convertColumnarToRowFormatTable(columnarTable) {
-    const tableKeys = Object.keys(columnarTable);
-    const tableRowsCount = columnarTable[tableKeys[0]].length;
-    const rowFormatTable = [];
-    for (let index = 0; index < tableRowsCount; index++) {
-        const tableItem = {};
-        for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {
-            const fieldName = tableKeys[keyIndex];
-            tableItem[fieldName] = columnarTable[fieldName][index];
-        }
-        rowFormatTable.push(tableItem);
-    }
-    return {
-        shape: 'object-row-table',
-        schema: columnarTable.schema,
-        data: rowFormatTable
-    };
-}

package/dist/tables/convert-table-to-arrow.d.ts

@@ -1,42 +0,0 @@
-/**
- * * Convert a loaders.gl Mesh to an Apache Arrow Table
- * @param mesh
- * @param metadata
- * @param batchSize
- * @returns
- *
-export function convertTableToArrow(table: Table, options?: {batchSize?: number}): ArrowTable {
-  const vectors: AbstractVector[] = [];
-  const fields: Field[] = [];
-
-  const length = getTableLength(table);
-  const batchSize = options?.batchSize || length;
-
-  const recordBatches: RecordBatch[] = [];
-  for (let i = 0; i < length; i += batchSize) {
-    for (let columnIndex = 0; columnIndex < getTableNumCols(table); ++columnIndex) {
-      const field_ = table.schema.fields[columnIndex];
-      const column = getTableColumnAt(table, columnIndex, i, batchSize);
-      const type = getArrowType(column);
-      const vector = getArrowVector(column);
-      const listType = new FixedSizeList(size, new Field('value', type));
-      const field = new Field(field_.name, listType, false, makeMeshAttributeMetadata(attribute));
-      const data = new Data(listType, 0, value.length / size, 0, undefined, [vector]);
-      const listVector = new FixedSizeListVector(data);
-      vectors.push(listVector);
-      fields.push(field);
-    }
-
-  for (const attributeKey in table.columns) {
-    const attribute = mesh.attributes[attributeKey];
-    const {value, size = 1} = attribute;
-  }
-
-  const schema = new ApacheArrowSchema(fields, table?.schema?.metadata || new Map<string, string>());
-  const recordBatch = new RecordBatch(schema, vectors[0].length, vectors);
-  const apacheArrowTable = new ApacheArrowTable(schema, recordBatch);
-
-  return apacheArrowTable;
-}
-*/
-//# sourceMappingURL=convert-table-to-arrow.d.ts.map

package/dist/tables/convert-table-to-arrow.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"convert-table-to-arrow.d.ts","sourceRoot":"","sources":["../../src/tables/convert-table-to-arrow.ts"],"names":[],"mappings":"AAoBA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAwCE"}

package/dist/tables/convert-table-to-arrow.js

@@ -1,59 +0,0 @@
-"use strict";
-// loaders.gl
-// SPDX-License-Identifier: MIT
-// Copyright (c) vis.gl contributors
-// import {
-//   Table as ApacheArrowTable,
-//   Schema as ApacheArrowSchema,
-//   RecordBatch,
-//   FixedSizeList,
-//   Field,
-//   Data,
-//   FixedSizeListVector
-// } from 'apache-arrow/Arrow.dom';
-// import {AbstractVector} from 'apache-arrow/vector';
-// import {Table} from '../../types/category-table';
-// import {getArrowType, getArrowVector} from './arrow-type-utils';
-// import {makeMeshAttributeMetadata} from './deduce-mesh-schema';
-// import {getTableLength, getTableNumCols} from './table-accessors';
-/**
- * * Convert a loaders.gl Mesh to an Apache Arrow Table
- * @param mesh
- * @param metadata
- * @param batchSize
- * @returns
- *
-export function convertTableToArrow(table: Table, options?: {batchSize?: number}): ArrowTable {
-  const vectors: AbstractVector[] = [];
-  const fields: Field[] = [];
-
-  const length = getTableLength(table);
-  const batchSize = options?.batchSize || length;
-
-  const recordBatches: RecordBatch[] = [];
-  for (let i = 0; i < length; i += batchSize) {
-    for (let columnIndex = 0; columnIndex < getTableNumCols(table); ++columnIndex) {
-      const field_ = table.schema.fields[columnIndex];
-      const column = getTableColumnAt(table, columnIndex, i, batchSize);
-      const type = getArrowType(column);
-      const vector = getArrowVector(column);
-      const listType = new FixedSizeList(size, new Field('value', type));
-      const field = new Field(field_.name, listType, false, makeMeshAttributeMetadata(attribute));
-      const data = new Data(listType, 0, value.length / size, 0, undefined, [vector]);
-      const listVector = new FixedSizeListVector(data);
-      vectors.push(listVector);
-      fields.push(field);
-    }
-
-  for (const attributeKey in table.columns) {
-    const attribute = mesh.attributes[attributeKey];
-    const {value, size = 1} = attribute;
-  }
-
-  const schema = new ApacheArrowSchema(fields, table?.schema?.metadata || new Map<string, string>());
-  const recordBatch = new RecordBatch(schema, vectors[0].length, vectors);
-  const apacheArrowTable = new ApacheArrowTable(schema, recordBatch);
-
-  return apacheArrowTable;
-}
-*/

package/dist/types.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAIA,KAAK,aAAa,GAAG,SAAS,GAAG,UAAU,GAAG,UAAU,GAAG,WAAW,GAAG,UAAU,GAAG,WAAW,CAAC;AAElG,KAAK,eAAe,GAAG,YAAY,GAAG,YAAY,CAAC;AAEnD,KAAK,UAAU,GAAG,aAAa,GAAG,eAAe,CAAC;AAElD,MAAM,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC;AAEnD,oBAAY,YAAY;IACtB,KAAK,IAAA;IACL,IAAI,IAAA;CACL"}

package/src/parsers/parse-arrow-sync.ts

@@ -1,44 +0,0 @@
-// loaders.gl
-// SPDX-License-Identifier: MIT
-// Copyright (c) vis.gl contributors
-
-import type {ArrayRowTable, ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';
-import type {ArrowTable} from '../lib/arrow-table';
-import {convertTable} from '@loaders.gl/schema';
-import * as arrow from 'apache-arrow';
-import {convertArrowToColumnarTable} from '../tables/convert-arrow-to-columnar-table';
-import {serializeArrowSchema} from '../schema/convert-arrow-schema';
-
-// Parses arrow to a columnar table
-export function parseArrowSync(
-  arrayBuffer,
-  options?: {shape?: 'arrow-table' | 'columnar-table' | 'object-row-table' | 'array-row-table'}
-): ArrowTable | ColumnarTable | ObjectRowTable | ArrayRowTable {
-  const apacheArrowTable = arrow.tableFromIPC([new Uint8Array(arrayBuffer)]);
-  const arrowTable: ArrowTable = {
-    shape: 'arrow-table',
-    schema: serializeArrowSchema(apacheArrowTable.schema),
-    data: apacheArrowTable
-  };
-
-  const shape = options?.shape || 'arrow-table';
-  switch (shape) {
-    case 'arrow-table':
-      return arrowTable;
-
-    case 'columnar-table':
-      return convertArrowToColumnarTable(arrowTable);
-
-    case 'object-row-table':
-      let columnarTable = convertArrowToColumnarTable(arrowTable);
-      return convertTable(columnarTable, 'object-row-table');
-
-    case 'array-row-table':
-      columnarTable = convertArrowToColumnarTable(arrowTable);
-      return convertTable(columnarTable, 'array-row-table');
-
-    default:
-      // TODO
-      throw new Error(shape);
-  }
-}

package/src/parsers/parse-geoarrow-in-batches.ts

@@ -1,16 +0,0 @@
-// loaders.gl
-// SPDX-License-Identifier: MIT
-// Copyright (c) vis.gl contributors
-
-import type {GeoJSONTableBatch} from '@loaders.gl/schema';
-import type {ArrowTableBatch} from '../lib/arrow-table';
-import {parseArrowInBatches} from './parse-arrow-in-batches';
-
-/**
- */
-export function parseGeoArrowInBatches(
-  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>
-): AsyncIterable<ArrowTableBatch | GeoJSONTableBatch> {
-  // | BinaryGeometry
-  return parseArrowInBatches(asyncIterator);
-}

package/src/parsers/parse-geoarrow-sync.ts

@@ -1,23 +0,0 @@
-// loaders.gl
-// SPDX-License-Identifier: MIT
-// Copyright (c) vis.gl contributors
-
-import type {GeoJSONTable} from '@loaders.gl/schema';
-import type {ArrowTable} from '../lib/arrow-table';
-import {parseArrowSync} from './parse-arrow-sync';
-import {convertArrowToGeoJSONTable} from '../tables/convert-arrow-to-geojson-table';
-
-// Parses arrow to a columnar table
-export function parseGeoArrowSync(
-  arrayBuffer,
-  options?: {shape?: 'arrow-table' | 'geojson-table' | 'binary-geometry'}
-): ArrowTable | GeoJSONTable {
-  // | BinaryGeometry
-  const table = parseArrowSync(arrayBuffer, {shape: 'arrow-table'}) as ArrowTable;
-  switch (options?.shape) {
-    case 'geojson-table':
-      return convertArrowToGeoJSONTable(table);
-    default:
-      return table;
-  }
-}

package/src/schema/arrow-type-utils.ts

@@ -1,55 +0,0 @@
-// loaders.gl
-// SPDX-License-Identifier: MIT
-// Copyright (c) vis.gl contributors
-
-import type {TypedArray} from '@loaders.gl/schema';
-import * as arrow from 'apache-arrow';
-
-/** Return an Apache Arrow Type instance that corresponds to the type of the elements in the supplied Typed Array */
-export function getArrowType(array: TypedArray): arrow.DataType {
-  switch (array.constructor) {
-    case Int8Array:
-      return new arrow.Int8();
-    case Uint8Array:
-      return new arrow.Uint8();
-    case Int16Array:
-      return new arrow.Int16();
-    case Uint16Array:
-      return new arrow.Uint16();
-    case Int32Array:
-      return new arrow.Int32();
-    case Uint32Array:
-      return new arrow.Uint32();
-    case Float32Array:
-      return new arrow.Float32();
-    case Float64Array:
-      return new arrow.Float64();
-    default:
-      throw new Error('array type not supported');
-  }
-}
-
-/*
-export function getArrowVector(array: TypedArray): AbstractVector {
-  switch (array.constructor) {
-    case Int8Array:
-      return Int8Vector.from(array);
-    case Uint8Array:
-      return Uint8Vector.from(array);
-    case Int16Array:
-      return Int16Vector.from(array);
-    case Uint16Array:
-      return Uint16Vector.from(array);
-    case Int32Array:
-      return Int32Vector.from(array);
-    case Uint32Array:
-      return Uint32Vector.from(array);
-    case Float32Array:
-      return Float32Vector.from(array);
-    case Float64Array:
-      return Float64Vector.from(array);
-    default:
-      throw new Error('array type not supported');
-  }
-}
-*/

package/src/tables/convert-arrow-to-columnar-table.ts

@@ -1,31 +0,0 @@
-// loaders.gl
-// SPDX-License-Identifier: MIT
-// Copyright (c) vis.gl contributors
-
-import type {ColumnarTable} from '@loaders.gl/schema';
-import type {ArrowTable} from '../lib/arrow-table';
-
-/**
- * Convert an Apache Arrow table to a ColumnarTable
- * @note Currently does not convert schema
- */
-export function convertArrowToColumnarTable(table: ArrowTable): ColumnarTable {
-  // TODO - avoid calling `getColumn` on columns we are not interested in?
-  // Add options object?
-
-  const arrowTable = table.data;
-  const columnarTable = {};
-
-  for (const field of arrowTable.schema.fields) {
-    // This (is intended to) coalesce all record batches into a single typed array
-    const arrowColumn = arrowTable.getChild(field.name);
-    const values = arrowColumn?.toArray();
-    columnarTable[field.name] = values;
-  }
-
-  return {
-    shape: 'columnar-table',
-    schema: table.schema,
-    data: columnarTable
-  };
-}

package/src/tables/convert-arrow-to-geojson-table.ts

@@ -1,66 +0,0 @@
-// loaders.gl
-// SPDX-License-Identifier: MIT
-// Copyright (c) vis.gl contributors
-
-import type {Feature, GeoJSONTable} from '@loaders.gl/schema';
-import * as arrow from 'apache-arrow';
-import type {ArrowTable} from '../lib/arrow-table';
-import {serializeArrowSchema, parseGeometryFromArrow} from '@loaders.gl/arrow';
-import {getGeometryColumnsFromSchema} from '@loaders.gl/gis';
-
-/**
- * Wrap an apache arrow table in a loaders.gl table wrapper.
- * From this additional conversions are available.
- * @param arrowTable
- * @returns
- */
-export function convertApacheArrowToArrowTable(arrowTable: arrow.Table): ArrowTable {
-  return {
-    shape: 'arrow-table',
-    schema: serializeArrowSchema(arrowTable.schema),
-    data: arrowTable
-  };
-}
-
-/**
- * Convert an Apache Arrow table to a GeoJSONTable
- * @note Currently does not convert schema
- */
-export function convertArrowToGeoJSONTable(table: ArrowTable): GeoJSONTable {
-  const arrowTable = table.data;
-  const schema = serializeArrowSchema(arrowTable.schema);
-  const geometryColumns = getGeometryColumnsFromSchema(schema);
-
-  // get encoding from geometryColumns['geometry']
-  const encoding = geometryColumns.geometry.encoding;
-
-  const features: Feature[] = [];
-
-  // Remove geometry columns
-  const propertyColumnNames = arrowTable.schema.fields
-    .map((field) => field.name)
-    // TODO - this deletes all geometry columns
-    .filter((name) => !(name in geometryColumns));
-  const propertiesTable = arrowTable.select(propertyColumnNames);
-
-  const arrowGeometryColumn = arrowTable.getChild('geometry');
-
-  for (let row = 0; row < arrowTable.numRows; row++) {
-    // get the geometry value from arrow geometry column
-    // Note that type can vary
-    const arrowGeometry = arrowGeometryColumn?.get(row);
-    // parse arrow geometry to geojson feature
-    const feature = parseGeometryFromArrow(arrowGeometry, encoding);
-    if (feature) {
-      const properties = propertiesTable.get(row)?.toJSON() || {};
-      features.push({type: 'Feature', geometry: feature, properties});
-    }
-  }
-
-  return {
-    shape: 'geojson-table',
-    type: 'FeatureCollection',
-    schema: table.schema,
-    features
-  };
-}

package/src/tables/convert-columnar-to-row-table.ts

@@ -1,31 +0,0 @@
-// loaders.gl
-// SPDX-License-Identifier: MIT
-// Copyright (c) vis.gl contributors
-
-import type {ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';
-
-/**
- *
- * @note - should be part of schema module
- */
-export function convertColumnarToRowFormatTable(columnarTable: ColumnarTable): ObjectRowTable {
-  const tableKeys = Object.keys(columnarTable);
-  const tableRowsCount = columnarTable[tableKeys[0]].length;
-
-  const rowFormatTable: {}[] = [];
-
-  for (let index = 0; index < tableRowsCount; index++) {
-    const tableItem = {};
-    for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {
-      const fieldName = tableKeys[keyIndex];
-      tableItem[fieldName] = columnarTable[fieldName][index];
-    }
-    rowFormatTable.push(tableItem);
-  }
-
-  return {
-    shape: 'object-row-table',
-    schema: columnarTable.schema,
-    data: rowFormatTable
-  };
-}

package/src/tables/convert-table-to-arrow.ts

@@ -1,61 +0,0 @@
-// loaders.gl
-// SPDX-License-Identifier: MIT
-// Copyright (c) vis.gl contributors
-
-// import {
-//   Table as ApacheArrowTable,
-//   Schema as ApacheArrowSchema,
-//   RecordBatch,
-//   FixedSizeList,
-//   Field,
-//   Data,
-//   FixedSizeListVector
-// } from 'apache-arrow/Arrow.dom';
-// import {AbstractVector} from 'apache-arrow/vector';
-
-// import {Table} from '../../types/category-table';
-// import {getArrowType, getArrowVector} from './arrow-type-utils';
-// import {makeMeshAttributeMetadata} from './deduce-mesh-schema';
-// import {getTableLength, getTableNumCols} from './table-accessors';
-
-/**
- * * Convert a loaders.gl Mesh to an Apache Arrow Table
- * @param mesh
- * @param metadata
- * @param batchSize
- * @returns
- *
-export function convertTableToArrow(table: Table, options?: {batchSize?: number}): ArrowTable {
-  const vectors: AbstractVector[] = [];
-  const fields: Field[] = [];
-
-  const length = getTableLength(table);
-  const batchSize = options?.batchSize || length;
-
-  const recordBatches: RecordBatch[] = [];
-  for (let i = 0; i < length; i += batchSize) {
-    for (let columnIndex = 0; columnIndex < getTableNumCols(table); ++columnIndex) {
-      const field_ = table.schema.fields[columnIndex];
-      const column = getTableColumnAt(table, columnIndex, i, batchSize);
-      const type = getArrowType(column);
-      const vector = getArrowVector(column);
-      const listType = new FixedSizeList(size, new Field('value', type));
-      const field = new Field(field_.name, listType, false, makeMeshAttributeMetadata(attribute));
-      const data = new Data(listType, 0, value.length / size, 0, undefined, [vector]);
-      const listVector = new FixedSizeListVector(data);
-      vectors.push(listVector);
-      fields.push(field);
-    }
-
-  for (const attributeKey in table.columns) {
-    const attribute = mesh.attributes[attributeKey];
-    const {value, size = 1} = attribute;
-  }
-
-  const schema = new ApacheArrowSchema(fields, table?.schema?.metadata || new Map<string, string>());
-  const recordBatch = new RecordBatch(schema, vectors[0].length, vectors);
-  const apacheArrowTable = new ApacheArrowTable(schema, recordBatch);
-
-  return apacheArrowTable;
-}
-*/