@loaders.gl/arrow 4.3.2 → 4.4.0-alpha.2

package/dist/workers/triangulation-worker.js

@@ -3,7 +3,7 @@
 // Copyright (c) vis.gl contributors
 import * as arrow from 'apache-arrow';
 import { createWorker } from '@loaders.gl/worker-utils';
-import { getTriangleIndices, getBinaryGeometriesFromArrow } from "../lib/geoarrow/convert-geoarrow-to-binary-geometry.js";
+import { getTriangleIndices, convertGeoArrowToBinaryFeatureCollection } from '@loaders.gl/gis';
 createWorker(async (data, options = {}) => {
     const input = data;
     const operation = input?.operation;
@@ -44,7 +44,7 @@ function parseGeoArrowBatch(data) {
     if (geometryColumn) {
         // NOTE: for a rebuild arrow.Vector, there is only one chunk, so chunkIndex is always 0
         const options = { calculateMeanCenters, triangle, chunkIndex: 0, chunkOffset };
-        binaryDataFromGeoArrow = getBinaryGeometriesFromArrow(geometryColumn, geometryEncoding, options);
+        binaryDataFromGeoArrow = convertGeoArrowToBinaryFeatureCollection(geometryColumn, geometryEncoding, options);
         // NOTE: here binaryGeometry will be copied to main thread
         return {
             binaryDataFromGeoArrow,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/arrow",
-  "version": "4.3.2",
+  "version": "4.4.0-alpha.2",
   "description": "Simple columnar table loader for the Apache Arrow format",
   "license": "MIT",
   "type": "module",
@@ -57,16 +57,17 @@
     "build-worker2": "esbuild src/workers/arrow-worker.ts --bundle --outfile=dist/arrow-worker.js --platform=browser --external:{stream}"
   },
   "dependencies": {
-    "@loaders.gl/gis": "4.3.2",
-    "@loaders.gl/loader-utils": "4.3.2",
-    "@loaders.gl/schema": "4.3.2",
-    "@loaders.gl/wkt": "4.3.2",
-    "@loaders.gl/worker-utils": "4.3.2",
+    "@loaders.gl/gis": "4.4.0-alpha.2",
+    "@loaders.gl/loader-utils": "4.4.0-alpha.2",
+    "@loaders.gl/schema": "4.4.0-alpha.2",
+    "@loaders.gl/schema-utils": "4.4.0-alpha.2",
+    "@loaders.gl/wkt": "4.4.0-alpha.2",
+    "@loaders.gl/worker-utils": "4.4.0-alpha.2",
     "@math.gl/polygon": "^4.1.0",
-    "apache-arrow": ">= 15.0.0"
+    "apache-arrow": ">= 16.1.0"
   },
   "peerDependencies": {
-    "@loaders.gl/core": "^4.3.0"
+    "@loaders.gl/core": "4.4.0-alpha.1"
   },
-  "gitHead": "b4203b8703f64a38d6f79a3113bc7bb51d45c93a"
+  "gitHead": "3d9fed050eabdc0812ddf2f4d5fb9914a34ee0c2"
 }

package/src/exports/arrow-format.ts

@@ -0,0 +1,21 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Format} from '@loaders.gl/loader-utils';
+
+/** ArrowJS table loader */
+export const ArrowFormat = {
+  name: 'Apache Arrow',
+  id: 'arrow',
+  module: 'arrow',
+  category: 'table',
+  extensions: ['arrow', 'feather'],
+  mimeTypes: [
+    'application/vnd.apache.arrow.file',
+    'application/vnd.apache.arrow.stream',
+    'application/octet-stream'
+  ],
+  binary: true,
+  tests: ['ARROW']
+} as const satisfies Format;

package/src/exports/arrow-loader.ts

@@ -3,7 +3,7 @@
 // Copyright (c) vis.gl contributors
 
 import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';
-import type {ArrowTable} from '../schema/arrow-table-type';
+import type {ArrowTable} from '@loaders.gl/schema';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.

package/src/exports/geoarrow-loader.ts

@@ -3,9 +3,8 @@
 // Copyright (c) vis.gl contributors
 
 import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';
+import type {ArrowTable, ArrowTableBatch, BinaryGeometry} from '@loaders.gl/schema';
 import {ArrowWorkerLoader} from './arrow-loader';
-import type {BinaryGeometry} from '@loaders.gl/schema';
-import type {ArrowTable, ArrowTableBatch} from '../schema/arrow-table-type';
 
 export type GeoArrowLoaderOptions = LoaderOptions & {
   arrow?: {

package/src/geoarrow-loader.ts

@@ -3,8 +3,12 @@
 // Copyright (c) vis.gl contributors
 
 import type {LoaderWithParser} from '@loaders.gl/loader-utils';
-import type {GeoJSONTable, GeoJSONTableBatch} from '@loaders.gl/schema';
-import type {ArrowTable, ArrowTableBatch} from './schema/arrow-table-type';
+import type {
+  ArrowTable,
+  ArrowTableBatch,
+  GeoJSONTable,
+  GeoJSONTableBatch
+} from '@loaders.gl/schema';
 import {parseGeoArrowSync, parseGeoArrowInBatches} from './lib/parsers/parse-geoarrow';
 import type {GeoArrowLoaderOptions} from './exports/geoarrow-loader';
 import {GeoArrowWorkerLoader} from './exports/geoarrow-loader';

package/src/geoarrow-writer.ts

@@ -1,7 +1,7 @@
 // import type {} from '@loaders.gl/loader-utils';
 
 import type {WriterWithEncoder, WriterOptions} from '@loaders.gl/loader-utils';
-import {GeoJSONTable, BinaryGeometry} from '@loaders.gl/schema';
+import type {GeoJSONTable, BinaryGeometry} from '@loaders.gl/schema';
 import {encodeGeoArrowSync} from './lib/encoders/encode-geoarrow';
 
 // __VERSION__ is injected by babel-plugin-version-inline

package/src/index.ts

@@ -3,16 +3,12 @@
 // Copyright (c) vis.gl contributors
 
 // Types
-export type {ArrowTable, ArrowTableBatch} from './schema/arrow-table-type';
 export {VECTOR_TYPES} from './lib/types';
-import {ArrowTableBatchAggregator} from './schema/arrow-table-batch';
-
-// Make the ArrowBatch type available
-import {TableBatchBuilder} from '@loaders.gl/schema';
-TableBatchBuilder.ArrowBatch = ArrowTableBatchAggregator;
 
 // Arrow loader / Writer
 
+export {ArrowFormat} from './exports/arrow-format';
+
 export type {ArrowLoaderOptions} from './exports/arrow-loader';
 export {ArrowWorkerLoader} from './exports/arrow-loader';
 export {ArrowLoader} from './arrow-loader';
@@ -23,48 +19,12 @@ export {ArrowWriter} from './arrow-writer';
 export {GeoArrowWorkerLoader} from './exports/geoarrow-loader';
 export {GeoArrowLoader} from './geoarrow-loader';
 
-// Schema utils
-export {
-  convertArrowToSchema,
-  convertSchemaToArrow,
-
-  // DETAILED FUNCTIONS
-  serializeArrowSchema,
-  deserializeArrowSchema,
-  serializeArrowMetadata,
-  deserializeArrowMetadata,
-  serializeArrowField,
-  deserializeArrowField,
-  serializeArrowType,
-  deserializeArrowType
-} from './lib//tables/convert-arrow-schema';
-
-// Table utils
-export {convertArrowToTable} from './lib/tables/convert-arrow-to-table';
-export {convertTableToArrow} from './lib/tables/convert-table-to-arrow';
-
 // EXPERIMENTAL
 
 // Arrow Utils
-export type {GeoArrowEncoding} from '@loaders.gl/gis';
 // getGeometryColumnsFromArrowTable,
 // getGeoArrowEncoding
 
-export type {
-  BinaryDataFromGeoArrow,
-  BinaryGeometriesFromArrowOptions
-} from './lib/geoarrow/convert-geoarrow-to-binary-geometry';
-export {
-  getBinaryGeometryTemplate,
-  getBinaryGeometriesFromArrow,
-  getTriangleIndices,
-  getMeanCentersFromBinaryGeometries
-} from './lib/geoarrow/convert-geoarrow-to-binary-geometry';
-
-export {updateBoundsFromGeoArrowSamples} from './lib/geoarrow/get-arrow-bounds';
-
-export {parseGeometryFromArrow} from './lib/geoarrow/convert-geoarrow-to-geojson-geometry';
-
 // EXPERIMENTAL WORKER
 export {hardClone} from './workers/hard-clone';
 

package/src/lib/parsers/parse-arrow.ts

@@ -3,10 +3,9 @@
 // Copyright (c) vis.gl contributors
 
 import * as arrow from 'apache-arrow';
-import type {Table} from '@loaders.gl/schema';
-import type {ArrowTableBatch} from '../../schema/arrow-table-type';
+import type {Table, ArrowTableBatch} from '@loaders.gl/schema';
 import {ArrowLoaderOptions} from '../../exports/arrow-loader';
-import {convertArrowToTable} from '../tables/convert-arrow-to-table';
+import {convertArrowToTable} from '@loaders.gl/schema-utils';
 
 /** Parses arrow to a loaders.gl table. Defaults to `arrow-table` */
 export function parseArrowSync(arrayBuffer, options?: {shape?: Table['shape']}): Table {

package/src/lib/parsers/parse-geoarrow.ts

@@ -2,10 +2,14 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import type {GeoJSONTable, GeoJSONTableBatch} from '@loaders.gl/schema';
-import type {ArrowTable, ArrowTableBatch} from '../../schema/arrow-table-type';
+import type {
+  GeoJSONTable,
+  GeoJSONTableBatch,
+  ArrowTable,
+  ArrowTableBatch
+} from '@loaders.gl/schema';
 import {parseArrowSync, parseArrowInBatches} from './parse-arrow';
-import {convertArrowToTable} from '../tables/convert-arrow-to-table';
+import {convertGeoArrowToTable} from '@loaders.gl/gis';
 
 // Parses arrow to a columnar table
 export function parseGeoArrowSync(
@@ -16,7 +20,7 @@ export function parseGeoArrowSync(
   const table = parseArrowSync(arrayBuffer, {shape: 'arrow-table'}) as ArrowTable;
   switch (options?.shape) {
     case 'geojson-table':
-      return convertArrowToTable(table.data, 'geojson-table');
+      return convertGeoArrowToTable(table.data, 'geojson-table');
     default:
       return table;
   }

package/src/triangulate-on-worker.ts

@@ -5,7 +5,8 @@
 import * as arrow from 'apache-arrow';
 import type {WorkerOptions} from '@loaders.gl/worker-utils';
 import {processOnWorker} from '@loaders.gl/worker-utils';
-import {BinaryDataFromGeoArrow, GeoArrowEncoding} from '@loaders.gl/arrow';
+import type {GeoArrowEncoding} from '@loaders.gl/geoarrow';
+import {BinaryDataFromGeoArrow} from '@loaders.gl/gis';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.

package/src/workers/triangulation-worker.ts

@@ -6,9 +6,9 @@ import * as arrow from 'apache-arrow';
 import {createWorker} from '@loaders.gl/worker-utils';
 import {
   getTriangleIndices,
-  getBinaryGeometriesFromArrow,
+  convertGeoArrowToBinaryFeatureCollection,
   BinaryDataFromGeoArrow
-} from '../lib/geoarrow/convert-geoarrow-to-binary-geometry';
+} from '@loaders.gl/gis';
 import type {
   TriangulationWorkerInput,
   TriangulateInput,
@@ -75,7 +75,7 @@ function parseGeoArrowBatch(data: ParseGeoArrowInput): ParseGeoArrowResult {
   if (geometryColumn) {
     // NOTE: for a rebuild arrow.Vector, there is only one chunk, so chunkIndex is always 0
     const options = {calculateMeanCenters, triangle, chunkIndex: 0, chunkOffset};
-    binaryDataFromGeoArrow = getBinaryGeometriesFromArrow(
+    binaryDataFromGeoArrow = convertGeoArrowToBinaryFeatureCollection(
       geometryColumn,
       geometryEncoding,
       options

package/dist/lib/geoarrow/convert-geoarrow-to-binary-geometry.d.ts

@@ -1,74 +0,0 @@
-import * as arrow from 'apache-arrow';
-import { BinaryFeatureCollection as BinaryFeatures } from '@loaders.gl/schema';
-import { GeoArrowEncoding } from '@loaders.gl/gis';
-/**
- * Binary data from geoarrow column and can be used by e.g. deck.gl GeojsonLayer
- */
-export type BinaryDataFromGeoArrow = {
-    /** Binary format geometries, an array of BinaryFeatureCollection */
-    binaryGeometries: BinaryFeatures[];
-    /** Boundary of the binary geometries */
-    bounds: [number, number, number, number];
-    /** Feature types of the binary geometries */
-    featureTypes: {
-        polygon: boolean;
-        point: boolean;
-        line: boolean;
-    };
-    /** (Optional) mean centers of the binary geometries for e.g. polygon filtering */
-    meanCenters?: number[][];
-};
-/**
- * binary geometry template, see deck.gl BinaryGeometry
- */
-export declare function getBinaryGeometryTemplate(): {
-    globalFeatureIds: {
-        value: Uint32Array;
-        size: number;
-    };
-    positions: {
-        value: Float32Array;
-        size: number;
-    };
-    properties: never[];
-    numericProps: {};
-    featureIds: {
-        value: Uint32Array;
-        size: number;
-    };
-};
-export type BinaryGeometriesFromArrowOptions = {
-    /** option to specify which chunk to get binary geometries from, for progressive rendering */
-    chunkIndex?: number;
-    /** The offset (beginning index of rows) of input chunk. Used for reconstructing globalFeatureIds in web workers */
-    chunkOffset?: number;
-    /** option to get mean centers from geometries, for polygon filtering */
-    calculateMeanCenters?: boolean;
-    /** option to compute the triangle indices by tesselating polygons */
-    triangulate?: boolean;
-};
-/**
- * get binary geometries from geoarrow column
- *
- * @param geoColumn the geoarrow column, e.g. arrowTable.getChildAt(geoColumnIndex)
- * @param geoEncoding the geo encoding of the geoarrow column, e.g. getGeoArrowEncoding(arrowTable.schema, geoColumnName)
- * @param options options for getting binary geometries {meanCenter: boolean}
- * @returns BinaryDataFromGeoArrow
- */
-export declare function getBinaryGeometriesFromArrow(geoColumn: arrow.Vector, geoEncoding: GeoArrowEncoding, options?: BinaryGeometriesFromArrowOptions): BinaryDataFromGeoArrow;
-/**
- * Get mean centers from binary geometries
- * @param binaryGeometries binary geometries from geoarrow column, an array of BinaryFeatureCollection
- * @returns mean centers of the binary geometries
- */
-export declare function getMeanCentersFromBinaryGeometries(binaryGeometries: BinaryFeatures[]): number[][];
-/**
- * get triangle indices. Allows deck.gl to skip performing costly triangulation on main thread.
- * @param polygonIndices Indices within positions of the start of each simple Polygon
- * @param primitivePolygonIndices Indices within positions of the start of each primitive Polygon/ring
- * @param flatCoordinateArray Array of x, y or x, y, z positions
- * @param nDim - number of dimensions per position
- * @returns triangle indices or null if invalid polygon and earcut fails
- */
-export declare function getTriangleIndices(polygonIndices: Uint16Array, primitivePolygonIndices: Int32Array, flatCoordinateArray: Float64Array, nDim: number): Uint32Array | null;
-//# sourceMappingURL=convert-geoarrow-to-binary-geometry.d.ts.map

package/dist/lib/geoarrow/convert-geoarrow-to-binary-geometry.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"convert-geoarrow-to-binary-geometry.d.ts","sourceRoot":"","sources":["../../../src/lib/geoarrow/convert-geoarrow-to-binary-geometry.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,KAAK,MAAM,cAAc,CAAC;AAEtC,OAAO,EAAC,uBAAuB,IAAI,cAAc,EAAC,MAAM,oBAAoB,CAAC;AAC7E,OAAO,EAAC,gBAAgB,EAAC,MAAM,iBAAiB,CAAC;AAajD;;GAEG;AACH,MAAM,MAAM,sBAAsB,GAAG;IACnC,oEAAoE;IACpE,gBAAgB,EAAE,cAAc,EAAE,CAAC;IACnC,wCAAwC;IACxC,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACzC,6CAA6C;IAC7C,YAAY,EAAE;QAAC,OAAO,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,OAAO,CAAC;QAAC,IAAI,EAAE,OAAO,CAAA;KAAC,CAAC;IAChE,kFAAkF;IAClF,WAAW,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC;CAC1B,CAAC;AAsBF;;GAEG;AACH,wBAAgB,yBAAyB;;;;;;;;;;;;;;;EAQxC;AAED,MAAM,MAAM,gCAAgC,GAAG;IAC7C,6FAA6F;IAC7F,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,mHAAmH;IACnH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,wEAAwE;IACxE,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,qEAAqE;IACrE,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB,CAAC;AAEF;;;;;;;GAOG;AACH,wBAAgB,4BAA4B,CAC1C,SAAS,EAAE,KAAK,CAAC,MAAM,EACvB,WAAW,EAAE,gBAAgB,EAC7B,OAAO,CAAC,EAAE,gCAAgC,GACzC,sBAAsB,CAiFxB;AAED;;;;GAIG;AACH,wBAAgB,kCAAkC,CAAC,gBAAgB,EAAE,cAAc,EAAE,GAAG,MAAM,EAAE,EAAE,CAiCjG;AA2ED;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAChC,cAAc,EAAE,WAAW,EAC3B,uBAAuB,EAAE,UAAU,EACnC,mBAAmB,EAAE,YAAY,EACjC,IAAI,EAAE,MAAM,GACX,WAAW,GAAG,IAAI,CAyCpB"}