kepler.gl 3.1.0 → 3.1.1

package/src/layers/src/index.ts

@@ -53,8 +53,7 @@ export type {
   LayerBaseConfig,
   VisualChannelDomain,
   VisualChannel,
-  VisualChannelDescription,
-  FindDefaultLayerPropsReturnValue
+  VisualChannelDescription
 } from './base-layer';
 export * from './base-layer';
 

package/src/layers/src/layer-text-label.ts

@@ -69,7 +69,9 @@ export const formatTextLabelData = ({
 
     const getTextAccessor: (d: {index: number}) => string = textLabelAccessor(tl)(dataContainer);
     let characterSet;
+    let getText: typeof getTextAccessor | arrow.Vector = getTextAccessor;
 
+    let rebuildArrowTextVector = true;
     if (
       !triggerChanged?.[`getLabelCharacterSet-${i}`] &&
       oldLayerData &&
@@ -77,6 +79,8 @@ export const formatTextLabelData = ({
       oldLayerData.textLabels[i]
     ) {
       characterSet = oldLayerData.textLabels[i].characterSet;
+      getText = oldLayerData.textLabels[i].getText;
+      rebuildArrowTextVector = false;
     } else {
       if (data instanceof arrow.Table) {
         // we don't filter out arrow tables,
@@ -100,13 +104,20 @@ export const formatTextLabelData = ({
       }
     }
 
-    let getText: typeof getTextAccessor | arrow.Vector = getTextAccessor;
     // For Arrow Layers getText has to be an arrow vector.
     // For now check here for ArrowTable, not ArrowDataContainer.
-    if (data instanceof arrow.Table && dataContainer instanceof ArrowDataContainer) {
-      // TODO the data has to be a column of string type.
-      // as numerical and other columns types lack valueOffsets prop.
+    if (
+      rebuildArrowTextVector &&
+      data instanceof arrow.Table &&
+      dataContainer instanceof ArrowDataContainer
+    ) {
       getText = dataContainer.getColumn(tl.field.fieldIdx);
+      try {
+        getText = getArrowTextVector(getText as arrow.Vector, getTextAccessor);
+      } catch (error) {
+        // empty text labels
+        getText = getArrowTextVector(getText, () => ' ');
+      }
     }
 
     return {
@@ -115,3 +126,41 @@ export const formatTextLabelData = ({
     };
   });
 };
+
+/**
+ * Get an arrow vector suitable to render text labels with arrow layers.
+ * @param getText A candidate arrow vector to use for text labels.
+ * @param getTextAccessor Text label accessor.
+ */
+export const getArrowTextVector = (
+  candidateTextVector: arrow.Vector,
+  getTextAccessor: ({index}: {index: number}) => string
+): arrow.Vector => {
+  // if the passed vector is suitable for text labels
+  if (arrow.DataType.isUtf8(candidateTextVector?.type)) {
+    return candidateTextVector;
+  }
+
+  // create utf8 vector from source vector with the same number of batches.
+  // @ts-expect-error
+  const offsets = candidateTextVector._offsets;
+  const numOffsets = offsets.length;
+  const batchVectors: arrow.Vector[] = [];
+  const datum = {index: 0};
+  for (let batchIndex = 0; batchIndex < numOffsets - 1; batchIndex++) {
+    const batchStart = offsets[batchIndex];
+    const batchEnd = offsets[batchIndex + 1];
+
+    const batchLabels: string[] = [];
+    for (let rowIndex = batchStart; rowIndex < batchEnd; ++rowIndex) {
+      datum.index = rowIndex;
+      batchLabels.push(getTextAccessor(datum));
+    }
+
+    batchVectors.push(arrow.vectorFromArray(batchLabels, new arrow.Utf8()));
+  }
+
+  const input = batchVectors.flatMap(x => x.data).flat(Number.POSITIVE_INFINITY);
+
+  return new arrow.Vector(input);
+};

package/src/layers/src/layer-utils.ts

@@ -5,12 +5,16 @@ import * as arrow from 'apache-arrow';
 import {Feature, BBox} from 'geojson';
 import {getGeoMetadata} from '@loaders.gl/gis';
 
+import {GEOARROW_EXTENSIONS, GEOARROW_METADATA_KEY} from '@kepler.gl/constants';
+import {KeplerTable} from '@kepler.gl/table';
 import {
   Field,
   ProtoDatasetField,
   FieldPair,
   SupportedColumnMode,
-  LayerColumn
+  LayerColumn,
+  LayerColumns,
+  RGBColor
 } from '@kepler.gl/types';
 import {DataContainerInterface, ArrowDataContainer} from '@kepler.gl/utils';
 import {
@@ -19,7 +23,6 @@ import {
   BinaryGeometriesFromArrowOptions,
   updateBoundsFromGeoArrowSamples
 } from '@loaders.gl/arrow';
-import {EXTENSION_NAME} from '@kepler.gl/deckgl-arrow-layers';
 
 import {WKBLoader} from '@loaders.gl/wkt';
 import {geojsonToBinary} from '@loaders.gl/gis';
@@ -33,7 +36,26 @@ import {
 
 import {DeckGlGeoTypes, GeojsonDataMaps} from './geojson-layer/geojson-utils';
 
-export function assignPointPairToLayerColumn(pair: FieldPair, hasAlt: boolean) {
+export type FindDefaultLayerProps = {
+  label: string;
+  color?: RGBColor;
+  isVisible?: boolean;
+  columns?: Record<string, LayerColumn>;
+};
+
+export type FindDefaultLayerPropsReturnValue = {
+  /** Layer props to create layers by default when a dataset is added */
+  props: FindDefaultLayerProps[];
+  /** layer props of possible alternative layer configurations, not created by default */
+  altProps?: FindDefaultLayerProps[];
+  /** Already found layer configurations */
+  foundLayers?: (FindDefaultLayerProps & {type: string})[];
+};
+
+export function assignPointPairToLayerColumn(
+  pair: FieldPair,
+  hasAlt: boolean
+): Record<string, LayerColumn> {
   const {lat, lng, altitude} = pair.pair;
   if (!hasAlt) {
     return {lat, lng};
@@ -149,7 +171,6 @@ function getBinaryGeometriesFromWKBArrow(
 }
 
 export function getGeojsonLayerMetaFromArrow({
-  dataContainer,
   geoColumn,
   geoField,
   chunkIndex
@@ -159,7 +180,7 @@ export function getGeojsonLayerMetaFromArrow({
   geoField: ProtoDatasetField;
   chunkIndex?: number;
 }): GeojsonLayerMetaProps {
-  const encoding = geoField?.metadata?.get('ARROW:extension:name');
+  const encoding = geoField?.metadata?.get(GEOARROW_METADATA_KEY);
   const options: BinaryGeometriesFromArrowOptions = {
     ...(chunkIndex !== undefined && chunkIndex >= 0
       ? {
@@ -172,7 +193,7 @@ export function getGeojsonLayerMetaFromArrow({
   };
 
   // getBinaryGeometriesFromArrow doesn't support geoarrow.wkb
-  if (encoding === EXTENSION_NAME.WKB) {
+  if (encoding === GEOARROW_EXTENSIONS.WKB) {
     return getBinaryGeometriesFromWKBArrow(geoColumn, options);
   }
 
@@ -220,7 +241,7 @@ export function getHoveredObjectFromArrow(
     const rawGeometry = col?.get(objectInfo.index);
 
     const field = fieldAccessor(dataContainer);
-    const encoding = field?.metadata?.get('ARROW:extension:name');
+    const encoding = field?.metadata?.get(GEOARROW_METADATA_KEY);
 
     const hoveredFeature = parseGeometryFromArrow(rawGeometry, encoding);
 
@@ -265,7 +286,7 @@ export function getGeoArrowPointFields(fields: Field[]): Field[] {
   return fields.filter(field => {
     return (
       field.type === 'geoarrow' &&
-      field.metadata?.get('ARROW:extension:name') === EXTENSION_NAME.POINT
+      field.metadata?.get(GEOARROW_METADATA_KEY) === GEOARROW_EXTENSIONS.POINT
     );
   });
 }
@@ -291,7 +312,7 @@ export function createGeoArrowPointVector(
   const precision = 2;
 
   const metadata = new Map();
-  metadata.set('ARROW:extension:name', EXTENSION_NAME.POINT);
+  metadata.set(GEOARROW_METADATA_KEY, GEOARROW_EXTENSIONS.POINT);
 
   const childField = new arrow.Field('xyz', new arrow.Float(precision), false, metadata);
   const fixedSizeList = new arrow.FixedSizeList(numCoords, childField);
@@ -411,3 +432,61 @@ export function getBoundsFromArrowMetadata(
 
   return false;
 }
+
+/**
+ * Finds and returns the first satisfied column mode based on the provided columns and fields.
+ * @param supportedColumnModes - An array of supported column modes to check.
+ * @param columns - The available columns.
+ * @param fields - Optional table fields to be used for extra verification.
+ * @returns The first column mode that satisfies the required conditions, or undefined if none match.
+ */
+export function getSatisfiedColumnMode(
+  columnModes: SupportedColumnMode[] | null,
+  columns: LayerColumns | undefined,
+  fields?: KeplerTable['fields']
+): SupportedColumnMode | undefined {
+  return columnModes?.find(mode => {
+    return mode.requiredColumns?.every(requriedCol => {
+      const column = columns?.[requriedCol];
+      if (column?.value) {
+        if (mode.verifyField && fields?.[column.fieldIdx]) {
+          const field = fields[column.fieldIdx];
+          return mode.verifyField(field);
+        }
+        return true;
+      }
+      return false;
+    });
+  });
+}
+
+/**
+ * Returns true if the field is of geoarrow point format.
+ * @param field A field.
+ * @returns Returns true if the field is of geoarrow point format.
+ */
+export function isGeoArrowPointField(field: Field) {
+  return (
+    field.type === 'geoarrow' &&
+    field.metadata?.get(GEOARROW_METADATA_KEY) === GEOARROW_EXTENSIONS.POINT
+  );
+}
+
+/**
+ * Create default geoarrow column props based on the dataset.
+ * @param dataset A dataset to create layer props from.
+ * @returns  geoarrow column props.
+ */
+export function getGeoArrowPointLayerProps(dataset: KeplerTable) {
+  const {label} = dataset;
+  const altProps: FindDefaultLayerProps[] = [];
+  dataset.fields.forEach(field => {
+    if (isGeoArrowPointField(field)) {
+      altProps.push({
+        label: (typeof label === 'string' && label.replace(/\.[^/.]+$/, '')) || field.name,
+        columns: {geoarrow: {value: field.name, fieldIdx: field.fieldIdx}}
+      });
+    }
+  });
+  return altProps;
+}

package/src/layers/src/line-layer/line-layer.ts

@@ -231,7 +231,7 @@ export default class LineLayer extends ArcLayer {
     };
   }
 
-  static findDefaultLayerProps({fields, fieldPairs = []}: KeplerTable): {
+  static findDefaultLayerProps({fieldPairs = []}: KeplerTable): {
     props: {color?: RGBColor; columns: LineLayerColumnsConfig; label: string}[];
   } {
     if (fieldPairs.length < 2) {

package/src/layers/src/point-layer/point-layer.ts

@@ -36,9 +36,12 @@ import {
   assignPointPairToLayerColumn,
   isLayerHoveredFromArrow,
   getBoundsFromArrowMetadata,
+  getGeoArrowPointLayerProps,
+  isGeoArrowPointField,
   createGeoArrowPointVector,
   getFilteredIndex,
-  getNeighbors
+  getNeighbors,
+  FindDefaultLayerProps
 } from '../layer-utils';
 import {getGeojsonPointDataMaps, GeojsonPointDataMaps} from '../geojson-layer/geojson-utils';
 import {
@@ -150,12 +153,14 @@ const SUPPORTED_COLUMN_MODES = [
   {
     key: COLUMN_MODE_GEOJSON,
     label: 'GeoJSON Feature',
-    requiredColumns: geojsonRequiredColumns
+    requiredColumns: geojsonRequiredColumns,
+    verifyField: f => !isGeoArrowPointField(f)
   },
   {
     key: COLUMN_MODE_GEOARROW,
     label: 'Geoarrow Points',
-    requiredColumns: geoarrowRequiredColumns
+    requiredColumns: geoarrowRequiredColumns,
+    verifyField: f => isGeoArrowPointField(f)
   }
 ];
 const DEFAULT_COLUMN_MODE = COLUMN_MODE_POINTS;
@@ -322,13 +327,10 @@ export default class PointLayer extends Layer {
     return this;
   }
 
-  static findDefaultLayerProps({fieldPairs = [], type}: KeplerTable) {
-    const props: {
-      label: string;
-      color?: RGBColor;
-      isVisible?: boolean;
-      columns?: PointLayerColumnsConfig;
-    }[] = [];
+  static findDefaultLayerProps(dataset: KeplerTable) {
+    const {fieldPairs = [], type} = dataset;
+
+    const props: FindDefaultLayerProps[] = [];
 
     if (type === DatasetType.VECTOR_TILE) {
       return {props};
@@ -362,7 +364,9 @@ export default class PointLayer extends Layer {
       props.push(prop);
     });
 
-    return {props};
+    const altProps = getGeoArrowPointLayerProps(dataset);
+
+    return {props, altProps};
   }
 
   getDefaultLayerConfig(props: LayerBaseConfigPartial) {

package/src/layers/src/trip-layer/trip-layer.ts

@@ -244,7 +244,7 @@ export default class TripLayer extends Layer {
 
   static findDefaultLayerProps(
     {label, fields = [], dataContainer, id}: KeplerTable,
-    foundLayers: any[]
+    foundLayers?: any[]
   ) {
     const geojsonColumns = fields.filter(f => f.type === 'geojson').map(f => f.name);
 
@@ -268,7 +268,7 @@ export default class TripLayer extends Layer {
         })),
 
         // if a geojson layer is created from this column, delete it
-        foundLayers: foundLayers.filter(
+        foundLayers: foundLayers?.filter(
           prop =>
             prop.type !== 'geojson' ||
             prop.dataId !== id ||
@@ -407,6 +407,8 @@ export default class TripLayer extends Layer {
           ...columnConfig,
           columnMode: COLUMN_MODE_TABLE
         });
+      } else {
+        return this;
       }
     }
 

package/src/layers/src/vector-tile/abstract-tile-layer.ts

@@ -40,6 +40,7 @@ import {
 } from '../base-layer';
 import TileDataset from './common-tile/tile-dataset';
 import {isIndexedField, isDomainQuantiles} from './common-tile/tile-utils';
+import {FindDefaultLayerPropsReturnValue} from '../layer-utils';
 
 const DEFAULT_ELEVATION = 500;
 export const DEFAULT_RADIUS = 1;
@@ -64,6 +65,7 @@ export type AbstractTileLayerVisConfigSettings = {
 export type LayerData = {
   minZoom?: number;
   maxZoom?: number;
+  bounds?: number[];
   getPointRadius?: () => number;
 };
 
@@ -163,9 +165,7 @@ export default abstract class AbstractTileLayer<
 
   protected abstract initTileDataset(): TileDataset<T, I>;
 
-  static findDefaultLayerProps(dataset: KeplerDataset): {
-    props: {dataId: string; label?: string; isVisible: boolean}[];
-  } {
+  static findDefaultLayerProps(dataset: KeplerDataset): FindDefaultLayerPropsReturnValue {
     if (!isTileDataset(dataset)) {
       return {props: []};
     }
@@ -409,10 +409,16 @@ export default abstract class AbstractTileLayer<
     });
 
     const metadata = dataset?.metadata as LayerData | undefined;
+    const metadataToData = metadata
+      ? {
+          minZoom: metadata.minZoom,
+          maxZoom: metadata.maxZoom,
+          bounds: metadata.bounds
+        }
+      : {};
 
     return {
-      ...(metadata ? {minZoom: metadata.minZoom} : {}),
-      ...(metadata ? {maxZoom: metadata.maxZoom} : {}),
+      ...metadataToData,
       ...accessors
     };
   }

package/src/layers/src/vector-tile/vector-tile-layer.ts

@@ -5,7 +5,7 @@ import {FeatureCollection, Feature} from 'geojson';
 
 import {Layer as DeckLayer} from '@deck.gl/core/typed';
 import {_Tile2DHeader as Tile2DHeader} from '@deck.gl/geo-layers/typed';
-import {GeoJsonLayer} from '@deck.gl/layers/typed';
+import {GeoJsonLayer, PathLayer} from '@deck.gl/layers/typed';
 import {MVTSource, MVTTileSource} from '@loaders.gl/mvt';
 import {PMTilesSource, PMTilesTileSource} from '@loaders.gl/pmtiles';
 import GL from '@luma.gl/constants';
@@ -53,6 +53,7 @@ import {
   VisualChannelDomain,
   VisualChannelField
 } from '../base-layer';
+import {FindDefaultLayerPropsReturnValue} from '../layer-utils';
 
 import AbstractTileLayer, {
   LayerData as CommonLayerData,
@@ -174,6 +175,35 @@ export type VectorTileLayerVisConfigSettings = Merge<
   }
 >;
 
+export function tileLayerBoundsLayer(id: string, props: {bounds?: number[]}): DeckLayer[] {
+  const {bounds} = props;
+  if (bounds?.length !== 4) return [];
+
+  const data = [
+    {
+      path: [
+        [bounds[0], bounds[1]],
+        [bounds[2], bounds[1]],
+        [bounds[2], bounds[3]],
+        [bounds[0], bounds[3]],
+        [bounds[0], bounds[1]]
+      ]
+    }
+  ];
+
+  const layer = new PathLayer({
+    id: `${id}-vector-tile-bounds`,
+    data,
+    getPath: d => d.path,
+    getColor: [128, 128, 128, 255],
+    getWidth: 1,
+    widthUnits: 'pixels',
+    pickable: false
+  });
+
+  return [layer];
+}
+
 export default class VectorTileLayer extends AbstractTileLayer<VectorTile, Feature[]> {
   declare config: VectorTileLayerConfig;
   declare visConfigSettings: VectorTileLayerVisConfigSettings;
@@ -186,9 +216,7 @@ export default class VectorTileLayer extends AbstractTileLayer<VectorTile, Featu
 
   meta = {};
 
-  static findDefaultLayerProps(dataset: KeplerDataset): {
-    props: {dataId: string; label?: string; isVisible: boolean}[];
-  } {
+  static findDefaultLayerProps(dataset: KeplerDataset): FindDefaultLayerPropsReturnValue {
     if (dataset.type !== DatasetType.VECTOR_TILE) {
       return {props: []};
     }
@@ -626,6 +654,7 @@ export default class VectorTileLayer extends AbstractTileLayer<VectorTile, Featu
               })
             ]
           : [])
+        // ...tileLayerBoundsLayer(defaultLayerProps.id, data),
       ];
 
       return layers;

package/src/localization/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@kepler.gl/localization",
   "author": "Shan He <shan@uber.com>",
-  "version": "3.1.0",
+  "version": "3.1.1",
   "description": "kepler.gl constants used by kepler.gl components, actions and reducers",
   "license": "MIT",
   "main": "dist/index.js",

package/src/processors/dist/data-processor.d.ts

@@ -166,7 +166,21 @@ export declare function processKeplerglDataset(rawData: object | object[], schem
  * @returns dataset containing `fields` and `rows` or null
  */
 export declare function processArrowTable(arrowTable: ArrowTable): ProcessorResult | null;
-export declare function arrowSchemaToFields(schema: arrow.Schema): Field[];
+/**
+ * Extracts GeoArrow metadata from an Apache Arrow table schema.
+ * For geoparquet files geoarrow metadata isn't present in fields, so extract extra info from schema.
+ * @param table The Apache Arrow table to extract metadata from.
+ * @returns An object mapping column names to their GeoArrow encoding type.
+ * @throws Logs an error message if parsing of metadata fails.
+ */
+export declare function getGeoArrowMetadataFromSchema(table: arrow.Table): Record<string, string>;
+/**
+ * Converts an Apache Arrow table schema into an array of Kepler.gl field objects.
+ * @param table The Apache Arrow table whose schema needs to be converted.
+ * @param fieldTypeSuggestions Optional mapping of field names to suggested field types.
+ * @returns An array of field objects suitable for Kepler.gl.
+ */
+export declare function arrowSchemaToFields(table: arrow.Table, fieldTypeSuggestions?: Record<string, string>): Field[];
 /**
  * Parse arrow batches returned from parseInBatches()
  *