@loaders.gl/arrow 4.0.0-beta.8 → 4.0.1

package/dist/dist.dev.js

@@ -31,7 +31,17 @@ var __exports__ = (() => {
     ArrowWorkerLoader: () => ArrowLoader,
     ArrowWriter: () => ArrowWriter,
     VECTOR_TYPES: () => VECTOR_TYPES,
-    getArrowType: () => getArrowType
+    deserializeArrowField: () => deserializeArrowField,
+    deserializeArrowMetadata: () => deserializeArrowMetadata,
+    deserializeArrowSchema: () => deserializeArrowSchema,
+    deserializeArrowType: () => deserializeArrowType,
+    getArrowType: () => getArrowType,
+    getBinaryGeometriesFromArrow: () => getBinaryGeometriesFromArrow,
+    parseGeometryFromArrow: () => parseGeometryFromArrow,
+    serializeArrowField: () => serializeArrowField,
+    serializeArrowMetadata: () => serializeArrowMetadata,
+    serializeArrowSchema: () => serializeArrowSchema,
+    serializeArrowType: () => serializeArrowType
   });
 
   // ../schema/src/lib/table/batches/base-table-batch-aggregator.ts
@@ -2105,6 +2115,11 @@ var __exports__ = (() => {
     proto.ArrayType = Int32Array;
     return proto[Symbol.toStringTag] = "Date";
   })(Date_.prototype);
+  var DateDay = class extends Date_ {
+    constructor() {
+      super(DateUnit.DAY);
+    }
+  };
   var DateMillisecond = class extends Date_ {
     constructor() {
       super(DateUnit.MILLISECOND);
@@ -2138,6 +2153,26 @@ var __exports__ = (() => {
     proto.bitWidth = null;
     return proto[Symbol.toStringTag] = "Time";
   })(Time_.prototype);
+  var TimeSecond = class extends Time_ {
+    constructor() {
+      super(TimeUnit.SECOND, 32);
+    }
+  };
+  var TimeMillisecond = class extends Time_ {
+    constructor() {
+      super(TimeUnit.MILLISECOND, 32);
+    }
+  };
+  var TimeMicrosecond = class extends Time_ {
+    constructor() {
+      super(TimeUnit.MICROSECOND, 64);
+    }
+  };
+  var TimeNanosecond = class extends Time_ {
+    constructor() {
+      super(TimeUnit.NANOSECOND, 64);
+    }
+  };
   var Timestamp_ = class extends DataType {
     constructor(unit, timezone) {
       super();
@@ -2158,6 +2193,26 @@ var __exports__ = (() => {
     proto.ArrayType = Int32Array;
     return proto[Symbol.toStringTag] = "Timestamp";
   })(Timestamp_.prototype);
+  var TimestampSecond = class extends Timestamp_ {
+    constructor(timezone) {
+      super(TimeUnit.SECOND, timezone);
+    }
+  };
+  var TimestampMillisecond = class extends Timestamp_ {
+    constructor(timezone) {
+      super(TimeUnit.MILLISECOND, timezone);
+    }
+  };
+  var TimestampMicrosecond = class extends Timestamp_ {
+    constructor(timezone) {
+      super(TimeUnit.MICROSECOND, timezone);
+    }
+  };
+  var TimestampNanosecond = class extends Timestamp_ {
+    constructor(timezone) {
+      super(TimeUnit.NANOSECOND, timezone);
+    }
+  };
   var Interval_ = class extends DataType {
     constructor(unit) {
       super();
@@ -2176,6 +2231,16 @@ var __exports__ = (() => {
     proto.ArrayType = Int32Array;
     return proto[Symbol.toStringTag] = "Interval";
   })(Interval_.prototype);
+  var IntervalDayTime = class extends Interval_ {
+    constructor() {
+      super(IntervalUnit.DAY_TIME);
+    }
+  };
+  var IntervalYearMonth = class extends Interval_ {
+    constructor() {
+      super(IntervalUnit.YEAR_MONTH);
+    }
+  };
   var List = class extends DataType {
     constructor(child) {
       super();
@@ -12273,7 +12338,7 @@ return true;`);
   RecordBatchFileWriter["throughDOM"] = recordBatchWriterThroughDOMStream;
   RecordBatchStreamWriter["throughDOM"] = recordBatchWriterThroughDOMStream;
 
-  // src/lib/convert-table.ts
+  // src/tables/convert-arrow-to-table.ts
   function convertApacheArrowToArrowTable(arrowTable) {
     return {
       shape: "arrow-table",
@@ -12413,6 +12478,238 @@ return true;`);
     }
   }
 
+  // src/schema/convert-arrow-schema.ts
+  function serializeArrowSchema(arrowSchema) {
+    return {
+      fields: arrowSchema.fields.map((arrowField) => serializeArrowField(arrowField)),
+      metadata: serializeArrowMetadata(arrowSchema.metadata)
+    };
+  }
+  function deserializeArrowSchema(schema) {
+    return new Schema2(schema.fields.map((field) => deserializeArrowField(field)), deserializeArrowMetadata(schema.metadata));
+  }
+  function serializeArrowMetadata(arrowMetadata) {
+    return Object.fromEntries(arrowMetadata);
+  }
+  function deserializeArrowMetadata(metadata) {
+    return metadata ? new Map(Object.entries(metadata)) : /* @__PURE__ */ new Map();
+  }
+  function serializeArrowField(field) {
+    return {
+      name: field.name,
+      type: serializeArrowType(field.type),
+      nullable: field.nullable,
+      metadata: serializeArrowMetadata(field.metadata)
+    };
+  }
+  function deserializeArrowField(field) {
+    return new Field2(field.name, deserializeArrowType(field.type), field.nullable, deserializeArrowMetadata(field.metadata));
+  }
+  function serializeArrowType(arrowType) {
+    switch (arrowType.constructor) {
+      case Null:
+        return "null";
+      case Binary:
+        return "binary";
+      case Bool:
+        return "bool";
+      case Int_:
+        const intType = arrowType;
+        return `${intType.isSigned ? "u" : ""}int${intType.bitWidth}`;
+      case Int8:
+        return "int8";
+      case Int16:
+        return "int16";
+      case Int32:
+        return "int32";
+      case Int64:
+        return "int64";
+      case Uint8:
+        return "uint8";
+      case Uint16:
+        return "uint16";
+      case Uint32:
+        return "uint32";
+      case Uint64:
+        return "uint64";
+      case Float:
+        const precision = arrowType.precision;
+        switch (precision) {
+          case Precision.HALF:
+            return "float16";
+          case Precision.SINGLE:
+            return "float32";
+          case Precision.DOUBLE:
+            return "float64";
+          default:
+            return "float16";
+        }
+      case Float16:
+        return "float16";
+      case Float32:
+        return "float32";
+      case Float64:
+        return "float64";
+      case Utf8:
+        return "utf8";
+      case Date:
+        const dateUnit = arrowType.unit;
+        return dateUnit === DateUnit.DAY ? "date-day" : "date-millisecond";
+      case DateDay:
+        return "date-day";
+      case DateMillisecond:
+        return "date-millisecond";
+      case Time_:
+        const timeUnit = arrowType.unit;
+        switch (timeUnit) {
+          case TimeUnit.SECOND:
+            return "time-second";
+          case TimeUnit.MILLISECOND:
+            return "time-millisecond";
+          case TimeUnit.MICROSECOND:
+            return "time-microsecond";
+          case TimeUnit.NANOSECOND:
+            return "time-nanosecond";
+          default:
+            return "time-second";
+        }
+      case TimeMillisecond:
+        return "time-millisecond";
+      case TimeSecond:
+        return "time-second";
+      case TimeMicrosecond:
+        return "time-microsecond";
+      case TimeNanosecond:
+        return "time-nanosecond";
+      case Timestamp_:
+        const timeStampUnit = arrowType.unit;
+        switch (timeStampUnit) {
+          case TimeUnit.SECOND:
+            return "timestamp-second";
+          case TimeUnit.MILLISECOND:
+            return "timestamp-millisecond";
+          case TimeUnit.MICROSECOND:
+            return "timestamp-microsecond";
+          case TimeUnit.NANOSECOND:
+            return "timestamp-nanosecond";
+          default:
+            return "timestamp-second";
+        }
+      case TimestampSecond:
+        return "timestamp-second";
+      case TimestampMillisecond:
+        return "timestamp-millisecond";
+      case TimestampMicrosecond:
+        return "timestamp-microsecond";
+      case TimestampNanosecond:
+        return "timestamp-nanosecond";
+      case Interval_:
+        const intervalUnit = arrowType.unit;
+        switch (intervalUnit) {
+          case IntervalUnit.DAY_TIME:
+            return "interval-daytime";
+          case IntervalUnit.YEAR_MONTH:
+            return "interval-yearmonth";
+          default:
+            return "interval-daytime";
+        }
+      case IntervalDayTime:
+        return "interval-daytime";
+      case IntervalYearMonth:
+        return "interval-yearmonth";
+      case List:
+        const listType = arrowType;
+        const listField = listType.valueField;
+        return {
+          type: "list",
+          children: [serializeArrowField(listField)]
+        };
+      case FixedSizeList:
+        return {
+          type: "fixed-size-list",
+          listSize: arrowType.listSize,
+          children: [serializeArrowField(arrowType.children[0])]
+        };
+      default:
+        throw new Error("array type not supported");
+    }
+  }
+  function deserializeArrowType(dataType) {
+    if (typeof dataType === "object") {
+      switch (dataType.type) {
+        case "list":
+          const field = deserializeArrowField(dataType.children[0]);
+          return new List(field);
+        case "fixed-size-list":
+          const child = deserializeArrowField(dataType.children[0]);
+          return new FixedSizeList(dataType.listSize, child);
+        case "struct":
+          const children = dataType.children.map((arrowField) => deserializeArrowField(arrowField));
+          return new Struct(children);
+        default:
+          throw new Error("array type not supported");
+      }
+    }
+    switch (dataType) {
+      case "null":
+        return new Null();
+      case "binary":
+        return new Binary();
+      case "bool":
+        return new Bool();
+      case "int8":
+        return new Int8();
+      case "int16":
+        return new Int16();
+      case "int32":
+        return new Int32();
+      case "int64":
+        return new Int64();
+      case "uint8":
+        return new Uint8();
+      case "uint16":
+        return new Uint16();
+      case "uint32":
+        return new Uint32();
+      case "uint64":
+        return new Uint64();
+      case "float16":
+        return new Float16();
+      case "float32":
+        return new Float32();
+      case "float64":
+        return new Float64();
+      case "utf8":
+        return new Utf8();
+      case "date-day":
+        return new DateDay();
+      case "date-millisecond":
+        return new DateMillisecond();
+      case "time-second":
+        return new TimeSecond();
+      case "time-millisecond":
+        return new TimeMillisecond();
+      case "time-microsecond":
+        return new TimeMicrosecond();
+      case "time-nanosecond":
+        return new TimeNanosecond();
+      case "timestamp-second":
+        return new TimestampSecond();
+      case "timestamp-millisecond":
+        return new TimestampMillisecond();
+      case "timestamp-microsecond":
+        return new TimestampMicrosecond();
+      case "timestamp-nanosecond":
+        return new TimestampNanosecond();
+      case "interval-daytime":
+        return new IntervalDayTime();
+      case "interval-yearmonth":
+        return new IntervalYearMonth();
+      default:
+        throw new Error("array type not supported");
+    }
+  }
+
   // src/types.ts
   var VECTOR_TYPES = function(VECTOR_TYPES2) {
     VECTOR_TYPES2[VECTOR_TYPES2["FLOAT"] = 0] = "FLOAT";
@@ -12457,6 +12754,357 @@ return true;`);
     options: {}
   };
 
+  // src/geoarrow/get-arrow-bounds.ts
+  function updateBoundsFromGeoArrowSamples(flatCoords, nDim, bounds, sampleSize = 100) {
+    const numberOfFeatures = flatCoords.length / nDim;
+    const sampleStep = Math.max(Math.floor(numberOfFeatures / sampleSize), 1);
+    const newBounds = [...bounds];
+    for (let i = 0; i < numberOfFeatures; i += sampleStep) {
+      const lng = flatCoords[i * nDim];
+      const lat = flatCoords[i * nDim + 1];
+      if (lng < bounds[0]) {
+        newBounds[0] = lng;
+      }
+      if (lat < newBounds[1]) {
+        newBounds[1] = lat;
+      }
+      if (lng > newBounds[2]) {
+        newBounds[2] = lng;
+      }
+      if (lat > newBounds[3]) {
+        newBounds[3] = lat;
+      }
+    }
+    return newBounds;
+  }
+
+  // src/geoarrow/convert-geoarrow-to-binary-geometry.ts
+  var BINARY_GEOMETRY_TEMPLATE = {
+    globalFeatureIds: {
+      value: new Uint32Array(0),
+      size: 1
+    },
+    positions: {
+      value: new Float32Array(0),
+      size: 2
+    },
+    properties: [],
+    numericProps: {},
+    featureIds: {
+      value: new Uint32Array(0),
+      size: 1
+    }
+  };
+  function getBinaryGeometriesFromArrow(geoColumn, geoEncoding) {
+    const featureTypes = {
+      polygon: geoEncoding === "geoarrow.multipolygon" || geoEncoding === "geoarrow.polygon",
+      point: geoEncoding === "geoarrow.multipoint" || geoEncoding === "geoarrow.point",
+      line: geoEncoding === "geoarrow.multilinestring" || geoEncoding === "geoarrow.linestring"
+    };
+    const chunks = geoColumn.data;
+    const bounds = [Infinity, Infinity, -Infinity, -Infinity];
+    let globalFeatureIdOffset = 0;
+    const binaryGeometries = [];
+    for (let c = 0; c < chunks.length; c++) {
+      const geometries = chunks[c];
+      const {
+        featureIds,
+        flatCoordinateArray,
+        nDim,
+        geomOffset
+      } = getBinaryGeometriesFromChunk(geometries, geoEncoding);
+      const numOfVertices = flatCoordinateArray.length / nDim;
+      const globalFeatureIds = new Uint32Array(numOfVertices);
+      for (let i = 0; i < numOfVertices; i++) {
+        globalFeatureIds[i] = featureIds[i] + globalFeatureIdOffset;
+      }
+      const binaryContent = {
+        globalFeatureIds: {
+          value: globalFeatureIds,
+          size: 1
+        },
+        positions: {
+          value: flatCoordinateArray,
+          size: nDim
+        },
+        featureIds: {
+          value: featureIds,
+          size: 1
+        },
+        properties: [...Array(geometries.length).keys()].map((i) => ({
+          index: i + globalFeatureIdOffset
+        }))
+      };
+      globalFeatureIdOffset += geometries.length;
+      binaryGeometries.push({
+        shape: "binary-feature-collection",
+        points: {
+          type: "Point",
+          ...BINARY_GEOMETRY_TEMPLATE,
+          ...featureTypes.point ? binaryContent : {}
+        },
+        lines: {
+          type: "LineString",
+          ...BINARY_GEOMETRY_TEMPLATE,
+          ...featureTypes.line ? binaryContent : {},
+          pathIndices: {
+            value: featureTypes.line ? geomOffset : new Uint16Array(0),
+            size: 1
+          }
+        },
+        polygons: {
+          type: "Polygon",
+          ...BINARY_GEOMETRY_TEMPLATE,
+          ...featureTypes.polygon ? binaryContent : {},
+          polygonIndices: {
+            value: featureTypes.polygon ? geomOffset : new Uint16Array(0),
+            size: 1
+          },
+          primitivePolygonIndices: {
+            value: featureTypes.polygon ? geomOffset : new Uint16Array(0),
+            size: 1
+          }
+        }
+      });
+      updateBoundsFromGeoArrowSamples(flatCoordinateArray, nDim, bounds);
+    }
+    return {
+      binaryGeometries,
+      bounds,
+      featureTypes
+    };
+  }
+  function getBinaryGeometriesFromChunk(chunk, geoEncoding) {
+    switch (geoEncoding) {
+      case "geoarrow.point":
+      case "geoarrow.multipoint":
+        return getBinaryPointsFromChunk(chunk, geoEncoding);
+      case "geoarrow.linestring":
+      case "geoarrow.multilinestring":
+        return getBinaryLinesFromChunk(chunk, geoEncoding);
+      case "geoarrow.polygon":
+      case "geoarrow.multipolygon":
+        return getBinaryPolygonsFromChunk(chunk, geoEncoding);
+      default:
+        throw Error("invalid geoarrow encoding");
+    }
+  }
+  function getBinaryPolygonsFromChunk(chunk, geoEncoding) {
+    const isMultiPolygon = geoEncoding === "geoarrow.multipolygon";
+    const polygonData = isMultiPolygon ? chunk.children[0] : chunk;
+    const ringData = polygonData.children[0];
+    const pointData = ringData.children[0];
+    const coordData = pointData.children[0];
+    const nDim = pointData.stride;
+    const geomOffset = ringData.valueOffsets;
+    const flatCoordinateArray = coordData.values;
+    const geometryIndicies = new Uint16Array(chunk.length + 1);
+    for (let i = 0; i < chunk.length; i++) {
+      geometryIndicies[i] = geomOffset[chunk.valueOffsets[i]];
+    }
+    geometryIndicies[chunk.length] = flatCoordinateArray.length / nDim;
+    const numOfVertices = flatCoordinateArray.length / nDim;
+    const featureIds = new Uint32Array(numOfVertices);
+    for (let i = 0; i < chunk.length - 1; i++) {
+      const startIdx = geomOffset[chunk.valueOffsets[i]];
+      const endIdx = geomOffset[chunk.valueOffsets[i + 1]];
+      for (let j = startIdx; j < endIdx; j++) {
+        featureIds[j] = i;
+      }
+    }
+    return {
+      featureIds,
+      flatCoordinateArray,
+      nDim,
+      geomOffset,
+      geometryIndicies
+    };
+  }
+  function getBinaryLinesFromChunk(chunk, geoEncoding) {
+    const isMultiLineString = geoEncoding === "geoarrow.multilinestring";
+    const lineData = isMultiLineString ? chunk.children[0] : chunk;
+    const pointData = lineData.children[0];
+    const coordData = pointData.children[0];
+    const nDim = pointData.stride;
+    const geomOffset = lineData.valueOffsets;
+    const flatCoordinateArray = coordData.values;
+    const geometryIndicies = new Uint16Array(0);
+    const numOfVertices = flatCoordinateArray.length / nDim;
+    const featureIds = new Uint32Array(numOfVertices);
+    for (let i = 0; i < chunk.length; i++) {
+      const startIdx = geomOffset[i];
+      const endIdx = geomOffset[i + 1];
+      for (let j = startIdx; j < endIdx; j++) {
+        featureIds[j] = i;
+      }
+    }
+    return {
+      featureIds,
+      flatCoordinateArray,
+      nDim,
+      geomOffset,
+      geometryIndicies
+    };
+  }
+  function getBinaryPointsFromChunk(chunk, geoEncoding) {
+    const isMultiPoint = geoEncoding === "geoarrow.multipoint";
+    const pointData = isMultiPoint ? chunk.children[0] : chunk;
+    const coordData = pointData.children[0];
+    const nDim = pointData.stride;
+    const flatCoordinateArray = coordData.values;
+    const geometryIndicies = new Uint16Array(0);
+    const geomOffset = new Int32Array(0);
+    const numOfVertices = flatCoordinateArray.length / nDim;
+    const featureIds = new Uint32Array(numOfVertices);
+    for (let i = 0; i < chunk.length; i++) {
+      featureIds[i] = i;
+    }
+    return {
+      featureIds,
+      flatCoordinateArray,
+      nDim,
+      geomOffset,
+      geometryIndicies
+    };
+  }
+
+  // src/geoarrow/convert-geoarrow-to-geojson.ts
+  function parseGeometryFromArrow(rawData) {
+    const encoding = rawData.encoding?.toLowerCase();
+    const data = rawData.data;
+    if (!encoding || !data) {
+      return null;
+    }
+    let geometry;
+    switch (encoding) {
+      case "geoarrow.multipolygon":
+        geometry = arrowMultiPolygonToFeature(data);
+        break;
+      case "geoarrow.polygon":
+        geometry = arrowPolygonToFeature(data);
+        break;
+      case "geoarrow.multipoint":
+        geometry = arrowMultiPointToFeature(data);
+        break;
+      case "geoarrow.point":
+        geometry = arrowPointToFeature(data);
+        break;
+      case "geoarrow.multilinestring":
+        geometry = arrowMultiLineStringToFeature(data);
+        break;
+      case "geoarrow.linestring":
+        geometry = arrowLineStringToFeature(data);
+        break;
+      default: {
+        throw Error(`GeoArrow encoding not supported ${encoding}`);
+      }
+    }
+    return {
+      type: "Feature",
+      geometry,
+      properties: {}
+    };
+  }
+  function arrowMultiPolygonToFeature(arrowMultiPolygon) {
+    const multiPolygon = [];
+    for (let m = 0; m < arrowMultiPolygon.length; m++) {
+      const arrowPolygon = arrowMultiPolygon.get(m);
+      const polygon = [];
+      for (let i = 0; arrowPolygon && i < arrowPolygon?.length; i++) {
+        const arrowRing = arrowPolygon?.get(i);
+        const ring = [];
+        for (let j = 0; arrowRing && j < arrowRing.length; j++) {
+          const arrowCoord = arrowRing.get(j);
+          const coord = Array.from(arrowCoord);
+          ring.push(coord);
+        }
+        polygon.push(ring);
+      }
+      multiPolygon.push(polygon);
+    }
+    const geometry = {
+      type: "MultiPolygon",
+      coordinates: multiPolygon
+    };
+    return geometry;
+  }
+  function arrowPolygonToFeature(arrowPolygon) {
+    const polygon = [];
+    for (let i = 0; arrowPolygon && i < arrowPolygon.length; i++) {
+      const arrowRing = arrowPolygon.get(i);
+      const ring = [];
+      for (let j = 0; arrowRing && j < arrowRing.length; j++) {
+        const arrowCoord = arrowRing.get(j);
+        const coords = Array.from(arrowCoord);
+        ring.push(coords);
+      }
+      polygon.push(ring);
+    }
+    const geometry = {
+      type: "Polygon",
+      coordinates: polygon
+    };
+    return geometry;
+  }
+  function arrowMultiPointToFeature(arrowMultiPoint) {
+    const multiPoint = [];
+    for (let i = 0; arrowMultiPoint && i < arrowMultiPoint.length; i++) {
+      const arrowPoint = arrowMultiPoint.get(i);
+      if (arrowPoint) {
+        const coord = Array.from(arrowPoint);
+        multiPoint.push(coord);
+      }
+    }
+    const geometry = {
+      type: "MultiPoint",
+      coordinates: multiPoint
+    };
+    return geometry;
+  }
+  function arrowPointToFeature(arrowPoint) {
+    const point = Array.from(arrowPoint);
+    const geometry = {
+      type: "Point",
+      coordinates: point
+    };
+    return geometry;
+  }
+  function arrowMultiLineStringToFeature(arrowMultiLineString) {
+    const multiLineString = [];
+    for (let i = 0; arrowMultiLineString && i < arrowMultiLineString.length; i++) {
+      const arrowLineString = arrowMultiLineString.get(i);
+      const lineString = [];
+      for (let j = 0; arrowLineString && j < arrowLineString.length; j++) {
+        const arrowCoord = arrowLineString.get(j);
+        if (arrowCoord) {
+          const coords = Array.from(arrowCoord);
+          lineString.push(coords);
+        }
+      }
+      multiLineString.push(lineString);
+    }
+    const geometry = {
+      type: "MultiLineString",
+      coordinates: multiLineString
+    };
+    return geometry;
+  }
+  function arrowLineStringToFeature(arrowLineString) {
+    const lineString = [];
+    for (let i = 0; arrowLineString && i < arrowLineString.length; i++) {
+      const arrowCoord = arrowLineString.get(i);
+      if (arrowCoord) {
+        const coords = Array.from(arrowCoord);
+        lineString.push(coords);
+      }
+    }
+    const geometry = {
+      type: "LineString",
+      coordinates: lineString
+    };
+    return geometry;
+  }
+
   // src/index.ts
   TableBatchBuilder.ArrowBatch = ArrowTableBatchAggregator;
   var ArrowLoader2 = {