@loaders.gl/scan 5.0.0-alpha.4 → 5.0.0-alpha.5

package/dist/dist.min.js

@@ -15938,6 +15938,12 @@ return true;`);
           type: "list",
           children: [serializeArrowField(listField)]
         };
+      case LargeList2:
+        const largeListType = arrowType;
+        return {
+          type: "large-list",
+          children: [serializeArrowField(largeListType.valueField)]
+        };
       case FixedSizeList2:
         const fixedSizeList = arrowType;
         return {
@@ -15951,6 +15957,15 @@ return true;`);
           type: "struct",
           children: structType.children.map((arrowField) => serializeArrowField(arrowField))
         };
+      case SparseUnion:
+      case DenseUnion:
+        const unionType = arrowType;
+        return {
+          type: unionType instanceof DenseUnion ? "dense-union" : "sparse-union",
+          typeIds: unionType.typeIds,
+          children: unionType.children.map((arrowField) => serializeArrowField(arrowField)),
+          typeIdToChildIndex: { ...unionType.typeIdToChildIndex }
+        };
       case Dictionary:
         const dictionaryType = arrowType;
         return {
@@ -15979,6 +15994,10 @@ return true;`);
           const field = deserializeArrowField(dataType.children[0], options);
           return new List2(field);
         }
+        case "large-list": {
+          const field = deserializeArrowField(dataType.children[0], options);
+          return new LargeList2(field);
+        }
         case "fixed-size-list": {
           const child = deserializeArrowField(dataType.children[0], options);
           return new FixedSizeList2(dataType.listSize, child);
@@ -15991,6 +16010,13 @@ return true;`);
           );
           return new Struct(children);
         }
+        case "sparse-union":
+        case "dense-union": {
+          const children = dataType.children.map(
+            (arrowField) => deserializeArrowField(arrowField, options)
+          );
+          return dataType.type === "dense-union" ? new DenseUnion(dataType.typeIds, children) : new SparseUnion(dataType.typeIds, children);
+        }
         case "dictionary": {
           return new Dictionary(
             deserializeArrowType(dataType.dictionary, options),
@@ -17176,6 +17202,9 @@ return true;`);
     if (left instanceof Date && right instanceof Date) {
       return compareNumbers2(left.getTime(), right.getTime());
     }
+    if (typeof left === "number" && right instanceof Date) {
+      return compareNumbers2(left, right.getTime());
+    }
     if (left instanceof Uint8Array && right instanceof Uint8Array) {
       return compareBinary(left, right);
     }

package/dist/hudi.cjs

@@ -3135,6 +3135,12 @@ function serializeArrowType(arrowType) {
         type: "list",
         children: [serializeArrowField(listField)]
       };
+    case arrow.LargeList:
+      const largeListType = arrowType;
+      return {
+        type: "large-list",
+        children: [serializeArrowField(largeListType.valueField)]
+      };
     case arrow.FixedSizeList:
       const fixedSizeList = arrowType;
       return {
@@ -3148,6 +3154,15 @@ function serializeArrowType(arrowType) {
         type: "struct",
         children: structType.children.map((arrowField) => serializeArrowField(arrowField))
       };
+    case arrow.SparseUnion:
+    case arrow.DenseUnion:
+      const unionType = arrowType;
+      return {
+        type: unionType instanceof arrow.DenseUnion ? "dense-union" : "sparse-union",
+        typeIds: unionType.typeIds,
+        children: unionType.children.map((arrowField) => serializeArrowField(arrowField)),
+        typeIdToChildIndex: { ...unionType.typeIdToChildIndex }
+      };
     case arrow.Dictionary:
       const dictionaryType = arrowType;
       return {
@@ -3176,6 +3191,10 @@ function deserializeArrowType(dataType, options) {
         const field = deserializeArrowField(dataType.children[0], options);
         return new arrow.List(field);
       }
+      case "large-list": {
+        const field = deserializeArrowField(dataType.children[0], options);
+        return new arrow.LargeList(field);
+      }
       case "fixed-size-list": {
         const child = deserializeArrowField(dataType.children[0], options);
         return new arrow.FixedSizeList(dataType.listSize, child);
@@ -3188,6 +3207,13 @@ function deserializeArrowType(dataType, options) {
         );
         return new arrow.Struct(children);
       }
+      case "sparse-union":
+      case "dense-union": {
+        const children = dataType.children.map(
+          (arrowField) => deserializeArrowField(arrowField, options)
+        );
+        return dataType.type === "dense-union" ? new arrow.DenseUnion(dataType.typeIds, children) : new arrow.SparseUnion(dataType.typeIds, children);
+      }
       case "dictionary": {
         return new arrow.Dictionary(
           deserializeArrowType(dataType.dictionary, options),
@@ -3825,8 +3851,85 @@ var init_deprecated = __esm({
   }
 });
 
-// ../schema-utils/src/index.ts
+// ../schema/src/metadata.ts
+function getMetadataValue(metadata, key) {
+  return metadata instanceof Map ? metadata.get(key) || null : metadata[key] || null;
+}
+function setMetadataValue(metadata, key, value) {
+  if (metadata instanceof Map) {
+    metadata.set(key, value);
+  } else {
+    metadata[key] = value;
+  }
+}
+function getGeoMetadata(metadata) {
+  if (!metadata) {
+    return null;
+  }
+  const stringifiedGeoMetadata = getMetadataValue(metadata, "geo");
+  const geoMetadata = stringifiedGeoMetadata && parseJSONStringMetadata(stringifiedGeoMetadata);
+  if (!geoMetadata) {
+    return null;
+  }
+  for (const column of Object.values(geoMetadata.columns || {})) {
+    if (column && typeof column === "object" && typeof column.encoding === "string") {
+      column.encoding = column.encoding.toLowerCase();
+    }
+  }
+  return geoMetadata;
+}
+function unpackGeoMetadata(metadata) {
+  const geoMetadata = getGeoMetadata(metadata);
+  if (!geoMetadata) {
+    return;
+  }
+  const { version: version2, primary_column: primaryColumn, columns } = geoMetadata;
+  if (version2) {
+    setMetadataValue(metadata, "geo.version", version2);
+  }
+  if (primaryColumn) {
+    setMetadataValue(metadata, "geo.primary_column", primaryColumn);
+  }
+  setMetadataValue(metadata, "geo.columns", Object.keys(columns || {}).join(","));
+}
+function unpackJSONStringMetadata(metadata, metadataKey) {
+  const stringifiedMetadata = getMetadataValue(metadata, metadataKey);
+  const json = stringifiedMetadata && parseJSONStringMetadata(stringifiedMetadata);
+  for (const [key, value] of Object.entries(json || {})) {
+    setMetadataValue(
+      metadata,
+      `${metadataKey}.${key}`,
+      typeof value === "string" ? value : JSON.stringify(value)
+    );
+  }
+}
+function parseJSONStringMetadata(stringifiedMetadata) {
+  if (!stringifiedMetadata) {
+    return null;
+  }
+  try {
+    const metadata = JSON.parse(stringifiedMetadata);
+    return metadata && typeof metadata === "object" ? metadata : null;
+  } catch {
+    return null;
+  }
+}
+var init_metadata = __esm({
+  "../schema/src/metadata.ts"() {
+    "use strict";
+  }
+});
+
+// ../schema/src/index.ts
 var init_src3 = __esm({
+  "../schema/src/index.ts"() {
+    "use strict";
+    init_metadata();
+  }
+});
+
+// ../schema-utils/src/index.ts
+var init_src4 = __esm({
   "../schema-utils/src/index.ts"() {
     "use strict";
     init_convert_arrow_schema();
@@ -4815,35 +4918,26 @@ var init_arrow_table_transport = __esm({
   "../arrow/src/lib/utils/arrow-table-transport.ts"() {
     "use strict";
     arrow6 = __toESM(require("apache-arrow"), 1);
-    init_src3();
+    init_src4();
     init_split_arrow_buffers();
   }
 });
 
 // ../gis/src/lib/geoarrow/metadata-utils.ts
-function getMetadataValue(metadata, key) {
-  return metadata instanceof Map ? metadata.get(key) || null : metadata[key] || null;
-}
-function setMetadataValue(metadata, key, value) {
-  if (metadata instanceof Map) {
-    metadata.set(key, value);
-  } else {
-    metadata[key] = value;
-  }
-}
 var init_metadata_utils = __esm({
   "../gis/src/lib/geoarrow/metadata-utils.ts"() {
     "use strict";
+    init_src3();
   }
 });
 
 // ../gis/src/lib/geoarrow/geoparquet-metadata.ts
-function getGeoMetadata(metadata) {
+function getGeoMetadata2(metadata) {
   if (!metadata) {
     return null;
   }
   const stringifiedGeoMetadata = getMetadataValue(metadata, "geo");
-  const geoMetadata = stringifiedGeoMetadata && parseJSONStringMetadata(stringifiedGeoMetadata);
+  const geoMetadata = stringifiedGeoMetadata && parseJSONStringMetadata2(stringifiedGeoMetadata);
   if (!geoMetadata) {
     return null;
   }
@@ -4854,32 +4948,7 @@ function getGeoMetadata(metadata) {
   }
   return geoMetadata;
 }
-function unpackGeoMetadata(metadata) {
-  const geoMetadata = getGeoMetadata(metadata);
-  if (!geoMetadata) {
-    return;
-  }
-  const { version: version2, primary_column, columns } = geoMetadata;
-  if (version2) {
-    setMetadataValue(metadata, "geo.version", version2);
-  }
-  if (primary_column) {
-    setMetadataValue(metadata, "geo.primary_column", primary_column);
-  }
-  setMetadataValue(metadata, "geo.columns", Object.keys(columns || {}).join(","));
-}
-function unpackJSONStringMetadata(metadata, metadataKey) {
-  const stringifiedMetadata = getMetadataValue(metadata, metadataKey);
-  const json = stringifiedMetadata && parseJSONStringMetadata(stringifiedMetadata);
-  for (const [key, value] of Object.entries(json || {})) {
-    setMetadataValue(
-      metadata,
-      `${metadataKey}.${key}`,
-      typeof value === "string" ? value : JSON.stringify(value)
-    );
-  }
-}
-function parseJSONStringMetadata(stringifiedMetadata) {
+function parseJSONStringMetadata2(stringifiedMetadata) {
   if (!stringifiedMetadata) {
     return null;
   }
@@ -4901,16 +4970,15 @@ var init_geoparquet_metadata = __esm({
 });
 
 // ../gis/src/index.ts
-var init_src4 = __esm({
+var init_src5 = __esm({
   "../gis/src/index.ts"() {
     "use strict";
-    init_metadata_utils();
     init_geoparquet_metadata();
   }
 });
 
 // ../arrow/src/index.ts
-var init_src5 = __esm({
+var init_src6 = __esm({
   "../arrow/src/index.ts"() {
     "use strict";
     init_indexed_arrow_table();
@@ -5154,10 +5222,11 @@ function getGeoArrowMetadataFromGeoParquetField(field, columnMetadata) {
   if (!extensionName) {
     return null;
   }
-  if (normalizedEncoding === "wkb" && field.type !== "binary") {
+  const isBinaryGeospatialEncoding = normalizedEncoding === "wkb" || normalizedEncoding === "geometry" || normalizedEncoding === "geography";
+  if (isBinaryGeospatialEncoding && !isBinaryFieldType(field.type)) {
     return null;
   }
-  if (normalizedEncoding !== "wkb" && (field.type === "binary" || field.type === "utf8")) {
+  if (!isBinaryGeospatialEncoding && (isBinaryFieldType(field.type) || isStringFieldType(field.type))) {
     return null;
   }
   const extensionMetadata = {};
@@ -5171,18 +5240,34 @@ function getGeoArrowMetadataFromGeoParquetField(field, columnMetadata) {
   if (columnMetadata.edges && columnMetadata.edges !== "planar") {
     extensionMetadata.edges = columnMetadata.edges;
   }
+  if (normalizedEncoding === "geography" && !extensionMetadata.edges) {
+    extensionMetadata.edges = "spherical";
+  }
+  for (const [metadataKey, metadataValue] of Object.entries(columnMetadata)) {
+    if (metadataValue !== void 0 && metadataValue !== null && !["encoding", "geometry_types", "crs", "edges"].includes(metadataKey)) {
+      extensionMetadata[metadataKey] = metadataValue;
+    }
+  }
   return Object.keys(extensionMetadata).length > 0 ? { extensionName, extensionMetadata } : { extensionName };
 }
+function isBinaryFieldType(fieldType) {
+  return typeof fieldType === "string" && (fieldType === "binary" || fieldType === "binary-view");
+}
+function isStringFieldType(fieldType) {
+  return typeof fieldType === "string" && (fieldType === "utf8" || fieldType === "utf8-view");
+}
 var arrow7, GEOARROW_EXTENSION_NAME_KEY, GEOARROW_EXTENSION_METADATA_KEY, GEOPARQUET_TO_GEOARROW_ENCODINGS;
 var init_geospatial_metadata = __esm({
   "../parquet/src/lib/geo/geospatial-metadata.ts"() {
     "use strict";
     arrow7 = __toESM(require("apache-arrow"), 1);
-    init_src4();
+    init_src3();
     GEOARROW_EXTENSION_NAME_KEY = "ARROW:extension:name";
     GEOARROW_EXTENSION_METADATA_KEY = "ARROW:extension:metadata";
     GEOPARQUET_TO_GEOARROW_ENCODINGS = {
       wkb: "geoarrow.wkb",
+      geometry: "geoarrow.wkb",
+      geography: "geoarrow.wkb",
       point: "geoarrow.point",
       linestring: "geoarrow.linestring",
       polygon: "geoarrow.polygon",
@@ -5208,7 +5293,7 @@ async function getSchemaFromParquetReader(reader) {
 var init_get_parquet_schema = __esm({
   "../parquet/src/lib/parsers/get-parquet-schema.ts"() {
     "use strict";
-    init_src4();
+    init_src3();
     init_convert_schema_from_parquet();
     init_geospatial_metadata();
   }
@@ -5320,7 +5405,7 @@ function doLongitudeIntervalsIntersect(firstWest, firstEast, secondWest, secondE
 var init_geoparquet_covering = __esm({
   "../parquet/src/lib/geo/geoparquet-covering.ts"() {
     "use strict";
-    init_src4();
+    init_src3();
   }
 });
 
@@ -12044,7 +12129,7 @@ var init_parse_bson = __esm({
 });
 
 // ../bson/src/index.ts
-var init_src6 = __esm({
+var init_src7 = __esm({
   "../bson/src/index.ts"() {
     "use strict";
     init_bson_writer();
@@ -12453,7 +12538,7 @@ var PARQUET_LOGICAL_TYPES, kMillisPerDay;
 var init_types = __esm({
   "../parquet/src/parquetjs/schema/types.ts"() {
     "use strict";
-    init_src6();
+    init_src7();
     init_binary_utils();
     PARQUET_LOGICAL_TYPES = {
       BOOLEAN: {
@@ -16474,7 +16559,7 @@ var init_default_codecs = __esm({
 });
 
 // ../compression/src/index.ts
-var init_src7 = __esm({
+var init_src8 = __esm({
   "../compression/src/index.ts"() {
     "use strict";
     init_compression();
@@ -16587,7 +16672,7 @@ var PARQUET_COMPRESSION_METHODS, ParquetSnappyDecompressor;
 var init_compression2 = __esm({
   "../parquet/src/parquetjs/compression.ts"() {
     "use strict";
-    init_src7();
+    init_src8();
     init_snappy_decompressor_hysnappy();
     init_src2();
     init_binary_utils();
@@ -22749,10 +22834,10 @@ var _preloadParquetSourceLoader, _createMetadataDataSource, ParquetSourceLoaderB
 var init_parquet_source_loader = __esm({
   "../parquet/src/parquet-source-loader.ts"() {
     "use strict";
-    init_src5();
+    init_src6();
     init_src2();
-    init_src3();
     init_src4();
+    init_src5();
     init_get_parquet_schema();
     init_geoparquet_covering();
     init_parquet_bloom_filter();
@@ -22935,7 +23020,7 @@ var init_parquet_source_loader = __esm({
           );
           const projectedColumns = normalizeColumns(readOptions.columns, initialization.schema);
           const predicateColumns = physicalPlan.predicate ? getParquetPredicateColumns(physicalPlan.predicate) : [];
-          const geoMetadata = readOptions.bbox ? getGeoMetadata(initialization.schema.metadata) : void 0;
+          const geoMetadata = readOptions.bbox ? getGeoMetadata2(initialization.schema.metadata) : void 0;
           const geometryColumn = readOptions.bbox ? readOptions.geometryColumn || (geoMetadata == null ? void 0 : geoMetadata.primary_column) : void 0;
           const filterColumns = [
             .../* @__PURE__ */ new Set([...predicateColumns, ...geometryColumn ? [geometryColumn] : []])