@loaders.gl/parquet 5.0.0-alpha.1 → 5.0.0-alpha.2

package/dist/index.d.ts

@@ -1,10 +1,13 @@
 export { ParquetFormat } from "./parquet-format.js";
 export type { ParquetLoaderOptions, ParquetJSLoaderOptions } from "./parquet-loader-options.js";
-export { ParquetLoader } from "./parquet-loader.js";
+export { ParquetLoader } from "./parquet-loader-types.js";
 export { GeoParquetLoader } from "./geoparquet-loader.js";
-export { ParquetJSLoader } from "./parquet-js-loader.js";
+export { ParquetJSLoader } from "./parquet-js-loader-types.js";
+export { ParquetSourceLoader } from "./parquet-source-loader-types.js";
+export type { ParquetBatch, ParquetBatchMetadata, ParquetBatchProvenance, ParquetColumnChunkMetadata, ParquetColumnChunkStatistics, ParquetDatasetBatch, ParquetDatasetBatchProvenance, ParquetDatasetBoundingBox, ParquetDatasetFile, ParquetDatasetFileCollection, ParquetDatasetFileProvider, ParquetDatasetFileQuery, ParquetDatasetFiles, ParquetDatasetPartitionValue, ParquetDatasetReadOptions, ParquetDatasetSourceOptions, ParquetDatasetTelemetry, ParquetMetadataRequestOptions, ParquetObjectVersion, ParquetComparisonPredicate, ParquetInPredicate, ParquetLogicalPredicate, ParquetNotPredicate, ParquetNullPredicate, ParquetPredicate, ParquetPredicateProperty, ParquetPredicateValue, ParquetRangeRequestOptions, ParquetReadOptions, ParquetRowGroupMetadata, ParquetSourceBatch, ParquetSourceLoaderOptions, ParquetSourceMetadata, ParquetSourceReadOptions, ParquetTelemetry, ParquetTelemetryEvent } from "./parquet-source-types.js";
+export { PARQUET_SOURCE_CAPABILITIES, type ParquetSourceCapabilities } from "./parquet-source-capabilities.js";
 export { ParquetWriter } from "./parquet-writer.js";
-export type { ParquetJSWriterOptions } from "./parquet-js-writer.js";
+export type { ParquetJSWriterEncoding, ParquetJSWriterOptions } from "./parquet-js-writer.js";
 export { ParquetJSWriter } from "./parquet-js-writer.js";
 export { preloadCompressions } from "./parquetjs/compression.js";
 export { ParquetSchema } from "./parquetjs/schema/schema.js";

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAMA,OAAO,EAAC,aAAa,EAAC,4BAAyB;AAE/C,YAAY,EAAC,oBAAoB,EAAE,sBAAsB,EAAC,oCAAiC;AAC3F,OAAO,EAAC,aAAa,EAAC,4BAAyB;AAC/C,OAAO,EAAC,gBAAgB,EAAC,+BAA4B;AACrD,OAAO,EAAC,eAAe,EAAC,+BAA4B;AAEpD,OAAO,EAAC,aAAa,EAAC,4BAAyB;AAC/C,YAAY,EAAC,sBAAsB,EAAC,+BAA4B;AAChE,OAAO,EAAC,eAAe,EAAC,+BAA4B;AAIpD,OAAO,EAAC,mBAAmB,EAAC,mCAAgC;AAE5D,OAAO,EAAC,aAAa,EAAC,qCAAkC;AACxD,OAAO,EAAC,aAAa,EAAC,6CAA0C;AAChE,OAAO,EAAC,cAAc,EAAC,+CAA4C;AAEnE,OAAO,EACL,oBAAoB,EACpB,oBAAoB,IAAI,2BAA2B,EACpD,mDAAgD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAMA,OAAO,EAAC,aAAa,EAAC,4BAAyB;AAE/C,YAAY,EAAC,oBAAoB,EAAE,sBAAsB,EAAC,oCAAiC;AAC3F,OAAO,EAAC,aAAa,EAAC,kCAA+B;AACrD,OAAO,EAAC,gBAAgB,EAAC,+BAA4B;AACrD,OAAO,EAAC,eAAe,EAAC,qCAAkC;AAE1D,OAAO,EAAC,mBAAmB,EAAC,yCAAsC;AAClE,YAAY,EACV,YAAY,EACZ,oBAAoB,EACpB,sBAAsB,EACtB,0BAA0B,EAC1B,4BAA4B,EAC5B,mBAAmB,EACnB,6BAA6B,EAC7B,yBAAyB,EACzB,kBAAkB,EAClB,4BAA4B,EAC5B,0BAA0B,EAC1B,uBAAuB,EACvB,mBAAmB,EACnB,4BAA4B,EAC5B,yBAAyB,EACzB,2BAA2B,EAC3B,uBAAuB,EACvB,6BAA6B,EAC7B,oBAAoB,EACpB,0BAA0B,EAC1B,kBAAkB,EAClB,uBAAuB,EACvB,mBAAmB,EACnB,oBAAoB,EACpB,gBAAgB,EAChB,wBAAwB,EACxB,qBAAqB,EACrB,0BAA0B,EAC1B,kBAAkB,EAClB,uBAAuB,EACvB,kBAAkB,EAClB,0BAA0B,EAC1B,qBAAqB,EACrB,wBAAwB,EACxB,gBAAgB,EAChB,qBAAqB,EACtB,kCAA+B;AAChC,OAAO,EACL,2BAA2B,EAC3B,KAAK,yBAAyB,EAC/B,yCAAsC;AAEvC,OAAO,EAAC,aAAa,EAAC,4BAAyB;AAC/C,YAAY,EAAC,uBAAuB,EAAE,sBAAsB,EAAC,+BAA4B;AACzF,OAAO,EAAC,eAAe,EAAC,+BAA4B;AAIpD,OAAO,EAAC,mBAAmB,EAAC,mCAAgC;AAE5D,OAAO,EAAC,aAAa,EAAC,qCAAkC;AACxD,OAAO,EAAC,aAAa,EAAC,6CAA0C;AAChE,OAAO,EAAC,cAAc,EAAC,+CAA4C;AAEnE,OAAO,EACL,oBAAoB,EACpB,oBAAoB,IAAI,2BAA2B,EACpD,mDAAgD"}

package/dist/index.js

@@ -3,9 +3,11 @@
 // Copyright (c) vis.gl contributors
 // import {ArrowTable, ArrowTableBatch} from '@loaders.gl/arrow';
 export { ParquetFormat } from "./parquet-format.js";
-export { ParquetLoader } from "./parquet-loader.js";
+export { ParquetLoader } from "./parquet-loader-types.js";
 export { GeoParquetLoader } from "./geoparquet-loader.js";
-export { ParquetJSLoader } from "./parquet-js-loader.js";
+export { ParquetJSLoader } from "./parquet-js-loader-types.js";
+export { ParquetSourceLoader } from "./parquet-source-loader-types.js";
+export { PARQUET_SOURCE_CAPABILITIES } from "./parquet-source-capabilities.js";
 export { ParquetWriter } from "./parquet-writer.js";
 export { ParquetJSWriter } from "./parquet-js-writer.js";
 // EXPERIMENTAL - expose the internal parquetjs API

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAEpC,iEAAiE;AAEjE,OAAO,EAAC,aAAa,EAAC,4BAAyB;AAG/C,OAAO,EAAC,aAAa,EAAC,4BAAyB;AAC/C,OAAO,EAAC,gBAAgB,EAAC,+BAA4B;AACrD,OAAO,EAAC,eAAe,EAAC,+BAA4B;AAEpD,OAAO,EAAC,aAAa,EAAC,4BAAyB;AAE/C,OAAO,EAAC,eAAe,EAAC,+BAA4B;AAEpD,mDAAmD;AAEnD,OAAO,EAAC,mBAAmB,EAAC,mCAAgC;AAE5D,OAAO,EAAC,aAAa,EAAC,qCAAkC;AACxD,OAAO,EAAC,aAAa,EAAC,6CAA0C;AAChE,OAAO,EAAC,cAAc,EAAC,+CAA4C;AAEnE,OAAO,EACL,oBAAoB,EACpB,oBAAoB,IAAI,2BAA2B,EACpD,mDAAgD"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAEpC,iEAAiE;AAEjE,OAAO,EAAC,aAAa,EAAC,4BAAyB;AAG/C,OAAO,EAAC,aAAa,EAAC,kCAA+B;AACrD,OAAO,EAAC,gBAAgB,EAAC,+BAA4B;AACrD,OAAO,EAAC,eAAe,EAAC,qCAAkC;AAE1D,OAAO,EAAC,mBAAmB,EAAC,yCAAsC;AAuClE,OAAO,EACL,2BAA2B,EAE5B,yCAAsC;AAEvC,OAAO,EAAC,aAAa,EAAC,4BAAyB;AAE/C,OAAO,EAAC,eAAe,EAAC,+BAA4B;AAEpD,mDAAmD;AAEnD,OAAO,EAAC,mBAAmB,EAAC,mCAAgC;AAE5D,OAAO,EAAC,aAAa,EAAC,qCAAkC;AACxD,OAAO,EAAC,aAAa,EAAC,6CAA0C;AAChE,OAAO,EAAC,cAAc,EAAC,+CAA4C;AAEnE,OAAO,EACL,oBAAoB,EACpB,oBAAoB,IAAI,2BAA2B,EACpD,mDAAgD"}

package/dist/lib/arrow/convert-columns-to-row-group.js

@@ -1,2 +1,5 @@
 "use strict";
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
 //# sourceMappingURL=convert-columns-to-row-group.js.map

package/dist/lib/arrow/convert-columns-to-row-group.js.map

@@ -1 +1 @@
-{"version":3,"file":"convert-columns-to-row-group.js","sourceRoot":"","sources":["../../../src/lib/arrow/convert-columns-to-row-group.ts"],"names":[],"mappings":""}
+{"version":3,"file":"convert-columns-to-row-group.js","sourceRoot":"","sources":["../../../src/lib/arrow/convert-columns-to-row-group.ts"],"names":[],"mappings":";AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC"}

package/dist/lib/arrow/convert-schema-from-parquet.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"convert-schema-from-parquet.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/convert-schema-from-parquet.ts"],"names":[],"mappings":"AAIA,OAAO,EAAC,MAAM,EAAS,QAAQ,EAAC,MAAM,oBAAoB,CAAC;AAE3D,OAAO,KAAK,EAAC,aAAa,EAAC,yCAAsC;AACjE,OAAO,KAAK,EAAgC,WAAW,EAAC,0CAAuC;AAC/F,OAAO,EAAC,YAAY,EAAC,0CAAuC;AAE5D,eAAO,MAAM,oBAAoB,EAAE;KAAE,IAAI,IAAI,WAAW,GAAG,QAAQ;CA+BlE,CAAC;AAEF,wBAAgB,oBAAoB,CAClC,aAAa,EAAE,aAAa,EAC5B,eAAe,EAAE,YAAY,GAAG,IAAI,GACnC,MAAM,CAUR"}
+{"version":3,"file":"convert-schema-from-parquet.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/convert-schema-from-parquet.ts"],"names":[],"mappings":"AAIA,OAAO,EAAC,MAAM,EAAS,QAAQ,EAAC,MAAM,oBAAoB,CAAC;AAE3D,OAAO,KAAK,EAAC,aAAa,EAAC,yCAAsC;AACjE,OAAO,KAAK,EAAkB,WAAW,EAAmB,0CAAuC;AACnG,OAAO,EAAC,YAAY,EAAC,0CAAuC;AAE5D,eAAO,MAAM,oBAAoB,EAAE;KAAE,IAAI,IAAI,WAAW,GAAG,QAAQ;CAwClE,CAAC;AAEF,wBAAgB,oBAAoB,CAClC,aAAa,EAAE,aAAa,EAC5B,eAAe,EAAE,YAAY,GAAG,IAAI,GACnC,MAAM,CAUR"}

package/dist/lib/arrow/convert-schema-from-parquet.js

@@ -4,19 +4,22 @@
 export const PARQUET_TYPE_MAPPING = {
     BOOLEAN: 'bool',
     INT32: 'int32',
-    INT64: 'float64',
+    INT64: 'int64',
     INT96: 'float64',
     FLOAT: 'float32',
     DOUBLE: 'float64',
     BYTE_ARRAY: 'binary',
     FIXED_LEN_BYTE_ARRAY: 'binary',
     UTF8: 'utf8',
-    DATE: 'int32',
-    TIME_MILLIS: 'int64',
-    TIME_MICROS: 'int64',
-    TIMESTAMP_MILLIS: 'int64',
-    TIMESTAMP_MICROS: 'int64',
-    UINT_8: 'int32',
+    ENUM: 'utf8',
+    DATE: 'date-day',
+    TIME_MILLIS: 'time-millisecond',
+    TIME_MICROS: 'time-microsecond',
+    TIME_NANOS: 'time-nanosecond',
+    TIMESTAMP_MILLIS: 'timestamp-millisecond',
+    TIMESTAMP_MICROS: 'timestamp-microsecond',
+    TIMESTAMP_NANOS: 'timestamp-nanosecond',
+    UINT_8: 'uint8',
     UINT_16: 'uint16',
     UINT_32: 'uint32',
     UINT_64: 'uint64',
@@ -24,14 +27,20 @@ export const PARQUET_TYPE_MAPPING = {
     INT_16: 'int16',
     INT_32: 'int32',
     INT_64: 'int64',
+    UUID: { type: 'fixed-size-binary', byteWidth: 16 },
+    FLOAT16: 'float16',
+    UNKNOWN: 'null',
+    VARIANT: 'binary',
+    GEOMETRY: 'binary',
+    GEOGRAPHY: 'binary',
     JSON: 'binary',
     BSON: 'binary',
     // TODO check interal type
     INTERVAL: 'binary',
-    DECIMAL_INT32: 'float32',
-    DECIMAL_INT64: 'float64',
-    DECIMAL_BYTE_ARRAY: 'float64',
-    DECIMAL_FIXED_LEN_BYTE_ARRAY: 'float64'
+    DECIMAL_INT32: { type: 'decimal', bitWidth: 128, precision: 38, scale: 0 },
+    DECIMAL_INT64: { type: 'decimal', bitWidth: 128, precision: 38, scale: 0 },
+    DECIMAL_BYTE_ARRAY: { type: 'decimal', bitWidth: 128, precision: 38, scale: 0 },
+    DECIMAL_FIXED_LEN_BYTE_ARRAY: { type: 'decimal', bitWidth: 128, precision: 38, scale: 0 }
 };
 export function convertParquetSchema(parquetSchema, parquetMetadata) {
     const fields = getFields(parquetSchema.schema);
@@ -45,29 +54,80 @@ export function convertParquetSchema(parquetSchema, parquetMetadata) {
 function getFields(schema) {
     const fields = [];
     for (const name in schema) {
-        const field = schema[name];
-        if (field.fields) {
-            const children = getFields(field.fields);
-            fields.push({ name, type: { type: 'struct', children }, nullable: field.optional });
+        fields.push(getField(name, schema[name]));
+    }
+    return fields;
+}
+/** Converts one Parquet field, preserving repeated values as Arrow lists. */
+function getField(name, field) {
+    const elementField = field.fields
+        ? {
+            name,
+            type: { type: 'struct', children: getFields(field.fields) },
+            nullable: Boolean(field.optional),
+            metadata: getFieldMetadata(field)
+        }
+        : {
+            name,
+            type: getFieldType(field),
+            nullable: Boolean(field.optional),
+            metadata: getFieldMetadata(field)
+        };
+    if (!field.repeated) {
+        return elementField;
+    }
+    return {
+        name,
+        type: {
+            type: 'list',
+            children: [{ ...elementField, name: 'element', nullable: false }]
+        },
+        nullable: Boolean(field.optional),
+        metadata: elementField.metadata
+    };
+}
+/** Returns the exact serialized Arrow type for one decoded Parquet field. */
+function getFieldType(field) {
+    if (field.logicalType?.type === 'DECIMAL') {
+        const precision = field.precision ?? field.presision;
+        const scale = field.scale;
+        if (!precision || scale === undefined) {
+            throw new Error('Parquet DECIMAL logical type requires precision and scale');
         }
-        else {
-            const type = PARQUET_TYPE_MAPPING[field.type];
-            const metadata = getFieldMetadata(field);
-            const arrowField = { name, type, nullable: field.optional, metadata };
-            fields.push(arrowField);
+        if (precision > 76) {
+            throw new Error(`Parquet DECIMAL precision ${precision} exceeds Arrow Decimal256`);
         }
+        return { type: 'decimal', bitWidth: precision <= 38 ? 128 : 256, precision, scale };
     }
-    return fields;
+    if (field.logicalType?.type === 'UUID') {
+        return { type: 'fixed-size-binary', byteWidth: field.typeLength || 16 };
+    }
+    return PARQUET_TYPE_MAPPING[field.type];
 }
+/** Returns defined physical field properties as Arrow-compatible string metadata. */
 function getFieldMetadata(field) {
     let metadata;
     for (const key in field) {
-        if (key !== 'name') {
-            let value = field[key] || '';
-            value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
-            metadata = metadata || {};
-            metadata[key] = value;
+        const fieldValue = field[key];
+        if (key === 'name' ||
+            key === 'fields' ||
+            (key === 'physicalType' && fieldValue === field.type) ||
+            (key === 'presision' && field.precision !== undefined) ||
+            fieldValue === undefined) {
+            continue;
+        }
+        const metadataValue = typeof fieldValue === 'string'
+            ? fieldValue
+            : typeof fieldValue === 'boolean'
+                ? fieldValue
+                    ? 'true'
+                    : 'false'
+                : JSON.stringify(fieldValue);
+        if (metadataValue === undefined) {
+            continue;
         }
+        metadata ||= {};
+        metadata[key] = metadataValue;
     }
     return metadata;
 }

package/dist/lib/arrow/convert-schema-from-parquet.js.map

@@ -1 +1 @@
-{"version":3,"file":"convert-schema-from-parquet.js","sourceRoot":"","sources":["../../../src/lib/arrow/convert-schema-from-parquet.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAQpC,MAAM,CAAC,MAAM,oBAAoB,GAAsC;IACrE,OAAO,EAAE,MAAM;IACf,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,SAAS;IAChB,KAAK,EAAE,SAAS;IAChB,KAAK,EAAE,SAAS;IAChB,MAAM,EAAE,SAAS;IACjB,UAAU,EAAE,QAAQ;IACpB,oBAAoB,EAAE,QAAQ;IAC9B,IAAI,EAAE,MAAM;IACZ,IAAI,EAAE,OAAO;IACb,WAAW,EAAE,OAAO;IACpB,WAAW,EAAE,OAAO;IACpB,gBAAgB,EAAE,OAAO;IACzB,gBAAgB,EAAE,OAAO;IACzB,MAAM,EAAE,OAAO;IACf,OAAO,EAAE,QAAQ;IACjB,OAAO,EAAE,QAAQ;IACjB,OAAO,EAAE,QAAQ;IACjB,KAAK,EAAE,MAAM;IACb,MAAM,EAAE,OAAO;IACf,MAAM,EAAE,OAAO;IACf,MAAM,EAAE,OAAO;IACf,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,QAAQ;IACd,0BAA0B;IAC1B,QAAQ,EAAE,QAAQ;IAClB,aAAa,EAAE,SAAS;IACxB,aAAa,EAAE,SAAS;IACxB,kBAAkB,EAAE,SAAS;IAC7B,4BAA4B,EAAE,SAAS;CACxC,CAAC;AAEF,MAAM,UAAU,oBAAoB,CAClC,aAA4B,EAC5B,eAAoC;IAEpC,MAAM,MAAM,GAAG,SAAS,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,eAAe,IAAI,iBAAiB,CAAC,eAAe,CAAC,CAAC;IAEvE,MAAM,MAAM,GAAW;QACrB,MAAM;QACN,QAAQ,EAAE,QAAQ,IAAI,EAAE;KACzB,CAAC;IAEF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,SAAS,CAAC,MAAuB;IACxC,MAAM,MAAM,GAAY,EAAE,CAAC;IAE3B,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAE3B,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACzC,MAAM,CAAC,IAAI,CAAC,EAAC,IAAI,EAAE,IAAI,EAAE,EAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAC,CAAC,CAAC;QAClF,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,GAAG,oBAAoB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,QAAQ,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;YACzC,MAAM,UAAU,GAAG,EAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,QAAQ,EAAC,CAAC;YACpE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAmB;IAC3C,IAAI,QAA4C,CAAC;IAEjD,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;QACxB,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;YACnB,IAAI,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;YAC7B,KAAK,GAAG,OAAO,KAAK,CAAC,GAAG,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACjF,QAAQ,GAAG,QAAQ,IAAI,EAAE,CAAC;YAC1B,QAAQ,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACxB,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,iBAAiB,CAAC,eAA6B;IACtD,IAAI,QAA4C,CAAC;IAEjD,MAAM,YAAY,GAAG,eAAe,CAAC,kBAAkB,IAAI,EAAE,CAAC;IAC9D,KAAK,MAAM,EAAC,GAAG,EAAE,KAAK,EAAC,IAAI,YAAY,EAAE,CAAC;QACxC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,QAAQ,GAAG,QAAQ,IAAI,EAAE,CAAC;YAC1B,QAAQ,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACxB,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}
+{"version":3,"file":"convert-schema-from-parquet.js","sourceRoot":"","sources":["../../../src/lib/arrow/convert-schema-from-parquet.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAQpC,MAAM,CAAC,MAAM,oBAAoB,GAAsC;IACrE,OAAO,EAAE,MAAM;IACf,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,SAAS;IAChB,KAAK,EAAE,SAAS;IAChB,MAAM,EAAE,SAAS;IACjB,UAAU,EAAE,QAAQ;IACpB,oBAAoB,EAAE,QAAQ;IAC9B,IAAI,EAAE,MAAM;IACZ,IAAI,EAAE,MAAM;IACZ,IAAI,EAAE,UAAU;IAChB,WAAW,EAAE,kBAAkB;IAC/B,WAAW,EAAE,kBAAkB;IAC/B,UAAU,EAAE,iBAAiB;IAC7B,gBAAgB,EAAE,uBAAuB;IACzC,gBAAgB,EAAE,uBAAuB;IACzC,eAAe,EAAE,sBAAsB;IACvC,MAAM,EAAE,OAAO;IACf,OAAO,EAAE,QAAQ;IACjB,OAAO,EAAE,QAAQ;IACjB,OAAO,EAAE,QAAQ;IACjB,KAAK,EAAE,MAAM;IACb,MAAM,EAAE,OAAO;IACf,MAAM,EAAE,OAAO;IACf,MAAM,EAAE,OAAO;IACf,IAAI,EAAE,EAAC,IAAI,EAAE,mBAAmB,EAAE,SAAS,EAAE,EAAE,EAAC;IAChD,OAAO,EAAE,SAAS;IAClB,OAAO,EAAE,MAAM;IACf,OAAO,EAAE,QAAQ;IACjB,QAAQ,EAAE,QAAQ;IAClB,SAAS,EAAE,QAAQ;IACnB,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,QAAQ;IACd,0BAA0B;IAC1B,QAAQ,EAAE,QAAQ;IAClB,aAAa,EAAE,EAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAC;IACxE,aAAa,EAAE,EAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAC;IACxE,kBAAkB,EAAE,EAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAC;IAC7E,4BAA4B,EAAE,EAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAC;CACxF,CAAC;AAEF,MAAM,UAAU,oBAAoB,CAClC,aAA4B,EAC5B,eAAoC;IAEpC,MAAM,MAAM,GAAG,SAAS,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,eAAe,IAAI,iBAAiB,CAAC,eAAe,CAAC,CAAC;IAEvE,MAAM,MAAM,GAAW;QACrB,MAAM;QACN,QAAQ,EAAE,QAAQ,IAAI,EAAE;KACzB,CAAC;IAEF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,SAAS,CAAC,MAAwB;IACzC,MAAM,MAAM,GAAY,EAAE,CAAC;IAE3B,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,6EAA6E;AAC7E,SAAS,QAAQ,CAAC,IAAY,EAAE,KAAsB;IACpD,MAAM,YAAY,GAAU,KAAK,CAAC,MAAM;QACtC,CAAC,CAAC;YACE,IAAI;YACJ,IAAI,EAAE,EAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,EAAC;YACzD,QAAQ,EAAE,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC;YACjC,QAAQ,EAAE,gBAAgB,CAAC,KAAK,CAAC;SAClC;QACH,CAAC,CAAC;YACE,IAAI;YACJ,IAAI,EAAE,YAAY,CAAC,KAAK,CAAC;YACzB,QAAQ,EAAE,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC;YACjC,QAAQ,EAAE,gBAAgB,CAAC,KAAK,CAAC;SAClC,CAAC;IAEN,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;QACpB,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,OAAO;QACL,IAAI;QACJ,IAAI,EAAE;YACJ,IAAI,EAAE,MAAM;YACZ,QAAQ,EAAE,CAAC,EAAC,GAAG,YAAY,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,EAAC,CAAC;SAChE;QACD,QAAQ,EAAE,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC;QACjC,QAAQ,EAAE,YAAY,CAAC,QAAQ;KAChC,CAAC;AACJ,CAAC;AAED,6EAA6E;AAC7E,SAAS,YAAY,CAAC,KAAsB;IAC1C,IAAI,KAAK,CAAC,WAAW,EAAE,IAAI,KAAK,SAAS,EAAE,CAAC;QAC1C,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,SAAS,CAAC;QACrD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;QAC1B,IAAI,CAAC,SAAS,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;QAC/E,CAAC;QACD,IAAI,SAAS,GAAG,EAAE,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,6BAA6B,SAAS,2BAA2B,CAAC,CAAC;QACrF,CAAC;QACD,OAAO,EAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,KAAK,EAAC,CAAC;IACpF,CAAC;IACD,IAAI,KAAK,CAAC,WAAW,EAAE,IAAI,KAAK,MAAM,EAAE,CAAC;QACvC,OAAO,EAAC,IAAI,EAAE,mBAAmB,EAAE,SAAS,EAAE,KAAK,CAAC,UAAU,IAAI,EAAE,EAAC,CAAC;IACxE,CAAC;IACD,OAAO,oBAAoB,CAAC,KAAK,CAAC,IAAK,CAAC,CAAC;AAC3C,CAAC;AAED,qFAAqF;AACrF,SAAS,gBAAgB,CAAC,KAAsB;IAC9C,IAAI,QAA4C,CAAC;IAEjD,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;QACxB,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;QAC9B,IACE,GAAG,KAAK,MAAM;YACd,GAAG,KAAK,QAAQ;YAChB,CAAC,GAAG,KAAK,cAAc,IAAI,UAAU,KAAK,KAAK,CAAC,IAAI,CAAC;YACrD,CAAC,GAAG,KAAK,WAAW,IAAI,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC;YACtD,UAAU,KAAK,SAAS,EACxB,CAAC;YACD,SAAS;QACX,CAAC;QACD,MAAM,aAAa,GACjB,OAAO,UAAU,KAAK,QAAQ;YAC5B,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,OAAO,UAAU,KAAK,SAAS;gBAC/B,CAAC,CAAC,UAAU;oBACV,CAAC,CAAC,MAAM;oBACR,CAAC,CAAC,OAAO;gBACX,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QACnC,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;YAChC,SAAS;QACX,CAAC;QACD,QAAQ,KAAK,EAAE,CAAC;QAChB,QAAQ,CAAC,GAAG,CAAC,GAAG,aAAa,CAAC;IAChC,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,iBAAiB,CAAC,eAA6B;IACtD,IAAI,QAA4C,CAAC;IAEjD,MAAM,YAAY,GAAG,eAAe,CAAC,kBAAkB,IAAI,EAAE,CAAC;IAC9D,KAAK,MAAM,EAAC,GAAG,EAAE,KAAK,EAAC,IAAI,YAAY,EAAE,CAAC;QACxC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,QAAQ,GAAG,QAAQ,IAAI,EAAE,CAAC;YAC1B,QAAQ,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACxB,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/dist/lib/arrow/convert-schema-to-parquet.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"convert-schema-to-parquet.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/convert-schema-to-parquet.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAEV,WAAW,EACZ,0CAAuC;AAExC,OAAO,EACL,MAAM,EAGN,QAAQ,EACT,MAAM,oBAAoB,CAAC;AAE5B,eAAO,MAAM,oBAAoB,EAAE;KAAE,IAAI,IAAI,WAAW,GAAG,QAAQ;CA+BlE,CAAC;AAEF,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAK7D"}
+{"version":3,"file":"convert-schema-to-parquet.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/convert-schema-to-parquet.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAEV,WAAW,EACZ,0CAAuC;AAExC,OAAO,EACL,MAAM,EAGN,QAAQ,EACT,MAAM,oBAAoB,CAAC;AAE5B,eAAO,MAAM,oBAAoB,EAAE;KAAE,IAAI,IAAI,WAAW,GAAG,QAAQ;CAwClE,CAAC;AAEF,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAK7D"}

package/dist/lib/arrow/convert-schema-to-parquet.js

@@ -4,19 +4,22 @@
 export const PARQUET_TYPE_MAPPING = {
     BOOLEAN: 'bool',
     INT32: 'int32',
-    INT64: 'float64',
+    INT64: 'int64',
     INT96: 'float64',
     FLOAT: 'float32',
     DOUBLE: 'float64',
     BYTE_ARRAY: 'binary',
     FIXED_LEN_BYTE_ARRAY: 'binary',
     UTF8: 'utf8',
-    DATE: 'int32',
-    TIME_MILLIS: 'int64',
-    TIME_MICROS: 'int64',
-    TIMESTAMP_MILLIS: 'int64',
-    TIMESTAMP_MICROS: 'int64',
-    UINT_8: 'int32',
+    ENUM: 'utf8',
+    DATE: 'date-day',
+    TIME_MILLIS: 'time-millisecond',
+    TIME_MICROS: 'time-microsecond',
+    TIME_NANOS: 'time-nanosecond',
+    TIMESTAMP_MILLIS: 'timestamp-millisecond',
+    TIMESTAMP_MICROS: 'timestamp-microsecond',
+    TIMESTAMP_NANOS: 'timestamp-nanosecond',
+    UINT_8: 'uint8',
     UINT_16: 'uint16',
     UINT_32: 'uint32',
     UINT_64: 'uint64',
@@ -24,6 +27,12 @@ export const PARQUET_TYPE_MAPPING = {
     INT_16: 'int16',
     INT_32: 'int32',
     INT_64: 'int64',
+    UUID: { type: 'fixed-size-binary', byteWidth: 16 },
+    FLOAT16: 'float16',
+    UNKNOWN: 'null',
+    VARIANT: 'binary',
+    GEOMETRY: 'binary',
+    GEOGRAPHY: 'binary',
     JSON: 'binary',
     BSON: 'binary',
     // TODO check interval type

package/dist/lib/arrow/convert-schema-to-parquet.js.map

@@ -1 +1 @@
-{"version":3,"file":"convert-schema-to-parquet.js","sourceRoot":"","sources":["../../../src/lib/arrow/convert-schema-to-parquet.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAepC,MAAM,CAAC,MAAM,oBAAoB,GAAsC;IACrE,OAAO,EAAE,MAAM;IACf,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,SAAS;IAChB,KAAK,EAAE,SAAS;IAChB,KAAK,EAAE,SAAS;IAChB,MAAM,EAAE,SAAS;IACjB,UAAU,EAAE,QAAQ;IACpB,oBAAoB,EAAE,QAAQ;IAC9B,IAAI,EAAE,MAAM;IACZ,IAAI,EAAE,OAAO;IACb,WAAW,EAAE,OAAO;IACpB,WAAW,EAAE,OAAO;IACpB,gBAAgB,EAAE,OAAO;IACzB,gBAAgB,EAAE,OAAO;IACzB,MAAM,EAAE,OAAO;IACf,OAAO,EAAE,QAAQ;IACjB,OAAO,EAAE,QAAQ;IACjB,OAAO,EAAE,QAAQ;IACjB,KAAK,EAAE,MAAM;IACb,MAAM,EAAE,OAAO;IACf,MAAM,EAAE,OAAO;IACf,MAAM,EAAE,OAAO;IACf,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,QAAQ;IACd,2BAA2B;IAC3B,QAAQ,EAAE,QAAQ;IAClB,aAAa,EAAE,SAAS;IACxB,aAAa,EAAE,SAAS;IACxB,kBAAkB,EAAE,SAAS;IAC7B,4BAA4B,EAAE,SAAS;CACxC,CAAC;AAEF,MAAM,UAAU,sBAAsB,CAAC,MAAc;IACnD,MAAM,MAAM,GAAG,EAAE,CAAC,CAAC,4BAA4B;IAE/C,+BAA+B;IAC/B,OAAO,EAAC,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAC,CAAC;AAChC,CAAC;AAED,sDAAsD;AACtD,gCAAgC;AAEhC,iCAAiC;AACjC,kCAAkC;AAElC,oBAAoB;AACpB,qDAAqD;AACrD,sBAAsB;AACtB,oDAAoD;AACpD,qFAAqF;AACrF,kCAAkC;AAClC,eAAe;AACf,4DAA4D;AAC5D,kDAAkD;AAClD,uFAAuF;AACvF,iCAAiC;AACjC,QAAQ;AACR,MAAM;AAEN,mBAAmB;AACnB,IAAI;AAEJ,wEAAwE;AACxE,gCAAgC;AAEhC,+BAA+B;AAC/B,4BAA4B;AAC5B,gGAAgG;AAChG,kCAAkC;AAClC,QAAQ;AACR,MAAM;AAEN,qBAAqB;AACrB,IAAI"}
+{"version":3,"file":"convert-schema-to-parquet.js","sourceRoot":"","sources":["../../../src/lib/arrow/convert-schema-to-parquet.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAepC,MAAM,CAAC,MAAM,oBAAoB,GAAsC;IACrE,OAAO,EAAE,MAAM;IACf,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,SAAS;IAChB,KAAK,EAAE,SAAS;IAChB,MAAM,EAAE,SAAS;IACjB,UAAU,EAAE,QAAQ;IACpB,oBAAoB,EAAE,QAAQ;IAC9B,IAAI,EAAE,MAAM;IACZ,IAAI,EAAE,MAAM;IACZ,IAAI,EAAE,UAAU;IAChB,WAAW,EAAE,kBAAkB;IAC/B,WAAW,EAAE,kBAAkB;IAC/B,UAAU,EAAE,iBAAiB;IAC7B,gBAAgB,EAAE,uBAAuB;IACzC,gBAAgB,EAAE,uBAAuB;IACzC,eAAe,EAAE,sBAAsB;IACvC,MAAM,EAAE,OAAO;IACf,OAAO,EAAE,QAAQ;IACjB,OAAO,EAAE,QAAQ;IACjB,OAAO,EAAE,QAAQ;IACjB,KAAK,EAAE,MAAM;IACb,MAAM,EAAE,OAAO;IACf,MAAM,EAAE,OAAO;IACf,MAAM,EAAE,OAAO;IACf,IAAI,EAAE,EAAC,IAAI,EAAE,mBAAmB,EAAE,SAAS,EAAE,EAAE,EAAC;IAChD,OAAO,EAAE,SAAS;IAClB,OAAO,EAAE,MAAM;IACf,OAAO,EAAE,QAAQ;IACjB,QAAQ,EAAE,QAAQ;IAClB,SAAS,EAAE,QAAQ;IACnB,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,QAAQ;IACd,2BAA2B;IAC3B,QAAQ,EAAE,QAAQ;IAClB,aAAa,EAAE,SAAS;IACxB,aAAa,EAAE,SAAS;IACxB,kBAAkB,EAAE,SAAS;IAC7B,4BAA4B,EAAE,SAAS;CACxC,CAAC;AAEF,MAAM,UAAU,sBAAsB,CAAC,MAAc;IACnD,MAAM,MAAM,GAAG,EAAE,CAAC,CAAC,4BAA4B;IAE/C,+BAA+B;IAC/B,OAAO,EAAC,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAC,CAAC;AAChC,CAAC;AAED,sDAAsD;AACtD,gCAAgC;AAEhC,iCAAiC;AACjC,kCAAkC;AAElC,oBAAoB;AACpB,qDAAqD;AACrD,sBAAsB;AACtB,oDAAoD;AACpD,qFAAqF;AACrF,kCAAkC;AAClC,eAAe;AACf,4DAA4D;AAC5D,kDAAkD;AAClD,uFAAuF;AACvF,iCAAiC;AACjC,QAAQ;AACR,MAAM;AAEN,mBAAmB;AACnB,IAAI;AAEJ,wEAAwE;AACxE,gCAAgC;AAEhC,+BAA+B;AAC/B,4BAA4B;AAC5B,gGAAgG;AAChG,kCAAkC;AAClC,QAAQ;AACR,MAAM;AAEN,qBAAqB;AACrB,IAAI"}

package/dist/lib/arrow/create-nested-arrow-vector.d.ts

@@ -0,0 +1,8 @@
+import * as arrow from 'apache-arrow';
+import type { ParquetField, ParquetRowGroup } from "../../parquetjs/schema/declare.js";
+/**
+ * Creates an Arrow List vector directly from a repeated Parquet primitive or repeated struct.
+ * Returns undefined for shapes whose repetition tree is not supported by the direct path.
+ */
+export declare function createNestedArrowVector(arrowType: arrow.DataType, parquetField: ParquetField, rowGroup: ParquetRowGroup, start: number, end: number): arrow.Vector | undefined;
+//# sourceMappingURL=create-nested-arrow-vector.d.ts.map

package/dist/lib/arrow/create-nested-arrow-vector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"create-nested-arrow-vector.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/create-nested-arrow-vector.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,KAAK,MAAM,cAAc,CAAC;AAEtC,OAAO,KAAK,EAEV,YAAY,EACZ,eAAe,EAChB,0CAAuC;AAgCxC;;;GAGG;AACH,wBAAgB,uBAAuB,CACrC,SAAS,EAAE,KAAK,CAAC,QAAQ,EACzB,YAAY,EAAE,YAAY,EAC1B,QAAQ,EAAE,eAAe,EACzB,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,GACV,KAAK,CAAC,MAAM,GAAG,SAAS,CAS1B"}