@loaders.gl/parquet 4.0.0-alpha.9 → 4.0.0-beta.2

package/dist/es5/lib/geo/decode-geo-metadata.js

@@ -5,21 +5,40 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.getGeoMetadata = getGeoMetadata;
+exports.parseJSONStringMetadata = parseJSONStringMetadata;
 exports.setGeoMetadata = setGeoMetadata;
 exports.unpackGeoMetadata = unpackGeoMetadata;
+exports.unpackJSONStringMetadata = unpackJSONStringMetadata;
 var _slicedToArray2 = _interopRequireDefault(require("@babel/runtime/helpers/slicedToArray"));
-function getGeoMetadata(schema) {
-  var stringifiedGeoMetadata = schema.metadata.geo;
-  if (!stringifiedGeoMetadata) {
+var _typeof2 = _interopRequireDefault(require("@babel/runtime/helpers/typeof"));
+function parseJSONStringMetadata(schema, metadataKey) {
+  var stringifiedMetadata = schema.metadata[metadataKey];
+  if (!stringifiedMetadata) {
     return null;
   }
   try {
-    var geoMetadata = JSON.parse(stringifiedGeoMetadata);
-    return geoMetadata;
+    var metadata = JSON.parse(stringifiedMetadata);
+    if (!metadata || (0, _typeof2.default)(metadata) !== 'object') {
+      return null;
+    }
+    return metadata;
   } catch (_unused) {
     return null;
   }
 }
+function unpackJSONStringMetadata(schema, metadataKey) {
+  var json = parseJSONStringMetadata(schema, metadataKey);
+  for (var _i = 0, _Object$entries = Object.entries(json || {}); _i < _Object$entries.length; _i++) {
+    var _Object$entries$_i = (0, _slicedToArray2.default)(_Object$entries[_i], 2),
+      _key = _Object$entries$_i[0],
+      value = _Object$entries$_i[1];
+    schema.metadata["".concat(metadataKey, ".").concat(_key)] = typeof value === 'string' ? value : JSON.stringify(value);
+  }
+}
+function getGeoMetadata(schema) {
+  var geoMetadata = parseJSONStringMetadata(schema, 'geo');
+  return geoMetadata;
+}
 function setGeoMetadata(schema, geoMetadata) {
   var stringifiedGeoMetadata = JSON.stringify(geoMetadata);
   schema.metadata.geo = stringifiedGeoMetadata;
@@ -40,9 +59,9 @@ function unpackGeoMetadata(schema) {
   }
   schema.metadata['geo.columns'] = Object.keys(columns || {}).join('');
   var _loop = function _loop() {
-    var _Object$entries$_i = (0, _slicedToArray2.default)(_Object$entries[_i], 2),
-      columnName = _Object$entries$_i[0],
-      columnMetadata = _Object$entries$_i[1];
+    var _Object$entries2$_i = (0, _slicedToArray2.default)(_Object$entries2[_i2], 2),
+      columnName = _Object$entries2$_i[0],
+      columnMetadata = _Object$entries2$_i[1];
     var field = schema.fields.find(function (field) {
       return field.name === columnName;
     });
@@ -53,24 +72,41 @@ function unpackGeoMetadata(schema) {
       unpackGeoFieldMetadata(field, columnMetadata);
     }
   };
-  for (var _i = 0, _Object$entries = Object.entries(columns || {}); _i < _Object$entries.length; _i++) {
+  for (var _i2 = 0, _Object$entries2 = Object.entries(columns || {}); _i2 < _Object$entries2.length; _i2++) {
     _loop();
   }
 }
 function unpackGeoFieldMetadata(field, columnMetadata) {
-  for (var _i2 = 0, _Object$entries2 = Object.entries(columnMetadata || {}); _i2 < _Object$entries2.length; _i2++) {
-    var _Object$entries2$_i = (0, _slicedToArray2.default)(_Object$entries2[_i2], 2),
-      _key = _Object$entries2$_i[0],
-      value = _Object$entries2$_i[1];
-    switch (_key) {
+  for (var _i3 = 0, _Object$entries3 = Object.entries(columnMetadata || {}); _i3 < _Object$entries3.length; _i3++) {
+    var _Object$entries3$_i = (0, _slicedToArray2.default)(_Object$entries3[_i3], 2),
+      _key2 = _Object$entries3$_i[0],
+      value = _Object$entries3$_i[1];
+    switch (_key2) {
       case 'geometry_type':
-        setFieldMetadata(field, "geo.".concat(_key), value.join(','));
+        setFieldMetadata(field, "geo.".concat(_key2), value.join(','));
         break;
       case 'bbox':
+        setFieldMetadata(field, "geo.crs.".concat(_key2), JSON.stringify(value));
+        break;
       case 'crs':
+        for (var _i4 = 0, _Object$entries4 = Object.entries(value || {}); _i4 < _Object$entries4.length; _i4++) {
+          var _Object$entries4$_i = (0, _slicedToArray2.default)(_Object$entries4[_i4], 2),
+            crsKey = _Object$entries4$_i[0],
+            crsValue = _Object$entries4$_i[1];
+          switch (crsKey) {
+            case 'id':
+              var crsId = (0, _typeof2.default)(crsValue) === 'object' ? "".concat(crsValue === null || crsValue === void 0 ? void 0 : crsValue.authority, ":").concat(crsValue === null || crsValue === void 0 ? void 0 : crsValue.code) : JSON.stringify(crsValue);
+              setFieldMetadata(field, "geo.crs.".concat(crsKey), crsId);
+              break;
+            default:
+              setFieldMetadata(field, "geo.crs.".concat(crsKey), typeof crsValue === 'string' ? crsValue : JSON.stringify(crsValue));
+              break;
+          }
+        }
+        break;
       case 'edges':
       default:
-        setFieldMetadata(field, "geo.".concat(_key), typeof value === 'string' ? value : JSON.stringify(value));
+        setFieldMetadata(field, "geo.".concat(_key2), typeof value === 'string' ? value : JSON.stringify(value));
     }
   }
 }

package/dist/es5/lib/geo/decode-geo-metadata.js.map

@@ -1 +1 @@
-{"version":3,"file":"decode-geo-metadata.js","names":["getGeoMetadata","schema","stringifiedGeoMetadata","metadata","geo","geoMetadata","JSON","parse","_unused","setGeoMetadata","stringify","unpackGeoMetadata","version","primary_column","columns","Object","keys","join","_loop","_Object$entries$_i","_slicedToArray2","default","_Object$entries","_i","columnName","columnMetadata","field","fields","find","name","setFieldMetadata","unpackGeoFieldMetadata","entries","length","_i2","_Object$entries2","_Object$entries2$_i","key","value","concat"],"sources":["../../../../src/lib/geo/decode-geo-metadata.ts"],"sourcesContent":["// loaders.gl, MIT license\nimport {Schema, Field} from '@loaders.gl/schema';\n\n/* eslint-disable camelcase */\n\n/** A geoarrow / geoparquet geo metadata object (stored in stringified form in the top level metadata 'geo' key) */\nexport type GeoMetadata = {\n  version?: string;\n  primary_column?: string;\n  columns: Record<string, GeoColumnMetadata>;\n  [key: string]: unknown;\n};\n\n/** A geoarrow / geoparquet geo metadata for one geometry column  */\nexport type GeoColumnMetadata = {\n  bounding_box?:\n    | [number, number, number, number]\n    | [number, number, number, number, number, number];\n  crs?: string;\n  geometry_type?: string[];\n  edges?: string;\n  [key: string]: unknown;\n};\n\n/**\n * Reads the GeoMetadata object from the metadata\n * @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata */\nexport function getGeoMetadata(schema: Schema): GeoMetadata | null {\n  const stringifiedGeoMetadata = schema.metadata.geo;\n  if (!stringifiedGeoMetadata) {\n    return null;\n  }\n\n  try {\n    const geoMetadata = JSON.parse(stringifiedGeoMetadata) as GeoMetadata;\n    return geoMetadata;\n  } catch {\n    return null;\n  }\n}\n\n/**\n * Stores a geoarrow / geoparquet geo metadata object in the schema\n * @note geoarrow / geoparquet geo metadata is a single stringified JSON field\n */\nexport function setGeoMetadata(schema: Schema, geoMetadata: GeoMetadata): void {\n  const stringifiedGeoMetadata = JSON.stringify(geoMetadata);\n  schema.metadata.geo = stringifiedGeoMetadata;\n}\n\n/**\n * Unpacks geo metadata into separate metadata fields (parses the long JSON string)\n * @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata\n */\nexport function unpackGeoMetadata(schema: Schema): void {\n  const geoMetadata = getGeoMetadata(schema);\n  if (!geoMetadata) {\n    return;\n  }\n\n  // Store Parquet Schema Level Metadata\n\n  const {version, primary_column, columns} = geoMetadata;\n  if (version) {\n    schema.metadata['geo.version'] = version;\n  }\n\n  if (primary_column) {\n    schema.metadata['geo.primary_column'] = primary_column;\n  }\n\n  // store column names as comma separated list\n  schema.metadata['geo.columns'] = Object.keys(columns || {}).join('');\n\n  for (const [columnName, columnMetadata] of Object.entries(columns || {})) {\n    const field = schema.fields.find((field) => field.name === columnName);\n    if (field) {\n      if (field.name === primary_column) {\n        setFieldMetadata(field, 'geo.primary_field', 'true');\n      }\n      unpackGeoFieldMetadata(field, columnMetadata);\n    }\n  }\n}\n\nfunction unpackGeoFieldMetadata(field: Field, columnMetadata): void {\n  for (const [key, value] of Object.entries(columnMetadata || {})) {\n    switch (key) {\n      case 'geometry_type':\n        setFieldMetadata(field, `geo.${key}`, (value as string[]).join(','));\n        break;\n      case 'bbox':\n      case 'crs':\n      case 'edges':\n      default:\n        setFieldMetadata(\n          field,\n          `geo.${key}`,\n          typeof value === 'string' ? value : JSON.stringify(value)\n        );\n    }\n  }\n}\n\nfunction setFieldMetadata(field: Field, key: string, value: string): void {\n  field.metadata = field.metadata || {};\n  field.metadata[key] = value;\n}\n"],"mappings":";;;;;;;;;;AA2BO,SAASA,cAAcA,CAACC,MAAc,EAAsB;EACjE,IAAMC,sBAAsB,GAAGD,MAAM,CAACE,QAAQ,CAACC,GAAG;EAClD,IAAI,CAACF,sBAAsB,EAAE;IAC3B,OAAO,IAAI;EACb;EAEA,IAAI;IACF,IAAMG,WAAW,GAAGC,IAAI,CAACC,KAAK,CAACL,sBAAsB,CAAgB;IACrE,OAAOG,WAAW;EACpB,CAAC,CAAC,OAAAG,OAAA,EAAM;IACN,OAAO,IAAI;EACb;AACF;AAMO,SAASC,cAAcA,CAACR,MAAc,EAAEI,WAAwB,EAAQ;EAC7E,IAAMH,sBAAsB,GAAGI,IAAI,CAACI,SAAS,CAACL,WAAW,CAAC;EAC1DJ,MAAM,CAACE,QAAQ,CAACC,GAAG,GAAGF,sBAAsB;AAC9C;AAMO,SAASS,iBAAiBA,CAACV,MAAc,EAAQ;EACtD,IAAMI,WAAW,GAAGL,cAAc,CAACC,MAAM,CAAC;EAC1C,IAAI,CAACI,WAAW,EAAE;IAChB;EACF;EAIA,IAAOO,OAAO,GAA6BP,WAAW,CAA/CO,OAAO;IAAEC,cAAc,GAAaR,WAAW,CAAtCQ,cAAc;IAAEC,OAAO,GAAIT,WAAW,CAAtBS,OAAO;EACvC,IAAIF,OAAO,EAAE;IACXX,MAAM,CAACE,QAAQ,CAAC,aAAa,CAAC,GAAGS,OAAO;EAC1C;EAEA,IAAIC,cAAc,EAAE;IAClBZ,MAAM,CAACE,QAAQ,CAAC,oBAAoB,CAAC,GAAGU,cAAc;EACxD;EAGAZ,MAAM,CAACE,QAAQ,CAAC,aAAa,CAAC,GAAGY,MAAM,CAACC,IAAI,CAACF,OAAO,IAAI,CAAC,CAAC,CAAC,CAACG,IAAI,CAAC,EAAE,CAAC;EAAC,IAAAC,KAAA,YAAAA,MAAA,EAEK;IAArE,IAAAC,kBAAA,OAAAC,eAAA,CAAAC,OAAA,EAAAC,eAAA,CAAAC,EAAA;MAAOC,UAAU,GAAAL,kBAAA;MAAEM,cAAc,GAAAN,kBAAA;IACpC,IAAMO,KAAK,GAAGzB,MAAM,CAAC0B,MAAM,CAACC,IAAI,CAAC,UAACF,KAAK;MAAA,OAAKA,KAAK,CAACG,IAAI,KAAKL,UAAU;IAAA,EAAC;IACtE,IAAIE,KAAK,EAAE;MACT,IAAIA,KAAK,CAACG,IAAI,KAAKhB,cAAc,EAAE;QACjCiB,gBAAgB,CAACJ,KAAK,EAAE,mBAAmB,EAAE,MAAM,CAAC;MACtD;MACAK,sBAAsB,CAACL,KAAK,EAAED,cAAc,CAAC;IAC/C;EACF,CAAC;EARD,SAAAF,EAAA,MAAAD,eAAA,GAA2CP,MAAM,CAACiB,OAAO,CAAClB,OAAO,IAAI,CAAC,CAAC,CAAC,EAAAS,EAAA,GAAAD,eAAA,CAAAW,MAAA,EAAAV,EAAA;IAAAL,KAAA;EAAA;AAS1E;AAEA,SAASa,sBAAsBA,CAACL,KAAY,EAAED,cAAc,EAAQ;EAClE,SAAAS,GAAA,MAAAC,gBAAA,GAA2BpB,MAAM,CAACiB,OAAO,CAACP,cAAc,IAAI,CAAC,CAAC,CAAC,EAAAS,GAAA,GAAAC,gBAAA,CAAAF,MAAA,EAAAC,GAAA,IAAE;IAA5D,IAAAE,mBAAA,OAAAhB,eAAA,CAAAC,OAAA,EAAAc,gBAAA,CAAAD,GAAA;MAAOG,IAAG,GAAAD,mBAAA;MAAEE,KAAK,GAAAF,mBAAA;IACpB,QAAQC,IAAG;MACT,KAAK,eAAe;QAClBP,gBAAgB,CAACJ,KAAK,SAAAa,MAAA,CAASF,IAAG,GAAKC,KAAK,CAAcrB,IAAI,CAAC,GAAG,CAAC,CAAC;QACpE;MACF,KAAK,MAAM;MACX,KAAK,KAAK;MACV,KAAK,OAAO;MACZ;QACEa,gBAAgB,CACdJ,KAAK,SAAAa,MAAA,CACEF,IAAG,GACV,OAAOC,KAAK,KAAK,QAAQ,GAAGA,KAAK,GAAGhC,IAAI,CAACI,SAAS,CAAC4B,KAAK,CAC1D,CAAC;IACL;EACF;AACF;AAEA,SAASR,gBAAgBA,CAACJ,KAAY,EAAEW,GAAW,EAAEC,KAAa,EAAQ;EACxEZ,KAAK,CAACvB,QAAQ,GAAGuB,KAAK,CAACvB,QAAQ,IAAI,CAAC,CAAC;EACrCuB,KAAK,CAACvB,QAAQ,CAACkC,GAAG,CAAC,GAAGC,KAAK;AAC7B"}
+{"version":3,"file":"decode-geo-metadata.js","names":["parseJSONStringMetadata","schema","metadataKey","stringifiedMetadata","metadata","JSON","parse","_typeof2","default","_unused","unpackJSONStringMetadata","json","_i","_Object$entries","Object","entries","length","_Object$entries$_i","_slicedToArray2","key","value","concat","stringify","getGeoMetadata","geoMetadata","setGeoMetadata","stringifiedGeoMetadata","geo","unpackGeoMetadata","version","primary_column","columns","keys","join","_loop","_Object$entries2$_i","_Object$entries2","_i2","columnName","columnMetadata","field","fields","find","name","setFieldMetadata","unpackGeoFieldMetadata","_i3","_Object$entries3","_Object$entries3$_i","_i4","_Object$entries4","_Object$entries4$_i","crsKey","crsValue","crsId","authority","code"],"sources":["../../../../src/lib/geo/decode-geo-metadata.ts"],"sourcesContent":["// loaders.gl, MIT license\nimport {Schema, Field} from '@loaders.gl/schema';\n\n/* eslint-disable camelcase */\n\ntype GeometryType =\n  | 'Point'\n  | 'LineString'\n  | 'Polygon'\n  | 'MultiPoint'\n  | 'MultiLineString'\n  | 'MultiPolygon'\n  | 'GeometryCollection'\n  | 'Point Z'\n  | 'LineString Z'\n  | 'Polygon Z'\n  | 'MultiPoint Z'\n  | 'MultiLineString Z'\n  | 'MultiPolygon Z'\n  | 'GeometryCollection Z';\n\n/**\n * A geoarrow / geoparquet geo metadata object\n * (stored in stringified form in the top level metadata 'geo' key)\n * @see https://github.com/opengeospatial/geoparquet/blob/main/format-specs/geoparquet.md\n * @see https://github.com/geoarrow/geoarrow/blob/main/metadata.md\n * */\nexport type GeoMetadata = {\n  version?: string;\n  primary_column?: string;\n  columns: Record<string, GeoColumnMetadata>;\n  [key: string]: unknown;\n};\n\n/** A geoarrow / geoparquet geo metadata for one geometry column  */\nexport type GeoColumnMetadata = {\n  encoding: 'wkb' | 'wkt';\n  geometry_types: GeometryType[];\n  crs?: object | null;\n  orientation?: 'counterclockwise';\n  bbox?: [number, number, number, number] | [number, number, number, number, number, number];\n  edges?: 'planar' | 'spherical';\n  epoch?: number;\n  [key: string]: unknown;\n};\n\n/** Parse a key with stringified arrow metadata */\nexport function parseJSONStringMetadata(\n  schema: Schema,\n  metadataKey: string\n): Record<string, unknown> | null {\n  const stringifiedMetadata = schema.metadata[metadataKey];\n  if (!stringifiedMetadata) {\n    return null;\n  }\n\n  try {\n    const metadata = JSON.parse(stringifiedMetadata);\n    if (!metadata || typeof metadata !== 'object') {\n      return null;\n    }\n    return metadata;\n  } catch {\n    return null;\n  }\n}\n\nexport function unpackJSONStringMetadata(schema: Schema, metadataKey: string): void {\n  const json = parseJSONStringMetadata(schema, metadataKey);\n  for (const [key, value] of Object.entries(json || {})) {\n    schema.metadata[`${metadataKey}.${key}`] =\n      typeof value === 'string' ? value : JSON.stringify(value);\n  }\n}\n\n// GEO METADATA\n\n/**\n * Reads the GeoMetadata object from the metadata\n * @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata */\nexport function getGeoMetadata(schema: Schema): GeoMetadata | null {\n  const geoMetadata = parseJSONStringMetadata(schema, 'geo') as GeoMetadata;\n  return geoMetadata;\n}\n\n/**\n * Stores a geoarrow / geoparquet geo metadata object in the schema\n * @note geoarrow / geoparquet geo metadata is a single stringified JSON field\n */\nexport function setGeoMetadata(schema: Schema, geoMetadata: GeoMetadata): void {\n  const stringifiedGeoMetadata = JSON.stringify(geoMetadata);\n  schema.metadata.geo = stringifiedGeoMetadata;\n}\n\n/**\n * Unpacks geo metadata into separate metadata fields (parses the long JSON string)\n * @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata\n */\nexport function unpackGeoMetadata(schema: Schema): void {\n  const geoMetadata = getGeoMetadata(schema);\n  if (!geoMetadata) {\n    return;\n  }\n\n  // Store Parquet Schema Level Metadata\n\n  const {version, primary_column, columns} = geoMetadata;\n  if (version) {\n    schema.metadata['geo.version'] = version;\n  }\n\n  if (primary_column) {\n    schema.metadata['geo.primary_column'] = primary_column;\n  }\n\n  // store column names as comma separated list\n  schema.metadata['geo.columns'] = Object.keys(columns || {}).join('');\n\n  for (const [columnName, columnMetadata] of Object.entries(columns || {})) {\n    const field = schema.fields.find((field) => field.name === columnName);\n    if (field) {\n      if (field.name === primary_column) {\n        setFieldMetadata(field, 'geo.primary_field', 'true');\n      }\n      unpackGeoFieldMetadata(field, columnMetadata);\n    }\n  }\n}\n\n// eslint-disable-next-line complexity\nfunction unpackGeoFieldMetadata(field: Field, columnMetadata): void {\n  for (const [key, value] of Object.entries(columnMetadata || {})) {\n    switch (key) {\n      case 'geometry_type':\n        setFieldMetadata(field, `geo.${key}`, (value as string[]).join(','));\n        break;\n      case 'bbox':\n        setFieldMetadata(field, `geo.crs.${key}`, JSON.stringify(value));\n        break;\n      case 'crs':\n        for (const [crsKey, crsValue] of Object.entries(value || {})) {\n          switch (crsKey) {\n            case 'id':\n              const crsId =\n                typeof crsValue === 'object'\n                  ? `${crsValue?.authority}:${crsValue?.code}`\n                  : JSON.stringify(crsValue);\n              setFieldMetadata(field, `geo.crs.${crsKey}`, crsId);\n              break;\n            default:\n              setFieldMetadata(\n                field,\n                `geo.crs.${crsKey}`,\n                typeof crsValue === 'string' ? crsValue : JSON.stringify(crsValue)\n              );\n              break;\n          }\n        }\n        break;\n      case 'edges':\n      default:\n        setFieldMetadata(\n          field,\n          `geo.${key}`,\n          typeof value === 'string' ? value : JSON.stringify(value)\n        );\n    }\n  }\n}\n\nfunction setFieldMetadata(field: Field, key: string, value: string): void {\n  field.metadata = field.metadata || {};\n  field.metadata[key] = value;\n}\n"],"mappings":";;;;;;;;;;;;;AA+CO,SAASA,uBAAuBA,CACrCC,MAAc,EACdC,WAAmB,EACa;EAChC,IAAMC,mBAAmB,GAAGF,MAAM,CAACG,QAAQ,CAACF,WAAW,CAAC;EACxD,IAAI,CAACC,mBAAmB,EAAE;IACxB,OAAO,IAAI;EACb;EAEA,IAAI;IACF,IAAMC,QAAQ,GAAGC,IAAI,CAACC,KAAK,CAACH,mBAAmB,CAAC;IAChD,IAAI,CAACC,QAAQ,IAAI,IAAAG,QAAA,CAAAC,OAAA,EAAOJ,QAAQ,MAAK,QAAQ,EAAE;MAC7C,OAAO,IAAI;IACb;IACA,OAAOA,QAAQ;EACjB,CAAC,CAAC,OAAAK,OAAA,EAAM;IACN,OAAO,IAAI;EACb;AACF;AAEO,SAASC,wBAAwBA,CAACT,MAAc,EAAEC,WAAmB,EAAQ;EAClF,IAAMS,IAAI,GAAGX,uBAAuB,CAACC,MAAM,EAAEC,WAAW,CAAC;EACzD,SAAAU,EAAA,MAAAC,eAAA,GAA2BC,MAAM,CAACC,OAAO,CAACJ,IAAI,IAAI,CAAC,CAAC,CAAC,EAAAC,EAAA,GAAAC,eAAA,CAAAG,MAAA,EAAAJ,EAAA,IAAE;IAAlD,IAAAK,kBAAA,OAAAC,eAAA,CAAAV,OAAA,EAAAK,eAAA,CAAAD,EAAA;MAAOO,IAAG,GAAAF,kBAAA;MAAEG,KAAK,GAAAH,kBAAA;IACpBhB,MAAM,CAACG,QAAQ,IAAAiB,MAAA,CAAInB,WAAW,OAAAmB,MAAA,CAAIF,IAAG,EAAG,GACtC,OAAOC,KAAK,KAAK,QAAQ,GAAGA,KAAK,GAAGf,IAAI,CAACiB,SAAS,CAACF,KAAK,CAAC;EAC7D;AACF;AAOO,SAASG,cAAcA,CAACtB,MAAc,EAAsB;EACjE,IAAMuB,WAAW,GAAGxB,uBAAuB,CAACC,MAAM,EAAE,KAAK,CAAgB;EACzE,OAAOuB,WAAW;AACpB;AAMO,SAASC,cAAcA,CAACxB,MAAc,EAAEuB,WAAwB,EAAQ;EAC7E,IAAME,sBAAsB,GAAGrB,IAAI,CAACiB,SAAS,CAACE,WAAW,CAAC;EAC1DvB,MAAM,CAACG,QAAQ,CAACuB,GAAG,GAAGD,sBAAsB;AAC9C;AAMO,SAASE,iBAAiBA,CAAC3B,MAAc,EAAQ;EACtD,IAAMuB,WAAW,GAAGD,cAAc,CAACtB,MAAM,CAAC;EAC1C,IAAI,CAACuB,WAAW,EAAE;IAChB;EACF;EAIA,IAAOK,OAAO,GAA6BL,WAAW,CAA/CK,OAAO;IAAEC,cAAc,GAAaN,WAAW,CAAtCM,cAAc;IAAEC,OAAO,GAAIP,WAAW,CAAtBO,OAAO;EACvC,IAAIF,OAAO,EAAE;IACX5B,MAAM,CAACG,QAAQ,CAAC,aAAa,CAAC,GAAGyB,OAAO;EAC1C;EAEA,IAAIC,cAAc,EAAE;IAClB7B,MAAM,CAACG,QAAQ,CAAC,oBAAoB,CAAC,GAAG0B,cAAc;EACxD;EAGA7B,MAAM,CAACG,QAAQ,CAAC,aAAa,CAAC,GAAGU,MAAM,CAACkB,IAAI,CAACD,OAAO,IAAI,CAAC,CAAC,CAAC,CAACE,IAAI,CAAC,EAAE,CAAC;EAAC,IAAAC,KAAA,YAAAA,MAAA,EAEK;IAArE,IAAAC,mBAAA,OAAAjB,eAAA,CAAAV,OAAA,EAAA4B,gBAAA,CAAAC,GAAA;MAAOC,UAAU,GAAAH,mBAAA;MAAEI,cAAc,GAAAJ,mBAAA;IACpC,IAAMK,KAAK,GAAGvC,MAAM,CAACwC,MAAM,CAACC,IAAI,CAAC,UAACF,KAAK;MAAA,OAAKA,KAAK,CAACG,IAAI,KAAKL,UAAU;IAAA,EAAC;IACtE,IAAIE,KAAK,EAAE;MACT,IAAIA,KAAK,CAACG,IAAI,KAAKb,cAAc,EAAE;QACjCc,gBAAgB,CAACJ,KAAK,EAAE,mBAAmB,EAAE,MAAM,CAAC;MACtD;MACAK,sBAAsB,CAACL,KAAK,EAAED,cAAc,CAAC;IAC/C;EACF,CAAC;EARD,SAAAF,GAAA,MAAAD,gBAAA,GAA2CtB,MAAM,CAACC,OAAO,CAACgB,OAAO,IAAI,CAAC,CAAC,CAAC,EAAAM,GAAA,GAAAD,gBAAA,CAAApB,MAAA,EAAAqB,GAAA;IAAAH,KAAA;EAAA;AAS1E;AAGA,SAASW,sBAAsBA,CAACL,KAAY,EAAED,cAAc,EAAQ;EAClE,SAAAO,GAAA,MAAAC,gBAAA,GAA2BjC,MAAM,CAACC,OAAO,CAACwB,cAAc,IAAI,CAAC,CAAC,CAAC,EAAAO,GAAA,GAAAC,gBAAA,CAAA/B,MAAA,EAAA8B,GAAA,IAAE;IAA5D,IAAAE,mBAAA,OAAA9B,eAAA,CAAAV,OAAA,EAAAuC,gBAAA,CAAAD,GAAA;MAAO3B,KAAG,GAAA6B,mBAAA;MAAE5B,KAAK,GAAA4B,mBAAA;IACpB,QAAQ7B,KAAG;MACT,KAAK,eAAe;QAClByB,gBAAgB,CAACJ,KAAK,SAAAnB,MAAA,CAASF,KAAG,GAAKC,KAAK,CAAca,IAAI,CAAC,GAAG,CAAC,CAAC;QACpE;MACF,KAAK,MAAM;QACTW,gBAAgB,CAACJ,KAAK,aAAAnB,MAAA,CAAaF,KAAG,GAAId,IAAI,CAACiB,SAAS,CAACF,KAAK,CAAC,CAAC;QAChE;MACF,KAAK,KAAK;QACR,SAAA6B,GAAA,MAAAC,gBAAA,GAAiCpC,MAAM,CAACC,OAAO,CAACK,KAAK,IAAI,CAAC,CAAC,CAAC,EAAA6B,GAAA,GAAAC,gBAAA,CAAAlC,MAAA,EAAAiC,GAAA,IAAE;UAAzD,IAAAE,mBAAA,OAAAjC,eAAA,CAAAV,OAAA,EAAA0C,gBAAA,CAAAD,GAAA;YAAOG,MAAM,GAAAD,mBAAA;YAAEE,QAAQ,GAAAF,mBAAA;UAC1B,QAAQC,MAAM;YACZ,KAAK,IAAI;cACP,IAAME,KAAK,GACT,IAAA/C,QAAA,CAAAC,OAAA,EAAO6C,QAAQ,MAAK,QAAQ,MAAAhC,MAAA,CACrBgC,QAAQ,aAARA,QAAQ,uBAARA,QAAQ,CAAEE,SAAS,OAAAlC,MAAA,CAAIgC,QAAQ,aAARA,QAAQ,uBAARA,QAAQ,CAAEG,IAAI,IACxCnD,IAAI,CAACiB,SAAS,CAAC+B,QAAQ,CAAC;cAC9BT,gBAAgB,CAACJ,KAAK,aAAAnB,MAAA,CAAa+B,MAAM,GAAIE,KAAK,CAAC;cACnD;YACF;cACEV,gBAAgB,CACdJ,KAAK,aAAAnB,MAAA,CACM+B,MAAM,GACjB,OAAOC,QAAQ,KAAK,QAAQ,GAAGA,QAAQ,GAAGhD,IAAI,CAACiB,SAAS,CAAC+B,QAAQ,CACnE,CAAC;cACD;UACJ;QACF;QACA;MACF,KAAK,OAAO;MACZ;QACET,gBAAgB,CACdJ,KAAK,SAAAnB,MAAA,CACEF,KAAG,GACV,OAAOC,KAAK,KAAK,QAAQ,GAAGA,KAAK,GAAGf,IAAI,CAACiB,SAAS,CAACF,KAAK,CAC1D,CAAC;IACL;EACF;AACF;AAEA,SAASwB,gBAAgBA,CAACJ,KAAY,EAAErB,GAAW,EAAEC,KAAa,EAAQ;EACxEoB,KAAK,CAACpC,QAAQ,GAAGoC,KAAK,CAACpC,QAAQ,IAAI,CAAC,CAAC;EACrCoC,KAAK,CAACpC,QAAQ,CAACe,GAAG,CAAC,GAAGC,KAAK;AAC7B"}

package/dist/es5/lib/geo/{geoparquet-schema.js → geoparquet-metadata-schema.js}

@@ -3,8 +3,8 @@
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
-exports.default = void 0;
-var _default = {
+exports.GEOPARQUET_METADATA_JSON_SCHEMA = void 0;
+var GEOPARQUET_METADATA_JSON_SCHEMA = {
   $schema: 'http://json-schema.org/draft-07/schema#',
   title: 'GeoParquet',
   description: 'Parquet metadata included in the geo field.',
@@ -79,5 +79,5 @@ var _default = {
     }
   }
 };
-exports.default = _default;
-//# sourceMappingURL=geoparquet-schema.js.map
+exports.GEOPARQUET_METADATA_JSON_SCHEMA = GEOPARQUET_METADATA_JSON_SCHEMA;
+//# sourceMappingURL=geoparquet-metadata-schema.js.map

package/dist/es5/lib/geo/geoparquet-metadata-schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"geoparquet-metadata-schema.js","names":["GEOPARQUET_METADATA_JSON_SCHEMA","$schema","title","description","type","required","properties","version","const","primary_column","minLength","columns","minProperties","patternProperties","encoding","geometry_types","uniqueItems","items","pattern","crs","oneOf","$ref","edges","enum","orientation","bbox","minItems","maxItems","epoch","additionalProperties","exports"],"sources":["../../../../src/lib/geo/geoparquet-metadata-schema.ts"],"sourcesContent":["// loaders.gl, MIT license\n\n/* eslint-disable camelcase */\n\n/**\n * Geoparquet JSON schema for geo metadata\n * @see https://github.com/geoarrow/geoarrow/blob/main/metadata.md\n * @see https://github.com/opengeospatial/geoparquet/blob/main/format-specs/geoparquet.md\n */\nexport const GEOPARQUET_METADATA_JSON_SCHEMA = {\n  $schema: 'http://json-schema.org/draft-07/schema#',\n  title: 'GeoParquet',\n  description: 'Parquet metadata included in the geo field.',\n  type: 'object',\n  required: ['version', 'primary_column', 'columns'],\n  properties: {\n    version: {type: 'string', const: '1.0.0-beta.1'},\n    primary_column: {type: 'string', minLength: 1},\n    columns: {\n      type: 'object',\n      minProperties: 1,\n      patternProperties: {\n        '.+': {\n          type: 'object',\n          required: ['encoding', 'geometry_types'],\n          properties: {\n            encoding: {type: 'string', const: 'WKB'},\n            geometry_types: {\n              type: 'array',\n              uniqueItems: true,\n              items: {\n                type: 'string',\n                pattern: '^(GeometryCollection|(Multi)?(Point|LineString|Polygon))( Z)?$'\n              }\n            },\n            crs: {\n              oneOf: [\n                {\n                  $ref: 'https://proj.org/schemas/v0.5/projjson.schema.json'\n                },\n                {type: 'null'}\n              ]\n            },\n            edges: {type: 'string', enum: ['planar', 'spherical']},\n            orientation: {type: 'string', const: 'counterclockwise'},\n            bbox: {\n              type: 'array',\n              items: {type: 'number'},\n              oneOf: [\n                {\n                  description: '2D bbox consisting of (xmin, ymin, xmax, ymax)',\n                  minItems: 4,\n                  maxItems: 4\n                },\n                {\n                  description: '3D bbox consisting of (xmin, ymin, zmin, xmax, ymax, zmax)',\n                  minItems: 6,\n                  maxItems: 6\n                }\n              ]\n            },\n            epoch: {type: 'number'}\n          }\n        }\n      },\n      additionalProperties: false\n    }\n  }\n};\n"],"mappings":";;;;;;AASO,IAAMA,+BAA+B,GAAG;EAC7CC,OAAO,EAAE,yCAAyC;EAClDC,KAAK,EAAE,YAAY;EACnBC,WAAW,EAAE,6CAA6C;EAC1DC,IAAI,EAAE,QAAQ;EACdC,QAAQ,EAAE,CAAC,SAAS,EAAE,gBAAgB,EAAE,SAAS,CAAC;EAClDC,UAAU,EAAE;IACVC,OAAO,EAAE;MAACH,IAAI,EAAE,QAAQ;MAAEI,KAAK,EAAE;IAAc,CAAC;IAChDC,cAAc,EAAE;MAACL,IAAI,EAAE,QAAQ;MAAEM,SAAS,EAAE;IAAC,CAAC;IAC9CC,OAAO,EAAE;MACPP,IAAI,EAAE,QAAQ;MACdQ,aAAa,EAAE,CAAC;MAChBC,iBAAiB,EAAE;QACjB,IAAI,EAAE;UACJT,IAAI,EAAE,QAAQ;UACdC,QAAQ,EAAE,CAAC,UAAU,EAAE,gBAAgB,CAAC;UACxCC,UAAU,EAAE;YACVQ,QAAQ,EAAE;cAACV,IAAI,EAAE,QAAQ;cAAEI,KAAK,EAAE;YAAK,CAAC;YACxCO,cAAc,EAAE;cACdX,IAAI,EAAE,OAAO;cACbY,WAAW,EAAE,IAAI;cACjBC,KAAK,EAAE;gBACLb,IAAI,EAAE,QAAQ;gBACdc,OAAO,EAAE;cACX;YACF,CAAC;YACDC,GAAG,EAAE;cACHC,KAAK,EAAE,CACL;gBACEC,IAAI,EAAE;cACR,CAAC,EACD;gBAACjB,IAAI,EAAE;cAAM,CAAC;YAElB,CAAC;YACDkB,KAAK,EAAE;cAAClB,IAAI,EAAE,QAAQ;cAAEmB,IAAI,EAAE,CAAC,QAAQ,EAAE,WAAW;YAAC,CAAC;YACtDC,WAAW,EAAE;cAACpB,IAAI,EAAE,QAAQ;cAAEI,KAAK,EAAE;YAAkB,CAAC;YACxDiB,IAAI,EAAE;cACJrB,IAAI,EAAE,OAAO;cACba,KAAK,EAAE;gBAACb,IAAI,EAAE;cAAQ,CAAC;cACvBgB,KAAK,EAAE,CACL;gBACEjB,WAAW,EAAE,gDAAgD;gBAC7DuB,QAAQ,EAAE,CAAC;gBACXC,QAAQ,EAAE;cACZ,CAAC,EACD;gBACExB,WAAW,EAAE,4DAA4D;gBACzEuB,QAAQ,EAAE,CAAC;gBACXC,QAAQ,EAAE;cACZ,CAAC;YAEL,CAAC;YACDC,KAAK,EAAE;cAACxB,IAAI,EAAE;YAAQ;UACxB;QACF;MACF,CAAC;MACDyB,oBAAoB,EAAE;IACxB;EACF;AACF,CAAC;AAACC,OAAA,CAAA9B,+BAAA,GAAAA,+BAAA"}

package/dist/es5/lib/geo/geoparquet-metadata-schema.json

@@ -0,0 +1,60 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "GeoParquet",
+  "description": "Parquet metadata included in the geo field.",
+  "type": "object",
+  "required": ["version", "primary_column", "columns"],
+  "properties": {
+    "version": {"type": "string", "const": "1.0.0-beta.1"},
+    "primary_column": {"type": "string", "minLength": 1},
+    "columns": {
+      "type": "object",
+      "minProperties": 1,
+      "patternProperties": {
+        ".+": {
+          "type": "object",
+          "required": ["encoding", "geometry_types"],
+          "properties": {
+            "encoding": {"type": "string", "const": "WKB"},
+            "geometry_types": {
+              "type": "array",
+              "uniqueItems": true,
+              "items": {
+                "type": "string",
+                "pattern": "^(GeometryCollection|(Multi)?(Point|LineString|Polygon))( Z)?$"
+              }
+            },
+            "crs": {
+              "oneOf": [
+                {
+                  "$ref": "https://proj.org/schemas/v0.5/projjson.schema.json"
+                },
+                {"type": "null"}
+              ]
+            },
+            "edges": {"type": "string", "enum": ["planar", "spherical"]},
+            "orientation": {"type": "string", "const": "counterclockwise"},
+            "bbox": {
+              "type": "array",
+              "items": {"type": "number"},
+              "oneOf": [
+                {
+                  "description": "2D bbox consisting of (xmin, ymin, xmax, ymax)",
+                  "minItems": 4,
+                  "maxItems": 4
+                },
+                {
+                  "description": "3D bbox consisting of (xmin, ymin, zmin, xmax, ymax, zmax)",
+                  "minItems": 6,
+                  "maxItems": 6
+                }
+              ]
+            },
+            "epoch": {"type": "number"}
+          }
+        }
+      },
+      "additionalProperties": false
+    }
+  }
+}

package/dist/es5/lib/parsers/get-parquet-schema.js

@@ -0,0 +1,41 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.getSchemaFromParquetReader = getSchemaFromParquetReader;
+var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
+var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
+var _convertSchemaFromParquet = require("../arrow/convert-schema-from-parquet");
+var _decodeGeoMetadata = require("../geo/decode-geo-metadata");
+function getSchemaFromParquetReader(_x) {
+  return _getSchemaFromParquetReader.apply(this, arguments);
+}
+function _getSchemaFromParquetReader() {
+  _getSchemaFromParquetReader = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee(reader) {
+    var parquetSchema, parquetMetadata, schema;
+    return _regenerator.default.wrap(function _callee$(_context) {
+      while (1) switch (_context.prev = _context.next) {
+        case 0:
+          _context.next = 2;
+          return reader.getSchema();
+        case 2:
+          parquetSchema = _context.sent;
+          _context.next = 5;
+          return reader.getFileMetadata();
+        case 5:
+          parquetMetadata = _context.sent;
+          schema = (0, _convertSchemaFromParquet.convertParquetSchema)(parquetSchema, parquetMetadata);
+          (0, _decodeGeoMetadata.unpackGeoMetadata)(schema);
+          (0, _decodeGeoMetadata.unpackJSONStringMetadata)(schema, 'pandas');
+          return _context.abrupt("return", schema);
+        case 10:
+        case "end":
+          return _context.stop();
+      }
+    }, _callee);
+  }));
+  return _getSchemaFromParquetReader.apply(this, arguments);
+}
+//# sourceMappingURL=get-parquet-schema.js.map

package/dist/es5/lib/parsers/get-parquet-schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"get-parquet-schema.js","names":["_convertSchemaFromParquet","require","_decodeGeoMetadata","getSchemaFromParquetReader","_x","_getSchemaFromParquetReader","apply","arguments","_asyncToGenerator2","default","_regenerator","mark","_callee","reader","parquetSchema","parquetMetadata","schema","wrap","_callee$","_context","prev","next","getSchema","sent","getFileMetadata","convertParquetSchema","unpackGeoMetadata","unpackJSONStringMetadata","abrupt","stop"],"sources":["../../../../src/lib/parsers/get-parquet-schema.ts"],"sourcesContent":["// loaders.gl\nimport {Schema} from '@loaders.gl/schema';\nimport {ParquetReader} from '../../parquetjs/parser/parquet-reader';\nimport {convertParquetSchema} from '../arrow/convert-schema-from-parquet';\nimport {unpackGeoMetadata, unpackJSONStringMetadata} from '../geo/decode-geo-metadata';\n\nexport async function getSchemaFromParquetReader(reader: ParquetReader): Promise<Schema> {\n  const parquetSchema = await reader.getSchema();\n  const parquetMetadata = await reader.getFileMetadata();\n  const schema = convertParquetSchema(parquetSchema, parquetMetadata);\n  unpackGeoMetadata(schema);\n  unpackJSONStringMetadata(schema, 'pandas');\n  return schema;\n}\n"],"mappings":";;;;;;;;;AAGA,IAAAA,yBAAA,GAAAC,OAAA;AACA,IAAAC,kBAAA,GAAAD,OAAA;AAAuF,SAEjEE,0BAA0BA,CAAAC,EAAA;EAAA,OAAAC,2BAAA,CAAAC,KAAA,OAAAC,SAAA;AAAA;AAAA,SAAAF,4BAAA;EAAAA,2BAAA,OAAAG,kBAAA,CAAAC,OAAA,EAAAC,YAAA,CAAAD,OAAA,CAAAE,IAAA,CAAzC,SAAAC,QAA0CC,MAAqB;IAAA,IAAAC,aAAA,EAAAC,eAAA,EAAAC,MAAA;IAAA,OAAAN,YAAA,CAAAD,OAAA,CAAAQ,IAAA,UAAAC,SAAAC,QAAA;MAAA,kBAAAA,QAAA,CAAAC,IAAA,GAAAD,QAAA,CAAAE,IAAA;QAAA;UAAAF,QAAA,CAAAE,IAAA;UAAA,OACxCR,MAAM,CAACS,SAAS,CAAC,CAAC;QAAA;UAAxCR,aAAa,GAAAK,QAAA,CAAAI,IAAA;UAAAJ,QAAA,CAAAE,IAAA;UAAA,OACWR,MAAM,CAACW,eAAe,CAAC,CAAC;QAAA;UAAhDT,eAAe,GAAAI,QAAA,CAAAI,IAAA;UACfP,MAAM,GAAG,IAAAS,8CAAoB,EAACX,aAAa,EAAEC,eAAe,CAAC;UACnE,IAAAW,oCAAiB,EAACV,MAAM,CAAC;UACzB,IAAAW,2CAAwB,EAACX,MAAM,EAAE,QAAQ,CAAC;UAAC,OAAAG,QAAA,CAAAS,MAAA,WACpCZ,MAAM;QAAA;QAAA;UAAA,OAAAG,QAAA,CAAAU,IAAA;MAAA;IAAA,GAAAjB,OAAA;EAAA,CACd;EAAA,OAAAP,2BAAA,CAAAC,KAAA,OAAAC,SAAA;AAAA"}

package/dist/es5/lib/parsers/parse-parquet-to-columns.js

@@ -12,9 +12,9 @@ var _awaitAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpe
 var _wrapAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/wrapAsyncGenerator"));
 var _loaderUtils = require("@loaders.gl/loader-utils");
 var _parquetReader = require("../../parquetjs/parser/parquet-reader");
-var _convertSchemaFromParquet = require("../arrow/convert-schema-from-parquet");
 var _shred = require("../../parquetjs/schema/shred");
-var _decodeGeoMetadata = require("../geo/decode-geo-metadata");
+var _getParquetSchema = require("./get-parquet-schema");
+var _bufferPolyfill = require("../../buffer-polyfill");
 function _asyncIterator(iterable) { var method, async, sync, retry = 2; for ("undefined" != typeof Symbol && (async = Symbol.asyncIterator, sync = Symbol.iterator); retry--;) { if (async && null != (method = iterable[async])) return method.call(iterable); if (sync && null != (method = iterable[sync])) return new AsyncFromSyncIterator(method.call(iterable)); async = "@@asyncIterator", sync = "@@iterator"; } throw new TypeError("Object is not async iterable"); }
 function AsyncFromSyncIterator(s) { function AsyncFromSyncIteratorContinuation(r) { if (Object(r) !== r) return Promise.reject(new TypeError(r + " is not an object.")); var done = r.done; return Promise.resolve(r.value).then(function (value) { return { value: value, done: done }; }); } return AsyncFromSyncIterator = function AsyncFromSyncIterator(s) { this.s = s, this.n = s.next; }, AsyncFromSyncIterator.prototype = { s: null, n: null, next: function next() { return AsyncFromSyncIteratorContinuation(this.n.apply(this.s, arguments)); }, return: function _return(value) { var ret = this.s.return; return void 0 === ret ? Promise.resolve({ value: value, done: !0 }) : AsyncFromSyncIteratorContinuation(ret.apply(this.s, arguments)); }, throw: function _throw(value) { var thr = this.s.return; return void 0 === thr ? Promise.reject(value) : AsyncFromSyncIteratorContinuation(thr.apply(this.s, arguments)); } }, new AsyncFromSyncIterator(s); }
 function parseParquetInColumns(_x3, _x4) {
@@ -22,21 +22,24 @@ function parseParquetInColumns(_x3, _x4) {
 }
 function _parseParquetInColumns() {
   _parseParquetInColumns = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee2(arrayBuffer, options) {
-    var blob, _iteratorAbruptCompletion, _didIteratorError, _iteratorError, _iterator, _step, batch;
+    var blob, file, reader, _iteratorAbruptCompletion, _didIteratorError, _iteratorError, _iterator, _step, batch;
     return _regenerator.default.wrap(function _callee2$(_context2) {
       while (1) switch (_context2.prev = _context2.next) {
         case 0:
+          (0, _bufferPolyfill.installBufferPolyfill)();
           blob = new Blob([arrayBuffer]);
+          file = (0, _loaderUtils.makeReadableFile)(blob);
+          reader = new _parquetReader.ParquetReader(file);
           _iteratorAbruptCompletion = false;
           _didIteratorError = false;
-          _context2.prev = 3;
-          _iterator = _asyncIterator(parseParquetFileInColumnarBatches(blob, options));
-        case 5:
-          _context2.next = 7;
+          _context2.prev = 6;
+          _iterator = _asyncIterator(parseParquetFileInColumnarBatches(reader, options));
+        case 8:
+          _context2.next = 10;
           return _iterator.next();
-        case 7:
+        case 10:
           if (!(_iteratorAbruptCompletion = !(_step = _context2.sent).done)) {
-            _context2.next = 13;
+            _context2.next = 16;
             break;
           }
           batch = _step.value;
@@ -45,45 +48,45 @@ function _parseParquetInColumns() {
             schema: batch.schema,
             data: batch.data
           });
-        case 10:
+        case 13:
           _iteratorAbruptCompletion = false;
-          _context2.next = 5;
+          _context2.next = 8;
           break;
-        case 13:
-          _context2.next = 19;
+        case 16:
+          _context2.next = 22;
           break;
-        case 15:
-          _context2.prev = 15;
-          _context2.t0 = _context2["catch"](3);
+        case 18:
+          _context2.prev = 18;
+          _context2.t0 = _context2["catch"](6);
           _didIteratorError = true;
           _iteratorError = _context2.t0;
-        case 19:
-          _context2.prev = 19;
-          _context2.prev = 20;
+        case 22:
+          _context2.prev = 22;
+          _context2.prev = 23;
           if (!(_iteratorAbruptCompletion && _iterator.return != null)) {
-            _context2.next = 24;
+            _context2.next = 27;
             break;
           }
-          _context2.next = 24;
+          _context2.next = 27;
           return _iterator.return();
-        case 24:
-          _context2.prev = 24;
+        case 27:
+          _context2.prev = 27;
           if (!_didIteratorError) {
-            _context2.next = 27;
+            _context2.next = 30;
             break;
           }
           throw _iteratorError;
-        case 27:
-          return _context2.finish(24);
-        case 28:
-          return _context2.finish(19);
-        case 29:
-          throw new Error('empty table');
         case 30:
+          return _context2.finish(27);
+        case 31:
+          return _context2.finish(22);
+        case 32:
+          throw new Error('empty table');
+        case 33:
         case "end":
           return _context2.stop();
       }
-    }, _callee2, null, [[3, 15, 19, 29], [20,, 24, 28]]);
+    }, _callee2, null, [[6, 18, 22, 32], [23,, 27, 31]]);
   }));
   return _parseParquetInColumns.apply(this, arguments);
 }
@@ -91,80 +94,76 @@ function parseParquetFileInColumnarBatches(_x, _x2) {
   return _parseParquetFileInColumnarBatches.apply(this, arguments);
 }
 function _parseParquetFileInColumnarBatches() {
-  _parseParquetFileInColumnarBatches = (0, _wrapAsyncGenerator2.default)(_regenerator.default.mark(function _callee(blob, options) {
-    var file, reader, parquetSchema, parquetMetadata, schema, rowGroups, _iteratorAbruptCompletion2, _didIteratorError2, _iteratorError2, _iterator2, _step2, rowGroup;
+  _parseParquetFileInColumnarBatches = (0, _wrapAsyncGenerator2.default)(_regenerator.default.mark(function _callee(reader, options) {
+    var schema, parquetSchema, rowGroups, _iteratorAbruptCompletion2, _didIteratorError2, _iteratorError2, _iterator2, _step2, rowGroup;
     return _regenerator.default.wrap(function _callee$(_context) {
       while (1) switch (_context.prev = _context.next) {
         case 0:
-          file = (0, _loaderUtils.makeReadableFile)(blob);
-          reader = new _parquetReader.ParquetReader(file);
-          _context.next = 4;
+          _context.next = 2;
+          return (0, _awaitAsyncGenerator2.default)((0, _getParquetSchema.getSchemaFromParquetReader)(reader));
+        case 2:
+          schema = _context.sent;
+          _context.next = 5;
           return (0, _awaitAsyncGenerator2.default)(reader.getSchema());
-        case 4:
+        case 5:
           parquetSchema = _context.sent;
-          _context.next = 7;
-          return (0, _awaitAsyncGenerator2.default)(reader.getFileMetadata());
-        case 7:
-          parquetMetadata = _context.sent;
-          schema = (0, _convertSchemaFromParquet.convertParquetSchema)(parquetSchema, parquetMetadata);
-          (0, _decodeGeoMetadata.unpackGeoMetadata)(schema);
           rowGroups = reader.rowGroupIterator(options === null || options === void 0 ? void 0 : options.parquet);
           _iteratorAbruptCompletion2 = false;
           _didIteratorError2 = false;
-          _context.prev = 13;
+          _context.prev = 9;
           _iterator2 = _asyncIterator(rowGroups);
-        case 15:
-          _context.next = 17;
+        case 11:
+          _context.next = 13;
           return (0, _awaitAsyncGenerator2.default)(_iterator2.next());
-        case 17:
+        case 13:
           if (!(_iteratorAbruptCompletion2 = !(_step2 = _context.sent).done)) {
-            _context.next = 24;
+            _context.next = 20;
             break;
           }
           rowGroup = _step2.value;
-          _context.next = 21;
-          return convertRowGroupToTableBatch(parquetSchema, rowGroup, schema);
-        case 21:
+          _context.next = 17;
+          return convertRowGroupToTableBatch(rowGroup, parquetSchema, schema);
+        case 17:
           _iteratorAbruptCompletion2 = false;
-          _context.next = 15;
+          _context.next = 11;
           break;
-        case 24:
-          _context.next = 30;
+        case 20:
+          _context.next = 26;
           break;
-        case 26:
-          _context.prev = 26;
-          _context.t0 = _context["catch"](13);
+        case 22:
+          _context.prev = 22;
+          _context.t0 = _context["catch"](9);
           _didIteratorError2 = true;
           _iteratorError2 = _context.t0;
-        case 30:
-          _context.prev = 30;
-          _context.prev = 31;
+        case 26:
+          _context.prev = 26;
+          _context.prev = 27;
           if (!(_iteratorAbruptCompletion2 && _iterator2.return != null)) {
-            _context.next = 35;
+            _context.next = 31;
             break;
           }
-          _context.next = 35;
+          _context.next = 31;
           return (0, _awaitAsyncGenerator2.default)(_iterator2.return());
-        case 35:
-          _context.prev = 35;
+        case 31:
+          _context.prev = 31;
           if (!_didIteratorError2) {
-            _context.next = 38;
+            _context.next = 34;
             break;
           }
           throw _iteratorError2;
-        case 38:
-          return _context.finish(35);
-        case 39:
-          return _context.finish(30);
-        case 40:
+        case 34:
+          return _context.finish(31);
+        case 35:
+          return _context.finish(26);
+        case 36:
         case "end":
           return _context.stop();
       }
-    }, _callee, null, [[13, 26, 30, 40], [31,, 35, 39]]);
+    }, _callee, null, [[9, 22, 26, 36], [27,, 31, 35]]);
   }));
   return _parseParquetFileInColumnarBatches.apply(this, arguments);
 }
-function convertRowGroupToTableBatch(parquetSchema, rowGroup, schema) {
+function convertRowGroupToTableBatch(rowGroup, parquetSchema, schema) {
   var data = (0, _shred.materializeColumns)(parquetSchema, rowGroup);
   return {
     shape: 'columnar-table',

package/dist/es5/lib/parsers/parse-parquet-to-columns.js.map

@@ -1 +1 @@
-{"version":3,"file":"parse-parquet-to-columns.js","names":["_loaderUtils","require","_parquetReader","_convertSchemaFromParquet","_shred","_decodeGeoMetadata","_asyncIterator","iterable","method","async","sync","retry","Symbol","asyncIterator","iterator","call","AsyncFromSyncIterator","TypeError","s","AsyncFromSyncIteratorContinuation","r","Object","Promise","reject","done","resolve","value","then","n","next","prototype","apply","arguments","return","_return","ret","throw","_throw","thr","parseParquetInColumns","_x3","_x4","_parseParquetInColumns","_asyncToGenerator2","default","_regenerator","mark","_callee2","arrayBuffer","options","blob","_iteratorAbruptCompletion","_didIteratorError","_iteratorError","_iterator","_step","batch","wrap","_callee2$","_context2","prev","Blob","parseParquetFileInColumnarBatches","sent","abrupt","shape","schema","data","t0","finish","Error","stop","_x","_x2","_parseParquetFileInColumnarBatches","_wrapAsyncGenerator2","_callee","file","reader","parquetSchema","parquetMetadata","rowGroups","_iteratorAbruptCompletion2","_didIteratorError2","_iteratorError2","_iterator2","_step2","rowGroup","_callee$","_context","makeReadableFile","ParquetReader","_awaitAsyncGenerator2","getSchema","getFileMetadata","convertParquetSchema","unpackGeoMetadata","rowGroupIterator","parquet","convertRowGroupToTableBatch","materializeColumns","batchType","length","rowCount"],"sources":["../../../../src/lib/parsers/parse-parquet-to-columns.ts"],"sourcesContent":["// loaders.gl, MIT license\n\n// import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';\nimport {ColumnarTable, ColumnarTableBatch, Schema} from '@loaders.gl/schema';\nimport {makeReadableFile} from '@loaders.gl/loader-utils';\nimport type {ParquetLoaderOptions} from '../../parquet-loader';\nimport {ParquetReader} from '../../parquetjs/parser/parquet-reader';\nimport {ParquetRowGroup} from '../../parquetjs/schema/declare';\nimport {ParquetSchema} from '../../parquetjs/schema/schema';\nimport {convertParquetSchema} from '../arrow/convert-schema-from-parquet';\nimport {materializeColumns} from '../../parquetjs/schema/shred';\n// import {convertParquetRowGroupToColumns} from '../arrow/convert-row-group-to-columns';\nimport {unpackGeoMetadata} from '../geo/decode-geo-metadata';\n\nexport async function parseParquetInColumns(\n  arrayBuffer: ArrayBuffer,\n  options?: ParquetLoaderOptions\n): Promise<ColumnarTable> {\n  const blob = new Blob([arrayBuffer]);\n  for await (const batch of parseParquetFileInColumnarBatches(blob, options)) {\n    return {\n      shape: 'columnar-table',\n      schema: batch.schema,\n      data: batch.data\n    };\n  }\n  throw new Error('empty table');\n}\n\nexport async function* parseParquetFileInColumnarBatches(\n  blob: Blob,\n  options?: ParquetLoaderOptions\n): AsyncIterable<ColumnarTableBatch> {\n  const file = makeReadableFile(blob);\n  const reader = new ParquetReader(file);\n  const parquetSchema = await reader.getSchema();\n  const parquetMetadata = await reader.getFileMetadata();\n  const schema = convertParquetSchema(parquetSchema, parquetMetadata);\n  unpackGeoMetadata(schema);\n  const rowGroups = reader.rowGroupIterator(options?.parquet);\n  for await (const rowGroup of rowGroups) {\n    yield convertRowGroupToTableBatch(parquetSchema, rowGroup, schema);\n  }\n}\n\nfunction convertRowGroupToTableBatch(\n  parquetSchema: ParquetSchema,\n  rowGroup: ParquetRowGroup,\n  schema: Schema\n): ColumnarTableBatch {\n  // const data = convertParquetRowGroupToColumns(schema, rowGroup);\n  const data = materializeColumns(parquetSchema, rowGroup);\n  return {\n    shape: 'columnar-table',\n    batchType: 'data',\n    schema,\n    data,\n    length: rowGroup.rowCount\n  };\n}\n"],"mappings":";;;;;;;;;;;;AAIA,IAAAA,YAAA,GAAAC,OAAA;AAEA,IAAAC,cAAA,GAAAD,OAAA;AAGA,IAAAE,yBAAA,GAAAF,OAAA;AACA,IAAAG,MAAA,GAAAH,OAAA;AAEA,IAAAI,kBAAA,GAAAJ,OAAA;AAA6D,SAAAK,eAAAC,QAAA,QAAAC,MAAA,EAAAC,KAAA,EAAAC,IAAA,EAAAC,KAAA,iCAAAC,MAAA,KAAAH,KAAA,GAAAG,MAAA,CAAAC,aAAA,EAAAH,IAAA,GAAAE,MAAA,CAAAE,QAAA,GAAAH,KAAA,WAAAF,KAAA,aAAAD,MAAA,GAAAD,QAAA,CAAAE,KAAA,WAAAD,MAAA,CAAAO,IAAA,CAAAR,QAAA,OAAAG,IAAA,aAAAF,MAAA,GAAAD,QAAA,CAAAG,IAAA,eAAAM,qBAAA,CAAAR,MAAA,CAAAO,IAAA,CAAAR,QAAA,IAAAE,KAAA,sBAAAC,IAAA,6BAAAO,SAAA;AAAA,SAAAD,sBAAAE,CAAA,aAAAC,kCAAAC,CAAA,QAAAC,MAAA,CAAAD,CAAA,MAAAA,CAAA,SAAAE,OAAA,CAAAC,MAAA,KAAAN,SAAA,CAAAG,CAAA,+BAAAI,IAAA,GAAAJ,CAAA,CAAAI,IAAA,SAAAF,OAAA,CAAAG,OAAA,CAAAL,CAAA,CAAAM,KAAA,EAAAC,IAAA,WAAAD,KAAA,aAAAA,KAAA,EAAAA,KAAA,EAAAF,IAAA,EAAAA,IAAA,iBAAAR,qBAAA,YAAAA,sBAAAE,CAAA,SAAAA,CAAA,GAAAA,CAAA,OAAAU,CAAA,GAAAV,CAAA,CAAAW,IAAA,KAAAb,qBAAA,CAAAc,SAAA,KAAAZ,CAAA,QAAAU,CAAA,QAAAC,IAAA,WAAAA,KAAA,WAAAV,iCAAA,MAAAS,CAAA,CAAAG,KAAA,MAAAb,CAAA,EAAAc,SAAA,OAAAC,MAAA,WAAAC,QAAAR,KAAA,QAAAS,GAAA,QAAAjB,CAAA,CAAAe,MAAA,oBAAAE,GAAA,GAAAb,OAAA,CAAAG,OAAA,GAAAC,KAAA,EAAAA,KAAA,EAAAF,IAAA,UAAAL,iCAAA,CAAAgB,GAAA,CAAAJ,KAAA,MAAAb,CAAA,EAAAc,SAAA,OAAAI,KAAA,WAAAC,OAAAX,KAAA,QAAAY,GAAA,QAAApB,CAAA,CAAAe,MAAA,oBAAAK,GAAA,GAAAhB,OAAA,CAAAC,MAAA,CAAAG,KAAA,IAAAP,iCAAA,CAAAmB,GAAA,CAAAP,KAAA,MAAAb,CAAA,EAAAc,SAAA,aAAAhB,qBAAA,CAAAE,CAAA;AAAA,SAEvCqB,qBAAqBA,CAAAC,GAAA,EAAAC,GAAA;EAAA,OAAAC,sBAAA,CAAAX,KAAA,OAAAC,SAAA;AAAA;AAAA,SAAAU,uBAAA;EAAAA,sBAAA,OAAAC,kBAAA,CAAAC,OAAA,EAAAC,YAAA,CAAAD,OAAA,CAAAE,IAAA,CAApC,SAAAC,SACLC,WAAwB,EACxBC,OAA8B;IAAA,IAAAC,IAAA,EAAAC,yBAAA,EAAAC,iBAAA,EAAAC,cAAA,EAAAC,SAAA,EAAAC,KAAA,EAAAC,KAAA;IAAA,OAAAX,YAAA,CAAAD,OAAA,CAAAa,IAAA,UAAAC,UAAAC,SAAA;MAAA,kBAAAA,SAAA,CAAAC,IAAA,GAAAD,SAAA,CAAA9B,IAAA;QAAA;UAExBqB,IAAI,GAAG,IAAIW,IAAI,CAAC,CAACb,WAAW,CAAC,CAAC;UAAAG,yBAAA;UAAAC,iBAAA;UAAAO,SAAA,CAAAC,IAAA;UAAAN,SAAA,GAAAhD,cAAA,CACVwD,iCAAiC,CAACZ,IAAI,EAAED,OAAO,CAAC;QAAA;UAAAU,SAAA,CAAA9B,IAAA;UAAA,OAAAyB,SAAA,CAAAzB,IAAA;QAAA;UAAA,MAAAsB,yBAAA,KAAAI,KAAA,GAAAI,SAAA,CAAAI,IAAA,EAAAvC,IAAA;YAAAmC,SAAA,CAAA9B,IAAA;YAAA;UAAA;UAAzD2B,KAAK,GAAAD,KAAA,CAAA7B,KAAA;UAAA,OAAAiC,SAAA,CAAAK,MAAA,WACb;YACLC,KAAK,EAAE,gBAAgB;YACvBC,MAAM,EAAEV,KAAK,CAACU,MAAM;YACpBC,IAAI,EAAEX,KAAK,CAACW;UACd,CAAC;QAAA;UAAAhB,yBAAA;UAAAQ,SAAA,CAAA9B,IAAA;UAAA;QAAA;UAAA8B,SAAA,CAAA9B,IAAA;UAAA;QAAA;UAAA8B,SAAA,CAAAC,IAAA;UAAAD,SAAA,CAAAS,EAAA,GAAAT,SAAA;UAAAP,iBAAA;UAAAC,cAAA,GAAAM,SAAA,CAAAS,EAAA;QAAA;UAAAT,SAAA,CAAAC,IAAA;UAAAD,SAAA,CAAAC,IAAA;UAAA,MAAAT,yBAAA,IAAAG,SAAA,CAAArB,MAAA;YAAA0B,SAAA,CAAA9B,IAAA;YAAA;UAAA;UAAA8B,SAAA,CAAA9B,IAAA;UAAA,OAAAyB,SAAA,CAAArB,MAAA;QAAA;UAAA0B,SAAA,CAAAC,IAAA;UAAA,KAAAR,iBAAA;YAAAO,SAAA,CAAA9B,IAAA;YAAA;UAAA;UAAA,MAAAwB,cAAA;QAAA;UAAA,OAAAM,SAAA,CAAAU,MAAA;QAAA;UAAA,OAAAV,SAAA,CAAAU,MAAA;QAAA;UAAA,MAEG,IAAIC,KAAK,CAAC,aAAa,CAAC;QAAA;QAAA;UAAA,OAAAX,SAAA,CAAAY,IAAA;MAAA;IAAA,GAAAxB,QAAA;EAAA,CAC/B;EAAA,OAAAL,sBAAA,CAAAX,KAAA,OAAAC,SAAA;AAAA;AAAA,SAEsB8B,iCAAiCA,CAAAU,EAAA,EAAAC,GAAA;EAAA,OAAAC,kCAAA,CAAA3C,KAAA,OAAAC,SAAA;AAAA;AAAA,SAAA0C,mCAAA;EAAAA,kCAAA,OAAAC,oBAAA,CAAA/B,OAAA,EAAAC,YAAA,CAAAD,OAAA,CAAAE,IAAA,CAAjD,SAAA8B,QACL1B,IAAU,EACVD,OAA8B;IAAA,IAAA4B,IAAA,EAAAC,MAAA,EAAAC,aAAA,EAAAC,eAAA,EAAAd,MAAA,EAAAe,SAAA,EAAAC,0BAAA,EAAAC,kBAAA,EAAAC,eAAA,EAAAC,UAAA,EAAAC,MAAA,EAAAC,QAAA;IAAA,OAAA1C,YAAA,CAAAD,OAAA,CAAAa,IAAA,UAAA+B,SAAAC,QAAA;MAAA,kBAAAA,QAAA,CAAA7B,IAAA,GAAA6B,QAAA,CAAA5D,IAAA;QAAA;UAExBgD,IAAI,GAAG,IAAAa,6BAAgB,EAACxC,IAAI,CAAC;UAC7B4B,MAAM,GAAG,IAAIa,4BAAa,CAACd,IAAI,CAAC;UAAAY,QAAA,CAAA5D,IAAA;UAAA,WAAA+D,qBAAA,CAAAhD,OAAA,EACVkC,MAAM,CAACe,SAAS,CAAC,CAAC;QAAA;UAAxCd,aAAa,GAAAU,QAAA,CAAA1B,IAAA;UAAA0B,QAAA,CAAA5D,IAAA;UAAA,WAAA+D,qBAAA,CAAAhD,OAAA,EACWkC,MAAM,CAACgB,eAAe,CAAC,CAAC;QAAA;UAAhDd,eAAe,GAAAS,QAAA,CAAA1B,IAAA;UACfG,MAAM,GAAG,IAAA6B,8CAAoB,EAAChB,aAAa,EAAEC,eAAe,CAAC;UACnE,IAAAgB,oCAAiB,EAAC9B,MAAM,CAAC;UACnBe,SAAS,GAAGH,MAAM,CAACmB,gBAAgB,CAAChD,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAEiD,OAAO,CAAC;UAAAhB,0BAAA;UAAAC,kBAAA;UAAAM,QAAA,CAAA7B,IAAA;UAAAyB,UAAA,GAAA/E,cAAA,CAC9B2E,SAAS;QAAA;UAAAQ,QAAA,CAAA5D,IAAA;UAAA,WAAA+D,qBAAA,CAAAhD,OAAA,EAAAyC,UAAA,CAAAxD,IAAA;QAAA;UAAA,MAAAqD,0BAAA,KAAAI,MAAA,GAAAG,QAAA,CAAA1B,IAAA,EAAAvC,IAAA;YAAAiE,QAAA,CAAA5D,IAAA;YAAA;UAAA;UAArB0D,QAAQ,GAAAD,MAAA,CAAA5D,KAAA;UAAA+D,QAAA,CAAA5D,IAAA;UACvB,OAAMsE,2BAA2B,CAACpB,aAAa,EAAEQ,QAAQ,EAAErB,MAAM,CAAC;QAAA;UAAAgB,0BAAA;UAAAO,QAAA,CAAA5D,IAAA;UAAA;QAAA;UAAA4D,QAAA,CAAA5D,IAAA;UAAA;QAAA;UAAA4D,QAAA,CAAA7B,IAAA;UAAA6B,QAAA,CAAArB,EAAA,GAAAqB,QAAA;UAAAN,kBAAA;UAAAC,eAAA,GAAAK,QAAA,CAAArB,EAAA;QAAA;UAAAqB,QAAA,CAAA7B,IAAA;UAAA6B,QAAA,CAAA7B,IAAA;UAAA,MAAAsB,0BAAA,IAAAG,UAAA,CAAApD,MAAA;YAAAwD,QAAA,CAAA5D,IAAA;YAAA;UAAA;UAAA4D,QAAA,CAAA5D,IAAA;UAAA,WAAA+D,qBAAA,CAAAhD,OAAA,EAAAyC,UAAA,CAAApD,MAAA;QAAA;UAAAwD,QAAA,CAAA7B,IAAA;UAAA,KAAAuB,kBAAA;YAAAM,QAAA,CAAA5D,IAAA;YAAA;UAAA;UAAA,MAAAuD,eAAA;QAAA;UAAA,OAAAK,QAAA,CAAApB,MAAA;QAAA;UAAA,OAAAoB,QAAA,CAAApB,MAAA;QAAA;QAAA;UAAA,OAAAoB,QAAA,CAAAlB,IAAA;MAAA;IAAA,GAAAK,OAAA;EAAA,CAErE;EAAA,OAAAF,kCAAA,CAAA3C,KAAA,OAAAC,SAAA;AAAA;AAED,SAASmE,2BAA2BA,CAClCpB,aAA4B,EAC5BQ,QAAyB,EACzBrB,MAAc,EACM;EAEpB,IAAMC,IAAI,GAAG,IAAAiC,yBAAkB,EAACrB,aAAa,EAAEQ,QAAQ,CAAC;EACxD,OAAO;IACLtB,KAAK,EAAE,gBAAgB;IACvBoC,SAAS,EAAE,MAAM;IACjBnC,MAAM,EAANA,MAAM;IACNC,IAAI,EAAJA,IAAI;IACJmC,MAAM,EAAEf,QAAQ,CAACgB;EACnB,CAAC;AACH"}
+{"version":3,"file":"parse-parquet-to-columns.js","names":["_loaderUtils","require","_parquetReader","_shred","_getParquetSchema","_bufferPolyfill","_asyncIterator","iterable","method","async","sync","retry","Symbol","asyncIterator","iterator","call","AsyncFromSyncIterator","TypeError","s","AsyncFromSyncIteratorContinuation","r","Object","Promise","reject","done","resolve","value","then","n","next","prototype","apply","arguments","return","_return","ret","throw","_throw","thr","parseParquetInColumns","_x3","_x4","_parseParquetInColumns","_asyncToGenerator2","default","_regenerator","mark","_callee2","arrayBuffer","options","blob","file","reader","_iteratorAbruptCompletion","_didIteratorError","_iteratorError","_iterator","_step","batch","wrap","_callee2$","_context2","prev","installBufferPolyfill","Blob","makeReadableFile","ParquetReader","parseParquetFileInColumnarBatches","sent","abrupt","shape","schema","data","t0","finish","Error","stop","_x","_x2","_parseParquetFileInColumnarBatches","_wrapAsyncGenerator2","_callee","parquetSchema","rowGroups","_iteratorAbruptCompletion2","_didIteratorError2","_iteratorError2","_iterator2","_step2","rowGroup","_callee$","_context","_awaitAsyncGenerator2","getSchemaFromParquetReader","getSchema","rowGroupIterator","parquet","convertRowGroupToTableBatch","materializeColumns","batchType","length","rowCount"],"sources":["../../../../src/lib/parsers/parse-parquet-to-columns.ts"],"sourcesContent":["// loaders.gl, MIT license\n\nimport {ColumnarTable, ColumnarTableBatch, Schema} from '@loaders.gl/schema';\nimport {makeReadableFile} from '@loaders.gl/loader-utils';\nimport type {ParquetLoaderOptions} from '../../parquet-loader';\nimport {ParquetReader} from '../../parquetjs/parser/parquet-reader';\nimport {ParquetRowGroup} from '../../parquetjs/schema/declare';\nimport {ParquetSchema} from '../../parquetjs/schema/schema';\nimport {materializeColumns} from '../../parquetjs/schema/shred';\nimport {getSchemaFromParquetReader} from './get-parquet-schema';\nimport {installBufferPolyfill} from '../../buffer-polyfill';\n\nexport async function parseParquetInColumns(\n  arrayBuffer: ArrayBuffer,\n  options?: ParquetLoaderOptions\n): Promise<ColumnarTable> {\n  installBufferPolyfill();\n  const blob = new Blob([arrayBuffer]);\n  const file = makeReadableFile(blob);\n  const reader = new ParquetReader(file);\n\n  for await (const batch of parseParquetFileInColumnarBatches(reader, options)) {\n    return {\n      shape: 'columnar-table',\n      schema: batch.schema,\n      data: batch.data\n    };\n  }\n  throw new Error('empty table');\n}\n\nexport async function* parseParquetFileInColumnarBatches(\n  reader: ParquetReader,\n  options?: ParquetLoaderOptions\n): AsyncIterable<ColumnarTableBatch> {\n  // Extract schema and geo metadata\n  const schema = await getSchemaFromParquetReader(reader);\n\n  const parquetSchema = await reader.getSchema();\n\n  // Iterate over row batches\n  const rowGroups = reader.rowGroupIterator(options?.parquet);\n  for await (const rowGroup of rowGroups) {\n    yield convertRowGroupToTableBatch(rowGroup, parquetSchema, schema);\n  }\n}\n\nfunction convertRowGroupToTableBatch(\n  rowGroup: ParquetRowGroup,\n  parquetSchema: ParquetSchema,\n  schema: Schema\n): ColumnarTableBatch {\n  // const data = convertParquetRowGroupToColumns(schema, rowGroup);\n  const data = materializeColumns(parquetSchema, rowGroup);\n  return {\n    shape: 'columnar-table',\n    batchType: 'data',\n    schema,\n    data,\n    length: rowGroup.rowCount\n  };\n}\n"],"mappings":";;;;;;;;;;;;AAGA,IAAAA,YAAA,GAAAC,OAAA;AAEA,IAAAC,cAAA,GAAAD,OAAA;AAGA,IAAAE,MAAA,GAAAF,OAAA;AACA,IAAAG,iBAAA,GAAAH,OAAA;AACA,IAAAI,eAAA,GAAAJ,OAAA;AAA4D,SAAAK,eAAAC,QAAA,QAAAC,MAAA,EAAAC,KAAA,EAAAC,IAAA,EAAAC,KAAA,iCAAAC,MAAA,KAAAH,KAAA,GAAAG,MAAA,CAAAC,aAAA,EAAAH,IAAA,GAAAE,MAAA,CAAAE,QAAA,GAAAH,KAAA,WAAAF,KAAA,aAAAD,MAAA,GAAAD,QAAA,CAAAE,KAAA,WAAAD,MAAA,CAAAO,IAAA,CAAAR,QAAA,OAAAG,IAAA,aAAAF,MAAA,GAAAD,QAAA,CAAAG,IAAA,eAAAM,qBAAA,CAAAR,MAAA,CAAAO,IAAA,CAAAR,QAAA,IAAAE,KAAA,sBAAAC,IAAA,6BAAAO,SAAA;AAAA,SAAAD,sBAAAE,CAAA,aAAAC,kCAAAC,CAAA,QAAAC,MAAA,CAAAD,CAAA,MAAAA,CAAA,SAAAE,OAAA,CAAAC,MAAA,KAAAN,SAAA,CAAAG,CAAA,+BAAAI,IAAA,GAAAJ,CAAA,CAAAI,IAAA,SAAAF,OAAA,CAAAG,OAAA,CAAAL,CAAA,CAAAM,KAAA,EAAAC,IAAA,WAAAD,KAAA,aAAAA,KAAA,EAAAA,KAAA,EAAAF,IAAA,EAAAA,IAAA,iBAAAR,qBAAA,YAAAA,sBAAAE,CAAA,SAAAA,CAAA,GAAAA,CAAA,OAAAU,CAAA,GAAAV,CAAA,CAAAW,IAAA,KAAAb,qBAAA,CAAAc,SAAA,KAAAZ,CAAA,QAAAU,CAAA,QAAAC,IAAA,WAAAA,KAAA,WAAAV,iCAAA,MAAAS,CAAA,CAAAG,KAAA,MAAAb,CAAA,EAAAc,SAAA,OAAAC,MAAA,WAAAC,QAAAR,KAAA,QAAAS,GAAA,QAAAjB,CAAA,CAAAe,MAAA,oBAAAE,GAAA,GAAAb,OAAA,CAAAG,OAAA,GAAAC,KAAA,EAAAA,KAAA,EAAAF,IAAA,UAAAL,iCAAA,CAAAgB,GAAA,CAAAJ,KAAA,MAAAb,CAAA,EAAAc,SAAA,OAAAI,KAAA,WAAAC,OAAAX,KAAA,QAAAY,GAAA,QAAApB,CAAA,CAAAe,MAAA,oBAAAK,GAAA,GAAAhB,OAAA,CAAAC,MAAA,CAAAG,KAAA,IAAAP,iCAAA,CAAAmB,GAAA,CAAAP,KAAA,MAAAb,CAAA,EAAAc,SAAA,aAAAhB,qBAAA,CAAAE,CAAA;AAAA,SAEtCqB,qBAAqBA,CAAAC,GAAA,EAAAC,GAAA;EAAA,OAAAC,sBAAA,CAAAX,KAAA,OAAAC,SAAA;AAAA;AAAA,SAAAU,uBAAA;EAAAA,sBAAA,OAAAC,kBAAA,CAAAC,OAAA,EAAAC,YAAA,CAAAD,OAAA,CAAAE,IAAA,CAApC,SAAAC,SACLC,WAAwB,EACxBC,OAA8B;IAAA,IAAAC,IAAA,EAAAC,IAAA,EAAAC,MAAA,EAAAC,yBAAA,EAAAC,iBAAA,EAAAC,cAAA,EAAAC,SAAA,EAAAC,KAAA,EAAAC,KAAA;IAAA,OAAAb,YAAA,CAAAD,OAAA,CAAAe,IAAA,UAAAC,UAAAC,SAAA;MAAA,kBAAAA,SAAA,CAAAC,IAAA,GAAAD,SAAA,CAAAhC,IAAA;QAAA;UAE9B,IAAAkC,qCAAqB,EAAC,CAAC;UACjBb,IAAI,GAAG,IAAIc,IAAI,CAAC,CAAChB,WAAW,CAAC,CAAC;UAC9BG,IAAI,GAAG,IAAAc,6BAAgB,EAACf,IAAI,CAAC;UAC7BE,MAAM,GAAG,IAAIc,4BAAa,CAACf,IAAI,CAAC;UAAAE,yBAAA;UAAAC,iBAAA;UAAAO,SAAA,CAAAC,IAAA;UAAAN,SAAA,GAAAlD,cAAA,CAEZ6D,iCAAiC,CAACf,MAAM,EAAEH,OAAO,CAAC;QAAA;UAAAY,SAAA,CAAAhC,IAAA;UAAA,OAAA2B,SAAA,CAAA3B,IAAA;QAAA;UAAA,MAAAwB,yBAAA,KAAAI,KAAA,GAAAI,SAAA,CAAAO,IAAA,EAAA5C,IAAA;YAAAqC,SAAA,CAAAhC,IAAA;YAAA;UAAA;UAA3D6B,KAAK,GAAAD,KAAA,CAAA/B,KAAA;UAAA,OAAAmC,SAAA,CAAAQ,MAAA,WACb;YACLC,KAAK,EAAE,gBAAgB;YACvBC,MAAM,EAAEb,KAAK,CAACa,MAAM;YACpBC,IAAI,EAAEd,KAAK,CAACc;UACd,CAAC;QAAA;UAAAnB,yBAAA;UAAAQ,SAAA,CAAAhC,IAAA;UAAA;QAAA;UAAAgC,SAAA,CAAAhC,IAAA;UAAA;QAAA;UAAAgC,SAAA,CAAAC,IAAA;UAAAD,SAAA,CAAAY,EAAA,GAAAZ,SAAA;UAAAP,iBAAA;UAAAC,cAAA,GAAAM,SAAA,CAAAY,EAAA;QAAA;UAAAZ,SAAA,CAAAC,IAAA;UAAAD,SAAA,CAAAC,IAAA;UAAA,MAAAT,yBAAA,IAAAG,SAAA,CAAAvB,MAAA;YAAA4B,SAAA,CAAAhC,IAAA;YAAA;UAAA;UAAAgC,SAAA,CAAAhC,IAAA;UAAA,OAAA2B,SAAA,CAAAvB,MAAA;QAAA;UAAA4B,SAAA,CAAAC,IAAA;UAAA,KAAAR,iBAAA;YAAAO,SAAA,CAAAhC,IAAA;YAAA;UAAA;UAAA,MAAA0B,cAAA;QAAA;UAAA,OAAAM,SAAA,CAAAa,MAAA;QAAA;UAAA,OAAAb,SAAA,CAAAa,MAAA;QAAA;UAAA,MAEG,IAAIC,KAAK,CAAC,aAAa,CAAC;QAAA;QAAA;UAAA,OAAAd,SAAA,CAAAe,IAAA;MAAA;IAAA,GAAA7B,QAAA;EAAA,CAC/B;EAAA,OAAAL,sBAAA,CAAAX,KAAA,OAAAC,SAAA;AAAA;AAAA,SAEsBmC,iCAAiCA,CAAAU,EAAA,EAAAC,GAAA;EAAA,OAAAC,kCAAA,CAAAhD,KAAA,OAAAC,SAAA;AAAA;AAAA,SAAA+C,mCAAA;EAAAA,kCAAA,OAAAC,oBAAA,CAAApC,OAAA,EAAAC,YAAA,CAAAD,OAAA,CAAAE,IAAA,CAAjD,SAAAmC,QACL7B,MAAqB,EACrBH,OAA8B;IAAA,IAAAsB,MAAA,EAAAW,aAAA,EAAAC,SAAA,EAAAC,0BAAA,EAAAC,kBAAA,EAAAC,eAAA,EAAAC,UAAA,EAAAC,MAAA,EAAAC,QAAA;IAAA,OAAA5C,YAAA,CAAAD,OAAA,CAAAe,IAAA,UAAA+B,SAAAC,QAAA;MAAA,kBAAAA,QAAA,CAAA7B,IAAA,GAAA6B,QAAA,CAAA9D,IAAA;QAAA;UAAA8D,QAAA,CAAA9D,IAAA;UAAA,WAAA+D,qBAAA,CAAAhD,OAAA,EAGT,IAAAiD,4CAA0B,EAACzC,MAAM,CAAC;QAAA;UAAjDmB,MAAM,GAAAoB,QAAA,CAAAvB,IAAA;UAAAuB,QAAA,CAAA9D,IAAA;UAAA,WAAA+D,qBAAA,CAAAhD,OAAA,EAEgBQ,MAAM,CAAC0C,SAAS,CAAC,CAAC;QAAA;UAAxCZ,aAAa,GAAAS,QAAA,CAAAvB,IAAA;UAGbe,SAAS,GAAG/B,MAAM,CAAC2C,gBAAgB,CAAC9C,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAE+C,OAAO,CAAC;UAAAZ,0BAAA;UAAAC,kBAAA;UAAAM,QAAA,CAAA7B,IAAA;UAAAyB,UAAA,GAAAjF,cAAA,CAC9B6E,SAAS;QAAA;UAAAQ,QAAA,CAAA9D,IAAA;UAAA,WAAA+D,qBAAA,CAAAhD,OAAA,EAAA2C,UAAA,CAAA1D,IAAA;QAAA;UAAA,MAAAuD,0BAAA,KAAAI,MAAA,GAAAG,QAAA,CAAAvB,IAAA,EAAA5C,IAAA;YAAAmE,QAAA,CAAA9D,IAAA;YAAA;UAAA;UAArB4D,QAAQ,GAAAD,MAAA,CAAA9D,KAAA;UAAAiE,QAAA,CAAA9D,IAAA;UACvB,OAAMoE,2BAA2B,CAACR,QAAQ,EAAEP,aAAa,EAAEX,MAAM,CAAC;QAAA;UAAAa,0BAAA;UAAAO,QAAA,CAAA9D,IAAA;UAAA;QAAA;UAAA8D,QAAA,CAAA9D,IAAA;UAAA;QAAA;UAAA8D,QAAA,CAAA7B,IAAA;UAAA6B,QAAA,CAAAlB,EAAA,GAAAkB,QAAA;UAAAN,kBAAA;UAAAC,eAAA,GAAAK,QAAA,CAAAlB,EAAA;QAAA;UAAAkB,QAAA,CAAA7B,IAAA;UAAA6B,QAAA,CAAA7B,IAAA;UAAA,MAAAsB,0BAAA,IAAAG,UAAA,CAAAtD,MAAA;YAAA0D,QAAA,CAAA9D,IAAA;YAAA;UAAA;UAAA8D,QAAA,CAAA9D,IAAA;UAAA,WAAA+D,qBAAA,CAAAhD,OAAA,EAAA2C,UAAA,CAAAtD,MAAA;QAAA;UAAA0D,QAAA,CAAA7B,IAAA;UAAA,KAAAuB,kBAAA;YAAAM,QAAA,CAAA9D,IAAA;YAAA;UAAA;UAAA,MAAAyD,eAAA;QAAA;UAAA,OAAAK,QAAA,CAAAjB,MAAA;QAAA;UAAA,OAAAiB,QAAA,CAAAjB,MAAA;QAAA;QAAA;UAAA,OAAAiB,QAAA,CAAAf,IAAA;MAAA;IAAA,GAAAK,OAAA;EAAA,CAErE;EAAA,OAAAF,kCAAA,CAAAhD,KAAA,OAAAC,SAAA;AAAA;AAED,SAASiE,2BAA2BA,CAClCR,QAAyB,EACzBP,aAA4B,EAC5BX,MAAc,EACM;EAEpB,IAAMC,IAAI,GAAG,IAAA0B,yBAAkB,EAAChB,aAAa,EAAEO,QAAQ,CAAC;EACxD,OAAO;IACLnB,KAAK,EAAE,gBAAgB;IACvB6B,SAAS,EAAE,MAAM;IACjB5B,MAAM,EAANA,MAAM;IACNC,IAAI,EAAJA,IAAI;IACJ4B,MAAM,EAAEX,QAAQ,CAACY;EACnB,CAAC;AACH"}