@loaders.gl/parquet 4.0.0-alpha.5 → 4.0.0-alpha.6

package/dist/es5/parquetjs/utils/read-utils.js

@@ -0,0 +1,120 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.decodeFileMetadata = decodeFileMetadata;
+exports.decodePageHeader = decodePageHeader;
+exports.decodeThrift = decodeThrift;
+exports.fieldIndexOf = fieldIndexOf;
+exports.getBitWidth = getBitWidth;
+exports.getThriftEnum = getThriftEnum;
+exports.serializeThrift = serializeThrift;
+var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
+var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
+var _assertThisInitialized2 = _interopRequireDefault(require("@babel/runtime/helpers/assertThisInitialized"));
+var _inherits2 = _interopRequireDefault(require("@babel/runtime/helpers/inherits"));
+var _possibleConstructorReturn2 = _interopRequireDefault(require("@babel/runtime/helpers/possibleConstructorReturn"));
+var _getPrototypeOf2 = _interopRequireDefault(require("@babel/runtime/helpers/getPrototypeOf"));
+var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
+var _thrift = require("thrift");
+var _parquetThrift = require("../parquet-thrift");
+function _createSuper(Derived) { var hasNativeReflectConstruct = _isNativeReflectConstruct(); return function _createSuperInternal() { var Super = (0, _getPrototypeOf2.default)(Derived), result; if (hasNativeReflectConstruct) { var NewTarget = (0, _getPrototypeOf2.default)(this).constructor; result = Reflect.construct(Super, arguments, NewTarget); } else { result = Super.apply(this, arguments); } return (0, _possibleConstructorReturn2.default)(this, result); }; }
+function _isNativeReflectConstruct() { if (typeof Reflect === "undefined" || !Reflect.construct) return false; if (Reflect.construct.sham) return false; if (typeof Proxy === "function") return true; try { Boolean.prototype.valueOf.call(Reflect.construct(Boolean, [], function () {})); return true; } catch (e) { return false; } }
+var UFramedTransport = function (_TFramedTransport) {
+  (0, _inherits2.default)(UFramedTransport, _TFramedTransport);
+  var _super = _createSuper(UFramedTransport);
+  function UFramedTransport() {
+    var _this;
+    (0, _classCallCheck2.default)(this, UFramedTransport);
+    for (var _len = arguments.length, args = new Array(_len), _key = 0; _key < _len; _key++) {
+      args[_key] = arguments[_key];
+    }
+    _this = _super.call.apply(_super, [this].concat(args));
+    (0, _defineProperty2.default)((0, _assertThisInitialized2.default)(_this), "readPos", 0);
+    return _this;
+  }
+  return (0, _createClass2.default)(UFramedTransport);
+}(_thrift.TFramedTransport);
+function serializeThrift(obj) {
+  var output = [];
+  var transport = new _thrift.TBufferedTransport(undefined, function (buf) {
+    output.push(buf);
+  });
+  var protocol = new _thrift.TCompactProtocol(transport);
+  obj.write(protocol);
+  transport.flush();
+  return Buffer.concat(output);
+}
+function decodeThrift(obj, buf, offset) {
+  if (!offset) {
+    offset = 0;
+  }
+  var transport = new UFramedTransport(buf);
+  transport.readPos = offset;
+  var protocol = new _thrift.TCompactProtocol(transport);
+  obj.read(protocol);
+  return transport.readPos - offset;
+}
+function getThriftEnum(klass, value) {
+  for (var k in klass) {
+    if (klass[k] === value) {
+      return k;
+    }
+  }
+  throw new Error('Invalid ENUM value');
+}
+function decodeFileMetadata(buf, offset) {
+  if (!offset) {
+    offset = 0;
+  }
+  var transport = new UFramedTransport(buf);
+  transport.readPos = offset;
+  var protocol = new _thrift.TCompactProtocol(transport);
+  var metadata = _parquetThrift.FileMetaData.read(protocol);
+  return {
+    length: transport.readPos - offset,
+    metadata: metadata
+  };
+}
+function decodePageHeader(buf, offset) {
+  if (!offset) {
+    offset = 0;
+  }
+  var transport = new UFramedTransport(buf);
+  transport.readPos = offset;
+  var protocol = new _thrift.TCompactProtocol(transport);
+  var pageHeader = _parquetThrift.PageHeader.read(protocol);
+  return {
+    length: transport.readPos - offset,
+    pageHeader: pageHeader
+  };
+}
+function getBitWidth(val) {
+  if (val === 0) {
+    return 0;
+  }
+  return Math.ceil(Math.log2(val + 1));
+}
+function fieldIndexOf(arr, elem) {
+  for (var j = 0; j < arr.length; j++) {
+    if (arr[j].length > elem.length) {
+      continue;
+    }
+    var m = true;
+    for (var i = 0; i < elem.length; i++) {
+      if (arr[j][i] === elem[i] || arr[j][i] === '+' || arr[j][i] === '#') {
+        continue;
+      }
+      if (i >= arr[j].length && arr[j][arr[j].length - 1] === '#') {
+        continue;
+      }
+      m = false;
+      break;
+    }
+    if (m) return j;
+  }
+  return -1;
+}
+//# sourceMappingURL=read-utils.js.map

package/dist/es5/parquetjs/utils/read-utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"read-utils.js","names":["_thrift","require","_parquetThrift","_createSuper","Derived","hasNativeReflectConstruct","_isNativeReflectConstruct","_createSuperInternal","Super","_getPrototypeOf2","default","result","NewTarget","constructor","Reflect","construct","arguments","apply","_possibleConstructorReturn2","sham","Proxy","Boolean","prototype","valueOf","call","e","UFramedTransport","_TFramedTransport","_inherits2","_super","_this","_classCallCheck2","_len","length","args","Array","_key","concat","_defineProperty2","_assertThisInitialized2","_createClass2","TFramedTransport","serializeThrift","obj","output","transport","TBufferedTransport","undefined","buf","push","protocol","TCompactProtocol","write","flush","Buffer","decodeThrift","offset","readPos","read","getThriftEnum","klass","value","k","Error","decodeFileMetadata","metadata","FileMetaData","decodePageHeader","pageHeader","PageHeader","getBitWidth","val","Math","ceil","log2","fieldIndexOf","arr","elem","j","m","i"],"sources":["../../../../src/parquetjs/utils/read-utils.ts"],"sourcesContent":["import {TBufferedTransport, TCompactProtocol, TFramedTransport} from 'thrift';\nimport {FileMetaData, PageHeader} from '../parquet-thrift';\n\nclass UFramedTransport extends TFramedTransport {\n  public readPos: number = 0;\n}\n\n/**\n * Helper function that serializes a thrift object into a buffer\n */\nexport function serializeThrift(obj: any): Buffer {\n  const output: Buffer[] = [];\n\n  const transport = new TBufferedTransport(undefined, (buf) => {\n    output.push(buf as Buffer);\n  });\n\n  const protocol = new TCompactProtocol(transport);\n  obj.write(protocol);\n  transport.flush();\n\n  return Buffer.concat(output);\n}\n\nexport function decodeThrift(obj: any, buf: Buffer, offset?: number) {\n  if (!offset) {\n    // tslint:disable-next-line:no-parameter-reassignment\n    offset = 0;\n  }\n\n  const transport = new UFramedTransport(buf);\n  transport.readPos = offset;\n  const protocol = new TCompactProtocol(transport);\n  obj.read(protocol);\n  return transport.readPos - offset;\n}\n\n/**\n * FIXME not ideal that this is linear\n */\nexport function getThriftEnum(klass: any, value: number | string): string {\n  for (const k in klass) {\n    if (klass[k] === value) {\n      return k;\n    }\n  }\n  throw new Error('Invalid ENUM value');\n}\n\nexport function decodeFileMetadata(buf: Buffer, offset?: number) {\n  if (!offset) {\n    // tslint:disable-next-line:no-parameter-reassignment\n    offset = 0;\n  }\n\n  const transport = new UFramedTransport(buf);\n  transport.readPos = offset;\n  const protocol = new TCompactProtocol(transport);\n  const metadata = FileMetaData.read(protocol);\n  return {length: transport.readPos - offset, metadata};\n}\n\nexport function decodePageHeader(buf: Buffer, offset?: number) {\n  if (!offset) {\n    // tslint:disable-next-line:no-parameter-reassignment\n    offset = 0;\n  }\n\n  const transport = new UFramedTransport(buf);\n  transport.readPos = offset;\n  const protocol = new TCompactProtocol(transport);\n  const pageHeader = PageHeader.read(protocol);\n  return {length: transport.readPos - offset, pageHeader};\n}\n\n/**\n * Get the number of bits required to store a given value\n */\nexport function getBitWidth(val: number): number {\n  if (val === 0) {\n    return 0;\n    // tslint:disable-next-line:no-else-after-return\n  }\n  return Math.ceil(Math.log2(val + 1));\n}\n\n// Supports MQTT path wildcards\n// + all immediate children\n// # all descendents\nexport function fieldIndexOf(arr: string[][], elem: string[]): number {\n  for (let j = 0; j < arr.length; j++) {\n    if (arr[j].length > elem.length) {\n      continue; // eslint-disable-line no-continue\n    }\n    let m = true;\n    for (let i = 0; i < elem.length; i++) {\n      if (arr[j][i] === elem[i] || arr[j][i] === '+' || arr[j][i] === '#') {\n        continue; // eslint-disable-line no-continue\n      }\n      if (i >= arr[j].length && arr[j][arr[j].length - 1] === '#') {\n        continue; // eslint-disable-line no-continue\n      }\n      m = false;\n      break;\n    }\n    if (m) return j;\n  }\n  return -1;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,cAAA,GAAAD,OAAA;AAA2D,SAAAE,aAAAC,OAAA,QAAAC,yBAAA,GAAAC,yBAAA,oBAAAC,qBAAA,QAAAC,KAAA,OAAAC,gBAAA,CAAAC,OAAA,EAAAN,OAAA,GAAAO,MAAA,MAAAN,yBAAA,QAAAO,SAAA,OAAAH,gBAAA,CAAAC,OAAA,QAAAG,WAAA,EAAAF,MAAA,GAAAG,OAAA,CAAAC,SAAA,CAAAP,KAAA,EAAAQ,SAAA,EAAAJ,SAAA,YAAAD,MAAA,GAAAH,KAAA,CAAAS,KAAA,OAAAD,SAAA,gBAAAE,2BAAA,CAAAR,OAAA,QAAAC,MAAA;AAAA,SAAAL,0BAAA,eAAAQ,OAAA,qBAAAA,OAAA,CAAAC,SAAA,oBAAAD,OAAA,CAAAC,SAAA,CAAAI,IAAA,2BAAAC,KAAA,oCAAAC,OAAA,CAAAC,SAAA,CAAAC,OAAA,CAAAC,IAAA,CAAAV,OAAA,CAAAC,SAAA,CAAAM,OAAA,8CAAAI,CAAA;AAAA,IAErDC,gBAAgB,aAAAC,iBAAA;EAAA,IAAAC,UAAA,CAAAlB,OAAA,EAAAgB,gBAAA,EAAAC,iBAAA;EAAA,IAAAE,MAAA,GAAA1B,YAAA,CAAAuB,gBAAA;EAAA,SAAAA,iBAAA;IAAA,IAAAI,KAAA;IAAA,IAAAC,gBAAA,CAAArB,OAAA,QAAAgB,gBAAA;IAAA,SAAAM,IAAA,GAAAhB,SAAA,CAAAiB,MAAA,EAAAC,IAAA,OAAAC,KAAA,CAAAH,IAAA,GAAAI,IAAA,MAAAA,IAAA,GAAAJ,IAAA,EAAAI,IAAA;MAAAF,IAAA,CAAAE,IAAA,IAAApB,SAAA,CAAAoB,IAAA;IAAA;IAAAN,KAAA,GAAAD,MAAA,CAAAL,IAAA,CAAAP,KAAA,CAAAY,MAAA,SAAAQ,MAAA,CAAAH,IAAA;IAAA,IAAAI,gBAAA,CAAA5B,OAAA,MAAA6B,uBAAA,CAAA7B,OAAA,EAAAoB,KAAA,cACK,CAAC;IAAA,OAAAA,KAAA;EAAA;EAAA,WAAAU,aAAA,CAAA9B,OAAA,EAAAgB,gBAAA;AAAA,EADGe,wBAAgB;AAOxC,SAASC,eAAeA,CAACC,GAAQ,EAAU;EAChD,IAAMC,MAAgB,GAAG,EAAE;EAE3B,IAAMC,SAAS,GAAG,IAAIC,0BAAkB,CAACC,SAAS,EAAE,UAACC,GAAG,EAAK;IAC3DJ,MAAM,CAACK,IAAI,CAACD,GAAa,CAAC;EAC5B,CAAC,CAAC;EAEF,IAAME,QAAQ,GAAG,IAAIC,wBAAgB,CAACN,SAAS,CAAC;EAChDF,GAAG,CAACS,KAAK,CAACF,QAAQ,CAAC;EACnBL,SAAS,CAACQ,KAAK,CAAC,CAAC;EAEjB,OAAOC,MAAM,CAACjB,MAAM,CAACO,MAAM,CAAC;AAC9B;AAEO,SAASW,YAAYA,CAACZ,GAAQ,EAAEK,GAAW,EAAEQ,MAAe,EAAE;EACnE,IAAI,CAACA,MAAM,EAAE;IAEXA,MAAM,GAAG,CAAC;EACZ;EAEA,IAAMX,SAAS,GAAG,IAAInB,gBAAgB,CAACsB,GAAG,CAAC;EAC3CH,SAAS,CAACY,OAAO,GAAGD,MAAM;EAC1B,IAAMN,QAAQ,GAAG,IAAIC,wBAAgB,CAACN,SAAS,CAAC;EAChDF,GAAG,CAACe,IAAI,CAACR,QAAQ,CAAC;EAClB,OAAOL,SAAS,CAACY,OAAO,GAAGD,MAAM;AACnC;AAKO,SAASG,aAAaA,CAACC,KAAU,EAAEC,KAAsB,EAAU;EACxE,KAAK,IAAMC,CAAC,IAAIF,KAAK,EAAE;IACrB,IAAIA,KAAK,CAACE,CAAC,CAAC,KAAKD,KAAK,EAAE;MACtB,OAAOC,CAAC;IACV;EACF;EACA,MAAM,IAAIC,KAAK,CAAC,oBAAoB,CAAC;AACvC;AAEO,SAASC,kBAAkBA,CAAChB,GAAW,EAAEQ,MAAe,EAAE;EAC/D,IAAI,CAACA,MAAM,EAAE;IAEXA,MAAM,GAAG,CAAC;EACZ;EAEA,IAAMX,SAAS,GAAG,IAAInB,gBAAgB,CAACsB,GAAG,CAAC;EAC3CH,SAAS,CAACY,OAAO,GAAGD,MAAM;EAC1B,IAAMN,QAAQ,GAAG,IAAIC,wBAAgB,CAACN,SAAS,CAAC;EAChD,IAAMoB,QAAQ,GAAGC,2BAAY,CAACR,IAAI,CAACR,QAAQ,CAAC;EAC5C,OAAO;IAACjB,MAAM,EAAEY,SAAS,CAACY,OAAO,GAAGD,MAAM;IAAES,QAAQ,EAARA;EAAQ,CAAC;AACvD;AAEO,SAASE,gBAAgBA,CAACnB,GAAW,EAAEQ,MAAe,EAAE;EAC7D,IAAI,CAACA,MAAM,EAAE;IAEXA,MAAM,GAAG,CAAC;EACZ;EAEA,IAAMX,SAAS,GAAG,IAAInB,gBAAgB,CAACsB,GAAG,CAAC;EAC3CH,SAAS,CAACY,OAAO,GAAGD,MAAM;EAC1B,IAAMN,QAAQ,GAAG,IAAIC,wBAAgB,CAACN,SAAS,CAAC;EAChD,IAAMuB,UAAU,GAAGC,yBAAU,CAACX,IAAI,CAACR,QAAQ,CAAC;EAC5C,OAAO;IAACjB,MAAM,EAAEY,SAAS,CAACY,OAAO,GAAGD,MAAM;IAAEY,UAAU,EAAVA;EAAU,CAAC;AACzD;AAKO,SAASE,WAAWA,CAACC,GAAW,EAAU;EAC/C,IAAIA,GAAG,KAAK,CAAC,EAAE;IACb,OAAO,CAAC;EAEV;EACA,OAAOC,IAAI,CAACC,IAAI,CAACD,IAAI,CAACE,IAAI,CAACH,GAAG,GAAG,CAAC,CAAC,CAAC;AACtC;AAKO,SAASI,YAAYA,CAACC,GAAe,EAAEC,IAAc,EAAU;EACpE,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGF,GAAG,CAAC3C,MAAM,EAAE6C,CAAC,EAAE,EAAE;IACnC,IAAIF,GAAG,CAACE,CAAC,CAAC,CAAC7C,MAAM,GAAG4C,IAAI,CAAC5C,MAAM,EAAE;MAC/B;IACF;IACA,IAAI8C,CAAC,GAAG,IAAI;IACZ,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGH,IAAI,CAAC5C,MAAM,EAAE+C,CAAC,EAAE,EAAE;MACpC,IAAIJ,GAAG,CAACE,CAAC,CAAC,CAACE,CAAC,CAAC,KAAKH,IAAI,CAACG,CAAC,CAAC,IAAIJ,GAAG,CAACE,CAAC,CAAC,CAACE,CAAC,CAAC,KAAK,GAAG,IAAIJ,GAAG,CAACE,CAAC,CAAC,CAACE,CAAC,CAAC,KAAK,GAAG,EAAE;QACnE;MACF;MACA,IAAIA,CAAC,IAAIJ,GAAG,CAACE,CAAC,CAAC,CAAC7C,MAAM,IAAI2C,GAAG,CAACE,CAAC,CAAC,CAACF,GAAG,CAACE,CAAC,CAAC,CAAC7C,MAAM,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE;QAC3D;MACF;MACA8C,CAAC,GAAG,KAAK;MACT;IACF;IACA,IAAIA,CAAC,EAAE,OAAOD,CAAC;EACjB;EACA,OAAO,CAAC,CAAC;AACX"}

package/dist/es5/workers/parquet-worker.js

@@ -0,0 +1,6 @@
+"use strict";
+
+var _loaderUtils = require("@loaders.gl/loader-utils");
+var _index = require("../index");
+(0, _loaderUtils.createLoaderWorker)(_index.ParquetLoader);
+//# sourceMappingURL=parquet-worker.js.map

package/dist/es5/workers/parquet-worker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-worker.js","names":["_loaderUtils","require","_index","createLoaderWorker","ParquetLoader"],"sources":["../../../src/workers/parquet-worker.ts"],"sourcesContent":["import {createLoaderWorker} from '@loaders.gl/loader-utils';\nimport {ParquetLoader} from '../index';\n\ncreateLoaderWorker(ParquetLoader);\n"],"mappings":";;AAAA,IAAAA,YAAA,GAAAC,OAAA;AACA,IAAAC,MAAA,GAAAD,OAAA;AAEA,IAAAE,+BAAkB,EAACC,oBAAa,CAAC"}

package/dist/esm/bundle.js

@@ -0,0 +1,4 @@
+const moduleExports = require('./index');
+globalThis.loaders = globalThis.loaders || {};
+module.exports = Object.assign(globalThis.loaders, moduleExports);
+//# sourceMappingURL=bundle.js.map

package/dist/esm/bundle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bundle.js","names":["moduleExports","require","globalThis","loaders","module","exports","Object","assign"],"sources":["../../src/bundle.ts"],"sourcesContent":["// @ts-nocheck\nconst moduleExports = require('./index');\nglobalThis.loaders = globalThis.loaders || {};\nmodule.exports = Object.assign(globalThis.loaders, moduleExports);\n"],"mappings":"AACA,MAAMA,aAAa,GAAGC,OAAO,CAAC,SAAS,CAAC;AACxCC,UAAU,CAACC,OAAO,GAAGD,UAAU,CAACC,OAAO,IAAI,CAAC,CAAC;AAC7CC,MAAM,CAACC,OAAO,GAAGC,MAAM,CAACC,MAAM,CAACL,UAAU,CAACC,OAAO,EAAEH,aAAa,CAAC"}

package/dist/esm/constants.js

@@ -0,0 +1,6 @@
+export const PARQUET_MAGIC = 'PAR1';
+export const PARQUET_MAGIC_ENCRYPTED = 'PARE';
+export const PARQUET_VERSION = 1;
+export const PARQUET_RDLVL_TYPE = 'INT32';
+export const PARQUET_RDLVL_ENCODING = 'RLE';
+//# sourceMappingURL=constants.js.map

package/dist/esm/constants.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"constants.js","names":["PARQUET_MAGIC","PARQUET_MAGIC_ENCRYPTED","PARQUET_VERSION","PARQUET_RDLVL_TYPE","PARQUET_RDLVL_ENCODING"],"sources":["../../src/constants.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/**\n * Parquet File Magic String\n */\nexport const PARQUET_MAGIC = 'PAR1';\nexport const PARQUET_MAGIC_ENCRYPTED = 'PARE';\n\n/**\n * Parquet File Format Version\n */\nexport const PARQUET_VERSION = 1;\n\n/**\n * Internal type used for repetition/definition levels\n */\nexport const PARQUET_RDLVL_TYPE = 'INT32';\nexport const PARQUET_RDLVL_ENCODING = 'RLE';\n"],"mappings":"AAIA,OAAO,MAAMA,aAAa,GAAG,MAAM;AACnC,OAAO,MAAMC,uBAAuB,GAAG,MAAM;AAK7C,OAAO,MAAMC,eAAe,GAAG,CAAC;AAKhC,OAAO,MAAMC,kBAAkB,GAAG,OAAO;AACzC,OAAO,MAAMC,sBAAsB,GAAG,KAAK"}

package/dist/esm/index.js

@@ -0,0 +1,31 @@
+import { ParquetLoader as ParquetWorkerLoader, ParquetLoader as ParquetColumnarWorkerLoader } from './parquet-loader';
+import { parseParquet, parseParquetFileInBatches } from './lib/parsers/parse-parquet-to-rows';
+import { parseParquetInColumns, parseParquetFileInColumnarBatches } from './lib/parsers/parse-parquet-to-columns';
+import { parseParquetWasm } from './lib/wasm/parse-parquet-wasm';
+import { ParquetWasmLoader as ParquetWasmWorkerLoader } from './parquet-wasm-loader';
+export { ParquetWorkerLoader, ParquetWasmWorkerLoader };
+export const ParquetLoader = {
+  ...ParquetWorkerLoader,
+  parse: parseParquet,
+  parseFileInBatches: parseParquetFileInBatches
+};
+export const ParquetColumnarLoader = {
+  ...ParquetColumnarWorkerLoader,
+  parse: parseParquetInColumns,
+  parseFileInBatches: parseParquetFileInColumnarBatches
+};
+export const ParquetWasmLoader = {
+  ...ParquetWasmWorkerLoader,
+  parse: parseParquetWasm
+};
+export { ParquetWriter as _ParquetWriter } from './parquet-writer';
+export { ParquetWasmWriter } from './parquet-wasm-writer';
+export { preloadCompressions } from './parquetjs/compression';
+export { ParquetSchema } from './parquetjs/schema/schema';
+export { ParquetReader } from './parquetjs/parser/parquet-reader';
+export { ParquetEncoder } from './parquetjs/encoder/parquet-encoder';
+export { convertParquetSchema, convertParquetSchema as convertParquetToArrowSchema } from './lib/arrow/convert-schema-from-parquet';
+export const _typecheckParquetLoader = ParquetLoader;
+export { default as geoJSONSchema } from './lib/geo/geoparquet-schema';
+export { getGeoMetadata, setGeoMetadata, unpackGeoMetadata } from './lib/geo/decode-geo-metadata';
+//# sourceMappingURL=index.js.map

package/dist/esm/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","names":["ParquetLoader","ParquetWorkerLoader","ParquetColumnarWorkerLoader","parseParquet","parseParquetFileInBatches","parseParquetInColumns","parseParquetFileInColumnarBatches","parseParquetWasm","ParquetWasmLoader","ParquetWasmWorkerLoader","parse","parseFileInBatches","ParquetColumnarLoader","ParquetWriter","_ParquetWriter","ParquetWasmWriter","preloadCompressions","ParquetSchema","ParquetReader","ParquetEncoder","convertParquetSchema","convertParquetToArrowSchema","_typecheckParquetLoader","default","geoJSONSchema","getGeoMetadata","setGeoMetadata","unpackGeoMetadata"],"sources":["../../src/index.ts"],"sourcesContent":["// loaders.gl, MIT license\n\nimport type {LoaderWithParser} from '@loaders.gl/loader-utils';\nimport type {\n  ObjectRowTable,\n  ObjectRowTableBatch,\n  ColumnarTable,\n  ColumnarTableBatch\n} from '@loaders.gl/schema';\nimport type {Table as ArrowTable} from 'apache-arrow';\n\n// ParquetLoader\n\nimport {\n  ParquetLoader as ParquetWorkerLoader,\n  ParquetLoader as ParquetColumnarWorkerLoader,\n  ParquetLoaderOptions\n} from './parquet-loader';\nimport {parseParquet, parseParquetFileInBatches} from './lib/parsers/parse-parquet-to-rows';\nimport {\n  parseParquetInColumns,\n  parseParquetFileInColumnarBatches\n} from './lib/parsers/parse-parquet-to-columns';\n\nimport {parseParquetWasm, ParquetWasmLoaderOptions} from './lib/wasm/parse-parquet-wasm';\nimport {ParquetWasmLoader as ParquetWasmWorkerLoader} from './parquet-wasm-loader';\n\nexport {ParquetWorkerLoader, ParquetWasmWorkerLoader};\n\n/** ParquetJS table loader */\nexport const ParquetLoader: LoaderWithParser<\n  ObjectRowTable,\n  ObjectRowTableBatch,\n  ParquetLoaderOptions\n> = {\n  ...ParquetWorkerLoader,\n  parse: parseParquet,\n  parseFileInBatches: parseParquetFileInBatches\n};\n\n/** ParquetJS table loader */\n// @ts-expect-error\nexport const ParquetColumnarLoader: LoaderWithParser<\n  ColumnarTable,\n  ColumnarTableBatch,\n  ParquetLoaderOptions\n> = {\n  ...ParquetColumnarWorkerLoader,\n  parse: parseParquetInColumns,\n  parseFileInBatches: parseParquetFileInColumnarBatches\n};\n\nexport const ParquetWasmLoader: LoaderWithParser<ArrowTable, never, ParquetWasmLoaderOptions> = {\n  ...ParquetWasmWorkerLoader,\n  parse: parseParquetWasm\n};\n\n// ParquetWriter\n\nexport {ParquetWriter as _ParquetWriter} from './parquet-writer';\nexport {ParquetWasmWriter} from './parquet-wasm-writer';\n\n// EXPERIMENTAL - expose the internal parquetjs API\n\nexport {preloadCompressions} from './parquetjs/compression';\n\nexport {ParquetSchema} from './parquetjs/schema/schema';\nexport {ParquetReader} from './parquetjs/parser/parquet-reader';\nexport {ParquetEncoder} from './parquetjs/encoder/parquet-encoder';\n\nexport {\n  convertParquetSchema,\n  convertParquetSchema as convertParquetToArrowSchema\n} from './lib/arrow/convert-schema-from-parquet';\n\n// TESTS\nexport const _typecheckParquetLoader: LoaderWithParser = ParquetLoader;\n\n// Geo Metadata\nexport {default as geoJSONSchema} from './lib/geo/geoparquet-schema';\n\nexport type {GeoMetadata} from './lib/geo/decode-geo-metadata';\nexport {getGeoMetadata, setGeoMetadata, unpackGeoMetadata} from './lib/geo/decode-geo-metadata';\n"],"mappings":"AAaA,SACEA,aAAa,IAAIC,mBAAmB,EACpCD,aAAa,IAAIE,2BAA2B,QAEvC,kBAAkB;AACzB,SAAQC,YAAY,EAAEC,yBAAyB,QAAO,qCAAqC;AAC3F,SACEC,qBAAqB,EACrBC,iCAAiC,QAC5B,wCAAwC;AAE/C,SAAQC,gBAAgB,QAAiC,+BAA+B;AACxF,SAAQC,iBAAiB,IAAIC,uBAAuB,QAAO,uBAAuB;AAElF,SAAQR,mBAAmB,EAAEQ,uBAAuB;AAGpD,OAAO,MAAMT,aAIZ,GAAG;EACF,GAAGC,mBAAmB;EACtBS,KAAK,EAAEP,YAAY;EACnBQ,kBAAkB,EAAEP;AACtB,CAAC;AAID,OAAO,MAAMQ,qBAIZ,GAAG;EACF,GAAGV,2BAA2B;EAC9BQ,KAAK,EAAEL,qBAAqB;EAC5BM,kBAAkB,EAAEL;AACtB,CAAC;AAED,OAAO,MAAME,iBAAgF,GAAG;EAC9F,GAAGC,uBAAuB;EAC1BC,KAAK,EAAEH;AACT,CAAC;AAID,SAAQM,aAAa,IAAIC,cAAc,QAAO,kBAAkB;AAChE,SAAQC,iBAAiB,QAAO,uBAAuB;AAIvD,SAAQC,mBAAmB,QAAO,yBAAyB;AAE3D,SAAQC,aAAa,QAAO,2BAA2B;AACvD,SAAQC,aAAa,QAAO,mCAAmC;AAC/D,SAAQC,cAAc,QAAO,qCAAqC;AAElE,SACEC,oBAAoB,EACpBA,oBAAoB,IAAIC,2BAA2B,QAC9C,yCAAyC;AAGhD,OAAO,MAAMC,uBAAyC,GAAGtB,aAAa;AAGtE,SAAQuB,OAAO,IAAIC,aAAa,QAAO,6BAA6B;AAGpE,SAAQC,cAAc,EAAEC,cAAc,EAAEC,iBAAiB,QAAO,+BAA+B"}

package/dist/esm/lib/arrow/convert-columns-to-row-group.js

@@ -0,0 +1,2 @@
+
+//# sourceMappingURL=convert-columns-to-row-group.js.map

package/dist/esm/lib/arrow/convert-columns-to-row-group.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"convert-columns-to-row-group.js","names":[],"sources":["../../../../src/lib/arrow/convert-columns-to-row-group.ts"],"sourcesContent":[""],"mappings":""}

package/dist/esm/lib/arrow/convert-row-group-to-columns.js

@@ -0,0 +1,8 @@
+export function convertParquetRowGroupToColumns(schema, rowGroup) {
+  const columns = {};
+  for (const [columnName, data] of Object.entries(rowGroup.columnData)) {
+    columns[columnName] = columns[columnName] || data.values;
+  }
+  return columns;
+}
+//# sourceMappingURL=convert-row-group-to-columns.js.map

package/dist/esm/lib/arrow/convert-row-group-to-columns.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"convert-row-group-to-columns.js","names":["convertParquetRowGroupToColumns","schema","rowGroup","columns","columnName","data","Object","entries","columnData","values"],"sources":["../../../../src/lib/arrow/convert-row-group-to-columns.ts"],"sourcesContent":["// loaders.gl, MIT license\n\nimport {Schema} from '@loaders.gl/schema';\nimport {ParquetRowGroup} from '@loaders.gl/parquet/parquetjs/schema/declare';\n\nexport function convertParquetRowGroupToColumns(\n  schema: Schema,\n  rowGroup: ParquetRowGroup\n): Record<string, any[]> {\n  const columns: Record<string, any[]> = {};\n  for (const [columnName, data] of Object.entries(rowGroup.columnData)) {\n    columns[columnName] = columns[columnName] || data.values;\n  }\n  return columns;\n}\n"],"mappings":"AAKA,OAAO,SAASA,+BAA+BA,CAC7CC,MAAc,EACdC,QAAyB,EACF;EACvB,MAAMC,OAA8B,GAAG,CAAC,CAAC;EACzC,KAAK,MAAM,CAACC,UAAU,EAAEC,IAAI,CAAC,IAAIC,MAAM,CAACC,OAAO,CAACL,QAAQ,CAACM,UAAU,CAAC,EAAE;IACpEL,OAAO,CAACC,UAAU,CAAC,GAAGD,OAAO,CAACC,UAAU,CAAC,IAAIC,IAAI,CAACI,MAAM;EAC1D;EACA,OAAON,OAAO;AAChB"}

package/dist/esm/lib/arrow/convert-schema-from-parquet.js

@@ -0,0 +1,95 @@
+export const PARQUET_TYPE_MAPPING = {
+  BOOLEAN: 'bool',
+  INT32: 'int32',
+  INT64: 'float64',
+  INT96: 'float64',
+  FLOAT: 'float32',
+  DOUBLE: 'float64',
+  BYTE_ARRAY: 'binary',
+  FIXED_LEN_BYTE_ARRAY: 'binary',
+  UTF8: 'utf8',
+  DATE: 'int32',
+  TIME_MILLIS: 'int64',
+  TIME_MICROS: 'int64',
+  TIMESTAMP_MILLIS: 'int64',
+  TIMESTAMP_MICROS: 'int64',
+  UINT_8: 'int32',
+  UINT_16: 'uint16',
+  UINT_32: 'uint32',
+  UINT_64: 'uint64',
+  INT_8: 'int8',
+  INT_16: 'int16',
+  INT_32: 'int32',
+  INT_64: 'int64',
+  JSON: 'binary',
+  BSON: 'binary',
+  INTERVAL: 'binary',
+  DECIMAL_INT32: 'float32',
+  DECIMAL_INT64: 'float64',
+  DECIMAL_BYTE_ARRAY: 'float64',
+  DECIMAL_FIXED_LEN_BYTE_ARRAY: 'float64'
+};
+export function convertParquetSchema(parquetSchema, parquetMetadata) {
+  const fields = getFields(parquetSchema.schema);
+  const metadata = parquetMetadata && getSchemaMetadata(parquetMetadata);
+  const schema = {
+    fields,
+    metadata: metadata || {}
+  };
+  return schema;
+}
+function getFields(schema) {
+  const fields = [];
+  for (const name in schema) {
+    const field = schema[name];
+    if (field.fields) {
+      const children = getFields(field.fields);
+      fields.push({
+        name,
+        type: {
+          type: 'struct',
+          children
+        },
+        nullable: field.optional
+      });
+    } else {
+      const type = PARQUET_TYPE_MAPPING[field.type];
+      const metadata = getFieldMetadata(field);
+      const arrowField = {
+        name,
+        type,
+        nullable: field.optional,
+        metadata
+      };
+      fields.push(arrowField);
+    }
+  }
+  return fields;
+}
+function getFieldMetadata(field) {
+  let metadata;
+  for (const key in field) {
+    if (key !== 'name') {
+      let value = field[key] || '';
+      value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
+      metadata = metadata || {};
+      metadata[key] = value;
+    }
+  }
+  return metadata;
+}
+function getSchemaMetadata(parquetMetadata) {
+  let metadata;
+  const keyValueList = parquetMetadata.key_value_metadata || [];
+  for (const {
+    key,
+    value
+  } of keyValueList) {
+    if (typeof value === 'string') {
+      metadata = metadata || {};
+      metadata[key] = value;
+    }
+  }
+  return metadata;
+}
+//# sourceMappingURL=convert-schema-from-parquet.js.map

package/dist/esm/lib/arrow/convert-schema-from-parquet.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"convert-schema-from-parquet.js","names":["PARQUET_TYPE_MAPPING","BOOLEAN","INT32","INT64","INT96","FLOAT","DOUBLE","BYTE_ARRAY","FIXED_LEN_BYTE_ARRAY","UTF8","DATE","TIME_MILLIS","TIME_MICROS","TIMESTAMP_MILLIS","TIMESTAMP_MICROS","UINT_8","UINT_16","UINT_32","UINT_64","INT_8","INT_16","INT_32","INT_64","JSON","BSON","INTERVAL","DECIMAL_INT32","DECIMAL_INT64","DECIMAL_BYTE_ARRAY","DECIMAL_FIXED_LEN_BYTE_ARRAY","convertParquetSchema","parquetSchema","parquetMetadata","fields","getFields","schema","metadata","getSchemaMetadata","name","field","children","push","type","nullable","optional","getFieldMetadata","arrowField","key","value","stringify","keyValueList","key_value_metadata"],"sources":["../../../../src/lib/arrow/convert-schema-from-parquet.ts"],"sourcesContent":["// loaders.gl, MIT license\n\nimport {Schema, Field, DataType} from '@loaders.gl/schema';\n\nimport type {ParquetSchema} from '../../parquetjs/schema/schema';\nimport type {FieldDefinition, ParquetField, ParquetType} from '../../parquetjs/schema/declare';\nimport {FileMetaData} from '../../parquetjs/parquet-thrift';\n\nexport const PARQUET_TYPE_MAPPING: {[type in ParquetType]: DataType} = {\n  BOOLEAN: 'bool',\n  INT32: 'int32',\n  INT64: 'float64',\n  INT96: 'float64',\n  FLOAT: 'float32',\n  DOUBLE: 'float64',\n  BYTE_ARRAY: 'binary',\n  FIXED_LEN_BYTE_ARRAY: 'binary',\n  UTF8: 'utf8',\n  DATE: 'int32',\n  TIME_MILLIS: 'int64',\n  TIME_MICROS: 'int64',\n  TIMESTAMP_MILLIS: 'int64',\n  TIMESTAMP_MICROS: 'int64',\n  UINT_8: 'int32',\n  UINT_16: 'uint16',\n  UINT_32: 'uint32',\n  UINT_64: 'uint64',\n  INT_8: 'int8',\n  INT_16: 'int16',\n  INT_32: 'int32',\n  INT_64: 'int64',\n  JSON: 'binary',\n  BSON: 'binary',\n  // TODO check interal type\n  INTERVAL: 'binary',\n  DECIMAL_INT32: 'float32',\n  DECIMAL_INT64: 'float64',\n  DECIMAL_BYTE_ARRAY: 'float64',\n  DECIMAL_FIXED_LEN_BYTE_ARRAY: 'float64'\n};\n\nexport function convertParquetSchema(\n  parquetSchema: ParquetSchema,\n  parquetMetadata: FileMetaData | null\n): Schema {\n  const fields = getFields(parquetSchema.schema);\n  const metadata = parquetMetadata && getSchemaMetadata(parquetMetadata);\n\n  const schema: Schema = {\n    fields,\n    metadata: metadata || {}\n  };\n\n  return schema;\n}\n\nfunction getFields(schema: FieldDefinition): Field[] {\n  const fields: Field[] = [];\n\n  for (const name in schema) {\n    const field = schema[name];\n\n    if (field.fields) {\n      const children = getFields(field.fields);\n      fields.push({name, type: {type: 'struct', children}, nullable: field.optional});\n    } else {\n      const type = PARQUET_TYPE_MAPPING[field.type];\n      const metadata = getFieldMetadata(field);\n      const arrowField = {name, type, nullable: field.optional, metadata};\n      fields.push(arrowField);\n    }\n  }\n\n  return fields;\n}\n\nfunction getFieldMetadata(field: ParquetField): Record<string, string> | undefined {\n  let metadata: Record<string, string> | undefined;\n\n  for (const key in field) {\n    if (key !== 'name') {\n      let value = field[key] || '';\n      value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];\n      metadata = metadata || {};\n      metadata[key] = value;\n    }\n  }\n\n  return metadata;\n}\n\nfunction getSchemaMetadata(parquetMetadata: FileMetaData): Record<string, string> | undefined {\n  let metadata: Record<string, string> | undefined;\n\n  const keyValueList = parquetMetadata.key_value_metadata || [];\n  for (const {key, value} of keyValueList) {\n    if (typeof value === 'string') {\n      metadata = metadata || {};\n      metadata[key] = value;\n    }\n  }\n\n  return metadata;\n}\n"],"mappings":"AAQA,OAAO,MAAMA,oBAAuD,GAAG;EACrEC,OAAO,EAAE,MAAM;EACfC,KAAK,EAAE,OAAO;EACdC,KAAK,EAAE,SAAS;EAChBC,KAAK,EAAE,SAAS;EAChBC,KAAK,EAAE,SAAS;EAChBC,MAAM,EAAE,SAAS;EACjBC,UAAU,EAAE,QAAQ;EACpBC,oBAAoB,EAAE,QAAQ;EAC9BC,IAAI,EAAE,MAAM;EACZC,IAAI,EAAE,OAAO;EACbC,WAAW,EAAE,OAAO;EACpBC,WAAW,EAAE,OAAO;EACpBC,gBAAgB,EAAE,OAAO;EACzBC,gBAAgB,EAAE,OAAO;EACzBC,MAAM,EAAE,OAAO;EACfC,OAAO,EAAE,QAAQ;EACjBC,OAAO,EAAE,QAAQ;EACjBC,OAAO,EAAE,QAAQ;EACjBC,KAAK,EAAE,MAAM;EACbC,MAAM,EAAE,OAAO;EACfC,MAAM,EAAE,OAAO;EACfC,MAAM,EAAE,OAAO;EACfC,IAAI,EAAE,QAAQ;EACdC,IAAI,EAAE,QAAQ;EAEdC,QAAQ,EAAE,QAAQ;EAClBC,aAAa,EAAE,SAAS;EACxBC,aAAa,EAAE,SAAS;EACxBC,kBAAkB,EAAE,SAAS;EAC7BC,4BAA4B,EAAE;AAChC,CAAC;AAED,OAAO,SAASC,oBAAoBA,CAClCC,aAA4B,EAC5BC,eAAoC,EAC5B;EACR,MAAMC,MAAM,GAAGC,SAAS,CAACH,aAAa,CAACI,MAAM,CAAC;EAC9C,MAAMC,QAAQ,GAAGJ,eAAe,IAAIK,iBAAiB,CAACL,eAAe,CAAC;EAEtE,MAAMG,MAAc,GAAG;IACrBF,MAAM;IACNG,QAAQ,EAAEA,QAAQ,IAAI,CAAC;EACzB,CAAC;EAED,OAAOD,MAAM;AACf;AAEA,SAASD,SAASA,CAACC,MAAuB,EAAW;EACnD,MAAMF,MAAe,GAAG,EAAE;EAE1B,KAAK,MAAMK,IAAI,IAAIH,MAAM,EAAE;IACzB,MAAMI,KAAK,GAAGJ,MAAM,CAACG,IAAI,CAAC;IAE1B,IAAIC,KAAK,CAACN,MAAM,EAAE;MAChB,MAAMO,QAAQ,GAAGN,SAAS,CAACK,KAAK,CAACN,MAAM,CAAC;MACxCA,MAAM,CAACQ,IAAI,CAAC;QAACH,IAAI;QAAEI,IAAI,EAAE;UAACA,IAAI,EAAE,QAAQ;UAAEF;QAAQ,CAAC;QAAEG,QAAQ,EAAEJ,KAAK,CAACK;MAAQ,CAAC,CAAC;IACjF,CAAC,MAAM;MACL,MAAMF,IAAI,GAAG1C,oBAAoB,CAACuC,KAAK,CAACG,IAAI,CAAC;MAC7C,MAAMN,QAAQ,GAAGS,gBAAgB,CAACN,KAAK,CAAC;MACxC,MAAMO,UAAU,GAAG;QAACR,IAAI;QAAEI,IAAI;QAAEC,QAAQ,EAAEJ,KAAK,CAACK,QAAQ;QAAER;MAAQ,CAAC;MACnEH,MAAM,CAACQ,IAAI,CAACK,UAAU,CAAC;IACzB;EACF;EAEA,OAAOb,MAAM;AACf;AAEA,SAASY,gBAAgBA,CAACN,KAAmB,EAAsC;EACjF,IAAIH,QAA4C;EAEhD,KAAK,MAAMW,GAAG,IAAIR,KAAK,EAAE;IACvB,IAAIQ,GAAG,KAAK,MAAM,EAAE;MAClB,IAAIC,KAAK,GAAGT,KAAK,CAACQ,GAAG,CAAC,IAAI,EAAE;MAC5BC,KAAK,GAAG,OAAOT,KAAK,CAACQ,GAAG,CAAC,KAAK,QAAQ,GAAGxB,IAAI,CAAC0B,SAAS,CAACV,KAAK,CAACQ,GAAG,CAAC,CAAC,GAAGR,KAAK,CAACQ,GAAG,CAAC;MAChFX,QAAQ,GAAGA,QAAQ,IAAI,CAAC,CAAC;MACzBA,QAAQ,CAACW,GAAG,CAAC,GAAGC,KAAK;IACvB;EACF;EAEA,OAAOZ,QAAQ;AACjB;AAEA,SAASC,iBAAiBA,CAACL,eAA6B,EAAsC;EAC5F,IAAII,QAA4C;EAEhD,MAAMc,YAAY,GAAGlB,eAAe,CAACmB,kBAAkB,IAAI,EAAE;EAC7D,KAAK,MAAM;IAACJ,GAAG;IAAEC;EAAK,CAAC,IAAIE,YAAY,EAAE;IACvC,IAAI,OAAOF,KAAK,KAAK,QAAQ,EAAE;MAC7BZ,QAAQ,GAAGA,QAAQ,IAAI,CAAC,CAAC;MACzBA,QAAQ,CAACW,GAAG,CAAC,GAAGC,KAAK;IACvB;EACF;EAEA,OAAOZ,QAAQ;AACjB"}

package/dist/esm/lib/arrow/convert-schema-to-parquet.js

@@ -0,0 +1,39 @@
+export const PARQUET_TYPE_MAPPING = {
+  BOOLEAN: 'bool',
+  INT32: 'int32',
+  INT64: 'float64',
+  INT96: 'float64',
+  FLOAT: 'float32',
+  DOUBLE: 'float64',
+  BYTE_ARRAY: 'binary',
+  FIXED_LEN_BYTE_ARRAY: 'binary',
+  UTF8: 'utf8',
+  DATE: 'int32',
+  TIME_MILLIS: 'int64',
+  TIME_MICROS: 'int64',
+  TIMESTAMP_MILLIS: 'int64',
+  TIMESTAMP_MICROS: 'int64',
+  UINT_8: 'int32',
+  UINT_16: 'uint16',
+  UINT_32: 'uint32',
+  UINT_64: 'uint64',
+  INT_8: 'int8',
+  INT_16: 'int16',
+  INT_32: 'int32',
+  INT_64: 'int64',
+  JSON: 'binary',
+  BSON: 'binary',
+  INTERVAL: 'binary',
+  DECIMAL_INT32: 'float32',
+  DECIMAL_INT64: 'float64',
+  DECIMAL_BYTE_ARRAY: 'float64',
+  DECIMAL_FIXED_LEN_BYTE_ARRAY: 'float64'
+};
+export function convertToParquetSchema(schema) {
+  const fields = [];
+  return {
+    fields,
+    metadata: {}
+  };
+}
+//# sourceMappingURL=convert-schema-to-parquet.js.map

package/dist/esm/lib/arrow/convert-schema-to-parquet.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"convert-schema-to-parquet.js","names":["PARQUET_TYPE_MAPPING","BOOLEAN","INT32","INT64","INT96","FLOAT","DOUBLE","BYTE_ARRAY","FIXED_LEN_BYTE_ARRAY","UTF8","DATE","TIME_MILLIS","TIME_MICROS","TIMESTAMP_MILLIS","TIMESTAMP_MICROS","UINT_8","UINT_16","UINT_32","UINT_64","INT_8","INT_16","INT_32","INT_64","JSON","BSON","INTERVAL","DECIMAL_INT32","DECIMAL_INT64","DECIMAL_BYTE_ARRAY","DECIMAL_FIXED_LEN_BYTE_ARRAY","convertToParquetSchema","schema","fields","metadata"],"sources":["../../../../src/lib/arrow/convert-schema-to-parquet.ts"],"sourcesContent":["// loaders.gl, MIT license\n\n// import type {ParquetSchema} from '../../parquetjs/schema/schema';\nimport type {\n  // FieldDefinition, ParquetField,\n  ParquetType\n} from '../../parquetjs/schema/declare';\n\nimport {\n  Schema,\n  // Struct,\n  // Field,\n  DataType\n} from '@loaders.gl/schema';\n\nexport const PARQUET_TYPE_MAPPING: {[type in ParquetType]: DataType} = {\n  BOOLEAN: 'bool',\n  INT32: 'int32',\n  INT64: 'float64',\n  INT96: 'float64',\n  FLOAT: 'float32',\n  DOUBLE: 'float64',\n  BYTE_ARRAY: 'binary',\n  FIXED_LEN_BYTE_ARRAY: 'binary',\n  UTF8: 'utf8',\n  DATE: 'int32',\n  TIME_MILLIS: 'int64',\n  TIME_MICROS: 'int64',\n  TIMESTAMP_MILLIS: 'int64',\n  TIMESTAMP_MICROS: 'int64',\n  UINT_8: 'int32',\n  UINT_16: 'uint16',\n  UINT_32: 'uint32',\n  UINT_64: 'uint64',\n  INT_8: 'int8',\n  INT_16: 'int16',\n  INT_32: 'int32',\n  INT_64: 'int64',\n  JSON: 'binary',\n  BSON: 'binary',\n  // TODO check interval type\n  INTERVAL: 'binary',\n  DECIMAL_INT32: 'float32',\n  DECIMAL_INT64: 'float64',\n  DECIMAL_BYTE_ARRAY: 'float64',\n  DECIMAL_FIXED_LEN_BYTE_ARRAY: 'float64'\n};\n\nexport function convertToParquetSchema(schema: Schema): Schema {\n  const fields = []; // getFields(schema.fields);\n\n  // TODO add metadata if needed.\n  return {fields, metadata: {}};\n}\n\n// function getFields(schema: Field[]): Definition[] {\n//   const fields: Field[] = [];\n\n//   for (const name in schema) {\n//     const field = schema[name];\n\n//     // @ts-ignore\n//     const children = field.children as DataType[];\n//     if (children) {\n//       const childField = getFields(field.fields);\n//       const nestedField = new Field(name, new Struct(childField), field.optional);\n//       fields.push(nestedField);\n//     } else {\n//       const FieldType = PARQUET_TYPE_MAPPING[field.type];\n//       const metadata = getFieldMetadata(field);\n//       const arrowField = new Field(name, new FieldType(), field.optional, metadata);\n//       fields.push(arrowField);\n//     }\n//   }\n\n//   return fields;\n// }\n\n// function getFieldMetadata(field: ParquetField): Map<string, string> {\n//   const metadata = new Map();\n\n//   for (const key in field) {\n//     if (key !== 'name') {\n//       const value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];\n//       metadata.set(key, value);\n//     }\n//   }\n\n//   return metadata;\n// }\n"],"mappings":"AAeA,OAAO,MAAMA,oBAAuD,GAAG;EACrEC,OAAO,EAAE,MAAM;EACfC,KAAK,EAAE,OAAO;EACdC,KAAK,EAAE,SAAS;EAChBC,KAAK,EAAE,SAAS;EAChBC,KAAK,EAAE,SAAS;EAChBC,MAAM,EAAE,SAAS;EACjBC,UAAU,EAAE,QAAQ;EACpBC,oBAAoB,EAAE,QAAQ;EAC9BC,IAAI,EAAE,MAAM;EACZC,IAAI,EAAE,OAAO;EACbC,WAAW,EAAE,OAAO;EACpBC,WAAW,EAAE,OAAO;EACpBC,gBAAgB,EAAE,OAAO;EACzBC,gBAAgB,EAAE,OAAO;EACzBC,MAAM,EAAE,OAAO;EACfC,OAAO,EAAE,QAAQ;EACjBC,OAAO,EAAE,QAAQ;EACjBC,OAAO,EAAE,QAAQ;EACjBC,KAAK,EAAE,MAAM;EACbC,MAAM,EAAE,OAAO;EACfC,MAAM,EAAE,OAAO;EACfC,MAAM,EAAE,OAAO;EACfC,IAAI,EAAE,QAAQ;EACdC,IAAI,EAAE,QAAQ;EAEdC,QAAQ,EAAE,QAAQ;EAClBC,aAAa,EAAE,SAAS;EACxBC,aAAa,EAAE,SAAS;EACxBC,kBAAkB,EAAE,SAAS;EAC7BC,4BAA4B,EAAE;AAChC,CAAC;AAED,OAAO,SAASC,sBAAsBA,CAACC,MAAc,EAAU;EAC7D,MAAMC,MAAM,GAAG,EAAE;EAGjB,OAAO;IAACA,MAAM;IAAEC,QAAQ,EAAE,CAAC;EAAC,CAAC;AAC/B"}

package/dist/esm/lib/geo/decode-geo-metadata.js

@@ -0,0 +1,62 @@
+export function getGeoMetadata(schema) {
+  const stringifiedGeoMetadata = schema.metadata.geo;
+  if (!stringifiedGeoMetadata) {
+    return null;
+  }
+  try {
+    const geoMetadata = JSON.parse(stringifiedGeoMetadata);
+    return geoMetadata;
+  } catch {
+    return null;
+  }
+}
+export function setGeoMetadata(schema, geoMetadata) {
+  const stringifiedGeoMetadata = JSON.stringify(geoMetadata);
+  schema.metadata.geo = stringifiedGeoMetadata;
+}
+export function unpackGeoMetadata(schema) {
+  const geoMetadata = getGeoMetadata(schema);
+  if (!geoMetadata) {
+    return;
+  }
+  const {
+    version,
+    primary_column,
+    columns
+  } = geoMetadata;
+  if (version) {
+    schema.metadata['geo.version'] = version;
+  }
+  if (primary_column) {
+    schema.metadata['geo.primary_column'] = primary_column;
+  }
+  schema.metadata['geo.columns'] = Object.keys(columns || {}).join('');
+  for (const [columnName, columnMetadata] of Object.entries(columns || {})) {
+    const field = schema.fields.find(field => field.name === columnName);
+    if (field) {
+      if (field.name === primary_column) {
+        setFieldMetadata(field, 'geo.primary_field', 'true');
+      }
+      unpackGeoFieldMetadata(field, columnMetadata);
+    }
+  }
+}
+function unpackGeoFieldMetadata(field, columnMetadata) {
+  for (const [key, value] of Object.entries(columnMetadata || {})) {
+    switch (key) {
+      case 'geometry_type':
+        setFieldMetadata(field, "geo.".concat(key), value.join(','));
+        break;
+      case 'bbox':
+      case 'crs':
+      case 'edges':
+      default:
+        setFieldMetadata(field, "geo.".concat(key), typeof value === 'string' ? value : JSON.stringify(value));
+    }
+  }
+}
+function setFieldMetadata(field, key, value) {
+  field.metadata = field.metadata || {};
+  field.metadata[key] = value;
+}
+//# sourceMappingURL=decode-geo-metadata.js.map

package/dist/esm/lib/geo/decode-geo-metadata.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"decode-geo-metadata.js","names":["getGeoMetadata","schema","stringifiedGeoMetadata","metadata","geo","geoMetadata","JSON","parse","setGeoMetadata","stringify","unpackGeoMetadata","version","primary_column","columns","Object","keys","join","columnName","columnMetadata","entries","field","fields","find","name","setFieldMetadata","unpackGeoFieldMetadata","key","value","concat"],"sources":["../../../../src/lib/geo/decode-geo-metadata.ts"],"sourcesContent":["// loaders.gl, MIT license\nimport {Schema, Field} from '@loaders.gl/schema';\n\n/* eslint-disable camelcase */\n\n/** A geoarrow / geoparquet geo metadata object (stored in stringified form in the top level metadata 'geo' key) */\nexport type GeoMetadata = {\n  version?: string;\n  primary_column?: string;\n  columns: Record<string, GeoColumnMetadata>;\n  [key: string]: unknown;\n};\n\n/** A geoarrow / geoparquet geo metadata for one geometry column  */\nexport type GeoColumnMetadata = {\n  bounding_box?:\n    | [number, number, number, number]\n    | [number, number, number, number, number, number];\n  crs?: string;\n  geometry_type?: string[];\n  edges?: string;\n  [key: string]: unknown;\n};\n\n/**\n * Reads the GeoMetadata object from the metadata\n * @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata */\nexport function getGeoMetadata(schema: Schema): GeoMetadata | null {\n  const stringifiedGeoMetadata = schema.metadata.geo;\n  if (!stringifiedGeoMetadata) {\n    return null;\n  }\n\n  try {\n    const geoMetadata = JSON.parse(stringifiedGeoMetadata) as GeoMetadata;\n    return geoMetadata;\n  } catch {\n    return null;\n  }\n}\n\n/**\n * Stores a geoarrow / geoparquet geo metadata object in the schema\n * @note geoarrow / geoparquet geo metadata is a single stringified JSON field\n */\nexport function setGeoMetadata(schema: Schema, geoMetadata: GeoMetadata): void {\n  const stringifiedGeoMetadata = JSON.stringify(geoMetadata);\n  schema.metadata.geo = stringifiedGeoMetadata;\n}\n\n/**\n * Unpacks geo metadata into separate metadata fields (parses the long JSON string)\n * @note geoarrow / parquet schema is stringified into a single key-value pair in the parquet metadata\n */\nexport function unpackGeoMetadata(schema: Schema): void {\n  const geoMetadata = getGeoMetadata(schema);\n  if (!geoMetadata) {\n    return;\n  }\n\n  // Store Parquet Schema Level Metadata\n\n  const {version, primary_column, columns} = geoMetadata;\n  if (version) {\n    schema.metadata['geo.version'] = version;\n  }\n\n  if (primary_column) {\n    schema.metadata['geo.primary_column'] = primary_column;\n  }\n\n  // store column names as comma separated list\n  schema.metadata['geo.columns'] = Object.keys(columns || {}).join('');\n\n  for (const [columnName, columnMetadata] of Object.entries(columns || {})) {\n    const field = schema.fields.find((field) => field.name === columnName);\n    if (field) {\n      if (field.name === primary_column) {\n        setFieldMetadata(field, 'geo.primary_field', 'true');\n      }\n      unpackGeoFieldMetadata(field, columnMetadata);\n    }\n  }\n}\n\nfunction unpackGeoFieldMetadata(field: Field, columnMetadata): void {\n  for (const [key, value] of Object.entries(columnMetadata || {})) {\n    switch (key) {\n      case 'geometry_type':\n        setFieldMetadata(field, `geo.${key}`, (value as string[]).join(','));\n        break;\n      case 'bbox':\n      case 'crs':\n      case 'edges':\n      default:\n        setFieldMetadata(\n          field,\n          `geo.${key}`,\n          typeof value === 'string' ? value : JSON.stringify(value)\n        );\n    }\n  }\n}\n\nfunction setFieldMetadata(field: Field, key: string, value: string): void {\n  field.metadata = field.metadata || {};\n  field.metadata[key] = value;\n}\n"],"mappings":"AA2BA,OAAO,SAASA,cAAcA,CAACC,MAAc,EAAsB;EACjE,MAAMC,sBAAsB,GAAGD,MAAM,CAACE,QAAQ,CAACC,GAAG;EAClD,IAAI,CAACF,sBAAsB,EAAE;IAC3B,OAAO,IAAI;EACb;EAEA,IAAI;IACF,MAAMG,WAAW,GAAGC,IAAI,CAACC,KAAK,CAACL,sBAAsB,CAAgB;IACrE,OAAOG,WAAW;EACpB,CAAC,CAAC,MAAM;IACN,OAAO,IAAI;EACb;AACF;AAMA,OAAO,SAASG,cAAcA,CAACP,MAAc,EAAEI,WAAwB,EAAQ;EAC7E,MAAMH,sBAAsB,GAAGI,IAAI,CAACG,SAAS,CAACJ,WAAW,CAAC;EAC1DJ,MAAM,CAACE,QAAQ,CAACC,GAAG,GAAGF,sBAAsB;AAC9C;AAMA,OAAO,SAASQ,iBAAiBA,CAACT,MAAc,EAAQ;EACtD,MAAMI,WAAW,GAAGL,cAAc,CAACC,MAAM,CAAC;EAC1C,IAAI,CAACI,WAAW,EAAE;IAChB;EACF;EAIA,MAAM;IAACM,OAAO;IAAEC,cAAc;IAAEC;EAAO,CAAC,GAAGR,WAAW;EACtD,IAAIM,OAAO,EAAE;IACXV,MAAM,CAACE,QAAQ,CAAC,aAAa,CAAC,GAAGQ,OAAO;EAC1C;EAEA,IAAIC,cAAc,EAAE;IAClBX,MAAM,CAACE,QAAQ,CAAC,oBAAoB,CAAC,GAAGS,cAAc;EACxD;EAGAX,MAAM,CAACE,QAAQ,CAAC,aAAa,CAAC,GAAGW,MAAM,CAACC,IAAI,CAACF,OAAO,IAAI,CAAC,CAAC,CAAC,CAACG,IAAI,CAAC,EAAE,CAAC;EAEpE,KAAK,MAAM,CAACC,UAAU,EAAEC,cAAc,CAAC,IAAIJ,MAAM,CAACK,OAAO,CAACN,OAAO,IAAI,CAAC,CAAC,CAAC,EAAE;IACxE,MAAMO,KAAK,GAAGnB,MAAM,CAACoB,MAAM,CAACC,IAAI,CAAEF,KAAK,IAAKA,KAAK,CAACG,IAAI,KAAKN,UAAU,CAAC;IACtE,IAAIG,KAAK,EAAE;MACT,IAAIA,KAAK,CAACG,IAAI,KAAKX,cAAc,EAAE;QACjCY,gBAAgB,CAACJ,KAAK,EAAE,mBAAmB,EAAE,MAAM,CAAC;MACtD;MACAK,sBAAsB,CAACL,KAAK,EAAEF,cAAc,CAAC;IAC/C;EACF;AACF;AAEA,SAASO,sBAAsBA,CAACL,KAAY,EAAEF,cAAc,EAAQ;EAClE,KAAK,MAAM,CAACQ,GAAG,EAAEC,KAAK,CAAC,IAAIb,MAAM,CAACK,OAAO,CAACD,cAAc,IAAI,CAAC,CAAC,CAAC,EAAE;IAC/D,QAAQQ,GAAG;MACT,KAAK,eAAe;QAClBF,gBAAgB,CAACJ,KAAK,SAAAQ,MAAA,CAASF,GAAG,GAAKC,KAAK,CAAcX,IAAI,CAAC,GAAG,CAAC,CAAC;QACpE;MACF,KAAK,MAAM;MACX,KAAK,KAAK;MACV,KAAK,OAAO;MACZ;QACEQ,gBAAgB,CACdJ,KAAK,SAAAQ,MAAA,CACEF,GAAG,GACV,OAAOC,KAAK,KAAK,QAAQ,GAAGA,KAAK,GAAGrB,IAAI,CAACG,SAAS,CAACkB,KAAK,CAC1D,CAAC;IACL;EACF;AACF;AAEA,SAASH,gBAAgBA,CAACJ,KAAY,EAAEM,GAAW,EAAEC,KAAa,EAAQ;EACxEP,KAAK,CAACjB,QAAQ,GAAGiB,KAAK,CAACjB,QAAQ,IAAI,CAAC,CAAC;EACrCiB,KAAK,CAACjB,QAAQ,CAACuB,GAAG,CAAC,GAAGC,KAAK;AAC7B"}

package/dist/esm/lib/geo/geoparquet-schema.js

@@ -0,0 +1,76 @@
+export default {
+  $schema: 'http://json-schema.org/draft-07/schema#',
+  title: 'GeoParquet',
+  description: 'Parquet metadata included in the geo field.',
+  type: 'object',
+  required: ['version', 'primary_column', 'columns'],
+  properties: {
+    version: {
+      type: 'string',
+      const: '1.0.0-beta.1'
+    },
+    primary_column: {
+      type: 'string',
+      minLength: 1
+    },
+    columns: {
+      type: 'object',
+      minProperties: 1,
+      patternProperties: {
+        '.+': {
+          type: 'object',
+          required: ['encoding', 'geometry_types'],
+          properties: {
+            encoding: {
+              type: 'string',
+              const: 'WKB'
+            },
+            geometry_types: {
+              type: 'array',
+              uniqueItems: true,
+              items: {
+                type: 'string',
+                pattern: '^(GeometryCollection|(Multi)?(Point|LineString|Polygon))( Z)?$'
+              }
+            },
+            crs: {
+              oneOf: [{
+                $ref: 'https://proj.org/schemas/v0.5/projjson.schema.json'
+              }, {
+                type: 'null'
+              }]
+            },
+            edges: {
+              type: 'string',
+              enum: ['planar', 'spherical']
+            },
+            orientation: {
+              type: 'string',
+              const: 'counterclockwise'
+            },
+            bbox: {
+              type: 'array',
+              items: {
+                type: 'number'
+              },
+              oneOf: [{
+                description: '2D bbox consisting of (xmin, ymin, xmax, ymax)',
+                minItems: 4,
+                maxItems: 4
+              }, {
+                description: '3D bbox consisting of (xmin, ymin, zmin, xmax, ymax, zmax)',
+                minItems: 6,
+                maxItems: 6
+              }]
+            },
+            epoch: {
+              type: 'number'
+            }
+          }
+        }
+      },
+      additionalProperties: false
+    }
+  }
+};
+//# sourceMappingURL=geoparquet-schema.js.map

package/dist/esm/lib/geo/geoparquet-schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"geoparquet-schema.js","names":["$schema","title","description","type","required","properties","version","const","primary_column","minLength","columns","minProperties","patternProperties","encoding","geometry_types","uniqueItems","items","pattern","crs","oneOf","$ref","edges","enum","orientation","bbox","minItems","maxItems","epoch","additionalProperties"],"sources":["../../../../src/lib/geo/geoparquet-schema.ts"],"sourcesContent":["// loaders.gl, MIT license\n\n/* eslint-disable camelcase */\n\n/**\n * Geoparquet JSON schema for geo metadata\n * @see https://github.com/geoarrow/geoarrow/blob/main/metadata.md\n * @see https://github.com/opengeospatial/geoparquet/blob/main/format-specs/geoparquet.md\n */\nexport default {\n  $schema: 'http://json-schema.org/draft-07/schema#',\n  title: 'GeoParquet',\n  description: 'Parquet metadata included in the geo field.',\n  type: 'object',\n  required: ['version', 'primary_column', 'columns'],\n  properties: {\n    version: {type: 'string', const: '1.0.0-beta.1'},\n    primary_column: {type: 'string', minLength: 1},\n    columns: {\n      type: 'object',\n      minProperties: 1,\n      patternProperties: {\n        '.+': {\n          type: 'object',\n          required: ['encoding', 'geometry_types'],\n          properties: {\n            encoding: {type: 'string', const: 'WKB'},\n            geometry_types: {\n              type: 'array',\n              uniqueItems: true,\n              items: {\n                type: 'string',\n                pattern: '^(GeometryCollection|(Multi)?(Point|LineString|Polygon))( Z)?$'\n              }\n            },\n            crs: {\n              oneOf: [\n                {\n                  $ref: 'https://proj.org/schemas/v0.5/projjson.schema.json'\n                },\n                {type: 'null'}\n              ]\n            },\n            edges: {type: 'string', enum: ['planar', 'spherical']},\n            orientation: {type: 'string', const: 'counterclockwise'},\n            bbox: {\n              type: 'array',\n              items: {type: 'number'},\n              oneOf: [\n                {\n                  description: '2D bbox consisting of (xmin, ymin, xmax, ymax)',\n                  minItems: 4,\n                  maxItems: 4\n                },\n                {\n                  description: '3D bbox consisting of (xmin, ymin, zmin, xmax, ymax, zmax)',\n                  minItems: 6,\n                  maxItems: 6\n                }\n              ]\n            },\n            epoch: {type: 'number'}\n          }\n        }\n      },\n      additionalProperties: false\n    }\n  }\n};\n"],"mappings":"AASA,eAAe;EACbA,OAAO,EAAE,yCAAyC;EAClDC,KAAK,EAAE,YAAY;EACnBC,WAAW,EAAE,6CAA6C;EAC1DC,IAAI,EAAE,QAAQ;EACdC,QAAQ,EAAE,CAAC,SAAS,EAAE,gBAAgB,EAAE,SAAS,CAAC;EAClDC,UAAU,EAAE;IACVC,OAAO,EAAE;MAACH,IAAI,EAAE,QAAQ;MAAEI,KAAK,EAAE;IAAc,CAAC;IAChDC,cAAc,EAAE;MAACL,IAAI,EAAE,QAAQ;MAAEM,SAAS,EAAE;IAAC,CAAC;IAC9CC,OAAO,EAAE;MACPP,IAAI,EAAE,QAAQ;MACdQ,aAAa,EAAE,CAAC;MAChBC,iBAAiB,EAAE;QACjB,IAAI,EAAE;UACJT,IAAI,EAAE,QAAQ;UACdC,QAAQ,EAAE,CAAC,UAAU,EAAE,gBAAgB,CAAC;UACxCC,UAAU,EAAE;YACVQ,QAAQ,EAAE;cAACV,IAAI,EAAE,QAAQ;cAAEI,KAAK,EAAE;YAAK,CAAC;YACxCO,cAAc,EAAE;cACdX,IAAI,EAAE,OAAO;cACbY,WAAW,EAAE,IAAI;cACjBC,KAAK,EAAE;gBACLb,IAAI,EAAE,QAAQ;gBACdc,OAAO,EAAE;cACX;YACF,CAAC;YACDC,GAAG,EAAE;cACHC,KAAK,EAAE,CACL;gBACEC,IAAI,EAAE;cACR,CAAC,EACD;gBAACjB,IAAI,EAAE;cAAM,CAAC;YAElB,CAAC;YACDkB,KAAK,EAAE;cAAClB,IAAI,EAAE,QAAQ;cAAEmB,IAAI,EAAE,CAAC,QAAQ,EAAE,WAAW;YAAC,CAAC;YACtDC,WAAW,EAAE;cAACpB,IAAI,EAAE,QAAQ;cAAEI,KAAK,EAAE;YAAkB,CAAC;YACxDiB,IAAI,EAAE;cACJrB,IAAI,EAAE,OAAO;cACba,KAAK,EAAE;gBAACb,IAAI,EAAE;cAAQ,CAAC;cACvBgB,KAAK,EAAE,CACL;gBACEjB,WAAW,EAAE,gDAAgD;gBAC7DuB,QAAQ,EAAE,CAAC;gBACXC,QAAQ,EAAE;cACZ,CAAC,EACD;gBACExB,WAAW,EAAE,4DAA4D;gBACzEuB,QAAQ,EAAE,CAAC;gBACXC,QAAQ,EAAE;cACZ,CAAC;YAEL,CAAC;YACDC,KAAK,EAAE;cAACxB,IAAI,EAAE;YAAQ;UACxB;QACF;MACF,CAAC;MACDyB,oBAAoB,EAAE;IACxB;EACF;AACF,CAAC"}

package/dist/esm/lib/parsers/parse-parquet-to-columns.js

@@ -0,0 +1,39 @@
+import { makeReadableFile } from '@loaders.gl/loader-utils';
+import { ParquetReader } from '../../parquetjs/parser/parquet-reader';
+import { convertParquetSchema } from '../arrow/convert-schema-from-parquet';
+import { materializeColumns } from '../../parquetjs/schema/shred';
+import { unpackGeoMetadata } from '../geo/decode-geo-metadata';
+export async function parseParquetInColumns(arrayBuffer, options) {
+  const blob = new Blob([arrayBuffer]);
+  for await (const batch of parseParquetFileInColumnarBatches(blob, options)) {
+    return {
+      shape: 'columnar-table',
+      schema: batch.schema,
+      data: batch.data
+    };
+  }
+  throw new Error('empty table');
+}
+export async function* parseParquetFileInColumnarBatches(blob, options) {
+  const file = makeReadableFile(blob);
+  const reader = new ParquetReader(file);
+  const parquetSchema = await reader.getSchema();
+  const parquetMetadata = await reader.getFileMetadata();
+  const schema = convertParquetSchema(parquetSchema, parquetMetadata);
+  unpackGeoMetadata(schema);
+  const rowGroups = reader.rowGroupIterator(options === null || options === void 0 ? void 0 : options.parquet);
+  for await (const rowGroup of rowGroups) {
+    yield convertRowGroupToTableBatch(parquetSchema, rowGroup, schema);
+  }
+}
+function convertRowGroupToTableBatch(parquetSchema, rowGroup, schema) {
+  const data = materializeColumns(parquetSchema, rowGroup);
+  return {
+    shape: 'columnar-table',
+    batchType: 'data',
+    schema,
+    data,
+    length: rowGroup.rowCount
+  };
+}
+//# sourceMappingURL=parse-parquet-to-columns.js.map

package/dist/esm/lib/parsers/parse-parquet-to-columns.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-parquet-to-columns.js","names":["makeReadableFile","ParquetReader","convertParquetSchema","materializeColumns","unpackGeoMetadata","parseParquetInColumns","arrayBuffer","options","blob","Blob","batch","parseParquetFileInColumnarBatches","shape","schema","data","Error","file","reader","parquetSchema","getSchema","parquetMetadata","getFileMetadata","rowGroups","rowGroupIterator","parquet","rowGroup","convertRowGroupToTableBatch","batchType","length","rowCount"],"sources":["../../../../src/lib/parsers/parse-parquet-to-columns.ts"],"sourcesContent":["// loaders.gl, MIT license\n\n// import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';\nimport {ColumnarTable, ColumnarTableBatch, Schema} from '@loaders.gl/schema';\nimport {makeReadableFile} from '@loaders.gl/loader-utils';\nimport type {ParquetLoaderOptions} from '../../parquet-loader';\nimport {ParquetReader} from '../../parquetjs/parser/parquet-reader';\nimport {ParquetRowGroup} from '../../parquetjs/schema/declare';\nimport {ParquetSchema} from '../../parquetjs/schema/schema';\nimport {convertParquetSchema} from '../arrow/convert-schema-from-parquet';\nimport {materializeColumns} from '../../parquetjs/schema/shred';\n// import {convertParquetRowGroupToColumns} from '../arrow/convert-row-group-to-columns';\nimport {unpackGeoMetadata} from '../geo/decode-geo-metadata';\n\nexport async function parseParquetInColumns(\n  arrayBuffer: ArrayBuffer,\n  options?: ParquetLoaderOptions\n): Promise<ColumnarTable> {\n  const blob = new Blob([arrayBuffer]);\n  for await (const batch of parseParquetFileInColumnarBatches(blob, options)) {\n    return {\n      shape: 'columnar-table',\n      schema: batch.schema,\n      data: batch.data\n    };\n  }\n  throw new Error('empty table');\n}\n\nexport async function* parseParquetFileInColumnarBatches(\n  blob: Blob,\n  options?: ParquetLoaderOptions\n): AsyncIterable<ColumnarTableBatch> {\n  const file = makeReadableFile(blob);\n  const reader = new ParquetReader(file);\n  const parquetSchema = await reader.getSchema();\n  const parquetMetadata = await reader.getFileMetadata();\n  const schema = convertParquetSchema(parquetSchema, parquetMetadata);\n  unpackGeoMetadata(schema);\n  const rowGroups = reader.rowGroupIterator(options?.parquet);\n  for await (const rowGroup of rowGroups) {\n    yield convertRowGroupToTableBatch(parquetSchema, rowGroup, schema);\n  }\n}\n\nfunction convertRowGroupToTableBatch(\n  parquetSchema: ParquetSchema,\n  rowGroup: ParquetRowGroup,\n  schema: Schema\n): ColumnarTableBatch {\n  // const data = convertParquetRowGroupToColumns(schema, rowGroup);\n  const data = materializeColumns(parquetSchema, rowGroup);\n  return {\n    shape: 'columnar-table',\n    batchType: 'data',\n    schema,\n    data,\n    length: rowGroup.rowCount\n  };\n}\n"],"mappings":"AAIA,SAAQA,gBAAgB,QAAO,0BAA0B;AAEzD,SAAQC,aAAa,QAAO,uCAAuC;AAGnE,SAAQC,oBAAoB,QAAO,sCAAsC;AACzE,SAAQC,kBAAkB,QAAO,8BAA8B;AAE/D,SAAQC,iBAAiB,QAAO,4BAA4B;AAE5D,OAAO,eAAeC,qBAAqBA,CACzCC,WAAwB,EACxBC,OAA8B,EACN;EACxB,MAAMC,IAAI,GAAG,IAAIC,IAAI,CAAC,CAACH,WAAW,CAAC,CAAC;EACpC,WAAW,MAAMI,KAAK,IAAIC,iCAAiC,CAACH,IAAI,EAAED,OAAO,CAAC,EAAE;IAC1E,OAAO;MACLK,KAAK,EAAE,gBAAgB;MACvBC,MAAM,EAAEH,KAAK,CAACG,MAAM;MACpBC,IAAI,EAAEJ,KAAK,CAACI;IACd,CAAC;EACH;EACA,MAAM,IAAIC,KAAK,CAAC,aAAa,CAAC;AAChC;AAEA,OAAO,gBAAgBJ,iCAAiCA,CACtDH,IAAU,EACVD,OAA8B,EACK;EACnC,MAAMS,IAAI,GAAGhB,gBAAgB,CAACQ,IAAI,CAAC;EACnC,MAAMS,MAAM,GAAG,IAAIhB,aAAa,CAACe,IAAI,CAAC;EACtC,MAAME,aAAa,GAAG,MAAMD,MAAM,CAACE,SAAS,CAAC,CAAC;EAC9C,MAAMC,eAAe,GAAG,MAAMH,MAAM,CAACI,eAAe,CAAC,CAAC;EACtD,MAAMR,MAAM,GAAGX,oBAAoB,CAACgB,aAAa,EAAEE,eAAe,CAAC;EACnEhB,iBAAiB,CAACS,MAAM,CAAC;EACzB,MAAMS,SAAS,GAAGL,MAAM,CAACM,gBAAgB,CAAChB,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAEiB,OAAO,CAAC;EAC3D,WAAW,MAAMC,QAAQ,IAAIH,SAAS,EAAE;IACtC,MAAMI,2BAA2B,CAACR,aAAa,EAAEO,QAAQ,EAAEZ,MAAM,CAAC;EACpE;AACF;AAEA,SAASa,2BAA2BA,CAClCR,aAA4B,EAC5BO,QAAyB,EACzBZ,MAAc,EACM;EAEpB,MAAMC,IAAI,GAAGX,kBAAkB,CAACe,aAAa,EAAEO,QAAQ,CAAC;EACxD,OAAO;IACLb,KAAK,EAAE,gBAAgB;IACvBe,SAAS,EAAE,MAAM;IACjBd,MAAM;IACNC,IAAI;IACJc,MAAM,EAAEH,QAAQ,CAACI;EACnB,CAAC;AACH"}