@loaders.gl/parquet 3.0.12 → 3.1.0-alpha.4

package/dist/esm/bundle.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/bundle.ts"],"names":["moduleExports","require","_global","window","global","loaders","module","exports","Object","assign"],"mappings":"AACA,MAAMA,aAAa,GAAGC,OAAO,CAAC,SAAD,CAA7B;;AACA,MAAMC,OAAO,GAAG,OAAOC,MAAP,KAAkB,WAAlB,GAAgCC,MAAhC,GAAyCD,MAAzD;;AACAD,OAAO,CAACG,OAAR,GAAkBH,OAAO,CAACG,OAAR,IAAmB,EAArC;AACAC,MAAM,CAACC,OAAP,GAAiBC,MAAM,CAACC,MAAP,CAAcP,OAAO,CAACG,OAAtB,EAA+BL,aAA/B,CAAjB","sourcesContent":["// @ts-nocheck\nconst moduleExports = require('./index');\nconst _global = typeof window === 'undefined' ? global : window;\n_global.loaders = _global.loaders || {};\nmodule.exports = Object.assign(_global.loaders, moduleExports);\n"],"file":"bundle.js"}
+{"version":3,"sources":["../../src/bundle.ts"],"names":["moduleExports","require","globalThis","loaders","module","exports","Object","assign"],"mappings":"AACA,MAAMA,aAAa,GAAGC,OAAO,CAAC,SAAD,CAA7B;;AACAC,UAAU,CAACC,OAAX,GAAqBD,UAAU,CAACC,OAAX,IAAsB,EAA3C;AACAC,MAAM,CAACC,OAAP,GAAiBC,MAAM,CAACC,MAAP,CAAcL,UAAU,CAACC,OAAzB,EAAkCH,aAAlC,CAAjB","sourcesContent":["// @ts-nocheck\nconst moduleExports = require('./index');\nglobalThis.loaders = globalThis.loaders || {};\nmodule.exports = Object.assign(globalThis.loaders, moduleExports);\n"],"file":"bundle.js"}

package/dist/esm/constants.js

@@ -0,0 +1,6 @@
+export const PARQUET_MAGIC = 'PAR1';
+export const PARQUET_MAGIC_ENCRYPTED = 'PARE';
+export const PARQUET_VERSION = 1;
+export const PARQUET_RDLVL_TYPE = 'INT32';
+export const PARQUET_RDLVL_ENCODING = 'RLE';
+//# sourceMappingURL=constants.js.map

package/dist/esm/constants.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/constants.ts"],"names":["PARQUET_MAGIC","PARQUET_MAGIC_ENCRYPTED","PARQUET_VERSION","PARQUET_RDLVL_TYPE","PARQUET_RDLVL_ENCODING"],"mappings":"AAIA,OAAO,MAAMA,aAAa,GAAG,MAAtB;AACP,OAAO,MAAMC,uBAAuB,GAAG,MAAhC;AAKP,OAAO,MAAMC,eAAe,GAAG,CAAxB;AAKP,OAAO,MAAMC,kBAAkB,GAAG,OAA3B;AACP,OAAO,MAAMC,sBAAsB,GAAG,KAA/B","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/**\n * Parquet File Magic String\n */\nexport const PARQUET_MAGIC = 'PAR1';\nexport const PARQUET_MAGIC_ENCRYPTED = 'PARE';\n\n/**\n * Parquet File Format Version\n */\nexport const PARQUET_VERSION = 1;\n\n/**\n * Internal type used for repetition/definition levels\n */\nexport const PARQUET_RDLVL_TYPE = 'INT32';\nexport const PARQUET_RDLVL_ENCODING = 'RLE';\n"],"file":"constants.js"}

package/dist/esm/index.js

@@ -1,6 +1,16 @@
-export { ParquetReader, ParquetEnvelopeReader } from './parquetjs/reader';
-export { ParquetWriter, ParquetEnvelopeWriter, ParquetTransformer } from './parquetjs/writer';
-export { ParquetSchema } from './parquetjs/schema/schema';
-export { ParquetLoader, ParquetWorkerLoader } from './parquet-loader';
+import { ParquetLoader as ParquetWorkerLoader } from './parquet-loader';
+import { parseParquet, parseParquetFileInBatches } from './lib/parse-parquet';
+export { ParquetWorkerLoader };
+export const ParquetLoader = { ...ParquetWorkerLoader,
+  parse: parseParquet,
+  parseFileInBatches: parseParquetFileInBatches
+};
 export { ParquetWriter as _ParquetWriter } from './parquet-writer';
+export { preloadCompressions } from './parquetjs/compression';
+export { ParquetEnvelopeReader } from './parquetjs/parser/parquet-envelope-reader';
+export { ParquetReader } from './parquetjs/parser/parquet-reader';
+export { ParquetWriter, ParquetEnvelopeWriter, ParquetTransformer } from './parquetjs/encoder/writer';
+export { ParquetSchema } from './parquetjs/schema/schema';
+export { convertParquetToArrowSchema } from './lib/convert-schema';
+export const _typecheckParquetLoader = ParquetLoader;
 //# sourceMappingURL=index.js.map

package/dist/esm/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/index.ts"],"names":["ParquetReader","ParquetEnvelopeReader","ParquetWriter","ParquetEnvelopeWriter","ParquetTransformer","ParquetSchema","ParquetLoader","ParquetWorkerLoader","_ParquetWriter"],"mappings":"AAAA,SAAQA,aAAR,EAAuBC,qBAAvB,QAAmD,oBAAnD;AACA,SAAQC,aAAR,EAAuBC,qBAAvB,EAA8CC,kBAA9C,QAAuE,oBAAvE;AACA,SAAQC,aAAR,QAA4B,2BAA5B;AAEA,SAAQC,aAAR,EAAuBC,mBAAvB,QAAiD,kBAAjD;AACA,SAAQL,aAAa,IAAIM,cAAzB,QAA8C,kBAA9C","sourcesContent":["export {ParquetReader, ParquetEnvelopeReader} from './parquetjs/reader';\nexport {ParquetWriter, ParquetEnvelopeWriter, ParquetTransformer} from './parquetjs/writer';\nexport {ParquetSchema} from './parquetjs/schema/schema';\n\nexport {ParquetLoader, ParquetWorkerLoader} from './parquet-loader';\nexport {ParquetWriter as _ParquetWriter} from './parquet-writer';\n"],"file":"index.js"}
+{"version":3,"sources":["../../src/index.ts"],"names":["ParquetLoader","ParquetWorkerLoader","parseParquet","parseParquetFileInBatches","parse","parseFileInBatches","ParquetWriter","_ParquetWriter","preloadCompressions","ParquetEnvelopeReader","ParquetReader","ParquetEnvelopeWriter","ParquetTransformer","ParquetSchema","convertParquetToArrowSchema","_typecheckParquetLoader"],"mappings":"AAIA,SAAQA,aAAa,IAAIC,mBAAzB,QAAmD,kBAAnD;AACA,SAAQC,YAAR,EAAsBC,yBAAtB,QAAsD,qBAAtD;AAEA,SAAQF,mBAAR;AAGA,OAAO,MAAMD,aAAa,GAAG,EAC3B,GAAGC,mBADwB;AAE3BG,EAAAA,KAAK,EAAEF,YAFoB;AAG3BG,EAAAA,kBAAkB,EAAEF;AAHO,CAAtB;AAQP,SAAQG,aAAa,IAAIC,cAAzB,QAA8C,kBAA9C;AAIA,SAAQC,mBAAR,QAAkC,yBAAlC;AAEA,SAAQC,qBAAR,QAAoC,4CAApC;AACA,SAAQC,aAAR,QAA4B,mCAA5B;AACA,SAAQJ,aAAR,EAAuBK,qBAAvB,EAA8CC,kBAA9C,QAAuE,4BAAvE;AACA,SAAQC,aAAR,QAA4B,2BAA5B;AACA,SAAQC,2BAAR,QAA0C,sBAA1C;AAGA,OAAO,MAAMC,uBAAyC,GAAGf,aAAlD","sourcesContent":["import type {LoaderWithParser} from '@loaders.gl/loader-utils';\n\n// ParquetLoader\n\nimport {ParquetLoader as ParquetWorkerLoader} from './parquet-loader';\nimport {parseParquet, parseParquetFileInBatches} from './lib/parse-parquet';\n\nexport {ParquetWorkerLoader};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n  ...ParquetWorkerLoader,\n  parse: parseParquet,\n  parseFileInBatches: parseParquetFileInBatches\n};\n\n// ParquetWriter\n\nexport {ParquetWriter as _ParquetWriter} from './parquet-writer';\n\n// EXPERIMENTAL - expose the internal parquetjs API\n\nexport {preloadCompressions} from './parquetjs/compression';\n\nexport {ParquetEnvelopeReader} from './parquetjs/parser/parquet-envelope-reader';\nexport {ParquetReader} from './parquetjs/parser/parquet-reader';\nexport {ParquetWriter, ParquetEnvelopeWriter, ParquetTransformer} from './parquetjs/encoder/writer';\nexport {ParquetSchema} from './parquetjs/schema/schema';\nexport {convertParquetToArrowSchema} from './lib/convert-schema';\n\n// TESTS\nexport const _typecheckParquetLoader: LoaderWithParser = ParquetLoader;\n"],"file":"index.js"}

package/dist/esm/lib/convert-schema.js

@@ -0,0 +1,71 @@
+import { Schema, Struct, Field, Bool, Float64, Int32, Float32, Binary, Utf8, Int64, Uint16, Uint32, Uint64, Int8, Int16 } from '@loaders.gl/schema';
+export const PARQUET_TYPE_MAPPING = {
+  BOOLEAN: Bool,
+  INT32: Int32,
+  INT64: Float64,
+  INT96: Float64,
+  FLOAT: Float32,
+  DOUBLE: Float64,
+  BYTE_ARRAY: Binary,
+  FIXED_LEN_BYTE_ARRAY: Binary,
+  UTF8: Utf8,
+  DATE: Int32,
+  TIME_MILLIS: Int64,
+  TIME_MICROS: Int64,
+  TIMESTAMP_MILLIS: Int64,
+  TIMESTAMP_MICROS: Int64,
+  UINT_8: Int32,
+  UINT_16: Uint16,
+  UINT_32: Uint32,
+  UINT_64: Uint64,
+  INT_8: Int8,
+  INT_16: Int16,
+  INT_32: Int32,
+  INT_64: Int64,
+  JSON: Binary,
+  BSON: Binary,
+  INTERVAL: Binary,
+  DECIMAL_INT32: Float32,
+  DECIMAL_INT64: Float64,
+  DECIMAL_BYTE_ARRAY: Float64,
+  DECIMAL_FIXED_LEN_BYTE_ARRAY: Float64
+};
+export function convertParquetToArrowSchema(parquetSchema) {
+  const fields = getFields(parquetSchema.schema);
+  return new Schema(fields);
+}
+
+function getFieldMetadata(field) {
+  const metadata = new Map();
+
+  for (const key in field) {
+    if (key !== 'name') {
+      const value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
+      metadata.set(key, value);
+    }
+  }
+
+  return metadata;
+}
+
+function getFields(schema) {
+  const fields = [];
+
+  for (const name in schema) {
+    const field = schema[name];
+
+    if (field.fields) {
+      const childField = getFields(field.fields);
+      const nestedField = new Field(name, new Struct(childField), field.optional);
+      fields.push(nestedField);
+    } else {
+      const FieldType = PARQUET_TYPE_MAPPING[field.type];
+      const metadata = getFieldMetadata(field);
+      const arrowField = new Field(name, new FieldType(), field.optional, metadata);
+      fields.push(arrowField);
+    }
+  }
+
+  return fields;
+}
+//# sourceMappingURL=convert-schema.js.map

package/dist/esm/lib/convert-schema.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/lib/convert-schema.ts"],"names":["Schema","Struct","Field","Bool","Float64","Int32","Float32","Binary","Utf8","Int64","Uint16","Uint32","Uint64","Int8","Int16","PARQUET_TYPE_MAPPING","BOOLEAN","INT32","INT64","INT96","FLOAT","DOUBLE","BYTE_ARRAY","FIXED_LEN_BYTE_ARRAY","UTF8","DATE","TIME_MILLIS","TIME_MICROS","TIMESTAMP_MILLIS","TIMESTAMP_MICROS","UINT_8","UINT_16","UINT_32","UINT_64","INT_8","INT_16","INT_32","INT_64","JSON","BSON","INTERVAL","DECIMAL_INT32","DECIMAL_INT64","DECIMAL_BYTE_ARRAY","DECIMAL_FIXED_LEN_BYTE_ARRAY","convertParquetToArrowSchema","parquetSchema","fields","getFields","schema","getFieldMetadata","field","metadata","Map","key","value","stringify","set","name","childField","nestedField","optional","push","FieldType","type","arrowField"],"mappings":"AAGA,SACEA,MADF,EAEEC,MAFF,EAGEC,KAHF,EAKEC,IALF,EAMEC,OANF,EAOEC,KAPF,EAQEC,OARF,EASEC,MATF,EAUEC,IAVF,EAWEC,KAXF,EAYEC,MAZF,EAaEC,MAbF,EAcEC,MAdF,EAeEC,IAfF,EAgBEC,KAhBF,QAiBO,oBAjBP;AAmBA,OAAO,MAAMC,oBAA8D,GAAG;AAC5EC,EAAAA,OAAO,EAAEb,IADmE;AAE5Ec,EAAAA,KAAK,EAAEZ,KAFqE;AAG5Ea,EAAAA,KAAK,EAAEd,OAHqE;AAI5Ee,EAAAA,KAAK,EAAEf,OAJqE;AAK5EgB,EAAAA,KAAK,EAAEd,OALqE;AAM5Ee,EAAAA,MAAM,EAAEjB,OANoE;AAO5EkB,EAAAA,UAAU,EAAEf,MAPgE;AAQ5EgB,EAAAA,oBAAoB,EAAEhB,MARsD;AAS5EiB,EAAAA,IAAI,EAAEhB,IATsE;AAU5EiB,EAAAA,IAAI,EAAEpB,KAVsE;AAW5EqB,EAAAA,WAAW,EAAEjB,KAX+D;AAY5EkB,EAAAA,WAAW,EAAElB,KAZ+D;AAa5EmB,EAAAA,gBAAgB,EAAEnB,KAb0D;AAc5EoB,EAAAA,gBAAgB,EAAEpB,KAd0D;AAe5EqB,EAAAA,MAAM,EAAEzB,KAfoE;AAgB5E0B,EAAAA,OAAO,EAAErB,MAhBmE;AAiB5EsB,EAAAA,OAAO,EAAErB,MAjBmE;AAkB5EsB,EAAAA,OAAO,EAAErB,MAlBmE;AAmB5EsB,EAAAA,KAAK,EAAErB,IAnBqE;AAoB5EsB,EAAAA,MAAM,EAAErB,KApBoE;AAqB5EsB,EAAAA,MAAM,EAAE/B,KArBoE;AAsB5EgC,EAAAA,MAAM,EAAE5B,KAtBoE;AAuB5E6B,EAAAA,IAAI,EAAE/B,MAvBsE;AAwB5EgC,EAAAA,IAAI,EAAEhC,MAxBsE;AA0B5EiC,EAAAA,QAAQ,EAAEjC,MA1BkE;AA2B5EkC,EAAAA,aAAa,EAAEnC,OA3B6D;AA4B5EoC,EAAAA,aAAa,EAAEtC,OA5B6D;AA6B5EuC,EAAAA,kBAAkB,EAAEvC,OA7BwD;AA8B5EwC,EAAAA,4BAA4B,EAAExC;AA9B8C,CAAvE;AAiCP,OAAO,SAASyC,2BAAT,CAAqCC,aAArC,EAA2E;AAChF,QAAMC,MAAM,GAAGC,SAAS,CAACF,aAAa,CAACG,MAAf,CAAxB;AAGA,SAAO,IAAIjD,MAAJ,CAAW+C,MAAX,CAAP;AACD;;AAED,SAASG,gBAAT,CAA0BC,KAA1B,EAAoE;AAClE,QAAMC,QAAQ,GAAG,IAAIC,GAAJ,EAAjB;;AAEA,OAAK,MAAMC,GAAX,IAAkBH,KAAlB,EAAyB;AACvB,QAAIG,GAAG,KAAK,MAAZ,EAAoB;AAClB,YAAMC,KAAK,GAAG,OAAOJ,KAAK,CAACG,GAAD,CAAZ,KAAsB,QAAtB,GAAiChB,IAAI,CAACkB,SAAL,CAAeL,KAAK,CAACG,GAAD,CAApB,CAAjC,GAA8DH,KAAK,CAACG,GAAD,CAAjF;AACAF,MAAAA,QAAQ,CAACK,GAAT,CAAaH,GAAb,EAAkBC,KAAlB;AACD;AACF;;AAED,SAAOH,QAAP;AACD;;AAED,SAASJ,SAAT,CAAmBC,MAAnB,EAAqD;AACnD,QAAMF,MAAe,GAAG,EAAxB;;AAEA,OAAK,MAAMW,IAAX,IAAmBT,MAAnB,EAA2B;AACzB,UAAME,KAAK,GAAGF,MAAM,CAACS,IAAD,CAApB;;AAEA,QAAIP,KAAK,CAACJ,MAAV,EAAkB;AAChB,YAAMY,UAAU,GAAGX,SAAS,CAACG,KAAK,CAACJ,MAAP,CAA5B;AACA,YAAMa,WAAW,GAAG,IAAI1D,KAAJ,CAAUwD,IAAV,EAAgB,IAAIzD,MAAJ,CAAW0D,UAAX,CAAhB,EAAwCR,KAAK,CAACU,QAA9C,CAApB;AACAd,MAAAA,MAAM,CAACe,IAAP,CAAYF,WAAZ;AACD,KAJD,MAIO;AACL,YAAMG,SAAS,GAAGhD,oBAAoB,CAACoC,KAAK,CAACa,IAAP,CAAtC;AACA,YAAMZ,QAAQ,GAAGF,gBAAgB,CAACC,KAAD,CAAjC;AACA,YAAMc,UAAU,GAAG,IAAI/D,KAAJ,CAAUwD,IAAV,EAAgB,IAAIK,SAAJ,EAAhB,EAAiCZ,KAAK,CAACU,QAAvC,EAAiDT,QAAjD,CAAnB;AACAL,MAAAA,MAAM,CAACe,IAAP,CAAYG,UAAZ;AACD;AACF;;AAED,SAAOlB,MAAP;AACD","sourcesContent":["import type {ParquetSchema} from '../parquetjs/schema/schema';\nimport type {FieldDefinition, ParquetField, ParquetType} from '../parquetjs/schema/declare';\n\nimport {\n  Schema,\n  Struct,\n  Field,\n  DataType,\n  Bool,\n  Float64,\n  Int32,\n  Float32,\n  Binary,\n  Utf8,\n  Int64,\n  Uint16,\n  Uint32,\n  Uint64,\n  Int8,\n  Int16\n} from '@loaders.gl/schema';\n\nexport const PARQUET_TYPE_MAPPING: {[type in ParquetType]: typeof DataType} = {\n  BOOLEAN: Bool,\n  INT32: Int32,\n  INT64: Float64,\n  INT96: Float64,\n  FLOAT: Float32,\n  DOUBLE: Float64,\n  BYTE_ARRAY: Binary,\n  FIXED_LEN_BYTE_ARRAY: Binary,\n  UTF8: Utf8,\n  DATE: Int32,\n  TIME_MILLIS: Int64,\n  TIME_MICROS: Int64,\n  TIMESTAMP_MILLIS: Int64,\n  TIMESTAMP_MICROS: Int64,\n  UINT_8: Int32,\n  UINT_16: Uint16,\n  UINT_32: Uint32,\n  UINT_64: Uint64,\n  INT_8: Int8,\n  INT_16: Int16,\n  INT_32: Int32,\n  INT_64: Int64,\n  JSON: Binary,\n  BSON: Binary,\n  // TODO check interal type\n  INTERVAL: Binary,\n  DECIMAL_INT32: Float32,\n  DECIMAL_INT64: Float64,\n  DECIMAL_BYTE_ARRAY: Float64,\n  DECIMAL_FIXED_LEN_BYTE_ARRAY: Float64\n};\n\nexport function convertParquetToArrowSchema(parquetSchema: ParquetSchema): Schema {\n  const fields = getFields(parquetSchema.schema);\n\n  // TODO add metadata if needed.\n  return new Schema(fields);\n}\n\nfunction getFieldMetadata(field: ParquetField): Map<string, string> {\n  const metadata = new Map();\n\n  for (const key in field) {\n    if (key !== 'name') {\n      const value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];\n      metadata.set(key, value);\n    }\n  }\n\n  return metadata;\n}\n\nfunction getFields(schema: FieldDefinition): Field[] {\n  const fields: Field[] = [];\n\n  for (const name in schema) {\n    const field = schema[name];\n\n    if (field.fields) {\n      const childField = getFields(field.fields);\n      const nestedField = new Field(name, new Struct(childField), field.optional);\n      fields.push(nestedField);\n    } else {\n      const FieldType = PARQUET_TYPE_MAPPING[field.type];\n      const metadata = getFieldMetadata(field);\n      const arrowField = new Field(name, new FieldType(), field.optional, metadata);\n      fields.push(arrowField);\n    }\n  }\n\n  return fields;\n}\n"],"file":"convert-schema.js"}

package/dist/esm/lib/parse-parquet.js

@@ -0,0 +1,28 @@
+import { ParquetReader } from '../parquetjs/parser/parquet-reader';
+export async function parseParquet(arrayBuffer, options) {
+  const blob = new Blob([arrayBuffer]);
+
+  for await (const batch of parseParquetFileInBatches(blob, options)) {
+    return batch;
+  }
+
+  return null;
+}
+export async function* parseParquetFileInBatches(blob, options) {
+  const reader = await ParquetReader.openBlob(blob);
+  const rows = [];
+
+  try {
+    const cursor = reader.getCursor();
+    let record;
+
+    while (record = await cursor.next()) {
+      rows.push(record);
+    }
+  } finally {
+    await reader.close();
+  }
+
+  yield rows;
+}
+//# sourceMappingURL=parse-parquet.js.map

package/dist/esm/lib/parse-parquet.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/lib/parse-parquet.ts"],"names":["ParquetReader","parseParquet","arrayBuffer","options","blob","Blob","batch","parseParquetFileInBatches","reader","openBlob","rows","cursor","getCursor","record","next","push","close"],"mappings":"AAGA,SAAQA,aAAR,QAA4B,oCAA5B;AAEA,OAAO,eAAeC,YAAf,CAA4BC,WAA5B,EAAsDC,OAAtD,EAAsF;AAC3F,QAAMC,IAAI,GAAG,IAAIC,IAAJ,CAAS,CAACH,WAAD,CAAT,CAAb;;AACA,aAAW,MAAMI,KAAjB,IAA0BC,yBAAyB,CAACH,IAAD,EAAOD,OAAP,CAAnD,EAAoE;AAClE,WAAOG,KAAP;AACD;;AACD,SAAO,IAAP;AACD;AAED,OAAO,gBAAgBC,yBAAhB,CAA0CH,IAA1C,EAAsDD,OAAtD,EAAsF;AAC3F,QAAMK,MAAM,GAAG,MAAMR,aAAa,CAACS,QAAd,CAAuBL,IAAvB,CAArB;AACA,QAAMM,IAAa,GAAG,EAAtB;;AACA,MAAI;AACF,UAAMC,MAAM,GAAGH,MAAM,CAACI,SAAP,EAAf;AACA,QAAIC,MAAJ;;AACA,WAAQA,MAAM,GAAG,MAAMF,MAAM,CAACG,IAAP,EAAvB,EAAuC;AACrCJ,MAAAA,IAAI,CAACK,IAAL,CAAUF,MAAV;AACD;AACF,GAND,SAMU;AACR,UAAML,MAAM,CAACQ,KAAP,EAAN;AACD;;AACD,QAAMN,IAAN;AACD","sourcesContent":["// import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';\nimport type {ParquetLoaderOptions} from '../parquet-loader';\n\nimport {ParquetReader} from '../parquetjs/parser/parquet-reader';\n\nexport async function parseParquet(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) {\n  const blob = new Blob([arrayBuffer]);\n  for await (const batch of parseParquetFileInBatches(blob, options)) {\n    return batch;\n  }\n  return null;\n}\n\nexport async function* parseParquetFileInBatches(blob: Blob, options?: ParquetLoaderOptions) {\n  const reader = await ParquetReader.openBlob(blob);\n  const rows: any[][] = [];\n  try {\n    const cursor = reader.getCursor();\n    let record: any[] | null;\n    while ((record = await cursor.next())) {\n      rows.push(record);\n    }\n  } finally {\n    await reader.close();\n  }\n  yield rows;\n}\n"],"file":"parse-parquet.js"}

package/dist/esm/lib/read-array-buffer.js

@@ -0,0 +1,9 @@
+export async function readArrayBuffer(file, start, length) {
+  if (file instanceof Blob) {
+    const slice = file.slice(start, start + length);
+    return await slice.arrayBuffer();
+  }
+
+  return await file.read(start, start + length);
+}
+//# sourceMappingURL=read-array-buffer.js.map

package/dist/esm/lib/read-array-buffer.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/lib/read-array-buffer.ts"],"names":["readArrayBuffer","file","start","length","Blob","slice","arrayBuffer","read"],"mappings":"AAEA,OAAO,eAAeA,eAAf,CACLC,IADK,EAELC,KAFK,EAGLC,MAHK,EAIiB;AACtB,MAAIF,IAAI,YAAYG,IAApB,EAA0B;AACxB,UAAMC,KAAK,GAAGJ,IAAI,CAACI,KAAL,CAAWH,KAAX,EAAkBA,KAAK,GAAGC,MAA1B,CAAd;AACA,WAAO,MAAME,KAAK,CAACC,WAAN,EAAb;AACD;;AACD,SAAO,MAAML,IAAI,CAACM,IAAL,CAAUL,KAAV,EAAiBA,KAAK,GAAGC,MAAzB,CAAb;AACD","sourcesContent":["// Random-Access read\n\nexport async function readArrayBuffer(\n  file: Blob | ArrayBuffer | any,\n  start: number,\n  length: number\n): Promise<ArrayBuffer> {\n  if (file instanceof Blob) {\n    const slice = file.slice(start, start + length);\n    return await slice.arrayBuffer();\n  }\n  return await file.read(start, start + length);\n}\n\n/**\n * Read a slice of a Blob or File, without loading the entire file into memory\n * The trick when reading File objects is to read successive \"slices\" of the File\n * Per spec https://w3c.github.io/FileAPI/, slicing a File only updates the start and end fields\n * Actually reading from file happens in `readAsArrayBuffer`\n * @param blob to read\n export async function readBlob(blob: Blob): Promise<ArrayBuffer> {\n  return await new Promise((resolve, reject) => {\n    const fileReader = new FileReader();\n    fileReader.onload = (event: ProgressEvent<FileReader>) =>\n      resolve(event?.target?.result as ArrayBuffer);\n    // TODO - reject with a proper Error\n    fileReader.onerror = (error: ProgressEvent<FileReader>) => reject(error);\n    fileReader.readAsArrayBuffer(blob);\n  });\n}\n*/\n"],"file":"read-array-buffer.js"}

package/dist/esm/parquet-loader.js

@@ -1,12 +1,11 @@
-import { ParquetReader } from './parquetjs/reader';
-const VERSION = typeof "3.0.12" !== 'undefined' ? "3.0.12" : 'latest';
+const VERSION = typeof "3.1.0-alpha.4" !== 'undefined' ? "3.1.0-alpha.4" : 'latest';
 const DEFAULT_PARQUET_LOADER_OPTIONS = {
   parquet: {
     type: 'object-row-table',
     url: undefined
   }
 };
-export const ParquetWorkerLoader = {
+export const ParquetLoader = {
   name: 'Apache Parquet',
   id: 'parquet',
   module: 'parquet',
@@ -16,27 +15,8 @@ export const ParquetWorkerLoader = {
   extensions: ['parquet'],
   mimeTypes: ['application/octet-stream'],
   binary: true,
+  tests: ['PAR1', 'PARE'],
   options: DEFAULT_PARQUET_LOADER_OPTIONS
 };
-export const ParquetLoader = { ...ParquetWorkerLoader,
-  parse
-};
-
-async function parse(arrayBuffer, options) {
-  const reader = await ParquetReader.openArrayBuffer(arrayBuffer);
-  const rows = [];
-
-  try {
-    const cursor = reader.getCursor();
-    let record;
-
-    while (record = await cursor.next()) {
-      rows.push(record);
-    }
-  } finally {
-    await reader.close();
-  }
-
-  return rows;
-}
+export const _typecheckParquetLoader = ParquetLoader;
 //# sourceMappingURL=parquet-loader.js.map

package/dist/esm/parquet-loader.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/parquet-loader.ts"],"names":["ParquetReader","VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","parquet","type","url","undefined","ParquetWorkerLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","options","ParquetLoader","parse","arrayBuffer","reader","openArrayBuffer","rows","cursor","getCursor","record","next","push","close"],"mappings":"AACA,SAAQA,aAAR,QAA4B,oBAA5B;AAIA,MAAMC,OAAO,GAAG,oBAAuB,WAAvB,cAAmD,QAAnE;AASA,MAAMC,8BAAoD,GAAG;AAC3DC,EAAAA,OAAO,EAAE;AACPC,IAAAA,IAAI,EAAE,kBADC;AAEPC,IAAAA,GAAG,EAAEC;AAFE;AADkD,CAA7D;AAQA,OAAO,MAAMC,mBAA2B,GAAG;AACzCC,EAAAA,IAAI,EAAE,gBADmC;AAEzCC,EAAAA,EAAE,EAAE,SAFqC;AAGzCC,EAAAA,MAAM,EAAE,SAHiC;AAIzCC,EAAAA,OAAO,EAAEV,OAJgC;AAKzCW,EAAAA,MAAM,EAAE,IALiC;AAMzCC,EAAAA,QAAQ,EAAE,OAN+B;AAOzCC,EAAAA,UAAU,EAAE,CAAC,SAAD,CAP6B;AAQzCC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CAR8B;AASzCC,EAAAA,MAAM,EAAE,IATiC;AAWzCC,EAAAA,OAAO,EAAEf;AAXgC,CAApC;AAeP,OAAO,MAAMgB,aAA+B,GAAG,EAC7C,GAAGX,mBAD0C;AAE7CY,EAAAA;AAF6C,CAAxC;;AAKP,eAAeA,KAAf,CAAqBC,WAArB,EAA+CH,OAA/C,EAA+E;AAC7E,QAAMI,MAAM,GAAG,MAAMrB,aAAa,CAACsB,eAAd,CAA8BF,WAA9B,CAArB;AACA,QAAMG,IAAa,GAAG,EAAtB;;AACA,MAAI;AACF,UAAMC,MAAM,GAAGH,MAAM,CAACI,SAAP,EAAf;AACA,QAAIC,MAAJ;;AACA,WAAQA,MAAM,GAAG,MAAMF,MAAM,CAACG,IAAP,EAAvB,EAAuC;AACrCJ,MAAAA,IAAI,CAACK,IAAL,CAAUF,MAAV;AACD;AACF,GAND,SAMU;AACR,UAAML,MAAM,CAACQ,KAAP,EAAN;AACD;;AACD,SAAON,IAAP;AACD","sourcesContent":["import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';\nimport {ParquetReader} from './parquetjs/reader';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetLoaderOptions = LoaderOptions & {\n  parquet?: {\n    type?: 'object-row-table';\n    url?: string;\n  };\n};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS: ParquetLoaderOptions = {\n  parquet: {\n    type: 'object-row-table',\n    url: undefined\n  }\n};\n\n/** ParquetJS table loader */\nexport const ParquetWorkerLoader: Loader = {\n  name: 'Apache Parquet',\n  id: 'parquet',\n  module: 'parquet',\n  version: VERSION,\n  worker: true,\n  category: 'table',\n  extensions: ['parquet'],\n  mimeTypes: ['application/octet-stream'],\n  binary: true,\n  // tests: ['PARQUET'],\n  options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\n/** ParquetJS table loader */\nexport const ParquetLoader: LoaderWithParser = {\n  ...ParquetWorkerLoader,\n  parse\n};\n\nasync function parse(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) {\n  const reader = await ParquetReader.openArrayBuffer(arrayBuffer);\n  const rows: any[][] = [];\n  try {\n    const cursor = reader.getCursor();\n    let record: any[] | null;\n    while ((record = await cursor.next())) {\n      rows.push(record);\n    }\n  } finally {\n    await reader.close();\n  }\n  return rows;\n}\n"],"file":"parquet-loader.js"}
+{"version":3,"sources":["../../src/parquet-loader.ts"],"names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","parquet","type","url","undefined","ParquetLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","tests","options","_typecheckParquetLoader"],"mappings":"AAIA,MAAMA,OAAO,GAAG,2BAAuB,WAAvB,qBAAmD,QAAnE;AASA,MAAMC,8BAAoD,GAAG;AAC3DC,EAAAA,OAAO,EAAE;AACPC,IAAAA,IAAI,EAAE,kBADC;AAEPC,IAAAA,GAAG,EAAEC;AAFE;AADkD,CAA7D;AAQA,OAAO,MAAMC,aAAa,GAAG;AAC3BC,EAAAA,IAAI,EAAE,gBADqB;AAE3BC,EAAAA,EAAE,EAAE,SAFuB;AAG3BC,EAAAA,MAAM,EAAE,SAHmB;AAI3BC,EAAAA,OAAO,EAAEV,OAJkB;AAK3BW,EAAAA,MAAM,EAAE,IALmB;AAM3BC,EAAAA,QAAQ,EAAE,OANiB;AAO3BC,EAAAA,UAAU,EAAE,CAAC,SAAD,CAPe;AAQ3BC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CARgB;AAS3BC,EAAAA,MAAM,EAAE,IATmB;AAU3BC,EAAAA,KAAK,EAAE,CAAC,MAAD,EAAS,MAAT,CAVoB;AAW3BC,EAAAA,OAAO,EAAEhB;AAXkB,CAAtB;AAcP,OAAO,MAAMiB,uBAA+B,GAAGZ,aAAxC","sourcesContent":["import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetLoaderOptions = LoaderOptions & {\n  parquet?: {\n    type?: 'object-row-table';\n    url?: string;\n  };\n};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS: ParquetLoaderOptions = {\n  parquet: {\n    type: 'object-row-table',\n    url: undefined\n  }\n};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n  name: 'Apache Parquet',\n  id: 'parquet',\n  module: 'parquet',\n  version: VERSION,\n  worker: true,\n  category: 'table',\n  extensions: ['parquet'],\n  mimeTypes: ['application/octet-stream'],\n  binary: true,\n  tests: ['PAR1', 'PARE'],\n  options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nexport const _typecheckParquetLoader: Loader = ParquetLoader;\n"],"file":"parquet-loader.js"}

package/dist/esm/parquet-writer.js

@@ -1,4 +1,4 @@
-const VERSION = typeof "3.0.12" !== 'undefined' ? "3.0.12" : 'latest';
+const VERSION = typeof "3.1.0-alpha.4" !== 'undefined' ? "3.1.0-alpha.4" : 'latest';
 const DEFAULT_PARQUET_LOADER_OPTIONS = {};
 export const ParquetWriter = {
   name: 'Apache Parquet',

package/dist/esm/parquet-writer.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/parquet-writer.ts"],"names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","ParquetWriter","name","id","module","version","extensions","mimeTypes","encodeSync","binary","options","data","ArrayBuffer"],"mappings":"AAIA,MAAMA,OAAO,GAAG,oBAAuB,WAAvB,cAAmD,QAAnE;AAIA,MAAMC,8BAA8B,GAAG,EAAvC;AAEA,OAAO,MAAMC,aAAqB,GAAG;AACnCC,EAAAA,IAAI,EAAE,gBAD6B;AAEnCC,EAAAA,EAAE,EAAE,SAF+B;AAGnCC,EAAAA,MAAM,EAAE,SAH2B;AAInCC,EAAAA,OAAO,EAAEN,OAJ0B;AAKnCO,EAAAA,UAAU,EAAE,CAAC,SAAD,CALuB;AAMnCC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CANwB;AAOnCC,EAAAA,UAPmC;AAQnCC,EAAAA,MAAM,EAAE,IAR2B;AASnCC,EAAAA,OAAO,EAAEV;AAT0B,CAA9B;;AAYP,SAASQ,UAAT,CAAoBG,IAApB,EAA0BD,OAA1B,EAA0D;AACxD,SAAO,IAAIE,WAAJ,CAAgB,CAAhB,CAAP;AACD","sourcesContent":["import type {Writer} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetWriterOptions = {};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS = {};\n\nexport const ParquetWriter: Writer = {\n  name: 'Apache Parquet',\n  id: 'parquet',\n  module: 'parquet',\n  version: VERSION,\n  extensions: ['parquet'],\n  mimeTypes: ['application/octet-stream'],\n  encodeSync,\n  binary: true,\n  options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nfunction encodeSync(data, options?: ParquetWriterOptions) {\n  return new ArrayBuffer(0);\n}\n"],"file":"parquet-writer.js"}
+{"version":3,"sources":["../../src/parquet-writer.ts"],"names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","ParquetWriter","name","id","module","version","extensions","mimeTypes","encodeSync","binary","options","data","ArrayBuffer"],"mappings":"AAIA,MAAMA,OAAO,GAAG,2BAAuB,WAAvB,qBAAmD,QAAnE;AAIA,MAAMC,8BAA8B,GAAG,EAAvC;AAEA,OAAO,MAAMC,aAAqB,GAAG;AACnCC,EAAAA,IAAI,EAAE,gBAD6B;AAEnCC,EAAAA,EAAE,EAAE,SAF+B;AAGnCC,EAAAA,MAAM,EAAE,SAH2B;AAInCC,EAAAA,OAAO,EAAEN,OAJ0B;AAKnCO,EAAAA,UAAU,EAAE,CAAC,SAAD,CALuB;AAMnCC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CANwB;AAOnCC,EAAAA,UAPmC;AAQnCC,EAAAA,MAAM,EAAE,IAR2B;AASnCC,EAAAA,OAAO,EAAEV;AAT0B,CAA9B;;AAYP,SAASQ,UAAT,CAAoBG,IAApB,EAA0BD,OAA1B,EAA0D;AACxD,SAAO,IAAIE,WAAJ,CAAgB,CAAhB,CAAP;AACD","sourcesContent":["import type {Writer} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetWriterOptions = {};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS = {};\n\nexport const ParquetWriter: Writer = {\n  name: 'Apache Parquet',\n  id: 'parquet',\n  module: 'parquet',\n  version: VERSION,\n  extensions: ['parquet'],\n  mimeTypes: ['application/octet-stream'],\n  encodeSync,\n  binary: true,\n  options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nfunction encodeSync(data, options?: ParquetWriterOptions) {\n  return new ArrayBuffer(0);\n}\n"],"file":"parquet-writer.js"}

package/dist/esm/parquetjs/codecs/dictionary.js

@@ -0,0 +1,12 @@
+import { decodeValues as decodeRleValues } from './rle';
+export function decodeValues(type, cursor, count, opts) {
+  opts.bitWidth = cursor.buffer.slice(cursor.offset, cursor.offset + 1).readInt8(0);
+  cursor.offset += 1;
+  return decodeRleValues(type, cursor, count, { ...opts,
+    disableEnvelope: true
+  });
+}
+export function encodeValues(type, cursor, count, opts) {
+  throw new Error('Encode dictionary functionality is not supported');
+}
+//# sourceMappingURL=dictionary.js.map

package/dist/esm/parquetjs/codecs/dictionary.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../src/parquetjs/codecs/dictionary.ts"],"names":["decodeValues","decodeRleValues","type","cursor","count","opts","bitWidth","buffer","slice","offset","readInt8","disableEnvelope","encodeValues","Error"],"mappings":"AAAA,SAAQA,YAAY,IAAIC,eAAxB,QAA8C,OAA9C;AAEA,OAAO,SAASD,YAAT,CAAsBE,IAAtB,EAA4BC,MAA5B,EAAoCC,KAApC,EAA2CC,IAA3C,EAAiD;AACtDA,EAAAA,IAAI,CAACC,QAAL,GAAgBH,MAAM,CAACI,MAAP,CAAcC,KAAd,CAAoBL,MAAM,CAACM,MAA3B,EAAmCN,MAAM,CAACM,MAAP,GAAgB,CAAnD,EAAsDC,QAAtD,CAA+D,CAA/D,CAAhB;AACAP,EAAAA,MAAM,CAACM,MAAP,IAAiB,CAAjB;AACA,SAAOR,eAAe,CAACC,IAAD,EAAOC,MAAP,EAAeC,KAAf,EAAsB,EAAC,GAAGC,IAAJ;AAAUM,IAAAA,eAAe,EAAE;AAA3B,GAAtB,CAAtB;AACD;AAED,OAAO,SAASC,YAAT,CAAsBV,IAAtB,EAA4BC,MAA5B,EAAoCC,KAApC,EAA2CC,IAA3C,EAAiD;AACtD,QAAM,IAAIQ,KAAJ,CAAU,kDAAV,CAAN;AACD","sourcesContent":["import {decodeValues as decodeRleValues} from './rle';\n\nexport function decodeValues(type, cursor, count, opts) {\n  opts.bitWidth = cursor.buffer.slice(cursor.offset, cursor.offset + 1).readInt8(0);\n  cursor.offset += 1;\n  return decodeRleValues(type, cursor, count, {...opts, disableEnvelope: true});\n}\n\nexport function encodeValues(type, cursor, count, opts) {\n  throw new Error('Encode dictionary functionality is not supported');\n}\n"],"file":"dictionary.js"}

package/dist/esm/parquetjs/codecs/index.js

@@ -1,5 +1,6 @@
 import * as PLAIN from './plain';
 import * as RLE from './rle';
+import * as DICTIONARY from './dictionary';
 export * from './declare';
 export const PARQUET_CODECS = {
   PLAIN: {
@@ -9,6 +10,14 @@ export const PARQUET_CODECS = {
   RLE: {
     encodeValues: RLE.encodeValues,
     decodeValues: RLE.decodeValues
+  },
+  PLAIN_DICTIONARY: {
+    encodeValues: DICTIONARY.encodeValues,
+    decodeValues: DICTIONARY.decodeValues
+  },
+  RLE_DICTIONARY: {
+    encodeValues: DICTIONARY.encodeValues,
+    decodeValues: DICTIONARY.decodeValues
   }
 };
 //# sourceMappingURL=index.js.map

package/dist/esm/parquetjs/codecs/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../src/parquetjs/codecs/index.ts"],"names":["PLAIN","RLE","PARQUET_CODECS","encodeValues","decodeValues"],"mappings":"AAGA,OAAO,KAAKA,KAAZ,MAAuB,SAAvB;AACA,OAAO,KAAKC,GAAZ,MAAqB,OAArB;AAEA,cAAc,WAAd;AAEA,OAAO,MAAMC,cAAqD,GAAG;AACnEF,EAAAA,KAAK,EAAE;AACLG,IAAAA,YAAY,EAAEH,KAAK,CAACG,YADf;AAELC,IAAAA,YAAY,EAAEJ,KAAK,CAACI;AAFf,GAD4D;AAKnEH,EAAAA,GAAG,EAAE;AACHE,IAAAA,YAAY,EAAEF,GAAG,CAACE,YADf;AAEHC,IAAAA,YAAY,EAAEH,GAAG,CAACG;AAFf;AAL8D,CAA9D","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport type {ParquetCodec} from '../schema/declare';\nimport type {ParquetCodecKit} from './declare';\nimport * as PLAIN from './plain';\nimport * as RLE from './rle';\n\nexport * from './declare';\n\nexport const PARQUET_CODECS: Record<ParquetCodec, ParquetCodecKit> = {\n  PLAIN: {\n    encodeValues: PLAIN.encodeValues,\n    decodeValues: PLAIN.decodeValues\n  },\n  RLE: {\n    encodeValues: RLE.encodeValues,\n    decodeValues: RLE.decodeValues\n  }\n};\n"],"file":"index.js"}
+{"version":3,"sources":["../../../../src/parquetjs/codecs/index.ts"],"names":["PLAIN","RLE","DICTIONARY","PARQUET_CODECS","encodeValues","decodeValues","PLAIN_DICTIONARY","RLE_DICTIONARY"],"mappings":"AAGA,OAAO,KAAKA,KAAZ,MAAuB,SAAvB;AACA,OAAO,KAAKC,GAAZ,MAAqB,OAArB;AACA,OAAO,KAAKC,UAAZ,MAA4B,cAA5B;AAEA,cAAc,WAAd;AAEA,OAAO,MAAMC,cAAqD,GAAG;AACnEH,EAAAA,KAAK,EAAE;AACLI,IAAAA,YAAY,EAAEJ,KAAK,CAACI,YADf;AAELC,IAAAA,YAAY,EAAEL,KAAK,CAACK;AAFf,GAD4D;AAKnEJ,EAAAA,GAAG,EAAE;AACHG,IAAAA,YAAY,EAAEH,GAAG,CAACG,YADf;AAEHC,IAAAA,YAAY,EAAEJ,GAAG,CAACI;AAFf,GAL8D;AAUnEC,EAAAA,gBAAgB,EAAE;AAEhBF,IAAAA,YAAY,EAAEF,UAAU,CAACE,YAFT;AAGhBC,IAAAA,YAAY,EAAEH,UAAU,CAACG;AAHT,GAViD;AAgBnEE,EAAAA,cAAc,EAAE;AAEdH,IAAAA,YAAY,EAAEF,UAAU,CAACE,YAFX;AAGdC,IAAAA,YAAY,EAAEH,UAAU,CAACG;AAHX;AAhBmD,CAA9D","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport type {ParquetCodec} from '../schema/declare';\nimport type {ParquetCodecKit} from './declare';\nimport * as PLAIN from './plain';\nimport * as RLE from './rle';\nimport * as DICTIONARY from './dictionary';\n\nexport * from './declare';\n\nexport const PARQUET_CODECS: Record<ParquetCodec, ParquetCodecKit> = {\n  PLAIN: {\n    encodeValues: PLAIN.encodeValues,\n    decodeValues: PLAIN.decodeValues\n  },\n  RLE: {\n    encodeValues: RLE.encodeValues,\n    decodeValues: RLE.decodeValues\n  },\n  // Using the PLAIN_DICTIONARY enum value is deprecated in the Parquet 2.0 specification.\n  PLAIN_DICTIONARY: {\n    // @ts-ignore\n    encodeValues: DICTIONARY.encodeValues,\n    decodeValues: DICTIONARY.decodeValues\n  },\n  // Prefer using RLE_DICTIONARY in a data page and PLAIN in a dictionary page for Parquet 2.0+ files.\n  RLE_DICTIONARY: {\n    // @ts-ignore\n    encodeValues: DICTIONARY.encodeValues,\n    decodeValues: DICTIONARY.decodeValues\n  }\n};\n"],"file":"index.js"}

package/dist/esm/parquetjs/codecs/rle.js

@@ -111,7 +111,7 @@ function decodeRunRepeated(cursor, count, opts) {
   let value = 0;
 
   for (let i = 0; i < Math.ceil(bitWidth / 8); i++) {
-    value <<= 8;
+    value << 8;
     value += cursor.buffer[cursor.offset];
     cursor.offset += 1;
   }
@@ -143,7 +143,7 @@ function encodeRunRepeated(value, count, opts) {
 
   for (let i = 0; i < buf.length; i++) {
     buf.writeUInt8(value & 0xff, i);
-    value >>= 8;
+    value >> 8;
   }
 
   return Buffer.concat([Buffer.from(varint.encode(count << 1)), buf]);

package/dist/esm/parquetjs/codecs/rle.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../src/parquetjs/codecs/rle.ts"],"names":["varint","encodeValues","type","values","opts","Error","map","x","parseInt","buf","Buffer","alloc","run","repeats","i","length","concat","encodeRunBitpacked","encodeRunRepeated","push","disableEnvelope","envelope","writeUInt32LE","undefined","copy","decodeValues","cursor","count","offset","header","decode","buffer","encodingLength","decodeRunBitpacked","decodeRunRepeated","slice","bitWidth","Array","fill","b","Math","floor","value","ceil","from","encode","writeUInt8"],"mappings":"AAIA,OAAOA,MAAP,MAAmB,QAAnB;AAGA,OAAO,SAASC,YAAT,CACLC,IADK,EAELC,MAFK,EAGLC,IAHK,EAIG;AACR,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,UAAQH,IAAR;AACE,SAAK,SAAL;AACA,SAAK,OAAL;AACA,SAAK,OAAL;AAEEC,MAAAA,MAAM,GAAGA,MAAM,CAACG,GAAP,CAAYC,CAAD,IAAOC,QAAQ,CAACD,CAAD,EAAI,EAAJ,CAA1B,CAAT;AACA;;AAEF;AACE,YAAM,IAAIF,KAAJ,CAAW,qBAAoBH,IAAK,EAApC,CAAN;AATJ;;AAYA,MAAIO,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,CAAb,CAAV;AACA,MAAIC,GAAU,GAAG,EAAjB;AACA,MAAIC,OAAO,GAAG,CAAd;;AAEA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAA3B,EAAmCD,CAAC,EAApC,EAAwC;AAGtC,QAAID,OAAO,KAAK,CAAZ,IAAiBD,GAAG,CAACG,MAAJ,GAAa,CAAb,KAAmB,CAApC,IAAyCZ,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAjE,EAA0E;AAExE,UAAIF,GAAG,CAACG,MAAR,EAAgB;AACdN,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACAQ,QAAAA,GAAG,GAAG,EAAN;AACD;;AACDC,MAAAA,OAAO,GAAG,CAAV;AACD,KAPD,MAOO,IAAIA,OAAO,GAAG,CAAV,IAAeV,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAvC,EAAgD;AACrDD,MAAAA,OAAO,IAAI,CAAX;AACD,KAFM,MAEA;AAEL,UAAIA,OAAJ,EAAa;AACXJ,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACW,CAAC,GAAG,CAAL,CAAP,EAAgBD,OAAhB,EAAyBT,IAAzB,CAAvB,CAAd,CAAN;AACAS,QAAAA,OAAO,GAAG,CAAV;AACD;;AACDD,MAAAA,GAAG,CAACO,IAAJ,CAAShB,MAAM,CAACW,CAAD,CAAf;AACD;AACF;;AAED,MAAID,OAAJ,EAAa;AACXJ,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACA,MAAM,CAACY,MAAP,GAAgB,CAAjB,CAAP,EAA4BF,OAA5B,EAAqCT,IAArC,CAAvB,CAAd,CAAN;AACD,GAFD,MAEO,IAAIQ,GAAG,CAACG,MAAR,EAAgB;AACrBN,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACD;;AAED,MAAIA,IAAI,CAACgB,eAAT,EAA0B;AACxB,WAAOX,GAAP;AACD;;AAED,QAAMY,QAAQ,GAAGX,MAAM,CAACC,KAAP,CAAaF,GAAG,CAACM,MAAJ,GAAa,CAA1B,CAAjB;AACAM,EAAAA,QAAQ,CAACC,aAAT,CAAuBb,GAAG,CAACM,MAA3B,EAAmCQ,SAAnC;AACAd,EAAAA,GAAG,CAACe,IAAJ,CAASH,QAAT,EAAmB,CAAnB;AAEA,SAAOA,QAAP;AACD;AAED,OAAO,SAASI,YAAT,CACLvB,IADK,EAELwB,MAFK,EAGLC,KAHK,EAILvB,IAJK,EAKK;AACV,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,MAAI,CAACD,IAAI,CAACgB,eAAV,EAA2B;AACzBM,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAED,MAAIzB,MAAgB,GAAG,EAAvB;;AACA,SAAOA,MAAM,CAACY,MAAP,GAAgBY,KAAvB,EAA8B;AAC5B,UAAME,MAAM,GAAG7B,MAAM,CAAC8B,MAAP,CAAcJ,MAAM,CAACK,MAArB,EAA6BL,MAAM,CAACE,MAApC,CAAf;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiB5B,MAAM,CAACgC,cAAP,CAAsBH,MAAtB,CAAjB;;AACA,QAAIA,MAAM,GAAG,CAAb,EAAgB;AACd,YAAMF,KAAK,GAAG,CAACE,MAAM,IAAI,CAAX,IAAgB,CAA9B;AACA1B,MAAAA,MAAM,CAACgB,IAAP,CAAY,GAAGc,kBAAkB,CAACP,MAAD,EAASC,KAAT,EAAgBvB,IAAhB,CAAjC;AACD,KAHD,MAGO;AACL,YAAMuB,KAAK,GAAGE,MAAM,IAAI,CAAxB;AACA1B,MAAAA,MAAM,CAACgB,IAAP,CAAY,GAAGe,iBAAiB,CAACR,MAAD,EAASC,KAAT,EAAgBvB,IAAhB,CAAhC;AACD;AACF;;AACDD,EAAAA,MAAM,GAAGA,MAAM,CAACgC,KAAP,CAAa,CAAb,EAAgBR,KAAhB,CAAT;;AAEA,MAAIxB,MAAM,CAACY,MAAP,KAAkBY,KAAtB,EAA6B;AAC3B,UAAM,IAAItB,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,SAAOF,MAAP;AACD;;AAED,SAAS8B,kBAAT,CACEP,MADF,EAEEC,KAFF,EAGEvB,IAHF,EAIY;AAEV,QAAMgC,QAAgB,GAAGhC,IAAI,CAACgC,QAA9B;;AAEA,MAAIT,KAAK,GAAG,CAAR,KAAc,CAAlB,EAAqB;AACnB,UAAM,IAAItB,KAAJ,CAAU,yBAAV,CAAN;AACD;;AAGD,QAAMF,MAAM,GAAG,IAAIkC,KAAJ,CAAUV,KAAV,EAAiBW,IAAjB,CAAsB,CAAtB,CAAf;;AACA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGT,KAA/B,EAAsCY,CAAC,EAAvC,EAA2C;AACzC,QAAIb,MAAM,CAACK,MAAP,CAAcL,MAAM,CAACE,MAAP,GAAgBY,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAA9B,IAAoD,KAAKA,CAAC,GAAG,CAAjE,EAAqE;AACnEpC,MAAAA,MAAM,CAACqC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,IAAoC,KAAKG,CAAC,GAAGH,QAA7C;AACD;AACF;;AAEDV,EAAAA,MAAM,CAACE,MAAP,IAAiBQ,QAAQ,IAAIT,KAAK,GAAG,CAAZ,CAAzB;AACA,SAAOxB,MAAP;AACD;;AAED,SAAS+B,iBAAT,CACER,MADF,EAEEC,KAFF,EAGEvB,IAHF,EAIY;AAEV,QAAMgC,QAAgB,GAAGhC,IAAI,CAACgC,QAA9B;AAEA,MAAIM,KAAK,GAAG,CAAZ;;AACA,OAAK,IAAI5B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG0B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAApB,EAA6CtB,CAAC,EAA9C,EAAkD;AAChD4B,IAAAA,KAAK,KAAK,CAAV;AACAA,IAAAA,KAAK,IAAIhB,MAAM,CAACK,MAAP,CAAcL,MAAM,CAACE,MAArB,CAAT;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAGD,SAAO,IAAIS,KAAJ,CAAUV,KAAV,EAAiBW,IAAjB,CAAsBI,KAAtB,CAAP;AACD;;AAED,SAASzB,kBAAT,CAA4Bd,MAA5B,EAA8CC,IAA9C,EAAiF;AAE/E,QAAMgC,QAAgB,GAAGhC,IAAI,CAACgC,QAA9B;;AAEA,OAAK,IAAItB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAAP,GAAgB,CAApC,EAAuCD,CAAC,EAAxC,EAA4C;AAC1CX,IAAAA,MAAM,CAACgB,IAAP,CAAY,CAAZ;AACD;;AAED,QAAMV,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa6B,IAAI,CAACG,IAAL,CAAUP,QAAQ,IAAIjC,MAAM,CAACY,MAAP,GAAgB,CAApB,CAAlB,CAAb,CAAZ;;AACA,OAAK,IAAIwB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGjC,MAAM,CAACY,MAAtC,EAA8CwB,CAAC,EAA/C,EAAmD;AACjD,QAAI,CAACpC,MAAM,CAACqC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,GAAoC,KAAKG,CAAC,GAAGH,QAA9C,IAA2D,CAA/D,EAAkE;AAChE3B,MAAAA,GAAG,CAAC+B,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAAD,CAAH,IAA0B,KAAKA,CAAC,GAAG,CAAnC;AACD;AACF;;AAED,SAAO7B,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACkC,IAAP,CAAY5C,MAAM,CAAC6C,MAAP,CAAgB1C,MAAM,CAACY,MAAP,GAAgB,CAAjB,IAAuB,CAAxB,GAA6B,CAA3C,CAAZ,CAAD,EAA6DN,GAA7D,CAAd,CAAP;AACD;;AAED,SAASS,iBAAT,CAA2BwB,KAA3B,EAA0Cf,KAA1C,EAAyDvB,IAAzD,EAA4F;AAE1F,QAAMgC,QAAgB,GAAGhC,IAAI,CAACgC,QAA9B;AAEA,QAAM3B,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa6B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAAb,CAAZ;;AAEA,OAAK,IAAItB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGL,GAAG,CAACM,MAAxB,EAAgCD,CAAC,EAAjC,EAAqC;AACnCL,IAAAA,GAAG,CAACqC,UAAJ,CAAeJ,KAAK,GAAG,IAAvB,EAA6B5B,CAA7B;AACA4B,IAAAA,KAAK,KAAK,CAAV;AACD;;AAED,SAAOhC,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACkC,IAAP,CAAY5C,MAAM,CAAC6C,MAAP,CAAclB,KAAK,IAAI,CAAvB,CAAZ,CAAD,EAAyClB,GAAzC,CAAd,CAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport type {PrimitiveType} from '../schema/declare';\nimport type {CursorBuffer, ParquetCodecOptions} from './declare';\nimport varint from 'varint';\n\n// eslint-disable-next-line max-statements, complexity\nexport function encodeValues(\n  type: PrimitiveType,\n  values: any[],\n  opts: ParquetCodecOptions\n): Buffer {\n  if (!('bitWidth' in opts)) {\n    throw new Error('bitWidth is required');\n  }\n\n  switch (type) {\n    case 'BOOLEAN':\n    case 'INT32':\n    case 'INT64':\n      // tslint:disable-next-line:no-parameter-reassignment\n      values = values.map((x) => parseInt(x, 10));\n      break;\n\n    default:\n      throw new Error(`unsupported type: ${type}`);\n  }\n\n  let buf = Buffer.alloc(0);\n  let run: any[] = [];\n  let repeats = 0;\n\n  for (let i = 0; i < values.length; i++) {\n    // If we are at the beginning of a run and the next value is same we start\n    // collecting repeated values\n    if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {\n      // If we have any data in runs we need to encode them\n      if (run.length) {\n        buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n        run = [];\n      }\n      repeats = 1;\n    } else if (repeats > 0 && values[i] === values[i - 1]) {\n      repeats += 1;\n    } else {\n      // If values changes we need to post any previous repeated values\n      if (repeats) {\n        buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)]);\n        repeats = 0;\n      }\n      run.push(values[i]);\n    }\n  }\n\n  if (repeats) {\n    buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)]);\n  } else if (run.length) {\n    buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n  }\n\n  if (opts.disableEnvelope) {\n    return buf;\n  }\n\n  const envelope = Buffer.alloc(buf.length + 4);\n  envelope.writeUInt32LE(buf.length, undefined);\n  buf.copy(envelope, 4);\n\n  return envelope;\n}\n\nexport function decodeValues(\n  type: PrimitiveType,\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  if (!('bitWidth' in opts)) {\n    throw new Error('bitWidth is required');\n  }\n\n  if (!opts.disableEnvelope) {\n    cursor.offset += 4;\n  }\n\n  let values: number[] = [];\n  while (values.length < count) {\n    const header = varint.decode(cursor.buffer, cursor.offset);\n    cursor.offset += varint.encodingLength(header);\n    if (header & 1) {\n      const count = (header >> 1) * 8;\n      values.push(...decodeRunBitpacked(cursor, count, opts));\n    } else {\n      const count = header >> 1;\n      values.push(...decodeRunRepeated(cursor, count, opts));\n    }\n  }\n  values = values.slice(0, count);\n\n  if (values.length !== count) {\n    throw new Error('invalid RLE encoding');\n  }\n\n  return values;\n}\n\nfunction decodeRunBitpacked(\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  if (count % 8 !== 0) {\n    throw new Error('must be a multiple of 8');\n  }\n\n  // tslint:disable-next-line:prefer-array-literal\n  const values = new Array(count).fill(0);\n  for (let b = 0; b < bitWidth * count; b++) {\n    if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & (1 << b % 8)) {\n      values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;\n    }\n  }\n\n  cursor.offset += bitWidth * (count / 8);\n  return values;\n}\n\nfunction decodeRunRepeated(\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  let value = 0;\n  for (let i = 0; i < Math.ceil(bitWidth / 8); i++) {\n    value <<= 8;\n    value += cursor.buffer[cursor.offset];\n    cursor.offset += 1;\n  }\n\n  // tslint:disable-next-line:prefer-array-literal\n  return new Array(count).fill(value);\n}\n\nfunction encodeRunBitpacked(values: number[], opts: ParquetCodecOptions): Buffer {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  for (let i = 0; i < values.length % 8; i++) {\n    values.push(0);\n  }\n\n  const buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));\n  for (let b = 0; b < bitWidth * values.length; b++) {\n    if ((values[Math.floor(b / bitWidth)] & (1 << b % bitWidth)) > 0) {\n      buf[Math.floor(b / 8)] |= 1 << b % 8;\n    }\n  }\n\n  return Buffer.concat([Buffer.from(varint.encode(((values.length / 8) << 1) | 1)), buf]);\n}\n\nfunction encodeRunRepeated(value: number, count: number, opts: ParquetCodecOptions): Buffer {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  const buf = Buffer.alloc(Math.ceil(bitWidth / 8));\n\n  for (let i = 0; i < buf.length; i++) {\n    buf.writeUInt8(value & 0xff, i);\n    value >>= 8;\n  }\n\n  return Buffer.concat([Buffer.from(varint.encode(count << 1)), buf]);\n}\n"],"file":"rle.js"}
+{"version":3,"sources":["../../../../src/parquetjs/codecs/rle.ts"],"names":["varint","encodeValues","type","values","opts","Error","map","x","parseInt","buf","Buffer","alloc","run","repeats","i","length","concat","encodeRunBitpacked","encodeRunRepeated","push","disableEnvelope","envelope","writeUInt32LE","undefined","copy","decodeValues","cursor","count","offset","header","decode","buffer","encodingLength","decodeRunBitpacked","decodeRunRepeated","slice","bitWidth","Array","fill","b","Math","floor","value","ceil","from","encode","writeUInt8"],"mappings":"AAIA,OAAOA,MAAP,MAAmB,QAAnB;AAGA,OAAO,SAASC,YAAT,CACLC,IADK,EAELC,MAFK,EAGLC,IAHK,EAIG;AACR,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,UAAQH,IAAR;AACE,SAAK,SAAL;AACA,SAAK,OAAL;AACA,SAAK,OAAL;AAEEC,MAAAA,MAAM,GAAGA,MAAM,CAACG,GAAP,CAAYC,CAAD,IAAOC,QAAQ,CAACD,CAAD,EAAI,EAAJ,CAA1B,CAAT;AACA;;AAEF;AACE,YAAM,IAAIF,KAAJ,CAAW,qBAAoBH,IAAK,EAApC,CAAN;AATJ;;AAYA,MAAIO,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa,CAAb,CAAV;AACA,MAAIC,GAAU,GAAG,EAAjB;AACA,MAAIC,OAAO,GAAG,CAAd;;AAEA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAA3B,EAAmCD,CAAC,EAApC,EAAwC;AAGtC,QAAID,OAAO,KAAK,CAAZ,IAAiBD,GAAG,CAACG,MAAJ,GAAa,CAAb,KAAmB,CAApC,IAAyCZ,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAjE,EAA0E;AAExE,UAAIF,GAAG,CAACG,MAAR,EAAgB;AACdN,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACAQ,QAAAA,GAAG,GAAG,EAAN;AACD;;AACDC,MAAAA,OAAO,GAAG,CAAV;AACD,KAPD,MAOO,IAAIA,OAAO,GAAG,CAAV,IAAeV,MAAM,CAACW,CAAD,CAAN,KAAcX,MAAM,CAACW,CAAC,GAAG,CAAL,CAAvC,EAAgD;AACrDD,MAAAA,OAAO,IAAI,CAAX;AACD,KAFM,MAEA;AAEL,UAAIA,OAAJ,EAAa;AACXJ,QAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACW,CAAC,GAAG,CAAL,CAAP,EAAgBD,OAAhB,EAAyBT,IAAzB,CAAvB,CAAd,CAAN;AACAS,QAAAA,OAAO,GAAG,CAAV;AACD;;AACDD,MAAAA,GAAG,CAACO,IAAJ,CAAShB,MAAM,CAACW,CAAD,CAAf;AACD;AACF;;AAED,MAAID,OAAJ,EAAa;AACXJ,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMS,iBAAiB,CAACf,MAAM,CAACA,MAAM,CAACY,MAAP,GAAgB,CAAjB,CAAP,EAA4BF,OAA5B,EAAqCT,IAArC,CAAvB,CAAd,CAAN;AACD,GAFD,MAEO,IAAIQ,GAAG,CAACG,MAAR,EAAgB;AACrBN,IAAAA,GAAG,GAAGC,MAAM,CAACM,MAAP,CAAc,CAACP,GAAD,EAAMQ,kBAAkB,CAACL,GAAD,EAAMR,IAAN,CAAxB,CAAd,CAAN;AACD;;AAED,MAAIA,IAAI,CAACgB,eAAT,EAA0B;AACxB,WAAOX,GAAP;AACD;;AAED,QAAMY,QAAQ,GAAGX,MAAM,CAACC,KAAP,CAAaF,GAAG,CAACM,MAAJ,GAAa,CAA1B,CAAjB;AACAM,EAAAA,QAAQ,CAACC,aAAT,CAAuBb,GAAG,CAACM,MAA3B,EAAmCQ,SAAnC;AACAd,EAAAA,GAAG,CAACe,IAAJ,CAASH,QAAT,EAAmB,CAAnB;AAEA,SAAOA,QAAP;AACD;AAED,OAAO,SAASI,YAAT,CACLvB,IADK,EAELwB,MAFK,EAGLC,KAHK,EAILvB,IAJK,EAKK;AACV,MAAI,EAAE,cAAcA,IAAhB,CAAJ,EAA2B;AACzB,UAAM,IAAIC,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,MAAI,CAACD,IAAI,CAACgB,eAAV,EAA2B;AACzBM,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAED,MAAIzB,MAAgB,GAAG,EAAvB;;AACA,SAAOA,MAAM,CAACY,MAAP,GAAgBY,KAAvB,EAA8B;AAC5B,UAAME,MAAM,GAAG7B,MAAM,CAAC8B,MAAP,CAAcJ,MAAM,CAACK,MAArB,EAA6BL,MAAM,CAACE,MAApC,CAAf;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiB5B,MAAM,CAACgC,cAAP,CAAsBH,MAAtB,CAAjB;;AACA,QAAIA,MAAM,GAAG,CAAb,EAAgB;AACd,YAAMF,KAAK,GAAG,CAACE,MAAM,IAAI,CAAX,IAAgB,CAA9B;AACA1B,MAAAA,MAAM,CAACgB,IAAP,CAAY,GAAGc,kBAAkB,CAACP,MAAD,EAASC,KAAT,EAAgBvB,IAAhB,CAAjC;AACD,KAHD,MAGO;AACL,YAAMuB,KAAK,GAAGE,MAAM,IAAI,CAAxB;AACA1B,MAAAA,MAAM,CAACgB,IAAP,CAAY,GAAGe,iBAAiB,CAACR,MAAD,EAASC,KAAT,EAAgBvB,IAAhB,CAAhC;AACD;AACF;;AACDD,EAAAA,MAAM,GAAGA,MAAM,CAACgC,KAAP,CAAa,CAAb,EAAgBR,KAAhB,CAAT;;AAEA,MAAIxB,MAAM,CAACY,MAAP,KAAkBY,KAAtB,EAA6B;AAC3B,UAAM,IAAItB,KAAJ,CAAU,sBAAV,CAAN;AACD;;AAED,SAAOF,MAAP;AACD;;AAED,SAAS8B,kBAAT,CACEP,MADF,EAEEC,KAFF,EAGEvB,IAHF,EAIY;AAEV,QAAMgC,QAAgB,GAAGhC,IAAI,CAACgC,QAA9B;;AAEA,MAAIT,KAAK,GAAG,CAAR,KAAc,CAAlB,EAAqB;AACnB,UAAM,IAAItB,KAAJ,CAAU,yBAAV,CAAN;AACD;;AAGD,QAAMF,MAAM,GAAG,IAAIkC,KAAJ,CAAUV,KAAV,EAAiBW,IAAjB,CAAsB,CAAtB,CAAf;;AACA,OAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGT,KAA/B,EAAsCY,CAAC,EAAvC,EAA2C;AACzC,QAAIb,MAAM,CAACK,MAAP,CAAcL,MAAM,CAACE,MAAP,GAAgBY,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAA9B,IAAoD,KAAKA,CAAC,GAAG,CAAjE,EAAqE;AACnEpC,MAAAA,MAAM,CAACqC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,IAAoC,KAAKG,CAAC,GAAGH,QAA7C;AACD;AACF;;AAEDV,EAAAA,MAAM,CAACE,MAAP,IAAiBQ,QAAQ,IAAIT,KAAK,GAAG,CAAZ,CAAzB;AACA,SAAOxB,MAAP;AACD;;AAED,SAAS+B,iBAAT,CACER,MADF,EAEEC,KAFF,EAGEvB,IAHF,EAIY;AAEV,QAAMgC,QAAgB,GAAGhC,IAAI,CAACgC,QAA9B;AAEA,MAAIM,KAAK,GAAG,CAAZ;;AACA,OAAK,IAAI5B,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG0B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAApB,EAA6CtB,CAAC,EAA9C,EAAkD;AAEhD4B,IAAAA,KAAK,IAAI,CAAT;AACAA,IAAAA,KAAK,IAAIhB,MAAM,CAACK,MAAP,CAAcL,MAAM,CAACE,MAArB,CAAT;AACAF,IAAAA,MAAM,CAACE,MAAP,IAAiB,CAAjB;AACD;;AAGD,SAAO,IAAIS,KAAJ,CAAUV,KAAV,EAAiBW,IAAjB,CAAsBI,KAAtB,CAAP;AACD;;AAED,SAASzB,kBAAT,CAA4Bd,MAA5B,EAA8CC,IAA9C,EAAiF;AAE/E,QAAMgC,QAAgB,GAAGhC,IAAI,CAACgC,QAA9B;;AAEA,OAAK,IAAItB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGX,MAAM,CAACY,MAAP,GAAgB,CAApC,EAAuCD,CAAC,EAAxC,EAA4C;AAC1CX,IAAAA,MAAM,CAACgB,IAAP,CAAY,CAAZ;AACD;;AAED,QAAMV,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa6B,IAAI,CAACG,IAAL,CAAUP,QAAQ,IAAIjC,MAAM,CAACY,MAAP,GAAgB,CAApB,CAAlB,CAAb,CAAZ;;AACA,OAAK,IAAIwB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGH,QAAQ,GAAGjC,MAAM,CAACY,MAAtC,EAA8CwB,CAAC,EAA/C,EAAmD;AACjD,QAAI,CAACpC,MAAM,CAACqC,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAGH,QAAf,CAAD,CAAN,GAAoC,KAAKG,CAAC,GAAGH,QAA9C,IAA2D,CAA/D,EAAkE;AAChE3B,MAAAA,GAAG,CAAC+B,IAAI,CAACC,KAAL,CAAWF,CAAC,GAAG,CAAf,CAAD,CAAH,IAA0B,KAAKA,CAAC,GAAG,CAAnC;AACD;AACF;;AAED,SAAO7B,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACkC,IAAP,CAAY5C,MAAM,CAAC6C,MAAP,CAAgB1C,MAAM,CAACY,MAAP,GAAgB,CAAjB,IAAuB,CAAxB,GAA6B,CAA3C,CAAZ,CAAD,EAA6DN,GAA7D,CAAd,CAAP;AACD;;AAED,SAASS,iBAAT,CAA2BwB,KAA3B,EAA0Cf,KAA1C,EAAyDvB,IAAzD,EAA4F;AAE1F,QAAMgC,QAAgB,GAAGhC,IAAI,CAACgC,QAA9B;AAEA,QAAM3B,GAAG,GAAGC,MAAM,CAACC,KAAP,CAAa6B,IAAI,CAACG,IAAL,CAAUP,QAAQ,GAAG,CAArB,CAAb,CAAZ;;AAEA,OAAK,IAAItB,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAGL,GAAG,CAACM,MAAxB,EAAgCD,CAAC,EAAjC,EAAqC;AACnCL,IAAAA,GAAG,CAACqC,UAAJ,CAAeJ,KAAK,GAAG,IAAvB,EAA6B5B,CAA7B;AAEA4B,IAAAA,KAAK,IAAI,CAAT;AACD;;AAED,SAAOhC,MAAM,CAACM,MAAP,CAAc,CAACN,MAAM,CAACkC,IAAP,CAAY5C,MAAM,CAAC6C,MAAP,CAAclB,KAAK,IAAI,CAAvB,CAAZ,CAAD,EAAyClB,GAAzC,CAAd,CAAP;AACD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport type {PrimitiveType} from '../schema/declare';\nimport type {CursorBuffer, ParquetCodecOptions} from './declare';\nimport varint from 'varint';\n\n// eslint-disable-next-line max-statements, complexity\nexport function encodeValues(\n  type: PrimitiveType,\n  values: any[],\n  opts: ParquetCodecOptions\n): Buffer {\n  if (!('bitWidth' in opts)) {\n    throw new Error('bitWidth is required');\n  }\n\n  switch (type) {\n    case 'BOOLEAN':\n    case 'INT32':\n    case 'INT64':\n      // tslint:disable-next-line:no-parameter-reassignment\n      values = values.map((x) => parseInt(x, 10));\n      break;\n\n    default:\n      throw new Error(`unsupported type: ${type}`);\n  }\n\n  let buf = Buffer.alloc(0);\n  let run: any[] = [];\n  let repeats = 0;\n\n  for (let i = 0; i < values.length; i++) {\n    // If we are at the beginning of a run and the next value is same we start\n    // collecting repeated values\n    if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {\n      // If we have any data in runs we need to encode them\n      if (run.length) {\n        buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n        run = [];\n      }\n      repeats = 1;\n    } else if (repeats > 0 && values[i] === values[i - 1]) {\n      repeats += 1;\n    } else {\n      // If values changes we need to post any previous repeated values\n      if (repeats) {\n        buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)]);\n        repeats = 0;\n      }\n      run.push(values[i]);\n    }\n  }\n\n  if (repeats) {\n    buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)]);\n  } else if (run.length) {\n    buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);\n  }\n\n  if (opts.disableEnvelope) {\n    return buf;\n  }\n\n  const envelope = Buffer.alloc(buf.length + 4);\n  envelope.writeUInt32LE(buf.length, undefined);\n  buf.copy(envelope, 4);\n\n  return envelope;\n}\n\nexport function decodeValues(\n  type: PrimitiveType,\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  if (!('bitWidth' in opts)) {\n    throw new Error('bitWidth is required');\n  }\n\n  if (!opts.disableEnvelope) {\n    cursor.offset += 4;\n  }\n\n  let values: number[] = [];\n  while (values.length < count) {\n    const header = varint.decode(cursor.buffer, cursor.offset);\n    cursor.offset += varint.encodingLength(header);\n    if (header & 1) {\n      const count = (header >> 1) * 8;\n      values.push(...decodeRunBitpacked(cursor, count, opts));\n    } else {\n      const count = header >> 1;\n      values.push(...decodeRunRepeated(cursor, count, opts));\n    }\n  }\n  values = values.slice(0, count);\n\n  if (values.length !== count) {\n    throw new Error('invalid RLE encoding');\n  }\n\n  return values;\n}\n\nfunction decodeRunBitpacked(\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  if (count % 8 !== 0) {\n    throw new Error('must be a multiple of 8');\n  }\n\n  // tslint:disable-next-line:prefer-array-literal\n  const values = new Array(count).fill(0);\n  for (let b = 0; b < bitWidth * count; b++) {\n    if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & (1 << b % 8)) {\n      values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;\n    }\n  }\n\n  cursor.offset += bitWidth * (count / 8);\n  return values;\n}\n\nfunction decodeRunRepeated(\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): number[] {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  let value = 0;\n  for (let i = 0; i < Math.ceil(bitWidth / 8); i++) {\n    // eslint-disable-next-line\n    value << 8; //  TODO - this looks wrong\n    value += cursor.buffer[cursor.offset];\n    cursor.offset += 1;\n  }\n\n  // tslint:disable-next-line:prefer-array-literal\n  return new Array(count).fill(value);\n}\n\nfunction encodeRunBitpacked(values: number[], opts: ParquetCodecOptions): Buffer {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  for (let i = 0; i < values.length % 8; i++) {\n    values.push(0);\n  }\n\n  const buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));\n  for (let b = 0; b < bitWidth * values.length; b++) {\n    if ((values[Math.floor(b / bitWidth)] & (1 << b % bitWidth)) > 0) {\n      buf[Math.floor(b / 8)] |= 1 << b % 8;\n    }\n  }\n\n  return Buffer.concat([Buffer.from(varint.encode(((values.length / 8) << 1) | 1)), buf]);\n}\n\nfunction encodeRunRepeated(value: number, count: number, opts: ParquetCodecOptions): Buffer {\n  // @ts-ignore\n  const bitWidth: number = opts.bitWidth;\n\n  const buf = Buffer.alloc(Math.ceil(bitWidth / 8));\n\n  for (let i = 0; i < buf.length; i++) {\n    buf.writeUInt8(value & 0xff, i);\n    // eslint-disable-next-line\n    value >> 8; //  TODO - this looks wrong\n  }\n\n  return Buffer.concat([Buffer.from(varint.encode(count << 1)), buf]);\n}\n"],"file":"rle.js"}

package/dist/esm/parquetjs/compression.js

@@ -1,80 +1,66 @@
-import * as Util from './util';
-import zlib from 'zlib';
-import snappyjs from 'snappyjs';
-let brotli;
-let lzo;
-let lz4js;
-export const PARQUET_COMPRESSION_METHODS = {
-  UNCOMPRESSED: {
-    deflate: deflate_identity,
-    inflate: inflate_identity
-  },
-  GZIP: {
-    deflate: deflate_gzip,
-    inflate: inflate_gzip
-  },
-  SNAPPY: {
-    deflate: deflate_snappy,
-    inflate: inflate_snappy
-  },
-  LZO: {
-    deflate: deflate_lzo,
-    inflate: inflate_lzo
-  },
-  BROTLI: {
-    deflate: deflate_brotli,
-    inflate: inflate_brotli
+import { NoCompression, GZipCompression, SnappyCompression, BrotliCompression, LZOCompression, LZ4Compression, ZstdCompression } from '@loaders.gl/compression';
+import { toArrayBuffer, toBuffer } from './utils/buffer-utils';
+import brotliDecompress from 'brotli/decompress';
+import lz4js from 'lz4js';
+import lzo from 'lzo';
+import { ZstdCodec } from 'zstd-codec';
+const modules = {
+  brotli: {
+    decompress: brotliDecompress,
+    compress: () => {
+      throw new Error('brotli compress');
+    }
   },
-  LZ4: {
-    deflate: deflate_lz4,
-    inflate: inflate_lz4
-  }
+  lz4js,
+  lzo,
+  'zstd-codec': ZstdCodec
 };
-export function deflate(method, value) {
-  if (!(method in PARQUET_COMPRESSION_METHODS)) {
-    throw new Error(`invalid compression method: ${method}`);
-  }
-
-  return PARQUET_COMPRESSION_METHODS[method].deflate(value);
-}
-
-function deflate_identity(value) {
-  return value;
-}
-
-function deflate_gzip(value) {
-  return zlib.gzipSync(value);
-}
-
-function deflate_snappy(value) {
-  return snappyjs.compress(value);
+export const PARQUET_COMPRESSION_METHODS = {
+  UNCOMPRESSED: new NoCompression(),
+  GZIP: new GZipCompression(),
+  SNAPPY: new SnappyCompression(),
+  BROTLI: new BrotliCompression({
+    modules
+  }),
+  LZ4: new LZ4Compression({
+    modules
+  }),
+  LZ4_RAW: new LZ4Compression({
+    modules
+  }),
+  LZO: new LZOCompression({
+    modules
+  }),
+  ZSTD: new ZstdCompression({
+    modules
+  })
+};
+export async function preloadCompressions(options) {
+  const compressions = Object.values(PARQUET_COMPRESSION_METHODS);
+  return await Promise.all(compressions.map(compression => compression.preload()));
 }
+export async function deflate(method, value) {
+  const compression = PARQUET_COMPRESSION_METHODS[method];
 
-function deflate_lzo(value) {
-  lzo = lzo || Util.load('lzo');
-  return lzo.compress(value);
-}
+  if (!compression) {
+    throw new Error(`parquet: invalid compression method: ${method}`);
+  }
 
-function deflate_brotli(value) {
-  brotli = brotli || Util.load('brotli');
-  const result = brotli.compress(value, {
-    mode: 0,
-    quality: 8,
-    lgwin: 22
-  });
-  return result ? Buffer.from(result) : Buffer.alloc(0);
+  const inputArrayBuffer = toArrayBuffer(value);
+  const compressedArrayBuffer = await compression.compress(inputArrayBuffer);
+  return toBuffer(compressedArrayBuffer);
 }
+export async function decompress(method, value, size) {
+  const compression = PARQUET_COMPRESSION_METHODS[method];
 
-function deflate_lz4(value) {
-  lz4js = lz4js || Util.load('lz4js');
-
-  try {
-    return Buffer.from(lz4js.compress(value));
-  } catch (err) {
-    throw err;
+  if (!compression) {
+    throw new Error(`parquet: invalid compression method: ${method}`);
   }
-}
 
+  const inputArrayBuffer = toArrayBuffer(value);
+  const compressedArrayBuffer = await compression.decompress(inputArrayBuffer, size);
+  return toBuffer(compressedArrayBuffer);
+}
 export function inflate(method, value, size) {
   if (!(method in PARQUET_COMPRESSION_METHODS)) {
     throw new Error(`invalid compression method: ${method}`);
@@ -82,41 +68,4 @@ export function inflate(method, value, size) {
 
   return PARQUET_COMPRESSION_METHODS[method].inflate(value, size);
 }
-
-function inflate_identity(value) {
-  return value;
-}
-
-function inflate_gzip(value) {
-  return zlib.gunzipSync(value);
-}
-
-function inflate_snappy(value) {
-  return snappyjs.uncompress(value);
-}
-
-function inflate_lzo(value, size) {
-  lzo = lzo || Util.load('lzo');
-  return lzo.decompress(value, size);
-}
-
-function inflate_lz4(value, size) {
-  lz4js = lz4js || Util.load('lz4js');
-
-  try {
-    return Buffer.from(lz4js.decompress(value, size));
-  } catch (err) {
-    throw err;
-  }
-}
-
-function inflate_brotli(value) {
-  brotli = brotli || Util.load('brotli');
-
-  if (!value.length) {
-    return Buffer.alloc(0);
-  }
-
-  return Buffer.from(brotli.decompress(value));
-}
 //# sourceMappingURL=compression.js.map