@loaders.gl/arrow 3.3.0-alpha.1 → 3.3.0-alpha.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/dist/arrow-loader.d.ts +1 -1
  2. package/dist/arrow-loader.d.ts.map +1 -1
  3. package/dist/arrow-worker.js +1288 -1304
  4. package/dist/arrow-writer.d.ts +1 -1
  5. package/dist/arrow-writer.d.ts.map +1 -1
  6. package/dist/dist.min.js +1289 -1305
  7. package/dist/es5/arrow-loader.js +2 -1
  8. package/dist/es5/arrow-loader.js.map +1 -1
  9. package/dist/es5/arrow-writer.js +1 -3
  10. package/dist/es5/arrow-writer.js.map +1 -1
  11. package/dist/es5/bundle.js +0 -1
  12. package/dist/es5/bundle.js.map +1 -1
  13. package/dist/es5/index.js +8 -25
  14. package/dist/es5/index.js.map +1 -1
  15. package/dist/es5/lib/arrow-table-batch.js +1 -36
  16. package/dist/es5/lib/arrow-table-batch.js.map +1 -1
  17. package/dist/es5/lib/encode-arrow.js +1 -12
  18. package/dist/es5/lib/encode-arrow.js.map +1 -1
  19. package/dist/es5/lib/parse-arrow-in-batches.js +62 -117
  20. package/dist/es5/lib/parse-arrow-in-batches.js.map +1 -1
  21. package/dist/es5/lib/parse-arrow-sync.js +0 -11
  22. package/dist/es5/lib/parse-arrow-sync.js.map +1 -1
  23. package/dist/es5/types.js +0 -1
  24. package/dist/es5/types.js.map +1 -1
  25. package/dist/es5/workers/arrow-worker.js +0 -2
  26. package/dist/es5/workers/arrow-worker.js.map +1 -1
  27. package/dist/esm/arrow-loader.js +3 -1
  28. package/dist/esm/arrow-loader.js.map +1 -1
  29. package/dist/esm/arrow-writer.js +4 -2
  30. package/dist/esm/arrow-writer.js.map +1 -1
  31. package/dist/esm/bundle.js +1 -1
  32. package/dist/esm/bundle.js.map +1 -1
  33. package/dist/esm/index.js +7 -1
  34. package/dist/esm/index.js.map +1 -1
  35. package/dist/esm/lib/arrow-table-batch.js +0 -14
  36. package/dist/esm/lib/arrow-table-batch.js.map +1 -1
  37. package/dist/esm/lib/encode-arrow.js +0 -3
  38. package/dist/esm/lib/encode-arrow.js.map +1 -1
  39. package/dist/esm/lib/parse-arrow-in-batches.js +7 -6
  40. package/dist/esm/lib/parse-arrow-in-batches.js.map +1 -1
  41. package/dist/esm/lib/parse-arrow-sync.js +2 -9
  42. package/dist/esm/lib/parse-arrow-sync.js.map +1 -1
  43. package/dist/esm/types.js +0 -1
  44. package/dist/esm/types.js.map +1 -1
  45. package/dist/esm/workers/arrow-worker.js.map +1 -1
  46. package/dist/lib/encode-arrow.d.ts +1 -1
  47. package/dist/lib/encode-arrow.d.ts.map +1 -1
  48. package/dist/lib/parse-arrow-in-batches.d.ts.map +1 -1
  49. package/dist/lib/parse-arrow-in-batches.js +1 -0
  50. package/dist/types.d.ts +4 -4
  51. package/dist/types.d.ts.map +1 -1
  52. package/package.json +4 -4
  53. package/src/lib/parse-arrow-in-batches.ts +1 -0
@@ -1,28 +1,29 @@
1
+
1
2
  import { RecordBatchReader } from 'apache-arrow';
3
+
2
4
  export function parseArrowInBatches(asyncIterator) {
5
+
3
6
  async function* makeArrowAsyncIterator() {
4
7
  const readers = RecordBatchReader.readAll(asyncIterator);
5
-
6
8
  for await (const reader of readers) {
7
9
  for await (const batch of reader) {
8
10
  yield processBatch(batch);
9
11
  }
10
-
11
12
  break;
12
13
  }
13
14
  }
14
15
 
15
16
  return makeArrowAsyncIterator();
16
17
  }
17
-
18
18
  function processBatch(batch) {
19
19
  const values = {
20
20
  metadata: batch.schema.metadata,
21
21
  length: batch.length
22
22
  };
23
- batch.schema.fields.forEach(({
24
- name
25
- }, index) => {
23
+ batch.schema.fields.forEach((_ref, index) => {
24
+ let {
25
+ name
26
+ } = _ref;
26
27
  values[name] = batch.getChildAt(index).toArray();
27
28
  });
28
29
  return values;
@@ -1 +1 @@
1
- {"version":3,"sources":["../../../src/lib/parse-arrow-in-batches.ts"],"names":["RecordBatchReader","parseArrowInBatches","asyncIterator","makeArrowAsyncIterator","readers","readAll","reader","batch","processBatch","values","metadata","schema","length","fields","forEach","name","index","getChildAt","toArray"],"mappings":"AACA,SAAQA,iBAAR,QAAgC,cAAhC;AAKA,OAAO,SAASC,mBAAT,CACLC,aADK,EAEe;AAmBpB,kBAAgBC,sBAAhB,GAAyC;AACvC,UAAMC,OAAO,GAAGJ,iBAAiB,CAACK,OAAlB,CAA0BH,aAA1B,CAAhB;;AACA,eAAW,MAAMI,MAAjB,IAA2BF,OAA3B,EAAoC;AAClC,iBAAW,MAAMG,KAAjB,IAA0BD,MAA1B,EAAkC;AAChC,cAAME,YAAY,CAACD,KAAD,CAAlB;AACD;;AACD;AACD;AACF;;AACD,SAAOJ,sBAAsB,EAA7B;AACD;;AAED,SAASK,YAAT,CAAsBD,KAAtB,EAA6B;AAC3B,QAAME,MAAM,GAAG;AACbC,IAAAA,QAAQ,EAAEH,KAAK,CAACI,MAAN,CAAaD,QADV;AAEbE,IAAAA,MAAM,EAAEL,KAAK,CAACK;AAFD,GAAf;AAIAL,EAAAA,KAAK,CAACI,MAAN,CAAaE,MAAb,CAAoBC,OAApB,CAA4B,CAAC;AAACC,IAAAA;AAAD,GAAD,EAASC,KAAT,KAAmB;AAC7CP,IAAAA,MAAM,CAACM,IAAD,CAAN,GAAeR,KAAK,CAACU,UAAN,CAAiBD,KAAjB,EAAwBE,OAAxB,EAAf;AACD,GAFD;AAGA,SAAOT,MAAP;AACD","sourcesContent":["// TODO - this import defeats the sophisticated typescript checking in ArrowJS\nimport {RecordBatchReader} from 'apache-arrow';\n// import {isIterable} from '@loaders.gl/core';\n\n/**\n */\nexport function parseArrowInBatches(\n asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>\n): AsyncIterable<any> {\n // Creates the appropriate RecordBatchReader subclasses from the input\n // This will also close the underlying source in case of early termination or errors\n\n // As an optimization, return a non-async iterator\n /*\n if (isIterable(readers)) {\n function* makeArrowIterator() {\n for (const reader of readers) {\n for (const batch of reader) {\n yield processBatch(batch, reader);\n }\n break; // only processing one stream of batches\n }\n }\n const arrowIterator = makeArrowIterator();\n }\n */\n\n async function* makeArrowAsyncIterator() {\n const readers = RecordBatchReader.readAll(asyncIterator);\n for await (const reader of readers) {\n for await (const batch of reader) {\n yield processBatch(batch);\n }\n break; // only processing one stream of batches\n }\n }\n return makeArrowAsyncIterator();\n}\n\nfunction processBatch(batch) {\n const values = {\n metadata: batch.schema.metadata,\n length: batch.length\n };\n batch.schema.fields.forEach(({name}, index) => {\n values[name] = batch.getChildAt(index).toArray();\n });\n return values;\n}\n"],"file":"parse-arrow-in-batches.js"}
1
+ {"version":3,"file":"parse-arrow-in-batches.js","names":["RecordBatchReader","parseArrowInBatches","asyncIterator","makeArrowAsyncIterator","readers","readAll","reader","batch","processBatch","values","metadata","schema","length","fields","forEach","index","name","getChildAt","toArray"],"sources":["../../../src/lib/parse-arrow-in-batches.ts"],"sourcesContent":["// TODO - this import defeats the sophisticated typescript checking in ArrowJS\nimport {RecordBatchReader} from 'apache-arrow';\n// import {isIterable} from '@loaders.gl/core';\n\n/**\n */\nexport function parseArrowInBatches(\n asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>\n): AsyncIterable<any> {\n // Creates the appropriate RecordBatchReader subclasses from the input\n // This will also close the underlying source in case of early termination or errors\n\n // As an optimization, return a non-async iterator\n /*\n if (isIterable(readers)) {\n function* makeArrowIterator() {\n for (const reader of readers) {\n for (const batch of reader) {\n yield processBatch(batch, reader);\n }\n break; // only processing one stream of batches\n }\n }\n const arrowIterator = makeArrowIterator();\n }\n */\n\n async function* makeArrowAsyncIterator() {\n // @ts-expect-error Fixed in arrow 9\n const readers = RecordBatchReader.readAll(asyncIterator);\n for await (const reader of readers) {\n for await (const batch of reader) {\n yield processBatch(batch);\n }\n break; // only processing one stream of batches\n }\n }\n return makeArrowAsyncIterator();\n}\n\nfunction processBatch(batch) {\n const values = {\n metadata: batch.schema.metadata,\n length: batch.length\n };\n batch.schema.fields.forEach(({name}, index) => {\n values[name] = batch.getChildAt(index).toArray();\n });\n return values;\n}\n"],"mappings":";AACA,SAAQA,iBAAiB,QAAO,cAAc;;AAK9C,OAAO,SAASC,mBAAmB,CACjCC,aAAiE,EAC7C;;EAmBpB,gBAAgBC,sBAAsB,GAAG;IAEvC,MAAMC,OAAO,GAAGJ,iBAAiB,CAACK,OAAO,CAACH,aAAa,CAAC;IACxD,WAAW,MAAMI,MAAM,IAAIF,OAAO,EAAE;MAClC,WAAW,MAAMG,KAAK,IAAID,MAAM,EAAE;QAChC,MAAME,YAAY,CAACD,KAAK,CAAC;MAC3B;MACA;IACF;EACF;;EACA,OAAOJ,sBAAsB,EAAE;AACjC;AAEA,SAASK,YAAY,CAACD,KAAK,EAAE;EAC3B,MAAME,MAAM,GAAG;IACbC,QAAQ,EAAEH,KAAK,CAACI,MAAM,CAACD,QAAQ;IAC/BE,MAAM,EAAEL,KAAK,CAACK;EAChB,CAAC;EACDL,KAAK,CAACI,MAAM,CAACE,MAAM,CAACC,OAAO,CAAC,OAASC,KAAK,KAAK;IAAA,IAAlB;MAACC;IAAI,CAAC;IACjCP,MAAM,CAACO,IAAI,CAAC,GAAGT,KAAK,CAACU,UAAU,CAACF,KAAK,CAAC,CAACG,OAAO,EAAE;EAClD,CAAC,CAAC;EACF,OAAOT,MAAM;AACf"}
@@ -1,44 +1,37 @@
1
1
  import { Table } from 'apache-arrow';
2
+
2
3
  export default function parseArrowSync(arrayBuffer, options) {
3
4
  var _options$arrow;
4
-
5
5
  const arrowTable = Table.from([new Uint8Array(arrayBuffer)]);
6
+
6
7
  const columnarTable = {};
7
8
  arrowTable.schema.fields.forEach(field => {
8
9
  const arrowColumn = arrowTable.getColumn(field.name);
9
10
  const values = arrowColumn.toArray();
10
11
  columnarTable[field.name] = values;
11
12
  });
12
-
13
13
  switch (options === null || options === void 0 ? void 0 : (_options$arrow = options.arrow) === null || _options$arrow === void 0 ? void 0 : _options$arrow.shape) {
14
14
  case 'arrow-table':
15
15
  return arrowTable;
16
-
17
16
  case 'object-row-table':
18
17
  return convertColumnarToRowFormatTable(columnarTable);
19
-
20
18
  case 'columnar-table':
21
19
  default:
22
20
  return columnarTable;
23
21
  }
24
22
  }
25
-
26
23
  function convertColumnarToRowFormatTable(columnarTable) {
27
24
  const tableKeys = Object.keys(columnarTable);
28
25
  const tableRowsCount = columnarTable[tableKeys[0]].length;
29
26
  const rowFormatTable = [];
30
-
31
27
  for (let index = 0; index < tableRowsCount; index++) {
32
28
  const tableItem = {};
33
-
34
29
  for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {
35
30
  const fieldName = tableKeys[keyIndex];
36
31
  tableItem[fieldName] = columnarTable[fieldName][index];
37
32
  }
38
-
39
33
  rowFormatTable.push(tableItem);
40
34
  }
41
-
42
35
  return rowFormatTable;
43
36
  }
44
37
  //# sourceMappingURL=parse-arrow-sync.js.map
@@ -1 +1 @@
1
- {"version":3,"sources":["../../../src/lib/parse-arrow-sync.ts"],"names":["Table","parseArrowSync","arrayBuffer","options","arrowTable","from","Uint8Array","columnarTable","schema","fields","forEach","field","arrowColumn","getColumn","name","values","toArray","arrow","shape","convertColumnarToRowFormatTable","tableKeys","Object","keys","tableRowsCount","length","rowFormatTable","index","tableItem","keyIndex","fieldName","push"],"mappings":"AACA,SAAQA,KAAR,QAAoB,cAApB;AAGA,eAAe,SAASC,cAAT,CAAwBC,WAAxB,EAAqCC,OAArC,EAAmE;AAAA;;AAChF,QAAMC,UAAU,GAAGJ,KAAK,CAACK,IAAN,CAAW,CAAC,IAAIC,UAAJ,CAAeJ,WAAf,CAAD,CAAX,CAAnB;AAMA,QAAMK,aAAa,GAAG,EAAtB;AAEAH,EAAAA,UAAU,CAACI,MAAX,CAAkBC,MAAlB,CAAyBC,OAAzB,CAAkCC,KAAD,IAAW;AAE1C,UAAMC,WAAW,GAAGR,UAAU,CAACS,SAAX,CAAqBF,KAAK,CAACG,IAA3B,CAApB;AACA,UAAMC,MAAM,GAAGH,WAAW,CAACI,OAAZ,EAAf;AACAT,IAAAA,aAAa,CAACI,KAAK,CAACG,IAAP,CAAb,GAA4BC,MAA5B;AACD,GALD;;AAOA,UAAQZ,OAAR,aAAQA,OAAR,yCAAQA,OAAO,CAAEc,KAAjB,mDAAQ,eAAgBC,KAAxB;AACE,SAAK,aAAL;AACE,aAAOd,UAAP;;AACF,SAAK,kBAAL;AACE,aAAOe,+BAA+B,CAACZ,aAAD,CAAtC;;AACF,SAAK,gBAAL;AACA;AACE,aAAOA,aAAP;AAPJ;AASD;;AAED,SAASY,+BAAT,CAAyCZ,aAAzC,EAAwD;AACtD,QAAMa,SAAS,GAAGC,MAAM,CAACC,IAAP,CAAYf,aAAZ,CAAlB;AACA,QAAMgB,cAAc,GAAGhB,aAAa,CAACa,SAAS,CAAC,CAAD,CAAV,CAAb,CAA4BI,MAAnD;AACA,QAAMC,cAAoB,GAAG,EAA7B;;AAEA,OAAK,IAAIC,KAAK,GAAG,CAAjB,EAAoBA,KAAK,GAAGH,cAA5B,EAA4CG,KAAK,EAAjD,EAAqD;AACnD,UAAMC,SAAS,GAAG,EAAlB;;AACA,SAAK,IAAIC,QAAQ,GAAG,CAApB,EAAuBA,QAAQ,GAAGR,SAAS,CAACI,MAA5C,EAAoDI,QAAQ,EAA5D,EAAgE;AAC9D,YAAMC,SAAS,GAAGT,SAAS,CAACQ,QAAD,CAA3B;AACAD,MAAAA,SAAS,CAACE,SAAD,CAAT,GAAuBtB,aAAa,CAACsB,SAAD,CAAb,CAAyBH,KAAzB,CAAvB;AACD;;AACDD,IAAAA,cAAc,CAACK,IAAf,CAAoBH,SAApB;AACD;;AACD,SAAOF,cAAP;AACD","sourcesContent":["import type {ArrowLoaderOptions} from '../arrow-loader';\nimport {Table} from 'apache-arrow';\n\n// Parses arrow to a columnar table\nexport default function parseArrowSync(arrayBuffer, options?: ArrowLoaderOptions) {\n const arrowTable = Table.from([new Uint8Array(arrayBuffer)]);\n\n // Extract columns\n\n // TODO - avoid calling `getColumn` on columns we are not interested in?\n // Add options object?\n const columnarTable = {};\n\n arrowTable.schema.fields.forEach((field) => {\n // This (is intended to) coalesce all record batches into a single typed array\n const arrowColumn = arrowTable.getColumn(field.name);\n const values = arrowColumn.toArray();\n columnarTable[field.name] = values;\n });\n\n switch (options?.arrow?.shape) {\n case 'arrow-table':\n return arrowTable;\n case 'object-row-table':\n return convertColumnarToRowFormatTable(columnarTable);\n case 'columnar-table':\n default:\n return columnarTable;\n }\n}\n\nfunction convertColumnarToRowFormatTable(columnarTable) {\n const tableKeys = Object.keys(columnarTable);\n const tableRowsCount = columnarTable[tableKeys[0]].length;\n const rowFormatTable: {}[] = [];\n\n for (let index = 0; index < tableRowsCount; index++) {\n const tableItem = {};\n for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {\n const fieldName = tableKeys[keyIndex];\n tableItem[fieldName] = columnarTable[fieldName][index];\n }\n rowFormatTable.push(tableItem);\n }\n return rowFormatTable;\n}\n"],"file":"parse-arrow-sync.js"}
1
+ {"version":3,"file":"parse-arrow-sync.js","names":["Table","parseArrowSync","arrayBuffer","options","arrowTable","from","Uint8Array","columnarTable","schema","fields","forEach","field","arrowColumn","getColumn","name","values","toArray","arrow","shape","convertColumnarToRowFormatTable","tableKeys","Object","keys","tableRowsCount","length","rowFormatTable","index","tableItem","keyIndex","fieldName","push"],"sources":["../../../src/lib/parse-arrow-sync.ts"],"sourcesContent":["import type {ArrowLoaderOptions} from '../arrow-loader';\nimport {Table} from 'apache-arrow';\n\n// Parses arrow to a columnar table\nexport default function parseArrowSync(arrayBuffer, options?: ArrowLoaderOptions) {\n const arrowTable = Table.from([new Uint8Array(arrayBuffer)]);\n\n // Extract columns\n\n // TODO - avoid calling `getColumn` on columns we are not interested in?\n // Add options object?\n const columnarTable = {};\n\n arrowTable.schema.fields.forEach((field) => {\n // This (is intended to) coalesce all record batches into a single typed array\n const arrowColumn = arrowTable.getColumn(field.name);\n const values = arrowColumn.toArray();\n columnarTable[field.name] = values;\n });\n\n switch (options?.arrow?.shape) {\n case 'arrow-table':\n return arrowTable;\n case 'object-row-table':\n return convertColumnarToRowFormatTable(columnarTable);\n case 'columnar-table':\n default:\n return columnarTable;\n }\n}\n\nfunction convertColumnarToRowFormatTable(columnarTable) {\n const tableKeys = Object.keys(columnarTable);\n const tableRowsCount = columnarTable[tableKeys[0]].length;\n const rowFormatTable: {}[] = [];\n\n for (let index = 0; index < tableRowsCount; index++) {\n const tableItem = {};\n for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {\n const fieldName = tableKeys[keyIndex];\n tableItem[fieldName] = columnarTable[fieldName][index];\n }\n rowFormatTable.push(tableItem);\n }\n return rowFormatTable;\n}\n"],"mappings":"AACA,SAAQA,KAAK,QAAO,cAAc;;AAGlC,eAAe,SAASC,cAAc,CAACC,WAAW,EAAEC,OAA4B,EAAE;EAAA;EAChF,MAAMC,UAAU,GAAGJ,KAAK,CAACK,IAAI,CAAC,CAAC,IAAIC,UAAU,CAACJ,WAAW,CAAC,CAAC,CAAC;;EAM5D,MAAMK,aAAa,GAAG,CAAC,CAAC;EAExBH,UAAU,CAACI,MAAM,CAACC,MAAM,CAACC,OAAO,CAAEC,KAAK,IAAK;IAE1C,MAAMC,WAAW,GAAGR,UAAU,CAACS,SAAS,CAACF,KAAK,CAACG,IAAI,CAAC;IACpD,MAAMC,MAAM,GAAGH,WAAW,CAACI,OAAO,EAAE;IACpCT,aAAa,CAACI,KAAK,CAACG,IAAI,CAAC,GAAGC,MAAM;EACpC,CAAC,CAAC;EAEF,QAAQZ,OAAO,aAAPA,OAAO,yCAAPA,OAAO,CAAEc,KAAK,mDAAd,eAAgBC,KAAK;IAC3B,KAAK,aAAa;MAChB,OAAOd,UAAU;IACnB,KAAK,kBAAkB;MACrB,OAAOe,+BAA+B,CAACZ,aAAa,CAAC;IACvD,KAAK,gBAAgB;IACrB;MACE,OAAOA,aAAa;EAAC;AAE3B;AAEA,SAASY,+BAA+B,CAACZ,aAAa,EAAE;EACtD,MAAMa,SAAS,GAAGC,MAAM,CAACC,IAAI,CAACf,aAAa,CAAC;EAC5C,MAAMgB,cAAc,GAAGhB,aAAa,CAACa,SAAS,CAAC,CAAC,CAAC,CAAC,CAACI,MAAM;EACzD,MAAMC,cAAoB,GAAG,EAAE;EAE/B,KAAK,IAAIC,KAAK,GAAG,CAAC,EAAEA,KAAK,GAAGH,cAAc,EAAEG,KAAK,EAAE,EAAE;IACnD,MAAMC,SAAS,GAAG,CAAC,CAAC;IACpB,KAAK,IAAIC,QAAQ,GAAG,CAAC,EAAEA,QAAQ,GAAGR,SAAS,CAACI,MAAM,EAAEI,QAAQ,EAAE,EAAE;MAC9D,MAAMC,SAAS,GAAGT,SAAS,CAACQ,QAAQ,CAAC;MACrCD,SAAS,CAACE,SAAS,CAAC,GAAGtB,aAAa,CAACsB,SAAS,CAAC,CAACH,KAAK,CAAC;IACxD;IACAD,cAAc,CAACK,IAAI,CAACH,SAAS,CAAC;EAChC;EACA,OAAOF,cAAc;AACvB"}
package/dist/esm/types.js CHANGED
@@ -1,5 +1,4 @@
1
1
  export let VECTOR_TYPES;
2
-
3
2
  (function (VECTOR_TYPES) {
4
3
  VECTOR_TYPES[VECTOR_TYPES["FLOAT"] = 0] = "FLOAT";
5
4
  VECTOR_TYPES[VECTOR_TYPES["DATE"] = 1] = "DATE";
@@ -1 +1 @@
1
- {"version":3,"sources":["../../src/types.ts"],"names":["VECTOR_TYPES"],"mappings":"AAgBA,WAAYA,YAAZ;;WAAYA,Y;AAAAA,EAAAA,Y,CAAAA,Y;AAAAA,EAAAA,Y,CAAAA,Y;GAAAA,Y,KAAAA,Y","sourcesContent":["type TypedIntArray =\n | Int8Array\n | Uint8Array\n | Int16Array\n | Uint16Array\n | Int32Array\n | Uint32Array\n | Int32Array\n | Uint32Array;\n\ntype TypedFloatArray = Float32Array | Float64Array;\n\ntype TypedArray = TypedIntArray | TypedFloatArray;\n\nexport type AnyArrayType = Array<any> | TypedArray;\n\nexport enum VECTOR_TYPES {\n FLOAT,\n DATE\n}\n"],"file":"types.js"}
1
+ {"version":3,"file":"types.js","names":["VECTOR_TYPES"],"sources":["../../src/types.ts"],"sourcesContent":["type TypedIntArray =\n | Int8Array\n | Uint8Array\n | Int16Array\n | Uint16Array\n | Int32Array\n | Uint32Array\n | Int32Array\n | Uint32Array;\n\ntype TypedFloatArray = Float32Array | Float64Array;\n\ntype TypedArray = TypedIntArray | TypedFloatArray;\n\nexport type AnyArrayType = Array<any> | TypedArray;\n\nexport enum VECTOR_TYPES {\n FLOAT,\n DATE\n}\n"],"mappings":"AAgBA,WAAYA,YAAY;AAGvB,WAHWA,YAAY;EAAZA,YAAY,CAAZA,YAAY;EAAZA,YAAY,CAAZA,YAAY;AAAA,GAAZA,YAAY,KAAZA,YAAY"}
@@ -1 +1 @@
1
- {"version":3,"sources":["../../../src/workers/arrow-worker.ts"],"names":["createLoaderWorker","ArrowLoader"],"mappings":"AAAA,SAAQA,kBAAR,QAAiC,0BAAjC;AACA,SAAQC,WAAR,QAA0B,UAA1B;AAEAD,kBAAkB,CAACC,WAAD,CAAlB","sourcesContent":["import {createLoaderWorker} from '@loaders.gl/loader-utils';\nimport {ArrowLoader} from '../index';\n\ncreateLoaderWorker(ArrowLoader);\n"],"file":"arrow-worker.js"}
1
+ {"version":3,"file":"arrow-worker.js","names":["createLoaderWorker","ArrowLoader"],"sources":["../../../src/workers/arrow-worker.ts"],"sourcesContent":["import {createLoaderWorker} from '@loaders.gl/loader-utils';\nimport {ArrowLoader} from '../index';\n\ncreateLoaderWorker(ArrowLoader);\n"],"mappings":"AAAA,SAAQA,kBAAkB,QAAO,0BAA0B;AAC3D,SAAQC,WAAW,QAAO,UAAU;AAEpCD,kBAAkB,CAACC,WAAW,CAAC"}
@@ -1,5 +1,5 @@
1
1
  import { AnyArrayType } from '../types';
2
- declare type ColumnarTable = {
2
+ type ColumnarTable = {
3
3
  name: string;
4
4
  array: AnyArrayType;
5
5
  type: number;
@@ -1 +1 @@
1
- {"version":3,"file":"encode-arrow.d.ts","sourceRoot":"","sources":["../../src/lib/encode-arrow.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,YAAY,EAAe,MAAM,UAAU,CAAC;AAEpD,aAAK,aAAa,GAAG;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,YAAY,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd,EAAE,CAAC;AAEJ;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,aAAa,GAAG,WAAW,CAWhE"}
1
+ {"version":3,"file":"encode-arrow.d.ts","sourceRoot":"","sources":["../../src/lib/encode-arrow.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,YAAY,EAAe,MAAM,UAAU,CAAC;AAEpD,KAAK,aAAa,GAAG;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,YAAY,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd,EAAE,CAAC;AAEJ;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,aAAa,GAAG,WAAW,CAWhE"}
@@ -1 +1 @@
1
- {"version":3,"file":"parse-arrow-in-batches.d.ts","sourceRoot":"","sources":["../../src/lib/parse-arrow-in-batches.ts"],"names":[],"mappings":"AAIA;GACG;AACH,wBAAgB,mBAAmB,CACjC,aAAa,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,GAChE,aAAa,CAAC,GAAG,CAAC,CA6BpB"}
1
+ {"version":3,"file":"parse-arrow-in-batches.d.ts","sourceRoot":"","sources":["../../src/lib/parse-arrow-in-batches.ts"],"names":[],"mappings":"AAIA;GACG;AACH,wBAAgB,mBAAmB,CACjC,aAAa,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,GAChE,aAAa,CAAC,GAAG,CAAC,CA8BpB"}
@@ -24,6 +24,7 @@ function parseArrowInBatches(asyncIterator) {
24
24
  }
25
25
  */
26
26
  async function* makeArrowAsyncIterator() {
27
+ // @ts-expect-error Fixed in arrow 9
27
28
  const readers = apache_arrow_1.RecordBatchReader.readAll(asyncIterator);
28
29
  for await (const reader of readers) {
29
30
  for await (const batch of reader) {
package/dist/types.d.ts CHANGED
@@ -1,7 +1,7 @@
1
- declare type TypedIntArray = Int8Array | Uint8Array | Int16Array | Uint16Array | Int32Array | Uint32Array | Int32Array | Uint32Array;
2
- declare type TypedFloatArray = Float32Array | Float64Array;
3
- declare type TypedArray = TypedIntArray | TypedFloatArray;
4
- export declare type AnyArrayType = Array<any> | TypedArray;
1
+ type TypedIntArray = Int8Array | Uint8Array | Int16Array | Uint16Array | Int32Array | Uint32Array | Int32Array | Uint32Array;
2
+ type TypedFloatArray = Float32Array | Float64Array;
3
+ type TypedArray = TypedIntArray | TypedFloatArray;
4
+ export type AnyArrayType = Array<any> | TypedArray;
5
5
  export declare enum VECTOR_TYPES {
6
6
  FLOAT = 0,
7
7
  DATE = 1
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,aAAK,aAAa,GACd,SAAS,GACT,UAAU,GACV,UAAU,GACV,WAAW,GACX,UAAU,GACV,WAAW,GACX,UAAU,GACV,WAAW,CAAC;AAEhB,aAAK,eAAe,GAAG,YAAY,GAAG,YAAY,CAAC;AAEnD,aAAK,UAAU,GAAG,aAAa,GAAG,eAAe,CAAC;AAElD,oBAAY,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC;AAEnD,oBAAY,YAAY;IACtB,KAAK,IAAA;IACL,IAAI,IAAA;CACL"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,KAAK,aAAa,GACd,SAAS,GACT,UAAU,GACV,UAAU,GACV,WAAW,GACX,UAAU,GACV,WAAW,GACX,UAAU,GACV,WAAW,CAAC;AAEhB,KAAK,eAAe,GAAG,YAAY,GAAG,YAAY,CAAC;AAEnD,KAAK,UAAU,GAAG,aAAa,GAAG,eAAe,CAAC;AAElD,MAAM,MAAM,YAAY,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC;AAEnD,oBAAY,YAAY;IACtB,KAAK,IAAA;IACL,IAAI,IAAA;CACL"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@loaders.gl/arrow",
3
- "version": "3.3.0-alpha.1",
3
+ "version": "3.3.0-alpha.11",
4
4
  "description": "Simple columnar table loader for the Apache Arrow format",
5
5
  "license": "MIT",
6
6
  "publishConfig": {
@@ -36,9 +36,9 @@
36
36
  "build-worker": "esbuild src/workers/arrow-worker.ts --bundle --outfile=dist/arrow-worker.js --define:__VERSION__=\\\"$npm_package_version\\\""
37
37
  },
38
38
  "dependencies": {
39
- "@loaders.gl/loader-utils": "3.3.0-alpha.1",
40
- "@loaders.gl/schema": "3.3.0-alpha.1",
39
+ "@loaders.gl/loader-utils": "3.3.0-alpha.11",
40
+ "@loaders.gl/schema": "3.3.0-alpha.11",
41
41
  "apache-arrow": "^4.0.0"
42
42
  },
43
- "gitHead": "85ed2df7d17dd0d204c8bd8feb44779f70bc693b"
43
+ "gitHead": "1863a8875f63d578c0fdf5bdb3b7d6827f799db9"
44
44
  }
@@ -26,6 +26,7 @@ export function parseArrowInBatches(
26
26
  */
27
27
 
28
28
  async function* makeArrowAsyncIterator() {
29
+ // @ts-expect-error Fixed in arrow 9
29
30
  const readers = RecordBatchReader.readAll(asyncIterator);
30
31
  for await (const reader of readers) {
31
32
  for await (const batch of reader) {