@loaders.gl/arrow 3.1.0-beta.5 → 3.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundle.js +5 -15452
- package/dist/dist.min.js +15452 -0
- package/dist/es5/arrow-loader.js +4 -4
- package/dist/es5/arrow-loader.js.map +1 -1
- package/dist/es5/arrow-writer.js +3 -3
- package/dist/es5/arrow-writer.js.map +1 -1
- package/dist/es5/bundle.js +1 -1
- package/dist/es5/bundle.js.map +1 -1
- package/dist/es5/index.js +40 -7
- package/dist/es5/index.js.map +1 -1
- package/dist/es5/lib/arrow-table-batch.js +78 -33
- package/dist/es5/lib/arrow-table-batch.js.map +1 -1
- package/dist/es5/lib/encode-arrow.js +25 -9
- package/dist/es5/lib/encode-arrow.js.map +1 -1
- package/dist/es5/lib/parse-arrow-in-batches.js +172 -12
- package/dist/es5/lib/parse-arrow-in-batches.js.map +1 -1
- package/dist/es5/lib/parse-arrow-sync.js +12 -12
- package/dist/es5/lib/parse-arrow-sync.js.map +1 -1
- package/dist/es5/types.js +1 -1
- package/dist/esm/arrow-loader.js +1 -1
- package/dist/esm/arrow-loader.js.map +1 -1
- package/dist/esm/arrow-writer.js +1 -1
- package/dist/esm/arrow-writer.js.map +1 -1
- package/package.json +6 -6
|
@@ -10,12 +10,12 @@ var _apacheArrow = require("apache-arrow");
|
|
|
10
10
|
function parseArrowSync(arrayBuffer, options) {
|
|
11
11
|
var _options$arrow;
|
|
12
12
|
|
|
13
|
-
|
|
13
|
+
var arrowTable = _apacheArrow.Table.from([new Uint8Array(arrayBuffer)]);
|
|
14
14
|
|
|
15
|
-
|
|
16
|
-
arrowTable.schema.fields.forEach(field
|
|
17
|
-
|
|
18
|
-
|
|
15
|
+
var columnarTable = {};
|
|
16
|
+
arrowTable.schema.fields.forEach(function (field) {
|
|
17
|
+
var arrowColumn = arrowTable.getColumn(field.name);
|
|
18
|
+
var values = arrowColumn.toArray();
|
|
19
19
|
columnarTable[field.name] = values;
|
|
20
20
|
});
|
|
21
21
|
|
|
@@ -33,15 +33,15 @@ function parseArrowSync(arrayBuffer, options) {
|
|
|
33
33
|
}
|
|
34
34
|
|
|
35
35
|
function convertColumnarToRowFormatTable(columnarTable) {
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
36
|
+
var tableKeys = Object.keys(columnarTable);
|
|
37
|
+
var tableRowsCount = columnarTable[tableKeys[0]].length;
|
|
38
|
+
var rowFormatTable = [];
|
|
39
39
|
|
|
40
|
-
for (
|
|
41
|
-
|
|
40
|
+
for (var index = 0; index < tableRowsCount; index++) {
|
|
41
|
+
var tableItem = {};
|
|
42
42
|
|
|
43
|
-
for (
|
|
44
|
-
|
|
43
|
+
for (var keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {
|
|
44
|
+
var fieldName = tableKeys[keyIndex];
|
|
45
45
|
tableItem[fieldName] = columnarTable[fieldName][index];
|
|
46
46
|
}
|
|
47
47
|
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/lib/parse-arrow-sync.ts"],"names":["parseArrowSync","arrayBuffer","options","arrowTable","Table","from","Uint8Array","columnarTable","schema","fields","forEach","field","arrowColumn","getColumn","name","values","toArray","arrow","shape","convertColumnarToRowFormatTable","tableKeys","Object","keys","tableRowsCount","length","rowFormatTable","index","tableItem","keyIndex","fieldName","push"],"mappings":";;;;;;;AACA;;AAGe,SAASA,cAAT,CAAwBC,WAAxB,EAAqCC,OAArC,EAAmE;AAAA;;AAChF,
|
|
1
|
+
{"version":3,"sources":["../../../src/lib/parse-arrow-sync.ts"],"names":["parseArrowSync","arrayBuffer","options","arrowTable","Table","from","Uint8Array","columnarTable","schema","fields","forEach","field","arrowColumn","getColumn","name","values","toArray","arrow","shape","convertColumnarToRowFormatTable","tableKeys","Object","keys","tableRowsCount","length","rowFormatTable","index","tableItem","keyIndex","fieldName","push"],"mappings":";;;;;;;AACA;;AAGe,SAASA,cAAT,CAAwBC,WAAxB,EAAqCC,OAArC,EAAmE;AAAA;;AAChF,MAAMC,UAAU,GAAGC,mBAAMC,IAAN,CAAW,CAAC,IAAIC,UAAJ,CAAeL,WAAf,CAAD,CAAX,CAAnB;;AAMA,MAAMM,aAAa,GAAG,EAAtB;AAEAJ,EAAAA,UAAU,CAACK,MAAX,CAAkBC,MAAlB,CAAyBC,OAAzB,CAAiC,UAACC,KAAD,EAAW;AAE1C,QAAMC,WAAW,GAAGT,UAAU,CAACU,SAAX,CAAqBF,KAAK,CAACG,IAA3B,CAApB;AACA,QAAMC,MAAM,GAAGH,WAAW,CAACI,OAAZ,EAAf;AACAT,IAAAA,aAAa,CAACI,KAAK,CAACG,IAAP,CAAb,GAA4BC,MAA5B;AACD,GALD;;AAOA,UAAQb,OAAR,aAAQA,OAAR,yCAAQA,OAAO,CAAEe,KAAjB,mDAAQ,eAAgBC,KAAxB;AACE,SAAK,aAAL;AACE,aAAOf,UAAP;;AACF,SAAK,kBAAL;AACE,aAAOgB,+BAA+B,CAACZ,aAAD,CAAtC;;AACF,SAAK,gBAAL;AACA;AACE,aAAOA,aAAP;AAPJ;AASD;;AAED,SAASY,+BAAT,CAAyCZ,aAAzC,EAAwD;AACtD,MAAMa,SAAS,GAAGC,MAAM,CAACC,IAAP,CAAYf,aAAZ,CAAlB;AACA,MAAMgB,cAAc,GAAGhB,aAAa,CAACa,SAAS,CAAC,CAAD,CAAV,CAAb,CAA4BI,MAAnD;AACA,MAAMC,cAAoB,GAAG,EAA7B;;AAEA,OAAK,IAAIC,KAAK,GAAG,CAAjB,EAAoBA,KAAK,GAAGH,cAA5B,EAA4CG,KAAK,EAAjD,EAAqD;AACnD,QAAMC,SAAS,GAAG,EAAlB;;AACA,SAAK,IAAIC,QAAQ,GAAG,CAApB,EAAuBA,QAAQ,GAAGR,SAAS,CAACI,MAA5C,EAAoDI,QAAQ,EAA5D,EAAgE;AAC9D,UAAMC,SAAS,GAAGT,SAAS,CAACQ,QAAD,CAA3B;AACAD,MAAAA,SAAS,CAACE,SAAD,CAAT,GAAuBtB,aAAa,CAACsB,SAAD,CAAb,CAAyBH,KAAzB,CAAvB;AACD;;AACDD,IAAAA,cAAc,CAACK,IAAf,CAAoBH,SAApB;AACD;;AACD,SAAOF,cAAP;AACD","sourcesContent":["import type {ArrowLoaderOptions} from '../arrow-loader';\nimport {Table} from 'apache-arrow';\n\n// Parses arrow to a columnar table\nexport default function parseArrowSync(arrayBuffer, options?: ArrowLoaderOptions) {\n const arrowTable = Table.from([new Uint8Array(arrayBuffer)]);\n\n // Extract columns\n\n // TODO - avoid calling `getColumn` on columns we are not interested in?\n // Add options object?\n const columnarTable = {};\n\n arrowTable.schema.fields.forEach((field) => {\n // This (is intended to) coalesce all record batches into a single typed array\n const arrowColumn = arrowTable.getColumn(field.name);\n const values = arrowColumn.toArray();\n columnarTable[field.name] = values;\n });\n\n switch (options?.arrow?.shape) {\n case 'arrow-table':\n return arrowTable;\n case 'object-row-table':\n return convertColumnarToRowFormatTable(columnarTable);\n case 'columnar-table':\n default:\n return columnarTable;\n }\n}\n\nfunction convertColumnarToRowFormatTable(columnarTable) {\n const tableKeys = Object.keys(columnarTable);\n const tableRowsCount = columnarTable[tableKeys[0]].length;\n const rowFormatTable: {}[] = [];\n\n for (let index = 0; index < tableRowsCount; index++) {\n const tableItem = {};\n for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {\n const fieldName = tableKeys[keyIndex];\n tableItem[fieldName] = columnarTable[fieldName][index];\n }\n rowFormatTable.push(tableItem);\n }\n return rowFormatTable;\n}\n"],"file":"parse-arrow-sync.js"}
|
package/dist/es5/types.js
CHANGED
package/dist/esm/arrow-loader.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/arrow-loader.ts"],"names":["VERSION","DEFAULT_ARROW_LOADER_OPTIONS","arrow","shape","ArrowLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","tests","options","_typecheckArrowLoader"],"mappings":"AAIA,MAAMA,OAAO,GAAG,
|
|
1
|
+
{"version":3,"sources":["../../src/arrow-loader.ts"],"names":["VERSION","DEFAULT_ARROW_LOADER_OPTIONS","arrow","shape","ArrowLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","tests","options","_typecheckArrowLoader"],"mappings":"AAIA,MAAMA,OAAO,GAAG,mBAAuB,WAAvB,aAAmD,QAAnE;AAQA,MAAMC,4BAA4B,GAAG;AACnCC,EAAAA,KAAK,EAAE;AACLC,IAAAA,KAAK,EAAE;AADF;AAD4B,CAArC;AAOA,OAAO,MAAMC,WAAW,GAAG;AACzBC,EAAAA,IAAI,EAAE,cADmB;AAEzBC,EAAAA,EAAE,EAAE,OAFqB;AAGzBC,EAAAA,MAAM,EAAE,OAHiB;AAIzBC,EAAAA,OAAO,EAAER,OAJgB;AAKzBS,EAAAA,MAAM,EAAE,IALiB;AAMzBC,EAAAA,QAAQ,EAAE,OANe;AAOzBC,EAAAA,UAAU,EAAE,CAAC,OAAD,EAAU,SAAV,CAPa;AAQzBC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CARc;AASzBC,EAAAA,MAAM,EAAE,IATiB;AAUzBC,EAAAA,KAAK,EAAE,CAAC,OAAD,CAVkB;AAWzBC,EAAAA,OAAO,EAAEd;AAXgB,CAApB;AAcP,OAAO,MAAMe,qBAA6B,GAAGZ,WAAtC","sourcesContent":["import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ArrowLoaderOptions = LoaderOptions & {\n arrow?: {\n shape: 'arrow-table' | 'columnar-table' | 'row-table' | 'array-row-table' | 'object-row-table';\n };\n};\n\nconst DEFAULT_ARROW_LOADER_OPTIONS = {\n arrow: {\n shape: 'columnar-table'\n }\n};\n\n/** ArrowJS table loader */\nexport const ArrowLoader = {\n name: 'Apache Arrow',\n id: 'arrow',\n module: 'arrow',\n version: VERSION,\n worker: true,\n category: 'table',\n extensions: ['arrow', 'feather'],\n mimeTypes: ['application/octet-stream'],\n binary: true,\n tests: ['ARROW'],\n options: DEFAULT_ARROW_LOADER_OPTIONS\n};\n\nexport const _typecheckArrowLoader: Loader = ArrowLoader;\n"],"file":"arrow-loader.js"}
|
package/dist/esm/arrow-writer.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { encodeArrowSync } from './lib/encode-arrow';
|
|
2
|
-
const VERSION = typeof "3.1.
|
|
2
|
+
const VERSION = typeof "3.1.2" !== 'undefined' ? "3.1.2" : 'latest';
|
|
3
3
|
export const ArrowWriter = {
|
|
4
4
|
name: 'Apache Arrow',
|
|
5
5
|
id: 'arrow',
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/arrow-writer.ts"],"names":["encodeArrowSync","VERSION","ArrowWriter","name","id","module","version","extensions","mimeTypes","encodeSync","binary","options","data"],"mappings":"AAEA,SAAQA,eAAR,QAA8B,oBAA9B;AAIA,MAAMC,OAAO,GAAG,
|
|
1
|
+
{"version":3,"sources":["../../src/arrow-writer.ts"],"names":["encodeArrowSync","VERSION","ArrowWriter","name","id","module","version","extensions","mimeTypes","encodeSync","binary","options","data"],"mappings":"AAEA,SAAQA,eAAR,QAA8B,oBAA9B;AAIA,MAAMC,OAAO,GAAG,mBAAuB,WAAvB,aAAmD,QAAnE;AAKA,OAAO,MAAMC,WAAW,GAAG;AACzBC,EAAAA,IAAI,EAAE,cADmB;AAEzBC,EAAAA,EAAE,EAAE,OAFqB;AAGzBC,EAAAA,MAAM,EAAE,OAHiB;AAIzBC,EAAAA,OAAO,EAAEL,OAJgB;AAKzBM,EAAAA,UAAU,EAAE,CAAC,OAAD,EAAU,SAAV,CALa;AAMzBC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CANc;AAOzBC,EAAAA,UAPyB;AAQzBC,EAAAA,MAAM,EAAE,IARiB;AASzBC,EAAAA,OAAO,EAAE;AATgB,CAApB;;AAYP,SAASF,UAAT,CAAoBG,IAApB,EAA0BD,OAA1B,EAAwD;AACtD,SAAOX,eAAe,CAACY,IAAD,CAAtB;AACD","sourcesContent":["// import type {Writer} from '@loaders.gl/loader-utils';\nimport type {WriterOptions} from '@loaders.gl/loader-utils';\nimport {encodeArrowSync} from './lib/encode-arrow';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\ntype ArrowWriterOptions = WriterOptions;\n\n/** Apache Arrow writer */\nexport const ArrowWriter = {\n name: 'Apache Arrow',\n id: 'arrow',\n module: 'arrow',\n version: VERSION,\n extensions: ['arrow', 'feather'],\n mimeTypes: ['application/octet-stream'],\n encodeSync,\n binary: true,\n options: {}\n};\n\nfunction encodeSync(data, options?: ArrowWriterOptions) {\n return encodeArrowSync(data);\n}\n"],"file":"arrow-writer.js"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@loaders.gl/arrow",
|
|
3
|
-
"version": "3.1.
|
|
3
|
+
"version": "3.1.2",
|
|
4
4
|
"description": "Simple columnar table loader for the Apache Arrow format",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"publishConfig": {
|
|
@@ -32,16 +32,16 @@
|
|
|
32
32
|
},
|
|
33
33
|
"scripts": {
|
|
34
34
|
"pre-build": "npm run build-worker && npm run build-bundle",
|
|
35
|
-
"build-bundle": "esbuild src/bundle.ts --bundle --outfile=dist/
|
|
35
|
+
"build-bundle": "esbuild src/bundle.ts --bundle --outfile=dist/dist.min.js",
|
|
36
36
|
"build-worker": "esbuild src/workers/arrow-worker.ts --bundle --outfile=dist/arrow-worker.js"
|
|
37
37
|
},
|
|
38
38
|
"dependencies": {
|
|
39
|
-
"@loaders.gl/loader-utils": "3.1.
|
|
40
|
-
"@loaders.gl/schema": "3.1.
|
|
39
|
+
"@loaders.gl/loader-utils": "3.1.2",
|
|
40
|
+
"@loaders.gl/schema": "3.1.2",
|
|
41
41
|
"apache-arrow": "^4.0.0"
|
|
42
42
|
},
|
|
43
43
|
"peerDependencies": {
|
|
44
|
-
"@loaders.gl/core": "3.1.0
|
|
44
|
+
"@loaders.gl/core": "3.1.0"
|
|
45
45
|
},
|
|
46
|
-
"gitHead": "
|
|
46
|
+
"gitHead": "5c25bb71a2ac8558ecedf2256cc925427b2a0506"
|
|
47
47
|
}
|