@loaders.gl/arrow 4.0.0-beta.7 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +7 -7
- package/dist/arrow-loader.d.ts.map +1 -1
- package/dist/arrow-loader.js.map +1 -1
- package/dist/arrow-worker.js +1 -1
- package/dist/lib/arrow-table.d.ts.map +1 -1
- package/dist/lib/arrow-table.js.map +1 -1
- package/dist/lib/convert-table.d.ts.map +1 -1
- package/dist/lib/convert-table.js.map +1 -1
- package/dist/schema/arrow-type-utils.d.ts.map +1 -1
- package/dist/schema/arrow-type-utils.js.map +1 -1
- package/dist/schema/convert-schema-arrow.d.ts.map +1 -1
- package/dist/schema/convert-schema-arrow.js.map +1 -1
- package/dist/schema/convert-table-to-arrow.d.ts.map +1 -1
- package/dist/schema/convert-table-to-arrow.js.map +1 -1
- package/package.json +4 -4
- package/src/arrow-loader.ts +1 -0
- package/src/lib/arrow-table.ts +1 -0
- package/src/lib/convert-table.ts +1 -0
- package/src/schema/arrow-type-utils.ts +1 -0
- package/src/schema/convert-schema-arrow.ts +1 -0
- package/src/schema/convert-table-to-arrow.ts +1 -0
package/LICENSE
CHANGED
|
@@ -1,9 +1,6 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
This software includes parts of PhiloGL (https://github.com/philogb/philogl)
|
|
4
|
-
under MIT license. PhiloGL parts Copyright © 2013 Sencha Labs.
|
|
1
|
+
loaders.gl is licensed under the MIT license
|
|
5
2
|
|
|
6
|
-
|
|
3
|
+
Copyright (c) vis.gl contributors
|
|
7
4
|
|
|
8
5
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
9
6
|
of this software and associated documentation files (the "Software"), to deal
|
|
@@ -23,8 +20,12 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
|
23
20
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
24
21
|
THE SOFTWARE.
|
|
25
22
|
|
|
23
|
+
---
|
|
24
|
+
|
|
25
|
+
Copyright (c) 2015 Uber Technologies, Inc.
|
|
26
26
|
|
|
27
|
-
loaders.gl includes certain files from Cesium (https://github.com/AnalyticalGraphicsInc/cesium)
|
|
27
|
+
loaders.gl includes certain files from Cesium (https://github.com/AnalyticalGraphicsInc/cesium)
|
|
28
|
+
under the Apache 2 License (found in the submodule: modules/3d-tiles):)
|
|
28
29
|
|
|
29
30
|
Copyright 2011-2018 CesiumJS Contributors
|
|
30
31
|
|
|
@@ -38,4 +39,3 @@ distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
38
39
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
39
40
|
See the License for the specific language governing permissions and limitations under the License.
|
|
40
41
|
|
|
41
|
-
Cesium-derived code can be found in the submodule: modules/3d-tiles
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"arrow-loader.d.ts","sourceRoot":"","sources":["../src/arrow-loader.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"arrow-loader.d.ts","sourceRoot":"","sources":["../src/arrow-loader.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,MAAM,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAMlD,MAAM,MAAM,kBAAkB,GAAG,aAAa,GAAG;IAC/C,KAAK,CAAC,EAAE;QACN,KAAK,EAAE,aAAa,GAAG,gBAAgB,GAAG,WAAW,GAAG,iBAAiB,GAAG,kBAAkB,CAAC;KAChG,CAAC;CACH,CAAC;AAEF,2BAA2B;AAC3B,eAAO,MAAM,WAAW,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,kBAAkB,CAoBrE,CAAC"}
|
package/dist/arrow-loader.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"arrow-loader.js","names":["VERSION","__VERSION__","ArrowLoader","name","id","module","version","category","extensions","mimeTypes","binary","tests","options","arrow","shape"],"sources":["../src/arrow-loader.ts"],"sourcesContent":["// loaders.gl, MIT license\nimport type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\nimport type {ArrowTable} from './lib/arrow-table';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ArrowLoaderOptions = LoaderOptions & {\n arrow?: {\n shape: 'arrow-table' | 'columnar-table' | 'row-table' | 'array-row-table' | 'object-row-table';\n };\n};\n\n/** ArrowJS table loader */\nexport const ArrowLoader: Loader<ArrowTable, never, ArrowLoaderOptions> = {\n name: 'Apache Arrow',\n id: 'arrow',\n module: 'arrow',\n version: VERSION,\n // worker: true,\n category: 'table',\n extensions: ['arrow', 'feather'],\n mimeTypes: [\n 'application/vnd.apache.arrow.file',\n 'application/vnd.apache.arrow.stream',\n 'application/octet-stream'\n ],\n binary: true,\n tests: ['ARROW'],\n options: {\n arrow: {\n shape: 'columnar-table'\n }\n }\n};\n"],"mappings":"
|
|
1
|
+
{"version":3,"file":"arrow-loader.js","names":["VERSION","__VERSION__","ArrowLoader","name","id","module","version","category","extensions","mimeTypes","binary","tests","options","arrow","shape"],"sources":["../src/arrow-loader.ts"],"sourcesContent":["// loaders.gl, MIT license\n// Copyright (c) vis.gl contributors\nimport type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\nimport type {ArrowTable} from './lib/arrow-table';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ArrowLoaderOptions = LoaderOptions & {\n arrow?: {\n shape: 'arrow-table' | 'columnar-table' | 'row-table' | 'array-row-table' | 'object-row-table';\n };\n};\n\n/** ArrowJS table loader */\nexport const ArrowLoader: Loader<ArrowTable, never, ArrowLoaderOptions> = {\n name: 'Apache Arrow',\n id: 'arrow',\n module: 'arrow',\n version: VERSION,\n // worker: true,\n category: 'table',\n extensions: ['arrow', 'feather'],\n mimeTypes: [\n 'application/vnd.apache.arrow.file',\n 'application/vnd.apache.arrow.stream',\n 'application/octet-stream'\n ],\n binary: true,\n tests: ['ARROW'],\n options: {\n arrow: {\n shape: 'columnar-table'\n }\n }\n};\n"],"mappings":"AAOA,MAAMA,OAAO,GAAG,OAAOC,WAAW,KAAK,WAAW,GAAGA,WAAW,GAAG,QAAQ;AAS3E,OAAO,MAAMC,WAA0D,GAAG;EACxEC,IAAI,EAAE,cAAc;EACpBC,EAAE,EAAE,OAAO;EACXC,MAAM,EAAE,OAAO;EACfC,OAAO,EAAEN,OAAO;EAEhBO,QAAQ,EAAE,OAAO;EACjBC,UAAU,EAAE,CAAC,OAAO,EAAE,SAAS,CAAC;EAChCC,SAAS,EAAE,CACT,mCAAmC,EACnC,qCAAqC,EACrC,0BAA0B,CAC3B;EACDC,MAAM,EAAE,IAAI;EACZC,KAAK,EAAE,CAAC,OAAO,CAAC;EAChBC,OAAO,EAAE;IACPC,KAAK,EAAE;MACLC,KAAK,EAAE;IACT;EACF;AACF,CAAC"}
|
package/dist/arrow-worker.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"arrow-table.d.ts","sourceRoot":"","sources":["../../src/lib/arrow-table.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"arrow-table.d.ts","sourceRoot":"","sources":["../../src/lib/arrow-table.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,KAAK,EAAE,MAAM,EAAC,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAC,KAAK,IAAI,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAE5D;;;GAGG;AACH,MAAM,MAAM,UAAU,GAAG;IACvB,KAAK,EAAE,aAAa,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,gBAAgB,CAAC;CACxB,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,eAAe,GAAG,KAAK,GAAG;IACpC,KAAK,EAAE,aAAa,CAAC;IACrB,UAAU,CAAC,EAAE,UAAU,GAAG,SAAS,CAAC;IACpC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,gBAAgB,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"arrow-table.js","names":[],"sources":["../../src/lib/arrow-table.ts"],"sourcesContent":["// loaders.gl, MIT license\n\nimport type {Batch, Schema} from '@loaders.gl/schema';\nimport type {Table as ApacheArrowTable} from 'apache-arrow';\n\n/**\n * A table organized as an Apache Arrow table\n * @note This is a variant of the type from loaders.gl/schema\n */\nexport type ArrowTable = {\n shape: 'arrow-table';\n schema?: Schema;\n data: ApacheArrowTable;\n};\n\n/**\n * Batch for a table organized as an Apache Arrow table\n * @note This is a variant of the type from loaders.gl/schema\n */\nexport type ArrowTableBatch = Batch & {\n shape: 'arrow-table';\n schemaType?: 'explicit' | 'deduced';\n schema?: Schema;\n data: ApacheArrowTable; // ApacheRecordBatch;\n length: number;\n};\n"],"mappings":""}
|
|
1
|
+
{"version":3,"file":"arrow-table.js","names":[],"sources":["../../src/lib/arrow-table.ts"],"sourcesContent":["// loaders.gl, MIT license\n// Copyright (c) vis.gl contributors\n\nimport type {Batch, Schema} from '@loaders.gl/schema';\nimport type {Table as ApacheArrowTable} from 'apache-arrow';\n\n/**\n * A table organized as an Apache Arrow table\n * @note This is a variant of the type from loaders.gl/schema\n */\nexport type ArrowTable = {\n shape: 'arrow-table';\n schema?: Schema;\n data: ApacheArrowTable;\n};\n\n/**\n * Batch for a table organized as an Apache Arrow table\n * @note This is a variant of the type from loaders.gl/schema\n */\nexport type ArrowTableBatch = Batch & {\n shape: 'arrow-table';\n schemaType?: 'explicit' | 'deduced';\n schema?: Schema;\n data: ApacheArrowTable; // ApacheRecordBatch;\n length: number;\n};\n"],"mappings":""}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"convert-table.d.ts","sourceRoot":"","sources":["../../src/lib/convert-table.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"convert-table.d.ts","sourceRoot":"","sources":["../../src/lib/convert-table.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,aAAa,EAAE,cAAc,EAAC,MAAM,oBAAoB,CAAC;AACtE,OAAO,KAAK,EAAC,KAAK,IAAI,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAC5D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,eAAe,CAAC;AAE9C;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAAC,UAAU,EAAE,gBAAgB,GAAG,UAAU,CAKvF;AAED;;;GAGG;AACH,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,UAAU,GAAG,aAAa,CAkB5E;AAED;;;GAGG;AACH,wBAAgB,+BAA+B,CAAC,aAAa,EAAE,aAAa,GAAG,cAAc,CAmB5F"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"convert-table.js","names":["convertApacheArrowToArrowTable","arrowTable","shape","data","convertArrowToColumnarTable","table","columnarTable","field","schema","fields","arrowColumn","getChild","name","values","toArray","convertColumnarToRowFormatTable","tableKeys","Object","keys","tableRowsCount","length","rowFormatTable","index","tableItem","keyIndex","fieldName","push"],"sources":["../../src/lib/convert-table.ts"],"sourcesContent":["// loaders.gl, MIT license\nimport type {ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';\nimport type {Table as ApacheArrowTable} from 'apache-arrow';\nimport type {ArrowTable} from './arrow-table';\n\n/**\n * Wrap an apache arrow table in a loaders.gl table wrapper.\n * From this additional conversions are available.\n * @param arrowTable\n * @returns\n */\nexport function convertApacheArrowToArrowTable(arrowTable: ApacheArrowTable): ArrowTable {\n return {\n shape: 'arrow-table',\n data: arrowTable\n };\n}\n\n/**\n * Convert an Apache Arrow table to a ColumnarTable\n * @note Currently does not convert schema\n */\nexport function convertArrowToColumnarTable(table: ArrowTable): ColumnarTable {\n // TODO - avoid calling `getColumn` on columns we are not interested in?\n // Add options object?\n\n const arrowTable = table.data;\n const columnarTable = {};\n\n for (const field of arrowTable.schema.fields) {\n // This (is intended to) coalesce all record batches into a single typed array\n const arrowColumn = arrowTable.getChild(field.name);\n const values = arrowColumn?.toArray();\n columnarTable[field.name] = values;\n }\n\n return {\n shape: 'columnar-table',\n data: columnarTable\n };\n}\n\n/**\n *\n * @note - should be part of schema module\n */\nexport function convertColumnarToRowFormatTable(columnarTable: ColumnarTable): ObjectRowTable {\n const tableKeys = Object.keys(columnarTable);\n const tableRowsCount = columnarTable[tableKeys[0]].length;\n\n const rowFormatTable: {}[] = [];\n\n for (let index = 0; index < tableRowsCount; index++) {\n const tableItem = {};\n for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {\n const fieldName = tableKeys[keyIndex];\n tableItem[fieldName] = columnarTable[fieldName][index];\n }\n rowFormatTable.push(tableItem);\n }\n\n return {\n shape: 'object-row-table',\n data: rowFormatTable\n };\n}\n"],"mappings":"
|
|
1
|
+
{"version":3,"file":"convert-table.js","names":["convertApacheArrowToArrowTable","arrowTable","shape","data","convertArrowToColumnarTable","table","columnarTable","field","schema","fields","arrowColumn","getChild","name","values","toArray","convertColumnarToRowFormatTable","tableKeys","Object","keys","tableRowsCount","length","rowFormatTable","index","tableItem","keyIndex","fieldName","push"],"sources":["../../src/lib/convert-table.ts"],"sourcesContent":["// loaders.gl, MIT license\n// Copyright (c) vis.gl contributors\nimport type {ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';\nimport type {Table as ApacheArrowTable} from 'apache-arrow';\nimport type {ArrowTable} from './arrow-table';\n\n/**\n * Wrap an apache arrow table in a loaders.gl table wrapper.\n * From this additional conversions are available.\n * @param arrowTable\n * @returns\n */\nexport function convertApacheArrowToArrowTable(arrowTable: ApacheArrowTable): ArrowTable {\n return {\n shape: 'arrow-table',\n data: arrowTable\n };\n}\n\n/**\n * Convert an Apache Arrow table to a ColumnarTable\n * @note Currently does not convert schema\n */\nexport function convertArrowToColumnarTable(table: ArrowTable): ColumnarTable {\n // TODO - avoid calling `getColumn` on columns we are not interested in?\n // Add options object?\n\n const arrowTable = table.data;\n const columnarTable = {};\n\n for (const field of arrowTable.schema.fields) {\n // This (is intended to) coalesce all record batches into a single typed array\n const arrowColumn = arrowTable.getChild(field.name);\n const values = arrowColumn?.toArray();\n columnarTable[field.name] = values;\n }\n\n return {\n shape: 'columnar-table',\n data: columnarTable\n };\n}\n\n/**\n *\n * @note - should be part of schema module\n */\nexport function convertColumnarToRowFormatTable(columnarTable: ColumnarTable): ObjectRowTable {\n const tableKeys = Object.keys(columnarTable);\n const tableRowsCount = columnarTable[tableKeys[0]].length;\n\n const rowFormatTable: {}[] = [];\n\n for (let index = 0; index < tableRowsCount; index++) {\n const tableItem = {};\n for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {\n const fieldName = tableKeys[keyIndex];\n tableItem[fieldName] = columnarTable[fieldName][index];\n }\n rowFormatTable.push(tableItem);\n }\n\n return {\n shape: 'object-row-table',\n data: rowFormatTable\n };\n}\n"],"mappings":"AAYA,OAAO,SAASA,8BAA8BA,CAACC,UAA4B,EAAc;EACvF,OAAO;IACLC,KAAK,EAAE,aAAa;IACpBC,IAAI,EAAEF;EACR,CAAC;AACH;AAMA,OAAO,SAASG,2BAA2BA,CAACC,KAAiB,EAAiB;EAI5E,MAAMJ,UAAU,GAAGI,KAAK,CAACF,IAAI;EAC7B,MAAMG,aAAa,GAAG,CAAC,CAAC;EAExB,KAAK,MAAMC,KAAK,IAAIN,UAAU,CAACO,MAAM,CAACC,MAAM,EAAE;IAE5C,MAAMC,WAAW,GAAGT,UAAU,CAACU,QAAQ,CAACJ,KAAK,CAACK,IAAI,CAAC;IACnD,MAAMC,MAAM,GAAGH,WAAW,aAAXA,WAAW,uBAAXA,WAAW,CAAEI,OAAO,CAAC,CAAC;IACrCR,aAAa,CAACC,KAAK,CAACK,IAAI,CAAC,GAAGC,MAAM;EACpC;EAEA,OAAO;IACLX,KAAK,EAAE,gBAAgB;IACvBC,IAAI,EAAEG;EACR,CAAC;AACH;AAMA,OAAO,SAASS,+BAA+BA,CAACT,aAA4B,EAAkB;EAC5F,MAAMU,SAAS,GAAGC,MAAM,CAACC,IAAI,CAACZ,aAAa,CAAC;EAC5C,MAAMa,cAAc,GAAGb,aAAa,CAACU,SAAS,CAAC,CAAC,CAAC,CAAC,CAACI,MAAM;EAEzD,MAAMC,cAAoB,GAAG,EAAE;EAE/B,KAAK,IAAIC,KAAK,GAAG,CAAC,EAAEA,KAAK,GAAGH,cAAc,EAAEG,KAAK,EAAE,EAAE;IACnD,MAAMC,SAAS,GAAG,CAAC,CAAC;IACpB,KAAK,IAAIC,QAAQ,GAAG,CAAC,EAAEA,QAAQ,GAAGR,SAAS,CAACI,MAAM,EAAEI,QAAQ,EAAE,EAAE;MAC9D,MAAMC,SAAS,GAAGT,SAAS,CAACQ,QAAQ,CAAC;MACrCD,SAAS,CAACE,SAAS,CAAC,GAAGnB,aAAa,CAACmB,SAAS,CAAC,CAACH,KAAK,CAAC;IACxD;IACAD,cAAc,CAACK,IAAI,CAACH,SAAS,CAAC;EAChC;EAEA,OAAO;IACLrB,KAAK,EAAE,kBAAkB;IACzBC,IAAI,EAAEkB;EACR,CAAC;AACH"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"arrow-type-utils.d.ts","sourceRoot":"","sources":["../../src/schema/arrow-type-utils.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"arrow-type-utils.d.ts","sourceRoot":"","sources":["../../src/schema/arrow-type-utils.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,oBAAoB,CAAC;AACnD,OAAO,EACL,QAAQ,EAiBT,MAAM,cAAc,CAAC;AAGtB,wBAAgB,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,QAAQ,CAqBxD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"arrow-type-utils.js","names":["Float32","Float64","Int16","Int32","Int8","Uint16","Uint32","Uint8","getArrowType","array","constructor","Int8Array","Uint8Array","Int16Array","Uint16Array","Int32Array","Uint32Array","Float32Array","Float64Array","Error"],"sources":["../../src/schema/arrow-type-utils.ts"],"sourcesContent":["// loaders.gl, MIT license\n\nimport type {TypedArray} from '@loaders.gl/schema';\nimport {\n DataType,\n Float32,\n Float64,\n Int16,\n Int32,\n Int8,\n Uint16,\n Uint32,\n Uint8\n // Int8Vector,\n // Uint8Vector,\n // Int16Vector,\n // Uint16Vector,\n // Int32Vector,\n // Uint32Vector,\n // Float32Vector,\n // Float64Vector\n} from 'apache-arrow';\n// import {AbstractVector} from 'apache-arrow/vector';\n\nexport function getArrowType(array: TypedArray): DataType {\n switch (array.constructor) {\n case Int8Array:\n return new Int8();\n case Uint8Array:\n return new Uint8();\n case Int16Array:\n return new Int16();\n case Uint16Array:\n return new Uint16();\n case Int32Array:\n return new Int32();\n case Uint32Array:\n return new Uint32();\n case Float32Array:\n return new Float32();\n case Float64Array:\n return new Float64();\n default:\n throw new Error('array type not supported');\n }\n}\n\n/*\nexport function getArrowVector(array: TypedArray): AbstractVector {\n switch (array.constructor) {\n case Int8Array:\n return Int8Vector.from(array);\n case Uint8Array:\n return Uint8Vector.from(array);\n case Int16Array:\n return Int16Vector.from(array);\n case Uint16Array:\n return Uint16Vector.from(array);\n case Int32Array:\n return Int32Vector.from(array);\n case Uint32Array:\n return Uint32Vector.from(array);\n case Float32Array:\n return Float32Vector.from(array);\n case Float64Array:\n return Float64Vector.from(array);\n default:\n throw new Error('array type not supported');\n }\n}\n*/\n"],"mappings":"
|
|
1
|
+
{"version":3,"file":"arrow-type-utils.js","names":["Float32","Float64","Int16","Int32","Int8","Uint16","Uint32","Uint8","getArrowType","array","constructor","Int8Array","Uint8Array","Int16Array","Uint16Array","Int32Array","Uint32Array","Float32Array","Float64Array","Error"],"sources":["../../src/schema/arrow-type-utils.ts"],"sourcesContent":["// loaders.gl, MIT license\n// Copyright (c) vis.gl contributors\n\nimport type {TypedArray} from '@loaders.gl/schema';\nimport {\n DataType,\n Float32,\n Float64,\n Int16,\n Int32,\n Int8,\n Uint16,\n Uint32,\n Uint8\n // Int8Vector,\n // Uint8Vector,\n // Int16Vector,\n // Uint16Vector,\n // Int32Vector,\n // Uint32Vector,\n // Float32Vector,\n // Float64Vector\n} from 'apache-arrow';\n// import {AbstractVector} from 'apache-arrow/vector';\n\nexport function getArrowType(array: TypedArray): DataType {\n switch (array.constructor) {\n case Int8Array:\n return new Int8();\n case Uint8Array:\n return new Uint8();\n case Int16Array:\n return new Int16();\n case Uint16Array:\n return new Uint16();\n case Int32Array:\n return new Int32();\n case Uint32Array:\n return new Uint32();\n case Float32Array:\n return new Float32();\n case Float64Array:\n return new Float64();\n default:\n throw new Error('array type not supported');\n }\n}\n\n/*\nexport function getArrowVector(array: TypedArray): AbstractVector {\n switch (array.constructor) {\n case Int8Array:\n return Int8Vector.from(array);\n case Uint8Array:\n return Uint8Vector.from(array);\n case Int16Array:\n return Int16Vector.from(array);\n case Uint16Array:\n return Uint16Vector.from(array);\n case Int32Array:\n return Int32Vector.from(array);\n case Uint32Array:\n return Uint32Vector.from(array);\n case Float32Array:\n return Float32Vector.from(array);\n case Float64Array:\n return Float64Vector.from(array);\n default:\n throw new Error('array type not supported');\n }\n}\n*/\n"],"mappings":"AAIA,SAEEA,OAAO,EACPC,OAAO,EACPC,KAAK,EACLC,KAAK,EACLC,IAAI,EACJC,MAAM,EACNC,MAAM,EACNC,KAAK,QASA,cAAc;AAGrB,OAAO,SAASC,YAAYA,CAACC,KAAiB,EAAY;EACxD,QAAQA,KAAK,CAACC,WAAW;IACvB,KAAKC,SAAS;MACZ,OAAO,IAAIP,IAAI,CAAC,CAAC;IACnB,KAAKQ,UAAU;MACb,OAAO,IAAIL,KAAK,CAAC,CAAC;IACpB,KAAKM,UAAU;MACb,OAAO,IAAIX,KAAK,CAAC,CAAC;IACpB,KAAKY,WAAW;MACd,OAAO,IAAIT,MAAM,CAAC,CAAC;IACrB,KAAKU,UAAU;MACb,OAAO,IAAIZ,KAAK,CAAC,CAAC;IACpB,KAAKa,WAAW;MACd,OAAO,IAAIV,MAAM,CAAC,CAAC;IACrB,KAAKW,YAAY;MACf,OAAO,IAAIjB,OAAO,CAAC,CAAC;IACtB,KAAKkB,YAAY;MACf,OAAO,IAAIjB,OAAO,CAAC,CAAC;IACtB;MACE,MAAM,IAAIkB,KAAK,CAAC,0BAA0B,CAAC;EAC/C;AACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"convert-schema-arrow.d.ts","sourceRoot":"","sources":["../../src/schema/convert-schema-arrow.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"convert-schema-arrow.d.ts","sourceRoot":"","sources":["../../src/schema/convert-schema-arrow.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,cAAc,EAAC,MAAM,oBAAoB,CAAC;AAChF,OAAO,EACL,KAAK,IAAI,UAAU,EACnB,MAAM,IAAI,WAAW,EACrB,QAAQ,IAAI,aAAa,EAkC1B,MAAM,cAAc,CAAC;AAEtB,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,WAAW,GAAG,MAAM,CAKrE;AAED,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAKlE;AAED,wBAAgB,sBAAsB,CAAC,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,cAAc,CAEzF;AAED,wBAAgB,wBAAwB,CAAC,QAAQ,CAAC,EAAE,cAAc,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAEvF;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,UAAU,GAAG,KAAK,CAO5D;AAED,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,KAAK,GAAG,UAAU,CAO9D;AAED,+EAA+E;AAE/E,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,aAAa,GAAG,QAAQ,CAqErE;AAED,+EAA+E;AAE/E,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,GAAG,aAAa,CA0EtE"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"convert-schema-arrow.js","names":["Field","ArrowField","Schema","ArrowSchema","Null","Binary","Bool","Int8","Int16","Int32","Int64","Uint8","Uint16","Uint32","Uint64","Float16","Float32","Float64","Utf8","DateDay","DateMillisecond","TimeMillisecond","TimeSecond","TimestampSecond","TimestampMillisecond","TimestampMicrosecond","TimestampNanosecond","IntervalDayTime","IntervalYearMonth","FixedSizeList","Struct","serializeArrowSchema","arrowSchema","fields","map","arrowField","serializeArrowField","metadata","serializeArrowMetadata","deserializeArrowSchema","schema","field","deserializeArrowField","deserializeArrowMetadata","arrowMetadata","Object","fromEntries","Map","entries","name","type","serializeArrowType","nullable","deserializeArrowType","arrowType","constructor","listSize","children","Error","dataType","child"],"sources":["../../src/schema/convert-schema-arrow.ts"],"sourcesContent":["// loaders.gl, MIT license\n\nimport type {DataType, Field, Schema, SchemaMetadata} from '@loaders.gl/schema';\nimport {\n Field as ArrowField,\n Schema as ArrowSchema,\n DataType as ArrowDataType,\n Null,\n Binary,\n Bool,\n // Int,\n Int8,\n Int16,\n Int32,\n Int64,\n Uint8,\n Uint16,\n Uint32,\n Uint64,\n // Float,\n Float16,\n Float32,\n Float64,\n Utf8,\n // Date,\n DateDay,\n DateMillisecond,\n // Time,\n TimeMillisecond,\n TimeSecond,\n // Timestamp,\n TimestampSecond,\n TimestampMillisecond,\n TimestampMicrosecond,\n TimestampNanosecond,\n // Interval,\n IntervalDayTime,\n IntervalYearMonth,\n FixedSizeList,\n Struct\n} from 'apache-arrow';\n\nexport function serializeArrowSchema(arrowSchema: ArrowSchema): Schema {\n return {\n fields: arrowSchema.fields.map((arrowField) => serializeArrowField(arrowField)),\n metadata: serializeArrowMetadata(arrowSchema.metadata)\n };\n}\n\nexport function deserializeArrowSchema(schema: Schema): ArrowSchema {\n return new ArrowSchema(\n schema.fields.map((field) => deserializeArrowField(field)),\n deserializeArrowMetadata(schema.metadata)\n );\n}\n\nexport function serializeArrowMetadata(arrowMetadata: Map<string, string>): SchemaMetadata {\n return Object.fromEntries(arrowMetadata);\n}\n\nexport function deserializeArrowMetadata(metadata?: SchemaMetadata): Map<string, string> {\n return metadata ? new Map(Object.entries(metadata)) : new Map<string, string>();\n}\n\nexport function serializeArrowField(field: ArrowField): Field {\n return {\n name: field.name,\n type: serializeArrowType(field.type),\n nullable: field.nullable,\n metadata: serializeArrowMetadata(field.metadata)\n };\n}\n\nexport function deserializeArrowField(field: Field): ArrowField {\n return new ArrowField(\n field.name,\n deserializeArrowType(field.type),\n field.nullable,\n deserializeArrowMetadata(field.metadata)\n );\n}\n\n/** Converts a serializable loaders.gl data type to hydrated arrow data type */\n// eslint-disable-next-line complexity\nexport function serializeArrowType(arrowType: ArrowDataType): DataType {\n switch (arrowType.constructor) {\n case Null:\n return 'null';\n case Binary:\n return 'binary';\n case Bool:\n return 'bool';\n // case Int: return 'int';\n case Int8:\n return 'int8';\n case Int16:\n return 'int16';\n case Int32:\n return 'int32';\n case Int64:\n return 'int64';\n case Uint8:\n return 'uint8';\n case Uint16:\n return 'uint16';\n case Uint32:\n return 'uint32';\n case Uint64:\n return 'uint64';\n // case Float: return 'float';\n case Float16:\n return 'float16';\n case Float32:\n return 'float32';\n case Float64:\n return 'float64';\n case Utf8:\n return 'utf8';\n // case Date: return 'date';\n case DateDay:\n return 'date-day';\n case DateMillisecond:\n return 'date-millisecond';\n // case Time: return 'time';\n case TimeMillisecond:\n return 'time-millisecond';\n case TimeSecond:\n return 'time-second';\n // case Timestamp: return 'timestamp';\n case TimestampSecond:\n return 'timestamp-second';\n case TimestampMillisecond:\n return 'timestamp-millisecond';\n case TimestampMicrosecond:\n return 'timestamp-microsecond';\n case TimestampNanosecond:\n return 'timestamp-nanosecond';\n // case Interval: return 'interval';\n case IntervalDayTime:\n return 'interval-daytime';\n case IntervalYearMonth:\n return 'interval-yearmonth';\n case FixedSizeList:\n return {\n type: 'fixed-size-list',\n listSize: (arrowType as FixedSizeList).listSize,\n children: [serializeArrowField((arrowType as FixedSizeList).children[0])]\n };\n // case Struct:\n // return {type: 'struct', children: (arrowType as Struct).children};\n default:\n throw new Error('array type not supported');\n }\n}\n\n/** Converts a serializable loaders.gl data type to hydrated arrow data type */\n// eslint-disable-next-line complexity\nexport function deserializeArrowType(dataType: DataType): ArrowDataType {\n if (typeof dataType === 'object') {\n switch (dataType.type) {\n case 'fixed-size-list':\n const child = deserializeArrowField(dataType.children[0]);\n return new FixedSizeList(dataType.listSize, child);\n case 'struct':\n const children = dataType.children.map((arrowField) => deserializeArrowField(arrowField));\n return new Struct(children);\n default:\n throw new Error('array type not supported');\n }\n }\n\n switch (dataType) {\n case 'null':\n return new Null();\n case 'binary':\n return new Binary();\n case 'bool':\n return new Bool();\n // case 'int': return new Int();\n case 'int8':\n return new Int8();\n case 'int16':\n return new Int16();\n case 'int32':\n return new Int32();\n case 'int64':\n return new Int64();\n case 'uint8':\n return new Uint8();\n case 'uint16':\n return new Uint16();\n case 'uint32':\n return new Uint32();\n case 'uint64':\n return new Uint64();\n // case 'float': return new Float();\n case 'float16':\n return new Float16();\n case 'float32':\n return new Float32();\n case 'float64':\n return new Float64();\n case 'utf8':\n return new Utf8();\n // case 'date': return new Date();\n case 'date-day':\n return new DateDay();\n case 'date-millisecond':\n return new DateMillisecond();\n // case 'time': return new Time();\n case 'time-millisecond':\n return new TimeMillisecond();\n case 'time-second':\n return new TimeSecond();\n // case 'timestamp': return new Timestamp();\n case 'timestamp-second':\n return new TimestampSecond();\n case 'timestamp-millisecond':\n return new TimestampMillisecond();\n case 'timestamp-microsecond':\n return new TimestampMicrosecond();\n case 'timestamp-nanosecond':\n return new TimestampNanosecond();\n // case 'interval': return new Interval();\n case 'interval-daytime':\n return new IntervalDayTime();\n case 'interval-yearmonth':\n return new IntervalYearMonth();\n default:\n throw new Error('array type not supported');\n }\n}\n"],"mappings":"AAGA,SACEA,KAAK,IAAIC,UAAU,EACnBC,MAAM,IAAIC,WAAW,EAErBC,IAAI,EACJC,MAAM,EACNC,IAAI,EAEJC,IAAI,EACJC,KAAK,EACLC,KAAK,EACLC,KAAK,EACLC,KAAK,EACLC,MAAM,EACNC,MAAM,EACNC,MAAM,EAENC,OAAO,EACPC,OAAO,EACPC,OAAO,EACPC,IAAI,EAEJC,OAAO,EACPC,eAAe,EAEfC,eAAe,EACfC,UAAU,EAEVC,eAAe,EACfC,oBAAoB,EACpBC,oBAAoB,EACpBC,mBAAmB,EAEnBC,eAAe,EACfC,iBAAiB,EACjBC,aAAa,EACbC,MAAM,QACD,cAAc;AAErB,OAAO,SAASC,oBAAoBA,CAACC,WAAwB,EAAU;EACrE,OAAO;IACLC,MAAM,EAAED,WAAW,CAACC,MAAM,CAACC,GAAG,CAAEC,UAAU,IAAKC,mBAAmB,CAACD,UAAU,CAAC,CAAC;IAC/EE,QAAQ,EAAEC,sBAAsB,CAACN,WAAW,CAACK,QAAQ;EACvD,CAAC;AACH;AAEA,OAAO,SAASE,sBAAsBA,CAACC,MAAc,EAAe;EAClE,OAAO,IAAIrC,WAAW,CACpBqC,MAAM,CAACP,MAAM,CAACC,GAAG,CAAEO,KAAK,IAAKC,qBAAqB,CAACD,KAAK,CAAC,CAAC,EAC1DE,wBAAwB,CAACH,MAAM,CAACH,QAAQ,CAC1C,CAAC;AACH;AAEA,OAAO,SAASC,sBAAsBA,CAACM,aAAkC,EAAkB;EACzF,OAAOC,MAAM,CAACC,WAAW,CAACF,aAAa,CAAC;AAC1C;AAEA,OAAO,SAASD,wBAAwBA,CAACN,QAAyB,EAAuB;EACvF,OAAOA,QAAQ,GAAG,IAAIU,GAAG,CAACF,MAAM,CAACG,OAAO,CAACX,QAAQ,CAAC,CAAC,GAAG,IAAIU,GAAG,CAAiB,CAAC;AACjF;AAEA,OAAO,SAASX,mBAAmBA,CAACK,KAAiB,EAAS;EAC5D,OAAO;IACLQ,IAAI,EAAER,KAAK,CAACQ,IAAI;IAChBC,IAAI,EAAEC,kBAAkB,CAACV,KAAK,CAACS,IAAI,CAAC;IACpCE,QAAQ,EAAEX,KAAK,CAACW,QAAQ;IACxBf,QAAQ,EAAEC,sBAAsB,CAACG,KAAK,CAACJ,QAAQ;EACjD,CAAC;AACH;AAEA,OAAO,SAASK,qBAAqBA,CAACD,KAAY,EAAc;EAC9D,OAAO,IAAIxC,UAAU,CACnBwC,KAAK,CAACQ,IAAI,EACVI,oBAAoB,CAACZ,KAAK,CAACS,IAAI,CAAC,EAChCT,KAAK,CAACW,QAAQ,EACdT,wBAAwB,CAACF,KAAK,CAACJ,QAAQ,CACzC,CAAC;AACH;AAIA,OAAO,SAASc,kBAAkBA,CAACG,SAAwB,EAAY;EACrE,QAAQA,SAAS,CAACC,WAAW;IAC3B,KAAKnD,IAAI;MACP,OAAO,MAAM;IACf,KAAKC,MAAM;MACT,OAAO,QAAQ;IACjB,KAAKC,IAAI;MACP,OAAO,MAAM;IAEf,KAAKC,IAAI;MACP,OAAO,MAAM;IACf,KAAKC,KAAK;MACR,OAAO,OAAO;IAChB,KAAKC,KAAK;MACR,OAAO,OAAO;IAChB,KAAKC,KAAK;MACR,OAAO,OAAO;IAChB,KAAKC,KAAK;MACR,OAAO,OAAO;IAChB,KAAKC,MAAM;MACT,OAAO,QAAQ;IACjB,KAAKC,MAAM;MACT,OAAO,QAAQ;IACjB,KAAKC,MAAM;MACT,OAAO,QAAQ;IAEjB,KAAKC,OAAO;MACV,OAAO,SAAS;IAClB,KAAKC,OAAO;MACV,OAAO,SAAS;IAClB,KAAKC,OAAO;MACV,OAAO,SAAS;IAClB,KAAKC,IAAI;MACP,OAAO,MAAM;IAEf,KAAKC,OAAO;MACV,OAAO,UAAU;IACnB,KAAKC,eAAe;MAClB,OAAO,kBAAkB;IAE3B,KAAKC,eAAe;MAClB,OAAO,kBAAkB;IAC3B,KAAKC,UAAU;MACb,OAAO,aAAa;IAEtB,KAAKC,eAAe;MAClB,OAAO,kBAAkB;IAC3B,KAAKC,oBAAoB;MACvB,OAAO,uBAAuB;IAChC,KAAKC,oBAAoB;MACvB,OAAO,uBAAuB;IAChC,KAAKC,mBAAmB;MACtB,OAAO,sBAAsB;IAE/B,KAAKC,eAAe;MAClB,OAAO,kBAAkB;IAC3B,KAAKC,iBAAiB;MACpB,OAAO,oBAAoB;IAC7B,KAAKC,aAAa;MAChB,OAAO;QACLqB,IAAI,EAAE,iBAAiB;QACvBM,QAAQ,EAAGF,SAAS,CAAmBE,QAAQ;QAC/CC,QAAQ,EAAE,CAACrB,mBAAmB,CAAEkB,SAAS,CAAmBG,QAAQ,CAAC,CAAC,CAAC,CAAC;MAC1E,CAAC;IAGH;MACE,MAAM,IAAIC,KAAK,CAAC,0BAA0B,CAAC;EAC/C;AACF;AAIA,OAAO,SAASL,oBAAoBA,CAACM,QAAkB,EAAiB;EACtE,IAAI,OAAOA,QAAQ,KAAK,QAAQ,EAAE;IAChC,QAAQA,QAAQ,CAACT,IAAI;MACnB,KAAK,iBAAiB;QACpB,MAAMU,KAAK,GAAGlB,qBAAqB,CAACiB,QAAQ,CAACF,QAAQ,CAAC,CAAC,CAAC,CAAC;QACzD,OAAO,IAAI5B,aAAa,CAAC8B,QAAQ,CAACH,QAAQ,EAAEI,KAAK,CAAC;MACpD,KAAK,QAAQ;QACX,MAAMH,QAAQ,GAAGE,QAAQ,CAACF,QAAQ,CAACvB,GAAG,CAAEC,UAAU,IAAKO,qBAAqB,CAACP,UAAU,CAAC,CAAC;QACzF,OAAO,IAAIL,MAAM,CAAC2B,QAAQ,CAAC;MAC7B;QACE,MAAM,IAAIC,KAAK,CAAC,0BAA0B,CAAC;IAC/C;EACF;EAEA,QAAQC,QAAQ;IACd,KAAK,MAAM;MACT,OAAO,IAAIvD,IAAI,CAAC,CAAC;IACnB,KAAK,QAAQ;MACX,OAAO,IAAIC,MAAM,CAAC,CAAC;IACrB,KAAK,MAAM;MACT,OAAO,IAAIC,IAAI,CAAC,CAAC;IAEnB,KAAK,MAAM;MACT,OAAO,IAAIC,IAAI,CAAC,CAAC;IACnB,KAAK,OAAO;MACV,OAAO,IAAIC,KAAK,CAAC,CAAC;IACpB,KAAK,OAAO;MACV,OAAO,IAAIC,KAAK,CAAC,CAAC;IACpB,KAAK,OAAO;MACV,OAAO,IAAIC,KAAK,CAAC,CAAC;IACpB,KAAK,OAAO;MACV,OAAO,IAAIC,KAAK,CAAC,CAAC;IACpB,KAAK,QAAQ;MACX,OAAO,IAAIC,MAAM,CAAC,CAAC;IACrB,KAAK,QAAQ;MACX,OAAO,IAAIC,MAAM,CAAC,CAAC;IACrB,KAAK,QAAQ;MACX,OAAO,IAAIC,MAAM,CAAC,CAAC;IAErB,KAAK,SAAS;MACZ,OAAO,IAAIC,OAAO,CAAC,CAAC;IACtB,KAAK,SAAS;MACZ,OAAO,IAAIC,OAAO,CAAC,CAAC;IACtB,KAAK,SAAS;MACZ,OAAO,IAAIC,OAAO,CAAC,CAAC;IACtB,KAAK,MAAM;MACT,OAAO,IAAIC,IAAI,CAAC,CAAC;IAEnB,KAAK,UAAU;MACb,OAAO,IAAIC,OAAO,CAAC,CAAC;IACtB,KAAK,kBAAkB;MACrB,OAAO,IAAIC,eAAe,CAAC,CAAC;IAE9B,KAAK,kBAAkB;MACrB,OAAO,IAAIC,eAAe,CAAC,CAAC;IAC9B,KAAK,aAAa;MAChB,OAAO,IAAIC,UAAU,CAAC,CAAC;IAEzB,KAAK,kBAAkB;MACrB,OAAO,IAAIC,eAAe,CAAC,CAAC;IAC9B,KAAK,uBAAuB;MAC1B,OAAO,IAAIC,oBAAoB,CAAC,CAAC;IACnC,KAAK,uBAAuB;MAC1B,OAAO,IAAIC,oBAAoB,CAAC,CAAC;IACnC,KAAK,sBAAsB;MACzB,OAAO,IAAIC,mBAAmB,CAAC,CAAC;IAElC,KAAK,kBAAkB;MACrB,OAAO,IAAIC,eAAe,CAAC,CAAC;IAC9B,KAAK,oBAAoB;MACvB,OAAO,IAAIC,iBAAiB,CAAC,CAAC;IAChC;MACE,MAAM,IAAI8B,KAAK,CAAC,0BAA0B,CAAC;EAC/C;AACF"}
|
|
1
|
+
{"version":3,"file":"convert-schema-arrow.js","names":["Field","ArrowField","Schema","ArrowSchema","Null","Binary","Bool","Int8","Int16","Int32","Int64","Uint8","Uint16","Uint32","Uint64","Float16","Float32","Float64","Utf8","DateDay","DateMillisecond","TimeMillisecond","TimeSecond","TimestampSecond","TimestampMillisecond","TimestampMicrosecond","TimestampNanosecond","IntervalDayTime","IntervalYearMonth","FixedSizeList","Struct","serializeArrowSchema","arrowSchema","fields","map","arrowField","serializeArrowField","metadata","serializeArrowMetadata","deserializeArrowSchema","schema","field","deserializeArrowField","deserializeArrowMetadata","arrowMetadata","Object","fromEntries","Map","entries","name","type","serializeArrowType","nullable","deserializeArrowType","arrowType","constructor","listSize","children","Error","dataType","child"],"sources":["../../src/schema/convert-schema-arrow.ts"],"sourcesContent":["// loaders.gl, MIT license\n// Copyright (c) vis.gl contributors\n\nimport type {DataType, Field, Schema, SchemaMetadata} from '@loaders.gl/schema';\nimport {\n Field as ArrowField,\n Schema as ArrowSchema,\n DataType as ArrowDataType,\n Null,\n Binary,\n Bool,\n // Int,\n Int8,\n Int16,\n Int32,\n Int64,\n Uint8,\n Uint16,\n Uint32,\n Uint64,\n // Float,\n Float16,\n Float32,\n Float64,\n Utf8,\n // Date,\n DateDay,\n DateMillisecond,\n // Time,\n TimeMillisecond,\n TimeSecond,\n // Timestamp,\n TimestampSecond,\n TimestampMillisecond,\n TimestampMicrosecond,\n TimestampNanosecond,\n // Interval,\n IntervalDayTime,\n IntervalYearMonth,\n FixedSizeList,\n Struct\n} from 'apache-arrow';\n\nexport function serializeArrowSchema(arrowSchema: ArrowSchema): Schema {\n return {\n fields: arrowSchema.fields.map((arrowField) => serializeArrowField(arrowField)),\n metadata: serializeArrowMetadata(arrowSchema.metadata)\n };\n}\n\nexport function deserializeArrowSchema(schema: Schema): ArrowSchema {\n return new ArrowSchema(\n schema.fields.map((field) => deserializeArrowField(field)),\n deserializeArrowMetadata(schema.metadata)\n );\n}\n\nexport function serializeArrowMetadata(arrowMetadata: Map<string, string>): SchemaMetadata {\n return Object.fromEntries(arrowMetadata);\n}\n\nexport function deserializeArrowMetadata(metadata?: SchemaMetadata): Map<string, string> {\n return metadata ? new Map(Object.entries(metadata)) : new Map<string, string>();\n}\n\nexport function serializeArrowField(field: ArrowField): Field {\n return {\n name: field.name,\n type: serializeArrowType(field.type),\n nullable: field.nullable,\n metadata: serializeArrowMetadata(field.metadata)\n };\n}\n\nexport function deserializeArrowField(field: Field): ArrowField {\n return new ArrowField(\n field.name,\n deserializeArrowType(field.type),\n field.nullable,\n deserializeArrowMetadata(field.metadata)\n );\n}\n\n/** Converts a serializable loaders.gl data type to hydrated arrow data type */\n// eslint-disable-next-line complexity\nexport function serializeArrowType(arrowType: ArrowDataType): DataType {\n switch (arrowType.constructor) {\n case Null:\n return 'null';\n case Binary:\n return 'binary';\n case Bool:\n return 'bool';\n // case Int: return 'int';\n case Int8:\n return 'int8';\n case Int16:\n return 'int16';\n case Int32:\n return 'int32';\n case Int64:\n return 'int64';\n case Uint8:\n return 'uint8';\n case Uint16:\n return 'uint16';\n case Uint32:\n return 'uint32';\n case Uint64:\n return 'uint64';\n // case Float: return 'float';\n case Float16:\n return 'float16';\n case Float32:\n return 'float32';\n case Float64:\n return 'float64';\n case Utf8:\n return 'utf8';\n // case Date: return 'date';\n case DateDay:\n return 'date-day';\n case DateMillisecond:\n return 'date-millisecond';\n // case Time: return 'time';\n case TimeMillisecond:\n return 'time-millisecond';\n case TimeSecond:\n return 'time-second';\n // case Timestamp: return 'timestamp';\n case TimestampSecond:\n return 'timestamp-second';\n case TimestampMillisecond:\n return 'timestamp-millisecond';\n case TimestampMicrosecond:\n return 'timestamp-microsecond';\n case TimestampNanosecond:\n return 'timestamp-nanosecond';\n // case Interval: return 'interval';\n case IntervalDayTime:\n return 'interval-daytime';\n case IntervalYearMonth:\n return 'interval-yearmonth';\n case FixedSizeList:\n return {\n type: 'fixed-size-list',\n listSize: (arrowType as FixedSizeList).listSize,\n children: [serializeArrowField((arrowType as FixedSizeList).children[0])]\n };\n // case Struct:\n // return {type: 'struct', children: (arrowType as Struct).children};\n default:\n throw new Error('array type not supported');\n }\n}\n\n/** Converts a serializable loaders.gl data type to hydrated arrow data type */\n// eslint-disable-next-line complexity\nexport function deserializeArrowType(dataType: DataType): ArrowDataType {\n if (typeof dataType === 'object') {\n switch (dataType.type) {\n case 'fixed-size-list':\n const child = deserializeArrowField(dataType.children[0]);\n return new FixedSizeList(dataType.listSize, child);\n case 'struct':\n const children = dataType.children.map((arrowField) => deserializeArrowField(arrowField));\n return new Struct(children);\n default:\n throw new Error('array type not supported');\n }\n }\n\n switch (dataType) {\n case 'null':\n return new Null();\n case 'binary':\n return new Binary();\n case 'bool':\n return new Bool();\n // case 'int': return new Int();\n case 'int8':\n return new Int8();\n case 'int16':\n return new Int16();\n case 'int32':\n return new Int32();\n case 'int64':\n return new Int64();\n case 'uint8':\n return new Uint8();\n case 'uint16':\n return new Uint16();\n case 'uint32':\n return new Uint32();\n case 'uint64':\n return new Uint64();\n // case 'float': return new Float();\n case 'float16':\n return new Float16();\n case 'float32':\n return new Float32();\n case 'float64':\n return new Float64();\n case 'utf8':\n return new Utf8();\n // case 'date': return new Date();\n case 'date-day':\n return new DateDay();\n case 'date-millisecond':\n return new DateMillisecond();\n // case 'time': return new Time();\n case 'time-millisecond':\n return new TimeMillisecond();\n case 'time-second':\n return new TimeSecond();\n // case 'timestamp': return new Timestamp();\n case 'timestamp-second':\n return new TimestampSecond();\n case 'timestamp-millisecond':\n return new TimestampMillisecond();\n case 'timestamp-microsecond':\n return new TimestampMicrosecond();\n case 'timestamp-nanosecond':\n return new TimestampNanosecond();\n // case 'interval': return new Interval();\n case 'interval-daytime':\n return new IntervalDayTime();\n case 'interval-yearmonth':\n return new IntervalYearMonth();\n default:\n throw new Error('array type not supported');\n }\n}\n"],"mappings":"AAIA,SACEA,KAAK,IAAIC,UAAU,EACnBC,MAAM,IAAIC,WAAW,EAErBC,IAAI,EACJC,MAAM,EACNC,IAAI,EAEJC,IAAI,EACJC,KAAK,EACLC,KAAK,EACLC,KAAK,EACLC,KAAK,EACLC,MAAM,EACNC,MAAM,EACNC,MAAM,EAENC,OAAO,EACPC,OAAO,EACPC,OAAO,EACPC,IAAI,EAEJC,OAAO,EACPC,eAAe,EAEfC,eAAe,EACfC,UAAU,EAEVC,eAAe,EACfC,oBAAoB,EACpBC,oBAAoB,EACpBC,mBAAmB,EAEnBC,eAAe,EACfC,iBAAiB,EACjBC,aAAa,EACbC,MAAM,QACD,cAAc;AAErB,OAAO,SAASC,oBAAoBA,CAACC,WAAwB,EAAU;EACrE,OAAO;IACLC,MAAM,EAAED,WAAW,CAACC,MAAM,CAACC,GAAG,CAAEC,UAAU,IAAKC,mBAAmB,CAACD,UAAU,CAAC,CAAC;IAC/EE,QAAQ,EAAEC,sBAAsB,CAACN,WAAW,CAACK,QAAQ;EACvD,CAAC;AACH;AAEA,OAAO,SAASE,sBAAsBA,CAACC,MAAc,EAAe;EAClE,OAAO,IAAIrC,WAAW,CACpBqC,MAAM,CAACP,MAAM,CAACC,GAAG,CAAEO,KAAK,IAAKC,qBAAqB,CAACD,KAAK,CAAC,CAAC,EAC1DE,wBAAwB,CAACH,MAAM,CAACH,QAAQ,CAC1C,CAAC;AACH;AAEA,OAAO,SAASC,sBAAsBA,CAACM,aAAkC,EAAkB;EACzF,OAAOC,MAAM,CAACC,WAAW,CAACF,aAAa,CAAC;AAC1C;AAEA,OAAO,SAASD,wBAAwBA,CAACN,QAAyB,EAAuB;EACvF,OAAOA,QAAQ,GAAG,IAAIU,GAAG,CAACF,MAAM,CAACG,OAAO,CAACX,QAAQ,CAAC,CAAC,GAAG,IAAIU,GAAG,CAAiB,CAAC;AACjF;AAEA,OAAO,SAASX,mBAAmBA,CAACK,KAAiB,EAAS;EAC5D,OAAO;IACLQ,IAAI,EAAER,KAAK,CAACQ,IAAI;IAChBC,IAAI,EAAEC,kBAAkB,CAACV,KAAK,CAACS,IAAI,CAAC;IACpCE,QAAQ,EAAEX,KAAK,CAACW,QAAQ;IACxBf,QAAQ,EAAEC,sBAAsB,CAACG,KAAK,CAACJ,QAAQ;EACjD,CAAC;AACH;AAEA,OAAO,SAASK,qBAAqBA,CAACD,KAAY,EAAc;EAC9D,OAAO,IAAIxC,UAAU,CACnBwC,KAAK,CAACQ,IAAI,EACVI,oBAAoB,CAACZ,KAAK,CAACS,IAAI,CAAC,EAChCT,KAAK,CAACW,QAAQ,EACdT,wBAAwB,CAACF,KAAK,CAACJ,QAAQ,CACzC,CAAC;AACH;AAIA,OAAO,SAASc,kBAAkBA,CAACG,SAAwB,EAAY;EACrE,QAAQA,SAAS,CAACC,WAAW;IAC3B,KAAKnD,IAAI;MACP,OAAO,MAAM;IACf,KAAKC,MAAM;MACT,OAAO,QAAQ;IACjB,KAAKC,IAAI;MACP,OAAO,MAAM;IAEf,KAAKC,IAAI;MACP,OAAO,MAAM;IACf,KAAKC,KAAK;MACR,OAAO,OAAO;IAChB,KAAKC,KAAK;MACR,OAAO,OAAO;IAChB,KAAKC,KAAK;MACR,OAAO,OAAO;IAChB,KAAKC,KAAK;MACR,OAAO,OAAO;IAChB,KAAKC,MAAM;MACT,OAAO,QAAQ;IACjB,KAAKC,MAAM;MACT,OAAO,QAAQ;IACjB,KAAKC,MAAM;MACT,OAAO,QAAQ;IAEjB,KAAKC,OAAO;MACV,OAAO,SAAS;IAClB,KAAKC,OAAO;MACV,OAAO,SAAS;IAClB,KAAKC,OAAO;MACV,OAAO,SAAS;IAClB,KAAKC,IAAI;MACP,OAAO,MAAM;IAEf,KAAKC,OAAO;MACV,OAAO,UAAU;IACnB,KAAKC,eAAe;MAClB,OAAO,kBAAkB;IAE3B,KAAKC,eAAe;MAClB,OAAO,kBAAkB;IAC3B,KAAKC,UAAU;MACb,OAAO,aAAa;IAEtB,KAAKC,eAAe;MAClB,OAAO,kBAAkB;IAC3B,KAAKC,oBAAoB;MACvB,OAAO,uBAAuB;IAChC,KAAKC,oBAAoB;MACvB,OAAO,uBAAuB;IAChC,KAAKC,mBAAmB;MACtB,OAAO,sBAAsB;IAE/B,KAAKC,eAAe;MAClB,OAAO,kBAAkB;IAC3B,KAAKC,iBAAiB;MACpB,OAAO,oBAAoB;IAC7B,KAAKC,aAAa;MAChB,OAAO;QACLqB,IAAI,EAAE,iBAAiB;QACvBM,QAAQ,EAAGF,SAAS,CAAmBE,QAAQ;QAC/CC,QAAQ,EAAE,CAACrB,mBAAmB,CAAEkB,SAAS,CAAmBG,QAAQ,CAAC,CAAC,CAAC,CAAC;MAC1E,CAAC;IAGH;MACE,MAAM,IAAIC,KAAK,CAAC,0BAA0B,CAAC;EAC/C;AACF;AAIA,OAAO,SAASL,oBAAoBA,CAACM,QAAkB,EAAiB;EACtE,IAAI,OAAOA,QAAQ,KAAK,QAAQ,EAAE;IAChC,QAAQA,QAAQ,CAACT,IAAI;MACnB,KAAK,iBAAiB;QACpB,MAAMU,KAAK,GAAGlB,qBAAqB,CAACiB,QAAQ,CAACF,QAAQ,CAAC,CAAC,CAAC,CAAC;QACzD,OAAO,IAAI5B,aAAa,CAAC8B,QAAQ,CAACH,QAAQ,EAAEI,KAAK,CAAC;MACpD,KAAK,QAAQ;QACX,MAAMH,QAAQ,GAAGE,QAAQ,CAACF,QAAQ,CAACvB,GAAG,CAAEC,UAAU,IAAKO,qBAAqB,CAACP,UAAU,CAAC,CAAC;QACzF,OAAO,IAAIL,MAAM,CAAC2B,QAAQ,CAAC;MAC7B;QACE,MAAM,IAAIC,KAAK,CAAC,0BAA0B,CAAC;IAC/C;EACF;EAEA,QAAQC,QAAQ;IACd,KAAK,MAAM;MACT,OAAO,IAAIvD,IAAI,CAAC,CAAC;IACnB,KAAK,QAAQ;MACX,OAAO,IAAIC,MAAM,CAAC,CAAC;IACrB,KAAK,MAAM;MACT,OAAO,IAAIC,IAAI,CAAC,CAAC;IAEnB,KAAK,MAAM;MACT,OAAO,IAAIC,IAAI,CAAC,CAAC;IACnB,KAAK,OAAO;MACV,OAAO,IAAIC,KAAK,CAAC,CAAC;IACpB,KAAK,OAAO;MACV,OAAO,IAAIC,KAAK,CAAC,CAAC;IACpB,KAAK,OAAO;MACV,OAAO,IAAIC,KAAK,CAAC,CAAC;IACpB,KAAK,OAAO;MACV,OAAO,IAAIC,KAAK,CAAC,CAAC;IACpB,KAAK,QAAQ;MACX,OAAO,IAAIC,MAAM,CAAC,CAAC;IACrB,KAAK,QAAQ;MACX,OAAO,IAAIC,MAAM,CAAC,CAAC;IACrB,KAAK,QAAQ;MACX,OAAO,IAAIC,MAAM,CAAC,CAAC;IAErB,KAAK,SAAS;MACZ,OAAO,IAAIC,OAAO,CAAC,CAAC;IACtB,KAAK,SAAS;MACZ,OAAO,IAAIC,OAAO,CAAC,CAAC;IACtB,KAAK,SAAS;MACZ,OAAO,IAAIC,OAAO,CAAC,CAAC;IACtB,KAAK,MAAM;MACT,OAAO,IAAIC,IAAI,CAAC,CAAC;IAEnB,KAAK,UAAU;MACb,OAAO,IAAIC,OAAO,CAAC,CAAC;IACtB,KAAK,kBAAkB;MACrB,OAAO,IAAIC,eAAe,CAAC,CAAC;IAE9B,KAAK,kBAAkB;MACrB,OAAO,IAAIC,eAAe,CAAC,CAAC;IAC9B,KAAK,aAAa;MAChB,OAAO,IAAIC,UAAU,CAAC,CAAC;IAEzB,KAAK,kBAAkB;MACrB,OAAO,IAAIC,eAAe,CAAC,CAAC;IAC9B,KAAK,uBAAuB;MAC1B,OAAO,IAAIC,oBAAoB,CAAC,CAAC;IACnC,KAAK,uBAAuB;MAC1B,OAAO,IAAIC,oBAAoB,CAAC,CAAC;IACnC,KAAK,sBAAsB;MACzB,OAAO,IAAIC,mBAAmB,CAAC,CAAC;IAElC,KAAK,kBAAkB;MACrB,OAAO,IAAIC,eAAe,CAAC,CAAC;IAC9B,KAAK,oBAAoB;MACvB,OAAO,IAAIC,iBAAiB,CAAC,CAAC;IAChC;MACE,MAAM,IAAI8B,KAAK,CAAC,0BAA0B,CAAC;EAC/C;AACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"convert-table-to-arrow.d.ts","sourceRoot":"","sources":["../../src/schema/convert-table-to-arrow.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"convert-table-to-arrow.d.ts","sourceRoot":"","sources":["../../src/schema/convert-table-to-arrow.ts"],"names":[],"mappings":"AAmBA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAwCE"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"convert-table-to-arrow.js","names":[],"sources":["../../src/schema/convert-table-to-arrow.ts"],"sourcesContent":["// loaders.gl, MIT license\n\n// import {\n// Table as ApacheArrowTable,\n// Schema as ApacheArrowSchema,\n// RecordBatch,\n// FixedSizeList,\n// Field,\n// Data,\n// FixedSizeListVector\n// } from 'apache-arrow/Arrow.dom';\n// import {AbstractVector} from 'apache-arrow/vector';\n\n// import {Table} from '../../types/category-table';\n// import {getArrowType, getArrowVector} from './arrow-type-utils';\n// import {makeMeshAttributeMetadata} from './deduce-mesh-schema';\n// import {getTableLength, getTableNumCols} from './table-accessors';\n\n/**\n * * Convert a loaders.gl Mesh to an Apache Arrow Table\n * @param mesh\n * @param metadata\n * @param batchSize\n * @returns\n *\nexport function convertTableToArrow(table: Table, options?: {batchSize?: number}): ArrowTable {\n const vectors: AbstractVector[] = [];\n const fields: Field[] = [];\n\n const length = getTableLength(table);\n const batchSize = options?.batchSize || length;\n\n const recordBatches: RecordBatch[] = [];\n for (let i = 0; i < length; i += batchSize) {\n for (let columnIndex = 0; columnIndex < getTableNumCols(table); ++columnIndex) {\n const field_ = table.schema.fields[columnIndex];\n const column = getTableColumnAt(table, columnIndex, i, batchSize);\n const type = getArrowType(column);\n const vector = getArrowVector(column);\n const listType = new FixedSizeList(size, new Field('value', type));\n const field = new Field(field_.name, listType, false, makeMeshAttributeMetadata(attribute));\n const data = new Data(listType, 0, value.length / size, 0, undefined, [vector]);\n const listVector = new FixedSizeListVector(data);\n vectors.push(listVector);\n fields.push(field);\n }\n\n for (const attributeKey in table.columns) {\n const attribute = mesh.attributes[attributeKey];\n const {value, size = 1} = attribute;\n }\n\n const schema = new ApacheArrowSchema(fields, table?.schema?.metadata || new Map<string, string>());\n const recordBatch = new RecordBatch(schema, vectors[0].length, vectors);\n const apacheArrowTable = new ApacheArrowTable(schema, recordBatch);\n\n return apacheArrowTable;\n}\n*/\n"],"mappings":""}
|
|
1
|
+
{"version":3,"file":"convert-table-to-arrow.js","names":[],"sources":["../../src/schema/convert-table-to-arrow.ts"],"sourcesContent":["// loaders.gl, MIT license\n// Copyright (c) vis.gl contributors\n\n// import {\n// Table as ApacheArrowTable,\n// Schema as ApacheArrowSchema,\n// RecordBatch,\n// FixedSizeList,\n// Field,\n// Data,\n// FixedSizeListVector\n// } from 'apache-arrow/Arrow.dom';\n// import {AbstractVector} from 'apache-arrow/vector';\n\n// import {Table} from '../../types/category-table';\n// import {getArrowType, getArrowVector} from './arrow-type-utils';\n// import {makeMeshAttributeMetadata} from './deduce-mesh-schema';\n// import {getTableLength, getTableNumCols} from './table-accessors';\n\n/**\n * * Convert a loaders.gl Mesh to an Apache Arrow Table\n * @param mesh\n * @param metadata\n * @param batchSize\n * @returns\n *\nexport function convertTableToArrow(table: Table, options?: {batchSize?: number}): ArrowTable {\n const vectors: AbstractVector[] = [];\n const fields: Field[] = [];\n\n const length = getTableLength(table);\n const batchSize = options?.batchSize || length;\n\n const recordBatches: RecordBatch[] = [];\n for (let i = 0; i < length; i += batchSize) {\n for (let columnIndex = 0; columnIndex < getTableNumCols(table); ++columnIndex) {\n const field_ = table.schema.fields[columnIndex];\n const column = getTableColumnAt(table, columnIndex, i, batchSize);\n const type = getArrowType(column);\n const vector = getArrowVector(column);\n const listType = new FixedSizeList(size, new Field('value', type));\n const field = new Field(field_.name, listType, false, makeMeshAttributeMetadata(attribute));\n const data = new Data(listType, 0, value.length / size, 0, undefined, [vector]);\n const listVector = new FixedSizeListVector(data);\n vectors.push(listVector);\n fields.push(field);\n }\n\n for (const attributeKey in table.columns) {\n const attribute = mesh.attributes[attributeKey];\n const {value, size = 1} = attribute;\n }\n\n const schema = new ApacheArrowSchema(fields, table?.schema?.metadata || new Map<string, string>());\n const recordBatch = new RecordBatch(schema, vectors[0].length, vectors);\n const apacheArrowTable = new ApacheArrowTable(schema, recordBatch);\n\n return apacheArrowTable;\n}\n*/\n"],"mappings":""}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@loaders.gl/arrow",
|
|
3
|
-
"version": "4.0.0
|
|
3
|
+
"version": "4.0.0",
|
|
4
4
|
"description": "Simple columnar table loader for the Apache Arrow format",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -46,9 +46,9 @@
|
|
|
46
46
|
"build-worker2": "esbuild src/workers/arrow-worker.ts --bundle --outfile=dist/arrow-worker.js --platform=browser --external:{stream}"
|
|
47
47
|
},
|
|
48
48
|
"dependencies": {
|
|
49
|
-
"@loaders.gl/loader-utils": "4.0.0
|
|
50
|
-
"@loaders.gl/schema": "4.0.0
|
|
49
|
+
"@loaders.gl/loader-utils": "4.0.0",
|
|
50
|
+
"@loaders.gl/schema": "4.0.0",
|
|
51
51
|
"apache-arrow": "^13.0.0"
|
|
52
52
|
},
|
|
53
|
-
"gitHead": "
|
|
53
|
+
"gitHead": "9b4211dc0ecd4134a1638ac0a29c5ea9008fd971"
|
|
54
54
|
}
|
package/src/arrow-loader.ts
CHANGED
package/src/lib/arrow-table.ts
CHANGED
package/src/lib/convert-table.ts
CHANGED