@loaders.gl/schema 3.3.0-alpha.1 → 3.3.0-alpha.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/category/common.d.ts +3 -3
- package/dist/category/common.d.ts.map +1 -1
- package/dist/category/gis.d.ts +23 -23
- package/dist/category/gis.d.ts.map +1 -1
- package/dist/category/image/image.d.ts +3 -3
- package/dist/category/image/image.d.ts.map +1 -1
- package/dist/category/mesh/convert-mesh.d.ts +1 -1
- package/dist/category/mesh/convert-mesh.d.ts.map +1 -1
- package/dist/category/mesh/mesh-types.d.ts +4 -4
- package/dist/category/mesh/mesh-types.d.ts.map +1 -1
- package/dist/category/mesh/mesh-utils.d.ts +2 -2
- package/dist/category/mesh/mesh-utils.d.ts.map +1 -1
- package/dist/category/table/table-types.d.ts +8 -8
- package/dist/category/table/table-types.d.ts.map +1 -1
- package/dist/category/texture/texture.d.ts +3 -3
- package/dist/category/texture/texture.d.ts.map +1 -1
- package/dist/es5/bundle.js +0 -1
- package/dist/es5/bundle.js.map +1 -1
- package/dist/es5/category/common.js.map +1 -1
- package/dist/es5/category/gis.js.map +1 -1
- package/dist/es5/category/image/image.js.map +1 -1
- package/dist/es5/category/mesh/convert-mesh.js +2 -12
- package/dist/es5/category/mesh/convert-mesh.js.map +1 -1
- package/dist/es5/category/mesh/deduce-mesh-schema.js +1 -10
- package/dist/es5/category/mesh/deduce-mesh-schema.js.map +1 -1
- package/dist/es5/category/mesh/mesh-to-arrow-table.js +2 -9
- package/dist/es5/category/mesh/mesh-to-arrow-table.js.map +1 -1
- package/dist/es5/category/mesh/mesh-types.js.map +1 -1
- package/dist/es5/category/mesh/mesh-utils.js +1 -6
- package/dist/es5/category/mesh/mesh-utils.js.map +1 -1
- package/dist/es5/category/table/deduce-table-schema.js +0 -11
- package/dist/es5/category/table/deduce-table-schema.js.map +1 -1
- package/dist/es5/category/table/table-types.js.map +1 -1
- package/dist/es5/category/texture/texture.js.map +1 -1
- package/dist/es5/index.js +100 -112
- package/dist/es5/index.js.map +1 -1
- package/dist/es5/lib/arrow/arrow-like-type-utils.js +0 -10
- package/dist/es5/lib/arrow/arrow-like-type-utils.js.map +1 -1
- package/dist/es5/lib/arrow/arrow-type-utils.js +0 -19
- package/dist/es5/lib/arrow/arrow-type-utils.js.map +1 -1
- package/dist/es5/lib/arrow/get-type-info.js +0 -6
- package/dist/es5/lib/arrow/get-type-info.js.map +1 -1
- package/dist/es5/lib/batches/base-table-batch-aggregator.js +0 -13
- package/dist/es5/lib/batches/base-table-batch-aggregator.js.map +1 -1
- package/dist/es5/lib/batches/columnar-table-batch-aggregator.js +6 -23
- package/dist/es5/lib/batches/columnar-table-batch-aggregator.js.map +1 -1
- package/dist/es5/lib/batches/row-table-batch-aggregator.js +0 -13
- package/dist/es5/lib/batches/row-table-batch-aggregator.js.map +1 -1
- package/dist/es5/lib/batches/table-batch-aggregator.js.map +1 -1
- package/dist/es5/lib/batches/table-batch-builder.js +21 -37
- package/dist/es5/lib/batches/table-batch-builder.js.map +1 -1
- package/dist/es5/lib/schema/impl/enum.js +0 -1
- package/dist/es5/lib/schema/impl/enum.js.map +1 -1
- package/dist/es5/lib/schema/impl/field.js +0 -7
- package/dist/es5/lib/schema/impl/field.js.map +1 -1
- package/dist/es5/lib/schema/impl/schema.js +6 -35
- package/dist/es5/lib/schema/impl/schema.js.map +1 -1
- package/dist/es5/lib/schema/impl/type.js +74 -248
- package/dist/es5/lib/schema/impl/type.js.map +1 -1
- package/dist/es5/lib/schema/schema.js +70 -74
- package/dist/es5/lib/schema/schema.js.map +1 -1
- package/dist/es5/lib/schema-utils/deduce-column-type.js +0 -3
- package/dist/es5/lib/schema-utils/deduce-column-type.js.map +1 -1
- package/dist/es5/lib/utils/assert.js +0 -1
- package/dist/es5/lib/utils/assert.js.map +1 -1
- package/dist/es5/lib/utils/async-queue.js +10 -50
- package/dist/es5/lib/utils/async-queue.js.map +1 -1
- package/dist/es5/lib/utils/row-utils.js +1 -10
- package/dist/es5/lib/utils/row-utils.js.map +1 -1
- package/dist/es5/types.js.map +1 -1
- package/dist/esm/bundle.js +1 -1
- package/dist/esm/bundle.js.map +1 -1
- package/dist/esm/category/common.js.map +1 -1
- package/dist/esm/category/gis.js.map +1 -1
- package/dist/esm/category/image/image.js.map +1 -1
- package/dist/esm/category/mesh/convert-mesh.js +1 -5
- package/dist/esm/category/mesh/convert-mesh.js.map +1 -1
- package/dist/esm/category/mesh/deduce-mesh-schema.js +2 -6
- package/dist/esm/category/mesh/deduce-mesh-schema.js.map +1 -1
- package/dist/esm/category/mesh/mesh-to-arrow-table.js +1 -3
- package/dist/esm/category/mesh/mesh-to-arrow-table.js.map +1 -1
- package/dist/esm/category/mesh/mesh-types.js.map +1 -1
- package/dist/esm/category/mesh/mesh-utils.js +3 -5
- package/dist/esm/category/mesh/mesh-utils.js.map +1 -1
- package/dist/esm/category/table/deduce-table-schema.js +2 -11
- package/dist/esm/category/table/deduce-table-schema.js.map +1 -1
- package/dist/esm/category/table/table-types.js.map +1 -1
- package/dist/esm/category/texture/texture.js.map +1 -1
- package/dist/esm/index.js +5 -0
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/lib/arrow/arrow-like-type-utils.js +0 -8
- package/dist/esm/lib/arrow/arrow-like-type-utils.js.map +1 -1
- package/dist/esm/lib/arrow/arrow-type-utils.js +0 -16
- package/dist/esm/lib/arrow/arrow-type-utils.js.map +1 -1
- package/dist/esm/lib/arrow/get-type-info.js +0 -3
- package/dist/esm/lib/arrow/get-type-info.js.map +1 -1
- package/dist/esm/lib/batches/base-table-batch-aggregator.js +0 -16
- package/dist/esm/lib/batches/base-table-batch-aggregator.js.map +1 -1
- package/dist/esm/lib/batches/columnar-table-batch-aggregator.js +0 -20
- package/dist/esm/lib/batches/columnar-table-batch-aggregator.js.map +1 -1
- package/dist/esm/lib/batches/row-table-batch-aggregator.js +0 -17
- package/dist/esm/lib/batches/row-table-batch-aggregator.js.map +1 -1
- package/dist/esm/lib/batches/table-batch-aggregator.js.map +1 -1
- package/dist/esm/lib/batches/table-batch-builder.js +4 -35
- package/dist/esm/lib/batches/table-batch-builder.js.map +1 -1
- package/dist/esm/lib/schema/impl/enum.js +2 -1
- package/dist/esm/lib/schema/impl/enum.js.map +1 -1
- package/dist/esm/lib/schema/impl/field.js +3 -10
- package/dist/esm/lib/schema/impl/field.js.map +1 -1
- package/dist/esm/lib/schema/impl/schema.js +9 -20
- package/dist/esm/lib/schema/impl/schema.js.map +1 -1
- package/dist/esm/lib/schema/impl/type.js +19 -113
- package/dist/esm/lib/schema/impl/type.js.map +1 -1
- package/dist/esm/lib/schema/schema.js +3 -1
- package/dist/esm/lib/schema/schema.js.map +1 -1
- package/dist/esm/lib/schema-utils/deduce-column-type.js +2 -2
- package/dist/esm/lib/schema-utils/deduce-column-type.js.map +1 -1
- package/dist/esm/lib/utils/assert.js +1 -0
- package/dist/esm/lib/utils/assert.js.map +1 -1
- package/dist/esm/lib/utils/async-queue.js +3 -25
- package/dist/esm/lib/utils/async-queue.js.map +1 -1
- package/dist/esm/lib/utils/row-utils.js +2 -8
- package/dist/esm/lib/utils/row-utils.js.map +1 -1
- package/dist/esm/types.js.map +1 -1
- package/dist/index.js +1 -1
- package/dist/lib/batches/columnar-table-batch-aggregator.d.ts +1 -1
- package/dist/lib/batches/columnar-table-batch-aggregator.d.ts.map +1 -1
- package/dist/lib/batches/row-table-batch-aggregator.js +1 -1
- package/dist/lib/batches/table-batch-builder.d.ts +2 -2
- package/dist/lib/batches/table-batch-builder.d.ts.map +1 -1
- package/dist/lib/schema/impl/schema.d.ts +1 -1
- package/dist/lib/schema/impl/schema.d.ts.map +1 -1
- package/dist/lib/schema/impl/type.d.ts +4 -4
- package/dist/lib/schema/impl/type.d.ts.map +1 -1
- package/dist/lib/schema-utils/deduce-column-type.js +1 -1
- package/dist/types.d.ts +4 -4
- package/dist/types.d.ts.map +1 -1
- package/package.json +3 -4
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../../src/lib/batches/table-batch-builder.ts"],"names":["DEFAULT_OPTIONS","shape","batchSize","batchDebounceMs","limit","_limitMB","ERR_MESSAGE","TableBatchBuilder","schema","options","Date","now","Boolean","totalLength","totalBytes","row","limitReached","rowBytes","_estimateRowMB","Array","isArray","addArrayRow","addObjectRow","aggregator","TableBatchType","_getTableBatchType","chunk","ArrayBuffer","bytesUsed","byteLength","length","isChunkComplete","_isFull","_getBatch","Object","keys","rowCount","lastBatchEmittedMs","normalizedBatch","getBatch","count","batchCount","assign","BaseTableBatchAggregator","RowTableBatchAggregator","ColumnarTableBatchAggregator","ArrowBatch","Error"],"mappings":";;;;;;;;;;;;;;;AAGA;;AACA;;AACA;;;;;;AAgBA,IAAMA,eAAmD,GAAG;AAC1DC,EAAAA,KAAK,EAAE,iBADmD;AAE1DC,EAAAA,SAAS,EAAE,MAF+C;AAG1DC,EAAAA,eAAe,EAAE,CAHyC;AAI1DC,EAAAA,KAAK,EAAE,CAJmD;AAK1DC,EAAAA,QAAQ,EAAE;AALgD,CAA5D;AAQA,IAAMC,WAAW,GAAG,mBAApB;;IAGqBC,iB;AAenB,6BAAYC,MAAZ,EAA4BC,OAA5B,EAAgE;AAAA;AAAA;AAAA;AAAA,sDAXd,IAWc;AAAA,sDAVnC,CAUmC;AAAA,qDATpC,CASoC;AAAA,2DAR7B,KAQ6B;AAAA,8DAP3BC,IAAI,CAACC,GAAL,EAO2B;AAAA,uDANlC,CAMkC;AAAA,sDALnC,CAKmC;AAAA,oDAJrC,CAIqC;AAC9D,SAAKH,MAAL,GAAcA,MAAd;AACA,SAAKC,OAAL,mCAAmBT,eAAnB,GAAuCS,OAAvC;AACD;;;;WAED,wBAAwB;AAAA;;AACtB,UAAIG,OAAO,kBAAC,KAAKH,OAAN,kDAAC,cAAcL,KAAf,CAAP,IAAgC,KAAKS,WAAL,IAAoB,KAAKJ,OAAL,CAAaL,KAArE,EAA4E;AAC1E,eAAO,IAAP;AACD;;AACD,UAAIQ,OAAO,mBAAC,KAAKH,OAAN,mDAAC,eAAcJ,QAAf,CAAP,IAAmC,KAAKS,UAAL,GAAkB,GAAlB,IAAyB,KAAKL,OAAL,CAAaJ,QAA7E,EAAuF;AACrF,eAAO,IAAP;AACD;;AACD,aAAO,KAAP;AACD;;;WAGD,gBAAOU,GAAP,EAAuD;AACrD,UAAI,KAAKC,YAAL,EAAJ,EAAyB;AACvB;AACD;;AACD,WAAKH,WAAL;AACA,WAAKI,QAAL,GAAgB,KAAKA,QAAL,IAAiB,KAAKC,cAAL,CAAoBH,GAApB,CAAjC;AACA,WAAKD,UAAL,IAAmB,KAAKG,QAAxB;;AACA,UAAIE,KAAK,CAACC,OAAN,CAAcL,GAAd,CAAJ,EAAwB;AACtB,aAAKM,WAAL,CAAiBN,GAAjB;AACD,OAFD,MAEO;AACL,aAAKO,YAAL,CAAkBP,GAAlB;AACD;AACF;;;WAGD,qBAAsBA,GAAtB,EAAkC;AAChC,UAAI,CAAC,KAAKQ,UAAV,EAAsB;AACpB,YAAMC,cAAc,GAAG,KAAKC,kBAAL,EAAvB;;AACA,aAAKF,UAAL,GAAkB,IAAIC,cAAJ,CAAmB,KAAKhB,MAAxB,EAAgC,KAAKC,OAArC,CAAlB;AACD;;AACD,WAAKc,UAAL,CAAgBF,WAAhB,CAA4BN,GAA5B;AACD;;;WAGD,sBAAuBA,GAAvB,EAA+D;AAC7D,UAAI,CAAC,KAAKQ,UAAV,EAAsB;AACpB,YAAMC,cAAc,GAAG,KAAKC,kBAAL,EAAvB;;AACA,aAAKF,UAAL,GAAkB,IAAIC,cAAJ,CAAmB,KAAKhB,MAAxB,EAAgC,KAAKC,OAArC,CAAlB;AACD;;AACD,WAAKc,UAAL,CAAgBD,YAAhB,CAA6BP,GAA7B;AACD;;;WAGD,uBAAcW,KAAd,EAAiD;AAC/C,UAAIA,KAAK,YAAYC,WAArB,EAAkC;AAChC,aAAKC,SAAL,IAAkBF,KAAK,CAACG,UAAxB;AACD;;AACD,UAAI,OAAOH,KAAP,KAAiB,QAArB,EAA+B;AAC7B,aAAKE,SAAL,IAAkBF,KAAK,CAACI,MAAxB;AACD;;AACD,WAAKC,eAAL,GAAuB,IAAvB;AACD;;;WAED,sBAAatB,OAAb,EAA2D;AACzD,aAAO,KAAKuB,OAAL,KAAiB,KAAKC,SAAL,CAAexB,OAAf,CAAjB,GAA2C,IAAlD;AACD;;;WAED,uBAAcA,OAAd,EAA4D;AAC1D,aAAO,KAAKwB,SAAL,CAAexB,OAAf,CAAP;AACD;;;WAID,wBAAeM,GAAf,EAA4C;AAC1C,aAAOI,KAAK,CAACC,OAAN,CAAcL,GAAd,IAAqBA,GAAG,CAACe,MAAJ,GAAa,CAAlC,GAAsCI,MAAM,CAACC,IAAP,CAAYpB,GAAZ,EAAiBe,MAAjB,GAA0B,CAAvE;AACD;;;WAED,mBAA2B;AAEzB,UAAI,CAAC,KAAKP,UAAN,IAAoB,KAAKA,UAAL,CAAgBa,QAAhB,OAA+B,CAAvD,EAA0D;AACxD,eAAO,KAAP;AACD;;AAID,UAAI,KAAK3B,OAAL,CAAaP,SAAb,KAA2B,MAA/B,EAAuC;AACrC,YAAI,CAAC,KAAK6B,eAAV,EAA2B;AACzB,iBAAO,KAAP;AACD;AACF,OAJD,MAIO,IAAI,KAAKtB,OAAL,CAAaP,SAAb,GAAyB,KAAKqB,UAAL,CAAgBa,QAAhB,EAA7B,EAAyD;AAC9D,eAAO,KAAP;AACD;;AAGD,UAAI,KAAK3B,OAAL,CAAaN,eAAb,GAA+BO,IAAI,CAACC,GAAL,KAAa,KAAK0B,kBAArD,EAAyE;AACvE,eAAO,KAAP;AACD;;AAGD,WAAKN,eAAL,GAAuB,KAAvB;AACA,WAAKM,kBAAL,GAA0B3B,IAAI,CAACC,GAAL,EAA1B;AACA,aAAO,IAAP;AACD;;;WAKD,mBAAkBF,OAAlB,EAAgE;AAC9D,UAAI,CAAC,KAAKc,UAAV,EAAsB;AACpB,eAAO,IAAP;AACD;;AAGD,UAAId,OAAJ,aAAIA,OAAJ,eAAIA,OAAO,CAAEmB,SAAb,EAAwB;AACtB,aAAKA,SAAL,GAAiBnB,OAAO,CAACmB,SAAzB;AACD;;AACD,UAAMU,eAAe,GAAG,KAAKf,UAAL,CAAgBgB,QAAhB,EAAxB;AACAD,MAAAA,eAAe,CAACE,KAAhB,GAAwB,KAAKC,UAA7B;AACAH,MAAAA,eAAe,CAACV,SAAhB,GAA4B,KAAKA,SAAjC;AACAM,MAAAA,MAAM,CAACQ,MAAP,CAAcJ,eAAd,EAA+B7B,OAA/B;AAEA,WAAKgC,UAAL;AACA,WAAKlB,UAAL,GAAkB,IAAlB;AACA,aAAOe,eAAP;AACD;;;WAED,8BAAoD;AAClD,cAAQ,KAAK7B,OAAL,CAAaR,KAArB;AACE,aAAK,WAAL;AACE,iBAAO0C,iCAAP;;AACF,aAAK,iBAAL;AACA,aAAK,kBAAL;AACE,iBAAOC,gCAAP;;AACF,aAAK,gBAAL;AACE,iBAAOC,qCAAP;;AACF,aAAK,aAAL;AACE,cAAI,CAACtC,iBAAiB,CAACuC,UAAvB,EAAmC;AACjC,kBAAM,IAAIC,KAAJ,CAAUzC,WAAV,CAAN;AACD;;AACD,iBAAOC,iBAAiB,CAACuC,UAAzB;;AACF;AACE,gBAAM,IAAIC,KAAJ,CAAUzC,WAAV,CAAN;AAdJ;AAgBD;;;;;;8BA1JkBC,iB","sourcesContent":["import type {Schema} from '../schema/schema';\nimport type {TableBatch} from '../../category/table/table-types';\nimport type {TableBatchAggregator, TableBatchConstructor} from './table-batch-aggregator';\nimport BaseTableBatchAggregator from './base-table-batch-aggregator';\nimport RowTableBatchAggregator from './row-table-batch-aggregator';\nimport ColumnarTableBatchAggregator from './columnar-table-batch-aggregator';\n\n// TODO define interface instead\ntype TableBatchBuilderOptions = {\n shape: 'row-table' | 'array-row-table' | 'object-row-table' | 'columnar-table' | 'arrow-table';\n batchSize?: number | 'auto';\n batchDebounceMs?: number;\n limit: number;\n _limitMB: number;\n};\n\ntype GetBatchOptions = {\n bytesUsed?: number;\n [key: string]: any;\n};\n\nconst DEFAULT_OPTIONS: Required<TableBatchBuilderOptions> = {\n shape: 'array-row-table',\n batchSize: 'auto',\n batchDebounceMs: 0,\n limit: 0,\n _limitMB: 0\n};\n\nconst ERR_MESSAGE = 'TableBatchBuilder';\n\n/** Incrementally builds batches from a stream of rows */\nexport default class TableBatchBuilder {\n schema: Schema;\n options: Required<TableBatchBuilderOptions>;\n\n private aggregator: TableBatchAggregator | null = null;\n private batchCount: number = 0;\n private bytesUsed: number = 0;\n private isChunkComplete: boolean = false;\n private lastBatchEmittedMs: number = Date.now();\n private totalLength: number = 0;\n private totalBytes: number = 0;\n private rowBytes: number = 0;\n\n static ArrowBatch?: TableBatchConstructor;\n\n constructor(schema: Schema, options?: TableBatchBuilderOptions) {\n this.schema = schema;\n this.options = {...DEFAULT_OPTIONS, ...options};\n }\n\n limitReached(): boolean {\n if (Boolean(this.options?.limit) && this.totalLength >= this.options.limit) {\n return true;\n }\n if (Boolean(this.options?._limitMB) && this.totalBytes / 1e6 >= this.options._limitMB) {\n return true;\n }\n return false;\n }\n\n /** @deprecated Use addArrayRow or addObjectRow */\n addRow(row: any[] | {[columnName: string]: any}): void {\n if (this.limitReached()) {\n return;\n }\n this.totalLength++;\n this.rowBytes = this.rowBytes || this._estimateRowMB(row);\n this.totalBytes += this.rowBytes;\n if (Array.isArray(row)) {\n this.addArrayRow(row);\n } else {\n this.addObjectRow(row);\n }\n }\n\n /** Add one row to the batch */\n protected addArrayRow(row: any[]) {\n if (!this.aggregator) {\n const TableBatchType = this._getTableBatchType();\n this.aggregator = new TableBatchType(this.schema, this.options);\n }\n this.aggregator.addArrayRow(row);\n }\n\n /** Add one row to the batch */\n protected addObjectRow(row: {[columnName: string]: any}): void {\n if (!this.aggregator) {\n const TableBatchType = this._getTableBatchType();\n this.aggregator = new TableBatchType(this.schema, this.options);\n }\n this.aggregator.addObjectRow(row);\n }\n\n /** Mark an incoming raw memory chunk has completed */\n chunkComplete(chunk: ArrayBuffer | string): void {\n if (chunk instanceof ArrayBuffer) {\n this.bytesUsed += chunk.byteLength;\n }\n if (typeof chunk === 'string') {\n this.bytesUsed += chunk.length;\n }\n this.isChunkComplete = true;\n }\n\n getFullBatch(options?: GetBatchOptions): TableBatch | null {\n return this._isFull() ? this._getBatch(options) : null;\n }\n\n getFinalBatch(options?: GetBatchOptions): TableBatch | null {\n return this._getBatch(options);\n }\n\n // INTERNAL\n\n _estimateRowMB(row: any[] | object): number {\n return Array.isArray(row) ? row.length * 8 : Object.keys(row).length * 8;\n }\n\n private _isFull(): boolean {\n // No batch, not ready\n if (!this.aggregator || this.aggregator.rowCount() === 0) {\n return false;\n }\n\n // if batchSize === 'auto' we wait for chunk to complete\n // if batchSize === number, ensure we have enough rows\n if (this.options.batchSize === 'auto') {\n if (!this.isChunkComplete) {\n return false;\n }\n } else if (this.options.batchSize > this.aggregator.rowCount()) {\n return false;\n }\n\n // Debounce batches\n if (this.options.batchDebounceMs > Date.now() - this.lastBatchEmittedMs) {\n return false;\n }\n\n // Emit batch\n this.isChunkComplete = false;\n this.lastBatchEmittedMs = Date.now();\n return true;\n }\n\n /**\n * bytesUsed can be set via chunkComplete or via getBatch*\n */\n private _getBatch(options?: GetBatchOptions): TableBatch | null {\n if (!this.aggregator) {\n return null;\n }\n\n // TODO - this can overly increment bytes used?\n if (options?.bytesUsed) {\n this.bytesUsed = options.bytesUsed;\n }\n const normalizedBatch = this.aggregator.getBatch() as TableBatch;\n normalizedBatch.count = this.batchCount;\n normalizedBatch.bytesUsed = this.bytesUsed;\n Object.assign(normalizedBatch, options);\n\n this.batchCount++;\n this.aggregator = null;\n return normalizedBatch;\n }\n\n private _getTableBatchType(): TableBatchConstructor {\n switch (this.options.shape) {\n case 'row-table':\n return BaseTableBatchAggregator;\n case 'array-row-table':\n case 'object-row-table':\n return RowTableBatchAggregator;\n case 'columnar-table':\n return ColumnarTableBatchAggregator;\n case 'arrow-table':\n if (!TableBatchBuilder.ArrowBatch) {\n throw new Error(ERR_MESSAGE);\n }\n return TableBatchBuilder.ArrowBatch;\n default:\n throw new Error(ERR_MESSAGE);\n }\n }\n}\n"],"file":"table-batch-builder.js"}
|
|
1
|
+
{"version":3,"file":"table-batch-builder.js","names":["DEFAULT_OPTIONS","shape","batchSize","batchDebounceMs","limit","_limitMB","ERR_MESSAGE","TableBatchBuilder","schema","options","Date","now","Boolean","totalLength","totalBytes","row","limitReached","rowBytes","_estimateRowMB","Array","isArray","addArrayRow","addObjectRow","aggregator","TableBatchType","_getTableBatchType","chunk","ArrayBuffer","bytesUsed","byteLength","length","isChunkComplete","_isFull","_getBatch","Object","keys","rowCount","lastBatchEmittedMs","normalizedBatch","getBatch","count","batchCount","assign","BaseTableBatchAggregator","RowTableBatchAggregator","ColumnarTableBatchAggregator","ArrowBatch","Error"],"sources":["../../../../src/lib/batches/table-batch-builder.ts"],"sourcesContent":["import type {Schema} from '../schema/schema';\nimport type {TableBatch} from '../../category/table/table-types';\nimport type {TableBatchAggregator, TableBatchConstructor} from './table-batch-aggregator';\nimport BaseTableBatchAggregator from './base-table-batch-aggregator';\nimport RowTableBatchAggregator from './row-table-batch-aggregator';\nimport ColumnarTableBatchAggregator from './columnar-table-batch-aggregator';\n\n// TODO define interface instead\ntype TableBatchBuilderOptions = {\n shape: 'row-table' | 'array-row-table' | 'object-row-table' | 'columnar-table' | 'arrow-table';\n batchSize?: number | 'auto';\n batchDebounceMs?: number;\n limit: number;\n _limitMB: number;\n};\n\ntype GetBatchOptions = {\n bytesUsed?: number;\n [key: string]: any;\n};\n\nconst DEFAULT_OPTIONS: Required<TableBatchBuilderOptions> = {\n shape: 'array-row-table',\n batchSize: 'auto',\n batchDebounceMs: 0,\n limit: 0,\n _limitMB: 0\n};\n\nconst ERR_MESSAGE = 'TableBatchBuilder';\n\n/** Incrementally builds batches from a stream of rows */\nexport default class TableBatchBuilder {\n schema: Schema;\n options: Required<TableBatchBuilderOptions>;\n\n private aggregator: TableBatchAggregator | null = null;\n private batchCount: number = 0;\n private bytesUsed: number = 0;\n private isChunkComplete: boolean = false;\n private lastBatchEmittedMs: number = Date.now();\n private totalLength: number = 0;\n private totalBytes: number = 0;\n private rowBytes: number = 0;\n\n static ArrowBatch?: TableBatchConstructor;\n\n constructor(schema: Schema, options?: TableBatchBuilderOptions) {\n this.schema = schema;\n this.options = {...DEFAULT_OPTIONS, ...options};\n }\n\n limitReached(): boolean {\n if (Boolean(this.options?.limit) && this.totalLength >= this.options.limit) {\n return true;\n }\n if (Boolean(this.options?._limitMB) && this.totalBytes / 1e6 >= this.options._limitMB) {\n return true;\n }\n return false;\n }\n\n /** @deprecated Use addArrayRow or addObjectRow */\n addRow(row: any[] | {[columnName: string]: any}): void {\n if (this.limitReached()) {\n return;\n }\n this.totalLength++;\n this.rowBytes = this.rowBytes || this._estimateRowMB(row);\n this.totalBytes += this.rowBytes;\n if (Array.isArray(row)) {\n this.addArrayRow(row);\n } else {\n this.addObjectRow(row);\n }\n }\n\n /** Add one row to the batch */\n protected addArrayRow(row: any[]) {\n if (!this.aggregator) {\n const TableBatchType = this._getTableBatchType();\n this.aggregator = new TableBatchType(this.schema, this.options);\n }\n this.aggregator.addArrayRow(row);\n }\n\n /** Add one row to the batch */\n protected addObjectRow(row: {[columnName: string]: any}): void {\n if (!this.aggregator) {\n const TableBatchType = this._getTableBatchType();\n this.aggregator = new TableBatchType(this.schema, this.options);\n }\n this.aggregator.addObjectRow(row);\n }\n\n /** Mark an incoming raw memory chunk has completed */\n chunkComplete(chunk: ArrayBuffer | string): void {\n if (chunk instanceof ArrayBuffer) {\n this.bytesUsed += chunk.byteLength;\n }\n if (typeof chunk === 'string') {\n this.bytesUsed += chunk.length;\n }\n this.isChunkComplete = true;\n }\n\n getFullBatch(options?: GetBatchOptions): TableBatch | null {\n return this._isFull() ? this._getBatch(options) : null;\n }\n\n getFinalBatch(options?: GetBatchOptions): TableBatch | null {\n return this._getBatch(options);\n }\n\n // INTERNAL\n\n _estimateRowMB(row: any[] | object): number {\n return Array.isArray(row) ? row.length * 8 : Object.keys(row).length * 8;\n }\n\n private _isFull(): boolean {\n // No batch, not ready\n if (!this.aggregator || this.aggregator.rowCount() === 0) {\n return false;\n }\n\n // if batchSize === 'auto' we wait for chunk to complete\n // if batchSize === number, ensure we have enough rows\n if (this.options.batchSize === 'auto') {\n if (!this.isChunkComplete) {\n return false;\n }\n } else if (this.options.batchSize > this.aggregator.rowCount()) {\n return false;\n }\n\n // Debounce batches\n if (this.options.batchDebounceMs > Date.now() - this.lastBatchEmittedMs) {\n return false;\n }\n\n // Emit batch\n this.isChunkComplete = false;\n this.lastBatchEmittedMs = Date.now();\n return true;\n }\n\n /**\n * bytesUsed can be set via chunkComplete or via getBatch*\n */\n private _getBatch(options?: GetBatchOptions): TableBatch | null {\n if (!this.aggregator) {\n return null;\n }\n\n // TODO - this can overly increment bytes used?\n if (options?.bytesUsed) {\n this.bytesUsed = options.bytesUsed;\n }\n const normalizedBatch = this.aggregator.getBatch() as TableBatch;\n normalizedBatch.count = this.batchCount;\n normalizedBatch.bytesUsed = this.bytesUsed;\n Object.assign(normalizedBatch, options);\n\n this.batchCount++;\n this.aggregator = null;\n return normalizedBatch;\n }\n\n private _getTableBatchType(): TableBatchConstructor {\n switch (this.options.shape) {\n case 'row-table':\n return BaseTableBatchAggregator;\n case 'array-row-table':\n case 'object-row-table':\n return RowTableBatchAggregator;\n case 'columnar-table':\n return ColumnarTableBatchAggregator;\n case 'arrow-table':\n if (!TableBatchBuilder.ArrowBatch) {\n throw new Error(ERR_MESSAGE);\n }\n return TableBatchBuilder.ArrowBatch;\n default:\n throw new Error(ERR_MESSAGE);\n }\n }\n}\n"],"mappings":";;;;;;;;;;AAGA;AACA;AACA;AAA6E;AAAA;AAgB7E,IAAMA,eAAmD,GAAG;EAC1DC,KAAK,EAAE,iBAAiB;EACxBC,SAAS,EAAE,MAAM;EACjBC,eAAe,EAAE,CAAC;EAClBC,KAAK,EAAE,CAAC;EACRC,QAAQ,EAAE;AACZ,CAAC;AAED,IAAMC,WAAW,GAAG,mBAAmB;;AAAC,IAGnBC,iBAAiB;EAepC,2BAAYC,MAAc,EAAEC,OAAkC,EAAE;IAAA;IAAA;IAAA;IAAA,kDAXd,IAAI;IAAA,kDACzB,CAAC;IAAA,iDACF,CAAC;IAAA,uDACM,KAAK;IAAA,0DACHC,IAAI,CAACC,GAAG,EAAE;IAAA,mDACjB,CAAC;IAAA,kDACF,CAAC;IAAA,gDACH,CAAC;IAK1B,IAAI,CAACH,MAAM,GAAGA,MAAM;IACpB,IAAI,CAACC,OAAO,mCAAOT,eAAe,GAAKS,OAAO,CAAC;EACjD;EAAC;IAAA;IAAA,OAED,wBAAwB;MAAA;MACtB,IAAIG,OAAO,kBAAC,IAAI,CAACH,OAAO,kDAAZ,cAAcL,KAAK,CAAC,IAAI,IAAI,CAACS,WAAW,IAAI,IAAI,CAACJ,OAAO,CAACL,KAAK,EAAE;QAC1E,OAAO,IAAI;MACb;MACA,IAAIQ,OAAO,mBAAC,IAAI,CAACH,OAAO,mDAAZ,eAAcJ,QAAQ,CAAC,IAAI,IAAI,CAACS,UAAU,GAAG,GAAG,IAAI,IAAI,CAACL,OAAO,CAACJ,QAAQ,EAAE;QACrF,OAAO,IAAI;MACb;MACA,OAAO,KAAK;IACd;;EAAC;IAAA;IAAA;IAGD,gBAAOU,GAAwC,EAAQ;MACrD,IAAI,IAAI,CAACC,YAAY,EAAE,EAAE;QACvB;MACF;MACA,IAAI,CAACH,WAAW,EAAE;MAClB,IAAI,CAACI,QAAQ,GAAG,IAAI,CAACA,QAAQ,IAAI,IAAI,CAACC,cAAc,CAACH,GAAG,CAAC;MACzD,IAAI,CAACD,UAAU,IAAI,IAAI,CAACG,QAAQ;MAChC,IAAIE,KAAK,CAACC,OAAO,CAACL,GAAG,CAAC,EAAE;QACtB,IAAI,CAACM,WAAW,CAACN,GAAG,CAAC;MACvB,CAAC,MAAM;QACL,IAAI,CAACO,YAAY,CAACP,GAAG,CAAC;MACxB;IACF;;EAAC;IAAA;IAAA;IAGD,qBAAsBA,GAAU,EAAE;MAChC,IAAI,CAAC,IAAI,CAACQ,UAAU,EAAE;QACpB,IAAMC,cAAc,GAAG,IAAI,CAACC,kBAAkB,EAAE;QAChD,IAAI,CAACF,UAAU,GAAG,IAAIC,cAAc,CAAC,IAAI,CAAChB,MAAM,EAAE,IAAI,CAACC,OAAO,CAAC;MACjE;MACA,IAAI,CAACc,UAAU,CAACF,WAAW,CAACN,GAAG,CAAC;IAClC;;EAAC;IAAA;IAAA;IAGD,sBAAuBA,GAAgC,EAAQ;MAC7D,IAAI,CAAC,IAAI,CAACQ,UAAU,EAAE;QACpB,IAAMC,cAAc,GAAG,IAAI,CAACC,kBAAkB,EAAE;QAChD,IAAI,CAACF,UAAU,GAAG,IAAIC,cAAc,CAAC,IAAI,CAAChB,MAAM,EAAE,IAAI,CAACC,OAAO,CAAC;MACjE;MACA,IAAI,CAACc,UAAU,CAACD,YAAY,CAACP,GAAG,CAAC;IACnC;;EAAC;IAAA;IAAA;IAGD,uBAAcW,KAA2B,EAAQ;MAC/C,IAAIA,KAAK,YAAYC,WAAW,EAAE;QAChC,IAAI,CAACC,SAAS,IAAIF,KAAK,CAACG,UAAU;MACpC;MACA,IAAI,OAAOH,KAAK,KAAK,QAAQ,EAAE;QAC7B,IAAI,CAACE,SAAS,IAAIF,KAAK,CAACI,MAAM;MAChC;MACA,IAAI,CAACC,eAAe,GAAG,IAAI;IAC7B;EAAC;IAAA;IAAA,OAED,sBAAatB,OAAyB,EAAqB;MACzD,OAAO,IAAI,CAACuB,OAAO,EAAE,GAAG,IAAI,CAACC,SAAS,CAACxB,OAAO,CAAC,GAAG,IAAI;IACxD;EAAC;IAAA;IAAA,OAED,uBAAcA,OAAyB,EAAqB;MAC1D,OAAO,IAAI,CAACwB,SAAS,CAACxB,OAAO,CAAC;IAChC;;EAAC;IAAA;IAAA;;IAID,wBAAeM,GAAmB,EAAU;MAC1C,OAAOI,KAAK,CAACC,OAAO,CAACL,GAAG,CAAC,GAAGA,GAAG,CAACe,MAAM,GAAG,CAAC,GAAGI,MAAM,CAACC,IAAI,CAACpB,GAAG,CAAC,CAACe,MAAM,GAAG,CAAC;IAC1E;EAAC;IAAA;IAAA,OAED,mBAA2B;MAEzB,IAAI,CAAC,IAAI,CAACP,UAAU,IAAI,IAAI,CAACA,UAAU,CAACa,QAAQ,EAAE,KAAK,CAAC,EAAE;QACxD,OAAO,KAAK;MACd;;MAIA,IAAI,IAAI,CAAC3B,OAAO,CAACP,SAAS,KAAK,MAAM,EAAE;QACrC,IAAI,CAAC,IAAI,CAAC6B,eAAe,EAAE;UACzB,OAAO,KAAK;QACd;MACF,CAAC,MAAM,IAAI,IAAI,CAACtB,OAAO,CAACP,SAAS,GAAG,IAAI,CAACqB,UAAU,CAACa,QAAQ,EAAE,EAAE;QAC9D,OAAO,KAAK;MACd;;MAGA,IAAI,IAAI,CAAC3B,OAAO,CAACN,eAAe,GAAGO,IAAI,CAACC,GAAG,EAAE,GAAG,IAAI,CAAC0B,kBAAkB,EAAE;QACvE,OAAO,KAAK;MACd;;MAGA,IAAI,CAACN,eAAe,GAAG,KAAK;MAC5B,IAAI,CAACM,kBAAkB,GAAG3B,IAAI,CAACC,GAAG,EAAE;MACpC,OAAO,IAAI;IACb;;EAAC;IAAA;IAAA;IAKD,mBAAkBF,OAAyB,EAAqB;MAC9D,IAAI,CAAC,IAAI,CAACc,UAAU,EAAE;QACpB,OAAO,IAAI;MACb;;MAGA,IAAId,OAAO,aAAPA,OAAO,eAAPA,OAAO,CAAEmB,SAAS,EAAE;QACtB,IAAI,CAACA,SAAS,GAAGnB,OAAO,CAACmB,SAAS;MACpC;MACA,IAAMU,eAAe,GAAG,IAAI,CAACf,UAAU,CAACgB,QAAQ,EAAgB;MAChED,eAAe,CAACE,KAAK,GAAG,IAAI,CAACC,UAAU;MACvCH,eAAe,CAACV,SAAS,GAAG,IAAI,CAACA,SAAS;MAC1CM,MAAM,CAACQ,MAAM,CAACJ,eAAe,EAAE7B,OAAO,CAAC;MAEvC,IAAI,CAACgC,UAAU,EAAE;MACjB,IAAI,CAAClB,UAAU,GAAG,IAAI;MACtB,OAAOe,eAAe;IACxB;EAAC;IAAA;IAAA,OAED,8BAAoD;MAClD,QAAQ,IAAI,CAAC7B,OAAO,CAACR,KAAK;QACxB,KAAK,WAAW;UACd,OAAO0C,iCAAwB;QACjC,KAAK,iBAAiB;QACtB,KAAK,kBAAkB;UACrB,OAAOC,gCAAuB;QAChC,KAAK,gBAAgB;UACnB,OAAOC,qCAA4B;QACrC,KAAK,aAAa;UAChB,IAAI,CAACtC,iBAAiB,CAACuC,UAAU,EAAE;YACjC,MAAM,IAAIC,KAAK,CAACzC,WAAW,CAAC;UAC9B;UACA,OAAOC,iBAAiB,CAACuC,UAAU;QACrC;UACE,MAAM,IAAIC,KAAK,CAACzC,WAAW,CAAC;MAAC;IAEnC;EAAC;EAAA;AAAA;AAAA;AAAA,8BA1JkBC,iBAAiB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"
|
|
1
|
+
{"version":3,"file":"enum.js","names":["Type"],"sources":["../../../../../src/lib/schema/impl/enum.ts"],"sourcesContent":["// This code is adapted from ArrowJS https://github.com/apache/arrow\n// under Apache license http://www.apache.org/licenses/LICENSE-2.0\n\n/**\n * Main data type enumeration.\n *\n * Data types in this library are all *logical*. They can be expressed as\n * either a primitive physical type (bytes or bits of some fixed size), a\n * nested type consisting of other data types, or another data type (e.g. a\n * timestamp encoded as an int64).\n *\n * **Note**: Only enum values 0-17 (NONE through Map) are written to an Arrow\n * IPC payload.\n *\n * The rest of the values are specified here so TypeScript can narrow the type\n * signatures further beyond the base Arrow Types. The Arrow DataTypes include\n * metadata like `bitWidth` that impact the type signatures of the values we\n * accept and return.\n *\n * For example, the `Int8Vector` reads 1-byte numbers from an `Int8Array`, an\n * `Int32Vector` reads a 4-byte number from an `Int32Array`, and an `Int64Vector`\n * reads a pair of 4-byte lo, hi 32-bit integers as a zero-copy slice from the\n * underlying `Int32Array`.\n *\n * Library consumers benefit by knowing the narrowest type, since we can ensure\n * the types across all public methods are propagated, and never bail to `any`.\n * These values are _never_ used at runtime, and they will _never_ be written\n * to the flatbuffers metadata of serialized Arrow IPC payloads.\n */\nexport enum Type {\n /** The default placeholder type */\n NONE = 0,\n /** A NULL type having no physical storage */\n Null = 1,\n /** Signed or unsigned 8, 16, 32, or 64-bit little-endian integer */\n Int = 2,\n /** 2, 4, or 8-byte floating point value */\n Float = 3,\n /** Variable-length bytes (no guarantee of UTF8-ness) */\n Binary = 4,\n /** UTF8 variable-length string as List<Char> */\n Utf8 = 5,\n /** Boolean as 1 bit, LSB bit-packed ordering */\n Bool = 6,\n /** Precision-and-scale-based decimal type. Storage type depends on the parameters. */\n Decimal = 7,\n /** int32_t days or int64_t milliseconds since the UNIX epoch */\n Date = 8,\n /** Time as signed 32 or 64-bit integer, representing either seconds, milliseconds, microseconds, or nanoseconds since midnight since midnight */\n Time = 9,\n /** Exact timestamp encoded with int64 since UNIX epoch (Default unit millisecond) */\n Timestamp = 10,\n /** YEAR_MONTH or DAY_TIME interval in SQL style */\n Interval = 11,\n /** A list of some logical data type */\n List = 12,\n /** Struct of logical types */\n Struct = 13,\n /** Union of logical types */\n Union = 14,\n /** Fixed-size binary. Each value occupies the same number of bytes */\n FixedSizeBinary = 15,\n /** Fixed-size list. Each value occupies the same number of bytes */\n FixedSizeList = 16,\n /** Map of named logical types */\n Map = 17,\n\n /** Dictionary aka Category type */\n Dictionary = -1,\n Int8 = -2,\n Int16 = -3,\n Int32 = -4,\n Int64 = -5,\n Uint8 = -6,\n Uint16 = -7,\n Uint32 = -8,\n Uint64 = -9,\n Float16 = -10,\n Float32 = -11,\n Float64 = -12,\n DateDay = -13,\n DateMillisecond = -14,\n TimestampSecond = -15,\n TimestampMillisecond = -16,\n TimestampMicrosecond = -17,\n TimestampNanosecond = -18,\n TimeSecond = -19,\n TimeMillisecond = -20,\n TimeMicrosecond = -21,\n TimeNanosecond = -22,\n DenseUnion = -23,\n SparseUnion = -24,\n IntervalDayTime = -25,\n IntervalYearMonth = -26\n}\n"],"mappings":";;;;;;IA6BYA,IAAI;AAAA;AAAA,WAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;EAAJA,IAAI,CAAJA,IAAI;AAAA,GAAJA,IAAI,oBAAJA,IAAI"}
|
|
@@ -1,18 +1,13 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
|
|
4
|
-
|
|
5
4
|
Object.defineProperty(exports, "__esModule", {
|
|
6
5
|
value: true
|
|
7
6
|
});
|
|
8
7
|
exports.default = void 0;
|
|
9
|
-
|
|
10
8
|
var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
|
|
11
|
-
|
|
12
9
|
var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
|
|
13
|
-
|
|
14
10
|
var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
|
|
15
|
-
|
|
16
11
|
var Field = function () {
|
|
17
12
|
function Field(name, type) {
|
|
18
13
|
var nullable = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : false;
|
|
@@ -27,7 +22,6 @@ var Field = function () {
|
|
|
27
22
|
this.nullable = nullable;
|
|
28
23
|
this.metadata = metadata;
|
|
29
24
|
}
|
|
30
|
-
|
|
31
25
|
(0, _createClass2.default)(Field, [{
|
|
32
26
|
key: "typeId",
|
|
33
27
|
get: function get() {
|
|
@@ -51,6 +45,5 @@ var Field = function () {
|
|
|
51
45
|
}]);
|
|
52
46
|
return Field;
|
|
53
47
|
}();
|
|
54
|
-
|
|
55
48
|
exports.default = Field;
|
|
56
49
|
//# sourceMappingURL=field.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"
|
|
1
|
+
{"version":3,"file":"field.js","names":["Field","name","type","nullable","metadata","Map","typeId","other"],"sources":["../../../../../src/lib/schema/impl/field.ts"],"sourcesContent":["import {DataType} from './type';\n\n/**\n * ArrowJS `Field` API-compatible class for row-based tables\n * https://loaders.gl/arrowjs/docs/api-reference/field\n * A field holds name, nullable, and metadata information about a table \"column\"\n * A Schema is essentially a list of fields\n */\nexport default class Field {\n name: string;\n type: DataType;\n nullable: boolean;\n metadata: Map<string, string>;\n\n constructor(\n name: string,\n type: DataType,\n nullable = false,\n metadata: Map<string, string> = new Map()\n ) {\n this.name = name;\n this.type = type;\n this.nullable = nullable;\n this.metadata = metadata;\n }\n\n get typeId(): number {\n return this.type && this.type.typeId;\n }\n\n clone(): Field {\n return new Field(this.name, this.type, this.nullable, this.metadata);\n }\n\n compareTo(other: this): boolean {\n return (\n this.name === other.name &&\n this.type === other.type &&\n this.nullable === other.nullable &&\n this.metadata === other.metadata\n );\n }\n\n toString(): string {\n return `${this.type}${this.nullable ? ', nullable' : ''}${\n this.metadata ? `, metadata: ${this.metadata}` : ''\n }`;\n }\n}\n"],"mappings":";;;;;;;;;;IAQqBA,KAAK;EAMxB,eACEC,IAAY,EACZC,IAAc,EAGd;IAAA,IAFAC,QAAQ,uEAAG,KAAK;IAAA,IAChBC,QAA6B,uEAAG,IAAIC,GAAG,EAAE;IAAA;IAAA;IAAA;IAAA;IAAA;IAEzC,IAAI,CAACJ,IAAI,GAAGA,IAAI;IAChB,IAAI,CAACC,IAAI,GAAGA,IAAI;IAChB,IAAI,CAACC,QAAQ,GAAGA,QAAQ;IACxB,IAAI,CAACC,QAAQ,GAAGA,QAAQ;EAC1B;EAAC;IAAA;IAAA,KAED,eAAqB;MACnB,OAAO,IAAI,CAACF,IAAI,IAAI,IAAI,CAACA,IAAI,CAACI,MAAM;IACtC;EAAC;IAAA;IAAA,OAED,iBAAe;MACb,OAAO,IAAIN,KAAK,CAAC,IAAI,CAACC,IAAI,EAAE,IAAI,CAACC,IAAI,EAAE,IAAI,CAACC,QAAQ,EAAE,IAAI,CAACC,QAAQ,CAAC;IACtE;EAAC;IAAA;IAAA,OAED,mBAAUG,KAAW,EAAW;MAC9B,OACE,IAAI,CAACN,IAAI,KAAKM,KAAK,CAACN,IAAI,IACxB,IAAI,CAACC,IAAI,KAAKK,KAAK,CAACL,IAAI,IACxB,IAAI,CAACC,QAAQ,KAAKI,KAAK,CAACJ,QAAQ,IAChC,IAAI,CAACC,QAAQ,KAAKG,KAAK,CAACH,QAAQ;IAEpC;EAAC;IAAA;IAAA,OAED,oBAAmB;MACjB,iBAAU,IAAI,CAACF,IAAI,SAAG,IAAI,CAACC,QAAQ,GAAG,YAAY,GAAG,EAAE,SACrD,IAAI,CAACC,QAAQ,yBAAkB,IAAI,CAACA,QAAQ,IAAK,EAAE;IAEvD;EAAC;EAAA;AAAA;AAAA"}
|
|
@@ -1,29 +1,20 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
|
|
4
|
-
|
|
5
4
|
Object.defineProperty(exports, "__esModule", {
|
|
6
5
|
value: true
|
|
7
6
|
});
|
|
8
7
|
exports.default = void 0;
|
|
9
|
-
|
|
10
8
|
var _toConsumableArray2 = _interopRequireDefault(require("@babel/runtime/helpers/toConsumableArray"));
|
|
11
|
-
|
|
12
9
|
var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
|
|
13
|
-
|
|
14
10
|
var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
|
|
15
|
-
|
|
16
11
|
var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
|
|
17
|
-
|
|
18
12
|
var _assert = require("../../utils/assert");
|
|
19
|
-
|
|
20
13
|
function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it.return != null) it.return(); } finally { if (didErr) throw err; } } }; }
|
|
21
|
-
|
|
22
14
|
function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
|
|
23
|
-
|
|
24
15
|
function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
|
|
25
|
-
|
|
26
16
|
var Schema = function () {
|
|
17
|
+
|
|
27
18
|
function Schema(fields, metadata) {
|
|
28
19
|
(0, _classCallCheck2.default)(this, Schema);
|
|
29
20
|
(0, _defineProperty2.default)(this, "fields", void 0);
|
|
@@ -36,37 +27,32 @@ var Schema = function () {
|
|
|
36
27
|
|
|
37
28
|
(0, _createClass2.default)(Schema, [{
|
|
38
29
|
key: "compareTo",
|
|
39
|
-
value:
|
|
30
|
+
value:
|
|
31
|
+
function compareTo(other) {
|
|
40
32
|
if (this.metadata !== other.metadata) {
|
|
41
33
|
return false;
|
|
42
34
|
}
|
|
43
|
-
|
|
44
35
|
if (this.fields.length !== other.fields.length) {
|
|
45
36
|
return false;
|
|
46
37
|
}
|
|
47
|
-
|
|
48
38
|
for (var i = 0; i < this.fields.length; ++i) {
|
|
49
39
|
if (!this.fields[i].compareTo(other.fields[i])) {
|
|
50
40
|
return false;
|
|
51
41
|
}
|
|
52
42
|
}
|
|
53
|
-
|
|
54
43
|
return true;
|
|
55
44
|
}
|
|
56
45
|
}, {
|
|
57
46
|
key: "select",
|
|
58
47
|
value: function select() {
|
|
59
48
|
var nameMap = Object.create(null);
|
|
60
|
-
|
|
61
49
|
for (var _len = arguments.length, columnNames = new Array(_len), _key = 0; _key < _len; _key++) {
|
|
62
50
|
columnNames[_key] = arguments[_key];
|
|
63
51
|
}
|
|
64
|
-
|
|
65
52
|
for (var _i = 0, _columnNames = columnNames; _i < _columnNames.length; _i++) {
|
|
66
53
|
var name = _columnNames[_i];
|
|
67
54
|
nameMap[name] = true;
|
|
68
55
|
}
|
|
69
|
-
|
|
70
56
|
var selectedFields = this.fields.filter(function (field) {
|
|
71
57
|
return nameMap[field.name];
|
|
72
58
|
});
|
|
@@ -76,11 +62,9 @@ var Schema = function () {
|
|
|
76
62
|
key: "selectAt",
|
|
77
63
|
value: function selectAt() {
|
|
78
64
|
var _this = this;
|
|
79
|
-
|
|
80
65
|
for (var _len2 = arguments.length, columnIndices = new Array(_len2), _key2 = 0; _key2 < _len2; _key2++) {
|
|
81
66
|
columnIndices[_key2] = arguments[_key2];
|
|
82
67
|
}
|
|
83
|
-
|
|
84
68
|
var selectedFields = columnIndices.map(function (index) {
|
|
85
69
|
return _this.fields[index];
|
|
86
70
|
}).filter(Boolean);
|
|
@@ -91,7 +75,6 @@ var Schema = function () {
|
|
|
91
75
|
value: function assign(schemaOrFields) {
|
|
92
76
|
var fields;
|
|
93
77
|
var metadata = this.metadata;
|
|
94
|
-
|
|
95
78
|
if (schemaOrFields instanceof Schema) {
|
|
96
79
|
var otherSchema = schemaOrFields;
|
|
97
80
|
fields = otherSchema.fields;
|
|
@@ -101,10 +84,8 @@ var Schema = function () {
|
|
|
101
84
|
}
|
|
102
85
|
|
|
103
86
|
var fieldMap = Object.create(null);
|
|
104
|
-
|
|
105
87
|
var _iterator = _createForOfIteratorHelper(this.fields),
|
|
106
|
-
|
|
107
|
-
|
|
88
|
+
_step;
|
|
108
89
|
try {
|
|
109
90
|
for (_iterator.s(); !(_step = _iterator.n()).done;) {
|
|
110
91
|
var field = _step.value;
|
|
@@ -115,10 +96,8 @@ var Schema = function () {
|
|
|
115
96
|
} finally {
|
|
116
97
|
_iterator.f();
|
|
117
98
|
}
|
|
118
|
-
|
|
119
99
|
var _iterator2 = _createForOfIteratorHelper(fields),
|
|
120
|
-
|
|
121
|
-
|
|
100
|
+
_step2;
|
|
122
101
|
try {
|
|
123
102
|
for (_iterator2.s(); !(_step2 = _iterator2.n()).done;) {
|
|
124
103
|
var _field = _step2.value;
|
|
@@ -129,30 +108,23 @@ var Schema = function () {
|
|
|
129
108
|
} finally {
|
|
130
109
|
_iterator2.f();
|
|
131
110
|
}
|
|
132
|
-
|
|
133
111
|
var mergedFields = Object.values(fieldMap);
|
|
134
112
|
return new Schema(mergedFields, metadata);
|
|
135
113
|
}
|
|
136
114
|
}]);
|
|
137
115
|
return Schema;
|
|
138
116
|
}();
|
|
139
|
-
|
|
140
117
|
exports.default = Schema;
|
|
141
|
-
|
|
142
118
|
function checkNames(fields) {
|
|
143
119
|
var usedNames = {};
|
|
144
|
-
|
|
145
120
|
var _iterator3 = _createForOfIteratorHelper(fields),
|
|
146
|
-
|
|
147
|
-
|
|
121
|
+
_step3;
|
|
148
122
|
try {
|
|
149
123
|
for (_iterator3.s(); !(_step3 = _iterator3.n()).done;) {
|
|
150
124
|
var field = _step3.value;
|
|
151
|
-
|
|
152
125
|
if (usedNames[field.name]) {
|
|
153
126
|
console.warn('Schema: duplicated field name', field.name, field);
|
|
154
127
|
}
|
|
155
|
-
|
|
156
128
|
usedNames[field.name] = true;
|
|
157
129
|
}
|
|
158
130
|
} catch (err) {
|
|
@@ -161,7 +133,6 @@ function checkNames(fields) {
|
|
|
161
133
|
_iterator3.f();
|
|
162
134
|
}
|
|
163
135
|
}
|
|
164
|
-
|
|
165
136
|
function mergeMaps(m1, m2) {
|
|
166
137
|
return new Map([].concat((0, _toConsumableArray2.default)(m1 || new Map()), (0, _toConsumableArray2.default)(m2 || new Map())));
|
|
167
138
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"
|
|
1
|
+
{"version":3,"file":"schema.js","names":["Schema","fields","metadata","assert","Array","isArray","checkNames","Map","other","length","i","compareTo","nameMap","Object","create","columnNames","name","selectedFields","filter","field","columnIndices","map","index","Boolean","schemaOrFields","otherSchema","mergeMaps","fieldMap","mergedFields","values","usedNames","console","warn","m1","m2"],"sources":["../../../../../src/lib/schema/impl/schema.ts"],"sourcesContent":["import {assert} from '../../utils/assert';\nimport Field from './field';\n\nexport type SchemaMetadata = Map<string, any>;\n\n/**\n * ArrowJS `Schema` API-compatible class for row-based tables (returned from `DataTable`)\n * https://loaders.gl/arrowjs/docs/api-reference/schema\n */\nexport default class Schema {\n fields: Field[];\n // TODO - Arrow just allows Map<string, string>\n metadata: SchemaMetadata;\n\n constructor(fields: Field[], metadata?: SchemaMetadata) {\n assert(Array.isArray(fields));\n checkNames(fields);\n // For kepler fields, create arrow compatible `Fields` that have kepler fields as `metadata`\n this.fields = fields;\n this.metadata = metadata || new Map();\n }\n\n // TODO - arrow only seems to compare fields, not metadata\n compareTo(other: Schema): boolean {\n if (this.metadata !== other.metadata) {\n return false;\n }\n if (this.fields.length !== other.fields.length) {\n return false;\n }\n for (let i = 0; i < this.fields.length; ++i) {\n if (!this.fields[i].compareTo(other.fields[i])) {\n return false;\n }\n }\n return true;\n }\n\n select(...columnNames: string[]): Schema {\n // Ensure column names reference valid fields\n const nameMap = Object.create(null);\n for (const name of columnNames) {\n nameMap[name] = true;\n }\n const selectedFields = this.fields.filter((field) => nameMap[field.name]);\n return new Schema(selectedFields, this.metadata);\n }\n\n selectAt(...columnIndices: number[]): Schema {\n // Ensure column indices reference valid fields\n const selectedFields = columnIndices.map((index) => this.fields[index]).filter(Boolean);\n return new Schema(selectedFields, this.metadata);\n }\n\n assign(schemaOrFields: Schema | Field[]): Schema {\n let fields: Field[];\n let metadata: SchemaMetadata = this.metadata;\n\n if (schemaOrFields instanceof Schema) {\n const otherSchema = schemaOrFields;\n fields = otherSchema.fields;\n metadata = mergeMaps(mergeMaps(new Map(), this.metadata), otherSchema.metadata);\n } else {\n fields = schemaOrFields;\n }\n\n // Create a merged list of fields, overwrite fields in place, new fields at end\n const fieldMap: {[key: string]: Field} = Object.create(null);\n\n for (const field of this.fields) {\n fieldMap[field.name] = field;\n }\n\n for (const field of fields) {\n fieldMap[field.name] = field;\n }\n\n const mergedFields = Object.values(fieldMap);\n\n return new Schema(mergedFields, metadata);\n }\n}\n\n// Warn if any duplicated field names\nfunction checkNames(fields: Field[]): void {\n const usedNames: Record<string, boolean> = {};\n for (const field of fields) {\n if (usedNames[field.name]) {\n // eslint-disable-next-line\n console.warn('Schema: duplicated field name', field.name, field);\n }\n usedNames[field.name] = true;\n }\n}\n\nfunction mergeMaps<T>(m1: T, m2: T): T {\n // @ts-ignore\n return new Map([...(m1 || new Map()), ...(m2 || new Map())]);\n}\n"],"mappings":";;;;;;;;;;;AAAA;AAA0C;AAAA;AAAA;AAAA,IASrBA,MAAM;;EAKzB,gBAAYC,MAAe,EAAEC,QAAyB,EAAE;IAAA;IAAA;IAAA;IACtD,IAAAC,cAAM,EAACC,KAAK,CAACC,OAAO,CAACJ,MAAM,CAAC,CAAC;IAC7BK,UAAU,CAACL,MAAM,CAAC;IAElB,IAAI,CAACA,MAAM,GAAGA,MAAM;IACpB,IAAI,CAACC,QAAQ,GAAGA,QAAQ,IAAI,IAAIK,GAAG,EAAE;EACvC;;EAAC;IAAA;IAAA;IAGD,mBAAUC,KAAa,EAAW;MAChC,IAAI,IAAI,CAACN,QAAQ,KAAKM,KAAK,CAACN,QAAQ,EAAE;QACpC,OAAO,KAAK;MACd;MACA,IAAI,IAAI,CAACD,MAAM,CAACQ,MAAM,KAAKD,KAAK,CAACP,MAAM,CAACQ,MAAM,EAAE;QAC9C,OAAO,KAAK;MACd;MACA,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG,IAAI,CAACT,MAAM,CAACQ,MAAM,EAAE,EAAEC,CAAC,EAAE;QAC3C,IAAI,CAAC,IAAI,CAACT,MAAM,CAACS,CAAC,CAAC,CAACC,SAAS,CAACH,KAAK,CAACP,MAAM,CAACS,CAAC,CAAC,CAAC,EAAE;UAC9C,OAAO,KAAK;QACd;MACF;MACA,OAAO,IAAI;IACb;EAAC;IAAA;IAAA,OAED,kBAAyC;MAEvC,IAAME,OAAO,GAAGC,MAAM,CAACC,MAAM,CAAC,IAAI,CAAC;MAAC,kCAF5BC,WAAW;QAAXA,WAAW;MAAA;MAGnB,gCAAmBA,WAAW,kCAAE;QAA3B,IAAMC,IAAI;QACbJ,OAAO,CAACI,IAAI,CAAC,GAAG,IAAI;MACtB;MACA,IAAMC,cAAc,GAAG,IAAI,CAAChB,MAAM,CAACiB,MAAM,CAAC,UAACC,KAAK;QAAA,OAAKP,OAAO,CAACO,KAAK,CAACH,IAAI,CAAC;MAAA,EAAC;MACzE,OAAO,IAAIhB,MAAM,CAACiB,cAAc,EAAE,IAAI,CAACf,QAAQ,CAAC;IAClD;EAAC;IAAA;IAAA,OAED,oBAA6C;MAAA;MAAA,mCAAjCkB,aAAa;QAAbA,aAAa;MAAA;MAEvB,IAAMH,cAAc,GAAGG,aAAa,CAACC,GAAG,CAAC,UAACC,KAAK;QAAA,OAAK,KAAI,CAACrB,MAAM,CAACqB,KAAK,CAAC;MAAA,EAAC,CAACJ,MAAM,CAACK,OAAO,CAAC;MACvF,OAAO,IAAIvB,MAAM,CAACiB,cAAc,EAAE,IAAI,CAACf,QAAQ,CAAC;IAClD;EAAC;IAAA;IAAA,OAED,gBAAOsB,cAAgC,EAAU;MAC/C,IAAIvB,MAAe;MACnB,IAAIC,QAAwB,GAAG,IAAI,CAACA,QAAQ;MAE5C,IAAIsB,cAAc,YAAYxB,MAAM,EAAE;QACpC,IAAMyB,WAAW,GAAGD,cAAc;QAClCvB,MAAM,GAAGwB,WAAW,CAACxB,MAAM;QAC3BC,QAAQ,GAAGwB,SAAS,CAACA,SAAS,CAAC,IAAInB,GAAG,EAAE,EAAE,IAAI,CAACL,QAAQ,CAAC,EAAEuB,WAAW,CAACvB,QAAQ,CAAC;MACjF,CAAC,MAAM;QACLD,MAAM,GAAGuB,cAAc;MACzB;;MAGA,IAAMG,QAAgC,GAAGd,MAAM,CAACC,MAAM,CAAC,IAAI,CAAC;MAAC,2CAEzC,IAAI,CAACb,MAAM;QAAA;MAAA;QAA/B,oDAAiC;UAAA,IAAtBkB,KAAK;UACdQ,QAAQ,CAACR,KAAK,CAACH,IAAI,CAAC,GAAGG,KAAK;QAC9B;MAAC;QAAA;MAAA;QAAA;MAAA;MAAA,4CAEmBlB,MAAM;QAAA;MAAA;QAA1B,uDAA4B;UAAA,IAAjBkB,MAAK;UACdQ,QAAQ,CAACR,MAAK,CAACH,IAAI,CAAC,GAAGG,MAAK;QAC9B;MAAC;QAAA;MAAA;QAAA;MAAA;MAED,IAAMS,YAAY,GAAGf,MAAM,CAACgB,MAAM,CAACF,QAAQ,CAAC;MAE5C,OAAO,IAAI3B,MAAM,CAAC4B,YAAY,EAAE1B,QAAQ,CAAC;IAC3C;EAAC;EAAA;AAAA;AAAA;AAIH,SAASI,UAAU,CAACL,MAAe,EAAQ;EACzC,IAAM6B,SAAkC,GAAG,CAAC,CAAC;EAAC,4CAC1B7B,MAAM;IAAA;EAAA;IAA1B,uDAA4B;MAAA,IAAjBkB,KAAK;MACd,IAAIW,SAAS,CAACX,KAAK,CAACH,IAAI,CAAC,EAAE;QAEzBe,OAAO,CAACC,IAAI,CAAC,+BAA+B,EAAEb,KAAK,CAACH,IAAI,EAAEG,KAAK,CAAC;MAClE;MACAW,SAAS,CAACX,KAAK,CAACH,IAAI,CAAC,GAAG,IAAI;IAC9B;EAAC;IAAA;EAAA;IAAA;EAAA;AACH;AAEA,SAASU,SAAS,CAAIO,EAAK,EAAEC,EAAK,EAAK;EAErC,OAAO,IAAI3B,GAAG,4CAAM0B,EAAE,IAAI,IAAI1B,GAAG,EAAE,oCAAO2B,EAAE,IAAI,IAAI3B,GAAG,EAAE,GAAG;AAC9D"}
|