@deephaven/console 0.46.1-beta.4 → 0.46.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/csv/CsvInputBar.d.ts +1 -1
- package/dist/csv/CsvInputBar.d.ts.map +1 -1
- package/dist/csv/CsvInputBar.js.map +1 -1
- package/dist/csv/CsvParser.d.ts +4 -2
- package/dist/csv/CsvParser.d.ts.map +1 -1
- package/dist/csv/CsvParser.js +21 -7
- package/dist/csv/CsvParser.js.map +1 -1
- package/dist/csv/CsvTypeParser.d.ts +4 -3
- package/dist/csv/CsvTypeParser.d.ts.map +1 -1
- package/dist/csv/CsvTypeParser.js +16 -7
- package/dist/csv/CsvTypeParser.js.map +1 -1
- package/dist/csv/ZipStreamHelper.d.ts +14 -0
- package/dist/csv/ZipStreamHelper.d.ts.map +1 -0
- package/dist/csv/ZipStreamHelper.js +15 -0
- package/dist/csv/ZipStreamHelper.js.map +1 -0
- package/package.json +13 -13
|
@@ -43,7 +43,7 @@ declare class CsvInputBar extends Component<CsvInputBarProps, CsvInputBarState>
|
|
|
43
43
|
handleTableName(event: ChangeEvent<HTMLInputElement>): void;
|
|
44
44
|
toggleFirstRowHeaders(): void;
|
|
45
45
|
handleUpload(event: FormEvent<HTMLFormElement>): void;
|
|
46
|
-
handleFile(file: Blob | JSZipObject, isZip?: boolean): void;
|
|
46
|
+
handleFile(file: File | Blob | JSZipObject, isZip?: boolean): void;
|
|
47
47
|
handleZipFile(zipFile: File): void;
|
|
48
48
|
handleProgress(progressValue: number): boolean;
|
|
49
49
|
handleCancelInProgress(): void;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CsvInputBar.d.ts","sourceRoot":"","sources":["../../src/csv/CsvInputBar.tsx"],"names":[],"mappings":"AAAA,OAAc,EACZ,WAAW,EACX,SAAS,EACT,SAAS,EACT,YAAY,EACZ,SAAS,EACV,MAAM,OAAO,CAAC;AAEf,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,OAAO,CAAC;AAEzC,OAAO,KAAK,EAAE,UAAU,EAAS,MAAM,wBAAwB,CAAC;AAIhE,OAAO,SAAS,MAAM,aAAa,CAAC;AACpC,OAAO,UAAU,MAAM,cAAc,CAAC;AACtC,OAAO,oBAAoB,CAAC;AAU5B,UAAU,gBAAgB;IACxB,OAAO,EAAE,UAAU,CAAC;IACpB,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IACpC,OAAO,EAAE,MAAM,IAAI,CAAC;IACpB,QAAQ,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;IACnC,OAAO,EAAE,CAAC,CAAC,EAAE,OAAO,KAAK,IAAI,CAAC;IAC9B,IAAI,EAAE,IAAI,CAAC;IACX,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,IAAI,CAAC;IACzC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,KAAK,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;CACnD;AAED,UAAU,gBAAgB;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,OAAO,CAAC;IACtB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,YAAY,EAAE,OAAO,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,OAAO,UAAU,CAAC,KAAK,CAAC;IACpC,MAAM,EAAE,SAAS,GAAG,IAAI,CAAC;CAC1B;AACD;;GAEG;AACH,cAAM,WAAY,SAAQ,SAAS,CAAC,gBAAgB,EAAE,gBAAgB,CAAC;IACrE,MAAM,CAAC,YAAY;;;;MAIjB;gBAEU,KAAK,EAAE,gBAAgB;IA2BnC,kBAAkB,CAAC,SAAS,EAAE,gBAAgB,GAAG,IAAI;IA8CrD,oBAAoB,IAAI,IAAI;IAO5B,QAAQ,EAAE,SAAS,CAAC,gBAAgB,CAAC,CAAC;IAEtC,YAAY,IAAI,IAAI;IAKpB,WAAW,CAAC,CAAC,EAAE,OAAO,GAAG,IAAI;IAO7B,eAAe,CAAC,KAAK,EAAE,WAAW,CAAC,gBAAgB,CAAC,GAAG,IAAI;IAI3D,qBAAqB,IAAI,IAAI;IAK7B,YAAY,CAAC,KAAK,EAAE,SAAS,CAAC,eAAe,CAAC,GAAG,IAAI;IAmBrD,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,WAAW,EAAE,KAAK,UAAQ,GAAG,IAAI;
|
|
1
|
+
{"version":3,"file":"CsvInputBar.d.ts","sourceRoot":"","sources":["../../src/csv/CsvInputBar.tsx"],"names":[],"mappings":"AAAA,OAAc,EACZ,WAAW,EACX,SAAS,EACT,SAAS,EACT,YAAY,EACZ,SAAS,EACV,MAAM,OAAO,CAAC;AAEf,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,OAAO,CAAC;AAEzC,OAAO,KAAK,EAAE,UAAU,EAAS,MAAM,wBAAwB,CAAC;AAIhE,OAAO,SAAS,MAAM,aAAa,CAAC;AACpC,OAAO,UAAU,MAAM,cAAc,CAAC;AACtC,OAAO,oBAAoB,CAAC;AAU5B,UAAU,gBAAgB;IACxB,OAAO,EAAE,UAAU,CAAC;IACpB,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IACpC,OAAO,EAAE,MAAM,IAAI,CAAC;IACpB,QAAQ,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;IACnC,OAAO,EAAE,CAAC,CAAC,EAAE,OAAO,KAAK,IAAI,CAAC;IAC9B,IAAI,EAAE,IAAI,CAAC;IACX,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,IAAI,CAAC;IACzC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,KAAK,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;CACnD;AAED,UAAU,gBAAgB;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,OAAO,CAAC;IACtB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,YAAY,EAAE,OAAO,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,IAAI,EAAE,MAAM,OAAO,UAAU,CAAC,KAAK,CAAC;IACpC,MAAM,EAAE,SAAS,GAAG,IAAI,CAAC;CAC1B;AACD;;GAEG;AACH,cAAM,WAAY,SAAQ,SAAS,CAAC,gBAAgB,EAAE,gBAAgB,CAAC;IACrE,MAAM,CAAC,YAAY;;;;MAIjB;gBAEU,KAAK,EAAE,gBAAgB;IA2BnC,kBAAkB,CAAC,SAAS,EAAE,gBAAgB,GAAG,IAAI;IA8CrD,oBAAoB,IAAI,IAAI;IAO5B,QAAQ,EAAE,SAAS,CAAC,gBAAgB,CAAC,CAAC;IAEtC,YAAY,IAAI,IAAI;IAKpB,WAAW,CAAC,CAAC,EAAE,OAAO,GAAG,IAAI;IAO7B,eAAe,CAAC,KAAK,EAAE,WAAW,CAAC,gBAAgB,CAAC,GAAG,IAAI;IAI3D,qBAAqB,IAAI,IAAI;IAK7B,YAAY,CAAC,KAAK,EAAE,SAAS,CAAC,eAAe,CAAC,GAAG,IAAI;IAmBrD,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI,GAAG,WAAW,EAAE,KAAK,UAAQ,GAAG,IAAI;IAgDhE,aAAa,CAAC,OAAO,EAAE,IAAI,GAAG,IAAI;IA2BlC,cAAc,CAAC,aAAa,EAAE,MAAM,GAAG,OAAO;IAY9C,sBAAsB,IAAI,IAAI;IAa9B,SAAS,IAAI,IAAI;IAOjB,qBAAqB,CAAC,KAAK,EAAE,WAAW,CAAC,iBAAiB,CAAC,GAAG,IAAI;IAMlE,MAAM,IAAI,YAAY;CAkFvB;AAED,eAAe,WAAW,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CsvInputBar.js","names":["React","Component","classNames","Button","Checkbox","Log","DbNameValidator","CsvOverlay","CsvParser","CsvFormats","log","module","TYPE_OPTIONS","Object","entries","TYPES","map","key","value","name","CsvInputBar","constructor","props","handleUpload","bind","handleCancel","handleError","handleTableName","toggleFirstRowHeaders","handleProgress","handleCancelInProgress","handleQueryTypeChange","inputRef","createRef","state","tableName","tableNameSet","isFirstRowHeaders","showProgress","progressValue","type","DEFAULT_TYPE","parser","componentDidUpdate","prevProps","file","paste","dotIndex","lastIndexOf","fileTableName","legalizeTableName","substring","setState","current","focus","fromExtension","undefined","AUTO","componentWillUnmount","cancel","onClose","e","onError","error","event","target","stopPropagation","preventDefault","endsWith","handleZipFile","handleFile","Blob","isZip","info","File","size","session","timeZone","onInProgress","handleParseDone","tables","length","bindTableToVariable","then","openTable","catch","mergeTables","table","forEach","t","close","onFileCompleted","readHeaders","onProgress","parse","zipFile","onUpdate","unzip","Error","files","csvFound","i","f","isValidExtension","onOpenTable","render","isNameInvalid","Boolean","width"],"sources":["../../src/csv/CsvInputBar.tsx"],"sourcesContent":["import React, {\n ChangeEvent,\n Component,\n FormEvent,\n ReactElement,\n RefObject,\n} from 'react';\nimport classNames from 'classnames';\nimport type { JSZipObject } from 'jszip';\nimport { Button, Checkbox } from '@deephaven/components';\nimport type { IdeSession, Table } from '@deephaven/jsapi-types';\nimport Log from '@deephaven/log';\nimport { DbNameValidator } from '@deephaven/utils';\nimport CsvOverlay from './CsvOverlay';\nimport CsvParser from './CsvParser';\nimport CsvFormats from './CsvFormats';\nimport './CsvInputBar.scss';\n\nconst log = Log.module('CsvInputBar');\n\nconst TYPE_OPTIONS = Object.entries(CsvFormats.TYPES).map(([key, value]) => (\n <option key={key} value={key}>\n {value.name}\n </option>\n));\n\ninterface CsvInputBarProps {\n session: IdeSession;\n onOpenTable: (name: string) => void;\n onClose: () => void;\n onUpdate: (update: string) => void;\n onError: (e: unknown) => void;\n file: File;\n paste?: string;\n onInProgress: (boolean: boolean) => void;\n timeZone: string;\n unzip?: (zipFile: File) => Promise<JSZipObject[]>;\n}\n\ninterface CsvInputBarState {\n tableName: string;\n tableNameSet: boolean;\n isFirstRowHeaders: boolean;\n showProgress: boolean;\n progressValue: number;\n type: keyof typeof CsvFormats.TYPES;\n parser: CsvParser | null;\n}\n/**\n * Input controls for CSV upload.\n */\nclass CsvInputBar extends Component<CsvInputBarProps, CsvInputBarState> {\n static defaultProps = {\n file: null,\n paste: null,\n unzip: null,\n };\n\n constructor(props: CsvInputBarProps) {\n super(props);\n\n this.handleUpload = this.handleUpload.bind(this);\n this.handleCancel = this.handleCancel.bind(this);\n this.handleError = this.handleError.bind(this);\n this.handleTableName = this.handleTableName.bind(this);\n this.toggleFirstRowHeaders = this.toggleFirstRowHeaders.bind(this);\n this.handleProgress = this.handleProgress.bind(this);\n this.handleCancelInProgress = this.handleCancelInProgress.bind(this);\n this.handleQueryTypeChange = this.handleQueryTypeChange.bind(this);\n\n this.inputRef = React.createRef();\n\n this.state = {\n tableName: '',\n tableNameSet: false,\n isFirstRowHeaders: true,\n showProgress: false,\n progressValue: 0,\n type: CsvFormats.DEFAULT_TYPE,\n parser: null,\n };\n }\n\n // React documentation says it is fine to update state inside an if statment\n /* eslint-disable react/no-did-update-set-state */\n componentDidUpdate(prevProps: CsvInputBarProps): void {\n const { file, paste } = this.props;\n const { tableName, tableNameSet } = this.state;\n // Set the table name from a file\n if (!tableNameSet && file != null && !tableName) {\n const dotIndex = file.name.lastIndexOf('.');\n const fileTableName = DbNameValidator.legalizeTableName(\n file.name.substring(0, dotIndex)\n );\n this.setState({\n tableName: fileTableName,\n tableNameSet: true,\n });\n this.inputRef.current?.focus();\n } else if (\n (file == null && prevProps.file != null) ||\n (paste == null && prevProps.paste != null)\n ) {\n // The file or paste was unstaged\n this.setState({\n tableName: '',\n tableNameSet: false,\n });\n }\n\n // Focus the name input field on paste\n if (paste != null && paste !== '' && !tableName && this.inputRef.current) {\n this.inputRef.current.focus();\n }\n\n // Determine parser type by file extension\n if (file != null && file !== prevProps.file) {\n this.setState({\n type: CsvFormats.fromExtension(file.name),\n });\n } else if (\n paste != null &&\n paste !== '' &&\n (prevProps.paste === undefined || prevProps.paste === '')\n ) {\n this.setState({\n type: CsvFormats.AUTO,\n });\n }\n }\n\n componentWillUnmount(): void {\n const { parser } = this.state;\n if (parser) {\n parser.cancel();\n }\n }\n\n inputRef: RefObject<HTMLInputElement>;\n\n handleCancel(): void {\n const { onClose } = this.props;\n onClose();\n }\n\n handleError(e: unknown): void {\n const { onClose, onError } = this.props;\n log.error(e);\n onError(e);\n onClose();\n }\n\n handleTableName(event: ChangeEvent<HTMLInputElement>): void {\n this.setState({ tableName: event.target.value, tableNameSet: true });\n }\n\n toggleFirstRowHeaders(): void {\n const { isFirstRowHeaders } = this.state;\n this.setState({ isFirstRowHeaders: !isFirstRowHeaders });\n }\n\n handleUpload(event: FormEvent<HTMLFormElement>): void {\n event.stopPropagation();\n event.preventDefault();\n const { file, paste } = this.props;\n if (file != null) {\n if (file.name.endsWith('.zip')) {\n this.handleZipFile(file);\n } else {\n this.handleFile(file);\n }\n } else if (paste !== undefined && paste !== '') {\n this.handleFile(\n new Blob([paste], {\n type: 'text/plain',\n })\n );\n }\n }\n\n handleFile(file: Blob | JSZipObject, isZip = false): void {\n log.info(\n `Starting CSV parser for ${\n file instanceof File ? file.name : 'pasted values'\n } ${isZip ? '' : (file as Blob).size} bytes`\n );\n const { session, timeZone, onInProgress } = this.props;\n const { tableName, isFirstRowHeaders, type } = this.state;\n const handleParseDone = (tables: Table[]) => {\n // Do not bother merging just one table\n if (tables.length === 1) {\n session\n .bindTableToVariable(tables[0], tableName)\n .then(() => this.openTable())\n .catch(e => this.handleError(e));\n } else {\n session\n .mergeTables(tables)\n .then(table => {\n session\n .bindTableToVariable(table, tableName)\n .then(() => this.openTable())\n .catch(e => this.handleError(e));\n tables.forEach(t => t.close());\n })\n .catch(e => this.handleError(e));\n }\n };\n const parser = new CsvParser({\n onFileCompleted: handleParseDone,\n session,\n file,\n type: CsvFormats.TYPES[type],\n readHeaders: isFirstRowHeaders,\n onProgress: this.handleProgress,\n onError: this.handleError,\n timeZone,\n isZip,\n });\n parser.parse();\n this.setState({\n showProgress: true,\n parser,\n });\n // Note that calling onClose will set in progress to false\n onInProgress(true);\n }\n\n handleZipFile(zipFile: File): void {\n const { onUpdate, unzip } = this.props;\n if (unzip == null) {\n this.handleError(new Error('No support for zip files available.'));\n return;\n }\n\n unzip(zipFile)\n .then(files => {\n let csvFound = false;\n // Find the first Csv file in the zip and use that\n for (let i = 0; i < files.length; i += 1) {\n const f = files[i];\n if (CsvOverlay.isValidExtension(f.name)) {\n onUpdate(`Loading ${f.name} as csv from ${zipFile.name}`);\n csvFound = true;\n this.handleFile(f, true);\n break;\n }\n }\n if (!csvFound) {\n throw new Error(`No csv file found in ${zipFile.name}`);\n }\n })\n .catch(e => this.handleError(e));\n }\n\n handleProgress(progressValue: number): boolean {\n const { showProgress } = this.state;\n if (showProgress) {\n this.setState({\n progressValue,\n });\n }\n // Indicates to the caller that the upload has been cancelled\n return !showProgress;\n }\n\n // Cancels an in progress upload\n handleCancelInProgress(): void {\n const { onInProgress } = this.props;\n const { parser } = this.state;\n if (parser) {\n parser.cancel();\n }\n this.setState({\n showProgress: false,\n progressValue: 0,\n });\n onInProgress(false);\n }\n\n openTable(): void {\n const { onOpenTable, onClose } = this.props;\n const { tableName } = this.state;\n onOpenTable(tableName);\n onClose();\n }\n\n handleQueryTypeChange(event: ChangeEvent<HTMLSelectElement>): void {\n this.setState({\n type: event.target.value as keyof typeof CsvFormats.TYPES,\n });\n }\n\n render(): ReactElement {\n const { file, paste } = this.props;\n const { tableName, isFirstRowHeaders, showProgress, progressValue, type } =\n this.state;\n // A blank table name is invalid for pasted values\n const isNameInvalid = Boolean(paste) && !tableName;\n return (\n <div className=\"csv-input-bar\">\n {!showProgress && (\n <form\n onSubmit={this.handleUpload}\n className=\"csv-input-bar-container form-inline\"\n >\n <div className=\"form-group\">\n <label htmlFor=\"tableNameInput\">Table name</label>\n <input\n ref={this.inputRef}\n id=\"tableNameInput\"\n type=\"text\"\n className={classNames('form-control', {\n 'is-invalid': isNameInvalid,\n })}\n value={tableName}\n onChange={this.handleTableName}\n />\n </div>\n <div className=\"form-group\">\n <label htmlFor=\"formatSelect\">File format</label>\n <select\n id=\"formatSelect\"\n className=\"custom-select\"\n value={type}\n onChange={this.handleQueryTypeChange}\n >\n {TYPE_OPTIONS}\n </select>\n </div>\n <Checkbox\n className=\"firstRowHeaders\"\n checked={isFirstRowHeaders}\n onChange={this.toggleFirstRowHeaders}\n >\n First row is column headers\n </Checkbox>\n <div className=\"csv-input-buttons form-group\">\n <Button kind=\"secondary\" onClick={this.handleCancel}>\n Cancel\n </Button>\n <Button\n kind=\"primary\"\n type=\"submit\"\n disabled={\n !(file !== null || (paste !== undefined && paste !== '')) ||\n !tableName\n }\n >\n Upload\n </Button>\n </div>\n </form>\n )}\n {showProgress && (\n <div className=\"csv-progress-container\">\n <label>Uploading Table</label>\n <div className=\"progress\">\n <div\n className=\"progress-bar bg-primary\"\n style={{ width: `${progressValue}%` }}\n aria-valuenow={progressValue}\n aria-valuemin={0}\n aria-valuemax={100}\n />\n </div>\n <label>{progressValue}%</label>\n <Button kind=\"primary\" onClick={this.handleCancelInProgress}>\n Cancel\n </Button>\n </div>\n )}\n </div>\n );\n }\n}\n\nexport default CsvInputBar;\n"],"mappings":";;;AAAA,OAAOA,KAAK,IAEVC,SAAS,QAIJ,OAAO;AACd,OAAOC,UAAU,MAAM,YAAY;AAEnC,SAASC,MAAM,EAAEC,QAAQ,QAAQ,uBAAuB;AAExD,OAAOC,GAAG,MAAM,gBAAgB;AAChC,SAASC,eAAe,QAAQ,kBAAkB;AAAC,OAC5CC,UAAU;AAAA,OACVC,SAAS;AAAA,OACTC,UAAU;AAAA;AAAA;AAAA;AAGjB,IAAMC,GAAG,GAAGL,GAAG,CAACM,MAAM,CAAC,aAAa,CAAC;AAErC,IAAMC,YAAY,GAAGC,MAAM,CAACC,OAAO,CAACL,UAAU,CAACM,KAAK,CAAC,CAACC,GAAG,CAAC;EAAA,IAAC,CAACC,GAAG,EAAEC,KAAK,CAAC;EAAA,oBACrE;IAAkB,KAAK,EAAED,GAAI;IAAA,UAC1BC,KAAK,CAACC;EAAI,GADAF,GAAG,CAEP;AAAA,CACV,CAAC;AAwBF;AACA;AACA;AACA,MAAMG,WAAW,SAASnB,SAAS,CAAqC;EAOtEoB,WAAW,CAACC,KAAuB,EAAE;IACnC,KAAK,CAACA,KAAK,CAAC;IAAC;IAEb,IAAI,CAACC,YAAY,GAAG,IAAI,CAACA,YAAY,CAACC,IAAI,CAAC,IAAI,CAAC;IAChD,IAAI,CAACC,YAAY,GAAG,IAAI,CAACA,YAAY,CAACD,IAAI,CAAC,IAAI,CAAC;IAChD,IAAI,CAACE,WAAW,GAAG,IAAI,CAACA,WAAW,CAACF,IAAI,CAAC,IAAI,CAAC;IAC9C,IAAI,CAACG,eAAe,GAAG,IAAI,CAACA,eAAe,CAACH,IAAI,CAAC,IAAI,CAAC;IACtD,IAAI,CAACI,qBAAqB,GAAG,IAAI,CAACA,qBAAqB,CAACJ,IAAI,CAAC,IAAI,CAAC;IAClE,IAAI,CAACK,cAAc,GAAG,IAAI,CAACA,cAAc,CAACL,IAAI,CAAC,IAAI,CAAC;IACpD,IAAI,CAACM,sBAAsB,GAAG,IAAI,CAACA,sBAAsB,CAACN,IAAI,CAAC,IAAI,CAAC;IACpE,IAAI,CAACO,qBAAqB,GAAG,IAAI,CAACA,qBAAqB,CAACP,IAAI,CAAC,IAAI,CAAC;IAElE,IAAI,CAACQ,QAAQ,gBAAGhC,KAAK,CAACiC,SAAS,EAAE;IAEjC,IAAI,CAACC,KAAK,GAAG;MACXC,SAAS,EAAE,EAAE;MACbC,YAAY,EAAE,KAAK;MACnBC,iBAAiB,EAAE,IAAI;MACvBC,YAAY,EAAE,KAAK;MACnBC,aAAa,EAAE,CAAC;MAChBC,IAAI,EAAE/B,UAAU,CAACgC,YAAY;MAC7BC,MAAM,EAAE;IACV,CAAC;EACH;;EAEA;EACA;EACAC,kBAAkB,CAACC,SAA2B,EAAQ;IACpD,IAAM;MAAEC,IAAI;MAAEC;IAAM,CAAC,GAAG,IAAI,CAACxB,KAAK;IAClC,IAAM;MAAEa,SAAS;MAAEC;IAAa,CAAC,GAAG,IAAI,CAACF,KAAK;IAC9C;IACA,IAAI,CAACE,YAAY,IAAIS,IAAI,IAAI,IAAI,IAAI,CAACV,SAAS,EAAE;MAAA;MAC/C,IAAMY,QAAQ,GAAGF,IAAI,CAAC1B,IAAI,CAAC6B,WAAW,CAAC,GAAG,CAAC;MAC3C,IAAMC,aAAa,GAAG3C,eAAe,CAAC4C,iBAAiB,CACrDL,IAAI,CAAC1B,IAAI,CAACgC,SAAS,CAAC,CAAC,EAAEJ,QAAQ,CAAC,CACjC;MACD,IAAI,CAACK,QAAQ,CAAC;QACZjB,SAAS,EAAEc,aAAa;QACxBb,YAAY,EAAE;MAChB,CAAC,CAAC;MACF,6BAAI,CAACJ,QAAQ,CAACqB,OAAO,0DAArB,sBAAuBC,KAAK,EAAE;IAChC,CAAC,MAAM,IACJT,IAAI,IAAI,IAAI,IAAID,SAAS,CAACC,IAAI,IAAI,IAAI,IACtCC,KAAK,IAAI,IAAI,IAAIF,SAAS,CAACE,KAAK,IAAI,IAAK,EAC1C;MACA;MACA,IAAI,CAACM,QAAQ,CAAC;QACZjB,SAAS,EAAE,EAAE;QACbC,YAAY,EAAE;MAChB,CAAC,CAAC;IACJ;;IAEA;IACA,IAAIU,KAAK,IAAI,IAAI,IAAIA,KAAK,KAAK,EAAE,IAAI,CAACX,SAAS,IAAI,IAAI,CAACH,QAAQ,CAACqB,OAAO,EAAE;MACxE,IAAI,CAACrB,QAAQ,CAACqB,OAAO,CAACC,KAAK,EAAE;IAC/B;;IAEA;IACA,IAAIT,IAAI,IAAI,IAAI,IAAIA,IAAI,KAAKD,SAAS,CAACC,IAAI,EAAE;MAC3C,IAAI,CAACO,QAAQ,CAAC;QACZZ,IAAI,EAAE/B,UAAU,CAAC8C,aAAa,CAACV,IAAI,CAAC1B,IAAI;MAC1C,CAAC,CAAC;IACJ,CAAC,MAAM,IACL2B,KAAK,IAAI,IAAI,IACbA,KAAK,KAAK,EAAE,KACXF,SAAS,CAACE,KAAK,KAAKU,SAAS,IAAIZ,SAAS,CAACE,KAAK,KAAK,EAAE,CAAC,EACzD;MACA,IAAI,CAACM,QAAQ,CAAC;QACZZ,IAAI,EAAE/B,UAAU,CAACgD;MACnB,CAAC,CAAC;IACJ;EACF;EAEAC,oBAAoB,GAAS;IAC3B,IAAM;MAAEhB;IAAO,CAAC,GAAG,IAAI,CAACR,KAAK;IAC7B,IAAIQ,MAAM,EAAE;MACVA,MAAM,CAACiB,MAAM,EAAE;IACjB;EACF;EAIAlC,YAAY,GAAS;IACnB,IAAM;MAAEmC;IAAQ,CAAC,GAAG,IAAI,CAACtC,KAAK;IAC9BsC,OAAO,EAAE;EACX;EAEAlC,WAAW,CAACmC,CAAU,EAAQ;IAC5B,IAAM;MAAED,OAAO;MAAEE;IAAQ,CAAC,GAAG,IAAI,CAACxC,KAAK;IACvCZ,GAAG,CAACqD,KAAK,CAACF,CAAC,CAAC;IACZC,OAAO,CAACD,CAAC,CAAC;IACVD,OAAO,EAAE;EACX;EAEAjC,eAAe,CAACqC,KAAoC,EAAQ;IAC1D,IAAI,CAACZ,QAAQ,CAAC;MAAEjB,SAAS,EAAE6B,KAAK,CAACC,MAAM,CAAC/C,KAAK;MAAEkB,YAAY,EAAE;IAAK,CAAC,CAAC;EACtE;EAEAR,qBAAqB,GAAS;IAC5B,IAAM;MAAES;IAAkB,CAAC,GAAG,IAAI,CAACH,KAAK;IACxC,IAAI,CAACkB,QAAQ,CAAC;MAAEf,iBAAiB,EAAE,CAACA;IAAkB,CAAC,CAAC;EAC1D;EAEAd,YAAY,CAACyC,KAAiC,EAAQ;IACpDA,KAAK,CAACE,eAAe,EAAE;IACvBF,KAAK,CAACG,cAAc,EAAE;IACtB,IAAM;MAAEtB,IAAI;MAAEC;IAAM,CAAC,GAAG,IAAI,CAACxB,KAAK;IAClC,IAAIuB,IAAI,IAAI,IAAI,EAAE;MAChB,IAAIA,IAAI,CAAC1B,IAAI,CAACiD,QAAQ,CAAC,MAAM,CAAC,EAAE;QAC9B,IAAI,CAACC,aAAa,CAACxB,IAAI,CAAC;MAC1B,CAAC,MAAM;QACL,IAAI,CAACyB,UAAU,CAACzB,IAAI,CAAC;MACvB;IACF,CAAC,MAAM,IAAIC,KAAK,KAAKU,SAAS,IAAIV,KAAK,KAAK,EAAE,EAAE;MAC9C,IAAI,CAACwB,UAAU,CACb,IAAIC,IAAI,CAAC,CAACzB,KAAK,CAAC,EAAE;QAChBN,IAAI,EAAE;MACR,CAAC,CAAC,CACH;IACH;EACF;EAEA8B,UAAU,CAACzB,IAAwB,EAAuB;IAAA,IAArB2B,KAAK,uEAAG,KAAK;IAChD9D,GAAG,CAAC+D,IAAI,mCAEJ5B,IAAI,YAAY6B,IAAI,GAAG7B,IAAI,CAAC1B,IAAI,GAAG,eAAe,cAChDqD,KAAK,GAAG,EAAE,GAAI3B,IAAI,CAAU8B,IAAI,YACrC;IACD,IAAM;MAAEC,OAAO;MAAEC,QAAQ;MAAEC;IAAa,CAAC,GAAG,IAAI,CAACxD,KAAK;IACtD,IAAM;MAAEa,SAAS;MAAEE,iBAAiB;MAAEG;IAAK,CAAC,GAAG,IAAI,CAACN,KAAK;IACzD,IAAM6C,eAAe,GAAIC,MAAe,IAAK;MAC3C;MACA,IAAIA,MAAM,CAACC,MAAM,KAAK,CAAC,EAAE;QACvBL,OAAO,CACJM,mBAAmB,CAACF,MAAM,CAAC,CAAC,CAAC,EAAE7C,SAAS,CAAC,CACzCgD,IAAI,CAAC,MAAM,IAAI,CAACC,SAAS,EAAE,CAAC,CAC5BC,KAAK,CAACxB,CAAC,IAAI,IAAI,CAACnC,WAAW,CAACmC,CAAC,CAAC,CAAC;MACpC,CAAC,MAAM;QACLe,OAAO,CACJU,WAAW,CAACN,MAAM,CAAC,CACnBG,IAAI,CAACI,KAAK,IAAI;UACbX,OAAO,CACJM,mBAAmB,CAACK,KAAK,EAAEpD,SAAS,CAAC,CACrCgD,IAAI,CAAC,MAAM,IAAI,CAACC,SAAS,EAAE,CAAC,CAC5BC,KAAK,CAACxB,CAAC,IAAI,IAAI,CAACnC,WAAW,CAACmC,CAAC,CAAC,CAAC;UAClCmB,MAAM,CAACQ,OAAO,CAACC,CAAC,IAAIA,CAAC,CAACC,KAAK,EAAE,CAAC;QAChC,CAAC,CAAC,CACDL,KAAK,CAACxB,CAAC,IAAI,IAAI,CAACnC,WAAW,CAACmC,CAAC,CAAC,CAAC;MACpC;IACF,CAAC;IACD,IAAMnB,MAAM,GAAG,IAAIlC,SAAS,CAAC;MAC3BmF,eAAe,EAAEZ,eAAe;MAChCH,OAAO;MACP/B,IAAI;MACJL,IAAI,EAAE/B,UAAU,CAACM,KAAK,CAACyB,IAAI,CAAC;MAC5BoD,WAAW,EAAEvD,iBAAiB;MAC9BwD,UAAU,EAAE,IAAI,CAAChE,cAAc;MAC/BiC,OAAO,EAAE,IAAI,CAACpC,WAAW;MACzBmD,QAAQ;MACRL;IACF,CAAC,CAAC;IACF9B,MAAM,CAACoD,KAAK,EAAE;IACd,IAAI,CAAC1C,QAAQ,CAAC;MACZd,YAAY,EAAE,IAAI;MAClBI;IACF,CAAC,CAAC;IACF;IACAoC,YAAY,CAAC,IAAI,CAAC;EACpB;EAEAT,aAAa,CAAC0B,OAAa,EAAQ;IACjC,IAAM;MAAEC,QAAQ;MAAEC;IAAM,CAAC,GAAG,IAAI,CAAC3E,KAAK;IACtC,IAAI2E,KAAK,IAAI,IAAI,EAAE;MACjB,IAAI,CAACvE,WAAW,CAAC,IAAIwE,KAAK,CAAC,qCAAqC,CAAC,CAAC;MAClE;IACF;IAEAD,KAAK,CAACF,OAAO,CAAC,CACXZ,IAAI,CAACgB,KAAK,IAAI;MACb,IAAIC,QAAQ,GAAG,KAAK;MACpB;MACA,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGF,KAAK,CAAClB,MAAM,EAAEoB,CAAC,IAAI,CAAC,EAAE;QACxC,IAAMC,CAAC,GAAGH,KAAK,CAACE,CAAC,CAAC;QAClB,IAAI9F,UAAU,CAACgG,gBAAgB,CAACD,CAAC,CAACnF,IAAI,CAAC,EAAE;UACvC6E,QAAQ,mBAAYM,CAAC,CAACnF,IAAI,0BAAgB4E,OAAO,CAAC5E,IAAI,EAAG;UACzDiF,QAAQ,GAAG,IAAI;UACf,IAAI,CAAC9B,UAAU,CAACgC,CAAC,EAAE,IAAI,CAAC;UACxB;QACF;MACF;MACA,IAAI,CAACF,QAAQ,EAAE;QACb,MAAM,IAAIF,KAAK,gCAAyBH,OAAO,CAAC5E,IAAI,EAAG;MACzD;IACF,CAAC,CAAC,CACDkE,KAAK,CAACxB,CAAC,IAAI,IAAI,CAACnC,WAAW,CAACmC,CAAC,CAAC,CAAC;EACpC;EAEAhC,cAAc,CAACU,aAAqB,EAAW;IAC7C,IAAM;MAAED;IAAa,CAAC,GAAG,IAAI,CAACJ,KAAK;IACnC,IAAII,YAAY,EAAE;MAChB,IAAI,CAACc,QAAQ,CAAC;QACZb;MACF,CAAC,CAAC;IACJ;IACA;IACA,OAAO,CAACD,YAAY;EACtB;;EAEA;EACAR,sBAAsB,GAAS;IAC7B,IAAM;MAAEgD;IAAa,CAAC,GAAG,IAAI,CAACxD,KAAK;IACnC,IAAM;MAAEoB;IAAO,CAAC,GAAG,IAAI,CAACR,KAAK;IAC7B,IAAIQ,MAAM,EAAE;MACVA,MAAM,CAACiB,MAAM,EAAE;IACjB;IACA,IAAI,CAACP,QAAQ,CAAC;MACZd,YAAY,EAAE,KAAK;MACnBC,aAAa,EAAE;IACjB,CAAC,CAAC;IACFuC,YAAY,CAAC,KAAK,CAAC;EACrB;EAEAM,SAAS,GAAS;IAChB,IAAM;MAAEoB,WAAW;MAAE5C;IAAQ,CAAC,GAAG,IAAI,CAACtC,KAAK;IAC3C,IAAM;MAAEa;IAAU,CAAC,GAAG,IAAI,CAACD,KAAK;IAChCsE,WAAW,CAACrE,SAAS,CAAC;IACtByB,OAAO,EAAE;EACX;EAEA7B,qBAAqB,CAACiC,KAAqC,EAAQ;IACjE,IAAI,CAACZ,QAAQ,CAAC;MACZZ,IAAI,EAAEwB,KAAK,CAACC,MAAM,CAAC/C;IACrB,CAAC,CAAC;EACJ;EAEAuF,MAAM,GAAiB;IACrB,IAAM;MAAE5D,IAAI;MAAEC;IAAM,CAAC,GAAG,IAAI,CAACxB,KAAK;IAClC,IAAM;MAAEa,SAAS;MAAEE,iBAAiB;MAAEC,YAAY;MAAEC,aAAa;MAAEC;IAAK,CAAC,GACvE,IAAI,CAACN,KAAK;IACZ;IACA,IAAMwE,aAAa,GAAGC,OAAO,CAAC7D,KAAK,CAAC,IAAI,CAACX,SAAS;IAClD,oBACE;MAAK,SAAS,EAAC,eAAe;MAAA,WAC3B,CAACG,YAAY,iBACZ;QACE,QAAQ,EAAE,IAAI,CAACf,YAAa;QAC5B,SAAS,EAAC,qCAAqC;QAAA,wBAE/C;UAAK,SAAS,EAAC,YAAY;UAAA,wBACzB;YAAO,OAAO,EAAC,gBAAgB;YAAA,UAAC;UAAU,EAAQ,eAClD;YACE,GAAG,EAAE,IAAI,CAACS,QAAS;YACnB,EAAE,EAAC,gBAAgB;YACnB,IAAI,EAAC,MAAM;YACX,SAAS,EAAE9B,UAAU,CAAC,cAAc,EAAE;cACpC,YAAY,EAAEwG;YAChB,CAAC,CAAE;YACH,KAAK,EAAEvE,SAAU;YACjB,QAAQ,EAAE,IAAI,CAACR;UAAgB,EAC/B;QAAA,EACE,eACN;UAAK,SAAS,EAAC,YAAY;UAAA,wBACzB;YAAO,OAAO,EAAC,cAAc;YAAA,UAAC;UAAW,EAAQ,eACjD;YACE,EAAE,EAAC,cAAc;YACjB,SAAS,EAAC,eAAe;YACzB,KAAK,EAAEa,IAAK;YACZ,QAAQ,EAAE,IAAI,CAACT,qBAAsB;YAAA,UAEpCnB;UAAY,EACN;QAAA,EACL,eACN,KAAC,QAAQ;UACP,SAAS,EAAC,iBAAiB;UAC3B,OAAO,EAAEyB,iBAAkB;UAC3B,QAAQ,EAAE,IAAI,CAACT,qBAAsB;UAAA,UACtC;QAED,EAAW,eACX;UAAK,SAAS,EAAC,8BAA8B;UAAA,wBAC3C,KAAC,MAAM;YAAC,IAAI,EAAC,WAAW;YAAC,OAAO,EAAE,IAAI,CAACH,YAAa;YAAA,UAAC;UAErD,EAAS,eACT,KAAC,MAAM;YACL,IAAI,EAAC,SAAS;YACd,IAAI,EAAC,QAAQ;YACb,QAAQ,EACN,EAAEoB,IAAI,KAAK,IAAI,IAAKC,KAAK,KAAKU,SAAS,IAAIV,KAAK,KAAK,EAAG,CAAC,IACzD,CAACX,SACF;YAAA,UACF;UAED,EAAS;QAAA,EACL;MAAA,EAET,EACAG,YAAY,iBACX;QAAK,SAAS,EAAC,wBAAwB;QAAA,wBACrC;UAAA,UAAO;QAAe,EAAQ,eAC9B;UAAK,SAAS,EAAC,UAAU;UAAA,uBACvB;YACE,SAAS,EAAC,yBAAyB;YACnC,KAAK,EAAE;cAAEsE,KAAK,YAAKrE,aAAa;YAAI,CAAE;YACtC,iBAAeA,aAAc;YAC7B,iBAAe,CAAE;YACjB,iBAAe;UAAI;QACnB,EACE,eACN;UAAA,WAAQA,aAAa,EAAC,GAAC;QAAA,EAAQ,eAC/B,KAAC,MAAM;UAAC,IAAI,EAAC,SAAS;UAAC,OAAO,EAAE,IAAI,CAACT,sBAAuB;UAAA,UAAC;QAE7D,EAAS;MAAA,EAEZ;IAAA,EACG;EAEV;AACF;AAAC,gBApUKV,WAAW,kBACO;EACpByB,IAAI,EAAE,IAAI;EACVC,KAAK,EAAE,IAAI;EACXmD,KAAK,EAAE;AACT,CAAC;AAiUH,eAAe7E,WAAW"}
|
|
1
|
+
{"version":3,"file":"CsvInputBar.js","names":["React","Component","classNames","Button","Checkbox","Log","DbNameValidator","CsvOverlay","CsvParser","CsvFormats","log","module","TYPE_OPTIONS","Object","entries","TYPES","map","key","value","name","CsvInputBar","constructor","props","handleUpload","bind","handleCancel","handleError","handleTableName","toggleFirstRowHeaders","handleProgress","handleCancelInProgress","handleQueryTypeChange","inputRef","createRef","state","tableName","tableNameSet","isFirstRowHeaders","showProgress","progressValue","type","DEFAULT_TYPE","parser","componentDidUpdate","prevProps","file","paste","dotIndex","lastIndexOf","fileTableName","legalizeTableName","substring","setState","current","focus","fromExtension","undefined","AUTO","componentWillUnmount","cancel","onClose","e","onError","error","event","target","stopPropagation","preventDefault","endsWith","handleZipFile","handleFile","Blob","isZip","info","File","size","session","timeZone","onInProgress","handleParseDone","tables","length","bindTableToVariable","then","openTable","catch","mergeTables","table","forEach","t","close","onFileCompleted","readHeaders","onProgress","parse","zipFile","onUpdate","unzip","Error","files","csvFound","i","f","isValidExtension","onOpenTable","render","isNameInvalid","Boolean","width"],"sources":["../../src/csv/CsvInputBar.tsx"],"sourcesContent":["import React, {\n ChangeEvent,\n Component,\n FormEvent,\n ReactElement,\n RefObject,\n} from 'react';\nimport classNames from 'classnames';\nimport type { JSZipObject } from 'jszip';\nimport { Button, Checkbox } from '@deephaven/components';\nimport type { IdeSession, Table } from '@deephaven/jsapi-types';\nimport Log from '@deephaven/log';\nimport { DbNameValidator } from '@deephaven/utils';\nimport CsvOverlay from './CsvOverlay';\nimport CsvParser from './CsvParser';\nimport CsvFormats from './CsvFormats';\nimport './CsvInputBar.scss';\n\nconst log = Log.module('CsvInputBar');\n\nconst TYPE_OPTIONS = Object.entries(CsvFormats.TYPES).map(([key, value]) => (\n <option key={key} value={key}>\n {value.name}\n </option>\n));\n\ninterface CsvInputBarProps {\n session: IdeSession;\n onOpenTable: (name: string) => void;\n onClose: () => void;\n onUpdate: (update: string) => void;\n onError: (e: unknown) => void;\n file: File;\n paste?: string;\n onInProgress: (boolean: boolean) => void;\n timeZone: string;\n unzip?: (zipFile: File) => Promise<JSZipObject[]>;\n}\n\ninterface CsvInputBarState {\n tableName: string;\n tableNameSet: boolean;\n isFirstRowHeaders: boolean;\n showProgress: boolean;\n progressValue: number;\n type: keyof typeof CsvFormats.TYPES;\n parser: CsvParser | null;\n}\n/**\n * Input controls for CSV upload.\n */\nclass CsvInputBar extends Component<CsvInputBarProps, CsvInputBarState> {\n static defaultProps = {\n file: null,\n paste: null,\n unzip: null,\n };\n\n constructor(props: CsvInputBarProps) {\n super(props);\n\n this.handleUpload = this.handleUpload.bind(this);\n this.handleCancel = this.handleCancel.bind(this);\n this.handleError = this.handleError.bind(this);\n this.handleTableName = this.handleTableName.bind(this);\n this.toggleFirstRowHeaders = this.toggleFirstRowHeaders.bind(this);\n this.handleProgress = this.handleProgress.bind(this);\n this.handleCancelInProgress = this.handleCancelInProgress.bind(this);\n this.handleQueryTypeChange = this.handleQueryTypeChange.bind(this);\n\n this.inputRef = React.createRef();\n\n this.state = {\n tableName: '',\n tableNameSet: false,\n isFirstRowHeaders: true,\n showProgress: false,\n progressValue: 0,\n type: CsvFormats.DEFAULT_TYPE,\n parser: null,\n };\n }\n\n // React documentation says it is fine to update state inside an if statment\n /* eslint-disable react/no-did-update-set-state */\n componentDidUpdate(prevProps: CsvInputBarProps): void {\n const { file, paste } = this.props;\n const { tableName, tableNameSet } = this.state;\n // Set the table name from a file\n if (!tableNameSet && file != null && !tableName) {\n const dotIndex = file.name.lastIndexOf('.');\n const fileTableName = DbNameValidator.legalizeTableName(\n file.name.substring(0, dotIndex)\n );\n this.setState({\n tableName: fileTableName,\n tableNameSet: true,\n });\n this.inputRef.current?.focus();\n } else if (\n (file == null && prevProps.file != null) ||\n (paste == null && prevProps.paste != null)\n ) {\n // The file or paste was unstaged\n this.setState({\n tableName: '',\n tableNameSet: false,\n });\n }\n\n // Focus the name input field on paste\n if (paste != null && paste !== '' && !tableName && this.inputRef.current) {\n this.inputRef.current.focus();\n }\n\n // Determine parser type by file extension\n if (file != null && file !== prevProps.file) {\n this.setState({\n type: CsvFormats.fromExtension(file.name),\n });\n } else if (\n paste != null &&\n paste !== '' &&\n (prevProps.paste === undefined || prevProps.paste === '')\n ) {\n this.setState({\n type: CsvFormats.AUTO,\n });\n }\n }\n\n componentWillUnmount(): void {\n const { parser } = this.state;\n if (parser) {\n parser.cancel();\n }\n }\n\n inputRef: RefObject<HTMLInputElement>;\n\n handleCancel(): void {\n const { onClose } = this.props;\n onClose();\n }\n\n handleError(e: unknown): void {\n const { onClose, onError } = this.props;\n log.error(e);\n onError(e);\n onClose();\n }\n\n handleTableName(event: ChangeEvent<HTMLInputElement>): void {\n this.setState({ tableName: event.target.value, tableNameSet: true });\n }\n\n toggleFirstRowHeaders(): void {\n const { isFirstRowHeaders } = this.state;\n this.setState({ isFirstRowHeaders: !isFirstRowHeaders });\n }\n\n handleUpload(event: FormEvent<HTMLFormElement>): void {\n event.stopPropagation();\n event.preventDefault();\n const { file, paste } = this.props;\n if (file != null) {\n if (file.name.endsWith('.zip')) {\n this.handleZipFile(file);\n } else {\n this.handleFile(file);\n }\n } else if (paste !== undefined && paste !== '') {\n this.handleFile(\n new Blob([paste], {\n type: 'text/plain',\n })\n );\n }\n }\n\n handleFile(file: File | Blob | JSZipObject, isZip = false): void {\n log.info(\n `Starting CSV parser for ${\n file instanceof File ? file.name : 'pasted values'\n } ${isZip ? '' : (file as Blob).size} bytes`\n );\n const { session, timeZone, onInProgress } = this.props;\n const { tableName, isFirstRowHeaders, type } = this.state;\n const handleParseDone = (tables: Table[]) => {\n // Do not bother merging just one table\n if (tables.length === 1) {\n session\n .bindTableToVariable(tables[0], tableName)\n .then(() => this.openTable())\n .catch(e => this.handleError(e));\n } else {\n session\n .mergeTables(tables)\n .then(table => {\n session\n .bindTableToVariable(table, tableName)\n .then(() => this.openTable())\n .catch(e => this.handleError(e));\n tables.forEach(t => t.close());\n })\n .catch(e => this.handleError(e));\n }\n };\n const parser = new CsvParser({\n onFileCompleted: handleParseDone,\n session,\n file,\n type: CsvFormats.TYPES[type],\n readHeaders: isFirstRowHeaders,\n onProgress: this.handleProgress,\n onError: this.handleError,\n timeZone,\n isZip,\n });\n parser.parse();\n this.setState({\n showProgress: true,\n parser,\n });\n // Note that calling onClose will set in progress to false\n onInProgress(true);\n }\n\n handleZipFile(zipFile: File): void {\n const { onUpdate, unzip } = this.props;\n if (unzip == null) {\n this.handleError(new Error('No support for zip files available.'));\n return;\n }\n\n unzip(zipFile)\n .then(files => {\n let csvFound = false;\n // Find the first Csv file in the zip and use that\n for (let i = 0; i < files.length; i += 1) {\n const f = files[i];\n if (CsvOverlay.isValidExtension(f.name)) {\n onUpdate(`Loading ${f.name} as csv from ${zipFile.name}`);\n csvFound = true;\n this.handleFile(f, true);\n break;\n }\n }\n if (!csvFound) {\n throw new Error(`No csv file found in ${zipFile.name}`);\n }\n })\n .catch(e => this.handleError(e));\n }\n\n handleProgress(progressValue: number): boolean {\n const { showProgress } = this.state;\n if (showProgress) {\n this.setState({\n progressValue,\n });\n }\n // Indicates to the caller that the upload has been cancelled\n return !showProgress;\n }\n\n // Cancels an in progress upload\n handleCancelInProgress(): void {\n const { onInProgress } = this.props;\n const { parser } = this.state;\n if (parser) {\n parser.cancel();\n }\n this.setState({\n showProgress: false,\n progressValue: 0,\n });\n onInProgress(false);\n }\n\n openTable(): void {\n const { onOpenTable, onClose } = this.props;\n const { tableName } = this.state;\n onOpenTable(tableName);\n onClose();\n }\n\n handleQueryTypeChange(event: ChangeEvent<HTMLSelectElement>): void {\n this.setState({\n type: event.target.value as keyof typeof CsvFormats.TYPES,\n });\n }\n\n render(): ReactElement {\n const { file, paste } = this.props;\n const { tableName, isFirstRowHeaders, showProgress, progressValue, type } =\n this.state;\n // A blank table name is invalid for pasted values\n const isNameInvalid = Boolean(paste) && !tableName;\n return (\n <div className=\"csv-input-bar\">\n {!showProgress && (\n <form\n onSubmit={this.handleUpload}\n className=\"csv-input-bar-container form-inline\"\n >\n <div className=\"form-group\">\n <label htmlFor=\"tableNameInput\">Table name</label>\n <input\n ref={this.inputRef}\n id=\"tableNameInput\"\n type=\"text\"\n className={classNames('form-control', {\n 'is-invalid': isNameInvalid,\n })}\n value={tableName}\n onChange={this.handleTableName}\n />\n </div>\n <div className=\"form-group\">\n <label htmlFor=\"formatSelect\">File format</label>\n <select\n id=\"formatSelect\"\n className=\"custom-select\"\n value={type}\n onChange={this.handleQueryTypeChange}\n >\n {TYPE_OPTIONS}\n </select>\n </div>\n <Checkbox\n className=\"firstRowHeaders\"\n checked={isFirstRowHeaders}\n onChange={this.toggleFirstRowHeaders}\n >\n First row is column headers\n </Checkbox>\n <div className=\"csv-input-buttons form-group\">\n <Button kind=\"secondary\" onClick={this.handleCancel}>\n Cancel\n </Button>\n <Button\n kind=\"primary\"\n type=\"submit\"\n disabled={\n !(file !== null || (paste !== undefined && paste !== '')) ||\n !tableName\n }\n >\n Upload\n </Button>\n </div>\n </form>\n )}\n {showProgress && (\n <div className=\"csv-progress-container\">\n <label>Uploading Table</label>\n <div className=\"progress\">\n <div\n className=\"progress-bar bg-primary\"\n style={{ width: `${progressValue}%` }}\n aria-valuenow={progressValue}\n aria-valuemin={0}\n aria-valuemax={100}\n />\n </div>\n <label>{progressValue}%</label>\n <Button kind=\"primary\" onClick={this.handleCancelInProgress}>\n Cancel\n </Button>\n </div>\n )}\n </div>\n );\n }\n}\n\nexport default CsvInputBar;\n"],"mappings":";;;AAAA,OAAOA,KAAK,IAEVC,SAAS,QAIJ,OAAO;AACd,OAAOC,UAAU,MAAM,YAAY;AAEnC,SAASC,MAAM,EAAEC,QAAQ,QAAQ,uBAAuB;AAExD,OAAOC,GAAG,MAAM,gBAAgB;AAChC,SAASC,eAAe,QAAQ,kBAAkB;AAAC,OAC5CC,UAAU;AAAA,OACVC,SAAS;AAAA,OACTC,UAAU;AAAA;AAAA;AAAA;AAGjB,IAAMC,GAAG,GAAGL,GAAG,CAACM,MAAM,CAAC,aAAa,CAAC;AAErC,IAAMC,YAAY,GAAGC,MAAM,CAACC,OAAO,CAACL,UAAU,CAACM,KAAK,CAAC,CAACC,GAAG,CAAC;EAAA,IAAC,CAACC,GAAG,EAAEC,KAAK,CAAC;EAAA,oBACrE;IAAkB,KAAK,EAAED,GAAI;IAAA,UAC1BC,KAAK,CAACC;EAAI,GADAF,GAAG,CAEP;AAAA,CACV,CAAC;AAwBF;AACA;AACA;AACA,MAAMG,WAAW,SAASnB,SAAS,CAAqC;EAOtEoB,WAAW,CAACC,KAAuB,EAAE;IACnC,KAAK,CAACA,KAAK,CAAC;IAAC;IAEb,IAAI,CAACC,YAAY,GAAG,IAAI,CAACA,YAAY,CAACC,IAAI,CAAC,IAAI,CAAC;IAChD,IAAI,CAACC,YAAY,GAAG,IAAI,CAACA,YAAY,CAACD,IAAI,CAAC,IAAI,CAAC;IAChD,IAAI,CAACE,WAAW,GAAG,IAAI,CAACA,WAAW,CAACF,IAAI,CAAC,IAAI,CAAC;IAC9C,IAAI,CAACG,eAAe,GAAG,IAAI,CAACA,eAAe,CAACH,IAAI,CAAC,IAAI,CAAC;IACtD,IAAI,CAACI,qBAAqB,GAAG,IAAI,CAACA,qBAAqB,CAACJ,IAAI,CAAC,IAAI,CAAC;IAClE,IAAI,CAACK,cAAc,GAAG,IAAI,CAACA,cAAc,CAACL,IAAI,CAAC,IAAI,CAAC;IACpD,IAAI,CAACM,sBAAsB,GAAG,IAAI,CAACA,sBAAsB,CAACN,IAAI,CAAC,IAAI,CAAC;IACpE,IAAI,CAACO,qBAAqB,GAAG,IAAI,CAACA,qBAAqB,CAACP,IAAI,CAAC,IAAI,CAAC;IAElE,IAAI,CAACQ,QAAQ,gBAAGhC,KAAK,CAACiC,SAAS,EAAE;IAEjC,IAAI,CAACC,KAAK,GAAG;MACXC,SAAS,EAAE,EAAE;MACbC,YAAY,EAAE,KAAK;MACnBC,iBAAiB,EAAE,IAAI;MACvBC,YAAY,EAAE,KAAK;MACnBC,aAAa,EAAE,CAAC;MAChBC,IAAI,EAAE/B,UAAU,CAACgC,YAAY;MAC7BC,MAAM,EAAE;IACV,CAAC;EACH;;EAEA;EACA;EACAC,kBAAkB,CAACC,SAA2B,EAAQ;IACpD,IAAM;MAAEC,IAAI;MAAEC;IAAM,CAAC,GAAG,IAAI,CAACxB,KAAK;IAClC,IAAM;MAAEa,SAAS;MAAEC;IAAa,CAAC,GAAG,IAAI,CAACF,KAAK;IAC9C;IACA,IAAI,CAACE,YAAY,IAAIS,IAAI,IAAI,IAAI,IAAI,CAACV,SAAS,EAAE;MAAA;MAC/C,IAAMY,QAAQ,GAAGF,IAAI,CAAC1B,IAAI,CAAC6B,WAAW,CAAC,GAAG,CAAC;MAC3C,IAAMC,aAAa,GAAG3C,eAAe,CAAC4C,iBAAiB,CACrDL,IAAI,CAAC1B,IAAI,CAACgC,SAAS,CAAC,CAAC,EAAEJ,QAAQ,CAAC,CACjC;MACD,IAAI,CAACK,QAAQ,CAAC;QACZjB,SAAS,EAAEc,aAAa;QACxBb,YAAY,EAAE;MAChB,CAAC,CAAC;MACF,6BAAI,CAACJ,QAAQ,CAACqB,OAAO,0DAArB,sBAAuBC,KAAK,EAAE;IAChC,CAAC,MAAM,IACJT,IAAI,IAAI,IAAI,IAAID,SAAS,CAACC,IAAI,IAAI,IAAI,IACtCC,KAAK,IAAI,IAAI,IAAIF,SAAS,CAACE,KAAK,IAAI,IAAK,EAC1C;MACA;MACA,IAAI,CAACM,QAAQ,CAAC;QACZjB,SAAS,EAAE,EAAE;QACbC,YAAY,EAAE;MAChB,CAAC,CAAC;IACJ;;IAEA;IACA,IAAIU,KAAK,IAAI,IAAI,IAAIA,KAAK,KAAK,EAAE,IAAI,CAACX,SAAS,IAAI,IAAI,CAACH,QAAQ,CAACqB,OAAO,EAAE;MACxE,IAAI,CAACrB,QAAQ,CAACqB,OAAO,CAACC,KAAK,EAAE;IAC/B;;IAEA;IACA,IAAIT,IAAI,IAAI,IAAI,IAAIA,IAAI,KAAKD,SAAS,CAACC,IAAI,EAAE;MAC3C,IAAI,CAACO,QAAQ,CAAC;QACZZ,IAAI,EAAE/B,UAAU,CAAC8C,aAAa,CAACV,IAAI,CAAC1B,IAAI;MAC1C,CAAC,CAAC;IACJ,CAAC,MAAM,IACL2B,KAAK,IAAI,IAAI,IACbA,KAAK,KAAK,EAAE,KACXF,SAAS,CAACE,KAAK,KAAKU,SAAS,IAAIZ,SAAS,CAACE,KAAK,KAAK,EAAE,CAAC,EACzD;MACA,IAAI,CAACM,QAAQ,CAAC;QACZZ,IAAI,EAAE/B,UAAU,CAACgD;MACnB,CAAC,CAAC;IACJ;EACF;EAEAC,oBAAoB,GAAS;IAC3B,IAAM;MAAEhB;IAAO,CAAC,GAAG,IAAI,CAACR,KAAK;IAC7B,IAAIQ,MAAM,EAAE;MACVA,MAAM,CAACiB,MAAM,EAAE;IACjB;EACF;EAIAlC,YAAY,GAAS;IACnB,IAAM;MAAEmC;IAAQ,CAAC,GAAG,IAAI,CAACtC,KAAK;IAC9BsC,OAAO,EAAE;EACX;EAEAlC,WAAW,CAACmC,CAAU,EAAQ;IAC5B,IAAM;MAAED,OAAO;MAAEE;IAAQ,CAAC,GAAG,IAAI,CAACxC,KAAK;IACvCZ,GAAG,CAACqD,KAAK,CAACF,CAAC,CAAC;IACZC,OAAO,CAACD,CAAC,CAAC;IACVD,OAAO,EAAE;EACX;EAEAjC,eAAe,CAACqC,KAAoC,EAAQ;IAC1D,IAAI,CAACZ,QAAQ,CAAC;MAAEjB,SAAS,EAAE6B,KAAK,CAACC,MAAM,CAAC/C,KAAK;MAAEkB,YAAY,EAAE;IAAK,CAAC,CAAC;EACtE;EAEAR,qBAAqB,GAAS;IAC5B,IAAM;MAAES;IAAkB,CAAC,GAAG,IAAI,CAACH,KAAK;IACxC,IAAI,CAACkB,QAAQ,CAAC;MAAEf,iBAAiB,EAAE,CAACA;IAAkB,CAAC,CAAC;EAC1D;EAEAd,YAAY,CAACyC,KAAiC,EAAQ;IACpDA,KAAK,CAACE,eAAe,EAAE;IACvBF,KAAK,CAACG,cAAc,EAAE;IACtB,IAAM;MAAEtB,IAAI;MAAEC;IAAM,CAAC,GAAG,IAAI,CAACxB,KAAK;IAClC,IAAIuB,IAAI,IAAI,IAAI,EAAE;MAChB,IAAIA,IAAI,CAAC1B,IAAI,CAACiD,QAAQ,CAAC,MAAM,CAAC,EAAE;QAC9B,IAAI,CAACC,aAAa,CAACxB,IAAI,CAAC;MAC1B,CAAC,MAAM;QACL,IAAI,CAACyB,UAAU,CAACzB,IAAI,CAAC;MACvB;IACF,CAAC,MAAM,IAAIC,KAAK,KAAKU,SAAS,IAAIV,KAAK,KAAK,EAAE,EAAE;MAC9C,IAAI,CAACwB,UAAU,CACb,IAAIC,IAAI,CAAC,CAACzB,KAAK,CAAC,EAAE;QAChBN,IAAI,EAAE;MACR,CAAC,CAAC,CACH;IACH;EACF;EAEA8B,UAAU,CAACzB,IAA+B,EAAuB;IAAA,IAArB2B,KAAK,uEAAG,KAAK;IACvD9D,GAAG,CAAC+D,IAAI,mCAEJ5B,IAAI,YAAY6B,IAAI,GAAG7B,IAAI,CAAC1B,IAAI,GAAG,eAAe,cAChDqD,KAAK,GAAG,EAAE,GAAI3B,IAAI,CAAU8B,IAAI,YACrC;IACD,IAAM;MAAEC,OAAO;MAAEC,QAAQ;MAAEC;IAAa,CAAC,GAAG,IAAI,CAACxD,KAAK;IACtD,IAAM;MAAEa,SAAS;MAAEE,iBAAiB;MAAEG;IAAK,CAAC,GAAG,IAAI,CAACN,KAAK;IACzD,IAAM6C,eAAe,GAAIC,MAAe,IAAK;MAC3C;MACA,IAAIA,MAAM,CAACC,MAAM,KAAK,CAAC,EAAE;QACvBL,OAAO,CACJM,mBAAmB,CAACF,MAAM,CAAC,CAAC,CAAC,EAAE7C,SAAS,CAAC,CACzCgD,IAAI,CAAC,MAAM,IAAI,CAACC,SAAS,EAAE,CAAC,CAC5BC,KAAK,CAACxB,CAAC,IAAI,IAAI,CAACnC,WAAW,CAACmC,CAAC,CAAC,CAAC;MACpC,CAAC,MAAM;QACLe,OAAO,CACJU,WAAW,CAACN,MAAM,CAAC,CACnBG,IAAI,CAACI,KAAK,IAAI;UACbX,OAAO,CACJM,mBAAmB,CAACK,KAAK,EAAEpD,SAAS,CAAC,CACrCgD,IAAI,CAAC,MAAM,IAAI,CAACC,SAAS,EAAE,CAAC,CAC5BC,KAAK,CAACxB,CAAC,IAAI,IAAI,CAACnC,WAAW,CAACmC,CAAC,CAAC,CAAC;UAClCmB,MAAM,CAACQ,OAAO,CAACC,CAAC,IAAIA,CAAC,CAACC,KAAK,EAAE,CAAC;QAChC,CAAC,CAAC,CACDL,KAAK,CAACxB,CAAC,IAAI,IAAI,CAACnC,WAAW,CAACmC,CAAC,CAAC,CAAC;MACpC;IACF,CAAC;IACD,IAAMnB,MAAM,GAAG,IAAIlC,SAAS,CAAC;MAC3BmF,eAAe,EAAEZ,eAAe;MAChCH,OAAO;MACP/B,IAAI;MACJL,IAAI,EAAE/B,UAAU,CAACM,KAAK,CAACyB,IAAI,CAAC;MAC5BoD,WAAW,EAAEvD,iBAAiB;MAC9BwD,UAAU,EAAE,IAAI,CAAChE,cAAc;MAC/BiC,OAAO,EAAE,IAAI,CAACpC,WAAW;MACzBmD,QAAQ;MACRL;IACF,CAAC,CAAC;IACF9B,MAAM,CAACoD,KAAK,EAAE;IACd,IAAI,CAAC1C,QAAQ,CAAC;MACZd,YAAY,EAAE,IAAI;MAClBI;IACF,CAAC,CAAC;IACF;IACAoC,YAAY,CAAC,IAAI,CAAC;EACpB;EAEAT,aAAa,CAAC0B,OAAa,EAAQ;IACjC,IAAM;MAAEC,QAAQ;MAAEC;IAAM,CAAC,GAAG,IAAI,CAAC3E,KAAK;IACtC,IAAI2E,KAAK,IAAI,IAAI,EAAE;MACjB,IAAI,CAACvE,WAAW,CAAC,IAAIwE,KAAK,CAAC,qCAAqC,CAAC,CAAC;MAClE;IACF;IAEAD,KAAK,CAACF,OAAO,CAAC,CACXZ,IAAI,CAACgB,KAAK,IAAI;MACb,IAAIC,QAAQ,GAAG,KAAK;MACpB;MACA,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGF,KAAK,CAAClB,MAAM,EAAEoB,CAAC,IAAI,CAAC,EAAE;QACxC,IAAMC,CAAC,GAAGH,KAAK,CAACE,CAAC,CAAC;QAClB,IAAI9F,UAAU,CAACgG,gBAAgB,CAACD,CAAC,CAACnF,IAAI,CAAC,EAAE;UACvC6E,QAAQ,mBAAYM,CAAC,CAACnF,IAAI,0BAAgB4E,OAAO,CAAC5E,IAAI,EAAG;UACzDiF,QAAQ,GAAG,IAAI;UACf,IAAI,CAAC9B,UAAU,CAACgC,CAAC,EAAE,IAAI,CAAC;UACxB;QACF;MACF;MACA,IAAI,CAACF,QAAQ,EAAE;QACb,MAAM,IAAIF,KAAK,gCAAyBH,OAAO,CAAC5E,IAAI,EAAG;MACzD;IACF,CAAC,CAAC,CACDkE,KAAK,CAACxB,CAAC,IAAI,IAAI,CAACnC,WAAW,CAACmC,CAAC,CAAC,CAAC;EACpC;EAEAhC,cAAc,CAACU,aAAqB,EAAW;IAC7C,IAAM;MAAED;IAAa,CAAC,GAAG,IAAI,CAACJ,KAAK;IACnC,IAAII,YAAY,EAAE;MAChB,IAAI,CAACc,QAAQ,CAAC;QACZb;MACF,CAAC,CAAC;IACJ;IACA;IACA,OAAO,CAACD,YAAY;EACtB;;EAEA;EACAR,sBAAsB,GAAS;IAC7B,IAAM;MAAEgD;IAAa,CAAC,GAAG,IAAI,CAACxD,KAAK;IACnC,IAAM;MAAEoB;IAAO,CAAC,GAAG,IAAI,CAACR,KAAK;IAC7B,IAAIQ,MAAM,EAAE;MACVA,MAAM,CAACiB,MAAM,EAAE;IACjB;IACA,IAAI,CAACP,QAAQ,CAAC;MACZd,YAAY,EAAE,KAAK;MACnBC,aAAa,EAAE;IACjB,CAAC,CAAC;IACFuC,YAAY,CAAC,KAAK,CAAC;EACrB;EAEAM,SAAS,GAAS;IAChB,IAAM;MAAEoB,WAAW;MAAE5C;IAAQ,CAAC,GAAG,IAAI,CAACtC,KAAK;IAC3C,IAAM;MAAEa;IAAU,CAAC,GAAG,IAAI,CAACD,KAAK;IAChCsE,WAAW,CAACrE,SAAS,CAAC;IACtByB,OAAO,EAAE;EACX;EAEA7B,qBAAqB,CAACiC,KAAqC,EAAQ;IACjE,IAAI,CAACZ,QAAQ,CAAC;MACZZ,IAAI,EAAEwB,KAAK,CAACC,MAAM,CAAC/C;IACrB,CAAC,CAAC;EACJ;EAEAuF,MAAM,GAAiB;IACrB,IAAM;MAAE5D,IAAI;MAAEC;IAAM,CAAC,GAAG,IAAI,CAACxB,KAAK;IAClC,IAAM;MAAEa,SAAS;MAAEE,iBAAiB;MAAEC,YAAY;MAAEC,aAAa;MAAEC;IAAK,CAAC,GACvE,IAAI,CAACN,KAAK;IACZ;IACA,IAAMwE,aAAa,GAAGC,OAAO,CAAC7D,KAAK,CAAC,IAAI,CAACX,SAAS;IAClD,oBACE;MAAK,SAAS,EAAC,eAAe;MAAA,WAC3B,CAACG,YAAY,iBACZ;QACE,QAAQ,EAAE,IAAI,CAACf,YAAa;QAC5B,SAAS,EAAC,qCAAqC;QAAA,wBAE/C;UAAK,SAAS,EAAC,YAAY;UAAA,wBACzB;YAAO,OAAO,EAAC,gBAAgB;YAAA,UAAC;UAAU,EAAQ,eAClD;YACE,GAAG,EAAE,IAAI,CAACS,QAAS;YACnB,EAAE,EAAC,gBAAgB;YACnB,IAAI,EAAC,MAAM;YACX,SAAS,EAAE9B,UAAU,CAAC,cAAc,EAAE;cACpC,YAAY,EAAEwG;YAChB,CAAC,CAAE;YACH,KAAK,EAAEvE,SAAU;YACjB,QAAQ,EAAE,IAAI,CAACR;UAAgB,EAC/B;QAAA,EACE,eACN;UAAK,SAAS,EAAC,YAAY;UAAA,wBACzB;YAAO,OAAO,EAAC,cAAc;YAAA,UAAC;UAAW,EAAQ,eACjD;YACE,EAAE,EAAC,cAAc;YACjB,SAAS,EAAC,eAAe;YACzB,KAAK,EAAEa,IAAK;YACZ,QAAQ,EAAE,IAAI,CAACT,qBAAsB;YAAA,UAEpCnB;UAAY,EACN;QAAA,EACL,eACN,KAAC,QAAQ;UACP,SAAS,EAAC,iBAAiB;UAC3B,OAAO,EAAEyB,iBAAkB;UAC3B,QAAQ,EAAE,IAAI,CAACT,qBAAsB;UAAA,UACtC;QAED,EAAW,eACX;UAAK,SAAS,EAAC,8BAA8B;UAAA,wBAC3C,KAAC,MAAM;YAAC,IAAI,EAAC,WAAW;YAAC,OAAO,EAAE,IAAI,CAACH,YAAa;YAAA,UAAC;UAErD,EAAS,eACT,KAAC,MAAM;YACL,IAAI,EAAC,SAAS;YACd,IAAI,EAAC,QAAQ;YACb,QAAQ,EACN,EAAEoB,IAAI,KAAK,IAAI,IAAKC,KAAK,KAAKU,SAAS,IAAIV,KAAK,KAAK,EAAG,CAAC,IACzD,CAACX,SACF;YAAA,UACF;UAED,EAAS;QAAA,EACL;MAAA,EAET,EACAG,YAAY,iBACX;QAAK,SAAS,EAAC,wBAAwB;QAAA,wBACrC;UAAA,UAAO;QAAe,EAAQ,eAC9B;UAAK,SAAS,EAAC,UAAU;UAAA,uBACvB;YACE,SAAS,EAAC,yBAAyB;YACnC,KAAK,EAAE;cAAEsE,KAAK,YAAKrE,aAAa;YAAI,CAAE;YACtC,iBAAeA,aAAc;YAC7B,iBAAe,CAAE;YACjB,iBAAe;UAAI;QACnB,EACE,eACN;UAAA,WAAQA,aAAa,EAAC,GAAC;QAAA,EAAQ,eAC/B,KAAC,MAAM;UAAC,IAAI,EAAC,SAAS;UAAC,OAAO,EAAE,IAAI,CAACT,sBAAuB;UAAA,UAAC;QAE7D,EAAS;MAAA,EAEZ;IAAA,EACG;EAEV;AACF;AAAC,gBApUKV,WAAW,kBACO;EACpByB,IAAI,EAAE,IAAI;EACVC,KAAK,EAAE,IAAI;EACXmD,KAAK,EAAE;AACT,CAAC;AAiUH,eAAe7E,WAAW"}
|
package/dist/csv/CsvParser.d.ts
CHANGED
|
@@ -6,7 +6,7 @@ import { CsvTypes } from './CsvFormats';
|
|
|
6
6
|
interface CsvParserConstructor {
|
|
7
7
|
onFileCompleted: (tables: Table[]) => void;
|
|
8
8
|
session: IdeSession;
|
|
9
|
-
file: Blob | JSZipObject;
|
|
9
|
+
file: File | Blob | JSZipObject;
|
|
10
10
|
type: CsvTypes;
|
|
11
11
|
readHeaders: boolean;
|
|
12
12
|
onProgress: (progressValue: number) => boolean;
|
|
@@ -23,7 +23,7 @@ declare class CsvParser {
|
|
|
23
23
|
constructor({ onFileCompleted, session, file, type, readHeaders, onProgress, onError, timeZone, isZip, }: CsvParserConstructor);
|
|
24
24
|
onFileCompleted: (tables: Table[]) => void;
|
|
25
25
|
session: IdeSession;
|
|
26
|
-
file: Blob | JSZipObject;
|
|
26
|
+
file: File | Blob | JSZipObject;
|
|
27
27
|
isZip: boolean;
|
|
28
28
|
type: CsvTypes;
|
|
29
29
|
readHeaders: boolean;
|
|
@@ -33,6 +33,8 @@ declare class CsvParser {
|
|
|
33
33
|
tables: Table[];
|
|
34
34
|
headers?: string[];
|
|
35
35
|
types?: string[];
|
|
36
|
+
rowCount: number;
|
|
37
|
+
rowsProcessed: number;
|
|
36
38
|
chunks: number;
|
|
37
39
|
totalChunks: number;
|
|
38
40
|
isComplete: boolean;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CsvParser.d.ts","sourceRoot":"","sources":["../../src/csv/CsvParser.ts"],"names":[],"mappings":";AAAA,OAAa,EAAE,gBAAgB,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAGxE,OAAO,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAChE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,OAAO,CAAC;AAEzC,OAAO,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"CsvParser.d.ts","sourceRoot":"","sources":["../../src/csv/CsvParser.ts"],"names":[],"mappings":";AAAA,OAAa,EAAE,gBAAgB,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAGxE,OAAO,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAChE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,OAAO,CAAC;AAEzC,OAAO,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AASxC,UAAU,oBAAoB;IAC5B,eAAe,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,IAAI,CAAC;IAC3C,OAAO,EAAE,UAAU,CAAC;IACpB,IAAI,EAAE,IAAI,GAAG,IAAI,GAAG,WAAW,CAAC;IAChC,IAAI,EAAE,QAAQ,CAAC;IACf,WAAW,EAAE,OAAO,CAAC;IACrB,UAAU,EAAE,CAAC,aAAa,EAAE,MAAM,KAAK,OAAO,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,EAAE,OAAO,KAAK,IAAI,CAAC;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;GAEG;AACH,cAAM,SAAS;IAEb,MAAM,CAAC,eAAe,eAAgB,MAAM,KAAG,MAAM,EAAE,CAMrD;IAEF,MAAM,CAAC,uBAAuB,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM;gBAYrC,EACV,eAAe,EACf,OAAO,EACP,IAAI,EACJ,IAAI,EACJ,WAAkB,EAClB,UAAU,EACV,OAAO,EACP,QAAQ,EACR,KAAK,GACN,EAAE,oBAAoB;IAyCvB,eAAe,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,IAAI,CAAC;IAE3C,OAAO,EAAE,UAAU,CAAC;IAEpB,IAAI,EAAE,IAAI,GAAG,IAAI,GAAG,WAAW,CAAC;IAEhC,KAAK,EAAE,OAAO,CAAC;IAEf,IAAI,EAAE,QAAQ,CAAC;IAEf,WAAW,EAAE,OAAO,CAAC;IAErB,QAAQ,EAAE,MAAM,CAAC;IAEjB,UAAU,EAAE,CAAC,aAAa,EAAE,MAAM,KAAK,OAAO,CAAC;IAE/C,OAAO,EAAE,CAAC,CAAC,EAAE,OAAO,KAAK,IAAI,CAAC;IAE9B,MAAM,EAAE,KAAK,EAAE,CAAC;IAEhB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAEnB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IAEjB,QAAQ,EAAE,MAAM,CAAC;IAEjB,aAAa,EAAE,MAAM,CAAC;IAEtB,MAAM,EAAE,MAAM,CAAC;IAEf,WAAW,EAAE,MAAM,CAAC;IAEpB,UAAU,EAAE,OAAO,CAAC;IAEpB,WAAW,EAAE,MAAM,CAAC;IAEpB,kBAAkB,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC;IAEhC,eAAe,EAAE,MAAM,CAAC;IAExB,WAAW,EAAE,OAAO,CAAC;IAErB,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,IAAI,GAAG,MAAM,CAAC,cAAc,CAAC,CAAC;IAEhE,MAAM,IAAI,IAAI;IAId,SAAS,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,EAAE;IAoB5D,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAIhC,KAAK,IAAI,IAAI;IA+Bb,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAkChE,iBAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAgB5D,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI;IAWrD,iBAAiB,CACf,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EAAE,EAAE,EACnB,MAAM,EAAE,MAAM,GAAG,IAAI,GACpB,IAAI;IAyDP,cAAc,CAAC,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,IAAI;IAWnD,WAAW,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI;IAKjC,gBAAgB,CAAC,QAAQ,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;CAGtD;AAED,eAAe,SAAS,CAAC"}
|
package/dist/csv/CsvParser.js
CHANGED
|
@@ -5,6 +5,7 @@ import Papa from 'papaparse';
|
|
|
5
5
|
import Log from '@deephaven/log';
|
|
6
6
|
import { assertNotNull, DbNameValidator } from '@deephaven/utils';
|
|
7
7
|
import CsvTypeParser from "./CsvTypeParser.js";
|
|
8
|
+
import makeZipStreamHelper from "./ZipStreamHelper.js";
|
|
8
9
|
var log = Log.module('CsvParser');
|
|
9
10
|
|
|
10
11
|
// This is based on jszip streaming ~15 KB chunks to papa parse
|
|
@@ -49,6 +50,8 @@ class CsvParser {
|
|
|
49
50
|
_defineProperty(this, "tables", void 0);
|
|
50
51
|
_defineProperty(this, "headers", void 0);
|
|
51
52
|
_defineProperty(this, "types", void 0);
|
|
53
|
+
_defineProperty(this, "rowCount", void 0);
|
|
54
|
+
_defineProperty(this, "rowsProcessed", void 0);
|
|
52
55
|
_defineProperty(this, "chunks", void 0);
|
|
53
56
|
_defineProperty(this, "totalChunks", void 0);
|
|
54
57
|
_defineProperty(this, "isComplete", void 0);
|
|
@@ -67,6 +70,8 @@ class CsvParser {
|
|
|
67
70
|
this.onProgress = onProgress;
|
|
68
71
|
this.onError = onError;
|
|
69
72
|
this.tables = [];
|
|
73
|
+
this.rowCount = 0;
|
|
74
|
+
this.rowsProcessed = 0;
|
|
70
75
|
this.chunks = 0;
|
|
71
76
|
this.totalChunks = isZip ? 0 : Math.ceil(file.size / Papa.LocalChunkSize);
|
|
72
77
|
this.isComplete = false;
|
|
@@ -112,13 +117,19 @@ class CsvParser {
|
|
|
112
117
|
return value === this.type.nullString ? '' : value;
|
|
113
118
|
}
|
|
114
119
|
parse() {
|
|
115
|
-
var handleParseDone = types => {
|
|
116
|
-
var toParse = this.isZip ? this.file.nodeStream(
|
|
117
|
-
// JsZip types are incorrect, thus the funny casting
|
|
118
|
-
// Actual parameter is 'nodebuffer'
|
|
119
|
-
'nodebuffer', this.handleNodeUpdate) : this.file;
|
|
120
|
+
var handleParseDone = (types, rowCount) => {
|
|
120
121
|
this.types = types;
|
|
121
|
-
|
|
122
|
+
this.rowCount = rowCount;
|
|
123
|
+
if (this.file instanceof File || this.file instanceof Blob) {
|
|
124
|
+
Papa.parse(this.file, this.config);
|
|
125
|
+
} else {
|
|
126
|
+
var zipStream = makeZipStreamHelper(this.file, this.handleNodeUpdate);
|
|
127
|
+
// This is actually a stream, but papaparse TS doesn't like it
|
|
128
|
+
Papa.parse(zipStream, this.config);
|
|
129
|
+
// The stream needs to be manually resumed since jszip starts paused
|
|
130
|
+
// Papaparse does not call resume and assumes the stream is already reading
|
|
131
|
+
zipStream.resume();
|
|
132
|
+
}
|
|
122
133
|
};
|
|
123
134
|
var typeParser = new CsvTypeParser(handleParseDone, this.file, this.readHeaders, this.config, this.type.nullString, this.onProgress, this.onError, this.totalChunks, this.isZip, this.type.shouldTrim);
|
|
124
135
|
typeParser.parse();
|
|
@@ -204,6 +215,7 @@ class CsvParser {
|
|
|
204
215
|
}
|
|
205
216
|
assertNotNull(this.headers);
|
|
206
217
|
assertNotNull(types);
|
|
218
|
+
this.rowsProcessed += columns[0].length;
|
|
207
219
|
session.newTable(this.headers, types, columns, this.timeZone).then(table => {
|
|
208
220
|
if (this.isCancelled) {
|
|
209
221
|
log.debug2('CSV parser cancelled.');
|
|
@@ -222,7 +234,9 @@ class CsvParser {
|
|
|
222
234
|
if (totalChunks > 0) {
|
|
223
235
|
progress = Math.round(tables.length / totalChunks * 50) + 50;
|
|
224
236
|
} else {
|
|
225
|
-
|
|
237
|
+
// The zip file can be read entirely while in the middle of parsing
|
|
238
|
+
// Since we know the number of rows from the type parsing, use that for progress
|
|
239
|
+
progress = Math.round(this.rowsProcessed / this.rowCount * 50) + 50;
|
|
226
240
|
}
|
|
227
241
|
log.debug2("CSV parser progress ".concat(progress));
|
|
228
242
|
onProgress(progress);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CsvParser.js","names":["Papa","Log","assertNotNull","DbNameValidator","CsvTypeParser","log","module","ZIP_CONSOLIDATE_CHUNKS","CsvParser","generateHeaderRecursive","n","header","char","concat","Math","floor","String","fromCharCode","constructor","onFileCompleted","session","file","type","readHeaders","onProgress","onError","timeZone","isZip","tables","chunks","totalChunks","ceil","size","LocalChunkSize","isComplete","zipProgress","numConsolidated","isCancelled","handleChunk","bind","handleError","handleComplete","handleNodeUpdate","handleCreateTable","config","delimiter","newline","escapeChar","dynamicTyping","error","skipEmptyLines","chunk","complete","fastMode","cancel","transpose","numColumns","array","numRows","length","columns","Array","fill","map","r","row","Error","c","value","shouldTrim","trim","nullCheck","nullString","parse","handleParseDone","types","toParse","nodeStream","typeParser","result","parser","debug2","abort","forEach","t","close","data","headers","legalizeColumnNames","slice","generateHeaders","consolidateChunks","index","e","consolidatedChunks","i","uploadConsolidatedChunks","toUpload","undefined","pause","newTable","then","table","resume","progress","round","catch","results","meta","aborted","metadata","percent","push"],"sources":["../../src/csv/CsvParser.ts"],"sourcesContent":["import Papa, { ParseLocalConfig, Parser, ParseResult } from 'papaparse';\nimport Log from '@deephaven/log';\nimport { assertNotNull, DbNameValidator } from '@deephaven/utils';\nimport type { IdeSession, Table } from '@deephaven/jsapi-types';\nimport type { JSZipObject } from 'jszip';\nimport CsvTypeParser from './CsvTypeParser';\nimport { CsvTypes } from './CsvFormats';\n\nconst log = Log.module('CsvParser');\n\n// This is based on jszip streaming ~15 KB chunks to papa parse\n// Want to consolidate to ~10 MB chunks\nconst ZIP_CONSOLIDATE_CHUNKS = 650;\n\ninterface CsvParserConstructor {\n onFileCompleted: (tables: Table[]) => void;\n session: IdeSession;\n file: Blob | JSZipObject;\n type: CsvTypes;\n readHeaders: boolean;\n onProgress: (progressValue: number) => boolean;\n onError: (e: unknown) => void;\n timeZone: string;\n isZip: boolean;\n}\n\n/**\n * Parser a CSV file in chunks and returns a table handle for each chunk.\n */\nclass CsvParser {\n // Generates column names A-Z, AA-AZ, BA-BZ, etc...\n static generateHeaders = (numColumns: number): string[] => {\n const headers = [];\n for (let i = 0; i < numColumns; i += 1) {\n headers.push(CsvParser.generateHeaderRecursive(i));\n }\n return headers;\n };\n\n static generateHeaderRecursive(n: number): string {\n let header = '';\n let char = n;\n if (n >= 26) {\n header = header.concat(\n CsvParser.generateHeaderRecursive(Math.floor(n / 26) - 1)\n );\n char = n % 26;\n }\n return header.concat(String.fromCharCode(65 + char));\n }\n\n constructor({\n onFileCompleted,\n session,\n file,\n type,\n readHeaders = true,\n onProgress,\n onError,\n timeZone,\n isZip,\n }: CsvParserConstructor) {\n this.onFileCompleted = onFileCompleted;\n this.session = session;\n this.file = file;\n this.isZip = isZip;\n this.type = type;\n this.readHeaders = readHeaders;\n this.timeZone = timeZone;\n this.onProgress = onProgress;\n this.onError = onError;\n this.tables = [];\n this.chunks = 0;\n this.totalChunks = isZip\n ? 0\n : Math.ceil((file as Blob).size / Papa.LocalChunkSize);\n this.isComplete = false;\n this.zipProgress = 0;\n this.numConsolidated = 0;\n this.isCancelled = false;\n\n this.handleChunk = this.handleChunk.bind(this);\n this.handleError = this.handleError.bind(this);\n this.handleComplete = this.handleComplete.bind(this);\n this.handleNodeUpdate = this.handleNodeUpdate.bind(this);\n this.handleCreateTable = this.handleCreateTable.bind(this);\n\n this.config = {\n delimiter: type.delimiter,\n newline: type.newline as '\\r\\n' | '\\n' | '\\r' | undefined,\n escapeChar: type.escapeChar,\n dynamicTyping: false,\n error: this.handleError,\n skipEmptyLines: type.skipEmptyLines,\n chunk: this.handleChunk,\n complete: this.handleComplete,\n fastMode: false,\n };\n }\n\n onFileCompleted: (tables: Table[]) => void;\n\n session: IdeSession;\n\n file: Blob | JSZipObject;\n\n isZip: boolean;\n\n type: CsvTypes;\n\n readHeaders: boolean;\n\n timeZone: string;\n\n onProgress: (progressValue: number) => boolean;\n\n onError: (e: unknown) => void;\n\n tables: Table[];\n\n headers?: string[];\n\n types?: string[];\n\n chunks: number;\n\n totalChunks: number;\n\n isComplete: boolean;\n\n zipProgress: number;\n\n consolidatedChunks?: string[][];\n\n numConsolidated: number;\n\n isCancelled: boolean;\n\n config: ParseLocalConfig<unknown, Blob | NodeJS.ReadableStream>;\n\n cancel(): void {\n this.isCancelled = true;\n }\n\n transpose(numColumns: number, array: string[][]): string[][] {\n const numRows = array.length;\n const columns = new Array(numColumns)\n .fill(null)\n .map(() => new Array(numRows));\n for (let r = 0; r < numRows; r += 1) {\n const row = array[r];\n if (row.length < numColumns) {\n throw new Error(\n `Insufficient columns. Expected ${numColumns} but found ${row.length}\\n${row}`\n );\n }\n for (let c = 0; c < numColumns; c += 1) {\n const value = this.type.shouldTrim ? array[r][c].trim() : array[r][c];\n columns[c][r] = this.nullCheck(value);\n }\n }\n return columns as string[][];\n }\n\n nullCheck(value: string): string {\n return value === this.type.nullString ? '' : value;\n }\n\n parse(): void {\n const handleParseDone = (types: string[]) => {\n const toParse = this.isZip\n ? (this.file as JSZipObject).nodeStream(\n // JsZip types are incorrect, thus the funny casting\n // Actual parameter is 'nodebuffer'\n 'nodebuffer' as 'nodestream',\n this.handleNodeUpdate\n )\n : (this.file as Blob);\n this.types = types;\n Papa.parse(toParse, this.config);\n };\n const typeParser = new CsvTypeParser(\n handleParseDone,\n this.file,\n this.readHeaders,\n this.config,\n this.type.nullString,\n this.onProgress,\n this.onError,\n this.totalChunks,\n this.isZip,\n this.type.shouldTrim\n );\n typeParser.parse();\n }\n\n handleChunk(result: ParseResult<string[]>, parser: Parser): void {\n const { readHeaders, onError, handleCreateTable, isZip, tables } = this;\n if (this.isCancelled) {\n log.debug2('CSV parser cancelled.');\n parser.abort();\n tables.forEach(t => t.close());\n return;\n }\n let { data } = result;\n if (!this.headers) {\n if (readHeaders) {\n this.headers = DbNameValidator.legalizeColumnNames(data[0]);\n data = data.slice(1);\n } else {\n this.headers = CsvParser.generateHeaders(data[0].length);\n }\n }\n\n let columns: string[][] = [];\n try {\n columns = this.transpose(this.headers.length, data);\n if (isZip) {\n // Zip file chunks are tiny, so consolidate them to avoid create thousands of small tables\n this.consolidateChunks(columns, parser);\n } else {\n const index = this.chunks;\n this.chunks += 1;\n handleCreateTable(index, columns, parser);\n }\n } catch (e: unknown) {\n onError(e);\n }\n }\n\n consolidateChunks(columns: string[][], parser: Parser): void {\n if (!this.consolidatedChunks) {\n this.consolidatedChunks = columns.slice();\n } else {\n for (let i = 0; i < columns.length; i += 1) {\n this.consolidatedChunks[i] = this.consolidatedChunks[i].concat(\n columns[i]\n );\n }\n }\n this.numConsolidated += 1;\n if (this.numConsolidated >= ZIP_CONSOLIDATE_CHUNKS || this.isComplete) {\n this.uploadConsolidatedChunks(parser);\n }\n }\n\n uploadConsolidatedChunks(parser: Parser | null): void {\n const { handleCreateTable } = this;\n const index = this.chunks;\n this.chunks += 1;\n const toUpload = this.consolidatedChunks?.slice();\n this.consolidatedChunks = undefined;\n this.numConsolidated = 0;\n assertNotNull(toUpload);\n handleCreateTable(index, toUpload, parser);\n }\n\n handleCreateTable(\n index: number,\n columns: string[][],\n parser: Parser | null\n ): void {\n const {\n session,\n tables,\n onFileCompleted,\n totalChunks,\n types,\n onProgress,\n onError,\n } = this;\n if (parser) {\n parser.pause();\n }\n assertNotNull(this.headers);\n assertNotNull(types);\n session\n .newTable(this.headers, types, columns, this.timeZone)\n .then(table => {\n if (this.isCancelled) {\n log.debug2('CSV parser cancelled.');\n if (parser) {\n parser.abort();\n }\n tables.forEach(t => t.close());\n return;\n }\n if (parser) {\n parser.resume();\n }\n tables[index] = table;\n // This accounts for 50% of parsing plus 50% already done by the type parser\n let progress = 0;\n if (totalChunks > 0) {\n progress = Math.round((tables.length / totalChunks) * 50) + 50;\n } else {\n progress = Math.round(50 + this.zipProgress / 2);\n }\n log.debug2(`CSV parser progress ${progress}`);\n onProgress(progress);\n if (this.isComplete && tables.length === this.chunks) {\n log.debug2('CSV parser complete.');\n onFileCompleted(tables);\n }\n })\n .catch(e => {\n if (!this.isComplete && parser) {\n parser.abort();\n }\n onError(e);\n });\n }\n\n handleComplete(results: ParseResult<unknown>): void {\n // results is undefined for a succesful parse, but has meta data for an abort\n if (results == null || results.meta.aborted == null) {\n this.isComplete = true;\n // Check if there are any consolidated chunks left over from a zip file\n if (this.consolidatedChunks) {\n this.uploadConsolidatedChunks(null);\n }\n }\n }\n\n handleError(error: unknown): void {\n const { onError } = this;\n onError(error);\n }\n\n handleNodeUpdate(metadata: { percent: number }): void {\n this.zipProgress = metadata.percent;\n }\n}\n\nexport default CsvParser;\n"],"mappings":";;;AAAA,OAAOA,IAAI,MAAiD,WAAW;AACvE,OAAOC,GAAG,MAAM,gBAAgB;AAChC,SAASC,aAAa,EAAEC,eAAe,QAAQ,kBAAkB;AAAC,OAG3DC,aAAa;AAGpB,IAAMC,GAAG,GAAGJ,GAAG,CAACK,MAAM,CAAC,WAAW,CAAC;;AAEnC;AACA;AACA,IAAMC,sBAAsB,GAAG,GAAG;AAclC;AACA;AACA;AACA,MAAMC,SAAS,CAAC;EACd;;EASA,OAAOC,uBAAuB,CAACC,CAAS,EAAU;IAChD,IAAIC,MAAM,GAAG,EAAE;IACf,IAAIC,IAAI,GAAGF,CAAC;IACZ,IAAIA,CAAC,IAAI,EAAE,EAAE;MACXC,MAAM,GAAGA,MAAM,CAACE,MAAM,CACpBL,SAAS,CAACC,uBAAuB,CAACK,IAAI,CAACC,KAAK,CAACL,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAC1D;MACDE,IAAI,GAAGF,CAAC,GAAG,EAAE;IACf;IACA,OAAOC,MAAM,CAACE,MAAM,CAACG,MAAM,CAACC,YAAY,CAAC,EAAE,GAAGL,IAAI,CAAC,CAAC;EACtD;EAEAM,WAAW,OAUc;IAAA,IAVb;MACVC,eAAe;MACfC,OAAO;MACPC,IAAI;MACJC,IAAI;MACJC,WAAW,GAAG,IAAI;MAClBC,UAAU;MACVC,OAAO;MACPC,QAAQ;MACRC;IACoB,CAAC;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IACrB,IAAI,CAACR,eAAe,GAAGA,eAAe;IACtC,IAAI,CAACC,OAAO,GAAGA,OAAO;IACtB,IAAI,CAACC,IAAI,GAAGA,IAAI;IAChB,IAAI,CAACM,KAAK,GAAGA,KAAK;IAClB,IAAI,CAACL,IAAI,GAAGA,IAAI;IAChB,IAAI,CAACC,WAAW,GAAGA,WAAW;IAC9B,IAAI,CAACG,QAAQ,GAAGA,QAAQ;IACxB,IAAI,CAACF,UAAU,GAAGA,UAAU;IAC5B,IAAI,CAACC,OAAO,GAAGA,OAAO;IACtB,IAAI,CAACG,MAAM,GAAG,EAAE;IAChB,IAAI,CAACC,MAAM,GAAG,CAAC;IACf,IAAI,CAACC,WAAW,GAAGH,KAAK,GACpB,CAAC,GACDb,IAAI,CAACiB,IAAI,CAAEV,IAAI,CAAUW,IAAI,GAAGhC,IAAI,CAACiC,cAAc,CAAC;IACxD,IAAI,CAACC,UAAU,GAAG,KAAK;IACvB,IAAI,CAACC,WAAW,GAAG,CAAC;IACpB,IAAI,CAACC,eAAe,GAAG,CAAC;IACxB,IAAI,CAACC,WAAW,GAAG,KAAK;IAExB,IAAI,CAACC,WAAW,GAAG,IAAI,CAACA,WAAW,CAACC,IAAI,CAAC,IAAI,CAAC;IAC9C,IAAI,CAACC,WAAW,GAAG,IAAI,CAACA,WAAW,CAACD,IAAI,CAAC,IAAI,CAAC;IAC9C,IAAI,CAACE,cAAc,GAAG,IAAI,CAACA,cAAc,CAACF,IAAI,CAAC,IAAI,CAAC;IACpD,IAAI,CAACG,gBAAgB,GAAG,IAAI,CAACA,gBAAgB,CAACH,IAAI,CAAC,IAAI,CAAC;IACxD,IAAI,CAACI,iBAAiB,GAAG,IAAI,CAACA,iBAAiB,CAACJ,IAAI,CAAC,IAAI,CAAC;IAE1D,IAAI,CAACK,MAAM,GAAG;MACZC,SAAS,EAAEvB,IAAI,CAACuB,SAAS;MACzBC,OAAO,EAAExB,IAAI,CAACwB,OAA2C;MACzDC,UAAU,EAAEzB,IAAI,CAACyB,UAAU;MAC3BC,aAAa,EAAE,KAAK;MACpBC,KAAK,EAAE,IAAI,CAACT,WAAW;MACvBU,cAAc,EAAE5B,IAAI,CAAC4B,cAAc;MACnCC,KAAK,EAAE,IAAI,CAACb,WAAW;MACvBc,QAAQ,EAAE,IAAI,CAACX,cAAc;MAC7BY,QAAQ,EAAE;IACZ,CAAC;EACH;EA0CAC,MAAM,GAAS;IACb,IAAI,CAACjB,WAAW,GAAG,IAAI;EACzB;EAEAkB,SAAS,CAACC,UAAkB,EAAEC,KAAiB,EAAc;IAC3D,IAAMC,OAAO,GAAGD,KAAK,CAACE,MAAM;IAC5B,IAAMC,OAAO,GAAG,IAAIC,KAAK,CAACL,UAAU,CAAC,CAClCM,IAAI,CAAC,IAAI,CAAC,CACVC,GAAG,CAAC,MAAM,IAAIF,KAAK,CAACH,OAAO,CAAC,CAAC;IAChC,KAAK,IAAIM,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGN,OAAO,EAAEM,CAAC,IAAI,CAAC,EAAE;MACnC,IAAMC,GAAG,GAAGR,KAAK,CAACO,CAAC,CAAC;MACpB,IAAIC,GAAG,CAACN,MAAM,GAAGH,UAAU,EAAE;QAC3B,MAAM,IAAIU,KAAK,0CACqBV,UAAU,wBAAcS,GAAG,CAACN,MAAM,eAAKM,GAAG,EAC7E;MACH;MACA,KAAK,IAAIE,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGX,UAAU,EAAEW,CAAC,IAAI,CAAC,EAAE;QACtC,IAAMC,KAAK,GAAG,IAAI,CAAC9C,IAAI,CAAC+C,UAAU,GAAGZ,KAAK,CAACO,CAAC,CAAC,CAACG,CAAC,CAAC,CAACG,IAAI,EAAE,GAAGb,KAAK,CAACO,CAAC,CAAC,CAACG,CAAC,CAAC;QACrEP,OAAO,CAACO,CAAC,CAAC,CAACH,CAAC,CAAC,GAAG,IAAI,CAACO,SAAS,CAACH,KAAK,CAAC;MACvC;IACF;IACA,OAAOR,OAAO;EAChB;EAEAW,SAAS,CAACH,KAAa,EAAU;IAC/B,OAAOA,KAAK,KAAK,IAAI,CAAC9C,IAAI,CAACkD,UAAU,GAAG,EAAE,GAAGJ,KAAK;EACpD;EAEAK,KAAK,GAAS;IACZ,IAAMC,eAAe,GAAIC,KAAe,IAAK;MAC3C,IAAMC,OAAO,GAAG,IAAI,CAACjD,KAAK,GACrB,IAAI,CAACN,IAAI,CAAiBwD,UAAU;MACnC;MACA;MACA,YAAY,EACZ,IAAI,CAACnC,gBAAgB,CACtB,GACA,IAAI,CAACrB,IAAa;MACvB,IAAI,CAACsD,KAAK,GAAGA,KAAK;MAClB3E,IAAI,CAACyE,KAAK,CAACG,OAAO,EAAE,IAAI,CAAChC,MAAM,CAAC;IAClC,CAAC;IACD,IAAMkC,UAAU,GAAG,IAAI1E,aAAa,CAClCsE,eAAe,EACf,IAAI,CAACrD,IAAI,EACT,IAAI,CAACE,WAAW,EAChB,IAAI,CAACqB,MAAM,EACX,IAAI,CAACtB,IAAI,CAACkD,UAAU,EACpB,IAAI,CAAChD,UAAU,EACf,IAAI,CAACC,OAAO,EACZ,IAAI,CAACK,WAAW,EAChB,IAAI,CAACH,KAAK,EACV,IAAI,CAACL,IAAI,CAAC+C,UAAU,CACrB;IACDS,UAAU,CAACL,KAAK,EAAE;EACpB;EAEAnC,WAAW,CAACyC,MAA6B,EAAEC,MAAc,EAAQ;IAC/D,IAAM;MAAEzD,WAAW;MAAEE,OAAO;MAAEkB,iBAAiB;MAAEhB,KAAK;MAAEC;IAAO,CAAC,GAAG,IAAI;IACvE,IAAI,IAAI,CAACS,WAAW,EAAE;MACpBhC,GAAG,CAAC4E,MAAM,CAAC,uBAAuB,CAAC;MACnCD,MAAM,CAACE,KAAK,EAAE;MACdtD,MAAM,CAACuD,OAAO,CAACC,CAAC,IAAIA,CAAC,CAACC,KAAK,EAAE,CAAC;MAC9B;IACF;IACA,IAAI;MAAEC;IAAK,CAAC,GAAGP,MAAM;IACrB,IAAI,CAAC,IAAI,CAACQ,OAAO,EAAE;MACjB,IAAIhE,WAAW,EAAE;QACf,IAAI,CAACgE,OAAO,GAAGpF,eAAe,CAACqF,mBAAmB,CAACF,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3DA,IAAI,GAAGA,IAAI,CAACG,KAAK,CAAC,CAAC,CAAC;MACtB,CAAC,MAAM;QACL,IAAI,CAACF,OAAO,GAAG/E,SAAS,CAACkF,eAAe,CAACJ,IAAI,CAAC,CAAC,CAAC,CAAC3B,MAAM,CAAC;MAC1D;IACF;IAEA,IAAIC,OAAmB,GAAG,EAAE;IAC5B,IAAI;MACFA,OAAO,GAAG,IAAI,CAACL,SAAS,CAAC,IAAI,CAACgC,OAAO,CAAC5B,MAAM,EAAE2B,IAAI,CAAC;MACnD,IAAI3D,KAAK,EAAE;QACT;QACA,IAAI,CAACgE,iBAAiB,CAAC/B,OAAO,EAAEoB,MAAM,CAAC;MACzC,CAAC,MAAM;QACL,IAAMY,KAAK,GAAG,IAAI,CAAC/D,MAAM;QACzB,IAAI,CAACA,MAAM,IAAI,CAAC;QAChBc,iBAAiB,CAACiD,KAAK,EAAEhC,OAAO,EAAEoB,MAAM,CAAC;MAC3C;IACF,CAAC,CAAC,OAAOa,CAAU,EAAE;MACnBpE,OAAO,CAACoE,CAAC,CAAC;IACZ;EACF;EAEAF,iBAAiB,CAAC/B,OAAmB,EAAEoB,MAAc,EAAQ;IAC3D,IAAI,CAAC,IAAI,CAACc,kBAAkB,EAAE;MAC5B,IAAI,CAACA,kBAAkB,GAAGlC,OAAO,CAAC6B,KAAK,EAAE;IAC3C,CAAC,MAAM;MACL,KAAK,IAAIM,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGnC,OAAO,CAACD,MAAM,EAAEoC,CAAC,IAAI,CAAC,EAAE;QAC1C,IAAI,CAACD,kBAAkB,CAACC,CAAC,CAAC,GAAG,IAAI,CAACD,kBAAkB,CAACC,CAAC,CAAC,CAAClF,MAAM,CAC5D+C,OAAO,CAACmC,CAAC,CAAC,CACX;MACH;IACF;IACA,IAAI,CAAC3D,eAAe,IAAI,CAAC;IACzB,IAAI,IAAI,CAACA,eAAe,IAAI7B,sBAAsB,IAAI,IAAI,CAAC2B,UAAU,EAAE;MACrE,IAAI,CAAC8D,wBAAwB,CAAChB,MAAM,CAAC;IACvC;EACF;EAEAgB,wBAAwB,CAAChB,MAAqB,EAAQ;IAAA;IACpD,IAAM;MAAErC;IAAkB,CAAC,GAAG,IAAI;IAClC,IAAMiD,KAAK,GAAG,IAAI,CAAC/D,MAAM;IACzB,IAAI,CAACA,MAAM,IAAI,CAAC;IAChB,IAAMoE,QAAQ,4BAAG,IAAI,CAACH,kBAAkB,0DAAvB,sBAAyBL,KAAK,EAAE;IACjD,IAAI,CAACK,kBAAkB,GAAGI,SAAS;IACnC,IAAI,CAAC9D,eAAe,GAAG,CAAC;IACxBlC,aAAa,CAAC+F,QAAQ,CAAC;IACvBtD,iBAAiB,CAACiD,KAAK,EAAEK,QAAQ,EAAEjB,MAAM,CAAC;EAC5C;EAEArC,iBAAiB,CACfiD,KAAa,EACbhC,OAAmB,EACnBoB,MAAqB,EACf;IACN,IAAM;MACJ5D,OAAO;MACPQ,MAAM;MACNT,eAAe;MACfW,WAAW;MACX6C,KAAK;MACLnD,UAAU;MACVC;IACF,CAAC,GAAG,IAAI;IACR,IAAIuD,MAAM,EAAE;MACVA,MAAM,CAACmB,KAAK,EAAE;IAChB;IACAjG,aAAa,CAAC,IAAI,CAACqF,OAAO,CAAC;IAC3BrF,aAAa,CAACyE,KAAK,CAAC;IACpBvD,OAAO,CACJgF,QAAQ,CAAC,IAAI,CAACb,OAAO,EAAEZ,KAAK,EAAEf,OAAO,EAAE,IAAI,CAAClC,QAAQ,CAAC,CACrD2E,IAAI,CAACC,KAAK,IAAI;MACb,IAAI,IAAI,CAACjE,WAAW,EAAE;QACpBhC,GAAG,CAAC4E,MAAM,CAAC,uBAAuB,CAAC;QACnC,IAAID,MAAM,EAAE;UACVA,MAAM,CAACE,KAAK,EAAE;QAChB;QACAtD,MAAM,CAACuD,OAAO,CAACC,CAAC,IAAIA,CAAC,CAACC,KAAK,EAAE,CAAC;QAC9B;MACF;MACA,IAAIL,MAAM,EAAE;QACVA,MAAM,CAACuB,MAAM,EAAE;MACjB;MACA3E,MAAM,CAACgE,KAAK,CAAC,GAAGU,KAAK;MACrB;MACA,IAAIE,QAAQ,GAAG,CAAC;MAChB,IAAI1E,WAAW,GAAG,CAAC,EAAE;QACnB0E,QAAQ,GAAG1F,IAAI,CAAC2F,KAAK,CAAE7E,MAAM,CAAC+B,MAAM,GAAG7B,WAAW,GAAI,EAAE,CAAC,GAAG,EAAE;MAChE,CAAC,MAAM;QACL0E,QAAQ,GAAG1F,IAAI,CAAC2F,KAAK,CAAC,EAAE,GAAG,IAAI,CAACtE,WAAW,GAAG,CAAC,CAAC;MAClD;MACA9B,GAAG,CAAC4E,MAAM,+BAAwBuB,QAAQ,EAAG;MAC7ChF,UAAU,CAACgF,QAAQ,CAAC;MACpB,IAAI,IAAI,CAACtE,UAAU,IAAIN,MAAM,CAAC+B,MAAM,KAAK,IAAI,CAAC9B,MAAM,EAAE;QACpDxB,GAAG,CAAC4E,MAAM,CAAC,sBAAsB,CAAC;QAClC9D,eAAe,CAACS,MAAM,CAAC;MACzB;IACF,CAAC,CAAC,CACD8E,KAAK,CAACb,CAAC,IAAI;MACV,IAAI,CAAC,IAAI,CAAC3D,UAAU,IAAI8C,MAAM,EAAE;QAC9BA,MAAM,CAACE,KAAK,EAAE;MAChB;MACAzD,OAAO,CAACoE,CAAC,CAAC;IACZ,CAAC,CAAC;EACN;EAEApD,cAAc,CAACkE,OAA6B,EAAQ;IAClD;IACA,IAAIA,OAAO,IAAI,IAAI,IAAIA,OAAO,CAACC,IAAI,CAACC,OAAO,IAAI,IAAI,EAAE;MACnD,IAAI,CAAC3E,UAAU,GAAG,IAAI;MACtB;MACA,IAAI,IAAI,CAAC4D,kBAAkB,EAAE;QAC3B,IAAI,CAACE,wBAAwB,CAAC,IAAI,CAAC;MACrC;IACF;EACF;EAEAxD,WAAW,CAACS,KAAc,EAAQ;IAChC,IAAM;MAAExB;IAAQ,CAAC,GAAG,IAAI;IACxBA,OAAO,CAACwB,KAAK,CAAC;EAChB;EAEAP,gBAAgB,CAACoE,QAA6B,EAAQ;IACpD,IAAI,CAAC3E,WAAW,GAAG2E,QAAQ,CAACC,OAAO;EACrC;AACF;AAAC,gBA/SKvG,SAAS,qBAEagD,UAAkB,IAAe;EACzD,IAAM+B,OAAO,GAAG,EAAE;EAClB,KAAK,IAAIQ,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGvC,UAAU,EAAEuC,CAAC,IAAI,CAAC,EAAE;IACtCR,OAAO,CAACyB,IAAI,CAACxG,SAAS,CAACC,uBAAuB,CAACsF,CAAC,CAAC,CAAC;EACpD;EACA,OAAOR,OAAO;AAChB,CAAC;AAySH,eAAe/E,SAAS"}
|
|
1
|
+
{"version":3,"file":"CsvParser.js","names":["Papa","Log","assertNotNull","DbNameValidator","CsvTypeParser","makeZipStreamHelper","log","module","ZIP_CONSOLIDATE_CHUNKS","CsvParser","generateHeaderRecursive","n","header","char","concat","Math","floor","String","fromCharCode","constructor","onFileCompleted","session","file","type","readHeaders","onProgress","onError","timeZone","isZip","tables","rowCount","rowsProcessed","chunks","totalChunks","ceil","size","LocalChunkSize","isComplete","zipProgress","numConsolidated","isCancelled","handleChunk","bind","handleError","handleComplete","handleNodeUpdate","handleCreateTable","config","delimiter","newline","escapeChar","dynamicTyping","error","skipEmptyLines","chunk","complete","fastMode","cancel","transpose","numColumns","array","numRows","length","columns","Array","fill","map","r","row","Error","c","value","shouldTrim","trim","nullCheck","nullString","parse","handleParseDone","types","File","Blob","zipStream","resume","typeParser","result","parser","debug2","abort","forEach","t","close","data","headers","legalizeColumnNames","slice","generateHeaders","consolidateChunks","index","e","consolidatedChunks","i","uploadConsolidatedChunks","toUpload","undefined","pause","newTable","then","table","progress","round","catch","results","meta","aborted","metadata","percent","push"],"sources":["../../src/csv/CsvParser.ts"],"sourcesContent":["import Papa, { ParseLocalConfig, Parser, ParseResult } from 'papaparse';\nimport Log from '@deephaven/log';\nimport { assertNotNull, DbNameValidator } from '@deephaven/utils';\nimport type { IdeSession, Table } from '@deephaven/jsapi-types';\nimport type { JSZipObject } from 'jszip';\nimport CsvTypeParser from './CsvTypeParser';\nimport { CsvTypes } from './CsvFormats';\nimport makeZipStreamHelper from './ZipStreamHelper';\n\nconst log = Log.module('CsvParser');\n\n// This is based on jszip streaming ~15 KB chunks to papa parse\n// Want to consolidate to ~10 MB chunks\nconst ZIP_CONSOLIDATE_CHUNKS = 650;\n\ninterface CsvParserConstructor {\n onFileCompleted: (tables: Table[]) => void;\n session: IdeSession;\n file: File | Blob | JSZipObject;\n type: CsvTypes;\n readHeaders: boolean;\n onProgress: (progressValue: number) => boolean;\n onError: (e: unknown) => void;\n timeZone: string;\n isZip: boolean;\n}\n\n/**\n * Parser a CSV file in chunks and returns a table handle for each chunk.\n */\nclass CsvParser {\n // Generates column names A-Z, AA-AZ, BA-BZ, etc...\n static generateHeaders = (numColumns: number): string[] => {\n const headers = [];\n for (let i = 0; i < numColumns; i += 1) {\n headers.push(CsvParser.generateHeaderRecursive(i));\n }\n return headers;\n };\n\n static generateHeaderRecursive(n: number): string {\n let header = '';\n let char = n;\n if (n >= 26) {\n header = header.concat(\n CsvParser.generateHeaderRecursive(Math.floor(n / 26) - 1)\n );\n char = n % 26;\n }\n return header.concat(String.fromCharCode(65 + char));\n }\n\n constructor({\n onFileCompleted,\n session,\n file,\n type,\n readHeaders = true,\n onProgress,\n onError,\n timeZone,\n isZip,\n }: CsvParserConstructor) {\n this.onFileCompleted = onFileCompleted;\n this.session = session;\n this.file = file;\n this.isZip = isZip;\n this.type = type;\n this.readHeaders = readHeaders;\n this.timeZone = timeZone;\n this.onProgress = onProgress;\n this.onError = onError;\n this.tables = [];\n this.rowCount = 0;\n this.rowsProcessed = 0;\n this.chunks = 0;\n this.totalChunks = isZip\n ? 0\n : Math.ceil((file as Blob).size / Papa.LocalChunkSize);\n this.isComplete = false;\n this.zipProgress = 0;\n this.numConsolidated = 0;\n this.isCancelled = false;\n\n this.handleChunk = this.handleChunk.bind(this);\n this.handleError = this.handleError.bind(this);\n this.handleComplete = this.handleComplete.bind(this);\n this.handleNodeUpdate = this.handleNodeUpdate.bind(this);\n this.handleCreateTable = this.handleCreateTable.bind(this);\n\n this.config = {\n delimiter: type.delimiter,\n newline: type.newline as '\\r\\n' | '\\n' | '\\r' | undefined,\n escapeChar: type.escapeChar,\n dynamicTyping: false,\n error: this.handleError,\n skipEmptyLines: type.skipEmptyLines,\n chunk: this.handleChunk,\n complete: this.handleComplete,\n fastMode: false,\n };\n }\n\n onFileCompleted: (tables: Table[]) => void;\n\n session: IdeSession;\n\n file: File | Blob | JSZipObject;\n\n isZip: boolean;\n\n type: CsvTypes;\n\n readHeaders: boolean;\n\n timeZone: string;\n\n onProgress: (progressValue: number) => boolean;\n\n onError: (e: unknown) => void;\n\n tables: Table[];\n\n headers?: string[];\n\n types?: string[];\n\n rowCount: number;\n\n rowsProcessed: number;\n\n chunks: number;\n\n totalChunks: number;\n\n isComplete: boolean;\n\n zipProgress: number;\n\n consolidatedChunks?: string[][];\n\n numConsolidated: number;\n\n isCancelled: boolean;\n\n config: ParseLocalConfig<unknown, Blob | NodeJS.ReadableStream>;\n\n cancel(): void {\n this.isCancelled = true;\n }\n\n transpose(numColumns: number, array: string[][]): string[][] {\n const numRows = array.length;\n const columns = new Array(numColumns)\n .fill(null)\n .map(() => new Array(numRows));\n for (let r = 0; r < numRows; r += 1) {\n const row = array[r];\n if (row.length < numColumns) {\n throw new Error(\n `Insufficient columns. Expected ${numColumns} but found ${row.length}\\n${row}`\n );\n }\n for (let c = 0; c < numColumns; c += 1) {\n const value = this.type.shouldTrim ? array[r][c].trim() : array[r][c];\n columns[c][r] = this.nullCheck(value);\n }\n }\n return columns as string[][];\n }\n\n nullCheck(value: string): string {\n return value === this.type.nullString ? '' : value;\n }\n\n parse(): void {\n const handleParseDone = (types: string[], rowCount: number) => {\n this.types = types;\n this.rowCount = rowCount;\n\n if (this.file instanceof File || this.file instanceof Blob) {\n Papa.parse(this.file, this.config);\n } else {\n const zipStream = makeZipStreamHelper(this.file, this.handleNodeUpdate);\n // This is actually a stream, but papaparse TS doesn't like it\n Papa.parse(zipStream as unknown as Blob, this.config);\n // The stream needs to be manually resumed since jszip starts paused\n // Papaparse does not call resume and assumes the stream is already reading\n zipStream.resume();\n }\n };\n const typeParser = new CsvTypeParser(\n handleParseDone,\n this.file,\n this.readHeaders,\n this.config,\n this.type.nullString,\n this.onProgress,\n this.onError,\n this.totalChunks,\n this.isZip,\n this.type.shouldTrim\n );\n typeParser.parse();\n }\n\n handleChunk(result: ParseResult<string[]>, parser: Parser): void {\n const { readHeaders, onError, handleCreateTable, isZip, tables } = this;\n if (this.isCancelled) {\n log.debug2('CSV parser cancelled.');\n parser.abort();\n tables.forEach(t => t.close());\n return;\n }\n let { data } = result;\n if (!this.headers) {\n if (readHeaders) {\n this.headers = DbNameValidator.legalizeColumnNames(data[0]);\n data = data.slice(1);\n } else {\n this.headers = CsvParser.generateHeaders(data[0].length);\n }\n }\n\n let columns: string[][] = [];\n try {\n columns = this.transpose(this.headers.length, data);\n if (isZip) {\n // Zip file chunks are tiny, so consolidate them to avoid create thousands of small tables\n this.consolidateChunks(columns, parser);\n } else {\n const index = this.chunks;\n this.chunks += 1;\n handleCreateTable(index, columns, parser);\n }\n } catch (e: unknown) {\n onError(e);\n }\n }\n\n consolidateChunks(columns: string[][], parser: Parser): void {\n if (!this.consolidatedChunks) {\n this.consolidatedChunks = columns.slice();\n } else {\n for (let i = 0; i < columns.length; i += 1) {\n this.consolidatedChunks[i] = this.consolidatedChunks[i].concat(\n columns[i]\n );\n }\n }\n this.numConsolidated += 1;\n if (this.numConsolidated >= ZIP_CONSOLIDATE_CHUNKS || this.isComplete) {\n this.uploadConsolidatedChunks(parser);\n }\n }\n\n uploadConsolidatedChunks(parser: Parser | null): void {\n const { handleCreateTable } = this;\n const index = this.chunks;\n this.chunks += 1;\n const toUpload = this.consolidatedChunks?.slice();\n this.consolidatedChunks = undefined;\n this.numConsolidated = 0;\n assertNotNull(toUpload);\n handleCreateTable(index, toUpload, parser);\n }\n\n handleCreateTable(\n index: number,\n columns: string[][],\n parser: Parser | null\n ): void {\n const {\n session,\n tables,\n onFileCompleted,\n totalChunks,\n types,\n onProgress,\n onError,\n } = this;\n if (parser) {\n parser.pause();\n }\n assertNotNull(this.headers);\n assertNotNull(types);\n\n this.rowsProcessed += columns[0].length;\n\n session\n .newTable(this.headers, types, columns, this.timeZone)\n .then(table => {\n if (this.isCancelled) {\n log.debug2('CSV parser cancelled.');\n if (parser) {\n parser.abort();\n }\n tables.forEach(t => t.close());\n return;\n }\n if (parser) {\n parser.resume();\n }\n tables[index] = table;\n // This accounts for 50% of parsing plus 50% already done by the type parser\n let progress = 0;\n if (totalChunks > 0) {\n progress = Math.round((tables.length / totalChunks) * 50) + 50;\n } else {\n // The zip file can be read entirely while in the middle of parsing\n // Since we know the number of rows from the type parsing, use that for progress\n progress = Math.round((this.rowsProcessed / this.rowCount) * 50) + 50;\n }\n log.debug2(`CSV parser progress ${progress}`);\n onProgress(progress);\n if (this.isComplete && tables.length === this.chunks) {\n log.debug2('CSV parser complete.');\n onFileCompleted(tables);\n }\n })\n .catch(e => {\n if (!this.isComplete && parser) {\n parser.abort();\n }\n onError(e);\n });\n }\n\n handleComplete(results: ParseResult<unknown>): void {\n // results is undefined for a succesful parse, but has meta data for an abort\n if (results == null || results.meta.aborted == null) {\n this.isComplete = true;\n // Check if there are any consolidated chunks left over from a zip file\n if (this.consolidatedChunks) {\n this.uploadConsolidatedChunks(null);\n }\n }\n }\n\n handleError(error: unknown): void {\n const { onError } = this;\n onError(error);\n }\n\n handleNodeUpdate(metadata: { percent: number }): void {\n this.zipProgress = metadata.percent;\n }\n}\n\nexport default CsvParser;\n"],"mappings":";;;AAAA,OAAOA,IAAI,MAAiD,WAAW;AACvE,OAAOC,GAAG,MAAM,gBAAgB;AAChC,SAASC,aAAa,EAAEC,eAAe,QAAQ,kBAAkB;AAAC,OAG3DC,aAAa;AAAA,OAEbC,mBAAmB;AAE1B,IAAMC,GAAG,GAAGL,GAAG,CAACM,MAAM,CAAC,WAAW,CAAC;;AAEnC;AACA;AACA,IAAMC,sBAAsB,GAAG,GAAG;AAclC;AACA;AACA;AACA,MAAMC,SAAS,CAAC;EACd;;EASA,OAAOC,uBAAuB,CAACC,CAAS,EAAU;IAChD,IAAIC,MAAM,GAAG,EAAE;IACf,IAAIC,IAAI,GAAGF,CAAC;IACZ,IAAIA,CAAC,IAAI,EAAE,EAAE;MACXC,MAAM,GAAGA,MAAM,CAACE,MAAM,CACpBL,SAAS,CAACC,uBAAuB,CAACK,IAAI,CAACC,KAAK,CAACL,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAC1D;MACDE,IAAI,GAAGF,CAAC,GAAG,EAAE;IACf;IACA,OAAOC,MAAM,CAACE,MAAM,CAACG,MAAM,CAACC,YAAY,CAAC,EAAE,GAAGL,IAAI,CAAC,CAAC;EACtD;EAEAM,WAAW,OAUc;IAAA,IAVb;MACVC,eAAe;MACfC,OAAO;MACPC,IAAI;MACJC,IAAI;MACJC,WAAW,GAAG,IAAI;MAClBC,UAAU;MACVC,OAAO;MACPC,QAAQ;MACRC;IACoB,CAAC;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IACrB,IAAI,CAACR,eAAe,GAAGA,eAAe;IACtC,IAAI,CAACC,OAAO,GAAGA,OAAO;IACtB,IAAI,CAACC,IAAI,GAAGA,IAAI;IAChB,IAAI,CAACM,KAAK,GAAGA,KAAK;IAClB,IAAI,CAACL,IAAI,GAAGA,IAAI;IAChB,IAAI,CAACC,WAAW,GAAGA,WAAW;IAC9B,IAAI,CAACG,QAAQ,GAAGA,QAAQ;IACxB,IAAI,CAACF,UAAU,GAAGA,UAAU;IAC5B,IAAI,CAACC,OAAO,GAAGA,OAAO;IACtB,IAAI,CAACG,MAAM,GAAG,EAAE;IAChB,IAAI,CAACC,QAAQ,GAAG,CAAC;IACjB,IAAI,CAACC,aAAa,GAAG,CAAC;IACtB,IAAI,CAACC,MAAM,GAAG,CAAC;IACf,IAAI,CAACC,WAAW,GAAGL,KAAK,GACpB,CAAC,GACDb,IAAI,CAACmB,IAAI,CAAEZ,IAAI,CAAUa,IAAI,GAAGnC,IAAI,CAACoC,cAAc,CAAC;IACxD,IAAI,CAACC,UAAU,GAAG,KAAK;IACvB,IAAI,CAACC,WAAW,GAAG,CAAC;IACpB,IAAI,CAACC,eAAe,GAAG,CAAC;IACxB,IAAI,CAACC,WAAW,GAAG,KAAK;IAExB,IAAI,CAACC,WAAW,GAAG,IAAI,CAACA,WAAW,CAACC,IAAI,CAAC,IAAI,CAAC;IAC9C,IAAI,CAACC,WAAW,GAAG,IAAI,CAACA,WAAW,CAACD,IAAI,CAAC,IAAI,CAAC;IAC9C,IAAI,CAACE,cAAc,GAAG,IAAI,CAACA,cAAc,CAACF,IAAI,CAAC,IAAI,CAAC;IACpD,IAAI,CAACG,gBAAgB,GAAG,IAAI,CAACA,gBAAgB,CAACH,IAAI,CAAC,IAAI,CAAC;IACxD,IAAI,CAACI,iBAAiB,GAAG,IAAI,CAACA,iBAAiB,CAACJ,IAAI,CAAC,IAAI,CAAC;IAE1D,IAAI,CAACK,MAAM,GAAG;MACZC,SAAS,EAAEzB,IAAI,CAACyB,SAAS;MACzBC,OAAO,EAAE1B,IAAI,CAAC0B,OAA2C;MACzDC,UAAU,EAAE3B,IAAI,CAAC2B,UAAU;MAC3BC,aAAa,EAAE,KAAK;MACpBC,KAAK,EAAE,IAAI,CAACT,WAAW;MACvBU,cAAc,EAAE9B,IAAI,CAAC8B,cAAc;MACnCC,KAAK,EAAE,IAAI,CAACb,WAAW;MACvBc,QAAQ,EAAE,IAAI,CAACX,cAAc;MAC7BY,QAAQ,EAAE;IACZ,CAAC;EACH;EA8CAC,MAAM,GAAS;IACb,IAAI,CAACjB,WAAW,GAAG,IAAI;EACzB;EAEAkB,SAAS,CAACC,UAAkB,EAAEC,KAAiB,EAAc;IAC3D,IAAMC,OAAO,GAAGD,KAAK,CAACE,MAAM;IAC5B,IAAMC,OAAO,GAAG,IAAIC,KAAK,CAACL,UAAU,CAAC,CAClCM,IAAI,CAAC,IAAI,CAAC,CACVC,GAAG,CAAC,MAAM,IAAIF,KAAK,CAACH,OAAO,CAAC,CAAC;IAChC,KAAK,IAAIM,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGN,OAAO,EAAEM,CAAC,IAAI,CAAC,EAAE;MACnC,IAAMC,GAAG,GAAGR,KAAK,CAACO,CAAC,CAAC;MACpB,IAAIC,GAAG,CAACN,MAAM,GAAGH,UAAU,EAAE;QAC3B,MAAM,IAAIU,KAAK,0CACqBV,UAAU,wBAAcS,GAAG,CAACN,MAAM,eAAKM,GAAG,EAC7E;MACH;MACA,KAAK,IAAIE,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGX,UAAU,EAAEW,CAAC,IAAI,CAAC,EAAE;QACtC,IAAMC,KAAK,GAAG,IAAI,CAAChD,IAAI,CAACiD,UAAU,GAAGZ,KAAK,CAACO,CAAC,CAAC,CAACG,CAAC,CAAC,CAACG,IAAI,EAAE,GAAGb,KAAK,CAACO,CAAC,CAAC,CAACG,CAAC,CAAC;QACrEP,OAAO,CAACO,CAAC,CAAC,CAACH,CAAC,CAAC,GAAG,IAAI,CAACO,SAAS,CAACH,KAAK,CAAC;MACvC;IACF;IACA,OAAOR,OAAO;EAChB;EAEAW,SAAS,CAACH,KAAa,EAAU;IAC/B,OAAOA,KAAK,KAAK,IAAI,CAAChD,IAAI,CAACoD,UAAU,GAAG,EAAE,GAAGJ,KAAK;EACpD;EAEAK,KAAK,GAAS;IACZ,IAAMC,eAAe,GAAG,CAACC,KAAe,EAAEhD,QAAgB,KAAK;MAC7D,IAAI,CAACgD,KAAK,GAAGA,KAAK;MAClB,IAAI,CAAChD,QAAQ,GAAGA,QAAQ;MAExB,IAAI,IAAI,CAACR,IAAI,YAAYyD,IAAI,IAAI,IAAI,CAACzD,IAAI,YAAY0D,IAAI,EAAE;QAC1DhF,IAAI,CAAC4E,KAAK,CAAC,IAAI,CAACtD,IAAI,EAAE,IAAI,CAACyB,MAAM,CAAC;MACpC,CAAC,MAAM;QACL,IAAMkC,SAAS,GAAG5E,mBAAmB,CAAC,IAAI,CAACiB,IAAI,EAAE,IAAI,CAACuB,gBAAgB,CAAC;QACvE;QACA7C,IAAI,CAAC4E,KAAK,CAACK,SAAS,EAAqB,IAAI,CAAClC,MAAM,CAAC;QACrD;QACA;QACAkC,SAAS,CAACC,MAAM,EAAE;MACpB;IACF,CAAC;IACD,IAAMC,UAAU,GAAG,IAAI/E,aAAa,CAClCyE,eAAe,EACf,IAAI,CAACvD,IAAI,EACT,IAAI,CAACE,WAAW,EAChB,IAAI,CAACuB,MAAM,EACX,IAAI,CAACxB,IAAI,CAACoD,UAAU,EACpB,IAAI,CAAClD,UAAU,EACf,IAAI,CAACC,OAAO,EACZ,IAAI,CAACO,WAAW,EAChB,IAAI,CAACL,KAAK,EACV,IAAI,CAACL,IAAI,CAACiD,UAAU,CACrB;IACDW,UAAU,CAACP,KAAK,EAAE;EACpB;EAEAnC,WAAW,CAAC2C,MAA6B,EAAEC,MAAc,EAAQ;IAC/D,IAAM;MAAE7D,WAAW;MAAEE,OAAO;MAAEoB,iBAAiB;MAAElB,KAAK;MAAEC;IAAO,CAAC,GAAG,IAAI;IACvE,IAAI,IAAI,CAACW,WAAW,EAAE;MACpBlC,GAAG,CAACgF,MAAM,CAAC,uBAAuB,CAAC;MACnCD,MAAM,CAACE,KAAK,EAAE;MACd1D,MAAM,CAAC2D,OAAO,CAACC,CAAC,IAAIA,CAAC,CAACC,KAAK,EAAE,CAAC;MAC9B;IACF;IACA,IAAI;MAAEC;IAAK,CAAC,GAAGP,MAAM;IACrB,IAAI,CAAC,IAAI,CAACQ,OAAO,EAAE;MACjB,IAAIpE,WAAW,EAAE;QACf,IAAI,CAACoE,OAAO,GAAGzF,eAAe,CAAC0F,mBAAmB,CAACF,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3DA,IAAI,GAAGA,IAAI,CAACG,KAAK,CAAC,CAAC,CAAC;MACtB,CAAC,MAAM;QACL,IAAI,CAACF,OAAO,GAAGnF,SAAS,CAACsF,eAAe,CAACJ,IAAI,CAAC,CAAC,CAAC,CAAC7B,MAAM,CAAC;MAC1D;IACF;IAEA,IAAIC,OAAmB,GAAG,EAAE;IAC5B,IAAI;MACFA,OAAO,GAAG,IAAI,CAACL,SAAS,CAAC,IAAI,CAACkC,OAAO,CAAC9B,MAAM,EAAE6B,IAAI,CAAC;MACnD,IAAI/D,KAAK,EAAE;QACT;QACA,IAAI,CAACoE,iBAAiB,CAACjC,OAAO,EAAEsB,MAAM,CAAC;MACzC,CAAC,MAAM;QACL,IAAMY,KAAK,GAAG,IAAI,CAACjE,MAAM;QACzB,IAAI,CAACA,MAAM,IAAI,CAAC;QAChBc,iBAAiB,CAACmD,KAAK,EAAElC,OAAO,EAAEsB,MAAM,CAAC;MAC3C;IACF,CAAC,CAAC,OAAOa,CAAU,EAAE;MACnBxE,OAAO,CAACwE,CAAC,CAAC;IACZ;EACF;EAEAF,iBAAiB,CAACjC,OAAmB,EAAEsB,MAAc,EAAQ;IAC3D,IAAI,CAAC,IAAI,CAACc,kBAAkB,EAAE;MAC5B,IAAI,CAACA,kBAAkB,GAAGpC,OAAO,CAAC+B,KAAK,EAAE;IAC3C,CAAC,MAAM;MACL,KAAK,IAAIM,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGrC,OAAO,CAACD,MAAM,EAAEsC,CAAC,IAAI,CAAC,EAAE;QAC1C,IAAI,CAACD,kBAAkB,CAACC,CAAC,CAAC,GAAG,IAAI,CAACD,kBAAkB,CAACC,CAAC,CAAC,CAACtF,MAAM,CAC5DiD,OAAO,CAACqC,CAAC,CAAC,CACX;MACH;IACF;IACA,IAAI,CAAC7D,eAAe,IAAI,CAAC;IACzB,IAAI,IAAI,CAACA,eAAe,IAAI/B,sBAAsB,IAAI,IAAI,CAAC6B,UAAU,EAAE;MACrE,IAAI,CAACgE,wBAAwB,CAAChB,MAAM,CAAC;IACvC;EACF;EAEAgB,wBAAwB,CAAChB,MAAqB,EAAQ;IAAA;IACpD,IAAM;MAAEvC;IAAkB,CAAC,GAAG,IAAI;IAClC,IAAMmD,KAAK,GAAG,IAAI,CAACjE,MAAM;IACzB,IAAI,CAACA,MAAM,IAAI,CAAC;IAChB,IAAMsE,QAAQ,4BAAG,IAAI,CAACH,kBAAkB,0DAAvB,sBAAyBL,KAAK,EAAE;IACjD,IAAI,CAACK,kBAAkB,GAAGI,SAAS;IACnC,IAAI,CAAChE,eAAe,GAAG,CAAC;IACxBrC,aAAa,CAACoG,QAAQ,CAAC;IACvBxD,iBAAiB,CAACmD,KAAK,EAAEK,QAAQ,EAAEjB,MAAM,CAAC;EAC5C;EAEAvC,iBAAiB,CACfmD,KAAa,EACblC,OAAmB,EACnBsB,MAAqB,EACf;IACN,IAAM;MACJhE,OAAO;MACPQ,MAAM;MACNT,eAAe;MACfa,WAAW;MACX6C,KAAK;MACLrD,UAAU;MACVC;IACF,CAAC,GAAG,IAAI;IACR,IAAI2D,MAAM,EAAE;MACVA,MAAM,CAACmB,KAAK,EAAE;IAChB;IACAtG,aAAa,CAAC,IAAI,CAAC0F,OAAO,CAAC;IAC3B1F,aAAa,CAAC4E,KAAK,CAAC;IAEpB,IAAI,CAAC/C,aAAa,IAAIgC,OAAO,CAAC,CAAC,CAAC,CAACD,MAAM;IAEvCzC,OAAO,CACJoF,QAAQ,CAAC,IAAI,CAACb,OAAO,EAAEd,KAAK,EAAEf,OAAO,EAAE,IAAI,CAACpC,QAAQ,CAAC,CACrD+E,IAAI,CAACC,KAAK,IAAI;MACb,IAAI,IAAI,CAACnE,WAAW,EAAE;QACpBlC,GAAG,CAACgF,MAAM,CAAC,uBAAuB,CAAC;QACnC,IAAID,MAAM,EAAE;UACVA,MAAM,CAACE,KAAK,EAAE;QAChB;QACA1D,MAAM,CAAC2D,OAAO,CAACC,CAAC,IAAIA,CAAC,CAACC,KAAK,EAAE,CAAC;QAC9B;MACF;MACA,IAAIL,MAAM,EAAE;QACVA,MAAM,CAACH,MAAM,EAAE;MACjB;MACArD,MAAM,CAACoE,KAAK,CAAC,GAAGU,KAAK;MACrB;MACA,IAAIC,QAAQ,GAAG,CAAC;MAChB,IAAI3E,WAAW,GAAG,CAAC,EAAE;QACnB2E,QAAQ,GAAG7F,IAAI,CAAC8F,KAAK,CAAEhF,MAAM,CAACiC,MAAM,GAAG7B,WAAW,GAAI,EAAE,CAAC,GAAG,EAAE;MAChE,CAAC,MAAM;QACL;QACA;QACA2E,QAAQ,GAAG7F,IAAI,CAAC8F,KAAK,CAAE,IAAI,CAAC9E,aAAa,GAAG,IAAI,CAACD,QAAQ,GAAI,EAAE,CAAC,GAAG,EAAE;MACvE;MACAxB,GAAG,CAACgF,MAAM,+BAAwBsB,QAAQ,EAAG;MAC7CnF,UAAU,CAACmF,QAAQ,CAAC;MACpB,IAAI,IAAI,CAACvE,UAAU,IAAIR,MAAM,CAACiC,MAAM,KAAK,IAAI,CAAC9B,MAAM,EAAE;QACpD1B,GAAG,CAACgF,MAAM,CAAC,sBAAsB,CAAC;QAClClE,eAAe,CAACS,MAAM,CAAC;MACzB;IACF,CAAC,CAAC,CACDiF,KAAK,CAACZ,CAAC,IAAI;MACV,IAAI,CAAC,IAAI,CAAC7D,UAAU,IAAIgD,MAAM,EAAE;QAC9BA,MAAM,CAACE,KAAK,EAAE;MAChB;MACA7D,OAAO,CAACwE,CAAC,CAAC;IACZ,CAAC,CAAC;EACN;EAEAtD,cAAc,CAACmE,OAA6B,EAAQ;IAClD;IACA,IAAIA,OAAO,IAAI,IAAI,IAAIA,OAAO,CAACC,IAAI,CAACC,OAAO,IAAI,IAAI,EAAE;MACnD,IAAI,CAAC5E,UAAU,GAAG,IAAI;MACtB;MACA,IAAI,IAAI,CAAC8D,kBAAkB,EAAE;QAC3B,IAAI,CAACE,wBAAwB,CAAC,IAAI,CAAC;MACrC;IACF;EACF;EAEA1D,WAAW,CAACS,KAAc,EAAQ;IAChC,IAAM;MAAE1B;IAAQ,CAAC,GAAG,IAAI;IACxBA,OAAO,CAAC0B,KAAK,CAAC;EAChB;EAEAP,gBAAgB,CAACqE,QAA6B,EAAQ;IACpD,IAAI,CAAC5E,WAAW,GAAG4E,QAAQ,CAACC,OAAO;EACrC;AACF;AAAC,gBA7TK1G,SAAS,qBAEakD,UAAkB,IAAe;EACzD,IAAMiC,OAAO,GAAG,EAAE;EAClB,KAAK,IAAIQ,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGzC,UAAU,EAAEyC,CAAC,IAAI,CAAC,EAAE;IACtCR,OAAO,CAACwB,IAAI,CAAC3G,SAAS,CAACC,uBAAuB,CAAC0F,CAAC,CAAC,CAAC;EACpD;EACA,OAAOR,OAAO;AAChB,CAAC;AAuTH,eAAenF,SAAS"}
|
|
@@ -15,9 +15,9 @@ declare class CsvTypeParser {
|
|
|
15
15
|
static checkLocalTime(value: string): string;
|
|
16
16
|
static getTypeFromUnknown(value: string): string;
|
|
17
17
|
static getNumberType(value: string): string;
|
|
18
|
-
constructor(onFileCompleted: (types: string[]) => void, file: Blob | JSZipObject, readHeaders: boolean, parentConfig: ParseLocalConfig<unknown, Blob | NodeJS.ReadableStream>, nullString: string | null, onProgress: (progressValue: number) => boolean, onError: (e: unknown) => void, totalChunks: number, isZip: boolean, shouldTrim: boolean);
|
|
19
|
-
onFileCompleted: (types: string[]) => void;
|
|
20
|
-
file: Blob | JSZipObject;
|
|
18
|
+
constructor(onFileCompleted: (types: string[], rowCount: number) => void, file: File | Blob | JSZipObject, readHeaders: boolean, parentConfig: ParseLocalConfig<unknown, Blob | NodeJS.ReadableStream>, nullString: string | null, onProgress: (progressValue: number) => boolean, onError: (e: unknown) => void, totalChunks: number, isZip: boolean, shouldTrim: boolean);
|
|
19
|
+
onFileCompleted: (types: string[], rowCount: number) => void;
|
|
20
|
+
file: File | Blob | JSZipObject;
|
|
21
21
|
readHeaders: boolean;
|
|
22
22
|
nullString: string | null;
|
|
23
23
|
onProgress: (progressValue: number) => boolean;
|
|
@@ -25,6 +25,7 @@ declare class CsvTypeParser {
|
|
|
25
25
|
types?: string[];
|
|
26
26
|
chunks: number;
|
|
27
27
|
totalChunks: number;
|
|
28
|
+
rowCount: number;
|
|
28
29
|
isZip: boolean;
|
|
29
30
|
shouldTrim: boolean;
|
|
30
31
|
zipProgress: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CsvTypeParser.d.ts","sourceRoot":"","sources":["../../src/csv/CsvTypeParser.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,OAAO,CAAC;AAEzC,OAAa,EAAE,MAAM,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"CsvTypeParser.d.ts","sourceRoot":"","sources":["../../src/csv/CsvTypeParser.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,OAAO,CAAC;AAEzC,OAAa,EAAE,MAAM,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAiBxE;;GAEG;AACH,cAAM,aAAa;IACjB,MAAM,CAAC,aAAa,CAClB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,UAAU,EAAE,MAAM,GAAG,IAAI,GACxB,MAAM;IA4BT,MAAM,CAAC,mBAAmB,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO;IAM9C,MAAM,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAS1C,MAAM,CAAC,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAavC,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IASzC,MAAM,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAQ1C,MAAM,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAQ3C,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAQ5C,MAAM,CAAC,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAsBhD,MAAM,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;gBAqBzC,eAAe,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,KAAK,IAAI,EAC5D,IAAI,EAAE,IAAI,GAAG,IAAI,GAAG,WAAW,EAC/B,WAAW,EAAE,OAAO,EACpB,YAAY,EAAE,gBAAgB,CAAC,OAAO,EAAE,IAAI,GAAG,MAAM,CAAC,cAAc,CAAC,EACrE,UAAU,EAAE,MAAM,GAAG,IAAI,EACzB,UAAU,EAAE,CAAC,aAAa,EAAE,MAAM,KAAK,OAAO,EAC9C,OAAO,EAAE,CAAC,CAAC,EAAE,OAAO,KAAK,IAAI,EAC7B,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,OAAO,EACd,UAAU,EAAE,OAAO;IA4BrB,eAAe,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IAE7D,IAAI,EAAE,IAAI,GAAG,IAAI,GAAG,WAAW,CAAC;IAEhC,WAAW,EAAE,OAAO,CAAC;IAErB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAE1B,UAAU,EAAE,CAAC,aAAa,EAAE,MAAM,KAAK,OAAO,CAAC;IAE/C,OAAO,EAAE,CAAC,CAAC,EAAE,OAAO,KAAK,IAAI,CAAC;IAE9B,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IAEjB,MAAM,EAAE,MAAM,CAAC;IAEf,WAAW,EAAE,MAAM,CAAC;IAEpB,QAAQ,EAAE,MAAM,CAAC;IAEjB,KAAK,EAAE,OAAO,CAAC;IAEf,UAAU,EAAE,OAAO,CAAC;IAEpB,WAAW,EAAE,MAAM,CAAC;IAEpB,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,IAAI,GAAG,MAAM,CAAC,cAAc,CAAC,CAAC;IAEhE,KAAK,IAAI,IAAI;IAab,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IAqDhE,cAAc,CAAC,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,IAAI;IAmBnD,WAAW,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI;IAKjC,gBAAgB,CAAC,QAAQ,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;CAGtD;AAED,eAAe,aAAa,CAAC"}
|
|
@@ -7,7 +7,8 @@ import { assertNotNull } from '@deephaven/utils';
|
|
|
7
7
|
import Papa from 'papaparse';
|
|
8
8
|
// Intentionally using isNaN rather than Number.isNaN
|
|
9
9
|
/* eslint-disable no-restricted-globals */
|
|
10
|
-
import NewTableColumnTypes from "./NewTableColumnTypes.js";
|
|
10
|
+
import NewTableColumnTypes from "./NewTableColumnTypes.js";
|
|
11
|
+
import makeZipStreamHelper from "./ZipStreamHelper.js"; // Initially column types start as unknown
|
|
11
12
|
var UNKNOWN = 'unknown';
|
|
12
13
|
var MAX_INT = 2147483647;
|
|
13
14
|
var MIN_INT = -2147483648;
|
|
@@ -136,6 +137,7 @@ class CsvTypeParser {
|
|
|
136
137
|
_defineProperty(this, "types", void 0);
|
|
137
138
|
_defineProperty(this, "chunks", void 0);
|
|
138
139
|
_defineProperty(this, "totalChunks", void 0);
|
|
140
|
+
_defineProperty(this, "rowCount", void 0);
|
|
139
141
|
_defineProperty(this, "isZip", void 0);
|
|
140
142
|
_defineProperty(this, "shouldTrim", void 0);
|
|
141
143
|
_defineProperty(this, "zipProgress", void 0);
|
|
@@ -148,6 +150,7 @@ class CsvTypeParser {
|
|
|
148
150
|
this.onError = onError;
|
|
149
151
|
this.chunks = 0;
|
|
150
152
|
this.totalChunks = totalChunks;
|
|
153
|
+
this.rowCount = 0;
|
|
151
154
|
this.isZip = isZip;
|
|
152
155
|
this.shouldTrim = shouldTrim;
|
|
153
156
|
this.zipProgress = 0;
|
|
@@ -162,11 +165,16 @@ class CsvTypeParser {
|
|
|
162
165
|
});
|
|
163
166
|
}
|
|
164
167
|
parse() {
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
168
|
+
if (this.file instanceof File || this.file instanceof Blob) {
|
|
169
|
+
Papa.parse(this.file, this.config);
|
|
170
|
+
} else {
|
|
171
|
+
var zipStream = makeZipStreamHelper(this.file, this.handleNodeUpdate);
|
|
172
|
+
// This is actually a stream, but papaparse TS doesn't like it
|
|
173
|
+
Papa.parse(zipStream, this.config);
|
|
174
|
+
// The stream needs to be manually resumed since jszip starts paused
|
|
175
|
+
// Papaparse does not call resume and assumes the stream is already reading
|
|
176
|
+
zipStream.resume();
|
|
177
|
+
}
|
|
170
178
|
}
|
|
171
179
|
handleChunk(result, parser) {
|
|
172
180
|
var {
|
|
@@ -184,6 +192,7 @@ class CsvTypeParser {
|
|
|
184
192
|
data = data.slice(1);
|
|
185
193
|
}
|
|
186
194
|
}
|
|
195
|
+
this.rowCount += data.length;
|
|
187
196
|
assertNotNull(this.types);
|
|
188
197
|
var cloneTypes = [...this.types];
|
|
189
198
|
data.forEach(row => {
|
|
@@ -222,7 +231,7 @@ class CsvTypeParser {
|
|
|
222
231
|
// If the type is still unknown or a local time, just map it to a string.
|
|
223
232
|
// Local times are not supported by the backend in DHC, and probably should have more context to parse safely anyway (such as a date or a time zone).
|
|
224
233
|
// In these cases, we just map it to a string, and the user can use an `.update_view` later if they want to parse it into a different type.
|
|
225
|
-
type === UNKNOWN || type === NewTableColumnTypes.LOCAL_TIME ? NewTableColumnTypes.STRING : type));
|
|
234
|
+
type === UNKNOWN || type === NewTableColumnTypes.LOCAL_TIME ? NewTableColumnTypes.STRING : type), this.rowCount);
|
|
226
235
|
}
|
|
227
236
|
}
|
|
228
237
|
handleError(error) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CsvTypeParser.js","names":["assertNotNull","Papa","NewTableColumnTypes","UNKNOWN","MAX_INT","MIN_INT","DATE_TIME_REGEX","LOCAL_TIME_REGEX","CsvTypeParser","determineType","value","type","nullString","STRING","INTEGER","checkInteger","LONG","checkLong","DOUBLE","checkDouble","BOOLEAN","checkBoolean","DATE_TIME","checkDateTime","LOCAL_TIME","checkLocalTime","getTypeFromUnknown","isNotParsableNumber","s","isNaN","noCommas","replace","getNumberType","includes","lower","toLowerCase","test","maxIntLength","startsWith","length","intValue","parseInt","constructor","onFileCompleted","file","readHeaders","parentConfig","onProgress","onError","totalChunks","isZip","shouldTrim","chunks","zipProgress","handleChunk","bind","handleComplete","handleError","handleNodeUpdate","config","error","chunk","complete","parse","toParse","nodeStream","result","parser","data","types","abort","Array","fill","slice","cloneTypes","forEach","row","i","trim","progress","Math","round","isCancelled","results","meta","aborted","map","metadata","percent"],"sources":["../../src/csv/CsvTypeParser.ts"],"sourcesContent":["import type { JSZipObject } from 'jszip';\nimport { assertNotNull } from '@deephaven/utils';\nimport Papa, { Parser, ParseResult, ParseLocalConfig } from 'papaparse';\n// Intentionally using isNaN rather than Number.isNaN\n/* eslint-disable no-restricted-globals */\nimport NewTableColumnTypes from './NewTableColumnTypes';\n\n// Initially column types start as unknown\nconst UNKNOWN = 'unknown';\n\nconst MAX_INT = 2147483647;\nconst MIN_INT = -2147483648;\n\nconst DATE_TIME_REGEX =\n /^[0-9]{4}-[0-1][0-9]-[0-3][0-9][ T][0-2][0-9]:[0-5][0-9]:[0-6][0-9](?:\\.[0-9]{1,9})?(?: [a-zA-Z]+)?$/;\nconst LOCAL_TIME_REGEX =\n /^([0-9]+T)?([0-9]+):([0-9]+)(:[0-9]+)?(?:\\.[0-9]{1,9})?$/;\n\n/**\n * Determines the type of each column in a CSV file by parsing it and looking at every value.\n */\nclass CsvTypeParser {\n static determineType(\n value: string,\n type: string,\n nullString: string | null\n ): string {\n if (!value || value === nullString) {\n // A null tells us nothing about the type\n return type;\n }\n\n switch (type) {\n case NewTableColumnTypes.STRING:\n // Strings never get promoted\n return NewTableColumnTypes.STRING;\n case NewTableColumnTypes.INTEGER:\n return CsvTypeParser.checkInteger(value);\n case NewTableColumnTypes.LONG:\n return CsvTypeParser.checkLong(value);\n case NewTableColumnTypes.DOUBLE:\n return CsvTypeParser.checkDouble(value);\n case NewTableColumnTypes.BOOLEAN:\n return CsvTypeParser.checkBoolean(value);\n case NewTableColumnTypes.DATE_TIME:\n return CsvTypeParser.checkDateTime(value);\n case NewTableColumnTypes.LOCAL_TIME:\n return CsvTypeParser.checkLocalTime(value);\n default:\n return CsvTypeParser.getTypeFromUnknown(value);\n }\n }\n\n // Allows for cusomt rules in addition to isNaN\n static isNotParsableNumber(s: string): boolean {\n return (\n isNaN(s as unknown as number) || s === 'Infinity' || s === '-Infinity'\n );\n }\n\n static checkInteger(value: string): string {\n const noCommas = value.replace(/,/g, '');\n if (CsvTypeParser.isNotParsableNumber(noCommas)) {\n return NewTableColumnTypes.STRING;\n }\n\n return CsvTypeParser.getNumberType(noCommas);\n }\n\n static checkLong(value: string): string {\n const noCommas = value.replace(/,/g, '');\n if (CsvTypeParser.isNotParsableNumber(noCommas)) {\n return NewTableColumnTypes.STRING;\n }\n\n if (noCommas.includes('.')) {\n return NewTableColumnTypes.DOUBLE;\n }\n\n return NewTableColumnTypes.LONG;\n }\n\n static checkDouble(value: string): string {\n const noCommas = value.replace(/,/g, '');\n if (CsvTypeParser.isNotParsableNumber(noCommas)) {\n return NewTableColumnTypes.STRING;\n }\n\n return NewTableColumnTypes.DOUBLE;\n }\n\n static checkBoolean(value: string): string {\n const lower = value.toLowerCase();\n if (lower === 'true' || lower === 'false') {\n return NewTableColumnTypes.BOOLEAN;\n }\n return NewTableColumnTypes.STRING;\n }\n\n static checkDateTime(value: string): string {\n if (DATE_TIME_REGEX.test(value)) {\n return NewTableColumnTypes.DATE_TIME;\n }\n\n return NewTableColumnTypes.STRING;\n }\n\n static checkLocalTime(value: string): string {\n if (LOCAL_TIME_REGEX.test(value)) {\n return NewTableColumnTypes.LOCAL_TIME;\n }\n\n return NewTableColumnTypes.STRING;\n }\n\n static getTypeFromUnknown(value: string): string {\n const noCommas = value.replace(/,/g, '');\n if (CsvTypeParser.isNotParsableNumber(noCommas)) {\n const lower = value.toLowerCase();\n if (lower === 'true' || lower === 'false') {\n return NewTableColumnTypes.BOOLEAN;\n }\n\n if (DATE_TIME_REGEX.test(value) && value.includes(':')) {\n return NewTableColumnTypes.DATE_TIME;\n }\n\n if (LOCAL_TIME_REGEX.test(value)) {\n return NewTableColumnTypes.LOCAL_TIME;\n }\n\n return NewTableColumnTypes.STRING;\n }\n\n return CsvTypeParser.getNumberType(noCommas);\n }\n\n static getNumberType(value: string): string {\n if (value.includes('.')) {\n return NewTableColumnTypes.DOUBLE;\n }\n\n // Fast length check\n const maxIntLength = value.startsWith('-') ? 11 : 10;\n if (value.length > maxIntLength) {\n return NewTableColumnTypes.LONG;\n }\n\n // Slower parseInt check\n const intValue = parseInt(value, 10);\n if (intValue > MAX_INT || intValue < MIN_INT) {\n return NewTableColumnTypes.LONG;\n }\n\n return NewTableColumnTypes.INTEGER;\n }\n\n constructor(\n onFileCompleted: (types: string[]) => void,\n file: Blob | JSZipObject,\n readHeaders: boolean,\n parentConfig: ParseLocalConfig<unknown, Blob | NodeJS.ReadableStream>,\n nullString: string | null,\n onProgress: (progressValue: number) => boolean,\n onError: (e: unknown) => void,\n totalChunks: number,\n isZip: boolean,\n shouldTrim: boolean\n ) {\n this.onFileCompleted = onFileCompleted;\n this.file = file;\n this.readHeaders = readHeaders;\n this.nullString = nullString;\n this.onProgress = onProgress;\n this.onError = onError;\n this.chunks = 0;\n this.totalChunks = totalChunks;\n this.isZip = isZip;\n this.shouldTrim = shouldTrim;\n this.zipProgress = 0;\n\n this.handleChunk = this.handleChunk.bind(this);\n this.handleComplete = this.handleComplete.bind(this);\n this.handleError = this.handleError.bind(this);\n this.handleNodeUpdate = this.handleNodeUpdate.bind(this);\n\n this.config = {\n ...parentConfig,\n error: this.handleError,\n chunk: this.handleChunk,\n complete: this.handleComplete,\n };\n }\n\n onFileCompleted: (types: string[]) => void;\n\n file: Blob | JSZipObject;\n\n readHeaders: boolean;\n\n nullString: string | null;\n\n onProgress: (progressValue: number) => boolean;\n\n onError: (e: unknown) => void;\n\n types?: string[];\n\n chunks: number;\n\n totalChunks: number;\n\n isZip: boolean;\n\n shouldTrim: boolean;\n\n zipProgress: number;\n\n config: ParseLocalConfig<unknown, Blob | NodeJS.ReadableStream>;\n\n parse(): void {\n const toParse = this.isZip\n ? (this.file as JSZipObject).nodeStream(\n // JsZip types are incorrect, thus the funny casting\n // Actual parameter is 'nodebuffer'\n 'nodebuffer' as 'nodestream',\n this.handleNodeUpdate\n )\n : (this.file as Blob);\n Papa.parse(toParse, this.config);\n }\n\n handleChunk(result: ParseResult<string[]>, parser: Parser): void {\n let { data } = result;\n if (!this.types) {\n if (data == null || data.length === 0) {\n parser.abort();\n this.onError('Error parsing CSV: no data in file.');\n return;\n }\n this.types = new Array(data[0].length).fill(UNKNOWN);\n if (this.readHeaders) {\n // If headers are being read from the file, remove them from type analysis\n data = data.slice(1);\n }\n }\n\n assertNotNull(this.types);\n\n const cloneTypes = [...this.types];\n\n data.forEach(row => {\n if (row.length >= cloneTypes.length) {\n for (let i = 0; i < cloneTypes.length; i += 1) {\n cloneTypes[i] = CsvTypeParser.determineType(\n this.shouldTrim ? row[i].trim() : row[i],\n cloneTypes[i],\n this.nullString\n );\n }\n this.types = cloneTypes;\n } else {\n parser.abort();\n this.onError(\n `Error parsing CSV: Insufficient data in row.\\nExpected length ${cloneTypes.length} but found ${row.length}.\\nRow = ${row}`\n );\n }\n });\n\n this.chunks += 1;\n // 50 because the type parsing accounts for 50% of the parsing\n let progress = 0;\n if (this.totalChunks > 0) {\n progress = Math.round((this.chunks / this.totalChunks) * 50);\n } else {\n progress = Math.round(this.zipProgress / 2);\n }\n const isCancelled = this.onProgress(progress);\n if (isCancelled) {\n parser.abort();\n }\n }\n\n handleComplete(results: ParseResult<unknown>): void {\n const { types, onFileCompleted } = this;\n // results is undefined for a succesful parse, but has meta data for an abort\n assertNotNull(types);\n if (results == null || !results.meta.aborted) {\n onFileCompleted(\n types.map(type =>\n // If the type is still unknown or a local time, just map it to a string.\n // Local times are not supported by the backend in DHC, and probably should have more context to parse safely anyway (such as a date or a time zone).\n // In these cases, we just map it to a string, and the user can use an `.update_view` later if they want to parse it into a different type.\n type === UNKNOWN || type === NewTableColumnTypes.LOCAL_TIME\n ? NewTableColumnTypes.STRING\n : type\n )\n );\n }\n }\n\n handleError(error: unknown): void {\n const { onError } = this;\n onError(error);\n }\n\n handleNodeUpdate(metadata: { percent: number }): void {\n this.zipProgress = metadata.percent;\n }\n}\n\nexport default CsvTypeParser;\n"],"mappings":";;;;;AACA,SAASA,aAAa,QAAQ,kBAAkB;AAChD,OAAOC,IAAI,MAAiD,WAAW;AACvE;AACA;AAAA,OACOC,mBAAmB,kCAE1B;AACA,IAAMC,OAAO,GAAG,SAAS;AAEzB,IAAMC,OAAO,GAAG,UAAU;AAC1B,IAAMC,OAAO,GAAG,CAAC,UAAU;AAE3B,IAAMC,eAAe,GACnB,sGAAsG;AACxG,IAAMC,gBAAgB,GACpB,0DAA0D;;AAE5D;AACA;AACA;AACA,MAAMC,aAAa,CAAC;EAClB,OAAOC,aAAa,CAClBC,KAAa,EACbC,IAAY,EACZC,UAAyB,EACjB;IACR,IAAI,CAACF,KAAK,IAAIA,KAAK,KAAKE,UAAU,EAAE;MAClC;MACA,OAAOD,IAAI;IACb;IAEA,QAAQA,IAAI;MACV,KAAKT,mBAAmB,CAACW,MAAM;QAC7B;QACA,OAAOX,mBAAmB,CAACW,MAAM;MACnC,KAAKX,mBAAmB,CAACY,OAAO;QAC9B,OAAON,aAAa,CAACO,YAAY,CAACL,KAAK,CAAC;MAC1C,KAAKR,mBAAmB,CAACc,IAAI;QAC3B,OAAOR,aAAa,CAACS,SAAS,CAACP,KAAK,CAAC;MACvC,KAAKR,mBAAmB,CAACgB,MAAM;QAC7B,OAAOV,aAAa,CAACW,WAAW,CAACT,KAAK,CAAC;MACzC,KAAKR,mBAAmB,CAACkB,OAAO;QAC9B,OAAOZ,aAAa,CAACa,YAAY,CAACX,KAAK,CAAC;MAC1C,KAAKR,mBAAmB,CAACoB,SAAS;QAChC,OAAOd,aAAa,CAACe,aAAa,CAACb,KAAK,CAAC;MAC3C,KAAKR,mBAAmB,CAACsB,UAAU;QACjC,OAAOhB,aAAa,CAACiB,cAAc,CAACf,KAAK,CAAC;MAC5C;QACE,OAAOF,aAAa,CAACkB,kBAAkB,CAAChB,KAAK,CAAC;IAAC;EAErD;;EAEA;EACA,OAAOiB,mBAAmB,CAACC,CAAS,EAAW;IAC7C,OACEC,KAAK,CAACD,CAAC,CAAsB,IAAIA,CAAC,KAAK,UAAU,IAAIA,CAAC,KAAK,WAAW;EAE1E;EAEA,OAAOb,YAAY,CAACL,KAAa,EAAU;IACzC,IAAMoB,QAAQ,GAAGpB,KAAK,CAACqB,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;IACxC,IAAIvB,aAAa,CAACmB,mBAAmB,CAACG,QAAQ,CAAC,EAAE;MAC/C,OAAO5B,mBAAmB,CAACW,MAAM;IACnC;IAEA,OAAOL,aAAa,CAACwB,aAAa,CAACF,QAAQ,CAAC;EAC9C;EAEA,OAAOb,SAAS,CAACP,KAAa,EAAU;IACtC,IAAMoB,QAAQ,GAAGpB,KAAK,CAACqB,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;IACxC,IAAIvB,aAAa,CAACmB,mBAAmB,CAACG,QAAQ,CAAC,EAAE;MAC/C,OAAO5B,mBAAmB,CAACW,MAAM;IACnC;IAEA,IAAIiB,QAAQ,CAACG,QAAQ,CAAC,GAAG,CAAC,EAAE;MAC1B,OAAO/B,mBAAmB,CAACgB,MAAM;IACnC;IAEA,OAAOhB,mBAAmB,CAACc,IAAI;EACjC;EAEA,OAAOG,WAAW,CAACT,KAAa,EAAU;IACxC,IAAMoB,QAAQ,GAAGpB,KAAK,CAACqB,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;IACxC,IAAIvB,aAAa,CAACmB,mBAAmB,CAACG,QAAQ,CAAC,EAAE;MAC/C,OAAO5B,mBAAmB,CAACW,MAAM;IACnC;IAEA,OAAOX,mBAAmB,CAACgB,MAAM;EACnC;EAEA,OAAOG,YAAY,CAACX,KAAa,EAAU;IACzC,IAAMwB,KAAK,GAAGxB,KAAK,CAACyB,WAAW,EAAE;IACjC,IAAID,KAAK,KAAK,MAAM,IAAIA,KAAK,KAAK,OAAO,EAAE;MACzC,OAAOhC,mBAAmB,CAACkB,OAAO;IACpC;IACA,OAAOlB,mBAAmB,CAACW,MAAM;EACnC;EAEA,OAAOU,aAAa,CAACb,KAAa,EAAU;IAC1C,IAAIJ,eAAe,CAAC8B,IAAI,CAAC1B,KAAK,CAAC,EAAE;MAC/B,OAAOR,mBAAmB,CAACoB,SAAS;IACtC;IAEA,OAAOpB,mBAAmB,CAACW,MAAM;EACnC;EAEA,OAAOY,cAAc,CAACf,KAAa,EAAU;IAC3C,IAAIH,gBAAgB,CAAC6B,IAAI,CAAC1B,KAAK,CAAC,EAAE;MAChC,OAAOR,mBAAmB,CAACsB,UAAU;IACvC;IAEA,OAAOtB,mBAAmB,CAACW,MAAM;EACnC;EAEA,OAAOa,kBAAkB,CAAChB,KAAa,EAAU;IAC/C,IAAMoB,QAAQ,GAAGpB,KAAK,CAACqB,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;IACxC,IAAIvB,aAAa,CAACmB,mBAAmB,CAACG,QAAQ,CAAC,EAAE;MAC/C,IAAMI,KAAK,GAAGxB,KAAK,CAACyB,WAAW,EAAE;MACjC,IAAID,KAAK,KAAK,MAAM,IAAIA,KAAK,KAAK,OAAO,EAAE;QACzC,OAAOhC,mBAAmB,CAACkB,OAAO;MACpC;MAEA,IAAId,eAAe,CAAC8B,IAAI,CAAC1B,KAAK,CAAC,IAAIA,KAAK,CAACuB,QAAQ,CAAC,GAAG,CAAC,EAAE;QACtD,OAAO/B,mBAAmB,CAACoB,SAAS;MACtC;MAEA,IAAIf,gBAAgB,CAAC6B,IAAI,CAAC1B,KAAK,CAAC,EAAE;QAChC,OAAOR,mBAAmB,CAACsB,UAAU;MACvC;MAEA,OAAOtB,mBAAmB,CAACW,MAAM;IACnC;IAEA,OAAOL,aAAa,CAACwB,aAAa,CAACF,QAAQ,CAAC;EAC9C;EAEA,OAAOE,aAAa,CAACtB,KAAa,EAAU;IAC1C,IAAIA,KAAK,CAACuB,QAAQ,CAAC,GAAG,CAAC,EAAE;MACvB,OAAO/B,mBAAmB,CAACgB,MAAM;IACnC;;IAEA;IACA,IAAMmB,YAAY,GAAG3B,KAAK,CAAC4B,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE;IACpD,IAAI5B,KAAK,CAAC6B,MAAM,GAAGF,YAAY,EAAE;MAC/B,OAAOnC,mBAAmB,CAACc,IAAI;IACjC;;IAEA;IACA,IAAMwB,QAAQ,GAAGC,QAAQ,CAAC/B,KAAK,EAAE,EAAE,CAAC;IACpC,IAAI8B,QAAQ,GAAGpC,OAAO,IAAIoC,QAAQ,GAAGnC,OAAO,EAAE;MAC5C,OAAOH,mBAAmB,CAACc,IAAI;IACjC;IAEA,OAAOd,mBAAmB,CAACY,OAAO;EACpC;EAEA4B,WAAW,CACTC,eAA0C,EAC1CC,IAAwB,EACxBC,WAAoB,EACpBC,YAAqE,EACrElC,UAAyB,EACzBmC,UAA8C,EAC9CC,OAA6B,EAC7BC,WAAmB,EACnBC,KAAc,EACdC,UAAmB,EACnB;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IACA,IAAI,CAACR,eAAe,GAAGA,eAAe;IACtC,IAAI,CAACC,IAAI,GAAGA,IAAI;IAChB,IAAI,CAACC,WAAW,GAAGA,WAAW;IAC9B,IAAI,CAACjC,UAAU,GAAGA,UAAU;IAC5B,IAAI,CAACmC,UAAU,GAAGA,UAAU;IAC5B,IAAI,CAACC,OAAO,GAAGA,OAAO;IACtB,IAAI,CAACI,MAAM,GAAG,CAAC;IACf,IAAI,CAACH,WAAW,GAAGA,WAAW;IAC9B,IAAI,CAACC,KAAK,GAAGA,KAAK;IAClB,IAAI,CAACC,UAAU,GAAGA,UAAU;IAC5B,IAAI,CAACE,WAAW,GAAG,CAAC;IAEpB,IAAI,CAACC,WAAW,GAAG,IAAI,CAACA,WAAW,CAACC,IAAI,CAAC,IAAI,CAAC;IAC9C,IAAI,CAACC,cAAc,GAAG,IAAI,CAACA,cAAc,CAACD,IAAI,CAAC,IAAI,CAAC;IACpD,IAAI,CAACE,WAAW,GAAG,IAAI,CAACA,WAAW,CAACF,IAAI,CAAC,IAAI,CAAC;IAC9C,IAAI,CAACG,gBAAgB,GAAG,IAAI,CAACA,gBAAgB,CAACH,IAAI,CAAC,IAAI,CAAC;IAExD,IAAI,CAACI,MAAM,mCACNb,YAAY;MACfc,KAAK,EAAE,IAAI,CAACH,WAAW;MACvBI,KAAK,EAAE,IAAI,CAACP,WAAW;MACvBQ,QAAQ,EAAE,IAAI,CAACN;IAAc,EAC9B;EACH;EA4BAO,KAAK,GAAS;IACZ,IAAMC,OAAO,GAAG,IAAI,CAACd,KAAK,GACrB,IAAI,CAACN,IAAI,CAAiBqB,UAAU;IACnC;IACA;IACA,YAAY,EACZ,IAAI,CAACP,gBAAgB,CACtB,GACA,IAAI,CAACd,IAAa;IACvB3C,IAAI,CAAC8D,KAAK,CAACC,OAAO,EAAE,IAAI,CAACL,MAAM,CAAC;EAClC;EAEAL,WAAW,CAACY,MAA6B,EAAEC,MAAc,EAAQ;IAC/D,IAAI;MAAEC;IAAK,CAAC,GAAGF,MAAM;IACrB,IAAI,CAAC,IAAI,CAACG,KAAK,EAAE;MACf,IAAID,IAAI,IAAI,IAAI,IAAIA,IAAI,CAAC7B,MAAM,KAAK,CAAC,EAAE;QACrC4B,MAAM,CAACG,KAAK,EAAE;QACd,IAAI,CAACtB,OAAO,CAAC,qCAAqC,CAAC;QACnD;MACF;MACA,IAAI,CAACqB,KAAK,GAAG,IAAIE,KAAK,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC7B,MAAM,CAAC,CAACiC,IAAI,CAACrE,OAAO,CAAC;MACpD,IAAI,IAAI,CAAC0C,WAAW,EAAE;QACpB;QACAuB,IAAI,GAAGA,IAAI,CAACK,KAAK,CAAC,CAAC,CAAC;MACtB;IACF;IAEAzE,aAAa,CAAC,IAAI,CAACqE,KAAK,CAAC;IAEzB,IAAMK,UAAU,GAAG,CAAC,GAAG,IAAI,CAACL,KAAK,CAAC;IAElCD,IAAI,CAACO,OAAO,CAACC,GAAG,IAAI;MAClB,IAAIA,GAAG,CAACrC,MAAM,IAAImC,UAAU,CAACnC,MAAM,EAAE;QACnC,KAAK,IAAIsC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGH,UAAU,CAACnC,MAAM,EAAEsC,CAAC,IAAI,CAAC,EAAE;UAC7CH,UAAU,CAACG,CAAC,CAAC,GAAGrE,aAAa,CAACC,aAAa,CACzC,IAAI,CAAC0C,UAAU,GAAGyB,GAAG,CAACC,CAAC,CAAC,CAACC,IAAI,EAAE,GAAGF,GAAG,CAACC,CAAC,CAAC,EACxCH,UAAU,CAACG,CAAC,CAAC,EACb,IAAI,CAACjE,UAAU,CAChB;QACH;QACA,IAAI,CAACyD,KAAK,GAAGK,UAAU;MACzB,CAAC,MAAM;QACLP,MAAM,CAACG,KAAK,EAAE;QACd,IAAI,CAACtB,OAAO,yEACuD0B,UAAU,CAACnC,MAAM,wBAAcqC,GAAG,CAACrC,MAAM,sBAAYqC,GAAG,EAC1H;MACH;IACF,CAAC,CAAC;IAEF,IAAI,CAACxB,MAAM,IAAI,CAAC;IAChB;IACA,IAAI2B,QAAQ,GAAG,CAAC;IAChB,IAAI,IAAI,CAAC9B,WAAW,GAAG,CAAC,EAAE;MACxB8B,QAAQ,GAAGC,IAAI,CAACC,KAAK,CAAE,IAAI,CAAC7B,MAAM,GAAG,IAAI,CAACH,WAAW,GAAI,EAAE,CAAC;IAC9D,CAAC,MAAM;MACL8B,QAAQ,GAAGC,IAAI,CAACC,KAAK,CAAC,IAAI,CAAC5B,WAAW,GAAG,CAAC,CAAC;IAC7C;IACA,IAAM6B,WAAW,GAAG,IAAI,CAACnC,UAAU,CAACgC,QAAQ,CAAC;IAC7C,IAAIG,WAAW,EAAE;MACff,MAAM,CAACG,KAAK,EAAE;IAChB;EACF;EAEAd,cAAc,CAAC2B,OAA6B,EAAQ;IAClD,IAAM;MAAEd,KAAK;MAAE1B;IAAgB,CAAC,GAAG,IAAI;IACvC;IACA3C,aAAa,CAACqE,KAAK,CAAC;IACpB,IAAIc,OAAO,IAAI,IAAI,IAAI,CAACA,OAAO,CAACC,IAAI,CAACC,OAAO,EAAE;MAC5C1C,eAAe,CACb0B,KAAK,CAACiB,GAAG,CAAC3E,IAAI;MACZ;MACA;MACA;MACAA,IAAI,KAAKR,OAAO,IAAIQ,IAAI,KAAKT,mBAAmB,CAACsB,UAAU,GACvDtB,mBAAmB,CAACW,MAAM,GAC1BF,IAAI,CACT,CACF;IACH;EACF;EAEA8C,WAAW,CAACG,KAAc,EAAQ;IAChC,IAAM;MAAEZ;IAAQ,CAAC,GAAG,IAAI;IACxBA,OAAO,CAACY,KAAK,CAAC;EAChB;EAEAF,gBAAgB,CAAC6B,QAA6B,EAAQ;IACpD,IAAI,CAAClC,WAAW,GAAGkC,QAAQ,CAACC,OAAO;EACrC;AACF;AAEA,eAAehF,aAAa"}
|
|
1
|
+
{"version":3,"file":"CsvTypeParser.js","names":["assertNotNull","Papa","NewTableColumnTypes","makeZipStreamHelper","UNKNOWN","MAX_INT","MIN_INT","DATE_TIME_REGEX","LOCAL_TIME_REGEX","CsvTypeParser","determineType","value","type","nullString","STRING","INTEGER","checkInteger","LONG","checkLong","DOUBLE","checkDouble","BOOLEAN","checkBoolean","DATE_TIME","checkDateTime","LOCAL_TIME","checkLocalTime","getTypeFromUnknown","isNotParsableNumber","s","isNaN","noCommas","replace","getNumberType","includes","lower","toLowerCase","test","maxIntLength","startsWith","length","intValue","parseInt","constructor","onFileCompleted","file","readHeaders","parentConfig","onProgress","onError","totalChunks","isZip","shouldTrim","chunks","rowCount","zipProgress","handleChunk","bind","handleComplete","handleError","handleNodeUpdate","config","error","chunk","complete","parse","File","Blob","zipStream","resume","result","parser","data","types","abort","Array","fill","slice","cloneTypes","forEach","row","i","trim","progress","Math","round","isCancelled","results","meta","aborted","map","metadata","percent"],"sources":["../../src/csv/CsvTypeParser.ts"],"sourcesContent":["import type { JSZipObject } from 'jszip';\nimport { assertNotNull } from '@deephaven/utils';\nimport Papa, { Parser, ParseResult, ParseLocalConfig } from 'papaparse';\n// Intentionally using isNaN rather than Number.isNaN\n/* eslint-disable no-restricted-globals */\nimport NewTableColumnTypes from './NewTableColumnTypes';\nimport makeZipStreamHelper from './ZipStreamHelper';\n\n// Initially column types start as unknown\nconst UNKNOWN = 'unknown';\n\nconst MAX_INT = 2147483647;\nconst MIN_INT = -2147483648;\n\nconst DATE_TIME_REGEX =\n /^[0-9]{4}-[0-1][0-9]-[0-3][0-9][ T][0-2][0-9]:[0-5][0-9]:[0-6][0-9](?:\\.[0-9]{1,9})?(?: [a-zA-Z]+)?$/;\nconst LOCAL_TIME_REGEX =\n /^([0-9]+T)?([0-9]+):([0-9]+)(:[0-9]+)?(?:\\.[0-9]{1,9})?$/;\n\n/**\n * Determines the type of each column in a CSV file by parsing it and looking at every value.\n */\nclass CsvTypeParser {\n static determineType(\n value: string,\n type: string,\n nullString: string | null\n ): string {\n if (!value || value === nullString) {\n // A null tells us nothing about the type\n return type;\n }\n\n switch (type) {\n case NewTableColumnTypes.STRING:\n // Strings never get promoted\n return NewTableColumnTypes.STRING;\n case NewTableColumnTypes.INTEGER:\n return CsvTypeParser.checkInteger(value);\n case NewTableColumnTypes.LONG:\n return CsvTypeParser.checkLong(value);\n case NewTableColumnTypes.DOUBLE:\n return CsvTypeParser.checkDouble(value);\n case NewTableColumnTypes.BOOLEAN:\n return CsvTypeParser.checkBoolean(value);\n case NewTableColumnTypes.DATE_TIME:\n return CsvTypeParser.checkDateTime(value);\n case NewTableColumnTypes.LOCAL_TIME:\n return CsvTypeParser.checkLocalTime(value);\n default:\n return CsvTypeParser.getTypeFromUnknown(value);\n }\n }\n\n // Allows for cusomt rules in addition to isNaN\n static isNotParsableNumber(s: string): boolean {\n return (\n isNaN(s as unknown as number) || s === 'Infinity' || s === '-Infinity'\n );\n }\n\n static checkInteger(value: string): string {\n const noCommas = value.replace(/,/g, '');\n if (CsvTypeParser.isNotParsableNumber(noCommas)) {\n return NewTableColumnTypes.STRING;\n }\n\n return CsvTypeParser.getNumberType(noCommas);\n }\n\n static checkLong(value: string): string {\n const noCommas = value.replace(/,/g, '');\n if (CsvTypeParser.isNotParsableNumber(noCommas)) {\n return NewTableColumnTypes.STRING;\n }\n\n if (noCommas.includes('.')) {\n return NewTableColumnTypes.DOUBLE;\n }\n\n return NewTableColumnTypes.LONG;\n }\n\n static checkDouble(value: string): string {\n const noCommas = value.replace(/,/g, '');\n if (CsvTypeParser.isNotParsableNumber(noCommas)) {\n return NewTableColumnTypes.STRING;\n }\n\n return NewTableColumnTypes.DOUBLE;\n }\n\n static checkBoolean(value: string): string {\n const lower = value.toLowerCase();\n if (lower === 'true' || lower === 'false') {\n return NewTableColumnTypes.BOOLEAN;\n }\n return NewTableColumnTypes.STRING;\n }\n\n static checkDateTime(value: string): string {\n if (DATE_TIME_REGEX.test(value)) {\n return NewTableColumnTypes.DATE_TIME;\n }\n\n return NewTableColumnTypes.STRING;\n }\n\n static checkLocalTime(value: string): string {\n if (LOCAL_TIME_REGEX.test(value)) {\n return NewTableColumnTypes.LOCAL_TIME;\n }\n\n return NewTableColumnTypes.STRING;\n }\n\n static getTypeFromUnknown(value: string): string {\n const noCommas = value.replace(/,/g, '');\n if (CsvTypeParser.isNotParsableNumber(noCommas)) {\n const lower = value.toLowerCase();\n if (lower === 'true' || lower === 'false') {\n return NewTableColumnTypes.BOOLEAN;\n }\n\n if (DATE_TIME_REGEX.test(value) && value.includes(':')) {\n return NewTableColumnTypes.DATE_TIME;\n }\n\n if (LOCAL_TIME_REGEX.test(value)) {\n return NewTableColumnTypes.LOCAL_TIME;\n }\n\n return NewTableColumnTypes.STRING;\n }\n\n return CsvTypeParser.getNumberType(noCommas);\n }\n\n static getNumberType(value: string): string {\n if (value.includes('.')) {\n return NewTableColumnTypes.DOUBLE;\n }\n\n // Fast length check\n const maxIntLength = value.startsWith('-') ? 11 : 10;\n if (value.length > maxIntLength) {\n return NewTableColumnTypes.LONG;\n }\n\n // Slower parseInt check\n const intValue = parseInt(value, 10);\n if (intValue > MAX_INT || intValue < MIN_INT) {\n return NewTableColumnTypes.LONG;\n }\n\n return NewTableColumnTypes.INTEGER;\n }\n\n constructor(\n onFileCompleted: (types: string[], rowCount: number) => void,\n file: File | Blob | JSZipObject,\n readHeaders: boolean,\n parentConfig: ParseLocalConfig<unknown, Blob | NodeJS.ReadableStream>,\n nullString: string | null,\n onProgress: (progressValue: number) => boolean,\n onError: (e: unknown) => void,\n totalChunks: number,\n isZip: boolean,\n shouldTrim: boolean\n ) {\n this.onFileCompleted = onFileCompleted;\n this.file = file;\n this.readHeaders = readHeaders;\n this.nullString = nullString;\n this.onProgress = onProgress;\n this.onError = onError;\n this.chunks = 0;\n this.totalChunks = totalChunks;\n this.rowCount = 0;\n this.isZip = isZip;\n this.shouldTrim = shouldTrim;\n this.zipProgress = 0;\n\n this.handleChunk = this.handleChunk.bind(this);\n this.handleComplete = this.handleComplete.bind(this);\n this.handleError = this.handleError.bind(this);\n this.handleNodeUpdate = this.handleNodeUpdate.bind(this);\n\n this.config = {\n ...parentConfig,\n error: this.handleError,\n chunk: this.handleChunk,\n complete: this.handleComplete,\n };\n }\n\n onFileCompleted: (types: string[], rowCount: number) => void;\n\n file: File | Blob | JSZipObject;\n\n readHeaders: boolean;\n\n nullString: string | null;\n\n onProgress: (progressValue: number) => boolean;\n\n onError: (e: unknown) => void;\n\n types?: string[];\n\n chunks: number;\n\n totalChunks: number;\n\n rowCount: number;\n\n isZip: boolean;\n\n shouldTrim: boolean;\n\n zipProgress: number;\n\n config: ParseLocalConfig<unknown, Blob | NodeJS.ReadableStream>;\n\n parse(): void {\n if (this.file instanceof File || this.file instanceof Blob) {\n Papa.parse(this.file, this.config);\n } else {\n const zipStream = makeZipStreamHelper(this.file, this.handleNodeUpdate);\n // This is actually a stream, but papaparse TS doesn't like it\n Papa.parse(zipStream as unknown as Blob, this.config);\n // The stream needs to be manually resumed since jszip starts paused\n // Papaparse does not call resume and assumes the stream is already reading\n zipStream.resume();\n }\n }\n\n handleChunk(result: ParseResult<string[]>, parser: Parser): void {\n let { data } = result;\n if (!this.types) {\n if (data == null || data.length === 0) {\n parser.abort();\n this.onError('Error parsing CSV: no data in file.');\n return;\n }\n this.types = new Array(data[0].length).fill(UNKNOWN);\n if (this.readHeaders) {\n // If headers are being read from the file, remove them from type analysis\n data = data.slice(1);\n }\n }\n\n this.rowCount += data.length;\n\n assertNotNull(this.types);\n\n const cloneTypes = [...this.types];\n\n data.forEach(row => {\n if (row.length >= cloneTypes.length) {\n for (let i = 0; i < cloneTypes.length; i += 1) {\n cloneTypes[i] = CsvTypeParser.determineType(\n this.shouldTrim ? row[i].trim() : row[i],\n cloneTypes[i],\n this.nullString\n );\n }\n this.types = cloneTypes;\n } else {\n parser.abort();\n this.onError(\n `Error parsing CSV: Insufficient data in row.\\nExpected length ${cloneTypes.length} but found ${row.length}.\\nRow = ${row}`\n );\n }\n });\n\n this.chunks += 1;\n // 50 because the type parsing accounts for 50% of the parsing\n let progress = 0;\n if (this.totalChunks > 0) {\n progress = Math.round((this.chunks / this.totalChunks) * 50);\n } else {\n progress = Math.round(this.zipProgress / 2);\n }\n const isCancelled = this.onProgress(progress);\n if (isCancelled) {\n parser.abort();\n }\n }\n\n handleComplete(results: ParseResult<unknown>): void {\n const { types, onFileCompleted } = this;\n // results is undefined for a succesful parse, but has meta data for an abort\n assertNotNull(types);\n if (results == null || !results.meta.aborted) {\n onFileCompleted(\n types.map(type =>\n // If the type is still unknown or a local time, just map it to a string.\n // Local times are not supported by the backend in DHC, and probably should have more context to parse safely anyway (such as a date or a time zone).\n // In these cases, we just map it to a string, and the user can use an `.update_view` later if they want to parse it into a different type.\n type === UNKNOWN || type === NewTableColumnTypes.LOCAL_TIME\n ? NewTableColumnTypes.STRING\n : type\n ),\n this.rowCount\n );\n }\n }\n\n handleError(error: unknown): void {\n const { onError } = this;\n onError(error);\n }\n\n handleNodeUpdate(metadata: { percent: number }): void {\n this.zipProgress = metadata.percent;\n }\n}\n\nexport default CsvTypeParser;\n"],"mappings":";;;;;AACA,SAASA,aAAa,QAAQ,kBAAkB;AAChD,OAAOC,IAAI,MAAiD,WAAW;AACvE;AACA;AAAA,OACOC,mBAAmB;AAAA,OACnBC,mBAAmB,8BAE1B;AACA,IAAMC,OAAO,GAAG,SAAS;AAEzB,IAAMC,OAAO,GAAG,UAAU;AAC1B,IAAMC,OAAO,GAAG,CAAC,UAAU;AAE3B,IAAMC,eAAe,GACnB,sGAAsG;AACxG,IAAMC,gBAAgB,GACpB,0DAA0D;;AAE5D;AACA;AACA;AACA,MAAMC,aAAa,CAAC;EAClB,OAAOC,aAAa,CAClBC,KAAa,EACbC,IAAY,EACZC,UAAyB,EACjB;IACR,IAAI,CAACF,KAAK,IAAIA,KAAK,KAAKE,UAAU,EAAE;MAClC;MACA,OAAOD,IAAI;IACb;IAEA,QAAQA,IAAI;MACV,KAAKV,mBAAmB,CAACY,MAAM;QAC7B;QACA,OAAOZ,mBAAmB,CAACY,MAAM;MACnC,KAAKZ,mBAAmB,CAACa,OAAO;QAC9B,OAAON,aAAa,CAACO,YAAY,CAACL,KAAK,CAAC;MAC1C,KAAKT,mBAAmB,CAACe,IAAI;QAC3B,OAAOR,aAAa,CAACS,SAAS,CAACP,KAAK,CAAC;MACvC,KAAKT,mBAAmB,CAACiB,MAAM;QAC7B,OAAOV,aAAa,CAACW,WAAW,CAACT,KAAK,CAAC;MACzC,KAAKT,mBAAmB,CAACmB,OAAO;QAC9B,OAAOZ,aAAa,CAACa,YAAY,CAACX,KAAK,CAAC;MAC1C,KAAKT,mBAAmB,CAACqB,SAAS;QAChC,OAAOd,aAAa,CAACe,aAAa,CAACb,KAAK,CAAC;MAC3C,KAAKT,mBAAmB,CAACuB,UAAU;QACjC,OAAOhB,aAAa,CAACiB,cAAc,CAACf,KAAK,CAAC;MAC5C;QACE,OAAOF,aAAa,CAACkB,kBAAkB,CAAChB,KAAK,CAAC;IAAC;EAErD;;EAEA;EACA,OAAOiB,mBAAmB,CAACC,CAAS,EAAW;IAC7C,OACEC,KAAK,CAACD,CAAC,CAAsB,IAAIA,CAAC,KAAK,UAAU,IAAIA,CAAC,KAAK,WAAW;EAE1E;EAEA,OAAOb,YAAY,CAACL,KAAa,EAAU;IACzC,IAAMoB,QAAQ,GAAGpB,KAAK,CAACqB,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;IACxC,IAAIvB,aAAa,CAACmB,mBAAmB,CAACG,QAAQ,CAAC,EAAE;MAC/C,OAAO7B,mBAAmB,CAACY,MAAM;IACnC;IAEA,OAAOL,aAAa,CAACwB,aAAa,CAACF,QAAQ,CAAC;EAC9C;EAEA,OAAOb,SAAS,CAACP,KAAa,EAAU;IACtC,IAAMoB,QAAQ,GAAGpB,KAAK,CAACqB,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;IACxC,IAAIvB,aAAa,CAACmB,mBAAmB,CAACG,QAAQ,CAAC,EAAE;MAC/C,OAAO7B,mBAAmB,CAACY,MAAM;IACnC;IAEA,IAAIiB,QAAQ,CAACG,QAAQ,CAAC,GAAG,CAAC,EAAE;MAC1B,OAAOhC,mBAAmB,CAACiB,MAAM;IACnC;IAEA,OAAOjB,mBAAmB,CAACe,IAAI;EACjC;EAEA,OAAOG,WAAW,CAACT,KAAa,EAAU;IACxC,IAAMoB,QAAQ,GAAGpB,KAAK,CAACqB,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;IACxC,IAAIvB,aAAa,CAACmB,mBAAmB,CAACG,QAAQ,CAAC,EAAE;MAC/C,OAAO7B,mBAAmB,CAACY,MAAM;IACnC;IAEA,OAAOZ,mBAAmB,CAACiB,MAAM;EACnC;EAEA,OAAOG,YAAY,CAACX,KAAa,EAAU;IACzC,IAAMwB,KAAK,GAAGxB,KAAK,CAACyB,WAAW,EAAE;IACjC,IAAID,KAAK,KAAK,MAAM,IAAIA,KAAK,KAAK,OAAO,EAAE;MACzC,OAAOjC,mBAAmB,CAACmB,OAAO;IACpC;IACA,OAAOnB,mBAAmB,CAACY,MAAM;EACnC;EAEA,OAAOU,aAAa,CAACb,KAAa,EAAU;IAC1C,IAAIJ,eAAe,CAAC8B,IAAI,CAAC1B,KAAK,CAAC,EAAE;MAC/B,OAAOT,mBAAmB,CAACqB,SAAS;IACtC;IAEA,OAAOrB,mBAAmB,CAACY,MAAM;EACnC;EAEA,OAAOY,cAAc,CAACf,KAAa,EAAU;IAC3C,IAAIH,gBAAgB,CAAC6B,IAAI,CAAC1B,KAAK,CAAC,EAAE;MAChC,OAAOT,mBAAmB,CAACuB,UAAU;IACvC;IAEA,OAAOvB,mBAAmB,CAACY,MAAM;EACnC;EAEA,OAAOa,kBAAkB,CAAChB,KAAa,EAAU;IAC/C,IAAMoB,QAAQ,GAAGpB,KAAK,CAACqB,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;IACxC,IAAIvB,aAAa,CAACmB,mBAAmB,CAACG,QAAQ,CAAC,EAAE;MAC/C,IAAMI,KAAK,GAAGxB,KAAK,CAACyB,WAAW,EAAE;MACjC,IAAID,KAAK,KAAK,MAAM,IAAIA,KAAK,KAAK,OAAO,EAAE;QACzC,OAAOjC,mBAAmB,CAACmB,OAAO;MACpC;MAEA,IAAId,eAAe,CAAC8B,IAAI,CAAC1B,KAAK,CAAC,IAAIA,KAAK,CAACuB,QAAQ,CAAC,GAAG,CAAC,EAAE;QACtD,OAAOhC,mBAAmB,CAACqB,SAAS;MACtC;MAEA,IAAIf,gBAAgB,CAAC6B,IAAI,CAAC1B,KAAK,CAAC,EAAE;QAChC,OAAOT,mBAAmB,CAACuB,UAAU;MACvC;MAEA,OAAOvB,mBAAmB,CAACY,MAAM;IACnC;IAEA,OAAOL,aAAa,CAACwB,aAAa,CAACF,QAAQ,CAAC;EAC9C;EAEA,OAAOE,aAAa,CAACtB,KAAa,EAAU;IAC1C,IAAIA,KAAK,CAACuB,QAAQ,CAAC,GAAG,CAAC,EAAE;MACvB,OAAOhC,mBAAmB,CAACiB,MAAM;IACnC;;IAEA;IACA,IAAMmB,YAAY,GAAG3B,KAAK,CAAC4B,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE;IACpD,IAAI5B,KAAK,CAAC6B,MAAM,GAAGF,YAAY,EAAE;MAC/B,OAAOpC,mBAAmB,CAACe,IAAI;IACjC;;IAEA;IACA,IAAMwB,QAAQ,GAAGC,QAAQ,CAAC/B,KAAK,EAAE,EAAE,CAAC;IACpC,IAAI8B,QAAQ,GAAGpC,OAAO,IAAIoC,QAAQ,GAAGnC,OAAO,EAAE;MAC5C,OAAOJ,mBAAmB,CAACe,IAAI;IACjC;IAEA,OAAOf,mBAAmB,CAACa,OAAO;EACpC;EAEA4B,WAAW,CACTC,eAA4D,EAC5DC,IAA+B,EAC/BC,WAAoB,EACpBC,YAAqE,EACrElC,UAAyB,EACzBmC,UAA8C,EAC9CC,OAA6B,EAC7BC,WAAmB,EACnBC,KAAc,EACdC,UAAmB,EACnB;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IAAA;IACA,IAAI,CAACR,eAAe,GAAGA,eAAe;IACtC,IAAI,CAACC,IAAI,GAAGA,IAAI;IAChB,IAAI,CAACC,WAAW,GAAGA,WAAW;IAC9B,IAAI,CAACjC,UAAU,GAAGA,UAAU;IAC5B,IAAI,CAACmC,UAAU,GAAGA,UAAU;IAC5B,IAAI,CAACC,OAAO,GAAGA,OAAO;IACtB,IAAI,CAACI,MAAM,GAAG,CAAC;IACf,IAAI,CAACH,WAAW,GAAGA,WAAW;IAC9B,IAAI,CAACI,QAAQ,GAAG,CAAC;IACjB,IAAI,CAACH,KAAK,GAAGA,KAAK;IAClB,IAAI,CAACC,UAAU,GAAGA,UAAU;IAC5B,IAAI,CAACG,WAAW,GAAG,CAAC;IAEpB,IAAI,CAACC,WAAW,GAAG,IAAI,CAACA,WAAW,CAACC,IAAI,CAAC,IAAI,CAAC;IAC9C,IAAI,CAACC,cAAc,GAAG,IAAI,CAACA,cAAc,CAACD,IAAI,CAAC,IAAI,CAAC;IACpD,IAAI,CAACE,WAAW,GAAG,IAAI,CAACA,WAAW,CAACF,IAAI,CAAC,IAAI,CAAC;IAC9C,IAAI,CAACG,gBAAgB,GAAG,IAAI,CAACA,gBAAgB,CAACH,IAAI,CAAC,IAAI,CAAC;IAExD,IAAI,CAACI,MAAM,mCACNd,YAAY;MACfe,KAAK,EAAE,IAAI,CAACH,WAAW;MACvBI,KAAK,EAAE,IAAI,CAACP,WAAW;MACvBQ,QAAQ,EAAE,IAAI,CAACN;IAAc,EAC9B;EACH;EA8BAO,KAAK,GAAS;IACZ,IAAI,IAAI,CAACpB,IAAI,YAAYqB,IAAI,IAAI,IAAI,CAACrB,IAAI,YAAYsB,IAAI,EAAE;MAC1DlE,IAAI,CAACgE,KAAK,CAAC,IAAI,CAACpB,IAAI,EAAE,IAAI,CAACgB,MAAM,CAAC;IACpC,CAAC,MAAM;MACL,IAAMO,SAAS,GAAGjE,mBAAmB,CAAC,IAAI,CAAC0C,IAAI,EAAE,IAAI,CAACe,gBAAgB,CAAC;MACvE;MACA3D,IAAI,CAACgE,KAAK,CAACG,SAAS,EAAqB,IAAI,CAACP,MAAM,CAAC;MACrD;MACA;MACAO,SAAS,CAACC,MAAM,EAAE;IACpB;EACF;EAEAb,WAAW,CAACc,MAA6B,EAAEC,MAAc,EAAQ;IAC/D,IAAI;MAAEC;IAAK,CAAC,GAAGF,MAAM;IACrB,IAAI,CAAC,IAAI,CAACG,KAAK,EAAE;MACf,IAAID,IAAI,IAAI,IAAI,IAAIA,IAAI,CAAChC,MAAM,KAAK,CAAC,EAAE;QACrC+B,MAAM,CAACG,KAAK,EAAE;QACd,IAAI,CAACzB,OAAO,CAAC,qCAAqC,CAAC;QACnD;MACF;MACA,IAAI,CAACwB,KAAK,GAAG,IAAIE,KAAK,CAACH,IAAI,CAAC,CAAC,CAAC,CAAChC,MAAM,CAAC,CAACoC,IAAI,CAACxE,OAAO,CAAC;MACpD,IAAI,IAAI,CAAC0C,WAAW,EAAE;QACpB;QACA0B,IAAI,GAAGA,IAAI,CAACK,KAAK,CAAC,CAAC,CAAC;MACtB;IACF;IAEA,IAAI,CAACvB,QAAQ,IAAIkB,IAAI,CAAChC,MAAM;IAE5BxC,aAAa,CAAC,IAAI,CAACyE,KAAK,CAAC;IAEzB,IAAMK,UAAU,GAAG,CAAC,GAAG,IAAI,CAACL,KAAK,CAAC;IAElCD,IAAI,CAACO,OAAO,CAACC,GAAG,IAAI;MAClB,IAAIA,GAAG,CAACxC,MAAM,IAAIsC,UAAU,CAACtC,MAAM,EAAE;QACnC,KAAK,IAAIyC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGH,UAAU,CAACtC,MAAM,EAAEyC,CAAC,IAAI,CAAC,EAAE;UAC7CH,UAAU,CAACG,CAAC,CAAC,GAAGxE,aAAa,CAACC,aAAa,CACzC,IAAI,CAAC0C,UAAU,GAAG4B,GAAG,CAACC,CAAC,CAAC,CAACC,IAAI,EAAE,GAAGF,GAAG,CAACC,CAAC,CAAC,EACxCH,UAAU,CAACG,CAAC,CAAC,EACb,IAAI,CAACpE,UAAU,CAChB;QACH;QACA,IAAI,CAAC4D,KAAK,GAAGK,UAAU;MACzB,CAAC,MAAM;QACLP,MAAM,CAACG,KAAK,EAAE;QACd,IAAI,CAACzB,OAAO,yEACuD6B,UAAU,CAACtC,MAAM,wBAAcwC,GAAG,CAACxC,MAAM,sBAAYwC,GAAG,EAC1H;MACH;IACF,CAAC,CAAC;IAEF,IAAI,CAAC3B,MAAM,IAAI,CAAC;IAChB;IACA,IAAI8B,QAAQ,GAAG,CAAC;IAChB,IAAI,IAAI,CAACjC,WAAW,GAAG,CAAC,EAAE;MACxBiC,QAAQ,GAAGC,IAAI,CAACC,KAAK,CAAE,IAAI,CAAChC,MAAM,GAAG,IAAI,CAACH,WAAW,GAAI,EAAE,CAAC;IAC9D,CAAC,MAAM;MACLiC,QAAQ,GAAGC,IAAI,CAACC,KAAK,CAAC,IAAI,CAAC9B,WAAW,GAAG,CAAC,CAAC;IAC7C;IACA,IAAM+B,WAAW,GAAG,IAAI,CAACtC,UAAU,CAACmC,QAAQ,CAAC;IAC7C,IAAIG,WAAW,EAAE;MACff,MAAM,CAACG,KAAK,EAAE;IAChB;EACF;EAEAhB,cAAc,CAAC6B,OAA6B,EAAQ;IAClD,IAAM;MAAEd,KAAK;MAAE7B;IAAgB,CAAC,GAAG,IAAI;IACvC;IACA5C,aAAa,CAACyE,KAAK,CAAC;IACpB,IAAIc,OAAO,IAAI,IAAI,IAAI,CAACA,OAAO,CAACC,IAAI,CAACC,OAAO,EAAE;MAC5C7C,eAAe,CACb6B,KAAK,CAACiB,GAAG,CAAC9E,IAAI;MACZ;MACA;MACA;MACAA,IAAI,KAAKR,OAAO,IAAIQ,IAAI,KAAKV,mBAAmB,CAACuB,UAAU,GACvDvB,mBAAmB,CAACY,MAAM,GAC1BF,IAAI,CACT,EACD,IAAI,CAAC0C,QAAQ,CACd;IACH;EACF;EAEAK,WAAW,CAACG,KAAc,EAAQ;IAChC,IAAM;MAAEb;IAAQ,CAAC,GAAG,IAAI;IACxBA,OAAO,CAACa,KAAK,CAAC;EAChB;EAEAF,gBAAgB,CAAC+B,QAA6B,EAAQ;IACpD,IAAI,CAACpC,WAAW,GAAGoC,QAAQ,CAACC,OAAO;EACrC;AACF;AAEA,eAAenF,aAAa"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { JSZipObject, OnUpdateCallback, JSZipStreamHelper } from 'jszip';
|
|
2
|
+
/**
|
|
3
|
+
* This is used to help papaparse understand our stream.
|
|
4
|
+
* It uses these fields for feature detection, but never actually calls read()
|
|
5
|
+
* https://github.com/mholt/PapaParse/blob/master/papaparse.js#L244
|
|
6
|
+
*/
|
|
7
|
+
interface ZipStreamHelper extends JSZipStreamHelper<string> {
|
|
8
|
+
readable: boolean;
|
|
9
|
+
read(): void;
|
|
10
|
+
removeListener(): void;
|
|
11
|
+
}
|
|
12
|
+
export default function makeZipStreamHelper(zipObj: JSZipObject, onUpdate: OnUpdateCallback): ZipStreamHelper;
|
|
13
|
+
export {};
|
|
14
|
+
//# sourceMappingURL=ZipStreamHelper.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ZipStreamHelper.d.ts","sourceRoot":"","sources":["../../src/csv/ZipStreamHelper.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,OAAO,CAAC;AAE9E;;;;GAIG;AACH,UAAU,eAAgB,SAAQ,iBAAiB,CAAC,MAAM,CAAC;IACzD,QAAQ,EAAE,OAAO,CAAC;IAClB,IAAI,IAAI,IAAI,CAAC;IACb,cAAc,IAAI,IAAI,CAAC;CACxB;AAED,MAAM,CAAC,OAAO,UAAU,mBAAmB,CACzC,MAAM,EAAE,WAAW,EACnB,QAAQ,EAAE,gBAAgB,mBAiB3B"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This is used to help papaparse understand our stream.
|
|
3
|
+
* It uses these fields for feature detection, but never actually calls read()
|
|
4
|
+
* https://github.com/mholt/PapaParse/blob/master/papaparse.js#L244
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
export default function makeZipStreamHelper(zipObj, onUpdate) {
|
|
8
|
+
var helper = zipObj.internalStream('string');
|
|
9
|
+
helper.readable = true;
|
|
10
|
+
helper.read = () => false;
|
|
11
|
+
helper.removeListener = () => false;
|
|
12
|
+
helper.on('data', (_, metadata) => onUpdate(metadata));
|
|
13
|
+
return helper;
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=ZipStreamHelper.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ZipStreamHelper.js","names":["makeZipStreamHelper","zipObj","onUpdate","helper","internalStream","readable","read","removeListener","on","_","metadata"],"sources":["../../src/csv/ZipStreamHelper.ts"],"sourcesContent":["import type { JSZipObject, OnUpdateCallback, JSZipStreamHelper } from 'jszip';\n\n/**\n * This is used to help papaparse understand our stream.\n * It uses these fields for feature detection, but never actually calls read()\n * https://github.com/mholt/PapaParse/blob/master/papaparse.js#L244\n */\ninterface ZipStreamHelper extends JSZipStreamHelper<string> {\n readable: boolean;\n read(): void;\n removeListener(): void;\n}\n\nexport default function makeZipStreamHelper(\n zipObj: JSZipObject,\n onUpdate: OnUpdateCallback\n) {\n const helper: ZipStreamHelper = (\n zipObj as JSZipObject & {\n // The type could be anything except nodebuffer from https://stuk.github.io/jszip/documentation/api_zipobject/internal_stream.html\n // We only need it as a string though\n // JSZip types don't include this method for some reason\n internalStream(type: 'string'): JSZipStreamHelper<string>;\n }\n ).internalStream('string') as ZipStreamHelper;\n\n helper.readable = true;\n helper.read = () => false;\n helper.removeListener = () => false;\n helper.on('data', (_, metadata) => onUpdate(metadata));\n\n return helper;\n}\n"],"mappings":"AAEA;AACA;AACA;AACA;AACA;;AAOA,eAAe,SAASA,mBAAmB,CACzCC,MAAmB,EACnBC,QAA0B,EAC1B;EACA,IAAMC,MAAuB,GAC3BF,MAAM,CAMNG,cAAc,CAAC,QAAQ,CAAoB;EAE7CD,MAAM,CAACE,QAAQ,GAAG,IAAI;EACtBF,MAAM,CAACG,IAAI,GAAG,MAAM,KAAK;EACzBH,MAAM,CAACI,cAAc,GAAG,MAAM,KAAK;EACnCJ,MAAM,CAACK,EAAE,CAAC,MAAM,EAAE,CAACC,CAAC,EAAEC,QAAQ,KAAKR,QAAQ,CAACQ,QAAQ,CAAC,CAAC;EAEtD,OAAOP,MAAM;AACf"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@deephaven/console",
|
|
3
|
-
"version": "0.46.1
|
|
3
|
+
"version": "0.46.1",
|
|
4
4
|
"description": "Deephaven Console",
|
|
5
5
|
"author": "Deephaven Data Labs LLC",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -23,15 +23,15 @@
|
|
|
23
23
|
"build:sass": "sass --embed-sources --load-path=../../node_modules ./src:./dist"
|
|
24
24
|
},
|
|
25
25
|
"dependencies": {
|
|
26
|
-
"@deephaven/chart": "^0.46.1
|
|
27
|
-
"@deephaven/components": "^0.46.1
|
|
28
|
-
"@deephaven/icons": "^0.46.
|
|
29
|
-
"@deephaven/jsapi-bootstrap": "^0.46.1
|
|
30
|
-
"@deephaven/jsapi-types": "^0.46.
|
|
31
|
-
"@deephaven/log": "^0.46.
|
|
32
|
-
"@deephaven/react-hooks": "^0.46.1
|
|
33
|
-
"@deephaven/storage": "^0.46.
|
|
34
|
-
"@deephaven/utils": "^0.46.1
|
|
26
|
+
"@deephaven/chart": "^0.46.1",
|
|
27
|
+
"@deephaven/components": "^0.46.1",
|
|
28
|
+
"@deephaven/icons": "^0.46.0",
|
|
29
|
+
"@deephaven/jsapi-bootstrap": "^0.46.1",
|
|
30
|
+
"@deephaven/jsapi-types": "^0.46.0",
|
|
31
|
+
"@deephaven/log": "^0.46.0",
|
|
32
|
+
"@deephaven/react-hooks": "^0.46.1",
|
|
33
|
+
"@deephaven/storage": "^0.46.0",
|
|
34
|
+
"@deephaven/utils": "^0.46.1",
|
|
35
35
|
"@fortawesome/react-fontawesome": "^0.2.0",
|
|
36
36
|
"classnames": "^2.3.1",
|
|
37
37
|
"linkifyjs": "^4.1.0",
|
|
@@ -51,8 +51,8 @@
|
|
|
51
51
|
"react-dom": "^17.x"
|
|
52
52
|
},
|
|
53
53
|
"devDependencies": {
|
|
54
|
-
"@deephaven/jsapi-shim": "^0.46.
|
|
55
|
-
"@deephaven/mocks": "^0.46.
|
|
54
|
+
"@deephaven/jsapi-shim": "^0.46.0",
|
|
55
|
+
"@deephaven/mocks": "^0.46.0"
|
|
56
56
|
},
|
|
57
57
|
"files": [
|
|
58
58
|
"dist"
|
|
@@ -63,5 +63,5 @@
|
|
|
63
63
|
"publishConfig": {
|
|
64
64
|
"access": "public"
|
|
65
65
|
},
|
|
66
|
-
"gitHead": "
|
|
66
|
+
"gitHead": "c24d1911aa97d13ece3bebfae41b9ec75f33f580"
|
|
67
67
|
}
|