gridviz 2.0.40 → 2.0.41

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "gridviz",
-    "version": "2.0.40",
+    "version": "2.0.41",
     "description": "Visualization tool for gridded statistics",
     "keywords": [
         "statistics",
@@ -47,7 +47,6 @@
         "src/**/*.js"
     ],
     "dependencies": {
-        "apache-arrow": "^10.0.1",
         "d3-color": "^3.1.0",
         "d3-fetch": "^3.0.1",
         "d3-format": "^3.1.0",

package/src/App.js

@@ -7,7 +7,6 @@ import { Layer } from './Layer.js'
 import { Dataset } from './Dataset.js'
 import { Tooltip } from './Tooltip.js'
 import { CSVGrid } from './dataset/CSVGrid.js'
-//import { ParquetGrid } from './dataset/ParquetGrid';
 import { TiledGrid } from './dataset/TiledGrid.js'
 import { BackgroundLayer } from './BackgroundLayer.js'
 import { BackgroundLayerWMS } from './BackgroundLayerWMS.js'
@@ -450,18 +449,6 @@ export class App {
         )
     }
 
-    /**
-     * Make a parquet grid dataset.
-     *
-     * @param {string} url The URL of the dataset.
-     * @param {number} resolution The dataset resolution in geographical unit.
-     * @param {object=} opts The parameters of the dataset.
-     * @returns {Dataset}
-     */
-    /*makeParquetGridDataset(url, resolution, opts) {
-        return new Dataset([new ParquetGrid(url, resolution, opts).getData(undefined, () => { this.cg.redraw(); })], [], opts)
-    }*/
-
     /**
      * Make a tiled grid dataset.
      *
@@ -483,22 +470,6 @@ export class App {
 
     //multi scale dataset creation
 
-    /**
-     * Make a multi scale parquet grid dataset.
-     *
-     * @param {Array.<number>} resolutions
-     * @param {function(number):string} resToURL
-     * @param {{preprocess?:function(import('./Dataset').Cell):boolean}} opts
-     * @returns {Dataset}
-     */
-    /*makeMultiScaleParquetGridDataset(resolutions, resToURL, opts) {
-        return Dataset.make(
-            resolutions,
-            (res) => new ParquetGrid(resToURL(res), res, opts).getData(undefined, () => { this.cg.redraw(); }),
-            opts
-        )
-    }*/
-
     /**
      * Make a multi scale CSV grid dataset.
      *
@@ -555,19 +526,6 @@ export class App {
         return this.addLayerFromDataset(ds, styles, opts)
     }
 
-    /**
-     * Add a layer from a parquet grid dataset.
-     *
-     * @param {string} url The URL of the dataset.
-     * @param {number} resolution The dataset resolution in geographical unit.
-     * @param {Array.<import('./Style').Style>} styles The styles, ordered in drawing order.
-     * @param {object=} opts The parameters of the dataset and layer.
-     * @returns {this}
-     */
-    /*addParquetGridLayer(url, resolution, styles, opts) {
-        const ds = this.makeParquetGridDataset(url, resolution, opts)
-        return this.addLayerFromDataset(ds, styles, opts);
-    }*/
 
     /**
      *
@@ -595,20 +553,6 @@ export class App {
         return this.addLayerFromDataset(ds, styles, opts)
     }
 
-    /**
-     * Add a layer from a parquet grid dataset.
-     *
-     * @param {Array.<number>} resolutions
-     * @param {function(number):string} resToURL
-     * @param {Array.<import('./Style').Style>} styles The styles, ordered in drawing order.
-     * @param {object=} opts The parameters of the dataset and layer.
-     * @returns {this}
-     */
-    /*addMultiScaleParquetGridLayer(resolutions, resToURL, styles, opts) {
-        const ds = this.makeMultiScaleParquetGridDataset(resolutions, resToURL, opts)
-        return this.addLayerFromDataset(ds, styles, opts);
-    }*/
-
     /**
      * @param {Array.<number>} resolutions
      * @param {function(number):string} resToURL

package/src/DatasetComponent.js

@@ -1,8 +1,6 @@
 //@ts-check
 'use strict'
 
-/** @typedef {"CSV"|"PARQUET"} Format */
-
 /**
  * A dataset component, of grid cells.
  * @abstract

package/src/dataset/TiledGrid.js

@@ -11,10 +11,9 @@ import { monitor, monitorDuration } from '../utils/Utils.js'
 
 // external
 import { json, csv } from 'd3-fetch'
-import { tableFromIPC } from 'apache-arrow'
 
 /**
- * A tiled dataset, composed of CSV (or parquet) tiles.
+ * A tiled dataset, composed of CSV tiles.
  *
  * @author Joseph Davies, Julien Gaffuri
  */
@@ -22,7 +21,7 @@ export class TiledGrid extends DatasetComponent {
     /**
      * @param {string} url The URL of the dataset.
      * @param {App} app The application.
-     * @param {{preprocess?:(function(import("../Dataset").Cell):boolean), readParquetFun?:Function }} opts
+     * @param {{preprocess?:(function(import("../Dataset").Cell):boolean) }} opts
      */
     constructor(url, app, opts = {}) {
         super(url, 0, opts)
@@ -53,10 +52,6 @@ export class TiledGrid extends DatasetComponent {
          * */
         this.cache = {}
 
-        /**
-         * @type {Function|undefined}
-         * @private  */
-        this.readParquetFun = opts.readParquetFun
     }
 
     /**
@@ -67,7 +62,7 @@ export class TiledGrid extends DatasetComponent {
      */
     loadInfo(callback) {
         if (!this.info && this.infoLoadingStatus === 'notLoaded') {
-            ;(async () => {
+            ; (async () => {
                 try {
                     const data = await json(this.url + 'info.json')
                     this.info = data
@@ -93,7 +88,7 @@ export class TiledGrid extends DatasetComponent {
      */
     getTilingEnvelope(e) {
         if (!this.info) {
-            this.loadInfo(() => {})
+            this.loadInfo(() => { })
             return
         }
 
@@ -131,8 +126,6 @@ export class TiledGrid extends DatasetComponent {
         /** @type {import("../Dataset").Envelope} */
         const gb = this.info.tilingBounds
 
-        const format = this.info.format
-
         for (let xT = Math.max(tb.xMin, gb.xMin); xT <= Math.min(tb.xMax, gb.xMax); xT++) {
             for (let yT = Math.max(tb.yMin, gb.yMin); yT <= Math.min(tb.yMax, gb.yMax); yT++) {
                 //prepare cache
@@ -145,13 +138,12 @@ export class TiledGrid extends DatasetComponent {
 
                 //mark tile as loading
                 this.cache[xT][yT] = 'loading'
-                ;(async () => {
-                    //request tile
-                    /** @type {Array.<import("../Dataset").Cell>}  */
-                    let cells
+                    ; (async () => {
+                        //request tile
+                        /** @type {Array.<import("../Dataset").Cell>}  */
+                        let cells
 
-                    try {
-                        if (!format || format === 'CSV') {
+                        try {
                             /** @type {Array.<import("../Dataset").Cell>}  */
                             // @ts-ignore
                             const data = await csv(this.url + xT + '/' + yT + '.csv')
@@ -171,80 +163,56 @@ export class TiledGrid extends DatasetComponent {
                             }
 
                             if (monitor) monitorDuration('preprocess / filter')
-                        } else if (format === 'PARQUET') {
-                            if (!this.readParquetFun)
-                                throw new Error('readParquet function needed for parquet dataset')
-
-                            const resp = await fetch(this.url + xT + '/' + yT + '.parquet')
-                            const parquetUint8Array = new Uint8Array(await resp.arrayBuffer())
-                            const arrowUint8Array = this.readParquetFun(parquetUint8Array)
-                            const t = tableFromIPC(arrowUint8Array)
-
-                            cells = []
-                            for (const e of t) {
-                                //get cell
-                                const c = e.toJSON()
-
-                                //preprocess/filter
-                                if (this.preprocess) {
-                                    const b = this.preprocess(c)
-                                    if (b == false) continue
-                                    cells.push(c)
-                                } else {
-                                    cells.push(c)
-                                }
-                            }
-                        } else throw new Error('Tiled format not supported: ' + format)
-                    } catch (error) {
-                        //mark as failed
-                        this.cache[xT][yT] = 'failed'
-                        return
-                    }
-
-                    //store tile in cache
-                    if (!this.info) {
-                        console.error('Tile info inknown')
-                        return
-                    }
-                    const tile_ = new GridTile(cells, xT, yT, this.info)
-                    this.cache[xT][yT] = tile_
-
-                    if (monitor) monitorDuration('storage')
-
-                    //if no redraw is specified, then leave
-                    if (!redrawFun) return
-
-                    //check if redraw is really needed, that is if:
-
-                    // 1. the dataset belongs to a layer which is visible at the current zoom level
-                    let redraw = false
-                    //go through the layers
-                    const zf = this.app.getZoomFactor()
-                    for (const lay of this.app.layers) {
-                        if (!lay.visible) continue
-                        if (lay.getDatasetComponent(zf) != this) continue
-                        //found one layer. No need to seek more.
-                        redraw = true
-                        break
-                    }
-                    if (monitor) monitorDuration('check redraw 1')
-
-                    if (!redraw) return
-
-                    // 2. the tile is within the view, that is its geo envelope intersects the viewer geo envelope.
-                    const env = this.app.updateExtentGeo()
-                    const envT = tile_.extGeo
-                    if (env.xMax <= envT.xMin) return
-                    if (env.xMin >= envT.xMax) return
-                    if (env.yMax <= envT.yMin) return
-                    if (env.yMin >= envT.yMax) return
-
-                    if (monitor) monitorDuration('check redraw 2')
-                    if (monitor) monitorDuration('*** TiledGrid parse end')
-
-                    //redraw
-                    redrawFun()
-                })()
+                        } catch (error) {
+                            //mark as failed
+                            this.cache[xT][yT] = 'failed'
+                            return
+                        }
+
+                        //store tile in cache
+                        if (!this.info) {
+                            console.error('Tile info inknown')
+                            return
+                        }
+                        const tile_ = new GridTile(cells, xT, yT, this.info)
+                        this.cache[xT][yT] = tile_
+
+                        if (monitor) monitorDuration('storage')
+
+                        //if no redraw is specified, then leave
+                        if (!redrawFun) return
+
+                        //check if redraw is really needed, that is if:
+
+                        // 1. the dataset belongs to a layer which is visible at the current zoom level
+                        let redraw = false
+                        //go through the layers
+                        const zf = this.app.getZoomFactor()
+                        for (const lay of this.app.layers) {
+                            if (!lay.visible) continue
+                            if (lay.getDatasetComponent(zf) != this) continue
+                            //found one layer. No need to seek more.
+                            redraw = true
+                            break
+                        }
+                        if (monitor) monitorDuration('check redraw 1')
+
+                        if (!redraw) return
+
+                        // 2. the tile is within the view, that is its geo envelope intersects the viewer geo envelope.
+                        const env = this.app.updateExtentGeo()
+                        const envT = tile_.extGeo
+                        if (env.xMax <= envT.xMin) return
+                        if (env.xMin >= envT.xMax) return
+                        if (env.yMax <= envT.yMin) return
+                        if (env.yMin >= envT.yMax) return
+
+                        if (monitor) monitorDuration('check redraw 2')
+                        if (monitor) monitorDuration('*** TiledGrid parse end')
+
+                        //redraw
+                        redrawFun()
+                    })()
             }
         }
         return this

package/src/index.js

@@ -12,7 +12,6 @@ export { DatasetComponent } from './DatasetComponent.js'
 // export dataset types
 export { TiledGrid } from './dataset/TiledGrid.js'
 export { CSVGrid } from './dataset/CSVGrid.js'
-//export { ParquetGrid } from "./dataset/ParquetGrid"
 //export { GeoTIFF } from "./dataset/GeoTIFF"
 
 // export styles
@@ -53,6 +52,9 @@ export * from './utils/stretching.js'
 
 export { getClass } from './utils/Utils.js'
 
+
+
+
 import { GeoCanvas } from './GeoCanvas.js'
 import { geoAzimuthalEqualArea } from 'd3-geo'
 

package/src/dataset/ParquetGrid.js

@@ -1,122 +0,0 @@
-//@ts-check
-'use strict'
-
-import { DatasetComponent } from '../DatasetComponent.js'
-import { tableFromIPC } from 'apache-arrow'
-//import { readParquet } from "parquet-wasm"
-
-/**
- * A dataset composed of a single parquet file (not tiled).
- *
- * @author Julien Gaffuri
- */
-export class ParquetGrid extends DatasetComponent {
-    /**
-     * @param {string} url The URL of the dataset.
-     * @param {number} resolution The dataset resolution in geographical unit.
-     * @param {{preprocess?:(function(import("../Dataset").Cell):boolean), readParquetFun?:Function}} opts
-     */
-    constructor(url, resolution, opts = {}) {
-        super(url, resolution, opts)
-
-        /**
-         * @private
-         * @type {Array.<import("../Dataset").Cell>} */
-        this.cells = []
-
-        /**
-         * @type {string}
-         * @private  */
-        this.infoLoadingStatus = 'notLoaded'
-
-        if (!opts.readParquetFun) throw new Error('readParquet function needed for parquet dataset')
-
-        /**
-         * @type {Function}
-         * @private  */
-        this.readParquetFun = opts.readParquetFun
-    }
-
-    /**
-     * Request data within a geographic envelope.
-     *
-     * @param {import("../Dataset").Envelope|undefined} e
-     * @param {function():void} redraw
-     */
-    getData(e, redraw) {
-        //check if data already loaded
-        if (this.infoLoadingStatus != 'notLoaded') return this
-
-        //load data
-        this.infoLoadingStatus = 'loading'
-        ;(async () => {
-            try {
-                const resp = await fetch(this.url)
-                const parquetUint8Array = new Uint8Array(await resp.arrayBuffer())
-                const arrowUint8Array = this.readParquetFun(parquetUint8Array)
-
-                const t = tableFromIPC(arrowUint8Array)
-                //see https://arrow.apache.org/docs/js/
-                //https://loaders.gl/arrowjs/docs/developer-guide/tables#record-tojson-and-toarray
-
-                this.cells = []
-                for (const e of t) {
-                    //get cell
-                    const c = e.toJSON()
-
-                    //preprocess/filter
-                    if (this.preprocess) {
-                        const b = this.preprocess(c)
-                        if (b == false) continue
-                        this.cells.push(c)
-                    } else {
-                        this.cells.push(c)
-                    }
-                }
-
-                //TODO check if redraw is necessary
-                //that is if the dataset belongs to a layer which is visible at the current zoom level
-
-                //execute the callback, usually a draw function
-                if (redraw) redraw()
-
-                this.infoLoadingStatus = 'loaded'
-            } catch (error) {
-                //mark as failed
-                this.infoLoadingStatus = 'failed'
-                this.cells = []
-            }
-        })()
-
-        return this
-    }
-
-    /**
-     * Fill the view cache with all cells which are within a geographical envelope.
-     *
-     * @param {import("../Dataset").Envelope} extGeo
-     * @returns {void}
-     */
-    updateViewCache(extGeo) {
-        //data not loaded yet
-        if (!this.cells) return
-
-        this.cellsViewCache = []
-        for (const cell of this.cells) {
-            if (+cell.x + this.resolution < extGeo.xMin) continue
-            if (+cell.x - this.resolution > extGeo.xMax) continue
-            if (+cell.y + this.resolution < extGeo.yMin) continue
-            if (+cell.y - this.resolution > extGeo.yMax) continue
-            this.cellsViewCache.push(cell)
-        }
-    }
-}
-
-/**
- * Retrieve parquet wasm decoder
- */
-const getReadParquet = async () => {
-    //const parquetModule = await import("https://unpkg.com/parquet-wasm@0.4.0-beta.5/esm/arrow2.js");
-    //await parquetModule.default();
-    //return parquetModule.readParquet;
-}