gridviz 2.0.27 → 2.0.29

package/src/dataset/GeoTIFF.js

@@ -0,0 +1,127 @@
+//@ts-check
+'use strict'
+
+import { DatasetComponent } from '../DatasetComponent'
+//import { fromUrl } from "geotiff"
+
+/**
+ * @todo
+ * A dataset composed of a single GeoTiff file.
+ *
+ * @author Joseph Davies, Julien Gaffuri
+ */
+export class GeoTIFF extends DatasetComponent {
+    /**
+     * @param {string} url The URL of the dataset.
+     * @param {number} resolution The dataset resolution in geographical unit.
+     * @param {{preprocess?:(function(import("../Dataset").Cell):boolean)}} opts
+     */
+    constructor(url, resolution, opts = {}) {
+        super(url, resolution, opts)
+
+        /**
+         * @private
+         * @type {Array.<import("../Dataset").Cell>} */
+        this.cells = []
+
+        /**
+         * @type {string}
+         * @private  */
+        this.infoLoadingStatus = 'notLoaded'
+    }
+
+    /**
+     * Request data within a geographic envelope.
+     *
+     * @param {import("../Dataset").Envelope|undefined} e
+     * @param {function():void} redraw
+     */
+    getData(e, redraw) {
+        //check if data already loaded
+        if (this.infoLoadingStatus != 'notLoaded') return this
+
+        //load data
+        this.infoLoadingStatus = 'loading'
+
+        /*
+
+        fromUrl(this.url)
+            .then(tiff => {
+                console.log(tiff);
+
+                (async function () {
+                    const image = await tiff.getImage();
+                    const width = image.getWidth();
+                    const height = image.getHeight();
+                    const tileWidth = image.getTileWidth();
+                    const tileHeight = image.getTileHeight();
+                    const samplesPerPixel = image.getSamplesPerPixel();
+
+                    //for geotiff
+                    const origin = image.getOrigin();
+                    const resolution = image.getResolution();
+                    const bbox = image.getBoundingBox();
+
+                    console.log(image)
+                    console.log(width, height)
+                    console.log(tileWidth, tileHeight)
+                    console.log(samplesPerPixel)
+                    console.log(origin)
+                    console.log(resolution)
+                    console.log(bbox)
+
+                    console.log("-------")
+
+                    const data = await image.readRasters();
+                    console.log(data)
+
+                })()
+
+                //convert coordinates in numbers
+                //for (const c of data) { c.x = +c.x; c.y = +c.y; }
+
+                //this.cells = data;
+
+                //execute preprocess, if any
+                //if (this.preprocess) for (const c of this.cells) this.preprocess(c);
+
+                //TODO check if redraw is necessary
+                //that is if the dataset belongs to a layer which is visible at the current zoom level
+
+                //execute the callback, usually a draw function
+                if (redraw) redraw()
+
+                this.infoLoadingStatus = "loaded";
+
+
+            })
+            .catch(() => {
+                //mark as failed
+                this.infoLoadingStatus = "failed";
+                this.cells = []
+            });
+*/
+
+        return this
+    }
+
+    /**
+     * Fill the view cache with all cells which are within a geographical envelope.
+     *
+     * @param {import("../Dataset").Envelope} extGeo
+     * @returns {void}
+     */
+    updateViewCache(extGeo) {
+        //data not loaded yet
+        if (!this.cells) return
+
+        this.cellsViewCache = []
+        for (const cell of this.cells) {
+            if (+cell.x + this.resolution < extGeo.xMin) continue
+            if (+cell.x - this.resolution > extGeo.xMax) continue
+            if (+cell.y + this.resolution < extGeo.yMin) continue
+            if (+cell.y - this.resolution > extGeo.yMax) continue
+            this.cellsViewCache.push(cell)
+        }
+    }
+}

package/src/dataset/GridTile.js

@@ -0,0 +1,41 @@
+//@ts-check
+'use strict'
+
+/**
+ * A grid tile.
+ *
+ * @author Joseph Davies, Julien Gaffuri
+ */
+export class GridTile {
+    /**
+     * @param {Array.<import("../Dataset").Cell>} cells The tile cells.
+     * @param {number} xT The X position of the tile.
+     * @param {number} yT The Y position of the tile.
+     * @param {import("./CSVGrid").GridInfo} gridInfo The grid info object.
+     */
+    constructor(cells, xT, yT, gridInfo) {
+        /** @type {Array.<import("../Dataset").Cell>} */
+        this.cells = cells
+        /** @type {number} */
+        this.x = xT
+        /** @type {number} */
+        this.y = yT
+
+        const r = gridInfo.resolutionGeo
+        const s = gridInfo.tileSizeCell
+
+        /** @type {import("../Dataset").Envelope} */
+        this.extGeo = {
+            xMin: gridInfo.originPoint.x + r * s * this.x,
+            xMax: gridInfo.originPoint.x + r * s * (this.x + 1),
+            yMin: gridInfo.originPoint.y + r * s * this.y,
+            yMax: gridInfo.originPoint.y + r * s * (this.y + 1),
+        }
+
+        //convert cell coordinates into geographical coordinates
+        for (let cell of this.cells) {
+            cell.x = this.extGeo.xMin + cell.x * r
+            cell.y = this.extGeo.yMin + cell.y * r
+        }
+    }
+}

package/src/dataset/ParquetGrid.js

@@ -0,0 +1,123 @@
+//@ts-check
+'use strict'
+
+import { DatasetComponent } from '../DatasetComponent'
+import { tableFromIPC } from 'apache-arrow'
+//import { readParquet } from "parquet-wasm"
+
+/**
+ * A dataset composed of a single parquet file (not tiled).
+ *
+ * @author Julien Gaffuri
+ */
+export class ParquetGrid extends DatasetComponent {
+    /**
+     * @param {string} url The URL of the dataset.
+     * @param {number} resolution The dataset resolution in geographical unit.
+     * @param {{preprocess?:(function(import("../Dataset").Cell):boolean), readParquetFun?:Function}} opts
+     */
+    constructor(url, resolution, opts = {}) {
+        super(url, resolution, opts)
+
+        /**
+         * @private
+         * @type {Array.<import("../Dataset").Cell>} */
+        this.cells = []
+
+        /**
+         * @type {string}
+         * @private  */
+        this.infoLoadingStatus = 'notLoaded'
+
+        if (!opts.readParquetFun) throw new Error('readParquet function needed for parquet dataset')
+
+        /**
+         * @type {Function}
+         * @private  */
+        this.readParquetFun = opts.readParquetFun
+    }
+
+    /**
+     * Request data within a geographic envelope.
+     *
+     * @param {import("../Dataset").Envelope|undefined} e
+     * @param {function():void} redraw
+     */
+    getData(e, redraw) {
+        //check if data already loaded
+        if (this.infoLoadingStatus != 'notLoaded') return this
+
+        //load data
+        this.infoLoadingStatus = 'loading'
+
+        ;(async () => {
+            try {
+                const resp = await fetch(this.url)
+                const parquetUint8Array = new Uint8Array(await resp.arrayBuffer())
+                const arrowUint8Array = this.readParquetFun(parquetUint8Array)
+
+                const t = tableFromIPC(arrowUint8Array)
+                //see https://arrow.apache.org/docs/js/
+                //https://loaders.gl/arrowjs/docs/developer-guide/tables#record-tojson-and-toarray
+
+                this.cells = []
+                for (const e of t) {
+                    //get cell
+                    const c = e.toJSON()
+
+                    //preprocess/filter
+                    if (this.preprocess) {
+                        const b = this.preprocess(c)
+                        if (b == false) continue
+                        this.cells.push(c)
+                    } else {
+                        this.cells.push(c)
+                    }
+                }
+
+                //TODO check if redraw is necessary
+                //that is if the dataset belongs to a layer which is visible at the current zoom level
+
+                //execute the callback, usually a draw function
+                if (redraw) redraw()
+
+                this.infoLoadingStatus = 'loaded'
+            } catch (error) {
+                //mark as failed
+                this.infoLoadingStatus = 'failed'
+                this.cells = []
+            }
+        })()
+
+        return this
+    }
+
+    /**
+     * Fill the view cache with all cells which are within a geographical envelope.
+     *
+     * @param {import("../Dataset").Envelope} extGeo
+     * @returns {void}
+     */
+    updateViewCache(extGeo) {
+        //data not loaded yet
+        if (!this.cells) return
+
+        this.cellsViewCache = []
+        for (const cell of this.cells) {
+            if (+cell.x + this.resolution < extGeo.xMin) continue
+            if (+cell.x - this.resolution > extGeo.xMax) continue
+            if (+cell.y + this.resolution < extGeo.yMin) continue
+            if (+cell.y - this.resolution > extGeo.yMax) continue
+            this.cellsViewCache.push(cell)
+        }
+    }
+}
+
+/**
+ * Retrieve parquet wasm decoder
+ */
+const getReadParquet = async () => {
+    //const parquetModule = await import("https://unpkg.com/parquet-wasm@0.4.0-beta.5/esm/arrow2.js");
+    //await parquetModule.default();
+    //return parquetModule.readParquet;
+}

package/src/dataset/TiledGrid.js

@@ -0,0 +1,294 @@
+//@ts-check
+'use strict'
+
+/** @typedef {{ dims: object, crs: string, tileSizeCell: number, originPoint: {x:number,y:number}, resolutionGeo: number, tilingBounds:import("../Dataset").Envelope, format:import("../DatasetComponent").Format }} GridInfo */
+
+import { json, csv } from 'd3-fetch'
+import { GridTile } from './GridTile'
+import { App } from '../App'
+import { DatasetComponent } from '../DatasetComponent'
+import { monitor, monitorDuration } from '../utils/Utils'
+import { tableFromIPC } from 'apache-arrow'
+
+/**
+ * A tiled dataset, composed of CSV (or parquet) tiles.
+ *
+ * @author Joseph Davies, Julien Gaffuri
+ */
+export class TiledGrid extends DatasetComponent {
+    /**
+     * @param {string} url The URL of the dataset.
+     * @param {App} app The application.
+     * @param {{preprocess?:(function(import("../Dataset").Cell):boolean), readParquetFun?:Function }} opts
+     */
+    constructor(url, app, opts = {}) {
+        super(url, 0, opts)
+
+        /**
+         * The app being used.
+         * @type {App}
+         */
+        this.app = app
+
+        /**
+         * The grid info object, from the info.json file.
+         *  @type {GridInfo | undefined}
+         * @private
+         *  */
+        this.info = undefined
+
+        /**
+         * @type {string}
+         * @private  */
+        this.infoLoadingStatus = 'notLoaded'
+
+        /**
+         * The cache of the loaded tiles. It is double indexed: by xT and then yT.
+         * Example: this.cache[xT][yT] returns the tile at [xT][yT] location.
+         *
+         * @type {object}
+         * */
+        this.cache = {}
+
+        /**
+         * @type {Function|undefined}
+         * @private  */
+        this.readParquetFun = opts.readParquetFun
+    }
+
+    /**
+     * Load the info.json from the url.
+     *
+     * @param {function():void} callback
+     * @returns this
+     */
+    loadInfo(callback) {
+        if (!this.info && this.infoLoadingStatus === 'notLoaded') {
+            ;(async () => {
+                try {
+                    const data = await json(this.url + 'info.json')
+                    this.info = data
+                    this.resolution = data.resolutionGeo
+                    this.infoLoadingStatus = 'loaded'
+                    if (callback) callback()
+                } catch (error) {
+                    //mark as failed
+                    this.infoLoadingStatus = 'failed'
+                }
+            })()
+        } else if (callback && (this.infoLoadingStatus === 'loaded' || this.infoLoadingStatus === 'failed'))
+            callback()
+        return this
+    }
+
+    /**
+     * Compute a tiling envelope from a geographical envelope.
+     * This is the function to use to know which tiles to download for a geographical view.
+     *
+     * @param {import("../Dataset").Envelope} e
+     * @returns {import("../Dataset").Envelope|undefined}
+     */
+    getTilingEnvelope(e) {
+        if (!this.info) {
+            this.loadInfo(() => {})
+            return
+        }
+
+        const po = this.info.originPoint,
+            r = this.info.resolutionGeo,
+            s = this.info.tileSizeCell
+
+        return {
+            xMin: Math.floor((e.xMin - po.x) / (r * s)),
+            xMax: Math.floor((e.xMax - po.x) / (r * s)),
+            yMin: Math.floor((e.yMin - po.y) / (r * s)),
+            yMax: Math.floor((e.yMax - po.y) / (r * s)),
+        }
+    }
+
+    /**
+     * Request data within a geographic envelope.
+     *
+     * @param {import("../Dataset").Envelope} extGeo
+     * @param {function():void} redrawFun
+     * @returns {this}
+     */
+    getData(extGeo, redrawFun) {
+        //TODO empty cache when it gets too big ?
+
+        //check if info has been loaded
+        if (!this.info) return this
+
+        //tiles within the scope
+        /** @type {import("../Dataset").Envelope|undefined} */
+        const tb = this.getTilingEnvelope(extGeo)
+        if (!tb) return this
+
+        //grid bounds
+        /** @type {import("../Dataset").Envelope} */
+        const gb = this.info.tilingBounds
+
+        const format = this.info.format
+
+        for (let xT = Math.max(tb.xMin, gb.xMin); xT <= Math.min(tb.xMax, gb.xMax); xT++) {
+            for (let yT = Math.max(tb.yMin, gb.yMin); yT <= Math.min(tb.yMax, gb.yMax); yT++) {
+                //prepare cache
+                if (!this.cache[xT]) this.cache[xT] = {}
+
+                //check if tile exists in the cache
+                /** @type {GridTile} */
+                let tile = this.cache[xT][yT]
+                if (tile) continue
+
+                //mark tile as loading
+                this.cache[xT][yT] = 'loading'
+
+                ;(async () => {
+                    //request tile
+                    /** @type {Array.<import("../Dataset").Cell>}  */
+                    let cells
+
+                    try {
+                        if (!format || format === 'CSV') {
+                            /** @type {Array.<import("../Dataset").Cell>}  */
+                            // @ts-ignore
+                            const data = await csv(this.url + xT + '/' + yT + '.csv')
+
+                            if (monitor) monitorDuration('*** TiledGrid parse start')
+
+                            //preprocess/filter
+                            if (this.preprocess) {
+                                cells = []
+                                for (const c of data) {
+                                    const b = this.preprocess(c)
+                                    if (b == false) continue
+                                    cells.push(c)
+                                }
+                            } else {
+                                cells = data
+                            }
+
+                            if (monitor) monitorDuration('preprocess / filter')
+                        } else if (format === 'PARQUET') {
+                            if (!this.readParquetFun)
+                                throw new Error('readParquet function needed for parquet dataset')
+
+                            const resp = await fetch(this.url + xT + '/' + yT + '.parquet')
+                            const parquetUint8Array = new Uint8Array(await resp.arrayBuffer())
+                            const arrowUint8Array = this.readParquetFun(parquetUint8Array)
+                            const t = tableFromIPC(arrowUint8Array)
+
+                            cells = []
+                            for (const e of t) {
+                                //get cell
+                                const c = e.toJSON()
+
+                                //preprocess/filter
+                                if (this.preprocess) {
+                                    const b = this.preprocess(c)
+                                    if (b == false) continue
+                                    cells.push(c)
+                                } else {
+                                    cells.push(c)
+                                }
+                            }
+                        } else throw new Error('Tiled format not supported: ' + format)
+                    } catch (error) {
+                        //mark as failed
+                        this.cache[xT][yT] = 'failed'
+                        return
+                    }
+
+                    //store tile in cache
+                    if (!this.info) {
+                        console.error('Tile info inknown')
+                        return
+                    }
+                    const tile_ = new GridTile(cells, xT, yT, this.info)
+                    this.cache[xT][yT] = tile_
+
+                    if (monitor) monitorDuration('storage')
+
+                    //if no redraw is specified, then leave
+                    if (!redrawFun) return
+
+                    //check if redraw is really needed, that is if:
+
+                    // 1. the dataset belongs to a layer which is visible at the current zoom level
+                    let redraw = false
+                    //go through the layers
+                    const zf = this.app.getZoomFactor()
+                    for (const lay of this.app.layers) {
+                        if (!lay.visible) continue
+                        if (lay.getDatasetComponent(zf) != this) continue
+                        //found one layer. No need to seek more.
+                        redraw = true
+                        break
+                    }
+                    if (monitor) monitorDuration('check redraw 1')
+
+                    if (!redraw) return
+
+                    // 2. the tile is within the view, that is its geo envelope intersects the viewer geo envelope.
+                    const env = this.app.updateExtentGeo()
+                    const envT = tile_.extGeo
+                    if (env.xMax <= envT.xMin) return
+                    if (env.xMin >= envT.xMax) return
+                    if (env.yMax <= envT.yMin) return
+                    if (env.yMin >= envT.yMax) return
+
+                    if (monitor) monitorDuration('check redraw 2')
+                    if (monitor) monitorDuration('*** TiledGrid parse end')
+
+                    //redraw
+                    redrawFun()
+                })()
+            }
+        }
+        return this
+    }
+
+    /**
+     * Fill the view cache with all cells which are within a geographical envelope.
+     * @abstract
+     * @param {import("../Dataset").Envelope} extGeo
+     * @returns {void}
+     */
+    updateViewCache(extGeo) {
+        //
+        this.cellsViewCache = []
+
+        //check if info has been loaded
+        if (!this.info) return
+
+        //tiles within the scope
+        /** @type {import("../Dataset").Envelope|undefined} */
+        const tb = this.getTilingEnvelope(extGeo)
+        if (!tb) return
+
+        //grid bounds
+        /** @type {import("../Dataset").Envelope} */
+        const gb = this.info.tilingBounds
+
+        for (let xT = Math.max(tb.xMin, gb.xMin); xT <= Math.min(tb.xMax, gb.xMax); xT++) {
+            if (!this.cache[xT]) continue
+            for (let yT = Math.max(tb.yMin, gb.yMin); yT <= Math.min(tb.yMax, gb.yMax); yT++) {
+                //get tile
+                /** @type {GridTile} */
+                const tile = this.cache[xT][yT]
+                if (!tile || typeof tile === 'string') continue
+
+                //get cells
+                //this.cellsViewCache = this.cellsViewCache.concat(tile.cells)
+
+                for (const cell of tile.cells) {
+                    if (+cell.x + this.resolution < extGeo.xMin) continue
+                    if (+cell.x - this.resolution > extGeo.xMax) continue
+                    if (+cell.y + this.resolution < extGeo.yMin) continue
+                    if (+cell.y - this.resolution > extGeo.yMax) continue
+                    this.cellsViewCache.push(cell)
+                }
+            }
+        }
+    }
+}