gridviz 3.0.40 → 3.1.0

package/src/dataset/TiledGrid.js

@@ -5,11 +5,23 @@
 
 // internal
 import { Dataset } from '../core/Dataset.js'
+import { isVisibleAt } from '../core/Drawable.js'
 //import { monitor, monitorDuration } from '../utils/Utils.js'
 
 // external
 import { json, csv } from 'd3-fetch'
 
+// The tiles that are loaded are kept, so that going back to a place needs no new request. They are kept as lists of cell objects, which
+// take a lot of memory (the education grid of the Statistical Atlas reached 1.5 GB after some minutes of panning and zooming), so the
+// number of cells kept by all the TiledGrid together is limited (TiledGrid.maxCachedCells): when there are more, the tiles that were used
+// the longest time ago are dropped, and loaded again if the map needs them. The tiles of the view on the screen, and those used in the
+// last KEEP_MS ms, are never dropped.
+const KEEP_MS = 3000
+/** All the tiles kept, of every dataset. */
+const keptTiles = new Set()
+let keptCells = 0
+const now = () => (typeof performance !== 'undefined' ? performance.now() : Date.now())
+
 /**
  * A tiled dataset, composed of CSV tiles.
  *
@@ -85,7 +97,7 @@ export class TiledGrid extends Dataset {
         if (!this.map || !this.map.layers) return false
         const z = this.map.getZoom()
         for (const lay of this.map.layers) {
-            if (lay.visible && !lay.visible(z)) continue
+            if (!isVisibleAt(lay, z)) continue
             if (!lay.getDataset) continue
             if (lay.getDataset(z) == this) return true
         }
@@ -141,8 +153,6 @@ export class TiledGrid extends Dataset {
         const yMin = Math.max(tb.yMin, gbYMin)
         const yMax = Math.min(tb.yMax, gbYMax)
 
-        const totalTiles = (xMax - xMin + 1) * (yMax - yMin + 1)
-        let processedTiles = 0
         const tilePromises = []
 
         // Iterate over tiles within bounds
@@ -150,9 +160,13 @@ export class TiledGrid extends Dataset {
             for (let yT = yMin; yT <= yMax; yT++) {
                 if (!this.cache[xT]) this.cache[xT] = {}
 
-                // Skip already loaded tiles or retry failed ones
-                if (this.cache[xT][yT] && this.cache[xT][yT] !== 'failed') {
-                    ++processedTiles
+                // Skip the tiles loaded, under way, or that failed recently
+                const kept = this.cache[xT][yT]
+                const retry =
+                    kept && kept.failed && !kept.missing && now() - kept.failed >= TiledGrid.retryFailedAfterMs
+                if (kept && !retry) {
+                    // in the view: not to be dropped
+                    if (kept.cells) kept.lastUsed = now()
                     continue
                 }
 
@@ -160,18 +174,40 @@ export class TiledGrid extends Dataset {
                 this.cache[xT][yT] = 'loading'
 
                 tilePromises.push(
-                    this.loadTile(xT, yT, signal)
-                        .then((tile) => {
+                    this.loadTile(xT, yT, signal).then(
+                        (tile) => {
                             this.cache[xT][yT] = tile
+                            this._keep(tile)
 
                             // Check if this is the last tile
-                            const isLastTile = ++processedTiles === totalTiles
+                            // (tiles under way for another request count: the one that ends last is the one that draws)
+                            const isLastTile = !this._anyLoading(xMin, xMax, yMin, yMax)
                             this.checkAndRedraw(tile, isLastTile)
-                        })
-                        .catch(() => {
-                            this.cache[xT][yT] = 'failed'
-                            ++processedTiles
-                        })
+                        },
+                        (error) => {
+                            if (error && error.name === 'AbortError') {
+                                // cancelled, not failed: the next request loads it
+                                delete this.cache[xT][yT]
+                                return
+                            }
+                            // Most of the time a tile that fails is one the server doesn't have (the tiles of a grid are
+                            // often sparse). It is remembered as failed, and not asked for again at every redraw.
+                            // A 404 is final ('missing'). Any other failure is tried again after retryFailedAfterMs: a
+                            // browser doesn't give the status of a response without CORS headers, so a 404 from such a
+                            // server can't be told from a network error.
+                            this.cache[xT][yT] = {
+                                failed: now(),
+                                missing: /^(404|410)\b/.test(error && error.message),
+                            }
+                            // it changes nothing on the map, except when it was waited for to draw
+                            if (
+                                this.onlyDrawWhenAllTilesReady &&
+                                !this._anyLoading(xMin, xMax, yMin, yMax) &&
+                                this.isShown()
+                            )
+                                this.map.redraw()
+                        }
+                    )
                 )
             }
         }
@@ -180,6 +216,25 @@ export class TiledGrid extends Dataset {
         return this
     }
 
+    /**
+     * Tells if some tiles of a tiling envelope are still being loaded, by this request or any other one.
+     *
+     * @param {number} xMin
+     * @param {number} xMax
+     * @param {number} yMin
+     * @param {number} yMax
+     * @returns {boolean}
+     * @private
+     */
+    _anyLoading(xMin, xMax, yMin, yMax) {
+        for (let xT = xMin; xT <= xMax; xT++) {
+            const column = this.cache[xT]
+            if (!column) continue
+            for (let yT = yMin; yT <= yMax; yT++) if (column[yT] === 'loading') return true
+        }
+        return false
+    }
+
     /**
      * Load a tile.
      *
@@ -220,19 +275,59 @@ export class TiledGrid extends Dataset {
         // 1. the dataset belongs to a layer which is visible at the current zoom level
         if (!this.isShown()) return
 
+        // Waiting for all the tiles: drawn when the last one arrives, wherever it is (the tiles asked for are those of the
+        // view and of a margin around it, so the last one to arrive may be outside the view)
+        if (this.onlyDrawWhenAllTilesReady) {
+            if (isLastTile) this.map.redraw()
+            return
+        }
+
         // Check if tile intersects the current view
         const env = this.map.updateExtentGeo()
         const { xMin, xMax, yMin, yMax } = tile.extGeo
         if (env.xMax <= xMin || env.xMin >= xMax || env.yMax <= yMin || env.yMin >= yMax) return
 
         // Trigger redraw
-        if (this.onlyDrawWhenAllTilesReady) {
-            if (isLastTile) {
-                this.map.redraw()
-            }
-        } else {
-            this.map.redraw()
+        this.map.redraw()
+    }
+
+    /**
+     * Counts a tile that was loaded in the cells kept, and drops the tiles used the longest time ago when there are too many.
+     *
+     * @param {any} tile
+     * @private
+     */
+    _keep(tile) {
+        // a failed tile is not a tile
+        if (!tile || typeof tile !== 'object' || !tile.cells) return
+        tile.lastUsed = now()
+        tile.owner = this
+        keptTiles.add(tile)
+        keptCells += tile.cells.length
+        if (keptCells <= TiledGrid.maxCachedCells) return
+        const limit = now() - KEEP_MS
+        const unused = [...keptTiles].filter((t) => t.lastUsed < limit).sort((a, b) => a.lastUsed - b.lastUsed)
+        for (const t of unused) {
+            if (keptCells <= TiledGrid.maxCachedCells) break
+            t.owner._drop(t)
+        }
+    }
+
+    /**
+     * Forgets a tile: it is loaded again when the map needs it.
+     *
+     * @param {any} tile
+     * @private
+     */
+    _drop(tile) {
+        const column = this.cache[tile.x]
+        if (column && column[tile.y] === tile) {
+            delete column[tile.y]
+            if (!Object.keys(column).length) delete this.cache[tile.x]
         }
+        if (keptTiles.delete(tile)) keptCells -= tile.cells.length
+        tile.owner = undefined
+        tile.cells = []
     }
 
     /**
@@ -263,7 +358,8 @@ export class TiledGrid extends Dataset {
                 //get tile
                 /** @type {object} */
                 const tile = this.cache[xT][yT]
-                if (!tile || typeof tile === 'string') continue
+                if (!tile || !tile.cells) continue
+                tile.lastUsed = now()
 
                 //get cells
                 //this.cellsViewCache = this.cellsViewCache.concat(tile.cells)
@@ -280,6 +376,22 @@ export class TiledGrid extends Dataset {
     }
 }
 
+/**
+ * The most cells kept in memory by the tiles of all the TiledGrid together (a cell is an object of the values of a place, some hundred
+ * bytes: about 250 MB for a million). It can be changed at any time.
+ *
+ * @type {number}
+ */
+TiledGrid.maxCachedCells = 1500000
+
+/**
+ * How long, in ms, a tile that failed to load is remembered, before it is requested again. It doesn't apply to a tile the
+ * server answered 404 or 410 for: that one isn't requested again. It can be changed at any time.
+ *
+ * @type {number}
+ */
+TiledGrid.retryFailedAfterMs = 30000
+
 function getGridTile(cells, xT, yT, gridInfo) {
     const tile = {}
 

package/src/layer/GridLayer.js

@@ -1,194 +1,195 @@
-//@ts-check
-'use strict'
-
-import { Layer } from '../core/Layer.js'
-
-/**
- * A layer, which specifies a dataset to be shown with specified styles.
- *
- * @module layer
- * @author Joseph Davies, Julien Gaffuri
- */
-export class GridLayer extends Layer {
-    /**
-     * @param {import("../core/Dataset").Dataset|import("../core/MultiResolutionDataset").MultiResolutionDataset} dataset The dataset to show.
-     * @param {Array.<import("../core/Style").Style>} styles The styles, ordered in drawing order.
-     * @param {{visible?:function(number):boolean,alpha?:function(number):number,blendOperation?:function(number):GlobalCompositeOperation,minPixelsPerCell?:number,cellInfoHTML?:'none'|function(import("../core/Dataset").Cell):string}} opts
-     */
-    constructor(dataset, styles, opts = {}) {
-        super(opts)
-        opts = opts || {}
-
-        /** @type {import("../core/Dataset").Dataset|import("../core/MultiResolutionDataset").MultiResolutionDataset} */
-        this.dataset = dataset
-
-        /** @type {Array.<import("../core/Style").Style>} */
-        this.styles = styles
-
-        /**
-         * This parameter is used when the dataset is a MultiResolutionDataset.
-         * It defines the minimum number of pixels a grid cell should have to select the dataset to display based on its resolution.
-         * A low value, means that the map will be more detailled (smaller cells).
-         * A high value, means that the map will be less detailled (larger cells).
-         * This value should be higher than 1, otherwise it means a grid cell is smaller than the screen resolution.
-         * For more complex cell representations that require some more map space, this value should be higher.
-         * @type {number} */
-        this.minPixelsPerCell = opts.minPixelsPerCell || 3
-
-        /**
-         * The function returning cell information as HTML.
-         * This is typically used for tooltip information.
-         * Set to 'none' to disable cell info.
-         * @type {'none'|function(import("../core/Dataset").Cell, number):string} */
-        this.cellInfoHTML = opts.cellInfoHTML || GridLayer.defaultCellInfoHTML
-    }
-
-    /** */
-    draw(geoCanvas, legend) {
-        //get zoom level
-        const z = geoCanvas.view.z
-        const ctx = geoCanvas.offscreenCtx
-
-        //get layer dataset component
-        /** @type {import('../core/Dataset.js').Dataset|undefined} */
-        const dsc = this.getDataset(z)
-        if (!dsc) return
-
-        //launch data download, if necessary
-        dsc.getData(geoCanvas.extGeo)
-
-        //update dataset view cache
-        dsc.updateViewCache(geoCanvas.extGeo)
-
-
-        //set layer alpha and blend mode
-        if (this.alpha || this.blendOperation) {
-            ctx.save()
-            if (this.alpha) ctx.globalAlpha = this.alpha(z)
-            if (this.blendOperation) ctx.globalCompositeOperation = this.blendOperation(z)
-        }
-
-        // tell the legends where they are drawn, so that they can fit it (width:'fill')
-        if (legend) for (const s of this.styles) GridLayer.setLegendContainer(legend.node(), s.legends, s['styles'])
-
-        //draw cells, style by style
-        for (const s of this.styles) {
-            //check if style is visible
-            if (s.visible && !s.visible(z)) continue
-
-            // set style alpha and blend mode
-            //TODO: multiply style alpha by layer alpha ?
-            if (s.alpha || s.blendOperation) {
-                ctx.save()
-                if (s.alpha) ctx.globalAlpha = s.alpha(z)
-                if (s.blendOperation) ctx.globalCompositeOperation = s.blendOperation(z)
-            }
-
-            //set affin transform to draw with geographical coordinates
-            geoCanvas.setCanvasTransform()
-
-            //draw with style
-            s.draw(dsc.getViewCache(), geoCanvas, dsc.getResolution())
-
-            //draw style filter
-            if (s.filterColor) s.drawFilter(geoCanvas)
-
-            //restore default alpha and blend operation - before style change
-            if (s.alpha || s.blendOperation) ctx.restore()
-        }
-
-        //restore default alpha and blend operation - before layer change
-        if (this.alpha || this.blendOperation) ctx.restore()
-
-        //add legend element
-        if (legend) {
-            for (const s of this.styles) {
-                //check if style is visible
-                if (s.visible && !s.visible(z)) continue
-                GridLayer.addLegends(legend, s.legends, s, geoCanvas)
-
-                //case for styles of styles, like kernel smoothing
-                //TODO do better
-                if (s['styles']) {
-                    for (const s2 of s['styles']) {
-                        if (s2.visible && !s2.visible(z)) continue
-                        GridLayer.addLegends(legend, s2.legends, s2, geoCanvas)
-                    }
-                }
-            }
-        }
-    }
-
-    /** @private */
-    static setLegendContainer(container, legends, subStyles) {
-        // legends can be nested in arrays, like in Style.updateLegendsRecursive
-        const set = (lg) => {
-            if (Array.isArray(lg)) lg.forEach(set)
-            else if (lg) lg.container = container
-        }
-        set(legends)
-        for (const s2 of subStyles || []) GridLayer.setLegendContainer(container, s2.legends, undefined)
-    }
-
-    /** @private */
-    static addLegends(legendComp, lg, style, geoCanvas) {
-        if (Array.isArray(lg)) for (const lg_ of lg) this.addLegends(legendComp, lg_, style, geoCanvas)
-        else {
-            // so that hovering a swatch can highlight the cells of the style on the map
-            lg.style_ = style
-            lg.redraw = () => (geoCanvas.redraw ? geoCanvas.redraw() : undefined)
-            // The legends go in the order they are drawn, after the one placed before them. A hovered legend is
-            // still in the container (see Map.redrawNow) and is left where it is: moving it would lose its mouseleave,
-            // and appending the others after it would put it above the legends that come before it.
-            const container = legendComp.node()
-            const el = lg.div.node()
-            const previous = container._gridvizLegendCursor
-            if (previous) {
-                if (el.previousSibling !== previous) previous.after(el)
-            } else if (container.firstChild !== el) container.prepend(el)
-            container._gridvizLegendCursor = el
-        }
-    }
-
-    /**
-     * Return the relevant dataset component for a specified zoom.
-     *
-     * @param {number} z
-     * @returns {import("../core/Dataset").Dataset|undefined}
-     * */
-    getDataset(z) {
-        return this.dataset.getDataset(z, this.minPixelsPerCell)
-    }
-
-    /**
-     * Set/get style stack.
-     *
-     * @param {undefined|import("../core/Style").Style|Array.<import("../core/Style").Style>} styles
-     * @returns { this | Array.<import("../core/Style").Style> }
-     */
-    styles_(styles) {
-        if (arguments.length === 0) return this.styles
-        if (arguments.length === 1)
-            if (Array.isArray(styles)) this.styles = styles
-            else this.styles = [styles]
-        else this.styles = arguments
-        return this
-    }
-
-    /**
-     * The default function returning cell information as HTML.
-     * This is typically used for tooltip information.
-     *
-     * @param {import("../core/Dataset").Cell} cell
-     * @returns {string}
-     */
-    static defaultCellInfoHTML(cell) {
-        const buf = []
-        for (const key of Object.keys(cell)) {
-            if (key === 'x') continue
-            if (key === 'y') continue
-            buf.push('<b>', key, '</b>', ' : ', cell[key], '<br>')
-        }
-        return buf.join('')
-    }
-}
+//@ts-check
+'use strict'
+
+import { Layer } from '../core/Layer.js'
+import { isVisibleAt } from '../core/Drawable.js'
+
+/**
+ * A layer, which specifies a dataset to be shown with specified styles.
+ *
+ * @module layer
+ * @author Joseph Davies, Julien Gaffuri
+ */
+export class GridLayer extends Layer {
+    /**
+     * @param {import("../core/Dataset").Dataset|import("../core/MultiResolutionDataset").MultiResolutionDataset} dataset The dataset to show.
+     * @param {Array.<import("../core/Style").Style>} styles The styles, ordered in drawing order.
+     * @param {{visible?:function(number):boolean,alpha?:function(number):number,blendOperation?:function(number):GlobalCompositeOperation,minPixelsPerCell?:number,cellInfoHTML?:'none'|function(import("../core/Dataset").Cell):string}} opts
+     */
+    constructor(dataset, styles, opts = {}) {
+        super(opts)
+        opts = opts || {}
+
+        /** @type {import("../core/Dataset").Dataset|import("../core/MultiResolutionDataset").MultiResolutionDataset} */
+        this.dataset = dataset
+
+        /** @type {Array.<import("../core/Style").Style>} */
+        this.styles = styles
+
+        /**
+         * This parameter is used when the dataset is a MultiResolutionDataset.
+         * It defines the minimum number of pixels a grid cell should have to select the dataset to display based on its resolution.
+         * A low value, means that the map will be more detailled (smaller cells).
+         * A high value, means that the map will be less detailled (larger cells).
+         * This value should be higher than 1, otherwise it means a grid cell is smaller than the screen resolution.
+         * For more complex cell representations that require some more map space, this value should be higher.
+         * @type {number} */
+        this.minPixelsPerCell = opts.minPixelsPerCell || 3
+
+        /**
+         * The function returning cell information as HTML.
+         * This is typically used for tooltip information.
+         * Set to 'none' to disable cell info.
+         * @type {'none'|function(import("../core/Dataset").Cell, number):string} */
+        this.cellInfoHTML = opts.cellInfoHTML || GridLayer.defaultCellInfoHTML
+    }
+
+    /** */
+    draw(geoCanvas, legend) {
+        //get zoom level
+        const z = geoCanvas.view.z
+        const ctx = geoCanvas.offscreenCtx
+
+        //get layer dataset component
+        /** @type {import('../core/Dataset.js').Dataset|undefined} */
+        const dsc = this.getDataset(z)
+        if (!dsc) return
+
+        //launch data download, if necessary
+        dsc.getData(geoCanvas.extGeo)
+
+        //update dataset view cache
+        dsc.updateViewCache(geoCanvas.extGeo)
+
+
+        //set layer alpha and blend mode
+        if (this.alpha || this.blendOperation) {
+            ctx.save()
+            if (this.alpha) ctx.globalAlpha = this.alpha(z)
+            if (this.blendOperation) ctx.globalCompositeOperation = this.blendOperation(z)
+        }
+
+        // tell the legends where they are drawn, so that they can fit it (width:'fill')
+        if (legend) for (const s of this.styles) GridLayer.setLegendContainer(legend.node(), s.legends, s['styles'])
+
+        //draw cells, style by style
+        for (const s of this.styles) {
+            //check if style is visible
+            if (!isVisibleAt(s, z)) continue
+
+            // set style alpha and blend mode
+            //TODO: multiply style alpha by layer alpha ?
+            if (s.alpha || s.blendOperation) {
+                ctx.save()
+                if (s.alpha) ctx.globalAlpha = s.alpha(z)
+                if (s.blendOperation) ctx.globalCompositeOperation = s.blendOperation(z)
+            }
+
+            //set affin transform to draw with geographical coordinates
+            geoCanvas.setCanvasTransform()
+
+            //draw with style
+            s.draw(dsc.getViewCache(), geoCanvas, dsc.getResolution())
+
+            //draw style filter
+            if (s.filterColor) s.drawFilter(geoCanvas)
+
+            //restore default alpha and blend operation - before style change
+            if (s.alpha || s.blendOperation) ctx.restore()
+        }
+
+        //restore default alpha and blend operation - before layer change
+        if (this.alpha || this.blendOperation) ctx.restore()
+
+        //add legend element
+        if (legend) {
+            for (const s of this.styles) {
+                //check if style is visible
+                if (!isVisibleAt(s, z)) continue
+                GridLayer.addLegends(legend, s.legends, s, geoCanvas)
+
+                //case for styles of styles, like kernel smoothing
+                //TODO do better
+                if (s['styles']) {
+                    for (const s2 of s['styles']) {
+                        if (!isVisibleAt(s2, z)) continue
+                        GridLayer.addLegends(legend, s2.legends, s2, geoCanvas)
+                    }
+                }
+            }
+        }
+    }
+
+    /** @private */
+    static setLegendContainer(container, legends, subStyles) {
+        // legends can be nested in arrays, like in Style.updateLegendsRecursive
+        const set = (lg) => {
+            if (Array.isArray(lg)) lg.forEach(set)
+            else if (lg) lg.container = container
+        }
+        set(legends)
+        for (const s2 of subStyles || []) GridLayer.setLegendContainer(container, s2.legends, undefined)
+    }
+
+    /** @private */
+    static addLegends(legendComp, lg, style, geoCanvas) {
+        if (Array.isArray(lg)) for (const lg_ of lg) this.addLegends(legendComp, lg_, style, geoCanvas)
+        else {
+            // so that hovering a swatch can highlight the cells of the style on the map
+            lg.style_ = style
+            lg.redraw = () => (geoCanvas.redraw ? geoCanvas.redraw() : undefined)
+            // The legends go in the order they are drawn, after the one placed before them. A hovered legend is
+            // still in the container (see Map.redrawNow) and is left where it is: moving it would lose its mouseleave,
+            // and appending the others after it would put it above the legends that come before it.
+            const container = legendComp.node()
+            const el = lg.div.node()
+            const previous = container._gridvizLegendCursor
+            if (previous) {
+                if (el.previousSibling !== previous) previous.after(el)
+            } else if (container.firstChild !== el) container.prepend(el)
+            container._gridvizLegendCursor = el
+        }
+    }
+
+    /**
+     * Return the relevant dataset component for a specified zoom.
+     *
+     * @param {number} z
+     * @returns {import("../core/Dataset").Dataset|undefined}
+     * */
+    getDataset(z) {
+        return this.dataset.getDataset(z, this.minPixelsPerCell)
+    }
+
+    /**
+     * Set/get style stack.
+     *
+     * @param {undefined|import("../core/Style").Style|Array.<import("../core/Style").Style>} styles
+     * @returns { this | Array.<import("../core/Style").Style> }
+     */
+    styles_(styles) {
+        if (arguments.length === 0) return this.styles
+        if (arguments.length === 1)
+            if (Array.isArray(styles)) this.styles = styles
+            else this.styles = [styles]
+        else this.styles = arguments
+        return this
+    }
+
+    /**
+     * The default function returning cell information as HTML.
+     * This is typically used for tooltip information.
+     *
+     * @param {import("../core/Dataset").Cell} cell
+     * @returns {string}
+     */
+    static defaultCellInfoHTML(cell) {
+        const buf = []
+        for (const key of Object.keys(cell)) {
+            if (key === 'x') continue
+            if (key === 'y') continue
+            buf.push('<b>', key, '</b>', ' : ', cell[key], '<br>')
+        }
+        return buf.join('')
+    }
+}