@signalk/charts-plugin 3.6.2 → 3.8.0

package/plugin/chartDownloader.js

@@ -3,38 +3,98 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ChartDownloader = exports.ChartSeedingManager = exports.Status = void 0;
-const fs_1 = __importDefault(require("fs"));
+exports.ChartDownloader = exports.ChartSeedingManager = exports.FailureKind = exports.TileSource = exports.State = void 0;
 const path_1 = __importDefault(require("path"));
-const p_limit_1 = __importDefault(require("p-limit"));
-const geojson_antimeridian_cut_1 = __importDefault(require("geojson-antimeridian-cut"));
-const boolean_intersects_1 = __importDefault(require("@turf/boolean-intersects"));
-const bbox_1 = require("@turf/bbox");
-const helpers_1 = require("@turf/helpers");
 const check_disk_space_1 = __importDefault(require("check-disk-space"));
 const projection_1 = require("./projection");
-const tileServer_1 = require("./tileServer");
-var Status;
-(function (Status) {
-    Status[Status["Stopped"] = 0] = "Stopped";
-    Status[Status["Running"] = 1] = "Running";
-})(Status || (exports.Status = Status = {}));
+const cacheLayout_1 = require("./cacheLayout");
+// All DB-touching helpers (delete / purge / vacuum / region-tile listing) need
+// the raw node:sqlite handle exposed by @signalk/mbtiles. Reaching through
+// `provider._mbtilesHandle!._db!` worked but lied about the contract: a proxy
+// provider whose mbtiles file failed to open at startup has neither, and the
+// non-null assertions papered over the resulting TypeError. Centralising here
+// gives a single descriptive failure surface and removes seven `!` operators
+// from call sites.
+function requireMbtilesDb(provider) {
+    const db = provider._mbtilesHandle?._db;
+    if (!db) {
+        throw new Error(`Provider "${provider.identifier}": mbtiles cache not open. ` +
+            `Configure the provider with proxy:true and ensure the cache path ` +
+            `is writable; this operation needs the raw SQLite handle.`);
+    }
+    return db;
+}
+const chartDownloaderTileHelpers_1 = require("./chartDownloaderTileHelpers");
+const chartDownloaderMBTilesHelpers_1 = require("./chartDownloaderMBTilesHelpers");
+var State;
+(function (State) {
+    State[State["Stopped"] = 0] = "Stopped";
+    State[State["Running"] = 1] = "Running";
+})(State || (exports.State = State = {}));
+var JobType;
+(function (JobType) {
+    JobType[JobType["None"] = 0] = "None";
+    JobType[JobType["Seed"] = 1] = "Seed";
+    JobType[JobType["Delete"] = 2] = "Delete";
+})(JobType || (JobType = {}));
+var TileSource;
+(function (TileSource) {
+    TileSource[TileSource["None"] = 0] = "None";
+    TileSource[TileSource["FetchedFromCache"] = 1] = "FetchedFromCache";
+    TileSource[TileSource["FetchedFromRemote"] = 2] = "FetchedFromRemote";
+})(TileSource || (exports.TileSource = TileSource = {}));
+// Categorised failure kinds for tile fetches. Lets the worker keep
+// per-job counts so an admin staring at "500000 failed" can tell whether
+// the upstream is timing out, the auth is wrong, or the network is gone.
+// Used in TileFetchResult.failure and surfaced via job.info().failures.
+var FailureKind;
+(function (FailureKind) {
+    FailureKind["None"] = "none";
+    FailureKind["Timeout"] = "timeout";
+    FailureKind["HttpClient"] = "http_4xx";
+    FailureKind["HttpServer"] = "http_5xx";
+    FailureKind["Network"] = "network";
+    FailureKind["NoRemoteUrl"] = "no_remote_url";
+})(FailureKind || (exports.FailureKind = FailureKind = {}));
+// Default tile-fetch timeout. Bumped from 5s to 10s — some legitimate
+// tile providers (NOAA WMS, custom inland-water sources) take 5-15s on
+// a cold-cache backend tile and a 5s default marked them as failures.
+// Per-provider override available via OnlineChartProvider.timeoutMs and
+// TokenProviderConfig.tile.timeoutMs (added on the same change).
+const TILE_FETCH_TIMEOUT_MS = 10000;
+// Default backoff before a single retry on a transient 5xx (502/503).
+// Picked to be long enough that an upstream momentary blip clears, short
+// enough that 32 workers don't all stall on a hard outage. Configurable
+// at runtime via ChartDownloader.tileRetryBackoffMs (test harness sets
+// it to 0 to keep mocked-503 tests fast).
+const DEFAULT_TILE_RETRY_BACKOFF_MS = 1500;
 class ChartSeedingManager {
     static ActiveJobs = {};
-    static async createJob(resourcesApi, chartsPath, provider, maxZoom, regionGUID = undefined, bbox = undefined, tile = undefined) {
-        const downloader = new ChartDownloader(resourcesApi, chartsPath, provider);
-        // Init must complete before the job is usable; callers get back a job that
-        // knows its tile set and totalTiles. Without awaiting, a follow-up "start"
-        // action would race the init reads of this.tiles.
-        if (regionGUID)
-            await downloader.initializeJobFromRegion(regionGUID, maxZoom);
-        else if (bbox)
-            await downloader.initializeJobFromBBox(bbox, maxZoom);
-        else if (tile)
-            await downloader.initializeJobFromTile(tile, maxZoom);
-        else
-            throw new Error('createJob requires regionGUID, bbox, or tile');
+    /**
+     * Reset all module-level static state owned by ChartSeedingManager and
+     * ChartDownloader so a test harness can run from a known baseline.
+     *
+     * The plugin's static fields outlive a `stop()` / `start()` cycle in
+     * production (which is intended — process-global gates and ID sequences),
+     * but in a test process running 200+ tests in sequence the residue
+     * shows up as cross-test interference: nextJobId drifts upward, an
+     * earlier low-disk-space probe leaves CachingEnabled=false for 2s,
+     * stale CacheStatistics from one provider get seen by an unrelated
+     * test asserting "stats are an object".
+     *
+     * Production code never calls this. The test app's beforeEach does.
+     */
+    static resetForTests() {
+        this.ActiveJobs = {};
+        ChartDownloader.resetForTests();
+    }
+    static async createJob(cachePath, provider, options, minZoom, maxZoom, feature = undefined, bbox = undefined) {
+        const downloader = new ChartDownloader(cachePath, provider, options);
         this.ActiveJobs[downloader.ID] = downloader;
+        if (feature)
+            downloader.initalizeJobFromFeature(feature, minZoom, maxZoom);
+        else if (bbox)
+            downloader.initializeJobFromBBox(bbox, minZoom, maxZoom);
         return downloader;
     }
     // Cancel all running jobs and clear the registry. Used when the plugin stops
@@ -48,206 +108,389 @@ class ChartSeedingManager {
 }
 exports.ChartSeedingManager = ChartSeedingManager;
 class ChartDownloader {
-    resourcesApi;
-    chartsPath;
+    cachePath;
     provider;
+    options;
     static MINIMUM_FREE_DISK_SPACE = 1024 * 1024 * 1024; // 1 GB
     static nextJobId = 1;
+    static CachingEnabled = true;
+    static CacheStatistics = new Map();
+    static TilesCached = 0;
+    static lastDiskSpaceCheck = 0;
+    static lastDiskSpaceResult = true;
+    // See DEFAULT_TILE_RETRY_BACKOFF_MS. Public so the test harness can drop
+    // it to 0; production code should treat it as read-only.
+    static tileRetryBackoffMs = DEFAULT_TILE_RETRY_BACKOFF_MS;
     id = ChartDownloader.nextJobId++;
-    status = Status.Stopped;
+    type = JobType.None;
+    state = State.Stopped;
+    status = 'Idle';
     totalTiles = 0;
     downloadedTiles = 0;
     failedTiles = 0;
     cachedTiles = 0;
-    concurrentDownloadsLimit = 20;
+    deletedTiles = 0;
+    // Per-job failure histogram, keyed by FailureKind. Surfaced via
+    // info().failures so the UI / admin can tell timeouts apart from
+    // 5xx apart from auth failures without grepping logs. Only the
+    // first failure of each kind is logged in detail (sampleLogged)
+    // so a job with 500k failed tiles doesn't drown the log buffer.
+    failureCounts = {};
+    sampleLogged = new Set();
     areaDescription = '';
     cancelRequested = false;
-    tiles = [];
-    tilesToDownload = [];
-    constructor(resourcesApi, chartsPath, provider) {
-        this.resourcesApi = resourcesApi;
-        this.chartsPath = chartsPath;
+    regionName = 'Unnamed';
+    tiles;
+    tilesInDB;
+    constructor(cachePath, provider, options) {
+        this.cachePath = cachePath;
         this.provider = provider;
+        this.options = options;
     }
     get ID() {
         return this.id;
     }
-    async initializeJobFromRegion(regionGUID, maxZoom) {
-        const region = (await this.resourcesApi.getResource('regions', regionGUID));
-        const geojson = this.convertRegionToGeoJSON(region);
-        this.tiles = this.getTilesForGeoJSON(geojson, this.provider.minzoom, maxZoom);
-        this.tilesToDownload = await this.filterCachedTiles(this.tiles);
-        this.status = Status.Stopped;
-        this.totalTiles = this.tiles.length;
-        this.cachedTiles = this.totalTiles - this.tilesToDownload.length;
-        this.areaDescription = `Region: ${region?.name ?? ''}`;
-    }
-    async initializeJobFromBBox(bbox, maxZoom) {
-        this.tiles = this.getTilesForBBox(bbox, maxZoom);
-        this.tilesToDownload = await this.filterCachedTiles(this.tiles);
-        this.status = Status.Stopped;
-        this.totalTiles = this.tiles.length;
-        this.cachedTiles = this.totalTiles - this.tilesToDownload.length;
-        this.areaDescription = `BBox: [${bbox.join(', ')}]`;
+    async initalizeJobFromFeature(feature, minZoom, maxZoom) {
+        const geojson = (0, chartDownloaderTileHelpers_1.convertFeatureToGeoJSON)(feature);
+        this.tiles = () => (0, chartDownloaderTileHelpers_1.getTilesForGeoJSON)(geojson, minZoom, maxZoom);
+        // Defer the requireMbtilesDb call to factory invocation: the handle may
+        // not be open yet when initialize* runs (e.g. createJob before the
+        // proxy mbtiles is reconciled), but it must be open by the time the
+        // generator is iterated.
+        this.tilesInDB = () => (0, chartDownloaderMBTilesHelpers_1.getMBTilesForPolygon)(requireMbtilesDb(this.provider), geojson, minZoom, maxZoom);
+        this.totalTiles = (0, chartDownloaderTileHelpers_1.countTilesAdaptiveIterative)(geojson, minZoom, maxZoom);
+        this.state = State.Stopped;
+        this.regionName = feature?.properties?.name ?? 'Unnamed';
+        this.areaDescription = `Region: ${this.regionName}`;
     }
-    async initializeJobFromTile(tile, maxZoom) {
-        this.tiles = this.getSubTiles(tile, maxZoom);
-        this.tilesToDownload = await this.filterCachedTiles(this.tiles);
-        this.status = Status.Stopped;
-        this.totalTiles = this.tiles.length;
-        this.cachedTiles = this.totalTiles - this.tilesToDownload.length;
-        this.areaDescription = `Tile: [${tile.x}, ${tile.y}, ${tile.z}]`;
+    async initializeJobFromBBox(bbox, minZoom, maxZoom) {
+        const geojson = (0, chartDownloaderTileHelpers_1.convertBboxToGeoJSON)(bbox);
+        this.tiles = () => (0, chartDownloaderTileHelpers_1.getTilesForGeoJSON)(geojson, minZoom, maxZoom);
+        this.tilesInDB = () => (0, chartDownloaderMBTilesHelpers_1.getMBTilesForPolygon)(requireMbtilesDb(this.provider), geojson, minZoom, maxZoom);
+        this.totalTiles = (0, chartDownloaderTileHelpers_1.countTilesAdaptiveIterative)(geojson, minZoom, maxZoom);
+        this.state = State.Stopped;
+        this.areaDescription = `BBox: [${bbox
+            .map((v) => Number(v).toFixed(3))
+            .join(', ')}]`;
     }
-    static DISK_CHECK_INTERVAL_MS = 30_000;
     /**
      * Download map tiles for a specific area.
      */
     async seedCache() {
         // Guard against double-start: a second call while Running would share
         // counters and concurrency slots with the first and corrupt progress.
-        if (this.status === Status.Running)
+        if (this.state === State.Running)
             return;
+        const concurrency = 32;
         this.cancelRequested = false;
-        this.status = Status.Running;
-        this.tilesToDownload = await this.filterCachedTiles(this.tiles);
+        this.state = State.Running;
         this.downloadedTiles = 0;
+        this.cachedTiles = 0;
         this.failedTiles = 0;
-        this.cachedTiles = this.totalTiles - this.tilesToDownload.length;
-        const limit = (0, p_limit_1.default)(this.concurrentDownloadsLimit); // concurrent download limit
-        let lastDiskCheck = 0;
-        const tasks = this.tilesToDownload.map((tile) => limit(async () => {
-            if (this.cancelRequested)
-                return;
-            // Time-based (rather than tile-count-based) disk-space probing: a
-            // tight per-1000-tile cadence fired hundreds of times on a large
-            // bbox, whereas real disk consumption grows with wall-clock time.
-            const now = Date.now();
-            if (now - lastDiskCheck >= ChartDownloader.DISK_CHECK_INTERVAL_MS) {
-                lastDiskCheck = now;
+        this.failureCounts = {};
+        this.sampleLogged = new Set();
+        this.type = JobType.Seed;
+        console.log(`[charts-plugin] job ${this.id} started seeding ` +
+            `provider=${this.provider.identifier} ` +
+            `area="${this.areaDescription}" totalTiles=${this.totalTiles}`);
+        const tileIterator = this.tiles();
+        let generatorDone = false;
+        const nextTile = () => {
+            if (generatorDone || this.cancelRequested)
+                return null;
+            const { value, done } = tileIterator.next();
+            if (done) {
+                generatorDone = true;
+                return null;
+            }
+            return value;
+        };
+        const worker = async () => {
+            while (!this.cancelRequested) {
+                const tile = nextTile();
+                if (!tile)
+                    return;
+                if (!ChartDownloader.CachingEnabled) {
+                    this.cancelRequested = true;
+                    this.status = 'Insufficient disk space, canceling job';
+                    return;
+                }
                 try {
-                    const { free } = await (0, check_disk_space_1.default)(this.chartsPath);
-                    if (free < ChartDownloader.MINIMUM_FREE_DISK_SPACE) {
-                        console.warn(`Low disk space. Stopping download.`);
-                        this.cancelRequested = true;
-                        return;
+                    const result = await ChartDownloader.getTileFromCacheOrRemote(this.cachePath, this.provider, tile, this.options.refetch);
+                    if (result.source === TileSource.None) {
+                        this.failedTiles++;
+                        const kind = result.failure?.kind ?? FailureKind.Network;
+                        this.failureCounts[kind] = (this.failureCounts[kind] ?? 0) + 1;
+                        // Log the first instance of each failure kind in this job —
+                        // gives an operator something to grep for ("job 42 timeout
+                        // first at z=12 x=2057 y=1364 status=undefined") without
+                        // 500k log lines if every tile fails.
+                        if (!this.sampleLogged.has(kind)) {
+                            this.sampleLogged.add(kind);
+                            console.warn(`[charts-plugin] job ${this.id} ` +
+                                `provider=${this.provider.identifier} ` +
+                                `first ${kind} failure at z=${tile.z} x=${tile.x} y=${tile.y}` +
+                                (result.failure?.status
+                                    ? ` status=${result.failure.status}`
+                                    : ''));
+                        }
+                    }
+                    else if (result.source === TileSource.FetchedFromCache) {
+                        this.cachedTiles++;
+                    }
+                    else {
+                        this.downloadedTiles++;
                     }
+                    this.totalTiles = Math.max(this.totalTiles, this.downloadedTiles + this.failedTiles + this.cachedTiles);
                 }
                 catch (err) {
-                    console.error(`Error checking disk space:`, err);
-                    this.cancelRequested = true;
-                    return;
+                    this.failedTiles++;
+                    const msg = err.message ?? String(err);
+                    this.failureCounts[FailureKind.Network] =
+                        (this.failureCounts[FailureKind.Network] ?? 0) + 1;
+                    if (!this.sampleLogged.has(FailureKind.Network)) {
+                        this.sampleLogged.add(FailureKind.Network);
+                        console.warn(`[charts-plugin] job ${this.id} ` +
+                            `provider=${this.provider.identifier} ` +
+                            `first uncaught failure at z=${tile.z} x=${tile.x} y=${tile.y}: ${msg}`);
+                    }
                 }
             }
-            const buffer = await ChartDownloader.getTileFromCacheOrRemote(this.chartsPath, this.provider, tile);
-            // Re-check after the await: the job may have been cancelled while the
-            // fetch was in flight. Still-running fetches would otherwise keep
-            // mutating counters after status flips to Stopped.
-            if (this.cancelRequested)
-                return;
-            if (buffer === null) {
-                this.failedTiles++;
-            }
-            else {
-                this.downloadedTiles++;
+        };
+        this.status = 'Seeding';
+        const workers = Array.from({ length: concurrency }, () => worker());
+        await Promise.all(workers);
+        this.state = State.Stopped;
+        if (this.options.mbtiles) {
+            try {
+                this.status = 'Creating MBTiles';
+                const safeRegionName = this.regionName
+                    .normalize('NFKC') // normalize unicode
+                    .replace(/[^a-zA-Z0-9_-]/g, '_') // allow only safe chars
+                    .replace(/_+/g, '_') // collapse repeats
+                    .replace(/^_+|_+$/g, '') // trim underscores
+                    .slice(0, 100); // limit length
+                // Snapshot exports go to ${cachePath}/exports/, which is excluded from
+                // the chart scanner — the file is a build artifact for offline transfer
+                // (USB stick to another SK server), not a chart the running plugin
+                // serves. Re-running the job overwrites the file freely; no scanner
+                // handle holds it open.
+                const baseDir = path_1.default.resolve(this.cachePath, cacheLayout_1.EXPORTS_DIR);
+                const filePath = path_1.default.resolve((0, cacheLayout_1.exportMbtilesPath)(this.cachePath, safeRegionName, this.provider.identifier));
+                // Same separator-aware check as the migrate endpoint: a naive
+                // startsWith would miss sibling-directory prefix collisions.
+                if (filePath !== baseDir && !filePath.startsWith(baseDir + path_1.default.sep)) {
+                    throw new Error('Invalid path detected');
+                }
+                // The export copies the whole seeded set into a second mbtiles, so it
+                // needs roughly as much free space again. Bail before opening the file
+                // rather than filling the disk and corrupting the working cache.
+                if (!(await ChartDownloader.hasDiskSpace(baseDir))) {
+                    throw new Error('Insufficient disk space for mbtiles export');
+                }
+                const mbtiles = await (0, chartDownloaderMBTilesHelpers_1.openOrCreateMbtiles)(filePath, this.provider);
+                const iterator = this.tiles();
+                for (const tile of iterator) {
+                    const buffer = await ChartDownloader.getTileFromMbTiles(this.provider._mbtilesHandle, tile);
+                    if (buffer) {
+                        await ChartDownloader.cacheTileToMbTiles(mbtiles, tile, buffer);
+                    }
+                }
+                mbtiles._db?.exec('PRAGMA wal_checkpoint(TRUNCATE);');
             }
-        }));
-        // allSettled ensures every in-flight task completes before we flip back
-        // to Stopped; Promise.all would resolve on the first rejection while
-        // other tasks still incremented counters in the background.
-        const results = await Promise.allSettled(tasks);
-        for (const r of results) {
-            if (r.status === 'rejected') {
-                console.error('Error downloading tile:', r.reason);
+            catch (err) {
+                // The tile seed itself already succeeded (state is Stopped); only the
+                // optional export failed. Record it and rethrow so the HTTP handler's
+                // .catch() can log it without an unhandled rejection crashing the
+                // server.
+                this.status = `Failed: ${err.message}`;
+                console.error(`[charts-plugin] job ${this.id} mbtiles export failed: ${err.message}`);
+                throw err;
             }
         }
-        this.status = Status.Stopped;
+        this.status = 'Completed';
+        console.log(`[charts-plugin] job ${this.id} completed ` +
+            `provider=${this.provider.identifier} ` +
+            `downloaded=${this.downloadedTiles} cached=${this.cachedTiles} ` +
+            `failed=${this.failedTiles} ` +
+            `failures=${JSON.stringify(this.failureCounts)}`);
     }
     async deleteCache() {
-        this.status = Status.Running;
-        for (const tile of this.tiles) {
-            if (this.cancelRequested)
-                break;
-            const tilePath = path_1.default.join(this.chartsPath, `${this.provider.name}`, `${tile.z}`, `${tile.x}`, `${tile.y}.${this.provider.format}`);
-            try {
-                await fs_1.default.promises.unlink(tilePath);
-                this.cachedTiles = Math.max(this.cachedTiles - 1, 0);
-            }
-            catch (err) {
-                if (err.code !== 'ENOENT') {
-                    console.error(`Error deleting cached tile ${tilePath}:`, err);
-                }
+        this.state = State.Running;
+        this.deletedTiles = 0;
+        this.type = JobType.Delete;
+        this.status = 'Deleting tiles';
+        console.log(`[charts-plugin] job ${this.id} delete started ` +
+            `provider=${this.provider.identifier} totalTiles=${this.totalTiles}`);
+        try {
+            const db = requireMbtilesDb(this.provider);
+            await (0, chartDownloaderMBTilesHelpers_1.deleteTilesInChunks)(db, this.tilesInDB(), 1000, (deleted) => {
+                console.log(`[charts-plugin] job ${this.id} deleted ${deleted} / ${this.totalTiles}`);
+            });
+            this.status = 'Purging orphaned images';
+            await (0, chartDownloaderMBTilesHelpers_1.purgeAllOrphanImages)(db, 1000, (deleted, totalDeleted) => {
+                this.deletedTiles += deleted;
+                console.log(`[charts-plugin] job ${this.id} purged ${totalDeleted} orphans ` +
+                    `(last chunk ${deleted})`);
+            });
+            if (this.options.vacuum) {
+                this.status = 'Vacuuming MBTiles database';
+                (0, chartDownloaderMBTilesHelpers_1.vacuumMbtiles)(db);
             }
+            this.status = 'Completed';
+            this.state = State.Stopped;
+            console.log(`[charts-plugin] job ${this.id} delete completed ` +
+                `provider=${this.provider.identifier} ` +
+                `deletedTiles=${this.deletedTiles}`);
+        }
+        catch (err) {
+            // Record the failure on the job and reset state so the job doesn't sit
+            // wedged in 'Running' forever. Rethrow so callers / tests still observe
+            // it — the HTTP handler attaches a .catch() to keep the rejection from
+            // becoming an unhandled rejection that crashes the server.
+            this.state = State.Stopped;
+            this.status = `Failed: ${err.message}`;
+            console.error(`[charts-plugin] job ${this.id} delete failed: ${err.message}`);
+            throw err;
         }
-        this.status = Status.Stopped;
     }
     cancelJob() {
         this.cancelRequested = true;
     }
-    async filterCachedTiles(allTiles) {
-        // Bound the concurrent fs.access calls. 100k+ tiles in a large bbox would
-        // otherwise fire all accesses at once, risking EMFILE on default rlimit
-        // and spiking the event loop.
-        const limit = (0, p_limit_1.default)(64);
-        const checks = allTiles.map((tile) => limit(async () => {
-            const tilePath = path_1.default.join(this.chartsPath, this.provider.name, `${tile.z}`, `${tile.x}`, `${tile.y}.${this.provider.format}`);
-            try {
-                await fs_1.default.promises.access(tilePath); // file exists
-                return null; // filter out cached tile
-            }
-            catch (err) {
-                if (err.code === 'ENOENT') {
-                    return tile; // file does not exist → uncached
+    info() {
+        const progress = () => {
+            if (this.totalTiles > 0) {
+                if (this.type === JobType.Seed) {
+                    return ((this.downloadedTiles + this.cachedTiles + this.failedTiles) /
+                        this.totalTiles);
+                }
+                else if (this.type === JobType.Delete) {
+                    return this.deletedTiles / this.totalTiles;
                 }
-                console.error('Unexpected fs error:', err);
-                return tile; // treat unknown errors as uncached
             }
-        }));
-        const results = await Promise.all(checks);
-        return results.filter((t) => t !== null);
-    }
-    info() {
+            return 0;
+        };
         return {
             id: this.id,
+            type: this.type,
             chartName: this.provider.name,
             regionName: this.areaDescription,
             totalTiles: this.totalTiles,
             downloadedTiles: this.downloadedTiles,
             cachedTiles: this.cachedTiles,
             failedTiles: this.failedTiles,
-            progress: this.totalTiles > 0
-                ? (this.downloadedTiles + this.cachedTiles + this.failedTiles) /
-                    this.totalTiles
-                : 0,
-            status: this.status
+            // Per-failure-kind histogram. Empty {} for jobs with no failures.
+            // Surfaced so the UI / admin can see "1234 timeouts, 5 auth, 200 5xx"
+            // instead of an undifferentiated failedTiles count.
+            failures: { ...this.failureCounts },
+            deletedTiles: this.deletedTiles,
+            progress: progress(),
+            status: this.status,
+            state: this.state
         };
     }
-    static async getTileFromCacheOrRemote(chartsPath, provider, tile) {
-        const tilePath = path_1.default.join(chartsPath, `${provider.name}`, `${tile.z}`, `${tile.x}`, `${tile.y}.${provider.format}`);
+    static async getTileFromCacheOrRemote(chartsPath, provider, tile, overwrite = false) {
+        let stats = ChartDownloader.CacheStatistics.get(provider.identifier);
+        if (!stats) {
+            stats = { requests: 0, hits: 0, misses: 0, failures: 0 };
+            ChartDownloader.CacheStatistics.set(provider.identifier, stats);
+        }
+        stats.requests++;
+        // Try MBTiles cache first
+        if (!overwrite) {
+            const mbtilesBuffer = await ChartDownloader.getTileFromMbTiles(provider._mbtilesHandle, tile);
+            if (mbtilesBuffer) {
+                stats.hits++;
+                return { buffer: mbtilesBuffer, source: TileSource.FetchedFromCache };
+            }
+        }
+        // Cache miss: fetch from remote
+        const { buffer, failure } = await ChartDownloader.fetchTileFromRemoteDetailed(provider, tile);
+        ChartDownloader.CachingEnabled =
+            await ChartDownloader.hasDiskSpace(chartsPath);
+        if (ChartDownloader.CachingEnabled && buffer) {
+            stats.misses++;
+            //Writing to mbtiles must be awaited
+            await ChartDownloader.cacheTileToMbTiles(provider._mbtilesHandle, tile, buffer);
+            return {
+                buffer,
+                source: TileSource.FetchedFromRemote,
+                failure: { kind: FailureKind.None }
+            };
+        }
+        stats.failures++;
+        return { buffer: null, source: TileSource.None, failure };
+    }
+    static async getTileFromMbTiles(mbtilesHandle, tile) {
+        if (!mbtilesHandle) {
+            return null;
+        }
         try {
-            const data = await fs_1.default.promises.readFile(tilePath);
-            return data;
+            const cachedBuffer = await new Promise((resolve, reject) => {
+                mbtilesHandle.getTile(tile.z, tile.x, tile.y, (err, data) => {
+                    if (err) {
+                        // MBTiles returns an error when tile does not exist
+                        if (err.message?.includes('Tile does not exist')) {
+                            return resolve(null);
+                        }
+                        return reject(err);
+                    }
+                    resolve(data);
+                });
+            });
+            if (cachedBuffer) {
+                return cachedBuffer;
+            }
         }
         catch (err) {
-            //Cache miss, proceed to fetch from remote
+            console.error('Failed to read tile from MBTiles cache:', err);
         }
-        const buffer = await this.fetchTileFromRemote(provider, tile);
-        if (buffer) {
+        return null;
+    }
+    static async cacheTileToMbTiles(mbtilesHandle, tile, buffer) {
+        if (buffer && ChartDownloader.CachingEnabled) {
             try {
-                await fs_1.default.promises.mkdir(path_1.default.dirname(tilePath), { recursive: true });
-                await fs_1.default.promises.writeFile(tilePath, buffer);
+                await new Promise((resolve, reject) => {
+                    mbtilesHandle.putTile(tile.z, tile.x, tile.y, buffer, (err) => {
+                        if (err)
+                            reject(err);
+                        else
+                            resolve();
+                    });
+                });
+                ChartDownloader.TilesCached++;
             }
             catch (err) {
-                console.error(`Error writing tile ${tilePath}:`, err);
+                console.error('Failed to write tile to mbtiles cache:', err);
             }
         }
-        return buffer;
     }
-    static async fetchTileFromRemote(provider, tile, timeoutMs = 5000) {
+    /**
+     * Detailed-result variant of fetchTileFromRemote. Returns the buffer on
+     * success and a `failure` discriminator on every other path so the
+     * caller can attribute "failed" tiles to a specific cause (timeout vs
+     * 5xx vs network) for diagnostic logs and per-job counters.
+     *
+     * Retries once on 502/503 with TILE_RETRY_BACKOFF_MS backoff: those are
+     * the response codes that genuinely benefit from one more try (LB hiccup,
+     * backend cold-start). 504 is excluded because it means the upstream
+     * already waited; another wait makes the per-tile latency intolerable.
+     * 4xx are not retried (auth or "tile genuinely doesn't exist" — neither
+     * gets better by trying again).
+     */
+    static async fetchTileFromRemoteDetailed(provider, tile, timeoutMs = TILE_FETCH_TIMEOUT_MS) {
+        // Token providers fetch their token lazily; awaiting here ensures
+        // remoteUrl/headers below template against a non-stale token. Fast-path
+        // when the cached token is still inside its TTL: the call returns
+        // synchronously without I/O.
+        if (provider._tokenProvider) {
+            await provider._tokenProvider.ensureFreshToken();
+        }
         // Local (non-proxy) providers have no remoteUrl; the POST /cache endpoint
         // is open to any provider, so callers can still land here and should get
         // a well-defined null rather than a crash.
         if (!provider.remoteUrl) {
-            return null;
+            return { buffer: null, failure: { kind: FailureKind.NoRemoteUrl } };
         }
         let url = provider.remoteUrl
             .replace('{z}', tile.z.toString())
@@ -272,175 +515,107 @@ class ChartDownloader {
                 url = url.replace('{bbox_3857}', `${mx1},${my1},${mx2},${my2}`);
             }
         }
-        const controller = new AbortController();
-        const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
-        try {
-            const response = await fetch(url, {
-                headers: provider.headers,
-                signal: controller.signal
-            });
-            // Clear the abort timer as soon as the response head is in. A long body
-            // read was otherwise racing the timeout and could be aborted mid-stream
-            // while the caller waited on arrayBuffer().
-            clearTimeout(timeoutId);
-            if (!response.ok) {
-                return null;
+        let lastFailure = { kind: FailureKind.Network };
+        for (let attempt = 0; attempt < 2; attempt++) {
+            if (attempt > 0) {
+                await new Promise((r) => setTimeout(r, ChartDownloader.tileRetryBackoffMs));
             }
-            const arrayBuffer = await response.arrayBuffer();
-            return Buffer.from(arrayBuffer);
-        }
-        catch (_err) {
-            clearTimeout(timeoutId);
-            return null;
-        }
-    }
-    getSubTiles(tile, maxZoom) {
-        const tiles = [tile];
-        for (let z = tile.z + 1; z <= maxZoom; z++) {
-            const zoomDiff = z - tile.z;
-            const factor = Math.pow(2, zoomDiff);
-            const startX = tile.x * factor;
-            const startY = tile.y * factor;
-            for (let x = startX; x < startX + factor; x++) {
-                for (let y = startY; y < startY + factor; y++) {
-                    tiles.push({ x, y, z });
+            const controller = new AbortController();
+            const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+            try {
+                const response = await fetch(url, {
+                    headers: provider.headers,
+                    signal: controller.signal
+                });
+                // Clear the abort timer as soon as the response head is in. A long
+                // body read was otherwise racing the timeout and could be aborted
+                // mid-stream while the caller waited on arrayBuffer().
+                clearTimeout(timeoutId);
+                if (response.ok) {
+                    const arrayBuffer = await response.arrayBuffer();
+                    return {
+                        buffer: Buffer.from(arrayBuffer),
+                        failure: { kind: FailureKind.None }
+                    };
+                }
+                // For token providers, an upstream 401/403 most likely means the
+                // cached token has expired or been revoked. Invalidate so the next
+                // fetch refreshes it. 5xx and other non-auth errors don't trigger
+                // this — they're not about credentials.
+                if ((response.status === 401 || response.status === 403) &&
+                    provider._tokenProvider) {
+                    provider._tokenProvider.invalidateToken();
                 }
+                const kind = response.status >= 500
+                    ? FailureKind.HttpServer
+                    : FailureKind.HttpClient;
+                lastFailure = { kind, status: response.status };
+                // Retry only on 502/503 — those are the LB/backend-blip codes
+                // worth a second try. 504 already implies a long wait upstream;
+                // 4xx won't change on retry.
+                if (response.status === 502 || response.status === 503)
+                    continue;
+                return { buffer: null, failure: lastFailure };
+            }
+            catch (err) {
+                clearTimeout(timeoutId);
+                // AbortError vs other network errors: the timeout's controller.abort
+                // throws an AbortError. Treat any other thrown error as Network.
+                const isAbort = err.name === 'AbortError' ||
+                    controller.signal.aborted;
+                lastFailure = {
+                    kind: isAbort ? FailureKind.Timeout : FailureKind.Network
+                };
+                // Don't retry on timeout (already slow) or generic network error
+                // (likely DNS/route failure — won't recover in 1.5s).
+                return { buffer: null, failure: lastFailure };
             }
         }
-        return tiles;
+        return { buffer: null, failure: lastFailure };
     }
     /**
-     * Get all tiles that intersect a bounding box up to a maximum zoom level.
-     * bbox = [minLon, minLat, maxLon, maxLat]
+     * Backwards-compatible wrapper for callers that only need the buffer.
+     * Internal callers should prefer fetchTileFromRemoteDetailed for the
+     * categorised failure information.
      */
-    getTilesForBBox(bbox, maxZoom) {
-        const tiles = [];
-        const [minLon, minLat, maxLon, maxLat] = bbox;
-        const crossesAntiMeridian = minLon > maxLon;
-        // Respect the provider's minzoom: low zooms outside the provider's range
-        // would 404 from the remote and just inflate totalTiles.
-        const minZoom = Math.max(tileServer_1.MIN_ZOOM, this.provider.minzoom ?? tileServer_1.MIN_ZOOM);
-        // Helper to process a lon/lat box normally. lonLatToTileXY returns
-        // tile-Y increasing southward, so for a box with minLat < maxLat the
-        // south edge yields the larger tile-Y.
-        const processBBox = (lo1, la1, lo2, la2) => {
-            for (let z = minZoom; z <= maxZoom; z++) {
-                const [minX, maxY] = (0, projection_1.lonLatToTile)(lo1, la1, z); // SW corner
-                const [maxX, minY] = (0, projection_1.lonLatToTile)(lo2, la2, z); // NE corner
-                for (let x = minX; x <= maxX; x++) {
-                    for (let y = minY; y <= maxY; y++) {
-                        tiles.push({ x, y, z });
-                    }
-                }
-            }
-        };
-        if (!crossesAntiMeridian) {
-            // normal
-            processBBox(minLon, minLat, maxLon, maxLat);
-        }
-        else {
-            // crosses antimeridian — split into two boxes:
-            // [minLon -> 180] and [-180 -> maxLon]
-            processBBox(minLon, minLat, 180, maxLat);
-            processBBox(-180, minLat, maxLon, maxLat);
-        }
-        return tiles;
-    }
-    getTilesForGeoJSON(geojson, zoomMin = 1, zoomMax = 14) {
-        const tiles = [];
-        for (const feature of geojson.features) {
-            if (feature.geometry.type !== 'Polygon' &&
-                feature.geometry.type !== 'MultiPolygon') {
-                console.warn('Skipping non-polygon feature');
-                continue;
-            }
-            const boundingBox = (0, bbox_1.bbox)(feature.geometry); // [minX, minY, maxX, maxY]
-            for (let z = zoomMin; z <= zoomMax; z++) {
-                const [minX, minY] = (0, projection_1.lonLatToTile)(boundingBox[0], boundingBox[3], z); // top-left
-                const [maxX, maxY] = (0, projection_1.lonLatToTile)(boundingBox[2], boundingBox[1], z); // bottom-right
-                for (let x = minX; x <= maxX; x++) {
-                    for (let y = minY; y <= maxY; y++) {
-                        // Cheap AABB pre-filter avoids allocating a turf polygon and
-                        // running booleanIntersects for tiles that can't possibly
-                        // overlap the feature's bbox. Saves 90%+ of the turf work on
-                        // concave regions.
-                        const [tMinLon, tMinLat, tMaxLon, tMaxLat] = (0, projection_1.tileToBBox)(x, y, z);
-                        if (tMaxLon < boundingBox[0] ||
-                            tMinLon > boundingBox[2] ||
-                            tMaxLat < boundingBox[1] ||
-                            tMinLat > boundingBox[3]) {
-                            continue;
-                        }
-                        const tilePoly = this.bboxPolygon([
-                            tMinLon,
-                            tMinLat,
-                            tMaxLon,
-                            tMaxLat
-                        ]);
-                        if ((0, boolean_intersects_1.default)(feature, tilePoly)) {
-                            tiles.push({ x, y, z });
-                        }
-                    }
-                }
-            }
-        }
-        return tiles;
+    static async fetchTileFromRemote(provider, tile, timeoutMs = TILE_FETCH_TIMEOUT_MS) {
+        const { buffer } = await ChartDownloader.fetchTileFromRemoteDetailed(provider, tile, timeoutMs);
+        return buffer;
     }
-    convertRegionToGeoJSON(region) {
-        const feature = region.feature;
-        if (!feature || feature.type !== 'Feature' || !feature.geometry) {
-            throw new Error('Invalid region: missing feature or geometry');
+    static async hasDiskSpace(path) {
+        const now = Date.now();
+        // cache for 2 seconds
+        if (now - ChartDownloader.lastDiskSpaceCheck < 2000) {
+            return ChartDownloader.lastDiskSpaceResult;
         }
-        const geoFeature = {
-            type: 'Feature',
-            id: feature.id || undefined,
-            geometry: feature.geometry,
-            properties: {
-                name: region.name || '',
-                description: region.description || '',
-                timestamp: region.timestamp || '',
-                source: region.$source || '',
-                ...(feature.properties || {})
-            }
-        };
-        const splitGeoFeature = (0, geojson_antimeridian_cut_1.default)(geoFeature);
-        const features = [];
-        const pushFeaturePolygon = (orig, coords, idx) => {
-            const poly = {
-                type: 'Feature',
-                id: idx != null && orig.id ? `${orig.id}-${idx}` : orig.id,
-                geometry: {
-                    type: 'Polygon',
-                    coordinates: coords
-                },
-                properties: orig.properties || {}
-            };
-            features.push(poly);
-        };
-        const f = splitGeoFeature;
-        if (f.geometry && f.geometry.type === 'MultiPolygon') {
-            const coords = f.geometry.coordinates;
-            coords.forEach((ring, i) => pushFeaturePolygon(f, ring, i));
+        try {
+            const { free } = await (0, check_disk_space_1.default)(path);
+            ChartDownloader.lastDiskSpaceResult =
+                free >= ChartDownloader.MINIMUM_FREE_DISK_SPACE;
         }
-        else if (f.geometry && f.geometry.type === 'Polygon') {
-            features.push(f);
+        catch (err) {
+            ChartDownloader.lastDiskSpaceResult = false;
         }
-        return {
-            type: 'FeatureCollection',
-            features
-        };
+        ChartDownloader.lastDiskSpaceCheck = now;
+        return ChartDownloader.lastDiskSpaceResult;
     }
-    bboxPolygon(boundingBox) {
-        const [minLon, minLat, maxLon, maxLat] = boundingBox;
-        return (0, helpers_1.polygon)([
-            [
-                [minLon, minLat],
-                [maxLon, minLat],
-                [maxLon, maxLat],
-                [minLon, maxLat],
-                [minLon, minLat]
-            ]
-        ]);
+    static getStatistics() {
+        return Object.fromEntries(ChartDownloader.CacheStatistics);
+    }
+    /**
+     * Reset the static fields owned by ChartDownloader. See
+     * ChartSeedingManager.resetForTests for why this exists.
+     */
+    static resetForTests() {
+        ChartDownloader.nextJobId = 1;
+        ChartDownloader.CachingEnabled = true;
+        ChartDownloader.CacheStatistics = new Map();
+        ChartDownloader.TilesCached = 0;
+        ChartDownloader.lastDiskSpaceCheck = 0;
+        ChartDownloader.lastDiskSpaceResult = true;
+        // Skip the production 1.5s backoff for mocked-503 tests; production
+        // value is restored per the constant on plugin reload.
+        ChartDownloader.tileRetryBackoffMs = 0;
     }
 }
 exports.ChartDownloader = ChartDownloader;