coastlines 0.0.6-alpha.0 → 0.0.8-alpha.0

package/README.md

@@ -111,6 +111,18 @@ the GSHHG zip. `coastlines/manifest.json` is the machine-readable form of this
 table, with the bytes and sha256 of each file and, under `sources`, of the
 zip itself.
 
+### Installed, built, or cached
+
+| Where | What | Size | How it reaches you |
+| --- | --- | ---: | --- |
+| `data/` | the 36 files above | 78 MB, 29 MB packed | installed with the package |
+| `geo/` | six cut bands, land and water in square cells | 896 MB | built by `npm run sync`; not shipped |
+| `cache/` | the GSHHG zip, six Natural Earth 10m water files | 167 MB | fetched once by sync, kept |
+
+The cut bands are Natural Earth 10m and GSHHG h and f, land and lakes, each
+a record in `catalogue.json`: grid, inputs and pins, cutter and build date.
+How cells reach a boat that installs the package is open ([#17](https://github.com/mark-brannan/coastlines/issues/17)).
+
 ### Sources and licences
 
 Natural Earth is public domain. GSHHG (Wessel & Smith, the Global
@@ -146,7 +158,7 @@ release and refuses any byte that disagrees with `manifest.json`; `npm pack`
 runs it first, so a tarball never ships a file the manifest did not promise.
 
 ```shell
-npm run sync          # fetch what is missing, verify everything (45 MB + 149 MB zip)
+npm run sync          # fetch what is missing, verify everything, cut the cells (~2 min, 6 GB heap)
 npm run sync:check    # verify only, no network
 npm run sync:refresh  # move the pins: refetch all and rewrite bytes and sha256
 ```

package/catalogue.json

@@ -1046,7 +1046,9 @@
       "licence": "public domain",
       "version": "v5.1.2",
       "date": "2022-05-13",
-      "caveat": "Holds to about z12 but has no narrow channels"
+      "caveat": "Holds to about z12 but has no narrow channels",
+      "cutter": "coastlines/lib/cut.js",
+      "built": "2026-10-15"
     },
     "bytesPerCell": null,
     "bytes": null,
@@ -1086,7 +1088,9 @@
       "licence": "LGPL-3.0-or-later",
       "version": "2.3.7",
       "date": "2017-06-15",
-      "caveat": "Holds to about z13; draws the Seattle ship canal. Its licence files are owned here and travel with every GSHHG band"
+      "caveat": "Holds to about z13; draws the Seattle ship canal. Its licence files are owned here and travel with every GSHHG band",
+      "cutter": "coastlines/lib/cut.js",
+      "built": "2026-10-15"
     },
     "bytesPerCell": null,
     "bytes": null,
@@ -1118,7 +1122,9 @@
       "licence": "LGPL-3.0-or-later",
       "version": "2.3.7",
       "date": "2017-06-15",
-      "caveat": "Full resolution; about 70,000 cells"
+      "caveat": "Full resolution; about 70,000 cells",
+      "cutter": "coastlines/lib/cut.js",
+      "built": "2026-10-15"
     },
     "bytesPerCell": null,
     "bytes": null,
@@ -1169,7 +1175,9 @@
       "licence": "public domain",
       "version": "v5.1.2",
       "date": "2022-05-13",
-      "caveat": "Rivers and lakes with their regional supplements; whole features go into every cell they reach. Not in the manifest, so each file is pinned here"
+      "caveat": "Rivers and lakes with their regional supplements; whole features go into every cell they reach. Not in the manifest, so each file is pinned here",
+      "cutter": "coastlines/lib/cut.js",
+      "built": "2026-10-15"
     },
     "bytesPerCell": null,
     "bytes": null,
@@ -1247,7 +1255,9 @@
       "licence": "LGPL-3.0-or-later",
       "version": "2.3.7",
       "date": "2017-06-15",
-      "caveat": "Lakes no Natural Earth file has (Dian Chi, Lago Mascardi); levels 2 and 3 with river-lakes"
+      "caveat": "Lakes no Natural Earth file has (Dian Chi, Lago Mascardi); levels 2 and 3 with river-lakes",
+      "cutter": "coastlines/lib/cut.js",
+      "built": "2026-10-15"
     },
     "bytesPerCell": null,
     "bytes": null,
@@ -1311,7 +1321,9 @@
       "licence": "LGPL-3.0-or-later",
       "version": "2.3.7",
       "date": "2017-06-15",
-      "caveat": "As gshhg-h-lakes at full resolution"
+      "caveat": "As gshhg-h-lakes at full resolution",
+      "cutter": "coastlines/lib/cut.js",
+      "built": "2026-10-15"
     },
     "bytesPerCell": null,
     "bytes": null,

package/lib/bands.js

@@ -16,6 +16,9 @@ import { charts } from './charts.js';
 
 const require = createRequire(import.meta.url);
 
+/** What a cut band's `provenance.cutter` names: the module that builds its cells. */
+export const CUTTER = 'coastlines/lib/cut.js';
+
 /** The parsed catalogue.json: one record per band, whole Natural Earth files first. */
 export const catalogue = require('../catalogue.json');
 
@@ -72,6 +75,20 @@ function wholeFileDrift(record) {
   return problems;
 }
 
+/** A cut band names its cutter and the day it was built, and draws on one upstream only. */
+function cutDrift(record) {
+  const problems = [];
+  const { cutter, built } = record.provenance;
+  if (cutter !== CUTTER) problems.push(`${record.id}: cutter ${cutter} is not ${CUTTER}`);
+  if (!/^\d{4}-\d{2}-\d{2}$/.test(built ?? ''))
+    problems.push(`${record.id}: built ${built} is not a date`);
+  const sources = new Set(
+    inputsOf(record).map((i) => (typeof i === 'string' ? find(i).source : i.source)),
+  );
+  if (sources.size !== 1) problems.push(`${record.id}: mixes sources ${[...sources].join(', ')}`);
+  return problems;
+}
+
 /**
  * Where the catalogue and the manifest disagree, as sentences; empty when they agree. Every
  * record's upstream URLs, members and sha256s must be what its inputs pin, its licence, version
@@ -105,6 +122,7 @@ export function drift() {
         if (date !== from.date) problems.push(`${record.id}: date ${date} is not ${key}'s`);
       }
       if (record.grid === null) problems.push(...wholeFileDrift(record));
+      else problems.push(...cutDrift(record));
     } catch (error) {
       problems.push(`${record.id}: ${error.message}`);
     }

package/lib/cut.js

@@ -0,0 +1,138 @@
+// Builds the catalogue's `cells` sources: each cut band's inputs, read through `read`, cut into
+// square lon/lat cells under geo/<dir>/ (<xi>_<yi>.json and index.json). A band is cut from one
+// upstream only, so no geometry file mixes sources.
+import { mkdir, rename, rm, writeFile } from 'node:fs/promises';
+import { band, CUTTER, inputsOf } from './bands.js';
+import { find } from './catalogue.js';
+import { shapefilePolygons } from './shapefile.js';
+import { cutCells, cutLakes, cutWater, dbfRecords, gshhgLakes, joinWater } from './detail-cells.js';
+
+/** The folder under geo/ of a band's `cells` source: `geo/f/` is `f`. */
+export const cellsDir = (record) =>
+  record.sources.find((source) => source.kind === 'cells').url.replace(/^geo\/|\/$/g, '');
+
+const json = (bytes) => JSON.parse(new TextDecoder().decode(bytes));
+
+function polygons(geojson) {
+  const out = [];
+  for (const { geometry } of geojson.features) {
+    if (geometry?.type === 'Polygon') out.push(geometry.coordinates);
+    else if (geometry?.type === 'MultiPolygon') for (const p of geometry.coordinates) out.push(p);
+  }
+  return out;
+}
+
+function lines(geojson) {
+  const out = [];
+  for (const { geometry } of geojson.features) {
+    if (geometry?.type === 'LineString') out.push(geometry.coordinates);
+    else if (geometry?.type === 'MultiLineString')
+      for (const l of geometry.coordinates) out.push(l);
+  }
+  return out;
+}
+
+/**
+ * The cells of a band: Map(key -> cell). `read(input)` returns the bytes of one input, as
+ * `inputsOf(record)` lists them.
+ */
+export async function cutBand(record, read) {
+  const size = record.grid.cellDeg;
+  const bytes = await Promise.all(inputsOf(record).map(read));
+  if (record.id === 'ne-10m-land') {
+    const [land, coast, minor] = bytes.map(json);
+    return cutCells(
+      { land: polygons(land), coast: [...lines(coast), ...polygons(minor).map((p) => p[0])] },
+      size,
+    );
+  }
+  if (record.layer === 'land') {
+    const land = shapefilePolygons(bytes[0]);
+    return cutCells({ land, coast: land.flat() }, size);
+  }
+  if (record.layer === 'water') {
+    const [rivers, lakes] = [bytes.slice(0, 3), bytes.slice(3)].map((half) =>
+      joinWater(half.map(json)),
+    );
+    return cutWater(rivers, lakes, { size });
+  }
+  if (record.layer === 'lakes') {
+    const [l2, d2, l3, d3] = bytes;
+    return cutLakes(
+      gshhgLakes(shapefilePolygons(l2), dbfRecords(d2), shapefilePolygons(l3), dbfRecords(d3)),
+      size,
+    );
+  }
+  throw new Error(`${record.id}: no cutter for layer ${record.layer}`);
+}
+
+/** Writes a band's cells to `root`/geo/<dir>/, complete or not at all; returns the cell count. */
+export async function writeBand(record, cells, root) {
+  const final = new URL(`geo/${cellsDir(record)}/`, root);
+  const partial = new URL(`geo/${cellsDir(record)}.partial/`, root);
+  await rm(partial, { recursive: true, force: true });
+  await mkdir(partial, { recursive: true });
+  // in batches: "f" has some 70,000 cells, past an open-file limit if written at once
+  const all = [...cells];
+  for (let i = 0; i < all.length; i += 512)
+    await Promise.all(
+      all
+        .slice(i, i + 512)
+        .map(([k, c]) => writeFile(new URL(`${k}.json`, partial), JSON.stringify(c))),
+    );
+  await writeFile(
+    new URL('index.json', partial),
+    JSON.stringify({ cell: record.grid.cellDeg, cells: [...cells.keys()] }),
+  );
+  await rm(final, { recursive: true, force: true });
+  await rename(partial, final);
+  return cells.size;
+}
+
+/**
+ * catalogue.json's text with one record's `provenance.built` set to `day`, every other byte as it
+ * was. Throws rather than touch the wrong record: the id must be there, and the `built` must sit
+ * in that record, before the next `"id":`.
+ */
+export function stampBuilt(text, id, day) {
+  const at = text.indexOf(`"id": "${id}"`);
+  if (at < 0) throw new Error(`${id}: not in catalogue.json`);
+  const end = text.indexOf('"id": "', at + 1);
+  const record = text.slice(at, end < 0 ? undefined : end);
+  const stamped = record.replace(/("built": )"[^"]*"/, `$1"${day}"`);
+  if (stamped === record && !record.includes(`"built": "${day}"`))
+    throw new Error(`${id}: no provenance.built to stamp`);
+  return text.slice(0, at) + stamped + (end < 0 ? '' : text.slice(end));
+}
+
+/** The cut bands whose inputs are GSHHG's, which the notice below covers. */
+export const modifiedGshhg = (catalogue) =>
+  catalogue.filter(
+    (r) =>
+      r.grid &&
+      inputsOf(r).some((i) => (typeof i === 'string' ? find(i).source : i.source) === 'gshhg'),
+  );
+
+/** The text of geo/gshhg-modified.txt for the bands built, with the dates they were cut. */
+export function modifiedNotice(records) {
+  const rows = records
+    .map((r) => `  geo/${cellsDir(r)}/   ${r.id}, cut ${r.provenance.built}`)
+    .join('\n');
+  return `GSHHG, modified
+===============
+
+The cell files below are modified GSHHG ${band(records[0].id).provenance.version} (Wessel & Smith), not the
+published shapefiles. They were made by ${CUTTER} from the pinned zip: polygons
+clipped to square lon/lat cells, shorelines split where they pass from one cell to the next,
+coordinates rounded to 5 decimals, and each land polygon annotated with the area and colour of the
+island it is a piece of.
+
+${rows}
+
+Licence: GNU Lesser General Public License version 3 or later, conveyed under section 5(a) of the
+GNU General Public License version 3, which asks for a notice that the work was modified and the
+date of the modification: this file. The unmodified originals are fetched, byte for byte, by
+\`npm run sync\`; their copyright and permission notice and the licence text are in
+data/gshhg-licence.txt and data/gshhg-copying-lesser.txt and apply to the cells too.
+`;
+}

package/lib/detail-cells.js

@@ -0,0 +1,313 @@
+// Cuts Natural Earth 10m land and coast into square lon/lat cells, so a close view fetches only the
+// few cells around it rather than the whole 10m world. Land polygons are clipped to each cell (fill
+// is even-odd, so the cut edges vanish; each cell's `m` holds, for its land polygons in order, the
+// area in km2 and colour of the island each is a piece of, worked out before the cut); coast lines are split where they pass from one cell to the
+// next, each piece keeping the shared point so the line stays joined. Coordinates are rounded to
+// `digits` decimals (5 is about a metre). Cell keys are `${xi}_${yi}`, xi = floor(lon / size).
+
+import { islandInfo } from './island.js';
+
+const key = (xi, yi) => `${xi}_${yi}`;
+
+// Sutherland-Hodgman against one axis-aligned half-plane: keep points with p[axis] <= v (keep < 0)
+// or >= v (keep > 0). `ring` is open (last point not repeated).
+function clipHalf(ring, axis, v, keep) {
+  const out = [],
+    inside = (p) => (keep < 0 ? p[axis] <= v : p[axis] >= v);
+  for (let i = 0; i < ring.length; i++) {
+    const a = ring[i],
+      b = ring[(i + 1) % ring.length],
+      ia = inside(a),
+      ib = inside(b);
+    if (ia) out.push(a);
+    if (ia !== ib) {
+      const t = (v - a[axis]) / (b[axis] - a[axis]);
+      const p = [a[0] + t * (b[0] - a[0]), a[1] + t * (b[1] - a[1])];
+      p[axis] = v;
+      out.push(p);
+    }
+  }
+  return out;
+}
+
+function ringBox(ring) {
+  let x0 = Infinity,
+    y0 = Infinity,
+    x1 = -Infinity,
+    y1 = -Infinity;
+  for (const [x, y] of ring) {
+    if (x < x0) x0 = x;
+    if (x > x1) x1 = x;
+    if (y < y0) y0 = y;
+    if (y > y1) y1 = y;
+  }
+  return [x0, y0, x1, y1];
+}
+
+// Polygons ([[outer, ...holes]], open rings) into cells by recursive halving along cell lines, so
+// each point is clipped about log2(cells) times rather than once per cell.
+function splitPolygons(polys, box, size, emit) {
+  if (!polys.length) return;
+  const [x0, y0, x1, y1] = box,
+    nx = Math.round((x1 - x0) / size),
+    ny = Math.round((y1 - y0) / size);
+  if (nx <= 1 && ny <= 1) {
+    emit(Math.round(x0 / size), Math.round(y0 / size), polys);
+    return;
+  }
+  const axis = nx >= ny ? 0 : 1,
+    mid = axis === 0 ? x0 + Math.floor(nx / 2) * size : y0 + Math.floor(ny / 2) * size;
+  const lo = [],
+    hi = [];
+  for (const poly of polys) {
+    const b = ringBox(poly[0]);
+    if (b[axis + 2] <= mid) {
+      lo.push(poly);
+      continue;
+    }
+    if (b[axis] >= mid) {
+      hi.push(poly);
+      continue;
+    }
+    for (const [side, keep] of [
+      [lo, -1],
+      [hi, 1],
+    ]) {
+      const [outer, ...holes] = poly.map((r) => clipHalf(r, axis, mid, keep));
+      if (outer.length >= 3)
+        side.push(Object.assign([outer, ...holes.filter((r) => r.length >= 3)], { isl: poly.isl }));
+    }
+  }
+  const loBox = axis === 0 ? [x0, y0, mid, y1] : [x0, y0, x1, mid],
+    hiBox = axis === 0 ? [mid, y0, x1, y1] : [x0, mid, x1, y1];
+  splitPolygons(lo, loBox, size, emit);
+  splitPolygons(hi, hiBox, size, emit);
+}
+
+// `land`: GeoJSON polygons as [[outer, ...holes]] of closed [lon, lat] rings; `coast`: lines of
+// [lon, lat]. Returns Map(key -> { land, coast, m }), rings closed again, coordinates rounded; `m`
+// holds `meta` of the whole polygon each land piece was cut from (by default its island's area and colour).
+const islandMeta = (poly) => {
+  const i = islandInfo(poly);
+  return [i.km2, i.hue];
+};
+export function cutCells({ land, coast }, size = 5, digits = 5, meta = islandMeta) {
+  const f = 10 ** digits,
+    round = (p) => [Math.round(p[0] * f) / f, Math.round(p[1] * f) / f];
+  const tidy = (pts) =>
+    pts.map(round).filter((p, i, a) => i === 0 || p[0] !== a[i - 1][0] || p[1] !== a[i - 1][1]);
+  const cells = new Map(),
+    cell = (k) => cells.get(k) ?? cells.set(k, { land: [], coast: [], m: [] }).get(k);
+  const ids = coast.some((l) => l.id !== undefined); // and if lines carry an `id`, `cm` holds each piece's
+
+  const open = (r) =>
+    r.length > 1 && r[0][0] === r[r.length - 1][0] && r[0][1] === r[r.length - 1][1]
+      ? r.slice(0, -1)
+      : r;
+  splitPolygons(
+    land.map((poly) => Object.assign(poly.map(open), { isl: meta(poly) })),
+    [-180, -90, 180, 90],
+    size,
+    (xi, yi, polys) => {
+      const c = cell(key(xi, yi)),
+        out = c.land;
+      for (const poly of polys) {
+        const rings = poly
+          .map(tidy)
+          .filter((r) => r.length >= 3)
+          .map((r) => [...r, r[0]]);
+        if (rings.length) {
+          out.push(rings);
+          c.m.push(poly.isl);
+        }
+      }
+    },
+  );
+
+  // A segment belongs to the cell holding its midpoint; a run of segments in one cell is one piece.
+  const cellOf = (a, b) =>
+    key(Math.floor((a[0] + b[0]) / 2 / size), Math.floor((a[1] + b[1]) / 2 / size));
+  const put = (k, piece, id) => {
+    const c = cell(k);
+    c.coast.push(piece);
+    if (ids) (c.cm ??= []).push(id);
+  };
+  for (const line of coast) {
+    let piece = null,
+      at = null;
+    for (let i = 0; i < line.length - 1; i++) {
+      const k = cellOf(line[i], line[i + 1]);
+      if (k !== at) {
+        if (piece) put(at, piece, line.id);
+        piece = [line[i]];
+        at = k;
+      }
+      piece.push(line[i + 1]);
+    }
+    if (piece) put(at, piece, line.id);
+  }
+  for (const c of cells.values()) {
+    const lines = c.coast.map(tidy),
+      keep = lines.map((l) => l.length > 1);
+    c.coast = lines.filter((_, i) => keep[i]);
+    if (c.cm) c.cm = c.cm.filter((_, i) => keep[i]);
+  }
+  return cells;
+}
+
+// The Natural Earth 10m water downloads, the world file first and then the regional supplements,
+// which hold what the world file leaves out (Lake Washington, Como, the Snohomish) and repeat none of
+// it. joinWater makes one collection of several, in this order, so cutWater's ids stay unique.
+export const WATER_FILES = {
+  rivers: [
+    'ne_10m_rivers_lake_centerlines.geojson',
+    'ne_10m_rivers_north_america.geojson',
+    'ne_10m_rivers_europe.geojson',
+  ],
+  lakes: [
+    'ne_10m_lakes.geojson',
+    'ne_10m_lakes_north_america.geojson',
+    'ne_10m_lakes_europe.geojson',
+  ],
+};
+export const joinWater = (geos) => ({
+  type: 'FeatureCollection',
+  features: geos.flatMap((g) => g?.features ?? []),
+});
+
+// Natural Earth rivers and lakes into the same square cells, for scripts/water.mjs past WATER_ZOOM.
+// A feature goes whole into every cell it reaches (a river, each cell its segments' midpoints and
+// vertices fall in; a lake, each cell its box meets), so a river keeps one wiggle and a lake one
+// outline wherever the view is cut; its index in the source file is its GeoJSON `id`, by which
+// the scene keeps one copy of a feature fetched from several cells. Properties are cut to `keep`.
+// Returns Map(key -> { rivers, lakes }), each a FeatureCollection.
+export function cutWater(
+  riversGeo,
+  lakesGeo,
+  {
+    keep = { rivers: ['name', 'featurecla', 'min_zoom'], lakes: ['name', 'min_zoom'] },
+    size = 5,
+    digits = 5,
+  } = {},
+) {
+  const f = 10 ** digits,
+    round = (c) =>
+      typeof c[0] === 'number'
+        ? [Math.round(c[0] * f) / f, Math.round(c[1] * f) / f]
+        : c.map(round);
+  const cells = new Map(),
+    cell = (k) => cells.get(k) ?? cells.set(k, { rivers: [], lakes: [] }).get(k);
+  // 180 east and 90 north fall in the last cell, not one past it
+  const xiOf = (lon) => Math.min(180 / size - 1, Math.floor(lon / size)),
+    yiOf = (lat) => Math.min(90 / size - 1, Math.floor(lat / size));
+  const at = (lon, lat) => key(xiOf(lon), yiOf(lat));
+  const lines = (g) =>
+    g.type === 'LineString'
+      ? [g.coordinates]
+      : g.type === 'MultiLineString'
+        ? g.coordinates
+        : g.type === 'Polygon'
+          ? g.coordinates
+          : g.type === 'MultiPolygon'
+            ? g.coordinates.flat()
+            : [];
+  const reach = {
+    rivers: (g) => {
+      const ks = new Set();
+      for (const l of lines(g))
+        l.forEach((p, i) => {
+          ks.add(at(p[0], p[1]));
+          if (i) ks.add(at((p[0] + l[i - 1][0]) / 2, (p[1] + l[i - 1][1]) / 2));
+        });
+      return ks;
+    },
+    lakes: (g) => {
+      const [x0, y0, x1, y1] = ringBox(lines(g).flat()),
+        ks = new Set();
+      for (let xi = xiOf(x0); xi <= xiOf(x1); xi++)
+        for (let yi = yiOf(y0); yi <= yiOf(y1); yi++) ks.add(key(xi, yi));
+      return ks;
+    },
+  };
+  for (const [kind, geo] of [
+    ['rivers', riversGeo],
+    ['lakes', lakesGeo],
+  ]) {
+    (geo?.features ?? []).forEach((ft, id) => {
+      if (!ft.geometry) return;
+      const slim = {
+        type: 'Feature',
+        id,
+        properties: Object.fromEntries(keep[kind].map((k) => [k, ft.properties[k]])),
+        geometry: { type: ft.geometry.type, coordinates: round(ft.geometry.coordinates) },
+      };
+      for (const k of reach[kind](ft.geometry)) cell(k)[kind].push(slim);
+    });
+  }
+  for (const [k, c] of cells)
+    cells.set(k, {
+      rivers: { type: 'FeatureCollection', features: c.rivers },
+      lakes: { type: 'FeatureCollection', features: c.lakes },
+    });
+  return cells;
+}
+
+// GSHHG lakes (level 2, river-lakes among them) with their islands (level 3) as holes, from the
+// shapefiles' polygons and dBASE records, for cutLakes. Each lake carries its GSHHG id and its
+// whole lon/lat box, which its cut pieces keep: the scene drops a 10m lake inside one.
+export function dbfRecords(bytes) {
+  const b = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  const n = b.readUInt32LE(4),
+    headLength = b.readUInt16LE(8),
+    recordLength = b.readUInt16LE(10),
+    fields = [];
+  for (let at = 32, o = 1; b[at] !== 0x0d; at += 32) {
+    const name = b.toString('latin1', at, at + 11).replace(/\0.*/s, ''),
+      length = b[at + 16];
+    fields.push([name, o, length]);
+    o += length;
+  }
+  return Array.from({ length: n }, (_, i) =>
+    Object.fromEntries(
+      fields.map(([k, o, l]) => [
+        k,
+        b
+          .toString(
+            'latin1',
+            headLength + i * recordLength + o,
+            headLength + i * recordLength + o + l,
+          )
+          .trim(),
+      ]),
+    ),
+  );
+}
+export function gshhgLakes(lakes, lakeRecords, islands, islandRecords) {
+  const byId = new Map(
+    lakes.map((poly, i) => [
+      lakeRecords[i].id,
+      Object.assign([...poly], { id: +lakeRecords[i].id }),
+    ]),
+  );
+  islands.forEach((poly, i) => byId.get(islandRecords[i].parent_id)?.push(poly[0]));
+  return [...byId.values()];
+}
+// Lakes into cells as land is (cutCells): pieces clipped to each cell to fill, and the shores split
+// at cell edges to ink, so a cut edge is never drawn. `m` holds, per piece, [id, x0, y0, x1, y1]: the
+// lake's id and whole box, rounded out to `digits`; `cm` each shore piece's lake id.
+export function cutLakes(lakes, size, digits = 5) {
+  const f = 10 ** digits;
+  const shores = lakes.flatMap((lake) =>
+    lake.map((ring) => Object.assign([...ring], { id: lake.id })),
+  );
+  return cutCells({ land: lakes, coast: shores }, size, digits, (poly) => {
+    const [x0, y0, x1, y1] = ringBox(poly[0]);
+    return [
+      poly.id,
+      Math.floor(x0 * f) / f,
+      Math.floor(y0 * f) / f,
+      Math.ceil(x1 * f) / f,
+      Math.ceil(y1 * f) / f,
+    ];
+  });
+}

package/lib/fetch-retry.js

@@ -0,0 +1,37 @@
+// A download that tries again on a network error, a body cut off mid-stream, or
+// an HTTP 5xx; never on a 4xx. Three attempts, waiting 2 s and then 6 s between
+// them. The caller's sha256 check stays the only trust: a retry changes how
+// bytes arrive, not whether they are believed.
+export const WAITS_MS = [2000, 6000];
+
+const sleepFor = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+/** One attempt: the bytes, or a failure, with `retry` false when trying again cannot help. */
+async function attempt(url, fetchImpl) {
+  let response;
+  try {
+    response = await fetchImpl(url);
+    if (response.ok) return { bytes: new Uint8Array(await response.arrayBuffer()) };
+  } catch (error) {
+    return { failure: error, retry: true };
+  }
+  await response.body?.cancel();
+  return {
+    failure: new Error(`${url}: HTTP ${response.status}`),
+    retry: response.status >= 500,
+  };
+}
+
+/** The bytes at `url`; throws on a 4xx at once, or with the last failure once the waits run out. */
+export async function fetchBytesWithRetry(
+  url,
+  { fetchImpl = fetch, sleep = sleepFor, waits = WAITS_MS, onRetry = () => {} } = {},
+) {
+  for (let tried = 0; ; tried++) {
+    const { bytes, failure, retry } = await attempt(url, fetchImpl);
+    if (bytes) return bytes;
+    if (!retry || tried >= waits.length) throw failure;
+    onRetry(failure, tried + 1, waits.length + 1, waits[tried]);
+    await sleep(waits[tried]);
+  }
+}

package/lib/island.js

@@ -0,0 +1,51 @@
+// What the land treatments need to know about an island: its area in km2 and a colour that stays
+// with it. Computed once per land polygon, from the whole polygon (so before lib/detail-cells.js
+// cuts it into cells, which carry the result along), and shared by the scene and the cutter.
+
+export const ISLAND_KM2 = 5000; // coloured islands are the land polygons smaller than this
+export const ISLAND_HUES = 3; // red, green, gold
+
+const R_KM = 6371.0088,
+  RAD = Math.PI / 180;
+
+// Area of a [lon, lat] ring (open or closed) in km2: equirectangular about the ring's own latitude,
+// good to a percent or two at this size.
+function ringKm2(ring) {
+  const lat0 = ring.reduce((s, p) => s + p[1], 0) / ring.length,
+    lon0 = ring[0][0],
+    c = Math.cos(lat0 * RAD);
+  const pt = ([lon, lat]) => [(((lon - lon0 + 540) % 360) - 180) * c, lat];
+  let s = 0;
+  for (let i = 0; i < ring.length; i++) {
+    const [x1, y1] = pt(ring[i]),
+      [x2, y2] = pt(ring[(i + 1) % ring.length]);
+    s += x1 * y2 - x2 * y1;
+  }
+  return (Math.abs(s) / 2) * (R_KM * RAD) ** 2;
+}
+
+// { km2, hue } of a polygon [outer, ...holes]. The hue comes from the centre of the outer ring's
+// box, to a tenth of a degree: the same island has the same colour in every view and, to within a
+// tenth of a degree of disagreement between data sets, at every zoom.
+export function islandInfo(poly) {
+  const km2 = Math.round(ringKm2(poly[0]) - poly.slice(1).reduce((s, r) => s + ringKm2(r), 0));
+  let x0 = Infinity,
+    x1 = -Infinity,
+    y0 = Infinity,
+    y1 = -Infinity;
+  const lon0 = poly[0][0][0];
+  for (const [lon, lat] of poly[0]) {
+    const x = ((lon - lon0 + 540) % 360) - 180;
+    if (x < x0) x0 = x;
+    if (x > x1) x1 = x;
+    if (lat < y0) y0 = lat;
+    if (lat > y1) y1 = lat;
+  }
+  let h =
+    Math.imul(Math.round((lon0 + (x0 + x1) / 2) * 10), 0x9e3779b1) ^
+    Math.imul(Math.round((y0 + y1) * 5), 0x85ebca6b);
+  h ^= h >>> 15;
+  h = Math.imul(h, 0x2c1b3c6d);
+  h ^= h >>> 13;
+  return { km2, hue: (h >>> 0) % ISLAND_HUES };
+}

package/lib/provider.js

@@ -0,0 +1,202 @@
+// The scene's data side: which band a view draws from, which of its cells, and what it falls back
+// to. A drawing library pulls rings from here through its port, an async iterable of rings, so
+// neither package imports the other; the caller owns the projection and hands over a view.
+//
+// A view is the lon/lat frame a caller paints and its scale at the centre:
+// `{ lat, metresPerPixel, lonMin, lonMax, latMin, latMax }`. The frame may run past the
+// antimeridian. The catalogue keys bands by web-mercator zoom (a 512 px world at zoom 0), so the
+// scale is turned into that zoom at the view's latitude to choose one.
+//
+// A band of cells keeps `<dir>/<xi>_<yi>.json`, `grid.cellDeg` degrees square, each
+// `{ land, coast }`, and `<dir>/index.json`, `{ cell, cells }`, listing the cells that hold
+// anything, under its `cells` source. A view draws from the finest band whose zoom it has passed,
+// over the cells within `grid.marginDeg` of its frame. For that view, a band whose index cannot be
+// read is skipped, and a band with a cell that cannot be read gives way to the band below it; the
+// last of these is the whole file. Both are read again for the next view.
+// The JSON itself, not lib/bands.js, so the provider runs in a browser too.
+import catalogue from '../catalogue.json' with { type: 'json' };
+
+const EARTH_R = 6378137; // metres, web mercator's sphere
+const WORLD_PX = 512; // the width of the world at zoom 0, as the catalogue's zooms count it
+const RAD = Math.PI / 180;
+
+/** The web-mercator zoom of a view's scale at its centre. */
+// Rounded, so a view at a band's edge zoom, turned into a scale and back, does not pass it.
+export const zoomOf = (view) =>
+  Math.round(
+    1e9 *
+      Math.log2(
+        (2 * Math.PI * EARTH_R * Math.cos(view.lat * RAD)) / (WORLD_PX * view.metresPerPixel),
+      ),
+  ) / 1e9;
+
+// What each kind of ring is read from: the bands are cut from both, and a cell carries both.
+const KINDS = {
+  coast: { layer: 'land', part: 'coast', whole: 'ne-coastline-50m' },
+  land: { layer: 'land', part: 'land', whole: 'ne-land-50m' },
+};
+
+// { id, dir, zoom, inclusive, cell, margin, bytesPerCell } of a cut record: `dir` is its `cells`
+// source's url, `zoom` the zoom it draws past (or from, when `inclusive`), `cell` and `margin` its
+// grid in degrees.
+const bandOf = (r) => ({
+  id: r.id,
+  dir: r.sources.find((s) => s.kind === 'cells').url.replace(/\/$/, ''),
+  zoom: r.zoom.min,
+  inclusive: r.zoom.minInclusive,
+  cell: r.grid.cellDeg,
+  margin: r.grid.marginDeg,
+  bytesPerCell: r.bytesPerCell,
+});
+const passed = (zoom, b) => (b.inclusive ? zoom >= b.zoom : zoom > b.zoom);
+
+const cellKey = (xi, yi) => `${xi}_${yi}`;
+
+// The band's cells, '<dir>/<xi>_<yi>', that meet the view's frame widened by its margin and that
+// the band's index (`known`) lists.
+function cellsAround(b, view, known) {
+  const deg = b.cell,
+    n = 360 / deg;
+  const lonMin = view.lonMin - b.margin,
+    lonMax = view.lonMax + b.margin;
+  const latMin = view.latMin - b.margin,
+    latMax = view.latMax + b.margin;
+  const keys = new Set();
+  for (let xi = Math.floor(lonMin / deg); xi <= Math.floor(lonMax / deg); xi++) {
+    const wx = ((((xi + n / 2) % n) + n) % n) - n / 2; // wrapped across the antimeridian
+    for (
+      let yi = Math.max(-90 / deg, Math.floor(latMin / deg));
+      yi <= Math.min(90 / deg - 1, Math.floor(latMax / deg));
+      yi++
+    ) {
+      if (known.has(cellKey(wx, yi))) keys.add(`${b.dir}/${cellKey(wx, yi)}`);
+    }
+  }
+  return [...keys];
+}
+
+// The rings of a GeoJSON object: a line's points, a polygon's rings.
+function geojsonRings(geo) {
+  const of = (g) => {
+    if (!g) return [];
+    if (g.type === 'FeatureCollection') return g.features.flatMap(of);
+    if (g.type === 'Feature') return of(g.geometry);
+    if (g.type === 'GeometryCollection') return g.geometries.flatMap(of);
+    if (g.type === 'LineString') return [g.coordinates];
+    if (g.type === 'MultiLineString' || g.type === 'Polygon') return g.coordinates;
+    if (g.type === 'MultiPolygon') return g.coordinates.flat();
+    return [];
+  };
+  return of(geo);
+}
+
+/**
+ * A provider of rings for views. `read(path)` resolves to the parsed JSON at a catalogue source
+ * path (`geo/h/12_45.json`, `data/ne-coastline-50m.geojson`) and rejects when it cannot. `kind`
+ * is `coast`, the coastline's lines, or `land`, the land polygons' rings.
+ *
+ * `rings(view)` yields the view's rings; `held(view)` says whether they are all in hand, so
+ * drawing them fetches nothing; `cost(view)` is what drawing it would fetch: the band, the cells
+ * not yet held, and their bytes where the catalogue knows them.
+ */
+export function provider({ read, kind = 'coast' }) {
+  const { layer, part, whole } = KINDS[kind] ?? {};
+  if (!layer) throw new Error(`unknown kind ${kind}`);
+  const bands = catalogue
+    .filter((r) => r.layer === layer && r.grid)
+    .map(bandOf)
+    .sort((a, b) => a.zoom - b.zoom);
+  const file = catalogue.find((r) => r.id === whole);
+  const wholePath = file.sources.find((s) => s.kind === 'file').url;
+
+  const index = new Map(); // dir -> promise of its Set of cell keys, or null when unreadable
+  const loadIndex = (b) => {
+    if (!index.has(b.dir))
+      index.set(
+        b.dir,
+        read(`${b.dir}/index.json`).then(
+          (i) => new Set(i.cells),
+          () => {
+            index.delete(b.dir); // skipped for this view, tried again for the next
+            return null;
+          },
+        ),
+      );
+    return index.get(b.dir);
+  };
+
+  const pending = new Map(); // path -> promise of its rings
+  const got = new Map(); // path -> its rings, once read
+  const fetch = (path, ringsOf) => {
+    if (!pending.has(path))
+      pending.set(
+        path,
+        read(path).then(
+          (json) => {
+            const rings = ringsOf(json);
+            got.set(path, rings);
+            return rings;
+          },
+          (e) => {
+            pending.delete(path); // a failed read is tried again for the next view
+            throw e;
+          },
+        ),
+      );
+    return pending.get(path);
+  };
+  const cellRings = (cell) => (part === 'land' ? cell.land.flat() : cell.coast);
+
+  // The bands the view may draw from, finest first, each with its index: passed, with an index.
+  async function candidates(view) {
+    const zoom = zoomOf(view);
+    const all = bands.filter((b) => passed(zoom, b));
+    const known = await Promise.all(all.map(loadIndex));
+    return all
+      .map((b, i) => ({ b, known: known[i] }))
+      .filter((c) => c.known !== null)
+      .reverse();
+  }
+
+  // What the view draws from: the finest candidate band with every cell it needs read, else the
+  // whole file.
+  async function draw(view) {
+    for (const { b, known } of await candidates(view)) {
+      const keys = cellsAround(b, view, known);
+      try {
+        const parts = await Promise.all(keys.map((k) => fetch(`${k}.json`, cellRings)));
+        return { band: b.id, parts };
+      } catch {
+        // a cell it lacks: the band below draws the whole view, rather than leave a hole
+      }
+    }
+    return { band: file.id, parts: [await fetch(wholePath, geojsonRings)] };
+  }
+
+  // The band the view would ask for first, and the paths it needs.
+  async function wants(view) {
+    const [first] = await candidates(view);
+    if (!first) return { band: file.id, paths: [wholePath], bytes: () => file.bytes };
+    const { b, known } = first;
+    const paths = cellsAround(b, view, known).map((k) => `${k}.json`);
+    return {
+      band: b.id,
+      paths,
+      bytes: (n) => (b.bytesPerCell == null ? null : n * b.bytesPerCell),
+    };
+  }
+
+  return {
+    async *rings(view) {
+      for (const p of (await draw(view)).parts) yield* p;
+    },
+    async held(view) {
+      return (await wants(view)).paths.every((p) => got.has(p));
+    },
+    async cost(view) {
+      const { band, paths, bytes } = await wants(view);
+      const cells = paths.filter((p) => !got.has(p)).length;
+      return { band, cells, bytes: cells ? bytes(cells) : 0 };
+    },
+  };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "coastlines",
-  "version": "0.0.6-alpha.0",
+  "version": "0.0.8-alpha.0",
   "description": "Natural Earth geography on a boat, installed rather than fetched: the marine catalogue at three scales, byte-for-byte as published, sha256-pinned, as plain GeoJSON today and a Signal K resource provider next",
   "type": "module",
   "main": "lib/index.js",
@@ -10,6 +10,7 @@
     "./charts.json": "./charts.json",
     "./catalogue": "./lib/bands.js",
     "./catalogue.json": "./catalogue.json",
+    "./provider": "./lib/provider.js",
     "./data/*": "./data/*.geojson",
     "./data/*.shp": "./data/*.shp",
     "./data/*.txt": "./data/*.txt",
@@ -18,10 +19,10 @@
   },
   "scripts": {
     "test": "node --test",
-    "sync": "node scripts/sync.mjs",
-    "sync:check": "node scripts/sync.mjs --check",
-    "sync:refresh": "node scripts/sync.mjs --refresh",
-    "prepack": "node scripts/sync.mjs",
+    "sync": "node --max-old-space-size=6144 scripts/sync.mjs",
+    "sync:check": "node --max-old-space-size=6144 scripts/sync.mjs --check",
+    "sync:refresh": "node --max-old-space-size=6144 scripts/sync.mjs --refresh",
+    "prepack": "node scripts/sync.mjs --no-cells",
     "build:sheet": "node scripts/render-sheet.mjs",
     "build:readme": "node scripts/catalogue-table.mjs",
     "build:pages": "node scripts/build-pages.mjs",