coastlines 0.0.7-alpha.0 → 0.0.9-alpha.0

package/README.md

@@ -111,6 +111,32 @@ the GSHHG zip. `coastlines/manifest.json` is the machine-readable form of this
 table, with the bytes and sha256 of each file and, under `sources`, of the
 zip itself.
 
+### Installed, built, or cached
+
+| Where | What | Size | How it reaches you |
+| --- | --- | ---: | --- |
+| `data/` | the 36 files above | 78 MB, 29 MB packed | installed with the package |
+| `geo/` | six cut bands, land and water in square cells | 549 MB | built by `npm run sync`; not shipped |
+| `cache/` | the GSHHG zip, six Natural Earth 10m water files | 167 MB | fetched once by sync, kept |
+
+Each cut band is a record in `catalogue.json`: grid, inputs and pins, cutter
+and build date. Cells and size are measured from the build (file content;
+4 KB disk blocks bring `geo/` to about 900 MB on disk). `cache/` holds the
+inputs those builds draw on, pinned but never shipped.
+
+<!-- cut-bands:start -->
+| Band | Layer | Zoom | Cell | Cells | Size | Source |
+| --- | --- | --- | --- | ---: | ---: | --- |
+| `ne-10m-land` | land | 9–10 | 5° (+2°) | 1,456 | 18.0 MB | Natural Earth v5.1.2, public domain |
+| `gshhg-h-land` | land | 10–12 | 1° (+0.5°) | 19,273 | 68.3 MB | GSHHG 2.3.7, LGPL-3.0-or-later |
+| `gshhg-f-land` | land | 12–∞ | 0.5° (+0.25°) | 70,692 | 383.8 MB | GSHHG 2.3.7, LGPL-3.0-or-later |
+| `ne-10m-water` | water | 7–∞ | 5° (+2°) | 712 | 31.0 MB | Natural Earth v5.1.2, public domain |
+| `gshhg-h-lakes` | lakes | 10–12 | 1° (+0.5°) | 4,068 | 11.4 MB | GSHHG 2.3.7, LGPL-3.0-or-later |
+| `gshhg-f-lakes` | lakes | 12–∞ | 0.5° (+0.25°) | 8,699 | 36.6 MB | GSHHG 2.3.7, LGPL-3.0-or-later |
+<!-- cut-bands:end -->
+
+How cells reach a boat that installs the package is open ([#17](https://github.com/mark-brannan/coastlines/issues/17)).
+
 ### Sources and licences
 
 Natural Earth is public domain. GSHHG (Wessel & Smith, the Global
@@ -146,7 +172,7 @@ release and refuses any byte that disagrees with `manifest.json`; `npm pack`
 runs it first, so a tarball never ships a file the manifest did not promise.
 
 ```shell
-npm run sync          # fetch what is missing, verify everything (45 MB + 149 MB zip)
+npm run sync          # fetch what is missing, verify everything, cut the cells (~2 min, 6 GB heap)
 npm run sync:check    # verify only, no network
 npm run sync:refresh  # move the pins: refetch all and rewrite bytes and sha256
 ```

package/catalogue.json

@@ -1046,10 +1046,13 @@
       "licence": "public domain",
       "version": "v5.1.2",
       "date": "2022-05-13",
-      "caveat": "Holds to about z12 but has no narrow channels"
+      "caveat": "Holds to about z12 but has no narrow channels",
+      "cutter": "coastlines/lib/cut.js",
+      "built": "2026-10-15"
     },
-    "bytesPerCell": null,
-    "bytes": null,
+    "cells": 1456,
+    "bytesPerCell": 12954,
+    "bytes": 18861693,
     "from": ["ne-land-10m", "ne-coastline-10m", "ne-minor-islands-10m"],
     "inputs": [],
     "sources": [
@@ -1086,10 +1089,13 @@
       "licence": "LGPL-3.0-or-later",
       "version": "2.3.7",
       "date": "2017-06-15",
-      "caveat": "Holds to about z13; draws the Seattle ship canal. Its licence files are owned here and travel with every GSHHG band"
+      "caveat": "Holds to about z13; draws the Seattle ship canal. Its licence files are owned here and travel with every GSHHG band",
+      "cutter": "coastlines/lib/cut.js",
+      "built": "2026-10-15"
     },
-    "bytesPerCell": null,
-    "bytes": null,
+    "cells": 19273,
+    "bytesPerCell": 3717,
+    "bytes": 71636830,
     "from": [],
     "inputs": ["gshhg-land-h", "gshhg-licence", "gshhg-copying-lesser"],
     "sources": [
@@ -1118,10 +1124,13 @@
       "licence": "LGPL-3.0-or-later",
       "version": "2.3.7",
       "date": "2017-06-15",
-      "caveat": "Full resolution; about 70,000 cells"
+      "caveat": "Full resolution; about 70,000 cells",
+      "cutter": "coastlines/lib/cut.js",
+      "built": "2026-10-15"
     },
-    "bytesPerCell": null,
-    "bytes": null,
+    "cells": 70692,
+    "bytesPerCell": 5693,
+    "bytes": 402415178,
     "from": [],
     "inputs": [
       {
@@ -1169,10 +1178,13 @@
       "licence": "public domain",
       "version": "v5.1.2",
       "date": "2022-05-13",
-      "caveat": "Rivers and lakes with their regional supplements; whole features go into every cell they reach. Not in the manifest, so each file is pinned here"
+      "caveat": "Rivers and lakes with their regional supplements; whole features go into every cell they reach. Not in the manifest, so each file is pinned here",
+      "cutter": "coastlines/lib/cut.js",
+      "built": "2026-10-15"
     },
-    "bytesPerCell": null,
-    "bytes": null,
+    "cells": 712,
+    "bytesPerCell": 45613,
+    "bytes": 32476615,
     "from": [],
     "inputs": [
       {
@@ -1247,10 +1259,13 @@
       "licence": "LGPL-3.0-or-later",
       "version": "2.3.7",
       "date": "2017-06-15",
-      "caveat": "Lakes no Natural Earth file has (Dian Chi, Lago Mascardi); levels 2 and 3 with river-lakes"
+      "caveat": "Lakes no Natural Earth file has (Dian Chi, Lago Mascardi); levels 2 and 3 with river-lakes",
+      "cutter": "coastlines/lib/cut.js",
+      "built": "2026-10-15"
     },
-    "bytesPerCell": null,
-    "bytes": null,
+    "cells": 4068,
+    "bytesPerCell": 2935,
+    "bytes": 11939267,
     "from": [],
     "inputs": [
       {
@@ -1311,10 +1326,13 @@
       "licence": "LGPL-3.0-or-later",
       "version": "2.3.7",
       "date": "2017-06-15",
-      "caveat": "As gshhg-h-lakes at full resolution"
+      "caveat": "As gshhg-h-lakes at full resolution",
+      "cutter": "coastlines/lib/cut.js",
+      "built": "2026-10-15"
     },
-    "bytesPerCell": null,
-    "bytes": null,
+    "cells": 8699,
+    "bytesPerCell": 4417,
+    "bytes": 38424112,
     "from": [],
     "inputs": [
       {

package/lib/acquire.js

@@ -0,0 +1,45 @@
+// Pinned-source acquisition: give it `{url, sha256}` (a manifest pin, or any
+// source pinned the same way) and get back verified bytes from a local cache.
+// Cache hit: no fetch. Miss, or a cached file that no longer matches: fetch
+// with retry, check, keep. A wrong sha256 throws and nothing is stored or
+// retried -- retry changes how bytes arrive, never whether they are believed.
+import { mkdir, readFile, rename, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { digest } from './verify.js';
+import { fetchBytesWithRetry } from './fetch-retry.js';
+
+async function cached(file) {
+  try {
+    return await readFile(file);
+  } catch (error) {
+    if (error.code === 'ENOENT') return null;
+    throw error;
+  }
+}
+
+/**
+ * The bytes pinned by `{url, sha256, bytes?}`, from `cacheDir/<name>` when they are there and
+ * right, else fetched, checked and kept. `name` defaults to the url's last path segment (empty for a url ending in /) and must be a file name, not a
+ * path. `fetchImpl`, `sleep`, `waits` and `onRetry` go to the retry; `onFetch(url)` is told before a download.
+ */
+export async function acquire(
+  { url, sha256, bytes: length },
+  { cacheDir, name = new URL(url).pathname.split('/').pop(), onFetch, ...retry },
+) {
+  if (!name || name !== name.split(/[/\\]/).pop() || name === '..' || name === '.')
+    throw new Error(`${name}: a cache name is a file name, not a path`);
+  const file = join(cacheDir, name);
+  const held = await cached(file);
+  if (held && digest(held) === sha256) return held;
+  onFetch?.(url);
+  const bytes = await fetchBytesWithRetry(url, retry);
+  const actual = digest(bytes);
+  if (actual !== sha256) throw new Error(`${name}: ${actual}, pinned ${sha256}`);
+  if (length !== undefined && bytes.byteLength !== length)
+    throw new Error(`${name}: ${bytes.byteLength} bytes, pinned ${length}`);
+  await mkdir(cacheDir, { recursive: true });
+  // Sibling then rename: an interrupted run leaves no truncated file to fail every later one.
+  await writeFile(`${file}.partial`, bytes);
+  await rename(`${file}.partial`, file);
+  return bytes;
+}

package/lib/bands.js

@@ -16,6 +16,9 @@ import { charts } from './charts.js';
 
 const require = createRequire(import.meta.url);
 
+/** What a cut band's `provenance.cutter` names: the module that builds its cells. */
+export const CUTTER = 'coastlines/lib/cut.js';
+
 /** The parsed catalogue.json: one record per band, whole Natural Earth files first. */
 export const catalogue = require('../catalogue.json');
 
@@ -46,6 +49,14 @@ export function inputsOf(record) {
   return [...own, ...inherited];
 }
 
+/** The source keys a record's geometry is built from; a geometry file must name exactly one. */
+export const sourcesOf = (record) =>
+  new Set(
+    inputsOf(record).map((input) =>
+      typeof input === 'string' ? find(input).source : input.source,
+    ),
+  );
+
 /** The zoom range charts.json draws a whole file at, across every chart that lists it; nulls when none does. */
 function chartZoom(id) {
   const layers = charts.flatMap((chart) => chart.layers).filter((layer) => layer.id === id);
@@ -72,6 +83,16 @@ function wholeFileDrift(record) {
   return problems;
 }
 
+/** A cut band names its cutter and the day it was built. */
+function cutDrift(record) {
+  const problems = [];
+  const { cutter, built } = record.provenance;
+  if (cutter !== CUTTER) problems.push(`${record.id}: cutter ${cutter} is not ${CUTTER}`);
+  if (!/^\d{4}-\d{2}-\d{2}$/.test(built ?? ''))
+    problems.push(`${record.id}: built ${built} is not a date`);
+  return problems;
+}
+
 /**
  * Where the catalogue and the manifest disagree, as sentences; empty when they agree. Every
  * record's upstream URLs, members and sha256s must be what its inputs pin, its licence, version
@@ -93,9 +114,11 @@ export function drift() {
         if (JSON.stringify(record.provenance[field]) !== want)
           problems.push(`${record.id}: ${field} differs from its inputs`);
       }
-      const keys = new Set(
-        inputs.map((input) => (typeof input === 'string' ? find(input).source : input.source)),
-      );
+      const keys = sourcesOf(record);
+      if (keys.size !== 1)
+        problems.push(
+          `${record.id}: built from ${keys.size} sources (${[...keys].join(', ')}), not one; each licence stays in its own file`,
+        );
       for (const key of keys) {
         const from = manifest.sources[key];
         if (licence !== from.licence)
@@ -105,6 +128,7 @@ export function drift() {
         if (date !== from.date) problems.push(`${record.id}: date ${date} is not ${key}'s`);
       }
       if (record.grid === null) problems.push(...wholeFileDrift(record));
+      else problems.push(...cutDrift(record));
     } catch (error) {
       problems.push(`${record.id}: ${error.message}`);
     }

package/lib/cut.js

@@ -0,0 +1,140 @@
+// Builds the catalogue's `cells` sources: each cut band's inputs, read through `read`, cut into
+// square lon/lat cells under geo/<dir>/ (<xi>_<yi>.json and index.json). A band is cut from one
+// upstream only, so no geometry file mixes sources.
+import { mkdir, rename, rm, writeFile } from 'node:fs/promises';
+import { band, CUTTER, inputsOf, sourcesOf } from './bands.js';
+import { find } from './catalogue.js';
+import { shapefilePolygons } from './shapefile.js';
+import { cutCells, cutLakes, cutWater, dbfRecords, gshhgLakes, joinWater } from './detail-cells.js';
+
+/** The folder under geo/ of a band's `cells` source: `geo/f/` is `f`. */
+export const cellsDir = (record) =>
+  record.sources.find((source) => source.kind === 'cells').url.replace(/^geo\/|\/$/g, '');
+
+const json = (bytes) => JSON.parse(new TextDecoder().decode(bytes));
+
+function polygons(geojson) {
+  const out = [];
+  for (const { geometry } of geojson.features) {
+    if (geometry?.type === 'Polygon') out.push(geometry.coordinates);
+    else if (geometry?.type === 'MultiPolygon') for (const p of geometry.coordinates) out.push(p);
+  }
+  return out;
+}
+
+function lines(geojson) {
+  const out = [];
+  for (const { geometry } of geojson.features) {
+    if (geometry?.type === 'LineString') out.push(geometry.coordinates);
+    else if (geometry?.type === 'MultiLineString')
+      for (const l of geometry.coordinates) out.push(l);
+  }
+  return out;
+}
+
+/**
+ * The cells of a band: Map(key -> cell). `read(input)` returns the bytes of one input, as
+ * `inputsOf(record)` lists them.
+ */
+export async function cutBand(record, read) {
+  if (sourcesOf(record).size !== 1)
+    throw new Error(`${record.id}: inputs from more than one source`);
+  const size = record.grid.cellDeg;
+  const bytes = await Promise.all(inputsOf(record).map(read));
+  if (record.id === 'ne-10m-land') {
+    const [land, coast, minor] = bytes.map(json);
+    return cutCells(
+      { land: polygons(land), coast: [...lines(coast), ...polygons(minor).map((p) => p[0])] },
+      size,
+    );
+  }
+  if (record.layer === 'land') {
+    const land = shapefilePolygons(bytes[0]);
+    return cutCells({ land, coast: land.flat() }, size);
+  }
+  if (record.layer === 'water') {
+    const [rivers, lakes] = [bytes.slice(0, 3), bytes.slice(3)].map((half) =>
+      joinWater(half.map(json)),
+    );
+    return cutWater(rivers, lakes, { size });
+  }
+  if (record.layer === 'lakes') {
+    const [l2, d2, l3, d3] = bytes;
+    return cutLakes(
+      gshhgLakes(shapefilePolygons(l2), dbfRecords(d2), shapefilePolygons(l3), dbfRecords(d3)),
+      size,
+    );
+  }
+  throw new Error(`${record.id}: no cutter for layer ${record.layer}`);
+}
+
+/** Writes a band's cells to `root`/geo/<dir>/, complete or not at all; returns the cell count. */
+export async function writeBand(record, cells, root) {
+  const final = new URL(`geo/${cellsDir(record)}/`, root);
+  const partial = new URL(`geo/${cellsDir(record)}.partial/`, root);
+  await rm(partial, { recursive: true, force: true });
+  await mkdir(partial, { recursive: true });
+  // in batches: "f" has some 70,000 cells, past an open-file limit if written at once
+  const all = [...cells];
+  for (let i = 0; i < all.length; i += 512)
+    await Promise.all(
+      all
+        .slice(i, i + 512)
+        .map(([k, c]) => writeFile(new URL(`${k}.json`, partial), JSON.stringify(c))),
+    );
+  await writeFile(
+    new URL('index.json', partial),
+    JSON.stringify({ cell: record.grid.cellDeg, cells: [...cells.keys()] }),
+  );
+  await rm(final, { recursive: true, force: true });
+  await rename(partial, final);
+  return cells.size;
+}
+
+/**
+ * catalogue.json's text with one record's `provenance.built` set to `day`, every other byte as it
+ * was. Throws rather than touch the wrong record: the id must be there, and the `built` must sit
+ * in that record, before the next `"id":`.
+ */
+export function stampBuilt(text, id, day) {
+  const at = text.indexOf(`"id": "${id}"`);
+  if (at < 0) throw new Error(`${id}: not in catalogue.json`);
+  const end = text.indexOf('"id": "', at + 1);
+  const record = text.slice(at, end < 0 ? undefined : end);
+  const stamped = record.replace(/("built": )"[^"]*"/, `$1"${day}"`);
+  if (stamped === record && !record.includes(`"built": "${day}"`))
+    throw new Error(`${id}: no provenance.built to stamp`);
+  return text.slice(0, at) + stamped + (end < 0 ? '' : text.slice(end));
+}
+
+/** The cut bands whose inputs are GSHHG's, which the notice below covers. */
+export const modifiedGshhg = (catalogue) =>
+  catalogue.filter(
+    (r) =>
+      r.grid &&
+      inputsOf(r).some((i) => (typeof i === 'string' ? find(i).source : i.source) === 'gshhg'),
+  );
+
+/** The text of geo/gshhg-modified.txt for the bands built, with the dates they were cut. */
+export function modifiedNotice(records) {
+  const rows = records
+    .map((r) => `  geo/${cellsDir(r)}/   ${r.id}, cut ${r.provenance.built}`)
+    .join('\n');
+  return `GSHHG, modified
+===============
+
+The cell files below are modified GSHHG ${band(records[0].id).provenance.version} (Wessel & Smith), not the
+published shapefiles. They were made by ${CUTTER} from the pinned zip: polygons
+clipped to square lon/lat cells, shorelines split where they pass from one cell to the next,
+coordinates rounded to 5 decimals, and each land polygon annotated with the area and colour of the
+island it is a piece of.
+
+${rows}
+
+Licence: GNU Lesser General Public License version 3 or later, conveyed under section 5(a) of the
+GNU General Public License version 3, which asks for a notice that the work was modified and the
+date of the modification: this file. The unmodified originals are fetched, byte for byte, by
+\`npm run sync\`; their copyright and permission notice and the licence text are in
+data/gshhg-licence.txt and data/gshhg-copying-lesser.txt and apply to the cells too.
+`;
+}

package/lib/detail-cells.js

@@ -0,0 +1,313 @@
+// Cuts Natural Earth 10m land and coast into square lon/lat cells, so a close view fetches only the
+// few cells around it rather than the whole 10m world. Land polygons are clipped to each cell (fill
+// is even-odd, so the cut edges vanish; each cell's `m` holds, for its land polygons in order, the
+// area in km2 and colour of the island each is a piece of, worked out before the cut); coast lines are split where they pass from one cell to the
+// next, each piece keeping the shared point so the line stays joined. Coordinates are rounded to
+// `digits` decimals (5 is about a metre). Cell keys are `${xi}_${yi}`, xi = floor(lon / size).
+
+import { islandInfo } from './island.js';
+
+const key = (xi, yi) => `${xi}_${yi}`;
+
+// Sutherland-Hodgman against one axis-aligned half-plane: keep points with p[axis] <= v (keep < 0)
+// or >= v (keep > 0). `ring` is open (last point not repeated).
+function clipHalf(ring, axis, v, keep) {
+  const out = [],
+    inside = (p) => (keep < 0 ? p[axis] <= v : p[axis] >= v);
+  for (let i = 0; i < ring.length; i++) {
+    const a = ring[i],
+      b = ring[(i + 1) % ring.length],
+      ia = inside(a),
+      ib = inside(b);
+    if (ia) out.push(a);
+    if (ia !== ib) {
+      const t = (v - a[axis]) / (b[axis] - a[axis]);
+      const p = [a[0] + t * (b[0] - a[0]), a[1] + t * (b[1] - a[1])];
+      p[axis] = v;
+      out.push(p);
+    }
+  }
+  return out;
+}
+
+function ringBox(ring) {
+  let x0 = Infinity,
+    y0 = Infinity,
+    x1 = -Infinity,
+    y1 = -Infinity;
+  for (const [x, y] of ring) {
+    if (x < x0) x0 = x;
+    if (x > x1) x1 = x;
+    if (y < y0) y0 = y;
+    if (y > y1) y1 = y;
+  }
+  return [x0, y0, x1, y1];
+}
+
+// Polygons ([[outer, ...holes]], open rings) into cells by recursive halving along cell lines, so
+// each point is clipped about log2(cells) times rather than once per cell.
+function splitPolygons(polys, box, size, emit) {
+  if (!polys.length) return;
+  const [x0, y0, x1, y1] = box,
+    nx = Math.round((x1 - x0) / size),
+    ny = Math.round((y1 - y0) / size);
+  if (nx <= 1 && ny <= 1) {
+    emit(Math.round(x0 / size), Math.round(y0 / size), polys);
+    return;
+  }
+  const axis = nx >= ny ? 0 : 1,
+    mid = axis === 0 ? x0 + Math.floor(nx / 2) * size : y0 + Math.floor(ny / 2) * size;
+  const lo = [],
+    hi = [];
+  for (const poly of polys) {
+    const b = ringBox(poly[0]);
+    if (b[axis + 2] <= mid) {
+      lo.push(poly);
+      continue;
+    }
+    if (b[axis] >= mid) {
+      hi.push(poly);
+      continue;
+    }
+    for (const [side, keep] of [
+      [lo, -1],
+      [hi, 1],
+    ]) {
+      const [outer, ...holes] = poly.map((r) => clipHalf(r, axis, mid, keep));
+      if (outer.length >= 3)
+        side.push(Object.assign([outer, ...holes.filter((r) => r.length >= 3)], { isl: poly.isl }));
+    }
+  }
+  const loBox = axis === 0 ? [x0, y0, mid, y1] : [x0, y0, x1, mid],
+    hiBox = axis === 0 ? [mid, y0, x1, y1] : [x0, mid, x1, y1];
+  splitPolygons(lo, loBox, size, emit);
+  splitPolygons(hi, hiBox, size, emit);
+}
+
+// `land`: GeoJSON polygons as [[outer, ...holes]] of closed [lon, lat] rings; `coast`: lines of
+// [lon, lat]. Returns Map(key -> { land, coast, m }), rings closed again, coordinates rounded; `m`
+// holds `meta` of the whole polygon each land piece was cut from (by default its island's area and colour).
+const islandMeta = (poly) => {
+  const i = islandInfo(poly);
+  return [i.km2, i.hue];
+};
+export function cutCells({ land, coast }, size = 5, digits = 5, meta = islandMeta) {
+  const f = 10 ** digits,
+    round = (p) => [Math.round(p[0] * f) / f, Math.round(p[1] * f) / f];
+  const tidy = (pts) =>
+    pts.map(round).filter((p, i, a) => i === 0 || p[0] !== a[i - 1][0] || p[1] !== a[i - 1][1]);
+  const cells = new Map(),
+    cell = (k) => cells.get(k) ?? cells.set(k, { land: [], coast: [], m: [] }).get(k);
+  const ids = coast.some((l) => l.id !== undefined); // and if lines carry an `id`, `cm` holds each piece's
+
+  const open = (r) =>
+    r.length > 1 && r[0][0] === r[r.length - 1][0] && r[0][1] === r[r.length - 1][1]
+      ? r.slice(0, -1)
+      : r;
+  splitPolygons(
+    land.map((poly) => Object.assign(poly.map(open), { isl: meta(poly) })),
+    [-180, -90, 180, 90],
+    size,
+    (xi, yi, polys) => {
+      const c = cell(key(xi, yi)),
+        out = c.land;
+      for (const poly of polys) {
+        const rings = poly
+          .map(tidy)
+          .filter((r) => r.length >= 3)
+          .map((r) => [...r, r[0]]);
+        if (rings.length) {
+          out.push(rings);
+          c.m.push(poly.isl);
+        }
+      }
+    },
+  );
+
+  // A segment belongs to the cell holding its midpoint; a run of segments in one cell is one piece.
+  const cellOf = (a, b) =>
+    key(Math.floor((a[0] + b[0]) / 2 / size), Math.floor((a[1] + b[1]) / 2 / size));
+  const put = (k, piece, id) => {
+    const c = cell(k);
+    c.coast.push(piece);
+    if (ids) (c.cm ??= []).push(id);
+  };
+  for (const line of coast) {
+    let piece = null,
+      at = null;
+    for (let i = 0; i < line.length - 1; i++) {
+      const k = cellOf(line[i], line[i + 1]);
+      if (k !== at) {
+        if (piece) put(at, piece, line.id);
+        piece = [line[i]];
+        at = k;
+      }
+      piece.push(line[i + 1]);
+    }
+    if (piece) put(at, piece, line.id);
+  }
+  for (const c of cells.values()) {
+    const lines = c.coast.map(tidy),
+      keep = lines.map((l) => l.length > 1);
+    c.coast = lines.filter((_, i) => keep[i]);
+    if (c.cm) c.cm = c.cm.filter((_, i) => keep[i]);
+  }
+  return cells;
+}
+
+// The Natural Earth 10m water downloads, the world file first and then the regional supplements,
+// which hold what the world file leaves out (Lake Washington, Como, the Snohomish) and repeat none of
+// it. joinWater makes one collection of several, in this order, so cutWater's ids stay unique.
+export const WATER_FILES = {
+  rivers: [
+    'ne_10m_rivers_lake_centerlines.geojson',
+    'ne_10m_rivers_north_america.geojson',
+    'ne_10m_rivers_europe.geojson',
+  ],
+  lakes: [
+    'ne_10m_lakes.geojson',
+    'ne_10m_lakes_north_america.geojson',
+    'ne_10m_lakes_europe.geojson',
+  ],
+};
+export const joinWater = (geos) => ({
+  type: 'FeatureCollection',
+  features: geos.flatMap((g) => g?.features ?? []),
+});
+
+// Natural Earth rivers and lakes into the same square cells, for scripts/water.mjs past WATER_ZOOM.
+// A feature goes whole into every cell it reaches (a river, each cell its segments' midpoints and
+// vertices fall in; a lake, each cell its box meets), so a river keeps one wiggle and a lake one
+// outline wherever the view is cut; its index in the source file is its GeoJSON `id`, by which
+// the scene keeps one copy of a feature fetched from several cells. Properties are cut to `keep`.
+// Returns Map(key -> { rivers, lakes }), each a FeatureCollection.
+export function cutWater(
+  riversGeo,
+  lakesGeo,
+  {
+    keep = { rivers: ['name', 'featurecla', 'min_zoom'], lakes: ['name', 'min_zoom'] },
+    size = 5,
+    digits = 5,
+  } = {},
+) {
+  const f = 10 ** digits,
+    round = (c) =>
+      typeof c[0] === 'number'
+        ? [Math.round(c[0] * f) / f, Math.round(c[1] * f) / f]
+        : c.map(round);
+  const cells = new Map(),
+    cell = (k) => cells.get(k) ?? cells.set(k, { rivers: [], lakes: [] }).get(k);
+  // 180 east and 90 north fall in the last cell, not one past it
+  const xiOf = (lon) => Math.min(180 / size - 1, Math.floor(lon / size)),
+    yiOf = (lat) => Math.min(90 / size - 1, Math.floor(lat / size));
+  const at = (lon, lat) => key(xiOf(lon), yiOf(lat));
+  const lines = (g) =>
+    g.type === 'LineString'
+      ? [g.coordinates]
+      : g.type === 'MultiLineString'
+        ? g.coordinates
+        : g.type === 'Polygon'
+          ? g.coordinates
+          : g.type === 'MultiPolygon'
+            ? g.coordinates.flat()
+            : [];
+  const reach = {
+    rivers: (g) => {
+      const ks = new Set();
+      for (const l of lines(g))
+        l.forEach((p, i) => {
+          ks.add(at(p[0], p[1]));
+          if (i) ks.add(at((p[0] + l[i - 1][0]) / 2, (p[1] + l[i - 1][1]) / 2));
+        });
+      return ks;
+    },
+    lakes: (g) => {
+      const [x0, y0, x1, y1] = ringBox(lines(g).flat()),
+        ks = new Set();
+      for (let xi = xiOf(x0); xi <= xiOf(x1); xi++)
+        for (let yi = yiOf(y0); yi <= yiOf(y1); yi++) ks.add(key(xi, yi));
+      return ks;
+    },
+  };
+  for (const [kind, geo] of [
+    ['rivers', riversGeo],
+    ['lakes', lakesGeo],
+  ]) {
+    (geo?.features ?? []).forEach((ft, id) => {
+      if (!ft.geometry) return;
+      const slim = {
+        type: 'Feature',
+        id,
+        properties: Object.fromEntries(keep[kind].map((k) => [k, ft.properties[k]])),
+        geometry: { type: ft.geometry.type, coordinates: round(ft.geometry.coordinates) },
+      };
+      for (const k of reach[kind](ft.geometry)) cell(k)[kind].push(slim);
+    });
+  }
+  for (const [k, c] of cells)
+    cells.set(k, {
+      rivers: { type: 'FeatureCollection', features: c.rivers },
+      lakes: { type: 'FeatureCollection', features: c.lakes },
+    });
+  return cells;
+}
+
+// GSHHG lakes (level 2, river-lakes among them) with their islands (level 3) as holes, from the
+// shapefiles' polygons and dBASE records, for cutLakes. Each lake carries its GSHHG id and its
+// whole lon/lat box, which its cut pieces keep: the scene drops a 10m lake inside one.
+export function dbfRecords(bytes) {
+  const b = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  const n = b.readUInt32LE(4),
+    headLength = b.readUInt16LE(8),
+    recordLength = b.readUInt16LE(10),
+    fields = [];
+  for (let at = 32, o = 1; b[at] !== 0x0d; at += 32) {
+    const name = b.toString('latin1', at, at + 11).replace(/\0.*/s, ''),
+      length = b[at + 16];
+    fields.push([name, o, length]);
+    o += length;
+  }
+  return Array.from({ length: n }, (_, i) =>
+    Object.fromEntries(
+      fields.map(([k, o, l]) => [
+        k,
+        b
+          .toString(
+            'latin1',
+            headLength + i * recordLength + o,
+            headLength + i * recordLength + o + l,
+          )
+          .trim(),
+      ]),
+    ),
+  );
+}
+export function gshhgLakes(lakes, lakeRecords, islands, islandRecords) {
+  const byId = new Map(
+    lakes.map((poly, i) => [
+      lakeRecords[i].id,
+      Object.assign([...poly], { id: +lakeRecords[i].id }),
+    ]),
+  );
+  islands.forEach((poly, i) => byId.get(islandRecords[i].parent_id)?.push(poly[0]));
+  return [...byId.values()];
+}
+// Lakes into cells as land is (cutCells): pieces clipped to each cell to fill, and the shores split
+// at cell edges to ink, so a cut edge is never drawn. `m` holds, per piece, [id, x0, y0, x1, y1]: the
+// lake's id and whole box, rounded out to `digits`; `cm` each shore piece's lake id.
+export function cutLakes(lakes, size, digits = 5) {
+  const f = 10 ** digits;
+  const shores = lakes.flatMap((lake) =>
+    lake.map((ring) => Object.assign([...ring], { id: lake.id })),
+  );
+  return cutCells({ land: lakes, coast: shores }, size, digits, (poly) => {
+    const [x0, y0, x1, y1] = ringBox(poly[0]);
+    return [
+      poly.id,
+      Math.floor(x0 * f) / f,
+      Math.floor(y0 * f) / f,
+      Math.ceil(x1 * f) / f,
+      Math.ceil(y1 * f) / f,
+    ];
+  });
+}

package/lib/island.js

@@ -0,0 +1,51 @@
+// What the land treatments need to know about an island: its area in km2 and a colour that stays
+// with it. Computed once per land polygon, from the whole polygon (so before lib/detail-cells.js
+// cuts it into cells, which carry the result along), and shared by the scene and the cutter.
+
+export const ISLAND_KM2 = 5000; // coloured islands are the land polygons smaller than this
+export const ISLAND_HUES = 3; // red, green, gold
+
+const R_KM = 6371.0088,
+  RAD = Math.PI / 180;
+
+// Area of a [lon, lat] ring (open or closed) in km2: equirectangular about the ring's own latitude,
+// good to a percent or two at this size.
+function ringKm2(ring) {
+  const lat0 = ring.reduce((s, p) => s + p[1], 0) / ring.length,
+    lon0 = ring[0][0],
+    c = Math.cos(lat0 * RAD);
+  const pt = ([lon, lat]) => [(((lon - lon0 + 540) % 360) - 180) * c, lat];
+  let s = 0;
+  for (let i = 0; i < ring.length; i++) {
+    const [x1, y1] = pt(ring[i]),
+      [x2, y2] = pt(ring[(i + 1) % ring.length]);
+    s += x1 * y2 - x2 * y1;
+  }
+  return (Math.abs(s) / 2) * (R_KM * RAD) ** 2;
+}
+
+// { km2, hue } of a polygon [outer, ...holes]. The hue comes from the centre of the outer ring's
+// box, to a tenth of a degree: the same island has the same colour in every view and, to within a
+// tenth of a degree of disagreement between data sets, at every zoom.
+export function islandInfo(poly) {
+  const km2 = Math.round(ringKm2(poly[0]) - poly.slice(1).reduce((s, r) => s + ringKm2(r), 0));
+  let x0 = Infinity,
+    x1 = -Infinity,
+    y0 = Infinity,
+    y1 = -Infinity;
+  const lon0 = poly[0][0][0];
+  for (const [lon, lat] of poly[0]) {
+    const x = ((lon - lon0 + 540) % 360) - 180;
+    if (x < x0) x0 = x;
+    if (x > x1) x1 = x;
+    if (lat < y0) y0 = lat;
+    if (lat > y1) y1 = lat;
+  }
+  let h =
+    Math.imul(Math.round((lon0 + (x0 + x1) / 2) * 10), 0x9e3779b1) ^
+    Math.imul(Math.round((y0 + y1) * 5), 0x85ebca6b);
+  h ^= h >>> 15;
+  h = Math.imul(h, 0x2c1b3c6d);
+  h ^= h >>> 13;
+  return { km2, hue: (h >>> 0) % ISLAND_HUES };
+}

package/lib/provider.js

@@ -3,7 +3,8 @@
 // neither package imports the other; the caller owns the projection and hands over a view.
 //
 // A view is the lon/lat frame a caller paints and its scale at the centre:
-// `{ lat, metresPerPixel, lonMin, lonMax, latMin, latMax }`. The frame may run past the
+// `{ lat, resolution, lonMin, lonMax, latMin, latMax }`, `resolution` in metres per pixel (the old
+// name, `metresPerPixel`, still reads, deprecated). The frame may run past the
 // antimeridian. The catalogue keys bands by web-mercator zoom (a 512 px world at zoom 0), so the
 // scale is turned into that zoom at the view's latitude to choose one.
 //
@@ -20,13 +21,20 @@ const EARTH_R = 6378137; // metres, web mercator's sphere
 const WORLD_PX = 512; // the width of the world at zoom 0, as the catalogue's zooms count it
 const RAD = Math.PI / 180;
 
+// A view's metres per pixel: `resolution`, or the deprecated `metresPerPixel` when that is all it has.
+const resolutionOf = (view) => {
+  if (view.resolution === undefined && view.metresPerPixel !== undefined)
+    warnOnce('metresPerPixel', 'resolution');
+  return view.resolution ?? view.metresPerPixel;
+};
+
 /** The web-mercator zoom of a view's scale at its centre. */
 // Rounded, so a view at a band's edge zoom, turned into a scale and back, does not pass it.
 export const zoomOf = (view) =>
   Math.round(
     1e9 *
       Math.log2(
-        (2 * Math.PI * EARTH_R * Math.cos(view.lat * RAD)) / (WORLD_PX * view.metresPerPixel),
+        (2 * Math.PI * EARTH_R * Math.cos(view.lat * RAD)) / (WORLD_PX * resolutionOf(view)),
       ),
   ) / 1e9;
 
@@ -95,9 +103,10 @@ function geojsonRings(geo) {
  * path (`geo/h/12_45.json`, `data/ne-coastline-50m.geojson`) and rejects when it cannot. `kind`
  * is `coast`, the coastline's lines, or `land`, the land polygons' rings.
  *
- * `rings(view)` yields the view's rings; `held(view)` says whether they are all in hand, so
- * drawing them fetches nothing; `cost(view)` is what drawing it would fetch: the band, the cells
- * not yet held, and their bytes where the catalogue knows them.
+ * `rings(view)` yields the view's rings; `loaded(view)` says whether they are all in hand, so
+ * drawing them fetches nothing; `pending(view)` is what drawing it would fetch: the band, the cells
+ * not yet loaded, and their bytes where the catalogue knows them. `held` and `cost` are the old
+ * names of those two.
  */
 export function provider({ read, kind = 'coast' }) {
   const { layer, part, whole } = KINDS[kind] ?? {};
@@ -125,11 +134,11 @@ export function provider({ read, kind = 'coast' }) {
     return index.get(b.dir);
   };
 
-  const pending = new Map(); // path -> promise of its rings
+  const inflight = new Map(); // path -> promise of its rings
   const got = new Map(); // path -> its rings, once read
   const fetch = (path, ringsOf) => {
-    if (!pending.has(path))
-      pending.set(
+    if (!inflight.has(path))
+      inflight.set(
         path,
         read(path).then(
           (json) => {
@@ -138,12 +147,12 @@ export function provider({ read, kind = 'coast' }) {
             return rings;
           },
           (e) => {
-            pending.delete(path); // a failed read is tried again for the next view
+            inflight.delete(path); // a failed read is tried again for the next view
             throw e;
           },
         ),
       );
-    return pending.get(path);
+    return inflight.get(path);
   };
   const cellRings = (cell) => (part === 'land' ? cell.land.flat() : cell.coast);
 
@@ -186,17 +195,38 @@ export function provider({ read, kind = 'coast' }) {
     };
   }
 
+  async function loaded(view) {
+    return (await wants(view)).paths.every((p) => got.has(p));
+  }
+
+  async function pending(view) {
+    const { band, paths, bytes } = await wants(view);
+    const cells = paths.filter((p) => !got.has(p)).length;
+    return { band, cells, bytes: cells ? bytes(cells) : 0 };
+  }
+
   return {
     async *rings(view) {
       for (const p of (await draw(view)).parts) yield* p;
     },
-    async held(view) {
-      return (await wants(view)).paths.every((p) => got.has(p));
+    loaded,
+    pending,
+    /** @deprecated Use `loaded`. */
+    held(view) {
+      warnOnce('held', 'loaded');
+      return loaded(view);
     },
-    async cost(view) {
-      const { band, paths, bytes } = await wants(view);
-      const cells = paths.filter((p) => !got.has(p)).length;
-      return { band, cells, bytes: cells ? bytes(cells) : 0 };
+    /** @deprecated Use `pending`. */
+    cost(view) {
+      warnOnce('cost', 'pending');
+      return pending(view);
     },
   };
 }
+
+const warned = new Set();
+function warnOnce(old, now) {
+  if (warned.has(old)) return;
+  warned.add(old);
+  console.warn(`coastlines provider: \`${old}\` is deprecated, use \`${now}\``);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "coastlines",
-  "version": "0.0.7-alpha.0",
+  "version": "0.0.9-alpha.0",
   "description": "Natural Earth geography on a boat, installed rather than fetched: the marine catalogue at three scales, byte-for-byte as published, sha256-pinned, as plain GeoJSON today and a Signal K resource provider next",
   "type": "module",
   "main": "lib/index.js",
@@ -11,6 +11,7 @@
     "./catalogue": "./lib/bands.js",
     "./catalogue.json": "./catalogue.json",
     "./provider": "./lib/provider.js",
+    "./acquire": "./lib/acquire.js",
     "./data/*": "./data/*.geojson",
     "./data/*.shp": "./data/*.shp",
     "./data/*.txt": "./data/*.txt",
@@ -19,10 +20,10 @@
   },
   "scripts": {
     "test": "node --test",
-    "sync": "node scripts/sync.mjs",
-    "sync:check": "node scripts/sync.mjs --check",
-    "sync:refresh": "node scripts/sync.mjs --refresh",
-    "prepack": "node scripts/sync.mjs",
+    "sync": "node --max-old-space-size=6144 scripts/sync.mjs",
+    "sync:check": "node --max-old-space-size=6144 scripts/sync.mjs --check",
+    "sync:refresh": "node --max-old-space-size=6144 scripts/sync.mjs --refresh",
+    "prepack": "node scripts/sync.mjs --check --no-cells",
     "build:sheet": "node scripts/render-sheet.mjs",
     "build:readme": "node scripts/catalogue-table.mjs",
     "build:pages": "node scripts/build-pages.mjs",