coastlines 0.0.8-alpha.0 → 0.0.9-alpha.0

package/README.md

@@ -116,11 +116,25 @@ zip itself.
 | Where | What | Size | How it reaches you |
 | --- | --- | ---: | --- |
 | `data/` | the 36 files above | 78 MB, 29 MB packed | installed with the package |
-| `geo/` | six cut bands, land and water in square cells | 896 MB | built by `npm run sync`; not shipped |
+| `geo/` | six cut bands, land and water in square cells | 549 MB | built by `npm run sync`; not shipped |
 | `cache/` | the GSHHG zip, six Natural Earth 10m water files | 167 MB | fetched once by sync, kept |
 
-The cut bands are Natural Earth 10m and GSHHG h and f, land and lakes, each
-a record in `catalogue.json`: grid, inputs and pins, cutter and build date.
+Each cut band is a record in `catalogue.json`: grid, inputs and pins, cutter
+and build date. Cells and size are measured from the build (file content;
+4 KB disk blocks bring `geo/` to about 900 MB on disk). `cache/` holds the
+inputs those builds draw on, pinned but never shipped.
+
+<!-- cut-bands:start -->
+| Band | Layer | Zoom | Cell | Cells | Size | Source |
+| --- | --- | --- | --- | ---: | ---: | --- |
+| `ne-10m-land` | land | 9–10 | 5° (+2°) | 1,456 | 18.0 MB | Natural Earth v5.1.2, public domain |
+| `gshhg-h-land` | land | 10–12 | 1° (+0.5°) | 19,273 | 68.3 MB | GSHHG 2.3.7, LGPL-3.0-or-later |
+| `gshhg-f-land` | land | 12–∞ | 0.5° (+0.25°) | 70,692 | 383.8 MB | GSHHG 2.3.7, LGPL-3.0-or-later |
+| `ne-10m-water` | water | 7–∞ | 5° (+2°) | 712 | 31.0 MB | Natural Earth v5.1.2, public domain |
+| `gshhg-h-lakes` | lakes | 10–12 | 1° (+0.5°) | 4,068 | 11.4 MB | GSHHG 2.3.7, LGPL-3.0-or-later |
+| `gshhg-f-lakes` | lakes | 12–∞ | 0.5° (+0.25°) | 8,699 | 36.6 MB | GSHHG 2.3.7, LGPL-3.0-or-later |
+<!-- cut-bands:end -->
+
 How cells reach a boat that installs the package is open ([#17](https://github.com/mark-brannan/coastlines/issues/17)).
 
 ### Sources and licences

package/catalogue.json

@@ -1050,8 +1050,9 @@
       "cutter": "coastlines/lib/cut.js",
       "built": "2026-10-15"
     },
-    "bytesPerCell": null,
-    "bytes": null,
+    "cells": 1456,
+    "bytesPerCell": 12954,
+    "bytes": 18861693,
     "from": ["ne-land-10m", "ne-coastline-10m", "ne-minor-islands-10m"],
     "inputs": [],
     "sources": [
@@ -1092,8 +1093,9 @@
       "cutter": "coastlines/lib/cut.js",
       "built": "2026-10-15"
     },
-    "bytesPerCell": null,
-    "bytes": null,
+    "cells": 19273,
+    "bytesPerCell": 3717,
+    "bytes": 71636830,
     "from": [],
     "inputs": ["gshhg-land-h", "gshhg-licence", "gshhg-copying-lesser"],
     "sources": [
@@ -1126,8 +1128,9 @@
       "cutter": "coastlines/lib/cut.js",
       "built": "2026-10-15"
     },
-    "bytesPerCell": null,
-    "bytes": null,
+    "cells": 70692,
+    "bytesPerCell": 5693,
+    "bytes": 402415178,
     "from": [],
     "inputs": [
       {
@@ -1179,8 +1182,9 @@
       "cutter": "coastlines/lib/cut.js",
       "built": "2026-10-15"
     },
-    "bytesPerCell": null,
-    "bytes": null,
+    "cells": 712,
+    "bytesPerCell": 45613,
+    "bytes": 32476615,
     "from": [],
     "inputs": [
       {
@@ -1259,8 +1263,9 @@
       "cutter": "coastlines/lib/cut.js",
       "built": "2026-10-15"
     },
-    "bytesPerCell": null,
-    "bytes": null,
+    "cells": 4068,
+    "bytesPerCell": 2935,
+    "bytes": 11939267,
     "from": [],
     "inputs": [
       {
@@ -1325,8 +1330,9 @@
       "cutter": "coastlines/lib/cut.js",
       "built": "2026-10-15"
     },
-    "bytesPerCell": null,
-    "bytes": null,
+    "cells": 8699,
+    "bytesPerCell": 4417,
+    "bytes": 38424112,
     "from": [],
     "inputs": [
       {

package/lib/acquire.js

@@ -0,0 +1,45 @@
+// Pinned-source acquisition: give it `{url, sha256}` (a manifest pin, or any
+// source pinned the same way) and get back verified bytes from a local cache.
+// Cache hit: no fetch. Miss, or a cached file that no longer matches: fetch
+// with retry, check, keep. A wrong sha256 throws and nothing is stored or
+// retried -- retry changes how bytes arrive, never whether they are believed.
+import { mkdir, readFile, rename, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { digest } from './verify.js';
+import { fetchBytesWithRetry } from './fetch-retry.js';
+
+async function cached(file) {
+  try {
+    return await readFile(file);
+  } catch (error) {
+    if (error.code === 'ENOENT') return null;
+    throw error;
+  }
+}
+
+/**
+ * The bytes pinned by `{url, sha256, bytes?}`, from `cacheDir/<name>` when they are there and
+ * right, else fetched, checked and kept. `name` defaults to the url's last path segment (empty for a url ending in /) and must be a file name, not a
+ * path. `fetchImpl`, `sleep`, `waits` and `onRetry` go to the retry; `onFetch(url)` is told before a download.
+ */
+export async function acquire(
+  { url, sha256, bytes: length },
+  { cacheDir, name = new URL(url).pathname.split('/').pop(), onFetch, ...retry },
+) {
+  if (!name || name !== name.split(/[/\\]/).pop() || name === '..' || name === '.')
+    throw new Error(`${name}: a cache name is a file name, not a path`);
+  const file = join(cacheDir, name);
+  const held = await cached(file);
+  if (held && digest(held) === sha256) return held;
+  onFetch?.(url);
+  const bytes = await fetchBytesWithRetry(url, retry);
+  const actual = digest(bytes);
+  if (actual !== sha256) throw new Error(`${name}: ${actual}, pinned ${sha256}`);
+  if (length !== undefined && bytes.byteLength !== length)
+    throw new Error(`${name}: ${bytes.byteLength} bytes, pinned ${length}`);
+  await mkdir(cacheDir, { recursive: true });
+  // Sibling then rename: an interrupted run leaves no truncated file to fail every later one.
+  await writeFile(`${file}.partial`, bytes);
+  await rename(`${file}.partial`, file);
+  return bytes;
+}

package/lib/bands.js

@@ -49,6 +49,14 @@ export function inputsOf(record) {
   return [...own, ...inherited];
 }
 
+/** The source keys a record's geometry is built from; a geometry file must name exactly one. */
+export const sourcesOf = (record) =>
+  new Set(
+    inputsOf(record).map((input) =>
+      typeof input === 'string' ? find(input).source : input.source,
+    ),
+  );
+
 /** The zoom range charts.json draws a whole file at, across every chart that lists it; nulls when none does. */
 function chartZoom(id) {
   const layers = charts.flatMap((chart) => chart.layers).filter((layer) => layer.id === id);
@@ -75,17 +83,13 @@ function wholeFileDrift(record) {
   return problems;
 }
 
-/** A cut band names its cutter and the day it was built, and draws on one upstream only. */
+/** A cut band names its cutter and the day it was built. */
 function cutDrift(record) {
   const problems = [];
   const { cutter, built } = record.provenance;
   if (cutter !== CUTTER) problems.push(`${record.id}: cutter ${cutter} is not ${CUTTER}`);
   if (!/^\d{4}-\d{2}-\d{2}$/.test(built ?? ''))
     problems.push(`${record.id}: built ${built} is not a date`);
-  const sources = new Set(
-    inputsOf(record).map((i) => (typeof i === 'string' ? find(i).source : i.source)),
-  );
-  if (sources.size !== 1) problems.push(`${record.id}: mixes sources ${[...sources].join(', ')}`);
   return problems;
 }
 
@@ -110,9 +114,11 @@ export function drift() {
         if (JSON.stringify(record.provenance[field]) !== want)
           problems.push(`${record.id}: ${field} differs from its inputs`);
       }
-      const keys = new Set(
-        inputs.map((input) => (typeof input === 'string' ? find(input).source : input.source)),
-      );
+      const keys = sourcesOf(record);
+      if (keys.size !== 1)
+        problems.push(
+          `${record.id}: built from ${keys.size} sources (${[...keys].join(', ')}), not one; each licence stays in its own file`,
+        );
       for (const key of keys) {
         const from = manifest.sources[key];
         if (licence !== from.licence)

package/lib/cut.js

@@ -2,7 +2,7 @@
 // square lon/lat cells under geo/<dir>/ (<xi>_<yi>.json and index.json). A band is cut from one
 // upstream only, so no geometry file mixes sources.
 import { mkdir, rename, rm, writeFile } from 'node:fs/promises';
-import { band, CUTTER, inputsOf } from './bands.js';
+import { band, CUTTER, inputsOf, sourcesOf } from './bands.js';
 import { find } from './catalogue.js';
 import { shapefilePolygons } from './shapefile.js';
 import { cutCells, cutLakes, cutWater, dbfRecords, gshhgLakes, joinWater } from './detail-cells.js';
@@ -37,6 +37,8 @@ function lines(geojson) {
  * `inputsOf(record)` lists them.
  */
 export async function cutBand(record, read) {
+  if (sourcesOf(record).size !== 1)
+    throw new Error(`${record.id}: inputs from more than one source`);
   const size = record.grid.cellDeg;
   const bytes = await Promise.all(inputsOf(record).map(read));
   if (record.id === 'ne-10m-land') {

package/lib/provider.js

@@ -3,7 +3,8 @@
 // neither package imports the other; the caller owns the projection and hands over a view.
 //
 // A view is the lon/lat frame a caller paints and its scale at the centre:
-// `{ lat, metresPerPixel, lonMin, lonMax, latMin, latMax }`. The frame may run past the
+// `{ lat, resolution, lonMin, lonMax, latMin, latMax }`, `resolution` in metres per pixel (the old
+// name, `metresPerPixel`, still reads, deprecated). The frame may run past the
 // antimeridian. The catalogue keys bands by web-mercator zoom (a 512 px world at zoom 0), so the
 // scale is turned into that zoom at the view's latitude to choose one.
 //
@@ -20,13 +21,20 @@ const EARTH_R = 6378137; // metres, web mercator's sphere
 const WORLD_PX = 512; // the width of the world at zoom 0, as the catalogue's zooms count it
 const RAD = Math.PI / 180;
 
+// A view's metres per pixel: `resolution`, or the deprecated `metresPerPixel` when that is all it has.
+const resolutionOf = (view) => {
+  if (view.resolution === undefined && view.metresPerPixel !== undefined)
+    warnOnce('metresPerPixel', 'resolution');
+  return view.resolution ?? view.metresPerPixel;
+};
+
 /** The web-mercator zoom of a view's scale at its centre. */
 // Rounded, so a view at a band's edge zoom, turned into a scale and back, does not pass it.
 export const zoomOf = (view) =>
   Math.round(
     1e9 *
       Math.log2(
-        (2 * Math.PI * EARTH_R * Math.cos(view.lat * RAD)) / (WORLD_PX * view.metresPerPixel),
+        (2 * Math.PI * EARTH_R * Math.cos(view.lat * RAD)) / (WORLD_PX * resolutionOf(view)),
       ),
   ) / 1e9;
 
@@ -95,9 +103,10 @@ function geojsonRings(geo) {
  * path (`geo/h/12_45.json`, `data/ne-coastline-50m.geojson`) and rejects when it cannot. `kind`
  * is `coast`, the coastline's lines, or `land`, the land polygons' rings.
  *
- * `rings(view)` yields the view's rings; `held(view)` says whether they are all in hand, so
- * drawing them fetches nothing; `cost(view)` is what drawing it would fetch: the band, the cells
- * not yet held, and their bytes where the catalogue knows them.
+ * `rings(view)` yields the view's rings; `loaded(view)` says whether they are all in hand, so
+ * drawing them fetches nothing; `pending(view)` is what drawing it would fetch: the band, the cells
+ * not yet loaded, and their bytes where the catalogue knows them. `held` and `cost` are the old
+ * names of those two.
  */
 export function provider({ read, kind = 'coast' }) {
   const { layer, part, whole } = KINDS[kind] ?? {};
@@ -125,11 +134,11 @@ export function provider({ read, kind = 'coast' }) {
     return index.get(b.dir);
   };
 
-  const pending = new Map(); // path -> promise of its rings
+  const inflight = new Map(); // path -> promise of its rings
   const got = new Map(); // path -> its rings, once read
   const fetch = (path, ringsOf) => {
-    if (!pending.has(path))
-      pending.set(
+    if (!inflight.has(path))
+      inflight.set(
         path,
         read(path).then(
           (json) => {
@@ -138,12 +147,12 @@ export function provider({ read, kind = 'coast' }) {
             return rings;
           },
           (e) => {
-            pending.delete(path); // a failed read is tried again for the next view
+            inflight.delete(path); // a failed read is tried again for the next view
             throw e;
           },
         ),
       );
-    return pending.get(path);
+    return inflight.get(path);
   };
   const cellRings = (cell) => (part === 'land' ? cell.land.flat() : cell.coast);
 
@@ -186,17 +195,38 @@ export function provider({ read, kind = 'coast' }) {
     };
   }
 
+  async function loaded(view) {
+    return (await wants(view)).paths.every((p) => got.has(p));
+  }
+
+  async function pending(view) {
+    const { band, paths, bytes } = await wants(view);
+    const cells = paths.filter((p) => !got.has(p)).length;
+    return { band, cells, bytes: cells ? bytes(cells) : 0 };
+  }
+
   return {
     async *rings(view) {
       for (const p of (await draw(view)).parts) yield* p;
     },
-    async held(view) {
-      return (await wants(view)).paths.every((p) => got.has(p));
+    loaded,
+    pending,
+    /** @deprecated Use `loaded`. */
+    held(view) {
+      warnOnce('held', 'loaded');
+      return loaded(view);
     },
-    async cost(view) {
-      const { band, paths, bytes } = await wants(view);
-      const cells = paths.filter((p) => !got.has(p)).length;
-      return { band, cells, bytes: cells ? bytes(cells) : 0 };
+    /** @deprecated Use `pending`. */
+    cost(view) {
+      warnOnce('cost', 'pending');
+      return pending(view);
     },
   };
 }
+
+const warned = new Set();
+function warnOnce(old, now) {
+  if (warned.has(old)) return;
+  warned.add(old);
+  console.warn(`coastlines provider: \`${old}\` is deprecated, use \`${now}\``);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "coastlines",
-  "version": "0.0.8-alpha.0",
+  "version": "0.0.9-alpha.0",
   "description": "Natural Earth geography on a boat, installed rather than fetched: the marine catalogue at three scales, byte-for-byte as published, sha256-pinned, as plain GeoJSON today and a Signal K resource provider next",
   "type": "module",
   "main": "lib/index.js",
@@ -11,6 +11,7 @@
     "./catalogue": "./lib/bands.js",
     "./catalogue.json": "./catalogue.json",
     "./provider": "./lib/provider.js",
+    "./acquire": "./lib/acquire.js",
     "./data/*": "./data/*.geojson",
     "./data/*.shp": "./data/*.shp",
     "./data/*.txt": "./data/*.txt",
@@ -22,7 +23,7 @@
     "sync": "node --max-old-space-size=6144 scripts/sync.mjs",
     "sync:check": "node --max-old-space-size=6144 scripts/sync.mjs --check",
     "sync:refresh": "node --max-old-space-size=6144 scripts/sync.mjs --refresh",
-    "prepack": "node scripts/sync.mjs --no-cells",
+    "prepack": "node scripts/sync.mjs --check --no-cells",
     "build:sheet": "node scripts/render-sheet.mjs",
     "build:readme": "node scripts/catalogue-table.mjs",
     "build:pages": "node scripts/build-pages.mjs",