@mailwoman/soil 9.2.0

package/lib/sdk/cell-tiers.ts

@@ -0,0 +1,310 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   The two phases between the streamed touches and the artifact a consumer reads: the stored containment
+ *   index, and the reduction above it.
+ *
+ *   BOTH READ THE TOUCH TABLE, AND ONLY ONE OF THEM COMPACTS. `resolveCells` writes the tiers a probe walks
+ *   and collapses uniform interiors parent-ward; `reduceCells` reads the UNCOMPACTED touches, because a
+ *   compacted parent no longer names the cells the reduction has to answer.
+ *
+ *   THE REDUCTION IS THE SLOW PHASE AND ITS MEMORY IS BOUNDED BY CONSTRUCTION. A lattice of 49 point tests
+ *   per sampled cell, over a delineation cache that is cleared whole rather than evicted one entry at a
+ *   time — see {@link GEOMETRY_CACHE_ENTRIES}. Memory stays flat in row count, which is the property the poi
+ *   build lost when a reader materialized instead of streaming.
+ */
+
+import { expandShortCellInt, shortCellToInt, type H3Cell } from "@mailwoman/spatial"
+import { beginBatched } from "@mailwoman/sqlite/batched"
+import type { DatabaseClient } from "@mailwoman/sqlite/client"
+import { compactCells, getResolution } from "h3-js"
+
+import { SoilCellContainment, type SoilCapabilityCellTable, type SoilDatabase } from "#schema"
+import { mapUnitProfile, reduceCell, type CellCandidate, type MapUnitProfile } from "#sdk/reduce"
+
+/**
+ * Resolve the touch table into the stored containment index.
+ *
+ * Compaction happens HERE and only on the whole side. It is expected to yield close to nothing on this layer, which is
+ * the inversion the survey predicts: compaction needs a uniform interior, and 85.4% of `IA153`'s delineations are
+ * smaller than one resolution-9 cell.
+ */
+export function resolveCells(
+	database: DatabaseClient<SoilDatabase>,
+	indexResolution: number
+): { wholeRows: number; partialRows: number; resolutions: number[] } {
+	database.exec("CREATE INDEX build_cell_touch_area_cell ON build_cell_touch (area_id, resolution, is_full, h3_cell)")
+
+	const groups = database
+		.prepare("SELECT DISTINCT area_id, resolution FROM build_cell_touch WHERE is_full = 1")
+		.all() as Array<{ area_id: string; resolution: number }>
+
+	const insertCell = database.prepare(
+		"INSERT OR REPLACE INTO soil_map_unit_cell (h3_cell, resolution, area_id, containment) VALUES (?, ?, ?, ?)"
+	)
+
+	const resolutions = new Set<number>()
+
+	let wholeRows = 0
+
+	// One group per (delineation, resolution): `compactCells` takes a single resolution, and an adaptively-indexed layer
+	// has several. Pooling them throws; compacting only the target-resolution group would silently drop every coarsened
+	// delineation's interior.
+	database.exec("BEGIN")
+
+	for (const { area_id: areaID, resolution } of groups) {
+		const whole = database
+			.prepare("SELECT DISTINCT h3_cell FROM build_cell_touch WHERE area_id = ? AND resolution = ? AND is_full = 1")
+			.all(areaID, resolution) as Array<{ h3_cell: number }>
+
+		for (const cell of compactCells(whole.map((row) => expandShortCellInt(row.h3_cell, resolution)))) {
+			const cellResolution = getResolution(cell)
+
+			resolutions.add(cellResolution)
+			insertCell.run(shortCellToInt(cell as H3Cell), cellResolution, areaID, SoilCellContainment.Whole)
+
+			wholeRows++
+		}
+	}
+
+	database.exec("COMMIT")
+
+	// The partial rows are every touch that is not whole for its own delineation. `INSERT OR REPLACE` above already put
+	// the whole rows in, and the primary key is `(h3_cell, area_id)`, so this insert must skip them explicitly rather
+	// than rely on the key: a partial row replacing a whole one would demote an answered cell to a ray cast.
+	database.exec(
+		"INSERT OR IGNORE INTO soil_map_unit_cell (h3_cell, resolution, area_id, containment) " +
+			"SELECT DISTINCT t.h3_cell, t.resolution, t.area_id, 'partial' FROM build_cell_touch t WHERE t.is_full = 0"
+	)
+
+	const partialRows = (
+		database.prepare("SELECT count(*) AS n FROM soil_map_unit_cell WHERE containment = 'partial'").get() as {
+			n: number
+		}
+	).n
+
+	for (const row of database.prepare("SELECT DISTINCT resolution FROM soil_map_unit_cell").all() as Array<{
+		resolution: number
+	}>) {
+		resolutions.add(row.resolution)
+	}
+
+	resolutions.add(indexResolution)
+
+	return { wholeRows, partialRows, resolutions: [...resolutions].toSorted((left, right) => left - right) }
+}
+
+/**
+ * Cells reduced per progress report. The reduction is the slow phase — a lattice of 49 point tests per sampled cell —
+ * so it reports often enough that a long run is visibly alive.
+ */
+const REDUCE_PROGRESS_STRIDE = 50_000
+
+/**
+ * One delineation's geometry, as the reduction reads it.
+ */
+interface StoredDelineation {
+	mukey: string
+	min_lat: number
+	min_lon: number
+	max_lat: number
+	max_lon: number
+	rings: Uint8Array
+}
+
+/**
+ * How many delineations the reduction keeps in memory at once.
+ *
+ * Sized to bound the phase rather than to hold everything: the pilot region's median delineation encodes to roughly 1.4
+ * kB, so 200,000 of them is a few hundred megabytes — comfortable, and far below the 2.5 million a whole state holds.
+ * Memory stays flat in row count, which is the property the poi build lost when a reader materialized instead of
+ * streaming.
+ */
+const GEOMETRY_CACHE_ENTRIES = 200_000
+
+/**
+ * Reduce the touch table into `soil_capability_cell`.
+ *
+ * Reads the touch table rather than `soil_map_unit_cell` on purpose: the stored index is compacted on the whole side,
+ * so a compacted parent no longer names the cells the reduction has to answer. The touch table is the uncompacted truth
+ * about which delineation reaches which cell.
+ */
+export function reduceCells(
+	database: DatabaseClient<SoilDatabase>,
+	indexResolution: number,
+	onProgress?: (message: string) => void
+): {
+	cells: number
+	sampled: number
+	topClassUnderHalf: number
+	classless: number
+	unsampled: number
+	candidatePairs: number
+} {
+	const profiles = readMapUnitProfiles(database)
+
+	const insert = database.prepare(
+		"INSERT OR REPLACE INTO soil_capability_cell (h3_cell, class_shares, unrated_share, notrateable_share, nodata_share, other_share, mapped_share, top_class, top_class_share, weighting, delineations) " +
+			"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
+	)
+
+	const selectArea = database.prepare(
+		"SELECT mukey, min_lat, min_lon, max_lat, max_lon, rings FROM soil_map_unit_area WHERE area_id = ?"
+	)
+
+	const rows = database
+		.prepare("SELECT h3_cell, resolution, area_id, is_full FROM build_cell_touch ORDER BY resolution, h3_cell, area_id")
+		.iterate() as Iterable<{ h3_cell: number; resolution: number; area_id: string; is_full: number }>
+
+	let cells = 0
+	let sampled = 0
+	let topClassUnderHalf = 0
+	let classless = 0
+	let unsampled = 0
+	let candidatePairs = 0
+
+	let currentCell: number | undefined
+	let currentResolution = indexResolution
+	let candidates: CellCandidate[] = []
+
+	// A delineation is named by every cell it reaches — 5.35 of them per cell at resolution 9 on the pilot region — so a
+	// naive read fetches each ring blob once per touch. At Iowa's scale that is millions of blob reads of ground already
+	// in memory. The cache is bounded and CLEARED WHOLE when it fills rather than evicted one at a time: h3 cell integers
+	// carry their ancestry in their high bits, so a scan in `h3_cell` order visits neighbours together and a cleared cache
+	// refills with the delineations the next run of cells actually names.
+	const geometry = new Map<string, StoredDelineation>()
+
+	const batch = beginBatched(database, { rowsPerCommit: REDUCE_PROGRESS_STRIDE })
+
+	const flush = (): void => {
+		if (currentCell === undefined || !candidates.length) return
+
+		const cell = expandShortCellInt(currentCell, currentResolution)
+
+		const reducedCell = reduceCell(cell, currentResolution, candidates, profiles, currentCell)
+
+		candidatePairs += candidates.length
+
+		if (reducedCell.row.mapped_share <= 0) {
+			unsampled++
+
+			return
+		}
+
+		cells++
+
+		if (reducedCell.sampled) {
+			sampled++
+		}
+
+		if (reducedCell.topClassUnderHalf) {
+			topClassUnderHalf++
+		}
+
+		if (!reducedCell.row.top_class) {
+			classless++
+		}
+
+		insertRow(insert, reducedCell.row)
+
+		if (batch.rowWritten()) {
+			onProgress?.(`${cells.toLocaleString()} cells reduced`)
+		}
+	}
+
+	for (const row of rows) {
+		if (row.h3_cell !== currentCell || row.resolution !== currentResolution) {
+			flush()
+
+			currentCell = row.h3_cell
+			currentResolution = row.resolution
+			candidates = []
+		}
+
+		let area = geometry.get(row.area_id)
+
+		if (!area) {
+			area = selectArea.get(row.area_id) as StoredDelineation | undefined
+
+			if (!area) {
+				throw new Error(
+					`soil build: the touch table names delineation ${row.area_id}, which soil_map_unit_area does not hold — the ingest and the reduction disagree about what was written`
+				)
+			}
+
+			if (geometry.size >= GEOMETRY_CACHE_ENTRIES) {
+				geometry.clear()
+			}
+
+			geometry.set(row.area_id, area)
+		}
+
+		candidates.push({
+			areaID: row.area_id,
+			mukey: area.mukey,
+			containment: row.is_full ? SoilCellContainment.Whole : SoilCellContainment.Partial,
+			minLat: area.min_lat,
+			minLon: area.min_lon,
+			maxLat: area.max_lat,
+			maxLon: area.max_lon,
+			rings: area.rings,
+		})
+	}
+
+	flush()
+
+	batch.commit()
+
+	return { cells, sampled, topClassUnderHalf, classless, unsampled, candidatePairs }
+}
+
+function insertRow(statement: ReturnType<DatabaseClient["prepare"]>, row: SoilCapabilityCellTable): void {
+	statement.run(
+		row.h3_cell,
+		row.class_shares,
+		row.unrated_share,
+		row.notrateable_share,
+		row.nodata_share,
+		row.other_share,
+		row.mapped_share,
+		row.top_class,
+		row.top_class_share,
+		row.weighting,
+		row.delineations
+	)
+}
+
+/**
+ * Every map unit's per-unit-area profile, computed once and reused for every cell it reaches.
+ */
+function readMapUnitProfiles(database: DatabaseClient<SoilDatabase>): Map<string, MapUnitProfile> {
+	const componentsByMukey = new Map<
+		string,
+		Array<{ comppct_r: number; compkind: string | null; nirrcapcl: string | null }>
+	>()
+
+	for (const row of database
+		.prepare("SELECT mukey, comppct_r, compkind, nirrcapcl FROM soil_component")
+		.all() as Array<{ mukey: string; comppct_r: number; compkind: string | null; nirrcapcl: string | null }>) {
+		const list = componentsByMukey.get(row.mukey)
+
+		if (list) {
+			list.push(row)
+		} else {
+			componentsByMukey.set(row.mukey, [row])
+		}
+	}
+
+	const profiles = new Map<string, MapUnitProfile>()
+
+	for (const row of database.prepare("SELECT mukey, no_mapping FROM soil_map_unit").all() as Array<{
+		mukey: string
+		no_mapping: number
+	}>) {
+		profiles.set(row.mukey, mapUnitProfile(row, componentsByMukey.get(row.mukey) ?? []))
+	}
+
+	return profiles
+}

package/lib/sdk/cells.ts

@@ -0,0 +1,224 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   The delineation-keyed cell index, and the two numbers the index resolution is chosen from.
+ *
+ *   THE CLASSIFIER ITSELF LIVES IN `@mailwoman/spatial`, because the traps it guards are properties of
+ *   h3-js rather than of SSURGO: a centre-containment polyfill drops every polygon smaller than a cell, an
+ *   exhausted WASM allocator reports success and returns zeros, and the allocator is sized from the
+ *   bounding box. The layer contract states all three as requirements on every polygon builder. What is
+ *   soil-shaped is the ACCUMULATOR below, which keys on the delineation rather than on a hazard class,
+ *   because the reduction weights by the area a delineation covers.
+ *
+ *   EXPECT THE `partial` SHARE TO INVERT AGAINST THE FLOOD LAYER, AND DO NOT READ THAT AS A DEFECT. Flood
+ *   polygons are large against their cells, so most cells fall wholly inside one zone and `compactCells`
+ *   collapses long uniform interiors. Soil delineations are the opposite: 85.4% of `IA153`'s 17,966 of them
+ *   are smaller than one resolution-9 cell, and the median is 24,863 m² against a 105,333 m² cell. Small
+ *   polygons against large cells means most cells are crossed by a boundary — so the `partial` share should
+ *   be HIGH, `compactCells` should yield close to nothing, and the index alone will rarely answer a point
+ *   probe. That is not an argument against storing the geometry; it is the argument for why this layer
+ *   carries the reduced `soil_capability_cell` alongside the index rather than relying on the index the way
+ *   the flood layer can.
+ *
+ *   TWO NUMBERS GET REPORTED AT EACH CANDIDATE RESOLUTION, AND THEY MOVE IN OPPOSITE DIRECTIONS. The
+ *   `partial` cell share says whether the containment index answers most probes alone. The share of cells
+ *   whose top class holds less than half the cell says whether the layer is answering or hedging — the
+ *   cell-grain analogue of NRCS's own `niccdcdpct` distribution, which reads 3.3% below half nationally.
+ *   Going coarser improves the first and worsens the second, and picking between them is what the
+ *   measurement is for.
+ */
+
+import {
+	classifyFeatureCells,
+	compactAcrossResolutions,
+	shortCellToInt,
+	type FeatureCells,
+	type H3Cell,
+	type MultiPolygonRings,
+} from "@mailwoman/spatial"
+import { getResolution } from "h3-js"
+
+/**
+ * The label this layer's classifier failures carry.
+ */
+export const SOIL_CELL_LABEL = "soil cells"
+
+/**
+ * Classify one delineation — {@link classifyFeatureCells} with this layer's label bound.
+ */
+export function classifyDelineationCells(
+	polygons: MultiPolygonRings,
+	targetResolution: number,
+	areaID: string
+): FeatureCells {
+	return classifyFeatureCells(polygons, targetResolution, areaID, SOIL_CELL_LABEL)
+}
+
+/**
+ * What one resolution's index came out as.
+ */
+export interface SoilCellIndexMeasurement {
+	resolution: number
+	/**
+	 * Cells the layer reaches at all.
+	 */
+	touchedCells: number
+	/**
+	 * Cells lying wholly inside a single delineation, before compaction.
+	 */
+	wholeCells: number
+	/**
+	 * Cells a delineation boundary crosses.
+	 */
+	partialCells: number
+	/**
+	 * `partialCells / touchedCells` — the share of in-layer probes that cannot be answered from the index alone.
+	 */
+	partialShare: number
+	/**
+	 * Whole cells after `compactCells`. Expected to be close to `wholeCells` here rather than far below it: compaction
+	 * needs a uniform interior, and small delineations do not produce one.
+	 */
+	compactedWholeCells: number
+	/**
+	 * `(cell, delineation)` pairs the index stores.
+	 */
+	cellDelineationPairs: number
+	/**
+	 * The mean number of delineations reaching a cell — the direct measure of how mixed a cell is before any rating is
+	 * read, and the number that rises as the resolution coarsens.
+	 */
+	meanDelineationsPerCell: number
+	/**
+	 * Delineations whose bounding box forced a coarser resolution than the target.
+	 */
+	coarsenedFeatures: number
+	/**
+	 * The resolutions actually present, finest last.
+	 */
+	resolutions: number[]
+}
+
+/**
+ * Accumulate one resolution's cell index over a stream of delineations.
+ *
+ * Held as short-cell STRINGS rather than the integers the tables store, because `compactCells` is an h3-js function
+ * over full indexes and round-tripping through the integer form at every step would cost more than the strings do.
+ */
+export class SoilCellIndex {
+	readonly resolution: number
+
+	readonly #whole = new Set<string>()
+	readonly #touched = new Set<string>()
+	/**
+	 * `cell → delineation ids`. Every touched cell, so the mean below is over the real population rather than over the
+	 * fringe alone.
+	 */
+	readonly #byCell = new Map<string, Set<string>>()
+
+	#coarsened = 0
+
+	constructor(resolution: number) {
+		this.resolution = resolution
+	}
+
+	/**
+	 * Fold one delineation's classification in.
+	 */
+	add(areaID: string, cells: FeatureCells): void {
+		if (cells.resolution !== this.resolution) {
+			this.#coarsened++
+		}
+
+		for (const cell of cells.whole) {
+			this.#whole.add(cell)
+			this.#record(cell, areaID)
+		}
+
+		for (const cell of cells.partial) {
+			this.#record(cell, areaID)
+		}
+	}
+
+	#record(cell: string, areaID: string): void {
+		this.#touched.add(cell)
+
+		let areas = this.#byCell.get(cell)
+
+		if (!areas) {
+			areas = new Set()
+
+			this.#byCell.set(cell, areas)
+		}
+
+		areas.add(areaID)
+	}
+
+	/**
+	 * Compact the whole-cell set and report the measurement.
+	 *
+	 * Compaction is applied to the WHOLE set only — a partial cell's parent is not partial in any useful sense, and
+	 * compacting it would claim the fringe covers ground it does not.
+	 */
+	finish(): SoilCellIndexMeasurement {
+		const compacted = compactAcrossResolutions(this.#whole)
+
+		const resolutions = new Set<number>()
+
+		for (const cell of this.#touched) {
+			resolutions.add(getResolution(cell))
+		}
+
+		let pairs = 0
+
+		for (const areas of this.#byCell.values()) {
+			pairs += areas.size
+		}
+
+		const touched = this.#touched.size
+		const partial = touched - this.#whole.size
+
+		return {
+			resolution: this.resolution,
+			touchedCells: touched,
+			wholeCells: this.#whole.size,
+			partialCells: partial,
+			partialShare: touched ? partial / touched : 0,
+			compactedWholeCells: compacted.length,
+			cellDelineationPairs: pairs,
+			meanDelineationsPerCell: touched ? pairs / touched : 0,
+			coarsenedFeatures: this.#coarsened,
+			resolutions: [...resolutions].toSorted((left, right) => left - right),
+		}
+	}
+}
+
+/**
+ * The measurement as markdown table ROWS — what a build receipt carries, one line per element so a caller printing them
+ * never has to split a joined string back apart.
+ */
+export function formatSoilResolutionRows(
+	measurements: ReadonlyArray<SoilCellIndexMeasurement & { mixedCellShare?: number }>
+): string[] {
+	return [
+		"| res | touched cells | whole | partial | partial share | whole after compaction | (cell, delineation) pairs | mean delineations/cell | top class under half |",
+		"| --- | ------------- | ----- | ------- | ------------- | ---------------------- | ------------------------- | ---------------------- | -------------------- |",
+		...measurements.map(
+			(m) =>
+				`| ${m.resolution} | ${m.touchedCells.toLocaleString()} | ${m.wholeCells.toLocaleString()} | ` +
+				`${m.partialCells.toLocaleString()} | ${(m.partialShare * 100).toFixed(1)}% | ` +
+				`${m.compactedWholeCells.toLocaleString()} | ${m.cellDelineationPairs.toLocaleString()} | ` +
+				`${m.meanDelineationsPerCell.toFixed(2)} | ` +
+				`${m.mixedCellShare === undefined ? "—" : `${(m.mixedCellShare * 100).toFixed(1)}%`} |`
+		),
+	]
+}
+
+/**
+ * The cell integer a short-cell string stores as.
+ */
+export function cellToShortInt(cell: string): number {
+	return shortCellToInt(cell as H3Cell)
+}