@mailwoman/soil 9.4.0 → 10.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/README.md +118 -116
  2. package/lib/index.ts +66 -98
  3. package/lib/paths.ts +24 -0
  4. package/lib/schema.ts +188 -120
  5. package/lib/vocabulary.ts +40 -107
  6. package/out/index.d.ts +52 -70
  7. package/out/index.d.ts.map +1 -1
  8. package/out/index.js +24 -72
  9. package/out/index.js.map +1 -1
  10. package/out/paths.d.ts +19 -0
  11. package/out/paths.d.ts.map +1 -0
  12. package/out/paths.js +21 -0
  13. package/out/paths.js.map +1 -0
  14. package/out/schema.d.ts +187 -119
  15. package/out/schema.d.ts.map +1 -1
  16. package/out/schema.js +31 -31
  17. package/out/schema.js.map +1 -1
  18. package/out/sdk/acquire.d.ts +15 -24
  19. package/out/sdk/acquire.d.ts.map +1 -1
  20. package/out/sdk/acquire.js +5 -20
  21. package/out/sdk/acquire.js.map +1 -1
  22. package/out/sdk/build-soil.d.ts +67 -70
  23. package/out/sdk/build-soil.d.ts.map +1 -1
  24. package/out/sdk/build-soil.js +65 -95
  25. package/out/sdk/build-soil.js.map +1 -1
  26. package/out/sdk/cell-tiers.d.ts +7 -19
  27. package/out/sdk/cell-tiers.d.ts.map +1 -1
  28. package/out/sdk/cell-tiers.js +19 -38
  29. package/out/sdk/cell-tiers.js.map +1 -1
  30. package/out/sdk/cells.d.ts +15 -41
  31. package/out/sdk/cells.d.ts.map +1 -1
  32. package/out/sdk/cells.js +11 -38
  33. package/out/sdk/cells.js.map +1 -1
  34. package/out/sdk/client.d.ts +23 -48
  35. package/out/sdk/client.d.ts.map +1 -1
  36. package/out/sdk/client.js +19 -63
  37. package/out/sdk/client.js.map +1 -1
  38. package/out/sdk/download.d.ts +24 -47
  39. package/out/sdk/download.d.ts.map +1 -1
  40. package/out/sdk/download.js +14 -54
  41. package/out/sdk/download.js.map +1 -1
  42. package/out/sdk/ingest/chunk.d.ts +77 -0
  43. package/out/sdk/ingest/chunk.d.ts.map +1 -0
  44. package/out/sdk/{ingest-chunk.js → ingest/chunk.js} +19 -17
  45. package/out/sdk/ingest/chunk.js.map +1 -0
  46. package/out/sdk/ingest/worker.d.ts +9 -0
  47. package/out/sdk/ingest/worker.d.ts.map +1 -0
  48. package/out/{scripts/ingest-chunk.js → sdk/ingest/worker.js} +11 -10
  49. package/out/sdk/ingest/worker.js.map +1 -0
  50. package/out/sdk/ingest.d.ts +40 -52
  51. package/out/sdk/ingest.d.ts.map +1 -1
  52. package/out/sdk/ingest.js +18 -61
  53. package/out/sdk/ingest.js.map +1 -1
  54. package/out/sdk/measure-resolutions.d.ts +5 -16
  55. package/out/sdk/measure-resolutions.d.ts.map +1 -1
  56. package/out/sdk/measure-resolutions.js +3 -15
  57. package/out/sdk/measure-resolutions.js.map +1 -1
  58. package/out/sdk/reduce.d.ts +33 -59
  59. package/out/sdk/reduce.d.ts.map +1 -1
  60. package/out/sdk/reduce.js +47 -78
  61. package/out/sdk/reduce.js.map +1 -1
  62. package/out/sdk/survey-area.d.ts +16 -48
  63. package/out/sdk/survey-area.d.ts.map +1 -1
  64. package/out/sdk/survey-area.js +33 -77
  65. package/out/sdk/survey-area.js.map +1 -1
  66. package/out/sdk/tabular.d.ts +32 -31
  67. package/out/sdk/tabular.d.ts.map +1 -1
  68. package/out/sdk/tabular.js +58 -56
  69. package/out/sdk/tabular.js.map +1 -1
  70. package/out/sdk/test-kit.d.ts +57 -0
  71. package/out/sdk/test-kit.d.ts.map +1 -0
  72. package/out/{test-kit.js → sdk/test-kit.js} +18 -39
  73. package/out/sdk/test-kit.js.map +1 -0
  74. package/out/sdk/verify.d.ts +15 -51
  75. package/out/sdk/verify.d.ts.map +1 -1
  76. package/out/sdk/verify.js +19 -77
  77. package/out/sdk/verify.js.map +1 -1
  78. package/out/vocabulary.d.ts +37 -103
  79. package/out/vocabulary.d.ts.map +1 -1
  80. package/out/vocabulary.js +34 -107
  81. package/out/vocabulary.js.map +1 -1
  82. package/package.json +36 -190
  83. package/{lib/sdk → sdk}/acquire.ts +17 -27
  84. package/{lib/sdk → sdk}/build-soil.ts +114 -128
  85. package/{lib/sdk → sdk}/cell-tiers.ts +20 -39
  86. package/{lib/sdk → sdk}/cells.ts +17 -43
  87. package/sdk/client.ts +147 -0
  88. package/sdk/download.ts +131 -0
  89. package/{lib/sdk/ingest-chunk.ts → sdk/ingest/chunk.ts} +34 -26
  90. package/{lib/scripts/ingest-chunk.ts → sdk/ingest/worker.ts} +10 -9
  91. package/sdk/ingest.ts +253 -0
  92. package/{lib/sdk → sdk}/measure-resolutions.ts +5 -16
  93. package/sdk/reduce.ts +344 -0
  94. package/{lib/sdk → sdk}/survey-area.ts +39 -83
  95. package/{lib/sdk → sdk}/tabular.ts +62 -59
  96. package/{lib → sdk}/test-kit.ts +18 -40
  97. package/{lib/sdk → sdk}/verify.ts +29 -85
  98. package/lib/sdk/client.ts +0 -184
  99. package/lib/sdk/download.ts +0 -161
  100. package/lib/sdk/index.ts +0 -20
  101. package/lib/sdk/ingest.ts +0 -278
  102. package/lib/sdk/reduce.ts +0 -375
  103. package/out/scripts/ingest-chunk.d.ts +0 -11
  104. package/out/scripts/ingest-chunk.d.ts.map +0 -1
  105. package/out/scripts/ingest-chunk.js.map +0 -1
  106. package/out/sdk/index.d.ts +0 -20
  107. package/out/sdk/index.d.ts.map +0 -1
  108. package/out/sdk/index.js +0 -20
  109. package/out/sdk/index.js.map +0 -1
  110. package/out/sdk/ingest-chunk.d.ts +0 -73
  111. package/out/sdk/ingest-chunk.d.ts.map +0 -1
  112. package/out/sdk/ingest-chunk.js.map +0 -1
  113. package/out/test-kit.d.ts +0 -79
  114. package/out/test-kit.d.ts.map +0 -1
  115. package/out/test-kit.js.map +0 -1
@@ -3,30 +3,11 @@
3
3
  * @license AGPL-3.0
4
4
  * @author Teffen Ellis, et al.
5
5
  *
6
- * Build `soil.db` — the sealed polygon layer, from the survey areas NRCS publishes.
6
+ * Builds the sealed `soil.db` polygon layer from NRCS soil survey areas.
7
7
  *
8
- * THE ACCUMULATION IS IN SQL, NOT IN A MAP. Classification is per delineation and shared with the
9
- * resolution measurement, but where the touches GO differs on purpose: the measuring instrument holds
10
- * them in memory because it is comparing candidate resolutions in one pass, and the builder streams them
11
- * into a temporary table because memory has to stay flat in row count. A polygon layer's touches
12
- * outnumber its features, and Iowa is 99 survey areas.
13
- *
14
- * ABSENCE IS NO ROW, IN EVERY TABLE. Land outside a published survey area gets no `layer_coverage` row and
15
- * no summary row — never a zero, never an empty histogram. Inside a published area the coverage row says
16
- * `designated` at completeness 1.0, because NRCS declares its own mapping complete for those areas at its
17
- * own scale; a coverage cell reached ONLY by `NOTCOM` and access-denied polygons gets no row either,
18
- * because the polygon exists and the soil mapping behind it does not.
19
- *
20
- * A COVERAGE ROW LICENSES ONLY THAT THE AUTHORITY MAPPED HERE. The reading is the class distribution, and
21
- * a cell that is 100% `unrated_share` is `designated`-complete and carries no capability reading
22
- * whatsoever. That pairing is not a corner case: 17.1% of national components carry no capability rating,
23
- * and for the irrigated rating the figure is 85.1%.
24
- *
25
- * THE AREA CROSS-CHECK USES THE AUTHORITY'S OWN PUBLISHED FIGURE. `legend.areaacres` is what NRCS states
26
- * the survey area covers — 378,800 acres for `IA153`, which is 1,532.9 km² against the 1,532.5 km² an
27
- * independent projected measurement of the same shapefile reports. Comparing the spherical ring sum
28
- * against it catches a hole read as an exterior ring, whose absence is silent: such a polygon is
29
- * well-formed and simply answers "inside" for ground the authority did not map.
8
+ * The builder streams cell touches into a temporary SQL table so that memory stays flat as the row
9
+ * count grows. Absent data has no row in any table. Land outside a survey area and coverage cells that
10
+ * only `notcom` or access-denied polygons reach, get no coverage row.
30
11
  */
31
12
 
32
13
  import { readFileSize } from "@mailwoman/core/fs/readers"
@@ -52,14 +33,15 @@ import {
52
33
  type ParsedGeometry,
53
34
  } from "@mailwoman/spatial"
54
35
  import type { DatabaseClient } from "@mailwoman/sqlite/client"
55
- import { buildSealedArtifact } from "@mailwoman/sqlite/sealed-build"
36
+ import { buildSealedArtifact } from "@mailwoman/sqlite/sealed/build"
56
37
  import { cellToLatLng } from "h3-js"
38
+ import type { PathBuilderLike } from "path-ts"
57
39
 
58
40
  import { createSoilTables, type SoilDatabase, type SoilSurveyAreaTable } from "#schema"
59
41
  import { reduceCells, resolveCells } from "#sdk/cell-tiers"
60
42
  import type { SoilFeatureSource } from "#sdk/ingest"
61
- import type { SoilChunkResult } from "#sdk/ingest-chunk"
62
- import { ingestSoilChunk } from "#sdk/ingest-chunk"
43
+ import type { SoilChunkResult } from "#sdk/ingest/chunk"
44
+ import { ingestSoilChunk } from "#sdk/ingest/chunk"
63
45
  import { WEIGHT_LATTICE_DEPTH } from "#sdk/reduce"
64
46
  import type { SurveyAreaAttributes } from "#sdk/survey-area"
65
47
  import {
@@ -72,105 +54,115 @@ import {
72
54
  } from "#vocabulary"
73
55
 
74
56
  /**
75
- * Schema version of the domain tables. Bumped when a column changes meaning, never for an added column a reader can
76
- * ignore.
57
+ * The schema version of the domain tables.
58
+ *
59
+ * It changes when a column changes meaning.
60
+ * An added column that readers can ignore does not change it.
77
61
  */
78
62
  export const SOIL_SCHEMA_VERSION = 1
79
63
 
80
64
  /**
81
- * Delineation ids per chunk process.
65
+ * The number of delineation ids per chunk process.
82
66
  *
83
- * Sized against the ceiling the flood layer measured rather than guessed: single-process runs over that product died
84
- * after roughly 510,000 and 798,000 features as h3's WASM heap fragmented. 100,000 leaves five times that margin, and
85
- * the cost of a smaller number is one interpreter start per chunk. Iowa's largest survey area holds well under it, so
86
- * on this build the bound costs one process per area — which is what makes an area's failure nameable.
67
+ * A single process over a large polygon layer fails after several hundred thousand
68
+ * features because the h3 wasm heap fragments.
69
+ * This size stays well below that limit.
87
70
  */
88
71
  export const DEFAULT_CHUNK_SIZE = 100_000
89
72
 
90
73
  /**
91
- * Square metres in an acre — the unit `legend.areaacres` publishes in.
74
+ * Square meters per acre, the unit of `legend.areaacres`.
92
75
  */
93
76
  const M2_PER_ACRE = 4046.8564224
94
77
 
95
78
  /**
96
- * The relative gap between the ring-area total and the authority's own published acreage that fails the build.
79
+ * The relative gap between the ring-area total and the published acreage above which the build fails.
97
80
  *
98
- * Two percent. The comparison is a spherical ring area against a figure NRCS itself warns "may differ from that
99
- * measured using GIS software due to different measuring techniques and rounding practices, or due to the fact that the
100
- * value has been adjusted so that the sum total of all map units in the legend equals that listed for soil survey area"
101
- * — so an exact test would be brittle. Two percent sits far above the 0.03% `IA153` measures and far below the error a
102
- * hole-blind read produces, which the zoning survey measured at 4.1% over a whole national layer.
81
+ * NRCS warns that its acreage can differ from a GIS measurement, so an exact
82
+ * comparison would fail on valid data.
83
+ * A hole read as an exterior ring produces a much larger gap than this tolerance.
103
84
  */
104
85
  const AREA_TOLERANCE = 0.02
105
86
 
106
87
  /**
107
- * One survey area, ready to build: where its geometry is, and what its tabular export said.
88
+ * One survey area ready to build, with its geometry and tabular attributes.
108
89
  */
109
90
  export interface SurveyAreaInput {
110
91
  attributes: SurveyAreaAttributes
111
92
  /**
112
- * The map-unit polygon shapefile.
93
+ * The path of the map-unit polygon shapefile.
113
94
  */
114
95
  shapefilePath?: string
115
96
  /**
116
- * The survey area's own outline, already read.
97
+ * The survey area's outline.
117
98
  */
118
99
  outline: ParsedGeometry
119
100
  /**
120
- * An in-process feature source. Correct for a fixture and for anything small; the batched path builds one of these
121
- * per chunk, so the two share one implementation.
101
+ * An in-process feature source, used by fixtures.
102
+ *
103
+ * The batched path builds one of these per chunk, so both paths share one implementation.
122
104
  */
123
105
  source?: SoilFeatureSource
124
106
  /**
125
- * The delineation count the shapefile declares, when the caller has already read it.
107
+ * The delineation count that the shapefile declares, when the caller has already read it.
126
108
  */
127
109
  declaredFeatureCount?: number
128
110
  }
129
111
 
112
+ /**
113
+ * Options for {@link buildSoilDatabase}.
114
+ */
130
115
  export interface BuildSoilOptions {
131
116
  /**
132
- * The survey areas to build, in the order they should be ingested.
117
+ * The survey areas to build, in ingest order.
133
118
  */
134
119
  areas: ReadonlyArray<SurveyAreaInput>
135
120
  /**
136
- * The region the layer name carries — the pilot's is `ia`.
121
+ * The region code in the layer name, such as `ia`.
137
122
  */
138
123
  region: string
139
124
  /**
140
- * Where the sealed artifact lands. The build writes beside it and swaps.
125
+ * The path of the sealed artifact.
126
+ *
127
+ * The build writes a temporary file beside it and then swaps it in.
141
128
  */
142
- out: string
129
+ out: PathBuilderLike
143
130
  /**
144
- * The refresh the build ingested — `layer_manifest.version` and `source_vintage`.
131
+ * The survey refresh date, written to `layer_manifest.version` and `source_vintage`.
145
132
  */
146
133
  sourceVintage: string
147
134
  buildCmd: string
148
135
  buildSHA: string
149
136
  /**
150
- * ISO-8601, supplied by the caller. Never generated here: the contract says so, and a library-generated timestamp
151
- * makes two builds of the same inputs differ.
137
+ * The ISO-8601 creation time, supplied by the caller so that two builds of the same inputs are identical.
152
138
  */
153
139
  createdAt: string
154
140
  /**
155
- * The resolution the cell index and the reduction are built at — chosen from the measurement.
141
+ * The H3 resolution of the cell index and the reduction.
156
142
  */
157
143
  indexResolution: number
158
144
  /**
159
- * The resolution `layer_coverage` rows are keyed at. Must be coarser than the index resolution.
145
+ * The H3 resolution of the `layer_coverage` rows.
146
+ * It must be coarser than the index resolution.
160
147
  */
161
148
  coverageResolution: number
162
149
  /**
163
- * Delineation ids per chunk process. See {@link DEFAULT_CHUNK_SIZE}.
150
+ * The number of delineation ids per chunk process.
151
+ * See {@link DEFAULT_CHUNK_SIZE}.
164
152
  */
165
153
  chunkSize?: number
166
154
  /**
167
- * Run the ingest IN THIS PROCESS rather than spawning chunk children. Only for fixtures, which carry no shapefile for
168
- * a child to open.
155
+ * Whether to run the ingest in this process instead of in chunk processes.
156
+ *
157
+ * Only fixtures use it, because they have no shapefile for a child process to open.
169
158
  */
170
159
  inProcess?: boolean
171
160
  onProgress?: (message: string) => void
172
161
  }
173
162
 
163
+ /**
164
+ * The counts and measurements that {@link buildSoilDatabase} reports.
165
+ */
174
166
  export interface BuildSoilResult {
175
167
  out: string
176
168
  region: string
@@ -183,28 +175,30 @@ export interface BuildSoilResult {
183
175
  wholeCellRows: number
184
176
  partialCellRows: number
185
177
  /**
186
- * `partialCellRows / (wholeCellRows + partialCellRows)` over the STORED index rows. The whole side is compacted, so
187
- * this is not the same number the resolution was chosen on and is reported separately.
178
+ * The value of `partialCellRows / (wholeCellRows + partialCellRows)` over the stored index rows.
179
+ *
180
+ * Whole cells are compacted, so this differs from the partial share used to choose the resolution.
188
181
  */
189
182
  storedPartialShare: number
190
183
  capabilityCells: number
191
184
  /**
192
- * Cells the lattice was used for, rather than the whole-cell fast path.
185
+ * The number of cells reduced with the sampling lattice instead of the whole-cell fast path.
193
186
  */
194
187
  sampledCells: number
195
188
  /**
196
- * Cells whose top class covers less than half the cell — the §4.7 number, taken off the shipping artifact rather than
197
- * out of a separate harness.
189
+ * The number of cells whose top class covers less than half the cell.
198
190
  */
199
191
  topClassUnderHalfCells: number
200
192
  topClassUnderHalfShare: number
201
193
  /**
202
- * Cells with NO class at all: the survey mapped them and rated nothing there.
194
+ * The number of cells that the survey mapped but did not rate.
203
195
  */
204
196
  classlessCells: number
205
197
  /**
206
- * Cells the index touched that no lattice point landed inside. Dropped rather than stored as an all-zero
207
- * distribution, and counted because a large number would mean the lattice is too coarse for this geometry.
198
+ * The number of indexed cells that no lattice point landed in.
199
+ *
200
+ * These cells are dropped.
201
+ * A large count means the lattice is too coarse for the geometry.
208
202
  */
209
203
  unsampledCells: number
210
204
  meanDelineationsPerCell: number
@@ -212,25 +206,27 @@ export interface BuildSoilResult {
212
206
  storedResolutions: number[]
213
207
  coverageCells: number
214
208
  /**
215
- * Coverage cells inside a survey-area outline that no delineation with soil mapping reached. Reported rather than
216
- * smoothed over: SSURGO is wall-to-wall inside a published area, so a large number means the outline and the
217
- * delineations disagree.
209
+ * The number of coverage cells inside a survey-area outline that no mapped delineation reached.
210
+ *
211
+ * SSURGO covers every part of a published area, so a large count means the outline
212
+ * and the delineations disagree.
218
213
  */
219
214
  coverageCellsWithoutMapping: number
220
215
  /**
221
- * The area readings in square kilometres against the authority's own published figure, with the witness stated. The
222
- * `known` count of areas that publish an acreage is what a receipt reads the absence against.
216
+ * The ring-area totals in square kilometers, compared with the published acreage.
217
+ *
218
+ * Its `known` count is the number of survey areas that publish an acreage.
223
219
  */
224
220
  area: AreaAgreementReading
225
221
  sizeBytes: number
226
222
  }
227
223
 
228
224
  /**
229
- * Build the layer.
225
+ * Builds the soil layer.
230
226
  *
231
- * @throws {Error} On a delineation the classifier refuses, a streamed count that disagrees with the shapefile's own
232
- * declaration, an area total that disagrees with the authority's published acreage, or a set of outlines that yields
233
- * no interior coverage cell at all.
227
+ * @throws {Error} When the classifier rejects a delineation, when a streamed count
228
+ * differs from the shapefile's declared count, when the area total differs from the
229
+ * published acreage, or when the outlines contain no interior coverage cell.
234
230
  */
235
231
  export async function buildSoilDatabase(options: BuildSoilOptions): Promise<BuildSoilResult> {
236
232
  if (options.coverageResolution >= options.indexResolution) {
@@ -252,17 +248,16 @@ export async function buildSoilDatabase(options: BuildSoilOptions): Promise<Buil
252
248
  await createLayerManifestTable(kdb)
253
249
  await createLayerCoverageTable(kdb)
254
250
 
255
- // The touch table exists only for this build and is dropped before the artifact is sealed. No primary key while
256
- // loading: the resolution queries below read it through indexes created once the load is done, and a clustered
257
- // key would sort every insert against an ingest order nothing controls.
251
+ // The touch table is dropped before the artifact is sealed.
252
+ // It has no primary key because a clustered key would sort every insert.
253
+ // The resolution queries use indexes that are created after the load.
258
254
  kdb.exec(
259
255
  "CREATE TABLE build_cell_touch (h3_cell INTEGER NOT NULL, resolution INTEGER NOT NULL, area_id TEXT NOT NULL, is_full INTEGER NOT NULL)"
260
256
  )
261
257
  },
262
258
  ingest: async (kdb) => {
263
- // Attributes FIRST because the ingest needs one thing out of them — which map units have no soil mapping behind
264
- // them — and because a delineation whose map unit is missing must fail while the artifact is still empty rather
265
- // than after millions of geometry rows are written.
259
+ // The attributes are written first because the ingest needs the map units that have no soil mapping.
260
+ // The build writes them first so a delineation with a missing map unit fails early.
266
261
  writeAttributes(kdb, options.areas)
267
262
 
268
263
  if (!options.inProcess) return undefined
@@ -293,10 +288,10 @@ export async function buildSoilDatabase(options: BuildSoilOptions): Promise<Buil
293
288
  writeSurveyAreaRows(kdb, options, coverage.cellsByArea)
294
289
  writeVocabularyRows(kdb, options.areas)
295
290
 
296
- // THE SPINE KEY NAMES THE TABLE A CONSUMER JOINS ON, table-qualified, per the layer contract. For this layer
297
- // that is the REDUCTION rather than the containment index: `soil_capability_cell` holds one row per cell at one
298
- // resolution, which `soil_map_unit_cell` does not — it is keyed `(cell, delineation)` and is mixed-resolution by
299
- // construction, so it is a tier the reader walks rather than a key a consumer joins.
291
+ // The spine key is the column that consumers join on.
292
+ // It points at `soil_capability_cell`, which has one row per cell at one resolution.
293
+ // The `soil_map_unit_cell` table is keyed by cell and delineation at mixed
294
+ // resolutions, so consumers cannot join on it.
300
295
  await writeLayerManifest(
301
296
  kdb,
302
297
  polygonLayerManifest(options, {
@@ -314,7 +309,7 @@ export async function buildSoilDatabase(options: BuildSoilOptions): Promise<Buil
314
309
  const totalCellRows = cells.wholeRows + cells.partialRows
315
310
 
316
311
  return {
317
- out: options.out,
312
+ out: options.out.toString(),
318
313
  region: options.region,
319
314
  surveyAreas: options.areas.length,
320
315
  delineations: ingested.delineations,
@@ -345,7 +340,7 @@ export async function buildSoilDatabase(options: BuildSoilOptions): Promise<Buil
345
340
  }
346
341
 
347
342
  /**
348
- * What the whole ingest produced, however many processes it took.
343
+ * The combined result of every ingest chunk.
349
344
  */
350
345
  interface StreamResult {
351
346
  delineations: number
@@ -358,10 +353,9 @@ interface StreamResult {
358
353
  }
359
354
 
360
355
  /**
361
- * Add up what the chunks reported.
356
+ * Sums the results of the ingest chunks.
362
357
  *
363
- * Exported for its own test: the coverage-cell arithmetic is the one part of the batched path a fixture build cannot
364
- * reach, and getting it wrong produces a well-formed artifact that under-reports how many delineations a cell holds.
358
+ * It is exported for a unit test because fixture builds do not exercise the batched path.
365
359
  */
366
360
  export function aggregateChunks(chunks: ReadonlyArray<SoilChunkResult>): StreamResult {
367
361
  const byArea = new Map<string, number>()
@@ -381,8 +375,7 @@ export function aggregateChunks(chunks: ReadonlyArray<SoilChunkResult>): StreamR
381
375
 
382
376
  byArea.set(chunk.areaSymbol, (byArea.get(chunk.areaSymbol) ?? 0) + chunk.delineations)
383
377
 
384
- // A coverage cell straddles chunk boundaries — a range of feature ids is not a region, and a coverage cell can
385
- // straddle two survey areas — so the counts ADD rather than replace.
378
+ // One coverage cell can appear in several chunks and in two survey areas, so the counts are summed.
386
379
  mergeCountsInto(observedByCoverageCell, chunk.observedByCoverageCell)
387
380
  mergeCountsInto(mappedByCoverageCell, chunk.mappedByCoverageCell)
388
381
  }
@@ -391,9 +384,7 @@ export function aggregateChunks(chunks: ReadonlyArray<SoilChunkResult>): StreamR
391
384
  }
392
385
 
393
386
  /**
394
- * Refuse a build that streamed fewer delineations than a survey area declares.
395
- *
396
- * A short read builds a smaller county and reports success, which is the partial result that must throw.
387
+ * Throws when the streamed delineation count of a survey area differs from its declared count.
397
388
  */
398
389
  function assertDelineationCounts(areas: ReadonlyArray<SurveyAreaInput>, streamed: StreamResult): void {
399
390
  for (const area of areas) {
@@ -412,11 +403,11 @@ function assertDelineationCounts(areas: ReadonlyArray<SurveyAreaInput>, streamed
412
403
  }
413
404
 
414
405
  /**
415
- * Refuse an artifact whose rings do not add up to the acreage the authority publishes.
406
+ * Throws when the ring-area total differs from the published acreage by more than {@link AREA_TOLERANCE}.
416
407
  *
417
- * The message carries the hole-blind total beside the nested one, because the gap between them is the diagnosis: a hole
418
- * read as an exterior ring answers "inside" for every point in it. A build over survey areas that publish no acreage
419
- * has no witness — the reading's own type says so, and the `known` count is what a receipt names.
408
+ * The error message includes the total with holes ignored, because a hole read
409
+ * as an exterior ring causes the gap.
410
+ * When no survey area publishes an acreage, the check is skipped.
420
411
  */
421
412
  function assertAreaAgreement(areas: ReadonlyArray<SurveyAreaInput>, streamed: StreamResult): AreaAgreementReading {
422
413
  let publishedAcres = 0
@@ -447,7 +438,7 @@ function assertAreaAgreement(areas: ReadonlyArray<SurveyAreaInput>, streamed: St
447
438
  }
448
439
 
449
440
  /**
450
- * Write every survey area's map units, components and vocabulary before any geometry is streamed.
441
+ * Writes every survey area's map units and components.
451
442
  */
452
443
  function writeAttributes(database: DatabaseClient<SoilDatabase>, areas: ReadonlyArray<SurveyAreaInput>): void {
453
444
  const insertMapUnit = database.prepare(
@@ -499,7 +490,7 @@ function writeAttributes(database: DatabaseClient<SoilDatabase>, areas: Readonly
499
490
  }
500
491
 
501
492
  /**
502
- * The map units with no soil mapping behind them, from what was just written.
493
+ * Returns the keys of the map units that have no soil mapping.
503
494
  */
504
495
  function noMappingMukeys(areas: ReadonlyArray<SurveyAreaInput>): Set<string> {
505
496
  const mukeys = new Set<string>()
@@ -516,7 +507,8 @@ function noMappingMukeys(areas: ReadonlyArray<SurveyAreaInput>): Set<string> {
516
507
  }
517
508
 
518
509
  /**
519
- * The in-process ingest — one chunk per survey area, all in this interpreter. Fixtures only.
510
+ * Ingests one chunk per survey area in this process.
511
+ * Only fixtures use it.
520
512
  */
521
513
  async function ingestInProcess(
522
514
  database: DatabaseClient<SoilDatabase>,
@@ -547,13 +539,13 @@ async function ingestInProcess(
547
539
  }
548
540
 
549
541
  /**
550
- * Run the ingest as a sequence of bounded child processes, one per range of one survey area's FIDs. The shared chunk
551
- * contract — the parent's no-handle rule, and the fail-loud handling of a chunk that dies or prints nothing — lives
552
- * with `ingestChunkArguments` and `runChunkProcess`.
542
+ * Runs the ingest as a sequence of child processes, one per FID range of one survey area.
543
+ *
544
+ * The shared chunk protocol and its error handling live in `ingestChunkArguments` and `runChunkProcess`.
553
545
  */
554
546
  async function runBatchedIngest(tmpPath: string, options: BuildSoilOptions): Promise<SoilChunkResult[]> {
555
547
  const chunkSize = options.chunkSize ?? DEFAULT_CHUNK_SIZE
556
- const script = resolveModulePath("@mailwoman/soil/scripts/ingest-chunk")
548
+ const script = resolveModulePath("@mailwoman/soil/sdk/ingest/worker")
557
549
  const noMapping = [...noMappingMukeys(options.areas)]
558
550
  const chunks: SoilChunkResult[] = []
559
551
 
@@ -602,22 +594,16 @@ async function runBatchedIngest(tmpPath: string, options: BuildSoilOptions): Pro
602
594
  }
603
595
 
604
596
  /**
605
- * The coverage rows: one per interior cell of the built footprint that soil mapping actually reaches, and none outside.
597
+ * Returns one coverage row per interior cell of the built footprint that soil mapping reaches.
606
598
  *
607
- * THE INTERIOR TEST RUNS ONCE OVER THE UNION OF EVERY OUTLINE BUILT, NOT PER SURVEY AREA, and the difference is most of
608
- * a state. The test is conservative — it keeps only cells lying WHOLLY inside — so applied per area it drops every cell
609
- * a county border crosses. Measured on Polk County alone at resolution 6: 20 interior cells against the roughly 42 the
610
- * county spans by area, so more than half of it would read `unknown` while sitting inside a survey the build had
611
- * ingested. Run over the union, only the OUTER border of the built set is dropped, which is the honest edge: beyond it
612
- * lies ground this artifact does not hold.
599
+ * The interior test keeps only cells that lie wholly inside the footprint.
600
+ * It runs once over the union of all outlines, because running it per survey area
601
+ * would drop every cell that crosses an internal county border.
613
602
  *
614
- * The conservatism itself stays. A cell wrongly called interior would state that an authority determined a location it
615
- * never looked at, and a point in the dropped strip reading `unknown` is the truthful answer for ground the built set
616
- * may or may not reach.
603
+ * Only cells on the outer border of the built set are dropped.
617
604
  *
618
- * `observed_rows` counts the delineations reaching the cell, which is what the contract's column means. A cell reached
619
- * only by `NOTCOM` and access-denied polygons gets NO ROW — the polygon exists, the soil mapping behind it does not,
620
- * and the survey's §3.2 puts that case with the absences rather than with the coverage.
605
+ * The `observed_rows` column counts the delineations that reach the cell.
606
+ * A cell that only `notcom` and access-denied polygons reach gets no row, because it has no soil mapping.
621
607
  */
622
608
  function buildCoverageCells(
623
609
  options: BuildSoilOptions,
@@ -655,8 +641,8 @@ function buildCoverageCells(
655
641
  continue
656
642
  }
657
643
 
658
- // Attributed by the cell's CENTRE, so each row is counted for exactly one survey area even where the cell straddles
659
- // two. The count is a per-area receipt, not part of the coverage claim — the claim is the row set itself.
644
+ // The cell's center decides its survey area, so each cell counts for one area only.
645
+ // This per-area count is informational and does not affect the coverage rows.
660
646
  const [latitude, longitude] = cellToLatLng(cell)
661
647
  const owner = options.areas.find((input) => geometryContains(input.outline, longitude, latitude))
662
648
 
@@ -671,10 +657,10 @@ function buildCoverageCells(
671
657
  }
672
658
 
673
659
  /**
674
- * One outline's polygons, in the `MultiPolygon` coordinate shape, whichever areal type it arrived as.
660
+ * Returns an outline's polygons in `MultiPolygon` coordinate form.
675
661
  *
676
- * @throws {TypeError} When the outline is not areal. A survey area whose footprint cannot be read would silently
677
- * contribute nothing to the union, and the coverage over it would simply be absent.
662
+ * @throws {TypeError} When the outline is not a polygon or multipolygon.
663
+ * The build would silently drop that survey area's coverage if it skipped the outline.
678
664
  */
679
665
  function outlinePolygons(outline: ParsedGeometry): MultiPolygonRings {
680
666
  const polygons = arealPolygons(outline)
@@ -687,7 +673,7 @@ function outlinePolygons(outline: ParsedGeometry): MultiPolygonRings {
687
673
  }
688
674
 
689
675
  /**
690
- * Insert one row per survey area.
676
+ * Inserts one row per survey area.
691
677
  */
692
678
  function writeSurveyAreaRows(
693
679
  database: DatabaseClient<SoilDatabase>,
@@ -754,10 +740,10 @@ function writeSurveyAreaRows(
754
740
  }
755
741
 
756
742
  /**
757
- * Insert the authority's declared domains, plus the weighting the shares were produced under.
743
+ * Inserts the authority's declared domains and the share weighting.
758
744
  *
759
- * The weighting rides in the vocabulary table as well as on every row: the row-level copy is what a consumer reads, and
760
- * this one carries the sentence that says what it MEANS, which no column can.
745
+ * Every cell row stores the weighting code.
746
+ * The vocabulary row adds the description of that code.
761
747
  */
762
748
  function writeVocabularyRows(database: DatabaseClient<SoilDatabase>, areas: ReadonlyArray<SurveyAreaInput>): void {
763
749
  const insert = database.prepare(
@@ -3,17 +3,7 @@
3
3
  * @license AGPL-3.0
4
4
  * @author Teffen Ellis, et al.
5
5
  *
6
- * The two phases between the streamed touches and the artifact a consumer reads: the stored containment
7
- * index, and the reduction above it.
8
- *
9
- * BOTH READ THE TOUCH TABLE, AND ONLY ONE OF THEM COMPACTS. `resolveCells` writes the tiers a probe walks
10
- * and collapses uniform interiors parent-ward; `reduceCells` reads the UNCOMPACTED touches, because a
11
- * compacted parent no longer names the cells the reduction has to answer.
12
- *
13
- * THE REDUCTION IS THE SLOW PHASE AND ITS MEMORY IS BOUNDED BY CONSTRUCTION. A lattice of 49 point tests
14
- * per sampled cell, over a delineation cache that is cleared whole rather than evicted one entry at a
15
- * time — see {@link GEOMETRY_CACHE_ENTRIES}. Memory stays flat in row count, which is the property the poi
16
- * build lost when a reader materialized instead of streaming.
6
+ * Turns the build's cell-touch table into the containment index and the per-cell capability reduction.
17
7
  */
18
8
 
19
9
  import { expandShortCellInt, shortCellToInt, type H3Cell } from "@mailwoman/spatial"
@@ -25,11 +15,10 @@ import { SoilCellContainment, type SoilCapabilityCellTable, type SoilDatabase }
25
15
  import { mapUnitProfile, reduceCell, type CellCandidate, type MapUnitProfile } from "#sdk/reduce"
26
16
 
27
17
  /**
28
- * Resolve the touch table into the stored containment index.
18
+ * Writes the touch table into the `soil_map_unit_cell` containment index.
29
19
  *
30
- * Compaction happens HERE and only on the whole side. It is expected to yield close to nothing on this layer, which is
31
- * the inversion the survey predicts: compaction needs a uniform interior, and 85.4% of `IA153`'s delineations are
32
- * smaller than one resolution-9 cell.
20
+ * Only whole cells are compacted.
21
+ * Most soil delineations are smaller than one index cell, so compaction removes few rows on this layer.
33
22
  */
34
23
  export function resolveCells(
35
24
  database: DatabaseClient<SoilDatabase>,
@@ -49,9 +38,8 @@ export function resolveCells(
49
38
 
50
39
  let wholeRows = 0
51
40
 
52
- // One group per (delineation, resolution): `compactCells` takes a single resolution, and an adaptively-indexed layer
53
- // has several. Pooling them throws; compacting only the target-resolution group would silently drop every coarsened
54
- // delineation's interior.
41
+ // Each group is one delineation at one resolution, because `compactCells` throws on mixed resolutions.
42
+ // Coarsened delineations are indexed at other resolutions, so every group must be compacted.
55
43
  database.exec("BEGIN")
56
44
 
57
45
  for (const { area_id: areaID, resolution } of groups) {
@@ -71,9 +59,8 @@ export function resolveCells(
71
59
 
72
60
  database.exec("COMMIT")
73
61
 
74
- // The partial rows are every touch that is not whole for its own delineation. `INSERT OR REPLACE` above already put
75
- // the whole rows in, and the primary key is `(h3_cell, area_id)`, so this insert must skip them explicitly rather
76
- // than rely on the key: a partial row replacing a whole one would demote an answered cell to a ray cast.
62
+ // The partial rows use `INSERT OR IGNORE` so that they never replace a whole row with the same key.
63
+ // A replaced whole row would force the reader to ray-cast a cell it could answer directly.
77
64
  database.exec(
78
65
  "INSERT OR IGNORE INTO soil_map_unit_cell (h3_cell, resolution, area_id, containment) " +
79
66
  "SELECT DISTINCT t.h3_cell, t.resolution, t.area_id, 'partial' FROM build_cell_touch t WHERE t.is_full = 0"
@@ -97,13 +84,12 @@ export function resolveCells(
97
84
  }
98
85
 
99
86
  /**
100
- * Cells reduced per progress report. The reduction is the slow phase — a lattice of 49 point tests per sampled cell —
101
- * so it reports often enough that a long run is visibly alive.
87
+ * The number of cells reduced between progress reports and commits.
102
88
  */
103
89
  const REDUCE_PROGRESS_STRIDE = 50_000
104
90
 
105
91
  /**
106
- * One delineation's geometry, as the reduction reads it.
92
+ * One delineation's geometry as the reduction reads it.
107
93
  */
108
94
  interface StoredDelineation {
109
95
  mukey: string
@@ -115,21 +101,17 @@ interface StoredDelineation {
115
101
  }
116
102
 
117
103
  /**
118
- * How many delineations the reduction keeps in memory at once.
104
+ * The maximum number of delineations that the reduction keeps in memory.
119
105
  *
120
- * Sized to bound the phase rather than to hold everything: the pilot region's median delineation encodes to roughly 1.4
121
- * kB, so 200,000 of them is a few hundred megabytes — comfortable, and far below the 2.5 million a whole state holds.
122
- * Memory stays flat in row count, which is the property the poi build lost when a reader materialized instead of
123
- * streaming.
106
+ * The limit keeps memory bounded when a state has millions of delineations.
124
107
  */
125
108
  const GEOMETRY_CACHE_ENTRIES = 200_000
126
109
 
127
110
  /**
128
- * Reduce the touch table into `soil_capability_cell`.
111
+ * Reduces the touch table into `soil_capability_cell`.
129
112
  *
130
- * Reads the touch table rather than `soil_map_unit_cell` on purpose: the stored index is compacted on the whole side,
131
- * so a compacted parent no longer names the cells the reduction has to answer. The touch table is the uncompacted truth
132
- * about which delineation reaches which cell.
113
+ * It reads the touch table because `soil_map_unit_cell` is compacted.
114
+ * A compacted parent cell hides the index cells that the reduction must answer.
133
115
  */
134
116
  export function reduceCells(
135
117
  database: DatabaseClient<SoilDatabase>,
@@ -169,11 +151,10 @@ export function reduceCells(
169
151
  let currentResolution = indexResolution
170
152
  let candidates: CellCandidate[] = []
171
153
 
172
- // A delineation is named by every cell it reaches — 5.35 of them per cell at resolution 9 on the pilot region — so a
173
- // naive read fetches each ring blob once per touch. At Iowa's scale that is millions of blob reads of ground already
174
- // in memory. The cache is bounded and CLEARED WHOLE when it fills rather than evicted one at a time: h3 cell integers
175
- // carry their ancestry in their high bits, so a scan in `h3_cell` order visits neighbours together and a cleared cache
176
- // refills with the delineations the next run of cells actually names.
154
+ // Each delineation touches several cells, so the cache avoids reading its rings once per touch.
155
+ // The cache is cleared whole when it fills.
156
+ // The scan runs in `h3_cell` order and visits neighboring cells together.
157
+ // After a clear, the cache refills with the delineations that the next cells need.
177
158
  const geometry = new Map<string, StoredDelineation>()
178
159
 
179
160
  const batch = beginBatched(database, { rowsPerCommit: REDUCE_PROGRESS_STRIDE })
@@ -277,7 +258,7 @@ function insertRow(statement: ReturnType<DatabaseClient["prepare"]>, row: SoilCa
277
258
  }
278
259
 
279
260
  /**
280
- * Every map unit's per-unit-area profile, computed once and reused for every cell it reaches.
261
+ * Computes every map unit's profile once so that each cell reuses it.
281
262
  */
282
263
  function readMapUnitProfiles(database: DatabaseClient<SoilDatabase>): Map<string, MapUnitProfile> {
283
264
  const componentsByMukey = new Map<