@mailwoman/soil 9.4.0 → 10.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +118 -116
- package/lib/index.ts +66 -98
- package/lib/paths.ts +24 -0
- package/lib/schema.ts +188 -120
- package/lib/vocabulary.ts +40 -107
- package/out/index.d.ts +52 -70
- package/out/index.d.ts.map +1 -1
- package/out/index.js +24 -72
- package/out/index.js.map +1 -1
- package/out/paths.d.ts +19 -0
- package/out/paths.d.ts.map +1 -0
- package/out/paths.js +21 -0
- package/out/paths.js.map +1 -0
- package/out/schema.d.ts +187 -119
- package/out/schema.d.ts.map +1 -1
- package/out/schema.js +31 -31
- package/out/schema.js.map +1 -1
- package/out/sdk/acquire.d.ts +15 -24
- package/out/sdk/acquire.d.ts.map +1 -1
- package/out/sdk/acquire.js +5 -20
- package/out/sdk/acquire.js.map +1 -1
- package/out/sdk/build-soil.d.ts +67 -70
- package/out/sdk/build-soil.d.ts.map +1 -1
- package/out/sdk/build-soil.js +65 -95
- package/out/sdk/build-soil.js.map +1 -1
- package/out/sdk/cell-tiers.d.ts +7 -19
- package/out/sdk/cell-tiers.d.ts.map +1 -1
- package/out/sdk/cell-tiers.js +19 -38
- package/out/sdk/cell-tiers.js.map +1 -1
- package/out/sdk/cells.d.ts +15 -41
- package/out/sdk/cells.d.ts.map +1 -1
- package/out/sdk/cells.js +11 -38
- package/out/sdk/cells.js.map +1 -1
- package/out/sdk/client.d.ts +23 -48
- package/out/sdk/client.d.ts.map +1 -1
- package/out/sdk/client.js +19 -63
- package/out/sdk/client.js.map +1 -1
- package/out/sdk/download.d.ts +24 -47
- package/out/sdk/download.d.ts.map +1 -1
- package/out/sdk/download.js +14 -54
- package/out/sdk/download.js.map +1 -1
- package/out/sdk/ingest/chunk.d.ts +77 -0
- package/out/sdk/ingest/chunk.d.ts.map +1 -0
- package/out/sdk/{ingest-chunk.js → ingest/chunk.js} +19 -17
- package/out/sdk/ingest/chunk.js.map +1 -0
- package/out/sdk/ingest/worker.d.ts +9 -0
- package/out/sdk/ingest/worker.d.ts.map +1 -0
- package/out/{scripts/ingest-chunk.js → sdk/ingest/worker.js} +11 -10
- package/out/sdk/ingest/worker.js.map +1 -0
- package/out/sdk/ingest.d.ts +40 -52
- package/out/sdk/ingest.d.ts.map +1 -1
- package/out/sdk/ingest.js +18 -61
- package/out/sdk/ingest.js.map +1 -1
- package/out/sdk/measure-resolutions.d.ts +5 -16
- package/out/sdk/measure-resolutions.d.ts.map +1 -1
- package/out/sdk/measure-resolutions.js +3 -15
- package/out/sdk/measure-resolutions.js.map +1 -1
- package/out/sdk/reduce.d.ts +33 -59
- package/out/sdk/reduce.d.ts.map +1 -1
- package/out/sdk/reduce.js +47 -78
- package/out/sdk/reduce.js.map +1 -1
- package/out/sdk/survey-area.d.ts +16 -48
- package/out/sdk/survey-area.d.ts.map +1 -1
- package/out/sdk/survey-area.js +33 -77
- package/out/sdk/survey-area.js.map +1 -1
- package/out/sdk/tabular.d.ts +32 -31
- package/out/sdk/tabular.d.ts.map +1 -1
- package/out/sdk/tabular.js +58 -56
- package/out/sdk/tabular.js.map +1 -1
- package/out/sdk/test-kit.d.ts +57 -0
- package/out/sdk/test-kit.d.ts.map +1 -0
- package/out/{test-kit.js → sdk/test-kit.js} +18 -39
- package/out/sdk/test-kit.js.map +1 -0
- package/out/sdk/verify.d.ts +15 -51
- package/out/sdk/verify.d.ts.map +1 -1
- package/out/sdk/verify.js +19 -77
- package/out/sdk/verify.js.map +1 -1
- package/out/vocabulary.d.ts +37 -103
- package/out/vocabulary.d.ts.map +1 -1
- package/out/vocabulary.js +34 -107
- package/out/vocabulary.js.map +1 -1
- package/package.json +36 -190
- package/{lib/sdk → sdk}/acquire.ts +17 -27
- package/{lib/sdk → sdk}/build-soil.ts +114 -128
- package/{lib/sdk → sdk}/cell-tiers.ts +20 -39
- package/{lib/sdk → sdk}/cells.ts +17 -43
- package/sdk/client.ts +147 -0
- package/sdk/download.ts +131 -0
- package/{lib/sdk/ingest-chunk.ts → sdk/ingest/chunk.ts} +34 -26
- package/{lib/scripts/ingest-chunk.ts → sdk/ingest/worker.ts} +10 -9
- package/sdk/ingest.ts +253 -0
- package/{lib/sdk → sdk}/measure-resolutions.ts +5 -16
- package/sdk/reduce.ts +344 -0
- package/{lib/sdk → sdk}/survey-area.ts +39 -83
- package/{lib/sdk → sdk}/tabular.ts +62 -59
- package/{lib → sdk}/test-kit.ts +18 -40
- package/{lib/sdk → sdk}/verify.ts +29 -85
- package/lib/sdk/client.ts +0 -184
- package/lib/sdk/download.ts +0 -161
- package/lib/sdk/index.ts +0 -20
- package/lib/sdk/ingest.ts +0 -278
- package/lib/sdk/reduce.ts +0 -375
- package/out/scripts/ingest-chunk.d.ts +0 -11
- package/out/scripts/ingest-chunk.d.ts.map +0 -1
- package/out/scripts/ingest-chunk.js.map +0 -1
- package/out/sdk/index.d.ts +0 -20
- package/out/sdk/index.d.ts.map +0 -1
- package/out/sdk/index.js +0 -20
- package/out/sdk/index.js.map +0 -1
- package/out/sdk/ingest-chunk.d.ts +0 -73
- package/out/sdk/ingest-chunk.d.ts.map +0 -1
- package/out/sdk/ingest-chunk.js.map +0 -1
- package/out/test-kit.d.ts +0 -79
- package/out/test-kit.d.ts.map +0 -1
- package/out/test-kit.js.map +0 -1
|
@@ -3,30 +3,11 @@
|
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*
|
|
6
|
-
*
|
|
6
|
+
* Builds the sealed `soil.db` polygon layer from NRCS soil survey areas.
|
|
7
7
|
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
* into a temporary table because memory has to stay flat in row count. A polygon layer's touches
|
|
12
|
-
* outnumber its features, and Iowa is 99 survey areas.
|
|
13
|
-
*
|
|
14
|
-
* ABSENCE IS NO ROW, IN EVERY TABLE. Land outside a published survey area gets no `layer_coverage` row and
|
|
15
|
-
* no summary row — never a zero, never an empty histogram. Inside a published area the coverage row says
|
|
16
|
-
* `designated` at completeness 1.0, because NRCS declares its own mapping complete for those areas at its
|
|
17
|
-
* own scale; a coverage cell reached ONLY by `NOTCOM` and access-denied polygons gets no row either,
|
|
18
|
-
* because the polygon exists and the soil mapping behind it does not.
|
|
19
|
-
*
|
|
20
|
-
* A COVERAGE ROW LICENSES ONLY THAT THE AUTHORITY MAPPED HERE. The reading is the class distribution, and
|
|
21
|
-
* a cell that is 100% `unrated_share` is `designated`-complete and carries no capability reading
|
|
22
|
-
* whatsoever. That pairing is not a corner case: 17.1% of national components carry no capability rating,
|
|
23
|
-
* and for the irrigated rating the figure is 85.1%.
|
|
24
|
-
*
|
|
25
|
-
* THE AREA CROSS-CHECK USES THE AUTHORITY'S OWN PUBLISHED FIGURE. `legend.areaacres` is what NRCS states
|
|
26
|
-
* the survey area covers — 378,800 acres for `IA153`, which is 1,532.9 km² against the 1,532.5 km² an
|
|
27
|
-
* independent projected measurement of the same shapefile reports. Comparing the spherical ring sum
|
|
28
|
-
* against it catches a hole read as an exterior ring, whose absence is silent: such a polygon is
|
|
29
|
-
* well-formed and simply answers "inside" for ground the authority did not map.
|
|
8
|
+
* The builder streams cell touches into a temporary SQL table so that memory stays flat as the row
|
|
9
|
+
* count grows. Absent data has no row in any table. Land outside a survey area and coverage cells that
|
|
10
|
+
* only `notcom` or access-denied polygons reach, get no coverage row.
|
|
30
11
|
*/
|
|
31
12
|
|
|
32
13
|
import { readFileSize } from "@mailwoman/core/fs/readers"
|
|
@@ -52,14 +33,15 @@ import {
|
|
|
52
33
|
type ParsedGeometry,
|
|
53
34
|
} from "@mailwoman/spatial"
|
|
54
35
|
import type { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
55
|
-
import { buildSealedArtifact } from "@mailwoman/sqlite/sealed
|
|
36
|
+
import { buildSealedArtifact } from "@mailwoman/sqlite/sealed/build"
|
|
56
37
|
import { cellToLatLng } from "h3-js"
|
|
38
|
+
import type { PathBuilderLike } from "path-ts"
|
|
57
39
|
|
|
58
40
|
import { createSoilTables, type SoilDatabase, type SoilSurveyAreaTable } from "#schema"
|
|
59
41
|
import { reduceCells, resolveCells } from "#sdk/cell-tiers"
|
|
60
42
|
import type { SoilFeatureSource } from "#sdk/ingest"
|
|
61
|
-
import type { SoilChunkResult } from "#sdk/ingest
|
|
62
|
-
import { ingestSoilChunk } from "#sdk/ingest
|
|
43
|
+
import type { SoilChunkResult } from "#sdk/ingest/chunk"
|
|
44
|
+
import { ingestSoilChunk } from "#sdk/ingest/chunk"
|
|
63
45
|
import { WEIGHT_LATTICE_DEPTH } from "#sdk/reduce"
|
|
64
46
|
import type { SurveyAreaAttributes } from "#sdk/survey-area"
|
|
65
47
|
import {
|
|
@@ -72,105 +54,115 @@ import {
|
|
|
72
54
|
} from "#vocabulary"
|
|
73
55
|
|
|
74
56
|
/**
|
|
75
|
-
*
|
|
76
|
-
*
|
|
57
|
+
* The schema version of the domain tables.
|
|
58
|
+
*
|
|
59
|
+
* It changes when a column changes meaning.
|
|
60
|
+
* An added column that readers can ignore does not change it.
|
|
77
61
|
*/
|
|
78
62
|
export const SOIL_SCHEMA_VERSION = 1
|
|
79
63
|
|
|
80
64
|
/**
|
|
81
|
-
*
|
|
65
|
+
* The number of delineation ids per chunk process.
|
|
82
66
|
*
|
|
83
|
-
*
|
|
84
|
-
*
|
|
85
|
-
*
|
|
86
|
-
* on this build the bound costs one process per area — which is what makes an area's failure nameable.
|
|
67
|
+
* A single process over a large polygon layer fails after several hundred thousand
|
|
68
|
+
* features because the h3 wasm heap fragments.
|
|
69
|
+
* This size stays well below that limit.
|
|
87
70
|
*/
|
|
88
71
|
export const DEFAULT_CHUNK_SIZE = 100_000
|
|
89
72
|
|
|
90
73
|
/**
|
|
91
|
-
* Square
|
|
74
|
+
* Square meters per acre, the unit of `legend.areaacres`.
|
|
92
75
|
*/
|
|
93
76
|
const M2_PER_ACRE = 4046.8564224
|
|
94
77
|
|
|
95
78
|
/**
|
|
96
|
-
* The relative gap between the ring-area total and the
|
|
79
|
+
* The relative gap between the ring-area total and the published acreage above which the build fails.
|
|
97
80
|
*
|
|
98
|
-
*
|
|
99
|
-
*
|
|
100
|
-
*
|
|
101
|
-
* — so an exact test would be brittle. Two percent sits far above the 0.03% `IA153` measures and far below the error a
|
|
102
|
-
* hole-blind read produces, which the zoning survey measured at 4.1% over a whole national layer.
|
|
81
|
+
* NRCS warns that its acreage can differ from a GIS measurement, so an exact
|
|
82
|
+
* comparison would fail on valid data.
|
|
83
|
+
* A hole read as an exterior ring produces a much larger gap than this tolerance.
|
|
103
84
|
*/
|
|
104
85
|
const AREA_TOLERANCE = 0.02
|
|
105
86
|
|
|
106
87
|
/**
|
|
107
|
-
* One survey area
|
|
88
|
+
* One survey area ready to build, with its geometry and tabular attributes.
|
|
108
89
|
*/
|
|
109
90
|
export interface SurveyAreaInput {
|
|
110
91
|
attributes: SurveyAreaAttributes
|
|
111
92
|
/**
|
|
112
|
-
* The map-unit polygon shapefile.
|
|
93
|
+
* The path of the map-unit polygon shapefile.
|
|
113
94
|
*/
|
|
114
95
|
shapefilePath?: string
|
|
115
96
|
/**
|
|
116
|
-
* The survey area's
|
|
97
|
+
* The survey area's outline.
|
|
117
98
|
*/
|
|
118
99
|
outline: ParsedGeometry
|
|
119
100
|
/**
|
|
120
|
-
* An in-process feature source
|
|
121
|
-
*
|
|
101
|
+
* An in-process feature source, used by fixtures.
|
|
102
|
+
*
|
|
103
|
+
* The batched path builds one of these per chunk, so both paths share one implementation.
|
|
122
104
|
*/
|
|
123
105
|
source?: SoilFeatureSource
|
|
124
106
|
/**
|
|
125
|
-
* The delineation count the shapefile declares, when the caller has already read it.
|
|
107
|
+
* The delineation count that the shapefile declares, when the caller has already read it.
|
|
126
108
|
*/
|
|
127
109
|
declaredFeatureCount?: number
|
|
128
110
|
}
|
|
129
111
|
|
|
112
|
+
/**
|
|
113
|
+
* Options for {@link buildSoilDatabase}.
|
|
114
|
+
*/
|
|
130
115
|
export interface BuildSoilOptions {
|
|
131
116
|
/**
|
|
132
|
-
* The survey areas to build, in
|
|
117
|
+
* The survey areas to build, in ingest order.
|
|
133
118
|
*/
|
|
134
119
|
areas: ReadonlyArray<SurveyAreaInput>
|
|
135
120
|
/**
|
|
136
|
-
* The region the layer name
|
|
121
|
+
* The region code in the layer name, such as `ia`.
|
|
137
122
|
*/
|
|
138
123
|
region: string
|
|
139
124
|
/**
|
|
140
|
-
*
|
|
125
|
+
* The path of the sealed artifact.
|
|
126
|
+
*
|
|
127
|
+
* The build writes a temporary file beside it and then swaps it in.
|
|
141
128
|
*/
|
|
142
|
-
out:
|
|
129
|
+
out: PathBuilderLike
|
|
143
130
|
/**
|
|
144
|
-
* The refresh
|
|
131
|
+
* The survey refresh date, written to `layer_manifest.version` and `source_vintage`.
|
|
145
132
|
*/
|
|
146
133
|
sourceVintage: string
|
|
147
134
|
buildCmd: string
|
|
148
135
|
buildSHA: string
|
|
149
136
|
/**
|
|
150
|
-
* ISO-8601, supplied by the caller
|
|
151
|
-
* makes two builds of the same inputs differ.
|
|
137
|
+
* The ISO-8601 creation time, supplied by the caller so that two builds of the same inputs are identical.
|
|
152
138
|
*/
|
|
153
139
|
createdAt: string
|
|
154
140
|
/**
|
|
155
|
-
* The resolution the cell index and the reduction
|
|
141
|
+
* The H3 resolution of the cell index and the reduction.
|
|
156
142
|
*/
|
|
157
143
|
indexResolution: number
|
|
158
144
|
/**
|
|
159
|
-
* The resolution `layer_coverage` rows
|
|
145
|
+
* The H3 resolution of the `layer_coverage` rows.
|
|
146
|
+
* It must be coarser than the index resolution.
|
|
160
147
|
*/
|
|
161
148
|
coverageResolution: number
|
|
162
149
|
/**
|
|
163
|
-
*
|
|
150
|
+
* The number of delineation ids per chunk process.
|
|
151
|
+
* See {@link DEFAULT_CHUNK_SIZE}.
|
|
164
152
|
*/
|
|
165
153
|
chunkSize?: number
|
|
166
154
|
/**
|
|
167
|
-
*
|
|
168
|
-
*
|
|
155
|
+
* Whether to run the ingest in this process instead of in chunk processes.
|
|
156
|
+
*
|
|
157
|
+
* Only fixtures use it, because they have no shapefile for a child process to open.
|
|
169
158
|
*/
|
|
170
159
|
inProcess?: boolean
|
|
171
160
|
onProgress?: (message: string) => void
|
|
172
161
|
}
|
|
173
162
|
|
|
163
|
+
/**
|
|
164
|
+
* The counts and measurements that {@link buildSoilDatabase} reports.
|
|
165
|
+
*/
|
|
174
166
|
export interface BuildSoilResult {
|
|
175
167
|
out: string
|
|
176
168
|
region: string
|
|
@@ -183,28 +175,30 @@ export interface BuildSoilResult {
|
|
|
183
175
|
wholeCellRows: number
|
|
184
176
|
partialCellRows: number
|
|
185
177
|
/**
|
|
186
|
-
* `partialCellRows / (wholeCellRows + partialCellRows)` over the
|
|
187
|
-
*
|
|
178
|
+
* The value of `partialCellRows / (wholeCellRows + partialCellRows)` over the stored index rows.
|
|
179
|
+
*
|
|
180
|
+
* Whole cells are compacted, so this differs from the partial share used to choose the resolution.
|
|
188
181
|
*/
|
|
189
182
|
storedPartialShare: number
|
|
190
183
|
capabilityCells: number
|
|
191
184
|
/**
|
|
192
|
-
*
|
|
185
|
+
* The number of cells reduced with the sampling lattice instead of the whole-cell fast path.
|
|
193
186
|
*/
|
|
194
187
|
sampledCells: number
|
|
195
188
|
/**
|
|
196
|
-
*
|
|
197
|
-
* out of a separate harness.
|
|
189
|
+
* The number of cells whose top class covers less than half the cell.
|
|
198
190
|
*/
|
|
199
191
|
topClassUnderHalfCells: number
|
|
200
192
|
topClassUnderHalfShare: number
|
|
201
193
|
/**
|
|
202
|
-
*
|
|
194
|
+
* The number of cells that the survey mapped but did not rate.
|
|
203
195
|
*/
|
|
204
196
|
classlessCells: number
|
|
205
197
|
/**
|
|
206
|
-
*
|
|
207
|
-
*
|
|
198
|
+
* The number of indexed cells that no lattice point landed in.
|
|
199
|
+
*
|
|
200
|
+
* These cells are dropped.
|
|
201
|
+
* A large count means the lattice is too coarse for the geometry.
|
|
208
202
|
*/
|
|
209
203
|
unsampledCells: number
|
|
210
204
|
meanDelineationsPerCell: number
|
|
@@ -212,25 +206,27 @@ export interface BuildSoilResult {
|
|
|
212
206
|
storedResolutions: number[]
|
|
213
207
|
coverageCells: number
|
|
214
208
|
/**
|
|
215
|
-
*
|
|
216
|
-
*
|
|
217
|
-
*
|
|
209
|
+
* The number of coverage cells inside a survey-area outline that no mapped delineation reached.
|
|
210
|
+
*
|
|
211
|
+
* SSURGO covers every part of a published area, so a large count means the outline
|
|
212
|
+
* and the delineations disagree.
|
|
218
213
|
*/
|
|
219
214
|
coverageCellsWithoutMapping: number
|
|
220
215
|
/**
|
|
221
|
-
* The area
|
|
222
|
-
*
|
|
216
|
+
* The ring-area totals in square kilometers, compared with the published acreage.
|
|
217
|
+
*
|
|
218
|
+
* Its `known` count is the number of survey areas that publish an acreage.
|
|
223
219
|
*/
|
|
224
220
|
area: AreaAgreementReading
|
|
225
221
|
sizeBytes: number
|
|
226
222
|
}
|
|
227
223
|
|
|
228
224
|
/**
|
|
229
|
-
*
|
|
225
|
+
* Builds the soil layer.
|
|
230
226
|
*
|
|
231
|
-
* @throws {Error}
|
|
232
|
-
*
|
|
233
|
-
*
|
|
227
|
+
* @throws {Error} When the classifier rejects a delineation, when a streamed count
|
|
228
|
+
* differs from the shapefile's declared count, when the area total differs from the
|
|
229
|
+
* published acreage, or when the outlines contain no interior coverage cell.
|
|
234
230
|
*/
|
|
235
231
|
export async function buildSoilDatabase(options: BuildSoilOptions): Promise<BuildSoilResult> {
|
|
236
232
|
if (options.coverageResolution >= options.indexResolution) {
|
|
@@ -252,17 +248,16 @@ export async function buildSoilDatabase(options: BuildSoilOptions): Promise<Buil
|
|
|
252
248
|
await createLayerManifestTable(kdb)
|
|
253
249
|
await createLayerCoverageTable(kdb)
|
|
254
250
|
|
|
255
|
-
// The touch table
|
|
256
|
-
//
|
|
257
|
-
//
|
|
251
|
+
// The touch table is dropped before the artifact is sealed.
|
|
252
|
+
// It has no primary key because a clustered key would sort every insert.
|
|
253
|
+
// The resolution queries use indexes that are created after the load.
|
|
258
254
|
kdb.exec(
|
|
259
255
|
"CREATE TABLE build_cell_touch (h3_cell INTEGER NOT NULL, resolution INTEGER NOT NULL, area_id TEXT NOT NULL, is_full INTEGER NOT NULL)"
|
|
260
256
|
)
|
|
261
257
|
},
|
|
262
258
|
ingest: async (kdb) => {
|
|
263
|
-
//
|
|
264
|
-
// them
|
|
265
|
-
// than after millions of geometry rows are written.
|
|
259
|
+
// The attributes are written first because the ingest needs the map units that have no soil mapping.
|
|
260
|
+
// The build writes them first so a delineation with a missing map unit fails early.
|
|
266
261
|
writeAttributes(kdb, options.areas)
|
|
267
262
|
|
|
268
263
|
if (!options.inProcess) return undefined
|
|
@@ -293,10 +288,10 @@ export async function buildSoilDatabase(options: BuildSoilOptions): Promise<Buil
|
|
|
293
288
|
writeSurveyAreaRows(kdb, options, coverage.cellsByArea)
|
|
294
289
|
writeVocabularyRows(kdb, options.areas)
|
|
295
290
|
|
|
296
|
-
//
|
|
297
|
-
//
|
|
298
|
-
//
|
|
299
|
-
//
|
|
291
|
+
// The spine key is the column that consumers join on.
|
|
292
|
+
// It points at `soil_capability_cell`, which has one row per cell at one resolution.
|
|
293
|
+
// The `soil_map_unit_cell` table is keyed by cell and delineation at mixed
|
|
294
|
+
// resolutions, so consumers cannot join on it.
|
|
300
295
|
await writeLayerManifest(
|
|
301
296
|
kdb,
|
|
302
297
|
polygonLayerManifest(options, {
|
|
@@ -314,7 +309,7 @@ export async function buildSoilDatabase(options: BuildSoilOptions): Promise<Buil
|
|
|
314
309
|
const totalCellRows = cells.wholeRows + cells.partialRows
|
|
315
310
|
|
|
316
311
|
return {
|
|
317
|
-
out: options.out,
|
|
312
|
+
out: options.out.toString(),
|
|
318
313
|
region: options.region,
|
|
319
314
|
surveyAreas: options.areas.length,
|
|
320
315
|
delineations: ingested.delineations,
|
|
@@ -345,7 +340,7 @@ export async function buildSoilDatabase(options: BuildSoilOptions): Promise<Buil
|
|
|
345
340
|
}
|
|
346
341
|
|
|
347
342
|
/**
|
|
348
|
-
*
|
|
343
|
+
* The combined result of every ingest chunk.
|
|
349
344
|
*/
|
|
350
345
|
interface StreamResult {
|
|
351
346
|
delineations: number
|
|
@@ -358,10 +353,9 @@ interface StreamResult {
|
|
|
358
353
|
}
|
|
359
354
|
|
|
360
355
|
/**
|
|
361
|
-
*
|
|
356
|
+
* Sums the results of the ingest chunks.
|
|
362
357
|
*
|
|
363
|
-
*
|
|
364
|
-
* reach, and getting it wrong produces a well-formed artifact that under-reports how many delineations a cell holds.
|
|
358
|
+
* It is exported for a unit test because fixture builds do not exercise the batched path.
|
|
365
359
|
*/
|
|
366
360
|
export function aggregateChunks(chunks: ReadonlyArray<SoilChunkResult>): StreamResult {
|
|
367
361
|
const byArea = new Map<string, number>()
|
|
@@ -381,8 +375,7 @@ export function aggregateChunks(chunks: ReadonlyArray<SoilChunkResult>): StreamR
|
|
|
381
375
|
|
|
382
376
|
byArea.set(chunk.areaSymbol, (byArea.get(chunk.areaSymbol) ?? 0) + chunk.delineations)
|
|
383
377
|
|
|
384
|
-
//
|
|
385
|
-
// straddle two survey areas — so the counts ADD rather than replace.
|
|
378
|
+
// One coverage cell can appear in several chunks and in two survey areas, so the counts are summed.
|
|
386
379
|
mergeCountsInto(observedByCoverageCell, chunk.observedByCoverageCell)
|
|
387
380
|
mergeCountsInto(mappedByCoverageCell, chunk.mappedByCoverageCell)
|
|
388
381
|
}
|
|
@@ -391,9 +384,7 @@ export function aggregateChunks(chunks: ReadonlyArray<SoilChunkResult>): StreamR
|
|
|
391
384
|
}
|
|
392
385
|
|
|
393
386
|
/**
|
|
394
|
-
*
|
|
395
|
-
*
|
|
396
|
-
* A short read builds a smaller county and reports success, which is the partial result that must throw.
|
|
387
|
+
* Throws when the streamed delineation count of a survey area differs from its declared count.
|
|
397
388
|
*/
|
|
398
389
|
function assertDelineationCounts(areas: ReadonlyArray<SurveyAreaInput>, streamed: StreamResult): void {
|
|
399
390
|
for (const area of areas) {
|
|
@@ -412,11 +403,11 @@ function assertDelineationCounts(areas: ReadonlyArray<SurveyAreaInput>, streamed
|
|
|
412
403
|
}
|
|
413
404
|
|
|
414
405
|
/**
|
|
415
|
-
*
|
|
406
|
+
* Throws when the ring-area total differs from the published acreage by more than {@link AREA_TOLERANCE}.
|
|
416
407
|
*
|
|
417
|
-
* The message
|
|
418
|
-
*
|
|
419
|
-
*
|
|
408
|
+
* The error message includes the total with holes ignored, because a hole read
|
|
409
|
+
* as an exterior ring causes the gap.
|
|
410
|
+
* When no survey area publishes an acreage, the check is skipped.
|
|
420
411
|
*/
|
|
421
412
|
function assertAreaAgreement(areas: ReadonlyArray<SurveyAreaInput>, streamed: StreamResult): AreaAgreementReading {
|
|
422
413
|
let publishedAcres = 0
|
|
@@ -447,7 +438,7 @@ function assertAreaAgreement(areas: ReadonlyArray<SurveyAreaInput>, streamed: St
|
|
|
447
438
|
}
|
|
448
439
|
|
|
449
440
|
/**
|
|
450
|
-
*
|
|
441
|
+
* Writes every survey area's map units and components.
|
|
451
442
|
*/
|
|
452
443
|
function writeAttributes(database: DatabaseClient<SoilDatabase>, areas: ReadonlyArray<SurveyAreaInput>): void {
|
|
453
444
|
const insertMapUnit = database.prepare(
|
|
@@ -499,7 +490,7 @@ function writeAttributes(database: DatabaseClient<SoilDatabase>, areas: Readonly
|
|
|
499
490
|
}
|
|
500
491
|
|
|
501
492
|
/**
|
|
502
|
-
*
|
|
493
|
+
* Returns the keys of the map units that have no soil mapping.
|
|
503
494
|
*/
|
|
504
495
|
function noMappingMukeys(areas: ReadonlyArray<SurveyAreaInput>): Set<string> {
|
|
505
496
|
const mukeys = new Set<string>()
|
|
@@ -516,7 +507,8 @@ function noMappingMukeys(areas: ReadonlyArray<SurveyAreaInput>): Set<string> {
|
|
|
516
507
|
}
|
|
517
508
|
|
|
518
509
|
/**
|
|
519
|
-
*
|
|
510
|
+
* Ingests one chunk per survey area in this process.
|
|
511
|
+
* Only fixtures use it.
|
|
520
512
|
*/
|
|
521
513
|
async function ingestInProcess(
|
|
522
514
|
database: DatabaseClient<SoilDatabase>,
|
|
@@ -547,13 +539,13 @@ async function ingestInProcess(
|
|
|
547
539
|
}
|
|
548
540
|
|
|
549
541
|
/**
|
|
550
|
-
*
|
|
551
|
-
*
|
|
552
|
-
*
|
|
542
|
+
* Runs the ingest as a sequence of child processes, one per FID range of one survey area.
|
|
543
|
+
*
|
|
544
|
+
* The shared chunk protocol and its error handling live in `ingestChunkArguments` and `runChunkProcess`.
|
|
553
545
|
*/
|
|
554
546
|
async function runBatchedIngest(tmpPath: string, options: BuildSoilOptions): Promise<SoilChunkResult[]> {
|
|
555
547
|
const chunkSize = options.chunkSize ?? DEFAULT_CHUNK_SIZE
|
|
556
|
-
const script = resolveModulePath("@mailwoman/soil/
|
|
548
|
+
const script = resolveModulePath("@mailwoman/soil/sdk/ingest/worker")
|
|
557
549
|
const noMapping = [...noMappingMukeys(options.areas)]
|
|
558
550
|
const chunks: SoilChunkResult[] = []
|
|
559
551
|
|
|
@@ -602,22 +594,16 @@ async function runBatchedIngest(tmpPath: string, options: BuildSoilOptions): Pro
|
|
|
602
594
|
}
|
|
603
595
|
|
|
604
596
|
/**
|
|
605
|
-
*
|
|
597
|
+
* Returns one coverage row per interior cell of the built footprint that soil mapping reaches.
|
|
606
598
|
*
|
|
607
|
-
*
|
|
608
|
-
*
|
|
609
|
-
*
|
|
610
|
-
* county spans by area, so more than half of it would read `unknown` while sitting inside a survey the build had
|
|
611
|
-
* ingested. Run over the union, only the OUTER border of the built set is dropped, which is the honest edge: beyond it
|
|
612
|
-
* lies ground this artifact does not hold.
|
|
599
|
+
* The interior test keeps only cells that lie wholly inside the footprint.
|
|
600
|
+
* It runs once over the union of all outlines, because running it per survey area
|
|
601
|
+
* would drop every cell that crosses an internal county border.
|
|
613
602
|
*
|
|
614
|
-
*
|
|
615
|
-
* never looked at, and a point in the dropped strip reading `unknown` is the truthful answer for ground the built set
|
|
616
|
-
* may or may not reach.
|
|
603
|
+
* Only cells on the outer border of the built set are dropped.
|
|
617
604
|
*
|
|
618
|
-
* `observed_rows` counts the delineations
|
|
619
|
-
* only
|
|
620
|
-
* and the survey's §3.2 puts that case with the absences rather than with the coverage.
|
|
605
|
+
* The `observed_rows` column counts the delineations that reach the cell.
|
|
606
|
+
* A cell that only `notcom` and access-denied polygons reach gets no row, because it has no soil mapping.
|
|
621
607
|
*/
|
|
622
608
|
function buildCoverageCells(
|
|
623
609
|
options: BuildSoilOptions,
|
|
@@ -655,8 +641,8 @@ function buildCoverageCells(
|
|
|
655
641
|
continue
|
|
656
642
|
}
|
|
657
643
|
|
|
658
|
-
//
|
|
659
|
-
//
|
|
644
|
+
// The cell's center decides its survey area, so each cell counts for one area only.
|
|
645
|
+
// This per-area count is informational and does not affect the coverage rows.
|
|
660
646
|
const [latitude, longitude] = cellToLatLng(cell)
|
|
661
647
|
const owner = options.areas.find((input) => geometryContains(input.outline, longitude, latitude))
|
|
662
648
|
|
|
@@ -671,10 +657,10 @@ function buildCoverageCells(
|
|
|
671
657
|
}
|
|
672
658
|
|
|
673
659
|
/**
|
|
674
|
-
*
|
|
660
|
+
* Returns an outline's polygons in `MultiPolygon` coordinate form.
|
|
675
661
|
*
|
|
676
|
-
* @throws {TypeError} When the outline is not
|
|
677
|
-
*
|
|
662
|
+
* @throws {TypeError} When the outline is not a polygon or multipolygon.
|
|
663
|
+
* The build would silently drop that survey area's coverage if it skipped the outline.
|
|
678
664
|
*/
|
|
679
665
|
function outlinePolygons(outline: ParsedGeometry): MultiPolygonRings {
|
|
680
666
|
const polygons = arealPolygons(outline)
|
|
@@ -687,7 +673,7 @@ function outlinePolygons(outline: ParsedGeometry): MultiPolygonRings {
|
|
|
687
673
|
}
|
|
688
674
|
|
|
689
675
|
/**
|
|
690
|
-
*
|
|
676
|
+
* Inserts one row per survey area.
|
|
691
677
|
*/
|
|
692
678
|
function writeSurveyAreaRows(
|
|
693
679
|
database: DatabaseClient<SoilDatabase>,
|
|
@@ -754,10 +740,10 @@ function writeSurveyAreaRows(
|
|
|
754
740
|
}
|
|
755
741
|
|
|
756
742
|
/**
|
|
757
|
-
*
|
|
743
|
+
* Inserts the authority's declared domains and the share weighting.
|
|
758
744
|
*
|
|
759
|
-
*
|
|
760
|
-
*
|
|
745
|
+
* Every cell row stores the weighting code.
|
|
746
|
+
* The vocabulary row adds the description of that code.
|
|
761
747
|
*/
|
|
762
748
|
function writeVocabularyRows(database: DatabaseClient<SoilDatabase>, areas: ReadonlyArray<SurveyAreaInput>): void {
|
|
763
749
|
const insert = database.prepare(
|
|
@@ -3,17 +3,7 @@
|
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*
|
|
6
|
-
*
|
|
7
|
-
* index, and the reduction above it.
|
|
8
|
-
*
|
|
9
|
-
* BOTH READ THE TOUCH TABLE, AND ONLY ONE OF THEM COMPACTS. `resolveCells` writes the tiers a probe walks
|
|
10
|
-
* and collapses uniform interiors parent-ward; `reduceCells` reads the UNCOMPACTED touches, because a
|
|
11
|
-
* compacted parent no longer names the cells the reduction has to answer.
|
|
12
|
-
*
|
|
13
|
-
* THE REDUCTION IS THE SLOW PHASE AND ITS MEMORY IS BOUNDED BY CONSTRUCTION. A lattice of 49 point tests
|
|
14
|
-
* per sampled cell, over a delineation cache that is cleared whole rather than evicted one entry at a
|
|
15
|
-
* time — see {@link GEOMETRY_CACHE_ENTRIES}. Memory stays flat in row count, which is the property the poi
|
|
16
|
-
* build lost when a reader materialized instead of streaming.
|
|
6
|
+
* Turns the build's cell-touch table into the containment index and the per-cell capability reduction.
|
|
17
7
|
*/
|
|
18
8
|
|
|
19
9
|
import { expandShortCellInt, shortCellToInt, type H3Cell } from "@mailwoman/spatial"
|
|
@@ -25,11 +15,10 @@ import { SoilCellContainment, type SoilCapabilityCellTable, type SoilDatabase }
|
|
|
25
15
|
import { mapUnitProfile, reduceCell, type CellCandidate, type MapUnitProfile } from "#sdk/reduce"
|
|
26
16
|
|
|
27
17
|
/**
|
|
28
|
-
*
|
|
18
|
+
* Writes the touch table into the `soil_map_unit_cell` containment index.
|
|
29
19
|
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
* smaller than one resolution-9 cell.
|
|
20
|
+
* Only whole cells are compacted.
|
|
21
|
+
* Most soil delineations are smaller than one index cell, so compaction removes few rows on this layer.
|
|
33
22
|
*/
|
|
34
23
|
export function resolveCells(
|
|
35
24
|
database: DatabaseClient<SoilDatabase>,
|
|
@@ -49,9 +38,8 @@ export function resolveCells(
|
|
|
49
38
|
|
|
50
39
|
let wholeRows = 0
|
|
51
40
|
|
|
52
|
-
//
|
|
53
|
-
//
|
|
54
|
-
// delineation's interior.
|
|
41
|
+
// Each group is one delineation at one resolution, because `compactCells` throws on mixed resolutions.
|
|
42
|
+
// Coarsened delineations are indexed at other resolutions, so every group must be compacted.
|
|
55
43
|
database.exec("BEGIN")
|
|
56
44
|
|
|
57
45
|
for (const { area_id: areaID, resolution } of groups) {
|
|
@@ -71,9 +59,8 @@ export function resolveCells(
|
|
|
71
59
|
|
|
72
60
|
database.exec("COMMIT")
|
|
73
61
|
|
|
74
|
-
// The partial rows
|
|
75
|
-
//
|
|
76
|
-
// than rely on the key: a partial row replacing a whole one would demote an answered cell to a ray cast.
|
|
62
|
+
// The partial rows use `INSERT OR IGNORE` so that they never replace a whole row with the same key.
|
|
63
|
+
// A replaced whole row would force the reader to ray-cast a cell it could answer directly.
|
|
77
64
|
database.exec(
|
|
78
65
|
"INSERT OR IGNORE INTO soil_map_unit_cell (h3_cell, resolution, area_id, containment) " +
|
|
79
66
|
"SELECT DISTINCT t.h3_cell, t.resolution, t.area_id, 'partial' FROM build_cell_touch t WHERE t.is_full = 0"
|
|
@@ -97,13 +84,12 @@ export function resolveCells(
|
|
|
97
84
|
}
|
|
98
85
|
|
|
99
86
|
/**
|
|
100
|
-
*
|
|
101
|
-
* so it reports often enough that a long run is visibly alive.
|
|
87
|
+
* The number of cells reduced between progress reports and commits.
|
|
102
88
|
*/
|
|
103
89
|
const REDUCE_PROGRESS_STRIDE = 50_000
|
|
104
90
|
|
|
105
91
|
/**
|
|
106
|
-
* One delineation's geometry
|
|
92
|
+
* One delineation's geometry as the reduction reads it.
|
|
107
93
|
*/
|
|
108
94
|
interface StoredDelineation {
|
|
109
95
|
mukey: string
|
|
@@ -115,21 +101,17 @@ interface StoredDelineation {
|
|
|
115
101
|
}
|
|
116
102
|
|
|
117
103
|
/**
|
|
118
|
-
*
|
|
104
|
+
* The maximum number of delineations that the reduction keeps in memory.
|
|
119
105
|
*
|
|
120
|
-
*
|
|
121
|
-
* kB, so 200,000 of them is a few hundred megabytes — comfortable, and far below the 2.5 million a whole state holds.
|
|
122
|
-
* Memory stays flat in row count, which is the property the poi build lost when a reader materialized instead of
|
|
123
|
-
* streaming.
|
|
106
|
+
* The limit keeps memory bounded when a state has millions of delineations.
|
|
124
107
|
*/
|
|
125
108
|
const GEOMETRY_CACHE_ENTRIES = 200_000
|
|
126
109
|
|
|
127
110
|
/**
|
|
128
|
-
*
|
|
111
|
+
* Reduces the touch table into `soil_capability_cell`.
|
|
129
112
|
*
|
|
130
|
-
*
|
|
131
|
-
*
|
|
132
|
-
* about which delineation reaches which cell.
|
|
113
|
+
* It reads the touch table because `soil_map_unit_cell` is compacted.
|
|
114
|
+
* A compacted parent cell hides the index cells that the reduction must answer.
|
|
133
115
|
*/
|
|
134
116
|
export function reduceCells(
|
|
135
117
|
database: DatabaseClient<SoilDatabase>,
|
|
@@ -169,11 +151,10 @@ export function reduceCells(
|
|
|
169
151
|
let currentResolution = indexResolution
|
|
170
152
|
let candidates: CellCandidate[] = []
|
|
171
153
|
|
|
172
|
-
//
|
|
173
|
-
//
|
|
174
|
-
//
|
|
175
|
-
//
|
|
176
|
-
// refills with the delineations the next run of cells actually names.
|
|
154
|
+
// Each delineation touches several cells, so the cache avoids reading its rings once per touch.
|
|
155
|
+
// The cache is cleared whole when it fills.
|
|
156
|
+
// The scan runs in `h3_cell` order and visits neighboring cells together.
|
|
157
|
+
// After a clear, the cache refills with the delineations that the next cells need.
|
|
177
158
|
const geometry = new Map<string, StoredDelineation>()
|
|
178
159
|
|
|
179
160
|
const batch = beginBatched(database, { rowsPerCommit: REDUCE_PROGRESS_STRIDE })
|
|
@@ -277,7 +258,7 @@ function insertRow(statement: ReturnType<DatabaseClient["prepare"]>, row: SoilCa
|
|
|
277
258
|
}
|
|
278
259
|
|
|
279
260
|
/**
|
|
280
|
-
*
|
|
261
|
+
* Computes every map unit's profile once so that each cell reuses it.
|
|
281
262
|
*/
|
|
282
263
|
function readMapUnitProfiles(database: DatabaseClient<SoilDatabase>): Map<string, MapUnitProfile> {
|
|
283
264
|
const componentsByMukey = new Map<
|