@mailwoman/soil 9.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +221 -0
- package/lib/index.ts +474 -0
- package/lib/schema.ts +493 -0
- package/lib/scripts/ingest-chunk.ts +47 -0
- package/lib/sdk/acquire.ts +124 -0
- package/lib/sdk/build-soil.ts +778 -0
- package/lib/sdk/cell-tiers.ts +310 -0
- package/lib/sdk/cells.ts +224 -0
- package/lib/sdk/client.ts +273 -0
- package/lib/sdk/download.ts +161 -0
- package/lib/sdk/index.ts +20 -0
- package/lib/sdk/ingest-chunk.ts +182 -0
- package/lib/sdk/ingest.ts +271 -0
- package/lib/sdk/measure-resolutions.ts +81 -0
- package/lib/sdk/reduce.ts +375 -0
- package/lib/sdk/survey-area.ts +467 -0
- package/lib/sdk/tabular.ts +296 -0
- package/lib/sdk/verify.ts +369 -0
- package/lib/test-kit.ts +255 -0
- package/lib/vocabulary.ts +241 -0
- package/out/index.d.ts +199 -0
- package/out/index.d.ts.map +1 -0
- package/out/index.js +255 -0
- package/out/index.js.map +1 -0
- package/out/schema.d.ts +369 -0
- package/out/schema.d.ts.map +1 -0
- package/out/schema.js +186 -0
- package/out/schema.js.map +1 -0
- package/out/scripts/ingest-chunk.d.ts +11 -0
- package/out/scripts/ingest-chunk.d.ts.map +1 -0
- package/out/scripts/ingest-chunk.js +42 -0
- package/out/scripts/ingest-chunk.js.map +1 -0
- package/out/sdk/acquire.d.ts +60 -0
- package/out/sdk/acquire.d.ts.map +1 -0
- package/out/sdk/acquire.js +74 -0
- package/out/sdk/acquire.js.map +1 -0
- package/out/sdk/build-soil.d.ts +197 -0
- package/out/sdk/build-soil.d.ts.map +1 -0
- package/out/sdk/build-soil.js +434 -0
- package/out/sdk/build-soil.js.map +1 -0
- package/out/sdk/cell-tiers.d.ts +47 -0
- package/out/sdk/cell-tiers.d.ts.map +1 -0
- package/out/sdk/cell-tiers.js +194 -0
- package/out/sdk/cell-tiers.js.map +1 -0
- package/out/sdk/cells.d.ts +118 -0
- package/out/sdk/cells.d.ts.map +1 -0
- package/out/sdk/cells.js +140 -0
- package/out/sdk/cells.js.map +1 -0
- package/out/sdk/client.d.ts +120 -0
- package/out/sdk/client.d.ts.map +1 -0
- package/out/sdk/client.js +214 -0
- package/out/sdk/client.js.map +1 -0
- package/out/sdk/download.d.ts +84 -0
- package/out/sdk/download.d.ts.map +1 -0
- package/out/sdk/download.js +117 -0
- package/out/sdk/download.js.map +1 -0
- package/out/sdk/index.d.ts +20 -0
- package/out/sdk/index.d.ts.map +1 -0
- package/out/sdk/index.js +20 -0
- package/out/sdk/index.js.map +1 -0
- package/out/sdk/ingest-chunk.d.ts +73 -0
- package/out/sdk/ingest-chunk.d.ts.map +1 -0
- package/out/sdk/ingest-chunk.js +98 -0
- package/out/sdk/ingest-chunk.js.map +1 -0
- package/out/sdk/ingest.d.ts +132 -0
- package/out/sdk/ingest.d.ts.map +1 -0
- package/out/sdk/ingest.js +163 -0
- package/out/sdk/ingest.js.map +1 -0
- package/out/sdk/measure-resolutions.d.ts +39 -0
- package/out/sdk/measure-resolutions.d.ts.map +1 -0
- package/out/sdk/measure-resolutions.js +52 -0
- package/out/sdk/measure-resolutions.js.map +1 -0
- package/out/sdk/reduce.d.ts +122 -0
- package/out/sdk/reduce.d.ts.map +1 -0
- package/out/sdk/reduce.js +266 -0
- package/out/sdk/reduce.js.map +1 -0
- package/out/sdk/survey-area.d.ts +105 -0
- package/out/sdk/survey-area.d.ts.map +1 -0
- package/out/sdk/survey-area.js +346 -0
- package/out/sdk/survey-area.js.map +1 -0
- package/out/sdk/tabular.d.ts +106 -0
- package/out/sdk/tabular.d.ts.map +1 -0
- package/out/sdk/tabular.js +194 -0
- package/out/sdk/tabular.js.map +1 -0
- package/out/sdk/verify.d.ts +135 -0
- package/out/sdk/verify.d.ts.map +1 -0
- package/out/sdk/verify.js +221 -0
- package/out/sdk/verify.js.map +1 -0
- package/out/test-kit.d.ts +80 -0
- package/out/test-kit.d.ts.map +1 -0
- package/out/test-kit.js +228 -0
- package/out/test-kit.js.map +1 -0
- package/out/vocabulary.d.ts +179 -0
- package/out/vocabulary.d.ts.map +1 -0
- package/out/vocabulary.js +215 -0
- package/out/vocabulary.js.map +1 -0
- package/package.json +276 -0
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* The pipe-delimited NASIS export inside a survey-area archive: which file holds which table, which
|
|
7
|
+
* column sits at which position, and the authority's own declared domains.
|
|
8
|
+
*
|
|
9
|
+
* THE FILES CARRY NO HEADER AND THE ARCHIVE SHIPS THE SCHEMA. `mstab.txt` maps a logical table name to
|
|
10
|
+
* the file base name that holds it (`component` → `comp.txt`, `sacatalog` → `sacatlog.txt` — neither is
|
|
11
|
+
* guessable), and `mstabcol.txt` gives every column's ordinal position. So the reader looks the positions
|
|
12
|
+
* up rather than hard-coding them, and {@link readTable} THROWS on a requested column the shipped
|
|
13
|
+
* dictionary does not declare. A reader that quietly returned `undefined` for a renamed column would turn
|
|
14
|
+
* "the source changed" into "there is none of it", at exactly the measurement boundary where that lie
|
|
15
|
+
* costs the most.
|
|
16
|
+
*
|
|
17
|
+
* QUOTE HANDLING IS NOT OPTIONAL HERE, AND THE MEASUREMENT determines the result. `sacatlog.txt` holds 594 newline
|
|
18
|
+
* bytes and exactly ONE record: its `fgdcmetadata` column carries a 43,251-character XML document with
|
|
19
|
+
* embedded newlines. `mstabcol.txt` — the column dictionary itself — holds 913 newlines and 865 records.
|
|
20
|
+
* A line-splitting reader gets 594 malformed rows from a one-row file, every one of them well-formed
|
|
21
|
+
* enough to keep going.
|
|
22
|
+
*
|
|
23
|
+
* THE DECLARED DOMAINS COME OUT OF THE ARCHIVE TOO, which is stronger than transcribing them.
|
|
24
|
+
* `msdomdet.txt` carries every `Choice` column's members WITH the authority's own prose definition —
|
|
25
|
+
* capability classes 1 through 8, subclasses `c`/`e`/`s`/`w`, the 28 conditional farmland
|
|
26
|
+
* classifications, the six component kinds. The layer stores them and validates against them.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import { readLocalBuffer } from "@mailwoman/core/fs/readers"
|
|
30
|
+
import { join, type PathBuilderLike } from "path-ts"
|
|
31
|
+
import { CSVSpliterator } from "spliterator"
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* One parsed record: the raw column strings, in the authority's declared order.
|
|
35
|
+
*/
|
|
36
|
+
export type TabularRow = ReadonlyArray<string>
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Read one pipe-delimited export file into rows.
|
|
40
|
+
*
|
|
41
|
+
* `enableQuoteHandling` is what makes the embedded newlines above survive; `header: false` is what keeps the first
|
|
42
|
+
* record from being eaten, since these files carry none.
|
|
43
|
+
*/
|
|
44
|
+
async function readPipeDelimited(path: PathBuilderLike): Promise<TabularRow[]> {
|
|
45
|
+
const rows: TabularRow[] = []
|
|
46
|
+
|
|
47
|
+
for (const row of CSVSpliterator.from(await readLocalBuffer(path), {
|
|
48
|
+
mode: "array",
|
|
49
|
+
header: false,
|
|
50
|
+
columnDelimiter: "|",
|
|
51
|
+
enableQuoteHandling: true,
|
|
52
|
+
})) {
|
|
53
|
+
// `String` ON A DECLARED STRING IS NOT REDUNDANT HERE. The emitter's array mode is TYPED `string[]` and coerces
|
|
54
|
+
// numeric-looking columns at RUNTIME — measured: `mukey` comes back as the number 412818, not `"412818"` — so a row
|
|
55
|
+
// passed through untouched stops joining against the shapefile's own `MUKEY`. An empty column arrives as `""`
|
|
56
|
+
// rather than as null, which is why nothing here has to decide what a missing value means.
|
|
57
|
+
rows.push(row.map((value) => String(value)))
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
return rows
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* The archive's own description of itself: table → file, and table → column positions.
|
|
65
|
+
*/
|
|
66
|
+
export interface TabularDictionary {
|
|
67
|
+
/**
|
|
68
|
+
* Logical table name → file base name, from `mstab.txt`.
|
|
69
|
+
*/
|
|
70
|
+
files: ReadonlyMap<string, string>
|
|
71
|
+
/**
|
|
72
|
+
* Logical table name → column name → zero-based position, from `mstabcol.txt`.
|
|
73
|
+
*/
|
|
74
|
+
columns: ReadonlyMap<string, ReadonlyMap<string, number>>
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Column positions in `mstab.txt` and `mstabcol.txt` themselves.
|
|
79
|
+
*
|
|
80
|
+
* These two are the ONLY positions this module hard-codes, and they cannot be looked up because they are what the
|
|
81
|
+
* lookup is built from. Both files declare themselves in `mstabcol.txt`, so the assertions below check the bootstrap
|
|
82
|
+
* against the archive's own account of it rather than trusting it.
|
|
83
|
+
*/
|
|
84
|
+
const MSTAB_TABLE_NAME = 0
|
|
85
|
+
const MSTAB_FILE_NAME = 4
|
|
86
|
+
const MSTABCOL_TABLE_NAME = 0
|
|
87
|
+
const MSTABCOL_POSITION = 1
|
|
88
|
+
const MSTABCOL_COLUMN_NAME = 2
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Declared widths of the two bootstrap files, asserted before either is read as a dictionary. A different width means
|
|
92
|
+
* the metadata format changed, and reading positions out of a changed format is how a builder mis-reads every column at
|
|
93
|
+
* once.
|
|
94
|
+
*/
|
|
95
|
+
const MSTAB_WIDTH = 5
|
|
96
|
+
const MSTABCOL_WIDTH = 14
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Read the archive's table and column dictionaries.
|
|
100
|
+
*
|
|
101
|
+
* @throws {Error} When either bootstrap file is missing or is not the declared width.
|
|
102
|
+
*/
|
|
103
|
+
export async function readTabularDictionary(tabularDirectory: PathBuilderLike): Promise<TabularDictionary> {
|
|
104
|
+
const mstab = await readPipeDelimited(join(tabularDirectory, "mstab.txt"))
|
|
105
|
+
const mstabcol = await readPipeDelimited(join(tabularDirectory, "mstabcol.txt"))
|
|
106
|
+
|
|
107
|
+
assertWidth(mstab, MSTAB_WIDTH, "mstab.txt")
|
|
108
|
+
assertWidth(mstabcol, MSTABCOL_WIDTH, "mstabcol.txt")
|
|
109
|
+
|
|
110
|
+
const files = new Map<string, string>()
|
|
111
|
+
|
|
112
|
+
for (const row of mstab) {
|
|
113
|
+
files.set(row[MSTAB_TABLE_NAME]!, row[MSTAB_FILE_NAME]!)
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
const columns = new Map<string, Map<string, number>>()
|
|
117
|
+
|
|
118
|
+
for (const row of mstabcol) {
|
|
119
|
+
const table = row[MSTABCOL_TABLE_NAME]!
|
|
120
|
+
const position = Number(row[MSTABCOL_POSITION])
|
|
121
|
+
|
|
122
|
+
let byName = columns.get(table)
|
|
123
|
+
|
|
124
|
+
if (!byName) {
|
|
125
|
+
byName = new Map()
|
|
126
|
+
|
|
127
|
+
columns.set(table, byName)
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
byName.set(row[MSTABCOL_COLUMN_NAME]!, position - 1)
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
return { files, columns }
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function assertWidth(rows: ReadonlyArray<TabularRow>, width: number, name: string): void {
|
|
137
|
+
const widths = new Set(rows.map((row) => row.length))
|
|
138
|
+
|
|
139
|
+
if (widths.size !== 1 || !widths.has(width)) {
|
|
140
|
+
throw new Error(
|
|
141
|
+
`soil tabular: ${name} holds rows of width ${[...widths].join(", ")}, expected ${width} — the shipped metadata format changed, and reading column positions out of a changed format mis-reads every column at once`
|
|
142
|
+
)
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* A reader over one logical table, projecting the columns a caller names.
|
|
148
|
+
*
|
|
149
|
+
* The projection is by NAME and a missing name throws, which is the whole point: this is the shape that produced the
|
|
150
|
+
* repository's worst measurement bugs, where a silently dropped column read downstream as an empty world.
|
|
151
|
+
*/
|
|
152
|
+
export interface TabularTable {
|
|
153
|
+
/**
|
|
154
|
+
* One record per row, already projected to the requested columns.
|
|
155
|
+
*/
|
|
156
|
+
rows: ReadonlyArray<Record<string, string>>
|
|
157
|
+
/**
|
|
158
|
+
* How many records the file held, before projection.
|
|
159
|
+
*/
|
|
160
|
+
recordCount: number
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Read a logical SSURGO table, projecting `wanted` columns.
|
|
165
|
+
*
|
|
166
|
+
* @throws {Error} When the archive declares no file for the table, when a requested column is not in the shipped
|
|
167
|
+
* dictionary, or when a record is narrower than the position a requested column sits at.
|
|
168
|
+
*/
|
|
169
|
+
export async function readTable(
|
|
170
|
+
tabularDirectory: PathBuilderLike,
|
|
171
|
+
dictionary: TabularDictionary,
|
|
172
|
+
table: string,
|
|
173
|
+
wanted: ReadonlyArray<string>
|
|
174
|
+
): Promise<TabularTable> {
|
|
175
|
+
const file = dictionary.files.get(table)
|
|
176
|
+
|
|
177
|
+
if (!file) {
|
|
178
|
+
throw new Error(`soil tabular: the archive's mstab.txt declares no file for table ${JSON.stringify(table)}`)
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
const positions = dictionary.columns.get(table)
|
|
182
|
+
|
|
183
|
+
if (!positions) {
|
|
184
|
+
throw new Error(`soil tabular: the archive's mstabcol.txt declares no columns for table ${JSON.stringify(table)}`)
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
const projection: Array<[string, number]> = []
|
|
188
|
+
|
|
189
|
+
for (const column of wanted) {
|
|
190
|
+
const position = positions.get(column)
|
|
191
|
+
|
|
192
|
+
if (position === undefined) {
|
|
193
|
+
throw new Error(
|
|
194
|
+
`soil tabular: table ${table} declares no column ${JSON.stringify(column)} — the shipped dictionary names ${positions.size} columns, and projecting away a column a caller asked for would read downstream as an absence`
|
|
195
|
+
)
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
projection.push([column, position])
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
const rows: Array<Record<string, string>> = []
|
|
202
|
+
const raw = await readPipeDelimited(join(tabularDirectory, `${file}.txt`))
|
|
203
|
+
|
|
204
|
+
for (const [index, row] of raw.entries()) {
|
|
205
|
+
const record: Record<string, string> = {}
|
|
206
|
+
|
|
207
|
+
for (const [column, position] of projection) {
|
|
208
|
+
if (position >= row.length) {
|
|
209
|
+
throw new Error(
|
|
210
|
+
`soil tabular: ${file}.txt record ${index + 1} holds ${row.length} columns, but ${table}.${column} sits at position ${position + 1}`
|
|
211
|
+
)
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
record[column] = row[position]!
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
rows.push(record)
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
return { rows, recordCount: raw.length }
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* One declared domain member, with the authority's own definition.
|
|
225
|
+
*/
|
|
226
|
+
export interface DomainMember {
|
|
227
|
+
domain: string
|
|
228
|
+
code: string
|
|
229
|
+
definition: string
|
|
230
|
+
sequence: number
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* Column positions in `msdomdet.txt`. Declared in `mstabcol.txt` under table `msdomdet`, so unlike the two bootstrap
|
|
235
|
+
* files above these could be looked up — they are named here because the domain read runs before any dictionary-driven
|
|
236
|
+
* read and the file is five columns wide by its own declaration.
|
|
237
|
+
*/
|
|
238
|
+
const MSDOMDET_WIDTH = 5
|
|
239
|
+
|
|
240
|
+
/**
|
|
241
|
+
* Read the authority's declared domains out of the archive.
|
|
242
|
+
*
|
|
243
|
+
* @throws {Error} When the file is not the declared width.
|
|
244
|
+
*/
|
|
245
|
+
export async function readDeclaredDomains(tabularDirectory: PathBuilderLike): Promise<DomainMember[]> {
|
|
246
|
+
const rows = await readPipeDelimited(join(tabularDirectory, "msdomdet.txt"))
|
|
247
|
+
|
|
248
|
+
assertWidth(rows, MSDOMDET_WIDTH, "msdomdet.txt")
|
|
249
|
+
|
|
250
|
+
return rows.map((row) => ({
|
|
251
|
+
domain: row[0]!,
|
|
252
|
+
sequence: Number(row[1]),
|
|
253
|
+
code: row[2]!,
|
|
254
|
+
definition: row[3]!,
|
|
255
|
+
}))
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/**
|
|
259
|
+
* The declared members of one domain, keyed by code.
|
|
260
|
+
*/
|
|
261
|
+
export function domainCodes(members: ReadonlyArray<DomainMember>, domain: string): Set<string> {
|
|
262
|
+
const codes = new Set<string>()
|
|
263
|
+
|
|
264
|
+
for (const member of members) {
|
|
265
|
+
if (member.domain === domain) {
|
|
266
|
+
codes.add(member.code)
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
return codes
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* `M/D/YYYY H:MM:SS` (and the `MM/DD/YYYY HH:MM:SS` the tabular export writes) to an ISO date.
|
|
275
|
+
*
|
|
276
|
+
* The two channels spell the same instant differently — Soil Data Access answers `9/9/2025 1:57:25 PM` and the shipped
|
|
277
|
+
* `sacatlog.txt` writes `09/09/2025 13:57:25` — and the download URL needs `2025-09-09`. Parsing to a date rather than
|
|
278
|
+
* slicing the string is what makes both channels agree.
|
|
279
|
+
*
|
|
280
|
+
* @throws {Error} When the value is not one of those shapes. A freshness date guessed wrong asks the download host for
|
|
281
|
+
* a file that does not exist, and the host answers 400 rather than 404, which reads as a bad request rather than a
|
|
282
|
+
* bad date.
|
|
283
|
+
*/
|
|
284
|
+
export function saverestToISODate(value: string): string {
|
|
285
|
+
const matched = /^(\d{1,2})\/(\d{1,2})\/(\d{4})/u.exec(value.trim())
|
|
286
|
+
|
|
287
|
+
if (!matched) {
|
|
288
|
+
throw new Error(
|
|
289
|
+
`soil tabular: cannot read ${JSON.stringify(value)} as a saverest date — expected M/D/YYYY, which is what both Soil Data Access and the shipped sacatlog.txt write`
|
|
290
|
+
)
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
const [, month, day, year] = matched
|
|
294
|
+
|
|
295
|
+
return `${year}-${month!.padStart(2, "0")}-${day!.padStart(2, "0")}`
|
|
296
|
+
}
|
|
@@ -0,0 +1,369 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* The two-path agreement check, and its negative half.
|
|
7
|
+
*
|
|
8
|
+
* POSITIVE HALF. A sample of points is answered from the sealed artifact and then re-asked of Soil Data
|
|
9
|
+
* Access — the same authority, a different distribution channel, and geometry this package has never
|
|
10
|
+
* touched. What is compared is the MAP UNIT the two channels put at the point, which is the thing a
|
|
11
|
+
* conversion can get wrong; comparing the derived capability class instead would let a wrong delineation
|
|
12
|
+
* agree by accident whenever two neighbouring map units happen to share a class.
|
|
13
|
+
*
|
|
14
|
+
* NEGATIVE HALF, AND IT MATTERS AS MUCH. A sample of points in states with no rows must come back
|
|
15
|
+
* `unknown` — no coverage row at all — and never a low-capability reading. The positive half alone would
|
|
16
|
+
* pass on an artifact that answered class 8 for the whole planet.
|
|
17
|
+
*
|
|
18
|
+
* A DISAGREEMENT NEAR A DELINEATION EDGE IS NOT A DEFECT, AND THE DISTANCE IS MEASURED TO THE EDGE RATHER
|
|
19
|
+
* THAN TO THE NEAREST VERTEX. A point a centimetre from a long edge can be metres from every vertex of it
|
|
20
|
+
* — the flood layer's one near-miss read 1.58 m to vertices and 0.009 m to edges, a 9 mm difference
|
|
21
|
+
* overstated 175-fold. Measuring vertices makes the boundary tolerance far stricter than it reads, which
|
|
22
|
+
* is how a rendering difference gets reported as a conversion defect.
|
|
23
|
+
*
|
|
24
|
+
* THE ARTIFACT'S OWN ANSWER IS THE CELL SUMMARY, AND THE POINT'S MAP UNIT IS UNDER IT. So the comparison
|
|
25
|
+
* reaches the GEOMETRY — the truth table — rather than the reduction: the reduction is a per-cell
|
|
26
|
+
* distribution and has no single map unit to compare. That makes this a check on the CONVERSION, which is
|
|
27
|
+
* what it is for; the reduction is checked by the fixtures and by the share-sum invariant.
|
|
28
|
+
*
|
|
29
|
+
* IT REACHES IT THROUGH THE CELL INDEX, NOT THROUGH A BOUNDING-BOX SCAN. A `WHERE min_lat <= ? AND …` over
|
|
30
|
+
* the geometry table reads like a prefilter and is a full table scan: none of those columns is indexed and
|
|
31
|
+
* every row carries a ring blob, so at the pilot's 2.7 million delineations it reads gigabytes per point.
|
|
32
|
+
* Naming the point's cell is a primary-key range scan over a `WITHOUT ROWID` table, which is the whole
|
|
33
|
+
* reason the index exists.
|
|
34
|
+
*/
|
|
35
|
+
|
|
36
|
+
import {
|
|
37
|
+
decodeRings,
|
|
38
|
+
interiorPointOfEncodedRings,
|
|
39
|
+
pointInEncodedRings,
|
|
40
|
+
segmentDistanceMetres,
|
|
41
|
+
shortCellToInt,
|
|
42
|
+
type H3Cell,
|
|
43
|
+
} from "@mailwoman/spatial"
|
|
44
|
+
import { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
45
|
+
import { cellToParent, latLngToCell } from "h3-js"
|
|
46
|
+
|
|
47
|
+
import { SoilCapabilityLookup, SoilReadingKind } from "#index"
|
|
48
|
+
import type { SoilDatabase } from "#schema"
|
|
49
|
+
import type { SoilDataAccessClient } from "#sdk/client"
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* One point, both verdicts, and whether they agree.
|
|
53
|
+
*/
|
|
54
|
+
export interface SoilAgreementRow {
|
|
55
|
+
label: string
|
|
56
|
+
latitude: number
|
|
57
|
+
longitude: number
|
|
58
|
+
/**
|
|
59
|
+
* The map unit the artifact's own geometry puts here.
|
|
60
|
+
*/
|
|
61
|
+
localMukey: string | null
|
|
62
|
+
/**
|
|
63
|
+
* The map unit Soil Data Access puts here.
|
|
64
|
+
*/
|
|
65
|
+
serviceMukey: string | null
|
|
66
|
+
outcome: "agree" | "disagree" | "boundary_tolerance"
|
|
67
|
+
/**
|
|
68
|
+
* Metres from the point to the nearest EDGE of the delineation the artifact matched.
|
|
69
|
+
*
|
|
70
|
+
* Carried on every row rather than only the tolerated ones, because it is what separates a real defect from two
|
|
71
|
+
* channels rendering the same edge differently — and a receipt that omits it forces a re-run.
|
|
72
|
+
*/
|
|
73
|
+
nearestEdgeMetres?: number
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* The negative half: a point the authority's published surveys do not reach.
|
|
78
|
+
*/
|
|
79
|
+
export interface SoilOutsideRow {
|
|
80
|
+
label: string
|
|
81
|
+
latitude: number
|
|
82
|
+
longitude: number
|
|
83
|
+
kind: SoilReadingKind
|
|
84
|
+
/**
|
|
85
|
+
* True when the artifact answered `unknown` — the only acceptable reading outside the built survey areas.
|
|
86
|
+
*/
|
|
87
|
+
passed: boolean
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export interface VerifySoilResult {
|
|
91
|
+
agreement: SoilAgreementRow[]
|
|
92
|
+
agreed: number
|
|
93
|
+
disagreed: number
|
|
94
|
+
boundaryTolerance: number
|
|
95
|
+
outside: SoilOutsideRow[]
|
|
96
|
+
outsidePassed: number
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Points outside the pilot region, named. Each is a place, not a bare pair of numbers: a coordinate a reader cannot
|
|
101
|
+
* name is a coordinate nobody can check.
|
|
102
|
+
*
|
|
103
|
+
* Every neighbouring state is included, because the failure this half catches is a footprint that leaked past the
|
|
104
|
+
* survey-area outlines — and a footprint accidentally clipped to "the Midwest" would pass a one-state check. Two of
|
|
105
|
+
* these sit close to the Iowa border on purpose: the outline test is conservative, so a near-border point must read
|
|
106
|
+
* unknown rather than borrow Iowa's coverage.
|
|
107
|
+
*/
|
|
108
|
+
export const OUTSIDE_PILOT_POINTS: ReadonlyArray<{ label: string; latitude: number; longitude: number }> = [
|
|
109
|
+
{ label: "Lincoln, Nebraska", latitude: 40.8136, longitude: -96.7026 },
|
|
110
|
+
{ label: "Omaha, Nebraska (Iowa border)", latitude: 41.2565, longitude: -95.9345 },
|
|
111
|
+
{ label: "Minneapolis, Minnesota", latitude: 44.9778, longitude: -93.265 },
|
|
112
|
+
{ label: "Albert Lea, Minnesota (Iowa border)", latitude: 43.6478, longitude: -93.3683 },
|
|
113
|
+
{ label: "Madison, Wisconsin", latitude: 43.0731, longitude: -89.4012 },
|
|
114
|
+
{ label: "Rockford, Illinois", latitude: 42.2711, longitude: -89.094 },
|
|
115
|
+
{ label: "Kansas City, Missouri", latitude: 39.0997, longitude: -94.5786 },
|
|
116
|
+
{ label: "Sioux Falls, South Dakota", latitude: 43.5446, longitude: -96.7311 },
|
|
117
|
+
]
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* How close to a delineation edge a disagreement is attributed to the channels' differing rendering rather than to the
|
|
121
|
+
* conversion.
|
|
122
|
+
*
|
|
123
|
+
* One metre. The published shapefile carries nine decimals through this package's ingest and Soil Data Access renders
|
|
124
|
+
* its own geometry independently; NRCS's own positional-accuracy statement says the difference between a boundary's
|
|
125
|
+
* field location and its digitized location "is unknown", so this tolerance is about the two RENDERINGS agreeing rather
|
|
126
|
+
* than about ground truth. One metre is far below the median delineation, which is 24,863 m² — about 158 m across.
|
|
127
|
+
*/
|
|
128
|
+
const BOUNDARY_TOLERANCE_METRES = 1
|
|
129
|
+
|
|
130
|
+
export interface VerifySoilOptions {
|
|
131
|
+
databasePath: string
|
|
132
|
+
client: Pick<SoilDataAccessClient, "mukeyAtPoint">
|
|
133
|
+
/**
|
|
134
|
+
* Points to re-ask the service about. A caller samples them from the artifact — see {@link sampleAgreementPoints}.
|
|
135
|
+
*/
|
|
136
|
+
points: ReadonlyArray<{ label: string; latitude: number; longitude: number }>
|
|
137
|
+
outsidePoints?: ReadonlyArray<{ label: string; latitude: number; longitude: number }>
|
|
138
|
+
onProgress?: (message: string) => void
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Run both halves.
|
|
143
|
+
*/
|
|
144
|
+
export async function verifySoilDatabase(options: VerifySoilOptions): Promise<VerifySoilResult> {
|
|
145
|
+
const database = new DatabaseClient<SoilDatabase>(options.databasePath, { readOnly: true })
|
|
146
|
+
const lookup = new SoilCapabilityLookup({ databasePath: options.databasePath })
|
|
147
|
+
|
|
148
|
+
// Read once: the stored index is mixed-resolution because the whole tier is compacted, and a probe that assumed one
|
|
149
|
+
// resolution would read every row at the others as an absence.
|
|
150
|
+
const resolutions = (
|
|
151
|
+
database.prepare("SELECT DISTINCT resolution FROM soil_map_unit_cell ORDER BY resolution").all() as Array<{
|
|
152
|
+
resolution: number
|
|
153
|
+
}>
|
|
154
|
+
).map((row) => row.resolution)
|
|
155
|
+
|
|
156
|
+
const { indexResolution } = lookup.identity
|
|
157
|
+
|
|
158
|
+
try {
|
|
159
|
+
const agreement: SoilAgreementRow[] = []
|
|
160
|
+
|
|
161
|
+
for (const point of options.points) {
|
|
162
|
+
const local = localDelineationAt(database, resolutions, indexResolution, point.latitude, point.longitude)
|
|
163
|
+
const serviceMukey = (await options.client.mukeyAtPoint(point.latitude, point.longitude)) ?? null
|
|
164
|
+
|
|
165
|
+
const nearEdge = local.nearestEdgeMetres !== undefined && local.nearestEdgeMetres <= BOUNDARY_TOLERANCE_METRES
|
|
166
|
+
|
|
167
|
+
agreement.push({
|
|
168
|
+
...point,
|
|
169
|
+
localMukey: local.mukey,
|
|
170
|
+
serviceMukey,
|
|
171
|
+
outcome: local.mukey === serviceMukey ? "agree" : nearEdge ? "boundary_tolerance" : "disagree",
|
|
172
|
+
...(local.nearestEdgeMetres === undefined ? {} : { nearestEdgeMetres: local.nearestEdgeMetres }),
|
|
173
|
+
})
|
|
174
|
+
|
|
175
|
+
options.onProgress?.(`${agreement.length}/${options.points.length} points compared`)
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
const outside: SoilOutsideRow[] = []
|
|
179
|
+
|
|
180
|
+
for (const point of options.outsidePoints ?? OUTSIDE_PILOT_POINTS) {
|
|
181
|
+
const reading = lookup.lookup(point.latitude, point.longitude)
|
|
182
|
+
|
|
183
|
+
outside.push({ ...point, kind: reading.kind, passed: reading.kind === SoilReadingKind.Unknown })
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
return {
|
|
187
|
+
agreement,
|
|
188
|
+
agreed: agreement.filter((row) => row.outcome === "agree").length,
|
|
189
|
+
disagreed: agreement.filter((row) => row.outcome === "disagree").length,
|
|
190
|
+
boundaryTolerance: agreement.filter((row) => row.outcome === "boundary_tolerance").length,
|
|
191
|
+
outside,
|
|
192
|
+
outsidePassed: outside.filter((row) => row.passed).length,
|
|
193
|
+
}
|
|
194
|
+
} finally {
|
|
195
|
+
lookup[Symbol.dispose]()
|
|
196
|
+
await database.destroy()
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* The candidate delineations reaching a point, found THROUGH THE CELL INDEX rather than by scanning the geometry table.
|
|
202
|
+
*
|
|
203
|
+
* A bounding-box `WHERE` over `soil_map_unit_area` reads as a prefilter and is a FULL TABLE SCAN: none of those columns
|
|
204
|
+
* is indexed, the rows carry the ring blobs, and at the pilot's scale that is 2.7 million rows and several gigabytes
|
|
205
|
+
* read per point. The cell index exists to make exactly this question cheap — `soil_map_unit_cell` is `WITHOUT ROWID`
|
|
206
|
+
* keyed `(h3_cell, area_id)`, so naming the point's cell is a primary-key range scan.
|
|
207
|
+
*
|
|
208
|
+
* EVERY STORED RESOLUTION IS PROBED, not just the index one. The whole tier is compacted parent-ward, so a delineation
|
|
209
|
+
* that fills a run of cells is stored at a coarser resolution and a probe at the index resolution alone would read it
|
|
210
|
+
* as an absence — the same ancestor walk the reader does, and the same false negative it avoids.
|
|
211
|
+
*/
|
|
212
|
+
function candidateDelineations(
|
|
213
|
+
database: DatabaseClient<SoilDatabase>,
|
|
214
|
+
resolutions: readonly number[],
|
|
215
|
+
indexResolution: number,
|
|
216
|
+
latitude: number,
|
|
217
|
+
longitude: number
|
|
218
|
+
): Array<{ mukey: string; rings: Uint8Array }> {
|
|
219
|
+
const selectCandidates = database.prepare(
|
|
220
|
+
"SELECT c.area_id AS area_id, a.mukey AS mukey, a.rings AS rings FROM soil_map_unit_cell c " +
|
|
221
|
+
"JOIN soil_map_unit_area a ON a.area_id = c.area_id WHERE c.h3_cell = ?"
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
const indexCell = latLngToCell(latitude, longitude, indexResolution) as H3Cell
|
|
225
|
+
const seen = new Set<string>()
|
|
226
|
+
const candidates: Array<{ mukey: string; rings: Uint8Array }> = []
|
|
227
|
+
|
|
228
|
+
for (const resolution of resolutions) {
|
|
229
|
+
const cell = resolution === indexResolution ? indexCell : (cellToParent(indexCell, resolution) as H3Cell)
|
|
230
|
+
|
|
231
|
+
for (const row of selectCandidates.all(shortCellToInt(cell)) as Array<{
|
|
232
|
+
area_id: string
|
|
233
|
+
mukey: string
|
|
234
|
+
rings: Uint8Array
|
|
235
|
+
}>) {
|
|
236
|
+
// DEDUPE ON THE DELINEATION, NEVER ON ITS MAP UNIT. A delineation reached through two resolutions is one
|
|
237
|
+
// delineation and must be tested once; two DIFFERENT delineations of the same map unit are two shapes covering
|
|
238
|
+
// different ground and must both be tested. Keying on the map unit drops the second, and it drops it silently —
|
|
239
|
+
// the point test simply finds nothing and the row reads as a disagreement with the authority. Measured at Iowa
|
|
240
|
+
// scale: one point in 60, where the artifact's own geometry does contain the point and the index-driven read
|
|
241
|
+
// could not reach the delineation that holds it.
|
|
242
|
+
if (seen.has(row.area_id)) continue
|
|
243
|
+
|
|
244
|
+
seen.add(row.area_id)
|
|
245
|
+
candidates.push({ mukey: row.mukey, rings: row.rings })
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
return candidates
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/**
|
|
253
|
+
* Which map unit the ARTIFACT's own geometry puts at a point, and how far the point is from that delineation's nearest
|
|
254
|
+
* edge.
|
|
255
|
+
*
|
|
256
|
+
* The cell index narrows; the ray cast decides. The edge distance is measured against every candidate, so a near-miss
|
|
257
|
+
* is reported with a distance rather than with nothing.
|
|
258
|
+
*/
|
|
259
|
+
function localDelineationAt(
|
|
260
|
+
database: DatabaseClient<SoilDatabase>,
|
|
261
|
+
resolutions: readonly number[],
|
|
262
|
+
indexResolution: number,
|
|
263
|
+
latitude: number,
|
|
264
|
+
longitude: number
|
|
265
|
+
): { mukey: string | null; nearestEdgeMetres?: number } {
|
|
266
|
+
const candidates = candidateDelineations(database, resolutions, indexResolution, latitude, longitude)
|
|
267
|
+
|
|
268
|
+
let mukey: string | null = null
|
|
269
|
+
let nearest = Infinity
|
|
270
|
+
|
|
271
|
+
for (const candidate of candidates) {
|
|
272
|
+
const distance = nearestEdgeDistance(candidate.rings, longitude, latitude)
|
|
273
|
+
|
|
274
|
+
if (distance < nearest) {
|
|
275
|
+
nearest = distance
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
if (mukey === null && pointInEncodedRings(candidate.rings, longitude, latitude)) {
|
|
279
|
+
mukey = candidate.mukey
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
return Number.isFinite(nearest) ? { mukey, nearestEdgeMetres: nearest } : { mukey }
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/**
|
|
287
|
+
* Metres from a point to the nearest edge of an encoded ring set.
|
|
288
|
+
*
|
|
289
|
+
* Decoding here rather than walking the blob directly: this runs a few hundred times in a verification, not per
|
|
290
|
+
* geocode, and the decoded form is what makes the segment walk readable.
|
|
291
|
+
*/
|
|
292
|
+
function nearestEdgeDistance(blob: Uint8Array, lon: number, lat: number): number {
|
|
293
|
+
const { polygons } = decodeRings(blob)
|
|
294
|
+
|
|
295
|
+
let nearest = Infinity
|
|
296
|
+
|
|
297
|
+
for (const rings of polygons) {
|
|
298
|
+
for (const ring of rings) {
|
|
299
|
+
for (let index = 2; index < ring.length; index += 2) {
|
|
300
|
+
const distance = segmentDistanceMetres(
|
|
301
|
+
lon,
|
|
302
|
+
lat,
|
|
303
|
+
[ring[index - 2]!, ring[index - 1]!],
|
|
304
|
+
[ring[index]!, ring[index + 1]!]
|
|
305
|
+
)
|
|
306
|
+
|
|
307
|
+
if (distance < nearest) {
|
|
308
|
+
nearest = distance
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
return nearest
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/**
|
|
318
|
+
* Draw a reproducible sample of points from the artifact.
|
|
319
|
+
*
|
|
320
|
+
* The draw is a deterministic stride over the primary key, not a random one, so a re-run compares the same points and a
|
|
321
|
+
* disagreement can be looked at rather than re-rolled.
|
|
322
|
+
*
|
|
323
|
+
* ONE ROW IS READ PER SAMPLE POINT AND NO MORE. A `WHERE rowid % stride = 0` scan looks like the same thing and is not:
|
|
324
|
+
* it walks the table itself, which means reading every ring blob to keep a few dozen. `ORDER BY area_id LIMIT 1 OFFSET
|
|
325
|
+
* n` walks the primary-key index to the offset and fetches exactly the row it lands on.
|
|
326
|
+
*/
|
|
327
|
+
export function sampleAgreementPoints(
|
|
328
|
+
databasePath: string,
|
|
329
|
+
options: { count?: number } = {}
|
|
330
|
+
): Array<{ label: string; latitude: number; longitude: number }> {
|
|
331
|
+
const count = options.count ?? 60
|
|
332
|
+
using database = new DatabaseClient<SoilDatabase>(databasePath, { readOnly: true })
|
|
333
|
+
|
|
334
|
+
const total = (database.prepare("SELECT count(*) AS n FROM soil_map_unit_area").get() as { n: number }).n
|
|
335
|
+
const stride = Math.max(1, Math.floor(total / Math.max(1, count)))
|
|
336
|
+
|
|
337
|
+
// One OFFSET probe per sample point rather than one materialized key list. The list looks cheap because it reads
|
|
338
|
+
// only the primary key, and at the pilot's 2.7 million delineations it is still 2.7 million strings held to keep
|
|
339
|
+
// sixty of them.
|
|
340
|
+
const selectByOffset = database.prepare(
|
|
341
|
+
"SELECT area_id, mukey, min_lat, min_lon, max_lat, max_lon, rings FROM soil_map_unit_area ORDER BY area_id LIMIT 1 OFFSET ?"
|
|
342
|
+
)
|
|
343
|
+
|
|
344
|
+
const points: Array<{ label: string; latitude: number; longitude: number }> = []
|
|
345
|
+
|
|
346
|
+
for (let index = 0; index < total && points.length < count; index += stride) {
|
|
347
|
+
const area = selectByOffset.get(index) as
|
|
348
|
+
| {
|
|
349
|
+
area_id: string
|
|
350
|
+
mukey: string
|
|
351
|
+
min_lat: number
|
|
352
|
+
min_lon: number
|
|
353
|
+
max_lat: number
|
|
354
|
+
max_lon: number
|
|
355
|
+
rings: Uint8Array
|
|
356
|
+
}
|
|
357
|
+
| undefined
|
|
358
|
+
|
|
359
|
+
if (!area) continue
|
|
360
|
+
|
|
361
|
+
const interior = interiorPointOfEncodedRings(area, 7)
|
|
362
|
+
|
|
363
|
+
if (!interior) continue
|
|
364
|
+
|
|
365
|
+
points.push({ label: `map unit ${area.mukey} delineation ${area.area_id}`, ...interior })
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
return points
|
|
369
|
+
}
|