@mailwoman/soil 9.4.0 → 10.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +118 -116
- package/lib/index.ts +66 -98
- package/lib/paths.ts +24 -0
- package/lib/schema.ts +188 -120
- package/lib/vocabulary.ts +40 -107
- package/out/index.d.ts +52 -70
- package/out/index.d.ts.map +1 -1
- package/out/index.js +24 -72
- package/out/index.js.map +1 -1
- package/out/paths.d.ts +19 -0
- package/out/paths.d.ts.map +1 -0
- package/out/paths.js +21 -0
- package/out/paths.js.map +1 -0
- package/out/schema.d.ts +187 -119
- package/out/schema.d.ts.map +1 -1
- package/out/schema.js +31 -31
- package/out/schema.js.map +1 -1
- package/out/sdk/acquire.d.ts +15 -24
- package/out/sdk/acquire.d.ts.map +1 -1
- package/out/sdk/acquire.js +5 -20
- package/out/sdk/acquire.js.map +1 -1
- package/out/sdk/build-soil.d.ts +67 -70
- package/out/sdk/build-soil.d.ts.map +1 -1
- package/out/sdk/build-soil.js +65 -95
- package/out/sdk/build-soil.js.map +1 -1
- package/out/sdk/cell-tiers.d.ts +7 -19
- package/out/sdk/cell-tiers.d.ts.map +1 -1
- package/out/sdk/cell-tiers.js +19 -38
- package/out/sdk/cell-tiers.js.map +1 -1
- package/out/sdk/cells.d.ts +15 -41
- package/out/sdk/cells.d.ts.map +1 -1
- package/out/sdk/cells.js +11 -38
- package/out/sdk/cells.js.map +1 -1
- package/out/sdk/client.d.ts +23 -48
- package/out/sdk/client.d.ts.map +1 -1
- package/out/sdk/client.js +19 -63
- package/out/sdk/client.js.map +1 -1
- package/out/sdk/download.d.ts +24 -47
- package/out/sdk/download.d.ts.map +1 -1
- package/out/sdk/download.js +14 -54
- package/out/sdk/download.js.map +1 -1
- package/out/sdk/ingest/chunk.d.ts +77 -0
- package/out/sdk/ingest/chunk.d.ts.map +1 -0
- package/out/sdk/{ingest-chunk.js → ingest/chunk.js} +19 -17
- package/out/sdk/ingest/chunk.js.map +1 -0
- package/out/sdk/ingest/worker.d.ts +9 -0
- package/out/sdk/ingest/worker.d.ts.map +1 -0
- package/out/{scripts/ingest-chunk.js → sdk/ingest/worker.js} +11 -10
- package/out/sdk/ingest/worker.js.map +1 -0
- package/out/sdk/ingest.d.ts +40 -52
- package/out/sdk/ingest.d.ts.map +1 -1
- package/out/sdk/ingest.js +18 -61
- package/out/sdk/ingest.js.map +1 -1
- package/out/sdk/measure-resolutions.d.ts +5 -16
- package/out/sdk/measure-resolutions.d.ts.map +1 -1
- package/out/sdk/measure-resolutions.js +3 -15
- package/out/sdk/measure-resolutions.js.map +1 -1
- package/out/sdk/reduce.d.ts +33 -59
- package/out/sdk/reduce.d.ts.map +1 -1
- package/out/sdk/reduce.js +47 -78
- package/out/sdk/reduce.js.map +1 -1
- package/out/sdk/survey-area.d.ts +16 -48
- package/out/sdk/survey-area.d.ts.map +1 -1
- package/out/sdk/survey-area.js +33 -77
- package/out/sdk/survey-area.js.map +1 -1
- package/out/sdk/tabular.d.ts +32 -31
- package/out/sdk/tabular.d.ts.map +1 -1
- package/out/sdk/tabular.js +58 -56
- package/out/sdk/tabular.js.map +1 -1
- package/out/sdk/test-kit.d.ts +57 -0
- package/out/sdk/test-kit.d.ts.map +1 -0
- package/out/{test-kit.js → sdk/test-kit.js} +18 -39
- package/out/sdk/test-kit.js.map +1 -0
- package/out/sdk/verify.d.ts +15 -51
- package/out/sdk/verify.d.ts.map +1 -1
- package/out/sdk/verify.js +19 -77
- package/out/sdk/verify.js.map +1 -1
- package/out/vocabulary.d.ts +37 -103
- package/out/vocabulary.d.ts.map +1 -1
- package/out/vocabulary.js +34 -107
- package/out/vocabulary.js.map +1 -1
- package/package.json +36 -190
- package/{lib/sdk → sdk}/acquire.ts +17 -27
- package/{lib/sdk → sdk}/build-soil.ts +114 -128
- package/{lib/sdk → sdk}/cell-tiers.ts +20 -39
- package/{lib/sdk → sdk}/cells.ts +17 -43
- package/sdk/client.ts +147 -0
- package/sdk/download.ts +131 -0
- package/{lib/sdk/ingest-chunk.ts → sdk/ingest/chunk.ts} +34 -26
- package/{lib/scripts/ingest-chunk.ts → sdk/ingest/worker.ts} +10 -9
- package/sdk/ingest.ts +253 -0
- package/{lib/sdk → sdk}/measure-resolutions.ts +5 -16
- package/sdk/reduce.ts +344 -0
- package/{lib/sdk → sdk}/survey-area.ts +39 -83
- package/{lib/sdk → sdk}/tabular.ts +62 -59
- package/{lib → sdk}/test-kit.ts +18 -40
- package/{lib/sdk → sdk}/verify.ts +29 -85
- package/lib/sdk/client.ts +0 -184
- package/lib/sdk/download.ts +0 -161
- package/lib/sdk/index.ts +0 -20
- package/lib/sdk/ingest.ts +0 -278
- package/lib/sdk/reduce.ts +0 -375
- package/out/scripts/ingest-chunk.d.ts +0 -11
- package/out/scripts/ingest-chunk.d.ts.map +0 -1
- package/out/scripts/ingest-chunk.js.map +0 -1
- package/out/sdk/index.d.ts +0 -20
- package/out/sdk/index.d.ts.map +0 -1
- package/out/sdk/index.js +0 -20
- package/out/sdk/index.js.map +0 -1
- package/out/sdk/ingest-chunk.d.ts +0 -73
- package/out/sdk/ingest-chunk.d.ts.map +0 -1
- package/out/sdk/ingest-chunk.js.map +0 -1
- package/out/test-kit.d.ts +0 -79
- package/out/test-kit.d.ts.map +0 -1
- package/out/test-kit.js.map +0 -1
|
@@ -2,34 +2,11 @@
|
|
|
2
2
|
* @copyright Sister Software
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
|
-
*
|
|
6
|
-
* One survey area's attributes, its own metadata, and its mapped footprint.
|
|
7
|
-
*
|
|
8
|
-
* THE FOOTPRINT IS THE SURVEY-AREA OUTLINE, NEVER THE UNION OF THE RATED POLYGONS. `NOTCOM`,
|
|
9
|
-
* access-denied and `NOTPUB` map units are INSIDE the footprint and carry no rating, so a footprint taken
|
|
10
|
-
* from the rated set would report them as unmapped when the authority has declared exactly what they are.
|
|
11
|
-
* The archive ships the outline as its own shapefile — `soilsa_a_<areasymbol>.shp`, one feature — which is
|
|
12
|
-
* why this layer never has to reconstruct it.
|
|
13
|
-
*
|
|
14
|
-
* THE REFRESH DATE IS NOT THE SURVEY DATE, AND CONFLATING THEM IS THE CURRENCY LIE THIS FILE EXISTS TO
|
|
15
|
-
* PREVENT. `IA153` carries `saverest` 2025-09-09 and version 28, and the FGDC lineage inside the same
|
|
16
|
-
* archive cites `Soil Survey of Polk County, Iowa`, 1:15,840, **1960**. The dataset's own
|
|
17
|
-
* time-period-of-content runs 1998-09-22 to 2025-09-09, so a consumer reading that as survey currency
|
|
18
|
-
* reads it wrong by sixty-five years. Both dates are stored, apart, with the title the older one came
|
|
19
|
-
* from so it is checkable rather than assertible.
|
|
20
|
-
*
|
|
21
|
-
* TWO SCALES, ALSO DIFFERENT FACTS. `legend.projectscale` is 12,000 for `IA153` — the scale the map units
|
|
22
|
-
* were digitized at. The 1960 source citation's own `srcscale` is 15,840 — the scale the ground was
|
|
23
|
-
* walked at. Storing one as the other would answer the enlargement caveat's question wrongly.
|
|
24
|
-
*
|
|
25
|
-
* THE LICENCE IS CHECKED PER SURVEY AREA, against the `useconst` element of the metadata that area ships.
|
|
26
|
-
* An area whose use constraints no longer say "This is public information" is a licence change, and a
|
|
27
|
-
* build that absorbed one would ship an artifact under terms nobody checked. The text is boilerplate
|
|
28
|
-
* repeated across SSURGO, which is why asserting it is cheap and why a change in it is loud.
|
|
29
5
|
*/
|
|
30
6
|
|
|
31
|
-
import { parseJSONStrict } from "@mailwoman/core/json"
|
|
7
|
+
import { parseJSONStrict, stringifyJSON } from "@mailwoman/core/json"
|
|
32
8
|
import { runFile } from "@mailwoman/core/process"
|
|
9
|
+
import { normalizeWhitespace } from "@mailwoman/core/strings/format"
|
|
33
10
|
import type { ParsedGeometry } from "@mailwoman/spatial"
|
|
34
11
|
import type { PathBuilderLike } from "path-ts"
|
|
35
12
|
|
|
@@ -45,10 +22,8 @@ import {
|
|
|
45
22
|
} from "#vocabulary"
|
|
46
23
|
|
|
47
24
|
/**
|
|
48
|
-
* The declared domains this layer validates against
|
|
49
|
-
*
|
|
50
|
-
* `capability_class` is shared by `nirrcapcl`, `irrcapcl` and `muaggatt.niccdcd`, which is why one domain covers three
|
|
51
|
-
* columns.
|
|
25
|
+
* The declared domains this layer validates against and stores; `capability_class`
|
|
26
|
+
* covers `nirrcapcl`, `irrcapcl` and `muaggatt.niccdcd`.
|
|
52
27
|
*/
|
|
53
28
|
export const STORED_DOMAINS = [
|
|
54
29
|
"capability_class",
|
|
@@ -72,8 +47,8 @@ export interface SurveyAreaAttributes {
|
|
|
72
47
|
sourceScale: number | null
|
|
73
48
|
mappingScale: number | null
|
|
74
49
|
/**
|
|
75
|
-
* The area the authority publishes for this survey area, in acres —
|
|
76
|
-
* compares against.
|
|
50
|
+
* The area the authority publishes for this survey area, in acres —
|
|
51
|
+
* the independent witness the ring-area check compares against.
|
|
77
52
|
*/
|
|
78
53
|
areaAcres: number | null
|
|
79
54
|
mapUnits: SoilMapUnitTable[]
|
|
@@ -84,8 +59,9 @@ export interface SurveyAreaAttributes {
|
|
|
84
59
|
/**
|
|
85
60
|
* Read one survey area's tabular export.
|
|
86
61
|
*
|
|
87
|
-
* @throws {Error} When the metadata's use constraints no longer
|
|
88
|
-
*
|
|
62
|
+
* @throws {Error} When the metadata's use constraints no longer include the
|
|
63
|
+
* public-information sentence, when a `Choice` column holds a value outside the
|
|
64
|
+
* authority's own declared domain, or when the export declares no legend row.
|
|
89
65
|
*/
|
|
90
66
|
export async function readSurveyAreaAttributes(
|
|
91
67
|
tabularDirectory: PathBuilderLike,
|
|
@@ -170,8 +146,8 @@ export async function readSurveyAreaAttributes(
|
|
|
170
146
|
return {
|
|
171
147
|
cokey: row.cokey!,
|
|
172
148
|
mukey: row.mukey!,
|
|
173
|
-
// A blank `comppct_r` is a component with no declared weight
|
|
174
|
-
//
|
|
149
|
+
// A blank `comppct_r` is a component with no declared weight, recorded as zero
|
|
150
|
+
// rather than dropped so the component still appears.
|
|
175
151
|
comppct_r: row.comppct_r ? Number(row.comppct_r) : 0,
|
|
176
152
|
compname: nullable(row.compname),
|
|
177
153
|
compkind: nullable(row.compkind),
|
|
@@ -225,11 +201,8 @@ export async function readSurveyAreaAttributes(
|
|
|
225
201
|
/**
|
|
226
202
|
* A polygon the authority drew with no soil mapping behind it.
|
|
227
203
|
*
|
|
228
|
-
*
|
|
229
|
-
*
|
|
230
|
-
* carrying NO components at all. A map unit with no components has nothing to rate whatever it is called, and reading
|
|
231
|
-
* it as "rated nothing" rather than "no mapping" would put it in `unrated_share` — a claim that the survey looked and
|
|
232
|
-
* declined, when it did not look.
|
|
204
|
+
* A map unit with no components has no component to rate, so it reads as no mapping
|
|
205
|
+
* rather than assigned no rating.
|
|
233
206
|
*/
|
|
234
207
|
function isNoMapping(musym: string, muname: string, componentCount: number): boolean {
|
|
235
208
|
if (SSURGO_NO_MAPPING_SYMBOLS.has(musym.toUpperCase())) return true
|
|
@@ -242,10 +215,8 @@ function isNoMapping(musym: string, muname: string, componentCount: number): boo
|
|
|
242
215
|
/**
|
|
243
216
|
* Refuse a value outside the authority's own declared domain.
|
|
244
217
|
*
|
|
245
|
-
*
|
|
246
|
-
*
|
|
247
|
-
* NULL is a real state in every one of these columns and means something specific — for `nirrcapcl` it means the survey
|
|
248
|
-
* did not rate the component, which is not class 8.
|
|
218
|
+
* A blank is a real NULL state rather than a violation, recording that the
|
|
219
|
+
* survey did not rate the component.
|
|
249
220
|
*/
|
|
250
221
|
function assertDeclared(declared: ReadonlySet<string>, value: string | undefined, domain: string, where: string): void {
|
|
251
222
|
if (!value) return
|
|
@@ -253,7 +224,7 @@ function assertDeclared(declared: ReadonlySet<string>, value: string | undefined
|
|
|
253
224
|
if (declared.has(value)) return
|
|
254
225
|
|
|
255
226
|
throw new Error(
|
|
256
|
-
`soil survey area: ${where} holds ${
|
|
227
|
+
`soil survey area: ${where} holds ${stringifyJSON(value)}, which is not in the authority's declared ${domain} domain (${declared.size} members, read from the archive's own msdomdet.txt) — an unknown code is a source-schema change, and coercing it would turn "the source changed" into "there is nothing here"`
|
|
257
228
|
)
|
|
258
229
|
}
|
|
259
230
|
|
|
@@ -262,12 +233,9 @@ function nullable(value: string | undefined): string | null {
|
|
|
262
233
|
}
|
|
263
234
|
|
|
264
235
|
/**
|
|
265
|
-
* The
|
|
236
|
+
* The nccpi v3.0 overall index per component.
|
|
266
237
|
*
|
|
267
|
-
*
|
|
268
|
-
* one row per component at {@link COINTERP_OVERALL_RULE_DEPTH}: 369 of 369 components on `IA153`, of which 327 carry a
|
|
269
|
-
* value. Sub-rules at greater depths are the submodels (corn, soybeans, small grains, cotton), which this layer does
|
|
270
|
-
* not carry.
|
|
238
|
+
* Sub-rules at greater depths are submodels this layer does not include.
|
|
271
239
|
*/
|
|
272
240
|
async function readNCCPI(
|
|
273
241
|
tabularDirectory: PathBuilderLike,
|
|
@@ -296,15 +264,15 @@ async function readNCCPI(
|
|
|
296
264
|
}
|
|
297
265
|
|
|
298
266
|
/**
|
|
299
|
-
* What the shipped FGDC metadata
|
|
267
|
+
* What the shipped FGDC metadata records about this survey area's dates and its license.
|
|
300
268
|
*/
|
|
301
269
|
export interface FGDCMetadata {
|
|
302
270
|
/**
|
|
303
|
-
* The citation's own `pubdate
|
|
271
|
+
* The citation's own `pubdate` as an ISO date — the refresh rather than the survey date.
|
|
304
272
|
*/
|
|
305
273
|
publicationDate: string
|
|
306
274
|
/**
|
|
307
|
-
* The
|
|
275
|
+
* The oldest source citation date in the lineage, as an ISO date or a bare year.
|
|
308
276
|
*/
|
|
309
277
|
oldestSourceDate: string | null
|
|
310
278
|
oldestSourceTitle: string | null
|
|
@@ -312,24 +280,17 @@ export interface FGDCMetadata {
|
|
|
312
280
|
}
|
|
313
281
|
|
|
314
282
|
/**
|
|
315
|
-
* Read the metadata
|
|
316
|
-
*
|
|
317
|
-
* Targeted extraction rather than a general XML parse, and NOT for want of a parser — `@mailwoman/core` ships
|
|
318
|
-
* `htmlparser2`. A parser RECOVERS an unclosed element by giving it the rest of the document as its content, and the
|
|
319
|
-
* two values below that throw would then stamp the artifact with that content instead. {@link elementText} answers
|
|
320
|
-
* `undefined` for an element it cannot read, which is what makes the throw reachable. Every value this reader cannot
|
|
321
|
-
* find is reported as `null` EXCEPT the publication date and the licence sentence, which throw — those two decide the
|
|
322
|
-
* artifact's vintage and whether it may be shipped at all, and neither has a safe default.
|
|
283
|
+
* Read the metadata nrcs ships inside the archive.
|
|
323
284
|
*
|
|
324
|
-
* @throws {Error} When the metadata
|
|
325
|
-
*
|
|
285
|
+
* @throws {Error} When the metadata has no publication date, or its use constraints
|
|
286
|
+
* no longer include the public-information sentence.
|
|
326
287
|
*/
|
|
327
288
|
export function readFGDCMetadata(xml: string, areaSymbol: string): FGDCMetadata {
|
|
328
289
|
const useConstraints = elementText(xml, "useconst")
|
|
329
290
|
|
|
330
291
|
if (!useConstraints?.includes(SSURGO_PUBLIC_INFORMATION_SENTENCE)) {
|
|
331
292
|
throw new Error(
|
|
332
|
-
`soil survey area: ${areaSymbol}'s FGDC use constraints do not carry ${
|
|
293
|
+
`soil survey area: ${areaSymbol}'s FGDC use constraints do not carry ${stringifyJSON(SSURGO_PUBLIC_INFORMATION_SENTENCE)} — that sentence is the grant this layer ships on, so a survey area without it must not be built into a distributable artifact`
|
|
333
294
|
)
|
|
334
295
|
}
|
|
335
296
|
|
|
@@ -353,14 +314,14 @@ export function readFGDCMetadata(xml: string, areaSymbol: string): FGDCMetadata
|
|
|
353
314
|
}
|
|
354
315
|
|
|
355
316
|
/**
|
|
356
|
-
* The lineage's source citations
|
|
317
|
+
* The lineage's source citations — what the polygons rest on and when each was made.
|
|
357
318
|
*/
|
|
358
319
|
function readSourceCitations(xml: string): Array<{ date: string; title: string; scale: number | null }> {
|
|
359
320
|
const citations: Array<{ date: string; title: string; scale: number | null }> = []
|
|
360
321
|
|
|
361
322
|
for (const body of elementBlocks(xml, "srcinfo")) {
|
|
362
|
-
// `caldate`
|
|
363
|
-
//
|
|
323
|
+
// `caldate` stores a single date.
|
|
324
|
+
// `begdate` stores a range's start date in this layer.
|
|
364
325
|
const date = elementText(body, "caldate") ?? elementText(body, "begdate")
|
|
365
326
|
|
|
366
327
|
if (!date) continue
|
|
@@ -369,7 +330,7 @@ function readSourceCitations(xml: string): Array<{ date: string; title: string;
|
|
|
369
330
|
|
|
370
331
|
citations.push({
|
|
371
332
|
date: date.trim(),
|
|
372
|
-
title: (elementText(body, "title") ?? "")
|
|
333
|
+
title: normalizeWhitespace(elementText(body, "title") ?? ""),
|
|
373
334
|
scale: scale ? Number(scale) : null,
|
|
374
335
|
})
|
|
375
336
|
}
|
|
@@ -378,13 +339,10 @@ function readSourceCitations(xml: string): Array<{ date: string; title: string;
|
|
|
378
339
|
}
|
|
379
340
|
|
|
380
341
|
/**
|
|
381
|
-
* The text of the first `<name>` element, whitespace
|
|
342
|
+
* The text of the first `<name>` element, with whitespace preserved.
|
|
382
343
|
*
|
|
383
|
-
*
|
|
384
|
-
*
|
|
385
|
-
* partner: the lazy run re-scans to the end from every candidate start. The input here is a 43,251-character document
|
|
386
|
-
* that arrived over the network inside a downloaded archive, so "a malformed one cannot happen" is not a claim this
|
|
387
|
-
* reader gets to make. Two `indexOf` calls answer the same question in one pass.
|
|
344
|
+
* Two `indexOf` calls avoid the polynomial backtracking a regex takes on a document
|
|
345
|
+
* whose opening tag has no closing partner.
|
|
388
346
|
*/
|
|
389
347
|
function elementText(xml: string, name: string): string | undefined {
|
|
390
348
|
const open = `<${name}>`
|
|
@@ -395,14 +353,12 @@ function elementText(xml: string, name: string): string | undefined {
|
|
|
395
353
|
const from = start + open.length
|
|
396
354
|
const end = xml.indexOf(`</${name}>`, from)
|
|
397
355
|
|
|
398
|
-
// An element with no closing tag is unreadable
|
|
399
|
-
// mean the value could not be read rather than that it is blank.
|
|
356
|
+
// An element with no closing tag is unreadable rather than empty, the same answer an absent element gets.
|
|
400
357
|
return end === -1 ? undefined : xml.slice(from, end)
|
|
401
358
|
}
|
|
402
359
|
|
|
403
360
|
/**
|
|
404
|
-
* Every `<name>` element's inner text
|
|
405
|
-
* for the same reason.
|
|
361
|
+
* Every `<name>` element's inner text in document order, the repeating counterpart of {@link elementText}.
|
|
406
362
|
*/
|
|
407
363
|
function elementBlocks(xml: string, name: string): string[] {
|
|
408
364
|
const open = `<${name}>`
|
|
@@ -427,8 +383,8 @@ function elementBlocks(xml: string, name: string): string[] {
|
|
|
427
383
|
}
|
|
428
384
|
|
|
429
385
|
/**
|
|
430
|
-
*
|
|
431
|
-
*
|
|
386
|
+
* A bare year stays bare, since padding it to January 1 would invent a precision
|
|
387
|
+
* the citation does not claim.
|
|
432
388
|
*/
|
|
433
389
|
function normalizeFGDCDate(value: string): string {
|
|
434
390
|
const trimmed = value.trim()
|
|
@@ -440,10 +396,10 @@ function normalizeFGDCDate(value: string): string {
|
|
|
440
396
|
/**
|
|
441
397
|
* Read the survey area's own outline shapefile as a GeoJSON geometry.
|
|
442
398
|
*
|
|
443
|
-
* @throws {Error} When the shapefile holds anything other than exactly one feature.
|
|
444
|
-
*
|
|
399
|
+
* @throws {Error} When the shapefile holds anything other than exactly one feature.
|
|
400
|
+
* The function would silently choose which ground the coverage claim describes if it took the first feature.
|
|
445
401
|
*/
|
|
446
|
-
export async function readSurveyAreaOutline(shapefilePath:
|
|
402
|
+
export async function readSurveyAreaOutline(shapefilePath: PathBuilderLike): Promise<ParsedGeometry> {
|
|
447
403
|
const { stdout } = await runFile("ogr2ogr", ["-f", "GeoJSON", "/vsistdout/", "-t_srs", "EPSG:4326", shapefilePath], {
|
|
448
404
|
maxBuffer: 256 * 1024 * 1024,
|
|
449
405
|
})
|
|
@@ -3,31 +3,29 @@
|
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
6
|
+
* Reads the pipe-delimited NASIS export inside a survey-area archive. It maps files to tables and columns to
|
|
7
|
+
* positions, then reads the authority's declared domains.
|
|
8
8
|
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
* "the source changed" into "there is none of it", at exactly the measurement boundary where that lie
|
|
15
|
-
* costs the most.
|
|
9
|
+
* The files have no headers. The archive supplies their schema. `mstab.txt` maps each logical table name to a
|
|
10
|
+
* filename (`component` → `comp.txt`, `sacatalog` → `sacatlog.txt`). The filenames cannot be inferred. `mstabcol.txt`
|
|
11
|
+
* gives each column's ordinal position. The reader looks up those positions instead of hard-coding them.
|
|
12
|
+
* {@link readTable} throws when a requested column is absent from the shipped dictionary. An `undefined` result
|
|
13
|
+
* for a renamed column would turn a source change into an apparent absence. That would corrupt the measurement.
|
|
16
14
|
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
* enough to keep going.
|
|
15
|
+
* Quote handling affects the parsed result. `sacatlog.txt` contains 594 newline bytes and one record. Its
|
|
16
|
+
* `fgdcmetadata` column contains a 43,251-character XML document with embedded newlines. `mstabcol.txt`, the column
|
|
17
|
+
* dictionary, contains 913 newlines and 865 records. A line-splitting reader would create 594 malformed rows from
|
|
18
|
+
* the one-row `sacatlog.txt` file. Each malformed row would still look valid enough to process.
|
|
22
19
|
*
|
|
23
|
-
*
|
|
24
|
-
* `msdomdet.txt`
|
|
25
|
-
* capability classes 1 through 8, subclasses `c`/`e`/`s`/`w`,
|
|
26
|
-
*
|
|
20
|
+
* The archive also supplies its declared domains. Those values remove the need to transcribe them by hand.
|
|
21
|
+
* `msdomdet.txt` lists every `Choice` column's members and the authority's prose definition. It includes
|
|
22
|
+
* capability classes 1 through 8, subclasses `c`/`e`/`s`/`w`, 28 conditional farmland classifications and six
|
|
23
|
+
* component kinds. The layer stores and validates those values.
|
|
27
24
|
*/
|
|
28
25
|
|
|
29
26
|
import { readLocalBuffer } from "@mailwoman/core/fs/readers"
|
|
30
|
-
import {
|
|
27
|
+
import { stringifyJSON } from "@mailwoman/core/json"
|
|
28
|
+
import { PathBuilder, type PathBuilderLike, resolvePathBuilder } from "path-ts"
|
|
31
29
|
import { CSVSpliterator } from "spliterator"
|
|
32
30
|
|
|
33
31
|
/**
|
|
@@ -38,30 +36,24 @@ export type TabularRow = ReadonlyArray<string>
|
|
|
38
36
|
/**
|
|
39
37
|
* Read one pipe-delimited export file into rows.
|
|
40
38
|
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
39
|
+
* Quote-aware parsing preserves embedded newlines; `header: false` keeps the first record
|
|
40
|
+
* because these files have no header.
|
|
43
41
|
*/
|
|
44
42
|
async function readPipeDelimited(path: PathBuilderLike): Promise<TabularRow[]> {
|
|
45
43
|
const rows: TabularRow[] = []
|
|
46
44
|
|
|
47
45
|
for (const row of CSVSpliterator.from(await readLocalBuffer(path), {
|
|
48
|
-
mode: "array",
|
|
49
46
|
header: false,
|
|
50
47
|
columnDelimiter: "|",
|
|
51
|
-
enableQuoteHandling: true,
|
|
52
48
|
})) {
|
|
53
|
-
|
|
54
|
-
// numeric-looking columns at RUNTIME — measured: `mukey` comes back as the number 412818, not `"412818"` — so a row
|
|
55
|
-
// passed through untouched stops joining against the shapefile's own `MUKEY`. An empty column arrives as `""`
|
|
56
|
-
// rather than as null, which is why nothing here has to decide what a missing value means.
|
|
57
|
-
rows.push(row.map((value) => String(value)))
|
|
49
|
+
rows.push(row)
|
|
58
50
|
}
|
|
59
51
|
|
|
60
52
|
return rows
|
|
61
53
|
}
|
|
62
54
|
|
|
63
55
|
/**
|
|
64
|
-
*
|
|
56
|
+
* Maps each table to a file and to its column positions, as recorded in the archive's dictionary.
|
|
65
57
|
*/
|
|
66
58
|
export interface TabularDictionary {
|
|
67
59
|
/**
|
|
@@ -77,9 +69,11 @@ export interface TabularDictionary {
|
|
|
77
69
|
/**
|
|
78
70
|
* Column positions in `mstab.txt` and `mstabcol.txt` themselves.
|
|
79
71
|
*
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
72
|
+
* This module hard-codes only these two positions.
|
|
73
|
+
* It cannot look them up because they initialize the lookup.
|
|
74
|
+
*
|
|
75
|
+
* `mstabcol.txt` declares positions for both files.
|
|
76
|
+
* The assertions below check the bootstrap against the archive's own dictionary.
|
|
83
77
|
*/
|
|
84
78
|
const MSTAB_TABLE_NAME = 0
|
|
85
79
|
const MSTAB_FILE_NAME = 4
|
|
@@ -88,9 +82,10 @@ const MSTABCOL_POSITION = 1
|
|
|
88
82
|
const MSTABCOL_COLUMN_NAME = 2
|
|
89
83
|
|
|
90
84
|
/**
|
|
91
|
-
* Declared widths of the two bootstrap files, asserted before either is read as a dictionary.
|
|
92
|
-
*
|
|
93
|
-
*
|
|
85
|
+
* Declared widths of the two bootstrap files, asserted before either is read as a dictionary.
|
|
86
|
+
*
|
|
87
|
+
* A different width means the metadata format changed.
|
|
88
|
+
* Changed-format positions could misread every column.
|
|
94
89
|
*/
|
|
95
90
|
const MSTAB_WIDTH = 5
|
|
96
91
|
const MSTABCOL_WIDTH = 14
|
|
@@ -101,8 +96,9 @@ const MSTABCOL_WIDTH = 14
|
|
|
101
96
|
* @throws {Error} When either bootstrap file is missing or is not the declared width.
|
|
102
97
|
*/
|
|
103
98
|
export async function readTabularDictionary(tabularDirectory: PathBuilderLike): Promise<TabularDictionary> {
|
|
104
|
-
const
|
|
105
|
-
const
|
|
99
|
+
const directory = PathBuilder.from(tabularDirectory)
|
|
100
|
+
const mstab = await readPipeDelimited(directory("mstab.txt"))
|
|
101
|
+
const mstabcol = await readPipeDelimited(directory("mstabcol.txt"))
|
|
106
102
|
|
|
107
103
|
assertWidth(mstab, MSTAB_WIDTH, "mstab.txt")
|
|
108
104
|
assertWidth(mstabcol, MSTABCOL_WIDTH, "mstabcol.txt")
|
|
@@ -146,8 +142,9 @@ function assertWidth(rows: ReadonlyArray<TabularRow>, width: number, name: strin
|
|
|
146
142
|
/**
|
|
147
143
|
* A reader over one logical table, projecting the columns a caller names.
|
|
148
144
|
*
|
|
149
|
-
* The projection
|
|
150
|
-
*
|
|
145
|
+
* The projection selects columns by name and throws when a name is missing.
|
|
146
|
+
* This prevents a silently dropped column from appearing downstream as an empty dataset,
|
|
147
|
+
* the failure behind the repository's worst measurement bugs.
|
|
151
148
|
*/
|
|
152
149
|
export interface TabularTable {
|
|
153
150
|
/**
|
|
@@ -161,10 +158,10 @@ export interface TabularTable {
|
|
|
161
158
|
}
|
|
162
159
|
|
|
163
160
|
/**
|
|
164
|
-
* Read a logical
|
|
161
|
+
* Read a logical ssurgo table, projecting `wanted` columns.
|
|
165
162
|
*
|
|
166
|
-
* @throws {Error} When the archive declares no file for the table, when a requested column is not in
|
|
167
|
-
*
|
|
163
|
+
* @throws {Error} When the archive declares no file for the table, when a requested column is not in
|
|
164
|
+
* the shipped dictionary, or when a record is narrower than the position a requested column sits at.
|
|
168
165
|
*/
|
|
169
166
|
export async function readTable(
|
|
170
167
|
tabularDirectory: PathBuilderLike,
|
|
@@ -175,13 +172,13 @@ export async function readTable(
|
|
|
175
172
|
const file = dictionary.files.get(table)
|
|
176
173
|
|
|
177
174
|
if (!file) {
|
|
178
|
-
throw new Error(`soil tabular: the archive's mstab.txt declares no file for table ${
|
|
175
|
+
throw new Error(`soil tabular: the archive's mstab.txt declares no file for table ${stringifyJSON(table)}`)
|
|
179
176
|
}
|
|
180
177
|
|
|
181
178
|
const positions = dictionary.columns.get(table)
|
|
182
179
|
|
|
183
180
|
if (!positions) {
|
|
184
|
-
throw new Error(`soil tabular: the archive's mstabcol.txt declares no columns for table ${
|
|
181
|
+
throw new Error(`soil tabular: the archive's mstabcol.txt declares no columns for table ${stringifyJSON(table)}`)
|
|
185
182
|
}
|
|
186
183
|
|
|
187
184
|
const projection: Array<[string, number]> = []
|
|
@@ -191,7 +188,7 @@ export async function readTable(
|
|
|
191
188
|
|
|
192
189
|
if (position === undefined) {
|
|
193
190
|
throw new Error(
|
|
194
|
-
`soil tabular: table ${table} declares no column ${
|
|
191
|
+
`soil tabular: table ${table} declares no column ${stringifyJSON(column)} — the shipped dictionary names ${positions.size} columns, and projecting away a column a caller asked for would read downstream as an absence`
|
|
195
192
|
)
|
|
196
193
|
}
|
|
197
194
|
|
|
@@ -199,7 +196,7 @@ export async function readTable(
|
|
|
199
196
|
}
|
|
200
197
|
|
|
201
198
|
const rows: Array<Record<string, string>> = []
|
|
202
|
-
const raw = await readPipeDelimited(
|
|
199
|
+
const raw = await readPipeDelimited(resolvePathBuilder(tabularDirectory, `${file}.txt`))
|
|
203
200
|
|
|
204
201
|
for (const [index, row] of raw.entries()) {
|
|
205
202
|
const record: Record<string, string> = {}
|
|
@@ -231,9 +228,12 @@ export interface DomainMember {
|
|
|
231
228
|
}
|
|
232
229
|
|
|
233
230
|
/**
|
|
234
|
-
* Column positions in `msdomdet.txt`.
|
|
235
|
-
*
|
|
236
|
-
*
|
|
231
|
+
* Column positions in `msdomdet.txt`.
|
|
232
|
+
*
|
|
233
|
+
* Declared in `mstabcol.txt` under table `msdomdet`, so unlike the two bootstrap
|
|
234
|
+
* files above these could be looked up.
|
|
235
|
+
* They are listed here because the domain read runs before any dictionary-driven read
|
|
236
|
+
* and the file is five columns wide by its own declaration.
|
|
237
237
|
*/
|
|
238
238
|
const MSDOMDET_WIDTH = 5
|
|
239
239
|
|
|
@@ -243,7 +243,7 @@ const MSDOMDET_WIDTH = 5
|
|
|
243
243
|
* @throws {Error} When the file is not the declared width.
|
|
244
244
|
*/
|
|
245
245
|
export async function readDeclaredDomains(tabularDirectory: PathBuilderLike): Promise<DomainMember[]> {
|
|
246
|
-
const rows = await readPipeDelimited(
|
|
246
|
+
const rows = await readPipeDelimited(resolvePathBuilder(tabularDirectory, "msdomdet.txt"))
|
|
247
247
|
|
|
248
248
|
assertWidth(rows, MSDOMDET_WIDTH, "msdomdet.txt")
|
|
249
249
|
|
|
@@ -271,23 +271,26 @@ export function domainCodes(members: ReadonlyArray<DomainMember>, domain: string
|
|
|
271
271
|
}
|
|
272
272
|
|
|
273
273
|
/**
|
|
274
|
-
* `M/D/
|
|
274
|
+
* `M/D/yyyy H:MM:SS` (and the `MM/DD/yyyy HH:MM:SS` the tabular export writes) to an ISO date.
|
|
275
|
+
*
|
|
276
|
+
* Soil Data Access writes `9/9/2025 1:57:25 PM`.
|
|
277
|
+
* The shipped `sacatlog.txt` writes the same instant as `09/09/2025 13:57:25`.
|
|
275
278
|
*
|
|
276
|
-
* The
|
|
277
|
-
*
|
|
278
|
-
* slicing the string is what makes both channels agree.
|
|
279
|
+
* The download URL needs `2025-09-09`.
|
|
280
|
+
* Both channels agree only when the code parses the value as a date instead of slicing the string.
|
|
279
281
|
*
|
|
280
|
-
* @throws {Error} When the value is not one of those shapes.
|
|
281
|
-
*
|
|
282
|
-
*
|
|
282
|
+
* @throws {Error} When the value is not one of those shapes.
|
|
283
|
+
* An incorrect freshness date requests a nonexistent file.
|
|
284
|
+
* The host returns 400 for the request.
|
|
285
|
+
* A 404 would identify a missing file.
|
|
283
286
|
*/
|
|
284
|
-
// repo-health-ignore export-name-affix -- parses the survey's M/D/
|
|
287
|
+
// repo-health-ignore export-name-affix -- parses the survey's M/D/yyyy form; `isoDate` formats a Date and reads none.
|
|
285
288
|
export function saverestToISODate(value: string): string {
|
|
286
289
|
const matched = /^(\d{1,2})\/(\d{1,2})\/(\d{4})/u.exec(value.trim())
|
|
287
290
|
|
|
288
291
|
if (!matched) {
|
|
289
292
|
throw new Error(
|
|
290
|
-
`soil tabular: cannot read ${
|
|
293
|
+
`soil tabular: cannot read ${stringifyJSON(value)} as a saverest date — expected M/D/YYYY, which is what both Soil Data Access and the shipped sacatlog.txt write`
|
|
291
294
|
)
|
|
292
295
|
}
|
|
293
296
|
|
package/{lib → sdk}/test-kit.ts
RENAMED
|
@@ -2,24 +2,10 @@
|
|
|
2
2
|
* @copyright Sister Software
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
|
-
*
|
|
6
|
-
* Hand-built survey areas for the fixture rung: geometry, attributes and an outline, with no network and
|
|
7
|
-
* no GDAL in the loop.
|
|
8
|
-
*
|
|
9
|
-
* A FIXTURE RUNG THAT COULD ONLY RUN THROUGH ogr2ogr WOULD TEST THE CONVERSION ON THE MACHINES THAT HAVE
|
|
10
|
-
* IT AND NOTHING AT ALL ON THE ONES THAT DO NOT. What these fixtures exercise is the whole database half —
|
|
11
|
-
* the declared-domain check, the cell classification, the area-weighted reduction, the four absence
|
|
12
|
-
* shares, the coverage rows, the manifest and the seal.
|
|
13
|
-
*
|
|
14
|
-
* THE ABSENCE CASES ARE THE POINT, AND IOWA HAS NONE OF THEM. Every Iowa survey area is fully digitized,
|
|
15
|
-
* so `NOTCOM`, `NOTPUB` and access-denied map units never appear in the live build — which means the only
|
|
16
|
-
* place `nodata_share` can be exercised is here. The same is true of a component whose rating is NULL for
|
|
17
|
-
* the not-rateable reason and of a class-8 rating: both exist in Iowa but sparsely, and a fixture pins the
|
|
18
|
-
* behaviour rather than hoping a county contains one.
|
|
19
5
|
*/
|
|
20
6
|
|
|
21
|
-
// The exterior and hole ring builders live in `@mailwoman/spatial
|
|
22
|
-
//
|
|
7
|
+
// The exterior and hole ring builders live in `@mailwoman/spatial`; a second copy
|
|
8
|
+
// is a second place for a hole to stop being one.
|
|
23
9
|
import { rectangleRing } from "@mailwoman/spatial"
|
|
24
10
|
|
|
25
11
|
import type { SoilComponentTable, SoilMapUnitTable } from "#schema"
|
|
@@ -27,22 +13,19 @@ import type { SoilDelineation, SoilFeatureSource } from "#sdk/ingest"
|
|
|
27
13
|
import type { SurveyAreaAttributes } from "#sdk/survey-area"
|
|
28
14
|
|
|
29
15
|
/**
|
|
30
|
-
*
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
/**
|
|
34
|
-
* Where the fixture geometry sits — central Iowa, so the cells it produces are the ones a real build would use.
|
|
16
|
+
* Where the fixture geometry sits — central Iowa, so the cells it produces
|
|
17
|
+
* are the ones a real build would use.
|
|
35
18
|
*/
|
|
36
19
|
export const FIXTURE_ORIGIN = { lat: 41.6, lon: -93.6 }
|
|
37
20
|
|
|
38
21
|
/**
|
|
39
|
-
* Degrees per fixture square side
|
|
40
|
-
* square has both an interior and a fringe.
|
|
22
|
+
* Degrees per fixture square side, about 1.6 km here — several resolution-9 cells across,
|
|
23
|
+
* so a square has both an interior and a fringe.
|
|
41
24
|
*/
|
|
42
25
|
export const FIXTURE_SIDE = 0.015
|
|
43
26
|
|
|
44
27
|
/**
|
|
45
|
-
* The fixture map units
|
|
28
|
+
* The fixture map units, each exercising exactly one reading.
|
|
46
29
|
*/
|
|
47
30
|
export function fixtureMapUnits(areaSymbol = "XX001"): SoilMapUnitTable[] {
|
|
48
31
|
return [
|
|
@@ -115,8 +98,8 @@ export function fixtureMapUnits(areaSymbol = "XX001"): SoilMapUnitTable[] {
|
|
|
115
98
|
}
|
|
116
99
|
|
|
117
100
|
/**
|
|
118
|
-
* The fixture components
|
|
119
|
-
* report as
|
|
101
|
+
* The fixture components; `mu-mixed` is 45/35/20 across three classes, the case a
|
|
102
|
+
* winner-class schema would report as class 2 and this one as a mixture.
|
|
120
103
|
*/
|
|
121
104
|
export function fixtureComponents(): SoilComponentTable[] {
|
|
122
105
|
return [
|
|
@@ -124,9 +107,10 @@ export function fixtureComponents(): SoilComponentTable[] {
|
|
|
124
107
|
component("co-mixed-2", "mu-mixed", 35, "Series", "3", "e"),
|
|
125
108
|
component("co-mixed-3", "mu-mixed", 20, "Series", "6", "s"),
|
|
126
109
|
component("co-class8", "mu-class8", 100, "Series", "8", "s"),
|
|
127
|
-
// A miscellaneous area with no rating
|
|
110
|
+
// A miscellaneous area with no rating is not rateable.
|
|
111
|
+
// That differs from unrated soil and class 8.
|
|
128
112
|
component("co-water", "mu-water", 100, "Miscellaneous area", null, null),
|
|
129
|
-
// A
|
|
113
|
+
// A soil type the survey did not rate: unrated.
|
|
130
114
|
component("co-unrated", "mu-unrated", 100, "Series", null, null),
|
|
131
115
|
// A minority component small enough to fall under the truncation floor once the lattice splits it.
|
|
132
116
|
component("co-tail", "mu-mixed", 1, "Series", "7", "e"),
|
|
@@ -178,8 +162,8 @@ export function fixtureDomains(): SurveyAreaAttributes["domains"] {
|
|
|
178
162
|
}
|
|
179
163
|
|
|
180
164
|
/**
|
|
181
|
-
* The fixture delineations
|
|
182
|
-
*
|
|
165
|
+
* The fixture delineations include mixed, class-8, water and unrated squares, plus a `notcom` square.
|
|
166
|
+
* They sit left to right, each on its own ground area.
|
|
183
167
|
*/
|
|
184
168
|
export function fixtureDelineations(areaSymbol = "XX001"): SoilDelineation[] {
|
|
185
169
|
const { lat, lon } = FIXTURE_ORIGIN
|
|
@@ -194,12 +178,8 @@ export function fixtureDelineations(areaSymbol = "XX001"): SoilDelineation[] {
|
|
|
194
178
|
}
|
|
195
179
|
|
|
196
180
|
/**
|
|
197
|
-
* The outline covering every fixture delineation, with margin
|
|
198
|
-
*
|
|
199
|
-
* The margin is nearly a degree because the coverage test is CONSERVATIVE: `interiorCoverageCellSet` keeps only cells
|
|
200
|
-
* lying wholly inside the outline, and a resolution-6 cell is about 36 km across. An outline the size of the fixture
|
|
201
|
-
* squares yields zero interior cells and the build refuses — correctly, since an artifact with no coverage rows answers
|
|
202
|
-
* unknown everywhere while reporting success.
|
|
181
|
+
* The outline covering every fixture delineation, with margin because the coverage
|
|
182
|
+
* test keeps only cells lying wholly inside the outline.
|
|
203
183
|
*/
|
|
204
184
|
export function fixtureOutline(margin = 0.75): { type: "Polygon"; coordinates: number[][][] } {
|
|
205
185
|
const { lat, lon } = FIXTURE_ORIGIN
|
|
@@ -231,10 +211,8 @@ export function fixtureSource(delineations: SoilDelineation[], areaSymbol = "XX0
|
|
|
231
211
|
}
|
|
232
212
|
|
|
233
213
|
/**
|
|
234
|
-
* One fixture survey area's attributes
|
|
235
|
-
*
|
|
236
|
-
* `areaAcres` is left NULL on purpose: the area cross-check compares against what the AUTHORITY publishes, and a
|
|
237
|
-
* fixture that invented an acreage would be checking this package's arithmetic against itself.
|
|
214
|
+
* One fixture survey area's attributes; `areaAcres` is left NULL on purpose so the area
|
|
215
|
+
* cross-check compares against the authority rather than this package's own arithmetic.
|
|
238
216
|
*/
|
|
239
217
|
export function fixtureAttributes(areaSymbol = "XX001"): SurveyAreaAttributes {
|
|
240
218
|
return {
|