@mailwoman/soil 9.4.0 → 10.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +118 -116
- package/lib/index.ts +66 -98
- package/lib/paths.ts +24 -0
- package/lib/schema.ts +188 -120
- package/lib/vocabulary.ts +40 -107
- package/out/index.d.ts +52 -70
- package/out/index.d.ts.map +1 -1
- package/out/index.js +24 -72
- package/out/index.js.map +1 -1
- package/out/paths.d.ts +19 -0
- package/out/paths.d.ts.map +1 -0
- package/out/paths.js +21 -0
- package/out/paths.js.map +1 -0
- package/out/schema.d.ts +187 -119
- package/out/schema.d.ts.map +1 -1
- package/out/schema.js +31 -31
- package/out/schema.js.map +1 -1
- package/out/sdk/acquire.d.ts +15 -24
- package/out/sdk/acquire.d.ts.map +1 -1
- package/out/sdk/acquire.js +5 -20
- package/out/sdk/acquire.js.map +1 -1
- package/out/sdk/build-soil.d.ts +67 -70
- package/out/sdk/build-soil.d.ts.map +1 -1
- package/out/sdk/build-soil.js +65 -95
- package/out/sdk/build-soil.js.map +1 -1
- package/out/sdk/cell-tiers.d.ts +7 -19
- package/out/sdk/cell-tiers.d.ts.map +1 -1
- package/out/sdk/cell-tiers.js +19 -38
- package/out/sdk/cell-tiers.js.map +1 -1
- package/out/sdk/cells.d.ts +15 -41
- package/out/sdk/cells.d.ts.map +1 -1
- package/out/sdk/cells.js +11 -38
- package/out/sdk/cells.js.map +1 -1
- package/out/sdk/client.d.ts +23 -48
- package/out/sdk/client.d.ts.map +1 -1
- package/out/sdk/client.js +19 -63
- package/out/sdk/client.js.map +1 -1
- package/out/sdk/download.d.ts +24 -47
- package/out/sdk/download.d.ts.map +1 -1
- package/out/sdk/download.js +14 -54
- package/out/sdk/download.js.map +1 -1
- package/out/sdk/ingest/chunk.d.ts +77 -0
- package/out/sdk/ingest/chunk.d.ts.map +1 -0
- package/out/sdk/{ingest-chunk.js → ingest/chunk.js} +19 -17
- package/out/sdk/ingest/chunk.js.map +1 -0
- package/out/sdk/ingest/worker.d.ts +9 -0
- package/out/sdk/ingest/worker.d.ts.map +1 -0
- package/out/{scripts/ingest-chunk.js → sdk/ingest/worker.js} +11 -10
- package/out/sdk/ingest/worker.js.map +1 -0
- package/out/sdk/ingest.d.ts +40 -52
- package/out/sdk/ingest.d.ts.map +1 -1
- package/out/sdk/ingest.js +18 -61
- package/out/sdk/ingest.js.map +1 -1
- package/out/sdk/measure-resolutions.d.ts +5 -16
- package/out/sdk/measure-resolutions.d.ts.map +1 -1
- package/out/sdk/measure-resolutions.js +3 -15
- package/out/sdk/measure-resolutions.js.map +1 -1
- package/out/sdk/reduce.d.ts +33 -59
- package/out/sdk/reduce.d.ts.map +1 -1
- package/out/sdk/reduce.js +47 -78
- package/out/sdk/reduce.js.map +1 -1
- package/out/sdk/survey-area.d.ts +16 -48
- package/out/sdk/survey-area.d.ts.map +1 -1
- package/out/sdk/survey-area.js +33 -77
- package/out/sdk/survey-area.js.map +1 -1
- package/out/sdk/tabular.d.ts +32 -31
- package/out/sdk/tabular.d.ts.map +1 -1
- package/out/sdk/tabular.js +58 -56
- package/out/sdk/tabular.js.map +1 -1
- package/out/sdk/test-kit.d.ts +57 -0
- package/out/sdk/test-kit.d.ts.map +1 -0
- package/out/{test-kit.js → sdk/test-kit.js} +18 -39
- package/out/sdk/test-kit.js.map +1 -0
- package/out/sdk/verify.d.ts +15 -51
- package/out/sdk/verify.d.ts.map +1 -1
- package/out/sdk/verify.js +19 -77
- package/out/sdk/verify.js.map +1 -1
- package/out/vocabulary.d.ts +37 -103
- package/out/vocabulary.d.ts.map +1 -1
- package/out/vocabulary.js +34 -107
- package/out/vocabulary.js.map +1 -1
- package/package.json +36 -190
- package/{lib/sdk → sdk}/acquire.ts +17 -27
- package/{lib/sdk → sdk}/build-soil.ts +114 -128
- package/{lib/sdk → sdk}/cell-tiers.ts +20 -39
- package/{lib/sdk → sdk}/cells.ts +17 -43
- package/sdk/client.ts +147 -0
- package/sdk/download.ts +131 -0
- package/{lib/sdk/ingest-chunk.ts → sdk/ingest/chunk.ts} +34 -26
- package/{lib/scripts/ingest-chunk.ts → sdk/ingest/worker.ts} +10 -9
- package/sdk/ingest.ts +253 -0
- package/{lib/sdk → sdk}/measure-resolutions.ts +5 -16
- package/sdk/reduce.ts +344 -0
- package/{lib/sdk → sdk}/survey-area.ts +39 -83
- package/{lib/sdk → sdk}/tabular.ts +62 -59
- package/{lib → sdk}/test-kit.ts +18 -40
- package/{lib/sdk → sdk}/verify.ts +29 -85
- package/lib/sdk/client.ts +0 -184
- package/lib/sdk/download.ts +0 -161
- package/lib/sdk/index.ts +0 -20
- package/lib/sdk/ingest.ts +0 -278
- package/lib/sdk/reduce.ts +0 -375
- package/out/scripts/ingest-chunk.d.ts +0 -11
- package/out/scripts/ingest-chunk.d.ts.map +0 -1
- package/out/scripts/ingest-chunk.js.map +0 -1
- package/out/sdk/index.d.ts +0 -20
- package/out/sdk/index.d.ts.map +0 -1
- package/out/sdk/index.js +0 -20
- package/out/sdk/index.js.map +0 -1
- package/out/sdk/ingest-chunk.d.ts +0 -73
- package/out/sdk/ingest-chunk.d.ts.map +0 -1
- package/out/sdk/ingest-chunk.js.map +0 -1
- package/out/test-kit.d.ts +0 -79
- package/out/test-kit.d.ts.map +0 -1
- package/out/test-kit.js.map +0 -1
package/out/sdk/download.js
CHANGED
|
@@ -2,75 +2,38 @@
|
|
|
2
2
|
* @copyright Sister Software
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
|
-
*
|
|
6
|
-
* Acquire one survey area's published archive — 13 to 41 MB streamed to disk and unzipped.
|
|
7
|
-
*
|
|
8
|
-
* THE TRANSFER ITSELF LIVES IN `@mailwoman/core/utils`, and `streamToDisk` carries why a file transfer of
|
|
9
|
-
* this size keeps raw `fetch` instead of going through `APIClient`, plus the `.part`-rename rule. What is
|
|
10
|
-
* soil's, and stays here, is the URL shape, the cache key, and the two facts below that the shared
|
|
11
|
-
* transfer is told rather than assumes: the progress stride and what a 400 means. The METADATA reads
|
|
12
|
-
* around this one do go through `APIClient` — see `client.ts`.
|
|
13
|
-
*
|
|
14
|
-
* FRESHNESS IS `sacatalog.saverest`, NEVER A LENGTH PROBE, AND THE HOST LEAVES NO CHOICE. It answers `HEAD`
|
|
15
|
-
* with HTTP 405 (`allow: GET`) and IGNORES `Range`: a request with `Range: bytes=0-0` returned HTTP 200 and
|
|
16
|
-
* transferred the whole 27,598,377 bytes in 7.23 s. So "check the size first" starts a real download. The
|
|
17
|
-
* cache is keyed on the version date the tabular service reports instead, and a vintage already on disk is
|
|
18
|
-
* never re-fetched. The `Range` behaviour is PATH-SPECIFIC rather than host-wide — `/DataAvailability/`
|
|
19
|
-
* does answer 206 — so a client must probe per path rather than conclude from one.
|
|
20
|
-
*
|
|
21
|
-
* THE FILENAME EMBEDS THE VERSION DATE AND A WRONG ONE IS AN HTTP 400. Not a 404: asking for a date the
|
|
22
|
-
* host does not hold reads as a malformed request rather than a missing file, which is why the date comes
|
|
23
|
-
* from the catalogue rather than from a guess. The square brackets must be sent literally, so the URL is
|
|
24
|
-
* built with them percent-encoded.
|
|
25
|
-
*
|
|
26
|
-
* TWO CACHE VARIANTS EXIST AND THE BARE ONE IS WANTED. `wss_SSA_IA153_[2025-09-09].zip` is 25,474,922 bytes;
|
|
27
|
-
* `wss_SSA_IA153_soildb_IA_2003_[2025-09-09].zip` is 27,598,377 and differs only by an EMPTY Microsoft Access
|
|
28
|
-
* template container for a workflow this program does not use. Confirmed on a second area (`IA015`:
|
|
29
|
-
* 38,981,269 against 41,104,724 bytes) and on a third that ships no template at all (`TX299`, 13,455,641
|
|
30
|
-
* bytes, 97 files, no `.mdb`).
|
|
31
5
|
*/
|
|
32
6
|
import { tryStat } from "@mailwoman/core/fs/readers";
|
|
33
7
|
import { makeDirectories } from "@mailwoman/core/fs/writers";
|
|
34
8
|
import { runFile } from "@mailwoman/core/process";
|
|
35
9
|
import { streamToDisk } from "@mailwoman/core/utils";
|
|
36
|
-
import {
|
|
10
|
+
import { PathBuilder } from "path-ts";
|
|
37
11
|
/**
|
|
38
|
-
*
|
|
39
|
-
* which lists `GET /{CacheName}/{FileName}`.
|
|
12
|
+
* Points to the Web Soil Survey download cache that serves survey-area archives.
|
|
40
13
|
*/
|
|
41
14
|
export const WSS_SSA_CACHE_URL = "https://websoilsurvey.sc.egov.usda.gov/DSD/Download/Cache/SSA";
|
|
42
15
|
/**
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
* The brackets are percent-encoded rather than sent raw: they are not valid in a URL path, and a client that sends them
|
|
46
|
-
* literally depends on the fetcher tolerating them.
|
|
16
|
+
* Returns the archive URL for one survey area at one version date, with the brackets
|
|
17
|
+
* around the date percent-encoded because they are not valid in a URL path.
|
|
47
18
|
*/
|
|
48
19
|
export function surveyAreaArchiveURL(areaSymbol, versionDate) {
|
|
49
20
|
return `${WSS_SSA_CACHE_URL}/wss_SSA_${areaSymbol}_%5B${versionDate}%5D.zip`;
|
|
50
21
|
}
|
|
51
|
-
/**
|
|
52
|
-
* Bytes between progress reports. Smaller than the shared default because these archives are 13–41 MB, and the default
|
|
53
|
-
* stride would leave the smallest of them reporting once.
|
|
54
|
-
*/
|
|
55
22
|
const PROGRESS_STRIDE_BYTES = 8 * 1024 * 1024;
|
|
56
|
-
/**
|
|
57
|
-
* What this host answers for a version date it does not hold. NOT a 404: it reads as a malformed request rather than a
|
|
58
|
-
* missing file, which is why the message below says so and why the date comes from the catalogue rather than a guess.
|
|
59
|
-
*/
|
|
60
23
|
const UNKNOWN_VERSION_STATUS = 400;
|
|
61
24
|
/**
|
|
62
|
-
*
|
|
25
|
+
* Downloads and unzips one survey area into the cache, skipping completed steps.
|
|
26
|
+
* Returns the paths to its files.
|
|
63
27
|
*
|
|
64
|
-
*
|
|
65
|
-
* complete archive — the same discipline the database build uses, for the same reason.
|
|
28
|
+
* The download goes through a `.part` file so an interrupted transfer never looks like a complete archive.
|
|
66
29
|
*
|
|
67
|
-
* @throws {Error} When the host answers anything but 200, or when the extracted
|
|
68
|
-
*
|
|
30
|
+
* @throws {Error} When the host answers anything but 200, or when the extracted
|
|
31
|
+
* tree lacks the `spatial` or `tabular` directory.
|
|
69
32
|
*/
|
|
70
33
|
export async function downloadSurveyArea(options) {
|
|
71
|
-
const vintageDirectory =
|
|
72
|
-
const root =
|
|
73
|
-
const archivePath =
|
|
34
|
+
const vintageDirectory = PathBuilder.from(options.cacheRoot)(options.versionDate);
|
|
35
|
+
const root = vintageDirectory(options.areaSymbol);
|
|
36
|
+
const archivePath = vintageDirectory(`wss_SSA_${options.areaSymbol}.zip`);
|
|
74
37
|
if (!(await tryStat(root))) {
|
|
75
38
|
await makeDirectories(vintageDirectory);
|
|
76
39
|
if (await tryStat(archivePath)) {
|
|
@@ -85,21 +48,18 @@ export async function downloadSurveyArea(options) {
|
|
|
85
48
|
describeStatus: (status) => status === UNKNOWN_VERSION_STATUS
|
|
86
49
|
? " — this host answers 400 rather than 404 for a version date it does not hold, so check the date against sacatalog.saverest"
|
|
87
50
|
: undefined,
|
|
88
|
-
// Every progress line names the area, because a full acquisition interleaves hundreds of them.
|
|
89
51
|
...(options.onProgress
|
|
90
52
|
? { onProgress: (message) => options.onProgress?.(`${options.areaSymbol}: ${message}`) }
|
|
91
53
|
: {}),
|
|
92
54
|
});
|
|
93
55
|
}
|
|
94
|
-
// The archive holds its files under an `<AREASYMBOL>/` root already, so it unzips into the vintage directory
|
|
95
|
-
// rather than into a directory named for itself.
|
|
96
56
|
await runFile("unzip", ["-o", "-q", archivePath, "-d", vintageDirectory]);
|
|
97
57
|
}
|
|
98
58
|
else {
|
|
99
59
|
options.onProgress?.(`${options.areaSymbol}: already extracted for ${options.versionDate}`);
|
|
100
60
|
}
|
|
101
|
-
const spatialDirectory =
|
|
102
|
-
const tabularDirectory =
|
|
61
|
+
const spatialDirectory = root("spatial");
|
|
62
|
+
const tabularDirectory = root("tabular");
|
|
103
63
|
for (const directory of [spatialDirectory, tabularDirectory]) {
|
|
104
64
|
if (!(await tryStat(directory))) {
|
|
105
65
|
throw new Error(`soil download: ${options.areaSymbol} extracted without a ${directory} directory — every survey area publishes both spatial/ and tabular/, so this archive is not the product`);
|
package/out/sdk/download.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download.js","sourceRoot":"","sources":["../../
|
|
1
|
+
{"version":3,"file":"download.js","sourceRoot":"","sources":["../../sdk/download.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAA;AACjD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AACpD,OAAO,EAAE,WAAW,EAAwB,MAAM,SAAS,CAAA;AAE3D;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,+DAA+D,CAAA;AAEhG;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,UAAkB,EAAE,WAAmB;IAC3E,OAAO,GAAG,iBAAiB,YAAY,UAAU,OAAO,WAAW,SAAS,CAAA;AAC7E,CAAC;AAsBD,MAAM,qBAAqB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAA;AAE7C,MAAM,sBAAsB,GAAG,GAAG,CAAA;AAuBlC;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAAkC;IAC1E,MAAM,gBAAgB,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAA;IACjF,MAAM,IAAI,GAAG,gBAAgB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAA;IACjD,MAAM,WAAW,GAAG,gBAAgB,CAAC,WAAW,OAAO,CAAC,UAAU,MAAM,CAAC,CAAA;IAEzE,IAAI,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAC5B,MAAM,eAAe,CAAC,gBAAgB,CAAC,CAAA;QAEvC,IAAI,MAAM,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;YAChC,OAAO,CAAC,UAAU,EAAE,CAAC,GAAG,OAAO,CAAC,UAAU,iBAAiB,OAAO,CAAC,WAAW,qBAAqB,CAAC,CAAA;QACrG,CAAC;aAAM,CAAC;YACP,MAAM,YAAY,CAAC;gBAClB,GAAG,EAAE,oBAAoB,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,WAAW,CAAC;gBAClE,WAAW,EAAE,WAAW;gBACxB,OAAO,EAAE,eAAe;gBACxB,mBAAmB,EAAE,qBAAqB;gBAC1C,cAAc,EAAE,CAAC,MAAM,EAAE,EAAE,CAC1B,MAAM,KAAK,sBAAsB;oBAChC,CAAC,CAAC,4HAA4H;oBAC9H,CAAC,CAAC,SAAS;gBAEb,GAAG,CAAC,OAAO,CAAC,UAAU;oBACrB,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,OAAe,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,GAAG,OAAO,CAAC,UAAU,KAAK,OAAO,EAAE,CAAC,EAAE;oBAChG,CAAC,CAAC,EAAE,CAAC;aACN,CAAC,CAAA;QACH,CAAC;QAED,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,gBAAgB,CAAC,CAAC,CAAA;IAC1E,CAAC;SAAM,CAAC;QACP,OAAO,CAAC,UAAU,EAAE,CAAC,GAAG,OAAO,CAAC,UAAU,2BAA2B,OAAO,CAAC,WAAW,EAAE,CAAC,CAAA;IAC5F,CAAC;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,SAAS,CAAC,CAAA;IACxC,MAAM,gBAAgB,GAAG,IAAI,CAAC,SAAS,CAAC,CAAA;IAExC,KAAK,MAAM,SAAS,IAAI,CAAC,gBAAgB,EAAE,gBAAgB,CAAC,EAAE,CAAC;QAC9D,IAAI,CAAC,CAAC,MAAM,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CACd,kBAAkB,OAAO,CAAC,UAAU,wBAAwB,SAAS,yGAAyG,CAC9K,CAAA;QACF,CAAC;IACF,CAAC;IAED,OAAO;QACN,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,IAAI;QACJ,gBAAgB;QAChB,gBAAgB;QAChB,WAAW;KACX,CAAA;AACF,CAAC"}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* This streaming pass writes each delineation to `soil_map_unit_area` and the build's touch table.
|
|
7
|
+
* It can process part of one survey area as a unit of work.
|
|
8
|
+
*
|
|
9
|
+
* H3's wasm heap cannot be reset from JavaScript. It also cannot sustain unlimited polyfill calls.
|
|
10
|
+
* Two runs over the flood layer's real product stopped after roughly 510,000 and 798,000 features.
|
|
11
|
+
* The same geometries classify in milliseconds in a fresh process. A build that completes only when
|
|
12
|
+
* fragmentation stays low is not reproducible. {@linkcode buildSoilDatabase} bounds each process to a range
|
|
13
|
+
* of the shapefile's FIDs. Each range gets a separate process with a fresh heap. Iowa's 99 survey areas
|
|
14
|
+
* contain far more delineations together than each area contains by itself. The bound therefore applies per range.
|
|
15
|
+
*
|
|
16
|
+
* The chunk appends rows to a database created by the parent. The parent seals that database and adds up
|
|
17
|
+
* the counts returned by each chunk. Chunks run one at a time against the file, so only one writer uses it.
|
|
18
|
+
*/
|
|
19
|
+
import type { DatabaseClient } from "@mailwoman/sqlite/client";
|
|
20
|
+
import type { SoilDatabase } from "#schema";
|
|
21
|
+
import type { SoilFeatureSource } from "#sdk/ingest";
|
|
22
|
+
/**
|
|
23
|
+
* What one chunk produced.
|
|
24
|
+
*
|
|
25
|
+
* Every field is JSON-serializable, because a chunk normally reports across a process boundary.
|
|
26
|
+
*/
|
|
27
|
+
export interface SoilChunkResult {
|
|
28
|
+
areaSymbol: string;
|
|
29
|
+
delineations: number;
|
|
30
|
+
/**
|
|
31
|
+
* Delineations whose bounding box forced a resolution coarser than the target.
|
|
32
|
+
*/
|
|
33
|
+
coarsened: number;
|
|
34
|
+
/**
|
|
35
|
+
* `[coverageCell, delineationsReachingIt]` pairs — an array rather than a `Map`
|
|
36
|
+
* so it survives the process boundary.
|
|
37
|
+
*/
|
|
38
|
+
observedByCoverageCell: Array<[number, number]>;
|
|
39
|
+
/**
|
|
40
|
+
* The same, counting only delineations whose map unit has soil mapping behind it.
|
|
41
|
+
*
|
|
42
|
+
* Separate from the total because the coverage rule depends on this value.
|
|
43
|
+
* A cell reached only by `notcom` or access-denied polygons lies inside a published
|
|
44
|
+
* survey area and contains no digitized soil mapping.
|
|
45
|
+
*
|
|
46
|
+
* Section 3.2 of the survey specification assigns no row to that cell.
|
|
47
|
+
*/
|
|
48
|
+
mappedByCoverageCell: Array<[number, number]>;
|
|
49
|
+
/**
|
|
50
|
+
* Square meters computed from the encoded rings, with holes and with every ring treated as exterior.
|
|
51
|
+
*/
|
|
52
|
+
area: {
|
|
53
|
+
nestedM2: number;
|
|
54
|
+
allExteriorM2: number;
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
export interface IngestSoilChunkOptions {
|
|
58
|
+
source: SoilFeatureSource;
|
|
59
|
+
indexResolution: number;
|
|
60
|
+
coverageResolution: number;
|
|
61
|
+
/**
|
|
62
|
+
* The map units with no soil mapping behind them — `notcom`, `notpub`,
|
|
63
|
+
* access denied, or no readable component weights.
|
|
64
|
+
*
|
|
65
|
+
* Passed in rather than joined here so the chunk stays a streaming pass over geometry.
|
|
66
|
+
*/
|
|
67
|
+
noMappingMukeys: ReadonlySet<string>;
|
|
68
|
+
onProgress?: (message: string) => void;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Stream one chunk of one survey area into `database`.
|
|
72
|
+
*
|
|
73
|
+
* @throws {Error} On a delineation the classifier refuses — which includes the
|
|
74
|
+
* allocator's silent zero-cell answer.
|
|
75
|
+
*/
|
|
76
|
+
export declare function ingestSoilChunk(database: DatabaseClient<SoilDatabase>, options: IngestSoilChunkOptions): Promise<SoilChunkResult>;
|
|
77
|
+
//# sourceMappingURL=chunk.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunk.d.ts","sourceRoot":"","sources":["../../../sdk/ingest/chunk.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAIH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AAE9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAE3C,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAiBpD;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC/B,UAAU,EAAE,MAAM,CAAA;IAClB,YAAY,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IACjB;;;OAGG;IACH,sBAAsB,EAAE,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;IAC/C;;;;;;;;OAQG;IACH,oBAAoB,EAAE,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;IAC7C;;OAEG;IACH,IAAI,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAA;KAAE,CAAA;CACjD;AAED,MAAM,WAAW,sBAAsB;IACtC,MAAM,EAAE,iBAAiB,CAAA;IACzB,eAAe,EAAE,MAAM,CAAA;IACvB,kBAAkB,EAAE,MAAM,CAAA;IAC1B;;;;;OAKG;IACH,eAAe,EAAE,WAAW,CAAC,MAAM,CAAC,CAAA;IACpC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;CACtC;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CACpC,QAAQ,EAAE,cAAc,CAAC,YAAY,CAAC,EACtC,OAAO,EAAE,sBAAsB,GAC7B,OAAO,CAAC,eAAe,CAAC,CA2F1B"}
|
|
@@ -3,28 +3,29 @@
|
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
6
|
+
* This streaming pass writes each delineation to `soil_map_unit_area` and the build's touch table.
|
|
7
|
+
* It can process part of one survey area as a unit of work.
|
|
8
8
|
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
* Iowa's 99 survey areas hold far more delineations together than any one of them does, which is why the
|
|
16
|
-
* bound is per RANGE rather than per area.
|
|
9
|
+
* H3's wasm heap cannot be reset from JavaScript. It also cannot sustain unlimited polyfill calls.
|
|
10
|
+
* Two runs over the flood layer's real product stopped after roughly 510,000 and 798,000 features.
|
|
11
|
+
* The same geometries classify in milliseconds in a fresh process. A build that completes only when
|
|
12
|
+
* fragmentation stays low is not reproducible. {@linkcode buildSoilDatabase} bounds each process to a range
|
|
13
|
+
* of the shapefile's FIDs. Each range gets a separate process with a fresh heap. Iowa's 99 survey areas
|
|
14
|
+
* contain far more delineations together than each area contains by itself. The bound therefore applies per range.
|
|
17
15
|
*
|
|
18
|
-
*
|
|
19
|
-
* counts
|
|
20
|
-
* and no locking to reason about.
|
|
16
|
+
* The chunk appends rows to a database created by the parent. The parent seals that database and adds up
|
|
17
|
+
* the counts returned by each chunk. Chunks run one at a time against the file, so only one writer uses it.
|
|
21
18
|
*/
|
|
22
19
|
import { addCoverageCells, encodeRings, ringAreaReadings, ringsBoundingBox, shortCellToInt } from "@mailwoman/spatial";
|
|
23
20
|
import { beginBatched } from "@mailwoman/sqlite/batched";
|
|
24
21
|
import { classifyDelineationCells } from "#sdk/cells";
|
|
25
22
|
/**
|
|
26
|
-
* Rows per bulk-insert transaction.
|
|
27
|
-
*
|
|
23
|
+
* Rows per bulk-insert transaction.
|
|
24
|
+
*
|
|
25
|
+
* Chosen for the geometry table.
|
|
26
|
+
* Each row contains a blob.
|
|
27
|
+
*
|
|
28
|
+
* A larger transaction grows the write-ahead file without improving throughput.
|
|
28
29
|
*/
|
|
29
30
|
const INSERT_TRANSACTION_ROWS = 5000;
|
|
30
31
|
/**
|
|
@@ -34,7 +35,8 @@ const PROGRESS_STRIDE = 20_000;
|
|
|
34
35
|
/**
|
|
35
36
|
* Stream one chunk of one survey area into `database`.
|
|
36
37
|
*
|
|
37
|
-
* @throws {Error} On a delineation the classifier refuses — which includes the
|
|
38
|
+
* @throws {Error} On a delineation the classifier refuses — which includes the
|
|
39
|
+
* allocator's silent zero-cell answer.
|
|
38
40
|
*/
|
|
39
41
|
export async function ingestSoilChunk(database, options) {
|
|
40
42
|
const insertArea = database.prepare("INSERT INTO soil_map_unit_area (area_id, mukey, areasymbol, min_lat, min_lon, max_lat, max_lon, rings) " +
|
|
@@ -95,4 +97,4 @@ export async function ingestSoilChunk(database, options) {
|
|
|
95
97
|
area: { nestedM2: nestedArea, allExteriorM2: allExteriorArea },
|
|
96
98
|
};
|
|
97
99
|
}
|
|
98
|
-
//# sourceMappingURL=
|
|
100
|
+
//# sourceMappingURL=chunk.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunk.js","sourceRoot":"","sources":["../../../sdk/ingest/chunk.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,gBAAgB,EAAE,WAAW,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AACtH,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAA;AAIxD,OAAO,EAAE,wBAAwB,EAAE,MAAM,YAAY,CAAA;AAGrD;;;;;;;GAOG;AACH,MAAM,uBAAuB,GAAG,IAAI,CAAA;AAEpC;;GAEG;AACH,MAAM,eAAe,GAAG,MAAM,CAAA;AAiD9B;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACpC,QAAsC,EACtC,OAA+B;IAE/B,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAClC,yGAAyG;QACxG,iCAAiC,CAClC,CAAA;IAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CACnC,0FAA0F,CAC1F,CAAA;IAED,MAAM,sBAAsB,GAAG,IAAI,GAAG,EAAkB,CAAA;IACxD,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAAkB,CAAA;IAEtD,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,UAAU,GAAG,CAAC,CAAA;IAClB,IAAI,eAAe,GAAG,CAAC,CAAA;IAEvB,MAAM,KAAK,GAAG,YAAY,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,uBAAuB,EAAE,CAAC,CAAA;IAEhF,IAAI,CAAC;QACJ,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,EAAE,CAAC;YAC/D,MAAM,IAAI,GAAG,gBAAgB,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAA;YACnD,MAAM,KAAK,GAAG,gBAAgB,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAA;YAEpD,UAAU,IAAI,KAAK,CAAC,MAAM,CAAA;YAC1B,eAAe,IAAI,KAAK,CAAC,WAAW,CAAA;YAEpC,UAAU,CAAC,GAAG,CACb,WAAW,CAAC,MAAM,EAClB,WAAW,CAAC,KAAK,EACjB,WAAW,CAAC,UAAU,EACtB,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,MAAM,EACX,WAAW,CAAC,WAAW,CAAC,QAAQ,CAAC,CACjC,CAAA;YAED,MAAM,UAAU,GAAG,wBAAwB,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,eAAe,EAAE,WAAW,CAAC,MAAM,CAAC,CAAA;YAE9G,IAAI,UAAU,CAAC,UAAU,KAAK,OAAO,CAAC,eAAe,EAAE,CAAC;gBACvD,SAAS,EAAE,CAAA;YACZ,CAAC;YAED,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAA;YAEvC,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;gBACrC,WAAW,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC,UAAU,EAAE,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;gBACnF,gBAAgB,CAAC,aAAa,EAAE,IAAI,EAAE,UAAU,CAAC,UAAU,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAA;YACzF,CAAC;YAED,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;gBACvC,WAAW,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC,UAAU,EAAE,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;gBACnF,gBAAgB,CAAC,aAAa,EAAE,IAAI,EAAE,UAAU,CAAC,UAAU,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAA;YACzF,CAAC;YAED,MAAM,UAAU,GAAG,CAAC,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,CAAA;YAElE,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;gBAC1C,sBAAsB,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;gBAE7F,IAAI,UAAU,EAAE,CAAC;oBAChB,oBAAoB,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,oBAAoB,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;gBAC1F,CAAC;YACF,CAAC;YAED,YAAY,EAAE,CAAA;YAEd,KAAK,CAAC,UAAU,EAAE,CAAA;YAElB,IAAI,YAAY,GAAG,eAAe,KAAK,CAAC,EAAE,CAAC;gBAC1C,OAAO,CAAC,UAAU,EAAE,CAAC,GAAG,YAAY,CAAC,cAAc,EAAE,6BAA6B,CAAC,CAAA;YACpF,CAAC;QACF,CAAC;QAED,KAAK,CAAC,MAAM,EAAE,CAAA;IACf,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,KAAK,CAAC,eAAe,EAAE,CAAA;QAEvB,MAAM,KAAK,CAAA;IACZ,CAAC;IAED,OAAO;QACN,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,UAAU;QACrC,YAAY;QACZ,SAAS;QACT,sBAAsB,EAAE,CAAC,GAAG,sBAAsB,CAAC;QACnD,oBAAoB,EAAE,CAAC,GAAG,oBAAoB,CAAC;QAC/C,IAAI,EAAE,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,eAAe,EAAE;KAC9D,CAAA;AACF,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../../../sdk/ingest/worker.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG"}
|
|
@@ -3,14 +3,12 @@
|
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*
|
|
6
|
-
* One chunk of the soil ingest
|
|
7
|
-
* The process boundary and the stdout contract live with `runIngestChunkScript`; what stays here is only
|
|
8
|
-
* this product's flags and its feature-source constructor.
|
|
6
|
+
* One chunk of the soil ingest as its own process, spawned by `buildSoilDatabase` and never run by hand.
|
|
9
7
|
*/
|
|
10
8
|
import { requiredArgument } from "@mailwoman/core/scripting/arguments";
|
|
11
9
|
import { INGEST_CHUNK_FLAGS, runIngestChunkScript } from "@mailwoman/core/scripting/ingest-chunk-script";
|
|
12
10
|
import { createShapefileFeatureSource } from "#sdk/ingest";
|
|
13
|
-
import { ingestSoilChunk } from "#sdk/ingest
|
|
11
|
+
import { ingestSoilChunk } from "#sdk/ingest/chunk";
|
|
14
12
|
await runIngestChunkScript({
|
|
15
13
|
context: "soil ingest-chunk",
|
|
16
14
|
options: {
|
|
@@ -27,16 +25,19 @@ await runIngestChunkScript({
|
|
|
27
25
|
areaSymbol: requiredArgument("soil ingest-chunk", "area-symbol", values["area-symbol"]),
|
|
28
26
|
fidFrom: Number(requiredArgument("soil ingest-chunk", "fid-from", values["fid-from"])),
|
|
29
27
|
fidTo: Number(requiredArgument("soil ingest-chunk", "fid-to", values["fid-to"])),
|
|
30
|
-
// A range's own count is not knowable up front — `ogrinfo` reports the layer's total
|
|
31
|
-
//
|
|
28
|
+
// A range's own count is not knowable up front — `ogrinfo` reports the layer's total
|
|
29
|
+
// and no narrower count — so the chunk makes no assertion about its size
|
|
30
|
+
// and the parent checks the per-area sum against the shapefile's.
|
|
32
31
|
declaredFeatureCount: 0,
|
|
33
32
|
}),
|
|
34
33
|
indexResolution: chunk.indexResolution,
|
|
35
34
|
coverageResolution: chunk.coverageResolution,
|
|
36
|
-
// An empty string is an empty set
|
|
37
|
-
//
|
|
38
|
-
|
|
35
|
+
// An empty string is an empty set rather than "every map unit": a build
|
|
36
|
+
// where every map unit has soil mapping passes one.
|
|
37
|
+
// `"".split(",")` yields one empty element that has to be dropped
|
|
38
|
+
// rather than joined against as a mukey.
|
|
39
|
+
noMappingMukeys: new Set((values["no-mapping-mukeys"] ?? "").split(",").filter((mukey) => mukey.length)),
|
|
39
40
|
onProgress: chunk.onProgress,
|
|
40
41
|
}),
|
|
41
42
|
});
|
|
42
|
-
//# sourceMappingURL=
|
|
43
|
+
//# sourceMappingURL=worker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../../sdk/ingest/worker.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAA;AACtE,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,+CAA+C,CAAA;AAIxG,OAAO,EAAE,4BAA4B,EAAE,MAAM,aAAa,CAAA;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AAEnD,MAAM,oBAAoB,CAAC;IAC1B,OAAO,EAAE,mBAAmB;IAC5B,OAAO,EAAE;QACR,GAAG,kBAAkB;QACrB,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QAC7B,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QACjC,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QAC9B,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QAC5B,mBAAmB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;KACvC;IACD,GAAG,EAAE,KAAK,EAAE,QAAsC,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,CACpE,eAAe,CAAC,QAAQ,EAAE;QACzB,MAAM,EAAE,MAAM,4BAA4B,CAAC;YAC1C,aAAa,EAAE,gBAAgB,CAAC,mBAAmB,EAAE,WAAW,EAAE,MAAM,CAAC,SAAS,CAAC;YACnF,UAAU,EAAE,gBAAgB,CAAC,mBAAmB,EAAE,aAAa,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC;YACvF,OAAO,EAAE,MAAM,CAAC,gBAAgB,CAAC,mBAAmB,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;YACtF,KAAK,EAAE,MAAM,CAAC,gBAAgB,CAAC,mBAAmB,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;YAChF,qFAAqF;YACrF,yEAAyE;YACzE,kEAAkE;YAClE,oBAAoB,EAAE,CAAC;SACvB,CAAC;QACF,eAAe,EAAE,KAAK,CAAC,eAAe;QACtC,kBAAkB,EAAE,KAAK,CAAC,kBAAkB;QAC5C,wEAAwE;QACxE,oDAAoD;QACpD,kEAAkE;QAClE,yCAAyC;QACzC,eAAe,EAAE,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACxG,UAAU,EAAE,KAAK,CAAC,UAAU;KAC5B,CAAC;CACH,CAAC,CAAA"}
|
package/out/sdk/ingest.d.ts
CHANGED
|
@@ -2,47 +2,26 @@
|
|
|
2
2
|
* @copyright Sister Software
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
|
-
*
|
|
6
|
-
* Read a survey area's published shapefiles as a stream of WGS84 delineations, through ogr2ogr.
|
|
7
|
-
*
|
|
8
|
-
* OGR IS BUILD TOOLING, NEVER A SERVE DEPENDENCY (SCOPE invariant 6). It converts the authority's geometry
|
|
9
|
-
* into the structure the runtime probes, and nothing downstream of this module knows GDAL exists.
|
|
10
|
-
*
|
|
11
|
-
* THE SOURCE IS ALREADY IN WGS84, AND CHECKING IT IS STILL THE CHECK. Each `.prj` is an ESRI WKT reading
|
|
12
|
-
* `GEOGCS["GCS_WGS_1984",…]`, which GDAL resolves to EPSG:4326 — so no reprojection is needed before H3.
|
|
13
|
-
* The authority code is asserted anyway before a single feature is read, and the reprojected stream is
|
|
14
|
-
* asserted against the layer's own declared extent, which is the check that catches a coordinate-order
|
|
15
|
-
* mistake the projection check cannot see.
|
|
16
|
-
*
|
|
17
|
-
* THE DATUM GUARD RUNS EVEN THOUGH THE ANSWER IS THE IDENTITY, AND THAT IS THE POINT. PROJ substitutes a
|
|
18
|
-
* ballpark datum shift SILENTLY when the accurate grid is missing — measured on the flood layer at 3.4 m
|
|
19
|
-
* over an entire country, visible only as eight disagreements out of 59 against the authority's own
|
|
20
|
-
* service. For an EPSG:4326 source `projinfo` answers `Null geographic offset from WGS 84 to WGS 84, 0 m,
|
|
21
|
-
* World.` and the guard passes in one process. Skipping it on the reasoning that this source needs no
|
|
22
|
-
* shift is how the guard comes to be missing on the day a source arrives that does.
|
|
23
|
-
*
|
|
24
|
-
* THE ID IS THE SHAPEFILE'S OWN FID, AND IT HAS TO BE, because SSURGO publishes no per-delineation key:
|
|
25
|
-
* `MUKEY` names the MAP UNIT and one map unit has many delineations — `IA153` holds 17,966 delineations
|
|
26
|
-
* across 152 map units. So `area_id` is `<areasymbol>:<fid>`, which is stable across runs and is what makes
|
|
27
|
-
* a bounded chunk name the same features every time.
|
|
28
5
|
*/
|
|
29
6
|
import { type MultiPolygonRings } from "@mailwoman/spatial";
|
|
7
|
+
import { PathBuilder, type PathBuilderLike } from "path-ts";
|
|
30
8
|
/**
|
|
31
|
-
*
|
|
9
|
+
* Returns the path of the shapefile that holds a survey area's map-unit polygons.
|
|
32
10
|
*/
|
|
33
|
-
export declare function mapUnitShapefile(spatialDirectory:
|
|
11
|
+
export declare function mapUnitShapefile(spatialDirectory: PathBuilderLike, areaSymbol: string): string;
|
|
34
12
|
/**
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
13
|
+
* Returns the path of the shapefile holding a survey area's outline.
|
|
14
|
+
*
|
|
15
|
+
* The build takes the footprint from this outline because unrated map units such as `notcom` lie inside it.
|
|
16
|
+
* A union of rated polygons would leave them out.
|
|
38
17
|
*/
|
|
39
|
-
export declare function surveyAreaShapefile(spatialDirectory:
|
|
18
|
+
export declare function surveyAreaShapefile(spatialDirectory: PathBuilderLike, areaSymbol: string): PathBuilder;
|
|
40
19
|
/**
|
|
41
20
|
* One map-unit delineation, reprojected to WGS84.
|
|
42
21
|
*/
|
|
43
22
|
export interface SoilDelineation {
|
|
44
23
|
/**
|
|
45
|
-
* `<areasymbol>:<fid>`.
|
|
24
|
+
* The delineation's identifier, formed as `<areasymbol>:<fid>`.
|
|
46
25
|
*/
|
|
47
26
|
areaID: string;
|
|
48
27
|
mukey: string;
|
|
@@ -50,80 +29,89 @@ export interface SoilDelineation {
|
|
|
50
29
|
polygons: MultiPolygonRings;
|
|
51
30
|
}
|
|
52
31
|
/**
|
|
53
|
-
*
|
|
32
|
+
* The metadata that a shapefile declares, read before any feature.
|
|
54
33
|
*/
|
|
55
34
|
export interface SoilSourceIdentity {
|
|
56
35
|
epsg: number;
|
|
57
36
|
featureCount: number;
|
|
58
37
|
layer: string;
|
|
59
38
|
/**
|
|
60
|
-
* The layer's
|
|
39
|
+
* The layer's declared extent as `[minLon, minLat, maxLon, maxLat]`.
|
|
61
40
|
*/
|
|
62
41
|
bbox: readonly [number, number, number, number];
|
|
63
42
|
}
|
|
43
|
+
/**
|
|
44
|
+
* Options for reading one SSURGO map-unit shapefile.
|
|
45
|
+
*/
|
|
64
46
|
export interface SoilIngestOptions {
|
|
65
47
|
shapefilePath: string;
|
|
66
48
|
/**
|
|
67
|
-
*
|
|
49
|
+
* The layer inside the shapefile.
|
|
50
|
+
*
|
|
51
|
+
* The default is the file's base name.
|
|
52
|
+
* The ESRI driver reports that name.
|
|
68
53
|
*/
|
|
69
54
|
layer?: string;
|
|
70
55
|
/**
|
|
71
|
-
* The EPSG code the source must declare.
|
|
56
|
+
* The EPSG code that the source must declare.
|
|
57
|
+
* The default is the SSURGO source projection.
|
|
72
58
|
*/
|
|
73
59
|
expectEPSG?: number;
|
|
74
60
|
/**
|
|
75
|
-
*
|
|
61
|
+
* The inclusive range of shapefile FIDs to read.
|
|
62
|
+
* A build can use it to read one chunk.
|
|
76
63
|
*/
|
|
77
64
|
fidFrom?: number;
|
|
78
65
|
fidTo?: number;
|
|
79
66
|
/**
|
|
80
|
-
*
|
|
67
|
+
* The maximum number of features to read.
|
|
68
|
+
* Fixture and smoke runs use it.
|
|
81
69
|
*/
|
|
82
70
|
limit?: number;
|
|
83
71
|
}
|
|
84
72
|
/**
|
|
85
|
-
*
|
|
73
|
+
* Reads the projection, feature count, layer name and extent that a shapefile declares.
|
|
86
74
|
*
|
|
87
|
-
*
|
|
88
|
-
* reports no feature count.
|
|
75
|
+
* It throws when the shapefile declares a projection other than `expectEPSG`.
|
|
89
76
|
*/
|
|
90
77
|
export declare function readSoilSourceIdentity(options: SoilIngestOptions): Promise<SoilSourceIdentity>;
|
|
91
78
|
/**
|
|
92
|
-
*
|
|
79
|
+
* Streams a shapefile's map-unit delineations reprojected to WGS84.
|
|
93
80
|
*
|
|
94
|
-
*
|
|
95
|
-
*
|
|
81
|
+
* It throws when a feature lacks geometry or `mukey`.
|
|
82
|
+
* It also throws when a vertex falls outside the declared extent.
|
|
96
83
|
*
|
|
97
|
-
*
|
|
98
|
-
* the declared extent.
|
|
84
|
+
* The extent check catches swapped coordinate axes that the projection check misses.
|
|
99
85
|
*/
|
|
100
86
|
export declare function readSoilDelineations(options: SoilIngestOptions & {
|
|
101
87
|
bbox: readonly [number, number, number, number];
|
|
102
88
|
}): AsyncGenerator<SoilDelineation>;
|
|
103
89
|
/**
|
|
104
|
-
*
|
|
90
|
+
* Supplies a soil build with its delineations and the source's declared identity.
|
|
105
91
|
*
|
|
106
|
-
* The builder takes
|
|
107
|
-
*
|
|
108
|
-
* reduction, the coverage rows, the manifest and the seal.
|
|
92
|
+
* The builder takes this instead of a path so that fixtures can run the database
|
|
93
|
+
* build without network access or GDAL.
|
|
109
94
|
*/
|
|
110
95
|
export interface SoilFeatureSource {
|
|
111
96
|
areaSymbol: string;
|
|
112
97
|
/**
|
|
113
|
-
*
|
|
114
|
-
*
|
|
98
|
+
* The feature count that the source declares.
|
|
99
|
+
*
|
|
100
|
+
* The build throws when its streamed total differs from this count.
|
|
115
101
|
*/
|
|
116
102
|
declaredFeatureCount: number;
|
|
117
103
|
layer: string;
|
|
118
104
|
epsg: number;
|
|
119
105
|
/**
|
|
120
|
-
* A description of where
|
|
106
|
+
* A description of where the delineations came from, such as the shapefile path.
|
|
121
107
|
*/
|
|
122
108
|
origin: string;
|
|
123
109
|
delineations: () => AsyncIterable<SoilDelineation>;
|
|
124
110
|
}
|
|
125
111
|
/**
|
|
126
|
-
*
|
|
112
|
+
* Creates a {@link SoilFeatureSource} for one survey area's shapefile.
|
|
113
|
+
*
|
|
114
|
+
* It reads the shapefile's identity immediately and streams features when asked.
|
|
127
115
|
*/
|
|
128
116
|
export declare function createShapefileFeatureSource(options: SoilIngestOptions & {
|
|
129
117
|
areaSymbol: string;
|
package/out/sdk/ingest.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../../
|
|
1
|
+
{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../../sdk/ingest.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAiD,KAAK,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AAG1G,OAAO,EAAY,WAAW,EAAE,KAAK,eAAe,EAAE,MAAM,SAAS,CAAA;AAQrE;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,gBAAgB,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,CAE9F;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,gBAAgB,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,GAAG,WAAW,CAEtG;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;IACb,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,iBAAiB,CAAA;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC,IAAI,EAAE,MAAM,CAAA;IACZ,YAAY,EAAE,MAAM,CAAA;IACpB,KAAK,EAAE,MAAM,CAAA;IAEb;;OAEG;IACH,IAAI,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAA;CAC/C;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC,aAAa,EAAE,MAAM,CAAA;IAErB;;;;;OAKG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;IAEd;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAA;IAEnB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;IAEd;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;CACd;AAED;;;;GAIG;AACH,wBAAsB,sBAAsB,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAepG;AAsBD;;;;;;;GAOG;AACH,wBAAuB,oBAAoB,CAC1C,OAAO,EAAE,iBAAiB,GAAG;IAAE,IAAI,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,GAC9E,cAAc,CAAC,eAAe,CAAC,CAsBjC;AAkCD;;;;;GAKG;AACH,MAAM,WAAW,iBAAiB;IACjC,UAAU,EAAE,MAAM,CAAA;IAElB;;;;OAIG;IACH,oBAAoB,EAAE,MAAM,CAAA;IAC5B,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,EAAE,MAAM,CAAA;IAEZ;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,MAAM,aAAa,CAAC,eAAe,CAAC,CAAA;CAClD;AAED;;;;GAIG;AACH,wBAAsB,4BAA4B,CACjD,OAAO,EAAE,iBAAiB,GAAG;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,oBAAoB,CAAC,EAAE,MAAM,CAAA;CAAE,GAChF,OAAO,CAAC,iBAAiB,CAAC,CAgB5B"}
|