mailwoman 10.5.0 → 10.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/commands/corpus/build.tsx +14 -0
- package/cli/commands/corpus/download.tsx +33 -7
- package/cli/commands/corpus/fetch.tsx +1 -0
- package/cli/commands/corpus/overlay-manifest.tsx +7 -0
- package/cli/commands/corpus/slice/index.tsx +17 -2
- package/cli/commands/corpus/source-register.tsx +6 -0
- package/cli/commands/corpus/sub/venue/extract.tsx +4 -4
- package/cli/commands/corpus/upload.tsx +6 -5
- package/cli/commands/eval/pins.tsx +4 -4
- package/cli/commands/gazetteer/build/poi/coverage.tsx +2 -2
- package/cli/commands/gazetteer/build/poi/index.tsx +4 -4
- package/cli/commands/gazetteer/inspect/sync/index.tsx +42 -12
- package/cli/commands/gazetteer/pair-index.tsx +26 -49
- package/cli/commands/gazetteer/repos-sync.tsx +40 -28
- package/cli/commands/release/merge-admin.tsx +3 -3
- package/cli/doctor/checks.ts +20 -2
- package/cli/doctor/runner.ts +37 -1
- package/cli/kit/progress.ts +103 -0
- package/cli/kit.ts +1 -0
- package/lib/browser-runtime/classify.ts +4 -3
- package/lib/browser-runtime/resources.ts +2 -1
- package/lib/data/bom.ts +3 -0
- package/lib/data/bundles.ts +10 -3
- package/lib/data/inventory.ts +12 -0
- package/lib/data/obligations.ts +35 -0
- package/lib/fork-entity.ts +6 -0
- package/lib/geocode/layer-paths.ts +43 -0
- package/lib/geocode/osm-layers.ts +47 -0
- package/lib/geocode/parse-priors.ts +1 -1
- package/lib/geocode/result.ts +30 -3
- package/lib/geocode/session.ts +3 -9
- package/lib/plus-code-override.ts +1 -0
- package/man/mailwoman.1 +1 -1
- package/out/browser-runtime/classify.d.ts.map +1 -1
- package/out/browser-runtime/classify.js +4 -3
- package/out/browser-runtime/classify.js.map +1 -1
- package/out/browser-runtime/resources.d.ts +2 -1
- package/out/browser-runtime/resources.d.ts.map +1 -1
- package/out/browser-runtime/resources.js +2 -1
- package/out/browser-runtime/resources.js.map +1 -1
- package/out/cli/commands/corpus/build.d.ts +4 -0
- package/out/cli/commands/corpus/build.d.ts.map +1 -1
- package/out/cli/commands/corpus/build.js +8 -1
- package/out/cli/commands/corpus/build.js.map +1 -1
- package/out/cli/commands/corpus/download.d.ts.map +1 -1
- package/out/cli/commands/corpus/download.js +30 -6
- package/out/cli/commands/corpus/download.js.map +1 -1
- package/out/cli/commands/corpus/fetch.d.ts +1 -1
- package/out/cli/commands/corpus/fetch.d.ts.map +1 -1
- package/out/cli/commands/corpus/fetch.js +1 -0
- package/out/cli/commands/corpus/fetch.js.map +1 -1
- package/out/cli/commands/corpus/overlay-manifest.d.ts +4 -0
- package/out/cli/commands/corpus/overlay-manifest.d.ts.map +1 -1
- package/out/cli/commands/corpus/overlay-manifest.js +6 -0
- package/out/cli/commands/corpus/overlay-manifest.js.map +1 -1
- package/out/cli/commands/corpus/slice/index.d.ts +4 -0
- package/out/cli/commands/corpus/slice/index.d.ts.map +1 -1
- package/out/cli/commands/corpus/slice/index.js +12 -2
- package/out/cli/commands/corpus/slice/index.js.map +1 -1
- package/out/cli/commands/corpus/source-register.d.ts +5 -0
- package/out/cli/commands/corpus/source-register.d.ts.map +1 -1
- package/out/cli/commands/corpus/source-register.js +6 -0
- package/out/cli/commands/corpus/source-register.js.map +1 -1
- package/out/cli/commands/corpus/sub/venue/extract.js +4 -4
- package/out/cli/commands/corpus/sub/venue/extract.js.map +1 -1
- package/out/cli/commands/corpus/upload.d.ts.map +1 -1
- package/out/cli/commands/corpus/upload.js +2 -4
- package/out/cli/commands/corpus/upload.js.map +1 -1
- package/out/cli/commands/eval/pins.js +4 -4
- package/out/cli/commands/eval/pins.js.map +1 -1
- package/out/cli/commands/gazetteer/build/poi/coverage.js +2 -2
- package/out/cli/commands/gazetteer/build/poi/coverage.js.map +1 -1
- package/out/cli/commands/gazetteer/build/poi/index.js +4 -4
- package/out/cli/commands/gazetteer/build/poi/index.js.map +1 -1
- package/out/cli/commands/gazetteer/inspect/sync/index.d.ts.map +1 -1
- package/out/cli/commands/gazetteer/inspect/sync/index.js +30 -7
- package/out/cli/commands/gazetteer/inspect/sync/index.js.map +1 -1
- package/out/cli/commands/gazetteer/pair-index.d.ts.map +1 -1
- package/out/cli/commands/gazetteer/pair-index.js +23 -43
- package/out/cli/commands/gazetteer/pair-index.js.map +1 -1
- package/out/cli/commands/gazetteer/repos-sync.d.ts.map +1 -1
- package/out/cli/commands/gazetteer/repos-sync.js +22 -26
- package/out/cli/commands/gazetteer/repos-sync.js.map +1 -1
- package/out/cli/commands/release/merge-admin.js +3 -3
- package/out/cli/commands/release/merge-admin.js.map +1 -1
- package/out/cli/doctor/checks.d.ts +1 -1
- package/out/cli/doctor/checks.d.ts.map +1 -1
- package/out/cli/doctor/checks.js +17 -2
- package/out/cli/doctor/checks.js.map +1 -1
- package/out/cli/doctor/runner.d.ts +6 -1
- package/out/cli/doctor/runner.d.ts.map +1 -1
- package/out/cli/doctor/runner.js +24 -2
- package/out/cli/doctor/runner.js.map +1 -1
- package/out/cli/kit/progress.d.ts +55 -0
- package/out/cli/kit/progress.d.ts.map +1 -0
- package/out/cli/kit/progress.js +46 -0
- package/out/cli/kit/progress.js.map +1 -0
- package/out/cli/kit.d.ts +1 -0
- package/out/cli/kit.d.ts.map +1 -1
- package/out/cli/kit.js +1 -0
- package/out/cli/kit.js.map +1 -1
- package/out/data/bom.d.ts.map +1 -1
- package/out/data/bom.js +3 -0
- package/out/data/bom.js.map +1 -1
- package/out/data/bundles.d.ts.map +1 -1
- package/out/data/bundles.js +10 -3
- package/out/data/bundles.js.map +1 -1
- package/out/data/inventory.d.ts +12 -0
- package/out/data/inventory.d.ts.map +1 -1
- package/out/data/inventory.js.map +1 -1
- package/out/data/obligations.d.ts +12 -0
- package/out/data/obligations.d.ts.map +1 -1
- package/out/data/obligations.js +25 -0
- package/out/data/obligations.js.map +1 -1
- package/out/fork-entity.d.ts +4 -0
- package/out/fork-entity.d.ts.map +1 -1
- package/out/fork-entity.js +2 -0
- package/out/fork-entity.js.map +1 -1
- package/out/geocode/layer-paths.d.ts +25 -0
- package/out/geocode/layer-paths.d.ts.map +1 -1
- package/out/geocode/layer-paths.js +27 -0
- package/out/geocode/layer-paths.js.map +1 -1
- package/out/geocode/osm-layers.d.ts +27 -0
- package/out/geocode/osm-layers.d.ts.map +1 -0
- package/out/geocode/osm-layers.js +42 -0
- package/out/geocode/osm-layers.js.map +1 -0
- package/out/geocode/parse-priors.js +1 -1
- package/out/geocode/result.d.ts.map +1 -1
- package/out/geocode/result.js +15 -0
- package/out/geocode/result.js.map +1 -1
- package/out/geocode/session.d.ts.map +1 -1
- package/out/geocode/session.js +3 -8
- package/out/geocode/session.js.map +1 -1
- package/out/plus-code-override.d.ts.map +1 -1
- package/out/plus-code-override.js +1 -0
- package/out/plus-code-override.js.map +1 -1
- package/out/tools/coverage/core.js +2 -2
- package/out/tools/coverage/core.js.map +1 -1
- package/out/tools/coverage/jurisdictions.d.ts.map +1 -1
- package/out/tools/coverage/jurisdictions.js +4 -3
- package/out/tools/coverage/jurisdictions.js.map +1 -1
- package/out/tools/dev-tools/codex/postcode-formats.d.ts +12 -0
- package/out/tools/dev-tools/codex/postcode-formats.d.ts.map +1 -0
- package/out/tools/dev-tools/codex/postcode-formats.js +98 -0
- package/out/tools/dev-tools/codex/postcode-formats.js.map +1 -0
- package/out/tools/dev-tools/codex/street-orders.d.ts.map +1 -1
- package/out/tools/dev-tools/codex/street-orders.js +8 -5
- package/out/tools/dev-tools/codex/street-orders.js.map +1 -1
- package/out/tools/dev-tools/corpus/gb-os-open-golden.run.d.ts +24 -0
- package/out/tools/dev-tools/corpus/gb-os-open-golden.run.d.ts.map +1 -0
- package/out/tools/dev-tools/corpus/gb-os-open-golden.run.js +74 -0
- package/out/tools/dev-tools/corpus/gb-os-open-golden.run.js.map +1 -0
- package/out/tools/dev-tools/corpus/nad-skeletons.run.d.ts +16 -0
- package/out/tools/dev-tools/corpus/nad-skeletons.run.d.ts.map +1 -0
- package/out/tools/dev-tools/corpus/nad-skeletons.run.js +82 -0
- package/out/tools/dev-tools/corpus/nad-skeletons.run.js.map +1 -0
- package/out/tools/dev-tools/corpus/street/attestation.run.d.ts +6 -3
- package/out/tools/dev-tools/corpus/street/attestation.run.d.ts.map +1 -1
- package/out/tools/dev-tools/corpus/street/attestation.run.js +17 -19
- package/out/tools/dev-tools/corpus/street/attestation.run.js.map +1 -1
- package/out/tools/dev-tools/corpus/variety-sample.run.d.ts +17 -0
- package/out/tools/dev-tools/corpus/variety-sample.run.d.ts.map +1 -0
- package/out/tools/dev-tools/corpus/variety-sample.run.js +77 -0
- package/out/tools/dev-tools/corpus/variety-sample.run.js.map +1 -0
- package/out/tools/dev-tools/gb-fragment-board.run.d.ts +4 -2
- package/out/tools/dev-tools/gb-fragment-board.run.d.ts.map +1 -1
- package/out/tools/dev-tools/gb-fragment-board.run.js +6 -4
- package/out/tools/dev-tools/gb-fragment-board.run.js.map +1 -1
- package/out/tools/dev-tools/heldout-geography-eval.run.js +9 -9
- package/out/tools/dev-tools/heldout-geography-eval.run.js.map +1 -1
- package/out/tools/dev-tools/lint-mdx-angles.js +2 -2
- package/out/tools/dev-tools/lint-mdx-angles.js.map +1 -1
- package/out/tools/dev-tools/probe/gb-anchor-fire.run.d.ts +1 -1
- package/out/tools/dev-tools/probe/gb-anchor-fire.run.js +2 -2
- package/out/tools/dev-tools/probe/gb-anchor-fire.run.js.map +1 -1
- package/out/tools/dev-tools/score/anchor-v2-boards.run.js +5 -2
- package/out/tools/dev-tools/score/anchor-v2-boards.run.js.map +1 -1
- package/out/tools/eval-harness/external-arenas.d.ts +1 -1
- package/out/tools/eval-harness/external-arenas.d.ts.map +1 -1
- package/out/tools/eval-harness/external-arenas.js +29 -9
- package/out/tools/eval-harness/external-arenas.js.map +1 -1
- package/out/tools/eval-harness/gauntlet/harness.js +2 -2
- package/out/tools/eval-harness/gauntlet/harness.js.map +1 -1
- package/out/tools/eval-harness/heldout/per-country.d.ts +63 -0
- package/out/tools/eval-harness/heldout/per-country.d.ts.map +1 -0
- package/out/tools/eval-harness/heldout/per-country.js +104 -0
- package/out/tools/eval-harness/heldout/per-country.js.map +1 -0
- package/out/tools/eval-harness/heldout/per-country.run.d.ts +13 -0
- package/out/tools/eval-harness/heldout/per-country.run.d.ts.map +1 -0
- package/out/tools/eval-harness/heldout/per-country.run.js +50 -0
- package/out/tools/eval-harness/heldout/per-country.run.js.map +1 -0
- package/out/tools/gazetteer-pipeline/postcode/geonames/tail.js +1 -1
- package/out/tools/gazetteer-pipeline/postcode/ni/osm/database.d.ts +5 -2
- package/out/tools/gazetteer-pipeline/postcode/ni/osm/database.d.ts.map +1 -1
- package/out/tools/gazetteer-pipeline/postcode/ni/osm/database.js +5 -2
- package/out/tools/gazetteer-pipeline/postcode/ni/osm/database.js.map +1 -1
- package/out/tools/gazetteer-pipeline/repos/audit.js +2 -2
- package/out/tools/gazetteer-pipeline/repos/audit.js.map +1 -1
- package/out/tools/gazetteer-pipeline/repos/sync.d.ts +18 -1
- package/out/tools/gazetteer-pipeline/repos/sync.d.ts.map +1 -1
- package/out/tools/gazetteer-pipeline/repos/sync.js +58 -32
- package/out/tools/gazetteer-pipeline/repos/sync.js.map +1 -1
- package/out/tools/gazetteer-pipeline/stamp-manifest.d.ts.map +1 -1
- package/out/tools/gazetteer-pipeline/stamp-manifest.js +2 -6
- package/out/tools/gazetteer-pipeline/stamp-manifest.js.map +1 -1
- package/out/tools/generate-clients.d.ts.map +1 -1
- package/out/tools/generate-clients.js +15 -7
- package/out/tools/generate-clients.js.map +1 -1
- package/out/tools/release-tools/publish/demo-assets.js +2 -2
- package/out/tools/release-tools/publish/demo-assets.js.map +1 -1
- package/out/tools/release-tools/publish/hf.d.ts.map +1 -1
- package/out/tools/release-tools/publish/hf.js +21 -18
- package/out/tools/release-tools/publish/hf.js.map +1 -1
- package/out/tools/tiles/publish.d.ts.map +1 -1
- package/out/tools/tiles/publish.js +2 -2
- package/out/tools/tiles/publish.js.map +1 -1
- package/package.json +42 -39
- package/tools/coverage/core.ts +2 -2
- package/tools/coverage/jurisdictions.ts +5 -3
- package/tools/dev-tools/codex/postcode-formats.ts +119 -0
- package/tools/dev-tools/codex/street-orders.ts +8 -5
- package/tools/dev-tools/corpus/gb-os-open-golden.run.ts +101 -0
- package/tools/dev-tools/corpus/nad-skeletons.run.ts +100 -0
- package/tools/dev-tools/corpus/street/attestation.run.ts +19 -21
- package/tools/dev-tools/corpus/variety-sample.run.ts +88 -0
- package/tools/dev-tools/gb-fragment-board.run.ts +6 -4
- package/tools/dev-tools/heldout-geography-eval.run.ts +13 -10
- package/tools/dev-tools/lint-mdx-angles.ts +2 -2
- package/tools/dev-tools/probe/gb-anchor-fire.run.ts +2 -2
- package/tools/dev-tools/score/anchor-v2-boards.run.ts +7 -2
- package/tools/eval-harness/conformance/punctuation.jsonl +1 -1
- package/tools/eval-harness/external-arenas.ts +29 -12
- package/tools/eval-harness/gauntlet/harness.ts +2 -2
- package/tools/eval-harness/heldout/per-country.run.ts +64 -0
- package/tools/eval-harness/heldout/per-country.ts +160 -0
- package/tools/eval-harness/invariance/suite.jsonl +16 -9
- package/tools/gazetteer-pipeline/postcode/geonames/tail.ts +1 -1
- package/tools/gazetteer-pipeline/postcode/ni/osm/database.ts +5 -2
- package/tools/gazetteer-pipeline/repos/audit.ts +2 -2
- package/tools/gazetteer-pipeline/repos/sync.ts +79 -32
- package/tools/gazetteer-pipeline/stamp-manifest.ts +2 -6
- package/tools/generate-clients.ts +16 -7
- package/tools/release-tools/publish/demo-assets.ts +2 -2
- package/tools/release-tools/publish/hf.ts +21 -19
- package/tools/tiles/publish.ts +2 -2
- package/lib/process.ts +0 -36
- package/out/process.d.ts +0 -15
- package/out/process.d.ts.map +0 -1
- package/out/process.js +0 -27
- package/out/process.js.map +0 -1
|
@@ -112,6 +112,12 @@ export const spec = {
|
|
|
112
112
|
type: "string",
|
|
113
113
|
description: "Comma-separated license prefixes to refuse, e.g. ODbL,CC-BY-SA",
|
|
114
114
|
},
|
|
115
|
+
"unshippable-reason": {
|
|
116
|
+
type: "string",
|
|
117
|
+
description:
|
|
118
|
+
"Admit sources whose register rows lack an elected decision permitting transform and train, for a " +
|
|
119
|
+
"local experiment. States why; the corpus manifest records it as not shippable",
|
|
120
|
+
},
|
|
115
121
|
},
|
|
116
122
|
} as const satisfies CommandSpec
|
|
117
123
|
|
|
@@ -201,6 +207,7 @@ const CorpusBuild: CommandComponent<typeof spec> = ({ options }) => {
|
|
|
201
207
|
profile: options.profile as BuildProfile,
|
|
202
208
|
licensePolicy: options.licensePolicy as LicensePolicy,
|
|
203
209
|
excludeLicenses: options.excludeLicenses ? compileLicenseExcludes(options.excludeLicenses) : undefined,
|
|
210
|
+
unshippableReason: options.unshippableReason,
|
|
204
211
|
onProgress: (name, message) => setStage({ name, message }),
|
|
205
212
|
})
|
|
206
213
|
|
|
@@ -214,6 +221,7 @@ const CorpusBuild: CommandComponent<typeof spec> = ({ options }) => {
|
|
|
214
221
|
unresolvedLicenseRows: m.admitted_unresolved_license_rows,
|
|
215
222
|
refusedByEligibility: m.excluded_by_eligibility,
|
|
216
223
|
ineligibleSources: Object.keys(m.ineligible_sources),
|
|
224
|
+
unshippableSources: [...new Set(m.source_rights.gaps.map((gap) => gap.source))],
|
|
217
225
|
}
|
|
218
226
|
})
|
|
219
227
|
|
|
@@ -243,6 +251,12 @@ const CorpusBuild: CommandComponent<typeof spec> = ({ options }) => {
|
|
|
243
251
|
. MANIFEST.json records each reason under `ineligible_sources`.
|
|
244
252
|
</Text>
|
|
245
253
|
) : null}
|
|
254
|
+
{done.unshippableSources.length ? (
|
|
255
|
+
<Text color="red">
|
|
256
|
+
NOT SHIPPABLE: --unshippable-reason admitted {done.unshippableSources.join(", ")}, whose register rows lack
|
|
257
|
+
a permission for transform and train. MANIFEST.json records each gap under `source_rights`.
|
|
258
|
+
</Text>
|
|
259
|
+
) : null}
|
|
246
260
|
<Text dimColor>{options.out}</Text>
|
|
247
261
|
</Box>
|
|
248
262
|
)
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
15
|
import { dataRootPath } from "@mailwoman/core/data-root"
|
|
16
|
+
import { runProcess } from "@mailwoman/core/process"
|
|
16
17
|
import { Box, Text } from "ink"
|
|
17
18
|
import { useState } from "react"
|
|
18
19
|
|
|
@@ -52,17 +53,26 @@ const CorpusDownload: CommandComponent<typeof spec> = ({ options }) => {
|
|
|
52
53
|
}
|
|
53
54
|
|
|
54
55
|
useCommandTask(async () => {
|
|
55
|
-
const { $ } = await import("zx")
|
|
56
|
-
|
|
57
56
|
const rcloneBase = `:s3:${options.bucket}`
|
|
58
|
-
const dryFlag = options.dryRun ? "--dry-run" :
|
|
57
|
+
const dryFlag = options.dryRun ? ["--dry-run"] : []
|
|
59
58
|
const out = options.outDir
|
|
60
59
|
|
|
61
60
|
// Step 0: Download v0.3.0 corpus
|
|
62
61
|
updateStep(0, { status: "running" })
|
|
63
62
|
|
|
64
63
|
try {
|
|
65
|
-
await
|
|
64
|
+
await runProcess("rclone", [
|
|
65
|
+
"sync",
|
|
66
|
+
`${rcloneBase}/corpus/v0.3.0/`,
|
|
67
|
+
`${out}/corpus/versioned/v0.3.0/corpus-v0.3.0/`,
|
|
68
|
+
"--progress",
|
|
69
|
+
"--transfers",
|
|
70
|
+
"8",
|
|
71
|
+
"--checkers",
|
|
72
|
+
"16",
|
|
73
|
+
...dryFlag,
|
|
74
|
+
])
|
|
75
|
+
|
|
66
76
|
updateStep(0, { status: "done" })
|
|
67
77
|
} catch (error: unknown) {
|
|
68
78
|
const e = error as Record<string, unknown>
|
|
@@ -75,7 +85,16 @@ const CorpusDownload: CommandComponent<typeof spec> = ({ options }) => {
|
|
|
75
85
|
updateStep(1, { status: "running" })
|
|
76
86
|
|
|
77
87
|
try {
|
|
78
|
-
await
|
|
88
|
+
await runProcess("rclone", [
|
|
89
|
+
"sync",
|
|
90
|
+
`${rcloneBase}/corpus/v0.4.0/`,
|
|
91
|
+
`${out}/corpus/versioned/v0.4.0/corpus-v0.4.0/`,
|
|
92
|
+
"--progress",
|
|
93
|
+
"--transfers",
|
|
94
|
+
"4",
|
|
95
|
+
...dryFlag,
|
|
96
|
+
])
|
|
97
|
+
|
|
79
98
|
updateStep(1, { status: "done" })
|
|
80
99
|
} catch (error: unknown) {
|
|
81
100
|
const e = error as Record<string, unknown>
|
|
@@ -88,7 +107,14 @@ const CorpusDownload: CommandComponent<typeof spec> = ({ options }) => {
|
|
|
88
107
|
updateStep(2, { status: "running" })
|
|
89
108
|
|
|
90
109
|
try {
|
|
91
|
-
await
|
|
110
|
+
await runProcess("rclone", [
|
|
111
|
+
"sync",
|
|
112
|
+
`${rcloneBase}/models/tokenizer/`,
|
|
113
|
+
`${out}/models/tokenizer/`,
|
|
114
|
+
"--progress",
|
|
115
|
+
...dryFlag,
|
|
116
|
+
])
|
|
117
|
+
|
|
92
118
|
updateStep(2, { status: "done" })
|
|
93
119
|
} catch (error: unknown) {
|
|
94
120
|
const e = error as Record<string, unknown>
|
|
@@ -101,7 +127,7 @@ const CorpusDownload: CommandComponent<typeof spec> = ({ options }) => {
|
|
|
101
127
|
updateStep(3, { status: "running" })
|
|
102
128
|
|
|
103
129
|
try {
|
|
104
|
-
await
|
|
130
|
+
await runProcess("rclone", ["sync", `${rcloneBase}/corpus-python/`, "./corpus-python/", "--progress", ...dryFlag])
|
|
105
131
|
updateStep(3, { status: "done" })
|
|
106
132
|
} catch (error: unknown) {
|
|
107
133
|
const e = error as Record<string, unknown>
|
|
@@ -46,6 +46,12 @@ export const spec = {
|
|
|
46
46
|
"Split per parquet, comma-separated. Unset puts every file in train. val or test is accepted only for a file `corpus split-slice` wrote, whose name ends .val.parquet or .test.parquet.",
|
|
47
47
|
},
|
|
48
48
|
note: { type: "string", required: true, description: "Manifest note" },
|
|
49
|
+
"unshippable-reason": {
|
|
50
|
+
type: "string",
|
|
51
|
+
description:
|
|
52
|
+
"Admit sources whose register rows lack an elected decision permitting transform and train, for a " +
|
|
53
|
+
"local experiment. States why; the manifest records the corpus as not shippable",
|
|
54
|
+
},
|
|
49
55
|
"replace-source": {
|
|
50
56
|
type: "string",
|
|
51
57
|
description:
|
|
@@ -92,6 +98,7 @@ const Cmd: CommandComponent<typeof spec> = ({ options }) => {
|
|
|
92
98
|
})),
|
|
93
99
|
note: options.note,
|
|
94
100
|
replaceSources: options.replaceSource ? extractDelimited(options.replaceSource) : [],
|
|
101
|
+
unshippableReason: options.unshippableReason,
|
|
95
102
|
})
|
|
96
103
|
|
|
97
104
|
return "done"
|
|
@@ -8,7 +8,7 @@ import { openWriteStream } from "@mailwoman/core/fs/streams"
|
|
|
8
8
|
import { movePath, removePathIfPresent, writeLocalJSONFile } from "@mailwoman/core/fs/writers"
|
|
9
9
|
import { CommandError } from "@mailwoman/core/scripting/command"
|
|
10
10
|
import type { RecipeOptions } from "@mailwoman/corpus"
|
|
11
|
-
import { createRecipeLineWriter } from "@mailwoman/corpus/recipes/scaffold"
|
|
11
|
+
import { createRecipeLineWriter, admittingRecipeLineWriter } from "@mailwoman/corpus/recipes/scaffold"
|
|
12
12
|
import { Box, Text } from "ink"
|
|
13
13
|
|
|
14
14
|
import {
|
|
@@ -65,6 +65,9 @@ export const spec = {
|
|
|
65
65
|
"negative-fraction": stringOption("confound-negative fraction"),
|
|
66
66
|
"venue-names": stringOption("venue, sub-venue: Overture place-names parquet"),
|
|
67
67
|
"venue-region-fraction": stringOption("venue: share of locality rows that also carry the region"),
|
|
68
|
+
"unshippable-reason": stringOption(
|
|
69
|
+
"Admit a source whose register rows lack a permission for transform and train, for a local experiment; the invocation record marks the output not shippable"
|
|
70
|
+
),
|
|
68
71
|
},
|
|
69
72
|
} as const satisfies CommandSpec
|
|
70
73
|
|
|
@@ -141,7 +144,18 @@ const CorpusRecipeRun: CommandComponent<typeof spec> = ({ options, args }) => {
|
|
|
141
144
|
const stagedPath = `${options.out}.partial`
|
|
142
145
|
const stream = openWriteStream(stagedPath, { encoding: "utf8" })
|
|
143
146
|
|
|
144
|
-
const
|
|
147
|
+
const { openSourceRightsAdmission } = await import("@mailwoman/corpus/source-register/corpus-inputs")
|
|
148
|
+
|
|
149
|
+
const rights = await openSourceRightsAdmission({
|
|
150
|
+
context: `recipe ${name}`,
|
|
151
|
+
unshippableReason: options.unshippableReason ?? null,
|
|
152
|
+
}).catch((error: unknown) => {
|
|
153
|
+
stream.destroy()
|
|
154
|
+
|
|
155
|
+
throw error
|
|
156
|
+
})
|
|
157
|
+
|
|
158
|
+
const write = admittingRecipeLineWriter(createRecipeLineWriter(stream), rights)
|
|
145
159
|
|
|
146
160
|
let stats: Awaited<ReturnType<typeof recipe.run>>
|
|
147
161
|
|
|
@@ -173,6 +187,7 @@ const CorpusRecipeRun: CommandComponent<typeof spec> = ({ options, args }) => {
|
|
|
173
187
|
read: stats.read ?? null,
|
|
174
188
|
ranAt: new Date().toISOString(),
|
|
175
189
|
options: Object.fromEntries(Object.entries(opts).filter(([, value]) => value !== undefined)),
|
|
190
|
+
source_rights: rights.record(),
|
|
176
191
|
},
|
|
177
192
|
`${options.out}.invocation.json`
|
|
178
193
|
)
|
|
@@ -33,6 +33,11 @@ export const spec = {
|
|
|
33
33
|
default: "packages/corpus/data/source-resolutions.json",
|
|
34
34
|
description: "Source fields a review resolved, applied over the research pass's placeholders",
|
|
35
35
|
},
|
|
36
|
+
additions: {
|
|
37
|
+
type: "string",
|
|
38
|
+
default: "packages/corpus/data/source-additions.json",
|
|
39
|
+
description: "Source rows a later pass added, in the research CSV's columns, read after its rows",
|
|
40
|
+
},
|
|
36
41
|
"register-version": { type: "string", default: "0.1.0", description: "Register version" },
|
|
37
42
|
"authored-at": { type: "string", description: "Date the research pass was taken, YYYY-MM-DD" },
|
|
38
43
|
"source-version": { type: "string", description: "Research document the pass was written against" },
|
|
@@ -55,6 +60,7 @@ const CorpusSourceRegister: CommandComponent<typeof spec> = ({ options }) => {
|
|
|
55
60
|
outPath: options.out,
|
|
56
61
|
decisionsPath: options.decisions,
|
|
57
62
|
resolutionsPath: options.resolutions,
|
|
63
|
+
additionsPath: options.additions,
|
|
58
64
|
version: options.registerVersion,
|
|
59
65
|
authoredAt: options.authoredAt,
|
|
60
66
|
sourceVersion: options.sourceVersion,
|
|
@@ -40,12 +40,12 @@ export const spec = {
|
|
|
40
40
|
|
|
41
41
|
const CorpusSubVenueExtract: CommandComponent<typeof spec> = ({ options }) => {
|
|
42
42
|
const state = useCommandTask(async () => {
|
|
43
|
-
// @mailwoman/osm is
|
|
44
|
-
//
|
|
43
|
+
// @mailwoman/osm is an optional peer that an operator installs to opt in to OpenStreetMap data, so a
|
|
44
|
+
// static import would break every install without it. Lazy-load it and say how to install it when absent.
|
|
45
45
|
const { writeSubVenueJSONL } = await import("@mailwoman/osm/sdk/extract/subvenue").catch(() => {
|
|
46
46
|
throw new Error(
|
|
47
|
-
"corpus sub-venue-extract requires @mailwoman/osm,
|
|
48
|
-
"
|
|
47
|
+
"corpus sub-venue-extract requires @mailwoman/osm, an optional package — install it with " +
|
|
48
|
+
"`npm install @mailwoman/osm` (its README states the OpenStreetMap license terms)."
|
|
49
49
|
)
|
|
50
50
|
})
|
|
51
51
|
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
|
|
10
10
|
import { dataRootPath } from "@mailwoman/core/data-root"
|
|
11
11
|
import { pathExists } from "@mailwoman/core/fs/readers/stat"
|
|
12
|
+
import { runProcess } from "@mailwoman/core/process"
|
|
12
13
|
import { extractDelimited } from "@mailwoman/core/scripting/arguments"
|
|
13
14
|
import { childEnv } from "@mailwoman/core/scripting/utils"
|
|
14
15
|
import type { BuildCorpusManifest } from "@mailwoman/corpus/build"
|
|
@@ -235,8 +236,6 @@ const CorpusUpload: CommandComponent<typeof spec> = ({ options }) => {
|
|
|
235
236
|
const [steps, setSteps] = useState<Step[]>([])
|
|
236
237
|
|
|
237
238
|
const state = useCommandTask(async () => {
|
|
238
|
-
const { $ } = await import("zx")
|
|
239
|
-
|
|
240
239
|
const corpusRoot = PathBuilder.from(options.corpusDir ?? dataRootPath("corpus", "versioned"))
|
|
241
240
|
|
|
242
241
|
const directories = extractDelimited(options.corpusDirectory)
|
|
@@ -350,9 +349,11 @@ const CorpusUpload: CommandComponent<typeof spec> = ({ options }) => {
|
|
|
350
349
|
update(index, { status: "running" })
|
|
351
350
|
|
|
352
351
|
try {
|
|
353
|
-
await
|
|
354
|
-
|
|
355
|
-
|
|
352
|
+
await runProcess(
|
|
353
|
+
"rclone",
|
|
354
|
+
["sync", job.source.toString(), job.dest, ...job.extra, ...dry, ...R2_RETRIES, "--stats-one-line"],
|
|
355
|
+
{ env }
|
|
356
|
+
)
|
|
356
357
|
|
|
357
358
|
update(index, { status: "done", detail: options.dryRun ? "would sync" : "synced" })
|
|
358
359
|
} catch (error: unknown) {
|
|
@@ -17,8 +17,8 @@
|
|
|
17
17
|
import { gitHead } from "@mailwoman/core/git"
|
|
18
18
|
import { parseJSONStrict, stringifyJSON } from "@mailwoman/core/json"
|
|
19
19
|
import { repoRootPath } from "@mailwoman/core/paths"
|
|
20
|
+
import { runProcess } from "@mailwoman/core/process"
|
|
20
21
|
import { Box, Text } from "ink"
|
|
21
|
-
import { $ } from "zx"
|
|
22
22
|
|
|
23
23
|
import { type CommandSpec, CommandTaskResult, type CommandComponent, useCommandTask } from "#cli-kit"
|
|
24
24
|
|
|
@@ -68,17 +68,17 @@ async function reportStalePins(drift: string, testPath: string): Promise<string>
|
|
|
68
68
|
`Run \`mailwoman eval pins --update\`, commit ${testPath}, and this issue closes on the next green audit.`
|
|
69
69
|
|
|
70
70
|
const search = `in:title ${stringifyJSON(STALE_PIN_ISSUE_TITLE)}`
|
|
71
|
-
const issueList = await
|
|
71
|
+
const issueList = await runProcess("gh", ["issue", "list", "--state", "open", "--search", search, "--json", "number"])
|
|
72
72
|
const existing = parseJSONStrict<Array<{ number: number }>>(issueList.stdout)
|
|
73
73
|
const first = existing[0]
|
|
74
74
|
|
|
75
75
|
if (first) {
|
|
76
|
-
await
|
|
76
|
+
await runProcess("gh", ["issue", "comment", String(first.number), "--body", body])
|
|
77
77
|
|
|
78
78
|
return `updated issue #${first.number}`
|
|
79
79
|
}
|
|
80
80
|
|
|
81
|
-
await
|
|
81
|
+
await runProcess("gh", ["issue", "create", "--title", STALE_PIN_ISSUE_TITLE, "--body", body])
|
|
82
82
|
|
|
83
83
|
return "opened the audit issue"
|
|
84
84
|
}
|
|
@@ -117,8 +117,8 @@ const GazetteerBuildPOICoverage: CommandComponent<typeof spec> = ({ options }) =
|
|
|
117
117
|
const out = options.out ?? poiDatabasePath(`poi-coverage-${options.category}-${slugify(region)}.db`)
|
|
118
118
|
const buildSHA = resolveBuildSHA(repoRootPath())
|
|
119
119
|
|
|
120
|
-
// Dynamic import is required: @mailwoman/osm is
|
|
121
|
-
// import breaks the published CLI on
|
|
120
|
+
// Dynamic import is required: @mailwoman/osm is an optional peer, so a top-level
|
|
121
|
+
// import breaks the published CLI on an install without it.
|
|
122
122
|
const [{ extractOSMBoundary }, { extractOSMPOIs, tagRuleFromOSMTag }] = await Promise.all([
|
|
123
123
|
import("@mailwoman/osm/sdk/extract/boundary"),
|
|
124
124
|
import("@mailwoman/osm/sdk/extract/poi"),
|
|
@@ -115,10 +115,10 @@ const GazetteerBuildPOI: CommandComponent<typeof spec> = ({ options }) => {
|
|
|
115
115
|
// twice, once here to derive coverage, once as buildPOIDatabase's `rows` injection point.
|
|
116
116
|
const rows: POISourceRow[] = []
|
|
117
117
|
|
|
118
|
-
//
|
|
119
|
-
//
|
|
120
|
-
//
|
|
121
|
-
//
|
|
118
|
+
// Dynamic import, required: @mailwoman/osm is an optional peer that an operator installs
|
|
119
|
+
// to opt in to OpenStreetMap data, and this dependency belongs only on the selected build
|
|
120
|
+
// path, so a top-level import breaks the whole published CLI on an install without it.
|
|
121
|
+
// The smoke test's clean-install leg is what catches it.
|
|
122
122
|
// The osm source branch is build-local by design.
|
|
123
123
|
// It may only resolve its SDK when a run selects it.
|
|
124
124
|
const { extractOSMPOIs } = await import("@mailwoman/osm/sdk/extract/poi")
|
|
@@ -21,6 +21,9 @@ import { ProgressBar } from "@inkjs/ui"
|
|
|
21
21
|
import type { RepositorySource, SynchronizeAction } from "@mailwoman/core"
|
|
22
22
|
import { wofReposPath } from "@mailwoman/core/data-root"
|
|
23
23
|
import { ByteFormatter } from "@mailwoman/core/fs/formatters"
|
|
24
|
+
import { GitClient } from "@mailwoman/core/git"
|
|
25
|
+
import { parseJSONStrict } from "@mailwoman/core/json"
|
|
26
|
+
import { runFile } from "@mailwoman/core/process"
|
|
24
27
|
import { formatQuantity } from "@mailwoman/core/resources/locale"
|
|
25
28
|
import { CommandError } from "@mailwoman/core/scripting/command"
|
|
26
29
|
import { Box, Text } from "ink"
|
|
@@ -33,7 +36,9 @@ import {
|
|
|
33
36
|
type CommandSpec,
|
|
34
37
|
CommandTaskResult,
|
|
35
38
|
type CommandComponent,
|
|
39
|
+
ProgressLines,
|
|
36
40
|
useCommandTask,
|
|
41
|
+
useProgressFeed,
|
|
37
42
|
} from "#cli-kit"
|
|
38
43
|
import {
|
|
39
44
|
assertDestinationNotARepoName,
|
|
@@ -92,12 +97,25 @@ interface SyncPlan {
|
|
|
92
97
|
* failure surfaces as an opaque spawn error from deep inside the task.
|
|
93
98
|
*/
|
|
94
99
|
async function discoverRepos(): Promise<DiscoveredRepo[]> {
|
|
95
|
-
const { $ } = await import("zx")
|
|
96
|
-
|
|
97
100
|
try {
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
+
const { stdout } = await runFile("gh", [
|
|
102
|
+
"repo",
|
|
103
|
+
"list",
|
|
104
|
+
WOF_REPO_OWNER,
|
|
105
|
+
"--no-archived",
|
|
106
|
+
"--limit",
|
|
107
|
+
"1000",
|
|
108
|
+
"--json",
|
|
109
|
+
"name",
|
|
110
|
+
"--json",
|
|
111
|
+
"url",
|
|
112
|
+
"--json",
|
|
113
|
+
"diskUsage",
|
|
114
|
+
])
|
|
115
|
+
|
|
116
|
+
return parseJSONStrict<Array<{ name: string; url: string; diskUsage: number }>>(stdout).map(
|
|
117
|
+
({ name, url, diskUsage }) => ({ name, url, diskUsageKB: diskUsage })
|
|
118
|
+
)
|
|
101
119
|
} catch (error) {
|
|
102
120
|
throw new CommandError(
|
|
103
121
|
`Could not list ${WOF_REPO_OWNER} through \`gh\`. Install the GitHub CLI and run \`gh auth login\`.`,
|
|
@@ -110,6 +128,7 @@ const WOFSync: CommandComponent<typeof spec, [string?]> = ({ options, args }) =>
|
|
|
110
128
|
const [plan, setPlan] = useState<SyncPlan>()
|
|
111
129
|
const [active, setActive] = useState<readonly string[]>([])
|
|
112
130
|
const [doneCount, setDoneCount] = useState(0)
|
|
131
|
+
const progress = useProgressFeed()
|
|
113
132
|
|
|
114
133
|
const state = useCommandTask(
|
|
115
134
|
async () => {
|
|
@@ -197,11 +216,18 @@ const WOFSync: CommandComponent<typeof spec, [string?]> = ({ options, args }) =>
|
|
|
197
216
|
let action: SynchronizeAction | null = null
|
|
198
217
|
let failure: string | null = null
|
|
199
218
|
|
|
219
|
+
await using client = new GitClient({ cwd: destination })
|
|
220
|
+
|
|
221
|
+
const unwatch = progress.watch(client, source.name)
|
|
222
|
+
|
|
200
223
|
try {
|
|
201
|
-
action = await synchronizeRepo(source, destination)
|
|
224
|
+
action = await synchronizeRepo(source, destination, client)
|
|
202
225
|
} catch (error) {
|
|
203
226
|
ok = false
|
|
204
227
|
failure = error instanceof Error ? error.message : String(error)
|
|
228
|
+
} finally {
|
|
229
|
+
unwatch()
|
|
230
|
+
progress.clear(source.name)
|
|
205
231
|
}
|
|
206
232
|
|
|
207
233
|
const elapsed = `${((performance.now() - startedAt) / 1000).toFixed(1)}s`
|
|
@@ -260,12 +286,16 @@ const WOFSync: CommandComponent<typeof spec, [string?]> = ({ options, args }) =>
|
|
|
260
286
|
{ByteFormatter.formatSI(plan.selection.totalDiskUsageKB * 1024)}
|
|
261
287
|
</Text>
|
|
262
288
|
|
|
263
|
-
{active
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
289
|
+
{active
|
|
290
|
+
.filter((name) => !progress.entries.has(name))
|
|
291
|
+
.map((name) => (
|
|
292
|
+
<Text key={name} color="cyan">
|
|
293
|
+
{" "}
|
|
294
|
+
⠋ {name}
|
|
295
|
+
</Text>
|
|
296
|
+
))}
|
|
297
|
+
|
|
298
|
+
<ProgressLines entries={progress.entries} />
|
|
269
299
|
|
|
270
300
|
<Box paddingX={1}>
|
|
271
301
|
<ProgressBar value={percentage} />
|
|
@@ -20,22 +20,6 @@ import {
|
|
|
20
20
|
useCommandTask,
|
|
21
21
|
} from "#cli-kit"
|
|
22
22
|
|
|
23
|
-
/**
|
|
24
|
-
* The expected distinct-pair count for a GB build using only the PPD CSV.
|
|
25
|
-
*
|
|
26
|
-
* A mismatch means the fold changed, so the artifact needs investigation before use.
|
|
27
|
-
*/
|
|
28
|
-
const EXPECTED_GB_PAIR_COUNT = 19_209
|
|
29
|
-
|
|
30
|
-
/**
|
|
31
|
-
* The GB pair count before folding, printed for context only.
|
|
32
|
-
*
|
|
33
|
-
* It exceeds {@link EXPECTED_GB_PAIR_COUNT} because the fold merges punctuation
|
|
34
|
-
* variants such as "St Helens" and "St.
|
|
35
|
-
* Helens".
|
|
36
|
-
*/
|
|
37
|
-
const RUNG3_PRE_FOLD_CENSUS_LINE_COUNT = 19_431
|
|
38
|
-
|
|
39
23
|
/**
|
|
40
24
|
* The expected distinct-pair count for a US build.
|
|
41
25
|
*
|
|
@@ -52,9 +36,9 @@ const EXPECTED_US_PAIR_COUNT = 47_878
|
|
|
52
36
|
*/
|
|
53
37
|
const PROBE_PAIRS_BY_COUNTRY: Readonly<Record<string, ReadonlyArray<readonly [city: string, district: string]>>> = {
|
|
54
38
|
gb: [
|
|
55
|
-
["Fishburn", "Stockton-on-Tees"],
|
|
56
39
|
["Shoreditch", "London"],
|
|
57
|
-
["
|
|
40
|
+
["Didsbury", "Manchester"],
|
|
41
|
+
["Henbury", "Bristol"],
|
|
58
42
|
],
|
|
59
43
|
nz: [
|
|
60
44
|
["Plimmerton", "Porirua"],
|
|
@@ -89,10 +73,9 @@ const PROBE_PAIRS_BY_COUNTRY: Readonly<Record<string, ReadonlyArray<readonly [ci
|
|
|
89
73
|
}
|
|
90
74
|
|
|
91
75
|
/**
|
|
92
|
-
* The parent tag for each source that does not report one itself: the `--source`
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
* line may override its default with `parentTag`.
|
|
76
|
+
* The parent tag for each source that does not report one itself: the `--source` CSV's
|
|
77
|
+
* `district` column is the town above the suburb in NZ, while `--borough-db` and `--ban-dir`
|
|
78
|
+
* supply a tag per pair and a `--pairs-jsonl` line may override its default with `parentTag`.
|
|
96
79
|
*/
|
|
97
80
|
const SOURCE_PARENT_TAGS = {
|
|
98
81
|
registerDistrict: "locality",
|
|
@@ -137,7 +120,6 @@ export const spec = {
|
|
|
137
120
|
|
|
138
121
|
const GazetteerPairIndex: CommandComponent<typeof spec> = ({ options }) => {
|
|
139
122
|
const state = useCommandTask(async () => {
|
|
140
|
-
const { dataRootPath } = await import("@mailwoman/core/data-root")
|
|
141
123
|
const { md5File, md5Hex } = await import("@mailwoman/core/utils")
|
|
142
124
|
const { normalizeFSTToken } = await import("@mailwoman/neural/fst-prior")
|
|
143
125
|
const { PairIndexResolver, serializePairIndex } = await import("@mailwoman/neural/pair")
|
|
@@ -148,10 +130,19 @@ const GazetteerPairIndex: CommandComponent<typeof spec> = ({ options }) => {
|
|
|
148
130
|
|
|
149
131
|
const country = options.country.toLowerCase()
|
|
150
132
|
|
|
151
|
-
//
|
|
152
|
-
//
|
|
153
|
-
|
|
154
|
-
|
|
133
|
+
// country-branch: the only Great Britain tuple CSV this command has read is HM Land Registry
|
|
134
|
+
// Price Paid Data, whose address fields packages/corpus/data/license-decisions.json
|
|
135
|
+
// refuses (`unchecked-access-free-gb-gb-property-building-1`).
|
|
136
|
+
// Great Britain builds from the secondary sources alone, and a per-source
|
|
137
|
+
// license check would replace this branch.
|
|
138
|
+
if (options.source && country === "gb") {
|
|
139
|
+
throw new Error(
|
|
140
|
+
`pair-index: a Great Britain build reads no tuple CSV (--source ${options.source}). ` +
|
|
141
|
+
`The register refuses Price Paid Data's address fields; pass --borough-db and --pairs-jsonl instead.`
|
|
142
|
+
)
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
const sourcePath = options.source
|
|
155
146
|
|
|
156
147
|
if (sourcePath && !(await pathExists(sourcePath))) {
|
|
157
148
|
throw new Error(`pair-index: source CSV not found: ${sourcePath}`)
|
|
@@ -159,7 +150,7 @@ const GazetteerPairIndex: CommandComponent<typeof spec> = ({ options }) => {
|
|
|
159
150
|
|
|
160
151
|
if (!sourcePath && !options.boroughDB && !options.pairsJSONL && !options.banDir) {
|
|
161
152
|
throw new Error(
|
|
162
|
-
`pair-index: country "${country}" has no
|
|
153
|
+
`pair-index: country "${country}" has no source — pass --source, --borough-db, --pairs-jsonl or --ban-dir, ` +
|
|
163
154
|
`or the build would write an empty index.`
|
|
164
155
|
)
|
|
165
156
|
}
|
|
@@ -192,7 +183,6 @@ const GazetteerPairIndex: CommandComponent<typeof spec> = ({ options }) => {
|
|
|
192
183
|
}
|
|
193
184
|
}
|
|
194
185
|
|
|
195
|
-
let secondaryPairsAdded = 0
|
|
196
186
|
let banFiles: readonly PathBuilderLike[] = []
|
|
197
187
|
|
|
198
188
|
if (options.boroughDB) {
|
|
@@ -202,9 +192,7 @@ const GazetteerPairIndex: CommandComponent<typeof spec> = ({ options }) => {
|
|
|
202
192
|
builder.addRow(pair.child, pair.parent, pair.parentTag)
|
|
203
193
|
}
|
|
204
194
|
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
console.error(`pair-index: +${secondaryPairsAdded} distinct borough pairs (WOF admin DB)`)
|
|
195
|
+
console.error(`pair-index: +${builder.distinctCount - before} distinct borough pairs (WOF admin DB)`)
|
|
208
196
|
}
|
|
209
197
|
|
|
210
198
|
if (options.banDir) {
|
|
@@ -217,8 +205,6 @@ const GazetteerPairIndex: CommandComponent<typeof spec> = ({ options }) => {
|
|
|
217
205
|
builder.addRow(pair.child, pair.parent, pair.parentTag)
|
|
218
206
|
}
|
|
219
207
|
|
|
220
|
-
secondaryPairsAdded += builder.distinctCount - before
|
|
221
|
-
|
|
222
208
|
console.error(
|
|
223
209
|
`pair-index: +${builder.distinctCount - before} distinct lieu-dit pairs ` +
|
|
224
210
|
`(${filesRead} départements, ${rowsWithLieuDit.toLocaleString()} rows with a clean lieu-dit)`
|
|
@@ -237,8 +223,6 @@ const GazetteerPairIndex: CommandComponent<typeof spec> = ({ options }) => {
|
|
|
237
223
|
builder.addRow(pair.child, pair.parent, pair.parentTag ?? SOURCE_PARENT_TAGS.secondaryPairsJSONL)
|
|
238
224
|
}
|
|
239
225
|
|
|
240
|
-
secondaryPairsAdded += builder.distinctCount - before
|
|
241
|
-
|
|
242
226
|
console.error(
|
|
243
227
|
`pair-index: +${builder.distinctCount - before} distinct secondary pairs (${path.split("/").pop()})`
|
|
244
228
|
)
|
|
@@ -317,22 +301,15 @@ const GazetteerPairIndex: CommandComponent<typeof spec> = ({ options }) => {
|
|
|
317
301
|
]
|
|
318
302
|
|
|
319
303
|
const preHoldoutCount = built.entries.length
|
|
320
|
-
const preFoldSuffix = ` (pre-fold rung-3 census: ${RUNG3_PRE_FOLD_CENSUS_LINE_COUNT.toLocaleString()} lines)`
|
|
321
304
|
|
|
322
305
|
const checkLine =
|
|
323
|
-
country === "
|
|
306
|
+
country === "us"
|
|
324
307
|
? options.holdoutFraction > 0
|
|
325
|
-
? `(cross-check skipped under --holdout-fraction; ${preHoldoutCount.toLocaleString()} distinct pairs before holdout, expects ${
|
|
326
|
-
: preHoldoutCount ===
|
|
327
|
-
? `CROSS-CHECK PASS: ${preHoldoutCount.toLocaleString()} distinct pairs (
|
|
328
|
-
: `CROSS-CHECK BLOCKED: ${preHoldoutCount.toLocaleString()} distinct pairs !=
|
|
329
|
-
:
|
|
330
|
-
? options.holdoutFraction > 0
|
|
331
|
-
? `(cross-check skipped under --holdout-fraction; ${preHoldoutCount.toLocaleString()} distinct pairs before holdout, expects ${EXPECTED_US_PAIR_COUNT.toLocaleString()})`
|
|
332
|
-
: preHoldoutCount === EXPECTED_US_PAIR_COUNT
|
|
333
|
-
? `CROSS-CHECK PASS: ${preHoldoutCount.toLocaleString()} distinct pairs (all WOF-sourced; no postal register)`
|
|
334
|
-
: `CROSS-CHECK BLOCKED: ${preHoldoutCount.toLocaleString()} distinct pairs != ${EXPECTED_US_PAIR_COUNT.toLocaleString()} — a WOF snapshot refresh moves this number, so re-anchor the constant DELIBERATELY (with the diff inspected) rather than trusting the artifact`
|
|
335
|
-
: `(cross-check only registered for gb/us; ${preHoldoutCount.toLocaleString()} distinct pairs)`
|
|
308
|
+
? `(cross-check skipped under --holdout-fraction; ${preHoldoutCount.toLocaleString()} distinct pairs before holdout, expects ${EXPECTED_US_PAIR_COUNT.toLocaleString()})`
|
|
309
|
+
: preHoldoutCount === EXPECTED_US_PAIR_COUNT
|
|
310
|
+
? `CROSS-CHECK PASS: ${preHoldoutCount.toLocaleString()} distinct pairs (all WOF-sourced; no postal register)`
|
|
311
|
+
: `CROSS-CHECK BLOCKED: ${preHoldoutCount.toLocaleString()} distinct pairs != ${EXPECTED_US_PAIR_COUNT.toLocaleString()} — a WOF snapshot refresh moves this number, so re-anchor the constant DELIBERATELY (with the diff inspected) rather than trusting the artifact`
|
|
312
|
+
: `(cross-check only registered for us; ${preHoldoutCount.toLocaleString()} distinct pairs)`
|
|
336
313
|
|
|
337
314
|
const holdoutLine =
|
|
338
315
|
options.holdoutFraction > 0
|