@mailwoman/corpus 9.4.0 → 10.1.0

package/out/source-register/types.js

@@ -0,0 +1,263 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   This module defines the address-source register's types. The register has a jurisdiction table and a source table.
+ */
+/**
+ * The research state of a jurisdiction's premise-address backbone.
+ *
+ * The letters match the global address corpus specification and are separate
+ * from locale tiers and layer tiers.
+ */
+export const BackboneState = {
+    /**
+     * A verified open nationwide, or effectively nationwide, official premise-address
+     * source, subject to its own terms.
+     */
+    Verified: "A",
+    /**
+     * A strong open official source that is federated, partial, establishment-only
+     * or territory-specific, or whose national completeness has not been measured.
+     */
+    VerifiedPartial: "A~",
+    /**
+     * An official address system exists.
+     *
+     * Bulk access, openness, coverage or redistribution rights are incomplete.
+     */
+    Restricted: "B",
+    /**
+     * The research pass found no permissive nationwide premise corpus.
+     * One may still exist.
+     */
+    Unverified: "C",
+    /**
+     * A jurisdiction where routing and hand-verified grammar outweigh an ordinary street hierarchy.
+     */
+    Exceptional: "D",
+};
+/**
+ * What the research pass did for a jurisdiction.
+ */
+export const JurisdictionResearchState = {
+    /**
+     * Research resolved at least one national source, so the jurisdiction must have source rows.
+     */
+    Seeded: "seeded",
+    /**
+     * No ordinary national registry exists, as in an uninhabited territory or a treaty area.
+     *
+     * The row records the reason.
+     */
+    Exception: "exception",
+    /**
+     * Research has neither resolved a source nor ruled one out.
+     *
+     * The row records the reason.
+     */
+    Unexamined: "unexamined",
+};
+/**
+ * How far this repository's review of one source has progressed.
+ */
+export const SourceStatus = {
+    /**
+     * Research confirmed that the register exists and is the national authority.
+     *
+     * Its address fields, bulk access and terms have no review record.
+     */
+    VerifiedAuthority: "verified-authority",
+    /**
+     * A row retained from the earlier memo without a recheck.
+     * These rows lack a publisher and a URL.
+     */
+    RetainedOriginal: "retained-original",
+    /**
+     * Research confirmed that a bulk corpus is reachable.
+     *
+     * This state by itself does not make a source eligible for ingest.
+     */
+    VerifiedCorpus: "verified-corpus",
+    /**
+     * A bulk corpus is reachable but the examined copy is no longer updated.
+     *
+     * The row points to the national portal instead.
+     * This state differs from `VerifiedCorpus` so a filter for reachable corpora excludes it.
+     */
+    VerifiedCorpusStale: "verified-corpus-stale",
+};
+/**
+ * The closed list of sectors that source rows may use.
+ *
+ * The audit rejects a row with any other sector.
+ */
+export const REGISTER_SECTORS = [
+    "business-legal",
+    "business-register",
+    "charity-nonprofit",
+    "education",
+    "environment-industry",
+    "finance",
+    "food-pharma",
+    "health",
+    "mixed-regulatory",
+    "procurement-grants",
+    "professional-licensing",
+    "property-building",
+    "public-administration",
+    "telecom",
+    "transport",
+];
+/**
+ * Whether a source provides coordinates; `Unresolved` differs from `Absent`.
+ */
+export const SourceGeometry = {
+    Present: "present",
+    /**
+     * A reviewer checked and found no coordinates.
+     */
+    Absent: "absent",
+    /**
+     * A reviewer checked and found coordinates on some records.
+     */
+    Partial: "partial",
+    /**
+     * The source has no coordinate review record.
+     */
+    Unresolved: "unresolved",
+};
+/**
+ * The research pass that added a row to the table.
+ *
+ * A `retained-original` row has no other provenance.
+ */
+export const ResearchPass = {
+    WebResearch: "2026-09-18-web-research",
+    OriginalMemo: "original-memo",
+    /**
+     * The rights review that read each candidate publisher's own terms.
+     *
+     * The pass rewrites a row when reading the publisher settles something the earlier pass
+     * recorded wrongly, such as a row naming two publishers or a URL that now answers 404.
+     */
+    RightsReview: "2026-09-30-rights-review",
+    /**
+     * The decoder-exposure survey, which recorded the authoritative address source of
+     * each jurisdiction the exposure arc admits through OpenStreetMap.
+     * It located each authority and did not read its terms.
+     */
+    DecoderExposureSurvey: "2026-10-03-decoder-exposure",
+};
+/**
+ * The review state of a source's terms.
+ *
+ * An `unchecked` source cannot enter a training build because its recorded label describes only access cost.
+ */
+export const LicenseReviewState = {
+    /**
+     * The publisher's terms have no review record.
+     *
+     * The research pass's license text is kept verbatim in
+     * {@linkcode UncheckedLicense.publisherStatement} and grants no license.
+     */
+    Unchecked: "unchecked",
+    /**
+     * A reviewer read the terms, elected one grant, retrieved a copy and recorded the reason.
+     */
+    Elected: "elected",
+    /**
+     * A reviewer read the terms and rejected the source.
+     * The row records the reason.
+     */
+    Refused: "refused",
+};
+/**
+ * The acts this repository performs on a source, in pipeline order.
+ *
+ * A grant can permit some and stay silent on others, so license decisions record a permission per operation.
+ */
+export const SourceOperation = {
+    /**
+     * Bulk copy of the published dataset into `$MAILWOMAN_DATA_ROOT`.
+     */
+    Fetch: "fetch",
+    /**
+     * Postal-address extraction from the copied file.
+     */
+    Extract: "extract",
+    /**
+     * Transformation of extracted rows into a token and tag corpus.
+     */
+    Transform: "transform",
+    /**
+     * Model fitting on the transformed rows.
+     */
+    Train: "train",
+    /**
+     * Redistribution of raw or transformed rows as a database, lexicon or index.
+     */
+    RedistributeData: "redistribute-data",
+    /**
+     * Redistribution of weights trained on the rows.
+     */
+    RedistributeModel: "redistribute-model",
+    /**
+     * Commercial sublicensing of the result.
+     */
+    CommercialSublicense: "commercial-sublicense",
+};
+/**
+ * What a grant permits for one operation.
+ *
+ * `unreviewed` means the terms lack an operation-specific review.
+ * Eligibility checks treat it as blocking.
+ * The state stays distinct from `refused`.
+ */
+export const OperationPermission = {
+    Permitted: "permitted",
+    Refused: "refused",
+    Unreviewed: "unreviewed",
+};
+/**
+ * The legal basis of a permission.
+ *
+ * Most permissions rest on a publisher grant.
+ * The other values mark permissions with a different basis.
+ */
+export const PermissionBasis = {
+    PublisherGrant: "publisher-grant",
+    StatutoryException: "statutory-exception",
+    OtherDocumented: "other-documented",
+};
+/**
+ * What a review found about personal data in one publication, a question separate from licensing.
+ *
+ * `Present` blocks ingest.
+ * A source without a review is also blocked.
+ */
+export const PersonalDataReading = {
+    /**
+     * A review found no record about an identified or identifiable natural person.
+     */
+    Absent: "absent",
+    /**
+     * A review found such records.
+     * Analysis remains incomplete.
+     */
+    Present: "present",
+    /**
+     * A review found such records. {@link PersonalDataReview.record} points to the completed analysis.
+     */
+    Assessed: "assessed",
+};
+/**
+ * Source fields that no row resolves yet.
+ *
+ * The audit checks both directions.
+ * A listed field must be absent from every row.
+ * An unlisted field must appear on at least one row.
+ */
+export const UNRESOLVED_FIELDS = ["addressRoles", "upstreamLineage", "coverage", "personalDataReview"];
+//# sourceMappingURL=types.js.map

package/out/source-register/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../lib/source-register/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAMH;;;;;GAKG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG;IAC5B;;;OAGG;IACH,QAAQ,EAAE,GAAG;IACb;;;OAGG;IACH,eAAe,EAAE,IAAI;IACrB;;;;OAIG;IACH,UAAU,EAAE,GAAG;IACf;;;OAGG;IACH,UAAU,EAAE,GAAG;IACf;;OAEG;IACH,WAAW,EAAE,GAAG;CACP,CAAA;AAOV;;GAEG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG;IACxC;;OAEG;IACH,MAAM,EAAE,QAAQ;IAChB;;;;OAIG;IACH,SAAS,EAAE,WAAW;IACtB;;;;OAIG;IACH,UAAU,EAAE,YAAY;CACf,CAAA;AAOV;;GAEG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG;IAC3B;;;;OAIG;IACH,iBAAiB,EAAE,oBAAoB;IACvC;;;OAGG;IACH,gBAAgB,EAAE,mBAAmB;IACrC;;;;OAIG;IACH,cAAc,EAAE,iBAAiB;IACjC;;;;;OAKG;IACH,mBAAmB,EAAE,uBAAuB;CACnC,CAAA;AAOV;;;;GAIG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC/B,gBAAgB;IAChB,mBAAmB;IACnB,mBAAmB;IACnB,WAAW;IACX,sBAAsB;IACtB,SAAS;IACT,aAAa;IACb,QAAQ;IACR,kBAAkB;IAClB,oBAAoB;IACpB,wBAAwB;IACxB,mBAAmB;IACnB,uBAAuB;IACvB,SAAS;IACT,WAAW;CACF,CAAA;AAOV;;GAEG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG;IAC7B,OAAO,EAAE,SAAS;IAClB;;OAEG;IACH,MAAM,EAAE,QAAQ;IAChB;;OAEG;IACH,OAAO,EAAE,SAAS;IAClB;;OAEG;IACH,UAAU,EAAE,YAAY;CACf,CAAA;AAOV;;;;GAIG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG;IAC3B,WAAW,EAAE,yBAAyB;IACtC,YAAY,EAAE,eAAe;IAC7B;;;;;OAKG;IACH,YAAY,EAAE,0BAA0B;IACxC;;;;OAIG;IACH,qBAAqB,EAAE,6BAA6B;CAC3C,CAAA;AAOV;;;;GAIG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG;IACjC;;;;;OAKG;IACH,SAAS,EAAE,WAAW;IACtB;;OAEG;IACH,OAAO,EAAE,SAAS;IAClB;;;OAGG;IACH,OAAO,EAAE,SAAS;CACT,CAAA;AAuBV;;;;GAIG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG;IAC9B;;OAEG;IACH,KAAK,EAAE,OAAO;IACd;;OAEG;IACH,OAAO,EAAE,SAAS;IAClB;;OAEG;IACH,SAAS,EAAE,WAAW;IACtB;;OAEG;IACH,KAAK,EAAE,OAAO;IACd;;OAEG;IACH,gBAAgB,EAAE,mBAAmB;IACrC;;OAEG;IACH,iBAAiB,EAAE,oBAAoB;IACvC;;OAEG;IACH,oBAAoB,EAAE,uBAAuB;CACpC,CAAA;AAOV;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG;IAClC,SAAS,EAAE,WAAW;IACtB,OAAO,EAAE,SAAS;IAClB,UAAU,EAAE,YAAY;CACf,CAAA;AAOV;;;;;GAKG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG;IAC9B,cAAc,EAAE,iBAAiB;IACjC,kBAAkB,EAAE,qBAAqB;IACzC,eAAe,EAAE,kBAAkB;CAC1B,CAAA;AAsHV;;;;;GAKG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG;IAClC;;OAEG;IACH,MAAM,EAAE,QAAQ;IAChB;;;OAGG;IACH,OAAO,EAAE,SAAS;IAClB;;OAEG;IACH,QAAQ,EAAE,UAAU;CACX,CAAA;AAmGV;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,cAAc,EAAE,iBAAiB,EAAE,UAAU,EAAE,oBAAoB,CAAU,CAAA"}

package/out/source-register.d.ts

@@ -0,0 +1,19 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   The address-source register — which jurisdictions exist, what research has resolved for each and what is known
+ *   about every source's terms.
+ *
+ *   It is a backlog made checkable. Almost every source row is refused on several conditions at once, so ask
+ *   {@linkcode ingestEligibilityProblems} for the reasons rather than reading a status as permission.
+ */
+export { addressSourceRegisterPath, applyLicenseDecisions, auditAddressSourceRegister, electedLicenseLabel, INGEST_OPERATIONS, ingestEligibilityProblems, MODEL_RELEASE_OPERATIONS, permissionFor, readAddressSourceRegister, registerContentDigest, } from "#source-register/register";
+export { BackboneState, JurisdictionResearchState, LicenseReviewState, OperationPermission, PermissionBasis, PersonalDataReading, REGISTER_SECTORS, ResearchPass, SourceGeometry, SourceOperation, SourceStatus, UNRESOLVED_FIELDS, } from "#source-register/types";
+export { auditTrainingManifest, freezeTrainingManifest, sourcesNotPermitting, trainingManifestDigest, } from "#source-register/training-manifest";
+export type { TrainingManifest, TrainingSourceRecord } from "#source-register/training-manifest";
+export { deriveEffectiveTrainingManifest, DrawEvidence, effectiveManifestDigest, epochMixtureAuditPath, ExclusionReason, provenanceDisagreement, provenanceRefusals, readConfigView, requiredAttributions, } from "#source-register/effective-manifest";
+export type { EffectiveConfigView, EffectiveSourceRecord, EffectiveTrainingManifest, EpochMixtureAudit, LabelSetContract, RequiredAttribution, } from "#source-register/effective-manifest";
+export type * from "#source-register/types";
+//# sourceMappingURL=source-register.d.ts.map

package/out/source-register.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"source-register.d.ts","sourceRoot":"","sources":["../lib/source-register.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EACN,yBAAyB,EACzB,qBAAqB,EACrB,0BAA0B,EAC1B,mBAAmB,EACnB,iBAAiB,EACjB,yBAAyB,EACzB,wBAAwB,EACxB,aAAa,EACb,yBAAyB,EACzB,qBAAqB,GACrB,MAAM,2BAA2B,CAAA;AAElC,OAAO,EACN,aAAa,EACb,yBAAyB,EACzB,kBAAkB,EAClB,mBAAmB,EACnB,eAAe,EACf,mBAAmB,EACnB,gBAAgB,EAChB,YAAY,EACZ,cAAc,EACd,eAAe,EACf,YAAY,EACZ,iBAAiB,GACjB,MAAM,wBAAwB,CAAA;AAE/B,OAAO,EACN,qBAAqB,EACrB,sBAAsB,EACtB,oBAAoB,EACpB,sBAAsB,GACtB,MAAM,oCAAoC,CAAA;AAE3C,YAAY,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,oCAAoC,CAAA;AAEhG,OAAO,EACN,+BAA+B,EAC/B,YAAY,EACZ,uBAAuB,EACvB,qBAAqB,EACrB,eAAe,EACf,sBAAsB,EACtB,kBAAkB,EAClB,cAAc,EACd,oBAAoB,GACpB,MAAM,qCAAqC,CAAA;AAE5C,YAAY,EACX,mBAAmB,EACnB,qBAAqB,EACrB,yBAAyB,EACzB,iBAAiB,EACjB,gBAAgB,EAChB,mBAAmB,GACnB,MAAM,qCAAqC,CAAA;AAE5C,mBAAmB,wBAAwB,CAAA"}

package/out/source-register.js

@@ -0,0 +1,16 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   The address-source register — which jurisdictions exist, what research has resolved for each and what is known
+ *   about every source's terms.
+ *
+ *   It is a backlog made checkable. Almost every source row is refused on several conditions at once, so ask
+ *   {@linkcode ingestEligibilityProblems} for the reasons rather than reading a status as permission.
+ */
+export { addressSourceRegisterPath, applyLicenseDecisions, auditAddressSourceRegister, electedLicenseLabel, INGEST_OPERATIONS, ingestEligibilityProblems, MODEL_RELEASE_OPERATIONS, permissionFor, readAddressSourceRegister, registerContentDigest, } from "#source-register/register";
+export { BackboneState, JurisdictionResearchState, LicenseReviewState, OperationPermission, PermissionBasis, PersonalDataReading, REGISTER_SECTORS, ResearchPass, SourceGeometry, SourceOperation, SourceStatus, UNRESOLVED_FIELDS, } from "#source-register/types";
+export { auditTrainingManifest, freezeTrainingManifest, sourcesNotPermitting, trainingManifestDigest, } from "#source-register/training-manifest";
+export { deriveEffectiveTrainingManifest, DrawEvidence, effectiveManifestDigest, epochMixtureAuditPath, ExclusionReason, provenanceDisagreement, provenanceRefusals, readConfigView, requiredAttributions, } from "#source-register/effective-manifest";
+//# sourceMappingURL=source-register.js.map

package/out/source-register.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"source-register.js","sourceRoot":"","sources":["../lib/source-register.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EACN,yBAAyB,EACzB,qBAAqB,EACrB,0BAA0B,EAC1B,mBAAmB,EACnB,iBAAiB,EACjB,yBAAyB,EACzB,wBAAwB,EACxB,aAAa,EACb,yBAAyB,EACzB,qBAAqB,GACrB,MAAM,2BAA2B,CAAA;AAElC,OAAO,EACN,aAAa,EACb,yBAAyB,EACzB,kBAAkB,EAClB,mBAAmB,EACnB,eAAe,EACf,mBAAmB,EACnB,gBAAgB,EAChB,YAAY,EACZ,cAAc,EACd,eAAe,EACf,YAAY,EACZ,iBAAiB,GACjB,MAAM,wBAAwB,CAAA;AAE/B,OAAO,EACN,qBAAqB,EACrB,sBAAsB,EACtB,oBAAoB,EACpB,sBAAsB,GACtB,MAAM,oCAAoC,CAAA;AAI3C,OAAO,EACN,+BAA+B,EAC/B,YAAY,EACZ,uBAAuB,EACvB,qBAAqB,EACrB,eAAe,EACf,sBAAsB,EACtB,kBAAkB,EAClB,cAAc,EACd,oBAAoB,GACpB,MAAM,qCAAqC,CAAA"}

package/out/south-asia/recipes/register.d.ts

@@ -0,0 +1,54 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `pk-register` and `bd-register` recipes, the typed forms of Pakistan and Bangladesh the country
+ *   templates do not render, over the OpenStreetMap corpus jsonl the `osm` adapter reads. The
+ *   adapter's own rows use the template shape (`4 38th Street, DHA Phase 6, Karachi`) and
+ *   (`24 Road 104, Dhaka - 1207`). These two represent the register:
+ *
+ *   - the house line, `House 4, Street 25, F-7/2, Islamabad` and `House 34, Road 4, Sector 9, Uttara,
+ *     Dhaka 1230`, with `House` in front of the number (untagged, the same rule as Singapore's `Blk`),
+ *     the numbered street, the scheme or sector as dependent locality, then the city
+ *   - the plain line with the trailing postcode and no dash, `58 Kalabagan 1st Ln, Dhaka 1205`, which
+ *     is what a person types where the template writes `Dhaka - 1205`
+ *
+ *   Islamabad's sector codes (`F-7/2`, `G-10/3`) are the one synthesized value. The register contains
+ *   them on a handful of rows, mostly inside `addr:city` (`F7/2 Islamabad`), so a row whose city is
+ *   Islamabad and whose scheme is unknown draws one from the capital's grid. Every other value is the
+ *   row's own. The components are the `osm` adapter's mapping (`componentsForOSMRow`), so the two
+ *   surfaces cannot disagree about what a field means.
+ *
+ *   ODbL: the rows inherit OpenStreetMap's share-alike license. A corpus build run under
+ *   `LicensePolicy.ShareAlikeFree` refuses them.
+ *
+ *   Run: mailwoman corpus slice pk-register --input <osm-pk.corpus.jsonl> --count N --seed S
+ */
+import { type CorpusRecipe } from "#recipes/scaffold";
+import { type CanonicalRow } from "#types";
+export type SouthAsiaRegister = "house" | "plain";
+export interface SouthAsiaRendering {
+    raw: string;
+    components: Record<string, string>;
+    register: SouthAsiaRegister;
+}
+/**
+ * Draw one Islamabad sector code, `F-7/2`.
+ */
+export declare function drawIslamabadSector(random: () => number): string;
+/**
+ * Render one register line from the adapter's components.
+ *
+ * @returns Null when the row has no house number or no locality: both registers need the two.
+ */
+export declare function renderSouthAsiaRegister(components: CanonicalRow["components"], random: () => number, register?: SouthAsiaRegister): SouthAsiaRendering | null;
+/**
+ * Pakistan: `House 4, Street 25, F-7/2, Islamabad` and `B230 Street 5, Block 3, Karachi`.
+ */
+export declare const pkRegisterRecipe: CorpusRecipe;
+/**
+ * Bangladesh: `House 34, Road 4, Sector 9, Uttara, Dhaka 1230` and `58 Kalabagan 1st Ln, Dhaka 1205`.
+ */
+export declare const bdRegisterRecipe: CorpusRecipe;
+//# sourceMappingURL=register.d.ts.map

package/out/south-asia/recipes/register.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"register.d.ts","sourceRoot":"","sources":["../../../lib/south-asia/recipes/register.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAMH,OAAO,EAAiB,KAAK,YAAY,EAAkD,MAAM,mBAAmB,CAAA;AAGpH,OAAO,EAAE,KAAK,YAAY,EAAiB,MAAM,QAAQ,CAAA;AAsBzD,MAAM,MAAM,iBAAiB,GAAG,OAAO,GAAG,OAAO,CAAA;AAEjD,MAAM,WAAW,kBAAkB;IAClC,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAClC,QAAQ,EAAE,iBAAiB,CAAA;CAC3B;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,MAAM,GAAG,MAAM,CAMhE;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CACtC,UAAU,EAAE,YAAY,CAAC,YAAY,CAAC,EACtC,MAAM,EAAE,MAAM,MAAM,EACpB,QAAQ,CAAC,EAAE,iBAAiB,GAC1B,kBAAkB,GAAG,IAAI,CAgC3B;AAiGD;;GAEG;AACH,eAAO,MAAM,gBAAgB,EAAE,YAK9B,CAAA;AAED;;GAEG;AACH,eAAO,MAAM,gBAAgB,EAAE,YAK9B,CAAA"}

package/out/south-asia/recipes/register.js

@@ -0,0 +1,174 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `pk-register` and `bd-register` recipes, the typed forms of Pakistan and Bangladesh the country
+ *   templates do not render, over the OpenStreetMap corpus jsonl the `osm` adapter reads. The
+ *   adapter's own rows use the template shape (`4 38th Street, DHA Phase 6, Karachi`) and
+ *   (`24 Road 104, Dhaka - 1207`). These two represent the register:
+ *
+ *   - the house line, `House 4, Street 25, F-7/2, Islamabad` and `House 34, Road 4, Sector 9, Uttara,
+ *     Dhaka 1230`, with `House` in front of the number (untagged, the same rule as Singapore's `Blk`),
+ *     the numbered street, the scheme or sector as dependent locality, then the city
+ *   - the plain line with the trailing postcode and no dash, `58 Kalabagan 1st Ln, Dhaka 1205`, which
+ *     is what a person types where the template writes `Dhaka - 1205`
+ *
+ *   Islamabad's sector codes (`F-7/2`, `G-10/3`) are the one synthesized value. The register contains
+ *   them on a handful of rows, mostly inside `addr:city` (`F7/2 Islamabad`), so a row whose city is
+ *   Islamabad and whose scheme is unknown draws one from the capital's grid. Every other value is the
+ *   row's own. The components are the `osm` adapter's mapping (`componentsForOSMRow`), so the two
+ *   surfaces cannot disagree about what a field means.
+ *
+ *   ODbL: the rows inherit OpenStreetMap's share-alike license. A corpus build run under
+ *   `LicensePolicy.ShareAlikeFree` refuses them.
+ *
+ *   Run: mailwoman corpus slice pk-register --input <osm-pk.corpus.jsonl> --count N --seed S
+ */
+import { stringifyJSON } from "@mailwoman/core/json";
+import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils";
+import { componentsForOSMRow, OSM_LICENSE, sameName } from "#adapters/osm/adapter";
+import { alignAndWrite, readTuples, recipeSourceID } from "#recipes/scaffold";
+import { defaultRecipeSource } from "#recipes/sources";
+import { SourceRegister } from "#registers";
+import { SurfaceOrigin } from "#types";
+/**
+ * Islamabad's residential sectors: the lettered rows E to I and the numbered columns
+ * developed by the Capital Development Authority.
+ * Each column splits into four sub-sectors.
+ */
+const ISLAMABAD_SECTOR_ROWS = ["E", "F", "G", "I"];
+const ISLAMABAD_SECTOR_COLUMNS = [6, 7, 8, 9, 10, 11];
+const ISLAMABAD_SUB_SECTORS = [1, 2, 3, 4];
+/**
+ * Share of rows rendered in the house register.
+ * The rest take the plain line.
+ */
+const P_HOUSE = 0.55;
+/**
+ * Share of house lines that spell the number `House No. 4` / `House # 4` rather than `House 4`.
+ */
+const P_HOUSE_NO = 0.25;
+/**
+ * Draw one Islamabad sector code, `F-7/2`.
+ */
+export function drawIslamabadSector(random) {
+    const row = ISLAMABAD_SECTOR_ROWS[Math.floor(random() * ISLAMABAD_SECTOR_ROWS.length)];
+    const column = ISLAMABAD_SECTOR_COLUMNS[Math.floor(random() * ISLAMABAD_SECTOR_COLUMNS.length)];
+    const sub = ISLAMABAD_SUB_SECTORS[Math.floor(random() * ISLAMABAD_SUB_SECTORS.length)];
+    return `${row}-${column}/${sub}`;
+}
+/**
+ * Render one register line from the adapter's components.
+ *
+ * @returns Null when the row has no house number or no locality: both registers need the two.
+ */
+export function renderSouthAsiaRegister(components, random, register) {
+    const { house_number: number, street, locality, postcode } = components;
+    if (!number || !street || !locality)
+        return null;
+    let dependent = components.dependent_locality;
+    if (!dependent && sameName(locality, "Islamabad")) {
+        dependent = drawIslamabadSector(random);
+    }
+    const chosen = register ?? (random() < P_HOUSE ? "house" : "plain");
+    const out = { house_number: number, street, locality };
+    const tail = postcode ? `${locality} ${postcode}` : locality;
+    if (postcode) {
+        out.postcode = postcode;
+    }
+    if (dependent) {
+        out.dependent_locality = dependent;
+    }
+    const middle = dependent ? `${street}, ${dependent}` : street;
+    if (chosen === "house") {
+        const spelling = random() < P_HOUSE_NO ? (random() < 0.5 ? "House No. " : "House # ") : "House ";
+        return { raw: `${spelling}${number}, ${middle}, ${tail}`, components: out, register: "house" };
+    }
+    return { raw: `${number} ${middle}, ${tail}`, components: out, register: "plain" };
+}
+function makeRecipe(name, country, locale, description) {
+    // The value is composed and read through the table rather than written out in the composition.
+    // so the vocabulary flip is one constant and a name the table does not record refuses here.
+    const source = defaultRecipeSource(`synth-${name}`);
+    return {
+        name,
+        description,
+        mode: "tuples",
+        options: [
+            { flag: "--count", description: "Rows to emit (default: every input row once)" },
+            {
+                flag: "--input",
+                description: `The OpenStreetMap ${country} corpus JSONL (osm-${country.toLowerCase()}.corpus.jsonl)`,
+            },
+        ],
+        async run(opts, write) {
+            if (!opts.input) {
+                throw new Error(`${name}: --input <osm-${country.toLowerCase()}.corpus.jsonl> is required`);
+            }
+            const random = makeMulberry32(opts.seed);
+            const sourceName = opts.sourceName ?? source;
+            const registers = {};
+            let read = 0;
+            let emitted = 0;
+            let skipped = 0;
+            let quarantined = 0;
+            for await (const tuple of readTuples(opts.input)) {
+                read++;
+                if (opts.count !== undefined && emitted >= opts.count)
+                    break;
+                const components = componentsForOSMRow(tuple);
+                const rendering = components ? renderSouthAsiaRegister(components, random) : null;
+                if (!rendering) {
+                    skipped++;
+                    continue;
+                }
+                if (opts.golden) {
+                    write(stringifyJSON({ raw: rendering.raw, components: rendering.components, country, locale }));
+                    emitted++;
+                    registers[rendering.register] = (registers[rendering.register] ?? 0) + 1;
+                    continue;
+                }
+                const ok = alignAndWrite(write, {
+                    raw: rendering.raw,
+                    components: rendering.components,
+                    country,
+                    locale,
+                    source: sourceName,
+                    source_id: recipeSourceID(sourceName, { raw: rendering.raw }),
+                    corpus_version: "0.1.0",
+                    license: OSM_LICENSE,
+                }, `${name}:${rendering.register}`, 
+                // The components come from the OpenStreetMap corpus jsonl the `osm` adapter wrote.
+                // This recipe re-orders them into the typed forms, so the address is real
+                // and the line is the recipe's.
+                // The OSM record is what `baseSourceID` names when the tuples file includes its id.
+                {
+                    register: SourceRegister.OpenStreetMap,
+                    surface: SurfaceOrigin.Composed,
+                    baseSourceID: tuple.sourceID ?? null,
+                });
+                if (ok) {
+                    emitted++;
+                    registers[rendering.register] = (registers[rendering.register] ?? 0) + 1;
+                }
+                else {
+                    quarantined++;
+                }
+            }
+            console.error(`  ${name}: ${emitted} emitted (${Object.entries(registers)
+                .map(([k, v]) => `${k} ${v}`)
+                .join(", ")}), ${skipped} skipped, ${quarantined} quarantined of ${read} read`);
+            return { read, emitted, skipped };
+        },
+    };
+}
+/**
+ * Pakistan: `House 4, Street 25, F-7/2, Islamabad` and `B230 Street 5, Block 3, Karachi`.
+ */
+export const pkRegisterRecipe = makeRecipe("pk-register", "PK", "en-PK", "Pakistan typed registers over the OpenStreetMap rows: the House line with the sector or scheme, the plain line with the trailing postcode");
+/**
+ * Bangladesh: `House 34, Road 4, Sector 9, Uttara, Dhaka 1230` and `58 Kalabagan 1st Ln, Dhaka 1205`.
+ */
+export const bdRegisterRecipe = makeRecipe("bd-register", "BD", "en-BD", "Bangladesh typed registers over the OpenStreetMap rows: the House line, the plain line with the trailing postcode and no dash");
+//# sourceMappingURL=register.js.map

package/out/south-asia/recipes/register.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"register.js","sourceRoot":"","sources":["../../../lib/south-asia/recipes/register.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AACpD,OAAO,EAAE,UAAU,IAAI,cAAc,EAAE,MAAM,uBAAuB,CAAA;AAEpE,OAAO,EAAE,mBAAmB,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAA;AAClF,OAAO,EAAE,aAAa,EAAqB,UAAU,EAAsB,cAAc,EAAE,MAAM,mBAAmB,CAAA;AACpH,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAC3C,OAAO,EAAqB,aAAa,EAAE,MAAM,QAAQ,CAAA;AAEzD;;;;GAIG;AACH,MAAM,qBAAqB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAU,CAAA;AAC3D,MAAM,wBAAwB,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,CAAU,CAAA;AAC9D,MAAM,qBAAqB,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAU,CAAA;AAEnD;;;GAGG;AACH,MAAM,OAAO,GAAG,IAAI,CAAA;AAEpB;;GAEG;AACH,MAAM,UAAU,GAAG,IAAI,CAAA;AAUvB;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,MAAoB;IACvD,MAAM,GAAG,GAAG,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAE,CAAA;IACvF,MAAM,MAAM,GAAG,wBAAwB,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,wBAAwB,CAAC,MAAM,CAAC,CAAE,CAAA;IAChG,MAAM,GAAG,GAAG,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAE,CAAA;IAEvF,OAAO,GAAG,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAA;AACjC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,uBAAuB,CACtC,UAAsC,EACtC,MAAoB,EACpB,QAA4B;IAE5B,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,UAAU,CAAA;IAEvE,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAA;IAEhD,IAAI,SAAS,GAAG,UAAU,CAAC,kBAAkB,CAAA;IAE7C,IAAI,CAAC,SAAS,IAAI,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,EAAE,CAAC;QACnD,SAAS,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAA;IACxC,CAAC;IAED,MAAM,MAAM,GAAG,QAAQ,IAAI,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAA;IACnE,MAAM,GAAG,GAA2B,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAA;IAC9E,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,QAAQ,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAA;IAE5D,IAAI,QAAQ,EAAE,CAAC;QACd,GAAG,CAAC,QAAQ,GAAG,QAAQ,CAAA;IACxB,CAAC;IAED,IAAI,SAAS,EAAE,CAAC;QACf,GAAG,CAAC,kBAAkB,GAAG,SAAS,CAAA;IACnC,CAAC;IAED,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,GAAG,MAAM,KAAK,SAAS,EAAE,CAAC,CAAC,CAAC,MAAM,CAAA;IAE7D,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;QACxB,MAAM,QAAQ,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAA;QAEhG,OAAO,EAAE,GAAG,EAAE,GAAG,QAAQ,GAAG,MAAM,KAAK,MAAM,KAAK,IAAI,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAA;IAC/F,CAAC;IAED,OAAO,EAAE,GAAG,EAAE,GAAG,MAAM,IAAI,MAAM,KAAK,IAAI,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAA;AACnF,CAAC;AAED,SAAS,UAAU,CAAC,IAAY,EAAE,OAAoB,EAAE,MAAc,EAAE,WAAmB;IAC1F,+FAA+F;IAC/F,4FAA4F;IAC5F,MAAM,MAAM,GAAG,mBAAmB,CAAC,SAAS,IAAI,EAAE,CAAC,CAAA;IAEnD,OAAO;QACN,IAAI;QACJ,WAAW;QACX,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE;YACR,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,8CAA8C,EAAE;YAChF;gBACC,IAAI,EAAE,SAAS;gBACf,WAAW,EAAE,qBAAqB,OAAO,sBAAsB,OAAO,CAAC,WAAW,EAAE,gBAAgB;aACpG;SACD;QACD,KAAK,CAAC,GAAG,CAAC,IAAmB,EAAE,KAAK;YACnC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,kBAAkB,OAAO,CAAC,WAAW,EAAE,4BAA4B,CAAC,CAAA;YAC5F,CAAC;YAED,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACxC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,MAAM,CAAA;YAC5C,MAAM,SAAS,GAA2B,EAAE,CAAA;YAC5C,IAAI,IAAI,GAAG,CAAC,CAAA;YACZ,IAAI,OAAO,GAAG,CAAC,CAAA;YACf,IAAI,OAAO,GAAG,CAAC,CAAA;YACf,IAAI,WAAW,GAAG,CAAC,CAAA;YAEnB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBAClD,IAAI,EAAE,CAAA;gBAEN,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,IAAI,IAAI,CAAC,KAAK;oBAAE,MAAK;gBAE5D,MAAM,UAAU,GAAG,mBAAmB,CAAC,KAAkD,CAAC,CAAA;gBAC1F,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,uBAAuB,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;gBAEjF,IAAI,CAAC,SAAS,EAAE,CAAC;oBAChB,OAAO,EAAE,CAAA;oBAET,SAAQ;gBACT,CAAC;gBAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;oBACjB,KAAK,CAAC,aAAa,CAAC,EAAE,GAAG,EAAE,SAAS,CAAC,GAAG,EAAE,UAAU,EAAE,SAAS,CAAC,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAA;oBAE/F,OAAO,EAAE,CAAA;oBACT,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;oBAExE,SAAQ;gBACT,CAAC;gBAED,MAAM,EAAE,GAAG,aAAa,CACvB,KAAK,EACL;oBACC,GAAG,EAAE,SAAS,CAAC,GAAG;oBAClB,UAAU,EAAE,SAAS,CAAC,UAAU;oBAChC,OAAO;oBACP,MAAM;oBACN,MAAM,EAAE,UAAU;oBAClB,SAAS,EAAE,cAAc,CAAC,UAAU,EAAE,EAAE,GAAG,EAAE,SAAS,CAAC,GAAG,EAAE,CAAC;oBAC7D,cAAc,EAAE,OAAO;oBACvB,OAAO,EAAE,WAAW;iBACpB,EACD,GAAG,IAAI,IAAI,SAAS,CAAC,QAAQ,EAAE;gBAC/B,mFAAmF;gBACnF,0EAA0E;gBAC1E,gCAAgC;gBAChC,oFAAoF;gBACpF;oBACC,QAAQ,EAAE,cAAc,CAAC,aAAa;oBACtC,OAAO,EAAE,aAAa,CAAC,QAAQ;oBAC/B,YAAY,EAAE,KAAK,CAAC,QAAQ,IAAI,IAAI;iBACpC,CACD,CAAA;gBAED,IAAI,EAAE,EAAE,CAAC;oBACR,OAAO,EAAE,CAAA;oBACT,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;gBACzE,CAAC;qBAAM,CAAC;oBACP,WAAW,EAAE,CAAA;gBACd,CAAC;YACF,CAAC;YAED,OAAO,CAAC,KAAK,CACZ,KAAK,IAAI,KAAK,OAAO,aAAa,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC;iBACzD,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;iBAC5B,IAAI,CAAC,IAAI,CAAC,MAAM,OAAO,aAAa,WAAW,mBAAmB,IAAI,OAAO,CAC/E,CAAA;YAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;QAClC,CAAC;KACD,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAiB,UAAU,CACvD,aAAa,EACb,IAAI,EACJ,OAAO,EACP,2IAA2I,CAC3I,CAAA;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAiB,UAAU,CACvD,aAAa,EACb,IAAI,EACJ,OAAO,EACP,+HAA+H,CAC/H,CAAA"}