@mailwoman/corpus 5.3.0 → 5.4.0

package/out/src/shard-recipes/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAiBH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAOhD,cAAc,eAAe,CAAA;AA0B7B,4BAA4B;AAC5B,eAAO,MAAM,aAAa,EAAE,WAAW,CAAC,MAAM,EAAE,WAAW,CAA4C,CAAA;AAEvG,+CAA+C;AAC/C,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,CAEpE;AAED,yCAAyC;AACzC,wBAAgB,gBAAgB,IAAI,SAAS,WAAW,EAAE,CAEzD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAkBH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAOhD,cAAc,eAAe,CAAA;AA2B7B,4BAA4B;AAC5B,eAAO,MAAM,aAAa,EAAE,WAAW,CAAC,MAAM,EAAE,WAAW,CAA4C,CAAA;AAEvG,+CAA+C;AAC/C,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,CAEpE;AAED,yCAAyC;AACzC,wBAAgB,gBAAgB,IAAI,SAAS,WAAW,EAAE,CAEzD"}

package/out/src/shard-recipes/index.js

@@ -18,6 +18,7 @@ import { germanRecipe } from "./german.js";
 import { houseVenueRecipe } from "./house-venue.js";
 import { intersectionRecipe } from "./intersection.js";
 import { localeRecipe } from "./locale.js";
+import { nlPostcodeRecipe } from "./nl-postcode.js";
 import { noStreetLedRecipe } from "./no-street-led.js";
 import { noStreetRecipe } from "./no-street.js";
 import { poBoxCedexRecipe } from "./po-box-cedex.js";
@@ -45,6 +46,7 @@ const RECIPES = [
     frAdminSplitRecipe,
     frBareStreetRecipe,
     czPcFirstPrepositionRecipe,
+    nlPostcodeRecipe,
     noStreetLedRecipe,
     siBareVillageRecipe,
     countryBalancedRecipe,

package/out/src/shard-recipes/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/shard-recipes/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AAC/D,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAA;AAC3D,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAA;AAC7D,OAAO,EAAE,0BAA0B,EAAE,MAAM,6BAA6B,CAAA;AACxE,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AACxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAA;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAEzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAA;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAA;AACrD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAA;AAEtC,cAAc,eAAe,CAAA;AAE7B,iDAAiD;AACjD,MAAM,OAAO,GAA2B;IACvC,YAAY;IACZ,gBAAgB;IAChB,iBAAiB;IACjB,cAAc;IACd,gBAAgB;IAChB,WAAW;IACX,gBAAgB;IAChB,UAAU;IACV,kBAAkB;IAClB,YAAY;IACZ,YAAY;IACZ,aAAa;IACb,kBAAkB;IAClB,kBAAkB;IAClB,0BAA0B;IAC1B,iBAAiB;IACjB,mBAAmB;IACnB,qBAAqB;IACrB,oBAAoB;IACpB,sBAAsB;CACtB,CAAA;AAED,4BAA4B;AAC5B,MAAM,CAAC,MAAM,aAAa,GAAqC,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;AAEvG,+CAA+C;AAC/C,MAAM,UAAU,cAAc,CAAC,IAAY;IAC1C,OAAO,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;AAC/B,CAAC;AAED,yCAAyC;AACzC,MAAM,UAAU,gBAAgB;IAC/B,OAAO,OAAO,CAAA;AACf,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/shard-recipes/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AAC/D,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAA;AAC3D,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAA;AAC7D,OAAO,EAAE,0BAA0B,EAAE,MAAM,6BAA6B,CAAA;AACxE,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AACxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAA;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAEzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAA;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAA;AACrD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAA;AAEtC,cAAc,eAAe,CAAA;AAE7B,iDAAiD;AACjD,MAAM,OAAO,GAA2B;IACvC,YAAY;IACZ,gBAAgB;IAChB,iBAAiB;IACjB,cAAc;IACd,gBAAgB;IAChB,WAAW;IACX,gBAAgB;IAChB,UAAU;IACV,kBAAkB;IAClB,YAAY;IACZ,YAAY;IACZ,aAAa;IACb,kBAAkB;IAClB,kBAAkB;IAClB,0BAA0B;IAC1B,gBAAgB;IAChB,iBAAiB;IACjB,mBAAmB;IACnB,qBAAqB;IACrB,oBAAoB;IACpB,sBAAsB;CACtB,CAAA;AAED,4BAA4B;AAC5B,MAAM,CAAC,MAAM,aAAa,GAAqC,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;AAEvG,+CAA+C;AAC/C,MAAM,UAAU,cAAc,CAAC,IAAY;IAC1C,OAAO,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;AAC/B,CAAC;AAED,yCAAyC;AACzC,MAAM,UAAU,gBAAgB;IAC/B,OAAO,OAAO,CAAA;AACf,CAAC"}

package/out/src/shard-recipes/nl-postcode.d.ts

@@ -0,0 +1,22 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `nl-postcode` — the Dutch full-form postcode shard (#924). The model reads the DIGITS-FIRST NL
+ *   postcode "1012 LG" as a house number + a 2-letter street ("1012 LG Amsterdam" → house_number
+ *   1012 / street "LG" / locality Amsterdam), and that spurious street context then pulls the
+ *   locality into the US situs tier (Amsterdam → Amsterdam, NY). Letters-first postcodes (UK
+ *   "SW1A 1AA") parse natively; the `\d{4} [A-Z]{2}` shape does not, and the soft query-shape prior
+ *   (0.9 log-odds) can't overcome the strong house-number reading of a leading 4-digit token.
+ *
+ *   This is the model-first fix as DATA (the #723/#901 discipline — teach the boundary, don't
+ *   override the decoder): real NL (street, number, postcode, city) tuples in the orders Dutch
+ *   addresses actually use, with the full postcode tagged as ONE postcode span. Both the SPACED
+ *   ("1012 LG", the failing form — a 2-token span) and UNSPACED ("1012LG", 1 token) forms are
+ *   emitted so the model learns the digits-first postcode regardless of spacing; the three orders
+ *   keep polarity balanced (the v1.9.9 lesson).
+ */
+import { type ShardRecipe } from "./scaffold.js";
+export declare const nlPostcodeRecipe: ShardRecipe;
+//# sourceMappingURL=nl-postcode.d.ts.map

package/out/src/shard-recipes/nl-postcode.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"nl-postcode.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/nl-postcode.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAA6C,KAAK,WAAW,EAAiB,MAAM,eAAe,CAAA;AAO1G,eAAO,MAAM,gBAAgB,EAAE,WA0E9B,CAAA"}

package/out/src/shard-recipes/nl-postcode.js

@@ -0,0 +1,94 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `nl-postcode` — the Dutch full-form postcode shard (#924). The model reads the DIGITS-FIRST NL
+ *   postcode "1012 LG" as a house number + a 2-letter street ("1012 LG Amsterdam" → house_number
+ *   1012 / street "LG" / locality Amsterdam), and that spurious street context then pulls the
+ *   locality into the US situs tier (Amsterdam → Amsterdam, NY). Letters-first postcodes (UK
+ *   "SW1A 1AA") parse natively; the `\d{4} [A-Z]{2}` shape does not, and the soft query-shape prior
+ *   (0.9 log-odds) can't overcome the strong house-number reading of a leading 4-digit token.
+ *
+ *   This is the model-first fix as DATA (the #723/#901 discipline — teach the boundary, don't
+ *   override the decoder): real NL (street, number, postcode, city) tuples in the orders Dutch
+ *   addresses actually use, with the full postcode tagged as ONE postcode span. Both the SPACED
+ *   ("1012 LG", the failing form — a 2-token span) and UNSPACED ("1012LG", 1 token) forms are
+ *   emitted so the model learns the digits-first postcode regardless of spacing; the three orders
+ *   keep polarity balanced (the v1.9.9 lesson).
+ */
+import { alignAndWrite, makeMulberry32, readTuples, shardSourceID } from "./scaffold.js";
+/** "1012LG" → "1012 LG". The tuples carry the unspaced OA form; the spaced form is the failing case. */
+function spacePostcode(pc) {
+    return pc.replace(/^(\d{4})([A-Z]{2})$/, "$1 $2");
+}
+export const nlPostcodeRecipe = {
+    name: "nl-postcode",
+    description: "NL full-form postcode (#924): teach '\\d{4} [A-Z]{2}' = postcode, not house#+street — spaced + unspaced, 3 orders",
+    mode: "tuples",
+    async run(opts, write) {
+        makeMulberry32(opts.seed);
+        let read = 0;
+        let emitted = 0;
+        let skipped = 0;
+        for await (const t of readTuples(opts.input)) {
+            read++;
+            const street = String(t.street ?? "").trim();
+            const city = String(t.locality ?? "").trim();
+            const number = String(t.number ?? "").trim();
+            const rawPostcode = String(t.postcode ?? "")
+                .trim()
+                .toUpperCase()
+                .replace(/\s+/g, "");
+            if (!street || !city || !number || !/^\d{4}[A-Z]{2}$/.test(rawPostcode)) {
+                skipped++;
+                continue;
+            }
+            // Spacing rotates so the model sees BOTH the failing spaced form and the unspaced form; the
+            // components.postcode value MUST match the raw form so alignment tags the right span.
+            const spaced = read % 2 === 0;
+            const postcode = spaced ? spacePostcode(rawPostcode) : rawPostcode;
+            // The three orders Dutch addresses use. `street number, postcode city` is canonical; the
+            // pc-first form is where the leading digits most strongly mis-read as a house number.
+            const order = read % 3;
+            let raw;
+            if (order === 0) {
+                raw = `${street} ${number}, ${postcode} ${city}`;
+            }
+            else if (order === 1) {
+                raw = `${postcode} ${city}, ${street} ${number}`;
+            }
+            else {
+                raw = `${city}, ${postcode}, ${street} ${number}`;
+            }
+            const components = {
+                street,
+                house_number: number,
+                postcode,
+                locality: city,
+            };
+            const source_id = shardSourceID("synth-nl-postcode", {
+                ...components,
+                o: String(order),
+                s: spaced ? "1" : "0",
+                v: String(read),
+            });
+            const canonical = {
+                raw,
+                components,
+                country: "NL",
+                locale: "nl-NL",
+                source: "synth-nl-postcode",
+                source_id,
+                corpus_version: "0.10.0",
+                license: "Synthetic — nl-postcode; (street, number, postcode, city) from OpenAddresses NL (per-source attribution in the model card)",
+            };
+            if (alignAndWrite(write, canonical, "nl-postcode"))
+                emitted++;
+            else
+                skipped++;
+        }
+        return { read, emitted, skipped };
+    },
+};
+//# sourceMappingURL=nl-postcode.js.map

package/out/src/shard-recipes/nl-postcode.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"nl-postcode.js","sourceRoot":"","sources":["../../../src/shard-recipes/nl-postcode.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,UAAU,EAAoB,aAAa,EAAE,MAAM,eAAe,CAAA;AAE1G,wGAAwG;AACxG,SAAS,aAAa,CAAC,EAAU;IAChC,OAAO,EAAE,CAAC,OAAO,CAAC,qBAAqB,EAAE,OAAO,CAAC,CAAA;AAClD,CAAC;AAED,MAAM,CAAC,MAAM,gBAAgB,GAAgB;IAC5C,IAAI,EAAE,aAAa;IACnB,WAAW,EACV,mHAAmH;IACpH,IAAI,EAAE,QAAQ;IACd,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACzB,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,KAAM,CAAC,EAAE,CAAC;YAC/C,IAAI,EAAE,CAAA;YACN,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YAC5C,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YAC5C,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YAC5C,MAAM,WAAW,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC;iBAC1C,IAAI,EAAE;iBACN,WAAW,EAAE;iBACb,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;YAErB,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;gBACzE,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YAED,4FAA4F;YAC5F,sFAAsF;YACtF,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAA;YAC7B,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAA;YAElE,yFAAyF;YACzF,sFAAsF;YACtF,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAA;YACtB,IAAI,GAAW,CAAA;YAEf,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBACjB,GAAG,GAAG,GAAG,MAAM,IAAI,MAAM,KAAK,QAAQ,IAAI,IAAI,EAAE,CAAA;YACjD,CAAC;iBAAM,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBACxB,GAAG,GAAG,GAAG,QAAQ,IAAI,IAAI,KAAK,MAAM,IAAI,MAAM,EAAE,CAAA;YACjD,CAAC;iBAAM,CAAC;gBACP,GAAG,GAAG,GAAG,IAAI,KAAK,QAAQ,KAAK,MAAM,IAAI,MAAM,EAAE,CAAA;YAClD,CAAC;YAED,MAAM,UAAU,GAA2B;gBAC1C,MAAM;gBACN,YAAY,EAAE,MAAM;gBACpB,QAAQ;gBACR,QAAQ,EAAE,IAAI;aACd,CAAA;YACD,MAAM,SAAS,GAAG,aAAa,CAAC,mBAAmB,EAAE;gBACpD,GAAG,UAAU;gBACb,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC;gBAChB,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;gBACrB,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC;aACf,CAAC,CAAA;YACF,MAAM,SAAS,GAAG;gBACjB,GAAG;gBACH,UAAU;gBACV,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,OAAO;gBACf,MAAM,EAAE,mBAAmB;gBAC3B,SAAS;gBACT,cAAc,EAAE,QAAQ;gBACxB,OAAO,EACN,4HAA4H;aAC7H,CAAA;YAED,IAAI,aAAa,CAAC,KAAK,EAAE,SAAS,EAAE,aAAa,CAAC;gBAAE,OAAO,EAAE,CAAA;;gBACxD,OAAO,EAAE,CAAA;QACf,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IAClC,CAAC;CACD,CAAA"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mailwoman/corpus",
-  "version": "5.3.0",
+  "version": "5.4.0",
   "description": "Mailwoman corpus pipeline: BIO-labeled dataset builder for the neural classifier.",
   "license": "AGPL-3.0-only OR LicenseRef-Commercial",
   "repository": {
@@ -26,9 +26,9 @@
   },
   "dependencies": {
     "@dsnp/parquetjs": "1.8.7",
-    "@mailwoman/codex": "5.3.0",
-    "@mailwoman/core": "5.3.0",
-    "@mailwoman/formatter": "5.3.0",
+    "@mailwoman/codex": "5.4.0",
+    "@mailwoman/core": "5.4.0",
+    "@mailwoman/formatter": "5.4.0",
     "csv-parse": "^5.6.0",
     "fast-glob": "^3.3.3",
     "fastest-levenshtein": "^1.0.16",