@mailwoman/resolver-wof-sqlite 8.0.0 → 8.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/address-point-interpolation.ts +9 -3
- package/address-point-schema.ts +32 -10
- package/address-point.ts +3 -0
- package/ancestry-backfill.ts +18 -5
- package/ancestry.ts +7 -2
- package/build-candidate.ts +29 -6
- package/build-slim.ts +40 -11
- package/candidate-fts.ts +1 -0
- package/candidate-lookup.ts +49 -15
- package/candidate-schema.ts +35 -11
- package/coincident-roles.ts +28 -6
- package/convention.ts +3 -1
- package/coverage-manifest-schema.ts +243 -0
- package/fst-autocomplete.ts +15 -9
- package/fst-builder.ts +65 -9
- package/fst-deserialize-web.ts +46 -9
- package/fst-matcher.ts +12 -5
- package/fst-serialize.ts +83 -9
- package/fst-types.ts +43 -0
- package/fts-query.ts +84 -0
- package/fts.ts +35 -9
- package/geo.ts +9 -3
- package/geonames-aliases.ts +116 -79
- package/geonames-postal.ts +25 -5
- package/index.ts +19 -0
- package/interpolation.ts +59 -55
- package/lookup.ts +103 -292
- package/name-score.ts +76 -0
- package/out/address-point-interpolation.d.ts.map +1 -1
- package/out/address-point-interpolation.js +4 -2
- package/out/address-point-interpolation.js.map +1 -1
- package/out/address-point-schema.d.ts +30 -10
- package/out/address-point-schema.d.ts.map +1 -1
- package/out/address-point-schema.js +6 -2
- package/out/address-point-schema.js.map +1 -1
- package/out/address-point.d.ts.map +1 -1
- package/out/address-point.js.map +1 -1
- package/out/ancestry-backfill.d.ts +6 -2
- package/out/ancestry-backfill.d.ts.map +1 -1
- package/out/ancestry-backfill.js +7 -3
- package/out/ancestry-backfill.js.map +1 -1
- package/out/ancestry.d.ts +6 -2
- package/out/ancestry.d.ts.map +1 -1
- package/out/ancestry.js +3 -1
- package/out/ancestry.js.map +1 -1
- package/out/build-candidate.d.ts +9 -3
- package/out/build-candidate.d.ts.map +1 -1
- package/out/build-candidate.js +5 -3
- package/out/build-candidate.js.map +1 -1
- package/out/build-slim.d.ts +15 -5
- package/out/build-slim.d.ts.map +1 -1
- package/out/build-slim.js +11 -5
- package/out/build-slim.js.map +1 -1
- package/out/candidate-fts.d.ts.map +1 -1
- package/out/candidate-fts.js.map +1 -1
- package/out/candidate-lookup.d.ts +16 -3
- package/out/candidate-lookup.d.ts.map +1 -1
- package/out/candidate-lookup.js +27 -11
- package/out/candidate-lookup.js.map +1 -1
- package/out/candidate-schema.d.ts +33 -11
- package/out/candidate-schema.d.ts.map +1 -1
- package/out/candidate-schema.js.map +1 -1
- package/out/coincident-roles.d.ts +16 -4
- package/out/coincident-roles.d.ts.map +1 -1
- package/out/coincident-roles.js +9 -3
- package/out/coincident-roles.js.map +1 -1
- package/out/convention.d.ts +3 -1
- package/out/convention.d.ts.map +1 -1
- package/out/convention.js.map +1 -1
- package/out/coverage-manifest-schema.d.ts +112 -0
- package/out/coverage-manifest-schema.d.ts.map +1 -0
- package/out/coverage-manifest-schema.js +154 -0
- package/out/coverage-manifest-schema.js.map +1 -0
- package/out/fst-autocomplete.d.ts +1 -1
- package/out/fst-autocomplete.d.ts.map +1 -1
- package/out/fst-autocomplete.js +11 -9
- package/out/fst-autocomplete.js.map +1 -1
- package/out/fst-builder.d.ts.map +1 -1
- package/out/fst-builder.js +42 -9
- package/out/fst-builder.js.map +1 -1
- package/out/fst-deserialize-web.d.ts.map +1 -1
- package/out/fst-deserialize-web.js +34 -9
- package/out/fst-deserialize-web.js.map +1 -1
- package/out/fst-matcher.d.ts +6 -2
- package/out/fst-matcher.d.ts.map +1 -1
- package/out/fst-matcher.js +9 -5
- package/out/fst-matcher.js.map +1 -1
- package/out/fst-serialize.d.ts.map +1 -1
- package/out/fst-serialize.js +62 -9
- package/out/fst-serialize.js.map +1 -1
- package/out/fst-types.d.ts +43 -0
- package/out/fst-types.d.ts.map +1 -1
- package/out/fts-query.d.ts +41 -0
- package/out/fts-query.d.ts.map +1 -0
- package/out/fts-query.js +75 -0
- package/out/fts-query.js.map +1 -0
- package/out/fts.d.ts +21 -7
- package/out/fts.d.ts.map +1 -1
- package/out/fts.js +10 -4
- package/out/fts.js.map +1 -1
- package/out/geo.d.ts +6 -2
- package/out/geo.d.ts.map +1 -1
- package/out/geo.js +3 -1
- package/out/geo.js.map +1 -1
- package/out/geonames-aliases.d.ts +12 -4
- package/out/geonames-aliases.d.ts.map +1 -1
- package/out/geonames-aliases.js +72 -67
- package/out/geonames-aliases.js.map +1 -1
- package/out/geonames-postal.d.ts +9 -3
- package/out/geonames-postal.d.ts.map +1 -1
- package/out/geonames-postal.js +7 -2
- package/out/geonames-postal.js.map +1 -1
- package/out/index.d.ts +2 -0
- package/out/index.d.ts.map +1 -1
- package/out/index.js +1 -0
- package/out/index.js.map +1 -1
- package/out/interpolation.d.ts +24 -6
- package/out/interpolation.d.ts.map +1 -1
- package/out/interpolation.js +32 -40
- package/out/interpolation.js.map +1 -1
- package/out/lookup.d.ts +3 -97
- package/out/lookup.d.ts.map +1 -1
- package/out/lookup.js +52 -184
- package/out/lookup.js.map +1 -1
- package/out/name-score.d.ts +28 -0
- package/out/name-score.d.ts.map +1 -0
- package/out/name-score.js +67 -0
- package/out/name-score.js.map +1 -0
- package/out/poi-lookup.d.ts +24 -8
- package/out/poi-lookup.d.ts.map +1 -1
- package/out/poi-lookup.js +27 -13
- package/out/poi-lookup.js.map +1 -1
- package/out/poi-schema.d.ts +42 -13
- package/out/poi-schema.d.ts.map +1 -1
- package/out/poi-schema.js +12 -3
- package/out/poi-schema.js.map +1 -1
- package/out/postal-city-alias-lookup.d.ts +18 -6
- package/out/postal-city-alias-lookup.d.ts.map +1 -1
- package/out/postal-city-alias-lookup.js.map +1 -1
- package/out/postal-city-alias-schema.d.ts +27 -9
- package/out/postal-city-alias-schema.d.ts.map +1 -1
- package/out/postal-city-alias-schema.js +3 -1
- package/out/postal-city-alias-schema.js.map +1 -1
- package/out/postal-city-candidate-schema.d.ts +15 -5
- package/out/postal-city-candidate-schema.d.ts.map +1 -1
- package/out/postal-city-candidate-schema.js +3 -1
- package/out/postal-city-candidate-schema.js.map +1 -1
- package/out/postcode-point-lookup.d.ts +6 -2
- package/out/postcode-point-lookup.d.ts.map +1 -1
- package/out/postcode-point-lookup.js +6 -2
- package/out/postcode-point-lookup.js.map +1 -1
- package/out/ranking-weights.d.ts +118 -0
- package/out/ranking-weights.d.ts.map +1 -0
- package/out/ranking-weights.js +44 -0
- package/out/ranking-weights.js.map +1 -0
- package/out/reverse.d.ts +9 -3
- package/out/reverse.d.ts.map +1 -1
- package/out/reverse.js +20 -6
- package/out/reverse.js.map +1 -1
- package/out/sharding.d.ts +3 -1
- package/out/sharding.d.ts.map +1 -1
- package/out/sharding.js +7 -5
- package/out/sharding.js.map +1 -1
- package/out/sqlite-convention-source.d.ts.map +1 -1
- package/out/sqlite-convention-source.js +3 -1
- package/out/sqlite-convention-source.js.map +1 -1
- package/out/street-centroid-schema.d.ts +33 -11
- package/out/street-centroid-schema.d.ts.map +1 -1
- package/out/street-centroid-schema.js +3 -1
- package/out/street-centroid-schema.js.map +1 -1
- package/out/street-centroid.d.ts.map +1 -1
- package/out/street-centroid.js +6 -2
- package/out/street-centroid.js.map +1 -1
- package/out/street-morphology-fst-builder.d.ts +6 -2
- package/out/street-morphology-fst-builder.d.ts.map +1 -1
- package/out/street-morphology-fst-builder.js +8 -7
- package/out/street-morphology-fst-builder.js.map +1 -1
- package/out/street-morphology-fst-loader.d.ts +67 -0
- package/out/street-morphology-fst-loader.d.ts.map +1 -0
- package/out/street-morphology-fst-loader.js +59 -0
- package/out/street-morphology-fst-loader.js.map +1 -0
- package/out/street-name-lookup.d.ts +9 -3
- package/out/street-name-lookup.d.ts.map +1 -1
- package/out/street-name-lookup.js +9 -7
- package/out/street-name-lookup.js.map +1 -1
- package/out/street-normalize.d.ts +3 -1
- package/out/street-normalize.d.ts.map +1 -1
- package/out/street-normalize.js +23 -13
- package/out/street-normalize.js.map +1 -1
- package/out/street-segment-schema.d.ts +68 -13
- package/out/street-segment-schema.d.ts.map +1 -1
- package/out/street-segment-schema.js +21 -2
- package/out/street-segment-schema.js.map +1 -1
- package/out/types.d.ts +18 -6
- package/out/types.d.ts.map +1 -1
- package/out/unified-schema.d.ts +1 -1
- package/out/unified-schema.d.ts.map +1 -1
- package/out/unified-schema.js +2 -2
- package/out/unified-schema.js.map +1 -1
- package/package.json +13 -5
- package/poi-lookup.ts +53 -21
- package/poi-schema.ts +43 -13
- package/postal-city-alias-lookup.ts +20 -6
- package/postal-city-alias-schema.ts +28 -9
- package/postal-city-candidate-schema.ts +15 -5
- package/postcode-point-lookup.ts +6 -2
- package/ranking-weights.ts +148 -0
- package/reverse.ts +47 -10
- package/sharding.ts +13 -6
- package/sqlite-convention-source.ts +4 -1
- package/street-centroid-schema.ts +35 -11
- package/street-centroid.ts +10 -3
- package/street-morphology-fst-builder.ts +25 -9
- package/street-morphology-fst-loader.ts +103 -0
- package/street-name-lookup.ts +19 -7
- package/street-normalize.ts +28 -13
- package/street-segment-schema.ts +83 -13
- package/types.ts +18 -6
- package/unified-schema.ts +11 -2
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Typed schema + read/write helpers for the candidate gazetteer's COVERAGE MANIFEST — the two
|
|
7
|
+
* country-keyed tables through which the artifact declares facts about ITSELF, so those facts are
|
|
8
|
+
* updated at gazetteer REBUILD, not by a hand-edited code PR after someone remembers:
|
|
9
|
+
*
|
|
10
|
+
* - `country_coverage`: the hard-country-filter coverage record (#743/#194) — per-country
|
|
11
|
+
* promote-gate verdicts + the measured hard-resolve rates that used to live in a code comment on
|
|
12
|
+
* `HARD_PLACE_COUNTRY_SAFELIST`. Presence = measured; `hard_filter_safe = 0` = measured and
|
|
13
|
+
* FAILED the gate (FI 69.5%, PL 77.8%) — distinguishable from a country never measured at all
|
|
14
|
+
* (the meaning-of-zero rule, `docs/articles/plan/reference/layer-contract.mdx`).
|
|
15
|
+
* - `country_bbox`: the coarse guard-B plausibility boxes that used to live in
|
|
16
|
+
* `resolver/plausibility.ts`'s `COUNTRY_BBOX`. An absent row fails open (never trips the guard),
|
|
17
|
+
* exactly like an absent key in the constant.
|
|
18
|
+
*
|
|
19
|
+
* Emission happens at build time (`mailwoman/gazetteer-pipeline/coverage-manifest.ts` owns the
|
|
20
|
+
* measured record and calls {@link writeGazetteerCoverageManifest} before the DB is sealed — never
|
|
21
|
+
* patch a shipped DB, rebuild). The read happens at open time ({@link WOFCandidateTableLookup}
|
|
22
|
+
* calls {@link readGazetteerCoverageManifest} in its constructor); an artifact that predates the
|
|
23
|
+
* manifest returns `undefined` and every consumer falls back to the code constants byte-identically.
|
|
24
|
+
*/
|
|
25
|
+
import { hardCountrySafelistFromCoverage, } from "@mailwoman/core/resolver";
|
|
26
|
+
import { sql } from "kysely";
|
|
27
|
+
import { hasTable } from "./sqlite-utils.js";
|
|
28
|
+
/**
|
|
29
|
+
* Table names the lookup probes (existence-gated, so a candidate.db built before the manifest is byte-stable).
|
|
30
|
+
*/
|
|
31
|
+
export const COUNTRY_COVERAGE_TABLE = "country_coverage";
|
|
32
|
+
/**
|
|
33
|
+
* Table of per-country bounding boxes, used to reject a placement that fell outside its own country.
|
|
34
|
+
*/
|
|
35
|
+
export const COUNTRY_BBOX_TABLE = "country_bbox";
|
|
36
|
+
/**
|
|
37
|
+
* Create `country_coverage` — a handful of small PK-probed rows, the WITHOUT ROWID sweet spot.
|
|
38
|
+
*/
|
|
39
|
+
export async function createCountryCoverageTable(db) {
|
|
40
|
+
await db.schema
|
|
41
|
+
.createTable(COUNTRY_COVERAGE_TABLE)
|
|
42
|
+
.ifNotExists()
|
|
43
|
+
.addColumn("country", "text", (c) => c.primaryKey())
|
|
44
|
+
.addColumn("hard_filter_safe", "integer", (c) => c.notNull())
|
|
45
|
+
.addColumn("hard_resolve_rate", "real")
|
|
46
|
+
.addColumn("sample_size", "integer")
|
|
47
|
+
.addColumn("measured_at", "text", (c) => c.notNull())
|
|
48
|
+
.addColumn("source", "text", (c) => c.notNull())
|
|
49
|
+
// `WITHOUT ROWID` has no first-class builder; the raw modifier is the idiomatic fallback.
|
|
50
|
+
.modifyEnd(sql `without rowid`)
|
|
51
|
+
.execute();
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Create `country_bbox` — same shape discipline as {@link createCountryCoverageTable}.
|
|
55
|
+
*/
|
|
56
|
+
export async function createCountryBBoxTable(db) {
|
|
57
|
+
await db.schema
|
|
58
|
+
.createTable(COUNTRY_BBOX_TABLE)
|
|
59
|
+
.ifNotExists()
|
|
60
|
+
.addColumn("country", "text", (c) => c.primaryKey())
|
|
61
|
+
.addColumn("lat_min", "real", (c) => c.notNull())
|
|
62
|
+
.addColumn("lat_max", "real", (c) => c.notNull())
|
|
63
|
+
.addColumn("lon_min", "real", (c) => c.notNull())
|
|
64
|
+
.addColumn("lon_max", "real", (c) => c.notNull())
|
|
65
|
+
.addColumn("source", "text", (c) => c.notNull())
|
|
66
|
+
.modifyEnd(sql `without rowid`)
|
|
67
|
+
.execute();
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Write the coverage manifest into a candidate DB UNDER CONSTRUCTION (pre-seal — a shipped DB is never patched, rebuild
|
|
71
|
+
* instead). Creates both tables and inserts the facts; call exactly once, from the gazetteer build.
|
|
72
|
+
*/
|
|
73
|
+
export async function writeGazetteerCoverageManifest(db, facts) {
|
|
74
|
+
await createCountryCoverageTable(db);
|
|
75
|
+
await createCountryBBoxTable(db);
|
|
76
|
+
if (facts.coverage.length) {
|
|
77
|
+
await db
|
|
78
|
+
.insertInto(COUNTRY_COVERAGE_TABLE)
|
|
79
|
+
.values(facts.coverage.map((f) => ({
|
|
80
|
+
country: f.country.toUpperCase(),
|
|
81
|
+
hard_filter_safe: f.hardFilterSafe ? 1 : 0,
|
|
82
|
+
hard_resolve_rate: f.hardResolveRate ?? null,
|
|
83
|
+
sample_size: f.sampleSize ?? null,
|
|
84
|
+
measured_at: f.measuredAt,
|
|
85
|
+
source: f.source,
|
|
86
|
+
})))
|
|
87
|
+
.execute();
|
|
88
|
+
}
|
|
89
|
+
if (facts.bboxes.length) {
|
|
90
|
+
await db
|
|
91
|
+
.insertInto(COUNTRY_BBOX_TABLE)
|
|
92
|
+
.values(facts.bboxes.map((f) => ({
|
|
93
|
+
country: f.country.toUpperCase(),
|
|
94
|
+
lat_min: f.latMin,
|
|
95
|
+
lat_max: f.latMax,
|
|
96
|
+
lon_min: f.lonMin,
|
|
97
|
+
lon_max: f.lonMax,
|
|
98
|
+
source: f.source,
|
|
99
|
+
})))
|
|
100
|
+
.execute();
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Read the coverage manifest from an OPEN candidate DB, or `undefined` when the artifact predates it (neither table
|
|
105
|
+
* present) — the signal for consumers to fall back to the code constants byte-identically. Synchronous raw reads on
|
|
106
|
+
* purpose: this runs inside {@link WOFCandidateTableLookup}'s synchronous constructor (the sync-reader carve-out in
|
|
107
|
+
* `AGENTS.md`), and the tables are a few dozen rows read once per open.
|
|
108
|
+
*/
|
|
109
|
+
export function readGazetteerCoverageManifest(db) {
|
|
110
|
+
const hasCoverage = hasTable(db, COUNTRY_COVERAGE_TABLE);
|
|
111
|
+
const hasBBox = hasTable(db, COUNTRY_BBOX_TABLE);
|
|
112
|
+
if (!hasCoverage && !hasBBox)
|
|
113
|
+
return undefined;
|
|
114
|
+
const countryCoverage = new Map();
|
|
115
|
+
if (hasCoverage) {
|
|
116
|
+
const rows = db
|
|
117
|
+
.prepare(`SELECT country, hard_filter_safe, hard_resolve_rate, sample_size, measured_at, source FROM ${COUNTRY_COVERAGE_TABLE}`)
|
|
118
|
+
.all();
|
|
119
|
+
for (const row of rows) {
|
|
120
|
+
const country = String(row.country).toUpperCase();
|
|
121
|
+
countryCoverage.set(country, {
|
|
122
|
+
country,
|
|
123
|
+
hardFilterSafe: Number(row.hard_filter_safe) !== 0,
|
|
124
|
+
...(row.hard_resolve_rate === null ? {} : { hardResolveRate: Number(row.hard_resolve_rate) }),
|
|
125
|
+
...(row.sample_size === null ? {} : { sampleSize: Number(row.sample_size) }),
|
|
126
|
+
measuredAt: String(row.measured_at),
|
|
127
|
+
source: String(row.source),
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
const countryBBoxes = new Map();
|
|
132
|
+
if (hasBBox) {
|
|
133
|
+
const rows = db
|
|
134
|
+
.prepare(`SELECT country, lat_min, lat_max, lon_min, lon_max, source FROM ${COUNTRY_BBOX_TABLE}`)
|
|
135
|
+
.all();
|
|
136
|
+
for (const row of rows) {
|
|
137
|
+
const country = String(row.country).toUpperCase();
|
|
138
|
+
countryBBoxes.set(country, {
|
|
139
|
+
country,
|
|
140
|
+
latMin: Number(row.lat_min),
|
|
141
|
+
latMax: Number(row.lat_max),
|
|
142
|
+
lonMin: Number(row.lon_min),
|
|
143
|
+
lonMax: Number(row.lon_max),
|
|
144
|
+
source: String(row.source),
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
return {
|
|
149
|
+
countryCoverage,
|
|
150
|
+
countryBBoxes,
|
|
151
|
+
hardCountrySafelist: hardCountrySafelistFromCoverage(countryCoverage.values()),
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
//# sourceMappingURL=coverage-manifest-schema.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"coverage-manifest-schema.js","sourceRoot":"","sources":["../coverage-manifest-schema.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAIH,OAAO,EACN,+BAA+B,GAI/B,MAAM,0BAA0B,CAAA;AACjC,OAAO,EAAE,GAAG,EAAe,MAAM,QAAQ,CAAA;AAEzC,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAA;AA0D5C;;GAEG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,kBAAkB,CAAA;AACxD;;GAEG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,cAAc,CAAA;AAEhD;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,EAAqC;IACrF,MAAM,EAAE,CAAC,MAAM;SACb,WAAW,CAAC,sBAAsB,CAAC;SACnC,WAAW,EAAE;SACb,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC;SACnD,SAAS,CAAC,kBAAkB,EAAE,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;SAC5D,SAAS,CAAC,mBAAmB,EAAE,MAAM,CAAC;SACtC,SAAS,CAAC,aAAa,EAAE,SAAS,CAAC;SACnC,SAAS,CAAC,aAAa,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;SACpD,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;QAChD,0FAA0F;SACzF,SAAS,CAAC,GAAG,CAAA,eAAe,CAAC;SAC7B,OAAO,EAAE,CAAA;AACZ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,EAAqC;IACjF,MAAM,EAAE,CAAC,MAAM;SACb,WAAW,CAAC,kBAAkB,CAAC;SAC/B,WAAW,EAAE;SACb,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC;SACnD,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;SAChD,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;SAChD,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;SAChD,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;SAChD,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;SAC/C,SAAS,CAAC,GAAG,CAAA,eAAe,CAAC;SAC7B,OAAO,EAAE,CAAA;AACZ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,8BAA8B,CACnD,EAAqC,EACrC,KAAuF;IAEvF,MAAM,0BAA0B,CAAC,EAAE,CAAC,CAAA;IACpC,MAAM,sBAAsB,CAAC,EAAE,CAAC,CAAA;IAEhC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,EAAE;aACN,UAAU,CAAC,sBAAsB,CAAC;aAClC,MAAM,CACN,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC1B,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE;YAChC,gBAAgB,EAAE,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1C,iBAAiB,EAAE,CAAC,CAAC,eAAe,IAAI,IAAI;YAC5C,WAAW,EAAE,CAAC,CAAC,UAAU,IAAI,IAAI;YACjC,WAAW,EAAE,CAAC,CAAC,UAAU;YACzB,MAAM,EAAE,CAAC,CAAC,MAAM;SAChB,CAAC,CAAC,CACH;aACA,OAAO,EAAE,CAAA;IACZ,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACzB,MAAM,EAAE;aACN,UAAU,CAAC,kBAAkB,CAAC;aAC9B,MAAM,CACN,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACxB,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE;YAChC,OAAO,EAAE,CAAC,CAAC,MAAM;YACjB,OAAO,EAAE,CAAC,CAAC,MAAM;YACjB,OAAO,EAAE,CAAC,CAAC,MAAM;YACjB,OAAO,EAAE,CAAC,CAAC,MAAM;YACjB,MAAM,EAAE,CAAC,CAAC,MAAM;SAChB,CAAC,CAAC,CACH;aACA,OAAO,EAAE,CAAA;IACZ,CAAC;AACF,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,6BAA6B,CAAC,EAAgB;IAC7D,MAAM,WAAW,GAAG,QAAQ,CAAC,EAAE,EAAE,sBAAsB,CAAC,CAAA;IACxD,MAAM,OAAO,GAAG,QAAQ,CAAC,EAAE,EAAE,kBAAkB,CAAC,CAAA;IAEhD,IAAI,CAAC,WAAW,IAAI,CAAC,OAAO;QAAE,OAAO,SAAS,CAAA;IAE9C,MAAM,eAAe,GAAG,IAAI,GAAG,EAA+B,CAAA;IAE9D,IAAI,WAAW,EAAE,CAAC;QACjB,MAAM,IAAI,GAAG,EAAE;aACb,OAAO,CACP,8FAA8F,sBAAsB,EAAE,CACtH;aACA,GAAG,EAAuC,CAAA;QAE5C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAA;YAEjD,eAAe,CAAC,GAAG,CAAC,OAAO,EAAE;gBAC5B,OAAO;gBACP,cAAc,EAAE,MAAM,CAAC,GAAG,CAAC,gBAAgB,CAAC,KAAK,CAAC;gBAClD,GAAG,CAAC,GAAG,CAAC,iBAAiB,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,MAAM,CAAC,GAAG,CAAC,iBAAiB,CAAC,EAAE,CAAC;gBAC7F,GAAG,CAAC,GAAG,CAAC,WAAW,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC5E,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC;gBACnC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;aAC1B,CAAC,CAAA;QACH,CAAC;IACF,CAAC;IAED,MAAM,aAAa,GAAG,IAAI,GAAG,EAA2B,CAAA;IAExD,IAAI,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,GAAG,EAAE;aACb,OAAO,CAAC,mEAAmE,kBAAkB,EAAE,CAAC;aAChG,GAAG,EAAmC,CAAA;QAExC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAA;YAEjD,aAAa,CAAC,GAAG,CAAC,OAAO,EAAE;gBAC1B,OAAO;gBACP,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC;gBAC3B,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC;gBAC3B,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC;gBAC3B,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC;gBAC3B,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;aAC1B,CAAC,CAAA;QACH,CAAC;IACF,CAAC;IAED,OAAO;QACN,eAAe;QACf,aAAa;QACb,mBAAmB,EAAE,+BAA+B,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC;KAC9E,CAAA;AACF,CAAC"}
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
* continuation edges (`token.startsWith(partial)`). This is what a char-level typeahead
|
|
17
17
|
* needs; without it "new yor" returns nothing useful. (#587)
|
|
18
18
|
*/
|
|
19
|
-
import { FSTMatcher } from "./fst-matcher.ts";
|
|
19
|
+
import type { FSTMatcher } from "./fst-matcher.ts";
|
|
20
20
|
export interface AutocompleteResult {
|
|
21
21
|
query: string;
|
|
22
22
|
normalizedTokens: string[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fst-autocomplete.d.ts","sourceRoot":"","sources":["../fst-autocomplete.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,UAAU,
|
|
1
|
+
{"version":3,"file":"fst-autocomplete.d.ts","sourceRoot":"","sources":["../fst-autocomplete.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAIlD,MAAM,WAAW,kBAAkB;IAClC,KAAK,EAAE,MAAM,CAAA;IACb,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,KAAK,EAAE,MAAM,CAAA;IACb,WAAW,EAAE,sBAAsB,EAAE,CAAA;CACrC;AAED,MAAM,WAAW,sBAAsB;IACtC,IAAI,EAAE,MAAM,CAAA;IACZ,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,MAAM,CAAA;IACb,WAAW,EAAE,MAAM,EAAE,CAAA;IACrB,UAAU,EAAE,MAAM,CAAA;IAClB,gBAAgB,EAAE,MAAM,EAAE,CAAA;CAC1B;AAED,MAAM,WAAW,gBAAgB;IAChC,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B;;;;OAIG;IACH,YAAY,CAAC,EAAE,OAAO,CAAA;CACtB;AAsBD;;GAEG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE,gBAAqB,GAAG,kBAAkB,CA8E5G"}
|
package/out/fst-autocomplete.js
CHANGED
|
@@ -16,8 +16,10 @@
|
|
|
16
16
|
* continuation edges (`token.startsWith(partial)`). This is what a char-level typeahead
|
|
17
17
|
* needs; without it "new yor" returns nothing useful. (#587)
|
|
18
18
|
*/
|
|
19
|
-
import {
|
|
20
|
-
/**
|
|
19
|
+
import { normalizeTokens } from "./fst-matcher.js";
|
|
20
|
+
/**
|
|
21
|
+
* Max accepting entries collected per BFS branch — keeps one dense branch from starving the search.
|
|
22
|
+
*/
|
|
21
23
|
const PER_BRANCH = 4;
|
|
22
24
|
/**
|
|
23
25
|
* The top-`k` entries by importance (descending). Avoids sorting/allocating when `entries` is small.
|
|
@@ -25,7 +27,7 @@ const PER_BRANCH = 4;
|
|
|
25
27
|
function topByImportance(entries, k) {
|
|
26
28
|
if (entries.length <= k)
|
|
27
29
|
return [...entries];
|
|
28
|
-
return [...entries].
|
|
30
|
+
return [...entries].toSorted((a, b) => b.importance - a.importance).slice(0, k);
|
|
29
31
|
}
|
|
30
32
|
/**
|
|
31
33
|
* Autocomplete from the current prefix. Returns suggestions ranked importance-descending.
|
|
@@ -34,12 +36,12 @@ export function autocomplete(fst, query, opts = {}) {
|
|
|
34
36
|
const maxSuggestions = opts.maxSuggestions ?? 10;
|
|
35
37
|
const maxExpansionDepth = opts.maxExpansionDepth ?? 2;
|
|
36
38
|
const normalizedTokens = normalizeTokens(query);
|
|
37
|
-
if (normalizedTokens.length
|
|
39
|
+
if (!normalizedTokens.length) {
|
|
38
40
|
return { query, normalizedTokens: [], depth: 0, suggestions: [] };
|
|
39
41
|
}
|
|
40
42
|
const seen = new Map();
|
|
41
43
|
const queue = [];
|
|
42
|
-
let depth
|
|
44
|
+
let depth;
|
|
43
45
|
const match = fst.walk(normalizedTokens);
|
|
44
46
|
if (match) {
|
|
45
47
|
// COMPLETE-token prefix landed on a state. Seed at the match state (accepting + continuations).
|
|
@@ -54,8 +56,8 @@ export function autocomplete(fst, query, opts = {}) {
|
|
|
54
56
|
else {
|
|
55
57
|
// PARTIAL last token — walk the complete prefix, complete the partial by prefix-filtering edges.
|
|
56
58
|
const complete = normalizedTokens.slice(0, -1);
|
|
57
|
-
const partial = normalizedTokens
|
|
58
|
-
const prefixState = complete.length
|
|
59
|
+
const partial = normalizedTokens.at(-1);
|
|
60
|
+
const prefixState = !complete.length ? 0 : (fst.walk(complete)?.stateID ?? undefined);
|
|
59
61
|
if (prefixState === undefined) {
|
|
60
62
|
return { query, normalizedTokens, depth: 0, suggestions: [] };
|
|
61
63
|
}
|
|
@@ -75,7 +77,7 @@ export function autocomplete(fst, query, opts = {}) {
|
|
|
75
77
|
// branch contributes only its top PER_BRANCH places: a state like "new london" has dozens of
|
|
76
78
|
// accepting entries and would otherwise blow the budget before the BFS ever reaches "new york"
|
|
77
79
|
// (the "new" state has 311 continuations). Per-branch capping keeps the search broad. (#587)
|
|
78
|
-
while (queue.length
|
|
80
|
+
while (queue.length && seen.size < maxSuggestions * 4) {
|
|
79
81
|
const item = queue.shift();
|
|
80
82
|
if (item.depth > maxExpansionDepth)
|
|
81
83
|
continue;
|
|
@@ -88,7 +90,7 @@ export function autocomplete(fst, query, opts = {}) {
|
|
|
88
90
|
}
|
|
89
91
|
}
|
|
90
92
|
}
|
|
91
|
-
let suggestions = [...seen.values()].
|
|
93
|
+
let suggestions = [...seen.values()].toSorted((a, b) => b.importance - a.importance);
|
|
92
94
|
if (opts.dedupeByName) {
|
|
93
95
|
suggestions = dedupeByName(suggestions);
|
|
94
96
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fst-autocomplete.js","sourceRoot":"","sources":["../fst-autocomplete.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;
|
|
1
|
+
{"version":3,"file":"fst-autocomplete.js","sourceRoot":"","sources":["../fst-autocomplete.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAGH,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA;AAqClD;;GAEG;AACH,MAAM,UAAU,GAAG,CAAC,CAAA;AAEpB;;GAEG;AACH,SAAS,eAAe,CAAC,OAA8B,EAAE,CAAS;IACjE,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,CAAC,GAAG,OAAO,CAAC,CAAA;IAE5C,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;AAChF,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,GAAe,EAAE,KAAa,EAAE,OAAyB,EAAE;IACvF,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,IAAI,EAAE,CAAA;IAChD,MAAM,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,IAAI,CAAC,CAAA;IACrD,MAAM,gBAAgB,GAAG,eAAe,CAAC,KAAK,CAAC,CAAA;IAE/C,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,CAAC;QAC9B,OAAO,EAAE,KAAK,EAAE,gBAAgB,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkC,CAAA;IACtD,MAAM,KAAK,GAAc,EAAE,CAAA;IAC3B,IAAI,KAAa,CAAA;IAEjB,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;IAExC,IAAI,KAAK,EAAE,CAAC;QACX,gGAAgG;QAChG,KAAK,GAAG,KAAK,CAAC,KAAK,CAAA;QAEnB,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YAClD,aAAa,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;QAC5C,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YACrD,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAC1E,CAAC;IACF,CAAC;SAAM,CAAC;QACP,iGAAiG;QACjG,MAAM,QAAQ,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QAC9C,MAAM,OAAO,GAAG,gBAAgB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAA;QACxC,MAAM,WAAW,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,OAAO,IAAI,SAAS,CAAC,CAAA;QAErF,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;YAC/B,OAAO,EAAE,KAAK,EAAE,gBAAgB,EAAE,KAAK,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;QAC9D,CAAC;QAED,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAA;QAEvB,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE,CAAC;YACnD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC;gBAAE,SAAQ;YAE7C,uFAAuF;YACvF,KAAK,MAAM,KAAK,IAAI,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,UAAU,CAAC,EAAE,CAAC;gBAClF,aAAa,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAA;YAC9D,CAAC;YAED,oFAAoF;YACpF,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAC1E,CAAC;IACF,CAAC;IAED,+FAA+F;IAC/F,6FAA6F;IAC7F,+FAA+F;IAC/F,6FAA6F;IAC7F,OAAO,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,GAAG,cAAc,GAAG,CAAC,EAAE,CAAC;QACvD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAG,CAAA;QAE3B,IAAI,IAAI,CAAC,KAAK,GAAG,iBAAiB;YAAE,SAAQ;QAE5C,KAAK,MAAM,KAAK,IAAI,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,UAAU,CAAC,EAAE,CAAC;YAC9E,aAAa,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;QAC5D,CAAC;QAED,IAAI,IAAI,CAAC,KAAK,GAAG,iBAAiB,EAAE,CAAC;YACpC,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBACpD,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;YACvG,CAAC;QACF,CAAC;IACF,CAAC;IAED,IAAI,WAAW,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAA;IAEpF,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;QACvB,WAAW,GAAG,YAAY,CAAC,WAAW,CAAC,CAAA;IACxC,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,gBAAgB,EAAE,KAAK,EAAE,WAAW,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,EAAE,CAAA;AAC7F,CAAC;AAED,SAAS,aAAa,CACrB,IAAyC,EACzC,KAAiB,EACjB,UAAkB,EAClB,gBAA0B;IAE1B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IAEtC,IAAI,QAAQ,IAAI,QAAQ,CAAC,UAAU,IAAI,UAAU;QAAE,OAAM;IAEzD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE;QACrB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,UAAU;QACV,gBAAgB,EAAE,CAAC,GAAG,gBAAgB,CAAC;KACvC,CAAC,CAAA;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,YAAY,CAAC,WAAqC;IAC1D,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAA;IACnC,MAAM,GAAG,GAA6B,EAAE,CAAA;IAExC,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAA;QAEhC,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAQ;QAChC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAClB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACZ,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC"}
|
package/out/fst-builder.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fst-builder.d.ts","sourceRoot":"","sources":["../fst-builder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAKH,OAAO,EAAE,UAAU,EAAmB,MAAM,kBAAkB,CAAA;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,EAA2B,MAAM,gBAAgB,CAAA;
|
|
1
|
+
{"version":3,"file":"fst-builder.d.ts","sourceRoot":"","sources":["../fst-builder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAKH,OAAO,EAAE,UAAU,EAAmB,MAAM,kBAAkB,CAAA;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,aAAa,EAA2B,MAAM,gBAAgB,CAAA;AAoC1G,wBAAgB,eAAe,CAAC,IAAI,EAAE,YAAY,GAAG;IACpD,OAAO,EAAE,UAAU,CAAA;IACnB,UAAU,EAAE,aAAa,CAAA;IACzB,MAAM,EAAE,cAAc,CAAA;CACtB,CAiSA"}
|
package/out/fst-builder.js
CHANGED
|
@@ -112,7 +112,7 @@ export function buildFSTFromWOF(opts) {
|
|
|
112
112
|
const popStmt = db.prepare("SELECT id, population FROM place_population");
|
|
113
113
|
const popRows = popStmt.all();
|
|
114
114
|
for (const row of popRows) {
|
|
115
|
-
const normalized = row.population > 0 ? Math.min(1
|
|
115
|
+
const normalized = row.population > 0 ? Math.min(1, Math.log2(1 + row.population / 1000) / 14) : 0;
|
|
116
116
|
importanceMap.set(row.id, normalized);
|
|
117
117
|
}
|
|
118
118
|
}
|
|
@@ -146,9 +146,31 @@ export function buildFSTFromWOF(opts) {
|
|
|
146
146
|
// Phase 5: Build the trie.
|
|
147
147
|
progress("trie", "Building trie");
|
|
148
148
|
const nodes = [{ edges: new Map(), places: [] }];
|
|
149
|
+
// Degenerate-surface curation (see BuildFSTOpts.excludeSurfaces). Applied to the WHOLE normalized
|
|
150
|
+
// surface only — a multi-token name containing a function word ("los angeles") is never affected.
|
|
151
|
+
const excludeSurfaces = opts.excludeSurfaces;
|
|
152
|
+
const excludeAllTokensOf = opts.excludeAllTokensOf;
|
|
153
|
+
let excludedCount = 0;
|
|
154
|
+
function isDegenerate(tokens) {
|
|
155
|
+
if (!tokens.length)
|
|
156
|
+
return false;
|
|
157
|
+
if (excludeSurfaces?.has(tokens.join(" ")))
|
|
158
|
+
return true;
|
|
159
|
+
if (excludeAllTokensOf !== undefined && tokens.every((t) => excludeAllTokensOf.has(t)))
|
|
160
|
+
return true;
|
|
161
|
+
return false;
|
|
162
|
+
}
|
|
163
|
+
// Surface-ambiguity classes (survey #4): a per-SURFACE fact, so the entry is cloned per insertion
|
|
164
|
+
// with its accepting surface's count attached (the same place under "nyc" and "new york city"
|
|
165
|
+
// records each surface's own ambiguity). Absent map → entries carry no count (back-compat bytes).
|
|
166
|
+
const surfaceCountryCounts = opts.surfaceCountryCounts;
|
|
149
167
|
function insertName(tokens, entry) {
|
|
150
|
-
if (tokens.length
|
|
151
|
-
return;
|
|
168
|
+
if (!tokens.length)
|
|
169
|
+
return false;
|
|
170
|
+
if (isDegenerate(tokens)) {
|
|
171
|
+
excludedCount++;
|
|
172
|
+
return false;
|
|
173
|
+
}
|
|
152
174
|
let stateID = 0;
|
|
153
175
|
for (const t of tokens) {
|
|
154
176
|
const node = nodes[stateID];
|
|
@@ -163,8 +185,15 @@ export function buildFSTFromWOF(opts) {
|
|
|
163
185
|
// Deduplicate: don't add the same wofID twice at the same state.
|
|
164
186
|
const existing = nodes[stateID].places;
|
|
165
187
|
if (!existing.some((p) => p.wofID === entry.wofID && p.placetype === entry.placetype)) {
|
|
166
|
-
|
|
188
|
+
if (surfaceCountryCounts !== undefined) {
|
|
189
|
+
const count = surfaceCountryCounts.get(tokens.join(" "));
|
|
190
|
+
existing.push({ ...entry, crossCountryBranches: Math.min(count ?? 1, 255) });
|
|
191
|
+
}
|
|
192
|
+
else {
|
|
193
|
+
existing.push(entry);
|
|
194
|
+
}
|
|
167
195
|
}
|
|
196
|
+
return true;
|
|
168
197
|
}
|
|
169
198
|
let insertCount = 0;
|
|
170
199
|
for (const row of sprRows) {
|
|
@@ -180,22 +209,23 @@ export function buildFSTFromWOF(opts) {
|
|
|
180
209
|
};
|
|
181
210
|
// Insert the primary name from spr.
|
|
182
211
|
const primaryTokens = normalizeTokens(row.name);
|
|
183
|
-
insertName(primaryTokens, entry)
|
|
184
|
-
|
|
212
|
+
if (insertName(primaryTokens, entry)) {
|
|
213
|
+
insertCount++;
|
|
214
|
+
}
|
|
185
215
|
// Insert alt names from the names table.
|
|
186
216
|
const altNames = namesByPlace.get(row.id) ?? [];
|
|
187
217
|
for (const altName of altNames) {
|
|
188
218
|
if (altName === row.name)
|
|
189
219
|
continue;
|
|
190
220
|
const altTokens = normalizeTokens(altName);
|
|
191
|
-
if (altTokens.length
|
|
192
|
-
insertName(altTokens, entry);
|
|
221
|
+
if (altTokens.length && altTokens.join(" ") !== primaryTokens.join(" ") && insertName(altTokens, entry)) {
|
|
193
222
|
insertCount++;
|
|
194
223
|
}
|
|
195
224
|
}
|
|
196
225
|
}
|
|
197
226
|
db.close();
|
|
198
|
-
progress("done", `Built trie: ${nodes.length} states, ${insertCount} name insertions`
|
|
227
|
+
progress("done", `Built trie: ${nodes.length} states, ${insertCount} name insertions` +
|
|
228
|
+
(excludedCount > 0 ? ` (${excludedCount} degenerate surfaces excluded)` : ""));
|
|
199
229
|
const edgeCount = nodes.reduce((sum, n) => sum + n.edges.size, 0);
|
|
200
230
|
const matcher = FSTMatcher.fromNodes(nodes);
|
|
201
231
|
const provenance = {
|
|
@@ -207,6 +237,9 @@ export function buildFSTFromWOF(opts) {
|
|
|
207
237
|
nameInsertions: insertCount,
|
|
208
238
|
importanceMatches: importanceMap.size,
|
|
209
239
|
sourceDB: opts.dbPath,
|
|
240
|
+
...(excludeSurfaces !== undefined || excludeAllTokensOf !== undefined
|
|
241
|
+
? { exclusionPolicy: opts.exclusionPolicy ?? "unspecified", excludedInsertions: excludedCount }
|
|
242
|
+
: {}),
|
|
210
243
|
};
|
|
211
244
|
return {
|
|
212
245
|
matcher,
|
package/out/fst-builder.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fst-builder.js","sourceRoot":"","sources":["../fst-builder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAG1C,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA;AAG9D,MAAM,kBAAkB,GAAkB;IACzC,SAAS;IACT,QAAQ;IACR,QAAQ;IACR,UAAU;IACV,YAAY;IACZ,SAAS;IACT,eAAe;CACf,CAAA;
|
|
1
|
+
{"version":3,"file":"fst-builder.js","sourceRoot":"","sources":["../fst-builder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAG1C,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA;AAG9D,MAAM,kBAAkB,GAAkB;IACzC,SAAS;IACT,QAAQ;IACR,QAAQ;IACR,UAAU;IACV,YAAY;IACZ,SAAS;IACT,eAAe;CACf,CAAA;AAED,MAAM,iBAAiB,GAAG,CAAC,IAAI,CAAC,CAAA;AAChC,MAAM,iBAAiB,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;AAuBrC,MAAM,UAAU,eAAe,CAAC,IAAkB;IAKjD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,iBAAiB,CAAA;IACrD,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,kBAAkB,CAAA;IACxD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,iBAAiB,CAAA;IACrD,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;IAE9C,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IAC7B,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;IAExD,uCAAuC;IACvC,QAAQ,CAAC,KAAK,EAAE,iCAAiC,SAAS,kBAAkB,UAAU,GAAG,CAAC,CAAA;IAC1F,MAAM,YAAY,GAAG,CAAC,GAAa,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAEpE,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,CACzB;;;uBAGqB,YAAY,CAAC,SAAS,CAAC;yBACrB,YAAY,CAAC,UAAU,CAAC,GAAG,CAClD,CAAA;IAED,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,SAAS,EAAE,GAAG,UAAU,CAAwB,CAAA;IAC/E,QAAQ,CAAC,KAAK,EAAE,UAAU,OAAO,CAAC,MAAM,SAAS,CAAC,CAAA;IAElD,uDAAuD;IACvD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAA;IAEzC,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC3B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAA;IACzB,CAAC;IAED,+FAA+F;IAC/F,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAAC,kFAAkF,CAAC,CAAA;IAEjH,6EAA6E;IAC7E,IAAI,YAAY,GAAyC,IAAI,CAAA;IAE7D,IAAI,CAAC;QACJ,YAAY,GAAG,EAAE,CAAC,OAAO,CACxB;;;;;;QAMK,CACL,CAAA;IACF,CAAC;IAAC,MAAM,CAAC;QACR,QAAQ,CAAC,WAAW,EAAE,oEAAoE,CAAC,CAAA;IAC5F,CAAC;IAED,SAAS,kBAAkB,CAAC,EAAU;QACrC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAE3B,IAAI,CAAC,GAAG;YAAE,OAAO,EAAE,CAAA;QAEnB,yDAAyD;QACzD,IAAI,GAAG,CAAC,SAAS,IAAI,CAAC,IAAI,YAAY,EAAE,CAAC;YACxC,MAAM,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC,EAAE,CAA8C,CAAA;YAEnF,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,CAAA;QACvE,CAAC;QAED,qCAAqC;QACrC,MAAM,KAAK,GAAa,EAAE,CAAA;QAC1B,IAAI,OAAO,GAAG,GAAG,CAAC,SAAS,CAAA;QAC3B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAS,CAAC,EAAE,CAAC,CAAC,CAAA;QAElC,OAAO,OAAO,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC1C,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;YACjB,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YACnB,IAAI,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;YAEpC,IAAI,CAAC,SAAS,EAAE,CAAC;gBAChB,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,OAAO,CAAkC,CAAA;gBAExE,IAAI,CAAC,OAAO;oBAAE,MAAK;gBACnB,SAAS,GAAG,OAAO,CAAA;gBACnB,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,SAAS,CAAC,CAAA;YAChC,CAAC;YAED,IAAI,SAAS,CAAC,SAAS,GAAG,CAAC,IAAI,SAAS,CAAC,SAAS,KAAK,OAAO,EAAE,CAAC;gBAChE,OAAO,GAAG,SAAS,CAAC,SAAS,CAAA;YAC9B,CAAC;iBAAM,CAAC;gBACP,MAAK;YACN,CAAC;QACF,CAAC;QAED,OAAO,KAAK,CAAA;IACb,CAAC;IAED,oFAAoF;IACpF,sFAAsF;IACtF,QAAQ,CAAC,YAAY,EAAE,yBAAyB,CAAC,CAAA;IACjD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAA;IAE/C,IAAI,CAAC;QACJ,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,CAAC,6CAA6C,CAAC,CAAA;QACzE,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,EAA0D,CAAA;QAErF,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC3B,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,UAAU,CAAC,CAAA;QAC1C,CAAC;QAED,QAAQ,CAAC,YAAY,EAAE,UAAU,aAAa,CAAC,IAAI,oBAAoB,CAAC,CAAA;IACzE,CAAC;IAAC,MAAM,CAAC;QACR,QAAQ,CAAC,YAAY,EAAE,wDAAwD,CAAC,CAAA;QAEhF,IAAI,CAAC;YACJ,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,CAAC,6CAA6C,CAAC,CAAA;YACzE,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,EAAgC,CAAA;YAE3D,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;gBAC3B,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;gBAClG,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,UAAU,CAAC,CAAA;YACtC,CAAC;QACF,CAAC;QAAC,MAAM,CAAC;YACR,QAAQ,CAAC,YAAY,EAAE,8CAA8C,CAAC,CAAA;QACvE,CAAC;IACF,CAAC;IAED,2CAA2C;IAC3C,QAAQ,CAAC,OAAO,EAAE,uBAAuB,CAAC,CAAA;IAC1C,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IACzC,MAAM,YAAY,GAAG,IAAI,GAAG,EAAoB,CAAA;IAEhD,MAAM,YAAY,GAAG,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;IAE5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC;QAC/C,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAA;QACxC,MAAM,cAAc,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAErD,MAAM,QAAQ,GAAG,YAAY;YAC5B,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,iEAAiE,cAAc,GAAG,CAAC;YAChG,CAAC,CAAC,EAAE,CAAC,OAAO,CACV,iEAAiE,cAAc,sBAAsB,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAC1I,CAAA;QAEH,MAAM,QAAQ,GAAG,CAAC,YAAY;YAC7B,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;YACxB,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,KAAK,EAAE,GAAG,SAAS,CAAC,CAAyB,CAAA;QAEhE,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,CAAA;YAE/C,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAClC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;YACxB,CAAC;YAED,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAA;QACnC,CAAC;IACF,CAAC;IAED,QAAQ,CAAC,OAAO,EAAE,oBAAoB,YAAY,CAAC,IAAI,SAAS,CAAC,CAAA;IAEjE,2BAA2B;IAC3B,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAC,CAAA;IACjC,MAAM,KAAK,GAAc,CAAC,EAAE,KAAK,EAAE,IAAI,GAAG,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAA;IAE3D,kGAAkG;IAClG,kGAAkG;IAClG,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,CAAA;IAC5C,MAAM,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAA;IAClD,IAAI,aAAa,GAAG,CAAC,CAAA;IAErB,SAAS,YAAY,CAAC,MAAgB;QACrC,IAAI,CAAC,MAAM,CAAC,MAAM;YAAE,OAAO,KAAK,CAAA;QAEhC,IAAI,eAAe,EAAE,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAAE,OAAO,IAAI,CAAA;QAEvD,IAAI,kBAAkB,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAAE,OAAO,IAAI,CAAA;QAEnG,OAAO,KAAK,CAAA;IACb,CAAC;IAED,kGAAkG;IAClG,8FAA8F;IAC9F,kGAAkG;IAClG,MAAM,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAA;IAEtD,SAAS,UAAU,CAAC,MAAgB,EAAE,KAAiB;QACtD,IAAI,CAAC,MAAM,CAAC,MAAM;YAAE,OAAO,KAAK,CAAA;QAEhC,IAAI,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1B,aAAa,EAAE,CAAA;YAEf,OAAO,KAAK,CAAA;QACb,CAAC;QAED,IAAI,OAAO,GAAG,CAAC,CAAA;QAEf,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACxB,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAE,CAAA;YAC5B,IAAI,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YAE5B,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;gBACxB,IAAI,GAAG,KAAK,CAAC,MAAM,CAAA;gBACnB,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,GAAG,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAA;gBAC5C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAA;YACxB,CAAC;YAED,OAAO,GAAG,IAAI,CAAA;QACf,CAAC;QAED,iEAAiE;QACjE,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAE,CAAC,MAAM,CAAA;QAEvC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;YACvF,IAAI,oBAAoB,KAAK,SAAS,EAAE,CAAC;gBACxC,MAAM,KAAK,GAAG,oBAAoB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;gBACxD,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,EAAE,oBAAoB,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;YAC7E,CAAC;iBAAM,CAAC;gBACP,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACrB,CAAC;QACF,CAAC;QAED,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,IAAI,WAAW,GAAG,CAAC,CAAA;IAEnB,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,WAAW,GAAG,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAE9C,MAAM,KAAK,GAAe;YACzB,KAAK,EAAE,GAAG,CAAC,EAAE;YACb,SAAS,EAAE,GAAG,CAAC,SAAwB;YACvC,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,WAAW;YACX,UAAU,EAAE,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC;YAC1C,GAAG,EAAE,GAAG,CAAC,QAAQ;YACjB,GAAG,EAAE,GAAG,CAAC,SAAS;SAClB,CAAA;QAED,oCAAoC;QACpC,MAAM,aAAa,GAAG,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAE/C,IAAI,UAAU,CAAC,aAAa,EAAE,KAAK,CAAC,EAAE,CAAC;YACtC,WAAW,EAAE,CAAA;QACd,CAAC;QAED,yCAAyC;QACzC,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,CAAA;QAE/C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAChC,IAAI,OAAO,KAAK,GAAG,CAAC,IAAI;gBAAE,SAAQ;YAClC,MAAM,SAAS,GAAG,eAAe,CAAC,OAAO,CAAC,CAAA;YAE1C,IAAI,SAAS,CAAC,MAAM,IAAI,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,CAAC;gBACzG,WAAW,EAAE,CAAA;YACd,CAAC;QACF,CAAC;IACF,CAAC;IAED,EAAE,CAAC,KAAK,EAAE,CAAA;IAEV,QAAQ,CACP,MAAM,EACN,eAAe,KAAK,CAAC,MAAM,YAAY,WAAW,kBAAkB;QACnE,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,aAAa,gCAAgC,CAAC,CAAC,CAAC,EAAE,CAAC,CAC9E,CAAA;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA;IACjE,MAAM,OAAO,GAAG,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IAE3C,MAAM,UAAU,GAAkB;QACjC,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACjC,SAAS;QACT,UAAU,EAAE,KAAK,CAAC,MAAM;QACxB,UAAU,EAAE,OAAO,CAAC,MAAM;QAC1B,SAAS;QACT,cAAc,EAAE,WAAW;QAC3B,iBAAiB,EAAE,aAAa,CAAC,IAAI;QACrC,QAAQ,EAAE,IAAI,CAAC,MAAM;QACrB,GAAG,CAAC,eAAe,KAAK,SAAS,IAAI,kBAAkB,KAAK,SAAS;YACpE,CAAC,CAAC,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,IAAI,aAAa,EAAE,kBAAkB,EAAE,aAAa,EAAE;YAC/F,CAAC,CAAC,EAAE,CAAC;KACN,CAAA;IAED,OAAO;QACN,OAAO;QACP,UAAU;QACV,MAAM,EAAE;YACP,UAAU,EAAE,KAAK,CAAC,MAAM;YACxB,UAAU,EAAE,OAAO,CAAC,MAAM;YAC1B,SAAS;YACT,UAAU,EAAE,WAAW;SACvB;KACD,CAAA;AACF,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fst-deserialize-web.d.ts","sourceRoot":"","sources":["../fst-deserialize-web.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAC7C,OAAO,KAAK,EAAE,aAAa,EAA2B,MAAM,gBAAgB,CAAA;
|
|
1
|
+
{"version":3,"file":"fst-deserialize-web.d.ts","sourceRoot":"","sources":["../fst-deserialize-web.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAC7C,OAAO,KAAK,EAAE,aAAa,EAA2B,MAAM,gBAAgB,CAAA;AAmD5E,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,WAAW,GAAG,UAAU,GAAG,UAAU,CAgH7E;AAED,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,WAAW,GAAG,UAAU,GAAG,aAAa,GAAG,SAAS,CAqB/F"}
|
|
@@ -10,13 +10,35 @@
|
|
|
10
10
|
* build-time operation).
|
|
11
11
|
*/
|
|
12
12
|
import { FSTMatcher } from "./fst-matcher.js";
|
|
13
|
+
/**
|
|
14
|
+
* Format version that widened the per-state edge and place counters from 16 to 32 bits, growing the state entry from 12
|
|
15
|
+
* to 16 bytes. Readers branch on it to stay backward-compatible with v2/v3 files.
|
|
16
|
+
*/
|
|
17
|
+
const VERSION_WIDE_STATE_COUNTERS = 4;
|
|
18
|
+
/**
|
|
19
|
+
* State-table entry size in bytes at or above {@link VERSION_WIDE_STATE_COUNTERS}.
|
|
20
|
+
*/
|
|
21
|
+
const WIDE_STATE_ENTRY_SIZE = 16;
|
|
22
|
+
/**
|
|
23
|
+
* State-table entry size in bytes below {@link VERSION_WIDE_STATE_COUNTERS}.
|
|
24
|
+
*/
|
|
25
|
+
const NARROW_STATE_ENTRY_SIZE = 12;
|
|
26
|
+
/**
|
|
27
|
+
* First format version carrying the trailing metadata block; older files simply have none.
|
|
28
|
+
*/
|
|
29
|
+
const VERSION_WITH_METADATA = 3;
|
|
13
30
|
const HEADER_SIZE = 32;
|
|
14
31
|
const EDGE_ENTRY_SIZE = 8;
|
|
15
32
|
const PLACE_ENTRY_SIZE = 56;
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
33
|
+
/**
|
|
34
|
+
* "FST\0".
|
|
35
|
+
*/
|
|
36
|
+
const MAGIC_BYTES = [0x46, 0x53, 0x54, 0x00];
|
|
37
|
+
/**
|
|
38
|
+
* Must track the serializer's VERSION (fst-serialize.ts, currently 4). The v3 provenance + v4 16-byte-state/u32-count
|
|
39
|
+
* layout logic below already matches the Node deserializer; only this gate was left stale at 2, so the browser FST
|
|
40
|
+
* loader rejected every real (v4) artifact.
|
|
41
|
+
*/
|
|
20
42
|
const MAX_VERSION = 4;
|
|
21
43
|
const PLACETYPE_ORDER = [
|
|
22
44
|
"country",
|
|
@@ -48,6 +70,8 @@ export function deserializeFSTWeb(input) {
|
|
|
48
70
|
throw new Error(`FST version ${version} unsupported (expected 1..${MAX_VERSION})`);
|
|
49
71
|
}
|
|
50
72
|
const isV2 = version >= 2;
|
|
73
|
+
// flags bit0 (survey #4, mirrors fst-serialize.ts): place rows carry surface-ambiguity data.
|
|
74
|
+
const hasAmbiguity = (view.getUint16(6, true) & 1) === 1;
|
|
51
75
|
const stateCount = view.getUint32(8, true);
|
|
52
76
|
const edgeCount = view.getUint32(12, true);
|
|
53
77
|
const _placeCount = view.getUint32(16, true);
|
|
@@ -69,7 +93,7 @@ export function deserializeFSTWeb(input) {
|
|
|
69
93
|
}
|
|
70
94
|
pos += stringBytes;
|
|
71
95
|
// --- State table ---
|
|
72
|
-
const stateEntrySize = version >=
|
|
96
|
+
const stateEntrySize = version >= VERSION_WIDE_STATE_COUNTERS ? WIDE_STATE_ENTRY_SIZE : NARROW_STATE_ENTRY_SIZE;
|
|
73
97
|
const stateTableStart = pos;
|
|
74
98
|
const edgeTableStart = stateTableStart + stateCount * stateEntrySize;
|
|
75
99
|
const placeTableStart = edgeTableStart + edgeCount * EDGE_ENTRY_SIZE;
|
|
@@ -78,8 +102,8 @@ export function deserializeFSTWeb(input) {
|
|
|
78
102
|
const sp = stateTableStart + si * stateEntrySize;
|
|
79
103
|
const edgeStart = view.getUint32(sp, true);
|
|
80
104
|
const placeStart = view.getUint32(sp + 4, true);
|
|
81
|
-
const edgeCountForState = version >=
|
|
82
|
-
const placeCountForState = version >=
|
|
105
|
+
const edgeCountForState = version >= VERSION_WIDE_STATE_COUNTERS ? view.getUint32(sp + 8, true) : view.getUint16(sp + 8, true);
|
|
106
|
+
const placeCountForState = version >= VERSION_WIDE_STATE_COUNTERS ? view.getUint32(sp + 12, true) : view.getUint16(sp + 10, true);
|
|
83
107
|
const edges = new Map();
|
|
84
108
|
for (let ei = 0; ei < edgeCountForState; ei++) {
|
|
85
109
|
const ep = edgeTableStart + (edgeStart + ei) * EDGE_ENTRY_SIZE;
|
|
@@ -97,7 +121,7 @@ export function deserializeFSTWeb(input) {
|
|
|
97
121
|
}
|
|
98
122
|
const rawImportance = isV2
|
|
99
123
|
? view.getFloat32(pp + 12, true)
|
|
100
|
-
: Math.min(1
|
|
124
|
+
: Math.min(1, Math.log2(1 + view.getUint32(pp + 12, true) / 1000) / 14);
|
|
101
125
|
places[pi] = {
|
|
102
126
|
wofID: view.getUint32(pp, true),
|
|
103
127
|
placetype: PLACETYPE_ORDER[view.getUint8(pp + 4)] ?? "locality",
|
|
@@ -106,6 +130,7 @@ export function deserializeFSTWeb(input) {
|
|
|
106
130
|
lat: view.getFloat32(pp + 16, true),
|
|
107
131
|
lon: view.getFloat32(pp + 20, true),
|
|
108
132
|
parentChain,
|
|
133
|
+
...(hasAmbiguity ? { crossCountryBranches: view.getUint8(pp + 6) } : {}),
|
|
109
134
|
};
|
|
110
135
|
}
|
|
111
136
|
nodes[si] = { edges, places };
|
|
@@ -119,7 +144,7 @@ export function readFSTProvenanceWeb(input) {
|
|
|
119
144
|
if (bytes.byteLength < HEADER_SIZE)
|
|
120
145
|
return undefined;
|
|
121
146
|
const version = view.getUint16(4, true);
|
|
122
|
-
if (version <
|
|
147
|
+
if (version < VERSION_WITH_METADATA)
|
|
123
148
|
return undefined;
|
|
124
149
|
const provenanceOffset = view.getUint32(28, true);
|
|
125
150
|
if (provenanceOffset === 0 || provenanceOffset >= bytes.byteLength)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fst-deserialize-web.js","sourceRoot":"","sources":["../fst-deserialize-web.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAG7C,MAAM,WAAW,GAAG,EAAE,CAAA;AACtB,MAAM,eAAe,GAAG,CAAC,CAAA;AACzB,MAAM,gBAAgB,GAAG,EAAE,CAAA;AAC3B,MAAM,WAAW,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAA
|
|
1
|
+
{"version":3,"file":"fst-deserialize-web.js","sourceRoot":"","sources":["../fst-deserialize-web.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAG7C;;;GAGG;AACH,MAAM,2BAA2B,GAAG,CAAC,CAAA;AAErC;;GAEG;AACH,MAAM,qBAAqB,GAAG,EAAE,CAAA;AAEhC;;GAEG;AACH,MAAM,uBAAuB,GAAG,EAAE,CAAA;AAElC;;GAEG;AACH,MAAM,qBAAqB,GAAG,CAAC,CAAA;AAE/B,MAAM,WAAW,GAAG,EAAE,CAAA;AACtB,MAAM,eAAe,GAAG,CAAC,CAAA;AACzB,MAAM,gBAAgB,GAAG,EAAE,CAAA;AAC3B;;GAEG;AACH,MAAM,WAAW,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAA;AAC5C;;;;GAIG;AACH,MAAM,WAAW,GAAG,CAAC,CAAA;AAErB,MAAM,eAAe,GAA2B;IAC/C,SAAS;IACT,QAAQ;IACR,QAAQ;IACR,UAAU;IACV,YAAY;IACZ,SAAS;IACT,eAAe;IACf,YAAY;IACZ,QAAQ;IACR,YAAY;IACZ,cAAc;CACd,CAAA;AAED,MAAM,UAAU,iBAAiB,CAAC,KAA+B;IAChE,MAAM,KAAK,GAAG,KAAK,YAAY,WAAW,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAA;IAC1E,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,CAAC,CAAA;IAC3E,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,OAAO,CAAC,CAAA;IAExC,IAAI,KAAK,CAAC,UAAU,GAAG,WAAW;QAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAA;IAEtF,IACC,KAAK,CAAC,CAAC,CAAC,KAAK,WAAW,CAAC,CAAC,CAAC;QAC3B,KAAK,CAAC,CAAC,CAAC,KAAK,WAAW,CAAC,CAAC,CAAC;QAC3B,KAAK,CAAC,CAAC,CAAC,KAAK,WAAW,CAAC,CAAC,CAAC;QAC3B,KAAK,CAAC,CAAC,CAAC,KAAK,WAAW,CAAC,CAAC,CAAC,EAC1B,CAAC;QACF,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAA;IACtC,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAA;IAEvC,IAAI,OAAO,GAAG,CAAC,IAAI,OAAO,GAAG,WAAW,EAAE,CAAC;QAC1C,MAAM,IAAI,KAAK,CAAC,eAAe,OAAO,6BAA6B,WAAW,GAAG,CAAC,CAAA;IACnF,CAAC;IAED,MAAM,IAAI,GAAG,OAAO,IAAI,CAAC,CAAA;IACzB,6FAA6F;IAC7F,MAAM,YAAY,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAA;IAExD,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAA;IAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,CAAC,CAAA;IAC1C,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,CAAC,CAAA;IAC5C,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,CAAC,CAAA;IAC5C,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,CAAC,CAAA;IAE5C,IAAI,GAAG,GAAG,WAAW,CAAA;IAErB,uBAAuB;IACvB,MAAM,UAAU,GAAG,IAAI,WAAW,CAAC,WAAW,GAAG,CAAC,CAAC,CAAA;IAEnD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,UAAU,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;QACzC,GAAG,IAAI,CAAC,CAAA;IACT,CAAC;IAED,MAAM,YAAY,GAAG,GAAG,CAAA;IACxB,MAAM,OAAO,GAAa,IAAI,KAAK,CAAC,WAAW,CAAC,CAAA;IAEhD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,YAAY,GAAG,UAAU,CAAC,CAAC,CAAE,CAAA;QAC3C,MAAM,GAAG,GAAG,YAAY,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,CAAE,CAAA;QAC7C,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAA;IACxD,CAAC;IAED,GAAG,IAAI,WAAW,CAAA;IAElB,sBAAsB;IACtB,MAAM,cAAc,GAAG,OAAO,IAAI,2BAA2B,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,uBAAuB,CAAA;IAC/G,MAAM,eAAe,GAAG,GAAG,CAAA;IAC3B,MAAM,cAAc,GAAG,eAAe,GAAG,UAAU,GAAG,cAAc,CAAA;IACpE,MAAM,eAAe,GAAG,cAAc,GAAG,SAAS,GAAG,eAAe,CAAA;IAEpE,MAAM,KAAK,GAAc,IAAI,KAAK,CAAC,UAAU,CAAC,CAAA;IAE9C,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,UAAU,EAAE,EAAE,EAAE,EAAE,CAAC;QACxC,MAAM,EAAE,GAAG,eAAe,GAAG,EAAE,GAAG,cAAc,CAAA;QAChD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,CAAC,CAAA;QAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,CAAC,CAAA;QAE/C,MAAM,iBAAiB,GACtB,OAAO,IAAI,2BAA2B,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,CAAC,CAAA;QAErG,MAAM,kBAAkB,GACvB,OAAO,IAAI,2BAA2B,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,EAAE,IAAI,CAAC,CAAA;QAEvG,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAA;QAEvC,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,iBAAiB,EAAE,EAAE,EAAE,EAAE,CAAC;YAC/C,MAAM,EAAE,GAAG,cAAc,GAAG,CAAC,SAAS,GAAG,EAAE,CAAC,GAAG,eAAe,CAAA;YAC9D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,CAAC,CAAA;YAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,CAAC,CAAA;YAC3C,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAE,EAAE,MAAM,CAAC,CAAA;QACvC,CAAC;QAED,MAAM,MAAM,GAAiB,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAA;QAE1D,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,kBAAkB,EAAE,EAAE,EAAE,EAAE,CAAC;YAChD,MAAM,EAAE,GAAG,eAAe,GAAG,CAAC,UAAU,GAAG,EAAE,CAAC,GAAG,gBAAgB,CAAA;YACjE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,CAAC,CAAA;YACtC,MAAM,WAAW,GAAa,EAAE,CAAA;YAEhC,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,QAAQ,EAAE,EAAE,EAAE,EAAE,CAAC;gBACtC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,CAAA;YACzD,CAAC;YAED,MAAM,aAAa,GAAG,IAAI;gBACzB,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,EAAE,IAAI,CAAC;gBAChC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,EAAE,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;YAExE,MAAM,CAAC,EAAE,CAAC,GAAG;gBACZ,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,CAAC;gBAC/B,SAAS,EAAE,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,UAAU;gBAC/D,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,CAAC,CAAE;gBAC5C,UAAU,EAAE,aAAa;gBACzB,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,EAAE,IAAI,CAAC;gBACnC,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,EAAE,IAAI,CAAC;gBACnC,WAAW;gBACX,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,oBAAoB,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACxE,CAAA;QACF,CAAC;QAED,KAAK,CAAC,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,CAAA;IAC9B,CAAC;IAED,OAAO,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;AACnC,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,KAA+B;IACnE,MAAM,KAAK,GAAG,KAAK,YAAY,WAAW,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAA;IAC1E,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,CAAC,CAAA;IAC3E,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,OAAO,CAAC,CAAA;IAExC,IAAI,KAAK,CAAC,UAAU,GAAG,WAAW;QAAE,OAAO,SAAS,CAAA;IACpD,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAA;IAEvC,IAAI,OAAO,GAAG,qBAAqB;QAAE,OAAO,SAAS,CAAA;IACrD,MAAM,gBAAgB,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,CAAC,CAAA;IAEjD,IAAI,gBAAgB,KAAK,CAAC,IAAI,gBAAgB,IAAI,KAAK,CAAC,UAAU;QAAE,OAAO,SAAS,CAAA;IAEpF,IAAI,CAAC;QACJ,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAA;QACtD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,gBAAgB,GAAG,CAAC,EAAE,gBAAgB,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,CAAA;QAEpG,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAkB,CAAA;IAC5C,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAA;IACjB,CAAC;AACF,CAAC"}
|
package/out/fst-matcher.d.ts
CHANGED
|
@@ -23,11 +23,15 @@ export declare class FSTMatcher {
|
|
|
23
23
|
continuations(stateID: number): FSTContinuation[];
|
|
24
24
|
query(text: string): FSTQueryResult;
|
|
25
25
|
get nodeCount(): number;
|
|
26
|
-
/**
|
|
26
|
+
/**
|
|
27
|
+
* Expose the internal node array for serialization.
|
|
28
|
+
*/
|
|
27
29
|
toNodes(): readonly FSTNode[];
|
|
28
30
|
static fromNodes(nodes: FSTNode[]): FSTMatcher;
|
|
29
31
|
}
|
|
30
|
-
/**
|
|
32
|
+
/**
|
|
33
|
+
* Normalize text into FST tokens: lowercase, NFKC, strip punctuation, split on whitespace.
|
|
34
|
+
*/
|
|
31
35
|
export declare function normalizeTokens(text: string): string[];
|
|
32
36
|
export type { FSTNode };
|
|
33
37
|
//# sourceMappingURL=fst-matcher.d.ts.map
|
package/out/fst-matcher.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fst-matcher.d.ts","sourceRoot":"","sources":["../fst-matcher.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAA;AAEjG,UAAU,OAAO;IAChB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC1B,MAAM,EAAE,UAAU,EAAE,CAAA;CACpB;AAED,qBAAa,UAAU;IACtB,OAAO,CAAC,KAAK,CAAW;gBAEZ,KAAK,EAAE,OAAO,EAAE;IAI5B,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED,IAAI,UAAU,IAAI,MAAM,CAQvB;IAED,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,cAAc,GAAG,IAAI;
|
|
1
|
+
{"version":3,"file":"fst-matcher.d.ts","sourceRoot":"","sources":["../fst-matcher.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAA;AAEjG,UAAU,OAAO;IAChB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC1B,MAAM,EAAE,UAAU,EAAE,CAAA;CACpB;AAED,qBAAa,UAAU;IACtB,OAAO,CAAC,KAAK,CAAW;gBAEZ,KAAK,EAAE,OAAO,EAAE;IAI5B,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED,IAAI,UAAU,IAAI,MAAM,CAQvB;IAED,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,cAAc,GAAG,IAAI;IAkB7C,QAAQ,CAAC,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,GAAG,cAAc,GAAG,IAAI;IAYpE,SAAS,CAAC,OAAO,EAAE,MAAM,GAAG,UAAU,EAAE;IAIxC,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,eAAe,EAAE;IAmBjD,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,cAAc;IAqCnC,IAAI,SAAS,IAAI,MAAM,CAEtB;IAED;;OAEG;IACH,OAAO,IAAI,SAAS,OAAO,EAAE;IAI7B,MAAM,CAAC,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,UAAU;CAG9C;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAOtD;AAED,YAAY,EAAE,OAAO,EAAE,CAAA"}
|
package/out/fst-matcher.js
CHANGED
|
@@ -24,11 +24,11 @@ export class FSTMatcher {
|
|
|
24
24
|
}
|
|
25
25
|
walk(tokens) {
|
|
26
26
|
let stateID = 0;
|
|
27
|
-
for (
|
|
27
|
+
for (const token of tokens) {
|
|
28
28
|
const node = this.nodes[stateID];
|
|
29
29
|
if (!node)
|
|
30
30
|
return null;
|
|
31
|
-
const next = node.edges.get(
|
|
31
|
+
const next = node.edges.get(token);
|
|
32
32
|
if (next === undefined)
|
|
33
33
|
return null;
|
|
34
34
|
stateID = next;
|
|
@@ -98,7 +98,9 @@ export class FSTMatcher {
|
|
|
98
98
|
get nodeCount() {
|
|
99
99
|
return this.nodes.length;
|
|
100
100
|
}
|
|
101
|
-
/**
|
|
101
|
+
/**
|
|
102
|
+
* Expose the internal node array for serialization.
|
|
103
|
+
*/
|
|
102
104
|
toNodes() {
|
|
103
105
|
return this.nodes;
|
|
104
106
|
}
|
|
@@ -106,12 +108,14 @@ export class FSTMatcher {
|
|
|
106
108
|
return new FSTMatcher(nodes);
|
|
107
109
|
}
|
|
108
110
|
}
|
|
109
|
-
/**
|
|
111
|
+
/**
|
|
112
|
+
* Normalize text into FST tokens: lowercase, NFKC, strip punctuation, split on whitespace.
|
|
113
|
+
*/
|
|
110
114
|
export function normalizeTokens(text) {
|
|
111
115
|
return text
|
|
112
116
|
.normalize("NFKC")
|
|
113
117
|
.toLowerCase()
|
|
114
|
-
.
|
|
118
|
+
.replaceAll(/[\p{P}\p{S}]/gu, "")
|
|
115
119
|
.split(/\s+/)
|
|
116
120
|
.filter((t) => t.length > 0);
|
|
117
121
|
}
|