h3-words 3.0.0

package/dist/scripts/build-wordlist.js

@@ -0,0 +1,473 @@
+/**
+ * Builds a small, curated wordlist (default 8192 words).
+ * Loads dictionaries, normalizes, filters, removes phonetic collisions.
+ *
+ * Run: npm run build:wordlist
+ * Output: scripts/words.json, scripts/checksum.txt
+ *
+ * Output is also written to ./words.json for the app to import.
+ */
+import fs from "node:fs";
+import path from "node:path";
+import crypto from "node:crypto";
+import { fileURLToPath } from "node:url";
+import wordlistEnglish from "wordlist-english";
+import doubleMetaphone from "double-metaphone";
+import * as badwords from "badwords-list";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const TARGET_SIZE = 8_192;
+
+// Profanity / slur blacklist (we keep it simple: exact-word match after normalize())
+const banned = new Set(
+  (badwords.array || [])
+    .map((w) => normalize(String(w)))
+    .filter(Boolean),
+);
+// Project-specific additions go here (normalized lowercase a-z only)
+[
+  // ── Core: violence, death, slurs, sexual, drugs, insults, misc ──
+  "rape", "die", "cut", "gun",
+  "war", "dead", "death", "dying",
+  "corpse", "fatal", "harm", "hurt",
+  "kill", "murder", "homicide", "suicide",
+  "torture", "victim", "violence", "weapon",
+  "blood", "wound", "stab", "shoot",
+  "bomb", "attack", "assault", "strike",
+  "battle", "warfare", "bullet", 
+  //"knife", "sword", "axe", "gut", "bullshit"
+  "gash", "gore",
+  "gory", "maim", "slay", "slit",
+  "hostage", "kidnap", "rot",
+  "handgun", "gunshot", "gunfire", "gunman",
+  "gunboat", "gunpoint", "wartime", "warlike",
+  "warpath", "warlord", "killing", "overkill",
+  "bombing", "bombard", "shooting", "weaponry",
+  "hurting", "hurtful", "harmful", "hateful",
+  "horrify", "terrify", "abusive", "cruelty",
+  "savagery", "suicidal", "sadistic", "penknife",
+  "deaden", "shotgun", "gunmen", "starve",
+  "starving", "beating", "behead", "thrash",
+  "bludgeon", "strangle", "hoodlum", "assassin",
+  "henchman", "redskin", "spastic", "niggling",
+  "arsehole", "schmuck", "slave",
+  "terror", "lynch", "hate", "abuse",
+  "racism", "racist", "race", "fascist",
+  "genocide", "cruel", "brutal", "savage",
+  "sexual", "sexist", "obscene", "sordid",
+  "nude", "orgy", "gay", "lesbian",
+  "pervert", "prurient", "sadism", 
+  // "sexy", "toxic", "fear", "mad", "ill"
+  "condom", "virgin", "lewd", "harlot",
+  "pubic", "ravish", "seduce", "misogyny",
+  "vaginae", "vaginal", "heroin", "addict",
+  "poison", "venom", "sin", "grief", "panic",
+  "horror",  "devil", "curse",
+  "wrath", "disease", "plague", "vomit",
+  "grave", "enemy", "abort", "evil",
+  "sinful", "hellish", "satanism", "warlock",
+  "fat", "bad", "dumb", "idiot",
+  "moron", "scum", "slag", "slur",
+  "guilty", "stupid", "dummy", "idiotic",
+  "lunatic", "madman", "crybaby", "madhouse",
+  "comma", "gag",
+  // "flu", "cry", "fail", "junk", "pain", "trash", "tear"
+  "rash", "sick", "colon",
+  "cough", "drown", "funeral", "ache",
+  "choke", "bore", "bury", "dump",
+  "err", "lie", "numb", "hang",
+  "hanging", "hung", "vice", "child",
+  "children", "affair", "extent", "regret",
+  "practise", "strip", "ashcan", "beaten",
+  "hating", "infant", "lethal", "abusing",
+  "bashing", "degrade", "deprave", "despise",
+  "lesbian", "teenage", "sixteen", "violate",
+  "violent", "aardvark", "abortion", "eighteen",
+  "fourteen", "fifteen", "teen", "nineteen",
+  "teenage", "hitherto", "invading", "invasion",
+  "maximise", "minimise", "optimise", "shouting",
+  "bra", "colic", "creak", "nymph",
+  "abduct", "cougar", "fetish", "morbid",
+  "pelvic", "rectal", "sexing", "sexism",
+  "asexual", "fascism", "jerking", "jinxing",
+  "macabre", "adultery", "bisexual", "erection",
+  "impotent", "flaccid", "colonist", "hooligan",
+  "christen", "rabbi", "emir", "fatty",
+  // ── Violence / Weapons / Fighting ──
+  "bash", "beat", "blade", "blast",
+  "brawl", "brute", "cage", "combat",
+  "crush", "cull", "fang", "feud",
+  "flog", "fury", "gang", "jail",
+  "lash", "loot", "lurk", "maul",
+  "prey", "punch", "purge", "rage",
+  "raid", "rant", "riot", "ruin",
+  "scald", "scalp", "slap", "slash",
+  "slug", "smash", "snarl", "stalk",
+  "swat", "thug", "thump", "whip",
+  "ambush", "avenge", "bandit", "invade",
+  "malice", "martyr", "mayhem", "napalm",
+  "ransom", "ravage", "revolt", "threat",
+  "afflict", "condemn", "convict", "crucify",
+  "culprit", "dungeon", "firearm", "harming",
+  "hostile", "inflict", "manhunt", "missile",
+  "outrage", "revenge", "treason", "arsonist",
+  "atrocity", "barbaric", "butchery", "cannibal",
+  "casualty", "corporal", "criminal", "desolate",
+  "despotic", "detonate", "dynamite", "frighten",
+  "fugitive", "imprison", "predator", "punching",
+  "ravaging", "sabotage", "savaging", "scalding",
+  "shrapnel", "slashing", "smashing", "stalking",
+  "stealing", "threaten", "throttle", "torching",
+  "vendetta", "wrecking",
+  // ── Death / Morbid / Horror ──
+  "crypt", "decay", "dirge", "doom",
+  "ghoul", "gloom", "pyre", "tomb",
+  "agony", "demon", "dread", "fiend",
+  "skull", "casket", "demise", "graven",
+  "cremate", "despair", "inferno", "serpent",
+  "sorcery", "tragedy", "turmoil", "vampire",
+  "vulture", "autopsy", "devilish", "dreadful",
+  "exorcism", "exorcist", "fatalism", "mournful",
+  "skeleton", "werewolf",
+  // ── Disgusting / Gross / Bodily ──
+  "bile", "cyst", "dung", "muck",
+  "ooze", "puke", "reek", "scab",
+  "scar", "slob", "slop", "snot",
+  "sore", "spew", "spit", "wart",
+  "welt", "bowel", "enema", "fetid",
+  "filth", "flesh", "phlegm", "snort",
+  "spasm", "maggot", "pimple", "putrid",
+  "rancid", "rotten", "sicken", "sludge",
+  "spleen", "vermin", "filthy", "gangrene",
+  "smallpox", "stinking", "virulent", "vomiting",
+  // ── Sexual / Body ──
+  "ogle", "teat", "womb", "bosom",
+  "harem", "erotic", "fondle", "sultry",
+  "indecent", "perverse",
+  // ── Offensive / Derogatory / Insults ──
+  "gob", "hag", "mob", "sod",
+  "jerk", "mock", "rude", "slum",
+  "vile", "beast", "bigot", "crazy",
+  "crude", "nasty", "scoff", "scorn",
+  "spite", "tramp", "asylum", "bedlam",
+  "defame", "defile", "deform", "deject",
+  "injure", "injury", "insult", "midget",
+  "ordeal", "orphan", "racial", "rancor",
+  "sadden", "stigma", "stolen", "stupor",
+  "anguish", "bigotry", "damning", "derange",
+  "disgust", "immoral", "leprosy", "mortify",
+  "painful", "uncouth", "vagrant", "neurotic",
+  "paranoia", "paranoid", "scornful", "spiteful",
+  "vengeful",
+  // ── Drugs / Alcohol / Vice ──
+  "dope", "drug", "booze", "drunk",
+  "opium", "tipsy", "torch", "toxin",
+  "alcohol", "drunken", "hashish", "morphine",
+  "syringe", "tobacco", "epidemic", "gambling",
+  "narcotic",
+  // ── Crime / derivatives ──
+  "aborting", "burying", "drowning", "raging",
+  "wail", "weep", "crime", "fraud",
+  "felon", "guilt", "havoc", "creep",
+  // ── Round 2: violence, death, disgusting, sexual, offensive, drugs ──
+  "bang", "duel", "fist", "foul",
+  "grim", "hurl", "raze", "rend",
+  "slam", "stun", "suck", "arson",
+  "ghost", "swear", "theft", "thief",
+  "wreck", "devour", "famine", "hijack",
+  "infect", "scream", "tragic", "vandal",
+  "anthrax", "berserk", "brutish", "burglar",
+  "dooming", "fearful", "larceny", "lashing",
+  "lurking", "machete", "rampage", "rancour",
+  "ransack", "ruining", "scourge", "amputate",
+  "coercive", "crucifix", "decimate", "disfavor",
+  "disgrace", "dishonor", "doomsday", "fatality",
+  "fearsome", "fiendish", "gangland", "horrific",
+  "injuring", "maltreat", "stampede", "coma",
+  "mourn", "mortal", "effigy", "embalm",
+  "epitaph", "obituary", "comatose", "stricken",
+  "mange", "mangy", "louse", "leech",
+  "groin", "scorpion", "toxicity", "lusty",
+  "lurid", "grope", "moaning", "puberty",
+  "sensual", "fondling", "foreskin", "seducing",
+  "cur", "foe", "woe", "cult",
+  "snob", "livid", "loony", "lousy",
+  "mania", "manic", "noose", "obese",
+  "scold", "amoral", "enmity", "extort",
+  "horrid", "infamy", "misery", "offend",
+  "harelip", "anorexia", "anorexic", "dementia",
+  "derelict", "derision", "shameful", "druggist",
+  "drunkard", "lobotomy", "overdose",
+  // ── Round 3: triggering (sexual, violent, derogatory, disease) ──
+  "moan", "rabid", "lice", "loin",
+  "hump", "kink", "loon", "oaf",
+  "gout", "yuck", "burp", "racy",
+  "runt", "bawdy", "booty", "buxom",
+  "dunce", "dusky", "elegy", "hick",
+  "ninny", "pansy", "pokey", "pokie",
+  "polio", "prude", "pudgy", "booby",
+  "skulk", "waif", "peon", "retch",
+  "saucy", "chaste", "climax", "idiocy",
+  "inmate", "nudist", "pummel", "urchin",
+  "hemlock", "heathen", "infidel", "jilting",
+  "lisping", "matador", "megaton", "piggish",
+  "prudish", "ranting", "rending", "servile",
+  "smiting", "thieve", "doleful", "ogling",
+  "snarling", "snorting", "humping", "spanking",
+  "burglary", "diatribe", "emaciate", "henchmen",
+  "insolent", "minstrel", "petulant", "polygamy",
+  "swarthy", "vagabond", "welshing", "acrimony",
+  // ── Culturally sensitive ──
+  "wigwam", "tomahawk",
+  // ── Obscure 3-letter ──
+  "sic", "qua", "avo", "caw",
+  "chi", "eke", "fen", "fez",
+  "fro", "gnu", "hew", "lop",
+  "lye", "mew", "nee", "ova",
+  "pap", "roe", "sac", "sop",
+  "sup", "wan", "wot", "zac",
+  "gee", "coo",
+  // ── Obscure 4-letter ──
+  "afto", "ambo", "arvo", "bani",
+  "baud", "bide", "bonz", "boor",
+  "bray", "calk", "cark", "cede",
+  "czar", "daub", "devo", "dour",
+  "eave", "esky", "fogy", "furl",
+  "gday", "geld", "gibe", "gild",
+  "gilt", "glib", "goad", "gybe",
+  "hale", "hark", "hart", "hind",
+  "hock", "hoon", "hove", "jute",
+  "lade", "lath", "laud", "lieu",
+  "loam", "loll", "lope", "lute",
+  "lyre", "mien", "mire", "morn",
+  "nigh", "nong", "onya", "pall",
+  "pate", "prow", "rort", "shod",
+  "sire", "toga", "tsar", "twee",
+  "wist", "woop", "yowl", "zack",
+  // ── Obscure 5-letter ──
+  "abhor", "acrid", "aftie", "allay",
+  "annul", "aorta", "arbor", "ardor",
+  "banal", "baulk", "bayou", "beady",
+  "befit", "belie", "beset", "biffo",
+  "bluey", "bonza", "bonze", "briny",
+  "broil", "bundy", "burro", "butte",
+  "byway", "calve", "chafe", "chaff",
+  "chasm", "cheep", "chide", "chook",
+  "circa", "clime", "cooee", "croon",
+  "curio", "decry", "deify", "deign",
+  "ditty", "doona", "dowdy", "downy",
+  "dowry", "droll", "dunno", "dunny",
+  "exalt", "extol", "exude", "exult",
+  "fiche", "filch", "flout", "fogey",
+  "foist", "fount", "frond", "galah",
+  "garbo", "genii", "gimme", "glade",
+  "gnarl", "gnash", "gonna", "goodo",
+  "gruel", "guile", "gulch", "halon",
+  "infix", "inlay", "junta", "kanga",
+  "kylie", "leery", "lemme", "lupin",
+  "lymph", "mamma", "miaow", "moult",
+  "mousy", "natty", "newsy", "nulla",
+  "octal", "oking", "olden", "passe",
+  "piety", "pique", "pithy", "primp",
+  "privy", "psalm", "quark", "radii",
+  "ravel", "razoo", "rebut", "remit",
+  "revue", "rouge", "rouse", "ruddy",
+  "ruing", "sabre", "unman",
+  // ── Obscure 6-letter ──
+  "acumen", "aflame", "amidst", "aplomb",
+  "arable", "ardent", "belfry", "bemoan",
+  "bingle", "blithe", "bodice", "bovine",
+  "bunyip", "bushel", "cavort", "chintz",
+  "cloven", "cogent", "cortex", "damsel",
+  "dazing", "desist", "dinkum", "disuse",
+  "dorsal", "endive", "fervor", "foment",
+  "geeing", "gentry", "girdle", "gutful",
+  "haling", "hobnob", "hubbub", "hymnal",
+  "jading", "jibing", "junket", "laming",
+  "laxity", "lichen", "loping", "lustre",
+  "madcap", "madmen", "meagre", "mohair",
+  "myopic", "nicety", "optima", "paltry",
+  "papacy", "papyri", "pedlar", "planar",
+  "ramrod", "rankle", "rebuke", "recant",
+  "regent", "repose", "revere", "rubric",
+  "stanch", "stolid", "theist", "thence",
+  "thresh", "thrice", "turgid",
+  // ── Obscure 7-letter ──
+  "askance", "attache", "belabor", "calculi",
+  "caustic", "cistern", "coarsen", "cogency",
+  "congeal", "corsage", "dualism", "dunging",
+  "earbash", "hackney", "hymning", "insipid",
+  "jocular", "jumbuck", "languid", "languor",
+  "latrine", "lexical", "lexicon", "liturgy",
+  "maudlin", "polemic", "rostrum", "sojourn",
+  "stoical", "swagman", "tankard", "treacle",
+  "vestige",
+  // ── Obscure 8-letter ──
+  "abstruse", "brackish", "cherubim", "claptrap",
+  "conjugal", "consular", "dillybag", "doggerel",
+  "esophagi", "filigree", "guttural", "indolent",
+  "larrikin", "mandible", "monastic", "opaquing",
+  "pastiche", "pedagogy", "primeval", "putative",
+  "skerrick", "taciturn", "teletype", "thesauri",
+  "ubiquity", "vehement", "venerate", "verbiage",
+  "vestment", "yarmulke",
+].forEach((w) => banned.add(normalize(w)));
+
+// ---------- helpers ----------
+
+function normalize(word) {
+  return word
+    .toLowerCase()
+    .normalize("NFKD")
+    .replace(/[^a-z]/g, "");
+}
+
+// BIP39-style-ish: readable, avoid awkward patterns. We can be stricter since we only need 8192.
+function isValid(word) {
+  // Allow 4-letter words to increase the tier 10/20 pool (more familiar words).
+  if (word.length < 3 || word.length > 8) return false;
+  if (banned.has(word)) return false;
+  if (/(.)\1\1/.test(word)) return false; // aaa
+  if (/(qzx|xq|jj)/.test(word)) return false; // ugly patterns
+  if (word.endsWith("s")) return false;
+  if (word.endsWith("ed")) return false;
+  if (word.endsWith("ly")) return false;
+  if (word.endsWith("er")) return false;
+  if (word.endsWith("est")) return false;
+  return true;
+}
+
+// ---------- load raw sources (frequency-ordered: common first) ----------
+
+// Track best (lowest) frequency tier per raw word
+const bestTierByRaw = new Map();
+const orderedRaw = [];
+
+// Popularity proxy: lower tier = more common.
+// We'll prioritize 10/20 strongly, and only fall back to 35/40 if needed.
+const frequencyTiers = [10, 20, 35, 40];
+const COMMON_MAX_TIER = 20;
+const WORD1_SIZE = 4096;
+[
+  "english",
+  "english/american",
+  "english/british",
+  "english/australian",
+  "english/canadian",
+].forEach((group) => {
+  frequencyTiers.forEach((freq) => {
+    const key = group + "/" + freq;
+    wordlistEnglish[key]?.forEach((w) => {
+      const prev = bestTierByRaw.get(w);
+      if (prev === undefined) {
+        bestTierByRaw.set(w, freq);
+        orderedRaw.push(w);
+      } else if (freq < prev) {
+        bestTierByRaw.set(w, freq);
+      }
+    });
+  });
+});
+
+console.log(`Raw words (frequency-ordered): ${orderedRaw.length}`);
+
+// ---------- normalize + filter (preserve frequency order) ----------
+
+// For normalized words, keep the best tier we've seen.
+const bestTierByNorm = new Map();
+let filtered = [];
+
+for (const w of orderedRaw) {
+  const n = normalize(w);
+  if (!n) continue;
+  if (!isValid(n)) continue;
+  const tier = bestTierByRaw.get(w) ?? 999;
+  const prevTier = bestTierByNorm.get(n);
+  if (prevTier === undefined || tier < prevTier) {
+    bestTierByNorm.set(n, tier);
+  }
+}
+
+for (const [w, tier] of bestTierByNorm.entries()) {
+  filtered.push({ w, tier });
+}
+
+console.log(`After normalization/filtering: ${filtered.length}`);
+
+// ---------- phonetic collision removal (prefer one word per metaphone code) ----------
+
+const metaphoneMap = new Map();
+const phoneticUnique = [];
+
+// Sort candidates by popularity first so we keep the "most common" word per metaphone code.
+filtered.sort((a, b) => a.tier - b.tier || a.w.length - b.w.length || a.w.localeCompare(b.w));
+
+for (const { w: word, tier } of filtered) {
+  const [primary] = doubleMetaphone(word);
+  if (!primary) continue;
+  if (metaphoneMap.has(primary)) continue;
+
+  metaphoneMap.set(primary, word);
+  phoneticUnique.push({ w: word, tier });
+}
+
+console.log(`After phonetic filtering (unique): ${phoneticUnique.length}`);
+
+// If we have enough phonetically unique, use them; otherwise fill from filtered
+const used = new Set(phoneticUnique.map((x) => x.w));
+let final = [...phoneticUnique];
+
+if (final.length < TARGET_SIZE) {
+  for (const { w: word, tier } of filtered) {
+    if (used.has(word)) continue;
+    final.push({ w: word, tier });
+    if (final.length >= TARGET_SIZE) break;
+  }
+}
+
+console.log(`After fill: ${final.length}`);
+
+if (final.length < TARGET_SIZE) {
+  throw new Error("Not enough words — relax filters or add sources");
+}
+
+// ---------- Ordering: popularity first, then short, then alpha ----------
+// This makes early indices skew toward common vocabulary.
+final.sort((a, b) => a.tier - b.tier || a.w.length - b.w.length || a.w.localeCompare(b.w));
+
+// ---------- truncate ----------
+
+// Ensure the "word1" pool is as curated/common as possible:
+// - Prefer to use ONLY tiers 10/20 if we have enough to fill the whole list.
+// - Always ensure the first WORD1_SIZE entries (used for word1 in encoding) come from tiers 10/20.
+const common = final.filter((x) => x.tier <= COMMON_MAX_TIER);
+const fallback = final.filter((x) => x.tier > COMMON_MAX_TIER);
+
+if (common.length < WORD1_SIZE) {
+  throw new Error(
+    `Not enough tier<=${COMMON_MAX_TIER} words for first ${WORD1_SIZE} entries. Got ${common.length}. Relax filters or add sources.`,
+  );
+}
+
+let ordered = common;
+if (common.length < TARGET_SIZE) {
+  ordered = common.concat(fallback);
+}
+
+// Force the first WORD1_SIZE to be common (10/20), even if we needed fallback later.
+const output = ordered.slice(0, TARGET_SIZE).map((x) => x.w);
+
+// ---------- write ----------
+
+const outDir = path.join(__dirname);
+const jsonStr = JSON.stringify(output, null, 2);
+
+fs.writeFileSync(path.join(outDir, "words.json"), jsonStr);
+fs.writeFileSync(
+  path.join(outDir, "checksum.txt"),
+  crypto.createHash("sha256").update(output.join("\n")).digest("hex")
+);
+
+console.log("✅ Done");
+console.log(`Final size: ${output.length}`);
+console.log(`Written to scripts/words.json`);

package/dist/scripts/checksum.txt

@@ -0,0 +1 @@
+969be24dcff1651fb48360a77d470021771b0f3f1ff93664d94fba8ad19d454f