farsit 0.0.0-stage → 1.0.0

package/dist/types/alphabet.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Persian orthography ↔ internal phoneme alphabet.
+ *
+ * Internal phonemes (one ASCII char each, borrowed from Persian G2P datasets):
+ *   vowels   a e o (short)  A i u (long)
+ *   S=ش  C=چ  Z=ژ  x=خ  q=ق/غ  ?=glottal stop (ع/ء/initial ا)
+ * Every other phoneme is the plain Latin letter.
+ */
+/** Letters that may carry a phoneme. ZWNJ (\u200c) is a silent in-word boundary. */
+export declare const ZWNJ = "\u200C";
+/** Fixed consonant value of each consonant letter. */
+export declare const CONSONANT: Readonly<Record<string, string>>;
+/**
+ * Every way a letter can be realised (before the optional short vowel that may follow it).
+ * Order = preference used by the aligner when several alignments are possible.
+ */
+export declare const REALIZATIONS: Readonly<Record<string, readonly string[]>>;
+export declare const SHORT_VOWELS: readonly ["", "a", "e", "o"];
+export declare const isPersianLetter: (ch: string) => boolean;

package/dist/types/constraints.d.ts

@@ -0,0 +1,16 @@
+/**
+ * A constraint narrows the labels the decoder may choose for one letter.
+ * Every rule layer (diacritics, morphology, wazn) speaks this one language,
+ * so the statistical decoder stays the single place where decisions are made.
+ */
+export interface Constraint {
+    /** Which layer produced it — shown by `explain`. */
+    source: 'diacritic' | 'morphology' | 'wazn';
+    /** Human-readable reason, e.g. "fatha → a". */
+    note: string;
+    test: (label: string) => boolean;
+}
+export type Constraints = Constraint[][];
+export declare const exact: (label: string) => (l: string) => boolean;
+export declare function toFilters(cs: Constraints): (((l: string) => boolean) | undefined)[];
+export declare function addConstraint(cs: Constraints, i: number, c: Constraint): void;

package/dist/types/core.d.ts

@@ -0,0 +1,68 @@
+import { type Constraints } from './constraints.js';
+import { Model, type ModelData } from './model.js';
+import { type AffixTable, type Segment } from './morphology.js';
+import { type RenderOptions } from './render.js';
+export interface Layers {
+    diacritics?: boolean;
+    morphology?: boolean;
+    wazn?: boolean;
+}
+export interface WordTrace {
+    input: string;
+    letters: string;
+    /** True when the word is one of the fixed FUNCTION_WORDS. */
+    functionWord?: boolean;
+    /** True when implicit tanvin (-an) was applied. */
+    tanvin?: boolean;
+    segments: Segment[];
+    wazn?: {
+        name: string;
+        root: string;
+        accepted: boolean;
+        scoreGap: number;
+    };
+    /** Per letter: chosen label, phonemes and what constrained it. */
+    steps: {
+        letter: string;
+        label: string;
+        constraints: {
+            source: string;
+            note: string;
+        }[];
+    }[];
+    phonemes: string;
+    output: string;
+    score: number;
+}
+export interface CoreData {
+    model: ModelData;
+    /** Learned ZWNJ affixes (see morphology.ts). */
+    affixes: AffixTable;
+    /** Accept margin per wazn name (log-prob units). Missing = disabled. */
+    waznMargins: Readonly<Record<string, number>>;
+    /** Skip wazn when the free decode of a one-segment word is fully familiar (no back-off) and this confident (log-prob per letter). */
+    waznGate?: number;
+}
+export declare class Core {
+    readonly data: CoreData;
+    readonly model: Model;
+    constructor(data: CoreData);
+    /** Pronounce one Persian word (letters + optional harakat + ZWNJ). `beam` is the Viterbi beam width. */
+    word(word: string, layers?: Layers, opt?: RenderOptions, beam?: number): WordTrace;
+    /** Confidence gate: leave a one-segment word to the model when its free decode is confident and fully familiar. */
+    private gated;
+    /** Try every matching pattern on the stem; the best-scoring one within its own dev-tuned margin wins. */
+    private bestWazn;
+    /** Implicit tanvin on a word ending in bare ا, or undefined when the structural gate says no. */
+    private implicitTanvin;
+}
+/** The diacritic + morphology constraints `Core.word` would use (training/tuning helper). */
+export declare function constraintsFor(core: Core, word: string, layers?: Layers): {
+    letters: string;
+    cs: Constraints;
+};
+/** Split text into Persian words and everything else. */
+export declare function tokenize(text: string): {
+    text: string;
+    persian: boolean;
+}[];

package/dist/types/diacritics.d.ts

@@ -0,0 +1,3 @@
+import { type Constraints } from './constraints.js';
+/** Harakat written by the author are the strongest evidence there is: turn them into constraints. */
+export declare function diacriticConstraints(letters: string, marks: string[], cs: Constraints): void;

package/dist/types/function-words.d.ts

@@ -0,0 +1,6 @@
+/**
+ * The only fixed pronunciations in the library: a small closed class of very common
+ * function words (conjunctions, prepositions, pronouns, particles) whose short
+ * spelling is ambiguous (و = "va", که = "ke"). Always on. No content words, names or places.
+ */
+export declare const FUNCTION_WORDS: Readonly<Record<string, string>>;

package/dist/types/generated/data.d.ts

@@ -0,0 +1,2 @@
+import type { CoreData } from '../core.js';
+export declare const DATA: CoreData;

package/dist/types/index.d.ts

@@ -0,0 +1,49 @@
+/**
+ * farsit — Persian → Finglish transliteration and URL slugs.
+ * Offline, deterministic, zero dependencies.
+ *
+ *   transliterate('استقامت')        // "esteghamat"
+ *   slugify('سفر به اصفهان ۱۴۰۳')   // "safar-be-esfahan-1403"
+ */
+import { type Layers, type WordTrace } from './core.js';
+import type { RenderOptions } from './render.js';
+export type { Layers, WordTrace } from './core.js';
+export { normalize } from './normalize.js';
+export { WAZNS } from './wazn.js';
+export interface TransliterateOptions extends RenderOptions {
+    /** Switch individual layers off (all on by default). */
+    layers?: Layers;
+    /** Viterbi beam width (default 8). Higher = slightly more accurate, slower. */
+    beam?: number;
+    /** Phrase-level ezafe: a word ending in the glide ی (‌های، درختای، دانشجوی، خانه‌ی) followed by another
+     * content word in the same phrase gets the ezafe vowel (haye ziba). Default true. Single words are unaffected. */
+    ezafe?: boolean;
+}
+export interface SlugifyOptions extends TransliterateOptions {
+    /** Default "-". */
+    separator?: string;
+    /** Default true. */
+    lowercase?: boolean;
+    /** Cut at a word boundary so the slug is at most this long. */
+    maxLength?: number;
+    /** Keep digits (Persian/Arabic digits become ASCII). Default true. */
+    keepDigits?: boolean;
+}
+/** Transliterate Persian text to Finglish. Non-Persian characters pass through (digits become ASCII). */
+export declare function transliterate(text: string, opt?: TransliterateOptions): string;
+/** Build a URL slug from Persian (or mixed) text. */
+export declare function slugify(text: string, opt?: SlugifyOptions): string;
+export interface Explanation {
+    input: string;
+    normalized: string;
+    output: string;
+    tokens: ({
+        text: string;
+        persian: false;
+    } | ({
+        persian: true;
+        ezafe?: boolean;
+    } & WordTrace))[];
+}
+/** Like `transliterate`, but returns every decision each layer made — for debugging and the demo. */
+export declare function explain(text: string, opt?: TransliterateOptions): Explanation;

package/dist/types/model.d.ts

@@ -0,0 +1,130 @@
+/**
+ * Statistical layer — a compact joint-sequence ("graphone") model decoded with Viterbi.
+ *
+ * Every letter is emitted together with its label (realization + following short vowel),
+ * e.g. ت|te. Two scores are combined for each candidate label:
+ *
+ *  1. Graphone n-gram  P(gᵢ | gᵢ₋ₙ₊₁ … gᵢ₋₁)   — left-to-right sequence knowledge
+ *  2. Context model    P(labelᵢ | letters around i) — sees the letters to the RIGHT too
+ *
+ * plus hard phonotactic rules (a Persian syllable never starts with two consonants).
+ * Callers can restrict the allowed labels per letter (diacritics, wazn, morphology):
+ * the decoder then finds the best path that satisfies those constraints.
+ *
+ * All probabilities are pre-computed offline (tools/) and quantised to one char.
+ */
+export interface ModelData {
+    /** n-gram order. */
+    order: number;
+    /** Label alphabet (phoneme strings). */
+    labels: readonly string[];
+    /** Graphone inventory: pairs of (letter char, label char), graphone id = index + 1 (0 = word boundary). */
+    graphones: string;
+    /**
+     * n-gram trie in depth-first order. Per node (full-order leaves excepted) its child count,
+     * then per child the gap to the previous sibling's graphone id. Back-off weights are not
+     * stored: they follow from the probabilities and are recomputed at load.
+     */
+    trie: string;
+    /** One logprob char per trie child, same order (0 = structural node with no probability). */
+    trieProbs: string;
+    /** One back-off weight char per history node (depth ≥ 1 with children), in DFS order. */
+    trieBows: string;
+    /** Context levels [left, right], nested, shortest first. */
+    levels: readonly (readonly [number, number])[];
+    /** Per level: window chars, count char n, then n × label char. */
+    contexts: readonly string[];
+    /** Per level: the logprob chars of `contexts`, in order. */
+    contextProbs: readonly string[];
+    /** Weight of the context model relative to the n-gram. */
+    lambda: number;
+    /** Quantisation step used for this model's logprobs. */
+    q?: number;
+}
+export declare const SYM_BASE = 48;
+/** Log-probabilities are stored as round(-ln p × Q) in one char. */
+export declare let Q: number;
+/** Training-time only: change the quantisation step. */
+export declare const setQ: (q: number) => void;
+export declare const PAD_LEFT = "^";
+export declare const PAD_RIGHT = "$";
+export declare const encSym: (n: number) => string;
+export declare const encBow: (b: number) => string;
+export declare const encLogp: (lp: number, off?: number) => string;
+export type LabelFilter = ((label: string) => boolean) | null | undefined;
+export interface DecodeResult {
+    labels: string[];
+    score: number;
+}
+export declare class Model {
+    readonly data: ModelData;
+    /** (node, digit) → child node. Node 0 is the root (empty history). */
+    private readonly child;
+    /** Per node: log p of its last graphone given its parent (NaN = not stored). */
+    private readonly lp;
+    /** Per node: back-off weight when used as a history. */
+    private readonly bow;
+    /** Per node: longest proper suffix that is a history node (-1 for the root). */
+    readonly link: Int32Array;
+    /** Per node: 1 if it has children (can serve as a decoder state). */
+    private readonly hasKids;
+    /** Context trie: (node, letter index) → child; per node an optional label distribution. */
+    private ctxChild;
+    /** Context node → its label distribution as a slice [ctxStart[n], ctxStart[n+1]) of ctxLabel/ctxLp (empty = none). */
+    private readonly ctxStart;
+    private readonly ctxLabel;
+    private readonly ctxLp;
+    private distBuf;
+    private readonly ctxOffsets;
+    private readonly charIdx;
+    private codeBuf;
+    /** For each letter: candidate graphone digits and label indices. */
+    private readonly byLetter;
+    private readonly maxL;
+    private readonly maxR;
+    private readonly shapes;
+    private readonly ctxPad;
+    /** Reusable back-pointer storage. */
+    private backLabel;
+    private backPrev;
+    /** Recombination table (see decode). */
+    private readonly stamp;
+    private readonly slotKey;
+    private readonly slotIdx;
+    private gen;
+    private readonly stepKey;
+    private readonly stepNext;
+    private readonly stepLp;
+    /** Reusable hypothesis buffers (current / next), grown on demand. */
+    private hyp;
+    private hypI;
+    private readonly topIdx;
+    private topK;
+    constructor(data: ModelData);
+    hasLetter(ch: string): boolean;
+    /** Back-off log-probability of digit `g` after history node `h`. */
+    private score;
+    /** Decoder state set by `step`. */
+    private nextState;
+    /**
+     * One walk down the back-off chain giving both the log-probability of `g` after state `h`
+     * and (in `nextState`) the state after it: the longest known history that is a suffix.
+     */
+    private step;
+    /**
+     * How "familiar" a decoded path is: the fraction of its graphones (end marker included) found
+     * without backing off from the full history. Near 1 only for spellings the model has
+     * effectively memorised — used to leave such words to the model.
+     */
+    familiarity(letters: string, labels: string[]): number;
+    /** Decoder state after emitting `g` from state `h`. */
+    private advance;
+    /** Longest-matching context node (with a distribution) for each position; 0 = none. */
+    private contextDists;
+    /**
+     * Viterbi beam search. `filters[i]` may restrict the labels allowed for letter i.
+     * Hypotheses with the same (n-gram state, phonotactic state) are merged; states are reduced
+     * to the longest history the model knows, so the merge is exact.
+     */
+    decode(letters: string, filters?: LabelFilter[], beam?: number): DecodeResult;
+}

package/dist/types/morphology.d.ts

@@ -0,0 +1,21 @@
+import { type Constraints } from './constraints.js';
+/**
+ * Morphology layer. Persian writes many affixes and compound parts as separate pieces
+ * joined by ZWNJ (نیم‌فاصله): می‌روم، کتاب‌ها، بزرگ‌ترین، برنامه‌نویسی.
+ * Pieces with a stable pronunciation become hard constraints, so the statistical model
+ * only has to guess the stem. The affix table is learned offline from the training
+ * split (tools/train.ts): keys are "P:<prefix>" / "S:<suffix>", values are labels joined by "|".
+ */
+export type AffixTable = Readonly<Record<string, string>>;
+export interface Segment {
+    text: string;
+    start: number;
+    kind: 'prefix' | 'stem' | 'suffix';
+}
+/** Split on ZWNJ, tag known affixes, and constrain their letters. Returns the segments for `explain`. */
+export declare function morphologyConstraints(letters: string, cs: Constraints, table: AffixTable): Segment[];
+/** Learn the affix table from aligned words: frequent ZWNJ pieces with one dominant reading. */
+export declare function learnAffixes(entries: readonly {
+    word: string;
+    labels: readonly string[];
+}[], minCount?: number, minShare?: number): Record<string, string>;

package/dist/types/normalize.d.ts

@@ -0,0 +1,19 @@
+/** Harakat we understand. Anything else in U+064B..U+065F is dropped. */
+export declare const DIACRITIC: {
+    readonly FATHA: "َ";
+    readonly KASRA: "ِ";
+    readonly DAMMA: "ُ";
+    readonly SUKUN: "ْ";
+    readonly SHADDA: "ّ";
+    readonly FATHATAN: "ً";
+    readonly KASRATAN: "ٍ";
+    readonly DAMMATAN: "ٌ";
+    readonly SUPERSCRIPT_ALEF: "ٰ";
+};
+/** Normalise text: unify letters, convert digits to ASCII, clean ZWNJ, keep useful harakat. */
+export declare function normalize(input: string): string;
+/** Split a normalised word into bare letters and per-letter diacritic marks. */
+export declare function splitDiacritics(word: string): {
+    letters: string;
+    marks: string[];
+};

package/dist/types/render.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Internal phonemes → Finglish.
+ *
+ * Conventions (common Iranian web usage, never "x"):
+ *   خ kh · ش sh · چ ch · ژ zh · ق/غ gh · آ/ا (long) a · ع/ء dropped · short vowels a e o
+ */
+export interface RenderOptions {
+    /** How to spell long /u/: "u" (default, e.g. "nur") or "oo" ("noor"). */
+    longU?: 'u' | 'oo';
+}
+export declare function render(phon: string, opt?: RenderOptions): string;

package/dist/types/tanvin.d.ts

@@ -0,0 +1,6 @@
+export declare function tanvinReason(w: string): string | undefined;
+/** Extra label constraints the tanvin reading implies: ta marbuta → …at-an; bare فَعل → CaCC-an. */
+export declare function tanvinShape(w: string): {
+    at: number;
+    test: (l: string) => boolean;
+}[];

package/dist/types/wazn.d.ts

@@ -0,0 +1,39 @@
+import { type Constraints } from './constraints.js';
+/**
+ * Arabic wazn (وزن) layer.
+ *
+ * Most Arabic loanwords in Persian are a three-letter root poured into a pattern:
+ * استقامت = است + ق ا م + ت on the pattern استفاله‌ت → "esteghamat".
+ * A pattern is written with ف ع ل as root slots; every letter gets a label template:
+ *   "?e" fixed label · "*" root consonant, no vowel · "*a" root + a · "**e" doubled root + e.
+ *
+ * Pattern matching only proposes; the statistical model decides. A pattern is accepted
+ * when the constrained decode scores within `margin` of the free decode. Margins are
+ * tuned on the dev set by tools/train.ts; patterns that do not help are switched off.
+ */
+export interface Wazn {
+    name: string;
+    /** Pattern letters, root slots as ف ع ل. */
+    form: string;
+    /** One label template per letter of `form`. */
+    labels: readonly string[];
+    /** Broken plurals (اخبار، مناطق) never take a second, sound plural ending (ات، ین، ون). */
+    brokenPlural?: boolean;
+    /** Only used as the stem of an implicit tanvin (فعلاً). */
+    tanvinOnly?: boolean;
+    /** Root must contain a letter only Arabic loans use (ث ح ذ ص ض ط ظ ع ق غ): keeps Persian فردا out. */
+    arabicOnly?: boolean;
+}
+export declare const WAZNS: readonly Wazn[];
+export interface WaznMatch {
+    wazn: Wazn;
+    root: string;
+    suffix: string;
+    labels: string[];
+    start: number;
+}
+/** All patterns that fit `stem` (located at `start` in the word). */
+export declare function matchWazn(stem: string, start?: number, enabled?: (w: Wazn) => boolean): WaznMatch[];
+/** Which dev-tuned margin governs a match: tanvin adverbs (کاملا) get their own, shared by all patterns. */
+export declare const marginKey: (m: WaznMatch) => string;
+export declare function waznConstraints(m: WaznMatch, cs: Constraints): void;

package/package.json

@@ -1,6 +1,78 @@
 {
   "name": "farsit",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "1.0.0",
+  "description": "Accurate Persian → Finglish (Latin) transliteration. No AI, no dictionary, zero deps. Slugs included.",
+  "keywords": [
+    "persian",
+    "farsi",
+    "slug",
+    "slugify",
+    "transliteration",
+    "finglish",
+    "pinglish",
+    "romanization",
+    "g2p",
+    "url",
+    "seo",
+    "zero-dependency",
+    "offline",
+    "typescript",
+    "farsit"
+  ],
+  "homepage": "https://github.com/amiryxe/farsit#readme",
+  "bugs": {
+    "url": "https://github.com/amiryxe/farsit/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/amiryxe/farsit.git"
+  },
+  "license": "MIT",
+  "author": "Amir Salehi (amiryxe)",
+  "type": "module",
+  "sideEffects": false,
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/index.d.ts",
+        "default": "./dist/index.js"
+      },
+      "require": {
+        "types": "./dist/index.d.cts",
+        "default": "./dist/index.cjs"
+      }
+    }
+  },
+  "files": [
+    "dist",
+    "LICENSE",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsup && tsc -p tsconfig.build.json && node tools/dts.mjs",
+    "train": "tsx tools/train.ts",
+    "eval": "tsx tools/eval.ts",
+    "bench": "tsx bench/bench.ts",
+    "test": "vitest run",
+    "typecheck": "tsc --noEmit -p .",
+    "demo": "vite demo --port 5180 --host",
+    "demo:build": "vite build demo",
+    "size": "node tools/size.mjs",
+    "prepublishOnly": "npm run typecheck && npm test && npm run build && npm run size"
+  },
+  "devDependencies": {
+    "@types/node": "^26.6.6",
+    "@typescript/typescript6": "^6.0.2",
+    "tsup": "^9.0.0",
+    "tsx": "^4.23.15",
+    "typescript": "^7.0.2",
+    "vite": "^8.3.5",
+    "vitest": "^5.0.5"
+  },
+  "overrides": {
+    "rollup": "4.67.3"
+  }
+}