@postalsys/otp-detect 0.0.1 → 0.1.0

package/dist/esm/candidates.js

@@ -0,0 +1,134 @@
+// Finds every token in a document that could be a one-time code. This step is deliberately
+// generous: a code it misses can never be found, while a wrong candidate only costs the scorer
+// one more row to reject.
+const TOKEN_RE = /[A-Za-z0-9]+(?:-[A-Za-z0-9]+)*/g;
+// Single characters separated by single spaces or line breaks: "4 8 2 9 1 7", or one digit per table cell
+const SPACED_RE = /(?<![A-Za-z0-9])[A-Z0-9](?:[ \n][A-Z0-9]){3,9}(?![A-Za-z0-9])/g;
+// Characters that glue a token to its neighbour into something that is not a code: 12.50, 10:30,
+// 1,000, a/b, user@host, snake_case
+const JOINERS = new Set(['.', ',', ':', '/', '@', '_', '\\', '=', '&', '?', '%', '+']);
+const ALNUM_RE = /[A-Za-z0-9]/;
+const MAX_CANDIDATES = 400;
+function isAlnum(ch) {
+    return ch !== undefined && ALNUM_RE.test(ch);
+}
+/** Whether a token without separators looks like a code */
+export function codeLike(core) {
+    const len = core.length;
+    if (len > 12 && len <= 40) {
+        // Long random validation codes mix upper case, lower case and digits (domain validation)
+        return /[A-Z]/.test(core) && /[a-z]/.test(core) && /\d/.test(core) && /^[A-Za-z0-9]+$/.test(core);
+    }
+    if (len < 4 || len > 12) {
+        return false;
+    }
+    if (/^\d+$/.test(core)) {
+        return len <= 10;
+    }
+    const hasDigit = /\d/.test(core);
+    if (!hasDigit) {
+        // Letter-only codes are printed in capitals (Steam Guard style: QWKZT)
+        return /^[A-Z]+$/.test(core) && len <= 10;
+    }
+    if (!/[A-Za-z]/.test(core)) {
+        return false;
+    }
+    // 21st, 100mg, 1080p, 12px, 5000mAh
+    return !/^\d+(?:[a-z]{1,3}|mAh|MB|GB|KB|kB|Hz)$/.test(core);
+}
+// Checked once per whitespace separated chunk, never per token, so a long blob stays linear
+const URL_OR_ADDRESS_RE = /:\/\/|^www\.|@[A-Za-z0-9-]+\.|\.[a-z]{2,6}\//i;
+const CHUNK_RE = /\S+/g;
+function gluedToNeighbour(text, start, end) {
+    const before = text[start - 1];
+    const after = text[end];
+    if (before !== undefined && JOINERS.has(before) && isAlnum(text[start - 2])) {
+        return true;
+    }
+    return after !== undefined && JOINERS.has(after) && isAlnum(text[end + 1]);
+}
+function make(text, start, end, kind, code) {
+    return { code, raw: text.slice(start, end), start, end, kind };
+}
+function tokenCandidates(text, out) {
+    const tokens = [];
+    CHUNK_RE.lastIndex = 0;
+    for (let chunk = CHUNK_RE.exec(text); chunk; chunk = CHUNK_RE.exec(text)) {
+        if (URL_OR_ADDRESS_RE.test(chunk[0])) {
+            continue;
+        }
+        TOKEN_RE.lastIndex = 0;
+        for (let m = TOKEN_RE.exec(chunk[0]); m; m = TOKEN_RE.exec(chunk[0])) {
+            const start = chunk.index + m.index;
+            const end = start + m[0].length;
+            if (!gluedToNeighbour(text, start, end)) {
+                tokens.push({ start, end, value: m[0] });
+                addToken(text, start, m[0], out);
+            }
+        }
+    }
+    addDigitGroups(text, tokens, out);
+}
+function addToken(text, start, value, out) {
+    const end = start + value.length;
+    if (!value.includes('-')) {
+        if (codeLike(value)) {
+            out.push(make(text, start, end, 'token', value));
+        }
+        return;
+    }
+    const parts = value.split('-');
+    const joined = parts.join('');
+    if (/^\d+$/.test(joined)) {
+        // 123-456, 12-34-56: typed without the dashes. Other digit runs are phone numbers and dates
+        if (parts.every(p => p.length >= 2 && p.length <= 4) && codeLike(joined)) {
+            out.push(make(text, start, end, 'grouped', joined));
+        }
+        return;
+    }
+    if (joined.length >= 4 && value.length <= 40 && /\d/.test(joined) && /[A-Za-z]/.test(joined)) {
+        // ABCD-1234, G-123456, 4a7c01e9-82f3b6d0: keys printed with their dashes. Digit-only runs
+        // that are not two to four even groups are phone numbers and dates (372-5-555-0100)
+        out.push(make(text, start, end, 'token', value));
+    }
+    let offset = start;
+    for (const part of parts) {
+        if (codeLike(part)) {
+            out.push(make(text, offset, offset + part.length, 'dashed-part', part));
+        }
+        offset += part.length + 1;
+    }
+}
+// "123 456" and "1234 5678": two or three digit groups separated by a single space
+function addDigitGroups(text, tokens, out) {
+    for (let i = 0; i < tokens.length; i++) {
+        let digits = '';
+        for (let j = i; j < tokens.length && j < i + 3; j++) {
+            const token = tokens[j];
+            const separatedBySpace = j === i || (token.start === tokens[j - 1].end + 1 && text[token.start - 1] === ' ');
+            if (!/^\d{2,4}$/.test(token.value) || !separatedBySpace) {
+                break;
+            }
+            digits += token.value;
+            if (j > i && digits.length >= 6 && digits.length <= 9) {
+                out.push(make(text, tokens[i].start, token.end, 'grouped', digits));
+            }
+        }
+    }
+}
+function spacedCandidates(text, out) {
+    SPACED_RE.lastIndex = 0;
+    for (let m = SPACED_RE.exec(text); m; m = SPACED_RE.exec(text)) {
+        const code = m[0].replace(/[ \n]/g, '');
+        if (/\d/.test(code)) {
+            out.push(make(text, m.index, m.index + m[0].length, 'spaced', code));
+        }
+    }
+}
+export function extractCandidates(doc) {
+    const out = [];
+    tokenCandidates(doc.text, out);
+    spacedCandidates(doc.text, out);
+    out.sort((a, b) => a.start - b.start || b.end - a.end);
+    return out.slice(0, MAX_CANDIDATES);
+}

package/dist/esm/doc.d.ts

@@ -0,0 +1,52 @@
+export declare const DEFAULT_FONT_SIZE = 16;
+export type Style = {
+    fontSize: number;
+    bold: boolean;
+    letterSpacing: boolean;
+    link: boolean;
+    heading: boolean;
+    mono: boolean;
+    hidden: boolean;
+    /** Inside quoted reply history or a forwarded message */
+    quote: boolean;
+};
+export declare const DEFAULT_STYLE: Readonly<Style>;
+export type Span = {
+    start: number;
+    end: number;
+};
+export type StyledSpan = Span & {
+    style: Readonly<Style>;
+};
+export type Doc = {
+    source: 'html' | 'text';
+    text: string;
+    /** Consecutive, non-overlapping, sorted by start; text between spans is whitespace */
+    spans: StyledSpan[];
+    /** Every block, innermost ones included, sorted by start */
+    blocks: Span[];
+    /** Offset where reply history or a forwarded message starts, text.length if none */
+    quoteStart: number;
+};
+export declare class DocBuilder {
+    private parts;
+    private length;
+    private last;
+    private spans;
+    private blocks;
+    private openBlocks;
+    private readonly source;
+    constructor(source: 'html' | 'text');
+    private append;
+    text(raw: string, style: Readonly<Style>): void;
+    private dropTrailingSpace;
+    newline(): void;
+    openBlock(): void;
+    closeBlock(): void;
+    finish(): Doc;
+}
+export declare function renderText(input: string): Doc;
+/** The style at a text offset; whitespace between spans takes the style of the span before it */
+export declare function styleAt(doc: Doc, offset: number): Readonly<Style>;
+/** The smallest block that contains the whole range */
+export declare function blockAround(doc: Doc, start: number, end: number): Span;

package/dist/esm/doc.js

@@ -0,0 +1,144 @@
+// The common representation both the HTML and the plain text body are turned into: visible text,
+// the style of every stretch of it, and the blocks (paragraphs, table cells, lines) it is split in.
+export const DEFAULT_FONT_SIZE = 16;
+export const DEFAULT_STYLE = Object.freeze({
+    fontSize: DEFAULT_FONT_SIZE,
+    bold: false,
+    letterSpacing: false,
+    link: false,
+    heading: false,
+    mono: false,
+    hidden: false,
+    quote: false
+});
+const SPACE_RE = /[\s\u00a0\u200b\u200c\u200d\u2028\u2029\ufeff]+/g;
+// "On Tue, 1 Jan 2025 John wrote:", Outlook and forward separators, and the same in Estonian
+const ATTRIBUTION_RE = /^(?:-{2,}\s*(?:original message|forwarded message|edastatud kiri|algne kiri)\s*-{2,}|-{5,}|on .{3,200}(?:wrote|kirjutas):|.{3,120}kirjutas(?: .{0,80})?:|from: .{3,200}\n(?:sent|date): )/im;
+export class DocBuilder {
+    constructor(source) {
+        // The output is kept as parts plus its length and last character: reading back a string that
+        // is being appended to makes V8 flatten it on every call, which is quadratic on large bodies
+        this.parts = [];
+        this.length = 0;
+        this.last = '';
+        this.spans = [];
+        this.blocks = [];
+        this.openBlocks = [];
+        this.source = source;
+    }
+    append(chunk) {
+        this.parts.push(chunk);
+        this.length += chunk.length;
+        this.last = chunk[chunk.length - 1];
+    }
+    text(raw, style) {
+        const collapsed = raw.replace(SPACE_RE, ' ');
+        const atLineStart = this.length === 0 || this.last === '\n' || this.last === ' ';
+        const chunk = atLineStart && collapsed.startsWith(' ') ? collapsed.slice(1) : collapsed;
+        if (!chunk) {
+            return;
+        }
+        const start = this.length;
+        this.append(chunk);
+        const span = this.spans[this.spans.length - 1];
+        if (span && span.style === style && span.end === start) {
+            span.end = this.length;
+        }
+        else {
+            this.spans.push({ start, end: this.length, style });
+        }
+    }
+    dropTrailingSpace() {
+        if (this.last !== ' ') {
+            return;
+        }
+        const part = this.parts.pop();
+        const rest = part.slice(0, -1);
+        this.length--;
+        if (rest) {
+            this.parts.push(rest);
+        }
+        const prev = this.parts[this.parts.length - 1];
+        this.last = prev ? prev[prev.length - 1] : '';
+        const span = this.spans[this.spans.length - 1];
+        if (span && span.end > this.length) {
+            span.end = this.length;
+        }
+    }
+    newline() {
+        this.dropTrailingSpace();
+        if (this.length && this.last !== '\n') {
+            this.append('\n');
+        }
+    }
+    openBlock() {
+        this.newline();
+        this.openBlocks.push(this.length);
+    }
+    closeBlock() {
+        const start = this.openBlocks.pop();
+        if (start === undefined) {
+            return;
+        }
+        // Text is collapsed as it is added, so at most one whitespace character trails it
+        const end = this.length - (this.last === '\n' || this.last === ' ' ? 1 : 0);
+        if (end > start) {
+            this.blocks.push({ start, end });
+        }
+        this.newline();
+    }
+    finish() {
+        while (this.openBlocks.length) {
+            this.closeBlock();
+        }
+        const text = this.parts.join('').replace(/\s+$/, '');
+        const spans = this.spans.filter(s => s.start < text.length).map(s => ({ ...s, end: Math.min(s.end, text.length) }));
+        // Every line is a block too, which covers plain text and HTML without block elements
+        for (const line of text.matchAll(/[^\n]+/g)) {
+            this.blocks.push({ start: line.index, end: line.index + line[0].length });
+        }
+        this.blocks.sort((a, b) => a.start - b.start || b.end - a.end);
+        const attribution = ATTRIBUTION_RE.exec(text);
+        return { source: this.source, text, spans, blocks: this.blocks, quoteStart: attribution ? attribution.index : text.length };
+    }
+}
+const QUOTED_STYLE = Object.freeze({ ...DEFAULT_STYLE, quote: true });
+export function renderText(input) {
+    const doc = new DocBuilder('text');
+    for (const line of input.replace(/\r\n?/g, '\n').split('\n')) {
+        doc.text(line, /^\s*>/.test(line) ? QUOTED_STYLE : DEFAULT_STYLE);
+        doc.newline();
+    }
+    return doc.finish();
+}
+/** The style at a text offset; whitespace between spans takes the style of the span before it */
+export function styleAt(doc, offset) {
+    let lo = 0;
+    let hi = doc.spans.length - 1;
+    let found;
+    while (lo <= hi) {
+        const mid = Math.floor((lo + hi) / 2);
+        const span = doc.spans[mid];
+        if (span.start <= offset) {
+            found = span;
+            lo = mid + 1;
+        }
+        else {
+            hi = mid - 1;
+        }
+    }
+    return found?.style ?? DEFAULT_STYLE;
+}
+/** The smallest block that contains the whole range */
+export function blockAround(doc, start, end) {
+    let best = { start: 0, end: doc.text.length };
+    for (const block of doc.blocks) {
+        if (block.start > start) {
+            break;
+        }
+        if (block.end >= end && block.end - block.start < best.end - best.start) {
+            best = block;
+        }
+    }
+    return best;
+}

package/dist/esm/features.d.ts

@@ -0,0 +1,37 @@
+import type { Candidate } from './candidates.js';
+import type { Doc } from './doc.js';
+export declare const FEATURE_NAMES: readonly ["len", "digits", "upper", "lower", "isDigits", "isUpperAlpha", "isMixedCase", "kindGrouped", "kindSpaced", "kindDashedPart", "hasDash", "leadingZero", "distinctRatio", "trivialSequence", "looksYear", "colonBefore", "hashBefore", "currencyNear", "plusOrParenBefore", "digitNearby", "percentAfter", "strongBefore", "strongAfter", "weakBefore", "weakAfter", "cueBefore", "cueAfter", "negBefore", "negAfter", "strongInLine", "weakInLine", "negInLine", "strongRank", "lineLen", "aloneOnLine", "blockLen", "aloneInBlock", "fontSize", "fontRatio", "isLargestFont", "bold", "letterSpacing", "link", "heading", "mono", "hidden", "quoted", "relPosition", "fromHtml", "occurrences", "inOtherDoc", "inSubject", "subjectStrong", "subjectWeak", "subjectNeg", "docStrong", "docCue", "docLen", "candidates", "distinctCodes", "candidateIndex", "promoBefore", "promoAfter", "promoInLine", "docPromo", "subjectPromo", "vowelRatio", "longestLetterRun", "caseChanges"];
+type Hits = {
+    start: number;
+    end: number;
+}[];
+/** Per-document data shared by all candidates of the document */
+export type DocContext = {
+    doc: Doc;
+    strong: Hits;
+    weak: Hits;
+    cue: Hits;
+    neg: Hits;
+    promo: Hits;
+    dominantFont: number;
+    maxCandidateFont: number;
+    candidates: Candidate[];
+    strongRanks: Map<Candidate, number>;
+    /** How many candidates of the document share each code */
+    codeCounts: Map<string, number>;
+};
+/** Message data shared by the documents of one email */
+export type MessageContext = {
+    subject: string;
+    otherText: string;
+    distinctCodes: number;
+    subjectStrong: boolean;
+    subjectWeak: boolean;
+    subjectNeg: boolean;
+    subjectPromo: boolean;
+};
+export declare function messageContext(subject: string, otherText: string, distinctCodes: number): MessageContext;
+export declare function docContext(doc: Doc, candidates: Candidate[]): DocContext;
+/** The feature vector of the candidate at `index` in ctx.candidates */
+export declare function candidateFeatures(ctx: DocContext, msg: MessageContext, index: number): number[];
+export {};

package/dist/esm/features.js

@@ -0,0 +1,314 @@
+// Turns each candidate into a fixed-length vector of numbers for the scorer. FEATURE_NAMES is the
+// contract with the trained model: training records the names it was fit on, and the model refuses
+// to load against a different list. Add features at the end and retrain; never reorder.
+import { blockAround, styleAt } from './doc.js';
+import { CUE, findAll, matches, NEGATIVE, PROMO, STRONG, WEAK } from './keywords.js';
+export const FEATURE_NAMES = [
+    // shape
+    'len',
+    'digits',
+    'upper',
+    'lower',
+    'isDigits',
+    'isUpperAlpha',
+    'isMixedCase',
+    'kindGrouped',
+    'kindSpaced',
+    'kindDashedPart',
+    'hasDash',
+    'leadingZero',
+    'distinctRatio',
+    'trivialSequence',
+    'looksYear',
+    // characters around it
+    'colonBefore',
+    'hashBefore',
+    'currencyNear',
+    'plusOrParenBefore',
+    'digitNearby',
+    'percentAfter',
+    // keywords, distances in characters, capped
+    'strongBefore',
+    'strongAfter',
+    'weakBefore',
+    'weakAfter',
+    'cueBefore',
+    'cueAfter',
+    'negBefore',
+    'negAfter',
+    'strongInLine',
+    'weakInLine',
+    'negInLine',
+    'strongRank',
+    // layout and style
+    'lineLen',
+    'aloneOnLine',
+    'blockLen',
+    'aloneInBlock',
+    'fontSize',
+    'fontRatio',
+    'isLargestFont',
+    'bold',
+    'letterSpacing',
+    'link',
+    'heading',
+    'mono',
+    'hidden',
+    'quoted',
+    'relPosition',
+    'fromHtml',
+    // message
+    'occurrences',
+    'inOtherDoc',
+    'inSubject',
+    'subjectStrong',
+    'subjectWeak',
+    'subjectNeg',
+    'docStrong',
+    'docCue',
+    'docLen',
+    'candidates',
+    'distinctCodes',
+    'candidateIndex',
+    // codes that prove nothing: promotions, referrals, invitations
+    'promoBefore',
+    'promoAfter',
+    'promoInLine',
+    'docPromo',
+    'subjectPromo',
+    // letters-only codes are random (QWKZT), words in capitals are not (WEBINAR)
+    'vowelRatio',
+    // identifiers from code and prose (loginFailedError, retry2fa) have word-like letter runs
+    'longestLetterRun',
+    'caseChanges'
+];
+const FAR = 1000;
+const CURRENCY_RE = /[€$£¥₽]|\b(?:eur|usd|gbp|sek|nok|dkk|rub)\b/i;
+export function messageContext(subject, otherText, distinctCodes) {
+    return {
+        subject,
+        otherText,
+        distinctCodes,
+        subjectStrong: matches(STRONG, subject),
+        subjectWeak: matches(WEAK, subject),
+        subjectNeg: matches(NEGATIVE, subject),
+        subjectPromo: matches(PROMO, subject)
+    };
+}
+// Hits come from one global regex, so they are sorted and do not overlap: starts and ends both
+// increase, and every lookup below is a binary search instead of a scan of an uncapped list.
+/** Index of the first hit whose `key` is at least `pos`, hits.length if none */
+function firstAtLeast(hits, key, pos) {
+    let lo = 0;
+    let hi = hits.length;
+    while (lo < hi) {
+        const mid = Math.floor((lo + hi) / 2);
+        if (hits[mid][key] < pos) {
+            lo = mid + 1;
+        }
+        else {
+            hi = mid;
+        }
+    }
+    return lo;
+}
+/** Distance from the end of the closest hit that ends at or before pos */
+function distanceBefore(hits, pos) {
+    const i = firstAtLeast(hits, 'end', pos + 1) - 1;
+    return i >= 0 ? Math.min(FAR, pos - hits[i].end) : FAR;
+}
+/** Distance to the start of the first hit that starts at or after pos */
+function distanceAfter(hits, pos) {
+    const i = firstAtLeast(hits, 'start', pos);
+    return i < hits.length ? Math.min(FAR, hits[i].start - pos) : FAR;
+}
+/** Whether a whole hit lies within [from, to]; the first hit starting at from has the smallest end */
+function inRange(hits, from, to) {
+    const i = firstAtLeast(hits, 'start', from);
+    return i < hits.length && hits[i].end <= to;
+}
+function dominantFontSize(doc) {
+    const chars = new Map();
+    for (const span of doc.spans) {
+        if (!span.style.hidden) {
+            const size = Math.round(span.style.fontSize);
+            chars.set(size, (chars.get(size) ?? 0) + span.end - span.start);
+        }
+    }
+    let best = 16;
+    let most = 0;
+    for (const [size, count] of chars) {
+        if (count > most) {
+            best = size;
+            most = count;
+        }
+    }
+    return best;
+}
+function rankByStrongDistance(strong, candidates) {
+    // Keys computed once; the sort is stable, so equal distances keep document order
+    const keyed = candidates.map(c => ({ c, key: Math.min(distanceBefore(strong, c.start), distanceAfter(strong, c.end)) }));
+    keyed.sort((a, b) => a.key - b.key);
+    return new Map(keyed.map((k, i) => [k.c, i]));
+}
+export function docContext(doc, candidates) {
+    const strong = findAll(STRONG, doc.text);
+    const codeCounts = new Map();
+    for (const c of candidates) {
+        codeCounts.set(c.code, (codeCounts.get(c.code) ?? 0) + 1);
+    }
+    return {
+        doc,
+        strong,
+        weak: findAll(WEAK, doc.text),
+        cue: findAll(CUE, doc.text),
+        neg: findAll(NEGATIVE, doc.text),
+        promo: findAll(PROMO, doc.text),
+        dominantFont: dominantFontSize(doc),
+        maxCandidateFont: Math.max(0, ...candidates.map(c => styleAt(doc, c.start).fontSize)),
+        candidates,
+        strongRanks: rankByStrongDistance(strong, candidates),
+        codeCounts
+    };
+}
+function lineOf(text, c) {
+    const start = text.lastIndexOf('\n', c.start - 1) + 1;
+    const nl = text.indexOf('\n', c.end);
+    const end = nl < 0 ? text.length : nl;
+    return { start, end, text: text.slice(start, end) };
+}
+function shapeFeatures(c) {
+    const code = c.code;
+    const digits = (code.match(/\d/g) ?? []).length;
+    const upper = (code.match(/[A-Z]/g) ?? []).length;
+    const lower = (code.match(/[a-z]/g) ?? []).length;
+    const trivial = /^(\d)\1+$/.test(code) || '0123456789'.includes(code) || '9876543210'.includes(code);
+    return [
+        code.length,
+        digits,
+        upper,
+        lower,
+        Number(digits === code.length),
+        Number(upper === code.length),
+        Number(upper > 0 && lower > 0),
+        Number(c.kind === 'grouped'),
+        Number(c.kind === 'spaced'),
+        Number(c.kind === 'dashed-part'),
+        Number(c.raw.includes('-')),
+        Number(code.startsWith('0')),
+        new Set(code).size / code.length,
+        Number(trivial),
+        Number(/^(?:19|20)\d\d$/.test(code))
+    ];
+}
+function surroundingFeatures(text, c) {
+    const before = text.slice(Math.max(0, c.start - 4), c.start);
+    const after = text.slice(c.end, c.end + 4);
+    return [
+        Number(/:\s*$/.test(before)),
+        Number(/#\s*$/.test(before)),
+        Number(CURRENCY_RE.test(before) || CURRENCY_RE.test(after)),
+        Number(/[+(]\s*$/.test(before)),
+        Number(/\d/.test(text.slice(Math.max(0, c.start - 3), c.start)) || /\d/.test(text.slice(c.end, c.end + 3))),
+        Number(/^\s*%/.test(after))
+    ];
+}
+function keywordFeatures(ctx, c, line) {
+    return [
+        distanceBefore(ctx.strong, c.start),
+        distanceAfter(ctx.strong, c.end),
+        distanceBefore(ctx.weak, c.start),
+        distanceAfter(ctx.weak, c.end),
+        distanceBefore(ctx.cue, c.start),
+        distanceAfter(ctx.cue, c.end),
+        distanceBefore(ctx.neg, c.start),
+        distanceAfter(ctx.neg, c.end),
+        Number(inRange(ctx.strong, line.start, line.end)),
+        Number(inRange(ctx.weak, line.start, line.end)),
+        Number(inRange(ctx.neg, line.start, line.end)),
+        ctx.strongRanks.get(c)
+    ];
+}
+function layoutFeatures(ctx, c, line) {
+    const { doc } = ctx;
+    const style = styleAt(doc, c.start);
+    const block = blockAround(doc, c.start, c.end);
+    const blockText = doc.text.slice(block.start, block.end).trim();
+    const lineText = line.text.trim();
+    return [
+        lineText.length,
+        Number(lineText === c.raw),
+        blockText.length,
+        Number(blockText === c.raw),
+        style.fontSize,
+        style.fontSize / (ctx.dominantFont || 16),
+        Number(style.fontSize >= ctx.maxCandidateFont),
+        Number(style.bold),
+        Number(style.letterSpacing),
+        Number(style.link),
+        Number(style.heading),
+        Number(style.mono),
+        Number(style.hidden),
+        Number(style.quote || c.start >= doc.quoteStart),
+        doc.text.length ? c.start / doc.text.length : 0,
+        Number(doc.source === 'html')
+    ];
+}
+function vowelRatio(code) {
+    const letters = code.match(/[A-Za-z]/g) ?? [];
+    if (!letters.length) {
+        return -1;
+    }
+    return letters.filter(ch => /[aeiouy]/i.test(ch)).length / letters.length;
+}
+function promoFeatures(ctx, msg, c, line) {
+    return [
+        distanceBefore(ctx.promo, c.start),
+        distanceAfter(ctx.promo, c.end),
+        Number(inRange(ctx.promo, line.start, line.end)),
+        ctx.promo.length,
+        Number(msg.subjectPromo)
+    ];
+}
+function messageFeatures(ctx, msg, c, index) {
+    return [
+        ctx.codeCounts.get(c.code),
+        Number(msg.otherText.includes(c.raw) || msg.otherText.includes(c.code)),
+        Number(msg.subject.includes(c.raw) || msg.subject.includes(c.code)),
+        Number(msg.subjectStrong),
+        Number(msg.subjectWeak),
+        Number(msg.subjectNeg),
+        ctx.strong.length,
+        Number(ctx.cue.length > 0),
+        Math.log1p(ctx.doc.text.length),
+        ctx.candidates.length,
+        msg.distinctCodes,
+        index
+    ];
+}
+function codeTextFeatures(code) {
+    return [
+        vowelRatio(code),
+        Math.max(0, ...(code.match(/[A-Za-z]+/g) ?? []).map(run => run.length)),
+        (code.match(/[a-z](?=[A-Z])|[A-Z](?=[a-z])/g) ?? []).length
+    ];
+}
+/** The feature vector of the candidate at `index` in ctx.candidates */
+export function candidateFeatures(ctx, msg, index) {
+    const c = ctx.candidates[index];
+    const text = ctx.doc.text;
+    const line = lineOf(text, c);
+    const row = [
+        ...shapeFeatures(c),
+        ...surroundingFeatures(text, c),
+        ...keywordFeatures(ctx, c, line),
+        ...layoutFeatures(ctx, c, line),
+        ...messageFeatures(ctx, msg, c, index),
+        ...promoFeatures(ctx, msg, c, line),
+        ...codeTextFeatures(c.code)
+    ];
+    if (row.length !== FEATURE_NAMES.length) {
+        throw new Error(`feature vector has ${row.length} values, expected ${FEATURE_NAMES.length}`);
+    }
+    return row;
+}

package/dist/esm/html.d.ts

@@ -0,0 +1,3 @@
+import type { Doc } from './doc.js';
+export declare function decodeEntities(input: string): string;
+export declare function renderHtml(html: string): Doc;