@roughen/core 0.3.0

package/dist-cjs/sentences.js

@@ -0,0 +1,98 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.segmentSentences = segmentSentences;
+const protected_tokens_js_1 = require("./protected-tokens.js");
+const abbreviations = new Set([
+    'dr', 'mr', 'mrs', 'ms', 'prof', 'sr', 'jr', 'st',
+    'e.g', 'i.e', 'etc', 'cf', 'vs', 'al',
+    'inc', 'ltd', 'co', 'corp', 'dept', 'est',
+]);
+/** Width of the letter code point ending at `end`, or 0. Surrogate pairs decode as `/u` does. */
+function letterBefore(text, end) {
+    const low = text.charCodeAt(end - 1);
+    const high = text.charCodeAt(end - 2);
+    const width = low >= 0xDC00 && low <= 0xDFFF && high >= 0xD800 && high <= 0xDBFF ? 2 : end > 0 ? 1 : 0;
+    return width && /^\p{L}$/u.test(text.slice(end - width, end)) ? width : 0;
+}
+/**
+ * The chain `/([\p{L}]+(?:\.[\p{L}]+)*)$/u` matches on `text.slice(0, end)`,
+ * scanned backward in time proportional to the chain instead of the text.
+ */
+function precedingChain(text, end) {
+    let start = end;
+    for (let width = 0;;) {
+        if ((width = letterBefore(text, start)))
+            start -= width;
+        else if (start < end && text[start - 1] === '.' && letterBefore(text, start - 1))
+            start--;
+        else
+            return text.slice(start, end);
+    }
+}
+/** Deterministic segmentation with original UTF-16 ranges; it does not infer authorship. */
+function segmentSentences(text, offset = 0) {
+    if (!Number.isSafeInteger(offset) || offset < 0)
+        throw new RangeError('Sentence offset must be a non-negative safe integer.');
+    const result = [];
+    const protectedRanges = (0, protected_tokens_js_1.protectedTokenRanges)(text);
+    const push = (start, end) => {
+        while (start < end && /\s/.test(text[start]))
+            start++;
+        while (end > start && /\s/.test(text[end - 1]))
+            end--;
+        if (end <= start)
+            return;
+        const sentence = text.slice(start, end);
+        const wordCount = (sentence.match(/[\p{L}\p{N}]+(?:['’.-][\p{L}\p{N}]+)*/gu) ?? []).length;
+        if (wordCount)
+            result.push({ text: sentence, range: [offset + start, offset + end], wordCount });
+    };
+    let start = 0;
+    let protectedIndex = 0;
+    // Every mark before `settled` shares a terminal group already found not to end a sentence.
+    let settled = 0;
+    const opener = /[\p{Lu}\p{Lt}\p{N}"'“‘([{]/uy;
+    for (let i = 0; i < text.length; i++) {
+        const char = text[i];
+        if (!/[.!?]/.test(char) || i < settled)
+            continue;
+        while (protectedIndex < protectedRanges.length && protectedRanges[protectedIndex][1] <= i)
+            protectedIndex++;
+        const protectedRange = protectedRanges[protectedIndex];
+        if (protectedRange && i >= protectedRange[0] && i < protectedRange[1])
+            continue;
+        if (char === '.') {
+            if (text[i - 1] === '.' || text[i + 1] === '.')
+                continue;
+            if (/\d/.test(text[i - 1] ?? '') && /\d/.test(text[i + 1] ?? ''))
+                continue;
+        }
+        let end = i + 1;
+        while (/[.!?]/.test(text[end] ?? '') && end < text.length)
+            end++;
+        while (/["'”’\])}]/.test(text[end] ?? '') && end < text.length)
+            end++;
+        let next = end;
+        while (next < text.length && /\s/.test(text[next]))
+            next++;
+        opener.lastIndex = next;
+        if (!(next === text.length || (next > end && opener.test(text)))) {
+            settled = end;
+            continue;
+        }
+        // The abbreviation lookback runs last, so only a real boundary candidate pays for it.
+        if (char === '.') {
+            const preceding = precedingChain(text, i);
+            if (abbreviations.has(preceding.toLowerCase()))
+                continue;
+            if (/^(?:\p{Lu}\.)*\p{Lu}$/u.test(preceding))
+                continue;
+        }
+        push(start, end);
+        start = next;
+        i = next - 1;
+    }
+    push(start, text.length);
+    return result;
+}
+//# sourceMappingURL=sentences.js.map

package/dist-cjs/sentences.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sentences.js","sourceRoot":"","sources":["../src/sentences.ts"],"names":[],"mappings":";;AA+BA,4CA4CC;AA1ED,+DAA6D;AAE7D,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC;IAC5B,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI;IACjD,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI;IACrC,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK;CAC1C,CAAC,CAAC;AAEH,iGAAiG;AACjG,SAAS,YAAY,CAAC,IAAY,EAAE,GAAW;IAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;IACrC,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;IACtC,MAAM,KAAK,GAAG,GAAG,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACvG,OAAO,KAAK,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AAC5E,CAAC;AAED;;;GAGG;AACH,SAAS,cAAc,CAAC,IAAY,EAAE,GAAW;IAC/C,IAAI,KAAK,GAAG,GAAG,CAAC;IAChB,KAAK,IAAI,KAAK,GAAG,CAAC,IAAK,CAAC;QACtB,IAAI,CAAC,KAAK,GAAG,YAAY,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAAE,KAAK,IAAI,KAAK,CAAC;aACnD,IAAI,KAAK,GAAG,GAAG,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,GAAG,IAAI,YAAY,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC;YAAE,KAAK,EAAE,CAAC;;YACrF,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACrC,CAAC;AACH,CAAC;AAED,4FAA4F;AAC5F,SAAgB,gBAAgB,CAAC,IAAY,EAAE,MAAM,GAAG,CAAC;IACvD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,sDAAsD,CAAC,CAAC;IAC9H,MAAM,MAAM,GAAe,EAAE,CAAC;IAC9B,MAAM,eAAe,GAAG,IAAA,0CAAoB,EAAC,IAAI,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,GAAW,EAAE,EAAE;QAC1C,OAAO,KAAK,GAAG,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAE,CAAC;YAAE,KAAK,EAAE,CAAC;QACvD,OAAO,GAAG,GAAG,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAE,CAAC;YAAE,GAAG,EAAE,CAAC;QACvD,IAAI,GAAG,IAAI,KAAK;YAAE,OAAO;QACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,yCAAyC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;QAC3F,IAAI,SAAS;YAAE,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,MAAM,GAAG,KAAK,EAAE,MAAM,GAAG,GAAG,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC;IACnG,CAAC,CAAC;IACF,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,2FAA2F;IAC3F,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,MAAM,MAAM,GAAG,8BAA8B,CAAC;IAC9C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAE,CAAC;QACtB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,OAAO;YAAE,SAAS;QACjD,OAAO,cAAc,GAAG,eAAe,CAAC,MAAM,IAAI,eAAe,CAAC,cAAc,CAAE,CAAC,CAAC,CAAC,IAAI,CAAC;YAAE,cAAc,EAAE,CAAC;QAC7G,MAAM,cAAc,GAAG,eAAe,CAAC,cAAc,CAAC,CAAC;QACvD,IAAI,cAAc,IAAI,CAAC,IAAI,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,CAAC,CAAC;YAAE,SAAS;QAChF,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACjB,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG;gBAAE,SAAS;YACzD,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;gBAAE,SAAS;QAC7E,CAAC;QACD,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QAChB,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM;YAAE,GAAG,EAAE,CAAC;QACjE,OAAO,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM;YAAE,GAAG,EAAE,CAAC;QACtE,IAAI,IAAI,GAAG,GAAG,CAAC;QACf,OAAO,IAAI,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAE,CAAC;YAAE,IAAI,EAAE,CAAC;QAC5D,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,GAAG,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YAAC,OAAO,GAAG,GAAG,CAAC;YAAC,SAAS;QAAC,CAAC;QAC9F,sFAAsF;QACtF,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACjB,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC1C,IAAI,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC;gBAAE,SAAS;YACzD,IAAI,wBAAwB,CAAC,IAAI,CAAC,SAAS,CAAC;gBAAE,SAAS;QACzD,CAAC;QACD,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAAC,KAAK,GAAG,IAAI,CAAC;QAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC;IAC/C,CAAC;IACD,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACzB,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist-cjs/types.d.ts

@@ -0,0 +1,140 @@
+/** All ranges are half-open UTF-16 offsets into the untouched source string. */
+export type Range = [number, number];
+export type Format = 'text' | 'md' | 'mdx' | 'html' | 'source';
+export type Severity = 'error' | 'warn' | 'info';
+export type FixSafety = 'safe' | 'careful' | 'manual';
+export interface Block {
+    kind: 'paragraph' | 'heading' | 'listItem' | 'blockquote' | 'code' | 'html';
+    /** A length-preserving prose view. Protected syntax is replaced with spaces. */
+    text: string;
+    sourceRange: Range;
+    depth?: number;
+    skip: boolean;
+}
+export interface Document {
+    source: string;
+    format: Format;
+    blocks: Block[];
+    protectedRanges: Range[];
+    /** Code: fences, spans, frontmatter, raw HTML code and expressions. A subset of protectedRanges. */
+    codeRanges: Range[];
+    /** Verbatim quotations and blockquotes. A subset of protectedRanges. */
+    quoteRanges: Range[];
+}
+export interface MarkdownLinkTarget {
+    /** Exact source spelling, including any Markdown backslash escapes. */
+    target: string;
+    sourceRange: Range;
+    kind: 'inline' | 'reference-definition';
+}
+export interface Sentence {
+    text: string;
+    range: Range;
+    wordCount: number;
+}
+export interface Finding {
+    ruleId: string;
+    severity: Severity;
+    message: string;
+    why: string;
+    range: Range;
+    loc: {
+        line: number;
+        column: number;
+        endLine: number;
+        endColumn: number;
+    };
+    fix?: {
+        range: Range;
+        text: string;
+    };
+    fixSafety: FixSafety;
+    /** Confidence in the described mechanical match, never authorship probability. */
+    confidence: number;
+    evidence?: Record<string, number | string>;
+}
+export type RuleFinding = Omit<Finding, 'ruleId' | 'severity' | 'loc'>;
+export type RuleSetting = Severity | 'off' | [Severity | 'off', Record<string, unknown>];
+/**
+ * Which human baseline the density rules measure against. 'article' is the
+ * 110 human articles every threshold was fit on; 'marketing' swaps in the
+ * thresholds fit on human marketing pages where they differ (list saturation).
+ */
+export type Register = 'article' | 'marketing';
+/** A character the project bans outright in copy, reported at every occurrence. */
+export interface BannedCharacter {
+    character: string;
+    use?: string;
+    why?: string;
+}
+/** A pattern (a regular expression source) the project bans outright in copy. */
+export interface BannedPattern {
+    pattern: string;
+    flags?: string;
+    message?: string;
+    use?: string;
+}
+export interface Config {
+    format?: Format;
+    fix?: boolean;
+    careful?: boolean;
+    register?: Register;
+    rules?: Record<string, RuleSetting>;
+    voice?: {
+        banned?: {
+            term: string;
+            use?: string;
+        }[];
+        required?: string[];
+        /** Absolute bans, whatever the density: em dashes on a site that forbids them, say. */
+        bannedCharacters?: (string | BannedCharacter)[];
+        bannedPatterns?: BannedPattern[];
+    };
+}
+export type DensityFamily = 'series' | 'contrast' | 'generalizer' | 'reveal' | 'emDash' | 'semicolon';
+/** Instances of each structural habit in editable body prose, with the words they're measured against. */
+export interface DensityProfile {
+    words: number;
+    instances: Record<DensityFamily, Range[]>;
+}
+export interface RuleContext {
+    document: Document;
+    config: Config;
+    register: Register;
+    options: Record<string, unknown>;
+    /** Sentences of editable body prose (paragraphs and list items), in source order. */
+    sentences: Sentence[];
+    profile: DensityProfile;
+}
+export interface Rule {
+    id: string;
+    severity: Severity;
+    description: string;
+    /** Registry order is the stable tie-breaker for overlapping fixes. */
+    check(context: RuleContext): RuleFinding[];
+}
+export interface Metrics {
+    sentenceWords: number[];
+    sentenceCount: number;
+    wordCount: number;
+    meanSentenceLength: number | null;
+    sentenceLengthCV: number | null;
+    meanAbsoluteSuccessiveDifference: number | null;
+}
+export interface Result {
+    /** With fix:true these findings refer to fixed, otherwise to the input. */
+    findings: Finding[];
+    /** No calibrated corpus exists yet. Never invent a texture or AI score. */
+    score: null;
+    calibration: {
+        status: 'unavailable';
+        reason: string;
+    };
+    metrics: Metrics;
+    fixed?: string;
+    fixes?: {
+        applied: number;
+        passes: number;
+        converged: boolean;
+    };
+}

package/dist-cjs/types.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map

package/dist-cjs/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}

package/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "@roughen/core",
+  "version": "0.3.0",
+  "description": "Deterministic, local prose linting with explained findings and safe fixes.",
+  "license": "MIT",
+  "keywords": [
+    "prose",
+    "linter",
+    "writing",
+    "ai-writing",
+    "copy",
+    "editing",
+    "style",
+    "deterministic"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "type": "module",
+  "main": "./dist-cjs/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/index.d.ts",
+        "default": "./dist/index.js"
+      },
+      "require": {
+        "types": "./dist-cjs/index.d.ts",
+        "default": "./dist-cjs/index.js"
+      }
+    }
+  },
+  "files": [
+    "dist",
+    "dist-cjs",
+    "README.md",
+    "LICENSE"
+  ],
+  "sideEffects": false,
+  "scripts": {
+    "build": "tsc -p tsconfig.json && tsc -p tsconfig.cjs.json && node scripts/cjs.mjs"
+  }
+}