humanizer-lint 0.0.0-stage → 0.2.3

package/scripts/utils/metrics.mjs

@@ -0,0 +1,73 @@
+import { deflateRawSync } from 'node:zlib';
+
+export function mean(values) {
+  return values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : 0;
+}
+
+export function standardDeviation(values) {
+  if (!values.length) return 0;
+  const average = mean(values);
+  return Math.sqrt(mean(values.map((value) => (value - average) ** 2)));
+}
+
+export function coefficientOfVariation(values) {
+  const average = mean(values);
+  return average ? standardDeviation(values) / average : 0;
+}
+
+export function adjacentDelta(values) {
+  if (values.length < 2) return 0;
+  let total = 0;
+  for (let index = 1; index < values.length; index += 1) total += Math.abs(values[index] - values[index - 1]);
+  return total / (values.length - 1);
+}
+
+export function mattr(words, requestedWindow) {
+  if (!words.length) return 0;
+  const window = Math.min(words.length, requestedWindow ?? Math.min(50, Math.max(10, Math.floor(words.length / 2))));
+  if (words.length <= window) return new Set(words).size / words.length;
+  let total = 0;
+  let windows = 0;
+  for (let index = 0; index <= words.length - window; index += 1) {
+    total += new Set(words.slice(index, index + window)).size / window;
+    windows += 1;
+  }
+  return total / windows;
+}
+
+export function compressionRatio(text) {
+  const buffer = Buffer.from(String(text ?? '').trim(), 'utf8');
+  return buffer.length ? deflateRawSync(buffer).length / buffer.length : 0;
+}
+
+export function punctuationRates(text, wordCount) {
+  const denominator = wordCount || 1;
+  const count = (pattern) => (String(text ?? '').match(pattern) ?? []).length / denominator;
+  return {
+    comma: count(/,/g),
+    semicolon: count(/;/g),
+    colon: count(/:/g),
+    question: count(/\?/g),
+    exclamation: count(/!/g),
+    parentheses: count(/[()]/g) / 2,
+    dash: count(/—|\s--\s/g),
+  };
+}
+
+export function calculateMetrics(tokenized, normalizedText) {
+  const lengths = tokenized.sentences.map((sentence) => sentence.wordCount);
+  const wordCount = tokenized.words.length;
+  const sentenceCount = lengths.length;
+  return {
+    wordCount,
+    sentenceCount,
+    meanSentenceLength: mean(lengths),
+    stdDev: standardDeviation(lengths),
+    burstinessCV: coefficientOfVariation(lengths),
+    adjacentDelta: adjacentDelta(lengths),
+    shortSentenceRatio: sentenceCount ? lengths.filter((length) => length <= 6).length / sentenceCount : 0,
+    longSentenceRatio: sentenceCount ? lengths.filter((length) => length >= 26).length / sentenceCount : 0,
+    mattr: mattr(tokenized.words),
+    compressionRatio: compressionRatio(normalizedText),
+  };
+}

package/scripts/utils/sanitizer.mjs

@@ -0,0 +1,48 @@
+const ENTITY = 'TOKEN_ENTITY';
+
+function isCompleteListItem(text) {
+  const words = text.match(/[\p{L}\p{N}]+/gu) ?? [];
+  return words.length >= 5 && /[.!?]["')\]]?$/.test(text.trim());
+}
+
+export function sanitizeMarkdown(input) {
+  let text = String(input ?? '').replace(/\r\n?/g, '\n');
+  text = text.replace(/^---\s*\n[\s\S]*?\n---\s*(?:\n|$)/, '');
+  text = text.replace(/^(?:```|~~~)[^\n]*\n[\s\S]*?^(?:```|~~~)\s*$/gm, '\n');
+  text = text.replace(/\[([^\]]+)\]\(https?:\/\/[^\s)]+\)/gi, '$1');
+  text = text.replace(/`[^`\n]+`/g, ENTITY);
+  text = text.replace(/https?:\/\/[^\s)]+/gi, (match) => {
+    const trailing = match.match(/[.!?,;:]+$/u)?.[0] ?? '';
+    return `${ENTITY}${trailing}`;
+  });
+
+  const sourceLines = text.split('\n');
+  const lines = [];
+  for (let index = 0; index < sourceLines.length; index += 1) {
+    const rawLine = sourceLines[index];
+    if (/^\s*#{1,6}\s+/.test(rawLine)) {
+      lines.push('');
+      continue;
+    }
+    if (/^\s*\|.*\|\s*$/.test(rawLine)) {
+      lines.push('');
+      continue;
+    }
+
+    let line = rawLine.replace(/^\s*>\s?/, '');
+    const list = line.match(/^\s*(?:[-*+]|\d+\.)\s+(.*)$/);
+    if (list) {
+      let item = list[1].trim();
+      let cursor = index + 1;
+      while (cursor < sourceLines.length && /^\s{2,}\S/.test(sourceLines[cursor])) {
+        item += ` ${sourceLines[cursor].replace(/^\s*>\s?/, '').trim()}`;
+        cursor += 1;
+      }
+      index = cursor - 1;
+      line = isCompleteListItem(item) ? item : '';
+    }
+    lines.push(line.trimEnd());
+  }
+
+  return lines.join('\n').replace(/\n{3,}/g, '\n\n').trim();
+}

package/scripts/utils/tokenizer.mjs

@@ -0,0 +1,82 @@
+const ABBREVIATIONS = {
+  id: ['dll.', 'dsb.', 'dst.', 'dkk.', 'hlm.', 'no.', 'dr.', 'prof.', 'ir.', 'hj.', 'pt.', 'cv.', 'tbk.', 'rp.', 'tgl.', 's.d.', 'a.n.', 'u.p.'],
+  en: ['mr.', 'mrs.', 'ms.', 'dr.', 'prof.', 'sr.', 'jr.', 'vs.', 'etc.', 'e.g.', 'i.e.', 'fig.', 'inc.', 'ltd.', 'co.', 'st.'],
+};
+
+const DOT = '\uE000';
+const DECIMAL = '\uE001';
+const ELLIPSIS = '\uE002';
+
+function escapeRegExp(value) {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+function protectAbbreviations(text, language) {
+  const items = language === 'auto'
+    ? [...new Set([...ABBREVIATIONS.id, ...ABBREVIATIONS.en])]
+    : ABBREVIATIONS[language] ?? ABBREVIATIONS.en;
+  let protectedText = text
+    .replace(/\.{2,}/g, ELLIPSIS)
+    .replace(/(\d)\.(\d)/g, `$1${DECIMAL}$2`);
+  for (const abbreviation of items.sort((a, b) => b.length - a.length)) {
+    protectedText = protectedText.replace(
+      new RegExp(`\\b${escapeRegExp(abbreviation)}`, 'gi'),
+      (match) => match.replaceAll('.', DOT),
+    );
+  }
+  return protectedText;
+}
+
+export function tokenizeWords(text) {
+  const normalized = String(text ?? '').replace(/—|--/g, ' ');
+  return (normalized.toLowerCase().match(/token_entity|[\p{L}\p{N}]+(?:['’][\p{L}\p{N}]+)?(?:-[\p{L}\p{N}]+)*/gu) ?? []);
+}
+
+export function tokenizeText(input, language = 'auto') {
+  const paragraphs = [];
+  const sentences = [];
+  const rawParagraphs = String(input ?? '').replace(/\r\n?/g, '\n').split(/\n\s*\n+/).map((value) => value.trim()).filter(Boolean);
+
+  for (const [paragraphOffset, rawParagraph] of rawParagraphs.entries()) {
+    const protectedText = protectAbbreviations(rawParagraph.replace(/\s+/g, ' '), language);
+    const markedText = protectedText.replace(/([.!?](?:["')\]”’]+)?)\s+/gu, '$1\n');
+    const parts = markedText.split('\n').map((value) => value
+      .replaceAll(DOT, '.')
+      .replaceAll(DECIMAL, '.')
+      .replaceAll(ELLIPSIS, '...')
+      .trim()).filter(Boolean);
+    const paragraphSentences = [];
+
+    for (const text of parts) {
+      const words = tokenizeWords(text);
+      if (!words.length) continue;
+      const sentence = {
+        text,
+        words,
+        wordCount: words.length,
+        paragraphIndex: paragraphOffset + 1,
+        sentenceIndex: sentences.length + 1,
+        sentenceInParagraph: paragraphSentences.length + 1,
+      };
+      paragraphSentences.push(sentence);
+      sentences.push(sentence);
+    }
+
+    if (paragraphSentences.length) {
+      paragraphs.push({
+        text: rawParagraph,
+        paragraphIndex: paragraphOffset + 1,
+        sentences: paragraphSentences,
+        words: paragraphSentences.flatMap((sentence) => sentence.words),
+      });
+    }
+  }
+
+  return {
+    paragraphs,
+    sentences,
+    words: sentences.flatMap((sentence) => sentence.words),
+  };
+}
+
+export { ABBREVIATIONS };