humanizer-lint 0.0.0-stage → 0.2.4

package/scripts/fingerprint.mjs

@@ -0,0 +1,127 @@
+#!/usr/bin/env node
+import { promises as fs, realpathSync } from 'node:fs';
+import path from 'node:path';
+import { parseArgs } from 'node:util';
+
+import { sanitizeMarkdown } from './utils/sanitizer.mjs';
+import { calculateMetrics, mean, punctuationRates } from './utils/metrics.mjs';
+import { tokenizeText } from './utils/tokenizer.mjs';
+
+const { mkdir, readFile, readdir, rename, stat, writeFile } = fs;
+
+function round(value, places = 4) {
+  const factor = 10 ** places;
+  return Math.round(value * factor) / factor;
+}
+
+function averageObjects(objects) {
+  const keys = [...new Set(objects.flatMap((object) => Object.keys(object)))];
+  return Object.fromEntries(keys.map((key) => [key, round(mean(objects.map((object) => object[key] ?? 0)))]));
+}
+
+export async function fingerprintSamples(files, options = {}) {
+  if (!Array.isArray(files) || files.length < 1 || files.length > 5) throw new Error('Fingerprint requires 1 to 5 sample files');
+  const samples = [];
+  for (const file of files) {
+    const raw = await readFile(file, 'utf8');
+    const sanitized = sanitizeMarkdown(raw);
+    const tokenized = tokenizeText(sanitized, 'auto');
+    const metrics = calculateMetrics(tokenized, sanitized);
+    if (metrics.wordCount < 20 || metrics.sentenceCount < 3) throw new Error(`Sample is too short: ${file}`);
+    samples.push({
+      ...metrics,
+      averageParagraphSentenceCount: mean(tokenized.paragraphs.map((paragraph) => paragraph.sentences.length)),
+      punctuationRates: punctuationRates(sanitized, metrics.wordCount),
+    });
+  }
+
+  const punctuation = averageObjects(samples.map((sample) => sample.punctuationRates));
+  const profile = {
+    name: options.name ?? 'custom-voice',
+    version: 1,
+    sampleCount: samples.length,
+    language: options.language ?? 'auto',
+    metrics: {
+      meanSentenceLength: round(mean(samples.map((sample) => sample.meanSentenceLength))),
+      stdDev: round(mean(samples.map((sample) => sample.stdDev))),
+      burstinessCV: round(mean(samples.map((sample) => sample.burstinessCV))),
+      adjacentDelta: round(mean(samples.map((sample) => sample.adjacentDelta))),
+      mattr: round(mean(samples.map((sample) => sample.mattr))),
+      averageParagraphSentenceCount: round(mean(samples.map((sample) => sample.averageParagraphSentenceCount))),
+      punctuationRates: punctuation,
+    },
+  };
+
+  if (options.output) {
+    const output = path.resolve(options.output);
+    await mkdir(path.dirname(output), { recursive: true });
+    const temporary = `${output}.tmp-${process.pid}`;
+    await writeFile(temporary, `${JSON.stringify(profile, null, 2)}\n`, 'utf8');
+    await rename(temporary, output);
+  }
+  return profile;
+}
+
+async function collectInput(input) {
+  const info = await stat(input);
+  if (info.isFile()) return [path.resolve(input)];
+  if (!info.isDirectory()) throw new Error('--input must be a file or directory');
+  const entries = await readdir(input, { withFileTypes: true });
+  return entries
+    .filter((entry) => entry.isFile() && /\.(?:md|txt)$/i.test(entry.name))
+    .map((entry) => path.resolve(input, entry.name))
+    .sort()
+    .slice(0, 5);
+}
+
+export async function runFingerprintCli(argv, io = {}) {
+  const stdout = io.stdout ?? ((value) => process.stdout.write(value));
+  const stderr = io.stderr ?? ((value) => process.stderr.write(value));
+  try {
+    const { values } = parseArgs({
+      args: argv,
+      options: {
+        input: { type: 'string' },
+        output: { type: 'string' },
+        name: { type: 'string' },
+        lang: { type: 'string', default: 'auto' },
+        help: { type: 'boolean' },
+      },
+      strict: true,
+      allowPositionals: false,
+    });
+    if (values.help) {
+      stdout('Usage: humanizer-fingerprint --input <file-or-dir> --output <profile.json> [--name <name>] [--lang id|en|auto]\n');
+      return 0;
+    }
+    if (!values.input || !values.output) throw new Error('--input and --output are required');
+    if (!['id', 'en', 'auto'].includes(values.lang)) throw new Error('--lang must be id, en, or auto');
+    const files = await collectInput(values.input);
+    if (!files.length) throw new Error('No .txt or .md samples found');
+    const profile = await fingerprintSamples(files, { output: values.output, name: values.name, language: values.lang });
+    stdout(`${JSON.stringify(profile, null, 2)}\n`);
+    return 0;
+  } catch (error) {
+    stderr(`ERROR: ${error.message}\n`);
+    return 2;
+  }
+}
+
+function currentModulePath() {
+  let value = decodeURIComponent(new URL(import.meta.url).pathname);
+  if (/^\/[A-Za-z]:/.test(value)) value = value.slice(1);
+  return path.resolve(value);
+}
+
+function isMainModule() {
+  if (!process.argv[1]) return false;
+  try {
+    return realpathSync(process.argv[1]) === realpathSync(currentModulePath());
+  } catch {
+    return false;
+  }
+}
+
+if (isMainModule()) {
+  process.exitCode = await runFingerprintCli(process.argv.slice(2));
+}

package/scripts/utils/metrics-core.mjs

@@ -0,0 +1,84 @@
+export function mean(values) {
+  return values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : 0;
+}
+
+export function standardDeviation(values) {
+  if (!values.length) return 0;
+  const average = mean(values);
+  return Math.sqrt(mean(values.map((value) => (value - average) ** 2)));
+}
+
+export function coefficientOfVariation(values) {
+  const average = mean(values);
+  return average ? standardDeviation(values) / average : 0;
+}
+
+export function adjacentDelta(values) {
+  if (values.length < 2) return 0;
+  let total = 0;
+  for (let index = 1; index < values.length; index += 1) total += Math.abs(values[index] - values[index - 1]);
+  return total / (values.length - 1);
+}
+
+export function mattr(words, requestedWindow) {
+  if (!words.length) return 0;
+  const window = Math.min(words.length, requestedWindow ?? Math.min(50, Math.max(10, Math.floor(words.length / 2))));
+  if (words.length <= window) return new Set(words).size / words.length;
+  let total = 0;
+  let windows = 0;
+  for (let index = 0; index <= words.length - window; index += 1) {
+    total += new Set(words.slice(index, index + window)).size / window;
+    windows += 1;
+  }
+  return total / windows;
+}
+
+export function punctuationRates(text, wordCount) {
+  const denominator = wordCount || 1;
+  const count = (pattern) => (String(text ?? '').match(pattern) ?? []).length / denominator;
+  return {
+    comma: count(/,/g),
+    semicolon: count(/;/g),
+    colon: count(/:/g),
+    question: count(/\?/g),
+    exclamation: count(/!/g),
+    parentheses: count(/[()]/g) / 2,
+    dash: count(/—|\s--\s/g),
+  };
+}
+
+export function calculateMetricsFromCompressionRatio(tokenized, compressionRatio) {
+  const lengths = tokenized.sentences.map((sentence) => sentence.wordCount);
+  const wordCount = tokenized.words.length;
+  const sentenceCount = lengths.length;
+  return {
+    wordCount,
+    sentenceCount,
+    meanSentenceLength: mean(lengths),
+    stdDev: standardDeviation(lengths),
+    burstinessCV: coefficientOfVariation(lengths),
+    adjacentDelta: adjacentDelta(lengths),
+    shortSentenceRatio: sentenceCount ? lengths.filter((length) => length <= 6).length / sentenceCount : 0,
+    longSentenceRatio: sentenceCount ? lengths.filter((length) => length >= 26).length / sentenceCount : 0,
+    mattr: mattr(tokenized.words),
+    compressionRatio,
+  };
+}
+
+export function createMetrics({ encode, deflateRaw }) {
+  if (typeof encode !== 'function' || typeof deflateRaw !== 'function') {
+    throw new TypeError('createMetrics requires encode and deflateRaw functions');
+  }
+
+  const compressionRatio = (text) => {
+    const bytes = encode(String(text ?? '').trim());
+    return bytes.length ? deflateRaw(bytes).length / bytes.length : 0;
+  };
+
+  const calculateMetrics = (tokenized, normalizedText) => calculateMetricsFromCompressionRatio(
+    tokenized,
+    compressionRatio(normalizedText),
+  );
+
+  return { compressionRatio, calculateMetrics };
+}

package/scripts/utils/metrics.mjs

@@ -0,0 +1,28 @@
+import { deflateRawSync } from 'node:zlib';
+
+import {
+  adjacentDelta,
+  coefficientOfVariation,
+  createMetrics,
+  mattr,
+  mean,
+  punctuationRates,
+  standardDeviation,
+} from './metrics-core.mjs';
+
+const metrics = createMetrics({
+  encode: (text) => Buffer.from(text, 'utf8'),
+  deflateRaw: (bytes) => deflateRawSync(bytes),
+});
+
+export {
+  adjacentDelta,
+  coefficientOfVariation,
+  mattr,
+  mean,
+  punctuationRates,
+  standardDeviation,
+};
+
+export const compressionRatio = metrics.compressionRatio;
+export const calculateMetrics = metrics.calculateMetrics;

package/scripts/utils/sanitizer.mjs

@@ -0,0 +1,48 @@
+const ENTITY = 'TOKEN_ENTITY';
+
+function isCompleteListItem(text) {
+  const words = text.match(/[\p{L}\p{N}]+/gu) ?? [];
+  return words.length >= 5 && /[.!?]["')\]]?$/.test(text.trim());
+}
+
+export function sanitizeMarkdown(input) {
+  let text = String(input ?? '').replace(/\r\n?/g, '\n');
+  text = text.replace(/^---\s*\n[\s\S]*?\n---\s*(?:\n|$)/, '');
+  text = text.replace(/^(?:```|~~~)[^\n]*\n[\s\S]*?^(?:```|~~~)\s*$/gm, '\n');
+  text = text.replace(/\[([^\]]+)\]\(https?:\/\/[^\s)]+\)/gi, '$1');
+  text = text.replace(/`[^`\n]+`/g, ENTITY);
+  text = text.replace(/https?:\/\/[^\s)]+/gi, (match) => {
+    const trailing = match.match(/[.!?,;:]+$/u)?.[0] ?? '';
+    return `${ENTITY}${trailing}`;
+  });
+
+  const sourceLines = text.split('\n');
+  const lines = [];
+  for (let index = 0; index < sourceLines.length; index += 1) {
+    const rawLine = sourceLines[index];
+    if (/^\s*#{1,6}\s+/.test(rawLine)) {
+      lines.push('');
+      continue;
+    }
+    if (/^\s*\|.*\|\s*$/.test(rawLine)) {
+      lines.push('');
+      continue;
+    }
+
+    let line = rawLine.replace(/^\s*>\s?/, '');
+    const list = line.match(/^\s*(?:[-*+]|\d+\.)\s+(.*)$/);
+    if (list) {
+      let item = list[1].trim();
+      let cursor = index + 1;
+      while (cursor < sourceLines.length && /^\s{2,}\S/.test(sourceLines[cursor])) {
+        item += ` ${sourceLines[cursor].replace(/^\s*>\s?/, '').trim()}`;
+        cursor += 1;
+      }
+      index = cursor - 1;
+      line = isCompleteListItem(item) ? item : '';
+    }
+    lines.push(line.trimEnd());
+  }
+
+  return lines.join('\n').replace(/\n{3,}/g, '\n\n').trim();
+}

package/scripts/utils/tokenizer.mjs

@@ -0,0 +1,82 @@
+const ABBREVIATIONS = {
+  id: ['dll.', 'dsb.', 'dst.', 'dkk.', 'hlm.', 'no.', 'dr.', 'prof.', 'ir.', 'hj.', 'pt.', 'cv.', 'tbk.', 'rp.', 'tgl.', 's.d.', 'a.n.', 'u.p.'],
+  en: ['mr.', 'mrs.', 'ms.', 'dr.', 'prof.', 'sr.', 'jr.', 'vs.', 'etc.', 'e.g.', 'i.e.', 'fig.', 'inc.', 'ltd.', 'co.', 'st.'],
+};
+
+const DOT = '\uE000';
+const DECIMAL = '\uE001';
+const ELLIPSIS = '\uE002';
+
+function escapeRegExp(value) {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+function protectAbbreviations(text, language) {
+  const items = language === 'auto'
+    ? [...new Set([...ABBREVIATIONS.id, ...ABBREVIATIONS.en])]
+    : ABBREVIATIONS[language] ?? ABBREVIATIONS.en;
+  let protectedText = text
+    .replace(/\.{2,}/g, ELLIPSIS)
+    .replace(/(\d)\.(\d)/g, `$1${DECIMAL}$2`);
+  for (const abbreviation of items.sort((a, b) => b.length - a.length)) {
+    protectedText = protectedText.replace(
+      new RegExp(`\\b${escapeRegExp(abbreviation)}`, 'gi'),
+      (match) => match.replaceAll('.', DOT),
+    );
+  }
+  return protectedText;
+}
+
+export function tokenizeWords(text) {
+  const normalized = String(text ?? '').replace(/—|--/g, ' ');
+  return (normalized.toLowerCase().match(/token_entity|[\p{L}\p{N}]+(?:['’][\p{L}\p{N}]+)?(?:-[\p{L}\p{N}]+)*/gu) ?? []);
+}
+
+export function tokenizeText(input, language = 'auto') {
+  const paragraphs = [];
+  const sentences = [];
+  const rawParagraphs = String(input ?? '').replace(/\r\n?/g, '\n').split(/\n\s*\n+/).map((value) => value.trim()).filter(Boolean);
+
+  for (const [paragraphOffset, rawParagraph] of rawParagraphs.entries()) {
+    const protectedText = protectAbbreviations(rawParagraph.replace(/\s+/g, ' '), language);
+    const markedText = protectedText.replace(/([.!?](?:["')\]”’]+)?)\s+/gu, '$1\n');
+    const parts = markedText.split('\n').map((value) => value
+      .replaceAll(DOT, '.')
+      .replaceAll(DECIMAL, '.')
+      .replaceAll(ELLIPSIS, '...')
+      .trim()).filter(Boolean);
+    const paragraphSentences = [];
+
+    for (const text of parts) {
+      const words = tokenizeWords(text);
+      if (!words.length) continue;
+      const sentence = {
+        text,
+        words,
+        wordCount: words.length,
+        paragraphIndex: paragraphOffset + 1,
+        sentenceIndex: sentences.length + 1,
+        sentenceInParagraph: paragraphSentences.length + 1,
+      };
+      paragraphSentences.push(sentence);
+      sentences.push(sentence);
+    }
+
+    if (paragraphSentences.length) {
+      paragraphs.push({
+        text: rawParagraph,
+        paragraphIndex: paragraphOffset + 1,
+        sentences: paragraphSentences,
+        words: paragraphSentences.flatMap((sentence) => sentence.words),
+      });
+    }
+  }
+
+  return {
+    paragraphs,
+    sentences,
+    words: sentences.flatMap((sentence) => sentence.words),
+  };
+}
+
+export { ABBREVIATIONS };