defluffisizer 1.0.0

package/skills/defluffisizer/scripts/defluff_lint.py

@@ -0,0 +1,319 @@
+#!/usr/bin/env python3
+"""
+Lightweight lint and comparison tool for defluffed writing.
+
+Examples:
+    python3 scripts/defluff_lint.py draft.txt
+    python3 scripts/defluff_lint.py --before before.txt --after after.txt
+"""
+
+import argparse
+import json
+import math
+import re
+import statistics
+import sys
+from collections import Counter
+from pathlib import Path
+
+EM_DASH = "\u2014"
+EN_DASH = "\u2013"
+
+FORMULAIC_PHRASES = [
+    "in today's",
+    "it is important to note",
+    "delve into",
+    "tap into",
+    "navigate the",
+    "meaningful outcomes",
+    "robust framework",
+    "seamless solution",
+    "transformative approach",
+    "unlock value",
+    "unlock actionable",
+]
+
+# Defensive contrast: a claim justified by knocking down a criticism nobody
+# raised, instead of just stating the claim. A close cousin of not-X-but-Y,
+# common enough to flag on its own.
+DEFENSIVE_CONTRAST_PHRASES = [
+    "no compromises",
+    "not just a",
+    "not a shortcut",
+    "not a band-aid",
+    "not a band aid",
+]
+
+EMPTY_AMPLIFIERS = [
+    "truly",
+    "genuinely",
+    "really",
+    "very",
+    "simply put",
+]
+
+VAGUE_WORDS = [
+    "landscape",
+    "realm",
+    "journey",
+    "ecosystem",
+    "leverage",
+    "unlock",
+    "synergy",
+    "holistic",
+    "impactful",
+    "transformative",
+    "seamless",
+    "robust",
+]
+
+WORD_RE = re.compile(r"[A-Za-z0-9']+")
+SENTENCE_SPLIT_RE = re.compile(r"(?<=[.!?])\s+")
+HYPHENATED_WORD_RE = re.compile(r"\b[A-Za-z0-9]+-[A-Za-z0-9]+\b")
+NOT_X_BUT_Y_RE = re.compile(r"\bnot\b[^.?!\n]{0,120}\bbut\b", re.IGNORECASE)
+# False contrast split across a sentence boundary, or joined with a dash,
+# instead of "but": "This is not a shortcut. It is a real pipeline." /
+# "It's not a rewrite — it's a rebuild." Same move as NOT_X_BUT_Y_RE, just
+# with the seam hidden behind punctuation other than "but".
+SPLIT_CONTRAST_RE = re.compile(
+    r"\b(?:this|it|that)(?:'s|\s+is)\s+not\b[^.?!\n—]{0,140}(?:[.!?]\s+|\s*—\s*)"
+    r"(?:it|this|that)(?:'s|\s+is)\b",
+    re.IGNORECASE,
+)
+TIGHT_EM_DASH_RE = re.compile(r"\S—\S")
+SPACED_EM_DASH_RE = re.compile(r"\s—\s")
+NUMBER_RE = re.compile(r"\b\d+(?:[.,:/-]\d+)*\b")
+PROPER_TOKEN_RE = re.compile(r"\b[A-Z][a-zA-Z0-9]+\b")
+PROPER_TOKEN_IGNORE = {
+    "A",
+    "An",
+    "And",
+    "As",
+    "At",
+    "But",
+    "By",
+    "For",
+    "From",
+    "If",
+    "In",
+    "Into",
+    "It",
+    "Its",
+    "Of",
+    "On",
+    "Or",
+    "Our",
+    "That",
+    "The",
+    "Their",
+    "There",
+    "These",
+    "This",
+    "Those",
+    "To",
+    "We",
+    "With",
+}
+
+
+def read_text(path_str):
+    return Path(path_str).read_text(encoding="utf-8")
+
+
+def words(text):
+    return WORD_RE.findall(text)
+
+
+def split_sentences(text):
+    parts = [part.strip() for part in SENTENCE_SPLIT_RE.split(text.strip()) if part.strip()]
+    if not parts and text.strip():
+        return [text.strip()]
+    return parts
+
+
+def count_phrase(text, phrase):
+    # Left boundary only: catches inflected forms of the same word
+    # ("seamlessly" for "seamless", "leverages" for "leverage") while still
+    # avoiding a mid-word false hit like "very" inside "delivery", where
+    # there is no boundary before the match at all.
+    pattern = r"\b" + re.escape(phrase)
+    return len(re.findall(pattern, text, re.IGNORECASE))
+
+
+def sentence_lengths(sentences):
+    return [len(words(sentence)) for sentence in sentences if words(sentence)]
+
+
+def repeated_openings(sentences, n=2):
+    openings = []
+    for sentence in sentences:
+        tokens = [token.lower() for token in WORD_RE.findall(sentence)]
+        if len(tokens) >= n:
+            openings.append(" ".join(tokens[:n]))
+    counts = Counter(openings)
+    return {opening: count for opening, count in counts.items() if count > 1}
+
+
+def analyze(text):
+    token_list = words(text)
+    sentences = split_sentences(text)
+    lengths = sentence_lengths(sentences)
+    proper_tokens = sorted(
+        token for token in set(PROPER_TOKEN_RE.findall(text)) if token not in PROPER_TOKEN_IGNORE
+    )
+    numbers = sorted(set(NUMBER_RE.findall(text)))
+    formulaic_counts = {phrase: count_phrase(text, phrase) for phrase in FORMULAIC_PHRASES}
+    vague_counts = {word: count_phrase(text, word) for word in VAGUE_WORDS}
+    defensive_counts = {phrase: count_phrase(text, phrase) for phrase in DEFENSIVE_CONTRAST_PHRASES}
+    amplifier_counts = {word: count_phrase(text, word) for word in EMPTY_AMPLIFIERS}
+    hyphenated_words = HYPHENATED_WORD_RE.findall(text)
+    repeated = repeated_openings(sentences)
+
+    # not_x_but_y and split_contrast both catch the same rhetorical move
+    # (false contrast for emphasis); split_contrast catches it when the
+    # writer hid the seam behind a period instead of joining with "but".
+    not_x_but_y_count = len(NOT_X_BUT_Y_RE.findall(text))
+    split_contrast_count = len(SPLIT_CONTRAST_RE.findall(text))
+
+    result = {
+        "word_count": len(token_list),
+        "sentence_count": len(sentences),
+        "avg_sentence_length": round(statistics.mean(lengths), 2) if lengths else 0,
+        "sentence_length_stddev": round(statistics.pstdev(lengths), 2) if len(lengths) > 1 else 0,
+        "em_dash_count": text.count(EM_DASH),
+        "tight_em_dash_count": len(TIGHT_EM_DASH_RE.findall(text)),
+        "spaced_em_dash_count": len(SPACED_EM_DASH_RE.findall(text)),
+        "en_dash_count": text.count(EN_DASH),
+        "hyphenated_word_count": len(hyphenated_words),
+        "hyphenated_words": hyphenated_words,
+        "not_x_but_y_count": not_x_but_y_count,
+        "split_contrast_count": split_contrast_count,
+        "false_contrast_count": not_x_but_y_count + split_contrast_count,
+        "formulaic_phrases": {key: value for key, value in formulaic_counts.items() if value},
+        "vague_words": {key: value for key, value in vague_counts.items() if value},
+        "defensive_contrast_phrases": {key: value for key, value in defensive_counts.items() if value},
+        "empty_amplifiers": {key: value for key, value in amplifier_counts.items() if value},
+        "repeated_openings": repeated,
+        "numbers": numbers,
+        "proper_tokens": proper_tokens,
+    }
+    return result
+
+
+def pattern_total(result):
+    return (
+        result["em_dash_count"]
+        + result["en_dash_count"]
+        + result["hyphenated_word_count"]
+        + result["false_contrast_count"]
+        + sum(result["formulaic_phrases"].values())
+        + sum(result["vague_words"].values())
+        + sum(result["defensive_contrast_phrases"].values())
+        + sum(result["empty_amplifiers"].values())
+    )
+
+
+def compare(before_text, after_text):
+    before = analyze(before_text)
+    after = analyze(after_text)
+
+    before_words = before["word_count"] or 1
+    compression_ratio = round(after["word_count"] / before_words, 3)
+
+    missing_numbers = sorted(set(before["numbers"]) - set(after["numbers"]))
+    missing_proper_tokens = sorted(set(before["proper_tokens"]) - set(after["proper_tokens"]))
+    repeated_delta = len(after["repeated_openings"]) - len(before["repeated_openings"])
+    banned_before = pattern_total(before)
+    banned_after = pattern_total(after)
+
+    score = 100
+    score -= min(30, banned_after * 5)
+    score -= min(10, sum(after["formulaic_phrases"].values()) * 2)
+    score -= min(10, sum(after["vague_words"].values()))
+    score -= 8 if compression_ratio > 1.05 else 0
+    score -= 6 if compression_ratio < 0.45 else 0
+    score -= min(10, len(missing_numbers) * 2)
+    score -= 4 if repeated_delta > 0 else 0
+    score = max(0, score)
+
+    return {
+        "before": before,
+        "after": after,
+        "compression_ratio": compression_ratio,
+        "banned_pattern_drop": banned_before - banned_after,
+        "missing_numbers": missing_numbers,
+        "missing_proper_tokens": missing_proper_tokens,
+        "score": score,
+    }
+
+
+def render_analysis(result, label):
+    lines = [
+        f"{label}:",
+        f"  words: {result['word_count']}",
+        f"  sentences: {result['sentence_count']}",
+        f"  avg sentence length: {result['avg_sentence_length']}",
+        f"  sentence length stddev: {result['sentence_length_stddev']}",
+        f"  em dashes: {result['em_dash_count']} (tight, no space: {result['tight_em_dash_count']}, spaced: {result['spaced_em_dash_count']})",
+        f"  en dashes: {result['en_dash_count']}",
+        f"  hyphenated words: {result['hyphenated_word_count']}",
+        f"  false contrast patterns: {result['false_contrast_count']} (not X but Y: {result['not_x_but_y_count']}, split across sentences: {result['split_contrast_count']})",
+    ]
+    if result["formulaic_phrases"]:
+        lines.append(f"  formulaic phrases: {json.dumps(result['formulaic_phrases'], ensure_ascii=False)}")
+    if result["defensive_contrast_phrases"]:
+        lines.append(f"  defensive contrast phrases: {json.dumps(result['defensive_contrast_phrases'], ensure_ascii=False)}")
+    if result["empty_amplifiers"]:
+        lines.append(f"  empty amplifiers: {json.dumps(result['empty_amplifiers'], ensure_ascii=False)}")
+    if result["vague_words"]:
+        lines.append(f"  vague words: {json.dumps(result['vague_words'], ensure_ascii=False)}")
+    if result["repeated_openings"]:
+        lines.append(f"  repeated openings: {json.dumps(result['repeated_openings'], ensure_ascii=False)}")
+    return "\n".join(lines)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Lint text for AI fluff patterns.")
+    parser.add_argument("path", nargs="?", help="Single text file to lint")
+    parser.add_argument("--before", help="Original text file")
+    parser.add_argument("--after", help="Rewritten text file")
+    parser.add_argument("--json", action="store_true", help="Print machine readable JSON")
+    args = parser.parse_args()
+
+    if args.before and args.after:
+        result = compare(read_text(args.before), read_text(args.after))
+        if args.json:
+            print(json.dumps(result, indent=2, ensure_ascii=False))
+            return
+        print(render_analysis(result["before"], "Before"))
+        print()
+        print(render_analysis(result["after"], "After"))
+        print()
+        print("Comparison:")
+        print(f"  compression ratio: {result['compression_ratio']}")
+        print(f"  banned pattern drop: {result['banned_pattern_drop']}")
+        print(f"  missing numbers: {result['missing_numbers'] or 'none'}")
+        print(f"  missing proper tokens: {result['missing_proper_tokens'] or 'none'}")
+        print(f"  score: {result['score']}/100")
+        return
+
+    if args.path:
+        result = analyze(read_text(args.path))
+        if args.json:
+            print(json.dumps(result, indent=2, ensure_ascii=False))
+            return
+        print(render_analysis(result, "Text"))
+        return
+
+    text = sys.stdin.read()
+    if not text.strip():
+        parser.error("provide a file path, --before and --after, or stdin")
+    result = analyze(text)
+    if args.json:
+        print(json.dumps(result, indent=2, ensure_ascii=False))
+        return
+    print(render_analysis(result, "Text"))
+
+
+if __name__ == "__main__":
+    main()

package/skills/defluffisizer/scripts/document_text_io.py

@@ -0,0 +1,272 @@
+#!/usr/bin/env python3
+"""
+Extract and apply text for multiple document formats used by defluffisizer.
+
+Examples:
+    python3 scripts/document_text_io.py probe report.docx
+    python3 scripts/document_text_io.py extract pitch.pptx --out editable.json
+    python3 scripts/document_text_io.py extract report.pdf --out report.txt
+
+    # apply without --output writes {stem}_defluffed{ext} next to the source:
+    python3 scripts/document_text_io.py apply pitch.pptx editable.json
+    # -> pitch_defluffed.pptx, beside pitch.pptx
+
+    python3 scripts/document_text_io.py apply report.pdf after.txt
+    # -> report_defluffed.docx, beside report.pdf (PDF has no native writer)
+
+    # --output-dir keeps the default name but changes the directory:
+    python3 scripts/document_text_io.py apply report.pdf after.txt --output-dir ~/Desktop/out
+
+    # --output overrides both name and directory:
+    python3 scripts/document_text_io.py apply report.docx after.txt --output final/report.docx
+"""
+
+import argparse
+import json
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+import pdf_write
+import pptx_rewrite
+
+DIRECT_TEXT_EXTENSIONS = {".txt", ".md", ".csv", ".tsv"}
+TEXTUTIL_EXTENSIONS = {".doc", ".docx", ".odt", ".rtf", ".html", ".htm", ".wordml"}
+PPTX_EXTENSIONS = {".pptx"}
+PDF_EXTENSIONS = {".pdf"}
+HTML_EXTENSIONS = {".html", ".htm"}
+
+TOOL_INSTALL_HINTS = {
+    "textutil": "textutil ships with macOS only. .doc/.docx/.odt/.rtf/.html support needs macOS.",
+    "pdftotext": "Install poppler: `brew install poppler` (macOS) or `apt install poppler-utils` (Debian/Ubuntu).",
+}
+
+
+def require_tool(tool_name):
+    if shutil.which(tool_name):
+        return
+    hint = TOOL_INSTALL_HINTS.get(tool_name, "")
+    message = f"Required tool not found on PATH: {tool_name}"
+    if hint:
+        message += f"\n{hint}"
+    raise SystemExit(message)
+
+
+def run_command(args):
+    subprocess.run(args, check=True)
+
+
+def suffix(path_str):
+    return Path(path_str).suffix.lower()
+
+
+def default_output_ext(source_ext):
+    """The extension a defluffed file gets when --output is not given.
+
+    Every format, PDF included, keeps its own extension: same format in,
+    same format out.
+    """
+    return source_ext
+
+
+def default_output_path(source, output_dir=None):
+    """Where a defluffed file lands when the caller does not name one.
+
+    Default is `{stem}_defluffed{ext}` next to the source file, matching
+    what the skill promises: the final output stays where the source was
+    unless the caller asks for a different directory.
+    """
+    source = Path(source)
+    directory = Path(output_dir) if output_dir else source.parent
+    ext = default_output_ext(source.suffix.lower())
+    return directory / f"{source.stem}_defluffed{ext}"
+
+
+def probe_format(path_str):
+    ext = suffix(path_str)
+    if ext in PPTX_EXTENSIONS:
+        return {
+            "format": ext,
+            "extract": "json",
+            "apply_back": True,
+            "layout_preservation": "yes",
+            "tool": "pptx_rewrite.py",
+        }
+    if ext in DIRECT_TEXT_EXTENSIONS:
+        return {
+            "format": ext,
+            "extract": "text",
+            "apply_back": True,
+            "layout_preservation": "n/a",
+            "tool": "direct",
+        }
+    if ext in TEXTUTIL_EXTENSIONS:
+        return {
+            "format": ext,
+            "extract": "text",
+            "apply_back": True,
+            "layout_preservation": "best_effort",
+            "tool": "textutil",
+        }
+    if ext in PDF_EXTENSIONS:
+        return {
+            "format": ext,
+            "extract": "text",
+            "apply_back": False,
+            "layout_preservation": "no",
+            "tool": "pdftotext",
+        }
+    return {
+        "format": ext or "unknown",
+        "extract": "unsupported",
+        "apply_back": False,
+        "layout_preservation": "unknown",
+        "tool": None,
+    }
+
+
+def extract_text(source, out_path):
+    source = Path(source)
+    out_path = Path(out_path)
+    ext = source.suffix.lower()
+
+    if ext in PPTX_EXTENSIONS:
+        payload = pptx_rewrite.extract_pptx(source)
+        out_path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+        return
+
+    if ext in DIRECT_TEXT_EXTENSIONS:
+        out_path.write_text(source.read_text(encoding="utf-8"), encoding="utf-8")
+        return
+
+    if ext in TEXTUTIL_EXTENSIONS:
+        require_tool("textutil")
+        run_command(["textutil", "-convert", "txt", str(source), "-output", str(out_path)])
+        return
+
+    if ext in PDF_EXTENSIONS:
+        require_tool("pdftotext")
+        run_command(["pdftotext", str(source), str(out_path)])
+        return
+
+    raise SystemExit(f"Unsupported source format for extract: {ext}")
+
+
+def write_pdf_output(rewritten, output):
+    """Render REWRITTEN to a real PDF at OUTPUT via headless Chrome.
+
+    If REWRITTEN is already .html, it is rendered as-is: this is how a
+    caller hands over a fully styled recreation (colors, tables, layout)
+    of the source instead of the generic plain-text fallback below. Any
+    other REWRITTEN is treated as plain text and wrapped in a minimal,
+    clean, single-column page, no attempt to clone the source's design.
+    """
+    rewritten = Path(rewritten)
+    if rewritten.suffix.lower() in HTML_EXTENSIONS:
+        pdf_write.html_to_pdf(rewritten, output)
+        return
+
+    html = pdf_write.text_to_html(rewritten.read_text(encoding="utf-8"))
+    tmp_html = output.with_suffix(".defluffisizer_tmp.html")
+    tmp_html.write_text(html, encoding="utf-8")
+    try:
+        pdf_write.html_to_pdf(tmp_html, output)
+    finally:
+        tmp_html.unlink(missing_ok=True)
+
+
+def apply_text(source, rewritten, output):
+    source = Path(source)
+    rewritten = Path(rewritten)
+    output = Path(output)
+    ext = source.suffix.lower()
+    output_ext = output.suffix.lower()
+
+    if ext in PPTX_EXTENSIONS:
+        if rewritten.suffix.lower() != ".json":
+            raise SystemExit("PPTX apply expects a replacement JSON file")
+        pptx_rewrite.apply_replacements(source, rewritten, output)
+        return
+
+    if output_ext == ".pdf":
+        write_pdf_output(rewritten, output)
+        return
+
+    if ext in DIRECT_TEXT_EXTENSIONS:
+        output.write_text(rewritten.read_text(encoding="utf-8"), encoding="utf-8")
+        return
+
+    if ext in TEXTUTIL_EXTENSIONS:
+        if output_ext in DIRECT_TEXT_EXTENSIONS:
+            output.write_text(rewritten.read_text(encoding="utf-8"), encoding="utf-8")
+            return
+        require_tool("textutil")
+        run_command(["textutil", "-convert", output_ext.lstrip("."), str(rewritten), "-output", str(output)])
+        return
+
+    if ext in PDF_EXTENSIONS:
+        if output_ext in DIRECT_TEXT_EXTENSIONS:
+            output.write_text(rewritten.read_text(encoding="utf-8"), encoding="utf-8")
+            return
+        if output_ext in TEXTUTIL_EXTENSIONS:
+            require_tool("textutil")
+            run_command(["textutil", "-convert", output_ext.lstrip("."), str(rewritten), "-output", str(output)])
+            return
+        raise SystemExit(f"Unsupported output format for PDF apply: {output_ext}")
+
+    raise SystemExit(f"Unsupported source format for apply: {ext}")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Document text IO for defluffisizer.")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    probe_parser = subparsers.add_parser("probe", help="Show format capabilities for a source file")
+    probe_parser.add_argument("source", help="Source file path")
+    probe_parser.add_argument("--json", action="store_true", help="Print JSON output")
+
+    extract_parser = subparsers.add_parser("extract", help="Extract editable text from a source file")
+    extract_parser.add_argument("source", help="Source file path")
+    extract_parser.add_argument("--out", required=True, help="Output text or JSON path")
+
+    apply_parser = subparsers.add_parser("apply", help="Apply rewritten content back into an artifact")
+    apply_parser.add_argument("source", help="Original source path")
+    apply_parser.add_argument("rewritten", help="Rewritten text file or PPTX replacement JSON")
+    apply_parser.add_argument(
+        "--output",
+        help="Output artifact path. Defaults to {stem}_defluffed{ext} next to the source file.",
+    )
+    apply_parser.add_argument(
+        "--output-dir",
+        help="Directory for the default output name, when --output is not given. Defaults to the source file's own directory.",
+    )
+
+    args = parser.parse_args()
+
+    if args.command == "probe":
+        info = probe_format(args.source)
+        if args.json:
+            print(json.dumps(info, indent=2, ensure_ascii=False))
+            return
+        print(f"format: {info['format']}")
+        print(f"extract: {info['extract']}")
+        print(f"apply back: {info['apply_back']}")
+        print(f"layout preservation: {info['layout_preservation']}")
+        print(f"tool: {info['tool']}")
+        return
+
+    if args.command == "extract":
+        extract_text(args.source, args.out)
+        return
+
+    if args.command == "apply":
+        output = Path(args.output) if args.output else default_output_path(args.source, args.output_dir)
+        output.parent.mkdir(parents=True, exist_ok=True)
+        apply_text(args.source, args.rewritten, output)
+        print(f"Wrote: {output}")
+        return
+
+
+if __name__ == "__main__":
+    main()