aslopcleaner 1.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 - 2026 Aron Homberg
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,154 @@
+# aslopcleaner
+
+High-performance CLI to normalize common LLM/AI Unicode punctuation and symbols into plain ASCII.
+
+## What it does
+
+- Recursively scans the current directory.
+- Uses `fast-glob` to skip expensive third-party/build/cache directories early.
+- Never opens `.env*`, SSH keys, certificate/key material, or password database files.
+- Skips files larger than **125 KiB**.
+- Uses a jump-sampled binary heuristic before reading as UTF-8.
+- Prompts once per file in interactive mode.
+- Replaces everything automatically in non-interactive mode with `-y`.
+
+## Default replacements
+
+### Dashes / bullets / arrows
+
+- `—` => `-`
+- `–` => `-`
+- `‒` => `-`
+- `―` => `--`
+- `‐` => `-`
+- `‑` => `-`
+- `−` => `-`
+- `→` => `=>`
+- `⇒` => `=>`
+- `⟶` => `=>`
+- `➜` => `=>`
+- `➔` => `=>`
+- `➝` => `=>`
+- `✔` => `-`
+- `✅` => `-`
+- `☑` => `-`
+- `✓` => `-`
+- `•` => `-`
+- `‣` => `-`
+- `◦` => `-`
+- `▪` => `-`
+- `·` => `-`
+- `●` => `-`
+- `○` => `-`
+
+### Quotes / punctuation / spacing
+
+- `“` => `"`
+- `”` => `"`
+- `„` => `"`
+- `‟` => `"`
+- `«` => `"`
+- `»` => `"`
+- `‹` => `'`
+- `›` => `'`
+- `‘` => `'`
+- `’` => `'`
+- `‚` => `'`
+- `‛` => `'`
+- `…` => `...`
+- `≤` => `<=`
+- `≥` => `>=`
+- `≠` => `!=`
+- NBSP / narrow NBSP / figure space => regular space
+- zero-width space / joiner / BOM => removed
+
+## Run it
+
+### NPM
+
+```bash
+npx aslopcleaner
+```
+
+### Bun
+
+```bash
+bunx aslopcleaner
+```
+
+### PNPM
+
+```bash
+pnpx aslopcleaner
+```
+
+### Yarn
+
+```bash
+yarn dlx aslopcleaner
+```
+
+### Local testing
+
+#### Node
+
+```bash
+node dist/cli.mjs
+```
+
+### Bun
+
+```bash
+bun run dist/cli.mjs
+```
+
+## Library usage
+
+You can also import `aslopcleaner` as a library to integrate Unicode normalization into your own tools:
+
+```bash
+bun add aslopcleaner
+pnpm install aslopcleaner
+yarn add aslopcleaner
+npm install aslopcleaner
+```
+
+```ts
+import {
+  findOccurrences,
+  applyOccurrences,
+  countByMatch,
+  scanDirectory,
+  isProbablyBinary,
+  shouldSkipSensitivePath,
+  REPLACEMENT_RULES,
+  REPLACEMENT_RULE_MAP,
+} from "aslopcleaner";
+
+// Scan a directory for files containing Unicode slop
+const { files, totalOccurrences } = await scanDirectory(process.cwd());
+
+// Find occurrences in a string
+const content = '"Hello" → world…';
+const matches = findOccurrences(content, REPLACEMENT_RULES);
+
+// Apply replacements
+const cleaned = applyOccurrences(content, matches, REPLACEMENT_RULE_MAP);
+// => '"Hello" => world...'
+
+// Count occurrences per symbol
+const counts = countByMatch(matches);
+// => Map { '"' => 1, '"' => 1, '→' => 1, '…' => 1 }
+
+// Check if a file is binary (skip before processing)
+const binary = await isProbablyBinary("image.png"); // true
+
+// Check if a path is sensitive (e.g. .env, SSH keys)
+shouldSkipSensitivePath(".env.production"); // true
+```
+
+### Exported types
+
+```ts
+import type { ReplacementRule, MatchOccurrence, ScanResult } from "aslopcleaner";
+```

package/dist/cli.mjs

@@ -0,0 +1,118 @@
+#!/usr/bin/env node
+import path from 'node:path';
+import process from 'node:process';
+import { readFile, writeFile } from 'node:fs/promises';
+import * as readline from 'node:readline/promises';
+import { R as REPLACEMENT_RULES, s as scanDirectory, d as shouldSkipSensitivePath, b as applyOccurrences, c as countByMatch, a as REPLACEMENT_RULE_MAP } from './scanner-ChvwTQMG.mjs';
+import 'fast-glob';
+
+function parseArgs(argv) {
+  const flags = new Set(argv);
+  return {
+    yes: flags.has("-y") || flags.has("--yes"),
+    help: flags.has("-h") || flags.has("--help")
+  };
+}
+function printHelp() {
+  console.log(`aslopcleaner
+
+Usage:
+  aslopcleaner [-y]
+
+Options:
+  -y, --yes   Replace every detected occurrence without prompting
+  -h, --help  Show help
+`);
+}
+function formatCounts(matches) {
+  const counts = countByMatch(matches);
+  const parts = [...counts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).map(([match, count]) => {
+    const replacement = REPLACEMENT_RULE_MAP.get(match) ?? "?";
+    return `${JSON.stringify(match)}=>${JSON.stringify(replacement)} x${count}`;
+  });
+  return parts.join(", ");
+}
+async function promptReplace(rl, filePath, matches) {
+  console.log(`
+${filePath}`);
+  console.log(`  ${matches.length} occurrence(s): ${formatCounts(matches)}`);
+  while (true) {
+    const answer = (await rl.question("  Replace and overwrite? [y/n] ")).trim().toLowerCase();
+    if (answer === "y") {
+      return true;
+    }
+    if (answer === "n") {
+      return false;
+    }
+  }
+}
+async function replaceFile(root, relativePath, matches) {
+  if (shouldSkipSensitivePath(relativePath)) {
+    return false;
+  }
+  const absolutePath = path.join(root, relativePath);
+  const original = await readFile(absolutePath, "utf8");
+  const updated = applyOccurrences(original, matches, REPLACEMENT_RULE_MAP);
+  if (updated === original) {
+    return false;
+  }
+  await writeFile(absolutePath, updated, "utf8");
+  return true;
+}
+async function main() {
+  const args = parseArgs(process.argv.slice(2));
+  if (args.help) {
+    printHelp();
+    return;
+  }
+  if (!args.yes && (!process.stdin.isTTY || !process.stdout.isTTY)) {
+    console.error("Interactive mode requires a TTY. Use -y for non-interactive replacement.");
+    process.exitCode = 1;
+    return;
+  }
+  const cwd = process.cwd();
+  console.log(`Scanning ${cwd}`);
+  console.log(`Loaded ${REPLACEMENT_RULES.length} ASCII normalization rule(s).`);
+  const scan = await scanDirectory(cwd);
+  const entries = [...scan.matchesByFile.entries()].sort((a, b) => a[0].localeCompare(b[0]));
+  if (entries.length === 0) {
+    console.log("No matching Unicode characters found.");
+    return;
+  }
+  const rl = args.yes ? null : readline.createInterface({
+    input: process.stdin,
+    output: process.stdout
+  });
+  let updatedFiles = 0;
+  let replacedOccurrences = 0;
+  try {
+    for (const [filePath, matches] of entries) {
+      const shouldReplace = args.yes || rl !== null && await promptReplace(rl, filePath, matches);
+      if (!shouldReplace) {
+        continue;
+      }
+      const changed = await replaceFile(cwd, filePath, matches);
+      if (!changed) {
+        continue;
+      }
+      updatedFiles += 1;
+      replacedOccurrences += matches.length;
+      console.log(`  updated ${filePath}`);
+    }
+  } finally {
+    await rl?.close();
+  }
+  console.log("\nDone.");
+  console.log(`  files with matches: ${entries.length}`);
+  console.log(`  files updated: ${updatedFiles}`);
+  console.log(`  occurrences replaced: ${replacedOccurrences}`);
+  console.log(`  files scanned after glob filtering: ${scan.scannedFiles}`);
+  console.log(`  skipped by sensitive path rules: ${scan.skippedBySensitivePattern}`);
+  console.log(`  skipped by size (>125 KiB): ${scan.skippedBySize}`);
+  console.log(`  skipped as binary: ${scan.skippedByBinary}`);
+}
+main().catch((error) => {
+  const message = error instanceof Error ? error.stack ?? error.message : String(error);
+  console.error(message);
+  process.exitCode = 1;
+});

package/dist/index.mjs

@@ -0,0 +1,5 @@
+#!/usr/bin/env node
+export { F as FAST_GLOB_IGNORE_PATTERNS, M as MAX_FILE_SIZE_BYTES, R as REPLACEMENT_RULES, a as REPLACEMENT_RULE_MAP, b as applyOccurrences, c as countByMatch, f as findOccurrences, i as isProbablyBinary, n as normalizeGlobPath, s as scanDirectory, d as shouldSkipSensitivePath } from './scanner-ChvwTQMG.mjs';
+import 'fast-glob';
+import 'node:path';
+import 'node:fs/promises';

package/dist/scanner-ChvwTQMG.mjs

@@ -0,0 +1,335 @@
+import fg from 'fast-glob';
+import path from 'node:path';
+import { open, stat, readFile } from 'node:fs/promises';
+
+const SKIPPED_DIRECTORIES = [
+  ".git",
+  ".hg",
+  ".svn",
+  ".idea",
+  ".vscode",
+  "node_modules",
+  "bower_components",
+  "vendor",
+  ".pnpm-store",
+  ".yarn",
+  ".turbo",
+  ".cache",
+  ".parcel-cache",
+  ".vite",
+  ".rollup.cache",
+  ".rspack-cache",
+  ".eslintcache",
+  ".stylelintcache",
+  ".next",
+  ".nuxt",
+  ".svelte-kit",
+  ".angular",
+  ".output",
+  "dist",
+  "build",
+  "out",
+  "target",
+  "bin",
+  "obj",
+  "coverage",
+  ".nyc_output",
+  "playwright-report",
+  "test-results",
+  "__pycache__",
+  ".venv",
+  "venv",
+  "env",
+  ".conda",
+  ".mypy_cache",
+  ".pytest_cache",
+  ".ruff_cache",
+  ".tox",
+  ".eggs",
+  ".gradle",
+  ".mvn",
+  ".settings",
+  ".bundle",
+  "_build",
+  "deps",
+  "dist-newstyle",
+  ".stack-work",
+  "storybook-static",
+  "tmp",
+  "temp",
+  "logs",
+  "log"
+];
+const SENSITIVE_EXACT_BASENAMES = /* @__PURE__ */ new Set([
+  "id_rsa",
+  "id_ed25519",
+  "known_hosts",
+  "authorized_keys",
+  ".npmrc",
+  ".pypirc",
+  ".netrc"
+]);
+const SENSITIVE_SUFFIXES = [
+  ".pem",
+  ".key",
+  ".p12",
+  ".pfx",
+  ".crt",
+  ".cer",
+  ".der",
+  ".csr",
+  ".p7b",
+  ".p7c",
+  ".jks",
+  ".keystore",
+  ".asc",
+  ".gpg",
+  ".kdbx"
+];
+const MAX_FILE_SIZE_BYTES = 125 * 1024;
+const FAST_GLOB_IGNORE_PATTERNS = SKIPPED_DIRECTORIES.flatMap((directory) => [
+  `${directory}/**`,
+  `**/${directory}/**`
+]);
+function shouldSkipSensitivePath(filePath) {
+  const baseName = path.basename(filePath).toLowerCase();
+  if (baseName === ".env" || baseName.startsWith(".env.")) {
+    return true;
+  }
+  if (SENSITIVE_EXACT_BASENAMES.has(baseName)) {
+    return true;
+  }
+  return SENSITIVE_SUFFIXES.some((suffix) => baseName.endsWith(suffix));
+}
+function normalizeGlobPath(filePath) {
+  return filePath.split(path.sep).join("/");
+}
+
+function findOccurrences(content, rules) {
+  const matches = [];
+  for (const rule of rules) {
+    let index = content.indexOf(rule.match);
+    if (index === -1) {
+      continue;
+    }
+    while (index !== -1) {
+      matches.push({ index, match: rule.match });
+      index = content.indexOf(rule.match, index + rule.match.length);
+    }
+  }
+  matches.sort((a, b) => a.index - b.index || a.match.length - b.match.length);
+  return matches;
+}
+function countByMatch(matches) {
+  const counts = /* @__PURE__ */ new Map();
+  for (const match of matches) {
+    counts.set(match.match, (counts.get(match.match) ?? 0) + 1);
+  }
+  return counts;
+}
+function applyOccurrences(content, matches, replacements) {
+  if (matches.length === 0) {
+    return content;
+  }
+  let cursor = 0;
+  let output = "";
+  for (const occurrence of matches) {
+    if (occurrence.index < cursor) {
+      continue;
+    }
+    const replacement = replacements.get(occurrence.match);
+    if (replacement === void 0) {
+      continue;
+    }
+    output += content.slice(cursor, occurrence.index);
+    output += replacement;
+    cursor = occurrence.index + occurrence.match.length;
+  }
+  output += content.slice(cursor);
+  return output;
+}
+
+const REPLACEMENT_RULES = [
+  { match: "\u2014", replacement: "-", description: "em dash" },
+  { match: "\u2013", replacement: "-", description: "en dash" },
+  { match: "\u2012", replacement: "-", description: "figure dash" },
+  { match: "\u2015", replacement: "--", description: "horizontal bar" },
+  { match: "\u2010", replacement: "-", description: "hyphen" },
+  { match: "\u2011", replacement: "-", description: "non-breaking hyphen" },
+  { match: "\u2212", replacement: "-", description: "minus sign" },
+  { match: "\u2192", replacement: "=>", description: "right arrow" },
+  { match: "\u21D2", replacement: "=>", description: "double right arrow" },
+  { match: "\u27F6", replacement: "=>", description: "long right arrow" },
+  { match: "\u279C", replacement: "=>", description: "heavy right arrow" },
+  { match: "\u2794", replacement: "=>", description: "black right arrow" },
+  { match: "\u279D", replacement: "=>", description: "drafting right arrow" },
+  { match: "\u2714", replacement: "-", description: "heavy check mark" },
+  { match: "\u2705", replacement: "-", description: "check mark button" },
+  { match: "\u2611", replacement: "-", description: "ballot box with check" },
+  { match: "\u2713", replacement: "-", description: "check mark" },
+  { match: "\u2022", replacement: "-", description: "bullet" },
+  { match: "\u2023", replacement: "-", description: "triangular bullet" },
+  { match: "\u25E6", replacement: "-", description: "white bullet" },
+  { match: "\u25AA", replacement: "-", description: "small square bullet" },
+  { match: "\xB7", replacement: "-", description: "middle dot bullet" },
+  { match: "\u25CF", replacement: "-", description: "black circle bullet" },
+  { match: "\u25CB", replacement: "-", description: "white circle bullet" },
+  { match: "\u201C", replacement: '"', description: "left double quote" },
+  { match: "\u201D", replacement: '"', description: "right double quote" },
+  { match: "\u201E", replacement: '"', description: "low double quote" },
+  { match: "\u201F", replacement: '"', description: "double high-reversed-9 quote" },
+  { match: "\xAB", replacement: '"', description: "left guillemet" },
+  { match: "\xBB", replacement: '"', description: "right guillemet" },
+  { match: "\u2039", replacement: "'", description: "left single guillemet" },
+  { match: "\u203A", replacement: "'", description: "right single guillemet" },
+  { match: "\u2018", replacement: "'", description: "left single quote" },
+  { match: "\u2019", replacement: "'", description: "right single quote / apostrophe" },
+  { match: "\u201A", replacement: "'", description: "low single quote" },
+  { match: "\u201B", replacement: "'", description: "single high-reversed-9 quote" },
+  { match: "\u2026", replacement: "...", description: "ellipsis" },
+  { match: "\u2264", replacement: "<=", description: "less-than-or-equal" },
+  { match: "\u2265", replacement: ">=", description: "greater-than-or-equal" },
+  { match: "\u2260", replacement: "!=", description: "not-equal" },
+  { match: "\xA0", replacement: " ", description: "no-break space" },
+  { match: "\u202F", replacement: " ", description: "narrow no-break space" },
+  { match: "\u2007", replacement: " ", description: "figure space" },
+  { match: "\u200B", replacement: "", description: "zero-width space" },
+  { match: "\u200C", replacement: "", description: "zero-width non-joiner" },
+  { match: "\u200D", replacement: "", description: "zero-width joiner" },
+  { match: "\u2060", replacement: "", description: "word joiner" },
+  { match: "\uFEFF", replacement: "", description: "byte-order mark" }
+];
+const REPLACEMENT_RULE_MAP = new Map(
+  REPLACEMENT_RULES.map((rule) => [rule.match, rule.replacement])
+);
+
+const SAMPLE_SIZE = 1024;
+const MAX_SAMPLES = 5;
+const SUSPICIOUS_RATIO = 0.15;
+function getSampleOffsets(size) {
+  if (size <= SAMPLE_SIZE) {
+    return [0];
+  }
+  const maxOffset = Math.max(0, size - SAMPLE_SIZE);
+  const offsets = /* @__PURE__ */ new Set([0, maxOffset]);
+  for (let i = 1; i < MAX_SAMPLES - 1; i += 1) {
+    offsets.add(Math.floor(maxOffset * i / (MAX_SAMPLES - 1)));
+  }
+  return [...offsets].sort((a, b) => a - b);
+}
+function scoreSample(buffer) {
+  let suspicious = 0;
+  let total = 0;
+  for (const byte of buffer) {
+    if (byte === 0) {
+      return 1;
+    }
+    total += 1;
+    const isAllowedControl = byte === 9 || byte === 10 || byte === 13 || byte === 12;
+    if (byte < 32 && !isAllowedControl || byte === 127) {
+      suspicious += 1;
+    }
+  }
+  return total === 0 ? 0 : suspicious / total;
+}
+async function isProbablyBinary(filePath) {
+  const handle = await open(filePath, "r");
+  try {
+    const stats = await handle.stat();
+    const offsets = getSampleOffsets(stats.size);
+    const sampleBuffer = Buffer.allocUnsafe(SAMPLE_SIZE);
+    for (const offset of offsets) {
+      const { bytesRead } = await handle.read(sampleBuffer, 0, SAMPLE_SIZE, offset);
+      if (bytesRead === 0) {
+        continue;
+      }
+      const score = scoreSample(sampleBuffer.subarray(0, bytesRead));
+      if (score >= SUSPICIOUS_RATIO) {
+        return true;
+      }
+    }
+    return false;
+  } finally {
+    await handle.close();
+  }
+}
+
+const DEFAULT_CONCURRENCY = 64;
+async function processFile(cwd, relativePath) {
+  const normalizedPath = normalizeGlobPath(relativePath);
+  if (shouldSkipSensitivePath(normalizedPath)) {
+    return { kind: "sensitive" };
+  }
+  const absolutePath = path.join(cwd, relativePath);
+  const fileStat = await stat(absolutePath);
+  if (fileStat.size > MAX_FILE_SIZE_BYTES) {
+    return { kind: "size" };
+  }
+  if (await isProbablyBinary(absolutePath)) {
+    return { kind: "binary" };
+  }
+  const content = await readFile(absolutePath, "utf8");
+  const matches = findOccurrences(content, REPLACEMENT_RULES);
+  if (matches.length === 0) {
+    return { kind: "none" };
+  }
+  return {
+    path: normalizedPath,
+    matches
+  };
+}
+async function scanDirectory(cwd) {
+  const matchesByFile = /* @__PURE__ */ new Map();
+  let scannedFiles = 0;
+  let skippedBySensitivePattern = 0;
+  let skippedBySize = 0;
+  let skippedByBinary = 0;
+  const stream = fg.stream("**/*", {
+    cwd,
+    onlyFiles: true,
+    dot: true,
+    followSymbolicLinks: false,
+    unique: true,
+    ignore: [...FAST_GLOB_IGNORE_PATTERNS]
+  });
+  const inFlight = /* @__PURE__ */ new Set();
+  const schedule = async (relativePath) => {
+    scannedFiles += 1;
+    try {
+      const result = await processFile(cwd, relativePath);
+      if ("kind" in result) {
+        if (result.kind === "sensitive") {
+          skippedBySensitivePattern += 1;
+        } else if (result.kind === "size") {
+          skippedBySize += 1;
+        } else if (result.kind === "binary") {
+          skippedByBinary += 1;
+        }
+        return;
+      }
+      matchesByFile.set(result.path, result.matches);
+    } catch {
+    }
+  };
+  for await (const entry of stream) {
+    const relativePath = String(entry);
+    const task = schedule(relativePath).finally(() => {
+      inFlight.delete(task);
+    });
+    inFlight.add(task);
+    if (inFlight.size >= DEFAULT_CONCURRENCY) {
+      await Promise.race(inFlight);
+    }
+  }
+  await Promise.all(inFlight);
+  return {
+    matchesByFile,
+    scannedFiles,
+    skippedByGlob: 0,
+    skippedBySensitivePattern,
+    skippedBySize,
+    skippedByBinary
+  };
+}
+
+export { FAST_GLOB_IGNORE_PATTERNS as F, MAX_FILE_SIZE_BYTES as M, REPLACEMENT_RULES as R, REPLACEMENT_RULE_MAP as a, applyOccurrences as b, countByMatch as c, shouldSkipSensitivePath as d, findOccurrences as f, isProbablyBinary as i, normalizeGlobPath as n, scanDirectory as s };

package/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "aslopcleaner",
+  "version": "1.0.0",
+  "description": "High-performance CLI to replace common LLM/AI Unicode punctuation and symbols with ASCII equivalents.",
+  "type": "module",
+  "bin": {
+    "aslopcleaner": "./dist/cli.mjs"
+  },
+  "exports": {
+    ".": {
+      "import": "./dist/index.mjs"
+    }
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/kyr0/defuss.git"
+  },
+  "author": "Aron Homberg <info@aron-homberg.de>",
+  "homepage": "https://github.com/kyr0/defuss",
+  "publishConfig": {
+    "access": "public"
+  },
+  "packageManager": "bun@1.3.9",
+  "sideEffects": false,
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "build": "rm -rf dist && pkgroll",
+    "start": "node dist/cli.mjs",
+    "dev": "node --enable-source-maps src/cli.ts",
+    "test": "vitest run",
+    "test:coverage": "vitest run --coverage",
+    "run:example": "tsx example/example.ts"
+  },
+  "keywords": [
+    "ascii",
+    "bun",
+    "node",
+    "cli",
+    "unicode",
+    "sanitizer",
+    "llm"
+  ],
+  "license": "MIT",
+  "dependencies": {
+    "fast-glob": "^3.3.3"
+  },
+  "devDependencies": {
+    "@types/node": "^25.5.0",
+    "@vitest/coverage-v8": "^3.1.1",
+    "pkgroll": "^2.11.3",
+    "tsx": "^4.19.4",
+    "vitest": "^3.1.1"
+  }
+}