formatferry 0.1.1 → 0.1.2

package/dist/index.js

@@ -22108,6 +22108,398 @@ function extractArticle(html, domProvider) {
   };
 }
 
+// ../../src/shared/token-estimator.ts
+var CJK_REGEX = /[\u4e00-\u9fff\u3400-\u4dbf\u3000-\u303f\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]/g;
+var MAX_INPUT_LENGTH = 1e6;
+function estimateTokens(text) {
+  if (!text || text.length === 0) {
+    return 0;
+  }
+  if (text.length > MAX_INPUT_LENGTH) {
+    return Math.round(text.length / 4);
+  }
+  const cjkMatches = text.match(CJK_REGEX);
+  const cjkCount = cjkMatches ? cjkMatches.length : 0;
+  const otherCount = text.length - cjkCount;
+  const cjkTokens = cjkCount / 1.5;
+  const otherTokens = otherCount / 4;
+  return Math.round(cjkTokens + otherTokens);
+}
+
+// ../../src/shared/toon-encoder.ts
+var DEFAULT_OPTIONS = {
+  indent: 0,
+  keyFolding: true,
+  tableFormat: "compact"
+};
+var ToonEncoder = class _ToonEncoder {
+  static encode(value, opts) {
+    const options = { ...DEFAULT_OPTIONS, ...opts };
+    return _ToonEncoder.encodeValue(value, options.indent, options);
+  }
+  static encodeValue(value, indent, opts) {
+    if (value === null) {
+      return "null";
+    }
+    if (typeof value === "boolean") {
+      return value ? "true" : "false";
+    }
+    if (typeof value === "number") {
+      return String(value);
+    }
+    if (typeof value === "string") {
+      return _ToonEncoder.encodeString(value);
+    }
+    if (Array.isArray(value)) {
+      return _ToonEncoder.encodeArray(value, indent, opts);
+    }
+    if (typeof value === "object") {
+      return _ToonEncoder.encodeObject(value, indent, opts);
+    }
+    return String(value);
+  }
+  static encodeString(str) {
+    const needsQuotes = str.includes(":") || str.includes("\n") || str.includes("	") || str.includes("|") || str.includes(",") || str.startsWith(" ") || str.endsWith(" ") || str === "";
+    if (needsQuotes) {
+      const escaped = str.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\n/g, "\\n").replace(/\t/g, "\\t");
+      return `"${escaped}"`;
+    }
+    return str;
+  }
+  static encodeArray(arr, indent, opts) {
+    if (arr.length === 0) {
+      return "[]";
+    }
+    if (_ToonEncoder.isTabularArray(arr)) {
+      return _ToonEncoder.encodeTabular(arr);
+    }
+    const prefix = "  ".repeat(indent);
+    const lines = arr.map((item) => {
+      if (typeof item === "object" && item !== null && !Array.isArray(item)) {
+        const obj = item;
+        const keys = Object.keys(obj);
+        if (keys.length > 0) {
+          const objLines = keys.map((key, idx) => {
+            const value = obj[key];
+            const encodedValue = _ToonEncoder.encodeValue(value, indent + 1, opts);
+            if (typeof value === "object" && value !== null && !Array.isArray(value)) {
+              const innerLines = encodedValue.split("\n");
+              if (innerLines.length > 1 && !encodedValue.startsWith("{}")) {
+                if (idx === 0) {
+                  return `${prefix}- ${key}:
+${innerLines.map((l) => `${prefix}    ${l.trimStart()}`).join("\n")}`;
+                }
+                return `${prefix}  ${key}:
+${innerLines.map((l) => `${prefix}    ${l.trimStart()}`).join("\n")}`;
+              }
+            }
+            if (idx === 0) {
+              return `${prefix}- ${key}: ${encodedValue}`;
+            }
+            return `${prefix}  ${key}: ${encodedValue}`;
+          });
+          return objLines.join("\n");
+        }
+      }
+      const encoded = _ToonEncoder.encodeValue(item, indent + 1, opts);
+      return `${prefix}- ${encoded}`;
+    });
+    return lines.join("\n");
+  }
+  static encodeObject(obj, indent, opts) {
+    const keys = Object.keys(obj);
+    if (keys.length === 0) {
+      return "{}";
+    }
+    if (opts.keyFolding && keys.length === 1) {
+      const key = keys[0];
+      const value = obj[key];
+      if (typeof value === "object" && value !== null && !Array.isArray(value) && Object.keys(value).length === 1) {
+        const foldedPath = _ToonEncoder.getFoldedPath(obj);
+        if (foldedPath) {
+          const { path, finalValue } = foldedPath;
+          const encodedValue = _ToonEncoder.encodeValue(finalValue, indent, opts);
+          const prefix2 = "  ".repeat(indent);
+          return `${prefix2}${path}: ${encodedValue}`;
+        }
+      }
+    }
+    const prefix = "  ".repeat(indent);
+    const lines = keys.map((key) => {
+      const value = obj[key];
+      const encodedValue = _ToonEncoder.encodeValue(value, indent + 1, opts);
+      if (typeof value === "object" && value !== null && !Array.isArray(value)) {
+        const innerLines = encodedValue.split("\n");
+        if (innerLines.length > 1 && !encodedValue.startsWith("{}")) {
+          return `${prefix}${key}:
+${innerLines.map((l) => `${prefix}  ${l.trimStart()}`).join("\n")}`;
+        }
+      }
+      if (Array.isArray(value) && value.length > 0 && !_ToonEncoder.isTabularArray(value)) {
+        return `${prefix}${key}:
+${encodedValue}`;
+      }
+      return `${prefix}${key}: ${encodedValue}`;
+    });
+    return lines.join("\n");
+  }
+  static getFoldedPath(obj, pathParts = []) {
+    const keys = Object.keys(obj);
+    if (keys.length !== 1) {
+      return null;
+    }
+    const key = keys[0];
+    const value = obj[key];
+    const newPath = [...pathParts, key];
+    if (typeof value === "object" && value !== null && !Array.isArray(value)) {
+      const innerKeys = Object.keys(value);
+      if (innerKeys.length === 1) {
+        return _ToonEncoder.getFoldedPath(value, newPath);
+      }
+    }
+    return {
+      path: newPath.join("."),
+      finalValue: value
+    };
+  }
+  static encodeTabular(arr) {
+    if (arr.length === 0) {
+      return "[]";
+    }
+    const keys = Object.keys(arr[0]);
+    const header = `|${keys.join("|")}`;
+    const rows = arr.map((row) => {
+      const values = keys.map((key) => {
+        const value = row[key];
+        if (value === null) return "null";
+        if (typeof value === "string") {
+          return value.replace(/\|/g, "\\|");
+        }
+        return String(value);
+      });
+      return `|${values.join("|")}`;
+    });
+    return [header, ...rows].join("\n");
+  }
+  static isTabularArray(arr) {
+    if (arr.length < 2) {
+      return false;
+    }
+    const firstItem = arr[0];
+    if (typeof firstItem !== "object" || firstItem === null || Array.isArray(firstItem)) {
+      return false;
+    }
+    const firstKeys = Object.keys(firstItem).sort().join(",");
+    for (let i = 1; i < arr.length; i++) {
+      const item = arr[i];
+      if (typeof item !== "object" || item === null || Array.isArray(item)) {
+        return false;
+      }
+      const itemKeys = Object.keys(item).sort().join(",");
+      if (itemKeys !== firstKeys) {
+        return false;
+      }
+      for (const key in item) {
+        const value = item[key];
+        if (typeof value === "object" && value !== null) {
+          return false;
+        }
+      }
+    }
+    for (const key in firstItem) {
+      const value = firstItem[key];
+      if (typeof value === "object" && value !== null) {
+        return false;
+      }
+    }
+    return true;
+  }
+};
+
+// ../../src/shared/toonify-service.ts
+var DEFAULT_OPTIONS2 = {
+  preserveCodeBlocks: true,
+  keyFolding: true
+};
+var ToonifyService = class {
+  static compress(markdown, options) {
+    const opts = { ...DEFAULT_OPTIONS2, ...options };
+    const original = markdown;
+    const originalTokenEstimate = estimateTokens(original);
+    if (!markdown || markdown.trim().length === 0) {
+      return {
+        original,
+        compressed: "",
+        originalTokenEstimate,
+        compressedTokenEstimate: 0,
+        reductionPercent: 100
+      };
+    }
+    const blocks = this.parseMarkdown(markdown, opts);
+    const document2 = { t: "md", blocks };
+    const encodeOptions = {
+      keyFolding: opts.keyFolding,
+      tableFormat: opts.tableFormat
+    };
+    const compressed = ToonEncoder.encode(document2, encodeOptions);
+    const compressedTokenEstimate = estimateTokens(compressed);
+    const reductionPercent = originalTokenEstimate > 0 ? Math.max(0, Math.min(100, Math.round(
+      (originalTokenEstimate - compressedTokenEstimate) / originalTokenEstimate * 100
+    ))) : 0;
+    return {
+      original,
+      compressed,
+      originalTokenEstimate,
+      compressedTokenEstimate,
+      reductionPercent
+    };
+  }
+  static estimateTokens(text) {
+    return estimateTokens(text);
+  }
+  static parseMarkdown(markdown, options) {
+    const blocks = [];
+    const lines = markdown.split("\n");
+    let i = 0;
+    while (i < lines.length) {
+      const line = lines[i];
+      if (line.match(/^```/)) {
+        const { block, endIndex } = this.parseCodeBlock(lines, i, options);
+        if (block) blocks.push(block);
+        i = endIndex + 1;
+        continue;
+      }
+      const headingMatch = line.match(/^(#{1,6})\s+(.+)$/);
+      if (headingMatch) {
+        blocks.push({
+          h: headingMatch[1].length,
+          text: headingMatch[2].trim()
+        });
+        i++;
+        continue;
+      }
+      if (line.includes("|") && line.trim().startsWith("|")) {
+        const { block, endIndex } = this.parseTable(lines, i);
+        if (block) blocks.push(block);
+        i = endIndex + 1;
+        continue;
+      }
+      const listMatch = line.match(/^(\s*)[-*]\s+(.+)$|^(\s*)\d+\.\s+(.+)$/);
+      if (listMatch) {
+        const { block, endIndex } = this.parseList(lines, i);
+        if (block) blocks.push(block);
+        i = endIndex + 1;
+        continue;
+      }
+      if (line.trim().length > 0) {
+        const { block, endIndex } = this.parseParagraph(lines, i);
+        if (block) blocks.push(block);
+        i = endIndex + 1;
+        continue;
+      }
+      i++;
+    }
+    return blocks;
+  }
+  static parseCodeBlock(lines, startIndex, options) {
+    const startLine = lines[startIndex];
+    const langMatch = startLine.match(/^```(\w*)$/);
+    const lang = langMatch ? langMatch[1] : "";
+    let endIndex = startIndex + 1;
+    const codeLines = [];
+    while (endIndex < lines.length) {
+      if (lines[endIndex].match(/^```$/)) {
+        break;
+      }
+      codeLines.push(lines[endIndex]);
+      endIndex++;
+    }
+    const codeText = codeLines.join("\n");
+    if (options.preserveCodeBlocks) {
+      return {
+        block: { code: { lang, text: codeText } },
+        endIndex
+      };
+    }
+    return {
+      block: { code: { lang, text: `[code:${codeText.length}chars]` } },
+      endIndex
+    };
+  }
+  static parseTable(lines, startIndex) {
+    const headerLine = lines[startIndex];
+    const headers = this.parseTableRow(headerLine);
+    let endIndex = startIndex + 1;
+    if (endIndex < lines.length && lines[endIndex].match(/^\|[\s:-]+\|[\s:-|]+$/)) {
+      endIndex++;
+    }
+    const rows = [];
+    while (endIndex < lines.length) {
+      const line = lines[endIndex];
+      if (!line.includes("|") || !line.trim().startsWith("|")) {
+        break;
+      }
+      rows.push(this.parseTableRow(line));
+      endIndex++;
+    }
+    return {
+      block: { table: { headers, rows } },
+      endIndex: endIndex - 1
+    };
+  }
+  static parseTableRow(line) {
+    return line.split("|").map((cell) => cell.trim()).filter((cell) => cell.length > 0);
+  }
+  static parseList(lines, startIndex) {
+    const items = [];
+    let endIndex = startIndex;
+    while (endIndex < lines.length) {
+      const line = lines[endIndex];
+      const match = line.match(/^(\s*)[-*]\s+(.+)$|^(\s*)\d+\.\s+(.+)$/);
+      if (!match) break;
+      items.push(match[2] || match[4]);
+      endIndex++;
+    }
+    return {
+      block: { list: items },
+      endIndex: endIndex - 1
+    };
+  }
+  static parseParagraph(lines, startIndex) {
+    const paragraphLines = [];
+    let endIndex = startIndex;
+    while (endIndex < lines.length) {
+      const line = lines[endIndex];
+      if (line.trim().length === 0) break;
+      if (line.match(/^#{1,6}\s+/)) break;
+      if (line.match(/^```/)) break;
+      if (line.match(/^[-*]\s+/) || line.match(/^\d+\.\s+/)) break;
+      if (line.includes("|") && line.trim().startsWith("|")) break;
+      paragraphLines.push(line);
+      endIndex++;
+    }
+    return {
+      block: { p: paragraphLines.join(" ").trim() },
+      endIndex: endIndex - 1
+    };
+  }
+};
+
+// src/toonify-helpers.ts
+function formatSingleStats(result) {
+  return `Toonify: ${result.originalTokenEstimate} \u2192 ${result.compressedTokenEstimate} tokens (${result.reductionPercent}% reduction)
+`;
+}
+function formatFileStats(filename, result) {
+  return `Toonify [${filename}]: ${result.originalTokenEstimate} \u2192 ${result.compressedTokenEstimate} tokens (${result.reductionPercent}% reduction)
+`;
+}
+function formatAggregateStats(totalOriginal, totalCompressed, fileCount) {
+  const aggregateReduction = totalOriginal > 0 ? Math.max(0, Math.min(100, Math.round((totalOriginal - totalCompressed) / totalOriginal * 100))) : 0;
+  return `Toonify total: ${totalOriginal} \u2192 ${totalCompressed} tokens (${aggregateReduction}% reduction, ${fileCount} files)
+`;
+}
+
 // src/commands/convert.ts
 var VALID_FORMATS = [
   "github",
@@ -22258,7 +22650,28 @@ async function urlToMarkdown(url, apiKey, format, proxyBaseUrl = process.env.FOR
   });
   return result.markdown;
 }
+function applyToonify(markdown) {
+  try {
+    const result = ToonifyService.compress(markdown);
+    process.stderr.write(formatSingleStats(result));
+    return result.compressed;
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    process.stderr.write(`Error: Toonify compression failed: ${message}
+`);
+    process.exit(1);
+  }
+}
 async function convertCommand(options) {
+  if (options.toonify) {
+    const provider = await resolveEntitlement();
+    if (!provider.hasFeature("token-optimization")) {
+      process.stderr.write(
+        "Error: Token optimization requires a Premium license.\nRun `formatferry auth --license-key <KEY>` to activate your license.\n"
+      );
+      process.exit(1);
+    }
+  }
   const format = validateFormat(options.format ?? "github");
   if (options.url) {
     if (options.input) {
@@ -22273,7 +22686,14 @@ async function convertCommand(options) {
       );
       process.exit(1);
     }
-    const markdown2 = await urlToMarkdown(options.url, apiKey, format);
+    let markdown2 = await urlToMarkdown(options.url, apiKey, format);
+    if (options.toonify) {
+      if (markdown2.trim()) {
+        markdown2 = applyToonify(markdown2);
+      } else {
+        markdown2 = "";
+      }
+    }
     if (options.output) {
       writeFileSync2(options.output, markdown2, "utf8");
     } else {
@@ -22324,6 +22744,13 @@ async function convertCommand(options) {
     }
     markdown = await htmlToMarkdown(html, format, applyAtlassian);
   }
+  if (options.toonify) {
+    if (markdown.trim()) {
+      markdown = applyToonify(markdown);
+    } else {
+      markdown = "";
+    }
+  }
   if (options.output) {
     writeFileSync2(options.output, markdown, "utf8");
   } else {
@@ -22379,6 +22806,14 @@ async function batchCommand(options) {
     );
     process.exit(1);
   }
+  if (options.toonify) {
+    if (!provider.hasFeature("token-optimization")) {
+      process.stderr.write(
+        "Error: Token optimization requires a Premium license.\nRun `formatferry auth --license-key <KEY>` to activate your license.\n"
+      );
+      process.exit(1);
+    }
+  }
   const glob = options.batch;
   if (!glob) {
     process.stderr.write("Error: --batch <glob> is required\n");
@@ -22393,9 +22828,29 @@ async function batchCommand(options) {
   const format = options.format ?? "github";
   if (options.outputDir) mkdirSync2(options.outputDir, { recursive: true });
   let successful = 0, failed = 0;
+  let totalOriginal = 0, totalCompressed = 0, toonifiedCount = 0;
   for (const filePath of files) {
     try {
-      const markdown = await convertFile(filePath, format);
+      let markdown = await convertFile(filePath, format);
+      if (options.toonify) {
+        if (markdown.trim()) {
+          try {
+            const result = ToonifyService.compress(markdown);
+            process.stderr.write(formatFileStats(basename(filePath), result));
+            markdown = result.compressed;
+            totalOriginal += result.originalTokenEstimate;
+            totalCompressed += result.compressedTokenEstimate;
+            toonifiedCount++;
+          } catch (err) {
+            process.stderr.write(`Error toonifying ${filePath}: ${err instanceof Error ? err.message : String(err)}
+`);
+            failed++;
+            continue;
+          }
+        } else {
+          markdown = "";
+        }
+      }
       writeFileSync3(getOutputPath(filePath, options.outputDir), markdown, "utf8");
       successful++;
     } catch (err) {
@@ -22404,6 +22859,9 @@ async function batchCommand(options) {
       failed++;
     }
   }
+  if (options.toonify && toonifiedCount > 0) {
+    process.stderr.write(formatAggregateStats(totalOriginal, totalCompressed, toonifiedCount));
+  }
   process.stderr.write(`
 Batch complete: ${successful} succeeded, ${failed} failed, ${files.length} total
 `);
@@ -22601,7 +23059,7 @@ function handleLogout() {
 
 // src/index.ts
 var VERSION = "0.1.0";
-program.name("formatferry").version(VERSION).description("Convert HTML, DOCX, PDF, XLSX, CSV to Markdown").option("-i, --input <path>", "Input file path").option("-o, --output <path>", "Output file path").option("-f, --format <format>", "Markdown format (github, commonmark, slack, discord, reddit, confluence, custom, rmarkdown)", "github").option("--url <url>", "Fetch article from URL").option("--batch <glob>", "Batch convert files (Premium)").option("--output-dir <dir>", "Output directory for batch").action(async (options) => {
+program.name("formatferry").version(VERSION).description("Convert HTML, DOCX, PDF, XLSX, CSV to Markdown").option("-i, --input <path>", "Input file path").option("-o, --output <path>", "Output file path").option("-f, --format <format>", "Markdown format (github, commonmark, slack, discord, reddit, confluence, custom, rmarkdown)", "github").option("--url <url>", "Fetch article from URL").option("--batch <glob>", "Batch convert files (Premium)").option("--output-dir <dir>", "Output directory for batch").option("--toonify", "Compress output to TOON format (Premium)").action(async (options) => {
   if (options.batch) {
     await batchCommand(options);
   } else {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "formatferry",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "FormatFerry CLI — Convert HTML, DOCX, PDF, XLSX, CSV to Markdown",
   "keywords": [
     "markdown",