formatferry 0.1.1 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/README.md +4 -0
  2. package/dist/index.js +518 -17
  3. package/package.json +1 -1
package/README.md CHANGED
@@ -214,6 +214,10 @@ formatferry -i page.html -f commonmark
214
214
  curl -s https://example.com | formatferry -f reddit
215
215
  ```
216
216
 
217
+ ## Tips
218
+
219
+ - For HTML documents over ~100 KB, prefer `-i <file>` over piping through stdin to avoid OS shell buffer limitations.
220
+
217
221
  ## Development
218
222
 
219
223
  ```bash
package/dist/index.js CHANGED
@@ -21981,9 +21981,11 @@ var CachedEntitlementProvider = class {
21981
21981
  // src/entitlement.ts
21982
21982
  var LICENSE_SERVER_URL = "https://formatferry.vibingfun.com";
21983
21983
  var VALIDATE_ENDPOINT = `${LICENSE_SERVER_URL}/v1/licenses/validate`;
21984
- var STARTUP_TIMEOUT_MS = 2e3;
21984
+ var DEFAULT_TIMEOUT_MS = 2e3;
21985
21985
  var STALE_CACHE_DAYS = 7;
21986
- async function resolveEntitlement() {
21986
+ var FRESH_CACHE_HOURS = 24;
21987
+ async function resolveEntitlement(opts = {}) {
21988
+ const { offline = false, timeoutMs = DEFAULT_TIMEOUT_MS } = opts;
21987
21989
  let config;
21988
21990
  try {
21989
21991
  config = readConfig();
@@ -22007,11 +22009,19 @@ async function resolveEntitlement() {
22007
22009
  return new FreeEntitlementProvider();
22008
22010
  }
22009
22011
  const { licenseKey, lastValidated } = config;
22012
+ if (offline) {
22013
+ return buildCachedProvider(config, lastValidated);
22014
+ }
22015
+ const cacheAgeMs = lastValidated ? Date.now() - new Date(lastValidated).getTime() : Infinity;
22016
+ const freshCacheMs = FRESH_CACHE_HOURS * 60 * 60 * 1e3;
22017
+ if (cacheAgeMs < freshCacheMs) {
22018
+ return buildCachedProvider(config, lastValidated);
22019
+ }
22010
22020
  let data = null;
22011
22021
  let networkError = false;
22012
22022
  try {
22013
22023
  const controller = new AbortController();
22014
- const timer = setTimeout(() => controller.abort(), STARTUP_TIMEOUT_MS);
22024
+ const timer = setTimeout(() => controller.abort(), timeoutMs);
22015
22025
  const response = await fetch(VALIDATE_ENDPOINT, {
22016
22026
  method: "POST",
22017
22027
  headers: { "Content-Type": "application/json" },
@@ -22046,22 +22056,24 @@ async function resolveEntitlement() {
22046
22056
  });
22047
22057
  return new CachedEntitlementProvider(freshState);
22048
22058
  }
22049
- const cacheAgeMs = lastValidated ? Date.now() - new Date(lastValidated).getTime() : Infinity;
22050
22059
  const staleCacheMs = STALE_CACHE_DAYS * 24 * 60 * 60 * 1e3;
22051
22060
  if (cacheAgeMs < staleCacheMs) {
22052
- const cachedState = {
22053
- tier: config.tier ?? "free",
22054
- features: config.features ?? [],
22055
- expiresAt: config.expiresAt ?? null,
22056
- lastValidated: lastValidated ?? ""
22057
- };
22058
- return new CachedEntitlementProvider(cachedState);
22061
+ return buildCachedProvider(config, lastValidated);
22059
22062
  }
22060
22063
  process.stderr.write(
22061
22064
  "Warning: Could not reach license server and cached license is older than 7 days. Operating in free tier. Please reconnect to validate your license.\n"
22062
22065
  );
22063
22066
  return new FreeEntitlementProvider();
22064
22067
  }
22068
+ function buildCachedProvider(config, lastValidated) {
22069
+ const cachedState = {
22070
+ tier: config.tier ?? "free",
22071
+ features: config.features ?? [],
22072
+ expiresAt: config.expiresAt ?? null,
22073
+ lastValidated: lastValidated ?? ""
22074
+ };
22075
+ return new CachedEntitlementProvider(cachedState);
22076
+ }
22065
22077
 
22066
22078
  // ../../src/shared/readability-processor.ts
22067
22079
  var import_readability = __toESM(require_readability(), 1);
@@ -22108,6 +22120,399 @@ function extractArticle(html, domProvider) {
22108
22120
  };
22109
22121
  }
22110
22122
 
22123
+ // ../../src/shared/token-estimator.ts
22124
+ var CJK_REGEX = /[\u4e00-\u9fff\u3400-\u4dbf\u3000-\u303f\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]/g;
22125
+ var MAX_INPUT_LENGTH = 1e6;
22126
+ function estimateTokens(text) {
22127
+ if (!text || text.length === 0) {
22128
+ return 0;
22129
+ }
22130
+ if (text.length > MAX_INPUT_LENGTH) {
22131
+ return Math.round(text.length / 4);
22132
+ }
22133
+ const cjkMatches = text.match(CJK_REGEX);
22134
+ const cjkCount = cjkMatches ? cjkMatches.length : 0;
22135
+ const otherCount = text.length - cjkCount;
22136
+ const cjkTokens = cjkCount / 1.5;
22137
+ const otherTokens = otherCount / 4;
22138
+ return Math.round(cjkTokens + otherTokens);
22139
+ }
22140
+
22141
+ // ../../src/shared/toon-encoder.ts
22142
+ var DEFAULT_OPTIONS = {
22143
+ indent: 0,
22144
+ keyFolding: true,
22145
+ tableFormat: "compact"
22146
+ };
22147
+ var ToonEncoder = class _ToonEncoder {
22148
+ static encode(value, opts) {
22149
+ const options = { ...DEFAULT_OPTIONS, ...opts };
22150
+ return _ToonEncoder.encodeValue(value, options.indent, options);
22151
+ }
22152
+ static encodeValue(value, indent, opts) {
22153
+ if (value === null) {
22154
+ return "null";
22155
+ }
22156
+ if (typeof value === "boolean") {
22157
+ return value ? "true" : "false";
22158
+ }
22159
+ if (typeof value === "number") {
22160
+ return String(value);
22161
+ }
22162
+ if (typeof value === "string") {
22163
+ return _ToonEncoder.encodeString(value);
22164
+ }
22165
+ if (Array.isArray(value)) {
22166
+ return _ToonEncoder.encodeArray(value, indent, opts);
22167
+ }
22168
+ if (typeof value === "object") {
22169
+ return _ToonEncoder.encodeObject(value, indent, opts);
22170
+ }
22171
+ return String(value);
22172
+ }
22173
+ static encodeString(str) {
22174
+ const needsQuotes = str.includes(":") || str.includes("\n") || str.includes(" ") || str.includes("|") || str.includes(",") || str.startsWith(" ") || str.endsWith(" ") || str === "";
22175
+ if (needsQuotes) {
22176
+ const escaped = str.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\n/g, "\\n").replace(/\t/g, "\\t");
22177
+ return `"${escaped}"`;
22178
+ }
22179
+ return str;
22180
+ }
22181
+ static encodeArray(arr, indent, opts) {
22182
+ if (arr.length === 0) {
22183
+ return "[]";
22184
+ }
22185
+ if (_ToonEncoder.isTabularArray(arr)) {
22186
+ return _ToonEncoder.encodeTabular(arr);
22187
+ }
22188
+ const prefix = " ".repeat(indent);
22189
+ const lines = arr.map((item) => {
22190
+ if (typeof item === "object" && item !== null && !Array.isArray(item)) {
22191
+ const obj = item;
22192
+ const keys = Object.keys(obj);
22193
+ if (keys.length > 0) {
22194
+ const objLines = keys.map((key, idx) => {
22195
+ const value = obj[key];
22196
+ const encodedValue = _ToonEncoder.encodeValue(value, indent + 1, opts);
22197
+ if (typeof value === "object" && value !== null && !Array.isArray(value)) {
22198
+ const innerLines = encodedValue.split("\n");
22199
+ if (innerLines.length > 1 && !encodedValue.startsWith("{}")) {
22200
+ if (idx === 0) {
22201
+ return `${prefix}- ${key}:
22202
+ ${innerLines.map((l) => `${prefix} ${l.trimStart()}`).join("\n")}`;
22203
+ }
22204
+ return `${prefix} ${key}:
22205
+ ${innerLines.map((l) => `${prefix} ${l.trimStart()}`).join("\n")}`;
22206
+ }
22207
+ }
22208
+ if (idx === 0) {
22209
+ return `${prefix}- ${key}: ${encodedValue}`;
22210
+ }
22211
+ return `${prefix} ${key}: ${encodedValue}`;
22212
+ });
22213
+ return objLines.join("\n");
22214
+ }
22215
+ }
22216
+ const encoded = _ToonEncoder.encodeValue(item, indent + 1, opts);
22217
+ return `${prefix}- ${encoded}`;
22218
+ });
22219
+ return lines.join("\n");
22220
+ }
22221
+ static encodeObject(obj, indent, opts) {
22222
+ const keys = Object.keys(obj);
22223
+ if (keys.length === 0) {
22224
+ return "{}";
22225
+ }
22226
+ if (opts.keyFolding && keys.length === 1) {
22227
+ const key = keys[0];
22228
+ const value = obj[key];
22229
+ if (typeof value === "object" && value !== null && !Array.isArray(value) && Object.keys(value).length === 1) {
22230
+ const foldedPath = _ToonEncoder.getFoldedPath(obj);
22231
+ if (foldedPath) {
22232
+ const { path, finalValue } = foldedPath;
22233
+ const encodedValue = _ToonEncoder.encodeValue(finalValue, indent, opts);
22234
+ const prefix2 = " ".repeat(indent);
22235
+ return `${prefix2}${path}: ${encodedValue}`;
22236
+ }
22237
+ }
22238
+ }
22239
+ const prefix = " ".repeat(indent);
22240
+ const lines = keys.map((key) => {
22241
+ const value = obj[key];
22242
+ const encodedValue = _ToonEncoder.encodeValue(value, indent + 1, opts);
22243
+ if (typeof value === "object" && value !== null && !Array.isArray(value)) {
22244
+ const innerLines = encodedValue.split("\n");
22245
+ if (innerLines.length > 1 && !encodedValue.startsWith("{}")) {
22246
+ return `${prefix}${key}:
22247
+ ${innerLines.map((l) => `${prefix} ${l.trimStart()}`).join("\n")}`;
22248
+ }
22249
+ }
22250
+ if (Array.isArray(value) && value.length > 0 && !_ToonEncoder.isTabularArray(value)) {
22251
+ return `${prefix}${key}:
22252
+ ${encodedValue}`;
22253
+ }
22254
+ return `${prefix}${key}: ${encodedValue}`;
22255
+ });
22256
+ return lines.join("\n");
22257
+ }
22258
+ static getFoldedPath(obj, pathParts = []) {
22259
+ const keys = Object.keys(obj);
22260
+ if (keys.length !== 1) {
22261
+ return null;
22262
+ }
22263
+ const key = keys[0];
22264
+ const value = obj[key];
22265
+ const newPath = [...pathParts, key];
22266
+ if (typeof value === "object" && value !== null && !Array.isArray(value)) {
22267
+ const innerKeys = Object.keys(value);
22268
+ if (innerKeys.length === 1) {
22269
+ return _ToonEncoder.getFoldedPath(value, newPath);
22270
+ }
22271
+ }
22272
+ return {
22273
+ path: newPath.join("."),
22274
+ finalValue: value
22275
+ };
22276
+ }
22277
+ static encodeTabular(arr) {
22278
+ if (arr.length === 0) {
22279
+ return "[]";
22280
+ }
22281
+ const keys = Object.keys(arr[0]);
22282
+ const header = `|${keys.join("|")}`;
22283
+ const rows = arr.map((row) => {
22284
+ const values = keys.map((key) => {
22285
+ const value = row[key];
22286
+ if (value === null) return "null";
22287
+ if (typeof value === "string") {
22288
+ return value.replace(/\|/g, "\\|");
22289
+ }
22290
+ return String(value);
22291
+ });
22292
+ return `|${values.join("|")}`;
22293
+ });
22294
+ return [header, ...rows].join("\n");
22295
+ }
22296
+ static isTabularArray(arr) {
22297
+ if (arr.length < 2) {
22298
+ return false;
22299
+ }
22300
+ const firstItem = arr[0];
22301
+ if (typeof firstItem !== "object" || firstItem === null || Array.isArray(firstItem)) {
22302
+ return false;
22303
+ }
22304
+ const firstKeys = Object.keys(firstItem).sort().join(",");
22305
+ for (let i = 1; i < arr.length; i++) {
22306
+ const item = arr[i];
22307
+ if (typeof item !== "object" || item === null || Array.isArray(item)) {
22308
+ return false;
22309
+ }
22310
+ const itemKeys = Object.keys(item).sort().join(",");
22311
+ if (itemKeys !== firstKeys) {
22312
+ return false;
22313
+ }
22314
+ for (const key in item) {
22315
+ const value = item[key];
22316
+ if (typeof value === "object" && value !== null) {
22317
+ return false;
22318
+ }
22319
+ }
22320
+ }
22321
+ for (const key in firstItem) {
22322
+ const value = firstItem[key];
22323
+ if (typeof value === "object" && value !== null) {
22324
+ return false;
22325
+ }
22326
+ }
22327
+ return true;
22328
+ }
22329
+ };
22330
+
22331
+ // ../../src/shared/toonify-service.ts
22332
+ var DEFAULT_OPTIONS2 = {
22333
+ preserveCodeBlocks: true,
22334
+ keyFolding: true
22335
+ };
22336
+ var ToonifyService = class {
22337
+ static compress(markdown, options) {
22338
+ const opts = { ...DEFAULT_OPTIONS2, ...options };
22339
+ const original = markdown;
22340
+ const originalTokenEstimate = estimateTokens(original);
22341
+ if (!markdown || markdown.trim().length === 0) {
22342
+ return {
22343
+ original,
22344
+ compressed: "",
22345
+ originalTokenEstimate,
22346
+ compressedTokenEstimate: 0,
22347
+ reductionPercent: 100
22348
+ };
22349
+ }
22350
+ const blocks = this.parseMarkdown(markdown, opts);
22351
+ const document2 = { t: "md", blocks };
22352
+ const encodeOptions = {
22353
+ keyFolding: opts.keyFolding,
22354
+ tableFormat: opts.tableFormat
22355
+ };
22356
+ const compressed = ToonEncoder.encode(document2, encodeOptions);
22357
+ const compressedTokenEstimate = estimateTokens(compressed);
22358
+ const reductionPercent = originalTokenEstimate > 0 ? Math.max(0, Math.min(100, Math.round(
22359
+ (originalTokenEstimate - compressedTokenEstimate) / originalTokenEstimate * 100
22360
+ ))) : 0;
22361
+ return {
22362
+ original,
22363
+ compressed,
22364
+ originalTokenEstimate,
22365
+ compressedTokenEstimate,
22366
+ reductionPercent
22367
+ };
22368
+ }
22369
+ static estimateTokens(text) {
22370
+ return estimateTokens(text);
22371
+ }
22372
+ static parseMarkdown(markdown, options) {
22373
+ const blocks = [];
22374
+ const lines = markdown.split("\n");
22375
+ let i = 0;
22376
+ while (i < lines.length) {
22377
+ const line = lines[i];
22378
+ if (line.match(/^```/)) {
22379
+ const { block, endIndex } = this.parseCodeBlock(lines, i, options);
22380
+ if (block) blocks.push(block);
22381
+ i = endIndex + 1;
22382
+ continue;
22383
+ }
22384
+ const headingMatch = line.match(/^(#{1,6})\s+(.+)$/);
22385
+ if (headingMatch) {
22386
+ blocks.push({
22387
+ h: headingMatch[1].length,
22388
+ text: headingMatch[2].trim()
22389
+ });
22390
+ i++;
22391
+ continue;
22392
+ }
22393
+ if (line.includes("|") && line.trim().startsWith("|")) {
22394
+ const { block, endIndex } = this.parseTable(lines, i);
22395
+ if (block) blocks.push(block);
22396
+ i = endIndex + 1;
22397
+ continue;
22398
+ }
22399
+ const listMatch = line.match(/^(\s*)[-*]\s+(.+)$|^(\s*)\d+\.\s+(.+)$/);
22400
+ if (listMatch) {
22401
+ const { block, endIndex } = this.parseList(lines, i);
22402
+ if (block) blocks.push(block);
22403
+ i = endIndex + 1;
22404
+ continue;
22405
+ }
22406
+ if (line.trim().length > 0) {
22407
+ const { block, endIndex } = this.parseParagraph(lines, i);
22408
+ if (block) blocks.push(block);
22409
+ i = endIndex + 1;
22410
+ continue;
22411
+ }
22412
+ i++;
22413
+ }
22414
+ return blocks;
22415
+ }
22416
+ static parseCodeBlock(lines, startIndex, options) {
22417
+ const startLine = lines[startIndex];
22418
+ const langMatch = startLine.match(/^```(\w*)$/);
22419
+ const lang = langMatch ? langMatch[1] : "";
22420
+ let endIndex = startIndex + 1;
22421
+ const codeLines = [];
22422
+ while (endIndex < lines.length) {
22423
+ if (lines[endIndex].match(/^```$/)) {
22424
+ break;
22425
+ }
22426
+ codeLines.push(lines[endIndex]);
22427
+ endIndex++;
22428
+ }
22429
+ const codeText = codeLines.join("\n");
22430
+ if (options.preserveCodeBlocks) {
22431
+ return {
22432
+ block: { code: { lang, text: codeText } },
22433
+ endIndex
22434
+ };
22435
+ }
22436
+ return {
22437
+ block: { code: { lang, text: `[code:${codeText.length}chars]` } },
22438
+ endIndex
22439
+ };
22440
+ }
22441
+ static parseTable(lines, startIndex) {
22442
+ const headerLine = lines[startIndex];
22443
+ const headers = this.parseTableRow(headerLine);
22444
+ let endIndex = startIndex + 1;
22445
+ if (endIndex < lines.length && lines[endIndex].match(/^\|[\s:-]+\|[\s:-|]+$/)) {
22446
+ endIndex++;
22447
+ }
22448
+ const rows = [];
22449
+ while (endIndex < lines.length) {
22450
+ const line = lines[endIndex];
22451
+ if (!line.includes("|") || !line.trim().startsWith("|")) {
22452
+ break;
22453
+ }
22454
+ rows.push(this.parseTableRow(line));
22455
+ endIndex++;
22456
+ }
22457
+ return {
22458
+ block: { table: { headers, rows } },
22459
+ endIndex: endIndex - 1
22460
+ };
22461
+ }
22462
+ static parseTableRow(line) {
22463
+ return line.split("|").map((cell) => cell.trim()).filter((cell) => cell.length > 0);
22464
+ }
22465
+ static parseList(lines, startIndex) {
22466
+ const items = [];
22467
+ let endIndex = startIndex;
22468
+ while (endIndex < lines.length) {
22469
+ const line = lines[endIndex];
22470
+ const match = line.match(/^(\s*)[-*]\s+(.+)$|^(\s*)\d+\.\s+(.+)$/);
22471
+ if (!match) break;
22472
+ items.push(match[2] || match[4]);
22473
+ endIndex++;
22474
+ }
22475
+ return {
22476
+ block: { list: items },
22477
+ endIndex: endIndex - 1
22478
+ };
22479
+ }
22480
+ static parseParagraph(lines, startIndex) {
22481
+ const paragraphLines = [];
22482
+ let endIndex = startIndex;
22483
+ while (endIndex < lines.length) {
22484
+ const line = lines[endIndex];
22485
+ if (line.trim().length === 0) break;
22486
+ if (line.match(/^#{1,6}\s+/)) break;
22487
+ if (line.match(/^```/)) break;
22488
+ if (line.match(/^[-*]\s+/) || line.match(/^\d+\.\s+/)) break;
22489
+ if (line.includes("|") && line.trim().startsWith("|")) break;
22490
+ paragraphLines.push(line);
22491
+ endIndex++;
22492
+ }
22493
+ return {
22494
+ block: { p: paragraphLines.join(" ").trim() },
22495
+ endIndex: endIndex - 1
22496
+ };
22497
+ }
22498
+ };
22499
+
22500
+ // src/toonify-helpers.ts
22501
+ var TOONIFY_MAX_INPUT_BYTES = 10 * 1024 * 1024;
22502
+ function formatSingleStats(result) {
22503
+ return `Toonify: ${result.originalTokenEstimate} \u2192 ${result.compressedTokenEstimate} tokens (${result.reductionPercent}% reduction)
22504
+ `;
22505
+ }
22506
+ function formatFileStats(filename, result) {
22507
+ return `Toonify [${filename}]: ${result.originalTokenEstimate} \u2192 ${result.compressedTokenEstimate} tokens (${result.reductionPercent}% reduction)
22508
+ `;
22509
+ }
22510
+ function formatAggregateStats(totalOriginal, totalCompressed, fileCount) {
22511
+ const aggregateReduction = totalOriginal > 0 ? Math.max(0, Math.min(100, Math.round((totalOriginal - totalCompressed) / totalOriginal * 100))) : 0;
22512
+ return `Toonify total: ${totalOriginal} \u2192 ${totalCompressed} tokens (${aggregateReduction}% reduction, ${fileCount} files)
22513
+ `;
22514
+ }
22515
+
22111
22516
  // src/commands/convert.ts
22112
22517
  var VALID_FORMATS = [
22113
22518
  "github",
@@ -22258,7 +22663,44 @@ async function urlToMarkdown(url, apiKey, format, proxyBaseUrl = process.env.FOR
22258
22663
  });
22259
22664
  return result.markdown;
22260
22665
  }
22666
+ function applyToonify(markdown) {
22667
+ if (Buffer.byteLength(markdown, "utf8") > TOONIFY_MAX_INPUT_BYTES) {
22668
+ process.stderr.write(
22669
+ `Error: Input too large for toonify (>${TOONIFY_MAX_INPUT_BYTES / 1024 / 1024} MB). Convert without --toonify or split the input.
22670
+ `
22671
+ );
22672
+ process.exit(1);
22673
+ }
22674
+ try {
22675
+ const result = ToonifyService.compress(markdown);
22676
+ process.stderr.write(formatSingleStats(result));
22677
+ return result.compressed;
22678
+ } catch (err) {
22679
+ const message = err instanceof Error ? err.message : String(err);
22680
+ process.stderr.write(`Error: Toonify compression failed: ${message}
22681
+ `);
22682
+ process.exit(1);
22683
+ }
22684
+ }
22261
22685
  async function convertCommand(options) {
22686
+ const parsedTimeout = options.timeout ? parseInt(options.timeout, 10) : void 0;
22687
+ if (parsedTimeout !== void 0 && (!Number.isFinite(parsedTimeout) || parsedTimeout <= 0)) {
22688
+ process.stderr.write("Error: --timeout must be a positive integer (milliseconds).\n");
22689
+ process.exit(1);
22690
+ }
22691
+ const entitlementOpts = {
22692
+ offline: options.offline,
22693
+ timeoutMs: parsedTimeout
22694
+ };
22695
+ if (options.toonify) {
22696
+ const provider = await resolveEntitlement(entitlementOpts);
22697
+ if (!provider.hasFeature("token-optimization")) {
22698
+ process.stderr.write(
22699
+ "Error: Token optimization requires a Premium license.\nRun `formatferry auth --license-key <KEY>` to activate your license.\n"
22700
+ );
22701
+ process.exit(1);
22702
+ }
22703
+ }
22262
22704
  const format = validateFormat(options.format ?? "github");
22263
22705
  if (options.url) {
22264
22706
  if (options.input) {
@@ -22273,7 +22715,14 @@ async function convertCommand(options) {
22273
22715
  );
22274
22716
  process.exit(1);
22275
22717
  }
22276
- const markdown2 = await urlToMarkdown(options.url, apiKey, format);
22718
+ let markdown2 = await urlToMarkdown(options.url, apiKey, format);
22719
+ if (options.toonify) {
22720
+ if (markdown2.trim()) {
22721
+ markdown2 = applyToonify(markdown2);
22722
+ } else {
22723
+ markdown2 = "";
22724
+ }
22725
+ }
22277
22726
  if (options.output) {
22278
22727
  writeFileSync2(options.output, markdown2, "utf8");
22279
22728
  } else {
@@ -22296,7 +22745,7 @@ async function convertCommand(options) {
22296
22745
  const html = fileBuffer.toString("utf8");
22297
22746
  let applyAtlassian = false;
22298
22747
  if (isAtlassianHTML(html)) {
22299
- const provider = await resolveEntitlement();
22748
+ const provider = await resolveEntitlement(entitlementOpts);
22300
22749
  if (!provider.hasFeature("atlassian")) {
22301
22750
  process.stderr.write(
22302
22751
  "Error: Atlassian HTML detected. A Premium license is required to process Atlassian content.\nRun `formatferry auth --license-key <KEY>` to activate your license.\n"
@@ -22313,7 +22762,7 @@ async function convertCommand(options) {
22313
22762
  const html = await readStdin();
22314
22763
  let applyAtlassian = false;
22315
22764
  if (isAtlassianHTML(html)) {
22316
- const provider = await resolveEntitlement();
22765
+ const provider = await resolveEntitlement(entitlementOpts);
22317
22766
  if (!provider.hasFeature("atlassian")) {
22318
22767
  process.stderr.write(
22319
22768
  "Error: Atlassian HTML detected. A Premium license is required to process Atlassian content.\nRun `formatferry auth --license-key <KEY>` to activate your license.\n"
@@ -22324,6 +22773,13 @@ async function convertCommand(options) {
22324
22773
  }
22325
22774
  markdown = await htmlToMarkdown(html, format, applyAtlassian);
22326
22775
  }
22776
+ if (options.toonify) {
22777
+ if (markdown.trim()) {
22778
+ markdown = applyToonify(markdown);
22779
+ } else {
22780
+ markdown = "";
22781
+ }
22782
+ }
22327
22783
  if (options.output) {
22328
22784
  writeFileSync2(options.output, markdown, "utf8");
22329
22785
  } else {
@@ -22372,13 +22828,30 @@ async function convertFile(filePath, format) {
22372
22828
  return c.markdown;
22373
22829
  }
22374
22830
  async function batchCommand(options) {
22375
- const provider = await resolveEntitlement();
22831
+ const parsedTimeout = options.timeout ? parseInt(options.timeout, 10) : void 0;
22832
+ if (parsedTimeout !== void 0 && (!Number.isFinite(parsedTimeout) || parsedTimeout <= 0)) {
22833
+ process.stderr.write("Error: --timeout must be a positive integer (milliseconds).\n");
22834
+ process.exit(1);
22835
+ }
22836
+ const entitlementOpts = {
22837
+ offline: options.offline,
22838
+ timeoutMs: parsedTimeout
22839
+ };
22840
+ const provider = await resolveEntitlement(entitlementOpts);
22376
22841
  if (!provider.hasFeature("batch")) {
22377
22842
  process.stderr.write(
22378
22843
  "Error: Batch processing requires a Premium license.\nRun `formatferry auth --license-key <KEY>` to activate your license.\n"
22379
22844
  );
22380
22845
  process.exit(1);
22381
22846
  }
22847
+ if (options.toonify) {
22848
+ if (!provider.hasFeature("token-optimization")) {
22849
+ process.stderr.write(
22850
+ "Error: Token optimization requires a Premium license.\nRun `formatferry auth --license-key <KEY>` to activate your license.\n"
22851
+ );
22852
+ process.exit(1);
22853
+ }
22854
+ }
22382
22855
  const glob = options.batch;
22383
22856
  if (!glob) {
22384
22857
  process.stderr.write("Error: --batch <glob> is required\n");
@@ -22393,9 +22866,34 @@ async function batchCommand(options) {
22393
22866
  const format = options.format ?? "github";
22394
22867
  if (options.outputDir) mkdirSync2(options.outputDir, { recursive: true });
22395
22868
  let successful = 0, failed = 0;
22869
+ let totalOriginal = 0, totalCompressed = 0, toonifiedCount = 0;
22396
22870
  for (const filePath of files) {
22397
22871
  try {
22398
- const markdown = await convertFile(filePath, format);
22872
+ let markdown = await convertFile(filePath, format);
22873
+ if (options.toonify) {
22874
+ if (markdown.trim()) {
22875
+ if (Buffer.byteLength(markdown, "utf8") > TOONIFY_MAX_INPUT_BYTES) {
22876
+ process.stderr.write(`Skipping toonify for ${basename(filePath)}: input exceeds ${TOONIFY_MAX_INPUT_BYTES / 1024 / 1024} MB
22877
+ `);
22878
+ } else {
22879
+ try {
22880
+ const result = ToonifyService.compress(markdown);
22881
+ process.stderr.write(formatFileStats(basename(filePath), result));
22882
+ markdown = result.compressed;
22883
+ totalOriginal += result.originalTokenEstimate;
22884
+ totalCompressed += result.compressedTokenEstimate;
22885
+ toonifiedCount++;
22886
+ } catch (err) {
22887
+ process.stderr.write(`Error toonifying ${filePath}: ${err instanceof Error ? err.message : String(err)}
22888
+ `);
22889
+ failed++;
22890
+ continue;
22891
+ }
22892
+ }
22893
+ } else {
22894
+ markdown = "";
22895
+ }
22896
+ }
22399
22897
  writeFileSync3(getOutputPath(filePath, options.outputDir), markdown, "utf8");
22400
22898
  successful++;
22401
22899
  } catch (err) {
@@ -22404,6 +22902,9 @@ async function batchCommand(options) {
22404
22902
  failed++;
22405
22903
  }
22406
22904
  }
22905
+ if (options.toonify && toonifiedCount > 0) {
22906
+ process.stderr.write(formatAggregateStats(totalOriginal, totalCompressed, toonifiedCount));
22907
+ }
22407
22908
  process.stderr.write(`
22408
22909
  Batch complete: ${successful} succeeded, ${failed} failed, ${files.length} total
22409
22910
  `);
@@ -22601,7 +23102,7 @@ function handleLogout() {
22601
23102
 
22602
23103
  // src/index.ts
22603
23104
  var VERSION = "0.1.0";
22604
- program.name("formatferry").version(VERSION).description("Convert HTML, DOCX, PDF, XLSX, CSV to Markdown").option("-i, --input <path>", "Input file path").option("-o, --output <path>", "Output file path").option("-f, --format <format>", "Markdown format (github, commonmark, slack, discord, reddit, confluence, custom, rmarkdown)", "github").option("--url <url>", "Fetch article from URL").option("--batch <glob>", "Batch convert files (Premium)").option("--output-dir <dir>", "Output directory for batch").action(async (options) => {
23105
+ program.name("formatferry").version(VERSION).description("Convert HTML, DOCX, PDF, XLSX, CSV to Markdown").option("-i, --input <path>", "Input file path").option("-o, --output <path>", "Output file path").option("-f, --format <format>", "Markdown format (github, commonmark, slack, discord, reddit, confluence, custom, rmarkdown)", "github").option("--url <url>", "Fetch article from URL").option("--batch <glob>", "Batch convert files (Premium)").option("--output-dir <dir>", "Output directory for batch").option("--toonify", "Compress output to TOON format (Premium)").option("--offline", "Skip entitlement check (use cached or free tier)").option("--timeout <ms>", "License server timeout in milliseconds (default: 2000)", "2000").action(async (options) => {
22605
23106
  if (options.batch) {
22606
23107
  await batchCommand(options);
22607
23108
  } else {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "formatferry",
3
- "version": "0.1.1",
3
+ "version": "0.1.3",
4
4
  "description": "FormatFerry CLI — Convert HTML, DOCX, PDF, XLSX, CSV to Markdown",
5
5
  "keywords": [
6
6
  "markdown",