formatferry 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/dist/index.js +518 -17
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -214,6 +214,10 @@ formatferry -i page.html -f commonmark
|
|
|
214
214
|
curl -s https://example.com | formatferry -f reddit
|
|
215
215
|
```
|
|
216
216
|
|
|
217
|
+
## Tips
|
|
218
|
+
|
|
219
|
+
- For HTML documents over ~100 KB, prefer `-i <file>` over piping through stdin to avoid OS shell buffer limitations.
|
|
220
|
+
|
|
217
221
|
## Development
|
|
218
222
|
|
|
219
223
|
```bash
|
package/dist/index.js
CHANGED
|
@@ -21981,9 +21981,11 @@ var CachedEntitlementProvider = class {
|
|
|
21981
21981
|
// src/entitlement.ts
|
|
21982
21982
|
var LICENSE_SERVER_URL = "https://formatferry.vibingfun.com";
|
|
21983
21983
|
var VALIDATE_ENDPOINT = `${LICENSE_SERVER_URL}/v1/licenses/validate`;
|
|
21984
|
-
var
|
|
21984
|
+
var DEFAULT_TIMEOUT_MS = 2e3;
|
|
21985
21985
|
var STALE_CACHE_DAYS = 7;
|
|
21986
|
-
|
|
21986
|
+
var FRESH_CACHE_HOURS = 24;
|
|
21987
|
+
async function resolveEntitlement(opts = {}) {
|
|
21988
|
+
const { offline = false, timeoutMs = DEFAULT_TIMEOUT_MS } = opts;
|
|
21987
21989
|
let config;
|
|
21988
21990
|
try {
|
|
21989
21991
|
config = readConfig();
|
|
@@ -22007,11 +22009,19 @@ async function resolveEntitlement() {
|
|
|
22007
22009
|
return new FreeEntitlementProvider();
|
|
22008
22010
|
}
|
|
22009
22011
|
const { licenseKey, lastValidated } = config;
|
|
22012
|
+
if (offline) {
|
|
22013
|
+
return buildCachedProvider(config, lastValidated);
|
|
22014
|
+
}
|
|
22015
|
+
const cacheAgeMs = lastValidated ? Date.now() - new Date(lastValidated).getTime() : Infinity;
|
|
22016
|
+
const freshCacheMs = FRESH_CACHE_HOURS * 60 * 60 * 1e3;
|
|
22017
|
+
if (cacheAgeMs < freshCacheMs) {
|
|
22018
|
+
return buildCachedProvider(config, lastValidated);
|
|
22019
|
+
}
|
|
22010
22020
|
let data = null;
|
|
22011
22021
|
let networkError = false;
|
|
22012
22022
|
try {
|
|
22013
22023
|
const controller = new AbortController();
|
|
22014
|
-
const timer = setTimeout(() => controller.abort(),
|
|
22024
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
22015
22025
|
const response = await fetch(VALIDATE_ENDPOINT, {
|
|
22016
22026
|
method: "POST",
|
|
22017
22027
|
headers: { "Content-Type": "application/json" },
|
|
@@ -22046,22 +22056,24 @@ async function resolveEntitlement() {
|
|
|
22046
22056
|
});
|
|
22047
22057
|
return new CachedEntitlementProvider(freshState);
|
|
22048
22058
|
}
|
|
22049
|
-
const cacheAgeMs = lastValidated ? Date.now() - new Date(lastValidated).getTime() : Infinity;
|
|
22050
22059
|
const staleCacheMs = STALE_CACHE_DAYS * 24 * 60 * 60 * 1e3;
|
|
22051
22060
|
if (cacheAgeMs < staleCacheMs) {
|
|
22052
|
-
|
|
22053
|
-
tier: config.tier ?? "free",
|
|
22054
|
-
features: config.features ?? [],
|
|
22055
|
-
expiresAt: config.expiresAt ?? null,
|
|
22056
|
-
lastValidated: lastValidated ?? ""
|
|
22057
|
-
};
|
|
22058
|
-
return new CachedEntitlementProvider(cachedState);
|
|
22061
|
+
return buildCachedProvider(config, lastValidated);
|
|
22059
22062
|
}
|
|
22060
22063
|
process.stderr.write(
|
|
22061
22064
|
"Warning: Could not reach license server and cached license is older than 7 days. Operating in free tier. Please reconnect to validate your license.\n"
|
|
22062
22065
|
);
|
|
22063
22066
|
return new FreeEntitlementProvider();
|
|
22064
22067
|
}
|
|
22068
|
+
function buildCachedProvider(config, lastValidated) {
|
|
22069
|
+
const cachedState = {
|
|
22070
|
+
tier: config.tier ?? "free",
|
|
22071
|
+
features: config.features ?? [],
|
|
22072
|
+
expiresAt: config.expiresAt ?? null,
|
|
22073
|
+
lastValidated: lastValidated ?? ""
|
|
22074
|
+
};
|
|
22075
|
+
return new CachedEntitlementProvider(cachedState);
|
|
22076
|
+
}
|
|
22065
22077
|
|
|
22066
22078
|
// ../../src/shared/readability-processor.ts
|
|
22067
22079
|
var import_readability = __toESM(require_readability(), 1);
|
|
@@ -22108,6 +22120,399 @@ function extractArticle(html, domProvider) {
|
|
|
22108
22120
|
};
|
|
22109
22121
|
}
|
|
22110
22122
|
|
|
22123
|
+
// ../../src/shared/token-estimator.ts
|
|
22124
|
+
var CJK_REGEX = /[\u4e00-\u9fff\u3400-\u4dbf\u3000-\u303f\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]/g;
|
|
22125
|
+
var MAX_INPUT_LENGTH = 1e6;
|
|
22126
|
+
function estimateTokens(text) {
|
|
22127
|
+
if (!text || text.length === 0) {
|
|
22128
|
+
return 0;
|
|
22129
|
+
}
|
|
22130
|
+
if (text.length > MAX_INPUT_LENGTH) {
|
|
22131
|
+
return Math.round(text.length / 4);
|
|
22132
|
+
}
|
|
22133
|
+
const cjkMatches = text.match(CJK_REGEX);
|
|
22134
|
+
const cjkCount = cjkMatches ? cjkMatches.length : 0;
|
|
22135
|
+
const otherCount = text.length - cjkCount;
|
|
22136
|
+
const cjkTokens = cjkCount / 1.5;
|
|
22137
|
+
const otherTokens = otherCount / 4;
|
|
22138
|
+
return Math.round(cjkTokens + otherTokens);
|
|
22139
|
+
}
|
|
22140
|
+
|
|
22141
|
+
// ../../src/shared/toon-encoder.ts
|
|
22142
|
+
var DEFAULT_OPTIONS = {
|
|
22143
|
+
indent: 0,
|
|
22144
|
+
keyFolding: true,
|
|
22145
|
+
tableFormat: "compact"
|
|
22146
|
+
};
|
|
22147
|
+
var ToonEncoder = class _ToonEncoder {
|
|
22148
|
+
static encode(value, opts) {
|
|
22149
|
+
const options = { ...DEFAULT_OPTIONS, ...opts };
|
|
22150
|
+
return _ToonEncoder.encodeValue(value, options.indent, options);
|
|
22151
|
+
}
|
|
22152
|
+
static encodeValue(value, indent, opts) {
|
|
22153
|
+
if (value === null) {
|
|
22154
|
+
return "null";
|
|
22155
|
+
}
|
|
22156
|
+
if (typeof value === "boolean") {
|
|
22157
|
+
return value ? "true" : "false";
|
|
22158
|
+
}
|
|
22159
|
+
if (typeof value === "number") {
|
|
22160
|
+
return String(value);
|
|
22161
|
+
}
|
|
22162
|
+
if (typeof value === "string") {
|
|
22163
|
+
return _ToonEncoder.encodeString(value);
|
|
22164
|
+
}
|
|
22165
|
+
if (Array.isArray(value)) {
|
|
22166
|
+
return _ToonEncoder.encodeArray(value, indent, opts);
|
|
22167
|
+
}
|
|
22168
|
+
if (typeof value === "object") {
|
|
22169
|
+
return _ToonEncoder.encodeObject(value, indent, opts);
|
|
22170
|
+
}
|
|
22171
|
+
return String(value);
|
|
22172
|
+
}
|
|
22173
|
+
static encodeString(str) {
|
|
22174
|
+
const needsQuotes = str.includes(":") || str.includes("\n") || str.includes(" ") || str.includes("|") || str.includes(",") || str.startsWith(" ") || str.endsWith(" ") || str === "";
|
|
22175
|
+
if (needsQuotes) {
|
|
22176
|
+
const escaped = str.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\n/g, "\\n").replace(/\t/g, "\\t");
|
|
22177
|
+
return `"${escaped}"`;
|
|
22178
|
+
}
|
|
22179
|
+
return str;
|
|
22180
|
+
}
|
|
22181
|
+
static encodeArray(arr, indent, opts) {
|
|
22182
|
+
if (arr.length === 0) {
|
|
22183
|
+
return "[]";
|
|
22184
|
+
}
|
|
22185
|
+
if (_ToonEncoder.isTabularArray(arr)) {
|
|
22186
|
+
return _ToonEncoder.encodeTabular(arr);
|
|
22187
|
+
}
|
|
22188
|
+
const prefix = " ".repeat(indent);
|
|
22189
|
+
const lines = arr.map((item) => {
|
|
22190
|
+
if (typeof item === "object" && item !== null && !Array.isArray(item)) {
|
|
22191
|
+
const obj = item;
|
|
22192
|
+
const keys = Object.keys(obj);
|
|
22193
|
+
if (keys.length > 0) {
|
|
22194
|
+
const objLines = keys.map((key, idx) => {
|
|
22195
|
+
const value = obj[key];
|
|
22196
|
+
const encodedValue = _ToonEncoder.encodeValue(value, indent + 1, opts);
|
|
22197
|
+
if (typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
22198
|
+
const innerLines = encodedValue.split("\n");
|
|
22199
|
+
if (innerLines.length > 1 && !encodedValue.startsWith("{}")) {
|
|
22200
|
+
if (idx === 0) {
|
|
22201
|
+
return `${prefix}- ${key}:
|
|
22202
|
+
${innerLines.map((l) => `${prefix} ${l.trimStart()}`).join("\n")}`;
|
|
22203
|
+
}
|
|
22204
|
+
return `${prefix} ${key}:
|
|
22205
|
+
${innerLines.map((l) => `${prefix} ${l.trimStart()}`).join("\n")}`;
|
|
22206
|
+
}
|
|
22207
|
+
}
|
|
22208
|
+
if (idx === 0) {
|
|
22209
|
+
return `${prefix}- ${key}: ${encodedValue}`;
|
|
22210
|
+
}
|
|
22211
|
+
return `${prefix} ${key}: ${encodedValue}`;
|
|
22212
|
+
});
|
|
22213
|
+
return objLines.join("\n");
|
|
22214
|
+
}
|
|
22215
|
+
}
|
|
22216
|
+
const encoded = _ToonEncoder.encodeValue(item, indent + 1, opts);
|
|
22217
|
+
return `${prefix}- ${encoded}`;
|
|
22218
|
+
});
|
|
22219
|
+
return lines.join("\n");
|
|
22220
|
+
}
|
|
22221
|
+
static encodeObject(obj, indent, opts) {
|
|
22222
|
+
const keys = Object.keys(obj);
|
|
22223
|
+
if (keys.length === 0) {
|
|
22224
|
+
return "{}";
|
|
22225
|
+
}
|
|
22226
|
+
if (opts.keyFolding && keys.length === 1) {
|
|
22227
|
+
const key = keys[0];
|
|
22228
|
+
const value = obj[key];
|
|
22229
|
+
if (typeof value === "object" && value !== null && !Array.isArray(value) && Object.keys(value).length === 1) {
|
|
22230
|
+
const foldedPath = _ToonEncoder.getFoldedPath(obj);
|
|
22231
|
+
if (foldedPath) {
|
|
22232
|
+
const { path, finalValue } = foldedPath;
|
|
22233
|
+
const encodedValue = _ToonEncoder.encodeValue(finalValue, indent, opts);
|
|
22234
|
+
const prefix2 = " ".repeat(indent);
|
|
22235
|
+
return `${prefix2}${path}: ${encodedValue}`;
|
|
22236
|
+
}
|
|
22237
|
+
}
|
|
22238
|
+
}
|
|
22239
|
+
const prefix = " ".repeat(indent);
|
|
22240
|
+
const lines = keys.map((key) => {
|
|
22241
|
+
const value = obj[key];
|
|
22242
|
+
const encodedValue = _ToonEncoder.encodeValue(value, indent + 1, opts);
|
|
22243
|
+
if (typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
22244
|
+
const innerLines = encodedValue.split("\n");
|
|
22245
|
+
if (innerLines.length > 1 && !encodedValue.startsWith("{}")) {
|
|
22246
|
+
return `${prefix}${key}:
|
|
22247
|
+
${innerLines.map((l) => `${prefix} ${l.trimStart()}`).join("\n")}`;
|
|
22248
|
+
}
|
|
22249
|
+
}
|
|
22250
|
+
if (Array.isArray(value) && value.length > 0 && !_ToonEncoder.isTabularArray(value)) {
|
|
22251
|
+
return `${prefix}${key}:
|
|
22252
|
+
${encodedValue}`;
|
|
22253
|
+
}
|
|
22254
|
+
return `${prefix}${key}: ${encodedValue}`;
|
|
22255
|
+
});
|
|
22256
|
+
return lines.join("\n");
|
|
22257
|
+
}
|
|
22258
|
+
static getFoldedPath(obj, pathParts = []) {
|
|
22259
|
+
const keys = Object.keys(obj);
|
|
22260
|
+
if (keys.length !== 1) {
|
|
22261
|
+
return null;
|
|
22262
|
+
}
|
|
22263
|
+
const key = keys[0];
|
|
22264
|
+
const value = obj[key];
|
|
22265
|
+
const newPath = [...pathParts, key];
|
|
22266
|
+
if (typeof value === "object" && value !== null && !Array.isArray(value)) {
|
|
22267
|
+
const innerKeys = Object.keys(value);
|
|
22268
|
+
if (innerKeys.length === 1) {
|
|
22269
|
+
return _ToonEncoder.getFoldedPath(value, newPath);
|
|
22270
|
+
}
|
|
22271
|
+
}
|
|
22272
|
+
return {
|
|
22273
|
+
path: newPath.join("."),
|
|
22274
|
+
finalValue: value
|
|
22275
|
+
};
|
|
22276
|
+
}
|
|
22277
|
+
static encodeTabular(arr) {
|
|
22278
|
+
if (arr.length === 0) {
|
|
22279
|
+
return "[]";
|
|
22280
|
+
}
|
|
22281
|
+
const keys = Object.keys(arr[0]);
|
|
22282
|
+
const header = `|${keys.join("|")}`;
|
|
22283
|
+
const rows = arr.map((row) => {
|
|
22284
|
+
const values = keys.map((key) => {
|
|
22285
|
+
const value = row[key];
|
|
22286
|
+
if (value === null) return "null";
|
|
22287
|
+
if (typeof value === "string") {
|
|
22288
|
+
return value.replace(/\|/g, "\\|");
|
|
22289
|
+
}
|
|
22290
|
+
return String(value);
|
|
22291
|
+
});
|
|
22292
|
+
return `|${values.join("|")}`;
|
|
22293
|
+
});
|
|
22294
|
+
return [header, ...rows].join("\n");
|
|
22295
|
+
}
|
|
22296
|
+
static isTabularArray(arr) {
|
|
22297
|
+
if (arr.length < 2) {
|
|
22298
|
+
return false;
|
|
22299
|
+
}
|
|
22300
|
+
const firstItem = arr[0];
|
|
22301
|
+
if (typeof firstItem !== "object" || firstItem === null || Array.isArray(firstItem)) {
|
|
22302
|
+
return false;
|
|
22303
|
+
}
|
|
22304
|
+
const firstKeys = Object.keys(firstItem).sort().join(",");
|
|
22305
|
+
for (let i = 1; i < arr.length; i++) {
|
|
22306
|
+
const item = arr[i];
|
|
22307
|
+
if (typeof item !== "object" || item === null || Array.isArray(item)) {
|
|
22308
|
+
return false;
|
|
22309
|
+
}
|
|
22310
|
+
const itemKeys = Object.keys(item).sort().join(",");
|
|
22311
|
+
if (itemKeys !== firstKeys) {
|
|
22312
|
+
return false;
|
|
22313
|
+
}
|
|
22314
|
+
for (const key in item) {
|
|
22315
|
+
const value = item[key];
|
|
22316
|
+
if (typeof value === "object" && value !== null) {
|
|
22317
|
+
return false;
|
|
22318
|
+
}
|
|
22319
|
+
}
|
|
22320
|
+
}
|
|
22321
|
+
for (const key in firstItem) {
|
|
22322
|
+
const value = firstItem[key];
|
|
22323
|
+
if (typeof value === "object" && value !== null) {
|
|
22324
|
+
return false;
|
|
22325
|
+
}
|
|
22326
|
+
}
|
|
22327
|
+
return true;
|
|
22328
|
+
}
|
|
22329
|
+
};
|
|
22330
|
+
|
|
22331
|
+
// ../../src/shared/toonify-service.ts
|
|
22332
|
+
var DEFAULT_OPTIONS2 = {
|
|
22333
|
+
preserveCodeBlocks: true,
|
|
22334
|
+
keyFolding: true
|
|
22335
|
+
};
|
|
22336
|
+
var ToonifyService = class {
|
|
22337
|
+
static compress(markdown, options) {
|
|
22338
|
+
const opts = { ...DEFAULT_OPTIONS2, ...options };
|
|
22339
|
+
const original = markdown;
|
|
22340
|
+
const originalTokenEstimate = estimateTokens(original);
|
|
22341
|
+
if (!markdown || markdown.trim().length === 0) {
|
|
22342
|
+
return {
|
|
22343
|
+
original,
|
|
22344
|
+
compressed: "",
|
|
22345
|
+
originalTokenEstimate,
|
|
22346
|
+
compressedTokenEstimate: 0,
|
|
22347
|
+
reductionPercent: 100
|
|
22348
|
+
};
|
|
22349
|
+
}
|
|
22350
|
+
const blocks = this.parseMarkdown(markdown, opts);
|
|
22351
|
+
const document2 = { t: "md", blocks };
|
|
22352
|
+
const encodeOptions = {
|
|
22353
|
+
keyFolding: opts.keyFolding,
|
|
22354
|
+
tableFormat: opts.tableFormat
|
|
22355
|
+
};
|
|
22356
|
+
const compressed = ToonEncoder.encode(document2, encodeOptions);
|
|
22357
|
+
const compressedTokenEstimate = estimateTokens(compressed);
|
|
22358
|
+
const reductionPercent = originalTokenEstimate > 0 ? Math.max(0, Math.min(100, Math.round(
|
|
22359
|
+
(originalTokenEstimate - compressedTokenEstimate) / originalTokenEstimate * 100
|
|
22360
|
+
))) : 0;
|
|
22361
|
+
return {
|
|
22362
|
+
original,
|
|
22363
|
+
compressed,
|
|
22364
|
+
originalTokenEstimate,
|
|
22365
|
+
compressedTokenEstimate,
|
|
22366
|
+
reductionPercent
|
|
22367
|
+
};
|
|
22368
|
+
}
|
|
22369
|
+
static estimateTokens(text) {
|
|
22370
|
+
return estimateTokens(text);
|
|
22371
|
+
}
|
|
22372
|
+
static parseMarkdown(markdown, options) {
|
|
22373
|
+
const blocks = [];
|
|
22374
|
+
const lines = markdown.split("\n");
|
|
22375
|
+
let i = 0;
|
|
22376
|
+
while (i < lines.length) {
|
|
22377
|
+
const line = lines[i];
|
|
22378
|
+
if (line.match(/^```/)) {
|
|
22379
|
+
const { block, endIndex } = this.parseCodeBlock(lines, i, options);
|
|
22380
|
+
if (block) blocks.push(block);
|
|
22381
|
+
i = endIndex + 1;
|
|
22382
|
+
continue;
|
|
22383
|
+
}
|
|
22384
|
+
const headingMatch = line.match(/^(#{1,6})\s+(.+)$/);
|
|
22385
|
+
if (headingMatch) {
|
|
22386
|
+
blocks.push({
|
|
22387
|
+
h: headingMatch[1].length,
|
|
22388
|
+
text: headingMatch[2].trim()
|
|
22389
|
+
});
|
|
22390
|
+
i++;
|
|
22391
|
+
continue;
|
|
22392
|
+
}
|
|
22393
|
+
if (line.includes("|") && line.trim().startsWith("|")) {
|
|
22394
|
+
const { block, endIndex } = this.parseTable(lines, i);
|
|
22395
|
+
if (block) blocks.push(block);
|
|
22396
|
+
i = endIndex + 1;
|
|
22397
|
+
continue;
|
|
22398
|
+
}
|
|
22399
|
+
const listMatch = line.match(/^(\s*)[-*]\s+(.+)$|^(\s*)\d+\.\s+(.+)$/);
|
|
22400
|
+
if (listMatch) {
|
|
22401
|
+
const { block, endIndex } = this.parseList(lines, i);
|
|
22402
|
+
if (block) blocks.push(block);
|
|
22403
|
+
i = endIndex + 1;
|
|
22404
|
+
continue;
|
|
22405
|
+
}
|
|
22406
|
+
if (line.trim().length > 0) {
|
|
22407
|
+
const { block, endIndex } = this.parseParagraph(lines, i);
|
|
22408
|
+
if (block) blocks.push(block);
|
|
22409
|
+
i = endIndex + 1;
|
|
22410
|
+
continue;
|
|
22411
|
+
}
|
|
22412
|
+
i++;
|
|
22413
|
+
}
|
|
22414
|
+
return blocks;
|
|
22415
|
+
}
|
|
22416
|
+
static parseCodeBlock(lines, startIndex, options) {
|
|
22417
|
+
const startLine = lines[startIndex];
|
|
22418
|
+
const langMatch = startLine.match(/^```(\w*)$/);
|
|
22419
|
+
const lang = langMatch ? langMatch[1] : "";
|
|
22420
|
+
let endIndex = startIndex + 1;
|
|
22421
|
+
const codeLines = [];
|
|
22422
|
+
while (endIndex < lines.length) {
|
|
22423
|
+
if (lines[endIndex].match(/^```$/)) {
|
|
22424
|
+
break;
|
|
22425
|
+
}
|
|
22426
|
+
codeLines.push(lines[endIndex]);
|
|
22427
|
+
endIndex++;
|
|
22428
|
+
}
|
|
22429
|
+
const codeText = codeLines.join("\n");
|
|
22430
|
+
if (options.preserveCodeBlocks) {
|
|
22431
|
+
return {
|
|
22432
|
+
block: { code: { lang, text: codeText } },
|
|
22433
|
+
endIndex
|
|
22434
|
+
};
|
|
22435
|
+
}
|
|
22436
|
+
return {
|
|
22437
|
+
block: { code: { lang, text: `[code:${codeText.length}chars]` } },
|
|
22438
|
+
endIndex
|
|
22439
|
+
};
|
|
22440
|
+
}
|
|
22441
|
+
static parseTable(lines, startIndex) {
|
|
22442
|
+
const headerLine = lines[startIndex];
|
|
22443
|
+
const headers = this.parseTableRow(headerLine);
|
|
22444
|
+
let endIndex = startIndex + 1;
|
|
22445
|
+
if (endIndex < lines.length && lines[endIndex].match(/^\|[\s:-]+\|[\s:-|]+$/)) {
|
|
22446
|
+
endIndex++;
|
|
22447
|
+
}
|
|
22448
|
+
const rows = [];
|
|
22449
|
+
while (endIndex < lines.length) {
|
|
22450
|
+
const line = lines[endIndex];
|
|
22451
|
+
if (!line.includes("|") || !line.trim().startsWith("|")) {
|
|
22452
|
+
break;
|
|
22453
|
+
}
|
|
22454
|
+
rows.push(this.parseTableRow(line));
|
|
22455
|
+
endIndex++;
|
|
22456
|
+
}
|
|
22457
|
+
return {
|
|
22458
|
+
block: { table: { headers, rows } },
|
|
22459
|
+
endIndex: endIndex - 1
|
|
22460
|
+
};
|
|
22461
|
+
}
|
|
22462
|
+
static parseTableRow(line) {
|
|
22463
|
+
return line.split("|").map((cell) => cell.trim()).filter((cell) => cell.length > 0);
|
|
22464
|
+
}
|
|
22465
|
+
static parseList(lines, startIndex) {
|
|
22466
|
+
const items = [];
|
|
22467
|
+
let endIndex = startIndex;
|
|
22468
|
+
while (endIndex < lines.length) {
|
|
22469
|
+
const line = lines[endIndex];
|
|
22470
|
+
const match = line.match(/^(\s*)[-*]\s+(.+)$|^(\s*)\d+\.\s+(.+)$/);
|
|
22471
|
+
if (!match) break;
|
|
22472
|
+
items.push(match[2] || match[4]);
|
|
22473
|
+
endIndex++;
|
|
22474
|
+
}
|
|
22475
|
+
return {
|
|
22476
|
+
block: { list: items },
|
|
22477
|
+
endIndex: endIndex - 1
|
|
22478
|
+
};
|
|
22479
|
+
}
|
|
22480
|
+
static parseParagraph(lines, startIndex) {
|
|
22481
|
+
const paragraphLines = [];
|
|
22482
|
+
let endIndex = startIndex;
|
|
22483
|
+
while (endIndex < lines.length) {
|
|
22484
|
+
const line = lines[endIndex];
|
|
22485
|
+
if (line.trim().length === 0) break;
|
|
22486
|
+
if (line.match(/^#{1,6}\s+/)) break;
|
|
22487
|
+
if (line.match(/^```/)) break;
|
|
22488
|
+
if (line.match(/^[-*]\s+/) || line.match(/^\d+\.\s+/)) break;
|
|
22489
|
+
if (line.includes("|") && line.trim().startsWith("|")) break;
|
|
22490
|
+
paragraphLines.push(line);
|
|
22491
|
+
endIndex++;
|
|
22492
|
+
}
|
|
22493
|
+
return {
|
|
22494
|
+
block: { p: paragraphLines.join(" ").trim() },
|
|
22495
|
+
endIndex: endIndex - 1
|
|
22496
|
+
};
|
|
22497
|
+
}
|
|
22498
|
+
};
|
|
22499
|
+
|
|
22500
|
+
// src/toonify-helpers.ts
|
|
22501
|
+
var TOONIFY_MAX_INPUT_BYTES = 10 * 1024 * 1024;
|
|
22502
|
+
function formatSingleStats(result) {
|
|
22503
|
+
return `Toonify: ${result.originalTokenEstimate} \u2192 ${result.compressedTokenEstimate} tokens (${result.reductionPercent}% reduction)
|
|
22504
|
+
`;
|
|
22505
|
+
}
|
|
22506
|
+
function formatFileStats(filename, result) {
|
|
22507
|
+
return `Toonify [${filename}]: ${result.originalTokenEstimate} \u2192 ${result.compressedTokenEstimate} tokens (${result.reductionPercent}% reduction)
|
|
22508
|
+
`;
|
|
22509
|
+
}
|
|
22510
|
+
function formatAggregateStats(totalOriginal, totalCompressed, fileCount) {
|
|
22511
|
+
const aggregateReduction = totalOriginal > 0 ? Math.max(0, Math.min(100, Math.round((totalOriginal - totalCompressed) / totalOriginal * 100))) : 0;
|
|
22512
|
+
return `Toonify total: ${totalOriginal} \u2192 ${totalCompressed} tokens (${aggregateReduction}% reduction, ${fileCount} files)
|
|
22513
|
+
`;
|
|
22514
|
+
}
|
|
22515
|
+
|
|
22111
22516
|
// src/commands/convert.ts
|
|
22112
22517
|
var VALID_FORMATS = [
|
|
22113
22518
|
"github",
|
|
@@ -22258,7 +22663,44 @@ async function urlToMarkdown(url, apiKey, format, proxyBaseUrl = process.env.FOR
|
|
|
22258
22663
|
});
|
|
22259
22664
|
return result.markdown;
|
|
22260
22665
|
}
|
|
22666
|
+
function applyToonify(markdown) {
|
|
22667
|
+
if (Buffer.byteLength(markdown, "utf8") > TOONIFY_MAX_INPUT_BYTES) {
|
|
22668
|
+
process.stderr.write(
|
|
22669
|
+
`Error: Input too large for toonify (>${TOONIFY_MAX_INPUT_BYTES / 1024 / 1024} MB). Convert without --toonify or split the input.
|
|
22670
|
+
`
|
|
22671
|
+
);
|
|
22672
|
+
process.exit(1);
|
|
22673
|
+
}
|
|
22674
|
+
try {
|
|
22675
|
+
const result = ToonifyService.compress(markdown);
|
|
22676
|
+
process.stderr.write(formatSingleStats(result));
|
|
22677
|
+
return result.compressed;
|
|
22678
|
+
} catch (err) {
|
|
22679
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
22680
|
+
process.stderr.write(`Error: Toonify compression failed: ${message}
|
|
22681
|
+
`);
|
|
22682
|
+
process.exit(1);
|
|
22683
|
+
}
|
|
22684
|
+
}
|
|
22261
22685
|
async function convertCommand(options) {
|
|
22686
|
+
const parsedTimeout = options.timeout ? parseInt(options.timeout, 10) : void 0;
|
|
22687
|
+
if (parsedTimeout !== void 0 && (!Number.isFinite(parsedTimeout) || parsedTimeout <= 0)) {
|
|
22688
|
+
process.stderr.write("Error: --timeout must be a positive integer (milliseconds).\n");
|
|
22689
|
+
process.exit(1);
|
|
22690
|
+
}
|
|
22691
|
+
const entitlementOpts = {
|
|
22692
|
+
offline: options.offline,
|
|
22693
|
+
timeoutMs: parsedTimeout
|
|
22694
|
+
};
|
|
22695
|
+
if (options.toonify) {
|
|
22696
|
+
const provider = await resolveEntitlement(entitlementOpts);
|
|
22697
|
+
if (!provider.hasFeature("token-optimization")) {
|
|
22698
|
+
process.stderr.write(
|
|
22699
|
+
"Error: Token optimization requires a Premium license.\nRun `formatferry auth --license-key <KEY>` to activate your license.\n"
|
|
22700
|
+
);
|
|
22701
|
+
process.exit(1);
|
|
22702
|
+
}
|
|
22703
|
+
}
|
|
22262
22704
|
const format = validateFormat(options.format ?? "github");
|
|
22263
22705
|
if (options.url) {
|
|
22264
22706
|
if (options.input) {
|
|
@@ -22273,7 +22715,14 @@ async function convertCommand(options) {
|
|
|
22273
22715
|
);
|
|
22274
22716
|
process.exit(1);
|
|
22275
22717
|
}
|
|
22276
|
-
|
|
22718
|
+
let markdown2 = await urlToMarkdown(options.url, apiKey, format);
|
|
22719
|
+
if (options.toonify) {
|
|
22720
|
+
if (markdown2.trim()) {
|
|
22721
|
+
markdown2 = applyToonify(markdown2);
|
|
22722
|
+
} else {
|
|
22723
|
+
markdown2 = "";
|
|
22724
|
+
}
|
|
22725
|
+
}
|
|
22277
22726
|
if (options.output) {
|
|
22278
22727
|
writeFileSync2(options.output, markdown2, "utf8");
|
|
22279
22728
|
} else {
|
|
@@ -22296,7 +22745,7 @@ async function convertCommand(options) {
|
|
|
22296
22745
|
const html = fileBuffer.toString("utf8");
|
|
22297
22746
|
let applyAtlassian = false;
|
|
22298
22747
|
if (isAtlassianHTML(html)) {
|
|
22299
|
-
const provider = await resolveEntitlement();
|
|
22748
|
+
const provider = await resolveEntitlement(entitlementOpts);
|
|
22300
22749
|
if (!provider.hasFeature("atlassian")) {
|
|
22301
22750
|
process.stderr.write(
|
|
22302
22751
|
"Error: Atlassian HTML detected. A Premium license is required to process Atlassian content.\nRun `formatferry auth --license-key <KEY>` to activate your license.\n"
|
|
@@ -22313,7 +22762,7 @@ async function convertCommand(options) {
|
|
|
22313
22762
|
const html = await readStdin();
|
|
22314
22763
|
let applyAtlassian = false;
|
|
22315
22764
|
if (isAtlassianHTML(html)) {
|
|
22316
|
-
const provider = await resolveEntitlement();
|
|
22765
|
+
const provider = await resolveEntitlement(entitlementOpts);
|
|
22317
22766
|
if (!provider.hasFeature("atlassian")) {
|
|
22318
22767
|
process.stderr.write(
|
|
22319
22768
|
"Error: Atlassian HTML detected. A Premium license is required to process Atlassian content.\nRun `formatferry auth --license-key <KEY>` to activate your license.\n"
|
|
@@ -22324,6 +22773,13 @@ async function convertCommand(options) {
|
|
|
22324
22773
|
}
|
|
22325
22774
|
markdown = await htmlToMarkdown(html, format, applyAtlassian);
|
|
22326
22775
|
}
|
|
22776
|
+
if (options.toonify) {
|
|
22777
|
+
if (markdown.trim()) {
|
|
22778
|
+
markdown = applyToonify(markdown);
|
|
22779
|
+
} else {
|
|
22780
|
+
markdown = "";
|
|
22781
|
+
}
|
|
22782
|
+
}
|
|
22327
22783
|
if (options.output) {
|
|
22328
22784
|
writeFileSync2(options.output, markdown, "utf8");
|
|
22329
22785
|
} else {
|
|
@@ -22372,13 +22828,30 @@ async function convertFile(filePath, format) {
|
|
|
22372
22828
|
return c.markdown;
|
|
22373
22829
|
}
|
|
22374
22830
|
async function batchCommand(options) {
|
|
22375
|
-
const
|
|
22831
|
+
const parsedTimeout = options.timeout ? parseInt(options.timeout, 10) : void 0;
|
|
22832
|
+
if (parsedTimeout !== void 0 && (!Number.isFinite(parsedTimeout) || parsedTimeout <= 0)) {
|
|
22833
|
+
process.stderr.write("Error: --timeout must be a positive integer (milliseconds).\n");
|
|
22834
|
+
process.exit(1);
|
|
22835
|
+
}
|
|
22836
|
+
const entitlementOpts = {
|
|
22837
|
+
offline: options.offline,
|
|
22838
|
+
timeoutMs: parsedTimeout
|
|
22839
|
+
};
|
|
22840
|
+
const provider = await resolveEntitlement(entitlementOpts);
|
|
22376
22841
|
if (!provider.hasFeature("batch")) {
|
|
22377
22842
|
process.stderr.write(
|
|
22378
22843
|
"Error: Batch processing requires a Premium license.\nRun `formatferry auth --license-key <KEY>` to activate your license.\n"
|
|
22379
22844
|
);
|
|
22380
22845
|
process.exit(1);
|
|
22381
22846
|
}
|
|
22847
|
+
if (options.toonify) {
|
|
22848
|
+
if (!provider.hasFeature("token-optimization")) {
|
|
22849
|
+
process.stderr.write(
|
|
22850
|
+
"Error: Token optimization requires a Premium license.\nRun `formatferry auth --license-key <KEY>` to activate your license.\n"
|
|
22851
|
+
);
|
|
22852
|
+
process.exit(1);
|
|
22853
|
+
}
|
|
22854
|
+
}
|
|
22382
22855
|
const glob = options.batch;
|
|
22383
22856
|
if (!glob) {
|
|
22384
22857
|
process.stderr.write("Error: --batch <glob> is required\n");
|
|
@@ -22393,9 +22866,34 @@ async function batchCommand(options) {
|
|
|
22393
22866
|
const format = options.format ?? "github";
|
|
22394
22867
|
if (options.outputDir) mkdirSync2(options.outputDir, { recursive: true });
|
|
22395
22868
|
let successful = 0, failed = 0;
|
|
22869
|
+
let totalOriginal = 0, totalCompressed = 0, toonifiedCount = 0;
|
|
22396
22870
|
for (const filePath of files) {
|
|
22397
22871
|
try {
|
|
22398
|
-
|
|
22872
|
+
let markdown = await convertFile(filePath, format);
|
|
22873
|
+
if (options.toonify) {
|
|
22874
|
+
if (markdown.trim()) {
|
|
22875
|
+
if (Buffer.byteLength(markdown, "utf8") > TOONIFY_MAX_INPUT_BYTES) {
|
|
22876
|
+
process.stderr.write(`Skipping toonify for ${basename(filePath)}: input exceeds ${TOONIFY_MAX_INPUT_BYTES / 1024 / 1024} MB
|
|
22877
|
+
`);
|
|
22878
|
+
} else {
|
|
22879
|
+
try {
|
|
22880
|
+
const result = ToonifyService.compress(markdown);
|
|
22881
|
+
process.stderr.write(formatFileStats(basename(filePath), result));
|
|
22882
|
+
markdown = result.compressed;
|
|
22883
|
+
totalOriginal += result.originalTokenEstimate;
|
|
22884
|
+
totalCompressed += result.compressedTokenEstimate;
|
|
22885
|
+
toonifiedCount++;
|
|
22886
|
+
} catch (err) {
|
|
22887
|
+
process.stderr.write(`Error toonifying ${filePath}: ${err instanceof Error ? err.message : String(err)}
|
|
22888
|
+
`);
|
|
22889
|
+
failed++;
|
|
22890
|
+
continue;
|
|
22891
|
+
}
|
|
22892
|
+
}
|
|
22893
|
+
} else {
|
|
22894
|
+
markdown = "";
|
|
22895
|
+
}
|
|
22896
|
+
}
|
|
22399
22897
|
writeFileSync3(getOutputPath(filePath, options.outputDir), markdown, "utf8");
|
|
22400
22898
|
successful++;
|
|
22401
22899
|
} catch (err) {
|
|
@@ -22404,6 +22902,9 @@ async function batchCommand(options) {
|
|
|
22404
22902
|
failed++;
|
|
22405
22903
|
}
|
|
22406
22904
|
}
|
|
22905
|
+
if (options.toonify && toonifiedCount > 0) {
|
|
22906
|
+
process.stderr.write(formatAggregateStats(totalOriginal, totalCompressed, toonifiedCount));
|
|
22907
|
+
}
|
|
22407
22908
|
process.stderr.write(`
|
|
22408
22909
|
Batch complete: ${successful} succeeded, ${failed} failed, ${files.length} total
|
|
22409
22910
|
`);
|
|
@@ -22601,7 +23102,7 @@ function handleLogout() {
|
|
|
22601
23102
|
|
|
22602
23103
|
// src/index.ts
|
|
22603
23104
|
var VERSION = "0.1.0";
|
|
22604
|
-
program.name("formatferry").version(VERSION).description("Convert HTML, DOCX, PDF, XLSX, CSV to Markdown").option("-i, --input <path>", "Input file path").option("-o, --output <path>", "Output file path").option("-f, --format <format>", "Markdown format (github, commonmark, slack, discord, reddit, confluence, custom, rmarkdown)", "github").option("--url <url>", "Fetch article from URL").option("--batch <glob>", "Batch convert files (Premium)").option("--output-dir <dir>", "Output directory for batch").action(async (options) => {
|
|
23105
|
+
program.name("formatferry").version(VERSION).description("Convert HTML, DOCX, PDF, XLSX, CSV to Markdown").option("-i, --input <path>", "Input file path").option("-o, --output <path>", "Output file path").option("-f, --format <format>", "Markdown format (github, commonmark, slack, discord, reddit, confluence, custom, rmarkdown)", "github").option("--url <url>", "Fetch article from URL").option("--batch <glob>", "Batch convert files (Premium)").option("--output-dir <dir>", "Output directory for batch").option("--toonify", "Compress output to TOON format (Premium)").option("--offline", "Skip entitlement check (use cached or free tier)").option("--timeout <ms>", "License server timeout in milliseconds (default: 2000)", "2000").action(async (options) => {
|
|
22605
23106
|
if (options.batch) {
|
|
22606
23107
|
await batchCommand(options);
|
|
22607
23108
|
} else {
|