@nexlibris/crosspoint-tools 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +24 -2
- package/convert.d.ts +6 -0
- package/convert.js +22 -0
- package/fonts.d.ts +29 -0
- package/fonts.js +54 -0
- package/formats.d.ts +22 -0
- package/formats.js +19 -0
- package/libraryIndex.d.ts +72 -0
- package/libraryIndex.js +191 -0
- package/package.json +21 -4
- package/sleepImage.d.ts +42 -0
- package/sleepImage.js +102 -0
- package/stardict.d.ts +41 -0
- package/stardict.js +172 -0
- package/xteink/image.d.ts +54 -0
- package/xteink/image.js +202 -0
- package/xteink/optimize.d.ts +33 -0
- package/xteink/optimize.js +157 -0
- package/xteink/options.d.ts +29 -0
- package/xteink/options.js +108 -0
- package/xteink/textsplit.d.ts +71 -0
- package/xteink/textsplit.js +431 -0
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/** Optimizer options, apart from the optimizer so the convert dialogs can offer them without loading it. */
|
|
2
|
+
import type { OptionField } from '@nexlibris/tools/options';
|
|
3
|
+
export interface ImageOptions {
|
|
4
|
+
/** JPEG quality, 1–100. */
|
|
5
|
+
quality: number;
|
|
6
|
+
grayscale: boolean;
|
|
7
|
+
/** Trim uniform margins; off by default, since sparse title pages can be zoomed a lot. */
|
|
8
|
+
autoCrop: boolean;
|
|
9
|
+
}
|
|
10
|
+
export interface OptimizeOptions extends ImageOptions {
|
|
11
|
+
/** Split oversized paragraphs, for the firmware's layout memory. */
|
|
12
|
+
splitParagraphs: boolean;
|
|
13
|
+
/**
|
|
14
|
+
* Split long spine files into ~7 KB ones. Off by default: the firmware now streams and indexes long files
|
|
15
|
+
* incrementally, and each piece shows as its own chapter.
|
|
16
|
+
*/
|
|
17
|
+
splitChapters: boolean;
|
|
18
|
+
}
|
|
19
|
+
export declare const DEFAULT_IMAGE_OPTIONS: ImageOptions;
|
|
20
|
+
export declare const DEFAULT_OPTIMIZE_OPTIONS: OptimizeOptions;
|
|
21
|
+
/** The options as asked for when converting, keyed as in `OptimizeOptions`. */
|
|
22
|
+
export declare const OPTIMIZE_FIELDS: OptionField[];
|
|
23
|
+
/** Options from the convert dialog's values (`OPTIMIZE_FIELDS`), with the defaults for any missing or out of range. */
|
|
24
|
+
export declare function optimizeOptionsFrom(values: Record<string, string>): OptimizeOptions;
|
|
25
|
+
/**
|
|
26
|
+
* The converter version for these options. Options are inputs to the output, so a copy made with other options
|
|
27
|
+
* reads as stale and is made again. The defaults give the plain version, which copies made before options existed have.
|
|
28
|
+
*/
|
|
29
|
+
export declare function optimizerVersion(options: OptimizeOptions): string;
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
//#region crosspoint-tools/src/xteink/options.ts
|
|
2
|
+
const DEFAULT_IMAGE_OPTIONS = {
|
|
3
|
+
quality: 85,
|
|
4
|
+
grayscale: true,
|
|
5
|
+
autoCrop: false
|
|
6
|
+
};
|
|
7
|
+
const DEFAULT_OPTIMIZE_OPTIONS = {
|
|
8
|
+
...DEFAULT_IMAGE_OPTIONS,
|
|
9
|
+
splitParagraphs: true,
|
|
10
|
+
splitChapters: false
|
|
11
|
+
};
|
|
12
|
+
/** The options as asked for when converting, keyed as in `OptimizeOptions`. */
|
|
13
|
+
const OPTIMIZE_FIELDS = [
|
|
14
|
+
{
|
|
15
|
+
key: "quality",
|
|
16
|
+
label: "Image quality",
|
|
17
|
+
default: String(DEFAULT_OPTIMIZE_OPTIONS.quality),
|
|
18
|
+
range: {
|
|
19
|
+
min: 1,
|
|
20
|
+
max: 100
|
|
21
|
+
},
|
|
22
|
+
help: "JPEG quality of images in the optimized copy. Lower is smaller and faster to open."
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
key: "grayscale",
|
|
26
|
+
label: "Images",
|
|
27
|
+
default: "yes",
|
|
28
|
+
choices: [{
|
|
29
|
+
value: "yes",
|
|
30
|
+
label: "Grayscale"
|
|
31
|
+
}, {
|
|
32
|
+
value: "no",
|
|
33
|
+
label: "Keep colour"
|
|
34
|
+
}],
|
|
35
|
+
help: "The screen is grayscale; colour images are larger for no gain on it."
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
key: "autoCrop",
|
|
39
|
+
label: "Crop image margins",
|
|
40
|
+
default: "no",
|
|
41
|
+
choices: [{
|
|
42
|
+
value: "no",
|
|
43
|
+
label: "No"
|
|
44
|
+
}, {
|
|
45
|
+
value: "yes",
|
|
46
|
+
label: "Yes"
|
|
47
|
+
}],
|
|
48
|
+
help: "Trims plain margins so illustrations fill the screen. Sparse pages can end up zoomed a lot."
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
key: "splitParagraphs",
|
|
52
|
+
label: "Split long paragraphs",
|
|
53
|
+
default: "yes",
|
|
54
|
+
choices: [{
|
|
55
|
+
value: "yes",
|
|
56
|
+
label: "Yes"
|
|
57
|
+
}, {
|
|
58
|
+
value: "no",
|
|
59
|
+
label: "No"
|
|
60
|
+
}],
|
|
61
|
+
help: "Splits very long paragraphs so the reader has the memory to lay them out."
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
key: "splitChapters",
|
|
65
|
+
label: "Split long chapters",
|
|
66
|
+
default: "no",
|
|
67
|
+
choices: [{
|
|
68
|
+
value: "no",
|
|
69
|
+
label: "No"
|
|
70
|
+
}, {
|
|
71
|
+
value: "yes",
|
|
72
|
+
label: "Yes"
|
|
73
|
+
}],
|
|
74
|
+
help: "Splits chapters into ~7 KB files, each shown as its own chapter. Only older firmware needs it."
|
|
75
|
+
}
|
|
76
|
+
];
|
|
77
|
+
/** Options from the convert dialog's values (`OPTIMIZE_FIELDS`), with the defaults for any missing or out of range. */
|
|
78
|
+
function optimizeOptionsFrom(values) {
|
|
79
|
+
const d = DEFAULT_OPTIMIZE_OPTIONS;
|
|
80
|
+
const quality = Math.round(Number(values.quality));
|
|
81
|
+
const flag = (key, fallback) => values[key] === "yes" ? true : values[key] === "no" ? false : fallback;
|
|
82
|
+
return {
|
|
83
|
+
quality: quality >= 1 && quality <= 100 ? quality : d.quality,
|
|
84
|
+
grayscale: flag("grayscale", d.grayscale),
|
|
85
|
+
autoCrop: flag("autoCrop", d.autoCrop),
|
|
86
|
+
splitParagraphs: flag("splitParagraphs", d.splitParagraphs),
|
|
87
|
+
splitChapters: flag("splitChapters", d.splitChapters)
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
/** The optimizer's own version: bump it when its output changes. */
|
|
91
|
+
const OPTIMIZER_VERSION = "2";
|
|
92
|
+
/**
|
|
93
|
+
* The converter version for these options. Options are inputs to the output, so a copy made with other options
|
|
94
|
+
* reads as stale and is made again. The defaults give the plain version, which copies made before options existed have.
|
|
95
|
+
*/
|
|
96
|
+
function optimizerVersion(options) {
|
|
97
|
+
const d = DEFAULT_OPTIMIZE_OPTIONS;
|
|
98
|
+
const parts = [
|
|
99
|
+
options.quality !== d.quality && `q${options.quality}`,
|
|
100
|
+
options.grayscale !== d.grayscale && (options.grayscale ? "gray" : "color"),
|
|
101
|
+
options.autoCrop !== d.autoCrop && (options.autoCrop ? "crop" : "nocrop"),
|
|
102
|
+
options.splitParagraphs !== d.splitParagraphs && (options.splitParagraphs ? "paras" : "nosplit"),
|
|
103
|
+
options.splitChapters !== d.splitChapters && (options.splitChapters ? "chapters" : "nochapters")
|
|
104
|
+
].filter(Boolean);
|
|
105
|
+
return [OPTIMIZER_VERSION, ...parts].join("+");
|
|
106
|
+
}
|
|
107
|
+
//#endregion
|
|
108
|
+
export { DEFAULT_IMAGE_OPTIONS, DEFAULT_OPTIMIZE_OPTIONS, OPTIMIZE_FIELDS, optimizeOptionsFrom, optimizerVersion };
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Text-side EPUB transformations for the low-RAM CrossPoint firmware, ported
|
|
3
|
+
* from `crosspoint_reader/textsplit.py` in the CrossPoint Calibre plugins (MIT).
|
|
4
|
+
*
|
|
5
|
+
* The device lays out one <p> at a time, holding every word in RAM, and caches
|
|
6
|
+
* whole spine files. On ~380 KB of RAM a multi-KB paragraph (~6 KB crashed an
|
|
7
|
+
* X4; ~1.2 KB is comfortable) runs it out of memory, and spine files beyond
|
|
8
|
+
* ~10 KB make indexing fragile. So this:
|
|
9
|
+
*
|
|
10
|
+
* - splits every <p> larger than PARA_LIMIT into ~PARA_TARGET-byte siblings, at
|
|
11
|
+
* sentence boundaries (inline tags kept whole), unless turned off,
|
|
12
|
+
* - if asked, splits spine files whose <body> exceeds SPLIT_LIMIT into
|
|
13
|
+
* ~CHUNK_TARGET-byte files, adding them to the manifest and spine (firmware
|
|
14
|
+
* that streams and indexes sections incrementally doesn't need this),
|
|
15
|
+
* - points `href="…#fragment"` links at the chunk that now holds the anchor,
|
|
16
|
+
* - moves base64 `data:` images into real files,
|
|
17
|
+
* - removes embedded fonts and page-list navs.
|
|
18
|
+
*
|
|
19
|
+
* Each transformation checks that the visible text is unchanged and is skipped
|
|
20
|
+
* if not, so this never corrupts a book. Sizes are in UTF-16 code units, close
|
|
21
|
+
* enough to the Python's characters.
|
|
22
|
+
*/
|
|
23
|
+
export declare const SPLIT_LIMIT = 9500;
|
|
24
|
+
export declare const CHUNK_TARGET = 7000;
|
|
25
|
+
export declare const PARA_LIMIT = 1600;
|
|
26
|
+
export declare const PARA_TARGET = 1200;
|
|
27
|
+
/** Splits markup into its top-level balanced nodes: elements (with their contents) and text runs. */
|
|
28
|
+
export declare function parseNodes(s: string): string[];
|
|
29
|
+
/**
|
|
30
|
+
* Splits a text run into pieces of ~target at sentence (or else word) boundaries.
|
|
31
|
+
* Lossless: boundary whitespace stays with the piece before it, so the pieces join back to `text`.
|
|
32
|
+
*/
|
|
33
|
+
export declare function splitTextSentences(text: string, target: number): string[];
|
|
34
|
+
/** Rewrites each <p> bigger than PARA_LIMIT as several sibling <p>s. A <p> holding block-level tags is left alone. */
|
|
35
|
+
export declare function splitBigParagraphs(html: string): {
|
|
36
|
+
html: string;
|
|
37
|
+
changed: boolean;
|
|
38
|
+
};
|
|
39
|
+
/** An oversized document as `[name, content]` chunks, the first keeping its name; [] if it needn't or can't split. */
|
|
40
|
+
export declare function splitXhtmlDoc(doc: string, basename: string): [string, string][];
|
|
41
|
+
/** The text a reader shows, with whitespace collapsed, for checking that a transformation kept it. */
|
|
42
|
+
export declare function visibleText(s: string): string;
|
|
43
|
+
/** An EPUB as entries in zip order, with text entries decoded, for the passes to rewrite. */
|
|
44
|
+
export interface EpubEntries {
|
|
45
|
+
/** Zip paths in the order they're written; `mimetype` goes first regardless. */
|
|
46
|
+
order: string[];
|
|
47
|
+
data: Map<string, Uint8Array | string>;
|
|
48
|
+
opfPath: string;
|
|
49
|
+
}
|
|
50
|
+
export interface TextSplitSummary {
|
|
51
|
+
paragraphs: number;
|
|
52
|
+
fileSplits: number;
|
|
53
|
+
fonts: number;
|
|
54
|
+
dataImages: number;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Runs the text passes over an EPUB's entries in place. `processImage` re-encodes images pulled out of `data:`
|
|
58
|
+
* URIs (as JPEG); without it they're kept as they are. `log` hears about each step and anything skipped.
|
|
59
|
+
* `split` picks the splitting passes; the cleanup passes always run.
|
|
60
|
+
*/
|
|
61
|
+
export declare function splitEpubText(entries: EpubEntries, processImage?: (data: Uint8Array, name: string) => Promise<Uint8Array>, log?: (message: string) => void, split?: {
|
|
62
|
+
paragraphs: boolean;
|
|
63
|
+
chapters: boolean;
|
|
64
|
+
}): Promise<TextSplitSummary>;
|
|
65
|
+
export declare const dirname: (path: string) => string;
|
|
66
|
+
export declare const basename: (path: string) => string;
|
|
67
|
+
export declare const join: (dir: string, name: string) => string;
|
|
68
|
+
/** Normalizes `.` and `..` segments. */
|
|
69
|
+
export declare function normalize(path: string): string;
|
|
70
|
+
/** A manifest href as a zip path. Hrefs are URLs, so they may be percent-encoded. */
|
|
71
|
+
export declare function resolveHref(opfDir: string, href: string): string;
|
|
@@ -0,0 +1,431 @@
|
|
|
1
|
+
//#region crosspoint-tools/src/xteink/textsplit.ts
|
|
2
|
+
/**
|
|
3
|
+
* Text-side EPUB transformations for the low-RAM CrossPoint firmware, ported
|
|
4
|
+
* from `crosspoint_reader/textsplit.py` in the CrossPoint Calibre plugins (MIT).
|
|
5
|
+
*
|
|
6
|
+
* The device lays out one <p> at a time, holding every word in RAM, and caches
|
|
7
|
+
* whole spine files. On ~380 KB of RAM a multi-KB paragraph (~6 KB crashed an
|
|
8
|
+
* X4; ~1.2 KB is comfortable) runs it out of memory, and spine files beyond
|
|
9
|
+
* ~10 KB make indexing fragile. So this:
|
|
10
|
+
*
|
|
11
|
+
* - splits every <p> larger than PARA_LIMIT into ~PARA_TARGET-byte siblings, at
|
|
12
|
+
* sentence boundaries (inline tags kept whole), unless turned off,
|
|
13
|
+
* - if asked, splits spine files whose <body> exceeds SPLIT_LIMIT into
|
|
14
|
+
* ~CHUNK_TARGET-byte files, adding them to the manifest and spine (firmware
|
|
15
|
+
* that streams and indexes sections incrementally doesn't need this),
|
|
16
|
+
* - points `href="…#fragment"` links at the chunk that now holds the anchor,
|
|
17
|
+
* - moves base64 `data:` images into real files,
|
|
18
|
+
* - removes embedded fonts and page-list navs.
|
|
19
|
+
*
|
|
20
|
+
* Each transformation checks that the visible text is unchanged and is skipped
|
|
21
|
+
* if not, so this never corrupts a book. Sizes are in UTF-16 code units, close
|
|
22
|
+
* enough to the Python's characters.
|
|
23
|
+
*/
|
|
24
|
+
const SPLIT_LIMIT = 9500;
|
|
25
|
+
const CHUNK_TARGET = 7e3;
|
|
26
|
+
const PARA_LIMIT = 1600;
|
|
27
|
+
const PARA_TARGET = 1200;
|
|
28
|
+
const VOID = [
|
|
29
|
+
"meta",
|
|
30
|
+
"link",
|
|
31
|
+
"img",
|
|
32
|
+
"br",
|
|
33
|
+
"hr",
|
|
34
|
+
"image",
|
|
35
|
+
"input",
|
|
36
|
+
"col",
|
|
37
|
+
"source"
|
|
38
|
+
];
|
|
39
|
+
const FONT_RE = /\.(otf|ttf|woff2?)$/i;
|
|
40
|
+
const XHTML_RE = /\.(xhtml|html|htm)$/i;
|
|
41
|
+
/** Block-level content makes a <p> unsplittable. */
|
|
42
|
+
const BLOCK_RE = /<(?:p|div|table|ul|ol)\b/;
|
|
43
|
+
/** Inline wrappers that may be split into several same-tag siblings; some books wrap whole paragraphs in one. */
|
|
44
|
+
const INLINE_WRAP = [
|
|
45
|
+
"span",
|
|
46
|
+
"em",
|
|
47
|
+
"i",
|
|
48
|
+
"b",
|
|
49
|
+
"strong",
|
|
50
|
+
"a",
|
|
51
|
+
"u",
|
|
52
|
+
"small",
|
|
53
|
+
"sub",
|
|
54
|
+
"sup"
|
|
55
|
+
];
|
|
56
|
+
const ELEMENT_RE = /^(<(\w+)[^>]*>)([\s\S]*)(<\/\2>)$/;
|
|
57
|
+
/** Splits markup into its top-level balanced nodes: elements (with their contents) and text runs. */
|
|
58
|
+
function parseNodes(s) {
|
|
59
|
+
const nodes = [];
|
|
60
|
+
let i = 0;
|
|
61
|
+
while (i < s.length) {
|
|
62
|
+
if (s[i] !== "<") {
|
|
63
|
+
const j = s.indexOf("<", i);
|
|
64
|
+
const end = j === -1 ? s.length : j;
|
|
65
|
+
nodes.push(s.slice(i, end));
|
|
66
|
+
i = end;
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
let depth = 0;
|
|
70
|
+
let j = i;
|
|
71
|
+
let k;
|
|
72
|
+
for (;;) {
|
|
73
|
+
k = s.indexOf(">", j);
|
|
74
|
+
if (k === -1) throw new Error("unterminated tag");
|
|
75
|
+
const tag = s.slice(j, k + 1);
|
|
76
|
+
const name = /^<\/?(\w+)/.exec(tag)?.[1]?.toLowerCase() ?? "";
|
|
77
|
+
if (tag.startsWith("<!--")) {
|
|
78
|
+
k = s.indexOf("-->", j);
|
|
79
|
+
if (k === -1) throw new Error("unterminated comment");
|
|
80
|
+
k += 2;
|
|
81
|
+
} else if (tag.startsWith("</")) depth--;
|
|
82
|
+
else if (!(tag.endsWith("/>") || VOID.includes(name) || tag.startsWith("<!") || tag.startsWith("<?"))) depth++;
|
|
83
|
+
if (depth <= 0) break;
|
|
84
|
+
j = s.indexOf("<", k);
|
|
85
|
+
if (j === -1) throw new Error("unbalanced markup");
|
|
86
|
+
}
|
|
87
|
+
nodes.push(s.slice(i, k + 1));
|
|
88
|
+
i = k + 1;
|
|
89
|
+
}
|
|
90
|
+
return nodes;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Splits a text run into pieces of ~target at sentence (or else word) boundaries.
|
|
94
|
+
* Lossless: boundary whitespace stays with the piece before it, so the pieces join back to `text`.
|
|
95
|
+
*/
|
|
96
|
+
function splitTextSentences(text, target) {
|
|
97
|
+
const pieces = [];
|
|
98
|
+
while (text.length > target) {
|
|
99
|
+
let cut = text.lastIndexOf(". ", target - 2);
|
|
100
|
+
if (cut !== -1) cut += 2;
|
|
101
|
+
else {
|
|
102
|
+
cut = text.lastIndexOf(" ", target - 1) + 1;
|
|
103
|
+
if (cut === 0) break;
|
|
104
|
+
}
|
|
105
|
+
pieces.push(text.slice(0, cut));
|
|
106
|
+
text = text.slice(cut);
|
|
107
|
+
}
|
|
108
|
+
pieces.push(text);
|
|
109
|
+
return pieces;
|
|
110
|
+
}
|
|
111
|
+
/** Greedily packs pieces into groups of about `target`, dropping groups that are only whitespace. */
|
|
112
|
+
function pack(pieces, target) {
|
|
113
|
+
const groups = [];
|
|
114
|
+
let current = [];
|
|
115
|
+
let length = 0;
|
|
116
|
+
const flush = () => {
|
|
117
|
+
if (current.join("").trim()) groups.push(current.join(""));
|
|
118
|
+
current = [];
|
|
119
|
+
length = 0;
|
|
120
|
+
};
|
|
121
|
+
for (const piece of pieces) {
|
|
122
|
+
if (length + piece.length > target && length > 0 && piece.trim()) flush();
|
|
123
|
+
current.push(piece);
|
|
124
|
+
length += piece.length;
|
|
125
|
+
}
|
|
126
|
+
flush();
|
|
127
|
+
return groups;
|
|
128
|
+
}
|
|
129
|
+
/** Packs inline content into ~PARA_TARGET chunks, splitting long text and long inline wrappers. Lossless. */
|
|
130
|
+
function groupInline(inner) {
|
|
131
|
+
return pack(parseNodes(inner).flatMap((token) => {
|
|
132
|
+
if (token.length <= 1600) return [token];
|
|
133
|
+
if (!token.startsWith("<")) return splitTextSentences(token, PARA_TARGET);
|
|
134
|
+
const m = ELEMENT_RE.exec(token);
|
|
135
|
+
if (m && INLINE_WRAP.includes(m[2].toLowerCase()) && !BLOCK_RE.test(m[3])) return groupInline(m[3]).map((g) => m[1] + g + m[4]);
|
|
136
|
+
return [token];
|
|
137
|
+
}), PARA_TARGET);
|
|
138
|
+
}
|
|
139
|
+
/** Rewrites each <p> bigger than PARA_LIMIT as several sibling <p>s. A <p> holding block-level tags is left alone. */
|
|
140
|
+
function splitBigParagraphs(html) {
|
|
141
|
+
const out = [];
|
|
142
|
+
let changed = false;
|
|
143
|
+
for (const node of parseNodes(html)) {
|
|
144
|
+
const m = node.startsWith("<p") ? /^(<p\b[^>]*>)([\s\S]*)(<\/p>)$/.exec(node) : null;
|
|
145
|
+
if (!m || m[2].length <= 1600 || BLOCK_RE.test(m[2])) {
|
|
146
|
+
if (node.startsWith("<") && !/^<(p|!|\/)/.test(node) && node.length > 1600) {
|
|
147
|
+
const mm = ELEMENT_RE.exec(node);
|
|
148
|
+
if (mm && !VOID.includes(mm[2])) {
|
|
149
|
+
const inner = splitBigParagraphs(mm[3]);
|
|
150
|
+
if (inner.changed) {
|
|
151
|
+
out.push(mm[1] + inner.html + mm[4]);
|
|
152
|
+
changed = true;
|
|
153
|
+
continue;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
out.push(node);
|
|
158
|
+
continue;
|
|
159
|
+
}
|
|
160
|
+
const [, open, inner, close] = m;
|
|
161
|
+
const groups = groupInline(inner);
|
|
162
|
+
if (groups.length < 2) {
|
|
163
|
+
out.push(node);
|
|
164
|
+
continue;
|
|
165
|
+
}
|
|
166
|
+
const continuation = open.replace(/\s+id="[^"]*"/, "");
|
|
167
|
+
out.push(groups.map((g, k) => (k === 0 ? open : continuation) + g + close).join(""));
|
|
168
|
+
changed = true;
|
|
169
|
+
}
|
|
170
|
+
return {
|
|
171
|
+
html: out.join(""),
|
|
172
|
+
changed
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
/** Packs nodes into chunks of about `target`, recursing into containers too big for one. */
|
|
176
|
+
function chunkNodes(nodes, target) {
|
|
177
|
+
const chunks = [];
|
|
178
|
+
let current = [];
|
|
179
|
+
let length = 0;
|
|
180
|
+
const flush = () => {
|
|
181
|
+
if (current.join("").trim()) chunks.push(current.join(""));
|
|
182
|
+
current = [];
|
|
183
|
+
length = 0;
|
|
184
|
+
};
|
|
185
|
+
for (const node of nodes) if (node.length > 9500 && node.startsWith("<")) {
|
|
186
|
+
const m = /^<(\w+)[^>]*>/.exec(node);
|
|
187
|
+
const open = m[0];
|
|
188
|
+
const close = `</${m[1]}>`;
|
|
189
|
+
flush();
|
|
190
|
+
for (const sub of chunkNodes(parseNodes(node.slice(open.length, -close.length)), target)) chunks.push(open + sub + close);
|
|
191
|
+
} else if (node.length > 9500) {
|
|
192
|
+
flush();
|
|
193
|
+
chunks.push(...splitTextSentences(node, target));
|
|
194
|
+
} else {
|
|
195
|
+
if (length + node.length > target && length > 0 && node.trim()) flush();
|
|
196
|
+
current.push(node);
|
|
197
|
+
length += node.length;
|
|
198
|
+
}
|
|
199
|
+
flush();
|
|
200
|
+
return chunks;
|
|
201
|
+
}
|
|
202
|
+
/** The body's extent in an (X)HTML document: its last `</body>`, as the Python's greedy match finds it. */
|
|
203
|
+
function bodyOf(doc) {
|
|
204
|
+
const m = /(<body[^>]*>)([\s\S]*)(<\/body>)/.exec(doc);
|
|
205
|
+
if (!m) return void 0;
|
|
206
|
+
const start = m.index + m[1].length;
|
|
207
|
+
return {
|
|
208
|
+
start,
|
|
209
|
+
end: start + m[2].length,
|
|
210
|
+
body: m[2]
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
/** An oversized document as `[name, content]` chunks, the first keeping its name; [] if it needn't or can't split. */
|
|
214
|
+
function splitXhtmlDoc(doc, basename) {
|
|
215
|
+
const body = bodyOf(doc);
|
|
216
|
+
if (!body || body.body.length <= 9500) return [];
|
|
217
|
+
const chunks = chunkNodes(parseNodes(body.body), CHUNK_TARGET);
|
|
218
|
+
if (chunks.length < 2) return [];
|
|
219
|
+
const dot = basename.lastIndexOf(".");
|
|
220
|
+
const [base, ext] = dot === -1 ? [basename, ""] : [basename.slice(0, dot), basename.slice(dot + 1)];
|
|
221
|
+
const head = doc.slice(0, body.start);
|
|
222
|
+
const tail = doc.slice(body.end);
|
|
223
|
+
return chunks.map((chunk, k) => [k === 0 ? basename : `${base}_ek${k}.${ext}`, head + chunk + tail]);
|
|
224
|
+
}
|
|
225
|
+
/** The text a reader shows, with whitespace collapsed, for checking that a transformation kept it. */
|
|
226
|
+
function visibleText(s) {
|
|
227
|
+
return (/<body.*?>([\s\S]*)<\/body>/.exec(s)?.[1] ?? s).replace(/<!--[\s\S]*?-->/g, "").replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim();
|
|
228
|
+
}
|
|
229
|
+
const decoder = new TextDecoder();
|
|
230
|
+
const text = (entries, path) => {
|
|
231
|
+
const value = entries.data.get(path);
|
|
232
|
+
return typeof value === "string" ? value : decoder.decode(value);
|
|
233
|
+
};
|
|
234
|
+
const attr = (tag, name) => new RegExp(`\\b${name}="([^"]*)"`).exec(tag)?.[1];
|
|
235
|
+
const escapeRegExp = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
236
|
+
/**
|
|
237
|
+
* Runs the text passes over an EPUB's entries in place. `processImage` re-encodes images pulled out of `data:`
|
|
238
|
+
* URIs (as JPEG); without it they're kept as they are. `log` hears about each step and anything skipped.
|
|
239
|
+
* `split` picks the splitting passes; the cleanup passes always run.
|
|
240
|
+
*/
|
|
241
|
+
async function splitEpubText(entries, processImage, log = () => void 0, split = {
|
|
242
|
+
paragraphs: true,
|
|
243
|
+
chapters: false
|
|
244
|
+
}) {
|
|
245
|
+
const result = {
|
|
246
|
+
paragraphs: 0,
|
|
247
|
+
fileSplits: 0,
|
|
248
|
+
fonts: 0,
|
|
249
|
+
dataImages: 0
|
|
250
|
+
};
|
|
251
|
+
const { opfPath } = entries;
|
|
252
|
+
let opf = text(entries, opfPath);
|
|
253
|
+
const opfDir = dirname(opfPath);
|
|
254
|
+
const items = /* @__PURE__ */ new Map();
|
|
255
|
+
for (const [tag] of opf.matchAll(/<(?:\w+:)?item\b[^>]*\/?>/g)) {
|
|
256
|
+
const id = attr(tag, "id");
|
|
257
|
+
const href = attr(tag, "href");
|
|
258
|
+
if (id && href) items.set(id, [href, tag]);
|
|
259
|
+
}
|
|
260
|
+
const spineIds = [...opf.matchAll(/<(?:\w+:)?itemref\b[^>]*\bidref="([^"]+)"/g)].map((m) => m[1]);
|
|
261
|
+
/** `<original basename>#<fragment>` to the basename of the chunk holding it now. */
|
|
262
|
+
const anchors = /* @__PURE__ */ new Map();
|
|
263
|
+
/** A chunk's zip path to the basename of the file it was split from. */
|
|
264
|
+
const chunkOrigin = /* @__PURE__ */ new Map();
|
|
265
|
+
for (const id of spineIds) {
|
|
266
|
+
const entry = items.get(id);
|
|
267
|
+
if (!entry) continue;
|
|
268
|
+
const [href, itemTag] = entry;
|
|
269
|
+
const path = resolveHref(opfDir, href);
|
|
270
|
+
if (!entries.data.has(path) || !XHTML_RE.test(path)) continue;
|
|
271
|
+
let doc = text(entries, path);
|
|
272
|
+
if (doc.includes("data:image/")) {
|
|
273
|
+
const extracted = await extractDataImages(entries, path, doc, opf, result, processImage, log);
|
|
274
|
+
doc = extracted.doc;
|
|
275
|
+
opf = extracted.opf;
|
|
276
|
+
entries.data.set(path, doc);
|
|
277
|
+
}
|
|
278
|
+
const original = visibleText(doc);
|
|
279
|
+
const body = split.paragraphs ? bodyOf(doc) : void 0;
|
|
280
|
+
if (body) {
|
|
281
|
+
let paragraphs = {
|
|
282
|
+
html: body.body,
|
|
283
|
+
changed: false
|
|
284
|
+
};
|
|
285
|
+
try {
|
|
286
|
+
paragraphs = splitBigParagraphs(body.body);
|
|
287
|
+
} catch (e) {
|
|
288
|
+
log(`${basename(path)}: paragraph pass skipped (${errorMessage(e)})`);
|
|
289
|
+
}
|
|
290
|
+
if (paragraphs.changed) {
|
|
291
|
+
const next = doc.slice(0, body.start) + paragraphs.html + doc.slice(body.end);
|
|
292
|
+
if (visibleText(next) === original) {
|
|
293
|
+
doc = next;
|
|
294
|
+
entries.data.set(path, doc);
|
|
295
|
+
result.paragraphs++;
|
|
296
|
+
} else log(`${basename(path)}: paragraph pass changed the text, reverted`);
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
if (!split.chapters) continue;
|
|
300
|
+
let pieces = [];
|
|
301
|
+
try {
|
|
302
|
+
pieces = splitXhtmlDoc(doc, basename(path));
|
|
303
|
+
} catch (e) {
|
|
304
|
+
log(`${basename(path)}: file split skipped (${errorMessage(e)})`);
|
|
305
|
+
}
|
|
306
|
+
if (!pieces.length) continue;
|
|
307
|
+
if (pieces.map(([, content]) => visibleText(content)).join(" ").replace(/\s+/g, " ").trim() !== original) {
|
|
308
|
+
log(`${basename(path)}: file split changed the text, reverted`);
|
|
309
|
+
continue;
|
|
310
|
+
}
|
|
311
|
+
const zipDir = dirname(path);
|
|
312
|
+
const hrefDir = dirname(href);
|
|
313
|
+
const origin = basename(href);
|
|
314
|
+
const insertAt = entries.order.indexOf(path);
|
|
315
|
+
pieces.forEach(([name, content], k) => {
|
|
316
|
+
const piecePath = join(zipDir, name);
|
|
317
|
+
entries.data.set(piecePath, content);
|
|
318
|
+
chunkOrigin.set(piecePath, origin);
|
|
319
|
+
if (k > 0) entries.order.splice(insertAt + k, 0, piecePath);
|
|
320
|
+
for (const [, fragment] of content.matchAll(/id="([^"]+)"/g)) anchors.set(`${origin}#${fragment}`, name);
|
|
321
|
+
});
|
|
322
|
+
const newItems = pieces.map(([name], k) => k === 0 ? itemTag : `<item id="${id}_ek${k}" href="${join(hrefDir, name)}" media-type="application/xhtml+xml"/>`);
|
|
323
|
+
const newRefs = pieces.map((_, k) => `<itemref idref="${k === 0 ? id : `${id}_ek${k}`}"/>`);
|
|
324
|
+
opf = opf.replace(itemTag, () => newItems.join("\n "));
|
|
325
|
+
opf = opf.replace(new RegExp(`<(?:\\w+:)?itemref\\b[^>]*\\bidref="${escapeRegExp(id)}"[^>]*\\/?>`), () => newRefs.join("\n "));
|
|
326
|
+
result.fileSplits++;
|
|
327
|
+
log(`${basename(path)} → ${pieces.length} files (max ${Math.max(...pieces.map(([, c]) => c.length))} B)`);
|
|
328
|
+
}
|
|
329
|
+
for (const path of [...entries.order]) {
|
|
330
|
+
if (!XHTML_RE.test(path) && !path.toLowerCase().endsWith(".ncx")) continue;
|
|
331
|
+
const before = text(entries, path);
|
|
332
|
+
let s = before;
|
|
333
|
+
if (anchors.size) {
|
|
334
|
+
s = s.replace(/((?:href|src)=")([^"#]+)#([^"]+)"/g, (_m, pre, file, fragment) => {
|
|
335
|
+
const chunk = anchors.get(`${basename(file)}#${fragment}`);
|
|
336
|
+
if (chunk && chunk !== basename(file)) file = join(dirname(file), chunk);
|
|
337
|
+
return `${pre}${file}#${fragment}"`;
|
|
338
|
+
});
|
|
339
|
+
const selfOrigin = chunkOrigin.get(path) ?? basename(path);
|
|
340
|
+
s = s.replace(/(href=")#([^"]+)"/g, (m, pre, fragment) => {
|
|
341
|
+
const chunk = anchors.get(`${selfOrigin}#${fragment}`);
|
|
342
|
+
return chunk && chunk !== basename(path) ? `${pre}${chunk}#${fragment}"` : m;
|
|
343
|
+
});
|
|
344
|
+
}
|
|
345
|
+
s = s.replace(/<nav[^>]*epub:type="page-list"[\s\S]*?<\/nav>/g, "");
|
|
346
|
+
if (s !== before) entries.data.set(path, s);
|
|
347
|
+
}
|
|
348
|
+
for (const path of [...entries.order]) if (FONT_RE.test(path)) {
|
|
349
|
+
entries.order.splice(entries.order.indexOf(path), 1);
|
|
350
|
+
entries.data.delete(path);
|
|
351
|
+
result.fonts++;
|
|
352
|
+
} else if (path.toLowerCase().endsWith(".css")) {
|
|
353
|
+
const css = text(entries, path);
|
|
354
|
+
const stripped = css.replace(/@font-face\s*\{[^}]*\}\s*/g, "");
|
|
355
|
+
if (stripped !== css) entries.data.set(path, stripped);
|
|
356
|
+
}
|
|
357
|
+
if (result.fonts) {
|
|
358
|
+
opf = opf.replace(/\s*<(?:\w+:)?item\b[^>]*\.(?:otf|ttf|woff2?)[^>]*\/?>/gi, "");
|
|
359
|
+
log(`removed ${result.fonts} embedded font(s)`);
|
|
360
|
+
}
|
|
361
|
+
entries.data.set(opfPath, opf);
|
|
362
|
+
if (result.paragraphs) log(`split oversized paragraphs in ${result.paragraphs} file(s)`);
|
|
363
|
+
return result;
|
|
364
|
+
}
|
|
365
|
+
const DATA_URI_RE = /(src|xlink:href)="data:image\/(png|jpe?g|gif|webp|bmp);base64,([^"]+)"/gi;
|
|
366
|
+
/** Moves base64 `data:` images in a spine file into zip entries with manifest items. */
|
|
367
|
+
async function extractDataImages(entries, path, doc, opf, result, processImage, log) {
|
|
368
|
+
const opfDir = dirname(entries.opfPath);
|
|
369
|
+
const replacements = /* @__PURE__ */ new Map();
|
|
370
|
+
for (const m of doc.matchAll(DATA_URI_RE)) {
|
|
371
|
+
const [whole, attribute, type, base64] = m;
|
|
372
|
+
if (replacements.has(whole)) continue;
|
|
373
|
+
let raw;
|
|
374
|
+
try {
|
|
375
|
+
raw = Uint8Array.from(atob(base64.replace(/\s+/g, "")), (c) => c.charCodeAt(0));
|
|
376
|
+
} catch {
|
|
377
|
+
continue;
|
|
378
|
+
}
|
|
379
|
+
const name = `dataimg_${result.dataImages++}`;
|
|
380
|
+
let data = raw;
|
|
381
|
+
let extension = type.toLowerCase() === "jpeg" ? "jpg" : type.toLowerCase();
|
|
382
|
+
if (processImage) try {
|
|
383
|
+
data = await processImage(raw, name);
|
|
384
|
+
extension = "jpg";
|
|
385
|
+
} catch (e) {
|
|
386
|
+
log(`data-URI image kept as it was (${errorMessage(e)})`);
|
|
387
|
+
}
|
|
388
|
+
const mediaType = extension === "jpg" ? "image/jpeg" : `image/${extension}`;
|
|
389
|
+
const file = `${name}.${extension}`;
|
|
390
|
+
const zipPath = join(dirname(path), file);
|
|
391
|
+
entries.data.set(zipPath, data);
|
|
392
|
+
entries.order.push(zipPath);
|
|
393
|
+
const item = `<item id="${name}" href="${relative(zipPath, opfDir)}" media-type="${mediaType}"/>`;
|
|
394
|
+
opf = opf.replace(/(<\/(?:\w+:)?manifest>)/, (close) => `${item}\n ${close}`);
|
|
395
|
+
log(`extracted ${(raw.length / 1024).toFixed(1)} KB data-URI image → ${file}`);
|
|
396
|
+
replacements.set(whole, `${attribute}="${file}"`);
|
|
397
|
+
}
|
|
398
|
+
return {
|
|
399
|
+
doc: doc.replace(DATA_URI_RE, (m) => replacements.get(m) ?? m),
|
|
400
|
+
opf
|
|
401
|
+
};
|
|
402
|
+
}
|
|
403
|
+
const errorMessage = (e) => e instanceof Error ? e.message : String(e);
|
|
404
|
+
const dirname = (path) => path.includes("/") ? path.slice(0, path.lastIndexOf("/")) : "";
|
|
405
|
+
const basename = (path) => path.slice(path.lastIndexOf("/") + 1);
|
|
406
|
+
const join = (dir, name) => dir ? `${dir}/${name}` : name;
|
|
407
|
+
/** Normalizes `.` and `..` segments. */
|
|
408
|
+
function normalize(path) {
|
|
409
|
+
const parts = [];
|
|
410
|
+
for (const part of path.split("/")) if (part === "..") parts.pop();
|
|
411
|
+
else if (part && part !== ".") parts.push(part);
|
|
412
|
+
return parts.join("/");
|
|
413
|
+
}
|
|
414
|
+
/** A manifest href as a zip path. Hrefs are URLs, so they may be percent-encoded. */
|
|
415
|
+
function resolveHref(opfDir, href) {
|
|
416
|
+
let decoded = href.split("#")[0];
|
|
417
|
+
try {
|
|
418
|
+
decoded = decodeURIComponent(decoded);
|
|
419
|
+
} catch {}
|
|
420
|
+
return normalize(join(opfDir, decoded));
|
|
421
|
+
}
|
|
422
|
+
/** `path` relative to the directory `from`. */
|
|
423
|
+
function relative(path, from) {
|
|
424
|
+
const target = path.split("/");
|
|
425
|
+
const base = from ? from.split("/") : [];
|
|
426
|
+
let common = 0;
|
|
427
|
+
while (common < base.length && common < target.length - 1 && base[common] === target[common]) common++;
|
|
428
|
+
return [...base.slice(common).map(() => ".."), ...target.slice(common)].join("/");
|
|
429
|
+
}
|
|
430
|
+
//#endregion
|
|
431
|
+
export { CHUNK_TARGET, PARA_LIMIT, PARA_TARGET, SPLIT_LIMIT, basename, dirname, join, normalize, parseNodes, resolveHref, splitBigParagraphs, splitEpubText, splitTextSentences, splitXhtmlDoc, visibleText };
|