mdast-util-to-markdown-cjk-friendly 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/index.d.mts +1 -5
- package/dist/index.mjs +7 -242
- package/dist/phrasing-CVe5_g2v.mjs +430 -0
- package/dist/phrasing.d.mts +18 -0
- package/dist/phrasing.mjs +2 -0
- package/package.json +10 -4
package/README.md
CHANGED
package/dist/index.d.mts
CHANGED
|
@@ -1,10 +1,6 @@
|
|
|
1
1
|
import { Options } from "mdast-util-to-markdown";
|
|
2
|
-
|
|
3
2
|
//#region src/index.d.ts
|
|
4
|
-
|
|
5
|
-
/**
|
|
6
|
-
* Add CJK-friendly `toMarkdown` handlers for emphasis and strong.
|
|
7
|
-
*/
|
|
3
|
+
/** Add CJK-friendly `toMarkdown` handlers for emphasis and strong. */
|
|
8
4
|
declare function cjkFriendlyToMarkdown(): Options;
|
|
9
5
|
//#endregion
|
|
10
6
|
export { cjkFriendlyToMarkdown };
|
package/dist/index.mjs
CHANGED
|
@@ -1,248 +1,13 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
|
|
1
|
+
import { n as cjkFriendlyPhrasingHandlers, t as cjkFriendlyAttention } from "./phrasing-CVe5_g2v.mjs";
|
|
2
|
+
import { defaultHandlers } from "mdast-util-to-markdown";
|
|
4
3
|
//#region src/index.ts
|
|
5
|
-
|
|
6
|
-
const encodedInsideBoundary = "&".codePointAt(0) ?? codes.eof;
|
|
7
|
-
/**
|
|
8
|
-
* Add CJK-friendly `toMarkdown` handlers for emphasis and strong.
|
|
9
|
-
*/
|
|
4
|
+
/** Add CJK-friendly `toMarkdown` handlers for emphasis and strong. */
|
|
10
5
|
function cjkFriendlyToMarkdown() {
|
|
11
6
|
return { handlers: {
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
7
|
+
...cjkFriendlyPhrasingHandlers,
|
|
8
|
+
emphasis: cjkFriendlyAttention(defaultHandlers.emphasis),
|
|
9
|
+
strong: cjkFriendlyAttention(defaultHandlers.strong)
|
|
15
10
|
} };
|
|
16
11
|
}
|
|
17
|
-
emphasis.peek = emphasisPeek;
|
|
18
|
-
strong.peek = strongPeek;
|
|
19
|
-
function emphasis(node, parent, state, info) {
|
|
20
|
-
return serializeAttention(node, parent, state, info, emphasisPeek(node, parent, state), 1);
|
|
21
|
-
}
|
|
22
|
-
function strong(node, parent, state, info) {
|
|
23
|
-
return serializeAttention(node, parent, state, info, strongPeek(node, parent, state), 2);
|
|
24
|
-
}
|
|
25
|
-
function emphasisPeek(_, _parent, state) {
|
|
26
|
-
return state.options.emphasis || "*";
|
|
27
|
-
}
|
|
28
|
-
function strongPeek(_, _parent, state) {
|
|
29
|
-
return state.options.strong || "*";
|
|
30
|
-
}
|
|
31
|
-
function serializeAttention(node, parent, state, info, marker, size) {
|
|
32
|
-
const sequence = marker.repeat(size);
|
|
33
|
-
const exit = state.enter(size === 1 ? "emphasis" : "strong");
|
|
34
|
-
const tracker = state.createTracker(info);
|
|
35
|
-
const before = tracker.move(sequence);
|
|
36
|
-
let between = tracker.move(state.containerPhrasing(node, {
|
|
37
|
-
after: marker,
|
|
38
|
-
before,
|
|
39
|
-
...tracker.current()
|
|
40
|
-
}));
|
|
41
|
-
const beforeBoundary = resolveBeforeBoundary(node, parent, state, info.before);
|
|
42
|
-
const afterBoundary = resolveAfterBoundary(node, parent, state, info.after);
|
|
43
|
-
const open = encodeInfoCjk(beforeBoundary, firstCodePoint(between), marker, "open");
|
|
44
|
-
if (open.inside && between) between = encodeFirstCodePoint(between);
|
|
45
|
-
const close = encodeInfoCjk({
|
|
46
|
-
current: lastCodePoint(between),
|
|
47
|
-
previous: codePointBeforeLast(between)
|
|
48
|
-
}, afterBoundary, marker, "close");
|
|
49
|
-
if (close.inside && between) between = encodeLastCodePoint(between);
|
|
50
|
-
const after = tracker.move(sequence);
|
|
51
|
-
exit();
|
|
52
|
-
const encodeAfterSupplementary = close.outside && shouldEncodeAfterSupplementaryText(parent, state);
|
|
53
|
-
state.attentionEncodeSurroundingInfo = {
|
|
54
|
-
after: close.outside && !encodeAfterSupplementary,
|
|
55
|
-
before: open.outside
|
|
56
|
-
};
|
|
57
|
-
getCjkFriendlyState(state).cjkFriendlyEncodeAfterSupplementaryText = encodeAfterSupplementary;
|
|
58
|
-
return before + between + after;
|
|
59
|
-
}
|
|
60
|
-
function text(node, _parent, state, info) {
|
|
61
|
-
const cjkFriendlyState = getCjkFriendlyState(state);
|
|
62
|
-
if (!cjkFriendlyState.cjkFriendlyEncodeAfterSupplementaryText) return state.safe(node.value, info);
|
|
63
|
-
cjkFriendlyState.cjkFriendlyEncodeAfterSupplementaryText = false;
|
|
64
|
-
const [first = ""] = [...node.value];
|
|
65
|
-
const rest = node.value.slice(first.length);
|
|
66
|
-
return `${encodeCharacterReference(first.codePointAt(0) ?? codes.eof)}${state.safe(rest, {
|
|
67
|
-
...info,
|
|
68
|
-
before: ";"
|
|
69
|
-
})}`;
|
|
70
|
-
}
|
|
71
|
-
function encodeInfoCjk(before, after, marker, target) {
|
|
72
|
-
const beforeKind = classifyBoundaryBefore(before);
|
|
73
|
-
const afterKind = classifyCharacter(after);
|
|
74
|
-
if (!isCjkOrIvs(beforeKind) && !isCjkOrIvs(afterKind)) return target === "open" ? encodeInfoFallback(beforeKind, afterKind, marker) : encodeInfoFallback(afterKind, beforeKind, marker);
|
|
75
|
-
const raw = {
|
|
76
|
-
inside: false,
|
|
77
|
-
outside: false
|
|
78
|
-
};
|
|
79
|
-
const preserveOutside = {
|
|
80
|
-
inside: true,
|
|
81
|
-
outside: false
|
|
82
|
-
};
|
|
83
|
-
const preserveInside = {
|
|
84
|
-
inside: false,
|
|
85
|
-
outside: true
|
|
86
|
-
};
|
|
87
|
-
const encodeBoth = {
|
|
88
|
-
inside: true,
|
|
89
|
-
outside: true
|
|
90
|
-
};
|
|
91
|
-
for (const candidate of [
|
|
92
|
-
raw,
|
|
93
|
-
preserveOutside,
|
|
94
|
-
preserveInside,
|
|
95
|
-
encodeBoth
|
|
96
|
-
]) {
|
|
97
|
-
const candidateBefore = target === "open" ? candidate.outside ? encodedBoundaryBeforeContext : before : candidate.inside ? encodedBoundaryBeforeContext : before;
|
|
98
|
-
const candidateAfter = target === "open" ? candidate.inside ? encodedBoundaryAfter : after : candidate.outside ? encodedBoundaryAfter : after;
|
|
99
|
-
if (target === "open" ? canOpen(marker, candidateBefore, candidateAfter) : canClose(marker, candidateBefore, candidateAfter)) return candidate;
|
|
100
|
-
}
|
|
101
|
-
return encodeBoth;
|
|
102
|
-
}
|
|
103
|
-
function encodeInfoFallback(outsideKind, insideKind, marker) {
|
|
104
|
-
if (isLetterLike(outsideKind)) return isLetterLike(insideKind) ? marker === "_" ? {
|
|
105
|
-
inside: true,
|
|
106
|
-
outside: true
|
|
107
|
-
} : {
|
|
108
|
-
inside: false,
|
|
109
|
-
outside: false
|
|
110
|
-
} : isUnicodeWhitespace(insideKind) ? {
|
|
111
|
-
inside: true,
|
|
112
|
-
outside: true
|
|
113
|
-
} : {
|
|
114
|
-
inside: false,
|
|
115
|
-
outside: true
|
|
116
|
-
};
|
|
117
|
-
if (isUnicodeWhitespace(outsideKind)) return isLetterLike(insideKind) ? {
|
|
118
|
-
inside: false,
|
|
119
|
-
outside: false
|
|
120
|
-
} : isUnicodeWhitespace(insideKind) ? {
|
|
121
|
-
inside: true,
|
|
122
|
-
outside: true
|
|
123
|
-
} : {
|
|
124
|
-
inside: false,
|
|
125
|
-
outside: false
|
|
126
|
-
};
|
|
127
|
-
return isUnicodeWhitespace(insideKind) ? {
|
|
128
|
-
inside: true,
|
|
129
|
-
outside: false
|
|
130
|
-
} : {
|
|
131
|
-
inside: false,
|
|
132
|
-
outside: false
|
|
133
|
-
};
|
|
134
|
-
}
|
|
135
|
-
function canOpen(marker, before, afterCode) {
|
|
136
|
-
const beforeKind = classifyBoundaryBefore(before);
|
|
137
|
-
const { close, open } = getAttentionSides(beforeKind, classifyCharacter(afterCode));
|
|
138
|
-
return marker === "_" ? open && (isSpaceOrPunctuation(beforeKind) || !close) : open;
|
|
139
|
-
}
|
|
140
|
-
function canClose(marker, before, afterCode) {
|
|
141
|
-
const afterKind = classifyCharacter(afterCode);
|
|
142
|
-
const { close, open } = getAttentionSides(classifyBoundaryBefore(before), afterKind);
|
|
143
|
-
return marker === "_" ? close && (isSpaceOrPunctuation(afterKind) || !open) : close;
|
|
144
|
-
}
|
|
145
|
-
function getAttentionSides(beforeKind, afterKind) {
|
|
146
|
-
const beforeNonCjkPunctuation = isNonCjkPunctuation(beforeKind);
|
|
147
|
-
const beforeSpaceOrNonCjkPunctuation = beforeNonCjkPunctuation || isUnicodeWhitespace(beforeKind);
|
|
148
|
-
const afterNonCjkPunctuation = isNonCjkPunctuation(afterKind);
|
|
149
|
-
const afterSpaceOrNonCjkPunctuation = afterNonCjkPunctuation || isUnicodeWhitespace(afterKind);
|
|
150
|
-
return {
|
|
151
|
-
open: !afterSpaceOrNonCjkPunctuation || afterNonCjkPunctuation && (beforeSpaceOrNonCjkPunctuation || isCjkOrIvs(beforeKind)),
|
|
152
|
-
close: !beforeSpaceOrNonCjkPunctuation || beforeNonCjkPunctuation && (afterSpaceOrNonCjkPunctuation || isCjk(afterKind))
|
|
153
|
-
};
|
|
154
|
-
}
|
|
155
|
-
function isLetterLike(kind) {
|
|
156
|
-
return !isUnicodeWhitespace(kind) && !isNonCjkPunctuation(kind);
|
|
157
|
-
}
|
|
158
|
-
function classifyBoundaryBefore(before) {
|
|
159
|
-
const kind = classifyCharacter(before.current);
|
|
160
|
-
return before.current === null || !isNonEmojiGeneralUseVS(kind) ? kind : classifyPrecedingCharacter(kind, () => before.previous, before.current);
|
|
161
|
-
}
|
|
162
|
-
function resolveBeforeBoundary(node, parent, state, fallback) {
|
|
163
|
-
let current = lastCodePoint(fallback);
|
|
164
|
-
let previous = codePointBeforeLast(fallback);
|
|
165
|
-
if (needsPreviousBoundaryRecovery(current) || needsPreviousContext(current, previous)) {
|
|
166
|
-
const siblingText = getAdjacentSiblingText(node, parent, state, -1);
|
|
167
|
-
if (siblingText) {
|
|
168
|
-
current = lastCodePoint(siblingText);
|
|
169
|
-
previous = codePointBeforeLast(siblingText);
|
|
170
|
-
}
|
|
171
|
-
}
|
|
172
|
-
return {
|
|
173
|
-
current,
|
|
174
|
-
previous
|
|
175
|
-
};
|
|
176
|
-
}
|
|
177
|
-
function resolveAfterBoundary(node, parent, state, fallback) {
|
|
178
|
-
const current = firstCodePoint(fallback);
|
|
179
|
-
if (!needsNextBoundaryRecovery(current)) return current;
|
|
180
|
-
const siblingText = getAdjacentSiblingText(node, parent, state, 1);
|
|
181
|
-
return siblingText ? firstCodePoint(siblingText) : current;
|
|
182
|
-
}
|
|
183
|
-
function needsPreviousBoundaryRecovery(codePoint) {
|
|
184
|
-
return codePoint !== null && 56320 <= codePoint && codePoint <= 57343;
|
|
185
|
-
}
|
|
186
|
-
function needsNextBoundaryRecovery(codePoint) {
|
|
187
|
-
return codePoint !== null && 55296 <= codePoint && codePoint <= 56319;
|
|
188
|
-
}
|
|
189
|
-
function needsPreviousContext(current, previous) {
|
|
190
|
-
return isNonEmojiGeneralUseVS(classifyCharacter(current)) && (previous === null || previous === codes.eof);
|
|
191
|
-
}
|
|
192
|
-
function getAdjacentSiblingText(node, parent, state, offset) {
|
|
193
|
-
if (!parent) return;
|
|
194
|
-
const stackIndex = state.indexStack.at(-1);
|
|
195
|
-
const siblings = parent.children;
|
|
196
|
-
const index = typeof stackIndex === "number" ? stackIndex : siblings.indexOf(node);
|
|
197
|
-
if (index < 0) return;
|
|
198
|
-
const sibling = siblings[index + offset];
|
|
199
|
-
return getNodeTextContent(sibling) || void 0;
|
|
200
|
-
}
|
|
201
|
-
function shouldEncodeAfterSupplementaryText(parent, state) {
|
|
202
|
-
if (!parent) return false;
|
|
203
|
-
const stackIndex = state.indexStack.at(-1);
|
|
204
|
-
if (typeof stackIndex !== "number") return false;
|
|
205
|
-
const sibling = parent.children[stackIndex + 1];
|
|
206
|
-
return sibling?.type === "text" && (firstCodePoint(sibling.value) ?? 0) > 65535;
|
|
207
|
-
}
|
|
208
|
-
function getCjkFriendlyState(state) {
|
|
209
|
-
return state;
|
|
210
|
-
}
|
|
211
|
-
function getNodeTextContent(node) {
|
|
212
|
-
if (!node || typeof node !== "object") return "";
|
|
213
|
-
if ("value" in node && typeof node.value === "string") return node.value;
|
|
214
|
-
if ("alt" in node && typeof node.alt === "string") return node.alt;
|
|
215
|
-
if ("children" in node && Array.isArray(node.children)) return node.children.map(getNodeTextContent).join("");
|
|
216
|
-
return "";
|
|
217
|
-
}
|
|
218
|
-
function encodeCharacterReference(codePoint) {
|
|
219
|
-
return `&#x${(codePoint ?? 0).toString(16).toUpperCase()};`;
|
|
220
|
-
}
|
|
221
|
-
function encodeFirstCodePoint(value) {
|
|
222
|
-
const [first = ""] = [...value];
|
|
223
|
-
return encodeCharacterReference(first.codePointAt(0) ?? codes.eof) + value.slice(first.length);
|
|
224
|
-
}
|
|
225
|
-
function encodeLastCodePoint(value) {
|
|
226
|
-
const characters = [...value];
|
|
227
|
-
const last = characters.pop();
|
|
228
|
-
return `${characters.join("")}${encodeCharacterReference(last?.codePointAt(0) ?? codes.eof)}`;
|
|
229
|
-
}
|
|
230
|
-
function codePointBeforeLast(value) {
|
|
231
|
-
const characters = [...value];
|
|
232
|
-
characters.pop();
|
|
233
|
-
return characters.at(-1)?.codePointAt(0) ?? codes.eof;
|
|
234
|
-
}
|
|
235
|
-
function firstCodePoint(value) {
|
|
236
|
-
return value.codePointAt(0) ?? codes.eof;
|
|
237
|
-
}
|
|
238
|
-
function lastCodePoint(value) {
|
|
239
|
-
return [...value].at(-1)?.codePointAt(0) ?? codes.eof;
|
|
240
|
-
}
|
|
241
|
-
const encodedBoundaryBeforeContext = {
|
|
242
|
-
current: encodedOutsideBoundary,
|
|
243
|
-
previous: codes.eof
|
|
244
|
-
};
|
|
245
|
-
const encodedBoundaryAfter = encodedInsideBoundary;
|
|
246
|
-
|
|
247
12
|
//#endregion
|
|
248
|
-
export { cjkFriendlyToMarkdown };
|
|
13
|
+
export { cjkFriendlyToMarkdown };
|
|
@@ -0,0 +1,430 @@
|
|
|
1
|
+
import { defaultHandlers } from "mdast-util-to-markdown";
|
|
2
|
+
import { asciiAlpha, asciiAlphanumeric, asciiPunctuation } from "micromark-util-character";
|
|
3
|
+
import { htmlBlockNames, htmlRawNames } from "micromark-util-html-tag-name";
|
|
4
|
+
import { classifyCharacter, classifyPrecedingCharacter, isCjk, isCjkOrIvs, isNonCjkPunctuation, isNonEmojiGeneralUseVS, isSpaceOrPunctuation, isUnicodeWhitespace } from "micromark-extension-cjk-friendly-util";
|
|
5
|
+
import { classifyCharacter as classifyCharacter$1 } from "micromark-util-classify-character";
|
|
6
|
+
//#region src/encode-info.ts
|
|
7
|
+
function firstCharacter(value) {
|
|
8
|
+
return value[Symbol.iterator]().next().value ?? "";
|
|
9
|
+
}
|
|
10
|
+
function lastCharacter(value, end = value.length) {
|
|
11
|
+
if (end === 0) return "";
|
|
12
|
+
const lastCodeUnit = value.charCodeAt(end - 1);
|
|
13
|
+
if (lastCodeUnit < 56320 || lastCodeUnit >= 57344 || end === 1 || (value.codePointAt(end - 2) ?? 0) < 65536) return value[end - 1];
|
|
14
|
+
return value.slice(end - 2, end);
|
|
15
|
+
}
|
|
16
|
+
function classifyBefore(value) {
|
|
17
|
+
const character = lastCharacter(value);
|
|
18
|
+
const current = character.codePointAt(0) ?? null;
|
|
19
|
+
const kind = classifyCharacter(current);
|
|
20
|
+
return current !== null && isNonEmojiGeneralUseVS(kind) ? classifyPrecedingCharacter(kind, () => lastCharacter(value, value.length - character.length).codePointAt(0) ?? null, current) : kind;
|
|
21
|
+
}
|
|
22
|
+
function sides(before, after, marker) {
|
|
23
|
+
const beforePunctuation = isNonCjkPunctuation(before);
|
|
24
|
+
const afterPunctuation = isNonCjkPunctuation(after);
|
|
25
|
+
const beforeSpaceOrPunctuation = beforePunctuation || isUnicodeWhitespace(before);
|
|
26
|
+
const afterSpaceOrPunctuation = afterPunctuation || isUnicodeWhitespace(after);
|
|
27
|
+
const open = !afterSpaceOrPunctuation || afterPunctuation && (beforeSpaceOrPunctuation || isCjkOrIvs(before));
|
|
28
|
+
const close = !beforeSpaceOrPunctuation || beforePunctuation && (afterSpaceOrPunctuation || isCjk(after));
|
|
29
|
+
return {
|
|
30
|
+
open: marker === "_" ? open && (isSpaceOrPunctuation(before) || !close) : open,
|
|
31
|
+
close: marker === "_" ? close && (isSpaceOrPunctuation(after) || !open) : close
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
function attentionSides(before, after, marker, cjk) {
|
|
35
|
+
if (cjk) return sides(classifyBefore(before), classifyCharacter(after.codePointAt(0) ?? null), marker);
|
|
36
|
+
const previous = classifyCharacter$1(lastCharacter(before).codePointAt(0) ?? null);
|
|
37
|
+
const next = classifyCharacter$1(after.codePointAt(0) ?? null);
|
|
38
|
+
const open = !next || next === 2 && Boolean(previous);
|
|
39
|
+
const close = !previous || previous === 2 && Boolean(next);
|
|
40
|
+
return {
|
|
41
|
+
open: marker === "_" ? open && (Boolean(previous) || !close) : open,
|
|
42
|
+
close: marker === "_" ? close && (Boolean(next) || !open) : close
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
function fallback(outside, inside, marker) {
|
|
46
|
+
if (outside === void 0) return inside === void 0 ? {
|
|
47
|
+
inside: marker === "_",
|
|
48
|
+
outside: marker === "_"
|
|
49
|
+
} : inside === 1 ? {
|
|
50
|
+
inside: true,
|
|
51
|
+
outside: true
|
|
52
|
+
} : {
|
|
53
|
+
inside: false,
|
|
54
|
+
outside: true
|
|
55
|
+
};
|
|
56
|
+
if (outside === 1) return {
|
|
57
|
+
inside: inside === 1,
|
|
58
|
+
outside: inside === 1
|
|
59
|
+
};
|
|
60
|
+
return {
|
|
61
|
+
inside: inside === 1,
|
|
62
|
+
outside: false
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
/** Choose boundary encodings without changing punctuation used by constructs. */
|
|
66
|
+
function encodeInfo(before, after, marker, opening, cjk) {
|
|
67
|
+
const beforeKind = classifyBefore(before);
|
|
68
|
+
const afterKind = classifyCharacter(after.codePointAt(0) ?? null);
|
|
69
|
+
if (!cjk || !isCjkOrIvs(beforeKind) && !isCjkOrIvs(afterKind)) {
|
|
70
|
+
const previous = cjk ? commonMarkKind(beforeKind) : classifyCharacter$1(lastCharacter(before).codePointAt(0) ?? null);
|
|
71
|
+
const next = cjk ? commonMarkKind(afterKind) : classifyCharacter$1(after.codePointAt(0) ?? null);
|
|
72
|
+
return opening ? fallback(previous, next, marker) : fallback(next, previous, marker);
|
|
73
|
+
}
|
|
74
|
+
for (const candidate of [
|
|
75
|
+
{
|
|
76
|
+
inside: false,
|
|
77
|
+
outside: false
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
inside: true,
|
|
81
|
+
outside: false
|
|
82
|
+
},
|
|
83
|
+
{
|
|
84
|
+
inside: false,
|
|
85
|
+
outside: true
|
|
86
|
+
},
|
|
87
|
+
{
|
|
88
|
+
inside: true,
|
|
89
|
+
outside: true
|
|
90
|
+
}
|
|
91
|
+
]) {
|
|
92
|
+
const encodedBefore = opening ? candidate.outside : candidate.inside;
|
|
93
|
+
const encodedAfter = opening ? candidate.inside : candidate.outside;
|
|
94
|
+
const forms = sides(encodedBefore ? classifyCharacter(";".codePointAt(0) ?? null) : beforeKind, encodedAfter ? classifyCharacter("&".codePointAt(0) ?? null) : afterKind, marker);
|
|
95
|
+
if (opening ? forms.open : forms.close) return candidate;
|
|
96
|
+
}
|
|
97
|
+
return {
|
|
98
|
+
inside: true,
|
|
99
|
+
outside: true
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
function commonMarkKind(kind) {
|
|
103
|
+
return isUnicodeWhitespace(kind) ? 1 : isNonCjkPunctuation(kind) ? 2 : void 0;
|
|
104
|
+
}
|
|
105
|
+
//#endregion
|
|
106
|
+
//#region src/phrasing.ts
|
|
107
|
+
/*!
|
|
108
|
+
* Phrasing algorithm adapted from mdast-util-to-markdown (commit 388140e).
|
|
109
|
+
* Copyright (c) Titus Wormer <tituswormer@gmail.com>
|
|
110
|
+
*
|
|
111
|
+
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
112
|
+
* of this software and associated documentation files (the "Software"), to deal
|
|
113
|
+
* in the Software without restriction, including without limitation the rights
|
|
114
|
+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
115
|
+
* copies of the Software, and to permit persons to whom the Software is
|
|
116
|
+
* furnished to do so, subject to the following conditions:
|
|
117
|
+
*
|
|
118
|
+
* The above copyright notice and this permission notice shall be included in
|
|
119
|
+
* all copies or substantial portions of the Software.
|
|
120
|
+
*
|
|
121
|
+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
122
|
+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
123
|
+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
124
|
+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
125
|
+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
126
|
+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
127
|
+
* SOFTWARE.
|
|
128
|
+
*/
|
|
129
|
+
/** Mark an attention handler for CJK-aware encoding and pairing. */
|
|
130
|
+
function cjkFriendlyAttention(handle) {
|
|
131
|
+
const attention = handle.attention;
|
|
132
|
+
if (!attention) throw new Error("Expected a handler with the attention API");
|
|
133
|
+
const result = (node, parent, state, info) => {
|
|
134
|
+
install(state);
|
|
135
|
+
return handle(node, parent, state, info);
|
|
136
|
+
};
|
|
137
|
+
result.peek = handle.peek;
|
|
138
|
+
result.attention = (node, state) => ({
|
|
139
|
+
...attention(node, state),
|
|
140
|
+
cjkFriendly: true
|
|
141
|
+
});
|
|
142
|
+
return result;
|
|
143
|
+
}
|
|
144
|
+
function install(state) {
|
|
145
|
+
state.containerPhrasing = (parent, info) => containerPhrasing(parent, state, info);
|
|
146
|
+
}
|
|
147
|
+
function withCjkPhrasing(handle) {
|
|
148
|
+
return (node, parent, state, info) => {
|
|
149
|
+
install(state);
|
|
150
|
+
return handle(node, parent, state, info);
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
/** Install the container before serializing a document or a phrasing block. */
|
|
154
|
+
const cjkFriendlyPhrasingHandlers = {
|
|
155
|
+
root: withCjkPhrasing(defaultHandlers.root),
|
|
156
|
+
paragraph: withCjkPhrasing(defaultHandlers.paragraph),
|
|
157
|
+
heading: withCjkPhrasing(defaultHandlers.heading)
|
|
158
|
+
};
|
|
159
|
+
function candidates(tokens, involved, runOf) {
|
|
160
|
+
const related = /* @__PURE__ */ new Set();
|
|
161
|
+
for (const token of tokens) {
|
|
162
|
+
const siblings = runOf.get(token);
|
|
163
|
+
if (siblings && token.attention && involved.some((d) => d.attention === token.attention)) for (const sibling of siblings.tokens) related.add(sibling.attention);
|
|
164
|
+
}
|
|
165
|
+
const ordered = [];
|
|
166
|
+
for (const token of tokens) if (token.attention && related.delete(token.attention)) ordered.push(token.attention);
|
|
167
|
+
return ordered.reverse();
|
|
168
|
+
}
|
|
169
|
+
function check(tokens, before, after) {
|
|
170
|
+
const runs = [];
|
|
171
|
+
const wrong = /* @__PURE__ */ new Map();
|
|
172
|
+
const runOf = /* @__PURE__ */ new Map();
|
|
173
|
+
let index = 0;
|
|
174
|
+
while (index < tokens.length) {
|
|
175
|
+
if (!tokens[index].attention) {
|
|
176
|
+
index++;
|
|
177
|
+
continue;
|
|
178
|
+
}
|
|
179
|
+
const marker = tokens[index].value.charAt(0);
|
|
180
|
+
let end = index + 1;
|
|
181
|
+
while (end < tokens.length && tokens[end].attention && tokens[end].value.charAt(0) === marker) end++;
|
|
182
|
+
const group = tokens.slice(index, end);
|
|
183
|
+
const { open, close } = attentionSides(index ? tokens[index - 1].value : before, end < tokens.length ? tokens[end].value : after, marker, group.some((token) => token.attention?.cjk));
|
|
184
|
+
const run = {
|
|
185
|
+
tokens: group,
|
|
186
|
+
markers: [],
|
|
187
|
+
start: 0,
|
|
188
|
+
end: 0,
|
|
189
|
+
open,
|
|
190
|
+
close,
|
|
191
|
+
split: marker === "*" || marker === "_"
|
|
192
|
+
};
|
|
193
|
+
for (const token of group) {
|
|
194
|
+
wrong.set(token, [...group]);
|
|
195
|
+
runOf.set(token, run);
|
|
196
|
+
for (let size = token.value.length; size > 0; size--) run.markers.push(token);
|
|
197
|
+
}
|
|
198
|
+
run.end = run.markers.length;
|
|
199
|
+
runs.push(run);
|
|
200
|
+
index = end;
|
|
201
|
+
}
|
|
202
|
+
pair(runs, wrong);
|
|
203
|
+
for (let index = 0; index < tokens.length; index++) {
|
|
204
|
+
const involved = wrong.get(tokens[index]);
|
|
205
|
+
if (involved) return {
|
|
206
|
+
attention: candidates(tokens, involved, runOf),
|
|
207
|
+
index
|
|
208
|
+
};
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
function pair(runs, wrong) {
|
|
212
|
+
let index = 0;
|
|
213
|
+
while (index < runs.length) {
|
|
214
|
+
const closer = runs[index];
|
|
215
|
+
const marker = closer.tokens[0].value.charAt(0);
|
|
216
|
+
const closerSize = closer.end - closer.start;
|
|
217
|
+
let open = closer.close && closerSize ? index : 0;
|
|
218
|
+
while (open--) {
|
|
219
|
+
const opener = runs[open];
|
|
220
|
+
const openerSize = opener.end - opener.start;
|
|
221
|
+
if (opener.tokens[0].value.charAt(0) === marker && opener.open && openerSize && (closer.split ? !((opener.close || closer.open) && closerSize % 3) || (openerSize + closerSize) % 3 : openerSize === opener.markers.length && openerSize === closerSize)) {
|
|
222
|
+
const size = closer.split ? openerSize > 1 && closerSize > 1 ? 2 : 1 : closerSize;
|
|
223
|
+
const opening = opener.markers.slice(opener.end - size, opener.end);
|
|
224
|
+
const closing = closer.markers.slice(closer.start, closer.start + size);
|
|
225
|
+
const openToken = opening[0];
|
|
226
|
+
const closeToken = closing[0];
|
|
227
|
+
if (openToken === opening[size - 1] && openToken.value.length === size && closeToken === closing[size - 1] && closeToken.value.length === size && openToken.attention === closeToken.attention) {
|
|
228
|
+
wrong.delete(openToken);
|
|
229
|
+
wrong.delete(closeToken);
|
|
230
|
+
} else {
|
|
231
|
+
const involved = [.../* @__PURE__ */ new Set([...opening, ...closing])];
|
|
232
|
+
for (const token of involved) wrong.get(token)?.push(...involved);
|
|
233
|
+
}
|
|
234
|
+
opener.end -= size;
|
|
235
|
+
closer.start += size;
|
|
236
|
+
break;
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
if (open === -1 || closer.start === closer.end) index++;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
function encode(tokens, start) {
|
|
243
|
+
const token = tokens[start ? 0 : tokens.length - 1];
|
|
244
|
+
if (!token || token.attention || !token.value) return;
|
|
245
|
+
const character = start ? firstCharacter(token.value) : lastCharacter(token.value);
|
|
246
|
+
const reference = `&#x${character.codePointAt(0)?.toString(16).toUpperCase()};`;
|
|
247
|
+
token.value = start ? reference + token.value.slice(character.length) : token.value.slice(0, -character.length) + reference;
|
|
248
|
+
}
|
|
249
|
+
function attentionSequences(type, markers, sizes) {
|
|
250
|
+
if (!markers.length || !sizes.length) throw new Error(`Cannot serialize ${type} as attention without markers and sizes`);
|
|
251
|
+
for (const size of sizes) if (!Number.isSafeInteger(size) || size < 1) throw new Error(`Invalid attention size for ${type}: ${size}`);
|
|
252
|
+
const sequences = [];
|
|
253
|
+
for (const marker of markers) {
|
|
254
|
+
if (marker.length !== 1 || !asciiPunctuation(marker.charCodeAt(0))) throw new Error(`Invalid attention marker for ${type}: ${marker}`);
|
|
255
|
+
for (const size of sizes) sequences.push(marker.repeat(size));
|
|
256
|
+
}
|
|
257
|
+
return sequences;
|
|
258
|
+
}
|
|
259
|
+
function htmlKind(value) {
|
|
260
|
+
if (value.charCodeAt(0) !== 60) return;
|
|
261
|
+
const next = value.charCodeAt(1);
|
|
262
|
+
if (next === 33) {
|
|
263
|
+
const code = value.charCodeAt(2);
|
|
264
|
+
if (code === 45) return 2;
|
|
265
|
+
if (code === 91) return 5;
|
|
266
|
+
if (asciiAlpha(code)) return 4;
|
|
267
|
+
return;
|
|
268
|
+
}
|
|
269
|
+
if (next === 63) return 3;
|
|
270
|
+
const closing = next === 47;
|
|
271
|
+
const start = closing ? 2 : 1;
|
|
272
|
+
let end = start;
|
|
273
|
+
let code = value.charCodeAt(end);
|
|
274
|
+
if (!asciiAlpha(code)) return;
|
|
275
|
+
while (code === 45 || asciiAlphanumeric(code)) code = value.charCodeAt(++end);
|
|
276
|
+
const name = value.slice(start, end).toLowerCase();
|
|
277
|
+
if (!closing && code !== 47 && htmlRawNames.includes(name)) return 1;
|
|
278
|
+
return htmlBlockNames.includes(name) ? 6 : 7;
|
|
279
|
+
}
|
|
280
|
+
function phrasing(parent, state, info) {
|
|
281
|
+
const children = parent.children;
|
|
282
|
+
const results = [];
|
|
283
|
+
let before = info.before;
|
|
284
|
+
state.indexStack.push(-1);
|
|
285
|
+
let tracker = state.createTracker(info);
|
|
286
|
+
try {
|
|
287
|
+
for (let index = 0; index < children.length; index++) {
|
|
288
|
+
const child = children[index];
|
|
289
|
+
state.indexStack[state.indexStack.length - 1] = index;
|
|
290
|
+
const next = children[index + 1];
|
|
291
|
+
let after = info.after;
|
|
292
|
+
if (next) {
|
|
293
|
+
const handle = state.handlers[next.type];
|
|
294
|
+
const peek = handle?.peek ?? handle;
|
|
295
|
+
after = peek ? firstCharacter(peek(next, parent, state, {
|
|
296
|
+
before: "",
|
|
297
|
+
after: "",
|
|
298
|
+
...tracker.current()
|
|
299
|
+
})) : "";
|
|
300
|
+
}
|
|
301
|
+
const previous = results[results.length - 1];
|
|
302
|
+
if (typeof previous === "string" && (before === "\r" || before === "\n") && child.type === "html" && htmlKind(child.value) !== 7) {
|
|
303
|
+
results[results.length - 1] = previous.replace(/(\r?\n|\r)$/, " ");
|
|
304
|
+
before = " ";
|
|
305
|
+
tracker = state.createTracker(info);
|
|
306
|
+
tracker.move(serialize(results, info.before, info.after));
|
|
307
|
+
}
|
|
308
|
+
const handle = state.handlers[child.type];
|
|
309
|
+
if (handle?.attention) {
|
|
310
|
+
const { construct, markers, sizes, cjkFriendly } = handle.attention(child, state);
|
|
311
|
+
const sequences = attentionSequences(child.type, markers, sizes);
|
|
312
|
+
const sequence = sequences[0];
|
|
313
|
+
const exit = state.enter(construct);
|
|
314
|
+
tracker.move(sequence);
|
|
315
|
+
let inside;
|
|
316
|
+
try {
|
|
317
|
+
inside = phrasing(child, state, {
|
|
318
|
+
...tracker.current(),
|
|
319
|
+
before: sequence,
|
|
320
|
+
after: sequence
|
|
321
|
+
});
|
|
322
|
+
tracker.move(serialize(inside, sequence, sequence));
|
|
323
|
+
tracker.move(sequence);
|
|
324
|
+
} finally {
|
|
325
|
+
exit();
|
|
326
|
+
}
|
|
327
|
+
results.push({
|
|
328
|
+
children: inside,
|
|
329
|
+
sequences,
|
|
330
|
+
cjk: Boolean(cjkFriendly)
|
|
331
|
+
});
|
|
332
|
+
before = lastCharacter(sequence);
|
|
333
|
+
} else {
|
|
334
|
+
const value = state.handle(child, parent, state, {
|
|
335
|
+
...tracker.current(),
|
|
336
|
+
after,
|
|
337
|
+
before
|
|
338
|
+
});
|
|
339
|
+
if (!value) continue;
|
|
340
|
+
tracker.move(value);
|
|
341
|
+
results.push(value);
|
|
342
|
+
before = value;
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
return results;
|
|
346
|
+
} finally {
|
|
347
|
+
state.indexStack.pop();
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
function render(items, chosen, before, after) {
|
|
351
|
+
const tokens = [];
|
|
352
|
+
let encodeAfter;
|
|
353
|
+
let previous = before;
|
|
354
|
+
for (let index = 0; index < items.length; index++) {
|
|
355
|
+
const item = items[index];
|
|
356
|
+
if (typeof item === "string") {
|
|
357
|
+
const token = {
|
|
358
|
+
value: item,
|
|
359
|
+
attention: void 0
|
|
360
|
+
};
|
|
361
|
+
if (encodeAfter && encodeAfter === firstCharacter(item)) encode([token], true);
|
|
362
|
+
encodeAfter = void 0;
|
|
363
|
+
tokens.push(token);
|
|
364
|
+
if (token.value) previous = token.value;
|
|
365
|
+
continue;
|
|
366
|
+
}
|
|
367
|
+
const sequence = chosen.get(item) || item.sequences[0];
|
|
368
|
+
const marker = sequence.charAt(0);
|
|
369
|
+
const next = items[index + 1];
|
|
370
|
+
const outsideAfter = next === void 0 ? after : typeof next === "string" ? next : chosen.get(next) || next.sequences[0];
|
|
371
|
+
const inside = render(item.children, chosen, sequence, sequence);
|
|
372
|
+
const head = inside[0]?.value ?? "";
|
|
373
|
+
const open = encodeInfo(previous, head, marker, true, item.cjk);
|
|
374
|
+
if (open.inside) encode(inside, true);
|
|
375
|
+
const close = encodeInfo(inside[inside.length - 1]?.value ?? "", outsideAfter, marker, false, item.cjk);
|
|
376
|
+
if (close.inside) encode(inside, false);
|
|
377
|
+
if (open.outside && lastCharacter(previous) !== "\n" && lastCharacter(previous) !== "\r") encode(tokens, false);
|
|
378
|
+
if (close.outside) encodeAfter = firstCharacter(outsideAfter);
|
|
379
|
+
tokens.push({
|
|
380
|
+
value: sequence,
|
|
381
|
+
attention: item
|
|
382
|
+
}, ...inside, {
|
|
383
|
+
value: sequence,
|
|
384
|
+
attention: item
|
|
385
|
+
});
|
|
386
|
+
previous = sequence;
|
|
387
|
+
}
|
|
388
|
+
return tokens;
|
|
389
|
+
}
|
|
390
|
+
function serialize(items, before, after) {
|
|
391
|
+
let chosen = /* @__PURE__ */ new Map();
|
|
392
|
+
const initial = render(items, chosen, before, after);
|
|
393
|
+
let tokens = initial;
|
|
394
|
+
let mistake = check(tokens, before, after);
|
|
395
|
+
while (mistake) {
|
|
396
|
+
let improvement;
|
|
397
|
+
for (const attention of mistake.attention) {
|
|
398
|
+
const current = chosen.get(attention) || attention.sequences[0];
|
|
399
|
+
for (const sequence of attention.sequences) {
|
|
400
|
+
if (sequence === current) continue;
|
|
401
|
+
const trial = new Map(chosen);
|
|
402
|
+
trial.set(attention, sequence);
|
|
403
|
+
const rendered = render(items, trial, before, after);
|
|
404
|
+
const next = check(rendered, before, after);
|
|
405
|
+
if (!next || next.index > mistake.index) {
|
|
406
|
+
improvement = {
|
|
407
|
+
chosen: trial,
|
|
408
|
+
tokens: rendered,
|
|
409
|
+
mistake: next
|
|
410
|
+
};
|
|
411
|
+
break;
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
if (improvement) break;
|
|
415
|
+
}
|
|
416
|
+
if (!improvement) {
|
|
417
|
+
tokens = initial;
|
|
418
|
+
break;
|
|
419
|
+
}
|
|
420
|
+
chosen = improvement.chosen;
|
|
421
|
+
tokens = improvement.tokens;
|
|
422
|
+
mistake = improvement.mistake;
|
|
423
|
+
}
|
|
424
|
+
return tokens.map((token) => token.value).join("");
|
|
425
|
+
}
|
|
426
|
+
function containerPhrasing(parent, state, info) {
|
|
427
|
+
return serialize(phrasing(parent, state, info), info.before, info.after);
|
|
428
|
+
}
|
|
429
|
+
//#endregion
|
|
430
|
+
export { cjkFriendlyPhrasingHandlers as n, cjkFriendlyAttention as t };
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { Attention, AttentionInfo, Handle, State } from "mdast-util-to-markdown";
|
|
2
|
+
//#region src/phrasing.d.ts
|
|
3
|
+
type AttentionHandle = Handle & {
|
|
4
|
+
peek?: Handle;
|
|
5
|
+
attention?: (node: Parameters<Attention>[0], state: State) => AttentionInfo & {
|
|
6
|
+
cjkFriendly?: boolean;
|
|
7
|
+
};
|
|
8
|
+
};
|
|
9
|
+
/** Mark an attention handler for CJK-aware encoding and pairing. */
|
|
10
|
+
declare function cjkFriendlyAttention(handle: Handle): AttentionHandle;
|
|
11
|
+
/** Install the container before serializing a document or a phrasing block. */
|
|
12
|
+
declare const cjkFriendlyPhrasingHandlers: {
|
|
13
|
+
root: Handle;
|
|
14
|
+
paragraph: Handle;
|
|
15
|
+
heading: Handle;
|
|
16
|
+
};
|
|
17
|
+
//#endregion
|
|
18
|
+
export { cjkFriendlyAttention, cjkFriendlyPhrasingHandlers };
|
package/package.json
CHANGED
|
@@ -1,12 +1,16 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mdast-util-to-markdown-cjk-friendly",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.1",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": {
|
|
7
7
|
"types": "./dist/index.d.mts",
|
|
8
8
|
"default": "./dist/index.mjs"
|
|
9
9
|
},
|
|
10
|
+
"./phrasing": {
|
|
11
|
+
"types": "./dist/phrasing.d.mts",
|
|
12
|
+
"default": "./dist/phrasing.mjs"
|
|
13
|
+
},
|
|
10
14
|
"./package.json": "./package.json"
|
|
11
15
|
},
|
|
12
16
|
"module": "./dist/index.mjs",
|
|
@@ -35,9 +39,11 @@
|
|
|
35
39
|
"description": "mdast-util-to-markdown extension to make Markdown emphasis output more friendly with Chinese, Japanese, and Korean (CJK)",
|
|
36
40
|
"sideEffects": false,
|
|
37
41
|
"dependencies": {
|
|
38
|
-
"mdast-util-to-markdown": "^2.1.
|
|
39
|
-
"micromark-util
|
|
40
|
-
"micromark-
|
|
42
|
+
"mdast-util-to-markdown": "^2.1.3",
|
|
43
|
+
"micromark-extension-cjk-friendly-util": "3.0.1",
|
|
44
|
+
"micromark-util-character": "^2.1.1",
|
|
45
|
+
"micromark-util-classify-character": "^2.0.1",
|
|
46
|
+
"micromark-util-html-tag-name": "^2.0.1"
|
|
41
47
|
},
|
|
42
48
|
"peerDependencies": {
|
|
43
49
|
"@types/mdast": "*"
|