@bastiangx/opencode-unmoji 0.1.1 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -4
- package/dist/core/emoji-data.d.ts +3 -0
- package/dist/core/emoji-data.d.ts.map +1 -1
- package/dist/core/emoji-data.js +46 -2
- package/dist/core/emoji-data.js.map +1 -1
- package/dist/core/processor.d.ts +12 -1
- package/dist/core/processor.d.ts.map +1 -1
- package/dist/core/processor.js +67 -12
- package/dist/core/processor.js.map +1 -1
- package/dist/{core/emoji-data-generated.d.ts → data/emoji-data-gen.d.ts} +1 -1
- package/dist/data/emoji-data-gen.d.ts.map +1 -0
- package/dist/{core/emoji-data-generated.js → data/emoji-data-gen.js} +7646 -5371
- package/dist/data/emoji-data-gen.js.map +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +135 -4
- package/dist/index.js.map +1 -1
- package/dist/plugin.bundled.js +7912 -5436
- package/dist/types/config.d.ts +19 -0
- package/dist/types/config.d.ts.map +1 -1
- package/dist/types/config.js +12 -0
- package/dist/types/config.js.map +1 -1
- package/dist/utils/code-detector.d.ts.map +1 -1
- package/dist/utils/code-detector.js +7 -1
- package/dist/utils/code-detector.js.map +1 -1
- package/dist/utils/grapheme.d.ts.map +1 -1
- package/dist/utils/grapheme.js +20 -2
- package/dist/utils/grapheme.js.map +1 -1
- package/package.json +19 -6
- package/dist/core/emoji-data-generated.d.ts.map +0 -1
- package/dist/core/emoji-data-generated.js.map +0 -1
- package/dist/streaming/hooks.d.ts +0 -12
- package/dist/streaming/hooks.d.ts.map +0 -1
- package/dist/streaming/hooks.js +0 -8
- package/dist/streaming/hooks.js.map +0 -1
- package/dist/streaming/sanitizer.d.ts +0 -66
- package/dist/streaming/sanitizer.d.ts.map +0 -1
- package/dist/streaming/sanitizer.js +0 -118
- package/dist/streaming/sanitizer.js.map +0 -1
- package/dist/streaming/state.d.ts +0 -10
- package/dist/streaming/state.d.ts.map +0 -1
- package/dist/streaming/state.js +0 -33
- package/dist/streaming/state.js.map +0 -1
- package/dist/utils/debug.d.ts +0 -18
- package/dist/utils/debug.d.ts.map +0 -1
- package/dist/utils/debug.js +0 -10
- package/dist/utils/debug.js.map +0 -1
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
<div align="center">
|
|
2
2
|
<a href="https://codeberg.org/bastiangx/opencode-unmoji">
|
|
3
|
-
<img height="600" src="https://github.com/user-attachments/assets/
|
|
3
|
+
<img height="600" src="https://github.com/user-attachments/assets/e1aaf6b4-a646-4708-8002-9dbac7236f95"/>
|
|
4
4
|
</a>
|
|
5
5
|
</div>
|
|
6
6
|
|
|
@@ -35,7 +35,7 @@ UNMOJI automatically removes _all_ _standard_ unicode emojis and glyphs from you
|
|
|
35
35
|
|
|
36
36
|
Specially useful when you are using models like Claude which seem to _LOVE_ spamming emojis on summaries, README files and documentations.
|
|
37
37
|
|
|
38
|
-
- ~
|
|
38
|
+
- ~ 30,000 emojis covered.
|
|
39
39
|
|
|
40
40
|
> _Thinking_ tokens as output targets are not supported.
|
|
41
41
|
|
|
@@ -53,7 +53,11 @@ Create and add the `unmoji.json` file to your OpenCode [config path](https://ope
|
|
|
53
53
|
"preserveInlineCode": true,
|
|
54
54
|
"skipQuotes": false,
|
|
55
55
|
"replacementStyle": "github",
|
|
56
|
-
"replacementFormat": "[:name:]"
|
|
56
|
+
"replacementFormat": "[:name:]",
|
|
57
|
+
"processMarkdown": true,
|
|
58
|
+
"markdownExtensions": [".md"],
|
|
59
|
+
"showNotification": true,
|
|
60
|
+
"notificationStyle": "thread"
|
|
57
61
|
}
|
|
58
62
|
```
|
|
59
63
|
|
|
@@ -66,6 +70,10 @@ Create and add the `unmoji.json` file to your OpenCode [config path](https://ope
|
|
|
66
70
|
| `skipQuotes` | `false` | `true`, `false` | Do not process emojis inside quoted text blocks |
|
|
67
71
|
| `replacementStyle` | `github` | `github`, `slack`, `discord`, `cldr`, `iamcal`, `joypixels` | Shortcode style used when mode is `replace` |
|
|
68
72
|
| `replacementFormat` | `[:name:]` | `:name:`, `[:name:]`, `{name}` | Template format for shortcode outputs |
|
|
73
|
+
| `processMarkdown` | `true` | `true`, `false` | Process emojis in markdown files written by tools |
|
|
74
|
+
| `markdownExtensions` | `[".md"]` | `.md`, `.mdx`, `.mdc`, `.markdown` (array) | File extensions treated as markdown for tool hooks |
|
|
75
|
+
| `showNotification` | `true` | `true`, `false` | Show notification when emojis are processed |
|
|
76
|
+
| `notificationStyle` | `thread` | `thread`, `thread-verbose`, `toast` | Notification display style |
|
|
69
77
|
|
|
70
78
|
#### `replace` Examples
|
|
71
79
|
|
|
@@ -103,13 +111,24 @@ Deploy successful 🚀 Bug fixed 🐛
|
|
|
103
111
|
Deploy successful [:rocket:] Bug fixed [:bug:]
|
|
104
112
|
```
|
|
105
113
|
|
|
114
|
+
### Markdown file processing
|
|
115
|
+
|
|
116
|
+
In addition to stripping/replacing emojis from agent text outputs, UNMOJI can also process markdown content that the agent writes to disk.
|
|
117
|
+
|
|
118
|
+
When `processMarkdown` is `true` (default):
|
|
119
|
+
|
|
120
|
+
- The `write` tool's `content` is processed for files whose `filePath` ends with one of `markdownExtensions`.
|
|
121
|
+
- The `edit` tool's `newString` is processed for matching markdown paths.
|
|
122
|
+
|
|
123
|
+
This ensures generated README.md, documentation, and other markdown files are kept emoji-free (or use text shortcodes) without affecting non-markdown files.
|
|
124
|
+
|
|
106
125
|
---
|
|
107
126
|
|
|
108
127
|
## Contributing
|
|
109
128
|
|
|
110
129
|
Contributions are welcome!
|
|
111
130
|
|
|
112
|
-
Follow the [Contributing guide](
|
|
131
|
+
Follow the [Contributing guide](https://codeberg.org/bastiangx/opencode-unmoji/src/branch/main/CONTRIBUTING.md) for instructions.
|
|
113
132
|
|
|
114
133
|
---
|
|
115
134
|
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import { type Result } from "neverthrow";
|
|
2
2
|
import type { EmojiDataError } from "../types/errors.js";
|
|
3
|
+
export declare const EXTRA_STRIPPABLE: Set<string>;
|
|
4
|
+
export declare function normalizeEmoji(grapheme: string): string;
|
|
5
|
+
export declare function isStrippable(grapheme: string, emojiSet: Set<string>): boolean;
|
|
3
6
|
export type ReplacementStyle = "github" | "slack" | "discord" | "cldr" | "iamcal" | "joypixels";
|
|
4
7
|
export declare function getEmojiSetSync(): Result<Set<string>, EmojiDataError>;
|
|
5
8
|
export declare function getEmojiMapSync(replacementStyle: ReplacementStyle): Result<Map<string, string>, EmojiDataError>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"emoji-data.d.ts","sourceRoot":"","sources":["../../src/core/emoji-data.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,MAAM,EAAW,MAAM,YAAY,CAAC;AAClD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"emoji-data.d.ts","sourceRoot":"","sources":["../../src/core/emoji-data.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,MAAM,EAAW,MAAM,YAAY,CAAC;AAClD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAiBzD,eAAO,MAAM,gBAAgB,aAI3B,CAAC;AAIH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAKvD;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,OAAO,CAe7E;AAED,MAAM,MAAM,gBAAgB,GACzB,QAAQ,GACR,OAAO,GACP,SAAS,GACT,MAAM,GACN,QAAQ,GACR,WAAW,CAAC;AAiBf,wBAAgB,eAAe,IAAI,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,cAAc,CAAC,CAQrE;AAED,wBAAgB,eAAe,CAC9B,gBAAgB,EAAE,gBAAgB,GAChC,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,cAAc,CAAC,CAoB7C;AAED,wBAAgB,iBAAiB,CAChC,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GACtB,MAAM,GAAG,IAAI,CAUf;AAaD,wBAAgB,cAAc,CAC7B,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,QAAQ,GAAG,UAAU,GAAG,QAAQ,EACxC,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GACtB,MAAM,CAIR"}
|
package/dist/core/emoji-data.js
CHANGED
|
@@ -1,5 +1,41 @@
|
|
|
1
1
|
import { ok, err } from "neverthrow";
|
|
2
|
-
import { EMOJI_SET, EMOJI_MAP_CLDR, EMOJI_MAP_GITHUB, EMOJI_MAP_IAMCAL, EMOJI_MAP_JOYPIXELS, } from "
|
|
2
|
+
import { EMOJI_SET, EMOJI_MAP_CLDR, EMOJI_MAP_GITHUB, EMOJI_MAP_IAMCAL, EMOJI_MAP_JOYPIXELS, } from "../data/emoji-data-gen.js";
|
|
3
|
+
// VS16 (U+FE0F) and VS15 (U+FE0E) - variation selectors that cause lookup mismatches
|
|
4
|
+
const VS16 = "\uFE0F";
|
|
5
|
+
const VS15 = "\uFE0E";
|
|
6
|
+
// Non-emoji glyphs that should still be stripped (not in Unicode Emoji property)
|
|
7
|
+
// U+1F241 = Tortoise Shell Bracketed CJK Unified Ideograph-4E09
|
|
8
|
+
// U+1F250 = Circled Ideograph Advantage
|
|
9
|
+
// U+1F251 = Circled Ideograph Accept
|
|
10
|
+
export const EXTRA_STRIPPABLE = new Set([
|
|
11
|
+
"\u{1F241}", // 🉁
|
|
12
|
+
"\u{1F250}", // 🉐
|
|
13
|
+
"\u{1F251}", // 🉑
|
|
14
|
+
]);
|
|
15
|
+
// Normalize emoji by removing variation selectors (VS16/VS15)
|
|
16
|
+
// Used as fallback when direct Set lookup fails
|
|
17
|
+
export function normalizeEmoji(grapheme) {
|
|
18
|
+
if (!grapheme.includes(VS16) && !grapheme.includes(VS15)) {
|
|
19
|
+
return grapheme;
|
|
20
|
+
}
|
|
21
|
+
return grapheme.replaceAll(VS16, "").replaceAll(VS15, "");
|
|
22
|
+
}
|
|
23
|
+
export function isStrippable(grapheme, emojiSet) {
|
|
24
|
+
// Direct lookup (fast path)
|
|
25
|
+
if (emojiSet.has(grapheme)) {
|
|
26
|
+
return true;
|
|
27
|
+
}
|
|
28
|
+
// Check extra strippables
|
|
29
|
+
if (EXTRA_STRIPPABLE.has(grapheme)) {
|
|
30
|
+
return true;
|
|
31
|
+
}
|
|
32
|
+
// Fallback: normalize and retry (handles VS16/VS15 mismatches)
|
|
33
|
+
const normalized = normalizeEmoji(grapheme);
|
|
34
|
+
if (normalized !== grapheme && emojiSet.has(normalized)) {
|
|
35
|
+
return true;
|
|
36
|
+
}
|
|
37
|
+
return false;
|
|
38
|
+
}
|
|
3
39
|
function _getShortcodePreset(style) {
|
|
4
40
|
if (style === "slack")
|
|
5
41
|
return "iamcal";
|
|
@@ -41,7 +77,15 @@ export function getEmojiMapSync(replacementStyle) {
|
|
|
41
77
|
return ok(map);
|
|
42
78
|
}
|
|
43
79
|
export function getEmojiShortcode(emoji, map) {
|
|
44
|
-
|
|
80
|
+
const direct = map.get(emoji);
|
|
81
|
+
if (direct) {
|
|
82
|
+
return direct;
|
|
83
|
+
}
|
|
84
|
+
const normalized = normalizeEmoji(emoji);
|
|
85
|
+
if (normalized !== emoji) {
|
|
86
|
+
return map.get(normalized) ?? null;
|
|
87
|
+
}
|
|
88
|
+
return null;
|
|
45
89
|
}
|
|
46
90
|
function generateHexFallback(emoji) {
|
|
47
91
|
const codepoints = [];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"emoji-data.js","sourceRoot":"","sources":["../../src/core/emoji-data.ts"],"names":[],"mappings":"AAAA,OAAO,EAAe,EAAE,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAElD,OAAO,EACN,SAAS,EACT,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,mBAAmB,GACnB,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"emoji-data.js","sourceRoot":"","sources":["../../src/core/emoji-data.ts"],"names":[],"mappings":"AAAA,OAAO,EAAe,EAAE,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAElD,OAAO,EACN,SAAS,EACT,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,mBAAmB,GACnB,MAAM,2BAA2B,CAAC;AAEnC,qFAAqF;AACrF,MAAM,IAAI,GAAG,QAAQ,CAAC;AACtB,MAAM,IAAI,GAAG,QAAQ,CAAC;AAEtB,iFAAiF;AACjF,gEAAgE;AAChE,wCAAwC;AACxC,qCAAqC;AACrC,MAAM,CAAC,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAS;IAC/C,WAAW,EAAE,KAAK;IAClB,WAAW,EAAE,KAAK;IAClB,WAAW,EAAE,KAAK;CAClB,CAAC,CAAC;AAEH,8DAA8D;AAC9D,gDAAgD;AAChD,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC9C,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1D,OAAO,QAAQ,CAAC;IACjB,CAAC;IACD,OAAO,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;AAC3D,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,QAAgB,EAAE,QAAqB;IACnE,4BAA4B;IAC5B,IAAI,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,0BAA0B;IAC1B,IAAI,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpC,OAAO,IAAI,CAAC;IACb,CAAC;IACD,+DAA+D;IAC/D,MAAM,UAAU,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IAC5C,IAAI,UAAU,KAAK,QAAQ,IAAI,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;QACzD,OAAO,IAAI,CAAC;IACb,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC;AAYD,SAAS,mBAAmB,CAAC,KAAuB;IACnD,IAAI,KAAK,KAAK,OAAO;QAAE,OAAO,QAAQ,CAAC;IACvC,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,WAAW,CAAC;IAC5C,OAAO,KAAK,CAAC;AACd,CAAC;AAED,MAAM,aAAa,GAAiD;IACnE,MAAM,EAAE,gBAAgB;IACxB,IAAI,EAAE,cAAc;IACpB,MAAM,EAAE,gBAAgB;IACxB,SAAS,EAAE,mBAAmB;CACrB,CAAC;AAEX,MAAM,UAAU,eAAe;IAC9B,IAAI,SAAS,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,GAAG,CAAC;YACV,MAAM,EAAE,YAAqB;YAC7B,IAAI,EAAE,eAAwB;SAC9B,CAAC,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,CAAC,SAAS,CAAC,CAAC;AACtB,CAAC;AAED,MAAM,UAAU,eAAe,CAC9B,gBAAkC;IAElC,MAAM,MAAM,GAAG,mBAAmB,CAAC,gBAAgB,CAAC,CAAC;IACrD,MAAM,GAAG,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IAElC,IAAI,CAAC,GAAG,EAAE,CAAC;QACV,OAAO,GAAG,CAAC;YACV,MAAM,EAAE,YAAqB;YAC7B,IAAI,EAAE,cAAuB;YAC7B,OAAO,EAAE,oCAAoC,MAAM,EAAE;SACrD,CAAC,CAAC;IACJ,CAAC;IAED,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACpB,OAAO,GAAG,CAAC;YACV,MAAM,EAAE,YAAqB;YAC7B,IAAI,EAAE,eAAwB;SAC9B,CAAC,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,iBAAiB,CAChC,KAAa,EACb,GAAwB;IAExB,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC9B,IAAI,MAAM,EAAE,CAAC;QACZ,OAAO,MAAM,CAAC;IACf,CAAC;IACD,MAAM,UAAU,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IACzC,IAAI,UAAU,KAAK,KAAK,EAAE,CAAC;QAC1B,OAAO,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC;IACpC,CAAC;IACD,OAAO,IAAI,CAAC;AACb,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAa;IACzC,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;QACxB,MAAM,EAAE,GAAG,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;QAC7B,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;YACtB,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACzD,CAAC;IACF,CAAC;IACD,OAAO,SAAS,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;AACxC,CAAC;AAED,MAAM,UAAU,cAAc,CAC7B,KAAa,EACb,MAAwC,EACxC,GAAwB;IAExB,MAAM,SAAS,GAAG,iBAAiB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,SAAS,IAAI,mBAAmB,CAAC,KAAK,CAAC,CAAC;IACrD,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AACrC,CAAC"}
|
package/dist/core/processor.d.ts
CHANGED
|
@@ -1,5 +1,16 @@
|
|
|
1
1
|
import type { UnmojiConfig } from "../types/config.js";
|
|
2
2
|
import type { Result } from "neverthrow";
|
|
3
3
|
import type { ProcessingError } from "../types/errors.js";
|
|
4
|
-
export
|
|
4
|
+
export type ProcessStats = {
|
|
5
|
+
removed: number;
|
|
6
|
+
lines: number[];
|
|
7
|
+
keptInlineCode: number;
|
|
8
|
+
keptCodeBlock: number;
|
|
9
|
+
keptQuote: number;
|
|
10
|
+
};
|
|
11
|
+
export type ProcessResult = {
|
|
12
|
+
text: string;
|
|
13
|
+
stats: ProcessStats;
|
|
14
|
+
};
|
|
15
|
+
export declare function processText(text: string, config: UnmojiConfig): Result<ProcessResult, ProcessingError>;
|
|
5
16
|
//# sourceMappingURL=processor.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"processor.d.ts","sourceRoot":"","sources":["../../src/core/processor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"processor.d.ts","sourceRoot":"","sources":["../../src/core/processor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAgB1D,MAAM,MAAM,YAAY,GAAG;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,YAAY,CAAC;CACpB,CAAC;AAEF,wBAAgB,WAAW,CAC1B,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,YAAY,GAClB,MAAM,CAAC,aAAa,EAAE,eAAe,CAAC,CA0HxC"}
|
package/dist/core/processor.js
CHANGED
|
@@ -1,10 +1,19 @@
|
|
|
1
1
|
import { ok, err } from "neverthrow";
|
|
2
|
-
import { detectCodeBlocks, detectQuotes,
|
|
2
|
+
import { detectCodeBlocks, detectQuotes, isInPreservedRange, } from "../utils/code-detector.js";
|
|
3
3
|
import { graphemes, mightContainEmoji } from "../utils/grapheme.js";
|
|
4
|
-
import { getEmojiSetSync, getEmojiMapSync, getReplacement, } from "./emoji-data.js";
|
|
4
|
+
import { getEmojiSetSync, getEmojiMapSync, getReplacement, isStrippable, } from "./emoji-data.js";
|
|
5
5
|
export function processText(text, config) {
|
|
6
6
|
if (!text || !mightContainEmoji(text)) {
|
|
7
|
-
return ok(
|
|
7
|
+
return ok({
|
|
8
|
+
text,
|
|
9
|
+
stats: {
|
|
10
|
+
removed: 0,
|
|
11
|
+
lines: [],
|
|
12
|
+
keptInlineCode: 0,
|
|
13
|
+
keptCodeBlock: 0,
|
|
14
|
+
keptQuote: 0,
|
|
15
|
+
},
|
|
16
|
+
});
|
|
8
17
|
}
|
|
9
18
|
const emojiSetResult = getEmojiSetSync();
|
|
10
19
|
if (emojiSetResult.isErr()) {
|
|
@@ -28,33 +37,79 @@ export function processText(text, config) {
|
|
|
28
37
|
}
|
|
29
38
|
emojiMap = emojiMapResult.value;
|
|
30
39
|
}
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
40
|
+
// Detect ranges separately to track kept emojis by type
|
|
41
|
+
const codeBlockRanges = config.preserveCodeBlocks
|
|
42
|
+
? detectCodeBlocks(text, {
|
|
43
|
+
preserveCodeBlocks: true,
|
|
44
|
+
preserveInlineCode: false,
|
|
45
|
+
})
|
|
46
|
+
: [];
|
|
47
|
+
const inlineCodeRanges = config.preserveInlineCode
|
|
48
|
+
? detectCodeBlocks(text, {
|
|
49
|
+
preserveCodeBlocks: false,
|
|
50
|
+
preserveInlineCode: true,
|
|
51
|
+
})
|
|
52
|
+
: [];
|
|
35
53
|
const quoteRanges = detectQuotes(text, {
|
|
36
54
|
preserveBlockquotes: config.skipQuotes,
|
|
37
55
|
preserveInlineQuotes: config.skipQuotes,
|
|
38
56
|
});
|
|
39
|
-
const preservedRanges = mergeRanges([codeRanges, quoteRanges]);
|
|
40
57
|
const chunks = [];
|
|
41
58
|
let currentIndex = 0;
|
|
59
|
+
let currentLine = 1;
|
|
60
|
+
let didChange = false;
|
|
61
|
+
let removedCount = 0;
|
|
62
|
+
let keptInlineCode = 0;
|
|
63
|
+
let keptCodeBlock = 0;
|
|
64
|
+
let keptQuote = 0;
|
|
65
|
+
const linesWithEmojisSet = new Set();
|
|
42
66
|
for (const grapheme of graphemes(text)) {
|
|
43
67
|
const graphemeLength = grapheme.length;
|
|
44
|
-
const
|
|
45
|
-
|
|
68
|
+
const inCodeBlock = isInPreservedRange(currentIndex, codeBlockRanges);
|
|
69
|
+
const inInlineCode = isInPreservedRange(currentIndex, inlineCodeRanges);
|
|
70
|
+
const inQuote = isInPreservedRange(currentIndex, quoteRanges);
|
|
71
|
+
const inPreserved = inCodeBlock || inInlineCode || inQuote;
|
|
72
|
+
const strippable = isStrippable(grapheme, emojiSet);
|
|
73
|
+
if (inPreserved || !strippable) {
|
|
46
74
|
chunks.push(grapheme);
|
|
75
|
+
// Count preserved emojis by type
|
|
76
|
+
if (inPreserved && strippable) {
|
|
77
|
+
if (inCodeBlock) {
|
|
78
|
+
keptCodeBlock++;
|
|
79
|
+
}
|
|
80
|
+
else if (inInlineCode) {
|
|
81
|
+
keptInlineCode++;
|
|
82
|
+
}
|
|
83
|
+
else if (inQuote) {
|
|
84
|
+
keptQuote++;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
47
87
|
}
|
|
48
88
|
else {
|
|
89
|
+
didChange = true;
|
|
90
|
+
removedCount++;
|
|
91
|
+
linesWithEmojisSet.add(currentLine);
|
|
49
92
|
if (config.mode === "replace" && emojiMap) {
|
|
50
93
|
const format = config.replacementFormat;
|
|
51
94
|
const replacement = getReplacement(grapheme, format, emojiMap);
|
|
52
95
|
chunks.push(replacement);
|
|
53
96
|
}
|
|
54
97
|
}
|
|
55
|
-
|
|
98
|
+
if (grapheme === "\n") {
|
|
99
|
+
currentLine++;
|
|
100
|
+
}
|
|
56
101
|
currentIndex += graphemeLength;
|
|
57
102
|
}
|
|
58
|
-
|
|
103
|
+
const stats = {
|
|
104
|
+
removed: removedCount,
|
|
105
|
+
lines: Array.from(linesWithEmojisSet),
|
|
106
|
+
keptInlineCode,
|
|
107
|
+
keptCodeBlock,
|
|
108
|
+
keptQuote,
|
|
109
|
+
};
|
|
110
|
+
if (!didChange) {
|
|
111
|
+
return ok({ text, stats });
|
|
112
|
+
}
|
|
113
|
+
return ok({ text: chunks.join(""), stats });
|
|
59
114
|
}
|
|
60
115
|
//# sourceMappingURL=processor.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"processor.js","sourceRoot":"","sources":["../../src/core/processor.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EACN,gBAAgB,EAChB,YAAY,EACZ,
|
|
1
|
+
{"version":3,"file":"processor.js","sourceRoot":"","sources":["../../src/core/processor.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EACN,gBAAgB,EAChB,YAAY,EACZ,kBAAkB,GAElB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACpE,OAAO,EACN,eAAe,EACf,eAAe,EACf,cAAc,EACd,YAAY,GACZ,MAAM,iBAAiB,CAAC;AAezB,MAAM,UAAU,WAAW,CAC1B,IAAY,EACZ,MAAoB;IAEpB,IAAI,CAAC,IAAI,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC;QACvC,OAAO,EAAE,CAAC;YACT,IAAI;YACJ,KAAK,EAAE;gBACN,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,EAAE;gBACT,cAAc,EAAE,CAAC;gBACjB,aAAa,EAAE,CAAC;gBAChB,SAAS,EAAE,CAAC;aACZ;SACD,CAAC,CAAC;IACJ,CAAC;IAED,MAAM,cAAc,GAAG,eAAe,EAAE,CAAC;IAEzC,IAAI,cAAc,CAAC,KAAK,EAAE,EAAE,CAAC;QAC5B,OAAO,GAAG,CAAC;YACV,MAAM,EAAE,YAAqB;YAC7B,IAAI,EAAE,wBAAiC;YACvC,KAAK,EAAE,cAAc,CAAC,KAAK;SAC3B,CAAC,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,CAAC;IAEtC,IAAI,QAAQ,GAA+B,IAAI,CAAC;IAChD,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,MAAM,CAAC,gBAAgB,IAAI,QAAQ,CAAC;QAClD,MAAM,cAAc,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAE9C,IAAI,cAAc,CAAC,KAAK,EAAE,EAAE,CAAC;YAC5B,OAAO,GAAG,CAAC;gBACV,MAAM,EAAE,YAAqB;gBAC7B,IAAI,EAAE,wBAAiC;gBACvC,KAAK,EAAE,cAAc,CAAC,KAAK;aAC3B,CAAC,CAAC;QACJ,CAAC;QAED,QAAQ,GAAG,cAAc,CAAC,KAAK,CAAC;IACjC,CAAC;IAED,wDAAwD;IACxD,MAAM,eAAe,GAAgB,MAAM,CAAC,kBAAkB;QAC7D,CAAC,CAAC,gBAAgB,CAAC,IAAI,EAAE;YACvB,kBAAkB,EAAE,IAAI;YACxB,kBAAkB,EAAE,KAAK;SACzB,CAAC;QACH,CAAC,CAAC,EAAE,CAAC;IAEN,MAAM,gBAAgB,GAAgB,MAAM,CAAC,kBAAkB;QAC9D,CAAC,CAAC,gBAAgB,CAAC,IAAI,EAAE;YACvB,kBAAkB,EAAE,KAAK;YACzB,kBAAkB,EAAE,IAAI;SACxB,CAAC;QACH,CAAC,CAAC,EAAE,CAAC;IAEN,MAAM,WAAW,GAAG,YAAY,CAAC,IAAI,EAAE;QACtC,mBAAmB,EAAE,MAAM,CAAC,UAAU;QACtC,oBAAoB,EAAE,MAAM,CAAC,UAAU;KACvC,CAAC,CAAC;IAEH,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAU,CAAC;IAE7C,KAAK,MAAM,QAAQ,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,MAAM,cAAc,GAAG,QAAQ,CAAC,MAAM,CAAC;QACvC,MAAM,WAAW,GAAG,kBAAkB,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;QACtE,MAAM,YAAY,GAAG,kBAAkB,CAAC,YAAY,EAAE,gBAAgB,CAAC,CAAC;QACxE,MAAM,OAAO,GAAG,kBAAkB,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;QAC9D,MAAM,WAAW,GAAG,WAAW,IAAI,YAAY,IAAI,OAAO,CAAC;QAC3D,MAAM,UAAU,GAAG,YAAY,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAEpD,IAAI,WAAW,IAAI,CAAC,UAAU,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACtB,iCAAiC;YACjC,IAAI,WAAW,IAAI,UAAU,EAAE,CAAC;gBAC/B,IAAI,WAAW,EAAE,CAAC;oBACjB,aAAa,EAAE,CAAC;gBACjB,CAAC;qBAAM,IAAI,YAAY,EAAE,CAAC;oBACzB,cAAc,EAAE,CAAC;gBAClB,CAAC;qBAAM,IAAI,OAAO,EAAE,CAAC;oBACpB,SAAS,EAAE,CAAC;gBACb,CAAC;YACF,CAAC;QACF,CAAC;aAAM,CAAC;YACP,SAAS,GAAG,IAAI,CAAC;YACjB,YAAY,EAAE,CAAC;YACf,kBAAkB,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YACpC,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,IAAI,QAAQ,EAAE,CAAC;gBAC3C,MAAM,MAAM,GAAG,MAAM,CAAC,iBAAiB,CAAC;gBACxC,MAAM,WAAW,GAAG,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;gBAC/D,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC1B,CAAC;QACF,CAAC;QAED,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;YACvB,WAAW,EAAE,CAAC;QACf,CAAC;QACD,YAAY,IAAI,cAAc,CAAC;IAChC,CAAC;IAED,MAAM,KAAK,GAAiB;QAC3B,OAAO,EAAE,YAAY;QACrB,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC;QACrC,cAAc;QACd,aAAa;QACb,SAAS;KACT,CAAC;IAEF,IAAI,CAAC,SAAS,EAAE,CAAC;QAChB,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5B,CAAC;IAED,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;AAC7C,CAAC"}
|
|
@@ -3,4 +3,4 @@ export declare const EMOJI_MAP_CLDR: Map<string, string>;
|
|
|
3
3
|
export declare const EMOJI_MAP_GITHUB: Map<string, string>;
|
|
4
4
|
export declare const EMOJI_MAP_IAMCAL: Map<string, string>;
|
|
5
5
|
export declare const EMOJI_MAP_JOYPIXELS: Map<string, string>;
|
|
6
|
-
//# sourceMappingURL=emoji-data-
|
|
6
|
+
//# sourceMappingURL=emoji-data-gen.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"emoji-data-gen.d.ts","sourceRoot":"","sources":["../../src/data/emoji-data-gen.ts"],"names":[],"mappings":"AAGA,eAAO,MAAM,SAAS,aA8+LpB,CAAC;AAEH,eAAO,MAAM,cAAc,qBA8+LzB,CAAC;AAEH,eAAO,MAAM,gBAAgB,qBA8+L3B,CAAC;AAEH,eAAO,MAAM,gBAAgB,qBA8+L3B,CAAC;AAEH,eAAO,MAAM,mBAAmB,qBA8+L9B,CAAC"}
|