@bastiangx/opencode-unmoji 0.1.1 → 0.3.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -52
- package/dist/core/emoji-data.d.ts +3 -0
- package/dist/core/emoji-data.d.ts.map +1 -1
- package/dist/core/emoji-data.js +46 -2
- package/dist/core/emoji-data.js.map +1 -1
- package/dist/core/processor.d.ts +12 -1
- package/dist/core/processor.d.ts.map +1 -1
- package/dist/core/processor.js +67 -12
- package/dist/core/processor.js.map +1 -1
- package/dist/{core/emoji-data-generated.d.ts → data/emoji-data-gen.d.ts} +1 -1
- package/dist/data/emoji-data-gen.d.ts.map +1 -0
- package/dist/{core/emoji-data-generated.js → data/emoji-data-gen.js} +7646 -5371
- package/dist/data/emoji-data-gen.js.map +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +153 -4
- package/dist/index.js.map +1 -1
- package/dist/plugin.bundled.js +7922 -5436
- package/dist/types/config.d.ts +12 -0
- package/dist/types/config.d.ts.map +1 -1
- package/dist/types/config.js +8 -0
- package/dist/types/config.js.map +1 -1
- package/dist/utils/code-detector.d.ts.map +1 -1
- package/dist/utils/code-detector.js +7 -1
- package/dist/utils/code-detector.js.map +1 -1
- package/dist/utils/grapheme.d.ts.map +1 -1
- package/dist/utils/grapheme.js +20 -2
- package/dist/utils/grapheme.js.map +1 -1
- package/package.json +20 -7
- package/dist/core/emoji-data-generated.d.ts.map +0 -1
- package/dist/core/emoji-data-generated.js.map +0 -1
- package/dist/streaming/hooks.d.ts +0 -12
- package/dist/streaming/hooks.d.ts.map +0 -1
- package/dist/streaming/hooks.js +0 -8
- package/dist/streaming/hooks.js.map +0 -1
- package/dist/streaming/sanitizer.d.ts +0 -66
- package/dist/streaming/sanitizer.d.ts.map +0 -1
- package/dist/streaming/sanitizer.js +0 -118
- package/dist/streaming/sanitizer.js.map +0 -1
- package/dist/streaming/state.d.ts +0 -10
- package/dist/streaming/state.d.ts.map +0 -1
- package/dist/streaming/state.js +0 -33
- package/dist/streaming/state.js.map +0 -1
- package/dist/utils/debug.d.ts +0 -18
- package/dist/utils/debug.d.ts.map +0 -1
- package/dist/utils/debug.js +0 -10
- package/dist/utils/debug.js.map +0 -1
package/README.md
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
<div align="center">
|
|
2
2
|
<a href="https://codeberg.org/bastiangx/opencode-unmoji">
|
|
3
|
-
<img
|
|
3
|
+
<img src="https://github.com/user-attachments/assets/01c5fa35-036d-4d7a-9feb-8becd744a5af"/>
|
|
4
4
|
</a>
|
|
5
5
|
</div>
|
|
6
6
|
|
|
7
7
|
# UNMOJI
|
|
8
8
|
|
|
9
|
-
<a href="https://
|
|
9
|
+
<a href="https://npmjs.com/package/@bastiangx/opencode-unmoji"><img src="https://img.shields.io/badge/npm-black?style=for-the-badge&logo=npm&logoSize=auto&labelColor=%23141010&color=%231F1A1A" alt="NPM Package"></a> <a href="https://npmjs.com/package/@bastiangx/opencode-unmoji"><img src="https://img.shields.io/npm/v/%40bastiangx%2Fopencode-unmoji?style=for-the-badge&label=version&labelColor=%23141010&color=%231F1A1A" alt="NPM Version"></a> <a href="https://npmjs.com/package/@bastiangx/opencode-unmoji"><img src="https://img.shields.io/npm/dm/%40bastiangx%2Fopencode-unmoji?style=for-the-badge&label=downloads&labelColor=%23141010&color=%231F1A1A" alt="NPM Downloads"></a> <a href="https://opencode.cafe/plugin/opencode-unmoji"><img src="https://img.shields.io/badge/opnecode_cafe-black?style=for-the-badge&logo=awesomelists&logoSize=auto&labelColor=%23141010&color=%231F1A1A" alt="OpenCode Cafe"></a> <a href="https://www.codeberg.org/bastiangx/opencode-unmoji"><img src="https://img.shields.io/badge/codeberg-black?style=for-the-badge&logo=codeberg&logoSize=auto&labelColor=%23141010&color=%231F1A1A" alt="codeberg"></a>
|
|
10
10
|
|
|
11
|
-
An [OpenCode](https://opencode.ai) plugin that strips
|
|
11
|
+
An [OpenCode](https://opencode.ai) plugin that strips _all kinds of_ emojis from your agent outputs.
|
|
12
12
|
|
|
13
|
-
<img height="
|
|
13
|
+
<img height="700" src="https://github.com/user-attachments/assets/fe606d89-01b0-4da6-81b7-5014d6aff972"/>
|
|
14
14
|
|
|
15
15
|
## Installation
|
|
16
16
|
|
|
@@ -33,22 +33,29 @@ Add the following to your `opencode.json` file:
|
|
|
33
33
|
|
|
34
34
|
UNMOJI automatically removes _all_ _standard_ unicode emojis and glyphs from your agent outputs.
|
|
35
35
|
|
|
36
|
-
|
|
36
|
+
Especially useful when you are using models like Claude which seem to _LOVE_ spamming emojis on summaries, README files and various documentations.
|
|
37
37
|
|
|
38
|
-
- ~
|
|
38
|
+
- ~ 30,000 emojis covered.
|
|
39
39
|
|
|
40
|
-
|
|
40
|
+
### Output targets
|
|
41
41
|
|
|
42
|
-
|
|
42
|
+
1. _Regular_ agent text and summaries in the main ui thread (format agnostic).
|
|
43
|
+
2. `newString` (new lines) in markdown files (.md, .mdx, .mdc, .markdown) when an agent calls `edit` tool.
|
|
44
|
+
3. Whole markdown file content when the agent calls `write` tool.
|
|
45
|
+
|
|
46
|
+
> _Thinking_ tokens as output targets are not supported at this time.
|
|
43
47
|
|
|
44
|
-
|
|
48
|
+
## Configuration
|
|
45
49
|
|
|
46
|
-
|
|
50
|
+
Create and add the `unmoji.json` file to your OpenCode [config path](https://opencode.ai/docs/config/).
|
|
47
51
|
|
|
48
52
|
```json
|
|
49
53
|
{
|
|
50
54
|
"enabled": true,
|
|
51
55
|
"mode": "strip",
|
|
56
|
+
"processMarkdown": true,
|
|
57
|
+
"markdownExtensions": [".md"],
|
|
58
|
+
"showNotification": true,
|
|
52
59
|
"preserveCodeBlocks": true,
|
|
53
60
|
"preserveInlineCode": true,
|
|
54
61
|
"skipQuotes": false,
|
|
@@ -57,60 +64,67 @@ Create and add the `unmoji.json` file to your OpenCode [config path](https://ope
|
|
|
57
64
|
}
|
|
58
65
|
```
|
|
59
66
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
|
63
|
-
|
|
|
64
|
-
| `
|
|
65
|
-
| `
|
|
66
|
-
| `
|
|
67
|
-
| `
|
|
68
|
-
| `
|
|
67
|
+
#### Full options spec
|
|
68
|
+
|
|
69
|
+
| Option | Default | Possible Values | Desc |
|
|
70
|
+
| :------------------- | :--------: | :---------------------------------------------------------- | :----------------------------------------------------- |
|
|
71
|
+
| `enabled` | `true` | `true`, `false` | Enable / disable the plugin without uninstalling it |
|
|
72
|
+
| `mode` | `strip` | `strip`, `replace` | `strip` removes emojis, `replace` puts shortcodes |
|
|
73
|
+
| `processMarkdown` | `true` | `true`, `false` | Process emojis in markdown files written by tools |
|
|
74
|
+
| `markdownExtensions` | `[".md"]` | `[".md", ".mdx", ".mdc", ".markdown"]` | File extensions treated as markdown for tool hooks |
|
|
75
|
+
| `showNotification` | `true` | `true`, `false` | Shown when emojis are removed |
|
|
76
|
+
| `preserveCodeBlocks` | `true` | `true`, `false` | Do not process emojis inside code blocks |
|
|
77
|
+
| `preserveInlineCode` | `true` | `true`, `false` | Do not process emojis inside inline code |
|
|
78
|
+
| `skipQuotes` | `false` | `true`, `false` | Do not process emojis inside quoted text blocks |
|
|
79
|
+
| `replacementStyle` | `github` | `github`, `slack`, `discord`, `cldr`, `iamcal`, `joypixels` | Shortcode style used when mode is `replace` |
|
|
80
|
+
| `replacementFormat` | `[:name:]` | `:name:`, `[:name:]`, `{name}` | Template format [literal string] for shortcode outputs |
|
|
69
81
|
|
|
70
|
-
|
|
82
|
+
---
|
|
71
83
|
|
|
72
|
-
|
|
84
|
+
## Contributing
|
|
73
85
|
|
|
74
|
-
|
|
75
|
-
{
|
|
76
|
-
"mode": "replace",
|
|
77
|
-
"replacementStyle": "github",
|
|
78
|
-
"replacementFormat": ":name:"
|
|
79
|
-
}
|
|
80
|
-
```
|
|
86
|
+
Contributions are welcome!
|
|
81
87
|
|
|
82
|
-
|
|
83
|
-
# Agent Output
|
|
84
|
-
Great work! 🎉 The tests are passing ✅
|
|
85
|
-
# Processed Output
|
|
86
|
-
Great work! :tada: The tests are passing :white_check_mark:
|
|
87
|
-
```
|
|
88
|
+
Follow the [Contributing guide](https://codeberg.org/bastiangx/opencode-unmoji/src/branch/main/CONTRIBUTING.md) for instructions.
|
|
88
89
|
|
|
89
|
-
|
|
90
|
+
---
|
|
90
91
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
92
|
+
---
|
|
93
|
+
|
|
94
|
+
License: MIT
|
|
95
|
+
|
|
96
|
+
## Emoji Test Section
|
|
97
|
+
|
|
98
|
+
This section contains various emojis for testing the stripping functionality .
|
|
99
|
+
|
|
100
|
+
### Heading with Emoji
|
|
98
101
|
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
102
|
+
Some text with multi-codepoint emoji:
|
|
103
|
+
|
|
104
|
+
- List item with skin tone emoji
|
|
105
|
+
- Another item with fire
|
|
106
|
+
|
|
107
|
+
```js
|
|
108
|
+
// Code block with emoji (should preserve)
|
|
109
|
+
console.log("Hello ");
|
|
104
110
|
```
|
|
105
111
|
|
|
106
|
-
|
|
112
|
+
Inline code with emoji: ` spark`
|
|
107
113
|
|
|
108
|
-
|
|
114
|
+
| Table | Column |
|
|
115
|
+
|-------|--------|
|
|
116
|
+
| Row | Data |
|
|
109
117
|
|
|
110
|
-
|
|
118
|
+
[Link with emoji ](https://example.com)
|
|
119
|
+
|
|
120
|
+
> Blockquote with emoji
|
|
121
|
+
|
|
122
|
+
Task list:
|
|
123
|
+
- [ ] Unchecked
|
|
124
|
+
- [x] Checked
|
|
111
125
|
|
|
112
|
-
|
|
126
|
+
Horizontal rule with adjacent emoji
|
|
113
127
|
|
|
114
128
|
---
|
|
115
129
|
|
|
116
|
-
|
|
130
|
+
More text with emoji
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import { type Result } from "neverthrow";
|
|
2
2
|
import type { EmojiDataError } from "../types/errors.js";
|
|
3
|
+
export declare const EXTRA_STRIPPABLE: Set<string>;
|
|
4
|
+
export declare function normalizeEmoji(grapheme: string): string;
|
|
5
|
+
export declare function isStrippable(grapheme: string, emojiSet: Set<string>): boolean;
|
|
3
6
|
export type ReplacementStyle = "github" | "slack" | "discord" | "cldr" | "iamcal" | "joypixels";
|
|
4
7
|
export declare function getEmojiSetSync(): Result<Set<string>, EmojiDataError>;
|
|
5
8
|
export declare function getEmojiMapSync(replacementStyle: ReplacementStyle): Result<Map<string, string>, EmojiDataError>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"emoji-data.d.ts","sourceRoot":"","sources":["../../src/core/emoji-data.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,MAAM,EAAW,MAAM,YAAY,CAAC;AAClD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"emoji-data.d.ts","sourceRoot":"","sources":["../../src/core/emoji-data.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,MAAM,EAAW,MAAM,YAAY,CAAC;AAClD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAiBzD,eAAO,MAAM,gBAAgB,aAI3B,CAAC;AAIH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAKvD;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,OAAO,CAe7E;AAED,MAAM,MAAM,gBAAgB,GACzB,QAAQ,GACR,OAAO,GACP,SAAS,GACT,MAAM,GACN,QAAQ,GACR,WAAW,CAAC;AAiBf,wBAAgB,eAAe,IAAI,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,cAAc,CAAC,CAQrE;AAED,wBAAgB,eAAe,CAC9B,gBAAgB,EAAE,gBAAgB,GAChC,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,cAAc,CAAC,CAoB7C;AAED,wBAAgB,iBAAiB,CAChC,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GACtB,MAAM,GAAG,IAAI,CAUf;AAaD,wBAAgB,cAAc,CAC7B,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,QAAQ,GAAG,UAAU,GAAG,QAAQ,EACxC,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GACtB,MAAM,CAIR"}
|
package/dist/core/emoji-data.js
CHANGED
|
@@ -1,5 +1,41 @@
|
|
|
1
1
|
import { ok, err } from "neverthrow";
|
|
2
|
-
import { EMOJI_SET, EMOJI_MAP_CLDR, EMOJI_MAP_GITHUB, EMOJI_MAP_IAMCAL, EMOJI_MAP_JOYPIXELS, } from "
|
|
2
|
+
import { EMOJI_SET, EMOJI_MAP_CLDR, EMOJI_MAP_GITHUB, EMOJI_MAP_IAMCAL, EMOJI_MAP_JOYPIXELS, } from "../data/emoji-data-gen.js";
|
|
3
|
+
// VS16 (U+FE0F) and VS15 (U+FE0E) - variation selectors that cause lookup mismatches
|
|
4
|
+
const VS16 = "\uFE0F";
|
|
5
|
+
const VS15 = "\uFE0E";
|
|
6
|
+
// Non-emoji glyphs that should still be stripped (not in Unicode Emoji property)
|
|
7
|
+
// U+1F241 = Tortoise Shell Bracketed CJK Unified Ideograph-4E09
|
|
8
|
+
// U+1F250 = Circled Ideograph Advantage
|
|
9
|
+
// U+1F251 = Circled Ideograph Accept
|
|
10
|
+
export const EXTRA_STRIPPABLE = new Set([
|
|
11
|
+
"\u{1F241}", // 🉁
|
|
12
|
+
"\u{1F250}", // 🉐
|
|
13
|
+
"\u{1F251}", // 🉑
|
|
14
|
+
]);
|
|
15
|
+
// Normalize emoji by removing variation selectors (VS16/VS15)
|
|
16
|
+
// Used as fallback when direct Set lookup fails
|
|
17
|
+
export function normalizeEmoji(grapheme) {
|
|
18
|
+
if (!grapheme.includes(VS16) && !grapheme.includes(VS15)) {
|
|
19
|
+
return grapheme;
|
|
20
|
+
}
|
|
21
|
+
return grapheme.replaceAll(VS16, "").replaceAll(VS15, "");
|
|
22
|
+
}
|
|
23
|
+
export function isStrippable(grapheme, emojiSet) {
|
|
24
|
+
// Direct lookup (fast path)
|
|
25
|
+
if (emojiSet.has(grapheme)) {
|
|
26
|
+
return true;
|
|
27
|
+
}
|
|
28
|
+
// Check extra strippables
|
|
29
|
+
if (EXTRA_STRIPPABLE.has(grapheme)) {
|
|
30
|
+
return true;
|
|
31
|
+
}
|
|
32
|
+
// Fallback: normalize and retry (handles VS16/VS15 mismatches)
|
|
33
|
+
const normalized = normalizeEmoji(grapheme);
|
|
34
|
+
if (normalized !== grapheme && emojiSet.has(normalized)) {
|
|
35
|
+
return true;
|
|
36
|
+
}
|
|
37
|
+
return false;
|
|
38
|
+
}
|
|
3
39
|
function _getShortcodePreset(style) {
|
|
4
40
|
if (style === "slack")
|
|
5
41
|
return "iamcal";
|
|
@@ -41,7 +77,15 @@ export function getEmojiMapSync(replacementStyle) {
|
|
|
41
77
|
return ok(map);
|
|
42
78
|
}
|
|
43
79
|
export function getEmojiShortcode(emoji, map) {
|
|
44
|
-
|
|
80
|
+
const direct = map.get(emoji);
|
|
81
|
+
if (direct) {
|
|
82
|
+
return direct;
|
|
83
|
+
}
|
|
84
|
+
const normalized = normalizeEmoji(emoji);
|
|
85
|
+
if (normalized !== emoji) {
|
|
86
|
+
return map.get(normalized) ?? null;
|
|
87
|
+
}
|
|
88
|
+
return null;
|
|
45
89
|
}
|
|
46
90
|
function generateHexFallback(emoji) {
|
|
47
91
|
const codepoints = [];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"emoji-data.js","sourceRoot":"","sources":["../../src/core/emoji-data.ts"],"names":[],"mappings":"AAAA,OAAO,EAAe,EAAE,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAElD,OAAO,EACN,SAAS,EACT,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,mBAAmB,GACnB,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"emoji-data.js","sourceRoot":"","sources":["../../src/core/emoji-data.ts"],"names":[],"mappings":"AAAA,OAAO,EAAe,EAAE,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAElD,OAAO,EACN,SAAS,EACT,cAAc,EACd,gBAAgB,EAChB,gBAAgB,EAChB,mBAAmB,GACnB,MAAM,2BAA2B,CAAC;AAEnC,qFAAqF;AACrF,MAAM,IAAI,GAAG,QAAQ,CAAC;AACtB,MAAM,IAAI,GAAG,QAAQ,CAAC;AAEtB,iFAAiF;AACjF,gEAAgE;AAChE,wCAAwC;AACxC,qCAAqC;AACrC,MAAM,CAAC,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAS;IAC/C,WAAW,EAAE,KAAK;IAClB,WAAW,EAAE,KAAK;IAClB,WAAW,EAAE,KAAK;CAClB,CAAC,CAAC;AAEH,8DAA8D;AAC9D,gDAAgD;AAChD,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC9C,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1D,OAAO,QAAQ,CAAC;IACjB,CAAC;IACD,OAAO,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;AAC3D,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,QAAgB,EAAE,QAAqB;IACnE,4BAA4B;IAC5B,IAAI,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC;IACb,CAAC;IACD,0BAA0B;IAC1B,IAAI,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpC,OAAO,IAAI,CAAC;IACb,CAAC;IACD,+DAA+D;IAC/D,MAAM,UAAU,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IAC5C,IAAI,UAAU,KAAK,QAAQ,IAAI,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;QACzD,OAAO,IAAI,CAAC;IACb,CAAC;IACD,OAAO,KAAK,CAAC;AACd,CAAC;AAYD,SAAS,mBAAmB,CAAC,KAAuB;IACnD,IAAI,KAAK,KAAK,OAAO;QAAE,OAAO,QAAQ,CAAC;IACvC,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,WAAW,CAAC;IAC5C,OAAO,KAAK,CAAC;AACd,CAAC;AAED,MAAM,aAAa,GAAiD;IACnE,MAAM,EAAE,gBAAgB;IACxB,IAAI,EAAE,cAAc;IACpB,MAAM,EAAE,gBAAgB;IACxB,SAAS,EAAE,mBAAmB;CACrB,CAAC;AAEX,MAAM,UAAU,eAAe;IAC9B,IAAI,SAAS,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,GAAG,CAAC;YACV,MAAM,EAAE,YAAqB;YAC7B,IAAI,EAAE,eAAwB;SAC9B,CAAC,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,CAAC,SAAS,CAAC,CAAC;AACtB,CAAC;AAED,MAAM,UAAU,eAAe,CAC9B,gBAAkC;IAElC,MAAM,MAAM,GAAG,mBAAmB,CAAC,gBAAgB,CAAC,CAAC;IACrD,MAAM,GAAG,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IAElC,IAAI,CAAC,GAAG,EAAE,CAAC;QACV,OAAO,GAAG,CAAC;YACV,MAAM,EAAE,YAAqB;YAC7B,IAAI,EAAE,cAAuB;YAC7B,OAAO,EAAE,oCAAoC,MAAM,EAAE;SACrD,CAAC,CAAC;IACJ,CAAC;IAED,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACpB,OAAO,GAAG,CAAC;YACV,MAAM,EAAE,YAAqB;YAC7B,IAAI,EAAE,eAAwB;SAC9B,CAAC,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,CAAC,GAAG,CAAC,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,iBAAiB,CAChC,KAAa,EACb,GAAwB;IAExB,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC9B,IAAI,MAAM,EAAE,CAAC;QACZ,OAAO,MAAM,CAAC;IACf,CAAC;IACD,MAAM,UAAU,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IACzC,IAAI,UAAU,KAAK,KAAK,EAAE,CAAC;QAC1B,OAAO,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC;IACpC,CAAC;IACD,OAAO,IAAI,CAAC;AACb,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAa;IACzC,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;QACxB,MAAM,EAAE,GAAG,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;QAC7B,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;YACtB,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACzD,CAAC;IACF,CAAC;IACD,OAAO,SAAS,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;AACxC,CAAC;AAED,MAAM,UAAU,cAAc,CAC7B,KAAa,EACb,MAAwC,EACxC,GAAwB;IAExB,MAAM,SAAS,GAAG,iBAAiB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,SAAS,IAAI,mBAAmB,CAAC,KAAK,CAAC,CAAC;IACrD,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AACrC,CAAC"}
|
package/dist/core/processor.d.ts
CHANGED
|
@@ -1,5 +1,16 @@
|
|
|
1
1
|
import type { UnmojiConfig } from "../types/config.js";
|
|
2
2
|
import type { Result } from "neverthrow";
|
|
3
3
|
import type { ProcessingError } from "../types/errors.js";
|
|
4
|
-
export
|
|
4
|
+
export type ProcessStats = {
|
|
5
|
+
removed: number;
|
|
6
|
+
lines: number[];
|
|
7
|
+
keptInlineCode: number;
|
|
8
|
+
keptCodeBlock: number;
|
|
9
|
+
keptQuote: number;
|
|
10
|
+
};
|
|
11
|
+
export type ProcessResult = {
|
|
12
|
+
text: string;
|
|
13
|
+
stats: ProcessStats;
|
|
14
|
+
};
|
|
15
|
+
export declare function processText(text: string, config: UnmojiConfig): Result<ProcessResult, ProcessingError>;
|
|
5
16
|
//# sourceMappingURL=processor.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"processor.d.ts","sourceRoot":"","sources":["../../src/core/processor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"processor.d.ts","sourceRoot":"","sources":["../../src/core/processor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAgB1D,MAAM,MAAM,YAAY,GAAG;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,YAAY,CAAC;CACpB,CAAC;AAEF,wBAAgB,WAAW,CAC1B,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,YAAY,GAClB,MAAM,CAAC,aAAa,EAAE,eAAe,CAAC,CA0HxC"}
|
package/dist/core/processor.js
CHANGED
|
@@ -1,10 +1,19 @@
|
|
|
1
1
|
import { ok, err } from "neverthrow";
|
|
2
|
-
import { detectCodeBlocks, detectQuotes,
|
|
2
|
+
import { detectCodeBlocks, detectQuotes, isInPreservedRange, } from "../utils/code-detector.js";
|
|
3
3
|
import { graphemes, mightContainEmoji } from "../utils/grapheme.js";
|
|
4
|
-
import { getEmojiSetSync, getEmojiMapSync, getReplacement, } from "./emoji-data.js";
|
|
4
|
+
import { getEmojiSetSync, getEmojiMapSync, getReplacement, isStrippable, } from "./emoji-data.js";
|
|
5
5
|
export function processText(text, config) {
|
|
6
6
|
if (!text || !mightContainEmoji(text)) {
|
|
7
|
-
return ok(
|
|
7
|
+
return ok({
|
|
8
|
+
text,
|
|
9
|
+
stats: {
|
|
10
|
+
removed: 0,
|
|
11
|
+
lines: [],
|
|
12
|
+
keptInlineCode: 0,
|
|
13
|
+
keptCodeBlock: 0,
|
|
14
|
+
keptQuote: 0,
|
|
15
|
+
},
|
|
16
|
+
});
|
|
8
17
|
}
|
|
9
18
|
const emojiSetResult = getEmojiSetSync();
|
|
10
19
|
if (emojiSetResult.isErr()) {
|
|
@@ -28,33 +37,79 @@ export function processText(text, config) {
|
|
|
28
37
|
}
|
|
29
38
|
emojiMap = emojiMapResult.value;
|
|
30
39
|
}
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
40
|
+
// Detect ranges separately to track kept emojis by type
|
|
41
|
+
const codeBlockRanges = config.preserveCodeBlocks
|
|
42
|
+
? detectCodeBlocks(text, {
|
|
43
|
+
preserveCodeBlocks: true,
|
|
44
|
+
preserveInlineCode: false,
|
|
45
|
+
})
|
|
46
|
+
: [];
|
|
47
|
+
const inlineCodeRanges = config.preserveInlineCode
|
|
48
|
+
? detectCodeBlocks(text, {
|
|
49
|
+
preserveCodeBlocks: false,
|
|
50
|
+
preserveInlineCode: true,
|
|
51
|
+
})
|
|
52
|
+
: [];
|
|
35
53
|
const quoteRanges = detectQuotes(text, {
|
|
36
54
|
preserveBlockquotes: config.skipQuotes,
|
|
37
55
|
preserveInlineQuotes: config.skipQuotes,
|
|
38
56
|
});
|
|
39
|
-
const preservedRanges = mergeRanges([codeRanges, quoteRanges]);
|
|
40
57
|
const chunks = [];
|
|
41
58
|
let currentIndex = 0;
|
|
59
|
+
let currentLine = 1;
|
|
60
|
+
let didChange = false;
|
|
61
|
+
let removedCount = 0;
|
|
62
|
+
let keptInlineCode = 0;
|
|
63
|
+
let keptCodeBlock = 0;
|
|
64
|
+
let keptQuote = 0;
|
|
65
|
+
const linesWithEmojisSet = new Set();
|
|
42
66
|
for (const grapheme of graphemes(text)) {
|
|
43
67
|
const graphemeLength = grapheme.length;
|
|
44
|
-
const
|
|
45
|
-
|
|
68
|
+
const inCodeBlock = isInPreservedRange(currentIndex, codeBlockRanges);
|
|
69
|
+
const inInlineCode = isInPreservedRange(currentIndex, inlineCodeRanges);
|
|
70
|
+
const inQuote = isInPreservedRange(currentIndex, quoteRanges);
|
|
71
|
+
const inPreserved = inCodeBlock || inInlineCode || inQuote;
|
|
72
|
+
const strippable = isStrippable(grapheme, emojiSet);
|
|
73
|
+
if (inPreserved || !strippable) {
|
|
46
74
|
chunks.push(grapheme);
|
|
75
|
+
// Count preserved emojis by type
|
|
76
|
+
if (inPreserved && strippable) {
|
|
77
|
+
if (inCodeBlock) {
|
|
78
|
+
keptCodeBlock++;
|
|
79
|
+
}
|
|
80
|
+
else if (inInlineCode) {
|
|
81
|
+
keptInlineCode++;
|
|
82
|
+
}
|
|
83
|
+
else if (inQuote) {
|
|
84
|
+
keptQuote++;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
47
87
|
}
|
|
48
88
|
else {
|
|
89
|
+
didChange = true;
|
|
90
|
+
removedCount++;
|
|
91
|
+
linesWithEmojisSet.add(currentLine);
|
|
49
92
|
if (config.mode === "replace" && emojiMap) {
|
|
50
93
|
const format = config.replacementFormat;
|
|
51
94
|
const replacement = getReplacement(grapheme, format, emojiMap);
|
|
52
95
|
chunks.push(replacement);
|
|
53
96
|
}
|
|
54
97
|
}
|
|
55
|
-
|
|
98
|
+
if (grapheme === "\n") {
|
|
99
|
+
currentLine++;
|
|
100
|
+
}
|
|
56
101
|
currentIndex += graphemeLength;
|
|
57
102
|
}
|
|
58
|
-
|
|
103
|
+
const stats = {
|
|
104
|
+
removed: removedCount,
|
|
105
|
+
lines: Array.from(linesWithEmojisSet),
|
|
106
|
+
keptInlineCode,
|
|
107
|
+
keptCodeBlock,
|
|
108
|
+
keptQuote,
|
|
109
|
+
};
|
|
110
|
+
if (!didChange) {
|
|
111
|
+
return ok({ text, stats });
|
|
112
|
+
}
|
|
113
|
+
return ok({ text: chunks.join(""), stats });
|
|
59
114
|
}
|
|
60
115
|
//# sourceMappingURL=processor.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"processor.js","sourceRoot":"","sources":["../../src/core/processor.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EACN,gBAAgB,EAChB,YAAY,EACZ,
|
|
1
|
+
{"version":3,"file":"processor.js","sourceRoot":"","sources":["../../src/core/processor.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EACN,gBAAgB,EAChB,YAAY,EACZ,kBAAkB,GAElB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACpE,OAAO,EACN,eAAe,EACf,eAAe,EACf,cAAc,EACd,YAAY,GACZ,MAAM,iBAAiB,CAAC;AAezB,MAAM,UAAU,WAAW,CAC1B,IAAY,EACZ,MAAoB;IAEpB,IAAI,CAAC,IAAI,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC;QACvC,OAAO,EAAE,CAAC;YACT,IAAI;YACJ,KAAK,EAAE;gBACN,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,EAAE;gBACT,cAAc,EAAE,CAAC;gBACjB,aAAa,EAAE,CAAC;gBAChB,SAAS,EAAE,CAAC;aACZ;SACD,CAAC,CAAC;IACJ,CAAC;IAED,MAAM,cAAc,GAAG,eAAe,EAAE,CAAC;IAEzC,IAAI,cAAc,CAAC,KAAK,EAAE,EAAE,CAAC;QAC5B,OAAO,GAAG,CAAC;YACV,MAAM,EAAE,YAAqB;YAC7B,IAAI,EAAE,wBAAiC;YACvC,KAAK,EAAE,cAAc,CAAC,KAAK;SAC3B,CAAC,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,CAAC;IAEtC,IAAI,QAAQ,GAA+B,IAAI,CAAC;IAChD,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,MAAM,CAAC,gBAAgB,IAAI,QAAQ,CAAC;QAClD,MAAM,cAAc,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAE9C,IAAI,cAAc,CAAC,KAAK,EAAE,EAAE,CAAC;YAC5B,OAAO,GAAG,CAAC;gBACV,MAAM,EAAE,YAAqB;gBAC7B,IAAI,EAAE,wBAAiC;gBACvC,KAAK,EAAE,cAAc,CAAC,KAAK;aAC3B,CAAC,CAAC;QACJ,CAAC;QAED,QAAQ,GAAG,cAAc,CAAC,KAAK,CAAC;IACjC,CAAC;IAED,wDAAwD;IACxD,MAAM,eAAe,GAAgB,MAAM,CAAC,kBAAkB;QAC7D,CAAC,CAAC,gBAAgB,CAAC,IAAI,EAAE;YACvB,kBAAkB,EAAE,IAAI;YACxB,kBAAkB,EAAE,KAAK;SACzB,CAAC;QACH,CAAC,CAAC,EAAE,CAAC;IAEN,MAAM,gBAAgB,GAAgB,MAAM,CAAC,kBAAkB;QAC9D,CAAC,CAAC,gBAAgB,CAAC,IAAI,EAAE;YACvB,kBAAkB,EAAE,KAAK;YACzB,kBAAkB,EAAE,IAAI;SACxB,CAAC;QACH,CAAC,CAAC,EAAE,CAAC;IAEN,MAAM,WAAW,GAAG,YAAY,CAAC,IAAI,EAAE;QACtC,mBAAmB,EAAE,MAAM,CAAC,UAAU;QACtC,oBAAoB,EAAE,MAAM,CAAC,UAAU;KACvC,CAAC,CAAC;IAEH,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAU,CAAC;IAE7C,KAAK,MAAM,QAAQ,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,MAAM,cAAc,GAAG,QAAQ,CAAC,MAAM,CAAC;QACvC,MAAM,WAAW,GAAG,kBAAkB,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;QACtE,MAAM,YAAY,GAAG,kBAAkB,CAAC,YAAY,EAAE,gBAAgB,CAAC,CAAC;QACxE,MAAM,OAAO,GAAG,kBAAkB,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;QAC9D,MAAM,WAAW,GAAG,WAAW,IAAI,YAAY,IAAI,OAAO,CAAC;QAC3D,MAAM,UAAU,GAAG,YAAY,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAEpD,IAAI,WAAW,IAAI,CAAC,UAAU,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACtB,iCAAiC;YACjC,IAAI,WAAW,IAAI,UAAU,EAAE,CAAC;gBAC/B,IAAI,WAAW,EAAE,CAAC;oBACjB,aAAa,EAAE,CAAC;gBACjB,CAAC;qBAAM,IAAI,YAAY,EAAE,CAAC;oBACzB,cAAc,EAAE,CAAC;gBAClB,CAAC;qBAAM,IAAI,OAAO,EAAE,CAAC;oBACpB,SAAS,EAAE,CAAC;gBACb,CAAC;YACF,CAAC;QACF,CAAC;aAAM,CAAC;YACP,SAAS,GAAG,IAAI,CAAC;YACjB,YAAY,EAAE,CAAC;YACf,kBAAkB,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YACpC,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,IAAI,QAAQ,EAAE,CAAC;gBAC3C,MAAM,MAAM,GAAG,MAAM,CAAC,iBAAiB,CAAC;gBACxC,MAAM,WAAW,GAAG,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;gBAC/D,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC1B,CAAC;QACF,CAAC;QAED,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;YACvB,WAAW,EAAE,CAAC;QACf,CAAC;QACD,YAAY,IAAI,cAAc,CAAC;IAChC,CAAC;IAED,MAAM,KAAK,GAAiB;QAC3B,OAAO,EAAE,YAAY;QACrB,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC;QACrC,cAAc;QACd,aAAa;QACb,SAAS;KACT,CAAC;IAEF,IAAI,CAAC,SAAS,EAAE,CAAC;QAChB,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5B,CAAC;IAED,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;AAC7C,CAAC"}
|
|
@@ -3,4 +3,4 @@ export declare const EMOJI_MAP_CLDR: Map<string, string>;
|
|
|
3
3
|
export declare const EMOJI_MAP_GITHUB: Map<string, string>;
|
|
4
4
|
export declare const EMOJI_MAP_IAMCAL: Map<string, string>;
|
|
5
5
|
export declare const EMOJI_MAP_JOYPIXELS: Map<string, string>;
|
|
6
|
-
//# sourceMappingURL=emoji-data-
|
|
6
|
+
//# sourceMappingURL=emoji-data-gen.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"emoji-data-gen.d.ts","sourceRoot":"","sources":["../../src/data/emoji-data-gen.ts"],"names":[],"mappings":"AAGA,eAAO,MAAM,SAAS,aA8+LpB,CAAC;AAEH,eAAO,MAAM,cAAc,qBA8+LzB,CAAC;AAEH,eAAO,MAAM,gBAAgB,qBA8+L3B,CAAC;AAEH,eAAO,MAAM,gBAAgB,qBA8+L3B,CAAC;AAEH,eAAO,MAAM,mBAAmB,qBA8+L9B,CAAC"}
|