humanizer-lint 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -5
- package/SKILL.md +7 -5
- package/package.json +3 -2
- package/rules/de-slop.json +190 -0
- package/rules/en-slop.json +50 -49
- package/rules/es-slop.json +190 -0
- package/rules/fr-slop.json +190 -0
- package/rules/id-slop.json +58 -57
- package/rules/ms-slop.json +190 -0
- package/rules/pt-slop.json +190 -0
- package/rules/zh-slop.json +190 -0
- package/scripts/analyze.mjs +9 -3
- package/scripts/core/analyzer-core.mjs +51 -24
- package/scripts/utils/tokenizer.mjs +149 -82
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
{
|
|
2
|
+
"language": "pt",
|
|
3
|
+
"stopwords": [
|
|
4
|
+
"de",
|
|
5
|
+
"a",
|
|
6
|
+
"o",
|
|
7
|
+
"que",
|
|
8
|
+
"e",
|
|
9
|
+
"do",
|
|
10
|
+
"da",
|
|
11
|
+
"em",
|
|
12
|
+
"um",
|
|
13
|
+
"para",
|
|
14
|
+
"é",
|
|
15
|
+
"com",
|
|
16
|
+
"não",
|
|
17
|
+
"uma",
|
|
18
|
+
"os",
|
|
19
|
+
"no",
|
|
20
|
+
"se",
|
|
21
|
+
"na",
|
|
22
|
+
"por",
|
|
23
|
+
"mais"
|
|
24
|
+
],
|
|
25
|
+
"patterns": [
|
|
26
|
+
{
|
|
27
|
+
"id": "PT001",
|
|
28
|
+
"type": "SLOP_PHRASE",
|
|
29
|
+
"pattern": "no\\s+mundo\\s+acelerado\\s+de\\s+hoje",
|
|
30
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
"id": "PT002",
|
|
34
|
+
"type": "SLOP_PHRASE",
|
|
35
|
+
"pattern": "vale\\s+ressaltar\\s+que",
|
|
36
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
"id": "PT003",
|
|
40
|
+
"type": "SLOP_PHRASE",
|
|
41
|
+
"pattern": "é\\s+importante\\s+notar\\s+que",
|
|
42
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
"id": "PT004",
|
|
46
|
+
"type": "SLOP_PHRASE",
|
|
47
|
+
"pattern": "não\\s+há\\s+como\\s+negar",
|
|
48
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
"id": "PT005",
|
|
52
|
+
"type": "SLOP_PHRASE",
|
|
53
|
+
"pattern": "em\\s+conclusão",
|
|
54
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
"id": "PT006",
|
|
58
|
+
"type": "SLOP_PHRASE",
|
|
59
|
+
"pattern": "em\\s+resumo",
|
|
60
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
61
|
+
},
|
|
62
|
+
{
|
|
63
|
+
"id": "PT007",
|
|
64
|
+
"type": "SLOP_PHRASE",
|
|
65
|
+
"pattern": "além\\s+disso",
|
|
66
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
67
|
+
},
|
|
68
|
+
{
|
|
69
|
+
"id": "PT008",
|
|
70
|
+
"type": "SLOP_PHRASE",
|
|
71
|
+
"pattern": "desempenha\\s+um\\s+papel\\s+crucial",
|
|
72
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
73
|
+
},
|
|
74
|
+
{
|
|
75
|
+
"id": "PT009",
|
|
76
|
+
"type": "SLOP_PHRASE",
|
|
77
|
+
"pattern": "uma\\s+tapeçaria\\s+de",
|
|
78
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
79
|
+
},
|
|
80
|
+
{
|
|
81
|
+
"id": "PT010",
|
|
82
|
+
"type": "SLOP_PHRASE",
|
|
83
|
+
"pattern": "mergulhar\\s+fundo",
|
|
84
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
85
|
+
},
|
|
86
|
+
{
|
|
87
|
+
"id": "PT011",
|
|
88
|
+
"type": "SLOP_PHRASE",
|
|
89
|
+
"pattern": "não\\s+apenas",
|
|
90
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
91
|
+
},
|
|
92
|
+
{
|
|
93
|
+
"id": "PT012",
|
|
94
|
+
"type": "SLOP_PHRASE",
|
|
95
|
+
"pattern": "mas\\s+também",
|
|
96
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
97
|
+
},
|
|
98
|
+
{
|
|
99
|
+
"id": "PT013",
|
|
100
|
+
"type": "SLOP_PHRASE",
|
|
101
|
+
"pattern": "na\\s+era\\s+digital",
|
|
102
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
103
|
+
},
|
|
104
|
+
{
|
|
105
|
+
"id": "PT014",
|
|
106
|
+
"type": "SLOP_PHRASE",
|
|
107
|
+
"pattern": "é\\s+fundamental\\s+lembrar",
|
|
108
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
109
|
+
},
|
|
110
|
+
{
|
|
111
|
+
"id": "PT015",
|
|
112
|
+
"type": "SLOP_PHRASE",
|
|
113
|
+
"pattern": "sem\\s+dúvida",
|
|
114
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
115
|
+
},
|
|
116
|
+
{
|
|
117
|
+
"id": "PT016",
|
|
118
|
+
"type": "SLOP_PHRASE",
|
|
119
|
+
"pattern": "no\\s+cenário\\s+atual",
|
|
120
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
121
|
+
},
|
|
122
|
+
{
|
|
123
|
+
"id": "PT017",
|
|
124
|
+
"type": "SLOP_PHRASE",
|
|
125
|
+
"pattern": "ao\\s+longo\\s+do\\s+tempo",
|
|
126
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
127
|
+
},
|
|
128
|
+
{
|
|
129
|
+
"id": "PT018",
|
|
130
|
+
"type": "SLOP_PHRASE",
|
|
131
|
+
"pattern": "em\\s+última\\s+análise",
|
|
132
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
133
|
+
},
|
|
134
|
+
{
|
|
135
|
+
"id": "PT019",
|
|
136
|
+
"type": "SLOP_PHRASE",
|
|
137
|
+
"pattern": "abrir\\s+caminho\\s+para",
|
|
138
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
139
|
+
},
|
|
140
|
+
{
|
|
141
|
+
"id": "PT020",
|
|
142
|
+
"type": "SLOP_PHRASE",
|
|
143
|
+
"pattern": "mudar\\s+o\\s+jogo",
|
|
144
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
145
|
+
},
|
|
146
|
+
{
|
|
147
|
+
"id": "PT021",
|
|
148
|
+
"type": "SLOP_PHRASE",
|
|
149
|
+
"pattern": "em\\s+um\\s+mundo\\s+em\\s+constante\\s+evolução",
|
|
150
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
151
|
+
},
|
|
152
|
+
{
|
|
153
|
+
"id": "PT022",
|
|
154
|
+
"type": "SLOP_PHRASE",
|
|
155
|
+
"pattern": "uma\\s+abordagem\\s+holística",
|
|
156
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
157
|
+
},
|
|
158
|
+
{
|
|
159
|
+
"id": "PT023",
|
|
160
|
+
"type": "SLOP_PHRASE",
|
|
161
|
+
"pattern": "insights\\s+valiosos",
|
|
162
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
163
|
+
},
|
|
164
|
+
{
|
|
165
|
+
"id": "PT024",
|
|
166
|
+
"type": "SLOP_PHRASE",
|
|
167
|
+
"pattern": "de\\s+forma\\s+contínua",
|
|
168
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
169
|
+
},
|
|
170
|
+
{
|
|
171
|
+
"id": "PT025",
|
|
172
|
+
"type": "SLOP_PHRASE",
|
|
173
|
+
"pattern": "vale\\s+a\\s+pena\\s+mencionar",
|
|
174
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
175
|
+
},
|
|
176
|
+
{
|
|
177
|
+
"id": "PT026",
|
|
178
|
+
"type": "DEFINITIONAL_OPENING",
|
|
179
|
+
"pattern": "^(?:No contexto de|No âmbito de|No mundo de) [\\w\\s]+,",
|
|
180
|
+
"scope": "paragraph-start",
|
|
181
|
+
"fixInstruction": "Start with the specific subject or event instead of a generic framing clause."
|
|
182
|
+
},
|
|
183
|
+
{
|
|
184
|
+
"id": "PT027",
|
|
185
|
+
"type": "STRUCTURAL_CLICHE",
|
|
186
|
+
"pattern": "\\bnão\\s+apenas\\b[\\s\\S]{0,100}?\\bmas\\s+também\\b",
|
|
187
|
+
"fixInstruction": "State both claims directly without the stock paired construction."
|
|
188
|
+
}
|
|
189
|
+
]
|
|
190
|
+
}
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
{
|
|
2
|
+
"language": "zh",
|
|
3
|
+
"stopwords": [
|
|
4
|
+
"的",
|
|
5
|
+
"了",
|
|
6
|
+
"和",
|
|
7
|
+
"是",
|
|
8
|
+
"在",
|
|
9
|
+
"我",
|
|
10
|
+
"有",
|
|
11
|
+
"而",
|
|
12
|
+
"及",
|
|
13
|
+
"与",
|
|
14
|
+
"着",
|
|
15
|
+
"或",
|
|
16
|
+
"一个",
|
|
17
|
+
"没有",
|
|
18
|
+
"我们",
|
|
19
|
+
"你们",
|
|
20
|
+
"他们",
|
|
21
|
+
"这个",
|
|
22
|
+
"那个",
|
|
23
|
+
"因此"
|
|
24
|
+
],
|
|
25
|
+
"patterns": [
|
|
26
|
+
{
|
|
27
|
+
"id": "ZH001",
|
|
28
|
+
"type": "SLOP_PHRASE",
|
|
29
|
+
"pattern": "在当今快节奏的世界中",
|
|
30
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
"id": "ZH002",
|
|
34
|
+
"type": "SLOP_PHRASE",
|
|
35
|
+
"pattern": "在数字化时代",
|
|
36
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
"id": "ZH003",
|
|
40
|
+
"type": "SLOP_PHRASE",
|
|
41
|
+
"pattern": "值得注意的是",
|
|
42
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
"id": "ZH004",
|
|
46
|
+
"type": "SLOP_PHRASE",
|
|
47
|
+
"pattern": "不可否认的是",
|
|
48
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
"id": "ZH005",
|
|
52
|
+
"type": "SLOP_PHRASE",
|
|
53
|
+
"pattern": "总而言之",
|
|
54
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
"id": "ZH006",
|
|
58
|
+
"type": "SLOP_PHRASE",
|
|
59
|
+
"pattern": "综上所述",
|
|
60
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
61
|
+
},
|
|
62
|
+
{
|
|
63
|
+
"id": "ZH007",
|
|
64
|
+
"type": "SLOP_PHRASE",
|
|
65
|
+
"pattern": "此外",
|
|
66
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
67
|
+
},
|
|
68
|
+
{
|
|
69
|
+
"id": "ZH008",
|
|
70
|
+
"type": "SLOP_PHRASE",
|
|
71
|
+
"pattern": "更重要的是",
|
|
72
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
73
|
+
},
|
|
74
|
+
{
|
|
75
|
+
"id": "ZH009",
|
|
76
|
+
"type": "SLOP_PHRASE",
|
|
77
|
+
"pattern": "扮演着至关重要的角色",
|
|
78
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
79
|
+
},
|
|
80
|
+
{
|
|
81
|
+
"id": "ZH010",
|
|
82
|
+
"type": "SLOP_PHRASE",
|
|
83
|
+
"pattern": "不仅",
|
|
84
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
85
|
+
},
|
|
86
|
+
{
|
|
87
|
+
"id": "ZH011",
|
|
88
|
+
"type": "SLOP_PHRASE",
|
|
89
|
+
"pattern": "而且",
|
|
90
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
91
|
+
},
|
|
92
|
+
{
|
|
93
|
+
"id": "ZH012",
|
|
94
|
+
"type": "SLOP_PHRASE",
|
|
95
|
+
"pattern": "深入探讨",
|
|
96
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
97
|
+
},
|
|
98
|
+
{
|
|
99
|
+
"id": "ZH013",
|
|
100
|
+
"type": "SLOP_PHRASE",
|
|
101
|
+
"pattern": "这不仅关乎",
|
|
102
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
103
|
+
},
|
|
104
|
+
{
|
|
105
|
+
"id": "ZH014",
|
|
106
|
+
"type": "SLOP_PHRASE",
|
|
107
|
+
"pattern": "见证了",
|
|
108
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
109
|
+
},
|
|
110
|
+
{
|
|
111
|
+
"id": "ZH015",
|
|
112
|
+
"type": "SLOP_PHRASE",
|
|
113
|
+
"pattern": "双刃剑",
|
|
114
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
115
|
+
},
|
|
116
|
+
{
|
|
117
|
+
"id": "ZH016",
|
|
118
|
+
"type": "SLOP_PHRASE",
|
|
119
|
+
"pattern": "在当前环境下",
|
|
120
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
121
|
+
},
|
|
122
|
+
{
|
|
123
|
+
"id": "ZH017",
|
|
124
|
+
"type": "SLOP_PHRASE",
|
|
125
|
+
"pattern": "毋庸置疑",
|
|
126
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
127
|
+
},
|
|
128
|
+
{
|
|
129
|
+
"id": "ZH018",
|
|
130
|
+
"type": "SLOP_PHRASE",
|
|
131
|
+
"pattern": "需要指出的是",
|
|
132
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
133
|
+
},
|
|
134
|
+
{
|
|
135
|
+
"id": "ZH019",
|
|
136
|
+
"type": "SLOP_PHRASE",
|
|
137
|
+
"pattern": "随着时代的发展",
|
|
138
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
139
|
+
},
|
|
140
|
+
{
|
|
141
|
+
"id": "ZH020",
|
|
142
|
+
"type": "SLOP_PHRASE",
|
|
143
|
+
"pattern": "从本质上讲",
|
|
144
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
145
|
+
},
|
|
146
|
+
{
|
|
147
|
+
"id": "ZH021",
|
|
148
|
+
"type": "SLOP_PHRASE",
|
|
149
|
+
"pattern": "在不断变化的格局中",
|
|
150
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
151
|
+
},
|
|
152
|
+
{
|
|
153
|
+
"id": "ZH022",
|
|
154
|
+
"type": "SLOP_PHRASE",
|
|
155
|
+
"pattern": "开启了新的篇章",
|
|
156
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
157
|
+
},
|
|
158
|
+
{
|
|
159
|
+
"id": "ZH023",
|
|
160
|
+
"type": "SLOP_PHRASE",
|
|
161
|
+
"pattern": "改变游戏规则",
|
|
162
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
163
|
+
},
|
|
164
|
+
{
|
|
165
|
+
"id": "ZH024",
|
|
166
|
+
"type": "SLOP_PHRASE",
|
|
167
|
+
"pattern": "无缝衔接",
|
|
168
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
169
|
+
},
|
|
170
|
+
{
|
|
171
|
+
"id": "ZH025",
|
|
172
|
+
"type": "SLOP_PHRASE",
|
|
173
|
+
"pattern": "全方位的方法",
|
|
174
|
+
"fixInstruction": "Replace this stock phrase with the concrete fact, actor, or action."
|
|
175
|
+
},
|
|
176
|
+
{
|
|
177
|
+
"id": "ZH026",
|
|
178
|
+
"type": "DEFINITIONAL_OPENING",
|
|
179
|
+
"pattern": "^(?:在.*?的背景下|在.*?领域中|在.*?世界里),",
|
|
180
|
+
"scope": "paragraph-start",
|
|
181
|
+
"fixInstruction": "Start with the specific subject or event instead of a generic framing clause."
|
|
182
|
+
},
|
|
183
|
+
{
|
|
184
|
+
"id": "ZH027",
|
|
185
|
+
"type": "STRUCTURAL_CLICHE",
|
|
186
|
+
"pattern": "不仅[\\s\\S]{0,100}?而且",
|
|
187
|
+
"fixInstruction": "State both claims directly without the stock paired construction."
|
|
188
|
+
}
|
|
189
|
+
]
|
|
190
|
+
}
|
package/scripts/analyze.mjs
CHANGED
|
@@ -9,6 +9,12 @@ import { calculateMetrics } from './utils/metrics.mjs';
|
|
|
9
9
|
const RULES = {
|
|
10
10
|
id: JSON.parse(readFileSync(new URL('../rules/id-slop.json', import.meta.url), 'utf8')),
|
|
11
11
|
en: JSON.parse(readFileSync(new URL('../rules/en-slop.json', import.meta.url), 'utf8')),
|
|
12
|
+
es: JSON.parse(readFileSync(new URL('../rules/es-slop.json', import.meta.url), 'utf8')),
|
|
13
|
+
ms: JSON.parse(readFileSync(new URL('../rules/ms-slop.json', import.meta.url), 'utf8')),
|
|
14
|
+
fr: JSON.parse(readFileSync(new URL('../rules/fr-slop.json', import.meta.url), 'utf8')),
|
|
15
|
+
de: JSON.parse(readFileSync(new URL('../rules/de-slop.json', import.meta.url), 'utf8')),
|
|
16
|
+
pt: JSON.parse(readFileSync(new URL('../rules/pt-slop.json', import.meta.url), 'utf8')),
|
|
17
|
+
zh: JSON.parse(readFileSync(new URL('../rules/zh-slop.json', import.meta.url), 'utf8')),
|
|
12
18
|
};
|
|
13
19
|
const DEFAULT_PROFILE = JSON.parse(readFileSync(new URL('../profiles/default-human.json', import.meta.url), 'utf8'));
|
|
14
20
|
const { analyzeText } = createAnalyzer({ rules: RULES, defaultProfile: DEFAULT_PROFILE, calculateMetrics });
|
|
@@ -16,7 +22,7 @@ const { analyzeText } = createAnalyzer({ rules: RULES, defaultProfile: DEFAULT_P
|
|
|
16
22
|
export { analyzeText };
|
|
17
23
|
|
|
18
24
|
function formatHuman(result) {
|
|
19
|
-
const lines = [`Status: ${result.status}`, `Score: ${result.score}`, `Language: ${result.language}`];
|
|
25
|
+
const lines = [`Status: ${result.status}`, `Score: ${result.score}`, `Language: ${result.language}`, `Rule pack: ${result.rulePack}`];
|
|
20
26
|
if (result.warning) lines.push(`Warning: ${result.warning}`);
|
|
21
27
|
lines.push('', 'Metrics:');
|
|
22
28
|
for (const [name, value] of Object.entries(result.metrics)) lines.push(` ${name}: ${value}`);
|
|
@@ -50,11 +56,11 @@ export async function runAnalyzeCli(argv, io = {}) {
|
|
|
50
56
|
allowPositionals: false,
|
|
51
57
|
});
|
|
52
58
|
if (values.help) {
|
|
53
|
-
stdout('Usage: humanizer-lint (--file <path> | --stdin) [--lang
|
|
59
|
+
stdout('Usage: humanizer-lint (--file <path> | --stdin) [--lang auto|universal|<ISO-639-1>] [--profile <path>] [--json]\n');
|
|
54
60
|
return 0;
|
|
55
61
|
}
|
|
56
62
|
if (Boolean(values.file) === Boolean(values.stdin)) throw new Error('Choose exactly one input: --file or --stdin');
|
|
57
|
-
if (
|
|
63
|
+
if (!/^(?:auto|universal|[a-z]{2})$/u.test(values.lang)) throw new Error('--lang must be auto, universal, or an ISO 639-1 code');
|
|
58
64
|
if (values.stdin && io.stdin == null && process.stdin.isTTY) throw new Error('--stdin requires piped input');
|
|
59
65
|
const text = values.file ? await new Promise((resolve, reject) => readFile(values.file, 'utf8', (error, data) => error ? reject(error) : resolve(data))) : await stdin();
|
|
60
66
|
const profile = values.profile ? JSON.parse(await new Promise((resolve, reject) => readFile(values.profile, 'utf8', (error, data) => error ? reject(error) : resolve(data)))) : undefined;
|
|
@@ -1,23 +1,45 @@
|
|
|
1
1
|
import { sanitizeMarkdown } from '../utils/sanitizer.mjs';
|
|
2
2
|
import { mean, punctuationRates } from '../utils/metrics-core.mjs';
|
|
3
|
-
import { tokenizeText } from '../utils/tokenizer.mjs';
|
|
3
|
+
import { isCJKText, tokenizeText } from '../utils/tokenizer.mjs';
|
|
4
4
|
|
|
5
|
-
const
|
|
6
|
-
const
|
|
5
|
+
const HAN = /[\u4e00-\u9fff]/gu;
|
|
6
|
+
const KANA = /[\u3040-\u30ff]/u;
|
|
7
|
+
const ARABIC = /[\u0600-\u06ff]/u;
|
|
8
|
+
const ID_EXCLUSIVE_WORDS = new Set(['tidak', 'karena', 'bisa', 'seperti', 'dari', 'bahwa']);
|
|
9
|
+
const MS_EXCLUSIVE_WORDS = new Set(['tak', 'kerana', 'boleh', 'bagi', 'daripada', 'bahawa', 'beliau']);
|
|
7
10
|
|
|
8
11
|
function round(value, places = 4) {
|
|
9
12
|
const factor = 10 ** places;
|
|
10
13
|
return Math.round(value * factor) / factor;
|
|
11
14
|
}
|
|
12
15
|
|
|
13
|
-
function detectLanguage(words) {
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
16
|
+
function detectLanguage(text, words, rules) {
|
|
17
|
+
if (KANA.test(text)) return 'ja';
|
|
18
|
+
if (ARABIC.test(text)) return 'ar';
|
|
19
|
+
const hanCount = (text.match(HAN) ?? []).length;
|
|
20
|
+
if (hanCount > 0 && isCJKText(text)) return 'zh';
|
|
21
|
+
|
|
22
|
+
const scores = new Map();
|
|
23
|
+
for (const [language, rule] of Object.entries(rules)) {
|
|
24
|
+
let score = 0;
|
|
25
|
+
const stopwords = new Set(rule.stopwords ?? []);
|
|
26
|
+
for (const word of words) if (stopwords.has(word)) score += 1;
|
|
27
|
+
scores.set(language, score);
|
|
28
|
+
}
|
|
29
|
+
if (scores.has('id') && scores.has('ms')) {
|
|
30
|
+
const idExclusive = words.filter((word) => ID_EXCLUSIVE_WORDS.has(word)).length;
|
|
31
|
+
const msExclusive = words.filter((word) => MS_EXCLUSIVE_WORDS.has(word)).length;
|
|
32
|
+
scores.set('id', scores.get('id') + idExclusive * 2);
|
|
33
|
+
scores.set('ms', scores.get('ms') + msExclusive * 2);
|
|
19
34
|
}
|
|
20
|
-
|
|
35
|
+
|
|
36
|
+
const [language, score] = [...scores.entries()].sort((left, right) => right[1] - left[1])[0] ?? ['universal', 0];
|
|
37
|
+
return score >= Math.max(2, Math.ceil(words.length * 0.04)) ? language : 'universal';
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function mattrThreshold(thresholds, language, tokenized) {
|
|
41
|
+
if (typeof thresholds.mattr?.[language] === 'number') return thresholds.mattr[language];
|
|
42
|
+
return isCJKText(tokenized.words.join('')) ? 0.68 : 0.72;
|
|
21
43
|
}
|
|
22
44
|
|
|
23
45
|
function locationFor(sentence) {
|
|
@@ -29,6 +51,7 @@ function regexFor(pattern) {
|
|
|
29
51
|
}
|
|
30
52
|
|
|
31
53
|
function patternViolations(tokenized, rules, wordCount) {
|
|
54
|
+
if (!rules?.patterns) return [];
|
|
32
55
|
const violations = [];
|
|
33
56
|
for (const rule of rules.patterns) {
|
|
34
57
|
const candidates = [];
|
|
@@ -122,7 +145,7 @@ function structuralViolations(tokenized, metrics) {
|
|
|
122
145
|
return violations;
|
|
123
146
|
}
|
|
124
147
|
|
|
125
|
-
function statisticalViolations(metrics, thresholds,
|
|
148
|
+
function statisticalViolations(metrics, thresholds, mattrTarget) {
|
|
126
149
|
const violations = [];
|
|
127
150
|
const cvTarget = metrics.sentenceCount >= 8 ? thresholds.burstinessCVLong : thresholds.burstinessCVShort;
|
|
128
151
|
if (metrics.burstinessCV < cvTarget) {
|
|
@@ -152,12 +175,12 @@ function statisticalViolations(metrics, thresholds, language) {
|
|
|
152
175
|
fixInstruction: 'Change neighboring sentence lengths by splitting or merging only where the content allows it.',
|
|
153
176
|
});
|
|
154
177
|
}
|
|
155
|
-
if (metrics.mattr <
|
|
178
|
+
if (metrics.mattr < mattrTarget) {
|
|
156
179
|
violations.push({
|
|
157
180
|
type: 'LOW_LEXICAL_DIVERSITY',
|
|
158
181
|
location: 'Whole text',
|
|
159
182
|
actual: round(metrics.mattr),
|
|
160
|
-
target:
|
|
183
|
+
target: mattrTarget,
|
|
161
184
|
fixInstruction: 'Replace repeated generic wording with precise terms already supported by the source.',
|
|
162
185
|
});
|
|
163
186
|
}
|
|
@@ -244,7 +267,7 @@ function voiceDeviation(metrics, tokenized, text, profile) {
|
|
|
244
267
|
return result < 0.0005 ? 0 : result;
|
|
245
268
|
}
|
|
246
269
|
|
|
247
|
-
function scoreResult(metrics, violations,
|
|
270
|
+
function scoreResult(metrics, violations, thresholds, mattrTarget) {
|
|
248
271
|
const cvTarget = metrics.sentenceCount >= 8 ? thresholds.burstinessCVLong : thresholds.burstinessCVShort;
|
|
249
272
|
let burstinessPenalty = metrics.burstinessCV < cvTarget ? Math.min(25, Math.round((cvTarget - metrics.burstinessCV) * 100)) : 0;
|
|
250
273
|
if (metrics.sentenceCount >= 8 && (metrics.shortSentenceRatio < thresholds.shortSentenceRatio || metrics.longSentenceRatio < thresholds.longSentenceRatio)) {
|
|
@@ -259,7 +282,7 @@ function scoreResult(metrics, violations, language, thresholds) {
|
|
|
259
282
|
const morphologyPenalty = Math.min(15,
|
|
260
283
|
(violations.some((violation) => violation.type === 'EXCESSIVE_NOMINALIZATION') ? 7.5 : 0)
|
|
261
284
|
+ Math.min(15, violations.filter((violation) => violation.type === 'PASSIVE_BLOATING').length * 7.5));
|
|
262
|
-
const vocabularyPenalty = metrics.mattr <
|
|
285
|
+
const vocabularyPenalty = metrics.mattr < mattrTarget ? 5 : 0;
|
|
263
286
|
const entropyPenalty = violations.some((violation) => ['LOW_ENTROPY_REPETITION', 'SYMMETRIC_PARAGRAPHS', 'REPETITIVE_STARTERS'].includes(violation.type)) ? 5 : 0;
|
|
264
287
|
const profilePenalty = violations.some((violation) => violation.type === 'VOICE_PROFILE_DEVIATION') ? 5 : 0;
|
|
265
288
|
const vocabularyAndProfilePenalty = Math.min(10, vocabularyPenalty + entropyPenalty + profilePenalty);
|
|
@@ -267,25 +290,27 @@ function scoreResult(metrics, violations, language, thresholds) {
|
|
|
267
290
|
}
|
|
268
291
|
|
|
269
292
|
export function createAnalyzer({ rules, defaultProfile, calculateMetrics, calculateMetricsAsync }) {
|
|
270
|
-
if (!rules
|
|
293
|
+
if (!rules || !defaultProfile || typeof calculateMetrics !== 'function') {
|
|
271
294
|
throw new TypeError('createAnalyzer requires rules, defaultProfile, and calculateMetrics');
|
|
272
295
|
}
|
|
273
296
|
|
|
274
297
|
const prepare = (input, options) => {
|
|
275
298
|
const sanitized = sanitizeMarkdown(input);
|
|
276
|
-
const
|
|
277
|
-
const
|
|
278
|
-
|
|
299
|
+
const requestedLanguage = String(options.language ?? 'auto').toLowerCase();
|
|
300
|
+
const initial = tokenizeText(sanitized, requestedLanguage);
|
|
301
|
+
const language = requestedLanguage === 'auto' ? detectLanguage(sanitized, initial.words, rules) : requestedLanguage;
|
|
279
302
|
return {
|
|
280
303
|
sanitized,
|
|
281
304
|
language,
|
|
282
|
-
|
|
305
|
+
rulePack: rules[language] ? language : 'universal',
|
|
306
|
+
tokenized: language === requestedLanguage ? initial : tokenizeText(sanitized, language),
|
|
283
307
|
};
|
|
284
308
|
};
|
|
285
309
|
|
|
286
|
-
const analyzePrepared = ({ sanitized, language, tokenized }, options, metrics) => {
|
|
310
|
+
const analyzePrepared = ({ sanitized, language, rulePack, tokenized }, options, metrics) => {
|
|
287
311
|
const profile = options.profile ?? defaultProfile;
|
|
288
312
|
const thresholds = { ...defaultProfile.thresholds, ...(profile.thresholds ?? {}) };
|
|
313
|
+
const mattrTarget = mattrThreshold(thresholds, language, tokenized);
|
|
289
314
|
const patternMatches = patternViolations(tokenized, rules[language], metrics.wordCount);
|
|
290
315
|
|
|
291
316
|
if (metrics.wordCount < thresholds.minimumWords || metrics.sentenceCount < thresholds.minimumSentences) {
|
|
@@ -294,13 +319,14 @@ export function createAnalyzer({ rules, defaultProfile, calculateMetrics, calcul
|
|
|
294
319
|
status: 'SKIP',
|
|
295
320
|
score: 100,
|
|
296
321
|
language,
|
|
322
|
+
rulePack,
|
|
297
323
|
warning: 'Text too short for statistical stylometry (< 3 sentences)',
|
|
298
324
|
metrics: Object.fromEntries(Object.entries({ ...metrics, nominalizationRatio: 0, passiveRatio: 0, voiceProfileDeviation: 0 }).map(([key, value]) => [key, typeof value === 'number' ? round(value) : value])),
|
|
299
325
|
violations,
|
|
300
326
|
};
|
|
301
327
|
}
|
|
302
328
|
|
|
303
|
-
const morphology = morphologyViolations(tokenized, rules[language]
|
|
329
|
+
const morphology = morphologyViolations(tokenized, rules[language]?.morphology);
|
|
304
330
|
metrics.nominalizationRatio = morphology.nominalizationRatio;
|
|
305
331
|
metrics.passiveRatio = morphology.passiveRatio;
|
|
306
332
|
metrics.voiceProfileDeviation = voiceDeviation(metrics, tokenized, sanitized, profile);
|
|
@@ -319,9 +345,9 @@ export function createAnalyzer({ rules, defaultProfile, calculateMetrics, calcul
|
|
|
319
345
|
...structuralViolations(tokenized, metrics),
|
|
320
346
|
...morphology.violations,
|
|
321
347
|
...profileViolations,
|
|
322
|
-
...statisticalViolations(metrics, thresholds,
|
|
348
|
+
...statisticalViolations(metrics, thresholds, mattrTarget),
|
|
323
349
|
];
|
|
324
|
-
const score = scoreResult(metrics, violations,
|
|
350
|
+
const score = scoreResult(metrics, violations, thresholds, mattrTarget);
|
|
325
351
|
const cvTarget = metrics.sentenceCount >= 8 ? thresholds.burstinessCVLong : thresholds.burstinessCVShort;
|
|
326
352
|
const forbidden = violations.some((violation) => violation.type === 'SLOP_PHRASE' || violation.type === 'DEFINITIONAL_OPENING');
|
|
327
353
|
const rhythmPass = metrics.adjacentDelta >= thresholds.adjacentDelta;
|
|
@@ -332,6 +358,7 @@ export function createAnalyzer({ rules, defaultProfile, calculateMetrics, calcul
|
|
|
332
358
|
status,
|
|
333
359
|
score,
|
|
334
360
|
language,
|
|
361
|
+
rulePack,
|
|
335
362
|
metrics: Object.fromEntries(Object.entries(metrics).map(([key, value]) => [key, typeof value === 'number' ? round(value) : value])),
|
|
336
363
|
violations,
|
|
337
364
|
};
|