humanizer-lint 0.2.3 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +556 -470
- package/SKILL.md +132 -116
- package/package.json +47 -25
- package/scripts/analyze.mjs +87 -408
- package/scripts/core/analyzer-core.mjs +352 -0
- package/scripts/utils/metrics-core.mjs +84 -0
- package/scripts/utils/metrics.mjs +28 -73
|
@@ -0,0 +1,352 @@
|
|
|
1
|
+
import { sanitizeMarkdown } from '../utils/sanitizer.mjs';
|
|
2
|
+
import { mean, punctuationRates } from '../utils/metrics-core.mjs';
|
|
3
|
+
import { tokenizeText } from '../utils/tokenizer.mjs';
|
|
4
|
+
|
|
5
|
+
const ID_WORDS = new Set(['yang', 'dan', 'di', 'ini', 'untuk', 'dengan', 'dari', 'pada', 'dalam', 'tidak', 'akan', 'atau', 'juga', 'oleh', 'adalah', 'sebagai', 'bahwa', 'dapat', 'karena', 'tersebut']);
|
|
6
|
+
const EN_WORDS = new Set(['the', 'and', 'of', 'to', 'in', 'is', 'that', 'for', 'it', 'with', 'as', 'on', 'are', 'this', 'by', 'be', 'at', 'or', 'from', 'an']);
|
|
7
|
+
|
|
8
|
+
function round(value, places = 4) {
|
|
9
|
+
const factor = 10 ** places;
|
|
10
|
+
return Math.round(value * factor) / factor;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
function detectLanguage(words) {
|
|
14
|
+
let id = 0;
|
|
15
|
+
let en = 0;
|
|
16
|
+
for (const word of words) {
|
|
17
|
+
if (ID_WORDS.has(word)) id += 1;
|
|
18
|
+
if (EN_WORDS.has(word)) en += 1;
|
|
19
|
+
}
|
|
20
|
+
return en > id ? 'en' : 'id';
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function locationFor(sentence) {
|
|
24
|
+
return `Paragraph ${sentence.paragraphIndex}, Sentence ${sentence.sentenceIndex}`;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function regexFor(pattern) {
|
|
28
|
+
return new RegExp(pattern, 'giu');
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function patternViolations(tokenized, rules, wordCount) {
|
|
32
|
+
const violations = [];
|
|
33
|
+
for (const rule of rules.patterns) {
|
|
34
|
+
const candidates = [];
|
|
35
|
+
const sentences = rule.scope === 'paragraph-start'
|
|
36
|
+
? tokenized.paragraphs.map((paragraph) => paragraph.sentences[0]).filter(Boolean)
|
|
37
|
+
: tokenized.sentences;
|
|
38
|
+
|
|
39
|
+
for (const sentence of sentences) {
|
|
40
|
+
const regex = regexFor(rule.pattern);
|
|
41
|
+
for (const match of sentence.text.matchAll(regex)) candidates.push({ sentence, match: match[0] });
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const allowed = rule.maxOccurrences == null
|
|
45
|
+
? 0
|
|
46
|
+
: rule.maxOccurrences * Math.max(1, Math.ceil(wordCount / (rule.perWords || wordCount || 1)));
|
|
47
|
+
for (const candidate of candidates.slice(allowed)) {
|
|
48
|
+
violations.push({
|
|
49
|
+
type: rule.type,
|
|
50
|
+
ruleId: rule.id,
|
|
51
|
+
location: locationFor(candidate.sentence),
|
|
52
|
+
match: candidate.match,
|
|
53
|
+
fixInstruction: rule.fixInstruction,
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
return violations;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function monotonyViolations(tokenized) {
|
|
61
|
+
const violations = [];
|
|
62
|
+
for (const paragraph of tokenized.paragraphs) {
|
|
63
|
+
for (let index = 0; index <= paragraph.sentences.length - 3; index += 1) {
|
|
64
|
+
const window = paragraph.sentences.slice(index, index + 3);
|
|
65
|
+
const lengths = window.map((sentence) => sentence.wordCount);
|
|
66
|
+
if (Math.abs(lengths[1] - lengths[0]) <= 3 && Math.abs(lengths[2] - lengths[1]) <= 3) {
|
|
67
|
+
violations.push({
|
|
68
|
+
type: 'MONOTONOUS_RHYTHM',
|
|
69
|
+
location: `Paragraph ${paragraph.paragraphIndex}, Sentences ${window[0].sentenceIndex}-${window[2].sentenceIndex}`,
|
|
70
|
+
lengths,
|
|
71
|
+
fixInstruction: `Sentence lengths are uniform (${lengths.join(', ')} words). Split one sentence into a short statement or merge two into a longer clause.`,
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
return violations;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function structuralViolations(tokenized, metrics) {
|
|
80
|
+
const violations = [];
|
|
81
|
+
if (metrics.wordCount >= 150 && metrics.compressionRatio < 0.42) {
|
|
82
|
+
violations.push({
|
|
83
|
+
type: 'LOW_ENTROPY_REPETITION',
|
|
84
|
+
location: 'Whole text',
|
|
85
|
+
fixInstruction: 'Remove repeated templates and vary concrete sentence structures.',
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
for (let index = 0; index <= tokenized.paragraphs.length - 3; index += 1) {
|
|
90
|
+
const window = tokenized.paragraphs.slice(index, index + 3);
|
|
91
|
+
const counts = window.map((paragraph) => paragraph.sentences.length);
|
|
92
|
+
if (counts[0] > 0 && counts.every((count) => count === counts[0])) {
|
|
93
|
+
violations.push({
|
|
94
|
+
type: 'SYMMETRIC_PARAGRAPHS',
|
|
95
|
+
location: `Paragraphs ${window[0].paragraphIndex}-${window[2].paragraphIndex}`,
|
|
96
|
+
counts,
|
|
97
|
+
fixInstruction: 'Merge or split a paragraph where the content supports a different rhythm.',
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
for (const paragraph of tokenized.paragraphs) {
|
|
103
|
+
const starters = new Map();
|
|
104
|
+
for (const sentence of paragraph.sentences) {
|
|
105
|
+
const starter = sentence.words[0];
|
|
106
|
+
if (!starter) continue;
|
|
107
|
+
const group = starters.get(starter) ?? [];
|
|
108
|
+
group.push(sentence.sentenceIndex);
|
|
109
|
+
starters.set(starter, group);
|
|
110
|
+
}
|
|
111
|
+
for (const [starter, indexes] of starters) {
|
|
112
|
+
if (indexes.length >= 3) {
|
|
113
|
+
violations.push({
|
|
114
|
+
type: 'REPETITIVE_STARTERS',
|
|
115
|
+
location: `Paragraph ${paragraph.paragraphIndex}, Sentences ${indexes.join(', ')}`,
|
|
116
|
+
match: starter,
|
|
117
|
+
fixInstruction: 'Change one sentence opening or combine sentences where the subject repeats.',
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
return violations;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function statisticalViolations(metrics, thresholds, language) {
|
|
126
|
+
const violations = [];
|
|
127
|
+
const cvTarget = metrics.sentenceCount >= 8 ? thresholds.burstinessCVLong : thresholds.burstinessCVShort;
|
|
128
|
+
if (metrics.burstinessCV < cvTarget) {
|
|
129
|
+
violations.push({
|
|
130
|
+
type: 'LOW_BURSTINESS',
|
|
131
|
+
location: 'Whole text',
|
|
132
|
+
actual: round(metrics.burstinessCV),
|
|
133
|
+
target: cvTarget,
|
|
134
|
+
fixInstruction: 'Vary sentence lengths by splitting one uniform sentence and merging another where the meaning supports it.',
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
if (metrics.sentenceCount >= 8 && (metrics.shortSentenceRatio < thresholds.shortSentenceRatio || metrics.longSentenceRatio < thresholds.longSentenceRatio)) {
|
|
138
|
+
violations.push({
|
|
139
|
+
type: 'MISSING_SENTENCE_RANGE',
|
|
140
|
+
location: 'Whole text',
|
|
141
|
+
shortSentenceRatio: round(metrics.shortSentenceRatio),
|
|
142
|
+
longSentenceRatio: round(metrics.longSentenceRatio),
|
|
143
|
+
fixInstruction: 'Add a supported sentence of 6 words or fewer and a supported sentence of 26 words or more until both distribution thresholds pass.',
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
if (metrics.adjacentDelta < thresholds.adjacentDelta) {
|
|
147
|
+
violations.push({
|
|
148
|
+
type: 'LOW_ADJACENT_DELTA',
|
|
149
|
+
location: 'Whole text',
|
|
150
|
+
actual: round(metrics.adjacentDelta),
|
|
151
|
+
target: thresholds.adjacentDelta,
|
|
152
|
+
fixInstruction: 'Change neighboring sentence lengths by splitting or merging only where the content allows it.',
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
if (metrics.mattr < thresholds.mattr[language]) {
|
|
156
|
+
violations.push({
|
|
157
|
+
type: 'LOW_LEXICAL_DIVERSITY',
|
|
158
|
+
location: 'Whole text',
|
|
159
|
+
actual: round(metrics.mattr),
|
|
160
|
+
target: thresholds.mattr[language],
|
|
161
|
+
fixInstruction: 'Replace repeated generic wording with precise terms already supported by the source.',
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
return violations;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function morphologyViolations(tokenized, morphology) {
|
|
168
|
+
if (!morphology) return { violations: [], nominalizationRatio: 0, passiveRatio: 0 };
|
|
169
|
+
const words = tokenized.words;
|
|
170
|
+
const nominalizationPattern = new RegExp(morphology.nominalizationPattern, 'iu');
|
|
171
|
+
const nominalizationWhitelist = new Set(morphology.nominalizationWhitelist);
|
|
172
|
+
const nominalizations = words.filter((word) => nominalizationPattern.test(word) && !nominalizationWhitelist.has(word));
|
|
173
|
+
const passivePattern = new RegExp(morphology.passivePattern, 'iu');
|
|
174
|
+
const loanwordExclusion = new RegExp(morphology.loanwordExclusionPattern, 'iu');
|
|
175
|
+
const loanwordSuffixExclusion = new RegExp(morphology.loanwordSuffixExclusionPattern ?? '$^', 'iu');
|
|
176
|
+
const passives = words.filter((word) => passivePattern.test(word) && !loanwordExclusion.test(word) && !loanwordSuffixExclusion.test(word));
|
|
177
|
+
const nominalizationRatio = words.length ? nominalizations.length / words.length : 0;
|
|
178
|
+
const passiveRatio = words.length ? passives.length / words.length : 0;
|
|
179
|
+
const violations = [];
|
|
180
|
+
|
|
181
|
+
if (nominalizationRatio > morphology.nominalizationMaxRatio) {
|
|
182
|
+
violations.push({
|
|
183
|
+
type: 'EXCESSIVE_NOMINALIZATION',
|
|
184
|
+
location: 'Whole text',
|
|
185
|
+
ratio: round(nominalizationRatio),
|
|
186
|
+
matches: [...new Set(nominalizations)],
|
|
187
|
+
fixInstruction: 'Replace abstract nominalizations with concrete actors and verbs.',
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
if (passiveRatio > morphology.passiveMaxRatio) {
|
|
191
|
+
violations.push({
|
|
192
|
+
type: 'PASSIVE_BLOATING',
|
|
193
|
+
location: 'Whole text',
|
|
194
|
+
ratio: round(passiveRatio),
|
|
195
|
+
matches: [...new Set(passives)],
|
|
196
|
+
fixInstruction: 'Name the actor and use an active verb where the actor is known.',
|
|
197
|
+
});
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
const openingPattern = new RegExp(morphology.subordinateOpeningPattern, 'iu');
|
|
201
|
+
let foundConsecutiveOpening = false;
|
|
202
|
+
for (const paragraph of tokenized.paragraphs) {
|
|
203
|
+
for (let index = 1; index < paragraph.sentences.length; index += 1) {
|
|
204
|
+
const previous = paragraph.sentences[index - 1];
|
|
205
|
+
const current = paragraph.sentences[index];
|
|
206
|
+
if (!openingPattern.test(previous.text) || !openingPattern.test(current.text)) continue;
|
|
207
|
+
violations.push({
|
|
208
|
+
type: 'PASSIVE_BLOATING',
|
|
209
|
+
location: `Paragraph ${paragraph.paragraphIndex}, Sentences ${previous.sentenceIndex}-${current.sentenceIndex}`,
|
|
210
|
+
match: 'Two or more consecutive sentences start with translationese subordinate clauses',
|
|
211
|
+
fixInstruction: 'Rewrite to start directly with the active subject (Subjek + Predikat).',
|
|
212
|
+
});
|
|
213
|
+
foundConsecutiveOpening = true;
|
|
214
|
+
break;
|
|
215
|
+
}
|
|
216
|
+
if (foundConsecutiveOpening) break;
|
|
217
|
+
}
|
|
218
|
+
return { violations, nominalizationRatio, passiveRatio };
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
function voiceDeviation(metrics, tokenized, text, profile) {
|
|
222
|
+
const targets = profile?.targets ?? profile?.metrics;
|
|
223
|
+
if (!targets) return 0;
|
|
224
|
+
const actual = {
|
|
225
|
+
meanSentenceLength: metrics.meanSentenceLength,
|
|
226
|
+
burstinessCV: metrics.burstinessCV,
|
|
227
|
+
adjacentDelta: metrics.adjacentDelta,
|
|
228
|
+
mattr: metrics.mattr,
|
|
229
|
+
averageParagraphSentenceCount: mean(tokenized.paragraphs.map((paragraph) => paragraph.sentences.length)),
|
|
230
|
+
};
|
|
231
|
+
const deviations = [];
|
|
232
|
+
for (const key of Object.keys(actual)) {
|
|
233
|
+
if (typeof targets[key] !== 'number') continue;
|
|
234
|
+
deviations.push(Math.min(3, Math.abs(actual[key] - targets[key]) / Math.max(Math.abs(targets[key]), 0.01)));
|
|
235
|
+
}
|
|
236
|
+
const targetPunctuation = targets.punctuationRates;
|
|
237
|
+
if (targetPunctuation) {
|
|
238
|
+
const actualPunctuation = punctuationRates(text, metrics.wordCount);
|
|
239
|
+
for (const key of Object.keys(targetPunctuation)) {
|
|
240
|
+
deviations.push(Math.min(3, Math.abs(actualPunctuation[key] - targetPunctuation[key]) / Math.max(Math.abs(targetPunctuation[key]), 0.01)));
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
const result = deviations.length ? mean(deviations) : 0;
|
|
244
|
+
return result < 0.0005 ? 0 : result;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
function scoreResult(metrics, violations, language, thresholds) {
|
|
248
|
+
const cvTarget = metrics.sentenceCount >= 8 ? thresholds.burstinessCVLong : thresholds.burstinessCVShort;
|
|
249
|
+
let burstinessPenalty = metrics.burstinessCV < cvTarget ? Math.min(25, Math.round((cvTarget - metrics.burstinessCV) * 100)) : 0;
|
|
250
|
+
if (metrics.sentenceCount >= 8 && (metrics.shortSentenceRatio < thresholds.shortSentenceRatio || metrics.longSentenceRatio < thresholds.longSentenceRatio)) {
|
|
251
|
+
burstinessPenalty = Math.min(25, burstinessPenalty + 8);
|
|
252
|
+
}
|
|
253
|
+
const rhythmPenalty = Math.min(20, violations.filter((violation) => violation.type === 'MONOTONOUS_RHYTHM').length * 6);
|
|
254
|
+
const lexicalPenalty = Math.min(30, violations.reduce((total, violation) => {
|
|
255
|
+
if (violation.type === 'SLOP_PHRASE' || violation.type === 'DEFINITIONAL_OPENING') return total + 5;
|
|
256
|
+
if (violation.type === 'STRUCTURAL_CLICHE') return total + 4;
|
|
257
|
+
return total;
|
|
258
|
+
}, 0));
|
|
259
|
+
const morphologyPenalty = Math.min(15,
|
|
260
|
+
(violations.some((violation) => violation.type === 'EXCESSIVE_NOMINALIZATION') ? 7.5 : 0)
|
|
261
|
+
+ Math.min(15, violations.filter((violation) => violation.type === 'PASSIVE_BLOATING').length * 7.5));
|
|
262
|
+
const vocabularyPenalty = metrics.mattr < thresholds.mattr[language] ? 5 : 0;
|
|
263
|
+
const entropyPenalty = violations.some((violation) => ['LOW_ENTROPY_REPETITION', 'SYMMETRIC_PARAGRAPHS', 'REPETITIVE_STARTERS'].includes(violation.type)) ? 5 : 0;
|
|
264
|
+
const profilePenalty = violations.some((violation) => violation.type === 'VOICE_PROFILE_DEVIATION') ? 5 : 0;
|
|
265
|
+
const vocabularyAndProfilePenalty = Math.min(10, vocabularyPenalty + entropyPenalty + profilePenalty);
|
|
266
|
+
return Math.max(0, Math.round(100 - burstinessPenalty - rhythmPenalty - lexicalPenalty - morphologyPenalty - vocabularyAndProfilePenalty));
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
export function createAnalyzer({ rules, defaultProfile, calculateMetrics, calculateMetricsAsync }) {
|
|
270
|
+
if (!rules?.id || !rules?.en || !defaultProfile || typeof calculateMetrics !== 'function') {
|
|
271
|
+
throw new TypeError('createAnalyzer requires rules, defaultProfile, and calculateMetrics');
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
const prepare = (input, options) => {
|
|
275
|
+
const sanitized = sanitizeMarkdown(input);
|
|
276
|
+
const initial = tokenizeText(sanitized, options.language ?? 'auto');
|
|
277
|
+
const language = options.language && options.language !== 'auto' ? options.language : detectLanguage(initial.words);
|
|
278
|
+
if (!rules[language]) throw new Error(`Unsupported language: ${language}`);
|
|
279
|
+
return {
|
|
280
|
+
sanitized,
|
|
281
|
+
language,
|
|
282
|
+
tokenized: language === (options.language ?? 'auto') ? initial : tokenizeText(sanitized, language),
|
|
283
|
+
};
|
|
284
|
+
};
|
|
285
|
+
|
|
286
|
+
const analyzePrepared = ({ sanitized, language, tokenized }, options, metrics) => {
|
|
287
|
+
const profile = options.profile ?? defaultProfile;
|
|
288
|
+
const thresholds = { ...defaultProfile.thresholds, ...(profile.thresholds ?? {}) };
|
|
289
|
+
const patternMatches = patternViolations(tokenized, rules[language], metrics.wordCount);
|
|
290
|
+
|
|
291
|
+
if (metrics.wordCount < thresholds.minimumWords || metrics.sentenceCount < thresholds.minimumSentences) {
|
|
292
|
+
const violations = patternMatches.filter((violation) => violation.type === 'SLOP_PHRASE');
|
|
293
|
+
return {
|
|
294
|
+
status: 'SKIP',
|
|
295
|
+
score: 100,
|
|
296
|
+
language,
|
|
297
|
+
warning: 'Text too short for statistical stylometry (< 3 sentences)',
|
|
298
|
+
metrics: Object.fromEntries(Object.entries({ ...metrics, nominalizationRatio: 0, passiveRatio: 0, voiceProfileDeviation: 0 }).map(([key, value]) => [key, typeof value === 'number' ? round(value) : value])),
|
|
299
|
+
violations,
|
|
300
|
+
};
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
const morphology = morphologyViolations(tokenized, rules[language].morphology);
|
|
304
|
+
metrics.nominalizationRatio = morphology.nominalizationRatio;
|
|
305
|
+
metrics.passiveRatio = morphology.passiveRatio;
|
|
306
|
+
metrics.voiceProfileDeviation = voiceDeviation(metrics, tokenized, sanitized, profile);
|
|
307
|
+
const profileTargets = profile.targets ?? profile.metrics ?? {};
|
|
308
|
+
const targetMean = Number(profileTargets.meanSentenceLength);
|
|
309
|
+
const meanDeviation = targetMean > 0 ? Math.abs(metrics.meanSentenceLength - targetMean) / targetMean : 0;
|
|
310
|
+
const profileViolations = options.profile != null && meanDeviation > 0.3 ? [{
|
|
311
|
+
type: 'VOICE_PROFILE_DEVIATION',
|
|
312
|
+
location: 'Global Document',
|
|
313
|
+
match: 'Mean sentence length deviates >30% from target voice profile',
|
|
314
|
+
fixInstruction: 'Adjust overall sentence complexity so mean sentence length approaches the target profile.',
|
|
315
|
+
}] : [];
|
|
316
|
+
const violations = [
|
|
317
|
+
...patternMatches,
|
|
318
|
+
...monotonyViolations(tokenized),
|
|
319
|
+
...structuralViolations(tokenized, metrics),
|
|
320
|
+
...morphology.violations,
|
|
321
|
+
...profileViolations,
|
|
322
|
+
...statisticalViolations(metrics, thresholds, language),
|
|
323
|
+
];
|
|
324
|
+
const score = scoreResult(metrics, violations, language, thresholds);
|
|
325
|
+
const cvTarget = metrics.sentenceCount >= 8 ? thresholds.burstinessCVLong : thresholds.burstinessCVShort;
|
|
326
|
+
const forbidden = violations.some((violation) => violation.type === 'SLOP_PHRASE' || violation.type === 'DEFINITIONAL_OPENING');
|
|
327
|
+
const rhythmPass = metrics.adjacentDelta >= thresholds.adjacentDelta;
|
|
328
|
+
const distributionPass = metrics.sentenceCount < 8 || (metrics.shortSentenceRatio >= thresholds.shortSentenceRatio && metrics.longSentenceRatio >= thresholds.longSentenceRatio);
|
|
329
|
+
const status = score >= thresholds.passScore && !forbidden && metrics.burstinessCV >= cvTarget && rhythmPass && distributionPass ? 'PASS' : 'FAIL';
|
|
330
|
+
|
|
331
|
+
return {
|
|
332
|
+
status,
|
|
333
|
+
score,
|
|
334
|
+
language,
|
|
335
|
+
metrics: Object.fromEntries(Object.entries(metrics).map(([key, value]) => [key, typeof value === 'number' ? round(value) : value])),
|
|
336
|
+
violations,
|
|
337
|
+
};
|
|
338
|
+
};
|
|
339
|
+
|
|
340
|
+
return {
|
|
341
|
+
analyzeText(input, options = {}) {
|
|
342
|
+
const prepared = prepare(input, options);
|
|
343
|
+
return analyzePrepared(prepared, options, calculateMetrics(prepared.tokenized, prepared.sanitized));
|
|
344
|
+
},
|
|
345
|
+
async analyzeTextAsync(input, options = {}) {
|
|
346
|
+
if (typeof calculateMetricsAsync !== 'function') return this.analyzeText(input, options);
|
|
347
|
+
const prepared = prepare(input, options);
|
|
348
|
+
const metrics = await calculateMetricsAsync(prepared.tokenized, prepared.sanitized);
|
|
349
|
+
return analyzePrepared(prepared, options, metrics);
|
|
350
|
+
},
|
|
351
|
+
};
|
|
352
|
+
}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
export function mean(values) {
|
|
2
|
+
return values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : 0;
|
|
3
|
+
}
|
|
4
|
+
|
|
5
|
+
export function standardDeviation(values) {
|
|
6
|
+
if (!values.length) return 0;
|
|
7
|
+
const average = mean(values);
|
|
8
|
+
return Math.sqrt(mean(values.map((value) => (value - average) ** 2)));
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export function coefficientOfVariation(values) {
|
|
12
|
+
const average = mean(values);
|
|
13
|
+
return average ? standardDeviation(values) / average : 0;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function adjacentDelta(values) {
|
|
17
|
+
if (values.length < 2) return 0;
|
|
18
|
+
let total = 0;
|
|
19
|
+
for (let index = 1; index < values.length; index += 1) total += Math.abs(values[index] - values[index - 1]);
|
|
20
|
+
return total / (values.length - 1);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function mattr(words, requestedWindow) {
|
|
24
|
+
if (!words.length) return 0;
|
|
25
|
+
const window = Math.min(words.length, requestedWindow ?? Math.min(50, Math.max(10, Math.floor(words.length / 2))));
|
|
26
|
+
if (words.length <= window) return new Set(words).size / words.length;
|
|
27
|
+
let total = 0;
|
|
28
|
+
let windows = 0;
|
|
29
|
+
for (let index = 0; index <= words.length - window; index += 1) {
|
|
30
|
+
total += new Set(words.slice(index, index + window)).size / window;
|
|
31
|
+
windows += 1;
|
|
32
|
+
}
|
|
33
|
+
return total / windows;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function punctuationRates(text, wordCount) {
|
|
37
|
+
const denominator = wordCount || 1;
|
|
38
|
+
const count = (pattern) => (String(text ?? '').match(pattern) ?? []).length / denominator;
|
|
39
|
+
return {
|
|
40
|
+
comma: count(/,/g),
|
|
41
|
+
semicolon: count(/;/g),
|
|
42
|
+
colon: count(/:/g),
|
|
43
|
+
question: count(/\?/g),
|
|
44
|
+
exclamation: count(/!/g),
|
|
45
|
+
parentheses: count(/[()]/g) / 2,
|
|
46
|
+
dash: count(/—|\s--\s/g),
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function calculateMetricsFromCompressionRatio(tokenized, compressionRatio) {
|
|
51
|
+
const lengths = tokenized.sentences.map((sentence) => sentence.wordCount);
|
|
52
|
+
const wordCount = tokenized.words.length;
|
|
53
|
+
const sentenceCount = lengths.length;
|
|
54
|
+
return {
|
|
55
|
+
wordCount,
|
|
56
|
+
sentenceCount,
|
|
57
|
+
meanSentenceLength: mean(lengths),
|
|
58
|
+
stdDev: standardDeviation(lengths),
|
|
59
|
+
burstinessCV: coefficientOfVariation(lengths),
|
|
60
|
+
adjacentDelta: adjacentDelta(lengths),
|
|
61
|
+
shortSentenceRatio: sentenceCount ? lengths.filter((length) => length <= 6).length / sentenceCount : 0,
|
|
62
|
+
longSentenceRatio: sentenceCount ? lengths.filter((length) => length >= 26).length / sentenceCount : 0,
|
|
63
|
+
mattr: mattr(tokenized.words),
|
|
64
|
+
compressionRatio,
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export function createMetrics({ encode, deflateRaw }) {
|
|
69
|
+
if (typeof encode !== 'function' || typeof deflateRaw !== 'function') {
|
|
70
|
+
throw new TypeError('createMetrics requires encode and deflateRaw functions');
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const compressionRatio = (text) => {
|
|
74
|
+
const bytes = encode(String(text ?? '').trim());
|
|
75
|
+
return bytes.length ? deflateRaw(bytes).length / bytes.length : 0;
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
const calculateMetrics = (tokenized, normalizedText) => calculateMetricsFromCompressionRatio(
|
|
79
|
+
tokenized,
|
|
80
|
+
compressionRatio(normalizedText),
|
|
81
|
+
);
|
|
82
|
+
|
|
83
|
+
return { compressionRatio, calculateMetrics };
|
|
84
|
+
}
|
|
@@ -1,73 +1,28 @@
|
|
|
1
|
-
import { deflateRawSync } from 'node:zlib';
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
export
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
let total = 0;
|
|
30
|
-
let windows = 0;
|
|
31
|
-
for (let index = 0; index <= words.length - window; index += 1) {
|
|
32
|
-
total += new Set(words.slice(index, index + window)).size / window;
|
|
33
|
-
windows += 1;
|
|
34
|
-
}
|
|
35
|
-
return total / windows;
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
export function compressionRatio(text) {
|
|
39
|
-
const buffer = Buffer.from(String(text ?? '').trim(), 'utf8');
|
|
40
|
-
return buffer.length ? deflateRawSync(buffer).length / buffer.length : 0;
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
export function punctuationRates(text, wordCount) {
|
|
44
|
-
const denominator = wordCount || 1;
|
|
45
|
-
const count = (pattern) => (String(text ?? '').match(pattern) ?? []).length / denominator;
|
|
46
|
-
return {
|
|
47
|
-
comma: count(/,/g),
|
|
48
|
-
semicolon: count(/;/g),
|
|
49
|
-
colon: count(/:/g),
|
|
50
|
-
question: count(/\?/g),
|
|
51
|
-
exclamation: count(/!/g),
|
|
52
|
-
parentheses: count(/[()]/g) / 2,
|
|
53
|
-
dash: count(/—|\s--\s/g),
|
|
54
|
-
};
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
export function calculateMetrics(tokenized, normalizedText) {
|
|
58
|
-
const lengths = tokenized.sentences.map((sentence) => sentence.wordCount);
|
|
59
|
-
const wordCount = tokenized.words.length;
|
|
60
|
-
const sentenceCount = lengths.length;
|
|
61
|
-
return {
|
|
62
|
-
wordCount,
|
|
63
|
-
sentenceCount,
|
|
64
|
-
meanSentenceLength: mean(lengths),
|
|
65
|
-
stdDev: standardDeviation(lengths),
|
|
66
|
-
burstinessCV: coefficientOfVariation(lengths),
|
|
67
|
-
adjacentDelta: adjacentDelta(lengths),
|
|
68
|
-
shortSentenceRatio: sentenceCount ? lengths.filter((length) => length <= 6).length / sentenceCount : 0,
|
|
69
|
-
longSentenceRatio: sentenceCount ? lengths.filter((length) => length >= 26).length / sentenceCount : 0,
|
|
70
|
-
mattr: mattr(tokenized.words),
|
|
71
|
-
compressionRatio: compressionRatio(normalizedText),
|
|
72
|
-
};
|
|
73
|
-
}
|
|
1
|
+
import { deflateRawSync } from 'node:zlib';
|
|
2
|
+
|
|
3
|
+
import {
|
|
4
|
+
adjacentDelta,
|
|
5
|
+
coefficientOfVariation,
|
|
6
|
+
createMetrics,
|
|
7
|
+
mattr,
|
|
8
|
+
mean,
|
|
9
|
+
punctuationRates,
|
|
10
|
+
standardDeviation,
|
|
11
|
+
} from './metrics-core.mjs';
|
|
12
|
+
|
|
13
|
+
const metrics = createMetrics({
|
|
14
|
+
encode: (text) => Buffer.from(text, 'utf8'),
|
|
15
|
+
deflateRaw: (bytes) => deflateRawSync(bytes),
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
export {
|
|
19
|
+
adjacentDelta,
|
|
20
|
+
coefficientOfVariation,
|
|
21
|
+
mattr,
|
|
22
|
+
mean,
|
|
23
|
+
punctuationRates,
|
|
24
|
+
standardDeviation,
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
export const compressionRatio = metrics.compressionRatio;
|
|
28
|
+
export const calculateMetrics = metrics.calculateMetrics;
|