dsh-bagu 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dsh-plugin.json +1 -1
- package/lib/analyze.js +19 -6
- package/lib/config.js +1 -1
- package/lib/lexicon.js +25 -5
- package/package.json +1 -1
- package/test/analyze.test.mjs +60 -0
package/dsh-plugin.json
CHANGED
package/lib/analyze.js
CHANGED
|
@@ -28,6 +28,7 @@ import {
|
|
|
28
28
|
EVAL_ADJECTIVES,
|
|
29
29
|
HEDGE_WORDS,
|
|
30
30
|
CONCRETE_MARKERS,
|
|
31
|
+
CONCRETE_WEIGHTS,
|
|
31
32
|
STRUCTURE_FRAMES,
|
|
32
33
|
REGISTERS,
|
|
33
34
|
HEDGE_BASELINE,
|
|
@@ -91,14 +92,17 @@ export function registerOf(text) {
|
|
|
91
92
|
export function concreteness(text) {
|
|
92
93
|
const src = String(text ?? '');
|
|
93
94
|
const hits = [];
|
|
95
|
+
let weighted = 0;
|
|
94
96
|
for (const [kind, re] of Object.entries(CONCRETE_MARKERS)) {
|
|
95
97
|
const g = new RegExp(re.source, 'g');
|
|
98
|
+
const w = CONCRETE_WEIGHTS[kind] ?? 1;
|
|
96
99
|
for (const m of src.matchAll(g)) {
|
|
97
|
-
hits.push({ kind, text: m[0], at: m.index });
|
|
100
|
+
hits.push({ kind, text: m[0], at: m.index, weight: w });
|
|
101
|
+
weighted += w;
|
|
98
102
|
}
|
|
99
103
|
}
|
|
100
|
-
const per100 = src.length ? (
|
|
101
|
-
return { hits, per100: Math.round(per100 * 100) / 100 };
|
|
104
|
+
const per100 = src.length ? (weighted / src.length) * 100 : 0;
|
|
105
|
+
return { hits, weighted, per100: Math.round(per100 * 100) / 100 };
|
|
102
106
|
}
|
|
103
107
|
|
|
104
108
|
/** 结构腔扫描。 */
|
|
@@ -162,7 +166,6 @@ export function analyze(input, opts = {}) {
|
|
|
162
166
|
const structures = scanStructures(text);
|
|
163
167
|
const totalStructures = structures.reduce((a, r) => a + r.count, 0);
|
|
164
168
|
const structurePer1k = per1k(totalStructures);
|
|
165
|
-
const allowed = Math.max(chars >= 200 ? 1 : 0, Math.round((baseline * chars) / 1000));
|
|
166
169
|
const overTimes = baseline ? Math.round((structurePer1k / baseline) * 10) / 10 : 0;
|
|
167
170
|
|
|
168
171
|
const flatHits = [];
|
|
@@ -179,18 +182,28 @@ export function analyze(input, opts = {}) {
|
|
|
179
182
|
const evalPer100 = per100(evalHits.length);
|
|
180
183
|
const aiWordPer100 = per100(aiWordHits.length);
|
|
181
184
|
|
|
182
|
-
|
|
185
|
+
// 六项坏信号相加,另加一个减分位:具体物足够多时往回拉一格。
|
|
186
|
+
// 否则「讲了十件亲手做过的事 + 用了两个抽象词」和「全是空话 + 两个抽象词」
|
|
187
|
+
// 在第 2 项上完全同分(issue #1 第 4 条)。
|
|
188
|
+
const raw1 =
|
|
183
189
|
(conc.per100 < concreteFloor ? 1 : 0) +
|
|
184
190
|
(abstractPer100 + evalPer100 >= 2 ? 1 : 0) +
|
|
185
191
|
(aiWordPer100 >= 1 || aiWordHits.length >= 2 ? 1 : 0) +
|
|
186
192
|
(structurePer1k > baseline * structureTimes ? 1 : 0) +
|
|
187
193
|
(hedgePer1k < hedgeFloor && chars > 40 ? 1 : 0) +
|
|
188
194
|
(summaryHits.length + fakeHits.length > 0 ? 1 : 0);
|
|
195
|
+
const credit = (conc.per100 >= concreteFloor * 2 ? 1 : 0);
|
|
196
|
+
const replaceable = Math.max(0, Math.min(6, raw1 - credit));
|
|
189
197
|
|
|
190
198
|
let band;
|
|
191
199
|
let bandLabel;
|
|
192
200
|
let bandNote;
|
|
193
|
-
if (
|
|
201
|
+
if (chars < 60) {
|
|
202
|
+
// 六十个字以下密度不稳:一两个词的增减就能跨过阈值(issue #1 第 8 条)
|
|
203
|
+
band = 'too-short';
|
|
204
|
+
bandLabel = '样本太短';
|
|
205
|
+
bandNote = '不到六十字,只看下面各项指标,判档这时不可靠。';
|
|
206
|
+
} else if (replaceable >= 4) {
|
|
194
207
|
band = 'heavy';
|
|
195
208
|
bandLabel = '换谁都行';
|
|
196
209
|
bandNote = '把这些句子搬到另一个人嘴里,读起来不会有差别。';
|
package/lib/config.js
CHANGED
package/lib/lexicon.js
CHANGED
|
@@ -143,8 +143,21 @@ export const CONCRETE_MARKERS = {
|
|
|
143
143
|
sense: /(?:看见|听到|闻|摸|尝|痛|酸|烫|凉|冷|热|饿|困|累|味道|声音|手感)/g,
|
|
144
144
|
/** 引号里的原话、书名号、代码 */
|
|
145
145
|
quoted: /(?:「[^」]{1,40}」|"[^"]{1,40}"|《[^》]{1,40}》|`[^`]{1,40}`)/g,
|
|
146
|
-
/**
|
|
147
|
-
ident: /(?:[A-Za-z_][A-Za-z0-9_
|
|
146
|
+
/** 具体到能指认的东西:路径、命令、英文标识符。必须带非字母字符或数字,否则 the/and/before 全命中(issue #1 第 1 条)。 */
|
|
147
|
+
ident: /(?:[A-Za-z_][A-Za-z0-9_]*[\/._:=-][\w./:-]{2,}|\/[\w./-]{3,}|\b[A-Z]{2,}\b|\d+[A-Za-z]+)/g,
|
|
148
|
+
};
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* 各类具体物的权重。路径串和版本号最容易凭空写出来,
|
|
152
|
+
* 亲身经历和身体感受最难(issue #1 第 2 条)。
|
|
153
|
+
*/
|
|
154
|
+
export const CONCRETE_WEIGHTS = {
|
|
155
|
+
experience: 1.5,
|
|
156
|
+
sense: 1.5,
|
|
157
|
+
time: 1.0,
|
|
158
|
+
number: 0.5,
|
|
159
|
+
ident: 0.5,
|
|
160
|
+
quoted: 0,
|
|
148
161
|
};
|
|
149
162
|
|
|
150
163
|
/** 结构腔。per1k 是真人评论语料的实测密度。 */
|
|
@@ -167,8 +180,15 @@ export const REGISTERS = {
|
|
|
167
180
|
steps: { label: '步骤', per1k: 2.6 },
|
|
168
181
|
};
|
|
169
182
|
|
|
170
|
-
/**
|
|
171
|
-
|
|
183
|
+
/**
|
|
184
|
+
* 真人留白词密度(每千字)。
|
|
185
|
+
*
|
|
186
|
+
* 这个数字我核不出来了 —— 原始语料不在这台机器上,注释里留的两组数
|
|
187
|
+
* (0.41~0.70 vs 2.0)差三到五倍,issue #1 第 6a 条指的就是这个。
|
|
188
|
+
* 按保守取值:用 0.55(注释区间上限附近),宁可少报也不误伤真人。
|
|
189
|
+
* 重新量过语料的人应该把这里改回实测值。
|
|
190
|
+
*/
|
|
191
|
+
export const HEDGE_BASELINE = 0.55;
|
|
172
192
|
|
|
173
193
|
/** 留白词低于这条线就报「句句在下断言」。 */
|
|
174
|
-
export const HEDGE_FLOOR = 0.
|
|
194
|
+
export const HEDGE_FLOOR = 0.2;
|
package/package.json
CHANGED
package/test/analyze.test.mjs
CHANGED
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
import { test } from 'node:test';
|
|
2
2
|
import assert from 'node:assert/strict';
|
|
3
|
+
import { readFileSync } from 'node:fs';
|
|
4
|
+
import { fileURLToPath } from 'node:url';
|
|
5
|
+
import { dirname, join } from 'node:path';
|
|
3
6
|
import { analyze, render, registerOf, stripCode, concreteness, scanStructures, cluster } from '../lib/analyze.js';
|
|
4
7
|
|
|
8
|
+
const here = dirname(fileURLToPath(import.meta.url));
|
|
9
|
+
|
|
5
10
|
test('典型八股判成 heavy', () => {
|
|
6
11
|
const r = analyze(
|
|
7
12
|
'本次修改已完成。首先调整了章节结构,其次新增了规则条款,最后优化了可读性。' +
|
|
@@ -90,3 +95,58 @@ test('同一个输入跑两次结果一致', () => {
|
|
|
90
95
|
const s = '首先这样,其次那样。总的来说,不是甲而是乙。';
|
|
91
96
|
assert.deepEqual(analyze(s).scores, analyze(s).scores);
|
|
92
97
|
});
|
|
98
|
+
|
|
99
|
+
// ── issue #1 的回归用例 ────────────────────────────────────
|
|
100
|
+
|
|
101
|
+
test('#1.1 英文普通单词不再算具体物', () => {
|
|
102
|
+
const r = analyze('The build pipeline uses a cache layer and the worker reads a manifest before it starts.');
|
|
103
|
+
assert.equal(r.hits.concrete.filter((h) => h.kind === 'ident' && /^[a-z]+$/.test(h.text)).length, 0);
|
|
104
|
+
assert.ok(r.scores.concretePer100 < 2, '纯英文散文该接近 0,实际 ' + r.scores.concretePer100);
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
test('#1.1 真正的标识符仍然计入', () => {
|
|
108
|
+
const r = analyze('把它放到 /usr/local/bin/node,然后设 EADDRINUSE 那个变量,路径是 C:/Users/x/y.js。');
|
|
109
|
+
const ids = r.hits.concrete.filter((h) => h.kind === 'ident').map((h) => h.text);
|
|
110
|
+
assert.ok(ids.some((t) => t.includes('/usr/local/bin/node')));
|
|
111
|
+
assert.ok(ids.some((t) => t.includes('EADDRINUSE')));
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
test('#1.2 experience 和 sense 的权重高于 number 和 ident', () => {
|
|
115
|
+
const a = concreteness('我昨天试了一下,手都酸了。');
|
|
116
|
+
const b = concreteness('版本 1.2.3,路径 /a/b/c,端口 8080。');
|
|
117
|
+
assert.ok(a.weighted > 0);
|
|
118
|
+
assert.ok(b.weighted > 0);
|
|
119
|
+
// a 是两条经历类,b 是两条数字/路径类,按权重 a 应该更高
|
|
120
|
+
assert.ok(a.weighted >= b.weighted, `experience 权重不足: a=${a.weighted} b=${b.weighted}`);
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
test('#1.4 具体物充足时往回减一格', () => {
|
|
124
|
+
const rich = analyze(
|
|
125
|
+
'昨晚十一点半我第三次跑那个脚本,又卡在 8443 端口。日志里只有 EADDRINUSE。' +
|
|
126
|
+
'我盯着看了半天才反应过来,后来把之前那个进程 kill 掉就好了,大概是我自己忘了关。'
|
|
127
|
+
);
|
|
128
|
+
const empty = analyze(
|
|
129
|
+
'这个方案具有重要意义,是关键的转折点,体现了深刻的洞察,值得每个人认真思考和学习。'
|
|
130
|
+
);
|
|
131
|
+
assert.ok(rich.scores.replaceable < empty.scores.replaceable,
|
|
132
|
+
`具体的那段不该比空话更可疑: rich=${rich.scores.replaceable} empty=${empty.scores.replaceable}`);
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
test('#1.8 六十个字以下不给判档', () => {
|
|
136
|
+
const r = analyze('这个挺好。');
|
|
137
|
+
assert.equal(r.band, 'too-short');
|
|
138
|
+
assert.ok(r.bandNote.includes('六十字'), 'bandNote: ' + r.bandNote);
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
test('#1.8 六十字以上照常判档', () => {
|
|
142
|
+
const r = analyze(
|
|
143
|
+
'本次修改已完成。首先调整了章节结构,其次新增了规则条款,最后优化了可读性。' +
|
|
144
|
+
'综上所述,这次改动具有重要意义,希望对你有帮助。'
|
|
145
|
+
);
|
|
146
|
+
assert.notEqual(r.band, 'too-short');
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
test('#1.6b 没有死代码留下的未使用变量', () => {
|
|
150
|
+
const src = readFileSync(join(here, '..', 'lib', 'analyze.js'), 'utf8');
|
|
151
|
+
assert.ok(!src.includes('const allowed ='), 'allowed 应该已经删掉');
|
|
152
|
+
});
|