mm_matchs 1.1.8 → 1.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +125 -125
- package/eslint.config.js +106 -0
- package/html.js +220 -220
- package/htmlformat.js +391 -395
- package/index.js +340 -311
- package/package.json +4 -2
- package/regex.js +80 -80
- package/test.js +206 -206
- package/test_html.js +2284 -2284
- package/index copy.js +0 -897
package/html.js
CHANGED
|
@@ -5,246 +5,246 @@ const htmlformat = require('./htmlformat');
|
|
|
5
5
|
* 解析Html页面的文章正文内容,基于文本密度的HTML正文提取类
|
|
6
6
|
* @class
|
|
7
7
|
*/
|
|
8
|
-
class
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
8
|
+
class HtmlToArticle {
|
|
9
|
+
/**
|
|
10
|
+
* 构造函数
|
|
11
|
+
* @param {object} arg
|
|
12
|
+
*/
|
|
13
|
+
constructor(arg) {
|
|
14
|
+
// 导入正则表达式
|
|
15
|
+
this.regex = require('./regex');
|
|
16
|
+
|
|
17
|
+
// 正则表达式过滤:正则表达式,要替换成的文本
|
|
18
|
+
this.filters = [{
|
|
19
|
+
rx: this.regex.script,
|
|
20
|
+
value: ''
|
|
21
|
+
}, {
|
|
22
|
+
rx: this.regex.style,
|
|
23
|
+
value: ''
|
|
24
|
+
}, {
|
|
25
|
+
rx: this.regex.comment,
|
|
26
|
+
value: ''
|
|
27
|
+
}, {
|
|
28
|
+
rx: this.regex.link,
|
|
29
|
+
value: '</a>\n'
|
|
30
|
+
}];
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* 是否使用追加模式,默认为false
|
|
34
|
+
* 使用追加模式后,会将符合过滤条件的所有文本提取出来
|
|
35
|
+
*/
|
|
36
|
+
this.append_mode = false;
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* 按行分析的深度,默认为6
|
|
40
|
+
*/
|
|
41
|
+
this.depth = 6;
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* 字符限定数,当分析的文本数量达到限定数则认为进入正文内容
|
|
45
|
+
* 默认180个字符数
|
|
46
|
+
*/
|
|
47
|
+
this.limit_count = 125;
|
|
48
|
+
|
|
49
|
+
// 确定文章正文头部时,向上查找,连续的空行到达this.headEmptyLines,则停止查找
|
|
50
|
+
this.head_empty_lines = 2;
|
|
51
|
+
|
|
52
|
+
// 用于确定文章结束的字符数
|
|
53
|
+
this.end_limit_char_count = 20;
|
|
54
|
+
}
|
|
55
55
|
}
|
|
56
56
|
|
|
57
57
|
/**
|
|
58
58
|
* 从给定的Html原始文本中获取正文信息
|
|
59
|
-
* @param {
|
|
60
|
-
* @
|
|
59
|
+
* @param {string} html
|
|
60
|
+
* @returns {string} 返回正文
|
|
61
61
|
*/
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
62
|
+
HtmlToArticle.prototype.getArticle = function(html) {
|
|
63
|
+
var htm = html.trim();
|
|
64
|
+
// 如果换行符的数量小于10,则认为html为压缩后的html
|
|
65
|
+
// 由于处理算法是按照行进行处理,需要为html标签添加换行符,便于处理
|
|
66
|
+
var arr = htm.split('\n');
|
|
67
|
+
if (arr.length < 10) {
|
|
68
|
+
htm = htm.replace(/>/g, '>\n');
|
|
69
|
+
}
|
|
70
|
+
var tabsize = 1;
|
|
71
|
+
var tabchar = ' ';
|
|
72
|
+
var max_char = 80;
|
|
73
|
+
htm = htmlformat(htm, tabsize, tabchar, max_char);
|
|
74
|
+
// 获取html,body标签内容
|
|
75
|
+
var body = '';
|
|
76
|
+
|
|
77
|
+
if (htm.indexOf('<article') !== -1) {
|
|
78
|
+
var mh = htm.match(eval(this.regex.article));
|
|
79
|
+
if (mh) {
|
|
80
|
+
body = mh[0];
|
|
81
|
+
}
|
|
82
|
+
} else {
|
|
83
|
+
if (htm.indexOf('<main') !== -1) {
|
|
84
|
+
var mh = htm.match(eval(this.regex.main));
|
|
85
|
+
if (mh) {
|
|
86
|
+
body = mh[0];
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
var mh = htm.match(eval(this.regex.body));
|
|
91
|
+
if (mh) {
|
|
92
|
+
body = mh[0];
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
// 过滤样式,脚本等不相干标签
|
|
96
|
+
var lt = this.filters;
|
|
97
|
+
var len = lt.length;
|
|
98
|
+
for (var i = 0; i < len; i++) {
|
|
99
|
+
var o = lt[i];
|
|
100
|
+
var rx = eval(o.rx);
|
|
101
|
+
body = body.replace(rx, o.value);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
var {
|
|
106
|
+
content,
|
|
107
|
+
content_tag
|
|
108
|
+
} = this.getContent(body);
|
|
109
|
+
|
|
110
|
+
var article = {
|
|
111
|
+
title: this.getTitle(htm),
|
|
112
|
+
datetime: this.getPublishDate(body),
|
|
113
|
+
content: content,
|
|
114
|
+
html: htmlformat(content_tag, tabsize, tabchar, max_char)
|
|
115
|
+
};
|
|
116
|
+
|
|
117
|
+
return article;
|
|
117
118
|
};
|
|
118
119
|
|
|
119
120
|
|
|
120
121
|
/**
|
|
121
122
|
* 获取标题
|
|
122
|
-
* @param {
|
|
123
|
-
* @
|
|
123
|
+
* @param {string} html
|
|
124
|
+
* @returns {string} 返回标题
|
|
124
125
|
*/
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
126
|
+
HtmlToArticle.prototype.getTitle = function(html) {
|
|
127
|
+
var titleFilter = /<title>[\s\S]*?<\/title>/gi;
|
|
128
|
+
var h1Filter = /<h1.*?>.*?<\/h1>/gi;
|
|
129
|
+
var clearFilter = /<.*?>/g;
|
|
130
|
+
|
|
131
|
+
var title = '';
|
|
132
|
+
var mh = html.match(titleFilter);
|
|
133
|
+
if (mh) {
|
|
134
|
+
title = mh[0].replace(clearFilter, '');
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
if (!title) {
|
|
138
|
+
// 正文的标题一般在h1中,比title中的标题更干净
|
|
139
|
+
mh = html.match(h1Filter);
|
|
140
|
+
if (mh) {
|
|
141
|
+
title = mh[0].replace(clearFilter, '');
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
return title;
|
|
144
145
|
};
|
|
145
146
|
|
|
146
147
|
/**
|
|
147
148
|
* 获取文章发布日期
|
|
148
|
-
* @param {
|
|
149
|
-
* @
|
|
149
|
+
* @param {string} html
|
|
150
|
+
* @returns {string} 返回时间字符串
|
|
150
151
|
*/
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
152
|
+
HtmlToArticle.prototype.getPublishDate = function(html) {
|
|
153
|
+
var text = html.replace(eval(this.regex.clear_tag), '');
|
|
154
|
+
var rx = eval(this.regex.datetime);
|
|
155
|
+
var mh = text.match(rx);
|
|
156
|
+
var result = '1900-01-01';
|
|
157
|
+
if (mh) {
|
|
158
|
+
result = mh[0].replace('年', '-').replace('月', '-').replace('日', '-').replace('时', ':').replace('分', ':').replace('秒',
|
|
159
|
+
'').replace(/\//g, '-');
|
|
160
|
+
}
|
|
161
|
+
return result;
|
|
161
162
|
};
|
|
162
163
|
|
|
163
164
|
/**
|
|
164
165
|
* 从body标签文本中分析正文内容
|
|
165
|
-
* @param {
|
|
166
|
-
* @param {
|
|
167
|
-
* @param {Striing} content_tag 返回文本正文,不包含标签
|
|
166
|
+
* @param {string} body_text
|
|
167
|
+
* @param {string} content_tag 返回文本正文,不包含标签
|
|
168
168
|
*/
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
169
|
+
HtmlToArticle.prototype.getContent = function(body_text) {
|
|
170
|
+
// 保存干净的文本内容,不包含标签
|
|
171
|
+
var lines = [];
|
|
172
|
+
|
|
173
|
+
// 保存原始内容,按行存储
|
|
174
|
+
var org_lines = body_text.split('\n');
|
|
175
|
+
|
|
176
|
+
// 去除每行的空白字符,剔除标签
|
|
177
|
+
for (var i = 0; i < org_lines.length; i++) {
|
|
178
|
+
var line_info = org_lines[i];
|
|
179
|
+
var str = line_info.replace(eval(this.regex.line_break), '\r\n').replace(eval(this.regex.clear_tag), '').trim();
|
|
180
|
+
if (str) {
|
|
181
|
+
lines.push(str);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
var sb = '';
|
|
185
|
+
var org_sb = '';
|
|
186
|
+
|
|
187
|
+
// 记录上一次统计的字符数量
|
|
188
|
+
var pre_text_len = 0;
|
|
189
|
+
|
|
190
|
+
// 记录文章正文的起始位置
|
|
191
|
+
var start_pos = -1;
|
|
192
|
+
var d = this.depth;
|
|
193
|
+
for (var i = 0; i < lines.length - d; i++) {
|
|
194
|
+
var len = 0;
|
|
195
|
+
for (var j = 0; j < d; j++) {
|
|
196
|
+
len += lines[i + j].length;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
if (start_pos == -1) // 还没有找到文章起始位置,需要判断起始位置
|
|
200
|
+
{
|
|
201
|
+
if (pre_text_len > this.limit_count && len > 0) // 如果上次查找的文本数量超过了限定字数,且当前行数字符数不为0,则认为是开始位置
|
|
202
|
+
{
|
|
203
|
+
// 查找文章起始位置, 如果向上查找,发现2行连续的空行则认为是头部
|
|
204
|
+
var empty_count = 0;
|
|
205
|
+
for (var j = i - 1; j > 0; j--) {
|
|
206
|
+
if (!lines[j]) {
|
|
207
|
+
empty_count++;
|
|
208
|
+
} else {
|
|
209
|
+
empty_count = 0;
|
|
210
|
+
}
|
|
211
|
+
if (empty_count == this.head_empty_lines) {
|
|
212
|
+
start_pos = j + this.head_empty_lines;
|
|
213
|
+
break;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
// 如果没有定位到文章头,则以当前查找位置作为文章头
|
|
217
|
+
if (start_pos == -1) {
|
|
218
|
+
start_pos = i;
|
|
219
|
+
}
|
|
220
|
+
// 填充发现的文章起始部分
|
|
221
|
+
for (var j = start_pos; j <= i; j++) {
|
|
222
|
+
sb += lines[j];
|
|
223
|
+
org_sb += org_lines[j];
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
} else {
|
|
227
|
+
//if (len == 0 && pre_text_len == 0) // 当前长度为0,且上一个长度也为0,则认为已经结束
|
|
228
|
+
if (len <= this.end_limit_char_count && pre_text_len < this.end_limit_char_count) // 当前长度为0,且上一个长度也为0,则认为已经结束
|
|
229
|
+
{
|
|
230
|
+
if (!this.end_limit_char_count) {
|
|
231
|
+
break;
|
|
232
|
+
}
|
|
233
|
+
start_pos = -1;
|
|
234
|
+
}
|
|
235
|
+
sb += lines[i];
|
|
236
|
+
org_sb += org_lines[i];
|
|
237
|
+
}
|
|
238
|
+
pre_text_len = len;
|
|
239
|
+
}
|
|
240
|
+
// 处理回车符,更好的将文本格式化输出
|
|
241
|
+
var content = sb;
|
|
242
|
+
// 输出带标签文本
|
|
243
|
+
var content_tag = org_sb;
|
|
244
|
+
return {
|
|
245
|
+
content,
|
|
246
|
+
content_tag
|
|
247
|
+
};
|
|
248
248
|
};
|
|
249
249
|
|
|
250
|
-
module.exports =
|
|
250
|
+
module.exports = HtmlToArticle;
|