elect 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/elect/cleaner.rb +18 -12
- data/lib/elect/cli.rb +1 -1
- data/lib/elect/pattern.rb +17 -0
- data/lib/elect/processor.rb +8 -6
- data/lib/elect/version.rb +1 -1
- data/lib/elect.rb +2 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 8f2c7494c8ded06589b2e6776d3adb086665072796c0bc29cc531416be1c74e2
|
|
4
|
+
data.tar.gz: 373fea6a80865af29e1790e1161980ffbda34eb6ba4c170e38c81e91aa9fc297
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6e4ba49802f8d14a3e081e430d304c67b1aafe16e1b9a5b73bb8b663b0210fa7f50a65725eb96fe927f620b3ea15324b5d26914a94ec6807ec9e77f6e80576a7
|
|
7
|
+
data.tar.gz: a57120d7a330804e1bdd62e8b26e7da115416437a9027c538612f3d4ad23261fb07b299f84e490df84ebc30d91463545bac67453192ac5e172c16df985c6144d
|
data/lib/elect/cleaner.rb
CHANGED
|
@@ -4,46 +4,52 @@
|
|
|
4
4
|
require_relative "pattern"
|
|
5
5
|
|
|
6
6
|
module Elect
|
|
7
|
-
# 清理器:接收文本,移除 AIGC
|
|
7
|
+
# 清理器:接收文本,移除 AIGC 水印块与 `> AI生成` 提示行。
|
|
8
8
|
#
|
|
9
9
|
module Cleaner
|
|
10
|
-
#
|
|
10
|
+
# 从文本中移除所有水印块与 AI 生成提示行。
|
|
11
11
|
#
|
|
12
12
|
# 策略:
|
|
13
|
-
# 1. 用 Pattern::REGEX
|
|
14
|
-
# 2.
|
|
15
|
-
# 3.
|
|
13
|
+
# 1. 用 Pattern::REGEX 全局替换为空串(水印块)
|
|
14
|
+
# 2. 用 Pattern::AI_GEN_LINE 全局替换为空串(> AI生成 行)
|
|
15
|
+
# 3. 清理因移除产生的连续空行(>2 连续空行压缩为 2)
|
|
16
|
+
# 4. 清理文件首尾多余空白
|
|
16
17
|
#
|
|
17
18
|
# 返回 [清理后文本, 移除数量]
|
|
18
19
|
#
|
|
19
20
|
def self.clean(text)
|
|
20
|
-
n =
|
|
21
|
+
n = detect(text)
|
|
21
22
|
return [text, 0] if n.zero?
|
|
22
23
|
|
|
24
|
+
# 探测原始换行符类型,压缩空行与补尾换行时保持一致,
|
|
25
|
+
# 避免 CRLF 文件被混入 LF(反之亦然)。
|
|
26
|
+
nl = text.include?("\r\n") ? "\r\n" : "\n"
|
|
27
|
+
|
|
23
28
|
cleaned = text.gsub(Pattern::REGEX, "")
|
|
29
|
+
cleaned = cleaned.gsub(Pattern::AI_GEN_LINE, "")
|
|
24
30
|
|
|
25
31
|
# 压缩连续空行(保留最多 2 换行 = 1 空行),兼容 CRLF
|
|
26
|
-
cleaned.gsub!(/(?:\r\n|\n){3,}/,
|
|
32
|
+
cleaned.gsub!(/(?:\r\n|\n){3,}/, nl + nl)
|
|
27
33
|
|
|
28
34
|
# 移除首尾多余空白
|
|
29
35
|
cleaned.strip!
|
|
30
36
|
|
|
31
37
|
# 确保文件结尾有且仅有一个换行
|
|
32
|
-
cleaned <<
|
|
38
|
+
cleaned << nl unless cleaned.empty?
|
|
33
39
|
|
|
34
40
|
[cleaned, n]
|
|
35
41
|
end
|
|
36
42
|
|
|
37
|
-
#
|
|
43
|
+
# 仅检测不修改,返回水印块 + AI 生成行总数
|
|
38
44
|
#
|
|
39
45
|
def self.detect(text)
|
|
40
|
-
Pattern.count(text)
|
|
46
|
+
Pattern.count(text) + Pattern.ai_gen_count(text)
|
|
41
47
|
end
|
|
42
48
|
|
|
43
|
-
#
|
|
49
|
+
# 判断文本是否含水印或 AI 生成提示行
|
|
44
50
|
#
|
|
45
51
|
def self.watermarked?(text)
|
|
46
|
-
Pattern.watermarked?(text)
|
|
52
|
+
Pattern.watermarked?(text) || Pattern.ai_gen_count(text).positive?
|
|
47
53
|
end
|
|
48
54
|
end
|
|
49
55
|
end
|
data/lib/elect/cli.rb
CHANGED
data/lib/elect/pattern.rb
CHANGED
|
@@ -47,6 +47,16 @@ module Elect
|
|
|
47
47
|
)
|
|
48
48
|
/x.freeze
|
|
49
49
|
|
|
50
|
+
# 以 `> AI生成` 开头的行(Markdown 块引用格式的 AI 生成提示标记)。
|
|
51
|
+
#
|
|
52
|
+
# 形如:
|
|
53
|
+
# > AI生成
|
|
54
|
+
# > AI生成:本节内容由 AI 生成
|
|
55
|
+
#
|
|
56
|
+
# 匹配整行并连同换行符一起删除,兼容 LF / CRLF。
|
|
57
|
+
#
|
|
58
|
+
AI_GEN_LINE = /^[ \t]*>[ \t]*AI生成[^\r\n]*\r?\n/.freeze
|
|
59
|
+
|
|
50
60
|
# 匹配水印块,返回 MatchData 或 nil
|
|
51
61
|
#
|
|
52
62
|
def self.match(text)
|
|
@@ -68,6 +78,13 @@ module Elect
|
|
|
68
78
|
s.scan(REGEX).size
|
|
69
79
|
end
|
|
70
80
|
|
|
81
|
+
# 统计以 `> AI生成` 开头的行数
|
|
82
|
+
#
|
|
83
|
+
def self.ai_gen_count(text)
|
|
84
|
+
s = sanitize(text)
|
|
85
|
+
s.scan(AI_GEN_LINE).size
|
|
86
|
+
end
|
|
87
|
+
|
|
71
88
|
# 提取所有水印块的文本(供调试用)
|
|
72
89
|
#
|
|
73
90
|
def self.extract(text)
|
data/lib/elect/processor.rb
CHANGED
|
@@ -68,15 +68,17 @@ module Elect
|
|
|
68
68
|
if dry_run
|
|
69
69
|
result.removed = n
|
|
70
70
|
result.status = :found
|
|
71
|
-
puts " [DRY] #{p} — 发现 #{n}
|
|
71
|
+
puts " [DRY] #{p} — 发现 #{n} 处水印/AI标记"
|
|
72
72
|
return result
|
|
73
73
|
end
|
|
74
74
|
|
|
75
75
|
cleaned, removed = Cleaner.clean(raw)
|
|
76
|
-
|
|
76
|
+
# 用二进制模式写回,避免 Windows 文本模式把 \r\n 里的 \n
|
|
77
|
+
# 二次转成 \r\n,产生 \r\r\n 导致每行后多出一个空行。
|
|
78
|
+
File.binwrite(p.to_s, cleaned)
|
|
77
79
|
result.removed = removed
|
|
78
80
|
result.status = :cleaned
|
|
79
|
-
puts " [OK] #{p} — 移除 #{removed}
|
|
81
|
+
puts " [OK] #{p} — 移除 #{removed} 处水印/AI标记" if verbose
|
|
80
82
|
result
|
|
81
83
|
end
|
|
82
84
|
|
|
@@ -129,9 +131,9 @@ module Elect
|
|
|
129
131
|
puts " 扫描完毕"
|
|
130
132
|
puts "=" * 50
|
|
131
133
|
puts " 扫描文件 : #{total}"
|
|
132
|
-
puts "
|
|
133
|
-
puts " 已清理 : #{cleaned} 个文件 (共移除 #{total_rm}
|
|
134
|
-
puts "
|
|
134
|
+
puts " 发现标记 : #{found + cleaned} 个文件"
|
|
135
|
+
puts " 已清理 : #{cleaned} 个文件 (共移除 #{total_rm} 处水印/AI标记)"
|
|
136
|
+
puts " 无标记 : #{clean} 个文件"
|
|
135
137
|
puts " 跳过 : #{skipped}"
|
|
136
138
|
puts " 错误 : #{errors}" if errors.positive?
|
|
137
139
|
puts "=" * 50
|
data/lib/elect/version.rb
CHANGED
data/lib/elect.rb
CHANGED