sunflower 0.3 → 0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +1 -1
- data/lib/sunflower/commontasks.rb +265 -250
- data/lib/sunflower/core.rb +288 -287
- data/lib/sunflower/listmaker.rb +160 -152
- data/scripts/ZDBOT.rb +61 -61
- data/scripts/aktualizacjapilkarzy.rb +339 -339
- data/scripts/changeimage.rb +41 -41
- data/scripts/fix-bold-in-headers.rb +41 -53
- data/scripts/fix-double-pipes.rb +30 -49
- data/scripts/fix-langs.rb +42 -42
- data/scripts/fix-multiple-same-refs.rb +101 -101
- data/scripts/fix-some-entities.rb +36 -43
- data/scripts/fix-unicode-control-chars.rb +30 -51
- data/scripts/insight.rb +132 -132
- data/scripts/lekkoatl-portal.rb +50 -50
- data/scripts/make-id2team-list.rb +31 -31
- data/scripts/recat.rb +27 -32
- data/scripts/wanted.rb +72 -72
- metadata +40 -62
data/lib/sunflower/listmaker.rb
CHANGED
@@ -1,152 +1,160 @@
|
|
1
|
-
# coding: utf-8
|
2
|
-
class Sunflower
|
3
|
-
# Makes a list of articles. Returns array of titles.
|
4
|
-
def make_list type, *parameters
|
5
|
-
type=type.downcase.gsub(/[^a-z]/, '')
|
6
|
-
first=parameters[0]
|
7
|
-
firstE=CGI.escape first.to_s
|
8
|
-
|
9
|
-
case type
|
10
|
-
when 'file'
|
11
|
-
f=File.open first
|
12
|
-
list=f.read.sub(/\357\273\277/,'').strip.split(/\r?\n/)
|
13
|
-
f.close
|
14
|
-
|
15
|
-
when 'page', 'pages'
|
16
|
-
list=parameters
|
17
|
-
|
18
|
-
when 'input'
|
19
|
-
puts 'Insert titles of articles to edit:'
|
20
|
-
puts 'Press [Enter] without inputting any text to finish.'
|
21
|
-
puts 'Press [Ctrl]+[C] to kill bot.'
|
22
|
-
list=[]
|
23
|
-
while true
|
24
|
-
input=gets.strip
|
25
|
-
break if input==''
|
26
|
-
|
27
|
-
list<<input
|
28
|
-
end
|
29
|
-
|
30
|
-
when 'categorieson'
|
31
|
-
r=self.API('action=query&prop=categories&cllimit=500&titles='+firstE)
|
32
|
-
list=r['query']['pages'].first['categories'].map{|v| v['title']}
|
33
|
-
|
34
|
-
when 'category'
|
35
|
-
r=self.API('action=query&list=categorymembers&cmprop=title&cmlimit=5000&cmtitle='+firstE)
|
36
|
-
list=r['query']['categorymembers'].map{|v| v['title']}
|
37
|
-
|
38
|
-
when 'categoryr', 'categoryrecursive'
|
39
|
-
list=[] #list of articles
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
list
|
56
|
-
|
57
|
-
when '
|
58
|
-
r=self.API('action=query&
|
59
|
-
list=r['query']['
|
60
|
-
|
61
|
-
when '
|
62
|
-
r=self.API('action=query&
|
63
|
-
list=r['query']['
|
64
|
-
|
65
|
-
when '
|
66
|
-
r=self.API('action=query&list=
|
67
|
-
list=r['query']['
|
68
|
-
|
69
|
-
when '
|
70
|
-
r=self.API('action=query&list=
|
71
|
-
list=r['query']['
|
72
|
-
|
73
|
-
when '
|
74
|
-
r=self.API('action=query&list=
|
75
|
-
list=r['query']['
|
76
|
-
|
77
|
-
when '
|
78
|
-
r=self.API('action=query&list=
|
79
|
-
list=r['query']['
|
80
|
-
|
81
|
-
when '
|
82
|
-
r=self.API('action=query&list=
|
83
|
-
list=r['query']['
|
84
|
-
|
85
|
-
when '
|
86
|
-
r=self.API('action=query&
|
87
|
-
list=r['query']['
|
88
|
-
|
89
|
-
when '
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
1
|
+
# coding: utf-8
|
2
|
+
class Sunflower
|
3
|
+
# Makes a list of articles. Returns array of titles.
|
4
|
+
def make_list type, *parameters
|
5
|
+
type=type.downcase.gsub(/[^a-z]/, '')
|
6
|
+
first=parameters[0]
|
7
|
+
firstE=CGI.escape first.to_s
|
8
|
+
|
9
|
+
case type
|
10
|
+
when 'file'
|
11
|
+
f=File.open first
|
12
|
+
list=f.read.sub(/\357\273\277/,'').strip.split(/\r?\n/)
|
13
|
+
f.close
|
14
|
+
|
15
|
+
when 'page', 'pages'
|
16
|
+
list=parameters
|
17
|
+
|
18
|
+
when 'input'
|
19
|
+
puts 'Insert titles of articles to edit:'
|
20
|
+
puts 'Press [Enter] without inputting any text to finish.'
|
21
|
+
puts 'Press [Ctrl]+[C] to kill bot.'
|
22
|
+
list=[]
|
23
|
+
while true
|
24
|
+
input=gets.strip
|
25
|
+
break if input==''
|
26
|
+
|
27
|
+
list<<input
|
28
|
+
end
|
29
|
+
|
30
|
+
when 'categorieson'
|
31
|
+
r=self.API('action=query&prop=categories&cllimit=500&titles='+firstE)
|
32
|
+
list=r['query']['pages'].first['categories'].map{|v| v['title']}
|
33
|
+
|
34
|
+
when 'category'
|
35
|
+
r=self.API('action=query&list=categorymembers&cmprop=title&cmlimit=5000&cmtitle='+firstE)
|
36
|
+
list=r['query']['categorymembers'].map{|v| v['title']}
|
37
|
+
|
38
|
+
when 'categoryr', 'categoryrecursive'
|
39
|
+
list = [] # list of articles
|
40
|
+
processed = []
|
41
|
+
cats_to_process = [first] # list of categories to be processes
|
42
|
+
while !cats_to_process.empty?
|
43
|
+
now = cats_to_process.shift
|
44
|
+
processed << now # make sure we do not get stuck in infinite loop
|
45
|
+
|
46
|
+
list2 = self.make_list 'category', now # get contents of first cat in list
|
47
|
+
|
48
|
+
# find categories and queue them
|
49
|
+
cats_to_process += list2
|
50
|
+
.select{|el| el=~/\AKategoria:/}
|
51
|
+
.reject{|el| processed.include? el or cats_to_process.include? el}
|
52
|
+
|
53
|
+
list += list2 # add articles to main list
|
54
|
+
end
|
55
|
+
list.uniq!
|
56
|
+
|
57
|
+
when 'linkson'
|
58
|
+
r=self.API('action=query&prop=links&pllimit=5000&titles='+firstE)
|
59
|
+
list=r['query']['pages'].first['links'].map{|v| v['title']}
|
60
|
+
|
61
|
+
when 'transclusionson', 'templateson'
|
62
|
+
r=self.API('action=query&prop=templates&tllimit=5000&titles='+firstE)
|
63
|
+
list=r['query']['pages'].first['templates'].map{|v| v['title']}
|
64
|
+
|
65
|
+
when 'usercontribs', 'contribs'
|
66
|
+
r=self.API('action=query&list=usercontribs&uclimit=5000&ucprop=title&ucuser='+firstE)
|
67
|
+
list=r['query']['usercontribs'].map{|v| v['title']}
|
68
|
+
|
69
|
+
when 'whatlinksto', 'whatlinkshere'
|
70
|
+
r=self.API('action=query&list=backlinks&bllimit=5000&bltitle='+firstE)
|
71
|
+
list=r['query']['backlinks'].map{|v| v['title']}
|
72
|
+
|
73
|
+
when 'whattranscludes', 'whatembeds'
|
74
|
+
r=self.API('action=query&list=embeddedin&eilimit=5000&eititle='+firstE)
|
75
|
+
list=r['query']['embeddedin'].map{|v| v['title']}
|
76
|
+
|
77
|
+
when 'image', 'imageusage'
|
78
|
+
r=self.API('action=query&list=imageusage&iulimit=5000&iutitle='+firstE)
|
79
|
+
list=r['query']['imageusage'].map{|v| v['title']}
|
80
|
+
|
81
|
+
when 'search'
|
82
|
+
r=self.API('action=query&list=search&srwhat=text&srlimit=5000&srnamespace='+(parameters[1]=='allns' ? CGI.escape('0|1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|100|101|102|103') : '0')+'&srsearch='+firstE)
|
83
|
+
list=r['query']['search'].map{|v| v['title']}
|
84
|
+
|
85
|
+
when 'searchtitles'
|
86
|
+
r=self.API('action=query&list=search&srwhat=title&srlimit=5000&srnamespace='+(parameters[1]=='allns' ? CGI.escape('0|1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|100|101|102|103') : '0')+'&srsearch='+firstE)
|
87
|
+
list=r['query']['search'].map{|v| v['title']}
|
88
|
+
|
89
|
+
when 'random'
|
90
|
+
r=self.API('action=query&list=random&rnnamespace=0&rnlimit='+firstE)
|
91
|
+
list=r['query']['random'].map{|v| v['title']}
|
92
|
+
|
93
|
+
when 'external', 'linksearch'
|
94
|
+
r=self.API('action=query&euprop=title&list=exturlusage&eulimit=5000&euquery='+firstE)
|
95
|
+
list=r['query']['exturlusage'].map{|v| v['title']}
|
96
|
+
|
97
|
+
when 'google'
|
98
|
+
limit=[parameters[1].to_i,999].min
|
99
|
+
from=0
|
100
|
+
list=[]
|
101
|
+
|
102
|
+
while from<limit
|
103
|
+
p=HTTP.get(URI.parse("http://www.google.pl/custom?q=kot&start=#{from}&sitesearch=#{@wikiURL}"))
|
104
|
+
p.scan(/<div class=g><h2 class=r><a href="http:\/\/#{@wikiURL}\/wiki\/([^#<>\[\]\|\{\}]+?)" class=l>/){
|
105
|
+
list<<CGI.unescape($1).gsub('_',' ')
|
106
|
+
}
|
107
|
+
|
108
|
+
from+=10
|
109
|
+
end
|
110
|
+
|
111
|
+
when 'grep', 'regex', 'regexp'
|
112
|
+
split=@wikiURL.split('.')
|
113
|
+
ns=(parameters[1] ? parameters[1].to_s.gsub(/\D/,'') : '0')
|
114
|
+
redirs=(parameters[2] ? '&redirects=on' : '')
|
115
|
+
list=[]
|
116
|
+
|
117
|
+
p=HTTP.get(URI.parse("http://toolserver.org/~nikola/grep.php?pattern=#{firstE}&lang=#{split[0]}&wiki=#{split[1]}&ns=#{ns}#{redirs}"))
|
118
|
+
p.scan(/<tr><td><a href="http:\/\/#{@wikiURL}\/wiki\/([^#<>\[\]\|\{\}]+?)(?:\?redirect=no|)">/){
|
119
|
+
list<<CGI.unescape($1).gsub('_',' ')
|
120
|
+
}
|
121
|
+
end
|
122
|
+
|
123
|
+
return list
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
if $0==__FILE__
|
128
|
+
puts 'What kind of list do you want to create?'
|
129
|
+
if !(t=ARGV.shift)
|
130
|
+
t=gets
|
131
|
+
else
|
132
|
+
t=t.strip
|
133
|
+
puts t
|
134
|
+
end
|
135
|
+
puts ''
|
136
|
+
|
137
|
+
puts 'Supply arguments to pass to listmaker:'
|
138
|
+
puts '(press [Enter] without writing anything to finish)'
|
139
|
+
arg=[]
|
140
|
+
ARGV.each do |i|
|
141
|
+
arg<<i.strip
|
142
|
+
puts i.strip
|
143
|
+
end
|
144
|
+
while (a=gets.strip)!=''
|
145
|
+
arg<<a
|
146
|
+
end
|
147
|
+
|
148
|
+
puts 'Making list, wait patiently...'
|
149
|
+
|
150
|
+
s=Sunflower.new
|
151
|
+
s.login
|
152
|
+
|
153
|
+
l=s.make_list(t, *arg)
|
154
|
+
l.sort!
|
155
|
+
f=File.open('list.txt','w')
|
156
|
+
f.write(l.join("\n"))
|
157
|
+
f.close
|
158
|
+
|
159
|
+
puts 'Done! List saved to "list.txt".'
|
160
|
+
end
|
data/scripts/ZDBOT.rb
CHANGED
@@ -1,62 +1,62 @@
|
|
1
|
-
require 'sunflower-core.rb'
|
2
|
-
require 'sunflower-commontasks.rb'
|
3
|
-
s=Sunflower.new
|
4
|
-
s.login
|
5
|
-
|
6
|
-
$summary='archiwizacja zadań'
|
7
|
-
|
8
|
-
pp=Page.get('Wikipedia:Zadania dla botów')
|
9
|
-
tasks=pp.text
|
10
|
-
|
11
|
-
tasksDone=[]
|
12
|
-
tasksError=[]
|
13
|
-
tasksOld=[]
|
14
|
-
|
15
|
-
tasks=tasks.gsub(/\n==\s*(.+?)\s*==\s*\{\{\/Status\|([^}]+)\}\}([\s\S]+?)(?=\r?\n==|\s*\Z)/) do
|
16
|
-
title=$1.strip
|
17
|
-
status=$2.strip
|
18
|
-
text=$3.strip
|
19
|
-
|
20
|
-
bval=''
|
21
|
-
|
22
|
-
if (['wykonane','zrobione','błąd','błędne','stare'].index(status)==nil)
|
23
|
-
bval=$&
|
24
|
-
elsif (status=='wykonane' || status=='zrobione')
|
25
|
-
tasksDone<<"== "+title+" ==\n{{/Status|"+status+"}}\n"+text
|
26
|
-
elsif (status=='błąd' || status=='błędne')
|
27
|
-
tasksError<<"== "+title+" ==\n{{/Status|"+status+"}}\n"+text
|
28
|
-
elsif (status=='stare')
|
29
|
-
tasksOld<<"== "+title+" ==\n{{/Status|"+status+"}}\n"+text
|
30
|
-
end
|
31
|
-
|
32
|
-
bval
|
33
|
-
end
|
34
|
-
|
35
|
-
puts 'Data loaded. Saving...'
|
36
|
-
|
37
|
-
p=Page.get('Wikipedia:Zadania_dla_botów/Archiwum/błędne')
|
38
|
-
p.append tasksError.join("\n\n") unless tasksError.empty?
|
39
|
-
p.save unless tasksError.empty?
|
40
|
-
puts 'Error - saved.'
|
41
|
-
|
42
|
-
p=Page.get('Wikipedia:Zadania_dla_botów/Archiwum/wykonane')
|
43
|
-
p.append tasksDone.join("\n\n") unless tasksDone.empty?
|
44
|
-
p.save unless tasksDone.empty?
|
45
|
-
puts 'Done - saved.'
|
46
|
-
|
47
|
-
p=Page.get('Wikipedia:Zadania_dla_botów/Archiwum/stare')
|
48
|
-
p.append tasksOld.join("\n\n") unless tasksOld.empty?
|
49
|
-
p.save unless tasksOld.empty?
|
50
|
-
puts 'Old - saved.'
|
51
|
-
|
52
|
-
pp.text=tasks
|
53
|
-
pp.save
|
54
|
-
puts 'Main - saved.'
|
55
|
-
|
56
|
-
# File.open('ZDBOT_main.txt','w').write(tasks)
|
57
|
-
# File.open('ZDBOT_done.txt','w').write(tasksDone.join("\n\n")) unless tasksDone.empty?
|
58
|
-
# File.open('ZDBOT_error.txt','w').write(tasksError.join("\n\n")) unless tasksError.empty?
|
59
|
-
# File.open('ZDBOT_old.txt','w').write(tasksOld.join("\n\n")) unless tasksOld.empty?
|
60
|
-
|
61
|
-
puts "Stats: done: #{tasksDone.length}; error: #{tasksError.length}; old: #{tasksOld.length}"
|
1
|
+
require 'sunflower-core.rb'
|
2
|
+
require 'sunflower-commontasks.rb'
|
3
|
+
s=Sunflower.new
|
4
|
+
s.login
|
5
|
+
|
6
|
+
$summary='archiwizacja zadań'
|
7
|
+
|
8
|
+
pp=Page.get('Wikipedia:Zadania dla botów')
|
9
|
+
tasks=pp.text
|
10
|
+
|
11
|
+
tasksDone=[]
|
12
|
+
tasksError=[]
|
13
|
+
tasksOld=[]
|
14
|
+
|
15
|
+
tasks=tasks.gsub(/\n==\s*(.+?)\s*==\s*\{\{\/Status\|([^}]+)\}\}([\s\S]+?)(?=\r?\n==|\s*\Z)/) do
|
16
|
+
title=$1.strip
|
17
|
+
status=$2.strip
|
18
|
+
text=$3.strip
|
19
|
+
|
20
|
+
bval=''
|
21
|
+
|
22
|
+
if (['wykonane','zrobione','błąd','błędne','stare'].index(status)==nil)
|
23
|
+
bval=$&
|
24
|
+
elsif (status=='wykonane' || status=='zrobione')
|
25
|
+
tasksDone<<"== "+title+" ==\n{{/Status|"+status+"}}\n"+text
|
26
|
+
elsif (status=='błąd' || status=='błędne')
|
27
|
+
tasksError<<"== "+title+" ==\n{{/Status|"+status+"}}\n"+text
|
28
|
+
elsif (status=='stare')
|
29
|
+
tasksOld<<"== "+title+" ==\n{{/Status|"+status+"}}\n"+text
|
30
|
+
end
|
31
|
+
|
32
|
+
bval
|
33
|
+
end
|
34
|
+
|
35
|
+
puts 'Data loaded. Saving...'
|
36
|
+
|
37
|
+
p=Page.get('Wikipedia:Zadania_dla_botów/Archiwum/błędne')
|
38
|
+
p.append tasksError.join("\n\n") unless tasksError.empty?
|
39
|
+
p.save unless tasksError.empty?
|
40
|
+
puts 'Error - saved.'
|
41
|
+
|
42
|
+
p=Page.get('Wikipedia:Zadania_dla_botów/Archiwum/wykonane')
|
43
|
+
p.append tasksDone.join("\n\n") unless tasksDone.empty?
|
44
|
+
p.save unless tasksDone.empty?
|
45
|
+
puts 'Done - saved.'
|
46
|
+
|
47
|
+
p=Page.get('Wikipedia:Zadania_dla_botów/Archiwum/stare')
|
48
|
+
p.append tasksOld.join("\n\n") unless tasksOld.empty?
|
49
|
+
p.save unless tasksOld.empty?
|
50
|
+
puts 'Old - saved.'
|
51
|
+
|
52
|
+
pp.text=tasks
|
53
|
+
pp.save
|
54
|
+
puts 'Main - saved.'
|
55
|
+
|
56
|
+
# File.open('ZDBOT_main.txt','w').write(tasks)
|
57
|
+
# File.open('ZDBOT_done.txt','w').write(tasksDone.join("\n\n")) unless tasksDone.empty?
|
58
|
+
# File.open('ZDBOT_error.txt','w').write(tasksError.join("\n\n")) unless tasksError.empty?
|
59
|
+
# File.open('ZDBOT_old.txt','w').write(tasksOld.join("\n\n")) unless tasksOld.empty?
|
60
|
+
|
61
|
+
puts "Stats: done: #{tasksDone.length}; error: #{tasksError.length}; old: #{tasksOld.length}"
|
62
62
|
gets
|