sunflower 0.3 → 0.4
Sign up to get free protection for your applications and to get access to all the features.
- data/README +1 -1
- data/lib/sunflower/commontasks.rb +265 -250
- data/lib/sunflower/core.rb +288 -287
- data/lib/sunflower/listmaker.rb +160 -152
- data/scripts/ZDBOT.rb +61 -61
- data/scripts/aktualizacjapilkarzy.rb +339 -339
- data/scripts/changeimage.rb +41 -41
- data/scripts/fix-bold-in-headers.rb +41 -53
- data/scripts/fix-double-pipes.rb +30 -49
- data/scripts/fix-langs.rb +42 -42
- data/scripts/fix-multiple-same-refs.rb +101 -101
- data/scripts/fix-some-entities.rb +36 -43
- data/scripts/fix-unicode-control-chars.rb +30 -51
- data/scripts/insight.rb +132 -132
- data/scripts/lekkoatl-portal.rb +50 -50
- data/scripts/make-id2team-list.rb +31 -31
- data/scripts/recat.rb +27 -32
- data/scripts/wanted.rb +72 -72
- metadata +40 -62
data/lib/sunflower/core.rb
CHANGED
@@ -1,287 +1,288 @@
|
|
1
|
-
# coding: utf-8
|
2
|
-
require 'rest-client'
|
3
|
-
require 'json'
|
4
|
-
require 'cgi'
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
#
|
9
|
-
# s.
|
10
|
-
#
|
11
|
-
#
|
12
|
-
#
|
13
|
-
#
|
14
|
-
#
|
15
|
-
#
|
16
|
-
#
|
17
|
-
#
|
18
|
-
#
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
r
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
@
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
"#{prefix}
|
116
|
-
"#{prefix}
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
r
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
r
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
#
|
154
|
-
#
|
155
|
-
#
|
156
|
-
#
|
157
|
-
#
|
158
|
-
#
|
159
|
-
#
|
160
|
-
#
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
@
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
count
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
@
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
r =
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
r =
|
224
|
-
r.
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
raise SunflowerError, '
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
end
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
end
|
287
|
-
|
1
|
+
# coding: utf-8
|
2
|
+
require 'rest-client'
|
3
|
+
require 'json'
|
4
|
+
require 'cgi'
|
5
|
+
|
6
|
+
class SunflowerError < StandardError; end
|
7
|
+
|
8
|
+
# Main class. To start working, you have to create new Sunflower:
|
9
|
+
# s = Sunflower.new('en.wikipedia.org')
|
10
|
+
# And then log in:
|
11
|
+
# s.login('Username','password')
|
12
|
+
#
|
13
|
+
# If you have ran setup, you can just use
|
14
|
+
# s = Sunflower.new.login
|
15
|
+
#
|
16
|
+
# Then you can request data from API using #API method.
|
17
|
+
#
|
18
|
+
# To log data to file, use #log method (works like puts). Use RestClient.log=<io> to log all requests.
|
19
|
+
#
|
20
|
+
# You can use multiple Sunflowers at once, to work on multiple wikis.
|
21
|
+
class Sunflower
|
22
|
+
VERSION = '0.4'
|
23
|
+
|
24
|
+
INVALID_CHARS = %w(# < > [ ] | { })
|
25
|
+
INVALID_CHARS_REGEX = Regexp.union *INVALID_CHARS
|
26
|
+
|
27
|
+
# Path to user data file.
|
28
|
+
def self.path
|
29
|
+
File.join(ENV['HOME'], 'sunflower-userdata')
|
30
|
+
end
|
31
|
+
|
32
|
+
# Options for this Sunflower.
|
33
|
+
attr_accessor :summary, :always_do_code_cleanup
|
34
|
+
|
35
|
+
attr_accessor :cookie, :headers, :wikiURL, :warnings, :log
|
36
|
+
|
37
|
+
# Initialize a new Sunflower working on a wiki with given URL, for ex. "pl.wikipedia.org".
|
38
|
+
def initialize url=nil
|
39
|
+
begin
|
40
|
+
r=File.read(Sunflower.path)
|
41
|
+
@userdata=r.split(/\r?\n/).map{|i| i.strip}
|
42
|
+
rescue
|
43
|
+
@userdata=[]
|
44
|
+
end
|
45
|
+
|
46
|
+
if !url
|
47
|
+
if !@userdata.empty?
|
48
|
+
url=@userdata[0]
|
49
|
+
else
|
50
|
+
raise SunflowerError, 'initialize: no URL supplied and no userdata found!'
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
@warnings=true
|
55
|
+
@log=false
|
56
|
+
|
57
|
+
@wikiURL=url
|
58
|
+
|
59
|
+
@loggedin=false
|
60
|
+
end
|
61
|
+
|
62
|
+
# Call the API. Returns a hash of JSON response. Request can be a HTTP request string or a hash.
|
63
|
+
def API request
|
64
|
+
if request.is_a? String
|
65
|
+
request += '&format=json'
|
66
|
+
elsif request.is_a? Hash
|
67
|
+
request.merge!({format:'json'})
|
68
|
+
end
|
69
|
+
|
70
|
+
resp = RestClient.post(
|
71
|
+
'http://'+@wikiURL+'/w/api.php',
|
72
|
+
request,
|
73
|
+
{:user_agent => "Sunflower #{VERSION} alpha", :cookies => @cookies}
|
74
|
+
)
|
75
|
+
JSON.parse resp.to_str
|
76
|
+
end
|
77
|
+
|
78
|
+
# Log in using given info.
|
79
|
+
def login user='', password=''
|
80
|
+
if user=='' || password==''
|
81
|
+
if !@userdata.empty?
|
82
|
+
user=@userdata[1] if user==''
|
83
|
+
password=@userdata[2] if password==''
|
84
|
+
else
|
85
|
+
raise SunflowerError, 'login: no user/pass supplied and no userdata found!'
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
raise SunflowerError, 'bad username!' if user =~ INVALID_CHARS_REGEX
|
90
|
+
|
91
|
+
|
92
|
+
# 1. get the login token
|
93
|
+
response = RestClient.post(
|
94
|
+
'http://'+@wikiURL+'/w/api.php?'+"action=login&lgname=#{CGI.escape user}&lgpassword=#{CGI.escape password}"+'&format=json',
|
95
|
+
nil,
|
96
|
+
{:user_agent => 'Sunflower alpha'}
|
97
|
+
)
|
98
|
+
|
99
|
+
@cookies = response.cookies
|
100
|
+
json = JSON.parse response.to_str
|
101
|
+
token, prefix = json['login']['token'], json['login']['cookieprefix']
|
102
|
+
|
103
|
+
|
104
|
+
# 2. actually log in
|
105
|
+
response = RestClient.post(
|
106
|
+
'http://'+@wikiURL+'/w/api.php?'+"action=login&lgname=#{CGI.escape user}&lgpassword=#{CGI.escape password}&lgtoken=#{token}"+'&format=json',
|
107
|
+
nil,
|
108
|
+
{:user_agent => 'Sunflower alpha', :cookies => @cookies}
|
109
|
+
)
|
110
|
+
|
111
|
+
json = JSON.parse response.to_str
|
112
|
+
|
113
|
+
@cookies = @cookies.merge(response.cookies).merge({
|
114
|
+
"#{prefix}UserName" => json['login']['lgusername'].to_s,
|
115
|
+
"#{prefix}UserID" => json['login']['lguserid'].to_s,
|
116
|
+
"#{prefix}Token" => json['login']['lgtoken'].to_s
|
117
|
+
})
|
118
|
+
|
119
|
+
|
120
|
+
raise SunflowerError, 'unable to log in (no cookies received)!' if !@cookies
|
121
|
+
|
122
|
+
|
123
|
+
# 3. confirm you did log in by checking the watchlist.
|
124
|
+
@loggedin=true
|
125
|
+
r=self.API('action=query&list=watchlistraw')
|
126
|
+
if r['error'] && r['error']['code']=='wrnotloggedin'
|
127
|
+
@loggedin=false
|
128
|
+
raise SunflowerError, 'unable to log in!'
|
129
|
+
end
|
130
|
+
|
131
|
+
# 4. check bot rights
|
132
|
+
r=self.API('action=query&list=allusers&aulimit=1&augroup=bot&aufrom='+(CGI.escape user))
|
133
|
+
unless r['query']['allusers'][0]['name']==user
|
134
|
+
warn 'Sunflower - this user does not have bot rights!' if @warnings
|
135
|
+
@is_bot=false
|
136
|
+
else
|
137
|
+
@is_bot=true
|
138
|
+
end
|
139
|
+
|
140
|
+
return self
|
141
|
+
end
|
142
|
+
|
143
|
+
def log t
|
144
|
+
File.open('log.txt','a'){|f| f.puts t}
|
145
|
+
end
|
146
|
+
|
147
|
+
def is_bot?
|
148
|
+
@is_bot
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
# Class representng single Wiki page. To load specified page, use #new/#get/#load method.
|
153
|
+
#
|
154
|
+
# When calling Page.new, at first only the text will be loaded - attributes and edit token will be loaded when needed, or when you call #preload_attrs.
|
155
|
+
#
|
156
|
+
# If you are using multiple Sunflowers, you have to specify which wiki this page belongs to using second argument of function; you can pass whole URL (same as when creating new Sunflower) or just language code.
|
157
|
+
#
|
158
|
+
# To save page, use #save/#put method. Optional argument is new title page, if ommited, page is saved at old title. Summary can be passed as second parameter. If it's ommited, s.summary is used. If it's empty too, error is raised.
|
159
|
+
#
|
160
|
+
# To get Sunflower instance which this page belongs to, use #sunflower of #belongs_to.
|
161
|
+
class Page
|
162
|
+
INVALID_CHARS = %w(# < > [ ] | { })
|
163
|
+
INVALID_CHARS_REGEX = Regexp.union *INVALID_CHARS
|
164
|
+
|
165
|
+
attr_accessor :text
|
166
|
+
attr_reader :orig_text
|
167
|
+
|
168
|
+
attr_reader :sunflower
|
169
|
+
alias :belongs_to :sunflower
|
170
|
+
|
171
|
+
attr_reader :pageid, :ns, :title, :touched, :lastrevid, :counter, :length, :starttimestamp, :edittoken, :protection #prop=info
|
172
|
+
|
173
|
+
# calling any of these accessors will fetch the data.
|
174
|
+
[:pageid, :ns, :title, :touched, :lastrevid, :counter, :length, :starttimestamp, :edittoken, :protection].each do |meth|
|
175
|
+
define_method meth do
|
176
|
+
preload_attrs unless @preloaded_attrs
|
177
|
+
instance_variable_get "@#{meth}"
|
178
|
+
end
|
179
|
+
end
|
180
|
+
|
181
|
+
def initialize title='', wiki=''
|
182
|
+
raise SunflowerError, 'title invalid: '+title if title =~ INVALID_CHARS_REGEX
|
183
|
+
|
184
|
+
@modulesExecd=[] #used by sunflower-commontasks.rb
|
185
|
+
@summaryAppend=[] #used by sunflower-commontasks.rb
|
186
|
+
|
187
|
+
@title=title
|
188
|
+
wiki=wiki+'.wikipedia.org' if wiki.index('.')==nil && wiki!=''
|
189
|
+
|
190
|
+
if wiki==''
|
191
|
+
count=ObjectSpace.each_object(Sunflower){|o| @sunflower=o}
|
192
|
+
raise SunflowerError, 'you must pass wiki name if using multiple Sunflowers at once!' if count>1
|
193
|
+
else
|
194
|
+
ObjectSpace.each_object(Sunflower){|o| @sunflower=o if o.wikiURL==wiki}
|
195
|
+
end
|
196
|
+
|
197
|
+
if title==''
|
198
|
+
@text=''
|
199
|
+
@orig_text=''
|
200
|
+
return
|
201
|
+
end
|
202
|
+
|
203
|
+
preload_text
|
204
|
+
end
|
205
|
+
|
206
|
+
def preload_text
|
207
|
+
r = @sunflower.API('action=query&prop=revisions&rvprop=content&titles='+CGI.escape(@title))
|
208
|
+
r = r['query']['pages'].first
|
209
|
+
if r['missing']
|
210
|
+
@text = ''
|
211
|
+
elsif r['invalid']
|
212
|
+
raise SunflowerError, 'title invalid: '+@title
|
213
|
+
else
|
214
|
+
@text = r['revisions'][0]['*']
|
215
|
+
end
|
216
|
+
|
217
|
+
@orig_text = @text
|
218
|
+
|
219
|
+
@preloaded_text = true
|
220
|
+
end
|
221
|
+
|
222
|
+
def preload_attrs
|
223
|
+
r = @sunflower.API('action=query&prop=info&inprop=protection&intoken=edit&titles='+CGI.escape(@title))
|
224
|
+
r = r['query']['pages'].first
|
225
|
+
r.each{|key, value|
|
226
|
+
self.instance_variable_set('@'+key, value)
|
227
|
+
}
|
228
|
+
|
229
|
+
@preloaded_attrs = true
|
230
|
+
end
|
231
|
+
|
232
|
+
def dump_to file
|
233
|
+
if file.respond_to? :write #probably file or IO
|
234
|
+
file.write @text
|
235
|
+
else #filename?
|
236
|
+
File.open(file.to_s, 'w'){|f| f.write @text}
|
237
|
+
end
|
238
|
+
end
|
239
|
+
|
240
|
+
def dump
|
241
|
+
self.dump_to @title.gsub(/[^a-zA-Z0-9\-]/,'_')+'.txt'
|
242
|
+
end
|
243
|
+
|
244
|
+
def save title=@title, summary=nil
|
245
|
+
preload_attrs unless @preloaded_attrs
|
246
|
+
|
247
|
+
summary = @sunflower.summary if !summary
|
248
|
+
|
249
|
+
raise SunflowerError, 'title invalid: '+title if title =~ INVALID_CHARS_REGEX
|
250
|
+
raise SunflowerError, 'no summary!' if (!summary or summary=='') && @summaryAppend.empty?
|
251
|
+
|
252
|
+
unless @summaryAppend.empty?
|
253
|
+
if !summary or summary==''
|
254
|
+
summary = @summaryAppend.uniq.join(', ')
|
255
|
+
else
|
256
|
+
summary = summary.sub(/,\s*\Z/, '') + ', ' + @summaryAppend.uniq.join(', ')
|
257
|
+
end
|
258
|
+
end
|
259
|
+
|
260
|
+
if @orig_text==@text && title==@title
|
261
|
+
@sunflower.log('Page '+title+' not saved - no changes.')
|
262
|
+
return
|
263
|
+
end
|
264
|
+
|
265
|
+
|
266
|
+
|
267
|
+
self.code_cleanup if @sunflower.always_do_code_cleanup && self.respond_to?('code_cleanup')
|
268
|
+
|
269
|
+
r=@sunflower.API("action=edit&bot=1&title=#{CGI.escape(title)}&text=#{CGI.escape(@text)}&summary=#{CGI.escape(summary)}&token=#{CGI.escape(@edittoken)}")# if @sunflower.isBot?
|
270
|
+
end
|
271
|
+
alias :put :save
|
272
|
+
|
273
|
+
def self.get title, wiki=''
|
274
|
+
Page.new(title, wiki)
|
275
|
+
end
|
276
|
+
|
277
|
+
def self.load title, wiki=''
|
278
|
+
Page.new(title, wiki)
|
279
|
+
end
|
280
|
+
end
|
281
|
+
|
282
|
+
class Hash
|
283
|
+
# just a lil patch
|
284
|
+
def first
|
285
|
+
self.values[0]
|
286
|
+
end
|
287
|
+
end
|
288
|
+
|