rubyhexagon 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/bin/e621_search ADDED
@@ -0,0 +1,116 @@
1
+ #!/usr/bin/env ruby
2
+ =begin
3
+ Copyright 2014 Maxine Red <maxine_red1@yahoo.com>
4
+
5
+ This file is part of rubyhexagon.
6
+
7
+ rubyhexagon is free software: you can redistribute it and/or modify
8
+ it under the terms of the GNU General Public License as published by
9
+ the Free Software Foundation, either version 3 of the License, or
10
+ (at your option) any later version.
11
+
12
+ rubyhexagon is distributed in the hope that it will be useful,
13
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+ GNU General Public License for more details.
16
+
17
+ You should have received a copy of the GNU General Public License
18
+ along with rubyhexagon. If not, see <http://www.gnu.org/licenses/>.
19
+ =end
20
+
21
+ file = File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__
22
+ $:.unshift(File.expand_path(File.dirname(file)+"/../lib"))
23
+
24
+ require "rubyhexagon"
25
+ require "readline"
26
+ require "socket"
27
+ require "base64"
28
+
29
+ include E621
30
+
31
+ E621::Config.config = File.expand_path("~/.hexagon/conf.json")
32
+ tags = Array.new
33
+ File.open(File.expand_path(E621::Config.paths["tags"])) do |f|
34
+ tags = f.read.parse
35
+ end
36
+ Readline.completion_proc = proc do |s|
37
+ add = s[0] if s.match(/^\-|\~/)
38
+ add = s[0,2] if s.match(/^\*_/)
39
+ s.sub!(/^\-|\~|(\*_)/,"")
40
+ s = Regexp.escape(s)
41
+ words = Array.new
42
+ words += Dir["*"].reject{|d|!File.directory?(d)}.map{|d|"name:#{File.basename(d)}"}
43
+ words += ["rating:s","rating:q","rating:e"]
44
+ words += ["id:","md5:","description:","desc:","pool:","set:","width:",\
45
+ "height:","score:","favcount:","views:","ratio:","parent:"]
46
+ words += ["type:jpg","type:png","type:gif","type:swf"]
47
+ bools = ["hassource","hasdescription","ischild","idparent","inpool"]
48
+ words += bools.map{|b|["#{b}:true","#{b}:false"]}.flatten
49
+ # Make this a one dimensional array again!
50
+ words += ["order:random"]
51
+ orders = ["id","score","views","set","favcount","tagcount","comments",\
52
+ "mpixels","filesize","ratio","desclength"]
53
+ words += orders.map{|b|["#{b}_asc","#{b}_desc"]}.flatten.map{|b|"order:#{b}"}
54
+ # Here we make all orders in both directions and keep it as just an one
55
+ # dimensional array.
56
+ words += tags.map{|t|t["name"]}
57
+ words = words.grep(/^#{s}/)
58
+ words.map!{|w|"#{add}#{w}"}
59
+ s = "#{add}#{s}"
60
+ words
61
+ end
62
+ Readline.completer_word_break_characters = " "
63
+ Readline.completion_append_character = " "
64
+ history = File.open(File.expand_path(E621::Config.paths["history"]),"a+")
65
+ history.read.split($/).each do |l|
66
+ Readline::HISTORY << l
67
+ end
68
+
69
+ def net_send(query)
70
+ socket = TCPSocket.new(E621::Config.config["server"],5621)
71
+ socket.puts query.to_json
72
+ res = socket.read.parse
73
+ if res["error"] then
74
+ $stderr.puts "Server error: #{res["message"]}"
75
+ abort
76
+ end
77
+ socket.close
78
+ return res
79
+ end
80
+
81
+ api = API.new("user")
82
+ prompt = "#{api.user}@e621.net/posts> "
83
+ Dir.chdir(File.expand_path(E621::Config.paths["posts"])) do
84
+ while buff = Readline.readline(prompt.bold("yellow"), false) do
85
+ next if buff == String.new
86
+ if !(buff == String.new || buff == Readline::HISTORY.to_a.last) then
87
+ Readline::HISTORY << buff
88
+ history.puts buff
89
+ end
90
+ buff = buff.split(/\s+/)
91
+ name = buff.shift.sub("name:","")
92
+ query = {"action"=>"search","name"=>name,"query"=>buff}
93
+ rid = net_send(query)["rid"]
94
+ posts,page,num = Array.new,{"posts"=>[2]},1
95
+ while page["posts"] != Array.new do
96
+ print "Fetching page #{num.pad(3," ")} (#{posts.length.pad(6," ")} posts fetched).\r"
97
+ page = net_send({"action"=>"page","page"=>num,"rid"=>rid})
98
+ posts += page["posts"] if page["posts"]
99
+ num += 1
100
+ end
101
+ puts
102
+ Dir.mkdir(name) unless File.exist?(name)
103
+ Dir.chdir(name) do
104
+ count = 1
105
+ posts.reverse.each do |post|
106
+ print "Downloading post #{count.pad(6," ")} of #{posts.length.pad(6, " ")}.\r"
107
+ query = {"action"=>"show", "post"=>post}
108
+ res = net_send(query)
109
+ File.open(res["name"],"w"){|f|f.print Base64.decode64(res["data"])}
110
+ count += 1
111
+ end
112
+ end
113
+ puts
114
+ end
115
+ end
116
+ puts
data/bin/e621_server ADDED
@@ -0,0 +1,158 @@
1
+ #!/usr/bin/env ruby
2
+ =begin
3
+ Copyright 2014 Maxine Red <maxine_red1@yahoo.com>
4
+
5
+ This file is part of rubyhexagon.
6
+
7
+ rubyhexagon is free software: you can redistribute it and/or modify
8
+ it under the terms of the GNU General Public License as published by
9
+ the Free Software Foundation, either version 3 of the License, or
10
+ (at your option) any later version.
11
+
12
+ rubyhexagon is distributed in the hope that it will be useful,
13
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+ GNU General Public License for more details.
16
+
17
+ You should have received a copy of the GNU General Public License
18
+ along with rubyhexagon. If not, see <http://www.gnu.org/licenses/>.
19
+ =end
20
+
21
+ $:.unshift(File.expand_path(File.dirname(__FILE__)+"/../lib"))
22
+
23
+ require "rubyhexagon"
24
+ require "thread"
25
+ require "logger"
26
+ require "socket"
27
+ require "digest"
28
+ require "base64"
29
+
30
+ include E621
31
+
32
+ Thread.abort_on_exception = true
33
+
34
+ E621::Config.config = File.expand_path("~/.hexagon/conf.json")
35
+ api = API.new
36
+ tasks = Hash.new
37
+ mt = Mutex.new
38
+ server = TCPServer.new(5621)
39
+ Process.daemon
40
+ @log = Logger.new(File.expand_path(E621::Config.paths["logging"]))
41
+ #@log = Logger.new(STDOUT)
42
+ #@log.level = Logger::DEBUG
43
+ =begin
44
+ 10.times do
45
+ Thread.new do
46
+ loop do
47
+ name,tclient,max,post = queue.pop
48
+ next unless post
49
+ Dir.mkdir(name) unless File.exist?(name)
50
+ post.download("#{name}/#{"0"*(7-post.id.to_s.length)}#{post.id}.#{post.md5}.#{post.file_ext}")
51
+ mt.synchronize do
52
+ unless tasks.has_key?(name) then
53
+ tasks.store(name,1)
54
+ else
55
+ tasks[name] += 1
56
+ end
57
+ count = tasks[name]
58
+ message = "Got #{" "*(max.to_s.length+2-count.to_s.length)}#{count}/#{max}"+" "*16
59
+ @log.debug message
60
+ tclient.puts message
61
+ tclient.close if count >= max
62
+ end
63
+ end
64
+ end
65
+ end
66
+ =end
67
+
68
+ @log.formatter = proc do |sev,dat,prog,msg|
69
+ "#{Time.now.strftime("%b %e, %Y %I:%M:%S %p")} #{msg}#$/"
70
+ end
71
+ @log.level = Logger::INFO
72
+ @log.info("New server instance started.")
73
+ Dir.chdir(File.expand_path("~/.hexagon")) do
74
+ Thread.new do
75
+ loop do
76
+ files,size,tm = Dir["cache/**/*"].reject{|x|!File.file?(x)},0,Array.new
77
+ files.each do |f|
78
+ size += File.size(f)
79
+ end
80
+ if size > 30*2**30 then
81
+ files.each do |f|
82
+ tm << {"file"=>f,"time"=>File.atime(f),"size"=>File.size(f)}
83
+ end
84
+ tm = tm.sort{|f1,f2|f1["time"]<=>f2["time"]}
85
+ while size > 30*2**30 do
86
+ unlink = tm.shift
87
+ size -= unlink["size"]
88
+ File.unlink(unlink["file"])
89
+ end
90
+ end
91
+ sleep 3600
92
+ end
93
+ end
94
+ begin
95
+ Dir["searches/*"].each{|f|File.unlink(f)}
96
+ loop do
97
+ client = server.accept
98
+ Thread.new(client) do |c|
99
+ begin
100
+ query = c.gets.parse
101
+ rescue
102
+ @log.error("[#{c.remote_address.ip_address}]: Error #$!.")
103
+ next
104
+ end
105
+ case query["action"]
106
+ when "search" then
107
+ name,query = query["name"],query["query"]
108
+ @log.info("[#{c.remote_address.ip_address}]: Search init #{name}/#{query.join("+")}.")
109
+ Dir.mkdir("searches") unless File.exist?("searches")
110
+ rid = Digest::SHA2.hexdigest(name+Time.now.strftime("%s"))
111
+ File.open("searches/#{rid}.json","w") do |f|
112
+ search = {"name"=>name,"query"=>query}
113
+ f.print search.to_json
114
+ end
115
+ rid = {"rid"=>rid}.to_json
116
+ c.print rid
117
+ when "page" then
118
+ page, rid = query["page"],query["rid"]
119
+ File.open("searches/#{rid}.json"){|f|query=f.read.parse}
120
+ @log.info("[#{c.remote_address.ip_address}]: Search page #{query["name"]}/#{page}.")
121
+ search = Search.new(query["query"])
122
+ search.page = page
123
+ posts = {"posts"=>search.posts}.to_json
124
+ c.print posts
125
+ when "show" then
126
+ post = Post.new(query["post"])
127
+ Dir.mkdir("cache") unless File.exist?("cache")
128
+ Dir.mkdir("cache/#{post.md5[0,2]}") unless File.exist?("cache/#{post.md5[0,2]}")
129
+ body = String.new
130
+ if File.exist?("cache/#{post.md5[0,2]}/#{post.md5}") then
131
+ cached = " (cached)"
132
+ File.open("cache/#{post.md5[0,2]}/#{post.md5}") do |f|
133
+ body = f.read
134
+ end
135
+ else
136
+ cached = ""
137
+ body = post.download_data
138
+ File.open("cache/#{post.md5[0,2]}/#{post.md5}","w") do |f|
139
+ f.print body
140
+ end
141
+ end
142
+ @log.info("[#{c.remote_address.ip_address}]: Show post #{query["post"]["id"].pad(6," ")}#{cached}.")
143
+ file = {"name"=>"#{post.id.pad(8)}.#{post.md5}.#{post.file_ext}","data"=>Base64.encode64(body)}
144
+ c.print file.to_json
145
+ end
146
+ c.close
147
+ end
148
+ end
149
+ rescue Errno
150
+ @log.error("Error: #$!!")
151
+ rescue => e
152
+ p e.class
153
+ @log.fatal("Fatal Error: #$!!")
154
+ raise
155
+ ensure
156
+ @log.info("Server instance terminated.")
157
+ end
158
+ end
data/bin/fav-sync ADDED
@@ -0,0 +1,82 @@
1
+ #!/usr/bin/env ruby
2
+ =begin
3
+ Copyright 2014 Maxine Red <maxine_red1@yahoo.com>
4
+
5
+ This file is part of rubyhexagon.
6
+
7
+ rubyhexagon is free software: you can redistribute it and/or modify
8
+ it under the terms of the GNU General Public License as published by
9
+ the Free Software Foundation, either version 3 of the License, or
10
+ (at your option) any later version.
11
+
12
+ rubyhexagon is distributed in the hope that it will be useful,
13
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+ GNU General Public License for more details.
16
+
17
+ You should have received a copy of the GNU General Public License
18
+ along with rubyhexagon. If not, see <http://www.gnu.org/licenses/>.
19
+ =end
20
+
21
+ loader = File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__
22
+ $:.unshift(File.expand_path(File.dirname(loader))+"/../lib")
23
+
24
+ require "rubyhexagon"
25
+
26
+ include E621
27
+
28
+ max_wait = 5
29
+ E621::Config.config = File.expand_path("~/.hexagon/conf.json")
30
+ api = API.new("user")
31
+ uid = api.get("index",{"name"=>api.user}).first["id"]
32
+ Dir.chdir(File.expand_path("~/pictures/e621/favorites"))
33
+ remote = Array.new
34
+ Search.new("fav:maxine_red order:id".split(" ")).each_post do |post|
35
+ string = ["0"*(7-post.id.to_s.length)+post.id.to_s,post.md5,post.file_ext].join(".")
36
+ remote << Array.new
37
+ if !File.exist?(string) then
38
+ post.download(string)
39
+ else
40
+ md5 = String.new
41
+ File.open(string){|f|md5 = Digest::MD5.hexdigest(f.read)}
42
+ next if md5 == post.md5
43
+ post.download(string)
44
+ end
45
+ puts "Downloaded post #{" "*(7-post.id.to_s.length)}#{post.id}."
46
+ sleep(rand*max_wait)
47
+ end
48
+ local = Dir["*"].reject{|x|x.match(/db$/)}.map{|x|x.sub(/\..+/,"").to_i}.sort.uniq
49
+ (remote-local).each do |id|
50
+ post = Post.new({"id"=>id})
51
+ f = post.unfavorite
52
+ if f["success"] then
53
+ print "\e[1;33mRemoved favorite on post #{" "*(7-post.id.to_s.length)}#{post.id}.\e[0m "
54
+ else
55
+ print "\e[1;31m#{f["reason"].to_s.gsub(/<.+?>/,"")}.\e[0m "
56
+ end
57
+ f = post.vote(1)
58
+ if f["success"] then
59
+ puts "\e[1;33mScored post #{" "*(7-post.id.to_s.length)}#{post.id} down.\e[0m"
60
+ else
61
+ puts "\e[1;31m#{f["reason"].to_s.gsub(/<.+?>/,"")}.\e[0m"
62
+ end
63
+ sleep(rand*max_wait)
64
+ end
65
+ (local-remote).each do |id|
66
+ next if id == 0
67
+ post = Post.new({"id"=>id})
68
+ f = post.favorite
69
+ if f["success"] then
70
+ print "\e[1;32mFavorited #{" "*(7-post.id.to_s.length)}#{post.id}.\e[0m "
71
+ else
72
+ print "\e[1;31m#{f["reason"].gsub(/<.+?>/,"")}.\e[0m "
73
+ #File.unlink(Dir["#{set["shortname"]}/*.#{"0"*(8-post.id.to_s.length)}#{post.id}.*"].first)
74
+ end
75
+ f = post.vote(1)
76
+ if f["success"] then
77
+ puts "\e[1;32mScored post #{" "*(7-post.id.to_s.length)}#{post.id} up.\e[0m"
78
+ else
79
+ puts "\e[1;31m#{f["reason"].to_s.gsub(/<.+?>/,"")}.\e[0m"
80
+ end
81
+ sleep(rand*max_wait)
82
+ end
data/bin/set-sync ADDED
@@ -0,0 +1,225 @@
1
+ #!/usr/bin/env ruby
2
+ =begin
3
+ Copyright 2014 Maxine Red <maxine_red1@yahoo.com>
4
+
5
+ This file is part of rubyhexagon.
6
+
7
+ rubyhexagon is free software: you can redistribute it and/or modify
8
+ it under the terms of the GNU General Public License as published by
9
+ the Free Software Foundation, either version 3 of the License, or
10
+ (at your option) any later version.
11
+
12
+ rubyhexagon is distributed in the hope that it will be useful,
13
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+ GNU General Public License for more details.
16
+
17
+ You should have received a copy of the GNU General Public License
18
+ along with rubyhexagon. If not, see <http://www.gnu.org/licenses/>.
19
+ =end
20
+
21
+ loader = File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__
22
+ $:.unshift(File.expand_path(File.dirname(loader))+"/../lib")
23
+
24
+ require "rubyhexagon"
25
+ require "logger"
26
+
27
+ include E621
28
+
29
+
30
+ def download(post,name,set)
31
+ artist = post.tags.split(/\s+/).map do |t|
32
+ @tags[@tags.index(t)] if @tags.index(t)
33
+ end
34
+ begin
35
+ artist = artist.compact.first.sub("_(artist)","")
36
+ rescue
37
+ artist = "unknown"
38
+ end
39
+ string = [post.created_at.to_i,post.id.pad(8),artist+"_"+name,post.file_ext]
40
+ string = string.join(".")
41
+ begin
42
+ api = API.new("post")
43
+ post = Post.new(api.post("show",{"id"=>post.id}))
44
+ post.download("#{name}/#{string}")
45
+ rescue
46
+ puts $!
47
+ sleep 1
48
+ retry
49
+ end
50
+ File.utime(post.created_at,post.created_at,"#{name}/#{string}")
51
+ @posts["downloads"] << post.id
52
+ jposts = @posts.to_json
53
+ set_files = File.expand_path("~/.hexagon/sets")
54
+ File.open(set_files+"/#{name}.json","w"){|f|f.print jposts}
55
+ s = "Downloaded post #{" "*(6-post.id.to_s.length)}#{post.id} from \"#{set["name"]}\"."
56
+ puts s
57
+ @log.info(s)
58
+ end
59
+ max_wait = 2.5
60
+ E621::Config.config = File.expand_path("~/.hexagon/conf.json")
61
+ api = API.new("user")
62
+ @log = Logger.new(File.expand_path(E621::Config.paths["logging"]))
63
+ @log.formatter = proc do |sev,dat,prog,msg|
64
+ "#{Time.now.strftime("%b %e, %Y %I:%M:%S %p")}: #{msg}#$/"
65
+ end
66
+ @log.level = Logger::INFO
67
+ @log.info("Program #$0 started.")
68
+ uid = api.get("index",{"name"=>api.user}).first["id"]
69
+ tags = File.expand_path("~/.hexagon/tags.json")
70
+ if !File.exist?(tags) then
71
+ @tags = Array.new
72
+ else
73
+ File.open(tags) do |f|
74
+ @tags = f.read.parse
75
+ end
76
+ end
77
+ @tags = @tags.sort{|k1,k2|k1["id"]<=>k2["id"]}
78
+ net = Net::HTTP.new("e621.net",443)
79
+ net.use_ssl = true
80
+ body,page = [2],1
81
+ until body == Array.new do
82
+ body = net.get("/tag/index.json?limit=0&order=count&after_id=#{@tags.last["id"]}&page=#{page}").body.parse
83
+ body = body.map{|x|x["name"] = x["name"].encode("us-ascii", :invalid => :replace, :undef => :replace, :replace => "");x}
84
+ @tags += body
85
+ page += 1
86
+ end
87
+ jtags = @tags.to_json
88
+ File.open(tags,"w"){|f|f.print jtags}
89
+ @tags = @tags.reject{|x|x["type"]!=1}.map{|x|x["name"]}
90
+ Dir.chdir(File.expand_path("~/Dropbox/Furry/e621/sets")) do
91
+ sets = Array.new
92
+ File.open(File.expand_path("~/.hexagon/sets.json")) do |f|
93
+ sets = f.read.parse
94
+ end
95
+ sets = sets.sort{|s1,s2|s1["id"]<=>s2["id"]}
96
+ set_files = File.expand_path("~/.hexagon/sets")
97
+ api = API.new("set")
98
+ sets.each do |set|
99
+ sid, owner, query = set["id"], set["owner"], set["search"]
100
+ set = api.get("show", {"id"=>sid})
101
+ posts = set["posts"].map{|post|Post.new(post)}
102
+ name = set["shortname"]
103
+ @log.info("Fetching set #{set["name"]}")
104
+ if !File.exists?(set_files+"/#{name}.json") then
105
+ File.open(set_files+"/#{name}.json","w"){|f|f.print "{\"downloads\":[]}"}
106
+ end
107
+ File.open(set_files+"/#{name}.json"){|f|@posts = f.read.parse}
108
+ Dir.mkdir(name) unless File.exist?(name)
109
+ posts.each do |post|
110
+ next if @posts["downloads"].include?(post.id)
111
+ download(post,name,set)
112
+ end
113
+ next if owner != uid
114
+ Search.new("fav:maxine_red #{query} order:id".split(" ")).each_post do |post|
115
+ next if @posts["downloads"].include?(post.id)
116
+ download(post,name,set)
117
+ sleep(rand*max_wait)
118
+ end
119
+ end
120
+ sets.each do |set|
121
+ sid, owner, query = set["id"], set["owner"], set["search"]
122
+ next if owner != uid
123
+ set = api.get("show", {"id"=>sid})
124
+ posts = set["posts"].map{|post|Post.new(post).id}
125
+ name = set["shortname"]
126
+ local = Dir["#{name}/*"].reject{|x|x.match(/db$/)}.map{|x|x.split(".")[1].to_i}.sort.uniq
127
+ (posts-local).each do |id|
128
+ post = Post.new({"id"=>id})
129
+ f = post.remove_from_set(sid)
130
+ if f["success"] then
131
+ s = "Removed Post #{" "*(6-post.id.to_s.length)}#{post.id} from \"#{set["name"]}\""
132
+ puts "\e[1;33m#{s}\e[0m"
133
+ @log.info(s)
134
+ else
135
+ s = "#{f["reason"].to_s.gsub(/<.+?>/,"")}."
136
+ puts "\e[1;31m#{s}\e[0m"
137
+ @log.info(s)
138
+ end
139
+ sleep(rand*max_wait)
140
+ end
141
+ (local-posts).each do |id|
142
+ next if id == 0
143
+ post = Post.new({"id"=>id})
144
+ f = post.add_to_set(sid)
145
+ if f["success"] then
146
+ s = "Added Post #{" "*(6-post.id.to_s.length)}#{post.id} to \"#{set["name"]}\"."
147
+ puts s
148
+ @log.info(s)
149
+ else
150
+ s = "#{f["reason"].gsub(/<.+?>/,"")}. Removing local file."
151
+ puts s
152
+ @log.info(s)
153
+ File.unlink(Dir["#{set["shortname"]}/*.#{"0"*(8-post.id.to_s.length)}#{post.id}.*"].first)
154
+ end
155
+ sleep(rand*max_wait)
156
+ end
157
+ end
158
+ end
159
+ Dir.chdir(File.expand_path("~/Dropbox/Furry/e621/pools")) do
160
+ pools = Array.new
161
+ File.open(File.expand_path("~/.hexagon/pools.json")) do |f|
162
+ pools = f.read.parse.sort.uniq
163
+ end
164
+ pool_files = File.expand_path("~/.hexagon/pools")
165
+ api = API.new("pool")
166
+ pools.each do |pool|
167
+ spool = {"updated_at"=>0, "posts"=>[]}
168
+ npool = api.get("show",{"id"=>pool})
169
+ pool = Pool.new(npool)
170
+ begin
171
+ pfile = pool_files+"/#{pool.id.pad(5)}.json"
172
+ rescue
173
+ @log.error("Error occured: #{npool.inspect}")
174
+ raise
175
+ end
176
+ if !File.exists?(pfile) then
177
+ File.open(pfile,"w"){|f|f.print spool.to_json}
178
+ spool = Pool.new(spool)
179
+ else
180
+ File.open(pfile){|f|spool = Pool.new(f.read.parse)}
181
+ end
182
+ posts = Array.new
183
+ next if pool.updated_at.to_i <= spool.updated_at.to_i
184
+ (pool.post_count/24.0).ceil.times do |page|
185
+ posts += pool.posts.map{|post|Post.new(post)}
186
+ pool = Pool.new(api.get("show",{"id"=>pool.id,"page"=>page+2}))
187
+ end
188
+ pool.name = pool.name.gsub("_"," ").encode("us-ascii", :invalid => :replace, :undef => :replace, :replace => "")
189
+ name = pool.name.gsub(/[^0-9, ,_,a-z,\-]/i,"").sub(/\s+$/,"")
190
+ Dir.mkdir(name) unless File.exist?(name)
191
+ @log.info("Fetching pool #{pool.name}")
192
+ posts.each_with_index do |post,id|
193
+ id = id.succ
194
+ next if spool.posts.include?(post.id)
195
+ artist = post.tags.split(/\s+/).map do |t|
196
+ @tags[@tags.index(t)] if @tags.index(t)
197
+ end
198
+ begin
199
+ artist = artist.compact.first.sub("_(artist)","")
200
+ rescue
201
+ artist = "unknown"
202
+ end
203
+ string = [pool.id.pad(5),id.pad(4),post.id.pad(8),artist,name.downcase.gsub(/\s/,"_"),post.file_ext]
204
+ string = string.join(".")
205
+ begin
206
+ post.download("#{name}/#{string}")
207
+ rescue
208
+ puts $!
209
+ sleep 1
210
+ retry
211
+ end
212
+ File.utime(post.created_at,post.created_at,"#{name}/#{string}")
213
+ spool.posts << post.id
214
+ jposts = spool.to_json
215
+ File.open(pfile,"w"){|f|f.print jposts}
216
+ s = "Downloaded post #{post.id.pad(6," ")} (#{id.pad(posts.length.to_s.length," ")}/#{posts.length}) from \"#{pool.name}\"."
217
+ puts s
218
+ @log.info(s)
219
+ sleep(rand*max_wait)
220
+ end
221
+ spool.updated_at = pool.updated_at
222
+ jposts = spool.to_json
223
+ File.open(pfile,"w"){|f|f.print jposts}
224
+ end
225
+ end