rubyhexagon 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bin/e621_search +116 -0
- data/bin/e621_server +158 -0
- data/bin/fav-sync +82 -0
- data/bin/set-sync +225 -0
- data/lib/api.rb +123 -0
- data/lib/config.rb +46 -0
- data/lib/container.rb +43 -0
- data/lib/pool.rb +50 -0
- data/lib/post.rb +106 -0
- data/lib/rubyhexagon.rb +34 -0
- data/lib/search.rb +73 -0
- data/lib/set.rb +43 -0
- data/lib/standard/error.rb +24 -0
- data/lib/standard/hash.rb +29 -0
- data/lib/standard/http.rb +83 -0
- data/lib/standard/int.rb +36 -0
- data/lib/standard/string.rb +84 -0
- data/lib/standard/time.rb +30 -0
- metadata +67 -0
data/bin/e621_search
ADDED
@@ -0,0 +1,116 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
=begin
|
3
|
+
Copyright 2014 Maxine Red <maxine_red1@yahoo.com>
|
4
|
+
|
5
|
+
This file is part of rubyhexagon.
|
6
|
+
|
7
|
+
rubyhexagon is free software: you can redistribute it and/or modify
|
8
|
+
it under the terms of the GNU General Public License as published by
|
9
|
+
the Free Software Foundation, either version 3 of the License, or
|
10
|
+
(at your option) any later version.
|
11
|
+
|
12
|
+
rubyhexagon is distributed in the hope that it will be useful,
|
13
|
+
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
14
|
+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
15
|
+
GNU General Public License for more details.
|
16
|
+
|
17
|
+
You should have received a copy of the GNU General Public License
|
18
|
+
along with rubyhexagon. If not, see <http://www.gnu.org/licenses/>.
|
19
|
+
=end
|
20
|
+
|
21
|
+
file = File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__
|
22
|
+
$:.unshift(File.expand_path(File.dirname(file)+"/../lib"))
|
23
|
+
|
24
|
+
require "rubyhexagon"
|
25
|
+
require "readline"
|
26
|
+
require "socket"
|
27
|
+
require "base64"
|
28
|
+
|
29
|
+
include E621
|
30
|
+
|
31
|
+
E621::Config.config = File.expand_path("~/.hexagon/conf.json")
|
32
|
+
tags = Array.new
|
33
|
+
File.open(File.expand_path(E621::Config.paths["tags"])) do |f|
|
34
|
+
tags = f.read.parse
|
35
|
+
end
|
36
|
+
Readline.completion_proc = proc do |s|
|
37
|
+
add = s[0] if s.match(/^\-|\~/)
|
38
|
+
add = s[0,2] if s.match(/^\*_/)
|
39
|
+
s.sub!(/^\-|\~|(\*_)/,"")
|
40
|
+
s = Regexp.escape(s)
|
41
|
+
words = Array.new
|
42
|
+
words += Dir["*"].reject{|d|!File.directory?(d)}.map{|d|"name:#{File.basename(d)}"}
|
43
|
+
words += ["rating:s","rating:q","rating:e"]
|
44
|
+
words += ["id:","md5:","description:","desc:","pool:","set:","width:",\
|
45
|
+
"height:","score:","favcount:","views:","ratio:","parent:"]
|
46
|
+
words += ["type:jpg","type:png","type:gif","type:swf"]
|
47
|
+
bools = ["hassource","hasdescription","ischild","idparent","inpool"]
|
48
|
+
words += bools.map{|b|["#{b}:true","#{b}:false"]}.flatten
|
49
|
+
# Make this a one dimensional array again!
|
50
|
+
words += ["order:random"]
|
51
|
+
orders = ["id","score","views","set","favcount","tagcount","comments",\
|
52
|
+
"mpixels","filesize","ratio","desclength"]
|
53
|
+
words += orders.map{|b|["#{b}_asc","#{b}_desc"]}.flatten.map{|b|"order:#{b}"}
|
54
|
+
# Here we make all orders in both directions and keep it as just an one
|
55
|
+
# dimensional array.
|
56
|
+
words += tags.map{|t|t["name"]}
|
57
|
+
words = words.grep(/^#{s}/)
|
58
|
+
words.map!{|w|"#{add}#{w}"}
|
59
|
+
s = "#{add}#{s}"
|
60
|
+
words
|
61
|
+
end
|
62
|
+
Readline.completer_word_break_characters = " "
|
63
|
+
Readline.completion_append_character = " "
|
64
|
+
history = File.open(File.expand_path(E621::Config.paths["history"]),"a+")
|
65
|
+
history.read.split($/).each do |l|
|
66
|
+
Readline::HISTORY << l
|
67
|
+
end
|
68
|
+
|
69
|
+
def net_send(query)
|
70
|
+
socket = TCPSocket.new(E621::Config.config["server"],5621)
|
71
|
+
socket.puts query.to_json
|
72
|
+
res = socket.read.parse
|
73
|
+
if res["error"] then
|
74
|
+
$stderr.puts "Server error: #{res["message"]}"
|
75
|
+
abort
|
76
|
+
end
|
77
|
+
socket.close
|
78
|
+
return res
|
79
|
+
end
|
80
|
+
|
81
|
+
api = API.new("user")
|
82
|
+
prompt = "#{api.user}@e621.net/posts> "
|
83
|
+
Dir.chdir(File.expand_path(E621::Config.paths["posts"])) do
|
84
|
+
while buff = Readline.readline(prompt.bold("yellow"), false) do
|
85
|
+
next if buff == String.new
|
86
|
+
if !(buff == String.new || buff == Readline::HISTORY.to_a.last) then
|
87
|
+
Readline::HISTORY << buff
|
88
|
+
history.puts buff
|
89
|
+
end
|
90
|
+
buff = buff.split(/\s+/)
|
91
|
+
name = buff.shift.sub("name:","")
|
92
|
+
query = {"action"=>"search","name"=>name,"query"=>buff}
|
93
|
+
rid = net_send(query)["rid"]
|
94
|
+
posts,page,num = Array.new,{"posts"=>[2]},1
|
95
|
+
while page["posts"] != Array.new do
|
96
|
+
print "Fetching page #{num.pad(3," ")} (#{posts.length.pad(6," ")} posts fetched).\r"
|
97
|
+
page = net_send({"action"=>"page","page"=>num,"rid"=>rid})
|
98
|
+
posts += page["posts"] if page["posts"]
|
99
|
+
num += 1
|
100
|
+
end
|
101
|
+
puts
|
102
|
+
Dir.mkdir(name) unless File.exist?(name)
|
103
|
+
Dir.chdir(name) do
|
104
|
+
count = 1
|
105
|
+
posts.reverse.each do |post|
|
106
|
+
print "Downloading post #{count.pad(6," ")} of #{posts.length.pad(6, " ")}.\r"
|
107
|
+
query = {"action"=>"show", "post"=>post}
|
108
|
+
res = net_send(query)
|
109
|
+
File.open(res["name"],"w"){|f|f.print Base64.decode64(res["data"])}
|
110
|
+
count += 1
|
111
|
+
end
|
112
|
+
end
|
113
|
+
puts
|
114
|
+
end
|
115
|
+
end
|
116
|
+
puts
|
data/bin/e621_server
ADDED
@@ -0,0 +1,158 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
=begin
|
3
|
+
Copyright 2014 Maxine Red <maxine_red1@yahoo.com>
|
4
|
+
|
5
|
+
This file is part of rubyhexagon.
|
6
|
+
|
7
|
+
rubyhexagon is free software: you can redistribute it and/or modify
|
8
|
+
it under the terms of the GNU General Public License as published by
|
9
|
+
the Free Software Foundation, either version 3 of the License, or
|
10
|
+
(at your option) any later version.
|
11
|
+
|
12
|
+
rubyhexagon is distributed in the hope that it will be useful,
|
13
|
+
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
14
|
+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
15
|
+
GNU General Public License for more details.
|
16
|
+
|
17
|
+
You should have received a copy of the GNU General Public License
|
18
|
+
along with rubyhexagon. If not, see <http://www.gnu.org/licenses/>.
|
19
|
+
=end
|
20
|
+
|
21
|
+
$:.unshift(File.expand_path(File.dirname(__FILE__)+"/../lib"))
|
22
|
+
|
23
|
+
require "rubyhexagon"
|
24
|
+
require "thread"
|
25
|
+
require "logger"
|
26
|
+
require "socket"
|
27
|
+
require "digest"
|
28
|
+
require "base64"
|
29
|
+
|
30
|
+
include E621
|
31
|
+
|
32
|
+
Thread.abort_on_exception = true
|
33
|
+
|
34
|
+
E621::Config.config = File.expand_path("~/.hexagon/conf.json")
|
35
|
+
api = API.new
|
36
|
+
tasks = Hash.new
|
37
|
+
mt = Mutex.new
|
38
|
+
server = TCPServer.new(5621)
|
39
|
+
Process.daemon
|
40
|
+
@log = Logger.new(File.expand_path(E621::Config.paths["logging"]))
|
41
|
+
#@log = Logger.new(STDOUT)
|
42
|
+
#@log.level = Logger::DEBUG
|
43
|
+
=begin
|
44
|
+
10.times do
|
45
|
+
Thread.new do
|
46
|
+
loop do
|
47
|
+
name,tclient,max,post = queue.pop
|
48
|
+
next unless post
|
49
|
+
Dir.mkdir(name) unless File.exist?(name)
|
50
|
+
post.download("#{name}/#{"0"*(7-post.id.to_s.length)}#{post.id}.#{post.md5}.#{post.file_ext}")
|
51
|
+
mt.synchronize do
|
52
|
+
unless tasks.has_key?(name) then
|
53
|
+
tasks.store(name,1)
|
54
|
+
else
|
55
|
+
tasks[name] += 1
|
56
|
+
end
|
57
|
+
count = tasks[name]
|
58
|
+
message = "Got #{" "*(max.to_s.length+2-count.to_s.length)}#{count}/#{max}"+" "*16
|
59
|
+
@log.debug message
|
60
|
+
tclient.puts message
|
61
|
+
tclient.close if count >= max
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
=end
|
67
|
+
|
68
|
+
@log.formatter = proc do |sev,dat,prog,msg|
|
69
|
+
"#{Time.now.strftime("%b %e, %Y %I:%M:%S %p")} #{msg}#$/"
|
70
|
+
end
|
71
|
+
@log.level = Logger::INFO
|
72
|
+
@log.info("New server instance started.")
|
73
|
+
Dir.chdir(File.expand_path("~/.hexagon")) do
|
74
|
+
Thread.new do
|
75
|
+
loop do
|
76
|
+
files,size,tm = Dir["cache/**/*"].reject{|x|!File.file?(x)},0,Array.new
|
77
|
+
files.each do |f|
|
78
|
+
size += File.size(f)
|
79
|
+
end
|
80
|
+
if size > 30*2**30 then
|
81
|
+
files.each do |f|
|
82
|
+
tm << {"file"=>f,"time"=>File.atime(f),"size"=>File.size(f)}
|
83
|
+
end
|
84
|
+
tm = tm.sort{|f1,f2|f1["time"]<=>f2["time"]}
|
85
|
+
while size > 30*2**30 do
|
86
|
+
unlink = tm.shift
|
87
|
+
size -= unlink["size"]
|
88
|
+
File.unlink(unlink["file"])
|
89
|
+
end
|
90
|
+
end
|
91
|
+
sleep 3600
|
92
|
+
end
|
93
|
+
end
|
94
|
+
begin
|
95
|
+
Dir["searches/*"].each{|f|File.unlink(f)}
|
96
|
+
loop do
|
97
|
+
client = server.accept
|
98
|
+
Thread.new(client) do |c|
|
99
|
+
begin
|
100
|
+
query = c.gets.parse
|
101
|
+
rescue
|
102
|
+
@log.error("[#{c.remote_address.ip_address}]: Error #$!.")
|
103
|
+
next
|
104
|
+
end
|
105
|
+
case query["action"]
|
106
|
+
when "search" then
|
107
|
+
name,query = query["name"],query["query"]
|
108
|
+
@log.info("[#{c.remote_address.ip_address}]: Search init #{name}/#{query.join("+")}.")
|
109
|
+
Dir.mkdir("searches") unless File.exist?("searches")
|
110
|
+
rid = Digest::SHA2.hexdigest(name+Time.now.strftime("%s"))
|
111
|
+
File.open("searches/#{rid}.json","w") do |f|
|
112
|
+
search = {"name"=>name,"query"=>query}
|
113
|
+
f.print search.to_json
|
114
|
+
end
|
115
|
+
rid = {"rid"=>rid}.to_json
|
116
|
+
c.print rid
|
117
|
+
when "page" then
|
118
|
+
page, rid = query["page"],query["rid"]
|
119
|
+
File.open("searches/#{rid}.json"){|f|query=f.read.parse}
|
120
|
+
@log.info("[#{c.remote_address.ip_address}]: Search page #{query["name"]}/#{page}.")
|
121
|
+
search = Search.new(query["query"])
|
122
|
+
search.page = page
|
123
|
+
posts = {"posts"=>search.posts}.to_json
|
124
|
+
c.print posts
|
125
|
+
when "show" then
|
126
|
+
post = Post.new(query["post"])
|
127
|
+
Dir.mkdir("cache") unless File.exist?("cache")
|
128
|
+
Dir.mkdir("cache/#{post.md5[0,2]}") unless File.exist?("cache/#{post.md5[0,2]}")
|
129
|
+
body = String.new
|
130
|
+
if File.exist?("cache/#{post.md5[0,2]}/#{post.md5}") then
|
131
|
+
cached = " (cached)"
|
132
|
+
File.open("cache/#{post.md5[0,2]}/#{post.md5}") do |f|
|
133
|
+
body = f.read
|
134
|
+
end
|
135
|
+
else
|
136
|
+
cached = ""
|
137
|
+
body = post.download_data
|
138
|
+
File.open("cache/#{post.md5[0,2]}/#{post.md5}","w") do |f|
|
139
|
+
f.print body
|
140
|
+
end
|
141
|
+
end
|
142
|
+
@log.info("[#{c.remote_address.ip_address}]: Show post #{query["post"]["id"].pad(6," ")}#{cached}.")
|
143
|
+
file = {"name"=>"#{post.id.pad(8)}.#{post.md5}.#{post.file_ext}","data"=>Base64.encode64(body)}
|
144
|
+
c.print file.to_json
|
145
|
+
end
|
146
|
+
c.close
|
147
|
+
end
|
148
|
+
end
|
149
|
+
rescue Errno
|
150
|
+
@log.error("Error: #$!!")
|
151
|
+
rescue => e
|
152
|
+
p e.class
|
153
|
+
@log.fatal("Fatal Error: #$!!")
|
154
|
+
raise
|
155
|
+
ensure
|
156
|
+
@log.info("Server instance terminated.")
|
157
|
+
end
|
158
|
+
end
|
data/bin/fav-sync
ADDED
@@ -0,0 +1,82 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
=begin
|
3
|
+
Copyright 2014 Maxine Red <maxine_red1@yahoo.com>
|
4
|
+
|
5
|
+
This file is part of rubyhexagon.
|
6
|
+
|
7
|
+
rubyhexagon is free software: you can redistribute it and/or modify
|
8
|
+
it under the terms of the GNU General Public License as published by
|
9
|
+
the Free Software Foundation, either version 3 of the License, or
|
10
|
+
(at your option) any later version.
|
11
|
+
|
12
|
+
rubyhexagon is distributed in the hope that it will be useful,
|
13
|
+
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
14
|
+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
15
|
+
GNU General Public License for more details.
|
16
|
+
|
17
|
+
You should have received a copy of the GNU General Public License
|
18
|
+
along with rubyhexagon. If not, see <http://www.gnu.org/licenses/>.
|
19
|
+
=end
|
20
|
+
|
21
|
+
loader = File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__
|
22
|
+
$:.unshift(File.expand_path(File.dirname(loader))+"/../lib")
|
23
|
+
|
24
|
+
require "rubyhexagon"
|
25
|
+
|
26
|
+
include E621
|
27
|
+
|
28
|
+
max_wait = 5
|
29
|
+
E621::Config.config = File.expand_path("~/.hexagon/conf.json")
|
30
|
+
api = API.new("user")
|
31
|
+
uid = api.get("index",{"name"=>api.user}).first["id"]
|
32
|
+
Dir.chdir(File.expand_path("~/pictures/e621/favorites"))
|
33
|
+
remote = Array.new
|
34
|
+
Search.new("fav:maxine_red order:id".split(" ")).each_post do |post|
|
35
|
+
string = ["0"*(7-post.id.to_s.length)+post.id.to_s,post.md5,post.file_ext].join(".")
|
36
|
+
remote << Array.new
|
37
|
+
if !File.exist?(string) then
|
38
|
+
post.download(string)
|
39
|
+
else
|
40
|
+
md5 = String.new
|
41
|
+
File.open(string){|f|md5 = Digest::MD5.hexdigest(f.read)}
|
42
|
+
next if md5 == post.md5
|
43
|
+
post.download(string)
|
44
|
+
end
|
45
|
+
puts "Downloaded post #{" "*(7-post.id.to_s.length)}#{post.id}."
|
46
|
+
sleep(rand*max_wait)
|
47
|
+
end
|
48
|
+
local = Dir["*"].reject{|x|x.match(/db$/)}.map{|x|x.sub(/\..+/,"").to_i}.sort.uniq
|
49
|
+
(remote-local).each do |id|
|
50
|
+
post = Post.new({"id"=>id})
|
51
|
+
f = post.unfavorite
|
52
|
+
if f["success"] then
|
53
|
+
print "\e[1;33mRemoved favorite on post #{" "*(7-post.id.to_s.length)}#{post.id}.\e[0m "
|
54
|
+
else
|
55
|
+
print "\e[1;31m#{f["reason"].to_s.gsub(/<.+?>/,"")}.\e[0m "
|
56
|
+
end
|
57
|
+
f = post.vote(1)
|
58
|
+
if f["success"] then
|
59
|
+
puts "\e[1;33mScored post #{" "*(7-post.id.to_s.length)}#{post.id} down.\e[0m"
|
60
|
+
else
|
61
|
+
puts "\e[1;31m#{f["reason"].to_s.gsub(/<.+?>/,"")}.\e[0m"
|
62
|
+
end
|
63
|
+
sleep(rand*max_wait)
|
64
|
+
end
|
65
|
+
(local-remote).each do |id|
|
66
|
+
next if id == 0
|
67
|
+
post = Post.new({"id"=>id})
|
68
|
+
f = post.favorite
|
69
|
+
if f["success"] then
|
70
|
+
print "\e[1;32mFavorited #{" "*(7-post.id.to_s.length)}#{post.id}.\e[0m "
|
71
|
+
else
|
72
|
+
print "\e[1;31m#{f["reason"].gsub(/<.+?>/,"")}.\e[0m "
|
73
|
+
#File.unlink(Dir["#{set["shortname"]}/*.#{"0"*(8-post.id.to_s.length)}#{post.id}.*"].first)
|
74
|
+
end
|
75
|
+
f = post.vote(1)
|
76
|
+
if f["success"] then
|
77
|
+
puts "\e[1;32mScored post #{" "*(7-post.id.to_s.length)}#{post.id} up.\e[0m"
|
78
|
+
else
|
79
|
+
puts "\e[1;31m#{f["reason"].to_s.gsub(/<.+?>/,"")}.\e[0m"
|
80
|
+
end
|
81
|
+
sleep(rand*max_wait)
|
82
|
+
end
|
data/bin/set-sync
ADDED
@@ -0,0 +1,225 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
=begin
|
3
|
+
Copyright 2014 Maxine Red <maxine_red1@yahoo.com>
|
4
|
+
|
5
|
+
This file is part of rubyhexagon.
|
6
|
+
|
7
|
+
rubyhexagon is free software: you can redistribute it and/or modify
|
8
|
+
it under the terms of the GNU General Public License as published by
|
9
|
+
the Free Software Foundation, either version 3 of the License, or
|
10
|
+
(at your option) any later version.
|
11
|
+
|
12
|
+
rubyhexagon is distributed in the hope that it will be useful,
|
13
|
+
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
14
|
+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
15
|
+
GNU General Public License for more details.
|
16
|
+
|
17
|
+
You should have received a copy of the GNU General Public License
|
18
|
+
along with rubyhexagon. If not, see <http://www.gnu.org/licenses/>.
|
19
|
+
=end
|
20
|
+
|
21
|
+
loader = File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__
|
22
|
+
$:.unshift(File.expand_path(File.dirname(loader))+"/../lib")
|
23
|
+
|
24
|
+
require "rubyhexagon"
|
25
|
+
require "logger"
|
26
|
+
|
27
|
+
include E621
|
28
|
+
|
29
|
+
|
30
|
+
def download(post,name,set)
|
31
|
+
artist = post.tags.split(/\s+/).map do |t|
|
32
|
+
@tags[@tags.index(t)] if @tags.index(t)
|
33
|
+
end
|
34
|
+
begin
|
35
|
+
artist = artist.compact.first.sub("_(artist)","")
|
36
|
+
rescue
|
37
|
+
artist = "unknown"
|
38
|
+
end
|
39
|
+
string = [post.created_at.to_i,post.id.pad(8),artist+"_"+name,post.file_ext]
|
40
|
+
string = string.join(".")
|
41
|
+
begin
|
42
|
+
api = API.new("post")
|
43
|
+
post = Post.new(api.post("show",{"id"=>post.id}))
|
44
|
+
post.download("#{name}/#{string}")
|
45
|
+
rescue
|
46
|
+
puts $!
|
47
|
+
sleep 1
|
48
|
+
retry
|
49
|
+
end
|
50
|
+
File.utime(post.created_at,post.created_at,"#{name}/#{string}")
|
51
|
+
@posts["downloads"] << post.id
|
52
|
+
jposts = @posts.to_json
|
53
|
+
set_files = File.expand_path("~/.hexagon/sets")
|
54
|
+
File.open(set_files+"/#{name}.json","w"){|f|f.print jposts}
|
55
|
+
s = "Downloaded post #{" "*(6-post.id.to_s.length)}#{post.id} from \"#{set["name"]}\"."
|
56
|
+
puts s
|
57
|
+
@log.info(s)
|
58
|
+
end
|
59
|
+
max_wait = 2.5
|
60
|
+
E621::Config.config = File.expand_path("~/.hexagon/conf.json")
|
61
|
+
api = API.new("user")
|
62
|
+
@log = Logger.new(File.expand_path(E621::Config.paths["logging"]))
|
63
|
+
@log.formatter = proc do |sev,dat,prog,msg|
|
64
|
+
"#{Time.now.strftime("%b %e, %Y %I:%M:%S %p")}: #{msg}#$/"
|
65
|
+
end
|
66
|
+
@log.level = Logger::INFO
|
67
|
+
@log.info("Program #$0 started.")
|
68
|
+
uid = api.get("index",{"name"=>api.user}).first["id"]
|
69
|
+
tags = File.expand_path("~/.hexagon/tags.json")
|
70
|
+
if !File.exist?(tags) then
|
71
|
+
@tags = Array.new
|
72
|
+
else
|
73
|
+
File.open(tags) do |f|
|
74
|
+
@tags = f.read.parse
|
75
|
+
end
|
76
|
+
end
|
77
|
+
@tags = @tags.sort{|k1,k2|k1["id"]<=>k2["id"]}
|
78
|
+
net = Net::HTTP.new("e621.net",443)
|
79
|
+
net.use_ssl = true
|
80
|
+
body,page = [2],1
|
81
|
+
until body == Array.new do
|
82
|
+
body = net.get("/tag/index.json?limit=0&order=count&after_id=#{@tags.last["id"]}&page=#{page}").body.parse
|
83
|
+
body = body.map{|x|x["name"] = x["name"].encode("us-ascii", :invalid => :replace, :undef => :replace, :replace => "");x}
|
84
|
+
@tags += body
|
85
|
+
page += 1
|
86
|
+
end
|
87
|
+
jtags = @tags.to_json
|
88
|
+
File.open(tags,"w"){|f|f.print jtags}
|
89
|
+
@tags = @tags.reject{|x|x["type"]!=1}.map{|x|x["name"]}
|
90
|
+
Dir.chdir(File.expand_path("~/Dropbox/Furry/e621/sets")) do
|
91
|
+
sets = Array.new
|
92
|
+
File.open(File.expand_path("~/.hexagon/sets.json")) do |f|
|
93
|
+
sets = f.read.parse
|
94
|
+
end
|
95
|
+
sets = sets.sort{|s1,s2|s1["id"]<=>s2["id"]}
|
96
|
+
set_files = File.expand_path("~/.hexagon/sets")
|
97
|
+
api = API.new("set")
|
98
|
+
sets.each do |set|
|
99
|
+
sid, owner, query = set["id"], set["owner"], set["search"]
|
100
|
+
set = api.get("show", {"id"=>sid})
|
101
|
+
posts = set["posts"].map{|post|Post.new(post)}
|
102
|
+
name = set["shortname"]
|
103
|
+
@log.info("Fetching set #{set["name"]}")
|
104
|
+
if !File.exists?(set_files+"/#{name}.json") then
|
105
|
+
File.open(set_files+"/#{name}.json","w"){|f|f.print "{\"downloads\":[]}"}
|
106
|
+
end
|
107
|
+
File.open(set_files+"/#{name}.json"){|f|@posts = f.read.parse}
|
108
|
+
Dir.mkdir(name) unless File.exist?(name)
|
109
|
+
posts.each do |post|
|
110
|
+
next if @posts["downloads"].include?(post.id)
|
111
|
+
download(post,name,set)
|
112
|
+
end
|
113
|
+
next if owner != uid
|
114
|
+
Search.new("fav:maxine_red #{query} order:id".split(" ")).each_post do |post|
|
115
|
+
next if @posts["downloads"].include?(post.id)
|
116
|
+
download(post,name,set)
|
117
|
+
sleep(rand*max_wait)
|
118
|
+
end
|
119
|
+
end
|
120
|
+
sets.each do |set|
|
121
|
+
sid, owner, query = set["id"], set["owner"], set["search"]
|
122
|
+
next if owner != uid
|
123
|
+
set = api.get("show", {"id"=>sid})
|
124
|
+
posts = set["posts"].map{|post|Post.new(post).id}
|
125
|
+
name = set["shortname"]
|
126
|
+
local = Dir["#{name}/*"].reject{|x|x.match(/db$/)}.map{|x|x.split(".")[1].to_i}.sort.uniq
|
127
|
+
(posts-local).each do |id|
|
128
|
+
post = Post.new({"id"=>id})
|
129
|
+
f = post.remove_from_set(sid)
|
130
|
+
if f["success"] then
|
131
|
+
s = "Removed Post #{" "*(6-post.id.to_s.length)}#{post.id} from \"#{set["name"]}\""
|
132
|
+
puts "\e[1;33m#{s}\e[0m"
|
133
|
+
@log.info(s)
|
134
|
+
else
|
135
|
+
s = "#{f["reason"].to_s.gsub(/<.+?>/,"")}."
|
136
|
+
puts "\e[1;31m#{s}\e[0m"
|
137
|
+
@log.info(s)
|
138
|
+
end
|
139
|
+
sleep(rand*max_wait)
|
140
|
+
end
|
141
|
+
(local-posts).each do |id|
|
142
|
+
next if id == 0
|
143
|
+
post = Post.new({"id"=>id})
|
144
|
+
f = post.add_to_set(sid)
|
145
|
+
if f["success"] then
|
146
|
+
s = "Added Post #{" "*(6-post.id.to_s.length)}#{post.id} to \"#{set["name"]}\"."
|
147
|
+
puts s
|
148
|
+
@log.info(s)
|
149
|
+
else
|
150
|
+
s = "#{f["reason"].gsub(/<.+?>/,"")}. Removing local file."
|
151
|
+
puts s
|
152
|
+
@log.info(s)
|
153
|
+
File.unlink(Dir["#{set["shortname"]}/*.#{"0"*(8-post.id.to_s.length)}#{post.id}.*"].first)
|
154
|
+
end
|
155
|
+
sleep(rand*max_wait)
|
156
|
+
end
|
157
|
+
end
|
158
|
+
end
|
159
|
+
Dir.chdir(File.expand_path("~/Dropbox/Furry/e621/pools")) do
|
160
|
+
pools = Array.new
|
161
|
+
File.open(File.expand_path("~/.hexagon/pools.json")) do |f|
|
162
|
+
pools = f.read.parse.sort.uniq
|
163
|
+
end
|
164
|
+
pool_files = File.expand_path("~/.hexagon/pools")
|
165
|
+
api = API.new("pool")
|
166
|
+
pools.each do |pool|
|
167
|
+
spool = {"updated_at"=>0, "posts"=>[]}
|
168
|
+
npool = api.get("show",{"id"=>pool})
|
169
|
+
pool = Pool.new(npool)
|
170
|
+
begin
|
171
|
+
pfile = pool_files+"/#{pool.id.pad(5)}.json"
|
172
|
+
rescue
|
173
|
+
@log.error("Error occured: #{npool.inspect}")
|
174
|
+
raise
|
175
|
+
end
|
176
|
+
if !File.exists?(pfile) then
|
177
|
+
File.open(pfile,"w"){|f|f.print spool.to_json}
|
178
|
+
spool = Pool.new(spool)
|
179
|
+
else
|
180
|
+
File.open(pfile){|f|spool = Pool.new(f.read.parse)}
|
181
|
+
end
|
182
|
+
posts = Array.new
|
183
|
+
next if pool.updated_at.to_i <= spool.updated_at.to_i
|
184
|
+
(pool.post_count/24.0).ceil.times do |page|
|
185
|
+
posts += pool.posts.map{|post|Post.new(post)}
|
186
|
+
pool = Pool.new(api.get("show",{"id"=>pool.id,"page"=>page+2}))
|
187
|
+
end
|
188
|
+
pool.name = pool.name.gsub("_"," ").encode("us-ascii", :invalid => :replace, :undef => :replace, :replace => "")
|
189
|
+
name = pool.name.gsub(/[^0-9, ,_,a-z,\-]/i,"").sub(/\s+$/,"")
|
190
|
+
Dir.mkdir(name) unless File.exist?(name)
|
191
|
+
@log.info("Fetching pool #{pool.name}")
|
192
|
+
posts.each_with_index do |post,id|
|
193
|
+
id = id.succ
|
194
|
+
next if spool.posts.include?(post.id)
|
195
|
+
artist = post.tags.split(/\s+/).map do |t|
|
196
|
+
@tags[@tags.index(t)] if @tags.index(t)
|
197
|
+
end
|
198
|
+
begin
|
199
|
+
artist = artist.compact.first.sub("_(artist)","")
|
200
|
+
rescue
|
201
|
+
artist = "unknown"
|
202
|
+
end
|
203
|
+
string = [pool.id.pad(5),id.pad(4),post.id.pad(8),artist,name.downcase.gsub(/\s/,"_"),post.file_ext]
|
204
|
+
string = string.join(".")
|
205
|
+
begin
|
206
|
+
post.download("#{name}/#{string}")
|
207
|
+
rescue
|
208
|
+
puts $!
|
209
|
+
sleep 1
|
210
|
+
retry
|
211
|
+
end
|
212
|
+
File.utime(post.created_at,post.created_at,"#{name}/#{string}")
|
213
|
+
spool.posts << post.id
|
214
|
+
jposts = spool.to_json
|
215
|
+
File.open(pfile,"w"){|f|f.print jposts}
|
216
|
+
s = "Downloaded post #{post.id.pad(6," ")} (#{id.pad(posts.length.to_s.length," ")}/#{posts.length}) from \"#{pool.name}\"."
|
217
|
+
puts s
|
218
|
+
@log.info(s)
|
219
|
+
sleep(rand*max_wait)
|
220
|
+
end
|
221
|
+
spool.updated_at = pool.updated_at
|
222
|
+
jposts = spool.to_json
|
223
|
+
File.open(pfile,"w"){|f|f.print jposts}
|
224
|
+
end
|
225
|
+
end
|