rubyhexagon 0.3.2 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/examples/db.rb +143 -0
- data/lib/rubyhexagon.rb +1 -1
- data/lib/standard/string.rb +3 -0
- metadata +3 -2
data/lib/examples/db.rb
ADDED
@@ -0,0 +1,143 @@
|
|
1
|
+
=begin
|
2
|
+
Copyright 2014 Maxine Red <maxine_red1@yahoo.com>
|
3
|
+
|
4
|
+
This file is part of rubyhexagon.
|
5
|
+
|
6
|
+
rubyhexagon is free software: you can redistribute it and/or modify
|
7
|
+
it under the terms of the GNU General Public License as published by
|
8
|
+
the Free Software Foundation, either version 3 of the License, or
|
9
|
+
(at your option) any later version.
|
10
|
+
|
11
|
+
rubyhexagon is distributed in the hope that it will be useful,
|
12
|
+
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
13
|
+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
14
|
+
GNU General Public License for more details.
|
15
|
+
|
16
|
+
You should have received a copy of the GNU General Public License
|
17
|
+
along with rubyhexagon. If not, see <http://www.gnu.org/licenses/>.
|
18
|
+
=end
|
19
|
+
|
20
|
+
require "logger"
|
21
|
+
require "thread"
|
22
|
+
require "pg"
|
23
|
+
|
24
|
+
include E621
|
25
|
+
|
26
|
+
$form = "%b %e, %Y %I:%M:%S %p"
|
27
|
+
|
28
|
+
require "search"
|
29
|
+
require "pool"
|
30
|
+
require "set"
|
31
|
+
|
32
|
+
Thread.abort_on_exception = true
|
33
|
+
|
34
|
+
class DB
|
35
|
+
def initialize(database)
|
36
|
+
@pg = PG.connect(dbname:database)
|
37
|
+
@mt = Mutex.new
|
38
|
+
end
|
39
|
+
def exec(query)
|
40
|
+
@mt.synchronize do
|
41
|
+
result = @pg.exec(query)
|
42
|
+
return result
|
43
|
+
end
|
44
|
+
end
|
45
|
+
def escape_string(str)
|
46
|
+
@pg.escape_string(str)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
class Sync
|
51
|
+
def initialize
|
52
|
+
E621::Config.config = File.expand_path("~/.hexagon/conf.json")
|
53
|
+
if $stdout.tty? then
|
54
|
+
@log = Logger.new($stdout)
|
55
|
+
@log.level = Logger::DEBUG
|
56
|
+
else
|
57
|
+
@log = Logger.new(File.expand_path(E621::Config.paths["logging"]))
|
58
|
+
@log.level = Logger::INFO
|
59
|
+
end
|
60
|
+
@log.formatter = proc do |sev,dat,prog,msg|
|
61
|
+
"#{Time.now.strftime($form)} [#{sev.pad(5)}] #{msg}#$/"
|
62
|
+
end
|
63
|
+
api = API.new("user") # Used for login
|
64
|
+
@log.info("Program #$0 started.")
|
65
|
+
@uid = api.get("index",{"name"=>api.user}).first["id"]
|
66
|
+
@max_wait = 1
|
67
|
+
Dir.chdir(File.expand_path(E621::Config.paths["files"]))
|
68
|
+
@db = DB.new("danbooru")
|
69
|
+
@queue = Queue.new
|
70
|
+
@mt = Mutex.new
|
71
|
+
end
|
72
|
+
|
73
|
+
def threads
|
74
|
+
return E621::Config.threads
|
75
|
+
end
|
76
|
+
|
77
|
+
def worker(mod)
|
78
|
+
if mod == "post" then
|
79
|
+
fetch_posts
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
def fetch_posts
|
84
|
+
lid = @db.exec("SELECT id FROM e621.posts ORDER BY id DESC LIMIT 1;").first
|
85
|
+
if lid then
|
86
|
+
lid = lid["id"].to_i
|
87
|
+
else
|
88
|
+
lid = 0
|
89
|
+
end
|
90
|
+
posts = Search.new(["order:id", "id:#{lid}.."])
|
91
|
+
@cols = ["id","tags","artist","created_at","score","md5","file_ext","description","rating","width","height","has_children","parent_id","file_size"]
|
92
|
+
posts.each_post do |post|
|
93
|
+
@queue << [post,"#{post.md5[0,2]}/#{post.md5[2,2]}/#{post.md5}.#{post.file_ext}"]
|
94
|
+
end
|
95
|
+
end
|
96
|
+
def download
|
97
|
+
Thread.current.exit if @queue.empty?
|
98
|
+
post,path = @queue.pop
|
99
|
+
m = String.new
|
100
|
+
path.split("/")[0,2].each do |d|
|
101
|
+
m += "#{d}/"
|
102
|
+
Dir.mkdir(m) unless File.exists?(m)
|
103
|
+
end
|
104
|
+
tries = 0
|
105
|
+
begin
|
106
|
+
post.download(path)
|
107
|
+
rescue
|
108
|
+
raise if tries > 8
|
109
|
+
tries += 1
|
110
|
+
sleep 2
|
111
|
+
retry
|
112
|
+
end
|
113
|
+
data = @cols.map do |x|
|
114
|
+
y=post.__send__(x.to_sym)
|
115
|
+
y = y.to_i if x == "created_at"
|
116
|
+
y = y.join(" ") if x == "artist"
|
117
|
+
y != nil ? @db.escape_string(y.to_s) : "NULL"
|
118
|
+
end
|
119
|
+
if !@db.exec("SELECT id FROM e621.posts WHERE id = #{post.id};").first then
|
120
|
+
query = "INSERT INTO e621.posts (#{@cols.join(",")}) VALUES ('#{data.join("','")}');".gsub("'NULL'","NULL")
|
121
|
+
@db.exec(query)
|
122
|
+
@mt.synchronize do
|
123
|
+
@log.info("Added post ##{post.id.pad(6," ")} to library.")
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
128
|
+
sync = Sync.new
|
129
|
+
posts = Thread.new do
|
130
|
+
sync.worker("post")
|
131
|
+
end
|
132
|
+
sleep 4
|
133
|
+
downloads = Array.new
|
134
|
+
10.times do
|
135
|
+
downloads << Thread.new do
|
136
|
+
loop do
|
137
|
+
sync.download
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
posts.join
|
143
|
+
downloads.each{|d|d.join}
|
data/lib/rubyhexagon.rb
CHANGED
data/lib/standard/string.rb
CHANGED
@@ -24,6 +24,9 @@ class String
|
|
24
24
|
def parse
|
25
25
|
JSON.parser.new(self).parse
|
26
26
|
end
|
27
|
+
def to_ascii
|
28
|
+
self.gsub("_"," ").encode("us-ascii", :invalid => :replace, :undef => :replace, :replace => "")
|
29
|
+
end
|
27
30
|
# Remove all tags inside of a string.
|
28
31
|
def clean
|
29
32
|
self.gsub(/<.+?>/,"")
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rubyhexagon
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2015-
|
12
|
+
date: 2015-06-11 00:00:00.000000000 Z
|
13
13
|
dependencies: []
|
14
14
|
description: Rubyhexagon provides Ruby bindings for the e621 [dot] net API.
|
15
15
|
email: maxine_red1@yahoo.com
|
@@ -21,6 +21,7 @@ files:
|
|
21
21
|
- lib/set.rb
|
22
22
|
- lib/examples/sync.rb
|
23
23
|
- lib/examples/add.rb
|
24
|
+
- lib/examples/db.rb
|
24
25
|
- lib/pool.rb
|
25
26
|
- lib/post.rb
|
26
27
|
- lib/standard/time.rb
|