rubyhexagon 0.3.2 → 0.4.0

data/lib/examples/db.rb

@@ -0,0 +1,143 @@
+=begin
+  Copyright 2014 Maxine Red <maxine_red1@yahoo.com>
+
+  This file is part of rubyhexagon.
+
+  rubyhexagon is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  rubyhexagon is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with rubyhexagon.  If not, see <http://www.gnu.org/licenses/>.
+=end
+
+require "logger"
+require "thread"
+require "pg"
+
+include E621
+
+$form = "%b %e, %Y %I:%M:%S %p"
+
+require "search"
+require "pool"
+require "set"
+
+Thread.abort_on_exception = true
+
+class DB
+  def initialize(database)
+    @pg = PG.connect(dbname:database)
+    @mt = Mutex.new
+  end
+  def exec(query)
+    @mt.synchronize do
+      result = @pg.exec(query)
+      return result
+    end
+  end
+  def escape_string(str)
+    @pg.escape_string(str)
+  end
+end
+
+class Sync
+  def initialize
+    E621::Config.config = File.expand_path("~/.hexagon/conf.json")
+    if $stdout.tty? then
+      @log = Logger.new($stdout)
+      @log.level = Logger::DEBUG
+    else
+      @log = Logger.new(File.expand_path(E621::Config.paths["logging"]))
+      @log.level = Logger::INFO
+    end
+    @log.formatter = proc do |sev,dat,prog,msg|
+      "#{Time.now.strftime($form)} [#{sev.pad(5)}] #{msg}#$/"
+    end
+    api = API.new("user") # Used for login
+    @log.info("Program #$0 started.")
+    @uid = api.get("index",{"name"=>api.user}).first["id"]
+    @max_wait = 1
+    Dir.chdir(File.expand_path(E621::Config.paths["files"]))
+    @db = DB.new("danbooru")
+    @queue = Queue.new
+    @mt = Mutex.new
+  end
+
+  def threads
+    return E621::Config.threads
+  end
+  
+  def worker(mod)
+    if mod == "post" then
+      fetch_posts
+    end
+  end
+
+  def fetch_posts
+    lid = @db.exec("SELECT id FROM e621.posts ORDER BY id DESC LIMIT 1;").first
+    if lid then
+      lid = lid["id"].to_i
+    else
+      lid = 0
+    end
+    posts = Search.new(["order:id", "id:#{lid}.."])
+    @cols = ["id","tags","artist","created_at","score","md5","file_ext","description","rating","width","height","has_children","parent_id","file_size"]
+    posts.each_post do |post|
+      @queue << [post,"#{post.md5[0,2]}/#{post.md5[2,2]}/#{post.md5}.#{post.file_ext}"]
+    end
+  end
+  def download
+    Thread.current.exit if @queue.empty?
+    post,path = @queue.pop
+    m = String.new
+    path.split("/")[0,2].each do |d|
+      m += "#{d}/"
+      Dir.mkdir(m) unless File.exists?(m)
+    end
+    tries = 0
+    begin
+      post.download(path)
+    rescue
+      raise if tries > 8
+      tries += 1
+      sleep 2
+      retry
+    end
+    data = @cols.map do |x|
+      y=post.__send__(x.to_sym)
+      y = y.to_i if x == "created_at"
+      y = y.join(" ") if x == "artist"
+      y != nil ? @db.escape_string(y.to_s) : "NULL"
+    end
+    if !@db.exec("SELECT id FROM e621.posts WHERE id = #{post.id};").first then
+      query = "INSERT INTO e621.posts (#{@cols.join(",")}) VALUES ('#{data.join("','")}');".gsub("'NULL'","NULL")
+      @db.exec(query)
+      @mt.synchronize do
+        @log.info("Added post ##{post.id.pad(6," ")} to library.")
+      end
+    end
+  end
+end
+sync = Sync.new
+posts = Thread.new do
+  sync.worker("post")
+end
+sleep 4
+downloads = Array.new
+10.times do
+  downloads << Thread.new do
+    loop do
+      sync.download
+    end
+  end
+end
+
+posts.join
+downloads.each{|d|d.join}

data/lib/rubyhexagon.rb

@@ -30,5 +30,5 @@ require "pool"
 
 module E621
   Name = "rubyhexagon"
-  Version = "0.3.2"
+  Version = "0.4.0"
 end

data/lib/standard/string.rb

@@ -24,6 +24,9 @@ class String
   def parse
     JSON.parser.new(self).parse
   end
+  def to_ascii
+    self.gsub("_"," ").encode("us-ascii", :invalid => :replace, :undef => :replace, :replace => "")
+  end
   # Remove all tags inside of a string.
   def clean
     self.gsub(/<.+?>/,"")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rubyhexagon
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-27 00:00:00.000000000 Z
+date: 2015-06-11 00:00:00.000000000 Z
 dependencies: []
 description: Rubyhexagon provides Ruby bindings for the e621 [dot] net API.
 email: maxine_red1@yahoo.com
@@ -21,6 +21,7 @@ files:
 - lib/set.rb
 - lib/examples/sync.rb
 - lib/examples/add.rb
+- lib/examples/db.rb
 - lib/pool.rb
 - lib/post.rb
 - lib/standard/time.rb