rubyhexagon 0.1.1 → 0.1.2

data/lib/examples/sync.rb

@@ -0,0 +1,196 @@
+=begin
+  Copyright 2014 Maxine Red <maxine_red1@yahoo.com>
+
+  This file is part of rubyhexagon.
+
+  rubyhexagon is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  rubyhexagon is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with rubyhexagon.  If not, see <http://www.gnu.org/licenses/>.
+=end
+
+require "logger"
+
+include E621
+
+mod = ARGV.shift
+
+$0 = "E621 #{mod.capitalize}-sync"
+$form = "%b %e, %Y %I:%M:%S %p"
+
+require mod
+
+class Sync
+  def initialize
+    if $stdout.tty? then
+      @log = Logger.new($stdout)
+      @log.level = Logger::DEBUG
+    else
+      @log = Logger.new(File.expand_path(E621::Config.paths["logging"]))
+      @log.level = Logger::INFO
+    end
+    @log.formatter = proc do |sev,dat,prog,msg|
+      "#{Time.now.strftime($form)} [#{sev.pad(5)}] #{msg}#$/"
+    end
+    E621::Config.config = File.expand_path("~/.hexagon/conf.json")
+    api = API.new("user") # Used for login
+    @log.info("Program #$0 started.")
+    @uid = api.get("index",{"name"=>api.user}).first["id"]
+    @max_wait = 1
+    tags
+  end
+  
+  def tags
+    tags = File.expand_path("~/.hexagon/tags.json")
+    if !File.exist?(tags) then
+      @tags = Array.new
+    else
+      File.open(tags) do |f|
+        @tags = f.read.parse
+      end
+    end
+    @tags = @tags.sort{|k1,k2|k1["id"]<=>k2["id"]}
+    api = API.new("tag")
+    body,page = [2],1
+    until body == Array.new do
+      body = api.get("index",{"limit"=>0,"order"=>"count","after_id"=>@tags.last["id"],"page"=>page})
+      body = body.map{|x|x["name"] = x["name"].encode("us-ascii", :invalid => :replace, :undef => :replace, :replace => "");x}
+      @tags += body
+      page += 1
+    end
+    jtags = @tags.to_json
+    File.open(tags,"w"){|f|f.print jtags}
+    @tags = @tags.reject{|x|x["type"]!=1}.map{|x|x["name"]}
+  end
+
+  def worker(mod)
+    @mod = mod
+    Dir.chdir(File.expand_path("~/Dropbox/Furry/e621/#{mod}s")) do
+      sets = Array.new
+      File.open(File.expand_path(E621::Config.paths["j#{mod}s"])) do |f|
+        sets = f.read.parse
+      end
+      @set_files = File.expand_path(E621::Config.paths["sets"])
+      api = API.new(mod)
+      sets.each do |oset|
+        id = mod == "set" ? oset["id"] : oset
+        if mod == "set" then
+          set = Set.new(api.get("show", {"id"=>id}))
+        elsif mod == "pool" then
+          set = Pool.new(api.get("show", {"id"=>id}))
+        end
+        set.name = set.name.gsub("_"," ")
+        @log.info("Fetching #{mod} \"#{set.name}\"")
+        set.name = "Love_Can_Be_Different" if set.name == "Love_can_be_different"
+        set.name = set.name.gsub("_"," ").encode("us-ascii", :invalid => :replace, :undef => :replace, :replace => "")
+        set.name = set.name.gsub(/[^0-9, ,_,a-z,\-]/i,"").sub(/\s+$/,"")
+        name = mod == "set" ? set.shortname : id.pad(5)
+        if !File.exists?(@set_files+"/#{name}.json") then
+          File.open(@set_files+"/#{name}.json","w"){|f|f.print "{\"downloads\":[],\"updated_at\":0}"}
+        end 
+        File.open(@set_files+"/#{name}.json"){|f|@posts = f.read.parse}
+        @posts.store("updated_at",0) if !@posts.has_key?("updated_at")
+        next if set.updated_at.to_i <= @posts["updated_at"].to_i
+        dname = mod == "pool" ? set.name : name
+        Dir.mkdir(dname) unless File.exist?(dname)
+        i = 1
+        set.each_post do |post|
+          next if @posts["downloads"].include?(post.id)
+          download(post,dname,set,i)
+          i += 1
+        end
+        advanced_set(oset) if mod == "set"
+      end
+    end
+    @posts["updated_at"] = set.updated_at.to_i
+    jposts = @posts.to_json
+    File.open(@set_files+"/#{name}.json","w"){|f|f.print jposts}
+  end
+
+  def set(set)
+    sid, owner, query = set["id"], set["owner"], set["search"]
+    if owner == uid then
+      Search.new("fav:maxine_red #{query} order:id".split(" ")).each_post do |post|
+        next if @posts["downloads"].include?(post.id)
+        download(post,name,set)
+      end
+      set = api.get("show", {"id"=>sid})
+      posts = set["posts"].map{|post|Post.new(post).id}
+      name = set["shortname"]
+      local = Dir["#{name}/*"].reject{|x|x.match(/db$/)}.map{|x|x.split(".")[1].to_i}.sort.uniq
+      (posts-local).each do |id|
+        post = Post.new({"id"=>id})
+        f = post.remove_from_set(sid)
+        if f["success"] then
+          s = "Removed Post #{" "*(6-post.id.to_s.length)}#{post.id} from \"#{set["name"]}\""
+          puts "#{Time.now.strftime($form)}: \e[1;33m#{s}\e[0m"
+          @log.info(s)
+        else
+          s = "#{f["reason"].to_s.gsub(/<.+?>/,"")}."
+          puts "#{Time.now.strftime($form)}: \e[1;31m#{s}\e[0m"
+          @log.info(s)
+        end
+      end
+      (local-posts).each do |id|
+        next if id == 0
+        post = Post.new({"id"=>id})
+        f = post.add_to_set(sid)
+        if f["success"] then
+          s = "Added Post #{" "*(6-post.id.to_s.length)}#{post.id} to \"#{set["name"]}\"."
+          puts "#{Time.now.strftime($form)}: #{s}"
+          @log.info(s)
+        else
+          s = "#{f["reason"].gsub(/<.+?>/,"")}. Removing local file."
+          puts "#{Time.now.strftime($form)}: #{s}"
+          @log.info(s)
+          File.unlink(Dir["#{set["shortname"]}/*.#{"0"*(8-post.id.to_s.length)}#{post.id}.*"].first)
+        end
+      end
+    end
+  end
+  def download(post,name,set,id=0)
+    artist = post.tags.split(/\s+/).map do |t|
+      @tags[@tags.index(t)] if @tags.index(t)
+    end
+    begin
+      artist = artist.compact.first.sub("_(artist)","")
+    rescue
+      artist = "unknown"
+    end
+    if @mod == "set" then
+      string = [post.created_at.to_i,post.id.pad(8),artist+"_"+name,post.file_ext]
+    elsif @mod == "pool" then
+      string = [set.id.pad(5),id.pad(4),post.id.pad(8),artist,name.downcase.gsub(/\s/,"_"),post.file_ext]
+    end
+    string = string.join(".")
+    tries = 0
+    begin
+      post.download("#{name}/#{string}")
+    rescue
+      raise if tries > 4
+      tries += 1
+      puts "#{Time.now.strftime($form)}: #$!"
+      @log.info("#$!")
+      sleep 1
+      retry
+    end
+    File.utime(post.created_at,post.created_at,"#{name}/#{string}")
+    @posts["downloads"] << post.id
+    jposts = @posts.to_json
+    File.open(@set_files+"/#{name}.json","w"){|f|f.print jposts}
+    s = "Downloaded post #{" "*(6-post.id.to_s.length)}#{post.id} from \"#{set.name}\"."
+    puts "#{Time.now.strftime($form)}: #{s}"
+    sleep(rand*@max_wait)
+    @log.info(s)
+  end
+end
+sync = Sync.new
+sync.worker(mod)

data/lib/rubyhexagon.rb

@@ -30,5 +30,5 @@ require "pool"
 
 module E621
   Name = "rubyhexagon"
-  Version = "0.1.1"
+  Version = "0.1.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rubyhexagon
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
   prerelease: 
 platform: ruby
 authors:
@@ -18,21 +18,22 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- lib/examples/add.rb
+- lib/examples/sync.rb
 - lib/config.rb
-- lib/set.rb
-- lib/standard/error.rb
-- lib/standard/http.rb
+- lib/post.rb
+- lib/rubyhexagon.rb
+- lib/pool.rb
+- lib/container.rb
 - lib/standard/string.rb
-- lib/standard/time.rb
+- lib/standard/http.rb
+- lib/standard/error.rb
 - lib/standard/int.rb
 - lib/standard/hash.rb
-- lib/api.rb
-- lib/rubyhexagon.rb
-- lib/pool.rb
+- lib/standard/time.rb
 - lib/search.rb
-- lib/container.rb
-- lib/post.rb
-- lib/examples/add.rb
+- lib/set.rb
+- lib/api.rb
 - bin/e621
 homepage: https://github.com/maxine-red/rubyhexagon
 licenses: