rubyhexagon 0.0.5 → 0.0.6

data/bin/set-sync

@@ -26,6 +26,8 @@ require "logger"
 
 include E621
 
+$0 = "E621 Set-sync"
+$form = "%b %e, %Y %I:%M:%S %p"
 
 def download(post,name,set)
   artist = post.tags.split(/\s+/).map do |t|
@@ -38,12 +40,16 @@ def download(post,name,set)
   end
   string = [post.created_at.to_i,post.id.pad(8),artist+"_"+name,post.file_ext]
   string = string.join(".")
+  tries = 0
   begin
     api = API.new("post")
     post = Post.new(api.post("show",{"id"=>post.id}))
     post.download("#{name}/#{string}")
   rescue
-    puts $!
+    raise if tries > 4
+    tries += 1
+    puts "#{Time.now.strftime($form)}: #$!"
+    @log.info("#$!")
     sleep 1
     retry
   end
@@ -53,173 +59,203 @@ def download(post,name,set)
   set_files = File.expand_path("~/.hexagon/sets")
   File.open(set_files+"/#{name}.json","w"){|f|f.print jposts}
   s = "Downloaded post #{" "*(6-post.id.to_s.length)}#{post.id} from \"#{set["name"]}\"."
-  puts s
+  puts "#{Time.now.strftime($form)}: #{s}"
   @log.info(s)
 end
-max_wait = 2.5
-E621::Config.config = File.expand_path("~/.hexagon/conf.json")
-api = API.new("user")
-@log = Logger.new(File.expand_path(E621::Config.paths["logging"]))
-@log.formatter = proc do |sev,dat,prog,msg|
-  "#{Time.now.strftime("%b %e, %Y %I:%M:%S %p")} [#{sev.pad(5)}] #{msg}#$/"
-end
-@log.level = Logger::INFO
-@log.info("Program #$0 started.")
-uid = api.get("index",{"name"=>api.user}).first["id"]
-tags = File.expand_path("~/.hexagon/tags.json")
-if !File.exist?(tags) then
-  @tags = Array.new
-else
-  File.open(tags) do |f|
-    @tags = f.read.parse
+begin
+  run,running = (File.expand_path("~/.hexagon/sets.run")),false
+  if File.exist?(run) then
+    running = true
+  else
+    File.open(run,"w"){|f|f.print $$}
   end
-end
-@tags = @tags.sort{|k1,k2|k1["id"]<=>k2["id"]}
-net = Net::HTTP.new("e621.net",443)
-net.use_ssl = true
-body,page = [2],1
-until body == Array.new do
-  body = net.get("/tag/index.json?limit=0&order=count&after_id=#{@tags.last["id"]}&page=#{page}").body.parse
-  body = body.map{|x|x["name"] = x["name"].encode("us-ascii", :invalid => :replace, :undef => :replace, :replace => "");x}
-  @tags += body
-  page += 1
-end
-jtags = @tags.to_json
-File.open(tags,"w"){|f|f.print jtags}
-@tags = @tags.reject{|x|x["type"]!=1}.map{|x|x["name"]}
-Dir.chdir(File.expand_path("~/Dropbox/Furry/e621/sets")) do
-  sets = Array.new
-  File.open(File.expand_path("~/.hexagon/sets.json")) do |f|
-    sets = f.read.parse
+  max_wait = 2.5
+  E621::Config.config = File.expand_path("~/.hexagon/conf.json")
+  api = API.new("user")
+  if $stdout.tty? then
+    @log = Logger.new($stdout)
+    @log.level = Logger::DEBUG
+  else
+    @log = Logger.new(File.expand_path(E621::Config.paths["logging"]))
+    @log.level = Logger::INFO
   end
-  sets = sets.sort{|s1,s2|s1["id"]<=>s2["id"]}
-  set_files = File.expand_path("~/.hexagon/sets")
-  api = API.new("set")
-  sets.each do |set|
-    sid, owner, query = set["id"], set["owner"], set["search"]
-    set = api.get("show", {"id"=>sid})
-    posts = set["posts"].map{|post|Post.new(post)}
-    name = set["shortname"]
-    @log.info("Fetching set #{set["name"]}")
-    if !File.exists?(set_files+"/#{name}.json") then
-      File.open(set_files+"/#{name}.json","w"){|f|f.print "{\"downloads\":[]}"}
-    end 
-    File.open(set_files+"/#{name}.json"){|f|@posts = f.read.parse}
-    Dir.mkdir(name) unless File.exist?(name)
-    posts.each do |post|
-      next if @posts["downloads"].include?(post.id)
-      download(post,name,set)
-    end
-    next if owner != uid
-    Search.new("fav:maxine_red #{query} order:id".split(" ")).each_post do |post|
-      next if @posts["downloads"].include?(post.id)
-      download(post,name,set)
-      sleep(rand*max_wait)
+  @log.formatter = proc do |sev,dat,prog,msg|
+    "#{Time.now.strftime("%b %e, %Y %I:%M:%S %p")} [#{sev.pad(5)}] #{msg}#$/"
+  end
+  @log.info("Program #$0 started.")
+  uid = api.get("index",{"name"=>api.user}).first["id"]
+  tags = File.expand_path("~/.hexagon/tags.json")
+  if !File.exist?(tags) then
+    @tags = Array.new
+  else
+    File.open(tags) do |f|
+      @tags = f.read.parse
     end
   end
-  sets.each do |set|
-    sid, owner, query = set["id"], set["owner"], set["search"]
-    next if owner != uid
-    set = api.get("show", {"id"=>sid})
-    posts = set["posts"].map{|post|Post.new(post).id}
-    name = set["shortname"]
-    local = Dir["#{name}/*"].reject{|x|x.match(/db$/)}.map{|x|x.split(".")[1].to_i}.sort.uniq
-    (posts-local).each do |id|
-      post = Post.new({"id"=>id})
-      f = post.remove_from_set(sid)
-      if f["success"] then
-        s = "Removed Post #{" "*(6-post.id.to_s.length)}#{post.id} from \"#{set["name"]}\""
-        puts "\e[1;33m#{s}\e[0m"
-        @log.info(s)
-      else
-        s = "#{f["reason"].to_s.gsub(/<.+?>/,"")}."
-        puts "\e[1;31m#{s}\e[0m"
-        @log.info(s)
+  @tags = @tags.sort{|k1,k2|k1["id"]<=>k2["id"]}
+  net = Net::HTTP.new("e621.net",443)
+  net.use_ssl = true
+  body,page = [2],1
+  until body == Array.new do
+    body = net.get("/tag/index.json?limit=0&order=count&after_id=#{@tags.last["id"]}&page=#{page}").body.parse
+    body = body.map{|x|x["name"] = x["name"].encode("us-ascii", :invalid => :replace, :undef => :replace, :replace => "");x}
+    @tags += body
+    page += 1
+  end
+  jtags = @tags.to_json
+  File.open(tags,"w"){|f|f.print jtags}
+  @tags = @tags.reject{|x|x["type"]!=1}.map{|x|x["name"]}
+  Dir.chdir(File.expand_path("~/Dropbox/Furry/e621/sets")) do
+    sets = Array.new
+    File.open(File.expand_path("~/.hexagon/sets.json")) do |f|
+      sets = f.read.parse
+    end
+    sets = sets.sort{|s1,s2|s1["id"]<=>s2["id"]}
+    set_files = File.expand_path("~/.hexagon/sets")
+    api = API.new("set")
+    sets.each do |set|
+      sid, owner, query = set["id"], set["owner"], set["search"]
+      set = api.get("show", {"id"=>sid})
+      @log.info("Fetching set #{set["name"]}")
+      posts = set["posts"].map{|post|Post.new(post)}
+      name = set["shortname"]
+      if !File.exists?(set_files+"/#{name}.json") then
+        File.open(set_files+"/#{name}.json","w"){|f|f.print "{\"downloads\":[]}"}
+      end 
+      File.open(set_files+"/#{name}.json"){|f|@posts = f.read.parse}
+      Dir.mkdir(name) unless File.exist?(name)
+      posts.each do |post|
+        next if @posts["downloads"].include?(post.id)
+        download(post,name,set)
+      end
+      next if owner != uid
+      Search.new("fav:maxine_red #{query} order:id".split(" ")).each_post do |post|
+        next if @posts["downloads"].include?(post.id)
+        download(post,name,set)
+        sleep(rand*max_wait)
       end
-      sleep(rand*max_wait)
     end
-    (local-posts).each do |id|
-      next if id == 0
-      post = Post.new({"id"=>id})
-      f = post.add_to_set(sid)
-      if f["success"] then
-        s = "Added Post #{" "*(6-post.id.to_s.length)}#{post.id} to \"#{set["name"]}\"."
-        puts s
-        @log.info(s)
-      else
-        s = "#{f["reason"].gsub(/<.+?>/,"")}. Removing local file."
-        puts s
-        @log.info(s)
-        File.unlink(Dir["#{set["shortname"]}/*.#{"0"*(8-post.id.to_s.length)}#{post.id}.*"].first)
+    sets.each do |set|
+      sid, owner, query = set["id"], set["owner"], set["search"]
+      next if owner != uid
+      set = api.get("show", {"id"=>sid})
+      posts = set["posts"].map{|post|Post.new(post).id}
+      name = set["shortname"]
+      local = Dir["#{name}/*"].reject{|x|x.match(/db$/)}.map{|x|x.split(".")[1].to_i}.sort.uniq
+      (posts-local).each do |id|
+        post = Post.new({"id"=>id})
+        f = post.remove_from_set(sid)
+        if f["success"] then
+          s = "Removed Post #{" "*(6-post.id.to_s.length)}#{post.id} from \"#{set["name"]}\""
+          puts "#{Time.now.strftime($form)}: \e[1;33m#{s}\e[0m"
+          @log.info(s)
+        else
+          s = "#{f["reason"].to_s.gsub(/<.+?>/,"")}."
+          puts "#{Time.now.strftime($form)}: \e[1;31m#{s}\e[0m"
+          @log.info(s)
+        end
+        sleep(rand*max_wait)
+      end
+      (local-posts).each do |id|
+        next if id == 0
+        post = Post.new({"id"=>id})
+        f = post.add_to_set(sid)
+        if f["success"] then
+          s = "Added Post #{" "*(6-post.id.to_s.length)}#{post.id} to \"#{set["name"]}\"."
+          puts "#{Time.now.strftime($form)}: #{s}"
+          @log.info(s)
+        else
+          s = "#{f["reason"].gsub(/<.+?>/,"")}. Removing local file."
+          puts "#{Time.now.strftime($form)}: #{s}"
+          @log.info(s)
+          File.unlink(Dir["#{set["shortname"]}/*.#{"0"*(8-post.id.to_s.length)}#{post.id}.*"].first)
+        end
+        sleep(rand*max_wait)
       end
-      sleep(rand*max_wait)
     end
   end
-end
-Dir.chdir(File.expand_path("~/Dropbox/Furry/e621/pools")) do
-  pools = Array.new
-  File.open(File.expand_path("~/.hexagon/pools.json")) do |f|
-    pools = f.read.parse.sort.uniq
-  end
-  pool_files = File.expand_path("~/.hexagon/pools")
-  api = API.new("pool")
-  pools.each do |pool|
-    spool = {"updated_at"=>0, "posts"=>[]}
-    npool = api.get("show",{"id"=>pool})
-    pool = Pool.new(npool)
-    begin
-      pfile = pool_files+"/#{pool.id.pad(5)}.json"
-    rescue
-      @log.error("Error occured: #{npool.inspect}")
-      raise
-    end
-    if !File.exists?(pfile) then
-      File.open(pfile,"w"){|f|f.print spool.to_json}
-      spool = Pool.new(spool)
-    else 
-      File.open(pfile){|f|spool = Pool.new(f.read.parse)}
-    end
-    posts = Array.new
-    next if pool.updated_at.to_i <= spool.updated_at.to_i
-    (pool.post_count/24.0).ceil.times do |page|
-      posts += pool.posts.map{|post|Post.new(post)}
-      pool = Pool.new(api.get("show",{"id"=>pool.id,"page"=>page+2}))
+  Dir.chdir(File.expand_path("~/Dropbox/Furry/e621/pools")) do
+    pools = Array.new
+    File.open(File.expand_path("~/.hexagon/pools.json")) do |f|
+      pools = f.read.parse.sort.uniq
     end
-    pool.name = pool.name.gsub("_"," ").encode("us-ascii", :invalid => :replace, :undef => :replace, :replace => "")
-    name = pool.name.gsub(/[^0-9, ,_,a-z,\-]/i,"").sub(/\s+$/,"")
-    Dir.mkdir(name) unless File.exist?(name)
-    @log.info("Fetching pool #{pool.name}")
-    posts.each_with_index do |post,id|
-      id = id.succ
-      next if spool.posts.include?(post.id)
-      artist = post.tags.split(/\s+/).map do |t|
-        @tags[@tags.index(t)] if @tags.index(t)
-      end
+    pool_files = File.expand_path("~/.hexagon/pools")
+    api = API.new("pool")
+    pools = pools-(1..600).to_a
+    pools.each do |id|
+      spool = {"updated_at"=>0, "posts"=>[]}
+      npool = api.get("show",{"id"=>id})
       begin
-        artist = artist.compact.first.sub("_(artist)","")
+        pool = Pool.new(npool)
+        @log.debug("Fetched pool \"#{pool.name.gsub("_"," ")}\"")
       rescue
-        artist = "unknown"
+        #$stderr.puts "#{pool.inspect}"
+        puts "#{Time.now.strftime($form)}: #{id} raised an error!"
+        @log.debug("Pool object: #{pool.inspect}")
+        next
       end
-      string = [pool.id.pad(5),id.pad(4),post.id.pad(8),artist,name.downcase.gsub(/\s/,"_"),post.file_ext]
-      string = string.join(".")
       begin
-        post.download("#{name}/#{string}")
+        pfile = pool_files+"/#{pool.id.pad(5)}.json"
       rescue
-        puts $!
-        sleep 1
-        retry
+        @log.error("Error occured: #{npool.inspect}")
+        raise
+      end
+      if !File.exists?(pfile) then
+        File.open(pfile,"w"){|f|f.print spool.to_json}
+        spool = Pool.new(spool)
+      else 
+        File.open(pfile){|f|spool = Pool.new(f.read.parse)}
+      end
+      posts = Array.new
+      @log.debug("Skipping pool \"#{pool.name.gsub("_"," ")}\" (#{pool.id.pad(4)})")
+      next if pool.updated_at.to_i <= spool.updated_at.to_i
+      (pool.post_count/24.0).ceil.times do |page|
+        posts += pool.posts.map{|post|Post.new(post)}
+        pool = Pool.new(api.get("show",{"id"=>pool.id,"page"=>page+2}))
+      end
+      pool.name = "Love_Can_Be_Different" if pool.name == "Love_can_be_different"
+      pool.name = pool.name.gsub("_"," ").encode("us-ascii", :invalid => :replace, :undef => :replace, :replace => "")
+      name = pool.name.gsub(/[^0-9, ,_,a-z,\-]/i,"").sub(/\s+$/,"")
+      Dir.mkdir(name) unless File.exist?(name)
+      @log.info("Fetching pool #{pool.name}")
+      posts.each_with_index do |post,id|
+        id = id.succ
+        next if spool.posts.include?(post.id)
+        artist = post.tags.split(/\s+/).map do |t|
+          @tags[@tags.index(t)] if @tags.index(t)
+        end
+        begin
+          artist = artist.compact.first.sub("_(artist)","")
+        rescue
+          artist = "unknown"
+        end
+        string = [pool.id.pad(5),id.pad(4),post.id.pad(8),artist,name.downcase.gsub(/\s/,"_"),post.file_ext]
+        string = string.join(".")
+        tries = 0
+        begin
+          post.download("#{name}/#{string}")
+        rescue
+          raise if tries > 4
+          tries += 1
+          puts "#{Time.now.strftime($form)}: #$!"
+          @log.info("#$!")
+          sleep 1
+          retry
+        end
+        File.utime(post.created_at,post.created_at,"#{name}/#{string}")
+        spool.posts << post.id
+        jposts = spool.to_json
+        File.open(pfile,"w"){|f|f.print jposts}
+        s = "Downloaded post #{post.id.pad(6," ")} (#{id.pad(posts.length.to_s.length," ")}/#{posts.length}) from \"#{pool.name}\"."
+        puts "#{Time.now.strftime($form)}: #{s}"
+        @log.info(s)
+        sleep(rand*max_wait)
       end
-      File.utime(post.created_at,post.created_at,"#{name}/#{string}")
-      spool.posts << post.id
+      spool.updated_at = pool.updated_at
       jposts = spool.to_json
       File.open(pfile,"w"){|f|f.print jposts}
-      s = "Downloaded post #{post.id.pad(6," ")} (#{id.pad(posts.length.to_s.length," ")}/#{posts.length}) from \"#{pool.name}\"."
-      puts s
-      @log.info(s)
-      sleep(rand*max_wait)
     end
-    spool.updated_at = pool.updated_at
-    jposts = spool.to_json
-    File.open(pfile,"w"){|f|f.print jposts}
   end
+ensure
+  File.unlink(run) unless running
 end

data/lib/examples/add.rb

@@ -0,0 +1,57 @@
+=begin
+  Copyright 2014 Maxine Red <maxine_red1@yahoo.com>
+
+  This file is part of rubyhexagon.
+
+  rubyhexagon is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  rubyhexagon is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with rubyhexagon.  If not, see <http://www.gnu.org/licenses/>.
+=end
+
+require "json"
+
+include E621
+
+mod = ARGV.shift
+
+$0 = "E621 #{mod}"
+
+E621::Config.config = File.expand_path("~/.hexagon/conf.json")
+Dir.chdir(File.expand_path(E621::Config.paths["home"])) do
+  pools, api = Array.new, API.new(mod)
+  File.open(E621::Config.paths["j#{mod}s"]) do |f|
+    pools = JSON.parser.new(f.read).parse
+  end
+  puts "Old count: #{pools.length}"
+  candidates = ARGV.map{|x|x.to_i}.reject{|x|x==0}
+  puts "Adding candidates: #{candidates.length}"
+  new = ARGV.map{|x|x.to_i}.reject{|x|x==0||pools.include?(x)}
+  new.each do |id|
+    pool = api.post("show",{"id"=>id})
+    if pool["id"] && mod == "pool" then
+      pools << id
+    elsif pool["id"] && mod == "set" then
+      pools << {"id"=>pool["id"],"owner"=>pool["user-id"]}
+    end
+  end
+  puts "Adding: #{new.length}"
+  if mod == "pool" then
+    pools = pools.sort.uniq
+  elsif mod == "set" then
+    pools = pools.sort{|k1,k2|k1["id"]<=>k2["id"]}.uniq
+  end
+  puts "New count: #{pools.length}"
+  pools = pools.to_json
+  File.open(E621::Config.paths["j#{mod}s"]) do |f|
+    f.print pools
+  end
+end

data/lib/rubyhexagon.rb

@@ -30,5 +30,5 @@ require "pool"
 
 module E621
   Name = "rubyhexagon"
-  Version = "0.0.5"
+  Version = "0.0.6"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rubyhexagon
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-15 00:00:00.000000000 Z
+date: 2015-03-01 00:00:00.000000000 Z
 dependencies: []
 description: Rubyhexagon provides Ruby bindings for the e621 [dot] net API.
 email: maxine_red1@yahoo.com
@@ -22,6 +22,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/pool.rb
+- lib/examples/add.rb
 - lib/rubyhexagon.rb
 - lib/standard/hash.rb
 - lib/standard/http.rb