rubyhexagon 0.0.1

data/bin/e621_search

@@ -0,0 +1,116 @@
+#!/usr/bin/env ruby
+=begin
+  Copyright 2014 Maxine Red <maxine_red1@yahoo.com>
+
+  This file is part of rubyhexagon.
+
+  rubyhexagon is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  rubyhexagon is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with rubyhexagon.  If not, see <http://www.gnu.org/licenses/>.
+=end
+
+file = File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__
+$:.unshift(File.expand_path(File.dirname(file)+"/../lib"))
+
+require "rubyhexagon"
+require "readline"
+require "socket"
+require "base64"
+
+include E621
+
+E621::Config.config = File.expand_path("~/.hexagon/conf.json")
+tags = Array.new
+File.open(File.expand_path(E621::Config.paths["tags"])) do |f|
+  tags = f.read.parse
+end
+Readline.completion_proc = proc do |s|
+  add = s[0] if s.match(/^\-|\~/)
+  add = s[0,2] if s.match(/^\*_/)
+  s.sub!(/^\-|\~|(\*_)/,"")
+  s = Regexp.escape(s)
+  words   = Array.new
+  words  += Dir["*"].reject{|d|!File.directory?(d)}.map{|d|"name:#{File.basename(d)}"}
+  words  += ["rating:s","rating:q","rating:e"]
+  words  += ["id:","md5:","description:","desc:","pool:","set:","width:",\
+    "height:","score:","favcount:","views:","ratio:","parent:"]
+  words  += ["type:jpg","type:png","type:gif","type:swf"]
+  bools   = ["hassource","hasdescription","ischild","idparent","inpool"]
+  words  += bools.map{|b|["#{b}:true","#{b}:false"]}.flatten
+  # Make this a one dimensional array again!
+  words  += ["order:random"]
+  orders  = ["id","score","views","set","favcount","tagcount","comments",\
+    "mpixels","filesize","ratio","desclength"]
+  words  += orders.map{|b|["#{b}_asc","#{b}_desc"]}.flatten.map{|b|"order:#{b}"}
+  # Here we make all orders in both directions and keep it as just an one
+  # dimensional array.
+  words  += tags.map{|t|t["name"]}
+  words = words.grep(/^#{s}/)
+  words.map!{|w|"#{add}#{w}"}
+  s = "#{add}#{s}"
+  words
+end
+Readline.completer_word_break_characters  = " "
+Readline.completion_append_character      = " "
+history = File.open(File.expand_path(E621::Config.paths["history"]),"a+")
+history.read.split($/).each do |l|
+  Readline::HISTORY << l
+end
+
+def net_send(query)
+  socket = TCPSocket.new(E621::Config.config["server"],5621)
+  socket.puts query.to_json
+  res = socket.read.parse
+  if res["error"] then
+    $stderr.puts "Server error: #{res["message"]}"
+    abort
+  end
+  socket.close
+  return res
+end
+
+api = API.new("user")
+prompt = "#{api.user}@e621.net/posts> "
+Dir.chdir(File.expand_path(E621::Config.paths["posts"])) do
+  while buff = Readline.readline(prompt.bold("yellow"), false) do
+    next if buff == String.new
+    if !(buff == String.new || buff == Readline::HISTORY.to_a.last) then
+      Readline::HISTORY << buff
+      history.puts buff
+    end
+    buff = buff.split(/\s+/)
+    name = buff.shift.sub("name:","")
+    query = {"action"=>"search","name"=>name,"query"=>buff}
+    rid = net_send(query)["rid"]
+    posts,page,num = Array.new,{"posts"=>[2]},1
+    while page["posts"] != Array.new do
+      print "Fetching page #{num.pad(3," ")} (#{posts.length.pad(6," ")} posts fetched).\r"
+      page = net_send({"action"=>"page","page"=>num,"rid"=>rid})
+      posts += page["posts"] if page["posts"]
+      num += 1
+    end
+    puts
+    Dir.mkdir(name) unless File.exist?(name)
+    Dir.chdir(name) do
+      count = 1
+      posts.reverse.each do |post|
+        print "Downloading post #{count.pad(6," ")} of #{posts.length.pad(6, " ")}.\r"
+        query = {"action"=>"show", "post"=>post}
+        res = net_send(query)
+        File.open(res["name"],"w"){|f|f.print Base64.decode64(res["data"])}
+        count += 1
+      end
+    end
+    puts
+  end
+end
+puts

data/bin/e621_server

@@ -0,0 +1,158 @@
+#!/usr/bin/env ruby
+=begin
+  Copyright 2014 Maxine Red <maxine_red1@yahoo.com>
+
+  This file is part of rubyhexagon.
+
+  rubyhexagon is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  rubyhexagon is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with rubyhexagon.  If not, see <http://www.gnu.org/licenses/>.
+=end
+
+$:.unshift(File.expand_path(File.dirname(__FILE__)+"/../lib"))
+
+require "rubyhexagon"
+require "thread"
+require "logger"
+require "socket"
+require "digest"
+require "base64"
+
+include E621
+
+Thread.abort_on_exception = true
+
+E621::Config.config = File.expand_path("~/.hexagon/conf.json")
+api = API.new
+tasks = Hash.new
+mt = Mutex.new
+server = TCPServer.new(5621)
+Process.daemon
+@log = Logger.new(File.expand_path(E621::Config.paths["logging"]))
+#@log = Logger.new(STDOUT)
+#@log.level = Logger::DEBUG
+=begin
+10.times do
+  Thread.new do 
+    loop do
+      name,tclient,max,post = queue.pop
+      next unless post
+      Dir.mkdir(name) unless File.exist?(name)
+      post.download("#{name}/#{"0"*(7-post.id.to_s.length)}#{post.id}.#{post.md5}.#{post.file_ext}")
+      mt.synchronize do
+        unless tasks.has_key?(name) then
+          tasks.store(name,1)
+        else
+          tasks[name] += 1
+        end
+        count = tasks[name]
+        message = "Got #{" "*(max.to_s.length+2-count.to_s.length)}#{count}/#{max}"+" "*16
+        @log.debug message
+        tclient.puts message
+        tclient.close if count >= max
+      end
+    end
+  end
+end
+=end
+
+@log.formatter = proc do |sev,dat,prog,msg|
+  "#{Time.now.strftime("%b %e, %Y %I:%M:%S %p")} #{msg}#$/"
+end
+@log.level = Logger::INFO
+@log.info("New server instance started.")
+Dir.chdir(File.expand_path("~/.hexagon")) do
+  Thread.new do
+    loop do
+      files,size,tm = Dir["cache/**/*"].reject{|x|!File.file?(x)},0,Array.new
+      files.each do |f|
+        size += File.size(f)
+      end
+      if size > 30*2**30 then
+        files.each do |f|
+          tm << {"file"=>f,"time"=>File.atime(f),"size"=>File.size(f)}
+        end
+        tm = tm.sort{|f1,f2|f1["time"]<=>f2["time"]}
+        while size > 30*2**30 do
+          unlink = tm.shift
+          size -= unlink["size"]
+          File.unlink(unlink["file"])
+        end
+      end
+      sleep 3600
+    end
+  end
+  begin
+    Dir["searches/*"].each{|f|File.unlink(f)}
+    loop do
+      client = server.accept
+      Thread.new(client) do |c|
+        begin
+          query = c.gets.parse
+        rescue
+          @log.error("[#{c.remote_address.ip_address}]: Error #$!.")
+          next
+        end
+        case query["action"]
+        when "search" then
+          name,query = query["name"],query["query"]
+          @log.info("[#{c.remote_address.ip_address}]: Search init #{name}/#{query.join("+")}.")
+          Dir.mkdir("searches") unless File.exist?("searches")
+          rid = Digest::SHA2.hexdigest(name+Time.now.strftime("%s"))
+          File.open("searches/#{rid}.json","w") do |f|
+            search = {"name"=>name,"query"=>query}
+            f.print search.to_json
+          end
+            rid = {"rid"=>rid}.to_json
+            c.print rid
+        when "page" then
+          page, rid = query["page"],query["rid"]
+          File.open("searches/#{rid}.json"){|f|query=f.read.parse}
+          @log.info("[#{c.remote_address.ip_address}]: Search page #{query["name"]}/#{page}.")
+          search = Search.new(query["query"])
+          search.page = page
+          posts = {"posts"=>search.posts}.to_json
+          c.print posts
+        when "show" then
+          post = Post.new(query["post"])
+          Dir.mkdir("cache") unless File.exist?("cache")
+          Dir.mkdir("cache/#{post.md5[0,2]}") unless File.exist?("cache/#{post.md5[0,2]}")
+          body = String.new
+          if File.exist?("cache/#{post.md5[0,2]}/#{post.md5}") then
+            cached = " (cached)"
+            File.open("cache/#{post.md5[0,2]}/#{post.md5}") do |f|
+              body = f.read
+            end
+          else
+            cached = ""
+            body = post.download_data
+            File.open("cache/#{post.md5[0,2]}/#{post.md5}","w") do |f|
+              f.print body
+            end
+          end
+          @log.info("[#{c.remote_address.ip_address}]: Show post #{query["post"]["id"].pad(6," ")}#{cached}.")
+          file = {"name"=>"#{post.id.pad(8)}.#{post.md5}.#{post.file_ext}","data"=>Base64.encode64(body)}
+          c.print file.to_json
+        end
+        c.close
+      end
+    end
+  rescue Errno
+     @log.error("Error: #$!!")
+  rescue => e
+    p e.class
+    @log.fatal("Fatal Error: #$!!")
+    raise
+  ensure
+    @log.info("Server instance terminated.")
+  end
+end

data/bin/fav-sync

@@ -0,0 +1,82 @@
+#!/usr/bin/env ruby
+=begin
+  Copyright 2014 Maxine Red <maxine_red1@yahoo.com>
+
+  This file is part of rubyhexagon.
+
+  rubyhexagon is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  rubyhexagon is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with rubyhexagon.  If not, see <http://www.gnu.org/licenses/>.
+=end
+
+loader = File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__
+$:.unshift(File.expand_path(File.dirname(loader))+"/../lib")
+
+require "rubyhexagon"
+
+include E621
+
+max_wait = 5
+E621::Config.config = File.expand_path("~/.hexagon/conf.json")
+api = API.new("user")
+uid = api.get("index",{"name"=>api.user}).first["id"]
+Dir.chdir(File.expand_path("~/pictures/e621/favorites"))
+remote = Array.new
+Search.new("fav:maxine_red order:id".split(" ")).each_post do |post|
+  string = ["0"*(7-post.id.to_s.length)+post.id.to_s,post.md5,post.file_ext].join(".")
+  remote << Array.new
+  if !File.exist?(string) then
+    post.download(string)
+  else
+    md5 = String.new
+    File.open(string){|f|md5 = Digest::MD5.hexdigest(f.read)}
+    next if md5 == post.md5
+    post.download(string)
+  end
+  puts "Downloaded post #{" "*(7-post.id.to_s.length)}#{post.id}."
+  sleep(rand*max_wait)
+end
+local = Dir["*"].reject{|x|x.match(/db$/)}.map{|x|x.sub(/\..+/,"").to_i}.sort.uniq
+(remote-local).each do |id|
+  post = Post.new({"id"=>id})
+  f = post.unfavorite
+  if f["success"] then
+    print "\e[1;33mRemoved favorite on post #{" "*(7-post.id.to_s.length)}#{post.id}.\e[0m "
+  else
+    print "\e[1;31m#{f["reason"].to_s.gsub(/<.+?>/,"")}.\e[0m "
+  end
+  f = post.vote(1)
+  if f["success"] then
+    puts "\e[1;33mScored post #{" "*(7-post.id.to_s.length)}#{post.id} down.\e[0m"
+  else
+    puts "\e[1;31m#{f["reason"].to_s.gsub(/<.+?>/,"")}.\e[0m"
+  end
+  sleep(rand*max_wait)
+end
+(local-remote).each do |id|
+  next if id == 0
+  post = Post.new({"id"=>id})
+  f = post.favorite
+  if f["success"] then
+    print "\e[1;32mFavorited #{" "*(7-post.id.to_s.length)}#{post.id}.\e[0m "
+  else
+    print "\e[1;31m#{f["reason"].gsub(/<.+?>/,"")}.\e[0m "
+    #File.unlink(Dir["#{set["shortname"]}/*.#{"0"*(8-post.id.to_s.length)}#{post.id}.*"].first)
+  end
+  f = post.vote(1)
+  if f["success"] then
+    puts "\e[1;32mScored post #{" "*(7-post.id.to_s.length)}#{post.id} up.\e[0m"
+  else
+    puts "\e[1;31m#{f["reason"].to_s.gsub(/<.+?>/,"")}.\e[0m"
+  end
+  sleep(rand*max_wait)
+end

data/bin/set-sync

@@ -0,0 +1,225 @@
+#!/usr/bin/env ruby
+=begin
+  Copyright 2014 Maxine Red <maxine_red1@yahoo.com>
+
+  This file is part of rubyhexagon.
+
+  rubyhexagon is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  rubyhexagon is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with rubyhexagon.  If not, see <http://www.gnu.org/licenses/>.
+=end
+
+loader = File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__
+$:.unshift(File.expand_path(File.dirname(loader))+"/../lib")
+
+require "rubyhexagon"
+require "logger"
+
+include E621
+
+
+def download(post,name,set)
+  artist = post.tags.split(/\s+/).map do |t|
+    @tags[@tags.index(t)] if @tags.index(t)
+  end
+  begin
+    artist = artist.compact.first.sub("_(artist)","")
+  rescue
+    artist = "unknown"
+  end
+  string = [post.created_at.to_i,post.id.pad(8),artist+"_"+name,post.file_ext]
+  string = string.join(".")
+  begin
+    api = API.new("post")
+    post = Post.new(api.post("show",{"id"=>post.id}))
+    post.download("#{name}/#{string}")
+  rescue
+    puts $!
+    sleep 1
+    retry
+  end
+  File.utime(post.created_at,post.created_at,"#{name}/#{string}")
+  @posts["downloads"] << post.id
+  jposts = @posts.to_json
+  set_files = File.expand_path("~/.hexagon/sets")
+  File.open(set_files+"/#{name}.json","w"){|f|f.print jposts}
+  s = "Downloaded post #{" "*(6-post.id.to_s.length)}#{post.id} from \"#{set["name"]}\"."
+  puts s
+  @log.info(s)
+end
+max_wait = 2.5
+E621::Config.config = File.expand_path("~/.hexagon/conf.json")
+api = API.new("user")
+@log = Logger.new(File.expand_path(E621::Config.paths["logging"]))
+@log.formatter = proc do |sev,dat,prog,msg|
+  "#{Time.now.strftime("%b %e, %Y %I:%M:%S %p")}: #{msg}#$/"
+end
+@log.level = Logger::INFO
+@log.info("Program #$0 started.")
+uid = api.get("index",{"name"=>api.user}).first["id"]
+tags = File.expand_path("~/.hexagon/tags.json")
+if !File.exist?(tags) then
+  @tags = Array.new
+else
+  File.open(tags) do |f|
+    @tags = f.read.parse
+  end
+end
+@tags = @tags.sort{|k1,k2|k1["id"]<=>k2["id"]}
+net = Net::HTTP.new("e621.net",443)
+net.use_ssl = true
+body,page = [2],1
+until body == Array.new do
+  body = net.get("/tag/index.json?limit=0&order=count&after_id=#{@tags.last["id"]}&page=#{page}").body.parse
+  body = body.map{|x|x["name"] = x["name"].encode("us-ascii", :invalid => :replace, :undef => :replace, :replace => "");x}
+  @tags += body
+  page += 1
+end
+jtags = @tags.to_json
+File.open(tags,"w"){|f|f.print jtags}
+@tags = @tags.reject{|x|x["type"]!=1}.map{|x|x["name"]}
+Dir.chdir(File.expand_path("~/Dropbox/Furry/e621/sets")) do
+  sets = Array.new
+  File.open(File.expand_path("~/.hexagon/sets.json")) do |f|
+    sets = f.read.parse
+  end
+  sets = sets.sort{|s1,s2|s1["id"]<=>s2["id"]}
+  set_files = File.expand_path("~/.hexagon/sets")
+  api = API.new("set")
+  sets.each do |set|
+    sid, owner, query = set["id"], set["owner"], set["search"]
+    set = api.get("show", {"id"=>sid})
+    posts = set["posts"].map{|post|Post.new(post)}
+    name = set["shortname"]
+    @log.info("Fetching set #{set["name"]}")
+    if !File.exists?(set_files+"/#{name}.json") then
+      File.open(set_files+"/#{name}.json","w"){|f|f.print "{\"downloads\":[]}"}
+    end 
+    File.open(set_files+"/#{name}.json"){|f|@posts = f.read.parse}
+    Dir.mkdir(name) unless File.exist?(name)
+    posts.each do |post|
+      next if @posts["downloads"].include?(post.id)
+      download(post,name,set)
+    end
+    next if owner != uid
+    Search.new("fav:maxine_red #{query} order:id".split(" ")).each_post do |post|
+      next if @posts["downloads"].include?(post.id)
+      download(post,name,set)
+      sleep(rand*max_wait)
+    end
+  end
+  sets.each do |set|
+    sid, owner, query = set["id"], set["owner"], set["search"]
+    next if owner != uid
+    set = api.get("show", {"id"=>sid})
+    posts = set["posts"].map{|post|Post.new(post).id}
+    name = set["shortname"]
+    local = Dir["#{name}/*"].reject{|x|x.match(/db$/)}.map{|x|x.split(".")[1].to_i}.sort.uniq
+    (posts-local).each do |id|
+      post = Post.new({"id"=>id})
+      f = post.remove_from_set(sid)
+      if f["success"] then
+        s = "Removed Post #{" "*(6-post.id.to_s.length)}#{post.id} from \"#{set["name"]}\""
+        puts "\e[1;33m#{s}\e[0m"
+        @log.info(s)
+      else
+        s = "#{f["reason"].to_s.gsub(/<.+?>/,"")}."
+        puts "\e[1;31m#{s}\e[0m"
+        @log.info(s)
+      end
+      sleep(rand*max_wait)
+    end
+    (local-posts).each do |id|
+      next if id == 0
+      post = Post.new({"id"=>id})
+      f = post.add_to_set(sid)
+      if f["success"] then
+        s = "Added Post #{" "*(6-post.id.to_s.length)}#{post.id} to \"#{set["name"]}\"."
+        puts s
+        @log.info(s)
+      else
+        s = "#{f["reason"].gsub(/<.+?>/,"")}. Removing local file."
+        puts s
+        @log.info(s)
+        File.unlink(Dir["#{set["shortname"]}/*.#{"0"*(8-post.id.to_s.length)}#{post.id}.*"].first)
+      end
+      sleep(rand*max_wait)
+    end
+  end
+end
+Dir.chdir(File.expand_path("~/Dropbox/Furry/e621/pools")) do
+  pools = Array.new
+  File.open(File.expand_path("~/.hexagon/pools.json")) do |f|
+    pools = f.read.parse.sort.uniq
+  end
+  pool_files = File.expand_path("~/.hexagon/pools")
+  api = API.new("pool")
+  pools.each do |pool|
+    spool = {"updated_at"=>0, "posts"=>[]}
+    npool = api.get("show",{"id"=>pool})
+    pool = Pool.new(npool)
+    begin
+      pfile = pool_files+"/#{pool.id.pad(5)}.json"
+    rescue
+      @log.error("Error occured: #{npool.inspect}")
+      raise
+    end
+    if !File.exists?(pfile) then
+      File.open(pfile,"w"){|f|f.print spool.to_json}
+      spool = Pool.new(spool)
+    else 
+      File.open(pfile){|f|spool = Pool.new(f.read.parse)}
+    end
+    posts = Array.new
+    next if pool.updated_at.to_i <= spool.updated_at.to_i
+    (pool.post_count/24.0).ceil.times do |page|
+      posts += pool.posts.map{|post|Post.new(post)}
+      pool = Pool.new(api.get("show",{"id"=>pool.id,"page"=>page+2}))
+    end
+    pool.name = pool.name.gsub("_"," ").encode("us-ascii", :invalid => :replace, :undef => :replace, :replace => "")
+    name = pool.name.gsub(/[^0-9, ,_,a-z,\-]/i,"").sub(/\s+$/,"")
+    Dir.mkdir(name) unless File.exist?(name)
+    @log.info("Fetching pool #{pool.name}")
+    posts.each_with_index do |post,id|
+      id = id.succ
+      next if spool.posts.include?(post.id)
+      artist = post.tags.split(/\s+/).map do |t|
+        @tags[@tags.index(t)] if @tags.index(t)
+      end
+      begin
+        artist = artist.compact.first.sub("_(artist)","")
+      rescue
+        artist = "unknown"
+      end
+      string = [pool.id.pad(5),id.pad(4),post.id.pad(8),artist,name.downcase.gsub(/\s/,"_"),post.file_ext]
+      string = string.join(".")
+      begin
+        post.download("#{name}/#{string}")
+      rescue
+        puts $!
+        sleep 1
+        retry
+      end
+      File.utime(post.created_at,post.created_at,"#{name}/#{string}")
+      spool.posts << post.id
+      jposts = spool.to_json
+      File.open(pfile,"w"){|f|f.print jposts}
+      s = "Downloaded post #{post.id.pad(6," ")} (#{id.pad(posts.length.to_s.length," ")}/#{posts.length}) from \"#{pool.name}\"."
+      puts s
+      @log.info(s)
+      sleep(rand*max_wait)
+    end
+    spool.updated_at = pool.updated_at
+    jposts = spool.to_json
+    File.open(pfile,"w"){|f|f.print jposts}
+  end
+end