saber 1.1.1 → 1.2.0

data/bin/saber.bb

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+SABER_TRACKER=bb saber $*

data/bin/saber.stp

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+SABER_TRACKER=stp saber $*

data/lib/saber.rb

@@ -3,6 +3,8 @@ require "pa"
 require "optimism"
 require "active_support/core_ext/module/attribute_accessors"
 require "active_support/concern"
+require "active_support/core_ext/numeric/bytes"
+require "saber/core_ext"
 
 module Saber
   autoload :VERSION, "saber/version"
@@ -12,11 +14,14 @@ module Saber
   autoload :Fetcher, "saber/fetcher"
   autoload :AutoFetcher, "saber/autofetcher"
   autoload :Tracker, "saber/tracker"
+  autoload :Tracker2, "saber/tracker2"
+  autoload :Book, "saber/book"
 
   Error = Class.new Exception
   FatalError = Class.new Exception
   Rc = Optimism.require "saber/rc", "~/.saberrc"
 
+
   class << self
     attr_accessor :ui
 

data/lib/saber/autofetcher/server.rb

@@ -65,7 +65,7 @@ module Saber
 
       # ["filea", "foo/filea", "foo/fileb"]
       def build_files(*names)
-        Pa.ls2_r(*names, :base_dir => Rc.p.remote_download, :file => true, :include => true) { |p,abs| not Pa.directory?(abs) }
+        Pa.ls2_r(*names, :base_dir => Rc.fetch.remote_dir, :file => true, :include => true) { |p,abs| not Pa.directory?(abs) }
       end
     end
   end

data/lib/saber/book.rb

@@ -0,0 +1,36 @@
+require "faraday"
+require "faraday_middleware"
+
+module Saber
+  class Book
+    # delegate to #populate
+    def self.populate(*args)
+      new.populate(*args)
+    end
+
+    attr_reader :client
+    
+    def initialize
+      @client = Faraday.new(url: Rc.api_url) {|c|
+        c.response :follow_redirects
+        c.response :json, :content_type => /\bjson$/
+
+        c.adapter Faraday.default_adapter
+      }
+    end
+
+    # @return [Hash] data
+    def populate(isbn, filename)
+      params = {}
+      rep = client.get("/books/#{isbn}", params)
+      data = rep.body
+
+      if data["status"] == 0
+        data["tracker"]
+      else
+        Saber.ui.error "Can't populate book -- #{isbn} #{filename}."
+        {}
+      end
+    end
+  end
+end

data/lib/saber/cli.rb

@@ -7,28 +7,29 @@ module Saber
     class_option "no-color",               banner: "Disable colorization in output", type: :boolean
     class_option "verbose", aliases: "-V", banner: "Enable verbose output mode", type: :boolean
     class_option "log",                    banner: "Log file", type: :string
-    class_option "force",   aliases: "-f", banner: "Fore writing even if file exists", type: :boolean
+    class_option "force",                  banner: "Fore writing even if file exists", type: :boolean
     class_option "tracker", aliases: "-t", banner: "tracker name", type: :string
-
-    attr_reader :o
+    class_option "dry-run", aliases: "-n", banner: "dry run", type: :boolean
 
     def initialize(*)
       super
-      o = @o = options.dup
+      self.options = self.options.dup
 
-      Saber.ui = if o["log"] then
+      Saber.ui = if options["log"] then
         require "logger"
-        UI::Logger.new(::Logger.new(o["log"]))
+        UI::Logger.new(::Logger.new(options["log"]))
       else
         the_shell = (options["no-color"] ? Thor::Shell::Basic.new : shell)
         UI::Shell.new(the_shell)
       end
 
-      Saber.ui.debug! if o["verbose"]
+      Saber.ui.debug! if options["verbose"]
 
       # Initialize environment in first time
       unless Rc.p.home.exists?
-        Pa.mkdir Rc.p.home
+        Pa.mkdir Rc.p.home 
+        Pa.mkdir "#{Rc.p.home}/templates"
+        Pa.mkdir "#{Rc.p.home}/database"
       end
     end
 
@@ -36,7 +37,7 @@ module Saber
     def clean
       require "saber/task/clean"
 
-      Task["clean"].invoke(:clean, [], o)
+      Task["clean"].invoke(:clean, [], options)
     end
 
     desc "server", "start saber-server daemon"
@@ -62,18 +63,25 @@ module Saber
       AutoFetcher::DRbClient.new.add(*names)
     end
 
-    desc "upload [options] <torrent_file/file ...>", "[make a torrent file and] upoad a torrent file to the site"
-    def upload(*files)
+    desc "upload [options] <format> <torrent_file/file ...>", "[make a torrent file and] upoad a torrent file to the site"
+    method_option "add", aliases: "-a", desc: "upload via add format.", type: :boolean
+    def upload(format0, *files)
       require "saber/task/upload"
+      format = format0.downcase
+      Saber.ui.error! "Don't support this format -- #{format0}" unless Rc.book_exts.include?(".#{format}")
 
-      Task["upload"].invoke(:upload, [options["tracker"] || ENV["SABER_TRACKER"], *files], o)
+      files = files.map{|v| v.dup} # unfrozen string.
+      Task["upload"].invoke(:upload, [options["tracker"] || ENV["SABER_TRACKER"], format, *wrap_file(*files)], options)
     end
 
-    desc "generate [options] <type> <filename> [arg ...]", %~generate a meta data file (alias: "g")~
-    def generate(type, filename, *args)
+    desc "generate [options] <type> [filename:isbn ...]", %~generate a meta data file (alias: "g")~
+    method_option "file", aliases: "-f", desc: "read files from file list", type: :string
+    def generate(type, *files)
       require "saber/task/generate"
 
-      Task["generate"].invoke(:generate, [options["tracker"] || ENV["SABER_TRACKER"], type, filename, *args], o)
+      files = File.read(options["file"]).split(/\n+/).map{|v| v.strip} if options["file"]
+      files = files.map{|v| name, isbn = v.split(":"); [*wrap_file(name), isbn]}
+      Task["generate"].invoke(:generate, [type, *files], options)
     end
     map "g" => "generate"
 
@@ -86,21 +94,39 @@ module Saber
     def send1(*names)
       require "saber/task/send"
 
-      Task["send"].invoke(:send1, names, o)
+      Task["send"].invoke(:send1, names, options)
     end
 
     desc "make [options] <file ..>", "make a torent file and send it to local and/or remote watch directory"
+    method_option "option", aliases: "-o", desc: "extra options passed to mktorrent", type: :string
+    method_option "file", aliases: "-f", desc: "read files from file list", type: :string
     def make(*files)
       require "saber/task/make"
 
-      Task["make"].invoke(:make, [options["tracker"] || ENV["SABER_TRACKER"], *files], o)
+      files = File.read(options["file"]).split(/\n+/).map{|v| v.strip.split(":")[0] } if options["file"]
+      Task["make"].invoke(:make, [options["tracker"] || ENV["SABER_TRACKER"], *files], options)
     end
 
-    desc "chd", "chd"
+    desc "chd", "NOT WORKING"
     def chd
       require "saber/task/chd"
 
-      Task["chd"].invoke(:chd, o)
+      Task["chd"].invoke(:chd, [], options)
+    end
+
+    desc "find_uploads", "NOT WORKING"
+    def find_uploads(page)
+      require "saber/task/find_uploads"
+
+      Task["find_uploads"].invoke(:find_uploads, [page], options)
+    end
+
+  private
+
+    # a.yml -> a
+    # a.epub.torrent -> a
+    def wrap_file(*names)
+      names.map{|v| Pa.delete_ext(v, *%w[.torrent .yml]).delete_ext2(*Rc.book_exts)}
     end
   end
 end

data/lib/saber/core_ext.rb

@@ -0,0 +1,68 @@
+class Hash
+  # From Rails 4.0
+
+  # Return a new hash with all keys converted by the block operation.
+  # This includes the keys from the root hash and from all
+  # nested hashes.
+  #
+  #  hash = { person: { name: 'Rob', age: '28' } }
+  #
+  #  hash.deep_transform_keys{ |key| key.to_s.upcase }
+  #  # => { "PERSON" => { "NAME" => "Rob", "AGE" => "28" } }
+  def deep_transform_keys(&block)
+    result = {}
+    each do |key, value|
+      result[yield(key)] = value.is_a?(Hash) ? value.deep_transform_keys(&block) : value
+    end
+    result
+  end
+
+  # Destructively convert all keys by using the block operation.
+  # This includes the keys from the root hash and from all
+  # nested hashes.
+  def deep_transform_keys!(&block)
+    keys.each do |key|
+      value = delete(key)
+      self[yield(key)] = value.is_a?(Hash) ? value.deep_transform_keys!(&block) : value
+    end
+    self
+  end
+
+  # Return a new hash with all keys converted to strings.
+  # This includes the keys from the root hash and from all
+  # nested hashes.
+  #
+  #   hash = { person: { name: 'Rob', age: '28' } }
+  #
+  #   hash.deep_stringify_keys
+  #   # => { "person" => { "name" => "Rob", "age" => "28" } }
+  def deep_stringify_keys
+    deep_transform_keys{ |key| key.to_s }
+  end
+
+  # Destructively convert all keys to strings.
+  # This includes the keys from the root hash and from all
+  # nested hashes.
+  def deep_stringify_keys!
+    deep_transform_keys!{ |key| key.to_s }
+  end
+
+  # Return a new hash with all keys converted to symbols, as long as
+  # they respond to +to_sym+. This includes the keys from the root hash
+  # and from all nested hashes.
+  #
+  #   hash = { 'person' => { 'name' => 'Rob', 'age' => '28' } }
+  #
+  #   hash.deep_symbolize_keys
+  #   # => { person: { name: "Rob", age: "28" } }
+  def deep_symbolize_keys
+    deep_transform_keys{ |key| key.to_sym rescue key }
+  end
+
+  # Destructively convert all keys to symbols, as long as they respond
+  # to +to_sym+. This includes the keys from the root hash and from all
+  # nested hashes.
+  def deep_symbolize_keys!
+    deep_transform_keys!{ |key| key.to_sym rescue key }
+  end
+end

data/lib/saber/fetcher.rb

@@ -17,7 +17,7 @@ module Saber
     def add(*files)
       files.each { |file|
         uri = "#{Rc.server.ftp}/#{file}"
-        gid = aria2_add([uri], :dir => Pa.dir2("#{Rc.p.fetcher_download}/#{file}"))
+        gid = aria2_add([uri], :dir => Pa.dir2("#{Rc.fetch.dir}/#{file}"))
         Saber.ui.debug "DOWNLOAD #{gid} #{uri}"
       }
     end
@@ -28,7 +28,7 @@ module Saber
       files = []
       Net::SSH.start(Rc.server.host, Rc.server.user) do |s|
         name = "'#{names.join("' '")}'"
-        cmd = "cd #{Rc.p.remote_download} && find #{name} -type f"
+        cmd = "cd #{Rc.fetch.remote_dir} && find #{name} -type f"
 
         rst = s.exec!(cmd)
         if rst =~ /^find: `|^cd:cd:/

data/lib/saber/rc.rb

@@ -1,9 +1,15 @@
+api_url = "http://saberapi.heroku.com"
 scgi_server = "http://localhost/RPC2"
+browser = [:firefox]
+book_exts = %w[.epub .mobi .pdf .txt .html .djvu .chm .cbr .cbz .azw3]
+book_formats = %w[epub mobi pdf txt html djvu chm cbr cbz azw3]
 
 p:
   root = Pa.expand("../../..", __FILE__)
   home = Pa("~/.saber")
   homerc = Pa("~/.saberrc")
+  database = Pa("~/.saber/database")
+  template = Pa("~/.saber/templates")
 
 aria2:
   rpc = "http://localhost:6800/rpc"

data/lib/saber/task.rb

@@ -18,6 +18,7 @@ module Saber
     autoload :Clean, "saber/task/clean"
     autoload :Make, "saber/task/make"
     autoload :Upload, "saber/task/upload"
+    autoload :FindUploads, "saber/task/find_uploads"
 
     @@tasks = {}
 
@@ -26,6 +27,8 @@ module Saber
 
     class << self
       def [](name)
+        require "saber/task/#{name}"
+
         tasks[name]
       end
     end

data/lib/saber/task/base.rb

@@ -8,11 +8,16 @@ module Saber
           Task.tasks[child.name.demodulize.underscore] = child
         end
 
-        # invoke a task
+        # delegate to #invoke
         def invoke(*args)
           new.invoke(*args)
         end
       end
+
+      def initialize(*)
+        super
+        self.options = self.options.dup
+      end
     end
   end
 end

data/lib/saber/task/chd.rb

@@ -1,10 +1,11 @@
+require "sys/filesystem"
+
 module Saber
   module Task
     class CHD < Base
 
       desc "chd", "chd"
       def chd(o={})
-        require "saber/tracker/chd"
         t = Tracker["chd"].new
 
         begin
@@ -14,7 +15,14 @@ module Saber
           t.update_cache(true)
 
           while true
-            t.add_torrents
+            # check free diskspace
+            s = Sys::Filesystem.stat(Rc.chd.diskspace_dir.p)
+            if s.block_size * s.blocks_free < Rc.chd.diskspace_limit
+              Saber.ui.say "::SKIP:: Reach low diskspace."
+            else
+              t.add_torrents
+            end
+
             sleep Rc.chd.update_interval
           end
         rescue Errno::ETIMEDOUT, Mechanize::ResponseCodeError, SocketError

data/lib/saber/task/clean.rb

@@ -6,8 +6,8 @@ module Saber
     class Clean < Base
       desc "clean", "clean"
       def clean
-        disk_files = Pa.ls2(Rc.p.download, absolute: true)
-        bt_files = Retort::Torrent.all.map{|t| Retort::Torrent.action("name", t.info_hash) }.map{|n| Pa.join2(Rc.p.download, n)}
+        disk_files = Pa.ls2(Rc.clean.dir, absolute: true)
+        bt_files = Retort::Torrent.all.map{|t| Retort::Torrent.action("name", t.info_hash) }.map{|n| Pa.join2(Rc.clean.dir, n)}
 
         (disk_files - bt_files).each { |file|
           Pa.rm_r file, :verbose => true

data/lib/saber/task/find_uploads.rb

@@ -0,0 +1,53 @@
+require "tagen/core/io"
+require "isbn"
+
+module Saber
+  module Task
+    class FindUploads < Base
+      desc "find_uploads", "find_uploads"
+      def find_uploads(page="1")
+        bib = Tracker["bib"].new(options)
+        stp = Tracker["stp"].new(options)
+        dir = Rc._fetch("find_uploads.dir", ".")
+        bib.agent.pluggable_parser["application/x-bittorrent"] = Mechanize::DirectorySaver.save_to(dir.to_s)
+        bib.login
+        stp.login
+
+        bib.browse(page) {|torrent|
+          title, isbn, download_link, filenames, tags = torrent[:title], torrent[:isbn], 
+            torrent[:download_link], torrent[:filenames], torrent[:tags]
+
+          generic_tags = convert_bibtags(tags)
+
+          begin
+            isbn = ISBN.thirteen(torrent[:isbn])
+          rescue ISBN::Invalid13DigitISBN # empty
+            next
+          end
+
+          if not stp.exists?(isbn: isbn)
+            bib.get(download_link)
+
+            Saber.ui.say "#{isbn} #{title}\n    #{filenames.join("\n    ")}"
+            File.append("list", "#{filenames[0]}:#{isbn}\n")
+
+            # local data
+            local_data = Pa.exists?("#{Rc.p.database}/#{isbn}.yml") ? YAML.load_file("#{Rc.p.database}/#{isbn}.yml") : {}
+            local_data.merge!({"tags" => generic_tags.join(", "), "bib.tags" => tags.join(", ")})
+            File.write "#{Rc.p.database}/#{isbn}.yml", YAML.dump(local_data)
+          end
+        }
+      end
+
+    private
+
+      def convert_bibtags(tags)
+        tags.map{|v| 
+          v.gsub(/ \(programming\)/i, '')
+        }.sort_by {|v| 
+          %w[fiction nonfiction].include?(v) ? -1 : tags.index(v)
+        }
+      end
+    end
+  end
+end

data/lib/saber/task/generate.rb

@@ -1,3 +1,5 @@
+require "isbn"
+
 module Saber
   module Task
     # Generate meta data file. 
@@ -5,34 +7,49 @@ module Saber
     # Usage
     # -----
     #
-    #     Task["generate"].invoke(:generate, ["bib", "ebook", "Hello.epub", isbn])
+    #     Task["generate"].invoke(:generate, ["ebook", ["harry_potter", isbn], ...])
     #     > generate Hello.epub.yml data file.
     #
     class Generate < Base
       include Thor::Actions
 
+      def self.source_paths
+        ["#{Rc.p.home}/templates"]
+      end
+
       source_root "#{Rc.p.root}/templates"
 
       desc "generate", "generate"
-      def generate(tracker_name, type, filename, *args)
-        require "saber/tracker/#{tracker_name}" if !args.empty?
-        template_file = find_in_source_paths("#{tracker_name}/#{type}.yml")
-        dest = "#{filename}.yml" 
-
-        if !args.empty? and Tracker[tracker_name].can_populate?(type)
-          require "yaml"
-          populate = {}
-          data = YAML.load_file(template_file)
-
-          tracker = Tracker[tracker_name].new
-          tracker.login
-          populate = tracker.populate(type, *args)
-          data.merge!(populate)
-
-          create_file dest, YAML.dump(data)
-        else
-          copy_file template_file, dest
-        end
+      def generate(type, *filenames)
+        template_file = find_in_source_paths("#{type}.yml")
+
+        filenames.each {|filename, isbn|
+          isbn = ISBN.thirteen(isbn) rescue nil
+          Saber.ui.say "Populating {#{isbn}} #{filename} ..."
+          dest = "#{filename}.yml" 
+
+          if isbn
+            require "yaml"
+            populate = {}
+            data = YAML.load_file(template_file)
+            data.merge! Book.populate(isbn, filename)
+            data.merge! YAML.load_file("#{Rc.p.database}/#{isbn}.yml") if Pa.exists?("#{Rc.p.database}/#{isbn}.yml")
+
+            # tags
+            if data["tags"]
+              Tracker.trackers.each {|name, tracker_class|
+                if data["#{name}.tags"].nil? and tracker_class.method_defined?(:convert_tags)
+                  tracker = tracker_class.new(options)
+                  data["#{name}.tags"] = tracker.convert_tags(*data["tags"].split(/, */)).join(", ")
+                end
+              }
+            end
+
+            create_file dest, YAML.dump(data)
+          else
+            copy_file template_file, dest
+          end
+        }
       end
     end
   end