kemonomachi-wanko 0.4.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c386e5f07a045d367717650f2f6fe54e2223b080
+  data.tar.gz: 90358f22e960700a46024b874a129cee3f2bfe17
+SHA512:
+  metadata.gz: 4f70b0fd6c5fff57da833b5599a756f1256e4f55a38bdb1b15637d203b541ec8f82a62df6379c8d73281de4260ade511f9aa29cfa6862d0830818a1748566029
+  data.tar.gz: 88b87e0b9b1f8f1b86d91cb78f879be33f52a56cdc62b7dbf34105c02f7dc3ad6bccc652270a7d528d0897dc79e2c71b50896e89385cdafb9586509e8489ae7e

data/LICENSE

@@ -0,0 +1,14 @@
+            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+                    Version 2, December 2004
+
+ Copyright (C) 2004 Sam Hocevar <sam@hocevar.net>
+
+ Everyone is permitted to copy and distribute verbatim or modified
+ copies of this license document, and changing it is allowed as long
+ as the name is changed.
+
+            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. You just DO WHAT THE FUCK YOU WANT TO.
+

data/README

@@ -0,0 +1,49 @@
+Wanko - RSS Torrent Fetcher
+
+Description
+-----------
+
+Fetches torrent file links from RSS feeds based on user-specified rules. The
+links can be printed to stdout or sent to Transmission for download, or the
+torrent files can be downloaded to a watchdir for processing by other torrent
+clients.
+
+Installation
+------------
+
+wanko can be installed from RubyGems:
+
+    gem install kemonomachi+wanko
+
+'kemonomachi+' is used as a ghetto namespace, since RubyGems doesn't support
+proper ones.
+
+Usage
+-----
+
+    wanko [options]
+
+The optional -c switch specifies the configuration directory, which defaults to
+$HOME/.wanko.
+
+See 'wanko --help' for usage info.
+
+Can be run manually, but creating a cron job to run it at regular intervals is
+recommended. It is not a daemon.
+
+Configuration
+-------------
+
+The config file is named 'config.yaml'. The example folder contains a full
+example config. RSS feeds, default fetch directory, fetch method and rules can
+be specified. Regexen are used to match torrents against rules.
+
+The config directory also holds 'history.yaml', which contains already read RSS
+items. This prevents fetching the same torrent multiple times.
+
+About the Name
+--------------
+
+Wanko (わんこ) is Japanese for 'doggy'. Because it's small and it fetches
+stuff for you.
+

data/bin/wanko

@@ -0,0 +1,8 @@
+#!/usr/bin/ruby
+
+require 'wanko'
+
+options = Wanko::parse_cli_switches ARGV
+
+options[:command].call options
+

data/lib/wanko.rb

@@ -0,0 +1,7 @@
+require 'wanko/data'
+require 'wanko/exception'
+require 'wanko/fetch'
+require 'wanko/read'
+require 'wanko/wanko'
+require 'wanko/write'
+

data/lib/wanko/command.rb

@@ -0,0 +1,73 @@
+require 'terminal-table'
+
+require 'wanko/read'
+require 'wanko/wanko'
+require 'wanko/write'
+
+module Wanko
+  module Command
+    def self.fetch(options)
+      config = begin
+        Wanko::Read.config options[:config_dir]
+      rescue Errno::ENOENT
+        abort 'Config file not found, aborting...'
+      end
+
+      [:feeds, :rules].each do |key|
+        warn 'WARN: No #{key} specified.' if config[key].empty?
+      end
+
+      history = Wanko::Read.history options[:config_dir]
+
+      torrents, new_history = Wanko::check_feeds config[:feeds], config[:rules], history
+
+      config[:fetcher].call torrents
+
+      Wanko::Write.history options[:config_dir], new_history
+    end
+
+    def self.add(options)
+      config = begin
+        Wanko::Read.raw_config options[:config_dir]
+      rescue Errno::ENOENT
+        abort 'Config file not found, aborting...'
+      end
+
+      id = if config[:rules].empty?
+             0
+           else
+             config[:rules].map {|x| x[:id]}.max + 1
+           end
+
+      rule = [id: id, regex: options[:regex], dir: options[:dir] || config[:base_dir]]
+
+      new_config = config.merge rules: config[:rules] + rule
+
+      Wanko::Write.config options[:config_dir], Wanko::Utility.stringify_keys(new_config)
+    end
+
+    def self.list(options)
+      config = begin
+        Wanko::Read.raw_config options[:config_dir]
+      rescue Errno::ENOENT
+        abort 'Config file not found, aborting...'
+      end
+
+      puts Terminal::Table.new(rows: config[:rules].map {|r| [r[:id], r[:regex], r[:dir]]},
+                               headings: ['ID', 'Regex', 'Directory']) {align_column 0, :right}
+    end
+
+    def self.remove(options)
+      config = begin
+        Wanko::Read.raw_config options[:config_dir]
+      rescue Errno::ENOENT
+        abort 'Config file not found, aborting...'
+      end
+
+      new_rules = config[:rules].reject {|rule| options[:ids].include? rule[:id]}
+
+      Wanko::Write.config options[:config_dir], Wanko::Utility.stringify_keys(config.merge rules: new_rules)
+    end
+  end
+end
+

data/lib/wanko/data.rb

@@ -0,0 +1,70 @@
+require 'json'
+
+module Wanko
+
+  # Data representations.
+  module Data
+
+    # Class for torrent data. Contains the name of and the link to a torrent,
+    # as well as the directory the torrent will be downloaded to.
+    class Torrent
+      def initialize(name, link, dir)
+        @name = name
+        @link = link
+        @dir = dir
+      end
+
+      attr_reader :name, :link, :dir
+
+      # Returns a Hash representation of this Torrent
+      def to_h()
+        {
+          name: name,
+          link: link,
+          dir: dir
+        }
+      end
+
+      # Returns a JSON representation of this Torrent
+      def to_json(state = nil)
+        to_h.to_json state
+      end
+    end
+
+    # Rule for matching against RSS items. Contains a regex for matching and a
+    # directory to download matched torrents to.
+    class Rule
+
+      # Public: Initialize a Rule object.
+      #
+      #regex - String or Regexp for matching.
+      #dir   - Directory to download matched torrents to.
+      def initialize(regex, dir)
+        @regex = Regexp.new regex, Regexp::IGNORECASE
+        @dir = dir
+      end
+
+      attr_reader :regex, :dir
+
+      # Public: Match the regex of this Rule against a String
+      #
+      # str - String to match.
+      #
+      # Returns true if str matches, false otherwise.
+      def =~(str)
+        regex =~ str
+      end
+
+      # Public: Compare this Rule to another object.
+      #
+      # other - Object to compare this Rule to.
+      #
+      # Returns true if the regex and dir fields of this Rule are equal to
+      #   the corresponding fields in other, false otherwise.
+      def ==(other)
+        regex == other.regex && dir == other.dir
+      end
+    end
+  end
+end
+

data/lib/wanko/exception.rb

@@ -0,0 +1,8 @@
+module Wanko
+  class Error < StandardError
+  end
+
+  class ConfigError < Wanko::Error
+  end
+end
+

data/lib/wanko/fetch.rb

@@ -0,0 +1,178 @@
+require 'json'
+require 'net/http'
+require 'open-uri'
+require 'yaml'
+
+require 'wanko/exception'
+require 'wanko/utility'
+
+module Wanko
+  
+  # Functions for fetching torrents in different ways. Several functions have
+  # side-effects, and ::fetcher_for returns Lambdas that have side-effects.
+  module Fetch
+
+    # Public: Create a lambda that fetches torrents when called.
+    #
+    # config - Hash with options for how to fetch torrents. :name is always
+    #          required and should be one of 'stdout', 'watchdir' and
+    #          'transmission'. Other options vary depending on the value of
+    #          :name, as follows:
+    #
+    #          name: 'stdout'
+    #            :format - Serialization format. See ::serialize for values.
+    #          name: 'transmission'
+    #            :host     - The daemon's host
+    #            :port     - The daemon's port
+    #            :path     - The daemon's path.
+    #            :user     - Username for Basic auth (optional)
+    #            :password - Password for Basic auth (optional)
+    #
+    # Returns a fetcher Lambda.
+    # Raises Wanko::ConfigError if :name option is not recognized.
+    def self.fetcher_for(config)
+      case config[:name]
+      when 'stdout'
+        ->(torrents) {$stdout.puts serialize(config[:format], torrents)}
+      when 'watchdir'
+        ->(torrents) {to_watchdir torrents}
+      when 'transmission'
+        url = make_transmission_url config[:host], config[:port], config[:path]
+   
+        auth = if config[:user] || config[:password]
+          config.select {|k, _| [:user, :password].include? k}
+        end
+
+        ->(torrents) {to_transmission url, auth, torrents}
+      else
+        raise Wanko::ConfigError, "Invalid fetcher name '#{config[:name]}'"
+      end
+    end
+
+    # Internal: Serialize torrents for the stdout fetcher.
+    #
+    # format   - Name of serialization method to use. Should be either 'json',
+    #            'simple' or 'yaml'.
+    # torrents - Torrents to serialize.
+    #
+    # Returns a String representation of torrents, in the given format.
+    # Raises Wanko::ConfigError if format is not recognized.
+    def self.serialize(format, torrents)
+      case format
+      when 'json'
+        torrents.to_json
+      when 'simple'
+        torrents.map(&:link).join "\n"
+      when 'yaml', nil
+        Utility.stringify_keys(torrents.map &:to_h).to_yaml
+      else
+        raise Wanko::ConfigError, "Invalid format '#{format}' for stdout fetcher."
+      end
+    end
+
+    # Internal: Download torrent files to watch directories. Creates any
+    # nonexisting directories.
+    #
+    # torrents - Torrents to download.
+    #
+    # Returns nothing.
+    def self.to_watchdir(torrents)
+      torrents.each do |tor|
+        FileUtils.mkdir_p tor.dir
+        
+        open tor.link do |remote|
+          File.binwrite File.join(tor.dir, "#{tor.name}.torrent"), remote.read
+        end
+      end
+    end
+
+    # Internal: Send RPC commands to Transmission daemon for downloading
+    # torrents.
+    #
+    # url      - URI object containing the URL of the daemon
+    # auth     - Hash with :user and :password for Basic auth, or falsey value
+    #            for no authorization.
+    # torrents - Array of torrents to download.
+    #
+    # Returns nothing.
+    def self.to_transmission(url, auth, torrents)
+      return if torrents.empty?
+
+      requests = torrents.map {|torrent| make_post_request url, make_transmission_rpc_command(torrent), auth}
+
+      Net::HTTP.start url.host, url.port do |transmission|
+        send_transmission_requests transmission, requests
+      end
+    end
+
+    # Internal: Send requests to a Transmission daemon. If a 409 Conflict
+    # response is recieved, update the 'X-transmission-Session-Id' header of
+    # the request and resend it.
+    #
+    # transmission - Net::HTTP connection to a Transmission daemon.
+    # reqs         - Net::HTTP::Post requests to send to Transmission.
+    #
+    # Returns nothing.
+    def self.send_transmission_requests(transmission, reqs)
+      session_id = ''
+
+      reqs.each do |req|
+        req['X-Transmission-Session-Id'] = session_id
+
+        response = transmission.request req
+
+        case response
+        when Net::HTTPConflict
+          session_id = response['X-Transmission-Session-Id']
+          redo
+        end
+      end
+    end
+
+    # Internal: Create a URI object for a Transmission daemon URL.
+    #
+    # host - Host part of the URL.
+    # port - Port part of the URL.
+    # path - Path part of the URL.
+    #
+    # Returns a URI object.
+    def self.make_transmission_url(host, port, path)
+      URI::HTTP.build host: host || '127.0.0.1', 
+                      port: port || 9091, 
+                      path: "#{path || '/transmission/'}rpc"
+    end
+
+    # Internal: Generate a JSON-encoded 'torrent-add' RPC command that can be
+    # sent to a Transmission daemon.
+    #
+    # torrent - Torrent to generate command for.
+    #
+    # Returns a String RPC command.
+    def self.make_transmission_rpc_command(torrent)
+      JSON.generate(
+        {
+          "method" => "torrent-add",
+          "arguments" => {
+            "filename" => torrent.link,
+            "download-dir" => torrent.dir
+          }
+        }
+      )
+    end
+
+    # Internal: Create an HTTP POST request.
+    #
+    # url  - URL this request will be sent to.
+    # body - String with body data.
+    # auth - Hash with Basic auth data, or falsey value for no authorization.
+    #
+    # Returns a Net::HTTP::Post request
+    def self.make_post_request(url, body, auth)
+      request = Net::HTTP::Post.new url
+      request.body = body
+      request.basic_auth auth[:user], auth[:password] if auth
+      request
+    end
+  end
+end
+

data/lib/wanko/read.rb

@@ -0,0 +1,88 @@
+require 'yaml'
+
+require 'wanko/data'
+require 'wanko/fetch'
+require 'wanko/utility'
+
+module Wanko
+
+  # Functions to read in data from files, urls and similar.
+  # All functions can be considered to rely on external state.
+  module Read
+
+    # Public: Read a config file, create a fetcher lambda and convert the
+    # rules.
+    #
+    # dir - Path to the directory containing the config file.
+    #
+    # Returns a Hash containing the configuration.
+    def self.config(dir)
+      config = raw_config dir
+
+      config.merge(
+        fetcher: Fetch.fetcher_for({name: 'stdout'}.merge Hash config[:fetcher]),
+        rules: Array(config[:rules]).map {|rule| convert_rule rule, config[:base_dir]}
+      )
+    end
+
+    # Public: Read a config file and supply some sensible defaults.
+    #
+    # dir - Path to the directory containing the config file.
+    #
+    # Returns a Hash containing the configuration.
+    def self.raw_config(dir)
+      config = Utility.symbolize_keys(YAML.load_file File.join(dir, 'config.yaml')) || {}
+
+      config.merge(
+        feeds: Array(config[:feeds]),
+        base_dir: config[:base_dir] || File.join(Dir.home, 'downloads'),
+        rules: Array(config[:rules])
+      )
+    end
+
+    # Internal: Convert a rule Hash into a Rule object, making the path
+    # absolute.
+    #
+    # rule     - Rule to convert. Has a required :regex entry and an optional
+    #            :dir entry.
+    # base_dir - Directory to use as base for relative paths, or as download
+    #            dir if the rule has no :dir entry.
+    #
+    # Returns an Array of converted Rules.
+    def self.convert_rule(rule, base_dir)
+      Data::Rule.new rule[:regex], File.absolute_path(rule[:dir] || '', base_dir)
+    end
+
+    # Public: Read an RSS feed history file.
+    #
+    # dir - Path to the directory containing the history file.
+    #
+    # Returns a Hash containing the history, or an empty Hash if the file does
+    #   not exist.
+    def self.history(dir)
+      history = begin
+        YAML.load_file File.join(dir, 'history.yaml')
+      rescue Errno::ENOENT
+        {}
+      end
+
+      # This is OK, since history is treated as read-only.
+      history.default = []
+
+      history
+    end
+
+    # Public: Read an RSS feed.
+    #
+    # url - Location of the feed. Must be openable by OpenURI.
+    #
+    # Returns an RSS::Rss object, or nil if the feed couldn't be read.
+    def self.feed(url)
+      begin
+        open(url, read_timeout: 10) {|rss| RSS::Parser.parse rss}
+      rescue OpenURI::HTTPError, Timeout::Error, Errno::ECONNREFUSED, SocketError => ex
+        warn "WARN: #{url} --> #{ex}"
+      end
+    end
+  end
+end

data/lib/wanko/utility.rb

@@ -0,0 +1,59 @@
+module Wanko
+
+  # Various utility functions.
+  module Utility
+
+    # Public: Recursively convert all keys of a Hash using the supplied block.
+    # Array values are traversed and each entry is converted. If arg is any
+    # other kind of object it is left untouched. If the conversion results in
+    # duplicate keys, later keys will overwrite earlier ones.
+    #
+    # Convenience functions exist for some often used operations, see
+    # ::symbolize_keys and ::stringify_keys.
+    #
+    # arg   - Object to convert.
+    # block - Block for converting the keys. Will be called with each key
+    # individually. Should return an object that can be used as a hash key.
+    #
+    # Examples
+    #
+    #   Utility.convert_keys h, &:to_sym
+    #
+    #   Utility.convert_keys(h) {|key| key.downcase.squeeze}
+    #
+    # Returns an object with converted hash keys.
+    def self.convert_keys(arg, &block)
+      case arg
+      when Hash
+        arg.each_with_object({}) { |(key, val), memo|
+          memo[yield(key)] = convert_keys(val, &block)
+        }
+      when Array
+        arg.map {|val| convert_keys val, &block}
+      else
+        arg
+      end
+    end
+
+    # Public: Recursively convert all keys in a Hash to Symbols. Does not check
+    # for duplicate keys. See ::convert_keys for more info.
+    #
+    # hash - Hash to convert. All keys must respond to #to_sym.
+    #
+    # Returns a structurally identical Hash with keys converted to Symbols.
+    def self.symbolize_keys(hash)
+      convert_keys hash, &:to_sym
+    end
+
+    # Public: Recursively convert all keys in a Hash to Strings. Does not check
+    # for duplicate keys. See ::convert_keys for more info.
+    #
+    # hash - Hash to convert. All keys must respond to #to_s.
+    #
+    # Returns a structurally identical Hash with keys converted to Strings.
+    def self.stringify_keys(hash)
+      convert_keys hash, &:to_s
+    end
+  end
+end
+

data/lib/wanko/wanko.rb

@@ -0,0 +1,111 @@
+require 'optparse'
+require 'rss'
+
+require 'wanko/command'
+require 'wanko/data'
+require 'wanko/read'
+
+module Wanko
+
+  # Public: Parse cli switches. Exits and prints a usage message when given
+  # -h, --help or an invalid switch.
+  #
+  # args - Array of switches to parse. Will not be altered.
+  #
+  # Returns a Hash containing the parsed options.
+  def self.parse_cli_switches(args)
+    options = {
+                command: Command.method(:fetch),
+                config_dir: File.join(Dir.home, ".wanko")
+              }
+
+    parser = OptionParser.new do |parser|
+      parser.banner = 'Usage: wanko [options]'
+
+      parser.separator ''
+      parser.separator 'Options:' 
+
+      parser.on '-c DIR', '--config_dir', 'Use a different config directory' do |dir|
+        options[:config_dir] = File.absolute_path dir
+      end
+
+      parser.on '-a REGEX', '--add', 'Add a fetch rule' do |regex|
+        options[:command] = Command.method :add
+        options[:regex] = regex
+      end
+
+      parser.on '-d DIR', '--directory', 'Optional directory for fetch rules added with -a' do |dir|
+        options[:dir] = dir
+      end
+
+      parser.on '-l', '--list', 'List all rules' do
+        options[:command] = Command.method :list
+      end
+
+      parser.on '-r ID', '--remove', Integer, 'Remove a fetch rule' do |id|
+        options[:command] = Command.method :remove
+        options[:ids] = [id]
+      end
+
+      parser.separator ''
+      parser.separator 'Other:'
+
+      parser.on '-h', '--help',
+              'Show this message' do
+        puts parser
+        exit
+      end
+    end
+
+    begin
+      parser.parse args
+    rescue OptionParser::InvalidOption
+      puts parser
+      exit
+    end
+
+    options
+  end
+
+  # Public: Check RSS feeds for new torrents matching a set of rules, excluding
+  # already read items.
+  #
+  # urls    - Array of feed urls to check.
+  # rules   - Array of Rules to match against.
+  # history - Hash with urls mapped to Arrays of already read items. Will be
+  #           accessed using the urls in the urls parameter. If a url is
+  #           missing, an empty Array will be substituted.
+  #
+  # Returns a pair [[Torrent], Hash] of matched items and an updated history
+  def self.check_feeds(urls, rules, history)
+    matches, new_history = urls.map { |url|
+      feed = Read.feed(url) or next [[], {url => history[url]}]
+
+      [
+        match(feed, rules, Array(history[url])),
+        {url => feed.items.map {|item| item.guid.content}}
+      ]
+    }.transpose
+
+    [matches.flatten, new_history.reduce(:merge)]
+  end
+
+  # Internal: Match a set of rules against the items of an RSS feed, excluding
+  # already read items.
+  #
+  # feed    - RSS object which items to search through.
+  # rules   - Array of Rules.
+  # history - Array of GUIDs. Items found in this array will be rejected.
+  #
+  # Returns an Array of Torrents representing the matched items.
+  def self.match(feed, rules, history)
+    feed.items
+    .reject {|item| history.include? item.guid.content}
+    .product(rules)
+    .select {|item, rule| rule =~ item.title}
+    .map { |item, rule|
+      Data::Torrent.new item.title, item.link, rule.dir
+    }
+  end
+end
+

data/lib/wanko/write.rb

@@ -0,0 +1,27 @@
+require 'yaml'
+
+module Wanko
+
+  # Functions for writing data out to files or similar. All functions can be
+  # considered to have destructive side-effects.
+  module Write
+
+    # Public: Write a YAML representation of an object to a file named
+    # 'history.yaml'.
+    #
+    # This function _will_ clobber an existing file.
+    #
+    # dir     - Path of the directory to write the file in.
+    # history - Object to write out. Responds to #to_yaml.
+    #
+    # Returns nothing
+    def self.history(dir, history)
+      File.write File.join(dir, 'history.yaml'), history.to_yaml
+    end
+
+    def self.config(dir, config)
+      File.write File.join(dir, 'config.yaml'), config.to_yaml
+    end
+  end
+end
+

metadata

@@ -0,0 +1,75 @@
+--- !ruby/object:Gem::Specification
+name: kemonomachi-wanko
+version: !ruby/object:Gem::Version
+  version: 0.4.2
+platform: ruby
+authors:
+- Ookami Kenrou
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-04-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: terminal-table
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+description: |2
+      Fetches torrent file links from RSS feeds based on user-specified rules. The
+      links can be printed to stdout or sent to Transmission for download, or the
+      torrent files can be downloaded to a watchdir for processing by other torrent
+      clients.
+email: ookamikenrou@gmail.com
+executables:
+- wanko
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README
+- bin/wanko
+- lib/wanko.rb
+- lib/wanko/command.rb
+- lib/wanko/data.rb
+- lib/wanko/exception.rb
+- lib/wanko/fetch.rb
+- lib/wanko/read.rb
+- lib/wanko/utility.rb
+- lib/wanko/wanko.rb
+- lib/wanko/write.rb
+homepage: https://github.com/kemonomachi/wanko
+licenses:
+- WTFPL
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements:
+- Transmission bittorrent client (optional)
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: RSS Torrent Fetcher
+test_files: []