nyaa 0.3.0 → 1.0.2

data/lib/nyaa/cli.rb

@@ -0,0 +1,75 @@
+# -*- encoding : utf-8 -*-
+module Nyaa
+  class CLI
+    def self.parse(args)
+      @config = {}
+
+      opts = OptionParser.new do |opt|
+        opt.banner = 'Usage: nyaa [options] QUERY'
+        opt.separator ''
+        opt.separator 'Specific options:'
+
+        @config[:category] = 'anime_english'
+        opt.on('-c', '--category=CATEGORY', 'Select a search category (default: anime_english)') do |cat|
+          @config[:category] = cat
+        end
+
+        @config[:filter] = 'show_all'
+        opt.on('-f', '--filter=FILTER', 'Filter for search query (default: show_all)') do |filter|
+          @config[:filter] = filter
+        end
+
+        @config[:output] = Dir.pwd
+        opt.on('-o', '--output-path=PATH', 'Output directory for downloads') do |output|
+          @config[:output] = output
+        end
+
+        opt.on('-b', '--batch', 'Batch mode for scripting (first page only)') do |batch|
+          @config[:batch] = batch
+        end
+
+        opt.on('-t', '--classic', 'Use the old interface (deprecated)') do |classic|
+          @config[:classic] = classic
+        end
+
+        opt.on('-v', '--version', 'Print version info') do
+          puts "nyaa #{Nyaa::VERSION}"
+          puts "Copyright (c) 2013 David Palma"
+          puts "https://github.com/mistofvongola/nyaa"
+          exit
+        end
+
+        opt.on_tail '-h', '--help', 'Show usage info' do
+          puts opts
+          puts %s{
+Categories:
+  anime_all, anime_raw, anime_english, anime_nonenglish, anime_music_video
+  books_all, books_raw, books_english, books_nonenglish
+  live_all, live_raw, live_english, live_nonenglish, live_promo
+  audio_all, audio_lossless, audio_lossy
+  pictures_all, pictures_photos, pictures_graphics,
+  software_all, software_apps, software_games
+
+Filters:
+  show_all, filter_remakes, trusted_only, aplus_only}
+          exit
+        end
+      end
+
+      # grab leftovers
+      @config[:query] = opts.parse!(args).join(' ')
+
+      unless Nyaa::CATS.has_key?(@config[:category].to_sym)
+        puts "#{@config[:category]} is not a valid category"
+        exit 1
+      end
+
+      unless Nyaa::FILS.has_key?(@config[:filter].to_sym)
+        puts "#{@config[:filter]} is not a valid filter"
+        exit 1
+      end
+
+      @config
+    end # parse
+  end # CLI
+end # Nyaa

data/lib/nyaa/constants.rb

@@ -0,0 +1,124 @@
+# -*- encoding : utf-8 -*-
+module Nyaa
+
+    BASE_URL = 'http://www.nyaa.eu/?page=search'
+    PSIZE = 100
+
+    CATS = {
+      :all_categories    => {
+        :id    => '0_0',
+        :title => 'All Categories'
+    },
+      :anime_all         => {
+        :id    => '1_0',
+        :title => 'Anime'
+      },
+      :anime_raw         => {
+        :id    => '1_11',
+        :title => 'Anime >> Raw Anime'
+      },
+      :anime_english     => {
+        :id    => '1_37',
+        :title => 'Anime >> English-translated Anime'
+      },
+      :anime_nonenglish  => {
+        :id    => '1_38',
+        :title => 'Anime >> Non-English-translated Anime'
+      },
+      :anime_music_video => {
+        :id    => '1_32',
+        :title => 'Anime >> Anime Music Video'
+      },
+      :books_all         => {
+        :id    => '2_0',
+        :title => 'Books'
+      },
+      :books_raw         => {
+        :id    => '2_13',
+        :title => 'Books >> Raw Books'
+      },
+      :books_english     => {
+        :id    => '2_12',
+        :title => 'Books >> English-scanlated Books'
+      },
+      :books_nonenglish  => {
+        :id    => '2_39',
+        :title => 'Books >> Non-English-scanlated Books'
+      },
+      :audio_all         => {
+        :id    => '3_0',
+        :title => 'Audio'
+      },
+      :audio_lossless    => {
+        :id    => '3_14',
+        :title => 'Audio >> Lossless Audio'
+      },
+      :audio_lossy       => {
+        :id    => '3_15',
+        :title => 'Audio >> Lossy Audio'
+      },
+      :pictures_all      => {
+        :id    => '4_0',
+        :title => 'Pictures'
+      },
+      :pictures_photos   => {
+        :id    => '4_17',
+        :title => 'Pictures >> Photos'
+      },
+      :pictures_graphics => {
+        :id    => '4_18',
+        :title => 'Pictures >> Graphics'
+      },
+      :live_all          => {
+        :id    => '5_0',
+        :title => 'Live Action'
+      },
+      :live_raw          => {
+        :id    => '5_20',
+        :title => 'Live Action >> Raw Live Action'
+      },
+      :live_english      => {
+        :id    => '5_19',
+        :title => 'Live Action >> English-translated Live Action'
+      },
+      :live_nonenglish   => {
+        :id    => '5_21',
+        :title => 'Live Action >> Non-English-translated Live Action'
+      },
+      :live_promo        => {
+        :id    => '5_22',
+        :title => 'Live Action >> Live Action Promotional Video'
+      },
+      :software_all      => {
+        :id    => '6_0',
+        :title => 'Software'
+      },
+      :software_apps     => {
+        :id    => '6_23',
+        :title => 'Software >> Applications'
+      },
+      :software_games    => {
+        :id    => '6_24',
+        :title => 'Software >> Games'
+      },
+    }
+
+    FILS = {
+      :show_all       => {
+        :id    => '0',
+        :title => 'Show all'
+      },
+      :filter_remakes => {
+        :id    => '1',
+        :title => 'Filter Remakes'
+      },
+      :trusted_only   => {
+        :id    => '2',
+        :title => 'Trusted only'
+      },
+      :aplus_only     => {
+        :id    => '3',
+        :title => 'A+ only'
+      },
+    }
+end

data/lib/nyaa/downloader.rb

@@ -0,0 +1,65 @@
+# -*- encoding : utf-8 -*-
+module Nyaa
+  class Downloader
+    attr_accessor :target, :destination, :retries
+    attr_accessor :response, :filename
+
+    def initialize(url, path, retries = 3)
+      self.target      = url
+      self.destination = sane_dir(path)
+      self.retries     = retries
+      self.response    = request
+      self.filename    = name_from_disposition
+
+      @fail = nil
+    end
+
+    def save
+      unless @fail
+        File.open("#{self.destination}/#{filename}", 'w') do |f|
+          f.write(self.response.body)
+        end
+      end
+    end
+
+    def failed?
+      @fail
+    end
+
+    private
+
+    def request
+      begin
+        response = RestClient.get(self.target)
+      rescue StandardError => e
+        if retries > 0
+          retries -= 1
+          sleep = 1
+          retry
+        end
+        @fail = true
+      end
+      @fail = false
+      response
+    end
+
+    # Filename from Content Disposition Header Field
+    # http://www.ietf.org/rfc/rfc2183.txt
+    def name_from_disposition
+      disp = self.response.headers[:content_disposition]
+      disp_filename = disp.split(/;\s+/).select { |v| v =~ /filename\s*=/ }[0]
+      re = /([""'])(?:(?=(\\?))\2.)*?\1/
+      if re.match(disp_filename)
+        filename = re.match(disp_filename).to_s.gsub(/\A['"]+|['"]+\Z/, "")
+      else
+        nil
+      end
+    end
+
+    def sane_dir(path)
+      path = Dir.pwd if path.nil? || !File.writable?(path)
+      FileUtils.mkdir_p path unless File.directory?(path)
+      path
+    end
+  end
+end

data/lib/nyaa/search.rb

@@ -0,0 +1,113 @@
+# -*- encoding : utf-8 -*-
+module Nyaa
+  class Search
+    attr_accessor :query, :category, :filter
+    attr_accessor :offset, :count, :results
+    attr_accessor :runid, :cachedir
+
+    def initialize(query, cat = nil, fil = nil)
+      self.query    = URI.escape(query)
+      self.category = cat ? CATS[cat.to_sym][:id] : '0_0'
+      self.filter   = fil ? FILS[fil.to_sym][:id] : '0'
+      self.offset   = 0
+      self.results  = []
+      self.count    = 1.0/0.0
+
+      self.runid    = Time.new.to_i
+      self.cachedir = File.expand_path('~/.nyaa/cache')
+      FileUtils.mkdir_p(cachedir) unless File.directory?(cachedir)
+    end
+
+    # Returns current batch (page) results
+    def get_results
+      if self.offset.zero?
+        batch = []
+      elsif self.offset == 1
+        batch = self.results[0, 100]
+      else # self.offset > 1
+        batch = self.results[(self.offset - 1) * 100, 100]
+      end
+      batch
+    end
+
+    def more
+      self.offset += 1
+      if self.results.length < self.count
+        extract(self.offset)
+      else
+        self.results = []
+        puts "No more results"
+      end
+        self
+    end
+
+    def cached(page)
+      cachefile = "#{self.cachedir}/cache_#{self.runid}_p#{page}"
+      p cachefile
+      return nil unless File.exists?(cachefile)
+
+      File.open(cachefile, 'rb') do |file|
+        begin
+          results = Marshal.load(file)
+        rescue => e
+          puts "ERROR: Failed to load #{cachefile}"
+          puts "#{e.backtrace}: #{e.message} (#{e.class})"
+        end
+      end
+      results
+    end
+
+    def purge
+      FileUtils.rm_rf Dir.glob("#{self.cachedir}/*")
+    end
+
+    private
+
+    def dump(page, results)
+      cachefile = "cache_#{self.runid}_p#{page}"
+      File.open("#{self.cachedir}/#{cachefile}", 'wb') do |file|
+        begin
+          Marshal.dump(results, file)
+        rescue => e
+          puts "ERROR: Failed to dump #{cachefile}"
+          puts "#{e.backtrace}: #{e.message} (#{e.class})"
+        ensure
+          file.close
+        end
+      end
+    end
+
+    def dump_json(page, results)
+      cachefile = "cache_#{self.runid}_p#{page}.json"
+      File.open("#{self.cachedir}/#{cachefile}", 'w') do |file|
+        begin
+          batch = []
+          results.each { |r| batch << r.to_hash }
+          file.write(batch.to_json)
+        rescue => e
+          puts "ERROR: Failed to dump #{cachefile}"
+          puts "#{e.backtrace}: #{e.message} (#{e.class})"
+        ensure
+          file.close
+        end
+      end
+    end
+
+    def extract(page)
+      raw = fetch(page)
+      doc = Nokogiri::HTML(raw)
+      self.count = doc.css('span.notice').text.match(/\d+/).to_s.to_i
+      rows = doc.css('div#main div.content table.tlist tr.tlistrow')
+      rows.each { |row| self.results << Torrent.new(row) }
+      dump(page, self.results)
+      #dump_json(page, self.results)
+    end
+
+    def fetch(page)
+      url = "#{BASE_URL}&offset=#{page}"
+      url << "&cats=#{self.category}&filter=#{self.filter}"
+      url << "&term=#{self.query}" unless self.query.empty?
+      open(url).read
+    end
+  end
+end

data/lib/nyaa/torrent.rb

@@ -0,0 +1,65 @@
+# -*- encoding : utf-8 -*-
+module Nyaa
+
+  class Torrent
+    attr_accessor :tid, :name, :info, :link
+    attr_accessor :filesize, :seeders, :leechers
+    attr_accessor :category, :status, :downloads, :comments
+    attr_accessor :health, :bytes
+
+    def initialize (row = nil)
+      self.tid      = row.css('td.tlistname').at('a')['href'][/tid=\d+/].gsub(/\D/,'')
+      self.name     = row.css('td.tlistname').at('a').text.strip
+      self.info     = row.css('td.tlistname').at('a')['href']
+      self.link     = row.css('td.tlistdownload').at('a')['href']
+      self.filesize = row.css('td.tlistsize').text
+      self.seeders  = row.css('td.tlistsn').text.to_i
+      self.leechers = row.css('td.tlistln').text.to_i
+
+      self.status    = state(row.values[0])
+      self.category  = row.css('td.tlisticon').at('a')['title']
+      self.downloads = row.css('td.tlistdn').text
+      self.comments  = row.css('td.tlistmn').text
+    end
+
+
+    def health( leech_weight = 0.5, seed_weight = 1.0 )
+      seeders.zero? ? 0 : seeders * seed_weight + leechers * leech_weight
+    end
+
+    def bytes
+      match = filesize.match(/([\d.]+)(.*)/)
+      if match
+        raw_size = match[1].to_f
+        case match[2].strip
+        when /gib/i then raw_size * 1000000000
+        when /mib/i then raw_size * 1000000
+        when /kib/i then raw_size * 1000
+        else nil
+        end
+      else
+        nil
+      end
+    end
+
+    def state(value)
+      case value
+      when 'trusted tlistrow' then status = 'Trusted'
+      when 'remake tlistrow'  then status = 'Remake'
+      when 'aplus tlistrow'   then status = 'A+'
+      when 'tlistrow'         then status = 'Normal'
+      else status = 'Normal'
+      end
+      status
+    end
+
+    def to_hash
+      hash = {}
+      instance_variables.each do |var|
+        hash[var.to_s.delete("@")] = instance_variable_get(var)
+      end
+      hash
+    end 
+
+  end
+end

data/lib/nyaa/ui.rb

@@ -0,0 +1,260 @@
+# -*- encoding : utf-8 -*-
+include Curses
+
+module Nyaa
+  class UI
+    attr_accessor :menusize, :page
+
+    def initialize (config, search)
+      @config = config
+      @status = { :text => 'Ready.', :type => :default }
+      setup_curses
+
+      # columns
+      @info_columns = %w[ Size SE LE ]
+      @info_column_width = 10
+      @name_column_width = 0
+
+      # commands
+      @commands = {
+        '?' => 'help',
+        'g' => 'get',
+        'i' => 'info',
+        'n' => 'next',
+        'p' => 'prev',
+        'q' => 'quit',
+      }
+
+      @search = search
+      @torrents = @search.more.results
+      @num_torrents = @torrents.size
+      harvester # start bg harvester
+      
+      @menusize = lines - 4 # lines - header + footer + status
+      @page = 1
+      @num_pages = (@search.count/@menusize.to_f).ceil
+      @offset = 0
+      # TODO: on hold state: prevent paging forward when waiting on results
+    end
+
+    def header
+      # pad info columns to column width
+      @info_columns = @info_columns.map do |column|
+        sprintf("%#{@info_column_width}s", column)
+      end
+      # key column width is whatever is leftover
+      @name_column_width = cols - (@info_columns.length * @info_column_width)
+
+      # header bar
+      cat = CATS[@config[:category].to_sym][:title]
+      header_text = sprintf " %-#{@name_column_width-1}s%s", "Nyaa - #{cat}", @info_columns.join
+      attrset(color_pair(1))
+      setpos(0,0)
+      addstr(sprintf "%-#{cols}s", header_text)
+    end
+
+    def footer
+      footer_text = @commands.map { |k,v| "#{k}: #{v}" }.join('  ')
+      attrset(color_pair(2))
+      setpos(lines - 1, 0)
+      addstr(sprintf " %-#{cols}s", footer_text)
+
+      search_summary = sprintf " %-14s %-14s %-14s",
+        "view: [#{@offset+1}-#{@offset+@menusize}]/#{@search.count}",
+        "recv: #{@num_torrents}/#{@search.count}",
+        "page: #{@page}/#{@num_pages}"
+      attrset(color_pair(2))
+      setpos(lines - 2, 0)
+      addstr(sprintf "%-#{cols}s", search_summary)
+    end
+
+    def status(text = nil, type = nil)
+      @status[:text] = text if text
+      @status[:type] = type
+
+      case @status[:type]
+      when :success then profile = 8
+      when :failure then profile = 9
+      else profile = 1
+      end
+
+      status_text = sprintf " Status: %-s", @status[:text]
+      attrset(color_pair(profile))
+      setpos(lines-3,0)
+      addstr(sprintf "%-#{cols}s", status_text)
+      refresh
+    end
+
+    def menu(highlight)
+      xpos = 1
+      attrset(color_pair(0))
+      setpos(xpos, 0)
+
+      (0..@menusize-1).each do |i|
+        if i < @search.count - @offset
+          if @torrents[@offset + i].nil?
+            status("Fetching more results, try again.", :failure)
+          else
+            line_text = sprintf("% -#{@name_column_width}s %9s %9s %9s",
+              truncate("#{@torrents[@offset + i].name}", @name_column_width),
+              @torrents[@offset + i].filesize,
+              @torrents[@offset + i].seeders,
+              @torrents[@offset + i].leechers)
+
+            attrset(color_pair(torrent_status(@torrents[@offset + i])))
+            setpos(xpos, 0)
+            # highlight the present choice
+            if highlight == i + 1
+              attron(A_STANDOUT)
+              addstr(line_text)
+              attroff(A_STANDOUT)
+            else
+              addstr(line_text)
+            end
+            xpos += 1
+          end
+        else
+          # blank lines if there's < @menusize of results
+          line_text = " "*cols
+          addstr(line_text)
+          xpos += 1
+        end 
+      end
+    end
+
+    # TODO: Pop up help window
+    def help
+    end
+
+    def move(cursor, increment)
+      if increment < 0 # negative, retreat!
+        if cursor == 1
+          if @offset == 0
+            cursor = 1
+          else
+            prev_page
+            cursor = menusize
+          end
+        else
+          cursor = cursor + increment
+        end
+      else # non-negative, advance!
+        if cursor == menusize
+          next_page
+          cursor = 1
+        else
+          cursor = cursor + increment
+        end
+      end
+    end
+
+    def get(choice)
+      torrent = @torrents[@offset + choice - 1]
+      download = Downloader.new(torrent.link, @config[:output])
+      download.save
+      unless download.failed?
+        status("Downloaded successful: #{torrent.tid}", :success)
+      else
+        status("Download failed (3 attempts): #{torrent.tid}", :failure)
+      end
+    end
+
+    def open(choice)
+      torrent = @torrents[@offset + choice - 1]
+      link = "#{torrent.info}"
+      if RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/ then
+          system("start #{link}")
+      elsif RbConfig::CONFIG['host_os'] =~ /darwin/ then
+          system("open '#{link}'", [:out, :err]=>'/dev/null')
+      elsif RbConfig::CONFIG['host_os'] =~ /linux/ then
+          system("xdg-open '#{link}'", [:out, :err]=>'/dev/null')
+      end
+      status("Opened '#{link}'", :success)
+    end
+
+    def next_page
+      status("Ready.")
+      unless @page + 1 > @num_pages
+        @page += 1
+      end
+
+      unless @offset + @menusize > @num_torrents
+        @offset += @menusize
+      end
+    end
+
+    def prev_page
+      unless @page - 1 < 1
+        @page += -1
+      end
+
+      unless @offset == 0
+        @offset -= @menusize
+      end
+    end
+
+    def resize_handler(cursor)
+      @menusize = lines - 4
+      menu(cursor)
+      refresh
+    end
+
+    def harvester
+      Thread.new do
+        until @num_torrents == @search.count
+          @torrents = @search.more.results
+          @num_torrents = @torrents.size
+          sleep 2
+        end
+        Thread.kill
+      end
+    end
+
+    private 
+
+    def setup_curses
+      noecho # don't show typed keys
+      init_screen # start curses mode
+      cbreak # disable line buffering
+      curs_set(0) # make cursor invisible
+      stdscr.keypad(true) # enable arrow keys
+
+      # set keyboard input timeout - sneaky way to manage refresh rate
+      timeout = 500 # milliseconds
+
+      if can_change_color?
+        start_color
+        init_pair(0, COLOR_WHITE, COLOR_BLACK)   # ui:default
+        init_pair(1, COLOR_BLACK, COLOR_CYAN)    # ui:header
+        init_pair(2, COLOR_BLACK, COLOR_YELLOW)  # ui:footer
+        init_pair(3, COLOR_MAGENTA, COLOR_BLACK) # ui:selected
+
+        init_pair(4, COLOR_WHITE, COLOR_BLACK) # torrent:normal
+        init_pair(5, COLOR_GREEN, COLOR_BLACK) # torrent:trusted
+        init_pair(6, COLOR_BLUE, COLOR_BLACK)  # torrent:aplus
+        init_pair(7, COLOR_RED, COLOR_BLACK)   # torrent:remake
+
+        init_pair(8, COLOR_BLACK, COLOR_GREEN) # type:success
+        init_pair(9, COLOR_BLACK, COLOR_RED)   # type:failure
+      end
+    end
+
+    def torrent_status(torrent)
+      case torrent.status
+      when 'Trusted' then 5
+      when 'A+'      then 6
+      when 'Remake'  then 7
+      else 4
+      end
+    end
+
+    def truncate(text, width)
+      if text.length > width
+        truncated = "#{text[0..width-4]}..."
+      else
+        text
+      end
+    end
+
+  end # UI
+end # Nyaa