nicorepo 0.0.8 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5501c9e31cdda24fbcf1bb355856b4f99a293bfd
-  data.tar.gz: e496adc8058c169ef52d5c3a808e22375d386d94
+  metadata.gz: bae618f5d97184adc26e5376e237e195903e70f0
+  data.tar.gz: 82a97cc15b7d6b3c2931d27108c22f9d0d9a5d08
 SHA512:
-  metadata.gz: e8701ba39746179bfd8f4e0d7203acfd03e4ccd2b41c23b5f218c1ac1ac14cead86291d0606ff84af8136e407e2591e2d66f16bb3f042989b751d6c2c4c6d9a4
-  data.tar.gz: ebd6d0210c61adaf63eaa39642040299296dd5898f94d25944736ce310b83fc201668cd192f920edc7c8333ec995e5dae30cf2f326ea94f433b1ba358652a8f3
+  metadata.gz: e92bf7c4e0f7916158d5cb8fc6d21ba0dc9ea3ce1d48a0a64775563eb1d0e5391cff14148d80c3c9d8606f0f9fe0ca5419d295290e79697ed93d07f7c6e2223b
+  data.tar.gz: a6343d1bb79878a5fe79b025db64b897fbb00e1f79dac1f5fe9f41319ce14e4f738dfd1cbbb3fd02b531ba69a3beb70bacf91b6dafdd8bf2f3b35bd706a66e02

data/README.md

@@ -1,12 +1,14 @@
 # Nicorepo
 
-Nicorepo is scraper and filter of nicorepo on nicovideo.
+Nicorepo is an (unofficial) API client for Nicorepo on nicovideo.jp
 
-- filter reports by kind of them
-- specify number and pages to fetch reports
-- open url in browser
+- Fetch nicorepo logs
+- Filter them by topics and a period
+- Provides a built-in CLI and commands
 
-It requires ruby 2.0.0.
+## Requirements
+
+- ruby 2.3.0
 
 ## Installation
 
@@ -24,11 +26,27 @@ Or install it yourself as:
 
 ## Usage
 
-### Authentication
+```rb
+client = Nicorepo::Client.new
+
+client.all
+# => Returns raw logs wrapped in Nicorepo::Report
+
+client.lives(10)
+# => Returns 10 live logs
+
+client.videos(10, from: Time.now - (3600 * 24), to: Time.now - (3600 * 48))
+# => Returns 10 uploaded video logs (at most) within yesterday
 
-Nicorepo supports reading netrc file.
+client.all.format
+# => Returns an array of logs formatted by Nicorepo::Report::DefaultFormatter
+```
+
+### Authentication for nicovideo.jp
 
-Add following lines to your netrc file (`~/.netrc`)
+Nicorepo requires your nicovideo account and reads them from the `~/.netrc` file.
+
+Add following lines to your `~/.netrc`.
 
 ```
 machine nicovideo.jp
@@ -42,48 +60,60 @@ And set the permission if not yet.
 $ chmod 0600 ~/.netrc
 ```
 
-### Start nicorepo cli
+### Start Nicorepo CLI
 
-    $ nicorepo
+```console
+$ nicorepo
+```
 
-You can use following commands in interactive cli to fetch nicorepos.
+You can use following commands in interactive cli.
 
 command               | alias | description
 ----------------------|-------|---------------------------------------------------------
-  all                 | a     | fetch all reports
-  videos              | v     | fetch only video reports
-  lives               | li    | fetch only live reports
-  show                | s     | show current reports
+  all                 | a     | fetch all logs
+  videos              | v     | fetch only video logs
+  lives               | li    | fetch only live logs
+  show                | s     | show current logs
   open REPORT-NUMBER  | o     | open the report url specified by number in your browser
   help [COMMAND]      | h     | Describe available commands or one specific command
   login               | lo    | re-login if your session is expired
   exit                | e     | exit interactive prompt
 
-Some commands have specific options `-n, --request-num=N`, `-p, --limit-page=N`.
-For example, if you want 20 video reports by searcing over 5 pages, the command will be,
+If you want only video logs, type `video` command.
 
-    > video -n20 -p5
+    > video
 
 Or you can also use aliases.
 
-    > v -n20 -p5
+    > v
+
+Some commands have options of `-n, --request-num=N` and `-p, --limit-page=N`.
+For example:
+
+    > lives -n20 -p5
 
-`-n` and `-p` are specifing options.
-You can omit it as you like then each commad uses default value.
+It fetches 20 live logs at a most, with limitation of max 5 pages.
+
+If you omit them the default values are used for that. (the defaults are configurable)
 
     > v
-    # => `video -n10 -p3`
+    # => `video -n10 -p10`
     > v -n20
-    # => `video -n20 -p3`
+    # => `video -n20 -p10`
     > v -p5
-    # => `video -n20 -p5`
+    # => `video -n20 -p10`
 
-And also, you can use `-l, --latest`, `-h, --hours`, `-d, --days` options to find reports by the specific period.
+And also, you can use `-l, --latest`, `-h, --hours` and `-d, --days` options to fetch logs in the specific period.
 
     > all -l
-    > lives -h 1
 
-### Configuration
+Collect all logs until reach the last fetched log.
+
+    > lives -h1
+
+Collect live logs until 1 hour ago. 
+
+#### Configuration for CLI
 
 You can configure default `request_num` and `limit_page` by adding `~/.nicorepo.yaml` if you want.
 Please refer the sample `nicorepo/.nicorepo.yaml.sample` or copy it to your home directory.
@@ -100,7 +130,7 @@ limit_page:
 ```
 
 - `general`: used in all commands
-- `all`, `videos`, `lives`: used in each command, has priority than `general`
+- `all`, `videos`, `lives`: used in each command, has higher priority than `general`
 
 ## Contributing
 

data/bin/nicorepo

@@ -1,6 +1,8 @@
 #!/usr/bin/env ruby
 
 require 'bundler/setup'
-require_relative '../lib/nicorepo'
+require 'pry'
+
+require_relative '../lib/nicorepo/cli'
 
 Nicorepo::Cli::Runner.run

data/lib/nicorepo.rb

@@ -1,52 +1,6 @@
-require 'mechanize'
-require 'nicorepo/reports'
-require 'nicorepo/parser'
-
-class Nicorepo
-  class LoginError < StandardError; end
-
-  PER_PAGE = 20
-  LOGIN_URL = 'https://secure.nicovideo.jp/secure/login?site=niconico'
-
-  attr_reader :agent
-
-  def initialize
-    @agent = Mechanize.new
-    @agent.ssl_version = 'TLSv1'
-    @agent.request_headers = { 'accept-language' => 'ja-JP', 'content-language' => 'ja-JP' }
-    @parser = Parser.new(@agent)
-    @logined = false
-  end
-
-  def login(mail, pass)
-    page = @agent.post(LOGIN_URL, mail: mail, password: pass)
-    if page.header["x-niconico-authflag"] == '0'
-      raise LoginError, "Failed to login"
-    else
-      @logined = true
-    end
-  end
-
-  def logined?
-    # TODO: page.header["x-niconico-auth-flag"] をチェックする？
-    #       現状一度ログインしたらfalseにならない
-    @logined
-  end
-
-  def all(request_num = PER_PAGE, since: nil)
-    limit_page = request_num / PER_PAGE + 1
-    Reports.new(@parser).fetch(request_num, limit_page, since: since)
-  end
-
-  def videos(request_num = 3, limit_page = 5, since: nil)
-    VideoReports.new(@parser).fetch(request_num, limit_page, since: since)
-  end
-
-  def lives(request_num = 3, limit_page = 5, since: nil)
-    LiveReports.new(@parser).fetch(request_num, limit_page, since: since)
-  end
-end
-
-require 'nicorepo/cli/cli'
 require "nicorepo/version"
-
+require 'nicorepo/client'
+require 'nicorepo/filter'
+require 'nicorepo/page'
+require 'nicorepo/report'
+require 'nicorepo/request'

data/lib/nicorepo/{cli/cli.rb → cli.rb}

@@ -1,10 +1,11 @@
+require 'nicorepo'
+require 'nicorepo/cli/configuration'
+
 require 'thor'
-require 'netrc'
 require 'launchy'
 require 'readline'
-require 'nicorepo/cli/config'
 
-class Nicorepo
+module Nicorepo
   module Cli
     class Runner
       def self.run
@@ -17,32 +18,14 @@ class Nicorepo
     end
 
     class Interactor < Thor
-      class LoginAccountError < StandardError; end
-      class ReportExistenceError < StandardError; end
-
       class << self
-        def start(*)
-          login unless repo.logined?
-          super
-        end
-
         # replaces by a blank for help because Interactor dosen't require the basename
         def basename
           ''
         end
 
-        def login
-          mail, pass = Netrc.read["nicovideo.jp"]
-          raise LoginAccountError, "machine nicovideo.jp is not defined in .netrc" if mail.nil? || pass.nil?
-          begin
-            repo.login(mail, pass)
-          rescue
-            raise LoginAccountError, "invalid mail or pass: mail = #{mail}"
-          end
-        end
-
         def repo
-          @repo ||= Nicorepo.new
+          @repo ||= Nicorepo::Client.new
         end
 
         def conf
@@ -54,9 +37,9 @@ class Nicorepo
         end
       end
 
-      desc "login", "re-login if your session is expired"
-      def login
-        login
+      desc "reset_session", "reset current session and try re-login at the next access. It is useful if your session is expired"
+      def reset_session
+        @repo.reset_session
       end
 
       desc "all", "fetch all reports"
@@ -66,7 +49,7 @@ class Nicorepo
       option :"hours", type: :numeric, aliases: :h
       def all
         request_num = options[:"request-num"] || conf.request_num("all")
-        request_options = { since: parse_since_options(options) }
+        request_options = { to: parse_time_options(options) }
         cache(repo.all(request_num, request_options))
         show
       end
@@ -80,8 +63,8 @@ class Nicorepo
       def videos
         request_num = options[:"request-num"] || conf.request_num("videos")
         limit_page  = options[:"limit-page"]  || conf.limit_page("videos")
-        request_options = { since: parse_since_options(options) }
-        cache(repo.videos(request_num, limit_page, request_options))
+        request_options = { limit_page: limit_page, to: parse_time_options(options) }
+        cache(repo.videos(request_num, request_options))
         show
       end
 
@@ -94,24 +77,29 @@ class Nicorepo
       def lives
         request_num = options[:"request-num"] || conf.request_num("lives")
         limit_page  = options[:"limit-page"]  || conf.limit_page("lives")
-        request_options = { since: parse_since_options(options) }
-        cache(repo.lives(request_num, limit_page, request_options))
+        request_options = { limit_page: limit_page, to: parse_time_options(options) }
+        cache(repo.lives(request_num, request_options))
         show
       end
 
       desc "show", "show current reports"
       option :more, type: :boolean, aliases: :m
       def show
-        showed_reports = options[:more] ? cached_reports : cached_reports[0, recent_tail]
-        showed_reports.each.with_index(1) do |report, i|
+        if cached_reports.size == 0
+          say "* No reports", :red
+          return
+        end
+
+        reports = options[:more] ? cached_reports : cached_reports[0, recent_tail]
+        reports.each.with_index(1) do |report, i|
           say "--- MORE ---", :blue if i == recent_tail + 1
-          say "[#{i}] #{report.body} at #{report.date.to_s}"
-          say "     #{report.title} (#{report.url})", :green
+          say "[#{i}] #{report[:topic]} from: #{report[:sender]} at: #{report[:created_at]}"
+          say "    #{report[:title]} (#{report[:url]})", :green
         end
         say "* last fetch time: #{cached_at}", :blue
       end
 
-      desc "open REPORT-NUMBER", "open the report url specified by number in your browser"
+      desc "open REPORT-NUMBER", "open the url in your browser. REPORT-NUMBER is shown in left of each report"
       def open(report_number)
         open_numbered_link(report_number.to_i)
       end
@@ -132,7 +120,7 @@ class Nicorepo
       end
 
       def cached_reports
-        self.class.cache[:reports] ||= []
+        self.class.cache[:report] ||= []
       end
 
       def recent_tail
@@ -143,13 +131,13 @@ class Nicorepo
         self.class.cache[:cached_at]
       end
 
-      def cache(reports)
-        cached_reports.unshift(reports).flatten!
-        self.class.cache[:recent_tail] = reports.size
+      def cache(new_report)
+        cached_reports.unshift(*new_report.format)
+        self.class.cache[:recent_tail] = new_report.size
         self.class.cache[:cached_at] = Time.now
       end
 
-      def parse_since_options(options)
+      def parse_time_options(options)
         case
         when options[:latest]
           cached_at
@@ -162,12 +150,16 @@ class Nicorepo
         end
       end
 
-      def open_numbered_link(request_num)
-        url = cached_reports[request_num - 1].url
-        raise ReportExistenceError, "report existence error: please fetch reports" if url.nil?
+      def open_numbered_link(num)
+        if num > cached_reports.size || num < 1
+          say "Unavailable report number", :red
+          return
+        end
+
+        url = cached_reports[num - 1][:url]
 
         Launchy.open(url, options) do |exception|
-          puts "Attempted to open #{url} and failed because #{exception}"
+          puts "Attempted to open #{url} but failed because of #{exception}"
           raise exception
         end
       end

data/lib/nicorepo/cli/{config.rb → configuration.rb}

@@ -1,6 +1,6 @@
 require 'yaml'
 
-class Nicorepo
+module Nicorepo
   module Cli
     class Configuration
 
@@ -8,7 +8,7 @@ class Nicorepo
 
       def initialize
         params = defaults.merge(load_config)
-        @request_nums  = params["request_num"]
+        @request_nums = params["request_num"]
         @limit_pages = params["limit_page"]
       end
 
@@ -35,7 +35,7 @@ class Nicorepo
             "general" => 10
           },
           "limit_page" => {
-            "general" => 3
+            "general" => 10
           }
         }
       end

data/lib/nicorepo/client.rb

@@ -0,0 +1,64 @@
+require 'netrc'
+
+module Nicorepo
+  class Client
+    class AccountLackError < StandardError; end
+
+    PER_PAGE = 25
+    MAX_PAGES_DEFAULT = 20
+
+    def session
+      # TODO: handle expiration
+      @session ||= (
+        mail, pass = Netrc.read["nicovideo.jp"]
+        raise AccountLackError, "mail and password is not defined in .netrc as a machine nicovideo.jp" if mail.nil? || pass.nil?
+        Request::Auth.new.login(mail, pass)
+      )
+    end
+
+    def reset_session
+      @session = nil
+    end
+
+    def all(request_num = PER_PAGE, options = {})
+      options = options.merge(limit_page: request_num / PER_PAGE + 1)
+
+      fetch(:all, request_num, options)
+    end
+
+    def videos(request_num, options = {})
+      fetch(:videos, request_num, options)
+    end
+
+    def lives(request_num, options = {})
+      fetch(:lives, request_num, options)
+    end
+
+    # === Params
+    #
+    # * limit_page - Integer
+    # * request_num - Integer
+    # * options - Hash
+    #
+    # === Options
+    #
+    # * from - Time or String
+    # * to - Time or String
+    #
+    # fetch reports `:from` newer `:to` older
+    #
+    def fetch(filter_type, request_num, options = {})
+      limit_page = options[:limit_page] || MAX_PAGES_DEFAULT
+      filter = Filter.new(filter_type, min_time: options[:to])
+      page = Page.new(session, filter, options[:from], options[:to])
+
+      # NOTE: filter は report に渡したほうがシンプルじゃない？
+      limit_page.times.each_with_object(Report.new(request_num)) do |_, report|
+        report.push(page)
+        break report if page.last_page? || report.reach_request_num?
+
+        page = page.next
+      end
+    end
+  end
+end

data/lib/nicorepo/filter.rb

@@ -0,0 +1,66 @@
+module Nicorepo
+  class Filter
+    def initialize(type_or_topic, options = {})
+      @rules = [].tap do |r|
+        r << (
+          case type_or_topic
+          when :all
+            All.new
+          when :videos
+            Equal.new('nicovideo.user.video.upload')
+          when :lives
+            Match.new(/^live/)
+          when String
+            Equal.new(type_or_topic)
+          when Regex
+            Match.new(type_or_topic)
+          else
+            fail "Given type or topic #{type_or_topic} is not supported"
+          end
+        )
+
+        r << MinTime.new(options[:min_time]) if options[:min_time]
+      end
+    end
+
+    def accepts?(data)
+      @rules.all? { |r| r.accepts?(data) }
+    end
+
+    class MinTime
+      def initialize(min_time)
+        @min_time = min_time
+      end
+
+      def accepts?(data)
+        Time.parse(data['createdAt']) > @min_time
+      end
+    end
+
+    class All
+      def accepts?(_)
+        true
+      end
+    end
+
+    class Equal
+      def initialize(topic)
+        @topic = topic
+      end
+
+      def accepts?(data)
+        data['topic'] == @topic
+      end
+    end
+
+    class Match
+      def initialize(topic_regexp)
+        @topic_regexp = topic_regexp
+      end
+
+      def accepts?(data)
+        data['topic'] =~ @topic_regexp
+      end
+    end
+  end
+end

data/lib/nicorepo/page.rb

@@ -0,0 +1,60 @@
+module Nicorepo
+  class Page
+    attr_reader :raw
+
+    def initialize(session, filter, from = nil, to = nil)
+      @session = session
+      @filter = filter
+      @cursor = Cursor.new(from).value
+      @min_cursor = Cursor.new(to).value
+
+      fetch
+    end
+
+    def next
+      fail 'Next page not found' if last_page?
+
+      self.class.new(@session, @filter, @next_cursor, @min_cursor)
+    end
+
+    def last_page?
+      return false unless @min_cursor && @next_cursor
+
+      @min_cursor > @next_cursor
+    end
+
+    private
+
+    def fetch
+      json = Request::Timeline.fetch(@session, cursor: @cursor)
+
+      @next_cursor = json['meta']['minId']
+      @raw = json['data'].select { |data| @filter.accepts?(data) }
+    end
+
+    class Cursor
+      attr_reader :value
+
+      def initialize(time_or_cursor)
+        @value = cursor?(time_or_cursor) ? time_or_cursor : value_from_time(time_or_cursor)
+      end
+
+      def to_s
+        @value
+      end
+
+      private
+
+      def value_from_time(time)
+        return unless time
+
+        time = Time.parse(time) if time === String
+        (time.to_f * 1000).floor.to_s
+      end
+
+      def cursor?(value)
+        value === String && cursor.match?(/\A\d{13}\z/)
+      end
+    end
+  end
+end

data/lib/nicorepo/report.rb

@@ -1,16 +1,107 @@
-class Nicorepo
+module Nicorepo
   class Report
-    attr_accessor :body, :title, :url, :author, :kind, :date
+    attr_reader :pages
 
-    ERROR_KIND = 'error'
+    def initialize(request_num)
+      @pages = []
+      @request_num = request_num
+    end
+
+    def push(page_or_pages)
+      pages = (page_or_pages.is_a?(Array) ? page_or_pages : [page_or_pages]).compact
+      @pages.push(*pages)
+      self
+    end
+
+    def size
+      raw.size
+    end
+
+    def raw
+      @pages.map(&:raw).flatten[0, @request_num]
+    end
+
+    def format(formatter = DefaultFormatter)
+      formatter.process_all(raw)
+    end
+
+    def reach_request_num?
+      size >= @request_num
+    end
+
+    class DefaultFormatter
+      require 'time'
+
+      class << self
+        def process_all(raw)
+          raw.map { |h| process(h) }.compact
+        end
+
+        def process(h)
+          return unless h
+
+          title, url = infer_body(h)
+          {
+            sender: nickname(h),
+            topic: topic(h),
+            title: title,
+            url: url,
+            created_at: created_at(h)
+          }
+        end
+
+        private
+
+        def nickname(h)
+          h.dig('senderNiconicoUser', 'nickname')
+        end
+
+        def topic(h)
+          case h['topic']
+          when 'nicovideo.user.video.upload'
+            '動画投稿'
+          when 'nicovideo.user.video.kiriban.play'
+            "#{h.dig('actionLog', 'viewCount')}再生達成"
+          when 'nicovideo.user.video.advertise'
+            '動画広告'
+          when 'nicovideo.user.mylist.add.video'
+            'マイリスト登録'
+          when 'live.user.program.onairs'
+            '生放送開始'
+          else
+            h['topic']
+          end
+        end
+
+        def created_at(h)
+          Time.parse(h['createdAt'])
+        end
+
+        # @return [String, String] title, url
+        def infer_body(h)
+          candidate_keys = %w(video program)
+          candidate_keys.each do |k|
+            if h[k]
+              id, title = h[k].fetch_values('id', 'title') if h[k]
+              return title, watch_url(id, k)
+            end
+          end
+
+          ['', '']
+        end
+
+        def watch_url(id, key)
+          base =
+            case key
+            when 'video'
+              "http://www.nicovideo.jp/watch/"
+            when 'program'
+              "http://live.nicovideo.jp/watch/"
+            end
 
-    def initialize(attrs)
-      @body   = attrs[:body]
-      @title  = attrs[:title]
-      @url    = attrs[:url]
-      @author = attrs[:author]
-      @kind   = attrs[:kind]
-      @date   = attrs[:date]
+          base + id
+        end
+      end
     end
   end
 end

data/lib/nicorepo/request.rb

@@ -0,0 +1,81 @@
+require 'net/http'
+require 'cgi'
+require 'openssl'
+
+module Nicorepo
+  module Request
+    class Auth
+      class LoginError < StandardError; end
+
+      attr_reader :session
+
+      def initialize
+        @session = ''
+      end
+
+      def login(mail, pass)
+        res = Net::HTTP.start(url.host, url.port, use_ssl: true, verify_mode: OpenSSL::SSL::VERIFY_NONE) do |client|
+          client.post(url.path, body(mail, pass))
+        end
+
+        cookies = res.get_fields('set-cookie').map { |str| CGI::Cookie.parse(str) }
+
+        session_cookie = cookies.detect { |c|
+          c['user_session']&.size > 0 && c['user_session'] != ['deleted']
+        }
+        fail LoginError, "invalid mail or password: mail => #{mail}" unless session_cookie
+
+        @session = session_cookie.fetch('user_session').first
+      end
+
+      private
+
+      def url
+        URI('https://secure.nicovideo.jp/secure/login?site=niconico')
+      end
+
+      def body(mail, pass)
+        "mail=#{mail}&password=#{pass}"
+      end
+    end
+
+    class Timeline
+      require 'json'
+
+      def self.fetch(session, cursor: nil)
+        new.fetch(session, cursor)
+      end
+
+      def fetch(session, cursor)
+        url = url(cursor)
+        res = Net::HTTP.start(url.host, url.port) do |client|
+          client.get(url.request_uri, header(session))
+        end
+
+        fail "Request timeline failed !! #{res.inspect}: #{res.to_hash}" if res.code != '200'
+
+        JSON.parse(res.body)
+      end
+
+      private
+
+      def header(session)
+        {
+          'Cookie' => CGI::Cookie.new('user_session', session).to_s,
+          'Connection' => 'keep-alive',
+        }
+      end
+
+      def url(cursor)
+        params = {
+          client_app: 'pc_myrepo',
+          cursor: cursor,
+        }.compact
+
+        URI('http://www.nicovideo.jp/api/nicorepo/timeline/my/all').tap do |url|
+          url.query = params.map { |k, v| "#{CGI.escape(k.to_s)}=#{v}" }.join('&')
+        end
+      end
+    end
+  end
+end

data/lib/nicorepo/version.rb

@@ -1,3 +1,3 @@
-class Nicorepo
-  VERSION = "0.0.8"
+module Nicorepo
+  VERSION = "1.0.0"
 end

data/nicorepo.gemspec

@@ -7,8 +7,8 @@ Gem::Specification.new do |spec|
   spec.version       = Nicorepo::VERSION
   spec.authors       = ["upinetree"]
   spec.email         = ["upinetree@gmail.com"]
-  spec.summary       = %q{Simple nicorepo scraper}
-  spec.description   = %q{Scrapes and filters nicorepo reports from nicovideo.}
+  spec.summary       = %q{Simple nicorepo client}
+  spec.description   = %q{Provides a simple API client for Nicorepo (on nicovideo.jp).}
   spec.homepage      = "https://github.com/upinetree/nicorepo"
   spec.license       = "MIT"
 
@@ -20,16 +20,17 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.required_ruby_version = ">= 2.0.0"
+  spec.required_ruby_version = ">= 2.3.0"
 
-  spec.add_dependency "mechanize", "~> 2.7"
   spec.add_dependency "launchy", "~> 2.4"
   spec.add_dependency "netrc", "~> 0.7"
   spec.add_dependency "thor", "~> 0.19"
 
-  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "bundler"
   spec.add_development_dependency "pry"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"
+  spec.add_development_dependency "webmock"
+  spec.add_development_dependency "vcr"
 end
 

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: nicorepo
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 1.0.0
 platform: ruby
 authors:
 - upinetree
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-05-30 00:00:00.000000000 Z
+date: 2017-07-13 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: mechanize
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.7'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.7'
 - !ruby/object:Gem::Dependency
   name: launchy
   requirement: !ruby/object:Gem::Requirement
@@ -70,16 +56,16 @@ dependencies:
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -122,7 +108,35 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: Scrapes and filters nicorepo reports from nicovideo.
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Provides a simple API client for Nicorepo (on nicovideo.jp).
 email:
 - upinetree@gmail.com
 executables:
@@ -141,11 +155,13 @@ files:
 - bin/nicorepo
 - exe/nicorepo
 - lib/nicorepo.rb
-- lib/nicorepo/cli/cli.rb
-- lib/nicorepo/cli/config.rb
-- lib/nicorepo/parser.rb
+- lib/nicorepo/cli.rb
+- lib/nicorepo/cli/configuration.rb
+- lib/nicorepo/client.rb
+- lib/nicorepo/filter.rb
+- lib/nicorepo/page.rb
 - lib/nicorepo/report.rb
-- lib/nicorepo/reports.rb
+- lib/nicorepo/request.rb
 - lib/nicorepo/version.rb
 - nicorepo.gemspec
 homepage: https://github.com/upinetree/nicorepo
@@ -160,7 +176,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.0.0
+      version: 2.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -171,5 +187,5 @@ rubyforge_project:
 rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
-summary: Simple nicorepo scraper
+summary: Simple nicorepo client
 test_files: []

data/lib/nicorepo/parser.rb

@@ -1,87 +0,0 @@
-class Nicorepo
-  class Parser
-    def initialize(agent)
-      @agent = agent
-    end
-
-    def parse_page(url)
-      page = @agent.get(url)
-      {
-        reports_attrs: parse_reports(page),
-        next_url: next_url(page)
-      }
-    end
-
-    private
-
-    def parse_reports(page)
-      nodes = report_nodes(page)
-      nodes.map { |node| report_attrs(node) }
-    end
-
-    def report_attrs(node)
-      {
-        body:   parse_body(node),
-        title:  parse_title(node),
-        url:    parse_url(node),
-        author: parse_author(node),
-        kind:   parse_kind(node),
-        date:   parse_date(node)
-      }
-    rescue => e
-      error_report(node, e)
-    end
-
-    def report_nodes(page)
-      page.search('.timeline/.log')
-    end
-
-    def next_url(page)
-      page.search('.next-page-link').first['href']
-    end
-
-    def parse_body(node)
-      node.search('.log-body').first.inner_text.gsub(/(\t|\r|\n)/, "")
-    end
-
-    def parse_title(node)
-      node.search('.log-target-info/a').first.inner_text
-    end
-
-    def parse_url(node)
-      node.search('.log-target-info/a').first['href']
-    end
-
-    def parse_author(node)
-      node.search('.log-body/a').first.inner_text
-    end
-
-    # example: 'log.log-community-video-upload' -> 'community-video-upload'
-    def parse_kind(node)
-      cls = node['class']
-      trim = 'log-'
-
-      index = cls.index(/#{trim}/)
-      return cls if index.nil?
-
-      index += trim.size
-      cls[index..cls.size]
-    end
-
-    def parse_date(node)
-      d = node.search('.log-footer-date/time').first['datetime']
-      Time.xmlschema(d).localtime
-    end
-
-    def error_report(node, e)
-      {
-        body:   node.inner_html.gsub(/\r|\n|\t/, ''),
-        title:  "An exception occured: #{e.message}\n#{e.backtrace}",
-        url:    '',
-        author: '',
-        kind:   Report::ERROR_KIND,
-        date:   ''
-      }
-    end
-  end
-end

data/lib/nicorepo/reports.rb

@@ -1,78 +0,0 @@
-require 'nicorepo/report'
-require 'forwardable'
-
-class Nicorepo
-  class Reports
-    extend Forwardable
-
-    TOP_URL = 'http://www.nicovideo.jp/my/top'
-
-    attr_reader :reports
-    def_delegators :@reports, :size
-
-    def initialize(parser)
-      @parser = parser
-      @reports = []
-    end
-
-    def fetch(request_num, limit_page, since: nil)
-      filter = {}
-      filter[:kind] = selected_kind
-      filter[:since] =
-        case since
-        when String
-          Time.parse since
-        when Time
-          since
-        else
-          nil
-        end
-      @reports = fetch_recursively(request_num, limit_page, filter)
-    end
-
-    private
-
-    def selected_kind
-      nil
-    end
-
-    def fetch_recursively(request_num, limit_page, filter = {}, url = TOP_URL)
-      return [] unless limit_page > 0
-
-      # fetch current page reports
-      page = @parser.parse_page(url)
-      reports = page[:reports_attrs].map { |attrs| Report.new(attrs) }
-
-      if filter[:since]
-        reach_oldest_page = (reports.last.date < filter[:since])
-        reports.reject! { |report| report.date < filter[:since] }
-      end
-      reports.select! { |report| report.kind =~ /#{filter[:kind]}|#{Report::ERROR_KIND}/ } if filter[:kind]
-
-      return reports[0, request_num] if reports.size >= request_num
-      return reports if filter[:since] && reach_oldest_page
-
-      # recursively fetch next reports
-      next_reports = fetch_recursively(request_num - reports.size, limit_page - 1, filter, page[:next_url])
-      reports += next_reports unless next_reports.nil?
-      reports
-    end
-  end
-
-  class VideoReports < Reports
-    private
-
-    def selected_kind
-      'video-upload'
-    end
-  end
-
-  class LiveReports < Reports
-    private
-
-    def selected_kind
-      'live'
-    end
-  end
-end
-