nicoquery 0.1.7 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
 - data/.rspec +2 -0
 - data/Gemfile.lock +1 -1
 - data/lib/nicoquery/object/mylist.rb +49 -12
 - data/lib/nicoquery/version.rb +1 -1
 - data/spec/api/mylist_rss_spec.rb +8 -9
 - data/spec/crawler/bulk_scraping_spec.rb +1 -1
 - data/spec/crawler/tag_search_spec.rb +14 -0
 - data/spec/fixture/tag_search_yukkuri_page1.rb +412 -0
 - data/spec/fixture/tag_search_yukkuri_page2.rb +412 -0
 - data/spec/nicoquery_spec.rb +26 -21
 - data/spec/object/mylist_spec.rb +51 -9
 - data/spec/object/tag_search_rss_spec.rb +5 -0
 - data/spec/spec_helper.rb +8 -0
 - metadata +7 -2
 
    
        checksums.yaml
    CHANGED
    
    | 
         @@ -1,7 +1,7 @@ 
     | 
|
| 
       1 
1 
     | 
    
         
             
            ---
         
     | 
| 
       2 
2 
     | 
    
         
             
            SHA1:
         
     | 
| 
       3 
     | 
    
         
            -
              metadata.gz:  
     | 
| 
       4 
     | 
    
         
            -
              data.tar.gz:  
     | 
| 
      
 3 
     | 
    
         
            +
              metadata.gz: 1d97bc8ec57f8b66e8c9ea690e626108d4623e93
         
     | 
| 
      
 4 
     | 
    
         
            +
              data.tar.gz: 83772ec763bc93b2f188101582fda451d4b1b6eb
         
     | 
| 
       5 
5 
     | 
    
         
             
            SHA512:
         
     | 
| 
       6 
     | 
    
         
            -
              metadata.gz:  
     | 
| 
       7 
     | 
    
         
            -
              data.tar.gz:  
     | 
| 
      
 6 
     | 
    
         
            +
              metadata.gz: 8ea629c1510df67bf7932eeca3787f79bd3b827e3e9864a6e9134d8c04cbb51a65466e75a6c62735c114231c4c8dc194bd4c42e5955eeb1c114096a251b4e05d
         
     | 
| 
      
 7 
     | 
    
         
            +
              data.tar.gz: eddde06634911d353a717a9fc7c93f13e4cffb7dcd8a8843a85187688ea5896b988317b0bf6d19dea5916ada42e274e4341e4c3ff517c37df321783af90fedea
         
     | 
    
        data/.rspec
    ADDED
    
    
    
        data/Gemfile.lock
    CHANGED
    
    
| 
         @@ -18,21 +18,21 @@ module NicoQuery 
     | 
|
| 
       18 
18 
     | 
    
         
             
                    'last_build_date',
         
     | 
| 
       19 
19 
     | 
    
         
             
                    'creator',
         
     | 
| 
       20 
20 
     | 
    
         
             
                  ].each do |field_name|
         
     | 
| 
       21 
     | 
    
         
            -
                    define_method(field_name)  
     | 
| 
      
 21 
     | 
    
         
            +
                    define_method(field_name) do
         
     | 
| 
      
 22 
     | 
    
         
            +
                      if available?
         
     | 
| 
      
 23 
     | 
    
         
            +
                        @source_object.meta.send field_name
         
     | 
| 
      
 24 
     | 
    
         
            +
                      else
         
     | 
| 
      
 25 
     | 
    
         
            +
                        nil
         
     | 
| 
      
 26 
     | 
    
         
            +
                      end
         
     | 
| 
      
 27 
     | 
    
         
            +
                    end
         
     | 
| 
       22 
28 
     | 
    
         
             
                  end
         
     | 
| 
       23 
29 
     | 
    
         | 
| 
       24 
30 
     | 
    
         
             
                  def initialize(mylist_id)
         
     | 
| 
       25 
31 
     | 
    
         
             
                    @movies = []
         
     | 
| 
       26 
32 
     | 
    
         
             
                    @mylist_id = mylist_id
         
     | 
| 
       27 
     | 
    
         
            -
                    @ 
     | 
| 
       28 
     | 
    
         
            -
                    @ 
     | 
| 
       29 
     | 
    
         
            -
             
     | 
| 
       30 
     | 
    
         
            -
                    return if @hash.items.nil?
         
     | 
| 
       31 
     | 
    
         
            -
                    @hash.items.map do |item|
         
     | 
| 
       32 
     | 
    
         
            -
                      movie = NicoQuery::Object::Movie.new item.video_id.presence || item.thread_id
         
     | 
| 
       33 
     | 
    
         
            -
                      movie.set_mylist_rss_source item
         
     | 
| 
       34 
     | 
    
         
            -
                      @movies.push movie
         
     | 
| 
       35 
     | 
    
         
            -
                    end
         
     | 
| 
      
 33 
     | 
    
         
            +
                    @response_xml = (NicoQuery::Api::MylistRSS.new mylist_id).get
         
     | 
| 
      
 34 
     | 
    
         
            +
                    @source_object = source_object
         
     | 
| 
      
 35 
     | 
    
         
            +
                    set_hash_items
         
     | 
| 
       36 
36 
     | 
    
         
             
                  end
         
     | 
| 
       37 
37 
     | 
    
         | 
| 
       38 
38 
     | 
    
         
             
                  def available?
         
     | 
| 
         @@ -40,11 +40,48 @@ module NicoQuery 
     | 
|
| 
       40 
40 
     | 
    
         
             
                  end
         
     | 
| 
       41 
41 
     | 
    
         | 
| 
       42 
42 
     | 
    
         
             
                  def forbidden?
         
     | 
| 
       43 
     | 
    
         
            -
                    @ 
     | 
| 
      
 43 
     | 
    
         
            +
                    @response_xml[:status_code] == 403
         
     | 
| 
       44 
44 
     | 
    
         
             
                  end
         
     | 
| 
       45 
45 
     | 
    
         | 
| 
       46 
46 
     | 
    
         
             
                  def exist?
         
     | 
| 
       47 
     | 
    
         
            -
                    @ 
     | 
| 
      
 47 
     | 
    
         
            +
                    @response_xml[:status_code] != 404
         
     | 
| 
      
 48 
     | 
    
         
            +
                  end
         
     | 
| 
      
 49 
     | 
    
         
            +
             
     | 
| 
      
 50 
     | 
    
         
            +
                  private
         
     | 
| 
      
 51 
     | 
    
         
            +
             
     | 
| 
      
 52 
     | 
    
         
            +
                  def source_object
         
     | 
| 
      
 53 
     | 
    
         
            +
                    NicoQuery::ObjectMapper::MylistRSS.new @response_xml[:body].presence || null_response_xml
         
     | 
| 
      
 54 
     | 
    
         
            +
                  end
         
     | 
| 
      
 55 
     | 
    
         
            +
             
     | 
| 
      
 56 
     | 
    
         
            +
                  def set_hash_items
         
     | 
| 
      
 57 
     | 
    
         
            +
                    @source_object.items.each do |item|
         
     | 
| 
      
 58 
     | 
    
         
            +
                      movie = NicoQuery::Object::Movie.new item.video_id.presence || item.thread_id
         
     | 
| 
      
 59 
     | 
    
         
            +
                      movie.set_mylist_rss_source item
         
     | 
| 
      
 60 
     | 
    
         
            +
                      @movies.push movie
         
     | 
| 
      
 61 
     | 
    
         
            +
                    end
         
     | 
| 
      
 62 
     | 
    
         
            +
                  end
         
     | 
| 
      
 63 
     | 
    
         
            +
             
     | 
| 
      
 64 
     | 
    
         
            +
                  def null_response_xml
         
     | 
| 
      
 65 
     | 
    
         
            +
                    <<-EOS
         
     | 
| 
      
 66 
     | 
    
         
            +
            <?xml version="1.0" encoding="utf-8"?>
         
     | 
| 
      
 67 
     | 
    
         
            +
            <rss version="2.0"
         
     | 
| 
      
 68 
     | 
    
         
            +
                 xmlns:dc="http://purl.org/dc/elements/1.1/"
         
     | 
| 
      
 69 
     | 
    
         
            +
                 xmlns:atom="http://www.w3.org/2005/Atom">
         
     | 
| 
      
 70 
     | 
    
         
            +
              <channel>
         
     | 
| 
      
 71 
     | 
    
         
            +
                <title></title>
         
     | 
| 
      
 72 
     | 
    
         
            +
                <link></link>
         
     | 
| 
      
 73 
     | 
    
         
            +
                <atom:link rel="self" type="application/rss+xml" href="http://www.nicovideo.jp/mylist/38350049?rss=2.0"/>
         
     | 
| 
      
 74 
     | 
    
         
            +
                <description></description>
         
     | 
| 
      
 75 
     | 
    
         
            +
                <pubDate></pubDate>
         
     | 
| 
      
 76 
     | 
    
         
            +
                <lastBuildDate></lastBuildDate>
         
     | 
| 
      
 77 
     | 
    
         
            +
                <generator></generator>
         
     | 
| 
      
 78 
     | 
    
         
            +
                <dc:creator></dc:creator>
         
     | 
| 
      
 79 
     | 
    
         
            +
                <language></language>
         
     | 
| 
      
 80 
     | 
    
         
            +
                <copyright></copyright>
         
     | 
| 
      
 81 
     | 
    
         
            +
                <docs></docs>
         
     | 
| 
      
 82 
     | 
    
         
            +
              </channel>
         
     | 
| 
      
 83 
     | 
    
         
            +
            </rss>
         
     | 
| 
      
 84 
     | 
    
         
            +
                    EOS
         
     | 
| 
       48 
85 
     | 
    
         
             
                  end
         
     | 
| 
       49 
86 
     | 
    
         
             
                end
         
     | 
| 
       50 
87 
     | 
    
         
             
              end
         
     | 
    
        data/lib/nicoquery/version.rb
    CHANGED
    
    
    
        data/spec/api/mylist_rss_spec.rb
    CHANGED
    
    | 
         @@ -6,15 +6,14 @@ require 'webmock/rspec' 
     | 
|
| 
       6 
6 
     | 
    
         
             
            describe "mylist_rss" do
         
     | 
| 
       7 
7 
     | 
    
         
             
              describe "when access an existing and public mylist" do
         
     | 
| 
       8 
8 
     | 
    
         
             
                before do
         
     | 
| 
       9 
     | 
    
         
            -
                   
     | 
| 
       10 
     | 
    
         
            -
                   
     | 
| 
       11 
     | 
    
         
            -
             
     | 
| 
       12 
     | 
    
         
            -
             
     | 
| 
       13 
     | 
    
         
            -
             
     | 
| 
       14 
     | 
    
         
            -
             
     | 
| 
       15 
     | 
    
         
            -
             
     | 
| 
       16 
     | 
    
         
            -
             
     | 
| 
       17 
     | 
    
         
            -
                  WebMock.disable!
         
     | 
| 
      
 9 
     | 
    
         
            +
                  WebMock.enable!
         
     | 
| 
      
 10 
     | 
    
         
            +
                  WebMock.stub_request(:get, "http://www.nicovideo.jp/mylist/18266317?numbers=1&rss=2.0").
         
     | 
| 
      
 11 
     | 
    
         
            +
                    with(:headers => {'Accept'=>'*/*; q=0.5, application/xml', 'Accept-Encoding'=>'gzip, deflate', 'User-Agent'=>'Ruby'}).
         
     | 
| 
      
 12 
     | 
    
         
            +
                    to_return(:status => 200, :body => Fixture.mylist_rss_18266317, :headers => {})
         
     | 
| 
      
 13 
     | 
    
         
            +
             
     | 
| 
      
 14 
     | 
    
         
            +
                  mylist_id = 18266317
         
     | 
| 
      
 15 
     | 
    
         
            +
                  @uri = "http://www.nicovideo.jp/mylist/#{mylist_id}?rss=2.0&numbers=1"
         
     | 
| 
      
 16 
     | 
    
         
            +
                  @instance = NicoQuery::Api::MylistRSS.new mylist_id
         
     | 
| 
       18 
17 
     | 
    
         
             
                end
         
     | 
| 
       19 
18 
     | 
    
         | 
| 
       20 
19 
     | 
    
         
             
                describe "get" do
         
     | 
| 
         @@ -1,4 +1,6 @@ 
     | 
|
| 
       1 
1 
     | 
    
         
             
            require "nicoquery/crawler/tag_search"
         
     | 
| 
      
 2 
     | 
    
         
            +
            require "fixture/tag_search_yukkuri_page1"
         
     | 
| 
      
 3 
     | 
    
         
            +
            require "fixture/tag_search_yukkuri_page2"
         
     | 
| 
       2 
4 
     | 
    
         | 
| 
       3 
5 
     | 
    
         | 
| 
       4 
6 
     | 
    
         
             
            describe "NicoQuery:Crawler" do
         
     | 
| 
         @@ -7,6 +9,14 @@ describe "NicoQuery:Crawler" do 
     | 
|
| 
       7 
9 
     | 
    
         
             
                  counter = 0
         
     | 
| 
       8 
10 
     | 
    
         
             
                  @acquired_movies = []
         
     | 
| 
       9 
11 
     | 
    
         | 
| 
      
 12 
     | 
    
         
            +
                  WebMock.stub_request(:get, "http://www.nicovideo.jp/tag/%E3%82%86%E3%81%A3%E3%81%8F%E3%82%8A%E5%AE%9F%E6%B3%81%E3%83%97%E3%83%AC%E3%82%A4?numbers=1&order=a&page=1&rss=2.0&sort=f").
         
     | 
| 
      
 13 
     | 
    
         
            +
                     with(:headers => {'Accept'=>'*/*; q=0.5, application/xml', 'Accept-Encoding'=>'gzip, deflate', 'User-Agent'=>'Ruby'}).
         
     | 
| 
      
 14 
     | 
    
         
            +
                     to_return(:status => 200, :body => Fixture.tag_search_yukkuri_page1, :headers => {})
         
     | 
| 
      
 15 
     | 
    
         
            +
             
     | 
| 
      
 16 
     | 
    
         
            +
                  WebMock.stub_request(:get, "http://www.nicovideo.jp/tag/%E3%82%86%E3%81%A3%E3%81%8F%E3%82%8A%E5%AE%9F%E6%B3%81%E3%83%97%E3%83%AC%E3%82%A4?numbers=1&order=a&page=2&rss=2.0&sort=f").
         
     | 
| 
      
 17 
     | 
    
         
            +
                     with(:headers => {'Accept'=>'*/*; q=0.5, application/xml', 'Accept-Encoding'=>'gzip, deflate', 'User-Agent'=>'Ruby'}).
         
     | 
| 
      
 18 
     | 
    
         
            +
                     to_return(:status => 200, :body => Fixture.tag_search_yukkuri_page2, :headers => {})
         
     | 
| 
      
 19 
     | 
    
         
            +
             
     | 
| 
       10 
20 
     | 
    
         
             
                  NicoQuery::Crawler::TagSearch.execute( tag: "ゆっくり実況プレイ",
         
     | 
| 
       11 
21 
     | 
    
         
             
                                                         sort: :published_at,
         
     | 
| 
       12 
22 
     | 
    
         
             
                                                         order: :asc
         
     | 
| 
         @@ -17,6 +27,10 @@ describe "NicoQuery:Crawler" do 
     | 
|
| 
       17 
27 
     | 
    
         
             
                  end
         
     | 
| 
       18 
28 
     | 
    
         
             
                end
         
     | 
| 
       19 
29 
     | 
    
         | 
| 
      
 30 
     | 
    
         
            +
                specify "it returns NicoQuery::Object::Movie instance in the block" do
         
     | 
| 
      
 31 
     | 
    
         
            +
                  expect(@acquired_movies[0]).to be_an_instance_of NicoQuery::Object::Movie
         
     | 
| 
      
 32 
     | 
    
         
            +
                end
         
     | 
| 
      
 33 
     | 
    
         
            +
             
     | 
| 
       20 
34 
     | 
    
         
             
                it "should sorted by published date" do
         
     | 
| 
       21 
35 
     | 
    
         
             
                  expect(@acquired_movies[0].publish_date).to be < @acquired_movies[1].publish_date
         
     | 
| 
       22 
36 
     | 
    
         
             
                end
         
     |