harunica 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ee4c6d19662c8be0e7fd0ffe472f9b3426f6d35d
-  data.tar.gz: b30bac0a332e19a8a5e22fd8ff848065836bd7ee
+  metadata.gz: cc57b37d7333f8c6ffc61d6ea007c3f531fb1d37
+  data.tar.gz: e0b398ca645b1fafb851d7c71fb587bbdbb89c4c
 SHA512:
-  metadata.gz: d74519046fd2672bc76784914f0720dbe8bee6c3ae0bc1f8d27a37d52ec26f6491c16c06b197ba3ed0425710972bbf17485113ed9b62c67728e8433d907ab7fa
-  data.tar.gz: 61284f26b82837aeb2a9329ffcf39e0ccaed18f4b3a7d7d153418a5e534992cb099fc319f04a20b95b02685a1085f303740a50faf723aff73cdf38427d8e0bd7
+  metadata.gz: c564b55142637589809e7c135a3dbb2f294129ffe25dba7e2cbcc50efd8201f906063b41bb71cd6fbc7f9c67e1fa3d97a630fd8281790e36e539eea5370a6b37
+  data.tar.gz: 416d39557acfcb5951fb6b15453d60f4e032c5f626e322c9fa67e9a9b962c18479d3995beca13b9500d5a8737c8ede56d478a5af0ab26f718a30d41825e68094

data/harunica.gemspec

@@ -24,6 +24,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.10"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec"
+  spec.add_development_dependency "glint"
 
   spec.add_dependency "nokogiri"
 end

data/lib/harunica/scraping/html.rb

@@ -1,13 +1,17 @@
+require 'nokogiri'
+require 'open-uri'
+
 module Harunica
   module Scraping
     class Html
-      def initialize(charset, text)
-        @charset = charset
-        @text = text
+      attr_reader :url
+
+      def initialize(url)
+        @url = Url.new(url)
       end
 
       def doc
-        @doc ||= ::Nokogiri::HTML.parse(@text, nil, @charset)
+        @doc ||= ::Nokogiri::HTML(::Kernel.open(url, &:read), nil, 'utf-8')
       end
     end
   end

data/lib/harunica/scraping/list_page.rb

@@ -1,4 +1,3 @@
-require 'open-uri'
 require 'nokogiri'
 
 module Harunica
@@ -23,31 +22,44 @@ module Harunica
         'a.pagerBtn.switchingBtn'
       ]
 
-      attr_reader :url
+      attr_reader :html
 
-      def initialize(url = INITIAL_URL)
-        @url = url
+      def initialize(html)
+        @html = html || Html.new(url: INITIAL_URL)
       end
 
-      def html
-        @html ||= ::Kernel.open(url) { |f| Html.new(f.charset, f.read) }
+      def video_pages
+        @video_pages ||= html.doc.css(VIDEO_SELECTORS.join(' ')).map do |e|
+          url = html.url.base + e.css('.itemContent .itemTitle a')[0].attr('href')
+          VideoPage.new(Html.new(url))
+        end
       end
 
-      def videos
-        @videos ||= html.doc.css(VIDEO_SELECTORS.join(' '))
+      # 現在のページ以降のすべてのページの検索結果に含まれる動画のリストを取得する.
+      def all_video_pages
+        current = self
+        a = []
+        while current
+          a += current.video_pages
+          current = current.next
+        end
+        a
       end
 
       def next_link
-        @next_link ||= html.doc.css(NEXT_LINK_SELECTOR.join(' ')).select { |e| e.content == '次へ' }
+        @next_link ||= begin
+          e = html.doc.css(NEXT_LINK_SELECTOR.join(' ')).select { |e| e.content == '次へ' }[0]
+          e && e.attr('href')
+        end
       end
 
       def next?
-        !next_link.empty?
+        !!next_link
       end
 
       def next
         if next?
-          self.class.new(next_link[0].attr('href'))
+          self.class.new(Html.new(next_link))
         else
           nil
         end

data/lib/harunica/scraping/url.rb

@@ -0,0 +1,16 @@
+module Harunica
+  module Scraping
+    class Url < ::String
+      def initialize(s)
+        super s.to_s
+      end
+
+      def base
+        @base ||= ::URI.split(self).instance_exec do
+          base = "#{self[0]}://#{self[2]}"
+          Url.new(self[3] ? "#{base}:#{self[3]}" : base)
+        end
+      end
+    end
+  end
+end

data/lib/harunica/scraping/video_page.rb

@@ -0,0 +1,23 @@
+module Harunica
+  module Scraping
+    class VideoPage
+      AUTHOR_SELECTORS = [
+        '#PAGEBODY',
+        '.wAdjust',
+        '.score-wrap',
+        '.score-item[itemprop="author"]',
+        'strong[itemprop="name"]'
+      ]
+
+      attr_reader :html
+
+      def initialize(html)
+        @html = html
+      end
+
+      def author
+        @author ||= html.doc.css(AUTHOR_SELECTORS.join(' '))[0].content
+      end
+    end
+  end
+end

data/lib/harunica/scraping.rb

@@ -2,6 +2,7 @@ module Harunica
   module Scraping
     autoload :Html, __dir__ + '/scraping/html'
     autoload :ListPage, __dir__ + '/scraping/list_page'
-    autoload :Videoset, __dir__ + '/scraping/videoset'
+    autoload :Url, __dir__ + '/scraping/url'
+    autoload :VideoPage, __dir__ + '/scraping/video_page'
   end
 end

data/lib/harunica/version.rb

@@ -1,3 +1,3 @@
 module Harunica
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: harunica
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - mosop
@@ -53,6 +53,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: glint
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -88,7 +102,8 @@ files:
 - lib/harunica/scraping.rb
 - lib/harunica/scraping/html.rb
 - lib/harunica/scraping/list_page.rb
-- lib/harunica/scraping/videoset.rb
+- lib/harunica/scraping/url.rb
+- lib/harunica/scraping/video_page.rb
 - lib/harunica/version.rb
 homepage: https://github.com/mosop/harunica-gem
 licenses:

data/lib/harunica/scraping/videoset.rb

@@ -1,16 +0,0 @@
-module Harunica
-  module Scraping
-    class Videoset
-      def items
-        @videos ||= begin
-          page = ListPage.new
-          a = page.videos
-          while page = page.next
-            a += page.videos
-          end
-          a
-        end
-      end
-    end
-  end
-end