xvideos_helper 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 1.9.3

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in xvideos_helper.gemspec
+gemspec
+gem 'nokogiri' # スクレイピング

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 YuheiNakasaka
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,85 @@
+# XvideosHelper
+
+XvideosHelper is a gem to support for adult site creater.This gem provides xvideo's data like movie url or movie page url for you with scraping the site easily.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'xvideos_helper'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install xvideos_helper
+
+## Usage
+
+### movies_of(target_url)
+
+- Get movie data
+
+```ruby
+    require 'xvideos_helper'
+    adult = XvideosHelper::Client.new
+    movie_data = adult.movies_of('http://www.xvideos.com') # default 35 objects
+
+    adult.movies_limit = 10
+    movie_data = adult.movies_of('http://www.xvideos.com') # return 10 objects
+```
+
+- return
+
+```ruby
+    {
+      0 => {
+        "movie_page_url"=> "http://jp.xvideos.com/video2017657/0/jp_kyoko_ayana_qc05-02_by_zeus4096_asian_cumshots_asian_swallow_japanese_chinese",
+        "movie_thumnail_url"=> "http://img100.xvideos.com/videos/thumbs/46/a0/69/46a069b72731e3c22ddf917d9fb1cbca/46a069b72731e3c22ddf917d9fb1cbca.4.jpg",
+        "description"=>"Jp Kyoko Ayana Qc05-02 By Zeus4096 asian  ...",
+        "duration"=>"(19min)",
+        "movie_quality"=>"Pornquality:98%"
+      },
+      1 => {
+        ...
+      }
+    }
+```
+
+### tag_data_lists
+
+- Get all tags data
+
+```ruby
+    require 'xvideos_helper'
+    sexy = XvideosHelper::Client.new
+    tags_data = sexy.tag_data_lists # default 1997 objects
+
+    sexy.tags_limit = 10
+    tags_data = sexy.tag_data_lists # return 10 objects
+```
+
+- return
+
+```ruby
+    {
+      0 => {
+        tag_name => 'sex'
+        tag_url => 'http://jp.xvideos.com/tags/sex'
+        tag_count => 320165
+      },
+      1 => {
+        ...
+      },
+    }
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/xvideos_helper

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+
+require 'xvideos_helper'

data/lib/xvideos_helper/client.rb

@@ -0,0 +1,41 @@
+# coding: utf-8
+require 'xvideos_helper/env'
+require 'xvideos_helper/crawler'
+
+module XvideosHelper
+  class Client
+    attr_accessor :domain, :tag_url
+    def initialize
+      @crawler = XvideosHelper::Crawler.new
+      @domain ||= Env::XVIDES_URL_JP
+      @tag_url ||= Env::XVIDES_TAG_URL
+    end
+
+    # get movie information according to target
+    def movies_of(target)
+      begin
+        return @crawler.get_data_from(target,'movie')
+      rescue Exception => e
+        raise e
+      end
+    end
+
+    # tag's url lists
+    def tag_data_lists
+      begin
+        return @crawler.get_data_from(@tag_url,'taglist')
+      rescue Exception => e
+        raise e
+      end
+    end
+
+    def movies_limit=(limit)
+      @crawler.movies_limit = limit
+    end
+
+    def tags_limit=(limit)
+      @crawler.tags_limit = limit
+    end
+
+  end
+end

data/lib/xvideos_helper/crawler.rb

@@ -0,0 +1,101 @@
+# coding: utf-8
+require 'open-uri'
+require "nokogiri"
+
+module XvideosHelper
+  class Crawler
+    attr_accessor :movies_limit,:tags_limit
+    def initialize
+      @domain ||= Env::XVIDES_URL_JP
+      @movies_limit ||= -1
+      @tags_limit ||= -1
+    end
+
+    def get_data_from(url,from)
+      begin
+        source = html(url)
+        if from == 'movie'
+          return parsed_movie_data(source)
+        elsif from == 'taglist'
+          return parsed_tag_data(source)
+        else
+          return {}
+        end
+      rescue Exception => e
+        raise e
+      end
+    end
+
+private
+    def html(url)
+      begin
+        return Nokogiri.HTML(open(url).read)
+      rescue Exception => e
+        raise e
+      end
+    end
+
+    # main crawler
+    def parsed_movie_data(data)
+      parsed_data = {}
+      index       = 0
+
+      data.xpath('//div[@class="thumbBlock"]/div[@class="thumbInside"]').each do |post|
+        begin
+          # limit
+          break if @movies_limit == index
+          parsed_data[index] = {}
+          # thumbnail infomation
+          post.search('div[@class="thumb"]/a').each do |a|
+            parsed_data[index]['movie_page_url'] = "#{@domain}#{a.attribute('href').value}"
+            parsed_data[index]['movie_thumnail_url'] = "#{a.children.attribute('src').value}"
+          end
+
+          # if script tag is contained
+          post.search('script').each do |elm|
+            parsed_data[index]['movie_page_url'] = @domain + (elm.children[0].content.match(/href="(.+?)">/))[1]
+            parsed_data[index]['movie_thumnail_url'] = (elm.children[0].content.match(/src="(.+?)"/))[1]
+            parsed_data[index]['description'] = (elm.children[0].content.match(/<p><a href=".+">(.+)<\/a><\/p>/))[1]
+          end
+
+          # description
+          post.search('p/a').each do |a|
+            parsed_data[index]['description'] = a.inner_text
+          end
+
+          # metadata
+          post.search('p[@class="metadata"]/span[@class="bg"]').each do |span|
+            text = span.inner_text.gsub(/(\t|\s|\n)+/,'')
+            parsed_data[index]['duration'] = (text.match(/\(.+\)/))[0]
+            parsed_data[index]['movie_quality'] = text.sub(/\(.+\)/,'')
+          end
+          index += 1
+        rescue Exception => e
+          raise e
+        end
+      end
+      return parsed_data
+    end
+
+    # tag list crawler
+    def parsed_tag_data(data)
+        parsed_data = {}
+        index       = 0
+        data.xpath('//div[@id="main"]/ul[@id="tags"]/li').each do |li|
+          begin
+            # limit
+            break if @tags_limit == index
+            parsed_data[index] = {}
+            # tag info
+            parsed_data[index]['tag_name'] = li.children.children.inner_text
+            parsed_data[index]['tag_url'] = "#{@domain}#{li.children.attribute('href').value}"
+            parsed_data[index]['tag_count'] = li.inner_text.sub(/.+\s/,'')
+            index += 1
+          rescue Exception => e
+            raise e
+          end
+        end
+        return parsed_data
+    end
+  end
+end

data/lib/xvideos_helper/env.rb

@@ -0,0 +1,5 @@
+# coding: utf-8
+module Env
+    XVIDES_URL_JP = 'http://jp.xvideos.com'
+    XVIDES_TAG_URL = "#{XVIDES_URL_JP}/tags"
+end

data/lib/xvideos_helper/version.rb

@@ -0,0 +1,3 @@
+module XvideosHelper
+  VERSION = "0.0.1"
+end

data/lib/xvideos_helper.rb

@@ -0,0 +1,5 @@
+require "xvideos_helper/version"
+require "xvideos_helper/client"
+
+module XvideosHelper
+end

data/spec/lib/client_spec.rb

@@ -0,0 +1,96 @@
+require 'spec_helper'
+
+describe "XvideosHelper::Client" do
+  context 'movies_of' do
+    it 'return valid values' do
+      @xh = XvideosHelper::Client.new
+      lists = @xh.movies_of("http://jp.xvideos.com/")
+      lists.count.should > 0
+      lists.each do |key,list|
+        list["movie_page_url"].should match(/^http:\/\/.+/)
+        list["movie_thumnail_url"].should match(/^http:\/\/.+/)
+        list["description"].should_not be_nil
+        list["duration"].should_not be_nil
+        list["movie_quality"].should_not be_nil
+      end
+    end
+
+    it 'raises error if invalid url' do
+      @xh = XvideosHelper::Client.new
+      lambda {@xh.movies_of("invalid://jp.xvideos.com/")}.should raise_error
+    end
+  end
+
+  context 'tag_data_lists' do
+    it 'return valid values' do
+      @xh = XvideosHelper::Client.new
+      @xh.tag_data_lists.count.should > 0
+      @xh.tag_data_lists.each do |key,list|
+        list['tag_name'].should_not be_nil
+        list['tag_url'].should match(/^http:\/\/.+/)
+        list['tag_count'].should_not be_nil
+      end
+    end
+
+    it 'raises error if invalid url' do
+      @xh = XvideosHelper::Client.new
+      @xh.tag_url = 'aaaaaaaa'
+      lambda {@xh.tag_data_lists}.should raise_error
+    end
+  end
+
+  context 'movies_limit=' do
+    it 'changes limit to 1' do
+      @xh = XvideosHelper::Client.new
+      @xh.movies_limit = 1
+      lists = @xh.movies_of("http://jp.xvideos.com/")
+      lists.count.should == 1
+      lists.each do |key,list|
+        list["movie_page_url"].should match(/^http:\/\/.+/)
+        list["movie_thumnail_url"].should match(/^http:\/\/.+/)
+        list["description"].should_not be_nil
+        list["duration"].should_not be_nil
+        list["movie_quality"].should_not be_nil
+      end
+    end
+
+    it 'changes limit to 0' do
+      @xh = XvideosHelper::Client.new
+      @xh.movies_limit = 0
+      lists = @xh.movies_of("http://jp.xvideos.com/")
+      lists.count.should == 0
+      lists.each do |key,list|
+        list["movie_page_url"].should match(/^http:\/\/.+/)
+        list["movie_thumnail_url"].should match(/^http:\/\/.+/)
+        list["description"].should_not be_nil
+        list["duration"].should_not be_nil
+        list["movie_quality"].should_not be_nil
+      end
+    end
+  end
+
+  context 'tags_limit=' do
+    it 'changes limit to 1' do
+      @xh = XvideosHelper::Client.new
+      @xh.tags_limit = 1
+      @xh.tag_data_lists.count.should == 1
+      @xh.tag_data_lists.each do |key,list|
+        list['tag_name'].should_not be_nil
+        list['tag_url'].should match(/^http:\/\/.+/)
+        list['tag_count'].should_not be_nil
+      end
+    end
+
+    it 'changes limit to 0' do
+      @xh = XvideosHelper::Client.new
+      @xh.tags_limit = 0
+      @xh.tag_data_lists.count.should == 0
+      @xh.tag_data_lists.each do |key,list|
+        list['tag_name'].should_not be_nil
+        list['tag_url'].should match(/^http:\/\/.+/)
+        list['tag_count'].should_not be_nil
+      end
+    end
+  end
+
+end

data/spec/lib/crawler_spec.rb

@@ -0,0 +1,97 @@
+require 'spec_helper'
+
+describe "XvideosHelper::Crawler" do
+  context 'get_data_from' do
+    it 'return valid movie information' do
+      @xh = XvideosHelper::Crawler.new
+      lists = @xh.get_data_from("http://jp.xvideos.com/",'movie')
+      lists.count.should > 0
+      lists.each do |key,list|
+        list["movie_page_url"].should match(/^http:\/\/.+/)
+        list["movie_thumnail_url"].should match(/^http:\/\/.+/)
+        list["description"].should_not be_nil
+        list["duration"].should_not be_nil
+        list["movie_quality"].should_not be_nil
+      end
+    end
+
+    it 'return 0 movie information' do
+      @xh = XvideosHelper::Crawler.new
+      @xh.movies_limit = 0
+      lists = @xh.get_data_from("http://jp.xvideos.com/",'movie')
+      lists.count.should == 0
+      lists.each do |key,list|
+        list["movie_page_url"].should match(/^http:\/\/.+/)
+        list["movie_thumnail_url"].should match(/^http:\/\/.+/)
+        list["description"].should_not be_nil
+        list["duration"].should_not be_nil
+        list["movie_quality"].should_not be_nil
+      end
+    end
+
+    it 'return 1 movie information' do
+      @xh = XvideosHelper::Crawler.new
+      @xh.movies_limit = 1
+      lists = @xh.get_data_from("http://jp.xvideos.com/",'movie')
+      lists.count.should == 1
+      lists.each do |key,list|
+        list["movie_page_url"].should match(/^http:\/\/.+/)
+        list["movie_thumnail_url"].should match(/^http:\/\/.+/)
+        list["description"].should_not be_nil
+        list["duration"].should_not be_nil
+        list["movie_quality"].should_not be_nil
+      end
+    end
+
+    it 'return valid tag lists' do
+      @xh = XvideosHelper::Crawler.new
+      lists = @xh.get_data_from("http://jp.xvideos.com/tags",'taglist')
+      lists.count.should > 0
+      lists.each do |key,list|
+        list['tag_name'].should_not be_nil
+        list['tag_url'].should match(/^http:\/\/.+/)
+        list['tag_count'].should_not be_nil
+      end
+    end
+
+    it 'return 0 tag list' do
+      @xh = XvideosHelper::Crawler.new
+      @xh.tags_limit = 0
+      lists = @xh.get_data_from("http://jp.xvideos.com/tags",'taglist')
+      lists.count.should == 0
+      lists.each do |key,list|
+        list['tag_name'].should_not be_nil
+        list['tag_url'].should match(/^http:\/\/.+/)
+        list['tag_count'].should_not be_nil
+      end
+    end
+
+    it 'return 1 tag list' do
+      @xh = XvideosHelper::Crawler.new
+      @xh.tags_limit = 1
+      lists = @xh.get_data_from("http://jp.xvideos.com/tags",'taglist')
+      lists.count.should == 1
+      lists.each do |key,list|
+        list['tag_name'].should_not be_nil
+        list['tag_url'].should match(/^http:\/\/.+/)
+        list['tag_count'].should_not be_nil
+      end
+    end
+
+    it 'return empty object if undifined name or the other objects' do
+      @xh = XvideosHelper::Crawler.new
+      lists = @xh.get_data_from("http://jp.xvideos.com/",'undifined_name')
+      lists.should == {}
+      lists = @xh.get_data_from("http://jp.xvideos.com/",{})
+      lists.should == {}
+      lists = @xh.get_data_from("http://jp.xvideos.com/",[])
+      lists.should == {}
+    end
+
+    it 'raises error if invalid url' do
+      @xh = XvideosHelper::Crawler.new
+      lambda {@xh.movies_of("invalid://jp.xvideos.com/",'movie')}.should raise_error
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'xvideos_helper'

data/spec/xvideos_helper_spec.rb

@@ -0,0 +1,7 @@
+require 'spec_helper'
+
+describe XvideosHelper do
+  it 'should have a version number' do
+    XvideosHelper::VERSION.should_not be_nil
+  end
+end

data/xvideos_helper.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'xvideos_helper/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "xvideos_helper"
+  spec.version       = XvideosHelper::VERSION
+  spec.authors       = ["YuheiNakasaka"]
+  spec.email         = ["yuhei.nakasaka@gmail.com"]
+  spec.description   = %q{xvideos_helper is a gem to support for adult site creater.}
+  spec.summary       = %q{This gem get movie infomation from xvideo.com.}
+  spec.homepage      = "https://github.com/YuheiNakasaka/xvideos_helper"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+end

metadata

@@ -0,0 +1,117 @@
+--- !ruby/object:Gem::Specification
+name: xvideos_helper
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- YuheiNakasaka
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-11-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: xvideos_helper is a gem to support for adult site creater.
+email:
+- yuhei.nakasaka@gmail.com
+executables:
+- xvideos_helper
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .travis.yml
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/xvideos_helper
+- lib/xvideos_helper.rb
+- lib/xvideos_helper/client.rb
+- lib/xvideos_helper/crawler.rb
+- lib/xvideos_helper/env.rb
+- lib/xvideos_helper/version.rb
+- spec/lib/client_spec.rb
+- spec/lib/crawler_spec.rb
+- spec/spec_helper.rb
+- spec/xvideos_helper_spec.rb
+- xvideos_helper.gemspec
+homepage: https://github.com/YuheiNakasaka/xvideos_helper
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: This gem get movie infomation from xvideo.com.
+test_files:
+- spec/lib/client_spec.rb
+- spec/lib/crawler_spec.rb
+- spec/spec_helper.rb
+- spec/xvideos_helper_spec.rb