ascraper 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8a95cfbe149bb77c19450a761e899ad8e9811fa2
+  data.tar.gz: 11958eb8b2195a5336ce6e4e4ce055b8df6c5773
+SHA512:
+  metadata.gz: e7fdecb114609b09b2b4f5d3f5f7d0aa74babd5d4c62011adb1342971452b705256cb7614df3b36f0dd875cd7e7a8b30433ee82d1507dcd1703fb1b4ca7f8377
+  data.tar.gz: ae55f31b711ee451c75c10d217ce54ea58ffa7a89ecb3fe03be21bd69d6c942ba58cd1a59805db0d9bfde61b552042199e9615abb62c2fa5f3a506b475885a31

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--warnings
+--require spec_helper

data/.ruby-gemset

@@ -0,0 +1 @@
+scraper

data/.ruby-version

@@ -0,0 +1 @@
+2.1.0

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in comics.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,5 @@
+guard :rspec do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Ahmet
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,19 @@
+# Scraper
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'scraper'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install scraper
+
+## Usage
+
+TODO: Write usage instructions here

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/lib/scraper.rb

@@ -0,0 +1,8 @@
+require "scraper/version"
+require 'nokogiri'
+require 'open-uri'
+require 'ostruct'
+
+["comics", "movies"].each do |inc|
+  require File.join(File.dirname(__FILE__), "scraper", inc)
+end

data/lib/scraper/comics.rb

@@ -0,0 +1,56 @@
+module Scraper
+  class Comics
+
+    def self.all(start_page = 1, last_page = 5)
+      results = []
+      start_page.upto(last_page).each do |i|
+        doc = Nokogiri::HTML(open("http://www.newcomic.org/page/#{i}/"))
+
+        doc.css('.story_short').each do |block|
+          result = OpenStruct.new
+
+          url = block.search('a').map {|a| a['href']}.first
+
+          result.title = block.at_css('.story_h').content
+          result.url = url
+
+          comic = Nokogiri::HTML(open(url))
+
+          result.main_image = main_image(comic)
+          result.other_images = other_images(comic)
+          result.download_link = download_link(comic)
+          result.tags = tags(comic)
+
+          results << result
+        end
+
+      end
+      results
+    end
+
+    def self.main_image(comic)
+      comic.at_css('.story_c').search('img').map {|a| a['src']}.first
+    end
+
+    def self.other_images(comic)
+      comic.at_css('.story_c').search('a').map {|a| a['href']}.collect! {|x| x if %r{\Ahttps?:\/\/.+\.(?:jpe?g|png)\z}.match(x) }.compact
+    end
+
+    def self.download_link(comic)
+      if comic.at_css('.story_c')
+        comic.at_css('.story_c').search('a').map {|a| a['href']}.first
+      end
+    end
+
+    def self.tags(comic)
+      if comic.at_xpath('//*[@id="dle-content"]/div/article/div[2]/p/i')
+        comic.at_xpath('//*[@id="dle-content"]/div/article/div[2]/p/i').content.gsub('Tags: ', '').split(',').collect(&:strip)
+      end
+    end
+
+    def self.last_page
+      Nokogiri::HTML(open('http://www.newcomic.org/')).search('#dle-content > div.basenavi > span.navigation > a:nth-child(12)').first.children[0].to_s.to_i
+    end
+
+  end
+end

data/lib/scraper/movies.rb

@@ -0,0 +1,36 @@
+module Scraper
+  class Movies
+
+    def self.all(start_page = 1, last_page = 5)
+      results = []
+      doc = Nokogiri::HTML(open('http://www.wrzko.eu/movies/'))
+
+      doc.css('.category_block').each do |block|
+        result = OpenStruct.new
+
+        url = block.at_css('.maintitle_base').search('a').map {|a| a['href']}.first
+
+        result.title = block.at_css('.maintitle_base').content.strip
+        result.description = block.at_css('.description').content
+        result.url = url
+
+        movie = Nokogiri::HTML(open(url))
+
+        # [0] NFO [1] Main Image [Rest] Other Images
+        images = movie.at_css('.image').search('a').map {|a| a['href']}.collect! {|x| x if %r{\Ahttps?:\/\/.+\.(?:jpe?g|png)\z}.match(x) }.compact.inspect
+
+        # movie.css('.postarea p').each do |p|
+          # raise p.inspect
+        # end
+
+        results << result
+      end
+      results
+    end
+
+    def self.last_page
+      Nokogiri::HTML(open('http://www.wrzko.eu/movies/')).css('#board_index > div.wp-pagenavi > div > span.pages').first.to_s.split(' ').last.tr('</span>', '')
+    end
+
+  end
+end

data/lib/scraper/version.rb

@@ -0,0 +1,3 @@
+module Scraper
+  VERSION = "0.0.1"
+end

data/scraper.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'scraper/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "ascraper"
+  spec.version       = Scraper::VERSION
+  spec.authors       = ["Ahmet"]
+  spec.email         = ["ahmet@kyan.com"]
+  spec.summary       = %q{A Scraper.}
+  spec.description   = %q{A Scraper.}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "guard"
+  spec.add_development_dependency "guard-rspec"
+  spec.add_dependency "nokogiri"
+end

data/spec/comics_spec.rb

@@ -0,0 +1,41 @@
+require 'rspec'
+require 'spec_helper'
+
+describe Scraper::Comics do
+  describe "Scraping the site" do
+    before(:each) do
+      @comics = Scraper::Comics.all(1,1)
+    end
+
+    it "should return 15 comics for first page" do
+      expect(@comics.count).to eq(15)
+    end
+  end
+
+  describe "Scraping a comic" do
+    before(:all) do
+      @comic = Scraper::Comics.all(1,1).first
+    end
+
+    it "should always return a title" do
+      puts @comic.title
+      expect(@comic.title).not_to be_empty
+    end
+
+    it "should always return a url" do
+      puts @comic.url
+      expect(@comic.url).not_to be_empty
+    end
+
+    it "should always return a main_image" do
+      puts @comic.main_image
+      expect(@comic.main_image).not_to be_empty
+    end
+
+    it "should always return a download_link" do
+      puts @comic.download_link
+      expect(@comic.download_link).not_to be_empty
+    end
+  end
+
+end

data/spec/movies_spec.rb

@@ -0,0 +1,37 @@
+require 'rspec'
+require 'spec_helper'
+
+describe Scraper::Movies do
+
+  describe "Scraping the site" do
+    before(:each) do
+      @movies = Scraper::Movies.all(1,1)
+    end
+
+    it "should return all 10 movies for first page" do
+      expect(@movies.count).to eq(10)
+    end
+  end
+
+  describe "Scraping a movie" do
+    before(:all) do
+      @movie = Scraper::Movies.all(1,1).first
+    end
+
+    it "should always return a title" do
+      puts @movie.title
+      expect(@movie.title).not_to be_empty
+    end
+
+    it "should always return a description" do
+      puts @movie.description
+      expect(@movie.description).not_to be_empty
+    end
+
+    it "should always return a url" do
+      puts @movie.url
+      expect(@movie.url).not_to be_empty
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,80 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause this
+# file to always be loaded, without a need to explicitly require it in any files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, make a
+# separate helper file that requires this one and then use it only in the specs
+# that actually need it.
+#
+# The `.rspec` file also contains a few flags that are not defaults but that
+# users commonly want.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+require_relative '../lib/scraper.rb'
+
+RSpec.configure do |config|
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+=begin
+  # These two settings work together to allow you to limit a spec run
+  # to individual examples or groups you care about by tagging them with
+  # `:focus` metadata. When nothing is tagged with `:focus`, all examples
+  # get run.
+  config.filter_run :focus
+  config.run_all_when_everything_filtered = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = 'doc'
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # Enable only the newer, non-monkey-patching expect syntax.
+    # For more details, see:
+    #   - http://myronmars.to/n/dev-blog/2012/06/rspecs-new-expectation-syntax
+    expectations.syntax = :expect
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Enable only the newer, non-monkey-patching expect syntax.
+    # For more details, see:
+    #   - http://teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+    mocks.syntax = :expect
+
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended.
+    mocks.verify_partial_doubles = true
+  end
+=end
+end

metadata

@@ -0,0 +1,148 @@
+--- !ruby/object:Gem::Specification
+name: ascraper
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Ahmet
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-07-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A Scraper.
+email:
+- ahmet@kyan.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".ruby-gemset"
+- ".ruby-version"
+- Gemfile
+- Guardfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/scraper.rb
+- lib/scraper/comics.rb
+- lib/scraper/movies.rb
+- lib/scraper/version.rb
+- scraper.gemspec
+- spec/comics_spec.rb
+- spec/movies_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: A Scraper.
+test_files:
+- spec/comics_spec.rb
+- spec/movies_spec.rb
+- spec/spec_helper.rb