ero_getter 0.1.0 → 0.1.1

data/.travis.yml

@@ -0,0 +1,12 @@
+language: ruby
+rvm:
+ - 1.8.7
+ - 1.9.2
+ - 1.9.3
+ - jruby-18mode
+ - jruby-19mode
+ - rbx-18mode
+ - rbx-19mode
+ - ruby-head
+ - jruby-head
+ - ree

data/Gemfile

@@ -3,6 +3,11 @@ source :rubygems
 gem 'activesupport'
 gem 'httpclient'
 gem 'rubyzip'
+gem 'nokogiri'
+
+platform :jruby do
+  gem 'jruby-openssl'
+end
 
 group :development, :test do
   gem 'rspec'

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.1.1

data/ero_getter.gemspec

@@ -0,0 +1,78 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = "ero_getter"
+  s.version = "0.1.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["masarakki"]
+  s.date = "2012-05-12"
+  s.description = "ero getter"
+  s.email = "masaki@hisme.net"
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+    ".rspec",
+    ".rvmrc",
+    ".travis.yml",
+    "Gemfile",
+    "LICENSE.txt",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "ero_getter.gemspec",
+    "lib/ero_getter.rb",
+    "lib/ero_getter/downloader.rb",
+    "lib/ero_getter/downloader/base.rb",
+    "spec/downloader/base_spec.rb",
+    "spec/downloader_spec.rb",
+    "spec/ero_getter_spec.rb",
+    "spec/samples/sample.html",
+    "spec/spec_helper.rb"
+  ]
+  s.homepage = "http://github.com/masarakki/ero_getter"
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = "1.8.24"
+  s.summary = "ero getter"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<activesupport>, [">= 0"])
+      s.add_runtime_dependency(%q<httpclient>, [">= 0"])
+      s.add_runtime_dependency(%q<rubyzip>, [">= 0"])
+      s.add_runtime_dependency(%q<nokogiri>, [">= 0"])
+      s.add_runtime_dependency(%q<jruby-openssl>, [">= 0"])
+      s.add_development_dependency(%q<rspec>, [">= 0"])
+      s.add_development_dependency(%q<bundler>, [">= 0"])
+      s.add_development_dependency(%q<jeweler>, [">= 0"])
+    else
+      s.add_dependency(%q<activesupport>, [">= 0"])
+      s.add_dependency(%q<httpclient>, [">= 0"])
+      s.add_dependency(%q<rubyzip>, [">= 0"])
+      s.add_dependency(%q<nokogiri>, [">= 0"])
+      s.add_dependency(%q<jruby-openssl>, [">= 0"])
+      s.add_dependency(%q<rspec>, [">= 0"])
+      s.add_dependency(%q<bundler>, [">= 0"])
+      s.add_dependency(%q<jeweler>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<activesupport>, [">= 0"])
+    s.add_dependency(%q<httpclient>, [">= 0"])
+    s.add_dependency(%q<rubyzip>, [">= 0"])
+    s.add_dependency(%q<nokogiri>, [">= 0"])
+    s.add_dependency(%q<jruby-openssl>, [">= 0"])
+    s.add_dependency(%q<rspec>, [">= 0"])
+    s.add_dependency(%q<bundler>, [">= 0"])
+    s.add_dependency(%q<jeweler>, [">= 0"])
+  end
+end
+

data/lib/ero_getter/downloader/base.rb

@@ -1,10 +1,12 @@
 require 'active_support/inflector'
 require 'httpclient'
+require 'nokogiri'
 
 class EroGetter::Downloader::Base
-  def initialize(url)
+  def initialize(url, direction = 0)
     raise unless url.match url_regex
     @url = url
+    @direction = direction
   end
 
   def base_dir
@@ -15,6 +17,18 @@ class EroGetter::Downloader::Base
     @http_client ||= HTTPClient.new
   end
 
+  def url
+    @url
+  end
+
+  def document
+    @document ||= Nokogiri::HTML(open(url).read)
+  end
+
+  def title
+    @title ||= document.title
+  end
+
   class << self
     def name(site_name)
       define_method(:name) do
@@ -28,5 +42,18 @@ class EroGetter::Downloader::Base
       end
       EroGetter.add_mapping(regex, self)
     end
+
+    def target(css_selector, &block)
+      define_method(:targets) do
+        unless instance_variable_defined?(:@targets)
+          items = document.css(css_selector).map do |elm|
+            yield(elm)
+          end
+          instance_variable_set(:@targets, items.compact)
+        end
+        instance_variable_get(:@targets)
+      end
+    end
   end
+
 end

data/lib/ero_getter.rb

@@ -16,10 +16,13 @@ class EroGetter
 
   class << self
     def url_mapping
-      @@url_mapping ||= {}
+      @url_mapping ||= {}
     end
     def add_mapping(regex, strategy)
       url_mapping[regex] = strategy
     end
+    def clean
+      @url_mapping = {}
+    end
   end
 end

data/spec/downloader/base_spec.rb

@@ -1,33 +1,45 @@
 require 'spec_helper'
 
 describe EroGetter::Downloader::Base do
-  let!(:regex) { %r{http://example.net/\d+.html} }
-  before(:all) do
+  let(:regex) { %r{http://example.net/\d+.html} }
+  before do
     _regex = regex
-    TestClass = Class.new(EroGetter::Downloader::Base) do
+    @klazz = Class.new(EroGetter::Downloader::Base) do
       name 'NijiEro BBS'
       url _regex
+      target "ul#sources li a" do |elm|
+        elm[:href]
+      end
     end
+    @klazz.stub(:to_s).and_return('TestClass')
   end
 
   describe "assign url_mapping" do
     it { EroGetter.url_mapping.should have_key regex }
-    it { EroGetter.url_mapping[regex].should == TestClass }
+    it { EroGetter.url_mapping[regex].should == @klazz }
   end
 
   describe :instance_methods do
-    subject { @klazz }
+    subject { @dl }
     context :good do
-      before { @klazz = TestClass.new('http://example.net/10101010.html') }
+      before do
+        @dl = @klazz.new('http://example.net/10101010.html')
+        @dl.stub(:open).and_return(File.open(sample_path('sample.html')))
+      end
       its(:name) { should == 'NijiEro BBS' }
       its(:url_regex) { should == regex }
       its(:base_dir) { should == 'test_class' }
       its(:http_client) { should be_a HTTPClient }
+      its(:document) { should be_a Nokogiri::HTML::Document }
+      its(:title) { should == 'EroGetter Server' }
+      its(:url) { should == 'http://example.net/10101010.html' }
+      its(:targets) { should == ['https://github.com/masarakki/ero_getter_server',
+          'https://github.com/masarakki/ero_getter_chrome_extension'] }
     end
     context :url_mismatch do
       it {
         lambda {
-          TestClass.new('http://example.com/10101010.html')
+          @klazz.new('http://example.com/10101010.html')
         }.should raise_error
       }
     end

data/spec/ero_getter_spec.rb

@@ -1,6 +1,10 @@
 require 'spec_helper'
 
 describe EroGetter do
+  before do
+    EroGetter.clean
+  end
+
   let(:regex) { %r{http://example.com/\d+.html} }
   subject { @ero_getter }
   describe :class_methods do

data/spec/samples/sample.html

@@ -0,0 +1,34 @@
+<!DOCTYPE html>
+<title>EroGetter Server</title>
+<link href='/bootstrap/css/bootstrap.css' rel='stylesheet' type='text/css' />
+<script src='http://ajax.googleapis.com/ajax/libs/jquery/1.7.2/jquery.min.js' type='text/javascript'></script>
+<div class='container'>
+  <h1>EroGetter Server</h1>
+  <div class='row'>
+    <div class='span6'>
+      <h2>Usage</h2>
+      <dl>
+        <dt>GET /</dt>
+        <dd>show usage and queues (this page)</dd>
+        <dt>POST /</dt>
+        <dd>add url to queue</dd>
+      </dl>
+      <h2>Source</h2>
+      <ul id='sources'>
+        <li>
+          <a href='https://github.com/masarakki/ero_getter_server'>EroGetter Server</a>
+        </li>
+        <li>
+          <a href='https://github.com/masarakki/ero_getter_chrome_extension'>EroGetter Chrome Extension</a>
+        </li>
+      </ul>
+    </div>
+    <div class='span6'>
+      <h1>process</h1>
+      <h2>pid</h2>
+      <p>7016</p>
+      <h2>queues</h2>
+      <p>empty</p>
+    </div>
+  </div>
+</div>

data/spec/spec_helper.rb

@@ -8,5 +8,10 @@ require 'ero_getter'
 Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
 
 RSpec.configure do |config|
-  
+
+end
+
+
+def sample_path(file)
+  File.join(File.dirname(__FILE__), 'samples', file)
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ero_getter
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -59,6 +59,38 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: jruby-openssl
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -118,17 +150,20 @@ files:
 - .document
 - .rspec
 - .rvmrc
+- .travis.yml
 - Gemfile
 - LICENSE.txt
 - README.rdoc
 - Rakefile
 - VERSION
+- ero_getter.gemspec
 - lib/ero_getter.rb
 - lib/ero_getter/downloader.rb
 - lib/ero_getter/downloader/base.rb
 - spec/downloader/base_spec.rb
 - spec/downloader_spec.rb
 - spec/ero_getter_spec.rb
+- spec/samples/sample.html
 - spec/spec_helper.rb
 homepage: http://github.com/masarakki/ero_getter
 licenses:
@@ -145,7 +180,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 72716279475598011
+      hash: -3104763422106775519
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: