ero_getter 0.1.6 → 1.0.0

data/Gemfile

@@ -2,7 +2,7 @@ source :rubygems
 
 gem 'activesupport'
 gem 'httpclient'
-gem 'rubyzip'
+gem 'zipruby'
 gem 'nokogiri'
 
 platforms :jruby do
@@ -11,6 +11,8 @@ end
 
 group :development, :test do
   gem 'rspec'
+  gem 'guard-rspec'
   gem 'bundler'
   gem 'jeweler'
+  gem 'fakeweb'
 end

data/Guardfile

@@ -0,0 +1,9 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard 'rspec', :version => 2 do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end
+

data/VERSION

@@ -1 +1 @@
-0.1.6
+1.0.0

data/ero_getter.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "ero_getter"
-  s.version = "0.1.6"
+  s.version = "1.0.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["masarakki"]
-  s.date = "2012-05-25"
+  s.date = "2012-05-26"
   s.description = "ero getter"
   s.email = "masaki@hisme.net"
   s.extra_rdoc_files = [
@@ -22,17 +22,22 @@ Gem::Specification.new do |s|
     ".rvmrc",
     ".travis.yml",
     "Gemfile",
+    "Guardfile",
     "LICENSE.txt",
     "README.rdoc",
     "Rakefile",
     "VERSION",
     "ero_getter.gemspec",
+    "lib/downloader/nijigazou_sokuhou.rb",
     "lib/ero_getter.rb",
-    "lib/ero_getter/downloader.rb",
-    "lib/ero_getter/downloader/base.rb",
-    "spec/downloader/base_spec.rb",
-    "spec/downloader_spec.rb",
+    "lib/ero_getter/base.rb",
+    "lib/ero_getter/utils.rb",
+    "spec/downloader/nijigazou_sokuhou_spec.rb",
+    "spec/ero_getter/base_spec.rb",
     "spec/ero_getter_spec.rb",
+    "spec/samples/nijigazou_sokuhou/first.html",
+    "spec/samples/nijigazou_sokuhou/last.html",
+    "spec/samples/nijigazou_sokuhou/middle.html",
     "spec/samples/sample.html",
     "spec/spec_helper.rb"
   ]
@@ -48,31 +53,37 @@ Gem::Specification.new do |s|
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<activesupport>, [">= 0"])
       s.add_runtime_dependency(%q<httpclient>, [">= 0"])
-      s.add_runtime_dependency(%q<rubyzip>, [">= 0"])
+      s.add_runtime_dependency(%q<zipruby>, [">= 0"])
       s.add_runtime_dependency(%q<nokogiri>, [">= 0"])
       s.add_runtime_dependency(%q<jruby-openssl>, [">= 0"])
       s.add_development_dependency(%q<rspec>, [">= 0"])
+      s.add_development_dependency(%q<guard-rspec>, [">= 0"])
       s.add_development_dependency(%q<bundler>, [">= 0"])
       s.add_development_dependency(%q<jeweler>, [">= 0"])
+      s.add_development_dependency(%q<fakeweb>, [">= 0"])
     else
       s.add_dependency(%q<activesupport>, [">= 0"])
       s.add_dependency(%q<httpclient>, [">= 0"])
-      s.add_dependency(%q<rubyzip>, [">= 0"])
+      s.add_dependency(%q<zipruby>, [">= 0"])
       s.add_dependency(%q<nokogiri>, [">= 0"])
       s.add_dependency(%q<jruby-openssl>, [">= 0"])
       s.add_dependency(%q<rspec>, [">= 0"])
+      s.add_dependency(%q<guard-rspec>, [">= 0"])
       s.add_dependency(%q<bundler>, [">= 0"])
       s.add_dependency(%q<jeweler>, [">= 0"])
+      s.add_dependency(%q<fakeweb>, [">= 0"])
     end
   else
     s.add_dependency(%q<activesupport>, [">= 0"])
     s.add_dependency(%q<httpclient>, [">= 0"])
-    s.add_dependency(%q<rubyzip>, [">= 0"])
+    s.add_dependency(%q<zipruby>, [">= 0"])
     s.add_dependency(%q<nokogiri>, [">= 0"])
     s.add_dependency(%q<jruby-openssl>, [">= 0"])
     s.add_dependency(%q<rspec>, [">= 0"])
+    s.add_dependency(%q<guard-rspec>, [">= 0"])
     s.add_dependency(%q<bundler>, [">= 0"])
     s.add_dependency(%q<jeweler>, [">= 0"])
+    s.add_dependency(%q<fakeweb>, [">= 0"])
   end
 end
 

data/lib/downloader/nijigazou_sokuhou.rb

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+require 'nokogiri'
+
+class NijigazouSokuhou < EroGetter::Base
+
+  name '二次画像速報'
+  url %r{http://nijigazo.2chblog.jp/archives/\d+.html}
+
+  target ".article-body-more > a > img" do |path|
+    path.parent[:href] if path.parent[:href] =~ /jpe?g|png|gif$/
+  end
+
+  sub_directory do
+    path = targets.first.split('/')
+    d = path[3..5].join('')
+    chara = path[6]
+    File.join(chara, d)
+  end
+
+  after ["//a[@rel='prev']", "//a[@rel='next']"] do |path|
+    path.text.match(Regexp.escape(title_part))
+  end
+
+  def title_part
+    @title_part ||= title.split(/:/).last.match(/(.+?)(その.+)?$/)[1].strip.gsub(/&amp;/, '&')
+  end
+
+end

data/lib/ero_getter/base.rb

@@ -0,0 +1,147 @@
+require 'active_support/inflector'
+require 'httpclient'
+require 'nokogiri'
+require 'open-uri'
+require 'zipruby'
+
+class EroGetter::Base
+  def initialize(url, direction = :none)
+    raise unless url.match url_regex
+    @url = url
+    @direction = direction
+  end
+
+  def base_dir
+    self.class.to_s.underscore
+  end
+
+  def directory
+    unless @dir
+      @dir = File.join(EroGetter.directory, base_dir, sub_directory)
+      EroGetter.mkdir(@dir)
+    end
+    @dir
+  end
+
+  def http_client
+    @http_client ||= HTTPClient.new
+  end
+
+  def url
+    @url
+  end
+
+  def direction
+    @direction
+  end
+
+  def document
+    @document ||= Nokogiri::HTML(open(url).read)
+  end
+
+  def title
+    @title ||= document.title
+  end
+
+  def run
+    targets.each do |target_url|
+      if target_url =~ /.*\.zip$/
+        save_zip(target_url)
+      else
+        save_image(target_url)
+      end
+    end
+    self.class.new(self.prev, :prev).run if run_prev?
+    self.class.new(self.next, :next).run if run_next?
+  end
+
+  def get_target(target)
+    response = http_client.get(target, :header => {:referer => url}, :follow_redirect => true)
+    raise unless response.status == 200
+    response
+  end
+
+  def save_image(target_url)
+    filename = File.basename(target_url)
+    response = get_target(target_url)
+    File.open(File.join(directory, filename), "wb") {|f| f.write response.body }
+  end
+
+  def save_zip(target_url)
+    response = get_target(target_url)
+    unzip(response.body).each do |filename, data|
+      File.open(File.join(dir, filename), "wb") {|f| f.write data }
+    end
+  end
+
+  def unzip(zip_data)
+    result = []
+    Zip::Archive.open_buffer(zip_data) do |archive|
+      archive.num_files.times do |i|
+        entry_name = archive.get_name(i)
+        archive.fopen(entry_name) do |f|
+          result << [f.name, f.read]
+        end
+      end
+    end
+    result
+  end
+
+  class << self
+    def name(site_name)
+      define_method(:name) do
+        site_name
+      end
+    end
+
+    def url(regex)
+      define_method(:url_regex) do
+        regex
+      end
+      EroGetter.add_mapping(regex, self)
+    end
+
+    def target(css_selector, &block)
+      define_method(:targets) do
+        unless instance_variable_defined?(:@targets)
+          items = document.css(css_selector).map do |elm|
+            yield(elm)
+          end
+          instance_variable_set(:@targets, items.compact)
+        end
+        instance_variable_get(:@targets)
+      end
+    end
+
+    def sub_directory(&block)
+      define_method(:sub_directory) do
+        unless instance_variable_defined?(:@sub_directory)
+          instance_variable_set(:@sub_directory, self.instance_eval(&block))
+        end
+        instance_variable_get(:@sub_directory)
+      end
+    end
+
+    def after(xpath, &block)
+      [:prev, :next].each_with_index do |method_name, index|
+        var_name = "@#{method_name}".to_sym
+        define_method(method_name) do
+          unless instance_variable_defined?(var_name)
+            tag = document.xpath(xpath[index]).first
+            instance_variable_set(var_name, instance_exec(tag, &block) ? tag[:href] : nil)
+          end
+          instance_variable_get(var_name)
+        end
+      end
+    end
+  end
+
+  private
+  def run_next?
+    direction != :prev && respond_to?(:next) && self.next != nil
+  end
+
+  def run_prev?
+    direction != :next && respond_to?(:prev) && self.prev != nil
+  end
+end

data/lib/ero_getter/utils.rb

@@ -0,0 +1,11 @@
+class EroGetter::Utils
+  class << self
+    def mkdir(path)
+      unless File.exists?(path)
+        basedir = File.dirname(path)
+        mkdir(basedir) unless File.exists?(basedir)
+        Dir.mkdir(path)
+      end
+    end
+  end
+end

data/lib/ero_getter.rb

@@ -1,5 +1,6 @@
 class EroGetter
-  autoload :Downloader, 'ero_getter/downloader'
+  autoload :Base, 'ero_getter/base'
+  autoload :Utils, 'ero_getter/utils'
 
   def detect(url)
     self.class.url_mapping.each_pair do |regex, klazz|
@@ -17,17 +18,37 @@ class EroGetter
 
   class << self
     def directory
-      File.join ENV['HOME'], 'ero_getter'
+      path = File.join ENV['HOME'], 'ero_getter'
+      puts path unless Dir.exists?(path)
+      path
+    end
+
+    def mkdir(path)
+      unless File.exists?(path)
+        basedir = File.dirname(path)
+        mkdir(basedir) unless File.exists?(basedir)
+        Dir.mkdir(path)
+      end
     end
 
     def url_mapping
       @url_mapping ||= {}
     end
+
     def add_mapping(regex, strategy)
       url_mapping[regex] = strategy
     end
+
     def clean
       @url_mapping = {}
     end
   end
 end
+
+Dir.glob(File.expand_path(File.dirname(__FILE__) + '/downloader/*.rb')) do |file|
+  begin
+    require file
+  rescue
+    puts "Load error in #{file}"
+  end
+end

data/spec/downloader/nijigazou_sokuhou_spec.rb

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+require 'spec_helper'
+
+describe NijigazouSokuhou do
+  let(:url) { 'http://nijigazo.2chblog.jp/archives/52249806.html' }
+  subject { @dl = NijigazouSokuhou.new(url) }
+
+  before do
+    fake(:get, url, 'nijigazou_sokuhou/middle.html')
+    EroGetter.stub(:mkdir)
+  end
+
+  describe :sample do
+    context :first_page do
+      before do
+        fake(:get, url, 'nijigazou_sokuhou/first.html')
+      end
+
+      its(:next) { should == url }
+      its(:prev) { should be_nil }
+    end
+
+    context :middle do
+      before do
+        fake(:get, url, 'nijigazou_sokuhou/middle.html')
+      end
+
+      its(:name) { should == '二次画像速報' }
+      its(:title) { should == '二次画像速報 : 【けいおん！】中野梓【あずにゃん】 その2' }
+      its(:title_part) { should == '【けいおん！】中野梓【あずにゃん】' }
+      its("targets.count") { should == 50 }
+      its("targets.first") { should == "http://img03.nijigazo.com/2012/05/11/azusa/0511azusa_0051.jpg" }
+      its(:sub_directory) { should == 'azusa/20120511' }
+      its(:prev) { should == 'http://nijigazo.2chblog.jp/archives/52249804.html' }
+      its(:next) { should == 'http://nijigazo.2chblog.jp/archives/52249808.html' }
+    end
+
+    context :last do
+      before do
+        fake(:get, url, 'nijigazou_sokuhou/last.html')
+      end
+
+      its(:next) { should be_nil }
+      its(:prev) { should == url }
+    end
+  end
+end

data/spec/ero_getter/base_spec.rb

@@ -0,0 +1,113 @@
+require 'spec_helper'
+
+describe EroGetter::Base do
+  let(:regex) { %r{http://example.net/\d+.html} }
+  before do
+    _regex = regex
+    fake(:get, regex, 'sample.html')
+    @klazz = Class.new(EroGetter::Base) do
+      name 'NijiEro BBS'
+      url _regex
+
+      target "ul#sources li a" do |elm|
+        elm[:href]
+      end
+
+      sub_directory do
+        targets.map{|x| x.split(%r{/}).last }.join('/')
+      end
+    end
+    @klazz.stub(:to_s).and_return('TestClass')
+  end
+
+  describe "assign url_mapping" do
+    it { EroGetter.url_mapping.should have_key regex }
+    it { EroGetter.url_mapping[regex].should == @klazz }
+  end
+
+  describe :instance_methods do
+    subject { @dl }
+    context :good do
+      before do
+        @dl = @klazz.new('http://example.net/10101010.html')
+        EroGetter.stub(:mkdir).and_return(true)
+        EroGetter.stub('directory').and_return('/tmp')
+      end
+      its(:name) { should == 'NijiEro BBS' }
+      its(:url_regex) { should == regex }
+      its(:base_dir) { should == 'test_class' }
+      its(:http_client) { should be_a HTTPClient }
+      its(:document) { should be_a Nokogiri::HTML::Document }
+      its(:title) { should == 'EroGetter Server' }
+      its(:url) { should == 'http://example.net/10101010.html' }
+      its(:direction) { should == :none }
+      its(:targets) { should == ['https://github.com/masarakki/ero_getter_server',
+          'https://github.com/masarakki/ero_getter_chrome_extension'] }
+      its(:sub_directory) { should == 'ero_getter_server/ero_getter_chrome_extension' }
+      its(:directory) { should == '/tmp/test_class/ero_getter_server/ero_getter_chrome_extension' }
+      describe :after_run do
+        context :not_set_after do
+          its(:run_next?) { should be_false }
+          its(:run_prev?) { should be_false }
+        end
+
+        context :direction_none do
+          context :has_next do
+            before do
+              @dl.stub(:next).and_return('hoge')
+            end
+            its(:run_next?) { should be_true }
+            its(:run_prev?) { should be_false }
+          end
+          context :has_prev do
+            before do
+              @dl.stub(:prev).and_return('hoge')
+            end
+            its(:run_next?) { should be_false }
+            its(:run_prev?) { should be_true }
+          end
+          context :has_next_and_prev do
+            before do
+              @dl.stub(:prev).and_return('hoge')
+              @dl.stub(:next).and_return('hoge')
+            end
+            its(:run_next?) { should be_true }
+            its(:run_prev?) { should be_true }
+          end
+        end
+
+        context :direction_prev do
+          before do
+            @dl.stub(:direction).and_return(:prev)
+          end
+          context :has_next do
+            before do
+              @dl.stub(:next).and_return('hoge')
+            end
+            its(:run_next?) { should be_false }
+          end
+        end
+
+        context :direction_next do
+          before do
+            @dl.stub(:direction).and_return(:next)
+          end
+          context :has_pref do
+            before do
+              @dl.stub(:prev).and_return('hoge')
+            end
+            its(:run_prev?) { should be_false }
+          end
+        end
+      end
+    end
+
+    context :url_mismatch do
+      it {
+        lambda {
+          @klazz.new('http://example.com/10101010.html')
+        }.should raise_error
+      }
+    end
+  end
+end