autopagerize 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in autopagerize.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 uu59
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,65 @@
+# Autopagerize gem
+
+Autopagerize gem works as such extensions:
+
+- [AutoPager Chrome](https://chrome.google.com/webstore/detail/autopager-chrome/mmgagnmbebdebebbcleklifnobamjonh?hl=ja)
+- [AutoPagerize](https://chrome.google.com/webstore/detail/autopagerize/igiofjhpmpihnifddepnpngfjhkfenbp)
+- [AutoPatchWork](https://chrome.google.com/webstore/detail/autopatchwork/aeolcjbaammbkgaiagooljfdepnjmkfd?hl=ja)
+
+## Usage
+
+You must download SITEINFO from <http://wedata.net/databases/AutoPagerize/items_all.json>
+
+    $ curl http://wedata.net/databases/AutoPagerize/items_all.json > siteinfo.json 
+
+    require "rubygems"
+    require "multi_json"
+    require "autopagerize"
+
+    siteinfo = MultiJson.load(File.read("siteinfo.json"))
+    url = "http://www.google.com/search?q=test"
+    page = Autopagerize.new(url, siteinfo)
+    puts page.processed_html
+
+## Advance Usage
+
+    require "rubygems"
+    require "multi_json"
+    require "autopagerize"
+
+    siteinfo = MultiJson.load(File.read("siteinfo.json"))
+    url = "http://apod.nasa.gov/apod/ap090903.html"
+    apod = Autopagerize.new(url, siteinfo, {
+      :maxpage => 30,
+      :headers => {
+        "User-Agent" => "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)",
+      },
+    })
+    apod.each do |page|
+      img = page.page_element.at_xpath('//a/img/..')
+      next unless img
+      puts URI.join(url, img.attributes["href"].to_s).to_s
+    end
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'autopagerize'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install autopagerize
+
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/autopagerize.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'autopagerize/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "autopagerize"
+  gem.version       = Autopagerize::VERSION
+  gem.authors       = ["uu59"]
+  gem.email         = ["k@uu59.org"]
+  gem.description   = %q{Concat paginated web pages to single page}
+  gem.summary       = %q{Concat paginated web pages to single page}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.add_dependency "addressable"
+  gem.add_dependency "httpclient"
+  gem.add_dependency "nokogiri"
+  gem.add_development_dependency "rspec"
+  gem.add_development_dependency "multi_json"
+end

data/lib/autopagerize.rb

@@ -0,0 +1,113 @@
+#require "autopagerize/version"
+require "addressable/uri"
+require "nokogiri"
+require "httpclient"
+
+class Autopagerize
+  include Enumerable
+
+  attr_reader :url, :client, :siteinfo, :options
+
+  def initialize(url, siteinfo, options = {})
+    @url = url
+    @siteinfo = siteinfo
+
+    @options = {
+      :maxpage => 10,
+      :headers => {},
+
+      # for internal/test use
+      :current_page => 1,
+      :httpclient => nil,
+      :site => nil,
+    }.merge(options)
+
+    @site = @options[:site]
+  end
+
+  def nextlink
+    return nil unless site
+    node = document.at_xpath(site["data"]["nextLink"])
+    return nil unless node
+    Addressable::URI.join(url, node.attributes["href"].to_s).to_s
+  end
+
+  def next
+    return nil if options[:maxpage] <= options[:current_page]
+    return nil if nextlink.nil?
+    @next ||= Autopagerize.new(nextlink, siteinfo, options.merge(:current_page => options[:current_page] + 1, :site => site))
+  end
+
+  def each
+    current = self
+    yield current
+    while current = current.next
+      yield current
+    end
+  end
+
+  def processed_document
+    @processed_document ||= begin
+      result = document.dup
+
+      # Insert rule:
+      # https://autopagerize.jottit.com/details_of_siteinfo_(ja)
+      before = site["data"]["insertBefore"]
+      if before.nil? || before.length == 0 || result.at_xpath(before).nil?
+        page = result.xpath(site["data"]["pageElement"]).last
+        point = Nokogiri::XML::Node.new("dummy_for_autopagerize", result.document)
+        page.after point
+      else
+        point = result.at_xpath(before)
+      end
+
+      @processed_page_elements = [self.page]
+      current = self
+      while current = current.next
+        point.before(current.page)
+        @processed_page_elements << current.page
+      end
+      point.remove
+      result
+    end
+  end
+
+  def processed_page_elements
+    processed_document
+    @processed_page_elements
+  end
+
+  def processed_html
+    processed_document.to_xml
+  end
+
+  def site
+    @site ||= siteinfo.find do |site|
+      /#{normalize_regex(site["data"]["url"])}/.match(url) && site["data"]["nextLink"] && document.at_xpath(site["data"]["nextLink"])
+    end
+  end
+
+  def client
+    options[:httpclient] ||= HTTPClient.new
+  end
+
+  def html
+    @html ||= client.get_content(url, nil, options[:headers])
+  end
+
+  def document
+    @document ||= Nokogiri::HTML.parse(html)
+  end
+
+  def page
+    document.xpath(site["data"]["pageElement"]).last
+  end
+  alias :page_element :page
+
+  private
+  def normalize_regex(re)
+    # to be quiet
+    # warning: nested repeat operator + and ? was replaced with '*': /^http:\/\/1stpower\.web\.fc2\.com\/comic(?:\d+)?/
+    re.gsub("+)?", "*)")
+  end
+end

data/lib/autopagerize/version.rb

@@ -0,0 +1,3 @@
+class Autopagerize
+  VERSION = "0.0.1"
+end

data/spec/autopagerize_spec.rb

@@ -0,0 +1,56 @@
+# -- coding: utf-8
+
+require "spec_helper"
+
+describe Autopagerize do
+  let(:siteinfo) do
+    json = MultiJson.dump [
+      {
+        :data => {
+          :url => "^http://foo/",
+          :nextLink => "nomatch",
+        },
+      },
+      {
+        :data => {
+          :url => "^http://bar/",
+          :pageElement => "//page",
+          :nextLink => "//a",
+        }
+      },
+      {
+        :data => {
+          :url => "^http://baz/",
+          :pageElement => "//foobar",
+          :nextLink => "//a",
+        }
+      },
+    ]
+    MultiJson.load(json)
+  end
+
+  it "should autopagerize" do
+    first = <<-HTML
+    <!doctype html>
+    <page>first page</page>
+    <a href="next.html">next link</a>
+    HTML
+
+    second = <<-HTML
+    <!doctype html>
+    <page>next.html</page>
+    <a href="morenext.html">go to final page</a>
+    HTML
+
+    third = <<-HTML
+    <!doctype html>
+    <page>final page</page>
+    HTML
+
+    client = HTTPClient.new
+    client.test_loopback_response << first << second << third
+    a = Autopagerize.new("http://bar/", siteinfo, :httpclient => client)
+    a.nextlink.should == "http://bar/next.html"
+    a.to_enum.to_a.length.should == 3
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,14 @@
+# -- coding: utf-8
+
+require "rubygems"
+require "bundler/setup"
+Bundler.require :default, :test, :development
+require "rspec-expectations"
+require "rspec/matchers/built_in/be"
+
+Dir["./spec/support/**/*.rb"].each{|file| require file }
+
+require File.expand_path("../../lib/autopagerize.rb", __FILE__)
+
+RSpec.configure do |config|
+end

metadata

@@ -0,0 +1,138 @@
+--- !ruby/object:Gem::Specification
+name: autopagerize
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- uu59
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-11-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: httpclient
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Concat paginated web pages to single page
+email:
+- k@uu59.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- autopagerize.gemspec
+- lib/autopagerize.rb
+- lib/autopagerize/version.rb
+- spec/autopagerize_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Concat paginated web pages to single page
+test_files:
+- spec/autopagerize_spec.rb
+- spec/spec_helper.rb
+has_rdoc: