docparser 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 829c9585c7ac42c2496a8b385f2f19d4486a5e10
+  data.tar.gz: 39f576740941b72e61babec808af6756adfeb1b8
+SHA512:
+  metadata.gz: 32a9dc03bb9f413641b25cb0e51c9951fa8e1a64a48f4b1c8cc8e1e3877cbaa87b1ac70738f85c20f7383a3446d5a37915ee53a09fd30099b00be553779f4f4a
+  data.tar.gz: b106a3fc8843a2d47f9d8c4f5ac65e39afc702fff8dcabfdfe89e0477731ac1c9576e9b795dcda53c927c8acc718846b2bb277a0e7e68654ead52cb97e612b3e

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+hackaday.*

data/.yardops

@@ -0,0 +1,2 @@
+README.md
+LICENSE

data/Gemfile

@@ -0,0 +1,9 @@
+ruby '2.0.0'
+
+source 'https://rubygems.org'
+gem 'nokogiri'
+gem 'parallel'
+gem 'axlsx'
+gem 'terminal-table'
+gem 'pageme'
+gem "json"

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Jurriaan Pruis
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,52 @@
+# DocParser
+
+Docs: http://rubydoc.info/github/jurriaan/docparser/
+
+DocParser is a web scraping/screen scraping tool.
+You can use it to easily scrape web sites.
+
+## Features
+
+- XPath and CSS support through Nokogiri
+- Support for loading of URLs throug open-uri
+- Support for parallel processing of the documents
+- 5 Output formats:
+  * CSV
+  * XLSX
+  * HTML
+  * YAML
+  * Screen (for debugging and development)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'docparser'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install docparser
+
+## Usage
+
+See example.rb
+
+## Todo
+
+- Tests
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## Contributors
+
+- Jurriaan Pruis

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/docparser.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'docparser/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "docparser"
+  spec.version       = DocParser::VERSION
+  spec.authors       = ["Jurriaan Pruis"]
+  spec.email         = ["email@jurriaanpruis.nl"]
+  spec.description   = %q{DocParser is a Ruby Gem for webscraping}
+  spec.summary       = %q{DocParser is a Ruby Gem for webscraping}
+  spec.homepage      = "https://github.com/jurriaan/docparser"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+end

data/example.rb

@@ -0,0 +1,22 @@
+#
+
+# An example of parsing a popular dutch website..
+# (C) 2013 Jurriaan Pruis
+#
+
+require 'docparser'
+include DocParser
+output = HTMLOutput.new filename: 'hackaday.html'
+output.header = 'Title', 'Author', 'Publication date', 'URL', 'Summary'
+parser = Parser.new(files: (1..20).map {|i| "http://hackaday.com/page/#{i}/"}, parallel: false, output: output)
+parser.parse! do
+  css('#content .post') do |post|
+    title_el = post.search('.entry-title a').first
+    title = title_el.content
+    author =post.search('.post-info .author .fn a').first.content
+    published_time = post.search('.post-info .date.published').first.content
+    url = title_el.attributes['href']
+    summary = post.search('.entry-content').first.content.strip
+    add_row title, author, published_time, url, summary
+  end
+end

data/lib/docparser.rb

@@ -0,0 +1 @@
+require 'docparser/parser'

data/lib/docparser/document.rb

@@ -0,0 +1,79 @@
+require 'set'
+module DocParser
+  # The Document class loads and parses the files.
+  # @see Parser
+  # @see Output
+  class Document
+    attr_reader :filename, :doc, :encoding, :results
+    def initialize(filename, encoding: 'utf-8', parser: nil)
+      if encoding == 'utf-8'
+        encodingstring = 'r:utf-8'
+      else
+        encodingstring = "r:#{encoding}:utf-8"
+      end
+
+      open(filename, encodingstring) do |f|
+        @doc = Nokogiri::HTML(f)
+      end
+
+      @encoding = encoding
+      @parser = parser
+      @filename = filename
+      @results = Array.new(@parser.outputs.length) { [] }
+    end
+
+    # Adds a row to an output
+    def add_row(*row, output: 0)
+      output = @parser.outputs.index(output) if output.is_a? Output
+      results[output] << row.flatten
+    end
+
+    # Extracts the document title
+    # @return [String] the title of the document
+    def title
+      @title ||= xpath_content('//head/title')
+    end
+
+    # @return [String] the source of the document
+    def html
+      @html ||= @doc.inner_html #TODO: ??
+    end
+
+    # Executes a xpath query
+    def xpath(query)
+      res = @doc.search(query)
+      res.each { |el| yield el } if block_given?
+    end
+
+    # Executes a xpath query and returns the content
+    # @return [String] the content of the HTML node
+    def xpath_content(query)
+      first = @doc.search(query).first
+      if first.nil?
+        nil
+      else
+        first.content
+      end
+    end
+
+    # Matches the HTML source using a regular expression
+    def regexp(regexp)
+      html.match(regexp) rescue nil
+    end
+
+    # Parses the document
+    # @return [Array] containing the parse results
+    def parse!(&block)
+      instance_exec(&block)
+      results
+    end
+
+    # @!visibility private
+    def inspect
+      "<Document file:'#{@filename}'>"
+    end
+
+    alias :css :xpath
+    alias :css_content :xpath_content
+  end
+end

data/lib/docparser/output.rb

@@ -0,0 +1,62 @@
+module DocParser
+  # The Output base class.
+  # All Output classes inherit from this one.
+  class Output
+    attr_reader :rowcount
+
+    # Creates a new output
+    # @param filename [String] Output filename
+    def initialize(filename: filename)
+      @rowcount = 0
+      @filename = filename
+      raise ArgumentError, 'Please specify a filename' if filename.empty?
+      @file = open filename, 'w'
+      open_file
+    end
+
+    # Stores the header
+    def header=(row)
+      @header = row
+      header
+    end
+
+    # Adds a row
+    def add_row(row)
+      @rowcount += 1
+      write_row row
+    end
+
+    # Closes output and IO
+    def close
+      footer
+      @file.close unless @file.closed?
+    end
+
+    # Called after the file is opened
+    def open_file
+      # do nothing
+    end
+
+    # Called after header is set
+    def header
+      # do nothing
+    end
+
+    # Called when a row is added
+    def write_row(row)
+      raise 'No row writer defined'
+    end
+
+    # Called before closing the file
+    def footer
+    end
+
+    # Displays information about the output
+    # @return [String] containing number of rows and file size
+    def summary
+      "%s:\t%d rows, %9.2f KiB" % [@filename,
+                                   @rowcount,
+                                   File.size(@filename) / 1024.0]
+    end
+  end
+end

data/lib/docparser/output/csv_output.rb

@@ -0,0 +1,20 @@
+require 'csv'
+module DocParser
+  # The CSVOutput class generates a CSV file containing all rows
+  # @see Output
+
+  class CSVOutput < Output
+    # @!visibility private
+    def open_file
+      @csv = CSV.new(@file, col_sep: ';')
+    end
+
+    def header
+      write_row @header
+    end
+
+    def write_row(row)
+      @csv << row
+    end
+  end
+end

data/lib/docparser/output/html_output.rb

@@ -0,0 +1,78 @@
+require 'cgi'
+module DocParser
+  # The XLSXOutput class generates an HTML file containing a table
+  # @see Output
+  class HTMLOutput < Output
+    # @!visibility private
+    HTMLHEADER = <<-EOS
+  <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
+   "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
+  <html>
+  <head>
+  <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+  <title>HTML output "#FILENAME#"</title>
+  <style type="text/css">
+  body {
+    font-family:"Helvetica Neue", Helvetica, Sans-Serif;
+    font-size:12px;
+  }
+  table {
+  border:1px solid #69c;
+  border-collapse:collapse;
+  font-size:12px;
+  text-align:left;
+  width:480px;
+  }
+  th {
+  border-bottom:1px dashed #69c;
+  color:#039;
+  font-size:14px;
+  font-weight:normal;
+  padding:12px 17px;
+  }
+  td {
+  color:#669;
+  padding:7px 17px;
+  white-space: pre;
+  }
+  tbody tr:hover td {
+  background:#d0dafd;
+  color:#339;
+  }
+  tbody tr:nth-child(even) {
+    background:#e0eaff;
+  }
+  </style>
+  </head>
+  <body>
+  <table>
+  EOS
+    # @!visibility private
+    HTMLFOOTER = <<-EOS
+    </tbody>
+    </table>
+    <p>#COUNT# rows</p>
+    </body>
+    </html>
+    EOS
+    def open_file
+      @file << HTMLHEADER.gsub('#FILENAME#', @filename)
+    end
+
+    def header
+      @file << '<thead><tr>'
+      @file << @header.map { |f| '<th>' + f + '</th>' }.join
+      @file << "</tr></thead>\n<tbody>\n"
+    end
+
+    def write_row(row)
+      @file << '<tr>'
+      @file << row.map { |f| '<td>' + CGI.escapeHTML(f.to_s) + '</td>' }.join
+      @file << "</tr>\n"
+    end
+
+    def footer
+      @file << HTMLFOOTER.gsub('#COUNT#', @rowcount.to_s)
+    end
+  end
+end

data/lib/docparser/output/json_output.rb

@@ -0,0 +1,30 @@
+require 'json'
+module DocParser
+  # The JSONOutput class generates a JSON file containing all rows as seperate
+  # JSON documents
+  # @see Output
+  class JSONOutput < Output
+    # @!visibility private
+    def open_file
+      @file << '['
+      @first = true
+      @doc = {}
+    end
+
+    def write_row(row)
+      if @first
+        @first = false
+      else
+        @file << ','
+      end
+      0.upto(@header.length - 1) do |counter|
+        @doc[@header[counter]] = row[counter] rescue ''
+      end
+      @file << JSON.dump(@doc)
+    end
+
+    def close
+      @file << ']'
+    end
+  end
+end

data/lib/docparser/output/multi_output.rb

@@ -0,0 +1,50 @@
+module DocParser
+  # The MultiOutput output combines multiple outputs.
+  # It creates a CSV, HTML, YAML and XLSX Output file
+  # @see CSVOutput
+  # @see HTMLOutput
+  # @see YAMLOutput
+  # @see XLSXOutput
+  # @see Output
+  class MultiOutput < Output
+    # @!visibility private
+    def initialize(**options)
+      @outputs = []
+      csvoptions = options.clone
+      csvoptions[:filename] += '.csv'
+      htmloptions = options.clone
+      htmloptions[:filename] += '.html'
+      yamloptions = options.clone
+      yamloptions[:filename] += '.yml'
+      xlsxoptions = options.clone
+      xlsxoptions[:filename] += '.xlsx'
+      jsonoptions = options.clone
+      jsonoptions[:filename] += '.json'
+      @outputs << CSVOutput.new(csvoptions)
+      @outputs << HTMLOutput.new(htmloptions)
+      @outputs << YAMLOutput.new(yamloptions)
+      @outputs << XLSXOutput.new(xlsxoptions)
+      @outputs << XLSXOutput.new(jsonoptions)
+    end
+
+    def header=(row)
+      @outputs.each { |out|  out.header = row.flatten }
+    end
+
+    def add_row(row)
+      @outputs.each { |out|  out.add_row row.flatten }
+    end
+
+    def rowcount
+      @outputs.min { |out| out.rowcount }.rowcount
+    end
+
+    def close
+      @outputs.each { |out|  out.close }
+    end
+
+    def summary
+      @outputs.map { |out|  out.summary }.join("\n")
+    end
+  end
+end

data/lib/docparser/output/screen_output.rb

@@ -0,0 +1,35 @@
+require 'terminal-table'
+require 'pageme'
+module DocParser
+  # This Output is can be used for debugging purposes.
+
+  # It pipes all rows through a pager
+  # @see Output
+  class ScreenOutput < Output
+    # @!visibility private
+
+    include PageMe
+
+    def initialize
+      @tables = []
+      @rowcount = 0
+    end
+
+    def close
+      page do |p|
+        p.puts "Showing all #{@tables.length} rows:\n\n"
+        @tables.each do |table|
+          p.puts table
+        end
+      end
+    end
+
+    def write_row(row)
+      out = []
+      0.upto(@header.length - 1) do |counter|
+        out << [@header[counter], row[counter]]
+      end
+      @tables << Terminal::Table.new(rows: out)
+    end
+  end
+end

data/lib/docparser/output/xlsx_output.rb

@@ -0,0 +1,28 @@
+require 'axlsx'
+module DocParser
+  # The XLSXOutput class generates Microsoft Excel compatible .xlsx files
+  # using the great axslx library
+  # @see Output
+  class XLSXOutput < Output
+    # @!visibility private
+    def open_file
+      @package = Axlsx::Package.new
+      @package.workbook.date1904 = false # Fix for OS X
+      @sheet = @package.workbook.add_worksheet
+      @file.close
+    end
+
+    def header
+      write_row @header
+    end
+
+    def write_row(row)
+      @sheet.add_row row
+    end
+
+    def close
+      @sheet.add_table "A1:#{@sheet.cells.last.r}", name: 'Data'
+      @package.serialize @filename
+    end
+  end
+end

data/lib/docparser/output/yaml_output.rb

@@ -0,0 +1,16 @@
+require 'yaml'
+module DocParser
+  # The YAMLOutput class generates a YAML file containing all rows as seperate
+  # YAML documents
+  # @see Output
+  class YAMLOutput < Output
+    # @!visibility private
+    def write_row(row)
+      @doc ||= {}
+      0.upto(@header.length - 1) do |counter|
+        @doc[@header[counter]] = row[counter] rescue ''
+      end
+      YAML.dump @doc, @file
+    end
+  end
+end

data/lib/docparser/parser.rb

@@ -0,0 +1,105 @@
+$:.unshift __dir__
+require 'rubygems'
+require 'bundler/setup'
+require 'version'
+require 'output'
+require 'document'
+require 'nokogiri'
+require 'open-uri'
+require 'parallel'
+require 'set'
+require 'output/screen_output.rb'
+require 'output/csv_output.rb'
+require 'output/html_output.rb'
+require 'output/xlsx_output.rb'
+require 'output/yaml_output.rb'
+require 'output/json_output.rb'
+require 'output/multi_output.rb'
+# {include:file:README.md}
+module DocParser
+  # The main parser class. This is the class you'll use to create your parser
+  # The real work happens in the Document class
+  # @see Document
+  class Parser
+    # @!visibility private
+    attr_reader :outputs
+
+    # Creates a new parser instance
+    # @param files [Array] An array containing URLs or paths to files
+    # @param quiet [Boolean] Be quiet
+    # @param encoding [String] The encoding to use for opening the files
+    # @param parallel [Boolean] Use parallel processing
+    # @param output [Output, Array] The output(s), defaults to a Screenoutput
+    # @param range [Range] Range of files to process (nil means process all)
+    # @param num_processes [Fixnum] Number of parallel processes
+    def initialize(files: [], quiet: false, encoding: 'utf-8', parallel: true,
+                   output: ScreenOutput.new, range: nil,
+                   num_processes: Parallel.processor_count + 1)
+      @quiet = quiet
+      @parallel = parallel
+      @num_processes = num_processes
+      @encoding = encoding
+      if output.is_a? Output
+        @outputs = []
+        @outputs << output
+      elsif output.is_a?(Array) && output.all? { |o| o.is_a? Output }
+        @outputs = output
+      else
+        raise ArgumentError, 'No outputs specified'
+      end
+      @files = if range
+        files[range]
+      else
+        files
+      end
+      log 'DocParser loaded..'
+      log "#{@files.length} files loaded (encoding: #{@encoding})"
+    end
+
+    #
+    # Parses the `files`
+    #
+    def parse!(&block)
+      log "Parsing #{@files.length} files."
+      start_time = Time.now
+      resultsets = Array.new(@outputs.length) { Set.new }
+
+      if @parallel && @num_processes > 1
+        log "Starting #{@num_processes} processes"
+        Parallel.map(@files, in_processes: @num_processes) do |file|
+          Document.new(file, encoding: @encoding, parser: self).parse!(&block)
+        end.each do |result|
+          result.each_with_index { |set, index| resultsets[index].merge(set) }
+        end
+        log 'Parallel processing finished, writing results..'
+      else
+        @files.each do |file|
+          doc = Document.new(file, encoding: @encoding, parser: self)
+          doc.parse!(&block).each_with_index do |set, index|
+            resultsets[index].merge(set)
+          end
+        end
+      end
+
+      log "\nSummary\n======="
+
+      @outputs.each_with_index do |output, index|
+        resultsets[index].each do |row|
+          output.add_row row
+        end
+        resultsets[index] = nil
+        output.close
+        log output.summary
+      end
+
+      log ''
+      log 'Done processing in %.2fs.' % (Time.now - start_time)
+    end
+
+    private
+
+    def log(str)
+      puts str unless @quiet
+    end
+  end
+end

data/lib/docparser/version.rb

@@ -0,0 +1,4 @@
+module DocParser
+  # The current version of DocParser
+  VERSION = '0.0.1'
+end

metadata

@@ -0,0 +1,93 @@
+--- !ruby/object:Gem::Specification
+name: docparser
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Jurriaan Pruis
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-04-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: DocParser is a Ruby Gem for webscraping
+email:
+- email@jurriaanpruis.nl
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .yardops
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- docparser.gemspec
+- example.rb
+- lib/docparser.rb
+- lib/docparser/document.rb
+- lib/docparser/output.rb
+- lib/docparser/output/csv_output.rb
+- lib/docparser/output/html_output.rb
+- lib/docparser/output/json_output.rb
+- lib/docparser/output/multi_output.rb
+- lib/docparser/output/screen_output.rb
+- lib/docparser/output/xlsx_output.rb
+- lib/docparser/output/yaml_output.rb
+- lib/docparser/parser.rb
+- lib/docparser/version.rb
+homepage: https://github.com/jurriaan/docparser
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: DocParser is a Ruby Gem for webscraping
+test_files: []
+has_rdoc: