docparser 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 829c9585c7ac42c2496a8b385f2f19d4486a5e10
-  data.tar.gz: 39f576740941b72e61babec808af6756adfeb1b8
+  metadata.gz: 0ba58a4708d78ae22fc79694754ddb70cc4fee63
+  data.tar.gz: cef89d6934e560633e8c3b05c8bb75e16e2c424b
 SHA512:
-  metadata.gz: 32a9dc03bb9f413641b25cb0e51c9951fa8e1a64a48f4b1c8cc8e1e3877cbaa87b1ac70738f85c20f7383a3446d5a37915ee53a09fd30099b00be553779f4f4a
-  data.tar.gz: b106a3fc8843a2d47f9d8c4f5ac65e39afc702fff8dcabfdfe89e0477731ac1c9576e9b795dcda53c927c8acc718846b2bb277a0e7e68654ead52cb97e612b3e
+  metadata.gz: e0db436a8578ca5d286c6a4946ea57a0c3ad38ed9d2db27803143fc3062c229c60dfe16ac156690d9e33b5f9041aa3bbff08b1d55f87f01467f18f97ef521d64
+  data.tar.gz: afca96d6dd7357fe08899d793e40ea6473e7bd9707318f93848cee8cd95c98d3ff57f6f9f8543a8b08174ceddf34d39aab83135fc242d89441d351c453bf7758

data/.gitignore

@@ -16,3 +16,5 @@ test/tmp
 test/version_tmp
 tmp
 hackaday.*
+
+.DS_Store

data/.rubocop.yml

@@ -0,0 +1,5 @@
+# Avoid methods longer than 10 lines of code
+MethodLength:
+  Enabled: true
+  CountComments: false  # count full line comments?
+  Max: 20

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+    - 2.0.0

data/Gemfile

@@ -6,4 +6,12 @@ gem 'parallel'
 gem 'axlsx'
 gem 'terminal-table'
 gem 'pageme'
-gem "json"
+gem 'json'
+gem 'log4r'
+
+group :test do
+  gem 'rake'
+  gem 'rubocop'
+  gem 'simplecov', :require => false
+  gem 'simple_mock'
+end

data/README.md

@@ -1,21 +1,27 @@
 # DocParser
 
-Docs: http://rubydoc.info/github/jurriaan/docparser/
+[![Build Status](https://travis-ci.org/jurriaan/docparser.png?branch=master)](https://travis-ci.org/jurriaan/docparser)
 
 DocParser is a web scraping/screen scraping tool.
+
 You can use it to easily scrape web sites.
 
+The gem is called [docparser](http://rubygems.org/gems/docparser).
+You can find the documentation [here](http://rubydoc.info/github/jurriaan/docparser/).
+
 ## Features
 
 - XPath and CSS support through Nokogiri
 - Support for loading of URLs throug open-uri
 - Support for parallel processing of the documents
-- 5 Output formats:
+- 6 Output formats:
   * CSV
   * XLSX
   * HTML
   * YAML
+  * JSON
   * Screen (for debugging and development)
+  * And more! (easy to extend)
 
 ## Installation
 
@@ -33,11 +39,12 @@ Or install it yourself as:
 
 ## Usage
 
-See example.rb
+See [example.rb](https://github.com/jurriaan/docparser/blob/master/example.rb)
 
 ## Todo
 
 - Tests
+- Better examples
 
 ## Contributing
 
@@ -49,4 +56,4 @@ See example.rb
 
 ## Contributors
 
-- Jurriaan Pruis
+- [Jurriaan Pruis](https://github.com/jurriaan)

data/Rakefile

@@ -1 +1,16 @@
 require "bundler/gem_tasks"
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'lib/docparser'
+  t.test_files = FileList['test/lib/**/*_test.rb']
+  t.verbose = true
+end
+
+task test: :rubocop
+
+task :rubocop do
+  sh 'rubocop'
+end
+
+task :default => :test

data/example.rb

@@ -1,21 +1,23 @@
 #
-
-# An example of parsing a popular dutch website..
+# An example of parsing hackaday.com
 # (C) 2013 Jurriaan Pruis
 #
+$LOAD_PATH.unshift __dir__
+require File.expand_path('lib/docparser.rb', __dir__)
+require 'tmpdir'
 
-require 'docparser'
 include DocParser
-output = HTMLOutput.new filename: 'hackaday.html'
+output = MultiOutput.new(filename: 'hackaday')
 output.header = 'Title', 'Author', 'Publication date', 'URL', 'Summary'
-parser = Parser.new(files: (1..20).map {|i| "http://hackaday.com/page/#{i}/"}, parallel: false, output: output)
+files = Dir[File.join(__dir__, 'test/support/hackaday/*.html')]
+parser = Parser.new(files: files, parallel: false, output: output)
 parser.parse! do
   css('#content .post') do |post|
     title_el = post.search('.entry-title a').first
     title = title_el.content
-    author =post.search('.post-info .author .fn a').first.content
+    author = post.search('.post-info .author .fn a').first.content
     published_time = post.search('.post-info .date.published').first.content
-    url = title_el.attributes['href']
+    url = title_el.attributes['href'].value
     summary = post.search('.entry-content').first.content.strip
     add_row title, author, published_time, url, summary
   end

data/lib/docparser.rb

@@ -1 +1,2 @@
+$LOAD_PATH.unshift __dir__
 require 'docparser/parser'

data/lib/docparser/document.rb

@@ -5,26 +5,29 @@ module DocParser
   # @see Output
   class Document
     attr_reader :filename, :doc, :encoding, :results
-    def initialize(filename, encoding: 'utf-8', parser: nil)
+    def initialize(filename: nil, encoding: 'utf-8', parser: nil)
       if encoding == 'utf-8'
         encodingstring = 'r:utf-8'
       else
         encodingstring = "r:#{encoding}:utf-8"
       end
-
+      @logger = Log4r::Logger.new('docparser::document')
+      @logger.debug { "Parsing #{filename}" }
       open(filename, encodingstring) do |f|
-        @doc = Nokogiri::HTML(f)
+        @html = f.read
+        @logger.warn "#{filename} is empty" if @html.empty?
+        @doc = Nokogiri(@html)
       end
-
       @encoding = encoding
       @parser = parser
       @filename = filename
-      @results = Array.new(@parser.outputs.length) { [] }
+      @results = Array.new(@parser.outputs ? @parser.outputs.length : 0) { [] }
     end
 
     # Adds a row to an output
     def add_row(*row, output: 0)
       output = @parser.outputs.index(output) if output.is_a? Output
+      @logger.debug { "#{filename}: Adding row #{row.flatten.to_s}" }
       results[output] << row.flatten
     end
 
@@ -36,13 +39,17 @@ module DocParser
 
     # @return [String] the source of the document
     def html
-      @html ||= @doc.inner_html #TODO: ??
+      @html
     end
 
     # Executes a xpath query
     def xpath(query)
       res = @doc.search(query)
-      res.each { |el| yield el } if block_given?
+      if block_given?
+        res.each { |el| yield el }
+      else
+        res
+      end
     end
 
     # Executes a xpath query and returns the content
@@ -58,7 +65,7 @@ module DocParser
 
     # Matches the HTML source using a regular expression
     def regexp(regexp)
-      html.match(regexp) rescue nil
+      html.match(regexp)
     end
 
     # Parses the document
@@ -70,10 +77,10 @@ module DocParser
 
     # @!visibility private
     def inspect
-      "<Document file:'#{@filename}'>"
+      "<Document file:'#{@filename}', encoding:'#{@encoding}'>"
     end
 
-    alias :css :xpath
-    alias :css_content :xpath_content
+    alias_method :css, :xpath
+    alias_method :css_content, :xpath_content
   end
 end

data/lib/docparser/output.rb

@@ -11,6 +11,8 @@ module DocParser
       @filename = filename
       raise ArgumentError, 'Please specify a filename' if filename.empty?
       @file = open filename, 'w'
+      classname = self.class.name.split('::').last
+      @logger = Log4r::Logger.new("docparser::output::#{classname}")
       open_file
     end
 
@@ -30,6 +32,9 @@ module DocParser
     def close
       footer
       @file.close unless @file.closed?
+      @logger.info "Finished writing"
+      size = File.size(@filename) / 1024.0
+      @logger.info sprintf("%s: %d rows, %.2f KiB", @filename, rowcount, size)
     end
 
     # Called after the file is opened
@@ -44,19 +49,14 @@ module DocParser
 
     # Called when a row is added
     def write_row(row)
-      raise 'No row writer defined'
+      raise NotImplementedError.new('No row writer defined')
     end
 
     # Called before closing the file
     def footer
     end
+  end
 
-    # Displays information about the output
-    # @return [String] containing number of rows and file size
-    def summary
-      "%s:\t%d rows, %9.2f KiB" % [@filename,
-                                   @rowcount,
-                                   File.size(@filename) / 1024.0]
-    end
+  class MissingHeaderException < StandardError
   end
 end

data/lib/docparser/output/html_output.rb

@@ -5,67 +5,73 @@ module DocParser
   class HTMLOutput < Output
     # @!visibility private
     HTMLHEADER = <<-EOS
-  <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
-   "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
-  <html>
-  <head>
-  <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
-  <title>HTML output "#FILENAME#"</title>
-  <style type="text/css">
-  body {
-    font-family:"Helvetica Neue", Helvetica, Sans-Serif;
-    font-size:12px;
-  }
-  table {
-  border:1px solid #69c;
-  border-collapse:collapse;
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
+ "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
+<html>
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+<title>HTML output "#FILENAME#"</title>
+<style type="text/css">
+body {
+  font-family:"Helvetica Neue", Helvetica, Sans-Serif;
   font-size:12px;
-  text-align:left;
-  width:480px;
-  }
-  th {
-  border-bottom:1px dashed #69c;
-  color:#039;
-  font-size:14px;
-  font-weight:normal;
-  padding:12px 17px;
-  }
-  td {
-  color:#669;
-  padding:7px 17px;
-  white-space: pre;
-  }
-  tbody tr:hover td {
-  background:#d0dafd;
-  color:#339;
-  }
-  tbody tr:nth-child(even) {
-    background:#e0eaff;
-  }
-  </style>
-  </head>
-  <body>
-  <table>
-  EOS
+}
+table {
+border:1px solid #69c;
+border-collapse:collapse;
+font-size:12px;
+text-align:left;
+width:480px;
+}
+th {
+border-bottom:1px dashed #69c;
+color:#039;
+font-size:14px;
+font-weight:normal;
+padding:12px 17px;
+}
+td {
+color:#669;
+padding:7px 17px;
+white-space: pre;
+}
+tbody tr:hover td {
+background:#d0dafd;
+color:#339;
+}
+tbody tr:nth-child(even) {
+  background:#e0eaff;
+}
+</style>
+</head>
+<body>
+<table>
+EOS
     # @!visibility private
     HTMLFOOTER = <<-EOS
-    </tbody>
-    </table>
-    <p>#COUNT# rows</p>
-    </body>
-    </html>
-    EOS
+</tbody>
+</table>
+<p>#COUNT# rows</p>
+</body>
+</html>
+EOS
     def open_file
       @file << HTMLHEADER.gsub('#FILENAME#', @filename)
     end
 
     def header
+      return if @header.nil? || @header.empty?
       @file << '<thead><tr>'
       @file << @header.map { |f| '<th>' + f + '</th>' }.join
       @file << "</tr></thead>\n<tbody>\n"
+      @tbody = true
     end
 
     def write_row(row)
+      unless @tbody
+        @file << "<tbody>\n"
+        @tbody = true
+      end
       @file << '<tr>'
       @file << row.map { |f| '<td>' + CGI.escapeHTML(f.to_s) + '</td>' }.join
       @file << "</tr>\n"

data/lib/docparser/output/json_output.rb

@@ -1,7 +1,7 @@
 require 'json'
 module DocParser
   # The JSONOutput class generates a JSON file containing all rows as seperate
-  # JSON documents
+  # Array elements
   # @see Output
   class JSONOutput < Output
     # @!visibility private
@@ -12,18 +12,23 @@ module DocParser
     end
 
     def write_row(row)
+      raise MissingHeaderException if @header.nil? || @header.length == 0
       if @first
         @first = false
       else
         @file << ','
       end
       0.upto(@header.length - 1) do |counter|
-        @doc[@header[counter]] = row[counter] rescue ''
+        if row.length > counter
+          @doc[@header[counter]] = row[counter]
+        else
+          @doc[@header[counter]] = ''
+        end
       end
       @file << JSON.dump(@doc)
     end
 
-    def close
+    def footer
       @file << ']'
     end
   end

data/lib/docparser/output/multi_output.rb

@@ -24,27 +24,23 @@ module DocParser
       @outputs << HTMLOutput.new(htmloptions)
       @outputs << YAMLOutput.new(yamloptions)
       @outputs << XLSXOutput.new(xlsxoptions)
-      @outputs << XLSXOutput.new(jsonoptions)
+      @outputs << JSONOutput.new(jsonoptions)
     end
 
     def header=(row)
-      @outputs.each { |out|  out.header = row.flatten }
+      @outputs.each { |out|  out.header = row }
     end
 
     def add_row(row)
-      @outputs.each { |out|  out.add_row row.flatten }
+      @outputs.each { |out|  out.add_row row }
     end
 
     def rowcount
-      @outputs.min { |out| out.rowcount }.rowcount
+      @outputs.map { |out| out.rowcount }.min
     end
 
     def close
       @outputs.each { |out|  out.close }
     end
-
-    def summary
-      @outputs.map { |out|  out.summary }.join("\n")
-    end
   end
 end