docparser 0.1.6 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9b7bee15c24dec4a95cb7d36e724f127360eb3b1
-  data.tar.gz: feee0df9857c9488bbd76d065f5fbdef781ec0b8
+  metadata.gz: 94ae9577e872555f07210deffe7c4ebc192ac617
+  data.tar.gz: cec0672efe1be1d5bab9d3bff313b4b5c8bc9513
 SHA512:
-  metadata.gz: fc6523cc590e56df0e22440b40c301c513647e56fe5ca18fc98a6d849d2cd084be29e011d242db9ee9327475d49082915e7a80faaf4077fa6c4cb056b1f51184
-  data.tar.gz: 99c44f0eabc3e58d6e463b3cac3ae0859ad448c647ccfecd73ab9e4eef9f709df18e79d15ac0ae19c511f0652a8aabca7608c610681fd63bb51bccb32e5335c7
+  metadata.gz: f1105cb8834ea0c03dd1e4672ae75792ef3a242b44a3a11d718ebcf444723ea7d0e42a64f23972c2c6faf1f7ddb5c7e10cb418094fad37d255c58ad3c1ebf0a2
+  data.tar.gz: 5afaaee3978d2ad2f50d980622540714645a02673f2ed9d3a645b415230141839dd32f31295c37a798427745a87bedb742a02c8eba11a6f379f0e6655514e58f

data/.travis.yml

@@ -1,6 +1,7 @@
 language: ruby
 rvm:
     - 2.0.0
+    - 2.1.1
     - ruby-head
     - jruby-head
 matrix:

data/Gemfile

@@ -1,13 +1,12 @@
-ruby '2.0.0'
 gemspec
 
 source 'https://rubygems.org'
 
 group :test do
-  gem 'minitest', '~> 5.0.8'
+  gem 'minitest', '~> 5.3.0'
   gem 'coveralls', require: false
   gem 'rake'
-  gem 'rubocop', '~> 0.13.1'
+  gem 'rubocop', '~> 0.18.1'
   gem 'simplecov', require: false
   gem 'simple_mock'
 end

data/README.md

@@ -5,7 +5,7 @@
 
 DocParser is a web scraping/screen scraping tool.
 
-You can use it to easily scrape web sites.
+You can use it to easily scrape information out of HTML documents.
 
 The gem is called [docparser](http://rubygems.org/gems/docparser).
 You can find the documentation [here](http://rubydoc.info/github/jurriaan/docparser/).
@@ -13,7 +13,6 @@ You can find the documentation [here](http://rubydoc.info/github/jurriaan/docpar
 ## Features
 
 - XPath and CSS support through Nokogiri
-- Support for loading of URLs throug open-uri
 - Support for parallel processing of the documents
 - 6 Output formats:
   * CSV

data/docparser.gemspec

@@ -19,11 +19,9 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
   spec.extra_rdoc_files = ['README.md', 'LICENSE']
 
-  spec.add_runtime_dependency 'nokogiri', '~> 1.6.0'
-  spec.add_runtime_dependency 'parallel', '~> 0.8.4'
+  spec.add_runtime_dependency 'nokogiri', '~> 1.6.1'
+  spec.add_runtime_dependency 'parallel', '~> 0.9.1'
   spec.add_runtime_dependency 'axlsx', '~> 2.0.1'
-  spec.add_runtime_dependency 'terminal-table', '~> 1.4.5'
-  spec.add_runtime_dependency 'pageme', '~> 0.0.3'
   spec.add_runtime_dependency 'log4r', '~> 1.1.10'
 
   spec.add_development_dependency 'yard'

data/lib/docparser/document.rb

@@ -4,7 +4,17 @@ module DocParser
   # @see Parser
   # @see Output
   class Document
-    attr_reader :filename, :doc, :encoding, :results
+    # @return [String] the filename of the current document
+    attr_reader :filename
+
+    # @return [Nokogiri::HTML::Document] a reference to the Nokogiri document
+    attr_reader :doc
+
+    # @return [String] the encoding of the document
+    attr_reader :encoding
+
+    # @return [Array] the results from this document
+    attr_reader :results
 
     # @return [String] the source of the document
     attr_reader :html

data/lib/docparser/output.rb

@@ -9,7 +9,7 @@ module DocParser
     def initialize(filename: filename)
       @rowcount = 0
       @filename = filename
-      raise ArgumentError, 'Please specify a filename' if filename.empty?
+      fail ArgumentError, 'Please specify a filename' if filename.empty?
       @file = open filename, 'w'
       classname = self.class.name.split('::').last
       @logger = Log4r::Logger.new("docparser::output::#{classname}")
@@ -49,7 +49,7 @@ module DocParser
 
     # Called when a row is added
     def write_row(row)
-      raise NotImplementedError.new('No row writer defined')
+      fail NotImplementedError, 'No row writer defined'
     end
 
     # Called before closing the file

data/lib/docparser/output/csv_output.rb

@@ -2,7 +2,6 @@ require 'csv'
 module DocParser
   # The CSVOutput class generates a CSV file containing all rows
   # @see Output
-
   class CSVOutput < Output
     # @!visibility private
     def open_file

data/lib/docparser/output/json_output.rb

@@ -11,7 +11,7 @@ module DocParser
     end
 
     def write_row(row)
-      raise MissingHeaderException if @header.nil? || @header.length == 0
+      fail MissingHeaderException if @header.nil? || @header.length == 0
 
       @file << ',' unless @file.pos <= 1
 

data/lib/docparser/output/yaml_output.rb

@@ -6,7 +6,7 @@ module DocParser
   class YAMLOutput < Output
     # @!visibility private
     def write_row(row)
-      raise MissingHeaderException if @header.nil? || @header.length == 0
+      fail MissingHeaderException if @header.nil? || @header.length == 0
       @doc ||= {}
 
       0.upto(@header.length - 1) do |counter|

data/lib/docparser/parser.rb

@@ -1,6 +1,5 @@
 require 'rubygems'
 require 'bundler/setup'
-require 'open-uri'
 require 'parallel'
 require 'set'
 require 'log4r'
@@ -8,7 +7,6 @@ require 'log4r/formatter/patternformatter'
 require 'docparser/version'
 require 'docparser/output'
 require 'docparser/document'
-require 'docparser/output/screen_output.rb'
 require 'docparser/output/csv_output.rb'
 require 'docparser/output/html_output.rb'
 require 'docparser/output/xlsx_output.rb'
@@ -86,7 +84,7 @@ module DocParser
       elsif output.is_a?(Array) && output.all? { |o| o.is_a? Output }
         @outputs = output
       elsif !output.nil?
-        raise ArgumentError, 'Invalid outputs specified'
+        fail ArgumentError, 'Invalid outputs specified'
       end
 
       @resultsets = Array.new(@outputs.length) { Set.new }
@@ -95,7 +93,7 @@ module DocParser
     def parallel_process(&block)
       @logger.info "Starting #{@num_processes} processes"
       option = RUBY_ENGINE == 'ruby' ? :in_processes : :in_threads
-      Parallel.map(@files, { option => @num_processes }) do |file|
+      Parallel.map(@files, option => @num_processes) do |file|
         # :nocov: #
         parse_doc(file, &block)
         # :nocov: #
@@ -129,6 +127,5 @@ module DocParser
         output.close
       end
     end
-
   end
 end

data/lib/docparser/version.rb

@@ -2,5 +2,5 @@
 # See README.md for information on using DocParser
 module DocParser
   # The current version of DocParser
-  VERSION = '0.1.6'
+  VERSION = '0.2.0'
 end

data/test/lib/docparser/document_test.rb

@@ -27,12 +27,6 @@ describe DocParser::Document do
     doc.xpath_content('xmltest > test').must_equal('Character Data')
   end
 
-  it 'should read remote contents' do
-    url = 'https://gist.github.com/jurriaan/3f2750aa546e3e6719cf/raw'
-    doc = DocParser::Document.new(filename: url, parser: @parser)
-    doc.html.must_equal(open(url).read)
-  end
-
   it 'should use the correct encoding' do
     file = File.join($SUPPORT_DIR, 'test_encoding.html')
     file2 = File.join($SUPPORT_DIR, 'test_encoding2.html')

data/test/lib/docparser/parser_test.rb

@@ -1,7 +1,7 @@
 require_relative '../../test_helper'
 describe DocParser::Parser do
   before do
-    SimpleCov.at_exit { }
+    SimpleCov.at_exit {}
   end
 
   after do

data/test/test_helper.rb

@@ -8,6 +8,7 @@ SimpleCov.start do
   # add_filter '/test/'
 end
 require 'minitest/autorun'
+require 'minitest/hell'
 require 'minitest/pride'
 require 'tempfile'
 require 'tmpdir'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: docparser
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.2.0
 platform: ruby
 authors:
 - Jurriaan Pruis
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-30 00:00:00.000000000 Z
+date: 2014-03-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -16,28 +16,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.0
+        version: 1.6.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.0
+        version: 1.6.1
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.4
+        version: 0.9.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.4
+        version: 0.9.1
 - !ruby/object:Gem::Dependency
   name: axlsx
   requirement: !ruby/object:Gem::Requirement
@@ -52,34 +52,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 2.0.1
-- !ruby/object:Gem::Dependency
-  name: terminal-table
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.4.5
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.4.5
-- !ruby/object:Gem::Dependency
-  name: pageme
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.0.3
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.0.3
 - !ruby/object:Gem::Dependency
   name: log4r
   requirement: !ruby/object:Gem::Requirement
@@ -164,7 +136,6 @@ files:
 - lib/docparser/output/json_output.rb
 - lib/docparser/output/multi_output.rb
 - lib/docparser/output/nil_output.rb
-- lib/docparser/output/screen_output.rb
 - lib/docparser/output/xlsx_output.rb
 - lib/docparser/output/yaml_output.rb
 - lib/docparser/parser.rb
@@ -178,7 +149,6 @@ files:
 - test/lib/docparser/output/json_output_test.rb
 - test/lib/docparser/output/multi_output_test.rb
 - test/lib/docparser/output/nil_output_test.rb
-- test/lib/docparser/output/screen_output_test.rb
 - test/lib/docparser/output/xlsx_output_test.rb
 - test/lib/docparser/output/yaml_output_test.rb
 - test/lib/docparser/output_test.rb
@@ -250,7 +220,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: DocParser is a Ruby Gem for webscraping
@@ -264,7 +234,6 @@ test_files:
 - test/lib/docparser/output/json_output_test.rb
 - test/lib/docparser/output/multi_output_test.rb
 - test/lib/docparser/output/nil_output_test.rb
-- test/lib/docparser/output/screen_output_test.rb
 - test/lib/docparser/output/xlsx_output_test.rb
 - test/lib/docparser/output/yaml_output_test.rb
 - test/lib/docparser/output_test.rb

data/lib/docparser/output/screen_output.rb

@@ -1,37 +0,0 @@
-require 'terminal-table'
-require 'pageme'
-module DocParser
-  # This Output can be used for debugging purposes.
-
-  # This output sends the results directly to the terminal and pipes all rows
-  # through a pager
-  # @see Output
-  class ScreenOutput < Output
-    # @!visibility private
-
-    include PageMe
-
-    def initialize
-      @tables = []
-      @rowcount = 0
-    end
-
-    def close
-      page do |p|
-        p.puts "Showing all #{@tables.length} rows:\n\n"
-        @tables.each do |table|
-          p.puts table
-        end
-      end
-    end
-
-    def write_row(row)
-      raise MissingHeaderException if @header.nil? || @header.length == 0
-      out = []
-      0.upto(@header.length - 1) do |counter|
-        out << [@header[counter], row[counter]]
-      end
-      @tables << Terminal::Table.new(rows: out)
-    end
-  end
-end

data/test/lib/docparser/output/screen_output_test.rb

@@ -1,54 +0,0 @@
-require_relative '../../../test_helper'
-require 'stringio'
-describe DocParser::ScreenOutput do
-  before do
-    Log4r::Logger['docparser'].level = Log4r::ERROR
-  end
-  after do
-    Log4r::Logger['docparser'].level = Log4r::INFO
-  end
-
-  it 'should not create a file' do
-    Dir.mktmpdir do |dir|
-      filename = File.join(dir, '*')
-      DocParser::ScreenOutput.new
-      Dir[filename].must_be_empty
-    end
-  end
-
-  it 'must give the correct rowcount' do
-    output = DocParser::ScreenOutput.new
-    output.header = 'test', 'the', 'header'
-    output.rowcount.must_equal 0
-    output.add_row %w(aap noot mies)
-    output.add_row %w(aap noot mies)
-    output.rowcount.must_equal 2
-  end
-
-  it 'must have a header' do
-    output = DocParser::ScreenOutput.new
-    lambda do
-      output.add_row %w(aap noot mies)
-    end.must_raise(DocParser::MissingHeaderException)
-  end
-
-  it 'must output the data after close' do
-    $out = StringIO.new
-    output = Class.new DocParser::ScreenOutput do
-      def page(*args, &p)
-        args << p
-        args.compact!
-        page_to $out, args
-      end
-    end.new
-    output.header = 'test', 'the', 'header'
-    output.add_row ['aap1' , '', 'mies']
-    output.add_row %w(aap2 mies1)
-    output.close
-    out = $out.string
-    out.must_include 'header'
-    out.must_include 'aap1'
-    out.must_include 'mies1'
-    out.must_include 'mies'
-  end
-end