jules 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f646200a27845170536bad15df78cd74f9a5fca4
+  data.tar.gz: c456c90b2e122bb9598323b751e6066671c21f1a
+SHA512:
+  metadata.gz: 5ab95e69f9e86dc1090bc9cf96be3439f00cd6af2a69225c100851ad54a53c830761e7b597832bd771d6b9820a4383145e56360266ac02a2f1518849bf7c314e
+  data.tar.gz: 041c7c4b5135becebb8d4e925a0f7035e27bf7ec3b9b806c8c43e3ddaa8aecc33a204841538471bc09bbb07004e1c2c6afdb8fb8f7b536eac4af3edfa476db82

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2014  Bart Olsthoorn, website: bartolsthoorn.nl
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,40 @@
+**Hi!** - Don't use this gem _yet_, it's still very much being developed.
+
+# Jules
+A data mining scraper with a high level of abstraction. It's capable of finding _lists_, _menus_, _titles_ and _contact data_.
+
+Jules uses semantics, patterns and NLP to find data, so you don't have to specify exactly where it is. You'll no longer have to make different scrapers for every new website you want to scrape.
+
+~~~ruby
+gem 'jules'
+~~~
+
+## Examples
+The following examples show you how to use Jules.
+### Lists
+~~~ruby
+html = File.open('web-page.html', 'rb') { |f| f.read }
+j = Jules::HTML(html)
+lists = j.lists
+~~~
+
+The following example gets lists only when they contain certain data types.
+~~~ruby
+j = Jules::HTML(html)
+lists = j.lists(
+  required: [:date, :price],
+  optional: [:download_link]
+)
+~~~
+
+### Jules Abstractions
+- Lists
+- Titles
+- Menus
+
+### Jules Data Types
+- Date *:date*
+- Price *:price*
+- Filesize *:filesize*
+- Download url *:download_url*
+- Telephone number *:telephone_number*

data/lib/damerau_levenshtein/sugar.rb

@@ -0,0 +1,9 @@
+module DamerauLevenshtein
+  # returns 1.0 for completely different strings
+  # returns 0.0 for completely identical strings
+  def self.relative(a, b)
+    length = [a.length, b.length].max
+    return DamerauLevenshtein.distance(a, b).to_f / length
+  end
+end
+DL = DamerauLevenshtein

data/lib/jules.rb

@@ -0,0 +1,25 @@
+require 'jules/version'
+
+require 'nokogiri'
+require 'whatlanguage'
+require 'damerau-levenshtein'
+
+require 'nokogiri/sugar'
+require 'damerau_levenshtein/sugar'
+
+require 'jules/abstractions/list'
+require 'jules/abstractions/title'
+
+require 'jules/miners/titles'
+require 'jules/miners/lists'
+
+require 'jules/document'
+
+module Jules
+  NAME = 'Jules'
+  LICENSE = 'See LICENSE for licensing details.'
+
+  def self.❨╯°□°❩╯︵┻━┻
+    puts 'Calm down, bro'
+  end
+end

data/lib/jules/abstractions/list.rb

@@ -0,0 +1,10 @@
+module Jules
+  module Abstractions
+    class List
+      attr_accessor :title, :content
+
+      def initialize(title, content)
+      end
+    end
+  end
+end

data/lib/jules/abstractions/title.rb

@@ -0,0 +1,21 @@
+module Jules
+  module Abstractions
+    class Title
+      attr_accessor :level, :text, :language
+
+      def initialize(level, text)
+        raise ArgumentError if level.class != Fixnum
+        raise ArgumentError if text.class != String
+
+        # H1 means level 1, etc.
+        @level = level
+
+        # Name contains the actual title data
+        @text = text
+
+        # Language detection
+        @language = text.language
+      end
+    end
+  end
+end

data/lib/jules/document.rb

@@ -0,0 +1,23 @@
+module Jules
+  class << self
+    def HTML(html, options = {})
+      raise ArgumentError if html.class != String
+      Jules::Document.new html
+    end
+  end
+
+  class Document
+    attr_accessor :html
+
+    def initialize(html)
+      @html = Nokogiri::HTML::Document.parse html
+    end
+
+    def titles
+      Jules::Miners.titles @html
+    end
+    def lists
+      Jules::Miners.lists @html
+    end
+  end
+end

data/lib/jules/miners/lists.rb

@@ -0,0 +1,114 @@
+module Jules
+  module Miners
+    class << self
+      def zebra_list?(list_items)
+        # Use outlines to see if lists are structured like
+        # AAAAAA        (stride 0)
+        # ABABAB        (stride 1)
+        # AABBAABB      (stride 2)
+        # AAABBBAAABBB  (stride 3)
+
+        outlines = list_items.map(&:to_outline)
+
+        # First test for non_zebra AAAAA
+        errors = []
+        outlines.each_with_index do |outline, i|
+          previous_outline =  outlines[i - 1]
+          errors << DL.relative(outline, previous_outline)
+        end
+        avg_error = errors.inject(:+) / errors.size
+        stride0_certainty = 1 - avg_error
+        if stride0_certainty == 1.0
+          return {stride: 0, certainty: stride0_certainty }
+        end
+
+        # Not certain it's AAAA, so continue to check for ABABAB
+        errors = []
+        outlines.each_with_index do |outline, i|
+          before_outline   =  outlines[i - 2]
+          previous_outline =  outlines[i - 1]
+          next_outline =      outlines[i + 1]
+
+          if previous_outline && next_outline
+            zebra_1 = DL.relative(previous_outline, next_outline)
+            zebra_2 = DL.relative(outline, before_outline)
+            # zebra should be close to 0.0
+            errors << (zebra_1 + zebra_2) / 2
+          end
+        end
+        avg_error = errors.inject(:+) / errors.size
+        stride1_certainty = 1 - avg_error
+        if stride1_certainty > stride0_certainty
+          {stride: 1, certainty: stride1_certainty}
+        else
+          {stride: 0, certainty: stride0_certainty}
+        end
+      end
+
+      def unzebrify(result)
+        result.each_with_index do |list, r|
+          nodes = list[:items].map{ |item| item[:node] }
+          zebra = zebra_list?(nodes)
+
+          if zebra[:stride] == 1
+            puts 'ZEBRA!'
+            # Merge nodes
+            #list[:items].each_with_index do |item, i|
+            #  list[:items][i+1][:node] = [
+            #    list[:items][i][:node],
+            #    list[:items][i+1][:node]
+            #  ]
+            #  list[:items].delete_at(i)
+            #end
+          end
+        end
+        result
+      end
+
+      def lists(html)
+        depth = html.deepest_level
+        result = []
+
+        depth.times do |level|
+          xpath = '/*' * (level + 1)
+          nodes = html.xpath(xpath)
+          items = []
+          last_node = nodes.first
+          nodes.each do |node|
+            next unless [:li, :div, :tr].include? node.name.to_sym
+
+            if items.last && items.last[:node].name != node.name
+              # Store items as collection when 2 or more found
+              if items.count > 1
+                result << {
+                  level: level,
+                  items: items }
+              end
+              items = []
+            end
+
+            # Node is same element family as previous node
+            # But it could still be part of a one multiple node item
+            if items.last
+              # Is current node different from the previous one?
+              if items.last[:node].to_outline != node.to_outline
+                # Next node outline same as previous node outline?
+              end
+            end
+            items << {
+              titles: Jules::Miners.titles(node),
+              node: node,
+              text: node.text
+            }
+          end
+          if items.count > 1
+            result << {
+              level: level,
+              items: items }
+          end
+        end
+        unzebrify(result)
+      end
+    end
+  end
+end

data/lib/jules/miners/titles.rb

@@ -0,0 +1,19 @@
+module Jules
+  module Miners
+    class << self
+      def titles(html)
+        titles = []
+
+        10.times do |i|
+          level = i + 1
+          html.xpath('.//h' + level.to_s).each do |title|
+            name = title.text
+            titles << Jules::Abstractions::Title.new(level, name)
+          end
+        end
+
+        titles
+      end
+    end
+  end
+end

data/lib/jules/version.rb

@@ -0,0 +1,3 @@
+module Jules
+  VERSION = '0.0.1'
+end

data/lib/nokogiri/sugar.rb

@@ -0,0 +1,33 @@
+# Add some sugar to Nokogiri
+# http://stackoverflow.com/questions/7176094/how-do-i-create-an-outline-of-the-html-tag-structure-on-the-page-using-nokogiri
+# http://stackoverflow.com/questions/5694759/how-do-you-calculate-the-number-of-levels-of-descendants-of-a-nokogiri-node
+class Nokogiri::XML::Node
+  def to_outline
+    children.find_all(&:element?).map(&:to_outline).join
+  end
+  def depth
+    ancestors.size
+    # The following is ~10x slower: xpath('count(ancestor::node())').to_i
+  end
+  def leaves
+    xpath('.//*[not(*)]').to_a
+  end
+  def height
+    tallest = leaves.map{ |leaf| leaf.depth }.max
+    tallest ? tallest - depth : 0
+  end
+  def deepest_leaves
+    by_height = leaves.group_by{ |leaf| leaf.depth }
+    by_height[ by_height.keys.max ]
+  end
+  def deepest_level
+    by_height = leaves.group_by{ |leaf| leaf.depth }
+    by_height.keys.max
+  end
+end
+
+class Nokogiri::XML::Element
+  def to_outline
+    "<#{name}>#{super}</#{name}>"
+  end
+end

metadata

@@ -0,0 +1,111 @@
+--- !ruby/object:Gem::Specification
+name: jules
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Bart Olsthoorn
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-05-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: whatlanguage
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: damerau-levenshtein
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: High level data mining scraper using patterns, semantics and NLP.
+email:
+- bartolsthoorn@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- lib/damerau_levenshtein/sugar.rb
+- lib/jules.rb
+- lib/jules/abstractions/list.rb
+- lib/jules/abstractions/title.rb
+- lib/jules/document.rb
+- lib/jules/miners/lists.rb
+- lib/jules/miners/titles.rb
+- lib/jules/version.rb
+- lib/nokogiri/sugar.rb
+homepage: http://github.com/bartolsthoorn/jules
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: High level data mining scraper using patterns, semantics and NLP.
+test_files: []