jules 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f646200a27845170536bad15df78cd74f9a5fca4
-  data.tar.gz: c456c90b2e122bb9598323b751e6066671c21f1a
+  metadata.gz: f4324208ffa6000790da1143ead1d0fa1c460dc5
+  data.tar.gz: ccd6ca4f08cf1b3e726b734bfed6b835ce8fa24d
 SHA512:
-  metadata.gz: 5ab95e69f9e86dc1090bc9cf96be3439f00cd6af2a69225c100851ad54a53c830761e7b597832bd771d6b9820a4383145e56360266ac02a2f1518849bf7c314e
-  data.tar.gz: 041c7c4b5135becebb8d4e925a0f7035e27bf7ec3b9b806c8c43e3ddaa8aecc33a204841538471bc09bbb07004e1c2c6afdb8fb8f7b536eac4af3edfa476db82
+  metadata.gz: 5cb30e73c092708515da863d4cbe399bf56b4f18192c506207c3dd922754cd805f1073b5ba7671289d847d9eac988a58caee43ce9df67c2b582c4de4776e9c55
+  data.tar.gz: 9d715e3dadd73bcc2636e25d740becbbfb081dbd1ab8a978ed0920cd3b74b8ca9fe368a41fd89cc399c8811bce33935db8c832823924ce958f211cdfddfd8275

data/README.md

@@ -1,40 +1,71 @@
-**Hi!** - Don't use this gem _yet_, it's still very much being developed.
+# jules
+> Experimental
 
-# Jules
-A data mining scraper with a high level of abstraction. It's capable of finding _lists_, _menus_, _titles_ and _contact data_.
-
-Jules uses semantics, patterns and NLP to find data, so you don't have to specify exactly where it is. You'll no longer have to make different scrapers for every new website you want to scrape.
+A fast new way to write scrapers.
+This is still an ongoing project.
 
 ~~~ruby
-gem 'jules'
+require 'open-uri'
+require 'jules'
+source = URI.parse('https://news.ycombinator.com').read
+
+filters = {
+  title:    'td.title a',
+  comments: [/(\d+) comments/, :optional],
+  points:   /(\d+) points/
+}
+
+items = Jules.collect(source, filters)
+# [{title: '2 years with Angular', comments: '95', points: '245'},
+#  {title: 'PolarSSL is now a part of ARM', comments: '13', points: '48'},
+#  {title: 'My boys love 1986 computing', comments: '25', points: '105'},
+#  {title: 'Kill init by touching a bunch of files', comments: '66', points: '102'},
+#  ...
 ~~~
 
-## Examples
-The following examples show you how to use Jules.
-### Lists
+## How?
+
+Jules uses the repitition of HTML structure. It rearranges the document using locality sensitive hashing (Simhash). It then continues to extract data using the user-specified filters.
+
+## Filters
+
+Filters can be
+- Strings (CSS selector / XPath query)
+- Regexp
+- Anonymous methods (`lambda`)
+
+By default filters are required fields. If a field is optional, mark it with `['#example', :optional]`.
+
+## Options
+
+### Enabled HTML elements
+By default, `div`, `tr` or `li` are enabled HTML elements for repitition. If a website wraps every item in `ul` and `div` elements, do this:
+
 ~~~ruby
-html = File.open('web-page.html', 'rb') { |f| f.read }
-j = Jules::HTML(html)
-lists = j.lists
+Jules.collect(html, filters, ['ul', 'div'])
 ~~~
 
-The following example gets lists only when they contain certain data types.
+### Examples
+#### The Onion
 ~~~ruby
-j = Jules::HTML(html)
-lists = j.lists(
-  required: [:date, :price],
-  optional: [:download_link]
-)
+require 'open-uri'
+require 'jules'
+source = URI.parse('http://www.theonion.com/search/?q=why').read
+
+filters = {
+  title:   'h1',
+  pubdate: /(\d{2}\.\d{2}\.\d{2})/,
+  img:     'img'
+}
+
+items = Jules.collect(source, filters)
+# [{title: 'Why Are We Leaving Facebook?', pubdate: '10.10.13', img: 'http://o.onionstatic.com/images/23/23823/16x9/350.jpg?0553'},
+#  {title: 'Why Are We Filing For Disability?', pubdate: '01.24.14', img: 'http://o.onionstatic.com/images/25/25070/16x9/350.jpg?8738'},
+#  {title: 'Why Are We Waiting To Have Children?', pubdate: '07.10.14', img: 'http://o.onionstatic.com/images/26/26746/16x9/350.jpg?7206'},
+#  {title: 'Why Are We Postponing The Wedding?', pubdate: '04.25.13', img: '/images/21/21801/16x9/350.jpg?8189'},
+#  {title: 'Why Are We Canceling Our Netflix Account?', pubdate: '01.09.14', img: 'http://o.onionstatic.com/images/24/24668/16x9/350.jpg?3803'},
+#  {title: 'Why Aren't We Watching The Olympics?', pubdate: '02.20.14', img: 'http://o.onionstatic.com/images/25/25345/16x9/350.jpg?2178'},
+#  ...
 ~~~
 
-### Jules Abstractions
-- Lists
-- Titles
-- Menus
-
-### Jules Data Types
-- Date *:date*
-- Price *:price*
-- Filesize *:filesize*
-- Download url *:download_url*
-- Telephone number *:telephone_number*
+See the [tests](test/) folder for more examples.

data/lib/enumerable/sugar.rb

@@ -0,0 +1,24 @@
+module Enumerable
+  def each_with_previous
+    self.inject(nil){|prev, curr| yield prev, curr; curr}
+    self
+  end
+end
+
+class Array
+  def find_by_partial_hash(hash)
+    self.select { |h| h.includes_hash?(hash) }
+  end
+end
+
+class Hash
+  def includes_hash?(other)
+    included = true
+
+    other.each do |key, value|
+      included &= self[key] == other[key]
+    end
+
+    included
+  end
+end

data/lib/jules.rb

@@ -1,25 +1,171 @@
 require 'jules/version'
-
 require 'nokogiri'
-require 'whatlanguage'
-require 'damerau-levenshtein'
+require 'simhash'
+require 'descriptive_statistics'
 
 require 'nokogiri/sugar'
-require 'damerau_levenshtein/sugar'
+require 'enumerable/sugar'
+require 'simhash/sugar'
 
-require 'jules/abstractions/list'
-require 'jules/abstractions/title'
+module Jules
+  SIMILARITY_THRESHOLD =  0.6
+  SIMHASH_BITLENGTH =     128
+  ELEMENTS = ['div', 'li', 'tr', 'article']
 
-require 'jules/miners/titles'
-require 'jules/miners/lists'
+  def self.collect(html, filters, elements=Jules::ELEMENTS)
+    unless html.is_a?(String) || html.is_a?(File)
+      raise ArgumentError, 'html not a String or File'
+    end
 
-require 'jules/document'
+    raise ArgumentError, 'filters argument empty' if filters.nil?
+    raise ArgumentError, 'filters not a Hash' unless filters.is_a? Hash
+    raise ArgumentError, 'elements not an Array' unless elements.is_a? Array
 
-module Jules
-  NAME = 'Jules'
-  LICENSE = 'See LICENSE for licensing details.'
+    document =  Nokogiri::HTML(html)
+    trees =     Jules.rearrange_trees(document, elements)
+    clusters =  Jules.cluster_trees(trees)
+    clusters =  Jules.grade_clusters(clusters, filters)
+    Jules.items(clusters, filters)
+  end
+
+  # Rearranges DOM trees with Simhash
+  def rearrange_trees(document, elements=Jules::ELEMENTS)
+    trees = []
+    document_length = document.inner_html.length
+    xpath = elements.map{ |x| '//' + x }.join('|')
+    document.search(xpath).each do |tree|
+      structure = Nokogiri::XML.remove_markup_outline(tree.to_outline).strip
+      if structure.empty?
+        tree.remove # This HTML tree does not contain any structure
+        next
+      end
+
+      # Tree should be smaller than 50% of document size
+      next if (tree.inner_html.length / document_length.to_f * 100) > 50
+
+      trees << {
+        node:  tree,
+        depth: tree.depth,
+        simhash: structure.simhash(hashbits: Jules::SIMHASH_BITLENGTH),
+        index:   tree.xpath('count(preceding-sibling::*)').to_i
+      }
+    end
+    trees.sort_by { |tree| tree[:simhash] }
+  end
+  module_function :rearrange_trees
+
+  # Cluster trees based on similarity
+  def cluster_trees(trees)
+    clusters, cluster = [], [trees[0]]
 
-  def self.❨╯°□°❩╯︵┻━┻
-    puts 'Calm down, bro'
+    trees.each_with_previous do |prev, tree|
+      next if prev.nil? # first item
+      similarity = Simhash.similarity(prev[:simhash], tree[:simhash])
+      if similarity < Jules::SIMILARITY_THRESHOLD
+        clusters << cluster
+        cluster = [tree]
+      else
+        cluster << tree
+      end
+    end
+    clusters << cluster if cluster.count > 0
+
+    # Reject clusters that only contain 1 tree
+    clusters.reject { |cluster| cluster.count < 2 }
   end
+  module_function :cluster_trees
+
+  # Grade clusters
+  def grade_clusters(clusters, filters)
+    # Map trees inside cluster hash, to make room for metadata
+    clusters.map! {|cluster| {trees: cluster} }
+
+    clusters.each do |cluster|
+      cluster[:score] = 0
+      cluster[:trees].each do |tree|
+        tree = filter_item(tree, filters)
+
+        cluster[:score] += tree[:item].to_h.count
+      end
+      cluster[:items] = cluster[:trees]
+        .sort_by { |tree| tree[:index] }
+        .map {     |tree| tree[:item] }
+        .reject {  |item| item.nil? }
+
+      # Bonus points if all trees are at same depth
+      depth_sd = cluster[:trees].map { |tree| tree[:depth] }.standard_deviation
+      cluster[:score] += 1 if depth_sd < 0.5
+      cluster[:score_ratio] = cluster[:score] / cluster[:trees].count.to_f
+    end
+
+    clusters
+      .reject{ |cluster| cluster[:items].count == 0 }
+      .sort_by { |cluster| cluster[:score] }
+  end
+  module_function :grade_clusters
+
+  # Try to find a single item in DOM tree
+  def filter_item(tree, filters)
+    filters.each do |key, filter|
+      filter = filter[0] if filter.class == Array # TODO
+
+      case filter 
+      when String
+        value = tree[:node].at(filter)
+        if value
+          tree[:item] ||= {}
+          if value.text.empty?
+            tree[:item][key] = value['src'] || value['href']
+          else
+            tree[:item][key] = value.text
+          end
+        end
+      when Regexp
+        match = filter.match(tree[:node].inner_html)
+        if match && match.captures
+          tree[:item] ||= {}
+          tree[:item][key] = match.captures[0]
+        end
+      when Proc
+        tree[:item][key] = filter.call(tree[:node])
+      else
+        raise ArgumentError, "#{filter} is not a valid filter type"
+      end
+    end
+    tree
+  end
+  module_function :filter_item
+
+  # Pick items from best cluster, or combine items from multiple clusters
+  def items(clusters, filters)
+    # Clusters need to have at least one item field per tree
+    clusters
+      .select!{|cluster| cluster[:score_ratio] >= 1.0 }
+
+    return [] if clusters.to_a.count == 0
+
+    # Find unique items, start with highest scoring clusters
+    items = []
+
+    # Pick two best groups
+    clusters = clusters.sort_by {|cluster| cluster[:score_ratio]}.reverse[0, 2]
+    clusters.each do |cluster|
+      cluster[:items].each do |item|
+        items << item
+      end
+    end
+    items.uniq!
+
+    # Keep best items of partial duplicates
+    items.delete_if do |item|
+      items.find_by_partial_hash(item).map(&:count).max > item.count
+    end
+    
+    return items
+  end
+  module_function :items
+
+
+  # Helper methods
+  def self.simhash(data); data.simhash(hashbits: Jules::SIMHASH_BITLENGTH); end
 end

data/lib/jules/version.rb

@@ -1,3 +1,3 @@
 module Jules
-  VERSION = '0.0.1'
+  VERSION = '0.1.0'
 end

data/lib/nokogiri/sugar.rb

@@ -1,33 +1,33 @@
 # Add some sugar to Nokogiri
-# http://stackoverflow.com/questions/7176094/how-do-i-create-an-outline-of-the-html-tag-structure-on-the-page-using-nokogiri
-# http://stackoverflow.com/questions/5694759/how-do-you-calculate-the-number-of-levels-of-descendants-of-a-nokogiri-node
 class Nokogiri::XML::Node
   def to_outline
     children.find_all(&:element?).map(&:to_outline).join
   end
   def depth
     ancestors.size
-    # The following is ~10x slower: xpath('count(ancestor::node())').to_i
-  end
-  def leaves
-    xpath('.//*[not(*)]').to_a
-  end
-  def height
-    tallest = leaves.map{ |leaf| leaf.depth }.max
-    tallest ? tallest - depth : 0
-  end
-  def deepest_leaves
-    by_height = leaves.group_by{ |leaf| leaf.depth }
-    by_height[ by_height.keys.max ]
-  end
-  def deepest_level
-    by_height = leaves.group_by{ |leaf| leaf.depth }
-    by_height.keys.max
   end
 end
 
 class Nokogiri::XML::Element
   def to_outline
-    "<#{name}>#{super}</#{name}>"
+    if self['id'] && self['class']
+      "<#{name} id=\"#{self['id']}\" class=\"#{self['class']}\">#{super}</#{name}>"
+    elsif self['id']
+      "<#{name} id=\"#{self['id']}\">#{super}</#{name}>"
+    elsif self['class']
+      "<#{name} class=\"#{self['class']}\">#{super}</#{name}>"
+    else
+      "<#{name}>#{super}</#{name}>"
+    end
+  end
+end
+
+module Nokogiri::XML
+  def self.remove_markup_outline(str)
+    f = Nokogiri::XML.fragment(str)
+    ['b', 'strong', 'strike', 'i', 'u'].each do |s|
+      f.search('.//' + s).remove
+    end
+    f.to_outline
   end
 end

data/lib/simhash/sugar.rb

@@ -0,0 +1,19 @@
+module Simhash
+  def self.hamming_distance(hash1, hash2)
+    (hash1 ^ hash2).to_s(2).count('1')
+  end
+
+  def self.similarity(hash1, hash2)
+    1 - (Simhash.hamming_distance(hash1, hash2) / Jules::SIMHASH_BITLENGTH.to_f)
+  end
+
+  # Bitwise left rotate
+  def self.lotate(hash, n=1)
+    (hash << n | hash >> (Jules::SIMHASH_BITLENGTH - n)) &
+      ('1'*Jules::SIMHASH_BITLENGTH).to_i(2)
+  end
+
+  # Cluster
+  def self.cluster(simhashes, threshold)
+  end
+end

metadata

@@ -1,89 +1,70 @@
 --- !ruby/object:Gem::Specification
 name: jules
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - Bart Olsthoorn
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-06 00:00:00.000000000 Z
+date: 2014-11-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.6'
 - !ruby/object:Gem::Dependency
-  name: whatlanguage
+  name: simhash
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.2'
 - !ruby/object:Gem::Dependency
-  name: damerau-levenshtein
+  name: descriptive_statistics
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-description: High level data mining scraper using patterns, semantics and NLP.
+        version: '2.4'
+description: Data mining scraper using local hashing.
 email:
 - bartolsthoorn@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- LICENSE
 - README.md
-- lib/damerau_levenshtein/sugar.rb
+- lib/enumerable/sugar.rb
 - lib/jules.rb
-- lib/jules/abstractions/list.rb
-- lib/jules/abstractions/title.rb
-- lib/jules/document.rb
-- lib/jules/miners/lists.rb
-- lib/jules/miners/titles.rb
 - lib/jules/version.rb
 - lib/nokogiri/sugar.rb
+- lib/simhash/sugar.rb
 homepage: http://github.com/bartolsthoorn/jules
 licenses:
 - MIT
@@ -104,8 +85,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.4
 signing_key: 
 specification_version: 4
-summary: High level data mining scraper using patterns, semantics and NLP.
+summary: Data mining scraper using local hashing.
 test_files: []

data/LICENSE

@@ -1,22 +0,0 @@
-Copyright (c) 2014  Bart Olsthoorn, website: bartolsthoorn.nl
-
-Permission is hereby granted, free of charge, to any person
-obtaining a copy of this software and associated documentation
-files (the "Software"), to deal in the Software without
-restriction, including without limitation the rights to use,
-copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the
-Software is furnished to do so, subject to the following
-conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
-OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
-HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-OTHER DEALINGS IN THE SOFTWARE.

data/lib/damerau_levenshtein/sugar.rb

@@ -1,9 +0,0 @@
-module DamerauLevenshtein
-  # returns 1.0 for completely different strings
-  # returns 0.0 for completely identical strings
-  def self.relative(a, b)
-    length = [a.length, b.length].max
-    return DamerauLevenshtein.distance(a, b).to_f / length
-  end
-end
-DL = DamerauLevenshtein

data/lib/jules/abstractions/list.rb

@@ -1,10 +0,0 @@
-module Jules
-  module Abstractions
-    class List
-      attr_accessor :title, :content
-
-      def initialize(title, content)
-      end
-    end
-  end
-end

data/lib/jules/abstractions/title.rb

@@ -1,21 +0,0 @@
-module Jules
-  module Abstractions
-    class Title
-      attr_accessor :level, :text, :language
-
-      def initialize(level, text)
-        raise ArgumentError if level.class != Fixnum
-        raise ArgumentError if text.class != String
-
-        # H1 means level 1, etc.
-        @level = level
-
-        # Name contains the actual title data
-        @text = text
-
-        # Language detection
-        @language = text.language
-      end
-    end
-  end
-end

data/lib/jules/document.rb

@@ -1,23 +0,0 @@
-module Jules
-  class << self
-    def HTML(html, options = {})
-      raise ArgumentError if html.class != String
-      Jules::Document.new html
-    end
-  end
-
-  class Document
-    attr_accessor :html
-
-    def initialize(html)
-      @html = Nokogiri::HTML::Document.parse html
-    end
-
-    def titles
-      Jules::Miners.titles @html
-    end
-    def lists
-      Jules::Miners.lists @html
-    end
-  end
-end

data/lib/jules/miners/lists.rb

@@ -1,114 +0,0 @@
-module Jules
-  module Miners
-    class << self
-      def zebra_list?(list_items)
-        # Use outlines to see if lists are structured like
-        # AAAAAA        (stride 0)
-        # ABABAB        (stride 1)
-        # AABBAABB      (stride 2)
-        # AAABBBAAABBB  (stride 3)
-
-        outlines = list_items.map(&:to_outline)
-
-        # First test for non_zebra AAAAA
-        errors = []
-        outlines.each_with_index do |outline, i|
-          previous_outline =  outlines[i - 1]
-          errors << DL.relative(outline, previous_outline)
-        end
-        avg_error = errors.inject(:+) / errors.size
-        stride0_certainty = 1 - avg_error
-        if stride0_certainty == 1.0
-          return {stride: 0, certainty: stride0_certainty }
-        end
-
-        # Not certain it's AAAA, so continue to check for ABABAB
-        errors = []
-        outlines.each_with_index do |outline, i|
-          before_outline   =  outlines[i - 2]
-          previous_outline =  outlines[i - 1]
-          next_outline =      outlines[i + 1]
-
-          if previous_outline && next_outline
-            zebra_1 = DL.relative(previous_outline, next_outline)
-            zebra_2 = DL.relative(outline, before_outline)
-            # zebra should be close to 0.0
-            errors << (zebra_1 + zebra_2) / 2
-          end
-        end
-        avg_error = errors.inject(:+) / errors.size
-        stride1_certainty = 1 - avg_error
-        if stride1_certainty > stride0_certainty
-          {stride: 1, certainty: stride1_certainty}
-        else
-          {stride: 0, certainty: stride0_certainty}
-        end
-      end
-
-      def unzebrify(result)
-        result.each_with_index do |list, r|
-          nodes = list[:items].map{ |item| item[:node] }
-          zebra = zebra_list?(nodes)
-
-          if zebra[:stride] == 1
-            puts 'ZEBRA!'
-            # Merge nodes
-            #list[:items].each_with_index do |item, i|
-            #  list[:items][i+1][:node] = [
-            #    list[:items][i][:node],
-            #    list[:items][i+1][:node]
-            #  ]
-            #  list[:items].delete_at(i)
-            #end
-          end
-        end
-        result
-      end
-
-      def lists(html)
-        depth = html.deepest_level
-        result = []
-
-        depth.times do |level|
-          xpath = '/*' * (level + 1)
-          nodes = html.xpath(xpath)
-          items = []
-          last_node = nodes.first
-          nodes.each do |node|
-            next unless [:li, :div, :tr].include? node.name.to_sym
-
-            if items.last && items.last[:node].name != node.name
-              # Store items as collection when 2 or more found
-              if items.count > 1
-                result << {
-                  level: level,
-                  items: items }
-              end
-              items = []
-            end
-
-            # Node is same element family as previous node
-            # But it could still be part of a one multiple node item
-            if items.last
-              # Is current node different from the previous one?
-              if items.last[:node].to_outline != node.to_outline
-                # Next node outline same as previous node outline?
-              end
-            end
-            items << {
-              titles: Jules::Miners.titles(node),
-              node: node,
-              text: node.text
-            }
-          end
-          if items.count > 1
-            result << {
-              level: level,
-              items: items }
-          end
-        end
-        unzebrify(result)
-      end
-    end
-  end
-end

data/lib/jules/miners/titles.rb

@@ -1,19 +0,0 @@
-module Jules
-  module Miners
-    class << self
-      def titles(html)
-        titles = []
-
-        10.times do |i|
-          level = i + 1
-          html.xpath('.//h' + level.to_s).each do |title|
-            name = title.text
-            titles << Jules::Abstractions::Title.new(level, name)
-          end
-        end
-
-        titles
-      end
-    end
-  end
-end