html-hierarchy-extractor 1.0.2 → 1.0.9

data/Rakefile

@@ -1,58 +0,0 @@
-# encoding: utf-8
-
-require 'rubygems'
-require 'bundler'
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts 'Run `bundle install` to install missing gems'
-  exit e.status_code
-end
-require 'rake'
-
-require 'jeweler'
-require_relative 'lib/version'
-Jeweler::Tasks.new do |gem|
-  # gem is a Gem::Specification...
-  # see http://guides.rubygems.org/specification-reference/ for more options
-  gem.name = 'html-hierarchy-extractor'
-  gem.version = HTMLHierarchyExtractorVersion.to_s
-  gem.homepage = 'http://github.com/pixelastic/html-hierarchy-extractor'
-  gem.license = 'MIT'
-  gem.summary = 'Extract HTML hierarchy (headings and content) into a' \
-                ' list of items'
-  gem.description = 'Take any arbitrary HTML as input and extract its' \
-                    ' hierarchy as a list of items, including parents and' \
-                    ' contents.' \
-                    'It is primarily intended to be used along with Algolia,' \
-                    ' to improve the relevance of searching into huge chunks' \
-                    ' of text'
-  gem.email = 'tim@pixelastic.com'
-  gem.authors = ['Tim Carry']
-  # dependencies defined in Gemfile
-end
-Jeweler::RubygemsDotOrgTasks.new
-
-require 'rake/testtask'
-Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
-  test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
-end
-
-require 'rspec/core'
-require 'rspec/core/rake_task'
-RSpec::Core::RakeTask.new(:spec) do |spec|
-  spec.rspec_opts = '--color --format documentation'
-  spec.pattern = FileList['spec/**/*_spec.rb']
-end
-task test: :spec
-
-desc 'Code coverage detail'
-task :coverage do
-  ENV['COVERAGE'] = 'true'
-  Rake::Task['spec'].execute
-end
-
-task default: :test

data/VERSION

@@ -1 +0,0 @@
-0.1.0

data/html-hierarchy-extractor.gemspec

@@ -1,99 +0,0 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
-# -*- encoding: utf-8 -*-
-# stub: html-hierarchy-extractor 1.0.2 ruby lib
-
-Gem::Specification.new do |s|
-  s.name = "html-hierarchy-extractor"
-  s.version = "1.0.2"
-
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.require_paths = ["lib"]
-  s.authors = ["Tim Carry"]
-  s.date = "2016-07-20"
-  s.description = "Take any arbitrary HTML as input and extract its hierarchy as a list of items, including parents and contents.It is primarily intended to be used along with Algolia, to improve the relevance of searching into huge chunks of text"
-  s.email = "tim@pixelastic.com"
-  s.extra_rdoc_files = [
-    "LICENSE.txt",
-    "README.md"
-  ]
-  s.files = [
-    ".coveralls.yml",
-    ".document",
-    ".rspec",
-    ".rubocop.yml",
-    ".travis.yml",
-    "CONTRIBUTING.md",
-    "Gemfile",
-    "Guardfile",
-    "LICENSE.txt",
-    "README.md",
-    "Rakefile",
-    "VERSION",
-    "html-hierarchy-extractor.gemspec",
-    "lib/html-hierarchy-extractor.rb",
-    "lib/version.rb",
-    "scripts/bump_version",
-    "scripts/check_flay",
-    "scripts/check_flog",
-    "scripts/coverage",
-    "scripts/git_hooks/pre-commit",
-    "scripts/git_hooks/pre-push",
-    "scripts/lint",
-    "scripts/release",
-    "scripts/test",
-    "scripts/test_ci",
-    "scripts/watch",
-    "spec/html_hierarchy_extractor_spec.rb",
-    "spec/spec_helper.rb",
-    "spec/spec_helper_simplecov.rb"
-  ]
-  s.homepage = "http://github.com/pixelastic/html-hierarchy-extractor"
-  s.licenses = ["MIT"]
-  s.rubygems_version = "2.4.8"
-  s.summary = "Extract HTML hierarchy (headings and content) into a list of items"
-
-  if s.respond_to? :specification_version then
-    s.specification_version = 4
-
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<awesome_print>, ["~> 1.6"])
-      s.add_runtime_dependency(%q<json>, ["~> 1.8"])
-      s.add_runtime_dependency(%q<nokogiri>, ["~> 1.6"])
-      s.add_development_dependency(%q<coveralls>, ["~> 0.8"])
-      s.add_development_dependency(%q<flay>, ["~> 2.6"])
-      s.add_development_dependency(%q<flog>, ["~> 4.3"])
-      s.add_development_dependency(%q<guard-rspec>, ["~> 4.6"])
-      s.add_development_dependency(%q<jeweler>, ["~> 2.0"])
-      s.add_development_dependency(%q<rspec>, ["~> 3.0"])
-      s.add_development_dependency(%q<rubocop>, ["~> 0.31"])
-      s.add_development_dependency(%q<simplecov>, ["~> 0.10"])
-    else
-      s.add_dependency(%q<awesome_print>, ["~> 1.6"])
-      s.add_dependency(%q<json>, ["~> 1.8"])
-      s.add_dependency(%q<nokogiri>, ["~> 1.6"])
-      s.add_dependency(%q<coveralls>, ["~> 0.8"])
-      s.add_dependency(%q<flay>, ["~> 2.6"])
-      s.add_dependency(%q<flog>, ["~> 4.3"])
-      s.add_dependency(%q<guard-rspec>, ["~> 4.6"])
-      s.add_dependency(%q<jeweler>, ["~> 2.0"])
-      s.add_dependency(%q<rspec>, ["~> 3.0"])
-      s.add_dependency(%q<rubocop>, ["~> 0.31"])
-      s.add_dependency(%q<simplecov>, ["~> 0.10"])
-    end
-  else
-    s.add_dependency(%q<awesome_print>, ["~> 1.6"])
-    s.add_dependency(%q<json>, ["~> 1.8"])
-    s.add_dependency(%q<nokogiri>, ["~> 1.6"])
-    s.add_dependency(%q<coveralls>, ["~> 0.8"])
-    s.add_dependency(%q<flay>, ["~> 2.6"])
-    s.add_dependency(%q<flog>, ["~> 4.3"])
-    s.add_dependency(%q<guard-rspec>, ["~> 4.6"])
-    s.add_dependency(%q<jeweler>, ["~> 2.0"])
-    s.add_dependency(%q<rspec>, ["~> 3.0"])
-    s.add_dependency(%q<rubocop>, ["~> 0.31"])
-    s.add_dependency(%q<simplecov>, ["~> 0.10"])
-  end
-end
-

data/lib/html-hierarchy-extractor.rb

@@ -1,144 +0,0 @@
-require 'nokogiri'
-require 'digest/md5'
-
-# Extract content from an HTML page in the form of items with associated
-# hierarchy data
-class HTMLHierarchyExtractor
-  def initialize(input, options: {})
-    @dom = Nokogiri::HTML(input)
-    default_options = {
-      css_selector: 'p'
-    }
-    @options = default_options.merge(options)
-  end
-
-  # Returns the outer HTML of a given node
-  #
-  # eg.
-  # <p>foo</p> => <p>foo</p>
-  def extract_html(node)
-    node.to_s.strip
-  end
-
-  # Returns the inner HTML of a given node
-  #
-  # eg.
-  # <p>foo</p> => foo
-  def extract_text(node)
-    node.content
-  end
-
-  # Returns the tag name of a given node
-  #
-  # eg
-  # <p>foo</p> => p
-  def extract_tag_name(node)
-    node.name.downcase
-  end
-
-  # Returns the anchor to the node
-  #
-  # eg.
-  # <h1 name="anchor">Foo</h1> => anchor
-  # <h1 id="anchor">Foo</h1> => anchor
-  # <h1><a name="anchor">Foo</a></h1> => anchor
-  def extract_anchor(node)
-    anchor = node.attr('name') || node.attr('id') || nil
-    return anchor unless anchor.nil?
-
-    # No anchor found directly in the header, search on children
-    subelement = node.css('[name],[id]')
-    return extract_anchor(subelement[0]) unless subelement.empty?
-
-    nil
-  end
-
-  ##
-  # Generate a unique identifier for the item
-  def uuid(item)
-    # We first get all the keys of the object, sorted alphabetically...
-    ordered_keys = item.keys.sort
-
-    # ...then we build a huge array of "key=value" pairs...
-    ordered_array = ordered_keys.map do |key|
-      value = item[key]
-      # We apply the method recursively on other hashes
-      value = uuid(value) if value.is_a?(Hash)
-      "#{key}=#{value}"
-    end
-
-    # ...then we build a unique md5 hash of it
-    Digest::MD5.hexdigest(ordered_array.join(','))
-  end
-
-  ##
-  # Get a relative numeric value of the importance of the heading
-  # 100 for top level, then -10 per heading
-  def heading_weight(heading_level)
-    weight = 100
-    return weight if heading_level.nil?
-    weight - ((heading_level + 1) * 10)
-  end
-
-  def extract
-    heading_selector = 'h1,h2,h3,h4,h5,h6'
-    # We select all nodes that match either the headings or the elements to
-    # extract. This will allow us to loop over it in order it appears in the DOM
-    all_selector = "#{heading_selector},#{@options[:css_selector]}"
-
-    items = []
-    current_hierarchy = {
-      lvl0: nil,
-      lvl1: nil,
-      lvl2: nil,
-      lvl3: nil,
-      lvl4: nil,
-      lvl5: nil
-    }
-    current_position = 0 # Position of the DOM node in the tree
-    current_lvl = nil # Current closest hierarchy level
-    current_anchor = nil # Current closest anchor
-
-    @dom.css(all_selector).each do |node|
-      # If it's a heading, we update our current hierarchy
-      if node.matches?(heading_selector)
-        # Which level heading is it?
-        current_lvl = extract_tag_name(node).gsub(/^h/, '').to_i - 1
-        # Update this level, and set all the following ones to nil
-        current_hierarchy["lvl#{current_lvl}".to_sym] = extract_text(node)
-        (current_lvl + 1..6).each do |lvl|
-          current_hierarchy["lvl#{lvl}".to_sym] = nil
-        end
-        # Update the anchor, if the new heading has one
-        new_anchor = extract_anchor(node)
-        current_anchor = new_anchor if new_anchor
-      end
-
-      # Stop if node is not to be extracted
-      next unless node.matches?(@options[:css_selector])
-
-      # Stop if node is empty
-      text = extract_text(node)
-      next if text.empty?
-
-      item = {
-        html: extract_html(node),
-        text: text,
-        tag_name: extract_tag_name(node),
-        hierarchy: current_hierarchy.clone,
-        anchor: current_anchor,
-        node: node,
-        weight: {
-          position: current_position,
-          heading: heading_weight(current_lvl)
-        }
-      }
-      item[:uuid] = uuid(item)
-      items << item
-
-      current_position += 1
-    end
-
-    items
-  end
-end

data/lib/version.rb

@@ -1,6 +0,0 @@
-# Expose gem version
-class HTMLHierarchyExtractorVersion
-  def self.to_s
-    '1.0.2'
-  end
-end

data/scripts/bump_version

@@ -1,47 +0,0 @@
-#!/usr/bin/env ruby
-require_relative '../lib/version.rb'
-
-# Simple script used to bump the version number
-class BumpVersion
-  def initialize(*args)
-    @type = args[0]
-    unless valid_type?(@type)
-      puts "Invalid bump type: #{@type}"
-      exit 1
-    end
-  end
-
-  def valid_type?(type)
-    %w(major minor patch).include?(type)
-  end
-
-  def bump(current_version, type)
-    major, minor, patch = current_version.split('.').map(&:to_i)
-    if type == 'major'
-      major += 1
-      minor = 0
-      patch = 0
-    end
-    if type == 'minor'
-      minor += 1
-      patch = 0
-    end
-    patch += 1 if type == 'patch'
-    "#{major}.#{minor}.#{patch}"
-  end
-
-  def run
-    old_version = HTMLHierarchyExtractorVersion.to_s
-    new_version = bump(old_version, @type)
-
-    script_dir = File.expand_path(File.dirname(__FILE__))
-    file = File.join(script_dir, '../lib/version.rb')
-    old_content = File.read(file)
-    new_content = old_content.gsub(old_version, new_version)
-    File.write(file, new_content)
-
-    `git add #{file}`
-    `git commit -m "chore(bump): Version bump to #{new_version}"`
-  end
-end
-BumpVersion.new(*ARGV).run

data/scripts/check_flay

@@ -1,30 +0,0 @@
-#!/usr/bin/env ruby
-
-MAX_SCORE = 45
-
-flay_lines = `flay -s ./lib/`.split("\n")
-
-errors = []
-flay_lines.each_with_index do |line, index|
-  # Skip header
-  next if index < 2
-
-  pattern = /^ *(.*): (.*)/
-  matches = line.match(pattern)
-  next if matches.nil?
-  score = matches[1].to_f
-
-  next if score < MAX_SCORE
-  errors << {
-    score: score,
-    file: matches[2]
-  }
-end
-
-exit 0 if errors.size == 0
-
-puts 'Flay test failed:'
-errors.sort_by { |a| a[:score] }.each do |error|
-  puts "#{error[:score]} / #{MAX_SCORE} in #{error[:file]}"
-end
-exit 1

data/scripts/check_flog

@@ -1,31 +0,0 @@
-#!/usr/bin/env ruby
-
-MAX_SCORE = 45
-
-flog_lines = `flog ./lib/`.split("\n")
-
-errors = []
-flog_lines.each_with_index do |line, index|
-  # Skip header
-  next if index < 3
-
-  pattern = /^ *(.*): (.*) (.*):[0-9]*/
-  matches = line.match(pattern)
-  next if matches.nil?
-  score = matches[1].to_f
-
-  next if score < MAX_SCORE
-  errors << {
-    score: score,
-    method: matches[2],
-    file: matches[3]
-  }
-end
-
-exit 0 if errors.size == 0
-
-puts 'Flog test failed:'
-errors.sort_by { |a| a[:score] }.each do |error|
-  puts "#{error[:score]} / #{MAX_SCORE}: #{error[:method]} in #{error[:file]}"
-end
-exit 1

data/scripts/coverage

@@ -1,3 +0,0 @@
-#!/usr/bin/env bash
-
-COVERAGE=1 bundle exec rspec

data/scripts/git_hooks/pre-commit

@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-
-# Succeed fast if we did not change any ruby file
-if ! git status --short | grep -q '\.rb$'; then
-  exit 0
-fi
-
-# Do not commit any focused or excluded tests
-if grep --color -r 'spec' -E -e '^( |\t)*(fit|fdescribe|xit|xdescribe)'; then
-  echo '✘ You have focused and/or skipped tests'
-  exit 1
-fi
-
-# Match style guide
-./scripts/lint || exit 1
-