text_rank 1.1.0

data/lib/text_rank/token_filter/stopwords.rb

@@ -0,0 +1,349 @@
+require 'set'
+
+module TextRank
+  module TokenFilter
+    ##
+    # Token filter to remove common stop word tokens
+    #
+    # = Example
+    #
+    #   Stopwords.new.filter!(%w[
+    #     but for what purpose was the earth formed to drive us mad
+    #   ])
+    #   => ["purpose", "earth", "formed", "drive", "mad"]
+    ##
+    class Stopwords
+
+      # Default English stop-word list.
+      STOP_WORDS = Set.new(%w[
+        a
+        about
+        above
+        across
+        after
+        afterwards
+        again
+        against
+        all
+        almost
+        alone
+        along
+        already
+        also
+        although
+        always
+        am
+        among
+        amongst
+        amoungst
+        amount
+        an
+        and
+        another
+        any
+        anyhow
+        anyone
+        anything
+        anyway
+        anywhere
+        are
+        around
+        as
+        at
+        back
+        be
+        became
+        because
+        become
+        becomes
+        becoming
+        been
+        before
+        beforehand
+        behind
+        being
+        below
+        beside
+        besides
+        between
+        beyond
+        bill
+        both
+        bottom
+        but
+        by
+        call
+        can
+        cannot
+        cant
+        co
+        con
+        could
+        couldnt
+        cry
+        de
+        describe
+        detail
+        do
+        done
+        down
+        due
+        during
+        each
+        eg
+        eight
+        either
+        eleven
+        else
+        elsewhere
+        empty
+        enough
+        etc
+        even
+        ever
+        every
+        everyone
+        everything
+        everywhere
+        except
+        few
+        fifteen
+        fify
+        fill
+        find
+        fire
+        first
+        five
+        for
+        former
+        formerly
+        forty
+        found
+        four
+        from
+        front
+        full
+        further
+        get
+        give
+        go
+        had
+        has
+        hasnt
+        have
+        he
+        hence
+        her
+        here
+        hereafter
+        hereby
+        herein
+        hereupon
+        hers
+        herself
+        him
+        himself
+        his
+        how
+        however
+        hundred
+        ie
+        if
+        in
+        inc
+        indeed
+        interest
+        into
+        is
+        it
+        its
+        itself
+        keep
+        last
+        latter
+        latterly
+        least
+        less
+        ltd
+        made
+        many
+        may
+        me
+        meanwhile
+        might
+        mill
+        mine
+        more
+        moreover
+        most
+        mostly
+        move
+        much
+        must
+        my
+        myself
+        name
+        namely
+        neither
+        never
+        nevertheless
+        next
+        nine
+        no
+        nobody
+        none
+        noone
+        nor
+        not
+        nothing
+        now
+        nowhere
+        of
+        off
+        often
+        on
+        once
+        one
+        only
+        onto
+        or
+        other
+        others
+        otherwise
+        our
+        ours
+        ourselves
+        out
+        over
+        own
+        part
+        per
+        perhaps
+        please
+        put
+        rather
+        re
+        same
+        see
+        seem
+        seemed
+        seeming
+        seems
+        serious
+        several
+        she
+        should
+        show
+        side
+        since
+        sincere
+        six
+        sixty
+        so
+        some
+        somehow
+        someone
+        something
+        sometime
+        sometimes
+        somewhere
+        still
+        such
+        system
+        take
+        ten
+        than
+        that
+        the
+        their
+        them
+        themselves
+        then
+        thence
+        there
+        thereafter
+        thereby
+        therefore
+        therein
+        thereupon
+        these
+        they
+        thickv
+        thin
+        third
+        this
+        those
+        though
+        three
+        through
+        throughout
+        thru
+        thus
+        to
+        together
+        too
+        top
+        toward
+        towards
+        twelve
+        twenty
+        two
+        un
+        under
+        until
+        up
+        upon
+        us
+        very
+        via
+        was
+        we
+        well
+        were
+        what
+        whatever
+        when
+        whence
+        whenever
+        where
+        whereafter
+        whereas
+        whereby
+        wherein
+        whereupon
+        wherever
+        whether
+        which
+        while
+        whither
+        who
+        whoever
+        whole
+        whom
+        whose
+        why
+        will
+        with
+        within
+        without
+        would
+        yet
+        you
+        your
+        yours
+        yourself
+        yourselves
+      ])
+
+      # Perform the filter
+      # @param tokens [Array<String>]
+      # @return [Array<String>]
+      def filter!(tokens)
+        tokens.delete_if do |token|
+          STOP_WORDS.include?(token.downcase)
+        end
+      end
+
+    end
+  end
+end

data/lib/text_rank/token_filter.rb

@@ -0,0 +1,18 @@
+module TextRank
+  ##
+  # Token filters can be used to pre-process potential tokens prior to creating
+  # a graph or executing PageRank.  Filters are typically used to throw out tokens
+  # which are not good candidates for keywords.  However, it is possible for a
+  # filter to add new tokens or to modify existing ones.
+  #
+  # Token filters are applied as a chain, so care should be taken to use them
+  # in the desired order.
+  ##
+  module TokenFilter
+
+    autoload :MinLength,    'text_rank/token_filter/min_length'
+    autoload :PartOfSpeech, 'text_rank/token_filter/part_of_speech'
+    autoload :Stopwords,    'text_rank/token_filter/stopwords'
+
+  end
+end

data/lib/text_rank/tokenizer/regex.rb

@@ -0,0 +1,26 @@
+module TextRank
+  module Tokenizer
+    ##
+    # Base tokenizer that tokenizes on any regular expression
+    #
+    # = Example
+    #
+    #  Regex.new(/:/).tokenize("i should:like to know:which is worse.")
+    #  => ["i should", "like to know", "which is worse"]
+    ##
+    class Regex
+
+      # @param regex [Regexp] to use for string splitting
+      def initialize(regex)
+        @regex = regex
+      end
+
+      # @param text [String] string to tokenize
+      # return [Array<String>] non-empty tokens
+      def tokenize(text)
+        text.split(@regex) - ['']
+      end
+
+    end
+  end
+end

data/lib/text_rank/tokenizer/whitespace.rb

@@ -0,0 +1,19 @@
+module TextRank
+  module Tokenizer
+    ##
+    # Tokenizer to split on any whitespace
+    #
+    # = Example
+    #
+    #  Whitespace.new.tokenize("i should:like to know:which is worse.")
+    #  => ["i", "should:like", "to", "know:which", "is", "worse."]
+    ##
+    class Whitespace < Regex
+
+      def initialize
+        super(/\s+/)
+      end
+
+    end
+  end
+end

data/lib/text_rank/tokenizer/words_and_punctuation.rb

@@ -0,0 +1,26 @@
+module TextRank
+  module Tokenizer
+    ##
+    # A tokenizer that preserves punctuation as their own tokens (which can be
+    # used, for example, by the [TokenFilter::PartOfSpeechBase] filter).
+    #
+    # = Example
+    #
+    #  WordsAndPunctuation.new.tokenize("i should:like to know:which is worse.")
+    #  => ["i", "should", ":", "like", "to", "know", ":", "which", "is", "worse", "."]
+    ##
+    class WordsAndPunctuation < Regex
+
+      def initialize
+        super(/
+          ([a-z][a-z0-9-]+)
+          |
+          ([\p{Punct}])
+          |
+          \s+
+        /xi)
+      end
+
+    end
+  end
+end

data/lib/text_rank/tokenizer.rb

@@ -0,0 +1,19 @@
+module TextRank
+  ##
+  # Tokenizers are responsible for transforming a single String of text into an
+  # array of potential keywords ("tokens").  There are no requirements of tokens
+  # other than to be non-empty.  When used in combination with token filters, it
+  # may make sense for a tokenizer to temporarily create tokens which might seem
+  # like ill-suited keywords.  The token filter may use these "bad" keywords to
+  # help inform its decision on which tokens to keep and which to drop.  An example
+  # of this is the part of speech token filter which uses punctuation tokens to
+  # help guess the part of speech of each non-punctuation token.
+  ##
+  module Tokenizer
+
+    autoload :Regex,                'text_rank/tokenizer/regex'
+    autoload :Whitespace,           'text_rank/tokenizer/whitespace'
+    autoload :WordsAndPunctuation,  'text_rank/tokenizer/words_and_punctuation'
+
+  end
+end

data/lib/text_rank/version.rb

@@ -0,0 +1,3 @@
+module TextRank
+  VERSION = '1.1.0'
+end

data/lib/text_rank.rb

@@ -0,0 +1,34 @@
+require 'page_rank'
+
+##
+# Provides convenience methods for quickly extracting keywords.
+#
+# @see README
+##
+module TextRank
+
+  autoload :CharFilter,       'text_rank/char_filter'
+  autoload :GraphStrategy,    'text_rank/graph_strategy'
+  autoload :KeywordExtractor, 'text_rank/keyword_extractor'
+  autoload :RankFilter,       'text_rank/rank_filter'
+  autoload :TokenFilter,      'text_rank/token_filter'
+  autoload :Tokenizer,        'text_rank/tokenizer'
+  autoload :VERSION,          'text_rank/version'
+
+  # A convenience method for quickly extracting keywords from text with default options
+  # @param text [String] text from which to extract keywords
+  # @option (see KeywordExtractor.basic)
+  # @return [Hash<String, Float>] of tokens and text rank (in descending order)
+  def self.extract_keywords(text, **options)
+    TextRank::KeywordExtractor.basic(**options).extract(text, **options)
+  end
+
+  # A convenience method for quickly extracting keywords from text with default advanced options
+  # @param (see extract_keywords)
+  # @option (see KeywordExtractor.advanced)
+  # @return (see extract_keywords)
+  def self.extract_keywords_advanced(text, **options)
+    TextRank::KeywordExtractor.advanced(**options).extract(text, **options)
+  end
+
+end

data/text_rank.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'text_rank/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'text_rank'
+  spec.version       = TextRank::VERSION
+  spec.authors       = ['David McCullars']
+  spec.email         = ['david.mccullars@gmail.com']
+
+  spec.summary       = %q{Implementation of TextRank solution to ranked keyword extraction}
+  spec.description   = %q{See https://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&ved=0ahUKEwjK9tfHxcvMAhVOzGMKHdaQBeEQFggdMAA&url=https%3A%2F%2Fweb.eecs.umich.edu%2F~mihalcea%2Fpapers%2Fmihalcea.emnlp04.pdf&usg=AFQjCNHL5SGlxLy4qmEg1yexaKGZK_Q7IA}
+  spec.homepage      = 'https://github.com/david-mccullars/text_rank'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler',    '~> 1.11'
+  spec.add_development_dependency 'rake',       '~> 10.0'
+  spec.add_development_dependency 'rspec',      '~> 3.0'
+  spec.add_development_dependency 'simplecov',  '~> 0.11'
+  spec.add_development_dependency 'codeclimate-test-reporter'
+
+  spec.add_development_dependency 'engtagger',  '~> 0.2.0' # Optional runtime dependency but needed for specs
+  spec.add_development_dependency 'nokogiri',   '~> 1.0'   # Optional runtime dependency but needed for specs
+end

metadata

@@ -0,0 +1,183 @@
+--- !ruby/object:Gem::Specification
+name: text_rank
+version: !ruby/object:Gem::Version
+  version: 1.1.0
+platform: ruby
+authors:
+- David McCullars
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-05-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.11'
+- !ruby/object:Gem::Dependency
+  name: codeclimate-test-reporter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: engtagger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+description: See https://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&ved=0ahUKEwjK9tfHxcvMAhVOzGMKHdaQBeEQFggdMAA&url=https%3A%2F%2Fweb.eecs.umich.edu%2F~mihalcea%2Fpapers%2Fmihalcea.emnlp04.pdf&usg=AFQjCNHL5SGlxLy4qmEg1yexaKGZK_Q7IA
+email:
+- david.mccullars@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".codeclimate.yml"
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".ruby-version"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/page_rank.rb
+- lib/page_rank/base.rb
+- lib/page_rank/dense.rb
+- lib/page_rank/sparse.rb
+- lib/text_rank.rb
+- lib/text_rank/char_filter.rb
+- lib/text_rank/char_filter/ascii_folding.rb
+- lib/text_rank/char_filter/lowercase.rb
+- lib/text_rank/char_filter/strip_email.rb
+- lib/text_rank/char_filter/strip_html.rb
+- lib/text_rank/char_filter/strip_possessive.rb
+- lib/text_rank/char_filter/undo_contractions.rb
+- lib/text_rank/graph_strategy.rb
+- lib/text_rank/graph_strategy/coocurrence.rb
+- lib/text_rank/keyword_extractor.rb
+- lib/text_rank/rank_filter.rb
+- lib/text_rank/rank_filter/collapse_adjacent.rb
+- lib/text_rank/token_filter.rb
+- lib/text_rank/token_filter/min_length.rb
+- lib/text_rank/token_filter/part_of_speech.rb
+- lib/text_rank/token_filter/stopwords.rb
+- lib/text_rank/tokenizer.rb
+- lib/text_rank/tokenizer/regex.rb
+- lib/text_rank/tokenizer/whitespace.rb
+- lib/text_rank/tokenizer/words_and_punctuation.rb
+- lib/text_rank/version.rb
+- text_rank.gemspec
+homepage: https://github.com/david-mccullars/text_rank
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Implementation of TextRank solution to ranked keyword extraction
+test_files: []
+has_rdoc: