RubyGems - text_rank - Versions diffs - 1.1.7 → 1.2.5 - Mend

text_rank 1.1.7 → 1.2.5

Files changed (38) hide show

checksums.yaml +5 -5
data/.codeclimate.yml +1 -6
data/.rubocop.yml +60 -1075
data/.ruby-version +1 -1
data/.travis.yml +13 -5
data/{LICENSE.txt → LICENSE} +0 -0
data/README.md +2 -1
data/bin/console +3 -3
data/lib/page_rank.rb +2 -0
data/lib/page_rank/base.rb +9 -8
data/lib/page_rank/dense.rb +2 -1
data/lib/page_rank/sparse.rb +6 -7
data/lib/text_rank.rb +12 -9
data/lib/text_rank/char_filter.rb +1 -1
data/lib/text_rank/char_filter/ascii_folding.rb +5 -1
data/lib/text_rank/char_filter/strip_possessive.rb +2 -2
data/lib/text_rank/char_filter/undo_contractions.rb +1 -137
data/lib/text_rank/char_filter/undo_contractions.yml +135 -0
data/lib/text_rank/fingerprint.rb +20 -28
data/lib/text_rank/fingerprint_overlap.rb +55 -0
data/lib/text_rank/graph_strategy/coocurrence.rb +15 -6
data/lib/text_rank/keyword_extractor.rb +32 -25
data/lib/text_rank/rank_filter/collapse_adjacent.rb +53 -25
data/lib/text_rank/rank_filter/normalize_probability.rb +2 -1
data/lib/text_rank/rank_filter/normalize_unit_vector.rb +2 -1
data/lib/text_rank/token_filter/part_of_speech.rb +0 -1
data/lib/text_rank/token_filter/stopwords.rb +1 -321
data/lib/text_rank/token_filter/stopwords.yml +317 -0
data/lib/text_rank/tokenizer.rb +1 -1
data/lib/text_rank/tokenizer/money.rb +11 -6
data/lib/text_rank/tokenizer/number.rb +4 -3
data/lib/text_rank/tokenizer/punctuation.rb +4 -1
data/lib/text_rank/tokenizer/url.rb +3 -0
data/lib/text_rank/tokenizer/whitespace.rb +4 -1
data/lib/text_rank/tokenizer/word.rb +5 -2
data/lib/text_rank/version.rb +3 -1
data/text_rank.gemspec +10 -10
metadata +48 -32

data/lib/text_rank/token_filter/stopwords.rb CHANGED

@@ -1,5 +1,3 @@
-require 'set'
 module TextRank
   module TokenFilter
     ##
@@ -15,325 +13,7 @@ module TextRank
     class Stopwords
       # Default English stop-word list.
-      STOP_WORDS = Set.new(%w[
-        a
-        about
-        above
-        across
-        after
-        afterwards
-        again
-        against
-        all
-        almost
-        alone
-        along
-        already
-        also
-        although
-        always
-        am
-        among
-        amongst
-        amoungst
-        amount
-        an
-        and
-        another
-        any
-        anyhow
-        anyone
-        anything
-        anyway
-        anywhere
-        are
-        around
-        as
-        at
-        back
-        be
-        became
-        because
-        become
-        becomes
-        becoming
-        been
-        before
-        beforehand
-        behind
-        being
-        below
-        beside
-        besides
-        between
-        beyond
-        bill
-        both
-        bottom
-        but
-        by
-        call
-        can
-        cannot
-        cant
-        co
-        con
-        could
-        couldnt
-        cry
-        de
-        describe
-        detail
-        do
-        done
-        down
-        due
-        during
-        each
-        eg
-        eight
-        either
-        eleven
-        else
-        elsewhere
-        empty
-        enough
-        etc
-        even
-        ever
-        every
-        everyone
-        everything
-        everywhere
-        except
-        few
-        fifteen
-        fify
-        fill
-        find
-        fire
-        first
-        five
-        for
-        former
-        formerly
-        forty
-        found
-        four
-        from
-        front
-        full
-        further
-        get
-        give
-        go
-        had
-        has
-        hasnt
-        have
-        he
-        hence
-        her
-        here
-        hereafter
-        hereby
-        herein
-        hereupon
-        hers
-        herself
-        him
-        himself
-        his
-        how
-        however
-        hundred
-        ie
-        if
-        in
-        inc
-        indeed
-        interest
-        into
-        is
-        it
-        its
-        itself
-        keep
-        last
-        latter
-        latterly
-        least
-        less
-        ltd
-        made
-        many
-        may
-        me
-        meanwhile
-        might
-        mill
-        mine
-        more
-        moreover
-        most
-        mostly
-        move
-        much
-        must
-        my
-        myself
-        name
-        namely
-        neither
-        never
-        nevertheless
-        next
-        nine
-        no
-        nobody
-        none
-        noone
-        nor
-        not
-        nothing
-        now
-        nowhere
-        of
-        off
-        often
-        on
-        once
-        one
-        only
-        onto
-        or
-        other
-        others
-        otherwise
-        our
-        ours
-        ourselves
-        out
-        over
-        own
-        part
-        per
-        perhaps
-        please
-        put
-        rather
-        re
-        same
-        see
-        seem
-        seemed
-        seeming
-        seems
-        serious
-        several
-        she
-        should
-        show
-        side
-        since
-        sincere
-        six
-        sixty
-        so
-        some
-        somehow
-        someone
-        something
-        sometime
-        sometimes
-        somewhere
-        still
-        such
-        system
-        take
-        ten
-        than
-        that
-        the
-        their
-        them
-        themselves
-        then
-        thence
-        there
-        thereafter
-        thereby
-        therefore
-        therein
-        thereupon
-        these
-        they
-        thickv
-        thin
-        third
-        this
-        those
-        though
-        three
-        through
-        throughout
-        thru
-        thus
-        to
-        together
-        too
-        top
-        toward
-        towards
-        twelve
-        twenty
-        two
-        un
-        under
-        until
-        up
-        upon
-        us
-        very
-        via
-        was
-        we
-        well
-        were
-        what
-        whatever
-        when
-        whence
-        whenever
-        where
-        whereafter
-        whereas
-        whereby
-        wherein
-        whereupon
-        wherever
-        whether
-        which
-        while
-        whither
-        who
-        whoever
-        whole
-        whom
-        whose
-        why
-        will
-        with
-        within
-        without
-        would
-        yet
-        you
-        your
-        yours
-        yourself
-        yourselves
-      ])
+      STOP_WORDS = Set.new(YAML.load_file(File.expand_path('stopwords.yml', __dir__)))
       # Perform the filter
       # @param tokens [Array<String>]

data/lib/text_rank/token_filter/stopwords.yml ADDED

@@ -0,0 +1,317 @@
+- a
+- about
+- above
+- across
+- after
+- afterwards
+- again
+- against
+- all
+- almost
+- alone
+- along
+- already
+- also
+- although
+- always
+- am
+- among
+- amongst
+- amoungst
+- amount
+- an
+- and
+- another
+- any
+- anyhow
+- anyone
+- anything
+- anyway
+- anywhere
+- are
+- around
+- as
+- at
+- back
+- be
+- became
+- because
+- become
+- becomes
+- becoming
+- been
+- before
+- beforehand
+- behind
+- being
+- below
+- beside
+- besides
+- between
+- beyond
+- bill
+- both
+- bottom
+- but
+- by
+- call
+- can
+- cannot
+- cant
+- co
+- con
+- could
+- couldnt
+- cry
+- de
+- describe
+- detail
+- do
+- done
+- down
+- due
+- during
+- each
+- eg
+- eight
+- either
+- eleven
+- else
+- elsewhere
+- empty
+- enough
+- etc
+- even
+- ever
+- every
+- everyone
+- everything
+- everywhere
+- except
+- few
+- fifteen
+- fify
+- fill
+- find
+- fire
+- first
+- five
+- for
+- former
+- formerly
+- forty
+- found
+- four
+- from
+- front
+- full
+- further
+- get
+- give
+- go
+- had
+- has
+- hasnt
+- have
+- he
+- hence
+- her
+- here
+- hereafter
+- hereby
+- herein
+- hereupon
+- hers
+- herself
+- him
+- himself
+- his
+- how
+- however
+- hundred
+- ie
+- if
+- in
+- inc
+- indeed
+- interest
+- into
+- is
+- it
+- its
+- itself
+- keep
+- last
+- latter
+- latterly
+- least
+- less
+- ltd
+- made
+- many
+- may
+- me
+- meanwhile
+- might
+- mill
+- mine
+- more
+- moreover
+- most
+- mostly
+- move
+- much
+- must
+- my
+- myself
+- name
+- namely
+- neither
+- never
+- nevertheless
+- next
+- nine
+- no
+- nobody
+- none
+- noone
+- nor
+- not
+- nothing
+- now
+- nowhere
+- of
+- off
+- often
+- on
+- once
+- one
+- only
+- onto
+- or
+- other
+- others
+- otherwise
+- our
+- ours
+- ourselves
+- out
+- over
+- own
+- part
+- per
+- perhaps
+- please
+- put
+- rather
+- re
+- same
+- see
+- seem
+- seemed
+- seeming
+- seems
+- serious
+- several
+- she
+- should
+- show
+- side
+- since
+- sincere
+- six
+- sixty
+- so
+- some
+- somehow
+- someone
+- something
+- sometime
+- sometimes
+- somewhere
+- still
+- such
+- system
+- take
+- ten
+- than
+- that
+- the
+- their
+- them
+- themselves
+- then
+- thence
+- there
+- thereafter
+- thereby
+- therefore
+- therein
+- thereupon
+- these
+- they
+- thickv
+- thin
+- third
+- this
+- those
+- though
+- three
+- through
+- throughout
+- thru
+- thus
+- to
+- together
+- too
+- top
+- toward
+- towards
+- twelve
+- twenty
+- two
+- un
+- under
+- until
+- up
+- upon
+- us
+- very
+- via
+- was
+- we
+- well
+- were
+- what
+- whatever
+- when
+- whence
+- whenever
+- where
+- whereafter
+- whereas
+- whereby
+- wherein
+- whereupon
+- wherever
+- whether
+- which
+- while
+- whither
+- who
+- whoever
+- whole
+- whom
+- whose
+- why
+- will
+- with
+- within
+- without
+- would
+- yet
+- you
+- your
+- yours
+- yourself
+- yourselves