bot_twitter_ebooks 3.3.0 → 3.3.1

data/data/nouns.txt

@@ -281,6 +281,7 @@ butcher
 butter
 button
 buzzard
+c-clamp
 cabbage
 cabinet
 cable
@@ -334,7 +335,6 @@ cauliflower
 cause
 caution
 cave
-c-clamp
 cd
 ceiling
 celery
@@ -705,18 +705,18 @@ ethernet
 euphonium
 evening
 event
+ex-husband
+ex-wife
 examination
 example
 exchange
 exclamation
 exhaust
-ex-husband
 existence
 expansion
 experience
 expert
 explanation
-ex-wife
 eye
 eyebrow
 eyelash
@@ -1227,8 +1227,8 @@ millimeter
 millisecond
 mind
 mine
-minibus
 mini-skirt
+minibus
 minister
 mint
 minute
@@ -1852,17 +1852,17 @@ stem
 step
 step-aunt
 step-brother
-stepdaughter
 step-daughter
 step-father
 step-grandfather
 step-grandmother
-stepmother
 step-mother
 step-sister
-stepson
 step-son
 step-uncle
+stepdaughter
+stepmother
+stepson
 steven
 stew
 stick
@@ -1928,6 +1928,7 @@ swordfish
 sycamore
 syrup
 system
+t-shirt
 table
 tablecloth
 tabletop
@@ -1998,8 +1999,8 @@ toast
 toe
 toenail
 toilet
-tomato
 tom-tom
+tomato
 ton
 tongue
 tooth
@@ -2039,7 +2040,6 @@ trowel
 truck
 trumpet
 trunk
-t-shirt
 tsunami
 tub
 tuba

data/lib/bot_twitter_ebooks/model.rb

@@ -154,7 +154,7 @@ module Ebooks
         end
       else
         log "Reading plaintext corpus from #{path} (if this is a json or csv file, please rename the file with an extension and reconsume)"
-        lines = content.split("\n")
+        lines = content.split("\n").reject { |l| l.start_with?('#') } # Remove commented lines
       end
 
       consume_lines(lines)
@@ -163,13 +163,12 @@ module Ebooks
     # Consume a sequence of lines
     # @param lines [Array<String>]
     def consume_lines(lines)
-      log "Removing commented lines and sorting mentions"
+      log "Removing rts and sorting mentions"
 
       statements = []
       mentions = []
       lines.each do |l|
-        next if l.start_with?('#') # Remove commented lines
-        next if l.include?(': "RT @') || l.include?(': "MT @') # Remove soft retweets
+        next if l.start_with?('RT @') || l.start_with?('MT @') # Remove soft retweets
 
         if l.include?('@')
           mentions << NLP.normalize(l)

data/lib/bot_twitter_ebooks/version.rb

@@ -1,3 +1,3 @@
 module Ebooks
-  VERSION = "3.3.0"
+  VERSION = "3.3.1"
 end

data/skeleton/stopwords.txt

@@ -1,3 +1,8 @@
+!
+'ll
+'ve
+.
+?
 a
 able
 about
@@ -8,6 +13,7 @@ according
 accordingly
 across
 act
+actual
 actually
 added
 adj
@@ -18,6 +24,7 @@ after
 afterwards
 again
 against
+ago
 ah
 all
 almost
@@ -47,6 +54,7 @@ apparently
 approximately
 are
 aren
+aren't
 arent
 arise
 around
@@ -54,13 +62,17 @@ as
 aside
 ask
 asking
+assume
 at
 auth
 available
 away
 awfully
+aww
 b
 back
+bad
+basically
 be
 became
 because
@@ -80,35 +92,54 @@ believe
 below
 beside
 besides
+best
+better
 between
 beyond
+big
 biol
+bit
+book
 both
 brief
 briefly
+btw
 but
 by
 c
 ca
+call
+called
 came
 can
-cannot
 can't
+cannot
+case
 cause
 causes
 certain
 certainly
+clearly
 co
 com
 come
 comes
+completely
 contain
 containing
 contains
+context
+cool
 could
 couldnt
+course
+current
 d
+damn
 date
+day
+dear
+definitely
 did
 didn't
 different
@@ -116,8 +147,9 @@ do
 does
 doesn't
 doing
-done
+don
 don't
+done
 down
 downwards
 due
@@ -136,6 +168,8 @@ elsewhere
 end
 ending
 enough
+entire
+entirely
 especially
 et
 et-al
@@ -148,12 +182,18 @@ everyone
 everything
 everywhere
 ex
+exactly
 except
 f
+fact
 far
+feel
 few
 ff
+ffs
 fifth
+finally
+find
 first
 five
 fix
@@ -164,9 +204,13 @@ for
 former
 formerly
 forth
+fortunately
 found
 four
+free
 from
+fuck
+fun
 further
 furthermore
 g
@@ -179,21 +223,38 @@ given
 gives
 giving
 go
+god
 goes
+going
 gone
+gonna
+good
 got
 gotten
+great
+guess
+guy
+guys
 h
 had
+hah
+haha
+hahaha
+half
 happens
+hard
 hardly
 has
 hasn't
+hate
 have
 haven't
 having
 he
+he's
+heard
 hed
+help
 hence
 her
 here
@@ -205,25 +266,42 @@ hereupon
 hers
 herself
 hes
+hey
 hi
 hid
+high
 him
 himself
 his
 hither
+hmm
+hmmm
+holy
 home
+hope
+hopefully
+house
 how
 howbeit
 however
+http
+https
+hullo
 hundred
 i
+i'd
+i'll
+i'm
+i've
 id
+idea
+idk
 ie
 if
-i'll
 im
 immediate
 immediately
+imo
 importance
 important
 in
@@ -232,17 +310,20 @@ indeed
 index
 information
 instead
+interesting
+internet
 into
 invention
 inward
 is
 isn't
 it
-itd
+it'd
 it'll
+it's
+itd
 its
 itself
-i've
 j
 just
 k
@@ -250,6 +331,8 @@ keep
 keeps
 kept
 kg
+kind
+kinda
 km
 know
 known
@@ -265,22 +348,29 @@ least
 less
 lest
 let
+let's
 lets
+life
 like
 liked
 likely
 line
+literally
 little
-'ll
+lol
+long
 look
 looking
 looks
+lot
+lots
 ltd
 m
 made
 mainly
 make
 makes
+man
 many
 may
 maybe
@@ -320,9 +410,12 @@ need
 needs
 neither
 never
+nevermind
 nevertheless
 new
+news
 next
+night
 nine
 ninety
 no
@@ -331,6 +424,7 @@ non
 none
 nonetheless
 noone
+nope
 nor
 normally
 nos
@@ -350,6 +444,7 @@ oh
 ok
 okay
 old
+omg
 omitted
 on
 once
@@ -357,6 +452,10 @@ one
 ones
 only
 onto
+ooh
+oooh
+oops
+open
 or
 ord
 other
@@ -379,11 +478,17 @@ part
 particular
 particularly
 past
+people
 per
 perhaps
+person
+phone
+place
 placed
+play
 please
 plus
+point
 poorly
 possible
 possibly
@@ -391,15 +496,18 @@ potentially
 pp
 predominantly
 present
+pretty
 previously
 primarily
 probably
+problem
 promptly
 proud
 provides
 put
 q
 que
+question
 quickly
 quite
 qv
@@ -408,8 +516,11 @@ ran
 rather
 rd
 re
+read
 readily
+real
 really
+reason
 recent
 recently
 ref
@@ -419,6 +530,7 @@ regardless
 regards
 related
 relatively
+remember
 research
 respectively
 resulted
@@ -427,12 +539,14 @@ results
 right
 run
 s
+sadly
 said
 same
 saw
 say
 saying
 says
+screen
 sec
 section
 see
@@ -445,13 +559,17 @@ seen
 self
 selves
 sent
+seriously
+set
 seven
 several
 shall
 she
-shed
 she'll
+she's
+shed
 shes
+shit
 should
 shouldn't
 show
@@ -479,13 +597,17 @@ somewhat
 somewhere
 soon
 sorry
+sort
+sounds
 specifically
 specified
 specify
 specifying
+start
 still
 stop
 strongly
+stuff
 sub
 substantially
 successfully
@@ -494,11 +616,16 @@ sufficiently
 suggest
 sup
 sure
+surely
+sweet
 t
 take
 taken
 taking
+talk
+talking
 tell
+tend
 tends
 th
 than
@@ -506,9 +633,11 @@ thank
 thanks
 thanx
 that
+that'd
 that'll
-thats
+that's
 that've
+thats
 the
 their
 theirs
@@ -517,55 +646,74 @@ themselves
 then
 thence
 there
+there'll
+there's
+there've
 thereafter
 thereby
 thered
 therefore
 therein
-there'll
 thereof
 therere
 theres
 thereto
 thereupon
-there've
 these
 they
-theyd
 they'll
-theyre
+they're
 they've
+theyd
+theyre
+thing
+things
 think
 this
+tho
 those
 thou
 though
 thoughh
+thought
 thousand
+three
 throug
 through
 throughout
 thru
 thus
 til
+time
+times
 tip
 to
+today
 together
+told
+tomorrow
+tonight
 too
 took
+totally
 toward
 towards
 tried
 tries
+true
 truly
 try
 trying
 ts
+tweet
 twice
+twitter
 two
 u
+ugh
 un
 under
+understand
 unfortunately
 unless
 unlike
@@ -587,30 +735,37 @@ usually
 v
 value
 various
-'ve
 very
 via
+video
 viz
 vol
 vols
 vs
 w
+wait
+wanna
 want
 wants
 was
 wasn't
 way
 we
+we'll
+we're
+we've
+wear
 wed
+weird
 welcome
-we'll
+well
 went
 were
 weren't
-we've
 what
-whatever
 what'll
+what's
+whatever
 whats
 when
 whence
@@ -629,215 +784,57 @@ while
 whim
 whither
 who
+who'll
+whoa
 whod
 whoever
 whole
-who'll
 whom
 whomever
+whoops
 whos
 whose
 why
 widely
+will
 willing
 wish
 with
 within
 without
+woah
 won't
+word
 words
+work
+working
+works
 world
 would
 wouldn't
+wow
+wrong
 www
 x
 y
+y'know
+yeah
+year
+years
+yep
 yes
 yet
 you
-youd
+you'd
 you'll
+you're
+you've
+youd
 your
 youre
 yours
 yourself
 yourselves
-you've
+yup
 z
 zero
-.
-?
-!
-
-http
-don
-people
-well
-will
-https
-time
-good
-thing
-twitter
-pretty
-it's
-i'm
-that's
-you're
-they're
-there's
-things
-yeah
-find
-going
-work
-point
-years
-guess
-bad
-problem
-real
-kind
-day
-better
-lot
-stuff
-i'd
-read
-thought
-idea
-case
-word
-hey
-person
-long
-Dear
-internet
-tweet
-he's
-feel
-wrong
-call
-hard
-phone
-ago
-literally
-remember
-reason
-called
-course
-bit
-question
-high
-today
-told
-man
-actual
-year
-three
-book
-assume
-life
-true
-best
-wow
-video
-times
-works
-fact
-completely
-totally
-imo
-open
-lol
-haha
-cool
-yep
-ooh
-great
-ugh
-tonight
-talk
-sounds
-hahaha
-whoa
-cool
-we're
-guys
-sweet
-fortunately
-hmm
-aren't
-sadly
-talking
-you'd
-place
-yup
-what's
-y'know
-basically
-god
-shit
-holy
-interesting
-news
-guy
-wait
-oooh
-gonna
-current
-let's
-tomorrow
-omg
-hate
-hope
-fuck
-oops
-night
-wear
-wanna
-fun
-finally
-whoops
-nevermind
-definitely
-context
-screen
-free
-exactly
-big
-house
-half
-working
-play
-heard
-hmmm
-damn
-woah
-tho
-set
-idk
-sort
-understand
-kinda
-seriously
-btw
-she's
-hah
-aww
-ffs
-it'd
-that'd
-hopefully
-non
-entirely
-lots
-entire
-tend
-hullo
-clearly
-surely
-weird
-start
-help
-nope