yawc 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 2d20985fb62b69440f25801ad9cf77bb3962fa2c
-  data.tar.gz: 32b4324643dbb06fc04249f4b50e165956c818c1
+SHA256:
+  metadata.gz: 8ff12033582388f6f58d93aa12e889f667794a4635f97ff8b0283705b5e246dd
+  data.tar.gz: '09514fb3345fe2e9c87b605ab339ee975ac5a00771dfbb7c64c0566d8dd3fcd5'
 SHA512:
-  metadata.gz: f673cc07ef2e057f0a6a66501b4b576795a20395b27ae72e2609f8e53bff3b5f7db0a11f969d49c8379bd81db1910a9fd628c193924573c3dfc0a938938d58fb
-  data.tar.gz: 5b5a37348f02764cc8e64050dde1f4afbb6086136f31de21a35a7037f945994ff680ee6559290cde863896e1f51a3eb2b740f3d03384f61b709fcf31b5b95a9f
+  metadata.gz: ab9ae685cbe33da67ec5cedd02eea7e7b0d1c9dd3e989affe26b02be0dc12a4bcb5cad5eb7aee7f4e7632ed7f521c4d3e98b121733c9300fb48598a5c2cf4b10
+  data.tar.gz: 5032e82ea749765f7529f3797ff5290a6eb9dc098a2c67626e2dd2853eadc24b389875d3342dd244ceba9dbb4010ba6c6359dc945658c4217250e8ecbc00cbf6

data/lib/yawc.rb

@@ -6,15 +6,706 @@
 #              count the number of words in a string and return the 
 #              result as a hash object.
 
-# note: A hashtag is identified as a word e.g. #liveblog
+# note: A hashtag is identified as a valid word e.g. #liveblog
+
+
+# source: https://www.ranks.nl/stopwords
+# see also: https://en.wikipedia.org/wiki/Stop_words
+
+require 'wordsdotdat'
+
+
+STOPWORDS = "
+a
+able
+about
+above
+abst
+accordance
+according
+accordingly
+across
+act
+actually
+added
+adj
+affected
+affecting
+affects
+after
+afterwards
+again
+against
+ah
+all
+almost
+alone
+along
+already
+also
+although
+always
+am
+among
+amongst
+an
+and
+announce
+another
+any
+anybody
+anyhow
+anymore
+anyone
+anything
+anyway
+anyways
+anywhere
+apparently
+approximately
+are
+aren
+arent
+arise
+around
+as
+aside
+ask
+asking
+at
+auth
+available
+away
+awfully
+b
+back
+be
+became
+because
+become
+becomes
+becoming
+been
+before
+beforehand
+begin
+beginning
+beginnings
+begins
+behind
+being
+believe
+below
+beside
+besides
+between
+beyond
+biol
+both
+brief
+briefly
+but
+by
+c
+ca
+came
+can
+cannot
+can't
+cause
+causes
+certain
+certainly
+co
+com
+come
+comes
+contain
+containing
+contains
+could
+couldnt
+d
+date
+did
+didn't
+different
+do
+does
+doesn't
+doing
+done
+don't
+down
+downwards
+due
+during
+e
+each
+ed
+edu
+effect
+eg
+eight
+eighty
+either
+else
+elsewhere
+end
+ending
+enough
+especially
+et
+et-al
+etc
+even
+ever
+every
+everybody
+everyone
+everything
+everywhere
+ex
+except
+f
+far
+few
+ff
+fifth
+first
+five
+fix
+followed
+following
+follows
+for
+former
+formerly
+forth
+found
+four
+from
+further
+furthermore
+g
+gave
+get
+gets
+getting
+give
+given
+gives
+giving
+go
+goes
+gone
+got
+gotten
+h
+had
+happens
+hardly
+has
+hasn't
+have
+haven't
+having
+he
+hed
+hence
+her
+here
+hereafter
+hereby
+herein
+heres
+hereupon
+hers
+herself
+hes
+hi
+hid
+him
+himself
+his
+hither
+home
+how
+howbeit
+however
+hundred
+i
+id
+ie
+if
+i'll
+im
+immediate
+immediately
+importance
+important
+in
+inc
+indeed
+index
+information
+instead
+into
+invention
+inward
+is
+isn't
+it
+itd
+it'll
+its
+itself
+i've
+j
+just
+k
+keep
+keeps
+kept
+kg
+km
+know
+known
+knows
+l
+largely
+last
+lately
+later
+latter
+latterly
+least
+less
+lest
+let
+lets
+like
+liked
+likely
+line
+little
+'ll
+look
+looking
+looks
+ltd
+m
+made
+mainly
+make
+makes
+many
+may
+maybe
+me
+mean
+means
+meantime
+meanwhile
+merely
+mg
+might
+million
+miss
+ml
+more
+moreover
+most
+mostly
+mr
+mrs
+much
+mug
+must
+my
+myself
+n
+na
+name
+namely
+nay
+nd
+near
+nearly
+necessarily
+necessary
+need
+needs
+neither
+never
+nevertheless
+new
+next
+nine
+ninety
+no
+nobody
+non
+none
+nonetheless
+noone
+nor
+normally
+nos
+not
+noted
+nothing
+now
+nowhere
+o
+obtain
+obtained
+obviously
+of
+off
+often
+oh
+ok
+okay
+old
+omitted
+on
+once
+one
+ones
+only
+onto
+or
+ord
+other
+others
+otherwise
+ought
+our
+ours
+ourselves
+out
+outside
+over
+overall
+owing
+own
+p
+page
+pages
+part
+particular
+particularly
+past
+per
+perhaps
+placed
+please
+plus
+poorly
+possible
+possibly
+potentially
+pp
+predominantly
+present
+previously
+primarily
+probably
+promptly
+proud
+provides
+put
+q
+que
+quickly
+quite
+qv
+r
+ran
+rather
+rd
+re
+readily
+really
+recent
+recently
+ref
+refs
+regarding
+regardless
+regards
+related
+relatively
+research
+respectively
+resulted
+resulting
+results
+right
+run
+s
+said
+same
+saw
+say
+saying
+says
+sec
+section
+see
+seeing
+seem
+seemed
+seeming
+seems
+seen
+self
+selves
+sent
+seven
+several
+shall
+she
+shed
+she'll
+shes
+should
+shouldn't
+show
+showed
+shown
+showns
+shows
+significant
+significantly
+similar
+similarly
+since
+six
+slightly
+so
+some
+somebody
+somehow
+someone
+somethan
+something
+sometime
+sometimes
+somewhat
+somewhere
+soon
+sorry
+specifically
+specified
+specify
+specifying
+still
+stop
+strongly
+sub
+substantially
+successfully
+such
+sufficiently
+suggest
+sup
+sure 	t
+take
+taken
+taking
+tell
+tends
+th
+than
+thank
+thanks
+thanx
+that
+that'll
+thats
+that've
+the
+their
+theirs
+them
+themselves
+then
+thence
+there
+thereafter
+thereby
+thered
+therefore
+therein
+there'll
+thereof
+therere
+theres
+thereto
+thereupon
+there've
+these
+they
+theyd
+they'll
+theyre
+they've
+think
+this
+those
+thou
+though
+thoughh
+thousand
+throug
+through
+throughout
+thru
+thus
+til
+tip
+to
+together
+too
+took
+toward
+towards
+tried
+tries
+truly
+try
+trying
+ts
+twice
+two
+u
+un
+under
+unfortunately
+unless
+unlike
+unlikely
+until
+unto
+up
+upon
+ups
+us
+use
+used
+useful
+usefully
+usefulness
+uses
+using
+usually
+v
+value
+various
+'ve
+very
+via
+viz
+vol
+vols
+vs
+w
+want
+wants
+was
+wasnt
+way
+we
+wed
+welcome
+we'll
+went
+were
+werent
+we've
+what
+whatever
+what'll
+whats
+when
+whence
+whenever
+where
+whereafter
+whereas
+whereby
+wherein
+wheres
+whereupon
+wherever
+whether
+which
+while
+whim
+whither
+who
+whod
+whoever
+whole
+who'll
+whom
+whomever
+whos
+whose
+why
+widely
+willing
+wish
+with
+within
+without
+wont
+words
+world
+would
+wouldnt
+www
+x
+y
+yes
+yet
+you
+youd
+you'll
+your
+youre
+yours
+yourself
+yourselves
+you've
+z
+zero
+"
 
 class Yawc
 
   attr_reader :to_h
+  
+  # level:
+  #    2 strips out ignore_words and stop_words
+  #    3 strips out dictionary words
 
-  def initialize(s)
+  def initialize(s, level: 2)
+    
 
-    h = words(s).split.group_by(&:to_s).\
+    @stopwords = STOPWORDS.strip.lines.map {|x| x.chomp}
+    
+    a = case level
+    when 2
+      words(s).split
+    when 3
+      list = words(s).split
+      list - WordsDotDat.words
+    end
+    
+    h = a.group_by(&:to_s).\
         inject({}){|r, x| r.merge(String.new(x[0]).\
                                     force_encoding("utf-8") => x[-1].length)}
     @to_h = h.sort_by(&:last).reverse.to_h
@@ -22,9 +713,16 @@ class Yawc
   end
 
   def words(s)
-
-    ignorewords = ['or', 'the', 'of', 'a', 'if', 'to', 'and', 
-                    'in', 'is', 'are', 'as', 'it', 'at']
+    
+    # words source:  http://norvig.com/mayzner.html
+    
+    ignorewords = ["the", "of", "and", "to", "in", "a", "is", "that", "for",
+                   "it", "as", "was", "with", "be", "by", "on", "not", "he", 
+                   "i", "this", "are", "or","his", "from", "at", "which", 
+                   "but", "have", "an", "had", "they", "you", "were", 
+                   "their", "one", "all", "we", "can", "her", "has", "there",
+                   "been", "if", "more", "when", "will", "would", "who", 
+                   "so", "no"]
 
     s.downcase. 
       gsub(/\w+'\w+/,'').  # remove words containing an apostrophe
@@ -32,7 +730,7 @@ class Yawc
       gsub(/\W[^a-z|#]+(\w+)/,' \1 ').     # remove 
       #             non-alpabetical characters from start or beginning of words
       gsub(/\s.\s/,' ').                              # remove single digits 
-      gsub(/\b(?:#{ignorewords.join('|')})\b/,'').   # ignore common words
+      gsub(/\b(?:#{(ignorewords + @stopwords).join('|')})\b/,'').   # ignore common words
       gsub(/\B[^\w#]\B+/,'')              # remove any other items which are 
       #                                 not words or hashtags
     

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: yawc
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - James Robertson
@@ -10,31 +10,55 @@ bindir: bin
 cert_chain:
 - |
   -----BEGIN CERTIFICATE-----
-  MIIDljCCAn6gAwIBAgIBATANBgkqhkiG9w0BAQUFADBIMRIwEAYDVQQDDAlnZW1t
-  YXN0ZXIxHjAcBgoJkiaJk/IsZAEZFg5qYW1lc3JvYmVydHNvbjESMBAGCgmSJomT
-  8ixkARkWAmV1MB4XDTE2MDUwNDE4MDQ1MVoXDTE3MDUwNDE4MDQ1MVowSDESMBAG
-  A1UEAwwJZ2VtbWFzdGVyMR4wHAYKCZImiZPyLGQBGRYOamFtZXNyb2JlcnRzb24x
-  EjAQBgoJkiaJk/IsZAEZFgJldTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC
-  ggEBALkLIqCSkG0h2NBcBbJa2AbsdLCU/rDiPh7OIDmLXQFAJqpELap47RP4vs6r
-  w4yT0kGZ0ukDp0B1uXlkpp1KCktGqViJYx1Ejt5aHmXEqViV5qgj0vpzNVvmzqp3
-  4z3ELf2yF4WOC2pHWqe5SK2/kYk9m4oSsdmZS/wXce7wPJN0P4IuCO1j8YiTENbP
-  8FwkTTu3zMb6YDpCQ0wgaXqF7kkPHV/n4QSlZfWNVxxPKtgKeNZbTmeWFP//zq/I
-  JBhDoJ0S3U1YikhENNgx72PN1QKuphoi4n6qwUEAQvNuhxY1f3aeM0FqBsEgbUHb
-  r9V+mJYkBcydwgkm7DK/1FEkJBMCAwEAAaOBijCBhzAJBgNVHRMEAjAAMAsGA1Ud
-  DwQEAwIEsDAdBgNVHQ4EFgQUNYthJw8TTY+XImiM2PM6p3gser4wJgYDVR0RBB8w
-  HYEbZ2VtbWFzdGVyQGphbWVzcm9iZXJ0c29uLmV1MCYGA1UdEgQfMB2BG2dlbW1h
-  c3RlckBqYW1lc3JvYmVydHNvbi5ldTANBgkqhkiG9w0BAQUFAAOCAQEAS1GrulAr
-  PxSUKrLSw+46KVljACEU0Y6c8Z1ShN5JyFTc2QzKVZMVnmaUQnp6K4hgyWvezApu
-  9dbY3Vy1dFRDgB4Y5cZGG38idhV3rhSEjDcI8xXWPUKDDplzIUFlN9/OzHhJGvP4
-  4m5mMTOqV83ttfxdv0ywErT8D4cjx6aZiHb3sfDTqICVmDN02Tt35rFEEIWkXlL6
-  6FXlxnu7ySBnvbbYcG74w/kq53Y9HQQKdi7FBDQWMpadRAi8c8u7wGKi5MwsvHjL
-  OlLHmevAiPqMymXu8xRan6xS9e+dvZ+AILldPh7L4F/kSGbrw8vknyGCQWmx+ObA
-  zkV6CoocsL1OSQ==
+  MIIEXjCCAsagAwIBAgIBATANBgkqhkiG9w0BAQsFADAsMSowKAYDVQQDDCFnZW1t
+  YXN0ZXIvREM9amFtZXNyb2JlcnRzb24vREM9ZXUwHhcNMjAwNzI4MDk1ODI0WhcN
+  MjEwNzI4MDk1ODI0WjAsMSowKAYDVQQDDCFnZW1tYXN0ZXIvREM9amFtZXNyb2Jl
+  cnRzb24vREM9ZXUwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQD6fRpj
+  VrIAOsB/fAMOe6/SUUsMcyLbEBUYylOHe+4ixNwiMy9BsTP6xRrKVi987OL1ZBXF
+  oKet/pUaCLva1AA9y69P4qfjW9zqNHzKAnrn/YZTiPKQMJcrkyBCubG+6A6uDUxW
+  vv2s+kS5v+R6gi7d5HQCHnCYKPm3RzTQPMD7cMQ669iXWAHAdQpQLjG3W6wr0xK8
+  zdP+9cEDUZJKaIW338TrF4eWVGW+RC4vaeEeg/KMc9xwiQMZSLBoEF6Oau4hunSc
+  CrgFViTH+/NSJENM1OuTmOmbEws5sVqsaNBLexwqEoeOTp6/waLO4AOCW9bI4nJp
+  BccpOa7Cklf9kT0NNV6HjTCQk64+DY8OFjCY4EuYjfeyjIbv2myzl+noXsSneJ9I
+  s7OgmLOnOAVXlSOfzAQJZQ6+zmFwokSThwOfAWucFuW7kMuiLUcVsA73JyCGrn5U
+  83A0EeeehBwBzVoQAxdqjNQJpTVBRlfY8TLzbQniLLkoBvcRNVBSnGqnMikCAwEA
+  AaOBijCBhzAJBgNVHRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQU53PybDES
+  EYibgRQQ/0BJMOlJw+QwJgYDVR0RBB8wHYEbZ2VtbWFzdGVyQGphbWVzcm9iZXJ0
+  c29uLmV1MCYGA1UdEgQfMB2BG2dlbW1hc3RlckBqYW1lc3JvYmVydHNvbi5ldTAN
+  BgkqhkiG9w0BAQsFAAOCAYEAof+ituc2P/XFNG+mkKZ0/Cdml7/01H/6k/VMXPVG
+  kM2o88i39TP++ISzOOQbUwar72qKyvRAQ4GWNNRCVstilcH8stTSZinOImzLYIxs
+  dz0/WApixbArdFVrJ6dW6KeKT+QRpIZFKhfoJFqx83AL6jrTtiuOaIpRSE1DDnMn
+  /BaP1ZfEXTNeWRsA0wDaWK01hMUFZ2+SFz3A+VZlJkwDCMFF1tuNt7kYVzaje65k
+  OlErp+A72LrWZDcB/MNv1+lJI/Vt3clOJRhPlxPNkQpzlMUV9FSeYe7PGvNhGp3M
+  K615Lde0paZzNCUDPMSIO3817y4tWg6JM4nzBmU00mpKHk4yeGGntzOpnFkClLjn
+  xZuq8tFnf/of3YsnL2FLHia3/j89WTbp92AROOrD01UdY9GrpGz6GYS+895WWg/p
+  Rm36O3mGfms74RMu+aJDblAxhrNNyKrGOdufnYQSQ8lq/Q8hnp98IIat0Dy5vMv6
+  zQC+ExzFRgU8HDVMYKWl/aLh
   -----END CERTIFICATE-----
-date: 2016-05-04 00:00:00.000000000 Z
-dependencies: []
+date: 2020-07-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: wordsdotdat
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.1
 description: 
-email: james@r0bertson.co.uk
+email: james@jamesrobertson.eu
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -59,8 +83,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Yet Another Word Counter (YAWC) makes it convenient to count  the number