wcc-text-analysis 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 53076bbd0f7310bd02cab7548cd71edb8417ad7bd320c4828c0f80dc72cdc0ad
+  data.tar.gz: e161b846a746e14d0912898d8028f5c954f2be6eeda844a16e202744962bfeb1
+SHA512:
+  metadata.gz: 1f07d19805a3723020d8d978c53c5c921958122d76f51a09e39b7501dca28df2239b49eacef6f2be154eacffd9bcc588ffdd7fd2c72dbd16b328e1305f556b4f
+  data.tar.gz: f685000411857bcb532f3e0bf46acfa1201ceb4bc53e470d3e011554489e676437e3048dcf798245f943bf471ec3d23b9dd057de544ba1728232194f3a5be24d

data/.gitignore

@@ -0,0 +1 @@
+Gemfile.lock

data/.rubocop.yml

@@ -0,0 +1,15 @@
+inherit_from: ../.rubocop.yml
+
+Metrics/BlockLength:
+  Exclude:
+    - 'spec/**/*'
+
+Style/ClassAndModuleChildren:
+  Enabled: false
+
+Style/MultilineBlockChain:
+  Exclude:
+    - 'spec/**/*'
+
+Rails/TimeZone:
+  Enabled: false

data/Gemfile

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in wcc-text-analysis.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,32 @@
+directories %w[lib spec]
+
+group :red_green_refactor, halt_on_fail: true do
+  guard :rspec, cmd: 'bundle exec rspec --order rand', all_on_start: false do
+    require 'guard/rspec/dsl'
+    dsl = Guard::RSpec::Dsl.new(self)
+
+    # RSpec files
+    rspec = dsl.rspec
+    watch(rspec.spec_helper) { rspec.spec_dir }
+    watch(rspec.spec_support) { rspec.spec_dir }
+    watch(rspec.spec_files)
+
+    # Ruby files
+    ruby = dsl.ruby
+    dsl.watch_spec_files_for(ruby.lib_files)
+  end
+
+  guard :rubocop, cli: ['--display-cop-names'] do
+    watch(%r{.+\.rb$})
+    watch(%r{(?:.+/)?\.rubocop(?:_todo)?\.yml$}) { |m| File.dirname(m[0]) }
+  end
+end
+
+group :autofix do
+  guard :rubocop, all_on_start: false, cli: ['--auto-correct', '--display-cop-names'] do
+    watch(%r{.+\.rb$})
+    watch(%r{(?:.+/)?\.rubocop(?:_todo)?\.yml$}) { |m| File.dirname(m[0]) }
+  end
+end
+
+scope group: :red_green_refactor

data/Rakefile

@@ -0,0 +1,3 @@
+require "bundler/gem_helper"
+
+Bundler::GemHelper.install_tasks

data/db/stop_words.txt

@@ -0,0 +1,721 @@
+a
+able
+about
+above
+according
+accordingly
+across
+actually
+after
+afterwards
+again
+against
+ain't
+aint
+ain´t
+ain’t
+all
+allow
+allows
+almost
+alone
+along
+already
+also
+although
+always
+am
+among
+amongst
+an
+and
+another
+any
+anybody
+anyhow
+anyone
+anything
+anyway
+anyways
+anywhere
+apart
+appear
+appreciate
+appropriate
+are
+aren't
+arent
+aren´t
+aren’t
+around
+as
+aside
+ask
+asking
+associated
+at
+available
+away
+awfully
+b
+be
+became
+because
+become
+becomes
+becoming
+been
+before
+beforehand
+behind
+being
+believe
+below
+beside
+besides
+best
+better
+between
+beyond
+both
+brief
+but
+by
+c
+came
+can
+can't
+cannot
+cant
+can´t
+can’t
+cause
+causes
+certain
+certainly
+changes
+clearly
+co
+com
+come
+comes
+concerning
+consequently
+consider
+considering
+contain
+containing
+contains
+corresponding
+could
+couldn't
+couldnt
+couldn´t
+couldn’t
+course
+currently
+d
+definitely
+described
+despite
+did
+didn't
+didnt
+didn´t
+didn’t
+different
+do
+does
+doesn't 
+doesnt 
+doesn´t 
+doesn’t 
+doing
+don't
+done
+dont
+don´t
+don’t
+down
+downwards
+during
+e
+each
+edu
+eg
+eight
+either
+else
+elsewhere
+enough
+entirely
+especially
+et
+etc
+even
+ever
+every
+everybody
+everyone
+everything
+everywhere
+ex
+exactly
+example
+except
+f
+far
+few
+fifth
+first
+five
+followed
+following
+follows
+for
+former
+formerly
+forth
+four
+from
+further
+furthermore
+g
+get
+gets
+getting
+given
+gives
+go
+goes
+going
+gone
+got
+gotten
+greetings
+h
+had
+hadn't
+hadnt
+hadn´t
+hadn’t
+happens
+hardly
+has
+hasn't
+hasnt
+hasn´t
+hasn’t
+have
+haven't
+havent
+haven´t
+haven’t
+having
+he
+he'd
+he'll
+he's
+hed
+hell
+hello
+help
+hence
+her
+here
+here's
+hereafter
+hereby
+herein
+heres
+hereupon
+here´s
+here’s
+hers
+herself
+hes
+he´d
+he´ll
+he´s
+he’d
+he’ll
+he’s
+hi
+him
+himself
+his
+hither
+hopefully
+how
+how's
+howbeit
+however
+hows
+how´s
+how’s
+i
+i'd
+i'll
+i'm
+i've
+id
+ie
+if
+ignored
+ill
+im
+immediate
+in
+inasmuch
+inc
+indeed
+indicate
+indicated
+indicates
+inner
+insofar
+instead
+into
+inward
+is
+isn't
+isnt
+isn´t
+isn’t
+it
+it's
+its
+itself
+it´s
+it’s
+ive
+i´d
+i´ll
+i´m
+i´ve
+i’d
+i’ll
+i’m
+i’ve
+j
+just
+k
+keep
+keeps
+kept
+know
+known
+knows
+l
+last
+lately
+later
+latter
+latterly
+least
+less
+lest
+let
+let's
+lets
+let´s
+let’s
+like
+liked
+likely
+little
+look
+looking
+looks
+ltd
+m
+mainly
+many
+may
+maybe
+me
+mean
+meanwhile
+merely
+might
+more
+moreover
+most
+mostly
+much
+must
+mustn't
+mustnt
+mustn´t
+mustn’t
+my
+myself
+n
+name
+namely
+nd
+near
+nearly
+necessary
+need
+needs
+neither
+never
+nevertheless
+new
+next
+nine
+no
+nobody
+non
+none
+noone
+nor
+normally
+not
+nothing
+novel
+now
+nowhere
+o
+obviously
+of
+off
+often
+oh
+ok
+okay
+old
+on
+once
+one
+ones
+only
+onto
+or
+other
+others
+otherwise
+ought
+our
+ours
+ourselves
+out
+outside
+over
+overall
+own
+p
+particular
+particularly
+per
+perhaps
+placed
+please
+plus
+possible
+presumably
+probably
+provides
+q
+que
+quite
+qv
+r
+rather
+rd
+re
+really
+reasonably
+regarding
+regardless
+regards
+relatively
+respectively
+right
+s
+said
+same
+saw
+say
+saying
+says
+second
+secondly
+see
+seeing
+seem
+seemed
+seeming
+seems
+seen
+self
+selves
+sensible
+sent
+serious
+seriously
+seven
+several
+shall
+shan't
+shant
+shan´t
+shan’t
+she
+she'd
+she'll
+she's
+shed
+shell
+shes
+she´d
+she´ll
+she´s
+she’d
+she’ll
+she’s
+should
+shouldn't
+shouldnt
+shouldn´t
+shouldn’t
+since
+six
+so
+some
+somebody
+somehow
+someone
+something
+sometime
+sometimes
+somewhat
+somewhere
+soon
+sorry
+specified
+specify
+specifying
+still
+sub
+such
+sup
+sure
+t
+take
+taken
+tell
+tends
+th
+than
+thank
+thanks
+thanx
+that
+that's
+thats
+that´s
+that’s
+the
+their
+theirs
+them
+themselves
+then
+thence
+there
+there's
+thereafter
+thereby
+therefore
+therein
+theres
+thereupon
+there´s
+there’s
+these
+they
+they'd
+they'll
+they're
+they've
+theyd
+theyll
+theyre
+theyve
+they´d
+they´ll
+they´re
+they´ve
+they’d
+they’ll
+they’re
+they’ve
+think
+third
+this
+thorough
+thoroughly
+those
+though
+three
+through
+throughout
+thru
+thus
+to
+together
+too
+took
+toward
+towards
+tried
+tries
+truly
+try
+trying
+twice
+two
+u
+un
+under
+unfortunately
+unless
+unlikely
+until
+unto
+up
+upon
+us
+use
+used
+useful
+uses
+using
+usually
+uucp
+v
+value
+various
+very
+via
+viz
+vs
+w
+want
+wants
+was
+wasn't
+wasnt
+wasn´t
+wasn’t
+way
+we
+we'd
+we'll
+we're
+we've
+wed
+welcome
+well
+went
+were
+weren't
+werent
+weren´t
+weren’t
+weve
+we´d
+we´ll
+we´re
+we´ve
+we’d
+we’ll
+we’re
+we’ve
+what
+what's
+whatever
+whats
+what´s
+what’s
+when
+when's
+whence
+whenever
+whens
+when´s
+when’s
+where
+where's
+whereafter
+whereas
+whereby
+wherein
+wheres
+whereupon
+wherever
+where´s
+where’s
+whether
+which
+while
+whither
+who
+who's
+whoever
+whole
+whom
+whos
+whose
+who´s
+who’s
+why
+why's
+whys
+why´s
+why’s
+will
+willing
+wish
+with
+within
+without
+won't
+wonder
+wont
+won´t
+won’t
+would
+wouldn't
+wouldnt
+wouldn´t
+wouldn’t
+x
+y
+yes
+yet
+you
+you'd
+you'll
+you're
+you've
+youd
+youll
+your
+youre
+yours
+yourself
+yourselves
+youve
+you´d
+you´ll
+you´re
+you´ve
+you’d
+you’ll
+you’re
+you’ve
+z
+zero

data/db/transcript_exclusions.txt

@@ -0,0 +1 @@
+thing

data/lib/wcc/text_analysis.rb

@@ -0,0 +1,71 @@
+require 'phrasie'
+require 'redcarpet'
+require 'redcarpet/render_strip'
+
+module WCC
+  class TextAnalysis
+    TEXT_BLACKLIST = /[…]/
+
+    attr_reader :normalized, :stripped
+
+    def self.extract_terms(file)
+      File.read("db/#{file}.txt").split("\n")
+    end
+
+    def initialize(string, stop_words: STOPWORDS)
+      @original = string
+      @stop_words = stop_words
+    end
+
+    def terms_by_frequency(phrase_length: 1, min_occurrences: 3)
+      @normalized ||= normalize(@original)
+      terms = ::Phrasie::Extractor.new
+        .phrases(@normalized, occur: min_occurrences)
+        .select { |t| t.last == phrase_length }
+        .map(&:first)
+      remove_ignored_tokens terms
+    end
+
+    def evaluate_length
+      @normalized ||= normalize(@original).downcase
+      @stripped ||= remove_ignored_tokens(tokenize(@normalized))
+
+      <<-OUTPUT.strip_heredoc
+        Original
+        \tLength: #{@original.length}
+        \tWordcount: #{tokenize(@original).length}
+
+        Normalized (removed markdown chars & whitespace)
+        \tLength: #{@normalized.length}
+        \tWordcount: #{tokenize(@normalized).length}
+
+        Processed (removed above & stopwords)
+        \tLength: #{@stripped.join(' ').length}
+        \tWordcount: #{@stripped.length}
+        \t   Unique: #{@stripped.uniq.length}
+        \t   Unique Length: #{@stripped.uniq.join(' ').length}
+      OUTPUT
+    end
+
+    private
+
+    def normalize(string)
+      ::Redcarpet::Markdown.new(::Redcarpet::Render::StripDown)
+        .render(CGI.unescapeHTML(string))
+        .gsub(TEXT_BLACKLIST, ' ')
+        .gsub(/\s+/, ' ')
+    end
+
+    def tokenize(string)
+      string.split(/\s+/)
+    end
+
+    def remove_ignored_tokens(string)
+      string - (@stop_words + EXCLUSIONS)
+    end
+
+    # Stopwords from http://www.ranks.nl/stopwords
+    STOPWORDS  = extract_terms("stop_words").freeze
+    EXCLUSIONS = extract_terms("transcript_exclusions").freeze
+  end
+end

data/lib/wcc/text_analysis/version.rb

@@ -0,0 +1,5 @@
+module WCC
+  class TextAnalysis
+    VERSION = '0.0.1'.freeze
+  end
+end

data/wcc-text-analysis.gemspec

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'wcc/text_analysis/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'wcc-text-analysis'
+  spec.version       = WCC::TextAnalysis::VERSION
+  spec.authors       = ['Watermark Dev']
+  spec.email         = ['dev@watermark.org']
+
+  spec.summary       = File.readlines(File.expand_path('README.md', __dir__)).join
+  spec.description   = 'Simple Library for NLP over bodies of text'
+  spec.homepage      = 'https://github.com/watermarkchurch/media/wcc-text-analysis'
+  spec.license       = 'MIT'
+
+  spec.required_ruby_version = '>= 2.3'
+
+  spec.files         =
+    `git ls-files -z`.split("\x0").reject do |f|
+      f.match(%r{^(test|spec|features)/})
+    end
+
+  spec.require_paths = ['lib']
+
+  spec.add_runtime_dependency 'phrasie', '0.1.5'
+  spec.add_runtime_dependency 'redcarpet', '~> 3.4'
+  spec.add_runtime_dependency 'wcc-base', '~> 0.3.1'
+
+  spec.add_development_dependency 'guard', '~> 2.15'
+  spec.add_development_dependency 'guard-rspec', '~> 4.7'
+  spec.add_development_dependency 'guard-rubocop', '~> 1.3'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rspec_junit_formatter', '~> 0.3.0'
+  spec.add_development_dependency 'rubocop', '~> 0.68.0'
+  spec.add_development_dependency 'rubocop-performance'
+end

metadata

@@ -0,0 +1,194 @@
+--- !ruby/object:Gem::Specification
+name: wcc-text-analysis
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Watermark Dev
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2021-03-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: phrasie
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.1.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.1.5
+- !ruby/object:Gem::Dependency
+  name: redcarpet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+- !ruby/object:Gem::Dependency
+  name: wcc-base
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.1
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.15'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.15'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.7'
+- !ruby/object:Gem::Dependency
+  name: guard-rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rspec_junit_formatter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.68.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.68.0
+- !ruby/object:Gem::Dependency
+  name: rubocop-performance
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Simple Library for NLP over bodies of text
+email:
+- dev@watermark.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rubocop.yml"
+- Gemfile
+- Guardfile
+- README.md
+- Rakefile
+- db/stop_words.txt
+- db/transcript_exclusions.txt
+- lib/wcc/text_analysis.rb
+- lib/wcc/text_analysis/version.rb
+- wcc-text-analysis.gemspec
+homepage: https://github.com/watermarkchurch/media/wcc-text-analysis
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.3'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key:
+specification_version: 4
+summary: ''
+test_files: []