automated_metareview 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.idea/automated_metareview.iml

@@ -0,0 +1,91 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="RUBY_MODULE" version="4">
+  <component name="FacetManager">
+    <facet type="gem" name="Gem">
+      <configuration>
+        <option name="GEM_APP_ROOT_PATH" value="$MODULE_DIR$" />
+        <option name="GEM_APP_TEST_PATH" value="" />
+        <option name="GEM_APP_LIB_PATH" value="" />
+      </configuration>
+    </facet>
+  </component>
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="library" scope="PROVIDED" name="RedCloth (v4.2.9, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="actionpack (v4.0.0, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="activemodel (v4.0.0, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="activerecord (v4.0.0, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="activerecord-deprecated_finders (v1.0.3, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="activesupport (v4.0.0, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="arel (v4.0.0, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="atomic (v1.1.14, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="bind-it (v0.2.7, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="builder (v3.1.4, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="bundler (v1.3.5, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="capistrano (v3.0.0, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="childprocess (v0.3.9, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="columnize (v0.3.6, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="daemons (v1.1.9, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="debugger (v1.6.2, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="debugger-linecache (v1.2.0, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="debugger-ruby_core_source (v1.2.3, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="delayed_job (v4.0.0, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="delayed_job_active_record (v4.0.0, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="diff-lcs (v1.2.4, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="edavis10-ruby-web-search (v0.0.2, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="engtagger (v0.1.2, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="erubis (v2.7.0, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="expertiza-authlogic (v3.2.0@09163c, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="fastercsv (v1.5.5, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="ffi (v1.9.0, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="ffi-aspell (v0.0.3, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="gchart (v1.0.0, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="gchartrb (v0.8, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="gdata (v1.1.2, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="hike (v1.2.3, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="hoptoad_notifier (v2.4.11, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="i18n (v0.6.5, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="jquery-rails (v3.0.4, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="json (v1.8.1, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="mini_portile (v0.5.2, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="minitest (v4.7.5, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="multi_json (v1.8.0, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="mysql (v2.9.1, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="net-scp (v1.1.2, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="net-ssh (v2.7.0, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="nokogiri (v1.6.0, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="open-uri-cached (v0.0.5, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rack (v1.5.2, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rack-test (v0.6.2, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="railties (v4.0.0, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rake (v10.1.0, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rgl (v0.4.0, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rjb (v1.4.8, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rspec-core (v2.14.5, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rspec-expectations (v2.14.3, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rspec-mocks (v2.14.3, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rspec-rails (v2.14.0, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rubyzip (v1.0.0, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rwordnet (v0.1.3, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="seer (v0.10.0, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="selenium-webdriver (v2.37.0, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="sprockets (v2.10.0, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="sqlite3 (v1.3.8, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="sqlite3-ruby (v1.3.3, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="sshkit (v1.1.0, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="stanford-core-nlp (v0.5.1, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="stream (v0.5, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="superfish-rails (v1.6.0.1, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="term-ansicolor (v1.2.2, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="thor (v0.18.1, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="thread_safe (v0.1.3, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="tilt (v1.4.1, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="tins (v0.12.0, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="tzinfo (v0.3.38, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="websocket (v1.0.7, ruby-1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="will_paginate (v3.0.5, ruby-1.9.3-p392) [gem]" level="application" />
+  </component>
+</module>
+

data/.idea/encodings.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Encoding" useUTFGuessing="true" native2AsciiForPropertiesFiles="false" />
+</project>
+

data/.idea/misc.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="ruby-1.9.3-p392" project-jdk-type="RUBY_SDK" />
+</project>
+

data/.idea/modules.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/automated_metareview.iml" filepath="$PROJECT_DIR$/.idea/automated_metareview.iml" />
+    </modules>
+  </component>
+</project>
+

data/.idea/scopes/scope_settings.xml

@@ -0,0 +1,5 @@
+<component name="DependencyValidationManager">
+  <state>
+    <option name="SKIP_IMPORT_STATEMENTS" value="false" />
+  </state>
+</component>

data/.idea/vcs.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="" />
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>
+

data/Gemfile

@@ -0,0 +1,42 @@
+source 'https://rubygems.org'
+
+
+
+gem 'bind-it'
+gem 'capistrano'
+gem 'delayed_job_active_record'
+gem 'edavis10-ruby-web-search'
+gem 'engtagger'
+gem 'expertiza-authlogic', git: 'https://github.com/expertiza/authlogic.git', :require => 'authlogic'
+gem 'fastercsv'
+gem 'ffi-aspell'
+gem 'gchart'
+gem 'gchartrb', :require => 'google_chart'
+gem 'gdata', :require => false
+gem 'hoptoad_notifier'
+gem 'jquery-rails'
+gem 'mysql'
+gem 'nokogiri'
+gem 'open-uri-cached'
+gem 'rake'
+gem 'RedCloth'
+gem 'rgl', :require => 'rgl/adjacency'
+gem 'rjb'
+gem 'rubyzip', :require => 'zip/zip'
+gem 'rwordnet'
+gem 'seer'
+gem 'sprockets'
+gem 'stanford-core-nlp'
+gem 'superfish-rails'
+gem 'will_paginate'
+
+group :development do
+  gem 'daemons'
+  gem 'debugger'
+  gem 'sqlite3-ruby', :require => 'sqlite3'
+  gem 'selenium-webdriver'
+end
+
+# Specify your gem's dependencies in automated_metareview.gemspec
+gemspec
+

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 AMEETA MURALIDHARAN
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# AutomatedMetareview
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'automated_metareview'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install automated_metareview
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/automated_metareview.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'automated_metareview/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "automated_metareview"
+  spec.version       = AutomatedMetareview::VERSION
+  spec.authors       = ["AMEETA MURALIDHARAN"]
+  spec.email         = ["fameeta@ncsu.edu"]
+  spec.description   = "automated metareview"
+  spec.summary       = "creation of gem for automated metareview"
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "engtagger"
+  spec.add_development_dependency "rspec-rails"
+  spec.add_development_dependency "activerecord"
+
+end

data/lib/automated_metareview.rb

@@ -0,0 +1,18 @@
+require 'automated_metareview/text_preprocessing'
+require 'automated_metareview/predict_class'
+require 'automated_metareview/degree_of_relevance'
+require 'automated_metareview/plagiarism_check'
+require 'automated_metareview/tone'
+require 'automated_metareview/text_quantity'
+require 'automated_metareview/constants'
+
+#gem install edavis10-ruby-web-search
+#gem install google-api-client
+
+class AutomatedMetareview #< ActiveRecord::Base
+  #belongs_to :response, :class_name => 'Response', :foreign_key => 'response_id'
+  #has_many :scores, :class_name => 'Score', :foreign_key => 'response_id', :dependent => :destroy
+  def self.hi
+    puts "hello"
+  end
+end

data/lib/automated_metareview/constants.rb

@@ -0,0 +1,208 @@
+require 'wordnet' 
+#necessary to access the data.{po} files in the wordnet/dict folder of the rwordnet gem
+include WordNet
+WordNet::SynsetType = {"n" => "noun", "v" => "verb", "a" => "adj", "r" => "adv"}
+    
+    
+# Create a parser object
+#frequently used general constants
+#prevtype tokens for graph generator
+NOUN = 1
+VERB = 2
+ADJ = 3
+ADV = 4
+#------------------------------------------#------------------------------------------
+#constants used by sentence state
+POSITIVE = 0
+SUGGESTIVE = 1
+NEGATED = 2
+NEGATIVE_WORD = 3
+NEGATIVE_DESCRIPTOR = 4
+NEGATIVE_PHRASE = 5
+#------------------------------------------#------------------------------------------
+
+#used by patternIdentify and predictClass and relevance, while comparing edges!
+EQUAL = 1.0
+DISTINCT = 2.0
+#------------------------------------------#------------------------------------------
+
+#constants used by graph generator
+WORDS = 10000 #to control the number of tokens/vertices a graph contains
+#thresholds for pruning edges during pattern selection
+ALPHA_FREQ = 2 #alpha - the frequency threshold (lower)
+BETA_FREQ = 10
+#------------------------------------------#------------------------------------------
+
+#threshold for tone
+THRESHOLD = 3 #number of level synonyms to check for tone identification
+#------------------------------------------#------------------------------------------
+
+#constants used by WordnetBasedSimilarity
+#initializing constants
+NOMATCH = 0 #distinct
+OVERLAPEXAM = 1 #overlapping examples
+OVERLAPDEFIN = 1 #overlapping definitions
+COMMONPARENTS = 2 #common parents
+MERONYM = 3 #paraphrasing
+HOLONYM = 3 #paraphrasing
+HYPONYM = 4 #paraphrasing
+HYPERNYM = 4 #paraphrasing
+SYNONYM = 5 #paraphrasing
+EXACT = 6 #exact strings
+  
+#negative matches
+NEGOVERLAPEXAM = -1 #overlapping examples
+NEGOVERLAPDEFIN = -1 #overlapping definitions
+NEGCOMMONPARENTS = -2 #common parents
+NEGMERONYM = -3 #paraphrasing
+NEGHOLONYM = -3 #paraphrasing
+NEGHYPONYM = -4 #paraphrasing
+NEGHYPERNYM = -4 #paraphrasing
+ANTONYM = -5 #antonyms
+NEGEXACT = -6 #exact strings
+#------------------------------------------#------------------------------------------
+
+#for plgiarism checking - n-gram
+NGRAM = 5 #5-gram matches
+#other constants for plagiarism checking
+ALL_RESPONSES_PLAGIARISED = 1
+SOME_RESPONSES_PLAGIARISED = 2
+#------------------------------------------#------------------------------------------
+
+#constants from text_preprocessing
+SIM_MATCH = 5
+SENTENCES = 100  #assuming each review has upto 5 sentences max.
+MAX = 3000
+#------------------------------------------#------------------------------------------
+
+=begin 
+   THE FIRST TWO SETS ARE COMMON WORDS DURING OVERLAP ACROSS DEFINITIONS OR EXAMPLES.
+   THE THIRD SET IS TO PREVENT FREQUENT WORDS FROM BEING COMPARED WITH OTHER TOKENS.
+=end   
+CLOSED_CLASS_WORDS = [".", ",", "THE", 
+        "AND", "A", "\"", "IN", "I", ":", "YOU", "IS", "TO", "OF", 
+        ")", "(", "IT", "FOR",  "!",  "?","THAT","ON", "WITH", "HAVE", 
+         "BE", "...",  "AS","THIS","WAS", "e.g.", "especially","BUT", "OR", "FROM","WHAT","NOT", "ARE",  
+        "MY", "AT",   "HE", "BY", "ONE","IF", "THEY", "YOUR","ALL",
+        "ME", "SO",  "AN",  "WE", "CAN", "WILL", "DO","ABOUT","JUST",
+         "OUT", "HIS",  "WHO", "WOULD","THERE","LIKE", "HAS","MORE","UP","NO",    
+        "THEM", "ANY", "THEIR", "IT'S", "ONLY", "WHEN","SOME","HAD", "DON'T",  
+        ";",  "I'M", "BEEN", "WHICH", "OTHER", "WERE", "HOW", "THEN", "NOW",
+        "HER",  "SHE",  "ALSO", "US", "VERY", "BECAUSE","THAN","WELL",
+        "AM",   "HIM", "INTO", "OUR", "COULD", "EVEN","MUCH","HERE",
+        "TOO","THESE", "THOSE","MAY", "WHERE", "MOST","SHOULD", "OVER", "WANT", "DID",   
+        "WHY", "OFF", "IT", "ITS", "I'VE","MANY","GOING", "THOSE", "DOES","PLEASE",  
+        "THAT'S",  "YOU'RE",  "DOWN","ANOTHER", "AROUND","CAN'T","DIDN'T", 
+         "MUST","YES", "EACH", "MAYBE","EVERY", "FEW", "DOESN'T",  
+        "I'LL", "OH", "ELSE", "HE'S", "THERE'S", "HI", "AWAY", "DOING","ISN'T",  
+        "OK", "THEY'RE", "YEAH", "MINE", "WE'RE", "WHAT'S", "SHALL","WON'T", 
+        "SHE'S", "HELLO", "OKAY", "HERE'S", "-", "LESS", "USED", "use", "associated", "having", "certain",
+        "etc", "etc.", "act", "purpose"]
+  
+# @invisible
+STOP_WORDS = ["a", "am", "an", "and", "any", "as", "at", "is", "it", "its","de", "by","i", 
+      "ie", "if", "in","no","of", "off", "or", "eg", "the", "too", "are", "the", "he",
+      "about", "above", "across", "after", "afterwards", "again", "against",
+      "all", "almost", "alone", "along", "already", "also", "although", "always", 
+      "among", "amongst", "amoungst", "amount",  "another", 
+      "anyhow", "anyone", "anything", "anyway", "anywhere", "are", "around",
+     "back", "be", "became", "because", "become", "becomes",
+      "becoming", "been", "before", "beforehand", "behind", "being", "below",
+       "beside", "besides", "between", "beyond", "bill", "both", "bottom",  "call", 
+       "can", "cannot", "cant", "co", "computer", "con", "could",
+      "couldnt", "cry",  "describe", "detail", "do", "done", "does", "down",
+      "due", "during", "each",  "eight", "either", "eleven", "else", "elsewhere",
+      "empty", "enough", "etc", "even", "ever", "every", "everyone", "everything",
+      "everywhere", "except", "few", "fifteen", "fify", "fill", "find",
+      "fire", "first", "five", "for", "former", "formerly", "forty", "found",
+      "four", "from", "front", "full", "further", "get", "give", "go",
+      "had", "has", "hasnt", "have", "he", "hence", "her", "here", "hereafter",
+      "hereby", "herein", "hereupon", "hers", "herself", "him", "himself", "his",
+      "how", "however", "hundred",  "inc", "indeed", 
+      "interest", "into",  "itself", "keep", "last", "latter",
+      "latterly", "least", "less", "ltd", "made", "many", "may", "me", 
+      "meanwhile", "might", "mill", "mine", "more", "moreover", "most", "mostly",
+      "move", "much", "must", "my", "myself", "name", "namely", "neither", 
+      "never", "nevertheless", "next", "nine",  "nobody", "none", "noone",
+      "nor", "not", "nothing", "now", "nowhere",  "often", "on", 
+      "once", "one", "only", "onto",  "other", "others", "otherwise", "our",
+      "ours", "ourselves", "out", "over", "own", "part", "per", "perhaps", 
+      "please", "put", "rather", "re", "same", "see", "seem", "seemed", "seeming",
+      "seems", "serious", "several", "she", "should", "show", "side", "since", 
+      "sincere", "six", "sixty", "so", "some", "somehow", "someone", "something",
+      "sometime", "sometimes", "somewhere", "still", "such", "system", "take", 
+      "ten", "than", "that", "the", "their", "them", "themselves", "then", 
+      "thence", "there", "thereafter", "thereby", "therefore", "therein", 
+      "thereupon", "these", "they", "thick", "thin", "third", "this", "those",
+      "though", "three", "through", "throughout", "thru", "thus", "to", 
+      "together",  "top", "toward", "towards", "twelve", "twenty", "two",
+      "un", "under", "until", "up", "upon", "us", "very", "via", "was", "we", 
+      "well", "were", "what", "whatever", "when", "whence", "whenever", "where", 
+      "whereafter", "whereas", "whereby", "wherein", "whereupon", "wherever", 
+      "whether", "which", "while", "whither", "who", "whoever", "whole", "whom", 
+      "whose", "why", "will", "with", "within", "without", "would", "yet", 
+      "you", "your", "yours", "yourself", "yourselves", "individual","individually"]
+    
+#tokens containing these words aren't compared with other tokens, because they don't add any meaning
+FREQUENT_WORDS = [ "a", "am", "an", "and", "any", "as", 
+      "at", "are","be", "by", "can", "did", "do", "does", "is", "it", "its", "i", "ie", "if", "in","no",
+      "or", "eg", "me", "my","of", "off", "oh", "our", "ours", "was", "have", "has", 
+       "so","she", "the", "too", "to","they", "their", "theirs", "that", "this", "then", 
+       "there","than", "up",  "us", "u", "his", "her", "hers",
+       "we", "with","were", "you", "your", "yours"]
+
+#puts FREQUENT_WORDS
+#------------------------------------------#------------------------------------------
+#constants used by sentence state - for identifying negations and suggestive words
+
+=begin
+If negative words were found and a neg. descriptor is seen => (-)(-) = (+)
+public static String[] NEGATIVE_DESCRIPTORS = new String[4879];
+=end
+NEGATIVE_DESCRIPTORS = [
+      #negated phrases (topical words I spotted in text)
+      "NOTHING", "nowhere", "scarcely", "scarce", "zero", "drawback",
+          "barely", "hardly", "deny", "refuse", "fail", "failed",
+          "without", "ambiguous", "ambiguity", "neither", "empty",
+        "deviation", "lacks", "lack", "lacking", "lacked", "abrupt", "abruptly", "somewhat", "copied", "copy",
+        "overbalanced", "missing", "poor",
+        "negative", "negatively", "underrepresented", "duplication", "wrong", "mistake", "mistakes","duplications",
+        "duplicate", "duplicated", "duplicating", "avoids", "messy", "deleted", "cumbersome", "strange",
+        "strangely", "misspell", "misspelling", "misspellings", "misspelt", "verbose", "confuse", "confusion", "confusing",
+        "confused", "confuses", "trivial", "triviality", "typo", "typos", "somewhat", "concerns", "concern",
+        "barring", "overuse", "repitition", "useless", "biased", "rushed", "absent", "wordy", "bad", "less", 
+        "unclear", "difficult", "vague", "briefly", "hard", "broken","replicate","replicated", "digress", "clutter",
+        "cluttered", "inadequate", "deviation", "contrived", "contrive", "horrid", "trouble","uneven", "unevenly", "alot",
+        "incorrect", "nonsense", "insufficient"]
+=begin
+  SENTENCE CLAUSE OR PHRASE FOLLOWING THESE WORDS CARRY A NEGATIVE MEANING (EITHER SUBTLE OR OVERT)
+=end
+NEGATED_WORDS = ["not", "n't", "WON'T", "DON'T", "DIDN'T", 
+        "DOESN'T", "WOULDN'T", "COULDN'T", "SHOULDN'T", "WASN'T", 
+        "WEREN'T", "AREN'T", "ISN'T", "HAVEN'T", "HASN'T", "HADN'T",  "NOBODY",  
+        "CAN'T","SHALLN'T", "MUSTN'T", "AIN'T", "cannot",        
+        #without the apostrophe
+        "cant", "dont", "wont","isnt","hasnt", "hadnt", "havent","arent", "werent", "wouldnt",
+        "didnt", "couldnt", "shouldnt", "mustnt", "shallnt",        
+        #other words that indicate negations (negative quantifiers)
+        "NO", "NEVER", "NONE"]
+          
+NEGATIVE_PHRASES = ["too concise",
+    "taken from", "been removed", "too long", "off topic",
+    "too short", "run on", "too much", "been directly", "similar to",
+    "at odds", "grammatical errors", "grammatical error", "available online",
+    "make up", "made up", "crammed up"]
+    
+SUGGESTIVE_WORDS = ["would", "could", "should",
+    "maybe", "perhaps", "might", "suggest", "suggests", "suggested", "advise", "advice","could've", "would've",
+    "should've", "might've", "may", #"will", "better", "can"
+    #when they provide sample examples
+    "eg",
+    #typos or colloqial
+     "I'd", "We'd", "they'd", "you'd"]
+
+#suggestive phrases
+SUGGESTIVE_PHRASES = ["for example","try adding", "instead of", #(explaining what isnt good and what potentially could be)
+    "little more", "try to", "need more", "needs to", "need to", "more detail",
+    "can be", "was expecting", "am expecting", "is required"]
+