cass 0.0.1

data/lib/cass/contrast.rb

@@ -0,0 +1,55 @@
+module Cass
+  
+  # Defines a single contrast on a document or documents.
+  # Currently, only comparisons between two pairs of words (i.e., computation of
+  # the interaction term; see Holtzman et al., submitted for details) is implemented.
+  class Contrast
+  
+    attr_accessor :words
+  
+    def initialize(words)
+      @words = words
+    end
+  
+    # Initialize a contrast from a string representation.
+    def self.parse(contrast)
+      words = contrast.split(/,*\s+/)
+      words = [words[0,2], words[2,2]] if (words.size == 4)
+      Contrast.new(words)
+    end
+  
+    # Apply the contrast to a document and return result as a string.
+    # See to_s method for format.
+    def apply(doc)
+      sim = doc.similarity(@words.flatten)
+      if sim.class == Array
+        puts "Error: #{doc.name} is missing the following operands: #{sim.join(", ")}"
+        return false
+      end
+      # Compute interaction term
+      if @words.flatten.size == 4
+        s = [doc.name, sim[0,2], sim[0,3], sim[1,2], sim[1,3]]
+        s[1,4] = s[1,4].map { |f| format("%0.4f", f).to_f }
+        s << s[1] - s[2] - s[3] + s[4]
+        s.join("\t")
+      else
+        [doc.name, sim[0,1]].join("\t")
+      end
+    end
+  
+    # Returns a string representation of the contrast.
+    # For interaction terms, returns five columns (the four pairs
+    # and the interaction term).
+    # For pairwise contrasts, just returns the similarity.
+    def to_s
+      if @words.flatten.size == 4
+        o = @words.flatten
+        "document\t#{o[0]}.#{o[2]}\t#{o[0]}.#{o[3]}\t#{o[1]}.#{o[2]}\t#{o[1]}.#{o[3]}\tinteraction"
+      else
+        "document\t#{@words.join(".")}"
+      end
+    end
+  
+  end
+  
+end

data/lib/cass/document.rb

@@ -0,0 +1,233 @@
+module Cass
+  
+  # A Document object represents a single document--
+  # can be either an entire file, or a subset.
+  class Document
+  
+    attr_accessor :name, :targets, :text, :lines, :clines, :context, :tindex, :unique
+  
+    # Create a new Document. Three arguments are required:
+    # * name: The name of the document (defaults to the filename)
+    # * targets: Either an array of target words, or an array of Contrasts from which targets will be extracted
+    # * text: A string of text (the contents of the document)
+    #
+    # The following (optional) settings can be passed in an options hash as the fourth argument:
+    # * context: A Context object to use (by default, a new one will be constructed)
+    # * skip_preproc: Skip most text preprocessing steps. This should only ever be used when creating a document derived from another document, where the text has already been processed.
+    # * max_lines: Maximum number of lines to use from the provided text. Note that this limit applies to the number of lines in the input text, NOT the number retained for analysis. By default, will use all lines.
+    # * recode: a hash containing words to recode in the text prior to analysis. For instance, if the key=>value pair 'liberal'=>'democrat' is passed, all occurrences of 'liberal' will be replaced with 'democrat'. This is useful when you want to analyze several words together as a single category, or for combining singular and plural forms of a word.
+    # * keep_case: By default, all words will be converted to lowercase. Passing this key in the options hash will preserve case in the text. Note that this will cause different cases of the same word to be handled as different words.
+    # * keep_special: By default, all non-alphabetical characters will be removed. Use this flag if you want special characters to be retained, with the same caveat as for keep_case.
+    # * parse_text: By default, it's assumed that the text is already broken up into sentences at desired boundaries (one sentence per line). If the parse_text key is passed, a parser will be called. Note that USING THIS OPTION IS NOT RECOMMENDED. You should generally preprocess the input text yourself to ensure it looks right before submitting it to analysis.
+    # * parser_basic: If parse_text is on, the Parser will try to call the Stanford Parser by default. If the Stanford Parser isn't installed properly, a backup basic parser will be used. Including the parser_basic flag will automatically revert to the basic parser instead of attempting to use the Stanford parser.
+    # * parser_regex: A custom regular expression that will be handed to the basic parser. Lines will be split at matches to the regex instead of the default (splitting only at newlines and periods). Note that parser_basic and parse_text must both be passed for this to work.
+    def initialize(name, targets, text, opts={})
+    
+      # Error checking...
+      if name.nil?
+        abort("Error: document has no name!")
+      elsif targets.nil? or targets.class != Array or targets.empty?
+        abort("Error: invalid target specification; targets must be an array of words or Contrasts.")
+      elsif text.nil?
+        abort("Error: no text provided!")
+      end
+    
+      # Set/initialize instance variables
+      @name, @text, @tindex = name, text, {}
+    
+      # Get list of words from contrasts if necessary
+      @targets = 
+      if targets[0].class == Contrast
+        targets = contrasts.inject([]) { |t, c| t += c.words.flatten }.uniq
+      else
+        targets
+      end
+    
+      # Index targets, parse text, and create Context
+      @targets.each_index { |i| @tindex[@targets[i]] = i }
+      parse(opts)
+      @context = context.nil? ? Context.new(self, opts) : @context
+    end
+  
+    # Parse raw text into sentences. When orig == false
+    # (e.g., when bootstrapping or splitting a document),
+    # skip certain preprocessing steps, on the assumption that
+    # these have already been performed.
+    def parse(opts={})
+      if opts.key?('skip_preproc')
+        @lines = (text.class == Array) ? @text : text.split(/[\r\n]+/)
+      else
+        puts "Converting to lowercase..." if VERBOSE
+        @text.downcase! unless opts.key?('keep_case')
+        @text.gsub!(/[^a-z \n]+/, '') unless opts.key('keep_special')
+        if opts.key?('recode')
+          puts "Recoding words..." if VERBOSE
+          opts['recode'].each { |k,v| @text.gsub!(/(^|\s+)(#{k})($|\s+)/, "\\1#{v}\\3") }
+        end
+        puts "Parsing text..." if VERBOSE
+        @lines = opts.key?('parse_text') ? Parser.parse(@text, opts) : @text.split(/[\r\n]+/)
+        @lines = @lines[0, opts['max_lines']] if opts.key?('max_lines')
+        trim!
+      end
+    end
+  
+    # Trim internal list of lines, keeping only those that contain
+    # at least one target word.
+    def trim!
+      puts "Deleting target-less lines..." if VERBOSE
+      ts = @targets.join("|")
+      #@lines.delete_if { |s| (s.split(/\s+/) & @targets).empty? }  # another way to do it
+      nl = @lines.size
+      @lines = @lines.grep(/(^|\s+)(#{ts})($|\s+)/)
+      puts "Keeping #{@lines.size} / #{nl} lines." if VERBOSE
+      self   
+    end
+  
+    # Randomly reorder lines of text.
+    # If clines is true, use the compacted lines variable, otherwise use all lines.
+    def permute(clines=false)
+      clines ? @clines.sort_by {rand} : @lines.sort_by {rand}
+    end
+  
+    # Same as permute, but replaces contents of current document.
+    def permute!(clines=false)
+      self.instance_variable_set("#{clines ? 'clines' : 'lines'}", permute(clines))
+      self
+    end
+  
+    # Resample n lines WITH replacement from text (for bootstrapping).
+    # n = number of lines to resample; defaults to full size of corpus.
+    # If clines is true, use the compacted lines variable, otherwise use all lines.
+    def resample(clines=false, n=nil)
+      n = @lines.size if n.nil? or n > @lines.size
+      max = @lines.size
+      Array.new(n).map { |i| clines ? @clines[rand(max)] : @lines[rand(max)] }
+    end
+  
+    # Same as resample, but replaces contents of current document.
+    def resample!(clines=false, n=nil)
+  		self.instance_variable_set("#{clines ? 'clines' : 'lines'}", resample(clines,n))
+      self
+    end
+  
+    # Split Document into n smaller random subsets,
+    # recalculating the context each time. Currently not used.
+    def split(n=10, recalc=true)
+      permute!
+      docs = []
+      n.times { |i|
+        text = @lines.slice!(0,(@lines.size.to_f/(n-i)).round)
+        context = recalc ? nil : @context
+        name = "#{@name}_split_#{(i+1)}"
+        docs[i] = Document.new(name, @targets, text, context, false)
+      }
+      docs
+    end
+  
+    # Bootstrap a distribution of n documents,
+    # recalculating the context each time. Note that this is currently
+    # not used anywhere because all the work is done in the Analysis class.
+    # def bootstrap(n=10, recalc=true)
+    #   permute!
+    #   docs = [self]
+    #   n.times { |i|
+    #     puts "Generating bootstrap ##{i+1}..." if VERBOSE
+    #     d = self.clone
+    #     d.name = "#{@name}_boot_#{(i+1)}"
+    #     d.resample!
+    #     d.context = Context.new(d) if recalc
+    #     d.text = nil  # Don't need to save this again, save time by deleting...
+    #     docs << d
+    #   }
+    #   docs    
+    # end
+  
+    # Drop all words that aren't in target list or context. Store as an array of arrays,
+    # with first element = array of targets and second = array of context words.
+    def compact
+  		puts "Compacting all lines..." if VERBOSE
+  		@clines = []
+  		@lines.each { |l|
+  			w = l.split(/\s+/).uniq
+  			targs = w.select { |s| @tindex.key?(s) }
+  			conts = w.delete_if { |s| !@context.key?(s) }
+  			@clines << [targs, conts]
+  		}
+  	end
+  
+    # Computes co-occurrence matrix between target words and the context.
+    # Stores a target-by-context integer matrix internally.
+    def cooccurrence(normalize_weights=false)
+      # puts "Generating co-occurrence matrix..." if VERBOSE
+      coocc = NMatrix.float(@targets.size, @context.size)
+  		compact if @clines.nil?
+      lc = 0  # line counter
+      @clines.each { |l|
+  			targs, conts = l
+  			targs.each { |t|
+  				conts.each { |c|
+  					next if t == c
+  					incr = normalize_weights ? 1.0/conts.size : 1
+  					coocc[@tindex[t], @context[c]] = coocc[@tindex[t], @context[c]].to_f + incr
+  				}
+  			}
+      }
+      @cooc = coocc  #.to_f
+      @corr = @cooc.corr#.collect { |i| i*i }  # Uncomment second half of line to square coefficients.
+      #p @corr.to_a
+      self
+    end
+  
+    # Return the requested subset of the similarity matrix.
+    # E.g., given the input ['apple', 'orange', 'elephant'],
+    # a 3 x 3 pairwise similarity matrix will be returned.
+    def similarity(words)
+      ind = @tindex.values_at(*words)
+      @corr[ind, ind]
+    end
+
+    # Computes the pairwise similarity between all possible target pairs
+    # and saves teh results to the specified file. Note that this will 
+    # produce an unmanageably large file if the number of 
+    # targets is very large!
+    # The returned string contains tab-delimited columns for:
+    # * Document name
+    # * First word in pair
+    # * Second word in pair
+    # * Similarity value (correlation)
+    def pairwise_similarity(filename)      
+      abort("Error: you must compute the similarity matrix first!") if @corr.nil?
+      outf = File.new(filename, 'w')
+      outf.sync = true
+      ind = @tindex.invert # For looking up words
+      outf.puts %w[Document Word1 Word2 Correlation].join("\t")
+      (dim = @corr.shape[0]).times { |i|
+        i.upto(dim-1) { |j| outf.puts [@name, ind[i], ind[j], @corr[i,j]] }
+      }
+    end
+    
+    # Print out summary information about the document.
+    # Optional arguments:
+    # * filename: if not nil, will save the results to location provided instead of printing.
+    # * list_context: if true, will print out the entire list (typically several thousand words) of words in the context.
+    # * word_count: print list of number of tokens in the document for each word, by descending frequency rank. Targets will be printed first. Note that the token counts reflect only the lines used in analysis (i.e., those that contain at least one target), and NOT the entire document. If you need word counts for an entire document, call Stats.word_count directly.
+    def summary(filename=nil, list_context=false, word_count=false)
+      
+      buffer = []
+      
+      # Basic info that always gets shown
+      buffer << "Summary for document '#{@name}':"
+      buffer << "#{@targets.size} target words (#{@targets.join (", ")})"
+      buffer << "#{@context.words.size} words in context."
+      buffer << "Using #{@clines.size} lines (containing at least one target word) for analysis."
+      
+      # Other options
+      buffer << "\n\nList of all words in context:#{@context.words}" if list_context
+      buffer << "\n\nNumber of tokens for all words included in analysis:\n#{Stats.word_count(@clines)}" if word_count
+      
+      filename.nil? ? puts(buffer) : File.new(filename, 'w').puts(buffer)
+   
+    end
+  end
+  
+end

data/lib/cass/extensions.rb

@@ -0,0 +1,16 @@
+# Added functionality in NArray matrix class.
+class NMatrix
+  
+  # Compute and return the column-wise correlation matrix for an NMatrix.
+  def corr
+    n = self.shape[0]
+    n.times { |i|
+      col = self[i,true]
+      sd = col.stddev
+      self[i,true] = ((col.sbt!(col.mean))/sd).to_a.flatten
+    }
+    #p self.to_a
+    (self.transpose*self)/(self.shape[1]-1).to_f
+  end
+  
+end

data/lib/cass/parser.rb

@@ -0,0 +1,43 @@
+module Cass
+  
+  # Parses a string (e.g., text read from a file) into sentences.
+  # Can use either the Stanford Natural Language Parser (if installed),
+  # or a barebones parser that splits text at line breaks and periods.
+  # Generally speaking, you shouldn't rely on the Parser class to 
+  # parse and sanitize your input texts for you. This class implements
+  # only barebones functionality, and there's no guarantee the resulting
+  # text will look the way you want it. You are strongly encouraged to 
+  # process all texts yourself beforehand, and use this functionality
+  # only as a last resort.
+  class Parser
+  
+    # Parses a string into sentences.If the Stanford Parser and 
+    # associated Ruby gem are installed (http://stanfordparser.rubyforge.org/),
+    # they will be called to do the job. If not, only basic parsing 
+    # will be performed: text will be split into sentences at newlines 
+    # and periods. Note that this is suboptimal and may generate problems
+    # for some documents.
+    def self.parse(text, opts={})
+      # Try to load Stanford Parser wrapper
+      begin
+        require 'stanfordparser'
+      rescue LoadError
+        puts "Error: stanfordparser gem couldn't load. Using barebones parsing mode instead. If you'd like to use" +
+            " the Stanford Parser, make sure all required components are installed (see http://stanfordparser.rubyforge.org/). You'll need to make sure the java library is installed, as well as the treebank and jrb gems."
+        spfail = true
+      end
+    
+      if spfail or opts.key?('parser_basic')
+        puts "Using a basic parser to split text into sentences. Note that this is intended as a last resort only; you are strongly encouraged to process all input texts yourself and make sure that lines are broken up the way you want them to be (with each line on a new line of text in the file). If you use this parser, we make no guarantees about the quality of the output."
+        rx = opts.key?('parser_regex') ? opts['parser_regex'] : "[\r\n\.]+"
+        text.split(/#{rx}/)
+      else
+        puts "Using the Stanford Parser to parse the text. Note that this could take a long time for large files!" if VERBOSE
+        parser = StanfordParser::DocumentPreprocessor.new
+        parser.getSentencesFromString(text)
+      end
+    end
+  
+  end
+  
+end

data/lib/cass/stats.rb

@@ -0,0 +1,40 @@
+module Cass
+  
+  # Collects miscellaneous descriptive statistic methods that
+  # may be useful. These are generally not hooked up to the 
+  # primary processing stream, and need to be called on an
+  # ad-hoc basis.
+  class Stats
+  
+    # Takes a string as input and prints out a list of all words encountered,
+    # sorted by their frequency count (in descending order).
+    # Words are separated by whitespace; no additional processing will be performed,
+    # so if you don't want special characters to define words, you need to preprocess
+    # the string before you call this method.
+    # Arguments:
+    # * text: the string to count token occurrences in.
+    # * stopwords: optional location of stopword file. Words in file will be excluded from count.
+    # * save: the filename to save the results to. If left nil, will print to screen.
+    def self.word_count(text, stopwords=nil, save=nil)
+      sw = {}
+      text = text.join(" ") if text.class == Array
+      File.new(stopwords).readlines.each { |l|  sw[l.strip] = 1 } if !stopwords.nil?
+      words = text.split(/\s+/)
+      counts = Hash.new(0)
+      words.each { |w| counts[w] += 1 if !sw.key?(w) }
+      counts = counts.sort { |a,b| b[1] <=> a[1] }.each { |l| "#{l[0]}: #{l[1]}" }
+      if save.nil?
+        puts counts
+      else
+        File.new(save, 'w').puts counts
+      end
+    end
+  
+    # Count the number of times a given token s occurs in text.
+    def self.string_tokens(text, s)
+      text.scan(/#{s}/).size
+    end
+  
+  end
+
+end

metadata

@@ -0,0 +1,112 @@
+--- !ruby/object:Gem::Specification 
+name: cass
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Tal Yarkoni
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-06-15 00:00:00 -06:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: narray
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 77
+        segments: 
+        - 0
+        - 5
+        - 9
+        - 7
+        version: 0.5.9.7
+  type: :runtime
+  version_requirements: *id001
+description: A set of tools for conducting Contrast Analyses of Semantic Similarity (CASS).
+email: tyarkoni@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- CHANGELOG
+- LICENSE
+- README.rdoc
+- lib/cass.rb
+- lib/cass/analysis.rb
+- lib/cass/context.rb
+- lib/cass/contrast.rb
+- lib/cass/document.rb
+- lib/cass/extensions.rb
+- lib/cass/parser.rb
+- lib/cass/stats.rb
+files: 
+- CHANGELOG
+- LICENSE
+- Manifest
+- README.rdoc
+- Rakefile
+- cass.gemspec
+- lib/cass.rb
+- lib/cass/analysis.rb
+- lib/cass/context.rb
+- lib/cass/contrast.rb
+- lib/cass/document.rb
+- lib/cass/extensions.rb
+- lib/cass/parser.rb
+- lib/cass/stats.rb
+has_rdoc: true
+homepage: http://casstools.org
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --line-numbers
+- --inline-source
+- --title
+- Cass
+- --main
+- README.rdoc
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 11
+      segments: 
+      - 1
+      - 2
+      version: "1.2"
+requirements: []
+
+rubyforge_project: cass
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: A set of tools for conducting Contrast Analyses of Semantic Similarity (CASS).
+test_files: []
+