shalmaneser-fred 1.2.0.rc4 → 1.2.rc5

data/lib/fred/slide_var.rb

@@ -0,0 +1,148 @@
+module Shalmaneser
+  ##########################################
+  module Fred
+    ################
+    # SlideVar:
+    # keeps a single sliding variable,
+    # has an iterator that yields each value of the slide as a pair
+    #  [lhs, rhs] to be passed on to FredConfigData.set_entry()
+    #
+    # Initialization with the value of a --slide command line parameter.
+    # Valid forms:
+    #
+    #  feature=<f>:<what>:<start>-<end>:<slide>
+    #       with f in { context, ngram, syn, grfunc, fe }
+    #            what in { weight, dist } (dist only available for context)
+    #            start, end, slide floats represented as strings
+    #
+    # <var>:<start>-<end>:<slide>
+    #       with var in { smoothing_lambda, window_size }
+    class SlideVar
+      attr_reader :var_name
+
+      def initialize(string, # value of --slide parameter
+                     exp)    # FredConfigData object
+
+        # keep start and end value and step size for the sliding
+        @startval = @endval = @step = @current = 0.0
+
+        # setting experiment file values for each step of the sliding:
+        # remember lhs and rhs of what needs to be set.
+        # rhs contains a string REPLACEME to be replaced by the current value
+        @exp_lhs = ""
+        @exp_rhs = ""
+        @var_name = ""
+        @remove_list_variable_regexp = nil # set non-nil if we need unset_list_entry()
+
+        if string == ""
+          # empty slide variable
+          return
+        end
+
+        if string =~ /^feature=(\w+):(\w+):([\d\.]+)-([\d\.]+):([\d\.]+)$/
+          # --slide feature=ngram:weight:0.8-4.0:0.3
+          # --slide feature=context:dist:0.7-0.9:0.05
+
+          featurename = $1
+          parname = $2
+          @startval = $3.to_f
+          @endval = $4.to_f
+          @step = $5.to_f
+
+          @exp_lhs = "feature"
+
+          if featurename == "context"
+            # both weight and dist possible
+
+            case parname
+            when "weight"
+              @exp_rhs = "#{featurename} REPLACEME #{exp.get_lf("feature", "context", "wtdist")}"
+            when "dist"
+              @exp_rhs = "#{featurename} #{exp.get_lf("feature", "context", "weight")} REPLACEME"
+            else
+              raise "Error in argument of --slide: I found a value of neither 'weight' nor 'dist': "+ parname
+            end
+
+            if exp.get_lf("feature", "context", "mwedist")
+              @exp_rhs << " mwedist"
+            end
+
+          else
+            # feature name not "context": only weight possible
+            unless parname == "weight"
+              raise "Error in argument of --slide: can only do 'weight', what I got is "+ parname
+            end
+
+            @exp_rhs = "#{featurename} REPLACEME"
+          end
+
+          @var_name = "feature #{featurename} #{parname}"
+          @remove_list_variable_regexp = Regexp.new("^#{featurename}\s")
+
+        elsif string =~ /^(\w+):([\d\.]+)-([\d\.]+):([\d\.]+)$/
+          # --slide window_size:0-4:1
+          # --slide smoothing_lambda:0.3-0.9:0.05
+
+          featurename = $1
+          case exp.get_type(featurename)
+          when "integer"
+            @startval = $2.to_i
+            @endval = $3.to_i
+            @step = $4.to_i
+          when "float"
+            @startval = $2.to_f
+            @endval = $3.to_f
+            @step = $4.to_f
+          else
+            raise "Unslidable variable "+ featurename
+          end
+
+          @exp_lhs = featurename
+          @exp_rhs = "REPLACEME"
+          @var_name = featurename
+
+        else
+          # not a valid argument to --slide
+
+          raise "Sorry, could not parse argument of --slide. \nI got: "+ string
+        end
+      end
+
+      ####
+      # iterate through each value of the slide variable (if there is a slide variable)
+      # and set it in the experiment file data structure
+      #
+      # also yield a descriptive text of the current setting
+      def each_slide_value(exp) # FredConfigData object
+
+        if empty?
+          # no slide variable
+
+          yield [0, ""]
+          return
+
+        else
+          # the slide variable is nonempty
+
+          @current = @startval
+
+          while @current <= @endval
+
+            if @remove_list_variable_regexp
+              # we have a list feature that we first need to unset before setting it
+              exp.unset_list_entry(@exp_lhs, @remove_list_variable_regexp)
+            end
+            exp.set_entry(@exp_lhs, @exp_rhs.sub(/REPLACEME/, @current.to_s))
+
+            yield [@current, @var_name + "=" + @current.to_s]
+            @current += @step
+          end
+        end
+      end
+
+      def empty?
+        @exp_lhs.empty?
+      end
+    end
+  end
+end

data/lib/fred/targets.rb

@@ -0,0 +1,136 @@
+require "fred/file_zipped"
+require 'fred/FredConventions' # !
+
+module Shalmaneser
+  module Fred
+    class Targets
+      attr_reader :targets_okay
+
+      ###
+      def initialize(exp,                 # experiment file object
+                     interpreter_class,   # SynInterpreter class, or nil
+                     mode)                # string: "r", "w", "a", as in files
+        @exp = exp
+        @interpreter_class = interpreter_class
+
+        # keep recorded targets here.
+        # try to read old list now.
+        @targets = {}
+
+        # write target info in the classifier directory.
+        # This is _not_ dependent on a potential split ID
+        @dir = File.new_dir(::Shalmaneser::Fred.fred_classifier_directory(@exp), "targets")
+
+        @targets_okay = true
+        case mode
+        when "w"
+        # start from scratch, no list of targets
+        when "a", "r"
+          # read existing file containing targets
+          begin
+            file = FileZipped.new(@dir + "targets.txt.gz")
+          rescue
+            # no pickle present: signal this
+            @targets_okay = false
+            return
+          end
+          file.each { |line|
+            line.chomp!
+            if line =~ /^LEMMA (.+) SENSES (.+)$/
+              lemmapos = $1
+              senses = $2.split
+              lemmapos.gsub!(/ /, '_')
+              #lemmapos.gsub!(/\.[A-Z]\./, '.')
+              @targets[lemmapos] = senses
+            end
+          }
+
+        else
+          $stderr.puts "Error: shouldn't be here."
+          exit 1
+        end
+
+        if ["w", "a"].include? mode
+          @record_targets = true
+        else
+          @record_targets = false
+        end
+      end
+
+      ###
+      # determine_targets:
+      # for a given SalsaTigerSentence,
+      # determine all targets,
+      # each as a _single_ main terminal node
+      #
+      # We need a single terminal node in order
+      # to compute the context window
+      #
+      # returns:
+      #  hash: target_IDs -> list of senses
+      #   where target_IDs is a pair [list of terminal IDs, main terminal ID]
+      #
+      #  where a sense is represented as a hash:
+      #  "sense": sense, a string
+      #  "obj":   FrameNode object
+      #  "all_targets": list of node IDs, may comprise more than a single node
+      #  "lex":   lemma, or multiword expression in canonical form
+      #  "sid": sentence ID
+      def determine_targets(sent)
+        raise "overwrite me"
+      end
+
+      ##
+      # returns a list of lemma-pos combined strings
+      def get_lemmas
+        return @targets.keys
+      end
+
+      ##
+      # access to lemmas and POS, returns a list of pairs [lemma, pos] (string*string)
+      def get_lemma_pos
+        @targets.keys.map { |lemmapos| ::Shalmaneser::Fred.fred_lemmapos_separate(lemmapos) }
+      end
+
+      ##
+      # access to senses
+      def get_senses(lemmapos) # string, result of fred_lemmapos_combine
+        @targets[lemmapos] ? @targets[lemmapos] : []
+      end
+
+      ##
+      # write file
+      def done_reading_targets
+        begin
+          file = FileZipped.new(@dir + "targets.txt.gz", "w")
+        rescue
+          $stderr.puts "Error: Could not write file #{@dir}targets.txt.gz"
+          exit 1
+        end
+
+        @targets.each_pair { |lemma, senses|
+          file.puts "LEMMA #{lemma} SENSES "+ senses.join(" ")
+        }
+
+        file.close
+      end
+
+      ###############################
+      protected
+
+      ##
+      # record: record occurrence of a lemma/sense pair
+      # <@targets> data structure
+      def record(target_info)
+        lemmapos = ::Shalmaneser::Fred.fred_lemmapos_combine(target_info["lex"], target_info["pos"])
+        unless @targets[lemmapos]
+          @targets[lemmapos] = []
+        end
+
+        unless @targets[lemmapos].include? target_info["sense"]
+          @targets[lemmapos] << target_info["sense"]
+        end
+      end
+    end
+  end
+end

data/lib/fred/toggle_var.rb

@@ -0,0 +1,61 @@
+module Shalmaneser
+  module Fred
+    ################
+    # ToggleVar:
+    # keeps a single toggle variable,
+    # and has a method that sets this toggle variable to a given value
+    # in the experiment file data structure.
+    class ToggleVar
+      attr_reader :var_name
+
+      def initialize(string, # part of value of --slide parameter, which has been split at :
+                     exp)    # FredConfigData object
+
+        if string =~ /^feature_dim=(\w+)$/
+          # feature dimension
+
+          @exp_lhs = "feature_dim"
+          @exp_rhs = $1
+          @unset_at_false = true # for false, un-set list valued parameter in set_value_to()
+          @var_name = "feature_dim #{@exp_rhs}"
+
+          unless ["word", "lemma", "pos", "ne"].include? @exp_rhs
+            raise "Unknown feature dimension "+ @exp_rhs
+          end
+
+        else
+          # normal variable
+          unless exp.get_type(string) == "bool"
+            raise "Unknown value in --toggle: "+ string
+          end
+
+          if ["use_fn_gf", "window_size"].include? string
+            raise "Sorry, cannot toggle #{string}, since this variable takes its effect during featurization."
+          end
+
+          @exp_lhs = string
+          @exp_rhs = "REPLACEME"
+          @unset_at_false = false # for false, set parameter to false in set_value_to
+          @var_name = @exp_lhs
+        end
+      end
+
+      ###
+      # set the value of my toggle variable to the given boolean
+      # in the given experiment file data structure.
+      #
+      # returns a descriptive text of the current setting
+      def set_value_to(boolean, # true, false
+                       exp)     # FredConfigData object
+
+        if @unset_at_false and not(boolean)
+          exp.unset_list_entry(@exp_lhs, @exp_rhs)
+        else
+          exp.set_entry(@exp_lhs, @exp_rhs.sub(/REPLACEME/, boolean.to_s))
+        end
+
+        return @var_name + "=" + boolean.to_s
+      end
+    end
+  end
+end

data/lib/fred/word_lemma_pos_ne.rb

@@ -0,0 +1,51 @@
+require 'salsa_tiger_xml/salsa_tiger_xml_helper'
+
+########################################
+# given a SynNode object representing a terminal,
+# return:
+# - the word
+# - the lemma
+# - the part of speech
+# - the named entity (if any)
+#
+# as a tuple
+#
+# WARNING: word and lemma are turned to lowercase
+module Shalmaneser
+module Fred
+module WordLemmaPosNe
+  # @param syn_obj [SynNode]
+  # @param i [SynInterpreter]
+  def word_lemma_pos_ne(syn_obj, i)
+    unless syn_obj.is_terminal?
+      $stderr.puts "Featurization warning: unexpectedly received non-terminal"
+      return [nil, nil, nil, nil]
+    end
+
+    word = syn_obj.word
+    if word
+      word.downcase!
+    end
+
+    lemma = i.lemma_backoff(syn_obj)
+    if lemma and STXML::SalsaTigerXMLHelper.unescape(lemma) == "<unknown>"
+      lemma = nil
+    end
+
+    if lemma
+      lemma.downcase!
+    end
+
+    pos = syn_obj.part_of_speech
+
+    ne = syn_obj.get_attribute("ne")
+
+    unless ne
+      ne = syn_obj.get_attribute("headof_ne")
+    end
+
+    [word, lemma, pos, ne]
+  end
+end
+end
+end

data/lib/fred/write_features_binary.rb

@@ -0,0 +1,95 @@
+###
+# Features for binary classifiers
+require 'fred/FredConventions' # !
+
+module Shalmaneser
+  module Fred
+    class WriteFeaturesBinary
+      def initialize(lemma,
+                     exp,
+                     dataset,
+                     feature_dir)
+        @dir = feature_dir
+        @lemma = lemma
+        @feature_dir = feature_dir
+
+        @negsense = exp.get("negsense")
+        unless @negsense
+          @negsense = "NONE"
+        end
+
+        # files: sense-> filename
+        @files = {}
+
+        # keep all instances such that, when a new sense comes around,
+        # we can write them for that sense
+        @instances = []
+      end
+
+
+      def write_instance(features, senses)
+        # sense we haven't seen before? Then we need to
+        # write the whole featurization file for that new sense
+        check_for_presence_of_senses(senses)
+
+        # write this new instance for all senses
+        @files.each_key { |sense_of_file|
+          write_to_sensefile(features, senses, sense_of_file)
+        }
+
+        # store instance in case another sense crops up later
+        @instances << [features, senses]
+      end
+
+
+      ###
+      def close
+        @files.each_value { |f| f.close }
+      end
+
+      ######
+      private
+
+      def check_for_presence_of_senses(senses)
+        senses.each { |sense|
+          # do we have a sense file for this sense?
+          unless @files[sense]
+            # open new file for this sense
+            @files[sense] = File.new(@feature_dir + ::Shalmaneser::Fred.fred_feature_filename(@lemma, sense, true), "w")
+            # filename = @feature_dir + Fred.fred_feature_filename(@lemma, sense, true)
+            # $stderr.puts "Starting new feature file #{filename}"
+
+            # and re-write all previous instances for it
+            @instances.each { |prev_features, prev_senses|
+              write_to_sensefile(prev_features, prev_senses,
+                                 sense)
+            }
+          end
+        }
+      end
+
+      ###
+      def write_to_sensefile(features, senses,
+                             sense_of_file)
+        # file to write to
+        f = @files[sense_of_file]
+
+        # print features
+        f.print features.map { |x|
+          x.to_s.gsub(/,/, "COMMA")
+        }.join(",")
+
+        f.print ","
+
+        # binarize target class
+        if senses.include? sense_of_file
+          # $stderr.puts "writing POS #{sense_of_file}"
+          f.puts sense_of_file.to_s
+        else
+          # $stderr.puts "writing NEG #{negsense}"
+          f.puts @negsense
+        end
+      end
+    end
+  end
+end

data/lib/fred/write_features_nary.rb

@@ -0,0 +1,51 @@
+require 'fred/FredConventions' # !
+module Shalmaneser
+  module Fred
+    ##############
+    # write features,
+    # either lemma-wise
+    # or lemma+sense-wise
+    # if lemma+sense-wise, write as binary classifier,
+    # i.e. map the target senses
+    #
+    # Use Delegator.
+    ###
+    # Features for N-ary classifiers
+    class WriteFeaturesNary
+      def initialize(lemma,
+                     exp,
+                     dataset,
+                     feature_dir)
+
+        @filename = feature_dir + ::Shalmaneser::Fred.fred_feature_filename(lemma)
+        @f = File.new(@filename, "w")
+        @handle_multilabel = exp.get("handle_multilabel")
+      end
+
+      def write_instance(features, senses)
+        @f.print features.map { |x|
+          x.to_s.gsub(/,/, "COMMA").gsub(/;/, "SEMICOLON")
+        }.join(",")
+
+        # possibly more than one sense? then use semicolon to separate
+        if @handle_multilabel == "keep"
+          # possibly more than one sense:
+          # separate by semicolon,
+          # and hope that the classifier knows this
+          @f.print ";"
+          @f.puts senses.map {|x|
+            x.to_s.gsub(/,/, "COMMA").gsub(/;/, "SEMICOLON")
+          }.join(",")
+        else
+          # one sense: just separate by comma
+          @f.print ","
+          @f.puts senses.first.to_s.gsub(/,/, "COMMA").gsub(/;/, "SEMICOLON")
+        end
+      end
+
+      def close
+        @f.close
+      end
+    end
+  end
+end

data/lib/fred/write_features_nary_or_binary.rb

@@ -0,0 +1,51 @@
+require "delegate"
+require 'fred/FredConventions'
+require 'fred/write_features_binary'
+require 'fred/write_features_nary'
+
+module Shalmaneser
+  module Fred
+    ########
+    # class writing features:
+    # delegating to either a binary or an n-ary writer
+    class WriteFeaturesNaryOrBinary < SimpleDelegator
+      ###
+      def initialize(lemma,
+                     exp,
+                     dataset)
+        feature_dir = WriteFeaturesNaryOrBinary.feature_dir(exp, dataset, "new")
+        if exp.get("binary_classifiers")
+          # binary classifiers
+          # $stderr.puts "Writing binary feature data."
+
+          # delegate writing to the binary feature writer
+          @writer = WriteFeaturesBinary.new(lemma, exp, dataset, feature_dir)
+          super(@writer)
+
+        else
+          # n-ary classifiers
+          # $stderr.puts "Writing n-ary feature data."
+
+          # delegate writing to the n-ary feature writer
+          @writer = WriteFeaturesNary.new(lemma, exp, dataset, feature_dir)
+          super(@writer)
+        end
+      end
+
+      def self.feature_dir(exp, dataset, mode = "existing")
+        ::Shalmaneser::Fred.fred_dirname(exp, dataset, "features", mode)
+      end
+
+      ###
+      def self.remove_files(exp, dataset)
+        feature_dir = WriteFeaturesNaryOrBinary.feature_dir(exp, dataset, "new")
+
+        Dir[feature_dir + ::Shalmaneser::Fred.fred_feature_filename("*")].each do |filename|
+          if File.exist? filename
+            File.delete(filename)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/shalmaneser/fred.rb

@@ -0,0 +1 @@
+# A dummy file to require for now.