shalmaneser-fred 1.2.0.rc4 → 1.2.rc5

data/lib/fred/fred_parameters.rb

@@ -0,0 +1,190 @@
+# FredParameters
+# Katrin Erk, April 05
+#
+# Frame disambiguation system:
+#  test different values for system parameters,
+#  construct text and graphical output
+
+# Salsa packages
+require 'fred/plot_and_r_eval'
+require 'fred/FredConventions' # !
+require 'fred/fred_split'
+require 'fred/fred_train'
+require 'fred/fred_test'
+require 'fred/fred_eval'
+require 'fred/toggle_var'
+require 'fred/slide_var'
+
+require 'logging'
+
+module Shalmaneser
+  module Fred
+    ##########################################
+    # main class of this package:
+    # try out different values for system parameters,
+    # and record the result.
+    #
+    # One value can be a slide variable, taking on several numerical values.
+    # 0 or more values can be toggle variables, taking on the values true and false.
+    # @todo AB: Reintroduce this task!!!
+    class FredParameters
+      #####
+      # @param [FredConfigData] exp
+      # @param [Hash] options hash: runtime option name (string) => value(string)
+      def initialize(exp, options)
+        @exp = exp
+
+        # evaluate runtime options:
+        # record the slide variable (if any) plus all toggle variables
+        @slide = SlideVar.new("", @exp)
+        @toggle = []
+        @outfile_prefix = "fred_parameters"
+
+        options.each_pair do |opt, arg|
+          case opt
+          when "--slide"
+            @slide = SlideVar.new(arg, @exp)
+
+          when "--toggle"
+            arg.split(":").each { |toggle_var|
+              @toggle << ToggleVar.new(toggle_var, @exp)
+            }
+
+          when "--output_to"
+            @outfile_prefix = arg
+
+          else
+            # case of unknown arguments has been dealt with by fred.rb
+          end
+        end
+      end
+
+      ####
+      def compute
+        LOGGER.info "Fred parameter exploration, experiment #{@exp.get("experiment_ID")}"
+        ##
+        # make a split of the training data
+        begin
+          feature_dir = ::Shalmaneser::Fred.fred_dirname(@exp, "train", "features")
+        rescue
+          $stderr.puts "To experiment with system parameters, please first featurize training data."
+          exit 1
+        end
+        # make new split ID from system time, and make a split with 80% training, 20% test data
+        splitID = Time.new.to_f.to_s
+        task_obj = FredSplit.new(@exp,
+                                 { "--logID" => splitID,
+                                   "--trainpercent" => "80",
+                                 },
+                                 true  # ignore unambiguous
+                                )
+        task_obj.compute
+
+        ##
+        # start recording results:
+
+        # text output file
+        begin
+          textout_file = File.new(@outfile_prefix + ".txt", "w")
+        rescue
+          raise "Could not write to output file #{@outfile_prefix}.txt"
+        end
+
+        # values_to_score: hash toggle_values_descr(string) =>
+        #                        hash slide_value(float) => score(float)
+        values_to_score = {}
+
+        # max_score: float, describing maximum score achieved
+        # max_setting: string, describing values for maximum score
+        max_score = 0.0
+        max_setting = ""
+
+        ##
+        # for each value of the toggle variables
+        0.upto(2**@toggle.length - 1) { |binary|
+
+          textout_line = ""
+
+          # re-set toggle values according to 'binary':
+          @toggle.each_index { |i|
+            # if the i-th bit is set in binary, set this
+            # boolean to true, else set it to false
+            if (binary & (2**i)) > 0
+              textout_line << @toggle[i].set_value_to(true, @exp) + " "
+            else
+              textout_line << @toggle[i].set_value_to(false, @exp) + " "
+            end
+          }
+
+          values_to_score[textout_line] = {}
+
+
+          ##
+          # for each value of the slide variable
+          @slide.each_slide_value(@exp) { |slide_value, slide_value_description|
+
+            ##
+            # progress bar
+            $stderr.puts "Parameter exploration: #{textout_line} #{slide_value_description}"
+
+            ##
+            # @exp has been modified to fit the current values of the
+            # slide and toggle variables.
+            # Now train, test, evaluate on the split we have constructed
+            task_obj = FredTrain.new(@exp, { "--logID" => splitID})
+            task_obj.compute
+            task_obj = FredTest.new(@exp,
+                                    { "--logID" => splitID,
+                                      "--nooutput"=> true
+                                    })
+            task_obj.compute
+            task_obj = FredEval.new(@exp, {"--logID" => splitID})
+            task_obj.compute(false)  # don't print evaluation results to file
+
+            ##
+            # read off F-score, record result
+            score = task_obj.f
+
+            textout_file.puts textout_line + slide_value_description + " : " + score.to_s
+            textout_file.flush
+            values_to_score[textout_line][slide_value] = score
+
+            if score > max_score
+              max_score = score
+              max_setting = textout_line + slide_value_description + " : " + score.to_s
+            end
+          }
+        }
+
+        ##
+        # remove split
+        FredSplit.remove_split(@exp, splitID)
+
+        ##
+        # plot outcome, report overall maximum
+
+        unless @slide.empty?
+          # gnuplot output only if some slide variable has been used
+          title = "Exploring #{@slide.var_name}, " + @toggle.map { |toggle_obj| toggle_obj.var_name }.join(", ")
+          PlotAndREval.gnuplot_direct(values_to_score,
+                                      title,
+                                      @slide.var_name,
+                                      "F-score",
+                                      @outfile_prefix + ".ps")
+        end
+
+        $stderr.puts "Parameter exploration finished."
+        $stderr.puts "Text output was written to #{@outfile_prefix}.txt"
+        unless @slide.empty?
+          $stderr.puts "Gnuplot output was written to #{@outfile_prefix}.ps"
+        end
+
+        unless max_setting.empty?
+          $stderr.puts "-----------------------"
+          $stderr.puts "Maximum score:"
+          $stderr.puts max_setting
+        end
+      end
+    end
+  end
+end

data/lib/fred/fred_split.rb

@@ -0,0 +1,86 @@
+# FredSplit
+# Katrin Erk April 05
+#
+# Frame disambiguation system:
+# make random split of the training data
+#
+# The split is computed on the basis of the Fred format
+# feature data.
+# The split is recorded in a separate split directory
+# with a very simple system:
+# - one file per feature file, same filename
+# - one line per instance line in feature file
+# - entry in that line is either 'train' or 'test'
+
+# Fred packages
+require 'fred/fred_split_pkg'
+require 'logging'
+
+module Shalmaneser
+  module Fred
+    class FredSplit
+      # @param [FredConfigData] exp
+      # @param [String] split_id
+      def self.remove_split(exp, split_id)
+        FredSplitPkg.remove_split(exp, split_id)
+      end
+
+      ###
+      # new
+      #
+      # evaluate runtime options and announce the task
+      def initialize(exp_obj, # FredConfigData object
+                     options, # hash: runtime option name (string) => value(string)
+                     ignore_unambiguous = false)
+
+        @exp = exp_obj
+        @ignore_unambiguous = ignore_unambiguous
+
+        # evaluate runtime options
+        @split_id = nil
+        @trainpercent = 0.9
+
+        options.each_pair do |opt, arg|
+          case opt
+          when "--logID"
+            @split_id = arg
+
+          # @ todo AB: Should be prepared in the ConfigData/OptParser.
+          when "--trainpercent"
+            @trainpercent = arg.to_f / 100.0
+          end
+        end
+
+        # sanity check: need a log ID
+        # @todo AB: Move it to OptParser
+        if @split_id.nil?
+          raise "I need a log ID, parameter --logID"
+        end
+
+        # @todo AB: Move it to OptParser
+        if @trainpercent <= 0.0 or @trainpercent >= 1.0
+          raise "Training percentage needs to be between 1 and 99. I got "+
+                (@trainpercent * 100.0).to_i.to_s
+        end
+
+        ##
+        # make a splitting object
+        @split_obj = FredSplitPkg.new(@exp)
+      end
+
+      ###
+      # compute
+      #
+      # do the splitting
+      def compute
+        # announce the task
+        LOGGER.info "Fred experiment #{@exp.get("experiment_ID")}: "\
+                    "Making split, using #{(@trainpercent * 100.0).to_i}% as training data."
+
+        FredSplitPkg.remove_split(@exp, @split_id)
+        @split_obj.make_new_split(@split_id, @trainpercent,
+                                  @ignore_unambiguous)
+      end
+    end
+  end
+end

data/lib/fred/fred_split_pkg.rb

@@ -0,0 +1,189 @@
+require "tempfile"
+require 'fileutils'
+require 'fred/targets'
+require 'fred/FredConventions' # !
+require 'fred/fred_error'
+
+require 'logging'
+
+module Shalmaneser
+  module Fred
+    # splitting package for WSD:
+    # compute a split for feature files (one item a line, CSV),
+    # and apply pre-computed split
+    # to produce new feature files accordingly
+    class FredSplitPkg
+      ###
+      # remove an old split
+      # @param [FredConfigData] exp object
+      # @param [String] split_id
+      def self.remove_split(exp, split_id)
+        begin
+          # split_dir = FredSplitPkg.split_dir(exp, split_id, "new")
+          split_dir = ::Shalmaneser::Fred.fred_dirname(exp, 'split', split_id, 'new')
+        rescue
+          # no split to be removed
+          return
+        end
+
+        FileUtils.rm_rf(split_dir)
+      end
+
+      def initialize(exp)
+        @exp = exp
+      end
+
+      # make a new split
+      def make_new_split(split_id,  # string: ID
+                         trainpercent, # float: percentage training data
+                         ignore_unambiguous = false)
+
+        # where to store the split?
+        split_dir = split_dir(@exp, split_id, "new")
+
+        lemmas_and_senses = Targets.new(@exp, nil, "r")
+        unless lemmas_and_senses.targets_okay
+          # error during initialization
+          raise FredError, "FredSplitPkg: Error: Could not read list of known targets, bailing out."
+        end
+
+        # Iterate through lemmas,
+        # split training feature files.
+        #
+        # Do the split only once per lemma,
+        # even if we have sense-specific feature files
+        feature_dir = ::Shalmaneser::Fred.fred_dirname(@exp, "train", "features")
+
+        lemmas_and_senses.get_lemmas.each { |lemma|
+          # construct split file
+          splitfilename = split_dir + fred_split_filename(lemma)
+          begin
+            splitfile = File.new(splitfilename, "w")
+          rescue
+            raise "Error: Couldn't write to file " + splitfilename
+          end
+
+          # find lemma-specific  feature file
+
+          filename = feature_dir + ::Shalmaneser::Fred.fred_feature_filename(lemma)
+
+          unless File.exist?(filename)
+            # try lemma+sense-specific feature file
+            file_pattern = ::Shalmaneser::Fred.fred_feature_filename(lemma, "*", true)
+            filename = Dir[feature_dir + file_pattern].first
+
+            unless filename
+              # no lemma+sense-specific feature file
+              LOGGER.warn "Warning: split: no feature file found for #{lemma}, skipping."
+              splitfile.close
+              next
+            end
+          end
+
+          # open feature file for reading
+          begin
+            file = File.new(filename)
+          rescue
+            raise "Couldn't read feature file " + filename
+          end
+
+          if ignore_unambiguous and
+            lemmas_and_senses.get_senses(lemma).length < 2
+            # unambiguous: ignore
+
+            while file.gets
+              splitfile.puts "ignore"
+            end
+
+          else
+            # read from feature file, classify at random
+            # as train or test,
+            # write result to splitfile
+
+            while file.gets
+              if rand < trainpercent
+                splitfile.puts "train"
+              else
+                splitfile.puts "test"
+              end
+            end
+          end
+
+          splitfile.close
+        }
+      end
+
+      # change feature files according to
+      # pre-computed split
+      #
+      #
+      # returns: tempfile containing featurized items,
+      # according to split,
+      # or nil if the split file wouldn't contain any data
+      def apply_split(filename, # feature file
+                      lemma,    # string: lemma that filename is about
+                      dataset,  # string: train, test
+                      split_id) # string: split ID
+
+        split_filename = split_dir(@exp, split_id) + fred_split_filename(lemma)
+
+        # read feature file and split file at the same time
+        # write to tempfile.
+        f_feat = File.new(filename)
+        f_split = File.new(split_filename)
+        f_out = Tempfile.new("fred_split")
+
+        num_yes = 0
+
+        f_feat.each do |line|
+          begin
+            split_part = f_split.readline.chomp
+          rescue
+            $stderr.puts "FredSplit error: split file too short."
+            $stderr.puts "skipping rest of featurization data."
+            $stderr.puts "Split file: #{split_filename}"
+            $stderr.puts "Feature file: #{filename}"
+            # @todo AB: FIXME
+            raise "HIER"
+
+            f_out.close
+            if num_yes > 0
+              return f_out
+            else
+              return nil
+            end
+          end
+
+          if split_part == dataset
+            # write training data, and this item is in the training
+            # part of the split,
+            # or write test data, and item is in test part
+            f_out.puts line
+            num_yes += 1
+          end
+        end
+
+        f_out.close
+        f_feat.close
+        f_split.close
+
+        if num_yes > 0
+          return f_out
+        else
+          return nil
+        end
+
+      end
+
+      private
+
+      def fred_split_filename(lemma)
+        "fred.split.#{lemma}"
+      end
+
+      def split_dir(exp, split_id, mode = "existing")
+        ::Shalmaneser::Fred.fred_dirname(exp, "split", split_id, mode)
+      end
+    end
+  end
+end