shalmaneser-fred 1.2.0.rc4 → 1.2.rc5

data/lib/fred/context_provider.rb

@@ -0,0 +1,55 @@
+require 'fred/abstract_context_provider'
+require 'salsa_tiger_xml/salsa_tiger_sentence'
+require 'salsa_tiger_xml/file_parts_parser'
+
+module Shalmaneser
+  module Fred
+
+    ####################################
+    # ContextProvider:
+    # subclass of AbstractContextProvider
+    # that assumes that the input text is a contiguous text
+    # and computes the context accordingly.
+    class ContextProvider < AbstractContextProvider
+      ###
+      # each_window: iterator
+      #
+      # given a directory with Salsa/Tiger XML data,
+      # iterate through the data,
+      # yielding each target word as soon as its context window is filled
+      # (or the last file is at an end)
+      def each_window(dir) # string: directory containing Salsa/Tiger XML data
+
+        # iterate through files in the directory.
+        # Try sorting filenames numerically, since this is
+        # what frprep mostly does with filenames
+        Dir[dir + "*.xml"].sort { |a, b|
+          File.basename(a, ".xml").to_i <=> File.basename(b, ".xml").to_i
+        }.each do |filename|
+          # progress bar
+          if @exp.get("verbose")
+            $stderr.puts "Featurizing #{File.basename(filename)}"
+          end
+          f = STXML::FilePartsParser.new(filename)
+          each_window_for_file(f) { |result| yield result }
+        end
+        # and empty the context array
+        each_remaining_target { |result| yield result }
+      end
+
+      ##################################
+      protected
+
+      ######################
+      # each_window_for_file: iterator
+      # same as each_window, but only for a single file
+      # (to be called from each_window())
+      def each_window_for_file(fpp) # FilePartsParser object: Salsa/Tiger XMl data
+        fpp.scan_s { |sent_string|
+          sent = STXML::SalsaTigerSentence.new(sent_string)
+          each_window_for_sent(sent) { |result| yield result }
+        }
+      end
+    end
+  end
+end

data/lib/fred/feature_extractors/fred_context_feature_extractor.rb

@@ -0,0 +1,48 @@
+require_relative 'fred_feature_extractor'
+
+module Shalmaneser
+  module Fred
+    #####
+    # context feature
+    class FredContextFeatureExtractor < FredFeatureExtractor
+
+      FredContextFeatureExtractor.announce_me
+
+      def self.feature_name
+        'context'
+      end
+
+      ###
+      def initialize(exp)
+        super(exp)
+
+        # cxsizes: list of context sizes chosen as features,
+        # encoded in metafeature labels
+        # written in a hash for fast access
+        @cxsizes = {}
+        @exp.get_lf("feature", "context").each do |cxsize|
+          @cxsizes["CX" + cxsize.to_s] = true
+        end
+      end
+
+      ###
+      def each_feature(feature_hash)
+        # grf#word#lemma#pos#ne
+        lemma_index = 2
+
+        feature_hash.each do |ftype, fvalues|
+          if @cxsizes[ftype]
+            # this is a context feature of a size chosen
+            # by the user for featurization
+
+            fvalues.each do |f|
+              next if f =~ /#####/
+              yield ftype + f.split("#")[lemma_index]
+            end
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/fred/feature_extractors/fred_context_pos_feature_extractor.rb

@@ -0,0 +1,48 @@
+require_relative 'fred_feature_extractor'
+
+module Shalmaneser
+  module Fred
+  #####
+    # context feature: POS separately, small contexts only
+    class FredContextPOSFeatureExtractor < FredFeatureExtractor
+      FredContextPOSFeatureExtractor.announce_me
+
+      def self.feature_name
+        'context_pos'
+      end
+
+      ###
+      def initialize(exp)
+        super(exp)
+
+        # cxsizes: list of context sizes chosen as features,
+        # encoded in metafeature labels
+        # written in a hash for fast access
+        @cxsizes = {}
+        @exp.get_lf("feature", "context").each do |cxsize|
+          if cxsize <= 10
+            @cxsizes["CX" + cxsize.to_s] = true
+          end
+        end
+
+        if @cxsizes.empty?
+          $stderr.puts "context_pos feature warning: will not be computed"
+          $stderr.puts "as there is no context of size <= 10"
+        end
+      end
+
+      ###
+      def each_feature(feature_hash)
+        # word#lemma#pos#ne
+        pos_index = 2
+        feature_hash.each do |ftype, fvalues|
+          if @cxsizes[ftype]
+            # this is a context feature of a size chosen
+            # by the user for featurization
+            fvalues.each { |f| yield "POS" + ftype + f.split("#")[pos_index] }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fred/feature_extractors/fred_feature_extractor.rb

@@ -0,0 +1,50 @@
+require 'fred/fred_feature_info'
+
+module Shalmaneser
+  module Fred
+    ##################################3
+    class FredFeatureExtractor
+      ###
+      # feature name:
+      # name by which you choose this feature
+      # in the experiment file
+      def FredFeatureExtractor.feature_name
+        raise "Overwrite me."
+      end
+
+      ###
+      # initialize with Fred experiment file object
+      def initialize(exp)
+        @exp = exp
+      end
+
+      ###
+      # compute features from meta-features
+      #
+      # argument: hash
+      # metafeature_label -> metafeatures
+      #  string -> array:string
+      #
+      # yields each feature as a string
+      def each_feature(feature_hash)
+        raise "overwrite me"
+      end
+
+      ######
+      protected
+
+      def FredFeatureExtractor.announce_me
+        # This check is obsolete since we require FeatureInfo.
+        # AB: In 1.9 constants are symbols.
+        if Module.constants.include?("FredFeatureInfo") or Module.constants.include?(:FredFeatureInfo)
+          # yup, we have a class to which we can announce ourselves
+          FredFeatureInfo.add_feature(self)
+        else
+          # no interface collector class
+          #      $stderr.puts "Feature #{self.name()} not announced: no RosyFeatureInfo."
+        end
+      end
+
+    end
+  end
+end

data/lib/fred/feature_extractors/fred_ngram_feature_extractor.rb

@@ -0,0 +1,65 @@
+require_relative 'fred_feature_extractor'
+
+module Shalmaneser
+  module Fred
+    #####
+    # bigram/trigram feature
+    class FredNgramFeatureExtractor < FredFeatureExtractor
+
+      FredNgramFeatureExtractor.announce_me
+
+      def self.feature_name
+        'ngram'
+      end
+
+      ###
+      def initialize(exp)
+        super(exp)
+
+        # cxsize: context size from which the ngram feature will be computed
+        # encoded in metafeature labels
+        # written in a hash for fast access
+        @cxsize = @exp.get_lf("feature", "context").detect do |cxsize|
+          cxsize >= 2
+        end
+
+        unless @cxsize
+          $stderr.puts "Warning: no context of size >= 2, so"
+          $stderr.puts "no ngram feature computed."
+        end
+      end
+
+      ###
+      def each_feature(feature_hash)
+        # word#lemma#pos#ne
+        lemma_index = 1
+        pos_index = 2
+
+        feature_hash.each do |ftype, fvalues|
+          if ftype == "CX" + @cxsize.to_s
+            # compute the ngram features from this context
+            # |fvalues| = 2*cxsize, that is, cxsize describes
+            # the length of a one-sided context window
+            # the bigram of features around the target
+            # concerns fvalues[cxsize-1] and fvalues[cxsize]
+            # the trigram of two words before, one word after includes
+            # fvalues[cxsize-2], fvalues[cxsize-1] and fvalues[cxsize]
+
+            [
+              [[-1, 0], "BLEM", lemma_index], # bigram of lemmas
+              [[-1, 0], "BPOS", pos_index],   # bigram of POSs
+              [[-2, -1, 0], "TLEM", lemma_index], # trigram of lemmas
+              [[-2, -1, 0], "TPOS", pos_index] # trigram of POSs
+            ].each do |f_indices, label, subindex|
+              fs = f_indices.map { |i| fvalues[@cxsize+i] }.compact
+              if fs.length == f_indices.length
+                # we successfully extracted entries for all the given indices
+                yield label + fs.map { |f| f.split("#")[subindex] }.join
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fred/feature_extractors/fred_syn_feature_extractor.rb

@@ -0,0 +1,33 @@
+require_relative 'fred_feature_extractor'
+
+module Shalmaneser
+  module Fred
+    #####
+    # syntax feature
+    class FredSynFeatureExtractor < FredFeatureExtractor
+
+      FredSynFeatureExtractor.announce_me
+
+      def self.feature_name
+        'syntax'
+      end
+
+      ###
+      def each_feature(feature_hash)
+        feature_hash.each do |ftype, fvalues|
+          case ftype
+          when "CH", "PA"
+            grf_index = 0
+            fvalues.each { |f| yield ftype + f.split("#")[grf_index] }
+          when "SI"
+            # parentlemma#grf#word#lemma#pos#ne
+            grf_index = 1
+            fvalues.each { |f| yield ftype + f.split("#")[grf_index] }
+          else
+            # not a syntactic metafeature
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fred/feature_extractors/fred_synsem_feature_extractor.rb

@@ -0,0 +1,32 @@
+require_relative 'fred_feature_extractor'
+
+module Shalmaneser
+  module Fred
+    #####
+    # syntax-plus-headword feature
+    class FredSynsemFeatureExtractor < FredFeatureExtractor
+
+      FredSynsemFeatureExtractor.announce_me
+
+      def self.feature_name
+        'synsem'
+      end
+
+      def each_feature(feature_hash)
+        feature_hash.each do |ftype, fvalues|
+          case ftype
+          when "CH", "PA"
+            # grf#word#lemma#pos#ne
+            fvalues.each { |f| yield ftype + "SEM" + f }
+          when "SI"
+            # parentlemma#grf#word#lemma#pos#ne
+            # remove parent lemma
+            fvalues.each { |f| yield ftype + "SEM" + f.split("#")[1..-1].join("#") }
+          else
+            # not a syntax feature
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fred/feature_extractors.rb

@@ -0,0 +1,5 @@
+require 'fred/feature_extractors/fred_context_feature_extractor'
+require 'fred/feature_extractors/fred_context_pos_feature_extractor'
+require 'fred/feature_extractors/fred_ngram_feature_extractor'
+require 'fred/feature_extractors/fred_syn_feature_extractor'
+require 'fred/feature_extractors/fred_synsem_feature_extractor'

data/lib/fred/file_zipped.rb

@@ -0,0 +1,43 @@
+require 'fred/fred_error'
+require 'logging'
+
+module Shalmaneser
+  module Fred
+    class FileZipped
+
+      # @todo Rewrite this class using stdlib.
+      # @return [IO]
+      # @param filename [String]
+      # @param mode [String]
+      # @raise [FredError] if some external error occured
+      def self.new(filename, mode = 'r')
+        # escape characters in the filename that
+        # would make the shell hiccup on the command
+        filename = filename.gsub(/([();:!?'`])/, 'XXSLASHXX\1')
+        filename = filename.gsub(/XXSLASHXX/, "\\")
+
+        unless %w{r w a}.include?(mode)
+          LOGGER.fatal "FileZipped error: only modes r, w, a are implemented. "\
+                       "I got: #{mode}."
+          raise FredError
+        end
+
+        begin
+          case mode
+          when "r"
+            unless File.exist?(filename)
+              raise FredError, 'File does not exist!'
+            end
+            return IO.popen("gunzip -c #{filename}")
+          when "w"
+            return IO.popen("gzip > #{filename}", "w")
+          when "a"
+            return IO.popen("gzip >> #{filename}", "w")
+          end
+        rescue => e
+          raise FredError, "Error opening file #{filename}.", e
+        end
+      end
+    end
+  end
+end

data/lib/fred/find_all_targets.rb

@@ -0,0 +1,94 @@
+require 'fred/targets'
+
+module Shalmaneser
+  module Fred
+    ########################################
+    class FindAllTargets < Targets
+      ###
+      # determine_targets:
+      # use all known lemmas, minus stopwords
+      def initialize(exp,
+                     interpreter_class)
+        # read target info from file
+        super(exp, interpreter_class, "r")
+        @training_lemmapos_pairs = get_lemma_pos
+
+        get_senses(@training_lemmapos_pairs)
+        # list of words to exclude from assignment, for now
+        @stoplemmas = [
+          "have",
+          "do",
+          "be"
+          #      "make"
+        ]
+
+      end
+
+      ####
+      #
+      # returns:
+      #  hash: target_IDs -> list of senses
+      #   where target_IDs is a pair [list of terminal IDs, main terminal ID]
+      #
+      #  where a sense is represented as a hash:
+      #  "sense": sense, a string
+      #  "obj":   FrameNode object
+      #  "all_targets": list of node IDs, may comprise more than a single node
+      #  "lex":   lemma, or multiword expression in canonical form
+      #  "sid": sentence ID
+      def determine_targets(sent) #SalsaTigerSentence object
+        # map target IDs to list of senses, in our case always [ nil ]
+        # because we assume that the senses of the targets we point out
+        # are unknown
+        retv = {}
+        # iterate through terminals of the sentence, check for inclusion
+        # of their lemma in @training_lemmas
+        sent.each_terminal { |node|
+          # we know this lemma from the training data,
+          # and it is not an auxiliary,
+          # and it is not in the stopword list
+          # and the node does not represent a preposition
+
+          ### modified by ines, 17.10.2008
+          lemma = @interpreter_class.lemma_backoff(node)
+          pos = @interpreter_class.category(node)
+
+          #     print "lemma ", lemma, " pos ", pos, "\n"
+          #      reg = /\.[ANV]/
+          #      if !reg.match(lemma)
+          #        if /verb/.match(pos)
+          #          lemma = lemma + ".V"
+          #        elsif /noun/.match(pos)
+          #          lemma = lemma + ".N"
+          #        elsif /adj/.match(pos)
+          #          lemma = lemma + ".A"
+          #        end
+          #        print "LEMMA ", lemma, " POS ", pos, "\n"
+          #      end
+
+          if (@training_lemmapos_pairs.include? [lemma, pos] and
+              not(@interpreter_class.auxiliary?(node)) and
+              not(@stoplemmas.include? lemma) and
+              not(pos == "prep"))
+            key = [ [ node.id ], node.id ]
+
+            # take this as a target.
+            retv[ key ] = [
+              {
+                "sense" => nil,
+                "obj" => nil,
+                "all_targets" => [ node.id ],
+                "lex" => lemma,
+                "pos" => pos,
+                "sid" => sent.id
+              } ]
+            # no recording of target info,
+            # since we haven't determined anything new
+          end
+        }
+
+        return retv
+      end
+    end
+  end
+end

data/lib/fred/find_targets_from_frames.rb

@@ -0,0 +1,92 @@
+require 'fred/targets'
+
+module Shalmaneser
+  module Fred
+    ########################################
+    class FindTargetsFromFrames < Targets
+      ###
+      # determine_targets:
+      # use existing frames to find targets
+      #
+      # returns:
+      #  hash: target_IDs -> list of senses
+      #   where target_IDs is a pair [list of terminal IDs, main terminal ID]
+      #
+      #  where a sense is represented as a hash:
+      #  "sense": sense, a string
+      #  "obj":   FrameNode object
+      #  "all_targets": list of node IDs, may comprise more than a single node
+      #  "lex":   lemma, or multiword expression in canonical form
+      #  "sid": sentence ID
+      def determine_targets(st_sent) #SalsaTigerSentence object
+        retv = {}
+        st_sent.each_frame { |frame_obj|
+          # instance-specific computation:
+          # target and target positions
+          # WARNING: at this moment, we are
+          # not considering true multiword targets for German.
+          # Remove the "no_mwe" parameter in main_node_of_expr
+          # to change this
+          term = nil
+          all_targets = nil
+          if frame_obj.target.nil? or frame_obj.target.children.empty?
+          # no target, nothing to record
+
+          elsif @exp.get("language") == "de"
+            # don't consider true multiword targets for German
+            all_targets = frame_obj.target.children
+            term = @interpreter_class.main_node_of_expr(all_targets, "no_mwe")
+
+          else
+            # for all other languages: try to figure out the head target word
+            # anyway
+            all_targets = frame_obj.target.children
+            term = @interpreter_class.main_node_of_expr(all_targets)
+          end
+
+          if term and term.is_splitword?
+            # don't use parts of a word as main node
+            term = term.parent
+          end
+          if term and term.is_terminal?
+            key = [all_targets.map { |t| t.id }, term.id]
+
+            unless retv[key]
+              retv[key] = []
+            end
+
+            pos = frame_obj.target.get_attribute("pos")
+            # gold POS available, may be in wrong form,
+            # i.e. not the same strings that @interpreter_class.category()
+            # would return
+            case pos
+            when /^[Vv]$/
+              pos = "verb"
+            when /^[Nn]$/
+              pos = "noun"
+            when /^[Aa]$/
+              pos = "adj"
+            when nil
+              pos = @interpreter_class.category(term)
+            end
+
+            target_info = {
+              "sense" => frame_obj.name,
+              "obj" => frame_obj,
+              "all_targets" => frame_obj.target.children.map { |ch| ch.id },
+              "lex" => frame_obj.target.get_attribute("lemma"),
+              "pos" => pos,
+              "sid" => st_sent.id
+            }
+            #print "lex ", frame_obj.target(), " und ",frame_obj.target().get_attribute("lemma"), "\n"
+            retv[key] << target_info
+            if @record_targets
+              record(target_info)
+            end
+          end
+        }
+        return retv
+      end
+    end
+  end
+end

data/lib/fred/fred.rb

@@ -1,47 +1,50 @@
 # AB: 2011-11-13
 # Initial import done, need to reimplement the whole interface.
-require 'fred/FredFeaturize'
-require 'fred/FredSplit'
-require 'fred/FredTrain'
-require 'fred/FredTest'
-require 'fred/FredEval'
+require 'fred/fred_featurize'
+require 'fred/fred_split'
+require 'fred/fred_train'
+require 'fred/fred_test'
+require 'fred/fred_eval'
+# Reintroduce this task.
+# require 'fred/fred_parameters'
 
-module Fred
-  class Fred
+require 'logging'
+require 'definitions'
 
-    def initialize(options)
-      @exp, @opts = options
-      @task = @opts['--task']
-    end
-    ##
-    # now perform the given task
+module Shalmaneser
+  module Fred
+    class Fred
+      def initialize(options)
+        @exp, @opts = options
+        @task = @opts['--task']
+      end
 
-    def assign
+      ##
+      # now perform the given task
+      def assign
+        # initialize task object
+        task = case @task
+               when "featurize"
+                 FredFeaturize.new(@exp, @opts)
+               when "refeaturize"
+                 FredFeaturize.new(@exp, @opts, "refeaturize" => true)
+               when "split"
+                 FredSplit.new(@exp, @opts)
+               when "train"
+                 FredTrain.new(@exp, @opts)
+               when "test"
+                 FredTest.new(@exp, @opts)
+               when "eval"
+                 FredEval.new(@exp, @opts)
+               else
+                 raise ArgumentError, "Wrong taks for #{PROGRAM_NAME}: #{@task}!"
+                 # @todo AB: this <else> condition should be impossible.
+                 #   Do it in OptionParser
+               end
 
-      # initialize task object
-      case @task
-      when "featurize"
-        task_obj = FredFeaturize.new(@exp, @opts)
-      when "refeaturize"
-        task_obj = FredFeaturize.new(@exp, @opts, "refeaturize" => true)
-      when "split"
-        task_obj = FredSplit.new(@exp, @opts)
-      when "train"
-        task_obj = FredTrain.new(@exp, @opts)
-      when "test"
-        task_obj = FredTest.new(@exp, @opts)
-      when "eval"
-        task_obj = FredEval.new(@exp, @opts)
-      else
-        raise "Shouldn't be here"
-        # @todo AB: this <else> condition should be unpossible
-        #     do in OptionParser
+        task.compute
+        LOGGER.info "#{PROGRAM_NAME} finished performing the task: #{task}!"
       end
-      
-      task_obj.compute
-      
-      $stderr.puts "Fred: Done."
-      
-    end
-  end # class Fred
-end # module Fred
+    end # class Fred
+  end # module Fred
+end

data/lib/fred/fred_error.rb

@@ -0,0 +1,15 @@
+module Shalmaneser
+  module Fred
+    class FredError < StandardError
+      # @param [String] msg A custom message for this exception.
+      # @param [Exception] nested_exception An external exception
+      #   which is reused to provide more information.
+      def initialize(msg = nil, nested_exception = nil)
+        if nested_exception
+          msg = "#{nested_exception.class}: #{nested_exception.message}\n#{msg}"
+        end
+        super(msg)
+      end
+    end
+  end
+end