shalmaneser-frappe 1.2.rc5

data/lib/frappe/interpreters/treetagger_interpreter.rb

@@ -0,0 +1,89 @@
+###############
+# an interpreter that only has Treetagger, no parser
+
+require_relative 'syn_interpreter'
+require 'logging'
+
+module Shalmaneser
+  module Frappe
+    class TreetaggerInterpreter < SynInterpreter
+      TreetaggerInterpreter.announce_me
+
+      ###
+      # names of the systems interpreted by this class:
+      # returns a hash service(string) -> system name (string),
+      # e.g.
+      # { "parser" => "collins", "lemmatizer" => "treetagger" }
+      def self.systems
+        {"pos_tagger" => "treetagger"}
+      end
+
+      ###
+      # names of additional systems that may be interpreted by this class
+      # returns a hash service(string) -> system name(string)
+      # same as names()
+      def self.optional_systems
+        {"lemmatizer" => "treetagger"}
+      end
+
+      ###
+      # generalize over POS tags.
+      #
+      # returns one of:
+      #
+      # adj:  adjective (phrase)
+      # adv:  adverb (phrase)
+      # card: numbers, quantity phrases
+      # con:  conjunction
+      # det:  determiner, including possessive/demonstrative pronouns etc.
+      # for:  foreign material
+      # noun: noun (phrase), including personal pronouns, proper names, expletives
+      # part: particles, truncated words (German compound parts)
+      # prep: preposition (phrase)
+      # pun:  punctuation, brackets, etc.
+      # sent: sentence
+      # top:  top node of a sentence
+      # verb: verb (phrase)
+      # nil:  something went wrong
+      #
+      # returns: string, or nil
+      def self.category(node) # SynNode
+        pt = TreetaggerInterpreter.pt(node)
+        # phrase type could not be determined
+        return nil if pt.nil?
+
+        case pt.to_s.strip.match(/^([^-]*)/)[1]
+        when /^JJ/, /(WH)?ADJP/, /^PDT/
+          "adj"
+        when /^RB/, /(WH)?ADVP/, /^UH/
+          "adv"
+        when /^CD/, /^QP/
+          "card"
+        when /^CC/, /^WRB/, /^CONJP/
+          "con"
+        when /^DT/, /^POS/
+          "det"
+        when /^FW/, /^SYM/
+          "for"
+        when /^N/, "WHAD", "WDT", /^PRP/, /^WHNP/, /^EX/, /^WP/
+          "noun"
+        when /^IN/, /^TO/, /(WH)?PP/, "RP", /^PR(T|N)/
+          "prep"
+        when /^PUNC/, /LRB/, /RRB/, /[,'".:;!?\(\)]/
+          "pun"
+        when /^S(s|bar|BAR|G|Q|BARQ|INV)?$/, /^UCP/, /^FRAG/, /^X/, /^INTJ/
+          "sent"
+        when /^TOP/
+          "top"
+        when /^TRACE/
+          "trace"
+        when /^V/, /^MD/
+          "verb"
+        else
+          LOGGER.warn "Unknown category/POS #{pt} (English data)."
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/frappe/one_parsed_file.rb

@@ -0,0 +1,31 @@
+# -*- encoding: utf-8 -*-
+# AB, 2010-11-25
+
+##############################
+# class for managing the parses of one file
+module Shalmaneser
+  module Frappe
+    class OneParsedFile
+      attr_reader :filename
+      # @param [String] filename  The core of filename for the parse file.
+      # @param [String] complete_filename The complete filename of the parse file.
+      # @param [Enumerable] obj_with_iterator object with each_sentence method, see above
+      def initialize(filename, complete_filename, obj_with_iterator)
+        @obj_with_iterator = obj_with_iterator
+        @filename = filename
+        @complete_filename = complete_filename
+      end
+
+      # yield each parse sentence as a tuple
+      # [ salsa/tiger xml sentence, tab format sentence, mapping]
+      # of a SalsaTigerSentence object, a FNTabSentence object,
+      # and a hash: FNTab sentence lineno(integer) -> array:SynNode
+      # pointing each tab word to one or more SalsaTigerSentence terminals
+      def each_sentence
+        @obj_with_iterator.each_sentence(@complete_filename) do |st_sent, tab_sent, mapping|
+          yield [st_sent, tab_sent, mapping]
+        end
+      end
+    end
+  end
+end

data/lib/frappe/opt_parser.rb

@@ -0,0 +1,92 @@
+# -*- encoding: utf-8 -*-
+
+# @author AB
+# @date 2010-11-25
+
+require 'optparse'
+require 'configuration/frappe_config_data'
+require 'definitions'
+require 'external_systems'
+require 'logging'
+
+module Shalmaneser
+  module Frappe
+    # This class parses options for FrPrep.
+    # @todo Remove explicit exits in this class.
+    class OptParser
+      # Main class method.
+      # OP expects cmd_args to be an array like ARGV.
+      def self.parse(cmd_args)
+        @prg_name = PROGRAM_NAME
+        @options = {}
+        parser = create_parser
+
+        # If no options provided print the help.
+        if cmd_args.empty?
+          msg = "You have to provide some options.\n"\
+                "Please start with <#{@prg_name} --help>."
+
+          $stderr.puts msg
+          exit(1)
+        end
+
+        # Parse ARGV and provide the options hash.
+        # Check if everything is correct and handle exceptions
+        begin
+          parser.parse(cmd_args)
+        rescue OptionParser::InvalidArgument => e
+          arg = e.message.split.last
+          $stderr.puts "The provided argument #{arg} is currently not supported!\n"\
+                       "Please consult <#{@prg_name} --help>."
+          exit(1)
+        rescue OptionParser::InvalidOption => e
+          $stderr.puts "You have provided an #{e.message}.\n"\
+                       "Please consult <#{@prg_name} --help>."
+          exit(1)
+        rescue
+          raise
+        end
+
+        # @todo Rename the config data class.
+        exp = ::Shalmaneser::Configuration::FrappeConfigData.new(@options[:exp_file])
+
+        # @todo AB: [2015-12-28 Mon 19:22]
+        #   Move this to ConfigData.
+        ExternalSystems.check_interfaces_abort_if_missing(exp)
+
+        exp
+      end
+
+      private
+
+      def self.create_parser
+        OptionParser.new do |opts|
+          opts.banner = "Fred and Rosy Preprocessor <Frappe>. Preprocessing stage before Fred and Rosy\n"\
+                        "for further frame/word sense disambiguation and semantic role assignment."\
+                        "\n"\
+                        "Usage: #{PROGRAM_NAME.downcase} -h|-e FILENAME"
+          opts.separator ''
+          opts.separator 'Program specific options:'
+
+          opts.on('-e', '--expfile FILENAME',
+                  'Provide the path to an experiment file.',
+                  "#{PROGRAM_NAME} will preprocess data according to the specifications",
+                  'given in your experiment file.',
+                  'This option is required!',
+                  'Also consider the documentation on format and features.'
+                 ) do |exp_file|
+            @options[:exp_file] = File.expand_path(exp_file)
+          end
+
+          opts.separator ''
+          opts.separator 'Common options:'
+
+          opts.on_tail('-h', '--help', 'Show this help message.') do
+            puts opts
+            exit
+          end
+        end
+      end # def self.parse
+    end # class OptParser
+  end # module Frappe
+end # Shalm

data/lib/frappe/path.rb

@@ -0,0 +1,199 @@
+#############################
+# class describing a path between two nodes
+#
+# provides access and output facilities for different aspects of the path
+#
+# this is the return value of SynInterpreter.path_between
+module Shalmaneser
+  module Frappe
+    class Path
+      attr_reader :startnode
+
+      ###
+      # initialize to empty path
+      def initialize(startnode)
+        @path = []
+        @cutoff_last_pt = false
+        set_startnode(startnode)
+      end
+
+      ###
+      # deep_clone:
+      # return clone of this path object,
+      #  with clone of this path rather than the same path
+      def deep_clone
+        new_path = self.clone
+        new_path.set_path(@path.clone)
+
+        return new_path
+      end
+
+      ###
+      def set_startnode(startnode)
+        @startnode = startnode
+
+        return self
+      end
+
+      ###
+      # iterate through the current path
+      #
+      # yield tuples
+      # [direction, edgelabel, nodelabel, endnode]
+      #  direction: string, U/D
+      #  edgelabel: string
+      #  nodelabel: string
+      #  endnode:   SynNode
+      def each_step
+        @path.each { |step|
+          yield step
+        }
+      end
+
+      ###
+      # empty?
+      # any steps in here?
+      def empty?
+        return @path.empty?
+      end
+
+      ###
+      # add one step to the beginning of the current path
+      def add_first_step(start_node,#SynNode
+                         direction, # string: U, D
+                         gf,        # string: edge label
+                         pt)
+        @path.unshift([direction, gf, pt, @startnode])
+        set_startnode(start_node)
+
+        return self
+      end
+
+
+      ###
+      # add one step to the end of the current path
+      def add_last_step(direction, # string: U, D
+                        gf,        # string: edge label
+                        pt,        # string: node label (of end_node)
+                        end_node)  # SynNode
+        @path << [direction, gf, pt, end_node]
+
+        return self
+      end
+
+      ###
+      # path length
+      def length
+        return @path.length
+      end
+
+      ###
+      #
+      def print(print_direction, # boolean. true: print direction
+                print_gf,        # boolean. true: print edgelabel
+                print_pt)        # boolean. true: print nodelabel
+
+        return print_aux(@path, print_direction, print_gf, print_pt)
+      end
+
+      ###
+      # print path from roof node to end
+      def print_downpart(print_direction,
+                         print_gf,
+                         print_pt)
+
+        roof, roof_index = compute_roof
+        if roof.nil? or @path.empty?
+          # no roof set
+          return ""
+
+        else
+          # roof node is in the middle
+          return print_aux(@path[roof_index..-1],
+                           print_direction, print_gf, print_pt)
+        end
+      end
+
+      ###
+      def lca
+        return compute_roof.first
+      end
+
+      ###
+      # cut off last node label in print and print_downpart?
+      def set_cutoff_last_pt_on_printing(bool) # Boolean
+        @cutoff_last_pt = bool
+      end
+
+      ########
+      protected
+
+      def set_path(new_path)
+        @path = new_path
+      end
+
+
+      ########
+      private
+
+      ###
+      # step through the path as long as direction is up.
+      # when direction starts to go "D", take current node as roof node
+      #
+      # returns: pair [roof node, roof node index] (SynNode, integer)
+      def compute_roof
+        node = @startnode
+        index = 0
+
+        each_step { |direction, edgelabel, nodelabel, endnode|
+          if direction =~ /D/
+            # down! the previous node was roof
+            return [node, index]
+          else
+            node = endnode
+            index += 1
+          end
+        }
+
+        # last node is roof
+        return [node, index]
+
+      end
+
+      ###
+      def print_aux(path,
+                    print_direction,
+                    print_gf,
+                    print_pt)
+        retv = ''
+        path.each { |step|
+          direction, gf, pt, _node = step.map { |entry|
+            if entry.nil?
+              "-"
+            else
+              entry
+            end
+          }
+
+          if print_direction
+            retv << direction + " "
+          end
+
+          if print_gf
+            retv << gf + " "
+          end
+
+          if print_pt
+            retv << pt + " "
+          end
+        }
+
+        if @cutoff_last_pt && print_pt && (retv =~ /^(.+ )\w+ $/)
+          return $1
+        else
+          return retv
+        end
+      end
+    end
+  end
+end

data/lib/frappe/plain_converter.rb

@@ -0,0 +1,59 @@
+require 'tokenizer'
+require 'tabular_format/fn_tab_format_file'
+
+module Shalmaneser
+  module Frappe
+    # A converter from plain text to Salsa Tab Format.
+    # Performs tokenization.
+    class PlainConverter
+      def initialize
+        # suffixes for different types of output files
+        @file_suffixes = {"lemma" => ".lemma", "pos" => ".pos", "tab" => ".tab", "stxml" => ".xml"}
+      end
+
+      ###############
+      # transform_plain:
+      #
+      # transformation for plaintext:
+      #
+      # transform to Tab format, separating punctuation from adjacent words
+      # @param input_dir [String] input directory
+      # @param output_dir [String] output directory
+      def transform_plain_dir(input_dir, output_dir)
+        Dir[input_dir + "*"].each do |plainfilename|
+          # open input and output file
+          # end output file name in "tab" because that is, at the moment, required
+          outfilename = output_dir + File.basename(plainfilename, '.*') + @file_suffixes["tab"]
+          plain_to_tab_file(plainfilename, outfilename)
+        end
+      end
+
+      ####
+      # transform plaintext file to Tab format file
+      # @param [String] input_filename string: name of input file
+      # @param [String] output_filename string: name of output file
+      def plain_to_tab_file(input_filename, output_filename)
+        sentences = File.open(input_filename) do |f|
+          # The file is supposed to contain one sentence per line.
+          f.readlines.map(&:chomp).map(&:strip).reject(&:empty?)
+        end
+        id = File.basename(input_filename, '.*')
+        t = Tokenizer::Tokenizer.new
+        File.open(output_filename, "w") do |f|
+          sentences.each_with_index do |sentence, idx|
+            # byebug
+            sentid = "#{id}_#{idx}"
+            sentence = t.tokenize(sentence)
+            sentence.each do |word|
+              # for each word, one line, entries in the line tab-separated
+              # the 'word' entry is the word, the 'lu_sent_ids' entry is the sentence ID sentid,
+              # all other entries (gf, pt, frame etc.) are not set
+              f.puts FNTabFormatFile.format_str("word" => word, "sent_id" => sentid)
+            end
+            f.puts
+          end
+        end
+      end
+    end
+  end
+end