shalmaneser-frappe 1.2.rc5

data/lib/frappe/syn_interface.rb

@@ -0,0 +1,76 @@
+#############################
+# abstract class, to be inherited:
+#
+# tabular format or SalsaTigerXML interface for modules
+# offering POS tagging, lemmatization, parsing etc.
+
+# Leave this commented until we've reworked ExternalSystems
+# since in causes circular requirements.
+# require 'external_systems'
+
+module Shalmaneser
+  module Frappe
+    class SynInterface
+      ###
+      # returns a string: the name of the system
+      # e.g. "Collins" or "TNT"
+      def self.system
+        raise NotImplementedError, "Overwrite me"
+      end
+
+      ###
+      # returns a string: the service offered
+      # one of "lemmatizer", "parser", "pos tagger"
+      def self.service
+        raise NotImplementedError, "Overwrite me"
+      end
+
+      ###
+      # initialize to set values for all subsequent processing
+      def initialize(program_path, # string: path to system
+                     insuffix,      # string: suffix of input files
+                     outsuffix,     # string: suffix for processed files
+                     var_hash = {}) # optional arguments in a hash
+
+        @program_path = program_path
+        @insuffix = insuffix
+        @outsuffix = outsuffix
+      end
+
+      ###
+      # process each file in in_dir with matching suffix,
+      # producing a file in out_dir with same name but the suffix replaced
+      #
+      # returns: nothing
+      def process_dir(in_dir,        # string: name of input directory
+                      out_dir)       # string: name of output directory
+
+        Dir["#{in_dir}*#{@insuffix}"].each do |infilename|
+          outfilename = "#{out_dir}#{File.basename(infilename, @insuffix)}#{@outsuffix}"
+          process_file(infilename, outfilename)
+        end
+      end
+
+      ###
+      # process one file, writing the result to outfilename
+      #
+      # returns: nothing
+      def process_file(infilename,   # string: name of input file
+                       outfilename)
+        raise NotImplementedError, "Overwrite me"
+      end
+
+      protected
+
+      def self.announce_me
+        if defined?(ExternalSystems)
+          # Yup, we have a class to which we can announce ourselves.
+          ExternalSystems.add_interface(self)
+        else
+          # no interface collector class
+          LOGGER.warn "Interface #{self} not announced: no ExternalSystems."
+        end
+      end
+    end
+  end
+end

data/lib/frappe/syn_interface_stxml.rb

@@ -0,0 +1,173 @@
+require_relative 'syn_interface'
+
+require 'salsa_tiger_xml/file_parts_parser'
+require 'salsa_tiger_xml/salsa_tiger_xml_helper'
+require 'salsa_tiger_xml/salsa_tiger_sentence'
+
+#############################
+# abstract class, to be inherited:
+#
+# SalsaTigerXML interface for modules
+# offering parsing etc.
+#
+# The input format for these classes is TabFormat or FNTabFormat
+module Shalmaneser
+  module Frappe
+    class SynInterfaceSTXML < SynInterface
+      ###
+      # initialize to set values for all subsequent processing
+      def initialize(program_path, # string: path to system
+                     insuffix,      # string: suffix of input files
+                     outsuffix,     # string: suffix for processed files
+                     stsuffix,      # string: suffix for Salsa/Tiger XML files
+                     var_hash = {}) # optional arguments in a hash
+        super(program_path, insuffix, outsuffix, var_hash)
+        @stsuffix = stsuffix
+      end
+
+      def to_stxml_dir(in_dir,   # string: name of dir with parse files
+                       out_dir)  # string: name of output dir
+
+        Dir["#{in_dir}*#{@outsuffix}"].each do |parsefilename|
+          stxmlfilename = "#{out_dir}#{File.basename(parsefilename, @outsuffix)}#{@stsuffix}"
+          to_stxml_file(parsefilename, stxmlfilename)
+        end
+      end
+
+      def to_stxml_file(infilename, outfilename)
+        raise NotImplementedError, "Overwrite me"
+      end
+
+      ###
+      # standard mapping:
+      #
+      # to be used as the mapping from tab sentence words to
+      # SalsaTigerSentence nodes returned by each_sentence:
+      # map the n-th word of the tab sentence to the n-th terminal of
+      # the SalsaTigerSentence
+      def self.standard_mapping(sent, tabsent)
+        retv = {}
+
+        if sent.nil?
+          retv = nil
+        else
+          terminals = sent.terminals_sorted
+          if tabsent
+            tabsent.each_line_parsed do |l|
+              if (t = terminals[l.get("lineno")])
+                retv[l.get("lineno")] = [t]
+              else
+                retv[l.get("lineno")] = []
+              end
+            end
+          end
+        end
+
+        retv
+      end
+
+
+      ###
+      # for a given processed file:
+      # yield each sentence as a tuple
+      #  [SalsaTigerSentence object, FNTabFormatSentence object, mapping]
+      # of
+      # - the sentence in SalsaTigerXML,
+      # - the matching tab format sentence
+      # - a mapping of terminals:
+      #   hash: line in tab sentence(integer) -> array:SynNode
+      #   mapping tab sentence nodes to matching nodes in the SalsaTigerSentence data structure
+      #
+      # default version: write Salsa/Tiger XML to tempfile, read back in
+      # and assume that each sentence in the tab file has a correspondent
+      # in the processed file (may not hold e.g. if the parser leaves out
+      # sentences it cannot process)
+      def each_sentence(infilename,  # string: name of processed file
+                        tab_dir = nil) # string: name of dir with input files
+        # (set either here or on initialization)
+        if tab_dir
+          @tab_dir = tab_dir
+        end
+
+        # write Salsa/Tiger XML to tempfile
+        tf = Tempfile.new("SynInterface")
+        tf.close
+        to_stxml_file(infilename, tf.path)
+        tf.flush
+
+        # get matching tab file, read
+        tab_reader = get_tab_reader(infilename)
+        tab_sentences = []
+        tab_reader.each_sentence { |s| tab_sentences << s }
+
+        # read Salsa/Tiger sentences and yield them
+        reader = STXML::FilePartsParser.new(tf.path)
+        sent_index = 0
+        reader.scan_s { |sent_string|
+          yield [
+            STXML::SalsaTigerSentence.new(sent_string, tab_sentences[sent_index]),
+            tab_sentences[sent_index],
+            SynInterfaceSTXML.standard_mapping(sent, tab_sentences[sent_index])
+          ]
+          sent_index += 1
+        }
+
+        # remove tempfile
+        tf.close(true)
+      end
+
+      #####################
+      protected
+
+
+      ###
+      # get tab format file for a given processed file
+      def get_tab_reader(infilename) # string: name of processed file
+        # find matching non-processed file for processed file
+        # assumption: directory with non-processed files
+        # has been set as @tab_dir
+
+        # sanity checks
+        unless @tab_dir
+          raise "Need to set tab directory"
+        end
+
+        # get matching tab file for this parser output file
+        tabfilename = @tab_dir+File.basename(infilename, @outsuffix)+ @insuffix
+        return FNTabFormatFile.new(tabfilename)
+      end
+
+
+      ###
+      # provide a XML representation for a sentence that couldn't be analyzed
+      # assuming a flat structure of all terminals, adding a virtual top node
+      def self.failed_sentence(tab_sent, sentid)
+        sent_obj = STXML::SalsaTigerSentence.empty_sentence(sentid.to_s)
+
+        sent_obj.set_attribute("failed","true")
+
+        topnode = sent_obj.add_syn("nt",
+                                   "NONE", # cat
+                                   nil, # word (doesn't matter)
+                                   nil, # pos (doesn't matter)
+                                   "500") # nonterminal counter
+
+        t_counter = 0
+
+        tab_sent.each_line_parsed {|line|
+          t_counter += 1
+          word = line.get("word")
+          pos = line.get("pos")
+          node = sent_obj.add_syn("t",
+                                  nil,  # cat (doesn't matter here)
+                                  STXML::SalsaTigerXMLHelper.escape(word), # word
+                                  pos,  # pos
+                                  t_counter.to_s)
+          topnode.add_child(node,nil)
+          node.add_parent(topnode, nil)
+        }
+        return sent_obj
+      end
+    end
+  end
+end

data/lib/frappe/syn_interface_tab.rb

@@ -0,0 +1,39 @@
+require_relative 'syn_interface'
+
+require 'tabular_format/fn_tab_format_file'
+
+#############################
+# abstract class, to be inherited:
+#
+# tabular format interface for modules
+# offering POS tagging, lemmatization etc.
+module Shalmaneser
+  module Frappe
+    class SynInterfaceTab < SynInterface
+
+      ##########
+      protected
+
+      # fntab_words_for_file:
+      # given a file in tab format, columns as in FNTabFormat,
+      # get the "word" entries and write them to a given file,
+      # one word per line, as input for processing
+      def SynInterfaceTab.fntab_words_to_file(infilename, # string: name of input file
+                                              outfile,    # stream: output file
+                                              sent_marker = "", # string: mark end of sentence how?
+                                              iso = nil)  # non-nil: assume utf-8, transform to iso-8859-1
+        corpusfile = FNTabFormatFile.new(infilename)
+        corpusfile.each_sentence {|s|
+          s.each_line_parsed {|line_obj|
+            if iso
+              outfile.puts UtfIso.to_iso_8859_1(line_obj.get("word"))
+            else
+              outfile.puts line_obj.get("word")
+            end
+          }
+          outfile.puts sent_marker
+        }
+      end
+    end
+  end
+end

data/lib/frappe/utf_iso.rb

@@ -0,0 +1,27 @@
+# KE changed July 05: now no inclusion of modules required,
+# and names changed from REXML.Encodign to UtfIso
+module Shalmaneser
+  module Frappe
+    module UtfIso
+      # Convert from UTF-8
+      def UtfIso.to_iso_8859_1(content)
+        array_utf8 = content.unpack('U*')
+        array_enc = []
+        array_utf8.each do |num|
+          if num <= 0xFF
+            array_enc << num
+          else
+            # Numeric entity (&#nnnn;); shard by  Stefan Scholl
+            # array_enc += to_iso_8859("&\##{num};").unpack('C*')
+          end
+        end
+        array_enc.pack('C*')
+      end
+
+      # Convert to UTF-8
+      def UtfIso.from_iso_8859_1(str)
+        str.unpack('C*').pack('U*')
+      end
+    end
+  end
+end

data/lib/shalmaneser/frappe.rb

@@ -0,0 +1 @@
+# A dummy file to require for now.

metadata

@@ -0,0 +1,130 @@
+--- !ruby/object:Gem::Specification
+name: shalmaneser-frappe
+version: !ruby/object:Gem::Version
+  version: 1.2.rc5
+platform: ruby
+authors:
+- Andrei Beliankou
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-01-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: tokenizer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: shalmaneser-lib
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.2.rc5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.2.rc5
+description: Frappe - Fred and Rosy PreProcEssor.
+email: arbox@yandex.ru
+executables: []
+extensions: []
+extra_rdoc_files:
+- README.md
+- LICENSE.md
+- CHANGELOG.md
+files:
+- ".yardopts"
+- CHANGELOG.md
+- LICENSE.md
+- README.md
+- lib/frappe/Ampersand.rb
+- lib/frappe/file_parser.rb
+- lib/frappe/fix_syn_sem_mapping.rb
+- lib/frappe/frappe.rb
+- lib/frappe/frappe_flat_syntax.rb
+- lib/frappe/frappe_read_stxml.rb
+- lib/frappe/interfaces/berkeley_interface.rb
+- lib/frappe/interfaces/collins_interface.rb
+- lib/frappe/interfaces/counter.rb
+- lib/frappe/interfaces/stanford_interface.rb
+- lib/frappe/interfaces/treetagger_interface.rb
+- lib/frappe/interfaces/treetagger_module.rb
+- lib/frappe/interfaces/treetagger_pos_interface.rb
+- lib/frappe/interpreters/berkeley_interpreter.rb
+- lib/frappe/interpreters/collins_tnt_interpreter.rb
+- lib/frappe/interpreters/collins_treetagger_interpreter.rb
+- lib/frappe/interpreters/empty_interpreter.rb
+- lib/frappe/interpreters/headz.rb
+- lib/frappe/interpreters/headz_helpers.rb
+- lib/frappe/interpreters/stanford_interpreter.rb
+- lib/frappe/interpreters/syn_interpreter.rb
+- lib/frappe/interpreters/tiger_interpreter.rb
+- lib/frappe/interpreters/treetagger_interpreter.rb
+- lib/frappe/one_parsed_file.rb
+- lib/frappe/opt_parser.rb
+- lib/frappe/path.rb
+- lib/frappe/plain_converter.rb
+- lib/frappe/salsa_tab_converter.rb
+- lib/frappe/salsa_tab_with_pos_converter.rb
+- lib/frappe/stxml_converter.rb
+- lib/frappe/syn_interface.rb
+- lib/frappe/syn_interface_stxml.rb
+- lib/frappe/syn_interface_tab.rb
+- lib/frappe/utf_iso.rb
+- lib/shalmaneser/frappe.rb
+homepage: https://github.com/arbox/shalmaneser
+licenses:
+- GPL-2.0
+metadata: {}
+post_install_message: 
+rdoc_options:
+- "-m"
+- README.md
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '='
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: FRAPPE
+test_files: []
+has_rdoc: