tcf2nif 0.2.1

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "tcf2nif"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/convpar

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+echo "Running parallel converters"
+
+rspec -t phanplain  > ./spec/logs/phanplain.log  2>&1 &
+rspec -t screwplain > ./spec/logs/screwplain.log 2>&1 &
+rspec -t misplain   > ./spec/logs/misplain.log   2>&1 &

data/exe/createturtle

@@ -0,0 +1,32 @@
+#! /usr/bin/env bash
+cd ../spec/out/plainprov
+rapper ./phantom.nt    -i ntriples -o turtle \
+  -f 'xmlns:ex="http://example.org/tcf2nif/example.txt#"' \
+  -f 'xmlns:xsd="http://www.w3.org/2001/XMLSchema#"' \
+  -f 'xmlns:prov="http://www.w3.org/ns/prov#"' \
+  -f 'xmlns:mond="http://petermenke.de/mond#"' \
+  -f 'xmlns:nif="http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#"' \
+  > ./phantom.ttl
+rapper ./screw.nt      -i ntriples -o turtle \
+  -f 'xmlns:ex="http://example.org/tcf2nif/example.txt#"' \
+  -f 'xmlns:xsd="http://www.w3.org/2001/XMLSchema#"' \
+  -f 'xmlns:prov="http://www.w3.org/ns/prov#"' \
+  -f 'xmlns:mond="http://petermenke.de/mond#"' \
+  -f 'xmlns:nif="http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#"' \
+  > ./screw.ttl
+rapper ./miserables.nt -i ntriples -o turtle \
+  -f 'xmlns:ex="http://example.org/tcf2nif/example.txt#"' \
+  -f 'xmlns:xsd="http://www.w3.org/2001/XMLSchema#"' \
+  -f 'xmlns:prov="http://www.w3.org/ns/prov#"' \
+  -f 'xmlns:mond="http://petermenke.de/mond#"' \
+  -f 'xmlns:nif="http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#"' \
+  > ./miserables.ttl
+
+cd ../modularized
+rapper ./phantom.n3    -i ntriples -o turtle \
+  -f 'xmlns:ex="http://example.org/tcf2nif/example.txt#"' \
+  -f 'xmlns:xsd="http://www.w3.org/2001/XMLSchema#"' \
+  -f 'xmlns:prov="http://www.w3.org/ns/prov#"' \
+  -f 'xmlns:mond="http://petermenke.de/mond#"' \
+  -f 'xmlns:nif="http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#"' \
+  > ./phantom.ttl

data/exe/tcf2nif

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), *%w[.. lib])
+require 'tcf2nif'
+require 'trollop'
+
+opts = Trollop::options do
+  synopsis 'Creates NIF documents from a TCF file (generated by WebLicht)'
+  usage "Usage: #{__FILE__} sourcefile [OPTIONS]"
+  opt :input_file,  "The file containing the plain text document to be converted", type: :string
+  opt :output_file, "The file name to be used for the output file", type: :string
+  opt :format,      "The output file format", type: :string, default: 'n3'
+end
+puts opts
+
+if opts[:input_file] && opts[:output_file] && opts[:format]
+  @tcf_file = File.open(opts[:input_file], 'r')
+  @tcf_doc = Tcf2Nif::TcfDocument.new(@tcf_file)
+  @trans = Tcf2Nif::Transformer.new(@tcf_doc, {})
+  @graph = @trans.transform(:noprov)
+  RDF::Writer.open(opts[:output_file], :format => :ntriples) do |writer|
+    writer << RDF::Repository.new do |repo|
+      repo << @graph
+    end
+  end
+else
+  Trollop::educate
+  exit(1)
+end

data/exe/txt2tcf

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), *%w[.. lib])
+require 'tcf2nif'
+require 'trollop'
+
+opts = Trollop::options do
+  synopsis 'Creates TCF files via the WebLicht service from a plain text file'
+  usage "Usage: #{__FILE__} sourcefile [OPTIONS]"
+  opt :input_file,  "The file containing the plain text document to be converted", type: :string
+  opt :output_file, "The file name to be used for the output file", type: :string
+  opt :token,       "The WebLicht WaaS API token", type: :string
+  opt :chain,       "The chain description XML document", type: :string
+end
+
+if opts[:input_file] && opts[:output_file] && opts[:chain]
+  if opts.token
+    waas_token = opts.token
+  else
+    waas_token = ENV['WAAS_TOKEN']
+  end
+  service_uri = "https://weblicht.sfs.uni-tuebingen.de/WaaS/api/1.0/chain/process"
+
+  puts "curl -X POST -F chains=@#{opts[:chain]} -F content=@#{opts[:input_file]} -F apikey=#{waas_token} #{service_uri} > #{opts[:output_file]}"
+       `curl -X POST -F chains=@#{opts[:chain]} -F content=@#{opts[:input_file]} -F apikey=#{waas_token} #{service_uri} > #{opts[:output_file]}`
+
+else
+  Trollop::educate
+  exit(1)
+end

data/lib/tcf2nif.rb

@@ -0,0 +1,55 @@
+# encoding: utf-8
+# This file is part of the tcf2nif gem.
+# Copyright (c) 2015 Peter Menke, SFB 673, Universität Bielefeld
+# http://www.sfb673.org
+#
+# tcf2nif is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as
+# published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version.
+#
+# tcf2nif is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with tcf2nif. If not, see
+# <http://www.gnu.org/licenses/>.
+
+require 'nokogiri'
+require 'rdf'
+require 'rdf/turtle'
+require "tcf2nif/version"
+require "tcf2nif/bounded_element"
+require "tcf2nif/annotation"
+require "tcf2nif/token"
+require "tcf2nif/named_entity_annotation"
+require "tcf2nif/geo_annotation"
+require "tcf2nif/tcf_document"
+require "tcf2nif/transformer"
+
+module Tcf2Nif
+
+  NIF  = RDF::Vocabulary.new("http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#")
+  PENN = RDF::Vocabulary.new("http://purl.org/olia/penn.owl#")
+  NERD = RDF::Vocabulary.new("http://nerd.eurecom.fr/ontology#")
+  GEO  = RDF::Vocabulary.new("http://www.w3.org/2003/01/geo/wgs84_pos#")
+  PROV = RDF::Vocabulary.new("http://www.w3.org/ns/prov#")
+  MOND = RDF::Vocabulary.new("http://petermenke.de/mond#")
+
+  STANDARD_PREFIXES = {
+          nif: Tcf2Nif::NIF,
+          rdfs: RDF::RDFS,
+          xsd: RDF::XSD,
+          penn: Tcf2Nif::PENN,
+          geo: Tcf2Nif::GEO,
+          nerd: Tcf2Nif::NERD,
+          mond: Tcf2Nif::MOND
+      }
+
+  def self.root
+    File.expand_path('../..',__FILE__)
+  end
+
+end

data/lib/tcf2nif/annotation.rb

@@ -0,0 +1,39 @@
+# encoding: utf-8
+# This file is part of the tcf2nif gem.
+# Copyright (c) 2015 Peter Menke, SFB 673, Universität Bielefeld
+# http://www.sfb673.org
+#
+# tcf2nif is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as
+# published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version.
+#
+# tcf2nif is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with tcf2nif. If not, see
+# <http://www.gnu.org/licenses/>.
+
+module Tcf2Nif
+
+  class Annotation
+
+    def initialize(tcf_document)
+      @tcf_document = tcf_document
+      @tokens = Array.new
+    end
+
+    def tokens
+      @tokens
+    end
+
+    def <<(token)
+      @tokens << token
+    end
+
+  end
+
+end

data/lib/tcf2nif/bounded_element.rb

@@ -0,0 +1,43 @@
+# encoding: utf-8
+# This file is part of the tcf2nif gem.
+# Copyright (c) 2015 Peter Menke, SFB 673, Universität Bielefeld
+# http://www.sfb673.org
+#
+# tcf2nif is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as
+# published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version.
+#
+# tcf2nif is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with tcf2nif. If not, see
+# <http://www.gnu.org/licenses/>.
+
+module Tcf2Nif
+
+  class BoundedElement
+
+    attr_accessor :begin_index
+    attr_accessor :end_index
+
+
+    def boundaries=(new_boundaries)
+      @begin_index=new_boundaries.first
+      @end_index=new_boundaries.last
+    end
+
+    def boundaries?
+      @begin_index && @end_index
+    end
+
+    def length
+      end_index - begin_index
+    end
+
+  end
+
+end

data/lib/tcf2nif/geo_annotation.rb

@@ -0,0 +1,26 @@
+# encoding: utf-8
+# This file is part of the tcf2nif gem.
+# Copyright (c) 2015 Peter Menke, SFB 673, Universität Bielefeld
+# http://www.sfb673.org
+#
+# tcf2nif is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as
+# published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version.
+#
+# tcf2nif is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with tcf2nif. If not, see
+# <http://www.gnu.org/licenses/>.
+
+module Tcf2Nif
+  class GeoAnnotation < Tcf2Nif::Annotation
+
+    attr_accessor :lat, :lon, :alt, :continent
+
+  end
+end

data/lib/tcf2nif/named_entity_annotation.rb

@@ -0,0 +1,28 @@
+# encoding: utf-8
+# This file is part of the tcf2nif gem.
+# Copyright (c) 2015 Peter Menke, SFB 673, Universität Bielefeld
+# http://www.sfb673.org
+#
+# tcf2nif is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as
+# published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version.
+#
+# tcf2nif is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with tcf2nif. If not, see
+# <http://www.gnu.org/licenses/>.
+
+module Tcf2Nif
+
+  class NamedEntityAnnotation < Tcf2Nif::Annotation
+
+    attr_accessor :category
+
+  end
+
+end

data/lib/tcf2nif/tcf_document.rb

@@ -0,0 +1,228 @@
+# encoding: utf-8
+# This file is part of the tcf2nif gem.
+# Copyright (c) 2015 Peter Menke, SFB 673, Universität Bielefeld
+# http://www.sfb673.org
+#
+# tcf2nif is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as
+# published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version.
+#
+# tcf2nif is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with tcf2nif. If not, see
+# <http://www.gnu.org/licenses/>.
+
+module Tcf2Nif
+  
+  class TcfDocument
+    
+    def initialize(io)
+      @doc = Nokogiri::XML(io)
+      # TODO add a method that reads the XML into Ruby structures
+      @tokens = Array.new
+      @named_entities = Array.new
+      @geo_annotations = Array.new
+      @id_map = Hash.new
+      @token_map = Hash.new
+      @dependency_map = Hash.new()
+
+      process_tokens
+      unless @tokens.all?{|t| t.boundaries? }
+        calculate_character_offsets
+      end
+
+      # TODO process pos and lemma information
+      process_pos
+      process_lemma
+
+      process_named_entities
+      process_geo_annotations
+      process_dependencies
+
+    end
+
+    def token_map
+      @token_map
+    end
+
+    def id_map
+      @id_map
+    end
+
+    def calculate_character_offsets
+      char_index = 0
+      tokens.each do |token|
+        new_index = text.index(token.form, char_index)
+        new_offset = new_index + token.form.length
+        token.boundaries= [new_index, new_offset]
+        char_index = new_offset
+      end
+    end
+
+    def text
+      #puts "texts:      %i" % @doc.xpath('.//tc:text/text()', 'wl' => 'http://www.dspin.de/data', 'tc' => 'http://www.dspin.de/data/textcorpus').size
+      #puts "text type:  %s" % @doc.xpath('.//tc:text/text()', 'wl' => 'http://www.dspin.de/data', 'tc' => 'http://www.dspin.de/data/textcorpus').class.name
+      #puts "first type: %s" % @doc.xpath('.//tc:text/text()', 'wl' => 'http://www.dspin.de/data', 'tc' => 'http://www.dspin.de/data/textcorpus').first.class.name
+      #puts "first cont: %s" % @doc.xpath('.//tc:text/text()', 'wl' => 'http://www.dspin.de/data', 'tc' => 'http://www.dspin.de/data/textcorpus').first.to_s.slice(0,128)
+
+      @text ||= @doc.xpath('.//tc:text/text()', 'wl' => 'http://www.dspin.de/data', 'tc' => 'http://www.dspin.de/data/textcorpus').first.to_s
+    end
+
+    def tokens
+      @tokens
+    end
+
+    def named_entities
+      @named_entities
+    end
+
+    def geo_annotations
+      @geo_annotations
+    end
+
+    def dependency_map
+      @dependency_map
+    end
+
+    def store_named_entity(named_entity_object)
+      @named_entities << named_entity_object
+    end
+
+
+    def xml_sentences
+      # /wl:D-Spin/tc:TextCorpus[1]/tc:text[1]
+      @xml_sentences ||= @doc.xpath('//tc:sentences/tc:sentence', 'tc' => 'http://www.dspin.de/data/textcorpus')
+    end
+    
+    def xml_tokens
+      @xml_tokens ||= @doc.xpath('//tc:tokens/tc:token', 'tc' => 'http://www.dspin.de/data/textcorpus')
+    end
+
+    def xml_named_entities
+      @xml_named_entities ||= @doc.xpath('//tc:namedEntities/tc:entity', 'tc' => 'http://www.dspin.de/data/textcorpus')
+    end
+
+    def xml_geo_annotations
+      @xml_geo_annotations ||= @doc.xpath('//tc:geo/tc:gpoint', 'tc' => 'http://www.dspin.de/data/textcorpus')
+    end
+
+    def xml_dependencies
+      @xml_dependencies ||= @doc.xpath('//tc:depparsing/tc:parse/tc:dependency', 'tc' => 'http://www.dspin.de/data/textcorpus')
+    end
+
+    # TODO add deep support for sentences and related tokens
+
+    def new_token(doc, xml_token)
+      token_object = Tcf2Nif::Token.new(doc, xml_token)
+      if xml_token.has_attribute?('start') && xml_token.has_attribute?('end')
+        token_object.boundaries= [xml_token['start'].to_i, xml_token['end'].to_i]
+      end
+      token_object
+    end
+
+    def store_token(token_object, xml_token)
+      @tokens << token_object
+      @id_map[xml_token['ID']] = token_object
+      @token_map[token_object] = xml_token['ID']
+    end
+
+    def token_for_id(xml_id)
+      @id_map[xml_id]
+    end
+
+    def id_for_token(token)
+      @token_map[token]
+    end
+
+
+    private
+
+    def process_tokens
+      xml_tokens.each do |xml_token|
+        token = new_token(@doc, xml_token)
+        store_token(token, xml_token)
+      end
+    end
+
+    def process_pos
+      xml_tags = @doc.xpath('//tc:POStags/tc:tag', 'tc' => 'http://www.dspin.de/data/textcorpus')
+      xml_tags.each do |tag|
+        val = tag.text
+        ref = tag['tokenIDs']
+        ref_obj = @id_map[ref]
+        if val && ref_obj
+          ref_obj.pos = val
+        end
+      end
+    end
+
+    def process_lemma
+      xml_lemmas = @doc.xpath('//tc:lemmas/tc:lemma', 'tc' => 'http://www.dspin.de/data/textcorpus')
+      xml_lemmas.each do |lemma|
+        val = lemma.text
+        ref = lemma['tokenIDs']
+        ref_obj = @id_map[ref]
+        if val && ref_obj
+          ref_obj.lemma = val
+        end
+      end
+    end
+
+    def process_named_entities
+      xml_named_entities.each do |ent|
+        nato = Tcf2Nif::NamedEntityAnnotation.new(@doc)
+        nato.category = ent['class']
+        token_refs = ent['tokenIDs'].split(/\s+/)
+        tokens = token_refs.collect{|r| token_for_id(r)}
+        tokens.each do |t|
+          nato << t
+        end
+        @named_entities << nato
+        #puts ent['class']
+        #puts tokens.collect{|t| t.form}.join(' ')
+      end
+    end
+
+    def process_geo_annotations
+      xml_geo_annotations.each do |anno|
+        geo = Tcf2Nif::GeoAnnotation.new(@doc)
+        geo.lat = anno['lat'].to_f
+        geo.lon = anno['lon'].to_f
+        geo.alt = anno['alt'].to_f
+        geo.continent = anno['continent']
+        token_refs = anno['tokenIDs'].split(/\s+/)
+        tokens = token_refs.collect{|r| token_for_id(r)}
+        tokens.each do |t|
+          geo << t
+        end
+        @geo_annotations << geo
+      end
+    end
+
+    def process_dependencies
+      xml_dependencies.each do |dep|
+        # <tc:dependency depIDs="t_4" func="ROOT"/>
+        # <tc:dependency govIDs="t_4" depIDs="t_2" func="aux"/>
+
+        depToken = token_for_id(dep['depIDs'])
+
+        if dep.has_attribute?('govIDs')
+          # non-root tag. func is also defined.
+          govToken = token_for_id(dep['govIDs'])
+          @dependency_map[[depToken,govToken]] = dep['func']
+        else
+          # root tag.
+
+        end
+
+      end
+    end
+    
+  end
+  
+end