opener-kaf-naf-parser 1.0.0

data/README.md

@@ -0,0 +1,46 @@
+kaf-naf-parser
+==============
+
+This library converts KAF to NAF and NAF to KAF. It also contains a webservice
+for doing exactly this.
+
+It uses slightly altered 
+distributions of the VUA_pylib and KafNafParserPy but it keeps the external 
+libraries contained, so that shouldn't influence anything on your system.
+
+This does mean however, that changes to the VUA_pylib and KafNafParserPy are not
+automatically resembled in the kaf-naf-parser.
+
+It ships with 3 command line tools and a webservice:
+
+```
+kaf-to-naf
+naf-to-kaf
+kaf-naf-parser
+kaf-naf-parser-server
+```
+
+It works like this:
+
+```
+cat some_kaf.kaf | kaf-to-naf > output.naf
+cat some_naf.naf | naf-to-kaf > output.kaf
+
+cat some_kaf.kaf | kaf-naf-parser --tonaf > output.naf
+```
+
+The webservice can be started like this:
+
+```
+naf-to-kaf-parser-server
+```
+And can then be accessed at: http://localhost:9292
+
+Easiest way to install the gem:
+
+```
+git clone git@github.com:opener-project/kaf-naf-parser.git
+cd kaf-naf-parser
+rake install
+```
+

data/bin/kaf-naf-parser

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/opener/kaf_naf_parser'
+
+cli = Opener::KafNafParser::CLI.new(:args => ARGV)
+
+cli.run(STDIN.tty? ? nil : STDIN.read)
+

data/bin/kaf-naf-parser-server

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+require 'rack'
+
+# Without calling `Rack::Server#options` manually the CLI arguments will never
+# be passed, thus the application can't be specified as a constructor argument.
+server                  = Rack::Server.new
+server.options[:config] = File.expand_path('../../config.ru', __FILE__)
+
+server.start

data/bin/kaf-to-naf

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/opener/kaf_naf_parser'
+
+cli = Opener::KafNafParser::CLI.new(:args => ARGV, :conversion=>"to-naf")
+
+cli.run(STDIN.tty? ? nil : STDIN.read)

data/bin/naf-to-kaf

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/opener/kaf_naf_parser'
+
+cli = Opener::KafNafParser::CLI.new(:args => ARGV, :conversion=>"to-kaf")
+
+cli.run(STDIN.tty? ? nil : STDIN.read)

data/config.ru

@@ -0,0 +1,4 @@
+require File.expand_path('../lib/opener/kaf_naf_parser', __FILE__)
+require File.expand_path('../lib/opener/kaf_naf_parser/server', __FILE__)
+
+run Opener::KafNafParser::Server

data/core/kaf-naf-parser.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+
+# This updates the load path to ensure that the local site-packages directory
+# can be used to load packages (e.g. a locally installed copy of lxml).
+
+import sys, getopt, os
+
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'site-packages/pre_build'))
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'site-packages/pre_install'))
+
+from KafNafParser import KafNafParser
+
+def main(argv):
+  conversion = ""
+  try:
+    opts, args = getopt.getopt(argv,"hkn",["tokaf","tonaf"])
+  except getopt.GetoptError:
+    print 'could not parse options. Correct usage: \n\n kaf-naf-parser.py --tokaf --tonaf'
+    sys.exit(2)
+  for opt, arg in opts:
+    if opt == '-h':
+      print 'test.py --tokaf --tonaf'
+      sys.exit()
+    elif opt in ("-k", "--tokaf"):
+      conversion = "to-kaf"
+    elif opt in ("-n", "--tonaf"):
+      conversion = "to-naf"
+
+  if conversion == "":
+    conversion = "kaf-naf"
+
+  obj = KafNafParser(sys.stdin)
+
+  if conversion == "to-kaf":
+    obj.to_kaf()
+  if conversion == "to-naf":
+    obj.to_naf()
+
+  obj.dump()
+
+if __name__ == "__main__":
+  main(sys.argv[1:])

data/core/site-packages/pre_build/KafNafParser-1.2-py2.7.egg-info/PKG-INFO

@@ -0,0 +1,10 @@
+Metadata-Version: 1.0
+Name: KafNafParser
+Version: 1.2
+Summary: Parser between KAF and NAF
+Home-page: https://github.com/cltl/KafNafParserPy
+Author: Ruben Izquierdo
+Author-email: r.izquierdobevia@vu.nl
+License: UNKNOWN
+Description: UNKNOWN
+Platform: UNKNOWN

data/core/site-packages/pre_build/KafNafParser-1.2-py2.7.egg-info/SOURCES.txt

@@ -0,0 +1,22 @@
+KafNafParser/KafNafParserMod.py
+KafNafParser/__init__.py
+KafNafParser/constituency_data.py
+KafNafParser/coreference_data.py
+KafNafParser/dependency_data.py
+KafNafParser/entity_data.py
+KafNafParser/external_references_data.py
+KafNafParser/features_data.py
+KafNafParser/header_data.py
+KafNafParser/opinion_data.py
+KafNafParser/references_data.py
+KafNafParser/span_data.py
+KafNafParser/term_data.py
+KafNafParser/term_sentiment_data.py
+KafNafParser/text_data.py
+KafNafParser.egg-info/PKG-INFO
+KafNafParser.egg-info/SOURCES.txt
+KafNafParser.egg-info/dependency_links.txt
+KafNafParser.egg-info/top_level.txt
+KafNafParser/feature_extractor/__init__.py
+KafNafParser/feature_extractor/constituency.py
+KafNafParser/feature_extractor/dependency.py

data/core/site-packages/pre_build/KafNafParser-1.2-py2.7.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

data/core/site-packages/pre_build/KafNafParser-1.2-py2.7.egg-info/installed-files.txt

@@ -0,0 +1,47 @@
+../KafNafParser/__init__.py
+../KafNafParser/header_data.py
+../KafNafParser/text_data.py
+../KafNafParser/term_data.py
+../KafNafParser/entity_data.py
+../KafNafParser/features_data.py
+../KafNafParser/opinion_data.py
+../KafNafParser/constituency_data.py
+../KafNafParser/dependency_data.py
+../KafNafParser/coreference_data.py
+../KafNafParser/references_data.py
+../KafNafParser/external_references_data.py
+../KafNafParser/span_data.py
+../KafNafParser/KafNafParserMod.py
+../KafNafParser/term_sentiment_data.py
+../KafNafParser/feature_extractor/dependency.py
+../KafNafParser/feature_extractor/constituency.py
+../KafNafParser/feature_extractor/__init__.py
+../KafNafParser/__init__.pyc
+../KafNafParser/header_data.pyc
+../KafNafParser/text_data.pyc
+../KafNafParser/term_data.pyc
+../KafNafParser/entity_data.pyc
+../KafNafParser/features_data.pyc
+../KafNafParser/opinion_data.pyc
+../KafNafParser/constituency_data.pyc
+../KafNafParser/dependency_data.pyc
+../KafNafParser/coreference_data.pyc
+../KafNafParser/references_data.pyc
+../KafNafParser/external_references_data.pyc
+../KafNafParser/span_data.pyc
+../KafNafParser/KafNafParserMod.pyc
+../KafNafParser/term_sentiment_data.pyc
+../KafNafParser/feature_extractor/dependency.pyc
+../KafNafParser/feature_extractor/constituency.pyc
+../KafNafParser/feature_extractor/__init__.pyc
+../../../kaf_example.xml
+../../../naf.dtd
+../../../naf_example.xml
+../../../test.py
+../../../README.md
+../../../LICENSE
+./
+SOURCES.txt
+dependency_links.txt
+top_level.txt
+PKG-INFO

data/core/site-packages/pre_build/KafNafParser-1.2-py2.7.egg-info/top_level.txt

@@ -0,0 +1 @@
+KafNafParser

data/core/site-packages/pre_build/KafNafParser/KafNafParserMod.py

@@ -0,0 +1,338 @@
+## LIST OF CHANGES
+# Ruben 8-nov-2013
+#    + included layers for entities, properties, opinions
+#    + renamed all classes to Cnameoftheclass
+# Ruben 15-nov-2013
+#	+ included constituency layer
+#
+# Ruben 19-nov-2013
+#	+ included dependency layer
+# Ruben 17-dec-2013
+#	+ modified all to red/write NAF and KAF
+#
+# Ruben 21-Feb-2014
+#	+ Included coreference layer
+
+
+__last_modified  = '17dec2013'
+
+from lxml import etree
+from header_data import *
+from text_data import *
+from term_data import *
+from entity_data import *
+from features_data import *
+from opinion_data import *
+from constituency_data import *
+from dependency_data import *
+from feature_extractor import Cdependency_extractor, Cconstituency_extractor
+from coreference_data import *
+from references_data import Creferences
+
+import sys
+
+
+
+class KafNafParser:
+	def __init__(self,filename):
+		self.tree = None
+		self.filename = filename
+		self.tree = etree.parse(filename,etree.XMLParser(remove_blank_text=True))
+		self.root = self.tree.getroot()
+		self.type = self.root.tag # KAF NAF
+
+		self.header = None
+		self.text_layer = None
+		self.term_layer = None
+		self.entity_layer = None
+		self.features_layer = None
+		self.opinion_layer = None
+		self.constituency_layer = None
+		self.dependency_layer = None
+		self.coreference_layer = None
+
+		## Specific feature extractor for complicated layers
+		self.my_dependency_extractor = None
+		self.my_constituency_extractor = None
+		##################################################
+
+		#######
+		self.dict_tokens_for_tid = None
+		##
+
+		self.lang = self.root.get('{http://www.w3.org/XML/1998/namespace}lang')
+		self.version = self.root.get('version')
+
+		if self.type == 'NAF':
+			node_header = self.root.find('nafHeader')
+		elif self.type == 'KAF':
+			node_header = self.root.find('kafHeader')
+
+		if node_header is not None:
+			self.header = CHeader(node_header,self.type)
+
+		# Text layer adapted to naf/kaf
+		node_text = self.root.find('text')
+		if node_text is not None:
+			self.text_layer = Ctext(node=node_text,type=self.type)
+
+		node_term = self.root.find('terms')
+		if node_term is not None:
+			self.term_layer = Cterms(node=node_term,type=self.type)
+
+		node_entity = self.root.find('entities')
+		if node_entity is not None:
+			self.entity_layer = Centities(node_entity,type=self.type)
+
+		node_features = self.root.find('features')
+		if node_features is not None:
+			self.features_layer = Cfeatures(node_features,type=self.type)
+
+		node_opinions = self.root.find('opinions')
+		if node_opinions is not None:
+			self.opinion_layer = Copinions(node_opinions,type=self.type)
+
+		# Definition KAF/NAF is the same
+		node_constituency = self.root.find('constituency')
+		if node_constituency is not None:
+			self.constituency_layer = Cconstituency(node_constituency)
+
+		# Definition KAF/NAF is the same
+		node_dependency = self.root.find('deps')
+		if node_dependency is not None:
+			self.dependency_layer = Cdependencies(node_dependency)
+
+		node_coreferences = self.root.find('coreferences')
+		if node_coreferences is not None:
+			self.coreference_layer = Ccoreferences(node_coreferences,type=self.type)
+
+	def get_type(self):
+		return self.type
+
+	def get_filename(self):
+		return self.filename
+
+	def to_kaf(self):
+		#Convert the root
+		if self.type == 'NAF':
+			self.root.tag = 'KAF'
+			self.type = 'KAF'
+
+		## Convert the header
+		if self.header is not None:
+			self.header.to_kaf()
+
+		## Convert the token layer
+		if self.text_layer is not None:
+			self.text_layer.to_kaf()
+
+		## Convert the term layer
+		if self.term_layer is not None:
+			self.term_layer.to_kaf()
+
+		## Convert the entity layer
+		if self.entity_layer is not None:
+			self.entity_layer.to_kaf()
+
+		## Convert the features layer
+		## There is no feature layer defined in NAF, but we assumed
+		## that is defined will be followin the same rules
+		if self.features_layer is not None:
+			self.features_layer.to_kaf()
+
+
+		##Convert the opinion layer
+		if self.opinion_layer is not None:
+			self.opinion_layer.to_kaf()
+
+		## Convert the constituency layer
+		## This layer is exactly the same in KAF/NAF
+		if self.constituency_layer is not None:
+			self.constituency_layer.to_kaf()	#Does nothing...
+
+
+		## Convert the dedepency layer
+		## It is not defined on KAF so we assme both will be similar
+		if self.dependency_layer is not None:
+			self.dependency_layer.to_kaf()
+
+		if self.coreference_layer is not None:
+			self.coreference_layer.to_kaf()
+
+
+	def to_naf(self):
+		#Convert the root
+		if self.type == 'KAF':
+			self.root.tag = self.type = 'NAF'
+
+		## Convert the header
+		if self.header is not None:
+			self.header.to_naf()
+
+		## Convert the token layer
+		if self.text_layer is not None:
+			self.text_layer.to_naf()
+
+
+		## Convert the term layer
+		if self.term_layer is not None:
+			self.term_layer.to_naf()
+
+
+		## Convert the entity layer
+		if self.entity_layer is not None:
+			self.entity_layer.to_naf()
+
+		## Convert the features layer
+		## There is no feature layer defined in NAF, but we assumed
+		## that is defined will be followin the same rules
+		if self.features_layer is not None:
+			self.features_layer.to_naf()
+
+
+		##Convert the opinion layer
+		if self.opinion_layer is not None:
+			self.opinion_layer.to_naf()
+
+
+		## Convert the constituency layer
+		## This layer is exactly the same in KAF/NAF
+		if self.constituency_layer is not None:
+			self.constituency_layer.to_naf()	#Does nothing...
+
+
+		## Convert the dedepency layer
+		## It is not defined on KAF so we assume both will be similar
+		if self.dependency_layer is not None:
+			self.dependency_layer.to_naf()	  #Does nothing...
+
+		if self.coreference_layer is not None:
+			self.coreference_layer.to_naf()
+
+
+
+	def print_constituency(self):
+		print self.constituency_layer
+
+	def get_trees(self):
+		if self.constituency_layer is not None:
+			for tree in self.constituency_layer.get_trees():
+				yield tree
+
+
+	def get_dependencies(self):
+		if self.dependency_layer is not None:
+			for dep in self.dependency_layer.get_dependencies():
+				yield dep
+
+	def get_language(self):
+		return self.lang
+
+	def get_tokens(self):
+		for token in self.text_layer:
+			yield token
+
+	def get_terms(self):
+		for term in self.term_layer:
+			yield term
+
+	def get_token(self,token_id):
+		if self.text_layer is not None:
+			return self.text_layer.get_wf(token_id)
+		else:
+			return None
+
+	def get_term(self,term_id):
+		if self.term_layer is not None:
+			return self.term_layer.get_term(term_id)
+		else:
+			return None
+
+	def get_properties(self):
+		if self.features_layer is not None:
+			for property in self.features_layer.get_properties():
+				yield property
+
+	def get_entities(self):
+		if self.entity_layer is not None:
+			for entity in self.entity_layer:
+				yield entity
+
+	def get_opinions(self):
+		if self.opinion_layer is not None:
+			for opinion in self.opinion_layer.get_opinions():
+				yield opinion
+
+
+	def dump(self,filename=sys.stdout):
+		self.tree.write(filename,encoding='UTF-8',pretty_print=True,xml_declaration=True)
+
+
+	def remove_opinion_layer(self):
+		if self.opinion_layer is not None:
+			this_node = self.opinion_layer.get_node()
+			self.root.remove(this_node)
+			self.opinion_layer = None
+
+	def remove_term_layer(self):
+		if self.term_layer is not None:
+			this_node = self.term_layer.get_node()
+			self.root.remove(this_node)
+			self.term_layer = None
+
+			if self.header is not None:
+				self.header.remove_lp('terms')
+
+	def get_constituency_extractor(self):
+		if self.constituency_layer is not None:	##Otherwise there are no constituens
+			if self.my_constituency_extractor is None:
+				self.my_constituency_extractor = Cconstituency_extractor(self)
+			return self.my_constituency_extractor
+		else:
+			return None
+
+	def get_dependency_extractor(self):
+		if self.dependency_layer is not None:	#otherwise there are no dependencies
+			if self.my_dependency_extractor is None:
+				self.my_dependency_extractor = Cdependency_extractor(self)
+			return self.my_dependency_extractor
+		else:
+			return None
+
+	## ADDING METHODS
+	def add_wf(self,wf_obj):
+		if self.text_layer is None:
+			self.text_layer = Ctext(type=self.type)
+			self.root.append(self.text_layer.get_node())
+		self.text_layer.add_wf(wf_obj)
+
+	def add_opinion(self,opinion_obj):
+		if self.opinion_layer is None:
+			self.opinion_layer = Copinions()
+			self.root.append(self.opinion_layer.get_node())
+		self.opinion_layer.add_opinion(opinion_obj)
+
+	def add_linguistic_processor(self, layer ,my_lp):
+		self.header.add_linguistic_processor(layer,my_lp)
+
+
+	def add_dependency(self,my_dep):
+		if self.dependency_layer is None:
+			self.dependency_layer = Cdependencies()
+			self.root.append(self.dependency_layer.get_node())
+		self.dependency_layer.add_dependency(my_dep)
+
+
+	## EXTRA FUNCTIONS
+	## Gets the token identifiers in the span of a term id
+	def get_dict_tokens_for_termid(self, term_id):
+		if self.dict_tokens_for_tid is None:
+			self.dict_tokens_for_tid = {}
+			for term in self.get_terms():
+				self.dict_tokens_for_tid[term.get_id()] = term.get_span().get_span_ids()
+
+		return self.dict_tokens_for_tid.get(term_id,[])
+
+
+
+