opener-property-tagger 2.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 449add6937b48b2b9c17a8174cc5790a2d366088
+  data.tar.gz: b64fc05e7b01f9720969607637e94e891a57c614
+SHA512:
+  metadata.gz: 24d9e2caf35674b6e33e495ba84467198b6adbfcfe3dab50e586e6ec6467e61b24e73810164a426ef7d4c7fdbecbea90255d0bc0caeff09585015505ae13d16f
+  data.tar.gz: 7969b6cb5409ab4c6141e68ef708ac85fa9fa8169c476c696fe40bc5c6452dfc3cdd590cce40bf306370d1a7f7f1901f19206d53cef63015eaa79fedfea31080

data/README.md

@@ -0,0 +1,62 @@
+#property-tagger#
+
+#Introduction#
+
+This module implements a tagger for hotel properties for Dutch, English, French, Italian, Spanish and German. It detects aspect words,
+for instance words related with "room", "cleanliness", "staff" or "breakfast" and links them with the correct aspect class.
+The input for this module has to be a valid KAF file with at lest the term layer, as the lemmas will be used for detecting the hotel properties. The output is also
+a KAF valid file extended with the property layer. This module works for all the languages within the OpeNER project (en,de,nl,fr,es,it) and the language is read from
+the input KAF file, from the lang attribute of the KAF element  (make suresure your preprocessors set properly this value or you might use the resources for a wrong language)
+````shell
+<KAF xml:lang="( en | nl | fr | it | de | es)">
+````
+
+#Requirements#
+* VUKafParserPy: parser in python for KAF files (https://github.com/opener-project/VU-kaf-parser)
+* lxml: library for processing xml in python
+
+
+
+#How to run the module with Python#
+You can run this module from the command line using Python. The main script is core/hotel_property_tagger_nl_en.py. This reads
+and writes the output to the standard output, generating some log information in the standard error output. To process the file
+"input.kaf" just type:
+````shell
+cat input.kaf | core/hotel_property_tagger_nl_en.py > output.kaf
+````
+
+This will read the KAF file in "input.kaf" and will store the constituent trees in "input.tree".
+
+The options of the program can be obtained calling to the program with the parameter -h:
+````shell
+core/hotel_property_tagger_nl_en.py -h
+usage: hotel_property_tagger_nl_en.py [-h] [--no-time] [--lexicon LEXICON]
+
+Tags a text with polarities at lemma level
+
+optional arguments:
+  -h, --help         show this help message and exit
+  --no-time          For not including timestamp in header
+  --lexicon LEXICON  Force to use this lexicon
+````
+
+The most important option is the --lexicon, which allows you to use your own lexicon with the program. The lexicon must be stored in a file and follow this format:
+````shell
+surf	verb	facilities
+surfer	noun	facilities
+surfing	verb	facilities
+```
+
+So, one aspect per line, with 3 fields separated by a tabulator, the first one is the word or span of words (in this case use whitespaces), then the part
+of speech (which actually it is not use, you can  include a dummy label) and finally the aspect class associated with the word. If you have created your lexicon
+in one file you could call to the program in this fashion:
+````shell
+$ cat my_input.kaf | python core/hotel_property_tagger_nl_en.py --lexicon path/to/your/lexicon/my_lexicon.txt > my_output.kaf
+````
+
+Contact
+------
+* Ruben Izquierdo
+* Vrije University of Amsterdam
+* ruben.izquierdobevia@vu.nl
+

data/bin/property-tagger

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/opener/property_tagger'
+
+cli = Opener::PropertyTagger::CLI.new(:args => ARGV)
+
+cli.run(STDIN.tty? ? nil : STDIN.read)

data/bin/property-tagger-server

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+require 'puma/cli'
+
+rack_config = File.expand_path('../../config.ru', __FILE__)
+
+cli = Puma::CLI.new([rack_config] + ARGV)
+cli.run

data/config.ru

@@ -0,0 +1,5 @@
+require File.expand_path('../lib/opener/property_tagger/hotel', __FILE__)
+require File.expand_path('../lib/opener/property_tagger/server', __FILE__)
+
+run Opener::PropertyTagger::Server
+

data/core/extract_aspects.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+
+from lxml import etree
+import sys
+#filename='/Users/ruben/CODE/VU-sentiment-lexicon-xml/VUSentimentLexicon/EN-lexicon/Sentiment-English-HotelDomain.xml'
+
+root = etree.parse(sys.stdin).getroot()
+
+for element in root.findall('Lexicon/LexicalEntry'):
+  ele_lemma = element.findall('Lemma')[0]
+  ele_domain = element.findall('Sense/Domain')[0]
+  pos = element.get('partOfSpeech','unknown_pos')
+  if ele_lemma is not None and ele_domain is not None:
+      lemma = ele_lemma.get('writtenForm','').lower()
+      aspect = ele_domain.get('aspect','').lower()
+      if lemma!='' and aspect!='':
+          print lemma.encode('utf-8')+'\t'+pos.encode('utf-8')+'\t'+aspect.encode('utf-8')
+

data/core/hotel_property_tagger_nl_en.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python
+
+import sys
+import argparse
+
+import codecs
+import os
+
+this_folder = os.path.dirname(os.path.realpath(__file__))
+
+# This updates the load path to ensure that the local site-packages directory
+# can be used to load packages (e.g. a locally installed copy of lxml).
+sys.path.append(os.path.join(this_folder, 'site-packages/pre_build'))
+
+from VUKafParserPy import KafParser
+from lxml import etree
+from collections import defaultdict
+
+__desc='VUA property tagger'
+__last_edited='4nov2013'
+__version='1.0'
+
+###
+__module_dir = os.path.dirname(__file__)
+max_ngram = 1
+verbose = False
+##
+
+
+########################################
+## Format of the file:
+#lemma pos aspect
+#lemma pos aspect
+########################################
+def loadAspects(my_lang,this_file=None):
+  my_aspects = {}
+  if this_file is not None:
+    aspects_filename = this_file
+  else:
+    aspects_filename = os.path.join(__module_dir,'data',my_lang,'aspects.txt')
+  
+  if not os.path.exists(aspects_filename):
+    print>>sys.stderr,'ERROR: file with aspects for the language',my_lang,'not found in',aspects_filename
+  else:
+    fic = codecs.open(aspects_filename,'r','utf-8')
+    for line in fic:
+      fields = line.strip().split('\t')
+      lemma,pos,aspect = fields
+      my_aspects[lemma] = aspect
+    fic.close()
+  return aspects_filename, my_aspects
+########################################
+
+
+
+###### MAIN ########
+
+argument_parser = argparse.ArgumentParser(description='Tags a text with polarities at lemma level')
+argument_parser.add_argument("--no-time",action="store_false", default=True, dest="my_time_stamp",help="For not including timestamp in header") 
+argument_parser.add_argument("--lexicon", action="store", default=None, dest="lexicon", help="Force to use this lexicon")
+
+arguments = argument_parser.parse_args()
+
+if not sys.stdin.isatty():
+    ## READING FROM A PIPE
+    pass
+else:
+    print>>sys.stderr,'Input stream required.'
+    print>>sys.stderr,'Example usage: cat myUTF8file.kaf.xml |',sys.argv[0]
+    print>>sys.stderr,sys.argv[0]+' -h  for help'
+    sys.exit(-1)
+
+
+## Load the tree and the list of terms with the id
+my_data = []
+try:
+  my_kaf_tree = KafParser(sys.stdin)
+except Exception as e:
+  print>>sys.stdout,'Error parsing input. Input is required to be KAF'
+  print>>sys.stdout,str(e)
+  sys.exit(2)
+
+
+## Get language from the KAF file
+my_lang  = my_kaf_tree.getLanguage()
+
+my_aspects_filename = my_aspects = None
+if arguments.lexicon is None:
+  if my_lang not in ['nl','en','de','fr','it','es']:
+    print>>sys.stdout,'Error in the language specified in your KAF. The language is ',my_lang,' and possible values for this module '
+    print>>sys.stdout,'are nl for Dutch ,en for English, es Spanish, fr French, it Italian or de German'
+    sys.exit(1)
+
+  my_aspects_filename, my_aspects = loadAspects(my_lang)
+else:
+  my_aspects_filename, my_aspects = loadAspects(my_lang,this_file=arguments.lexicon)
+  
+if verbose:
+  print>>sys.stderr,'Loaded ',len(my_aspects),'aspects from',my_aspects_filename
+
+
+for term in my_kaf_tree.getTerms():
+    my_data.append((term.getLemma(),term.getId()))
+if verbose: print>>sys.stderr,'Number of terms in the kaf file:',len(my_data)
+
+
+current_token = found = 0
+uniq_aspects = defaultdict(list)
+while current_token < len(my_data):
+    for tam_ngram in range(1,max_ngram+1):
+        # Build an n-gram of size tam_ngram and beginning in current_token
+        if current_token + tam_ngram <=  len(my_data):
+            ngram = ' '.join(lemma for lemma,_ in my_data[current_token:current_token+tam_ngram])
+            aspect = my_aspects.get(ngram,None)
+            if aspect is not None:
+                list_of_ids = [id for _,id in my_data[current_token:current_token+tam_ngram]]
+                uniq_aspects[aspect].append((list_of_ids,ngram))
+    current_token += 1
+
+
+## Code for generating the propery layer included in the Parser
+for aspect, list_of_lists in uniq_aspects.items():
+  for list_of_ids, str_text in list_of_lists:
+    my_kaf_tree.add_property(aspect,list_of_ids,str_text)
+
+my_kaf_tree.addLinguisticProcessor(__desc,__last_edited+'_'+__version,'features', arguments.my_time_stamp)
+my_kaf_tree.saveToFile(sys.stdout)
+
+
+
+
+
+

data/core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/PKG-INFO

@@ -0,0 +1,10 @@
+Metadata-Version: 1.0
+Name: VUKafParserPy
+Version: 1.0
+Summary: Library in python to parse kaf files
+Home-page: UNKNOWN
+Author: Ruben Izquierdo
+Author-email: r.izquierdobevia@vu.nl
+License: UNKNOWN
+Description: UNKNOWN
+Platform: UNKNOWN

data/core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/SOURCES.txt

@@ -0,0 +1,7 @@
+VUKafParserPy/KafDataObjectsMod.py
+VUKafParserPy/KafParserMod.py
+VUKafParserPy/__init__.py
+VUKafParserPy.egg-info/PKG-INFO
+VUKafParserPy.egg-info/SOURCES.txt
+VUKafParserPy.egg-info/dependency_links.txt
+VUKafParserPy.egg-info/top_level.txt

data/core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

data/core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/installed-files.txt

@@ -0,0 +1,11 @@
+../VUKafParserPy/KafParserMod.py
+../VUKafParserPy/__init__.py
+../VUKafParserPy/KafDataObjectsMod.py
+../VUKafParserPy/KafParserMod.pyc
+../VUKafParserPy/__init__.pyc
+../VUKafParserPy/KafDataObjectsMod.pyc
+./
+top_level.txt
+SOURCES.txt
+PKG-INFO
+dependency_links.txt

data/core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/top_level.txt

@@ -0,0 +1 @@
+VUKafParserPy

data/core/site-packages/pre_build/VUKafParserPy/KafDataObjectsMod.py

@@ -0,0 +1,165 @@
+class KafTermSentiment:
+  def __init__(self):
+    self.resource=None
+    self.polarity=None
+    self.strength=None
+    self.subjectivity=None
+    
+  def simpleInit(self,r,p,st,su,sm=None):
+    self.resource=r
+    self.polarity=p
+    self.strength=st
+    self.subjectivity=su
+    self.sentiment_modifier = sm
+    
+  def getPolarity(self):
+    return self.polarity
+  
+  def getSentimentModifier(self):
+    return self.sentiment_modifier
+    
+    
+class KafToken:
+  def __init__(self,wid, value, sent=None, para=None):
+    self.token_id = wid
+    self.value = value
+    self.sent = sent
+    self.para = para
+  
+
+class KafOpinionExpression:
+  def __init__(self,polarity,strength,targets):
+    self.polarity = polarity
+    self.strength = strength
+    self.targets = targets
+    
+  def __str__(self):
+    return 'Op_exp==> pol:'+self.polarity+' Str:'+self.strength+' ids:'+'-'.join(self.targets)
+    
+class KafOpinion:
+  def __init__(self,id,holders, targets, opi_exp):
+    self.id = id
+    self.holders = holders
+    self.targets = targets
+    self.opi_exp = opi_exp
+    
+  def __str__(self):
+    c='Opinion id'+self.id+'\n'
+    c+='  Holders: '+'-'.join(self.holders)+'\n'
+    c+='  Targets: '+'-'.join(self.targets)+'\n'
+    c+=str(self.opi_exp)
+    return c
+    
+    
+
+class KafSingleProperty:
+  def __init__(self,id,type,targets):
+    self.id = id
+    self.type = type
+    self.targets = targets
+    
+    
+  def get_id(self):
+    return self.id
+    
+  def get_type(self):
+    return self.type
+  
+  def get_span(self):
+    return self.targets
+  
+  def __str__(self):
+    return 'Id: '+self.id+' Type: '+self.type+' ids:'+' '.join(self.targets)
+
+
+class KafSingleEntity:
+  def __init__(self,id,type,targets):
+    self.id = id
+    self.type = type
+    self.targets = targets
+    
+  def get_id(self):
+    return self.id
+    
+  def get_type(self):
+    return self.type
+  
+  def get_span(self):
+    return self.targets
+  
+  def __str__(self):
+    return 'Id: '+self.id+' Type: '+self.type+' ids:'+' '.join(self.targets)
+
+class KafTerm:
+  def __init__(self):
+    self.tid = None
+    self.lemma = None
+    self.pos = None
+    self.morphofeat = None
+    self.sentiment = None
+    self.list_span_id = []
+    
+  def get_morphofeat(self):
+    return self.morphofeat
+    
+  def set_list_span_id(self, L):
+    self.list_span_id = L
+    
+  def get_list_span(self):
+    return self.list_span_id
+    
+  def get_polarity(self):
+    if self.sentiment != None:
+      return self.sentiment.getPolarity()
+    else:
+      return None
+      
+  def get_sentiment_modifier(self):
+    if self.sentiment != None:
+      return self.sentiment.getSentimentModifier()
+    else:
+      return None
+    
+    
+  def setSentiment(self,my_sent):
+    self.sentiment = my_sent
+    
+  def getSentiment(self):
+    return self.sentiment
+    
+  def getLemma(self): 
+      return self.lemma
+    
+  def setLemma(self,lemma):
+      self.lemma = lemma
+    
+  def getPos(self):
+      return self.pos
+    
+  def setPos(self,pos):
+      self.pos = pos
+      
+  def getId(self):
+      return self.tid
+      
+  def setId(self,id):
+      self.tid = id
+
+  def getShortPos(self):
+    if self.pos==None:
+      return None
+    auxpos=self.pos.lower()[0]
+    if auxpos == 'g': auxpos='a'
+    elif auxpos == 'a': auxpos='r'
+    return auxpos
+    
+  def __str__(self):
+    if self.tid and self.lemma and self.pos:
+        return self.tid+'\n\t'+self.lemma.encode('utf-8')+'\n\t'+self.pos
+    else:
+        return 'None'
+      
+    
+    
+    
+