opener-constituent-parser-nl 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ea12eef17f8f3e0bbf6cdeb7d7f85c1708bdf363
+  data.tar.gz: 436a65c189c21f054b8105393b1a6f4238780907
+SHA512:
+  metadata.gz: 16d4fff1cb2d8eb1f26f008e8af562e1bbeb0e1d2455f3089e276e6d3f1804c524321d1f107e5820f32187a3d6ae5f65d6d8c3bad97c693f93b9c0a9107fe232
+  data.tar.gz: 47c9c43d790a5bc9bc90dbb6833437021f9015ff976efceead9056dd56269fbedfa5ffde3b42d7f3d0e303eabc524c958517445ad144e15c05a68611122367f4

data/README.md

@@ -0,0 +1,41 @@
+[![Build Status](https://drone.io/github.com/opener-project/constituent-parser-nl/status.png)](https://drone.io/github.com/opener-project/constituent-parser-nl/latest)
+
+Constituent-parser-nl
+=======
+
+Introduction
+------------
+
+This is a parser for Dutch text using the Alpino parser (http://www.let.rug.nl/vannoord/alp/Alpino/). The input for this module has to be a valid
+KAF file with at least the text layer. The output will be the constituent trees in pennTreebank format for each of the sentences in the input KAF.
+The tokenization and sentence splitting is taken from the input KAF file, so if your input file has a wrong tokenization/splitting, the output could
+contain errors. The number of output constituent trees will be exactly the same as the number of sentences in your input KAF
+
+Requirements
+-----------
+* VUKafParserPy: parser in python for KAF files (https://github.com/opener-project/VU-kaf-parser)
+* lxml: library for processing xml in python
+* Alpino parser:http://www.let.rug.nl/vannoord/alp/Alpino/
+
+Installation
+-----------
+Clone the repository to your local machine and set the varible ALPINO_HOME in the file core/alpino_parser.py
+to point to your local folder of the Alpino parser.
+
+How to run the module with Python
+---------------------------------
+
+You can run this module from the command line using Python. The main script is core/alpino_parser.py. This script reads the KAF from the standard input
+and writes the output to the standard output, generating some log information in the standard error output. To process one file just run:
+````shell
+cat input.kaf | core/alpino_parser.py > input.tree
+````
+
+This will read the KAF file in "input.kaf" and will store the constituent trees in "input.tree"
+
+
+Contact
+------
+* Ruben Izquierdo
+* Vrije University of Amsterdam
+* ruben.izquierdobevia@vu.nl

data/bin/constituent-parser-nl

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/opener/constituent_parsers/nl'
+
+# STDIN.tty? returns `false` if data is being piped into the current process.
+if STDIN.tty?
+  input = nil
+else
+  input = STDIN.read
+end
+
+kernel = Opener::ConstituentParsers::NL.new(:args => ARGV)
+stdout, stderr, process = kernel.run(input)
+
+puts stdout

data/core/alpino_parser.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python
+
+import sys
+import getopt
+import os
+
+this_folder = os.path.dirname(os.path.realpath(__file__))
+
+# This updates the load path to ensure that the local site-packages directory
+# can be used to load packages (e.g. a locally installed copy of lxml).
+sys.path.append(os.path.join(this_folder, 'site-packages/pre_install'))
+
+import codecs
+from VUKafParserPy import KafParser
+from lxml import etree
+import tempfile
+from subprocess import Popen,PIPE
+import shutil
+import glob
+import logging
+
+from convert_penn_to_kaf import convert_penn_to_kaf_with_numtokens
+## LAST CHANGES ##
+# 20-dec-2013: modified to generate KAF output
+# 15-jan-2014: order in alpino XML does not math the order of tokens
+#       so the label "begin" in the xml is used to know which is the number of token of each <node>
+
+
+last_modified='21Jan2014'
+version="1.4"
+this_name = 'alpino kaf constituency parser'
+this_layer = 'constituents'
+
+#### SET THIS VARIABLE TO YOUR LOCAL FOLDER OF ALPINO
+ALPINO_HOME = os.environ['ALPINO_HOME']
+
+logging.basicConfig(stream=sys.stderr,format='%(asctime)s - %(levelname)s - %(message)s',level=logging.DEBUG)
+
+__module_dir = os.path.dirname(__file__)
+
+## Function to convert to penn treebank bracketd format 
+def node_to_penn(node):
+  children = node.getchildren()
+  if len(children) == 0:
+    word = node.get('word',None)
+    if word is not None:
+      #The attribute begin gives you the number of the token
+      word = word.replace('(','-LRB')
+      word = word.replace(')','-RRB-')
+      
+      
+      num_token = node.get('begin')
+
+      word = num_token+'#'+word
+      if node.get('rel') == 'hd': 
+        head = '=H'
+      else: 
+        head = ''
+      return '('+node.get('pos')+head+' '+word.encode('utf-8')+')'
+    else:
+      return ''
+  else:
+    str = '('+node.get('cat')+' '
+    for n in children:
+      str+=node_to_penn(n)
+    str+=')'
+    return str
+
+    
+def xml_to_penn(filename):
+    
+  ## Under certain condition, there is know bug of Alpino, it sets the encoding in the XML
+  ## to iso-8859-1, but the real encoding is UTF-8. So we need to force to use this encoding
+  
+  parser = etree.XMLParser(encoding='UTF-8')
+  tree = etree.parse(filename,parser)
+  
+  str = node_to_penn(tree.find('node'))
+  return str
+
+if not sys.stdin.isatty(): 
+    ## READING FROM A PIPE
+    pass
+else:
+    print>>sys.stderr,'Input stream required in KAF format at least with the text layer.'
+    print>>sys.stderr,'The language encoded in the KAF has to be Dutch, otherwise it will raise an error.'
+    print>>sys.stderr,'Example usage: cat myUTF8file.kaf.xml |',sys.argv[0]
+    sys.exit(-1)
+
+my_time_stamp = True
+try:
+  opts, args = getopt.getopt(sys.argv[1:],"",["no-time"])
+  for opt, arg in opts:
+    if opt == "--no-time":
+      my_time_stamp = False
+except getopt.GetoptError:
+  pass
+  
+  
+logging.debug('Loading and parsing KAF file ...')
+my_kaf = KafParser(sys.stdin)
+
+lang = my_kaf.getLanguage()
+if lang != 'nl':
+  print>>sys.stdout,'ERROR! Language is ',lang,' and must be nl (Dutch)'
+  sys.exit(-1)
+  
+logging.debug('Extracting sentences from the KAF')
+sentences = []
+current_sent = [] 
+term_ids = []
+current_sent_tid = []
+
+    
+lemma_for_termid = {}
+termid_for_token = {}
+
+for term in my_kaf.getTerms():
+    lemma_for_termid[term.getId()] = term.getLemma()
+    tokens_id = term.get_list_span()
+    for token_id in tokens_id:
+        termid_for_token[token_id] = term.getId()
+ 
+
+previous_sent = None
+for token,sent,token_id in my_kaf.getTokens():
+  ##To avoid using tokens that have no term linked
+  if token_id not in termid_for_token:
+    continue
+  if sent != previous_sent and previous_sent!=None:
+    sentences.append(current_sent)
+    current_sent = [token]
+    term_ids.append(current_sent_tid)
+    current_sent_tid = [termid_for_token[token_id]]
+  else:
+    current_sent.append(token)
+    current_sent_tid.append(termid_for_token[token_id])
+  previous_sent = sent
+  
+if len(current_sent) !=0:
+  sentences.append(current_sent)
+  term_ids.append(current_sent_tid)
+  
+  
+out_folder_alp = tempfile.mkdtemp()
+
+
+logging.debug('Calling to Alpino parser in '+ALPINO_HOME)
+logging.debug('Temporary folder: '+out_folder_alp)
+
+
+## CALL TO ALPINO
+alpino_bin = os.path.join(ALPINO_HOME,'bin','Alpino')
+cmd = alpino_bin+' end_hook=xml -flag treebank '+out_folder_alp+' -parse'
+alpino_pro = Popen(cmd,stdout=PIPE,stdin=PIPE,stderr=PIPE,shell=True)
+
+for sentence in sentences:
+  for token in sentence:
+    token = token.replace('[','\[')
+    token = token.replace(']','\]')
+    token = token.replace('|','\|')
+    #print>>sys.stderr,token.encode('utf-8'),
+    alpino_pro.stdin.write(token.encode('utf-8')+' ')
+  alpino_pro.stdin.write('\n')
+  #print>>sys.stderr
+alpino_pro.stdin.close()
+
+error_log = alpino_pro.stderr.read()
+#print>>sys.stderr,alpino_pro.stderr.read()
+
+# As we are not reading the stdout or stderr of the process, if we dont wait to it to be done
+# the parent will keep running without alpino be completed, and we will get empty XML files
+# If the parent reads from stdout or stderr, it waits to the child to be completed before keep running
+alpino_pro.wait()
+
+
+## There should be as many files as number of sentences in the KAF
+
+const = etree.Element('constituency')
+
+#for xml_file in glob.glob(os.path.join(out_folder_alp,'*.xml')):
+cnt_t = cnt_nt = cnt_edge = 0
+some_error = False
+for num_sent in range(len(sentences)):
+  xml_file = os.path.join(out_folder_alp,str(num_sent+1)+'.xml')    
+  if os.path.exists(xml_file):
+    logging.debug('Converting alpino XML to pennTreebank, sentence num '+str(num_sent+1))
+    penn_str = xml_to_penn(xml_file)
+    tree_node,cnt_t,cnt_nt,cnt_edge = convert_penn_to_kaf_with_numtokens(penn_str,term_ids[num_sent],logging,lemma_for_termid,cnt_t,cnt_nt,cnt_edge)
+  else:
+    tree_node = etree.Element('tree') #empty
+    some_error = True
+  const.append(tree_node)
+
+if some_error:
+  print>>sys.stderr,'POSSIBLE ERROR',error_log
+  value = -1
+else:
+  value = 0
+
+my_kaf.tree.getroot().append(const)
+my_kaf.addLinguisticProcessor(this_name, version+'_'+last_modified, this_layer, my_time_stamp)
+my_kaf.saveToFile(sys.stdout)
+  
+
+logging.debug('Number of sentences in the input KAF:  '+str(len(sentences)))
+logging.debug('PROCESS DONE')
+
+##Remove temporary stuff
+shutil.rmtree(out_folder_alp)
+#print out_folder_alp
+sys.exit(value)

data/core/convert_penn_to_kaf.py

@@ -0,0 +1,161 @@
+from lxml import etree
+from tree import Tree
+import logging
+
+
+
+## will be used as global variables to generate recursively the KAF constituent nodes
+NOTER='nonter'
+TER='ter'
+EDGE='edge'
+noter_cnt=0
+ter_cnt=0
+edge_cnt=0
+
+##This function generates a "tree" xml element as defined in KAF from a string containing
+##the penntreebank format and a list of term ids to do the linking
+'''           
+s = '(S (NP (DET The) (NN dog)) (VP (V ate) (NP (DET the) (NN cat))) (. .))'
+ids = ['t0 t1','t2','t3','t4','t5','t6']
+tree_node = create_constituency_layer(s, ids)
+e = etree.ElementTree(element=tree_node)
+e.write(sys.stdout,pretty_print=True)
+'''
+
+list_t = []
+cnt_t = 0
+list_nt = []
+cnt_nt =0
+list_edge = []
+cnt_edge =0
+
+def convert_penn_to_kaf_with_numtokens(tree_str,term_ids,logging,lemma_for_termid,off_t=0,off_nt=0,off_edge=0):
+    global list_t, list_nt,list_edge,cnt_t, cnt_nt, cnt_edge
+    list_t = []
+    list_nt = []
+    list_edge = []
+    cnt_t = off_t
+    cnt_nt = off_nt
+    cnt_edge = off_edge
+
+    this_tree = Tree(tree_str)
+    logging.debug('\n'+str(this_tree))    ##It has been already encoded using UTF8
+    for num, num_token_and_token in enumerate(this_tree.leaves()):
+        ## token is not used at all
+        ##print num,token,position,token_id
+        p = num_token_and_token.find('#')
+        num_token = int(num_token_and_token[:p])
+        position = this_tree.leaf_treeposition(num)
+        token_id = term_ids[int(num_token)]
+        this_tree[position] = token_id
+        logging.debug('Matching '+num_token_and_token+' with term id='+token_id+'  according to KAF lemma='+str(lemma_for_termid.get(token_id).encode('utf-8')))
+
+    ##Creat the ROOT
+    create_extra_root = False
+    nt_id = None
+    if create_extra_root:
+        nt_id = 'nter'+str(cnt_nt)
+        cnt_nt +=1
+        list_nt.append((nt_id,'ROOT'))
+    
+    visit_node(this_tree, nt_id)
+ 
+    root = etree.Element('tree')
+    nonter_heads = set()
+    #Nonter
+    labels_for_nt = {}
+    for nt_id, label in list_nt:
+        ##Checking the head
+        if len(label)>=2 and label[-1]=='H' and label[-2]=='=':
+            nonter_heads.add(nt_id)
+            label = label[:-2]
+        ele = etree.Element('nt', attrib={'id':nt_id,'label':label})
+        labels_for_nt[nt_id] = label
+        root.append(ele)
+    
+    ## Terminals
+    lemma_for_ter = {}
+    for ter_id, span_ids in list_t:
+        ele = etree.Element('t',attrib={'id':ter_id})
+        span = etree.Element('span')
+        ele.append(span)
+        for termid in span_ids.split(' '):
+            target = etree.Element('target',attrib={'id':termid})
+            span.append(target)
+        lemma_for_ter[ter_id] = lemma_for_termid[termid]
+        root.append(ele)
+        
+    ##Edges
+    #for edge_id,node_to,node_from in list_edge:
+    for edge_id, node_from, node_to in list_edge:
+        ele = etree.Element('edge',attrib={'id':edge_id,'from':node_from,'to':node_to})
+        
+        ## For the comment
+        ##Only non-ter
+        label_to = labels_for_nt.get(node_to)
+        
+        ##Could be ter or nonter
+        label_from = labels_for_nt.get(node_from)
+        if label_from is None:
+            label_from = lemma_for_ter.get(node_from,'kk')
+                                        
+        comment = '  '+(edge_id)+'  '+(label_to)+' <- '+(label_from)+' '
+        comment = comment.replace('--','-')
+        if node_from in nonter_heads:
+            ele.set('head','yes')
+        root.append(etree.Comment(comment))
+        root.append(ele)
+    
+    return root,cnt_t,cnt_nt,cnt_edge
+        
+
+
+def visit_node(node,id_parent=None):
+    global list_t, list_nt,list_edge,cnt_t, cnt_nt, cnt_edge
+
+    if isinstance(node,str): #is a terminal
+        ##Create the terminal
+        t_id = 'ter'+str(cnt_t)
+        cnt_t +=1
+        list_t.append((t_id,str(node)))
+        
+        ##Create the edge with the parent
+        edge_id = 'tre'+str(cnt_edge)
+        cnt_edge +=1
+        list_edge.append((edge_id,t_id,id_parent))
+    else:  #Is a non terminal
+        ##Create the nonterminal
+        nt_id = 'nter'+str(cnt_nt)
+        cnt_nt+=1
+        list_nt.append((nt_id,node.node))
+        
+        ##Create the linking with the parent
+        if id_parent is not None:
+            edge_id = 'tre'+str(cnt_edge)
+            cnt_edge +=1
+            list_edge.append((edge_id,nt_id,id_parent))
+        
+        ##Call to the child
+        for child in node:
+            visit_node(child,nt_id)
+            
+        
+    
+if __name__ == '__main__':
+    s = "(S (NP (DET 0#The) (NN 1#dog)) (VP (V 2#ate) (NP (DET 3#the) (NN 4#cat))) (. 5#.))"
+    ids = ['t0' ,'t1','t2','t3','t4','t5']
+    t= {}
+    t['t0']='The'
+    t['t1']='dog'
+    t['t2']='ate'
+    t['t3']='the'
+    t['t4']='cat'
+    t['t5']='.'
+    root = convert_penn_to_kaf_with_numtokens(s,ids,None,t)
+    import sys
+    etree.ElementTree(element=root).write(sys.stdout,pretty_print=1)
+
+
+
+    
+    

data/core/tree.py

@@ -0,0 +1,1438 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Text Trees
+#
+# Copyright (C) 2001-2012 NLTK Project
+# Author: Edward Loper <edloper@gradient.cis.upenn.edu>
+#         Steven Bird <sb@csse.unimelb.edu.au>
+#         Peter Ljunglöf <peter.ljunglof@gu.se>
+#         Nathan Bodenstab <bodenstab@cslu.ogi.edu> (tree transforms)
+# URL: <http://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Class for representing hierarchical language structures, such as
+syntax trees and morphological trees.
+"""
+
+# TODO: add LabelledTree (can be used for dependency trees)
+
+import re
+import string
+
+######################################################################
+## Trees
+######################################################################
+
+class Tree(list):
+    """
+    A Tree represents a hierarchical grouping of leaves and subtrees.
+    For example, each constituent in a syntax tree is represented by a single Tree.
+
+    A tree's children are encoded as a list of leaves and subtrees,
+    where a leaf is a basic (non-tree) value; and a subtree is a
+    nested Tree.
+
+        >>> from nltk.tree import Tree
+        >>> print Tree(1, [2, Tree(3, [4]), 5])
+        (1 2 (3 4) 5)
+        >>> vp = Tree('VP', [Tree('V', ['saw']),
+        ...                  Tree('NP', ['him'])])
+        >>> s = Tree('S', [Tree('NP', ['I']), vp])
+        >>> print s
+        (S (NP I) (VP (V saw) (NP him)))
+        >>> print s[1]
+        (VP (V saw) (NP him))
+        >>> print s[1,1]
+        (NP him)
+        >>> t = Tree("(S (NP I) (VP (V saw) (NP him)))")
+        >>> s == t
+        True
+        >>> t[1][1].node = "X"
+        >>> print t
+        (S (NP I) (VP (V saw) (X him)))
+        >>> t[0], t[1,1] = t[1,1], t[0]
+        >>> print t
+        (S (X him) (VP (V saw) (NP I)))
+
+    The length of a tree is the number of children it has.
+
+        >>> len(t)
+        2
+
+    Any other properties that a Tree defines are known as node
+    properties, and are used to add information about individual
+    hierarchical groupings.  For example, syntax trees use a NODE
+    property to label syntactic constituents with phrase tags, such as
+    "NP" and "VP".
+
+    Several Tree methods use "tree positions" to specify
+    children or descendants of a tree.  Tree positions are defined as
+    follows:
+
+      - The tree position *i* specifies a Tree's *i*\ th child.
+      - The tree position ``()`` specifies the Tree itself.
+      - If *p* is the tree position of descendant *d*, then
+        *p+i* specifies the *i*\ th child of *d*.
+
+    I.e., every tree position is either a single index *i*,
+    specifying ``tree[i]``; or a sequence *i1, i2, ..., iN*,
+    specifying ``tree[i1][i2]...[iN]``.
+
+    Construct a new tree.  This constructor can be called in one
+    of two ways:
+
+    - ``Tree(node, children)`` constructs a new tree with the
+        specified node value and list of children.
+
+    - ``Tree(s)`` constructs a new tree by parsing the string ``s``.
+        It is equivalent to calling the class method ``Tree.parse(s)``.
+    """
+    def __init__(self, node_or_str, children=None):
+        if children is None: 
+            if not isinstance(node_or_str, basestring):
+                raise TypeError("%s: Expected a node value and child list "
+                                "or a single string" % type(self).__name__)
+            tree = type(self).parse(node_or_str)
+            list.__init__(self, tree)
+            self.node = tree.node
+        elif isinstance(children, basestring):
+            raise TypeError("%s() argument 2 should be a list, not a "
+                            "string" % type(self).__name__)
+        else:
+            list.__init__(self, children)
+            self.node = node_or_str
+
+    #////////////////////////////////////////////////////////////
+    # Comparison operators
+    #////////////////////////////////////////////////////////////
+
+    def __eq__(self, other):
+        if not isinstance(other, Tree): return False
+        return self.node == other.node and list.__eq__(self, other)
+    def __ne__(self, other):
+        return not (self == other)
+    def __lt__(self, other):
+        if not isinstance(other, Tree): return False
+        return self.node < other.node or list.__lt__(self, other)
+    def __le__(self, other):
+        if not isinstance(other, Tree): return False
+        return self.node <= other.node or list.__le__(self, other)
+    def __gt__(self, other):
+        if not isinstance(other, Tree): return True
+        return self.node > other.node or list.__gt__(self, other)
+    def __ge__(self, other):
+        if not isinstance(other, Tree): return False
+        return self.node >= other.node or list.__ge__(self, other)
+
+    #////////////////////////////////////////////////////////////
+    # Disabled list operations
+    #////////////////////////////////////////////////////////////
+
+    def __mul__(self, v):
+        raise TypeError('Tree does not support multiplication')
+    def __rmul__(self, v):
+        raise TypeError('Tree does not support multiplication')
+    def __add__(self, v):
+        raise TypeError('Tree does not support addition')
+    def __radd__(self, v):
+        raise TypeError('Tree does not support addition')
+
+    #////////////////////////////////////////////////////////////
+    # Indexing (with support for tree positions)
+    #////////////////////////////////////////////////////////////
+
+    def __getitem__(self, index):
+        if isinstance(index, (int, slice)):
+            return list.__getitem__(self, index)
+        elif isinstance(index, (list, tuple)):
+            if len(index) == 0:
+                return self
+            elif len(index) == 1:
+                return self[index[0]]
+            else:
+                return self[index[0]][index[1:]]
+        else:
+            raise TypeError("%s indices must be integers, not %s" %
+                            (type(self).__name__, type(index).__name__))
+
+    def __setitem__(self, index, value):
+        if isinstance(index, (int, slice)):
+            return list.__setitem__(self, index, value)
+        elif isinstance(index, (list, tuple)):
+            if len(index) == 0:
+                raise IndexError('The tree position () may not be '
+                                 'assigned to.')
+            elif len(index) == 1:
+                self[index[0]] = value
+            else:
+                self[index[0]][index[1:]] = value
+        else:
+            raise TypeError("%s indices must be integers, not %s" %
+                            (type(self).__name__, type(index).__name__))
+
+    def __delitem__(self, index):
+        if isinstance(index, (int, slice)):
+            return list.__delitem__(self, index)
+        elif isinstance(index, (list, tuple)):
+            if len(index) == 0:
+                raise IndexError('The tree position () may not be deleted.')
+            elif len(index) == 1:
+                del self[index[0]]
+            else:
+                del self[index[0]][index[1:]]
+        else:
+            raise TypeError("%s indices must be integers, not %s" %
+                            (type(self).__name__, type(index).__name__))
+
+    #////////////////////////////////////////////////////////////
+    # Basic tree operations
+    #////////////////////////////////////////////////////////////
+
+    def leaves(self):
+        """
+        Return the leaves of the tree.
+
+            >>> t = Tree("(S (NP (D the) (N dog)) (VP (V chased) (NP (D the) (N cat))))")
+            >>> t.leaves()
+            ['the', 'dog', 'chased', 'the', 'cat']
+
+        :return: a list containing this tree's leaves.
+            The order reflects the order of the
+            leaves in the tree's hierarchical structure.
+        :rtype: list
+        """
+        leaves = []
+        for child in self:
+            if isinstance(child, Tree):
+                leaves.extend(child.leaves())
+            else:
+                leaves.append(child)
+        return leaves
+
+    def flatten(self):
+        """
+        Return a flat version of the tree, with all non-root non-terminals removed.
+
+            >>> t = Tree("(S (NP (D the) (N dog)) (VP (V chased) (NP (D the) (N cat))))")
+            >>> print t.flatten()
+            (S the dog chased the cat)
+
+        :return: a tree consisting of this tree's root connected directly to
+            its leaves, omitting all intervening non-terminal nodes.
+        :rtype: Tree
+        """
+        return Tree(self.node, self.leaves())
+
+    def height(self):
+        """
+        Return the height of the tree.
+
+            >>> t = Tree("(S (NP (D the) (N dog)) (VP (V chased) (NP (D the) (N cat))))")
+            >>> t.height()
+            5
+            >>> print t[0,0]
+            (D the)
+            >>> t[0,0].height()
+            2
+
+        :return: The height of this tree.  The height of a tree
+            containing no children is 1; the height of a tree
+            containing only leaves is 2; and the height of any other
+            tree is one plus the maximum of its children's
+            heights.
+        :rtype: int
+        """
+        max_child_height = 0
+        for child in self:
+            if isinstance(child, Tree):
+                max_child_height = max(max_child_height, child.height())
+            else:
+                max_child_height = max(max_child_height, 1)
+        return 1 + max_child_height
+
+    def treepositions(self, order='preorder'):
+        """
+            >>> t = Tree("(S (NP (D the) (N dog)) (VP (V chased) (NP (D the) (N cat))))")
+            >>> t.treepositions() # doctest: +ELLIPSIS
+            [(), (0,), (0, 0), (0, 0, 0), (0, 1), (0, 1, 0), (1,), (1, 0), (1, 0, 0), ...]
+            >>> for pos in t.treepositions('leaves'):
+            ...     t[pos] = t[pos][::-1].upper()
+            >>> print t
+            (S (NP (D EHT) (N GOD)) (VP (V DESAHC) (NP (D EHT) (N TAC))))
+
+        :param order: One of: ``preorder``, ``postorder``, ``bothorder``,
+            ``leaves``.
+        """
+        positions = []
+        if order in ('preorder', 'bothorder'): positions.append( () )
+        for i, child in enumerate(self):
+            if isinstance(child, Tree):
+                childpos = child.treepositions(order)
+                positions.extend((i,)+p for p in childpos)
+            else:
+                positions.append( (i,) )
+        if order in ('postorder', 'bothorder'): positions.append( () )
+        return positions
+
+    def subtrees(self, filter=None):
+        """
+        Generate all the subtrees of this tree, optionally restricted
+        to trees matching the filter function.
+
+            >>> t = Tree("(S (NP (D the) (N dog)) (VP (V chased) (NP (D the) (N cat))))")
+            >>> for s in t.subtrees(lambda t: t.height() == 2):
+            ...     print s
+            (D the)
+            (N dog)
+            (V chased)
+            (D the)
+            (N cat)
+
+        :type filter: function
+        :param filter: the function to filter all local trees
+        """
+        if not filter or filter(self):
+            yield self
+        for child in self:
+            if isinstance(child, Tree):
+                for subtree in child.subtrees(filter):
+                    yield subtree
+
+    def productions(self):
+        """
+        Generate the productions that correspond to the non-terminal nodes of the tree.
+        For each subtree of the form (P: C1 C2 ... Cn) this produces a production of the
+        form P -> C1 C2 ... Cn.
+
+            >>> t = Tree("(S (NP (D the) (N dog)) (VP (V chased) (NP (D the) (N cat))))")
+            >>> t.productions()
+            [S -> NP VP, NP -> D N, D -> 'the', N -> 'dog', VP -> V NP, V -> 'chased',
+            NP -> D N, D -> 'the', N -> 'cat']
+
+        :rtype: list(Production)
+        """
+
+        if not isinstance(self.node, basestring):
+            raise TypeError, 'Productions can only be generated from trees having node labels that are strings'
+
+        prods = [Production(Nonterminal(self.node), _child_names(self))]
+        for child in self:
+            if isinstance(child, Tree):
+                prods += child.productions()
+        return prods
+
+    def pos(self):
+        """
+        Return a sequence of pos-tagged words extracted from the tree.
+
+            >>> t = Tree("(S (NP (D the) (N dog)) (VP (V chased) (NP (D the) (N cat))))")
+            >>> t.pos()
+            [('the', 'D'), ('dog', 'N'), ('chased', 'V'), ('the', 'D'), ('cat', 'N')]
+
+        :return: a list of tuples containing leaves and pre-terminals (part-of-speech tags).
+            The order reflects the order of the leaves in the tree's hierarchical structure.
+        :rtype: list(tuple)
+        """
+        pos = []
+        for child in self:
+            if isinstance(child, Tree):
+                pos.extend(child.pos())
+            else:
+                pos.append((child, self.node))
+        return pos
+
+    def leaf_treeposition(self, index):
+        """
+        :return: The tree position of the ``index``-th leaf in this
+            tree.  I.e., if ``tp=self.leaf_treeposition(i)``, then
+            ``self[tp]==self.leaves()[i]``.
+
+        :raise IndexError: If this tree contains fewer than ``index+1``
+            leaves, or if ``index<0``.
+        """
+        if index < 0: raise IndexError('index must be non-negative')
+
+        stack = [(self, ())]
+        while stack:
+            value, treepos = stack.pop()
+            if not isinstance(value, Tree):
+                if index == 0: return treepos
+                else: index -= 1
+            else:
+                for i in range(len(value)-1, -1, -1):
+                    stack.append( (value[i], treepos+(i,)) )
+
+        raise IndexError('index must be less than or equal to len(self)')
+
+    def treeposition_spanning_leaves(self, start, end):
+        """
+        :return: The tree position of the lowest descendant of this
+            tree that dominates ``self.leaves()[start:end]``.
+        :raise ValueError: if ``end <= start``
+        """
+        if end <= start:
+            raise ValueError('end must be greater than start')
+        # Find the tree positions of the start & end leaves, and
+        # take the longest common subsequence.
+        start_treepos = self.leaf_treeposition(start)
+        end_treepos = self.leaf_treeposition(end-1)
+        # Find the first index where they mismatch:
+        for i in range(len(start_treepos)):
+            if i == len(end_treepos) or start_treepos[i] != end_treepos[i]:
+                return start_treepos[:i]
+        return start_treepos
+
+    #////////////////////////////////////////////////////////////
+    # Transforms
+    #////////////////////////////////////////////////////////////
+
+    def chomsky_normal_form(self, factor = "right", horzMarkov = None, vertMarkov = 0, childChar = "|", parentChar = "^"):
+        """
+        This method can modify a tree in three ways:
+
+          1. Convert a tree into its Chomsky Normal Form (CNF)
+             equivalent -- Every subtree has either two non-terminals
+             or one terminal as its children.  This process requires
+             the creation of more"artificial" non-terminal nodes.
+          2. Markov (vertical) smoothing of children in new artificial
+             nodes
+          3. Horizontal (parent) annotation of nodes
+
+        :param factor: Right or left factoring method (default = "right")
+        :type  factor: str = [left|right]
+        :param horzMarkov: Markov order for sibling smoothing in artificial nodes (None (default) = include all siblings)
+        :type  horzMarkov: int | None
+        :param vertMarkov: Markov order for parent smoothing (0 (default) = no vertical annotation)
+        :type  vertMarkov: int | None
+        :param childChar: A string used in construction of the artificial nodes, separating the head of the
+                          original subtree from the child nodes that have yet to be expanded (default = "|")
+        :type  childChar: str
+        :param parentChar: A string used to separate the node representation from its vertical annotation
+        :type  parentChar: str
+        """
+        from treetransforms import chomsky_normal_form
+        chomsky_normal_form(self, factor, horzMarkov, vertMarkov, childChar, parentChar)
+
+    def un_chomsky_normal_form(self, expandUnary = True, childChar = "|", parentChar = "^", unaryChar = "+"):
+        """
+        This method modifies the tree in three ways:
+
+          1. Transforms a tree in Chomsky Normal Form back to its
+             original structure (branching greater than two)
+          2. Removes any parent annotation (if it exists)
+          3. (optional) expands unary subtrees (if previously
+             collapsed with collapseUnary(...) )
+
+        :param expandUnary: Flag to expand unary or not (default = True)
+        :type  expandUnary: bool
+        :param childChar: A string separating the head node from its children in an artificial node (default = "|")
+        :type  childChar: str
+        :param parentChar: A sting separating the node label from its parent annotation (default = "^")
+        :type  parentChar: str
+        :param unaryChar: A string joining two non-terminals in a unary production (default = "+")
+        :type  unaryChar: str
+        """
+        from treetransforms import un_chomsky_normal_form
+        un_chomsky_normal_form(self, expandUnary, childChar, parentChar, unaryChar)
+
+    def collapse_unary(self, collapsePOS = False, collapseRoot = False, joinChar = "+"):
+        """
+        Collapse subtrees with a single child (ie. unary productions)
+        into a new non-terminal (Tree node) joined by 'joinChar'.
+        This is useful when working with algorithms that do not allow
+        unary productions, and completely removing the unary productions
+        would require loss of useful information.  The Tree is modified
+        directly (since it is passed by reference) and no value is returned.
+
+        :param collapsePOS: 'False' (default) will not collapse the parent of leaf nodes (ie.
+                            Part-of-Speech tags) since they are always unary productions
+        :type  collapsePOS: bool
+        :param collapseRoot: 'False' (default) will not modify the root production
+                             if it is unary.  For the Penn WSJ treebank corpus, this corresponds
+                             to the TOP -> productions.
+        :type collapseRoot: bool
+        :param joinChar: A string used to connect collapsed node values (default = "+")
+        :type  joinChar: str
+        """
+        from treetransforms import collapse_unary
+        collapse_unary(self, collapsePOS, collapseRoot, joinChar)
+
+    #////////////////////////////////////////////////////////////
+    # Convert, copy
+    #////////////////////////////////////////////////////////////
+
+    @classmethod
+    def convert(cls, tree):
+        """
+        Convert a tree between different subtypes of Tree.  ``cls`` determines
+        which class will be used to encode the new tree.
+
+        :type tree: Tree
+        :param tree: The tree that should be converted.
+        :return: The new Tree.
+        """
+        if isinstance(tree, Tree):
+            children = [cls.convert(child) for child in tree]
+            return cls(tree.node, children)
+        else:
+            return tree
+
+    def copy(self, deep=False):
+        if not deep: return type(self)(self.node, self)
+        else: return type(self).convert(self)
+
+    def _frozen_class(self): return ImmutableTree
+    def freeze(self, leaf_freezer=None):
+        frozen_class = self._frozen_class()
+        if leaf_freezer is None:
+            newcopy = frozen_class.convert(self)
+        else:
+            newcopy = self.copy(deep=True)
+            for pos in newcopy.treepositions('leaves'):
+                newcopy[pos] = leaf_freezer(newcopy[pos])
+            newcopy = frozen_class.convert(newcopy)
+        hash(newcopy) # Make sure the leaves are hashable.
+        return newcopy
+
+    #////////////////////////////////////////////////////////////
+    # Parsing
+    #////////////////////////////////////////////////////////////
+
+    @classmethod
+    def parse(cls, s, brackets='()', parse_node=None, parse_leaf=None,
+              node_pattern=None, leaf_pattern=None,
+              remove_empty_top_bracketing=False):
+        """
+        Parse a bracketed tree string and return the resulting tree.
+        Trees are represented as nested brackettings, such as::
+
+          (S (NP (NNP John)) (VP (V runs)))
+
+        :type s: str
+        :param s: The string to parse
+
+        :type brackets: str (length=2)
+        :param brackets: The bracket characters used to mark the
+            beginning and end of trees and subtrees.
+
+        :type parse_node: function
+        :type parse_leaf: function
+        :param parse_node, parse_leaf: If specified, these functions
+            are applied to the substrings of ``s`` corresponding to
+            nodes and leaves (respectively) to obtain the values for
+            those nodes and leaves.  They should have the following
+            signature:
+
+               parse_node(str) -> value
+
+            For example, these functions could be used to parse nodes
+            and leaves whose values should be some type other than
+            string (such as ``FeatStruct``).
+            Note that by default, node strings and leaf strings are
+            delimited by whitespace and brackets; to override this
+            default, use the ``node_pattern`` and ``leaf_pattern``
+            arguments.
+
+        :type node_pattern: str
+        :type leaf_pattern: str
+        :param node_pattern, leaf_pattern: Regular expression patterns
+            used to find node and leaf substrings in ``s``.  By
+            default, both nodes patterns are defined to match any
+            sequence of non-whitespace non-bracket characters.
+
+        :type remove_empty_top_bracketing: bool
+        :param remove_empty_top_bracketing: If the resulting tree has
+            an empty node label, and is length one, then return its
+            single child instead.  This is useful for treebank trees,
+            which sometimes contain an extra level of bracketing.
+
+        :return: A tree corresponding to the string representation ``s``.
+            If this class method is called using a subclass of Tree,
+            then it will return a tree of that type.
+        :rtype: Tree
+        """
+        if not isinstance(brackets, basestring) or len(brackets) != 2:
+            raise TypeError('brackets must be a length-2 string')
+        if re.search('\s', brackets):
+            raise TypeError('whitespace brackets not allowed')
+        # Construct a regexp that will tokenize the string.
+        open_b, close_b = brackets
+        open_pattern, close_pattern = (re.escape(open_b), re.escape(close_b))
+        if node_pattern is None:
+            node_pattern = '[^\s%s%s]+' % (open_pattern, close_pattern)
+        if leaf_pattern is None:
+            leaf_pattern = '[^\s%s%s]+' % (open_pattern, close_pattern)
+        token_re = re.compile('%s\s*(%s)?|%s|(%s)' % (
+            open_pattern, node_pattern, close_pattern, leaf_pattern))
+        # Walk through each token, updating a stack of trees.
+        stack = [(None, [])] # list of (node, children) tuples
+        for match in token_re.finditer(s):
+            token = match.group()
+            # Beginning of a tree/subtree
+            if token[0] == open_b:
+                if len(stack) == 1 and len(stack[0][1]) > 0:
+                    cls._parse_error(s, match, 'end-of-string')
+                node = token[1:].lstrip()
+                if parse_node is not None: node = parse_node(node)
+                stack.append((node, []))
+            # End of a tree/subtree
+            elif token == close_b:
+                if len(stack) == 1:
+                    if len(stack[0][1]) == 0:
+                        cls._parse_error(s, match, open_b)
+                    else:
+                        cls._parse_error(s, match, 'end-of-string')
+                node, children = stack.pop()
+                stack[-1][1].append(cls(node, children))
+            # Leaf node
+            else:
+                if len(stack) == 1:
+                    cls._parse_error(s, match, open_b)
+                if parse_leaf is not None: token = parse_leaf(token)
+                stack[-1][1].append(token)
+
+        # check that we got exactly one complete tree.
+        if len(stack) > 1:
+            cls._parse_error(s, 'end-of-string', close_b)
+        elif len(stack[0][1]) == 0:
+            cls._parse_error(s, 'end-of-string', open_b)
+        else:
+            assert stack[0][0] is None
+            assert len(stack[0][1]) == 1
+        tree = stack[0][1][0]
+
+        # If the tree has an extra level with node='', then get rid of
+        # it.  E.g.: "((S (NP ...) (VP ...)))"
+        if remove_empty_top_bracketing and tree.node == '' and len(tree) == 1:
+            tree = tree[0]
+        # return the tree.
+        return tree
+
+    @classmethod
+    def _parse_error(cls, s, match, expecting):
+        """
+        Display a friendly error message when parsing a tree string fails.
+        :param s: The string we're parsing.
+        :param match: regexp match of the problem token.
+        :param expecting: what we expected to see instead.
+        """
+        # Construct a basic error message
+        if match == 'end-of-string':
+            pos, token = len(s), 'end-of-string'
+        else:
+            pos, token = match.start(), match.group()
+        msg = '%s.parse(): expected %r but got %r\n%sat index %d.' % (
+            cls.__name__, expecting, token, ' '*12, pos)
+        # Add a display showing the error token itsels:
+        s = s.replace('\n', ' ').replace('\t', ' ')
+        offset = pos
+        if len(s) > pos+10:
+            s = s[:pos+10]+'...'
+        if pos > 10:
+            s = '...'+s[pos-10:]
+            offset = 13
+        msg += '\n%s"%s"\n%s^' % (' '*16, s, ' '*(17+offset))
+        raise ValueError(msg)
+
+    #////////////////////////////////////////////////////////////
+    # Visualization & String Representation
+    #////////////////////////////////////////////////////////////
+
+    def draw(self):
+        """
+        Open a new window containing a graphical diagram of this tree.
+        """
+        from nltk.draw.tree import draw_trees
+        draw_trees(self)
+
+    def __repr__(self):
+        childstr = ", ".join(repr(c) for c in self)
+        return '%s(%r, [%s])' % (type(self).__name__, self.node, childstr)
+
+    def __str__(self):
+        return self.pprint()
+
+    def pprint(self, margin=70, indent=0, nodesep='', parens='()', quotes=False):
+        """
+        :return: A pretty-printed string representation of this tree.
+        :rtype: str
+        :param margin: The right margin at which to do line-wrapping.
+        :type margin: int
+        :param indent: The indentation level at which printing
+            begins.  This number is used to decide how far to indent
+            subsequent lines.
+        :type indent: int
+        :param nodesep: A string that is used to separate the node
+            from the children.  E.g., the default value ``':'`` gives
+            trees like ``(S: (NP: I) (VP: (V: saw) (NP: it)))``.
+        """
+
+        # Try writing it on one line.
+        s = self._pprint_flat(nodesep, parens, quotes)
+        if len(s)+indent < margin:
+            return s
+
+        # If it doesn't fit on one line, then write it on multi-lines.
+        if isinstance(self.node, basestring):
+            s = '%s%s%s' % (parens[0], self.node, nodesep)
+        else:
+            s = '%s%r%s' % (parens[0], self.node, nodesep)
+        for child in self:
+            if isinstance(child, Tree):
+                s += '\n'+' '*(indent+2)+child.pprint(margin, indent+2,
+                                                  nodesep, parens, quotes)
+            elif isinstance(child, tuple):
+                s += '\n'+' '*(indent+2)+ "/".join(child)
+            elif isinstance(child, basestring) and not quotes:
+                s += '\n'+' '*(indent+2)+ '%s' % child
+            else:
+                s += '\n'+' '*(indent+2)+ '%r' % child
+        return s+parens[1]
+
+    def pprint_latex_qtree(self):
+        r"""
+        Returns a representation of the tree compatible with the
+        LaTeX qtree package. This consists of the string ``\Tree``
+        followed by the parse tree represented in bracketed notation.
+
+        For example, the following result was generated from a parse tree of
+        the sentence ``The announcement astounded us``::
+
+          \Tree [.I'' [.N'' [.D The ] [.N' [.N announcement ] ] ]
+              [.I' [.V'' [.V' [.V astounded ] [.N'' [.N' [.N us ] ] ] ] ] ] ]
+
+        See http://www.ling.upenn.edu/advice/latex.html for the LaTeX
+        style file for the qtree package.
+
+        :return: A latex qtree representation of this tree.
+        :rtype: str
+        """
+        return r'\Tree ' + self.pprint(indent=6, nodesep='', parens=('[.', ' ]'))
+
+    def _pprint_flat(self, nodesep, parens, quotes):
+        childstrs = []
+        for child in self:
+            if isinstance(child, Tree):
+                childstrs.append(child._pprint_flat(nodesep, parens, quotes))
+            elif isinstance(child, tuple):
+                childstrs.append("/".join(child))
+            elif isinstance(child, basestring) and not quotes:
+                childstrs.append('%s' % child)
+            else:
+                childstrs.append('%r' % child)
+        if isinstance(self.node, basestring):
+            return '%s%s%s %s%s' % (parens[0], self.node, nodesep,
+                                    string.join(childstrs), parens[1])
+        else:
+            return '%s%r%s %s%s' % (parens[0], self.node, nodesep,
+                                    string.join(childstrs), parens[1])
+
+
+class ImmutableTree(Tree):
+    def __init__(self, node_or_str, children=None):
+        super(ImmutableTree, self).__init__(node_or_str, children)
+        # Precompute our hash value.  This ensures that we're really
+        # immutable.  It also means we only have to calculate it once.
+        try:
+            self._hash = hash( (self.node, tuple(self)) )
+        except (TypeError, ValueError):
+            raise ValueError("%s: node value and children "
+                             "must be immutable" % type(self).__name__)
+
+    def __setitem__(self, index, value):
+        raise ValueError('%s may not be modified' % type(self).__name__)
+    def __setslice__(self, i, j, value):
+        raise ValueError('%s may not be modified' % type(self).__name__)
+    def __delitem__(self, index):
+        raise ValueError('%s may not be modified' % type(self).__name__)
+    def __delslice__(self, i, j):
+        raise ValueError('%s may not be modified' % type(self).__name__)
+    def __iadd__(self, other):
+        raise ValueError('%s may not be modified' % type(self).__name__)
+    def __imul__(self, other):
+        raise ValueError('%s may not be modified' % type(self).__name__)
+    def append(self, v):
+        raise ValueError('%s may not be modified' % type(self).__name__)
+    def extend(self, v):
+        raise ValueError('%s may not be modified' % type(self).__name__)
+    def pop(self, v=None):
+        raise ValueError('%s may not be modified' % type(self).__name__)
+    def remove(self, v):
+        raise ValueError('%s may not be modified' % type(self).__name__)
+    def reverse(self):
+        raise ValueError('%s may not be modified' % type(self).__name__)
+    def sort(self):
+        raise ValueError('%s may not be modified' % type(self).__name__)
+    def __hash__(self):
+        return self._hash
+
+    def _get_node(self):
+        """Get the node value"""
+        return self._node
+    def _set_node(self, value):
+        """
+        Set the node value.  This will only succeed the first time the
+        node value is set, which should occur in ImmutableTree.__init__().
+        """
+        if hasattr(self, 'node'):
+            raise ValueError('%s may not be modified' % type(self).__name__)
+        self._node = value
+    node = property(_get_node, _set_node)
+
+
+######################################################################
+## Parented trees
+######################################################################
+
+class AbstractParentedTree(Tree):
+    """
+    An abstract base class for a ``Tree`` that automatically maintains
+    pointers to parent nodes.  These parent pointers are updated
+    whenever any change is made to a tree's structure.  Two subclasses
+    are currently defined:
+
+      - ``ParentedTree`` is used for tree structures where each subtree
+        has at most one parent.  This class should be used in cases
+        where there is no"sharing" of subtrees.
+
+      - ``MultiParentedTree`` is used for tree structures where a
+        subtree may have zero or more parents.  This class should be
+        used in cases where subtrees may be shared.
+
+    Subclassing
+    ===========
+    The ``AbstractParentedTree`` class redefines all operations that
+    modify a tree's structure to call two methods, which are used by
+    subclasses to update parent information:
+
+      - ``_setparent()`` is called whenever a new child is added.
+      - ``_delparent()`` is called whenever a child is removed.
+    """
+
+    def __init__(self, node_or_str, children=None):
+        super(AbstractParentedTree, self).__init__(node_or_str, children)
+        # If children is None, the tree is parsed from node_or_str, and
+        # all parents will be set during parsing.
+        if children is not None:
+            # Otherwise we have to set the parent of the children.
+            # Iterate over self, and *not* children, because children
+            # might be an iterator.
+            for i, child in enumerate(self):
+                if isinstance(child, Tree):
+                    self._setparent(child, i, dry_run=True)
+            for i, child in enumerate(self):
+                if isinstance(child, Tree):
+                    self._setparent(child, i)
+
+    #////////////////////////////////////////////////////////////
+    # Parent management
+    #////////////////////////////////////////////////////////////
+
+    def _setparent(self, child, index, dry_run=False):
+        """
+        Update the parent pointer of ``child`` to point to ``self``.  This
+        method is only called if the type of ``child`` is ``Tree``;
+        i.e., it is not called when adding a leaf to a tree.  This method
+        is always called before the child is actually added to the
+        child list of ``self``.
+
+        :type child: Tree
+        :type index: int
+        :param index: The index of ``child`` in ``self``.
+        :raise TypeError: If ``child`` is a tree with an impropriate
+            type.  Typically, if ``child`` is a tree, then its type needs
+            to match the type of ``self``.  This prevents mixing of
+            different tree types (single-parented, multi-parented, and
+            non-parented).
+        :param dry_run: If true, the don't actually set the child's
+            parent pointer; just check for any error conditions, and
+            raise an exception if one is found.
+        """
+        raise NotImplementedError()
+
+    def _delparent(self, child, index):
+        """
+        Update the parent pointer of ``child`` to not point to self.  This
+        method is only called if the type of ``child`` is ``Tree``; i.e., it
+        is not called when removing a leaf from a tree.  This method
+        is always called before the child is actually removed from the
+        child list of ``self``.
+
+        :type child: Tree
+        :type index: int
+        :param index: The index of ``child`` in ``self``.
+        """
+        raise NotImplementedError()
+
+    #////////////////////////////////////////////////////////////
+    # Methods that add/remove children
+    #////////////////////////////////////////////////////////////
+    # Every method that adds or removes a child must make
+    # appropriate calls to _setparent() and _delparent().
+
+    def __delitem__(self, index):
+        # del ptree[start:stop]
+        if isinstance(index, slice):
+            start, stop, step = slice_bounds(self, index, allow_step=True)
+            # Clear all the children pointers.
+            for i in xrange(start, stop, step):
+                if isinstance(self[i], Tree):
+                    self._delparent(self[i], i)
+            # Delete the children from our child list.
+            super(AbstractParentedTree, self).__delitem__(index)
+
+        # del ptree[i]
+        elif isinstance(index, int):
+            if index < 0: index += len(self)
+            if index < 0: raise IndexError('index out of range')
+            # Clear the child's parent pointer.
+            if isinstance(self[index], Tree):
+                self._delparent(self[index], index)
+            # Remove the child from our child list.
+            super(AbstractParentedTree, self).__delitem__(index)
+
+        elif isinstance(index, (list, tuple)):
+            # del ptree[()]
+            if len(index) == 0:
+                raise IndexError('The tree position () may not be deleted.')
+            # del ptree[(i,)]
+            elif len(index) == 1:
+                del self[index[0]]
+            # del ptree[i1, i2, i3]
+            else:
+                del self[index[0]][index[1:]]
+
+        else:
+            raise TypeError("%s indices must be integers, not %s" %
+                            (type(self).__name__, type(index).__name__))
+
+    def __setitem__(self, index, value):
+        # ptree[start:stop] = value
+        if isinstance(index, slice):
+            start, stop, step = slice_bounds(self, index, allow_step=True)
+            # make a copy of value, in case it's an iterator
+            if not isinstance(value, (list, tuple)):
+                value = list(value)
+            # Check for any error conditions, so we can avoid ending
+            # up in an inconsistent state if an error does occur.
+            for i, child in enumerate(value):
+                if isinstance(child, Tree):
+                    self._setparent(child, start + i*step, dry_run=True)
+            # clear the child pointers of all parents we're removing
+            for i in xrange(start, stop, step):
+                if isinstance(self[i], Tree):
+                    self._delparent(self[i], i)
+            # set the child pointers of the new children.  We do this
+            # after clearing *all* child pointers, in case we're e.g.
+            # reversing the elements in a tree.
+            for i, child in enumerate(value):
+                if isinstance(child, Tree):
+                    self._setparent(child, start + i*step)
+            # finally, update the content of the child list itself.
+            super(AbstractParentedTree, self).__setitem__(index, value)
+
+        # ptree[i] = value
+        elif isinstance(index, int):
+            if index < 0: index += len(self)
+            if index < 0: raise IndexError('index out of range')
+            # if the value is not changing, do nothing.
+            if value is self[index]:
+                return
+            # Set the new child's parent pointer.
+            if isinstance(value, Tree):
+                self._setparent(value, index)
+            # Remove the old child's parent pointer
+            if isinstance(self[index], Tree):
+                self._delparent(self[index], index)
+            # Update our child list.
+            super(AbstractParentedTree, self).__setitem__(index, value)
+
+        elif isinstance(index, (list, tuple)):
+            # ptree[()] = value
+            if len(index) == 0:
+                raise IndexError('The tree position () may not be assigned to.')
+            # ptree[(i,)] = value
+            elif len(index) == 1:
+                self[index[0]] = value
+            # ptree[i1, i2, i3] = value
+            else:
+                self[index[0]][index[1:]] = value
+
+        else:
+            raise TypeError("%s indices must be integers, not %s" %
+                            (type(self).__name__, type(index).__name__))
+
+    def append(self, child):
+        if isinstance(child, Tree):
+            self._setparent(child, len(self))
+        super(AbstractParentedTree, self).append(child)
+
+    def extend(self, children):
+        for child in children:
+            if isinstance(child, Tree):
+                self._setparent(child, len(self))
+            super(AbstractParentedTree, self).append(child)
+
+    def insert(self, index, child):
+        # Handle negative indexes.  Note that if index < -len(self),
+        # we do *not* raise an IndexError, unlike __getitem__.  This
+        # is done for consistency with list.__getitem__ and list.index.
+        if index < 0: index += len(self)
+        if index < 0: index = 0
+        # Set the child's parent, and update our child list.
+        if isinstance(child, Tree):
+            self._setparent(child, index)
+        super(AbstractParentedTree, self).insert(index, child)
+
+    def pop(self, index=-1):
+        if index < 0: index += len(self)
+        if index < 0: raise IndexError('index out of range')
+        if isinstance(self[index], Tree):
+            self._delparent(self[index], index)
+        return super(AbstractParentedTree, self).pop(index)
+
+    # n.b.: like `list`, this is done by equality, not identity!
+    # To remove a specific child, use del ptree[i].
+    def remove(self, child):
+        index = self.index(child)
+        if isinstance(self[index], Tree):
+            self._delparent(self[index], index)
+        super(AbstractParentedTree, self).remove(child)
+
+    # We need to implement __getslice__ and friends, even though
+    # they're deprecated, because otherwise list.__getslice__ will get
+    # called (since we're subclassing from list).  Just delegate to
+    # __getitem__ etc., but use max(0, start) and max(0, stop) because
+    # because negative indices are already handled *before*
+    # __getslice__ is called; and we don't want to double-count them.
+    if hasattr(list, '__getslice__'):
+        def __getslice__(self, start, stop):
+            return self.__getitem__(slice(max(0, start), max(0, stop)))
+        def __delslice__(self, start, stop):
+            return self.__delitem__(slice(max(0, start), max(0, stop)))
+        def __setslice__(self, start, stop, value):
+            return self.__setitem__(slice(max(0, start), max(0, stop)), value)
+
+class ParentedTree(AbstractParentedTree):
+    """
+    A ``Tree`` that automatically maintains parent pointers for
+    single-parented trees.  The following are methods for querying 
+    the structure of a parented tree: ``parent``, ``parent_index``, 
+    ``left_sibling``, ``right_sibling``, ``root``, ``treeposition``.
+
+    Each ``ParentedTree`` may have at most one parent.  In
+    particular, subtrees may not be shared.  Any attempt to reuse a
+    single ``ParentedTree`` as a child of more than one parent (or
+    as multiple children of the same parent) will cause a
+    ``ValueError`` exception to be raised.
+
+    ``ParentedTrees`` should never be used in the same tree as ``Trees``
+    or ``MultiParentedTrees``.  Mixing tree implementations may result
+    in incorrect parent pointers and in ``TypeError`` exceptions.
+    """
+    def __init__(self, node_or_str, children=None):
+        self._parent = None
+        """The parent of this Tree, or None if it has no parent."""
+        super(ParentedTree, self).__init__(node_or_str, children)
+        if children is None:
+            # If children is None, the tree is parsed from node_or_str.
+            # After parsing, the parent of the immediate children 
+            # will point to an intermediate tree, not self. 
+            # We fix this by brute force:
+            for i, child in enumerate(self):
+                if isinstance(child, Tree):
+                    child._parent = None
+                    self._setparent(child, i)
+
+    def _frozen_class(self): return ImmutableParentedTree
+
+    #/////////////////////////////////////////////////////////////////
+    # Methods
+    #/////////////////////////////////////////////////////////////////
+
+    def parent(self):
+        """The parent of this tree, or None if it has no parent."""
+        return self._parent
+
+    def parent_index(self):
+        """
+        The index of this tree in its parent.  I.e.,
+        ``ptree.parent()[ptree.parent_index()] is ptree``.  Note that
+        ``ptree.parent_index()`` is not necessarily equal to
+        ``ptree.parent.index(ptree)``, since the ``index()`` method
+        returns the first child that is equal to its argument.
+        """
+        if self._parent is None: return None
+        for i, child in enumerate(self._parent):
+            if child is self: return i
+        assert False, 'expected to find self in self._parent!'
+
+    def left_sibling(self):
+        """The left sibling of this tree, or None if it has none."""
+        parent_index = self.parent_index()
+        if self._parent and parent_index > 0:
+            return self._parent[parent_index-1]
+        return None # no left sibling
+
+    def right_sibling(self):
+        """The right sibling of this tree, or None if it has none."""
+        parent_index = self.parent_index()
+        if self._parent and parent_index < (len(self._parent)-1):
+            return self._parent[parent_index+1]
+        return None # no right sibling
+
+    def root(self):
+        """
+        The root of this tree.  I.e., the unique ancestor of this tree
+        whose parent is None.  If ``ptree.parent()`` is None, then
+        ``ptree`` is its own root.
+        """
+        root = self
+        while root.parent() is not None:
+            root = root.parent()
+        return root
+
+    def treeposition(self):
+        """
+        The tree position of this tree, relative to the root of the
+        tree.  I.e., ``ptree.root[ptree.treeposition] is ptree``.
+        """
+        if self.parent() is None: return ()
+        else: return self.parent().treeposition() + (self.parent_index(),)
+
+
+    #/////////////////////////////////////////////////////////////////
+    # Parent Management
+    #/////////////////////////////////////////////////////////////////
+
+    def _delparent(self, child, index):
+        # Sanity checks
+        assert isinstance(child, ParentedTree)
+        assert self[index] is child
+        assert child._parent is self
+
+        # Delete child's parent pointer.
+        child._parent = None
+
+    def _setparent(self, child, index, dry_run=False):
+        # If the child's type is incorrect, then complain.
+        if not isinstance(child, ParentedTree):
+            raise TypeError('Can not insert a non-ParentedTree '+
+                            'into a ParentedTree')
+
+        # If child already has a parent, then complain.
+        if child._parent is not None:
+            raise ValueError('Can not insert a subtree that already '
+                             'has a parent.')
+
+        # Set child's parent pointer & index.
+        if not dry_run:
+            child._parent = self
+
+
+class MultiParentedTree(AbstractParentedTree):
+    """
+    A ``Tree`` that automatically maintains parent pointers for
+    multi-parented trees.  The following are methods for querying the 
+    structure of a multi-parented tree: ``parents()``, ``parent_indices()``, 
+    ``left_siblings()``, ``right_siblings()``, ``roots``, ``treepositions``.
+
+    Each ``MultiParentedTree`` may have zero or more parents.  In
+    particular, subtrees may be shared.  If a single
+    ``MultiParentedTree`` is used as multiple children of the same
+    parent, then that parent will appear multiple times in its
+    ``parents()`` method.
+
+    ``MultiParentedTrees`` should never be used in the same tree as
+    ``Trees`` or ``ParentedTrees``.  Mixing tree implementations may
+    result in incorrect parent pointers and in ``TypeError`` exceptions.
+    """
+    def __init__(self, node_or_str, children=None):
+        self._parents = []
+        """A list of this tree's parents.  This list should not
+           contain duplicates, even if a parent contains this tree
+           multiple times."""
+        super(MultiParentedTree, self).__init__(node_or_str, children)
+        if children is None:
+            # If children is None, the tree is parsed from node_or_str.
+            # After parsing, the parent(s) of the immediate children 
+            # will point to an intermediate tree, not self. 
+            # We fix this by brute force:
+            for i, child in enumerate(self):
+                if isinstance(child, Tree):
+                    child._parents = []
+                    self._setparent(child, i)
+
+    def _frozen_class(self): return ImmutableMultiParentedTree
+
+    #/////////////////////////////////////////////////////////////////
+    # Methods
+    #/////////////////////////////////////////////////////////////////
+
+    def parents(self):
+        """
+        The set of parents of this tree.  If this tree has no parents,
+        then ``parents`` is the empty set.  To check if a tree is used
+        as multiple children of the same parent, use the
+        ``parent_indices()`` method.
+
+        :type: list(MultiParentedTree)
+        """
+        return list(self._parents)
+
+    def left_siblings(self):
+        """
+        A list of all left siblings of this tree, in any of its parent
+        trees.  A tree may be its own left sibling if it is used as
+        multiple contiguous children of the same parent.  A tree may
+        appear multiple times in this list if it is the left sibling
+        of this tree with respect to multiple parents.
+
+        :type: list(MultiParentedTree)
+        """
+        return [parent[index-1]
+                for (parent, index) in self._get_parent_indices()
+                if index > 0]
+
+    def right_siblings(self):
+        """
+        A list of all right siblings of this tree, in any of its parent
+        trees.  A tree may be its own right sibling if it is used as
+        multiple contiguous children of the same parent.  A tree may
+        appear multiple times in this list if it is the right sibling
+        of this tree with respect to multiple parents.
+
+        :type: list(MultiParentedTree)
+        """
+        return [parent[index+1]
+                for (parent, index) in self._get_parent_indices()
+                if index < (len(parent)-1)]
+
+    def _get_parent_indices(self):
+        return [(parent, index)
+                for parent in self._parents
+                for index, child in enumerate(parent)
+                if child is self]
+
+    def roots(self):
+        """
+        The set of all roots of this tree.  This set is formed by
+        tracing all possible parent paths until trees with no parents
+        are found.
+
+        :type: list(MultiParentedTree)
+        """
+        return self._get_roots_helper({}).values()
+
+    def _get_roots_helper(self, result):
+        if self._parents:
+            for parent in self._parents:
+                parent._get_roots_helper(result)
+        else:
+            result[id(self)] = self
+        return result
+
+    def parent_indices(self, parent):
+        """
+        Return a list of the indices where this tree occurs as a child
+        of ``parent``.  If this child does not occur as a child of
+        ``parent``, then the empty list is returned.  The following is
+        always true::
+
+          for parent_index in ptree.parent_indices(parent):
+              parent[parent_index] is ptree
+        """
+        if parent not in self._parents: return []
+        else: return [index for (index, child) in enumerate(parent)
+                      if child is self]
+
+    def treepositions(self, root):
+        """
+        Return a list of all tree positions that can be used to reach
+        this multi-parented tree starting from ``root``.  I.e., the
+        following is always true::
+
+          for treepos in ptree.treepositions(root):
+              root[treepos] is ptree
+        """
+        if self is root:
+            return [()]
+        else:
+            return [treepos+(index,)
+                    for parent in self._parents
+                    for treepos in parent.treepositions(root)
+                    for (index, child) in enumerate(parent) if child is self]
+
+
+    #/////////////////////////////////////////////////////////////////
+    # Parent Management
+    #/////////////////////////////////////////////////////////////////
+
+    def _delparent(self, child, index):
+        # Sanity checks
+        assert isinstance(child, MultiParentedTree)
+        assert self[index] is child
+        assert len([p for p in child._parents if p is self]) == 1
+
+        # If the only copy of child in self is at index, then delete
+        # self from child's parent list.
+        for i, c in enumerate(self):
+            if c is child and i != index: break
+        else:
+            child._parents.remove(self)
+
+    def _setparent(self, child, index, dry_run=False):
+        # If the child's type is incorrect, then complain.
+        if not isinstance(child, MultiParentedTree):
+            raise TypeError('Can not insert a non-MultiParentedTree '+
+                            'into a MultiParentedTree')
+
+        # Add self as a parent pointer if it's not already listed.
+        if not dry_run:
+            for parent in child._parents:
+                if parent is self: break
+            else:
+                child._parents.append(self)
+
+class ImmutableParentedTree(ImmutableTree, ParentedTree):
+    pass
+
+class ImmutableMultiParentedTree(ImmutableTree, MultiParentedTree):
+    pass
+
+
+def _child_names(tree):
+    names = []
+    for child in tree:
+        if isinstance(child, Tree):
+            names.append(Nonterminal(child.node))
+        else:
+            names.append(child)
+    return names
+
+######################################################################
+## Parsing
+######################################################################
+
+def bracket_parse(s):
+    """
+    Use Tree.parse(s, remove_empty_top_bracketing=True) instead.
+    """
+    raise NameError("Use Tree.parse(s, remove_empty_top_bracketing=True) instead.")
+
+def sinica_parse(s):
+    """
+    Parse a Sinica Treebank string and return a tree.  Trees are represented as nested brackettings,
+    as shown in the following example (X represents a Chinese character):
+    S(goal:NP(Head:Nep:XX)|theme:NP(Head:Nhaa:X)|quantity:Dab:X|Head:VL2:X)#0(PERIODCATEGORY)
+
+    :return: A tree corresponding to the string representation.
+    :rtype: Tree
+    :param s: The string to be converted
+    :type s: str
+    """
+    tokens = re.split(r'([()| ])', s)
+    for i in range(len(tokens)):
+        if tokens[i] == '(':
+            tokens[i-1], tokens[i] = tokens[i], tokens[i-1]     # pull nonterminal inside parens
+        elif ':' in tokens[i]:
+            fields = tokens[i].split(':')
+            if len(fields) == 2:                                # non-terminal
+                tokens[i] = fields[1]
+            else:
+                tokens[i] = "(" + fields[-2] + " " + fields[-1] + ")"
+        elif tokens[i] == '|':
+            tokens[i] = ''
+
+    treebank_string = string.join(tokens)
+    return Tree.parse(treebank_string, remove_empty_top_bracketing=True)
+
+#    s = re.sub(r'^#[^\s]*\s', '', s)  # remove leading identifier
+#    s = re.sub(r'\w+:', '', s)       # remove role tags
+
+#    return s
+
+######################################################################
+## Demonstration
+######################################################################
+
+def demo():
+    """
+    A demonstration showing how Trees and Trees can be
+    used.  This demonstration creates a Tree, and loads a
+    Tree from the Treebank corpus,
+    and shows the results of calling several of their methods.
+    """
+
+    from nltk import tree
+
+    # Demonstrate tree parsing.
+    s = '(S (NP (DT the) (NN cat)) (VP (VBD ate) (NP (DT a) (NN cookie))))'
+    t = Tree(s)
+    print "Convert bracketed string into tree:"
+    print t
+    print t.__repr__()
+
+    print "Display tree properties:"
+    print t.node           # tree's constituent type
+    print t[0]             # tree's first child
+    print t[1]             # tree's second child
+    print t.height()
+    print t.leaves()
+    print t[1]
+    print t[1,1]
+    print t[1,1,0]
+
+    # Demonstrate tree modification.
+    the_cat = t[0]
+    the_cat.insert(1, tree.Tree.parse('(JJ big)'))
+    print "Tree modification:"
+    print t
+    t[1,1,1] = tree.Tree.parse('(NN cake)')
+    print t
+    print
+
+    # Tree transforms
+    print "Collapse unary:"
+    t.collapse_unary()
+    print t
+    print "Chomsky normal form:"
+    t.chomsky_normal_form()
+    print t
+    print
+
+    # Demonstrate probabilistic trees.
+    pt = tree.ProbabilisticTree('x', ['y', 'z'], prob=0.5)
+    print "Probabilistic Tree:"
+    print pt
+    print
+
+    # Demonstrate parsing of treebank output format.
+    t = tree.Tree.parse(t.pprint())
+    print "Convert tree to bracketed string and back again:"
+    print t
+    print
+
+    # Demonstrate LaTeX output
+    print "LaTeX output:"
+    print t.pprint_latex_qtree()
+    print
+
+    # Demonstrate Productions
+    print "Production output:"
+    print t.productions()
+    print
+
+    # Demonstrate tree nodes containing objects other than strings
+    t.node = ('test', 3)
+    print t
+
+__all__ = ['ImmutableProbabilisticTree', 'ImmutableTree', 'ProbabilisticMixIn',
+           'ProbabilisticTree', 'Tree', 'bracket_parse',
+           'sinica_parse', 'ParentedTree', 'MultiParentedTree',
+           'ImmutableParentedTree', 'ImmutableMultiParentedTree']
+
+if __name__ == "__main__":
+    import doctest
+    doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
+