bee_python 0.0.1

data/tools/epydoc/docstringparser.py

@@ -0,0 +1,1111 @@
+# epydoc -- Docstring processing
+#
+# Copyright (C) 2005 Edward Loper
+# Author: Edward Loper <edloper@loper.org>
+# URL: <http://epydoc.sf.net>
+#
+# $Id: docstringparser.py,v 1.1 2009/11/26 13:20:41 casa Exp $
+
+"""
+Parse docstrings and handle any fields it defines, such as C{@type}
+and C{@author}.  Fields are used to describe specific information
+about an object.  There are two classes of fields: X{simple fields}
+and X{special fields}.
+
+Simple fields are fields that get stored directly in an C{APIDoc}'s
+metadata dictionary, without any special processing.  The set of
+simple fields is defined by the list L{STANDARD_FIELDS}, whose
+elements are L{DocstringField}s.
+
+Special fields are fields that perform some sort of processing on the
+C{APIDoc}, or add information to attributes other than the metadata
+dictionary.  Special fields are are handled by field handler
+functions, which are registered using L{register_field_handler}.
+"""
+__docformat__ = 'epytext en'
+
+
+######################################################################
+## Imports
+######################################################################
+
+import re, sys
+from epydoc import markup
+from epydoc.markup import epytext
+from epydoc.apidoc import *
+from epydoc.docintrospecter import introspect_docstring_lineno
+from epydoc.util import py_src_filename
+from epydoc import log
+import epydoc.docparser
+import __builtin__, exceptions
+
+######################################################################
+# Docstring Fields
+######################################################################
+
+class DocstringField:
+    """
+    A simple docstring field, which can be used to describe specific
+    information about an object, such as its author or its version.
+    Simple docstring fields are fields that take no arguments, and
+    are displayed as simple sections.
+
+    @ivar tags: The set of tags that can be used to identify this
+        field.
+    @ivar singular: The label that should be used to identify this
+        field in the output, if the field contains one value.
+    @ivar plural: The label that should be used to identify this
+        field in the output, if the field contains multiple values.
+    @ivar short: If true, then multiple values should be combined
+        into a single comma-delimited list.  If false, then
+        multiple values should be listed separately in a bulleted
+        list.
+    @ivar multivalue: If true, then multiple values may be given
+        for this field; if false, then this field can only take a
+        single value, and a warning should be issued if it is
+        redefined.
+    @ivar takes_arg: If true, then this field expects an argument;
+        and a separate field section will be constructed for each
+        argument value.  The label (and plural label) should include
+        a '%s' to mark where the argument's string rep should be
+        added.
+    """
+    def __init__(self, tags, label, plural=None,
+                 short=0, multivalue=1, takes_arg=0,
+                 varnames=None):
+        if type(tags) in (list, tuple):
+            self.tags = tuple(tags)
+        elif type(tags) is str:
+            self.tags = (tags,)
+        else: raise TypeError('Bad tags: %s' % tags)
+        self.singular = label
+        if plural is None: self.plural = label
+        else: self.plural = plural
+        self.multivalue = multivalue
+        self.short = short
+        self.takes_arg = takes_arg
+        self.varnames = varnames or []
+
+    def __cmp__(self, other):
+        if not isinstance(other, DocstringField): return -1
+        return cmp(self.tags, other.tags)
+    
+    def __hash__(self):
+        return hash(self.tags)
+
+    def __repr__(self):
+        return '<Field: %s>' % self.tags[0]
+
+STANDARD_FIELDS = [
+    #: A list of the standard simple fields accepted by epydoc.  This
+    #: list can be augmented at run-time by a docstring with the special
+    #: C{@deffield} field.  The order in which fields are listed here
+    #: determines the order in which they will be displayed in the
+    #: output.
+    
+    # If it's deprecated, put that first.
+    DocstringField(['deprecated', 'depreciated'],
+             'Deprecated', multivalue=0, varnames=['__deprecated__']),
+
+    # Status info
+    DocstringField(['version'], 'Version', multivalue=0,
+                   varnames=['__version__']),
+    DocstringField(['date'], 'Date', multivalue=0,
+                   varnames=['__date__']),
+    DocstringField(['status'], 'Status', multivalue=0),
+    
+    # Bibliographic Info
+    DocstringField(['author', 'authors'], 'Author', 'Authors', short=1,
+                   varnames=['__author__', '__authors__']),
+    DocstringField(['contact'], 'Contact', 'Contacts', short=1,
+                   varnames=['__contact__']),
+    DocstringField(['organization', 'org'],
+                   'Organization', 'Organizations'),
+    DocstringField(['copyright', '(c)'], 'Copyright', multivalue=0,
+                   varnames=['__copyright__']),
+    DocstringField(['license'], 'License', multivalue=0,
+                   varnames=['__license__']),
+
+    # Various warnings etc.
+    DocstringField(['bug'], 'Bug', 'Bugs'),
+    DocstringField(['warning', 'warn'], 'Warning', 'Warnings'),
+    DocstringField(['attention'], 'Attention'),
+    DocstringField(['note'], 'Note', 'Notes'),
+
+    # Formal conditions
+    DocstringField(['requires', 'require', 'requirement'], 'Requires'),
+    DocstringField(['precondition', 'precond'],
+             'Precondition', 'Preconditions'),
+    DocstringField(['postcondition', 'postcond'],
+             'Postcondition', 'Postconditions'),
+    DocstringField(['invariant'], 'Invariant'),
+
+    # When was it introduced (version # or date)
+    DocstringField(['since'], 'Since', multivalue=0),
+
+    # Changes made
+    DocstringField(['change', 'changed'], 'Change Log'),
+                   
+    # Crossreferences
+    DocstringField(['see', 'seealso'], 'See Also', short=1),
+
+    # Future Work
+    DocstringField(['todo'], 'To Do', takes_arg=True),
+
+    # Permissions (used by zope-based projects)
+    DocstringField(['permission', 'permissions'], 'Permission', 'Permissions')
+    ]
+
+######################################################################
+#{ Docstring Parsing
+######################################################################
+
+DEFAULT_DOCFORMAT = 'epytext'
+"""The name of the default markup languge used to process docstrings."""
+
+# [xx] keep track of which ones we've already done, in case we're
+# asked to process one twice?  e.g., for @include we might have to
+# parse the included docstring earlier than we might otherwise..??
+
+def parse_docstring(api_doc, docindex, suppress_warnings=[]):
+    """
+    Process the given C{APIDoc}'s docstring.  In particular, populate
+    the C{APIDoc}'s C{descr} and C{summary} attributes, and add any
+    information provided by fields in the docstring.
+    
+    @param docindex: A DocIndex, used to find the containing
+        module (to look up the docformat); and to find any
+        user docfields defined by containing objects.
+    @param suppress_warnings: A set of objects for which docstring
+        warnings should be suppressed.
+    """
+    if api_doc.metadata is not UNKNOWN:
+        if not (isinstance(api_doc, RoutineDoc)
+                and api_doc.canonical_name[-1] == '__init__'):
+            log.debug("%s's docstring processed twice" %
+                      api_doc.canonical_name)
+        return
+        
+    initialize_api_doc(api_doc)
+
+    # If there's no docstring, then check for special variables (e.g.,
+    # __version__), and then return -- there's nothing else to do.
+    if (api_doc.docstring in (None, UNKNOWN)):
+        if isinstance(api_doc, NamespaceDoc):
+            for field in STANDARD_FIELDS + user_docfields(api_doc, docindex):
+                add_metadata_from_var(api_doc, field)
+        return
+
+    # Remove leading indentation from the docstring.
+    api_doc.docstring = unindent_docstring(api_doc.docstring)
+
+    # Decide which docformat is used by this module.
+    docformat = get_docformat(api_doc, docindex)
+
+    # A list of markup errors from parsing.
+    parse_errors = []
+    
+    # Extract a signature from the docstring, if it has one.  This
+    # overrides any signature we got via introspection/parsing.
+    if isinstance(api_doc, RoutineDoc):
+        parse_function_signature(api_doc, None, docformat, parse_errors)
+
+    # Parse the docstring.  Any errors encountered are stored as
+    # `ParseError` objects in the errors list.
+    parsed_docstring = markup.parse(api_doc.docstring, docformat,
+                                    parse_errors)
+        
+    # Divide the docstring into a description and a list of
+    # fields.
+    descr, fields = parsed_docstring.split_fields(parse_errors)
+    api_doc.descr = descr
+
+    field_warnings = []
+
+    # Handle the constructor fields that have been defined in the class
+    # docstring. This code assumes that a class docstring is parsed before
+    # the same class __init__ docstring.
+    if isinstance(api_doc, ClassDoc):
+
+        # Parse ahead the __init__ docstring for this class
+        initvar = api_doc.variables.get('__init__')
+        if initvar and isinstance(initvar.value, RoutineDoc):
+            init_api_doc = initvar.value
+            parse_docstring(init_api_doc, docindex, suppress_warnings)
+
+            parse_function_signature(init_api_doc, api_doc,
+                                     docformat, parse_errors)
+            init_fields = split_init_fields(fields, field_warnings)
+
+            # Process fields
+            for field in init_fields:
+                try:
+                    process_field(init_api_doc, docindex, field.tag(),
+                                    field.arg(), field.body())
+                except ValueError, e: field_warnings.append(str(e))
+
+    # Process fields
+    for field in fields:
+        try:
+            process_field(api_doc, docindex, field.tag(),
+                               field.arg(), field.body())
+        except ValueError, e: field_warnings.append(str(e))
+
+    # Check to make sure that all type parameters correspond to
+    # some documented parameter.
+    check_type_fields(api_doc, field_warnings)
+
+    # Check for special variables (e.g., __version__)
+    if isinstance(api_doc, NamespaceDoc):
+        for field in STANDARD_FIELDS + user_docfields(api_doc, docindex):
+            add_metadata_from_var(api_doc, field)
+
+    # Extract a summary
+    if api_doc.summary is None and api_doc.descr is not None:
+        api_doc.summary, api_doc.other_docs = api_doc.descr.summary()
+
+    # If the summary is empty, but the return field is not, then use
+    # the return field to generate a summary description.
+    if (isinstance(api_doc, RoutineDoc) and api_doc.summary is None and
+        api_doc.return_descr is not None):
+        s, o = api_doc.return_descr.summary()
+        api_doc.summary = RETURN_PDS + s
+        api_doc.other_docs = o
+
+    # [XX] Make sure we don't have types/param descrs for unknown
+    # vars/params?
+
+    # Report any errors that occured
+    if api_doc in suppress_warnings:
+        if parse_errors or field_warnings:
+            log.info("Suppressing docstring warnings for %s, since it "
+                     "is not included in the documented set." %
+                     api_doc.canonical_name)
+    else:
+        report_errors(api_doc, docindex, parse_errors, field_warnings)
+
+def add_metadata_from_var(api_doc, field):
+    for varname in field.varnames:
+        # Check if api_doc has a variable w/ the given name.
+        if varname not in api_doc.variables: continue
+
+        # Check moved here from before the for loop because we expect to
+        # reach rarely this point. The loop below is to be performed more than
+        # once only for fields with more than one varname, which currently is
+        # only 'author'.
+        for md in api_doc.metadata:
+            if field == md[0]:
+                return # We already have a value for this metadata.
+
+        var_doc = api_doc.variables[varname]
+        if var_doc.value is UNKNOWN: continue
+        val_doc = var_doc.value
+        value = []
+
+        # Try extracting the value from the pyval.
+        ok_types = (basestring, int, float, bool, type(None))
+        if val_doc.pyval is not UNKNOWN:
+            if isinstance(val_doc.pyval, ok_types):
+                value = [val_doc.pyval]
+            elif field.multivalue:
+                if isinstance(val_doc.pyval, (tuple, list)):
+                    for elt in val_doc.pyval:
+                        if not isinstance(elt, ok_types): break
+                    else:
+                        value = list(val_doc.pyval)
+
+        # Try extracting the value from the parse tree.
+        elif val_doc.toktree is not UNKNOWN:
+            try: value = [epydoc.docparser.parse_string(val_doc.toktree)]
+            except KeyboardInterrupt: raise
+            except: pass
+            if field.multivalue and not value:
+                try: value = epydoc.docparser.parse_string_list(val_doc.toktree)
+                except KeyboardInterrupt: raise
+                except: raise
+                
+        # Add any values that we found.
+        for elt in value:
+            if isinstance(elt, str):
+                elt = decode_with_backslashreplace(elt)
+            else:
+                elt = unicode(elt)
+            elt = epytext.ParsedEpytextDocstring(
+                epytext.parse_as_para(elt), inline=True)
+
+            # Add in the metadata and remove from the variables
+            api_doc.metadata.append( (field, varname, elt) )
+
+        # Remove the variable itself (unless it's documented)
+        if var_doc.docstring in (None, UNKNOWN):
+            del api_doc.variables[varname]
+            if api_doc.sort_spec is not UNKNOWN:
+                try: api_doc.sort_spec.remove(varname)
+                except ValueError: pass
+
+def initialize_api_doc(api_doc):
+    """A helper function for L{parse_docstring()} that initializes
+    the attributes that C{parse_docstring()} will write to."""
+    if api_doc.descr is UNKNOWN:
+        api_doc.descr = None
+    if api_doc.summary is UNKNOWN:
+        api_doc.summary = None
+    if api_doc.metadata is UNKNOWN:
+        api_doc.metadata = []
+    if isinstance(api_doc, RoutineDoc):
+        if api_doc.arg_descrs is UNKNOWN:
+            api_doc.arg_descrs = []
+        if api_doc.arg_types is UNKNOWN:
+            api_doc.arg_types = {}
+        if api_doc.return_descr is UNKNOWN:
+            api_doc.return_descr = None
+        if api_doc.return_type is UNKNOWN:
+            api_doc.return_type = None
+        if api_doc.exception_descrs is UNKNOWN:
+            api_doc.exception_descrs = []
+    if isinstance(api_doc, (VariableDoc, PropertyDoc)):
+        if api_doc.type_descr is UNKNOWN:
+            api_doc.type_descr = None
+    if isinstance(api_doc, NamespaceDoc):
+        if api_doc.group_specs is UNKNOWN:
+            api_doc.group_specs = []
+        if api_doc.sort_spec is UNKNOWN:
+            api_doc.sort_spec = []
+
+def split_init_fields(fields, warnings):
+    """
+    Remove the fields related to the constructor from a class docstring
+    fields list.
+
+    @param fields: The fields to process. The list will be modified in place
+    @type fields: C{list} of L{markup.Field}
+    @param warnings: A list to emit processing warnings
+    @type warnings: C{list}
+    @return: The C{fields} items to be applied to the C{__init__} method
+    @rtype: C{list} of L{markup.Field}
+    """
+    init_fields = []
+
+    # Split fields in lists according to their argument, keeping order.
+    arg_fields = {}
+    args_order = []
+    i = 0
+    while i < len(fields):
+        field = fields[i]
+
+        # gather together all the fields with the same arg
+        if field.arg() is not None:
+            arg_fields.setdefault(field.arg(), []).append(fields.pop(i))
+            args_order.append(field.arg())
+        else:
+            i += 1
+
+    # Now check that for each argument there is at most a single variable
+    # and a single parameter, and at most a single type for each of them.
+    for arg in args_order:
+        ff = arg_fields.pop(arg, None)
+        if ff is None:
+            continue
+
+        var = tvar = par = tpar = None
+        for field in ff:
+            if field.tag() in VARIABLE_TAGS:
+                if var is None:
+                    var = field
+                    fields.append(field)
+                else:
+                    warnings.append(
+                        "There is more than one variable named '%s'"
+                        % arg)
+            elif field.tag() in PARAMETER_TAGS:
+                if par is None:
+                    par = field
+                    init_fields.append(field)
+                else:
+                    warnings.append(
+                        "There is more than one parameter named '%s'"
+                        % arg)
+
+            elif field.tag() == 'type':
+                if var is None and par is None:
+                    # type before obj
+                    tvar = tpar = field
+                else:
+                    if var is not None and tvar is None:
+                        tvar = field
+                    if par is not None and tpar is None:
+                        tpar = field
+
+            elif field.tag() in EXCEPTION_TAGS:
+                init_fields.append(field)
+
+            else: # Unespected field
+                fields.append(field)
+
+        # Put selected types into the proper output lists
+        if tvar is not None:
+            if var is not None:
+                fields.append(tvar)
+            else:
+                pass # [xx] warn about type w/o object?
+
+        if tpar is not None:
+            if par is not None:
+                init_fields.append(tpar)
+            else:
+                pass # [xx] warn about type w/o object?
+
+    return init_fields
+
+def report_errors(api_doc, docindex, parse_errors, field_warnings):
+    """A helper function for L{parse_docstring()} that reports any
+    markup warnings and field warnings that we encountered while
+    processing C{api_doc}'s docstring."""
+    if not parse_errors and not field_warnings: return
+
+    # Get the name of the item containing the error, and the
+    # filename of its containing module.
+    name = api_doc.canonical_name
+    module = api_doc.defining_module
+    if module is not UNKNOWN and module.filename not in (None, UNKNOWN):
+        try: filename = py_src_filename(module.filename)
+        except: filename = module.filename
+    else:
+        filename = '??'
+
+    # [xx] Don't report markup errors for standard builtins.
+    # n.b. that we must use 'is' to compare pyvals here -- if we use
+    # 'in' or '==', then a user __cmp__ method might raise an
+    # exception, or lie.
+    if isinstance(api_doc, ValueDoc) and api_doc != module:
+        if module not in (None, UNKNOWN) and module.pyval is exceptions:
+            return
+        for builtin_val in __builtin__.__dict__.values():
+            if builtin_val is api_doc.pyval:
+                return
+        
+    # Get the start line of the docstring containing the error.
+    startline = api_doc.docstring_lineno
+    if startline in (None, UNKNOWN):
+        startline = introspect_docstring_lineno(api_doc)
+        if startline in (None, UNKNOWN):
+            startline = None
+
+    # Display a block header.
+    header = 'File %s, ' % filename
+    if startline is not None:
+        header += 'line %d, ' % startline
+    header += 'in %s' % name
+    log.start_block(header)
+    
+
+    # Display all parse errors.  But first, combine any errors
+    # with duplicate description messages.
+    if startline is None:
+        # remove dups, but keep original order:
+        dups = {}
+        for error in parse_errors:
+            message = error.descr()
+            if message not in dups:
+                log.docstring_warning(message)
+                dups[message] = 1
+    else:
+        # Combine line number fields for dup messages:
+        messages = {} # maps message -> list of linenum
+        for error in parse_errors:
+            error.set_linenum_offset(startline)
+            message = error.descr()
+            messages.setdefault(message, []).append(error.linenum())
+        message_items = messages.items()
+        message_items.sort(lambda a,b:cmp(min(a[1]), min(b[1])))
+        for message, linenums in message_items:
+            linenums = [n for n in linenums if n is not None]
+            if len(linenums) == 0:
+                log.docstring_warning(message)
+            elif len(linenums) == 1:
+                log.docstring_warning("Line %s: %s" % (linenums[0], message))
+            else:
+                linenums = ', '.join(['%s' % l for l in linenums])
+                log.docstring_warning("Lines %s: %s" % (linenums, message))
+
+    # Display all field warnings.
+    for warning in field_warnings:
+        log.docstring_warning(warning)
+
+    # End the message block.
+    log.end_block()
+
+RETURN_PDS = markup.parse('Returns:', markup='epytext')
+"""A ParsedDocstring containing the text 'Returns'.  This is used to
+construct summary descriptions for routines that have empty C{descr},
+but non-empty C{return_descr}."""
+RETURN_PDS._tree.children[0].attribs['inline'] = True
+
+######################################################################
+#{ Field Processing Error Messages
+######################################################################
+
+UNEXPECTED_ARG = '%r did not expect an argument'
+EXPECTED_ARG = '%r expected an argument'
+EXPECTED_SINGLE_ARG = '%r expected a single argument'
+BAD_CONTEXT = 'Invalid context for %r'
+REDEFINED = 'Redefinition of %s'
+UNKNOWN_TAG = 'Unknown field tag %r'
+BAD_PARAM = '@%s for unknown parameter %s'
+
+######################################################################
+#{ Field Processing
+######################################################################
+
+def process_field(api_doc, docindex, tag, arg, descr):
+    """
+    Process a single field, and use it to update C{api_doc}.  If
+    C{tag} is the name of a special field, then call its handler
+    function.  If C{tag} is the name of a simple field, then use
+    C{process_simple_field} to process it.  Otherwise, check if it's a
+    user-defined field, defined in this docstring or the docstring of
+    a containing object; and if so, process it with
+    C{process_simple_field}.
+
+    @param tag: The field's tag, such as C{'author'}
+    @param arg: The field's optional argument
+    @param descr: The description following the field tag and
+        argument.
+    @raise ValueError: If a problem was encountered while processing
+        the field.  The C{ValueError}'s string argument is an
+        explanation of the problem, which should be displayed as a
+        warning message.
+    """
+    # standard special fields
+    if tag in _field_dispatch_table:
+        handler = _field_dispatch_table[tag]
+        handler(api_doc, docindex, tag, arg, descr)
+        return
+
+    # standard simple fields & user-defined fields
+    for field in STANDARD_FIELDS + user_docfields(api_doc, docindex):
+        if tag in field.tags:
+            # [xx] check if it's redefined if it's not multivalue??
+            if not field.takes_arg:
+                _check(api_doc, tag, arg, expect_arg=False)
+            api_doc.metadata.append((field, arg, descr))
+            return
+
+    # If we didn't handle the field, then report a warning.
+    raise ValueError(UNKNOWN_TAG % tag)
+
+def user_docfields(api_doc, docindex):
+    """
+    Return a list of user defined fields that can be used for the
+    given object.  This list is taken from the given C{api_doc}, and
+    any of its containing C{NamepaceDoc}s.
+
+    @note: We assume here that a parent's docstring will always be
+        parsed before its childrens'.  This is indeed the case when we
+        are called via L{docbuilder.build_doc_index()}.  If a child's
+        docstring is parsed before its parents, then its parent won't
+        yet have had its C{extra_docstring_fields} attribute
+        initialized.
+    """
+    docfields = []
+    # Get any docfields from `api_doc` itself
+    if api_doc.extra_docstring_fields not in (None, UNKNOWN):
+        docfields += api_doc.extra_docstring_fields
+    # Get any docfields from `api_doc`'s ancestors
+    for i in range(len(api_doc.canonical_name)-1, 0, -1):
+        ancestor = docindex.get_valdoc(api_doc.canonical_name[:i])
+        if ancestor is not None \
+        and ancestor.extra_docstring_fields not in (None, UNKNOWN):
+            docfields += ancestor.extra_docstring_fields
+    return docfields
+
+_field_dispatch_table = {}
+def register_field_handler(handler, *field_tags):
+    """
+    Register the given field handler function for processing any
+    of the given field tags.  Field handler functions should
+    have the following signature:
+
+        >>> def field_handler(api_doc, docindex, tag, arg, descr):
+        ...     '''update api_doc in response to the field.'''
+
+    Where C{api_doc} is the documentation object to update;
+    C{docindex} is a L{DocIndex} that can be used to look up the
+    documentation for related objects; C{tag} is the field tag that
+    was used; C{arg} is the optional argument; and C{descr} is the
+    description following the field tag and argument.
+    """
+    for field_tag in field_tags:
+        _field_dispatch_table[field_tag] = handler
+
+######################################################################
+#{ Field Handler Functions
+######################################################################
+
+def process_summary_field(api_doc, docindex, tag, arg, descr):
+    """Store C{descr} in C{api_doc.summary}"""
+    _check(api_doc, tag, arg, expect_arg=False)
+    if api_doc.summary is not None:
+        raise ValueError(REDEFINED % tag)
+    api_doc.summary = descr
+
+def process_include_field(api_doc, docindex, tag, arg, descr):
+    """Copy the docstring contents from the object named in C{descr}"""
+    _check(api_doc, tag, arg, expect_arg=False)
+    # options:
+    #   a. just append the descr to our own
+    #   b. append descr and update metadata
+    #   c. append descr and process all fields.
+    # in any case, mark any errors we may find as coming from an
+    # imported docstring.
+    
+    # how does this interact with documentation inheritance??
+    raise ValueError('%s not implemented yet' % tag)
+
+def process_undocumented_field(api_doc, docindex, tag, arg, descr):
+    """Remove any documentation for the variables named in C{descr}"""
+    _check(api_doc, tag, arg, context=NamespaceDoc, expect_arg=False)
+    for ident in _descr_to_identifiers(descr):
+        var_name_re = re.compile('^%s$' % ident.replace('*', '(.*)'))
+        for var_name, var_doc in api_doc.variables.items():
+            if var_name_re.match(var_name):
+                # Remove the variable from `variables`.
+                api_doc.variables.pop(var_name, None)
+                if api_doc.sort_spec is not UNKNOWN:
+                    try: api_doc.sort_spec.remove(var_name)
+                    except ValueError: pass
+        # For modules, remove any submodules that match var_name_re.
+        if isinstance(api_doc, ModuleDoc):
+            removed = set([m for m in api_doc.submodules
+                           if var_name_re.match(m.canonical_name[-1])])
+            if removed:
+                # Remove the indicated submodules from this module.
+                api_doc.submodules = [m for m in api_doc.submodules
+                                      if m not in removed]
+                # Remove all ancestors of the indicated submodules
+                # from the docindex root.  E.g., if module x
+                # declares y to be undocumented, then x.y.z should
+                # also be undocumented.
+                for elt in docindex.root[:]:
+                    for m in removed:
+                        if m.canonical_name.dominates(elt.canonical_name):
+                            docindex.root.remove(elt)
+
+def process_group_field(api_doc, docindex, tag, arg, descr):
+    """Define a group named C{arg} containing the variables whose
+    names are listed in C{descr}."""
+    _check(api_doc, tag, arg, context=NamespaceDoc, expect_arg=True)
+    api_doc.group_specs.append( (arg, _descr_to_identifiers(descr)) )
+    # [xx] should this also set sort order?
+
+def process_deffield_field(api_doc, docindex, tag, arg, descr):
+    """Define a new custom field."""
+    _check(api_doc, tag, arg, expect_arg=True)
+    if api_doc.extra_docstring_fields is UNKNOWN:
+        api_doc.extra_docstring_fields = []
+    try:
+        docstring_field = _descr_to_docstring_field(arg, descr)
+        docstring_field.varnames.append("__%s__" % arg)
+        api_doc.extra_docstring_fields.append(docstring_field)
+    except ValueError, e:
+        raise ValueError('Bad %s: %s' % (tag, e))
+
+def process_raise_field(api_doc, docindex, tag, arg, descr):
+    """Record the fact that C{api_doc} can raise the exception named
+    C{tag} in C{api_doc.exception_descrs}."""
+    _check(api_doc, tag, arg, context=RoutineDoc, expect_arg='single')
+    try: name = DottedName(arg, strict=True)
+    except DottedName.InvalidDottedName: name = arg
+    api_doc.exception_descrs.append( (name, descr) )
+
+def process_sort_field(api_doc, docindex, tag, arg, descr):
+    _check(api_doc, tag, arg, context=NamespaceDoc, expect_arg=False)
+    api_doc.sort_spec = _descr_to_identifiers(descr) + api_doc.sort_spec
+
+# [xx] should I notice when they give a type for an unknown var?
+def process_type_field(api_doc, docindex, tag, arg, descr):
+    # In namespace, "@type var: ..." describes the type of a var.
+    if isinstance(api_doc, NamespaceDoc):
+        _check(api_doc, tag, arg, expect_arg='single')
+        set_var_type(api_doc, arg, descr)
+
+    # For variables & properties, "@type: ..." describes the variable.
+    elif isinstance(api_doc, (VariableDoc, PropertyDoc)):
+        _check(api_doc, tag, arg, expect_arg=False)
+        if api_doc.type_descr is not None:
+            raise ValueError(REDEFINED % tag)
+        api_doc.type_descr = descr
+
+    # For routines, "@type param: ..." describes a parameter.
+    elif isinstance(api_doc, RoutineDoc):
+        _check(api_doc, tag, arg, expect_arg='single')
+        if arg in api_doc.arg_types:
+            raise ValueError(REDEFINED % ('type for '+arg))
+        api_doc.arg_types[arg] = descr
+
+    else:
+        raise ValueError(BAD_CONTEXT % tag)
+        
+def process_var_field(api_doc, docindex, tag, arg, descr):
+    _check(api_doc, tag, arg, context=ModuleDoc, expect_arg=True)
+    for ident in re.split('[:;, ] *', arg):
+        set_var_descr(api_doc, ident, descr)
+        
+def process_cvar_field(api_doc, docindex, tag, arg, descr):
+    # If @cvar is used *within* a variable, then use it as the
+    # variable's description, and treat the variable as a class var.
+    if (isinstance(api_doc, VariableDoc) and
+        isinstance(api_doc.container, ClassDoc)):
+        _check(api_doc, tag, arg, expect_arg=False)
+        api_doc.is_instvar = False
+        api_doc.descr = markup.ConcatenatedDocstring(api_doc.descr, descr)
+        api_doc.summary, api_doc.other_docs = descr.summary()
+
+    # Otherwise, @cvar should be used in a class.
+    else:
+        _check(api_doc, tag, arg, context=ClassDoc, expect_arg=True)
+        for ident in re.split('[:;, ] *', arg):
+            set_var_descr(api_doc, ident, descr)
+            api_doc.variables[ident].is_instvar = False
+        
+def process_ivar_field(api_doc, docindex, tag, arg, descr):
+    # If @ivar is used *within* a variable, then use it as the
+    # variable's description, and treat the variable as an instvar.
+    if (isinstance(api_doc, VariableDoc) and
+        isinstance(api_doc.container, ClassDoc)):
+        _check(api_doc, tag, arg, expect_arg=False)
+        # require that there be no other descr?
+        api_doc.is_instvar = True
+        api_doc.descr = markup.ConcatenatedDocstring(api_doc.descr, descr)
+        api_doc.summary, api_doc.other_docs = descr.summary()
+
+    # Otherwise, @ivar should be used in a class.
+    else:
+        _check(api_doc, tag, arg, context=ClassDoc, expect_arg=True)
+        for ident in re.split('[:;, ] *', arg):
+            set_var_descr(api_doc, ident, descr)
+            api_doc.variables[ident].is_instvar = True
+
+# [xx] '@return: foo' used to get used as a descr if no other
+# descr was present.  is that still true?
+def process_return_field(api_doc, docindex, tag, arg, descr):
+    _check(api_doc, tag, arg, context=RoutineDoc, expect_arg=False)
+    if api_doc.return_descr is not None:
+        raise ValueError(REDEFINED % 'return value description')
+    api_doc.return_descr = descr
+
+def process_rtype_field(api_doc, docindex, tag, arg, descr):
+    _check(api_doc, tag, arg,
+           context=(RoutineDoc, PropertyDoc), expect_arg=False)
+    if isinstance(api_doc, RoutineDoc):
+        if api_doc.return_type is not None:
+            raise ValueError(REDEFINED % 'return value type')
+        api_doc.return_type = descr
+
+    elif isinstance(api_doc, PropertyDoc):
+        _check(api_doc, tag, arg, expect_arg=False)
+        if api_doc.type_descr is not None:
+            raise ValueError(REDEFINED % tag)
+        api_doc.type_descr = descr
+
+def process_arg_field(api_doc, docindex, tag, arg, descr):
+    _check(api_doc, tag, arg, context=RoutineDoc, expect_arg=True)
+    idents = re.split('[:;, ] *', arg)
+    api_doc.arg_descrs.append( (idents, descr) )
+    # Check to make sure that the documented parameter(s) are
+    # actually part of the function signature.
+    all_args = api_doc.all_args()
+    if all_args not in (['...'], UNKNOWN):
+        bad_params = ['"%s"' % i for i in idents if i not in all_args]
+        if bad_params:
+            raise ValueError(BAD_PARAM % (tag, ', '.join(bad_params)))
+
+def process_kwarg_field(api_doc, docindex, tag, arg, descr):
+    # [xx] these should -not- be checked if they exist..
+    # and listed separately or not??
+    _check(api_doc, tag, arg, context=RoutineDoc, expect_arg=True)
+    idents = re.split('[:;, ] *', arg)
+    api_doc.arg_descrs.append( (idents, descr) )
+
+register_field_handler(process_group_field, 'group')
+register_field_handler(process_deffield_field, 'deffield', 'newfield')
+register_field_handler(process_sort_field, 'sort')
+register_field_handler(process_summary_field, 'summary')
+register_field_handler(process_undocumented_field, 'undocumented')
+register_field_handler(process_include_field, 'include')
+register_field_handler(process_var_field, 'var', 'variable')
+register_field_handler(process_type_field, 'type')
+register_field_handler(process_cvar_field, 'cvar', 'cvariable')
+register_field_handler(process_ivar_field, 'ivar', 'ivariable')
+register_field_handler(process_return_field, 'return', 'returns')
+register_field_handler(process_rtype_field, 'rtype', 'returntype')
+register_field_handler(process_arg_field, 'arg', 'argument',
+                                          'parameter', 'param')
+register_field_handler(process_kwarg_field, 'kwarg', 'keyword', 'kwparam')
+register_field_handler(process_raise_field, 'raise', 'raises',
+                                            'except', 'exception')
+
+# Tags related to function parameters
+PARAMETER_TAGS = ('arg', 'argument', 'parameter', 'param',
+                  'kwarg', 'keyword', 'kwparam')
+
+# Tags related to variables in a class
+VARIABLE_TAGS = ('cvar', 'cvariable', 'ivar', 'ivariable')
+
+# Tags related to exceptions
+EXCEPTION_TAGS = ('raise', 'raises', 'except', 'exception')
+
+######################################################################
+#{ Helper Functions
+######################################################################
+
+def check_type_fields(api_doc, field_warnings):
+    """Check to make sure that all type fields correspond to some
+    documented parameter; if not, append a warning to field_warnings."""
+    if isinstance(api_doc, RoutineDoc):
+        for arg in api_doc.arg_types:
+            if arg not in api_doc.all_args():
+                for args, descr in api_doc.arg_descrs:
+                    if arg in args:
+                        break
+                else:
+                    field_warnings.append(BAD_PARAM % ('type', '"%s"' % arg))
+
+def set_var_descr(api_doc, ident, descr):
+    if ident not in api_doc.variables:
+        api_doc.variables[ident] = VariableDoc(
+            container=api_doc, name=ident,
+            canonical_name=api_doc.canonical_name+ident)
+                                      
+    var_doc = api_doc.variables[ident]
+    if var_doc.descr not in (None, UNKNOWN):
+        raise ValueError(REDEFINED % ('description for '+ident))
+    var_doc.descr = descr
+    if var_doc.summary in (None, UNKNOWN):
+        var_doc.summary, var_doc.other_docs = var_doc.descr.summary()
+
+def set_var_type(api_doc, ident, descr):
+    if ident not in api_doc.variables:
+        api_doc.variables[ident] = VariableDoc(
+            container=api_doc, name=ident,
+            canonical_name=api_doc.canonical_name+ident)
+        
+    var_doc = api_doc.variables[ident]
+    if var_doc.type_descr not in (None, UNKNOWN):
+        raise ValueError(REDEFINED % ('type for '+ident))
+    var_doc.type_descr = descr
+        
+def _check(api_doc, tag, arg, context=None, expect_arg=None):
+    if context is not None:
+        if not isinstance(api_doc, context):
+            raise ValueError(BAD_CONTEXT % tag)
+    if expect_arg is not None:
+        if expect_arg == True:
+            if arg is None:
+                raise ValueError(EXPECTED_ARG % tag)
+        elif expect_arg == False:
+            if arg is not None:
+                raise ValueError(UNEXPECTED_ARG % tag)
+        elif expect_arg == 'single':
+            if (arg is None or ' ' in arg):
+                raise ValueError(EXPECTED_SINGLE_ARG % tag)
+        else:
+            assert 0, 'bad value for expect_arg'
+
+def get_docformat(api_doc, docindex):
+    """
+    Return the name of the markup language that should be used to
+    parse the API documentation for the given object.
+    """
+    # Find the module that defines api_doc.
+    module = api_doc.defining_module
+    # Look up its docformat.
+    if module is not UNKNOWN and module.docformat not in (None, UNKNOWN):
+        docformat = module.docformat
+    else:
+        docformat = DEFAULT_DOCFORMAT
+    # Convert to lower case & strip region codes.
+    try: return docformat.lower().split()[0]
+    except: return DEFAULT_DOCFORMAT
+
+def unindent_docstring(docstring):
+    # [xx] copied from inspect.getdoc(); we can't use inspect.getdoc()
+    # itself, since it expects an object, not a string.
+    
+    if not docstring: return ''
+    lines = docstring.expandtabs().split('\n')
+
+    # Find minimum indentation of any non-blank lines after first line.
+    margin = sys.maxint
+    for line in lines[1:]:
+        content = len(line.lstrip())
+        if content:
+            indent = len(line) - content
+            margin = min(margin, indent)
+    # Remove indentation.
+    if lines:
+        lines[0] = lines[0].lstrip()
+    if margin < sys.maxint:
+        for i in range(1, len(lines)): lines[i] = lines[i][margin:]
+    # Remove any trailing (but not leading!) blank lines.
+    while lines and not lines[-1]:
+        lines.pop()
+    #while lines and not lines[0]:
+    #    lines.pop(0)
+    return '\n'.join(lines)
+                           
+_IDENTIFIER_LIST_REGEXP = re.compile(r'^[\w.\*]+([\s,:;]\s*[\w.\*]+)*$')
+def _descr_to_identifiers(descr):
+    """
+    Given a C{ParsedDocstring} that contains a list of identifiers,
+    return a list of those identifiers.  This is used by fields such
+    as C{@group} and C{@sort}, which expect lists of identifiers as
+    their values.  To extract the identifiers, the docstring is first
+    converted to plaintext, and then split.  The plaintext content of
+    the docstring must be a a list of identifiers, separated by
+    spaces, commas, colons, or semicolons.
+    
+    @rtype: C{list} of C{string}
+    @return: A list of the identifier names contained in C{descr}.
+    @type descr: L{markup.ParsedDocstring}
+    @param descr: A C{ParsedDocstring} containing a list of
+        identifiers.
+    @raise ValueError: If C{descr} does not contain a valid list of
+        identifiers.
+    """
+    idents = descr.to_plaintext(None).strip()
+    idents = re.sub(r'\s+', ' ', idents)
+    if not _IDENTIFIER_LIST_REGEXP.match(idents):
+        raise ValueError, 'Bad Identifier list: %r' % idents
+    rval = re.split('[:;, ] *', idents)
+    return rval
+    
+def _descr_to_docstring_field(arg, descr):
+    tags = [s.lower() for s in re.split('[:;, ] *', arg)]
+    descr = descr.to_plaintext(None).strip()
+    args = re.split('[:;,] *', descr)
+    if len(args) == 0 or len(args) > 3:
+        raise ValueError, 'Wrong number of arguments'
+    singular = args[0]
+    if len(args) >= 2: plural = args[1]
+    else: plural = None
+    short = 0
+    if len(args) >= 3:
+        if args[2] == 'short': short = 1
+        else: raise ValueError('Bad arg 2 (expected "short")')
+    return DocstringField(tags, singular, plural, short)
+
+######################################################################
+#{ Function Signature Extraction
+######################################################################
+
+# [XX] todo: add optional type modifiers?
+_SIGNATURE_RE = re.compile(
+    # Class name (for builtin methods)
+    r'^\s*((?P<self>\w+)\.)?' +
+    # The function name (must match exactly) [XX] not anymore!
+    r'(?P<func>\w+)' +
+    # The parameters
+    r'\((?P<params>(\s*\[?\s*\*{0,2}[\w\-\.]+(\s*=.+?)?'+
+    r'(\s*\[?\s*,\s*\]?\s*\*{0,2}[\w\-\.]+(\s*=.+?)?)*\]*)?)\s*\)' +
+    # The return value (optional)
+    r'(\s*(->)\s*(?P<return>\S.*?))?'+
+    # The end marker
+    r'\s*(\n|\s+(--|<=+>)\s+|$|\.\s+|\.\n)')
+"""A regular expression that is used to extract signatures from
+docstrings."""
+    
+def parse_function_signature(func_doc, doc_source, docformat, parse_errors):
+    """
+    Construct the signature for a builtin function or method from
+    its docstring.  If the docstring uses the standard convention
+    of including a signature in the first line of the docstring
+    (and formats that signature according to standard
+    conventions), then it will be used to extract a signature.
+    Otherwise, the signature will be set to a single varargs
+    variable named C{"..."}.
+
+    @param func_doc: The target object where to store parsed signature. Also
+        container of the docstring to parse if doc_source is C{None}
+    @type func_doc: L{RoutineDoc}
+    @param doc_source: Contains the docstring to parse. If C{None}, parse
+        L{func_doc} docstring instead
+    @type doc_source: L{APIDoc}
+    @rtype: C{None}
+    """
+    if doc_source is None:
+        doc_source = func_doc
+
+    # If there's no docstring, then don't do anything.
+    if not doc_source.docstring: return False
+
+    m = _SIGNATURE_RE.match(doc_source.docstring)
+    if m is None: return False
+
+    # Do I want to be this strict?
+    # Notice that __init__ must match the class name instead, if the signature
+    # comes from the class docstring
+#     if not (m.group('func') == func_doc.canonical_name[-1] or
+#             '_'+m.group('func') == func_doc.canonical_name[-1]):
+#         log.warning("Not extracting function signature from %s's "
+#                     "docstring, since the name doesn't match." %
+#                     func_doc.canonical_name)
+#         return False
+    
+    params = m.group('params')
+    rtype = m.group('return')
+    selfparam = m.group('self')
+    
+    # Extract the parameters from the signature.
+    func_doc.posargs = []
+    func_doc.vararg = None
+    func_doc.kwarg = None
+    if func_doc.posarg_defaults is UNKNOWN:
+        func_doc.posarg_defaults = []
+    if params:
+        # Figure out which parameters are optional.
+        while '[' in params or ']' in params:
+            m2 = re.match(r'(.*)\[([^\[\]]+)\](.*)', params)
+            if not m2: return False
+            (start, mid, end) = m2.groups()
+            mid = re.sub(r'((,|^)\s*[\w\-\.]+)', r'\1=...', mid)
+            params = start+mid+end
+
+        params = re.sub(r'=...=' , r'=', params)
+        for name in params.split(','):
+            if '=' in name:
+                (name, default_repr) = name.split('=',1)
+                default = GenericValueDoc(parse_repr=default_repr)
+            else:
+                default = None
+            name = name.strip()
+            if name == '...':
+                func_doc.vararg = '...'
+            elif name.startswith('**'):
+                func_doc.kwarg = name[2:]
+            elif name.startswith('*'):
+                func_doc.vararg = name[1:]
+            else:
+                func_doc.posargs.append(name)
+                if len(func_doc.posarg_defaults) < len(func_doc.posargs):
+                    func_doc.posarg_defaults.append(default)
+                elif default is not None:
+                    argnum = len(func_doc.posargs)-1
+                    func_doc.posarg_defaults[argnum] = default
+
+    # Extract the return type/value from the signature
+    if rtype:
+        func_doc.return_type = markup.parse(rtype, docformat, parse_errors,
+                                            inline=True)
+
+    # Add the self parameter, if it was specified.
+    if selfparam:
+        func_doc.posargs.insert(0, selfparam)
+        func_doc.posarg_defaults.insert(0, None)
+
+    # Remove the signature from the docstring.
+    doc_source.docstring = doc_source.docstring[m.end():]
+        
+    # We found a signature.
+    return True
+