ctags.rb 1.0.0

data/ext/vendor/exuberant-ctags/python.c

@@ -0,0 +1,771 @@
+/*
+*   $Id: python.c 720 2009-07-07 03:55:23Z dhiebert $
+*
+*   Copyright (c) 2000-2003, Darren Hiebert
+*
+*   This source code is released for free distribution under the terms of the
+*   GNU General Public License.
+*
+*   This module contains functions for generating tags for Python language
+*   files.
+*/
+/*
+*   INCLUDE FILES
+*/
+#include "general.h"  /* must always come first */
+
+#include <string.h>
+
+#include "entry.h"
+#include "options.h"
+#include "read.h"
+#include "main.h"
+#include "vstring.h"
+#include "routines.h"
+#include "debug.h"
+
+/*
+*   DATA DECLARATIONS
+*/
+typedef struct NestingLevel NestingLevel;
+typedef struct NestingLevels NestingLevels;
+
+struct NestingLevel
+{
+	int indentation;
+	vString *name;
+	int type;
+};
+
+struct NestingLevels
+{
+	NestingLevel *levels;
+	int n;					/* number of levels in use */
+	int allocated;
+};
+
+typedef enum {
+	K_CLASS, K_FUNCTION, K_MEMBER, K_VARIABLE, K_IMPORT
+} pythonKind;
+
+/*
+*   DATA DEFINITIONS
+*/
+static kindOption PythonKinds[] = {
+	{TRUE, 'c', "class",    "classes"},
+	{TRUE, 'f', "function", "functions"},
+	{TRUE, 'm', "member",   "class members"},
+    {TRUE, 'v', "variable", "variables"},
+    {TRUE, 'i', "namespace", "imports"}
+};
+
+static char const * const singletriple = "'''";
+static char const * const doubletriple = "\"\"\"";
+
+/*
+*   FUNCTION DEFINITIONS
+*/
+
+static NestingLevels *nestingLevelsNew (void)
+{
+	NestingLevels *nls = xCalloc (1, NestingLevels);
+	return nls;
+}
+
+static void nestingLevelsFree (NestingLevels *nls)
+{
+	int i;
+	for (i = 0; i < nls->allocated; i++)
+		vStringDelete(nls->levels[i].name);
+	if (nls->levels) eFree(nls->levels);
+	eFree(nls);
+}
+
+static void nestingLevelsPush (NestingLevels *nls,
+	const vString *name, int type)
+{
+	NestingLevel *nl = NULL;
+
+	if (nls->n >= nls->allocated)
+	{
+		nls->allocated++;
+		nls->levels = xRealloc(nls->levels,
+			nls->allocated, NestingLevel);
+		nls->levels[nls->n].name = vStringNew();
+	}
+	nl = &nls->levels[nls->n];
+	nls->n++;
+
+	vStringCopy(nl->name, name);
+	nl->type = type;
+}
+
+#if 0
+static NestingLevel *nestingLevelsGetCurrent (NestingLevels *nls)
+{
+	Assert (nls != NULL);
+
+	if (nls->n < 1)
+		return NULL;
+
+	return &nls->levels[nls->n - 1];
+}
+
+static void nestingLevelsPop (NestingLevels *nls)
+{
+	const NestingLevel *nl = nestingLevelsGetCurrent(nls);
+
+	Assert (nl != NULL);
+	vStringClear(nl->name);
+	nls->n--;
+}
+#endif
+
+static boolean isIdentifierFirstCharacter (int c)
+{
+	return (boolean) (isalpha (c) || c == '_');
+}
+
+static boolean isIdentifierCharacter (int c)
+{
+	return (boolean) (isalnum (c) || c == '_');
+}
+
+/* Given a string with the contents of a line directly after the "def" keyword,
+ * extract all relevant information and create a tag.
+ */
+static void makeFunctionTag (vString *const function,
+	vString *const parent, int is_class_parent, const char *arglist __unused__)
+{
+	tagEntryInfo tag;
+	initTagEntry (&tag, vStringValue (function));
+
+	tag.kindName = "function";
+	tag.kind = 'f';
+	/* tag.extensionFields.arglist = arglist; */
+
+	if (vStringLength (parent) > 0)
+	{
+		if (is_class_parent)
+		{
+			tag.kindName = "member";
+			tag.kind = 'm';
+			tag.extensionFields.scope [0] = "class";
+			tag.extensionFields.scope [1] = vStringValue (parent);
+		}
+		else
+		{
+			tag.extensionFields.scope [0] = "function";
+			tag.extensionFields.scope [1] = vStringValue (parent);
+		}
+	}
+
+	/* If a function starts with __, we mark it as file scope.
+	 * FIXME: What is the proper way to signal such attributes?
+	 * TODO: What does functions/classes starting with _ and __ mean in python?
+	 */
+	if (strncmp (vStringValue (function), "__", 2) == 0 &&
+		strcmp (vStringValue (function), "__init__") != 0)
+	{
+		tag.extensionFields.access = "private";
+		tag.isFileScope = TRUE;
+	}
+	else
+	{
+		tag.extensionFields.access = "public";
+	}
+	makeTagEntry (&tag);
+}
+
+/* Given a string with the contents of the line directly after the "class"
+ * keyword, extract all necessary information and create a tag.
+ */
+static void makeClassTag (vString *const class, vString *const inheritance,
+	vString *const parent, int is_class_parent)
+{
+	tagEntryInfo tag;
+	initTagEntry (&tag, vStringValue (class));
+	tag.kindName = "class";
+	tag.kind = 'c';
+	if (vStringLength (parent) > 0)
+	{
+		if (is_class_parent)
+		{
+			tag.extensionFields.scope [0] = "class";
+			tag.extensionFields.scope [1] = vStringValue (parent);
+		}
+		else
+		{
+			tag.extensionFields.scope [0] = "function";
+			tag.extensionFields.scope [1] = vStringValue (parent);
+		}
+	}
+	tag.extensionFields.inheritance = vStringValue (inheritance);
+	makeTagEntry (&tag);
+}
+
+static void makeVariableTag (vString *const var, vString *const parent)
+{
+	tagEntryInfo tag;
+	initTagEntry (&tag, vStringValue (var));
+	tag.kindName = "variable";
+	tag.kind = 'v';
+	if (vStringLength (parent) > 0)
+	{
+		tag.extensionFields.scope [0] = "class";
+		tag.extensionFields.scope [1] = vStringValue (parent);
+	}
+	makeTagEntry (&tag);
+}
+
+/* Skip a single or double quoted string. */
+static const char *skipString (const char *cp)
+{
+	const char *start = cp;
+	int escaped = 0;
+	for (cp++; *cp; cp++)
+	{
+		if (escaped)
+			escaped--;
+		else if (*cp == '\\')
+			escaped++;
+		else if (*cp == *start)
+			return cp + 1;
+	}
+	return cp;
+}
+
+/* Skip everything up to an identifier start. */
+static const char *skipEverything (const char *cp)
+{
+	for (; *cp; cp++)
+	{
+		if (*cp == '"' || *cp == '\'')
+		{
+			cp = skipString(cp);
+			if (!*cp) break;
+		}
+		if (isIdentifierFirstCharacter ((int) *cp))
+			return cp;
+	}
+	return cp;
+}
+
+/* Skip an identifier. */
+static const char *skipIdentifier (const char *cp)
+{
+	while (isIdentifierCharacter ((int) *cp))
+		cp++;
+	return cp;
+}
+
+static const char *findDefinitionOrClass (const char *cp)
+{
+	while (*cp)
+	{
+		cp = skipEverything (cp);
+		if (!strncmp(cp, "def", 3) || !strncmp(cp, "class", 5) ||
+			!strncmp(cp, "cdef", 4) || !strncmp(cp, "cpdef", 5))
+		{
+			return cp;
+		}
+		cp = skipIdentifier (cp);
+	}
+	return NULL;
+}
+
+static const char *skipSpace (const char *cp)
+{
+	while (isspace ((int) *cp))
+		++cp;
+	return cp;
+}
+
+/* Starting at ''cp'', parse an identifier into ''identifier''. */
+static const char *parseIdentifier (const char *cp, vString *const identifier)
+{
+	vStringClear (identifier);
+	while (isIdentifierCharacter ((int) *cp))
+	{
+		vStringPut (identifier, (int) *cp);
+		++cp;
+	}
+	vStringTerminate (identifier);
+	return cp;
+}
+
+static void parseClass (const char *cp, vString *const class,
+	vString *const parent, int is_class_parent)
+{
+	vString *const inheritance = vStringNew ();
+	vStringClear (inheritance);
+	cp = parseIdentifier (cp, class);
+	cp = skipSpace (cp);
+	if (*cp == '(')
+	{
+		++cp;
+		while (*cp != ')')
+		{
+			if (*cp == '\0')
+			{
+				/* Closing parenthesis can be in follow up line. */
+				cp = (const char *) fileReadLine ();
+				if (!cp) break;
+				vStringPut (inheritance, ' ');
+				continue;
+			}
+			vStringPut (inheritance, *cp);
+			++cp;
+		}
+		vStringTerminate (inheritance);
+	}
+	makeClassTag (class, inheritance, parent, is_class_parent);
+	vStringDelete (inheritance);
+}
+
+static void parseImports (const char *cp)
+{
+	const char *pos;
+	vString *name, *name_next;
+
+	cp = skipEverything (cp);
+
+	if ((pos = strstr (cp, "import")) == NULL)
+		return;
+
+	cp = pos + 6;
+
+	/* continue only if there is some space between the keyword and the identifier */
+	if (! isspace (*cp))
+		return;
+
+	cp++;
+	cp = skipSpace (cp);
+
+	name = vStringNew ();
+	name_next = vStringNew ();
+
+	cp = skipEverything (cp);
+	while (*cp)
+	{
+		cp = parseIdentifier (cp, name);
+
+		cp = skipEverything (cp);
+		/* we parse the next possible import statement as well to be able to ignore 'foo' in
+		 * 'import foo as bar' */
+		parseIdentifier (cp, name_next);
+
+		/* take the current tag only if the next one is not "as" */
+		if (strcmp (vStringValue (name_next), "as") != 0 &&
+			strcmp (vStringValue (name), "as") != 0)
+		{
+			makeSimpleTag (name, PythonKinds, K_IMPORT);
+		}
+	}
+	vStringDelete (name);
+	vStringDelete (name_next);
+}
+
+/* modified from get.c getArglistFromStr().
+ * warning: terminates rest of string past arglist!
+ * note: does not ignore brackets inside strings! */
+static char *parseArglist(const char *buf)
+{
+	char *start, *end;
+	int level;
+	if (NULL == buf)
+		return NULL;
+	if (NULL == (start = strchr(buf, '(')))
+		return NULL;
+	for (level = 1, end = start + 1; level > 0; ++end)
+	{
+		if ('\0' == *end)
+			break;
+		else if ('(' == *end)
+			++ level;
+		else if (')' == *end)
+			-- level;
+	}
+	*end = '\0';
+	return strdup(start);
+}
+
+static void parseFunction (const char *cp, vString *const def,
+	vString *const parent, int is_class_parent)
+{
+	char *arglist;
+
+	cp = parseIdentifier (cp, def);
+	arglist = parseArglist (cp);
+	makeFunctionTag (def, parent, is_class_parent, arglist);
+	eFree (arglist);
+}
+
+/* Get the combined name of a nested symbol. Classes are separated with ".",
+ * functions with "/". For example this code:
+ * class MyClass:
+ *     def myFunction:
+ *         def SubFunction:
+ *             class SubClass:
+ *                 def Method:
+ *                     pass
+ * Would produce this string:
+ * MyClass.MyFunction/SubFunction/SubClass.Method
+ */
+static boolean constructParentString(NestingLevels *nls, int indent,
+	vString *result)
+{
+	int i;
+	NestingLevel *prev = NULL;
+	int is_class = FALSE;
+	vStringClear (result);
+	for (i = 0; i < nls->n; i++)
+	{
+		NestingLevel *nl = nls->levels + i;
+		if (indent <= nl->indentation)
+			break;
+		if (prev)
+		{
+			vStringCatS(result, ".");	/* make Geany symbol list grouping work properly */
+/*
+			if (prev->type == K_CLASS)
+				vStringCatS(result, ".");
+			else
+				vStringCatS(result, "/");
+*/
+		}
+		vStringCat(result, nl->name);
+		is_class = (nl->type == K_CLASS);
+		prev = nl;
+	}
+	return is_class;
+}
+
+/* Check whether parent's indentation level is higher than the current level and
+ * if so, remove it.
+ */
+static void checkParent(NestingLevels *nls, int indent, vString *parent)
+{
+	int i;
+	NestingLevel *n;
+
+	for (i = 0; i < nls->n; i++)
+	{
+		n = nls->levels + i;
+		/* is there a better way to compare two vStrings? */
+		if (strcmp(vStringValue(parent), vStringValue(n->name)) == 0)
+		{
+			if (n && indent <= n->indentation)
+			{
+				/* remove this level by clearing its name */
+				vStringClear(n->name);
+			}
+			break;
+		}
+	}
+}
+
+static void addNestingLevel(NestingLevels *nls, int indentation,
+	const vString *name, boolean is_class)
+{
+	int i;
+	NestingLevel *nl = NULL;
+
+	for (i = 0; i < nls->n; i++)
+	{
+		nl = nls->levels + i;
+		if (indentation <= nl->indentation) break;
+	}
+	if (i == nls->n)
+	{
+		nestingLevelsPush(nls, name, 0);
+		nl = nls->levels + i;
+	}
+	else
+	{	/* reuse existing slot */
+		nls->n = i + 1;
+		vStringCopy(nl->name, name);
+	}
+	nl->indentation = indentation;
+	nl->type = is_class ? K_CLASS : !K_CLASS;
+}
+
+/* Return a pointer to the start of the next triple string, or NULL. Store
+ * the kind of triple string in "which" if the return is not NULL.
+ */
+static char const *find_triple_start(char const *string, char const **which)
+{
+	char const *cp = string;
+
+	for (; *cp; cp++)
+	{
+		if (*cp == '"' || *cp == '\'')
+		{
+			if (strncmp(cp, doubletriple, 3) == 0)
+			{
+				*which = doubletriple;
+				return cp;
+			}
+			if (strncmp(cp, singletriple, 3) == 0)
+			{
+				*which = singletriple;
+				return cp;
+			}
+			cp = skipString(cp);
+			if (!*cp) break;
+		}
+	}
+	return NULL;
+}
+
+/* Find the end of a triple string as pointed to by "which", and update "which"
+ * with any other triple strings following in the given string.
+ */
+static void find_triple_end(char const *string, char const **which)
+{
+	char const *s = string;
+	while (1)
+	{
+		/* Check if the string ends in the same line. */
+		s = strstr (s, *which);
+		if (!s) break;
+		s += 3;
+		*which = NULL;
+		/* If yes, check if another one starts in the same line. */
+		s = find_triple_start(s, which);
+		if (!s) break;
+		s += 3;
+	}
+}
+
+static const char *findVariable(const char *line)
+{
+	/* Parse global and class variable names (C.x) from assignment statements.
+	 * Object attributes (obj.x) are ignored.
+	 * Assignment to a tuple 'x, y = 2, 3' not supported.
+	 * TODO: ignore duplicate tags from reassignment statements. */
+	const char *cp, *sp, *eq, *start;
+
+	cp = strstr(line, "=");
+	if (!cp)
+		return NULL;
+	eq = cp + 1;
+	while (*eq)
+	{
+		if (*eq == '=')
+			return NULL;	/* ignore '==' operator and 'x=5,y=6)' function lines */
+		if (*eq == '(' || *eq == '#')
+			break;	/* allow 'x = func(b=2,y=2,' lines and comments at the end of line */
+		eq++;
+	}
+
+	/* go backwards to the start of the line, checking we have valid chars */
+	start = cp - 1;
+	while (start >= line && isspace ((int) *start))
+		--start;
+	while (start >= line && isIdentifierCharacter ((int) *start))
+		--start;
+	if (!isIdentifierFirstCharacter(*(start + 1)))
+		return NULL;
+	sp = start;
+	while (sp >= line && isspace ((int) *sp))
+		--sp;
+	if ((sp + 1) != line)	/* the line isn't a simple variable assignment */
+		return NULL;
+	/* the line is valid, parse the variable name */
+	++start;
+	return start;
+}
+
+/* Skip type declaration that optionally follows a cdef/cpdef */
+static const char *skipTypeDecl (const char *cp, boolean *is_class)
+{
+	const char *lastStart = cp, *ptr = cp;
+	int loopCount = 0;
+	ptr = skipSpace(cp);
+	if (!strncmp("extern", ptr, 6)) {
+		ptr += 6;
+		ptr = skipSpace(ptr);
+		if (!strncmp("from", ptr, 4)) { return NULL; }
+	}
+	if (!strncmp("class", ptr, 5)) {
+		ptr += 5 ;
+		*is_class = TRUE;
+		ptr = skipSpace(ptr);
+		return ptr;
+	}
+	/* limit so that we don't pick off "int item=obj()" */
+	while (*ptr && loopCount++ < 2) {
+		while (*ptr && *ptr != '=' && *ptr != '(' && !isspace(*ptr)) ptr++;
+		if (!*ptr || *ptr == '=') return NULL;
+		if (*ptr == '(') {
+		    return lastStart; /* if we stopped on a '(' we are done */
+		}
+		ptr = skipSpace(ptr);
+		lastStart = ptr;
+		while (*lastStart == '*') lastStart++;  /* cdef int *identifier */
+	}
+	return NULL;
+}
+
+static void findPythonTags (void)
+{
+	vString *const continuation = vStringNew ();
+	vString *const name = vStringNew ();
+	vString *const parent = vStringNew();
+
+	NestingLevels *const nesting_levels = nestingLevelsNew();
+
+	const char *line;
+	int line_skip = 0;
+	char const *longStringLiteral = NULL;
+
+	while ((line = (const char *) fileReadLine ()) != NULL)
+	{
+		const char *cp = line, *candidate;
+		char const *longstring;
+		char const *keyword, *variable;
+		int indent;
+
+		cp = skipSpace (cp);
+
+		if (*cp == '\0')  /* skip blank line */
+			continue;
+
+		/* Skip comment if we are not inside a multi-line string. */
+		if (*cp == '#' && !longStringLiteral)
+			continue;
+
+		/* Deal with line continuation. */
+		if (!line_skip) vStringClear(continuation);
+		vStringCatS(continuation, line);
+		vStringStripTrailing(continuation);
+		if (vStringLast(continuation) == '\\')
+		{
+			vStringChop(continuation);
+			vStringCatS(continuation, " ");
+			line_skip = 1;
+			continue;
+		}
+		cp = line = vStringValue(continuation);
+		cp = skipSpace (cp);
+		indent = cp - line;
+		line_skip = 0;
+
+		checkParent(nesting_levels, indent, parent);
+
+		/* Deal with multiline string ending. */
+		if (longStringLiteral)
+		{
+			find_triple_end(cp, &longStringLiteral);
+			continue;
+		}
+
+		/* Deal with multiline string start. */
+		longstring = find_triple_start(cp, &longStringLiteral);
+		if (longstring)
+		{
+			longstring += 3;
+			find_triple_end(longstring, &longStringLiteral);
+			/* We don't parse for any tags in the rest of the line. */
+			continue;
+		}
+
+		/* Deal with def and class keywords. */
+		keyword = findDefinitionOrClass (cp);
+		if (keyword)
+		{
+			boolean found = FALSE;
+			boolean is_class = FALSE;
+			if (!strncmp (keyword, "def ", 4))
+			{
+				cp = skipSpace (keyword + 3);
+				found = TRUE;
+			}
+			else if (!strncmp (keyword, "class ", 6))
+			{
+				cp = skipSpace (keyword + 5);
+				found = TRUE;
+				is_class = TRUE;
+			}
+			else if (!strncmp (keyword, "cdef ", 5))
+		    {
+		        cp = skipSpace(keyword + 4);
+		        candidate = skipTypeDecl (cp, &is_class);
+		        if (candidate)
+		        {
+		    		found = TRUE;
+		    		cp = candidate;
+		        }
+
+		    }
+    		else if (!strncmp (keyword, "cpdef ", 6))
+		    {
+		        cp = skipSpace(keyword + 5);
+		        candidate = skipTypeDecl (cp, &is_class);
+		        if (candidate)
+		        {
+		    		found = TRUE;
+		    		cp = candidate;
+		        }
+		    }
+
+			if (found)
+			{
+				boolean is_parent_class;
+
+				is_parent_class =
+					constructParentString(nesting_levels, indent, parent);
+
+				if (is_class)
+					parseClass (cp, name, parent, is_parent_class);
+				else
+					parseFunction(cp, name, parent, is_parent_class);
+
+				addNestingLevel(nesting_levels, indent, name, is_class);
+			}
+		}
+		/* Find global and class variables */
+		variable = findVariable(line);
+		if (variable)
+		{
+			const char *start = variable;
+			boolean parent_is_class;
+
+			vStringClear (name);
+			while (isIdentifierCharacter ((int) *start))
+			{
+				vStringPut (name, (int) *start);
+				++start;
+			}
+			vStringTerminate (name);
+
+			parent_is_class = constructParentString(nesting_levels, indent, parent);
+			/* skip variables in methods */
+			if (! parent_is_class && vStringLength(parent) > 0)
+				continue;
+
+			makeVariableTag (name, parent);
+		}
+		/* Find and parse imports */
+		parseImports(line);
+	}
+	/* Clean up all memory we allocated. */
+	vStringDelete (parent);
+	vStringDelete (name);
+	vStringDelete (continuation);
+	nestingLevelsFree (nesting_levels);
+}
+
+extern parserDefinition *PythonParser (void)
+{
+    static const char *const extensions[] = { "py", "pyx", "pxd", "pxi" ,"scons", NULL };
+	parserDefinition *def = parserNew ("Python");
+	def->kinds = PythonKinds;
+	def->kindCount = KIND_COUNT (PythonKinds);
+	def->extensions = extensions;
+	def->parser = findPythonTags;
+	return def;
+}
+
+/* vi:set tabstop=4 shiftwidth=4: */