redcar-javamateview 0.2

data/src/com/redcareditor/mate/ParseThunk.java

@@ -0,0 +1,71 @@
+package com.redcareditor.mate;
+
+import org.eclipse.swt.widgets.Display;
+
+public class ParseThunk implements Runnable {
+	int WAIT                     = 500;
+	int DELAY_IF_MODIFIED_WITHIN = 400;
+	int DELAY_FOR_USER_INPUT     = 10;
+	
+	public long timeCreated;
+	public long lastModificationTime;
+	public int parseFrom;
+	public boolean closed;
+	public int wait = WAIT;
+	
+	private Parser parser;
+	
+	public ParseThunk(Parser parser, int parseFrom) {
+		this.parser               = parser;
+		this.timeCreated          = System.currentTimeMillis();
+		this.lastModificationTime = System.currentTimeMillis();
+		this.parseFrom            = parseFrom;
+		// System.out.printf("New thunk. parseFrom:%d time: %d\n", parseFrom, timeCreated);
+		enqueue();
+	}
+	
+	public void enqueue() {
+		Display.getCurrent().timerExec(wait, this);
+	}
+	
+	public void stop() {
+		closed = true;
+	}
+	
+	public void run() {
+		if (closed)
+			return;
+		// System.out.printf("Run thunk. time: %s\n", System.currentTimeMillis());
+		if (lastModificationTime > System.currentTimeMillis() - DELAY_IF_MODIFIED_WITHIN) {
+			// System.out.printf("  Postponing thunk.\n", parseFrom);
+			Display.getCurrent().timerExec(wait, this);
+		}
+		else {
+			// System.out.printf("  Once, after 0.5 seconds, parse from %d.\n", parseFrom);
+			execute();
+		}
+	}
+	
+	public void delayAndUpdate(int lineIx) {
+		// System.out.printf("Delay thunk time: %s\n", System.currentTimeMillis());
+		this.lastModificationTime = System.currentTimeMillis();
+		this.parseFrom = Math.min(parseFrom, lineIx);
+	}
+	
+	public void execute() {
+		parser.parserScheduler.thunk = null;
+		//System.out.printf("Thunk: parseOnwards(%d)\n", parseFrom);
+		parseFrom = parser.parserScheduler.parseOnwards(parseFrom);
+		if (parseFrom == -1)
+			return;
+		if (parseFrom >= parser.getLineCount() - 1)
+			return;
+		if (parseFrom >= parser.parserScheduler.lastVisibleLine + ParserScheduler.LOOK_AHEAD - 5)
+			return;
+		if (parseFrom <= parser.getLineCount() - 1) {
+			wait = DELAY_FOR_USER_INPUT;
+			enqueue();
+		}
+	}
+}
+

data/src/com/redcareditor/mate/Parser.java

@@ -0,0 +1,627 @@
+package com.redcareditor.mate;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Logger;
+import java.util.logging.Handler;
+import java.util.logging.ConsoleHandler;
+import java.util.logging.Level;
+
+import org.eclipse.swt.custom.StyledText;
+import org.eclipse.swt.widgets.Display;
+import org.eclipse.swt.widgets.ScrollBar;
+
+import org.eclipse.jface.text.BadLocationException;
+import org.eclipse.jface.text.Document;
+import org.eclipse.jface.text.IRegion;
+
+import com.redcareditor.mate.ParserScheduler;
+import com.redcareditor.mate.document.MateDocument;
+import com.redcareditor.mate.document.MateTextLocation;
+import com.redcareditor.mate.document.swt.SwtMateDocument;
+import com.redcareditor.onig.Match;
+import com.redcareditor.onig.Rx;
+
+public class Parser {
+	public static int linesParsed = 0;
+	public Logger logger;
+
+	public Grammar grammar;
+	public MateText mateText;
+	public Document document;
+	public StyledText styledText;
+	public MateDocument mateDocument;
+	
+	public Scope root;
+	public ParserScheduler parserScheduler;
+	
+	public Parser(Grammar g, MateText m) {
+		g.initForUse();
+		grammar = g;
+		mateText = m;
+		styledText = m.getTextWidget();
+		makeRoot();
+		mateDocument = m.getMateDocument();
+		document     = (Document) m.getDocument();
+		parserScheduler = new ParserScheduler(this);
+		logger = Logger.getLogger("JMV.Parser ");
+		logger.setUseParentHandlers(false);
+		for (Handler h : logger.getHandlers()) {
+			logger.removeHandler(h);
+		}
+		logger.addHandler(MateText.consoleHandler());
+		logger.setLevel(Level.INFO);
+	}
+	
+	public void close() {
+		parserScheduler.close();
+	}
+	
+	public void setRoot(Scope root) {
+		this.root = root;
+		root.setMateText(mateText);
+	}
+	
+	public void makeRoot() {
+		this.root = new Scope(mateText, this.grammar.scopeName);
+		this.root.isOpen = true;
+		DoublePattern dp = new DoublePattern();
+		dp.name = this.grammar.name;
+		dp.patterns = this.grammar.patterns;
+		dp.grammar = this.grammar;
+		this.root.pattern = dp;
+	}
+	
+	public int getLineAtOffset(int offset) {
+		try {
+			return document.getLineOfOffset(offset);
+		} catch (BadLocationException e) {
+			System.out.printf("*** Warning BadLocationException offset:%d\n", offset);
+			e.printStackTrace();
+			return -1;
+		}
+	}
+	
+	public int getOffsetAtLine(int line) {
+		try {
+			return document.getLineOffset(line);
+		} catch (BadLocationException e) {
+			System.out.printf("*** Warning BadLocationException");
+			e.printStackTrace();
+			return -1;
+		}
+	}
+	
+	public int getLineCount() {
+		return document.getNumberOfLines();
+	}
+	
+	public int getCharCount() {
+		return document.getLength();
+	}
+	
+	public String getLine(int line) {
+		try {
+			IRegion region = document.getLineInformation(line);
+			return document.get(region.getOffset(), region.getLength());
+		} catch (BadLocationException e) {
+			System.out.printf("*** Warning BadLocationException");
+			e.printStackTrace();
+			return "";
+		}
+	}
+	
+
+	public boolean shouldColour() {
+		return (parserScheduler.modifyStart == -1);
+	}
+	
+	public void redrawLine(int lineIx) {
+		// System.out.printf("redrawLine(%d)\n", lineIx);
+		int startOffset = getOffsetAtLine(lineIx);
+		int endOffset;
+		int lineCount = getLineCount();
+		if (lineIx + 1 < lineCount) {
+			endOffset = getOffsetAtLine(lineIx + 1) - 1;
+		}
+		else {
+			endOffset = getCharCount();
+		}
+		styledText.redrawRange(startOffset, endOffset - startOffset, false);
+	}
+	
+	private Scope scopeBeforeStartOfLine(int lineIx) {
+		Scope startScope = this.root.scopeAt(lineIx, 0);
+		// System.out.printf("scopeBeforeStartOfLine: %s\n", startScope.pattern.name);
+		if (startScope.getStart().getLine() == lineIx) { // || startScope.pattern instanceof SinglePattern) {
+			startScope = startScope.containingDoubleScope(lineIx);
+			// System.out.printf("scopeBeforeStartOfLine: up %s\n", startScope.pattern.name);
+		}
+
+		return startScope;
+	}
+
+	private Scope scopeAfterEndOfLine(int lineIx, int lineLength) {
+		Scope endScope = this.root.scopeAt(lineIx, lineLength - 1);
+		if (endScope.getStart().getLine() == lineIx ) {
+			endScope = endScope.containingDoubleScope(lineIx);
+		}
+
+		return endScope;
+	}
+	
+	public void clearFrom(int offset) {
+		root.clearFrom(offset);
+	}
+	
+	public boolean parseLine(int lineIx) {
+		Parser.linesParsed++;
+		String line = getLine(lineIx) + "\n";
+		int length = line.length();
+		//if (length > 1)
+		//	logger.info(String.format("parseLine(%d) \"%s\"", lineIx, line.substring(0, line.length() - 1)));
+		//else
+		//	logger.info(String.format("parseLine(%d)", lineIx));
+		if (lineIx > parserScheduler.getParsedUpto())
+			parserScheduler.setParsedUpto(lineIx);
+		//System.out.printf("getParsedUpto: %d\n", getParsedUpto());
+		Scope startScope = scopeBeforeStartOfLine(lineIx);
+		Scope endScope1  = scopeAfterEndOfLine(lineIx, length);
+		//logger.info(String.format("  startScope is: %s", startScope.name));
+		//logger.info(String.format("  endScope1: %s", endScope1.name));
+		Scanner scanner = new Scanner(startScope, line, lineIx);
+		ArrayList<Scope> allScopes = new ArrayList<Scope>();
+		allScopes.add(startScope);
+		ArrayList<Scope> closedScopes = new ArrayList<Scope>();
+		ArrayList<Scope> removedScopes = new ArrayList<Scope>();
+		allScopes.add(startScope);
+		//logger.info(String.format("start pretty:\n%s", root.pretty(4)));
+		for (Marker m : scanner) {
+			Scope expectedScope = getExpectedScope(scanner.getCurrentScope(), lineIx, length, scanner.position);
+			// if (expectedScope != null)
+			// 	logger.info(String.format("expectedScope: %s (%d, %d)", expectedScope.name, expectedScope.getStart().getLine(), 
+			// 		           expectedScope.getStart().getLineOffset()));
+			// else
+			// 	logger.info("no expected scope");
+			// logger.info(String.format("  scope: %s %d-%d (line length: %d)", 
+			//				m.pattern.name, m.from, m.match.getCapture(0).end, length));
+			if (m.isCloseScope) {
+				//logger.info("     (closing)");
+				closeScope(scanner, expectedScope, lineIx, line, length, m, 
+							allScopes, closedScopes, removedScopes);
+			}
+			else if (m.pattern instanceof DoublePattern) {
+				//logger.info("     (opening)");
+				openScope(scanner, expectedScope, lineIx, line, length, m, 
+						   allScopes, closedScopes, removedScopes);
+			}
+			else {
+				//logger.info("     (single)");
+				singleScope(scanner, expectedScope, lineIx, line, length, m, 
+							 allScopes, closedScopes, removedScopes);
+			}
+			//System.out.printf("pretty:\n%s\n", root.pretty(2));
+			scanner.position = m.match.getByteCapture(0).end;
+		}
+		clearLine(lineIx, startScope, allScopes, closedScopes, removedScopes);
+		Scope endScope2 = scopeAfterEndOfLine(lineIx, length);
+		//logger.info(String.format("  end_scope2: %s\n", endScope2.name));
+		//System.out.printf("end pretty: %s\n", this.root.pretty(4));
+//		if (colourer != null) {
+//			// System.out.printf("before_uncolour_scopes\n");
+//			colourer.uncolourScopes(removedScopes);
+//			// System.out.printf("before_colour_line_with_scopes\n");
+//			colourer.colourLineWithScopes(allScopes);
+//			// System.out.printf("after_colour_line_with_scopes\n");
+//       ]
+//		else {
+//			// stdout.printf("no colourer");
+//		]
+		return (endScope1 != endScope2);
+	}
+
+	public Scope getExpectedScope(Scope currentScope, int line, int lineLength, int lineOffset) {
+//		System.out.printf("get_expected_scope(%s, %d, %d)\n", currentScope.name, line, lineOffset);
+		if (lineOffset == lineLength)
+			return null;
+		Scope expectedScope = currentScope.firstChildAfter(mateDocument.getTextLocation(line, lineOffset));
+//		System.out.printf("first_child_after: %s\n", expectedScope.name);
+		assert(expectedScope != currentScope);
+		if (expectedScope != null) {
+			if (expectedScope.getStart().getLine() != line)
+				expectedScope = null;
+			while (expectedScope != null && expectedScope.isCapture) {
+				expectedScope = expectedScope.parent;
+			}
+		}
+		return expectedScope;
+	}
+
+	public void closeScope(Scanner scanner, Scope expectedScope, int lineIx, String line, 
+			int length, Marker m, ArrayList<Scope> allScopes, 
+			ArrayList<Scope> closedScopes, ArrayList<Scope> removedScopes) {
+		String endMatchString = line.substring(m.from, m.match.getCapture(0).end);
+		
+		boolean is_ended            = !scanner.getCurrentScope().isOpen; //(scanner.getCurrentScope().getEnd() != null);
+		boolean equal_ends          = false;
+		boolean equal_inner_ends    = false;
+		boolean equal_match_strings = false;
+		
+		if (is_ended) {
+			equal_ends          = (scanner.getCurrentScope().getEnd().equals(mateDocument.getTextLocation(lineIx, m.match.getCapture(0).end)));
+			equal_inner_ends    = (scanner.getCurrentScope().getInnerEnd().equals(mateDocument.getTextLocation(lineIx, m.from)));
+			equal_match_strings = (scanner.getCurrentScope().endMatchString.equals(endMatchString));
+		}
+		
+		if (is_ended && equal_ends && equal_inner_ends && equal_match_strings) {
+			// System.out.printf("closing scope matches expected\n");
+			// we have already parsed this line and this scope ends here
+
+			// Re-add the captures from the end of the current scope to the 
+			// tracking arrays
+			for (Scope child : scanner.getCurrentScope().children) {
+				if (child.isCapture && 
+						child.getStart().getLine() == lineIx) {
+					if (!closedScopes.contains(child))
+						closedScopes.add(child);
+					if (!allScopes.contains(child))
+						allScopes.add(child);
+				}
+			}
+		}
+		else {
+			// System.out.printf("closing scope does not match expected\n");
+			// stdout.printf("closing scope at %d\n", m.from);
+//			if (colourer != null) {
+//				colourer.uncolourScope(scanner.getCurrentScope(), false);
+//			]
+			setInnerEndPosSafely(scanner.getCurrentScope(), m, lineIx, length, 0);
+			setEndPosSafely(scanner.getCurrentScope(), m, lineIx, length, 0);
+			scanner.getCurrentScope().isOpen = false;
+			scanner.getCurrentScope().endMatchString = endMatchString;
+			//stdout.printf("end_match_string: '%s'\n", scanner.current_scope.end_match_string);
+			if (expectedScope != null) {
+				scanner.getCurrentScope().removeChild(expectedScope);
+				removedScopes.add(expectedScope);
+				// @removed_scopes << expected_scope
+			}
+		}
+		removeCloseCaptures(scanner.getCurrentScope());
+		handleCaptures(lineIx, length, line, scanner.getCurrentScope(), m, allScopes, closedScopes);
+		removedScopes.add(scanner.getCurrentScope()); // so it gets uncoloured
+		closedScopes.add(scanner.getCurrentScope());
+		scanner.setCurrentScope(scanner.getCurrentScope().parent);
+		allScopes.add(scanner.getCurrentScope());
+	}
+
+
+	public void openScope(Scanner scanner, Scope expectedScope, int lineIx, 
+			String line, int length, Marker m,
+			ArrayList<Scope> allScopes, ArrayList<Scope> closedScopes, ArrayList<Scope> removedScopes ) {
+		// System.out.printf("[opening with %d patterns], \n", ((DoublePattern) m.pattern).patterns.size());
+		Scope s = new Scope(mateText, m.pattern.name);
+		s.pattern = m.pattern;
+		s.openMatch = m.match;
+		setStartPosSafely(s, m, lineIx, length, 0);
+		setInnerStartPosSafely(s, m, lineIx, length, 0);
+		s.beginMatchString = line.substring(m.from, m.match.getCapture(0).end);
+		
+		s.isOpen = true;
+		s.isCapture = false;
+		s.parent = scanner.getCurrentScope();
+		Scope newScope = s;
+		// is this a bug? captures aren't necessarily to be put into all_scopes yet surely?
+		if (expectedScope != null) {
+			// check mod ending scopes as the new one will not have a closing marker
+			// but the expected one will:
+			// stdout.printf("%s == %s and %s == %s and %s == %s and %s == %s and %s == %s",
+			// 			  name, other.name, pattern.name, other.pattern.name, start_loc().to_s(),
+			// 			  other.start_loc().to_s(), inner_start_loc().to_s(), other.inner_start_loc().to_s(),
+			// 			  begin_match_string, other.begin_match_string);
+			// if (!s.name.equals(expectedScope.name)) System.out.printf("different names\n");
+			// if (!s.pattern.name.equals(expectedScope.pattern.name)) System.out.printf("different patterns\n");
+			// if (!s.getStart().equals(expectedScope.getStart())) System.out.printf("different starts\n");
+			// if (!s.getInnerStart().equals(expectedScope.getInnerStart())) System.out.printf("different inner starts\n");
+			// if (!s.beginMatchString.equals(expectedScope.beginMatchString)) {
+			// 	System.out.printf("different beginMatchStrings '%s'/'%s'\n", s.beginMatchString, expectedScope.beginMatchString);
+			// }
+			if (s.surfaceIdenticalToModuloEnding(expectedScope)) {
+				// System.out.printf("surface_identical_mod_ending: keep expected\n");
+				// don't need to do anything as we have already found this,
+				// but let's keep the old scope since it will have children and what not.
+				newScope = expectedScope;
+				for (Scope child : expectedScope.children) {
+					// if (!child.isCapture) {
+						closedScopes.add(child);
+						allScopes.add(child);
+					// }
+				}
+				// handleCaptures(lineIx, length, line, s, m, allScopes, closedScopes);
+				scanner.setCurrentScope(expectedScope);
+			}
+			else {
+				// System.out.printf("surface_NOT_identical_mod_ending: replace expected\n");
+				if (s.overlapsWith(expectedScope)) {
+					scanner.getCurrentScope().removeChild(expectedScope);
+					// removed_scopes << expected_scope
+					removedScopes.add(expectedScope);
+				}
+				handleCaptures(lineIx, length, line, s, m, allScopes, closedScopes);
+				scanner.getCurrentScope().addChild(s);
+				scanner.setCurrentScope(s);
+			}
+		}
+		else {
+			handleCaptures(lineIx, length, line, s, m, allScopes, closedScopes);
+			scanner.getCurrentScope().addChild(s);
+			scanner.setCurrentScope(s);
+		}
+		allScopes.add(newScope);
+	}
+
+	private void singleScope(Scanner scanner, Scope expectedScope, int lineIx, 
+							String line, int length, Marker m, 
+							ArrayList<Scope> allScopes, ArrayList<Scope> closedScopes, 
+							ArrayList<Scope> removedScopes) {
+		Scope s = new Scope(this.mateText, m.pattern.name);
+		s.pattern = m.pattern;
+		s.openMatch = m.match;
+		setStartPosSafely(s, m, lineIx, length, 0);
+		setEndPosSafely(s, m, lineIx, length, 0);
+		s.isOpen = false;
+		s.isCapture = false;
+//		System.out.printf("beginMatchString '%s' %d - %d\n",  new String(line.getBytes(), m.from, m.match.getCapture(0).end - m.from), m.from, m.match.getCapture(0).end);
+		// s.beginMatchString = new String(line.getBytes(), m.from, m.match.getCapture(0).end - m.from); 
+		s.beginMatchString = line.substring(m.match.getCapture(0).start, m.match.getCapture(0).end);
+		s.parent = scanner.getCurrentScope();
+		Scope newScope = s;
+		if (expectedScope != null) {
+			if (s.surfaceIdenticalTo(expectedScope)) {
+				newScope = expectedScope;
+				for (Scope child : expectedScope.children) {
+					closedScopes.add(child);
+				}
+			}
+			else {
+				handleCaptures(lineIx, length, line, s, m, allScopes, closedScopes);
+				if (s.overlapsWith(expectedScope)) {
+					if (expectedScope == scanner.getCurrentScope()) {
+						// we expected this scope to close, but it doesn't
+					}
+					else {
+						scanner.getCurrentScope().removeChild(expectedScope);
+						// removed_scopes << expectedScope
+						removedScopes.add(expectedScope);
+					}
+				}
+				scanner.getCurrentScope().addChild(s);
+			}
+		}
+		else {
+			handleCaptures(lineIx, length, line, s, m, allScopes, closedScopes);
+			scanner.getCurrentScope().addChild(s);
+		}
+		allScopes.add(newScope);
+		closedScopes.add(newScope);
+	}
+
+	private boolean atEndOfNonFinalLine(int lineIx, int length, int to) {
+		return to == length && getLineCount() > lineIx+1;
+	}
+
+	public void setStartPosSafely(Scope scope, Marker m, int lineIx, int length, int cap) {
+		int to = m.match.getCapture(cap).start;
+		// if (atEndOfNonFinalLine(lineIx, length, to)) 
+		// 	scope.setStartPos(lineIx+1, 0, false);
+		// else
+			scope.setStartPos(lineIx, Math.min(to, length), false);
+	}
+
+	public void setInnerStartPosSafely(Scope scope, Marker m, int lineIx, int length, int cap) {
+		int to = m.match.getCapture(cap).start;
+		// if (atEndOfNonFinalLine(lineIx, length, to)) 
+		// 	scope.setInnerStartPos(lineIx+1, 0, false);
+		// else
+			scope.setInnerStartPos(lineIx, Math.min(to, length), false);
+	}
+
+	public void setInnerEndPosSafely(Scope scope, Marker m, int lineIx, int length, int cap) {
+		int from = m.match.getCapture(cap).start;
+		// if (atEndOfNonFinalLine(lineIx, length, from)) {
+		// 	scope.setInnerEndPos(lineIx, length, true);
+		// }
+		// else {
+			scope.setInnerEndPos(lineIx, Math.min(from, length-1), true);
+		// }
+	}
+	
+	public void setEndPosSafely(Scope scope, Marker m, int lineIx, int length, int cap) {
+		int to = m.match.getCapture(cap).end;
+		// if (atEndOfNonFinalLine(lineIx, length, to)) {
+		// 	scope.setEndPos(lineIx, length, true);
+		// }
+		// else {
+			scope.setEndPos(lineIx, Math.min(to, length-1), true);
+		// }
+	}
+	
+	// Opens scopes for captures AND creates closing regexp from
+	// captures if necessary.
+	public void handleCaptures(int lineIx, int length, String line, 
+			Scope scope, Marker m, ArrayList<Scope> allScopes, ArrayList<Scope> closedScopes) {
+		makeClosingRegex(line, scope, m);
+		collectChildCaptures(lineIx, length, scope, m, allScopes, closedScopes);
+	}
+
+	public Rx makeClosingRegex(String line, Scope scope, Marker m) {
+		if (m.pattern instanceof DoublePattern && !m.isCloseScope) {
+			DoublePattern dp = (DoublePattern) m.pattern;
+			// System.out.printf("making closing regex: %s\n", dp.endString);
+			Rx rx = Rx.createRx("\\\\(\\d+)");
+			Match match;
+			int pos = 0;
+			StringBuilder src = new StringBuilder("");
+			boolean found = false;
+			while ((match = rx.search(dp.endString, pos, (int) dp.endString.length())) != null) {
+				found = true;
+				src.append(dp.endString.substring(pos, match.getCapture(0).start));
+				String numstr = dp.endString.substring(match.getCapture(1).start, match.getCapture(1).end);
+				int num = Integer.parseInt(numstr);
+				// System.out.printf("capture found: %d\n", num);
+				String capstr = line.substring(m.match.getCapture(num).start, m.match.getCapture(num).end);
+				src.append(Rx.escape(capstr));
+				pos = match.getCapture(1).end;
+			}
+			if (found)
+				src.append(dp.endString.substring(pos, dp.endString.length()));
+			else
+				src.append(dp.endString);
+			// System.out.printf("close re: '%s'\n", src.toString());
+			scope.closingRegex = Rx.createRx(src.toString());
+		}
+		return null;
+	}
+	
+	public void removeCloseCaptures(Scope scope) {
+		int i = 0;
+		ArrayList<Scope> children = scope.children;
+		while (i < children.size()) {
+			Scope childScope = children.get(i);
+			if (childScope.isCloseCapture) {
+				children.remove(i);
+				i--;
+			}
+			i++;
+		}
+	}
+
+	public void collectChildCaptures(int lineIx, int length, Scope scope, 
+			Marker m, ArrayList<Scope> allScopes, ArrayList<Scope> closedScopes) {
+		Scope s;
+		Map<Integer, String> captures;
+		boolean isCloseCaptures = false;
+		if (m.pattern instanceof SinglePattern) {
+			captures = ((SinglePattern) m.pattern).captures;
+		}
+		else {
+			if (m.isCloseScope) {
+				isCloseCaptures = true;
+				captures = ((DoublePattern) m.pattern).endCaptures;
+			}
+			else {
+				captures = ((DoublePattern) m.pattern).beginCaptures;
+			}
+			if (((DoublePattern) m.pattern).bothCaptures != null) {
+				captures = ((DoublePattern) m.pattern).bothCaptures;
+			}
+		}
+		List<Scope> captureScopes = new ArrayList<Scope>();
+		// create capture scopes
+		if (captures != null) {
+			for (Integer cap : captures.keySet()) {
+				if (m.match.numCaptures() - 1 >= cap && m.match.getCapture(cap).start != -1) {
+					s = new Scope(mateText, captures.get(cap));
+					s.pattern = scope.pattern;
+					s.setStartPos(lineIx, Math.min(m.match.getCapture(cap).start, length-1), false);
+					setInnerEndPosSafely(s, m, lineIx, length, cap);
+					setEndPosSafely(s, m, lineIx, length, cap);
+					s.isOpen = false;
+					s.isCapture = true;
+					s.isCloseCapture = isCloseCaptures;
+					s.parent = scope;
+					captureScopes.add(s);
+					allScopes.add(s);
+					closedScopes.add(s);
+				}
+			}
+		}
+		// Now we arrange the capture scopes into a tree under the matched
+		// scope. We do this by processing the captures in order of offset and 
+		// length. For each capture, we check to see if it is a child of an already 
+		// placed capture, and if so we add it as a child (we know that the latest 
+		// such capture is the one to add it to by the ordering). If not we
+		// add it as a child of the matched scope.
+
+		List<Scope> placedScopes = new ArrayList<Scope>();
+		Scope parentScope;
+		while (captureScopes.size() > 0) {
+			s = null;
+			// find first and longest remaining scope (put it in 's')
+			for (Scope cs : captureScopes) {
+				if (s == null || 
+						cs.getStart().compareTo(s.getStart()) < 0 || 
+						(cs.getStart().compareTo(s.getStart()) == 0 && cs.getLength() > s.getLength())) {
+					s = cs;
+				}
+			}
+			//System.out.printf("arrange: %s, start: %d, length: %d\n", s.name, s.startOffset(), s.endOffset() - s.startOffset());
+			// look for somewhere to put it from placed_scopes
+			parentScope = null;
+			for (Scope ps : placedScopes) {
+				if (s.getStart().compareTo(ps.getStart()) >= 0 && s.getEnd().compareTo(ps.getEnd()) <= 0) {
+					parentScope = ps;
+				}
+			}
+			if (parentScope != null) {
+				parentScope.addChild(s);
+				s.parent = parentScope;
+			}
+			else {
+				scope.addChild(s);
+			}
+			// logger.info(String.format("    capture %s", s.name));
+			placedScopes.add(s);
+			captureScopes.remove(s);
+		}
+	}
+
+	private ArrayList<Scope> deleteAnyOnLineNotIn(Scope cs, int lineIx, ArrayList<Scope> allScopes) {
+		int startOffset = getOffsetAtLine(lineIx);
+		int endOffset;
+		if (lineIx == getLineCount() - 1)
+			endOffset = getCharCount();
+		else
+			endOffset = getOffsetAtLine(lineIx + 1);
+		return cs.deleteAnyBetweenNotIn(startOffset, endOffset, allScopes);
+	}
+
+	private void clearLine(int lineIx, Scope startScope, ArrayList<Scope> allScopes, 
+							ArrayList<Scope> closedScopes, ArrayList<Scope> removedScopes) {
+		// If we are reparsing, we might find that some scopes have disappeared,
+		// delete them:
+		Scope cs = startScope;
+		while (cs != null) {
+			// stdout.printf("  removing_scopes from: %s\n", cs.name);
+			ArrayList<Scope> newRemovedScopes = deleteAnyOnLineNotIn(cs, lineIx, allScopes);
+			removedScopes.addAll(newRemovedScopes);
+			cs = cs.parent;
+		}
+
+		// any that we expected to close on this line that now don't?
+		// first build list of scopes that close on this line (including ones
+		// that did but haven't been removed yet).
+		ArrayList<Scope> scopesThatClosedOnLine = new ArrayList<Scope>();
+		Scope ts = startScope;
+		while (ts.parent != null) {
+			if (ts.getInnerEnd().getLine() == lineIx) {
+				scopesThatClosedOnLine.add(ts);
+			}
+			ts = ts.parent;
+		}
+		for (Scope s : scopesThatClosedOnLine) {
+			if (!closedScopes.contains(s)) {
+				if (s.isCapture) {
+					s.parent.removeChild(s);
+					removedScopes.add(s);
+				}
+				else {
+					int line = getLineCount() - 1;
+					int lineOffset = getCharCount() - getOffsetAtLine(line);
+					s.removeEnd();
+					s.isOpen = true;
+				}
+			}
+		}
+	}
+}