@marrs/sxml 0.3.0

package/src/parser.js

@@ -0,0 +1,1012 @@
+import { Readable } from 'stream';
+import { Buffer_Trait, Sexp_Buffer_Trait } from './buffer.js'
+import { last, last_but, Interface } from './util.js';
+import { pre_html_chars, pre_hex_chars, pre_html_entity, pre_hex_entity, pre_qualified_ns, unqualified_xml } from './processors/index.js';
+
+// String catenation by joining an array is a bit faster
+// than using +.  Therefore we prefer to build an array
+// of string parts rather than appending to a preexisting
+// string variable.
+
+function is_found(idxOrStr, substr) {
+    if (void 0 === substr) {
+        return idxOrStr > -1;
+    }
+    return idxOrStr.indexOf(substr) > -1;
+}
+
+function identify_operator(token) {
+    if (token.length) {
+        var firstChar = token.charAt(0);
+        if (/\s/.test(firstChar)) {
+            return 'none';
+        }
+        switch(firstChar) {
+            case "@": return 'attr';
+            case ")": return 'none';
+            case "(": return 'bracket';
+            case "&": return 'specialchar';
+            case "<": return 'specialchar';
+            case ">": return 'specialchar';
+            default: return  'tag';
+        }
+    }
+    return 'none';
+}
+
+const iStackFrame = Interface.create({
+    assertions: {
+        processor: Interface.isDefined,
+        data: Interface.isArray
+    },
+    defaults: {
+        processor: null,
+        operator: '',
+        ast: [''],
+        namespace: null
+    },
+});
+
+
+function is_valid_attr_name(name) {
+    if (0 === name.length) return false;
+    if (is_found(name, '"')) return false;
+    if (is_found(name, "'")) return false;
+    if (is_found(name, ">")) return false;
+    if (is_found(name, "/")) return false;
+    if (is_found(name, "=")) return false;
+    // Reject control chars
+    if (/[\u0000-\u001F\u0020]/.test(name)) return false;
+    // Reject nonchars
+    if (/[\uFDD0-\uFDEF\uFFFE\uFFFF\u1FFFE\u1FFFF\u2FFFE\u2FFFF\u3FFFE\u3FFFF\u4FFFE\u4FFFF\u5FFFE\u5FFFF\u6FFFE\u6FFFF\u7FFFE\u7FFFF\u8FFFE\u8FFFF\u9FFFE\u9FFFF\uAFFFE\uAFFFF\uBFFFE\uBFFFF\uCFFFE\uCFFFF\uDFFFE\uDFFFF\uEFFFE\uEFFFF\uFFFFE\uFFFFF\u10FFFE\u10FFFF]/.test(name)) return false;
+    return true;
+}
+
+function is_char_quote_mark(ch) {
+    return ['"', "'"].indexOf(ch) > -1;
+}
+
+export function log_parse_error(data) {
+    console.info("TODO: LOG TO STDERR -", data.msg, "- line:", data.lineNumber, "token:", data.token);
+}
+
+function is_within_quote(isQuoting, idxQuote, idxString) {
+    if (idxQuote === idxString) {
+        throw new Error("Substring and quote char cannot begin at same index");
+    }
+    if (isQuoting) {
+        if (is_found(idxQuote)) {
+            return idxQuote > idxString;
+        }
+        return true;
+    }
+
+    if (is_found(idxQuote)) { 
+        return idxQuote < idxString;
+    }
+
+    return false;
+}
+
+function is_quoting_by(idx, idxQuote, isAlreadyQuoting) {
+    if (is_found(idxQuote) && idxQuote <= idx) {
+        return !isAlreadyQuoting;
+    }
+    return isAlreadyQuoting;
+}
+
+function index_of_closing_quote(str, quoteChar) {
+    var maybeClosingQuote = str.indexOf(quoteChar);
+    if (maybeClosingQuote < 0) return -1;
+    if (0 === maybeClosingQuote) return 0;
+    if ('\\' === str.charAt(maybeClosingQuote -1)) {
+        // XXX Could stack-overflow in theory
+        return maybeClosingQuote + 1 + index_of_closing_quote(str.substring(maybeClosingQuote +1), quoteChar);
+    }
+    return maybeClosingQuote;
+}
+
+function extend_default_processors(ext) {
+    const target = {};
+    Object.entries(defaultProcessors).forEach(([ns, proc]) => {
+        target[ns] = {
+            pre: proc.pre.slice(),
+            proc: proc.proc,
+        }
+    });
+    Object.entries(ext).forEach(([ns, proc]) => {
+        const extPre = proc?.pre;
+        const extProc = proc?.proc;
+        if (target[ns]) {
+            if(extPre && extPre.length) {
+                target[ns].pre = extPre;
+            }
+            if(extProc) {
+                if (typeof extProc !== 'function') {
+                    throw new Error(`${ns}.proc must be a function`);
+                }
+                target[ns].proc = extProc;
+            }
+        } else {
+            target[ns] = {
+                pre: extPre,
+                proc: extProc,
+            }
+        }
+    });
+    return target;
+}
+
+function make_pipeline(descriptor) {
+    if (typeof descriptor === 'function') return descriptor;
+    var pres = descriptor.pre || [];
+    var proc = descriptor.proc;
+    return function(event, stackFrame, content, state, callData) {
+        var args = [event, stackFrame, content, state, callData];
+        for (var i = 0; i < pres.length; i++) {
+            args = pres[i].apply(null, args);
+        }
+        return proc.apply(null, args);
+    };
+}
+
+function definitely_comes_before(idx1, idx2) {
+    if (idx1 > -1 && idx2 < 0) return true;
+    return idx1 > -1 && idx1 < idx2;
+}
+
+function track_line_state(content, data) {
+    var lastNl = content.lastIndexOf('\n');
+    if (lastNl >= 0) {
+        data.hadNewline = true;
+        data.currentLineIndent = content.substring(lastNl + 1);
+    } else {
+        data.currentLineIndent += content;
+    }
+}
+
+const defaultProcessor = 'html';
+
+const defaultProcessors = {
+    unqualified_xml: {
+        pre: [pre_hex_chars, pre_hex_entity],
+        proc: unqualified_xml
+    },
+    html: {
+        pre: [pre_html_chars, pre_html_entity],
+        proc: unqualified_xml
+    },
+    xsl: {
+        pre: [pre_qualified_ns, pre_hex_chars, pre_hex_entity],
+        proc: unqualified_xml
+    },
+};
+
+export function init_parse_state(initData) {
+    initData = initData || {};
+    var data = {
+        processor: defaultProcessor,
+        lineCount: 0,
+        offset: 0,
+        lexicalStack: {
+            frames: [],
+            push: function(x) {
+                return this.frames.push(x);
+            },
+            current_frame: function() {
+                return last(this.frames);
+            },
+            parent_frame: function() {
+                return last_but(1, this.frames);
+            },
+            processor: function() {
+                if (!this.frames.length) {
+                    return data.processor || defaultProcessor;
+                }
+                var currentFrame = this.current_frame();
+                var idxColon = (currentFrame.operator || '').indexOf(':');
+                switch (idxColon) {
+                    case -1: return currentFrame.processor || data.processor || defaultProcessor;
+                    case 0: return currentFrame.processor;
+                    default: {
+                        var currentNs = currentFrame.operator.substring(0, idxColon);
+                        if (data.processors[currentNs]) {
+                            return currentNs;
+                        }
+                        return defaultProcessor;
+                    }
+                }
+            },
+            pop: function(buf, isTagChildless) {
+                var event = isTagChildless? 'write-tag:no-children' : 'close-tag';
+                var poppedFrame = this.frames.pop();
+                var currentFrame = this.current_frame();
+                var poppedFrameAst = poppedFrame.ast;
+                switch (identify_operator(poppedFrame.operator)) {
+                    case 'tag':
+                        poppedFrame.processedData = data.processors[poppedFrame.processor](
+                            event,
+                            poppedFrame,
+                            '',
+                            data.processorState[poppedFrame.processor],
+                            {
+                                ast: poppedFrameAst,
+                                cursor: data.cursor,
+                                source: data.source,
+                                lineCount: data.lineCount
+                            }
+                );
+                    break;
+                    case 'attr':
+                        if (currentFrame) {
+                            switch (identify_operator(currentFrame.operator)) {
+                            case 'tag':
+                                currentFrame.ast.push(poppedFrameAst);
+                            break;
+                            case 'attr':
+                                currentFrame.ast.push(poppedFrameAst);
+                            break;
+                            default:
+                                console.error("Could not merge attributes. Stack frame did not represent attribute or tag.");
+                            }
+                        }
+                    break;
+                    default:
+                    break;
+                }
+                return poppedFrame;
+            }
+        },
+        state: null,
+        wsBuf: '',
+        lastChunk: '',
+        isPrettyFormatting: initData.isPrettyFormatting || false,
+        currentLineIndent: '',
+        hadNewline: false,
+        readerMacroName: '',
+        readerMacroArgs: ''
+    };
+
+    var source = initData.source;
+    if (source === undefined || source === null) {
+        data.source = process.stdin;
+    } else if (typeof source === 'string' || source instanceof Readable) {
+        data.source = source;
+    } else {
+        throw new Error('source must be a string or a Readable stream');
+    }
+    data.cursor = initData.cursor || 0;
+
+    data.processors = {};
+    data.processorState = {};
+
+    const processors = extend_default_processors(initData?.extensions || {});
+
+    Object.entries(processors).forEach(([ns, descriptor]) => {
+        data.processors[ns] = make_pipeline(descriptor);
+        data.processorState[ns] = iStackFrame.new();
+    });
+
+    data.readers = {
+        ext: function(data, args) {
+            data.lexicalStack.push(iStackFrame.new({ processor: args, isExtWrapper: true }));
+        },
+        _default: function(data, args) {
+            log_parse_error({ msg: "Unknown reader macro: #" + data.readerMacroName, lineNumber: data.lineCount });
+        }
+    };
+
+    if (initData.readers) {
+        for (let name in initData.readers) {
+            data.readers[name] = initData.readers[name];
+        }
+    }
+
+    data.barf_ws = function() {
+        var buf = this.wsBuf;
+        this.wsBuf = '';
+        return buf;
+    }
+
+    return data;
+}
+
+export function update_parse_state(strChunk, data, result) {
+    parse_chunk(strChunk, data, result);
+    data.offset += strChunk.length;
+    data.lastChunk = strChunk;
+}
+
+// The AST for lexical stack frame data is represented like so:
+// [ LEADING_WHITESPACE, TAG [LEADING_WHITESPACE, ATTR_NAME, ATTR_VALUE], ...]
+
+const AST_IDX_WS = 0;
+const AST_IDX_TAG = 1;
+const AST_IDX_NAME = 1;
+const AST_IDX_VAL = 2;
+const AST_IDX_AT1 = 2;
+
+export function parse_chunk(strChunk, data, result) {
+    var lineLength = strChunk.length;
+
+    var buf = Object.create(Sexp_Buffer_Trait);
+    Object.assign(buf, {
+        cursor: 0,
+        lineCount: data.lineCount,
+        str: strChunk,
+        token: '',
+        substr: strChunk,
+    });
+
+    var $1, $2, $3;
+    var count = 0;
+
+    while (true) {
+        if (data.state === null
+        && data.attrCapture !== undefined
+        && data.lexicalStack.frames.length === data.attrCaptureDepth) {
+            var $captured = result.splice(data.attrCapture).join('');
+            data.lexicalStack.current_frame().ast[AST_IDX_VAL] += $captured;
+            data.attrCapture = undefined;
+            data.state = '(@ _';
+        }
+        if (!buf.substr) {
+            data.cursor += buf.cursor;
+            return;
+        }
+        // Code:
+        //   #?  : Possible escape sequence
+        //   #_  : Acquire reader macro name
+        //   #_[ : Acquire reader macro args
+        //   #(  : Opened trapdoor
+        //   )#? : Closing trapdoor?
+        //   (?  : Discover purpose of opening bracket
+        //   (#  : Opening an s-exp for a tag
+        //   (# ?: Discover what comes after tag operator
+        //   (# (: Handle opener discovered within tag operand
+        //   (@ ?: Discover what comes after attr operator
+        //   (@1 : Handle first char of s-exp for attribute name
+        //   (@! : Bad attribute
+        //   (@_ : Acquire attribute name
+        //   (@ ": Parsing an attribute value wrapped in double quotes
+        //   (@ ': Parsing an attribute value wrapped in single quotes
+        //   (@ _: Parsing an attribute value not wrapped in quotes
+        //   (&  : Parse special char
+        //   (& 1: Parse the first operand of the special char function
+        //   (& _: Parse the remaining operands of the special char function
+        //   (& ): Resolve the special char function
+        //   ))  : Handle second closing bracket
+        switch (data.state) {
+            case '#?': {
+                if ('(' === buf.substr[0]) {
+                    data.state = '#(';
+                } else if (';' === buf.substr[0]) {
+                    data.state = '#;';
+                } else if (/[a-zA-Z0-9_<>=.!-]/.test(buf.substr[0])) {
+                    data.readerMacroName = buf.substr[0];
+                    data.state = '#_';
+                } else {
+                    result[result.length] = '#';
+                    data.state = null;
+                }
+                buf.step();
+            } break;
+            case '#_': {
+                var nameChars = (buf.substr.match(/^[a-zA-Z0-9_<>=?!.-]*/) || [''])[0];
+                data.readerMacroName += nameChars;
+                buf.step(nameChars.length);
+                if (!buf.substr) { return; }
+                if ('_' === data.readerMacroName[0]) {
+                    log_parse_error({ msg: "Reader macro names cannot start with underscore: #" + data.readerMacroName, lineNumber: data.lineCount });
+                    data.readerMacroName = '';
+                    data.state = null;
+                } else if ('[' === buf.substr[0]) {
+                    buf.step();
+                    data.state = '#_[';
+                } else {
+                    var reader = data.readers[data.readerMacroName] || data.readers._default;
+                    reader(data, null);
+                    data.readerMacroName = '';
+                    data.state = null;
+                }
+            } break;
+            case '#_[': {
+                var idxClose = buf.substr.indexOf(']');
+                if (idxClose < 0) {
+                    data.readerMacroArgs += buf.read_to_end();
+                    return;
+                }
+                data.readerMacroArgs += buf.read_to(idxClose);
+                buf.step();
+                var reader = data.readers[data.readerMacroName] || data.readers._default;
+                reader(data, data.readerMacroArgs);
+                data.readerMacroName = '';
+                data.readerMacroArgs = '';
+                data.state = null;
+            } break;
+            case '#(': {
+                result[result.length] = buf.read_to(')')
+                if (!buf.substr) {
+                    return;
+                } else {
+                    buf.step();
+                    data.state = ')#?';
+                }
+            } break;
+            case ')#?': {
+                if ('#' === buf.substr[0]) {
+                    buf.step();
+                    data.state = null;
+                } else {
+                    result[result.length] = ')';
+                    data.state = '#(';
+                }
+            } break;
+            case '#;': {
+                buf.skip_whitespace();
+                if (!buf.substr) { return; }
+                if ('(' === buf.substr[0]) {
+                    data.commentDepth = 1;
+                    buf.step();
+                    data.state = '#;(';
+                } else {
+                    data.state = null;
+                }
+            } break;
+            case '#;(': {
+                var i = 0;
+                var len = buf.substr.length;
+                while (i < len) {
+                    if (buf.substr[i] === '(') {
+                        data.commentDepth++;
+                    } else if (buf.substr[i] === ')') {
+                        data.commentDepth--;
+                        if (data.commentDepth === 0) {
+                            buf.reset(i);
+                            buf.step();
+                            data.state = null;
+                            break;
+                        }
+                    }
+                    i++;
+                }
+                if (data.state === '#;(') {
+                    buf.reset(len);
+                    return;
+                }
+            } break;
+            case '(?': {
+                if (!buf.substr) { return; }
+                $1 = identify_operator(buf.substr);
+                switch ($1) {
+                    case 'tag':
+                        $2 = data.lexicalStack.current_frame();
+                        if (data.isPrettyFormatting && data.hadNewline && $2) {
+                            $2.hasBlockChildren = true;
+                        }
+                        data.lexicalStack.push(iStackFrame.new({
+                            processor: null,
+                            operator: '',
+                            ast: [''], // 1st element contains whitespace
+                            namespace: ($2 || {}).namespace,
+                            indent: data.currentLineIndent
+                        }));
+                        data.hadNewline = false;
+                        data.state = '(#';
+                    break;
+                    case 'attr':
+                        buf.step();
+                        data.state = '(@1';
+                    break;
+                    case 'bracket':
+                        data.lexicalStack.push(iStackFrame.new({
+                            processor: data.processor,
+                            operator: '('
+                        }));
+                        result[result.length] = '(';
+                        buf.step();
+                        data.state = null;
+                    break;
+                    case 'specialchar': {
+                        var $c = buf.substr[0];
+                        var $isDirect = $c === '<' || $c === '>';
+                        data.lexicalStack.push(iStackFrame.new({
+                            processor: (data.lexicalStack.current_frame() || {}).processor || data.processor,
+                            operator: '&',
+                            ast: $isDirect ? [$c, 1] : [''],
+                            namespace: (
+                                data.lexicalStack.current_frame() || {}
+                            ).namespace,
+                        }));
+                        data.state = $isDirect ? '(& )' : '(&';
+                        buf.step();
+                    } break;
+                    default:
+                        result[result.length] = '(';
+                        data.state = null;
+                        // XXX Why don't we advance here?
+                        // We must be advancing somewhere else?
+                        // Is that ok?
+                }
+            } break;
+            case '(#': {
+                $1 = data.lexicalStack.current_frame();
+                $1.operator += buf.read_token();
+                $2 = $1.operator.indexOf(":");
+                if ($2 < 0) {
+                    $1.processor = last_but(1, data.lexicalStack.frames, {}).processor || data.processor;
+                } else {
+                    $1.processor = $1.operator.substring(0, $2);
+                    if (!data.processors[$1.processor]) {
+                        $1.processor = defaultProcessor;
+                    }
+                }
+                var $parentFrame = last_but(1, data.lexicalStack.frames, {});
+                if ($parentFrame.isExtWrapper) {
+                    $1.processor = $parentFrame.processor;
+                    data.lexicalStack.frames.splice(data.lexicalStack.frames.length - 2, 1);
+                }
+                if (!buf.substr) { return; }
+                $1.ast[AST_IDX_TAG] = $1.operator;
+                $1.ast[AST_IDX_AT1] = [];
+                data.state = '(# ?';
+            } break;
+            case '(# (': {
+                switch (identify_operator(buf.substr)) {
+                    case 'attr':
+                        data.lexicalStack.push(iStackFrame.new({
+                            processor: data.processor,
+                            operator: '@',
+                            ast: [data.barf_ws()],
+                            namespace: (
+                                data.lexicalStack.current_frame() || {}
+                            ).namespace,
+                        }));
+                        $1 = data.lexicalStack.current_frame().ast;
+                        buf.step();
+                        data.state = '(@1';
+                    break;
+                    case 'tag':
+                        $1 = data.lexicalStack.current_frame();
+                        if (data.isPrettyFormatting && is_found(data.wsBuf, '\n')) {
+                            $1.hasBlockChildren = true;
+                        }
+                        result[result.length] =
+                            data.processors[$1.processor](
+                                'open-tag',
+                                $1,
+                                '',
+                                data.processorState[$1.processor],
+                                {
+                                    ast: $1.ast,
+                                    cursor: data.cursor + buf.cursor,
+                                    source: data.source,
+                                    lineCount: data.lineCount
+                                }
+                            );
+                        data.lexicalStack.push(iStackFrame.new({
+                            processor: null,
+                            operator: '',
+                            ast: [''],
+                            namespace: $1.namespace
+                        }));
+                        data.lexicalStack.current_frame().ast[AST_IDX_WS] = (
+                            is_found(data.wsBuf, '\n')? data.barf_ws() : data.barf_ws().substring(1)
+                        );
+                        data.lexicalStack.current_frame().indent =
+                            (data.lexicalStack.current_frame().ast[AST_IDX_WS] || '').match(/[^\n]*$/)[0];
+                        data.state = '(#';
+                    break;
+                    case 'bracket':
+                        result[result.length] =
+                            data.processors[data.lexicalStack.processor()](
+                                'open-tag',
+                                data.lexicalStack.current_frame(),
+                                data.processorState[data.lexicalStack.processor()],
+                                {
+                                    cursor: data.cursor + buf.cursor,
+                                    source: data.source,
+                                    lineCount: data.lineCount
+                                }
+                            );
+                        data.lexicalStack.push(iStackFrame.new({
+                            processor: data.processor,
+                            operator: '('
+                        }));
+                        result[result.length] =
+                            data.processors[data.lexicalStack.processor()](
+                                'begin-escape-bracket',
+                                data.lexicalStack.current_frame(),
+                                null,
+                                {
+                                    cursor: data.cursor + buf.cursor,
+                                    source: data.source,
+                                    lineCount: data.lineCount
+                                }
+                            );
+                        buf.step();
+                        data.wsBuf = '';
+                        data.state = null;
+                    break;
+                    case 'specialchar': {
+                        $1 = data.lexicalStack.current_frame();
+                        result[result.length] = data.processors[$1.processor](
+                            'open-tag',
+                            $1,
+                            '',
+                            data.processorState[$1.processor],
+                            {
+                                ast: $1.ast,
+                                cursor: data.cursor + buf.cursor,
+                                source: data.source,
+                                lineCount: data.lineCount
+                            }
+                        );
+                        data.barf_ws();
+                        var $c2 = buf.substr[0];
+                        var $isDirect2 = $c2 === '<' || $c2 === '>';
+                        data.lexicalStack.push(iStackFrame.new({
+                            processor: $1.processor || data.processor,
+                            operator: '&',
+                            ast: $isDirect2 ? [$c2, 1] : [''],
+                            namespace: $1.namespace,
+                        }));
+                        data.state = $isDirect2 ? '(& )' : '(&';
+                        buf.step();
+                    } break;
+                    case 'none':
+                        $1 = data.lexicalStack.processor();
+                        result[result.length] =
+                            data.processors[$1](
+                                'open-tag',
+                                data.lexicalStack.current_frame(),
+                                data.processorState[data.lexicalStack.processor()],
+                                {
+                                    ast: $1.ast,
+                                    cursor: data.cursor + buf.cursor,
+                                    source: data.source,
+                                    lineCount: data.lineCount
+                                }
+                            );
+                        data.state = null;
+                        buf.step();
+                    break;
+                    default: {
+                        console.error(["ERR: ", identify_operator(buf.substr), "did not match a pattern"].join(' '));
+                    }
+                }
+            } break;
+            case '(# ?': {
+                data.wsBuf += buf.read_whitespace();
+                $1 = data.lexicalStack.current_frame().ast
+                if (!buf.substr) { return; }
+                if (buf.substr[0] === ')') {
+                    buf.step();
+                    $1[AST_IDX_AT1] = [data.barf_ws()];
+                    $2 = data.lexicalStack.pop(buf, true);
+                    result[result.length] = $2.processedData;
+                    data.state = null;
+                } else if (buf.substr[0] === '(') {
+                    buf.step();
+                    data.state = '(# (';
+                } else {
+                    var $frame = data.lexicalStack.current_frame();
+                    result[result.length] =
+                        data.processors[$frame.processor](
+                            'open-tag',
+                            $frame,
+                            data.processorState[$frame.processor],
+                            {
+                                ast: $1,
+                                cursor: data.cursor + buf.cursor,
+                                source: data.source,
+                                lineCount: data.lineCount
+                            }
+                        );
+                    // TODO: Work out how to handle this whitespace after opening tag.
+                    if (is_found(data.wsBuf, '\n')) {
+                        result[result.length] = data.barf_ws();
+                    } else {
+                        result[result.length] = data.barf_ws().substring(1);
+                    }
+                    data.state = null;
+                }
+            } break;
+            case '(@1': {
+                $1 = buf.substr[0];
+                switch ($1) {
+                    case ')':
+                        buf.step();
+                        data.state = null;
+                    break;
+                    default: {
+                        if (/[\s'"]/.test($1)) {
+                            data.state = '(@!';
+                        } else {
+                            buf.step();
+                            if (!data.lexicalStack.current_frame()) {
+                                data.lexicalStack.push(iStackFrame.new({
+                                    processor: data.processor,
+                                    operator: '@',
+                                    ast: []
+                                }));
+                            }
+                            data.lexicalStack.current_frame().ast[AST_IDX_NAME] = $1;
+                            data.state = '(@_';
+                        }
+                    }
+                }
+            } break;
+            case '(@!': {
+                // TODO: Put bad attributes into AST.
+                //   Maybe by appending to ws string.
+                result[result.length] = buf.read_to(')')
+                if (!buf.substr) {
+                    return;
+                }
+                buf.step();
+                data.state = null;
+            } break;
+            case '(@_': {
+                $1 = data.lexicalStack.current_frame();
+                $2 = $1.ast;
+                $2[AST_IDX_NAME] += buf.read_token();
+                if (!buf.substr) { return; }
+                if (!is_valid_attr_name($2[AST_IDX_NAME])) {
+                    // XXX Will read funny for super-long attribute names
+                    log_parse_error({
+                        msg: "Invalid attribute name",
+                        lineNumber: data.lineCount,
+                        token: last($2)[AST_IDX_NAME]
+                    });
+                }
+                $2[AST_IDX_VAL] = '';
+                data.state = '(@ ?';
+            } break;
+            case '(@ ?': {
+                buf.skip_whitespace();
+                if (!buf.substr) { return; }
+                $1 = buf.substr[0];
+                if (is_char_quote_mark($1)) {
+                    data.lexicalStack.current_frame().ast[AST_IDX_VAL] = $1;
+                    buf.step();
+                    data.state = '(@ ' + $1;
+                } else if (')' === $1) {
+                    data.lexicalStack.pop(buf);
+                    buf.step();
+                    data.state = '(# ?';
+                } else {
+                    data.lexicalStack.current_frame().ast[AST_IDX_VAL] = '"';
+                    buf.skip_whitespace();
+                    data.state = '(@ _';
+                }
+            } break;
+            case '(@ "':
+            case "(@ '": {
+                $1 = index_of_closing_quote(buf.substr, last(data.state));
+                $2 = data.lexicalStack.current_frame().ast;
+                if ($1 > -1) {
+                    $2[AST_IDX_VAL] += buf.read_to($1 +1);
+                    if (!(0 === $1 && '\\' === (last(data.lastChunk)))) {
+                        data.state = '(@ ?';
+                    }
+                } else {
+                    $2[AST_IDX_VAL] += buf.read_to_end();
+                    return;
+                }
+            } break;
+
+            case '(@ ?(': {
+                if (!buf.substr) { return; }
+                if (buf.substr[0] === '@') {
+                    data.lexicalStack.current_frame().ast[AST_IDX_VAL] += '(';
+                    data.state = '(@ _';
+                } else {
+                    data.attrCapture = result.length;
+                    data.attrCaptureDepth = data.lexicalStack.frames.length;
+                    data.state = '(?';
+                }
+            } break;
+
+            case '(@ _': {
+                $1 = buf.substr.indexOf(')');
+                $3 = buf.substr.indexOf('(');
+                $2 = data.lexicalStack.current_frame().ast;
+                if (definitely_comes_before($3, $1)) {
+                    $2[AST_IDX_VAL] += buf.read_to($3);
+                    buf.step();
+                    data.state = '(@ ?(';
+                } else if ($1 > -1) {
+                    $2[AST_IDX_VAL] += buf.read_to($1) + '"';
+                    data.state = '(@ ?';
+                } else {
+                    $2[AST_IDX_VAL] += buf.read_to_end();
+                    return;
+                }
+            } break;
+
+            case '(&': {
+                $1 = data.lexicalStack.current_frame();
+                $1.ast[0] += buf.read_token();
+                if (!buf.substr) { return }
+                if (')' === buf.substr[0]) {
+                    $1.ast[1] = 1;
+                    data.state = '(& )';
+                } else {
+                    $1.ast[1] = '';
+                    buf.step();
+                    data.state = '(& 1';
+                }
+            } break;
+
+            case '(& 1': {
+                $1 = data.lexicalStack.current_frame();
+                $1.ast[1] += buf.read_token()
+                if (!buf.substr) { return; }
+                $1.ast[1] = parseInt($1.ast[1], 10);
+                if (')' === buf.substr[0]) {
+                    data.state = '(& )';
+                } else {
+                    data.state = '(& _';
+                }
+            } break;
+
+            case '(& _': {
+                $1 = buf.substr.indexOf(')');
+                if ($1 < 0) { return; }
+                buf.reset($1);
+                data.state = '(& )';
+            } break;
+
+            case '(& )': {
+                $1 = data.lexicalStack.current_frame();
+                $2 = {'': '&', '&': '&', '<': '<', '>': '>'}[$1.ast[0]];
+                if ($2 !== undefined) {
+                    result[result.length] = data.processors[data.lexicalStack.processor()](
+                        'content',
+                        $1,
+                        $2.repeat($1.ast[1]),
+                        data.processorState[data.lexicalStack.processor()],
+                        {
+                            cursor: data.cursor + buf.cursor,
+                            lineCount: data.lineCount,
+                            source: data.source
+                        }
+                    );
+                } else {
+                    $2 = data.processors[data.lexicalStack.processor()](
+                        'entity',
+                        $1,
+                        $1.ast[0],
+                        data.processorState[data.lexicalStack.processor()],
+                        {
+                            cursor: data.cursor + buf.cursor,
+                            lineCount: data.lineCount,
+                            source: data.source
+                        }
+                    );
+                    for ($3 = 0; $3 < $1.ast[1]; ++$3) {
+                        result[result.length] = $2;
+                    }
+                }
+                buf.step();
+                data.lexicalStack.pop(buf);
+                data.state = null;
+            } break;
+
+            case '))': {
+                if (!buf.substr.length) {
+                    return;
+                }
+                if (')' === buf.substr[0]) {
+                    buf.step();
+                }
+                data.state = null;
+            } break;
+
+            default: {
+                // Not quoting an attribute value.  Any string that's
+                // within an s-exp is a text node (barring the operator).
+                //
+                // We can put anything up to a closing bracket straight
+                // to result if we're in a stack. Anything up to an opener
+                // if we're not.
+
+                var idxOpeningBracket = buf.substr.indexOf('(');
+                var idxClosingBracket = buf.substr.indexOf(')');
+                var idxHash = buf.substr.indexOf('#');
+
+                if (definitely_comes_before(idxClosingBracket, idxOpeningBracket)) {
+                    $3 = buf.read_to(idxClosingBracket);
+                    if (data.isPrettyFormatting) track_line_state($3, data);
+                    result[result.length] = data.processors[data.lexicalStack.processor()](
+                        'content',
+                        data.lexicalStack.current_frame(),
+                        $3,
+                        data.processorState[data.lexicalStack.processor()],
+                        {
+                            cursor: data.cursor + buf.cursor,
+                            lineCount: data.lineCount,
+                            source: data.source
+                        }
+                    );
+                    if (data.lexicalStack.frames.length) {
+                        if ('(' === data.lexicalStack.current_frame().operator) {
+                            result[result.length] = data.processors[data.lexicalStack.processor()]('end-escape-bracket');
+                            data.lexicalStack.pop(buf);
+                            data.state = '))';
+                        } else {
+                            $1 = data.lexicalStack.pop(buf)
+                            if ('tag' === identify_operator($1.operator)) {
+                                if (data.isPrettyFormatting && $1.hasBlockChildren) {
+                                    $2 = result[result.length - 1] || '';
+                                    if (!$2.endsWith('\n')) {
+                                        result[result.length] = '\n' + ($1.indent || '');
+                                    }
+                                }
+                                result[result.length] = $1.processedData;
+                            }
+                        }
+                    } else {
+                        result[result.length] = ')';
+                    }
+                    buf.step();
+                    continue;
+                }
+
+                if (definitely_comes_before(idxHash, idxOpeningBracket)) {
+                    $3 = buf.read_to(idxHash);
+                    if (data.isPrettyFormatting) track_line_state($3, data);
+                    result[result.length] = data.processors[data.lexicalStack.processor()](
+                        'content',
+                        data.lexicalStack.current_frame(),
+                        $3,
+                        data.processorState[data.lexicalStack.processor()],
+                        {
+                            cursor: data.cursor + buf.cursor,
+                            lineCount: data.lineCount,
+                            source: data.source
+                        }
+                    );
+                    buf.step();
+                    data.state = '#?';
+                    continue;
+                }
+
+                if (idxOpeningBracket > -1) {
+                    $3 = buf.read_to(idxOpeningBracket);
+                    if (data.isPrettyFormatting) track_line_state($3, data);
+                    result[result.length] = data.processors[data.lexicalStack.processor()](
+                        'content',
+                        data.lexicalStack.current_frame(),
+                        $3,
+                        data.processorState[data.lexicalStack.processor()],
+                        {
+                            cursor: data.cursor + buf.cursor,
+                            lineCount: data.lineCount,
+                            source: data.source
+                        }
+                    );
+                    buf.step();
+                    data.state = '(?';
+                    continue;
+                } else {
+                    $3 = buf.read_to_end();
+                    if (data.isPrettyFormatting) track_line_state($3, data);
+                    result[result.length] = data.processors[data.lexicalStack.processor()](
+                        'content',
+                        data.lexicalStack.current_frame(),
+                        $3,
+                        data.processorState[data.lexicalStack.processor()],
+                        {
+                            cursor: data.cursor + buf.cursor,
+                            lineCount: data.lineCount,
+                            source: data.source
+                        }
+                    );
+                    return;
+                }
+            }
+        }
+    }
+}