gollum 3.1.2 → 3.1.3

data/lib/gollum/public/gollum/livepreview/js/ace/lib/ace/mode/_test/Readme.md

@@ -0,0 +1,9 @@
+`tokens_<modeName>.json` files keep information about correct tokens and tokenizer states for all modes supported by ace.
+They are generated from `text_<modeName>.txt` or `demo/kitchen-sink/doc/*` with
+
+```sh
+node highlight_rules_test.js -gen
+```
+
+command.
+

data/lib/gollum/public/gollum/livepreview/js/ace/lib/ace/mode/_test/highlight_rules_test.js

@@ -0,0 +1,210 @@
+var fs = require("fs");
+var path = require("path");
+if (!fs.existsSync)
+    fs.existsSync = path.existsSync;
+
+require("amd-loader");
+
+var cwd = __dirname + "/";
+var root = path.normalize(cwd + Array(5).join("../"));
+
+function jsFileList(path, filter) {
+    if (!filter) filter = /_test/;
+    return fs.readdirSync(path).map(function(x) {
+        if (x.slice(-3) == ".js" && !filter.test(x) && !/\s/.test(x))
+            return x.slice(0, -3);
+    }).filter(Boolean);
+}
+
+function modeList() {
+    return jsFileList(cwd + "../", /_highlight_rules|_test|_worker|xml_util|_outdent|behaviour|completions/);
+}
+
+function checkModes() {
+    modeList().forEach(function(modeName) {
+        try {
+            var Mode = require("../" + modeName).Mode;
+        } catch(e) {
+            console.warn("Can't load mode :" + modeName, e);
+            return;
+        }
+        var m = new Mode();
+        if (!m.lineCommentStart && !m.blockComment)
+            console.warn("missing comment in " + modeName);
+        if (!m.$id)
+            console.warn("missing id in " + modeName);
+        var tokenizer = (new Mode).getTokenizer();
+        if (m.lineCommentStart) {
+            if (Array.isArray(m.lineCommentStart)) {
+                m.lineCommentStart.forEach(function(x) {
+                    testLineComment(tokenizer, x, modeName)
+                });
+            } else {
+                testLineComment(tokenizer, m.lineCommentStart, modeName)
+            }
+        }
+        // if (m.blockComment) {
+        //     var tokens = tok.getLineTokens(m.lineCommentStart, "start");
+        //     if (!/comment/.test(tokens[0]))
+        //         console.warn("broken lineCommentStart in " + modeName);
+        // }
+    });
+    
+    function testLineComment(tokenizer, commentStart, modeName) {
+        var tokens = tokenizer.getLineTokens(commentStart + " ", "start").tokens;
+        if (!/comment/.test(tokens[0].type))
+            console.warn("broken lineCommentStart in " + modeName);
+    }
+}
+
+function generateTestData() {
+    var docRoot = root + "/demo/kitchen-sink/docs";
+    var docs = fs.readdirSync(docRoot);
+    var specialDocs = fs.readdirSync(cwd);
+    var modes = modeList();
+
+    // console.log("Docs:", docs);
+    // console.log("Modes:", modes);
+
+    docs.forEach(function(docName) {
+        var p = docName.toLowerCase().split(".");
+        if (!p[1])
+            return;
+        var modeName;
+        if (modes.indexOf(p[0]) != -1)
+            modeName = p[0];
+        else if (modes.indexOf(p[1]) != -1)
+            modeName = p[1];
+        else
+            modeName = {"txt": "text", cpp: "c_cpp"}[p[1]];
+
+        var filePath = "text_" + modeName + ".txt";
+        if (specialDocs.indexOf(filePath) == -1) {
+            filePath = docRoot + "/" + docName;
+        } else {
+            filePath = cwd + filePath;
+        }
+
+        var text = fs.readFileSync(filePath, "utf8");
+        try {
+            var Mode = require("../" + modeName).Mode;
+        } catch(e) {
+            console.warn("Can't load mode :" + modeName, p, e);
+            return;
+        }
+        console.log(modeName);
+        var tokenizer = new Mode().getTokenizer();
+
+        var state = "start";
+        var data = text.split(/\r\n|\r|\n/).map(function(line) {
+            var data = tokenizer.getLineTokens(line, state);
+            var tmp = [];
+            tmp.push(JSON.stringify(data.state));
+            var tokenizedLine = "";
+            data.tokens.forEach(function(x) {
+                tokenizedLine += x.value;
+                tmp.push(JSON.stringify([x.type, x.value]));
+            });
+            if (tokenizedLine != line)
+                tmp.push(JSON.stringify(line));
+            state = data.state;
+            return tmp.join(",\n  ");
+        });
+        
+        var jsonStr = "[[\n   " + data.join("\n],[\n   ") + "\n]]";
+        fs.writeFileSync(cwd + "tokens_" + modeName + ".json", jsonStr, "utf8");
+    });
+}
+
+function test(startAt) {
+    var modes = fs.readdirSync(cwd).map(function(x) {
+        return (x.match(/tokens_(.*).json/) || {})[1];
+    }).filter(function(x){return !!x});
+
+    for (var i = Math.max(0, startAt||0); i < modes.length; i++)
+        testMode(modes[i], i);
+
+    console.log("\u001b[32m" + "all ok" + "\u001b[0m");
+}
+function testMode(modeName, i) {
+    console.log(padNumber(i+1, 3) + ") testing: \u001b[33m" + modeName + "\u001b[0m");
+
+    var text = fs.readFileSync(cwd + "tokens_" + modeName + ".json", "utf8");
+    var data = JSON.parse(text);
+    var Mode = require("../" + modeName).Mode;
+    var tokenizer = new Mode().getTokenizer();
+
+    var state = "start";
+    data.forEach(function(lineData) {
+        lineData.values = [];
+        lineData.types = [];
+        lineData.state = lineData.shift();
+        var line = null;
+        if (typeof lineData[lineData.length - 1] == "string")
+            line = lineData.pop();
+        lineData.forEach(function(x) {
+            lineData.types.push(x[0]);
+            lineData.values.push(x[1]);
+        });
+        if (typeof line != "string")
+            line = lineData.values.join("");
+
+        var tokens = tokenizer.getLineTokens(line, state);
+        var values = tokens.tokens.map(function(x) {return x.value;});
+        var types = tokens.tokens.map(function(x) {return x.type;});
+
+        var err = testEqual([
+            JSON.stringify(lineData.state), JSON.stringify(tokens.state),
+            lineData.types, types,
+            lineData.values, values]);
+        
+        if (err) {
+            console.log(line);
+            throw "error";
+        }
+
+        state = tokens.state;
+    });
+}
+function testEqual(a) {
+    var err;
+    if (a[0] + "" !== a[1] + "") {
+        console.log(a[0],a[1]);
+        err = 1;
+    }
+
+    if ( a[2] + "" !== a[3] + "" || a[4] + "" !== a[5] + "") {
+        arrayDiff(a[2],a[3]);
+        arrayDiff(a[4],a[5]);
+        err = 1;
+    }
+    return err;
+}
+function arrayDiff(a1, a2) {
+    var l = Math.max(a1.length, a2.length);
+    var out = [];
+    for (var i = 0; i < l; i++) {
+        out.push("\n", padNumber(i+1, 3), ") ");
+        if (a1[i] !== a2[i])
+            out.push("\u001b[31m", a1[i], "\u001b[0m != \u001b[32m", a2[i], "\u001b[0m");
+        else
+            out.push(a1[i]);
+    }
+    console.log(out.join(""));
+}
+function padNumber(num, digits) {
+    return ("      " + num).slice(-digits);
+}
+
+// cli
+var arg = process.argv[2];
+if (!arg)
+    test();
+else if (/--?g(en)?/.test(arg))
+    generateTestData(process.argv.splice(3));
+else if (/--?c(heck)?/.test(arg))
+    checkModes(process.argv.splice(3));
+else if (/\d+/.test(arg))
+    test(parseInt(process.argv[2],10) || 0);
+else
+    testMode(arg, -1);

data/lib/gollum/public/gollum/livepreview/js/ace/lib/ace/mode/_test/text_asciidoc.txt

@@ -0,0 +1,111 @@
+------------------------------------
+-----------------------------------
+_ita_lic_, *bo*ld*, +mo+no+, normal. 
+``double quoted'', `single quoted'.
+  normal, ^super^, ~sub~.
+
+''''
+Escaped:
+\_italic_, +++_italic_+++,
+t\__e__st, +++t__e__st+++,
++++<b>bold</b>+++, $$<b>normal</b>$$
+\&#182; &#182;
+\`not single quoted'
+\`\`not double quoted''
+
+
+[fffff]
++++++++++++++++++++++++++++++++++++++
+(C) {ss} ss
++++++++++++++++++++++++++++++++++++++
+
+.............
+callout <1>
+..............
+
+> 1
+1> 2
+<2> 3
+
+Escaped:
+\_italic_,
+t\__e__st, o__
+
+.optional title
+.............
+callout <1>
+..............
+
+
+:macro: dddd
+
+.lists
+. 1
+.. 2
+... d
+..... big
++
+continue +
+next line
+xi) no ++la+tin++
+
+xi) la++tin++
+2. num__ber__  [red]#red#
+--
+  5. f <<x6,ssss>> {macro}
+--
+image::path[beauty] ->--<= -- replacements
+
+ image::s
+sssss
+sss 
+sssss
+
+
+== 1
+  heading
+=== not a heading
+====================================
+
+==================================
+====4 
+NOTE: above
+
+NOTE: above
+
+[[x6]]
+WARNING: 
+
+[options[]]
+---------------------------
+literal
+---------------------------
+
+
+= Tables
+|====================================
+| _italic_ |  *bold* | text | (R)
+
+|====================================
+
+
+[more, options]
+///////////
+comment
+///////////
+// one line comment
+
+
+
+[red]#red text# [yellow-background]#on yellow#
+[big]#large# [red yellow-background big]*all bold*
+
+	
+https://site text <mail@i.am> callto:ace http://ace.ajaxorg.com[awesome]
+  .still normal text
+.break out thoug should not
+---------------------------
+///////////////////////////
+---------------------------
+
+

data/lib/gollum/public/gollum/livepreview/js/ace/lib/ace/mode/_test/text_coffee.txt

@@ -0,0 +1,57 @@
+#test: tokenize keyword
+    for (i 1..2)
+#test: tokenize regexp
+/"[a]/
+#test: tokenize functions
+foo = ({args}) ->
+    foo = ({a1, a2}) ->
+    foo = ({@a1, a2}) ->
+    foo : ({args}) ->
+    foo = ({args}) ->
+    foo = ({0abc}) ->
+    foo = ({/abc}) =>
+    foo = ({abc/}) ->
+    foo = ({#abc}) ->
+    foo = ({abc#}) ->
+    foo = ({)abc}) ->
+    foo = ({abc)}) ->
+    foo = ({a{bc}) ->
+    foo = ({}) ->
+    foo = ({ }) ->
+    foo : ({}) ->
+    foo = (args) ->
+    foo = (arg1, arg2) ->
+    foo = (arg1 = 1, arg2 = 'name') ->
+    foo = (@arg1 = /abc/, arg2 = 'name') ->
+    #test: tokenize function: invalid case:
+    foo=(/args) ->
+    foo = () ->
+    foo = ( ) ->
+    foo : ( ) ->
+    window.foo = (args) ->
+    foo = ->
+    foo = ->
+    foo : ->
+    #test: tokenize callback function
+    foo bar: 1, (args) ->
+    foo = (1, 2 (x) ->
+#test: tokenize class
+class Foo
+class Foo extends Bar
+#test: tokenize illegal name property
+foo.static.function
+#!test tokenize string with interpolation
+a = "#{ 22 / 7 + {x: "#{a + b}"} + 2}"
+" #{ "" + {} } )"  
+"""heredoc
+   """
+do ->
+    ###
+    herecomment
+    ###
+    re = /regex/imgy.test ///
+        heregex  # comment
+    ///imgy
+    this isnt: `just 
+       JavaScript`
+    undefined

data/lib/gollum/public/gollum/livepreview/js/ace/lib/ace/mode/_test/text_curly.txt

@@ -0,0 +1,9 @@
+tokenize Curly template{{test}}
+tokenize embedded script
+<script a='a'>var</script>'123'
+tokenize multiline attribute value with double quotes
+<a href="abc{{xyz}}
+def">
+tokenize multiline attribute value with single quotes
+<a href='abc
+def\"'>

data/lib/gollum/public/gollum/livepreview/js/ace/lib/ace/mode/_test/text_html.txt

@@ -0,0 +1,10 @@
+<html>
+<scripts>
+<script a='a'>var "</scripts></script>'123'
+</scripts>
+<a href="abc
+  def">
+<a href='abc
+def\'>
+
+</html>

data/lib/gollum/public/gollum/livepreview/js/ace/lib/ace/mode/_test/text_javascript.txt

@@ -1,3 +1,37 @@
+//test: tokenize 'standard' functions
+string.charCodeAt(23); document.getElementById('test'); console.log('Here it is');";
+test: /**tokenize doc*/ comment
+/**tokenize doc comment with @tag {}*/
+//test: tokenize parens
+    var line = "[{( )}]";
+//test tokenize arithmetic expression which looks like a regexp
+a/b/c
+a/=b/c
+//test tokenize reg exps
+a=/b/g
+a+/b/g
+a = 1 + /2 + 1/gimyxk
+a=/a/ / /a/
+case /a/.test(c)
+//test tokenize multi-line comment containing a single line comment
+noRegex
+/* foo // bar */
+canBeRegex;
+/* foo // bar */
+// test tokenize identifier with umlauts
+fu?e
+// test // is not a regexp
+{ // 123
+//test skipping escaped chars
+'Meh\\nNeh'
+console.log('\\u1232Feh'
+"test multiline\
+ strings"
+a='
+b="\
+still a string
+ 
+ 
 function foo(items, nada) {
     for (var i=0; i<items.length; i++) {
         alert(items[i] + "juhu\n");